From cd2231630f7d248dd249d99a8797989b76423266 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Fri, 4 Aug 2023 14:54:45 +0100
Subject: [PATCH 01/19] Improve updating of viewV2 state and restore
 grid<>builder sync for datasource definitions

---
 .../backend/DataTable/ViewV2DataTable.svelte  |  3 +-
 .../builder/src/stores/backend/viewsV2.js     | 73 ++++++++++++-------
 .../src/components/grid/stores/datasource.js  |  2 +-
 3 files changed, 50 insertions(+), 28 deletions(-)

diff --git a/packages/builder/src/components/backend/DataTable/ViewV2DataTable.svelte b/packages/builder/src/components/backend/DataTable/ViewV2DataTable.svelte
index 21b3e1c291..859ce11280 100644
--- a/packages/builder/src/components/backend/DataTable/ViewV2DataTable.svelte
+++ b/packages/builder/src/components/backend/DataTable/ViewV2DataTable.svelte
@@ -13,7 +13,8 @@
   }
 
   const handleGridViewUpdate = async e => {
-    viewsV2.replace(id, e.detail)
+    console.log("update")
+    viewsV2.replaceView(id, e.detail)
   }
 </script>
 
diff --git a/packages/builder/src/stores/backend/viewsV2.js b/packages/builder/src/stores/backend/viewsV2.js
index c8f38ff5e5..6fa3b52bb9 100644
--- a/packages/builder/src/stores/backend/viewsV2.js
+++ b/packages/builder/src/stores/backend/viewsV2.js
@@ -1,4 +1,4 @@
-import { writable, derived } from "svelte/store"
+import { writable, derived, get } from "svelte/store"
 import { tables } from "./"
 import { API } from "api"
 
@@ -30,44 +30,64 @@ export function createViewsV2Store() {
 
   const deleteView = async view => {
     await API.viewV2.delete(view.id)
-
-    // Update tables
-    tables.update(state => {
-      const table = state.list.find(table => table._id === view.tableId)
-      delete table.views[view.name]
-      return { ...state }
-    })
+    replaceView(view.id, null)
   }
 
   const create = async view => {
     const savedViewResponse = await API.viewV2.create(view)
     const savedView = savedViewResponse.data
-
-    // Update tables
-    tables.update(state => {
-      const table = state.list.find(table => table._id === view.tableId)
-      table.views[view.name] = savedView
-      return { ...state }
-    })
-
+    replaceView(savedView.id, savedView)
     return savedView
   }
 
   const save = async view => {
     const res = await API.viewV2.update(view)
     const savedView = res?.data
+    replaceView(view.id, savedView)
+  }
 
-    // Update tables
-    tables.update(state => {
-      const table = state.list.find(table => table._id === view.tableId)
-      if (table) {
-        if (view.originalName) {
-          delete table.views[view.originalName]
-        }
-        table.views[view.name] = savedView
-      }
-      return { ...state }
+  // Handles external updates of tables
+  const replaceView = (viewId, view) => {
+    console.log("replace", viewId, view)
+    if (!viewId) {
+      return
+    }
+    const existingView = get(derivedStore).list.find(view => view.id === viewId)
+    const tableIndex = get(tables).list.findIndex(table => {
+      return table._id === view?.tableId || table._id === existingView?.tableId
     })
+    if (tableIndex === -1) {
+      return
+    }
+
+    // Handle deletion
+    if (!view) {
+      tables.update(state => {
+        delete state.list[tableIndex].views[existingView.name]
+        return state
+      })
+      return
+    }
+
+    // Add new view
+    if (!existingView) {
+      tables.update(state => {
+        state.list[tableIndex].views[view.name] = view
+        return state
+      })
+    }
+
+    // Update existing view
+    else {
+      tables.update(state => {
+        // Remove old view
+        delete state.list[tableIndex].views[existingView.name]
+
+        // Add new view
+        state.list[tableIndex].views[view.name] = view
+        return state
+      })
+    }
   }
 
   return {
@@ -76,6 +96,7 @@ export function createViewsV2Store() {
     delete: deleteView,
     create,
     save,
+    replaceView,
   }
 }
 
diff --git a/packages/frontend-core/src/components/grid/stores/datasource.js b/packages/frontend-core/src/components/grid/stores/datasource.js
index 5ace5ceef8..d5017aa8d1 100644
--- a/packages/frontend-core/src/components/grid/stores/datasource.js
+++ b/packages/frontend-core/src/components/grid/stores/datasource.js
@@ -80,7 +80,7 @@ export const createActions = context => {
 
     // Broadcast change to external state can be updated, as this change
     // will not be received by the builder websocket because we caused it ourselves
-    dispatch("updatedefinition", newDefinition)
+    dispatch("updatedatasource", newDefinition)
   }
 
   // Adds a row to the datasource

From 0a87e3502e2ad720d49facf102f04f062cd1d74d Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Fri, 4 Aug 2023 14:58:12 +0100
Subject: [PATCH 02/19] Fix other user selection state for viewV2s

---
 .../components/backend/TableNavigator/TableNavigator.svelte    | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/packages/builder/src/components/backend/TableNavigator/TableNavigator.svelte b/packages/builder/src/components/backend/TableNavigator/TableNavigator.svelte
index f1965117ae..056a36c4a7 100644
--- a/packages/builder/src/components/backend/TableNavigator/TableNavigator.svelte
+++ b/packages/builder/src/components/backend/TableNavigator/TableNavigator.svelte
@@ -58,7 +58,8 @@
               $goto(`./view/v1/${encodeURIComponent(name)}`)
             }
           }}
-          selectedBy={$userSelectedResourceMap[name]}
+          selectedBy={$userSelectedResourceMap[name] ||
+            $userSelectedResourceMap[view.id]}
         >
           <EditViewPopover {view} />
         </NavItem>

From 938a5a445f87b32cc0672ba3feab58c3eb917eba Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Fri, 4 Aug 2023 15:04:58 +0100
Subject: [PATCH 03/19] Add multi dev collab for viewV2 definitions

---
 packages/server/src/api/controllers/view/viewsV2.ts | 12 +++++++++++-
 packages/server/src/sdk/app/views/index.ts          |  3 ++-
 2 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/packages/server/src/api/controllers/view/viewsV2.ts b/packages/server/src/api/controllers/view/viewsV2.ts
index 226b7201b8..ba97fff8b6 100644
--- a/packages/server/src/api/controllers/view/viewsV2.ts
+++ b/packages/server/src/api/controllers/view/viewsV2.ts
@@ -8,6 +8,7 @@ import {
   ViewV2,
   RequiredKeys,
 } from "@budibase/types"
+import { builderSocket } from "../../../websockets"
 
 async function parseSchemaUI(ctx: Ctx, view: CreateViewRequest) {
   if (!view.schema) {
@@ -86,6 +87,9 @@ export async function create(ctx: Ctx<CreateViewRequest, ViewResponse>) {
   ctx.body = {
     data: result,
   }
+
+  const table = await sdk.tables.getTable(tableId)
+  builderSocket?.emitTableUpdate(ctx, table)
 }
 
 export async function update(ctx: Ctx<UpdateViewRequest, ViewResponse>) {
@@ -118,11 +122,17 @@ export async function update(ctx: Ctx<UpdateViewRequest, ViewResponse>) {
   ctx.body = {
     data: result,
   }
+
+  const table = await sdk.tables.getTable(tableId)
+  builderSocket?.emitTableUpdate(ctx, table)
 }
 
 export async function remove(ctx: Ctx) {
   const { viewId } = ctx.params
 
-  await sdk.views.remove(viewId)
+  const view = await sdk.views.remove(viewId)
   ctx.status = 204
+
+  const table = await sdk.tables.getTable(view.tableId)
+  builderSocket?.emitTableUpdate(ctx, table)
 }
diff --git a/packages/server/src/sdk/app/views/index.ts b/packages/server/src/sdk/app/views/index.ts
index d87715f49b..aafaab3a36 100644
--- a/packages/server/src/sdk/app/views/index.ts
+++ b/packages/server/src/sdk/app/views/index.ts
@@ -56,7 +56,7 @@ export function isV2(view: View | ViewV2): view is ViewV2 {
   return (view as ViewV2).version === 2
 }
 
-export async function remove(viewId: string): Promise<void> {
+export async function remove(viewId: string): Promise<ViewV2> {
   const db = context.getAppDB()
 
   const view = await get(viewId)
@@ -67,6 +67,7 @@ export async function remove(viewId: string): Promise<void> {
 
   delete table.views![view?.name]
   await db.put(table)
+  return view
 }
 
 export function enrichSchema(view: View | ViewV2, tableSchema: TableSchema) {

From ffe82e18e993b7a402cfccdd6e9b319898bced77 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Fri, 4 Aug 2023 15:07:44 +0100
Subject: [PATCH 04/19] Remove log

---
 packages/builder/src/stores/backend/viewsV2.js | 1 -
 1 file changed, 1 deletion(-)

diff --git a/packages/builder/src/stores/backend/viewsV2.js b/packages/builder/src/stores/backend/viewsV2.js
index 6fa3b52bb9..dcc2c27152 100644
--- a/packages/builder/src/stores/backend/viewsV2.js
+++ b/packages/builder/src/stores/backend/viewsV2.js
@@ -48,7 +48,6 @@ export function createViewsV2Store() {
 
   // Handles external updates of tables
   const replaceView = (viewId, view) => {
-    console.log("replace", viewId, view)
     if (!viewId) {
       return
     }

From 8a9db6d8deef8d27ec8909e3ef038e781a809951 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Fri, 4 Aug 2023 15:15:43 +0100
Subject: [PATCH 05/19] Remove log

---
 .../src/components/backend/DataTable/ViewV2DataTable.svelte      | 1 -
 1 file changed, 1 deletion(-)

diff --git a/packages/builder/src/components/backend/DataTable/ViewV2DataTable.svelte b/packages/builder/src/components/backend/DataTable/ViewV2DataTable.svelte
index 859ce11280..e4e1adda36 100644
--- a/packages/builder/src/components/backend/DataTable/ViewV2DataTable.svelte
+++ b/packages/builder/src/components/backend/DataTable/ViewV2DataTable.svelte
@@ -13,7 +13,6 @@
   }
 
   const handleGridViewUpdate = async e => {
-    console.log("update")
     viewsV2.replaceView(id, e.detail)
   }
 </script>

From ef4ff87d8a3aacd73d23ef61f43b5a175c47a2b7 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Fri, 4 Aug 2023 15:17:30 +0100
Subject: [PATCH 06/19] Remove filter button on views for now

---
 .../src/components/backend/DataTable/ViewV2DataTable.svelte   | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/packages/builder/src/components/backend/DataTable/ViewV2DataTable.svelte b/packages/builder/src/components/backend/DataTable/ViewV2DataTable.svelte
index e4e1adda36..789f7d730a 100644
--- a/packages/builder/src/components/backend/DataTable/ViewV2DataTable.svelte
+++ b/packages/builder/src/components/backend/DataTable/ViewV2DataTable.svelte
@@ -26,9 +26,7 @@
     showAvatars={false}
     on:updatedatasource={handleGridViewUpdate}
   >
-    <svelte:fragment slot="filter">
-      <GridFilterButton />
-    </svelte:fragment>
+    <svelte:fragment slot="filter" />
     <svelte:fragment slot="controls">
       <GridCreateEditRowModal />
     </svelte:fragment>

From dd4b3047369d7b0d8fdaf5d0ab03eaa1303ac854 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Fri, 4 Aug 2023 15:20:30 +0100
Subject: [PATCH 07/19] Fix creating views

---
 .../buttons/grid/GridCreateViewButton.svelte          |  4 ++--
 .../GridCreateViewModal.svelte}                       | 11 +++++------
 2 files changed, 7 insertions(+), 8 deletions(-)
 rename packages/builder/src/components/backend/DataTable/modals/{CreateViewModal.svelte => grid/GridCreateViewModal.svelte} (82%)

diff --git a/packages/builder/src/components/backend/DataTable/buttons/grid/GridCreateViewButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/grid/GridCreateViewButton.svelte
index 33c416d7ef..3244ce3277 100644
--- a/packages/builder/src/components/backend/DataTable/buttons/grid/GridCreateViewButton.svelte
+++ b/packages/builder/src/components/backend/DataTable/buttons/grid/GridCreateViewButton.svelte
@@ -1,7 +1,7 @@
 <script>
   import { getContext } from "svelte"
   import { Modal, ActionButton } from "@budibase/bbui"
-  import CreateViewModal from "../../modals/CreateViewModal.svelte"
+  import GridCreateViewModal from "../../modals/grid/GridCreateViewModal.svelte"
 
   const { rows, columns } = getContext("grid")
 
@@ -14,5 +14,5 @@
   Add view
 </ActionButton>
 <Modal bind:this={modal}>
-  <CreateViewModal />
+  <GridCreateViewModal />
 </Modal>
diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateViewModal.svelte b/packages/builder/src/components/backend/DataTable/modals/grid/GridCreateViewModal.svelte
similarity index 82%
rename from packages/builder/src/components/backend/DataTable/modals/CreateViewModal.svelte
rename to packages/builder/src/components/backend/DataTable/modals/grid/GridCreateViewModal.svelte
index afb9efa53f..1531fd5187 100644
--- a/packages/builder/src/components/backend/DataTable/modals/CreateViewModal.svelte
+++ b/packages/builder/src/components/backend/DataTable/modals/grid/GridCreateViewModal.svelte
@@ -5,14 +5,13 @@
   import { viewsV2 } from "stores/backend"
   import { LuceneUtils } from "@budibase/frontend-core"
 
-  const { filter, sort, table } = getContext("grid")
+  const { filter, sort, definition } = getContext("grid")
 
   $: query = LuceneUtils.buildLuceneQuery($filter)
 
   let name
 
-  $: console.log($table)
-  $: views = Object.keys($table?.views || {})
+  $: views = Object.keys($definition?.views || {})
   $: nameExists = views.includes(name?.trim())
 
   const saveView = async () => {
@@ -20,14 +19,14 @@
     try {
       const newView = await viewsV2.create({
         name,
-        tableId: $table._id,
+        tableId: $definition._id,
         query,
         sort: {
           field: $sort.column,
           order: $sort.order,
         },
-        schema: $table.schema,
-        primaryDisplay: $table.primaryDisplay,
+        schema: $definition.schema,
+        primaryDisplay: $definition.primaryDisplay,
       })
       notifications.success(`View ${name} created`)
       $goto(`../../view/v2/${newView.id}`)

From 9652356ee0be4da060f5cc2a0244776d9a5f0df1 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Fri, 4 Aug 2023 15:26:06 +0100
Subject: [PATCH 08/19] Improve sanity checking of valid datasources to prevent
 erroneous API calls

---
 .../frontend-core/src/components/grid/stores/datasource.js | 7 +++++++
 packages/frontend-core/src/components/grid/stores/rows.js  | 2 +-
 packages/frontend-core/src/components/grid/stores/table.js | 5 +++++
 .../frontend-core/src/components/grid/stores/viewV2.js     | 7 +++++++
 4 files changed, 20 insertions(+), 1 deletion(-)

diff --git a/packages/frontend-core/src/components/grid/stores/datasource.js b/packages/frontend-core/src/components/grid/stores/datasource.js
index d5017aa8d1..61caa79734 100644
--- a/packages/frontend-core/src/components/grid/stores/datasource.js
+++ b/packages/frontend-core/src/components/grid/stores/datasource.js
@@ -98,10 +98,16 @@ export const createActions = context => {
     return await getAPI()?.actions.deleteRows(rows)
   }
 
+  // Gets a single row from a datasource
   const getRow = async id => {
     return await getAPI()?.actions.getRow(id)
   }
 
+  // Checks if a certain datasource config is valid
+  const isDatasourceValid = datasource => {
+    return getAPI()?.actions.isDatasourceValid(datasource)
+  }
+
   return {
     datasource: {
       ...datasource,
@@ -112,6 +118,7 @@ export const createActions = context => {
         updateRow,
         deleteRows,
         getRow,
+        isDatasourceValid,
       },
     },
   }
diff --git a/packages/frontend-core/src/components/grid/stores/rows.js b/packages/frontend-core/src/components/grid/stores/rows.js
index d876b5f8df..df881eefdd 100644
--- a/packages/frontend-core/src/components/grid/stores/rows.js
+++ b/packages/frontend-core/src/components/grid/stores/rows.js
@@ -98,7 +98,7 @@ export const createActions = context => {
     loading.set(true)
 
     // Abandon if we don't have a valid datasource
-    if (!$datasource) {
+    if (!datasource.actions.isDatasourceValid($datasource)) {
       return
     }
 
diff --git a/packages/frontend-core/src/components/grid/stores/table.js b/packages/frontend-core/src/components/grid/stores/table.js
index 5c3d2c9ef2..bf4eebcda4 100644
--- a/packages/frontend-core/src/components/grid/stores/table.js
+++ b/packages/frontend-core/src/components/grid/stores/table.js
@@ -25,6 +25,10 @@ export const createActions = context => {
     })
   }
 
+  const isDatasourceValid = datasource => {
+    return datasource?.type === "table" && datasource?.tableId
+  }
+
   const getRow = async id => {
     const res = await API.searchTable({
       tableId: get(datasource).tableId,
@@ -48,6 +52,7 @@ export const createActions = context => {
         updateRow: saveRow,
         deleteRows,
         getRow,
+        isDatasourceValid,
       },
     },
   }
diff --git a/packages/frontend-core/src/components/grid/stores/viewV2.js b/packages/frontend-core/src/components/grid/stores/viewV2.js
index b2d80acd1a..31ccba8c76 100644
--- a/packages/frontend-core/src/components/grid/stores/viewV2.js
+++ b/packages/frontend-core/src/components/grid/stores/viewV2.js
@@ -60,6 +60,12 @@ export const createActions = context => {
     return res?.rows?.[0]
   }
 
+  const isDatasourceValid = datasource => {
+    return (
+      datasource?.type === "viewV2" && datasource?.id && datasource?.tableId
+    )
+  }
+
   return {
     viewV2: {
       actions: {
@@ -69,6 +75,7 @@ export const createActions = context => {
         updateRow,
         deleteRows,
         getRow,
+        isDatasourceValid,
       },
     },
   }

From 7aa5211d8734e47efb0b2df1b25896a5ffa212f8 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Fri, 4 Aug 2023 15:40:46 +0100
Subject: [PATCH 09/19] Fix grid block

---
 .../client/src/components/app/GridBlock.svelte    | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/packages/client/src/components/app/GridBlock.svelte b/packages/client/src/components/app/GridBlock.svelte
index dd67a5657b..a771cca0e3 100644
--- a/packages/client/src/components/app/GridBlock.svelte
+++ b/packages/client/src/components/app/GridBlock.svelte
@@ -38,11 +38,11 @@
   class:in-builder={$builderStore.inBuilder}
 >
   <Grid
-    tableId={table?.tableId}
+    datasource={{
+      type: "table",
+      tableId: table?.tableId,
+    }}
     {API}
-    {allowAddRows}
-    {allowEditRows}
-    {allowDeleteRows}
     {stripeRows}
     {initialFilter}
     {initialSortColumn}
@@ -50,9 +50,12 @@
     {fixedRowHeight}
     {columnWhitelist}
     {schemaOverrides}
+    canAddRows={allowAddRows}
+    canEditRows={allowEditRows}
+    canDeleteRows={allowDeleteRows}
+    canExpandRows={false}
+    canSaveSchema={false}
     showControls={false}
-    allowExpandRows={false}
-    allowSchemaChanges={false}
     notifySuccess={notificationStore.actions.success}
     notifyError={notificationStore.actions.error}
   />

From aadcd0ae137889ef2977b64b21d692948bc81fcf Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 7 Aug 2023 17:20:53 +0300
Subject: [PATCH 10/19] Typing update table

---
 .../server/src/api/controllers/table/internal.ts   | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/packages/server/src/api/controllers/table/internal.ts b/packages/server/src/api/controllers/table/internal.ts
index 7317d11e18..2e8a857d22 100644
--- a/packages/server/src/api/controllers/table/internal.ts
+++ b/packages/server/src/api/controllers/table/internal.ts
@@ -9,7 +9,7 @@ import {
   fixAutoColumnSubType,
 } from "../../../utilities/rowProcessor"
 import { runStaticFormulaChecks } from "./bulkFormula"
-import { Table } from "@budibase/types"
+import { Table, ViewStatisticsSchema } from "@budibase/types"
 import { quotas } from "@budibase/pro"
 import isEqual from "lodash/isEqual"
 import { cloneDeep } from "lodash/fp"
@@ -36,7 +36,9 @@ function checkAutoColumns(table: Table, oldTable?: Table) {
 export async function save(ctx: any) {
   const db = context.getAppDB()
   const { rows, ...rest } = ctx.request.body
-  let tableToSave = {
+  let tableToSave: Table & {
+    _rename?: { old: string; updated: string } | null
+  } = {
     type: "table",
     _id: generateTableID(),
     views: {},
@@ -44,7 +46,7 @@ export async function save(ctx: any) {
   }
 
   // if the table obj had an _id then it will have been retrieved
-  let oldTable
+  let oldTable: Table | undefined
   if (ctx.request.body && ctx.request.body._id) {
     oldTable = await sdk.tables.getTable(ctx.request.body._id)
   }
@@ -97,7 +99,11 @@ export async function save(ctx: any) {
     const tableView = tableToSave.views[view]
     if (!tableView) continue
 
-    if (tableView.schema.group || tableView.schema.field) continue
+    if (
+      (tableView.schema as ViewStatisticsSchema).group ||
+      tableView.schema.field
+    )
+      continue
     tableView.schema = tableToSave.schema
   }
 

From b7dd40eb6a5132aa8452a14c54edcc0c4a25d73d Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 7 Aug 2023 17:21:36 +0300
Subject: [PATCH 11/19] Update view search to be called as a post

---
 packages/frontend-core/src/api/viewsV2.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/frontend-core/src/api/viewsV2.js b/packages/frontend-core/src/api/viewsV2.js
index f2848b55f3..cd89f1097a 100644
--- a/packages/frontend-core/src/api/viewsV2.js
+++ b/packages/frontend-core/src/api/viewsV2.js
@@ -24,7 +24,7 @@ export const buildViewV2Endpoints = API => ({
    * @param viewId the id of the view
    */
   fetch: async viewId => {
-    return await API.get({ url: `/api/v2/views/${viewId}/search` })
+    return await API.post({ url: `/api/v2/views/${viewId}/search` })
   },
   /**
    * Delete a view

From af2fac695811a5e852208f56361192a2b18a6a69 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 7 Aug 2023 17:21:58 +0300
Subject: [PATCH 12/19] Fix overriding view v2 schema

---
 packages/server/src/api/controllers/table/internal.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/server/src/api/controllers/table/internal.ts b/packages/server/src/api/controllers/table/internal.ts
index 2e8a857d22..a8074087ed 100644
--- a/packages/server/src/api/controllers/table/internal.ts
+++ b/packages/server/src/api/controllers/table/internal.ts
@@ -97,7 +97,7 @@ export async function save(ctx: any) {
   // update schema of non-statistics views when new columns are added
   for (let view in tableToSave.views) {
     const tableView = tableToSave.views[view]
-    if (!tableView) continue
+    if (!tableView || sdk.views.isV2(tableView)) continue
 
     if (
       (tableView.schema as ViewStatisticsSchema).group ||

From a72c94426f6e92e4fde4d205534598494bb3efa0 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Tue, 8 Aug 2023 09:29:05 +0100
Subject: [PATCH 13/19] Hydrate view filters and allow filter changes to be
 saved to views

---
 .../backend/DataTable/ViewV2DataTable.svelte  |  4 +-
 .../buttons/grid/GridFilterButton.svelte      |  3 --
 .../modals/grid/GridCreateViewModal.svelte    |  5 +-
 .../src/components/grid/stores/table.js       | 49 ++++++++++++-------
 .../src/components/grid/stores/viewV2.js      | 20 +++++++-
 5 files changed, 52 insertions(+), 29 deletions(-)

diff --git a/packages/builder/src/components/backend/DataTable/ViewV2DataTable.svelte b/packages/builder/src/components/backend/DataTable/ViewV2DataTable.svelte
index 789f7d730a..e4e1adda36 100644
--- a/packages/builder/src/components/backend/DataTable/ViewV2DataTable.svelte
+++ b/packages/builder/src/components/backend/DataTable/ViewV2DataTable.svelte
@@ -26,7 +26,9 @@
     showAvatars={false}
     on:updatedatasource={handleGridViewUpdate}
   >
-    <svelte:fragment slot="filter" />
+    <svelte:fragment slot="filter">
+      <GridFilterButton />
+    </svelte:fragment>
     <svelte:fragment slot="controls">
       <GridCreateEditRowModal />
     </svelte:fragment>
diff --git a/packages/builder/src/components/backend/DataTable/buttons/grid/GridFilterButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/grid/GridFilterButton.svelte
index ae4d39483e..f927f84876 100644
--- a/packages/builder/src/components/backend/DataTable/buttons/grid/GridFilterButton.svelte
+++ b/packages/builder/src/components/backend/DataTable/buttons/grid/GridFilterButton.svelte
@@ -4,9 +4,6 @@
 
   const { columns, datasource, filter, definition } = getContext("grid")
 
-  // Wipe filter whenever table ID changes to avoid using stale filters
-  $: $datasource, filter.set([])
-
   const onFilter = e => {
     filter.set(e.detail || [])
   }
diff --git a/packages/builder/src/components/backend/DataTable/modals/grid/GridCreateViewModal.svelte b/packages/builder/src/components/backend/DataTable/modals/grid/GridCreateViewModal.svelte
index 1531fd5187..ba55a3c540 100644
--- a/packages/builder/src/components/backend/DataTable/modals/grid/GridCreateViewModal.svelte
+++ b/packages/builder/src/components/backend/DataTable/modals/grid/GridCreateViewModal.svelte
@@ -3,12 +3,9 @@
   import { Input, notifications, ModalContent } from "@budibase/bbui"
   import { goto } from "@roxi/routify"
   import { viewsV2 } from "stores/backend"
-  import { LuceneUtils } from "@budibase/frontend-core"
 
   const { filter, sort, definition } = getContext("grid")
 
-  $: query = LuceneUtils.buildLuceneQuery($filter)
-
   let name
 
   $: views = Object.keys($definition?.views || {})
@@ -20,7 +17,7 @@
       const newView = await viewsV2.create({
         name,
         tableId: $definition._id,
-        query,
+        query: $filter,
         sort: {
           field: $sort.column,
           order: $sort.order,
diff --git a/packages/frontend-core/src/components/grid/stores/table.js b/packages/frontend-core/src/components/grid/stores/table.js
index bf4eebcda4..291e639572 100644
--- a/packages/frontend-core/src/components/grid/stores/table.js
+++ b/packages/frontend-core/src/components/grid/stores/table.js
@@ -61,38 +61,49 @@ export const createActions = context => {
 export const initialise = context => {
   const { datasource, fetch, filter, sort, definition } = context
 
+  // Wipe filter whenever table ID changes to avoid using stale filters
+  definition.subscribe(() => {
+    if (get(datasource)?.type !== "table") {
+      return
+    }
+    filter.set([])
+  })
+
   // Update fetch when filter changes
   filter.subscribe($filter => {
-    if (get(datasource)?.type === "table") {
-      get(fetch)?.update({
-        filter: $filter,
-      })
+    if (get(datasource)?.type !== "table") {
+      return
     }
+    get(fetch)?.update({
+      filter: $filter,
+    })
   })
 
   // Update fetch when sorting changes
   sort.subscribe($sort => {
-    if (get(datasource)?.type === "table") {
-      get(fetch)?.update({
-        sortOrder: $sort.order,
-        sortColumn: $sort.column,
-      })
+    if (get(datasource)?.type !== "table") {
+      return
     }
+    get(fetch)?.update({
+      sortOrder: $sort.order,
+      sortColumn: $sort.column,
+    })
   })
 
   // Ensure sorting UI reflects the fetch state whenever we reset the fetch,
   // which triggers a new definition
   definition.subscribe(() => {
-    if (get(datasource)?.type === "table") {
-      const $fetch = get(fetch)
-      if (!$fetch) {
-        return
-      }
-      const { sortColumn, sortOrder } = get($fetch)
-      sort.set({
-        column: sortColumn,
-        order: sortOrder,
-      })
+    if (get(datasource)?.type !== "table") {
+      return
     }
+    const $fetch = get(fetch)
+    if (!$fetch) {
+      return
+    }
+    const { sortColumn, sortOrder } = get($fetch)
+    sort.set({
+      column: sortColumn,
+      order: sortOrder,
+    })
   })
 }
diff --git a/packages/frontend-core/src/components/grid/stores/viewV2.js b/packages/frontend-core/src/components/grid/stores/viewV2.js
index 31ccba8c76..cb6f4cc3e6 100644
--- a/packages/frontend-core/src/components/grid/stores/viewV2.js
+++ b/packages/frontend-core/src/components/grid/stores/viewV2.js
@@ -82,9 +82,9 @@ export const createActions = context => {
 }
 
 export const initialise = context => {
-  const { definition, datasource, sort, rows } = context
+  const { definition, datasource, sort, rows, filter } = context
 
-  // For views, keep sort state in line with the view definition
+  // Keep sort and filter state in line with the view definition
   definition.subscribe($definition => {
     if (!$definition || get(datasource)?.type !== "viewV2") {
       return
@@ -93,6 +93,7 @@ export const initialise = context => {
       column: $definition.sort?.field,
       order: $definition.sort?.order,
     })
+    filter.set($definition.query || [])
   })
 
   // When sorting changes, ensure view definition is kept up to date
@@ -115,4 +116,19 @@ export const initialise = context => {
       await rows.actions.refreshData()
     }
   })
+
+  // When filters change, ensure view definition is kept up to date
+  filter.subscribe(async $filter => {
+    const $view = get(definition)
+    if (!$view || get(datasource)?.type !== "viewV2") {
+      return
+    }
+    if (JSON.stringify($filter) !== JSON.stringify($view.query)) {
+      await datasource.actions.saveDefinition({
+        ...$view,
+        query: $filter,
+      })
+      await rows.actions.refreshData()
+    }
+  })
 }

From 4db33b9d753b2b462f43ab6444b3540a6de6d8e4 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Tue, 8 Aug 2023 13:13:27 +0100
Subject: [PATCH 14/19] Enable pagination on views

---
 packages/frontend-core/src/api/viewsV2.js     | 28 +++++++++++++++--
 packages/frontend-core/src/fetch/DataFetch.js |  2 +-
 .../frontend-core/src/fetch/ViewV2Fetch.js    | 30 ++++++++++++++++---
 3 files changed, 53 insertions(+), 7 deletions(-)

diff --git a/packages/frontend-core/src/api/viewsV2.js b/packages/frontend-core/src/api/viewsV2.js
index cd89f1097a..25cb7d6884 100644
--- a/packages/frontend-core/src/api/viewsV2.js
+++ b/packages/frontend-core/src/api/viewsV2.js
@@ -22,9 +22,33 @@ export const buildViewV2Endpoints = API => ({
   /**
    * Fetches all rows in a view
    * @param viewId the id of the view
+   * @param paginate whether to paginate or not
+   * @param limit page size
+   * @param bookmark pagination cursor
+   * @param sort sort column
+   * @param sortOrder sort order
+   * @param sortType sort type (text or numeric)
    */
-  fetch: async viewId => {
-    return await API.post({ url: `/api/v2/views/${viewId}/search` })
+  fetch: async ({
+    viewId,
+    paginate,
+    limit,
+    bookmark,
+    sort,
+    sortOrder,
+    sortType,
+  }) => {
+    return await API.post({
+      url: `/api/v2/views/${viewId}/search`,
+      body: {
+        paginate,
+        limit,
+        bookmark,
+        sort,
+        sortOrder,
+        sortType,
+      },
+    })
   },
   /**
    * Delete a view
diff --git a/packages/frontend-core/src/fetch/DataFetch.js b/packages/frontend-core/src/fetch/DataFetch.js
index a9803747ae..28fa68afe9 100644
--- a/packages/frontend-core/src/fetch/DataFetch.js
+++ b/packages/frontend-core/src/fetch/DataFetch.js
@@ -116,7 +116,7 @@ export default class DataFetch {
   async getInitialData() {
     const { datasource, filter, paginate } = this.options
 
-    // Fetch datasource definition and extract filter and sort if configured
+    // Fetch datasource definition and extract sort properties if configured
     const definition = await this.getDefinition(datasource)
     if (definition?.sort?.field) {
       this.options.sortColumn = definition.sort.field
diff --git a/packages/frontend-core/src/fetch/ViewV2Fetch.js b/packages/frontend-core/src/fetch/ViewV2Fetch.js
index 9bc61f0383..3b8e432cd0 100644
--- a/packages/frontend-core/src/fetch/ViewV2Fetch.js
+++ b/packages/frontend-core/src/fetch/ViewV2Fetch.js
@@ -1,8 +1,12 @@
 import DataFetch from "./DataFetch.js"
+import { get } from "svelte/store"
 
 export default class ViewV2Fetch extends DataFetch {
   determineFeatureFlags() {
     return {
+      // The API does not actually support dynamic filtering, but since views
+      // have filters built in we don't want to perform client side filtering
+      // which would happen if we marked this as false
       supportsSearch: true,
       supportsSort: true,
       supportsPagination: true,
@@ -29,12 +33,30 @@ export default class ViewV2Fetch extends DataFetch {
   }
 
   async getData() {
-    const { datasource } = this.options
+    const { datasource, limit, sortColumn, sortOrder, sortType, paginate } =
+      this.options
+    const { cursor } = get(this.store)
     try {
-      const res = await this.API.viewV2.fetch(datasource.id)
-      return { rows: res?.rows || [] }
+      const res = await this.API.viewV2.fetch({
+        viewId: datasource.id,
+        paginate,
+        limit,
+        bookmark: cursor,
+        sort: sortColumn,
+        sortOrder,
+        sortType,
+      })
+      return {
+        rows: res?.rows || [],
+        hasNextPage: res?.hasNextPage || false,
+        cursor: res?.bookmark || null,
+      }
     } catch (error) {
-      return { rows: [] }
+      return {
+        rows: [],
+        hasNextPage: false,
+        error,
+      }
     }
   }
 }

From dcf26dd0cb6758b203fc3ea8f6bb777d364d4c79 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 9 Aug 2023 12:53:08 +0300
Subject: [PATCH 15/19] Fix overriding viewv2 schemas

---
 packages/server/src/api/controllers/table/internal.ts | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/packages/server/src/api/controllers/table/internal.ts b/packages/server/src/api/controllers/table/internal.ts
index a8074087ed..dc6e246a96 100644
--- a/packages/server/src/api/controllers/table/internal.ts
+++ b/packages/server/src/api/controllers/table/internal.ts
@@ -97,7 +97,13 @@ export async function save(ctx: any) {
   // update schema of non-statistics views when new columns are added
   for (let view in tableToSave.views) {
     const tableView = tableToSave.views[view]
-    if (!tableView || sdk.views.isV2(tableView)) continue
+    if (!tableView) continue
+
+    if (sdk.views.isV2(tableView)) {
+      // We don't want to modify views from the tables controller
+      tableToSave.views[view] = oldTable!.views![view]
+      continue
+    }
 
     if (
       (tableView.schema as ViewStatisticsSchema).group ||

From 06810d43d97b335ea75a9cfe89ff2148c4a93dd7 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Wed, 9 Aug 2023 15:09:53 +0100
Subject: [PATCH 16/19] Add support for viewV2 datasources in client library

---
 .../settings/controls/DataSourceSelect.svelte | 36 ++++++++++++++-----
 packages/client/src/utils/schema.js           |  2 ++
 .../frontend-core/src/fetch/ViewV2Fetch.js    |  2 +-
 3 files changed, 30 insertions(+), 10 deletions(-)

diff --git a/packages/builder/src/components/design/settings/controls/DataSourceSelect.svelte b/packages/builder/src/components/design/settings/controls/DataSourceSelect.svelte
index 6da1bd6187..69249d3ba5 100644
--- a/packages/builder/src/components/design/settings/controls/DataSourceSelect.svelte
+++ b/packages/builder/src/components/design/settings/controls/DataSourceSelect.svelte
@@ -39,15 +39,33 @@
     tableId: m._id,
     type: "table",
   }))
-  $: views = $tablesStore.list.reduce((acc, cur) => {
-    let viewsArr = Object.entries(cur.views || {}).map(([key, value]) => ({
-      label: key,
-      name: key,
-      ...value,
-      type: "view",
-    }))
-    return [...acc, ...viewsArr]
-  }, [])
+  $: viewsV1 = $tablesStore.list.reduce(
+    (acc, table) => [
+      ...acc,
+      ...Object.values(table.views || {})
+        .filter(view => view.version !== 2)
+        .map(view => ({
+          ...view,
+          label: view.name,
+          type: "view",
+        })),
+    ],
+    []
+  )
+  $: viewsV2 = $tablesStore.list.reduce(
+    (acc, table) => [
+      ...acc,
+      ...Object.values(table.views || {})
+        .filter(view => view.version === 2)
+        .map(view => ({
+          ...view,
+          label: view.name,
+          type: "viewV2",
+        })),
+    ],
+    []
+  )
+  $: views = [...(viewsV1 || []), ...(viewsV2 || [])]
   $: queries = $queriesStore.list
     .filter(q => showAllQueries || q.queryVerb === "read" || q.readable)
     .map(query => ({
diff --git a/packages/client/src/utils/schema.js b/packages/client/src/utils/schema.js
index e450c516d7..800e9b9be1 100644
--- a/packages/client/src/utils/schema.js
+++ b/packages/client/src/utils/schema.js
@@ -6,6 +6,7 @@ import RelationshipFetch from "@budibase/frontend-core/src/fetch/RelationshipFet
 import NestedProviderFetch from "@budibase/frontend-core/src/fetch/NestedProviderFetch.js"
 import FieldFetch from "@budibase/frontend-core/src/fetch/FieldFetch.js"
 import JSONArrayFetch from "@budibase/frontend-core/src/fetch/JSONArrayFetch.js"
+import ViewV2Fetch from "@budibase/frontend-core/src/fetch/ViewV2Fetch.js"
 
 /**
  * Fetches the schema of any kind of datasource.
@@ -21,6 +22,7 @@ export const fetchDatasourceSchema = async (
   const handler = {
     table: TableFetch,
     view: ViewFetch,
+    viewV2: ViewV2Fetch,
     query: QueryFetch,
     link: RelationshipFetch,
     provider: NestedProviderFetch,
diff --git a/packages/frontend-core/src/fetch/ViewV2Fetch.js b/packages/frontend-core/src/fetch/ViewV2Fetch.js
index 3b8e432cd0..83af9453ca 100644
--- a/packages/frontend-core/src/fetch/ViewV2Fetch.js
+++ b/packages/frontend-core/src/fetch/ViewV2Fetch.js
@@ -13,7 +13,7 @@ export default class ViewV2Fetch extends DataFetch {
     }
   }
 
-  async getSchema(datasource, definition) {
+  getSchema(datasource, definition) {
     return definition?.schema
   }
 

From 08b348c1d4932b18b3db02046baefe7f98ec470b Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Wed, 9 Aug 2023 16:22:12 +0100
Subject: [PATCH 17/19] Allow selecting viewV2s for grid blocks and form blocks

---
 .../settings/controls/TableSelect.svelte      | 44 +++++++++++++++----
 packages/client/manifest.json                 |  4 +-
 .../src/components/app/GridBlock.svelte       |  5 +--
 3 files changed, 39 insertions(+), 14 deletions(-)

diff --git a/packages/builder/src/components/design/settings/controls/TableSelect.svelte b/packages/builder/src/components/design/settings/controls/TableSelect.svelte
index 384bbe1e3a..03b739e09f 100644
--- a/packages/builder/src/components/design/settings/controls/TableSelect.svelte
+++ b/packages/builder/src/components/design/settings/controls/TableSelect.svelte
@@ -7,22 +7,50 @@
 
   const dispatch = createEventDispatcher()
 
-  $: tables = $tablesStore.list.map(m => ({
-    label: m.name,
-    tableId: m._id,
+  $: tables = $tablesStore.list.map(table => ({
+    ...table,
     type: "table",
+    label: table.name,
+    key: table._id,
   }))
+  $: views = $tablesStore.list.reduce(
+    (acc, table) => [
+      ...acc,
+      ...Object.values(table.views || {})
+        .filter(view => view.version === 2)
+        .map(view => ({
+          ...view,
+          type: "viewV2",
+          label: view.name,
+          key: view.id,
+        })),
+    ],
+    []
+  )
+  $: options = [...(tables || []), ...(views || [])]
+  $: {
+    // Migrate old table values before "key" existed
+    if (value && !value.key) {
+      console.log("migrate")
+      dispatch(
+        "change",
+        tables.find(x => x.tableId === value.tableId)
+      )
+    }
+  }
 
   const onChange = e => {
-    const dataSource = tables?.find(x => x.tableId === e.detail)
-    dispatch("change", dataSource)
+    dispatch(
+      "change",
+      options.find(x => x.key === e.detail)
+    )
   }
 </script>
 
 <Select
   on:change={onChange}
-  value={value?.tableId}
-  options={tables}
-  getOptionValue={x => x.tableId}
+  value={value?.key}
+  {options}
+  getOptionValue={x => x.key}
   getOptionLabel={x => x.label}
 />
diff --git a/packages/client/manifest.json b/packages/client/manifest.json
index b6a231917c..e8063103b4 100644
--- a/packages/client/manifest.json
+++ b/packages/client/manifest.json
@@ -5272,7 +5272,7 @@
       },
       {
         "type": "table",
-        "label": "Table",
+        "label": "Data",
         "key": "dataSource"
       },
       {
@@ -5547,7 +5547,7 @@
     "settings": [
       {
         "type": "table",
-        "label": "Table",
+        "label": "Data",
         "key": "table",
         "required": true
       },
diff --git a/packages/client/src/components/app/GridBlock.svelte b/packages/client/src/components/app/GridBlock.svelte
index a771cca0e3..a7c3b8eef2 100644
--- a/packages/client/src/components/app/GridBlock.svelte
+++ b/packages/client/src/components/app/GridBlock.svelte
@@ -38,10 +38,7 @@
   class:in-builder={$builderStore.inBuilder}
 >
   <Grid
-    datasource={{
-      type: "table",
-      tableId: table?.tableId,
-    }}
+    datasource={table}
     {API}
     {stripeRows}
     {initialFilter}

From 7a40ad66c6f9bcc322535cce6f0552b33bcc8458 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Wed, 9 Aug 2023 16:30:27 +0100
Subject: [PATCH 18/19] Strip hidden fields from viewV2 schema in client lib

---
 packages/client/src/utils/schema.js | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/packages/client/src/utils/schema.js b/packages/client/src/utils/schema.js
index 800e9b9be1..f20e724a6e 100644
--- a/packages/client/src/utils/schema.js
+++ b/packages/client/src/utils/schema.js
@@ -51,6 +51,15 @@ export const fetchDatasourceSchema = async (
     return null
   }
 
+  // Strip hidden fields from views
+  if (datasource.type === "viewV2") {
+    Object.keys(schema).forEach(field => {
+      if (!schema[field].visible) {
+        delete schema[field]
+      }
+    })
+  }
+
   // Enrich schema with relationships if required
   if (definition?.sql && options?.enrichRelationships) {
     const relationshipAdditions = await getRelationshipSchemaAdditions(schema)

From 5188abe4e542c902108fa8141a9b8a2ebdd70890 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Wed, 9 Aug 2023 16:39:23 +0100
Subject: [PATCH 19/19] Strip hidden fields from viewsV2 in design section

---
 packages/builder/src/builderStore/dataBinding.js | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/packages/builder/src/builderStore/dataBinding.js b/packages/builder/src/builderStore/dataBinding.js
index bbe116721a..615b032d92 100644
--- a/packages/builder/src/builderStore/dataBinding.js
+++ b/packages/builder/src/builderStore/dataBinding.js
@@ -734,9 +734,19 @@ export const getSchemaForDatasource = (asset, datasource, options) => {
     // Determine the schema from the backing entity if not already determined
     if (table && !schema) {
       if (type === "view") {
-        // For views, the schema is pulled from the `views` property of the
-        // table
+        // Old views
         schema = cloneDeep(table.views?.[datasource.name]?.schema)
+      } else if (type === "viewV2") {
+        // New views which are DS+
+        const view = table.views?.[datasource.name]
+        schema = cloneDeep(view?.schema)
+
+        // Strip hidden fields
+        Object.keys(schema || {}).forEach(field => {
+          if (!schema[field].visible) {
+            delete schema[field]
+          }
+        })
       } else if (
         type === "query" &&
         (options.formSchema || options.searchableSchema)