From 5e5dc902d1ce91fa7e112db28c4e98025cae2362 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Thu, 15 Jun 2023 09:04:21 +0100 Subject: [PATCH 1/5] Broadcast datasource change via websocket when making changes to tables --- packages/builder/src/stores/backend/tables.js | 2 - .../src/api/controllers/table/external.ts | 8 ++ packages/server/src/websockets/builder.ts | 80 +++++++++++++------ packages/server/src/websockets/grid.ts | 54 +++++++++---- packages/server/src/websockets/websocket.ts | 22 +++-- packages/types/src/sdk/websocket.ts | 4 + 6 files changed, 125 insertions(+), 45 deletions(-) diff --git a/packages/builder/src/stores/backend/tables.js b/packages/builder/src/stores/backend/tables.js index f8796712a8..d79ed6f072 100644 --- a/packages/builder/src/stores/backend/tables.js +++ b/packages/builder/src/stores/backend/tables.js @@ -1,5 +1,4 @@ import { get, writable, derived } from "svelte/store" -import { datasources } from "./" import { cloneDeep } from "lodash/fp" import { API } from "api" import { SWITCHABLE_TYPES } from "constants/backend" @@ -63,7 +62,6 @@ export function createTablesStore() { const savedTable = await API.saveTable(updatedTable) replaceTable(savedTable._id, savedTable) - await datasources.fetch() select(savedTable._id) return savedTable } diff --git a/packages/server/src/api/controllers/table/external.ts b/packages/server/src/api/controllers/table/external.ts index ee789ddd3a..f35691cb62 100644 --- a/packages/server/src/api/controllers/table/external.ts +++ b/packages/server/src/api/controllers/table/external.ts @@ -26,6 +26,7 @@ import { RelationshipTypes, } from "@budibase/types" import sdk from "../../../sdk" +import { builderSocket } from "../../../websockets" const { cloneDeep } = require("lodash/fp") async function makeTableRequest( @@ -318,6 +319,13 @@ export async function save(ctx: UserCtx) { datasource.entities[tableToSave.name] = tableToSave await db.put(datasource) + // Since tables are stored inside datasources, we need to notify clients + // that the datasource definition changed + const updatedDatasource = await db.get(datasource._id) + builderSocket?.emitDatasourceUpdate(ctx, updatedDatasource, { + includeOriginator: true, + }) + return tableToSave } diff --git a/packages/server/src/websockets/builder.ts b/packages/server/src/websockets/builder.ts index 0f2c43e5ab..53b0a10905 100644 --- a/packages/server/src/websockets/builder.ts +++ b/packages/server/src/websockets/builder.ts @@ -3,7 +3,13 @@ import { BaseSocket } from "./websocket" import { permissions, events } from "@budibase/backend-core" import http from "http" import Koa from "koa" -import { Datasource, Table, SocketSession, ContextUser } from "@budibase/types" +import { + Datasource, + Table, + SocketSession, + ContextUser, + SocketMessageOptions, +} from "@budibase/types" import { gridSocket } from "./index" import { clearLock, updateLock } from "../utilities/redis" import { Socket } from "socket.io" @@ -66,33 +72,61 @@ export default class BuilderSocket extends BaseSocket { } } - emitTableUpdate(ctx: any, table: Table) { - this.emitToRoom(ctx, ctx.appId, BuilderSocketEvent.TableChange, { - id: table._id, - table, - }) - gridSocket?.emitTableUpdate(ctx, table) + emitTableUpdate(ctx: any, table: Table, options?: SocketMessageOptions) { + this.emitToRoom( + ctx, + ctx.appId, + BuilderSocketEvent.TableChange, + { + id: table._id, + table, + }, + options + ) + gridSocket?.emitTableUpdate(ctx, table, options) } - emitTableDeletion(ctx: any, id: string) { - this.emitToRoom(ctx, ctx.appId, BuilderSocketEvent.TableChange, { - id, - table: null, - }) - gridSocket?.emitTableDeletion(ctx, id) + emitTableDeletion(ctx: any, id: string, options?: SocketMessageOptions) { + this.emitToRoom( + ctx, + ctx.appId, + BuilderSocketEvent.TableChange, + { + id, + table: null, + }, + options + ) + gridSocket?.emitTableDeletion(ctx, id, options) } - emitDatasourceUpdate(ctx: any, datasource: Datasource) { - this.emitToRoom(ctx, ctx.appId, BuilderSocketEvent.DatasourceChange, { - id: datasource._id, - datasource, - }) + emitDatasourceUpdate( + ctx: any, + datasource: Datasource, + options?: SocketMessageOptions + ) { + this.emitToRoom( + ctx, + ctx.appId, + BuilderSocketEvent.DatasourceChange, + { + id: datasource._id, + datasource, + }, + options + ) } - emitDatasourceDeletion(ctx: any, id: string) { - this.emitToRoom(ctx, ctx.appId, BuilderSocketEvent.DatasourceChange, { - id, - datasource: null, - }) + emitDatasourceDeletion(ctx: any, id: string, options?: SocketMessageOptions) { + this.emitToRoom( + ctx, + ctx.appId, + BuilderSocketEvent.DatasourceChange, + { + id, + datasource: null, + }, + options + ) } } diff --git a/packages/server/src/websockets/grid.ts b/packages/server/src/websockets/grid.ts index 6731c2d899..38ec091816 100644 --- a/packages/server/src/websockets/grid.ts +++ b/packages/server/src/websockets/grid.ts @@ -4,7 +4,7 @@ import { permissions } from "@budibase/backend-core" import http from "http" import Koa from "koa" import { getTableId } from "../api/controllers/row/utils" -import { Row, Table } from "@budibase/types" +import { Row, SocketMessageOptions, Table } from "@budibase/types" import { Socket } from "socket.io" import { GridSocketEvent } from "@budibase/shared-core" @@ -29,27 +29,51 @@ export default class GridSocket extends BaseSocket { }) } - emitRowUpdate(ctx: any, row: Row) { + emitRowUpdate(ctx: any, row: Row, options?: SocketMessageOptions) { const tableId = getTableId(ctx) - this.emitToRoom(ctx, tableId, GridSocketEvent.RowChange, { - id: row._id, - row, - }) + this.emitToRoom( + ctx, + tableId, + GridSocketEvent.RowChange, + { + id: row._id, + row, + }, + options + ) } - emitRowDeletion(ctx: any, id: string) { + emitRowDeletion(ctx: any, id: string, options?: SocketMessageOptions) { const tableId = getTableId(ctx) - this.emitToRoom(ctx, tableId, GridSocketEvent.RowChange, { id, row: null }) + this.emitToRoom( + ctx, + tableId, + GridSocketEvent.RowChange, + { id, row: null }, + options + ) } - emitTableUpdate(ctx: any, table: Table) { - this.emitToRoom(ctx, table._id!, GridSocketEvent.TableChange, { - id: table._id, - table, - }) + emitTableUpdate(ctx: any, table: Table, options?: SocketMessageOptions) { + this.emitToRoom( + ctx, + table._id!, + GridSocketEvent.TableChange, + { + id: table._id, + table, + }, + options + ) } - emitTableDeletion(ctx: any, id: string) { - this.emitToRoom(ctx, id, GridSocketEvent.TableChange, { id, table: null }) + emitTableDeletion(ctx: any, id: string, options?: SocketMessageOptions) { + this.emitToRoom( + ctx, + id, + GridSocketEvent.TableChange, + { id, table: null }, + options + ) } } diff --git a/packages/server/src/websockets/websocket.ts b/packages/server/src/websockets/websocket.ts index d8cc10bda4..59a9d698d7 100644 --- a/packages/server/src/websockets/websocket.ts +++ b/packages/server/src/websockets/websocket.ts @@ -9,7 +9,7 @@ import { createAdapter } from "@socket.io/redis-adapter" import { Socket } from "socket.io" import { getSocketPubSubClients } from "../utilities/redis" import { SocketEvent, SocketSessionTTL } from "@budibase/shared-core" -import { SocketSession } from "@budibase/types" +import { SocketSession, SocketMessageOptions } from "@budibase/types" export class BaseSocket { io: Server @@ -276,12 +276,24 @@ export class BaseSocket { this.io.sockets.emit(event, payload) } - // Emit an event to everyone in a room, including metadata of whom - // the originator of the request was - emitToRoom(ctx: any, room: string, event: string, payload: any) { + // Emit an event to everyone in a room + emitToRoom( + ctx: any, + room: string, + event: string, + payload: any, + options?: SocketMessageOptions + ) { + // By default, we include the session API of the originator so that they can ignore + // this event. If we want to include the originator then we leave it unset to that all + // clients will react to it. + let apiSessionId = null + if (!options?.includeOriginator) { + apiSessionId = ctx.headers?.[Header.SESSION_ID] + } this.io.in(room).emit(event, { ...payload, - apiSessionId: ctx.headers?.[Header.SESSION_ID], + apiSessionId, }) } } diff --git a/packages/types/src/sdk/websocket.ts b/packages/types/src/sdk/websocket.ts index 40e2654e82..770f44d9b2 100644 --- a/packages/types/src/sdk/websocket.ts +++ b/packages/types/src/sdk/websocket.ts @@ -7,3 +7,7 @@ export interface SocketSession { room?: string connectedAt: number } + +export interface SocketMessageOptions { + includeOriginator?: boolean +} From 99a8fc7c123986a430509296ea767926c6f4843d Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Thu, 15 Jun 2023 09:27:45 +0100 Subject: [PATCH 2/5] Revert websocket changes and just fetch datasources constantly --- .../backend/DataTable/DataTable.svelte | 13 ++- .../modals/CreateTableModal.svelte | 1 + .../popovers/EditTablePopover.svelte | 1 + .../src/components/grid/stores/columns.js | 6 +- .../src/api/controllers/table/external.ts | 9 ++- packages/server/src/websockets/builder.ts | 80 ++++++------------- packages/server/src/websockets/grid.ts | 54 ++++--------- packages/server/src/websockets/websocket.ts | 19 +---- packages/types/src/sdk/websocket.ts | 4 - 9 files changed, 63 insertions(+), 124 deletions(-) diff --git a/packages/builder/src/components/backend/DataTable/DataTable.svelte b/packages/builder/src/components/backend/DataTable/DataTable.svelte index 1b0c92bde0..4ed41c5b34 100644 --- a/packages/builder/src/components/backend/DataTable/DataTable.svelte +++ b/packages/builder/src/components/backend/DataTable/DataTable.svelte @@ -1,5 +1,5 @@
@@ -37,7 +46,7 @@ allowDeleteRows={!isUsersTable} schemaOverrides={isUsersTable ? userSchemaOverrides : null} showAvatars={false} - on:updatetable={e => tables.replaceTable(id, e.detail)} + on:updatetable={handleGridTableUpdate} > {#if isInternal} diff --git a/packages/builder/src/components/backend/TableNavigator/modals/CreateTableModal.svelte b/packages/builder/src/components/backend/TableNavigator/modals/CreateTableModal.svelte index bd1761620d..bfca91afaa 100644 --- a/packages/builder/src/components/backend/TableNavigator/modals/CreateTableModal.svelte +++ b/packages/builder/src/components/backend/TableNavigator/modals/CreateTableModal.svelte @@ -93,6 +93,7 @@ try { await beforeSave() table = await tables.save(newTable) + await datasources.fetch() await afterSave(table) } catch (e) { notifications.error(e) diff --git a/packages/builder/src/components/backend/TableNavigator/popovers/EditTablePopover.svelte b/packages/builder/src/components/backend/TableNavigator/popovers/EditTablePopover.svelte index 01c62d56f7..11ef60480b 100644 --- a/packages/builder/src/components/backend/TableNavigator/popovers/EditTablePopover.svelte +++ b/packages/builder/src/components/backend/TableNavigator/popovers/EditTablePopover.svelte @@ -65,6 +65,7 @@ const updatedTable = cloneDeep(table) updatedTable.name = updatedName await tables.save(updatedTable) + await datasources.fetch() notifications.success("Table renamed successfully") } diff --git a/packages/frontend-core/src/components/grid/stores/columns.js b/packages/frontend-core/src/components/grid/stores/columns.js index 024fdc29bc..82a26d923c 100644 --- a/packages/frontend-core/src/components/grid/stores/columns.js +++ b/packages/frontend-core/src/components/grid/stores/columns.js @@ -90,12 +90,12 @@ export const deriveStores = context => { // Update local state table.set(newTable) + // Update server + await API.saveTable(newTable) + // Broadcast change to external state can be updated, as this change // will not be received by the builder websocket because we caused it ourselves dispatch("updatetable", newTable) - - // Update server - await API.saveTable(newTable) } return { diff --git a/packages/server/src/api/controllers/table/external.ts b/packages/server/src/api/controllers/table/external.ts index f35691cb62..24d4242478 100644 --- a/packages/server/src/api/controllers/table/external.ts +++ b/packages/server/src/api/controllers/table/external.ts @@ -322,9 +322,7 @@ export async function save(ctx: UserCtx) { // Since tables are stored inside datasources, we need to notify clients // that the datasource definition changed const updatedDatasource = await db.get(datasource._id) - builderSocket?.emitDatasourceUpdate(ctx, updatedDatasource, { - includeOriginator: true, - }) + builderSocket?.emitDatasourceUpdate(ctx, updatedDatasource) return tableToSave } @@ -352,6 +350,11 @@ export async function destroy(ctx: UserCtx) { await db.put(datasource) + // Since tables are stored inside datasources, we need to notify clients + // that the datasource definition changed + const updatedDatasource = await db.get(datasource._id) + builderSocket?.emitDatasourceUpdate(ctx, updatedDatasource) + return tableToDelete } diff --git a/packages/server/src/websockets/builder.ts b/packages/server/src/websockets/builder.ts index 53b0a10905..0f2c43e5ab 100644 --- a/packages/server/src/websockets/builder.ts +++ b/packages/server/src/websockets/builder.ts @@ -3,13 +3,7 @@ import { BaseSocket } from "./websocket" import { permissions, events } from "@budibase/backend-core" import http from "http" import Koa from "koa" -import { - Datasource, - Table, - SocketSession, - ContextUser, - SocketMessageOptions, -} from "@budibase/types" +import { Datasource, Table, SocketSession, ContextUser } from "@budibase/types" import { gridSocket } from "./index" import { clearLock, updateLock } from "../utilities/redis" import { Socket } from "socket.io" @@ -72,61 +66,33 @@ export default class BuilderSocket extends BaseSocket { } } - emitTableUpdate(ctx: any, table: Table, options?: SocketMessageOptions) { - this.emitToRoom( - ctx, - ctx.appId, - BuilderSocketEvent.TableChange, - { - id: table._id, - table, - }, - options - ) - gridSocket?.emitTableUpdate(ctx, table, options) + emitTableUpdate(ctx: any, table: Table) { + this.emitToRoom(ctx, ctx.appId, BuilderSocketEvent.TableChange, { + id: table._id, + table, + }) + gridSocket?.emitTableUpdate(ctx, table) } - emitTableDeletion(ctx: any, id: string, options?: SocketMessageOptions) { - this.emitToRoom( - ctx, - ctx.appId, - BuilderSocketEvent.TableChange, - { - id, - table: null, - }, - options - ) - gridSocket?.emitTableDeletion(ctx, id, options) + emitTableDeletion(ctx: any, id: string) { + this.emitToRoom(ctx, ctx.appId, BuilderSocketEvent.TableChange, { + id, + table: null, + }) + gridSocket?.emitTableDeletion(ctx, id) } - emitDatasourceUpdate( - ctx: any, - datasource: Datasource, - options?: SocketMessageOptions - ) { - this.emitToRoom( - ctx, - ctx.appId, - BuilderSocketEvent.DatasourceChange, - { - id: datasource._id, - datasource, - }, - options - ) + emitDatasourceUpdate(ctx: any, datasource: Datasource) { + this.emitToRoom(ctx, ctx.appId, BuilderSocketEvent.DatasourceChange, { + id: datasource._id, + datasource, + }) } - emitDatasourceDeletion(ctx: any, id: string, options?: SocketMessageOptions) { - this.emitToRoom( - ctx, - ctx.appId, - BuilderSocketEvent.DatasourceChange, - { - id, - datasource: null, - }, - options - ) + emitDatasourceDeletion(ctx: any, id: string) { + this.emitToRoom(ctx, ctx.appId, BuilderSocketEvent.DatasourceChange, { + id, + datasource: null, + }) } } diff --git a/packages/server/src/websockets/grid.ts b/packages/server/src/websockets/grid.ts index 38ec091816..6731c2d899 100644 --- a/packages/server/src/websockets/grid.ts +++ b/packages/server/src/websockets/grid.ts @@ -4,7 +4,7 @@ import { permissions } from "@budibase/backend-core" import http from "http" import Koa from "koa" import { getTableId } from "../api/controllers/row/utils" -import { Row, SocketMessageOptions, Table } from "@budibase/types" +import { Row, Table } from "@budibase/types" import { Socket } from "socket.io" import { GridSocketEvent } from "@budibase/shared-core" @@ -29,51 +29,27 @@ export default class GridSocket extends BaseSocket { }) } - emitRowUpdate(ctx: any, row: Row, options?: SocketMessageOptions) { + emitRowUpdate(ctx: any, row: Row) { const tableId = getTableId(ctx) - this.emitToRoom( - ctx, - tableId, - GridSocketEvent.RowChange, - { - id: row._id, - row, - }, - options - ) + this.emitToRoom(ctx, tableId, GridSocketEvent.RowChange, { + id: row._id, + row, + }) } - emitRowDeletion(ctx: any, id: string, options?: SocketMessageOptions) { + emitRowDeletion(ctx: any, id: string) { const tableId = getTableId(ctx) - this.emitToRoom( - ctx, - tableId, - GridSocketEvent.RowChange, - { id, row: null }, - options - ) + this.emitToRoom(ctx, tableId, GridSocketEvent.RowChange, { id, row: null }) } - emitTableUpdate(ctx: any, table: Table, options?: SocketMessageOptions) { - this.emitToRoom( - ctx, - table._id!, - GridSocketEvent.TableChange, - { - id: table._id, - table, - }, - options - ) + emitTableUpdate(ctx: any, table: Table) { + this.emitToRoom(ctx, table._id!, GridSocketEvent.TableChange, { + id: table._id, + table, + }) } - emitTableDeletion(ctx: any, id: string, options?: SocketMessageOptions) { - this.emitToRoom( - ctx, - id, - GridSocketEvent.TableChange, - { id, table: null }, - options - ) + emitTableDeletion(ctx: any, id: string) { + this.emitToRoom(ctx, id, GridSocketEvent.TableChange, { id, table: null }) } } diff --git a/packages/server/src/websockets/websocket.ts b/packages/server/src/websockets/websocket.ts index 59a9d698d7..89fcc8869d 100644 --- a/packages/server/src/websockets/websocket.ts +++ b/packages/server/src/websockets/websocket.ts @@ -9,7 +9,7 @@ import { createAdapter } from "@socket.io/redis-adapter" import { Socket } from "socket.io" import { getSocketPubSubClients } from "../utilities/redis" import { SocketEvent, SocketSessionTTL } from "@budibase/shared-core" -import { SocketSession, SocketMessageOptions } from "@budibase/types" +import { SocketSession } from "@budibase/types" export class BaseSocket { io: Server @@ -277,23 +277,10 @@ export class BaseSocket { } // Emit an event to everyone in a room - emitToRoom( - ctx: any, - room: string, - event: string, - payload: any, - options?: SocketMessageOptions - ) { - // By default, we include the session API of the originator so that they can ignore - // this event. If we want to include the originator then we leave it unset to that all - // clients will react to it. - let apiSessionId = null - if (!options?.includeOriginator) { - apiSessionId = ctx.headers?.[Header.SESSION_ID] - } + emitToRoom(ctx: any, room: string, event: string, payload: any) { this.io.in(room).emit(event, { ...payload, - apiSessionId, + apiSessionId: ctx.headers?.[Header.SESSION_ID], }) } } diff --git a/packages/types/src/sdk/websocket.ts b/packages/types/src/sdk/websocket.ts index 770f44d9b2..40e2654e82 100644 --- a/packages/types/src/sdk/websocket.ts +++ b/packages/types/src/sdk/websocket.ts @@ -7,7 +7,3 @@ export interface SocketSession { room?: string connectedAt: number } - -export interface SocketMessageOptions { - includeOriginator?: boolean -} From 3cbebaf40da830ab80bcfc77ee5cc32afd5cb6e6 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Thu, 15 Jun 2023 09:29:46 +0100 Subject: [PATCH 3/5] Restore original comment --- packages/server/src/websockets/websocket.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/server/src/websockets/websocket.ts b/packages/server/src/websockets/websocket.ts index 89fcc8869d..d8cc10bda4 100644 --- a/packages/server/src/websockets/websocket.ts +++ b/packages/server/src/websockets/websocket.ts @@ -276,7 +276,8 @@ export class BaseSocket { this.io.sockets.emit(event, payload) } - // Emit an event to everyone in a room + // Emit an event to everyone in a room, including metadata of whom + // the originator of the request was emitToRoom(ctx: any, room: string, event: string, payload: any) { this.io.in(room).emit(event, { ...payload, From 9e535a1ca4a80c5a8dd84064d0d00f377b7d8bb3 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Thu, 15 Jun 2023 09:35:22 +0100 Subject: [PATCH 4/5] Account for table 'type' field meaning different things in different endpoints --- .../src/components/backend/DataTable/DataTable.svelte | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/builder/src/components/backend/DataTable/DataTable.svelte b/packages/builder/src/components/backend/DataTable/DataTable.svelte index 4ed41c5b34..981a619ebd 100644 --- a/packages/builder/src/components/backend/DataTable/DataTable.svelte +++ b/packages/builder/src/components/backend/DataTable/DataTable.svelte @@ -30,8 +30,11 @@ const handleGridTableUpdate = async e => { tables.replaceTable(id, e.detail) - // We need to refresh datasources when an external table changes - if (e.detail?.type === "external") { + // We need to refresh datasources when an external table changes. + // Type "external" may exist - sometimes type is "table" and sometimes it + // is "external" - it has different meanings in different endpoints. + // If we check both these then we hopefully catch all external tables. + if (e.detail?.type === "external" || e.detail?.sql) { await datasources.fetch() } } From f2ce876c5f2c5cbae4d7b5a04dac53e5cb2d79f5 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Thu, 15 Jun 2023 09:39:27 +0100 Subject: [PATCH 5/5] Refresh tables list when some other user adds a datasource --- packages/builder/src/stores/backend/datasources.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/builder/src/stores/backend/datasources.js b/packages/builder/src/stores/backend/datasources.js index 0815f9d766..af914cbee7 100644 --- a/packages/builder/src/stores/backend/datasources.js +++ b/packages/builder/src/stores/backend/datasources.js @@ -113,6 +113,10 @@ export function createDatasourcesStore() { ...state, list: [...state.list, datasource], })) + + // If this is a new datasource then we should refresh the tables list, + // because otherwise we'll never see the new tables + tables.fetch() } // Update existing datasource