From 7f96fbf7411f65ce1dca980395c7498b29a9fe70 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Fri, 12 May 2023 13:55:08 +0100 Subject: [PATCH 01/31] Remove single user restriction and notify users if they are the primary builder or not --- packages/builder/src/components/start/AppRow.svelte | 6 ------ .../pages/builder/portal/overview/[appId]/_layout.svelte | 7 ------- packages/server/src/api/controllers/application.ts | 3 ++- packages/server/src/middleware/builder.ts | 9 ++++----- 4 files changed, 6 insertions(+), 19 deletions(-) diff --git a/packages/builder/src/components/start/AppRow.svelte b/packages/builder/src/components/start/AppRow.svelte index 34d083a096..e2eadd5503 100644 --- a/packages/builder/src/components/start/AppRow.svelte +++ b/packages/builder/src/components/start/AppRow.svelte @@ -15,12 +15,6 @@ } const goToBuilder = () => { - if (app.lockedOther) { - notifications.error( - `App locked by ${app.lockedBy.email}. Please allow lock to expire or have them unlock this app.` - ) - return - } $goto(`../../app/${app.devId}`) } diff --git a/packages/builder/src/pages/builder/portal/overview/[appId]/_layout.svelte b/packages/builder/src/pages/builder/portal/overview/[appId]/_layout.svelte index 806589e421..cc244852d8 100644 --- a/packages/builder/src/pages/builder/portal/overview/[appId]/_layout.svelte +++ b/packages/builder/src/pages/builder/portal/overview/[appId]/_layout.svelte @@ -80,13 +80,6 @@ } const editApp = () => { - if (appLocked && !lockedByYou) { - const identifier = app?.lockedBy?.firstName || app?.lockedBy?.email - notifications.warning( - `App locked by ${identifier}. Please allow lock to expire or have them unlock this app.` - ) - return - } $goto(`../../../app/${app.devId}`) } diff --git a/packages/server/src/api/controllers/application.ts b/packages/server/src/api/controllers/application.ts index ff0dbe015b..815366785c 100644 --- a/packages/server/src/api/controllers/application.ts +++ b/packages/server/src/api/controllers/application.ts @@ -29,7 +29,7 @@ import { USERS_TABLE_SCHEMA } from "../../constants" import { buildDefaultDocs } from "../../db/defaultData/datasource_bb_default" import { removeAppFromUserRoles } from "../../utilities/workerRequests" import { stringToReadStream, isQsTrue } from "../../utilities" -import { getLocksById } from "../../utilities/redis" +import { getLocksById, doesUserHaveLock } from "../../utilities/redis" import { updateClientLibrary, backupClientLibrary, @@ -227,6 +227,7 @@ export async function fetchAppPackage(ctx: UserCtx) { screens, layouts, clientLibPath, + hasLock: await doesUserHaveLock(application.appId, ctx.user), } } diff --git a/packages/server/src/middleware/builder.ts b/packages/server/src/middleware/builder.ts index 5174f618a0..ffb2e2c002 100644 --- a/packages/server/src/middleware/builder.ts +++ b/packages/server/src/middleware/builder.ts @@ -35,12 +35,11 @@ async function checkDevAppLocks(ctx: BBContext) { if (!appId || !appId.startsWith(APP_DEV_PREFIX)) { return } - if (!(await doesUserHaveLock(appId, ctx.user))) { - ctx.throw(400, "User does not hold app lock.") - } - // they do have lock, update it - await updateLock(appId, ctx.user) + // If this user already owns the lock, then update it + if (await doesUserHaveLock(appId, ctx.user)) { + await updateLock(appId, ctx.user) + } } async function updateAppUpdatedAt(ctx: BBContext) { From 5f81584a14390938b8a67d2449454a562fe70fc4 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Fri, 12 May 2023 14:16:10 +0100 Subject: [PATCH 02/31] Update top bar when not the primary builder and prevent flashing during loading states --- .../src/builderStore/store/frontend.js | 6 +- .../src/components/deploy/AppActions.svelte | 190 +++++++++--------- .../builder/app/[application]/_layout.svelte | 141 +++++++------ 3 files changed, 182 insertions(+), 155 deletions(-) diff --git a/packages/builder/src/builderStore/store/frontend.js b/packages/builder/src/builderStore/store/frontend.js index e264dc099b..33c82778e3 100644 --- a/packages/builder/src/builderStore/store/frontend.js +++ b/packages/builder/src/builderStore/store/frontend.js @@ -39,6 +39,7 @@ import { makePropSafe as safe } from "@budibase/string-templates" import { getComponentFieldOptions } from "helpers/formFields" const INITIAL_FRONTEND_STATE = { + initialised: false, apps: [], name: "", url: "", @@ -70,6 +71,7 @@ const INITIAL_FRONTEND_STATE = { previewDevice: "desktop", highlightedSettingKey: null, builderSidePanel: false, + hasLock: true, // URL params selectedScreenId: null, @@ -112,7 +114,7 @@ export const getFrontendStore = () => { store.set({ ...INITIAL_FRONTEND_STATE }) }, initialise: async pkg => { - const { layouts, screens, application, clientLibPath } = pkg + const { layouts, screens, application, clientLibPath, hasLock } = pkg await store.actions.components.refreshDefinitions(application.appId) @@ -137,6 +139,8 @@ export const getFrontendStore = () => { upgradableVersion: application.upgradableVersion, navigation: application.navigation || {}, usedPlugins: application.usedPlugins || [], + hasLock, + initialised: true, })) screenHistoryStore.reset() automationHistoryStore.reset() diff --git a/packages/builder/src/components/deploy/AppActions.svelte b/packages/builder/src/components/deploy/AppActions.svelte index daed564204..9813237317 100644 --- a/packages/builder/src/components/deploy/AppActions.svelte +++ b/packages/builder/src/components/deploy/AppActions.svelte @@ -113,109 +113,113 @@ }) -
-
-
- -
- - - {#if isPublished} -
-
- -
- -
- - Your published app - - - {processStringSync( - "Last published {{ duration time 'millisecond' }} ago", - { - time: - new Date().getTime() - - new Date(latestDeployments[0].updatedAt).getTime(), - } - )} - - -
- - -
- -
- +{#if $store.hasLock} +
+
+
+
- {/if} + - {#if !isPublished} - - {/if} + {#if isPublished} +
+
+ +
+ +
+ + Your published app + + + {processStringSync( + "Last published {{ duration time 'millisecond' }} ago", + { + time: + new Date().getTime() - + new Date(latestDeployments[0].updatedAt).getTime(), + } + )} + + +
+ + +
+ +
+ +
+ {/if} - - + {#if !isPublished} { - store.update(state => { - state.builderSidePanel = true - return state - }) - }} - > - Users - - - -
-
+ tooltip="Your app has not been published yet" + disabled + /> + {/if} - - Are you sure you want to unpublish the app {selectedApp?.name}? - + + + { + store.update(state => { + state.builderSidePanel = true + return state + }) + }} + > + Users + + + +
+
+ + + Are you sure you want to unpublish the app {selectedApp?.name}? + +{/if}
- + {#if $store.hasLock} + + {/if}
From d752448403a01b3777a839425af423f1105586de Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Fri, 12 May 2023 14:33:00 +0100 Subject: [PATCH 03/31] Disable command palette and tour for non lock holders and forcefully prevent access to the design and automate pages --- .../pages/builder/app/[application]/_layout.svelte | 12 +++++++++--- .../app/[application]/automate/_layout.svelte | 9 +++++++++ .../builder/app/[application]/design/_layout.svelte | 12 ++++++++++++ 3 files changed, 30 insertions(+), 3 deletions(-) diff --git a/packages/builder/src/pages/builder/app/[application]/_layout.svelte b/packages/builder/src/pages/builder/app/[application]/_layout.svelte index 434a6c7770..2fb1b29c18 100644 --- a/packages/builder/src/pages/builder/app/[application]/_layout.svelte +++ b/packages/builder/src/pages/builder/app/[application]/_layout.svelte @@ -13,7 +13,6 @@ Modal, notifications, } from "@budibase/bbui" - import AppActions from "components/deploy/AppActions.svelte" import { API } from "api" import { isActive, goto, layout, redirect } from "@roxi/routify" @@ -30,7 +29,9 @@ let promise = getPackage() let hasSynced = false let commandPaletteModal + let loaded = false + $: loaded && initTour() $: selected = capitalise( $layout.children.find(layout => $isActive(layout.path))?.title ?? "data" ) @@ -43,6 +44,7 @@ await automationStore.actions.fetch() await roles.fetch() await flags.fetch() + loaded = true return pkg } catch (error) { notifications.error(`Error initialising app: ${error?.message}`) @@ -67,13 +69,18 @@ // Event handler for the command palette const handleKeyDown = e => { - if (e.key === "k" && (e.ctrlKey || e.metaKey)) { + if (e.key === "k" && (e.ctrlKey || e.metaKey) && $store.hasLock) { e.preventDefault() commandPaletteModal.toggle() } } const initTour = async () => { + // Skip tour if we don't have the lock + if (!$store.hasLock) { + return + } + // Check if onboarding is enabled. if (isEnabled(TENANT_FEATURE_FLAGS.ONBOARDING_TOUR)) { if (!$auth.user?.onboardedAt) { @@ -110,7 +117,6 @@ // check if user has beta access // const betaResponse = await API.checkBetaAccess($auth?.user?.email) // betaAccess = betaResponse.access - initTour() } catch (error) { notifications.error("Failed to sync with production database") } diff --git a/packages/builder/src/pages/builder/app/[application]/automate/_layout.svelte b/packages/builder/src/pages/builder/app/[application]/automate/_layout.svelte index 74dfe671ab..79ca5df168 100644 --- a/packages/builder/src/pages/builder/app/[application]/automate/_layout.svelte +++ b/packages/builder/src/pages/builder/app/[application]/automate/_layout.svelte @@ -8,6 +8,15 @@ import { onDestroy, onMount } from "svelte" import { syncURLToState } from "helpers/urlStateSync" import * as routify from "@roxi/routify" + import { store } from "builderStore" + import { redirect } from "@roxi/routify" + + // Prevent access for other users than the lock holder + $: { + if (!$store.hasLock) { + $redirect("../data") + } + } // Keep URL and state in sync for selected screen ID const stopSyncing = syncURLToState({ diff --git a/packages/builder/src/pages/builder/app/[application]/design/_layout.svelte b/packages/builder/src/pages/builder/app/[application]/design/_layout.svelte index ec21d909aa..d23514ae6d 100644 --- a/packages/builder/src/pages/builder/app/[application]/design/_layout.svelte +++ b/packages/builder/src/pages/builder/app/[application]/design/_layout.svelte @@ -1,2 +1,14 @@ + + From f8f970bf7ed82f32291890909ed97f7723205948 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Fri, 12 May 2023 16:13:32 +0100 Subject: [PATCH 04/31] Update grid websocket to send actual changes down to reduce API load --- .../src/components/grid/layout/Grid.svelte | 5 ++- .../src/components/grid/lib/websocket.js | 2 +- .../src/components/grid/stores/rows.js | 24 ++++++++---- .../server/src/api/controllers/row/index.ts | 37 ++++++++----------- .../server/src/api/controllers/row/utils.ts | 12 ++++++ packages/server/src/websockets/grid.ts | 12 ++++++ 6 files changed, 59 insertions(+), 33 deletions(-) diff --git a/packages/frontend-core/src/components/grid/layout/Grid.svelte b/packages/frontend-core/src/components/grid/layout/Grid.svelte index 2035ec4d39..3b0e74d882 100644 --- a/packages/frontend-core/src/components/grid/layout/Grid.svelte +++ b/packages/frontend-core/src/components/grid/layout/Grid.svelte @@ -1,5 +1,5 @@
{ }) socket.on("row-update", data => { if (data.id) { - rows.actions.refreshRow(data.id) + rows.actions.replaceRow(data.id, data.row) } }) socket.on("user-update", user => { diff --git a/packages/frontend-core/src/components/grid/stores/rows.js b/packages/frontend-core/src/components/grid/stores/rows.js index b6dc8c05d0..ee5ac6c1d2 100644 --- a/packages/frontend-core/src/components/grid/stores/rows.js +++ b/packages/frontend-core/src/components/grid/stores/rows.js @@ -268,27 +268,25 @@ export const deriveStores = context => { return res?.rows?.[0] } - // Refreshes a specific row, handling updates, addition or deletion - const refreshRow = async id => { - // Fetch row from the server again - const newRow = await fetchRow(id) - + // Replaces a row in state with the newly defined row, handling updates, + // addition and deletion + const replaceRow = (id, row) => { // Get index of row to check if it exists const $rows = get(rows) const $rowLookupMap = get(rowLookupMap) const index = $rowLookupMap[id] // Process as either an update, addition or deletion - if (newRow) { + if (row) { if (index != null) { // An existing row was updated rows.update(state => { - state[index] = { ...newRow } + state[index] = { ...row } return state }) } else { // A new row was created - handleNewRows([newRow]) + handleNewRows([row]) } } else if (index != null) { // A row was removed @@ -296,6 +294,15 @@ export const deriveStores = context => { } } + // Refreshes a specific row + const refreshRow = async id => { + // Fetch row from the server again + const row = await fetchRow(id) + + // Update local state + replaceRow(id, row) + } + // Refreshes all data const refreshData = () => { get(fetch)?.getInitialData() @@ -455,6 +462,7 @@ export const deriveStores = context => { hasRow, loadNextPage, refreshRow, + replaceRow, refreshData, refreshTableDefinition, }, diff --git a/packages/server/src/api/controllers/row/index.ts b/packages/server/src/api/controllers/row/index.ts index 349c4e72e7..55d2d27cce 100644 --- a/packages/server/src/api/controllers/row/index.ts +++ b/packages/server/src/api/controllers/row/index.ts @@ -4,6 +4,7 @@ import * as external from "./external" import { isExternalTable } from "../../../integrations/utils" import { Ctx } from "@budibase/types" import * as utils from "./utils" +import { gridSocket } from "../../../websockets" function pickApi(tableId: any) { if (isExternalTable(tableId)) { @@ -12,21 +13,9 @@ function pickApi(tableId: any) { return internal } -function getTableId(ctx: any) { - if (ctx.request.body && ctx.request.body.tableId) { - return ctx.request.body.tableId - } - if (ctx.params && ctx.params.tableId) { - return ctx.params.tableId - } - if (ctx.params && ctx.params.viewName) { - return ctx.params.viewName - } -} - export async function patch(ctx: any): Promise { const appId = ctx.appId - const tableId = getTableId(ctx) + const tableId = utils.getTableId(ctx) const body = ctx.request.body // if it doesn't have an _id then its save if (body && !body._id) { @@ -47,6 +36,7 @@ export async function patch(ctx: any): Promise { ctx.eventEmitter.emitRow(`row:update`, appId, row, table) ctx.message = `${table.name} updated successfully.` ctx.body = row + gridSocket?.emitRowUpdate(ctx, row) } catch (err) { ctx.throw(400, err) } @@ -54,7 +44,7 @@ export async function patch(ctx: any): Promise { export const save = async (ctx: any) => { const appId = ctx.appId - const tableId = getTableId(ctx) + const tableId = utils.getTableId(ctx) const body = ctx.request.body // if it has an ID already then its a patch if (body && body._id) { @@ -69,23 +59,24 @@ export const save = async (ctx: any) => { ctx.eventEmitter && ctx.eventEmitter.emitRow(`row:save`, appId, row, table) ctx.message = `${table.name} saved successfully` ctx.body = row + gridSocket?.emitRowUpdate(ctx, row) } export async function fetchView(ctx: any) { - const tableId = getTableId(ctx) + const tableId = utils.getTableId(ctx) ctx.body = await quotas.addQuery(() => pickApi(tableId).fetchView(ctx), { datasourceId: tableId, }) } export async function fetch(ctx: any) { - const tableId = getTableId(ctx) + const tableId = utils.getTableId(ctx) ctx.body = await quotas.addQuery(() => pickApi(tableId).fetch(ctx), { datasourceId: tableId, }) } export async function find(ctx: any) { - const tableId = getTableId(ctx) + const tableId = utils.getTableId(ctx) ctx.body = await quotas.addQuery(() => pickApi(tableId).find(ctx), { datasourceId: tableId, }) @@ -94,7 +85,7 @@ export async function find(ctx: any) { export async function destroy(ctx: any) { const appId = ctx.appId const inputs = ctx.request.body - const tableId = getTableId(ctx) + const tableId = utils.getTableId(ctx) let response, row if (inputs.rows) { let { rows } = await quotas.addQuery( @@ -107,6 +98,7 @@ export async function destroy(ctx: any) { response = rows for (let row of rows) { ctx.eventEmitter && ctx.eventEmitter.emitRow(`row:delete`, appId, row) + gridSocket?.emitRowDeletion(ctx, row._id) } } else { let resp = await quotas.addQuery(() => pickApi(tableId).destroy(ctx), { @@ -116,6 +108,7 @@ export async function destroy(ctx: any) { response = resp.response row = resp.row ctx.eventEmitter && ctx.eventEmitter.emitRow(`row:delete`, appId, row) + gridSocket?.emitRowDeletion(ctx, row._id) } ctx.status = 200 // for automations include the row that was deleted @@ -124,7 +117,7 @@ export async function destroy(ctx: any) { } export async function search(ctx: any) { - const tableId = getTableId(ctx) + const tableId = utils.getTableId(ctx) ctx.status = 200 ctx.body = await quotas.addQuery(() => pickApi(tableId).search(ctx), { datasourceId: tableId, @@ -132,7 +125,7 @@ export async function search(ctx: any) { } export async function validate(ctx: Ctx) { - const tableId = getTableId(ctx) + const tableId = utils.getTableId(ctx) // external tables are hard to validate currently if (isExternalTable(tableId)) { ctx.body = { valid: true } @@ -145,7 +138,7 @@ export async function validate(ctx: Ctx) { } export async function fetchEnrichedRow(ctx: any) { - const tableId = getTableId(ctx) + const tableId = utils.getTableId(ctx) ctx.body = await quotas.addQuery( () => pickApi(tableId).fetchEnrichedRow(ctx), { @@ -155,7 +148,7 @@ export async function fetchEnrichedRow(ctx: any) { } export const exportRows = async (ctx: any) => { - const tableId = getTableId(ctx) + const tableId = utils.getTableId(ctx) ctx.body = await quotas.addQuery(() => pickApi(tableId).exportRows(ctx), { datasourceId: tableId, }) diff --git a/packages/server/src/api/controllers/row/utils.ts b/packages/server/src/api/controllers/row/utils.ts index f6a87dd24c..f1edbf538b 100644 --- a/packages/server/src/api/controllers/row/utils.ts +++ b/packages/server/src/api/controllers/row/utils.ts @@ -154,3 +154,15 @@ export function cleanExportRows( return cleanRows } + +export function getTableId(ctx: any) { + if (ctx.request.body && ctx.request.body.tableId) { + return ctx.request.body.tableId + } + if (ctx.params && ctx.params.tableId) { + return ctx.params.tableId + } + if (ctx.params && ctx.params.viewName) { + return ctx.params.viewName + } +} diff --git a/packages/server/src/websockets/grid.ts b/packages/server/src/websockets/grid.ts index 886784cd2c..bb23ec8e18 100644 --- a/packages/server/src/websockets/grid.ts +++ b/packages/server/src/websockets/grid.ts @@ -3,6 +3,8 @@ import Socket from "./websocket" import { permissions } from "@budibase/backend-core" import http from "http" import Koa from "koa" +import { getTableId } from "../api/controllers/row/utils" +import { Row } from "@budibase/types" export default class GridSocket extends Socket { constructor(app: Koa, server: http.Server) { @@ -52,4 +54,14 @@ export default class GridSocket extends Socket { }) }) } + + emitRowUpdate(ctx: any, row: Row) { + const tableId = getTableId(ctx) + this.io.in(tableId).emit("row-update", { id: row._id, row }) + } + + emitRowDeletion(ctx: any, id: string) { + const tableId = getTableId(ctx) + this.io.in(tableId).emit("row-update", { id, row: null }) + } } From 46d8ad286489a38321cc8cc2fe205321204e00fe Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Mon, 15 May 2023 14:51:54 +0100 Subject: [PATCH 05/31] Restore proper functionality for showing other users inside grids --- .../src/components/grid/cells/GridCell.svelte | 12 +++++++++--- .../src/components/grid/layout/NewRow.svelte | 2 +- .../src/components/grid/stores/users.js | 4 ++-- packages/server/src/websockets/grid.ts | 2 +- 4 files changed, 13 insertions(+), 7 deletions(-) diff --git a/packages/frontend-core/src/components/grid/cells/GridCell.svelte b/packages/frontend-core/src/components/grid/cells/GridCell.svelte index 6589c18d07..d5e96c3ec4 100644 --- a/packages/frontend-core/src/components/grid/cells/GridCell.svelte +++ b/packages/frontend-core/src/components/grid/cells/GridCell.svelte @@ -112,10 +112,10 @@ z-index: 2; } .cell.focused { - --cell-color: var(--spectrum-global-color-blue-400); + --cell-color: var(--spectrum-global-color-blue-400) !important; } .cell.error { - --cell-color: var(--spectrum-global-color-red-500); + --cell-color: var(--spectrum-global-color-red-500) !important; } .cell.readonly { --cell-color: var(--spectrum-global-color-gray-600); @@ -141,7 +141,7 @@ left: 0; padding: 1px 4px 3px 4px; margin: 0 0 -2px 0; - background: var(--user-color); + background: var(--cell-color); border-radius: 2px; display: block; color: white; @@ -162,4 +162,10 @@ .error .label { background: var(--spectrum-global-color-red-500); } + .selected-other:not(.error) .label { + display: none; + } + .selected-other:not(.error):hover .label { + display: block; + } diff --git a/packages/frontend-core/src/components/grid/layout/NewRow.svelte b/packages/frontend-core/src/components/grid/layout/NewRow.svelte index 8048a4e2fa..97a31e743a 100644 --- a/packages/frontend-core/src/components/grid/layout/NewRow.svelte +++ b/packages/frontend-core/src/components/grid/layout/NewRow.svelte @@ -193,7 +193,7 @@ row={newRow} focused={$focusedCellId === cellId} width={column.width} - rowIdx={0} + rowIdx={offset === 0 ? 0 : 1} invertX={columnIdx >= $columnHorizontalInversionIndex} {invertY} > diff --git a/packages/frontend-core/src/components/grid/stores/users.js b/packages/frontend-core/src/components/grid/stores/users.js index 3a6ec5fb21..4cf4bdac52 100644 --- a/packages/frontend-core/src/components/grid/stores/users.js +++ b/packages/frontend-core/src/components/grid/stores/users.js @@ -85,11 +85,11 @@ export const deriveStores = context => { const updateUser = user => { const $users = get(users) - const index = $users.findIndex(x => x.id === user.id) - if (index === -1) { + if (!$users.some(x => x.id === user.id)) { users.set([...$users, user]) } else { users.update(state => { + const index = state.findIndex(x => x.id === user.id) state[index] = user return state.slice() }) diff --git a/packages/server/src/websockets/grid.ts b/packages/server/src/websockets/grid.ts index bb23ec8e18..4fb81e006c 100644 --- a/packages/server/src/websockets/grid.ts +++ b/packages/server/src/websockets/grid.ts @@ -40,7 +40,7 @@ export default class GridSocket extends Socket { // Handle users selecting a new cell socket.on("select-cell", cellId => { - socket.data.user.selectedCellId = cellId + socket.data.user.focusedCellId = cellId if (currentRoom) { socket.to(currentRoom).emit("user-update", socket.data.user) } From e2a860ea4c1b798464bb02c6b839e115e06e50ac Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Mon, 15 May 2023 16:43:37 +0100 Subject: [PATCH 06/31] Fix visual issues with other users selecting cells --- .../src/components/grid/cells/GridCell.svelte | 9 +++++---- .../frontend-core/src/components/grid/stores/rows.js | 3 --- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/packages/frontend-core/src/components/grid/cells/GridCell.svelte b/packages/frontend-core/src/components/grid/cells/GridCell.svelte index d5e96c3ec4..3ac9a3a399 100644 --- a/packages/frontend-core/src/components/grid/cells/GridCell.svelte +++ b/packages/frontend-core/src/components/grid/cells/GridCell.svelte @@ -94,14 +94,15 @@ } /* Cell border for cells with labels */ - .cell.error:after, - .cell.selected-other:not(.focused):after { + .cell.error:after { border-radius: 0 2px 2px 2px; } - .cell[data-row="0"].error:after, - .cell[data-row="0"].selected-other:not(.focused):after { + .cell[data-row="0"].error:after { border-radius: 2px 2px 2px 0; } + .cell.selected-other:not(.focused):after { + border-radius: 2px; + } /* Cell z-index */ .cell.error, diff --git a/packages/frontend-core/src/components/grid/stores/rows.js b/packages/frontend-core/src/components/grid/stores/rows.js index ee5ac6c1d2..f6c709b675 100644 --- a/packages/frontend-core/src/components/grid/stores/rows.js +++ b/packages/frontend-core/src/components/grid/stores/rows.js @@ -296,10 +296,7 @@ export const deriveStores = context => { // Refreshes a specific row const refreshRow = async id => { - // Fetch row from the server again const row = await fetchRow(id) - - // Update local state replaceRow(id, row) } From 913188052ab6ae447bacf060caa30937c2be46c6 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Tue, 16 May 2023 08:18:26 +0100 Subject: [PATCH 07/31] Fix cell colours with multiple users --- .../src/components/grid/cells/GridCell.svelte | 27 ++++++++++++------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/packages/frontend-core/src/components/grid/cells/GridCell.svelte b/packages/frontend-core/src/components/grid/cells/GridCell.svelte index 3ac9a3a399..3d4f2a37bc 100644 --- a/packages/frontend-core/src/components/grid/cells/GridCell.svelte +++ b/packages/frontend-core/src/components/grid/cells/GridCell.svelte @@ -15,7 +15,7 @@ const getStyle = (width, selectedUser) => { let style = `flex: 0 0 ${width}px;` if (selectedUser) { - style += `--cell-color:${selectedUser.color};` + style += `--user-color:${selectedUser.color};` } return style } @@ -112,14 +112,8 @@ .cell.focused { z-index: 2; } - .cell.focused { - --cell-color: var(--spectrum-global-color-blue-400) !important; - } - .cell.error { - --cell-color: var(--spectrum-global-color-red-500) !important; - } - .cell.readonly { - --cell-color: var(--spectrum-global-color-gray-600); + .cell.selected-other:hover { + z-index: 2; } .cell:not(.focused) { user-select: none; @@ -127,6 +121,21 @@ .cell:hover { cursor: default; } + + /* Cell color overrides */ + .cell.selected-other { + --cell-color: var(--user-color); + } + .cell.focused { + --cell-color: var(--spectrum-global-color-blue-400); + } + .cell.error { + --cell-color: var(--spectrum-global-color-red-500); + } + .cell.focused.readonly { + --cell-color: var(--spectrum-global-color-gray-600); + } + .cell.highlighted:not(.focused), .cell.focused.readonly { --cell-background: var(--cell-background-hover); From cb9d35f6aafb0b1c2688e596c0fe5dfc2999ab6b Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Tue, 16 May 2023 08:24:23 +0100 Subject: [PATCH 08/31] Fix validation error position for new rows --- .../frontend-core/src/components/grid/cells/GridCell.svelte | 1 - packages/frontend-core/src/components/grid/layout/NewRow.svelte | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/frontend-core/src/components/grid/cells/GridCell.svelte b/packages/frontend-core/src/components/grid/cells/GridCell.svelte index 3d4f2a37bc..63446a858c 100644 --- a/packages/frontend-core/src/components/grid/cells/GridCell.svelte +++ b/packages/frontend-core/src/components/grid/cells/GridCell.svelte @@ -165,7 +165,6 @@ .cell[data-row="0"] .label { bottom: auto; top: 100%; - border-radius: 0 2px 2px 2px; padding: 2px 4px 2px 4px; margin: -2px 0 0 0; } diff --git a/packages/frontend-core/src/components/grid/layout/NewRow.svelte b/packages/frontend-core/src/components/grid/layout/NewRow.svelte index 97a31e743a..12bab9c02c 100644 --- a/packages/frontend-core/src/components/grid/layout/NewRow.svelte +++ b/packages/frontend-core/src/components/grid/layout/NewRow.svelte @@ -167,7 +167,7 @@ focused={$focusedCellId === cellId} width={$stickyColumn.width} {updateValue} - rowIdx={0} + rowIdx={offset === 0 ? 0 : 1} {invertY} > {#if $stickyColumn?.schema?.autocolumn} From 6ed691be1d7b98de374293c73dffa82251c5f67b Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Tue, 16 May 2023 14:18:31 +0100 Subject: [PATCH 09/31] Improve builder socket --- packages/builder/src/builderStore/index.js | 2 + .../src/builderStore/store/frontend.js | 7 +++ .../builder/src/builderStore/store/users.js | 10 +++++ .../builder/src/builderStore/websocket.js | 26 +++++++++++ .../_components/UserAvatars.svelte | 33 ++++++++++++++ .../builder/app/[application]/_layout.svelte | 7 ++- .../src/components/grid/layout/Grid.svelte | 4 +- .../components/grid/layout/UserAvatars.svelte | 2 +- .../src/components/grid/lib/websocket.js | 21 ++------- .../src/components/grid/stores/users.js | 29 ------------- packages/frontend-core/src/utils/index.js | 1 + packages/frontend-core/src/utils/websocket.js | 23 ++++++++++ packages/server/src/websockets/builder.ts | 43 +++++++++++++++++++ packages/server/src/websockets/index.ts | 5 ++- packages/server/src/websockets/websocket.ts | 31 +++++++++++-- 15 files changed, 185 insertions(+), 59 deletions(-) create mode 100644 packages/builder/src/builderStore/store/users.js create mode 100644 packages/builder/src/builderStore/websocket.js create mode 100644 packages/builder/src/pages/builder/app/[application]/_components/UserAvatars.svelte create mode 100644 packages/frontend-core/src/utils/websocket.js create mode 100644 packages/server/src/websockets/builder.ts diff --git a/packages/builder/src/builderStore/index.js b/packages/builder/src/builderStore/index.js index d15cdb6e98..8183c75a20 100644 --- a/packages/builder/src/builderStore/index.js +++ b/packages/builder/src/builderStore/index.js @@ -2,6 +2,7 @@ import { getFrontendStore } from "./store/frontend" import { getAutomationStore } from "./store/automation" import { getTemporalStore } from "./store/temporal" import { getThemeStore } from "./store/theme" +import { getUsersStore } from "./store/users" import { derived } from "svelte/store" import { findComponent, findComponentPath } from "./componentUtils" import { RoleUtils } from "@budibase/frontend-core" @@ -12,6 +13,7 @@ export const store = getFrontendStore() export const automationStore = getAutomationStore() export const themeStore = getThemeStore() export const temporalStore = getTemporalStore() +export const usersStore = getUsersStore() // Setup history for screens export const screenHistoryStore = createHistoryStore({ diff --git a/packages/builder/src/builderStore/store/frontend.js b/packages/builder/src/builderStore/store/frontend.js index 33c82778e3..0a27f576a3 100644 --- a/packages/builder/src/builderStore/store/frontend.js +++ b/packages/builder/src/builderStore/store/frontend.js @@ -37,6 +37,7 @@ import { } from "builderStore/dataBinding" import { makePropSafe as safe } from "@budibase/string-templates" import { getComponentFieldOptions } from "helpers/formFields" +import { createBuilderWebsocket } from "builderStore/websocket" const INITIAL_FRONTEND_STATE = { initialised: false, @@ -84,10 +85,14 @@ const INITIAL_FRONTEND_STATE = { // Onboarding onboarding: false, tourNodes: null, + + // Multi user collab + users: [], } export const getFrontendStore = () => { const store = writable({ ...INITIAL_FRONTEND_STATE }) + let websocket // This is a fake implementation of a "patch" API endpoint to try and prevent // 409s. All screen doc mutations (aside from creation) use this function, @@ -112,9 +117,11 @@ export const getFrontendStore = () => { store.actions = { reset: () => { store.set({ ...INITIAL_FRONTEND_STATE }) + websocket?.disconnect() }, initialise: async pkg => { const { layouts, screens, application, clientLibPath, hasLock } = pkg + websocket = createBuilderWebsocket() await store.actions.components.refreshDefinitions(application.appId) diff --git a/packages/builder/src/builderStore/store/users.js b/packages/builder/src/builderStore/store/users.js new file mode 100644 index 0000000000..dfb592c1c5 --- /dev/null +++ b/packages/builder/src/builderStore/store/users.js @@ -0,0 +1,10 @@ +import { writable } from "svelte/store" + +export const getUsersStore = () => { + const initialValue = { + users: [], + } + const store = writable(initialValue) + + return store +} diff --git a/packages/builder/src/builderStore/websocket.js b/packages/builder/src/builderStore/websocket.js new file mode 100644 index 0000000000..d73ab08f9a --- /dev/null +++ b/packages/builder/src/builderStore/websocket.js @@ -0,0 +1,26 @@ +import { createWebsocket } from "@budibase/frontend-core" +import { store } from "builderStore" + +export const createBuilderWebsocket = () => { + const socket = createWebsocket("/socket/builder") + + socket.on("connect", () => { + socket.emit("get-users", null, response => { + console.log("conntected!", response.users) + store.update(state => ({ + ...state, + users: response.users, + })) + }) + }) + + socket.on("user-update", user => {}) + + socket.on("user-disconnect", user => {}) + + socket.on("connect_error", err => { + console.log("Failed to connect to builder websocket:", err.message) + }) + + return socket +} diff --git a/packages/builder/src/pages/builder/app/[application]/_components/UserAvatars.svelte b/packages/builder/src/pages/builder/app/[application]/_components/UserAvatars.svelte new file mode 100644 index 0000000000..a99c4bebee --- /dev/null +++ b/packages/builder/src/pages/builder/app/[application]/_components/UserAvatars.svelte @@ -0,0 +1,33 @@ + + +
+ {#each users as user} +
+ {user.email[0]} +
+ {/each} +
+ + diff --git a/packages/builder/src/pages/builder/app/[application]/_layout.svelte b/packages/builder/src/pages/builder/app/[application]/_layout.svelte index 2fb1b29c18..e145466b11 100644 --- a/packages/builder/src/pages/builder/app/[application]/_layout.svelte +++ b/packages/builder/src/pages/builder/app/[application]/_layout.svelte @@ -22,6 +22,7 @@ import TourWrap from "components/portal/onboarding/TourWrap.svelte" import TourPopover from "components/portal/onboarding/TourPopover.svelte" import BuilderSidePanel from "./_components/BuilderSidePanel.svelte" + import UserAvatars from "./_components/UserAvatars.svelte" import { TOUR_KEYS, TOURS } from "components/portal/onboarding/tours.js" export let application @@ -125,10 +126,7 @@ }) onDestroy(() => { - store.update(state => { - state.appId = null - return state - }) + store.actions.reset() }) @@ -211,6 +209,7 @@ {/if}
+
{/if} diff --git a/packages/frontend-core/src/components/grid/layout/Grid.svelte b/packages/frontend-core/src/components/grid/layout/Grid.svelte index 3b0e74d882..4d1ca1434f 100644 --- a/packages/frontend-core/src/components/grid/layout/Grid.svelte +++ b/packages/frontend-core/src/components/grid/layout/Grid.svelte @@ -24,7 +24,7 @@ import RowHeightButton from "../controls/RowHeightButton.svelte" import ColumnWidthButton from "../controls/ColumnWidthButton.svelte" import NewRow from "./NewRow.svelte" - import { createWebsocket } from "../lib/websocket" + import { createGridWebsocket } from "../lib/websocket" import { MaxCellRenderHeight, MaxCellRenderWidthOverflow, @@ -98,7 +98,7 @@ export const getContext = () => context // Initialise websocket for multi-user - onMount(() => createWebsocket(context)) + onMount(() => createGridWebsocket(context))
diff --git a/packages/frontend-core/src/components/grid/lib/websocket.js b/packages/frontend-core/src/components/grid/lib/websocket.js index ae67cd714d..7ac34b5a95 100644 --- a/packages/frontend-core/src/components/grid/lib/websocket.js +++ b/packages/frontend-core/src/components/grid/lib/websocket.js @@ -1,24 +1,9 @@ import { get } from "svelte/store" -import { io } from "socket.io-client" +import { createWebsocket } from "../../../utils" -export const createWebsocket = context => { +export const createGridWebsocket = context => { const { rows, tableId, users, userId, focusedCellId } = context - - // Determine connection info - const tls = location.protocol === "https:" - const proto = tls ? "wss:" : "ws:" - const host = location.hostname - const port = location.port || (tls ? 443 : 80) - const socket = io(`${proto}//${host}:${port}`, { - path: "/socket/grid", - // Cap reconnection attempts to 3 (total of 15 seconds before giving up) - reconnectionAttempts: 3, - // Delay reconnection attempt by 5 seconds - reconnectionDelay: 5000, - reconnectionDelayMax: 5000, - // Timeout after 4 seconds so we never stack requests - timeout: 4000, - }) + const socket = createWebsocket("/socket/grid") const connectToTable = tableId => { if (!socket.connected) { diff --git a/packages/frontend-core/src/components/grid/stores/users.js b/packages/frontend-core/src/components/grid/stores/users.js index 4cf4bdac52..5190de70da 100644 --- a/packages/frontend-core/src/components/grid/stores/users.js +++ b/packages/frontend-core/src/components/grid/stores/users.js @@ -21,35 +21,6 @@ export const createStores = () => { return 0 } }) - // Enrich users with colors - .map((user, idx) => { - // Generate random colour hue - let hue = 1 - for (let i = 0; i < user.email.length && i < 5; i++) { - hue *= user.email.charCodeAt(i + 1) - hue /= 17 - } - hue = hue % 360 - const color = - idx === 0 - ? "var(--spectrum-global-color-blue-400)" - : `hsl(${hue}, 50%, 40%)` - - // Generate friendly label - let label = user.email - if (user.firstName) { - label = user.firstName - if (user.lastName) { - label += ` ${user.lastName}` - } - } - - return { - ...user, - color, - label, - } - }) ) }, [] diff --git a/packages/frontend-core/src/utils/index.js b/packages/frontend-core/src/utils/index.js index 6ac7c65c62..16dc4f323a 100644 --- a/packages/frontend-core/src/utils/index.js +++ b/packages/frontend-core/src/utils/index.js @@ -3,3 +3,4 @@ export * as JSONUtils from "./json" export * as CookieUtils from "./cookies" export * as RoleUtils from "./roles" export * as Utils from "./utils" +export { createWebsocket } from "./websocket" diff --git a/packages/frontend-core/src/utils/websocket.js b/packages/frontend-core/src/utils/websocket.js new file mode 100644 index 0000000000..85c09acc14 --- /dev/null +++ b/packages/frontend-core/src/utils/websocket.js @@ -0,0 +1,23 @@ +import { io } from "socket.io-client" + +export const createWebsocket = path => { + if (!path) { + throw "A websocket path must be provided" + } + + // Determine connection info + const tls = location.protocol === "https:" + const proto = tls ? "wss:" : "ws:" + const host = location.hostname + const port = location.port || (tls ? 443 : 80) + return io(`${proto}//${host}:${port}`, { + path, + // Cap reconnection attempts to 3 (total of 15 seconds before giving up) + reconnectionAttempts: 3, + // Delay reconnection attempt by 5 seconds + reconnectionDelay: 5000, + reconnectionDelayMax: 5000, + // Timeout after 4 seconds so we never stack requests + timeout: 4000, + }) +} diff --git a/packages/server/src/websockets/builder.ts b/packages/server/src/websockets/builder.ts new file mode 100644 index 0000000000..5c23682a7b --- /dev/null +++ b/packages/server/src/websockets/builder.ts @@ -0,0 +1,43 @@ +import authorized from "../middleware/authorized" +import Socket from "./websocket" +import { permissions } from "@budibase/backend-core" +import http from "http" +import Koa from "koa" +import { Table } from "@budibase/types" + +export default class BuilderSocket extends Socket { + constructor(app: Koa, server: http.Server) { + super(app, server, "/socket/builder", [authorized(permissions.BUILDER)]) + + this.io.on("connection", socket => { + // Join a room for this app + const user = socket.data.user + const appId = socket.data.appId + socket.join(appId) + socket.to(appId).emit("user-update", socket.data.user) + console.log(`Builder user connected: ${user?.id}`) + + // Initial identification of connected spreadsheet + socket.on("get-users", async (payload, callback) => { + const sockets = await this.io.in(appId).fetchSockets() + callback({ + users: sockets.map(socket => socket.data.user), + id: user.id, + }) + }) + + // Disconnection cleanup + socket.on("disconnect", () => { + socket.to(appId).emit("user-disconnect", socket.data.user) + }) + }) + } + + emitTableUpdate(ctx: any, table: Table) { + this.io.in(ctx.appId).emit("table-update", { id: table._id, table }) + } + + emitTableDeletion(ctx: any, id: string) { + this.io.in(ctx.appId).emit("table-update", { id, table: null }) + } +} diff --git a/packages/server/src/websockets/index.ts b/packages/server/src/websockets/index.ts index 60cdfb8aed..2761c4d9da 100644 --- a/packages/server/src/websockets/index.ts +++ b/packages/server/src/websockets/index.ts @@ -1,14 +1,17 @@ import http from "http" import Koa from "koa" -import GridSocket from "./grid" import ClientAppSocket from "./client" +import GridSocket from "./grid" +import BuilderSocket from "./builder" let clientAppSocket: ClientAppSocket let gridSocket: GridSocket +let builderSocket: BuilderSocket export const initialise = (app: Koa, server: http.Server) => { clientAppSocket = new ClientAppSocket(app, server) gridSocket = new GridSocket(app, server) + builderSocket = new BuilderSocket(app, server) } export { clientAppSocket, gridSocket } diff --git a/packages/server/src/websockets/websocket.ts b/packages/server/src/websockets/websocket.ts index 1b34168f14..0e0e6d59a4 100644 --- a/packages/server/src/websockets/websocket.ts +++ b/packages/server/src/websockets/websocket.ts @@ -59,13 +59,36 @@ export default class Socket { for (let [idx, middleware] of middlewares.entries()) { await middleware(ctx, () => { if (idx === middlewares.length - 1) { - // Middlewares are finished. + // Middlewares are finished // Extract some data from our enriched koa context to persist // as metadata for the socket - socket.data.user = { - id: ctx.user._id, - email: ctx.user.email, + + // Add user info, including a deterministic color and friendly + // label + const { _id, email, firstName, lastName } = ctx.user + let hue = 1 + for (let i = 0; i < email.length && i < 5; i++) { + hue *= email.charCodeAt(i + 1) + hue /= 17 } + hue = hue % 360 + const color = `hsl(${hue}, 50%, 40%)` + let label = email + if (firstName) { + label = firstName + if (lastName) { + label += ` ${lastName}` + } + } + socket.data.user = { + id: _id, + email, + color, + label, + } + + // Add app ID to help split sockets into rooms + socket.data.appId = ctx.appId next() } }) From 3794d8e204ed624031d23cbd9cdbefd145815d24 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Thu, 18 May 2023 08:57:20 +0100 Subject: [PATCH 10/31] Simplify websocket comms for grid and improve builder socket --- packages/builder/src/builderStore/index.js | 4 +- .../src/builderStore/store/frontend.js | 5 +-- .../builder/src/builderStore/store/users.js | 45 ++++++++++++++++--- .../builder/src/builderStore/websocket.js | 22 ++++----- .../backend/DataTable/DataTable.svelte | 1 + .../builder/app/[application]/_layout.svelte | 4 +- .../src/components/grid/layout/Grid.svelte | 12 ++++- .../src/components/grid/lib/websocket.js | 3 +- .../src/components/grid/stores/users.js | 42 +++-------------- packages/server/src/websockets/grid.ts | 1 - 10 files changed, 74 insertions(+), 65 deletions(-) diff --git a/packages/builder/src/builderStore/index.js b/packages/builder/src/builderStore/index.js index 8183c75a20..9dca6a64e6 100644 --- a/packages/builder/src/builderStore/index.js +++ b/packages/builder/src/builderStore/index.js @@ -2,7 +2,7 @@ import { getFrontendStore } from "./store/frontend" import { getAutomationStore } from "./store/automation" import { getTemporalStore } from "./store/temporal" import { getThemeStore } from "./store/theme" -import { getUsersStore } from "./store/users" +import { getUserStore } from "./store/users" import { derived } from "svelte/store" import { findComponent, findComponentPath } from "./componentUtils" import { RoleUtils } from "@budibase/frontend-core" @@ -13,7 +13,7 @@ export const store = getFrontendStore() export const automationStore = getAutomationStore() export const themeStore = getThemeStore() export const temporalStore = getTemporalStore() -export const usersStore = getUsersStore() +export const userStore = getUserStore() // Setup history for screens export const screenHistoryStore = createHistoryStore({ diff --git a/packages/builder/src/builderStore/store/frontend.js b/packages/builder/src/builderStore/store/frontend.js index 0a27f576a3..8c2119b2b0 100644 --- a/packages/builder/src/builderStore/store/frontend.js +++ b/packages/builder/src/builderStore/store/frontend.js @@ -5,6 +5,7 @@ import { selectedComponent, screenHistoryStore, automationHistoryStore, + userStore, } from "builderStore" import { datasources, @@ -85,9 +86,6 @@ const INITIAL_FRONTEND_STATE = { // Onboarding onboarding: false, tourNodes: null, - - // Multi user collab - users: [], } export const getFrontendStore = () => { @@ -122,7 +120,6 @@ export const getFrontendStore = () => { initialise: async pkg => { const { layouts, screens, application, clientLibPath, hasLock } = pkg websocket = createBuilderWebsocket() - await store.actions.components.refreshDefinitions(application.appId) // Reset store state diff --git a/packages/builder/src/builderStore/store/users.js b/packages/builder/src/builderStore/store/users.js index dfb592c1c5..615346c060 100644 --- a/packages/builder/src/builderStore/store/users.js +++ b/packages/builder/src/builderStore/store/users.js @@ -1,10 +1,43 @@ -import { writable } from "svelte/store" +import { writable, get } from "svelte/store" -export const getUsersStore = () => { - const initialValue = { - users: [], +export const getUserStore = () => { + const store = writable([]) + + const init = users => { + store.set(users) } - const store = writable(initialValue) - return store + const updateUser = user => { + console.log(user) + const $users = get(store) + if (!$users.some(x => x.id === user.id)) { + store.set([...$users, user]) + } else { + store.update(state => { + const index = state.findIndex(x => x.id === user.id) + state[index] = user + return state.slice() + }) + } + } + + const removeUser = user => { + store.update(state => { + return state.filter(x => x.id !== user.id) + }) + } + + const reset = () => { + store.set([]) + } + + return { + ...store, + actions: { + init, + updateUser, + removeUser, + reset, + }, + } } diff --git a/packages/builder/src/builderStore/websocket.js b/packages/builder/src/builderStore/websocket.js index d73ab08f9a..d9b203edb0 100644 --- a/packages/builder/src/builderStore/websocket.js +++ b/packages/builder/src/builderStore/websocket.js @@ -1,26 +1,26 @@ import { createWebsocket } from "@budibase/frontend-core" -import { store } from "builderStore" +import { userStore } from "builderStore" export const createBuilderWebsocket = () => { const socket = createWebsocket("/socket/builder") socket.on("connect", () => { socket.emit("get-users", null, response => { - console.log("conntected!", response.users) - store.update(state => ({ - ...state, - users: response.users, - })) + userStore.actions.init(response.users) }) }) - socket.on("user-update", user => {}) - - socket.on("user-disconnect", user => {}) - + socket.on("user-update", userStore.actions.updateUser) + socket.on("user-disconnect", userStore.actions.removeUser) socket.on("connect_error", err => { console.log("Failed to connect to builder websocket:", err.message) }) - return socket + return { + ...socket, + disconnect: () => { + socket?.disconnect() + userStore.actions.reset() + }, + } } diff --git a/packages/builder/src/components/backend/DataTable/DataTable.svelte b/packages/builder/src/components/backend/DataTable/DataTable.svelte index dfe30a3711..b5211b7d9b 100644 --- a/packages/builder/src/components/backend/DataTable/DataTable.svelte +++ b/packages/builder/src/components/backend/DataTable/DataTable.svelte @@ -36,6 +36,7 @@ allowDeleteRows={!isUsersTable} schemaOverrides={isUsersTable ? userSchemaOverrides : null} on:updatetable={e => tables.updateTable(e.detail)} + showAvatars={false} > {#if isInternal} diff --git a/packages/builder/src/pages/builder/app/[application]/_layout.svelte b/packages/builder/src/pages/builder/app/[application]/_layout.svelte index e145466b11..409f3e071b 100644 --- a/packages/builder/src/pages/builder/app/[application]/_layout.svelte +++ b/packages/builder/src/pages/builder/app/[application]/_layout.svelte @@ -1,5 +1,5 @@
- + {#if showAvatars} + + {/if}
{#if $loaded} diff --git a/packages/frontend-core/src/components/grid/lib/websocket.js b/packages/frontend-core/src/components/grid/lib/websocket.js index 7ac34b5a95..e8cead849e 100644 --- a/packages/frontend-core/src/components/grid/lib/websocket.js +++ b/packages/frontend-core/src/components/grid/lib/websocket.js @@ -2,7 +2,7 @@ import { get } from "svelte/store" import { createWebsocket } from "../../../utils" export const createGridWebsocket = context => { - const { rows, tableId, users, userId, focusedCellId } = context + const { rows, tableId, users, focusedCellId } = context const socket = createWebsocket("/socket/grid") const connectToTable = tableId => { @@ -13,7 +13,6 @@ export const createGridWebsocket = context => { socket.emit("select-table", tableId, response => { // handle initial connection info users.set(response.users) - userId.set(response.id) }) } diff --git a/packages/frontend-core/src/components/grid/stores/users.js b/packages/frontend-core/src/components/grid/stores/users.js index 5190de70da..a160330590 100644 --- a/packages/frontend-core/src/components/grid/stores/users.js +++ b/packages/frontend-core/src/components/grid/stores/users.js @@ -2,56 +2,28 @@ import { writable, get, derived } from "svelte/store" export const createStores = () => { const users = writable([]) - const userId = writable(null) - - // Enrich users with unique colours - const enrichedUsers = derived( - [users, userId], - ([$users, $userId]) => { - return ( - $users - .slice() - // Place current user first - .sort((a, b) => { - if (a.id === $userId) { - return -1 - } else if (b.id === $userId) { - return 1 - } else { - return 0 - } - }) - ) - }, - [] - ) return { - users: { - ...users, - subscribe: enrichedUsers.subscribe, - }, - userId, + users, } } export const deriveStores = context => { - const { users, userId } = context + const { users, focusedCellId } = context // Generate a lookup map of cell ID to the user that has it selected, to make // lookups inside cells extremely fast const selectedCellMap = derived( - [users, userId], - ([$enrichedUsers, $userId]) => { + [users, focusedCellId], + ([$users, $focusedCellId]) => { let map = {} - $enrichedUsers.forEach(user => { - if (user.focusedCellId && user.id !== $userId) { + $users.forEach(user => { + if (user.focusedCellId && user.focusedCellId !== $focusedCellId) { map[user.focusedCellId] = user } }) return map - }, - {} + } ) const updateUser = user => { diff --git a/packages/server/src/websockets/grid.ts b/packages/server/src/websockets/grid.ts index 4fb81e006c..731c920e07 100644 --- a/packages/server/src/websockets/grid.ts +++ b/packages/server/src/websockets/grid.ts @@ -34,7 +34,6 @@ export default class GridSocket extends Socket { const sockets = await this.io.in(currentRoom).fetchSockets() callback({ users: sockets.map(socket => socket.data.user), - id: user.id, }) }) From cfa07a68aeae7e26ee2367d62b49554d08f95588 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Mon, 22 May 2023 15:59:44 +0100 Subject: [PATCH 11/31] Handle table, query and datasource events using builder socket --- .../builder/src/builderStore/websocket.js | 19 ++++- .../backend/DataTable/DataTable.svelte | 1 - .../builder/src/stores/backend/datasources.js | 36 ++++++++- packages/builder/src/stores/backend/tables.js | 76 +++++++++++-------- packages/builder/src/stores/backend/views.js | 32 +++++--- .../src/components/grid/lib/websocket.js | 29 +++++-- .../src/components/grid/stores/columns.js | 4 - .../server/src/api/controllers/datasource.ts | 4 + .../server/src/api/controllers/table/index.ts | 3 + .../server/src/api/controllers/view/index.ts | 10 +-- packages/server/src/websockets/builder.ts | 20 ++++- packages/server/src/websockets/grid.ts | 14 +++- packages/server/src/websockets/index.ts | 2 +- 13 files changed, 177 insertions(+), 73 deletions(-) diff --git a/packages/builder/src/builderStore/websocket.js b/packages/builder/src/builderStore/websocket.js index d9b203edb0..d0baae38c6 100644 --- a/packages/builder/src/builderStore/websocket.js +++ b/packages/builder/src/builderStore/websocket.js @@ -1,21 +1,34 @@ import { createWebsocket } from "@budibase/frontend-core" import { userStore } from "builderStore" +import { datasources, tables } from "stores/backend" export const createBuilderWebsocket = () => { const socket = createWebsocket("/socket/builder") + // Connection events socket.on("connect", () => { socket.emit("get-users", null, response => { userStore.actions.init(response.users) }) }) - - socket.on("user-update", userStore.actions.updateUser) - socket.on("user-disconnect", userStore.actions.removeUser) socket.on("connect_error", err => { console.log("Failed to connect to builder websocket:", err.message) }) + // User events + socket.on("user-update", userStore.actions.updateUser) + socket.on("user-disconnect", userStore.actions.removeUser) + + // Table events + socket.on("table-change", ({ id, table }) => { + tables.replaceTable(id, table) + }) + + // Table events + socket.on("datasource-change", ({ id, datasource }) => { + datasources.replaceDatasource(id, datasource) + }) + return { ...socket, disconnect: () => { diff --git a/packages/builder/src/components/backend/DataTable/DataTable.svelte b/packages/builder/src/components/backend/DataTable/DataTable.svelte index b5211b7d9b..79b49d2541 100644 --- a/packages/builder/src/components/backend/DataTable/DataTable.svelte +++ b/packages/builder/src/components/backend/DataTable/DataTable.svelte @@ -35,7 +35,6 @@ allowAddRows={!isUsersTable} allowDeleteRows={!isUsersTable} schemaOverrides={isUsersTable ? userSchemaOverrides : null} - on:updatetable={e => tables.updateTable(e.detail)} showAvatars={false} > diff --git a/packages/builder/src/stores/backend/datasources.js b/packages/builder/src/stores/backend/datasources.js index ed84bb8ee9..e414e080a2 100644 --- a/packages/builder/src/stores/backend/datasources.js +++ b/packages/builder/src/stores/backend/datasources.js @@ -1,4 +1,4 @@ -import { writable, derived } from "svelte/store" +import { writable, derived, get } from "svelte/store" import { queries, tables } from "./" import { API } from "api" @@ -89,6 +89,39 @@ export function createDatasourcesStore() { }) } + // Handles external updates of tables + const replaceDatasource = (datasourceId, datasource) => { + if (!datasourceId) { + return + } + + // Handle deletion + if (!datasource) { + store.update(state => ({ + ...state, + list: state.list.filter(x => x._id !== datasourceId), + })) + return + } + + // Add new datasource + const index = get(store).list.findIndex(x => x._id === datasource._id) + if (index === -1) { + store.update(state => ({ + ...state, + list: [...state.list, datasource], + })) + } + + // Update existing datasource + else if (datasource) { + store.update(state => { + state.list[index] = datasource + return state + }) + } + } + return { subscribe: derivedStore.subscribe, fetch, @@ -98,6 +131,7 @@ export function createDatasourcesStore() { save, delete: deleteDatasource, removeSchemaError, + replaceDatasource, } } diff --git a/packages/builder/src/stores/backend/tables.js b/packages/builder/src/stores/backend/tables.js index 75679532f3..ba900b7df9 100644 --- a/packages/builder/src/stores/backend/tables.js +++ b/packages/builder/src/stores/backend/tables.js @@ -22,18 +22,6 @@ export function createTablesStore() { })) } - const fetchTable = async tableId => { - const table = await API.fetchTableDefinition(tableId) - - store.update(state => { - const indexToUpdate = state.list.findIndex(t => t._id === table._id) - state.list[indexToUpdate] = table - return { - ...state, - } - }) - } - const select = tableId => { store.update(state => ({ ...state, @@ -74,20 +62,23 @@ export function createTablesStore() { } const savedTable = await API.saveTable(updatedTable) - await fetch() + replaceTable(table._id, savedTable) if (table.type === "external") { await datasources.fetch() } - await select(savedTable._id) + select(savedTable._id) return savedTable } const deleteTable = async table => { + if (!table?._id || !table?._rev) { + return + } await API.deleteTable({ - tableId: table?._id, - tableRev: table?._rev, + tableId: table._id, + tableRev: table._rev, }) - await fetch() + replaceTable(table._id, null) } const saveField = async ({ @@ -135,35 +126,56 @@ export function createTablesStore() { await save(draft) } - const updateTable = table => { - const index = get(store).list.findIndex(x => x._id === table._id) - if (index === -1) { + // Handles external updates of tables + const replaceTable = (tableId, table) => { + if (!tableId) { return } - // This function has to merge state as there discrepancies with the table - // API endpoints. The table list endpoint and get table endpoint use the - // "type" property to mean different things. - store.update(state => { - state.list[index] = { - ...table, - type: state.list[index].type, - } - return state - }) + // Handle deletion + if (!table) { + store.update(state => ({ + ...state, + list: state.list.filter(x => x._id !== tableId), + })) + return + } + + // Add new table + const index = get(store).list.findIndex(x => x._id === table._id) + if (index === -1) { + store.update(state => ({ + ...state, + list: [...state.list, table], + })) + } + + // Update existing table + else if (table) { + // This function has to merge state as there discrepancies with the table + // API endpoints. The table list endpoint and get table endpoint use the + // "type" property to mean different things. + store.update(state => { + state.list[index] = { + ...table, + type: state.list[index].type, + } + return state + }) + } } return { + ...store, subscribe: derivedStore.subscribe, fetch, - fetchTable, init: fetch, select, save, delete: deleteTable, saveField, deleteField, - updateTable, + replaceTable, } } diff --git a/packages/builder/src/stores/backend/views.js b/packages/builder/src/stores/backend/views.js index 71e9a44c5c..2f396096c8 100644 --- a/packages/builder/src/stores/backend/views.js +++ b/packages/builder/src/stores/backend/views.js @@ -1,4 +1,4 @@ -import { writable, get, derived } from "svelte/store" +import { writable, derived } from "svelte/store" import { tables } from "./" import { API } from "api" @@ -27,21 +27,31 @@ export function createViewsStore() { const deleteView = async view => { await API.deleteView(view) - await tables.fetch() + + // Update tables + tables.update(state => { + const table = state.list.find(table => table._id === view.tableId) + if (table) { + delete table.views[view.name] + } + return { ...state } + }) } const save = async view => { const savedView = await API.saveView(view) - const viewMeta = { - name: view.name, - ...savedView, - } - const viewTable = get(tables).list.find(table => table._id === view.tableId) - - if (view.originalName) delete viewTable.views[view.originalName] - viewTable.views[view.name] = viewMeta - await tables.save(viewTable) + // Update tables + tables.update(state => { + const table = state.list.find(table => table._id === view.tableId) + if (table) { + if (view.originalName) { + delete table.views[view.originalName] + } + table.views[view.name] = savedView + } + return { ...state } + }) } return { diff --git a/packages/frontend-core/src/components/grid/lib/websocket.js b/packages/frontend-core/src/components/grid/lib/websocket.js index e8cead849e..a62910c2fa 100644 --- a/packages/frontend-core/src/components/grid/lib/websocket.js +++ b/packages/frontend-core/src/components/grid/lib/websocket.js @@ -2,7 +2,7 @@ import { get } from "svelte/store" import { createWebsocket } from "../../../utils" export const createGridWebsocket = context => { - const { rows, tableId, users, focusedCellId } = context + const { rows, tableId, users, focusedCellId, table } = context const socket = createWebsocket("/socket/grid") const connectToTable = tableId => { @@ -16,23 +16,36 @@ export const createGridWebsocket = context => { }) } - // Event handlers + // Connection events socket.on("connect", () => { connectToTable(get(tableId)) }) - socket.on("row-update", data => { - if (data.id) { - rows.actions.replaceRow(data.id, data.row) - } + socket.on("connect_error", err => { + console.log("Failed to connect to grid websocket:", err.message) }) + + // User events socket.on("user-update", user => { users.actions.updateUser(user) }) socket.on("user-disconnect", user => { users.actions.removeUser(user) }) - socket.on("connect_error", err => { - console.log("Failed to connect to grid websocket:", err.message) + + // Row events + socket.on("row-change", data => { + if (data.id) { + rows.actions.replaceRow(data.id, data.row) + } + }) + + // Table events + socket.on("table-change", data => { + // Only update table if one exists. If the table was deleted then we don't + // want to know - let the builder navigate away + if (data.table) { + table.set(data.table) + } }) // Change websocket connection when table changes diff --git a/packages/frontend-core/src/components/grid/stores/columns.js b/packages/frontend-core/src/components/grid/stores/columns.js index e953977487..3b286ac0f0 100644 --- a/packages/frontend-core/src/components/grid/stores/columns.js +++ b/packages/frontend-core/src/components/grid/stores/columns.js @@ -90,10 +90,6 @@ export const deriveStores = context => { // Update local state table.set(newTable) - // Broadcast event so that we can keep sync with external state - // (e.g. data section which maintains a list of table definitions) - dispatch("updatetable", newTable) - // Update server await API.saveTable(newTable) } diff --git a/packages/server/src/api/controllers/datasource.ts b/packages/server/src/api/controllers/datasource.ts index 8f13e0e618..ecaf1ef58b 100644 --- a/packages/server/src/api/controllers/datasource.ts +++ b/packages/server/src/api/controllers/datasource.ts @@ -25,6 +25,7 @@ import { DatasourcePlus, } from "@budibase/types" import sdk from "../../sdk" +import { builderSocket } from "../../websockets" function getErrorTables(errors: any, errorType: string) { return Object.entries(errors) @@ -280,6 +281,7 @@ export async function update(ctx: UserCtx) { ctx.body = { datasource: await sdk.datasources.removeSecretSingle(datasource), } + builderSocket.emitDatasourceUpdate(ctx, datasource) } export async function save( @@ -322,6 +324,7 @@ export async function save( response.error = schemaError } ctx.body = response + builderSocket.emitDatasourceUpdate(ctx, datasource) } async function destroyInternalTablesBySourceId(datasourceId: string) { @@ -381,6 +384,7 @@ export async function destroy(ctx: UserCtx) { ctx.message = `Datasource deleted.` ctx.status = 200 + builderSocket.emitDatasourceDeletion(ctx, datasourceId) } export async function find(ctx: UserCtx) { diff --git a/packages/server/src/api/controllers/table/index.ts b/packages/server/src/api/controllers/table/index.ts index cbbda7b930..271f3e82fa 100644 --- a/packages/server/src/api/controllers/table/index.ts +++ b/packages/server/src/api/controllers/table/index.ts @@ -11,6 +11,7 @@ import { context, events } from "@budibase/backend-core" import { Table, UserCtx } from "@budibase/types" import sdk from "../../../sdk" import { jsonFromCsvString } from "../../../utilities/csv" +import { builderSocket } from "../../../websockets" function pickApi({ tableId, table }: { tableId?: string; table?: Table }) { if (table && !tableId) { @@ -77,6 +78,7 @@ export async function save(ctx: UserCtx) { ctx.eventEmitter && ctx.eventEmitter.emitTable(`table:save`, appId, savedTable) ctx.body = savedTable + builderSocket.emitTableUpdate(ctx, savedTable) } export async function destroy(ctx: UserCtx) { @@ -89,6 +91,7 @@ export async function destroy(ctx: UserCtx) { ctx.status = 200 ctx.table = deletedTable ctx.body = { message: `Table ${tableId} deleted.` } + builderSocket.emitTableDeletion(ctx, tableId) } export async function bulkImport(ctx: UserCtx) { diff --git a/packages/server/src/api/controllers/view/index.ts b/packages/server/src/api/controllers/view/index.ts index ed2302f723..28b0d0a81f 100644 --- a/packages/server/src/api/controllers/view/index.ts +++ b/packages/server/src/api/controllers/view/index.ts @@ -16,6 +16,7 @@ import { View, } from "@budibase/types" import { cleanExportRows } from "../row/utils" +import { builderSocket } from "../../../websockets" const { cloneDeep, isEqual } = require("lodash") @@ -48,7 +49,7 @@ export async function save(ctx: Ctx) { if (!view.meta.schema) { view.meta.schema = table.schema } - table.views[viewName] = view.meta + table.views[viewName] = { ...view.meta, name: viewName } if (originalName) { delete table.views[originalName] existingTable.views[viewName] = existingTable.views[originalName] @@ -56,10 +57,8 @@ export async function save(ctx: Ctx) { await db.put(table) await handleViewEvents(existingTable.views[viewName], table.views[viewName]) - ctx.body = { - ...table.views[viewToSave.name], - name: viewToSave.name, - } + ctx.body = table.views[viewName] + builderSocket.emitTableUpdate(ctx, table) } export async function calculationEvents(existingView: View, newView: View) { @@ -128,6 +127,7 @@ export async function destroy(ctx: Ctx) { await events.view.deleted(view) ctx.body = view + builderSocket.emitTableUpdate(ctx, table) } export async function exportView(ctx: Ctx) { diff --git a/packages/server/src/websockets/builder.ts b/packages/server/src/websockets/builder.ts index 5c23682a7b..ae32a7de18 100644 --- a/packages/server/src/websockets/builder.ts +++ b/packages/server/src/websockets/builder.ts @@ -3,7 +3,8 @@ import Socket from "./websocket" import { permissions } from "@budibase/backend-core" import http from "http" import Koa from "koa" -import { Table } from "@budibase/types" +import { Datasource, Table } from "@budibase/types" +import { gridSocket } from "./index" export default class BuilderSocket extends Socket { constructor(app: Koa, server: http.Server) { @@ -22,7 +23,6 @@ export default class BuilderSocket extends Socket { const sockets = await this.io.in(appId).fetchSockets() callback({ users: sockets.map(socket => socket.data.user), - id: user.id, }) }) @@ -34,10 +34,22 @@ export default class BuilderSocket extends Socket { } emitTableUpdate(ctx: any, table: Table) { - this.io.in(ctx.appId).emit("table-update", { id: table._id, table }) + this.io.in(ctx.appId).emit("table-change", { id: table._id, table }) + gridSocket.emitTableUpdate(table) } emitTableDeletion(ctx: any, id: string) { - this.io.in(ctx.appId).emit("table-update", { id, table: null }) + this.io.in(ctx.appId).emit("table-change", { id, table: null }) + gridSocket.emitTableDeletion(id) + } + + emitDatasourceUpdate(ctx: any, datasource: Datasource) { + this.io + .in(ctx.appId) + .emit("datasource-change", { id: datasource._id, datasource }) + } + + emitDatasourceDeletion(ctx: any, id: string) { + this.io.in(ctx.appId).emit("datasource-change", { id, datasource: null }) } } diff --git a/packages/server/src/websockets/grid.ts b/packages/server/src/websockets/grid.ts index 731c920e07..09b9ee0e3d 100644 --- a/packages/server/src/websockets/grid.ts +++ b/packages/server/src/websockets/grid.ts @@ -4,7 +4,7 @@ import { permissions } from "@budibase/backend-core" import http from "http" import Koa from "koa" import { getTableId } from "../api/controllers/row/utils" -import { Row } from "@budibase/types" +import { Row, Table } from "@budibase/types" export default class GridSocket extends Socket { constructor(app: Koa, server: http.Server) { @@ -56,11 +56,19 @@ export default class GridSocket extends Socket { emitRowUpdate(ctx: any, row: Row) { const tableId = getTableId(ctx) - this.io.in(tableId).emit("row-update", { id: row._id, row }) + this.io.in(tableId).emit("row-change", { id: row._id, row }) } emitRowDeletion(ctx: any, id: string) { const tableId = getTableId(ctx) - this.io.in(tableId).emit("row-update", { id, row: null }) + this.io.in(tableId).emit("row-change", { id, row: null }) + } + + emitTableUpdate(table: Table) { + this.io.in(table._id!).emit("table-change", { id: table._id, table }) + } + + emitTableDeletion(id: string) { + this.io.in(id).emit("table-change", { id, table: null }) } } diff --git a/packages/server/src/websockets/index.ts b/packages/server/src/websockets/index.ts index 2761c4d9da..b74a8adfca 100644 --- a/packages/server/src/websockets/index.ts +++ b/packages/server/src/websockets/index.ts @@ -14,4 +14,4 @@ export const initialise = (app: Koa, server: http.Server) => { builderSocket = new BuilderSocket(app, server) } -export { clientAppSocket, gridSocket } +export { clientAppSocket, gridSocket, builderSocket } From 53eceec9785444de29fa0b622f2ebc4e095cd619 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Mon, 22 May 2023 16:01:12 +0100 Subject: [PATCH 12/31] Update comments --- packages/builder/src/stores/backend/datasources.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/builder/src/stores/backend/datasources.js b/packages/builder/src/stores/backend/datasources.js index e414e080a2..98f6521bbb 100644 --- a/packages/builder/src/stores/backend/datasources.js +++ b/packages/builder/src/stores/backend/datasources.js @@ -89,7 +89,7 @@ export function createDatasourcesStore() { }) } - // Handles external updates of tables + // Handles external updates of datasources const replaceDatasource = (datasourceId, datasource) => { if (!datasourceId) { return From 5e480a1527314d9a7340335b747c85b8062fb251 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Thu, 25 May 2023 08:48:56 +0100 Subject: [PATCH 13/31] Use socker.io redis adapter to broadcast events to all server instances --- packages/backend-core/src/redis/utils.ts | 1 + packages/server/package.json | 1 + packages/server/src/app.ts | 1 - packages/server/src/startup.ts | 2 ++ packages/server/src/utilities/redis.ts | 21 +++++++++++++++++++-- packages/server/src/websockets/websocket.ts | 9 ++++++++- yarn.lock | 19 +++++++++++++++++++ 7 files changed, 50 insertions(+), 4 deletions(-) diff --git a/packages/backend-core/src/redis/utils.ts b/packages/backend-core/src/redis/utils.ts index 2c49ee4941..f8b815824c 100644 --- a/packages/backend-core/src/redis/utils.ts +++ b/packages/backend-core/src/redis/utils.ts @@ -27,6 +27,7 @@ export enum Databases { GENERIC_CACHE = "data_cache", WRITE_THROUGH = "writeThrough", LOCKS = "locks", + SOCKET_IO = "socket_io", } /** diff --git a/packages/server/package.json b/packages/server/package.json index 5ed4579667..31022a7c99 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -58,6 +58,7 @@ "@koa/router": "8.0.8", "@sendgrid/mail": "7.1.1", "@sentry/node": "6.17.7", + "@socket.io/redis-adapter": "^8.2.1", "airtable": "0.10.1", "arangojs": "7.2.0", "aws-sdk": "2.1030.0", diff --git a/packages/server/src/app.ts b/packages/server/src/app.ts index f8f82f9fdc..274de89c57 100644 --- a/packages/server/src/app.ts +++ b/packages/server/src/app.ts @@ -61,7 +61,6 @@ if (env.isProd()) { const server = http.createServer(app.callback()) destroyable(server) -initialiseWebsockets(app, server) let shuttingDown = false, errCode = 0 diff --git a/packages/server/src/startup.ts b/packages/server/src/startup.ts index 2fd59b7a8e..64a6c011c4 100644 --- a/packages/server/src/startup.ts +++ b/packages/server/src/startup.ts @@ -16,6 +16,7 @@ import * as bullboard from "./automations/bullboard" import * as pro from "@budibase/pro" import * as api from "./api" import sdk from "./sdk" +import { initialise as initialiseWebsockets } from "./websockets" let STARTUP_RAN = false @@ -64,6 +65,7 @@ export async function startup(app?: any, server?: any) { fileSystem.init() await redis.init() eventInit() + initialiseWebsockets(app, server) // run migrations on startup if not done via http // not recommended in a clustered environment diff --git a/packages/server/src/utilities/redis.ts b/packages/server/src/utilities/redis.ts index dc37baae58..ff1c863bf7 100644 --- a/packages/server/src/utilities/redis.ts +++ b/packages/server/src/utilities/redis.ts @@ -4,23 +4,33 @@ import { ContextUser } from "@budibase/types" const APP_DEV_LOCK_SECONDS = 600 const AUTOMATION_TEST_FLAG_SECONDS = 60 -let devAppClient: any, debounceClient: any, flagClient: any +let devAppClient: any, debounceClient: any, flagClient: any, socketClient: any -// we init this as we want to keep the connection open all the time +// We need to maintain a duplicate client for socket.io pub/sub +let socketSubClient: any + +// We init this as we want to keep the connection open all the time // reduces the performance hit export async function init() { devAppClient = new redis.Client(redis.utils.Databases.DEV_LOCKS) debounceClient = new redis.Client(redis.utils.Databases.DEBOUNCE) flagClient = new redis.Client(redis.utils.Databases.FLAGS) + socketClient = new redis.Client(redis.utils.Databases.SOCKET_IO) await devAppClient.init() await debounceClient.init() await flagClient.init() + await socketClient.init() + + // Duplicate the socket client for pub/sub + socketSubClient = socketClient.getClient().duplicate() } export async function shutdown() { if (devAppClient) await devAppClient.finish() if (debounceClient) await debounceClient.finish() if (flagClient) await flagClient.finish() + if (socketClient) await socketClient.finish() + if (socketSubClient) socketSubClient.disconnect() // shutdown core clients await redis.clients.shutdown() console.log("Redis shutdown") @@ -86,3 +96,10 @@ export async function checkTestFlag(id: string) { export async function clearTestFlag(id: string) { await devAppClient.delete(id) } + +export function getSocketPubSubClients() { + return { + pub: socketClient.getClient(), + sub: socketSubClient, + } +} diff --git a/packages/server/src/websockets/websocket.ts b/packages/server/src/websockets/websocket.ts index 0e0e6d59a4..c3a1019932 100644 --- a/packages/server/src/websockets/websocket.ts +++ b/packages/server/src/websockets/websocket.ts @@ -5,6 +5,8 @@ import Cookies from "cookies" import { userAgent } from "koa-useragent" import { auth } from "@budibase/backend-core" import currentApp from "../middleware/currentapp" +import { createAdapter } from "@socket.io/redis-adapter" +import { getSocketPubSubClients } from "../utilities/redis" export default class Socket { io: Server @@ -12,7 +14,7 @@ export default class Socket { constructor( app: Koa, server: http.Server, - path: string, + path: string = "/", additionalMiddlewares?: any[] ) { this.io = new Server(server, { @@ -97,6 +99,11 @@ export default class Socket { next(error) } }) + + // Instantiate redis adapter + const { pub, sub } = getSocketPubSubClients() + const opts = { key: `socket.io-${path}` } + this.io.adapter(createAdapter(pub, sub, opts)) } // Emit an event to all sockets diff --git a/yarn.lock b/yarn.lock index 086342b0f2..e37e9769eb 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4623,6 +4623,15 @@ resolved "https://registry.yarnpkg.com/@socket.io/component-emitter/-/component-emitter-3.1.0.tgz#96116f2a912e0c02817345b3c10751069920d553" integrity sha512-+9jVqKhRSpsc591z5vX+X5Yyw+he/HCB4iQ/RYxw35CEPaY1gnsNE43nf9n9AaYjAQrTiI/mOwKUKdUs9vf7Xg== +"@socket.io/redis-adapter@^8.2.1": + version "8.2.1" + resolved "https://registry.yarnpkg.com/@socket.io/redis-adapter/-/redis-adapter-8.2.1.tgz#36f75afc518d0e1fa4fa7c29e6d042f53ee7563b" + integrity sha512-6Dt7EZgGSBP0qvXeOKGx7NnSr2tPMbVDfDyL97zerZo+v69hMfL99skMCL3RKZlWVqLyRme2T0wcy3udHhtOsg== + dependencies: + debug "~4.3.1" + notepack.io "~3.0.1" + uid2 "1.0.0" + "@spectrum-css/accordion@3.0.24": version "3.0.24" resolved "https://registry.yarnpkg.com/@spectrum-css/accordion/-/accordion-3.0.24.tgz#f89066c120c57b0cfc9aba66d60c39fc1cf69f74" @@ -18641,6 +18650,11 @@ normalize-url@^6.0.1: resolved "https://registry.yarnpkg.com/normalize-url/-/normalize-url-6.1.0.tgz#40d0885b535deffe3f3147bec877d05fe4c5668a" integrity sha512-DlL+XwOy3NxAQ8xuC0okPgK46iuVNAK01YN7RueYBqqFeGsBjV9XmCAzAdgt+667bCl5kPh9EqKKDwnaPG1I7A== +notepack.io@~3.0.1: + version "3.0.1" + resolved "https://registry.yarnpkg.com/notepack.io/-/notepack.io-3.0.1.tgz#2c2c9de1bd4e64a79d34e33c413081302a0d4019" + integrity sha512-TKC/8zH5pXIAMVQio2TvVDTtPRX+DJPHDqjRbxogtFiByHyzKmy96RA0JtCQJ+WouyyL4A10xomQzgbUT+1jCg== + npm-bundled@^1.1.2: version "1.1.2" resolved "https://registry.yarnpkg.com/npm-bundled/-/npm-bundled-1.1.2.tgz#944c78789bd739035b70baa2ca5cc32b8d860bc1" @@ -24787,6 +24801,11 @@ uid2@0.0.x: resolved "https://registry.yarnpkg.com/uid2/-/uid2-0.0.4.tgz#033f3b1d5d32505f5ce5f888b9f3b667123c0a44" integrity sha512-IevTus0SbGwQzYh3+fRsAMTVVPOoIVufzacXcHPmdlle1jUpq7BRL+mw3dgeLanvGZdwwbWhRV6XrcFNdBmjWA== +uid2@1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/uid2/-/uid2-1.0.0.tgz#ef8d95a128d7c5c44defa1a3d052eecc17a06bfb" + integrity sha512-+I6aJUv63YAcY9n4mQreLUt0d4lvwkkopDNmpomkAUz0fAkEMV9pRWxN0EjhW1YfRhcuyHg2v3mwddCDW1+LFQ== + unbox-primitive@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/unbox-primitive/-/unbox-primitive-1.0.2.tgz#29032021057d5e6cdbd08c5129c226dff8ed6f9e" From 183fb3d29b5fc252b10f46935a0cc1179cec2cc6 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Tue, 23 May 2023 16:02:05 +0100 Subject: [PATCH 14/31] Fix issue with schema mutation in grids. Use displayName to represent grid column labels. Allow deletion of duplicated user columns --- .../backend/DataTable/DataTable.svelte | 10 +++++----- .../src/components/grid/stores/columns.js | 20 ++++++++++++++++--- 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/packages/builder/src/components/backend/DataTable/DataTable.svelte b/packages/builder/src/components/backend/DataTable/DataTable.svelte index 79b49d2541..7a8b622594 100644 --- a/packages/builder/src/components/backend/DataTable/DataTable.svelte +++ b/packages/builder/src/components/backend/DataTable/DataTable.svelte @@ -16,11 +16,11 @@ import GridEditColumnModal from "components/backend/DataTable/modals/grid/GridEditColumnModal.svelte" const userSchemaOverrides = { - firstName: { name: "First name", disabled: true }, - lastName: { name: "Last name", disabled: true }, - email: { name: "Email", disabled: true }, - roleId: { name: "Role", disabled: true }, - status: { name: "Status", disabled: true }, + firstName: { displayName: "First name", disabled: true }, + lastName: { displayName: "Last name", disabled: true }, + email: { displayName: "Email", disabled: true }, + roleId: { displayName: "Role", disabled: true }, + status: { displayName: "Status", disabled: true }, } $: id = $tables.selected?._id diff --git a/packages/frontend-core/src/components/grid/stores/columns.js b/packages/frontend-core/src/components/grid/stores/columns.js index 3b286ac0f0..46519b9226 100644 --- a/packages/frontend-core/src/components/grid/stores/columns.js +++ b/packages/frontend-core/src/components/grid/stores/columns.js @@ -112,10 +112,24 @@ export const initialise = context => { const schema = derived( [table, schemaOverrides], ([$table, $schemaOverrides]) => { - let newSchema = $table?.schema - if (!newSchema) { + if (!$table?.schema) { return null } + let newSchema = { ...$table?.schema } + + // Edge case to temporarily allow deletion of duplicated user + // fields that were saved with the "disabled" flag set. + // By overriding the saved schema we ensure only overrides can + // set the disabled flag. + // TODO: remove in future + Object.keys(newSchema).forEach(field => { + newSchema[field] = { + ...newSchema[field], + disabled: false, + } + }) + + // Apply schema overrides Object.keys($schemaOverrides || {}).forEach(field => { if (newSchema[field]) { newSchema[field] = { @@ -156,7 +170,7 @@ export const initialise = context => { fields .map(field => ({ name: field, - label: $schema[field].name || field, + label: $schema[field].displayName || field, schema: $schema[field], width: $schema[field].width || DefaultColumnWidth, visible: $schema[field].visible ?? true, From b29ea6120364ebf969814eca21ef15b4fde749a3 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Tue, 23 May 2023 18:01:04 +0100 Subject: [PATCH 15/31] Ensure text cells account for being used for number cells too --- .../frontend-core/src/components/grid/cells/TextCell.svelte | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/frontend-core/src/components/grid/cells/TextCell.svelte b/packages/frontend-core/src/components/grid/cells/TextCell.svelte index 533b030b5c..04485a6b50 100644 --- a/packages/frontend-core/src/components/grid/cells/TextCell.svelte +++ b/packages/frontend-core/src/components/grid/cells/TextCell.svelte @@ -52,7 +52,7 @@ {:else}
- {value || ""} + {value ?? ""}
{/if} From f246a982db89b7ad007ae69351172507fcb4d895 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Thu, 25 May 2023 10:45:00 +0100 Subject: [PATCH 16/31] Update lock icon and remove logs --- packages/builder/src/builderStore/store/users.js | 1 - packages/builder/src/components/common/AppLockModal.svelte | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/builder/src/builderStore/store/users.js b/packages/builder/src/builderStore/store/users.js index 615346c060..6de2adc90a 100644 --- a/packages/builder/src/builderStore/store/users.js +++ b/packages/builder/src/builderStore/store/users.js @@ -8,7 +8,6 @@ export const getUserStore = () => { } const updateUser = user => { - console.log(user) const $users = get(store) if (!$users.some(x => x.id === user.id)) { store.set([...$users, user]) diff --git a/packages/builder/src/components/common/AppLockModal.svelte b/packages/builder/src/components/common/AppLockModal.svelte index 875b4afce0..df491ead38 100644 --- a/packages/builder/src/components/common/AppLockModal.svelte +++ b/packages/builder/src/components/common/AppLockModal.svelte @@ -60,7 +60,7 @@ {#if lockedBy}
{ From cc7df474c91eff86a94b33125abc4ca03db3c744 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Fri, 26 May 2023 09:24:53 +0100 Subject: [PATCH 17/31] Standardise usage of user avatars and colours across the entire platform --- packages/bbui/src/Avatar/Avatar.svelte | 18 ++- .../builder/src/builderStore/store/users.js | 6 +- .../src/components/common/AppLockModal.svelte | 143 ------------------ .../src/components/start/AppRow.svelte | 29 ++-- .../_components/UserAvatars.svelte | 33 ++-- .../builder/app/[application]/_layout.svelte | 12 +- .../src/pages/builder/apps/index.svelte | 11 +- .../portal/_components/UserDropdown.svelte | 5 +- .../auditLogs/_components/UserRenderer.svelte | 13 +- .../portal/overview/[appId]/_layout.svelte | 2 - .../backups/_components/UserRenderer.svelte | 9 +- .../portal/overview/[appId]/overview.svelte | 15 +- .../portal/users/users/[userId].svelte | 19 +-- .../src/components/UserAvatar.svelte | 58 +++++++ .../src/components/grid/layout/Avatar.svelte | 24 --- .../components/grid/layout/UserAvatars.svelte | 16 +- .../src/components/grid/stores/users.js | 20 ++- .../frontend-core/src/components/index.js | 1 + packages/server/src/websockets/builder.ts | 22 ++- packages/server/src/websockets/grid.ts | 9 +- packages/server/src/websockets/websocket.ts | 26 +--- packages/shared-core/src/helpers/helpers.ts | 59 ++++++++ 22 files changed, 254 insertions(+), 296 deletions(-) delete mode 100644 packages/builder/src/components/common/AppLockModal.svelte create mode 100644 packages/frontend-core/src/components/UserAvatar.svelte delete mode 100644 packages/frontend-core/src/components/grid/layout/Avatar.svelte diff --git a/packages/bbui/src/Avatar/Avatar.svelte b/packages/bbui/src/Avatar/Avatar.svelte index 1e4cefd8ce..0faf50f55a 100644 --- a/packages/bbui/src/Avatar/Avatar.svelte +++ b/packages/bbui/src/Avatar/Avatar.svelte @@ -13,10 +13,12 @@ export let url = "" export let disabled = false export let initials = "JD" + export let color = null const DefaultColor = "#3aab87" - $: color = getColor(initials) + $: avatarColor = color || getColor(initials) + $: style = getStyle(size, avatarColor) const getColor = initials => { if (!initials?.length) { @@ -26,6 +28,12 @@ const hue = ((code % 26) / 26) * 360 return `hsl(${hue}, 50%, 50%)` } + + const getStyle = (sizeKey, color) => { + const size = `var(${sizes.get(sizeKey)})` + const fontSize = `calc(${size} / 2)` + return `width:${size}; height:${size}; font-size:${fontSize}; background:${color};` + } {#if url} @@ -37,13 +45,7 @@ style="width: var({sizes.get(size)}); height: var({sizes.get(size)});" /> {:else} -
+
{initials || ""}
{/if} diff --git a/packages/builder/src/builderStore/store/users.js b/packages/builder/src/builderStore/store/users.js index 6de2adc90a..18f343c884 100644 --- a/packages/builder/src/builderStore/store/users.js +++ b/packages/builder/src/builderStore/store/users.js @@ -9,11 +9,11 @@ export const getUserStore = () => { const updateUser = user => { const $users = get(store) - if (!$users.some(x => x.id === user.id)) { + if (!$users.some(x => x.sessionId === user.sessionId)) { store.set([...$users, user]) } else { store.update(state => { - const index = state.findIndex(x => x.id === user.id) + const index = state.findIndex(x => x.sessionId === user.sessionId) state[index] = user return state.slice() }) @@ -22,7 +22,7 @@ export const getUserStore = () => { const removeUser = user => { store.update(state => { - return state.filter(x => x.id !== user.id) + return state.filter(x => x.sessionId !== user.sessionId) }) } diff --git a/packages/builder/src/components/common/AppLockModal.svelte b/packages/builder/src/components/common/AppLockModal.svelte deleted file mode 100644 index df491ead38..0000000000 --- a/packages/builder/src/components/common/AppLockModal.svelte +++ /dev/null @@ -1,143 +0,0 @@ - - -{#if lockedBy} -
- { - e.stopPropagation() - appLockModal.show() - }} - /> -
-{/if} - - - - - - Apps are locked to prevent work being lost from overlapping changes - between your team. - - {#if lockedByYou && getExpiryDuration(app) > 0} - - {processStringSync( - "This lock will expire in {{ duration time 'millisecond' }} from now.", - { - time: getExpiryDuration(app), - } - )} - - {/if} -
- - - {#if lockedByYou} - - {/if} - -
- - - - - diff --git a/packages/builder/src/components/start/AppRow.svelte b/packages/builder/src/components/start/AppRow.svelte index 162958473e..0c011e89be 100644 --- a/packages/builder/src/components/start/AppRow.svelte +++ b/packages/builder/src/components/start/AppRow.svelte @@ -1,13 +1,16 @@
- {#each users as user} -
- {user.email[0]} -
+ {#each uniqueUsers as user} + {/each}
@@ -15,19 +25,4 @@ display: flex; gap: 4px; } - .avatar { - width: 24px; - height: 24px; - display: grid; - place-items: center; - color: white; - border-radius: 50%; - font-size: 12px; - font-weight: 700; - text-transform: uppercase; - line-height: 0; - } - .avatar:hover { - cursor: pointer; - } diff --git a/packages/builder/src/pages/builder/app/[application]/_layout.svelte b/packages/builder/src/pages/builder/app/[application]/_layout.svelte index 409f3e071b..4da0da44f9 100644 --- a/packages/builder/src/pages/builder/app/[application]/_layout.svelte +++ b/packages/builder/src/pages/builder/app/[application]/_layout.svelte @@ -218,7 +218,9 @@
{:then _} - +
+ +
{:catch error}

Something went wrong: {error.message}

{/await} @@ -254,6 +256,7 @@ box-sizing: border-box; align-items: stretch; border-bottom: var(--border-light); + z-index: 2; } .topleftnav { @@ -294,4 +297,11 @@ flex-direction: row; gap: 8px; } + + .body { + flex: 1 1 auto; + z-index: 1; + display: flex; + flex-direction: column; + } diff --git a/packages/builder/src/pages/builder/apps/index.svelte b/packages/builder/src/pages/builder/apps/index.svelte index 4b77671345..1806d51b69 100644 --- a/packages/builder/src/pages/builder/apps/index.svelte +++ b/packages/builder/src/pages/builder/apps/index.svelte @@ -5,7 +5,6 @@ Divider, ActionMenu, MenuItem, - Avatar, Page, Icon, Body, @@ -22,6 +21,8 @@ import { processStringSync } from "@budibase/string-templates" import Spaceman from "assets/bb-space-man.svg" import Logo from "assets/bb-emblem.svg" + import { UserAvatar } from "@budibase/frontend-core" + import { helpers } from "@budibase/shared-core" let loaded = false let userInfoModal @@ -96,11 +97,7 @@
- +
userInfoModal.show()}> @@ -125,7 +122,7 @@
- Hey {$auth.user.firstName || $auth.user.email} + Hey {helpers.getUserLabel($auth.user)} Welcome to the {$organisation.company} portal. Below you'll find the diff --git a/packages/builder/src/pages/builder/portal/_components/UserDropdown.svelte b/packages/builder/src/pages/builder/portal/_components/UserDropdown.svelte index 935d69812f..9faae70aa9 100644 --- a/packages/builder/src/pages/builder/portal/_components/UserDropdown.svelte +++ b/packages/builder/src/pages/builder/portal/_components/UserDropdown.svelte @@ -1,11 +1,12 @@ {#if row?.user?.email} @@ -19,7 +14,7 @@ on:focus={() => (showTooltip = true)} on:mouseleave={() => (showTooltip = false)} > - +
{#if showTooltip}
diff --git a/packages/builder/src/pages/builder/portal/overview/[appId]/_layout.svelte b/packages/builder/src/pages/builder/portal/overview/[appId]/_layout.svelte index cc244852d8..8ee469a914 100644 --- a/packages/builder/src/pages/builder/portal/overview/[appId]/_layout.svelte +++ b/packages/builder/src/pages/builder/portal/overview/[appId]/_layout.svelte @@ -24,7 +24,6 @@ import { AppStatus } from "constants" import analytics, { Events, EventSource } from "analytics" import { store } from "builderStore" - import AppLockModal from "components/common/AppLockModal.svelte" import EditableIcon from "components/common/EditableIcon.svelte" import { API } from "api" import ConfirmDialog from "components/common/ConfirmDialog.svelte" @@ -128,7 +127,6 @@ />
- - - - - + {#if !$isActive("./new")} + + + + + + + {/if}
diff --git a/packages/builder/src/pages/builder/app/[application]/data/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/index.svelte index b2aca1f7f3..47939f09b4 100644 --- a/packages/builder/src/pages/builder/app/[application]/data/index.svelte +++ b/packages/builder/src/pages/builder/app/[application]/data/index.svelte @@ -1,22 +1,17 @@ - - diff --git a/packages/builder/src/pages/builder/app/[application]/data/new.svelte b/packages/builder/src/pages/builder/app/[application]/data/new.svelte new file mode 100644 index 0000000000..f8e8fd85e7 --- /dev/null +++ b/packages/builder/src/pages/builder/app/[application]/data/new.svelte @@ -0,0 +1,257 @@ + + + + + + + + {#if integration?.auth?.type === "google"} + + {:else} + + {/if} + + +
+
+ {#if hasData} + + {/if} +
+
+ Add new data source +
+ +
+ Get started with our Budibase DB + +
+ +
+ + + + + + + + + +
+ +
+ Or connect to an external datasource +
+ +
+ {#each integrations as [key, value]} + handleIntegrationSelect(key)} + title={value.friendlyName} + description={value.type} + {disabled} + > + + + {/each} +
+
+ + diff --git a/packages/builder/src/pages/builder/portal/apps/onboarding/_components/DataPanel.svelte b/packages/builder/src/pages/builder/portal/apps/onboarding/_components/DataPanel.svelte deleted file mode 100644 index 9a7fffd893..0000000000 --- a/packages/builder/src/pages/builder/portal/apps/onboarding/_components/DataPanel.svelte +++ /dev/null @@ -1,13 +0,0 @@ - - -
- - -
diff --git a/packages/builder/src/pages/builder/portal/apps/onboarding/_components/DatasourceConfigPanel.svelte b/packages/builder/src/pages/builder/portal/apps/onboarding/_components/DatasourceConfigPanel.svelte deleted file mode 100644 index 2b44648279..0000000000 --- a/packages/builder/src/pages/builder/portal/apps/onboarding/_components/DatasourceConfigPanel.svelte +++ /dev/null @@ -1,120 +0,0 @@ - - -
- -
- - {#each Object.entries(fields) as [name, { type, default: defaultValue, required }]} - {#if type !== "boolean"} - {}} - label={formatName(name)} - {type} - /> - {/if} - {/each} - {#each Object.entries(fields) as [name, { type, default: defaultValue, required }]} - {#if type === "boolean"} - - {/if} - {/each} - -
- {#if isGoogle} - - {:else} - - {/if} -
- - diff --git a/packages/builder/src/pages/builder/portal/apps/onboarding/_components/ExampleApp.svelte b/packages/builder/src/pages/builder/portal/apps/onboarding/_components/ExampleApp.svelte index 3e970ac360..0b290decbf 100644 --- a/packages/builder/src/pages/builder/portal/apps/onboarding/_components/ExampleApp.svelte +++ b/packages/builder/src/pages/builder/portal/apps/onboarding/_components/ExampleApp.svelte @@ -1,6 +1,5 @@ - - - -
- {#if stage === "name"} - (stage = "data")} /> - {:else if googleComplete} -
- Please login to your Google account in the new tab which as opened to - continue. -
- {:else if integrationsLoading || creationLoading} -
- -
- {:else if stage === "data"} - (stage = "name")}> -
- handleCreateApp({ useSampleData: true })} - > -
-
- -
- Budibase Sample data -
- -
-
- -
-
- -
- Upload data (CSV or JSON) -
- -
- {#each Object.entries(plusIntegrations) as [integrationType, schema]} -
- (stage = integrationType)}> -
-
- -
- {schema.friendlyName} -
- -
- {/each} - - {:else if stage in plusIntegrations} - (stage = "data")} - onNext={data => { - const isGoogle = data.isGoogle - delete data.isGoogle - return handleCreateApp({ datasourceConfig: data, isGoogle }) - }} - /> - {:else} -

There was an problem. Please refresh the page and try again.

- {/if} +
- +
@@ -258,35 +77,4 @@ .full-width { width: 100%; } - .centered { - display: flex; - justify-content: center; - align-items: center; - min-height: 400px; - } - - .dataButton { - margin-bottom: 12px; - } - - .dataButtonContent { - display: flex; - align-items: center; - } - - .budibaseLogo { - height: 20px; - } - - .dataButtonIcon { - width: 22px; - display: flex; - justify-content: center; - margin-right: 16px; - } - - .dataButtonContent :global(svg) { - font-size: 18px; - color: white; - } diff --git a/packages/builder/src/stores/backend/tables.js b/packages/builder/src/stores/backend/tables.js index ba900b7df9..a36c91d1b1 100644 --- a/packages/builder/src/stores/backend/tables.js +++ b/packages/builder/src/stores/backend/tables.js @@ -63,9 +63,7 @@ export function createTablesStore() { const savedTable = await API.saveTable(updatedTable) replaceTable(table._id, savedTable) - if (table.type === "external") { - await datasources.fetch() - } + await datasources.fetch() select(savedTable._id) return savedTable } diff --git a/packages/frontend-core/src/api/app.js b/packages/frontend-core/src/api/app.js index 7868eef063..ce18bcc0c5 100644 --- a/packages/frontend-core/src/api/app.js +++ b/packages/frontend-core/src/api/app.js @@ -152,4 +152,10 @@ export const buildAppEndpoints = API => ({ url: `/api/${appId}/components/definitions`, }) }, + + addSampleData: async appId => { + return await API.post({ + url: `/api/applications/${appId}/sample`, + }) + }, }) diff --git a/packages/server/src/api/controllers/application.ts b/packages/server/src/api/controllers/application.ts index 815366785c..9c89b48b8a 100644 --- a/packages/server/src/api/controllers/application.ts +++ b/packages/server/src/api/controllers/application.ts @@ -26,7 +26,10 @@ import { env as envCore, } from "@budibase/backend-core" import { USERS_TABLE_SCHEMA } from "../../constants" -import { buildDefaultDocs } from "../../db/defaultData/datasource_bb_default" +import { + DEFAULT_BB_DATASOURCE_ID, + buildDefaultDocs, +} from "../../db/defaultData/datasource_bb_default" import { removeAppFromUserRoles } from "../../utilities/workerRequests" import { stringToReadStream, isQsTrue } from "../../utilities" import { getLocksById, doesUserHaveLock } from "../../utilities/redis" @@ -111,11 +114,7 @@ function checkAppName( } } -async function createInstance( - appId: string, - template: any, - includeSampleData: boolean -) { +async function createInstance(appId: string, template: any) { const db = context.getAppDB() await db.put({ _id: "_design/database", @@ -142,21 +141,25 @@ async function createInstance( } else { // create the users table await db.put(USERS_TABLE_SCHEMA) - - if (includeSampleData) { - // create ootb stock db - await addDefaultTables(db) - } } return { _id: appId } } -async function addDefaultTables(db: Database) { - const defaultDbDocs = buildDefaultDocs() +export const addSampleData = async (ctx: UserCtx) => { + const db = context.getAppDB() - // add in the default db data docs - tables, datasource, rows and links - await db.bulkDocs([...defaultDbDocs]) + try { + // Check if default datasource exists before creating it + await sdk.datasources.get(DEFAULT_BB_DATASOURCE_ID) + } catch (err: any) { + const defaultDbDocs = buildDefaultDocs() + + // add in the default db data docs - tables, datasource, rows and links + await db.bulkDocs([...defaultDbDocs]) + } + + ctx.status = 200 } export async function fetch(ctx: UserCtx) { @@ -248,16 +251,11 @@ async function performAppCreate(ctx: UserCtx) { if (ctx.request.files && ctx.request.files.templateFile) { instanceConfig.file = ctx.request.files.templateFile } - const includeSampleData = isQsTrue(ctx.request.body.sampleData) const tenantId = tenancy.isMultiTenant() ? tenancy.getTenantId() : null const appId = generateDevAppID(generateAppID(tenantId)) return await context.doInAppContext(appId, async () => { - const instance = await createInstance( - appId, - instanceConfig, - includeSampleData - ) + const instance = await createInstance(appId, instanceConfig) const db = context.getAppDB() let newApplication: App = { diff --git a/packages/server/src/api/routes/application.ts b/packages/server/src/api/routes/application.ts index 0aa88568f3..0c1fa364ff 100644 --- a/packages/server/src/api/routes/application.ts +++ b/packages/server/src/api/routes/application.ts @@ -38,6 +38,11 @@ router authorized(permissions.BUILDER), controller.revertClient ) + .post( + "/api/applications/:appId/sample", + authorized(permissions.BUILDER), + controller.addSampleData + ) .post( "/api/applications/:appId/publish", authorized(permissions.BUILDER), From 3f8828086749696c01fe892215cef9af5f273bc8 Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Tue, 30 May 2023 12:02:25 +0000 Subject: [PATCH 30/31] Bump version to 2.6.19-alpha.30 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index bace077bb9..344e5ffffc 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.6.19-alpha.29", + "version": "2.6.19-alpha.30", "npmClient": "yarn", "packages": [ "packages/backend-core", From 6230e62b9be5c99df928591195181e8ce09217d1 Mon Sep 17 00:00:00 2001 From: Rory Powell Date: Thu, 25 May 2023 12:09:12 +0100 Subject: [PATCH 31/31] Automation investigation logging + max doc size fix for automation log --- .../backend-core/src/logging/pino/logger.ts | 1 + packages/pro | 2 +- packages/server/package.json | 1 + .../server/src/automations/logging/index.ts | 21 +++++++++ packages/server/src/environment.ts | 1 + packages/server/src/threads/automation.ts | 44 ++++++++++++------- packages/server/src/threads/index.ts | 6 ++- packages/server/src/threads/utils.ts | 2 + yarn.lock | 7 +++ 9 files changed, 68 insertions(+), 17 deletions(-) diff --git a/packages/backend-core/src/logging/pino/logger.ts b/packages/backend-core/src/logging/pino/logger.ts index cebc78ffc7..c96bc83e04 100644 --- a/packages/backend-core/src/logging/pino/logger.ts +++ b/packages/backend-core/src/logging/pino/logger.ts @@ -96,6 +96,7 @@ if (!env.DISABLE_PINO_LOGGER) { const mergingObject: any = { err: error, + pid: process.pid, ...contextObject, } diff --git a/packages/pro b/packages/pro index 2adc101c1e..86c32b80e0 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 2adc101c1ede13f861f282d702f45b94ab91fd41 +Subproject commit 86c32b80e08d2f19b57dcc2a3159667ac5a86c21 diff --git a/packages/server/package.json b/packages/server/package.json index 9f053401d8..cb53a1618e 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -100,6 +100,7 @@ "mssql": "6.2.3", "mysql2": "2.3.3", "node-fetch": "2.6.7", + "object-sizeof": "2.6.1", "open": "8.4.0", "openai": "^3.2.1", "pg": "8.10.0", diff --git a/packages/server/src/automations/logging/index.ts b/packages/server/src/automations/logging/index.ts index e3cc9d273c..fd47990488 100644 --- a/packages/server/src/automations/logging/index.ts +++ b/packages/server/src/automations/logging/index.ts @@ -2,6 +2,23 @@ import env from "../../environment" import { AutomationResults, Automation, App } from "@budibase/types" import { automations } from "@budibase/pro" import { db as dbUtils } from "@budibase/backend-core" +import sizeof from "object-sizeof" + +const MAX_LOG_SIZE_MB = 5 +const MB_IN_BYTES = 1024 * 1024 + +function sanitiseResults(results: AutomationResults) { + const message = `[removed] - max results size of ${MAX_LOG_SIZE_MB}MB exceeded` + for (let step of results.steps) { + step.inputs = { + message, + } + step.outputs = { + message, + success: step.outputs.success + } + } +} export async function storeLog( automation: Automation, @@ -11,6 +28,10 @@ export async function storeLog( if (env.DISABLE_AUTOMATION_LOGS) { return } + const bytes = sizeof(results) + if ((bytes / MB_IN_BYTES) > MAX_LOG_SIZE_MB) { + sanitiseResults(results) + } await automations.logs.storeLog(automation, results) } diff --git a/packages/server/src/environment.ts b/packages/server/src/environment.ts index 3e5e475360..0ba708e7bb 100644 --- a/packages/server/src/environment.ts +++ b/packages/server/src/environment.ts @@ -80,6 +80,7 @@ const environment = { ENABLE_ANALYTICS: process.env.ENABLE_ANALYTICS, SELF_HOSTED: process.env.SELF_HOSTED, HTTP_MB_LIMIT: process.env.HTTP_MB_LIMIT, + FORKED_PROCESS_NAME: process.env.FORKED_PROCESS_NAME || "main", // old CLIENT_ID: process.env.CLIENT_ID, _set(key: string, value: any) { diff --git a/packages/server/src/threads/automation.ts b/packages/server/src/threads/automation.ts index 9db647b693..7959f60724 100644 --- a/packages/server/src/threads/automation.ts +++ b/packages/server/src/threads/automation.ts @@ -19,6 +19,7 @@ import { AutomationStatus, AutomationMetadata, AutomationJob, + AutomationData, } from "@budibase/types" import { LoopStep, @@ -480,7 +481,16 @@ class Orchestrator { } // store the logs for the automation run - await storeLog(this._automation, this.executionOutput) + try { + await storeLog(this._automation, this.executionOutput) + } catch (e: any) { + if (e.status === 413 && e.request?.data) { + // if content is too large we shouldn't log it + delete e.request.data + e.request.data = { message: "removed due to large size" } + } + logging.logAlert("Error writing automation log", e) + } if (isProdAppID(this._appId) && isRecurring(automation) && metadata) { await this.updateMetadata(metadata) } @@ -488,24 +498,28 @@ class Orchestrator { } } -export function execute(job: Job, callback: WorkerCallback) { +export function execute(job: Job, callback: WorkerCallback) { const appId = job.data.event.appId + const automationId = job.data.automation._id if (!appId) { throw new Error("Unable to execute, event doesn't contain app ID.") } - return context.doInAppContext(appId, async () => { - const envVars = await sdkUtils.getEnvironmentVariables() - // put into automation thread for whole context - await context.doInEnvironmentContext(envVars, async () => { - const automationOrchestrator = new Orchestrator(job) - try { - const response = await automationOrchestrator.execute() - callback(null, response) - } catch (err) { - callback(err) - } - }) - }) + if (!automationId) { + throw new Error("Unable to execute, event doesn't contain automation ID.") + } + return context.doInAutomationContext({ appId, automationId, task: async () => { + const envVars = await sdkUtils.getEnvironmentVariables() + // put into automation thread for whole context + await context.doInEnvironmentContext(envVars, async () => { + const automationOrchestrator = new Orchestrator(job) + try { + const response = await automationOrchestrator.execute() + callback(null, response) + } catch (err) { + callback(err) + } + }) + }}) } export function executeSynchronously(job: Job) { diff --git a/packages/server/src/threads/index.ts b/packages/server/src/threads/index.ts index 9b6bffa867..6c03e5b464 100644 --- a/packages/server/src/threads/index.ts +++ b/packages/server/src/threads/index.ts @@ -38,6 +38,7 @@ export class Thread { this.count = opts.count ? opts.count : 1 this.disableThreading = this.shouldDisableThreading() if (!this.disableThreading) { + console.debug(`[${env.FORKED_PROCESS_NAME}] initialising worker farm type=${type}`) const workerOpts: any = { autoStart: true, maxConcurrentWorkers: this.count, @@ -45,6 +46,7 @@ export class Thread { env: { ...process.env, FORKED_PROCESS: "1", + FORKED_PROCESS_NAME: type, }, }, } @@ -54,6 +56,8 @@ export class Thread { } this.workers = workerFarm(workerOpts, typeToFile(type), ["execute"]) Thread.workerRefs.push(this.workers) + } else { + console.debug(`[${env.FORKED_PROCESS_NAME}] skipping worker farm type=${type}`) } } @@ -73,7 +77,7 @@ export class Thread { worker.execute(job, (err: any, response: any) => { if (err && err.type === "TimeoutError") { reject( - new Error(`Query response time exceeded ${timeout}ms timeout.`) + new Error(`Thread timeout exceeded ${timeout}ms timeout.`) ) } else if (err) { reject(err) diff --git a/packages/server/src/threads/utils.ts b/packages/server/src/threads/utils.ts index 37c7a6d3f9..a0f3bbdc47 100644 --- a/packages/server/src/threads/utils.ts +++ b/packages/server/src/threads/utils.ts @@ -26,8 +26,10 @@ function makeVariableKey(queryId: string, variable: string) { export function threadSetup() { // don't run this if not threading if (env.isTest() || env.DISABLE_THREADING || !env.isInThread()) { + console.debug(`[${env.FORKED_PROCESS_NAME}] thread setup skipped`) return } + console.debug(`[${env.FORKED_PROCESS_NAME}] thread setup running`) db.init() } diff --git a/yarn.lock b/yarn.lock index c5aad94d5b..c365222ce3 100644 --- a/yarn.lock +++ b/yarn.lock @@ -19043,6 +19043,13 @@ object-keys@~0.4.0: resolved "https://registry.yarnpkg.com/object-keys/-/object-keys-0.4.0.tgz#28a6aae7428dd2c3a92f3d95f21335dd204e0336" integrity sha512-ncrLw+X55z7bkl5PnUvHwFK9FcGuFYo9gtjws2XtSzL+aZ8tm830P60WJ0dSmFVaSalWieW5MD7kEdnXda9yJw== +object-sizeof@2.6.1: + version "2.6.1" + resolved "https://registry.yarnpkg.com/object-sizeof/-/object-sizeof-2.6.1.tgz#1e2b6a01d182c268dbb07ee3403f539de45f63d3" + integrity sha512-a7VJ1Zx7ZuHceKwjgfsSqzV/X0PVGvpZz7ho3Dn4Cs0LLcR5e5WuV+gsbizmplD8s0nAXMJmckKB2rkSiPm/Gg== + dependencies: + buffer "^6.0.3" + object-visit@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/object-visit/-/object-visit-1.0.1.tgz#f79c4493af0c5377b59fe39d395e41042dd045bb"