From 2475f17bd9f4a269b5d02d7c8822490ccb1052c2 Mon Sep 17 00:00:00 2001 From: Maurits Lourens Date: Fri, 5 May 2023 17:33:01 +0200 Subject: [PATCH 01/46] update icon name to Delete --- packages/bbui/src/Form/Core/Dropzone.svelte | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/bbui/src/Form/Core/Dropzone.svelte b/packages/bbui/src/Form/Core/Dropzone.svelte index 64d851b1e4..e9ee75bd8b 100644 --- a/packages/bbui/src/Form/Core/Dropzone.svelte +++ b/packages/bbui/src/Form/Core/Dropzone.svelte @@ -165,7 +165,7 @@ {/if} {#if !disabled}
- +
{/if} @@ -209,7 +209,7 @@ {/if} {#if !disabled}
- +
{/if} From 7f96fbf7411f65ce1dca980395c7498b29a9fe70 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Fri, 12 May 2023 13:55:08 +0100 Subject: [PATCH 02/46] Remove single user restriction and notify users if they are the primary builder or not --- packages/builder/src/components/start/AppRow.svelte | 6 ------ .../pages/builder/portal/overview/[appId]/_layout.svelte | 7 ------- packages/server/src/api/controllers/application.ts | 3 ++- packages/server/src/middleware/builder.ts | 9 ++++----- 4 files changed, 6 insertions(+), 19 deletions(-) diff --git a/packages/builder/src/components/start/AppRow.svelte b/packages/builder/src/components/start/AppRow.svelte index 34d083a096..e2eadd5503 100644 --- a/packages/builder/src/components/start/AppRow.svelte +++ b/packages/builder/src/components/start/AppRow.svelte @@ -15,12 +15,6 @@ } const goToBuilder = () => { - if (app.lockedOther) { - notifications.error( - `App locked by ${app.lockedBy.email}. Please allow lock to expire or have them unlock this app.` - ) - return - } $goto(`../../app/${app.devId}`) } diff --git a/packages/builder/src/pages/builder/portal/overview/[appId]/_layout.svelte b/packages/builder/src/pages/builder/portal/overview/[appId]/_layout.svelte index 806589e421..cc244852d8 100644 --- a/packages/builder/src/pages/builder/portal/overview/[appId]/_layout.svelte +++ b/packages/builder/src/pages/builder/portal/overview/[appId]/_layout.svelte @@ -80,13 +80,6 @@ } const editApp = () => { - if (appLocked && !lockedByYou) { - const identifier = app?.lockedBy?.firstName || app?.lockedBy?.email - notifications.warning( - `App locked by ${identifier}. Please allow lock to expire or have them unlock this app.` - ) - return - } $goto(`../../../app/${app.devId}`) } diff --git a/packages/server/src/api/controllers/application.ts b/packages/server/src/api/controllers/application.ts index ff0dbe015b..815366785c 100644 --- a/packages/server/src/api/controllers/application.ts +++ b/packages/server/src/api/controllers/application.ts @@ -29,7 +29,7 @@ import { USERS_TABLE_SCHEMA } from "../../constants" import { buildDefaultDocs } from "../../db/defaultData/datasource_bb_default" import { removeAppFromUserRoles } from "../../utilities/workerRequests" import { stringToReadStream, isQsTrue } from "../../utilities" -import { getLocksById } from "../../utilities/redis" +import { getLocksById, doesUserHaveLock } from "../../utilities/redis" import { updateClientLibrary, backupClientLibrary, @@ -227,6 +227,7 @@ export async function fetchAppPackage(ctx: UserCtx) { screens, layouts, clientLibPath, + hasLock: await doesUserHaveLock(application.appId, ctx.user), } } diff --git a/packages/server/src/middleware/builder.ts b/packages/server/src/middleware/builder.ts index 5174f618a0..ffb2e2c002 100644 --- a/packages/server/src/middleware/builder.ts +++ b/packages/server/src/middleware/builder.ts @@ -35,12 +35,11 @@ async function checkDevAppLocks(ctx: BBContext) { if (!appId || !appId.startsWith(APP_DEV_PREFIX)) { return } - if (!(await doesUserHaveLock(appId, ctx.user))) { - ctx.throw(400, "User does not hold app lock.") - } - // they do have lock, update it - await updateLock(appId, ctx.user) + // If this user already owns the lock, then update it + if (await doesUserHaveLock(appId, ctx.user)) { + await updateLock(appId, ctx.user) + } } async function updateAppUpdatedAt(ctx: BBContext) { From 5f81584a14390938b8a67d2449454a562fe70fc4 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Fri, 12 May 2023 14:16:10 +0100 Subject: [PATCH 03/46] Update top bar when not the primary builder and prevent flashing during loading states --- .../src/builderStore/store/frontend.js | 6 +- .../src/components/deploy/AppActions.svelte | 190 +++++++++--------- .../builder/app/[application]/_layout.svelte | 141 +++++++------ 3 files changed, 182 insertions(+), 155 deletions(-) diff --git a/packages/builder/src/builderStore/store/frontend.js b/packages/builder/src/builderStore/store/frontend.js index e264dc099b..33c82778e3 100644 --- a/packages/builder/src/builderStore/store/frontend.js +++ b/packages/builder/src/builderStore/store/frontend.js @@ -39,6 +39,7 @@ import { makePropSafe as safe } from "@budibase/string-templates" import { getComponentFieldOptions } from "helpers/formFields" const INITIAL_FRONTEND_STATE = { + initialised: false, apps: [], name: "", url: "", @@ -70,6 +71,7 @@ const INITIAL_FRONTEND_STATE = { previewDevice: "desktop", highlightedSettingKey: null, builderSidePanel: false, + hasLock: true, // URL params selectedScreenId: null, @@ -112,7 +114,7 @@ export const getFrontendStore = () => { store.set({ ...INITIAL_FRONTEND_STATE }) }, initialise: async pkg => { - const { layouts, screens, application, clientLibPath } = pkg + const { layouts, screens, application, clientLibPath, hasLock } = pkg await store.actions.components.refreshDefinitions(application.appId) @@ -137,6 +139,8 @@ export const getFrontendStore = () => { upgradableVersion: application.upgradableVersion, navigation: application.navigation || {}, usedPlugins: application.usedPlugins || [], + hasLock, + initialised: true, })) screenHistoryStore.reset() automationHistoryStore.reset() diff --git a/packages/builder/src/components/deploy/AppActions.svelte b/packages/builder/src/components/deploy/AppActions.svelte index daed564204..9813237317 100644 --- a/packages/builder/src/components/deploy/AppActions.svelte +++ b/packages/builder/src/components/deploy/AppActions.svelte @@ -113,109 +113,113 @@ }) -
-
-
- -
- - - {#if isPublished} -
-
- -
- -
- - Your published app - - - {processStringSync( - "Last published {{ duration time 'millisecond' }} ago", - { - time: - new Date().getTime() - - new Date(latestDeployments[0].updatedAt).getTime(), - } - )} - - -
- - -
- -
- +{#if $store.hasLock} +
+
+
+
- {/if} + - {#if !isPublished} - - {/if} + {#if isPublished} +
+
+ +
+ +
+ + Your published app + + + {processStringSync( + "Last published {{ duration time 'millisecond' }} ago", + { + time: + new Date().getTime() - + new Date(latestDeployments[0].updatedAt).getTime(), + } + )} + + +
+ + +
+ +
+ +
+ {/if} - - + {#if !isPublished} { - store.update(state => { - state.builderSidePanel = true - return state - }) - }} - > - Users - - - -
-
+ tooltip="Your app has not been published yet" + disabled + /> + {/if} - - Are you sure you want to unpublish the app {selectedApp?.name}? - + + + { + store.update(state => { + state.builderSidePanel = true + return state + }) + }} + > + Users + + + +
+
+ + + Are you sure you want to unpublish the app {selectedApp?.name}? + +{/if}
- + {#if $store.hasLock} + + {/if}
From d752448403a01b3777a839425af423f1105586de Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Fri, 12 May 2023 14:33:00 +0100 Subject: [PATCH 04/46] Disable command palette and tour for non lock holders and forcefully prevent access to the design and automate pages --- .../pages/builder/app/[application]/_layout.svelte | 12 +++++++++--- .../app/[application]/automate/_layout.svelte | 9 +++++++++ .../builder/app/[application]/design/_layout.svelte | 12 ++++++++++++ 3 files changed, 30 insertions(+), 3 deletions(-) diff --git a/packages/builder/src/pages/builder/app/[application]/_layout.svelte b/packages/builder/src/pages/builder/app/[application]/_layout.svelte index 434a6c7770..2fb1b29c18 100644 --- a/packages/builder/src/pages/builder/app/[application]/_layout.svelte +++ b/packages/builder/src/pages/builder/app/[application]/_layout.svelte @@ -13,7 +13,6 @@ Modal, notifications, } from "@budibase/bbui" - import AppActions from "components/deploy/AppActions.svelte" import { API } from "api" import { isActive, goto, layout, redirect } from "@roxi/routify" @@ -30,7 +29,9 @@ let promise = getPackage() let hasSynced = false let commandPaletteModal + let loaded = false + $: loaded && initTour() $: selected = capitalise( $layout.children.find(layout => $isActive(layout.path))?.title ?? "data" ) @@ -43,6 +44,7 @@ await automationStore.actions.fetch() await roles.fetch() await flags.fetch() + loaded = true return pkg } catch (error) { notifications.error(`Error initialising app: ${error?.message}`) @@ -67,13 +69,18 @@ // Event handler for the command palette const handleKeyDown = e => { - if (e.key === "k" && (e.ctrlKey || e.metaKey)) { + if (e.key === "k" && (e.ctrlKey || e.metaKey) && $store.hasLock) { e.preventDefault() commandPaletteModal.toggle() } } const initTour = async () => { + // Skip tour if we don't have the lock + if (!$store.hasLock) { + return + } + // Check if onboarding is enabled. if (isEnabled(TENANT_FEATURE_FLAGS.ONBOARDING_TOUR)) { if (!$auth.user?.onboardedAt) { @@ -110,7 +117,6 @@ // check if user has beta access // const betaResponse = await API.checkBetaAccess($auth?.user?.email) // betaAccess = betaResponse.access - initTour() } catch (error) { notifications.error("Failed to sync with production database") } diff --git a/packages/builder/src/pages/builder/app/[application]/automate/_layout.svelte b/packages/builder/src/pages/builder/app/[application]/automate/_layout.svelte index 74dfe671ab..79ca5df168 100644 --- a/packages/builder/src/pages/builder/app/[application]/automate/_layout.svelte +++ b/packages/builder/src/pages/builder/app/[application]/automate/_layout.svelte @@ -8,6 +8,15 @@ import { onDestroy, onMount } from "svelte" import { syncURLToState } from "helpers/urlStateSync" import * as routify from "@roxi/routify" + import { store } from "builderStore" + import { redirect } from "@roxi/routify" + + // Prevent access for other users than the lock holder + $: { + if (!$store.hasLock) { + $redirect("../data") + } + } // Keep URL and state in sync for selected screen ID const stopSyncing = syncURLToState({ diff --git a/packages/builder/src/pages/builder/app/[application]/design/_layout.svelte b/packages/builder/src/pages/builder/app/[application]/design/_layout.svelte index ec21d909aa..d23514ae6d 100644 --- a/packages/builder/src/pages/builder/app/[application]/design/_layout.svelte +++ b/packages/builder/src/pages/builder/app/[application]/design/_layout.svelte @@ -1,2 +1,14 @@ + + From f8f970bf7ed82f32291890909ed97f7723205948 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Fri, 12 May 2023 16:13:32 +0100 Subject: [PATCH 05/46] Update grid websocket to send actual changes down to reduce API load --- .../src/components/grid/layout/Grid.svelte | 5 ++- .../src/components/grid/lib/websocket.js | 2 +- .../src/components/grid/stores/rows.js | 24 ++++++++---- .../server/src/api/controllers/row/index.ts | 37 ++++++++----------- .../server/src/api/controllers/row/utils.ts | 12 ++++++ packages/server/src/websockets/grid.ts | 12 ++++++ 6 files changed, 59 insertions(+), 33 deletions(-) diff --git a/packages/frontend-core/src/components/grid/layout/Grid.svelte b/packages/frontend-core/src/components/grid/layout/Grid.svelte index 2035ec4d39..3b0e74d882 100644 --- a/packages/frontend-core/src/components/grid/layout/Grid.svelte +++ b/packages/frontend-core/src/components/grid/layout/Grid.svelte @@ -1,5 +1,5 @@
{ }) socket.on("row-update", data => { if (data.id) { - rows.actions.refreshRow(data.id) + rows.actions.replaceRow(data.id, data.row) } }) socket.on("user-update", user => { diff --git a/packages/frontend-core/src/components/grid/stores/rows.js b/packages/frontend-core/src/components/grid/stores/rows.js index b6dc8c05d0..ee5ac6c1d2 100644 --- a/packages/frontend-core/src/components/grid/stores/rows.js +++ b/packages/frontend-core/src/components/grid/stores/rows.js @@ -268,27 +268,25 @@ export const deriveStores = context => { return res?.rows?.[0] } - // Refreshes a specific row, handling updates, addition or deletion - const refreshRow = async id => { - // Fetch row from the server again - const newRow = await fetchRow(id) - + // Replaces a row in state with the newly defined row, handling updates, + // addition and deletion + const replaceRow = (id, row) => { // Get index of row to check if it exists const $rows = get(rows) const $rowLookupMap = get(rowLookupMap) const index = $rowLookupMap[id] // Process as either an update, addition or deletion - if (newRow) { + if (row) { if (index != null) { // An existing row was updated rows.update(state => { - state[index] = { ...newRow } + state[index] = { ...row } return state }) } else { // A new row was created - handleNewRows([newRow]) + handleNewRows([row]) } } else if (index != null) { // A row was removed @@ -296,6 +294,15 @@ export const deriveStores = context => { } } + // Refreshes a specific row + const refreshRow = async id => { + // Fetch row from the server again + const row = await fetchRow(id) + + // Update local state + replaceRow(id, row) + } + // Refreshes all data const refreshData = () => { get(fetch)?.getInitialData() @@ -455,6 +462,7 @@ export const deriveStores = context => { hasRow, loadNextPage, refreshRow, + replaceRow, refreshData, refreshTableDefinition, }, diff --git a/packages/server/src/api/controllers/row/index.ts b/packages/server/src/api/controllers/row/index.ts index 349c4e72e7..55d2d27cce 100644 --- a/packages/server/src/api/controllers/row/index.ts +++ b/packages/server/src/api/controllers/row/index.ts @@ -4,6 +4,7 @@ import * as external from "./external" import { isExternalTable } from "../../../integrations/utils" import { Ctx } from "@budibase/types" import * as utils from "./utils" +import { gridSocket } from "../../../websockets" function pickApi(tableId: any) { if (isExternalTable(tableId)) { @@ -12,21 +13,9 @@ function pickApi(tableId: any) { return internal } -function getTableId(ctx: any) { - if (ctx.request.body && ctx.request.body.tableId) { - return ctx.request.body.tableId - } - if (ctx.params && ctx.params.tableId) { - return ctx.params.tableId - } - if (ctx.params && ctx.params.viewName) { - return ctx.params.viewName - } -} - export async function patch(ctx: any): Promise { const appId = ctx.appId - const tableId = getTableId(ctx) + const tableId = utils.getTableId(ctx) const body = ctx.request.body // if it doesn't have an _id then its save if (body && !body._id) { @@ -47,6 +36,7 @@ export async function patch(ctx: any): Promise { ctx.eventEmitter.emitRow(`row:update`, appId, row, table) ctx.message = `${table.name} updated successfully.` ctx.body = row + gridSocket?.emitRowUpdate(ctx, row) } catch (err) { ctx.throw(400, err) } @@ -54,7 +44,7 @@ export async function patch(ctx: any): Promise { export const save = async (ctx: any) => { const appId = ctx.appId - const tableId = getTableId(ctx) + const tableId = utils.getTableId(ctx) const body = ctx.request.body // if it has an ID already then its a patch if (body && body._id) { @@ -69,23 +59,24 @@ export const save = async (ctx: any) => { ctx.eventEmitter && ctx.eventEmitter.emitRow(`row:save`, appId, row, table) ctx.message = `${table.name} saved successfully` ctx.body = row + gridSocket?.emitRowUpdate(ctx, row) } export async function fetchView(ctx: any) { - const tableId = getTableId(ctx) + const tableId = utils.getTableId(ctx) ctx.body = await quotas.addQuery(() => pickApi(tableId).fetchView(ctx), { datasourceId: tableId, }) } export async function fetch(ctx: any) { - const tableId = getTableId(ctx) + const tableId = utils.getTableId(ctx) ctx.body = await quotas.addQuery(() => pickApi(tableId).fetch(ctx), { datasourceId: tableId, }) } export async function find(ctx: any) { - const tableId = getTableId(ctx) + const tableId = utils.getTableId(ctx) ctx.body = await quotas.addQuery(() => pickApi(tableId).find(ctx), { datasourceId: tableId, }) @@ -94,7 +85,7 @@ export async function find(ctx: any) { export async function destroy(ctx: any) { const appId = ctx.appId const inputs = ctx.request.body - const tableId = getTableId(ctx) + const tableId = utils.getTableId(ctx) let response, row if (inputs.rows) { let { rows } = await quotas.addQuery( @@ -107,6 +98,7 @@ export async function destroy(ctx: any) { response = rows for (let row of rows) { ctx.eventEmitter && ctx.eventEmitter.emitRow(`row:delete`, appId, row) + gridSocket?.emitRowDeletion(ctx, row._id) } } else { let resp = await quotas.addQuery(() => pickApi(tableId).destroy(ctx), { @@ -116,6 +108,7 @@ export async function destroy(ctx: any) { response = resp.response row = resp.row ctx.eventEmitter && ctx.eventEmitter.emitRow(`row:delete`, appId, row) + gridSocket?.emitRowDeletion(ctx, row._id) } ctx.status = 200 // for automations include the row that was deleted @@ -124,7 +117,7 @@ export async function destroy(ctx: any) { } export async function search(ctx: any) { - const tableId = getTableId(ctx) + const tableId = utils.getTableId(ctx) ctx.status = 200 ctx.body = await quotas.addQuery(() => pickApi(tableId).search(ctx), { datasourceId: tableId, @@ -132,7 +125,7 @@ export async function search(ctx: any) { } export async function validate(ctx: Ctx) { - const tableId = getTableId(ctx) + const tableId = utils.getTableId(ctx) // external tables are hard to validate currently if (isExternalTable(tableId)) { ctx.body = { valid: true } @@ -145,7 +138,7 @@ export async function validate(ctx: Ctx) { } export async function fetchEnrichedRow(ctx: any) { - const tableId = getTableId(ctx) + const tableId = utils.getTableId(ctx) ctx.body = await quotas.addQuery( () => pickApi(tableId).fetchEnrichedRow(ctx), { @@ -155,7 +148,7 @@ export async function fetchEnrichedRow(ctx: any) { } export const exportRows = async (ctx: any) => { - const tableId = getTableId(ctx) + const tableId = utils.getTableId(ctx) ctx.body = await quotas.addQuery(() => pickApi(tableId).exportRows(ctx), { datasourceId: tableId, }) diff --git a/packages/server/src/api/controllers/row/utils.ts b/packages/server/src/api/controllers/row/utils.ts index f6a87dd24c..f1edbf538b 100644 --- a/packages/server/src/api/controllers/row/utils.ts +++ b/packages/server/src/api/controllers/row/utils.ts @@ -154,3 +154,15 @@ export function cleanExportRows( return cleanRows } + +export function getTableId(ctx: any) { + if (ctx.request.body && ctx.request.body.tableId) { + return ctx.request.body.tableId + } + if (ctx.params && ctx.params.tableId) { + return ctx.params.tableId + } + if (ctx.params && ctx.params.viewName) { + return ctx.params.viewName + } +} diff --git a/packages/server/src/websockets/grid.ts b/packages/server/src/websockets/grid.ts index 886784cd2c..bb23ec8e18 100644 --- a/packages/server/src/websockets/grid.ts +++ b/packages/server/src/websockets/grid.ts @@ -3,6 +3,8 @@ import Socket from "./websocket" import { permissions } from "@budibase/backend-core" import http from "http" import Koa from "koa" +import { getTableId } from "../api/controllers/row/utils" +import { Row } from "@budibase/types" export default class GridSocket extends Socket { constructor(app: Koa, server: http.Server) { @@ -52,4 +54,14 @@ export default class GridSocket extends Socket { }) }) } + + emitRowUpdate(ctx: any, row: Row) { + const tableId = getTableId(ctx) + this.io.in(tableId).emit("row-update", { id: row._id, row }) + } + + emitRowDeletion(ctx: any, id: string) { + const tableId = getTableId(ctx) + this.io.in(tableId).emit("row-update", { id, row: null }) + } } From 46d8ad286489a38321cc8cc2fe205321204e00fe Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Mon, 15 May 2023 14:51:54 +0100 Subject: [PATCH 06/46] Restore proper functionality for showing other users inside grids --- .../src/components/grid/cells/GridCell.svelte | 12 +++++++++--- .../src/components/grid/layout/NewRow.svelte | 2 +- .../src/components/grid/stores/users.js | 4 ++-- packages/server/src/websockets/grid.ts | 2 +- 4 files changed, 13 insertions(+), 7 deletions(-) diff --git a/packages/frontend-core/src/components/grid/cells/GridCell.svelte b/packages/frontend-core/src/components/grid/cells/GridCell.svelte index 6589c18d07..d5e96c3ec4 100644 --- a/packages/frontend-core/src/components/grid/cells/GridCell.svelte +++ b/packages/frontend-core/src/components/grid/cells/GridCell.svelte @@ -112,10 +112,10 @@ z-index: 2; } .cell.focused { - --cell-color: var(--spectrum-global-color-blue-400); + --cell-color: var(--spectrum-global-color-blue-400) !important; } .cell.error { - --cell-color: var(--spectrum-global-color-red-500); + --cell-color: var(--spectrum-global-color-red-500) !important; } .cell.readonly { --cell-color: var(--spectrum-global-color-gray-600); @@ -141,7 +141,7 @@ left: 0; padding: 1px 4px 3px 4px; margin: 0 0 -2px 0; - background: var(--user-color); + background: var(--cell-color); border-radius: 2px; display: block; color: white; @@ -162,4 +162,10 @@ .error .label { background: var(--spectrum-global-color-red-500); } + .selected-other:not(.error) .label { + display: none; + } + .selected-other:not(.error):hover .label { + display: block; + } diff --git a/packages/frontend-core/src/components/grid/layout/NewRow.svelte b/packages/frontend-core/src/components/grid/layout/NewRow.svelte index 8048a4e2fa..97a31e743a 100644 --- a/packages/frontend-core/src/components/grid/layout/NewRow.svelte +++ b/packages/frontend-core/src/components/grid/layout/NewRow.svelte @@ -193,7 +193,7 @@ row={newRow} focused={$focusedCellId === cellId} width={column.width} - rowIdx={0} + rowIdx={offset === 0 ? 0 : 1} invertX={columnIdx >= $columnHorizontalInversionIndex} {invertY} > diff --git a/packages/frontend-core/src/components/grid/stores/users.js b/packages/frontend-core/src/components/grid/stores/users.js index 3a6ec5fb21..4cf4bdac52 100644 --- a/packages/frontend-core/src/components/grid/stores/users.js +++ b/packages/frontend-core/src/components/grid/stores/users.js @@ -85,11 +85,11 @@ export const deriveStores = context => { const updateUser = user => { const $users = get(users) - const index = $users.findIndex(x => x.id === user.id) - if (index === -1) { + if (!$users.some(x => x.id === user.id)) { users.set([...$users, user]) } else { users.update(state => { + const index = state.findIndex(x => x.id === user.id) state[index] = user return state.slice() }) diff --git a/packages/server/src/websockets/grid.ts b/packages/server/src/websockets/grid.ts index bb23ec8e18..4fb81e006c 100644 --- a/packages/server/src/websockets/grid.ts +++ b/packages/server/src/websockets/grid.ts @@ -40,7 +40,7 @@ export default class GridSocket extends Socket { // Handle users selecting a new cell socket.on("select-cell", cellId => { - socket.data.user.selectedCellId = cellId + socket.data.user.focusedCellId = cellId if (currentRoom) { socket.to(currentRoom).emit("user-update", socket.data.user) } From e2a860ea4c1b798464bb02c6b839e115e06e50ac Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Mon, 15 May 2023 16:43:37 +0100 Subject: [PATCH 07/46] Fix visual issues with other users selecting cells --- .../src/components/grid/cells/GridCell.svelte | 9 +++++---- .../frontend-core/src/components/grid/stores/rows.js | 3 --- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/packages/frontend-core/src/components/grid/cells/GridCell.svelte b/packages/frontend-core/src/components/grid/cells/GridCell.svelte index d5e96c3ec4..3ac9a3a399 100644 --- a/packages/frontend-core/src/components/grid/cells/GridCell.svelte +++ b/packages/frontend-core/src/components/grid/cells/GridCell.svelte @@ -94,14 +94,15 @@ } /* Cell border for cells with labels */ - .cell.error:after, - .cell.selected-other:not(.focused):after { + .cell.error:after { border-radius: 0 2px 2px 2px; } - .cell[data-row="0"].error:after, - .cell[data-row="0"].selected-other:not(.focused):after { + .cell[data-row="0"].error:after { border-radius: 2px 2px 2px 0; } + .cell.selected-other:not(.focused):after { + border-radius: 2px; + } /* Cell z-index */ .cell.error, diff --git a/packages/frontend-core/src/components/grid/stores/rows.js b/packages/frontend-core/src/components/grid/stores/rows.js index ee5ac6c1d2..f6c709b675 100644 --- a/packages/frontend-core/src/components/grid/stores/rows.js +++ b/packages/frontend-core/src/components/grid/stores/rows.js @@ -296,10 +296,7 @@ export const deriveStores = context => { // Refreshes a specific row const refreshRow = async id => { - // Fetch row from the server again const row = await fetchRow(id) - - // Update local state replaceRow(id, row) } From 913188052ab6ae447bacf060caa30937c2be46c6 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Tue, 16 May 2023 08:18:26 +0100 Subject: [PATCH 08/46] Fix cell colours with multiple users --- .../src/components/grid/cells/GridCell.svelte | 27 ++++++++++++------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/packages/frontend-core/src/components/grid/cells/GridCell.svelte b/packages/frontend-core/src/components/grid/cells/GridCell.svelte index 3ac9a3a399..3d4f2a37bc 100644 --- a/packages/frontend-core/src/components/grid/cells/GridCell.svelte +++ b/packages/frontend-core/src/components/grid/cells/GridCell.svelte @@ -15,7 +15,7 @@ const getStyle = (width, selectedUser) => { let style = `flex: 0 0 ${width}px;` if (selectedUser) { - style += `--cell-color:${selectedUser.color};` + style += `--user-color:${selectedUser.color};` } return style } @@ -112,14 +112,8 @@ .cell.focused { z-index: 2; } - .cell.focused { - --cell-color: var(--spectrum-global-color-blue-400) !important; - } - .cell.error { - --cell-color: var(--spectrum-global-color-red-500) !important; - } - .cell.readonly { - --cell-color: var(--spectrum-global-color-gray-600); + .cell.selected-other:hover { + z-index: 2; } .cell:not(.focused) { user-select: none; @@ -127,6 +121,21 @@ .cell:hover { cursor: default; } + + /* Cell color overrides */ + .cell.selected-other { + --cell-color: var(--user-color); + } + .cell.focused { + --cell-color: var(--spectrum-global-color-blue-400); + } + .cell.error { + --cell-color: var(--spectrum-global-color-red-500); + } + .cell.focused.readonly { + --cell-color: var(--spectrum-global-color-gray-600); + } + .cell.highlighted:not(.focused), .cell.focused.readonly { --cell-background: var(--cell-background-hover); From cb9d35f6aafb0b1c2688e596c0fe5dfc2999ab6b Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Tue, 16 May 2023 08:24:23 +0100 Subject: [PATCH 09/46] Fix validation error position for new rows --- .../frontend-core/src/components/grid/cells/GridCell.svelte | 1 - packages/frontend-core/src/components/grid/layout/NewRow.svelte | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/frontend-core/src/components/grid/cells/GridCell.svelte b/packages/frontend-core/src/components/grid/cells/GridCell.svelte index 3d4f2a37bc..63446a858c 100644 --- a/packages/frontend-core/src/components/grid/cells/GridCell.svelte +++ b/packages/frontend-core/src/components/grid/cells/GridCell.svelte @@ -165,7 +165,6 @@ .cell[data-row="0"] .label { bottom: auto; top: 100%; - border-radius: 0 2px 2px 2px; padding: 2px 4px 2px 4px; margin: -2px 0 0 0; } diff --git a/packages/frontend-core/src/components/grid/layout/NewRow.svelte b/packages/frontend-core/src/components/grid/layout/NewRow.svelte index 97a31e743a..12bab9c02c 100644 --- a/packages/frontend-core/src/components/grid/layout/NewRow.svelte +++ b/packages/frontend-core/src/components/grid/layout/NewRow.svelte @@ -167,7 +167,7 @@ focused={$focusedCellId === cellId} width={$stickyColumn.width} {updateValue} - rowIdx={0} + rowIdx={offset === 0 ? 0 : 1} {invertY} > {#if $stickyColumn?.schema?.autocolumn} From 6ed691be1d7b98de374293c73dffa82251c5f67b Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Tue, 16 May 2023 14:18:31 +0100 Subject: [PATCH 10/46] Improve builder socket --- packages/builder/src/builderStore/index.js | 2 + .../src/builderStore/store/frontend.js | 7 +++ .../builder/src/builderStore/store/users.js | 10 +++++ .../builder/src/builderStore/websocket.js | 26 +++++++++++ .../_components/UserAvatars.svelte | 33 ++++++++++++++ .../builder/app/[application]/_layout.svelte | 7 ++- .../src/components/grid/layout/Grid.svelte | 4 +- .../components/grid/layout/UserAvatars.svelte | 2 +- .../src/components/grid/lib/websocket.js | 21 ++------- .../src/components/grid/stores/users.js | 29 ------------- packages/frontend-core/src/utils/index.js | 1 + packages/frontend-core/src/utils/websocket.js | 23 ++++++++++ packages/server/src/websockets/builder.ts | 43 +++++++++++++++++++ packages/server/src/websockets/index.ts | 5 ++- packages/server/src/websockets/websocket.ts | 31 +++++++++++-- 15 files changed, 185 insertions(+), 59 deletions(-) create mode 100644 packages/builder/src/builderStore/store/users.js create mode 100644 packages/builder/src/builderStore/websocket.js create mode 100644 packages/builder/src/pages/builder/app/[application]/_components/UserAvatars.svelte create mode 100644 packages/frontend-core/src/utils/websocket.js create mode 100644 packages/server/src/websockets/builder.ts diff --git a/packages/builder/src/builderStore/index.js b/packages/builder/src/builderStore/index.js index d15cdb6e98..8183c75a20 100644 --- a/packages/builder/src/builderStore/index.js +++ b/packages/builder/src/builderStore/index.js @@ -2,6 +2,7 @@ import { getFrontendStore } from "./store/frontend" import { getAutomationStore } from "./store/automation" import { getTemporalStore } from "./store/temporal" import { getThemeStore } from "./store/theme" +import { getUsersStore } from "./store/users" import { derived } from "svelte/store" import { findComponent, findComponentPath } from "./componentUtils" import { RoleUtils } from "@budibase/frontend-core" @@ -12,6 +13,7 @@ export const store = getFrontendStore() export const automationStore = getAutomationStore() export const themeStore = getThemeStore() export const temporalStore = getTemporalStore() +export const usersStore = getUsersStore() // Setup history for screens export const screenHistoryStore = createHistoryStore({ diff --git a/packages/builder/src/builderStore/store/frontend.js b/packages/builder/src/builderStore/store/frontend.js index 33c82778e3..0a27f576a3 100644 --- a/packages/builder/src/builderStore/store/frontend.js +++ b/packages/builder/src/builderStore/store/frontend.js @@ -37,6 +37,7 @@ import { } from "builderStore/dataBinding" import { makePropSafe as safe } from "@budibase/string-templates" import { getComponentFieldOptions } from "helpers/formFields" +import { createBuilderWebsocket } from "builderStore/websocket" const INITIAL_FRONTEND_STATE = { initialised: false, @@ -84,10 +85,14 @@ const INITIAL_FRONTEND_STATE = { // Onboarding onboarding: false, tourNodes: null, + + // Multi user collab + users: [], } export const getFrontendStore = () => { const store = writable({ ...INITIAL_FRONTEND_STATE }) + let websocket // This is a fake implementation of a "patch" API endpoint to try and prevent // 409s. All screen doc mutations (aside from creation) use this function, @@ -112,9 +117,11 @@ export const getFrontendStore = () => { store.actions = { reset: () => { store.set({ ...INITIAL_FRONTEND_STATE }) + websocket?.disconnect() }, initialise: async pkg => { const { layouts, screens, application, clientLibPath, hasLock } = pkg + websocket = createBuilderWebsocket() await store.actions.components.refreshDefinitions(application.appId) diff --git a/packages/builder/src/builderStore/store/users.js b/packages/builder/src/builderStore/store/users.js new file mode 100644 index 0000000000..dfb592c1c5 --- /dev/null +++ b/packages/builder/src/builderStore/store/users.js @@ -0,0 +1,10 @@ +import { writable } from "svelte/store" + +export const getUsersStore = () => { + const initialValue = { + users: [], + } + const store = writable(initialValue) + + return store +} diff --git a/packages/builder/src/builderStore/websocket.js b/packages/builder/src/builderStore/websocket.js new file mode 100644 index 0000000000..d73ab08f9a --- /dev/null +++ b/packages/builder/src/builderStore/websocket.js @@ -0,0 +1,26 @@ +import { createWebsocket } from "@budibase/frontend-core" +import { store } from "builderStore" + +export const createBuilderWebsocket = () => { + const socket = createWebsocket("/socket/builder") + + socket.on("connect", () => { + socket.emit("get-users", null, response => { + console.log("conntected!", response.users) + store.update(state => ({ + ...state, + users: response.users, + })) + }) + }) + + socket.on("user-update", user => {}) + + socket.on("user-disconnect", user => {}) + + socket.on("connect_error", err => { + console.log("Failed to connect to builder websocket:", err.message) + }) + + return socket +} diff --git a/packages/builder/src/pages/builder/app/[application]/_components/UserAvatars.svelte b/packages/builder/src/pages/builder/app/[application]/_components/UserAvatars.svelte new file mode 100644 index 0000000000..a99c4bebee --- /dev/null +++ b/packages/builder/src/pages/builder/app/[application]/_components/UserAvatars.svelte @@ -0,0 +1,33 @@ + + +
+ {#each users as user} +
+ {user.email[0]} +
+ {/each} +
+ + diff --git a/packages/builder/src/pages/builder/app/[application]/_layout.svelte b/packages/builder/src/pages/builder/app/[application]/_layout.svelte index 2fb1b29c18..e145466b11 100644 --- a/packages/builder/src/pages/builder/app/[application]/_layout.svelte +++ b/packages/builder/src/pages/builder/app/[application]/_layout.svelte @@ -22,6 +22,7 @@ import TourWrap from "components/portal/onboarding/TourWrap.svelte" import TourPopover from "components/portal/onboarding/TourPopover.svelte" import BuilderSidePanel from "./_components/BuilderSidePanel.svelte" + import UserAvatars from "./_components/UserAvatars.svelte" import { TOUR_KEYS, TOURS } from "components/portal/onboarding/tours.js" export let application @@ -125,10 +126,7 @@ }) onDestroy(() => { - store.update(state => { - state.appId = null - return state - }) + store.actions.reset() }) @@ -211,6 +209,7 @@ {/if}
+
{/if} diff --git a/packages/frontend-core/src/components/grid/layout/Grid.svelte b/packages/frontend-core/src/components/grid/layout/Grid.svelte index 3b0e74d882..4d1ca1434f 100644 --- a/packages/frontend-core/src/components/grid/layout/Grid.svelte +++ b/packages/frontend-core/src/components/grid/layout/Grid.svelte @@ -24,7 +24,7 @@ import RowHeightButton from "../controls/RowHeightButton.svelte" import ColumnWidthButton from "../controls/ColumnWidthButton.svelte" import NewRow from "./NewRow.svelte" - import { createWebsocket } from "../lib/websocket" + import { createGridWebsocket } from "../lib/websocket" import { MaxCellRenderHeight, MaxCellRenderWidthOverflow, @@ -98,7 +98,7 @@ export const getContext = () => context // Initialise websocket for multi-user - onMount(() => createWebsocket(context)) + onMount(() => createGridWebsocket(context))
diff --git a/packages/frontend-core/src/components/grid/lib/websocket.js b/packages/frontend-core/src/components/grid/lib/websocket.js index ae67cd714d..7ac34b5a95 100644 --- a/packages/frontend-core/src/components/grid/lib/websocket.js +++ b/packages/frontend-core/src/components/grid/lib/websocket.js @@ -1,24 +1,9 @@ import { get } from "svelte/store" -import { io } from "socket.io-client" +import { createWebsocket } from "../../../utils" -export const createWebsocket = context => { +export const createGridWebsocket = context => { const { rows, tableId, users, userId, focusedCellId } = context - - // Determine connection info - const tls = location.protocol === "https:" - const proto = tls ? "wss:" : "ws:" - const host = location.hostname - const port = location.port || (tls ? 443 : 80) - const socket = io(`${proto}//${host}:${port}`, { - path: "/socket/grid", - // Cap reconnection attempts to 3 (total of 15 seconds before giving up) - reconnectionAttempts: 3, - // Delay reconnection attempt by 5 seconds - reconnectionDelay: 5000, - reconnectionDelayMax: 5000, - // Timeout after 4 seconds so we never stack requests - timeout: 4000, - }) + const socket = createWebsocket("/socket/grid") const connectToTable = tableId => { if (!socket.connected) { diff --git a/packages/frontend-core/src/components/grid/stores/users.js b/packages/frontend-core/src/components/grid/stores/users.js index 4cf4bdac52..5190de70da 100644 --- a/packages/frontend-core/src/components/grid/stores/users.js +++ b/packages/frontend-core/src/components/grid/stores/users.js @@ -21,35 +21,6 @@ export const createStores = () => { return 0 } }) - // Enrich users with colors - .map((user, idx) => { - // Generate random colour hue - let hue = 1 - for (let i = 0; i < user.email.length && i < 5; i++) { - hue *= user.email.charCodeAt(i + 1) - hue /= 17 - } - hue = hue % 360 - const color = - idx === 0 - ? "var(--spectrum-global-color-blue-400)" - : `hsl(${hue}, 50%, 40%)` - - // Generate friendly label - let label = user.email - if (user.firstName) { - label = user.firstName - if (user.lastName) { - label += ` ${user.lastName}` - } - } - - return { - ...user, - color, - label, - } - }) ) }, [] diff --git a/packages/frontend-core/src/utils/index.js b/packages/frontend-core/src/utils/index.js index 6ac7c65c62..16dc4f323a 100644 --- a/packages/frontend-core/src/utils/index.js +++ b/packages/frontend-core/src/utils/index.js @@ -3,3 +3,4 @@ export * as JSONUtils from "./json" export * as CookieUtils from "./cookies" export * as RoleUtils from "./roles" export * as Utils from "./utils" +export { createWebsocket } from "./websocket" diff --git a/packages/frontend-core/src/utils/websocket.js b/packages/frontend-core/src/utils/websocket.js new file mode 100644 index 0000000000..85c09acc14 --- /dev/null +++ b/packages/frontend-core/src/utils/websocket.js @@ -0,0 +1,23 @@ +import { io } from "socket.io-client" + +export const createWebsocket = path => { + if (!path) { + throw "A websocket path must be provided" + } + + // Determine connection info + const tls = location.protocol === "https:" + const proto = tls ? "wss:" : "ws:" + const host = location.hostname + const port = location.port || (tls ? 443 : 80) + return io(`${proto}//${host}:${port}`, { + path, + // Cap reconnection attempts to 3 (total of 15 seconds before giving up) + reconnectionAttempts: 3, + // Delay reconnection attempt by 5 seconds + reconnectionDelay: 5000, + reconnectionDelayMax: 5000, + // Timeout after 4 seconds so we never stack requests + timeout: 4000, + }) +} diff --git a/packages/server/src/websockets/builder.ts b/packages/server/src/websockets/builder.ts new file mode 100644 index 0000000000..5c23682a7b --- /dev/null +++ b/packages/server/src/websockets/builder.ts @@ -0,0 +1,43 @@ +import authorized from "../middleware/authorized" +import Socket from "./websocket" +import { permissions } from "@budibase/backend-core" +import http from "http" +import Koa from "koa" +import { Table } from "@budibase/types" + +export default class BuilderSocket extends Socket { + constructor(app: Koa, server: http.Server) { + super(app, server, "/socket/builder", [authorized(permissions.BUILDER)]) + + this.io.on("connection", socket => { + // Join a room for this app + const user = socket.data.user + const appId = socket.data.appId + socket.join(appId) + socket.to(appId).emit("user-update", socket.data.user) + console.log(`Builder user connected: ${user?.id}`) + + // Initial identification of connected spreadsheet + socket.on("get-users", async (payload, callback) => { + const sockets = await this.io.in(appId).fetchSockets() + callback({ + users: sockets.map(socket => socket.data.user), + id: user.id, + }) + }) + + // Disconnection cleanup + socket.on("disconnect", () => { + socket.to(appId).emit("user-disconnect", socket.data.user) + }) + }) + } + + emitTableUpdate(ctx: any, table: Table) { + this.io.in(ctx.appId).emit("table-update", { id: table._id, table }) + } + + emitTableDeletion(ctx: any, id: string) { + this.io.in(ctx.appId).emit("table-update", { id, table: null }) + } +} diff --git a/packages/server/src/websockets/index.ts b/packages/server/src/websockets/index.ts index 60cdfb8aed..2761c4d9da 100644 --- a/packages/server/src/websockets/index.ts +++ b/packages/server/src/websockets/index.ts @@ -1,14 +1,17 @@ import http from "http" import Koa from "koa" -import GridSocket from "./grid" import ClientAppSocket from "./client" +import GridSocket from "./grid" +import BuilderSocket from "./builder" let clientAppSocket: ClientAppSocket let gridSocket: GridSocket +let builderSocket: BuilderSocket export const initialise = (app: Koa, server: http.Server) => { clientAppSocket = new ClientAppSocket(app, server) gridSocket = new GridSocket(app, server) + builderSocket = new BuilderSocket(app, server) } export { clientAppSocket, gridSocket } diff --git a/packages/server/src/websockets/websocket.ts b/packages/server/src/websockets/websocket.ts index 1b34168f14..0e0e6d59a4 100644 --- a/packages/server/src/websockets/websocket.ts +++ b/packages/server/src/websockets/websocket.ts @@ -59,13 +59,36 @@ export default class Socket { for (let [idx, middleware] of middlewares.entries()) { await middleware(ctx, () => { if (idx === middlewares.length - 1) { - // Middlewares are finished. + // Middlewares are finished // Extract some data from our enriched koa context to persist // as metadata for the socket - socket.data.user = { - id: ctx.user._id, - email: ctx.user.email, + + // Add user info, including a deterministic color and friendly + // label + const { _id, email, firstName, lastName } = ctx.user + let hue = 1 + for (let i = 0; i < email.length && i < 5; i++) { + hue *= email.charCodeAt(i + 1) + hue /= 17 } + hue = hue % 360 + const color = `hsl(${hue}, 50%, 40%)` + let label = email + if (firstName) { + label = firstName + if (lastName) { + label += ` ${lastName}` + } + } + socket.data.user = { + id: _id, + email, + color, + label, + } + + // Add app ID to help split sockets into rooms + socket.data.appId = ctx.appId next() } }) From d6736d7763c9067c8c277536dbbe1e8fb0f9a9bd Mon Sep 17 00:00:00 2001 From: gitstart Date: Thu, 18 May 2023 06:32:25 +0000 Subject: [PATCH 11/46] fix: automation panel does not scroll Co-authored-by: jaysoni-ash42 --- .../builder/app/[application]/automate/_layout.svelte | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/builder/src/pages/builder/app/[application]/automate/_layout.svelte b/packages/builder/src/pages/builder/app/[application]/automate/_layout.svelte index 74dfe671ab..cabad1565c 100644 --- a/packages/builder/src/pages/builder/app/[application]/automate/_layout.svelte +++ b/packages/builder/src/pages/builder/app/[application]/automate/_layout.svelte @@ -32,7 +32,9 @@
- +
+ +
{#if $automationStore.automations?.length} @@ -81,6 +83,11 @@ grid-template-columns: 260px minmax(510px, 1fr) fit-content(500px); overflow: hidden; } + .list { + overflow-x: hidden; + overflow-y: auto; + display: flex; + } .content { position: relative; display: flex; From 3794d8e204ed624031d23cbd9cdbefd145815d24 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Thu, 18 May 2023 08:57:20 +0100 Subject: [PATCH 12/46] Simplify websocket comms for grid and improve builder socket --- packages/builder/src/builderStore/index.js | 4 +- .../src/builderStore/store/frontend.js | 5 +-- .../builder/src/builderStore/store/users.js | 45 ++++++++++++++++--- .../builder/src/builderStore/websocket.js | 22 ++++----- .../backend/DataTable/DataTable.svelte | 1 + .../builder/app/[application]/_layout.svelte | 4 +- .../src/components/grid/layout/Grid.svelte | 12 ++++- .../src/components/grid/lib/websocket.js | 3 +- .../src/components/grid/stores/users.js | 42 +++-------------- packages/server/src/websockets/grid.ts | 1 - 10 files changed, 74 insertions(+), 65 deletions(-) diff --git a/packages/builder/src/builderStore/index.js b/packages/builder/src/builderStore/index.js index 8183c75a20..9dca6a64e6 100644 --- a/packages/builder/src/builderStore/index.js +++ b/packages/builder/src/builderStore/index.js @@ -2,7 +2,7 @@ import { getFrontendStore } from "./store/frontend" import { getAutomationStore } from "./store/automation" import { getTemporalStore } from "./store/temporal" import { getThemeStore } from "./store/theme" -import { getUsersStore } from "./store/users" +import { getUserStore } from "./store/users" import { derived } from "svelte/store" import { findComponent, findComponentPath } from "./componentUtils" import { RoleUtils } from "@budibase/frontend-core" @@ -13,7 +13,7 @@ export const store = getFrontendStore() export const automationStore = getAutomationStore() export const themeStore = getThemeStore() export const temporalStore = getTemporalStore() -export const usersStore = getUsersStore() +export const userStore = getUserStore() // Setup history for screens export const screenHistoryStore = createHistoryStore({ diff --git a/packages/builder/src/builderStore/store/frontend.js b/packages/builder/src/builderStore/store/frontend.js index 0a27f576a3..8c2119b2b0 100644 --- a/packages/builder/src/builderStore/store/frontend.js +++ b/packages/builder/src/builderStore/store/frontend.js @@ -5,6 +5,7 @@ import { selectedComponent, screenHistoryStore, automationHistoryStore, + userStore, } from "builderStore" import { datasources, @@ -85,9 +86,6 @@ const INITIAL_FRONTEND_STATE = { // Onboarding onboarding: false, tourNodes: null, - - // Multi user collab - users: [], } export const getFrontendStore = () => { @@ -122,7 +120,6 @@ export const getFrontendStore = () => { initialise: async pkg => { const { layouts, screens, application, clientLibPath, hasLock } = pkg websocket = createBuilderWebsocket() - await store.actions.components.refreshDefinitions(application.appId) // Reset store state diff --git a/packages/builder/src/builderStore/store/users.js b/packages/builder/src/builderStore/store/users.js index dfb592c1c5..615346c060 100644 --- a/packages/builder/src/builderStore/store/users.js +++ b/packages/builder/src/builderStore/store/users.js @@ -1,10 +1,43 @@ -import { writable } from "svelte/store" +import { writable, get } from "svelte/store" -export const getUsersStore = () => { - const initialValue = { - users: [], +export const getUserStore = () => { + const store = writable([]) + + const init = users => { + store.set(users) } - const store = writable(initialValue) - return store + const updateUser = user => { + console.log(user) + const $users = get(store) + if (!$users.some(x => x.id === user.id)) { + store.set([...$users, user]) + } else { + store.update(state => { + const index = state.findIndex(x => x.id === user.id) + state[index] = user + return state.slice() + }) + } + } + + const removeUser = user => { + store.update(state => { + return state.filter(x => x.id !== user.id) + }) + } + + const reset = () => { + store.set([]) + } + + return { + ...store, + actions: { + init, + updateUser, + removeUser, + reset, + }, + } } diff --git a/packages/builder/src/builderStore/websocket.js b/packages/builder/src/builderStore/websocket.js index d73ab08f9a..d9b203edb0 100644 --- a/packages/builder/src/builderStore/websocket.js +++ b/packages/builder/src/builderStore/websocket.js @@ -1,26 +1,26 @@ import { createWebsocket } from "@budibase/frontend-core" -import { store } from "builderStore" +import { userStore } from "builderStore" export const createBuilderWebsocket = () => { const socket = createWebsocket("/socket/builder") socket.on("connect", () => { socket.emit("get-users", null, response => { - console.log("conntected!", response.users) - store.update(state => ({ - ...state, - users: response.users, - })) + userStore.actions.init(response.users) }) }) - socket.on("user-update", user => {}) - - socket.on("user-disconnect", user => {}) - + socket.on("user-update", userStore.actions.updateUser) + socket.on("user-disconnect", userStore.actions.removeUser) socket.on("connect_error", err => { console.log("Failed to connect to builder websocket:", err.message) }) - return socket + return { + ...socket, + disconnect: () => { + socket?.disconnect() + userStore.actions.reset() + }, + } } diff --git a/packages/builder/src/components/backend/DataTable/DataTable.svelte b/packages/builder/src/components/backend/DataTable/DataTable.svelte index dfe30a3711..b5211b7d9b 100644 --- a/packages/builder/src/components/backend/DataTable/DataTable.svelte +++ b/packages/builder/src/components/backend/DataTable/DataTable.svelte @@ -36,6 +36,7 @@ allowDeleteRows={!isUsersTable} schemaOverrides={isUsersTable ? userSchemaOverrides : null} on:updatetable={e => tables.updateTable(e.detail)} + showAvatars={false} > {#if isInternal} diff --git a/packages/builder/src/pages/builder/app/[application]/_layout.svelte b/packages/builder/src/pages/builder/app/[application]/_layout.svelte index e145466b11..409f3e071b 100644 --- a/packages/builder/src/pages/builder/app/[application]/_layout.svelte +++ b/packages/builder/src/pages/builder/app/[application]/_layout.svelte @@ -1,5 +1,5 @@
- + {#if showAvatars} + + {/if}
{#if $loaded} diff --git a/packages/frontend-core/src/components/grid/lib/websocket.js b/packages/frontend-core/src/components/grid/lib/websocket.js index 7ac34b5a95..e8cead849e 100644 --- a/packages/frontend-core/src/components/grid/lib/websocket.js +++ b/packages/frontend-core/src/components/grid/lib/websocket.js @@ -2,7 +2,7 @@ import { get } from "svelte/store" import { createWebsocket } from "../../../utils" export const createGridWebsocket = context => { - const { rows, tableId, users, userId, focusedCellId } = context + const { rows, tableId, users, focusedCellId } = context const socket = createWebsocket("/socket/grid") const connectToTable = tableId => { @@ -13,7 +13,6 @@ export const createGridWebsocket = context => { socket.emit("select-table", tableId, response => { // handle initial connection info users.set(response.users) - userId.set(response.id) }) } diff --git a/packages/frontend-core/src/components/grid/stores/users.js b/packages/frontend-core/src/components/grid/stores/users.js index 5190de70da..a160330590 100644 --- a/packages/frontend-core/src/components/grid/stores/users.js +++ b/packages/frontend-core/src/components/grid/stores/users.js @@ -2,56 +2,28 @@ import { writable, get, derived } from "svelte/store" export const createStores = () => { const users = writable([]) - const userId = writable(null) - - // Enrich users with unique colours - const enrichedUsers = derived( - [users, userId], - ([$users, $userId]) => { - return ( - $users - .slice() - // Place current user first - .sort((a, b) => { - if (a.id === $userId) { - return -1 - } else if (b.id === $userId) { - return 1 - } else { - return 0 - } - }) - ) - }, - [] - ) return { - users: { - ...users, - subscribe: enrichedUsers.subscribe, - }, - userId, + users, } } export const deriveStores = context => { - const { users, userId } = context + const { users, focusedCellId } = context // Generate a lookup map of cell ID to the user that has it selected, to make // lookups inside cells extremely fast const selectedCellMap = derived( - [users, userId], - ([$enrichedUsers, $userId]) => { + [users, focusedCellId], + ([$users, $focusedCellId]) => { let map = {} - $enrichedUsers.forEach(user => { - if (user.focusedCellId && user.id !== $userId) { + $users.forEach(user => { + if (user.focusedCellId && user.focusedCellId !== $focusedCellId) { map[user.focusedCellId] = user } }) return map - }, - {} + } ) const updateUser = user => { diff --git a/packages/server/src/websockets/grid.ts b/packages/server/src/websockets/grid.ts index 4fb81e006c..731c920e07 100644 --- a/packages/server/src/websockets/grid.ts +++ b/packages/server/src/websockets/grid.ts @@ -34,7 +34,6 @@ export default class GridSocket extends Socket { const sockets = await this.io.in(currentRoom).fetchSockets() callback({ users: sockets.map(socket => socket.data.user), - id: user.id, }) }) From b2f3455b1906befb82d84ad9ef5dc4dda69f3306 Mon Sep 17 00:00:00 2001 From: gitstart Date: Thu, 18 May 2023 11:48:57 +0000 Subject: [PATCH 13/46] update fix: automation panel does not scroll Co-authored-by: jaysoni-ash42 --- packages/builder/src/components/design/Panel.svelte | 1 + .../builder/app/[application]/automate/_layout.svelte | 9 +-------- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/packages/builder/src/components/design/Panel.svelte b/packages/builder/src/components/design/Panel.svelte index dbf42c51a5..3968292ba9 100644 --- a/packages/builder/src/components/design/Panel.svelte +++ b/packages/builder/src/components/design/Panel.svelte @@ -58,6 +58,7 @@ justify-content: flex-start; align-items: stretch; transition: width 130ms ease-out; + overflow: hidden; } .panel.borderLeft { border-left: var(--border-light); diff --git a/packages/builder/src/pages/builder/app/[application]/automate/_layout.svelte b/packages/builder/src/pages/builder/app/[application]/automate/_layout.svelte index cabad1565c..74dfe671ab 100644 --- a/packages/builder/src/pages/builder/app/[application]/automate/_layout.svelte +++ b/packages/builder/src/pages/builder/app/[application]/automate/_layout.svelte @@ -32,9 +32,7 @@
-
- -
+
{#if $automationStore.automations?.length} @@ -83,11 +81,6 @@ grid-template-columns: 260px minmax(510px, 1fr) fit-content(500px); overflow: hidden; } - .list { - overflow-x: hidden; - overflow-y: auto; - display: flex; - } .content { position: relative; display: flex; From cfa07a68aeae7e26ee2367d62b49554d08f95588 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Mon, 22 May 2023 15:59:44 +0100 Subject: [PATCH 14/46] Handle table, query and datasource events using builder socket --- .../builder/src/builderStore/websocket.js | 19 ++++- .../backend/DataTable/DataTable.svelte | 1 - .../builder/src/stores/backend/datasources.js | 36 ++++++++- packages/builder/src/stores/backend/tables.js | 76 +++++++++++-------- packages/builder/src/stores/backend/views.js | 32 +++++--- .../src/components/grid/lib/websocket.js | 29 +++++-- .../src/components/grid/stores/columns.js | 4 - .../server/src/api/controllers/datasource.ts | 4 + .../server/src/api/controllers/table/index.ts | 3 + .../server/src/api/controllers/view/index.ts | 10 +-- packages/server/src/websockets/builder.ts | 20 ++++- packages/server/src/websockets/grid.ts | 14 +++- packages/server/src/websockets/index.ts | 2 +- 13 files changed, 177 insertions(+), 73 deletions(-) diff --git a/packages/builder/src/builderStore/websocket.js b/packages/builder/src/builderStore/websocket.js index d9b203edb0..d0baae38c6 100644 --- a/packages/builder/src/builderStore/websocket.js +++ b/packages/builder/src/builderStore/websocket.js @@ -1,21 +1,34 @@ import { createWebsocket } from "@budibase/frontend-core" import { userStore } from "builderStore" +import { datasources, tables } from "stores/backend" export const createBuilderWebsocket = () => { const socket = createWebsocket("/socket/builder") + // Connection events socket.on("connect", () => { socket.emit("get-users", null, response => { userStore.actions.init(response.users) }) }) - - socket.on("user-update", userStore.actions.updateUser) - socket.on("user-disconnect", userStore.actions.removeUser) socket.on("connect_error", err => { console.log("Failed to connect to builder websocket:", err.message) }) + // User events + socket.on("user-update", userStore.actions.updateUser) + socket.on("user-disconnect", userStore.actions.removeUser) + + // Table events + socket.on("table-change", ({ id, table }) => { + tables.replaceTable(id, table) + }) + + // Table events + socket.on("datasource-change", ({ id, datasource }) => { + datasources.replaceDatasource(id, datasource) + }) + return { ...socket, disconnect: () => { diff --git a/packages/builder/src/components/backend/DataTable/DataTable.svelte b/packages/builder/src/components/backend/DataTable/DataTable.svelte index b5211b7d9b..79b49d2541 100644 --- a/packages/builder/src/components/backend/DataTable/DataTable.svelte +++ b/packages/builder/src/components/backend/DataTable/DataTable.svelte @@ -35,7 +35,6 @@ allowAddRows={!isUsersTable} allowDeleteRows={!isUsersTable} schemaOverrides={isUsersTable ? userSchemaOverrides : null} - on:updatetable={e => tables.updateTable(e.detail)} showAvatars={false} > diff --git a/packages/builder/src/stores/backend/datasources.js b/packages/builder/src/stores/backend/datasources.js index ed84bb8ee9..e414e080a2 100644 --- a/packages/builder/src/stores/backend/datasources.js +++ b/packages/builder/src/stores/backend/datasources.js @@ -1,4 +1,4 @@ -import { writable, derived } from "svelte/store" +import { writable, derived, get } from "svelte/store" import { queries, tables } from "./" import { API } from "api" @@ -89,6 +89,39 @@ export function createDatasourcesStore() { }) } + // Handles external updates of tables + const replaceDatasource = (datasourceId, datasource) => { + if (!datasourceId) { + return + } + + // Handle deletion + if (!datasource) { + store.update(state => ({ + ...state, + list: state.list.filter(x => x._id !== datasourceId), + })) + return + } + + // Add new datasource + const index = get(store).list.findIndex(x => x._id === datasource._id) + if (index === -1) { + store.update(state => ({ + ...state, + list: [...state.list, datasource], + })) + } + + // Update existing datasource + else if (datasource) { + store.update(state => { + state.list[index] = datasource + return state + }) + } + } + return { subscribe: derivedStore.subscribe, fetch, @@ -98,6 +131,7 @@ export function createDatasourcesStore() { save, delete: deleteDatasource, removeSchemaError, + replaceDatasource, } } diff --git a/packages/builder/src/stores/backend/tables.js b/packages/builder/src/stores/backend/tables.js index 75679532f3..ba900b7df9 100644 --- a/packages/builder/src/stores/backend/tables.js +++ b/packages/builder/src/stores/backend/tables.js @@ -22,18 +22,6 @@ export function createTablesStore() { })) } - const fetchTable = async tableId => { - const table = await API.fetchTableDefinition(tableId) - - store.update(state => { - const indexToUpdate = state.list.findIndex(t => t._id === table._id) - state.list[indexToUpdate] = table - return { - ...state, - } - }) - } - const select = tableId => { store.update(state => ({ ...state, @@ -74,20 +62,23 @@ export function createTablesStore() { } const savedTable = await API.saveTable(updatedTable) - await fetch() + replaceTable(table._id, savedTable) if (table.type === "external") { await datasources.fetch() } - await select(savedTable._id) + select(savedTable._id) return savedTable } const deleteTable = async table => { + if (!table?._id || !table?._rev) { + return + } await API.deleteTable({ - tableId: table?._id, - tableRev: table?._rev, + tableId: table._id, + tableRev: table._rev, }) - await fetch() + replaceTable(table._id, null) } const saveField = async ({ @@ -135,35 +126,56 @@ export function createTablesStore() { await save(draft) } - const updateTable = table => { - const index = get(store).list.findIndex(x => x._id === table._id) - if (index === -1) { + // Handles external updates of tables + const replaceTable = (tableId, table) => { + if (!tableId) { return } - // This function has to merge state as there discrepancies with the table - // API endpoints. The table list endpoint and get table endpoint use the - // "type" property to mean different things. - store.update(state => { - state.list[index] = { - ...table, - type: state.list[index].type, - } - return state - }) + // Handle deletion + if (!table) { + store.update(state => ({ + ...state, + list: state.list.filter(x => x._id !== tableId), + })) + return + } + + // Add new table + const index = get(store).list.findIndex(x => x._id === table._id) + if (index === -1) { + store.update(state => ({ + ...state, + list: [...state.list, table], + })) + } + + // Update existing table + else if (table) { + // This function has to merge state as there discrepancies with the table + // API endpoints. The table list endpoint and get table endpoint use the + // "type" property to mean different things. + store.update(state => { + state.list[index] = { + ...table, + type: state.list[index].type, + } + return state + }) + } } return { + ...store, subscribe: derivedStore.subscribe, fetch, - fetchTable, init: fetch, select, save, delete: deleteTable, saveField, deleteField, - updateTable, + replaceTable, } } diff --git a/packages/builder/src/stores/backend/views.js b/packages/builder/src/stores/backend/views.js index 71e9a44c5c..2f396096c8 100644 --- a/packages/builder/src/stores/backend/views.js +++ b/packages/builder/src/stores/backend/views.js @@ -1,4 +1,4 @@ -import { writable, get, derived } from "svelte/store" +import { writable, derived } from "svelte/store" import { tables } from "./" import { API } from "api" @@ -27,21 +27,31 @@ export function createViewsStore() { const deleteView = async view => { await API.deleteView(view) - await tables.fetch() + + // Update tables + tables.update(state => { + const table = state.list.find(table => table._id === view.tableId) + if (table) { + delete table.views[view.name] + } + return { ...state } + }) } const save = async view => { const savedView = await API.saveView(view) - const viewMeta = { - name: view.name, - ...savedView, - } - const viewTable = get(tables).list.find(table => table._id === view.tableId) - - if (view.originalName) delete viewTable.views[view.originalName] - viewTable.views[view.name] = viewMeta - await tables.save(viewTable) + // Update tables + tables.update(state => { + const table = state.list.find(table => table._id === view.tableId) + if (table) { + if (view.originalName) { + delete table.views[view.originalName] + } + table.views[view.name] = savedView + } + return { ...state } + }) } return { diff --git a/packages/frontend-core/src/components/grid/lib/websocket.js b/packages/frontend-core/src/components/grid/lib/websocket.js index e8cead849e..a62910c2fa 100644 --- a/packages/frontend-core/src/components/grid/lib/websocket.js +++ b/packages/frontend-core/src/components/grid/lib/websocket.js @@ -2,7 +2,7 @@ import { get } from "svelte/store" import { createWebsocket } from "../../../utils" export const createGridWebsocket = context => { - const { rows, tableId, users, focusedCellId } = context + const { rows, tableId, users, focusedCellId, table } = context const socket = createWebsocket("/socket/grid") const connectToTable = tableId => { @@ -16,23 +16,36 @@ export const createGridWebsocket = context => { }) } - // Event handlers + // Connection events socket.on("connect", () => { connectToTable(get(tableId)) }) - socket.on("row-update", data => { - if (data.id) { - rows.actions.replaceRow(data.id, data.row) - } + socket.on("connect_error", err => { + console.log("Failed to connect to grid websocket:", err.message) }) + + // User events socket.on("user-update", user => { users.actions.updateUser(user) }) socket.on("user-disconnect", user => { users.actions.removeUser(user) }) - socket.on("connect_error", err => { - console.log("Failed to connect to grid websocket:", err.message) + + // Row events + socket.on("row-change", data => { + if (data.id) { + rows.actions.replaceRow(data.id, data.row) + } + }) + + // Table events + socket.on("table-change", data => { + // Only update table if one exists. If the table was deleted then we don't + // want to know - let the builder navigate away + if (data.table) { + table.set(data.table) + } }) // Change websocket connection when table changes diff --git a/packages/frontend-core/src/components/grid/stores/columns.js b/packages/frontend-core/src/components/grid/stores/columns.js index e953977487..3b286ac0f0 100644 --- a/packages/frontend-core/src/components/grid/stores/columns.js +++ b/packages/frontend-core/src/components/grid/stores/columns.js @@ -90,10 +90,6 @@ export const deriveStores = context => { // Update local state table.set(newTable) - // Broadcast event so that we can keep sync with external state - // (e.g. data section which maintains a list of table definitions) - dispatch("updatetable", newTable) - // Update server await API.saveTable(newTable) } diff --git a/packages/server/src/api/controllers/datasource.ts b/packages/server/src/api/controllers/datasource.ts index 8f13e0e618..ecaf1ef58b 100644 --- a/packages/server/src/api/controllers/datasource.ts +++ b/packages/server/src/api/controllers/datasource.ts @@ -25,6 +25,7 @@ import { DatasourcePlus, } from "@budibase/types" import sdk from "../../sdk" +import { builderSocket } from "../../websockets" function getErrorTables(errors: any, errorType: string) { return Object.entries(errors) @@ -280,6 +281,7 @@ export async function update(ctx: UserCtx) { ctx.body = { datasource: await sdk.datasources.removeSecretSingle(datasource), } + builderSocket.emitDatasourceUpdate(ctx, datasource) } export async function save( @@ -322,6 +324,7 @@ export async function save( response.error = schemaError } ctx.body = response + builderSocket.emitDatasourceUpdate(ctx, datasource) } async function destroyInternalTablesBySourceId(datasourceId: string) { @@ -381,6 +384,7 @@ export async function destroy(ctx: UserCtx) { ctx.message = `Datasource deleted.` ctx.status = 200 + builderSocket.emitDatasourceDeletion(ctx, datasourceId) } export async function find(ctx: UserCtx) { diff --git a/packages/server/src/api/controllers/table/index.ts b/packages/server/src/api/controllers/table/index.ts index cbbda7b930..271f3e82fa 100644 --- a/packages/server/src/api/controllers/table/index.ts +++ b/packages/server/src/api/controllers/table/index.ts @@ -11,6 +11,7 @@ import { context, events } from "@budibase/backend-core" import { Table, UserCtx } from "@budibase/types" import sdk from "../../../sdk" import { jsonFromCsvString } from "../../../utilities/csv" +import { builderSocket } from "../../../websockets" function pickApi({ tableId, table }: { tableId?: string; table?: Table }) { if (table && !tableId) { @@ -77,6 +78,7 @@ export async function save(ctx: UserCtx) { ctx.eventEmitter && ctx.eventEmitter.emitTable(`table:save`, appId, savedTable) ctx.body = savedTable + builderSocket.emitTableUpdate(ctx, savedTable) } export async function destroy(ctx: UserCtx) { @@ -89,6 +91,7 @@ export async function destroy(ctx: UserCtx) { ctx.status = 200 ctx.table = deletedTable ctx.body = { message: `Table ${tableId} deleted.` } + builderSocket.emitTableDeletion(ctx, tableId) } export async function bulkImport(ctx: UserCtx) { diff --git a/packages/server/src/api/controllers/view/index.ts b/packages/server/src/api/controllers/view/index.ts index ed2302f723..28b0d0a81f 100644 --- a/packages/server/src/api/controllers/view/index.ts +++ b/packages/server/src/api/controllers/view/index.ts @@ -16,6 +16,7 @@ import { View, } from "@budibase/types" import { cleanExportRows } from "../row/utils" +import { builderSocket } from "../../../websockets" const { cloneDeep, isEqual } = require("lodash") @@ -48,7 +49,7 @@ export async function save(ctx: Ctx) { if (!view.meta.schema) { view.meta.schema = table.schema } - table.views[viewName] = view.meta + table.views[viewName] = { ...view.meta, name: viewName } if (originalName) { delete table.views[originalName] existingTable.views[viewName] = existingTable.views[originalName] @@ -56,10 +57,8 @@ export async function save(ctx: Ctx) { await db.put(table) await handleViewEvents(existingTable.views[viewName], table.views[viewName]) - ctx.body = { - ...table.views[viewToSave.name], - name: viewToSave.name, - } + ctx.body = table.views[viewName] + builderSocket.emitTableUpdate(ctx, table) } export async function calculationEvents(existingView: View, newView: View) { @@ -128,6 +127,7 @@ export async function destroy(ctx: Ctx) { await events.view.deleted(view) ctx.body = view + builderSocket.emitTableUpdate(ctx, table) } export async function exportView(ctx: Ctx) { diff --git a/packages/server/src/websockets/builder.ts b/packages/server/src/websockets/builder.ts index 5c23682a7b..ae32a7de18 100644 --- a/packages/server/src/websockets/builder.ts +++ b/packages/server/src/websockets/builder.ts @@ -3,7 +3,8 @@ import Socket from "./websocket" import { permissions } from "@budibase/backend-core" import http from "http" import Koa from "koa" -import { Table } from "@budibase/types" +import { Datasource, Table } from "@budibase/types" +import { gridSocket } from "./index" export default class BuilderSocket extends Socket { constructor(app: Koa, server: http.Server) { @@ -22,7 +23,6 @@ export default class BuilderSocket extends Socket { const sockets = await this.io.in(appId).fetchSockets() callback({ users: sockets.map(socket => socket.data.user), - id: user.id, }) }) @@ -34,10 +34,22 @@ export default class BuilderSocket extends Socket { } emitTableUpdate(ctx: any, table: Table) { - this.io.in(ctx.appId).emit("table-update", { id: table._id, table }) + this.io.in(ctx.appId).emit("table-change", { id: table._id, table }) + gridSocket.emitTableUpdate(table) } emitTableDeletion(ctx: any, id: string) { - this.io.in(ctx.appId).emit("table-update", { id, table: null }) + this.io.in(ctx.appId).emit("table-change", { id, table: null }) + gridSocket.emitTableDeletion(id) + } + + emitDatasourceUpdate(ctx: any, datasource: Datasource) { + this.io + .in(ctx.appId) + .emit("datasource-change", { id: datasource._id, datasource }) + } + + emitDatasourceDeletion(ctx: any, id: string) { + this.io.in(ctx.appId).emit("datasource-change", { id, datasource: null }) } } diff --git a/packages/server/src/websockets/grid.ts b/packages/server/src/websockets/grid.ts index 731c920e07..09b9ee0e3d 100644 --- a/packages/server/src/websockets/grid.ts +++ b/packages/server/src/websockets/grid.ts @@ -4,7 +4,7 @@ import { permissions } from "@budibase/backend-core" import http from "http" import Koa from "koa" import { getTableId } from "../api/controllers/row/utils" -import { Row } from "@budibase/types" +import { Row, Table } from "@budibase/types" export default class GridSocket extends Socket { constructor(app: Koa, server: http.Server) { @@ -56,11 +56,19 @@ export default class GridSocket extends Socket { emitRowUpdate(ctx: any, row: Row) { const tableId = getTableId(ctx) - this.io.in(tableId).emit("row-update", { id: row._id, row }) + this.io.in(tableId).emit("row-change", { id: row._id, row }) } emitRowDeletion(ctx: any, id: string) { const tableId = getTableId(ctx) - this.io.in(tableId).emit("row-update", { id, row: null }) + this.io.in(tableId).emit("row-change", { id, row: null }) + } + + emitTableUpdate(table: Table) { + this.io.in(table._id!).emit("table-change", { id: table._id, table }) + } + + emitTableDeletion(id: string) { + this.io.in(id).emit("table-change", { id, table: null }) } } diff --git a/packages/server/src/websockets/index.ts b/packages/server/src/websockets/index.ts index 2761c4d9da..b74a8adfca 100644 --- a/packages/server/src/websockets/index.ts +++ b/packages/server/src/websockets/index.ts @@ -14,4 +14,4 @@ export const initialise = (app: Koa, server: http.Server) => { builderSocket = new BuilderSocket(app, server) } -export { clientAppSocket, gridSocket } +export { clientAppSocket, gridSocket, builderSocket } From 53eceec9785444de29fa0b622f2ebc4e095cd619 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Mon, 22 May 2023 16:01:12 +0100 Subject: [PATCH 15/46] Update comments --- packages/builder/src/stores/backend/datasources.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/builder/src/stores/backend/datasources.js b/packages/builder/src/stores/backend/datasources.js index e414e080a2..98f6521bbb 100644 --- a/packages/builder/src/stores/backend/datasources.js +++ b/packages/builder/src/stores/backend/datasources.js @@ -89,7 +89,7 @@ export function createDatasourcesStore() { }) } - // Handles external updates of tables + // Handles external updates of datasources const replaceDatasource = (datasourceId, datasource) => { if (!datasourceId) { return From 5e480a1527314d9a7340335b747c85b8062fb251 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Thu, 25 May 2023 08:48:56 +0100 Subject: [PATCH 16/46] Use socker.io redis adapter to broadcast events to all server instances --- packages/backend-core/src/redis/utils.ts | 1 + packages/server/package.json | 1 + packages/server/src/app.ts | 1 - packages/server/src/startup.ts | 2 ++ packages/server/src/utilities/redis.ts | 21 +++++++++++++++++++-- packages/server/src/websockets/websocket.ts | 9 ++++++++- yarn.lock | 19 +++++++++++++++++++ 7 files changed, 50 insertions(+), 4 deletions(-) diff --git a/packages/backend-core/src/redis/utils.ts b/packages/backend-core/src/redis/utils.ts index 2c49ee4941..f8b815824c 100644 --- a/packages/backend-core/src/redis/utils.ts +++ b/packages/backend-core/src/redis/utils.ts @@ -27,6 +27,7 @@ export enum Databases { GENERIC_CACHE = "data_cache", WRITE_THROUGH = "writeThrough", LOCKS = "locks", + SOCKET_IO = "socket_io", } /** diff --git a/packages/server/package.json b/packages/server/package.json index 5ed4579667..31022a7c99 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -58,6 +58,7 @@ "@koa/router": "8.0.8", "@sendgrid/mail": "7.1.1", "@sentry/node": "6.17.7", + "@socket.io/redis-adapter": "^8.2.1", "airtable": "0.10.1", "arangojs": "7.2.0", "aws-sdk": "2.1030.0", diff --git a/packages/server/src/app.ts b/packages/server/src/app.ts index f8f82f9fdc..274de89c57 100644 --- a/packages/server/src/app.ts +++ b/packages/server/src/app.ts @@ -61,7 +61,6 @@ if (env.isProd()) { const server = http.createServer(app.callback()) destroyable(server) -initialiseWebsockets(app, server) let shuttingDown = false, errCode = 0 diff --git a/packages/server/src/startup.ts b/packages/server/src/startup.ts index 2fd59b7a8e..64a6c011c4 100644 --- a/packages/server/src/startup.ts +++ b/packages/server/src/startup.ts @@ -16,6 +16,7 @@ import * as bullboard from "./automations/bullboard" import * as pro from "@budibase/pro" import * as api from "./api" import sdk from "./sdk" +import { initialise as initialiseWebsockets } from "./websockets" let STARTUP_RAN = false @@ -64,6 +65,7 @@ export async function startup(app?: any, server?: any) { fileSystem.init() await redis.init() eventInit() + initialiseWebsockets(app, server) // run migrations on startup if not done via http // not recommended in a clustered environment diff --git a/packages/server/src/utilities/redis.ts b/packages/server/src/utilities/redis.ts index dc37baae58..ff1c863bf7 100644 --- a/packages/server/src/utilities/redis.ts +++ b/packages/server/src/utilities/redis.ts @@ -4,23 +4,33 @@ import { ContextUser } from "@budibase/types" const APP_DEV_LOCK_SECONDS = 600 const AUTOMATION_TEST_FLAG_SECONDS = 60 -let devAppClient: any, debounceClient: any, flagClient: any +let devAppClient: any, debounceClient: any, flagClient: any, socketClient: any -// we init this as we want to keep the connection open all the time +// We need to maintain a duplicate client for socket.io pub/sub +let socketSubClient: any + +// We init this as we want to keep the connection open all the time // reduces the performance hit export async function init() { devAppClient = new redis.Client(redis.utils.Databases.DEV_LOCKS) debounceClient = new redis.Client(redis.utils.Databases.DEBOUNCE) flagClient = new redis.Client(redis.utils.Databases.FLAGS) + socketClient = new redis.Client(redis.utils.Databases.SOCKET_IO) await devAppClient.init() await debounceClient.init() await flagClient.init() + await socketClient.init() + + // Duplicate the socket client for pub/sub + socketSubClient = socketClient.getClient().duplicate() } export async function shutdown() { if (devAppClient) await devAppClient.finish() if (debounceClient) await debounceClient.finish() if (flagClient) await flagClient.finish() + if (socketClient) await socketClient.finish() + if (socketSubClient) socketSubClient.disconnect() // shutdown core clients await redis.clients.shutdown() console.log("Redis shutdown") @@ -86,3 +96,10 @@ export async function checkTestFlag(id: string) { export async function clearTestFlag(id: string) { await devAppClient.delete(id) } + +export function getSocketPubSubClients() { + return { + pub: socketClient.getClient(), + sub: socketSubClient, + } +} diff --git a/packages/server/src/websockets/websocket.ts b/packages/server/src/websockets/websocket.ts index 0e0e6d59a4..c3a1019932 100644 --- a/packages/server/src/websockets/websocket.ts +++ b/packages/server/src/websockets/websocket.ts @@ -5,6 +5,8 @@ import Cookies from "cookies" import { userAgent } from "koa-useragent" import { auth } from "@budibase/backend-core" import currentApp from "../middleware/currentapp" +import { createAdapter } from "@socket.io/redis-adapter" +import { getSocketPubSubClients } from "../utilities/redis" export default class Socket { io: Server @@ -12,7 +14,7 @@ export default class Socket { constructor( app: Koa, server: http.Server, - path: string, + path: string = "/", additionalMiddlewares?: any[] ) { this.io = new Server(server, { @@ -97,6 +99,11 @@ export default class Socket { next(error) } }) + + // Instantiate redis adapter + const { pub, sub } = getSocketPubSubClients() + const opts = { key: `socket.io-${path}` } + this.io.adapter(createAdapter(pub, sub, opts)) } // Emit an event to all sockets diff --git a/yarn.lock b/yarn.lock index 086342b0f2..e37e9769eb 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4623,6 +4623,15 @@ resolved "https://registry.yarnpkg.com/@socket.io/component-emitter/-/component-emitter-3.1.0.tgz#96116f2a912e0c02817345b3c10751069920d553" integrity sha512-+9jVqKhRSpsc591z5vX+X5Yyw+he/HCB4iQ/RYxw35CEPaY1gnsNE43nf9n9AaYjAQrTiI/mOwKUKdUs9vf7Xg== +"@socket.io/redis-adapter@^8.2.1": + version "8.2.1" + resolved "https://registry.yarnpkg.com/@socket.io/redis-adapter/-/redis-adapter-8.2.1.tgz#36f75afc518d0e1fa4fa7c29e6d042f53ee7563b" + integrity sha512-6Dt7EZgGSBP0qvXeOKGx7NnSr2tPMbVDfDyL97zerZo+v69hMfL99skMCL3RKZlWVqLyRme2T0wcy3udHhtOsg== + dependencies: + debug "~4.3.1" + notepack.io "~3.0.1" + uid2 "1.0.0" + "@spectrum-css/accordion@3.0.24": version "3.0.24" resolved "https://registry.yarnpkg.com/@spectrum-css/accordion/-/accordion-3.0.24.tgz#f89066c120c57b0cfc9aba66d60c39fc1cf69f74" @@ -18641,6 +18650,11 @@ normalize-url@^6.0.1: resolved "https://registry.yarnpkg.com/normalize-url/-/normalize-url-6.1.0.tgz#40d0885b535deffe3f3147bec877d05fe4c5668a" integrity sha512-DlL+XwOy3NxAQ8xuC0okPgK46iuVNAK01YN7RueYBqqFeGsBjV9XmCAzAdgt+667bCl5kPh9EqKKDwnaPG1I7A== +notepack.io@~3.0.1: + version "3.0.1" + resolved "https://registry.yarnpkg.com/notepack.io/-/notepack.io-3.0.1.tgz#2c2c9de1bd4e64a79d34e33c413081302a0d4019" + integrity sha512-TKC/8zH5pXIAMVQio2TvVDTtPRX+DJPHDqjRbxogtFiByHyzKmy96RA0JtCQJ+WouyyL4A10xomQzgbUT+1jCg== + npm-bundled@^1.1.2: version "1.1.2" resolved "https://registry.yarnpkg.com/npm-bundled/-/npm-bundled-1.1.2.tgz#944c78789bd739035b70baa2ca5cc32b8d860bc1" @@ -24787,6 +24801,11 @@ uid2@0.0.x: resolved "https://registry.yarnpkg.com/uid2/-/uid2-0.0.4.tgz#033f3b1d5d32505f5ce5f888b9f3b667123c0a44" integrity sha512-IevTus0SbGwQzYh3+fRsAMTVVPOoIVufzacXcHPmdlle1jUpq7BRL+mw3dgeLanvGZdwwbWhRV6XrcFNdBmjWA== +uid2@1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/uid2/-/uid2-1.0.0.tgz#ef8d95a128d7c5c44defa1a3d052eecc17a06bfb" + integrity sha512-+I6aJUv63YAcY9n4mQreLUt0d4lvwkkopDNmpomkAUz0fAkEMV9pRWxN0EjhW1YfRhcuyHg2v3mwddCDW1+LFQ== + unbox-primitive@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/unbox-primitive/-/unbox-primitive-1.0.2.tgz#29032021057d5e6cdbd08c5129c226dff8ed6f9e" From 183fb3d29b5fc252b10f46935a0cc1179cec2cc6 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Tue, 23 May 2023 16:02:05 +0100 Subject: [PATCH 17/46] Fix issue with schema mutation in grids. Use displayName to represent grid column labels. Allow deletion of duplicated user columns --- .../backend/DataTable/DataTable.svelte | 10 +++++----- .../src/components/grid/stores/columns.js | 20 ++++++++++++++++--- 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/packages/builder/src/components/backend/DataTable/DataTable.svelte b/packages/builder/src/components/backend/DataTable/DataTable.svelte index 79b49d2541..7a8b622594 100644 --- a/packages/builder/src/components/backend/DataTable/DataTable.svelte +++ b/packages/builder/src/components/backend/DataTable/DataTable.svelte @@ -16,11 +16,11 @@ import GridEditColumnModal from "components/backend/DataTable/modals/grid/GridEditColumnModal.svelte" const userSchemaOverrides = { - firstName: { name: "First name", disabled: true }, - lastName: { name: "Last name", disabled: true }, - email: { name: "Email", disabled: true }, - roleId: { name: "Role", disabled: true }, - status: { name: "Status", disabled: true }, + firstName: { displayName: "First name", disabled: true }, + lastName: { displayName: "Last name", disabled: true }, + email: { displayName: "Email", disabled: true }, + roleId: { displayName: "Role", disabled: true }, + status: { displayName: "Status", disabled: true }, } $: id = $tables.selected?._id diff --git a/packages/frontend-core/src/components/grid/stores/columns.js b/packages/frontend-core/src/components/grid/stores/columns.js index 3b286ac0f0..46519b9226 100644 --- a/packages/frontend-core/src/components/grid/stores/columns.js +++ b/packages/frontend-core/src/components/grid/stores/columns.js @@ -112,10 +112,24 @@ export const initialise = context => { const schema = derived( [table, schemaOverrides], ([$table, $schemaOverrides]) => { - let newSchema = $table?.schema - if (!newSchema) { + if (!$table?.schema) { return null } + let newSchema = { ...$table?.schema } + + // Edge case to temporarily allow deletion of duplicated user + // fields that were saved with the "disabled" flag set. + // By overriding the saved schema we ensure only overrides can + // set the disabled flag. + // TODO: remove in future + Object.keys(newSchema).forEach(field => { + newSchema[field] = { + ...newSchema[field], + disabled: false, + } + }) + + // Apply schema overrides Object.keys($schemaOverrides || {}).forEach(field => { if (newSchema[field]) { newSchema[field] = { @@ -156,7 +170,7 @@ export const initialise = context => { fields .map(field => ({ name: field, - label: $schema[field].name || field, + label: $schema[field].displayName || field, schema: $schema[field], width: $schema[field].width || DefaultColumnWidth, visible: $schema[field].visible ?? true, From b29ea6120364ebf969814eca21ef15b4fde749a3 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Tue, 23 May 2023 18:01:04 +0100 Subject: [PATCH 18/46] Ensure text cells account for being used for number cells too --- .../frontend-core/src/components/grid/cells/TextCell.svelte | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/frontend-core/src/components/grid/cells/TextCell.svelte b/packages/frontend-core/src/components/grid/cells/TextCell.svelte index 533b030b5c..04485a6b50 100644 --- a/packages/frontend-core/src/components/grid/cells/TextCell.svelte +++ b/packages/frontend-core/src/components/grid/cells/TextCell.svelte @@ -52,7 +52,7 @@ {:else}
- {value || ""} + {value ?? ""}
{/if} From f246a982db89b7ad007ae69351172507fcb4d895 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Thu, 25 May 2023 10:45:00 +0100 Subject: [PATCH 19/46] Update lock icon and remove logs --- packages/builder/src/builderStore/store/users.js | 1 - packages/builder/src/components/common/AppLockModal.svelte | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/builder/src/builderStore/store/users.js b/packages/builder/src/builderStore/store/users.js index 615346c060..6de2adc90a 100644 --- a/packages/builder/src/builderStore/store/users.js +++ b/packages/builder/src/builderStore/store/users.js @@ -8,7 +8,6 @@ export const getUserStore = () => { } const updateUser = user => { - console.log(user) const $users = get(store) if (!$users.some(x => x.id === user.id)) { store.set([...$users, user]) diff --git a/packages/builder/src/components/common/AppLockModal.svelte b/packages/builder/src/components/common/AppLockModal.svelte index 875b4afce0..df491ead38 100644 --- a/packages/builder/src/components/common/AppLockModal.svelte +++ b/packages/builder/src/components/common/AppLockModal.svelte @@ -60,7 +60,7 @@ {#if lockedBy}
{ From cc7df474c91eff86a94b33125abc4ca03db3c744 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Fri, 26 May 2023 09:24:53 +0100 Subject: [PATCH 20/46] Standardise usage of user avatars and colours across the entire platform --- packages/bbui/src/Avatar/Avatar.svelte | 18 ++- .../builder/src/builderStore/store/users.js | 6 +- .../src/components/common/AppLockModal.svelte | 143 ------------------ .../src/components/start/AppRow.svelte | 29 ++-- .../_components/UserAvatars.svelte | 33 ++-- .../builder/app/[application]/_layout.svelte | 12 +- .../src/pages/builder/apps/index.svelte | 11 +- .../portal/_components/UserDropdown.svelte | 5 +- .../auditLogs/_components/UserRenderer.svelte | 13 +- .../portal/overview/[appId]/_layout.svelte | 2 - .../backups/_components/UserRenderer.svelte | 9 +- .../portal/overview/[appId]/overview.svelte | 15 +- .../portal/users/users/[userId].svelte | 19 +-- .../src/components/UserAvatar.svelte | 58 +++++++ .../src/components/grid/layout/Avatar.svelte | 24 --- .../components/grid/layout/UserAvatars.svelte | 16 +- .../src/components/grid/stores/users.js | 20 ++- .../frontend-core/src/components/index.js | 1 + packages/server/src/websockets/builder.ts | 22 ++- packages/server/src/websockets/grid.ts | 9 +- packages/server/src/websockets/websocket.ts | 26 +--- packages/shared-core/src/helpers/helpers.ts | 59 ++++++++ 22 files changed, 254 insertions(+), 296 deletions(-) delete mode 100644 packages/builder/src/components/common/AppLockModal.svelte create mode 100644 packages/frontend-core/src/components/UserAvatar.svelte delete mode 100644 packages/frontend-core/src/components/grid/layout/Avatar.svelte diff --git a/packages/bbui/src/Avatar/Avatar.svelte b/packages/bbui/src/Avatar/Avatar.svelte index 1e4cefd8ce..0faf50f55a 100644 --- a/packages/bbui/src/Avatar/Avatar.svelte +++ b/packages/bbui/src/Avatar/Avatar.svelte @@ -13,10 +13,12 @@ export let url = "" export let disabled = false export let initials = "JD" + export let color = null const DefaultColor = "#3aab87" - $: color = getColor(initials) + $: avatarColor = color || getColor(initials) + $: style = getStyle(size, avatarColor) const getColor = initials => { if (!initials?.length) { @@ -26,6 +28,12 @@ const hue = ((code % 26) / 26) * 360 return `hsl(${hue}, 50%, 50%)` } + + const getStyle = (sizeKey, color) => { + const size = `var(${sizes.get(sizeKey)})` + const fontSize = `calc(${size} / 2)` + return `width:${size}; height:${size}; font-size:${fontSize}; background:${color};` + } {#if url} @@ -37,13 +45,7 @@ style="width: var({sizes.get(size)}); height: var({sizes.get(size)});" /> {:else} -
+
{initials || ""}
{/if} diff --git a/packages/builder/src/builderStore/store/users.js b/packages/builder/src/builderStore/store/users.js index 6de2adc90a..18f343c884 100644 --- a/packages/builder/src/builderStore/store/users.js +++ b/packages/builder/src/builderStore/store/users.js @@ -9,11 +9,11 @@ export const getUserStore = () => { const updateUser = user => { const $users = get(store) - if (!$users.some(x => x.id === user.id)) { + if (!$users.some(x => x.sessionId === user.sessionId)) { store.set([...$users, user]) } else { store.update(state => { - const index = state.findIndex(x => x.id === user.id) + const index = state.findIndex(x => x.sessionId === user.sessionId) state[index] = user return state.slice() }) @@ -22,7 +22,7 @@ export const getUserStore = () => { const removeUser = user => { store.update(state => { - return state.filter(x => x.id !== user.id) + return state.filter(x => x.sessionId !== user.sessionId) }) } diff --git a/packages/builder/src/components/common/AppLockModal.svelte b/packages/builder/src/components/common/AppLockModal.svelte deleted file mode 100644 index df491ead38..0000000000 --- a/packages/builder/src/components/common/AppLockModal.svelte +++ /dev/null @@ -1,143 +0,0 @@ - - -{#if lockedBy} -
- { - e.stopPropagation() - appLockModal.show() - }} - /> -
-{/if} - - - - - - Apps are locked to prevent work being lost from overlapping changes - between your team. - - {#if lockedByYou && getExpiryDuration(app) > 0} - - {processStringSync( - "This lock will expire in {{ duration time 'millisecond' }} from now.", - { - time: getExpiryDuration(app), - } - )} - - {/if} -
- - - {#if lockedByYou} - - {/if} - -
- - - - - diff --git a/packages/builder/src/components/start/AppRow.svelte b/packages/builder/src/components/start/AppRow.svelte index 162958473e..0c011e89be 100644 --- a/packages/builder/src/components/start/AppRow.svelte +++ b/packages/builder/src/components/start/AppRow.svelte @@ -1,13 +1,16 @@
- {#each users as user} -
- {user.email[0]} -
+ {#each uniqueUsers as user} + {/each}
@@ -15,19 +25,4 @@ display: flex; gap: 4px; } - .avatar { - width: 24px; - height: 24px; - display: grid; - place-items: center; - color: white; - border-radius: 50%; - font-size: 12px; - font-weight: 700; - text-transform: uppercase; - line-height: 0; - } - .avatar:hover { - cursor: pointer; - } diff --git a/packages/builder/src/pages/builder/app/[application]/_layout.svelte b/packages/builder/src/pages/builder/app/[application]/_layout.svelte index 409f3e071b..4da0da44f9 100644 --- a/packages/builder/src/pages/builder/app/[application]/_layout.svelte +++ b/packages/builder/src/pages/builder/app/[application]/_layout.svelte @@ -218,7 +218,9 @@
{:then _} - +
+ +
{:catch error}

Something went wrong: {error.message}

{/await} @@ -254,6 +256,7 @@ box-sizing: border-box; align-items: stretch; border-bottom: var(--border-light); + z-index: 2; } .topleftnav { @@ -294,4 +297,11 @@ flex-direction: row; gap: 8px; } + + .body { + flex: 1 1 auto; + z-index: 1; + display: flex; + flex-direction: column; + } diff --git a/packages/builder/src/pages/builder/apps/index.svelte b/packages/builder/src/pages/builder/apps/index.svelte index 4b77671345..1806d51b69 100644 --- a/packages/builder/src/pages/builder/apps/index.svelte +++ b/packages/builder/src/pages/builder/apps/index.svelte @@ -5,7 +5,6 @@ Divider, ActionMenu, MenuItem, - Avatar, Page, Icon, Body, @@ -22,6 +21,8 @@ import { processStringSync } from "@budibase/string-templates" import Spaceman from "assets/bb-space-man.svg" import Logo from "assets/bb-emblem.svg" + import { UserAvatar } from "@budibase/frontend-core" + import { helpers } from "@budibase/shared-core" let loaded = false let userInfoModal @@ -96,11 +97,7 @@
- +
userInfoModal.show()}> @@ -125,7 +122,7 @@
- Hey {$auth.user.firstName || $auth.user.email} + Hey {helpers.getUserLabel($auth.user)} Welcome to the {$organisation.company} portal. Below you'll find the diff --git a/packages/builder/src/pages/builder/portal/_components/UserDropdown.svelte b/packages/builder/src/pages/builder/portal/_components/UserDropdown.svelte index 935d69812f..9faae70aa9 100644 --- a/packages/builder/src/pages/builder/portal/_components/UserDropdown.svelte +++ b/packages/builder/src/pages/builder/portal/_components/UserDropdown.svelte @@ -1,11 +1,12 @@ {#if row?.user?.email} @@ -19,7 +14,7 @@ on:focus={() => (showTooltip = true)} on:mouseleave={() => (showTooltip = false)} > - +
{#if showTooltip}
diff --git a/packages/builder/src/pages/builder/portal/overview/[appId]/_layout.svelte b/packages/builder/src/pages/builder/portal/overview/[appId]/_layout.svelte index cc244852d8..8ee469a914 100644 --- a/packages/builder/src/pages/builder/portal/overview/[appId]/_layout.svelte +++ b/packages/builder/src/pages/builder/portal/overview/[appId]/_layout.svelte @@ -24,7 +24,6 @@ import { AppStatus } from "constants" import analytics, { Events, EventSource } from "analytics" import { store } from "builderStore" - import AppLockModal from "components/common/AppLockModal.svelte" import EditableIcon from "components/common/EditableIcon.svelte" import { API } from "api" import ConfirmDialog from "components/common/ConfirmDialog.svelte" @@ -128,7 +127,6 @@ />
- - - - - + {#if !$isActive("./new")} + + + + + + + {/if}
diff --git a/packages/builder/src/pages/builder/app/[application]/data/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/index.svelte index b2aca1f7f3..47939f09b4 100644 --- a/packages/builder/src/pages/builder/app/[application]/data/index.svelte +++ b/packages/builder/src/pages/builder/app/[application]/data/index.svelte @@ -1,22 +1,17 @@ - - diff --git a/packages/builder/src/pages/builder/app/[application]/data/new.svelte b/packages/builder/src/pages/builder/app/[application]/data/new.svelte new file mode 100644 index 0000000000..f8e8fd85e7 --- /dev/null +++ b/packages/builder/src/pages/builder/app/[application]/data/new.svelte @@ -0,0 +1,257 @@ + + + + + + + + {#if integration?.auth?.type === "google"} + + {:else} + + {/if} + + +
+
+ {#if hasData} + + {/if} +
+
+ Add new data source +
+ +
+ Get started with our Budibase DB + +
+ +
+ + + + + + + + + +
+ +
+ Or connect to an external datasource +
+ +
+ {#each integrations as [key, value]} + handleIntegrationSelect(key)} + title={value.friendlyName} + description={value.type} + {disabled} + > + + + {/each} +
+
+ + diff --git a/packages/builder/src/pages/builder/portal/apps/onboarding/_components/DataPanel.svelte b/packages/builder/src/pages/builder/portal/apps/onboarding/_components/DataPanel.svelte deleted file mode 100644 index 9a7fffd893..0000000000 --- a/packages/builder/src/pages/builder/portal/apps/onboarding/_components/DataPanel.svelte +++ /dev/null @@ -1,13 +0,0 @@ - - -
- - -
diff --git a/packages/builder/src/pages/builder/portal/apps/onboarding/_components/DatasourceConfigPanel.svelte b/packages/builder/src/pages/builder/portal/apps/onboarding/_components/DatasourceConfigPanel.svelte deleted file mode 100644 index 2b44648279..0000000000 --- a/packages/builder/src/pages/builder/portal/apps/onboarding/_components/DatasourceConfigPanel.svelte +++ /dev/null @@ -1,120 +0,0 @@ - - -
- -
- - {#each Object.entries(fields) as [name, { type, default: defaultValue, required }]} - {#if type !== "boolean"} - {}} - label={formatName(name)} - {type} - /> - {/if} - {/each} - {#each Object.entries(fields) as [name, { type, default: defaultValue, required }]} - {#if type === "boolean"} - - {/if} - {/each} - -
- {#if isGoogle} - - {:else} - - {/if} -
- - diff --git a/packages/builder/src/pages/builder/portal/apps/onboarding/_components/ExampleApp.svelte b/packages/builder/src/pages/builder/portal/apps/onboarding/_components/ExampleApp.svelte index 3e970ac360..0b290decbf 100644 --- a/packages/builder/src/pages/builder/portal/apps/onboarding/_components/ExampleApp.svelte +++ b/packages/builder/src/pages/builder/portal/apps/onboarding/_components/ExampleApp.svelte @@ -1,6 +1,5 @@ - - - -
- {#if stage === "name"} - (stage = "data")} /> - {:else if googleComplete} -
- Please login to your Google account in the new tab which as opened to - continue. -
- {:else if integrationsLoading || creationLoading} -
- -
- {:else if stage === "data"} - (stage = "name")}> -
- handleCreateApp({ useSampleData: true })} - > -
-
- -
- Budibase Sample data -
- -
-
- -
-
- -
- Upload data (CSV or JSON) -
- -
- {#each Object.entries(plusIntegrations) as [integrationType, schema]} -
- (stage = integrationType)}> -
-
- -
- {schema.friendlyName} -
- -
- {/each} - - {:else if stage in plusIntegrations} - (stage = "data")} - onNext={data => { - const isGoogle = data.isGoogle - delete data.isGoogle - return handleCreateApp({ datasourceConfig: data, isGoogle }) - }} - /> - {:else} -

There was an problem. Please refresh the page and try again.

- {/if} +
- +
@@ -258,35 +77,4 @@ .full-width { width: 100%; } - .centered { - display: flex; - justify-content: center; - align-items: center; - min-height: 400px; - } - - .dataButton { - margin-bottom: 12px; - } - - .dataButtonContent { - display: flex; - align-items: center; - } - - .budibaseLogo { - height: 20px; - } - - .dataButtonIcon { - width: 22px; - display: flex; - justify-content: center; - margin-right: 16px; - } - - .dataButtonContent :global(svg) { - font-size: 18px; - color: white; - } diff --git a/packages/builder/src/stores/backend/tables.js b/packages/builder/src/stores/backend/tables.js index ba900b7df9..a36c91d1b1 100644 --- a/packages/builder/src/stores/backend/tables.js +++ b/packages/builder/src/stores/backend/tables.js @@ -63,9 +63,7 @@ export function createTablesStore() { const savedTable = await API.saveTable(updatedTable) replaceTable(table._id, savedTable) - if (table.type === "external") { - await datasources.fetch() - } + await datasources.fetch() select(savedTable._id) return savedTable } diff --git a/packages/frontend-core/src/api/app.js b/packages/frontend-core/src/api/app.js index 7868eef063..ce18bcc0c5 100644 --- a/packages/frontend-core/src/api/app.js +++ b/packages/frontend-core/src/api/app.js @@ -152,4 +152,10 @@ export const buildAppEndpoints = API => ({ url: `/api/${appId}/components/definitions`, }) }, + + addSampleData: async appId => { + return await API.post({ + url: `/api/applications/${appId}/sample`, + }) + }, }) diff --git a/packages/server/src/api/controllers/application.ts b/packages/server/src/api/controllers/application.ts index 815366785c..9c89b48b8a 100644 --- a/packages/server/src/api/controllers/application.ts +++ b/packages/server/src/api/controllers/application.ts @@ -26,7 +26,10 @@ import { env as envCore, } from "@budibase/backend-core" import { USERS_TABLE_SCHEMA } from "../../constants" -import { buildDefaultDocs } from "../../db/defaultData/datasource_bb_default" +import { + DEFAULT_BB_DATASOURCE_ID, + buildDefaultDocs, +} from "../../db/defaultData/datasource_bb_default" import { removeAppFromUserRoles } from "../../utilities/workerRequests" import { stringToReadStream, isQsTrue } from "../../utilities" import { getLocksById, doesUserHaveLock } from "../../utilities/redis" @@ -111,11 +114,7 @@ function checkAppName( } } -async function createInstance( - appId: string, - template: any, - includeSampleData: boolean -) { +async function createInstance(appId: string, template: any) { const db = context.getAppDB() await db.put({ _id: "_design/database", @@ -142,21 +141,25 @@ async function createInstance( } else { // create the users table await db.put(USERS_TABLE_SCHEMA) - - if (includeSampleData) { - // create ootb stock db - await addDefaultTables(db) - } } return { _id: appId } } -async function addDefaultTables(db: Database) { - const defaultDbDocs = buildDefaultDocs() +export const addSampleData = async (ctx: UserCtx) => { + const db = context.getAppDB() - // add in the default db data docs - tables, datasource, rows and links - await db.bulkDocs([...defaultDbDocs]) + try { + // Check if default datasource exists before creating it + await sdk.datasources.get(DEFAULT_BB_DATASOURCE_ID) + } catch (err: any) { + const defaultDbDocs = buildDefaultDocs() + + // add in the default db data docs - tables, datasource, rows and links + await db.bulkDocs([...defaultDbDocs]) + } + + ctx.status = 200 } export async function fetch(ctx: UserCtx) { @@ -248,16 +251,11 @@ async function performAppCreate(ctx: UserCtx) { if (ctx.request.files && ctx.request.files.templateFile) { instanceConfig.file = ctx.request.files.templateFile } - const includeSampleData = isQsTrue(ctx.request.body.sampleData) const tenantId = tenancy.isMultiTenant() ? tenancy.getTenantId() : null const appId = generateDevAppID(generateAppID(tenantId)) return await context.doInAppContext(appId, async () => { - const instance = await createInstance( - appId, - instanceConfig, - includeSampleData - ) + const instance = await createInstance(appId, instanceConfig) const db = context.getAppDB() let newApplication: App = { diff --git a/packages/server/src/api/routes/application.ts b/packages/server/src/api/routes/application.ts index 0aa88568f3..0c1fa364ff 100644 --- a/packages/server/src/api/routes/application.ts +++ b/packages/server/src/api/routes/application.ts @@ -38,6 +38,11 @@ router authorized(permissions.BUILDER), controller.revertClient ) + .post( + "/api/applications/:appId/sample", + authorized(permissions.BUILDER), + controller.addSampleData + ) .post( "/api/applications/:appId/publish", authorized(permissions.BUILDER), From 3f8828086749696c01fe892215cef9af5f273bc8 Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Tue, 30 May 2023 12:02:25 +0000 Subject: [PATCH 33/46] Bump version to 2.6.19-alpha.30 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index bace077bb9..344e5ffffc 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.6.19-alpha.29", + "version": "2.6.19-alpha.30", "npmClient": "yarn", "packages": [ "packages/backend-core", From 5a7f0ba58693bde1aa319318a6ed41bb02fa68ce Mon Sep 17 00:00:00 2001 From: Peter Clement Date: Tue, 30 May 2023 14:25:28 +0100 Subject: [PATCH 34/46] Fix performance issue with looping and context --- packages/server/src/threads/automation.ts | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/packages/server/src/threads/automation.ts b/packages/server/src/threads/automation.ts index 9db647b693..b9ef30cbca 100644 --- a/packages/server/src/threads/automation.ts +++ b/packages/server/src/threads/automation.ts @@ -37,8 +37,8 @@ const LOOP_STEP_ID = actions.BUILTIN_ACTION_DEFINITIONS.LOOP.stepId const CRON_STEP_ID = triggerDefs.CRON.stepId const STOPPED_STATUS = { success: true, status: AutomationStatus.STOPPED } -function getLoopIterations(loopStep: LoopStep, input: LoopInput) { - const binding = automationUtils.typecastForLooping(loopStep, input) +function getLoopIterations(loopStep: LoopStep) { + let binding = loopStep.inputs.binding if (!binding) { return 0 } @@ -68,7 +68,6 @@ class Orchestrator { constructor(job: AutomationJob) { let automation = job.data.automation let triggerOutput = job.data.event - let timeout = job.data.event.timeout const metadata = triggerOutput.metadata this._chainCount = metadata ? metadata.automationChainCount! : 0 this._appId = triggerOutput.appId as string @@ -277,22 +276,17 @@ class Orchestrator { if (loopStep) { input = await processObject(loopStep.inputs, this._context) - iterations = getLoopIterations(loopStep as LoopStep, input) + iterations = getLoopIterations(loopStep as LoopStep) } for (let index = 0; index < iterations; index++) { let originalStepInput = cloneDeep(step.inputs) // Handle if the user has set a max iteration count or if it reaches the max limit set by us if (loopStep && input.binding) { - let newInput: any = await processObject( - loopStep.inputs, - cloneDeep(this._context) - ) - let tempOutput = { items: loopSteps, iterations: iterationCount } try { - newInput.binding = automationUtils.typecastForLooping( + loopStep.inputs.binding = automationUtils.typecastForLooping( loopStep as LoopStep, - newInput + loopStep.inputs as LoopInput ) } catch (err) { this.updateContextAndOutput(loopStepNumber, step, tempOutput, { @@ -303,13 +297,12 @@ class Orchestrator { loopStep = undefined break } - let item = [] if ( typeof loopStep.inputs.binding === "string" && loopStep.inputs.option === "String" ) { - item = automationUtils.stringSplit(newInput.binding) + item = automationUtils.stringSplit(loopStep.inputs.binding) } else if (Array.isArray(loopStep.inputs.binding)) { item = loopStep.inputs.binding } @@ -351,6 +344,7 @@ class Orchestrator { } } } + if ( index === env.AUTOMATION_MAX_ITERATIONS || index === parseInt(loopStep.inputs.iterations) @@ -439,6 +433,7 @@ class Orchestrator { break } } + console.log("end of loop!") } if (loopStep && iterations === 0) { From 0a91e5bed1c8a9d73dfdefbceb8e8aa72aa70131 Mon Sep 17 00:00:00 2001 From: Peter Clement Date: Tue, 30 May 2023 14:26:49 +0100 Subject: [PATCH 35/46] update let to const --- packages/server/src/threads/automation.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/threads/automation.ts b/packages/server/src/threads/automation.ts index b9ef30cbca..af61f3bf0a 100644 --- a/packages/server/src/threads/automation.ts +++ b/packages/server/src/threads/automation.ts @@ -38,7 +38,7 @@ const CRON_STEP_ID = triggerDefs.CRON.stepId const STOPPED_STATUS = { success: true, status: AutomationStatus.STOPPED } function getLoopIterations(loopStep: LoopStep) { - let binding = loopStep.inputs.binding + const binding = loopStep.inputs.binding if (!binding) { return 0 } From 3119ba5a8db8245179437f9f4c04b46c1612edc3 Mon Sep 17 00:00:00 2001 From: Peter Clement Date: Tue, 30 May 2023 14:41:04 +0100 Subject: [PATCH 36/46] remove log --- packages/server/src/threads/automation.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/server/src/threads/automation.ts b/packages/server/src/threads/automation.ts index af61f3bf0a..a24def349f 100644 --- a/packages/server/src/threads/automation.ts +++ b/packages/server/src/threads/automation.ts @@ -344,7 +344,6 @@ class Orchestrator { } } } - if ( index === env.AUTOMATION_MAX_ITERATIONS || index === parseInt(loopStep.inputs.iterations) @@ -433,7 +432,6 @@ class Orchestrator { break } } - console.log("end of loop!") } if (loopStep && iterations === 0) { From 91a8888890925bb57c7cee7c5b35ce79b9b5c393 Mon Sep 17 00:00:00 2001 From: Pedro Silva Date: Tue, 30 May 2023 15:08:37 +0100 Subject: [PATCH 37/46] Add teardown logic for multitenancy --- .../src/account-api/api/apis/AccountAPI.ts | 6 ++++++ qa-core/src/jest/globalSetup.ts | 10 +++++---- qa-core/src/jest/globalTeardown.ts | 21 +++++++++++++++++-- 3 files changed, 31 insertions(+), 6 deletions(-) diff --git a/qa-core/src/account-api/api/apis/AccountAPI.ts b/qa-core/src/account-api/api/apis/AccountAPI.ts index a97fe29fd0..c18dde3d4a 100644 --- a/qa-core/src/account-api/api/apis/AccountAPI.ts +++ b/qa-core/src/account-api/api/apis/AccountAPI.ts @@ -58,4 +58,10 @@ export default class AccountAPI { } return [response, json] } + + async delete(accountID: string) { + const [response, json] = await this.client.del(`/api/accounts/${accountID}`) + expect(response).toHaveStatusCode(200) + return response + } } diff --git a/qa-core/src/jest/globalSetup.ts b/qa-core/src/jest/globalSetup.ts index e222e7c043..12d227df02 100644 --- a/qa-core/src/jest/globalSetup.ts +++ b/qa-core/src/jest/globalSetup.ts @@ -2,7 +2,7 @@ import { DEFAULT_TENANT_ID, logging } from "@budibase/backend-core" import { AccountInternalAPI } from "../account-api" import * as fixtures from "../internal-api/fixtures" import { BudibaseInternalAPI } from "../internal-api" -import { CreateAccountRequest, Feature } from "@budibase/types" +import { Account, CreateAccountRequest, Feature } from "@budibase/types" import env from "../environment" import { APIRequestOpts } from "../types" @@ -18,13 +18,13 @@ const API_OPTS: APIRequestOpts = { doExpect: false } // @ts-ignore global.qa = {} -async function createAccount() { +async function createAccount(): Promise<[CreateAccountRequest, Account]> { const account = fixtures.accounts.generateAccount() await accountsApi.accounts.validateEmail(account.email, API_OPTS) await accountsApi.accounts.validateTenantId(account.tenantId, API_OPTS) const [res, newAccount] = await accountsApi.accounts.create(account, API_OPTS) await updateLicense(newAccount.accountId) - return account + return [account, newAccount] } const UNLIMITED = { value: -1 } @@ -85,9 +85,11 @@ async function setup() { console.log(`Environment: ${JSON.stringify(env)}`) if (env.multiTenancy) { - const account = await createAccount() + const [account, newAccount] = await createAccount() // @ts-ignore global.qa.tenantId = account.tenantId + // @ts-ignore + global.qa.accountId = newAccount.accountId await loginAsAccount(account) } else { // @ts-ignore diff --git a/qa-core/src/jest/globalTeardown.ts b/qa-core/src/jest/globalTeardown.ts index 9cd7fff041..51f6046f4f 100644 --- a/qa-core/src/jest/globalTeardown.ts +++ b/qa-core/src/jest/globalTeardown.ts @@ -1,7 +1,24 @@ +import { AccountInternalAPI } from "../account-api" +import { BudibaseInternalAPI } from "../internal-api" +import { APIRequestOpts } from "../types" + +const accountsApi = new AccountInternalAPI({}) +const internalApi = new BudibaseInternalAPI({}) + +const API_OPTS: APIRequestOpts = { doExpect: false } + +async function deleteAccount() { + // @ts-ignore + const accountID = global.qa.accountId + await accountsApi.accounts.delete(accountID) +} + async function teardown() { console.log("\nGLOBAL TEARDOWN STARTING") - - // TODO: Delete account and apps after test run + const env = await internalApi.environment.getEnvironment(API_OPTS) + if (env.multiTenancy) { + await deleteAccount() + } console.log("GLOBAL TEARDOWN COMPLETE") } From 6230e62b9be5c99df928591195181e8ce09217d1 Mon Sep 17 00:00:00 2001 From: Rory Powell Date: Thu, 25 May 2023 12:09:12 +0100 Subject: [PATCH 38/46] Automation investigation logging + max doc size fix for automation log --- .../backend-core/src/logging/pino/logger.ts | 1 + packages/pro | 2 +- packages/server/package.json | 1 + .../server/src/automations/logging/index.ts | 21 +++++++++ packages/server/src/environment.ts | 1 + packages/server/src/threads/automation.ts | 44 ++++++++++++------- packages/server/src/threads/index.ts | 6 ++- packages/server/src/threads/utils.ts | 2 + yarn.lock | 7 +++ 9 files changed, 68 insertions(+), 17 deletions(-) diff --git a/packages/backend-core/src/logging/pino/logger.ts b/packages/backend-core/src/logging/pino/logger.ts index cebc78ffc7..c96bc83e04 100644 --- a/packages/backend-core/src/logging/pino/logger.ts +++ b/packages/backend-core/src/logging/pino/logger.ts @@ -96,6 +96,7 @@ if (!env.DISABLE_PINO_LOGGER) { const mergingObject: any = { err: error, + pid: process.pid, ...contextObject, } diff --git a/packages/pro b/packages/pro index 2adc101c1e..86c32b80e0 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 2adc101c1ede13f861f282d702f45b94ab91fd41 +Subproject commit 86c32b80e08d2f19b57dcc2a3159667ac5a86c21 diff --git a/packages/server/package.json b/packages/server/package.json index 9f053401d8..cb53a1618e 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -100,6 +100,7 @@ "mssql": "6.2.3", "mysql2": "2.3.3", "node-fetch": "2.6.7", + "object-sizeof": "2.6.1", "open": "8.4.0", "openai": "^3.2.1", "pg": "8.10.0", diff --git a/packages/server/src/automations/logging/index.ts b/packages/server/src/automations/logging/index.ts index e3cc9d273c..fd47990488 100644 --- a/packages/server/src/automations/logging/index.ts +++ b/packages/server/src/automations/logging/index.ts @@ -2,6 +2,23 @@ import env from "../../environment" import { AutomationResults, Automation, App } from "@budibase/types" import { automations } from "@budibase/pro" import { db as dbUtils } from "@budibase/backend-core" +import sizeof from "object-sizeof" + +const MAX_LOG_SIZE_MB = 5 +const MB_IN_BYTES = 1024 * 1024 + +function sanitiseResults(results: AutomationResults) { + const message = `[removed] - max results size of ${MAX_LOG_SIZE_MB}MB exceeded` + for (let step of results.steps) { + step.inputs = { + message, + } + step.outputs = { + message, + success: step.outputs.success + } + } +} export async function storeLog( automation: Automation, @@ -11,6 +28,10 @@ export async function storeLog( if (env.DISABLE_AUTOMATION_LOGS) { return } + const bytes = sizeof(results) + if ((bytes / MB_IN_BYTES) > MAX_LOG_SIZE_MB) { + sanitiseResults(results) + } await automations.logs.storeLog(automation, results) } diff --git a/packages/server/src/environment.ts b/packages/server/src/environment.ts index 3e5e475360..0ba708e7bb 100644 --- a/packages/server/src/environment.ts +++ b/packages/server/src/environment.ts @@ -80,6 +80,7 @@ const environment = { ENABLE_ANALYTICS: process.env.ENABLE_ANALYTICS, SELF_HOSTED: process.env.SELF_HOSTED, HTTP_MB_LIMIT: process.env.HTTP_MB_LIMIT, + FORKED_PROCESS_NAME: process.env.FORKED_PROCESS_NAME || "main", // old CLIENT_ID: process.env.CLIENT_ID, _set(key: string, value: any) { diff --git a/packages/server/src/threads/automation.ts b/packages/server/src/threads/automation.ts index 9db647b693..7959f60724 100644 --- a/packages/server/src/threads/automation.ts +++ b/packages/server/src/threads/automation.ts @@ -19,6 +19,7 @@ import { AutomationStatus, AutomationMetadata, AutomationJob, + AutomationData, } from "@budibase/types" import { LoopStep, @@ -480,7 +481,16 @@ class Orchestrator { } // store the logs for the automation run - await storeLog(this._automation, this.executionOutput) + try { + await storeLog(this._automation, this.executionOutput) + } catch (e: any) { + if (e.status === 413 && e.request?.data) { + // if content is too large we shouldn't log it + delete e.request.data + e.request.data = { message: "removed due to large size" } + } + logging.logAlert("Error writing automation log", e) + } if (isProdAppID(this._appId) && isRecurring(automation) && metadata) { await this.updateMetadata(metadata) } @@ -488,24 +498,28 @@ class Orchestrator { } } -export function execute(job: Job, callback: WorkerCallback) { +export function execute(job: Job, callback: WorkerCallback) { const appId = job.data.event.appId + const automationId = job.data.automation._id if (!appId) { throw new Error("Unable to execute, event doesn't contain app ID.") } - return context.doInAppContext(appId, async () => { - const envVars = await sdkUtils.getEnvironmentVariables() - // put into automation thread for whole context - await context.doInEnvironmentContext(envVars, async () => { - const automationOrchestrator = new Orchestrator(job) - try { - const response = await automationOrchestrator.execute() - callback(null, response) - } catch (err) { - callback(err) - } - }) - }) + if (!automationId) { + throw new Error("Unable to execute, event doesn't contain automation ID.") + } + return context.doInAutomationContext({ appId, automationId, task: async () => { + const envVars = await sdkUtils.getEnvironmentVariables() + // put into automation thread for whole context + await context.doInEnvironmentContext(envVars, async () => { + const automationOrchestrator = new Orchestrator(job) + try { + const response = await automationOrchestrator.execute() + callback(null, response) + } catch (err) { + callback(err) + } + }) + }}) } export function executeSynchronously(job: Job) { diff --git a/packages/server/src/threads/index.ts b/packages/server/src/threads/index.ts index 9b6bffa867..6c03e5b464 100644 --- a/packages/server/src/threads/index.ts +++ b/packages/server/src/threads/index.ts @@ -38,6 +38,7 @@ export class Thread { this.count = opts.count ? opts.count : 1 this.disableThreading = this.shouldDisableThreading() if (!this.disableThreading) { + console.debug(`[${env.FORKED_PROCESS_NAME}] initialising worker farm type=${type}`) const workerOpts: any = { autoStart: true, maxConcurrentWorkers: this.count, @@ -45,6 +46,7 @@ export class Thread { env: { ...process.env, FORKED_PROCESS: "1", + FORKED_PROCESS_NAME: type, }, }, } @@ -54,6 +56,8 @@ export class Thread { } this.workers = workerFarm(workerOpts, typeToFile(type), ["execute"]) Thread.workerRefs.push(this.workers) + } else { + console.debug(`[${env.FORKED_PROCESS_NAME}] skipping worker farm type=${type}`) } } @@ -73,7 +77,7 @@ export class Thread { worker.execute(job, (err: any, response: any) => { if (err && err.type === "TimeoutError") { reject( - new Error(`Query response time exceeded ${timeout}ms timeout.`) + new Error(`Thread timeout exceeded ${timeout}ms timeout.`) ) } else if (err) { reject(err) diff --git a/packages/server/src/threads/utils.ts b/packages/server/src/threads/utils.ts index 37c7a6d3f9..a0f3bbdc47 100644 --- a/packages/server/src/threads/utils.ts +++ b/packages/server/src/threads/utils.ts @@ -26,8 +26,10 @@ function makeVariableKey(queryId: string, variable: string) { export function threadSetup() { // don't run this if not threading if (env.isTest() || env.DISABLE_THREADING || !env.isInThread()) { + console.debug(`[${env.FORKED_PROCESS_NAME}] thread setup skipped`) return } + console.debug(`[${env.FORKED_PROCESS_NAME}] thread setup running`) db.init() } diff --git a/yarn.lock b/yarn.lock index c5aad94d5b..c365222ce3 100644 --- a/yarn.lock +++ b/yarn.lock @@ -19043,6 +19043,13 @@ object-keys@~0.4.0: resolved "https://registry.yarnpkg.com/object-keys/-/object-keys-0.4.0.tgz#28a6aae7428dd2c3a92f3d95f21335dd204e0336" integrity sha512-ncrLw+X55z7bkl5PnUvHwFK9FcGuFYo9gtjws2XtSzL+aZ8tm830P60WJ0dSmFVaSalWieW5MD7kEdnXda9yJw== +object-sizeof@2.6.1: + version "2.6.1" + resolved "https://registry.yarnpkg.com/object-sizeof/-/object-sizeof-2.6.1.tgz#1e2b6a01d182c268dbb07ee3403f539de45f63d3" + integrity sha512-a7VJ1Zx7ZuHceKwjgfsSqzV/X0PVGvpZz7ho3Dn4Cs0LLcR5e5WuV+gsbizmplD8s0nAXMJmckKB2rkSiPm/Gg== + dependencies: + buffer "^6.0.3" + object-visit@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/object-visit/-/object-visit-1.0.1.tgz#f79c4493af0c5377b59fe39d395e41042dd045bb" From ed6cd8144bb8bb0cffe595ecc1956bb728cf0361 Mon Sep 17 00:00:00 2001 From: Peter Clement Date: Tue, 30 May 2023 16:52:22 +0100 Subject: [PATCH 39/46] lint --- packages/server/src/threads/automation.ts | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/packages/server/src/threads/automation.ts b/packages/server/src/threads/automation.ts index a24def349f..6dde6a5c99 100644 --- a/packages/server/src/threads/automation.ts +++ b/packages/server/src/threads/automation.ts @@ -38,7 +38,7 @@ const CRON_STEP_ID = triggerDefs.CRON.stepId const STOPPED_STATUS = { success: true, status: AutomationStatus.STOPPED } function getLoopIterations(loopStep: LoopStep) { - const binding = loopStep.inputs.binding + let binding = loopStep.inputs.binding if (!binding) { return 0 } @@ -251,7 +251,7 @@ class Orchestrator { return } } - + const start = performance.now() for (let step of automation.definition.steps) { if (timeoutFlag) { break @@ -344,6 +344,7 @@ class Orchestrator { } } } + if ( index === env.AUTOMATION_MAX_ITERATIONS || index === parseInt(loopStep.inputs.iterations) @@ -432,6 +433,7 @@ class Orchestrator { break } } + console.log("end of loop!") } if (loopStep && iterations === 0) { @@ -472,6 +474,11 @@ class Orchestrator { } } + const end = performance.now() + const executionTime = end - start + + console.log(`Execution time: ${executionTime} milliseconds`) + // store the logs for the automation run await storeLog(this._automation, this.executionOutput) if (isProdAppID(this._appId) && isRecurring(automation) && metadata) { From 7c7bd4d5cba5f033c6f00362bda65092b9b6e1fc Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 30 May 2023 17:41:20 +0100 Subject: [PATCH 40/46] Fix for debugging with webstorm the old way (if desired), updating the builder middleware to be more multi-dev capable, ignoring 409s when attempting to update the last updated at for apps (if multiple devs hit at same time, only use one) also updating writethrough cache to retry once, with the extended TTL on locks, plus the multi-dev collab it can take a minute to update usage quota doc when a lot of updates occur at once. --- .../backend-core/src/cache/writethrough.ts | 2 +- .../backend-core/src/redis/redlockImpl.ts | 38 ++++++++++++------- packages/server/src/middleware/builder.ts | 35 ++++++++++------- .../server/src/utilities/fileSystem/app.ts | 18 +++++++-- packages/types/src/sdk/locks.ts | 1 + 5 files changed, 62 insertions(+), 32 deletions(-) diff --git a/packages/backend-core/src/cache/writethrough.ts b/packages/backend-core/src/cache/writethrough.ts index e64c116663..d399b17896 100644 --- a/packages/backend-core/src/cache/writethrough.ts +++ b/packages/backend-core/src/cache/writethrough.ts @@ -44,7 +44,7 @@ async function put( if (updateDb) { const lockResponse = await locks.doWithLock( { - type: LockType.TRY_ONCE, + type: LockType.TRY_TWICE, name: LockName.PERSIST_WRITETHROUGH, resource: key, ttl: 15000, diff --git a/packages/backend-core/src/redis/redlockImpl.ts b/packages/backend-core/src/redis/redlockImpl.ts index 55b891ea84..0698335b7c 100644 --- a/packages/backend-core/src/redis/redlockImpl.ts +++ b/packages/backend-core/src/redis/redlockImpl.ts @@ -4,10 +4,10 @@ import { LockOptions, LockType } from "@budibase/types" import * as context from "../context" import env from "../environment" -const getClient = async ( +async function getClient( type: LockType, opts?: Redlock.Options -): Promise => { +): Promise { if (type === LockType.CUSTOM) { return newRedlock(opts) } @@ -18,6 +18,9 @@ const getClient = async ( case LockType.TRY_ONCE: { return newRedlock(OPTIONS.TRY_ONCE) } + case LockType.TRY_TWICE: { + return newRedlock(OPTIONS.TRY_TWICE) + } case LockType.DEFAULT: { return newRedlock(OPTIONS.DEFAULT) } @@ -35,6 +38,9 @@ const OPTIONS = { // immediately throws an error if the lock is already held retryCount: 0, }, + TRY_TWICE: { + retryCount: 1, + }, TEST: { // higher retry count in unit tests // due to high contention. @@ -62,7 +68,7 @@ const OPTIONS = { }, } -const newRedlock = async (opts: Redlock.Options = {}) => { +export async function newRedlock(opts: Redlock.Options = {}) { let options = { ...OPTIONS.DEFAULT, ...opts } const redisWrapper = await getLockClient() const client = redisWrapper.getClient() @@ -81,22 +87,26 @@ type RedlockExecution = | SuccessfulRedlockExecution | UnsuccessfulRedlockExecution -export const doWithLock = async ( +function getLockName(opts: LockOptions) { + // determine lock name + // by default use the tenantId for uniqueness, unless using a system lock + const prefix = opts.systemLock ? "system" : context.getTenantId() + let name: string = `lock:${prefix}_${opts.name}` + // add additional unique name if required + if (opts.resource) { + name = name + `_${opts.resource}` + } + return name +} + +export async function doWithLock( opts: LockOptions, task: () => Promise -): Promise> => { +): Promise> { const redlock = await getClient(opts.type, opts.customOptions) let lock try { - // determine lock name - // by default use the tenantId for uniqueness, unless using a system lock - const prefix = opts.systemLock ? "system" : context.getTenantId() - let name: string = `lock:${prefix}_${opts.name}` - - // add additional unique name if required - if (opts.resource) { - name = name + `_${opts.resource}` - } + const name = getLockName(opts) // create the lock lock = await redlock.lock(name, opts.ttl) diff --git a/packages/server/src/middleware/builder.ts b/packages/server/src/middleware/builder.ts index ffb2e2c002..31c4da127c 100644 --- a/packages/server/src/middleware/builder.ts +++ b/packages/server/src/middleware/builder.ts @@ -9,8 +9,8 @@ import { checkDebounce, setDebounce, } from "../utilities/redis" -import { db as dbCore, cache, permissions } from "@budibase/backend-core" -import { BBContext, Database } from "@budibase/types" +import { db as dbCore, cache } from "@budibase/backend-core" +import { UserCtx, Database } from "@budibase/types" const DEBOUNCE_TIME_SEC = 30 @@ -23,7 +23,7 @@ const DEBOUNCE_TIME_SEC = 30 * through the authorized middleware * ****************************************************/ -async function checkDevAppLocks(ctx: BBContext) { +async function checkDevAppLocks(ctx: UserCtx) { const appId = ctx.appId // if any public usage, don't proceed @@ -42,7 +42,7 @@ async function checkDevAppLocks(ctx: BBContext) { } } -async function updateAppUpdatedAt(ctx: BBContext) { +async function updateAppUpdatedAt(ctx: UserCtx) { const appId = ctx.appId // if debouncing skip this update // get methods also aren't updating @@ -50,20 +50,29 @@ async function updateAppUpdatedAt(ctx: BBContext) { return } await dbCore.doWithDB(appId, async (db: Database) => { - const metadata = await db.get(DocumentType.APP_METADATA) - metadata.updatedAt = new Date().toISOString() + try { + const metadata = await db.get(DocumentType.APP_METADATA) + metadata.updatedAt = new Date().toISOString() - metadata.updatedBy = getGlobalIDFromUserMetadataID(ctx.user?.userId!) + metadata.updatedBy = getGlobalIDFromUserMetadataID(ctx.user?.userId!) - const response = await db.put(metadata) - metadata._rev = response.rev - await cache.app.invalidateAppMetadata(appId, metadata) - // set a new debounce record with a short TTL - await setDebounce(appId, DEBOUNCE_TIME_SEC) + const response = await db.put(metadata) + metadata._rev = response.rev + await cache.app.invalidateAppMetadata(appId, metadata) + // set a new debounce record with a short TTL + await setDebounce(appId, DEBOUNCE_TIME_SEC) + } catch (err: any) { + // if a 409 occurs, then multiple clients connected at the same time - ignore + if (err?.status === 409) { + return + } else { + throw err + } + } }) } -export default async function builder(ctx: BBContext) { +export default async function builder(ctx: UserCtx) { const appId = ctx.appId // this only functions within an app context if (!appId) { diff --git a/packages/server/src/utilities/fileSystem/app.ts b/packages/server/src/utilities/fileSystem/app.ts index 25600ee3f1..16681c2978 100644 --- a/packages/server/src/utilities/fileSystem/app.ts +++ b/packages/server/src/utilities/fileSystem/app.ts @@ -35,10 +35,20 @@ export const getComponentLibraryManifest = async (library: string) => { const filename = "manifest.json" if (env.isDev() || env.isTest()) { - const path = join(TOP_LEVEL_PATH, "packages/client", filename) - // always load from new so that updates are refreshed - delete require.cache[require.resolve(path)] - return require(path) + const paths = [ + join(TOP_LEVEL_PATH, "packages/client", filename), + join(process.cwd(), "client", filename), + ] + for (let path of paths) { + if (fs.existsSync(path)) { + // always load from new so that updates are refreshed + delete require.cache[require.resolve(path)] + return require(path) + } + } + throw new Error( + `Unable to find ${filename} in development environment (may need to build).` + ) } if (!appId) { diff --git a/packages/types/src/sdk/locks.ts b/packages/types/src/sdk/locks.ts index 6147308f7d..a35e7b379b 100644 --- a/packages/types/src/sdk/locks.ts +++ b/packages/types/src/sdk/locks.ts @@ -6,6 +6,7 @@ export enum LockType { * No retries will take place and no error will be thrown. */ TRY_ONCE = "try_once", + TRY_TWICE = "try_twice", DEFAULT = "default", DELAY_500 = "delay_500", CUSTOM = "custom", From b0783d373ac2b3376059623b0c8538969ddd0751 Mon Sep 17 00:00:00 2001 From: Rory Powell Date: Tue, 30 May 2023 19:16:36 +0100 Subject: [PATCH 41/46] Lint --- packages/server/src/automations/logging/index.ts | 4 ++-- packages/server/src/threads/automation.ts | 8 ++++++-- packages/server/src/threads/index.ts | 12 +++++++----- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/packages/server/src/automations/logging/index.ts b/packages/server/src/automations/logging/index.ts index fd47990488..9d16f15a67 100644 --- a/packages/server/src/automations/logging/index.ts +++ b/packages/server/src/automations/logging/index.ts @@ -15,7 +15,7 @@ function sanitiseResults(results: AutomationResults) { } step.outputs = { message, - success: step.outputs.success + success: step.outputs.success, } } } @@ -29,7 +29,7 @@ export async function storeLog( return } const bytes = sizeof(results) - if ((bytes / MB_IN_BYTES) > MAX_LOG_SIZE_MB) { + if (bytes / MB_IN_BYTES > MAX_LOG_SIZE_MB) { sanitiseResults(results) } await automations.logs.storeLog(automation, results) diff --git a/packages/server/src/threads/automation.ts b/packages/server/src/threads/automation.ts index 0d5586c235..60a07dab09 100644 --- a/packages/server/src/threads/automation.ts +++ b/packages/server/src/threads/automation.ts @@ -507,7 +507,10 @@ export function execute(job: Job, callback: WorkerCallback) { if (!automationId) { throw new Error("Unable to execute, event doesn't contain automation ID.") } - return context.doInAutomationContext({ appId, automationId, task: async () => { + return context.doInAutomationContext({ + appId, + automationId, + task: async () => { const envVars = await sdkUtils.getEnvironmentVariables() // put into automation thread for whole context await context.doInEnvironmentContext(envVars, async () => { @@ -519,7 +522,8 @@ export function execute(job: Job, callback: WorkerCallback) { callback(err) } }) - }}) + }, + }) } export function executeSynchronously(job: Job) { diff --git a/packages/server/src/threads/index.ts b/packages/server/src/threads/index.ts index 6c03e5b464..6afaa9bb4e 100644 --- a/packages/server/src/threads/index.ts +++ b/packages/server/src/threads/index.ts @@ -38,7 +38,9 @@ export class Thread { this.count = opts.count ? opts.count : 1 this.disableThreading = this.shouldDisableThreading() if (!this.disableThreading) { - console.debug(`[${env.FORKED_PROCESS_NAME}] initialising worker farm type=${type}`) + console.debug( + `[${env.FORKED_PROCESS_NAME}] initialising worker farm type=${type}` + ) const workerOpts: any = { autoStart: true, maxConcurrentWorkers: this.count, @@ -57,7 +59,9 @@ export class Thread { this.workers = workerFarm(workerOpts, typeToFile(type), ["execute"]) Thread.workerRefs.push(this.workers) } else { - console.debug(`[${env.FORKED_PROCESS_NAME}] skipping worker farm type=${type}`) + console.debug( + `[${env.FORKED_PROCESS_NAME}] skipping worker farm type=${type}` + ) } } @@ -76,9 +80,7 @@ export class Thread { function fire(worker: any) { worker.execute(job, (err: any, response: any) => { if (err && err.type === "TimeoutError") { - reject( - new Error(`Thread timeout exceeded ${timeout}ms timeout.`) - ) + reject(new Error(`Thread timeout exceeded ${timeout}ms timeout.`)) } else if (err) { reject(err) } else { From a1dbd6753509f1f904668ddca006b46cc005d870 Mon Sep 17 00:00:00 2001 From: Rory Powell Date: Tue, 30 May 2023 19:23:19 +0100 Subject: [PATCH 42/46] Remove debug log --- packages/server/src/threads/automation.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/server/src/threads/automation.ts b/packages/server/src/threads/automation.ts index 60a07dab09..563af93303 100644 --- a/packages/server/src/threads/automation.ts +++ b/packages/server/src/threads/automation.ts @@ -434,7 +434,6 @@ class Orchestrator { break } } - console.log("end of loop!") } if (loopStep && iterations === 0) { From e9fd89d0f19e5043132f66278bdf9cd2bf85f262 Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Tue, 30 May 2023 18:28:02 +0000 Subject: [PATCH 43/46] Bump version to 2.6.19-alpha.31 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index 344e5ffffc..d00adb3e92 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.6.19-alpha.30", + "version": "2.6.19-alpha.31", "npmClient": "yarn", "packages": [ "packages/backend-core", From 5249148d6cf4c5f686288a3085e01eed90e8e769 Mon Sep 17 00:00:00 2001 From: Michael Drury Date: Tue, 30 May 2023 20:20:22 +0100 Subject: [PATCH 44/46] Updating writethrough test to be aware of the double attempt locks. --- .../src/cache/tests/writethrough.spec.ts | 28 +++++++++++-------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/packages/backend-core/src/cache/tests/writethrough.spec.ts b/packages/backend-core/src/cache/tests/writethrough.spec.ts index e4c7cc6e64..d3125537a8 100644 --- a/packages/backend-core/src/cache/tests/writethrough.spec.ts +++ b/packages/backend-core/src/cache/tests/writethrough.spec.ts @@ -72,20 +72,26 @@ describe("writethrough", () => { writethrough.put({ ...current, value: 4 }), ]) - const newRev = responses.map(x => x.rev).find(x => x !== current._rev) - expect(newRev).toBeDefined() - expect(responses.map(x => x.rev)).toEqual( - expect.arrayContaining([current._rev, current._rev, newRev]) - ) - expectFunctionWasCalledTimesWith( - mocks.alerts.logWarn, - 2, - "Ignoring redlock conflict in write-through cache" - ) + // with a lock, this will work + const revs = responses.map(x => x.rev) + const startWith = ["3", "4", "5"] + const found = [] + let maxRev + for (let starting of startWith) { + for (let rev of revs) { + if (rev?.startsWith(starting)) { + found.push(starting) + } + if (rev?.startsWith("5")) { + maxRev = rev + } + } + } + expect(found.length).toBe(3) const output = await db.get(current._id) expect(output.value).toBe(4) - expect(output._rev).toBe(newRev) + expect(output._rev).toBe(maxRev) current = output }) From 99607ca06e058205d80b2186ae04154f2175369e Mon Sep 17 00:00:00 2001 From: Michael Drury Date: Tue, 30 May 2023 20:45:10 +0100 Subject: [PATCH 45/46] Reverting try twice change to writethrough. --- .../src/cache/tests/writethrough.spec.ts | 22 +++++-------------- .../backend-core/src/cache/writethrough.ts | 2 +- .../backend-core/src/redis/redlockImpl.ts | 1 - 3 files changed, 7 insertions(+), 18 deletions(-) diff --git a/packages/backend-core/src/cache/tests/writethrough.spec.ts b/packages/backend-core/src/cache/tests/writethrough.spec.ts index d3125537a8..92b073ed64 100644 --- a/packages/backend-core/src/cache/tests/writethrough.spec.ts +++ b/packages/backend-core/src/cache/tests/writethrough.spec.ts @@ -73,25 +73,15 @@ describe("writethrough", () => { ]) // with a lock, this will work - const revs = responses.map(x => x.rev) - const startWith = ["3", "4", "5"] - const found = [] - let maxRev - for (let starting of startWith) { - for (let rev of revs) { - if (rev?.startsWith(starting)) { - found.push(starting) - } - if (rev?.startsWith("5")) { - maxRev = rev - } - } - } - expect(found.length).toBe(3) + const newRev = responses.map(x => x.rev).find(x => x !== current._rev) + expect(newRev).toBeDefined() + expect(responses.map(x => x.rev)).toEqual( + expect.arrayContaining([current._rev, current._rev, newRev]) + ) const output = await db.get(current._id) expect(output.value).toBe(4) - expect(output._rev).toBe(maxRev) + expect(output._rev).toBe(newRev) current = output }) diff --git a/packages/backend-core/src/cache/writethrough.ts b/packages/backend-core/src/cache/writethrough.ts index d399b17896..e64c116663 100644 --- a/packages/backend-core/src/cache/writethrough.ts +++ b/packages/backend-core/src/cache/writethrough.ts @@ -44,7 +44,7 @@ async function put( if (updateDb) { const lockResponse = await locks.doWithLock( { - type: LockType.TRY_TWICE, + type: LockType.TRY_ONCE, name: LockName.PERSIST_WRITETHROUGH, resource: key, ttl: 15000, diff --git a/packages/backend-core/src/redis/redlockImpl.ts b/packages/backend-core/src/redis/redlockImpl.ts index 0698335b7c..7fe61a409e 100644 --- a/packages/backend-core/src/redis/redlockImpl.ts +++ b/packages/backend-core/src/redis/redlockImpl.ts @@ -122,7 +122,6 @@ export async function doWithLock( if (opts.type === LockType.TRY_ONCE) { // don't throw for try-once locks, they will always error // due to retry count (0) exceeded - console.warn(e) return { executed: false } } else { console.error(e) From 3f9edcff5cefc208c9da19a3c64683f3116bb467 Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Tue, 30 May 2023 22:01:08 +0000 Subject: [PATCH 46/46] Bump version to 2.6.19-alpha.32 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index d00adb3e92..e72132901e 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.6.19-alpha.31", + "version": "2.6.19-alpha.32", "npmClient": "yarn", "packages": [ "packages/backend-core",