From 1040407068bc0f32ce1b27a7edaf268447e66c74 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Fri, 9 Jun 2023 13:11:18 +0100 Subject: [PATCH 01/20] Reset rows and tick before updating schema when swapping table ID in grids --- .../frontend-core/src/components/grid/stores/rows.js | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/packages/frontend-core/src/components/grid/stores/rows.js b/packages/frontend-core/src/components/grid/stores/rows.js index 198c05025c..2020708fa8 100644 --- a/packages/frontend-core/src/components/grid/stores/rows.js +++ b/packages/frontend-core/src/components/grid/stores/rows.js @@ -2,6 +2,7 @@ import { writable, derived, get } from "svelte/store" import { fetchData } from "../../../fetch/fetchData" import { notifications } from "@budibase/bbui" import { NewRowID, RowPageSize } from "../lib/constants" +import { tick } from "svelte" const initialSortState = { column: null, @@ -124,13 +125,22 @@ export const deriveStores = context => { }) // Subscribe to changes of this fetch model - unsubscribe = newFetch.subscribe($fetch => { + unsubscribe = newFetch.subscribe(async $fetch => { if ($fetch.loaded && !$fetch.loading) { hasNextPage.set($fetch.hasNextPage) const $instanceLoaded = get(instanceLoaded) const resetRows = $fetch.resetKey !== lastResetKey + const previousResetKey = lastResetKey lastResetKey = $fetch.resetKey + // If resetting rows due to a table change, wipe data and wait for + // derived stores to compute. This prevents stale data being passed + // to cells when we save the new schema. + if (!$instanceLoaded && previousResetKey) { + rows.set([]) + await tick() + } + // Reset state properties when dataset changes if (!$instanceLoaded || resetRows) { table.set($fetch.definition) From 669a05b6406199ebf57508f40fd400aa95beb73b Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Tue, 13 Jun 2023 15:07:38 +0100 Subject: [PATCH 02/20] Fix field on change actions not properly checking for unchanged values --- .../client/src/components/app/forms/InnerForm.svelte | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/packages/client/src/components/app/forms/InnerForm.svelte b/packages/client/src/components/app/forms/InnerForm.svelte index 27e854781c..9eb21b0bae 100644 --- a/packages/client/src/components/app/forms/InnerForm.svelte +++ b/packages/client/src/components/app/forms/InnerForm.svelte @@ -283,7 +283,7 @@ // Skip if the value is the same if (!skipCheck && fieldState.value === value) { - return true + return false } // Update field state @@ -295,7 +295,7 @@ return state }) - return !error + return true } // Clears the value of a certain field back to the default value @@ -376,8 +376,9 @@ deregister, validate: () => { // Validate the field by force setting the same value again - const { fieldState } = get(getField(field)) - return setValue(fieldState.value, true) + const fieldInfo = getField(field) + setValue(get(fieldInfo).fieldState.value, true) + return !get(fieldInfo).fieldState.error }, } } From 5e5dc902d1ce91fa7e112db28c4e98025cae2362 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Thu, 15 Jun 2023 09:04:21 +0100 Subject: [PATCH 03/20] Broadcast datasource change via websocket when making changes to tables --- packages/builder/src/stores/backend/tables.js | 2 - .../src/api/controllers/table/external.ts | 8 ++ packages/server/src/websockets/builder.ts | 80 +++++++++++++------ packages/server/src/websockets/grid.ts | 54 +++++++++---- packages/server/src/websockets/websocket.ts | 22 +++-- packages/types/src/sdk/websocket.ts | 4 + 6 files changed, 125 insertions(+), 45 deletions(-) diff --git a/packages/builder/src/stores/backend/tables.js b/packages/builder/src/stores/backend/tables.js index f8796712a8..d79ed6f072 100644 --- a/packages/builder/src/stores/backend/tables.js +++ b/packages/builder/src/stores/backend/tables.js @@ -1,5 +1,4 @@ import { get, writable, derived } from "svelte/store" -import { datasources } from "./" import { cloneDeep } from "lodash/fp" import { API } from "api" import { SWITCHABLE_TYPES } from "constants/backend" @@ -63,7 +62,6 @@ export function createTablesStore() { const savedTable = await API.saveTable(updatedTable) replaceTable(savedTable._id, savedTable) - await datasources.fetch() select(savedTable._id) return savedTable } diff --git a/packages/server/src/api/controllers/table/external.ts b/packages/server/src/api/controllers/table/external.ts index ee789ddd3a..f35691cb62 100644 --- a/packages/server/src/api/controllers/table/external.ts +++ b/packages/server/src/api/controllers/table/external.ts @@ -26,6 +26,7 @@ import { RelationshipTypes, } from "@budibase/types" import sdk from "../../../sdk" +import { builderSocket } from "../../../websockets" const { cloneDeep } = require("lodash/fp") async function makeTableRequest( @@ -318,6 +319,13 @@ export async function save(ctx: UserCtx) { datasource.entities[tableToSave.name] = tableToSave await db.put(datasource) + // Since tables are stored inside datasources, we need to notify clients + // that the datasource definition changed + const updatedDatasource = await db.get(datasource._id) + builderSocket?.emitDatasourceUpdate(ctx, updatedDatasource, { + includeOriginator: true, + }) + return tableToSave } diff --git a/packages/server/src/websockets/builder.ts b/packages/server/src/websockets/builder.ts index 0f2c43e5ab..53b0a10905 100644 --- a/packages/server/src/websockets/builder.ts +++ b/packages/server/src/websockets/builder.ts @@ -3,7 +3,13 @@ import { BaseSocket } from "./websocket" import { permissions, events } from "@budibase/backend-core" import http from "http" import Koa from "koa" -import { Datasource, Table, SocketSession, ContextUser } from "@budibase/types" +import { + Datasource, + Table, + SocketSession, + ContextUser, + SocketMessageOptions, +} from "@budibase/types" import { gridSocket } from "./index" import { clearLock, updateLock } from "../utilities/redis" import { Socket } from "socket.io" @@ -66,33 +72,61 @@ export default class BuilderSocket extends BaseSocket { } } - emitTableUpdate(ctx: any, table: Table) { - this.emitToRoom(ctx, ctx.appId, BuilderSocketEvent.TableChange, { - id: table._id, - table, - }) - gridSocket?.emitTableUpdate(ctx, table) + emitTableUpdate(ctx: any, table: Table, options?: SocketMessageOptions) { + this.emitToRoom( + ctx, + ctx.appId, + BuilderSocketEvent.TableChange, + { + id: table._id, + table, + }, + options + ) + gridSocket?.emitTableUpdate(ctx, table, options) } - emitTableDeletion(ctx: any, id: string) { - this.emitToRoom(ctx, ctx.appId, BuilderSocketEvent.TableChange, { - id, - table: null, - }) - gridSocket?.emitTableDeletion(ctx, id) + emitTableDeletion(ctx: any, id: string, options?: SocketMessageOptions) { + this.emitToRoom( + ctx, + ctx.appId, + BuilderSocketEvent.TableChange, + { + id, + table: null, + }, + options + ) + gridSocket?.emitTableDeletion(ctx, id, options) } - emitDatasourceUpdate(ctx: any, datasource: Datasource) { - this.emitToRoom(ctx, ctx.appId, BuilderSocketEvent.DatasourceChange, { - id: datasource._id, - datasource, - }) + emitDatasourceUpdate( + ctx: any, + datasource: Datasource, + options?: SocketMessageOptions + ) { + this.emitToRoom( + ctx, + ctx.appId, + BuilderSocketEvent.DatasourceChange, + { + id: datasource._id, + datasource, + }, + options + ) } - emitDatasourceDeletion(ctx: any, id: string) { - this.emitToRoom(ctx, ctx.appId, BuilderSocketEvent.DatasourceChange, { - id, - datasource: null, - }) + emitDatasourceDeletion(ctx: any, id: string, options?: SocketMessageOptions) { + this.emitToRoom( + ctx, + ctx.appId, + BuilderSocketEvent.DatasourceChange, + { + id, + datasource: null, + }, + options + ) } } diff --git a/packages/server/src/websockets/grid.ts b/packages/server/src/websockets/grid.ts index 6731c2d899..38ec091816 100644 --- a/packages/server/src/websockets/grid.ts +++ b/packages/server/src/websockets/grid.ts @@ -4,7 +4,7 @@ import { permissions } from "@budibase/backend-core" import http from "http" import Koa from "koa" import { getTableId } from "../api/controllers/row/utils" -import { Row, Table } from "@budibase/types" +import { Row, SocketMessageOptions, Table } from "@budibase/types" import { Socket } from "socket.io" import { GridSocketEvent } from "@budibase/shared-core" @@ -29,27 +29,51 @@ export default class GridSocket extends BaseSocket { }) } - emitRowUpdate(ctx: any, row: Row) { + emitRowUpdate(ctx: any, row: Row, options?: SocketMessageOptions) { const tableId = getTableId(ctx) - this.emitToRoom(ctx, tableId, GridSocketEvent.RowChange, { - id: row._id, - row, - }) + this.emitToRoom( + ctx, + tableId, + GridSocketEvent.RowChange, + { + id: row._id, + row, + }, + options + ) } - emitRowDeletion(ctx: any, id: string) { + emitRowDeletion(ctx: any, id: string, options?: SocketMessageOptions) { const tableId = getTableId(ctx) - this.emitToRoom(ctx, tableId, GridSocketEvent.RowChange, { id, row: null }) + this.emitToRoom( + ctx, + tableId, + GridSocketEvent.RowChange, + { id, row: null }, + options + ) } - emitTableUpdate(ctx: any, table: Table) { - this.emitToRoom(ctx, table._id!, GridSocketEvent.TableChange, { - id: table._id, - table, - }) + emitTableUpdate(ctx: any, table: Table, options?: SocketMessageOptions) { + this.emitToRoom( + ctx, + table._id!, + GridSocketEvent.TableChange, + { + id: table._id, + table, + }, + options + ) } - emitTableDeletion(ctx: any, id: string) { - this.emitToRoom(ctx, id, GridSocketEvent.TableChange, { id, table: null }) + emitTableDeletion(ctx: any, id: string, options?: SocketMessageOptions) { + this.emitToRoom( + ctx, + id, + GridSocketEvent.TableChange, + { id, table: null }, + options + ) } } diff --git a/packages/server/src/websockets/websocket.ts b/packages/server/src/websockets/websocket.ts index d8cc10bda4..59a9d698d7 100644 --- a/packages/server/src/websockets/websocket.ts +++ b/packages/server/src/websockets/websocket.ts @@ -9,7 +9,7 @@ import { createAdapter } from "@socket.io/redis-adapter" import { Socket } from "socket.io" import { getSocketPubSubClients } from "../utilities/redis" import { SocketEvent, SocketSessionTTL } from "@budibase/shared-core" -import { SocketSession } from "@budibase/types" +import { SocketSession, SocketMessageOptions } from "@budibase/types" export class BaseSocket { io: Server @@ -276,12 +276,24 @@ export class BaseSocket { this.io.sockets.emit(event, payload) } - // Emit an event to everyone in a room, including metadata of whom - // the originator of the request was - emitToRoom(ctx: any, room: string, event: string, payload: any) { + // Emit an event to everyone in a room + emitToRoom( + ctx: any, + room: string, + event: string, + payload: any, + options?: SocketMessageOptions + ) { + // By default, we include the session API of the originator so that they can ignore + // this event. If we want to include the originator then we leave it unset to that all + // clients will react to it. + let apiSessionId = null + if (!options?.includeOriginator) { + apiSessionId = ctx.headers?.[Header.SESSION_ID] + } this.io.in(room).emit(event, { ...payload, - apiSessionId: ctx.headers?.[Header.SESSION_ID], + apiSessionId, }) } } diff --git a/packages/types/src/sdk/websocket.ts b/packages/types/src/sdk/websocket.ts index 40e2654e82..770f44d9b2 100644 --- a/packages/types/src/sdk/websocket.ts +++ b/packages/types/src/sdk/websocket.ts @@ -7,3 +7,7 @@ export interface SocketSession { room?: string connectedAt: number } + +export interface SocketMessageOptions { + includeOriginator?: boolean +} From 99a8fc7c123986a430509296ea767926c6f4843d Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Thu, 15 Jun 2023 09:27:45 +0100 Subject: [PATCH 04/20] Revert websocket changes and just fetch datasources constantly --- .../backend/DataTable/DataTable.svelte | 13 ++- .../modals/CreateTableModal.svelte | 1 + .../popovers/EditTablePopover.svelte | 1 + .../src/components/grid/stores/columns.js | 6 +- .../src/api/controllers/table/external.ts | 9 ++- packages/server/src/websockets/builder.ts | 80 ++++++------------- packages/server/src/websockets/grid.ts | 54 ++++--------- packages/server/src/websockets/websocket.ts | 19 +---- packages/types/src/sdk/websocket.ts | 4 - 9 files changed, 63 insertions(+), 124 deletions(-) diff --git a/packages/builder/src/components/backend/DataTable/DataTable.svelte b/packages/builder/src/components/backend/DataTable/DataTable.svelte index 1b0c92bde0..4ed41c5b34 100644 --- a/packages/builder/src/components/backend/DataTable/DataTable.svelte +++ b/packages/builder/src/components/backend/DataTable/DataTable.svelte @@ -1,5 +1,5 @@
@@ -37,7 +46,7 @@ allowDeleteRows={!isUsersTable} schemaOverrides={isUsersTable ? userSchemaOverrides : null} showAvatars={false} - on:updatetable={e => tables.replaceTable(id, e.detail)} + on:updatetable={handleGridTableUpdate} > {#if isInternal} diff --git a/packages/builder/src/components/backend/TableNavigator/modals/CreateTableModal.svelte b/packages/builder/src/components/backend/TableNavigator/modals/CreateTableModal.svelte index bd1761620d..bfca91afaa 100644 --- a/packages/builder/src/components/backend/TableNavigator/modals/CreateTableModal.svelte +++ b/packages/builder/src/components/backend/TableNavigator/modals/CreateTableModal.svelte @@ -93,6 +93,7 @@ try { await beforeSave() table = await tables.save(newTable) + await datasources.fetch() await afterSave(table) } catch (e) { notifications.error(e) diff --git a/packages/builder/src/components/backend/TableNavigator/popovers/EditTablePopover.svelte b/packages/builder/src/components/backend/TableNavigator/popovers/EditTablePopover.svelte index 01c62d56f7..11ef60480b 100644 --- a/packages/builder/src/components/backend/TableNavigator/popovers/EditTablePopover.svelte +++ b/packages/builder/src/components/backend/TableNavigator/popovers/EditTablePopover.svelte @@ -65,6 +65,7 @@ const updatedTable = cloneDeep(table) updatedTable.name = updatedName await tables.save(updatedTable) + await datasources.fetch() notifications.success("Table renamed successfully") } diff --git a/packages/frontend-core/src/components/grid/stores/columns.js b/packages/frontend-core/src/components/grid/stores/columns.js index 024fdc29bc..82a26d923c 100644 --- a/packages/frontend-core/src/components/grid/stores/columns.js +++ b/packages/frontend-core/src/components/grid/stores/columns.js @@ -90,12 +90,12 @@ export const deriveStores = context => { // Update local state table.set(newTable) + // Update server + await API.saveTable(newTable) + // Broadcast change to external state can be updated, as this change // will not be received by the builder websocket because we caused it ourselves dispatch("updatetable", newTable) - - // Update server - await API.saveTable(newTable) } return { diff --git a/packages/server/src/api/controllers/table/external.ts b/packages/server/src/api/controllers/table/external.ts index f35691cb62..24d4242478 100644 --- a/packages/server/src/api/controllers/table/external.ts +++ b/packages/server/src/api/controllers/table/external.ts @@ -322,9 +322,7 @@ export async function save(ctx: UserCtx) { // Since tables are stored inside datasources, we need to notify clients // that the datasource definition changed const updatedDatasource = await db.get(datasource._id) - builderSocket?.emitDatasourceUpdate(ctx, updatedDatasource, { - includeOriginator: true, - }) + builderSocket?.emitDatasourceUpdate(ctx, updatedDatasource) return tableToSave } @@ -352,6 +350,11 @@ export async function destroy(ctx: UserCtx) { await db.put(datasource) + // Since tables are stored inside datasources, we need to notify clients + // that the datasource definition changed + const updatedDatasource = await db.get(datasource._id) + builderSocket?.emitDatasourceUpdate(ctx, updatedDatasource) + return tableToDelete } diff --git a/packages/server/src/websockets/builder.ts b/packages/server/src/websockets/builder.ts index 53b0a10905..0f2c43e5ab 100644 --- a/packages/server/src/websockets/builder.ts +++ b/packages/server/src/websockets/builder.ts @@ -3,13 +3,7 @@ import { BaseSocket } from "./websocket" import { permissions, events } from "@budibase/backend-core" import http from "http" import Koa from "koa" -import { - Datasource, - Table, - SocketSession, - ContextUser, - SocketMessageOptions, -} from "@budibase/types" +import { Datasource, Table, SocketSession, ContextUser } from "@budibase/types" import { gridSocket } from "./index" import { clearLock, updateLock } from "../utilities/redis" import { Socket } from "socket.io" @@ -72,61 +66,33 @@ export default class BuilderSocket extends BaseSocket { } } - emitTableUpdate(ctx: any, table: Table, options?: SocketMessageOptions) { - this.emitToRoom( - ctx, - ctx.appId, - BuilderSocketEvent.TableChange, - { - id: table._id, - table, - }, - options - ) - gridSocket?.emitTableUpdate(ctx, table, options) + emitTableUpdate(ctx: any, table: Table) { + this.emitToRoom(ctx, ctx.appId, BuilderSocketEvent.TableChange, { + id: table._id, + table, + }) + gridSocket?.emitTableUpdate(ctx, table) } - emitTableDeletion(ctx: any, id: string, options?: SocketMessageOptions) { - this.emitToRoom( - ctx, - ctx.appId, - BuilderSocketEvent.TableChange, - { - id, - table: null, - }, - options - ) - gridSocket?.emitTableDeletion(ctx, id, options) + emitTableDeletion(ctx: any, id: string) { + this.emitToRoom(ctx, ctx.appId, BuilderSocketEvent.TableChange, { + id, + table: null, + }) + gridSocket?.emitTableDeletion(ctx, id) } - emitDatasourceUpdate( - ctx: any, - datasource: Datasource, - options?: SocketMessageOptions - ) { - this.emitToRoom( - ctx, - ctx.appId, - BuilderSocketEvent.DatasourceChange, - { - id: datasource._id, - datasource, - }, - options - ) + emitDatasourceUpdate(ctx: any, datasource: Datasource) { + this.emitToRoom(ctx, ctx.appId, BuilderSocketEvent.DatasourceChange, { + id: datasource._id, + datasource, + }) } - emitDatasourceDeletion(ctx: any, id: string, options?: SocketMessageOptions) { - this.emitToRoom( - ctx, - ctx.appId, - BuilderSocketEvent.DatasourceChange, - { - id, - datasource: null, - }, - options - ) + emitDatasourceDeletion(ctx: any, id: string) { + this.emitToRoom(ctx, ctx.appId, BuilderSocketEvent.DatasourceChange, { + id, + datasource: null, + }) } } diff --git a/packages/server/src/websockets/grid.ts b/packages/server/src/websockets/grid.ts index 38ec091816..6731c2d899 100644 --- a/packages/server/src/websockets/grid.ts +++ b/packages/server/src/websockets/grid.ts @@ -4,7 +4,7 @@ import { permissions } from "@budibase/backend-core" import http from "http" import Koa from "koa" import { getTableId } from "../api/controllers/row/utils" -import { Row, SocketMessageOptions, Table } from "@budibase/types" +import { Row, Table } from "@budibase/types" import { Socket } from "socket.io" import { GridSocketEvent } from "@budibase/shared-core" @@ -29,51 +29,27 @@ export default class GridSocket extends BaseSocket { }) } - emitRowUpdate(ctx: any, row: Row, options?: SocketMessageOptions) { + emitRowUpdate(ctx: any, row: Row) { const tableId = getTableId(ctx) - this.emitToRoom( - ctx, - tableId, - GridSocketEvent.RowChange, - { - id: row._id, - row, - }, - options - ) + this.emitToRoom(ctx, tableId, GridSocketEvent.RowChange, { + id: row._id, + row, + }) } - emitRowDeletion(ctx: any, id: string, options?: SocketMessageOptions) { + emitRowDeletion(ctx: any, id: string) { const tableId = getTableId(ctx) - this.emitToRoom( - ctx, - tableId, - GridSocketEvent.RowChange, - { id, row: null }, - options - ) + this.emitToRoom(ctx, tableId, GridSocketEvent.RowChange, { id, row: null }) } - emitTableUpdate(ctx: any, table: Table, options?: SocketMessageOptions) { - this.emitToRoom( - ctx, - table._id!, - GridSocketEvent.TableChange, - { - id: table._id, - table, - }, - options - ) + emitTableUpdate(ctx: any, table: Table) { + this.emitToRoom(ctx, table._id!, GridSocketEvent.TableChange, { + id: table._id, + table, + }) } - emitTableDeletion(ctx: any, id: string, options?: SocketMessageOptions) { - this.emitToRoom( - ctx, - id, - GridSocketEvent.TableChange, - { id, table: null }, - options - ) + emitTableDeletion(ctx: any, id: string) { + this.emitToRoom(ctx, id, GridSocketEvent.TableChange, { id, table: null }) } } diff --git a/packages/server/src/websockets/websocket.ts b/packages/server/src/websockets/websocket.ts index 59a9d698d7..89fcc8869d 100644 --- a/packages/server/src/websockets/websocket.ts +++ b/packages/server/src/websockets/websocket.ts @@ -9,7 +9,7 @@ import { createAdapter } from "@socket.io/redis-adapter" import { Socket } from "socket.io" import { getSocketPubSubClients } from "../utilities/redis" import { SocketEvent, SocketSessionTTL } from "@budibase/shared-core" -import { SocketSession, SocketMessageOptions } from "@budibase/types" +import { SocketSession } from "@budibase/types" export class BaseSocket { io: Server @@ -277,23 +277,10 @@ export class BaseSocket { } // Emit an event to everyone in a room - emitToRoom( - ctx: any, - room: string, - event: string, - payload: any, - options?: SocketMessageOptions - ) { - // By default, we include the session API of the originator so that they can ignore - // this event. If we want to include the originator then we leave it unset to that all - // clients will react to it. - let apiSessionId = null - if (!options?.includeOriginator) { - apiSessionId = ctx.headers?.[Header.SESSION_ID] - } + emitToRoom(ctx: any, room: string, event: string, payload: any) { this.io.in(room).emit(event, { ...payload, - apiSessionId, + apiSessionId: ctx.headers?.[Header.SESSION_ID], }) } } diff --git a/packages/types/src/sdk/websocket.ts b/packages/types/src/sdk/websocket.ts index 770f44d9b2..40e2654e82 100644 --- a/packages/types/src/sdk/websocket.ts +++ b/packages/types/src/sdk/websocket.ts @@ -7,7 +7,3 @@ export interface SocketSession { room?: string connectedAt: number } - -export interface SocketMessageOptions { - includeOriginator?: boolean -} From 3cbebaf40da830ab80bcfc77ee5cc32afd5cb6e6 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Thu, 15 Jun 2023 09:29:46 +0100 Subject: [PATCH 05/20] Restore original comment --- packages/server/src/websockets/websocket.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/server/src/websockets/websocket.ts b/packages/server/src/websockets/websocket.ts index 89fcc8869d..d8cc10bda4 100644 --- a/packages/server/src/websockets/websocket.ts +++ b/packages/server/src/websockets/websocket.ts @@ -276,7 +276,8 @@ export class BaseSocket { this.io.sockets.emit(event, payload) } - // Emit an event to everyone in a room + // Emit an event to everyone in a room, including metadata of whom + // the originator of the request was emitToRoom(ctx: any, room: string, event: string, payload: any) { this.io.in(room).emit(event, { ...payload, From 9e535a1ca4a80c5a8dd84064d0d00f377b7d8bb3 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Thu, 15 Jun 2023 09:35:22 +0100 Subject: [PATCH 06/20] Account for table 'type' field meaning different things in different endpoints --- .../src/components/backend/DataTable/DataTable.svelte | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/builder/src/components/backend/DataTable/DataTable.svelte b/packages/builder/src/components/backend/DataTable/DataTable.svelte index 4ed41c5b34..981a619ebd 100644 --- a/packages/builder/src/components/backend/DataTable/DataTable.svelte +++ b/packages/builder/src/components/backend/DataTable/DataTable.svelte @@ -30,8 +30,11 @@ const handleGridTableUpdate = async e => { tables.replaceTable(id, e.detail) - // We need to refresh datasources when an external table changes - if (e.detail?.type === "external") { + // We need to refresh datasources when an external table changes. + // Type "external" may exist - sometimes type is "table" and sometimes it + // is "external" - it has different meanings in different endpoints. + // If we check both these then we hopefully catch all external tables. + if (e.detail?.type === "external" || e.detail?.sql) { await datasources.fetch() } } From f2ce876c5f2c5cbae4d7b5a04dac53e5cb2d79f5 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Thu, 15 Jun 2023 09:39:27 +0100 Subject: [PATCH 07/20] Refresh tables list when some other user adds a datasource --- packages/builder/src/stores/backend/datasources.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/builder/src/stores/backend/datasources.js b/packages/builder/src/stores/backend/datasources.js index 0815f9d766..af914cbee7 100644 --- a/packages/builder/src/stores/backend/datasources.js +++ b/packages/builder/src/stores/backend/datasources.js @@ -113,6 +113,10 @@ export function createDatasourcesStore() { ...state, list: [...state.list, datasource], })) + + // If this is a new datasource then we should refresh the tables list, + // because otherwise we'll never see the new tables + tables.fetch() } // Update existing datasource From e512ecff9691dc00c30a6707a2137a045b08c9aa Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Wed, 21 Jun 2023 08:19:29 +0000 Subject: [PATCH 08/20] Bump version to 2.7.26 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index 9addf2b7c9..7692fc7340 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.7.25", + "version": "2.7.26", "npmClient": "yarn", "packages": [ "packages/backend-core", From 8e7e4f50e51491455a010f77439379b99cdd29ef Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Thu, 22 Jun 2023 07:22:22 +0000 Subject: [PATCH 09/20] Bump version to 2.7.27 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index 7692fc7340..9c3495c14f 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.7.26", + "version": "2.7.27", "npmClient": "yarn", "packages": [ "packages/backend-core", From 7fa4db003dd56d3988e4bec5cd602727942bfbb4 Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Thu, 22 Jun 2023 10:53:44 +0000 Subject: [PATCH 10/20] Bump version to 2.7.28 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index 9c3495c14f..b37bcac229 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.7.27", + "version": "2.7.28", "npmClient": "yarn", "packages": [ "packages/backend-core", From 959d3729dc7809acffab186fb2670529767d9b7e Mon Sep 17 00:00:00 2001 From: Dean Date: Thu, 22 Jun 2023 12:28:12 +0100 Subject: [PATCH 11/20] Added config for FontAwesome icons used by EasyMDE in the builder --- packages/bbui/src/Markdown/SpectrumMDE.svelte | 2 +- .../components/common/FontAwesomeIcon.svelte | 29 ++++++++++++++++++- 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/packages/bbui/src/Markdown/SpectrumMDE.svelte b/packages/bbui/src/Markdown/SpectrumMDE.svelte index 9b0832c91f..8e7b1bbdfd 100644 --- a/packages/bbui/src/Markdown/SpectrumMDE.svelte +++ b/packages/bbui/src/Markdown/SpectrumMDE.svelte @@ -87,7 +87,7 @@ border-color: var(--spectrum-global-color-gray-400); } /* Toolbar button color */ - :global(.EasyMDEContainer .editor-toolbar button i) { + :global(.EasyMDEContainer .editor-toolbar button) { color: var(--spectrum-global-color-gray-800); } /* Separator between toolbar buttons*/ diff --git a/packages/builder/src/components/common/FontAwesomeIcon.svelte b/packages/builder/src/components/common/FontAwesomeIcon.svelte index 364b3af25f..97d33f5790 100644 --- a/packages/builder/src/components/common/FontAwesomeIcon.svelte +++ b/packages/builder/src/components/common/FontAwesomeIcon.svelte @@ -9,6 +9,18 @@ faFileArrowUp, faChevronLeft, faCircleInfo, + faBold, + faItalic, + faHeading, + faQuoteLeft, + faListUl, + faListOl, + faLink, + faImage, + faEye, + faColumns, + faArrowsAlt, + faQuestionCircle, } from "@fortawesome/free-solid-svg-icons" import { faGithub, faDiscord } from "@fortawesome/free-brands-svg-icons" @@ -22,7 +34,22 @@ faEnvelope, faFileArrowUp, faChevronLeft, - faCircleInfo + faCircleInfo, + + // -- Required for easyMDE use in the builder. + faBold, + faItalic, + faHeading, + faQuoteLeft, + faListUl, + faListOl, + faLink, + faImage, + faEye, + faColumns, + faArrowsAlt, + faQuestionCircle + // -- ) dom.watch() From 5373b867e155cb91b5fcd441d8e3ab5000a0a583 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 22 Jun 2023 12:32:39 +0100 Subject: [PATCH 12/20] Fix for #10949 - the relationship types were always locked to be the same, which means that the foreign key wasn't being updated correctly as the backend was not aware of which side was the one-to-many side. --- .../Datasources/CreateEditRelationship.svelte | 12 ++++++++++- .../server/src/api/controllers/row/utils.ts | 20 +++++++++++++++++-- yarn.lock | 19 ++++++++++++++++-- 3 files changed, 46 insertions(+), 5 deletions(-) diff --git a/packages/builder/src/components/backend/Datasources/CreateEditRelationship.svelte b/packages/builder/src/components/backend/Datasources/CreateEditRelationship.svelte index 1413cd157e..4908512515 100644 --- a/packages/builder/src/components/backend/Datasources/CreateEditRelationship.svelte +++ b/packages/builder/src/components/backend/Datasources/CreateEditRelationship.svelte @@ -59,7 +59,6 @@ $: valid = getErrorCount(errors) === 0 && allRequiredAttributesSet() $: isManyToMany = relationshipType === RelationshipTypes.MANY_TO_MANY $: isManyToOne = relationshipType === RelationshipTypes.MANY_TO_ONE - $: toRelationship.relationshipType = fromRelationship?.relationshipType function getTable(id) { return plusTables.find(table => table._id === id) @@ -180,6 +179,16 @@ return getErrorCount(errors) === 0 } + function otherRelationshipType(type) { + if (type === RelationshipTypes.MANY_TO_ONE) { + return RelationshipTypes.ONE_TO_MANY + } else if (type === RelationshipTypes.ONE_TO_MANY) { + return RelationshipTypes.MANY_TO_ONE + } else if (type === RelationshipTypes.MANY_TO_MANY) { + return RelationshipTypes.MANY_TO_MANY + } + } + function buildRelationships() { const id = Helpers.uuid() //Map temporary variables @@ -200,6 +209,7 @@ ...toRelationship, tableId: fromId, name: fromColumn, + relationshipType: otherRelationshipType(relationshipType), through: throughId, type: "link", _id: id, diff --git a/packages/server/src/api/controllers/row/utils.ts b/packages/server/src/api/controllers/row/utils.ts index f1edbf538b..7309f0b7a5 100644 --- a/packages/server/src/api/controllers/row/utils.ts +++ b/packages/server/src/api/controllers/row/utils.ts @@ -3,10 +3,10 @@ import * as userController from "../user" import { FieldTypes } from "../../../constants" import { context } from "@budibase/backend-core" import { makeExternalQuery } from "../../../integrations/base/query" -import { Row, Table } from "@budibase/types" +import { FieldType, Row, Table, UserCtx } from "@budibase/types" import { Format } from "../view/exporters" -import { UserCtx } from "@budibase/types" import sdk from "../../../sdk" + const validateJs = require("validate.js") const { cloneDeep } = require("lodash/fp") @@ -20,6 +20,18 @@ validateJs.extend(validateJs.validators.datetime, { }, }) +function isForeignKey(key: string, table: Table) { + const relationships = Object.values(table.schema).filter( + column => column.type === FieldType.LINK + ) + for (let relationship of relationships) { + if (relationship.foreignKey === key) { + return true + } + } + return false +} + export async function getDatasourceAndQuery(json: any) { const datasourceId = json.endpoint.datasourceId const datasource = await sdk.datasources.get(datasourceId) @@ -65,6 +77,10 @@ export async function validate({ const column = fetchedTable.schema[fieldName] const constraints = cloneDeep(column.constraints) const type = column.type + // foreign keys are likely to be enriched + if (isForeignKey(fieldName, fetchedTable)) { + continue + } // formulas shouldn't validated, data will be deleted anyway if (type === FieldTypes.FORMULA || column.autocolumn) { continue diff --git a/yarn.lock b/yarn.lock index 9a57eb4377..981d17e96a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2489,6 +2489,11 @@ minimatch "^3.0.4" strip-json-comments "^3.1.1" +"@fontsource/source-sans-pro@^5.0.3": + version "5.0.3" + resolved "https://registry.yarnpkg.com/@fontsource/source-sans-pro/-/source-sans-pro-5.0.3.tgz#7d6e84a8169ba12fa5e6ce70757aa2ca7e74d855" + integrity sha512-mQnjuif/37VxwRloHZ+wQdoozd2VPWutbFSt1AuSkk7nFXIBQxHJLw80rgCF/osL0t7N/3Gx1V7UJuOX2zxzhQ== + "@fortawesome/fontawesome-common-types@6.3.0": version "6.3.0" resolved "https://registry.yarnpkg.com/@fortawesome/fontawesome-common-types/-/fontawesome-common-types-6.3.0.tgz#51f734e64511dbc3674cd347044d02f4dd26e86b" @@ -8408,7 +8413,7 @@ chmodr@1.2.0: resolved "https://registry.yarnpkg.com/chmodr/-/chmodr-1.2.0.tgz#720e96caa09b7f1cdbb01529b7d0ab6bc5e118b9" integrity sha512-Y5uI7Iq/Az6HgJEL6pdw7THVd7jbVOTPwsmcPOBjQL8e3N+pz872kzK5QxYGEy21iRys+iHWV0UZQXDFJo1hyA== -chokidar@3.5.3, chokidar@^3.0.0, chokidar@^3.5.1, chokidar@^3.5.2: +chokidar@3.5.3, chokidar@^3.0.0, chokidar@^3.5.1, chokidar@^3.5.2, chokidar@^3.5.3: version "3.5.3" resolved "https://registry.yarnpkg.com/chokidar/-/chokidar-3.5.3.tgz#1cf37c8707b932bd1af1ae22c0432e2acd1903bd" integrity sha512-Dr3sfKRP6oTcjf2JmUmFJfeVMvXBdegxB0iVQ5eb2V10uFJUCAS8OByZdVAyVb8xXNz3GjjTgj9kLWsZTqE6kw== @@ -11852,7 +11857,7 @@ fast-glob@3.2.7: merge2 "^1.3.0" micromatch "^4.0.4" -fast-glob@^3.0.3: +fast-glob@^3.0.3, fast-glob@^3.2.11: version "3.2.12" resolved "https://registry.yarnpkg.com/fast-glob/-/fast-glob-3.2.12.tgz#7f39ec99c2e6ab030337142da9e0c18f37afae80" integrity sha512-DVj4CQIYYow0BlaelwK1pHl5n5cRSJfM60UA0zK891sVInoPri2Ekj7+e1CT3/3qxXenpI+nBBmQAcJPJgaj4w== @@ -25402,6 +25407,16 @@ vite-node@0.29.8: picocolors "^1.0.0" vite "^3.0.0 || ^4.0.0" +vite-plugin-static-copy@^0.16.0: + version "0.16.0" + resolved "https://registry.yarnpkg.com/vite-plugin-static-copy/-/vite-plugin-static-copy-0.16.0.tgz#2f65227037f17fc99c0782fd0b344e962935e69e" + integrity sha512-dMVEg5Z2SwYRgQnHZaeokvSKB4p/TOTf65JU4sP3U6ccSBsukqdtDOjpmT+xzTFHAA8WJjcS31RMLjUdWQCBzw== + dependencies: + chokidar "^3.5.3" + fast-glob "^3.2.11" + fs-extra "^11.1.0" + picocolors "^1.0.0" + "vite@^3.0.0 || ^4.0.0": version "4.2.2" resolved "https://registry.yarnpkg.com/vite/-/vite-4.2.2.tgz#014c30e5163844f6e96d7fe18fbb702236516dc6" From 58d7293798f65c563938108cbd89042619b8c434 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 22 Jun 2023 13:25:37 +0100 Subject: [PATCH 13/20] PR suggestion. --- packages/server/src/api/controllers/row/utils.ts | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/packages/server/src/api/controllers/row/utils.ts b/packages/server/src/api/controllers/row/utils.ts index 7309f0b7a5..a213f14a08 100644 --- a/packages/server/src/api/controllers/row/utils.ts +++ b/packages/server/src/api/controllers/row/utils.ts @@ -24,12 +24,7 @@ function isForeignKey(key: string, table: Table) { const relationships = Object.values(table.schema).filter( column => column.type === FieldType.LINK ) - for (let relationship of relationships) { - if (relationship.foreignKey === key) { - return true - } - } - return false + return relationships.some(relationship => relationship.foreignKey === key) } export async function getDatasourceAndQuery(json: any) { From 35c394c05dcf02dd77471eb1258ba59f07f160db Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Thu, 22 Jun 2023 14:04:54 +0000 Subject: [PATCH 14/20] Bump version to 2.7.29 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index b37bcac229..c153d8ddbd 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.7.28", + "version": "2.7.29", "npmClient": "yarn", "packages": [ "packages/backend-core", From 0860a5b24810727d5c54e04857aa6f9e5d38e98c Mon Sep 17 00:00:00 2001 From: melohagan <101575380+melohagan@users.noreply.github.com> Date: Thu, 22 Jun 2023 17:35:47 +0100 Subject: [PATCH 15/20] Exclude entities from datasource env processing (#11000) --- packages/server/src/sdk/app/datasources/datasources.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/server/src/sdk/app/datasources/datasources.ts b/packages/server/src/sdk/app/datasources/datasources.ts index 74c2ff7eb0..21661dd1fd 100644 --- a/packages/server/src/sdk/app/datasources/datasources.ts +++ b/packages/server/src/sdk/app/datasources/datasources.ts @@ -36,11 +36,14 @@ export function checkDatasourceTypes(schema: Integration, config: any) { async function enrichDatasourceWithValues(datasource: Datasource) { const cloned = cloneDeep(datasource) const env = await getEnvironmentVariables() + //Do not process entities, as we do not want to process formulas + const { entities, ...clonedWithoutEntities } = cloned const processed = processObjectSync( - cloned, + clonedWithoutEntities, { env }, { onlyFound: true } ) as Datasource + processed.entities = entities const definition = await getDefinition(processed.source) processed.config = checkDatasourceTypes(definition!, processed.config) return { From c668200261bca6db3e2b3d55b9564db1ef276a20 Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Thu, 22 Jun 2023 16:39:05 +0000 Subject: [PATCH 16/20] Bump version to 2.7.30 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index c153d8ddbd..2700352198 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.7.29", + "version": "2.7.30", "npmClient": "yarn", "packages": [ "packages/backend-core", From aa3eee001c04fd29e37b2895d11816179b086742 Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Fri, 23 Jun 2023 08:10:33 +0000 Subject: [PATCH 17/20] Bump version to 2.7.31 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index 2700352198..7ab2de286c 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.7.30", + "version": "2.7.31", "npmClient": "yarn", "packages": [ "packages/backend-core", From c642ceef37d015a888e4d6adea7766920a62c081 Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Fri, 23 Jun 2023 10:57:37 +0000 Subject: [PATCH 18/20] Bump version to 2.7.32 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index 7ab2de286c..9730620cc3 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.7.31", + "version": "2.7.32", "npmClient": "yarn", "packages": [ "packages/backend-core", From 3aca0c0ed33e4481dc5b1a8e11c3c89e16a7fbd7 Mon Sep 17 00:00:00 2001 From: melohagan <101575380+melohagan@users.noreply.github.com> Date: Fri, 23 Jun 2023 14:48:07 +0100 Subject: [PATCH 19/20] Null safety (#11007) --- packages/server/src/integrations/mongodb.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/integrations/mongodb.ts b/packages/server/src/integrations/mongodb.ts index 417aa61f41..3f05dca954 100644 --- a/packages/server/src/integrations/mongodb.ts +++ b/packages/server/src/integrations/mongodb.ts @@ -383,7 +383,7 @@ class MongoIntegration implements IntegrationBase { createObjectIds(json: any): object { const self = this function interpolateObjectIds(json: any) { - for (let field of Object.keys(json)) { + for (let field of Object.keys(json || {})) { if (json[field] instanceof Object) { json[field] = self.createObjectIds(json[field]) } From 478897236272499d498178976005525929c15168 Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Fri, 23 Jun 2023 13:51:51 +0000 Subject: [PATCH 20/20] Bump version to 2.7.33 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index 9730620cc3..b43faee706 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.7.32", + "version": "2.7.33", "npmClient": "yarn", "packages": [ "packages/backend-core",