From b112995fd05cb86a7bbe9dc37bd2e248d9c12a17 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Tue, 16 May 2023 15:26:20 +0100 Subject: [PATCH 01/42] Disable hide column option in header cell context menu for sticky column --- .../src/components/grid/cells/HeaderCell.svelte | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/frontend-core/src/components/grid/cells/HeaderCell.svelte b/packages/frontend-core/src/components/grid/cells/HeaderCell.svelte index 72b0ad0ff1..21ee210233 100644 --- a/packages/frontend-core/src/components/grid/cells/HeaderCell.svelte +++ b/packages/frontend-core/src/components/grid/cells/HeaderCell.svelte @@ -196,7 +196,11 @@ Move right - Hide column + Hide column From 0a0e78c314d4cda86cfd9168341227f3f225f77d Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Tue, 16 May 2023 15:29:48 +0100 Subject: [PATCH 02/42] Fix grid keyboard shortcuts being hard to read in light theme --- .../src/components/grid/layout/KeyboardShortcut.svelte | 2 +- packages/frontend-core/src/components/grid/layout/NewRow.svelte | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/frontend-core/src/components/grid/layout/KeyboardShortcut.svelte b/packages/frontend-core/src/components/grid/layout/KeyboardShortcut.svelte index 5024a24ea7..cac39bbf2f 100644 --- a/packages/frontend-core/src/components/grid/layout/KeyboardShortcut.svelte +++ b/packages/frontend-core/src/components/grid/layout/KeyboardShortcut.svelte @@ -38,7 +38,7 @@ padding: 2px 6px; font-size: 12px; font-weight: 600; - background-color: var(--spectrum-global-color-gray-200); + background-color: var(--spectrum-global-color-gray-300); color: var(--spectrum-global-color-gray-700); border-radius: 4px; text-align: center; diff --git a/packages/frontend-core/src/components/grid/layout/NewRow.svelte b/packages/frontend-core/src/components/grid/layout/NewRow.svelte index 8048a4e2fa..f705881a8d 100644 --- a/packages/frontend-core/src/components/grid/layout/NewRow.svelte +++ b/packages/frontend-core/src/components/grid/layout/NewRow.svelte @@ -219,7 +219,7 @@ From 834202423f920d0ab1b4ad982b6e1f3a93a785f6 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Tue, 16 May 2023 16:42:36 +0100 Subject: [PATCH 03/42] Fix bulk deletion triggering on delete keypress after selecting then deselecting a row --- .../components/grid/cells/GutterCell.svelte | 11 +--------- .../grid/overlays/KeyboardManager.svelte | 5 +---- .../src/components/grid/stores/ui.js | 21 ++++++++++++++++++- 3 files changed, 22 insertions(+), 15 deletions(-) diff --git a/packages/frontend-core/src/components/grid/cells/GutterCell.svelte b/packages/frontend-core/src/components/grid/cells/GutterCell.svelte index 00b99c0711..588818f10d 100644 --- a/packages/frontend-core/src/components/grid/cells/GutterCell.svelte +++ b/packages/frontend-core/src/components/grid/cells/GutterCell.svelte @@ -21,16 +21,7 @@ svelteDispatch("select") const id = row?._id if (id) { - selectedRows.update(state => { - let newState = { - ...state, - [id]: !state[id], - } - if (!newState[id]) { - delete newState[id] - } - return newState - }) + selectedRows.actions.toggleRow(id) } } diff --git a/packages/frontend-core/src/components/grid/overlays/KeyboardManager.svelte b/packages/frontend-core/src/components/grid/overlays/KeyboardManager.svelte index c7fa0a5cb7..6d16acc7c5 100644 --- a/packages/frontend-core/src/components/grid/overlays/KeyboardManager.svelte +++ b/packages/frontend-core/src/components/grid/overlays/KeyboardManager.svelte @@ -224,10 +224,7 @@ if (!id || id === NewRowID) { return } - selectedRows.update(state => { - state[id] = !state[id] - return state - }) + selectedRows.actions.toggleRow(id) } onMount(() => { diff --git a/packages/frontend-core/src/components/grid/stores/ui.js b/packages/frontend-core/src/components/grid/stores/ui.js index 85afad8a90..b62e883437 100644 --- a/packages/frontend-core/src/components/grid/stores/ui.js +++ b/packages/frontend-core/src/components/grid/stores/ui.js @@ -25,14 +25,33 @@ export const createStores = () => { null ) + // Toggles whether a certain row ID is selected or not + const toggleSelectedRow = id => { + selectedRows.update(state => { + let newState = { + ...state, + [id]: !state[id], + } + if (!newState[id]) { + delete newState[id] + } + return newState + }) + } + return { focusedCellId, focusedCellAPI, focusedRowId, previousFocusedRowId, - selectedRows, hoveredRowId, rowHeight, + selectedRows: { + ...selectedRows, + actions: { + toggleRow: toggleSelectedRow, + }, + }, } } From ba9691ee12aa04d31089db3dc90690dfb079a0c0 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Tue, 16 May 2023 17:09:32 +0100 Subject: [PATCH 04/42] Add grid flag to stripe rows different colours --- .../src/components/grid/cells/DataCell.svelte | 2 ++ .../src/components/grid/cells/GridCell.svelte | 13 +++++++++---- .../src/components/grid/cells/GutterCell.svelte | 1 + .../src/components/grid/layout/Grid.svelte | 8 ++++++++ .../src/components/grid/layout/GridBody.svelte | 6 +++++- .../src/components/grid/layout/GridRow.svelte | 5 +++-- .../src/components/grid/layout/NewRow.svelte | 4 ++-- .../src/components/grid/layout/StickyColumn.svelte | 3 ++- 8 files changed, 32 insertions(+), 10 deletions(-) diff --git a/packages/frontend-core/src/components/grid/cells/DataCell.svelte b/packages/frontend-core/src/components/grid/cells/DataCell.svelte index f39b820632..cb8616a735 100644 --- a/packages/frontend-core/src/components/grid/cells/DataCell.svelte +++ b/packages/frontend-core/src/components/grid/cells/DataCell.svelte @@ -11,6 +11,7 @@ export let selected export let rowFocused export let rowIdx + export let topRow = false export let focused export let selectedUser export let column @@ -68,6 +69,7 @@ {highlighted} {selected} {rowIdx} + {topRow} {focused} {selectedUser} {readonly} diff --git a/packages/frontend-core/src/components/grid/cells/GridCell.svelte b/packages/frontend-core/src/components/grid/cells/GridCell.svelte index 6589c18d07..7e38a989d6 100644 --- a/packages/frontend-core/src/components/grid/cells/GridCell.svelte +++ b/packages/frontend-core/src/components/grid/cells/GridCell.svelte @@ -6,6 +6,7 @@ export let selectedUser = null export let error = null export let rowIdx + export let topRow = false export let defaultHeight = false export let center = false export let readonly = false @@ -31,13 +32,14 @@ class:readonly class:default-height={defaultHeight} class:selected-other={selectedUser != null} + class:alt={rowIdx % 2 === 1} + class:top={topRow} on:focus on:mousedown on:mouseup on:click on:contextmenu {style} - data-row={rowIdx} > {#if error}
@@ -70,6 +72,9 @@ width: 0; --cell-color: transparent; } + .cell.alt { + --cell-background: var(--cell-background-alt); + } .cell.default-height { height: var(--default-row-height); } @@ -98,8 +103,8 @@ .cell.selected-other:not(.focused):after { border-radius: 0 2px 2px 2px; } - .cell[data-row="0"].error:after, - .cell[data-row="0"].selected-other:not(.focused):after { + .cell.top.error:after, + .cell.top.selected-other:not(.focused):after { border-radius: 2px 2px 2px 0; } @@ -152,7 +157,7 @@ overflow: hidden; user-select: none; } - .cell[data-row="0"] .label { + .cell.top .label { bottom: auto; top: 100%; border-radius: 0 2px 2px 2px; diff --git a/packages/frontend-core/src/components/grid/cells/GutterCell.svelte b/packages/frontend-core/src/components/grid/cells/GutterCell.svelte index 588818f10d..56c4c20d54 100644 --- a/packages/frontend-core/src/components/grid/cells/GutterCell.svelte +++ b/packages/frontend-core/src/components/grid/cells/GutterCell.svelte @@ -38,6 +38,7 @@ highlighted={rowFocused || rowHovered} selected={rowSelected} {defaultHeight} + rowIdx={row?.__idx} >
{#if $$slots.default} diff --git a/packages/frontend-core/src/components/grid/layout/Grid.svelte b/packages/frontend-core/src/components/grid/layout/Grid.svelte index 2035ec4d39..314479f519 100644 --- a/packages/frontend-core/src/components/grid/layout/Grid.svelte +++ b/packages/frontend-core/src/components/grid/layout/Grid.svelte @@ -40,6 +40,7 @@ export let allowExpandRows = true export let allowEditRows = true export let allowDeleteRows = true + export let stripeRows = false // Unique identifier for DOM nodes inside this instance const rand = Math.random() @@ -54,6 +55,7 @@ allowExpandRows, allowEditRows, allowDeleteRows, + stripeRows, }) // Build up context @@ -88,6 +90,7 @@ allowExpandRows, allowEditRows, allowDeleteRows, + stripeRows, }) // Set context for children to consume @@ -105,6 +108,7 @@ id="grid-{rand}" class:is-resizing={$isResizing} class:is-reordering={$isReordering} + class:stripe={$config.stripeRows} style="--row-height:{$rowHeight}px; --default-row-height:{DefaultRowHeight}px; --gutter-width:{GutterWidth}px; --max-cell-render-height:{MaxCellRenderHeight}px; --max-cell-render-width-overflow:{MaxCellRenderWidthOverflow}px; --content-lines:{$contentLines};" >
@@ -167,6 +171,7 @@ /* Variables */ --cell-background: var(--spectrum-global-color-gray-50); --cell-background-hover: var(--spectrum-global-color-gray-100); + --cell-background-alt: var(--cell-background); --cell-padding: 8px; --cell-spacing: 4px; --cell-border: 1px solid var(--spectrum-global-color-gray-200); @@ -183,6 +188,9 @@ .grid.is-reordering :global(*) { cursor: grabbing !important; } + .grid.stripe { + --cell-background-alt: var(--spectrum-global-color-gray-75); + } .grid-data-outer, .grid-data-inner { diff --git a/packages/frontend-core/src/components/grid/layout/GridBody.svelte b/packages/frontend-core/src/components/grid/layout/GridBody.svelte index 67f5f03898..016369df49 100644 --- a/packages/frontend-core/src/components/grid/layout/GridBody.svelte +++ b/packages/frontend-core/src/components/grid/layout/GridBody.svelte @@ -36,7 +36,11 @@
{#each $renderedRows as row, idx} - = $rowVerticalInversionIndex} /> + = $rowVerticalInversionIndex} + /> {/each} {#if $config.allowAddRows && $renderedColumns.length}
= $columnHorizontalInversionIndex} highlighted={rowHovered || rowFocused || reorderSource === column.name} selected={rowSelected} - rowIdx={idx} + rowIdx={row.__idx} + topRow={top} focused={$focusedCellId === cellId} selectedUser={$selectedCellMap[cellId]} width={column.width} diff --git a/packages/frontend-core/src/components/grid/layout/NewRow.svelte b/packages/frontend-core/src/components/grid/layout/NewRow.svelte index f705881a8d..85b430f79b 100644 --- a/packages/frontend-core/src/components/grid/layout/NewRow.svelte +++ b/packages/frontend-core/src/components/grid/layout/NewRow.svelte @@ -167,7 +167,7 @@ focused={$focusedCellId === cellId} width={$stickyColumn.width} {updateValue} - rowIdx={0} + topRow={offset === 0} {invertY} > {#if $stickyColumn?.schema?.autocolumn} @@ -193,7 +193,7 @@ row={newRow} focused={$focusedCellId === cellId} width={column.width} - rowIdx={0} + topRow={offset === 0} invertX={columnIdx >= $columnHorizontalInversionIndex} {invertY} > diff --git a/packages/frontend-core/src/components/grid/layout/StickyColumn.svelte b/packages/frontend-core/src/components/grid/layout/StickyColumn.svelte index 6301112110..801772ed51 100644 --- a/packages/frontend-core/src/components/grid/layout/StickyColumn.svelte +++ b/packages/frontend-core/src/components/grid/layout/StickyColumn.svelte @@ -82,7 +82,8 @@ {rowFocused} selected={rowSelected} highlighted={rowHovered || rowFocused} - rowIdx={idx} + rowIdx={row.__idx} + topRow={idx === 0} focused={$focusedCellId === cellId} selectedUser={$selectedCellMap[cellId]} width={$stickyColumn.width} From cadd1b5a4e2a4491277bf07d3b456ce36e42576b Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Tue, 16 May 2023 19:48:14 +0100 Subject: [PATCH 05/42] Add automatic scrolling left/right when dragging to reorder columns --- .../src/components/grid/stores/reorder.js | 86 +++++++++++++++++-- 1 file changed, 77 insertions(+), 9 deletions(-) diff --git a/packages/frontend-core/src/components/grid/stores/reorder.js b/packages/frontend-core/src/components/grid/stores/reorder.js index de343987db..a99c1b1ab2 100644 --- a/packages/frontend-core/src/components/grid/stores/reorder.js +++ b/packages/frontend-core/src/components/grid/stores/reorder.js @@ -4,9 +4,10 @@ const reorderInitialState = { sourceColumn: null, targetColumn: null, breakpoints: [], - initialMouseX: null, - scrollLeft: 0, gridLeft: 0, + width: 0, + latestX: 0, + increment: 0, } export const createStores = () => { @@ -23,14 +24,24 @@ export const createStores = () => { } export const deriveStores = context => { - const { reorder, columns, visibleColumns, scroll, bounds, stickyColumn, ui } = - context + const { + reorder, + columns, + visibleColumns, + scroll, + bounds, + stickyColumn, + ui, + maxScrollLeft, + } = context + + let autoScrollInterval + let isAutoScrolling // Callback when dragging on a colum header and starting reordering const startReordering = (column, e) => { const $visibleColumns = get(visibleColumns) const $bounds = get(bounds) - const $scroll = get(scroll) const $stickyColumn = get(stickyColumn) ui.actions.blur() @@ -51,9 +62,8 @@ export const deriveStores = context => { sourceColumn: column, targetColumn: null, breakpoints, - initialMouseX: e.clientX, - scrollLeft: $scroll.left, gridLeft: $bounds.left, + width: $bounds.width, }) // Add listeners to handle mouse movement @@ -66,12 +76,44 @@ export const deriveStores = context => { // Callback when moving the mouse when reordering columns const onReorderMouseMove = e => { + // Immediately handle the current position + const x = e.clientX + reorder.update(state => ({ + ...state, + latestX: x, + })) + considerReorderPosition() + + // Check if we need to start auto-scrolling const $reorder = get(reorder) + const proximityCutoff = 140 + const speedFactor = 8 + const rightProximity = Math.max(0, $reorder.gridLeft + $reorder.width - x) + const leftProximity = Math.max(0, x - $reorder.gridLeft) + if (rightProximity < proximityCutoff) { + const weight = proximityCutoff - rightProximity + const increment = (weight / proximityCutoff) * speedFactor + reorder.update(state => ({ ...state, increment })) + startAutoScroll() + } else if (leftProximity < proximityCutoff) { + const weight = -1 * (proximityCutoff - leftProximity) + const increment = (weight / proximityCutoff) * speedFactor + reorder.update(state => ({ ...state, increment })) + startAutoScroll() + } else { + stopAutoScroll() + } + } + + // Actual logic to consider the current position and determine the new order + const considerReorderPosition = () => { + const $reorder = get(reorder) + const $scroll = get(scroll) // Compute the closest breakpoint to the current position let targetColumn let minDistance = Number.MAX_SAFE_INTEGER - const mouseX = e.clientX - $reorder.gridLeft + $reorder.scrollLeft + const mouseX = $reorder.latestX - $reorder.gridLeft + $scroll.left $reorder.breakpoints.forEach(point => { const distance = Math.abs(point.x - mouseX) if (distance < minDistance) { @@ -79,7 +121,6 @@ export const deriveStores = context => { targetColumn = point.column } }) - if (targetColumn !== $reorder.targetColumn) { reorder.update(state => ({ ...state, @@ -88,8 +129,35 @@ export const deriveStores = context => { } } + // Commences auto-scrolling in a certain direction, triggered when the mouse + // approaches the edges of the grid + const startAutoScroll = () => { + if (isAutoScrolling) { + return + } + isAutoScrolling = true + autoScrollInterval = setInterval(() => { + const $maxLeft = get(maxScrollLeft) + const { increment } = get(reorder) + scroll.update(state => ({ + ...state, + left: Math.max(0, Math.min($maxLeft, state.left + increment)), + })) + considerReorderPosition() + }, 10) + } + + // Stops auto scrolling + const stopAutoScroll = () => { + isAutoScrolling = false + clearInterval(autoScrollInterval) + } + // Callback when stopping reordering columns const stopReordering = async () => { + // Ensure auto-scrolling is stopped + stopAutoScroll() + // Swap position of columns let { sourceColumn, targetColumn } = get(reorder) moveColumn(sourceColumn, targetColumn) From efb45a850b626c485d7e347d2aa3ec867e5ab0a8 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Wed, 17 May 2023 08:07:07 +0100 Subject: [PATCH 06/42] Update z-indexes for add column button --- .../frontend-core/src/components/grid/layout/HeaderRow.svelte | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/frontend-core/src/components/grid/layout/HeaderRow.svelte b/packages/frontend-core/src/components/grid/layout/HeaderRow.svelte index be9fad00d0..9d6cc2275b 100644 --- a/packages/frontend-core/src/components/grid/layout/HeaderRow.svelte +++ b/packages/frontend-core/src/components/grid/layout/HeaderRow.svelte @@ -61,7 +61,7 @@ border-right: var(--cell-border); border-bottom: var(--cell-border); background: var(--spectrum-global-color-gray-100); - z-index: 20; + z-index: 1; } .add:hover { background: var(--spectrum-global-color-gray-200); From 475c962a5fdaab584626e57b051d136b64c44744 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Wed, 17 May 2023 08:16:47 +0100 Subject: [PATCH 07/42] Center align boolean fields --- .../frontend-core/src/components/grid/cells/BooleanCell.svelte | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/frontend-core/src/components/grid/cells/BooleanCell.svelte b/packages/frontend-core/src/components/grid/cells/BooleanCell.svelte index 52aecb07a7..c3449c0b39 100644 --- a/packages/frontend-core/src/components/grid/cells/BooleanCell.svelte +++ b/packages/frontend-core/src/components/grid/cells/BooleanCell.svelte @@ -37,6 +37,9 @@ .boolean-cell { padding: 2px var(--cell-padding); pointer-events: none; + flex: 1 1 auto; + display: flex; + justify-content: center; } .boolean-cell.editable { pointer-events: all; From 886b726627dd46ed235281c69660049074f9d067 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 23 May 2023 15:21:33 +0200 Subject: [PATCH 08/42] Verify datasouce before saving --- .../builder/src/builderStore/datasource.js | 7 +++++ .../modals/DatasourceConfigModal.svelte | 29 +++++++++++++++++-- packages/frontend-core/src/api/datasources.js | 11 +++++++ 3 files changed, 45 insertions(+), 2 deletions(-) diff --git a/packages/builder/src/builderStore/datasource.js b/packages/builder/src/builderStore/datasource.js index e12b318e1c..6509216a5b 100644 --- a/packages/builder/src/builderStore/datasource.js +++ b/packages/builder/src/builderStore/datasource.js @@ -2,6 +2,7 @@ import { datasources, tables } from "../stores/backend" import { IntegrationNames } from "../constants/backend" import { get } from "svelte/store" import cloneDeep from "lodash/cloneDeepWith" +import { API } from "api" function prepareData(config) { let datasource = {} @@ -37,3 +38,9 @@ export async function createRestDatasource(integration) { const config = cloneDeep(integration) return saveDatasource(config) } + +export async function validateDatasourceConfig(config) { + const datasource = prepareData(config) + const resp = await API.validateDatasource(datasource) + return resp +} diff --git a/packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte b/packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte index edbe55178f..36b535c2b8 100644 --- a/packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte +++ b/packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte @@ -4,7 +4,10 @@ import IntegrationConfigForm from "components/backend/DatasourceNavigator/TableIntegrationMenu/IntegrationConfigForm.svelte" import { IntegrationNames } from "constants/backend" import cloneDeep from "lodash/cloneDeepWith" - import { saveDatasource as save } from "builderStore/datasource" + import { + saveDatasource as save, + validateDatasourceConfig, + } from "builderStore/datasource" import { onMount } from "svelte" export let integration @@ -18,6 +21,27 @@ $: name = IntegrationNames[datasource.type] || datasource.name || datasource.type + async function validateConfig() { + function displayError(message) { + notifications.error(message ?? "Error validating datasource") + } + + let connected = false + try { + if (!datasource.name) { + datasource.name = name + } + + const resp = await validateDatasourceConfig(datasource) + if (!resp.connected) { + displayError(resp.error) + } + connected = resp.connected + } catch (err) { + displayError(err?.message) + } + return connected + } async function saveDatasource() { try { if (!datasource.name) { @@ -40,7 +64,8 @@ saveDatasource()} + onConfirm={() => + validateConfig().then(connected => connected && saveDatasource())} onCancel={() => modal.show()} confirmText={datasource.plus ? "Save and fetch tables" diff --git a/packages/frontend-core/src/api/datasources.js b/packages/frontend-core/src/api/datasources.js index eda7b3c860..16d19c512f 100644 --- a/packages/frontend-core/src/api/datasources.js +++ b/packages/frontend-core/src/api/datasources.js @@ -58,4 +58,15 @@ export const buildDatasourceEndpoints = API => ({ url: `/api/datasources/${datasourceId}/${datasourceRev}`, }) }, + + /** + * Validate a datasource configuration + * @param datasource the datasource configuration to validate + */ + validateDatasource: async datasource => { + return await API.post({ + url: `/api/datasources/verify`, + body: { datasource }, + }) + }, }) From d6b00efa908b9646eaf09e20d391df7c7a4c6636 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 23 May 2023 15:53:01 +0200 Subject: [PATCH 09/42] Verify datasouce before updating --- .../modals/DatasourceConfigModal.svelte | 15 ++++++------- .../datasource/[datasourceId]/index.svelte | 22 +++++++++++++++++++ 2 files changed, 29 insertions(+), 8 deletions(-) diff --git a/packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte b/packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte index 36b535c2b8..f5d57285bb 100644 --- a/packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte +++ b/packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte @@ -22,16 +22,11 @@ IntegrationNames[datasource.type] || datasource.name || datasource.type async function validateConfig() { - function displayError(message) { + const displayError = message => notifications.error(message ?? "Error validating datasource") - } let connected = false try { - if (!datasource.name) { - datasource.name = name - } - const resp = await validateDatasourceConfig(datasource) if (!resp.connected) { displayError(resp.error) @@ -42,7 +37,12 @@ } return connected } + async function saveDatasource() { + const valid = await validateConfig() + if (!valid) { + return false + } try { if (!datasource.name) { datasource.name = name @@ -64,8 +64,7 @@ - validateConfig().then(connected => connected && saveDatasource())} + onConfirm={() => saveDatasource()} onCancel={() => modal.show()} confirmText={datasource.plus ? "Save and fetch tables" diff --git a/packages/builder/src/pages/builder/app/[application]/data/datasource/[datasourceId]/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/datasource/[datasourceId]/index.svelte index 3a3a2495ed..511a7a81c8 100644 --- a/packages/builder/src/pages/builder/app/[application]/data/datasource/[datasourceId]/index.svelte +++ b/packages/builder/src/pages/builder/app/[application]/data/datasource/[datasourceId]/index.svelte @@ -20,6 +20,7 @@ import { isEqual } from "lodash" import { cloneDeep } from "lodash/fp" import ImportRestQueriesModal from "components/backend/DatasourceNavigator/modals/ImportRestQueriesModal.svelte" + import { API } from "api" const querySchema = { name: {}, @@ -45,7 +46,28 @@ } } + async function validateConfig() { + const displayError = message => + notifications.error(message ?? "Error validating datasource") + + let connected = false + try { + const resp = await API.validateDatasource(datasource) + if (!resp.connected) { + displayError(resp.error) + } + connected = resp.connected + } catch (err) { + displayError(err?.message) + } + return connected + } + const saveDatasource = async () => { + const valid = await validateConfig() + if (!valid) { + return false + } try { // Create datasource await datasources.save(datasource) From 3951788f7ed58931cf45fab2742c2557c44b3015 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 23 May 2023 15:55:36 +0200 Subject: [PATCH 10/42] Improve messaging --- .../[application]/data/datasource/[datasourceId]/index.svelte | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/builder/src/pages/builder/app/[application]/data/datasource/[datasourceId]/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/datasource/[datasourceId]/index.svelte index 511a7a81c8..850e92c772 100644 --- a/packages/builder/src/pages/builder/app/[application]/data/datasource/[datasourceId]/index.svelte +++ b/packages/builder/src/pages/builder/app/[application]/data/datasource/[datasourceId]/index.svelte @@ -54,7 +54,7 @@ try { const resp = await API.validateDatasource(datasource) if (!resp.connected) { - displayError(resp.error) + displayError(`Error connecting to the datasource: ` + resp.error) } connected = resp.connected } catch (err) { From 6659a697dec8ab2c8f254f86d33946fb65035588 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 23 May 2023 15:58:50 +0200 Subject: [PATCH 11/42] Improve messaging --- .../DatasourceNavigator/modals/DatasourceConfigModal.svelte | 2 +- .../[application]/data/datasource/[datasourceId]/index.svelte | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte b/packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte index f5d57285bb..5394510122 100644 --- a/packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte +++ b/packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte @@ -29,7 +29,7 @@ try { const resp = await validateDatasourceConfig(datasource) if (!resp.connected) { - displayError(resp.error) + displayError(`Unable to connect - ${resp.error}`) } connected = resp.connected } catch (err) { diff --git a/packages/builder/src/pages/builder/app/[application]/data/datasource/[datasourceId]/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/datasource/[datasourceId]/index.svelte index 850e92c772..f33860bb69 100644 --- a/packages/builder/src/pages/builder/app/[application]/data/datasource/[datasourceId]/index.svelte +++ b/packages/builder/src/pages/builder/app/[application]/data/datasource/[datasourceId]/index.svelte @@ -54,7 +54,7 @@ try { const resp = await API.validateDatasource(datasource) if (!resp.connected) { - displayError(`Error connecting to the datasource: ` + resp.error) + displayError(`Unable to connect - ${resp.error}`) } connected = resp.connected } catch (err) { From 3c3f67c3006194da916a46fb6e9e2756f8e066c5 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 23 May 2023 16:19:36 +0200 Subject: [PATCH 12/42] Handle name change on onboarding --- .../apps/onboarding/_components/NamePanel.svelte | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/packages/builder/src/pages/builder/portal/apps/onboarding/_components/NamePanel.svelte b/packages/builder/src/pages/builder/portal/apps/onboarding/_components/NamePanel.svelte index 1264b63531..0641e28766 100644 --- a/packages/builder/src/pages/builder/portal/apps/onboarding/_components/NamePanel.svelte +++ b/packages/builder/src/pages/builder/portal/apps/onboarding/_components/NamePanel.svelte @@ -31,6 +31,18 @@ return "Invalid URL" } } + + $: urlManuallySet = false + + const updateUrl = event => { + const appName = event.detail + if (urlManuallySet) { + return + } + + const parsedUrl = appName.toLowerCase().replace(/\s+/g, "-") + url = encodeURI(parsedUrl) + }
@@ -43,11 +55,13 @@ bind:value={name} bind:error={nameError} validate={validateName} + on:change={updateUrl} label="Name" /> (urlManuallySet = true)} validate={validateUrl} label="URL" /> From 2b2b25290022b1afb273d683360aff47e4a4c6d3 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 23 May 2023 16:43:10 +0200 Subject: [PATCH 13/42] Update messages --- .../DatasourceNavigator/modals/DatasourceConfigModal.svelte | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte b/packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte index 5394510122..09eb97a497 100644 --- a/packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte +++ b/packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte @@ -66,12 +66,9 @@ title={`Connect to ${name}`} onConfirm={() => saveDatasource()} onCancel={() => modal.show()} - confirmText={datasource.plus - ? "Save and fetch tables" - : "Save and continue to query"} + confirmText={datasource.plus ? "Connect" : "Save and continue to query"} cancelText="Back" showSecondaryButton={datasource.plus} - secondaryButtonText={datasource.plus ? "Skip table fetch" : undefined} secondaryAction={() => { skipFetch = true saveDatasource() From 7be824b2e6a7ca24ae1ccc1ffeca4aa50d9249f1 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 23 May 2023 16:45:30 +0200 Subject: [PATCH 14/42] Remove secondary action --- .../modals/DatasourceConfigModal.svelte | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte b/packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte index 09eb97a497..d9b39e95a3 100644 --- a/packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte +++ b/packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte @@ -15,7 +15,6 @@ // kill the reference so the input isn't saved let datasource = cloneDeep(integration) - let skipFetch = false let isValid = false $: name = @@ -47,7 +46,7 @@ if (!datasource.name) { datasource.name = name } - const resp = await save(datasource, skipFetch) + const resp = await save(datasource) $goto(`./datasource/${resp._id}`) notifications.success(`Datasource updated successfully.`) } catch (err) { @@ -56,10 +55,6 @@ return false } } - - onMount(() => { - skipFetch = false - }) { - skipFetch = true - saveDatasource() - return true - }} size="L" disabled={!isValid} > From 0c97f61cc890a0a8ecacd4efa5a8ade072385e46 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 23 May 2023 18:23:37 +0200 Subject: [PATCH 15/42] Validate feature toggle --- .../modals/CreateDatasourceModal.svelte | 1 + .../modals/DatasourceConfigModal.svelte | 19 +++++++++++++------ 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/packages/builder/src/components/backend/DatasourceNavigator/modals/CreateDatasourceModal.svelte b/packages/builder/src/components/backend/DatasourceNavigator/modals/CreateDatasourceModal.svelte index a07829eb58..290f25b941 100644 --- a/packages/builder/src/components/backend/DatasourceNavigator/modals/CreateDatasourceModal.svelte +++ b/packages/builder/src/components/backend/DatasourceNavigator/modals/CreateDatasourceModal.svelte @@ -53,6 +53,7 @@ config, schema: selected.datasource, auth: selected.auth, + features: selected.features || [], } if (selected.friendlyName) { integration.name = selected.friendlyName diff --git a/packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte b/packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte index d9b39e95a3..ee1f97bf40 100644 --- a/packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte +++ b/packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte @@ -8,7 +8,6 @@ saveDatasource as save, validateDatasourceConfig, } from "builderStore/datasource" - import { onMount } from "svelte" export let integration export let modal @@ -37,10 +36,14 @@ return connected } + $: shouldValidate = integration.features?.find(f => f === "connection") + async function saveDatasource() { - const valid = await validateConfig() - if (!valid) { - return false + if (shouldValidate) { + const valid = await validateConfig() + if (!valid) { + return false + } } try { if (!datasource.name) { @@ -48,7 +51,7 @@ } const resp = await save(datasource) $goto(`./datasource/${resp._id}`) - notifications.success(`Datasource updated successfully.`) + notifications.success(`Datasource created successfully.`) } catch (err) { notifications.error(err?.message ?? "Error saving datasource") // prevent the modal from closing @@ -61,7 +64,11 @@ title={`Connect to ${name}`} onConfirm={() => saveDatasource()} onCancel={() => modal.show()} - confirmText={datasource.plus ? "Connect" : "Save and continue to query"} + confirmText={datasource.plus + ? shouldValidate + ? "Connect" + : "Save and continue" + : "Save and continue to query"} cancelText="Back" showSecondaryButton={datasource.plus} size="L" From 237cda064c90ab79ef3f61ca576da2da9a52b496 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 24 May 2023 10:50:51 +0200 Subject: [PATCH 16/42] Simplify integration features --- .../modals/DatasourceConfigModal.svelte | 11 +++-------- .../data/datasource/[datasourceId]/index.svelte | 9 ++++++--- packages/builder/src/stores/portal/features.js | 2 +- packages/server/src/integrations/airtable.ts | 4 +++- packages/server/src/integrations/arangodb.ts | 4 +++- packages/server/src/integrations/couchdb.ts | 4 +++- packages/server/src/integrations/dynamodb.ts | 4 +++- packages/server/src/integrations/elasticsearch.ts | 4 +++- packages/server/src/integrations/firebase.ts | 4 +++- packages/server/src/integrations/googlesheets.ts | 8 ++++---- .../server/src/integrations/microsoftSqlServer.ts | 8 ++++---- packages/server/src/integrations/mongodb.ts | 4 +++- packages/server/src/integrations/mysql.ts | 8 ++++---- packages/server/src/integrations/oracle.ts | 8 ++++---- packages/server/src/integrations/postgres.ts | 8 ++++---- packages/server/src/integrations/redis.ts | 4 +++- packages/server/src/integrations/s3.ts | 4 +++- packages/server/src/integrations/snowflake.ts | 4 +++- packages/types/src/sdk/datasources.ts | 2 +- 19 files changed, 61 insertions(+), 43 deletions(-) diff --git a/packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte b/packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte index ee1f97bf40..e75109b1be 100644 --- a/packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte +++ b/packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte @@ -8,6 +8,7 @@ saveDatasource as save, validateDatasourceConfig, } from "builderStore/datasource" + import { DatasourceFeature } from "@budibase/types" export let integration export let modal @@ -36,10 +37,8 @@ return connected } - $: shouldValidate = integration.features?.find(f => f === "connection") - async function saveDatasource() { - if (shouldValidate) { + if (integration.features[DatasourceFeature.CONNECTION_CHECKING]) { const valid = await validateConfig() if (!valid) { return false @@ -64,11 +63,7 @@ title={`Connect to ${name}`} onConfirm={() => saveDatasource()} onCancel={() => modal.show()} - confirmText={datasource.plus - ? shouldValidate - ? "Connect" - : "Save and continue" - : "Save and continue to query"} + confirmText={datasource.plus ? "Connect" : "Save and continue to query"} cancelText="Back" showSecondaryButton={datasource.plus} size="L" diff --git a/packages/builder/src/pages/builder/app/[application]/data/datasource/[datasourceId]/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/datasource/[datasourceId]/index.svelte index f33860bb69..a853fcea0c 100644 --- a/packages/builder/src/pages/builder/app/[application]/data/datasource/[datasourceId]/index.svelte +++ b/packages/builder/src/pages/builder/app/[application]/data/datasource/[datasourceId]/index.svelte @@ -21,6 +21,7 @@ import { cloneDeep } from "lodash/fp" import ImportRestQueriesModal from "components/backend/DatasourceNavigator/modals/ImportRestQueriesModal.svelte" import { API } from "api" + import { DatasourceFeature } from "@budibase/types" const querySchema = { name: {}, @@ -64,9 +65,11 @@ } const saveDatasource = async () => { - const valid = await validateConfig() - if (!valid) { - return false + if (integration.features[DatasourceFeature.CONNECTION_CHECKING]) { + const valid = await validateConfig() + if (!valid) { + return false + } } try { // Create datasource diff --git a/packages/builder/src/stores/portal/features.js b/packages/builder/src/stores/portal/features.js index 4334fda3c4..e2ecb4c66a 100644 --- a/packages/builder/src/stores/portal/features.js +++ b/packages/builder/src/stores/portal/features.js @@ -1,7 +1,7 @@ import { writable } from "svelte/store" import { API } from "api" import { licensing } from "./licensing" -import { ConfigType } from "../../../../types/src/documents" +import { ConfigType } from "@budibase/types" export const createFeatureStore = () => { const internalStore = writable({ diff --git a/packages/server/src/integrations/airtable.ts b/packages/server/src/integrations/airtable.ts index a102caab76..a404a84b43 100644 --- a/packages/server/src/integrations/airtable.ts +++ b/packages/server/src/integrations/airtable.ts @@ -20,7 +20,9 @@ const SCHEMA: Integration = { "Airtable is a spreadsheet-database hybrid, with the features of a database but applied to a spreadsheet.", friendlyName: "Airtable", type: "Spreadsheet", - features: [DatasourceFeature.CONNECTION_CHECKING], + features: { + [DatasourceFeature.CONNECTION_CHECKING]: true, + }, datasource: { apiKey: { type: DatasourceFieldType.PASSWORD, diff --git a/packages/server/src/integrations/arangodb.ts b/packages/server/src/integrations/arangodb.ts index b486748a68..0127d25632 100644 --- a/packages/server/src/integrations/arangodb.ts +++ b/packages/server/src/integrations/arangodb.ts @@ -23,7 +23,9 @@ const SCHEMA: Integration = { type: "Non-relational", description: "ArangoDB is a scalable open-source multi-model database natively supporting graph, document and search. All supported data models & access patterns can be combined in queries allowing for maximal flexibility. ", - features: [DatasourceFeature.CONNECTION_CHECKING], + features: { + [DatasourceFeature.CONNECTION_CHECKING]: true, + }, datasource: { url: { type: DatasourceFieldType.STRING, diff --git a/packages/server/src/integrations/couchdb.ts b/packages/server/src/integrations/couchdb.ts index 4ccbd5456d..b55468fd93 100644 --- a/packages/server/src/integrations/couchdb.ts +++ b/packages/server/src/integrations/couchdb.ts @@ -20,7 +20,9 @@ const SCHEMA: Integration = { type: "Non-relational", description: "Apache CouchDB is an open-source document-oriented NoSQL database, implemented in Erlang.", - features: [DatasourceFeature.CONNECTION_CHECKING], + features: { + [DatasourceFeature.CONNECTION_CHECKING]: true, + }, datasource: { url: { type: DatasourceFieldType.STRING, diff --git a/packages/server/src/integrations/dynamodb.ts b/packages/server/src/integrations/dynamodb.ts index 28b42c7a54..bede4a7f7c 100644 --- a/packages/server/src/integrations/dynamodb.ts +++ b/packages/server/src/integrations/dynamodb.ts @@ -25,7 +25,9 @@ const SCHEMA: Integration = { "Amazon DynamoDB is a key-value and document database that delivers single-digit millisecond performance at any scale.", friendlyName: "DynamoDB", type: "Non-relational", - features: [DatasourceFeature.CONNECTION_CHECKING], + features: { + [DatasourceFeature.CONNECTION_CHECKING]: true, + }, datasource: { region: { type: DatasourceFieldType.STRING, diff --git a/packages/server/src/integrations/elasticsearch.ts b/packages/server/src/integrations/elasticsearch.ts index af52799c51..7ae0295298 100644 --- a/packages/server/src/integrations/elasticsearch.ts +++ b/packages/server/src/integrations/elasticsearch.ts @@ -22,7 +22,9 @@ const SCHEMA: Integration = { "Elasticsearch is a search engine based on the Lucene library. It provides a distributed, multitenant-capable full-text search engine with an HTTP web interface and schema-free JSON documents.", friendlyName: "ElasticSearch", type: "Non-relational", - features: [DatasourceFeature.CONNECTION_CHECKING], + features: { + [DatasourceFeature.CONNECTION_CHECKING]: true, + }, datasource: { url: { type: DatasourceFieldType.STRING, diff --git a/packages/server/src/integrations/firebase.ts b/packages/server/src/integrations/firebase.ts index 3907275f41..3f03f295a9 100644 --- a/packages/server/src/integrations/firebase.ts +++ b/packages/server/src/integrations/firebase.ts @@ -20,7 +20,9 @@ const SCHEMA: Integration = { type: "Non-relational", description: "Cloud Firestore is a flexible, scalable database for mobile, web, and server development from Firebase and Google Cloud.", - features: [DatasourceFeature.CONNECTION_CHECKING], + features: { + [DatasourceFeature.CONNECTION_CHECKING]: true, + }, datasource: { email: { type: DatasourceFieldType.STRING, diff --git a/packages/server/src/integrations/googlesheets.ts b/packages/server/src/integrations/googlesheets.ts index d1f3f9e950..aa1c2fb66f 100644 --- a/packages/server/src/integrations/googlesheets.ts +++ b/packages/server/src/integrations/googlesheets.ts @@ -66,10 +66,10 @@ const SCHEMA: Integration = { "Create and collaborate on online spreadsheets in real-time and from any device.", friendlyName: "Google Sheets", type: "Spreadsheet", - features: [ - DatasourceFeature.CONNECTION_CHECKING, - DatasourceFeature.FETCH_TABLE_NAMES, - ], + features: { + [DatasourceFeature.CONNECTION_CHECKING]: true, + [DatasourceFeature.FETCH_TABLE_NAMES]: true, + }, datasource: { spreadsheetId: { display: "Google Sheet URL", diff --git a/packages/server/src/integrations/microsoftSqlServer.ts b/packages/server/src/integrations/microsoftSqlServer.ts index a83630afbb..291aad8631 100644 --- a/packages/server/src/integrations/microsoftSqlServer.ts +++ b/packages/server/src/integrations/microsoftSqlServer.ts @@ -40,10 +40,10 @@ const SCHEMA: Integration = { "Microsoft SQL Server is a relational database management system developed by Microsoft. ", friendlyName: "MS SQL Server", type: "Relational", - features: [ - DatasourceFeature.CONNECTION_CHECKING, - DatasourceFeature.FETCH_TABLE_NAMES, - ], + features: { + [DatasourceFeature.CONNECTION_CHECKING]: true, + [DatasourceFeature.FETCH_TABLE_NAMES]: true, + }, datasource: { user: { type: DatasourceFieldType.STRING, diff --git a/packages/server/src/integrations/mongodb.ts b/packages/server/src/integrations/mongodb.ts index ee7302c501..417aa61f41 100644 --- a/packages/server/src/integrations/mongodb.ts +++ b/packages/server/src/integrations/mongodb.ts @@ -40,7 +40,9 @@ const getSchema = () => { type: "Non-relational", description: "MongoDB is a general purpose, document-based, distributed database built for modern application developers and for the cloud era.", - features: [DatasourceFeature.CONNECTION_CHECKING], + features: { + [DatasourceFeature.CONNECTION_CHECKING]: true, + }, datasource: { connectionString: { type: DatasourceFieldType.STRING, diff --git a/packages/server/src/integrations/mysql.ts b/packages/server/src/integrations/mysql.ts index d83ce300d3..0dbd628ea3 100644 --- a/packages/server/src/integrations/mysql.ts +++ b/packages/server/src/integrations/mysql.ts @@ -36,10 +36,10 @@ const SCHEMA: Integration = { type: "Relational", description: "MySQL Database Service is a fully managed database service to deploy cloud-native applications. ", - features: [ - DatasourceFeature.CONNECTION_CHECKING, - DatasourceFeature.FETCH_TABLE_NAMES, - ], + features: { + [DatasourceFeature.CONNECTION_CHECKING]: true, + [DatasourceFeature.FETCH_TABLE_NAMES]: true, + }, datasource: { host: { type: DatasourceFieldType.STRING, diff --git a/packages/server/src/integrations/oracle.ts b/packages/server/src/integrations/oracle.ts index afb7021a74..d8c366814f 100644 --- a/packages/server/src/integrations/oracle.ts +++ b/packages/server/src/integrations/oracle.ts @@ -50,10 +50,10 @@ const SCHEMA: Integration = { type: "Relational", description: "Oracle Database is an object-relational database management system developed by Oracle Corporation", - features: [ - DatasourceFeature.CONNECTION_CHECKING, - DatasourceFeature.FETCH_TABLE_NAMES, - ], + features: { + [DatasourceFeature.CONNECTION_CHECKING]: true, + [DatasourceFeature.FETCH_TABLE_NAMES]: true, + }, datasource: { host: { type: DatasourceFieldType.STRING, diff --git a/packages/server/src/integrations/postgres.ts b/packages/server/src/integrations/postgres.ts index 8633d78a88..47295716e6 100644 --- a/packages/server/src/integrations/postgres.ts +++ b/packages/server/src/integrations/postgres.ts @@ -52,10 +52,10 @@ const SCHEMA: Integration = { type: "Relational", description: "PostgreSQL, also known as Postgres, is a free and open-source relational database management system emphasizing extensibility and SQL compliance.", - features: [ - DatasourceFeature.CONNECTION_CHECKING, - DatasourceFeature.FETCH_TABLE_NAMES, - ], + features: { + [DatasourceFeature.CONNECTION_CHECKING]: true, + [DatasourceFeature.FETCH_TABLE_NAMES]: true, + }, datasource: { host: { type: DatasourceFieldType.STRING, diff --git a/packages/server/src/integrations/redis.ts b/packages/server/src/integrations/redis.ts index d71f66edc1..26666ac800 100644 --- a/packages/server/src/integrations/redis.ts +++ b/packages/server/src/integrations/redis.ts @@ -21,7 +21,9 @@ const SCHEMA: Integration = { "Redis is a caching tool, providing powerful key-value store capabilities.", friendlyName: "Redis", type: "Non-relational", - features: [DatasourceFeature.CONNECTION_CHECKING], + features: { + [DatasourceFeature.CONNECTION_CHECKING]: true, + }, datasource: { host: { type: "string", diff --git a/packages/server/src/integrations/s3.ts b/packages/server/src/integrations/s3.ts index 0f9848ed59..df9d21aab2 100644 --- a/packages/server/src/integrations/s3.ts +++ b/packages/server/src/integrations/s3.ts @@ -24,7 +24,9 @@ const SCHEMA: Integration = { "Amazon Simple Storage Service (Amazon S3) is an object storage service that offers industry-leading scalability, data availability, security, and performance.", friendlyName: "Amazon S3", type: "Object store", - features: [DatasourceFeature.CONNECTION_CHECKING], + features: { + [DatasourceFeature.CONNECTION_CHECKING]: true, + }, datasource: { region: { type: "string", diff --git a/packages/server/src/integrations/snowflake.ts b/packages/server/src/integrations/snowflake.ts index 9b743131ae..73d3d4f6af 100644 --- a/packages/server/src/integrations/snowflake.ts +++ b/packages/server/src/integrations/snowflake.ts @@ -22,7 +22,9 @@ const SCHEMA: Integration = { "Snowflake is a solution for data warehousing, data lakes, data engineering, data science, data application development, and securely sharing and consuming shared data.", friendlyName: "Snowflake", type: "Relational", - features: [DatasourceFeature.CONNECTION_CHECKING], + features: { + [DatasourceFeature.CONNECTION_CHECKING]: true, + }, datasource: { account: { type: "string", diff --git a/packages/types/src/sdk/datasources.ts b/packages/types/src/sdk/datasources.ts index 24ccea7e83..50ea063ca3 100644 --- a/packages/types/src/sdk/datasources.ts +++ b/packages/types/src/sdk/datasources.ts @@ -116,7 +116,7 @@ export interface Integration { docs: string plus?: boolean auth?: { type: string } - features?: DatasourceFeature[] + features?: Partial> relationships?: boolean description: string friendlyName: string From d3eb262bed58a535261017bb83c17a7652f66ccd Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 24 May 2023 11:07:49 +0200 Subject: [PATCH 17/42] Validate onboarding --- .../portal/apps/onboarding/index.svelte | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/packages/builder/src/pages/builder/portal/apps/onboarding/index.svelte b/packages/builder/src/pages/builder/portal/apps/onboarding/index.svelte index 546f5f57ba..18253bb8a2 100644 --- a/packages/builder/src/pages/builder/portal/apps/onboarding/index.svelte +++ b/packages/builder/src/pages/builder/portal/apps/onboarding/index.svelte @@ -18,6 +18,8 @@ import { Roles } from "constants/backend" import Spinner from "components/common/Spinner.svelte" import { helpers } from "@budibase/shared-core" + import { validateDatasourceConfig } from "builderStore/datasource" + import { DatasourceFeature } from "@budibase/types" let name = "My first app" let url = "my-first-app" @@ -108,7 +110,24 @@ isGoogle, }) => { let app + try { + if ( + datasourceConfig && + plusIntegrations[stage].features[DatasourceFeature.CONNECTION_CHECKING] + ) { + const resp = await validateDatasourceConfig({ + config: datasourceConfig, + type: stage, + }) + if (!resp.connected) { + notifications.error( + `Unable to connect - ${resp.error ?? "Error validating datasource"}` + ) + return false + } + } + app = await createApp(useSampleData) let datasource From 0992bf31200bba1f3344b389d4233663f26e9225 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 26 May 2023 00:17:07 +0200 Subject: [PATCH 18/42] Update ref --- packages/pro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pro b/packages/pro index aea8a4acb0..701739c4db 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit aea8a4acb0bae6a1036520bf4c6d8cae428cc7d9 +Subproject commit 701739c4dbfe8b318d94d1f96dff682b8c27b937 From 35d5df7ef0864f50da63f74af52efd1b69012157 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 26 May 2023 10:10:55 +0200 Subject: [PATCH 19/42] Use local refs --- packages/pro | 2 +- packages/server/package.json | 2 +- packages/worker/package.json | 2 +- yarn.lock | 60 +++++++++++++++++++++++------------- 4 files changed, 41 insertions(+), 25 deletions(-) diff --git a/packages/pro b/packages/pro index 701739c4db..2e50f86a53 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 701739c4dbfe8b318d94d1f96dff682b8c27b937 +Subproject commit 2e50f86a534c1e188415299fbf4a2dc835d0859e diff --git a/packages/server/package.json b/packages/server/package.json index 136eeb609b..2ec1395b71 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -47,7 +47,7 @@ "@apidevtools/swagger-parser": "10.0.3", "@budibase/backend-core": "0.0.1", "@budibase/client": "0.0.1", - "@budibase/pro": "develop", + "@budibase/pro": "0.0.1", "@budibase/shared-core": "0.0.1", "@budibase/string-templates": "0.0.1", "@budibase/types": "0.0.1", diff --git a/packages/worker/package.json b/packages/worker/package.json index dded20cf30..684bd8755b 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -38,7 +38,7 @@ "license": "GPL-3.0", "dependencies": { "@budibase/backend-core": "0.0.1", - "@budibase/pro": "develop", + "@budibase/pro": "0.0.1", "@budibase/string-templates": "0.0.1", "@budibase/types": "0.0.1", "@koa/router": "8.0.8", diff --git a/yarn.lock b/yarn.lock index 33225932ea..a8d197d134 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1869,25 +1869,6 @@ pouchdb-promise "^6.0.4" through2 "^2.0.0" -"@budibase/pro@develop": - version "2.6.19-alpha.4" - resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.6.19-alpha.4.tgz#5d4c885ac9ac4ccfb2f8896961aca903c1178750" - integrity sha512-iu2QzV8Z77c00muBSK+NVsZdug3lLD0lR+vcKancGEz1PPE4yNIH7g8jB6i8h9agArbx9S2ICeHQqGb6nQaGHQ== - dependencies: - "@budibase/backend-core" "2.6.19-alpha.4" - "@budibase/shared-core" "2.6.19-alpha.4" - "@budibase/string-templates" "2.6.19-alpha.4" - "@budibase/types" "2.6.19-alpha.4" - "@koa/router" "8.0.8" - bull "4.10.1" - joi "17.6.0" - jsonwebtoken "8.5.1" - lru-cache "^7.14.1" - memorystream "^0.3.1" - node-fetch "^2.6.1" - scim-patch "^0.7.0" - scim2-parse-filter "^0.2.8" - "@budibase/shared-core@2.6.19-alpha.4": version "2.6.19-alpha.4" resolved "https://registry.yarnpkg.com/@budibase/shared-core/-/shared-core-2.6.19-alpha.4.tgz#dd22dd0a18ee4d6739b629f461e5caec90706282" @@ -3543,6 +3524,11 @@ resolved "https://registry.yarnpkg.com/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.14.tgz#add4c98d341472a289190b424efbdb096991bb24" integrity sha512-XPSJHWmi394fuUuzDnGz1wiKqWfo1yXecHQMRf2l6hztTO+nPru658AyDngaBe7isIxEkRsPR3FZh+s7iVa4Uw== +"@jridgewell/sourcemap-codec@^1.4.13": + version "1.4.15" + resolved "https://registry.yarnpkg.com/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.15.tgz#d7c6e6755c78567a951e04ab52ef0fd26de59f32" + integrity sha512-eF2rxCRulEKXHTRiDrDy6erMYWqNw4LPdQ8UQA4huuxaQsVeRPFl2oM8oDGxMFhJUWZf9McpLtJasDDZb/Bpeg== + "@jridgewell/trace-mapping@0.3.9": version "0.3.9" resolved "https://registry.yarnpkg.com/@jridgewell/trace-mapping/-/trace-mapping-0.3.9.tgz#6534fd5933a53ba7cbf3a17615e273a0d1273ff9" @@ -4323,7 +4309,7 @@ dependencies: slash "^3.0.0" -"@rollup/plugin-commonjs@^16.0.0": +"@rollup/plugin-commonjs@16.0.0", "@rollup/plugin-commonjs@^16.0.0": version "16.0.0" resolved "https://registry.yarnpkg.com/@rollup/plugin-commonjs/-/plugin-commonjs-16.0.0.tgz#169004d56cd0f0a1d0f35915d31a036b0efe281f" integrity sha512-LuNyypCP3msCGVQJ7ki8PqYdpjfEkE/xtFa5DqlF+7IBD0JsfMZ87C58heSwIMint58sAUZbt3ITqOmdQv/dXw== @@ -4406,6 +4392,22 @@ "@rollup/pluginutils" "^3.1.0" magic-string "^0.25.7" +"@rollup/plugin-replace@^5.0.2": + version "5.0.2" + resolved "https://registry.yarnpkg.com/@rollup/plugin-replace/-/plugin-replace-5.0.2.tgz#45f53501b16311feded2485e98419acb8448c61d" + integrity sha512-M9YXNekv/C/iHHK+cvORzfRYfPbq0RDD8r0G+bMiTXjNGKulPnCT9O3Ss46WfhI6ZOCgApOP7xAdmCQJ+U2LAA== + dependencies: + "@rollup/pluginutils" "^5.0.1" + magic-string "^0.27.0" + +"@rollup/plugin-typescript@8.3.0": + version "8.3.0" + resolved "https://registry.yarnpkg.com/@rollup/plugin-typescript/-/plugin-typescript-8.3.0.tgz#bc1077fa5897b980fc27e376c4e377882c63e68b" + integrity sha512-I5FpSvLbtAdwJ+naznv+B4sjXZUcIvLLceYpITAn7wAP8W0wqc5noLdGIp9HGVntNhRWXctwPYrSSFQxtl0FPA== + dependencies: + "@rollup/pluginutils" "^3.1.0" + resolve "^1.17.0" + "@rollup/pluginutils@^3.0.8", "@rollup/pluginutils@^3.1.0": version "3.1.0" resolved "https://registry.yarnpkg.com/@rollup/pluginutils/-/pluginutils-3.1.0.tgz#706b4524ee6dc8b103b3c995533e5ad680c02b9b" @@ -12433,7 +12435,7 @@ fs.realpath@^1.0.0: resolved "https://registry.yarnpkg.com/fs.realpath/-/fs.realpath-1.0.0.tgz#1504ad2523158caa40db4a2787cb01411994ea4f" integrity sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw== -fsevents@^2.1.2, fsevents@^2.3.2, fsevents@~2.3.2: +fsevents@^2.1.2, fsevents@^2.3.2, fsevents@~2.3.1, fsevents@~2.3.2: version "2.3.2" resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-2.3.2.tgz#8a526f78b8fdf4623b709e0b975c52c24c02fd1a" integrity sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA== @@ -17481,6 +17483,13 @@ magic-string@^0.26.2: dependencies: sourcemap-codec "^1.4.8" +magic-string@^0.27.0: + version "0.27.0" + resolved "https://registry.yarnpkg.com/magic-string/-/magic-string-0.27.0.tgz#e4a3413b4bab6d98d2becffd48b4a257effdbbf3" + integrity sha512-8UnnX2PeRAPZuN12svgR9j7M1uWMovg/CEnIwIG0LFkXSJJe4PdfUGiTGl8V9bsBHFUtfVINcSyYxd7q+kx9fA== + dependencies: + "@jridgewell/sourcemap-codec" "^1.4.13" + make-dir@3.1.0, make-dir@^3.0.0, make-dir@^3.1.0: version "3.1.0" resolved "https://registry.yarnpkg.com/make-dir/-/make-dir-3.1.0.tgz#415e967046b3a7f1d185277d84aa58203726a13f" @@ -22307,6 +22316,13 @@ rollup-pluginutils@^2.3.1, rollup-pluginutils@^2.5.0, rollup-pluginutils@^2.6.0, dependencies: estree-walker "^0.6.1" +rollup@2.45.2: + version "2.45.2" + resolved "https://registry.yarnpkg.com/rollup/-/rollup-2.45.2.tgz#8fb85917c9f35605720e92328f3ccbfba6f78b48" + integrity sha512-kRRU7wXzFHUzBIv0GfoFFIN3m9oteY4uAsKllIpQDId5cfnkWF2J130l+27dzDju0E6MScKiV0ZM5Bw8m4blYQ== + optionalDependencies: + fsevents "~2.3.1" + rollup@^2.36.2, rollup@^2.44.0, rollup@^2.45.2, rollup@^2.79.1: version "2.79.1" resolved "https://registry.yarnpkg.com/rollup/-/rollup-2.79.1.tgz#bedee8faef7c9f93a2647ac0108748f497f081c7" @@ -24312,7 +24328,7 @@ timed-out@^4.0.1: resolved "https://registry.yarnpkg.com/timed-out/-/timed-out-4.0.1.tgz#f32eacac5a175bea25d7fab565ab3ed8741ef56f" integrity sha512-G7r3AhovYtr5YKOWQkta8RKAPb+J9IsO4uVmzjl8AZwfhs8UcUwTiD6gcJYSgOtzyjvQKrKYn41syHbUWMkafA== -timekeeper@2.2.0: +timekeeper@2.2.0, timekeeper@^2.2.0: version "2.2.0" resolved "https://registry.yarnpkg.com/timekeeper/-/timekeeper-2.2.0.tgz#9645731fce9e3280a18614a57a9d1b72af3ca368" integrity sha512-W3AmPTJWZkRwu+iSNxPIsLZ2ByADsOLbbLxe46UJyWj3mlYLlwucKiq+/dPm0l9wTzqoF3/2PH0AGFCebjq23A== From ee0fc258cb2c4d698075ad69fcaa8bb2f6163364 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 26 May 2023 10:11:18 +0200 Subject: [PATCH 20/42] Update yarn.lock --- yarn.lock | 67 ------------------------------------------------------- 1 file changed, 67 deletions(-) diff --git a/yarn.lock b/yarn.lock index a8d197d134..0d9d795b8a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1728,47 +1728,6 @@ resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw== -"@budibase/backend-core@2.6.19-alpha.4": - version "2.6.19-alpha.4" - resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-2.6.19-alpha.4.tgz#525dd7ba6c5db4404cf00d1165f79d34a1093826" - integrity sha512-1pfOr+J9xYawedVmvqpQ4b/8C2SQP4cKhFmSz5uErM2SCgbRj+JuzOUTPNX0vzAXPvat/kEegt79xThummDvhA== - dependencies: - "@budibase/nano" "10.1.2" - "@budibase/pouchdb-replication-stream" "1.2.10" - "@budibase/types" "2.6.19-alpha.4" - "@shopify/jest-koa-mocks" "5.0.1" - "@techpass/passport-openidconnect" "0.3.2" - aws-cloudfront-sign "2.2.0" - aws-sdk "2.1030.0" - bcrypt "5.0.1" - bcryptjs "2.4.3" - bull "4.10.1" - correlation-id "4.0.0" - dotenv "16.0.1" - emitter-listener "1.1.2" - ioredis "4.28.0" - joi "17.6.0" - jsonwebtoken "9.0.0" - koa-passport "4.1.4" - koa-pino-logger "4.0.0" - lodash "4.17.21" - lodash.isarguments "3.1.0" - node-fetch "2.6.7" - passport-google-oauth "2.0.0" - passport-jwt "4.0.0" - passport-local "1.0.0" - passport-oauth2-refresh "^2.1.0" - pino "8.11.0" - pino-http "8.3.3" - posthog-node "1.3.0" - pouchdb "7.3.0" - pouchdb-find "7.2.2" - redlock "4.2.0" - sanitize-s3-objectkey "0.0.1" - semver "7.3.7" - tar-fs "2.1.1" - uuid "8.3.2" - "@budibase/bbui@^0.9.139": version "0.9.190" resolved "https://registry.yarnpkg.com/@budibase/bbui/-/bbui-0.9.190.tgz#e1ec400ac90f556bfbc80fc23a04506f1585ea81" @@ -1869,13 +1828,6 @@ pouchdb-promise "^6.0.4" through2 "^2.0.0" -"@budibase/shared-core@2.6.19-alpha.4": - version "2.6.19-alpha.4" - resolved "https://registry.yarnpkg.com/@budibase/shared-core/-/shared-core-2.6.19-alpha.4.tgz#dd22dd0a18ee4d6739b629f461e5caec90706282" - integrity sha512-ac6iWSsgz70OYbdA+QHPLpTnRbIZ4OecVc6Y7gnEZ78hZ4S5da8a+73jswuy0/t4YsiT/gjukjzjoihg3NemVg== - dependencies: - "@budibase/types" "2.6.19-alpha.4" - "@budibase/standard-components@^0.9.139": version "0.9.139" resolved "https://registry.yarnpkg.com/@budibase/standard-components/-/standard-components-0.9.139.tgz#cf8e2b759ae863e469e50272b3ca87f2827e66e3" @@ -1894,25 +1846,6 @@ svelte-apexcharts "^1.0.2" svelte-flatpickr "^3.1.0" -"@budibase/string-templates@2.6.19-alpha.4": - version "2.6.19-alpha.4" - resolved "https://registry.yarnpkg.com/@budibase/string-templates/-/string-templates-2.6.19-alpha.4.tgz#92ebd69a6841174b8af91f338c4754ca7c402707" - integrity sha512-KsH3NlQcJibRj98Q8zQ3KQHhfSIWPQfvR80MmBTIe05llEZGox4re4pQQUnlMafaUEyNNtIqVnbTJ1XP0LmFng== - dependencies: - "@budibase/handlebars-helpers" "^0.11.8" - dayjs "^1.10.4" - handlebars "^4.7.6" - handlebars-utils "^1.0.6" - lodash "^4.17.20" - vm2 "^3.9.15" - -"@budibase/types@2.6.19-alpha.4": - version "2.6.19-alpha.4" - resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.6.19-alpha.4.tgz#bcf81699329d3f8509e4b0a489211f35b6cfa7ce" - integrity sha512-qFsXHZTSigcfCv02aTZGsf17vBT/MC+zK9ky7WZVX4h0sJiE0li4A66/tMaSDz3/vQ7ToPRhJK/p+LOWA/oceg== - dependencies: - scim-patch "^0.7.0" - "@bull-board/api@3.7.0": version "3.7.0" resolved "https://registry.yarnpkg.com/@bull-board/api/-/api-3.7.0.tgz#231f687187c0cb34e0b97f463917b6aaeb4ef6af" From 3db3b3e5687ce19f4b9f2d1a967efd91751fabab Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 26 May 2023 10:25:37 +0200 Subject: [PATCH 21/42] Update ref --- packages/pro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pro b/packages/pro index 2e50f86a53..e0b4b92531 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 2e50f86a534c1e188415299fbf4a2dc835d0859e +Subproject commit e0b4b925313a637a723ec6e22641b9abb41d6291 From 0ca747f6cd70079423a2438c362c48e0cd9004a9 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 26 May 2023 10:30:59 +0200 Subject: [PATCH 22/42] Update ref --- packages/pro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pro b/packages/pro index e0b4b92531..f556147417 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit e0b4b925313a637a723ec6e22641b9abb41d6291 +Subproject commit f55614741741982ba2dbc234dc45cfc3917f01c2 From cb2f63ada8495c5d78dfb6fae93d59409e2d2811 Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Fri, 26 May 2023 08:40:15 +0000 Subject: [PATCH 23/42] Bump version to 2.6.19-alpha.13 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index 6a1a54fb7b..b21582e8bc 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.6.19-alpha.12", + "version": "2.6.19-alpha.13", "npmClient": "yarn", "packages": [ "packages/backend-core", From e63222b58dbfb690a30416899ff225142668f9a8 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 26 May 2023 10:48:31 +0200 Subject: [PATCH 24/42] Update ref --- packages/pro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pro b/packages/pro index f556147417..622a8070b4 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit f55614741741982ba2dbc234dc45cfc3917f01c2 +Subproject commit 622a8070b4c8102e4d27e1f596080731f35728ad From 5f831fe5f884c98c53cf83e28c596dd5b040bca4 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 26 May 2023 10:58:48 +0200 Subject: [PATCH 25/42] Point pro to dev --- packages/pro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pro b/packages/pro index 622a8070b4..a97335b9bb 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 622a8070b4c8102e4d27e1f596080731f35728ad +Subproject commit a97335b9bb86998000258a9df4ccdadf37aa8229 From 25d27dd27f4d996a88fb28b1f8c4a34992b11ef6 Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Fri, 26 May 2023 09:03:29 +0000 Subject: [PATCH 26/42] Bump version to 2.6.19-alpha.14 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index b21582e8bc..eb75f40c92 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.6.19-alpha.13", + "version": "2.6.19-alpha.14", "npmClient": "yarn", "packages": [ "packages/backend-core", From e1a87f2072416dc319881fe19234f2f1c9656548 Mon Sep 17 00:00:00 2001 From: Dean Date: Fri, 26 May 2023 10:22:28 +0100 Subject: [PATCH 27/42] Reverted eval call to direct. Indirect appears to cause issues in cloud --- packages/server/src/db/inMemoryView.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/packages/server/src/db/inMemoryView.ts b/packages/server/src/db/inMemoryView.ts index b38c72b29b..4e9301f4ee 100644 --- a/packages/server/src/db/inMemoryView.ts +++ b/packages/server/src/db/inMemoryView.ts @@ -25,9 +25,8 @@ export async function runView( })) ) let fn = (doc: Document, emit: any) => emit(doc._id) - ;(0, eval)( - "fn = " + view?.map?.replace("function (doc)", "function (doc, emit)") - ) + // BUDI-7060 -> indirect eval call appears to cause issues in cloud + eval("fn = " + view?.map?.replace("function (doc)", "function (doc, emit)")) const queryFns: any = { meta: view.meta, map: fn, From aad4c350474f41c5b97d3df30375b7cab7c80028 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 26 May 2023 11:23:14 +0200 Subject: [PATCH 28/42] Point pro to dev --- packages/pro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pro b/packages/pro index a97335b9bb..6265117ac4 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit a97335b9bb86998000258a9df4ccdadf37aa8229 +Subproject commit 6265117ac441bb6b945b6320dea1f460e197233c From 316313a82e3975421ba58999027d8b5c1ff3a789 Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Fri, 26 May 2023 09:27:18 +0000 Subject: [PATCH 29/42] Bump version to 2.6.19-alpha.15 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index eb75f40c92..7936bec389 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.6.19-alpha.14", + "version": "2.6.19-alpha.15", "npmClient": "yarn", "packages": [ "packages/backend-core", From 64c0894457c194fd6d31aa0c83d0dec5d490d9e8 Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Fri, 26 May 2023 10:54:42 +0000 Subject: [PATCH 30/42] Bump version to 2.6.19-alpha.16 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index 7936bec389..56bb3ccc1d 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.6.19-alpha.15", + "version": "2.6.19-alpha.16", "npmClient": "yarn", "packages": [ "packages/backend-core", From 644d7e5d81298416848458ea960afa7c475f2ce3 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 26 May 2023 13:43:41 +0200 Subject: [PATCH 31/42] Update pro ref --- packages/pro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pro b/packages/pro index 6265117ac4..2d6f999586 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 6265117ac441bb6b945b6320dea1f460e197233c +Subproject commit 2d6f999586fcb62bc98b26416ee406f6328e6615 From deb0dc1b9909213c64cf7a45e2e3ce8eb166510b Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Fri, 26 May 2023 11:47:46 +0000 Subject: [PATCH 32/42] Bump version to 2.6.19-alpha.17 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index 56bb3ccc1d..f2dd31c691 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.6.19-alpha.16", + "version": "2.6.19-alpha.17", "npmClient": "yarn", "packages": [ "packages/backend-core", From 099f6fe311bdd39bd4e66591b8816cc517dd1d8e Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 26 May 2023 13:53:37 +0200 Subject: [PATCH 33/42] Build worker using build script --- packages/worker/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/worker/package.json b/packages/worker/package.json index 684bd8755b..68a308347f 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -13,7 +13,7 @@ ], "scripts": { "prebuild": "rimraf dist/", - "build": "cd ../.. && nx build @budibase/worker", + "build": "node ../../scripts/build.js", "build:dev": "yarn prebuild && tsc --build --watch --preserveWatchOutput", "run:docker": "node dist/index.js", "debug": "yarn build && node --expose-gc --inspect=9223 dist/index.js", From 489ca561b731fc8269aa1a7e71d150caaf52995c Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 26 May 2023 13:54:10 +0200 Subject: [PATCH 34/42] Check types --- .github/workflows/budibase_ci.yml | 5 ++++- package.json | 1 + packages/server/package.json | 1 + packages/worker/package.json | 1 + 4 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml index a15e3d9110..4e0bc8b3a9 100644 --- a/.github/workflows/budibase_ci.yml +++ b/.github/workflows/budibase_ci.yml @@ -44,7 +44,10 @@ jobs: node-version: 14.x cache: "yarn" - run: yarn - - run: yarn nx run-many -t=build --configuration=production + # Run build all the projects + - run: yarn build + # Check the types of the projects built via esbuild + - run: yarn check:types test-libraries: runs-on: ubuntu-latest diff --git a/package.json b/package.json index edbb82b892..096cfea9e5 100644 --- a/package.json +++ b/package.json @@ -34,6 +34,7 @@ "bootstrap": "./scripts/link-dependencies.sh && echo '***BOOTSTRAP ONLY REQUIRED FOR USE WITH ACCOUNT PORTAL***'", "build": "yarn nx run-many -t=build", "build:dev": "lerna run --stream prebuild && yarn nx run-many --target=build --output-style=dynamic --watch --preserveWatchOutput", + "check:types": "lerna run check:types --skip-nx-cache", "backend:bootstrap": "./scripts/scopeBackend.sh && yarn run bootstrap", "backend:build": "./scripts/scopeBackend.sh 'lerna run --stream build'", "build:sdk": "lerna run --stream build:sdk", diff --git a/packages/server/package.json b/packages/server/package.json index 2ec1395b71..0fc7ce53bd 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -11,6 +11,7 @@ "scripts": { "prebuild": "rimraf dist/", "build": "node ./scripts/build.js", + "check:types": "tsc -p tsconfig.build.json --noEmit", "postbuild": "copyfiles -f ../client/dist/budibase-client.js ../client/manifest.json client", "build:dev": "yarn prebuild && tsc --build --watch --preserveWatchOutput", "debug": "yarn build && node --expose-gc --inspect=9222 dist/index.js", diff --git a/packages/worker/package.json b/packages/worker/package.json index 68a308347f..0a589e17ec 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -14,6 +14,7 @@ "scripts": { "prebuild": "rimraf dist/", "build": "node ../../scripts/build.js", + "check:types": "tsc -p tsconfig.build.json --noEmit", "build:dev": "yarn prebuild && tsc --build --watch --preserveWatchOutput", "run:docker": "node dist/index.js", "debug": "yarn build && node --expose-gc --inspect=9223 dist/index.js", From 7a439b8e01123708fb5d5ccc47d9c81b3304ec0b Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 26 May 2023 13:59:50 +0200 Subject: [PATCH 35/42] Remove nx/esbuild refs --- package.json | 1 - packages/worker/project.json | 48 ------------------------------------ yarn.lock | 22 ----------------- 3 files changed, 71 deletions(-) delete mode 100644 packages/worker/project.json diff --git a/package.json b/package.json index 096cfea9e5..b866dc3e23 100644 --- a/package.json +++ b/package.json @@ -4,7 +4,6 @@ "devDependencies": { "@esbuild-plugins/node-resolve": "^0.2.2", "@esbuild-plugins/tsconfig-paths": "^0.1.2", - "@nx/esbuild": "16.2.1", "@nx/js": "16.2.1", "@rollup/plugin-json": "^4.0.2", "@typescript-eslint/parser": "5.45.0", diff --git a/packages/worker/project.json b/packages/worker/project.json deleted file mode 100644 index 65f9b2d567..0000000000 --- a/packages/worker/project.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "name": "@budibase/worker", - "$schema": "../../node_modules/nx/schemas/project-schema.json", - "targets": { - "build": { - "executor": "@nx/esbuild:esbuild", - "outputs": ["{options.outputPath}"], - "options": { - "main": "packages/worker/src/index.ts", - "outputPath": "packages/worker/dist", - "outputFileName": "index.js", - "tsConfig": "packages/worker/tsconfig.build.json", - "assets": [ - { - "glob": "**/*.hbs", - "input": "packages/worker/src/constants/templates", - "output": "." - } - ], - "external": ["graphql/*", "deasync", "mock-aws-s3", "nock"], - "format": ["cjs"], - "esbuildOptions": { - "outExtension": { - ".js": ".js" - }, - "sourcemap": true - }, - "minify": true, - "generatePackageJson": true, - "skipTypeCheck": true - }, - "configurations": { - "production": { - "skipTypeCheck": false, - "esbuildOptions": { - "sourcemap": false - } - } - }, - "dependsOn": [ - { - "projects": ["@budibase/types", "@budibase/string-templates"], - "target": "build" - } - ] - } - } -} diff --git a/yarn.lock b/yarn.lock index 0d9d795b8a..3eb037b40c 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3780,13 +3780,6 @@ dependencies: "@nx/devkit" "16.2.2" -"@nrwl/esbuild@16.2.1": - version "16.2.1" - resolved "https://registry.yarnpkg.com/@nrwl/esbuild/-/esbuild-16.2.1.tgz#83f8fbf2c7c541c220ccfff8be17a0e542aa7d01" - integrity sha512-qyNpdtPAzk2IhYmK5Qzn9gM1cvEtWFp6vJfgdbCKQngdqioYcJpGtdKhRm6Vcj8mFXF7zDgtUd2ecqJSiJU6VA== - dependencies: - "@nx/esbuild" "16.2.1" - "@nrwl/js@16.2.1": version "16.2.1" resolved "https://registry.yarnpkg.com/@nrwl/js/-/js-16.2.1.tgz#6eacfa1f0658ca1e288da86b6c38be4846f2779a" @@ -3839,21 +3832,6 @@ tmp "~0.2.1" tslib "^2.3.0" -"@nx/esbuild@16.2.1": - version "16.2.1" - resolved "https://registry.yarnpkg.com/@nx/esbuild/-/esbuild-16.2.1.tgz#157a408a617b5095ba1372ec27b03d6c9aa3b78c" - integrity sha512-jG4zsGc1EN+SfEBUky44+4cpj7M0Sf0v3pGWsw4hCOuBeLl6qW+eAcFypDXecDosS7ct/M2nEtlZP45IVDMZgQ== - dependencies: - "@nrwl/esbuild" "16.2.1" - "@nx/devkit" "16.2.1" - "@nx/js" "16.2.1" - chalk "^4.1.0" - dotenv "~10.0.0" - fast-glob "3.2.7" - fs-extra "^11.1.0" - tsconfig-paths "^4.1.2" - tslib "^2.3.0" - "@nx/js@16.2.1": version "16.2.1" resolved "https://registry.yarnpkg.com/@nx/js/-/js-16.2.1.tgz#41c8c2d610131fa064bbb2b6bb25bd67ed06be79" From 296de6b1b80e13bc88465adff9e544aea2d54d7e Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 26 May 2023 14:01:35 +0200 Subject: [PATCH 36/42] Fix dockerfile --- packages/worker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/worker/Dockerfile b/packages/worker/Dockerfile index 7b8f991013..a80ae00af1 100644 --- a/packages/worker/Dockerfile +++ b/packages/worker/Dockerfile @@ -12,7 +12,7 @@ RUN apk add --no-cache --virtual .gyp python3 make g++ RUN yarn global add pm2 -COPY dist/package.json . +COPY package.json . RUN yarn install --frozen-lockfile --production=true # Remove unneeded data from file system to reduce image size RUN apk del .gyp \ From baeb465a956c9a84899f45b53e459e066c940e49 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 26 May 2023 14:02:03 +0200 Subject: [PATCH 37/42] Fix dockerfile --- packages/worker/.dockerignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/worker/.dockerignore b/packages/worker/.dockerignore index c4815909ba..8f79e19685 100644 --- a/packages/worker/.dockerignore +++ b/packages/worker/.dockerignore @@ -1,3 +1,4 @@ * !/dist/ -!/docker_run.sh \ No newline at end of file +!/docker_run.sh +!/package.json \ No newline at end of file From 993348f7664cc71a61e85c63bfdf6f62a0d692e4 Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Fri, 26 May 2023 12:17:02 +0000 Subject: [PATCH 38/42] Bump version to 2.6.19-alpha.18 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index f2dd31c691..df67d8122c 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.6.19-alpha.17", + "version": "2.6.19-alpha.18", "npmClient": "yarn", "packages": [ "packages/backend-core", From 5d93ea6fffa4ca79337f2f65ed1224b718404329 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 26 May 2023 13:28:08 +0100 Subject: [PATCH 39/42] Adding environment variable file for dev:docker command. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index b866dc3e23..35452b024a 100644 --- a/package.json +++ b/package.json @@ -52,7 +52,7 @@ "dev:noserver": "yarn run kill-builder && lerna run --stream dev:stack:up && lerna run --stream --parallel dev:builder --ignore @budibase/backend-core --ignore @budibase/server --ignore @budibase/worker", "dev:server": "yarn run kill-server && lerna run --stream --parallel dev:builder --scope @budibase/worker --scope @budibase/server", "dev:built": "yarn run kill-all && cd packages/server && yarn dev:stack:up && cd ../../ && lerna run --stream --parallel dev:built", - "dev:docker": "yarn build && docker-compose -f hosting/docker-compose.dev.yaml -f hosting/docker-compose.build.yaml up --build --scale proxy-service=0 ", + "dev:docker": "yarn build && docker-compose -f hosting/docker-compose.build.yaml -f hosting/docker-compose.dev.yaml --env-file hosting/.env up --build --scale proxy-service=0", "test": "lerna run --stream test --stream", "lint:eslint": "eslint packages && eslint qa-core", "lint:prettier": "prettier --check \"packages/**/*.{js,ts,svelte}\" && prettier --write \"examples/**/*.{js,ts,svelte}\" && prettier --check \"qa-core/**/*.{js,ts,svelte}\"", From 844293cf988635f1f0f03a38bb01c5621af8bb20 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 26 May 2023 14:38:41 +0200 Subject: [PATCH 40/42] Reset pro dependencies before the npm push --- .github/workflows/release-develop.yml | 3 +++ .github/workflows/release-master.yml | 3 +++ scripts/resetProDependencies.js | 38 +++++++++++++++++++++++++++ 3 files changed, 44 insertions(+) create mode 100755 scripts/resetProDependencies.js diff --git a/.github/workflows/release-develop.yml b/.github/workflows/release-develop.yml index c724b717e2..ebd2f4c3ef 100644 --- a/.github/workflows/release-develop.yml +++ b/.github/workflows/release-develop.yml @@ -54,6 +54,9 @@ jobs: - run: yarn build --configuration=production - run: yarn build:sdk + - name: Reset pro dependencies + run: node scripts/resetProDependencies.js + - name: Publish budibase packages to NPM env: NPM_TOKEN: ${{ secrets.NPM_TOKEN }} diff --git a/.github/workflows/release-master.yml b/.github/workflows/release-master.yml index 4959194064..304b0de100 100644 --- a/.github/workflows/release-master.yml +++ b/.github/workflows/release-master.yml @@ -60,6 +60,9 @@ jobs: - run: yarn build --configuration=production - run: yarn build:sdk + - name: Reset pro dependencies + run: node scripts/resetProDependencies.js + - name: Publish budibase packages to NPM env: NPM_TOKEN: ${{ secrets.NPM_TOKEN }} diff --git a/scripts/resetProDependencies.js b/scripts/resetProDependencies.js new file mode 100755 index 0000000000..c6752c44c6 --- /dev/null +++ b/scripts/resetProDependencies.js @@ -0,0 +1,38 @@ +const fs = require("fs") +const path = require("path") +const { execSync } = require("child_process") + +// Get the list of workspaces with mismatched dependencies +const output = execSync("yarn --silent workspaces info --json", { + encoding: "utf-8", +}) +const data = JSON.parse(output) + +const packageJsonPath = path.join( + data["@budibase/pro"].location, + "package.json" +) +const packageJson = JSON.parse(fs.readFileSync(packageJsonPath, "utf-8")) + +let hasChanges = false + +const dependencies = data["@budibase/pro"].workspaceDependencies +dependencies.forEach(dependency => { + if (packageJson.dependencies?.[dependency]) { + packageJson.dependencies[dependency] = "0.0.1" + hasChanges = true + } + if (packageJson.devDependencies?.[dependency]) { + packageJson.devDependencies[dependency] = "0.0.1" + hasChanges = true + } + if (packageJson.peerDependencies?.[dependency]) { + packageJson.peerDependencies[dependency] = "0.0.1" + hasChanges = true + } +}) + +// Write changes to package.json if there are any +if (hasChanges) { + fs.writeFileSync(packageJsonPath, JSON.stringify(packageJson, null, 2) + "\n") +} From b8ccb67b47e0466f1999c2b975fd3d06b5183a66 Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Fri, 26 May 2023 12:49:46 +0000 Subject: [PATCH 41/42] Bump version to 2.6.19-alpha.19 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index df67d8122c..d8c2cc6192 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.6.19-alpha.18", + "version": "2.6.19-alpha.19", "npmClient": "yarn", "packages": [ "packages/backend-core", From a9b5d966dca4b8df4900bd2780b682d3f88286eb Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Fri, 26 May 2023 14:26:29 +0000 Subject: [PATCH 42/42] Bump version to 2.6.19-alpha.20 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index d8c2cc6192..5884929a22 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.6.19-alpha.19", + "version": "2.6.19-alpha.20", "npmClient": "yarn", "packages": [ "packages/backend-core",