From b839325a86e86dbec8adc9b94f9a3c63eda86ab4 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Fri, 26 Aug 2022 08:47:50 +0100 Subject: [PATCH 001/151] Add initial work on grid layout --- .../src/builderStore/store/frontend.js | 19 ++- .../[screenId]/_components/AppPreview.svelte | 2 + .../_components/settings/componentStyles.js | 61 ++++++++ .../new/_components/componentStructure.json | 3 +- packages/client/manifest.json | 9 ++ .../client/src/components/Component.svelte | 4 + .../client/src/components/app/Grid.svelte | 81 +++++++++++ packages/client/src/components/app/index.js | 1 + .../src/components/preview/DNDHandler.svelte | 137 ++++++++++++++++-- .../src/components/preview/Indicator.svelte | 100 +++++++++++++ .../components/preview/IndicatorSet.svelte | 1 + packages/client/src/index.js | 4 + packages/client/src/stores/builder.js | 15 +- packages/client/src/utils/styleable.js | 2 +- 14 files changed, 423 insertions(+), 16 deletions(-) create mode 100644 packages/client/src/components/app/Grid.svelte diff --git a/packages/builder/src/builderStore/store/frontend.js b/packages/builder/src/builderStore/store/frontend.js index 4d0653208c..9deeef15f1 100644 --- a/packages/builder/src/builderStore/store/frontend.js +++ b/packages/builder/src/builderStore/store/frontend.js @@ -439,7 +439,16 @@ export const getFrontendStore = () => { return { _id: Helpers.uuid(), _component: definition.component, - _styles: { normal: {}, hover: {}, active: {} }, + _styles: { + normal: { + "grid-column-start": 1, + "grid-column-end": 2, + "grid-row-start": 1, + "grid-row-end": 2, + }, + hover: {}, + active: {}, + }, _instanceName: `New ${definition.name}`, ...cloneDeep(props), ...extras, @@ -873,6 +882,14 @@ export const getFrontendStore = () => { } }) }, + updateStyles: async styles => { + await store.actions.components.patch(component => { + component._styles.normal = { + ...component._styles.normal, + ...styles, + } + }) + }, updateCustomStyle: async style => { await store.actions.components.patch(component => { component._styles.custom = style diff --git a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/AppPreview.svelte b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/AppPreview.svelte index 9f81effd1d..ba132053b7 100644 --- a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/AppPreview.svelte +++ b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/AppPreview.svelte @@ -143,6 +143,8 @@ } } else if (type === "update-prop") { await store.actions.components.updateSetting(data.prop, data.value) + } else if (type === "update-styles") { + await store.actions.components.updateStyles(data.styles) } else if (type === "delete-component" && data.id) { // Legacy type, can be deleted in future confirmDeleteComponent(data.id) diff --git a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/components/[componentId]/_components/settings/componentStyles.js b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/components/[componentId]/_components/settings/componentStyles.js index d4912241b3..710cbed9b1 100644 --- a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/components/[componentId]/_components/settings/componentStyles.js +++ b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/components/[componentId]/_components/settings/componentStyles.js @@ -1,6 +1,67 @@ import { Input, Select } from "@budibase/bbui" import ColorPicker from "components/design/settings/controls/ColorPicker.svelte" +export const grid = { + label: "Grid", + columns: "1fr 1fr", + settings: [ + { + label: "Col. start", + key: "grid-column-start", + control: Select, + placeholder: "Auto", + options: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], + }, + { + label: "Col. end", + key: "grid-column-end", + control: Select, + placeholder: "Auto", + options: [ + { label: "1", value: 2 }, + { label: "2", value: 3 }, + { label: "3", value: 4 }, + { label: "4", value: 5 }, + { label: "5", value: 6 }, + { label: "6", value: 7 }, + { label: "7", value: 8 }, + { label: "8", value: 9 }, + { label: "9", value: 10 }, + { label: "10", value: 11 }, + { label: "11", value: 12 }, + { label: "12", value: 13 }, + ], + }, + { + label: "Row start", + key: "grid-row-start", + control: Select, + placeholder: "Auto", + options: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], + }, + { + label: "Row end", + key: "grid-row-end", + control: Select, + placeholder: "Auto", + options: [ + { label: "1", value: 2 }, + { label: "2", value: 3 }, + { label: "3", value: 4 }, + { label: "4", value: 5 }, + { label: "5", value: 6 }, + { label: "6", value: 7 }, + { label: "7", value: 8 }, + { label: "8", value: 9 }, + { label: "9", value: 10 }, + { label: "10", value: 11 }, + { label: "11", value: 12 }, + { label: "12", value: 13 }, + ], + }, + ], +} + export const margin = { label: "Margin", columns: "1fr 1fr", diff --git a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/components/[componentId]/new/_components/componentStructure.json b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/components/[componentId]/new/_components/componentStructure.json index acd28c6a41..fddff9d0d9 100644 --- a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/components/[componentId]/new/_components/componentStructure.json +++ b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/components/[componentId]/new/_components/componentStructure.json @@ -13,7 +13,8 @@ "icon": "ClassicGridView", "children": [ "container", - "section" + "section", + "grid" ] }, { diff --git a/packages/client/manifest.json b/packages/client/manifest.json index 87cc8b2567..642b4e0f99 100644 --- a/packages/client/manifest.json +++ b/packages/client/manifest.json @@ -86,6 +86,7 @@ "hasChildren": true, "showSettingsBar": true, "styles": [ + "grid", "padding", "size", "background", @@ -4378,5 +4379,13 @@ "required": true } ] + }, + "grid": { + "name": "Grid", + "icon": "ViewGrid", + "hasChildren": true, + "styles": [ + "size" + ] } } \ No newline at end of file diff --git a/packages/client/src/components/Component.svelte b/packages/client/src/components/Component.svelte index a7f506a387..e55b5288e5 100644 --- a/packages/client/src/components/Component.svelte +++ b/packages/client/src/components/Component.svelte @@ -151,6 +151,10 @@ children: children.length, styles: { ...instance._styles, + normal: { + ...instance._styles?.normal, + ...(selected ? $builderStore.gridStyles : null), + }, id, empty: emptyState, interactive, diff --git a/packages/client/src/components/app/Grid.svelte b/packages/client/src/components/app/Grid.svelte new file mode 100644 index 0000000000..51c5162386 --- /dev/null +++ b/packages/client/src/components/app/Grid.svelte @@ -0,0 +1,81 @@ + + +
+
+ {#each coords as coord} +
+ {/each} +
+ + {#if $builderStore.isDragging} +
+ {#each coords as coord} +
+ {/each} +
+ {/if} +
+ + diff --git a/packages/client/src/components/app/index.js b/packages/client/src/components/app/index.js index 1c0d868433..b64e074115 100644 --- a/packages/client/src/components/app/index.js +++ b/packages/client/src/components/app/index.js @@ -34,6 +34,7 @@ export { default as spectrumcard } from "./SpectrumCard.svelte" export { default as tag } from "./Tag.svelte" export { default as markdownviewer } from "./MarkdownViewer.svelte" export { default as embeddedmap } from "./embedded-map/EmbeddedMap.svelte" +export { default as grid } from "./Grid.svelte" export * from "./charts" export * from "./forms" export * from "./table" diff --git a/packages/client/src/components/preview/DNDHandler.svelte b/packages/client/src/components/preview/DNDHandler.svelte index c37eb93afa..719f17efe3 100644 --- a/packages/client/src/components/preview/DNDHandler.svelte +++ b/packages/client/src/components/preview/DNDHandler.svelte @@ -12,7 +12,7 @@ import { get } from "svelte/store" import IndicatorSet from "./IndicatorSet.svelte" import DNDPositionIndicator from "./DNDPositionIndicator.svelte" - import { builderStore } from "stores" + import { builderStore, componentStore } from "stores" let dragInfo let dropInfo @@ -35,22 +35,41 @@ const getDOMNodeForComponent = component => { const parent = component.closest(".component") - const children = Array.from(parent.children) - return children[0] + const children = Array.from(parent?.children || []) + return children?.[0] } // Callback when initially starting a drag on a draggable component const onDragStart = e => { - const parent = e.target.closest(".component") - if (!parent?.classList.contains("draggable")) { + var img = new Image() + img.src = + "data:image/gif;base64,R0lGODlhAQABAIAAAAUEBAAAACwAAAAAAQABAAACAkQBADs=" + e.dataTransfer.setDragImage(img, 0, 0) + + // Resize component + if (e.target.classList.contains("anchor")) { + dragInfo = { + target: e.target.dataset.id, + side: e.target.dataset.side, + mode: "resize", + } + } else { + // Drag component + const parent = e.target.closest(".component") + if (!parent?.classList.contains("draggable")) { + return + } + dragInfo = { + target: parent.dataset.id, + parent: parent.dataset.parent, + mode: "move", + } + } + + if (!dragInfo) { return } - // Update state - dragInfo = { - target: parent.dataset.id, - parent: parent.dataset.parent, - } builderStore.actions.selectComponent(dragInfo.target) builderStore.actions.setDragging(true) @@ -71,20 +90,48 @@ } } + // Update grid styles + if ($builderStore.gridStyles) { + builderStore.actions.updateStyles($builderStore.gridStyles) + } + // Reset state and styles dragInfo = null dropInfo = null - builderStore.actions.setDragging(false) } // Callback when on top of a component const onDragOver = e => { // Skip if we aren't validly dragging currently - if (!dragInfo || !dropInfo) { + if (!dragInfo) { return } e.preventDefault() + + // Set drag info for grids if not set + if (!dragInfo.grid) { + const coord = e.target.closest(".grid-coord") + if (coord) { + const row = parseInt(coord.dataset.row) + const col = parseInt(coord.dataset.col) + const component = $componentStore.selectedComponent + const getStyle = x => parseInt(component._styles.normal?.[x] || "0") + dragInfo.grid = { + startRow: row, + startCol: col, + rowDeltaMin: 1 - getStyle("grid-row-start"), + rowDeltaMax: 13 - getStyle("grid-row-end"), + colDeltaMin: 1 - getStyle("grid-column-start"), + colDeltaMax: 13 - getStyle("grid-column-end"), + } + } + } + + if (!dropInfo) { + return + } + const { droppableInside, bounds } = dropInfo const { top, left, height, width } = bounds const mouseY = e.clientY @@ -147,6 +194,72 @@ return } + const coord = e.target.closest(".grid-coord") + if (coord && dragInfo.grid) { + const row = parseInt(coord.dataset.row) + const col = parseInt(coord.dataset.col) + const { mode, side, grid } = dragInfo + const { + startRow, + startCol, + rowDeltaMin, + rowDeltaMax, + colDeltaMin, + colDeltaMax, + } = grid + + const component = $componentStore.selectedComponent + const rowStart = parseInt( + component._styles.normal?.["grid-row-start"] || 0 + ) + const rowEnd = parseInt(component._styles.normal?.["grid-row-end"] || 0) + const colStart = parseInt( + component._styles.normal?.["grid-column-start"] || 0 + ) + const colEnd = parseInt( + component._styles.normal?.["grid-column-end"] || 0 + ) + + let rowDelta = row - startRow + let colDelta = col - startCol + + if (mode === "move") { + rowDelta = Math.min(Math.max(rowDelta, rowDeltaMin), rowDeltaMax) + colDelta = Math.min(Math.max(colDelta, colDeltaMin), colDeltaMax) + builderStore.actions.setGridStyles({ + "grid-row-start": rowStart + rowDelta, + "grid-row-end": rowEnd + rowDelta, + "grid-column-start": colStart + colDelta, + "grid-column-end": colEnd + colDelta, + }) + } else if (mode === "resize") { + let newStyles = {} + if (side === "right") { + newStyles["grid-column-end"] = colEnd + colDelta + } else if (side === "left") { + newStyles["grid-column-start"] = colStart + colDelta + } else if (side === "top") { + newStyles["grid-row-start"] = rowStart + rowDelta + } else if (side === "bottom") { + newStyles["grid-row-end"] = rowEnd + rowDelta + } else if (side === "bottom-right") { + newStyles["grid-column-end"] = colEnd + colDelta + newStyles["grid-row-end"] = rowEnd + rowDelta + } else if (side === "bottom-left") { + newStyles["grid-column-start"] = colStart + colDelta + newStyles["grid-row-end"] = rowEnd + rowDelta + } else if (side === "top-right") { + newStyles["grid-column-end"] = colEnd + colDelta + newStyles["grid-row-start"] = rowStart + rowDelta + } else if (side === "top-left") { + newStyles["grid-column-start"] = colStart + colDelta + newStyles["grid-row-start"] = rowStart + rowDelta + } + builderStore.actions.setGridStyles(newStyles) + } + } + return + const element = e.target.closest(".component:not(.block)") if ( element && diff --git a/packages/client/src/components/preview/Indicator.svelte b/packages/client/src/components/preview/Indicator.svelte index 1cb669bdc4..d450dd666a 100644 --- a/packages/client/src/components/preview/Indicator.svelte +++ b/packages/client/src/components/preview/Indicator.svelte @@ -13,6 +13,7 @@ export let transition = false export let line = false export let alignRight = false + export let componentId $: flipped = top < 24 @@ -40,6 +41,54 @@ {/if}
{/if} +
+
+
+
+
+
+
+
diff --git a/packages/client/src/components/preview/IndicatorSet.svelte b/packages/client/src/components/preview/IndicatorSet.svelte index 662741d100..99db893faa 100644 --- a/packages/client/src/components/preview/IndicatorSet.svelte +++ b/packages/client/src/components/preview/IndicatorSet.svelte @@ -127,6 +127,7 @@ height={indicator.height} text={idx === 0 ? text : null} icon={idx === 0 ? icon : null} + {componentId} {transition} {zIndex} {color} diff --git a/packages/client/src/index.js b/packages/client/src/index.js index b582dab4d3..b590ee9b00 100644 --- a/packages/client/src/index.js +++ b/packages/client/src/index.js @@ -32,6 +32,10 @@ const loadBudibase = () => { const enableDevTools = !get(builderStore).inBuilder && get(appStore).isDevApp devToolsStore.actions.setEnabled(enableDevTools) + if (get(builderStore).gridOffset) { + builderStore.actions.setDragging(false) + } + // Create app if one hasn't been created yet if (!app) { app = new ClientApp({ diff --git a/packages/client/src/stores/builder.js b/packages/client/src/stores/builder.js index 32eb956d52..77540b3205 100644 --- a/packages/client/src/stores/builder.js +++ b/packages/client/src/stores/builder.js @@ -40,6 +40,9 @@ const createBuilderStore = () => { updateProp: (prop, value) => { dispatchEvent("update-prop", { prop, value }) }, + updateStyles: styles => { + dispatchEvent("update-styles", { styles }) + }, keyDown: (key, ctrlKey) => { dispatchEvent("key-down", { key, ctrlKey }) }, @@ -67,7 +70,11 @@ const createBuilderStore = () => { if (dragging === get(store).isDragging) { return } - store.update(state => ({ ...state, isDragging: dragging })) + store.update(state => ({ + ...state, + isDragging: dragging, + gridStyles: null, + })) }, setEditMode: enabled => { if (enabled === get(store).editMode) { @@ -84,6 +91,12 @@ const createBuilderStore = () => { highlightSetting: setting => { dispatchEvent("highlight-setting", { setting }) }, + setGridStyles: styles => { + store.update(state => { + state.gridStyles = styles + return state + }) + }, } return { ...store, diff --git a/packages/client/src/utils/styleable.js b/packages/client/src/utils/styleable.js index b07a3213d9..5cbe74bb68 100644 --- a/packages/client/src/utils/styleable.js +++ b/packages/client/src/utils/styleable.js @@ -27,7 +27,7 @@ export const styleable = (node, styles = {}) => { const setupStyles = (newStyles = {}) => { let baseStyles = {} if (newStyles.empty) { - baseStyles.border = "2px dashed var(--spectrum-global-color-gray-600)" + // baseStyles.border = "2px dashed var(--spectrum-global-color-gray-600)" baseStyles.padding = "var(--spacing-l)" baseStyles.overflow = "hidden" } From 166064f6df8999ca4c3cf6560bba703d8de12342 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Fri, 26 Aug 2022 14:47:29 +0100 Subject: [PATCH 002/151] Don't clear grid dnd state when loading client library --- .../client/src/components/preview/DNDHandler.svelte | 1 + packages/client/src/index.js | 10 ++++++---- packages/client/src/stores/builder.js | 9 +++++++++ 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/packages/client/src/components/preview/DNDHandler.svelte b/packages/client/src/components/preview/DNDHandler.svelte index 719f17efe3..7a89b958ae 100644 --- a/packages/client/src/components/preview/DNDHandler.svelte +++ b/packages/client/src/components/preview/DNDHandler.svelte @@ -93,6 +93,7 @@ // Update grid styles if ($builderStore.gridStyles) { builderStore.actions.updateStyles($builderStore.gridStyles) + builderStore.actions.clearGridNextLoad() } // Reset state and styles diff --git a/packages/client/src/index.js b/packages/client/src/index.js index b590ee9b00..3c2f375ac8 100644 --- a/packages/client/src/index.js +++ b/packages/client/src/index.js @@ -9,6 +9,10 @@ loadSpectrumIcons() let app const loadBudibase = () => { + if (get(builderStore).clearGridNextLoad) { + builderStore.actions.setDragging(false) + } + // Update builder store with any builder flags builderStore.set({ inBuilder: !!window["##BUDIBASE_IN_BUILDER##"], @@ -21,6 +25,8 @@ const loadBudibase = () => { previewDevice: window["##BUDIBASE_PREVIEW_DEVICE##"], navigation: window["##BUDIBASE_PREVIEW_NAVIGATION##"], hiddenComponentIds: window["##BUDIBASE_HIDDEN_COMPONENT_IDS##"], + gridStyles: get(builderStore).gridStyles, + isDragging: get(builderStore).isDragging, }) // Set app ID - this window flag is set by both the preview and the real @@ -32,10 +38,6 @@ const loadBudibase = () => { const enableDevTools = !get(builderStore).inBuilder && get(appStore).isDevApp devToolsStore.actions.setEnabled(enableDevTools) - if (get(builderStore).gridOffset) { - builderStore.actions.setDragging(false) - } - // Create app if one hasn't been created yet if (!app) { app = new ClientApp({ diff --git a/packages/client/src/stores/builder.js b/packages/client/src/stores/builder.js index 77540b3205..1ab8cdb725 100644 --- a/packages/client/src/stores/builder.js +++ b/packages/client/src/stores/builder.js @@ -19,6 +19,8 @@ const createBuilderStore = () => { isDragging: false, navigation: null, hiddenComponentIds: [], + gridStyles: null, + clearGridNextLoad: false, // Legacy - allow the builder to specify a layout layout: null, @@ -74,6 +76,7 @@ const createBuilderStore = () => { ...state, isDragging: dragging, gridStyles: null, + clearGridNextLoad: false, })) }, setEditMode: enabled => { @@ -97,6 +100,12 @@ const createBuilderStore = () => { return state }) }, + clearGridNextLoad: () => { + store.update(state => { + state.clearGridNextLoad = true + return state + }) + }, } return { ...store, From 8592f79eaaacae58cedfbdc420c49f8ffdb8a9ca Mon Sep 17 00:00:00 2001 From: Maurits Lourens Date: Thu, 7 Jul 2022 23:06:57 +0200 Subject: [PATCH 003/151] 6538 - add validation to datasource config modal --- .../IntegrationConfigForm.svelte | 21 +++++++++++++++ .../modals/DatasourceConfigModal.svelte | 3 +++ .../src/helpers/validation/yup/index.js | 27 ++++++++++++++++++- 3 files changed, 50 insertions(+), 1 deletion(-) diff --git a/packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/IntegrationConfigForm.svelte b/packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/IntegrationConfigForm.svelte index 27358df0be..1f740bf35b 100644 --- a/packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/IntegrationConfigForm.svelte +++ b/packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/IntegrationConfigForm.svelte @@ -10,10 +10,14 @@ import KeyValueBuilder from "components/integration/KeyValueBuilder.svelte" import { capitalise } from "helpers" import { IntegrationTypes } from "constants/backend" + import { createValidationStore } from "helpers/validation/yup" + import { createEventDispatcher } from "svelte" export let datasource export let schema export let creating + const validation = createValidationStore() + const dispatch = createEventDispatcher() function filter([key, value]) { if (!value) { @@ -31,6 +35,17 @@ .filter(el => filter(el)) .map(([key]) => key) + // setup the validation for each required field + $: configKeys.forEach(key => { + if (schema[key].required) { + validation.addValidatorType(key, schema[key].type, schema[key].required) + } + }) + // run the validation whenever the config changes + $: validation.check(config) + // dispatch the validation result + $: dispatch("valid", $validation.valid) + let addButton function getDisplayName(key) { @@ -79,6 +94,9 @@ type={schema[configKey].type} on:change bind:value={config[configKey]} + on:blur={($validation.touched[configKey] = true)} + error={$validation.touched[configKey] && + $validation.errors[configKey]} />
{:else} @@ -88,6 +106,9 @@ type={schema[configKey].type} on:change bind:value={config[configKey]} + on:blur={($validation.touched[configKey] = true)} + error={$validation.touched[configKey] && + $validation.errors[configKey]} />
{/if} diff --git a/packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte b/packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte index c8a5bc96eb..edbe55178f 100644 --- a/packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte +++ b/packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte @@ -13,6 +13,7 @@ // kill the reference so the input isn't saved let datasource = cloneDeep(integration) let skipFetch = false + let isValid = false $: name = IntegrationNames[datasource.type] || datasource.name || datasource.type @@ -53,6 +54,7 @@ return true }} size="L" + disabled={!isValid} > (isValid = e.detail)} /> diff --git a/packages/builder/src/helpers/validation/yup/index.js b/packages/builder/src/helpers/validation/yup/index.js index 6783ad7e58..9b38adbf9d 100644 --- a/packages/builder/src/helpers/validation/yup/index.js +++ b/packages/builder/src/helpers/validation/yup/index.js @@ -1,5 +1,5 @@ import { capitalise } from "helpers" -import { object } from "yup" +import { object, string, number } from "yup" import { writable, get } from "svelte/store" import { notifications } from "@budibase/bbui" @@ -20,6 +20,30 @@ export const createValidationStore = () => { validator[propertyName] = propertyValidator } + const addValidatorType = (propertyName, type, required) => { + if (!type || !propertyName) { + return + } + + let propertyValidator + switch (type) { + case "number": + propertyValidator = number() + break + case "email": + propertyValidator = string().email() + break + default: + propertyValidator = string() + } + + if (required) { + propertyValidator = propertyValidator.required() + } + + validator[propertyName] = propertyValidator + } + const check = async values => { const obj = object().shape(validator) // clear the previous errors @@ -62,5 +86,6 @@ export const createValidationStore = () => { set: validation.set, check, addValidator, + addValidatorType, } } From 9c9d2de4a555ce2087fb1bfe3338ff9ff3a5589d Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 3 Oct 2022 11:56:31 +0100 Subject: [PATCH 004/151] Type updates for app backups work in pro + linting. --- packages/types/src/documents/global/index.ts | 1 + packages/types/src/documents/global/schedules.ts | 9 +++++++++ packages/types/src/sdk/licensing/feature.ts | 1 + 3 files changed, 11 insertions(+) create mode 100644 packages/types/src/documents/global/schedules.ts diff --git a/packages/types/src/documents/global/index.ts b/packages/types/src/documents/global/index.ts index 84684df369..edce416a3f 100644 --- a/packages/types/src/documents/global/index.ts +++ b/packages/types/src/documents/global/index.ts @@ -3,3 +3,4 @@ export * from "./user" export * from "./userGroup" export * from "./plugin" export * from "./quotas" +export * from "./schedules" diff --git a/packages/types/src/documents/global/schedules.ts b/packages/types/src/documents/global/schedules.ts new file mode 100644 index 0000000000..357a841a9a --- /dev/null +++ b/packages/types/src/documents/global/schedules.ts @@ -0,0 +1,9 @@ +export enum ScheduleType { + APP_BACKUP = "app_backup", +} + +export enum ScheduleRepeatPeriod { + DAILY = "daily", + WEEKLY = "weekly", + MONTHLY = "monthly", +} diff --git a/packages/types/src/sdk/licensing/feature.ts b/packages/types/src/sdk/licensing/feature.ts index cbd1f4a50c..f06a8d1382 100644 --- a/packages/types/src/sdk/licensing/feature.ts +++ b/packages/types/src/sdk/licensing/feature.ts @@ -1,3 +1,4 @@ export enum Feature { USER_GROUPS = "userGroups", + APP_BACKUPS = "appBackups", } From 29659813efa90cb5d78b03249a69a5ae08b1a99a Mon Sep 17 00:00:00 2001 From: Rory Powell Date: Mon, 3 Oct 2022 14:02:58 +0100 Subject: [PATCH 005/151] Add document and api types --- packages/types/src/api/web/app/backup.ts | 15 +++++++++ packages/types/src/api/web/app/index.ts | 1 + packages/types/src/api/web/index.ts | 2 ++ packages/types/src/api/web/schedule.ts | 15 +++++++++ packages/types/src/api/web/user.ts | 7 ++++ packages/types/src/documents/app/backup.ts | 21 ++++++++++++ packages/types/src/documents/app/index.ts | 1 + packages/types/src/documents/global/index.ts | 2 +- .../types/src/documents/global/schedule.ts | 32 +++++++++++++++++++ .../types/src/documents/global/schedules.ts | 9 ------ .../src/api/controllers/global/users.ts | 4 ++- packages/worker/src/sdk/users/users.ts | 7 +++- 12 files changed, 104 insertions(+), 12 deletions(-) create mode 100644 packages/types/src/api/web/app/backup.ts create mode 100644 packages/types/src/api/web/app/index.ts create mode 100644 packages/types/src/api/web/schedule.ts create mode 100644 packages/types/src/documents/app/backup.ts create mode 100644 packages/types/src/documents/global/schedule.ts delete mode 100644 packages/types/src/documents/global/schedules.ts diff --git a/packages/types/src/api/web/app/backup.ts b/packages/types/src/api/web/app/backup.ts new file mode 100644 index 0000000000..57ffba0d70 --- /dev/null +++ b/packages/types/src/api/web/app/backup.ts @@ -0,0 +1,15 @@ +import { AppBackupTrigger } from "../../../documents" + +export interface SearchAppBackupsRequest { + trigger: AppBackupTrigger + startDate: string + endDate: string +} + +export interface CreateAppBackupRequest { + name: string +} + +export interface UpdateAppBackupRequest { + name: string +} diff --git a/packages/types/src/api/web/app/index.ts b/packages/types/src/api/web/app/index.ts new file mode 100644 index 0000000000..1d73755cb6 --- /dev/null +++ b/packages/types/src/api/web/app/index.ts @@ -0,0 +1 @@ +export * from "./backup" diff --git a/packages/types/src/api/web/index.ts b/packages/types/src/api/web/index.ts index 0129fb38d9..1dbe22aa46 100644 --- a/packages/types/src/api/web/index.ts +++ b/packages/types/src/api/web/index.ts @@ -1,3 +1,5 @@ export * from "./analytics" export * from "./user" export * from "./errors" +export * from "./schedule" +export * from "./app" diff --git a/packages/types/src/api/web/schedule.ts b/packages/types/src/api/web/schedule.ts new file mode 100644 index 0000000000..bf762b1603 --- /dev/null +++ b/packages/types/src/api/web/schedule.ts @@ -0,0 +1,15 @@ +import { + ScheduleMetadata, + ScheduleRepeatPeriod, + ScheduleType, +} from "../../documents" + +export interface CreateScheduleRequest { + type: ScheduleType + name: string + startDate: string + repeat: ScheduleRepeatPeriod + metadata: ScheduleMetadata +} + +export interface UpdateScheduleRequest extends CreateScheduleRequest {} diff --git a/packages/types/src/api/web/user.ts b/packages/types/src/api/web/user.ts index c66d3203e8..98ffcdf360 100644 --- a/packages/types/src/api/web/user.ts +++ b/packages/types/src/api/web/user.ts @@ -44,3 +44,10 @@ export interface InviteUsersResponse { successful: { email: string }[] unsuccessful: { email: string; reason: string }[] } + +export interface SearchUsersRequest { + page?: string + email?: string + appId?: string + userIds?: string[] +} diff --git a/packages/types/src/documents/app/backup.ts b/packages/types/src/documents/app/backup.ts new file mode 100644 index 0000000000..a935ed5ba1 --- /dev/null +++ b/packages/types/src/documents/app/backup.ts @@ -0,0 +1,21 @@ +import { Document } from "../document" + +export enum AppBackupTrigger { + PUBLISH = "publish", + MANUAL = "manual", + SCHEDULED = "scheduled", +} + +export interface AppBackupContents { + datasources: string[] + screens: string[] + automations: string[] +} + +export interface AppBackup extends Document { + trigger: AppBackupTrigger + name: string + date: string + userId: string + contents: AppBackupContents +} diff --git a/packages/types/src/documents/app/index.ts b/packages/types/src/documents/app/index.ts index e8b29257fc..dad594b804 100644 --- a/packages/types/src/documents/app/index.ts +++ b/packages/types/src/documents/app/index.ts @@ -10,3 +10,4 @@ export * from "./view" export * from "../document" export * from "./row" export * from "./user" +export * from "./backup" diff --git a/packages/types/src/documents/global/index.ts b/packages/types/src/documents/global/index.ts index edce416a3f..9f779d4937 100644 --- a/packages/types/src/documents/global/index.ts +++ b/packages/types/src/documents/global/index.ts @@ -3,4 +3,4 @@ export * from "./user" export * from "./userGroup" export * from "./plugin" export * from "./quotas" -export * from "./schedules" +export * from "./schedule" diff --git a/packages/types/src/documents/global/schedule.ts b/packages/types/src/documents/global/schedule.ts new file mode 100644 index 0000000000..63bb9284fe --- /dev/null +++ b/packages/types/src/documents/global/schedule.ts @@ -0,0 +1,32 @@ +import { Document } from "../document" + +export enum ScheduleType { + APP_BACKUP = "app_backup", +} + +export enum ScheduleRepeatPeriod { + DAILY = "daily", + WEEKLY = "weekly", + MONTHLY = "monthly", +} + +export interface Schedule extends Document { + type: ScheduleType + name: string + startDate: string + repeat: ScheduleRepeatPeriod + metadata: ScheduleMetadata +} + +export type ScheduleMetadata = AppBackupScheduleMetadata + +export const isAppBackupMetadata = ( + type: ScheduleType, + metadata: ScheduleMetadata +): metadata is AppBackupScheduleMetadata => { + return type === ScheduleType.APP_BACKUP +} + +export interface AppBackupScheduleMetadata { + apps: string[] +} diff --git a/packages/types/src/documents/global/schedules.ts b/packages/types/src/documents/global/schedules.ts deleted file mode 100644 index 357a841a9a..0000000000 --- a/packages/types/src/documents/global/schedules.ts +++ /dev/null @@ -1,9 +0,0 @@ -export enum ScheduleType { - APP_BACKUP = "app_backup", -} - -export enum ScheduleRepeatPeriod { - DAILY = "daily", - WEEKLY = "weekly", - MONTHLY = "monthly", -} diff --git a/packages/worker/src/api/controllers/global/users.ts b/packages/worker/src/api/controllers/global/users.ts index 8894330f67..ea1df5b45a 100644 --- a/packages/worker/src/api/controllers/global/users.ts +++ b/packages/worker/src/api/controllers/global/users.ts @@ -7,6 +7,7 @@ import { CloudAccount, InviteUserRequest, InviteUsersRequest, + SearchUsersRequest, User, } from "@budibase/types" import { @@ -144,7 +145,8 @@ export const destroy = async (ctx: any) => { } export const search = async (ctx: any) => { - const paginated = await sdk.users.paginatedUsers(ctx.request.body) + const body = ctx.request.body as SearchUsersRequest + const paginated = await sdk.users.paginatedUsers(body) // user hashed password shouldn't ever be returned for (let user of paginated.data) { if (user) { diff --git a/packages/worker/src/sdk/users/users.ts b/packages/worker/src/sdk/users/users.ts index 775514ea5e..3b98c8ef52 100644 --- a/packages/worker/src/sdk/users/users.ts +++ b/packages/worker/src/sdk/users/users.ts @@ -27,6 +27,7 @@ import { MigrationType, PlatformUserByEmail, RowResponse, + SearchUsersRequest, User, } from "@budibase/types" import { sendEmail } from "../../utilities/email" @@ -56,7 +57,8 @@ export const paginatedUsers = async ({ page, email, appId, -}: { page?: string; email?: string; appId?: string } = {}) => { + userIds, +}: SearchUsersRequest = {}) => { const db = tenancy.getGlobalDB() // get one extra document, to have the next page const opts: any = { @@ -77,6 +79,9 @@ export const paginatedUsers = async ({ } else if (email) { userList = await usersCore.searchGlobalUsersByEmail(email, opts) property = "email" + } + if (userIds) { + // TODO: search users by userIds } else { // no search, query allDocs const response = await db.allDocs(dbUtils.getGlobalUserParams(null, opts)) From e78ed801b249fbf85ca7b1e886082c9a6e7640f5 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 3 Oct 2022 19:36:33 +0100 Subject: [PATCH 006/151] Linting. --- .../TestConfiguration/InternalAPIClient.ts | 6 ++---- .../internal-api/TestConfiguration/applications.ts | 8 ++------ .../config/internal-api/TestConfiguration/auth.ts | 6 +++--- .../config/internal-api/fixtures/applications.ts | 9 ++++----- .../tests/internal-api/applications/create.spec.ts | 13 +++++++------ 5 files changed, 18 insertions(+), 24 deletions(-) diff --git a/qa-core/src/config/internal-api/TestConfiguration/InternalAPIClient.ts b/qa-core/src/config/internal-api/TestConfiguration/InternalAPIClient.ts index bfcbb9f4e2..dafc2b1ff2 100644 --- a/qa-core/src/config/internal-api/TestConfiguration/InternalAPIClient.ts +++ b/qa-core/src/config/internal-api/TestConfiguration/InternalAPIClient.ts @@ -16,9 +16,7 @@ class InternalAPIClient { constructor(appId?: string) { if (!env.BUDIBASE_SERVER_URL) { - throw new Error( - "Must set BUDIBASE_SERVER_URL env var" - ) + throw new Error("Must set BUDIBASE_SERVER_URL env var") } this.host = `${env.BUDIBASE_SERVER_URL}/api` this.appId = appId @@ -55,4 +53,4 @@ class InternalAPIClient { put = this.apiCall("PUT") } -export default InternalAPIClient \ No newline at end of file +export default InternalAPIClient diff --git a/qa-core/src/config/internal-api/TestConfiguration/applications.ts b/qa-core/src/config/internal-api/TestConfiguration/applications.ts index 10e4a6657b..0c51487122 100644 --- a/qa-core/src/config/internal-api/TestConfiguration/applications.ts +++ b/qa-core/src/config/internal-api/TestConfiguration/applications.ts @@ -1,6 +1,4 @@ -import { - Application, -} from "@budibase/server/api/controllers/public/mapping/types" +import { Application } from "@budibase/server/api/controllers/public/mapping/types" import { App } from "@budibase/types" import { Response } from "node-fetch" import InternalAPIClient from "./InternalAPIClient" @@ -37,9 +35,7 @@ export default class AppApi { return [response, json] } - async create( - body: any - ): Promise<[Response, Partial]> { + async create(body: any): Promise<[Response, Partial]> { const response = await this.api.post(`/applications`, { body }) const json = await response.json() return [response, json] diff --git a/qa-core/src/config/internal-api/TestConfiguration/auth.ts b/qa-core/src/config/internal-api/TestConfiguration/auth.ts index 6ac53f24b6..d83c859ab3 100644 --- a/qa-core/src/config/internal-api/TestConfiguration/auth.ts +++ b/qa-core/src/config/internal-api/TestConfiguration/auth.ts @@ -9,11 +9,11 @@ export default class AuthApi { } async login(): Promise<[Response, any]> { - const response = await this.api.post(`/global/auth/default/login`, { + const response = await this.api.post(`/global/auth/default/login`, { body: { username: process.env.BB_ADMIN_USER_EMAIL, - password: process.env.BB_ADMIN_USER_PASSWORD - } + password: process.env.BB_ADMIN_USER_PASSWORD, + }, }) const cookie = response.headers.get("set-cookie") this.api.cookie = cookie as any diff --git a/qa-core/src/config/internal-api/fixtures/applications.ts b/qa-core/src/config/internal-api/fixtures/applications.ts index dfad7e0b46..9076a05e1b 100644 --- a/qa-core/src/config/internal-api/fixtures/applications.ts +++ b/qa-core/src/config/internal-api/fixtures/applications.ts @@ -1,10 +1,9 @@ import generator from "../../generator" -import { - Application, -} from "@budibase/server/api/controllers/public/mapping/types" +import { Application } from "@budibase/server/api/controllers/public/mapping/types" - -const generate = (overrides: Partial = {}): Partial => ({ +const generate = ( + overrides: Partial = {} +): Partial => ({ name: generator.word(), url: `/${generator.word()}`, ...overrides, diff --git a/qa-core/src/tests/internal-api/applications/create.spec.ts b/qa-core/src/tests/internal-api/applications/create.spec.ts index 81d43d9c91..2c934e0bd7 100644 --- a/qa-core/src/tests/internal-api/applications/create.spec.ts +++ b/qa-core/src/tests/internal-api/applications/create.spec.ts @@ -24,14 +24,14 @@ describe("Internal API - /applications endpoints", () => { useTemplate: "true", templateName: "Near Miss Register", templateKey: "app/near-miss-register", - templateFile: undefined + templateFile: undefined, }) } it("GET - fetch applications", async () => { await config.applications.create({ ...generateApp(), - useTemplate: false + useTemplate: false, }) const [response, apps] = await config.applications.fetch() expect(response).toHaveStatusCode(200) @@ -57,7 +57,7 @@ describe("Internal API - /applications endpoints", () => { expect(publish).toEqual({ _id: expect.any(String), appUrl: app.url, - status: "SUCCESS" + status: "SUCCESS", }) }) @@ -70,7 +70,8 @@ describe("Internal API - /applications endpoints", () => { config.applications.api.appId = app.appId // check preview renders - const [previewResponse, previewRenders] = await config.applications.canRender() + const [previewResponse, previewRenders] = + await config.applications.canRender() expect(previewResponse).toHaveStatusCode(200) expect(previewRenders).toBe(true) @@ -79,8 +80,8 @@ describe("Internal API - /applications endpoints", () => { // check published app renders config.applications.api.appId = db.getProdAppID(app.appId) - const [publishedAppResponse, publishedAppRenders] = await config.applications.canRender() + const [publishedAppResponse, publishedAppRenders] = + await config.applications.canRender() expect(publishedAppRenders).toBe(true) }) - }) From ef52bde670d5b73e5a742216de6573014d9d0af5 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 6 Oct 2022 19:10:45 +0100 Subject: [PATCH 007/151] Building out initial SDK work - converting some existing exporting work to typescript. --- .../api/controllers/{backup.js => backup.ts} | 12 +- packages/server/src/api/controllers/cloud.js | 11 +- packages/server/src/api/routes/backup.js | 10 -- packages/server/src/api/routes/backup.ts | 10 ++ packages/server/src/sdk/app/export.ts | 113 ++++++++++++++++++ packages/server/src/sdk/app/index.ts | 1 + packages/server/src/sdk/index.ts | 5 + .../server/src/utilities/fileSystem/index.js | 106 ---------------- 8 files changed, 139 insertions(+), 129 deletions(-) rename packages/server/src/api/controllers/{backup.js => backup.ts} (56%) delete mode 100644 packages/server/src/api/routes/backup.js create mode 100644 packages/server/src/api/routes/backup.ts create mode 100644 packages/server/src/sdk/app/export.ts create mode 100644 packages/server/src/sdk/app/index.ts create mode 100644 packages/server/src/sdk/index.ts diff --git a/packages/server/src/api/controllers/backup.js b/packages/server/src/api/controllers/backup.ts similarity index 56% rename from packages/server/src/api/controllers/backup.js rename to packages/server/src/api/controllers/backup.ts index 5e08c823ed..cd2ea415a4 100644 --- a/packages/server/src/api/controllers/backup.js +++ b/packages/server/src/api/controllers/backup.ts @@ -1,15 +1,15 @@ -const { streamBackup } = require("../../utilities/fileSystem") -const { events, context } = require("@budibase/backend-core") -const { DocumentType } = require("../../db/utils") -const { isQsTrue } = require("../../utilities") +import sdk from "../../sdk" +import { events, context } from "@budibase/backend-core" +import { DocumentType } from "../../db/utils" +import { isQsTrue } from "../../utilities" -exports.exportAppDump = async function (ctx) { +export async function exportAppDump(ctx: any) { let { appId, excludeRows } = ctx.query const appName = decodeURI(ctx.query.appname) excludeRows = isQsTrue(excludeRows) const backupIdentifier = `${appName}-export-${new Date().getTime()}.txt` ctx.attachment(backupIdentifier) - ctx.body = await streamBackup(appId, excludeRows) + ctx.body = await sdk.apps.exports.streamBackup(appId, excludeRows) await context.doInAppContext(appId, async () => { const appDb = context.getAppDB() diff --git a/packages/server/src/api/controllers/cloud.js b/packages/server/src/api/controllers/cloud.js index 1e6abb1d3b..55aa6bb548 100644 --- a/packages/server/src/api/controllers/cloud.js +++ b/packages/server/src/api/controllers/cloud.js @@ -1,14 +1,11 @@ const env = require("../../environment") const { getAllApps, getGlobalDBName } = require("@budibase/backend-core/db") -const { - exportDB, - sendTempFile, - readFileSync, -} = require("../../utilities/fileSystem") +const { sendTempFile, readFileSync } = require("../../utilities/fileSystem") const { stringToReadStream } = require("../../utilities") const { getGlobalDB } = require("@budibase/backend-core/tenancy") const { create } = require("./application") const { getDocParams, DocumentType, isDevAppID } = require("../../db/utils") +const sdk = require("../../sdk") async function createApp(appName, appImport) { const ctx = { @@ -27,7 +24,7 @@ exports.exportApps = async ctx => { ctx.throw(400, "Exporting only allowed in multi-tenant cloud environments.") } const apps = await getAllApps({ all: true }) - const globalDBString = await exportDB(getGlobalDBName(), { + const globalDBString = await sdk.apps.exports.exportDB(getGlobalDBName(), { filter: doc => !doc._id.startsWith(DocumentType.USER), }) let allDBs = { @@ -38,7 +35,7 @@ exports.exportApps = async ctx => { // only export the dev apps as they will be the latest, the user can republish the apps // in their self hosted environment if (isDevAppID(appId)) { - allDBs[app.name] = await exportDB(appId) + allDBs[app.name] = await sdk.apps.exports.exportDB(appId) } } const filename = `cloud-export-${new Date().getTime()}.txt` diff --git a/packages/server/src/api/routes/backup.js b/packages/server/src/api/routes/backup.js deleted file mode 100644 index 9f3b27e95a..0000000000 --- a/packages/server/src/api/routes/backup.js +++ /dev/null @@ -1,10 +0,0 @@ -const Router = require("@koa/router") -const controller = require("../controllers/backup") -const authorized = require("../../middleware/authorized") -const { BUILDER } = require("@budibase/backend-core/permissions") - -const router = new Router() - -router.get("/api/backups/export", authorized(BUILDER), controller.exportAppDump) - -module.exports = router diff --git a/packages/server/src/api/routes/backup.ts b/packages/server/src/api/routes/backup.ts new file mode 100644 index 0000000000..2473fa9f67 --- /dev/null +++ b/packages/server/src/api/routes/backup.ts @@ -0,0 +1,10 @@ +import Router from "@koa/router" +import * as controller from "../controllers/backup" +import authorized from "../../middleware/authorized" +import { BUILDER } from "@budibase/backend-core/permissions" + +const router = new Router() + +router.get("/api/backups/export", authorized(BUILDER), controller.exportAppDump) + +export default router diff --git a/packages/server/src/sdk/app/export.ts b/packages/server/src/sdk/app/export.ts new file mode 100644 index 0000000000..70c4446273 --- /dev/null +++ b/packages/server/src/sdk/app/export.ts @@ -0,0 +1,113 @@ +import { closeDB, dangerousGetDB, doWithDB } from "@budibase/backend-core/db" +import { budibaseTempDir } from "../../utilities/budibaseDir" +import { streamUpload } from "../../utilities/fileSystem/utilities" +import { ObjectStoreBuckets } from "../../constants" +import { + LINK_USER_METADATA_PREFIX, + TABLE_ROW_PREFIX, + USER_METDATA_PREFIX, +} from "../../db/utils" +import fs from "fs" +import env from "../../environment" +import { join } from "path" +const MemoryStream = require("memorystream") + +/** + * Exports a DB to either file or a variable (memory). + * @param {string} dbName the DB which is to be exported. + * @param {object} opts various options for the export, e.g. whether to stream, + * a filter function or the name of the export. + * @return {*} either a readable stream or a string + */ +export async function exportDB( + dbName: string, + opts: { stream?: boolean; filter?: any; exportName?: string } = {} +) { + // streaming a DB dump is a bit more complicated, can't close DB + if (opts?.stream) { + const db = dangerousGetDB(dbName) + const memStream = new MemoryStream() + memStream.on("end", async () => { + await closeDB(db) + }) + db.dump(memStream, { filter: opts?.filter }) + return memStream + } + + return doWithDB(dbName, async (db: any) => { + // Write the dump to file if required + if (opts?.exportName) { + const path = join(budibaseTempDir(), opts?.exportName) + const writeStream = fs.createWriteStream(path) + await db.dump(writeStream, { filter: opts?.filter }) + + // Upload the dump to the object store if self-hosted + if (env.SELF_HOSTED) { + await streamUpload( + ObjectStoreBuckets.BACKUPS, + join(dbName, opts?.exportName), + fs.createReadStream(path) + ) + } + + return fs.createReadStream(path) + } + + // Stringify the dump in memory if required + const memStream = new MemoryStream() + let appString = "" + memStream.on("data", (chunk: any) => { + appString += chunk.toString() + }) + await db.dump(memStream, { filter: opts?.filter }) + return appString + }) +} + +function defineFilter(excludeRows?: boolean) { + const ids = [USER_METDATA_PREFIX, LINK_USER_METADATA_PREFIX] + if (excludeRows) { + ids.push(TABLE_ROW_PREFIX) + } + return (doc: any) => + !ids.map(key => doc._id.includes(key)).reduce((prev, curr) => prev || curr) +} + +/** + * Local utility to back up the database state for an app, excluding global user + * data or user relationships. + * @param {string} appId The app to back up + * @param {object} config Config to send to export DB + * @param {boolean} excludeRows Flag to state whether the export should include data. + * @returns {*} either a string or a stream of the backup + */ +async function backupAppData( + appId: string, + config: any, + excludeRows?: boolean +) { + return await exportDB(appId, { + ...config, + filter: defineFilter(excludeRows), + }) +} + +/** + * Streams a backup of the database state for an app + * @param {string} appId The ID of the app which is to be backed up. + * @param {boolean} excludeRows Flag to state whether the export should include data. + * @returns {*} a readable stream of the backup which is written in real time + */ +export async function streamBackup(appId: string, excludeRows: boolean) { + return await backupAppData(appId, { stream: true }, excludeRows) +} + +/** + * Takes a copy of the database state for an app to the object store. + * @param {string} appId The ID of the app which is to be backed up. + * @param {string} backupName The name of the backup located in the object store. + * @return {*} a readable stream to the completed backup file + */ +export async function performBackup(appId: string, backupName: string) { + return await backupAppData(appId, { exportName: backupName }) +} diff --git a/packages/server/src/sdk/app/index.ts b/packages/server/src/sdk/app/index.ts new file mode 100644 index 0000000000..3927539bc8 --- /dev/null +++ b/packages/server/src/sdk/app/index.ts @@ -0,0 +1 @@ +export * as exports from "./export" diff --git a/packages/server/src/sdk/index.ts b/packages/server/src/sdk/index.ts new file mode 100644 index 0000000000..9199c1cc72 --- /dev/null +++ b/packages/server/src/sdk/index.ts @@ -0,0 +1,5 @@ +import * as apps from "./app" + +export default { + apps, +} diff --git a/packages/server/src/utilities/fileSystem/index.js b/packages/server/src/utilities/fileSystem/index.js index 4e9b13cca0..656ba4816c 100644 --- a/packages/server/src/utilities/fileSystem/index.js +++ b/packages/server/src/utilities/fileSystem/index.js @@ -2,17 +2,11 @@ const { budibaseTempDir } = require("../budibaseDir") const fs = require("fs") const { join } = require("path") const uuid = require("uuid/v4") -const { - doWithDB, - dangerousGetDB, - closeDB, -} = require("@budibase/backend-core/db") const { ObjectStoreBuckets } = require("../../constants") const { upload, retrieve, retrieveToTmp, - streamUpload, deleteFolder, downloadTarball, downloadTarballDirect, @@ -21,12 +15,6 @@ const { const { updateClientLibrary } = require("./clientLibrary") const { checkSlashesInUrl } = require("../") const env = require("../../environment") -const { - USER_METDATA_PREFIX, - LINK_USER_METADATA_PREFIX, - TABLE_ROW_PREFIX, -} = require("../../db/utils") -const MemoryStream = require("memorystream") const { getAppId } = require("@budibase/backend-core/context") const tar = require("tar") const fetch = require("node-fetch") @@ -124,100 +112,6 @@ exports.apiFileReturn = contents => { return fs.createReadStream(path) } -exports.defineFilter = excludeRows => { - const ids = [USER_METDATA_PREFIX, LINK_USER_METADATA_PREFIX] - if (excludeRows) { - ids.push(TABLE_ROW_PREFIX) - } - return doc => - !ids.map(key => doc._id.includes(key)).reduce((prev, curr) => prev || curr) -} - -/** - * Local utility to back up the database state for an app, excluding global user - * data or user relationships. - * @param {string} appId The app to backup - * @param {object} config Config to send to export DB - * @param {boolean} excludeRows Flag to state whether the export should include data. - * @returns {*} either a string or a stream of the backup - */ -const backupAppData = async (appId, config, excludeRows) => { - return await exports.exportDB(appId, { - ...config, - filter: exports.defineFilter(excludeRows), - }) -} - -/** - * Takes a copy of the database state for an app to the object store. - * @param {string} appId The ID of the app which is to be backed up. - * @param {string} backupName The name of the backup located in the object store. - * @return {*} a readable stream to the completed backup file - */ -exports.performBackup = async (appId, backupName) => { - return await backupAppData(appId, { exportName: backupName }) -} - -/** - * Streams a backup of the database state for an app - * @param {string} appId The ID of the app which is to be backed up. - * @param {boolean} excludeRows Flag to state whether the export should include data. - * @returns {*} a readable stream of the backup which is written in real time - */ -exports.streamBackup = async (appId, excludeRows) => { - return await backupAppData(appId, { stream: true }, excludeRows) -} - -/** - * Exports a DB to either file or a variable (memory). - * @param {string} dbName the DB which is to be exported. - * @param {string} exportName optional - provide a filename to write the backup to a file - * @param {boolean} stream optional - whether to perform a full backup - * @param {function} filter optional - a filter function to clear out any un-wanted docs. - * @return {*} either a readable stream or a string - */ -exports.exportDB = async (dbName, { stream, filter, exportName } = {}) => { - // streaming a DB dump is a bit more complicated, can't close DB - if (stream) { - const db = dangerousGetDB(dbName) - const memStream = new MemoryStream() - memStream.on("end", async () => { - await closeDB(db) - }) - db.dump(memStream, { filter }) - return memStream - } - - return doWithDB(dbName, async db => { - // Write the dump to file if required - if (exportName) { - const path = join(budibaseTempDir(), exportName) - const writeStream = fs.createWriteStream(path) - await db.dump(writeStream, { filter }) - - // Upload the dump to the object store if self hosted - if (env.SELF_HOSTED) { - await streamUpload( - ObjectStoreBuckets.BACKUPS, - join(dbName, exportName), - fs.createReadStream(path) - ) - } - - return fs.createReadStream(path) - } - - // Stringify the dump in memory if required - const memStream = new MemoryStream() - let appString = "" - memStream.on("data", chunk => { - appString += chunk.toString() - }) - await db.dump(memStream, { filter }) - return appString - }) -} - /** * Writes the provided contents to a temporary file, which can be used briefly. * @param {string} fileContents contents which will be written to a temp file. From 9efb8f98bc4a105e00680eae97ed6d0916ada783 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 7 Oct 2022 21:08:20 +0100 Subject: [PATCH 008/151] Updating koa versions to align with pro - types were inaccurate and couldn't be imported correctly. --- packages/server/package.json | 8 +- packages/server/src/api/routes/index.ts | 8 +- packages/server/yarn.lock | 118 +++++------------------- packages/worker/package.json | 1 - 4 files changed, 36 insertions(+), 99 deletions(-) diff --git a/packages/server/package.json b/packages/server/package.json index 2f2409850e..dba6bfbe79 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -86,7 +86,7 @@ "@bull-board/koa": "3.9.4", "@elastic/elasticsearch": "7.10.0", "@google-cloud/firestore": "5.0.2", - "@koa/router": "8.0.0", + "@koa/router": "8.0.8", "@sendgrid/mail": "7.1.1", "@sentry/node": "6.17.7", "airtable": "0.10.1", @@ -112,7 +112,7 @@ "js-yaml": "4.1.0", "jsonschema": "1.4.0", "knex": "0.95.15", - "koa": "2.7.0", + "koa": "2.13.4", "koa-body": "4.2.0", "koa-compress": "4.0.1", "koa-connect": "2.1.0", @@ -163,8 +163,8 @@ "@types/global-agent": "2.1.1", "@types/google-spreadsheet": "3.1.5", "@types/jest": "27.5.1", - "@types/koa": "2.13.4", - "@types/koa__router": "8.0.0", + "@types/koa": "2.13.5", + "@types/koa__router": "8.0.11", "@types/lodash": "4.14.180", "@types/mongodb": "3.6.3", "@types/node": "14.18.20", diff --git a/packages/server/src/api/routes/index.ts b/packages/server/src/api/routes/index.ts index 64f5f9cb89..1cf34d4a68 100644 --- a/packages/server/src/api/routes/index.ts +++ b/packages/server/src/api/routes/index.ts @@ -25,11 +25,15 @@ import devRoutes from "./dev" import cloudRoutes from "./cloud" import migrationRoutes from "./migrations" import pluginRoutes from "./plugin" +import Router from "@koa/router" +import { api } from "@budibase/pro" export { default as staticRoutes } from "./static" export { default as publicRoutes } from "./public" -export const mainRoutes = [ +const appBackupRoutes = api.appBackups +const scheduleRoutes = api.schedules +export const mainRoutes: Router[] = [ authRoutes, deployRoutes, layoutRoutes, @@ -59,4 +63,6 @@ export const mainRoutes = [ rowRoutes, migrationRoutes, pluginRoutes, + appBackupRoutes, + scheduleRoutes, ] diff --git a/packages/server/yarn.lock b/packages/server/yarn.lock index 18ab07c17f..bf95495815 100644 --- a/packages/server/yarn.lock +++ b/packages/server/yarn.lock @@ -2003,18 +2003,6 @@ resolved "https://registry.yarnpkg.com/@jsdevtools/ono/-/ono-7.1.3.tgz#9df03bbd7c696a5c58885c34aa06da41c8543796" integrity sha512-4JQNk+3mVzK3xh2rqd6RB4J46qUR19azEHBneZyTZM+c456qOrbbM/5xcR8huNCCcbVt7+UmizG6GuUvPvKUYg== -"@koa/router@8.0.0": - version "8.0.0" - resolved "https://registry.yarnpkg.com/@koa/router/-/router-8.0.0.tgz#fd4ffa6f03d8293a04c023cb4a22b612401fbe70" - integrity sha512-P70CGOGs6JPu/mnrd9lt6ESzlBXLHT/uTK8+5U4M7Oapt8la/tiZv2c7X9jq0ksFsM59RH3AwJYzKOuavDcjIw== - dependencies: - debug "^3.1.0" - http-errors "^1.3.1" - koa-compose "^3.0.0" - methods "^1.0.1" - path-to-regexp "^1.1.1" - urijs "^1.19.0" - "@koa/router@8.0.8": version "8.0.8" resolved "https://registry.yarnpkg.com/@koa/router/-/router-8.0.8.tgz#95f32d11373d03d89dcb63fabe9ac6f471095236" @@ -2851,7 +2839,7 @@ dependencies: "@types/koa" "*" -"@types/koa@*", "@types/koa@2.13.4": +"@types/koa@*": version "2.13.4" resolved "https://registry.yarnpkg.com/@types/koa/-/koa-2.13.4.tgz#10620b3f24a8027ef5cbae88b393d1b31205726b" integrity sha512-dfHYMfU+z/vKtQB7NUrthdAEiSvnLebvBjwHtfFmpZmB7em2N3WVQdHgnFq+xvyVgxW5jKDmjWfLD3lw4g4uTw== @@ -2865,10 +2853,24 @@ "@types/koa-compose" "*" "@types/node" "*" -"@types/koa__router@8.0.0": - version "8.0.0" - resolved "https://registry.yarnpkg.com/@types/koa__router/-/koa__router-8.0.0.tgz#057a7254a25df5bc93b42a1acacb2d99cd02d297" - integrity sha512-XaGqudqJyFOmByN+f9BrEIZEgLfBnvVtZlm/beuTxWpbWpMHiA+ZmA+mB5dsrbGemko61wUA+WG0jhUzMSq+JA== +"@types/koa@2.13.5": + version "2.13.5" + resolved "https://registry.yarnpkg.com/@types/koa/-/koa-2.13.5.tgz#64b3ca4d54e08c0062e89ec666c9f45443b21a61" + integrity sha512-HSUOdzKz3by4fnqagwthW/1w/yJspTgppyyalPVbgZf8jQWvdIXcVW5h2DGtw4zYntOaeRGx49r1hxoPWrD4aA== + dependencies: + "@types/accepts" "*" + "@types/content-disposition" "*" + "@types/cookies" "*" + "@types/http-assert" "*" + "@types/http-errors" "*" + "@types/keygrip" "*" + "@types/koa-compose" "*" + "@types/node" "*" + +"@types/koa__router@8.0.11": + version "8.0.11" + resolved "https://registry.yarnpkg.com/@types/koa__router/-/koa__router-8.0.11.tgz#d7b37e6db934fc072ea1baa2ab92bc8ac4564f3e" + integrity sha512-WXgKWpBsbS14kzmzD9LeFapOIa678h7zvUHxDwXwSx4ETKXhXLVUAToX6jZ/U7EihM7qwyD9W/BZvB0MRu7MTQ== dependencies: "@types/koa" "*" @@ -3523,7 +3525,7 @@ any-base@^1.1.0: resolved "https://registry.yarnpkg.com/any-base/-/any-base-1.1.0.tgz#ae101a62bc08a597b4c9ab5b7089d456630549fe" integrity sha512-uMgjozySS8adZZYePpaWs8cxB9/kdzmpX6SgJZ+wbz1K5eYk5QMYDVJaZKhxyIHUdnnJkfR7SVgStgH7LkGUyg== -any-promise@^1.0.0, any-promise@^1.1.0: +any-promise@^1.0.0: version "1.3.0" resolved "https://registry.yarnpkg.com/any-promise/-/any-promise-1.3.0.tgz#abc6afeedcea52e809cdc0376aed3ce39635d17f" integrity sha512-7UvmKalWRt1wgjL1RrGxoSJW/0QZFIegpeGvZG9kjp8vrRu55XTHbwnqq2GpXm9uLbcuhxm3IqX9OB4MZR1b2A== @@ -4907,14 +4909,6 @@ cookiejar@^2.1.0: resolved "https://registry.yarnpkg.com/cookiejar/-/cookiejar-2.1.3.tgz#fc7a6216e408e74414b90230050842dacda75acc" integrity sha512-JxbCBUdrfr6AQjOXrxoTvAMJO4HBTUIlBzslcJPAz+/KT8yk53fXun51u+RenNYvad/+Vc2DIz5o9UxlCDymFQ== -cookies@~0.7.1: - version "0.7.3" - resolved "https://registry.yarnpkg.com/cookies/-/cookies-0.7.3.tgz#7912ce21fbf2e8c2da70cf1c3f351aecf59dadfa" - integrity sha512-+gixgxYSgQLTaTIilDHAdlNPZDENDQernEMiIcZpYYP14zgHsCt4Ce1FEjFtcp6GefhozebB6orvhAAWx/IS0A== - dependencies: - depd "~1.1.2" - keygrip "~1.0.3" - cookies@~0.8.0: version "0.8.0" resolved "https://registry.yarnpkg.com/cookies/-/cookies-0.8.0.tgz#1293ce4b391740a8406e3c9870e828c4b54f3f90" @@ -5138,13 +5132,6 @@ debug@^3.1.0, debug@^3.2.6, debug@^3.2.7: dependencies: ms "^2.1.1" -debug@~3.1.0: - version "3.1.0" - resolved "https://registry.yarnpkg.com/debug/-/debug-3.1.0.tgz#5bb5a0672628b64149566ba16819e61518c67261" - integrity sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g== - dependencies: - ms "2.0.0" - debuglog@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/debuglog/-/debuglog-1.0.1.tgz#aa24ffb9ac3df9a2351837cfb2d279360cd78492" @@ -5710,11 +5697,6 @@ error-ex@^1.3.1: dependencies: is-arrayish "^0.2.1" -error-inject@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/error-inject/-/error-inject-1.0.0.tgz#e2b3d91b54aed672f309d950d154850fa11d4f37" - integrity sha512-JM8N6PytDbmIYm1IhPWlo8vr3NtfjhDY/1MhD/a5b/aad/USE8a0+NsqE9d5n+GVGmuNkPQWm4bFQWv18d8tMg== - error-stack-parser@^2.0.6: version "2.1.4" resolved "https://registry.yarnpkg.com/error-stack-parser/-/error-stack-parser-2.1.4.tgz#229cb01cdbfa84440bfa91876285b94680188286" @@ -7396,7 +7378,7 @@ http-errors@2.0.0: statuses "2.0.1" toidentifier "1.0.1" -http-errors@^1.3.1, http-errors@^1.6.3, http-errors@^1.7.3, http-errors@~1.8.0: +http-errors@^1.6.3, http-errors@^1.7.3, http-errors@~1.8.0: version "1.8.1" resolved "https://registry.yarnpkg.com/http-errors/-/http-errors-1.8.1.tgz#7c3f28577cbc8a207388455dbd62295ed07bd68c" integrity sha512-Kpk9Sm7NmI+RHhnj6OIWDI1d6fIoFAtFt9RLaTMRlg/8w49juAStsrBgp0Dp4OdxdVbRIeKhtCUvoi/RuAhO4g== @@ -9265,11 +9247,6 @@ jws@^4.0.0: jwa "^2.0.0" safe-buffer "^5.0.1" -keygrip@~1.0.3: - version "1.0.3" - resolved "https://registry.yarnpkg.com/keygrip/-/keygrip-1.0.3.tgz#399d709f0aed2bab0a059e0cdd3a5023a053e1dc" - integrity sha512-/PpesirAIfaklxUzp4Yb7xBper9MwP6hNRA6BGGUFCgbJ+BM5CKBtsoxinNXkLHAr+GXS1/lSlF2rP7cv5Fl+g== - keygrip@~1.1.0: version "1.1.0" resolved "https://registry.yarnpkg.com/keygrip/-/keygrip-1.1.0.tgz#871b1681d5e159c62a445b0c74b615e0917e7226" @@ -9355,13 +9332,6 @@ koa-body@4.2.0: co-body "^5.1.1" formidable "^1.1.1" -koa-compose@^3.0.0: - version "3.2.1" - resolved "https://registry.yarnpkg.com/koa-compose/-/koa-compose-3.2.1.tgz#a85ccb40b7d986d8e5a345b3a1ace8eabcf54de7" - integrity sha512-8gen2cvKHIZ35eDEik5WOo8zbVp9t4cP8p4hW4uE55waxolLRexKKrqfCpwhGVppnB40jWeF8bZeTVg99eZgPw== - dependencies: - any-promise "^1.1.0" - koa-compose@^4.1.0: version "4.1.0" resolved "https://registry.yarnpkg.com/koa-compose/-/koa-compose-4.1.0.tgz#507306b9371901db41121c812e923d0d67d3e877" @@ -9383,14 +9353,6 @@ koa-connect@2.1.0: resolved "https://registry.yarnpkg.com/koa-connect/-/koa-connect-2.1.0.tgz#16bce0a917c4cb24233aaac83fbc5b83804b4a1c" integrity sha512-O9pcFafHk0oQsBevlbTBlB9co+2RUQJ4zCzu3qJPmGlGoeEZkne+7gWDkecqDPSbCtED6LmhlQladxs6NjOnMQ== -koa-convert@^1.2.0: - version "1.2.0" - resolved "https://registry.yarnpkg.com/koa-convert/-/koa-convert-1.2.0.tgz#da40875df49de0539098d1700b50820cebcd21d0" - integrity sha512-K9XqjmEDStGX09v3oxR7t5uPRy0jqJdvodHa6wxWTHrTfDq0WUNnYTOOUZN6g8OM8oZQXprQASbiIXG2Ez8ehA== - dependencies: - co "^4.6.0" - koa-compose "^3.0.0" - koa-convert@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/koa-convert/-/koa-convert-2.0.0.tgz#86a0c44d81d40551bae22fee6709904573eea4f5" @@ -9492,37 +9454,7 @@ koa2-ratelimit@1.1.1: resolved "https://registry.yarnpkg.com/koa2-ratelimit/-/koa2-ratelimit-1.1.1.tgz#9c1d8257770e4a0a08063ba2ddcaf690fd457d23" integrity sha512-IpxGMdZqEhMykW0yYKGVB4vDEacPvSBH4hNpDL38ABj3W2KHNLujAljGEDg7eEjXvrRbXRSWXzANhV3c9v7nyg== -koa@2.7.0: - version "2.7.0" - resolved "https://registry.yarnpkg.com/koa/-/koa-2.7.0.tgz#7e00843506942b9d82c6cc33749f657c6e5e7adf" - integrity sha512-7ojD05s2Q+hFudF8tDLZ1CpCdVZw8JQELWSkcfG9bdtoTDzMmkRF6BQBU7JzIzCCOY3xd3tftiy/loHBUYaY2Q== - dependencies: - accepts "^1.3.5" - cache-content-type "^1.0.0" - content-disposition "~0.5.2" - content-type "^1.0.4" - cookies "~0.7.1" - debug "~3.1.0" - delegates "^1.0.0" - depd "^1.1.2" - destroy "^1.0.4" - error-inject "^1.0.0" - escape-html "^1.0.3" - fresh "~0.5.2" - http-assert "^1.3.0" - http-errors "^1.6.3" - is-generator-function "^1.0.7" - koa-compose "^4.1.0" - koa-convert "^1.2.0" - koa-is-json "^1.0.0" - on-finished "^2.3.0" - only "~0.0.2" - parseurl "^1.3.2" - statuses "^1.5.0" - type-is "^1.6.16" - vary "^1.1.2" - -koa@^2.13.1, koa@^2.13.4: +koa@2.13.4, koa@^2.13.1, koa@^2.13.4: version "2.13.4" resolved "https://registry.yarnpkg.com/koa/-/koa-2.13.4.tgz#ee5b0cb39e0b8069c38d115139c774833d32462e" integrity sha512-43zkIKubNbnrULWlHdN5h1g3SEKXOEzoAlRsHOTFpnlDu8JlAOZSMJBLULusuXRequboiwJcj5vtYXKB3k7+2g== @@ -10116,7 +10048,7 @@ merge2@^1.3.0, merge2@^1.4.1: resolved "https://registry.yarnpkg.com/merge2/-/merge2-1.4.1.tgz#4368892f885e907455a6fd7dc55c0c9d404990ae" integrity sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg== -methods@^1.0.1, methods@^1.1.1, methods@^1.1.2: +methods@^1.1.1, methods@^1.1.2: version "1.1.2" resolved "https://registry.yarnpkg.com/methods/-/methods-1.1.2.tgz#5529a4d67654134edcc5266656835b0f851afcee" integrity sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w== @@ -11202,7 +11134,7 @@ path-parser@^6.1.0: search-params "3.0.0" tslib "^1.10.0" -path-to-regexp@1.x, path-to-regexp@^1.1.1: +path-to-regexp@1.x: version "1.8.0" resolved "https://registry.yarnpkg.com/path-to-regexp/-/path-to-regexp-1.8.0.tgz#887b3ba9d84393e87a0a0b9f4cb756198b53548a" integrity sha512-n43JRhlUKUAlibEJhPeir1ncUID16QnEjNpwzNdO3Lm4ywrBpBZ5oLD0I6br9evr1Y9JTqwRtAh7JLoOzAQdVA== @@ -14203,7 +14135,7 @@ uri-js@^4.2.2: dependencies: punycode "^2.1.0" -urijs@^1.19.0, urijs@^1.19.2: +urijs@^1.19.2: version "1.19.11" resolved "https://registry.yarnpkg.com/urijs/-/urijs-1.19.11.tgz#204b0d6b605ae80bea54bea39280cdb7c9f923cc" integrity sha512-HXgFDgDommxn5/bIv0cnQZsPhHDA90NPHD6+c/v21U5+Sx5hoP8+dP9IZXBU1gIfvdRfhG8cel9QNPeionfcCQ== diff --git a/packages/worker/package.json b/packages/worker/package.json index 46927f8725..3ce04ac0f2 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -72,7 +72,6 @@ "devDependencies": { "@types/jest": "26.0.23", "@types/koa": "2.13.4", - "@types/koa-router": "7.4.4", "@types/koa__router": "8.0.11", "@types/node": "14.18.20", "@types/uuid": "8.3.4", From 1f36eec89a7faa297b6a589536b0110c6327639c Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 10 Oct 2022 20:08:59 +0100 Subject: [PATCH 009/151] Some updates towards supporting attachments in app exports. --- .../backend-core/src/objectStore/index.ts | 64 +++++++++++++++++-- packages/builder/src/stores/portal/apps.js | 13 ++++ packages/server/src/api/controllers/backup.ts | 2 +- packages/server/src/api/controllers/cloud.js | 2 +- .../src/api/controllers/static/index.ts | 4 +- packages/server/src/constants/index.js | 6 +- packages/server/src/sdk/app/export.ts | 59 ++++++++--------- .../src/utilities/fileSystem/utilities.js | 2 + packages/worker/src/sdk/users/users.ts | 11 +--- 9 files changed, 110 insertions(+), 53 deletions(-) diff --git a/packages/backend-core/src/objectStore/index.ts b/packages/backend-core/src/objectStore/index.ts index 17e002cc49..903ba28ed1 100644 --- a/packages/backend-core/src/objectStore/index.ts +++ b/packages/backend-core/src/objectStore/index.ts @@ -18,6 +18,10 @@ const STATE = { bucketCreationPromises: {}, } +type ListParams = { + ContinuationToken?: string +} + const CONTENT_TYPE_MAP: any = { html: "text/html", css: "text/css", @@ -93,7 +97,7 @@ export const ObjectStore = (bucket: any) => { * Given an object store and a bucket name this will make sure the bucket exists, * if it does not exist then it will create it. */ -export const makeSureBucketExists = async (client: any, bucketName: any) => { +export const makeSureBucketExists = async (client: any, bucketName: string) => { bucketName = sanitizeBucket(bucketName) try { await client @@ -168,8 +172,8 @@ export const upload = async ({ * through to the object store. */ export const streamUpload = async ( - bucketName: any, - filename: any, + bucketName: string, + filename: string, stream: any, extra = {} ) => { @@ -202,7 +206,7 @@ export const streamUpload = async ( * retrieves the contents of a file from the object store, if it is a known content type it * will be converted, otherwise it will be returned as a buffer stream. */ -export const retrieve = async (bucketName: any, filepath: any) => { +export const retrieve = async (bucketName: string, filepath: string) => { const objectStore = ObjectStore(bucketName) const params = { Bucket: sanitizeBucket(bucketName), @@ -217,10 +221,38 @@ export const retrieve = async (bucketName: any, filepath: any) => { } } +export const listAllObjects = async (bucketName: string, path: string) => { + const objectStore = ObjectStore(bucketName) + const list = (params: ListParams = {}) => { + return objectStore + .listObjectsV2({ + ...params, + Bucket: sanitizeBucket(bucketName), + Prefix: sanitizeKey(path), + }) + .promise() + } + let isTruncated = false, + token, + objects: AWS.S3.Types.Object[] = [] + do { + let params: ListParams = {} + if (token) { + params.ContinuationToken = token + } + const response = await list(params) + if (response.Contents) { + objects = objects.concat(response.Contents) + } + isTruncated = !!response.IsTruncated + } while (isTruncated) + return objects +} + /** * Same as retrieval function but puts to a temporary file. */ -export const retrieveToTmp = async (bucketName: any, filepath: any) => { +export const retrieveToTmp = async (bucketName: string, filepath: string) => { bucketName = sanitizeBucket(bucketName) filepath = sanitizeKey(filepath) const data = await retrieve(bucketName, filepath) @@ -229,10 +261,30 @@ export const retrieveToTmp = async (bucketName: any, filepath: any) => { return outputPath } +export const retrieveDirectory = async (bucketName: string, path: string) => { + let writePath = join(budibaseTempDir(), v4()) + const objects = await listAllObjects(bucketName, path) + let fullObjects = await Promise.all( + objects.map(obj => retrieve(bucketName, obj.Key!)) + ) + let count = 0 + for (let obj of objects) { + const filename = obj.Key! + const data = fullObjects[count++] + const possiblePath = filename.split("/") + if (possiblePath.length > 1) { + const dirs = possiblePath.slice(0, possiblePath.length - 1) + fs.mkdirSync(join(writePath, ...dirs), { recursive: true }) + } + fs.writeFileSync(join(writePath, ...possiblePath), data) + } + return writePath +} + /** * Delete a single file. */ -export const deleteFile = async (bucketName: any, filepath: any) => { +export const deleteFile = async (bucketName: string, filepath: string) => { const objectStore = ObjectStore(bucketName) await makeSureBucketExists(objectStore, bucketName) const params = { diff --git a/packages/builder/src/stores/portal/apps.js b/packages/builder/src/stores/portal/apps.js index a83e35e941..f84e0c973c 100644 --- a/packages/builder/src/stores/portal/apps.js +++ b/packages/builder/src/stores/portal/apps.js @@ -2,6 +2,9 @@ import { writable } from "svelte/store" import { AppStatus } from "../../constants" import { API } from "api" +// properties that should always come from the dev app, not the deployed +const DEV_PROPS = ["updatedBy", "updatedAt"] + const extractAppId = id => { const split = id?.split("_") || [] return split.length ? split[split.length - 1] : null @@ -57,9 +60,19 @@ export function createAppStore() { return } + let devProps = {} + if (appMap[id]) { + const entries = Object.entries(appMap[id]).filter( + ([key]) => DEV_PROPS.indexOf(key) !== -1 + ) + entries.forEach(entry => { + devProps[entry[0]] = entry[1] + }) + } appMap[id] = { ...appMap[id], ...app, + ...devProps, prodId: app.appId, prodRev: app._rev, } diff --git a/packages/server/src/api/controllers/backup.ts b/packages/server/src/api/controllers/backup.ts index cd2ea415a4..878a81e110 100644 --- a/packages/server/src/api/controllers/backup.ts +++ b/packages/server/src/api/controllers/backup.ts @@ -9,7 +9,7 @@ export async function exportAppDump(ctx: any) { excludeRows = isQsTrue(excludeRows) const backupIdentifier = `${appName}-export-${new Date().getTime()}.txt` ctx.attachment(backupIdentifier) - ctx.body = await sdk.apps.exports.streamBackup(appId, excludeRows) + ctx.body = await sdk.apps.exports.streamExportApp(appId, excludeRows) await context.doInAppContext(appId, async () => { const appDb = context.getAppDB() diff --git a/packages/server/src/api/controllers/cloud.js b/packages/server/src/api/controllers/cloud.js index 55aa6bb548..5de5141d74 100644 --- a/packages/server/src/api/controllers/cloud.js +++ b/packages/server/src/api/controllers/cloud.js @@ -35,7 +35,7 @@ exports.exportApps = async ctx => { // only export the dev apps as they will be the latest, the user can republish the apps // in their self hosted environment if (isDevAppID(appId)) { - allDBs[app.name] = await sdk.apps.exports.exportDB(appId) + allDBs[app.name] = await sdk.apps.exports.exportApp(appId) } } const filename = `cloud-export-${new Date().getTime()}.txt` diff --git a/packages/server/src/api/controllers/static/index.ts b/packages/server/src/api/controllers/static/index.ts index 08213c2cf8..f60dc12971 100644 --- a/packages/server/src/api/controllers/static/index.ts +++ b/packages/server/src/api/controllers/static/index.ts @@ -5,7 +5,7 @@ require("svelte/register") const send = require("koa-send") const { resolve, join } = require("../../../utilities/centralPath") const uuid = require("uuid") -const { ObjectStoreBuckets } = require("../../../constants") +const { ObjectStoreBuckets, ATTACHMENT_PATH } = require("../../../constants") const { processString } = require("@budibase/string-templates") const { loadHandlebarsFile, @@ -90,7 +90,7 @@ export const uploadFile = async function (ctx: any) { return prepareUpload({ file, - s3Key: `${ctx.appId}/attachments/${processedFileName}`, + s3Key: `${ctx.appId}/${ATTACHMENT_PATH}/${processedFileName}`, bucket: ObjectStoreBuckets.APPS, }) }) diff --git a/packages/server/src/constants/index.js b/packages/server/src/constants/index.js index c002c10f7b..c1a992f70e 100644 --- a/packages/server/src/constants/index.js +++ b/packages/server/src/constants/index.js @@ -1,6 +1,6 @@ const { BUILTIN_ROLE_IDS } = require("@budibase/backend-core/roles") const { UserStatus } = require("@budibase/backend-core/constants") -const { ObjectStoreBuckets } = require("@budibase/backend-core/objectStore") +const { objectStore } = require("@budibase/backend-core") exports.JobQueues = { AUTOMATIONS: "automationQueue", @@ -209,6 +209,8 @@ exports.AutomationErrors = { } // pass through the list from the auth/core lib -exports.ObjectStoreBuckets = ObjectStoreBuckets +exports.ObjectStoreBuckets = objectStore.ObjectStoreBuckets + +exports.ATTACHMENT_PATH = "attachments" exports.MAX_AUTOMATION_RECURRING_ERRORS = 5 diff --git a/packages/server/src/sdk/app/export.ts b/packages/server/src/sdk/app/export.ts index 70c4446273..bf7ea81cbc 100644 --- a/packages/server/src/sdk/app/export.ts +++ b/packages/server/src/sdk/app/export.ts @@ -1,7 +1,10 @@ -import { closeDB, dangerousGetDB, doWithDB } from "@budibase/backend-core/db" +import { db as dbCore } from "@budibase/backend-core" import { budibaseTempDir } from "../../utilities/budibaseDir" -import { streamUpload } from "../../utilities/fileSystem/utilities" -import { ObjectStoreBuckets } from "../../constants" +import { + streamUpload, + retrieveDirectory, +} from "../../utilities/fileSystem/utilities" +import { ObjectStoreBuckets, ATTACHMENT_PATH } from "../../constants" import { LINK_USER_METADATA_PREFIX, TABLE_ROW_PREFIX, @@ -25,16 +28,16 @@ export async function exportDB( ) { // streaming a DB dump is a bit more complicated, can't close DB if (opts?.stream) { - const db = dangerousGetDB(dbName) + const db = dbCore.dangerousGetDB(dbName) const memStream = new MemoryStream() memStream.on("end", async () => { - await closeDB(db) + await dbCore.closeDB(db) }) db.dump(memStream, { filter: opts?.filter }) return memStream } - return doWithDB(dbName, async (db: any) => { + return dbCore.doWithDB(dbName, async (db: any) => { // Write the dump to file if required if (opts?.exportName) { const path = join(budibaseTempDir(), opts?.exportName) @@ -49,18 +52,17 @@ export async function exportDB( fs.createReadStream(path) ) } - return fs.createReadStream(path) + } else { + // Stringify the dump in memory if required + const memStream = new MemoryStream() + let appString = "" + memStream.on("data", (chunk: any) => { + appString += chunk.toString() + }) + await db.dump(memStream, { filter: opts?.filter }) + return appString } - - // Stringify the dump in memory if required - const memStream = new MemoryStream() - let appString = "" - memStream.on("data", (chunk: any) => { - appString += chunk.toString() - }) - await db.dump(memStream, { filter: opts?.filter }) - return appString }) } @@ -81,12 +83,17 @@ function defineFilter(excludeRows?: boolean) { * @param {boolean} excludeRows Flag to state whether the export should include data. * @returns {*} either a string or a stream of the backup */ -async function backupAppData( +export async function exportApp( appId: string, - config: any, + config?: any, excludeRows?: boolean ) { - return await exportDB(appId, { + const attachmentsPath = `${dbCore.getProdAppID(appId)}/${ATTACHMENT_PATH}` + const tmpPath = await retrieveDirectory( + ObjectStoreBuckets.APPS, + attachmentsPath + ) + await exportDB(appId, { ...config, filter: defineFilter(excludeRows), }) @@ -98,16 +105,6 @@ async function backupAppData( * @param {boolean} excludeRows Flag to state whether the export should include data. * @returns {*} a readable stream of the backup which is written in real time */ -export async function streamBackup(appId: string, excludeRows: boolean) { - return await backupAppData(appId, { stream: true }, excludeRows) -} - -/** - * Takes a copy of the database state for an app to the object store. - * @param {string} appId The ID of the app which is to be backed up. - * @param {string} backupName The name of the backup located in the object store. - * @return {*} a readable stream to the completed backup file - */ -export async function performBackup(appId: string, backupName: string) { - return await backupAppData(appId, { exportName: backupName }) +export async function streamExportApp(appId: string, excludeRows: boolean) { + return await exportApp(appId, { stream: true }, excludeRows) } diff --git a/packages/server/src/utilities/fileSystem/utilities.js b/packages/server/src/utilities/fileSystem/utilities.js index 1c804c0142..01ba58f5bc 100644 --- a/packages/server/src/utilities/fileSystem/utilities.js +++ b/packages/server/src/utilities/fileSystem/utilities.js @@ -6,6 +6,7 @@ const { streamUpload, retrieve, retrieveToTmp, + retrieveDirectory, deleteFolder, uploadDirectory, downloadTarball, @@ -27,6 +28,7 @@ exports.upload = upload exports.streamUpload = streamUpload exports.retrieve = retrieve exports.retrieveToTmp = retrieveToTmp +exports.retrieveDirectory = retrieveDirectory exports.deleteFolder = deleteFolder exports.uploadDirectory = uploadDirectory exports.downloadTarball = downloadTarball diff --git a/packages/worker/src/sdk/users/users.ts b/packages/worker/src/sdk/users/users.ts index 3b98c8ef52..ce03a12587 100644 --- a/packages/worker/src/sdk/users/users.ts +++ b/packages/worker/src/sdk/users/users.ts @@ -99,16 +99,7 @@ export const paginatedUsers = async ({ */ export const getUser = async (userId: string) => { const db = tenancy.getGlobalDB() - let user - try { - user = await db.get(userId) - } catch (err: any) { - // no user found, just return nothing - if (err.status === 404) { - return {} - } - throw err - } + let user = await db.get(userId) if (user) { delete user.password } From 7c71f76b705374506ca24fed38e05216f15e9b21 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 11 Oct 2022 18:21:58 +0100 Subject: [PATCH 010/151] Export to tarball through tmp. --- packages/server/src/api/controllers/cloud.js | 119 +++++++++--------- packages/server/src/sdk/app/export.ts | 116 +++++++++++------ .../server/src/utilities/fileSystem/index.js | 4 + 3 files changed, 136 insertions(+), 103 deletions(-) diff --git a/packages/server/src/api/controllers/cloud.js b/packages/server/src/api/controllers/cloud.js index 5de5141d74..d766a7987f 100644 --- a/packages/server/src/api/controllers/cloud.js +++ b/packages/server/src/api/controllers/cloud.js @@ -1,24 +1,9 @@ const env = require("../../environment") const { getAllApps, getGlobalDBName } = require("@budibase/backend-core/db") -const { sendTempFile, readFileSync } = require("../../utilities/fileSystem") -const { stringToReadStream } = require("../../utilities") -const { getGlobalDB } = require("@budibase/backend-core/tenancy") -const { create } = require("./application") -const { getDocParams, DocumentType, isDevAppID } = require("../../db/utils") +const { streamFile } = require("../../utilities/fileSystem") +const { DocumentType, isDevAppID } = require("../../db/utils") const sdk = require("../../sdk") -async function createApp(appName, appImport) { - const ctx = { - request: { - body: { - templateString: appImport, - name: appName, - }, - }, - } - return create(ctx) -} - exports.exportApps = async ctx => { if (env.SELF_HOSTED || !env.MULTI_TENANCY) { ctx.throw(400, "Exporting only allowed in multi-tenant cloud environments.") @@ -27,29 +12,18 @@ exports.exportApps = async ctx => { const globalDBString = await sdk.apps.exports.exportDB(getGlobalDBName(), { filter: doc => !doc._id.startsWith(DocumentType.USER), }) - let allDBs = { - global: globalDBString, - } - for (let app of apps) { - const appId = app.appId || app._id - // only export the dev apps as they will be the latest, the user can republish the apps - // in their self hosted environment - if (isDevAppID(appId)) { - allDBs[app.name] = await sdk.apps.exports.exportApp(appId) - } - } - const filename = `cloud-export-${new Date().getTime()}.txt` - ctx.attachment(filename) - ctx.body = sendTempFile(JSON.stringify(allDBs)) -} - -async function getAllDocType(db, docType) { - const response = await db.allDocs( - getDocParams(docType, null, { - include_docs: true, - }) + // only export the dev apps as they will be the latest, the user can republish the apps + // in their self-hosted environment + let appIds = apps + .map(app => app.appId || app._id) + .filter(appId => isDevAppID(appId)) + const tmpPath = await sdk.apps.exports.exportMultipleApps( + appIds, + globalDBString ) - return response.rows.map(row => row.doc) + const filename = `cloud-export-${new Date().getTime()}.tar.gz` + ctx.attachment(filename) + ctx.body = streamFile(tmpPath) } async function hasBeenImported() { @@ -77,30 +51,51 @@ exports.importApps = async ctx => { "Import file is required and environment must be fresh to import apps." ) } - const importFile = ctx.request.files.importFile - const importString = readFileSync(importFile.path) - const dbs = JSON.parse(importString) - const globalDbImport = dbs.global - // remove from the list of apps - delete dbs.global - const globalDb = getGlobalDB() - // load the global db first - await globalDb.load(stringToReadStream(globalDbImport)) - for (let [appName, appImport] of Object.entries(dbs)) { - await createApp(appName, appImport) - } - // if there are any users make sure to remove them - let users = await getAllDocType(globalDb, DocumentType.USER) - let userDeletionPromises = [] - for (let user of users) { - userDeletionPromises.push(globalDb.remove(user._id, user._rev)) - } - if (userDeletionPromises.length > 0) { - await Promise.all(userDeletionPromises) - } - - await globalDb.bulkDocs(users) + // TODO: IMPLEMENT TARBALL EXTRACTION, APP IMPORT, ATTACHMENT IMPORT AND GLOBAL DB IMPORT + // async function getAllDocType(db, docType) { + // const response = await db.allDocs( + // getDocParams(docType, null, { + // include_docs: true, + // }) + // ) + // return response.rows.map(row => row.doc) + // } + // async function createApp(appName, appImport) { + // const ctx = { + // request: { + // body: { + // templateString: appImport, + // name: appName, + // }, + // }, + // } + // return create(ctx) + // } + // const importFile = ctx.request.files.importFile + // const importString = readFileSync(importFile.path) + // const dbs = JSON.parse(importString) + // const globalDbImport = dbs.global + // // remove from the list of apps + // delete dbs.global + // const globalDb = getGlobalDB() + // // load the global db first + // await globalDb.load(stringToReadStream(globalDbImport)) + // for (let [appName, appImport] of Object.entries(dbs)) { + // await createApp(appName, appImport) + // } + // + // // if there are any users make sure to remove them + // let users = await getAllDocType(globalDb, DocumentType.USER) + // let userDeletionPromises = [] + // for (let user of users) { + // userDeletionPromises.push(globalDb.remove(user._id, user._rev)) + // } + // if (userDeletionPromises.length > 0) { + // await Promise.all(userDeletionPromises) + // } + // + // await globalDb.bulkDocs(users) ctx.body = { message: "Apps successfully imported.", } diff --git a/packages/server/src/sdk/app/export.ts b/packages/server/src/sdk/app/export.ts index bf7ea81cbc..ad71062483 100644 --- a/packages/server/src/sdk/app/export.ts +++ b/packages/server/src/sdk/app/export.ts @@ -1,9 +1,7 @@ import { db as dbCore } from "@budibase/backend-core" import { budibaseTempDir } from "../../utilities/budibaseDir" -import { - streamUpload, - retrieveDirectory, -} from "../../utilities/fileSystem/utilities" +import { retrieveDirectory } from "../../utilities/fileSystem/utilities" +import { streamFile } from "../../utilities/fileSystem" import { ObjectStoreBuckets, ATTACHMENT_PATH } from "../../constants" import { LINK_USER_METADATA_PREFIX, @@ -11,10 +9,35 @@ import { USER_METDATA_PREFIX, } from "../../db/utils" import fs from "fs" -import env from "../../environment" import { join } from "path" +const uuid = require("uuid/v4") +const tar = require("tar") const MemoryStream = require("memorystream") +const DB_EXPORT_FILE = "db.txt" +const GLOBAL_DB_EXPORT_FILE = "global.txt" +type ExportOpts = { + filter?: any + exportPath?: string + tar?: boolean + excludeRows?: boolean +} + +function tarFiles(cwd: string, files: string[], exportName?: string) { + exportName = exportName ? `${exportName}.tar.gz` : "export.tar.gz" + tar.create( + { + sync: true, + gzip: true, + file: exportName, + recursive: true, + cwd, + }, + files + ) + return join(cwd, exportName) +} + /** * Exports a DB to either file or a variable (memory). * @param {string} dbName the DB which is to be exported. @@ -22,36 +45,13 @@ const MemoryStream = require("memorystream") * a filter function or the name of the export. * @return {*} either a readable stream or a string */ -export async function exportDB( - dbName: string, - opts: { stream?: boolean; filter?: any; exportName?: string } = {} -) { - // streaming a DB dump is a bit more complicated, can't close DB - if (opts?.stream) { - const db = dbCore.dangerousGetDB(dbName) - const memStream = new MemoryStream() - memStream.on("end", async () => { - await dbCore.closeDB(db) - }) - db.dump(memStream, { filter: opts?.filter }) - return memStream - } - +export async function exportDB(dbName: string, opts: ExportOpts = {}) { return dbCore.doWithDB(dbName, async (db: any) => { // Write the dump to file if required - if (opts?.exportName) { - const path = join(budibaseTempDir(), opts?.exportName) + if (opts?.exportPath) { + const path = opts?.exportPath const writeStream = fs.createWriteStream(path) await db.dump(writeStream, { filter: opts?.filter }) - - // Upload the dump to the object store if self-hosted - if (env.SELF_HOSTED) { - await streamUpload( - ObjectStoreBuckets.BACKUPS, - join(dbName, opts?.exportName), - fs.createReadStream(path) - ) - } return fs.createReadStream(path) } else { // Stringify the dump in memory if required @@ -79,24 +79,57 @@ function defineFilter(excludeRows?: boolean) { * Local utility to back up the database state for an app, excluding global user * data or user relationships. * @param {string} appId The app to back up - * @param {object} config Config to send to export DB - * @param {boolean} excludeRows Flag to state whether the export should include data. + * @param {object} config Config to send to export DB/attachment export * @returns {*} either a string or a stream of the backup */ -export async function exportApp( - appId: string, - config?: any, - excludeRows?: boolean -) { - const attachmentsPath = `${dbCore.getProdAppID(appId)}/${ATTACHMENT_PATH}` +export async function exportApp(appId: string, config?: ExportOpts) { + const prodAppId = dbCore.getProdAppID(appId) + const attachmentsPath = `${prodAppId}/${ATTACHMENT_PATH}` + // export attachments to tmp const tmpPath = await retrieveDirectory( ObjectStoreBuckets.APPS, attachmentsPath ) + // move out of app directory, simplify structure + fs.renameSync(join(tmpPath, attachmentsPath), join(tmpPath, ATTACHMENT_PATH)) + // remove the old app directory created by object export + fs.rmdirSync(join(tmpPath, prodAppId)) + // enforce an export of app DB to the tmp path + const dbPath = join(tmpPath, DB_EXPORT_FILE) await exportDB(appId, { ...config, - filter: defineFilter(excludeRows), + filter: defineFilter(config?.excludeRows), + exportPath: dbPath, }) + // if tar requested, return where the tarball is + if (config?.tar) { + // now the tmpPath contains both the DB export and attachments, tar this + return tarFiles(tmpPath, [ATTACHMENT_PATH, DB_EXPORT_FILE]) + } + // tar not requested, turn the directory where export is + else { + return tmpPath + } +} + +export async function exportMultipleApps( + appIds: string[], + globalDbContents?: string +) { + const tmpPath = join(budibaseTempDir(), uuid()) + let exportPromises: Promise[] = [] + const exportAndMove = async (appId: string) => { + const path = await exportApp(appId) + await fs.promises.rename(path, join(tmpPath, appId)) + } + for (let appId of appIds) { + exportPromises.push(exportAndMove(appId)) + } + await Promise.all(exportPromises) + if (globalDbContents) { + fs.writeFileSync(join(tmpPath, GLOBAL_DB_EXPORT_FILE), globalDbContents) + } + return tarFiles(tmpPath, [...appIds, GLOBAL_DB_EXPORT_FILE]) } /** @@ -106,5 +139,6 @@ export async function exportApp( * @returns {*} a readable stream of the backup which is written in real time */ export async function streamExportApp(appId: string, excludeRows: boolean) { - return await exportApp(appId, { stream: true }, excludeRows) + const tmpPath = await exportApp(appId, { excludeRows, tar: true }) + return streamFile(tmpPath) } diff --git a/packages/server/src/utilities/fileSystem/index.js b/packages/server/src/utilities/fileSystem/index.js index 656ba4816c..69bf38c819 100644 --- a/packages/server/src/utilities/fileSystem/index.js +++ b/packages/server/src/utilities/fileSystem/index.js @@ -112,6 +112,10 @@ exports.apiFileReturn = contents => { return fs.createReadStream(path) } +exports.streamFile = path => { + return fs.createReadStream(path) +} + /** * Writes the provided contents to a temporary file, which can be used briefly. * @param {string} fileContents contents which will be written to a temp file. From f237befbce935481aca9c6760d58c4d55d5e0756 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 11 Oct 2022 19:28:13 +0100 Subject: [PATCH 011/151] Some fixes + cleanup of tmp directory. --- .../backend-core/src/objectStore/index.ts | 1 + packages/server/src/api/controllers/backup.ts | 2 +- packages/server/src/sdk/app/export.ts | 43 +++++++++++++------ 3 files changed, 33 insertions(+), 13 deletions(-) diff --git a/packages/backend-core/src/objectStore/index.ts b/packages/backend-core/src/objectStore/index.ts index 903ba28ed1..4c88166375 100644 --- a/packages/backend-core/src/objectStore/index.ts +++ b/packages/backend-core/src/objectStore/index.ts @@ -263,6 +263,7 @@ export const retrieveToTmp = async (bucketName: string, filepath: string) => { export const retrieveDirectory = async (bucketName: string, path: string) => { let writePath = join(budibaseTempDir(), v4()) + fs.mkdirSync(writePath) const objects = await listAllObjects(bucketName, path) let fullObjects = await Promise.all( objects.map(obj => retrieve(bucketName, obj.Key!)) diff --git a/packages/server/src/api/controllers/backup.ts b/packages/server/src/api/controllers/backup.ts index 878a81e110..7fe197de76 100644 --- a/packages/server/src/api/controllers/backup.ts +++ b/packages/server/src/api/controllers/backup.ts @@ -7,7 +7,7 @@ export async function exportAppDump(ctx: any) { let { appId, excludeRows } = ctx.query const appName = decodeURI(ctx.query.appname) excludeRows = isQsTrue(excludeRows) - const backupIdentifier = `${appName}-export-${new Date().getTime()}.txt` + const backupIdentifier = `${appName}-export-${new Date().getTime()}.tar.gz` ctx.attachment(backupIdentifier) ctx.body = await sdk.apps.exports.streamExportApp(appId, excludeRows) diff --git a/packages/server/src/sdk/app/export.ts b/packages/server/src/sdk/app/export.ts index ad71062483..68aa873129 100644 --- a/packages/server/src/sdk/app/export.ts +++ b/packages/server/src/sdk/app/export.ts @@ -23,19 +23,19 @@ type ExportOpts = { excludeRows?: boolean } -function tarFiles(cwd: string, files: string[], exportName?: string) { - exportName = exportName ? `${exportName}.tar.gz` : "export.tar.gz" +function tarFilesToTmp(tmpDir: string, files: string[]) { + const exportFile = join(budibaseTempDir(), `${uuid()}.tar.gz`) tar.create( { sync: true, gzip: true, - file: exportName, + file: exportFile, recursive: true, - cwd, + cwd: tmpDir, }, files ) - return join(cwd, exportName) + return exportFile } /** @@ -52,7 +52,7 @@ export async function exportDB(dbName: string, opts: ExportOpts = {}) { const path = opts?.exportPath const writeStream = fs.createWriteStream(path) await db.dump(writeStream, { filter: opts?.filter }) - return fs.createReadStream(path) + return path } else { // Stringify the dump in memory if required const memStream = new MemoryStream() @@ -90,10 +90,16 @@ export async function exportApp(appId: string, config?: ExportOpts) { ObjectStoreBuckets.APPS, attachmentsPath ) - // move out of app directory, simplify structure - fs.renameSync(join(tmpPath, attachmentsPath), join(tmpPath, ATTACHMENT_PATH)) - // remove the old app directory created by object export - fs.rmdirSync(join(tmpPath, prodAppId)) + const downloadedPath = join(tmpPath, attachmentsPath), + tmpAttachmentPath = join(tmpPath, ATTACHMENT_PATH) + if (fs.existsSync(downloadedPath)) { + // move out of app directory, simplify structure + fs.renameSync(downloadedPath, tmpAttachmentPath) + // remove the old app directory created by object export + fs.rmdirSync(join(tmpPath, prodAppId)) + } else { + fs.mkdirSync(tmpAttachmentPath) + } // enforce an export of app DB to the tmp path const dbPath = join(tmpPath, DB_EXPORT_FILE) await exportDB(appId, { @@ -104,7 +110,10 @@ export async function exportApp(appId: string, config?: ExportOpts) { // if tar requested, return where the tarball is if (config?.tar) { // now the tmpPath contains both the DB export and attachments, tar this - return tarFiles(tmpPath, [ATTACHMENT_PATH, DB_EXPORT_FILE]) + const tarPath = tarFilesToTmp(tmpPath, [ATTACHMENT_PATH, DB_EXPORT_FILE]) + // cleanup the tmp export files as tarball returned + fs.rmSync(tmpPath, { recursive: true, force: true }) + return tarPath } // tar not requested, turn the directory where export is else { @@ -112,6 +121,13 @@ export async function exportApp(appId: string, config?: ExportOpts) { } } +/** + * Export all apps + global DB (if supplied) to a single tarball, this includes + * the attachments for each app as well. + * @param {string[]} appIds The IDs of the apps to be exported. + * @param {string} globalDbContents The contents of the global DB to export as well. + * @return {string} The path to the tarball. + */ export async function exportMultipleApps( appIds: string[], globalDbContents?: string @@ -129,7 +145,10 @@ export async function exportMultipleApps( if (globalDbContents) { fs.writeFileSync(join(tmpPath, GLOBAL_DB_EXPORT_FILE), globalDbContents) } - return tarFiles(tmpPath, [...appIds, GLOBAL_DB_EXPORT_FILE]) + const tarPath = tarFilesToTmp(tmpPath, [...appIds, GLOBAL_DB_EXPORT_FILE]) + // clear up the tmp path now tarball generated + fs.rmSync(tmpPath, { recursive: true, force: true }) + return tarPath } /** From 19133f08e6d3190b8600dccd6cfd7a34d242db57 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 11 Oct 2022 20:25:22 +0100 Subject: [PATCH 012/151] Adding import functionality - still need to update the attachments URL. --- .../backend-core/src/objectStore/index.ts | 26 ++++--- .../server/src/api/controllers/application.ts | 13 +--- .../src/api/controllers/static/index.ts | 4 +- packages/server/src/constants/index.js | 2 +- packages/server/src/sdk/app/constants.ts | 4 ++ .../src/sdk/app/{export.ts => exports.ts} | 15 ++-- packages/server/src/sdk/app/imports.ts | 71 +++++++++++++++++++ packages/server/src/sdk/app/index.ts | 3 +- .../server/src/utilities/fileSystem/index.js | 15 ---- 9 files changed, 107 insertions(+), 46 deletions(-) create mode 100644 packages/server/src/sdk/app/constants.ts rename packages/server/src/sdk/app/{export.ts => exports.ts} (93%) create mode 100644 packages/server/src/sdk/app/imports.ts diff --git a/packages/backend-core/src/objectStore/index.ts b/packages/backend-core/src/objectStore/index.ts index 4c88166375..e210f9dccc 100644 --- a/packages/backend-core/src/objectStore/index.ts +++ b/packages/backend-core/src/objectStore/index.ts @@ -36,16 +36,16 @@ const STRING_CONTENT_TYPES = [ ] // does normal sanitization and then swaps dev apps to apps -export function sanitizeKey(input: any) { +export function sanitizeKey(input: string) { return sanitize(sanitizeBucket(input)).replace(/\\/g, "/") } // simply handles the dev app to app conversion -export function sanitizeBucket(input: any) { +export function sanitizeBucket(input: string) { return input.replace(new RegExp(APP_DEV_PREFIX, "g"), APP_PREFIX) } -function publicPolicy(bucketName: any) { +function publicPolicy(bucketName: string) { return { Version: "2012-10-17", Statement: [ @@ -73,7 +73,7 @@ const PUBLIC_BUCKETS = [ * @return {Object} an S3 object store object, check S3 Nodejs SDK for usage. * @constructor */ -export const ObjectStore = (bucket: any) => { +export const ObjectStore = (bucket: string) => { const config: any = { s3ForcePathStyle: true, signatureVersion: "v4", @@ -295,7 +295,7 @@ export const deleteFile = async (bucketName: string, filepath: string) => { return objectStore.deleteObject(params) } -export const deleteFiles = async (bucketName: any, filepaths: any) => { +export const deleteFiles = async (bucketName: string, filepaths: string[]) => { const objectStore = ObjectStore(bucketName) await makeSureBucketExists(objectStore, bucketName) const params = { @@ -311,8 +311,8 @@ export const deleteFiles = async (bucketName: any, filepaths: any) => { * Delete a path, including everything within. */ export const deleteFolder = async ( - bucketName: any, - folder: any + bucketName: string, + folder: string ): Promise => { bucketName = sanitizeBucket(bucketName) folder = sanitizeKey(folder) @@ -345,9 +345,9 @@ export const deleteFolder = async ( } export const uploadDirectory = async ( - bucketName: any, - localPath: any, - bucketPath: any + bucketName: string, + localPath: string, + bucketPath: string ) => { bucketName = sanitizeBucket(bucketName) let uploads = [] @@ -379,7 +379,11 @@ exports.downloadTarballDirect = async ( await streamPipeline(response.body, zlib.Unzip(), tar.extract(path)) } -export const downloadTarball = async (url: any, bucketName: any, path: any) => { +export const downloadTarball = async ( + url: string, + bucketName: string, + path: string +) => { bucketName = sanitizeBucket(bucketName) path = sanitizeKey(path) const response = await fetch(url) diff --git a/packages/server/src/api/controllers/application.ts b/packages/server/src/api/controllers/application.ts index a7caf85e94..87bc7d0033 100644 --- a/packages/server/src/api/controllers/application.ts +++ b/packages/server/src/api/controllers/application.ts @@ -5,11 +5,7 @@ import { createRoutingView, createAllSearchIndex, } from "../../db/views/staticViews" -import { - getTemplateStream, - createApp, - deleteApp, -} from "../../utilities/fileSystem" +import { createApp, deleteApp } from "../../utilities/fileSystem" import { generateAppID, getLayoutParams, @@ -50,6 +46,7 @@ import { errors, events, migrations } from "@budibase/backend-core" import { App, Layout, Screen, MigrationType } from "@budibase/types" import { BASE_LAYOUT_PROP_IDS } from "../../constants/layouts" import { enrichPluginURLs } from "../../utilities/plugins" +import sdk from "../../sdk" const URL_REGEX_SLASH = /\/|\\/g @@ -153,11 +150,7 @@ async function createInstance(template: any) { throw "Error loading database dump from memory." } } else if (template && template.useTemplate === "true") { - /* istanbul ignore next */ - const { ok } = await db.load(await getTemplateStream(template)) - if (!ok) { - throw "Error loading database dump from template." - } + await sdk.apps.imports.importApp(appId, db, template) } else { // create the users table await db.put(USERS_TABLE_SCHEMA) diff --git a/packages/server/src/api/controllers/static/index.ts b/packages/server/src/api/controllers/static/index.ts index f60dc12971..ec2ea70820 100644 --- a/packages/server/src/api/controllers/static/index.ts +++ b/packages/server/src/api/controllers/static/index.ts @@ -5,7 +5,7 @@ require("svelte/register") const send = require("koa-send") const { resolve, join } = require("../../../utilities/centralPath") const uuid = require("uuid") -const { ObjectStoreBuckets, ATTACHMENT_PATH } = require("../../../constants") +const { ObjectStoreBuckets, ATTACHMENT_DIR } = require("../../../constants") const { processString } = require("@budibase/string-templates") const { loadHandlebarsFile, @@ -90,7 +90,7 @@ export const uploadFile = async function (ctx: any) { return prepareUpload({ file, - s3Key: `${ctx.appId}/${ATTACHMENT_PATH}/${processedFileName}`, + s3Key: `${ctx.appId}/${ATTACHMENT_DIR}/${processedFileName}`, bucket: ObjectStoreBuckets.APPS, }) }) diff --git a/packages/server/src/constants/index.js b/packages/server/src/constants/index.js index c1a992f70e..b9362cecf6 100644 --- a/packages/server/src/constants/index.js +++ b/packages/server/src/constants/index.js @@ -211,6 +211,6 @@ exports.AutomationErrors = { // pass through the list from the auth/core lib exports.ObjectStoreBuckets = objectStore.ObjectStoreBuckets -exports.ATTACHMENT_PATH = "attachments" +exports.ATTACHMENT_DIR = "attachments" exports.MAX_AUTOMATION_RECURRING_ERRORS = 5 diff --git a/packages/server/src/sdk/app/constants.ts b/packages/server/src/sdk/app/constants.ts new file mode 100644 index 0000000000..e831172683 --- /dev/null +++ b/packages/server/src/sdk/app/constants.ts @@ -0,0 +1,4 @@ +import { ATTACHMENT_DIR as attachmentDir } from "../../constants" +export const DB_EXPORT_FILE = "db.txt" +export const ATTACHMENT_DIR = attachmentDir +export const GLOBAL_DB_EXPORT_FILE = "global.txt" diff --git a/packages/server/src/sdk/app/export.ts b/packages/server/src/sdk/app/exports.ts similarity index 93% rename from packages/server/src/sdk/app/export.ts rename to packages/server/src/sdk/app/exports.ts index 68aa873129..1ecf69de68 100644 --- a/packages/server/src/sdk/app/export.ts +++ b/packages/server/src/sdk/app/exports.ts @@ -2,20 +2,23 @@ import { db as dbCore } from "@budibase/backend-core" import { budibaseTempDir } from "../../utilities/budibaseDir" import { retrieveDirectory } from "../../utilities/fileSystem/utilities" import { streamFile } from "../../utilities/fileSystem" -import { ObjectStoreBuckets, ATTACHMENT_PATH } from "../../constants" +import { ObjectStoreBuckets } from "../../constants" import { LINK_USER_METADATA_PREFIX, TABLE_ROW_PREFIX, USER_METDATA_PREFIX, } from "../../db/utils" +import { + DB_EXPORT_FILE, + GLOBAL_DB_EXPORT_FILE, + ATTACHMENT_DIR, +} from "./constants" import fs from "fs" import { join } from "path" const uuid = require("uuid/v4") const tar = require("tar") const MemoryStream = require("memorystream") -const DB_EXPORT_FILE = "db.txt" -const GLOBAL_DB_EXPORT_FILE = "global.txt" type ExportOpts = { filter?: any exportPath?: string @@ -84,14 +87,14 @@ function defineFilter(excludeRows?: boolean) { */ export async function exportApp(appId: string, config?: ExportOpts) { const prodAppId = dbCore.getProdAppID(appId) - const attachmentsPath = `${prodAppId}/${ATTACHMENT_PATH}` + const attachmentsPath = `${prodAppId}/${ATTACHMENT_DIR}` // export attachments to tmp const tmpPath = await retrieveDirectory( ObjectStoreBuckets.APPS, attachmentsPath ) const downloadedPath = join(tmpPath, attachmentsPath), - tmpAttachmentPath = join(tmpPath, ATTACHMENT_PATH) + tmpAttachmentPath = join(tmpPath, ATTACHMENT_DIR) if (fs.existsSync(downloadedPath)) { // move out of app directory, simplify structure fs.renameSync(downloadedPath, tmpAttachmentPath) @@ -110,7 +113,7 @@ export async function exportApp(appId: string, config?: ExportOpts) { // if tar requested, return where the tarball is if (config?.tar) { // now the tmpPath contains both the DB export and attachments, tar this - const tarPath = tarFilesToTmp(tmpPath, [ATTACHMENT_PATH, DB_EXPORT_FILE]) + const tarPath = tarFilesToTmp(tmpPath, [ATTACHMENT_DIR, DB_EXPORT_FILE]) // cleanup the tmp export files as tarball returned fs.rmSync(tmpPath, { recursive: true, force: true }) return tarPath diff --git a/packages/server/src/sdk/app/imports.ts b/packages/server/src/sdk/app/imports.ts new file mode 100644 index 0000000000..96629db39a --- /dev/null +++ b/packages/server/src/sdk/app/imports.ts @@ -0,0 +1,71 @@ +import { db as dbCore } from "@budibase/backend-core" +import { budibaseTempDir } from "../../utilities/budibaseDir" +import { DB_EXPORT_FILE, ATTACHMENT_DIR } from "./constants" +import { uploadDirectory } from "../../utilities/fileSystem/utilities" +import { ObjectStoreBuckets } from "../../constants" +import { join } from "path" +import fs from "fs" +const uuid = require("uuid/v4") +const tar = require("tar") + +type TemplateType = { + file?: { + type: string + path: string + } + key?: string +} + +/** + * This function manages temporary template files which are stored by Koa. + * @param {Object} template The template object retrieved from the Koa context object. + * @returns {Object} Returns a fs read stream which can be loaded into the database. + */ +async function getTemplateStream(template: TemplateType) { + if (template.file) { + return fs.createReadStream(template.file.path) + } else if (template.key) { + const [type, name] = template.key.split("/") + const tmpPath = await exports.downloadTemplate(type, name) + return fs.createReadStream(join(tmpPath, name, "db", "dump.txt")) + } +} + +export async function importApp( + appId: string, + db: PouchDB.Database, + template: TemplateType +) { + let prodAppId = dbCore.getProdAppID(appId) + let dbStream: any + if (template.file && template.file.type === "application/gzip") { + const tmpPath = join(budibaseTempDir(), uuid()) + fs.mkdirSync(tmpPath) + // extract the tarball + tar.extract({ + sync: true, + cwd: tmpPath, + file: template.file.path, + }) + const attachmentPath = join(tmpPath, ATTACHMENT_DIR) + // have to handle object import + if (fs.existsSync(attachmentPath)) { + await uploadDirectory( + ObjectStoreBuckets.APPS, + attachmentPath, + join(prodAppId, ATTACHMENT_DIR) + ) + } + dbStream = fs.createReadStream(join(tmpPath, DB_EXPORT_FILE)) + } else { + dbStream = await getTemplateStream(template) + } + // @ts-ignore + const { ok } = await db.load(dbStream) + if (!ok) { + throw "Error loading database dump from template." + } else { + // TODO: need to iterate over attachments and update their URLs + } + return ok +} diff --git a/packages/server/src/sdk/app/index.ts b/packages/server/src/sdk/app/index.ts index 3927539bc8..5ac7c04b2c 100644 --- a/packages/server/src/sdk/app/index.ts +++ b/packages/server/src/sdk/app/index.ts @@ -1 +1,2 @@ -export * as exports from "./export" +export * as exports from "./exports" +export * as imports from "./imports" diff --git a/packages/server/src/utilities/fileSystem/index.js b/packages/server/src/utilities/fileSystem/index.js index 69bf38c819..1eb8a481e5 100644 --- a/packages/server/src/utilities/fileSystem/index.js +++ b/packages/server/src/utilities/fileSystem/index.js @@ -74,21 +74,6 @@ exports.checkDevelopmentEnvironment = () => { } } -/** - * This function manages temporary template files which are stored by Koa. - * @param {Object} template The template object retrieved from the Koa context object. - * @returns {Object} Returns an fs read stream which can be loaded into the database. - */ -exports.getTemplateStream = async template => { - if (template.file) { - return fs.createReadStream(template.file.path) - } else { - const [type, name] = template.key.split("/") - const tmpPath = await exports.downloadTemplate(type, name) - return fs.createReadStream(join(tmpPath, name, "db", "dump.txt")) - } -} - /** * Used to retrieve a handlebars file from the system which will be used as a template. * This is allowable as the template handlebars files should be static and identical across From 8210233f3655575c66b59ddec4f6bfbfb17d6d73 Mon Sep 17 00:00:00 2001 From: Mel O'Hagan Date: Wed, 12 Oct 2022 10:22:54 +0100 Subject: [PATCH 013/151] Edit clone of column --- packages/builder/src/components/backend/DataTable/Table.svelte | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/builder/src/components/backend/DataTable/Table.svelte b/packages/builder/src/components/backend/DataTable/Table.svelte index 85b271dee8..adc35a333d 100644 --- a/packages/builder/src/components/backend/DataTable/Table.svelte +++ b/packages/builder/src/components/backend/DataTable/Table.svelte @@ -8,6 +8,7 @@ import CreateEditRow from "./modals/CreateEditRow.svelte" import CreateEditUser from "./modals/CreateEditUser.svelte" import CreateEditColumn from "./modals/CreateEditColumn.svelte" + import { cloneDeep } from "lodash/fp" import { TableNames, UNEDITABLE_USER_FIELDS, @@ -110,7 +111,7 @@ } const editColumn = field => { - editableColumn = schema?.[field] + editableColumn = cloneDeep(schema?.[field]) if (editableColumn) { editColumnModal.show() } From 8d7f40e443a6ee81f6cdfdc8651f98b4b45d927b Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 12 Oct 2022 17:02:23 +0100 Subject: [PATCH 014/151] Refactoring more to Typescript, adding the ability to use the _find API of CouchDB. --- .../src/db/{index.js => index.ts} | 54 ++++-- packages/backend-core/src/db/utils.ts | 30 +--- .../server/src/api/controllers/application.ts | 2 +- packages/server/src/api/controllers/backup.ts | 2 +- packages/server/src/api/controllers/cloud.js | 7 +- .../src/api/controllers/table/bulkFormula.js | 7 +- .../src/api/controllers/table/external.js | 8 +- .../server/src/api/controllers/table/index.js | 8 +- .../src/api/controllers/table/internal.ts | 10 +- .../server/src/api/controllers/table/utils.ts | 40 ----- .../server/src/api/controllers/view/index.js | 4 +- .../server/src/automations/automationUtils.js | 4 +- .../functions/backfill/app/tables.ts | 19 +-- .../src/sdk/app/{ => backups}/constants.ts | 0 .../src/sdk/app/{ => backups}/exports.ts | 10 +- .../src/sdk/app/{ => backups}/imports.ts | 33 +++- packages/server/src/sdk/app/backups/index.ts | 7 + packages/server/src/sdk/app/index.ts | 2 - packages/server/src/sdk/app/tables/index.ts | 60 +++++++ packages/server/src/sdk/index.ts | 6 +- packages/types/package.json | 3 +- packages/types/src/core/db.ts | 22 +++ packages/types/src/core/index.ts | 1 + packages/types/src/documents/app/table.ts | 1 + packages/types/src/index.ts | 1 + packages/types/yarn.lock | 158 ++++++++++++++++++ 26 files changed, 369 insertions(+), 130 deletions(-) rename packages/backend-core/src/db/{index.js => index.ts} (58%) rename packages/server/src/sdk/app/{ => backups}/constants.ts (100%) rename packages/server/src/sdk/app/{ => backups}/exports.ts (94%) rename packages/server/src/sdk/app/{ => backups}/imports.ts (62%) create mode 100644 packages/server/src/sdk/app/backups/index.ts delete mode 100644 packages/server/src/sdk/app/index.ts create mode 100644 packages/server/src/sdk/app/tables/index.ts create mode 100644 packages/types/src/core/db.ts create mode 100644 packages/types/src/core/index.ts diff --git a/packages/backend-core/src/db/index.js b/packages/backend-core/src/db/index.ts similarity index 58% rename from packages/backend-core/src/db/index.js rename to packages/backend-core/src/db/index.ts index aa6f7ebc2c..016155033a 100644 --- a/packages/backend-core/src/db/index.js +++ b/packages/backend-core/src/db/index.ts @@ -1,8 +1,11 @@ -const pouch = require("./pouch") -const env = require("../environment") +import pouch from "./pouch" +import env from "../environment" +import { checkSlashesInUrl } from "../helpers" +import fetch from "node-fetch" +import { PouchOptions, CouchFindOptions } from "@budibase/types" -const openDbs = [] -let PouchDB +const openDbs: string[] = [] +let PouchDB: any let initialised = false const dbList = new Set() @@ -14,8 +17,8 @@ if (env.MEMORY_LEAK_CHECK) { } const put = - dbPut => - async (doc, options = {}) => { + (dbPut: any) => + async (doc: any, options = {}) => { if (!doc.createdAt) { doc.createdAt = new Date().toISOString() } @@ -29,7 +32,7 @@ const checkInitialised = () => { } } -exports.init = opts => { +export async function init(opts: PouchOptions) { PouchDB = pouch.getPouch(opts) initialised = true } @@ -37,7 +40,7 @@ exports.init = opts => { // NOTE: THIS IS A DANGEROUS FUNCTION - USE WITH CAUTION // this function is prone to leaks, should only be used // in situations that using the function doWithDB does not work -exports.dangerousGetDB = (dbName, opts) => { +export async function dangerousGetDB(dbName: string, opts: any) { checkInitialised() if (env.isTest()) { dbList.add(dbName) @@ -53,7 +56,7 @@ exports.dangerousGetDB = (dbName, opts) => { // use this function if you have called dangerousGetDB - close // the databases you've opened once finished -exports.closeDB = async db => { +export async function closeDB(db: PouchDB.Database) { if (!db || env.isTest()) { return } @@ -71,7 +74,7 @@ exports.closeDB = async db => { // we have to use a callback for this so that we can close // the DB when we're done, without this manual requests would // need to close the database when done with it to avoid memory leaks -exports.doWithDB = async (dbName, cb, opts = {}) => { +export async function doWithDB(dbName: string, cb: any, opts = {}) { const db = exports.dangerousGetDB(dbName, opts) // need this to be async so that we can correctly close DB after all // async operations have been completed @@ -82,10 +85,39 @@ exports.doWithDB = async (dbName, cb, opts = {}) => { } } -exports.allDbs = () => { +export function allDbs() { if (!env.isTest()) { throw new Error("Cannot be used outside test environment.") } checkInitialised() return [...dbList] } + +export async function directCouchQuery( + path: string, + method: string = "GET", + body?: any +) { + let { url, cookie } = pouch.getCouchInfo() + const couchUrl = `${url}/${path}` + const params: any = { + method: method, + headers: { + Authorization: cookie, + }, + } + if (body && method !== "GET") { + params.body = body + } + const response = await fetch(checkSlashesInUrl(encodeURI(couchUrl)), params) + if (response.status < 300) { + return await response.json() + } else { + throw "Cannot connect to CouchDB instance" + } +} + +export async function directCouchFind(dbName: string, opts: CouchFindOptions) { + const json = await directCouchQuery(`${dbName}/_find`, "POST", opts) + return { rows: json.docs, bookmark: json.bookmark } +} diff --git a/packages/backend-core/src/db/utils.ts b/packages/backend-core/src/db/utils.ts index 1c4be7e366..c78c11217e 100644 --- a/packages/backend-core/src/db/utils.ts +++ b/packages/backend-core/src/db/utils.ts @@ -4,11 +4,8 @@ import env from "../environment" import { SEPARATOR, DocumentType, UNICODE_MAX, ViewName } from "./constants" import { getTenantId, getGlobalDB } from "../context" import { getGlobalDBName } from "./tenancy" -import fetch from "node-fetch" -import { doWithDB, allDbs } from "./index" -import { getCouchInfo } from "./pouch" +import { doWithDB, allDbs, directCouchQuery } from "./index" import { getAppMetadata } from "../cache/appMetadata" -import { checkSlashesInUrl } from "../helpers" import { isDevApp, isDevAppID, getProdAppID } from "./conversions" import { APP_PREFIX } from "./constants" import * as events from "../events" @@ -209,22 +206,11 @@ export async function getAllDbs(opts = { efficient: false }) { return allDbs() } let dbs: any[] = [] - let { url, cookie } = getCouchInfo() - async function addDbs(couchUrl: string) { - const response = await fetch(checkSlashesInUrl(encodeURI(couchUrl)), { - method: "GET", - headers: { - Authorization: cookie, - }, - }) - if (response.status === 200) { - let json = await response.json() - dbs = dbs.concat(json) - } else { - throw "Cannot connect to CouchDB instance" - } + async function addDbs(couchPath: string) { + const json = await directCouchQuery(couchPath) + dbs = dbs.concat(json) } - let couchUrl = `${url}/_all_dbs` + let couchPath = "/_all_dbs" let tenantId = getTenantId() if (!env.MULTI_TENANCY || (!efficient && tenantId === DEFAULT_TENANT_ID)) { // just get all DBs when: @@ -232,12 +218,12 @@ export async function getAllDbs(opts = { efficient: false }) { // - default tenant // - apps dbs don't contain tenant id // - non-default tenant dbs are filtered out application side in getAllApps - await addDbs(couchUrl) + await addDbs(couchPath) } else { // get prod apps - await addDbs(getStartEndKeyURL(couchUrl, DocumentType.APP, tenantId)) + await addDbs(getStartEndKeyURL(couchPath, DocumentType.APP, tenantId)) // get dev apps - await addDbs(getStartEndKeyURL(couchUrl, DocumentType.APP_DEV, tenantId)) + await addDbs(getStartEndKeyURL(couchPath, DocumentType.APP_DEV, tenantId)) // add global db name dbs.push(getGlobalDBName(tenantId)) } diff --git a/packages/server/src/api/controllers/application.ts b/packages/server/src/api/controllers/application.ts index 87bc7d0033..f3dca51f72 100644 --- a/packages/server/src/api/controllers/application.ts +++ b/packages/server/src/api/controllers/application.ts @@ -150,7 +150,7 @@ async function createInstance(template: any) { throw "Error loading database dump from memory." } } else if (template && template.useTemplate === "true") { - await sdk.apps.imports.importApp(appId, db, template) + await sdk.backups.importApp(appId, db, template) } else { // create the users table await db.put(USERS_TABLE_SCHEMA) diff --git a/packages/server/src/api/controllers/backup.ts b/packages/server/src/api/controllers/backup.ts index 7fe197de76..0ffda2c733 100644 --- a/packages/server/src/api/controllers/backup.ts +++ b/packages/server/src/api/controllers/backup.ts @@ -9,7 +9,7 @@ export async function exportAppDump(ctx: any) { excludeRows = isQsTrue(excludeRows) const backupIdentifier = `${appName}-export-${new Date().getTime()}.tar.gz` ctx.attachment(backupIdentifier) - ctx.body = await sdk.apps.exports.streamExportApp(appId, excludeRows) + ctx.body = await sdk.backups.streamExportApp(appId, excludeRows) await context.doInAppContext(appId, async () => { const appDb = context.getAppDB() diff --git a/packages/server/src/api/controllers/cloud.js b/packages/server/src/api/controllers/cloud.js index d766a7987f..323c7409db 100644 --- a/packages/server/src/api/controllers/cloud.js +++ b/packages/server/src/api/controllers/cloud.js @@ -9,7 +9,7 @@ exports.exportApps = async ctx => { ctx.throw(400, "Exporting only allowed in multi-tenant cloud environments.") } const apps = await getAllApps({ all: true }) - const globalDBString = await sdk.apps.exports.exportDB(getGlobalDBName(), { + const globalDBString = await sdk.backups.exportDB(getGlobalDBName(), { filter: doc => !doc._id.startsWith(DocumentType.USER), }) // only export the dev apps as they will be the latest, the user can republish the apps @@ -17,10 +17,7 @@ exports.exportApps = async ctx => { let appIds = apps .map(app => app.appId || app._id) .filter(appId => isDevAppID(appId)) - const tmpPath = await sdk.apps.exports.exportMultipleApps( - appIds, - globalDBString - ) + const tmpPath = await sdk.backups.exportMultipleApps(appIds, globalDBString) const filename = `cloud-export-${new Date().getTime()}.tar.gz` ctx.attachment(filename) ctx.body = streamFile(tmpPath) diff --git a/packages/server/src/api/controllers/table/bulkFormula.js b/packages/server/src/api/controllers/table/bulkFormula.js index d736c126f2..733c16d455 100644 --- a/packages/server/src/api/controllers/table/bulkFormula.js +++ b/packages/server/src/api/controllers/table/bulkFormula.js @@ -1,10 +1,11 @@ const { FieldTypes, FormulaTypes } = require("../../../constants") -const { getAllInternalTables, clearColumns } = require("./utils") +const { clearColumns } = require("./utils") const { doesContainStrings } = require("@budibase/string-templates") const { cloneDeep } = require("lodash/fp") const { isEqual, uniq } = require("lodash") const { updateAllFormulasInTable } = require("../row/staticFormula") const { getAppDB } = require("@budibase/backend-core/context") +const sdk = require("../../../sdk") function isStaticFormula(column) { return ( @@ -39,7 +40,7 @@ function getFormulaThatUseColumn(table, columnNames) { */ async function checkIfFormulaNeedsCleared(table, { oldTable, deletion }) { // start by retrieving all tables, remove the current table from the list - const tables = (await getAllInternalTables()).filter( + const tables = (await sdk.tables.getAllInternalTables()).filter( tbl => tbl._id !== table._id ) const schemaToUse = oldTable ? oldTable.schema : table.schema @@ -99,7 +100,7 @@ async function updateRelatedFormulaLinksOnTables( ) { const db = getAppDB() // start by retrieving all tables, remove the current table from the list - const tables = (await getAllInternalTables()).filter( + const tables = (await sdk.tables.getAllInternalTables()).filter( tbl => tbl._id !== table._id ) // clone the tables, so we can compare at end diff --git a/packages/server/src/api/controllers/table/external.js b/packages/server/src/api/controllers/table/external.js index d919e9dad7..fe9270fe1d 100644 --- a/packages/server/src/api/controllers/table/external.js +++ b/packages/server/src/api/controllers/table/external.js @@ -3,7 +3,6 @@ const { breakExternalTableId, } = require("../../../integrations/utils") const { - getTable, generateForeignKey, generateJunctionTableName, foreignKeyStructure, @@ -20,6 +19,7 @@ const csvParser = require("../../../utilities/csvParser") const { handleRequest } = require("../row/external") const { getAppDB } = require("@budibase/backend-core/context") const { events } = require("@budibase/backend-core") +const sdk = require("../../../sdk") async function makeTableRequest( datasource, @@ -181,7 +181,7 @@ exports.save = async function (ctx) { let oldTable if (ctx.request.body && ctx.request.body._id) { - oldTable = await getTable(ctx.request.body._id) + oldTable = await sdk.tables.getTable(ctx.request.body._id) } if (hasTypeChanged(tableToSave, oldTable)) { @@ -281,7 +281,7 @@ exports.save = async function (ctx) { } exports.destroy = async function (ctx) { - const tableToDelete = await getTable(ctx.params.tableId) + const tableToDelete = await sdk.tables.getTable(ctx.params.tableId) if (!tableToDelete || !tableToDelete.created) { ctx.throw(400, "Cannot delete tables which weren't created in Budibase.") } @@ -303,7 +303,7 @@ exports.destroy = async function (ctx) { } exports.bulkImport = async function (ctx) { - const table = await getTable(ctx.params.tableId) + const table = await sdk.tables.getTable(ctx.params.tableId) const { dataImport } = ctx.request.body if (!dataImport || !dataImport.schema || !dataImport.csvString) { ctx.throw(400, "Provided data import information is invalid.") diff --git a/packages/server/src/api/controllers/table/index.js b/packages/server/src/api/controllers/table/index.js index e6192457af..3a20f4dff6 100644 --- a/packages/server/src/api/controllers/table/index.js +++ b/packages/server/src/api/controllers/table/index.js @@ -4,8 +4,8 @@ const csvParser = require("../../../utilities/csvParser") const { isExternalTable, isSQL } = require("../../../integrations/utils") const { getDatasourceParams } = require("../../../db/utils") const { getAppDB } = require("@budibase/backend-core/context") -const { getTable, getAllInternalTables } = require("./utils") const { events } = require("@budibase/backend-core") +const sdk = require("../../../sdk") function pickApi({ tableId, table }) { if (table && !tableId) { @@ -23,7 +23,7 @@ function pickApi({ tableId, table }) { exports.fetch = async function (ctx) { const db = getAppDB() - const internal = await getAllInternalTables() + const internal = await sdk.tables.getAllInternalTables() const externalTables = await db.allDocs( getDatasourceParams("plus", { @@ -50,7 +50,7 @@ exports.fetch = async function (ctx) { exports.find = async function (ctx) { const tableId = ctx.params.tableId - ctx.body = await getTable(tableId) + ctx.body = await sdk.tables.getTable(tableId) } exports.save = async function (ctx) { @@ -101,7 +101,7 @@ exports.validateCSVSchema = async function (ctx) { const { csvString, schema = {}, tableId } = ctx.request.body let existingTable if (tableId) { - existingTable = await getTable(tableId) + existingTable = await sdk.tables.getTable(tableId) } let result = await csvParser.parse(csvString, schema) if (existingTable) { diff --git a/packages/server/src/api/controllers/table/internal.ts b/packages/server/src/api/controllers/table/internal.ts index 7e55c71aea..a50009b1f6 100644 --- a/packages/server/src/api/controllers/table/internal.ts +++ b/packages/server/src/api/controllers/table/internal.ts @@ -1,12 +1,7 @@ import { updateLinks, EventType } from "../../../db/linkedRows" import { getRowParams, generateTableID } from "../../../db/utils" import { FieldTypes } from "../../../constants" -import { - TableSaveFunctions, - hasTypeChanged, - getTable, - handleDataImport, -} from "./utils" +import { TableSaveFunctions, hasTypeChanged, handleDataImport } from "./utils" const { getAppDB } = require("@budibase/backend-core/context") import { isTest } from "../../../environment" import { @@ -19,6 +14,7 @@ import { quotas } from "@budibase/pro" import { isEqual } from "lodash" import { cloneDeep } from "lodash/fp" import env from "../../../environment" +import sdk from "../../../sdk" function checkAutoColumns(table: Table, oldTable: Table) { if (!table.schema) { @@ -188,7 +184,7 @@ export async function destroy(ctx: any) { } export async function bulkImport(ctx: any) { - const table = await getTable(ctx.params.tableId) + const table = await sdk.tables.getTable(ctx.params.tableId) const { dataImport } = ctx.request.body await handleDataImport(ctx.user, table, dataImport) return table diff --git a/packages/server/src/api/controllers/table/utils.ts b/packages/server/src/api/controllers/table/utils.ts index c2aa2f47c9..79c551b63a 100644 --- a/packages/server/src/api/controllers/table/utils.ts +++ b/packages/server/src/api/controllers/table/utils.ts @@ -256,46 +256,6 @@ class TableSaveFunctions { } } -export async function getAllInternalTables() { - const db = getAppDB() - const internalTables = await db.allDocs( - getTableParams(null, { - include_docs: true, - }) - ) - return internalTables.rows.map((tableDoc: any) => ({ - ...tableDoc.doc, - type: "internal", - sourceId: BudibaseInternalDB._id, - })) -} - -export async function getAllExternalTables(datasourceId: any) { - const db = getAppDB() - const datasource = await db.get(datasourceId) - if (!datasource || !datasource.entities) { - throw "Datasource is not configured fully." - } - return datasource.entities -} - -export async function getExternalTable(datasourceId: any, tableName: any) { - const entities = await getAllExternalTables(datasourceId) - return entities[tableName] -} - -export async function getTable(tableId: any) { - const db = getAppDB() - if (isExternalTable(tableId)) { - let { datasourceId, tableName } = breakExternalTableId(tableId) - const datasource = await db.get(datasourceId) - const table = await getExternalTable(datasourceId, tableName) - return { ...table, sql: isSQL(datasource) } - } else { - return db.get(tableId) - } -} - export async function checkForViewUpdates( table: any, rename: any, diff --git a/packages/server/src/api/controllers/view/index.js b/packages/server/src/api/controllers/view/index.js index b2c3a84c59..91657cfc21 100644 --- a/packages/server/src/api/controllers/view/index.js +++ b/packages/server/src/api/controllers/view/index.js @@ -3,12 +3,12 @@ const { apiFileReturn } = require("../../../utilities/fileSystem") const exporters = require("./exporters") const { saveView, getView, getViews, deleteView } = require("./utils") const { fetchView } = require("../row") -const { getTable } = require("../table/utils") const { FieldTypes } = require("../../../constants") const { getAppDB } = require("@budibase/backend-core/context") const { events } = require("@budibase/backend-core") const { DocumentType } = require("../../../db/utils") const { cloneDeep, isEqual } = require("lodash") +const sdk = require("../../../sdk") exports.fetch = async ctx => { ctx.body = await getViews() @@ -144,7 +144,7 @@ exports.exportView = async ctx => { let schema = view && view.meta && view.meta.schema const tableId = ctx.params.tableId || view.meta.tableId - const table = await getTable(tableId) + const table = await sdk.tables.getTable(tableId) if (!schema) { schema = table.schema } diff --git a/packages/server/src/automations/automationUtils.js b/packages/server/src/automations/automationUtils.js index 0646e453c2..06a79e6ab4 100644 --- a/packages/server/src/automations/automationUtils.js +++ b/packages/server/src/automations/automationUtils.js @@ -1,10 +1,10 @@ -const { getTable } = require("../api/controllers/table/utils") const { findHBSBlocks, decodeJSBinding, isJSBinding, encodeJSBinding, } = require("@budibase/string-templates") +const sdk = require("../sdk") /** * When values are input to the system generally they will be of type string as this is required for template strings. @@ -64,7 +64,7 @@ exports.cleanInputValues = (inputs, schema) => { * @returns {Promise} The cleaned up rows object, will should now have all the required primitive types. */ exports.cleanUpRow = async (tableId, row) => { - let table = await getTable(tableId) + let table = await sdk.tables.getTable(tableId) return exports.cleanInputValues(row, { properties: table.schema }) } diff --git a/packages/server/src/migrations/functions/backfill/app/tables.ts b/packages/server/src/migrations/functions/backfill/app/tables.ts index 150a3b4d4a..b6d896a1ca 100644 --- a/packages/server/src/migrations/functions/backfill/app/tables.ts +++ b/packages/server/src/migrations/functions/backfill/app/tables.ts @@ -1,18 +1,11 @@ import { events } from "@budibase/backend-core" -import { getTableParams } from "../../../../db/utils" -import { Table } from "@budibase/types" +import sdk from "../../../../sdk" -const getTables = async (appDb: any): Promise => { - const response = await appDb.allDocs( - getTableParams(null, { - include_docs: true, - }) - ) - return response.rows.map((row: any) => row.doc) -} - -export const backfill = async (appDb: any, timestamp: string | number) => { - const tables = await getTables(appDb) +export const backfill = async ( + appDb: PouchDB.Database, + timestamp: string | number +) => { + const tables = await sdk.tables.getAllInternalTables(appDb) for (const table of tables) { await events.table.created(table, timestamp) diff --git a/packages/server/src/sdk/app/constants.ts b/packages/server/src/sdk/app/backups/constants.ts similarity index 100% rename from packages/server/src/sdk/app/constants.ts rename to packages/server/src/sdk/app/backups/constants.ts diff --git a/packages/server/src/sdk/app/exports.ts b/packages/server/src/sdk/app/backups/exports.ts similarity index 94% rename from packages/server/src/sdk/app/exports.ts rename to packages/server/src/sdk/app/backups/exports.ts index 1ecf69de68..adf39f7e15 100644 --- a/packages/server/src/sdk/app/exports.ts +++ b/packages/server/src/sdk/app/backups/exports.ts @@ -1,13 +1,13 @@ import { db as dbCore } from "@budibase/backend-core" -import { budibaseTempDir } from "../../utilities/budibaseDir" -import { retrieveDirectory } from "../../utilities/fileSystem/utilities" -import { streamFile } from "../../utilities/fileSystem" -import { ObjectStoreBuckets } from "../../constants" +import { budibaseTempDir } from "../../../utilities/budibaseDir" +import { retrieveDirectory } from "../../../utilities/fileSystem/utilities" +import { streamFile } from "../../../utilities/fileSystem" +import { ObjectStoreBuckets } from "../../../constants" import { LINK_USER_METADATA_PREFIX, TABLE_ROW_PREFIX, USER_METDATA_PREFIX, -} from "../../db/utils" +} from "../../../db/utils" import { DB_EXPORT_FILE, GLOBAL_DB_EXPORT_FILE, diff --git a/packages/server/src/sdk/app/imports.ts b/packages/server/src/sdk/app/backups/imports.ts similarity index 62% rename from packages/server/src/sdk/app/imports.ts rename to packages/server/src/sdk/app/backups/imports.ts index 96629db39a..0ea7eeaa1b 100644 --- a/packages/server/src/sdk/app/imports.ts +++ b/packages/server/src/sdk/app/backups/imports.ts @@ -1,10 +1,12 @@ import { db as dbCore } from "@budibase/backend-core" -import { budibaseTempDir } from "../../utilities/budibaseDir" +import { budibaseTempDir } from "../../../utilities/budibaseDir" import { DB_EXPORT_FILE, ATTACHMENT_DIR } from "./constants" -import { uploadDirectory } from "../../utilities/fileSystem/utilities" -import { ObjectStoreBuckets } from "../../constants" +import { uploadDirectory } from "../../../utilities/fileSystem/utilities" +import { ObjectStoreBuckets, FieldTypes } from "../../../constants" import { join } from "path" import fs from "fs" +import sdk from "../../" +import { CouchFindOptions, Row } from "@budibase/types" const uuid = require("uuid/v4") const tar = require("tar") @@ -64,8 +66,29 @@ export async function importApp( const { ok } = await db.load(dbStream) if (!ok) { throw "Error loading database dump from template." - } else { - // TODO: need to iterate over attachments and update their URLs + } + // iterate through attachment documents and update them + const tables = await sdk.tables.getAllInternalTables(db) + for (let table of tables) { + const attachmentCols: string[] = [] + for (let [key, column] of Object.entries(table.schema)) { + if (column.type === FieldTypes.ATTACHMENT) { + attachmentCols.push(key) + } + } + // no attachment columns, nothing to do + if (attachmentCols.length === 0) { + continue + } + // use the CouchDB Mango query API to lookup rows that have attachments + const params: CouchFindOptions = { selector: {} } + attachmentCols.forEach(col => (params.selector[col] = { $exists: true })) + const { rows } = await dbCore.directCouchFind(db.name, params) + for (let row of rows) { + // TODO: + } + // write back the updated attachments + await db.bulkDocs(rows) } return ok } diff --git a/packages/server/src/sdk/app/backups/index.ts b/packages/server/src/sdk/app/backups/index.ts new file mode 100644 index 0000000000..fe7e4e1049 --- /dev/null +++ b/packages/server/src/sdk/app/backups/index.ts @@ -0,0 +1,7 @@ +import * as exportApps from "./exports" +import * as importApps from "./imports" + +export default { + ...exportApps, + ...importApps, +} diff --git a/packages/server/src/sdk/app/index.ts b/packages/server/src/sdk/app/index.ts deleted file mode 100644 index 5ac7c04b2c..0000000000 --- a/packages/server/src/sdk/app/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export * as exports from "./exports" -export * as imports from "./imports" diff --git a/packages/server/src/sdk/app/tables/index.ts b/packages/server/src/sdk/app/tables/index.ts new file mode 100644 index 0000000000..eeaf869055 --- /dev/null +++ b/packages/server/src/sdk/app/tables/index.ts @@ -0,0 +1,60 @@ +import { getAppDB } from "@budibase/backend-core/context" +import { BudibaseInternalDB, getTableParams } from "../../../db/utils" +import { + breakExternalTableId, + isExternalTable, + isSQL, +} from "../../../integrations/utils" +import { Table } from "@budibase/types" + +async function getAllInternalTables(db?: PouchDB.Database): Promise { + if (!db) { + db = getAppDB() as PouchDB.Database + } + const internalTables = await db.allDocs( + getTableParams(null, { + include_docs: true, + }) + ) + return internalTables.rows.map((tableDoc: any) => ({ + ...tableDoc.doc, + type: "internal", + sourceId: BudibaseInternalDB._id, + })) +} + +async function getAllExternalTables(datasourceId: any): Promise { + const db = getAppDB() + const datasource = await db.get(datasourceId) + if (!datasource || !datasource.entities) { + throw "Datasource is not configured fully." + } + return datasource.entities +} + +async function getExternalTable( + datasourceId: any, + tableName: any +): Promise { + const entities = await getAllExternalTables(datasourceId) + return entities[tableName] +} + +async function getTable(tableId: any): Promise
{ + const db = getAppDB() + if (isExternalTable(tableId)) { + let { datasourceId, tableName } = breakExternalTableId(tableId) + const datasource = await db.get(datasourceId) + const table = await getExternalTable(datasourceId, tableName) + return { ...table, sql: isSQL(datasource) } + } else { + return db.get(tableId) + } +} + +export default { + getAllInternalTables, + getAllExternalTables, + getExternalTable, + getTable, +} diff --git a/packages/server/src/sdk/index.ts b/packages/server/src/sdk/index.ts index 9199c1cc72..8cf9d21c41 100644 --- a/packages/server/src/sdk/index.ts +++ b/packages/server/src/sdk/index.ts @@ -1,5 +1,7 @@ -import * as apps from "./app" +import { default as backups } from "./app/backups" +import { default as tables } from "./app/tables" export default { - apps, + backups, + tables, } diff --git a/packages/types/package.json b/packages/types/package.json index 93271fb93e..0787093e46 100644 --- a/packages/types/package.json +++ b/packages/types/package.json @@ -16,6 +16,7 @@ "@types/koa": "2.13.4", "@types/node": "14.18.20", "rimraf": "3.0.2", - "typescript": "4.7.3" + "typescript": "4.7.3", + "@types/pouchdb": "6.4.0" } } diff --git a/packages/types/src/core/db.ts b/packages/types/src/core/db.ts new file mode 100644 index 0000000000..6d47a5c36b --- /dev/null +++ b/packages/types/src/core/db.ts @@ -0,0 +1,22 @@ +export type PouchOptions = { + inMemory: boolean + replication: boolean + onDisk: boolean + find: boolean +} + +export enum SortOption { + ASCENDING = "asc", + DESCENDING = "desc", +} + +export type CouchFindOptions = { + selector: PouchDB.Find.Selector + fields?: string[] + sort?: { + [key: string]: SortOption + }[] + limit?: number + skip?: number + bookmark?: string +} diff --git a/packages/types/src/core/index.ts b/packages/types/src/core/index.ts new file mode 100644 index 0000000000..9071393365 --- /dev/null +++ b/packages/types/src/core/index.ts @@ -0,0 +1 @@ +export * from "./db" diff --git a/packages/types/src/documents/app/table.ts b/packages/types/src/documents/app/table.ts index 72cff4f056..b98c562852 100644 --- a/packages/types/src/documents/app/table.ts +++ b/packages/types/src/documents/app/table.ts @@ -49,4 +49,5 @@ export interface Table extends Document { sourceId?: string relatedFormula?: string[] constrained?: string[] + sql?: boolean } diff --git a/packages/types/src/index.ts b/packages/types/src/index.ts index 4adb2fda97..92d2ceb050 100644 --- a/packages/types/src/index.ts +++ b/packages/types/src/index.ts @@ -1,3 +1,4 @@ export * from "./documents" export * from "./sdk" export * from "./api" +export * from "./core" diff --git a/packages/types/yarn.lock b/packages/types/yarn.lock index c80ff652ba..f225ffc442 100644 --- a/packages/types/yarn.lock +++ b/packages/types/yarn.lock @@ -39,6 +39,13 @@ "@types/keygrip" "*" "@types/node" "*" +"@types/debug@*": + version "4.1.7" + resolved "https://registry.yarnpkg.com/@types/debug/-/debug-4.1.7.tgz#7cc0ea761509124709b8b2d1090d8f6c17aadb82" + integrity sha512-9AonUzyTjXXhEOa0DnqpzZi6VHlqKMswga9EXjpXnnqxwLtdvPPtlO8evrI5D9S6asFRCQ6v+wpiUKbw+vKqyg== + dependencies: + "@types/ms" "*" + "@types/express-serve-static-core@^4.17.18": version "4.17.29" resolved "https://registry.yarnpkg.com/@types/express-serve-static-core/-/express-serve-static-core-4.17.29.tgz#2a1795ea8e9e9c91b4a4bbe475034b20c1ec711c" @@ -113,6 +120,11 @@ resolved "https://registry.yarnpkg.com/@types/mime/-/mime-1.3.2.tgz#93e25bf9ee75fe0fd80b594bc4feb0e862111b5a" integrity sha512-YATxVxgRqNH6nHEIsvg6k2Boc1JHI9ZbH5iWFFv/MTkchz3b1ieGDa5T0a9RznNdI0KhVbdbWSN+KWWrQZRxTw== +"@types/ms@*": + version "0.7.31" + resolved "https://registry.yarnpkg.com/@types/ms/-/ms-0.7.31.tgz#31b7ca6407128a3d2bbc27fe2d21b345397f6197" + integrity sha512-iiUgKzV9AuaEkZqkOLDIvlQiL6ltuZd9tGcW3gwpnX8JbuiuhFlEGmmFXEXkN50Cvq7Os88IY2v0dkDqXYWVgA== + "@types/node@*": version "18.0.6" resolved "https://registry.yarnpkg.com/@types/node/-/node-18.0.6.tgz#0ba49ac517ad69abe7a1508bc9b3a5483df9d5d7" @@ -123,6 +135,152 @@ resolved "https://registry.yarnpkg.com/@types/node/-/node-14.18.20.tgz#268f028b36eaf51181c3300252f605488c4f0650" integrity sha512-Q8KKwm9YqEmUBRsqJ2GWJDtXltBDxTdC4m5vTdXBolu2PeQh8LX+f6BTwU+OuXPu37fLxoN6gidqBmnky36FXA== +"@types/pouchdb-adapter-cordova-sqlite@*": + version "1.0.1" + resolved "https://registry.yarnpkg.com/@types/pouchdb-adapter-cordova-sqlite/-/pouchdb-adapter-cordova-sqlite-1.0.1.tgz#49e5ee6df7cc0c23196fcb340f43a560e74eb1d6" + integrity sha512-nqlXpW1ho3KBg1mUQvZgH2755y3z/rw4UA7ZJCPMRTHofxGMY8izRVw5rHBL4/7P615or0J2udpRYxgkT3D02g== + dependencies: + "@types/pouchdb-core" "*" + +"@types/pouchdb-adapter-fruitdown@*": + version "6.1.3" + resolved "https://registry.yarnpkg.com/@types/pouchdb-adapter-fruitdown/-/pouchdb-adapter-fruitdown-6.1.3.tgz#9b140ad9645cc56068728acf08ec19ac0046658e" + integrity sha512-Wz1Z1JLOW1hgmFQjqnSkmyyfH7by/iWb4abKn684WMvQfmxx6BxKJpJ4+eulkVPQzzgMMSgU1MpnQOm9FgRkbw== + dependencies: + "@types/pouchdb-core" "*" + +"@types/pouchdb-adapter-http@*": + version "6.1.3" + resolved "https://registry.yarnpkg.com/@types/pouchdb-adapter-http/-/pouchdb-adapter-http-6.1.3.tgz#6e592d5f48deb6274a21ddac1498dd308096bcf3" + integrity sha512-9Z4TLbF/KJWy/D2sWRPBA+RNU0odQimfdvlDX+EY7rGcd3aVoH8qjD/X0Xcd/0dfBH5pKrNIMFFQgW/TylRCmA== + dependencies: + "@types/pouchdb-core" "*" + +"@types/pouchdb-adapter-idb@*": + version "6.1.4" + resolved "https://registry.yarnpkg.com/@types/pouchdb-adapter-idb/-/pouchdb-adapter-idb-6.1.4.tgz#cb9a18864585d600820cd325f007614c5c3989cd" + integrity sha512-KIAXbkF4uYUz0ZwfNEFLtEkK44mEWopAsD76UhucH92XnJloBysav+TjI4FFfYQyTjoW3S1s6V+Z14CUJZ0F6w== + dependencies: + "@types/pouchdb-core" "*" + +"@types/pouchdb-adapter-leveldb@*": + version "6.1.3" + resolved "https://registry.yarnpkg.com/@types/pouchdb-adapter-leveldb/-/pouchdb-adapter-leveldb-6.1.3.tgz#17c7e75d75b992050bca15991e97fba575c61bb3" + integrity sha512-ex8NFqQGFwEpFi7AaZ5YofmuemfZNsL3nTFZBUCAKYMBkazQij1pe2ILLStSvJr0XS0qxgXjCEW19T5Wqiiskg== + dependencies: + "@types/pouchdb-core" "*" + +"@types/pouchdb-adapter-localstorage@*": + version "6.1.3" + resolved "https://registry.yarnpkg.com/@types/pouchdb-adapter-localstorage/-/pouchdb-adapter-localstorage-6.1.3.tgz#0dde02ba6b9d6073a295a20196563942ba9a54bd" + integrity sha512-oor040tye1KKiGLWYtIy7rRT7C2yoyX3Tf6elEJRpjOA7Ja/H8lKc4LaSh9ATbptIcES6MRqZDxtp7ly9hsW3Q== + dependencies: + "@types/pouchdb-core" "*" + +"@types/pouchdb-adapter-memory@*": + version "6.1.3" + resolved "https://registry.yarnpkg.com/@types/pouchdb-adapter-memory/-/pouchdb-adapter-memory-6.1.3.tgz#9eabdbc890fcf58960ee8b68b8685f837e75c844" + integrity sha512-gVbsIMzDzgZYThFVT4eVNsmuZwVm/4jDxP1sjlgc3qtDIxbtBhGgyNfcskwwz9Zu5Lv1avkDsIWvcxQhnvRlHg== + dependencies: + "@types/pouchdb-core" "*" + +"@types/pouchdb-adapter-node-websql@*": + version "6.1.3" + resolved "https://registry.yarnpkg.com/@types/pouchdb-adapter-node-websql/-/pouchdb-adapter-node-websql-6.1.3.tgz#aa18bc68af8cf509acd12c400010dcd5fab2243d" + integrity sha512-F/P+os6Jsa7CgHtH64+Z0HfwIcj0hIRB5z8gNhF7L7dxPWoAfkopK5H2gydrP3sQrlGyN4WInF+UJW/Zu1+FKg== + dependencies: + "@types/pouchdb-adapter-websql" "*" + "@types/pouchdb-core" "*" + +"@types/pouchdb-adapter-websql@*": + version "6.1.4" + resolved "https://registry.yarnpkg.com/@types/pouchdb-adapter-websql/-/pouchdb-adapter-websql-6.1.4.tgz#359fbe42ccac0ac90b492ddb8c32fafd0aa96d79" + integrity sha512-zMJQCtXC40hBsIDRn0GhmpeGMK0f9l/OGWfLguvczROzxxcOD7REI+e6SEmX7gJKw5JuMvlfuHzkQwjmvSJbtg== + dependencies: + "@types/pouchdb-core" "*" + +"@types/pouchdb-browser@*": + version "6.1.3" + resolved "https://registry.yarnpkg.com/@types/pouchdb-browser/-/pouchdb-browser-6.1.3.tgz#8f33d6ef58d6817d1f6d36979148a1c7f63244d8" + integrity sha512-EdYowrWxW9SWBMX/rux2eq7dbHi5Zeyzz+FF/IAsgQKnUxgeCO5VO2j4zTzos0SDyJvAQU+EYRc11r7xGn5tvA== + dependencies: + "@types/pouchdb-adapter-http" "*" + "@types/pouchdb-adapter-idb" "*" + "@types/pouchdb-adapter-websql" "*" + "@types/pouchdb-core" "*" + "@types/pouchdb-mapreduce" "*" + "@types/pouchdb-replication" "*" + +"@types/pouchdb-core@*": + version "7.0.10" + resolved "https://registry.yarnpkg.com/@types/pouchdb-core/-/pouchdb-core-7.0.10.tgz#d1ea1549e7fad6cb579f71459b1bc27252e06a5a" + integrity sha512-mKhjLlWWXyV3PTTjDhzDV1kc2dolO7VYFa75IoKM/hr8Er9eo8RIbS7mJLfC8r/C3p6ihZu9yZs1PWC1LQ0SOA== + dependencies: + "@types/debug" "*" + "@types/pouchdb-find" "*" + +"@types/pouchdb-find@*": + version "7.3.0" + resolved "https://registry.yarnpkg.com/@types/pouchdb-find/-/pouchdb-find-7.3.0.tgz#b917030e9f4bf6e56bf8c3b9fe4b2a25e989009a" + integrity sha512-sFPli5tBjGX9UfXioik1jUzPdcN84eV82n0lmEFuoPepWqkLjQcyri0eOa++HYOaNPyMDhKFBqEALEZivK2dRg== + dependencies: + "@types/pouchdb-core" "*" + +"@types/pouchdb-http@*": + version "6.1.3" + resolved "https://registry.yarnpkg.com/@types/pouchdb-http/-/pouchdb-http-6.1.3.tgz#09576c0d409da1f8dee34ec5b768415e2472ea52" + integrity sha512-0e9E5SqNOyPl/3FnEIbENssB4FlJsNYuOy131nxrZk36S+y1R/6qO7ZVRypWpGTqBWSuVd7gCsq2UDwO/285+w== + dependencies: + "@types/pouchdb-adapter-http" "*" + "@types/pouchdb-core" "*" + +"@types/pouchdb-mapreduce@*": + version "6.1.7" + resolved "https://registry.yarnpkg.com/@types/pouchdb-mapreduce/-/pouchdb-mapreduce-6.1.7.tgz#9ab32d1e0f234f1bf6d1e4c5d7e216e9e23ac0a3" + integrity sha512-WzBwm7tmO9QhfRzVaWT4v6JQSS/fG2OoUDrWrhX87rPe2Pn6laPvdK5li6myNRxCoI/l5e8Jd+oYBAFnaiFucA== + dependencies: + "@types/pouchdb-core" "*" + +"@types/pouchdb-node@*": + version "6.1.4" + resolved "https://registry.yarnpkg.com/@types/pouchdb-node/-/pouchdb-node-6.1.4.tgz#5214c0169fcfd2237d373380bbd65a934feb5dfb" + integrity sha512-wnTCH8X1JOPpNOfVhz8HW0AvmdHh6pt40MuRj0jQnK7QEHsHS79WujsKTKSOF8QXtPwpvCNSsI7ut7H7tfxxJQ== + dependencies: + "@types/pouchdb-adapter-http" "*" + "@types/pouchdb-adapter-leveldb" "*" + "@types/pouchdb-core" "*" + "@types/pouchdb-mapreduce" "*" + "@types/pouchdb-replication" "*" + +"@types/pouchdb-replication@*": + version "6.4.4" + resolved "https://registry.yarnpkg.com/@types/pouchdb-replication/-/pouchdb-replication-6.4.4.tgz#743406c90f13a988fa3e346ea74ce40acd170d00" + integrity sha512-BsE5LKpjJK4iAf6Fx5kyrMw+33V+Ip7uWldUnU2BYrrvtR+MLD22dcImm7DZN1st2wPPb91i0XEnQzvP0w1C/Q== + dependencies: + "@types/pouchdb-core" "*" + "@types/pouchdb-find" "*" + +"@types/pouchdb@6.4.0": + version "6.4.0" + resolved "https://registry.yarnpkg.com/@types/pouchdb/-/pouchdb-6.4.0.tgz#f9c41ca64b23029f9bf2eb4bf6956e6431cb79f8" + integrity sha512-eGCpX+NXhd5VLJuJMzwe3L79fa9+IDTrAG3CPaf4s/31PD56hOrhDJTSmRELSXuiqXr6+OHzzP0PldSaWsFt7w== + dependencies: + "@types/pouchdb-adapter-cordova-sqlite" "*" + "@types/pouchdb-adapter-fruitdown" "*" + "@types/pouchdb-adapter-http" "*" + "@types/pouchdb-adapter-idb" "*" + "@types/pouchdb-adapter-leveldb" "*" + "@types/pouchdb-adapter-localstorage" "*" + "@types/pouchdb-adapter-memory" "*" + "@types/pouchdb-adapter-node-websql" "*" + "@types/pouchdb-adapter-websql" "*" + "@types/pouchdb-browser" "*" + "@types/pouchdb-core" "*" + "@types/pouchdb-http" "*" + "@types/pouchdb-mapreduce" "*" + "@types/pouchdb-node" "*" + "@types/pouchdb-replication" "*" + "@types/qs@*": version "6.9.7" resolved "https://registry.yarnpkg.com/@types/qs/-/qs-6.9.7.tgz#63bb7d067db107cc1e457c303bc25d511febf6cb" From f5dd87f8f9d6c6077ec35000661a7a57f87bb3ed Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 12 Oct 2022 17:34:17 +0100 Subject: [PATCH 015/151] Adding first pass of attachments updating. --- packages/backend-core/src/db/index.ts | 4 +- .../server/src/sdk/app/backups/constants.ts | 2 +- .../server/src/sdk/app/backups/imports.ts | 84 +++++++++++++------ packages/types/src/documents/app/row.ts | 8 ++ 4 files changed, 71 insertions(+), 27 deletions(-) diff --git a/packages/backend-core/src/db/index.ts b/packages/backend-core/src/db/index.ts index 016155033a..2728b83682 100644 --- a/packages/backend-core/src/db/index.ts +++ b/packages/backend-core/src/db/index.ts @@ -32,7 +32,7 @@ const checkInitialised = () => { } } -export async function init(opts: PouchOptions) { +export async function init(opts?: PouchOptions) { PouchDB = pouch.getPouch(opts) initialised = true } @@ -40,7 +40,7 @@ export async function init(opts: PouchOptions) { // NOTE: THIS IS A DANGEROUS FUNCTION - USE WITH CAUTION // this function is prone to leaks, should only be used // in situations that using the function doWithDB does not work -export async function dangerousGetDB(dbName: string, opts: any) { +export async function dangerousGetDB(dbName: string, opts?: any) { checkInitialised() if (env.isTest()) { dbList.add(dbName) diff --git a/packages/server/src/sdk/app/backups/constants.ts b/packages/server/src/sdk/app/backups/constants.ts index e831172683..f022168846 100644 --- a/packages/server/src/sdk/app/backups/constants.ts +++ b/packages/server/src/sdk/app/backups/constants.ts @@ -1,4 +1,4 @@ -import { ATTACHMENT_DIR as attachmentDir } from "../../constants" +import { ATTACHMENT_DIR as attachmentDir } from "../../../constants" export const DB_EXPORT_FILE = "db.txt" export const ATTACHMENT_DIR = attachmentDir export const GLOBAL_DB_EXPORT_FILE = "global.txt" diff --git a/packages/server/src/sdk/app/backups/imports.ts b/packages/server/src/sdk/app/backups/imports.ts index 0ea7eeaa1b..297274e26b 100644 --- a/packages/server/src/sdk/app/backups/imports.ts +++ b/packages/server/src/sdk/app/backups/imports.ts @@ -1,4 +1,5 @@ import { db as dbCore } from "@budibase/backend-core" +import { TABLE_ROW_PREFIX } from "../../../db/utils" import { budibaseTempDir } from "../../../utilities/budibaseDir" import { DB_EXPORT_FILE, ATTACHMENT_DIR } from "./constants" import { uploadDirectory } from "../../../utilities/fileSystem/utilities" @@ -6,7 +7,7 @@ import { ObjectStoreBuckets, FieldTypes } from "../../../constants" import { join } from "path" import fs from "fs" import sdk from "../../" -import { CouchFindOptions, Row } from "@budibase/types" +import { CouchFindOptions, RowAttachment } from "@budibase/types" const uuid = require("uuid/v4") const tar = require("tar") @@ -18,6 +19,63 @@ type TemplateType = { key?: string } +async function updateAttachmentColumns( + prodAppId: string, + db: PouchDB.Database +) { + // iterate through attachment documents and update them + const tables = await sdk.tables.getAllInternalTables(db) + for (let table of tables) { + const attachmentCols: string[] = [] + for (let [key, column] of Object.entries(table.schema)) { + if (column.type === FieldTypes.ATTACHMENT) { + attachmentCols.push(key) + } + } + // no attachment columns, nothing to do + if (attachmentCols.length === 0) { + continue + } + // use the CouchDB Mango query API to lookup rows that have attachments + const params: CouchFindOptions = { + selector: { + _id: { + $regex: `^${TABLE_ROW_PREFIX}`, + }, + }, + } + attachmentCols.forEach(col => (params.selector[col] = { $exists: true })) + const { rows } = await dbCore.directCouchFind(db.name, params) + for (let row of rows) { + for (let column of attachmentCols) { + if (!Array.isArray(row[column])) { + continue + } + row[column] = row[column].map((attachment: RowAttachment) => { + // URL looks like: /prod-budi-app-assets/appId/attachments/file.csv + const urlParts = attachment.url.split("/") + // drop the first empty element + urlParts.shift() + // get the prefix + const prefix = urlParts.shift() + // remove the app ID + urlParts.shift() + // add new app ID + urlParts.unshift(prodAppId) + const key = urlParts.join("/") + return { + ...attachment, + key, + url: `/${prefix}/${key}`, + } + }) + } + } + // write back the updated attachments + await db.bulkDocs(rows) + } +} + /** * This function manages temporary template files which are stored by Koa. * @param {Object} template The template object retrieved from the Koa context object. @@ -67,28 +125,6 @@ export async function importApp( if (!ok) { throw "Error loading database dump from template." } - // iterate through attachment documents and update them - const tables = await sdk.tables.getAllInternalTables(db) - for (let table of tables) { - const attachmentCols: string[] = [] - for (let [key, column] of Object.entries(table.schema)) { - if (column.type === FieldTypes.ATTACHMENT) { - attachmentCols.push(key) - } - } - // no attachment columns, nothing to do - if (attachmentCols.length === 0) { - continue - } - // use the CouchDB Mango query API to lookup rows that have attachments - const params: CouchFindOptions = { selector: {} } - attachmentCols.forEach(col => (params.selector[col] = { $exists: true })) - const { rows } = await dbCore.directCouchFind(db.name, params) - for (let row of rows) { - // TODO: - } - // write back the updated attachments - await db.bulkDocs(rows) - } + await updateAttachmentColumns(prodAppId, db) return ok } diff --git a/packages/types/src/documents/app/row.ts b/packages/types/src/documents/app/row.ts index ee5c0231e7..2cac32279b 100644 --- a/packages/types/src/documents/app/row.ts +++ b/packages/types/src/documents/app/row.ts @@ -16,6 +16,14 @@ export enum FieldType { INTERNAL = "internal", } +export interface RowAttachment { + size: number + name: string + url: string + extension: string + key: string +} + export interface Row extends Document { type?: string tableId?: string From 23f95b8be2d67cb8b20e1ab9826820f26120614b Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Wed, 12 Oct 2022 17:34:55 +0100 Subject: [PATCH 016/151] making bucketnames configurable in helm chart --- charts/budibase/templates/app-service-deployment.yaml | 4 ++++ charts/budibase/templates/worker-service-deployment.yaml | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/charts/budibase/templates/app-service-deployment.yaml b/charts/budibase/templates/app-service-deployment.yaml index f72d1aef03..25a5dd410a 100644 --- a/charts/budibase/templates/app-service-deployment.yaml +++ b/charts/budibase/templates/app-service-deployment.yaml @@ -80,6 +80,10 @@ spec: value: {{ .Values.services.objectStore.url }} - name: PLUGIN_BUCKET_NAME value: {{ .Values.services.objectStore.pluginBucketName | default "plugins" | quote }} + - name: APPS_BUCKET_NAME + value: {{ .Values.services.objectStore.appsBucketName | default "apps" | quote }} + - name: GLOBAL_CLOUD_BUCKET_NAME + value: {{ .Values.services.objectStore.globalCloudBucketName | default "global" | quote }} - name: PORT value: {{ .Values.services.apps.port | quote }} {{ if .Values.services.worker.publicApiRateLimitPerSecond }} diff --git a/charts/budibase/templates/worker-service-deployment.yaml b/charts/budibase/templates/worker-service-deployment.yaml index b1c6110d95..1284b9c411 100644 --- a/charts/budibase/templates/worker-service-deployment.yaml +++ b/charts/budibase/templates/worker-service-deployment.yaml @@ -79,6 +79,10 @@ spec: value: {{ .Values.services.objectStore.url }} - name: PLUGIN_BUCKET_NAME value: {{ .Values.services.objectStore.pluginBucketName | default "plugins" | quote }} + - name: APPS_BUCKET_NAME + value: {{ .Values.services.objectStore.appsBucketName | default "apps" | quote }} + - name: GLOBAL_CLOUD_BUCKET_NAME + value: {{ .Values.services.objectStore.globalCloudBucketName | default "global" | quote }} - name: PORT value: {{ .Values.services.worker.port | quote }} - name: MULTI_TENANCY From 4da37058511480aa92d8e81df59bb7fc53af9f46 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 12 Oct 2022 17:37:52 +0100 Subject: [PATCH 017/151] Quick fixes to DB TS conversion. --- packages/backend-core/src/db/index.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/backend-core/src/db/index.ts b/packages/backend-core/src/db/index.ts index 2728b83682..7a2337359a 100644 --- a/packages/backend-core/src/db/index.ts +++ b/packages/backend-core/src/db/index.ts @@ -40,7 +40,7 @@ export async function init(opts?: PouchOptions) { // NOTE: THIS IS A DANGEROUS FUNCTION - USE WITH CAUTION // this function is prone to leaks, should only be used // in situations that using the function doWithDB does not work -export async function dangerousGetDB(dbName: string, opts?: any) { +export function dangerousGetDB(dbName: string, opts?: any) { checkInitialised() if (env.isTest()) { dbList.add(dbName) @@ -75,13 +75,13 @@ export async function closeDB(db: PouchDB.Database) { // the DB when we're done, without this manual requests would // need to close the database when done with it to avoid memory leaks export async function doWithDB(dbName: string, cb: any, opts = {}) { - const db = exports.dangerousGetDB(dbName, opts) + const db = dangerousGetDB(dbName, opts) // need this to be async so that we can correctly close DB after all // async operations have been completed try { return await cb(db) } finally { - await exports.closeDB(db) + await closeDB(db) } } From 2eae3f2a6c954eb49053b60615633f6ec7dfac9c Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 12 Oct 2022 17:57:31 +0100 Subject: [PATCH 018/151] Fixes for find functionality after testing. --- packages/backend-core/src/db/index.ts | 11 ++++++++++- packages/backend-core/src/db/utils.ts | 17 ++++++++--------- packages/server/src/sdk/index.ts | 8 +++++++- 3 files changed, 25 insertions(+), 11 deletions(-) diff --git a/packages/backend-core/src/db/index.ts b/packages/backend-core/src/db/index.ts index 7a2337359a..8374aecd8d 100644 --- a/packages/backend-core/src/db/index.ts +++ b/packages/backend-core/src/db/index.ts @@ -107,7 +107,8 @@ export async function directCouchQuery( }, } if (body && method !== "GET") { - params.body = body + params.body = JSON.stringify(body) + params.headers["Content-Type"] = "application/json" } const response = await fetch(checkSlashesInUrl(encodeURI(couchUrl)), params) if (response.status < 300) { @@ -117,6 +118,14 @@ export async function directCouchQuery( } } +export async function directCouchAllDbs(queryString?: string) { + let couchPath = "/_all_dbs" + if (queryString) { + couchPath += `?${queryString}` + } + return await directCouchQuery(couchPath) +} + export async function directCouchFind(dbName: string, opts: CouchFindOptions) { const json = await directCouchQuery(`${dbName}/_find`, "POST", opts) return { rows: json.docs, bookmark: json.bookmark } diff --git a/packages/backend-core/src/db/utils.ts b/packages/backend-core/src/db/utils.ts index c78c11217e..8d824d60bb 100644 --- a/packages/backend-core/src/db/utils.ts +++ b/packages/backend-core/src/db/utils.ts @@ -4,7 +4,7 @@ import env from "../environment" import { SEPARATOR, DocumentType, UNICODE_MAX, ViewName } from "./constants" import { getTenantId, getGlobalDB } from "../context" import { getGlobalDBName } from "./tenancy" -import { doWithDB, allDbs, directCouchQuery } from "./index" +import { doWithDB, allDbs, directCouchQuery, directCouchAllDbs } from "./index" import { getAppMetadata } from "../cache/appMetadata" import { isDevApp, isDevAppID, getProdAppID } from "./conversions" import { APP_PREFIX } from "./constants" @@ -188,9 +188,9 @@ export function getRoleParams(roleId = null, otherProps = {}) { return getDocParams(DocumentType.ROLE, roleId, otherProps) } -export function getStartEndKeyURL(base: any, baseKey: any, tenantId = null) { +export function getStartEndKeyURL(baseKey: any, tenantId = null) { const tenancy = tenantId ? `${SEPARATOR}${tenantId}` : "" - return `${base}?startkey="${baseKey}${tenancy}"&endkey="${baseKey}${tenancy}${UNICODE_MAX}"` + return `startkey="${baseKey}${tenancy}"&endkey="${baseKey}${tenancy}${UNICODE_MAX}"` } /** @@ -206,11 +206,10 @@ export async function getAllDbs(opts = { efficient: false }) { return allDbs() } let dbs: any[] = [] - async function addDbs(couchPath: string) { - const json = await directCouchQuery(couchPath) + async function addDbs(queryString?: string) { + const json = await directCouchAllDbs(queryString) dbs = dbs.concat(json) } - let couchPath = "/_all_dbs" let tenantId = getTenantId() if (!env.MULTI_TENANCY || (!efficient && tenantId === DEFAULT_TENANT_ID)) { // just get all DBs when: @@ -218,12 +217,12 @@ export async function getAllDbs(opts = { efficient: false }) { // - default tenant // - apps dbs don't contain tenant id // - non-default tenant dbs are filtered out application side in getAllApps - await addDbs(couchPath) + await addDbs() } else { // get prod apps - await addDbs(getStartEndKeyURL(couchPath, DocumentType.APP, tenantId)) + await addDbs(getStartEndKeyURL(DocumentType.APP, tenantId)) // get dev apps - await addDbs(getStartEndKeyURL(couchPath, DocumentType.APP_DEV, tenantId)) + await addDbs(getStartEndKeyURL(DocumentType.APP_DEV, tenantId)) // add global db name dbs.push(getGlobalDBName(tenantId)) } diff --git a/packages/server/src/sdk/index.ts b/packages/server/src/sdk/index.ts index 8cf9d21c41..85c01cdb44 100644 --- a/packages/server/src/sdk/index.ts +++ b/packages/server/src/sdk/index.ts @@ -1,7 +1,13 @@ import { default as backups } from "./app/backups" import { default as tables } from "./app/tables" -export default { +const toExport = { backups, tables, } + +// default export for TS +export default toExport + +// default export for JS +module.exports = toExport From bd52533f017d844bc662ec9150df8d71b6e0377b Mon Sep 17 00:00:00 2001 From: Budibase Release Bot <> Date: Wed, 12 Oct 2022 16:58:49 +0000 Subject: [PATCH 019/151] v2.0.30 --- lerna.json | 2 +- packages/backend-core/package.json | 4 ++-- packages/bbui/package.json | 4 ++-- packages/builder/package.json | 10 +++++----- packages/cli/package.json | 8 ++++---- packages/client/package.json | 8 ++++---- packages/frontend-core/package.json | 4 ++-- packages/sdk/package.json | 2 +- packages/server/package.json | 10 +++++----- packages/string-templates/package.json | 2 +- packages/types/package.json | 2 +- packages/worker/package.json | 8 ++++---- 12 files changed, 32 insertions(+), 32 deletions(-) diff --git a/lerna.json b/lerna.json index f5ecd09c34..96c2575602 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.0.29", + "version": "2.0.30", "npmClient": "yarn", "packages": [ "packages/*" diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json index 05606920a3..888483b56e 100644 --- a/packages/backend-core/package.json +++ b/packages/backend-core/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/backend-core", - "version": "2.0.29", + "version": "2.0.30", "description": "Budibase backend core libraries used in server and worker", "main": "dist/src/index.js", "types": "dist/src/index.d.ts", @@ -20,7 +20,7 @@ "test:watch": "jest --watchAll" }, "dependencies": { - "@budibase/types": "^2.0.29", + "@budibase/types": "^2.0.30", "@shopify/jest-koa-mocks": "5.0.1", "@techpass/passport-openidconnect": "0.3.2", "aws-sdk": "2.1030.0", diff --git a/packages/bbui/package.json b/packages/bbui/package.json index d453fcf3b1..9472f84bb4 100644 --- a/packages/bbui/package.json +++ b/packages/bbui/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/bbui", "description": "A UI solution used in the different Budibase projects.", - "version": "2.0.29", + "version": "2.0.30", "license": "MPL-2.0", "svelte": "src/index.js", "module": "dist/bbui.es.js", @@ -38,7 +38,7 @@ ], "dependencies": { "@adobe/spectrum-css-workflow-icons": "^1.2.1", - "@budibase/string-templates": "^2.0.29", + "@budibase/string-templates": "^2.0.30", "@spectrum-css/actionbutton": "^1.0.1", "@spectrum-css/actiongroup": "^1.0.1", "@spectrum-css/avatar": "^3.0.2", diff --git a/packages/builder/package.json b/packages/builder/package.json index 7fcfa6221f..25d833c378 100644 --- a/packages/builder/package.json +++ b/packages/builder/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/builder", - "version": "2.0.29", + "version": "2.0.30", "license": "GPL-3.0", "private": true, "scripts": { @@ -71,10 +71,10 @@ } }, "dependencies": { - "@budibase/bbui": "^2.0.29", - "@budibase/client": "^2.0.29", - "@budibase/frontend-core": "^2.0.29", - "@budibase/string-templates": "^2.0.29", + "@budibase/bbui": "^2.0.30", + "@budibase/client": "^2.0.30", + "@budibase/frontend-core": "^2.0.30", + "@budibase/string-templates": "^2.0.30", "@sentry/browser": "5.19.1", "@spectrum-css/page": "^3.0.1", "@spectrum-css/vars": "^3.0.1", diff --git a/packages/cli/package.json b/packages/cli/package.json index 0cf8656079..58e6aea835 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/cli", - "version": "2.0.29", + "version": "2.0.30", "description": "Budibase CLI, for developers, self hosting and migrations.", "main": "src/index.js", "bin": { @@ -26,9 +26,9 @@ "outputPath": "build" }, "dependencies": { - "@budibase/backend-core": "^2.0.29", - "@budibase/string-templates": "^2.0.29", - "@budibase/types": "^2.0.29", + "@budibase/backend-core": "^2.0.30", + "@budibase/string-templates": "^2.0.30", + "@budibase/types": "^2.0.30", "axios": "0.21.2", "chalk": "4.1.0", "cli-progress": "3.11.2", diff --git a/packages/client/package.json b/packages/client/package.json index 367c8b2a87..f01e9f4ea5 100644 --- a/packages/client/package.json +++ b/packages/client/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/client", - "version": "2.0.29", + "version": "2.0.30", "license": "MPL-2.0", "module": "dist/budibase-client.js", "main": "dist/budibase-client.js", @@ -19,9 +19,9 @@ "dev:builder": "rollup -cw" }, "dependencies": { - "@budibase/bbui": "^2.0.29", - "@budibase/frontend-core": "^2.0.29", - "@budibase/string-templates": "^2.0.29", + "@budibase/bbui": "^2.0.30", + "@budibase/frontend-core": "^2.0.30", + "@budibase/string-templates": "^2.0.30", "@spectrum-css/button": "^3.0.3", "@spectrum-css/card": "^3.0.3", "@spectrum-css/divider": "^1.0.3", diff --git a/packages/frontend-core/package.json b/packages/frontend-core/package.json index daafebee22..ace3316711 100644 --- a/packages/frontend-core/package.json +++ b/packages/frontend-core/package.json @@ -1,12 +1,12 @@ { "name": "@budibase/frontend-core", - "version": "2.0.29", + "version": "2.0.30", "description": "Budibase frontend core libraries used in builder and client", "author": "Budibase", "license": "MPL-2.0", "svelte": "src/index.js", "dependencies": { - "@budibase/bbui": "^2.0.29", + "@budibase/bbui": "^2.0.30", "lodash": "^4.17.21", "svelte": "^3.46.2" } diff --git a/packages/sdk/package.json b/packages/sdk/package.json index fbe5b7b637..438bcf8fb0 100644 --- a/packages/sdk/package.json +++ b/packages/sdk/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/sdk", - "version": "2.0.29", + "version": "2.0.30", "description": "Budibase Public API SDK", "author": "Budibase", "license": "MPL-2.0", diff --git a/packages/server/package.json b/packages/server/package.json index f387181441..5fd6803ea0 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/server", "email": "hi@budibase.com", - "version": "2.0.29", + "version": "2.0.30", "description": "Budibase Web Server", "main": "src/index.ts", "repository": { @@ -77,11 +77,11 @@ "license": "GPL-3.0", "dependencies": { "@apidevtools/swagger-parser": "10.0.3", - "@budibase/backend-core": "^2.0.29", - "@budibase/client": "^2.0.29", + "@budibase/backend-core": "^2.0.30", + "@budibase/client": "^2.0.30", "@budibase/pro": "2.0.29", - "@budibase/string-templates": "^2.0.29", - "@budibase/types": "^2.0.29", + "@budibase/string-templates": "^2.0.30", + "@budibase/types": "^2.0.30", "@bull-board/api": "3.7.0", "@bull-board/koa": "3.9.4", "@elastic/elasticsearch": "7.10.0", diff --git a/packages/string-templates/package.json b/packages/string-templates/package.json index 28da60a3e3..64b3e52bf7 100644 --- a/packages/string-templates/package.json +++ b/packages/string-templates/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/string-templates", - "version": "2.0.29", + "version": "2.0.30", "description": "Handlebars wrapper for Budibase templating.", "main": "src/index.cjs", "module": "dist/bundle.mjs", diff --git a/packages/types/package.json b/packages/types/package.json index d6f467ef88..209294dbee 100644 --- a/packages/types/package.json +++ b/packages/types/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/types", - "version": "2.0.29", + "version": "2.0.30", "description": "Budibase types", "main": "dist/index.js", "types": "dist/index.d.ts", diff --git a/packages/worker/package.json b/packages/worker/package.json index 64910f3708..a410b8dd84 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/worker", "email": "hi@budibase.com", - "version": "2.0.29", + "version": "2.0.30", "description": "Budibase background service", "main": "src/index.ts", "repository": { @@ -36,10 +36,10 @@ "author": "Budibase", "license": "GPL-3.0", "dependencies": { - "@budibase/backend-core": "^2.0.29", + "@budibase/backend-core": "^2.0.30", "@budibase/pro": "2.0.29", - "@budibase/string-templates": "^2.0.29", - "@budibase/types": "^2.0.29", + "@budibase/string-templates": "^2.0.30", + "@budibase/types": "^2.0.30", "@koa/router": "8.0.8", "@sentry/node": "6.17.7", "@techpass/passport-openidconnect": "0.3.2", From 847fcbc41e61fca4f168df0de9cca072664dacf1 Mon Sep 17 00:00:00 2001 From: Budibase Release Bot <> Date: Wed, 12 Oct 2022 17:02:51 +0000 Subject: [PATCH 020/151] Update pro version to 2.0.30 --- packages/server/package.json | 2 +- packages/server/yarn.lock | 30 +++++++++++++++--------------- packages/worker/package.json | 2 +- packages/worker/yarn.lock | 30 +++++++++++++++--------------- 4 files changed, 32 insertions(+), 32 deletions(-) diff --git a/packages/server/package.json b/packages/server/package.json index 5fd6803ea0..e2242dd1ef 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -79,7 +79,7 @@ "@apidevtools/swagger-parser": "10.0.3", "@budibase/backend-core": "^2.0.30", "@budibase/client": "^2.0.30", - "@budibase/pro": "2.0.29", + "@budibase/pro": "2.0.30", "@budibase/string-templates": "^2.0.30", "@budibase/types": "^2.0.30", "@bull-board/api": "3.7.0", diff --git a/packages/server/yarn.lock b/packages/server/yarn.lock index 97f550267c..35555f7f27 100644 --- a/packages/server/yarn.lock +++ b/packages/server/yarn.lock @@ -1094,12 +1094,12 @@ resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw== -"@budibase/backend-core@2.0.29": - version "2.0.29" - resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-2.0.29.tgz#d5856d49d8cc64790961631dfe0fface7f7be4e4" - integrity sha512-05mnl6YcucWrO1X6bVBYG6r7Yig/fIHbokLRfEvFFrZNe/EcRB3iLeOG1+2190dv5TbO/jhabS3kcrbDs54HHw== +"@budibase/backend-core@2.0.30": + version "2.0.30" + resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-2.0.30.tgz#5cc53e8560572d789ae687ec2181b629526c2f3f" + integrity sha512-+X4DwtMf1Xv4ygG7OJN0wOxZgjJ1wul1gUVOXJm44sRIEOx0LHpyx8xT4npC9e2VKKezpF5GZFE2IrB9lwWsoQ== dependencies: - "@budibase/types" "^2.0.29" + "@budibase/types" "^2.0.30" "@shopify/jest-koa-mocks" "5.0.1" "@techpass/passport-openidconnect" "0.3.2" aws-sdk "2.1030.0" @@ -1180,13 +1180,13 @@ svelte-flatpickr "^3.2.3" svelte-portal "^1.0.0" -"@budibase/pro@2.0.29": - version "2.0.29" - resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.0.29.tgz#169055bc39894f90341226fbff4a1601418d0b42" - integrity sha512-ELBoQ7/MXlgatCJNvTNXgF7DK02pfYx5Yy1s/2BJr4iGe26+5Q65ztiC7Jp+d/owese+f5kqKJRNuU1KINUfjQ== +"@budibase/pro@2.0.30": + version "2.0.30" + resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.0.30.tgz#4c739ed97e04d6dfc875dfb550262ef7085dc9d4" + integrity sha512-unel/xYLpeWUI+I0vstITH9UTVkjKS+SS0Ce4jCQlpMyOSB8zCie/sO6Ie2TakrI+vnuYuypnRT8fqlbgtrNyA== dependencies: - "@budibase/backend-core" "2.0.29" - "@budibase/types" "2.0.29" + "@budibase/backend-core" "2.0.30" + "@budibase/types" "2.0.30" "@koa/router" "8.0.8" joi "17.6.0" node-fetch "^2.6.1" @@ -1209,10 +1209,10 @@ svelte-apexcharts "^1.0.2" svelte-flatpickr "^3.1.0" -"@budibase/types@2.0.29", "@budibase/types@^2.0.29": - version "2.0.29" - resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.0.29.tgz#8b27f695aded7ad7523c4943deb556eadfb66c3c" - integrity sha512-wwpHgDwKff2UhNmKAdrzIxmDQ/crY77AZdFyWNpPvrHYIetyh2Kp5ikEKyZlYHTEpS2IPDE8EKn4coDeu+mGlQ== +"@budibase/types@2.0.30", "@budibase/types@^2.0.30": + version "2.0.30" + resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.0.30.tgz#ac3d2d714ebf2820875150739959e5f6b851aceb" + integrity sha512-46uCHPcLAAjAFRkE53skS7oLjI/Fa+2Mn4vZjIApa1/0JXyR2HnzTzJocbrQfYFTYLkdniMfLRticcKC/E2X1w== "@bull-board/api@3.7.0": version "3.7.0" diff --git a/packages/worker/package.json b/packages/worker/package.json index a410b8dd84..6723ba89e7 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -37,7 +37,7 @@ "license": "GPL-3.0", "dependencies": { "@budibase/backend-core": "^2.0.30", - "@budibase/pro": "2.0.29", + "@budibase/pro": "2.0.30", "@budibase/string-templates": "^2.0.30", "@budibase/types": "^2.0.30", "@koa/router": "8.0.8", diff --git a/packages/worker/yarn.lock b/packages/worker/yarn.lock index 3ecc2a4f02..b9bfaed432 100644 --- a/packages/worker/yarn.lock +++ b/packages/worker/yarn.lock @@ -291,12 +291,12 @@ resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw== -"@budibase/backend-core@2.0.29": - version "2.0.29" - resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-2.0.29.tgz#d5856d49d8cc64790961631dfe0fface7f7be4e4" - integrity sha512-05mnl6YcucWrO1X6bVBYG6r7Yig/fIHbokLRfEvFFrZNe/EcRB3iLeOG1+2190dv5TbO/jhabS3kcrbDs54HHw== +"@budibase/backend-core@2.0.30": + version "2.0.30" + resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-2.0.30.tgz#5cc53e8560572d789ae687ec2181b629526c2f3f" + integrity sha512-+X4DwtMf1Xv4ygG7OJN0wOxZgjJ1wul1gUVOXJm44sRIEOx0LHpyx8xT4npC9e2VKKezpF5GZFE2IrB9lwWsoQ== dependencies: - "@budibase/types" "^2.0.29" + "@budibase/types" "^2.0.30" "@shopify/jest-koa-mocks" "5.0.1" "@techpass/passport-openidconnect" "0.3.2" aws-sdk "2.1030.0" @@ -327,21 +327,21 @@ uuid "8.3.2" zlib "1.0.5" -"@budibase/pro@2.0.29": - version "2.0.29" - resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.0.29.tgz#169055bc39894f90341226fbff4a1601418d0b42" - integrity sha512-ELBoQ7/MXlgatCJNvTNXgF7DK02pfYx5Yy1s/2BJr4iGe26+5Q65ztiC7Jp+d/owese+f5kqKJRNuU1KINUfjQ== +"@budibase/pro@2.0.30": + version "2.0.30" + resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.0.30.tgz#4c739ed97e04d6dfc875dfb550262ef7085dc9d4" + integrity sha512-unel/xYLpeWUI+I0vstITH9UTVkjKS+SS0Ce4jCQlpMyOSB8zCie/sO6Ie2TakrI+vnuYuypnRT8fqlbgtrNyA== dependencies: - "@budibase/backend-core" "2.0.29" - "@budibase/types" "2.0.29" + "@budibase/backend-core" "2.0.30" + "@budibase/types" "2.0.30" "@koa/router" "8.0.8" joi "17.6.0" node-fetch "^2.6.1" -"@budibase/types@2.0.29", "@budibase/types@^2.0.29": - version "2.0.29" - resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.0.29.tgz#8b27f695aded7ad7523c4943deb556eadfb66c3c" - integrity sha512-wwpHgDwKff2UhNmKAdrzIxmDQ/crY77AZdFyWNpPvrHYIetyh2Kp5ikEKyZlYHTEpS2IPDE8EKn4coDeu+mGlQ== +"@budibase/types@2.0.30", "@budibase/types@^2.0.30": + version "2.0.30" + resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.0.30.tgz#ac3d2d714ebf2820875150739959e5f6b851aceb" + integrity sha512-46uCHPcLAAjAFRkE53skS7oLjI/Fa+2Mn4vZjIApa1/0JXyR2HnzTzJocbrQfYFTYLkdniMfLRticcKC/E2X1w== "@cspotcode/source-map-consumer@0.8.0": version "0.8.0" From d1c9a56e9a01cad3bae1c139a3cb29bb409ef2b2 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 12 Oct 2022 19:15:28 +0100 Subject: [PATCH 021/151] Full import implementation - needs further testing, untars the file with all apps, then adds each of them individually. --- .../pages/builder/portal/apps/index.svelte | 3 +- packages/server/src/api/controllers/cloud.js | 123 +++++++++++------- .../server/src/sdk/app/backups/exports.ts | 17 ++- .../server/src/sdk/app/backups/imports.ts | 36 +++-- 4 files changed, 112 insertions(+), 67 deletions(-) diff --git a/packages/builder/src/pages/builder/portal/apps/index.svelte b/packages/builder/src/pages/builder/portal/apps/index.svelte index a2fa954389..0b760c4b4a 100644 --- a/packages/builder/src/pages/builder/portal/apps/index.svelte +++ b/packages/builder/src/pages/builder/portal/apps/index.svelte @@ -21,7 +21,6 @@ import { API } from "api" import { onMount } from "svelte" import { apps, auth, admin, templates, licensing } from "stores/portal" - import download from "downloadjs" import { goto } from "@roxi/routify" import AppRow from "components/start/AppRow.svelte" import { AppStatus } from "constants" @@ -140,7 +139,7 @@ const initiateAppsExport = () => { try { - download(`/api/cloud/export`) + window.location = `/api/cloud/export` notifications.success("Apps exported successfully") } catch (err) { notifications.error(`Error exporting apps: ${err}`) diff --git a/packages/server/src/api/controllers/cloud.js b/packages/server/src/api/controllers/cloud.js index 323c7409db..9397bc69a6 100644 --- a/packages/server/src/api/controllers/cloud.js +++ b/packages/server/src/api/controllers/cloud.js @@ -1,9 +1,45 @@ const env = require("../../environment") const { getAllApps, getGlobalDBName } = require("@budibase/backend-core/db") +const { getGlobalDB } = require("@budibase/backend-core/tenancy") const { streamFile } = require("../../utilities/fileSystem") -const { DocumentType, isDevAppID } = require("../../db/utils") +const { stringToReadStream } = require("../../utilities") +const { + getDocParams, + DocumentType, + isDevAppID, + APP_PREFIX, +} = require("../../db/utils") +const { create } = require("./application") +const { join } = require("path") +const fs = require("fs") const sdk = require("../../sdk") +async function createApp(appName, appDirectory) { + const ctx = { + request: { + body: { + useTemplate: true, + name: appName, + }, + files: { + templateFile: { + path: appDirectory, + }, + }, + }, + } + return create(ctx) +} + +async function getAllDocType(db, docType) { + const response = await db.allDocs( + getDocParams(docType, null, { + include_docs: true, + }) + ) + return response.rows.map(row => row.doc) +} + exports.exportApps = async ctx => { if (env.SELF_HOSTED || !env.MULTI_TENANCY) { ctx.throw(400, "Exporting only allowed in multi-tenant cloud environments.") @@ -14,10 +50,13 @@ exports.exportApps = async ctx => { }) // only export the dev apps as they will be the latest, the user can republish the apps // in their self-hosted environment - let appIds = apps - .map(app => app.appId || app._id) - .filter(appId => isDevAppID(appId)) - const tmpPath = await sdk.backups.exportMultipleApps(appIds, globalDBString) + let appMetadata = apps + .filter(app => isDevAppID(app.appId || app._id)) + .map(app => ({ appId: app.appId || app._id, name: app.name })) + const tmpPath = await sdk.backups.exportMultipleApps( + appMetadata, + globalDBString + ) const filename = `cloud-export-${new Date().getTime()}.tar.gz` ctx.attachment(filename) ctx.body = streamFile(tmpPath) @@ -48,51 +87,37 @@ exports.importApps = async ctx => { "Import file is required and environment must be fresh to import apps." ) } + if (ctx.request.files.importFile.type !== "application/gzip") { + ctx.throw(400, "Import file must be a gzipped tarball.") + } - // TODO: IMPLEMENT TARBALL EXTRACTION, APP IMPORT, ATTACHMENT IMPORT AND GLOBAL DB IMPORT - // async function getAllDocType(db, docType) { - // const response = await db.allDocs( - // getDocParams(docType, null, { - // include_docs: true, - // }) - // ) - // return response.rows.map(row => row.doc) - // } - // async function createApp(appName, appImport) { - // const ctx = { - // request: { - // body: { - // templateString: appImport, - // name: appName, - // }, - // }, - // } - // return create(ctx) - // } - // const importFile = ctx.request.files.importFile - // const importString = readFileSync(importFile.path) - // const dbs = JSON.parse(importString) - // const globalDbImport = dbs.global - // // remove from the list of apps - // delete dbs.global - // const globalDb = getGlobalDB() - // // load the global db first - // await globalDb.load(stringToReadStream(globalDbImport)) - // for (let [appName, appImport] of Object.entries(dbs)) { - // await createApp(appName, appImport) - // } - // - // // if there are any users make sure to remove them - // let users = await getAllDocType(globalDb, DocumentType.USER) - // let userDeletionPromises = [] - // for (let user of users) { - // userDeletionPromises.push(globalDb.remove(user._id, user._rev)) - // } - // if (userDeletionPromises.length > 0) { - // await Promise.all(userDeletionPromises) - // } - // - // await globalDb.bulkDocs(users) + // initially get all the app databases out of the tarball + const tmpPath = sdk.backups.untarFile(ctx.request.file.importFile) + const globalDbImport = sdk.backups.getGlobalDBFile(tmpPath) + const appNames = fs + .readdirSync(tmpPath) + .filter(dir => dir.startsWith(APP_PREFIX)) + + const globalDb = getGlobalDB() + // load the global db first + await globalDb.load(stringToReadStream(globalDbImport)) + const appCreationPromises = [] + for (let appName of appNames) { + appCreationPromises.push(createApp(appName, join(tmpPath, appName))) + } + await Promise.all(appCreationPromises) + + // if there are any users make sure to remove them + let users = await getAllDocType(globalDb, DocumentType.USER) + let userDeletionPromises = [] + for (let user of users) { + userDeletionPromises.push(globalDb.remove(user._id, user._rev)) + } + if (userDeletionPromises.length > 0) { + await Promise.all(userDeletionPromises) + } + + await globalDb.bulkDocs(users) ctx.body = { message: "Apps successfully imported.", } diff --git a/packages/server/src/sdk/app/backups/exports.ts b/packages/server/src/sdk/app/backups/exports.ts index adf39f7e15..5a028c27a9 100644 --- a/packages/server/src/sdk/app/backups/exports.ts +++ b/packages/server/src/sdk/app/backups/exports.ts @@ -127,28 +127,33 @@ export async function exportApp(appId: string, config?: ExportOpts) { /** * Export all apps + global DB (if supplied) to a single tarball, this includes * the attachments for each app as well. - * @param {string[]} appIds The IDs of the apps to be exported. + * @param {object[]} appMetadata The IDs and names of apps to export. * @param {string} globalDbContents The contents of the global DB to export as well. * @return {string} The path to the tarball. */ export async function exportMultipleApps( - appIds: string[], + appMetadata: { appId: string; name: string }[], globalDbContents?: string ) { const tmpPath = join(budibaseTempDir(), uuid()) + fs.mkdirSync(tmpPath) let exportPromises: Promise[] = [] - const exportAndMove = async (appId: string) => { + // export each app to a directory, then move it into the complete export + const exportAndMove = async (appId: string, appName: string) => { const path = await exportApp(appId) await fs.promises.rename(path, join(tmpPath, appId)) } - for (let appId of appIds) { - exportPromises.push(exportAndMove(appId)) + for (let metadata of appMetadata) { + exportPromises.push(exportAndMove(metadata.appId, metadata.name)) } + // wait for all exports to finish await Promise.all(exportPromises) + // add the global DB contents if (globalDbContents) { fs.writeFileSync(join(tmpPath, GLOBAL_DB_EXPORT_FILE), globalDbContents) } - const tarPath = tarFilesToTmp(tmpPath, [...appIds, GLOBAL_DB_EXPORT_FILE]) + const appNames = appMetadata.map(metadata => metadata.name) + const tarPath = tarFilesToTmp(tmpPath, [...appNames, GLOBAL_DB_EXPORT_FILE]) // clear up the tmp path now tarball generated fs.rmSync(tmpPath, { recursive: true, force: true }) return tarPath diff --git a/packages/server/src/sdk/app/backups/imports.ts b/packages/server/src/sdk/app/backups/imports.ts index 297274e26b..13d8e7aab0 100644 --- a/packages/server/src/sdk/app/backups/imports.ts +++ b/packages/server/src/sdk/app/backups/imports.ts @@ -1,7 +1,11 @@ import { db as dbCore } from "@budibase/backend-core" import { TABLE_ROW_PREFIX } from "../../../db/utils" import { budibaseTempDir } from "../../../utilities/budibaseDir" -import { DB_EXPORT_FILE, ATTACHMENT_DIR } from "./constants" +import { + DB_EXPORT_FILE, + ATTACHMENT_DIR, + GLOBAL_DB_EXPORT_FILE, +} from "./constants" import { uploadDirectory } from "../../../utilities/fileSystem/utilities" import { ObjectStoreBuckets, FieldTypes } from "../../../constants" import { join } from "path" @@ -91,6 +95,22 @@ async function getTemplateStream(template: TemplateType) { } } +export function untarFile(file: { path: string }) { + const tmpPath = join(budibaseTempDir(), uuid()) + fs.mkdirSync(tmpPath) + // extract the tarball + tar.extract({ + sync: true, + cwd: tmpPath, + file: file.path, + }) + return tmpPath +} + +export function getGlobalDBFile(tmpPath: string) { + return fs.readFileSync(join(tmpPath, GLOBAL_DB_EXPORT_FILE), "utf8") +} + export async function importApp( appId: string, db: PouchDB.Database, @@ -98,15 +118,11 @@ export async function importApp( ) { let prodAppId = dbCore.getProdAppID(appId) let dbStream: any - if (template.file && template.file.type === "application/gzip") { - const tmpPath = join(budibaseTempDir(), uuid()) - fs.mkdirSync(tmpPath) - // extract the tarball - tar.extract({ - sync: true, - cwd: tmpPath, - file: template.file.path, - }) + const isTar = template.file && template.file.type === "application/gzip" + const isDirectory = + template.file && fs.lstatSync(template.file.path).isDirectory() + if (template.file && (isTar || isDirectory)) { + const tmpPath = isTar ? untarFile(template.file) : template.file.path const attachmentPath = join(tmpPath, ATTACHMENT_DIR) // have to handle object import if (fs.existsSync(attachmentPath)) { From 5d0dd41de34c851580f4cd541154bbbd4d6a82e5 Mon Sep 17 00:00:00 2001 From: Jonny McCullagh Date: Thu, 13 Oct 2022 11:57:00 +0100 Subject: [PATCH 022/151] single image mount nfs share if vars defined --- hosting/single/runner.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/hosting/single/runner.sh b/hosting/single/runner.sh index e02b33d771..9ab6875d10 100644 --- a/hosting/single/runner.sh +++ b/hosting/single/runner.sh @@ -18,6 +18,7 @@ declare -a DOCKER_VARS=("APP_PORT" "APPS_URL" "ARCHITECTURE" "BUDIBASE_ENVIRONME [[ -z "${WORKER_URL}" ]] && export WORKER_URL=http://localhost:4002 [[ -z "${APPS_URL}" ]] && export APPS_URL=http://localhost:4001 # export CUSTOM_DOMAIN=budi001.custom.com + # Azure App Service customisations if [[ "${TARGETBUILD}" = "aas" ]]; then DATA_DIR=/home @@ -26,6 +27,13 @@ else DATA_DIR=${DATA_DIR:-/data} fi +# Mount NFS or GCP Filestore if env vars exist for it +if [[ -z ${FILESHARE_IP} && -z ${FILESHARE_NAME} ]]; then + echo "Mount file share ${FILESHARE_IP}:/${FILESHARE_NAME} to ${DATA_DIR}" + mount -o nolock ${FILESHARE_IP}:/${FILESHARE_NAME} ${DATA_DIR} + echo "Mounting completed." +fi + if [ -f "${DATA_DIR}/.env" ]; then # Read in the .env file and export the variables for LINE in $(cat ${DATA_DIR}/.env); do export $LINE; done From 7b2e471d829507c44ff8af917d8835b27d691a55 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Thu, 13 Oct 2022 15:32:31 +0100 Subject: [PATCH 023/151] Only hide plugin search bar when no plugins exist --- .../src/pages/builder/portal/manage/plugins/index.svelte | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/builder/src/pages/builder/portal/manage/plugins/index.svelte b/packages/builder/src/pages/builder/portal/manage/plugins/index.svelte index 84722c27be..b9b84307a8 100644 --- a/packages/builder/src/pages/builder/portal/manage/plugins/index.svelte +++ b/packages/builder/src/pages/builder/portal/manage/plugins/index.svelte @@ -55,7 +55,7 @@ Add plugin - {#if filteredPlugins?.length} + {#if $plugins?.length}
diff --git a/packages/client/manifest.json b/packages/client/manifest.json index b6d4941e4c..d9c9b14676 100644 --- a/packages/client/manifest.json +++ b/packages/client/manifest.json @@ -3523,7 +3523,7 @@ "key": "filter" }, { - "type": "field", + "type": "field/sortable", "label": "Sort Column", "key": "sortColumn" }, @@ -3853,7 +3853,7 @@ "key": "filter" }, { - "type": "field", + "type": "field/sortable", "label": "Sort Column", "key": "sortColumn" }, @@ -4018,7 +4018,7 @@ "key": "filter" }, { - "type": "field", + "type": "field/sortable", "label": "Sort Column", "key": "sortColumn" }, @@ -4177,7 +4177,7 @@ "key": "filter" }, { - "type": "field", + "type": "field/sortable", "label": "Sort Column", "key": "sortColumn" }, From bfee51aeab32fa50d082f3b416e4d970aca52387 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Thu, 13 Oct 2022 16:11:53 +0100 Subject: [PATCH 026/151] Update formula tooltip to explain they cannot be sorted by --- .../components/backend/DataTable/modals/CreateEditColumn.svelte | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte index 21059b32dd..44a750cbb1 100644 --- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte +++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte @@ -487,7 +487,7 @@ ]} getOptionLabel={option => option.label} getOptionValue={option => option.value} - tooltip="Dynamic formula are calculated when retrieved, but cannot be filtered, + tooltip="Dynamic formula are calculated when retrieved, but cannot be filtered or sorted by, while static formula are calculated when the row is saved." /> {/if} From 35525bfedd9b03db51e22e29567c2817f0ae1e13 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 13 Oct 2022 17:27:04 +0100 Subject: [PATCH 027/151] Moving queue implementation into backend-core - so that pro can access. --- packages/backend-core/package.json | 1 + packages/backend-core/src/index.ts | 2 + packages/backend-core/src/queue/constants.ts | 4 + .../backend-core/src/queue/inMemoryQueue.ts | 127 +++++++++++++++++ packages/backend-core/src/queue/index.ts | 2 + packages/backend-core/src/queue/queue.ts | 47 +++++++ packages/backend-core/yarn.lock | 129 +++++++++++++++++- packages/server/src/automations/bullboard.js | 33 +---- packages/server/src/constants/index.js | 4 - 9 files changed, 314 insertions(+), 35 deletions(-) create mode 100644 packages/backend-core/src/queue/constants.ts create mode 100644 packages/backend-core/src/queue/inMemoryQueue.ts create mode 100644 packages/backend-core/src/queue/index.ts create mode 100644 packages/backend-core/src/queue/queue.ts diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json index abcf72491a..404b7da346 100644 --- a/packages/backend-core/package.json +++ b/packages/backend-core/package.json @@ -26,6 +26,7 @@ "aws-sdk": "2.1030.0", "bcrypt": "5.0.1", "bcryptjs": "2.4.3", + "bull": "^4.10.1", "dotenv": "16.0.1", "emitter-listener": "1.1.2", "ioredis": "4.28.0", diff --git a/packages/backend-core/src/index.ts b/packages/backend-core/src/index.ts index 42cad17620..659a56c051 100644 --- a/packages/backend-core/src/index.ts +++ b/packages/backend-core/src/index.ts @@ -19,6 +19,7 @@ import pino from "./pino" import * as middleware from "./middleware" import plugins from "./plugin" import encryption from "./security/encryption" +import * as queue from "./queue" // mimic the outer package exports import * as db from "./pkg/db" @@ -63,6 +64,7 @@ const core = { ...errorClasses, middleware, encryption, + queue, } export = core diff --git a/packages/backend-core/src/queue/constants.ts b/packages/backend-core/src/queue/constants.ts new file mode 100644 index 0000000000..d8fb3121a3 --- /dev/null +++ b/packages/backend-core/src/queue/constants.ts @@ -0,0 +1,4 @@ +export enum JobQueue { + AUTOMATIONS = "automationQueue", + APP_BACKUPS = "appBackupQueue", +} diff --git a/packages/backend-core/src/queue/inMemoryQueue.ts b/packages/backend-core/src/queue/inMemoryQueue.ts new file mode 100644 index 0000000000..80ee7362e4 --- /dev/null +++ b/packages/backend-core/src/queue/inMemoryQueue.ts @@ -0,0 +1,127 @@ +import events from "events" + +/** + * Bull works with a Job wrapper around all messages that contains a lot more information about + * the state of the message, this object constructor implements the same schema of Bull jobs + * for the sake of maintaining API consistency. + * @param {string} queue The name of the queue which the message will be carried on. + * @param {object} message The JSON message which will be passed back to the consumer. + * @returns {Object} A new job which can now be put onto the queue, this is mostly an + * internal structure so that an in memory queue can be easily swapped for a Bull queue. + */ +function newJob(queue: string, message: any) { + return { + timestamp: Date.now(), + queue: queue, + data: message, + } +} + +/** + * This is designed to replicate Bull (https://github.com/OptimalBits/bull) in memory as a sort of mock. + * It is relatively simple, using an event emitter internally to register when messages are available + * to the consumers - in can support many inputs and many consumers. + */ +class InMemoryQueue { + _name: string + _opts?: any + _messages: any[] + _emitter: EventEmitter + /** + * The constructor the queue, exactly the same as that of Bulls. + * @param {string} name The name of the queue which is being configured. + * @param {object|null} opts This is not used by the in memory queue as there is no real use + * case when in memory, but is the same API as Bull + */ + constructor(name: string, opts = null) { + this._name = name + this._opts = opts + this._messages = [] + this._emitter = new events.EventEmitter() + } + + /** + * Same callback API as Bull, each callback passed to this will consume messages as they are + * available. Please note this is a queue service, not a notification service, so each + * consumer will receive different messages. + * @param {function} func The callback function which will return a "Job", the same + * as the Bull API, within this job the property "data" contains the JSON message. Please + * note this is incredibly limited compared to Bull as in reality the Job would contain + * a lot more information about the queue and current status of Bull cluster. + */ + process(func: any) { + this._emitter.on("message", async () => { + if (this._messages.length <= 0) { + return + } + let msg = this._messages.shift() + let resp = func(msg) + if (resp.then != null) { + await resp + } + }) + } + + // simply puts a message to the queue and emits to the queue for processing + /** + * Simple function to replicate the add message functionality of Bull, putting + * a new message on the queue. This then emits an event which will be used to + * return the message to a consumer (if one is attached). + * @param {object} msg A message to be transported over the queue, this should be + * a JSON message as this is required by Bull. + * @param {boolean} repeat serves no purpose for the import queue. + */ + // eslint-disable-next-line no-unused-vars + add(msg: any, repeat: boolean) { + if (typeof msg !== "object") { + throw "Queue only supports carrying JSON." + } + this._messages.push(newJob(this._name, msg)) + this._emitter.emit("message") + } + + /** + * replicating the close function from bull, which waits for jobs to finish. + */ + async close() { + return [] + } + + /** + * This removes a cron which has been implemented, this is part of Bull API. + * @param {string} cronJobId The cron which is to be removed. + */ + removeRepeatableByKey(cronJobId: string) { + // TODO: implement for testing + console.log(cronJobId) + } + + /** + * Implemented for tests + */ + getRepeatableJobs() { + return [] + } + + // eslint-disable-next-line no-unused-vars + removeJobs(pattern: string) { + // no-op + } + + /** + * Implemented for tests + */ + async clean() { + return [] + } + + async getJob() { + return {} + } + + on() { + // do nothing + } +} + +export = InMemoryQueue diff --git a/packages/backend-core/src/queue/index.ts b/packages/backend-core/src/queue/index.ts new file mode 100644 index 0000000000..b7d565ba13 --- /dev/null +++ b/packages/backend-core/src/queue/index.ts @@ -0,0 +1,2 @@ +export * from "./queue" +export * from "./constants" diff --git a/packages/backend-core/src/queue/queue.ts b/packages/backend-core/src/queue/queue.ts new file mode 100644 index 0000000000..de2c738ca4 --- /dev/null +++ b/packages/backend-core/src/queue/queue.ts @@ -0,0 +1,47 @@ +import env from "../environment" +import { getRedisOptions } from "../redis/utils" +import { JobQueue } from "./constants" +import inMemoryQueue from "./inMemoryQueue" +import BullQueue from "bull" +import InMemoryQueue from "./inMemoryQueue" +const { opts, redisProtocolUrl } = getRedisOptions() + +const CLEANUP_PERIOD_MS = 60 * 1000 +let QUEUES: BullQueue.Queue[] | InMemoryQueue[] = [] +let cleanupInterval: NodeJS.Timeout + +async function cleanup() { + for (let queue of QUEUES) { + await queue.clean(CLEANUP_PERIOD_MS, "completed") + } +} + +export function createQueue(jobQueue: JobQueue) { + const queueConfig: any = redisProtocolUrl || { redis: opts } + let queue: any + if (env.isTest()) { + queue = new BullQueue(jobQueue, queueConfig) + } else { + queue = new inMemoryQueue(jobQueue, queueConfig) + } + QUEUES.push(queue) + if (!cleanupInterval) { + cleanupInterval = setInterval(cleanup, CLEANUP_PERIOD_MS) + // fire off an initial cleanup + cleanup().catch(err => { + console.error(`Unable to cleanup automation queue initially - ${err}`) + }) + } + return queue +} + +exports.shutdown = async () => { + if (QUEUES.length) { + clearInterval(cleanupInterval) + for (let queue of QUEUES) { + await queue.close() + } + QUEUES = [] + } + console.log("Queues shutdown") +} diff --git a/packages/backend-core/yarn.lock b/packages/backend-core/yarn.lock index 6bc9b63728..31d25320c7 100644 --- a/packages/backend-core/yarn.lock +++ b/packages/backend-core/yarn.lock @@ -291,6 +291,11 @@ resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw== +"@budibase/types@2.0.30-alpha.3": + version "2.0.30-alpha.3" + resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.0.30-alpha.3.tgz#cb55bcced75b711cc8a675284fbacaa8ebf1c0f2" + integrity sha512-rHeFVuNbSSE4fMnX6uyrM2r47m+neqFXlVNOkhHU9i7KoIcIZbEYInU8CjUFR2da3ruST9ajXjJ5UenX2+MnTg== + "@hapi/hoek@^9.0.0": version "9.3.0" resolved "https://registry.yarnpkg.com/@hapi/hoek/-/hoek-9.3.0.tgz#8368869dcb735be2e7f5cb7647de78e167a251fb" @@ -543,6 +548,36 @@ semver "^7.3.5" tar "^6.1.11" +"@msgpackr-extract/msgpackr-extract-darwin-arm64@2.1.2": + version "2.1.2" + resolved "https://registry.yarnpkg.com/@msgpackr-extract/msgpackr-extract-darwin-arm64/-/msgpackr-extract-darwin-arm64-2.1.2.tgz#9571b87be3a3f2c46de05585470bc4f3af2f6f00" + integrity sha512-TyVLn3S/+ikMDsh0gbKv2YydKClN8HaJDDpONlaZR+LVJmsxLFUgA+O7zu59h9+f9gX1aj/ahw9wqa6rosmrYQ== + +"@msgpackr-extract/msgpackr-extract-darwin-x64@2.1.2": + version "2.1.2" + resolved "https://registry.yarnpkg.com/@msgpackr-extract/msgpackr-extract-darwin-x64/-/msgpackr-extract-darwin-x64-2.1.2.tgz#bfbc6936ede2955218f5621a675679a5fe8e6f4c" + integrity sha512-YPXtcVkhmVNoMGlqp81ZHW4dMxK09msWgnxtsDpSiZwTzUBG2N+No2bsr7WMtBKCVJMSD6mbAl7YhKUqkp/Few== + +"@msgpackr-extract/msgpackr-extract-linux-arm64@2.1.2": + version "2.1.2" + resolved "https://registry.yarnpkg.com/@msgpackr-extract/msgpackr-extract-linux-arm64/-/msgpackr-extract-linux-arm64-2.1.2.tgz#22555e28382af2922e7450634c8a2f240bb9eb82" + integrity sha512-vHZ2JiOWF2+DN9lzltGbhtQNzDo8fKFGrf37UJrgqxU0yvtERrzUugnfnX1wmVfFhSsF8OxrfqiNOUc5hko1Zg== + +"@msgpackr-extract/msgpackr-extract-linux-arm@2.1.2": + version "2.1.2" + resolved "https://registry.yarnpkg.com/@msgpackr-extract/msgpackr-extract-linux-arm/-/msgpackr-extract-linux-arm-2.1.2.tgz#ffb6ae1beea7ac572b6be6bf2a8e8162ebdd8be7" + integrity sha512-42R4MAFeIeNn+L98qwxAt360bwzX2Kf0ZQkBBucJ2Ircza3asoY4CDbgiu9VWklq8gWJVSJSJBwDI+c/THiWkA== + +"@msgpackr-extract/msgpackr-extract-linux-x64@2.1.2": + version "2.1.2" + resolved "https://registry.yarnpkg.com/@msgpackr-extract/msgpackr-extract-linux-x64/-/msgpackr-extract-linux-x64-2.1.2.tgz#7caf62eebbfb1345de40f75e89666b3d4194755f" + integrity sha512-RjRoRxg7Q3kPAdUSC5EUUPlwfMkIVhmaRTIe+cqHbKrGZ4M6TyCA/b5qMaukQ/1CHWrqYY2FbKOAU8Hg0pQFzg== + +"@msgpackr-extract/msgpackr-extract-win32-x64@2.1.2": + version "2.1.2" + resolved "https://registry.yarnpkg.com/@msgpackr-extract/msgpackr-extract-win32-x64/-/msgpackr-extract-win32-x64-2.1.2.tgz#f2d8b9ddd8d191205ed26ce54aba3dfc5ae3e7c9" + integrity sha512-rIZVR48zA8hGkHIK7ED6+ZiXsjRCcAVBJbm8o89OKAMTmEAQ2QvoOxoiu3w2isAaWwzgtQIOFIqHwvZDyLKCvw== + "@shopify/jest-koa-mocks@5.0.1": version "5.0.1" resolved "https://registry.yarnpkg.com/@shopify/jest-koa-mocks/-/jest-koa-mocks-5.0.1.tgz#fba490b6b7985fbb571eb9974897d396a3642e94" @@ -1497,6 +1532,21 @@ buffer@^5.5.0, buffer@^5.6.0: base64-js "^1.3.1" ieee754 "^1.1.13" +bull@^4.10.1: + version "4.10.1" + resolved "https://registry.yarnpkg.com/bull/-/bull-4.10.1.tgz#f14974b6089358b62b495a2cbf838aadc098e43f" + integrity sha512-Fp21tRPb2EaZPVfmM+ONZKVz2RA+to+zGgaTLyCKt3JMSU8OOBqK8143OQrnGuGpsyE5G+9FevFAGhdZZfQP2g== + dependencies: + cron-parser "^4.2.1" + debuglog "^1.0.0" + get-port "^5.1.1" + ioredis "^4.28.5" + lodash "^4.17.21" + msgpackr "^1.5.2" + p-timeout "^3.2.0" + semver "^7.3.2" + uuid "^8.3.0" + cache-content-type@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/cache-content-type/-/cache-content-type-1.0.1.tgz#035cde2b08ee2129f4a8315ea8f00a00dba1453c" @@ -1764,6 +1814,13 @@ core-util-is@~1.0.0: resolved "https://registry.yarnpkg.com/core-util-is/-/core-util-is-1.0.3.tgz#a6042d3634c2b27e9328f837b965fac83808db85" integrity sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ== +cron-parser@^4.2.1: + version "4.6.0" + resolved "https://registry.yarnpkg.com/cron-parser/-/cron-parser-4.6.0.tgz#404c3fdbff10ae80eef6b709555d577ef2fd2e0d" + integrity sha512-guZNLMGUgg6z4+eGhmHGw7ft+v6OQeuHzd1gcLxCo9Yg/qoxmG3nindp2/uwGCLizEisf2H0ptqeVXeoCpP6FA== + dependencies: + luxon "^3.0.1" + cross-spawn@^7.0.3: version "7.0.3" resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-7.0.3.tgz#f73a85b9d5d41d045551c177e2882d4ac85728a6" @@ -1837,6 +1894,11 @@ debug@~3.1.0: dependencies: ms "2.0.0" +debuglog@^1.0.0: + version "1.0.1" + resolved "https://registry.yarnpkg.com/debuglog/-/debuglog-1.0.1.tgz#aa24ffb9ac3df9a2351837cfb2d279360cd78492" + integrity sha512-syBZ+rnAK3EgMsH2aYEOLUW7mZSY9Gb+0wUMCFsZvcmiz+HigA0LOcq/HoQqVuGG+EKykunc7QG2bzrponfaSw== + decimal.js@^10.2.1: version "10.3.1" resolved "https://registry.yarnpkg.com/decimal.js/-/decimal.js-10.3.1.tgz#d8c3a444a9c6774ba60ca6ad7261c3a94fd5e783" @@ -2318,6 +2380,11 @@ get-package-type@^0.1.0: resolved "https://registry.yarnpkg.com/get-package-type/-/get-package-type-0.1.0.tgz#8de2d803cff44df3bc6c456e6668b36c3926e11a" integrity sha512-pjzuKtY64GYfWizNAJ0fr9VqttZkNiK2iS430LtIHzjBEr6bX8Am2zm4sW4Ro5wjWW5cAlRL1qAMTcXbjNAO2Q== +get-port@^5.1.1: + version "5.1.1" + resolved "https://registry.yarnpkg.com/get-port/-/get-port-5.1.1.tgz#0469ed07563479de6efb986baf053dcd7d4e3193" + integrity sha512-g/Q1aTSDOxFpchXC4i8ZWvxA1lnPqx/JHqcpIw0/LX9T8x/GBbi6YnlN5nhaKIFkT8oFsscUKgDJYxfwfS6QsQ== + get-stream@^4.1.0: version "4.1.0" resolved "https://registry.yarnpkg.com/get-stream/-/get-stream-4.1.0.tgz#c1b255575f3dc21d59bfc79cd3d2b46b1c3a54b5" @@ -2652,6 +2719,23 @@ ioredis@4.28.0: redis-parser "^3.0.0" standard-as-callback "^2.1.0" +ioredis@^4.28.5: + version "4.28.5" + resolved "https://registry.yarnpkg.com/ioredis/-/ioredis-4.28.5.tgz#5c149e6a8d76a7f8fa8a504ffc85b7d5b6797f9f" + integrity sha512-3GYo0GJtLqgNXj4YhrisLaNNvWSNwSS2wS4OELGfGxH8I69+XfNdnmV1AyN+ZqMh0i7eX+SWjrwFKDBDgfBC1A== + dependencies: + cluster-key-slot "^1.1.0" + debug "^4.3.1" + denque "^1.1.0" + lodash.defaults "^4.2.0" + lodash.flatten "^4.4.0" + lodash.isarguments "^3.1.0" + p-map "^2.1.0" + redis-commands "1.7.0" + redis-errors "^1.2.0" + redis-parser "^3.0.0" + standard-as-callback "^2.1.0" + is-arrayish@^0.2.1: version "0.2.1" resolved "https://registry.yarnpkg.com/is-arrayish/-/is-arrayish-0.2.1.tgz#77c99840527aa8ecb1a8ba697b80645a7a926a9d" @@ -3725,6 +3809,11 @@ ltgt@2.2.1, ltgt@^2.1.2, ltgt@~2.2.0: resolved "https://registry.yarnpkg.com/ltgt/-/ltgt-2.2.1.tgz#f35ca91c493f7b73da0e07495304f17b31f87ee5" integrity sha512-AI2r85+4MquTw9ZYqabu4nMwy9Oftlfa/e/52t9IjtfG+mGBbTNdAoZ3RQKLHR6r0wQnwZnPIEh/Ya6XTWAKNA== +luxon@^3.0.1: + version "3.0.4" + resolved "https://registry.yarnpkg.com/luxon/-/luxon-3.0.4.tgz#d179e4e9f05e092241e7044f64aaa54796b03929" + integrity sha512-aV48rGUwP/Vydn8HT+5cdr26YYQiUZ42NM6ToMoaGKwYfWbfLeRkEu1wXWMHBZT6+KyLfcbbtVcoQFCbbPjKlw== + make-dir@^3.0.0, make-dir@^3.1.0: version "3.1.0" resolved "https://registry.yarnpkg.com/make-dir/-/make-dir-3.1.0.tgz#415e967046b3a7f1d185277d84aa58203726a13f" @@ -3872,6 +3961,27 @@ ms@^2.1.1, ms@^2.1.3: resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.3.tgz#574c8138ce1d2b5861f0b44579dbadd60c6615b2" integrity sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA== +msgpackr-extract@^2.1.2: + version "2.1.2" + resolved "https://registry.yarnpkg.com/msgpackr-extract/-/msgpackr-extract-2.1.2.tgz#56272030f3e163e1b51964ef8b1cd5e7240c03ed" + integrity sha512-cmrmERQFb19NX2JABOGtrKdHMyI6RUyceaPBQ2iRz9GnDkjBWFjNJC0jyyoOfZl2U/LZE3tQCCQc4dlRyA8mcA== + dependencies: + node-gyp-build-optional-packages "5.0.3" + optionalDependencies: + "@msgpackr-extract/msgpackr-extract-darwin-arm64" "2.1.2" + "@msgpackr-extract/msgpackr-extract-darwin-x64" "2.1.2" + "@msgpackr-extract/msgpackr-extract-linux-arm" "2.1.2" + "@msgpackr-extract/msgpackr-extract-linux-arm64" "2.1.2" + "@msgpackr-extract/msgpackr-extract-linux-x64" "2.1.2" + "@msgpackr-extract/msgpackr-extract-win32-x64" "2.1.2" + +msgpackr@^1.5.2: + version "1.7.2" + resolved "https://registry.yarnpkg.com/msgpackr/-/msgpackr-1.7.2.tgz#68d6debf5999d6b61abb6e7046a689991ebf7261" + integrity sha512-mWScyHTtG6TjivXX9vfIy2nBtRupaiAj0HQ2mtmpmYujAmqZmaaEVPaSZ1NKLMvicaMLFzEaMk0ManxMRg8rMQ== + optionalDependencies: + msgpackr-extract "^2.1.2" + napi-macros@~2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/napi-macros/-/napi-macros-2.0.0.tgz#2b6bae421e7b96eb687aa6c77a7858640670001b" @@ -3919,6 +4029,11 @@ node-forge@^0.7.1: resolved "https://registry.yarnpkg.com/node-forge/-/node-forge-0.7.6.tgz#fdf3b418aee1f94f0ef642cd63486c77ca9724ac" integrity sha512-sol30LUpz1jQFBjOKwbjxijiE3b6pjd74YwfD0fJOKPjF+fONKb2Yg8rYgS6+bK6VDl+/wfr4IYpC7jDzLUIfw== +node-gyp-build-optional-packages@5.0.3: + version "5.0.3" + resolved "https://registry.yarnpkg.com/node-gyp-build-optional-packages/-/node-gyp-build-optional-packages-5.0.3.tgz#92a89d400352c44ad3975010368072b41ad66c17" + integrity sha512-k75jcVzk5wnnc/FMxsf4udAoTEUv2jY3ycfdSd3yWu6Cnd1oee6/CfZJApyscA4FJOmdoixWwiwOyf16RzD5JA== + node-gyp-build@~4.1.0: version "4.1.1" resolved "https://registry.yarnpkg.com/node-gyp-build/-/node-gyp-build-4.1.1.tgz#d7270b5d86717068d114cc57fff352f96d745feb" @@ -4075,6 +4190,11 @@ p-cancelable@^1.0.0: resolved "https://registry.yarnpkg.com/p-cancelable/-/p-cancelable-1.1.0.tgz#d078d15a3af409220c886f1d9a0ca2e441ab26cc" integrity sha512-s73XxOZ4zpt1edZYZzvhqFa6uvQc1vwUa0K0BdtIZgQMAJj9IbebH+JkgKZc9h+B05PKHLOTl4ajG1BmNrVZlw== +p-finally@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/p-finally/-/p-finally-1.0.0.tgz#3fbcfb15b899a44123b34b6dcc18b724336a2cae" + integrity sha512-LICb2p9CB7FS+0eR1oqWnHhp0FljGLZCWBE9aix0Uye9W8LTQPwMTYVGWQWIw9RdQiDg4+epXQODwIYJtSJaow== + p-limit@^2.2.0: version "2.3.0" resolved "https://registry.yarnpkg.com/p-limit/-/p-limit-2.3.0.tgz#3dd33c647a214fdfffd835933eb086da0dc21db1" @@ -4094,6 +4214,13 @@ p-map@^2.1.0: resolved "https://registry.yarnpkg.com/p-map/-/p-map-2.1.0.tgz#310928feef9c9ecc65b68b17693018a665cea175" integrity sha512-y3b8Kpd8OAN444hxfBbFfj1FY/RjtTd8tzYwhUqNYXx0fXx2iX4maP4Qr6qhIKbQXI02wTLAda4fYUbDagTUFw== +p-timeout@^3.2.0: + version "3.2.0" + resolved "https://registry.yarnpkg.com/p-timeout/-/p-timeout-3.2.0.tgz#c7e17abc971d2a7962ef83626b35d635acf23dfe" + integrity sha512-rhIwUycgwwKcP9yTOOFK/AKsAopjjCakVqLHePO3CC6Mir1Z99xT+R63jZxAT5lFZLa2inS5h+ZS2GvR99/FBg== + dependencies: + p-finally "^1.0.0" + p-try@^2.0.0: version "2.2.0" resolved "https://registry.yarnpkg.com/p-try/-/p-try-2.2.0.tgz#cb2868540e313d61de58fafbe35ce9004d5540e6" @@ -5360,7 +5487,7 @@ uuid@8.1.0: resolved "https://registry.yarnpkg.com/uuid/-/uuid-8.1.0.tgz#6f1536eb43249f473abc6bd58ff983da1ca30d8d" integrity sha512-CI18flHDznR0lq54xBycOVmphdCYnQLKn8abKn7PXUiKUGdEd+/l9LWNJmugXel4hXq7S+RMNl34ecyC9TntWg== -uuid@8.3.2, uuid@^8.3.2: +uuid@8.3.2, uuid@^8.3.0, uuid@^8.3.2: version "8.3.2" resolved "https://registry.yarnpkg.com/uuid/-/uuid-8.3.2.tgz#80d5b5ced271bb9af6c445f21a1a04c606cefbe2" integrity sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg== diff --git a/packages/server/src/automations/bullboard.js b/packages/server/src/automations/bullboard.js index 3aac6c4fed..b0ccf14634 100644 --- a/packages/server/src/automations/bullboard.js +++ b/packages/server/src/automations/bullboard.js @@ -1,37 +1,15 @@ const { createBullBoard } = require("@bull-board/api") const { BullAdapter } = require("@bull-board/api/bullAdapter") const { KoaAdapter } = require("@bull-board/koa") -const env = require("../environment") -const Queue = env.isTest() - ? require("../utilities/queue/inMemoryQueue") - : require("bull") -const { JobQueues } = require("../constants") -const { utils } = require("@budibase/backend-core/redis") -const { opts, redisProtocolUrl } = utils.getRedisOptions() +const { queue } = require("@budibase/backend-core") const listeners = require("./listeners") -const CLEANUP_PERIOD_MS = 60 * 1000 -const queueConfig = redisProtocolUrl || { redis: opts } -let cleanupInternal = null - -let automationQueue = new Queue(JobQueues.AUTOMATIONS, queueConfig) +let automationQueue = queue.createQueue(queue.JobQueues.AUTOMATIONS) listeners.addListeners(automationQueue) -async function cleanup() { - await automationQueue.clean(CLEANUP_PERIOD_MS, "completed") -} - const PATH_PREFIX = "/bulladmin" exports.init = () => { - // cleanup the events every 5 minutes - if (!cleanupInternal) { - cleanupInternal = setInterval(cleanup, CLEANUP_PERIOD_MS) - // fire off an initial cleanup - cleanup().catch(err => { - console.error(`Unable to cleanup automation queue initially - ${err}`) - }) - } // Set up queues for bull board admin const queues = [automationQueue] const adapters = [] @@ -48,12 +26,7 @@ exports.init = () => { } exports.shutdown = async () => { - if (automationQueue) { - clearInterval(cleanupInternal) - await automationQueue.close() - automationQueue = null - } - console.log("Bull shutdown") + await queue.shutdown() } exports.queue = automationQueue diff --git a/packages/server/src/constants/index.js b/packages/server/src/constants/index.js index b9362cecf6..73bffeabd6 100644 --- a/packages/server/src/constants/index.js +++ b/packages/server/src/constants/index.js @@ -2,10 +2,6 @@ const { BUILTIN_ROLE_IDS } = require("@budibase/backend-core/roles") const { UserStatus } = require("@budibase/backend-core/constants") const { objectStore } = require("@budibase/backend-core") -exports.JobQueues = { - AUTOMATIONS: "automationQueue", -} - const FilterTypes = { STRING: "string", FUZZY: "fuzzy", From b6ca14aa85bf83784da4d23e546bbbc99eea177f Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 13 Oct 2022 17:39:26 +0100 Subject: [PATCH 028/151] Renaming some queue to automationQueue, getting build working. --- packages/backend-core/package.json | 1 + packages/backend-core/yarn.lock | 7 +++++++ packages/server/src/automations/bullboard.js | 4 ++-- packages/server/src/automations/index.js | 8 ++++---- packages/server/src/automations/triggers.js | 8 ++++---- packages/server/src/automations/utils.ts | 14 +++++++------- 6 files changed, 25 insertions(+), 17 deletions(-) diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json index 404b7da346..eb90fa4159 100644 --- a/packages/backend-core/package.json +++ b/packages/backend-core/package.json @@ -64,6 +64,7 @@ }, "devDependencies": { "@types/chance": "1.1.3", + "@types/ioredis": "^4.28.10", "@types/jest": "27.5.1", "@types/koa": "2.0.52", "@types/lodash": "4.14.180", diff --git a/packages/backend-core/yarn.lock b/packages/backend-core/yarn.lock index 31d25320c7..22b93d7b3e 100644 --- a/packages/backend-core/yarn.lock +++ b/packages/backend-core/yarn.lock @@ -768,6 +768,13 @@ resolved "https://registry.yarnpkg.com/@types/http-errors/-/http-errors-1.8.2.tgz#7315b4c4c54f82d13fa61c228ec5c2ea5cc9e0e1" integrity sha512-EqX+YQxINb+MeXaIqYDASb6U6FCHbWjkj4a1CKDBks3d/QiB2+PqBLyO72vLDgAO1wUI4O+9gweRcQK11bTL/w== +"@types/ioredis@^4.28.10": + version "4.28.10" + resolved "https://registry.yarnpkg.com/@types/ioredis/-/ioredis-4.28.10.tgz#40ceb157a4141088d1394bb87c98ed09a75a06ff" + integrity sha512-69LyhUgrXdgcNDv7ogs1qXZomnfOEnSmrmMFqKgt1XMJxmoOSG/u3wYy13yACIfKuMJ8IhKgHafDO3sx19zVQQ== + dependencies: + "@types/node" "*" + "@types/istanbul-lib-coverage@*", "@types/istanbul-lib-coverage@^2.0.0", "@types/istanbul-lib-coverage@^2.0.1": version "2.0.4" resolved "https://registry.yarnpkg.com/@types/istanbul-lib-coverage/-/istanbul-lib-coverage-2.0.4.tgz#8467d4b3c087805d63580480890791277ce35c44" diff --git a/packages/server/src/automations/bullboard.js b/packages/server/src/automations/bullboard.js index b0ccf14634..9dfa7546ec 100644 --- a/packages/server/src/automations/bullboard.js +++ b/packages/server/src/automations/bullboard.js @@ -4,7 +4,7 @@ const { KoaAdapter } = require("@bull-board/koa") const { queue } = require("@budibase/backend-core") const listeners = require("./listeners") -let automationQueue = queue.createQueue(queue.JobQueues.AUTOMATIONS) +let automationQueue = queue.createQueue(queue.JobQueue.AUTOMATIONS) listeners.addListeners(automationQueue) const PATH_PREFIX = "/bulladmin" @@ -29,4 +29,4 @@ exports.shutdown = async () => { await queue.shutdown() } -exports.queue = automationQueue +exports.automationQueue = automationQueue diff --git a/packages/server/src/automations/index.js b/packages/server/src/automations/index.js index 2baa868890..521991dd2c 100644 --- a/packages/server/src/automations/index.js +++ b/packages/server/src/automations/index.js @@ -1,5 +1,5 @@ const { processEvent } = require("./utils") -const { queue, shutdown } = require("./bullboard") +const { automationQueue, shutdown } = require("./bullboard") const { TRIGGER_DEFINITIONS, rebootTrigger } = require("./triggers") const { ACTION_DEFINITIONS } = require("./actions") @@ -8,7 +8,7 @@ const { ACTION_DEFINITIONS } = require("./actions") */ exports.init = async function () { // this promise will not complete - const promise = queue.process(async job => { + const promise = automationQueue.process(async job => { await processEvent(job) }) // on init we need to trigger any reboot automations @@ -17,13 +17,13 @@ exports.init = async function () { } exports.getQueues = () => { - return [queue] + return [automationQueue] } exports.shutdown = () => { return shutdown() } -exports.queue = queue +exports.automationQueue = automationQueue exports.TRIGGER_DEFINITIONS = TRIGGER_DEFINITIONS exports.ACTION_DEFINITIONS = ACTION_DEFINITIONS diff --git a/packages/server/src/automations/triggers.js b/packages/server/src/automations/triggers.js index 395390113a..6a4bbd8da6 100644 --- a/packages/server/src/automations/triggers.js +++ b/packages/server/src/automations/triggers.js @@ -4,7 +4,7 @@ const { coerce } = require("../utilities/rowProcessor") const { definitions } = require("./triggerInfo") const { isDevAppID } = require("../db/utils") // need this to call directly, so we can get a response -const { queue } = require("./bullboard") +const { automationQueue } = require("./bullboard") const { checkTestFlag } = require("../utilities/redis") const utils = require("./utils") const env = require("../environment") @@ -56,7 +56,7 @@ async function queueRelevantRowAutomations(event, eventType) { automationTrigger.inputs && automationTrigger.inputs.tableId === event.row.tableId ) { - await queue.add({ automation, event }, JOB_OPTS) + await automationQueue.add({ automation, event }, JOB_OPTS) } } }) @@ -110,7 +110,7 @@ exports.externalTrigger = async function ( if (getResponses) { return utils.processEvent({ data }) } else { - return queue.add(data, JOB_OPTS) + return automationQueue.add(data, JOB_OPTS) } } @@ -136,7 +136,7 @@ exports.rebootTrigger = async () => { timestamp: Date.now(), }, } - rebootEvents.push(queue.add(job, JOB_OPTS)) + rebootEvents.push(automationQueue.add(job, JOB_OPTS)) } } await Promise.all(rebootEvents) diff --git a/packages/server/src/automations/utils.ts b/packages/server/src/automations/utils.ts index 7e19486798..54b8078f30 100644 --- a/packages/server/src/automations/utils.ts +++ b/packages/server/src/automations/utils.ts @@ -1,7 +1,7 @@ import { Thread, ThreadType } from "../threads" import { definitions } from "./triggerInfo" import * as webhooks from "../api/controllers/webhook" -import { queue } from "./bullboard" +import { automationQueue } from "./bullboard" import newid from "../db/newid" import { updateEntityMetadata } from "../utilities" import { MetadataTypes, WebhookType } from "../constants" @@ -79,12 +79,12 @@ export function removeDeprecated(definitions: any) { // end the repetition and the job itself export async function disableAllCrons(appId: any) { const promises = [] - const jobs = await queue.getRepeatableJobs() + const jobs = await automationQueue.getRepeatableJobs() for (let job of jobs) { if (job.key.includes(`${appId}_cron`)) { - promises.push(queue.removeRepeatableByKey(job.key)) + promises.push(automationQueue.removeRepeatableByKey(job.key)) if (job.id) { - promises.push(queue.removeJobs(job.id)) + promises.push(automationQueue.removeJobs(job.id)) } } } @@ -92,8 +92,8 @@ export async function disableAllCrons(appId: any) { } export async function disableCron(jobId: string, jobKey: string) { - await queue.removeRepeatableByKey(jobKey) - await queue.removeJobs(jobId) + await automationQueue.removeRepeatableByKey(jobKey) + await automationQueue.removeJobs(jobId) console.log(`jobId=${jobId} disabled`) } @@ -141,7 +141,7 @@ export async function enableCronTrigger(appId: any, automation: Automation) { ) { // make a job id rather than letting Bull decide, makes it easier to handle on way out const jobId = `${appId}_cron_${newid()}` - const job: any = await queue.add( + const job: any = await automationQueue.add( { automation, event: { appId, timestamp: Date.now() }, From d620e54fdbb1e8ffdd5da6492adc36bc46b7fa45 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 13 Oct 2022 17:55:05 +0100 Subject: [PATCH 029/151] Handling listeners as part of queue creation, rather than external part. --- .../src/queue}/listeners.ts | 62 +++++++++++-------- packages/backend-core/src/queue/queue.ts | 12 ++-- packages/server/src/automations/bullboard.js | 8 ++- 3 files changed, 50 insertions(+), 32 deletions(-) rename packages/{server/src/automations => backend-core/src/queue}/listeners.ts (52%) diff --git a/packages/server/src/automations/listeners.ts b/packages/backend-core/src/queue/listeners.ts similarity index 52% rename from packages/server/src/automations/listeners.ts rename to packages/backend-core/src/queue/listeners.ts index 9f8667bd29..c5db628ef0 100644 --- a/packages/server/src/automations/listeners.ts +++ b/packages/backend-core/src/queue/listeners.ts @@ -1,78 +1,90 @@ -import { Queue, Job, JobId } from "bull" -import { AutomationEvent } from "../definitions/automations" -import * as automation from "../threads/automation" +import { Job, JobId, Queue } from "bull" +import { JobQueue } from "./constants" -export const addListeners = (queue: Queue) => { - logging(queue) - handleStalled(queue) +export type StalledFn = (job: Job) => Promise + +export const addListeners = ( + queue: Queue, + jobQueue: JobQueue, + removeStalled?: StalledFn +) => { + logging(queue, jobQueue) + if (removeStalled) { + handleStalled(queue, removeStalled) + } } -const handleStalled = (queue: Queue) => { +const handleStalled = (queue: Queue, removeStalled: StalledFn) => { queue.on("stalled", async (job: Job) => { - await automation.removeStalled(job as AutomationEvent) + await removeStalled(job) }) } -const logging = (queue: Queue) => { +const logging = (queue: Queue, jobQueue: JobQueue) => { + let eventType: string + switch (jobQueue) { + case JobQueue.AUTOMATIONS: + eventType = "automation-event" + break + case JobQueue.APP_BACKUPS: + eventType = "app-backup-event" + break + } if (process.env.NODE_DEBUG?.includes("bull")) { queue .on("error", (error: any) => { // An error occurred. - console.error(`automation-event=error error=${JSON.stringify(error)}`) + console.error(`${eventType}=error error=${JSON.stringify(error)}`) }) .on("waiting", (jobId: JobId) => { // A Job is waiting to be processed as soon as a worker is idling. - console.log(`automation-event=waiting jobId=${jobId}`) + console.log(`${eventType}=waiting jobId=${jobId}`) }) .on("active", (job: Job, jobPromise: any) => { // A job has started. You can use `jobPromise.cancel()`` to abort it. - console.log(`automation-event=active jobId=${job.id}`) + console.log(`${eventType}=active jobId=${job.id}`) }) .on("stalled", (job: Job) => { // A job has been marked as stalled. This is useful for debugging job // workers that crash or pause the event loop. console.error( - `automation-event=stalled jobId=${job.id} job=${JSON.stringify(job)}` + `${eventType}=stalled jobId=${job.id} job=${JSON.stringify(job)}` ) }) .on("progress", (job: Job, progress: any) => { // A job's progress was updated! console.log( - `automation-event=progress jobId=${job.id} progress=${progress}` + `${eventType}=progress jobId=${job.id} progress=${progress}` ) }) .on("completed", (job: Job, result) => { // A job successfully completed with a `result`. - console.log( - `automation-event=completed jobId=${job.id} result=${result}` - ) + console.log(`${eventType}=completed jobId=${job.id} result=${result}`) }) .on("failed", (job, err: any) => { // A job failed with reason `err`! - console.log(`automation-event=failed jobId=${job.id} error=${err}`) + console.log(`${eventType}=failed jobId=${job.id} error=${err}`) }) .on("paused", () => { // The queue has been paused. - console.log(`automation-event=paused`) + console.log(`${eventType}=paused`) }) .on("resumed", (job: Job) => { // The queue has been resumed. - console.log(`automation-event=paused jobId=${job.id}`) + console.log(`${eventType}=paused jobId=${job.id}`) }) .on("cleaned", (jobs: Job[], type: string) => { // Old jobs have been cleaned from the queue. `jobs` is an array of cleaned // jobs, and `type` is the type of jobs cleaned. - console.log( - `automation-event=cleaned length=${jobs.length} type=${type}` - ) + console.log(`${eventType}=cleaned length=${jobs.length} type=${type}`) }) .on("drained", () => { // Emitted every time the queue has processed all the waiting jobs (even if there can be some delayed jobs not yet processed) - console.log(`automation-event=drained`) + console.log(`${eventType}=drained`) }) .on("removed", (job: Job) => { // A job successfully removed. - console.log(`automation-event=removed jobId=${job.id}`) + console.log(`${eventType}=removed jobId=${job.id}`) }) } } diff --git a/packages/backend-core/src/queue/queue.ts b/packages/backend-core/src/queue/queue.ts index de2c738ca4..8f7ac79e7a 100644 --- a/packages/backend-core/src/queue/queue.ts +++ b/packages/backend-core/src/queue/queue.ts @@ -1,9 +1,9 @@ import env from "../environment" import { getRedisOptions } from "../redis/utils" import { JobQueue } from "./constants" -import inMemoryQueue from "./inMemoryQueue" -import BullQueue from "bull" import InMemoryQueue from "./inMemoryQueue" +import BullQueue from "bull" +import { addListeners, StalledFn } from "./listeners" const { opts, redisProtocolUrl } = getRedisOptions() const CLEANUP_PERIOD_MS = 60 * 1000 @@ -16,14 +16,18 @@ async function cleanup() { } } -export function createQueue(jobQueue: JobQueue) { +export function createQueue( + jobQueue: JobQueue, + removeStalled: StalledFn +): BullQueue.Queue { const queueConfig: any = redisProtocolUrl || { redis: opts } let queue: any if (env.isTest()) { queue = new BullQueue(jobQueue, queueConfig) } else { - queue = new inMemoryQueue(jobQueue, queueConfig) + queue = new InMemoryQueue(jobQueue, queueConfig) } + addListeners(queue, jobQueue, removeStalled) QUEUES.push(queue) if (!cleanupInterval) { cleanupInterval = setInterval(cleanup, CLEANUP_PERIOD_MS) diff --git a/packages/server/src/automations/bullboard.js b/packages/server/src/automations/bullboard.js index 9dfa7546ec..af2243f13d 100644 --- a/packages/server/src/automations/bullboard.js +++ b/packages/server/src/automations/bullboard.js @@ -2,10 +2,12 @@ const { createBullBoard } = require("@bull-board/api") const { BullAdapter } = require("@bull-board/api/bullAdapter") const { KoaAdapter } = require("@bull-board/koa") const { queue } = require("@budibase/backend-core") -const listeners = require("./listeners") +const automation = require("../threads/automation") -let automationQueue = queue.createQueue(queue.JobQueue.AUTOMATIONS) -listeners.addListeners(automationQueue) +let automationQueue = queue.createQueue( + queue.JobQueue.AUTOMATIONS, + automation.removeStalled +) const PATH_PREFIX = "/bulladmin" From 6ff52c64a38fee01aaef6ec12b5debdab7d6658a Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Thu, 13 Oct 2022 18:07:55 +0100 Subject: [PATCH 030/151] ipv6 change for master --- hosting/proxy/10-listen-on-ipv6-by-default.sh | 24 +++++++++++++++++++ hosting/proxy/Dockerfile | 1 + 2 files changed, 25 insertions(+) create mode 100644 hosting/proxy/10-listen-on-ipv6-by-default.sh diff --git a/hosting/proxy/10-listen-on-ipv6-by-default.sh b/hosting/proxy/10-listen-on-ipv6-by-default.sh new file mode 100644 index 0000000000..1d62732ea1 --- /dev/null +++ b/hosting/proxy/10-listen-on-ipv6-by-default.sh @@ -0,0 +1,24 @@ +#!/bin/sh +# vim:sw=4:ts=4:et + +set -e + +ME=$(basename $0) +NGINX_CONF_FILE="/etc/nginx/nginx.conf" +DEFAULT_CONF_FILE="/etc/nginx/conf.d/default.conf" + +# check if we have ipv6 available +if [ ! -f "/proc/net/if_inet6" ]; then + # ipv6 not available so delete lines from nginx conf + if [ -f "$NGINX_CONF_FILE" ]; then + sed -i '/listen \[::\]/d' $NGINX_CONF_FILE + fi + if [ -f "$DEFAULT_CONF_FILE" ]; then + sed -i '/listen \[::\]/d' $DEFAULT_CONF_FILE + fi + echo "$ME: info: ipv6 not available so delete lines from nginx conf" +else + echo "$ME: info: ipv6 is available so no need to delete lines from nginx conf" +fi + +exit 0 \ No newline at end of file diff --git a/hosting/proxy/Dockerfile b/hosting/proxy/Dockerfile index 298762aaf1..a9c94c06fa 100644 --- a/hosting/proxy/Dockerfile +++ b/hosting/proxy/Dockerfile @@ -5,6 +5,7 @@ FROM nginx:latest # override the output dir to output directly to /etc/nginx instead of /etc/nginx/conf.d ENV NGINX_ENVSUBST_OUTPUT_DIR=/etc/nginx COPY .generated-nginx.prod.conf /etc/nginx/templates/nginx.conf.template +COPY 10-listen-on-ipv6-by-default.sh /docker-entrypoint.d/10-listen-on-ipv6-by-default.sh # Error handling COPY error.html /usr/share/nginx/html/error.html From dea4338d54580bc7ccbaef5c980c68b17c2cca44 Mon Sep 17 00:00:00 2001 From: Budibase Release Bot <> Date: Fri, 14 Oct 2022 09:26:51 +0000 Subject: [PATCH 031/151] v2.0.31 --- lerna.json | 2 +- packages/backend-core/package.json | 4 ++-- packages/bbui/package.json | 4 ++-- packages/builder/package.json | 10 +++++----- packages/cli/package.json | 8 ++++---- packages/client/package.json | 8 ++++---- packages/frontend-core/package.json | 4 ++-- packages/sdk/package.json | 2 +- packages/server/package.json | 10 +++++----- packages/string-templates/package.json | 2 +- packages/types/package.json | 2 +- packages/worker/package.json | 8 ++++---- 12 files changed, 32 insertions(+), 32 deletions(-) diff --git a/lerna.json b/lerna.json index 96c2575602..7d825bcefb 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.0.30", + "version": "2.0.31", "npmClient": "yarn", "packages": [ "packages/*" diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json index 888483b56e..1c3957949b 100644 --- a/packages/backend-core/package.json +++ b/packages/backend-core/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/backend-core", - "version": "2.0.30", + "version": "2.0.31", "description": "Budibase backend core libraries used in server and worker", "main": "dist/src/index.js", "types": "dist/src/index.d.ts", @@ -20,7 +20,7 @@ "test:watch": "jest --watchAll" }, "dependencies": { - "@budibase/types": "^2.0.30", + "@budibase/types": "^2.0.31", "@shopify/jest-koa-mocks": "5.0.1", "@techpass/passport-openidconnect": "0.3.2", "aws-sdk": "2.1030.0", diff --git a/packages/bbui/package.json b/packages/bbui/package.json index 9472f84bb4..7f508ef0e0 100644 --- a/packages/bbui/package.json +++ b/packages/bbui/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/bbui", "description": "A UI solution used in the different Budibase projects.", - "version": "2.0.30", + "version": "2.0.31", "license": "MPL-2.0", "svelte": "src/index.js", "module": "dist/bbui.es.js", @@ -38,7 +38,7 @@ ], "dependencies": { "@adobe/spectrum-css-workflow-icons": "^1.2.1", - "@budibase/string-templates": "^2.0.30", + "@budibase/string-templates": "^2.0.31", "@spectrum-css/actionbutton": "^1.0.1", "@spectrum-css/actiongroup": "^1.0.1", "@spectrum-css/avatar": "^3.0.2", diff --git a/packages/builder/package.json b/packages/builder/package.json index 25d833c378..2f8591ef30 100644 --- a/packages/builder/package.json +++ b/packages/builder/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/builder", - "version": "2.0.30", + "version": "2.0.31", "license": "GPL-3.0", "private": true, "scripts": { @@ -71,10 +71,10 @@ } }, "dependencies": { - "@budibase/bbui": "^2.0.30", - "@budibase/client": "^2.0.30", - "@budibase/frontend-core": "^2.0.30", - "@budibase/string-templates": "^2.0.30", + "@budibase/bbui": "^2.0.31", + "@budibase/client": "^2.0.31", + "@budibase/frontend-core": "^2.0.31", + "@budibase/string-templates": "^2.0.31", "@sentry/browser": "5.19.1", "@spectrum-css/page": "^3.0.1", "@spectrum-css/vars": "^3.0.1", diff --git a/packages/cli/package.json b/packages/cli/package.json index 58e6aea835..1fcac0164a 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/cli", - "version": "2.0.30", + "version": "2.0.31", "description": "Budibase CLI, for developers, self hosting and migrations.", "main": "src/index.js", "bin": { @@ -26,9 +26,9 @@ "outputPath": "build" }, "dependencies": { - "@budibase/backend-core": "^2.0.30", - "@budibase/string-templates": "^2.0.30", - "@budibase/types": "^2.0.30", + "@budibase/backend-core": "^2.0.31", + "@budibase/string-templates": "^2.0.31", + "@budibase/types": "^2.0.31", "axios": "0.21.2", "chalk": "4.1.0", "cli-progress": "3.11.2", diff --git a/packages/client/package.json b/packages/client/package.json index f01e9f4ea5..7434bcc4c6 100644 --- a/packages/client/package.json +++ b/packages/client/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/client", - "version": "2.0.30", + "version": "2.0.31", "license": "MPL-2.0", "module": "dist/budibase-client.js", "main": "dist/budibase-client.js", @@ -19,9 +19,9 @@ "dev:builder": "rollup -cw" }, "dependencies": { - "@budibase/bbui": "^2.0.30", - "@budibase/frontend-core": "^2.0.30", - "@budibase/string-templates": "^2.0.30", + "@budibase/bbui": "^2.0.31", + "@budibase/frontend-core": "^2.0.31", + "@budibase/string-templates": "^2.0.31", "@spectrum-css/button": "^3.0.3", "@spectrum-css/card": "^3.0.3", "@spectrum-css/divider": "^1.0.3", diff --git a/packages/frontend-core/package.json b/packages/frontend-core/package.json index ace3316711..d871e129fc 100644 --- a/packages/frontend-core/package.json +++ b/packages/frontend-core/package.json @@ -1,12 +1,12 @@ { "name": "@budibase/frontend-core", - "version": "2.0.30", + "version": "2.0.31", "description": "Budibase frontend core libraries used in builder and client", "author": "Budibase", "license": "MPL-2.0", "svelte": "src/index.js", "dependencies": { - "@budibase/bbui": "^2.0.30", + "@budibase/bbui": "^2.0.31", "lodash": "^4.17.21", "svelte": "^3.46.2" } diff --git a/packages/sdk/package.json b/packages/sdk/package.json index 438bcf8fb0..c993a6b75d 100644 --- a/packages/sdk/package.json +++ b/packages/sdk/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/sdk", - "version": "2.0.30", + "version": "2.0.31", "description": "Budibase Public API SDK", "author": "Budibase", "license": "MPL-2.0", diff --git a/packages/server/package.json b/packages/server/package.json index e2242dd1ef..0c2a1dd20b 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/server", "email": "hi@budibase.com", - "version": "2.0.30", + "version": "2.0.31", "description": "Budibase Web Server", "main": "src/index.ts", "repository": { @@ -77,11 +77,11 @@ "license": "GPL-3.0", "dependencies": { "@apidevtools/swagger-parser": "10.0.3", - "@budibase/backend-core": "^2.0.30", - "@budibase/client": "^2.0.30", + "@budibase/backend-core": "^2.0.31", + "@budibase/client": "^2.0.31", "@budibase/pro": "2.0.30", - "@budibase/string-templates": "^2.0.30", - "@budibase/types": "^2.0.30", + "@budibase/string-templates": "^2.0.31", + "@budibase/types": "^2.0.31", "@bull-board/api": "3.7.0", "@bull-board/koa": "3.9.4", "@elastic/elasticsearch": "7.10.0", diff --git a/packages/string-templates/package.json b/packages/string-templates/package.json index 64b3e52bf7..970bf98e8c 100644 --- a/packages/string-templates/package.json +++ b/packages/string-templates/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/string-templates", - "version": "2.0.30", + "version": "2.0.31", "description": "Handlebars wrapper for Budibase templating.", "main": "src/index.cjs", "module": "dist/bundle.mjs", diff --git a/packages/types/package.json b/packages/types/package.json index 209294dbee..a7d589b405 100644 --- a/packages/types/package.json +++ b/packages/types/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/types", - "version": "2.0.30", + "version": "2.0.31", "description": "Budibase types", "main": "dist/index.js", "types": "dist/index.d.ts", diff --git a/packages/worker/package.json b/packages/worker/package.json index 6723ba89e7..8bc2e41db8 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/worker", "email": "hi@budibase.com", - "version": "2.0.30", + "version": "2.0.31", "description": "Budibase background service", "main": "src/index.ts", "repository": { @@ -36,10 +36,10 @@ "author": "Budibase", "license": "GPL-3.0", "dependencies": { - "@budibase/backend-core": "^2.0.30", + "@budibase/backend-core": "^2.0.31", "@budibase/pro": "2.0.30", - "@budibase/string-templates": "^2.0.30", - "@budibase/types": "^2.0.30", + "@budibase/string-templates": "^2.0.31", + "@budibase/types": "^2.0.31", "@koa/router": "8.0.8", "@sentry/node": "6.17.7", "@techpass/passport-openidconnect": "0.3.2", From b035a708f2933aee8a6f0e1e38936770d19008e3 Mon Sep 17 00:00:00 2001 From: Budibase Release Bot <> Date: Fri, 14 Oct 2022 09:30:15 +0000 Subject: [PATCH 032/151] Update pro version to 2.0.31 --- packages/server/package.json | 2 +- packages/server/yarn.lock | 30 +++++++++++++++--------------- packages/worker/package.json | 2 +- packages/worker/yarn.lock | 30 +++++++++++++++--------------- 4 files changed, 32 insertions(+), 32 deletions(-) diff --git a/packages/server/package.json b/packages/server/package.json index 0c2a1dd20b..ca9d1a17bc 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -79,7 +79,7 @@ "@apidevtools/swagger-parser": "10.0.3", "@budibase/backend-core": "^2.0.31", "@budibase/client": "^2.0.31", - "@budibase/pro": "2.0.30", + "@budibase/pro": "2.0.31", "@budibase/string-templates": "^2.0.31", "@budibase/types": "^2.0.31", "@bull-board/api": "3.7.0", diff --git a/packages/server/yarn.lock b/packages/server/yarn.lock index 35555f7f27..eaf629b010 100644 --- a/packages/server/yarn.lock +++ b/packages/server/yarn.lock @@ -1094,12 +1094,12 @@ resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw== -"@budibase/backend-core@2.0.30": - version "2.0.30" - resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-2.0.30.tgz#5cc53e8560572d789ae687ec2181b629526c2f3f" - integrity sha512-+X4DwtMf1Xv4ygG7OJN0wOxZgjJ1wul1gUVOXJm44sRIEOx0LHpyx8xT4npC9e2VKKezpF5GZFE2IrB9lwWsoQ== +"@budibase/backend-core@2.0.31": + version "2.0.31" + resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-2.0.31.tgz#199fddbbaf987dde7477f12e7f9ab7f4435bdafd" + integrity sha512-te7fMsEWflKbPiv8vxh+U31562Yn4W3vUBcHUm3Oer38rv0nprUJoyxO1YGBVcOJVNAEtNAtEdoPayFezU3YOw== dependencies: - "@budibase/types" "^2.0.30" + "@budibase/types" "^2.0.31" "@shopify/jest-koa-mocks" "5.0.1" "@techpass/passport-openidconnect" "0.3.2" aws-sdk "2.1030.0" @@ -1180,13 +1180,13 @@ svelte-flatpickr "^3.2.3" svelte-portal "^1.0.0" -"@budibase/pro@2.0.30": - version "2.0.30" - resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.0.30.tgz#4c739ed97e04d6dfc875dfb550262ef7085dc9d4" - integrity sha512-unel/xYLpeWUI+I0vstITH9UTVkjKS+SS0Ce4jCQlpMyOSB8zCie/sO6Ie2TakrI+vnuYuypnRT8fqlbgtrNyA== +"@budibase/pro@2.0.31": + version "2.0.31" + resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.0.31.tgz#238f3057f9fac654efeb65ec8fcce83caa364ac2" + integrity sha512-VUx6PT/1RqCp0UNANmooJcKeMrGQNCM/rbMhOvvtbqvWhRa1sYn00RdXTAzzy++plOS8vue+f/BFewrr1yYlug== dependencies: - "@budibase/backend-core" "2.0.30" - "@budibase/types" "2.0.30" + "@budibase/backend-core" "2.0.31" + "@budibase/types" "2.0.31" "@koa/router" "8.0.8" joi "17.6.0" node-fetch "^2.6.1" @@ -1209,10 +1209,10 @@ svelte-apexcharts "^1.0.2" svelte-flatpickr "^3.1.0" -"@budibase/types@2.0.30", "@budibase/types@^2.0.30": - version "2.0.30" - resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.0.30.tgz#ac3d2d714ebf2820875150739959e5f6b851aceb" - integrity sha512-46uCHPcLAAjAFRkE53skS7oLjI/Fa+2Mn4vZjIApa1/0JXyR2HnzTzJocbrQfYFTYLkdniMfLRticcKC/E2X1w== +"@budibase/types@2.0.31", "@budibase/types@^2.0.31": + version "2.0.31" + resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.0.31.tgz#699ccd746cff39d4439e6f01a62c38a9f1a98bad" + integrity sha512-Eyx9wEVi4ZHrJ6410aM98IW7EWrsmYJ0ixmm/wD2tIhb57N33YgFWobsdR/G1wJUTZnR76Ve2tAweDgXmvtscQ== "@bull-board/api@3.7.0": version "3.7.0" diff --git a/packages/worker/package.json b/packages/worker/package.json index 8bc2e41db8..e4379020b7 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -37,7 +37,7 @@ "license": "GPL-3.0", "dependencies": { "@budibase/backend-core": "^2.0.31", - "@budibase/pro": "2.0.30", + "@budibase/pro": "2.0.31", "@budibase/string-templates": "^2.0.31", "@budibase/types": "^2.0.31", "@koa/router": "8.0.8", diff --git a/packages/worker/yarn.lock b/packages/worker/yarn.lock index b9bfaed432..0d6085ef5a 100644 --- a/packages/worker/yarn.lock +++ b/packages/worker/yarn.lock @@ -291,12 +291,12 @@ resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw== -"@budibase/backend-core@2.0.30": - version "2.0.30" - resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-2.0.30.tgz#5cc53e8560572d789ae687ec2181b629526c2f3f" - integrity sha512-+X4DwtMf1Xv4ygG7OJN0wOxZgjJ1wul1gUVOXJm44sRIEOx0LHpyx8xT4npC9e2VKKezpF5GZFE2IrB9lwWsoQ== +"@budibase/backend-core@2.0.31": + version "2.0.31" + resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-2.0.31.tgz#199fddbbaf987dde7477f12e7f9ab7f4435bdafd" + integrity sha512-te7fMsEWflKbPiv8vxh+U31562Yn4W3vUBcHUm3Oer38rv0nprUJoyxO1YGBVcOJVNAEtNAtEdoPayFezU3YOw== dependencies: - "@budibase/types" "^2.0.30" + "@budibase/types" "^2.0.31" "@shopify/jest-koa-mocks" "5.0.1" "@techpass/passport-openidconnect" "0.3.2" aws-sdk "2.1030.0" @@ -327,21 +327,21 @@ uuid "8.3.2" zlib "1.0.5" -"@budibase/pro@2.0.30": - version "2.0.30" - resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.0.30.tgz#4c739ed97e04d6dfc875dfb550262ef7085dc9d4" - integrity sha512-unel/xYLpeWUI+I0vstITH9UTVkjKS+SS0Ce4jCQlpMyOSB8zCie/sO6Ie2TakrI+vnuYuypnRT8fqlbgtrNyA== +"@budibase/pro@2.0.31": + version "2.0.31" + resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.0.31.tgz#238f3057f9fac654efeb65ec8fcce83caa364ac2" + integrity sha512-VUx6PT/1RqCp0UNANmooJcKeMrGQNCM/rbMhOvvtbqvWhRa1sYn00RdXTAzzy++plOS8vue+f/BFewrr1yYlug== dependencies: - "@budibase/backend-core" "2.0.30" - "@budibase/types" "2.0.30" + "@budibase/backend-core" "2.0.31" + "@budibase/types" "2.0.31" "@koa/router" "8.0.8" joi "17.6.0" node-fetch "^2.6.1" -"@budibase/types@2.0.30", "@budibase/types@^2.0.30": - version "2.0.30" - resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.0.30.tgz#ac3d2d714ebf2820875150739959e5f6b851aceb" - integrity sha512-46uCHPcLAAjAFRkE53skS7oLjI/Fa+2Mn4vZjIApa1/0JXyR2HnzTzJocbrQfYFTYLkdniMfLRticcKC/E2X1w== +"@budibase/types@2.0.31", "@budibase/types@^2.0.31": + version "2.0.31" + resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.0.31.tgz#699ccd746cff39d4439e6f01a62c38a9f1a98bad" + integrity sha512-Eyx9wEVi4ZHrJ6410aM98IW7EWrsmYJ0ixmm/wD2tIhb57N33YgFWobsdR/G1wJUTZnR76Ve2tAweDgXmvtscQ== "@cspotcode/source-map-consumer@0.8.0": version "0.8.0" From b702c7482a5bdf8270137ead8ffaec4fbb6abc40 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 14 Oct 2022 13:26:42 +0100 Subject: [PATCH 033/151] Fixes for cronjob stop - correctly handle this without stalled job handle. --- packages/backend-core/src/queue/listeners.ts | 21 +++++++-- packages/backend-core/src/queue/queue.ts | 2 +- packages/server/src/automations/utils.ts | 10 ++-- .../server/src/definitions/automations.ts | 12 ----- packages/server/src/threads/automation.ts | 47 ++++++++----------- 5 files changed, 44 insertions(+), 48 deletions(-) diff --git a/packages/backend-core/src/queue/listeners.ts b/packages/backend-core/src/queue/listeners.ts index c5db628ef0..0e06fb5ef0 100644 --- a/packages/backend-core/src/queue/listeners.ts +++ b/packages/backend-core/src/queue/listeners.ts @@ -3,24 +3,35 @@ import { JobQueue } from "./constants" export type StalledFn = (job: Job) => Promise -export const addListeners = ( +export function addListeners( queue: Queue, jobQueue: JobQueue, removeStalled?: StalledFn -) => { +) { logging(queue, jobQueue) if (removeStalled) { handleStalled(queue, removeStalled) } } -const handleStalled = (queue: Queue, removeStalled: StalledFn) => { +function handleStalled(queue: Queue, removeStalled?: StalledFn) { queue.on("stalled", async (job: Job) => { - await removeStalled(job) + if (removeStalled) { + await removeStalled(job) + } else if (job.opts.repeat) { + const jobId = job.id + const repeatJobs = await queue.getRepeatableJobs() + for (let repeatJob of repeatJobs) { + if (repeatJob.id === jobId) { + await queue.removeRepeatableByKey(repeatJob.key) + } + } + console.log(`jobId=${jobId} disabled`) + } }) } -const logging = (queue: Queue, jobQueue: JobQueue) => { +function logging(queue: Queue, jobQueue: JobQueue) { let eventType: string switch (jobQueue) { case JobQueue.AUTOMATIONS: diff --git a/packages/backend-core/src/queue/queue.ts b/packages/backend-core/src/queue/queue.ts index 8f7ac79e7a..d83e421456 100644 --- a/packages/backend-core/src/queue/queue.ts +++ b/packages/backend-core/src/queue/queue.ts @@ -18,7 +18,7 @@ async function cleanup() { export function createQueue( jobQueue: JobQueue, - removeStalled: StalledFn + removeStalled?: StalledFn ): BullQueue.Queue { const queueConfig: any = redisProtocolUrl || { redis: opts } let queue: any diff --git a/packages/server/src/automations/utils.ts b/packages/server/src/automations/utils.ts index 54b8078f30..0eebcb21cf 100644 --- a/packages/server/src/automations/utils.ts +++ b/packages/server/src/automations/utils.ts @@ -91,9 +91,13 @@ export async function disableAllCrons(appId: any) { return Promise.all(promises) } -export async function disableCron(jobId: string, jobKey: string) { - await automationQueue.removeRepeatableByKey(jobKey) - await automationQueue.removeJobs(jobId) +export async function disableCronById(jobId: number | string) { + const repeatJobs = await automationQueue.getRepeatableJobs() + for (let repeatJob of repeatJobs) { + if (repeatJob.id === jobId) { + await automationQueue.removeRepeatableByKey(repeatJob.key) + } + } console.log(`jobId=${jobId} disabled`) } diff --git a/packages/server/src/definitions/automations.ts b/packages/server/src/definitions/automations.ts index ed1455c049..877a1b4579 100644 --- a/packages/server/src/definitions/automations.ts +++ b/packages/server/src/definitions/automations.ts @@ -27,18 +27,6 @@ export interface TriggerOutput { timestamp?: number } -export interface AutomationEvent { - data: { - automation: Automation - event: any - } - opts?: { - repeat?: { - jobId: string - } - } -} - export interface AutomationContext extends AutomationResults { steps: any[] trigger: any diff --git a/packages/server/src/threads/automation.ts b/packages/server/src/threads/automation.ts index 64ae9439d8..b4b290462e 100644 --- a/packages/server/src/threads/automation.ts +++ b/packages/server/src/threads/automation.ts @@ -1,6 +1,11 @@ import { default as threadUtils } from "./utils" +import { Job } from "bull" threadUtils.threadSetup() -import { isRecurring, disableCron, isErrorInOutput } from "../automations/utils" +import { + isRecurring, + disableCronById, + isErrorInOutput, +} from "../automations/utils" import { default as actions } from "../automations/actions" import { default as automationUtils } from "../automations/automationUtils" import { default as AutomationEmitter } from "../events/AutomationEmitter" @@ -13,7 +18,6 @@ import { LoopStep, LoopStepType, LoopInput, - AutomationEvent, TriggerOutput, AutomationContext, AutomationMetadata, @@ -73,19 +77,16 @@ class Orchestrator { _automation: Automation _emitter: any _context: AutomationContext - _repeat?: { jobId: string; jobKey: string } + _job: Job executionOutput: AutomationContext - constructor(automation: Automation, triggerOutput: TriggerOutput, opts: any) { + constructor(job: Job) { + let automation = job.data.automation, + triggerOutput = job.data.event const metadata = triggerOutput.metadata this._chainCount = metadata ? metadata.automationChainCount : 0 this._appId = triggerOutput.appId as string - if (opts?.repeat) { - this._repeat = { - jobId: opts.repeat.jobId, - jobKey: opts.repeat.key, - } - } + this._job = job const triggerStepId = automation.definition.trigger.stepId triggerOutput = this.cleanupTriggerOutputs(triggerStepId, triggerOutput) // remove from context @@ -134,7 +135,7 @@ class Orchestrator { } async stopCron(reason: string) { - if (!this._repeat) { + if (!this._job.opts.repeat) { return } logWarn( @@ -142,7 +143,7 @@ class Orchestrator { ) const automation = this._automation const trigger = automation.definition.trigger - await disableCron(this._repeat?.jobId, this._repeat?.jobKey) + await disableCronById(this._job.id) this.updateExecutionOutput( trigger.id, trigger.stepId, @@ -156,7 +157,7 @@ class Orchestrator { } async checkIfShouldStop(metadata: AutomationMetadata): Promise { - if (!metadata.errorCount || !this._repeat) { + if (!metadata.errorCount || !this._job.opts.repeat) { return false } if (metadata.errorCount >= MAX_AUTOMATION_RECURRING_ERRORS) { @@ -475,17 +476,13 @@ class Orchestrator { } } -export function execute(input: AutomationEvent, callback: WorkerCallback) { - const appId = input.data.event.appId +export function execute(job: Job, callback: WorkerCallback) { + const appId = job.data.event.appId if (!appId) { throw new Error("Unable to execute, event doesn't contain app ID.") } doInAppContext(appId, async () => { - const automationOrchestrator = new Orchestrator( - input.data.automation, - input.data.event, - input.opts - ) + const automationOrchestrator = new Orchestrator(job) try { const response = await automationOrchestrator.execute() callback(null, response) @@ -495,17 +492,13 @@ export function execute(input: AutomationEvent, callback: WorkerCallback) { }) } -export const removeStalled = async (input: AutomationEvent) => { - const appId = input.data.event.appId +export const removeStalled = async (job: Job) => { + const appId = job.data.event.appId if (!appId) { throw new Error("Unable to execute, event doesn't contain app ID.") } await doInAppContext(appId, async () => { - const automationOrchestrator = new Orchestrator( - input.data.automation, - input.data.event, - input.opts - ) + const automationOrchestrator = new Orchestrator(job) await automationOrchestrator.stopCron("stalled") }) } From 0bd2a18e46f09c51a75d19f6161a1a3c0bfb358a Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 14 Oct 2022 19:24:03 +0100 Subject: [PATCH 034/151] Main types and work for the CRUD operations of app backup backend in pro + the listeners to handle exporting apps from the server. --- packages/backend-core/package.json | 1 + packages/backend-core/src/db/constants.ts | 2 + .../backend-core/src/objectStore/index.ts | 3 +- packages/backend-core/src/queue/constants.ts | 4 +- packages/backend-core/src/queue/listeners.ts | 4 +- packages/backend-core/src/queue/queue.ts | 4 +- packages/backend-core/yarn.lock | 10 ++++- packages/server/src/app.ts | 2 + packages/server/src/automations/bullboard.js | 2 +- packages/server/src/sdk/app/backups/backup.ts | 39 +++++++++++++++++++ packages/server/src/sdk/app/backups/index.ts | 2 + packages/types/src/api/web/app/backup.ts | 1 + packages/types/src/documents/app/backup.ts | 37 +++++++++++++----- 13 files changed, 93 insertions(+), 18 deletions(-) create mode 100644 packages/server/src/sdk/app/backups/backup.ts diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json index eb90fa4159..17977decba 100644 --- a/packages/backend-core/package.json +++ b/packages/backend-core/package.json @@ -63,6 +63,7 @@ ] }, "devDependencies": { + "@types/bull": "^3.15.9", "@types/chance": "1.1.3", "@types/ioredis": "^4.28.10", "@types/jest": "27.5.1", diff --git a/packages/backend-core/src/db/constants.ts b/packages/backend-core/src/db/constants.ts index a61e8a2af2..8f8f45638b 100644 --- a/packages/backend-core/src/db/constants.ts +++ b/packages/backend-core/src/db/constants.ts @@ -21,6 +21,7 @@ export enum ViewName { ACCOUNT_BY_EMAIL = "account_by_email", PLATFORM_USERS_LOWERCASE = "platform_users_lowercase", USER_BY_GROUP = "by_group_user", + APP_BACKUP_BY_TRIGGER = "by_trigger", } export const DeprecatedViews = { @@ -49,6 +50,7 @@ export enum DocumentType { TABLE = "ta", DATASOURCE = "datasource", DATASOURCE_PLUS = "datasource_plus", + APP_BACKUP = "backup", } export const StaticDatabases = { diff --git a/packages/backend-core/src/objectStore/index.ts b/packages/backend-core/src/objectStore/index.ts index e210f9dccc..8453c9aee6 100644 --- a/packages/backend-core/src/objectStore/index.ts +++ b/packages/backend-core/src/objectStore/index.ts @@ -27,6 +27,7 @@ const CONTENT_TYPE_MAP: any = { css: "text/css", js: "application/javascript", json: "application/json", + gz: "application/gzip", } const STRING_CONTENT_TYPES = [ CONTENT_TYPE_MAP.html, @@ -149,7 +150,7 @@ export const upload = async ({ type, metadata, }: any) => { - const extension = [...filename.split(".")].pop() + const extension = filename.split(".").pop() const fileBytes = fs.readFileSync(path) const objectStore = ObjectStore(bucketName) diff --git a/packages/backend-core/src/queue/constants.ts b/packages/backend-core/src/queue/constants.ts index d8fb3121a3..e8323dacb8 100644 --- a/packages/backend-core/src/queue/constants.ts +++ b/packages/backend-core/src/queue/constants.ts @@ -1,4 +1,4 @@ export enum JobQueue { - AUTOMATIONS = "automationQueue", - APP_BACKUPS = "appBackupQueue", + AUTOMATION = "automationQueue", + APP_BACKUP = "appBackupQueue", } diff --git a/packages/backend-core/src/queue/listeners.ts b/packages/backend-core/src/queue/listeners.ts index 0e06fb5ef0..f264c3a84c 100644 --- a/packages/backend-core/src/queue/listeners.ts +++ b/packages/backend-core/src/queue/listeners.ts @@ -34,10 +34,10 @@ function handleStalled(queue: Queue, removeStalled?: StalledFn) { function logging(queue: Queue, jobQueue: JobQueue) { let eventType: string switch (jobQueue) { - case JobQueue.AUTOMATIONS: + case JobQueue.AUTOMATION: eventType = "automation-event" break - case JobQueue.APP_BACKUPS: + case JobQueue.APP_BACKUP: eventType = "app-backup-event" break } diff --git a/packages/backend-core/src/queue/queue.ts b/packages/backend-core/src/queue/queue.ts index d83e421456..6dd10ee091 100644 --- a/packages/backend-core/src/queue/queue.ts +++ b/packages/backend-core/src/queue/queue.ts @@ -16,10 +16,10 @@ async function cleanup() { } } -export function createQueue( +export function createQueue( jobQueue: JobQueue, removeStalled?: StalledFn -): BullQueue.Queue { +): BullQueue.Queue { const queueConfig: any = redisProtocolUrl || { redis: opts } let queue: any if (env.isTest()) { diff --git a/packages/backend-core/yarn.lock b/packages/backend-core/yarn.lock index 22b93d7b3e..d2831ca8fe 100644 --- a/packages/backend-core/yarn.lock +++ b/packages/backend-core/yarn.lock @@ -698,6 +698,14 @@ "@types/connect" "*" "@types/node" "*" +"@types/bull@^3.15.9": + version "3.15.9" + resolved "https://registry.yarnpkg.com/@types/bull/-/bull-3.15.9.tgz#e10e0901ec3762bff85716b3c580277960751c93" + integrity sha512-MPUcyPPQauAmynoO3ezHAmCOhbB0pWmYyijr/5ctaCqhbKWsjW0YCod38ZcLzUBprosfZ9dPqfYIcfdKjk7RNQ== + dependencies: + "@types/ioredis" "*" + "@types/redis" "^2.8.0" + "@types/chance@1.1.3": version "1.1.3" resolved "https://registry.yarnpkg.com/@types/chance/-/chance-1.1.3.tgz#d19fe9391288d60fdccd87632bfc9ab2b4523fea" @@ -768,7 +776,7 @@ resolved "https://registry.yarnpkg.com/@types/http-errors/-/http-errors-1.8.2.tgz#7315b4c4c54f82d13fa61c228ec5c2ea5cc9e0e1" integrity sha512-EqX+YQxINb+MeXaIqYDASb6U6FCHbWjkj4a1CKDBks3d/QiB2+PqBLyO72vLDgAO1wUI4O+9gweRcQK11bTL/w== -"@types/ioredis@^4.28.10": +"@types/ioredis@*", "@types/ioredis@^4.28.10": version "4.28.10" resolved "https://registry.yarnpkg.com/@types/ioredis/-/ioredis-4.28.10.tgz#40ceb157a4141088d1394bb87c98ed09a75a06ff" integrity sha512-69LyhUgrXdgcNDv7ogs1qXZomnfOEnSmrmMFqKgt1XMJxmoOSG/u3wYy13yACIfKuMJ8IhKgHafDO3sx19zVQQ== diff --git a/packages/server/src/app.ts b/packages/server/src/app.ts index 3bfe8976f7..776adb602d 100644 --- a/packages/server/src/app.ts +++ b/packages/server/src/app.ts @@ -37,6 +37,7 @@ import { } from "./utilities/workerRequests" import { watch } from "./watch" import { initialise as initialiseWebsockets } from "./websocket" +import sdk from "./sdk" const app = new Koa() @@ -108,6 +109,7 @@ module.exports = server.listen(env.PORT || 0, async () => { eventEmitter.emitPort(env.PORT) fileSystem.init() await redis.init() + await sdk.backups.init() // run migrations on startup if not done via http // not recommended in a clustered environment diff --git a/packages/server/src/automations/bullboard.js b/packages/server/src/automations/bullboard.js index af2243f13d..c4f33e07a9 100644 --- a/packages/server/src/automations/bullboard.js +++ b/packages/server/src/automations/bullboard.js @@ -5,7 +5,7 @@ const { queue } = require("@budibase/backend-core") const automation = require("../threads/automation") let automationQueue = queue.createQueue( - queue.JobQueue.AUTOMATIONS, + queue.JobQueue.AUTOMATION, automation.removeStalled ) diff --git a/packages/server/src/sdk/app/backups/backup.ts b/packages/server/src/sdk/app/backups/backup.ts new file mode 100644 index 0000000000..6a0d370653 --- /dev/null +++ b/packages/server/src/sdk/app/backups/backup.ts @@ -0,0 +1,39 @@ +import { backups } from "@budibase/pro" +import { objectStore, tenancy } from "@budibase/backend-core" +import { exportApp } from "./exports" +import { Job } from "bull" +import fs from "fs" +import env from "../../../environment" + +export async function init() { + await backups.addAppBackupProcessor(async (job: Job) => { + const appId = job.data.appId, + trigger = job.data.trigger, + name = job.data.name + const createdAt = new Date().toISOString() + const tarPath = await exportApp(appId, { tar: true }) + let filename = `${appId}/backup-${createdAt}.tar.gz` + // add the tenant to the bucket path if backing up within a multi-tenant environment + if (env.MULTI_TENANCY) { + const tenantId = tenancy.getTenantIDFromAppID(appId) + filename = `${tenantId}/${filename}` + } + const bucket = objectStore.ObjectStoreBuckets.BACKUPS + const metadata = { + appId, + createdAt, + trigger, + name, + } + await objectStore.upload({ + path: tarPath, + type: "application/gzip", + bucket, + filename, + metadata, + }) + await backups.storeAppBackupMetadata(filename, metadata) + // clear up the tarball after uploading it + fs.rmSync(tarPath) + }) +} diff --git a/packages/server/src/sdk/app/backups/index.ts b/packages/server/src/sdk/app/backups/index.ts index fe7e4e1049..94210c2c4c 100644 --- a/packages/server/src/sdk/app/backups/index.ts +++ b/packages/server/src/sdk/app/backups/index.ts @@ -1,7 +1,9 @@ import * as exportApps from "./exports" import * as importApps from "./imports" +import * as backup from "./backup" export default { ...exportApps, ...importApps, + ...backup, } diff --git a/packages/types/src/api/web/app/backup.ts b/packages/types/src/api/web/app/backup.ts index 57ffba0d70..092cf714b5 100644 --- a/packages/types/src/api/web/app/backup.ts +++ b/packages/types/src/api/web/app/backup.ts @@ -4,6 +4,7 @@ export interface SearchAppBackupsRequest { trigger: AppBackupTrigger startDate: string endDate: string + page?: string } export interface CreateAppBackupRequest { diff --git a/packages/types/src/documents/app/backup.ts b/packages/types/src/documents/app/backup.ts index a935ed5ba1..fb8ef61e3a 100644 --- a/packages/types/src/documents/app/backup.ts +++ b/packages/types/src/documents/app/backup.ts @@ -6,16 +6,35 @@ export enum AppBackupTrigger { SCHEDULED = "scheduled", } -export interface AppBackupContents { - datasources: string[] - screens: string[] - automations: string[] -} - export interface AppBackup extends Document { trigger: AppBackupTrigger name: string - date: string - userId: string - contents: AppBackupContents + createdAt: string + filename: string + appId: string + userId?: string + contents?: { + datasources: string[] + screens: string[] + automations: string[] + } +} + +export type AppBackupFetchOpts = { + trigger?: AppBackupTrigger + limit?: number + page?: string + paginate?: boolean + startDate?: string + endDate?: string +} + +export interface AppBackupQueueData { + trigger: AppBackupTrigger + name?: string + appId: string +} + +export interface AppBackupMetadata extends AppBackupQueueData { + createdAt: string } From 1373630b33af7739e55c6e79fe424f580616e431 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 14 Oct 2022 20:10:44 +0100 Subject: [PATCH 035/151] Minor fixes after testing manual backup system. --- packages/backend-core/src/context/index.ts | 3 ++ packages/server/src/api/routes/index.ts | 10 ++-- packages/server/src/sdk/app/backups/backup.ts | 50 ++++++++++--------- packages/types/src/sdk/koa.ts | 9 +++- 4 files changed, 41 insertions(+), 31 deletions(-) diff --git a/packages/backend-core/src/context/index.ts b/packages/backend-core/src/context/index.ts index 35eeee608b..7efe0e23f7 100644 --- a/packages/backend-core/src/context/index.ts +++ b/packages/backend-core/src/context/index.ts @@ -53,6 +53,9 @@ export const getTenantIDFromAppID = (appId: string) => { if (!appId) { return null } + if (!isMultiTenant()) { + return DEFAULT_TENANT_ID + } const split = appId.split(SEPARATOR) const hasDev = split[1] === DocumentType.DEV if ((hasDev && split.length === 3) || (!hasDev && split.length === 2)) { diff --git a/packages/server/src/api/routes/index.ts b/packages/server/src/api/routes/index.ts index 1cf34d4a68..02a4900077 100644 --- a/packages/server/src/api/routes/index.ts +++ b/packages/server/src/api/routes/index.ts @@ -34,6 +34,8 @@ export { default as publicRoutes } from "./public" const appBackupRoutes = api.appBackups const scheduleRoutes = api.schedules export const mainRoutes: Router[] = [ + appBackupRoutes, + backupRoutes, authRoutes, deployRoutes, layoutRoutes, @@ -53,16 +55,14 @@ export const mainRoutes: Router[] = [ permissionRoutes, datasourceRoutes, queryRoutes, - backupRoutes, metadataRoutes, devRoutes, cloudRoutes, - // these need to be handled last as they still use /api/:tableId - // this could be breaking as koa may recognise other routes as this - tableRoutes, rowRoutes, migrationRoutes, pluginRoutes, - appBackupRoutes, scheduleRoutes, + // these need to be handled last as they still use /api/:tableId + // this could be breaking as koa may recognise other routes as this + tableRoutes, ] diff --git a/packages/server/src/sdk/app/backups/backup.ts b/packages/server/src/sdk/app/backups/backup.ts index 6a0d370653..fb4bcc6022 100644 --- a/packages/server/src/sdk/app/backups/backup.ts +++ b/packages/server/src/sdk/app/backups/backup.ts @@ -10,30 +10,32 @@ export async function init() { const appId = job.data.appId, trigger = job.data.trigger, name = job.data.name - const createdAt = new Date().toISOString() - const tarPath = await exportApp(appId, { tar: true }) - let filename = `${appId}/backup-${createdAt}.tar.gz` - // add the tenant to the bucket path if backing up within a multi-tenant environment - if (env.MULTI_TENANCY) { - const tenantId = tenancy.getTenantIDFromAppID(appId) - filename = `${tenantId}/${filename}` - } - const bucket = objectStore.ObjectStoreBuckets.BACKUPS - const metadata = { - appId, - createdAt, - trigger, - name, - } - await objectStore.upload({ - path: tarPath, - type: "application/gzip", - bucket, - filename, - metadata, + const tenantId = tenancy.getTenantIDFromAppID(appId) + await tenancy.doInTenant(tenantId, async () => { + const createdAt = new Date().toISOString() + const tarPath = await exportApp(appId, { tar: true }) + let filename = `${appId}/backup-${createdAt}.tar.gz` + // add the tenant to the bucket path if backing up within a multi-tenant environment + if (env.MULTI_TENANCY) { + filename = `${tenantId}/${filename}` + } + const bucket = objectStore.ObjectStoreBuckets.BACKUPS + const metadata = { + appId, + createdAt, + trigger, + name, + } + await objectStore.upload({ + path: tarPath, + type: "application/gzip", + bucket, + filename, + metadata, + }) + await backups.storeAppBackupMetadata(filename, metadata) + // clear up the tarball after uploading it + fs.rmSync(tarPath) }) - await backups.storeAppBackupMetadata(filename, metadata) - // clear up the tarball after uploading it - fs.rmSync(tarPath) }) } diff --git a/packages/types/src/sdk/koa.ts b/packages/types/src/sdk/koa.ts index 8d419d5cf1..250b176d79 100644 --- a/packages/types/src/sdk/koa.ts +++ b/packages/types/src/sdk/koa.ts @@ -7,7 +7,12 @@ export interface ContextUser extends User { license: License } -export interface BBContext extends Context { +export interface BBContext { user?: ContextUser - body: any + request: { + body: any + } + params: any + body?: any + redirect?: any } From ecb47851d60760ac09867435bd53d95d04ca08a7 Mon Sep 17 00:00:00 2001 From: Jonny McCullagh Date: Mon, 17 Oct 2022 10:20:09 +0100 Subject: [PATCH 036/151] make proxy ipv6 customise script executable --- hosting/proxy/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/hosting/proxy/Dockerfile b/hosting/proxy/Dockerfile index a9c94c06fa..f11480fd73 100644 --- a/hosting/proxy/Dockerfile +++ b/hosting/proxy/Dockerfile @@ -6,6 +6,7 @@ FROM nginx:latest ENV NGINX_ENVSUBST_OUTPUT_DIR=/etc/nginx COPY .generated-nginx.prod.conf /etc/nginx/templates/nginx.conf.template COPY 10-listen-on-ipv6-by-default.sh /docker-entrypoint.d/10-listen-on-ipv6-by-default.sh +RUN chmod +x /docker-entrypoint.d/10-listen-on-ipv6-by-default.sh # Error handling COPY error.html /usr/share/nginx/html/error.html From bb4e3ba8cfa6cc95578b111df7bcf5a4b19381bd Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 17 Oct 2022 15:26:09 +0100 Subject: [PATCH 037/151] Adding in required events for app backup system. --- .../backend-core/src/events/publishers/backup.ts | 12 ++++++++++++ packages/backend-core/src/events/publishers/index.ts | 1 + packages/types/src/sdk/events/backup.ts | 7 +++++++ packages/types/src/sdk/events/event.ts | 3 +++ packages/types/src/sdk/events/index.ts | 1 + 5 files changed, 24 insertions(+) create mode 100644 packages/backend-core/src/events/publishers/backup.ts create mode 100644 packages/types/src/sdk/events/backup.ts diff --git a/packages/backend-core/src/events/publishers/backup.ts b/packages/backend-core/src/events/publishers/backup.ts new file mode 100644 index 0000000000..00b4f8db69 --- /dev/null +++ b/packages/backend-core/src/events/publishers/backup.ts @@ -0,0 +1,12 @@ +import { AppBackup, AppBackupRevertEvent, Event } from "@budibase/types" +import { publishEvent } from "../events" + +export async function appBackupRestored(backup: AppBackup) { + const properties: AppBackupRevertEvent = { + appId: backup.appId, + backupName: backup.name, + backupCreatedAt: backup.createdAt, + } + + await publishEvent(Event.APP_BACKUP_RESTORED, properties) +} diff --git a/packages/backend-core/src/events/publishers/index.ts b/packages/backend-core/src/events/publishers/index.ts index 6fe42c4bda..7306312a8f 100644 --- a/packages/backend-core/src/events/publishers/index.ts +++ b/packages/backend-core/src/events/publishers/index.ts @@ -19,3 +19,4 @@ export * as installation from "./installation" export * as backfill from "./backfill" export * as group from "./group" export * as plugin from "./plugin" +export * as backup from "./backup" diff --git a/packages/types/src/sdk/events/backup.ts b/packages/types/src/sdk/events/backup.ts new file mode 100644 index 0000000000..f3ddafcafc --- /dev/null +++ b/packages/types/src/sdk/events/backup.ts @@ -0,0 +1,7 @@ +import { BaseEvent } from "./event" + +export interface AppBackupRevertEvent extends BaseEvent { + appId: string + backupName: string + backupCreatedAt: string +} diff --git a/packages/types/src/sdk/events/event.ts b/packages/types/src/sdk/events/event.ts index 73e5315713..71caf2bf96 100644 --- a/packages/types/src/sdk/events/event.ts +++ b/packages/types/src/sdk/events/event.ts @@ -168,6 +168,9 @@ export enum Event { PLUGIN_INIT = "plugin:init", PLUGIN_IMPORTED = "plugin:imported", PLUGIN_DELETED = "plugin:deleted", + + // BACKUP + APP_BACKUP_RESTORED = "app:backup:restored", } // properties added at the final stage of the event pipeline diff --git a/packages/types/src/sdk/events/index.ts b/packages/types/src/sdk/events/index.ts index cc0c2b9aa1..5abc30f5b9 100644 --- a/packages/types/src/sdk/events/index.ts +++ b/packages/types/src/sdk/events/index.ts @@ -20,3 +20,4 @@ export * from "./backfill" export * from "./identification" export * from "./userGroup" export * from "./plugin" +export * from "./backup" From 65184704090de0fa8c1685269f419b1330c10e14 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Mon, 17 Oct 2022 17:32:23 +0100 Subject: [PATCH 038/151] updating variable name --- charts/budibase/templates/app-service-deployment.yaml | 2 +- charts/budibase/templates/worker-service-deployment.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/budibase/templates/app-service-deployment.yaml b/charts/budibase/templates/app-service-deployment.yaml index 25a5dd410a..af2a3ed544 100644 --- a/charts/budibase/templates/app-service-deployment.yaml +++ b/charts/budibase/templates/app-service-deployment.yaml @@ -83,7 +83,7 @@ spec: - name: APPS_BUCKET_NAME value: {{ .Values.services.objectStore.appsBucketName | default "apps" | quote }} - name: GLOBAL_CLOUD_BUCKET_NAME - value: {{ .Values.services.objectStore.globalCloudBucketName | default "global" | quote }} + value: {{ .Values.services.objectStore.globalBucketName | default "global" | quote }} - name: PORT value: {{ .Values.services.apps.port | quote }} {{ if .Values.services.worker.publicApiRateLimitPerSecond }} diff --git a/charts/budibase/templates/worker-service-deployment.yaml b/charts/budibase/templates/worker-service-deployment.yaml index 1284b9c411..e7dccfae1c 100644 --- a/charts/budibase/templates/worker-service-deployment.yaml +++ b/charts/budibase/templates/worker-service-deployment.yaml @@ -82,7 +82,7 @@ spec: - name: APPS_BUCKET_NAME value: {{ .Values.services.objectStore.appsBucketName | default "apps" | quote }} - name: GLOBAL_CLOUD_BUCKET_NAME - value: {{ .Values.services.objectStore.globalCloudBucketName | default "global" | quote }} + value: {{ .Values.services.objectStore.globalBucketName | default "global" | quote }} - name: PORT value: {{ .Values.services.worker.port | quote }} - name: MULTI_TENANCY From 873b20219cc03880be89f4cf352dbe208af72736 Mon Sep 17 00:00:00 2001 From: Budibase Release Bot <> Date: Mon, 17 Oct 2022 16:43:56 +0000 Subject: [PATCH 039/151] v2.0.32 --- lerna.json | 2 +- packages/backend-core/package.json | 4 ++-- packages/bbui/package.json | 4 ++-- packages/builder/package.json | 10 +++++----- packages/cli/package.json | 8 ++++---- packages/client/package.json | 8 ++++---- packages/frontend-core/package.json | 4 ++-- packages/sdk/package.json | 2 +- packages/server/package.json | 10 +++++----- packages/string-templates/package.json | 2 +- packages/types/package.json | 2 +- packages/worker/package.json | 8 ++++---- 12 files changed, 32 insertions(+), 32 deletions(-) diff --git a/lerna.json b/lerna.json index 7d825bcefb..824ebe4902 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.0.31", + "version": "2.0.32", "npmClient": "yarn", "packages": [ "packages/*" diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json index 1c3957949b..cba777c4c0 100644 --- a/packages/backend-core/package.json +++ b/packages/backend-core/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/backend-core", - "version": "2.0.31", + "version": "2.0.32", "description": "Budibase backend core libraries used in server and worker", "main": "dist/src/index.js", "types": "dist/src/index.d.ts", @@ -20,7 +20,7 @@ "test:watch": "jest --watchAll" }, "dependencies": { - "@budibase/types": "^2.0.31", + "@budibase/types": "^2.0.32", "@shopify/jest-koa-mocks": "5.0.1", "@techpass/passport-openidconnect": "0.3.2", "aws-sdk": "2.1030.0", diff --git a/packages/bbui/package.json b/packages/bbui/package.json index 7f508ef0e0..1300a954c3 100644 --- a/packages/bbui/package.json +++ b/packages/bbui/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/bbui", "description": "A UI solution used in the different Budibase projects.", - "version": "2.0.31", + "version": "2.0.32", "license": "MPL-2.0", "svelte": "src/index.js", "module": "dist/bbui.es.js", @@ -38,7 +38,7 @@ ], "dependencies": { "@adobe/spectrum-css-workflow-icons": "^1.2.1", - "@budibase/string-templates": "^2.0.31", + "@budibase/string-templates": "^2.0.32", "@spectrum-css/actionbutton": "^1.0.1", "@spectrum-css/actiongroup": "^1.0.1", "@spectrum-css/avatar": "^3.0.2", diff --git a/packages/builder/package.json b/packages/builder/package.json index 2f8591ef30..09e9dcd27a 100644 --- a/packages/builder/package.json +++ b/packages/builder/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/builder", - "version": "2.0.31", + "version": "2.0.32", "license": "GPL-3.0", "private": true, "scripts": { @@ -71,10 +71,10 @@ } }, "dependencies": { - "@budibase/bbui": "^2.0.31", - "@budibase/client": "^2.0.31", - "@budibase/frontend-core": "^2.0.31", - "@budibase/string-templates": "^2.0.31", + "@budibase/bbui": "^2.0.32", + "@budibase/client": "^2.0.32", + "@budibase/frontend-core": "^2.0.32", + "@budibase/string-templates": "^2.0.32", "@sentry/browser": "5.19.1", "@spectrum-css/page": "^3.0.1", "@spectrum-css/vars": "^3.0.1", diff --git a/packages/cli/package.json b/packages/cli/package.json index 1fcac0164a..1e4515b822 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/cli", - "version": "2.0.31", + "version": "2.0.32", "description": "Budibase CLI, for developers, self hosting and migrations.", "main": "src/index.js", "bin": { @@ -26,9 +26,9 @@ "outputPath": "build" }, "dependencies": { - "@budibase/backend-core": "^2.0.31", - "@budibase/string-templates": "^2.0.31", - "@budibase/types": "^2.0.31", + "@budibase/backend-core": "^2.0.32", + "@budibase/string-templates": "^2.0.32", + "@budibase/types": "^2.0.32", "axios": "0.21.2", "chalk": "4.1.0", "cli-progress": "3.11.2", diff --git a/packages/client/package.json b/packages/client/package.json index 7434bcc4c6..32409da298 100644 --- a/packages/client/package.json +++ b/packages/client/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/client", - "version": "2.0.31", + "version": "2.0.32", "license": "MPL-2.0", "module": "dist/budibase-client.js", "main": "dist/budibase-client.js", @@ -19,9 +19,9 @@ "dev:builder": "rollup -cw" }, "dependencies": { - "@budibase/bbui": "^2.0.31", - "@budibase/frontend-core": "^2.0.31", - "@budibase/string-templates": "^2.0.31", + "@budibase/bbui": "^2.0.32", + "@budibase/frontend-core": "^2.0.32", + "@budibase/string-templates": "^2.0.32", "@spectrum-css/button": "^3.0.3", "@spectrum-css/card": "^3.0.3", "@spectrum-css/divider": "^1.0.3", diff --git a/packages/frontend-core/package.json b/packages/frontend-core/package.json index d871e129fc..e0cef709b7 100644 --- a/packages/frontend-core/package.json +++ b/packages/frontend-core/package.json @@ -1,12 +1,12 @@ { "name": "@budibase/frontend-core", - "version": "2.0.31", + "version": "2.0.32", "description": "Budibase frontend core libraries used in builder and client", "author": "Budibase", "license": "MPL-2.0", "svelte": "src/index.js", "dependencies": { - "@budibase/bbui": "^2.0.31", + "@budibase/bbui": "^2.0.32", "lodash": "^4.17.21", "svelte": "^3.46.2" } diff --git a/packages/sdk/package.json b/packages/sdk/package.json index c993a6b75d..7903b47981 100644 --- a/packages/sdk/package.json +++ b/packages/sdk/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/sdk", - "version": "2.0.31", + "version": "2.0.32", "description": "Budibase Public API SDK", "author": "Budibase", "license": "MPL-2.0", diff --git a/packages/server/package.json b/packages/server/package.json index ca9d1a17bc..5574b4ca88 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/server", "email": "hi@budibase.com", - "version": "2.0.31", + "version": "2.0.32", "description": "Budibase Web Server", "main": "src/index.ts", "repository": { @@ -77,11 +77,11 @@ "license": "GPL-3.0", "dependencies": { "@apidevtools/swagger-parser": "10.0.3", - "@budibase/backend-core": "^2.0.31", - "@budibase/client": "^2.0.31", + "@budibase/backend-core": "^2.0.32", + "@budibase/client": "^2.0.32", "@budibase/pro": "2.0.31", - "@budibase/string-templates": "^2.0.31", - "@budibase/types": "^2.0.31", + "@budibase/string-templates": "^2.0.32", + "@budibase/types": "^2.0.32", "@bull-board/api": "3.7.0", "@bull-board/koa": "3.9.4", "@elastic/elasticsearch": "7.10.0", diff --git a/packages/string-templates/package.json b/packages/string-templates/package.json index 970bf98e8c..9fdceec45f 100644 --- a/packages/string-templates/package.json +++ b/packages/string-templates/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/string-templates", - "version": "2.0.31", + "version": "2.0.32", "description": "Handlebars wrapper for Budibase templating.", "main": "src/index.cjs", "module": "dist/bundle.mjs", diff --git a/packages/types/package.json b/packages/types/package.json index a7d589b405..33509d5929 100644 --- a/packages/types/package.json +++ b/packages/types/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/types", - "version": "2.0.31", + "version": "2.0.32", "description": "Budibase types", "main": "dist/index.js", "types": "dist/index.d.ts", diff --git a/packages/worker/package.json b/packages/worker/package.json index e4379020b7..eb5ffa2e73 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/worker", "email": "hi@budibase.com", - "version": "2.0.31", + "version": "2.0.32", "description": "Budibase background service", "main": "src/index.ts", "repository": { @@ -36,10 +36,10 @@ "author": "Budibase", "license": "GPL-3.0", "dependencies": { - "@budibase/backend-core": "^2.0.31", + "@budibase/backend-core": "^2.0.32", "@budibase/pro": "2.0.31", - "@budibase/string-templates": "^2.0.31", - "@budibase/types": "^2.0.31", + "@budibase/string-templates": "^2.0.32", + "@budibase/types": "^2.0.32", "@koa/router": "8.0.8", "@sentry/node": "6.17.7", "@techpass/passport-openidconnect": "0.3.2", From 64c09a9888c93e2f9771e02c457e98886ff54a1e Mon Sep 17 00:00:00 2001 From: Budibase Release Bot <> Date: Mon, 17 Oct 2022 16:47:08 +0000 Subject: [PATCH 040/151] Update pro version to 2.0.32 --- packages/server/package.json | 2 +- packages/server/yarn.lock | 30 +++++++++++++++--------------- packages/worker/package.json | 2 +- packages/worker/yarn.lock | 30 +++++++++++++++--------------- 4 files changed, 32 insertions(+), 32 deletions(-) diff --git a/packages/server/package.json b/packages/server/package.json index 5574b4ca88..9f49ac5767 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -79,7 +79,7 @@ "@apidevtools/swagger-parser": "10.0.3", "@budibase/backend-core": "^2.0.32", "@budibase/client": "^2.0.32", - "@budibase/pro": "2.0.31", + "@budibase/pro": "2.0.32", "@budibase/string-templates": "^2.0.32", "@budibase/types": "^2.0.32", "@bull-board/api": "3.7.0", diff --git a/packages/server/yarn.lock b/packages/server/yarn.lock index eaf629b010..fb3d88015c 100644 --- a/packages/server/yarn.lock +++ b/packages/server/yarn.lock @@ -1094,12 +1094,12 @@ resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw== -"@budibase/backend-core@2.0.31": - version "2.0.31" - resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-2.0.31.tgz#199fddbbaf987dde7477f12e7f9ab7f4435bdafd" - integrity sha512-te7fMsEWflKbPiv8vxh+U31562Yn4W3vUBcHUm3Oer38rv0nprUJoyxO1YGBVcOJVNAEtNAtEdoPayFezU3YOw== +"@budibase/backend-core@2.0.32": + version "2.0.32" + resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-2.0.32.tgz#638c89302cc834d7cc11c5ce97bb3305392b9c85" + integrity sha512-4Pmh1VFmXlwnh5B5Jq75h+i5hVvOEQmDRVRvbsNk5sYAk/xxVbf7/kIQJ0OuhuTA+WfY8EpQLDEnHGhdTeFhWA== dependencies: - "@budibase/types" "^2.0.31" + "@budibase/types" "^2.0.32" "@shopify/jest-koa-mocks" "5.0.1" "@techpass/passport-openidconnect" "0.3.2" aws-sdk "2.1030.0" @@ -1180,13 +1180,13 @@ svelte-flatpickr "^3.2.3" svelte-portal "^1.0.0" -"@budibase/pro@2.0.31": - version "2.0.31" - resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.0.31.tgz#238f3057f9fac654efeb65ec8fcce83caa364ac2" - integrity sha512-VUx6PT/1RqCp0UNANmooJcKeMrGQNCM/rbMhOvvtbqvWhRa1sYn00RdXTAzzy++plOS8vue+f/BFewrr1yYlug== +"@budibase/pro@2.0.32": + version "2.0.32" + resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.0.32.tgz#b091d5fda894cdce179f97441b9ed69ba7859f84" + integrity sha512-5LNXUGu+J6pH8iNfkv2jWxyBJhek0M3KtAJvTspREeK5BZ+dxHR2AxceIksKVFy3Tco7mGs2haSRQ0qActIgCg== dependencies: - "@budibase/backend-core" "2.0.31" - "@budibase/types" "2.0.31" + "@budibase/backend-core" "2.0.32" + "@budibase/types" "2.0.32" "@koa/router" "8.0.8" joi "17.6.0" node-fetch "^2.6.1" @@ -1209,10 +1209,10 @@ svelte-apexcharts "^1.0.2" svelte-flatpickr "^3.1.0" -"@budibase/types@2.0.31", "@budibase/types@^2.0.31": - version "2.0.31" - resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.0.31.tgz#699ccd746cff39d4439e6f01a62c38a9f1a98bad" - integrity sha512-Eyx9wEVi4ZHrJ6410aM98IW7EWrsmYJ0ixmm/wD2tIhb57N33YgFWobsdR/G1wJUTZnR76Ve2tAweDgXmvtscQ== +"@budibase/types@2.0.32", "@budibase/types@^2.0.32": + version "2.0.32" + resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.0.32.tgz#dda8989d7e59cdae8088447c568cabd7ea99dee5" + integrity sha512-ALTVPOdRlGFzI8AvYJcBncQiLxqM3coZV0Jc5fCJbSNZ9pObHxibhkCunGSevdEOC1l5p/vT7/B5bUD4fUvnsg== "@bull-board/api@3.7.0": version "3.7.0" diff --git a/packages/worker/package.json b/packages/worker/package.json index eb5ffa2e73..de35922a91 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -37,7 +37,7 @@ "license": "GPL-3.0", "dependencies": { "@budibase/backend-core": "^2.0.32", - "@budibase/pro": "2.0.31", + "@budibase/pro": "2.0.32", "@budibase/string-templates": "^2.0.32", "@budibase/types": "^2.0.32", "@koa/router": "8.0.8", diff --git a/packages/worker/yarn.lock b/packages/worker/yarn.lock index 0d6085ef5a..4955805d4e 100644 --- a/packages/worker/yarn.lock +++ b/packages/worker/yarn.lock @@ -291,12 +291,12 @@ resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw== -"@budibase/backend-core@2.0.31": - version "2.0.31" - resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-2.0.31.tgz#199fddbbaf987dde7477f12e7f9ab7f4435bdafd" - integrity sha512-te7fMsEWflKbPiv8vxh+U31562Yn4W3vUBcHUm3Oer38rv0nprUJoyxO1YGBVcOJVNAEtNAtEdoPayFezU3YOw== +"@budibase/backend-core@2.0.32": + version "2.0.32" + resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-2.0.32.tgz#638c89302cc834d7cc11c5ce97bb3305392b9c85" + integrity sha512-4Pmh1VFmXlwnh5B5Jq75h+i5hVvOEQmDRVRvbsNk5sYAk/xxVbf7/kIQJ0OuhuTA+WfY8EpQLDEnHGhdTeFhWA== dependencies: - "@budibase/types" "^2.0.31" + "@budibase/types" "^2.0.32" "@shopify/jest-koa-mocks" "5.0.1" "@techpass/passport-openidconnect" "0.3.2" aws-sdk "2.1030.0" @@ -327,21 +327,21 @@ uuid "8.3.2" zlib "1.0.5" -"@budibase/pro@2.0.31": - version "2.0.31" - resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.0.31.tgz#238f3057f9fac654efeb65ec8fcce83caa364ac2" - integrity sha512-VUx6PT/1RqCp0UNANmooJcKeMrGQNCM/rbMhOvvtbqvWhRa1sYn00RdXTAzzy++plOS8vue+f/BFewrr1yYlug== +"@budibase/pro@2.0.32": + version "2.0.32" + resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.0.32.tgz#b091d5fda894cdce179f97441b9ed69ba7859f84" + integrity sha512-5LNXUGu+J6pH8iNfkv2jWxyBJhek0M3KtAJvTspREeK5BZ+dxHR2AxceIksKVFy3Tco7mGs2haSRQ0qActIgCg== dependencies: - "@budibase/backend-core" "2.0.31" - "@budibase/types" "2.0.31" + "@budibase/backend-core" "2.0.32" + "@budibase/types" "2.0.32" "@koa/router" "8.0.8" joi "17.6.0" node-fetch "^2.6.1" -"@budibase/types@2.0.31", "@budibase/types@^2.0.31": - version "2.0.31" - resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.0.31.tgz#699ccd746cff39d4439e6f01a62c38a9f1a98bad" - integrity sha512-Eyx9wEVi4ZHrJ6410aM98IW7EWrsmYJ0ixmm/wD2tIhb57N33YgFWobsdR/G1wJUTZnR76Ve2tAweDgXmvtscQ== +"@budibase/types@2.0.32", "@budibase/types@^2.0.32": + version "2.0.32" + resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.0.32.tgz#dda8989d7e59cdae8088447c568cabd7ea99dee5" + integrity sha512-ALTVPOdRlGFzI8AvYJcBncQiLxqM3coZV0Jc5fCJbSNZ9pObHxibhkCunGSevdEOC1l5p/vT7/B5bUD4fUvnsg== "@cspotcode/source-map-consumer@0.8.0": version "0.8.0" From 8003f8b28342bb33564204e74d8f5f87508141e9 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 17 Oct 2022 18:50:52 +0100 Subject: [PATCH 041/151] Work for backup download endpoint. --- packages/types/src/documents/app/backup.ts | 2 ++ packages/types/src/sdk/koa.ts | 1 + 2 files changed, 3 insertions(+) diff --git a/packages/types/src/documents/app/backup.ts b/packages/types/src/documents/app/backup.ts index fb8ef61e3a..5fb602860e 100644 --- a/packages/types/src/documents/app/backup.ts +++ b/packages/types/src/documents/app/backup.ts @@ -10,6 +10,7 @@ export interface AppBackup extends Document { trigger: AppBackupTrigger name: string createdAt: string + createdBy?: string filename: string appId: string userId?: string @@ -31,6 +32,7 @@ export type AppBackupFetchOpts = { export interface AppBackupQueueData { trigger: AppBackupTrigger + createdBy?: string name?: string appId: string } diff --git a/packages/types/src/sdk/koa.ts b/packages/types/src/sdk/koa.ts index 250b176d79..e6f4a5094e 100644 --- a/packages/types/src/sdk/koa.ts +++ b/packages/types/src/sdk/koa.ts @@ -15,4 +15,5 @@ export interface BBContext { params: any body?: any redirect?: any + attachment: any } From f795cb0e33b727f09029934d41af9d8fc1b2b06c Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 17 Oct 2022 19:42:36 +0100 Subject: [PATCH 042/151] Reformatting types to allow queue to be used for import and export. --- packages/server/src/sdk/app/backups/backup.ts | 66 ++++++++++--------- .../server/src/sdk/app/backups/imports.ts | 2 +- packages/types/src/documents/app/backup.ts | 23 +++++-- 3 files changed, 55 insertions(+), 36 deletions(-) diff --git a/packages/server/src/sdk/app/backups/backup.ts b/packages/server/src/sdk/app/backups/backup.ts index fb4bcc6022..cb758536bd 100644 --- a/packages/server/src/sdk/app/backups/backup.ts +++ b/packages/server/src/sdk/app/backups/backup.ts @@ -5,37 +5,41 @@ import { Job } from "bull" import fs from "fs" import env from "../../../environment" -export async function init() { - await backups.addAppBackupProcessor(async (job: Job) => { - const appId = job.data.appId, - trigger = job.data.trigger, - name = job.data.name - const tenantId = tenancy.getTenantIDFromAppID(appId) - await tenancy.doInTenant(tenantId, async () => { - const createdAt = new Date().toISOString() - const tarPath = await exportApp(appId, { tar: true }) - let filename = `${appId}/backup-${createdAt}.tar.gz` - // add the tenant to the bucket path if backing up within a multi-tenant environment - if (env.MULTI_TENANCY) { - filename = `${tenantId}/${filename}` - } - const bucket = objectStore.ObjectStoreBuckets.BACKUPS - const metadata = { - appId, - createdAt, - trigger, - name, - } - await objectStore.upload({ - path: tarPath, - type: "application/gzip", - bucket, - filename, - metadata, - }) - await backups.storeAppBackupMetadata(filename, metadata) - // clear up the tarball after uploading it - fs.rmSync(tarPath) +async function importProcessor(job: Job) {} + +async function exportProcessor(job: Job) { + const appId = job.data.appId, + trigger = job.data.trigger, + name = job.data.name + const tenantId = tenancy.getTenantIDFromAppID(appId) + await tenancy.doInTenant(tenantId, async () => { + const createdAt = new Date().toISOString() + const tarPath = await exportApp(appId, { tar: true }) + let filename = `${appId}/backup-${createdAt}.tar.gz` + // add the tenant to the bucket path if backing up within a multi-tenant environment + if (env.MULTI_TENANCY) { + filename = `${tenantId}/${filename}` + } + const bucket = objectStore.ObjectStoreBuckets.BACKUPS + const metadata = { + appId, + createdAt, + trigger, + name, + } + await objectStore.upload({ + path: tarPath, + type: "application/gzip", + bucket, + filename, + metadata, }) + await backups.storeAppBackupMetadata(filename, metadata) + // clear up the tarball after uploading it + fs.rmSync(tarPath) }) } + +export async function init() { + await backups.addAppBackupProcessors(importProcessor, exportProcessor) +} diff --git a/packages/server/src/sdk/app/backups/imports.ts b/packages/server/src/sdk/app/backups/imports.ts index d09c4b3f02..60ce63d51e 100644 --- a/packages/server/src/sdk/app/backups/imports.ts +++ b/packages/server/src/sdk/app/backups/imports.ts @@ -1,5 +1,5 @@ import { db as dbCore } from "@budibase/backend-core" -import { APP_PREFIX, TABLE_ROW_PREFIX } from "../../../db/utils" +import { TABLE_ROW_PREFIX } from "../../../db/utils" import { budibaseTempDir } from "../../../utilities/budibaseDir" import { DB_EXPORT_FILE, diff --git a/packages/types/src/documents/app/backup.ts b/packages/types/src/documents/app/backup.ts index 5fb602860e..28e927c772 100644 --- a/packages/types/src/documents/app/backup.ts +++ b/packages/types/src/documents/app/backup.ts @@ -6,6 +6,11 @@ export enum AppBackupTrigger { SCHEDULED = "scheduled", } +export enum AppBackupEventType { + EXPORT = "export", + IMPORT = "import", +} + export interface AppBackup extends Document { trigger: AppBackupTrigger name: string @@ -31,12 +36,22 @@ export type AppBackupFetchOpts = { } export interface AppBackupQueueData { - trigger: AppBackupTrigger - createdBy?: string - name?: string + eventType: AppBackupEventType appId: string + export?: { + trigger: AppBackupTrigger + name?: string + createdBy?: string + } + import?: { + backupId: string + } } -export interface AppBackupMetadata extends AppBackupQueueData { +export interface AppBackupMetadata { + appId: string + trigger: AppBackupTrigger + name?: string + createdBy?: string createdAt: string } From d1e729ec55e96c988be85a69f986d78ac64b1f6b Mon Sep 17 00:00:00 2001 From: Jonny McCullagh Date: Tue, 18 Oct 2022 09:33:04 +0100 Subject: [PATCH 043/151] proxy IPv6 removal needs to happen after envsubst --- ...n-ipv6-by-default.sh => 80-listen-on-ipv6-by-default.sh} | 0 hosting/proxy/Dockerfile | 6 ++++-- 2 files changed, 4 insertions(+), 2 deletions(-) rename hosting/proxy/{10-listen-on-ipv6-by-default.sh => 80-listen-on-ipv6-by-default.sh} (100%) diff --git a/hosting/proxy/10-listen-on-ipv6-by-default.sh b/hosting/proxy/80-listen-on-ipv6-by-default.sh similarity index 100% rename from hosting/proxy/10-listen-on-ipv6-by-default.sh rename to hosting/proxy/80-listen-on-ipv6-by-default.sh diff --git a/hosting/proxy/Dockerfile b/hosting/proxy/Dockerfile index f11480fd73..c1b11b23f7 100644 --- a/hosting/proxy/Dockerfile +++ b/hosting/proxy/Dockerfile @@ -5,8 +5,10 @@ FROM nginx:latest # override the output dir to output directly to /etc/nginx instead of /etc/nginx/conf.d ENV NGINX_ENVSUBST_OUTPUT_DIR=/etc/nginx COPY .generated-nginx.prod.conf /etc/nginx/templates/nginx.conf.template -COPY 10-listen-on-ipv6-by-default.sh /docker-entrypoint.d/10-listen-on-ipv6-by-default.sh -RUN chmod +x /docker-entrypoint.d/10-listen-on-ipv6-by-default.sh +# IPv6 removal needs to happen after envsubst +RUN rm -rf /docker-entrypoint.d/10-listen-on-ipv6-by-default.sh +COPY 80-listen-on-ipv6-by-default.sh /docker-entrypoint.d/80-listen-on-ipv6-by-default.sh +RUN chmod +x /docker-entrypoint.d/80-listen-on-ipv6-by-default.sh # Error handling COPY error.html /usr/share/nginx/html/error.html From 15bbc788473c384fc3ed28f6476325de3a6cf6e0 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Tue, 18 Oct 2022 16:18:22 +0100 Subject: [PATCH 044/151] Add grid functionality separately to DND --- .../client/src/components/Component.svelte | 5 +- .../client/src/components/app/Grid.svelte | 45 +- .../src/components/preview/DNDHandler.svelte | 39 +- .../components/preview/GridDNDHandler.svelte | 656 ++++-------------- .../components/preview/HoverIndicator.svelte | 4 +- .../src/components/preview/Indicator.svelte | 72 +- .../components/preview/IndicatorSet.svelte | 21 + .../preview/SelectionIndicator.svelte | 5 +- .../src/components/preview/SettingsBar.svelte | 4 +- packages/client/src/index.js | 2 - packages/client/src/stores/builder.js | 7 + packages/client/src/stores/dnd.js | 12 - packages/client/src/stores/index.js | 1 - 13 files changed, 251 insertions(+), 622 deletions(-) diff --git a/packages/client/src/components/Component.svelte b/packages/client/src/components/Component.svelte index 742b93bb42..fe99c3e4cd 100644 --- a/packages/client/src/components/Component.svelte +++ b/packages/client/src/components/Component.svelte @@ -21,7 +21,6 @@ devToolsStore, componentStore, appStore, - dndIsDragging, dndComponentPath, } from "stores" import { Helpers } from "@budibase/bbui" @@ -163,7 +162,7 @@ // nested layers. Only reset this when dragging stops. let pad = false $: pad = pad || (interactive && hasChildren && inDndPath) - $: $dndIsDragging, (pad = false) + $: $builderStore.dragging, (pad = false) // Update component context $: store.set({ @@ -427,7 +426,7 @@ const scrollIntoView = () => { // Don't scroll into view if we selected this component because we were // starting dragging on it - if (get(dndIsDragging)) { + if (get(builderStore).dragging) { return } const node = document.getElementsByClassName(id)?.[0]?.children[0] diff --git a/packages/client/src/components/app/Grid.svelte b/packages/client/src/components/app/Grid.svelte index 51c5162386..9a588dbf17 100644 --- a/packages/client/src/components/app/Grid.svelte +++ b/packages/client/src/components/app/Grid.svelte @@ -1,11 +1,14 @@ -
+
{#each coords as coord}
{/each}
- {#if $builderStore.isDragging} -
- {#each coords as coord} -
- {/each} -
- {/if}
+{#if $builderStore.inBuilder && node} + +{/if} + diff --git a/packages/client/src/components/preview/SettingsBar.svelte b/packages/client/src/components/preview/SettingsBar.svelte index 8ef0c81af0..757d8b88db 100644 --- a/packages/client/src/components/preview/SettingsBar.svelte +++ b/packages/client/src/components/preview/SettingsBar.svelte @@ -3,7 +3,7 @@ import SettingsButton from "./SettingsButton.svelte" import SettingsColorPicker from "./SettingsColorPicker.svelte" import SettingsPicker from "./SettingsPicker.svelte" - import { builderStore, componentStore } from "stores" + import { builderStore, componentStore, dndIsDragging } from "stores" import { domDebounce } from "utils/domDebounce" const verticalOffset = 36 @@ -16,7 +16,7 @@ let measured = false $: definition = $componentStore.selectedComponentDefinition - $: showBar = definition?.showSettingsBar && !$builderStore.dragging + $: showBar = definition?.showSettingsBar && !$dndIsDragging $: settings = getBarSettings(definition) const getBarSettings = definition => { diff --git a/packages/client/src/stores/builder.js b/packages/client/src/stores/builder.js index 41fd9bb92d..30fcc39bad 100644 --- a/packages/client/src/stores/builder.js +++ b/packages/client/src/stores/builder.js @@ -19,7 +19,6 @@ const createBuilderStore = () => { navigation: null, hiddenComponentIds: [], usedPlugins: null, - dragging: false, // Legacy - allow the builder to specify a layout layout: null, @@ -41,8 +40,8 @@ const createBuilderStore = () => { updateProp: (prop, value) => { dispatchEvent("update-prop", { prop, value }) }, - updateStyles: styles => { - dispatchEvent("update-styles", { styles }) + updateStyles: (styles, id) => { + dispatchEvent("update-styles", { styles, id }) }, keyDown: (key, ctrlKey) => { dispatchEvent("key-down", { key, ctrlKey }) @@ -112,12 +111,6 @@ const createBuilderStore = () => { // Notify the builder so we can reload component definitions dispatchEvent("reload-plugin") }, - setDragging: dragging => { - store.update(state => { - state.dragging = dragging - return state - }) - }, } return { ...store, diff --git a/packages/client/src/stores/dnd.js b/packages/client/src/stores/dnd.js index d9363e4725..9f17a5aa90 100644 --- a/packages/client/src/stores/dnd.js +++ b/packages/client/src/stores/dnd.js @@ -87,3 +87,4 @@ export const dndIsNewComponent = derived( dndStore, $dndStore => $dndStore.source?.newComponentType != null ) +export const dndIsDragging = derived(dndStore, $dndStore => !!$dndStore.source) diff --git a/packages/client/src/stores/index.js b/packages/client/src/stores/index.js index e28ebf7bc4..c431302d43 100644 --- a/packages/client/src/stores/index.js +++ b/packages/client/src/stores/index.js @@ -21,6 +21,7 @@ export { dndParent, dndBounds, dndIsNewComponent, + dndIsDragging, } from "./dnd" // Context stores are layered and duplicated, so it is not a singleton From 81aa2fbfcc3e4b7c7962939bddb9c66994df195b Mon Sep 17 00:00:00 2001 From: Peter Clement Date: Tue, 18 Oct 2022 18:38:49 +0100 Subject: [PATCH 048/151] add base ui and backups tab --- .../overview/backups/ActionsRenderer.svelte | 25 +++ .../backups/AutomationsRenderer.svelte | 20 +++ .../portal/overview/backups/BackupsTab.svelte | 165 ++++++++++++++++++ .../overview/backups/CreateBackupModal.svelte | 21 +++ .../backups/DatasourceRenderer.svelte | 19 ++ .../overview/backups/ScreensRenderer.svelte | 19 ++ .../overview/backups/TriggerRenderer.svelte | 27 +++ .../overview/[application]/index.svelte | 9 +- 8 files changed, 300 insertions(+), 5 deletions(-) create mode 100644 packages/builder/src/components/portal/overview/backups/ActionsRenderer.svelte create mode 100644 packages/builder/src/components/portal/overview/backups/AutomationsRenderer.svelte create mode 100644 packages/builder/src/components/portal/overview/backups/BackupsTab.svelte create mode 100644 packages/builder/src/components/portal/overview/backups/CreateBackupModal.svelte create mode 100644 packages/builder/src/components/portal/overview/backups/DatasourceRenderer.svelte create mode 100644 packages/builder/src/components/portal/overview/backups/ScreensRenderer.svelte create mode 100644 packages/builder/src/components/portal/overview/backups/TriggerRenderer.svelte diff --git a/packages/builder/src/components/portal/overview/backups/ActionsRenderer.svelte b/packages/builder/src/components/portal/overview/backups/ActionsRenderer.svelte new file mode 100644 index 0000000000..5cf9388adc --- /dev/null +++ b/packages/builder/src/components/portal/overview/backups/ActionsRenderer.svelte @@ -0,0 +1,25 @@ + + +
+ Restore + +
+ +
+ + Delete +
+
+ + diff --git a/packages/builder/src/components/portal/overview/backups/AutomationsRenderer.svelte b/packages/builder/src/components/portal/overview/backups/AutomationsRenderer.svelte new file mode 100644 index 0000000000..df9a323c7c --- /dev/null +++ b/packages/builder/src/components/portal/overview/backups/AutomationsRenderer.svelte @@ -0,0 +1,20 @@ + + +
+ +
{value.length}
+
+ + diff --git a/packages/builder/src/components/portal/overview/backups/BackupsTab.svelte b/packages/builder/src/components/portal/overview/backups/BackupsTab.svelte new file mode 100644 index 0000000000..a95bb23ecb --- /dev/null +++ b/packages/builder/src/components/portal/overview/backups/BackupsTab.svelte @@ -0,0 +1,165 @@ + + +
+ +
+ + + {/if} + + + + + + + + diff --git a/packages/builder/src/components/portal/overview/backups/CreateBackupModal.svelte b/packages/builder/src/components/portal/overview/backups/CreateBackupModal.svelte new file mode 100644 index 0000000000..133e0b9443 --- /dev/null +++ b/packages/builder/src/components/portal/overview/backups/CreateBackupModal.svelte @@ -0,0 +1,21 @@ + + + createManualBackup(name)} + title="Create new backup" + confirmText="Create" + > + + diff --git a/packages/builder/src/components/portal/overview/backups/DatasourceRenderer.svelte b/packages/builder/src/components/portal/overview/backups/DatasourceRenderer.svelte new file mode 100644 index 0000000000..69f9aa7929 --- /dev/null +++ b/packages/builder/src/components/portal/overview/backups/DatasourceRenderer.svelte @@ -0,0 +1,19 @@ + + +
+ +
{value.length}
+
+ + diff --git a/packages/builder/src/components/portal/overview/backups/ScreensRenderer.svelte b/packages/builder/src/components/portal/overview/backups/ScreensRenderer.svelte new file mode 100644 index 0000000000..afafec0870 --- /dev/null +++ b/packages/builder/src/components/portal/overview/backups/ScreensRenderer.svelte @@ -0,0 +1,19 @@ + + +
+ +
{value.length}
+
+ + diff --git a/packages/builder/src/components/portal/overview/backups/TriggerRenderer.svelte b/packages/builder/src/components/portal/overview/backups/TriggerRenderer.svelte new file mode 100644 index 0000000000..81b50e7112 --- /dev/null +++ b/packages/builder/src/components/portal/overview/backups/TriggerRenderer.svelte @@ -0,0 +1,27 @@ + + +
+ {#if value === "PUBLISH"} + +
{value}
+ {:else if value === "MANUAL"} + +
{value}
+ {:else if value === "SCHEDULED"} + +
{value}
+ {/if} +
+ + diff --git a/packages/builder/src/pages/builder/portal/overview/[application]/index.svelte b/packages/builder/src/pages/builder/portal/overview/[application]/index.svelte index fe0e2443a2..c7ad77029a 100644 --- a/packages/builder/src/pages/builder/portal/overview/[application]/index.svelte +++ b/packages/builder/src/pages/builder/portal/overview/[application]/index.svelte @@ -33,6 +33,7 @@ import ExportAppModal from "components/start/ExportAppModal.svelte" import { checkIncomingDeploymentStatus } from "components/deploy/utils" import { onDestroy, onMount } from "svelte" + import BackupsTab from "components/portal/overview/backups/BackupsTab.svelte" export let application @@ -323,11 +324,9 @@ {/if} - {#if false} - -
Backups contents
-
- {/if} + + + From a367863d43da6255edc2edf20f65e02aef325dfb Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Tue, 18 Oct 2022 18:49:24 +0100 Subject: [PATCH 049/151] Allow normal DND in and out of grid children --- packages/client/manifest.json | 4 +++- .../client/src/components/preview/DNDHandler.svelte | 11 +++++++---- .../src/components/preview/GridDNDHandler.svelte | 8 +++++++- .../src/components/preview/HoverIndicator.svelte | 4 ---- .../client/src/components/preview/IndicatorSet.svelte | 5 +++-- 5 files changed, 20 insertions(+), 12 deletions(-) diff --git a/packages/client/manifest.json b/packages/client/manifest.json index 80ed9623cd..16f4bc10c6 100644 --- a/packages/client/manifest.json +++ b/packages/client/manifest.json @@ -4573,7 +4573,9 @@ "hasChildren": true, "styles": [ "size" - ] + ], + "illegalChildren": ["grid", "section"], + "allowedDirectChildren": [""] }, "formblock": { "name": "Form Block", diff --git a/packages/client/src/components/preview/DNDHandler.svelte b/packages/client/src/components/preview/DNDHandler.svelte index 43d1d52480..f556346ad2 100644 --- a/packages/client/src/components/preview/DNDHandler.svelte +++ b/packages/client/src/components/preview/DNDHandler.svelte @@ -23,7 +23,10 @@ $: drop = $dndStore.drop const insideGrid = e => { - return e.target?.closest?.(".grid") != null + return e.target + ?.closest?.(".component") + ?.parentNode?.closest?.(".component") + ?.childNodes[0]?.classList.contains("grid") } // Util to get the inner DOM node by a component ID @@ -218,7 +221,7 @@ // Callback when on top of a component. const onDragOver = e => { - if (!source || !target || insideGrid(e)) { + if (!source || !target) { return } handleEvent(e) @@ -226,7 +229,7 @@ // Callback when entering a potential drop target const onDragEnter = e => { - if (!source || insideGrid(e)) { + if (!source) { return } @@ -249,7 +252,7 @@ // Callback when dropping a drag on top of some component const onDrop = e => { - if (!source || !drop?.parent || drop?.index == null || insideGrid(e)) { + if (!source || !drop?.parent || drop?.index == null) { return } diff --git a/packages/client/src/components/preview/GridDNDHandler.svelte b/packages/client/src/components/preview/GridDNDHandler.svelte index 22fecc9d1d..2b1541e1e4 100644 --- a/packages/client/src/components/preview/GridDNDHandler.svelte +++ b/packages/client/src/components/preview/GridDNDHandler.svelte @@ -11,7 +11,13 @@ $: applyStyles(dragNode, gridStyles) const insideGrid = e => { - return e.target?.closest?.(".grid") || e.target.classList.contains("anchor") + return ( + e.target + ?.closest?.(".component") + ?.parentNode?.closest?.(".component") + ?.childNodes[0]?.classList.contains("grid") || + e.target.classList.contains("anchor") + ) } // Util to get the inner DOM node by a component ID diff --git a/packages/client/src/components/preview/HoverIndicator.svelte b/packages/client/src/components/preview/HoverIndicator.svelte index 380520d469..230edeb70c 100644 --- a/packages/client/src/components/preview/HoverIndicator.svelte +++ b/packages/client/src/components/preview/HoverIndicator.svelte @@ -9,21 +9,17 @@ const onMouseOver = e => { // Ignore if dragging if (e.buttons > 0) { - console.log("ignore") return } let newId - if (e.target.classList.contains("anchor")) { // Handle resize anchors newId = e.target.dataset.id - console.log("anchor", newId) } else { // Handle normal components const element = e.target.closest(".interactive.component") newId = element?.dataset?.id - console.log("normal", newId) } if (newId !== componentId) { diff --git a/packages/client/src/components/preview/IndicatorSet.svelte b/packages/client/src/components/preview/IndicatorSet.svelte index dc4355b30f..51a60ce981 100644 --- a/packages/client/src/components/preview/IndicatorSet.svelte +++ b/packages/client/src/components/preview/IndicatorSet.svelte @@ -34,8 +34,9 @@ return false } - // Check if we're a descendent of a grid - return domNode?.closest(".grid") != null + return component?.parentNode + ?.closest?.(".component") + ?.childNodes[0]?.classList.contains("grid") } const createIntersectionCallback = idx => entries => { From 7ae1e3a3eea73cd1cda0beb978faa498df115d66 Mon Sep 17 00:00:00 2001 From: Peter Clement Date: Tue, 18 Oct 2022 19:00:19 +0100 Subject: [PATCH 050/151] add apis and svelte store --- packages/builder/src/stores/portal/backups.js | 40 +++++++++++++++++++ packages/builder/src/stores/portal/index.js | 1 + packages/frontend-core/src/api/backups.js | 20 ++++++++++ packages/frontend-core/src/api/index.js | 2 + 4 files changed, 63 insertions(+) create mode 100644 packages/builder/src/stores/portal/backups.js create mode 100644 packages/frontend-core/src/api/backups.js diff --git a/packages/builder/src/stores/portal/backups.js b/packages/builder/src/stores/portal/backups.js new file mode 100644 index 0000000000..70b973a3e8 --- /dev/null +++ b/packages/builder/src/stores/portal/backups.js @@ -0,0 +1,40 @@ +import { writable } from "svelte/store" +import { API } from "api" + +export function createBackupsStore() { + const { subscribe, set } = writable([]) + + async function load() { + set([ + { + trigger: "PUBLISH", + name: "A Backup", + date: "1665407451", + userId: "Peter Clement", + contents: [ + { datasources: ["datasource1", "datasource2"] }, + { screens: ["screen1", "screen2"] }, + { automations: ["automation1", "automation2"] }, + ], + }, + ]) + } + + async function searchBackups(appId) { + return API.searchBackups(appId) + } + + async function createManualBackup(appId, name) { + let resp = API.createManualBackup(appId, name) + return resp + } + + return { + subscribe, + load, + createManualBackup, + searchBackups, + } +} + +export const backups = createBackupsStore() diff --git a/packages/builder/src/stores/portal/index.js b/packages/builder/src/stores/portal/index.js index fa7aa7e3cf..5406ddc3dc 100644 --- a/packages/builder/src/stores/portal/index.js +++ b/packages/builder/src/stores/portal/index.js @@ -9,3 +9,4 @@ export { templates } from "./templates" export { licensing } from "./licensing" export { groups } from "./groups" export { plugins } from "./plugins" +export { backups } from "./backups" diff --git a/packages/frontend-core/src/api/backups.js b/packages/frontend-core/src/api/backups.js new file mode 100644 index 0000000000..45079e3e7e --- /dev/null +++ b/packages/frontend-core/src/api/backups.js @@ -0,0 +1,20 @@ +export const buildBackupsEndpoints = API => ({ + /** + * Gets a list of users in the current tenant. + */ + searchBackups: async appId => { + return await API.post({ + url: `/api/apps/${appId}/backups/search`, + body: { + trigger: "MANUAL", + }, + }) + }, + + createManualBackup: async ({ appId, name }) => { + return await API.post({ + url: `/api/apps/${appId}/backups`, + body: { name }, + }) + }, +}) diff --git a/packages/frontend-core/src/api/index.js b/packages/frontend-core/src/api/index.js index 3b9bb5b57e..9a40b21351 100644 --- a/packages/frontend-core/src/api/index.js +++ b/packages/frontend-core/src/api/index.js @@ -25,6 +25,7 @@ import { buildViewEndpoints } from "./views" import { buildLicensingEndpoints } from "./licensing" import { buildGroupsEndpoints } from "./groups" import { buildPluginEndpoints } from "./plugins" +import { buildBackupsEndpoints } from "./backups" const defaultAPIClientConfig = { /** @@ -245,5 +246,6 @@ export const createAPIClient = config => { ...buildLicensingEndpoints(API), ...buildGroupsEndpoints(API), ...buildPluginEndpoints(API), + ...buildBackupsEndpoints(API), } } From bdc4e29b2da5d06d75ced004654c7422ffe6a8fc Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 18 Oct 2022 19:43:19 +0100 Subject: [PATCH 051/151] Finishing import processor - download backup, delete dev DB and then import over the top of this. Also includes a rollback feature if the backup fails to restore for whatever reason. --- packages/server/src/sdk/app/backups/backup.ts | 37 +++++++++++++++++-- 1 file changed, 34 insertions(+), 3 deletions(-) diff --git a/packages/server/src/sdk/app/backups/backup.ts b/packages/server/src/sdk/app/backups/backup.ts index 92f70a5a84..cc2338e274 100644 --- a/packages/server/src/sdk/app/backups/backup.ts +++ b/packages/server/src/sdk/app/backups/backup.ts @@ -1,17 +1,48 @@ import { backups } from "@budibase/pro" -import { objectStore, tenancy } from "@budibase/backend-core" +import { objectStore, tenancy, db as dbCore } from "@budibase/backend-core" import { AppBackupQueueData } from "@budibase/types" import { exportApp } from "./exports" +import { importApp } from "./imports" import { Job } from "bull" import fs from "fs" import env from "../../../environment" +async function removeExistingApp(devId: string) { + const devDb = dbCore.dangerousGetDB(devId, { skip_setup: true }) + await devDb.destroy() +} + async function importProcessor(job: Job) { const data: AppBackupQueueData = job.data const appId = data.appId, backupId = data.import!.backupId - const { path, metadata } = await backups.downloadAppBackup(backupId) - // start by removing app database and contents of bucket - which will be updated + const tenantId = tenancy.getTenantIDFromAppID(appId) + tenancy.doInTenant(tenantId, async () => { + const devAppId = dbCore.getDevAppID(appId) + const performImport = async (path: string) => { + await importApp(devAppId, dbCore.dangerousGetDB(devAppId), { + file: { + type: "application/gzip", + path, + }, + key: path, + }) + } + // initially export the current state to disk - incase something goes wrong + const backupTarPath = await exportApp(devAppId, { tar: true }) + // get the backup ready on disk + const { path } = await backups.downloadAppBackup(backupId) + // start by removing app database and contents of bucket - which will be updated + await removeExistingApp(devAppId) + try { + await performImport(path) + } catch (err) { + // rollback - clear up failed import and re-import the pre-backup + await removeExistingApp(devAppId) + await performImport(backupTarPath) + } + fs.rmSync(backupTarPath) + }) } async function exportProcessor(job: Job) { From 71439d4b825e2b4b13729c5058ef0c80fb84d13d Mon Sep 17 00:00:00 2001 From: Peter Clement Date: Wed, 19 Oct 2022 10:16:54 +0100 Subject: [PATCH 052/151] add datepicker range and test search --- packages/bbui/src/Form/Core/DatePicker.svelte | 3 ++- packages/bbui/src/Form/DatePicker.svelte | 3 ++- .../src/components/portal/overview/backups/BackupsTab.svelte | 5 +++++ 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/packages/bbui/src/Form/Core/DatePicker.svelte b/packages/bbui/src/Form/Core/DatePicker.svelte index 1a7ab59818..6f91890d61 100644 --- a/packages/bbui/src/Form/Core/DatePicker.svelte +++ b/packages/bbui/src/Form/Core/DatePicker.svelte @@ -17,7 +17,7 @@ export let timeOnly = false export let ignoreTimezones = false export let time24hr = false - + export let range = false const dispatch = createEventDispatcher() const flatpickrId = `${uuid()}-wrapper` let open = false @@ -41,6 +41,7 @@ time_24hr: time24hr || false, altFormat: timeOnly ? "H:i" : enableTime ? "F j Y, H:i" : "F j, Y", wrap: true, + mode: "range" || null, appendTo, disableMobile: "true", onReady: () => { diff --git a/packages/bbui/src/Form/DatePicker.svelte b/packages/bbui/src/Form/DatePicker.svelte index a0b102dbe8..e2d4547af4 100644 --- a/packages/bbui/src/Form/DatePicker.svelte +++ b/packages/bbui/src/Form/DatePicker.svelte @@ -14,7 +14,7 @@ export let placeholder = null export let appendTo = undefined export let ignoreTimezones = false - + export let range = false const dispatch = createEventDispatcher() const onChange = e => { @@ -34,6 +34,7 @@ {time24hr} {appendTo} {ignoreTimezones} + {range} on:change={onChange} /> diff --git a/packages/builder/src/components/portal/overview/backups/BackupsTab.svelte b/packages/builder/src/components/portal/overview/backups/BackupsTab.svelte index a95bb23ecb..cc47dba3ad 100644 --- a/packages/builder/src/components/portal/overview/backups/BackupsTab.svelte +++ b/packages/builder/src/components/portal/overview/backups/BackupsTab.svelte @@ -1,6 +1,7 @@ - -{#if style} -
-
-
-{/if} - - diff --git a/packages/client/src/components/preview/DNDPlaceholderOverlay.svelte b/packages/client/src/components/preview/DNDPlaceholderOverlay.svelte index 6ed2df6a87..0be7faff1b 100644 --- a/packages/client/src/components/preview/DNDPlaceholderOverlay.svelte +++ b/packages/client/src/components/preview/DNDPlaceholderOverlay.svelte @@ -6,7 +6,8 @@ let left, top, height, width const updatePosition = () => { - const node = document.getElementById(DNDPlaceholderID) + const node = + document.getElementsByClassName(DNDPlaceholderID)[0]?.childNodes[0] if (!node) { height = 0 width = 0 diff --git a/packages/client/src/components/preview/GridDNDHandler.svelte b/packages/client/src/components/preview/GridDNDHandler.svelte index 802a4c9b46..1cb4f92da6 100644 --- a/packages/client/src/components/preview/GridDNDHandler.svelte +++ b/packages/client/src/components/preview/GridDNDHandler.svelte @@ -1,6 +1,6 @@
-
{value.length}
+
{value?.length}
diff --git a/packages/builder/src/components/portal/overview/backups/DaysRenderer.svelte b/packages/builder/src/components/portal/overview/backups/DaysRenderer.svelte new file mode 100644 index 0000000000..c6fe631663 --- /dev/null +++ b/packages/builder/src/components/portal/overview/backups/DaysRenderer.svelte @@ -0,0 +1,18 @@ + + +
+ {value} + {`day${value == 1 ? "" : "s"} ago`} +
+ + diff --git a/packages/builder/src/components/portal/overview/backups/ScreensRenderer.svelte b/packages/builder/src/components/portal/overview/backups/ScreensRenderer.svelte index afafec0870..f192ee98f1 100644 --- a/packages/builder/src/components/portal/overview/backups/ScreensRenderer.svelte +++ b/packages/builder/src/components/portal/overview/backups/ScreensRenderer.svelte @@ -6,7 +6,7 @@
-
{value.length}
+
{value?.length}
diff --git a/packages/builder/src/components/portal/overview/backups/BackupsTab.svelte b/packages/builder/src/components/portal/overview/backups/BackupsTab.svelte index 5f58f2a8d7..9812279b24 100644 --- a/packages/builder/src/components/portal/overview/backups/BackupsTab.svelte +++ b/packages/builder/src/components/portal/overview/backups/BackupsTab.svelte @@ -12,14 +12,13 @@ import { backups } from "stores/portal" import { createPaginationStore } from "helpers/pagination" - import DatasourceRenderer from "./DatasourceRenderer.svelte" - import ScreensRenderer from "./ScreensRenderer.svelte" - import AutomationsRenderer from "./AutomationsRenderer.svelte" + import AppSizeRenderer from "./AppSizeRenderer.svelte" import CreateBackupModal from "./CreateBackupModal.svelte" - import TriggerRenderer from "./TriggerRenderer.svelte" import ActionsRenderer from "./ActionsRenderer.svelte" import DateRenderer from "./DateRenderer.svelte" - import DaysRenderer from "./DaysRenderer.svelte" + import UserRenderer from "./UserRenderer.svelte" + import StatusRenderer from "./StatusRenderer.svelte" + import TypeRenderer from "./TypeRenderer.svelte" export let app @@ -38,66 +37,47 @@ } const schema = { - trigger: { - displayName: "Trigger", - }, - days: { - displayName: null, - }, - - name: { - displayName: "Name", + type: { + displayName: "Type", }, createdAt: { displayName: "Date", }, - datasources: { - displayName: "Data", + name: { + displayName: "Name", }, - screens: { - displayName: "Screens", + appSize: { + displayName: "App size", }, - automations: { - displayName: "Automations", - }, - userId: { + createdBy: { displayName: "User", }, + status: { + displayName: "Status", + }, actions: { displayName: null, }, } const customRenderers = [ - { column: "datasources", component: DatasourceRenderer }, - { column: "screens", component: ScreensRenderer }, - { column: "automations", component: AutomationsRenderer }, - { column: "trigger", component: TriggerRenderer }, + { column: "appSize", component: AppSizeRenderer }, { column: "actions", component: ActionsRenderer }, { column: "createdAt", component: DateRenderer }, - { column: "days", component: DaysRenderer }, + { column: "createdBy", component: UserRenderer }, + { column: "status", component: StatusRenderer }, + { column: "type", component: TypeRenderer }, ] function flattenBackups(backups) { return backups.map(backup => { return { ...backup, - days: getDaysBetween(backup.timestamp), ...backup?.contents, } }) } - function getDaysBetween(date) { - const now = new Date() - const backupDate = new Date(date) - backupDate.setDate(backupDate.getDate() - 1) - const oneDay = 24 * 60 * 60 * 1000 - return now > backupDate - ? Math.round(Math.abs((now - backupDate) / oneDay)) - : 0 - } - async function fetchBackups(trigger, page) { const response = await backups.searchBackups({ appId: app.instance._id, @@ -123,6 +103,7 @@ } async function handleButtonClick({ detail }) { + console.log(detail.type) if (detail.type === "backupDelete") { await backups.deleteBackup({ appId: app.instance._id, diff --git a/packages/builder/src/components/portal/overview/backups/DatasourceRenderer.svelte b/packages/builder/src/components/portal/overview/backups/DatasourceRenderer.svelte index e69fa2f625..198339dae9 100644 --- a/packages/builder/src/components/portal/overview/backups/DatasourceRenderer.svelte +++ b/packages/builder/src/components/portal/overview/backups/DatasourceRenderer.svelte @@ -5,8 +5,10 @@
- -
{value || 0}
+ {#if value != null} + +
{value || 0}
+ {/if}
diff --git a/packages/builder/src/components/portal/overview/backups/ScreensRenderer.svelte b/packages/builder/src/components/portal/overview/backups/ScreensRenderer.svelte deleted file mode 100644 index e97f140da7..0000000000 --- a/packages/builder/src/components/portal/overview/backups/ScreensRenderer.svelte +++ /dev/null @@ -1,19 +0,0 @@ - - -
- -
{value || 0}
-
- - diff --git a/packages/builder/src/components/portal/overview/backups/StatusRenderer.svelte b/packages/builder/src/components/portal/overview/backups/StatusRenderer.svelte new file mode 100644 index 0000000000..8fd98f663e --- /dev/null +++ b/packages/builder/src/components/portal/overview/backups/StatusRenderer.svelte @@ -0,0 +1,10 @@ + + + + {status} + diff --git a/packages/builder/src/components/portal/overview/backups/TriggerRenderer.svelte b/packages/builder/src/components/portal/overview/backups/TriggerRenderer.svelte deleted file mode 100644 index f896cb03c5..0000000000 --- a/packages/builder/src/components/portal/overview/backups/TriggerRenderer.svelte +++ /dev/null @@ -1,32 +0,0 @@ - - -
- {#if value === "publish"} - -
{trigger}
- {:else if value === "manual"} - -
{trigger}
- {:else if value === "scheduled"} - -
{trigger}
- {:else if value === "restore"} - -
{trigger}
- {/if} -
- - diff --git a/packages/builder/src/components/portal/overview/backups/TypeRenderer.svelte b/packages/builder/src/components/portal/overview/backups/TypeRenderer.svelte new file mode 100644 index 0000000000..9057a2adee --- /dev/null +++ b/packages/builder/src/components/portal/overview/backups/TypeRenderer.svelte @@ -0,0 +1,20 @@ + + +
+ {trigger} + {type} +
+ + diff --git a/packages/builder/src/components/portal/overview/backups/AutomationsRenderer.svelte b/packages/builder/src/components/portal/overview/backups/UserRenderer.svelte similarity index 55% rename from packages/builder/src/components/portal/overview/backups/AutomationsRenderer.svelte rename to packages/builder/src/components/portal/overview/backups/UserRenderer.svelte index 08ea5fb118..42b25bcca8 100644 --- a/packages/builder/src/components/portal/overview/backups/AutomationsRenderer.svelte +++ b/packages/builder/src/components/portal/overview/backups/UserRenderer.svelte @@ -1,12 +1,14 @@
- -
{value || 0}
+ {#if value != null} +
{username}
+ {/if}
diff --git a/packages/frontend-core/src/api/backups.js b/packages/frontend-core/src/api/backups.js index 82fb9533d8..85df0eab1e 100644 --- a/packages/frontend-core/src/api/backups.js +++ b/packages/frontend-core/src/api/backups.js @@ -40,9 +40,10 @@ export const buildBackupsEndpoints = API => ({ }) }, - restoreBackup: async ({ appId, backupId }) => { + restoreBackup: async ({ appId, backupId, name }) => { return await API.post({ url: `/api/apps/${appId}/backups/${backupId}/import`, + body: { name }, }) }, diff --git a/packages/types/src/documents/app/backup.ts b/packages/types/src/documents/app/backup.ts index 520013a8bb..9179852331 100644 --- a/packages/types/src/documents/app/backup.ts +++ b/packages/types/src/documents/app/backup.ts @@ -59,6 +59,7 @@ export interface AppBackupQueueData { } import?: { backupId: string + nameForBackup: string createdBy?: string } } From 854cb23947f96ae55d1f3395b44e9c1d12b14f73 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 20 Oct 2022 19:07:10 +0100 Subject: [PATCH 075/151] Getting the import system to carry out a backup of the app before the restore. --- packages/server/src/sdk/app/backups/backup.ts | 145 +++++++++++------- packages/types/src/documents/app/backup.ts | 2 + 2 files changed, 93 insertions(+), 54 deletions(-) diff --git a/packages/server/src/sdk/app/backups/backup.ts b/packages/server/src/sdk/app/backups/backup.ts index 97fe12e63d..2741e41096 100644 --- a/packages/server/src/sdk/app/backups/backup.ts +++ b/packages/server/src/sdk/app/backups/backup.ts @@ -1,6 +1,11 @@ import { backups } from "@budibase/pro" import { db as dbCore, objectStore, tenancy } from "@budibase/backend-core" -import { AppBackupQueueData, AppBackupStatus } from "@budibase/types" +import { + AppBackupQueueData, + AppBackupStatus, + AppBackupTrigger, + AppBackupType, +} from "@budibase/types" import { exportApp } from "./exports" import { importApp } from "./imports" import { calculateBackupStats } from "../statistics" @@ -8,19 +13,96 @@ import { Job } from "bull" import fs from "fs" import env from "../../../environment" +type BackupOpts = { + doc?: { id: string; rev: string } + createdBy?: string +} + async function removeExistingApp(devId: string) { const devDb = dbCore.dangerousGetDB(devId, { skip_setup: true }) await devDb.destroy() } +async function runBackup( + name: string, + trigger: AppBackupTrigger, + tenantId: string, + appId: string, + opts?: BackupOpts +) { + const devAppId = dbCore.getDevAppID(appId), + prodAppId = dbCore.getProdAppID(appId) + const timestamp = new Date().toISOString() + const tarPath = await exportApp(devAppId, { tar: true }) + const contents = await calculateBackupStats(devAppId) + let filename = `${prodAppId}/backup-${timestamp}.tar.gz` + // add the tenant to the bucket path if backing up within a multi-tenant environment + if (env.MULTI_TENANCY) { + filename = `${tenantId}/${filename}` + } + const bucket = objectStore.ObjectStoreBuckets.BACKUPS + await objectStore.upload({ + path: tarPath, + type: "application/gzip", + bucket, + filename, + metadata: { + name, + trigger, + timestamp, + appId: prodAppId, + }, + }) + if (opts?.doc) { + await backups.updateBackupStatus( + opts.doc.id, + opts.doc.rev, + AppBackupStatus.COMPLETE, + contents, + filename + ) + } else { + await backups.storeAppBackupMetadata( + { + appId: prodAppId, + timestamp, + name, + trigger, + type: AppBackupType.BACKUP, + status: AppBackupStatus.COMPLETE, + contents, + createdBy: opts?.createdBy, + }, + { filename } + ) + } + // clear up the tarball after uploading it + fs.rmSync(tarPath) +} + async function importProcessor(job: Job) { const data: AppBackupQueueData = job.data const appId = data.appId, - backupId = data.import!.backupId - const tenantId = tenancy.getTenantIDFromAppID(appId) + backupId = data.import!.backupId, + nameForBackup = data.import!.nameForBackup, + createdBy = data.import!.createdBy + const tenantId = tenancy.getTenantIDFromAppID(appId) as string tenancy.doInTenant(tenantId, async () => { const devAppId = dbCore.getDevAppID(appId) - const performImport = async (path: string) => { + // initially export the current state to disk - incase something goes wrong + await runBackup( + nameForBackup, + AppBackupTrigger.RESTORING, + tenantId, + appId, + { createdBy } + ) + // get the backup ready on disk + const { path } = await backups.downloadAppBackup(backupId) + // start by removing app database and contents of bucket - which will be updated + await removeExistingApp(devAppId) + let status = AppBackupStatus.COMPLETE + try { await importApp(devAppId, dbCore.dangerousGetDB(devAppId), { file: { type: "application/gzip", @@ -28,26 +110,10 @@ async function importProcessor(job: Job) { }, key: path, }) - } - // initially export the current state to disk - incase something goes wrong - const backupTarPath = await exportApp(devAppId, { tar: true }) - // get the backup ready on disk - const { path } = await backups.downloadAppBackup(backupId) - // start by removing app database and contents of bucket - which will be updated - await removeExistingApp(devAppId) - try { - await performImport(path) } catch (err) { - // rollback - clear up failed import and re-import the pre-backup - await removeExistingApp(devAppId) - await performImport(backupTarPath) + status = AppBackupStatus.FAILED } - await backups.updateRestoreStatus( - data.docId, - data.docRev, - AppBackupStatus.COMPLETE - ) - fs.rmSync(backupTarPath) + await backups.updateRestoreStatus(data.docId, data.docRev, status) }) } @@ -56,40 +122,11 @@ async function exportProcessor(job: Job) { const appId = data.appId, trigger = data.export!.trigger, name = data.export!.name || `${trigger} - backup` - const tenantId = tenancy.getTenantIDFromAppID(appId) + const tenantId = tenancy.getTenantIDFromAppID(appId) as string await tenancy.doInTenant(tenantId, async () => { - const devAppId = dbCore.getDevAppID(appId), - prodAppId = dbCore.getProdAppID(appId) - const timestamp = new Date().toISOString() - const tarPath = await exportApp(devAppId, { tar: true }) - const contents = await calculateBackupStats(devAppId) - let filename = `${prodAppId}/backup-${timestamp}.tar.gz` - // add the tenant to the bucket path if backing up within a multi-tenant environment - if (env.MULTI_TENANCY) { - filename = `${tenantId}/${filename}` - } - const bucket = objectStore.ObjectStoreBuckets.BACKUPS - await objectStore.upload({ - path: tarPath, - type: "application/gzip", - bucket, - filename, - metadata: { - name, - trigger, - timestamp, - appId: prodAppId, - }, + return runBackup(name, trigger, tenantId, appId, { + doc: { id: data.docId, rev: data.docRev }, }) - await backups.updateBackupStatus( - data.docId, - data.docRev, - AppBackupStatus.COMPLETE, - contents, - filename - ) - // clear up the tarball after uploading it - fs.rmSync(tarPath) }) } diff --git a/packages/types/src/documents/app/backup.ts b/packages/types/src/documents/app/backup.ts index 520013a8bb..c9671e8269 100644 --- a/packages/types/src/documents/app/backup.ts +++ b/packages/types/src/documents/app/backup.ts @@ -16,6 +16,7 @@ export enum AppBackupTrigger { PUBLISH = "publish", MANUAL = "manual", SCHEDULED = "scheduled", + RESTORING = "restoring", } export interface AppBackupContents { @@ -59,6 +60,7 @@ export interface AppBackupQueueData { } import?: { backupId: string + nameForBackup: string createdBy?: string } } From 6a96d447e9805c47a61e7532821fd860f10d3792 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Thu, 20 Oct 2022 19:49:14 +0100 Subject: [PATCH 076/151] Ensure allowed component list is actually accurate and prevent any way around illegal component nesting --- .../src/builderStore/store/frontend.js | 4 +- .../new/_components/NewComponentPanel.svelte | 54 ++++++++++++++++--- 2 files changed, 49 insertions(+), 9 deletions(-) diff --git a/packages/builder/src/builderStore/store/frontend.js b/packages/builder/src/builderStore/store/frontend.js index b6d315f15d..97fad5f1bb 100644 --- a/packages/builder/src/builderStore/store/frontend.js +++ b/packages/builder/src/builderStore/store/frontend.js @@ -236,12 +236,12 @@ export const getFrontendStore = () => { } } - // Validate the entire tree and throw and error if an illegal child is + // Validate the entire tree and throw an error if an illegal child is // found anywhere const illegalChild = findIllegalChild(screen.props) if (illegalChild) { const def = store.actions.components.getDefinition(illegalChild) - throw `A ${def.name} can't be inserted here` + throw `You can't place a ${def.name} here` } }, save: async screen => { diff --git a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/components/[componentId]/new/_components/NewComponentPanel.svelte b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/components/[componentId]/new/_components/NewComponentPanel.svelte index ca8912a74f..0610be6d0a 100644 --- a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/components/[componentId]/new/_components/NewComponentPanel.svelte +++ b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/components/[componentId]/new/_components/NewComponentPanel.svelte @@ -11,9 +11,10 @@ notifications, } from "@budibase/bbui" import structure from "./componentStructure.json" - import { store, selectedComponent } from "builderStore" + import { store, selectedComponent, selectedScreen } from "builderStore" import { onMount } from "svelte" import { fly } from "svelte/transition" + import { findComponentPath } from "builderStore/componentUtils" let section = "components" let searchString @@ -21,8 +22,10 @@ let selectedIndex let componentList = [] - $: currentDefinition = store.actions.components.getDefinition( - $selectedComponent?._component + $: allowedComponents = getAllowedComponents( + $store.components, + $selectedScreen, + $selectedComponent ) $: enrichedStructure = enrichStructure( structure, @@ -31,13 +34,50 @@ ) $: filteredStructure = filterStructure( enrichedStructure, - section, - currentDefinition, + allowedComponents, searchString ) $: blocks = enrichedStructure.find(x => x.name === "Blocks").children $: orderMap = createComponentOrderMap(componentList) + const getAllowedComponents = (allComponents, screen, component) => { + const path = findComponentPath(screen?.props, component?._id) + if (!path?.length) { + return [] + } + + // Get initial set of allowed components + let allowedComponents = [] + const definition = store.actions.components.getDefinition( + component?._component + ) + if (definition.legalDirectChildren?.length) { + allowedComponents = definition.legalDirectChildren.map(x => { + return `@budibase/standard-components/${x}` + }) + } else { + allowedComponents = Object.keys(allComponents) + } + + // Build up list of illegal children from ancestors + let illegalChildren = definition.illegalChildren || [] + path.forEach(ancestor => { + const def = store.actions.components.getDefinition(ancestor._component) + const blacklist = def?.illegalChildren?.map(x => { + return `@budibase/standard-components/${x}` + }) + illegalChildren = [...illegalChildren, ...(blacklist || [])] + }) + illegalChildren = [...new Set(illegalChildren)] + + // Filter out illegal children from allowed components + allowedComponents = allowedComponents.filter(x => { + return !illegalChildren.includes(x) + }) + + return allowedComponents + } + // Creates a simple lookup map from an array, so we can find the selected // component much faster const createComponentOrderMap = list => { @@ -90,7 +130,7 @@ return enrichedStructure } - const filterStructure = (structure, section, currentDefinition, search) => { + const filterStructure = (structure, allowedComponents, search) => { selectedIndex = search ? 0 : null componentList = [] if (!structure?.length) { @@ -114,7 +154,7 @@ } // Check if the component is allowed as a child - return !currentDefinition?.illegalChildren?.includes(name) + return allowedComponents.includes(child.component) }) if (matchedChildren.length) { filteredStructure.push({ From 93c042d0eb47d27d235f64436cad252f9441aefd Mon Sep 17 00:00:00 2001 From: Peter Clement Date: Fri, 21 Oct 2022 14:10:08 +0100 Subject: [PATCH 077/151] fix download issue and pass name for backup after restore --- .../portal/overview/backups/ActionsRenderer.svelte | 12 +++--------- .../portal/overview/backups/BackupsTab.svelte | 2 +- .../portal/overview/backups/StatusRenderer.svelte | 2 +- packages/builder/src/stores/portal/backups.js | 4 ++-- 4 files changed, 7 insertions(+), 13 deletions(-) diff --git a/packages/builder/src/components/portal/overview/backups/ActionsRenderer.svelte b/packages/builder/src/components/portal/overview/backups/ActionsRenderer.svelte index 6a6a5a8060..bb09ad3f6d 100644 --- a/packages/builder/src/components/portal/overview/backups/ActionsRenderer.svelte +++ b/packages/builder/src/components/portal/overview/backups/ActionsRenderer.svelte @@ -11,8 +11,6 @@ import ConfirmDialog from "components/common/ConfirmDialog.svelte" import CreateRestoreModal from "./CreateRestoreModal.svelte" import { createEventDispatcher } from "svelte" - import download from "downloadjs" - import { backups } from "stores/portal" export let row @@ -20,7 +18,6 @@ let restoreDialog let updateDialog let name - let restoreBackupName let restoreBackupModal const dispatch = createEventDispatcher() @@ -30,7 +27,7 @@ type: "backupRestore", name, backupId: row._id, - restoreBackupName, + restoreBackupName: name, }) } @@ -48,12 +45,9 @@ name, }) } + async function downloadExport() { - let resp = await backups.downloadBackup({ - backupId: row._id, - appId: row.appId, - }) - download(resp, row.filename) + window.location = `/api/apps/${row.appId}/backups/${row._id}/file` } diff --git a/packages/builder/src/components/portal/overview/backups/BackupsTab.svelte b/packages/builder/src/components/portal/overview/backups/BackupsTab.svelte index 3d257db5a8..d435360362 100644 --- a/packages/builder/src/components/portal/overview/backups/BackupsTab.svelte +++ b/packages/builder/src/components/portal/overview/backups/BackupsTab.svelte @@ -107,7 +107,6 @@ } async function handleButtonClick({ detail }) { - console.log(detail.type) if (detail.type === "backupDelete") { await backups.deleteBackup({ appId: app.instance._id, @@ -118,6 +117,7 @@ await backups.restoreBackup({ appId: app.instance._id, backupId: detail.backupId, + name: detail.restoreBackupName, }) } else if (detail.type === "backupUpdate") { await backups.updateBackup({ diff --git a/packages/builder/src/components/portal/overview/backups/StatusRenderer.svelte b/packages/builder/src/components/portal/overview/backups/StatusRenderer.svelte index 8fd98f663e..610b080d37 100644 --- a/packages/builder/src/components/portal/overview/backups/StatusRenderer.svelte +++ b/packages/builder/src/components/portal/overview/backups/StatusRenderer.svelte @@ -2,7 +2,7 @@ import { Badge } from "@budibase/bbui" export let value = "started" - $: status = value.charAt(0).toUpperCase() + value.slice(1) + $: status = value?.charAt(0).toUpperCase() + value?.slice(1) diff --git a/packages/builder/src/stores/portal/backups.js b/packages/builder/src/stores/portal/backups.js index 2e0139cf6a..af336c73ab 100644 --- a/packages/builder/src/stores/portal/backups.js +++ b/packages/builder/src/stores/portal/backups.js @@ -15,8 +15,8 @@ export function createBackupsStore() { return API.searchBackups({ appId, trigger, page, startDate, endDate }) } - async function restoreBackup({ appId, backupId }) { - return API.restoreBackup({ appId, backupId }) + async function restoreBackup({ appId, backupId, name }) { + return API.restoreBackup({ appId, backupId, name }) } async function deleteBackup({ appId, backupId }) { From cdcfbc6dce5f4dd0119c1cf6fed9b4f40a1599c3 Mon Sep 17 00:00:00 2001 From: Dean Date: Fri, 21 Oct 2022 14:20:40 +0100 Subject: [PATCH 078/151] Initial commit for chart block component --- .../settings/ComponentSettingsSection.svelte | 29 +- .../new/_components/componentStructure.json | 3 +- packages/client/manifest.json | 471 ++++++++++++++++++ .../components/app/blocks/ChartBlock.svelte | 107 ++++ .../client/src/components/app/blocks/index.js | 1 + .../components/app/charts/ApexChart.svelte | 5 +- .../app/charts/ApexOptionsBuilder.js | 3 + 7 files changed, 605 insertions(+), 14 deletions(-) create mode 100644 packages/client/src/components/app/blocks/ChartBlock.svelte diff --git a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/components/[componentId]/_components/settings/ComponentSettingsSection.svelte b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/components/[componentId]/_components/settings/ComponentSettingsSection.svelte index efd3db8ff7..12a7eea87d 100644 --- a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/components/[componentId]/_components/settings/ComponentSettingsSection.svelte +++ b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/components/[componentId]/_components/settings/ComponentSettingsSection.svelte @@ -29,6 +29,10 @@ // Filter out settings which shouldn't be rendered sections.forEach(section => { + section.visible = shouldDisplay(instance, section) + if (!section.visible) { + return + } section.settings.forEach(setting => { setting.visible = canRenderControl(instance, setting, isScreen) }) @@ -46,17 +50,7 @@ } } - const canRenderControl = (instance, setting, isScreen) => { - // Prevent rendering on click setting for screens - if (setting?.type === "event" && isScreen) { - return false - } - - const control = getComponentForSetting(setting) - if (!control) { - return false - } - + const shouldDisplay = (instance, setting) => { // Parse dependant settings if (setting.dependsOn) { let dependantSetting = setting.dependsOn @@ -93,6 +87,19 @@ return true } + + const canRenderControl = (instance, setting, isScreen) => { + // Prevent rendering on click setting for screens + if (setting?.type === "event" && isScreen) { + return false + } + const control = getComponentForSetting(setting) + if (!control) { + return false + } + + return shouldDisplay(instance, setting) + } {#each sections as section, idx (section.name)} diff --git a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/components/[componentId]/new/_components/componentStructure.json b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/components/[componentId]/new/_components/componentStructure.json index 3d9cf7e2a3..3100c7467c 100644 --- a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/components/[componentId]/new/_components/componentStructure.json +++ b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/components/[componentId]/new/_components/componentStructure.json @@ -6,7 +6,8 @@ "tableblock", "cardsblock", "repeaterblock", - "formblock" + "formblock", + "chartblock" ] }, { diff --git a/packages/client/manifest.json b/packages/client/manifest.json index 763ac46b3f..93a53c15a3 100644 --- a/packages/client/manifest.json +++ b/packages/client/manifest.json @@ -3972,6 +3972,477 @@ } ] }, + "chartblock": { + "block": true, + "name": "Chart block", + "icon": "GraphPie", + "hasChildren": false, + "settings": [ + { + "type": "select", + "label": "Chart Type", + "key": "chartType", + "required": true, + "options": [ + { + "label": "Pie", + "value": "pie" + }, + { + "label": "Bar", + "value": "bar" + }, + { + "label": "Line", + "value": "line" + }, + { + "label": "Donut", + "value": "donut" + }, + { + "label": "Candlestick", + "value": "candlestick" + }, + { + "label": "Area", + "value": "area" + } + ] + }, + { + "type": "dataSource", + "label": "Data", + "key": "dataSource", + "required": true + }, + { + "type": "text", + "label": "Title", + "key": "chartTitle" + }, + { + "type": "filter", + "label": "Filtering", + "key": "filter" + }, + { + "type": "field", + "label": "Sort Column", + "key": "sortColumn" + }, + { + "type": "select", + "label": "Sort Order", + "key": "sortOrder", + "options": [ + "Ascending", + "Descending" + ], + "defaultValue": "Ascending" + }, + { + "type": "number", + "label": "Limit", + "key": "limit", + "defaultValue": 50 + }, + { + "type": "text", + "label": "Width", + "key": "width" + }, + { + "type": "text", + "label": "Height", + "key": "height", + "defaultValue": "400" + }, + { + "type": "select", + "label": "Colors", + "key": "palette", + "defaultValue": "Palette 1", + "options": [ + "Custom", + "Palette 1", + "Palette 2", + "Palette 3", + "Palette 4", + "Palette 5", + "Palette 6", + "Palette 7", + "Palette 8", + "Palette 9", + "Palette 10" + ] + }, + { + "type": "color", + "label": "C1", + "key": "c1", + "dependsOn": { + "setting": "palette", + "value": "Custom" + } + }, + { + "type": "color", + "label": "C2", + "key": "c2", + "dependsOn": { + "setting": "palette", + "value": "Custom" + } + }, + { + "type": "color", + "label": "C3", + "key": "c3", + "dependsOn": { + "setting": "palette", + "value": "Custom" + } + }, + { + "type": "color", + "label": "C4", + "key": "c4", + "dependsOn": { + "setting": "palette", + "value": "Custom" + } + }, + { + "type": "color", + "label": "C5", + "key": "c5", + "dependsOn": { + "setting": "palette", + "value": "Custom" + } + }, + { + "type": "boolean", + "label": "Data Labels", + "key": "dataLabels", + "defaultValue": false + }, + { + "type": "boolean", + "label": "Legend", + "key": "legend", + "defaultValue": true + }, + { + "type": "boolean", + "label": "Animate", + "key": "animate", + "defaultValue": false + }, + { + "section": true, + "name": "Pie Chart", + "icon": "GraphPie", + "dependsOn": { + "setting": "chartType", + "value": "pie" + }, + "settings": [ + { + "type": "field", + "label": "Label Col.", + "key": "labelColumn", + "dependsOn": "dataSource", + "required": true + }, + { + "type": "field", + "label": "Data Col.", + "key": "valueColumn", + "dependsOn": "dataSource", + "required": true + } + ] + }, + { + "section": true, + "name": "Donut Chart", + "icon": "GraphDonut", + "dependsOn": { + "setting": "chartType", + "value": "donut" + }, + "settings": [ + { + "type": "field", + "label": "Label Col.", + "key": "labelColumn", + "dependsOn": "dataSource", + "required": true + }, + { + "type": "field", + "label": "Data Col.", + "key": "valueColumn", + "dependsOn": "dataSource", + "required": true + } + ] + }, + { + "section": true, + "name": "Bar Chart", + "icon": "GraphBarVertical", + "dependsOn": { + "setting": "chartType", + "value": "bar" + }, + "settings": [ + { + "type": "field", + "label": "Label Col.", + "key": "labelColumn", + "dependsOn": "dataSource", + "required": true + }, + { + "type": "multifield", + "label": "Data Cols.", + "key": "valueColumns", + "dependsOn": "dataSource", + "required": true + }, + { + "type": "select", + "label": "Format", + "key": "yAxisUnits", + "options": [ + "Default", + "Thousands", + "Millions" + ], + "defaultValue": "Default" + }, + { + "type": "text", + "label": "Y Axis Label", + "key": "yAxisLabel" + }, + { + "type": "text", + "label": "X Axis Label", + "key": "xAxisLabel" + }, + { + "type": "boolean", + "label": "Stacked", + "key": "stacked", + "defaultValue": false + }, + { + "type": "boolean", + "label": "Horizontal", + "key": "horizontal", + "defaultValue": false + } + ] + }, + { + "section": true, + "name": "Line Chart", + "icon": "GraphTrend", + "dependsOn": { + "setting": "chartType", + "value": "line" + }, + "settings": [ + { + "type": "field", + "label": "Label Col.", + "key": "labelColumn", + "dependsOn": "dataSource", + "required": true + }, + { + "type": "multifield", + "label": "Data Cols.", + "key": "valueColumns", + "dependsOn": "dataSource", + "required": true + }, + { + "type": "select", + "label": "Format", + "key": "yAxisUnits", + "options": [ + "Default", + "Thousands", + "Millions" + ], + "defaultValue": "Default" + }, + { + "type": "text", + "label": "Y Axis Label", + "key": "yAxisLabel" + }, + { + "type": "text", + "label": "X Axis Label", + "key": "xAxisLabel" + }, + { + "type": "select", + "label": "Curve", + "key": "curve", + "options": [ + "Smooth", + "Straight", + "Stepline" + ], + "defaultValue": "Smooth" + } + ] + }, + { + "section": true, + "name": "Area Chart", + "icon": "GraphAreaStacked", + "dependsOn": { + "setting": "chartType", + "value": "area" + }, + "settings": [ + { + "type": "field", + "label": "Label Col.", + "key": "labelColumn", + "dependsOn": "dataSource", + "required": true + }, + { + "type": "multifield", + "label": "Data Cols.", + "key": "valueColumns", + "dependsOn": "dataSource", + "required": true + }, + { + "type": "select", + "label": "Format", + "key": "yAxisUnits", + "options": [ + "Default", + "Thousands", + "Millions" + ], + "defaultValue": "Default" + }, + { + "type": "text", + "label": "Y Axis Label", + "key": "yAxisLabel" + }, + { + "type": "text", + "label": "X Axis Label", + "key": "xAxisLabel" + }, + { + "type": "select", + "label": "Curve", + "key": "curve", + "options": [ + "Smooth", + "Straight", + "Stepline" + ], + "defaultValue": "Smooth" + }, + { + "type": "boolean", + "label": "Stacked", + "key": "stacked", + "defaultValue": true + }, + { + "type": "boolean", + "label": "Gradient", + "key": "gradient", + "defaultValue": false + } + ] + }, + { + "section": true, + "name": "Candlestick Chart", + "icon": "GraphBarVerticalStacked", + "dependsOn": { + "setting": "chartType", + "value": "candlestick" + }, + "settings": [ + { + "type": "field", + "label": "Date Col.", + "key": "dateColumn", + "dependsOn": "dataSource", + "required": true + }, + { + "type": "field", + "label": "Open Col.", + "key": "openColumn", + "dependsOn": "dataSource", + "required": true + }, + { + "type": "field", + "label": "Close Col.", + "key": "closeColumn", + "dependsOn": "dataSource", + "required": true + }, + { + "type": "field", + "label": "High Col.", + "key": "highColumn", + "dependsOn": "dataSource", + "required": true + }, + { + "type": "field", + "label": "Low Col.", + "key": "lowColumn", + "dependsOn": "dataSource", + "required": true + }, + { + "type": "select", + "label": "Format", + "key": "yAxisUnits", + "options": [ + "Default", + "Thousands", + "Millions" + ], + "defaultValue": "Default" + }, + { + "type": "text", + "label": "Y Axis Label", + "key": "yAxisLabel" + }, + { + "type": "text", + "label": "X Axis Label", + "key": "xAxisLabel" + } + ] + } + ] + }, "tableblock": { "block": true, "name": "Table block", diff --git a/packages/client/src/components/app/blocks/ChartBlock.svelte b/packages/client/src/components/app/blocks/ChartBlock.svelte new file mode 100644 index 0000000000..7a2f8469b8 --- /dev/null +++ b/packages/client/src/components/app/blocks/ChartBlock.svelte @@ -0,0 +1,107 @@ + + + + + {#if $component.empty} + + {:else} + + {/if} + + diff --git a/packages/client/src/components/app/blocks/index.js b/packages/client/src/components/app/blocks/index.js index 32b2b98c06..734bff2c0f 100644 --- a/packages/client/src/components/app/blocks/index.js +++ b/packages/client/src/components/app/blocks/index.js @@ -2,3 +2,4 @@ export { default as tableblock } from "./TableBlock.svelte" export { default as cardsblock } from "./CardsBlock.svelte" export { default as repeaterblock } from "./RepeaterBlock.svelte" export { default as formblock } from "./FormBlock.svelte" +export { default as chartblock } from "./ChartBlock.svelte" diff --git a/packages/client/src/components/app/charts/ApexChart.svelte b/packages/client/src/components/app/charts/ApexChart.svelte index 87d78bf5a2..eed8fb2e6c 100644 --- a/packages/client/src/components/app/charts/ApexChart.svelte +++ b/packages/client/src/components/app/charts/ApexChart.svelte @@ -24,8 +24,9 @@ display: flex !important; text-transform: capitalize; } - div :global(.apexcharts-yaxis-label), - div :global(.apexcharts-xaxis-label) { + div :global(.apexcharts-text.apexcharts-xaxis-title-text), + div :global(.apexcharts-text.apexcharts-yaxis-title-text), + div :global(.apexcharts-title-text) { fill: var(--spectrum-global-color-gray-600); } diff --git a/packages/client/src/components/app/charts/ApexOptionsBuilder.js b/packages/client/src/components/app/charts/ApexOptionsBuilder.js index 6b3e3a4440..04b5805df3 100644 --- a/packages/client/src/components/app/charts/ApexOptionsBuilder.js +++ b/packages/client/src/components/app/charts/ApexOptionsBuilder.js @@ -184,6 +184,9 @@ export class ApexOptionsBuilder { } palette(palette) { + if (!palette) { + return this + } return this.setOption( ["theme", "palette"], palette.toLowerCase().replace(/[\W]/g, "") From d17241d8d7c1c1a81ce6887eb97eae5c6c3727ba Mon Sep 17 00:00:00 2001 From: Peter Clement Date: Fri, 21 Oct 2022 16:00:10 +0100 Subject: [PATCH 079/151] refetch after backup / delete / update --- .../components/portal/overview/backups/BackupsTab.svelte | 8 ++++++-- .../portal/overview/backups/StatusRenderer.svelte | 7 ++++++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/packages/builder/src/components/portal/overview/backups/BackupsTab.svelte b/packages/builder/src/components/portal/overview/backups/BackupsTab.svelte index d435360362..7ba2bd8370 100644 --- a/packages/builder/src/components/portal/overview/backups/BackupsTab.svelte +++ b/packages/builder/src/components/portal/overview/backups/BackupsTab.svelte @@ -100,6 +100,7 @@ appId: app.instance._id, name, }) + await fetchBackups(trigger, page) notifications.success(response.message) } catch { notifications.error("Unable to create backup") @@ -112,20 +113,21 @@ appId: app.instance._id, backupId: detail.backupId, }) - await fetchBackups(app.instance._id, trigger, page) + await fetchBackups(trigger, page) } else if (detail.type === "backupRestore") { await backups.restoreBackup({ appId: app.instance._id, backupId: detail.backupId, name: detail.restoreBackupName, }) + await fetchBackups(trigger, page) } else if (detail.type === "backupUpdate") { await backups.updateBackup({ appId: app.instance._id, backupId: detail.backupId, name: detail.name, }) - await fetchBackups(app.instance._id, trigger, page) + await fetchBackups(trigger, page) } } @@ -138,6 +140,8 @@ placeholder="All" label="Trigger" options={Object.values(triggers)} + getOptionLabel={trigger => + trigger.charAt(0).toUpperCase() + trigger.slice(1)} bind:value={trigger} />
diff --git a/packages/builder/src/components/portal/overview/backups/StatusRenderer.svelte b/packages/builder/src/components/portal/overview/backups/StatusRenderer.svelte index 610b080d37..95834b024e 100644 --- a/packages/builder/src/components/portal/overview/backups/StatusRenderer.svelte +++ b/packages/builder/src/components/portal/overview/backups/StatusRenderer.svelte @@ -5,6 +5,11 @@ $: status = value?.charAt(0).toUpperCase() + value?.slice(1) - + {status} From 2a2f41a8615f1143af1012790c04175ba1c2a45a Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 21 Oct 2022 16:02:13 +0100 Subject: [PATCH 080/151] Fixing issue discovered by tests. --- packages/backend-core/src/queue/queue.ts | 2 +- .../server/src/api/controllers/deploy/index.ts | 16 ++++++++++++---- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/packages/backend-core/src/queue/queue.ts b/packages/backend-core/src/queue/queue.ts index 6dd10ee091..7eaafe38a8 100644 --- a/packages/backend-core/src/queue/queue.ts +++ b/packages/backend-core/src/queue/queue.ts @@ -22,7 +22,7 @@ export function createQueue( ): BullQueue.Queue { const queueConfig: any = redisProtocolUrl || { redis: opts } let queue: any - if (env.isTest()) { + if (!env.isTest()) { queue = new BullQueue(jobQueue, queueConfig) } else { queue = new InMemoryQueue(jobQueue, queueConfig) diff --git a/packages/server/src/api/controllers/deploy/index.ts b/packages/server/src/api/controllers/deploy/index.ts index a1cb905930..b3b875e397 100644 --- a/packages/server/src/api/controllers/deploy/index.ts +++ b/packages/server/src/api/controllers/deploy/index.ts @@ -20,6 +20,7 @@ import { import { events } from "@budibase/backend-core" import { backups } from "@budibase/pro" import { AppBackupTrigger } from "@budibase/types" +import env from "../../../environment" // the max time we can wait for an invalidation to complete before considering it failed const MAX_PENDING_TIME_MS = 30 * 60000 @@ -107,10 +108,17 @@ async function deployApp(deployment: any, userId: string) { const devAppId = getDevelopmentAppID(appId) const productionAppId = getProdAppID(appId) - // trigger backup initially - await backups.triggerAppBackup(productionAppId, AppBackupTrigger.PUBLISH, { - createdBy: userId, - }) + // can't do this in test + if (!env.isTest()) { + // trigger backup initially + await backups.triggerAppBackup( + productionAppId, + AppBackupTrigger.PUBLISH, + { + createdBy: userId, + } + ) + } const config: any = { source: devAppId, From 67b6821b376b377bd76a77e729e351fced116030 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Fri, 21 Oct 2022 16:52:46 +0100 Subject: [PATCH 081/151] Fix components being selected when starting dragging --- packages/client/src/components/Component.svelte | 2 +- packages/client/src/utils/computed.js | 0 2 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 packages/client/src/utils/computed.js diff --git a/packages/client/src/components/Component.svelte b/packages/client/src/components/Component.svelte index bc430ee92c..669cc8afdc 100644 --- a/packages/client/src/components/Component.svelte +++ b/packages/client/src/components/Component.svelte @@ -446,7 +446,7 @@ const scrollIntoView = () => { // Don't scroll into view if we selected this component because we were // starting dragging on it - if (get(builderStore).dragging) { + if (get(dndIsDragging)) { return } const node = document.getElementsByClassName(id)?.[0]?.children[0] diff --git a/packages/client/src/utils/computed.js b/packages/client/src/utils/computed.js new file mode 100644 index 0000000000..e69de29bb2 From f1714ab2a5c02d7ff25989ac4a1e0ca3a820de1d Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Fri, 21 Oct 2022 16:54:01 +0100 Subject: [PATCH 082/151] Improve performance by fixing multiple instances of redundant client app initialisations --- packages/builder/src/builderStore/store/frontend.js | 12 +++++------- .../design/[screenId]/_components/AppPreview.svelte | 8 ++++++-- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/packages/builder/src/builderStore/store/frontend.js b/packages/builder/src/builderStore/store/frontend.js index 97fad5f1bb..fc8b1b8427 100644 --- a/packages/builder/src/builderStore/store/frontend.js +++ b/packages/builder/src/builderStore/store/frontend.js @@ -667,16 +667,14 @@ export const getFrontendStore = () => { }) // Select the parent if cutting - if (cut) { + if (cut && selectParent) { const screen = get(selectedScreen) const parent = findComponentParent(screen?.props, component._id) if (parent) { - if (selectParent) { - store.update(state => { - state.selectedComponentId = parent._id - return state - }) - } + store.update(state => { + state.selectedComponentId = parent._id + return state + }) } } }, diff --git a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/AppPreview.svelte b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/AppPreview.svelte index 3c7edf096f..2a606233c2 100644 --- a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/AppPreview.svelte +++ b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/AppPreview.svelte @@ -86,7 +86,11 @@ : [], isBudibaseEvent: true, usedPlugins: $store.usedPlugins, - location: window.location, + location: { + protocol: window.location.protocol, + hostname: window.location.hostname, + port: window.location.port, + }, } // Refresh the preview when required @@ -189,7 +193,7 @@ // Cut and paste the component to the new destination if (source && destination) { - store.actions.components.copy(source, true) + store.actions.components.copy(source, true, false) await store.actions.components.paste(destination, data.mode) } } else if (type === "click-nav") { From 794db1a7db0aa3ce657bbde237e8313f02bc8278 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Fri, 21 Oct 2022 16:54:34 +0100 Subject: [PATCH 083/151] Make DND feel much smoother by persisting the end position of drops, and more performance by properly memoizing some state values --- .../src/components/preview/DNDHandler.svelte | 7 +- packages/client/src/index.js | 9 ++- packages/client/src/stores/dnd.js | 28 ++++--- packages/client/src/stores/index.js | 1 + packages/client/src/stores/screens.js | 79 ++++++++++++------- packages/client/src/utils/computed.js | 26 ++++++ 6 files changed, 107 insertions(+), 43 deletions(-) diff --git a/packages/client/src/components/preview/DNDHandler.svelte b/packages/client/src/components/preview/DNDHandler.svelte index f556346ad2..e04bd81750 100644 --- a/packages/client/src/components/preview/DNDHandler.svelte +++ b/packages/client/src/components/preview/DNDHandler.svelte @@ -57,7 +57,9 @@ } // Reset state - dndStore.actions.reset() + if (!$dndStore.dropped) { + dndStore.actions.reset() + } } // Callback when initially starting a drag on a draggable component @@ -251,7 +253,7 @@ } // Callback when dropping a drag on top of some component - const onDrop = e => { + const onDrop = () => { if (!source || !drop?.parent || drop?.index == null) { return } @@ -299,6 +301,7 @@ } if (legacyDropTarget && legacyDropMode) { + dndStore.actions.markDropped() builderStore.actions.moveComponent( source.id, legacyDropTarget, diff --git a/packages/client/src/index.js b/packages/client/src/index.js index e9c821c54f..e61df37d95 100644 --- a/packages/client/src/index.js +++ b/packages/client/src/index.js @@ -24,10 +24,6 @@ loadSpectrumIcons() let app const loadBudibase = async () => { - if (get(builderStore).clearGridNextLoad) { - builderStore.actions.setDragging(false) - } - // Update builder store with any builder flags builderStore.set({ ...get(builderStore), @@ -45,6 +41,11 @@ const loadBudibase = async () => { location: window["##BUDIBASE_LOCATION##"], }) + // Reset DND state if we completed a successful drop + if (get(dndStore).dropped) { + dndStore.actions.reset() + } + // Set app ID - this window flag is set by both the preview and the real // server rendered app HTML appStore.actions.setAppId(window["##BUDIBASE_APP_ID##"]) diff --git a/packages/client/src/stores/dnd.js b/packages/client/src/stores/dnd.js index 9f17a5aa90..eee338c167 100644 --- a/packages/client/src/stores/dnd.js +++ b/packages/client/src/stores/dnd.js @@ -1,4 +1,5 @@ import { writable, derived } from "svelte/store" +import { computed } from "../utils/computed.js" const createDndStore = () => { const initialState = { @@ -10,6 +11,9 @@ const createDndStore = () => { // Info about where the component would be dropped drop: null, + + // Whether the current drop has been completed successfully + dropped: false, } const store = writable(initialState) @@ -59,6 +63,13 @@ const createDndStore = () => { store.set(initialState) } + const markDropped = () => { + store.update(state => { + state.dropped = true + return state + }) + } + return { subscribe: store.subscribe, actions: { @@ -67,6 +78,7 @@ const createDndStore = () => { updateTarget, updateDrop, reset, + markDropped, }, } } @@ -77,14 +89,12 @@ export const dndStore = createDndStore() // performance by deriving any state that needs to be externally observed. // By doing this and using primitives, we can avoid invalidating other stores // or components which depend on DND state unless values actually change. -export const dndParent = derived(dndStore, $dndStore => $dndStore.drop?.parent) -export const dndIndex = derived(dndStore, $dndStore => $dndStore.drop?.index) -export const dndBounds = derived( +export const dndParent = computed(dndStore, x => x.drop?.parent) +export const dndIndex = computed(dndStore, x => x.drop?.index) +export const dndDropped = computed(dndStore, x => x.dropped) +export const dndBounds = computed(dndStore, x => x.source?.bounds) +export const dndIsDragging = computed(dndStore, x => !!x.source) +export const dndIsNewComponent = computed( dndStore, - $dndStore => $dndStore.source?.bounds + x => x.source?.newComponentType != null ) -export const dndIsNewComponent = derived( - dndStore, - $dndStore => $dndStore.source?.newComponentType != null -) -export const dndIsDragging = derived(dndStore, $dndStore => !!$dndStore.source) diff --git a/packages/client/src/stores/index.js b/packages/client/src/stores/index.js index c431302d43..7a06e55a67 100644 --- a/packages/client/src/stores/index.js +++ b/packages/client/src/stores/index.js @@ -22,6 +22,7 @@ export { dndBounds, dndIsNewComponent, dndIsDragging, + dndDropped, } from "./dnd" // Context stores are layered and duplicated, so it is not a singleton diff --git a/packages/client/src/stores/screens.js b/packages/client/src/stores/screens.js index fdfd5d5673..d9d91ad51a 100644 --- a/packages/client/src/stores/screens.js +++ b/packages/client/src/stores/screens.js @@ -2,7 +2,13 @@ import { derived } from "svelte/store" import { routeStore } from "./routes" import { builderStore } from "./builder" import { appStore } from "./app" -import { dndIndex, dndParent, dndIsNewComponent, dndBounds } from "./dnd.js" +import { + dndIndex, + dndParent, + dndIsNewComponent, + dndBounds, + dndDropped, +} from "./dnd.js" import { RoleUtils } from "@budibase/frontend-core" import { findComponentById, findComponentParent } from "../utils/components.js" import { Helpers } from "@budibase/bbui" @@ -18,6 +24,7 @@ const createScreenStore = () => { dndIndex, dndIsNewComponent, dndBounds, + dndDropped, ], ([ $appStore, @@ -27,6 +34,7 @@ const createScreenStore = () => { $dndIndex, $dndIsNewComponent, $dndBounds, + $dndDropped, ]) => { let activeLayout, activeScreen let screens @@ -64,39 +72,54 @@ const createScreenStore = () => { // Insert DND placeholder if required if (activeScreen && $dndParent && $dndIndex != null) { + const { selectedComponentId } = $builderStore + + // Extract and save the selected component as we need a reference to it + // later, and we may be removing it + let selectedParent = findComponentParent( + activeScreen.props, + selectedComponentId + ) + const selectedComponent = selectedParent?._children?.find( + x => x._id === selectedComponentId + ) + // Remove selected component from tree if we are moving an existing // component - const { selectedComponentId } = $builderStore - if (!$dndIsNewComponent) { - let selectedParent = findComponentParent( - activeScreen.props, - selectedComponentId + if (!$dndIsNewComponent && selectedParent) { + selectedParent._children = selectedParent._children?.filter( + x => x._id !== selectedComponentId ) - if (selectedParent) { - selectedParent._children = selectedParent._children?.filter( - x => x._id !== selectedComponentId - ) - } } - // Insert placeholder component - const placeholder = { - _component: "@budibase/standard-components/container", - _id: DNDPlaceholderID, - _styles: { - normal: { - width: `${$dndBounds?.width || 666}px`, - height: `${$dndBounds?.height || 666}px`, - opacity: 0, - }, - }, - static: true, - } - let parent = findComponentById(activeScreen.props, $dndParent) - if (!parent._children?.length) { - parent._children = [placeholder] + // Insert placeholder component if dragging, or artificially insert + // the dropped component in the new location if the drop completed + let componentToInsert + if ($dndDropped && !$dndIsNewComponent) { + componentToInsert = selectedComponent } else { - parent._children.splice($dndIndex, 0, placeholder) + componentToInsert = { + _component: "@budibase/standard-components/container", + _id: DNDPlaceholderID, + _styles: { + normal: { + width: `${$dndBounds?.width || 400}px`, + height: `${$dndBounds?.height || 200}px`, + opacity: 0, + }, + }, + static: true, + } + } + if (componentToInsert) { + let parent = findComponentById(activeScreen.props, $dndParent) + if (parent) { + if (!parent._children?.length) { + parent._children = [componentToInsert] + } else { + parent._children.splice($dndIndex, 0, componentToInsert) + } + } } } diff --git a/packages/client/src/utils/computed.js b/packages/client/src/utils/computed.js index e69de29bb2..02bf97710a 100644 --- a/packages/client/src/utils/computed.js +++ b/packages/client/src/utils/computed.js @@ -0,0 +1,26 @@ +import { writable } from "svelte/store" + +const getKey = value => { + if (value == null || typeof value !== "object") { + return value + } else { + return JSON.stringify(value) + } +} + +export const computed = (store, getValue) => { + const initialValue = getValue(store) + const computedStore = writable(initialValue) + let lastKey = getKey(initialValue) + + store.subscribe(state => { + const value = getValue(state) + const key = getKey(value) + if (key !== lastKey) { + lastKey = key + computedStore.set(value) + } + }) + + return computedStore +} From 4dc6b869a8f3ca89cc615e614d275e1491e3b335 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Fri, 21 Oct 2022 16:59:26 +0100 Subject: [PATCH 084/151] Lint and improve comments --- packages/client/src/stores/dnd.js | 2 +- packages/client/src/utils/computed.js | 34 ++++++++++++++++++--------- 2 files changed, 24 insertions(+), 12 deletions(-) diff --git a/packages/client/src/stores/dnd.js b/packages/client/src/stores/dnd.js index eee338c167..532dea0ad5 100644 --- a/packages/client/src/stores/dnd.js +++ b/packages/client/src/stores/dnd.js @@ -1,4 +1,4 @@ -import { writable, derived } from "svelte/store" +import { writable } from "svelte/store" import { computed } from "../utils/computed.js" const createDndStore = () => { diff --git a/packages/client/src/utils/computed.js b/packages/client/src/utils/computed.js index 02bf97710a..aa89e7ad1b 100644 --- a/packages/client/src/utils/computed.js +++ b/packages/client/src/utils/computed.js @@ -1,20 +1,22 @@ import { writable } from "svelte/store" -const getKey = value => { - if (value == null || typeof value !== "object") { - return value - } else { - return JSON.stringify(value) - } -} - -export const computed = (store, getValue) => { - const initialValue = getValue(store) +/** + * Extension of Svelte's built in "derived" stores, which the addition of deep + * comparison of non-primitives. Falls back to using shallow comparison for + * primitive types to avoid performance penalties. + * Useful for instances where a deep comparison is cheaper than an additional + * store invalidation. + * @param store the store to observer + * @param deriveValue the derivation function + * @returns {Writable<*>} a derived svelte store containing just the derived value + */ +export const computed = (store, deriveValue) => { + const initialValue = deriveValue(store) const computedStore = writable(initialValue) let lastKey = getKey(initialValue) store.subscribe(state => { - const value = getValue(state) + const value = deriveValue(state) const key = getKey(value) if (key !== lastKey) { lastKey = key @@ -24,3 +26,13 @@ export const computed = (store, getValue) => { return computedStore } + +// Helper function to serialise any value into a primitive which can be cheaply +// and shallowly compared +const getKey = value => { + if (value == null || typeof value !== "object") { + return value + } else { + return JSON.stringify(value) + } +} From 1064095d94cb877ad22150bee5242512a50a3490 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 21 Oct 2022 17:03:01 +0100 Subject: [PATCH 085/151] Updating frontend to allow searching by type and trigger. --- .../portal/overview/backups/BackupsTab.svelte | 51 ++++++++++++------- packages/builder/src/stores/portal/backups.js | 11 +++- packages/frontend-core/src/api/backups.js | 5 +- 3 files changed, 44 insertions(+), 23 deletions(-) diff --git a/packages/builder/src/components/portal/overview/backups/BackupsTab.svelte b/packages/builder/src/components/portal/overview/backups/BackupsTab.svelte index 7ba2bd8370..cb20013b9b 100644 --- a/packages/builder/src/components/portal/overview/backups/BackupsTab.svelte +++ b/packages/builder/src/components/portal/overview/backups/BackupsTab.svelte @@ -24,18 +24,31 @@ let backupData = null let modal - let trigger = null let pageInfo = createPaginationStore() - let startDate - let endDate + let filterOpt = null + let startDate = null + let endDate = null + let filters = getFilters() $: page = $pageInfo.page - $: fetchBackups(trigger, page, startDate, endDate) + $: fetchBackups(filterOpt, page, startDate, endDate) - const triggers = { - PUBLISH: "publish", - SCHEDULED: "scheduled", - MANUAL: "manual", + function getFilters() { + const options = [] + let types = ["backup"] + let triggers = ["manual", "publish", "scheduled", "restoring"] + for (let type of types) { + for (let trigger of triggers) { + let label = `${trigger} ${type}` + label = label.charAt(0).toUpperCase() + label?.slice(1) + options.push({ label, value: { type, trigger } }) + } + } + options.push({ + label: `Manual restore`, + value: { type: "restore", trigger: "manual" }, + }) + return options } const schema = { @@ -80,10 +93,10 @@ }) } - async function fetchBackups(trigger, page, startDate, endDate) { + async function fetchBackups(filters, page, startDate, endDate) { const response = await backups.searchBackups({ appId: app.instance._id, - trigger, + ...filters, page, startDate, endDate, @@ -100,7 +113,7 @@ appId: app.instance._id, name, }) - await fetchBackups(trigger, page) + await fetchBackups(filterOpt, page) notifications.success(response.message) } catch { notifications.error("Unable to create backup") @@ -113,21 +126,21 @@ appId: app.instance._id, backupId: detail.backupId, }) - await fetchBackups(trigger, page) + await fetchBackups(filterOpt, page) } else if (detail.type === "backupRestore") { await backups.restoreBackup({ appId: app.instance._id, backupId: detail.backupId, name: detail.restoreBackupName, }) - await fetchBackups(trigger, page) + await fetchBackups(filterOpt, page) } else if (detail.type === "backupUpdate") { await backups.updateBackup({ appId: app.instance._id, backupId: detail.backupId, name: detail.name, }) - await fetchBackups(trigger, page) + await fetchBackups(filterOpt, page) } } @@ -138,11 +151,11 @@
filter.value} - getOptionLabel={filter => filter.label} - bind:value={filterOpt} - /> -
-
- { - if (e.detail[0].length > 1) { - startDate = e.detail[0][0].toISOString() - endDate = e.detail[0][1].toISOString() - } - }} - /> -
+ {#if !$licensing.backupsEnabled} + + +
+ Backups + + Pro plan + +
+
+ + Backup your apps and restore them to their previous state. + {#if !$auth.accountPortalAccess && !$licensing.groupsEnabled && $admin.cloud} + Contact your account holder to upgrade your plan. + {/if} + +
+ +
+ + + +
+
+
+ {:else if backupData} + +
- {/if} - + + {:else if !backupData} + +
+ BackupsDefault + + You have no backups yet +
+ You can manually backup your app any time +
+
+ +
+
+
+
+ {/if} @@ -242,4 +309,21 @@ flex: 1; gap: var(--spacing-xl); } + + .title { + display: flex; + flex-direction: row; + align-items: center; + gap: var(--spacing-m); + } + + .align { + margin-top: 5%; + text-align: center; + } + + .pro-buttons { + display: flex; + gap: var(--spacing-m); + } diff --git a/packages/builder/src/stores/portal/licensing.js b/packages/builder/src/stores/portal/licensing.js index 179dac9689..59a1622c9f 100644 --- a/packages/builder/src/stores/portal/licensing.js +++ b/packages/builder/src/stores/portal/licensing.js @@ -14,6 +14,7 @@ export const createLicensingStore = () => { isFreePlan: true, // features groupsEnabled: false, + backupsEnabled: false, // the currently used quotas from the db quotaUsage: undefined, // derived quota metrics for percentages used @@ -56,12 +57,17 @@ export const createLicensingStore = () => { const groupsEnabled = license.features.includes( Constants.Features.USER_GROUPS ) + const backupsEnabled = license.features.includes( + Constants.Features.BACKUPS + ) + store.update(state => { return { ...state, license, isFreePlan, groupsEnabled, + backupsEnabled, } }) }, diff --git a/packages/frontend-core/src/constants.js b/packages/frontend-core/src/constants.js index 9a5acf8a9b..1eed492c25 100644 --- a/packages/frontend-core/src/constants.js +++ b/packages/frontend-core/src/constants.js @@ -113,6 +113,7 @@ export const ApiVersion = "1" export const Features = { USER_GROUPS: "userGroups", + BACKUPS: "appBackups", } // Role IDs From 1aca8756f663bdff694e4ed6a45dc5aa29918ce9 Mon Sep 17 00:00:00 2001 From: Peter Clement Date: Mon, 24 Oct 2022 15:09:32 +0100 Subject: [PATCH 117/151] pr comments --- packages/bbui/src/Form/DatePicker.svelte | 2 +- .../portal/overview/backups/ActionsRenderer.svelte | 1 - .../portal/overview/backups/StatusRenderer.svelte | 2 +- packages/builder/src/stores/portal/backups.js | 5 ----- packages/frontend-core/src/api/backups.js | 6 ------ 5 files changed, 2 insertions(+), 14 deletions(-) diff --git a/packages/bbui/src/Form/DatePicker.svelte b/packages/bbui/src/Form/DatePicker.svelte index 6a0ba4ad36..04ce8b5467 100644 --- a/packages/bbui/src/Form/DatePicker.svelte +++ b/packages/bbui/src/Form/DatePicker.svelte @@ -21,7 +21,7 @@ if (range) { // Flatpickr cant take two dates and work out what to display, needs to be provided a string. // Like - "Date1 to Date2". Hence passing in that specifically from the array - value = e.detail[1] + value = e?.detail[1] } else { value = e.detail } diff --git a/packages/builder/src/components/portal/overview/backups/ActionsRenderer.svelte b/packages/builder/src/components/portal/overview/backups/ActionsRenderer.svelte index bb09ad3f6d..8d1dfb52c5 100644 --- a/packages/builder/src/components/portal/overview/backups/ActionsRenderer.svelte +++ b/packages/builder/src/components/portal/overview/backups/ActionsRenderer.svelte @@ -101,7 +101,6 @@ warning={false} > -