From d90123e8de29a40ebfc15542ded58701de722da7 Mon Sep 17 00:00:00 2001 From: Dean Date: Thu, 8 Feb 2024 11:05:45 +0000 Subject: [PATCH 001/114] Wip --- .../NewScreen/CreateScreenModal.svelte | 83 +++++++++++++++---- 1 file changed, 69 insertions(+), 14 deletions(-) diff --git a/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/CreateScreenModal.svelte b/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/CreateScreenModal.svelte index a9d64afd19..ab8ccecf6e 100644 --- a/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/CreateScreenModal.svelte +++ b/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/CreateScreenModal.svelte @@ -2,6 +2,7 @@ import ScreenDetailsModal from "components/design/ScreenDetailsModal.svelte" import DatasourceModal from "./DatasourceModal.svelte" import ScreenRoleModal from "./ScreenRoleModal.svelte" + import FormTypeModal from "./FormTypeModal.svelte" import sanitizeUrl from "builderStore/store/screenTemplates/utils/sanitizeUrl" import { Modal, notifications } from "@budibase/bbui" import { store } from "builderStore" @@ -19,12 +20,17 @@ let screenDetailsModal let datasourceModal let screenAccessRoleModal + let formTypeModal // Cache variables for workflow let screenAccessRole = Roles.BASIC + let selectedTemplates = null + + let selectedDatasources = null let blankScreenUrl = null let screenMode = null + let formType = null // Creates an array of screens, checking and sanitising their URLs const createScreens = async ({ screens, screenAccessRole }) => { @@ -56,20 +62,21 @@ screen.routing.roleId = screenAccessRole // Create the screen - const response = await store.actions.screens.save(screen) - screenId = response._id + // const response = await store.actions.screens.save(screen) + // screenId = response._id // Add link in layout. We only ever actually create 1 screen now, even // for autoscreens, so it's always safe to do this. - await store.actions.links.save( - screen.routing.route, - capitalise(screen.routing.route.split("/")[1]) - ) + // await store.actions.links.save( + // screen.routing.route, + // capitalise(screen.routing.route.split("/")[1]) + // ) + console.log(screen) } // Go to new screen - $goto(`./${screenId}`) - store.actions.screens.select(screenId) + //$goto(`./${screenId}`) + //store.actions.screens.select(screenId) } catch (error) { console.log(error) notifications.error("Error creating screens") @@ -103,13 +110,15 @@ // Handler for NewScreenModal export const show = newMode => { mode = newMode - selectedTemplates = null + // selectedTemplates = null + selectedDatasources = null blankScreenUrl = null screenMode = mode pendingScreen = null screenAccessRole = Roles.BASIC + formType = null - if (mode === "table" || mode === "grid") { + if (mode === "table" || mode === "grid" || mode === "form") { datasourceModal.show() } else if (mode === "blank") { let templates = getTemplates($tables.list) @@ -124,8 +133,9 @@ } // Handler for DatasourceModal confirmation, move to screen access select - const confirmScreenDatasources = async ({ templates }) => { - selectedTemplates = templates + const confirmScreenDatasources = async ({ datasources }) => { + selectedDatasources = datasources + console.log("confirmScreenDatasources ", datasources) screenAccessRoleModal.show() } @@ -136,6 +146,14 @@ screenTemplate.autoTableId = template.resourceId return screenTemplate }) + console.log("selectedTemplates ", selectedTemplates) + /* + + id : "ROW_LIST_TEMPLATE" + name : "Employees - List" + resourceId : "ta_bb_employee" + + */ await createScreens({ screens, screenAccessRole }) } @@ -175,8 +193,14 @@ datasourceModal.show() } } + window.test = () => { + formTypeModal.show() + } + { + if (screenMode === "form") { + formTypeModal.show() + } else { + confirmScreenCreation() + } + }} bind:screenAccessRole + onCancel={roleSelectBack} screenUrl={blankScreenUrl} + confirmText={screenMode === "form" ? "Confirm" : "Done"} /> @@ -200,3 +231,27 @@ initialUrl={blankScreenUrl} /> + + { + console.log("hide") + //formType = null + }} +> + { + console.log("test confirm") + }} + onCancel={() => { + console.log("cancel") + formTypeModal.hide() + screenAccessRoleModal.show() + }} + on:select={e => { + console.log("form type selection ", e.detail) + formType = e.detail + }} + type={formType} + /> + From e9e5281e820589f82b091db479b5b72dc2b94db8 Mon Sep 17 00:00:00 2001 From: Dean Date: Wed, 14 Feb 2024 12:11:24 +0000 Subject: [PATCH 002/114] Initial commit form screen flow and tour refactor --- .../builder/src/builderStore/dataBinding.js | 2 +- .../src/builderStore/store/frontend.js | 6 +- .../store/screenTemplates/formScreen.js | 43 ++++++ .../store/screenTemplates/index.js | 7 +- .../src/components/deploy/AppActions.svelte | 9 +- .../ButtonConfiguration/ButtonSetting.svelte | 2 +- .../EditComponentPopover.svelte | 22 +-- .../controls/EditComponentPopover/index.js | 18 +++ .../FieldConfiguration/FieldSetting.svelte | 2 +- .../controls/FormStepConfiguration.svelte | 4 +- .../FieldSetting.svelte | 2 +- .../PrimaryColumnFieldSetting.svelte | 2 +- .../settings/controls/PropertyControl.svelte | 25 +++- .../portal/onboarding/TourPopover.svelte | 3 +- .../portal/onboarding/TourWrap.svelte | 29 ++-- .../steps/NewViewUpdateFormRowId.svelte | 17 +++ .../portal/onboarding/steps/index.js | 1 + .../src/components/portal/onboarding/tours.js | 119 +++++++++++++-- .../builder/app/[application]/_layout.svelte | 2 +- .../Component/ComponentSettingsPanel.svelte | 35 ++++- .../Component/ComponentSettingsSection.svelte | 4 +- .../[screenId]/_components/AppPreview.svelte | 2 +- .../NewScreen/CreateScreenModal.svelte | 137 +++++++++++------- .../NewScreen/DatasourceModal.svelte | 30 ++-- .../NewScreen/FormTypeModal.svelte | 78 ++++++++++ .../NewScreen/ScreenRoleModal.svelte | 3 +- .../NewScreen/{ => images}/blank.png | Bin .../_components/NewScreen/images/form.png | Bin 0 -> 22892 bytes .../NewScreen/{ => images}/grid.png | Bin .../NewScreen/{ => images}/table.png | Bin .../design/_components/NewScreen/index.svelte | 17 ++- packages/types/src/api/web/auth.ts | 1 + packages/types/src/documents/global/user.ts | 1 + .../worker/src/api/routes/validation/users.ts | 1 + 34 files changed, 478 insertions(+), 146 deletions(-) create mode 100644 packages/builder/src/builderStore/store/screenTemplates/formScreen.js rename packages/builder/src/components/design/settings/controls/{ => EditComponentPopover}/EditComponentPopover.svelte (79%) create mode 100644 packages/builder/src/components/design/settings/controls/EditComponentPopover/index.js create mode 100644 packages/builder/src/components/portal/onboarding/steps/NewViewUpdateFormRowId.svelte create mode 100644 packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/FormTypeModal.svelte rename packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/{ => images}/blank.png (100%) create mode 100644 packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/images/form.png rename packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/{ => images}/grid.png (100%) rename packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/{ => images}/table.png (100%) diff --git a/packages/builder/src/builderStore/dataBinding.js b/packages/builder/src/builderStore/dataBinding.js index edea3b9ec7..9cb7b3311b 100644 --- a/packages/builder/src/builderStore/dataBinding.js +++ b/packages/builder/src/builderStore/dataBinding.js @@ -1131,7 +1131,7 @@ export const getAllStateVariables = () => { "@budibase/standard-components/multistepformblockstep" ) - steps.forEach(step => { + steps?.forEach(step => { parseComponentSettings(stepDefinition, step) }) }) diff --git a/packages/builder/src/builderStore/store/frontend.js b/packages/builder/src/builderStore/store/frontend.js index 55208bb97e..456f0658fc 100644 --- a/packages/builder/src/builderStore/store/frontend.js +++ b/packages/builder/src/builderStore/store/frontend.js @@ -75,7 +75,7 @@ const INITIAL_FRONTEND_STATE = { theme: "", customTheme: {}, previewDevice: "desktop", - highlightedSettingKey: null, + highlightedSetting: null, propertyFocus: null, builderSidePanel: false, hasLock: true, @@ -1460,10 +1460,10 @@ export const getFrontendStore = () => { }, }, settings: { - highlight: key => { + highlight: (key, type) => { store.update(state => ({ ...state, - highlightedSettingKey: key, + highlightedSetting: { key, type: type || "info" }, })) }, propertyFocus: key => { diff --git a/packages/builder/src/builderStore/store/screenTemplates/formScreen.js b/packages/builder/src/builderStore/store/screenTemplates/formScreen.js new file mode 100644 index 0000000000..8ce46cd002 --- /dev/null +++ b/packages/builder/src/builderStore/store/screenTemplates/formScreen.js @@ -0,0 +1,43 @@ +import { Screen } from "./utils/Screen" +import { Component } from "./utils/Component" +import sanitizeUrl from "./utils/sanitizeUrl" + +export const FORM_TEMPLATE = "FORM_TEMPLATE" +export const formUrl = datasource => sanitizeUrl(`/${datasource.label}-form`) + +// Mode not really necessary +export default function (datasources, config) { + if (!Array.isArray(datasources)) { + return [] + } + return datasources.map(datasource => { + return { + name: `${datasource.label} - Form`, + create: () => createScreen(datasource, config), + id: FORM_TEMPLATE, + resourceId: datasource.resourceId, + } + }) +} + +const generateMultistepFormBlock = (dataSource, { actionType } = {}) => { + const multistepFormBlock = new Component( + "@budibase/standard-components/multistepformblock" + ) + multistepFormBlock + .customProps({ + actionType, + dataSource, + steps: [{}], + }) + .instanceName(`${dataSource.label} - Multistep Form block`) + return multistepFormBlock +} + +const createScreen = (datasource, config) => { + return new Screen() + .route(formUrl(datasource)) + .instanceName(`${datasource.label} - Form`) + .addChild(generateMultistepFormBlock(datasource, config)) + .json() +} diff --git a/packages/builder/src/builderStore/store/screenTemplates/index.js b/packages/builder/src/builderStore/store/screenTemplates/index.js index 3ff42fdec6..fff31cc070 100644 --- a/packages/builder/src/builderStore/store/screenTemplates/index.js +++ b/packages/builder/src/builderStore/store/screenTemplates/index.js @@ -1,7 +1,11 @@ import rowListScreen from "./rowListScreen" import createFromScratchScreen from "./createFromScratchScreen" +import formScreen from "./formScreen" -const allTemplates = datasources => [...rowListScreen(datasources)] +const allTemplates = datasources => [ + ...rowListScreen(datasources), + ...formScreen(datasources), +] // Allows us to apply common behaviour to all create() functions const createTemplateOverride = template => () => { @@ -19,6 +23,7 @@ export default datasources => { }) const fromScratch = enrichTemplate(createFromScratchScreen) const tableTemplates = allTemplates(datasources).map(enrichTemplate) + return [ fromScratch, ...tableTemplates.sort((templateA, templateB) => { diff --git a/packages/builder/src/components/deploy/AppActions.svelte b/packages/builder/src/components/deploy/AppActions.svelte index 7d14fd0e87..bf59c3a230 100644 --- a/packages/builder/src/components/deploy/AppActions.svelte +++ b/packages/builder/src/components/deploy/AppActions.svelte @@ -156,9 +156,10 @@ {/if}
@@ -204,7 +205,7 @@
- + Publish - import EditComponentPopover from "../EditComponentPopover.svelte" + import EditComponentPopover from "../EditComponentPopover/EditComponentPopover.svelte" import { Icon } from "@budibase/bbui" import { runtimeToReadableBinding } from "builderStore/dataBinding" import { isJSBinding } from "@budibase/string-templates" diff --git a/packages/builder/src/components/design/settings/controls/EditComponentPopover.svelte b/packages/builder/src/components/design/settings/controls/EditComponentPopover/EditComponentPopover.svelte similarity index 79% rename from packages/builder/src/components/design/settings/controls/EditComponentPopover.svelte rename to packages/builder/src/components/design/settings/controls/EditComponentPopover/EditComponentPopover.svelte index 04bb925873..af535a00f0 100644 --- a/packages/builder/src/components/design/settings/controls/EditComponentPopover.svelte +++ b/packages/builder/src/components/design/settings/controls/EditComponentPopover/EditComponentPopover.svelte @@ -3,7 +3,8 @@ import { store } from "builderStore" import { cloneDeep } from "lodash/fp" import { createEventDispatcher, getContext } from "svelte" - import ComponentSettingsSection from "../../../../pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ComponentSettingsSection.svelte" + import { customPositionHandler } from "." + import ComponentSettingsSection from "../../../../../pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ComponentSettingsSection.svelte" export let anchor export let componentInstance @@ -59,25 +60,6 @@ dispatch("change", nestedComponentInstance) } - - const customPositionHandler = (anchorBounds, eleBounds, cfg) => { - let { left, top } = cfg - let percentageOffset = 30 - // left-outside - left = anchorBounds.left - eleBounds.width - 18 - - // shift up from the anchor, if space allows - let offsetPos = Math.floor(eleBounds.height / 100) * percentageOffset - let defaultTop = anchorBounds.top - offsetPos - - if (window.innerHeight - defaultTop < eleBounds.height) { - top = window.innerHeight - eleBounds.height - 5 - } else { - top = anchorBounds.top - offsetPos - } - - return { ...cfg, left, top } - } { + let { left, top } = cfg + let percentageOffset = 30 + // left-outside + left = anchorBounds.left - eleBounds.width - 18 + + // shift up from the anchor, if space allows + let offsetPos = Math.floor(eleBounds.height / 100) * percentageOffset + let defaultTop = anchorBounds.top - offsetPos + + if (window.innerHeight - defaultTop < eleBounds.height) { + top = window.innerHeight - eleBounds.height - 5 + } else { + top = anchorBounds.top - offsetPos + } + + return { ...cfg, left, top } +} diff --git a/packages/builder/src/components/design/settings/controls/FieldConfiguration/FieldSetting.svelte b/packages/builder/src/components/design/settings/controls/FieldConfiguration/FieldSetting.svelte index 8c40c455c8..94ce698ff1 100644 --- a/packages/builder/src/components/design/settings/controls/FieldConfiguration/FieldSetting.svelte +++ b/packages/builder/src/components/design/settings/controls/FieldConfiguration/FieldSetting.svelte @@ -1,5 +1,5 @@
{#if label && !labelHidden} @@ -115,6 +120,16 @@
diff --git a/packages/builder/src/components/portal/onboarding/steps/index.js b/packages/builder/src/components/portal/onboarding/steps/index.js index 8e27748f36..6694ce97a7 100644 --- a/packages/builder/src/components/portal/onboarding/steps/index.js +++ b/packages/builder/src/components/portal/onboarding/steps/index.js @@ -1,3 +1,4 @@ export { default as OnboardingData } from "./OnboardingData.svelte" export { default as OnboardingDesign } from "./OnboardingDesign.svelte" export { default as OnboardingPublish } from "./OnboardingPublish.svelte" +export { default as NewViewUpdateFormRowId } from "./NewViewUpdateFormRowId.svelte" diff --git a/packages/builder/src/components/portal/onboarding/tours.js b/packages/builder/src/components/portal/onboarding/tours.js index 55fd4c4a9b..fdc00bf32d 100644 --- a/packages/builder/src/components/portal/onboarding/tours.js +++ b/packages/builder/src/components/portal/onboarding/tours.js @@ -2,8 +2,14 @@ import { get } from "svelte/store" import { store } from "builderStore" import { auth } from "stores/portal" import analytics from "analytics" -import { OnboardingData, OnboardingDesign, OnboardingPublish } from "./steps" +import { + OnboardingData, + OnboardingDesign, + OnboardingPublish, + NewViewUpdateFormRowId, +} from "./steps" import { API } from "api" +import { customPositionHandler } from "components/design/settings/controls/EditComponentPopover" const ONBOARDING_EVENT_PREFIX = "onboarding" @@ -14,11 +20,26 @@ export const TOUR_STEP_KEYS = { BUILDER_USER_MANAGEMENT: "builder-user-management", BUILDER_AUTOMATION_SECTION: "builder-automation-section", FEATURE_USER_MANAGEMENT: "feature-user-management", + BUILDER_FORM_CREATE_STEPS: "builder-form-create-steps", + BUILDER_FORM_VIEW_UPDATE_STEPS: "builder-form-view-update-steps", + BUILDER_FORM_ROW_ID: "builder-form-row-id", } export const TOUR_KEYS = { TOUR_BUILDER_ONBOARDING: "builder-onboarding", FEATURE_ONBOARDING: "feature-onboarding", + BUILDER_FORM_CREATE: "builder-form-create", + BUILDER_FORM_VIEW_UPDATE: "builder-form-view-update", +} + +const resetTourState = () => { + store.update(state => ({ + ...state, + tourNodes: undefined, + tourKey: undefined, + tourKeyStep: undefined, + onboarding: false, + })) } const endUserOnboarding = async ({ skipped = false } = {}) => { @@ -37,13 +58,7 @@ const endUserOnboarding = async ({ skipped = false } = {}) => { // Update the cached user await auth.getSelf() - store.update(state => ({ - ...state, - tourNodes: undefined, - tourKey: undefined, - tourKeyStep: undefined, - onboarding: false, - })) + resetTourState() } catch (e) { console.error("Onboarding failed", e) return false @@ -52,9 +67,28 @@ const endUserOnboarding = async ({ skipped = false } = {}) => { } } -const tourEvent = eventKey => { +const endTour = async ({ key, skipped = false } = {}) => { + const { tours = {} } = get(auth).user + tours[key] = new Date().toISOString() + + await API.updateSelf({ + tours, + }) + + if (skipped) { + tourEvent(key, skipped) + } + + // Update the cached user + await auth.getSelf() + + resetTourState() +} + +const tourEvent = (eventKey, skipped) => { analytics.captureEvent(`${ONBOARDING_EVENT_PREFIX}:${eventKey}`, { eventSource: EventSource.PORTAL, + skipped, }) } @@ -135,7 +169,74 @@ const getTours = () => { }, ], }, + [TOUR_KEYS.BUILDER_FORM_CREATE]: { + steps: [ + { + id: TOUR_STEP_KEYS.BUILDER_FORM_CREATE_STEPS, + title: "Add multiple steps", + body: `When faced with a sizable form, consider implementing a multi-step + approach to enhance user experience. Breaking the form into multiple steps + can significantly improve usability by making the process more digestible for your users.`, + query: "#steps-prop-control-wrap", + onComplete: () => { + store.actions.settings.highlight() + endTour({ key: TOUR_KEYS.BUILDER_FORM_CREATE }) + }, + onLoad: () => { + tourEvent(TOUR_STEP_KEYS.BUILDER_FORM_CREATE_STEPS) + store.actions.settings.highlight("steps", "info") + }, + positionHandler: customPositionHandler, + align: "left-outside", + }, + ], + }, + [TOUR_KEYS.BUILDER_FORM_VIEW_UPDATE]: { + steps: [ + { + id: TOUR_STEP_KEYS.BUILDER_FORM_ROW_ID, + title: "Add row ID to update a row", + layout: NewViewUpdateFormRowId, + query: "#rowId-prop-control-wrap", + onLoad: () => { + tourEvent(TOUR_STEP_KEYS.BUILDER_FORM_ROW_ID) + store.actions.settings.highlight("rowId", "info") + }, + positionHandler: customPositionHandler, + align: "left-outside", + }, + { + id: TOUR_STEP_KEYS.BUILDER_FORM_VIEW_UPDATE_STEPS, + title: "Add multiple steps", + body: `When faced with a sizable form, consider implementing a multi-step + approach to enhance user experience. Breaking the form into multiple steps + can significantly improve usability by making the process more digestible for your users.`, + query: "#steps-prop-control-wrap", + onComplete: () => { + store.actions.settings.highlight() + endTour({ key: TOUR_KEYS.BUILDER_FORM_VIEW_UPDATE }) + }, + onLoad: () => { + tourEvent(TOUR_STEP_KEYS.BUILDER_FORM_VIEW_UPDATE_STEPS) + store.actions.settings.highlight("steps", "info") + }, + positionHandler: customPositionHandler, + align: "left-outside", + }, + ], + onSkip: async () => { + store.actions.settings.highlight() + endTour({ key: TOUR_KEYS.BUILDER_FORM_VIEW_UPDATE, skipped: true }) + }, + }, } } export const TOURS = getTours() +export const TOURSBYSTEP = Object.keys(TOURS).reduce((acc, tour) => { + TOURS[tour].steps.forEach(element => { + acc[element.id] = element + acc[element.id]["tour"] = tour + }) + return acc +}, {}) diff --git a/packages/builder/src/pages/builder/app/[application]/_layout.svelte b/packages/builder/src/pages/builder/app/[application]/_layout.svelte index 1df2a90250..5a6e9c941e 100644 --- a/packages/builder/src/pages/builder/app/[application]/_layout.svelte +++ b/packages/builder/src/pages/builder/app/[application]/_layout.svelte @@ -151,7 +151,7 @@ {#each $layout.children as { path, title }} - + import Panel from "components/design/Panel.svelte" import { store, selectedComponent, selectedScreen } from "builderStore" + import { auth } from "stores/portal" import { getComponentName } from "builderStore/componentUtils" import ComponentSettingsSection from "./ComponentSettingsSection.svelte" import DesignSection from "./DesignSection.svelte" import CustomStylesSection from "./CustomStylesSection.svelte" import ConditionalUISection from "./ConditionalUISection.svelte" import { notifications, ActionButton } from "@budibase/bbui" + import TourWrap from "components/portal/onboarding/TourWrap.svelte" + import { + TOUR_STEP_KEYS, + TOUR_KEYS, + } from "components/portal/onboarding/tours.js" import { getBindableProperties, @@ -14,6 +20,12 @@ } from "builderStore/dataBinding" import { capitalise } from "helpers" + const { + BUILDER_FORM_CREATE_STEPS, + BUILDER_FORM_VIEW_UPDATE_STEPS, + BUILDER_FORM_ROW_ID, + } = TOUR_STEP_KEYS + const onUpdateName = async value => { try { await store.actions.components.updateSetting("_instanceName", value) @@ -43,7 +55,6 @@ $: id = $selectedComponent?._id $: id, (section = tabs[0]) - $: componentName = getComponentName(componentInstance) @@ -89,13 +100,21 @@
{#if section == "settings"} - + + + {/if} {#if section == "styles"} updateSetting(setting, val)} - highlighted={$store.highlightedSettingKey === setting.key} + highlighted={$store.highlightedSetting?.key === setting.key + ? $store.highlightedSetting + : null} propertyFocus={$store.propertyFocus === setting.key} info={setting.info} disableBindings={setting.disableBindings} diff --git a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/AppPreview.svelte b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/AppPreview.svelte index 011980bbe2..c9dc4f8982 100644 --- a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/AppPreview.svelte +++ b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/AppPreview.svelte @@ -161,7 +161,7 @@ } else if (type === "request-add-component") { toggleAddComponent() } else if (type === "highlight-setting") { - store.actions.settings.highlight(data.setting) + store.actions.settings.highlight(data.setting, "error") // Also scroll setting into view const selector = `#${data.setting}-prop-control` diff --git a/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/CreateScreenModal.svelte b/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/CreateScreenModal.svelte index 2a2459949d..a61e7551e7 100644 --- a/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/CreateScreenModal.svelte +++ b/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/CreateScreenModal.svelte @@ -4,14 +4,18 @@ import ScreenRoleModal from "./ScreenRoleModal.svelte" import FormTypeModal from "./FormTypeModal.svelte" import sanitizeUrl from "builderStore/store/screenTemplates/utils/sanitizeUrl" + import rowListScreen from "builderStore/store/screenTemplates/rowListScreen" + import formScreen from "builderStore/store/screenTemplates/formScreen" import { Modal, notifications } from "@budibase/bbui" import { store } from "builderStore" import { get } from "svelte/store" import getTemplates from "builderStore/store/screenTemplates" import { tables } from "stores/backend" + import { auth } from "stores/portal" import { Roles } from "constants/backend" import { capitalise } from "helpers" import { goto } from "@roxi/routify" + import { TOUR_KEYS } from "components/portal/onboarding/tours.js" let mode let pendingScreen @@ -25,7 +29,8 @@ // Cache variables for workflow let screenAccessRole = Roles.BASIC - let selectedTemplates = null + let templates = null + let screens = null let selectedDatasources = null let blankScreenUrl = null @@ -40,6 +45,7 @@ try { let screenId + let createdScreens = [] for (let screen of screens) { // Check we aren't clashing with an existing URL @@ -62,21 +68,19 @@ screen.routing.roleId = screenAccessRole // Create the screen - // const response = await store.actions.screens.save(screen) - // screenId = response._id + const response = await store.actions.screens.save(screen) + screenId = response._id + createdScreens.push(response) // Add link in layout. We only ever actually create 1 screen now, even // for autoscreens, so it's always safe to do this. - // await store.actions.links.save( - // screen.routing.route, - // capitalise(screen.routing.route.split("/")[1]) - // ) - console.log(screen) + await store.actions.links.save( + screen.routing.route, + capitalise(screen.routing.route.split("/")[1]) + ) } - // Go to new screen - //$goto(`./${screenId}`) - //store.actions.screens.select(screenId) + return createdScreens } catch (error) { console.error(error) notifications.error("Error creating screens") @@ -110,7 +114,8 @@ // Handler for NewScreenModal export const show = newMode => { mode = newMode - // selectedTemplates = null + templates = null + screens = null selectedDatasources = null blankScreenUrl = null screenMode = mode @@ -135,26 +140,24 @@ // Handler for DatasourceModal confirmation, move to screen access select const confirmScreenDatasources = async ({ datasources }) => { selectedDatasources = datasources - console.log("confirmScreenDatasources ", datasources) - screenAccessRoleModal.show() + if (screenMode === "form") { + formTypeModal.show() + } else { + screenAccessRoleModal.show() + } } // Handler for Datasource Screen Creation const completeDatasourceScreenCreation = async () => { - const screens = selectedTemplates.map(template => { + templates = rowListScreen(selectedDatasources) + + const screens = templates.map(template => { let screenTemplate = template.create() screenTemplate.autoTableId = template.resourceId return screenTemplate }) - console.log("selectedTemplates ", selectedTemplates) - /* - - id : "ROW_LIST_TEMPLATE" - name : "Employees - List" - resourceId : "ta_bb_employee" - - */ - await createScreens({ screens, screenAccessRole }) + const createdScreens = await createScreens({ screens, screenAccessRole }) + loadNewScreen(createdScreens) } const confirmScreenBlank = async ({ screenUrl }) => { @@ -171,7 +174,55 @@ return } pendingScreen.routing.route = screenUrl - await createScreens({ screens: [pendingScreen], screenAccessRole }) + const createdScreens = await createScreens({ + screens: [pendingScreen], + screenAccessRole, + }) + loadNewScreen(createdScreens) + } + + const onConfirmFormType = () => { + screenAccessRoleModal.show() + } + + const loadNewScreen = createdScreens => { + const lastScreen = createdScreens.slice(-1) + + // Go to new screen + $goto(`./${lastScreen._id}`) + store.actions.screens.select(lastScreen._id) + } + + const confirmFormScreenCreation = async () => { + templates = formScreen(selectedDatasources, { actionType: formType }) + screens = templates.map(template => { + let screenTemplate = template.create() + return screenTemplate + }) + const createdScreens = await createScreens({ screens, screenAccessRole }) + const lastScreen = createdScreens?.slice(-1)?.pop() + const mainComponent = lastScreen?.props?._children?.[0]._id + + if (formType === "Update" || formType === "Create") { + const associatedTour = + formType === "Update" + ? TOUR_KEYS.BUILDER_FORM_VIEW_UPDATE + : TOUR_KEYS.BUILDER_FORM_CREATE + + const tourRequired = !$auth?.user?.tours?.[associatedTour] + if (tourRequired) { + store.update(state => ({ + ...state, + tourStepKey: null, + tourNodes: null, + tourKey: associatedTour, + })) + } + } + + // Go to new screen + $goto(`./${lastScreen._id}/${mainComponent}`) + store.actions.screens.select(lastScreen._id) } // Submit screen config for creation. @@ -181,6 +232,8 @@ screenUrl: blankScreenUrl, screenAccessRole, }) + } else if (screenMode === "form") { + confirmFormScreenCreation() } else { completeDatasourceScreenCreation() } @@ -193,30 +246,16 @@ datasourceModal.show() } } - window.test = () => { - formTypeModal.show() - } - - + { - if (screenMode === "form") { - formTypeModal.show() - } else { - confirmScreenCreation() - } + confirmScreenCreation() }} bind:screenAccessRole onCancel={roleSelectBack} @@ -232,24 +271,14 @@ /> - { - console.log("hide") - //formType = null - }} -> + { - console.log("test confirm") - }} + onConfirm={onConfirmFormType} onCancel={() => { - console.log("cancel") formTypeModal.hide() - screenAccessRoleModal.show() + datasourceModal.show() }} on:select={e => { - console.log("form type selection ", e.detail) formType = e.detail }} type={formType} diff --git a/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/DatasourceModal.svelte b/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/DatasourceModal.svelte index 731c60a406..4348c17312 100644 --- a/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/DatasourceModal.svelte +++ b/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/DatasourceModal.svelte @@ -4,37 +4,33 @@ import ICONS from "components/backend/DatasourceNavigator/icons" import { IntegrationNames } from "constants" import { onMount } from "svelte" - import rowListScreen from "builderStore/store/screenTemplates/rowListScreen" import DatasourceTemplateRow from "./DatasourceTemplateRow.svelte" - export let mode export let onCancel export let onConfirm - export let initialScreens = [] - let selectedScreens = [...initialScreens] + let selectedSources = [] $: filteredSources = $datasources.list?.filter(datasource => { return datasource.source !== IntegrationNames.REST && datasource["entities"] }) const toggleSelection = datasource => { - const { resourceId } = datasource - if (selectedScreens.find(s => s.resourceId === resourceId)) { - selectedScreens = selectedScreens.filter( - screen => screen.resourceId !== resourceId + const exists = selectedSources.find( + d => d.resourceId === datasource.resourceId + ) + if (exists) { + selectedSources = selectedSources.filter( + d => d.resourceId === datasource.resourceId ) } else { - selectedScreens = [ - ...selectedScreens, - rowListScreen([datasource], mode)[0], - ] + selectedSources = [...selectedSources, datasource] } } const confirmDatasourceSelection = async () => { await onConfirm({ - templates: selectedScreens, + datasources: selectedSources, }) } @@ -54,7 +50,7 @@ cancelText="Back" onConfirm={confirmDatasourceSelection} {onCancel} - disabled={!selectedScreens.length} + disabled={!selectedSources.length} size="L" > @@ -85,8 +81,8 @@ resourceId: table._id, type: "table", }} - {@const selected = selectedScreens.find( - screen => screen.resourceId === tableDS.resourceId + {@const selected = selectedSources.find( + datasource => datasource.resourceId === tableDS.resourceId )} toggleSelection(tableDS)} @@ -103,7 +99,7 @@ tableId: view.tableId, type: "viewV2", }} - {@const selected = selectedScreens.find( + {@const selected = selectedSources.find( x => x.resourceId === viewDS.resourceId )} + import { ModalContent, Layout, Body, Label } from "@budibase/bbui" + import { createEventDispatcher } from "svelte" + + export let onCancel = () => {} + export let onConfirm = () => {} + export let type + + const dispatch = createEventDispatcher() + + + + + + +
{ + dispatch("select", "Create") + }} + > + Create a new row + For capturing and storing new data from your users +
+
{ + dispatch("select", "Update") + }} + > + Update an existing row + For viewing and updating existing data +
+
{ + dispatch("select", "View") + }} + > + View an existing row + For a read only view of your data +
+ + + + + diff --git a/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/ScreenRoleModal.svelte b/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/ScreenRoleModal.svelte index 5d73b7961c..9363523a63 100644 --- a/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/ScreenRoleModal.svelte +++ b/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/ScreenRoleModal.svelte @@ -10,6 +10,7 @@ export let onCancel export let screenUrl export let screenAccessRole + export let confirmText = "Done" let error @@ -41,7 +42,7 @@ lgxV!H3f7O5)=RcfUcw{s|5f6=U*RqB=9T8YjjKa^-qL@jEshoj0}~# zr<<*Vvkd^ilp35WrZ_A~HfpXdmr5Wd5-SK_t>&wVJ#G|5ql45gU5mkx6(W@$O*cBK z>CP^A+ic+D?yhi6(g@1%Z~mvfUSg5V`uw5 z_DnyziAg7)4d%xUU~~c6YKAGZqBavA^oHlYf{fE2^|Jn5Z~L{!J;1uEb^+bp9Vvm3 zIdF4tvp2u@DX?O38`BEQv&do_{`PfOz?N?nWC1V#{fc@k(_cBr?uv$9001iPe-99l zlSlN*MD$itl|$SIp`+0V3J&)(0RU70C0QvQ|Fx6g%_KG_PhHtU9{;hHaVR@H$qov- z^e)98@P^?ovp;=5AQ`&zL4@wj4(N|nm`$R~>Rf`@U;i<$-<1*Zo0}r6tTanT=6brz zr*SFrt?GV0wo6&r&6cBt%|oG256@OSfdO~h`W!ETC_H{VJUpFE%c0rXQBh*K>y4Ag zgtph6w%53R4+mZ=(^JRGwr-&wj(`4Ms4FBazl7WeMRUG9UQC5b^po?Lgh=i=PcEw~Hik>|)bVUmUK6No}f4{n|EPQgF{<7`dZ)^t)nTGXm6hEn(>vOkO zW)=3EJ>cFxUc=0Se$Q|F_VW>*06fedvi^G`P0keIwUaQbR{@u z^IhT1_-e@3$-Z#GEez@5{>#a_?X~gm>rJGGq?q8Z8-3h7$q|@+ql~hbzv<7-U;18H zx&)Im1vtcub9)m1(dL+N8(OP-tJCn@au3p%#q%rp>fneWtABL%zS&hZM`hpuy(fWT zamRNj$bys?y~mrz-uWLP=RLlQ(PgVRA^r`fW-SWpz9od=ovr*wE*?DNG!jqhV2yn7 zLB4@Ymr9=$@BaoJkNqZF5gr(C(Tjz4#PiIgz`*V5(Z?f!LjB7M~QD;uCsx(wZG z{*Sl-5>i1`#g!Ho)6Yh?={E#E2p#Hw@d*bGu?zWw_WF|kQ)H7wBUm3RH)ZgI|FCuj z+G~-ut{o(-7#Ylh5F)nmmD0y_%)Hfl;d|vHL$B!nw4;}~N>m>1IAl`uxc(wReV$#T z7Br`fBoWrB4n5-23!O5_JF#~?t|>Ds4Xc(QA$_GlH%T$8h->0Pvp+1p#wC3YIKds{ zRA(4@x?l+(Z^5Tb#6g|$vHxfbQ}c!dsTo0IJ4IT8F}&bc{IHSB3L{UTxYY4U6dmGC zmU$ofCH=DKzA*nQ`Rb$jd}T@RoB!*^LrhEMa%pYRpAcN_e*AnRpZMg$bA95rqPFsH zIhr{J)VM#9{-O;PufG&S(C+M4q7F{}nLD-7`S|zW+OH_UFA)B{U1PuK80=!VUc>sr zDDY714Kmk`A^J}X(gz&-1E)W;#-2;;h??8n^G%dK<5yFZ6a~lv{|UqIJO11C`uIx2 z_9h5#${|NUZP4psI`-mAos9B7?Zk<=R6S10(w2XypdWmh40-n#@xgH?i}vhBvGn%w ze=-nGsSQsbId;;n;A{j6Kb-dktpuVKN|*i9Az4cJObEf{gYy zwM}%1NJ?-^ln_Vb-~QCWtN;6&ePeLB+3c)p{Jxpx_k)%9h)VyH1dZ_|^kt3p#~!Xi zVS~re|E*A3zU`<2sU?XB+s0}3o7VYN4)T9`3%2pDWtZa*gn+coH+jfC9{#hC|DXw*3LFEFD28Er7# zMQ>!0z^+1+{8>6O@%r?r0$lY!M7IaKYf=``}Zx16<^PA zSI-h#orq`Puw=eCP_Eoy~dX%4$37GUaPWs7Y>*|S!suPnoae+G)}%u^{=-*N`$}9 z??}S9mpE@F5B#~wXFI;}VtW1Hjefm%^4xPzA)i;<6}^IhzPO$FO(OOIPtbquxt{Un zK-Sl$w9Vz%=q$w7cPcEbEbznH!;0uzw{eF7zJWl?{oYl?=tZ&C`z^!^wqO0p*TDk2 zy@!3B*-zg?xgG{z$O<2lp!ZkoKW1Xu{rLk!PV>F5)5;`{08w{!o}a!I8+@NsVkA#! zWyrH9zC-LcvsiJ^OPWGPTnTeHx}xQ161&YjF8mwc@jv-lp$2)CcY=kx3$MP$W%3ZPzx0nFSFan)c}IYccF!w zC{UD;U~>3(I+FOZV^m0zq7wQUg&a#=L5%56FZrR6YvWM2pks0>mM%MCrU0Q2Yg|G{ zhPgvbHnwemGL^L>8c7tG+xXmmRUrf~Y>vP9i9(#T4~R*alnx;1^Dj6UYU?Kq%Y_Ze z9(VWl0U-7pCIycm zyD%*rfHD9?>i#Y!0MCzY4ajIF>GV9C(I_gaV3lg}1#!N`M7_Ji;-dQIv>h(Un4ahW zT))F}@_WhV{j23M_q-O&d6hJffs_@U-2i9*Q#uKsvp^_i^R3Bq0fdNL8(D`76^2ZH zqQA{2`5k$9=H}?Is42$#(2yF-+6zErT=E&O0Din;U0;ZkY8F8OX(zHyId_Yopb;td z49_Y?PTrS*Dylj=WdM@fpq;?B*$X{LU-hCqpE5g;V{-T{`DhcY4UtQAa>uX?HB&dx z>SRn0qxeCk%TFwe#%L;|H8O9B{$+sNL}V0=5E|BXbJ(qL{gNA{u-L0frf8GSehXIY z3TM%0S#bcBQ^kS&wqGs^1rgJHF~+>9u&-&p@c6z=dY{hQuzfvtF-JLr<-#$%6;Ab3 z$KbRQ7&OEQG{D0WU>BV^Krp)9O6@zCnmdJx8xh!n z+07n{6l59(qMl(r{30xmgI0uyc5TZu4v;!f23tZek_Hz}x~`&HZSKvgSUHhJ)VC+9 zNB3_ww4zxqv8C;rZ+o>X#zM?gMBI8?=+NX)sH{VARWzdKzBx-UgT$y8V`C{kT^_j~uC)@^KV(LgY)Dt9KvJwImKf62W=0DcgIzbZZ!{m{ISbVsA-_%Xl``wbF&S%P#%K3LJYy(@Q6 zeA%-)Vjx;@dzVv$og&>RJA=X&mZntLWuL4RtM)Ocm%Ar1;};QT#cVGyqAbfQ+w>5n z+;&yjxWY6NQkFoH{;5+2h6s*m0U6YT4q?XQG#peMpD(qe4;K=?xbbQc&G&pFAsi-G+NycQ82P@_Z#2dQibP9g}HBq$p$6q%uvlHeZ%U&#JyZ z@ZA?GjrUL$r^cD4ApK}cMblxgB-~x|SJ#elT>S$XJX9$$FAj>KVw}ysEbz}+3FL6b-^eeLAduO0( z#^p)tgahCPKEqpXcE?;laz=?60yR6;k1hFss~jzRp36SGoRa<6GNm+sn1B3xyorc; zxM*9NWC~W<`({EZlmFhV{!~VKu{X(p2m(`yM-^ra_(BaKPCe8eOH7>sB1-=pM_1}m+ps6r0 zGn)1$Y{hi%le8X1me%6UtUtEkmNUnwp++oxMYVzY9f=0dOT?q7DN4cpHuEbXUu^&A z;pIlp@ABBHVPj_WVLEmtS8!wFuieBM{K~mvniik^B+O6vyMO$OWTr3^u$JACWpiE3sye2SCTn=&mo;-yrIx&Fli`ZKS zw=@S`Zng1Oe8Ktj&F*NrWlu1W;M;i3`$0P+l?p)6N)^(U2y9Gaz820AH5|RFy_&DM ziUAg)1&G!Z!UYXv${ofz=ZTT=bjf^V>&lvq%@CX)t`XIFbE(XK^Q2+Z((>EBvF~9* z_QOv)n7<8LQp{56*RlwzfeGSl4F-1-onBndE7hI_z(Hc_;%88C5_0$jf(N;4ur8Z! zRfK`=UXxNKMx_syRSVVmm`#v2lH|M84id{=y|KV{=k!|vzJ(x>rY)}4@_U|Hs0TxP z&&xbkd@=vzVkRW+^R7znBzd@lLD1SfFWvj-HTUCA1c-&nA;N3myWKP<71?KXFi&dAs7Gj>r_9Rh~ zrTQveQsyV$G%R_YU#bl(I72Tt!*qO7th@K>mR-WoZ2`agF4)mTxB&dBhd9E7T7RnY z%kXzS5kG=GgACc)&&rsHZcv+)k7QqwmNJz@mMY^ay;d z*+<~q18^QVcf_qF_nORVkS*TQ8A{h=k}RFdwOr8RgPZB_iC|pP@;UEajzbyoX4-%W zDbL!2(ZDDw`Vxh#GD~ zYQ_W|p-Nw3dl7hT=idHU=|0?)@eH8zvSWyxIW7byQyC~Axlp=#PydR;qbj|nyAr64 zV)xiF6ByimUQ_e-&kg6%hnG6Eqt)196dPAzCG(<0Pxwn37|NMA+uqM!B|QBT_S;K# zo<>%WAt}x&pEIrBN;}?qkRaG}pY{F2Dl@I;55@y5w0t4EEME-G{3^610uYbSbe`64 zBSG~CVg|RK+L>}<+rOuc@XC(LLC0edVP8@`N9Yu1LiM1{eYs`ZyKIUHjHa1mJ%;?u za+oH*U91bqAi<$`l6n4dGnPN+DpKQ8vPnF?CW4Vc->s;2n9InP`0xkKv#VTA8M-hm z+!@Z*aC`YF-Q?4uUQ$xdry0TSUGH_?6;S#*WwIv3VH*D(e=C_&)YuF#hNJ}7s#cD9 z31N-5<@<4C=5$lVmK?wl?!FN@KjlgN(Jl7AfpmsH5MHBqVTds$K1ZY5B=XhIEvWlG)8aSzWtp;s3 zkjXjckAqozPSFqz^dB^06M;%OI2jK+yMBKWIM|F4m-0!10pm3WcGWk_ZR zx~;Vleu}q~l#p@o1@2S$YrVnJd3@4@D+uB zNv!lO$|_ENC{68sZ0q+{In-(5sl;~@3u8UTgoIhqSfL%0u46l0$=q zIyCE-#DoGSEz_pC)UW=>(4K$H3MCIOtZv15y3DuL`H6s#J*Az$yaAH?9;7o}2mJ znDlq_!V{DEgyFP{;lNG?V=OQ`c2*OQ-%0f@eljNRA>>R{y4V8iXD>KkENNJ+_y$T( zes0j%`fZ;Pj>k~PS(|>RFRv@<+W6M;XGr(UFL@PHpwU250Ro8q>-*usMn+{q9>XXI z>wM~jrk;j87<)`MO9t+Y@fMDaSo1< zn7m7Dwjs&DAD;xM<Ukb4D+_L@m zmw$c0`=UF|UK)oFp8|#ndO;{NSsG;OjZ`@q#}usKDkxDE@gV^Z*+##r4%cZQHoyL4 z=#*+zfnY@54b(H*;01zp7abtH4w1al60zoH&1siH!)g#HVQW(DglHdlMI+=xiXI5g zF7DJLAZ$nuPRc<`43l&`CLiQIcEITsfY$k4j%Wxg0)*h&raSX7NG130A zjq{N8HKI?oDQUj>HZCzA!x?NGNlGdvY{i?>K{1c>Kld6rU=H7@N zm%*ml;_@S+>IR{Lmcd`dQ~AoECGni;3SP@V!5zTG#2eENFPQ0J0Czi>xQWutNB{Xp zWVacO2wMc!j6wq5dx))6#+{5-{AzoT$HPMtr`pN99)9D3l7zFaA0Zmz-D&bBh06FT zq6>KwwnMmbtHNrOIvTVSHZ)9yZ>ky-YkDygFIM7?Dd{Ez7+8rZ|o zsmR$?aQ%y`r(-K>EK;=`(&1Bwyffo=OSq1|4?kVRi?St>Un3Y>J!I;T&^h@IRB@az3{F}GKG+z>($z>RK8&V% zSj4j^?i0rt4M~&G2X1$a`_V=Z!!aC<2zLw~N-GBkU6Dsrug|j$v!~#FSxvoAo3{#V zIAJ~l>et2Hio4JF{erv~`z}Ux;uFOT3_Q=y6e?P{%&N*(ez@C-ZzlsBb9=1nq@5Yl zIhfl0dPf&OxX&jlCa)1ZJ5<2j?#6~8nqmt-?wq}a4@=riO66wGt{Q3RdFH19*}@Nt zK%C|?W69wmUmz|mL-87s?7k8&C)<31_sWp(wd|skTiJhfyVzF1)j#nD8>K3}7d9iz zpN5XdLp>;9%@-#4@8xS(Z8bF0NK%~P#8a-#RxzaHf=r#}SD**u^4cZ-gZWIhQgS>r zv^Fr)11#8+9Ah4GTDi8d9!cL-@=MgJU Yvyx9Zl9yZ6P*V{!Tpauu`1b++LgmxO zWa0D5ZvQhPmP(UkcM1*P+vs3{X*H~)q$MgOKzUnJ?y)j!gwVAaMA-1QHV00koIsZ; zj#Zwe1`8tF&X1(V0X>i4Q6q#R^iZ&;GziwHUC;i_{-BW5d#LgRs%)&t_pWEbDCp7s zez)l}GU2ss1482(6x`0T*^1H9(o9_@6L@}x2ut7S+m>^bpoYb<5RK9dfc7)Eg)9>h z@C;M!Jd9LOy`?yrBrp`eK2+ttK&d|HShRoLyZ6(2cvoBa_DIv=BA^jCnU{4ijzQak zsvA9aVj?-Z6gS!}W{gy(QvPS<0qURvoKrP$xaSu7Q$jmt2t@vZG;OvY>;#|IVP%ar z>qje&MTJ!yjTDQ4*d9&AuW=d}m%%l2N1!!w$@Mr8Tnl>N-a+v3rmjgzd5r79Rd;7M zK5*X=eN7eP%-ipoY?DPimA*@hnQn5k=BPd8)1DO0AObF~U6 zxyu%v_LM`5K#=0%b5W6(PGExcpN9L3(?iY>AXhg6-FteQu3^)7!&NtfhnBu{R|wjR zfgG2=u2TkBb*Kj2&lbtYIKhc<8$ZtbvGQ1TgRdun#>L=GL+6O{Yr1 zLM`;rdv3f{sTFga!XraW6`f}cd|yr08Jp9c%v88t6qEM7RJ+L%!94(bw|P}bfEt|B zyjMQy^~EIg0$Kz}vHw?4+3|U1a7(R1%dMY#`WBbp>b+I9Jafp9s=wqNs?~ zNEw7@=agzfLW)E!0obk75sbMXwZhFN|{0bYZFJTTu&Ywj&HjeR>l{5un+W6)spwaZS z=V-DVti`wW7;FXLHg~*{vPLaI{zg@rG*u`^?PD+Kh28w@Z;i%N6UYh`QL@dZv~c z%QxR{sqt%GCt_5P^oY$YSe&F%wz*0Aln7>0$_}U{mRf>sIl?i9B`rejTYn!u+Y&#T zq}ViaukUjsNvCRojY4XCB5Qw$?9f-zjSl5VEjDfTiar$ZOkX|^de40lkktnD&IocJ zo=b687{I5~<;b%rR4a;W&wiFY{G1A^)Qh8I>b!-JCXn}H)=uwE_y}CU(OOKo`eX9z z=+;#sZ&}PSRLsd##hM+c+q8;Xoncg1bq;7NpVnDAh6AL$)x?qLi=uW5sY-6x{YxB( z9-Q26X1ia5a@Fl0bJ){y;~>+U-?zREwG&&6RX=4N2Zto#xrp)n!*UZFZuTS==~=!E z6+j}iY;;JU8d8-_9x#PG)UzGlS`DjErf0Iu>u-7L6VY`q9L1djv>(GRBVW6`hMICW z@9@fT6TAW^B<{9H6}eJqFoy29T6|W6^sf6V2Ejm(XFxjHx_)qSM_O>G+a>i)^GJ85 zqz)R%&OP3^D5|SwhwY{R8L^7p^yP;g<*OLnlI_htJXZf?BPSi=er|Vnk$g7Oha{`a zPw1S-FJtJamz{HN9ok-&HkfcXcO~s9VLJH3OZws*Xl=->$hCv0IyD01h0?*XIyek?dH5X#uKs3kP8#hHkw#KAZ60NX)s=dx zvvLx^ad=z}s2f3^hARB#TcSSuVGARQ*Y1k9VnQz2FI#YX$afyauaN}POzDFtG$MT0 zeUQUey8u>rv6r)xk<7NkRDgV@MgG(J!}Dil`+l#<&p=I^XH?Jd$rJ$I$#C56?k8hS zU%?+O-PbNyO>lex@cAnj#tn$gHdN(0G9u~(*3kiO=n?fT^;j%K`eDDY4@L$~e67w# zRK{JQmDbc+O>D9*dW2>sozh!)_-4G@$Tjy(DhB7i3afOuZO)6n;hzvS;Z@f&gf2{G zY7DKZqJQ4N>|$z?^x$ACZH8$orz*^`#~x8fYR5gNi20S%y}1kK@oUjwjaSE42{`xw zy&sd#SNCcLZD5cAB4#nj+BUwGqJ^FgTx+Fs6P>K?7^j5`snbnMUS#8H#gTo=)f*XZ zvp2-%TF#+ab%N(#;r7S| zTR8xv@=oRN95(UyH-s6y$VzXso1Cc6j{tWb{!n~U?bd(G7-y5}K zGY=Gn6hrE1>Km#*xv-oxe_#QCr}(p^S>VD;co z$xvK(GCga*4&0=nt$vvG9yD87JDizA6F?v}SA8_~oj|l?F6*7Qc(wQYUA8T{w>XzQ z)PNKgX&i+XS=|7MdzyZ+CJM`Ct4@S@X31{xySlb_(`FcZ*<|Fs{Hda#y2-9j_)PRI zF?NzQdW zAjmtYu7;W2VG!_PYP@46DN5XYZcwE^%MUwPVF$Da6{B%BjD|?G=5vBH|9tt$1WGhD zR=g?^s-4W!2+*rNgle!nBGuB5pfQ0!@I+OTLw83hDV=~qA*Q&vH4K$HZ{6w6<{Q9+=y%a>cgV2X6IBoc-zIHl{W zFH^dq^!l-o5b4wmCxOY;+dBj$&K|Z+lqsNRu|KpWCi&enn|VBGKg!~6qf-$?&GcUp z0JeZ9*eMw5pfM5uu6kN0wob90IiP*>4p}no7f7E-9*i})p-cFbK{N0jmCFD9I7h>+ zAmQ|ql80_{J$C#+f5{HrTTYt6yYMH!Wiw>gfk9L`h`2$307N*jLI6>h&U>qOWU7ka ze-hlZ_WMD%yy6W~Q$>4)JMXs;{UxDR%jYcX7{?hYPPflY@pcGi$)tIIWy1XI1KZr0 zpzauUyQ;k2u$RG z2)Ou-IRLEwq5XGlIX?w>T+_ZP1XsJOdspBM$y$8HGGqK?a zwS)%y9_8~o-IIW}o-I>HVHJfY8e}ClzkrgLkil79DRdijlAX-Co`N?273msc8(wi! z<03OPWS-awzZDSv_?e-PTFrGB{6jNvUV*NnorBI0Pgw#%lRZ1G5^)e?F}BN#B7a^D z%C5SHUCp-=zB437o6kI~wQogs>=HG=3iV^d`=IGOeZ}I;J1RFpz$@oHR}^L7j7_uV zxbc4LoVo_CBKez!@p#!`Gd~97I%)Xw=gB$QLJK1>Nk!fXbMakv&2SCi__IGxCPGJo znT-q$lJrh5LCTu^uh{CJ@=tT}pEyQirF;lDyi{yI0|TPxka;yFZc42LAonoIwJ({~ z199Gd?=6N_&rdNFHtxl?I;-xlgh!bvy`}WP;r1lz29NvdQ!n=l!EMPXWomH-Ec_Hj zBnFCa!5YeIM_;4e`+Vb6yplB~t!@8IO*g49z-?3+Fg<-sf@6dcL)N@9sLxfV*&%%n z58pKAf%7pyt>67oZLHqb9vC(9pEf)>n3OA1C9?8JG)ETlcc-7v(>Vd2zV-wD&TyDu z>(8@x_G@Z@C;f^VQdRx;7_x*6XuVIo->mo;9;ES40jcnGf|%Q@f_BN|PCiEuJ@in0 zZqwJYL~?k=qu%|hD zJZ#>9$}|ekFm`Z3MfgZoc3ps3If}dnZT@dB&B&YyI!>%AeValI5iz`YCNi#*GCJ8d z&xE?FB-4`RZK=r{xp9$EdlJ zTLzGtdhH!S8kI9GSjW zzoWaqhiKV;u;lirPJI?dG2a^2hDb!cn6hi;bV|TR2JQUmnh7+TK zgy9|Mse-LTXEBKkh*PFEdJO&cxE+=%0RhU9-~Wxr^_y8HK`Q&#_i= z3xM$t2YyLZ9vz%%>iHCak7>UMbw1(ksnE z!_2zrqJC&P+2U1k*1WNVHHzweFK17AD3wG;5kFL5dc!~4Dtl0-n-y0p(3C| zKsMg)-Q(zX&ePosD=40c8_&l3GctqqbcV=h6ZjyV@;}fTcw5=ttO?QiS9``psKIx4 zwS&p0g=G9X$xt46LgF8@!20oX^0EZkmNGfpqs#4TWt{6}zcBk1QA`Dt)Ij8=-#9XC zA{f@V_myR{e%n;R96xi3j#}M*%v>#e*=9&NJZ-1%&4adgeUCG_RIxemj3O`4NfziG zNCM?Dfs$EvC|<+SInl)N$?cmt4A%ubmgk~%RIDT5z&_O|en|n-SI>26^y!EMK~GG> z^S{El#ouls!;s&hiFhWwMMxcO%?_664C-)buZUI6xi=xLZ1G>}JTghaT`f&}AHil_ zG6q(mMAXqFL2T}l8-#x+>6sGfOL~S&&PuBrh{ERFn2()l07gF#wzst#Wm+7S+TG*D z*v*NH;G?a_Bw24a%C?IKDRk5$1~Xv92^<$<+8QOKv#lS(&dcv?yj@3ivGH_2 zlw`HNJ9taJ@+Ls{Iv^aP&7ikNg0+3d>t>h=-&QEAW@8Z^tLq)-B<99e#3RWa}0O6o)pST{+ize+(h2#FYQ$Pyv&nX|$Xq z)(^W3jQA}F%Kkypy_}#y68X^N(k>_>KAZlwetRfE{ZNpVYNg2`;BN}IVJw0c03{`1K!8{`$~^VWM+vdfe@ebaE%~j=u&0rjE^SG zL4y6`wSTrFRtY(kk8-(VRTNA${ z&4W3jgo=nmP`!g7qW0CGkjpL$)V!GN#L8GSW91cm28dC66X3s=&|W-+N|4g4f94#- z*O<5-UC?`yKN0FgR9_cCqx)=2{{Ifoo(TUE2{pDRS;v zxG4Wsh2gc4+tW|WOBdgh3>YzcN*}&WxXka4S;Zcu0)=-jv^m{3YzEE6hZi9kvD8Kp zqcNmZEirmxu`kDG%&_vH$w{sx>F~IM-{eq4f)T9?R~t1Azf5u7x>|2l(V)YUuo0*t zFSD!J-^jMpWT?k4RUbMV>I49#<ZiW=QR4M7a_0AVh&Zyy7bNl^d6?t33{Rw{;Dlfm z!JGu1*Qz1IAE_cuDG`E>wZ!riZj4-{>J>_tq=5doktYw~n*4I)s=uQP&25d6Sn|}- z9qhKOW^%>(A@^9f@0k3pf5NYR&W40nb9QsHM_J^_3$n=zV>e5Ac=sOp8>WDFBU%iiN4zb}}#}`WEKYI#4sKoiyoZF;D z%YyhGMS6i@7^y#SUWO$5hL+~;NvQ}+IBw`mXz9?0H&ve-bS)iGZfm9yQJod3<4rLH z-aI*0;`cPo2^|*y@Wzsbx3k}uelV!*p4$;ESQKfJV(88?mP$~%+QNNpgM1Jo{X+xw zVBg&oXmgt$F4Hh}7*LGkMDfYdVd{tDt2m*a=RbKATvsj@)=*)gS5Qf8pv~i$p*M4F z*31*f?y;w&A?~fsM!(FnHR{=fW7HuhO`rHyabe45%7SjTl^eX3*X##y@$^v5^MVsP1Vfl?`2ozbgWD%qcsH$7;ep{Fd? zn{6dIYT{`wlnyL*`T-GHydQe1RUOAl{ePRH)%@p}<1@QwC+HtE-#Xa5DZQ6I{&JkZ z?CXiR;beSVo4%M0(6*kIh_`<5(+0YO4^5qSq*@0U&`Fu_q(W5W!t7qa*f{?8zdwOadjVE!A5c z*)7(YH8Ru2hV{LK#QMlmS`YAJ|M+>62l&e`m!p$l_v6eSDzkG{)AcrJ&lgh=n9Z|! zNVUsckr|Di@%cNtogq|CQj|G|M1!0&m`=l#*aVII7~$vaisM3(m=NyWnWT#x=EnyW z5mM6NPAVeCVX9QH3n5wkX?s`ZE(WV5{FiM6d&b`nLAjpm2KLDg+RIh*0=XtaYF#IkF1OWt5t!oVw_6edFH4=dcuY)0+Vp2X zg>t62vTJ^%(f^UYC_JUx_lgSq>H23%ed>9?e>vH&?i80BK-aeZc{%{`xAJw=6pd~O zWWZ$nviEt@`uaZt`>Pt*5Ni&vY}VR_)?69T5<`J04QF zlGtBg?>j(d^*ybUpd-rb0 zgPO0@jNthLU>+voL8@XSi<5HRD}XU3rOfSrzM-DX+#aGCpuCEz+}9Z>F$(7U@%6yq zFWDt&sh-YwRFYS5f2A=mCI)$Od3sJ>TohzeOa}2@*8)s~Av(@e{xjq?E`ulAfN1nu z`uV6o9%qDL!E*1Tn9)zr)?LY#=#8*;p0L8`7FgbNAR(X^gO896t3hR`KxH43fHR`} z^po}uaTz!XD~9ScuDKLLFpZ1(YtA1`C0c%?v#+CxP?ivmNnJ$|M}t{~W;WD5usGxb zOw#%(*LJC~q;sMigYvx<10-;r&|F0`V%AwwF}lC#TeOrn4y?u44aj!je9M+q1t_T5 z0?te6gxAl^Sw%gPXOtgVuSE8V;k~mvVZtbJ--njWlIekY#@Bk&CBGvLoXd%~%qM+< zF-#0mcK%8Xf`@2D0@Sn4~i>Z zKKRrt_Y_}}by3hm@Z_{Lj{Gb>&}Aw$jrN-ZFW;gQngflFez8ykQW5wkpR&rha(C)* z^Cg3uGh5Q!BGT+@k`Qd)CbL{dtQNc`&>XOY4=So+R6j(qBFe}#>}PrMGC(i_;R-)5 z3b=G8Fnmp6l!cw|Z1MX0h@?`L?q^e`Bq3>0H>~YOE1Mb`0^p9O7zg#`kls473boFh zYb);z6~WWz*SugDaAMhB@~PjTY!=3%Kq8nal$hJ^_kkUSU5FJS`|`PX-RK3STcN1` zkFyHp@I3+L+^=vw-Jn8ht7bi{icemb4?ayFU6hss0Nw+anqk`5jsi;_ML&pVL`|9b zsRKlqv1CV%kvQc`rvgT>-!yTzudwudgWRBzp1*NZH@O=`Bn$()4;Q_!E7=TTP_H6qO*s>lDGF9xFexab$Pex!=A-ru zRm;C@p)oK=0P*_P>9m=JLt|-=48R!kbr?%;k=(T`v1-F6Q@+)22=BSs z$9=_v)eSB!4r09z9XF_4bhaBs4m-q(2=5n-%*YMY!HHS`@;u=+z!7;^b?8znMGtq1 z?E%BsAQhMYPsn`zr}`S~SuGq#K2GXKRXpauJQl>hcZZPSdln1Y4q*|fho(UoS#fmu z8|<XCjr+d<-O7g6~0q(bBVbx7T+U9!BOgTU21i)E5PpYcUD~ z&26UXXKAL)&T*g_zPSA1-|`qO;WGhIJLdx9@e zC7edygp-tatQg^sXoMROp*85VjAJ-sDf*q6GO~@qe?}9LkMfwj_%Cumt5~4*2Aia1 zJSdm%rt`fz6pZxb-l(Bk+!L+RFxX@mmlJQGGb@0Ac1d^bCFvFon?BeFPw5s2&39KJ z!6pDLT&>i)dHtmfXJoH%)yB!)cB&5cD0U>49$EO5QlC&p2#FVK17<< z=3R_OJ#MDgw-VRip8!oAFYe)V!@aAe7+V6zwweZFy&Ps`%ZZTjpF*t4)uHAGUbdIsehzpODe}l>X~UKR#4ZNXkd=j9-zo2h~;>5`arn(-F*B=n&16 z1y!w@;^grB;wb#v(ZA^9!~`symg$rEqn|80qXOTWITcb70>*itvE(}?Ejxa64V|Vb zqmh+99jqCj9rQvWk?D;YLmG{275ZvvgGP_W8*qh{KH|a-H1oh*JI;y<4qMCP?F5HZ zLJ;nPLsh$CpA+i_MqWLsW|7UUe~WFCFsSb1Uh~_tIo)A2ETocSV0h2<>L0qQ=1-FK zQ_6I4=8SM1gbOwM(3I6mdEjN+Fd8h$;>p+tGXtpZ%a%#1x%&BVpCPHRMrYI1ZEg6_ z34=btE`$kUpP#d9asx>oOTwN8u;VnpUc3vOr!L?TD=!4);cjz`+?@SP&O-wYeL#in z1e)W&PQ3^{gdY4Ts=`_n$b=@8?JRK>o}}x6aF}!}Rk|#~^sBMzp(Kg1R;yBc)FW~2 zUd9AZF;eM!6I%9XLU(@NH{h+Trf%~OUCDG2rU4Y6C28K|v9Iu0fET`9o5CTs=xFRz z78VB2ivZ>mu_ji~p$pE8c1y>6(zB6{l+&uU9oGHd!d*GEl(c;jwPl#*xBTI#b**vS+@NTy;i_k-Iunz z;lh`NV~J;=*%4LtU&n2lbUs7kC&d1Qd#@6}37amQ!Kv}fp?uR|Z()ZwLq_to(tG~a z*BB5pJmVNrQYOM8|Il2cCaa8B0L62pL@M5q^(v;k?uY%{qd5R^rnUmlLtu=i*et0P#>S&(H zu)_VUCIS5p4DiTr+zD*j^E(DZgheXOptUN8?^m{02`4$Ho)$FzetU7$%Msvti#)RROW{bC}|5gMAm;wzb+ z1+yb}*~Tc?Lgxw2vH6IQqL;MsCAPz#x!MsFHClR1L?|=~S8Am&-1MdvHTYA?z*FMULl#vpt~sF7jpY6aX4oqCbMy zn&mCh*ya=9+Kr<2C%W+Q;2AW$3~@u!oo5ZBV+(_}Zw4)-=^f--<;3o>W}P=1E9LC}YpP9y`zt=$v|Y zZHchz)5Sz5S-IP%LWRy`8WV|^VOTSCF~n)?uU0~&jx(fx)jaDfb*^o^R*bwEE2m>B zF%X~*5k^q#lwNF#%D@abh{tLs{j#(2EpyGY7cT@m71d5p_wnU?p3!;Ys>I&Um%7;yp` z(fDFlYZM7uKu0xm+4e6Jpg9OP*VV1)VKDJ4WF%tc6YjM@1Ck|vrn$y|l&chCOQ)bz z!MwObA28T1H-ij5=^lZ-8;->bXsRpAbdbfY@1GeB-)D*+5*)9DAzJ957?3zswVX5u z!p%45ikOAlFiWG`3P#BR>*vymN^tUW;^cxmOwUeSKMM3TUdaKf@keDcKb7T(y$4?s zoo_s?=T|il7&t+EgpZURZ9iN3~U8yhWRBlqSZ*N;x=0l|ymCD+d-WGkvrD}hIha@Oa2n32~ zO&0g#_CW!vyv{NK6o82>Ce-8>X3abS*)A*8^tMvL4hTG^sd2#caRD{Niq=ITfp$=gA6XlZU&|^g6&p-st&h?AQ>EkqGyFN z^et?vdD#)g@GSXL?m7PB9@QvAd0wK?jE%U20N;icMF?DUdr6GzG!3y5}@&afu_<%F1&=tWT)o4GRIB z8t{w_n0{;^74{=jaqA!%Msmz0?N)2{NkMfXH3t%b1hakt_EGl^s1cE%KvMBus28|C z*XJcFuASK~SY}~Odea*astT$O1(|4nI=B`pN@Qd6ODLj13~LDDSy%z*=_?&D9TQSV zvN$$yF#DV)GjRnK6a~>Y>Mk_n6RA+ynGozC8%xJX+cNuNo&YHdgwlOdbV1RB6cv~; z%pNhy-8uaTs4>A82=-3EQO)qU(eqsQg>ohPbrA5ylFJwyFnv5$3KCE=BRz*>tg$M#UU^sM}SnD)(ODgtcEPhpTkpZkS5RF3pi%k)C z83?s9+3?zRsc5DJ#+bl%0Cm9hQP6a`%CB+^g`*3Cc~Jmw1e;!HCE2eBGay+mN>Wsg zKt*-CeqbzI*d?^F4v%N%LxGzYdiq@be-<>Sw%w!y>b}Z#N7ViZM zq=TaarVqy>#U;>OKD~{!cTZ<5=RtE<% zj$vRO_sCzydy2=}gHtq?02z(UVdSE5$^*LM5l zF2)It4Vd1->mHo2HI6*M=0KyE6Q$I4-l92*jy?6VbTLNHc z?dR%!apSibnD1Qeoe)xi<)66evZBO$JfJ#Y`dHX1GP~3n0oDVD69ab0PH3U4Gs!6? zOr>+HHbY)}o8Cl@krmb;_nuQ%ePEB5W;5*iko}Q|QlwV2(J<^+oZ^z57X_lhsxPVS zY^o?r&>tNeF#XsB6-L#SGu}<%=qeD=p4Ayis8YL(!`?>NS?diWU@x+*ZlARLypjJM zB(r7&+7s~F_uRfMWfV?cDB_|#>!R9@bwIk%7cB$)k$bmgmQ7H4(co-lmT7?O6PsBX z8!-L&NGe3CrJ}&;XP3mryQ~;1t%f516(@}ts20RyDL|rqq0JWLoU#S1#Hp59a5$tq z)+$lay`ap$5J`ES56Nu2I0UtfVBqJE*u!ZFhb1F9WUvrjLN71{~kEaXu4Z zN1#>?@fMtCTLCNV6GB#s*yN>KTq*{|G8@2Dq3}dBw+t0&siNiqf*+8zfB}heUkq2~ zO0#NLqACJl&9wHFlfS9hE)z5p-{&mI_F<)W4yH3=_PtRyWs8bxQq^3)KS!hQx(YB5 zu$WO&D3TYPCoO!z0@>N%)^*0sw85zIgZpE+ldVN!#zCq~MMDvo79S(3aJxR&>@&r4 zN`q%Zq17Sguc?1!Chqi@4w#M=;E49dnCe~zpfklC1bPnGZyZqJVVrcTwqtnG%bw*! zpoql56M+JiEjT}q=a6c&+7c>$p}?;C4$KHWqKJvT23AtA+9!AGGa(TK16TWE74m3oXkZB)u;%b@RC=*96coy6A{@q(S^i)8-L@$*ONuqC(tgHOG;a4aeIdQu zvcrbK677XhE)XvHjv>=}u8sPX6@gTA&lJlh71mjOx1e8-T4drva+OHNu$`RKNFgE? z9B7o^*u5~*v8$OQkB=P&bOyqHBo%oBmYcoo%w&)57*erJG?QMKsz*V5UuIa%wRpjl zL}sf8VHLXxA|N3$AY;j_q2RTR~{+)>=sDl>TXT2Mj zhGa6uX9}%N79gVP-4o(Spf20ft<)Hd4VZphJS9rksRs4U+sd>kAW&mc&P;zO(4Acm z=@GE*=Y}Fyteig^%FQsPnnAtorl=L2h;gwDie^X|P@10VrcWjY4XG~g!RL4-`+$HE z4j5+xLXwq_4VXRxF|B_dq@V%ofDuz5TL+YLhS2n$mHM}ilwe2uF?5m!v9k9}^pvI_ zdEF`P=E;goxZ9Uwrt4gyJ!w^UHb_a!AQ)--`-Q$|YGfSSqiTS)D)N6~W@&7|^kI+{ z%8>0S-^w_&6G8ZukxmUzvtyuwbifb}I`wVA3g6BO{n8ZJe60kW2)8ktJl6AT@# zIX${8__c5&r63V&6V-$cRjI228bxZ}#Wf#{y%R8vX^pu(OGU*bPB2SW{*Lqm6q|hc zw_7jmmtMZezU6(dOQfzDoz);y=)2!La`LQKRFC1->nRgV?r6$!FYPDmG}~H{Q~3;% z0toxyHWS5=w0?Lt_C zfdK9jQh*J>RDGS&m^8I|+KGTJ1c4D|Et&$fu3uZ0#S{Y{>RylC8cZk6dut%=CbLM9 zXz|iBW3(71g8&(|$uN%L&|JslJ3-(@BjlpbCKwZtg!n9Uz`)K?RmZ`J*@85)!c6p0 z1I*@HTK#D9J~Gcp8UTT;h?_PhjT%#j{ze7hvFn2Ar>gz2tAi-X8XN$hlrAJ&5K8AL zf=%ZbV>b>f)ek08%CXueQ5`eHy{_U~$~Xq3)B~}6H>nUNsN65XN>x#f28`Okry_w1 zmBvR$rbUQAy8WDM0PjufQLE|kplTFFF_plOVY*8A`Jo}os1LOI z?2B|gO9)p7nP>Nx0Y14Hi~w&cXjl-!r`A0CZ?`jH6qy}{@XT40T|vRwLZ$#R(gD_c z0n@Q}0nut6BHP@*`?s$Yu5?bl8Pn#Q6r=|h{zk*PgJg}VvvB}`ZnOvl1$yV_4n~xn za!p z0n?{hhHIOf$-1XPFz-Y}OciP%oz>)TY3G`?AOyM9tE7x8aE-_5XyH&FtGIB@UC z#|>o{F7-?;2XnRGyU+4^oU>qk>*^{-KCk2 z8@*>4gnE)(bxPzvLtMM;GY(jJE8ZYTH&XL6jlciY9F)wgAeCeUW6R~F7$4dX@4kE}T{dTwb1Sf;Q8 zvIELkzwI$Q$DF1p4~(H>`>D}0x7fNWZ9y;~sOmbe^xU$uRR>UqnLZ|xqB)>qO6&~y z9H*z?)(oRfi&)Ew?rv!Nq$`d2)%P#^{p$2@&}*tPds45rO}heRsvxr>l62%kw#Jxe z#)xO2drz3)#n(Wx&FBa~i2~RV*KTaU^n+7U!ghiN3_Gg8m`U~SkITrmg>Vf@n$bHZ z6u3ABw#2A=XMHc56Yd*=+uJxXR@D4i|35KP2D zfIG|59Z3=PK|o~puxd97c%i5Fj%~?+NacQ4pw{he50H!fR*shA5t*l=10z&1Nz?FTHn0i&cNe-HB&ttuFr6ctq z7d$Y6 z0HdV+qQLn`SUjU6TYR4tZR73j^A2D;Mrk0JGhq+EQ82^1IU^;eXR%$}}sYcLik%(+_DqAI+@40UYVxEOy0}qBxT3&4D6(AdV4{Txo7GwmfacGo~ zLhw_S-0=NyGAJB-2QZCE^WGFsQ%BmQRngQAe8Swpptcu=O`-G`3M!a%7Cp~ZVWQ5p zbx7+RWOF@Jp}Kd%*&NscU?R9jqMJZNKMTr`^olJLvqDXn6&ge!3WXK`%T<-QoXqkB zkag zs*JuQFr7PeD=dvwApxc;P}%cp*ukNu_o5lU-BT>SQ#8dkd>R1H+iYTrQ1o46&qS0p ztx}rKwaRo#RISV*C~l*kspP!O^MZn@mGz>4ay47S;#nA^quqNS@#(&!fZ^mAF(mk{0-Y{v^ zRzFwh$)wVv&F76}k8Dt=f?9Cdhm|T?)Sklx$V<$&=|^@S?7U+z9lIrnmJZN}7ag$* zR)I!14`P{=QWNW7jdV82?_CHgO4hIy3Pc0N&G>y)kcvMOo`{!y(*1e7V8876IyNh! z_M}#88)5+#!M3G}SC-i=fFuQ^Cj6cWJe2Wsy_mxQN0bwUs7w zFb=>r8irgtC2O3(<_-y60LL5*SXPR#e+wV7o~ zNlb=D0^_83Zu(8{223Z$xi*4Qb3Q>@OnG5K1f=d_wyZCe(+j;CwgguIwlWjz&Pkjn zx~BU$WeJ`7kUCqH*=s}<(Pfbq=nti27g^S}st*kI)%Txq!=fQl_Ca`#PyRc^dWQ=Y z5RC^PFnuU08f-)yM4L<(F;^2ulw+{zDc}#HO9@wk;34*tk__7HbPsMnpEUivZL;6Y zFm5z~$8c6OTW3nGijaj2VFX(HdFFXRDQgD2gPW1N(l*Anqfo(b{d~#Cpq0rkgTrH& z1=F!x0;{SEnnlO$K55|?_Ah#w9wiH*N(Zz*=MHF;wvwejSkm#QMbJztn$a?5SV&W` zjMC_6csyikCW>NI>kjtPJN;-%xeG8@BSjUKTULZQrvWk@Y%j8;zBHA*_Xeh4%`)5v zc`Y5r)_MBE`x69P@?Fj-fIt8}x~r3S1CE0k^sUJJRaCG&Mzep6f*R_^3vEuz#brz; z1O!S@sGCvh3yRtXczEk1{A; z%|-~BqYh);hne0bm_FNAQ-X90JdK30c?|;l7e=a9OkR8nBr2y*FcO7fFzo{=J7$YU zYR?qNku7N^6YIDSETTGwud#F#ocRKqmH@dpu_P8BQhTRXJ>DEFl!CV6Gf`<)gNaRT zg}H+@4IA->dt;_!S0khrA&y-OvOr=r+83kaV6}e*h5<=;006dy-cGb{EQ+9)EPnxP z_uq?=pghX#nL@5mo|hndl)L$)mJAE;K#RQR%aV>v;3;H|V35(wI!;xQ#>L(jn2y~Z zV7+?}9k8;By`VnDEIQu&%&jj1VWx>VnvE&$8KZ!`a&(2GkB;_8g-umZKmgVufd^Yl zGs(WUlHekD&-d|?Zi+h&kx(E5}_^vMrSU(@e{qmX5WRDrr9YS#sL*s;EmUbGB z&=ypF1`TE>ncW%K?AfK`NiZ%B>1J4d+G5?@#}MvaC;TC0eL~C%fWTug^?tzA-TpYj zzCTNGMf>(|n*E(`CxE&Tg;NDVD5V4#6ci~3*9sjDQ)EH_4S5bf;iN$_TDFoEfF=^c zp!W9y2->?X%hjp6ycG+lSzPt}2T+-EQmn%J%z%^ import { Body } from "@budibase/bbui" import CreationPage from "components/common/CreationPage.svelte" - import blankImage from "./blank.png" - import tableImage from "./table.png" - import gridImage from "./grid.png" + import blankImage from "./images/blank.png" + import tableImage from "./images/table.png" + import gridImage from "./images/grid.png" + import formImage from "./images/form.png" //optimized example import CreateScreenModal from "./CreateScreenModal.svelte" import { store } from "builderStore" @@ -54,6 +55,16 @@ View and manipulate rows on a grid
+ +
createScreenModal.show("form")}> +
+ +
+
+ Form + Capture data from your users +
+
diff --git a/packages/types/src/api/web/auth.ts b/packages/types/src/api/web/auth.ts index 46b1e8cec9..5ff0c3c1f5 100644 --- a/packages/types/src/api/web/auth.ts +++ b/packages/types/src/api/web/auth.ts @@ -18,6 +18,7 @@ export interface UpdateSelfRequest { password?: string forceResetPassword?: boolean onboardedAt?: string + tours?: Record } export interface UpdateSelfResponse { diff --git a/packages/types/src/documents/global/user.ts b/packages/types/src/documents/global/user.ts index 337855787f..ddb1e39c64 100644 --- a/packages/types/src/documents/global/user.ts +++ b/packages/types/src/documents/global/user.ts @@ -55,6 +55,7 @@ export interface User extends Document { dayPassRecordedAt?: string userGroups?: string[] onboardedAt?: string + tours?: Record scimInfo?: { isSync: true } & Record ssoId?: string } diff --git a/packages/worker/src/api/routes/validation/users.ts b/packages/worker/src/api/routes/validation/users.ts index dfc1e6fbbf..7b95de0f59 100644 --- a/packages/worker/src/api/routes/validation/users.ts +++ b/packages/worker/src/api/routes/validation/users.ts @@ -26,6 +26,7 @@ export const buildSelfSaveValidation = () => { firstName: OPTIONAL_STRING, lastName: OPTIONAL_STRING, onboardedAt: Joi.string().optional(), + tours: Joi.object().optional(), } return auth.joiValidator.body(Joi.object(schema).required().unknown(false)) } From b0cd3d4d03206c11d97821e2c860a2f22076a4c9 Mon Sep 17 00:00:00 2001 From: Dean Date: Thu, 15 Feb 2024 15:23:13 +0000 Subject: [PATCH 003/114] Refactored tours. Tours will end if a TourWrap is removed from screen. --- .../portal/onboarding/TourPopover.svelte | 19 ++++---- .../portal/onboarding/TourWrap.svelte | 44 +++++++++++++------ .../src/components/portal/onboarding/tours.js | 20 ++++++--- .../builder/app/[application]/_layout.svelte | 2 +- .../NewScreen/CreateScreenModal.svelte | 25 +++++------ .../design/_components/NewScreen/index.svelte | 2 +- .../builder/src/stores/builder/builder.js | 15 +++++-- .../src/stores/builder/tests/builder.test.js | 34 ++++++++++++-- 8 files changed, 109 insertions(+), 52 deletions(-) diff --git a/packages/builder/src/components/portal/onboarding/TourPopover.svelte b/packages/builder/src/components/portal/onboarding/TourPopover.svelte index 1140708593..e319d3bee4 100644 --- a/packages/builder/src/components/portal/onboarding/TourPopover.svelte +++ b/packages/builder/src/components/portal/onboarding/TourPopover.svelte @@ -1,6 +1,6 @@ {#if tourKey} @@ -100,6 +96,7 @@ dismissible={false} offset={15} handlePostionUpdate={tourStep?.positionHandler} + customZindex={3} >
diff --git a/packages/builder/src/components/portal/onboarding/TourWrap.svelte b/packages/builder/src/components/portal/onboarding/TourWrap.svelte index 9be6255f52..779a84f463 100644 --- a/packages/builder/src/components/portal/onboarding/TourWrap.svelte +++ b/packages/builder/src/components/portal/onboarding/TourWrap.svelte @@ -1,44 +1,62 @@ diff --git a/packages/builder/src/components/portal/onboarding/tours.js b/packages/builder/src/components/portal/onboarding/tours.js index 894f9c7894..f5e34518cb 100644 --- a/packages/builder/src/components/portal/onboarding/tours.js +++ b/packages/builder/src/components/portal/onboarding/tours.js @@ -32,14 +32,18 @@ export const TOUR_KEYS = { BUILDER_FORM_VIEW_UPDATE: "builder-form-view-update", } +export const getCurrentStepIdx = (steps, tourStepKey) => { + if (!steps?.length) { + return + } + if (steps?.length && !tourStepKey) { + return 0 + } + return steps.findIndex(step => step.id === tourStepKey) +} + const resetTourState = () => { - builderStore.update(state => ({ - ...state, - tourNodes: undefined, - tourKey: undefined, - tourKeyStep: undefined, - onboarding: false, - })) + builderStore.setTour() } const endUserOnboarding = async ({ skipped = false } = {}) => { @@ -58,6 +62,7 @@ const endUserOnboarding = async ({ skipped = false } = {}) => { // Update the cached user await auth.getSelf() + builderStore.endBuilderOnboarding() resetTourState() } catch (e) { console.error("Onboarding failed", e) @@ -222,6 +227,7 @@ const getTours = () => { }, positionHandler: customPositionHandler, align: "left-outside", + scrollIntoView: true, }, ], onSkip: async () => { diff --git a/packages/builder/src/pages/builder/app/[application]/_layout.svelte b/packages/builder/src/pages/builder/app/[application]/_layout.svelte index 474c17ffb7..f786fad017 100644 --- a/packages/builder/src/pages/builder/app/[application]/_layout.svelte +++ b/packages/builder/src/pages/builder/app/[application]/_layout.svelte @@ -95,7 +95,7 @@ const release_date = new Date("2023-03-01T00:00:00.000Z") const onboarded = new Date($auth.user?.onboardedAt) if (onboarded < release_date) { - builderStore.startTour(TOUR_KEYS.FEATURE_ONBOARDING) + builderStore.setTour(TOUR_KEYS.FEATURE_ONBOARDING) } } } diff --git a/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/CreateScreenModal.svelte b/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/CreateScreenModal.svelte index d263b6b983..c2a7a364e5 100644 --- a/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/CreateScreenModal.svelte +++ b/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/CreateScreenModal.svelte @@ -155,7 +155,7 @@ // Handler for Datasource Screen Creation const completeDatasourceScreenCreation = async () => { - templates = rowListScreen(selectedDatasources) + templates = rowListScreen(selectedDatasources, mode) const screens = templates.map(template => { let screenTemplate = template.create() @@ -192,10 +192,17 @@ } const loadNewScreen = createdScreens => { - const lastScreen = createdScreens.slice(-1) + const lastScreen = createdScreens.slice(-1)[0] // Go to new screen - $goto(`./${lastScreen._id}`) + if (lastScreen?.props?._children.length) { + // Focus on the main component for the streen type + const mainComponent = lastScreen?.props?._children?.[0]._id + $goto(`./${lastScreen._id}/${mainComponent}`) + } else { + $goto(`./${lastScreen._id}`) + } + screenStore.select(lastScreen._id) } @@ -206,8 +213,6 @@ return screenTemplate }) const createdScreens = await createScreens({ screens, screenAccessRole }) - const lastScreen = createdScreens?.slice(-1)?.pop() - const mainComponent = lastScreen?.props?._children?.[0]._id if (formType === "Update" || formType === "Create") { const associatedTour = @@ -217,18 +222,12 @@ const tourRequired = !$auth?.user?.tours?.[associatedTour] if (tourRequired) { - builderStore.update(state => ({ - ...state, - tourStepKey: null, - tourNodes: null, - tourKey: associatedTour, - })) + builderStore.setTour(associatedTour) } } // Go to new screen - $goto(`./${lastScreen._id}/${mainComponent}`) - screenStore.select(lastScreen._id) + loadNewScreen(createdScreens) } // Submit screen config for creation. diff --git a/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/index.svelte b/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/index.svelte index 6c3637a248..ff3b0beee9 100644 --- a/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/index.svelte +++ b/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/index.svelte @@ -4,7 +4,7 @@ import blankImage from "./images/blank.png" import tableImage from "./images/table.png" import gridImage from "./images/grid.png" - import formImage from "./images/form.png" //optimized example + import formImage from "./images/form.png" import CreateScreenModal from "./CreateScreenModal.svelte" import { screenStore } from "stores/builder" diff --git a/packages/builder/src/stores/builder/builder.js b/packages/builder/src/stores/builder/builder.js index 22b663af35..19253d2688 100644 --- a/packages/builder/src/stores/builder/builder.js +++ b/packages/builder/src/stores/builder/builder.js @@ -7,7 +7,7 @@ import { TOUR_KEYS } from "components/portal/onboarding/tours.js" export const INITIAL_BUILDER_STATE = { previousTopNavPath: {}, - highlightedSettingKey: null, + highlightedSetting: null, propertyFocus: null, builderSidePanel: false, onboarding: false, @@ -61,7 +61,7 @@ export class BuilderStore extends BudiStore { highlightSetting(key, type) { this.update(state => ({ ...state, - highlightedSetting: { key, type: type || "info" }, + highlightedSetting: key ? { key, type: type || "info" } : null, })) } @@ -135,9 +135,18 @@ export class BuilderStore extends BudiStore { })) } - startTour(tourKey) { + endBuilderOnboarding() { this.update(state => ({ ...state, + onboarding: false, + })) + } + + setTour(tourKey) { + this.update(state => ({ + ...state, + tourStepKey: null, + tourNodes: null, tourKey: tourKey, })) } diff --git a/packages/builder/src/stores/builder/tests/builder.test.js b/packages/builder/src/stores/builder/tests/builder.test.js index 7aac2489db..e6f52689aa 100644 --- a/packages/builder/src/stores/builder/tests/builder.test.js +++ b/packages/builder/src/stores/builder/tests/builder.test.js @@ -88,14 +88,42 @@ describe("Builder store", () => { ) }) - it("Sync a highlighted setting key to state", ctx => { - expect(ctx.test.store.highlightedSettingKey).toBeNull() + it("Sync a highlighted setting key to state. Default to info type", ctx => { + expect(ctx.test.store.highlightedSetting).toBeNull() ctx.test.builderStore.highlightSetting("testing") expect(ctx.test.store).toStrictEqual({ ...INITIAL_BUILDER_STATE, - highlightedSettingKey: "testing", + highlightedSetting: { + key: "testing", + type: "info", + }, + }) + }) + + it("Sync a highlighted setting key to state. Use provided type", ctx => { + expect(ctx.test.store.highlightedSetting).toBeNull() + + ctx.test.builderStore.highlightSetting("testing", "error") + + expect(ctx.test.store).toStrictEqual({ + ...INITIAL_BUILDER_STATE, + highlightedSetting: { + key: "testing", + type: "error", + }, + }) + }) + + it("Sync a highlighted setting key to state. Unset when no value is passed", ctx => { + expect(ctx.test.store.highlightedSetting).toBeNull() + + ctx.test.builderStore.highlightSetting("testing", "error") + ctx.test.builderStore.highlightSetting() + + expect(ctx.test.store).toStrictEqual({ + ...INITIAL_BUILDER_STATE, }) }) From 9da5467bfe5444e87a20022a9eb6a20d53de1a3f Mon Sep 17 00:00:00 2001 From: Dean Date: Thu, 15 Feb 2024 15:23:52 +0000 Subject: [PATCH 004/114] Remove unnecessary reset function --- .../builder/src/components/portal/onboarding/tours.js | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/packages/builder/src/components/portal/onboarding/tours.js b/packages/builder/src/components/portal/onboarding/tours.js index f5e34518cb..5cf6735970 100644 --- a/packages/builder/src/components/portal/onboarding/tours.js +++ b/packages/builder/src/components/portal/onboarding/tours.js @@ -42,10 +42,6 @@ export const getCurrentStepIdx = (steps, tourStepKey) => { return steps.findIndex(step => step.id === tourStepKey) } -const resetTourState = () => { - builderStore.setTour() -} - const endUserOnboarding = async ({ skipped = false } = {}) => { // Mark the users onboarding as complete // Clear all tour related state @@ -63,7 +59,7 @@ const endUserOnboarding = async ({ skipped = false } = {}) => { await auth.getSelf() builderStore.endBuilderOnboarding() - resetTourState() + builderStore.setTour() } catch (e) { console.error("Onboarding failed", e) return false @@ -87,7 +83,8 @@ const endTour = async ({ key, skipped = false } = {}) => { // Update the cached user await auth.getSelf() - resetTourState() + // Reset tour state + builderStore.setTour() } const tourEvent = (eventKey, skipped) => { From 86c6922bf4f3cedaef16e61569000d40c1beb326 Mon Sep 17 00:00:00 2001 From: Dean Date: Thu, 15 Feb 2024 15:25:07 +0000 Subject: [PATCH 005/114] Added in init flag to ensure that analytic clients only init once --- packages/builder/src/analytics/index.js | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/packages/builder/src/analytics/index.js b/packages/builder/src/analytics/index.js index 6bb10acdb5..3a80a05d7f 100644 --- a/packages/builder/src/analytics/index.js +++ b/packages/builder/src/analytics/index.js @@ -9,13 +9,17 @@ const intercom = new IntercomClient(process.env.INTERCOM_TOKEN) class AnalyticsHub { constructor() { this.clients = [posthog, intercom] + this.initialised = false } async activate() { // Check analytics are enabled const analyticsStatus = await API.getAnalyticsStatus() - if (analyticsStatus.enabled) { - this.clients.forEach(client => client.init()) + if (analyticsStatus.enabled && !this.initialised) { + this.clients.forEach(client => { + client.init() + }) + this.initialised = true } } From ddc51edee77793de33579e7d7446ce8cbdebc08c Mon Sep 17 00:00:00 2001 From: Dean Date: Thu, 15 Feb 2024 15:44:54 +0000 Subject: [PATCH 006/114] Fix to ensure the skip flag is also reset in the tour popover --- .../builder/src/components/portal/onboarding/TourPopover.svelte | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/builder/src/components/portal/onboarding/TourPopover.svelte b/packages/builder/src/components/portal/onboarding/TourPopover.svelte index e319d3bee4..6dd7fa96ee 100644 --- a/packages/builder/src/components/portal/onboarding/TourPopover.svelte +++ b/packages/builder/src/components/portal/onboarding/TourPopover.svelte @@ -26,6 +26,7 @@ tourStep = null popoverAnchor = null popover = null + skipping = false return } if (!tourSteps?.length) { From 7895292705fcb7197e7f846d070931fac761bc13 Mon Sep 17 00:00:00 2001 From: Dean Date: Thu, 15 Feb 2024 16:16:30 +0000 Subject: [PATCH 007/114] Add offset to custom positioning. Reusing existing prop --- packages/bbui/src/Actions/position_dropdown.js | 5 ++++- .../EditComponentPopover/EditComponentPopover.svelte | 1 + .../design/settings/controls/EditComponentPopover/index.js | 4 ++-- .../src/components/portal/onboarding/TourPopover.svelte | 2 +- 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/packages/bbui/src/Actions/position_dropdown.js b/packages/bbui/src/Actions/position_dropdown.js index cc169eac09..d259b9197a 100644 --- a/packages/bbui/src/Actions/position_dropdown.js +++ b/packages/bbui/src/Actions/position_dropdown.js @@ -35,7 +35,10 @@ export default function positionDropdown(element, opts) { } if (typeof customUpdate === "function") { - styles = customUpdate(anchorBounds, elementBounds, styles) + styles = customUpdate(anchorBounds, elementBounds, { + ...styles, + offset: opts.offset, + }) } else { // Determine vertical styles if (align === "right-outside") { diff --git a/packages/builder/src/components/design/settings/controls/EditComponentPopover/EditComponentPopover.svelte b/packages/builder/src/components/design/settings/controls/EditComponentPopover/EditComponentPopover.svelte index 5bbbfa283c..39e4bc2ada 100644 --- a/packages/builder/src/components/design/settings/controls/EditComponentPopover/EditComponentPopover.svelte +++ b/packages/builder/src/components/design/settings/controls/EditComponentPopover/EditComponentPopover.svelte @@ -86,6 +86,7 @@ showPopover={drawers.length === 0} clickOutsideOverride={drawers.length > 0} maxHeight={600} + offset={18} handlePostionUpdate={customPositionHandler} > diff --git a/packages/builder/src/components/design/settings/controls/EditComponentPopover/index.js b/packages/builder/src/components/design/settings/controls/EditComponentPopover/index.js index a32a1cd821..2dc3f60185 100644 --- a/packages/builder/src/components/design/settings/controls/EditComponentPopover/index.js +++ b/packages/builder/src/components/design/settings/controls/EditComponentPopover/index.js @@ -1,8 +1,8 @@ export const customPositionHandler = (anchorBounds, eleBounds, cfg) => { - let { left, top } = cfg + let { left, top, offset } = cfg let percentageOffset = 30 // left-outside - left = anchorBounds.left - eleBounds.width - 18 + left = anchorBounds.left - eleBounds.width - (offset || 5) // shift up from the anchor, if space allows let offsetPos = Math.floor(eleBounds.height / 100) * percentageOffset diff --git a/packages/builder/src/components/portal/onboarding/TourPopover.svelte b/packages/builder/src/components/portal/onboarding/TourPopover.svelte index 6dd7fa96ee..732ee40e11 100644 --- a/packages/builder/src/components/portal/onboarding/TourPopover.svelte +++ b/packages/builder/src/components/portal/onboarding/TourPopover.svelte @@ -95,7 +95,7 @@ anchor={popoverAnchor} maxWidth={300} dismissible={false} - offset={15} + offset={12} handlePostionUpdate={tourStep?.positionHandler} customZindex={3} > From 08d5a6174a1aa0850d1bb75f14107c22c3adf330 Mon Sep 17 00:00:00 2001 From: Dean Date: Thu, 15 Feb 2024 16:24:34 +0000 Subject: [PATCH 008/114] Lint --- .../components/design/settings/controls/PropertyControl.svelte | 1 - .../design/_components/NewScreen/CreateScreenModal.svelte | 2 -- .../design/_components/NewScreen/FormTypeModal.svelte | 3 +-- 3 files changed, 1 insertion(+), 5 deletions(-) diff --git a/packages/builder/src/components/design/settings/controls/PropertyControl.svelte b/packages/builder/src/components/design/settings/controls/PropertyControl.svelte index 3bfb7e9086..4cbf29e3ae 100644 --- a/packages/builder/src/components/design/settings/controls/PropertyControl.svelte +++ b/packages/builder/src/components/design/settings/controls/PropertyControl.svelte @@ -26,7 +26,6 @@ export let disableBindings = false export let wide - $: nullishValue = value == null || value === "" $: allBindings = getAllBindings(bindings, componentBindings, nested) $: safeValue = getSafeValue(value, defaultValue, allBindings) $: replaceBindings = val => readableToRuntimeBinding(allBindings, val) diff --git a/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/CreateScreenModal.svelte b/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/CreateScreenModal.svelte index c2a7a364e5..8c1a11289d 100644 --- a/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/CreateScreenModal.svelte +++ b/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/CreateScreenModal.svelte @@ -48,7 +48,6 @@ } try { - let screenId let createdScreens = [] for (let screen of screens) { @@ -73,7 +72,6 @@ // Create the screen const response = await screenStore.save(screen) - screenId = response._id createdScreens.push(response) // Add link in layout. We only ever actually create 1 screen now, even diff --git a/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/FormTypeModal.svelte b/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/FormTypeModal.svelte index cc0ffaea49..856552dec2 100644 --- a/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/FormTypeModal.svelte +++ b/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/FormTypeModal.svelte @@ -1,5 +1,5 @@
{#if label && !labelHidden} diff --git a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ComponentSettingsSection.svelte b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ComponentSettingsSection.svelte index b55be0728a..c7f8094084 100644 --- a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ComponentSettingsSection.svelte +++ b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ComponentSettingsSection.svelte @@ -177,9 +177,6 @@ defaultValue={setting.defaultValue} nested={setting.nested} onChange={val => updateSetting(setting, val)} - highlighted={$builderStore.highlightedSetting?.key === setting.key - ? $builderStore.highlightedSetting - : null} propertyFocus={$builderStore.propertyFocus === setting.key} info={setting.info} disableBindings={setting.disableBindings} From 2933571c62e9a8f58a24cdb73e5d7eed768b8812 Mon Sep 17 00:00:00 2001 From: Peter Clement Date: Wed, 28 Feb 2024 08:34:41 +0000 Subject: [PATCH 013/114] update runLuceneQuery in client to allow for all filter matching --- packages/shared-core/src/filters.ts | 59 +++++++++++++++++++++-------- 1 file changed, 44 insertions(+), 15 deletions(-) diff --git a/packages/shared-core/src/filters.ts b/packages/shared-core/src/filters.ts index 46d765a7b5..2c4861ed60 100644 --- a/packages/shared-core/src/filters.ts +++ b/packages/shared-core/src/filters.ts @@ -390,23 +390,52 @@ export const runLuceneQuery = (docs: any[], query?: SearchQuery) => { } ) - // Match a document against all criteria const docMatch = (doc: any) => { - return ( - stringMatch(doc) && - fuzzyMatch(doc) && - rangeMatch(doc) && - equalMatch(doc) && - notEqualMatch(doc) && - emptyMatch(doc) && - notEmptyMatch(doc) && - oneOf(doc) && - contains(doc) && - containsAny(doc) && - notContains(doc) - ) - } + // Determine active filters based on query object + const activeFilterKeys = Object.entries(query || {}) + .filter( + ([key, value]) => + !["allOr", "onEmptyFilter"].includes(key) && + Object.keys(value).length > 0 + ) + .map(([key]) => key) + // Apply filters dynamically based on activeFilterKeys + const results = activeFilterKeys.map(filterKey => { + switch (filterKey) { + case "string": + return stringMatch(doc) + case "fuzzy": + return fuzzyMatch(doc) + case "range": + return rangeMatch(doc) + case "equal": + return equalMatch(doc) + case "notEqual": + return notEqualMatch(doc) + case "empty": + return emptyMatch(doc) + case "notEmpty": + return notEmptyMatch(doc) + case "oneOf": + return oneOf(doc) + case "contains": + return contains(doc) + case "containsAny": + return containsAny(doc) + case "notContains": + return notContains(doc) + default: + return true // If the filter type is not recognized, default to true (assuming pass) + } + }) + + if (query!.allOr) { + return results.some(result => result === true) + } else { + return results.every(result => result === true) + } + } // Process all docs return docs.filter(docMatch) } From 3203cc3d72b8a95e4bc5e02bac1df7d1d52ab4c9 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 28 Feb 2024 16:27:14 +0000 Subject: [PATCH 014/114] Convert TableAPI. --- .../server/src/api/routes/tests/table.spec.ts | 8 +-- .../server/src/tests/utilities/api/base.ts | 65 +++++++++++++++++- .../server/src/tests/utilities/api/table.ts | 67 ++++--------------- 3 files changed, 81 insertions(+), 59 deletions(-) diff --git a/packages/server/src/api/routes/tests/table.spec.ts b/packages/server/src/api/routes/tests/table.spec.ts index ce119e56f0..4c83237a49 100644 --- a/packages/server/src/api/routes/tests/table.spec.ts +++ b/packages/server/src/api/routes/tests/table.spec.ts @@ -831,7 +831,7 @@ describe("/tables", () => { subtype: FieldSubtype.USERS, }, }, - { expectStatus: 400 } + { status: 400 } ) }) @@ -846,7 +846,7 @@ describe("/tables", () => { subtype: FieldSubtype.USERS, }, }, - { expectStatus: 400 } + { status: 400 } ) }) @@ -861,7 +861,7 @@ describe("/tables", () => { subtype: FieldSubtype.USERS, }, }, - { expectStatus: 400 } + { status: 400 } ) }) @@ -880,7 +880,7 @@ describe("/tables", () => { subtype: FieldSubtype.USERS, }, }, - { expectStatus: 400 } + { status: 400 } ) }) }) diff --git a/packages/server/src/tests/utilities/api/base.ts b/packages/server/src/tests/utilities/api/base.ts index 34120277c0..61f2c64610 100644 --- a/packages/server/src/tests/utilities/api/base.ts +++ b/packages/server/src/tests/utilities/api/base.ts @@ -2,7 +2,7 @@ import TestConfiguration from "../TestConfiguration" import { SuperTest, Test } from "supertest" export interface TestAPIOpts { - headers?: any + headers?: Record status?: number } @@ -14,4 +14,67 @@ export abstract class TestAPI { this.config = config this.request = config.request! } + + protected _get = async ( + url: string, + opts: TestAPIOpts = {} + ): Promise => { + return await this._request("get", url, undefined, opts) + } + + protected _post = async ( + url: string, + body: Record, + opts: TestAPIOpts = {} + ): Promise => { + return await this._request("post", url, body, opts) + } + + protected _put = async ( + url: string, + body: Record, + opts: TestAPIOpts = {} + ): Promise => { + return await this._request("put", url, body, opts) + } + + protected _patch = async ( + url: string, + body: Record, + opts: TestAPIOpts = {} + ): Promise => { + return await this._request("patch", url, body, opts) + } + + protected _delete = async ( + url: string, + body: Record, + opts: TestAPIOpts = {} + ): Promise => { + return await this._request("delete", url, body, opts) + } + + protected _request = async ( + method: "get" | "post" | "put" | "patch" | "delete", + url: string, + body?: Record, + opts: TestAPIOpts = {} + ): Promise => { + const { headers = {}, status = 200 } = opts + const response = await this.request[method](url) + .send(body) + .set(this.config.defaultHeaders()) + .set(headers) + .expect("Content-Type", /json/) + + if (response.status !== status) { + throw new Error( + `Expected status ${status} but got ${ + response.status + } with body ${JSON.stringify(response.body)}` + ) + } + + return response.body as T + } } diff --git a/packages/server/src/tests/utilities/api/table.ts b/packages/server/src/tests/utilities/api/table.ts index 5a9654e3bc..d2f16e0e1b 100644 --- a/packages/server/src/tests/utilities/api/table.ts +++ b/packages/server/src/tests/utilities/api/table.ts @@ -5,74 +5,33 @@ import { SaveTableResponse, Table, } from "@budibase/types" -import TestConfiguration from "../TestConfiguration" -import { TestAPI } from "./base" +import { TestAPI, TestAPIOpts } from "./base" export class TableAPI extends TestAPI { - constructor(config: TestConfiguration) { - super(config) - } - save = async ( data: SaveTableRequest, - { expectStatus } = { expectStatus: 200 } + opts?: TestAPIOpts ): Promise => { - const res = await this.request - .post(`/api/tables`) - .send(data) - .set(this.config.defaultHeaders()) - .expect("Content-Type", /json/) - - if (res.status !== expectStatus) { - throw new Error( - `Expected status ${expectStatus} but got ${ - res.status - } with body ${JSON.stringify(res.body)}` - ) - } - - return res.body + return await this._post("/api/tables", data, opts) } - fetch = async ( - { expectStatus } = { expectStatus: 200 } - ): Promise => { - const res = await this.request - .get(`/api/tables`) - .set(this.config.defaultHeaders()) - .expect("Content-Type", /json/) - .expect(expectStatus) - return res.body + fetch = async (opts?: TestAPIOpts): Promise => { + return await this._get("/api/tables", opts) } - get = async ( - tableId: string, - { expectStatus } = { expectStatus: 200 } - ): Promise => { - const res = await this.request - .get(`/api/tables/${tableId}`) - .set(this.config.defaultHeaders()) - .expect("Content-Type", /json/) - .expect(expectStatus) - return res.body + get = async (tableId: string, opts?: TestAPIOpts): Promise
=> { + return await this._get
(`/api/tables/${tableId}`, opts) } migrate = async ( tableId: string, data: MigrateRequest, - { expectStatus } = { expectStatus: 200 } + opts?: TestAPIOpts ): Promise => { - const res = await this.request - .post(`/api/tables/${tableId}/migrate`) - .send(data) - .set(this.config.defaultHeaders()) - if (res.status !== expectStatus) { - throw new Error( - `Expected status ${expectStatus} but got ${ - res.status - } with body ${JSON.stringify(res.body)}` - ) - } - return res.body + return await this._post( + `/api/tables/${tableId}/migrate`, + data, + opts + ) } } From e876d14b92673ec75fe942eb17195583a067ef25 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Wed, 28 Feb 2024 16:43:41 +0000 Subject: [PATCH 015/114] Ensure unsaved pending changes to rows are applied when changing cell --- .../src/components/grid/stores/rows.js | 51 ++++++++++++++----- .../src/components/grid/stores/ui.js | 10 ++++ .../src/components/grid/stores/validation.js | 22 +++++++- 3 files changed, 70 insertions(+), 13 deletions(-) diff --git a/packages/frontend-core/src/components/grid/stores/rows.js b/packages/frontend-core/src/components/grid/stores/rows.js index 34ba77afa2..f4b0e97942 100644 --- a/packages/frontend-core/src/components/grid/stores/rows.js +++ b/packages/frontend-core/src/components/grid/stores/rows.js @@ -328,25 +328,34 @@ export const createActions = context => { } // Patches a row with some changes - const updateRow = async (rowId, changes, options = { save: true }) => { + const updateRow = async ( + rowId, + changes, + options = { save: true, force: false } + ) => { const $rows = get(rows) const $rowLookupMap = get(rowLookupMap) const index = $rowLookupMap[rowId] const row = $rows[index] - if (index == null || !Object.keys(changes || {}).length) { + if (index == null) { + return + } + if (!options?.force && !Object.keys(changes || {}).length) { return } // Abandon if no changes - let same = true - for (let column of Object.keys(changes)) { - if (row[column] !== changes[column]) { - same = false - break + if (!options?.force) { + let same = true + for (let column of Object.keys(changes)) { + if (row[column] !== changes[column]) { + same = false + break + } + } + if (same) { + return } - } - if (same) { - return } // Immediately update state so that the change is reflected @@ -359,7 +368,7 @@ export const createActions = context => { })) // Stop here if we don't want to persist the change - if (!options?.save) { + if (!options?.save && !options?.force) { return } @@ -508,7 +517,14 @@ export const createActions = context => { } export const initialise = context => { - const { rowChangeCache, inProgressChanges, previousFocusedRowId } = context + const { + rowChangeCache, + inProgressChanges, + previousFocusedRowId, + previousFocusedCellId, + rows, + validation, + } = context // Wipe the row change cache when changing row previousFocusedRowId.subscribe(id => { @@ -519,4 +535,15 @@ export const initialise = context => { }) } }) + + // Ensure any unsaved changes are saved when changing cell + previousFocusedCellId.subscribe(id => { + const rowId = id?.split("-")[0] + const hasErrors = validation.actions.rowHasErrors(rowId) + const hasChanges = Object.keys(get(rowChangeCache)[rowId] || {}).length > 0 + const isSavingChanges = get(inProgressChanges)[rowId] + if (rowId && !hasErrors && hasChanges && !isSavingChanges) { + rows.actions.updateRow(rowId, null, { force: true }) + } + }) } diff --git a/packages/frontend-core/src/components/grid/stores/ui.js b/packages/frontend-core/src/components/grid/stores/ui.js index 129d6614e5..da0558bb5b 100644 --- a/packages/frontend-core/src/components/grid/stores/ui.js +++ b/packages/frontend-core/src/components/grid/stores/ui.js @@ -16,6 +16,7 @@ export const createStores = context => { const hoveredRowId = writable(null) const rowHeight = writable(get(props).fixedRowHeight || DefaultRowHeight) const previousFocusedRowId = writable(null) + const previousFocusedCellId = writable(null) const gridFocused = writable(false) const isDragging = writable(false) const buttonColumnWidth = writable(0) @@ -48,6 +49,7 @@ export const createStores = context => { focusedCellAPI, focusedRowId, previousFocusedRowId, + previousFocusedCellId, hoveredRowId, rowHeight, gridFocused, @@ -129,6 +131,7 @@ export const initialise = context => { const { focusedRowId, previousFocusedRowId, + previousFocusedCellId, rows, focusedCellId, selectedRows, @@ -181,6 +184,13 @@ export const initialise = context => { lastFocusedRowId = id }) + // Remember the last focused cell ID so that we can store the previous one + let lastFocusedCellId = null + focusedCellId.subscribe(id => { + previousFocusedCellId.set(lastFocusedCellId) + lastFocusedCellId = id + }) + // Remove hovered row when a cell is selected focusedCellId.subscribe(cell => { if (cell && get(hoveredRowId)) { diff --git a/packages/frontend-core/src/components/grid/stores/validation.js b/packages/frontend-core/src/components/grid/stores/validation.js index 9c3927f9c9..70db076593 100644 --- a/packages/frontend-core/src/components/grid/stores/validation.js +++ b/packages/frontend-core/src/components/grid/stores/validation.js @@ -1,8 +1,23 @@ -import { writable, get } from "svelte/store" +import { writable, get, derived } from "svelte/store" +// Normally we would break out actions into the explicit "createActions" +// function, but for validation all these actions are pure so can go into +// "createStores" instead to make dependency ordering simpler export const createStores = () => { const validation = writable({}) + // Derive which rows have errors so that we can use that info later + const rowErrorMap = derived(validation, $validation => { + let map = {} + Object.entries($validation).forEach(([key, error]) => { + // Extract row ID from all errored cell IDs + if (error) { + map[key.split("-")[0]] = true + } + }) + return map + }) + const setError = (cellId, error) => { if (!cellId) { return @@ -13,11 +28,16 @@ export const createStores = () => { })) } + const rowHasErrors = rowId => { + return get(rowErrorMap)[rowId] + } + return { validation: { ...validation, actions: { setError, + rowHasErrors, }, }, } From f1ed7af4393bc5cfeaa0d862b3ff591428f79e88 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 28 Feb 2024 16:55:45 +0000 Subject: [PATCH 016/114] Rework the API slightly. --- .../server/src/tests/utilities/api/base.ts | 90 ++++++++++++------- .../server/src/tests/utilities/api/table.ts | 28 +++--- 2 files changed, 72 insertions(+), 46 deletions(-) diff --git a/packages/server/src/tests/utilities/api/base.ts b/packages/server/src/tests/utilities/api/base.ts index 61f2c64610..e5818f7e23 100644 --- a/packages/server/src/tests/utilities/api/base.ts +++ b/packages/server/src/tests/utilities/api/base.ts @@ -1,71 +1,95 @@ import TestConfiguration from "../TestConfiguration" import { SuperTest, Test } from "supertest" +type Headers = Record + export interface TestAPIOpts { - headers?: Record + headers?: Headers status?: number } +export interface Expectations { + status?: number + contentType?: string | RegExp +} + +export interface RequestOpts { + headers?: Headers + body?: Record + fields?: Record + files?: Record + expectations?: Expectations +} + export abstract class TestAPI { config: TestConfiguration request: SuperTest - protected constructor(config: TestConfiguration) { + constructor(config: TestConfiguration) { this.config = config this.request = config.request! } protected _get = async ( url: string, - opts: TestAPIOpts = {} + expectations?: Expectations ): Promise => { - return await this._request("get", url, undefined, opts) + return await this._request("get", url, { expectations }) } - protected _post = async ( - url: string, - body: Record, - opts: TestAPIOpts = {} - ): Promise => { - return await this._request("post", url, body, opts) + protected _post = async (url: string, opts?: RequestOpts): Promise => { + return await this._request("post", url, opts) } - protected _put = async ( - url: string, - body: Record, - opts: TestAPIOpts = {} - ): Promise => { - return await this._request("put", url, body, opts) + protected _put = async (url: string, opts?: RequestOpts): Promise => { + return await this._request("put", url, opts) } - protected _patch = async ( - url: string, - body: Record, - opts: TestAPIOpts = {} - ): Promise => { - return await this._request("patch", url, body, opts) + protected _patch = async (url: string, opts?: RequestOpts): Promise => { + return await this._request("patch", url, opts) } protected _delete = async ( url: string, - body: Record, - opts: TestAPIOpts = {} + opts?: RequestOpts ): Promise => { - return await this._request("delete", url, body, opts) + return await this._request("delete", url, opts) } protected _request = async ( method: "get" | "post" | "put" | "patch" | "delete", url: string, - body?: Record, - opts: TestAPIOpts = {} + opts?: RequestOpts ): Promise => { - const { headers = {}, status = 200 } = opts - const response = await this.request[method](url) - .send(body) - .set(this.config.defaultHeaders()) - .set(headers) - .expect("Content-Type", /json/) + const { headers = {}, body, fields, files, expectations } = opts || {} + const { status = 200, contentType = /json/ } = expectations || {} + + let request = this.request[method](url).set(this.config.defaultHeaders()) + if (headers) { + request = request.set(headers) + } + if (body) { + request = request.send(body) + } + if (fields) { + for (const [key, value] of Object.entries(fields)) { + request = request.field(key, value) + } + } + if (files) { + for (const [key, value] of Object.entries(files)) { + request = request.attach(key, value) + } + } + if (contentType) { + if (contentType instanceof RegExp) { + request = request.expect("Content-Type", contentType) + } else { + request = request.expect("Content-Type", contentType) + } + } + + const response = await request if (response.status !== status) { throw new Error( diff --git a/packages/server/src/tests/utilities/api/table.ts b/packages/server/src/tests/utilities/api/table.ts index d2f16e0e1b..90ad0e04fb 100644 --- a/packages/server/src/tests/utilities/api/table.ts +++ b/packages/server/src/tests/utilities/api/table.ts @@ -5,33 +5,35 @@ import { SaveTableResponse, Table, } from "@budibase/types" -import { TestAPI, TestAPIOpts } from "./base" +import { Expectations, TestAPI } from "./base" export class TableAPI extends TestAPI { save = async ( data: SaveTableRequest, - opts?: TestAPIOpts + expectations?: Expectations ): Promise => { - return await this._post("/api/tables", data, opts) + return await this._post("/api/tables", { + body: data, + expectations, + }) } - fetch = async (opts?: TestAPIOpts): Promise => { - return await this._get("/api/tables", opts) + fetch = async (expectations?: Expectations): Promise => { + return await this._get("/api/tables", expectations) } - get = async (tableId: string, opts?: TestAPIOpts): Promise
=> { - return await this._get
(`/api/tables/${tableId}`, opts) + get = async (tableId: string, expectations: Expectations): Promise
=> { + return await this._get
(`/api/tables/${tableId}`, expectations) } migrate = async ( tableId: string, data: MigrateRequest, - opts?: TestAPIOpts + expectations?: Expectations ): Promise => { - return await this._post( - `/api/tables/${tableId}/migrate`, - data, - opts - ) + return await this._post(`/api/tables/${tableId}/migrate`, { + body: data, + expectations, + }) } } From 7a48fd85acfeac57b4a22d2237a9b549c7973b5d Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 28 Feb 2024 17:27:15 +0000 Subject: [PATCH 017/114] Migrate ApplicationAPI --- .../src/api/routes/tests/application.spec.ts | 2 +- .../appMigrations/tests/migrations.spec.ts | 14 +- .../src/tests/utilities/api/application.ts | 231 +++++++----------- .../server/src/tests/utilities/api/base.ts | 51 +++- .../server/src/tests/utilities/api/table.ts | 4 +- packages/types/src/api/web/application.ts | 6 + 6 files changed, 149 insertions(+), 159 deletions(-) diff --git a/packages/server/src/api/routes/tests/application.spec.ts b/packages/server/src/api/routes/tests/application.spec.ts index dbe4eb51ae..dc235dbd01 100644 --- a/packages/server/src/api/routes/tests/application.spec.ts +++ b/packages/server/src/api/routes/tests/application.spec.ts @@ -184,7 +184,7 @@ describe("/applications", () => { it("app should not sync if production", async () => { const { message } = await config.api.application.sync( app.appId.replace("_dev", ""), - { statusCode: 400 } + { status: 400 } ) expect(message).toEqual( diff --git a/packages/server/src/appMigrations/tests/migrations.spec.ts b/packages/server/src/appMigrations/tests/migrations.spec.ts index 5eb8535695..7af2346934 100644 --- a/packages/server/src/appMigrations/tests/migrations.spec.ts +++ b/packages/server/src/appMigrations/tests/migrations.spec.ts @@ -30,9 +30,9 @@ describe("migrations", () => { const appId = config.getAppId() - const response = await config.api.application.getRaw(appId) - - expect(response.headers[Header.MIGRATING_APP]).toBeUndefined() + await config.api.application.get(appId, { + headersNotPresent: [Header.MIGRATING_APP], + }) }) it("accessing an app that has pending migrations will attach the migrating header", async () => { @@ -46,8 +46,10 @@ describe("migrations", () => { func: async () => {}, }) - const response = await config.api.application.getRaw(appId) - - expect(response.headers[Header.MIGRATING_APP]).toEqual(appId) + await config.api.application.get(appId, { + headers: { + [Header.MIGRATING_APP]: appId, + }, + }) }) }) diff --git a/packages/server/src/tests/utilities/api/application.ts b/packages/server/src/tests/utilities/api/application.ts index 3951bba667..f215a98edb 100644 --- a/packages/server/src/tests/utilities/api/application.ts +++ b/packages/server/src/tests/utilities/api/application.ts @@ -1,12 +1,12 @@ -import { Response } from "supertest" import { App, + PublishResponse, type CreateAppRequest, type FetchAppDefinitionResponse, type FetchAppPackageResponse, } from "@budibase/types" import TestConfiguration from "../TestConfiguration" -import { TestAPI } from "./base" +import { Expectations, TestAPI } from "./base" import { AppStatus } from "../../../db/utils" import { constants } from "@budibase/backend-core" @@ -15,179 +15,124 @@ export class ApplicationAPI extends TestAPI { super(config) } - create = async (app: CreateAppRequest): Promise => { - const request = this.request - .post("/api/applications") - .set(this.config.defaultHeaders()) - .expect("Content-Type", /json/) - - for (const key of Object.keys(app)) { - request.field(key, (app as any)[key]) - } - - if (app.templateFile) { - request.attach("templateFile", app.templateFile) - } - - const result = await request - - if (result.statusCode !== 200) { - throw new Error(JSON.stringify(result.body)) - } - - return result.body as App + create = async ( + app: CreateAppRequest, + expectations?: Expectations + ): Promise => { + const files = app.templateFile ? { templateFile: app.templateFile } : {} + delete app.templateFile + return await this._post("/api/applications", { + fields: app, + files, + expectations, + }) } - delete = async (appId: string): Promise => { - await this.request - .delete(`/api/applications/${appId}`) - .set(this.config.defaultHeaders()) - .expect(200) + delete = async ( + appId: string, + expectations?: Expectations + ): Promise => { + await this._delete(`/api/applications/${appId}`, { expectations }) } - publish = async ( - appId: string - ): Promise<{ _id: string; status: string; appUrl: string }> => { - // While the publish endpoint does take an :appId parameter, it doesn't - // use it. It uses the appId from the context. - let headers = { - ...this.config.defaultHeaders(), - [constants.Header.APP_ID]: appId, - } - const result = await this.request - .post(`/api/applications/${appId}/publish`) - .set(headers) - .expect("Content-Type", /json/) - .expect(200) - return result.body as { _id: string; status: string; appUrl: string } + publish = async (appId: string): Promise => { + return await this._post( + `/api/applications/${appId}/publish`, + { + // While the publish endpoint does take an :appId parameter, it doesn't + // use it. It uses the appId from the context. + headers: { + [constants.Header.APP_ID]: appId, + }, + } + ) } unpublish = async (appId: string): Promise => { - await this.request - .post(`/api/applications/${appId}/unpublish`) - .set(this.config.defaultHeaders()) - .expect(204) + await this._post(`/api/applications/${appId}/unpublish`, { + expectations: { status: 204 }, + }) } sync = async ( appId: string, - { statusCode }: { statusCode: number } = { statusCode: 200 } + expectations?: Expectations ): Promise<{ message: string }> => { - const result = await this.request - .post(`/api/applications/${appId}/sync`) - .set(this.config.defaultHeaders()) - .expect("Content-Type", /json/) - .expect(statusCode) - return result.body + return await this._post<{ message: string }>( + `/api/applications/${appId}/sync`, + { expectations } + ) } - getRaw = async (appId: string): Promise => { - // While the appPackage endpoint does take an :appId parameter, it doesn't - // use it. It uses the appId from the context. - let headers = { - ...this.config.defaultHeaders(), - [constants.Header.APP_ID]: appId, - } - const result = await this.request - .get(`/api/applications/${appId}/appPackage`) - .set(headers) - .expect("Content-Type", /json/) - .expect(200) - return result - } - - get = async (appId: string): Promise => { - const result = await this.getRaw(appId) - return result.body.application as App + get = async (appId: string, expectations?: Expectations): Promise => { + return await this._get(`/api/applications/${appId}`, { + // While the get endpoint does take an :appId parameter, it doesn't use + // it. It uses the appId from the context. + headers: { + [constants.Header.APP_ID]: appId, + }, + expectations, + }) } getDefinition = async ( - appId: string + appId: string, + expectations?: Expectations ): Promise => { - const result = await this.request - .get(`/api/applications/${appId}/definition`) - .set(this.config.defaultHeaders()) - .expect("Content-Type", /json/) - .expect(200) - return result.body as FetchAppDefinitionResponse + return await this._get( + `/api/applications/${appId}/definition`, + { expectations } + ) } - getAppPackage = async (appId: string): Promise => { - const result = await this.request - .get(`/api/applications/${appId}/appPackage`) - .set(this.config.defaultHeaders()) - .expect("Content-Type", /json/) - .expect(200) - return result.body + getAppPackage = async ( + appId: string, + expectations?: Expectations + ): Promise => { + return await this._get( + `/api/applications/${appId}/appPackage`, + { expectations } + ) } update = async ( appId: string, - app: { name?: string; url?: string } + app: { name?: string; url?: string }, + expectations?: Expectations ): Promise => { - const request = this.request - .put(`/api/applications/${appId}`) - .set(this.config.defaultHeaders()) - .expect("Content-Type", /json/) - - for (const key of Object.keys(app)) { - request.field(key, (app as any)[key]) - } - - const result = await request - - if (result.statusCode !== 200) { - throw new Error(JSON.stringify(result.body)) - } - - return result.body as App + return await this._put(`/api/applications/${appId}`, { + fields: app, + expectations, + }) } - updateClient = async (appId: string): Promise => { - // While the updateClient endpoint does take an :appId parameter, it doesn't - // use it. It uses the appId from the context. - let headers = { - ...this.config.defaultHeaders(), - [constants.Header.APP_ID]: appId, - } - const response = await this.request - .post(`/api/applications/${appId}/client/update`) - .set(headers) - .expect("Content-Type", /json/) - - if (response.statusCode !== 200) { - throw new Error(JSON.stringify(response.body)) - } + updateClient = async ( + appId: string, + expectations?: Expectations + ): Promise => { + await this._post(`/api/applications/${appId}/client/update`, { + // While the updateClient endpoint does take an :appId parameter, it doesn't + // use it. It uses the appId from the context. + headers: { + [constants.Header.APP_ID]: appId, + }, + expectations, + }) } revertClient = async (appId: string): Promise => { - // While the revertClient endpoint does take an :appId parameter, it doesn't - // use it. It uses the appId from the context. - let headers = { - ...this.config.defaultHeaders(), - [constants.Header.APP_ID]: appId, - } - const response = await this.request - .post(`/api/applications/${appId}/client/revert`) - .set(headers) - .expect("Content-Type", /json/) - - if (response.statusCode !== 200) { - throw new Error(JSON.stringify(response.body)) - } + await this._post(`/api/applications/${appId}/client/revert`, { + // While the revertClient endpoint does take an :appId parameter, it doesn't + // use it. It uses the appId from the context. + headers: { + [constants.Header.APP_ID]: appId, + }, + }) } fetch = async ({ status }: { status?: AppStatus } = {}): Promise => { - let query = [] - if (status) { - query.push(`status=${status}`) - } - - const result = await this.request - .get(`/api/applications${query.length ? `?${query.join("&")}` : ""}`) - .set(this.config.defaultHeaders()) - .expect("Content-Type", /json/) - .expect(200) - return result.body as App[] + return await this._get("/api/applications", { + query: { status }, + }) } } diff --git a/packages/server/src/tests/utilities/api/base.ts b/packages/server/src/tests/utilities/api/base.ts index e5818f7e23..8325c1caf6 100644 --- a/packages/server/src/tests/utilities/api/base.ts +++ b/packages/server/src/tests/utilities/api/base.ts @@ -1,3 +1,4 @@ +import exp from "constants" import TestConfiguration from "../TestConfiguration" import { SuperTest, Test } from "supertest" @@ -11,10 +12,13 @@ export interface TestAPIOpts { export interface Expectations { status?: number contentType?: string | RegExp + headers?: Record + headersNotPresent?: string[] } export interface RequestOpts { headers?: Headers + query?: Record body?: Record fields?: Record files?: Record @@ -30,11 +34,8 @@ export abstract class TestAPI { this.request = config.request! } - protected _get = async ( - url: string, - expectations?: Expectations - ): Promise => { - return await this._request("get", url, { expectations }) + protected _get = async (url: string, opts?: RequestOpts): Promise => { + return await this._request("get", url, opts) } protected _post = async (url: string, opts?: RequestOpts): Promise => { @@ -61,9 +62,26 @@ export abstract class TestAPI { url: string, opts?: RequestOpts ): Promise => { - const { headers = {}, body, fields, files, expectations } = opts || {} + const { + headers = {}, + query = {}, + body, + fields, + files, + expectations, + } = opts || {} const { status = 200, contentType = /json/ } = expectations || {} + let queryParams = [] + for (const [key, value] of Object.entries(query)) { + if (value) { + queryParams.push(`${key}=${value}`) + } + } + if (queryParams.length) { + url += `?${queryParams.join("&")}` + } + let request = this.request[method](url).set(this.config.defaultHeaders()) if (headers) { request = request.set(headers) @@ -81,13 +99,22 @@ export abstract class TestAPI { request = request.attach(key, value) } } - if (contentType) { + if (contentType && status !== 204) { if (contentType instanceof RegExp) { request = request.expect("Content-Type", contentType) } else { request = request.expect("Content-Type", contentType) } } + if (expectations?.headers) { + for (const [key, value] of Object.entries(expectations.headers)) { + if (value instanceof RegExp) { + request = request.expect(key, value) + } else { + request = request.expect(key, value) + } + } + } const response = await request @@ -99,6 +126,16 @@ export abstract class TestAPI { ) } + if (expectations?.headersNotPresent) { + for (const header of expectations.headersNotPresent) { + if (response.headers[header]) { + throw new Error( + `Expected header ${header} not to be present, found value "${response.headers[header]}"` + ) + } + } + } + return response.body as T } } diff --git a/packages/server/src/tests/utilities/api/table.ts b/packages/server/src/tests/utilities/api/table.ts index 90ad0e04fb..b692e4ed1a 100644 --- a/packages/server/src/tests/utilities/api/table.ts +++ b/packages/server/src/tests/utilities/api/table.ts @@ -19,11 +19,11 @@ export class TableAPI extends TestAPI { } fetch = async (expectations?: Expectations): Promise => { - return await this._get("/api/tables", expectations) + return await this._get("/api/tables", { expectations }) } get = async (tableId: string, expectations: Expectations): Promise
=> { - return await this._get
(`/api/tables/${tableId}`, expectations) + return await this._get
(`/api/tables/${tableId}`, { expectations }) } migrate = async ( diff --git a/packages/types/src/api/web/application.ts b/packages/types/src/api/web/application.ts index 87a0bd6ef9..3d33fce5b1 100644 --- a/packages/types/src/api/web/application.ts +++ b/packages/types/src/api/web/application.ts @@ -27,3 +27,9 @@ export interface FetchAppPackageResponse { clientLibPath: string hasLock: boolean } + +export interface PublishResponse { + _id: string + status: string + appUrl: string +} From d9cffa1878e02fe40ec83a7ef9eaf83a36ee6e15 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 28 Feb 2024 17:43:39 +0000 Subject: [PATCH 018/114] Migrate AttachmentAPI. --- .../src/api/routes/tests/attachment.spec.ts | 6 +-- .../src/tests/utilities/api/application.ts | 5 -- .../src/tests/utilities/api/attachment.ts | 33 +++--------- .../server/src/tests/utilities/api/base.ts | 54 +++++++++++-------- 4 files changed, 43 insertions(+), 55 deletions(-) diff --git a/packages/server/src/api/routes/tests/attachment.spec.ts b/packages/server/src/api/routes/tests/attachment.spec.ts index e230b0688a..aa02ea898e 100644 --- a/packages/server/src/api/routes/tests/attachment.spec.ts +++ b/packages/server/src/api/routes/tests/attachment.spec.ts @@ -29,7 +29,7 @@ describe("/api/applications/:appId/sync", () => { let resp = (await config.api.attachment.process( "ohno.exe", Buffer.from([0]), - { expectStatus: 400 } + { status: 400 } )) as unknown as APIError expect(resp.message).toContain("invalid extension") }) @@ -40,7 +40,7 @@ describe("/api/applications/:appId/sync", () => { let resp = (await config.api.attachment.process( "OHNO.EXE", Buffer.from([0]), - { expectStatus: 400 } + { status: 400 } )) as unknown as APIError expect(resp.message).toContain("invalid extension") }) @@ -51,7 +51,7 @@ describe("/api/applications/:appId/sync", () => { undefined as any, undefined as any, { - expectStatus: 400, + status: 400, } )) as unknown as APIError expect(resp.message).toContain("No file provided") diff --git a/packages/server/src/tests/utilities/api/application.ts b/packages/server/src/tests/utilities/api/application.ts index f215a98edb..da3d7cefd8 100644 --- a/packages/server/src/tests/utilities/api/application.ts +++ b/packages/server/src/tests/utilities/api/application.ts @@ -5,16 +5,11 @@ import { type FetchAppDefinitionResponse, type FetchAppPackageResponse, } from "@budibase/types" -import TestConfiguration from "../TestConfiguration" import { Expectations, TestAPI } from "./base" import { AppStatus } from "../../../db/utils" import { constants } from "@budibase/backend-core" export class ApplicationAPI extends TestAPI { - constructor(config: TestConfiguration) { - super(config) - } - create = async ( app: CreateAppRequest, expectations?: Expectations diff --git a/packages/server/src/tests/utilities/api/attachment.ts b/packages/server/src/tests/utilities/api/attachment.ts index a466f1a67e..bb33ef04bb 100644 --- a/packages/server/src/tests/utilities/api/attachment.ts +++ b/packages/server/src/tests/utilities/api/attachment.ts @@ -1,35 +1,16 @@ -import { - APIError, - Datasource, - ProcessAttachmentResponse, -} from "@budibase/types" -import TestConfiguration from "../TestConfiguration" -import { TestAPI } from "./base" +import { ProcessAttachmentResponse } from "@budibase/types" +import { Expectations, TestAPI } from "./base" import fs from "fs" export class AttachmentAPI extends TestAPI { - constructor(config: TestConfiguration) { - super(config) - } - process = async ( name: string, file: Buffer | fs.ReadStream | string, - { expectStatus } = { expectStatus: 200 } + expectations?: Expectations ): Promise => { - const result = await this.request - .post(`/api/attachments/process`) - .attach("file", file, name) - .set(this.config.defaultHeaders()) - - if (result.statusCode !== expectStatus) { - throw new Error( - `Expected status ${expectStatus} but got ${ - result.statusCode - }, body: ${JSON.stringify(result.body)}` - ) - } - - return result.body + return await this._post(`/api/attachments/process`, { + files: { file: { name, file } }, + expectations, + }) } } diff --git a/packages/server/src/tests/utilities/api/base.ts b/packages/server/src/tests/utilities/api/base.ts index 8325c1caf6..9a6ff2e6ca 100644 --- a/packages/server/src/tests/utilities/api/base.ts +++ b/packages/server/src/tests/utilities/api/base.ts @@ -1,6 +1,6 @@ -import exp from "constants" import TestConfiguration from "../TestConfiguration" import { SuperTest, Test } from "supertest" +import { ReadStream } from "fs" type Headers = Record @@ -9,6 +9,22 @@ export interface TestAPIOpts { status?: number } +export interface AttachedFile { + name: string + file: Buffer | ReadStream | string +} + +function isAttachedFile(file: any): file is AttachedFile { + if (file === undefined) { + return false + } + const attachedFile = file as AttachedFile + return ( + Object.hasOwnProperty.call(attachedFile, "file") && + Object.hasOwnProperty.call(attachedFile, "name") + ) +} + export interface Expectations { status?: number contentType?: string | RegExp @@ -21,7 +37,10 @@ export interface RequestOpts { query?: Record body?: Record fields?: Record - files?: Record + files?: Record< + string, + Buffer | ReadStream | string | AttachedFile | undefined + > expectations?: Expectations } @@ -66,8 +85,8 @@ export abstract class TestAPI { headers = {}, query = {}, body, - fields, - files, + fields = {}, + files = {}, expectations, } = opts || {} const { status = 200, contentType = /json/ } = expectations || {} @@ -89,30 +108,23 @@ export abstract class TestAPI { if (body) { request = request.send(body) } - if (fields) { - for (const [key, value] of Object.entries(fields)) { - request = request.field(key, value) - } + for (const [key, value] of Object.entries(fields)) { + request = request.field(key, value) } - if (files) { - for (const [key, value] of Object.entries(files)) { - request = request.attach(key, value) + + for (const [key, value] of Object.entries(files)) { + if (isAttachedFile(value)) { + request = request.attach(key, value.file, value.name) + } else { + request = request.attach(key, value as any) } } if (contentType && status !== 204) { - if (contentType instanceof RegExp) { - request = request.expect("Content-Type", contentType) - } else { - request = request.expect("Content-Type", contentType) - } + request = request.expect("Content-Type", contentType as any) } if (expectations?.headers) { for (const [key, value] of Object.entries(expectations.headers)) { - if (value instanceof RegExp) { - request = request.expect(key, value) - } else { - request = request.expect(key, value) - } + request = request.expect(key, value as any) } } From e309282ff795a4752fa8c09f4a8b714f7c7a9af5 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 28 Feb 2024 17:46:16 +0000 Subject: [PATCH 019/114] Fix type checks. --- packages/server/src/tests/utilities/api/base.ts | 5 ----- packages/server/src/tests/utilities/api/table.ts | 5 ++++- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/packages/server/src/tests/utilities/api/base.ts b/packages/server/src/tests/utilities/api/base.ts index 9a6ff2e6ca..46fb7eb3b9 100644 --- a/packages/server/src/tests/utilities/api/base.ts +++ b/packages/server/src/tests/utilities/api/base.ts @@ -4,11 +4,6 @@ import { ReadStream } from "fs" type Headers = Record -export interface TestAPIOpts { - headers?: Headers - status?: number -} - export interface AttachedFile { name: string file: Buffer | ReadStream | string diff --git a/packages/server/src/tests/utilities/api/table.ts b/packages/server/src/tests/utilities/api/table.ts index b692e4ed1a..49105a3883 100644 --- a/packages/server/src/tests/utilities/api/table.ts +++ b/packages/server/src/tests/utilities/api/table.ts @@ -22,7 +22,10 @@ export class TableAPI extends TestAPI { return await this._get("/api/tables", { expectations }) } - get = async (tableId: string, expectations: Expectations): Promise
=> { + get = async ( + tableId: string, + expectations?: Expectations + ): Promise
=> { return await this._get
(`/api/tables/${tableId}`, { expectations }) } From 8488ff4144872d6e2dc5a9f4de502e0801627f07 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 29 Feb 2024 09:19:14 +0000 Subject: [PATCH 020/114] Print stack traces from inside request handler. --- .../src/middleware/errorHandling.ts | 23 ++++++++++------ .../server/src/tests/utilities/api/base.ts | 26 +++++++++++++++---- 2 files changed, 36 insertions(+), 13 deletions(-) diff --git a/packages/backend-core/src/middleware/errorHandling.ts b/packages/backend-core/src/middleware/errorHandling.ts index ebdd4107e9..ae5373a2e0 100644 --- a/packages/backend-core/src/middleware/errorHandling.ts +++ b/packages/backend-core/src/middleware/errorHandling.ts @@ -1,5 +1,6 @@ import { APIError } from "@budibase/types" import * as errors from "../errors" +import environment from "../environment" export async function errorHandling(ctx: any, next: any) { try { @@ -14,15 +15,21 @@ export async function errorHandling(ctx: any, next: any) { console.error(err) } - const error = errors.getPublicError(err) - const body: APIError = { - message: err.message, - status: status, - validationErrors: err.validation, - error, + if (environment.isTest()) { + ctx.body = { + message: err.message, + status: status, + error: errors.getPublicError(err), + stack: err.stack, + } + } else { + ctx.body = { + message: err.message, + status: status, + validationErrors: err.validation, + error: errors.getPublicError(err), + } } - - ctx.body = body } } diff --git a/packages/server/src/tests/utilities/api/base.ts b/packages/server/src/tests/utilities/api/base.ts index 46fb7eb3b9..42911628fd 100644 --- a/packages/server/src/tests/utilities/api/base.ts +++ b/packages/server/src/tests/utilities/api/base.ts @@ -126,11 +126,27 @@ export abstract class TestAPI { const response = await request if (response.status !== status) { - throw new Error( - `Expected status ${status} but got ${ - response.status - } with body ${JSON.stringify(response.body)}` - ) + let message = `Expected status ${status} but got ${response.status}` + + const stack = response.body.stack + delete response.body.stack + + if (response.body) { + message += `\n\nBody:` + const body = JSON.stringify(response.body, null, 2) + for (const line of body.split("\n")) { + message += `\n⏐ ${line}` + } + } + + if (stack) { + message += `\n\nStack from request handler:` + for (const line of stack.split("\n")) { + message += `\n⏐ ${line}` + } + } + + throw new Error(message) } if (expectations?.headersNotPresent) { From acecea570499d26ddd453157d6abedfc86a43dff Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Thu, 29 Feb 2024 10:30:38 +0000 Subject: [PATCH 021/114] Refactor grid row actions to be more explicit and remove extraneous flags --- .../src/components/grid/cells/DataCell.svelte | 4 +- .../grid/overlays/KeyboardManager.svelte | 4 +- .../src/components/grid/stores/rows.js | 100 +++++++++--------- 3 files changed, 54 insertions(+), 54 deletions(-) diff --git a/packages/frontend-core/src/components/grid/cells/DataCell.svelte b/packages/frontend-core/src/components/grid/cells/DataCell.svelte index cdaf28978a..d8cff26b9d 100644 --- a/packages/frontend-core/src/components/grid/cells/DataCell.svelte +++ b/packages/frontend-core/src/components/grid/cells/DataCell.svelte @@ -59,13 +59,13 @@ isReadonly: () => readonly, getType: () => column.schema.type, getValue: () => row[column.name], - setValue: (value, options = { save: true }) => { + setValue: (value, options = { apply: true }) => { validation.actions.setError(cellId, null) updateValue({ rowId: row._id, column: column.name, value, - save: options?.save, + apply: options?.apply, }) }, } diff --git a/packages/frontend-core/src/components/grid/overlays/KeyboardManager.svelte b/packages/frontend-core/src/components/grid/overlays/KeyboardManager.svelte index 8b0a0f0942..5e3a035d89 100644 --- a/packages/frontend-core/src/components/grid/overlays/KeyboardManager.svelte +++ b/packages/frontend-core/src/components/grid/overlays/KeyboardManager.svelte @@ -217,14 +217,14 @@ const type = $focusedCellAPI.getType() if (type === "number" && keyCodeIsNumber(keyCode)) { // Update the value locally but don't save it yet - $focusedCellAPI.setValue(parseInt(key), { save: false }) + $focusedCellAPI.setValue(parseInt(key), { apply: false }) $focusedCellAPI.focus() } else if ( ["string", "barcodeqr", "longform"].includes(type) && (keyCodeIsLetter(keyCode) || keyCodeIsNumber(keyCode)) ) { // Update the value locally but don't save it yet - $focusedCellAPI.setValue(key, { save: false }) + $focusedCellAPI.setValue(key, { apply: false }) $focusedCellAPI.focus() } } diff --git a/packages/frontend-core/src/components/grid/stores/rows.js b/packages/frontend-core/src/components/grid/stores/rows.js index f4b0e97942..c8d27da2e7 100644 --- a/packages/frontend-core/src/components/grid/stores/rows.js +++ b/packages/frontend-core/src/components/grid/stores/rows.js @@ -3,6 +3,7 @@ import { fetchData } from "../../../fetch" import { NewRowID, RowPageSize } from "../lib/constants" import { tick } from "svelte" import { Helpers } from "@budibase/bbui" +import { isValid } from "@budibase/string-templates" export const createStores = () => { const rows = writable([]) @@ -327,38 +328,31 @@ export const createActions = context => { get(fetch)?.getInitialData() } - // Patches a row with some changes - const updateRow = async ( - rowId, - changes, - options = { save: true, force: false } - ) => { + // Checks if a changeset for a row actually mutates the row or not + const changesAreValid = (row, changes) => { + const columns = Object.keys(changes || {}) + if (!row || !columns.length) { + return false + } + + // Ensure there is at least 1 column that creates a difference + return columns.some(column => row[column] !== changes[column]) + } + + // Patches a row with some changes in local state, and returns whether a + // valid pending change was made or not + const stashRowChanges = (rowId, changes) => { const $rows = get(rows) const $rowLookupMap = get(rowLookupMap) const index = $rowLookupMap[rowId] const row = $rows[index] - if (index == null) { - return - } - if (!options?.force && !Object.keys(changes || {}).length) { - return + + // Check this is a valid change + if (!row || !changesAreValid(row, changes)) { + return false } - // Abandon if no changes - if (!options?.force) { - let same = true - for (let column of Object.keys(changes)) { - if (row[column] !== changes[column]) { - same = false - break - } - } - if (same) { - return - } - } - - // Immediately update state so that the change is reflected + // Add change to cache rowChangeCache.update(state => ({ ...state, [rowId]: { @@ -366,26 +360,30 @@ export const createActions = context => { ...changes, }, })) + return true + } - // Stop here if we don't want to persist the change - if (!options?.save && !options?.force) { + // Saves any pending changes to a row + const applyRowChanges = async rowId => { + const $rows = get(rows) + const $rowLookupMap = get(rowLookupMap) + const index = $rowLookupMap[rowId] + const row = $rows[index] + if (row == null) { return } // Save change try { - inProgressChanges.update(state => ({ - ...state, - [rowId]: true, - })) + // Mark as in progress + inProgressChanges.update(state => ({ ...state, [rowId]: true })) // Update row - const saved = await datasource.actions.updateRow({ - ...cleanRow(row), - ...get(rowChangeCache)[rowId], - }) + const changes = get(rowChangeCache)[rowId] + const newRow = { ...cleanRow(row), ...changes } + const saved = await datasource.actions.updateRow(newRow) - // Update state after a successful change + // Update row state after a successful change if (saved?._id) { rows.update(state => { state[index] = saved @@ -395,6 +393,8 @@ export const createActions = context => { // Handle users table edge case await refreshRow(saved.id) } + + // Wipe row change cache now that we've saved the row rowChangeCache.update(state => { delete state[rowId] return state @@ -402,15 +402,17 @@ export const createActions = context => { } catch (error) { handleValidationError(rowId, error) } - inProgressChanges.update(state => ({ - ...state, - [rowId]: false, - })) + + // Mark as completed + inProgressChanges.update(state => ({ ...state, [rowId]: false })) } // Updates a value of a row - const updateValue = async ({ rowId, column, value, save = true }) => { - return await updateRow(rowId, { [column]: value }, { save }) + const updateValue = async ({ rowId, column, value, apply = true }) => { + const success = stashRowChanges(rowId, { [column]: value }) + if (success && apply) { + await applyRowChanges(rowId) + } } // Deletes an array of rows @@ -420,9 +422,7 @@ export const createActions = context => { } // Actually delete rows - rowsToDelete.forEach(row => { - delete row.__idx - }) + rowsToDelete.forEach(row => delete row.__idx) await datasource.actions.deleteRows(rowsToDelete) // Update state @@ -442,7 +442,7 @@ export const createActions = context => { newRow = newRows[i] // Ensure we have a unique _id. - // This means generating one for non DS+, overriting any that may already + // This means generating one for non DS+, overwriting any that may already // exist as we cannot allow duplicates. if (!$isDatasourcePlus) { newRow._id = Helpers.uuid() @@ -503,7 +503,7 @@ export const createActions = context => { duplicateRow, getRow, updateValue, - updateRow, + applyRowChanges, deleteRows, hasRow, loadNextPage, @@ -537,13 +537,13 @@ export const initialise = context => { }) // Ensure any unsaved changes are saved when changing cell - previousFocusedCellId.subscribe(id => { + previousFocusedCellId.subscribe(async id => { const rowId = id?.split("-")[0] const hasErrors = validation.actions.rowHasErrors(rowId) const hasChanges = Object.keys(get(rowChangeCache)[rowId] || {}).length > 0 const isSavingChanges = get(inProgressChanges)[rowId] if (rowId && !hasErrors && hasChanges && !isSavingChanges) { - rows.actions.updateRow(rowId, null, { force: true }) + await rows.actions.applyRowChanges(rowId) } }) } From 6b306266b5e322e5f3728a8392e7c55d5e82cd47 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 29 Feb 2024 11:09:39 +0000 Subject: [PATCH 022/114] Only show stack traces if you ask for them. --- .../src/middleware/errorHandling.ts | 26 +++++++++---------- .../server/src/tests/utilities/api/base.ts | 6 ++++- 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/packages/backend-core/src/middleware/errorHandling.ts b/packages/backend-core/src/middleware/errorHandling.ts index ae5373a2e0..2b8f7195ed 100644 --- a/packages/backend-core/src/middleware/errorHandling.ts +++ b/packages/backend-core/src/middleware/errorHandling.ts @@ -15,21 +15,19 @@ export async function errorHandling(ctx: any, next: any) { console.error(err) } - if (environment.isTest()) { - ctx.body = { - message: err.message, - status: status, - error: errors.getPublicError(err), - stack: err.stack, - } - } else { - ctx.body = { - message: err.message, - status: status, - validationErrors: err.validation, - error: errors.getPublicError(err), - } + let error: APIError = { + message: err.message, + status: status, + validationErrors: err.validation, + error: errors.getPublicError(err), } + + if (environment.isTest() && ctx.headers["x-budibase-include-stacktrace"]) { + // @ts-ignore + error.stack = err.stack + } + + ctx.body = error } } diff --git a/packages/server/src/tests/utilities/api/base.ts b/packages/server/src/tests/utilities/api/base.ts index 42911628fd..19fc7ed1f5 100644 --- a/packages/server/src/tests/utilities/api/base.ts +++ b/packages/server/src/tests/utilities/api/base.ts @@ -96,7 +96,11 @@ export abstract class TestAPI { url += `?${queryParams.join("&")}` } - let request = this.request[method](url).set(this.config.defaultHeaders()) + let request = this.request[method](url).set( + this.config.defaultHeaders({ + "x-budibase-include-stacktrace": "true", + }) + ) if (headers) { request = request.set(headers) } From bc723c7094db65e19eabbaff012f126a5037c571 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Thu, 29 Feb 2024 12:25:21 +0000 Subject: [PATCH 023/114] Lint --- packages/frontend-core/src/components/grid/stores/rows.js | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/frontend-core/src/components/grid/stores/rows.js b/packages/frontend-core/src/components/grid/stores/rows.js index c8d27da2e7..5dc9413ccd 100644 --- a/packages/frontend-core/src/components/grid/stores/rows.js +++ b/packages/frontend-core/src/components/grid/stores/rows.js @@ -3,7 +3,6 @@ import { fetchData } from "../../../fetch" import { NewRowID, RowPageSize } from "../lib/constants" import { tick } from "svelte" import { Helpers } from "@budibase/bbui" -import { isValid } from "@budibase/string-templates" export const createStores = () => { const rows = writable([]) From 5163434b08ce284a3171066d94de7b45ac76e98e Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 29 Feb 2024 14:33:09 +0000 Subject: [PATCH 024/114] Convert BackupAPI. --- .../src/api/routes/tests/backup.spec.ts | 21 +++---- .../server/src/tests/utilities/api/backup.ts | 61 +++++++++---------- .../server/src/tests/utilities/api/base.ts | 35 ++++++++--- 3 files changed, 62 insertions(+), 55 deletions(-) diff --git a/packages/server/src/api/routes/tests/backup.spec.ts b/packages/server/src/api/routes/tests/backup.spec.ts index becbeb5480..c862106d58 100644 --- a/packages/server/src/api/routes/tests/backup.spec.ts +++ b/packages/server/src/api/routes/tests/backup.spec.ts @@ -19,11 +19,8 @@ describe("/backups", () => { describe("/api/backups/export", () => { it("should be able to export app", async () => { - const { body, headers } = await config.api.backup.exportBasicBackup( - config.getAppId()! - ) + const body = await config.api.backup.exportBasicBackup(config.getAppId()!) expect(body instanceof Buffer).toBe(true) - expect(headers["content-type"]).toEqual("application/gzip") expect(events.app.exported).toBeCalledTimes(1) }) @@ -38,15 +35,13 @@ describe("/backups", () => { it("should infer the app name from the app", async () => { tk.freeze(mocks.date.MOCK_DATE) - const { headers } = await config.api.backup.exportBasicBackup( - config.getAppId()! - ) - - expect(headers["content-disposition"]).toEqual( - `attachment; filename="${ - config.getApp().name - }-export-${mocks.date.MOCK_DATE.getTime()}.tar.gz"` - ) + await config.api.backup.exportBasicBackup(config.getAppId()!, { + headers: { + "content-disposition": `attachment; filename="${ + config.getApp().name + }-export-${mocks.date.MOCK_DATE.getTime()}.tar.gz"`, + }, + }) }) }) diff --git a/packages/server/src/tests/utilities/api/backup.ts b/packages/server/src/tests/utilities/api/backup.ts index 8cd1e58a29..7c01b57108 100644 --- a/packages/server/src/tests/utilities/api/backup.ts +++ b/packages/server/src/tests/utilities/api/backup.ts @@ -2,42 +2,38 @@ import { CreateAppBackupResponse, ImportAppBackupResponse, } from "@budibase/types" -import TestConfiguration from "../TestConfiguration" -import { TestAPI } from "./base" +import { Expectations, TestAPI } from "./base" export class BackupAPI extends TestAPI { - constructor(config: TestConfiguration) { - super(config) - } - - exportBasicBackup = async (appId: string) => { - const result = await this.request - .post(`/api/backups/export?appId=${appId}`) - .set(this.config.defaultHeaders()) - .expect("Content-Type", /application\/gzip/) - .expect(200) - return { - body: result.body as Buffer, - headers: result.headers, + exportBasicBackup = async (appId: string, expectations?: Expectations) => { + const exp = { + ...expectations, + headers: { + ...expectations?.headers, + "Content-Type": "application/gzip", + }, } + return await this._post(`/api/backups/export`, { + query: { appId }, + expectations: exp, + }) } - createBackup = async (appId: string) => { - const result = await this.request - .post(`/api/apps/${appId}/backups`) - .set(this.config.defaultHeaders()) - .expect("Content-Type", /json/) - .expect(200) - return result.body as CreateAppBackupResponse + createBackup = async (appId: string, expectations?: Expectations) => { + return await this._post( + `/api/apps/${appId}/backups`, + { expectations } + ) } waitForBackupToComplete = async (appId: string, backupId: string) => { for (let i = 0; i < 10; i++) { await new Promise(resolve => setTimeout(resolve, 1000)) - const result = await this.request - .get(`/api/apps/${appId}/backups/${backupId}/file`) - .set(this.config.defaultHeaders()) - if (result.status === 200) { + const response = await this._requestRaw( + "get", + `/api/apps/${appId}/backups/${backupId}/file` + ) + if (response.status === 200) { return } } @@ -46,13 +42,12 @@ export class BackupAPI extends TestAPI { importBackup = async ( appId: string, - backupId: string + backupId: string, + expectations?: Expectations ): Promise => { - const result = await this.request - .post(`/api/apps/${appId}/backups/${backupId}/import`) - .set(this.config.defaultHeaders()) - .expect("Content-Type", /json/) - .expect(200) - return result.body as ImportAppBackupResponse + return await this._post( + `/api/apps/${appId}/backups/${backupId}/import`, + { expectations } + ) } } diff --git a/packages/server/src/tests/utilities/api/base.ts b/packages/server/src/tests/utilities/api/base.ts index 19fc7ed1f5..b3e5733bb0 100644 --- a/packages/server/src/tests/utilities/api/base.ts +++ b/packages/server/src/tests/utilities/api/base.ts @@ -1,5 +1,5 @@ import TestConfiguration from "../TestConfiguration" -import { SuperTest, Test } from "supertest" +import { SuperTest, Test, Response } from "supertest" import { ReadStream } from "fs" type Headers = Record @@ -22,7 +22,6 @@ function isAttachedFile(file: any): file is AttachedFile { export interface Expectations { status?: number - contentType?: string | RegExp headers?: Record headersNotPresent?: string[] } @@ -71,11 +70,11 @@ export abstract class TestAPI { return await this._request("delete", url, opts) } - protected _request = async ( + protected _requestRaw = async ( method: "get" | "post" | "put" | "patch" | "delete", url: string, opts?: RequestOpts - ): Promise => { + ): Promise => { const { headers = {}, query = {}, @@ -84,7 +83,12 @@ export abstract class TestAPI { files = {}, expectations, } = opts || {} - const { status = 200, contentType = /json/ } = expectations || {} + const { status = 200 } = expectations || {} + const expectHeaders = expectations?.headers || {} + + if (status !== 204 && !expectHeaders["Content-Type"]) { + expectHeaders["Content-Type"] = "application/json" + } let queryParams = [] for (const [key, value] of Object.entries(query)) { @@ -118,16 +122,29 @@ export abstract class TestAPI { request = request.attach(key, value as any) } } - if (contentType && status !== 204) { - request = request.expect("Content-Type", contentType as any) - } if (expectations?.headers) { for (const [key, value] of Object.entries(expectations.headers)) { + if (value === undefined) { + throw new Error( + `Got an undefined expected value for header "${key}", if you want to check for the absence of a header, use headersNotPresent` + ) + } request = request.expect(key, value as any) } } - const response = await request + return await request + } + + protected _request = async ( + method: "get" | "post" | "put" | "patch" | "delete", + url: string, + opts?: RequestOpts + ): Promise => { + const { expectations } = opts || {} + const { status = 200 } = expectations || {} + + const response = await this._requestRaw(method, url, opts) if (response.status !== status) { let message = `Expected status ${status} but got ${response.status}` From 4fbe03bbda0f62d62d0527d4bd90d66ac777c49e Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 29 Feb 2024 15:50:18 +0000 Subject: [PATCH 025/114] Migrate DatasourceAPI. --- .../routes/tests/queries/query.seq.spec.ts | 17 +++-- .../src/integration-test/postgres.spec.ts | 41 ++++++---- .../server/src/tests/utilities/api/base.ts | 15 +++- .../src/tests/utilities/api/datasource.ts | 75 ++++++++----------- packages/types/src/api/web/app/datasource.ts | 4 +- 5 files changed, 77 insertions(+), 75 deletions(-) diff --git a/packages/server/src/api/routes/tests/queries/query.seq.spec.ts b/packages/server/src/api/routes/tests/queries/query.seq.spec.ts index 2bbc8366ea..c5cb188cbc 100644 --- a/packages/server/src/api/routes/tests/queries/query.seq.spec.ts +++ b/packages/server/src/api/routes/tests/queries/query.seq.spec.ts @@ -397,15 +397,16 @@ describe("/queries", () => { }) it("should fail with invalid integration type", async () => { - const response = await config.api.datasource.create( - { - ...basicDatasource().datasource, - source: "INVALID_INTEGRATION" as SourceName, + const datasource: Datasource = { + ...basicDatasource().datasource, + source: "INVALID_INTEGRATION" as SourceName, + } + await config.api.datasource.create(datasource, { + status: 500, + body: { + message: "No datasource implementation found.", }, - { expectStatus: 500, rawResponse: true } - ) - - expect(response.body.message).toBe("No datasource implementation found.") + }) }) }) diff --git a/packages/server/src/integration-test/postgres.spec.ts b/packages/server/src/integration-test/postgres.spec.ts index 0031fe1136..ae6b66b0f5 100644 --- a/packages/server/src/integration-test/postgres.spec.ts +++ b/packages/server/src/integration-test/postgres.spec.ts @@ -1040,28 +1040,37 @@ describe("postgres integrations", () => { describe("POST /api/datasources/verify", () => { it("should be able to verify the connection", async () => { - const response = await config.api.datasource.verify({ - datasource: await databaseTestProviders.postgres.datasource(), - }) - expect(response.status).toBe(200) - expect(response.body.connected).toBe(true) + await config.api.datasource.verify( + { + datasource: await databaseTestProviders.postgres.datasource(), + }, + { + body: { + connected: true, + }, + } + ) }) it("should state an invalid datasource cannot connect", async () => { const dbConfig = await databaseTestProviders.postgres.datasource() - const response = await config.api.datasource.verify({ - datasource: { - ...dbConfig, - config: { - ...dbConfig.config, - password: "wrongpassword", + const response = await config.api.datasource.verify( + { + datasource: { + ...dbConfig, + config: { + ...dbConfig.config, + password: "wrongpassword", + }, }, }, - }) - - expect(response.status).toBe(200) - expect(response.body.connected).toBe(false) - expect(response.body.error).toBeDefined() + { + body: { + connected: false, + error: 'password authentication failed for user "postgres"', + }, + } + ) }) }) diff --git a/packages/server/src/tests/utilities/api/base.ts b/packages/server/src/tests/utilities/api/base.ts index b3e5733bb0..c79686980b 100644 --- a/packages/server/src/tests/utilities/api/base.ts +++ b/packages/server/src/tests/utilities/api/base.ts @@ -3,6 +3,10 @@ import { SuperTest, Test, Response } from "supertest" import { ReadStream } from "fs" type Headers = Record +type SuccessStatus = 200 | 201 | 204 +type ErrorStatus = 400 | 401 | 403 | 404 | 500 | 502 | 503 | 504 +type Status = SuccessStatus | ErrorStatus +type Method = "get" | "post" | "put" | "patch" | "delete" export interface AttachedFile { name: string @@ -21,9 +25,10 @@ function isAttachedFile(file: any): file is AttachedFile { } export interface Expectations { - status?: number + status?: Status headers?: Record headersNotPresent?: string[] + body?: Record } export interface RequestOpts { @@ -137,7 +142,7 @@ export abstract class TestAPI { } protected _request = async ( - method: "get" | "post" | "put" | "patch" | "delete", + method: Method, url: string, opts?: RequestOpts ): Promise => { @@ -180,6 +185,10 @@ export abstract class TestAPI { } } - return response.body as T + if (expectations?.body) { + expect(response.body).toMatchObject(expectations.body) + } + + return response.body } } diff --git a/packages/server/src/tests/utilities/api/datasource.ts b/packages/server/src/tests/utilities/api/datasource.ts index bcd7a71089..06aa9b4e1e 100644 --- a/packages/server/src/tests/utilities/api/datasource.ts +++ b/packages/server/src/tests/utilities/api/datasource.ts @@ -1,63 +1,48 @@ import { - CreateDatasourceRequest, Datasource, VerifyDatasourceRequest, + CreateDatasourceResponse, + UpdateDatasourceResponse, + UpdateDatasourceRequest, } from "@budibase/types" -import TestConfiguration from "../TestConfiguration" -import { TestAPI } from "./base" -import supertest from "supertest" +import { Expectations, TestAPI } from "./base" export class DatasourceAPI extends TestAPI { - constructor(config: TestConfiguration) { - super(config) - } - - create = async ( + create = async ( config: Datasource, - { - expectStatus, - rawResponse, - }: { expectStatus?: number; rawResponse?: B } = {} - ): Promise => { - const body: CreateDatasourceRequest = { - datasource: config, - tablesFilter: [], - } - const result = await this.request - .post(`/api/datasources`) - .send(body) - .set(this.config.defaultHeaders()) - .expect("Content-Type", /json/) - .expect(expectStatus || 200) - if (rawResponse) { - return result as any - } - return result.body.datasource + expectations?: Expectations + ): Promise => { + const response = await this._post( + `/api/datasources`, + { + body: { + datasource: config, + tablesFilter: [], + }, + expectations, + } + ) + return response.datasource } update = async ( - datasource: Datasource, - { expectStatus } = { expectStatus: 200 } + datasource: UpdateDatasourceRequest, + expectations?: Expectations ): Promise => { - const result = await this.request - .put(`/api/datasources/${datasource._id}`) - .send(datasource) - .set(this.config.defaultHeaders()) - .expect("Content-Type", /json/) - .expect(expectStatus) - return result.body.datasource as Datasource + const response = await this._put( + `/api/datasources/${datasource._id}`, + { body: datasource, expectations } + ) + return response.datasource } verify = async ( data: VerifyDatasourceRequest, - { expectStatus } = { expectStatus: 200 } + expectations?: Expectations ) => { - const result = await this.request - .post(`/api/datasources/verify`) - .send(data) - .set(this.config.defaultHeaders()) - .expect("Content-Type", /json/) - .expect(expectStatus) - return result + return await this._post(`/api/datasources/verify`, { + body: data, + expectations, + }) } } diff --git a/packages/types/src/api/web/app/datasource.ts b/packages/types/src/api/web/app/datasource.ts index 4a3d07a952..08dcd00c80 100644 --- a/packages/types/src/api/web/app/datasource.ts +++ b/packages/types/src/api/web/app/datasource.ts @@ -32,9 +32,7 @@ export interface FetchDatasourceInfoResponse { tableNames: string[] } -export interface UpdateDatasourceRequest extends Datasource { - datasource: Datasource -} +export type UpdateDatasourceRequest = Datasource export interface BuildSchemaFromSourceRequest { tablesFilter?: string[] From 1a2a77fc914f27ed25ed6b4663416c80ecf49319 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 29 Feb 2024 15:59:03 +0000 Subject: [PATCH 026/114] Migrate LegacyViewAPI --- packages/server/src/api/routes/tests/row.spec.ts | 14 +++++++------- .../server/src/integration-test/postgres.spec.ts | 2 +- .../server/src/tests/utilities/api/legacyView.ts | 16 ++++------------ 3 files changed, 12 insertions(+), 20 deletions(-) diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts index 726e493b2d..2e1b1c47b9 100644 --- a/packages/server/src/api/routes/tests/row.spec.ts +++ b/packages/server/src/api/routes/tests/row.spec.ts @@ -757,16 +757,16 @@ describe.each([ const row = await config.createRow() const rowUsage = await getRowUsage() - const res = await config.api.legacyView.get(table._id!) - expect(res.body.length).toEqual(1) - expect(res.body[0]._id).toEqual(row._id) + const rows = await config.api.legacyView.get(table._id!) + expect(rows.length).toEqual(1) + expect(rows[0]._id).toEqual(row._id) await assertRowUsage(rowUsage) }) it("should throw an error if view doesn't exist", async () => { const rowUsage = await getRowUsage() - await config.api.legacyView.get("derp", { expectStatus: 404 }) + await config.api.legacyView.get("derp", { status: 404 }) await assertRowUsage(rowUsage) }) @@ -781,9 +781,9 @@ describe.each([ const row = await config.createRow() const rowUsage = await getRowUsage() - const res = await config.api.legacyView.get(view.name) - expect(res.body.length).toEqual(1) - expect(res.body[0]._id).toEqual(row._id) + const rows = await config.api.legacyView.get(view.name) + expect(rows.length).toEqual(1) + expect(rows[0]._id).toEqual(row._id) await assertRowUsage(rowUsage) }) diff --git a/packages/server/src/integration-test/postgres.spec.ts b/packages/server/src/integration-test/postgres.spec.ts index ae6b66b0f5..2680d1a11b 100644 --- a/packages/server/src/integration-test/postgres.spec.ts +++ b/packages/server/src/integration-test/postgres.spec.ts @@ -1054,7 +1054,7 @@ describe("postgres integrations", () => { it("should state an invalid datasource cannot connect", async () => { const dbConfig = await databaseTestProviders.postgres.datasource() - const response = await config.api.datasource.verify( + await config.api.datasource.verify( { datasource: { ...dbConfig, diff --git a/packages/server/src/tests/utilities/api/legacyView.ts b/packages/server/src/tests/utilities/api/legacyView.ts index 63981cec5e..38ef70d62a 100644 --- a/packages/server/src/tests/utilities/api/legacyView.ts +++ b/packages/server/src/tests/utilities/api/legacyView.ts @@ -1,16 +1,8 @@ -import TestConfiguration from "../TestConfiguration" -import { TestAPI } from "./base" +import { Expectations, TestAPI } from "./base" +import { Row } from "@budibase/types" export class LegacyViewAPI extends TestAPI { - constructor(config: TestConfiguration) { - super(config) - } - - get = async (id: string, { expectStatus } = { expectStatus: 200 }) => { - return await this.request - .get(`/api/views/${id}`) - .set(this.config.defaultHeaders()) - .expect("Content-Type", /json/) - .expect(expectStatus) + get = async (id: string, expectations?: Expectations) => { + return await this._get(`/api/views/${id}`, { expectations }) } } From 46bec3c515293337a75601d7ed961af2c9b3784a Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 29 Feb 2024 17:33:36 +0000 Subject: [PATCH 027/114] Migrate PermissionAPI --- .../server/src/api/controllers/permission.ts | 39 ++++++---- .../src/api/routes/tests/permissions.spec.ts | 40 ++++++----- .../server/src/api/routes/tests/row.spec.ts | 8 +-- .../src/tests/utilities/api/permission.ts | 71 ++++++++----------- packages/types/src/api/web/app/permission.ts | 20 +++++- 5 files changed, 99 insertions(+), 79 deletions(-) diff --git a/packages/server/src/api/controllers/permission.ts b/packages/server/src/api/controllers/permission.ts index e2bd6c40e5..e12bf3655d 100644 --- a/packages/server/src/api/controllers/permission.ts +++ b/packages/server/src/api/controllers/permission.ts @@ -7,6 +7,10 @@ import { GetResourcePermsResponse, ResourcePermissionInfo, GetDependantResourcesResponse, + AddPermissionResponse, + AddPermissionRequest, + RemovePermissionRequest, + RemovePermissionResponse, } from "@budibase/types" import { getRoleParams } from "../../db/utils" import { @@ -16,9 +20,9 @@ import { import { removeFromArray } from "../../utilities" import sdk from "../../sdk" -const PermissionUpdateType = { - REMOVE: "remove", - ADD: "add", +enum PermissionUpdateType { + REMOVE = "remove", + ADD = "add", } const SUPPORTED_LEVELS = CURRENTLY_SUPPORTED_LEVELS @@ -39,7 +43,7 @@ async function updatePermissionOnRole( resourceId, level, }: { roleId: string; resourceId: string; level: PermissionLevel }, - updateType: string + updateType: PermissionUpdateType ) { const allowedAction = await sdk.permissions.resourceActionAllowed({ resourceId, @@ -107,11 +111,15 @@ async function updatePermissionOnRole( } const response = await db.bulkDocs(docUpdates) - return response.map((resp: any) => { + return response.map(resp => { const version = docUpdates.find(role => role._id === resp.id)?.version - resp._id = roles.getExternalRoleID(resp.id, version) - delete resp.id - return resp + const _id = roles.getExternalRoleID(resp.id, version) + return { + _id, + rev: resp.rev, + error: resp.error, + reason: resp.reason, + } }) } @@ -189,13 +197,14 @@ export async function getDependantResources( } } -export async function addPermission(ctx: UserCtx) { - ctx.body = await updatePermissionOnRole(ctx.params, PermissionUpdateType.ADD) +export async function addPermission(ctx: UserCtx) { + const params: AddPermissionRequest = ctx.params + ctx.body = await updatePermissionOnRole(params, PermissionUpdateType.ADD) } -export async function removePermission(ctx: UserCtx) { - ctx.body = await updatePermissionOnRole( - ctx.params, - PermissionUpdateType.REMOVE - ) +export async function removePermission( + ctx: UserCtx +) { + const params: RemovePermissionRequest = ctx.params + ctx.body = await updatePermissionOnRole(params, PermissionUpdateType.REMOVE) } diff --git a/packages/server/src/api/routes/tests/permissions.spec.ts b/packages/server/src/api/routes/tests/permissions.spec.ts index 129bc00b44..b9c9ba0359 100644 --- a/packages/server/src/api/routes/tests/permissions.spec.ts +++ b/packages/server/src/api/routes/tests/permissions.spec.ts @@ -45,7 +45,7 @@ describe("/permission", () => { table = (await config.createTable()) as typeof table row = await config.createRow() view = await config.api.viewV2.create({ tableId: table._id }) - perms = await config.api.permission.set({ + perms = await config.api.permission.add({ roleId: STD_ROLE_ID, resourceId: table._id, level: PermissionLevel.READ, @@ -88,13 +88,13 @@ describe("/permission", () => { }) it("should get resource permissions with multiple roles", async () => { - perms = await config.api.permission.set({ + perms = await config.api.permission.add({ roleId: HIGHER_ROLE_ID, resourceId: table._id, level: PermissionLevel.WRITE, }) const res = await config.api.permission.get(table._id) - expect(res.body).toEqual({ + expect(res).toEqual({ permissions: { read: { permissionType: "EXPLICIT", role: STD_ROLE_ID }, write: { permissionType: "EXPLICIT", role: HIGHER_ROLE_ID }, @@ -117,16 +117,19 @@ describe("/permission", () => { level: PermissionLevel.READ, }) - const response = await config.api.permission.set( + await config.api.permission.add( { roleId: STD_ROLE_ID, resourceId: table._id, level: PermissionLevel.EXECUTE, }, - { expectStatus: 403 } - ) - expect(response.message).toEqual( - "You are not allowed to 'read' the resource type 'datasource'" + { + status: 403, + body: { + message: + "You are not allowed to 'read' the resource type 'datasource'", + }, + } ) }) }) @@ -138,9 +141,9 @@ describe("/permission", () => { resourceId: table._id, level: PermissionLevel.READ, }) - expect(res.body[0]._id).toEqual(STD_ROLE_ID) + expect(res[0]._id).toEqual(STD_ROLE_ID) const permsRes = await config.api.permission.get(table._id) - expect(permsRes.body[STD_ROLE_ID]).toBeUndefined() + expect(permsRes.permissions[STD_ROLE_ID]).toBeUndefined() }) it("throw forbidden if the action is not allowed for the resource", async () => { @@ -156,10 +159,13 @@ describe("/permission", () => { resourceId: table._id, level: PermissionLevel.EXECUTE, }, - { expectStatus: 403 } - ) - expect(response.body.message).toEqual( - "You are not allowed to 'read' the resource type 'datasource'" + { + status: 403, + body: { + message: + "You are not allowed to 'read' the resource type 'datasource'", + }, + } ) }) }) @@ -203,7 +209,7 @@ describe("/permission", () => { }) it("should ignore the view permissions if the flag is not on", async () => { - await config.api.permission.set({ + await config.api.permission.add({ roleId: STD_ROLE_ID, resourceId: view.id, level: PermissionLevel.READ, @@ -224,7 +230,7 @@ describe("/permission", () => { it("should use the view permissions if the flag is on", async () => { mocks.licenses.useViewPermissions() - await config.api.permission.set({ + await config.api.permission.add({ roleId: STD_ROLE_ID, resourceId: view.id, level: PermissionLevel.READ, @@ -277,7 +283,7 @@ describe("/permission", () => { const res = await config.api.permission.get(legacyView.name) - expect(res.body).toEqual({ + expect(res).toEqual({ permissions: { read: { permissionType: "BASE", diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts index 2e1b1c47b9..7423157678 100644 --- a/packages/server/src/api/routes/tests/row.spec.ts +++ b/packages/server/src/api/routes/tests/row.spec.ts @@ -1523,7 +1523,7 @@ describe.each([ }) it("allow public users to fetch when permissions are explicit", async () => { - await config.api.permission.set({ + await config.api.permission.add({ roleId: roles.BUILTIN_ROLE_IDS.PUBLIC, level: PermissionLevel.READ, resourceId: viewId, @@ -1538,7 +1538,7 @@ describe.each([ }) it("allow public users to fetch when permissions are inherited", async () => { - await config.api.permission.set({ + await config.api.permission.add({ roleId: roles.BUILTIN_ROLE_IDS.PUBLIC, level: PermissionLevel.READ, resourceId: tableId, @@ -1553,12 +1553,12 @@ describe.each([ }) it("respects inherited permissions, not allowing not public views from public tables", async () => { - await config.api.permission.set({ + await config.api.permission.add({ roleId: roles.BUILTIN_ROLE_IDS.PUBLIC, level: PermissionLevel.READ, resourceId: tableId, }) - await config.api.permission.set({ + await config.api.permission.add({ roleId: roles.BUILTIN_ROLE_IDS.POWER, level: PermissionLevel.READ, resourceId: viewId, diff --git a/packages/server/src/tests/utilities/api/permission.ts b/packages/server/src/tests/utilities/api/permission.ts index ffa89e88f9..986796d9a1 100644 --- a/packages/server/src/tests/utilities/api/permission.ts +++ b/packages/server/src/tests/utilities/api/permission.ts @@ -1,52 +1,39 @@ -import { AnyDocument, PermissionLevel } from "@budibase/types" -import TestConfiguration from "../TestConfiguration" -import { TestAPI } from "./base" +import { + AddPermissionRequest, + AddPermissionResponse, + GetResourcePermsResponse, + RemovePermissionRequest, + RemovePermissionResponse, +} from "@budibase/types" +import { Expectations, TestAPI } from "./base" export class PermissionAPI extends TestAPI { - constructor(config: TestConfiguration) { - super(config) + get = async (resourceId: string, expectations?: Expectations) => { + return await this._get( + `/api/permission/${resourceId}`, + { expectations } + ) } - get = async ( - resourceId: string, - { expectStatus } = { expectStatus: 200 } - ) => { - return this.request - .get(`/api/permission/${resourceId}`) - .set(this.config.defaultHeaders()) - .expect("Content-Type", /json/) - .expect(expectStatus) - } - - set = async ( - { - roleId, - resourceId, - level, - }: { roleId: string; resourceId: string; level: PermissionLevel }, - { expectStatus } = { expectStatus: 200 } - ): Promise => { - const res = await this.request - .post(`/api/permission/${roleId}/${resourceId}/${level}`) - .set(this.config.defaultHeaders()) - .expect("Content-Type", /json/) - .expect(expectStatus) - return res.body + add = async ( + request: AddPermissionRequest, + expectations?: Expectations + ): Promise => { + const { roleId, resourceId, level } = request + return await this._post( + `/api/permission/${roleId}/${resourceId}/${level}`, + { expectations } + ) } revoke = async ( - { - roleId, - resourceId, - level, - }: { roleId: string; resourceId: string; level: PermissionLevel }, - { expectStatus } = { expectStatus: 200 } + request: RemovePermissionRequest, + expectations?: Expectations ) => { - const res = await this.request - .delete(`/api/permission/${roleId}/${resourceId}/${level}`) - .set(this.config.defaultHeaders()) - .expect("Content-Type", /json/) - .expect(expectStatus) - return res + const { roleId, resourceId, level } = request + return await this._delete( + `/api/permission/${roleId}/${resourceId}/${level}`, + { expectations } + ) } } diff --git a/packages/types/src/api/web/app/permission.ts b/packages/types/src/api/web/app/permission.ts index a8ab0e8084..ebe7a8bea3 100644 --- a/packages/types/src/api/web/app/permission.ts +++ b/packages/types/src/api/web/app/permission.ts @@ -1,4 +1,4 @@ -import { PlanType } from "../../../sdk" +import { PermissionLevel, PlanType } from "../../../sdk" export interface ResourcePermissionInfo { role: string @@ -14,3 +14,21 @@ export interface GetResourcePermsResponse { export interface GetDependantResourcesResponse { resourceByType?: Record } + +export interface AddedPermission { + _id: string + rev?: string + error?: string + reason?: string +} + +export type AddPermissionResponse = AddedPermission[] + +export interface AddPermissionRequest { + roleId: string + resourceId: string + level: PermissionLevel +} + +export interface RemovePermissionRequest extends AddPermissionRequest {} +export interface RemovePermissionResponse extends AddPermissionResponse {} From f5f81a5fb01496b627b7207959fb9044436b298d Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 1 Mar 2024 13:59:55 +0000 Subject: [PATCH 028/114] Fix tests. --- packages/server/src/api/routes/tests/role.spec.js | 2 +- packages/server/src/tests/utilities/api/base.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/server/src/api/routes/tests/role.spec.js b/packages/server/src/api/routes/tests/role.spec.js index a653b573b2..4575f9b213 100644 --- a/packages/server/src/api/routes/tests/role.spec.js +++ b/packages/server/src/api/routes/tests/role.spec.js @@ -93,7 +93,7 @@ describe("/roles", () => { it("should be able to get the role with a permission added", async () => { const table = await config.createTable() - await config.api.permission.set({ + await config.api.permission.add({ roleId: BUILTIN_ROLE_IDS.POWER, resourceId: table._id, level: PermissionLevel.READ, diff --git a/packages/server/src/tests/utilities/api/base.ts b/packages/server/src/tests/utilities/api/base.ts index c79686980b..df37c62f00 100644 --- a/packages/server/src/tests/utilities/api/base.ts +++ b/packages/server/src/tests/utilities/api/base.ts @@ -92,7 +92,7 @@ export abstract class TestAPI { const expectHeaders = expectations?.headers || {} if (status !== 204 && !expectHeaders["Content-Type"]) { - expectHeaders["Content-Type"] = "application/json" + expectHeaders["Content-Type"] = /^application\/json/ } let queryParams = [] From 16e9c5ff4e8f0a72c76c4246c704de0b1815e572 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 1 Mar 2024 14:33:18 +0000 Subject: [PATCH 029/114] Migrate QueryAPI --- .../server/src/api/controllers/query/index.ts | 24 ++++---- .../server/src/tests/utilities/api/query.ts | 55 ++++--------------- packages/types/src/api/web/index.ts | 1 + packages/types/src/api/web/query.ts | 20 +++++++ packages/types/src/documents/app/query.ts | 16 ------ 5 files changed, 46 insertions(+), 70 deletions(-) create mode 100644 packages/types/src/api/web/query.ts diff --git a/packages/server/src/api/controllers/query/index.ts b/packages/server/src/api/controllers/query/index.ts index 973718ba48..725de41c9a 100644 --- a/packages/server/src/api/controllers/query/index.ts +++ b/packages/server/src/api/controllers/query/index.ts @@ -17,10 +17,12 @@ import { QueryPreview, QuerySchema, FieldType, - type ExecuteQueryRequest, - type ExecuteQueryResponse, - type Row, + ExecuteQueryRequest, + ExecuteQueryResponse, + Row, QueryParameter, + PreviewQueryRequest, + PreviewQueryResponse, } from "@budibase/types" import { ValidQueryNameRegex, utils as JsonUtils } from "@budibase/shared-core" @@ -134,14 +136,16 @@ function enrichParameters( return requestParameters } -export async function preview(ctx: UserCtx) { +export async function preview( + ctx: UserCtx +) { const { datasource, envVars } = await sdk.datasources.getWithEnvVars( ctx.request.body.datasourceId ) - const query: QueryPreview = ctx.request.body // preview may not have a queryId as it hasn't been saved, but if it does // this stops dynamic variables from calling the same query - const { fields, parameters, queryVerb, transformer, queryId, schema } = query + const { fields, parameters, queryVerb, transformer, queryId, schema } = + ctx.request.body let existingSchema = schema if (queryId && !existingSchema) { @@ -266,9 +270,7 @@ export async function preview(ctx: UserCtx) { }, } - const { rows, keys, info, extra } = (await Runner.run( - inputs - )) as QueryResponse + const { rows, keys, info, extra } = await Runner.run(inputs) const { previewSchema, nestedSchemaFields } = getSchemaFields(rows, keys) // if existing schema, update to include any previous schema keys @@ -281,7 +283,7 @@ export async function preview(ctx: UserCtx) { } // remove configuration before sending event delete datasource.config - await events.query.previewed(datasource, query) + await events.query.previewed(datasource, ctx.request.body) ctx.body = { rows, nestedSchemaFields, @@ -295,7 +297,7 @@ export async function preview(ctx: UserCtx) { } async function execute( - ctx: UserCtx, + ctx: UserCtx, opts: any = { rowsOnly: false, isAutomation: false } ) { const db = context.getAppDB() diff --git a/packages/server/src/tests/utilities/api/query.ts b/packages/server/src/tests/utilities/api/query.ts index b0eac5c8b7..4b6e99fd6c 100644 --- a/packages/server/src/tests/utilities/api/query.ts +++ b/packages/server/src/tests/utilities/api/query.ts @@ -1,60 +1,29 @@ -import TestConfiguration from "../TestConfiguration" import { Query, - QueryPreview, - type ExecuteQueryRequest, - type ExecuteQueryResponse, + ExecuteQueryRequest, + ExecuteQueryResponse, + PreviewQueryRequest, + PreviewQueryResponse, } from "@budibase/types" import { TestAPI } from "./base" export class QueryAPI extends TestAPI { - constructor(config: TestConfiguration) { - super(config) - } - create = async (body: Query): Promise => { - const res = await this.request - .post(`/api/queries`) - .set(this.config.defaultHeaders()) - .send(body) - .expect("Content-Type", /json/) - - if (res.status !== 200) { - throw new Error(JSON.stringify(res.body)) - } - - return res.body as Query + return await this._post(`/api/queries`, { body }) } execute = async ( queryId: string, body?: ExecuteQueryRequest ): Promise => { - const res = await this.request - .post(`/api/v2/queries/${queryId}`) - .set(this.config.defaultHeaders()) - .send(body) - .expect("Content-Type", /json/) - - if (res.status !== 200) { - throw new Error(JSON.stringify(res.body)) - } - - return res.body + return await this._post(`/api/queries/${queryId}`, { + body, + }) } - previewQuery = async (queryPreview: QueryPreview) => { - const res = await this.request - .post(`/api/queries/preview`) - .send(queryPreview) - .set(this.config.defaultHeaders()) - .expect("Content-Type", /json/) - .expect(200) - - if (res.status !== 200) { - throw new Error(JSON.stringify(res.body)) - } - - return res.body + previewQuery = async (queryPreview: PreviewQueryRequest) => { + return await this._post(`/api/queries/preview`, { + body: queryPreview, + }) } } diff --git a/packages/types/src/api/web/index.ts b/packages/types/src/api/web/index.ts index 62d8ce8280..9a688a17a5 100644 --- a/packages/types/src/api/web/index.ts +++ b/packages/types/src/api/web/index.ts @@ -13,3 +13,4 @@ export * from "./searchFilter" export * from "./cookies" export * from "./automation" export * from "./layout" +export * from "./query" diff --git a/packages/types/src/api/web/query.ts b/packages/types/src/api/web/query.ts new file mode 100644 index 0000000000..40f4426d3e --- /dev/null +++ b/packages/types/src/api/web/query.ts @@ -0,0 +1,20 @@ +import { QueryPreview, QuerySchema } from "../../documents" + +export interface PreviewQueryRequest extends QueryPreview {} + +export interface PreviewQueryResponse { + rows: any[] + nestedSchemaFields: { [key: string]: { [key: string]: string | QuerySchema } } + schema: { [key: string]: string | QuerySchema } + info: any + extra: any +} + +export interface ExecuteQueryRequest { + parameters?: { [key: string]: string } + pagination?: any +} + +export interface ExecuteQueryResponse { + data: any[] +} diff --git a/packages/types/src/documents/app/query.ts b/packages/types/src/documents/app/query.ts index b1b0a1d780..3227666bf3 100644 --- a/packages/types/src/documents/app/query.ts +++ b/packages/types/src/documents/app/query.ts @@ -62,22 +62,6 @@ export interface PaginationValues { limit: number | null } -export interface PreviewQueryRequest extends Omit { - parameters: {} - flags?: { - urlName?: boolean - } -} - -export interface ExecuteQueryRequest { - parameters?: { [key: string]: string } - pagination?: any -} - -export interface ExecuteQueryResponse { - data: Row[] -} - export enum HttpMethod { GET = "GET", POST = "POST", From d59b3b628adc36c2959d3059c45ecb091d68a94c Mon Sep 17 00:00:00 2001 From: Peter Clement Date: Fri, 1 Mar 2024 14:59:30 +0000 Subject: [PATCH 030/114] Ensure title value is passed to client app --- packages/server/src/api/controllers/static/index.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/server/src/api/controllers/static/index.ts b/packages/server/src/api/controllers/static/index.ts index 5a3803e6d5..c718d5f704 100644 --- a/packages/server/src/api/controllers/static/index.ts +++ b/packages/server/src/api/controllers/static/index.ts @@ -170,6 +170,7 @@ export const serveApp = async function (ctx: Ctx) { if (!env.isJest()) { const plugins = objectStore.enrichPluginURLs(appInfo.usedPlugins) const { head, html, css } = AppComponent.render({ + title: branding?.platformTitle || `${appInfo.name}`, metaImage: branding?.metaImageUrl || "https://res.cloudinary.com/daog6scxm/image/upload/v1698759482/meta-images/plain-branded-meta-image-coral_ocxmgu.png", From 5d42804020c3e8ea8b7722ebc689f4ef2c0ec92f Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 1 Mar 2024 15:20:07 +0000 Subject: [PATCH 031/114] Migrate some of RowAPI, WIP --- .../server/src/api/controllers/row/index.ts | 2 +- .../server/src/api/routes/tests/row.spec.ts | 75 +++++++++---------- .../server/src/api/routes/tests/table.spec.ts | 15 ++-- .../src/automations/tests/createRow.spec.ts | 2 +- .../src/automations/tests/updateRow.spec.ts | 18 ++--- .../src/db/tests/linkController.spec.ts | 2 +- .../src/integration-test/postgres.spec.ts | 2 +- .../src/sdk/app/rows/tests/internal.spec.ts | 10 ++- .../src/tests/utilities/TestConfiguration.ts | 5 -- .../server/src/tests/utilities/api/base.ts | 5 +- .../server/src/tests/utilities/api/query.ts | 9 ++- .../server/src/tests/utilities/api/row.ts | 33 +++----- 12 files changed, 78 insertions(+), 100 deletions(-) diff --git a/packages/server/src/api/controllers/row/index.ts b/packages/server/src/api/controllers/row/index.ts index ec56919d12..54c294c42b 100644 --- a/packages/server/src/api/controllers/row/index.ts +++ b/packages/server/src/api/controllers/row/index.ts @@ -211,7 +211,7 @@ export async function validate(ctx: Ctx) { } } -export async function fetchEnrichedRow(ctx: any) { +export async function fetchEnrichedRow(ctx: UserCtx) { const tableId = utils.getTableId(ctx) ctx.body = await pickApi(tableId).fetchEnrichedRow(ctx) } diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts index 7423157678..38f28cc138 100644 --- a/packages/server/src/api/routes/tests/row.spec.ts +++ b/packages/server/src/api/routes/tests/row.spec.ts @@ -106,9 +106,6 @@ describe.each([ mocks.licenses.useCloudFree() }) - const loadRow = (id: string, tbl_Id: string, status = 200) => - config.api.row.get(tbl_Id, id, { expectStatus: status }) - const getRowUsage = async () => { const { total } = await config.doInContext(undefined, () => quotas.getCurrentUsageValues(QuotaUsageType.STATIC, StaticQuotaName.ROWS) @@ -235,7 +232,7 @@ describe.each([ const res = await config.api.row.get(tableId, existing._id!) - expect(res.body).toEqual({ + expect(res).toEqual({ ...existing, ...defaultRowFields, }) @@ -265,7 +262,7 @@ describe.each([ await config.createRow() await config.api.row.get(tableId, "1234567", { - expectStatus: 404, + status: 404, }) }) @@ -395,7 +392,7 @@ describe.each([ const createdRow = await config.createRow(row) const id = createdRow._id! - const saved = (await loadRow(id, table._id!)).body + const saved = await config.api.row.get(id, table._id!) expect(saved.stringUndefined).toBe(undefined) expect(saved.stringNull).toBe(null) @@ -476,8 +473,8 @@ describe.each([ ) const row = await config.api.row.get(table._id!, createRowResponse._id!) - expect(row.body.Story).toBeUndefined() - expect(row.body).toEqual({ + expect(row.Story).toBeUndefined() + expect(row).toEqual({ ...defaultRowFields, OrderID: 1111, Country: "Aussy", @@ -524,10 +521,10 @@ describe.each([ expect(row.name).toEqual("Updated Name") expect(row.description).toEqual(existing.description) - const savedRow = await loadRow(row._id!, table._id!) + const savedRow = await config.api.row.get(row._id!, table._id!) - expect(savedRow.body.description).toEqual(existing.description) - expect(savedRow.body.name).toEqual("Updated Name") + expect(savedRow.description).toEqual(existing.description) + expect(savedRow.name).toEqual("Updated Name") await assertRowUsage(rowUsage) }) @@ -582,8 +579,8 @@ describe.each([ }) let getResp = await config.api.row.get(table._id!, row._id!) - expect(getResp.body.user1[0]._id).toEqual(user1._id) - expect(getResp.body.user2[0]._id).toEqual(user2._id) + expect(getResp.user1[0]._id).toEqual(user1._id) + expect(getResp.user2[0]._id).toEqual(user2._id) let patchResp = await config.api.row.patch(table._id!, { _id: row._id!, @@ -595,8 +592,8 @@ describe.each([ expect(patchResp.user2[0]._id).toEqual(user2._id) getResp = await config.api.row.get(table._id!, row._id!) - expect(getResp.body.user1[0]._id).toEqual(user2._id) - expect(getResp.body.user2[0]._id).toEqual(user2._id) + expect(getResp.user1[0]._id).toEqual(user2._id) + expect(getResp.user2[0]._id).toEqual(user2._id) }) it("should be able to update relationships when both columns are same name", async () => { @@ -609,7 +606,7 @@ describe.each([ description: "test", relationship: [row._id], }) - row = (await config.api.row.get(table._id!, row._id!)).body + row = await config.api.row.get(table._id!, row._id!) expect(row.relationship.length).toBe(1) const resp = await config.api.row.patch(table._id!, { _id: row._id!, @@ -685,7 +682,7 @@ describe.each([ const res = await config.api.row.delete(table._id!, [row1, row2]) expect(res.body.length).toEqual(2) - await loadRow(row1._id!, table._id!, 404) + await config.api.row.get(row1._id!, table._id!, { status: 404 }) await assertRowUsage(rowUsage - 2) }) @@ -704,7 +701,7 @@ describe.each([ ]) expect(res.body.length).toEqual(3) - await loadRow(row1._id!, table._id!, 404) + await config.api.row.get(row1._id!, table._id!, { status: 404 }) await assertRowUsage(rowUsage - 3) }) @@ -715,7 +712,7 @@ describe.each([ const res = await config.api.row.delete(table._id!, row1) expect(res.body.id).toEqual(row1._id) - await loadRow(row1._id!, table._id!, 404) + await config.api.row.get(row1._id!, table._id!, { status: 404 }) await assertRowUsage(rowUsage - 1) }) @@ -841,8 +838,8 @@ describe.each([ linkedTable._id!, secondRow._id! ) - expect(resBasic.body.link.length).toBe(1) - expect(resBasic.body.link[0]).toEqual({ + expect(resBasic.link.length).toBe(1) + expect(resBasic.link[0]).toEqual({ _id: firstRow._id, primaryDisplay: firstRow.name, }) @@ -852,10 +849,10 @@ describe.each([ linkedTable._id!, secondRow._id! ) - expect(resEnriched.body.link.length).toBe(1) - expect(resEnriched.body.link[0]._id).toBe(firstRow._id) - expect(resEnriched.body.link[0].name).toBe("Test Contact") - expect(resEnriched.body.link[0].description).toBe("original description") + expect(resEnriched.link.length).toBe(1) + expect(resEnriched.link[0]._id).toBe(firstRow._id) + expect(resEnriched.link[0].name).toBe("Test Contact") + expect(resEnriched.link[0].description).toBe("original description") await assertRowUsage(rowUsage) }) }) @@ -1000,7 +997,7 @@ describe.each([ }) const row = await config.api.row.get(table._id!, newRow._id!) - expect(row.body).toEqual({ + expect(row).toEqual({ name: data.name, surname: data.surname, address: data.address, @@ -1010,9 +1007,9 @@ describe.each([ id: newRow.id, ...defaultRowFields, }) - expect(row.body._viewId).toBeUndefined() - expect(row.body.age).toBeUndefined() - expect(row.body.jobTitle).toBeUndefined() + expect(row._viewId).toBeUndefined() + expect(row.age).toBeUndefined() + expect(row.jobTitle).toBeUndefined() }) }) @@ -1042,7 +1039,7 @@ describe.each([ }) const row = await config.api.row.get(tableId, newRow._id!) - expect(row.body).toEqual({ + expect(row).toEqual({ ...newRow, name: newData.name, address: newData.address, @@ -1051,9 +1048,9 @@ describe.each([ id: newRow.id, ...defaultRowFields, }) - expect(row.body._viewId).toBeUndefined() - expect(row.body.age).toBeUndefined() - expect(row.body.jobTitle).toBeUndefined() + expect(row._viewId).toBeUndefined() + expect(row.age).toBeUndefined() + expect(row.jobTitle).toBeUndefined() }) }) @@ -1076,7 +1073,7 @@ describe.each([ await assertRowUsage(rowUsage - 1) await config.api.row.get(tableId, createdRow._id!, { - expectStatus: 404, + status: 404, }) }) @@ -1102,12 +1099,12 @@ describe.each([ await assertRowUsage(rowUsage - 2) await config.api.row.get(tableId, rows[0]._id!, { - expectStatus: 404, + status: 404, }) await config.api.row.get(tableId, rows[2]._id!, { - expectStatus: 404, + status: 404, }) - await config.api.row.get(tableId, rows[1]._id!, { expectStatus: 200 }) + await config.api.row.get(tableId, rows[1]._id!, { status: 200 }) }) }) @@ -1754,7 +1751,7 @@ describe.each([ } const row = await config.api.row.save(tableId, rowData) - const { body: retrieved } = await config.api.row.get(tableId, row._id!) + const retrieved = await config.api.row.get(tableId, row._id!) expect(retrieved).toEqual({ name: rowData.name, description: rowData.description, @@ -1781,7 +1778,7 @@ describe.each([ } const row = await config.api.row.save(tableId, rowData) - const { body: retrieved } = await config.api.row.get(tableId, row._id!) + const retrieved = await config.api.row.get(tableId, row._id!) expect(retrieved).toEqual({ name: rowData.name, description: rowData.description, diff --git a/packages/server/src/api/routes/tests/table.spec.ts b/packages/server/src/api/routes/tests/table.spec.ts index 4c83237a49..29465145a9 100644 --- a/packages/server/src/api/routes/tests/table.spec.ts +++ b/packages/server/src/api/routes/tests/table.spec.ts @@ -663,8 +663,7 @@ describe("/tables", () => { expect(migratedTable.schema["user column"]).toBeDefined() expect(migratedTable.schema["user relationship"]).not.toBeDefined() - const resp = await config.api.row.get(table._id!, testRow._id!) - const migratedRow = resp.body as Row + const migratedRow = await config.api.row.get(table._id!, testRow._id!) expect(migratedRow["user column"]).toBeDefined() expect(migratedRow["user relationship"]).not.toBeDefined() @@ -716,15 +715,13 @@ describe("/tables", () => { expect(migratedTable.schema["user column"]).toBeDefined() expect(migratedTable.schema["user relationship"]).not.toBeDefined() - const row1Migrated = (await config.api.row.get(table._id!, row1._id!)) - .body as Row + const row1Migrated = await config.api.row.get(table._id!, row1._id!) expect(row1Migrated["user relationship"]).not.toBeDefined() expect(row1Migrated["user column"].map((r: Row) => r._id)).toEqual( expect.arrayContaining([users[0]._id, users[1]._id]) ) - const row2Migrated = (await config.api.row.get(table._id!, row2._id!)) - .body as Row + const row2Migrated = await config.api.row.get(table._id!, row2._id!) expect(row2Migrated["user relationship"]).not.toBeDefined() expect(row2Migrated["user column"].map((r: Row) => r._id)).toEqual( expect.arrayContaining([users[1]._id, users[2]._id]) @@ -773,15 +770,13 @@ describe("/tables", () => { expect(migratedTable.schema["user column"]).toBeDefined() expect(migratedTable.schema["user relationship"]).not.toBeDefined() - const row1Migrated = (await config.api.row.get(table._id!, row1._id!)) - .body as Row + const row1Migrated = await config.api.row.get(table._id!, row1._id!) expect(row1Migrated["user relationship"]).not.toBeDefined() expect(row1Migrated["user column"].map((r: Row) => r._id)).toEqual( expect.arrayContaining([users[0]._id, users[1]._id]) ) - const row2Migrated = (await config.api.row.get(table._id!, row2._id!)) - .body as Row + const row2Migrated = await config.api.row.get(table._id!, row2._id!) expect(row2Migrated["user relationship"]).not.toBeDefined() expect(row2Migrated["user column"].map((r: Row) => r._id)).toEqual([ users[2]._id, diff --git a/packages/server/src/automations/tests/createRow.spec.ts b/packages/server/src/automations/tests/createRow.spec.ts index 0615fcdd97..0098be39a5 100644 --- a/packages/server/src/automations/tests/createRow.spec.ts +++ b/packages/server/src/automations/tests/createRow.spec.ts @@ -24,7 +24,7 @@ describe("test the create row action", () => { expect(res.id).toBeDefined() expect(res.revision).toBeDefined() expect(res.success).toEqual(true) - const gottenRow = await config.getRow(table._id, res.id) + const gottenRow = await config.api.row.get(table._id, res.id) expect(gottenRow.name).toEqual("test") expect(gottenRow.description).toEqual("test") }) diff --git a/packages/server/src/automations/tests/updateRow.spec.ts b/packages/server/src/automations/tests/updateRow.spec.ts index b64c52147d..76823e8a11 100644 --- a/packages/server/src/automations/tests/updateRow.spec.ts +++ b/packages/server/src/automations/tests/updateRow.spec.ts @@ -36,7 +36,7 @@ describe("test the update row action", () => { it("should be able to run the action", async () => { const res = await setup.runStep(setup.actions.UPDATE_ROW.stepId, inputs) expect(res.success).toEqual(true) - const updatedRow = await config.getRow(table._id!, res.id) + const updatedRow = await config.api.row.get(table._id!, res.id) expect(updatedRow.name).toEqual("Updated name") expect(updatedRow.description).not.toEqual("") }) @@ -87,8 +87,8 @@ describe("test the update row action", () => { }) let getResp = await config.api.row.get(table._id!, row._id!) - expect(getResp.body.user1[0]._id).toEqual(user1._id) - expect(getResp.body.user2[0]._id).toEqual(user2._id) + expect(getResp.user1[0]._id).toEqual(user1._id) + expect(getResp.user2[0]._id).toEqual(user2._id) let stepResp = await setup.runStep(setup.actions.UPDATE_ROW.stepId, { rowId: row._id, @@ -103,8 +103,8 @@ describe("test the update row action", () => { expect(stepResp.success).toEqual(true) getResp = await config.api.row.get(table._id!, row._id!) - expect(getResp.body.user1[0]._id).toEqual(user2._id) - expect(getResp.body.user2[0]._id).toEqual(user2._id) + expect(getResp.user1[0]._id).toEqual(user2._id) + expect(getResp.user2[0]._id).toEqual(user2._id) }) it("should overwrite links if those links are not set and we ask it do", async () => { @@ -140,8 +140,8 @@ describe("test the update row action", () => { }) let getResp = await config.api.row.get(table._id!, row._id!) - expect(getResp.body.user1[0]._id).toEqual(user1._id) - expect(getResp.body.user2[0]._id).toEqual(user2._id) + expect(getResp.user1[0]._id).toEqual(user1._id) + expect(getResp.user2[0]._id).toEqual(user2._id) let stepResp = await setup.runStep(setup.actions.UPDATE_ROW.stepId, { rowId: row._id, @@ -163,7 +163,7 @@ describe("test the update row action", () => { expect(stepResp.success).toEqual(true) getResp = await config.api.row.get(table._id!, row._id!) - expect(getResp.body.user1[0]._id).toEqual(user2._id) - expect(getResp.body.user2).toBeUndefined() + expect(getResp.user1[0]._id).toEqual(user2._id) + expect(getResp.user2).toBeUndefined() }) }) diff --git a/packages/server/src/db/tests/linkController.spec.ts b/packages/server/src/db/tests/linkController.spec.ts index ae1922db27..4f41fd3838 100644 --- a/packages/server/src/db/tests/linkController.spec.ts +++ b/packages/server/src/db/tests/linkController.spec.ts @@ -100,7 +100,7 @@ describe("test the link controller", () => { const { _id } = await config.createRow( basicLinkedRow(t1._id!, row._id!, linkField) ) - return config.getRow(t1._id!, _id!) + return config.api.row.get(t1._id!, _id!) } it("should be able to confirm if two table schemas are equal", async () => { diff --git a/packages/server/src/integration-test/postgres.spec.ts b/packages/server/src/integration-test/postgres.spec.ts index 2680d1a11b..7c14bc2b69 100644 --- a/packages/server/src/integration-test/postgres.spec.ts +++ b/packages/server/src/integration-test/postgres.spec.ts @@ -398,7 +398,7 @@ describe("postgres integrations", () => { expect(res.status).toBe(200) expect(res.body).toEqual(updatedRow) - const persistedRow = await config.getRow( + const persistedRow = await config.api.row.get( primaryPostgresTable._id!, row.id ) diff --git a/packages/server/src/sdk/app/rows/tests/internal.spec.ts b/packages/server/src/sdk/app/rows/tests/internal.spec.ts index 3908ef83ed..877bd1e6dc 100644 --- a/packages/server/src/sdk/app/rows/tests/internal.spec.ts +++ b/packages/server/src/sdk/app/rows/tests/internal.spec.ts @@ -98,7 +98,10 @@ describe("sdk >> rows >> internal", () => { }, }) - const persistedRow = await config.getRow(table._id!, response.row._id!) + const persistedRow = await config.api.row.get( + table._id!, + response.row._id! + ) expect(persistedRow).toEqual({ ...row, type: "row", @@ -157,7 +160,10 @@ describe("sdk >> rows >> internal", () => { }, }) - const persistedRow = await config.getRow(table._id!, response.row._id!) + const persistedRow = await config.api.row.get( + table._id!, + response.row._id! + ) expect(persistedRow).toEqual({ ...row, type: "row", diff --git a/packages/server/src/tests/utilities/TestConfiguration.ts b/packages/server/src/tests/utilities/TestConfiguration.ts index 21605b7a5e..35ca2982c0 100644 --- a/packages/server/src/tests/utilities/TestConfiguration.ts +++ b/packages/server/src/tests/utilities/TestConfiguration.ts @@ -712,11 +712,6 @@ export default class TestConfiguration { return this.api.row.save(tableId, config) } - async getRow(tableId: string, rowId: string): Promise { - const res = await this.api.row.get(tableId, rowId) - return res.body - } - async getRows(tableId: string) { if (!tableId && this.table) { tableId = this.table._id! diff --git a/packages/server/src/tests/utilities/api/base.ts b/packages/server/src/tests/utilities/api/base.ts index df37c62f00..d69a254f16 100644 --- a/packages/server/src/tests/utilities/api/base.ts +++ b/packages/server/src/tests/utilities/api/base.ts @@ -3,9 +3,6 @@ import { SuperTest, Test, Response } from "supertest" import { ReadStream } from "fs" type Headers = Record -type SuccessStatus = 200 | 201 | 204 -type ErrorStatus = 400 | 401 | 403 | 404 | 500 | 502 | 503 | 504 -type Status = SuccessStatus | ErrorStatus type Method = "get" | "post" | "put" | "patch" | "delete" export interface AttachedFile { @@ -25,7 +22,7 @@ function isAttachedFile(file: any): file is AttachedFile { } export interface Expectations { - status?: Status + status?: number headers?: Record headersNotPresent?: string[] body?: Record diff --git a/packages/server/src/tests/utilities/api/query.ts b/packages/server/src/tests/utilities/api/query.ts index 4b6e99fd6c..32866314ff 100644 --- a/packages/server/src/tests/utilities/api/query.ts +++ b/packages/server/src/tests/utilities/api/query.ts @@ -16,9 +16,12 @@ export class QueryAPI extends TestAPI { queryId: string, body?: ExecuteQueryRequest ): Promise => { - return await this._post(`/api/queries/${queryId}`, { - body, - }) + return await this._post( + `/api/v2/queries/${queryId}`, + { + body, + } + ) } previewQuery = async (queryPreview: PreviewQueryRequest) => { diff --git a/packages/server/src/tests/utilities/api/row.ts b/packages/server/src/tests/utilities/api/row.ts index 936c906f9f..edcca5fa84 100644 --- a/packages/server/src/tests/utilities/api/row.ts +++ b/packages/server/src/tests/utilities/api/row.ts @@ -9,42 +9,27 @@ import { SearchRowResponse, SearchParams, } from "@budibase/types" -import TestConfiguration from "../TestConfiguration" -import { TestAPI } from "./base" +import { Expectations, TestAPI } from "./base" export class RowAPI extends TestAPI { - constructor(config: TestConfiguration) { - super(config) - } - get = async ( sourceId: string, rowId: string, - { expectStatus } = { expectStatus: 200 } + expectations?: Expectations ) => { - const request = this.request - .get(`/api/${sourceId}/rows/${rowId}`) - .set(this.config.defaultHeaders()) - .expect(expectStatus) - if (expectStatus !== 404) { - request.expect("Content-Type", /json/) - } - return request + return await this._get(`/api/${sourceId}/rows/${rowId}`, { + expectations, + }) } getEnriched = async ( sourceId: string, rowId: string, - { expectStatus } = { expectStatus: 200 } + expectations?: Expectations ) => { - const request = this.request - .get(`/api/${sourceId}/${rowId}/enrich`) - .set(this.config.defaultHeaders()) - .expect(expectStatus) - if (expectStatus !== 404) { - request.expect("Content-Type", /json/) - } - return request + return await this._get(`/api/${sourceId}/${rowId}/enrich`, { + expectations, + }) } save = async ( From d5c6ab86482d983d62199c529c79bb4fbbe82261 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 1 Mar 2024 16:03:52 +0000 Subject: [PATCH 032/114] Fix tests. --- packages/server/src/api/routes/tests/row.spec.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts index 38f28cc138..f1c8eb38a6 100644 --- a/packages/server/src/api/routes/tests/row.spec.ts +++ b/packages/server/src/api/routes/tests/row.spec.ts @@ -392,7 +392,7 @@ describe.each([ const createdRow = await config.createRow(row) const id = createdRow._id! - const saved = await config.api.row.get(id, table._id!) + const saved = await config.api.row.get(table._id!, id) expect(saved.stringUndefined).toBe(undefined) expect(saved.stringNull).toBe(null) @@ -521,7 +521,7 @@ describe.each([ expect(row.name).toEqual("Updated Name") expect(row.description).toEqual(existing.description) - const savedRow = await config.api.row.get(row._id!, table._id!) + const savedRow = await config.api.row.get(table._id!, row._id!) expect(savedRow.description).toEqual(existing.description) expect(savedRow.name).toEqual("Updated Name") @@ -682,7 +682,7 @@ describe.each([ const res = await config.api.row.delete(table._id!, [row1, row2]) expect(res.body.length).toEqual(2) - await config.api.row.get(row1._id!, table._id!, { status: 404 }) + await config.api.row.get(table._id!, row1._id!, { status: 404 }) await assertRowUsage(rowUsage - 2) }) @@ -701,7 +701,7 @@ describe.each([ ]) expect(res.body.length).toEqual(3) - await config.api.row.get(row1._id!, table._id!, { status: 404 }) + await config.api.row.get(table._id!, row1._id!, { status: 404 }) await assertRowUsage(rowUsage - 3) }) @@ -712,7 +712,7 @@ describe.each([ const res = await config.api.row.delete(table._id!, row1) expect(res.body.id).toEqual(row1._id) - await config.api.row.get(row1._id!, table._id!, { status: 404 }) + await config.api.row.get(table._id!, row1._id!, { status: 404 }) await assertRowUsage(rowUsage - 1) }) From a639ba91d38dbcf61ef2d546615f8521ede88f1f Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 1 Mar 2024 16:38:31 +0000 Subject: [PATCH 033/114] Migrate RowAPI.save --- .../server/src/api/routes/tests/row.spec.ts | 2 +- .../server/src/tests/utilities/api/row.ts | 19 +++++-------------- 2 files changed, 6 insertions(+), 15 deletions(-) diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts index f1c8eb38a6..69e268641a 100644 --- a/packages/server/src/api/routes/tests/row.spec.ts +++ b/packages/server/src/api/routes/tests/row.spec.ts @@ -1486,7 +1486,7 @@ describe.each([ email: "joe@joe.com", roles: {}, }, - { expectStatus: 400 } + { status: 400 } ) expect(response.message).toBe("Cannot create new user entry.") }) diff --git a/packages/server/src/tests/utilities/api/row.ts b/packages/server/src/tests/utilities/api/row.ts index edcca5fa84..7e3a0442f6 100644 --- a/packages/server/src/tests/utilities/api/row.ts +++ b/packages/server/src/tests/utilities/api/row.ts @@ -35,21 +35,12 @@ export class RowAPI extends TestAPI { save = async ( tableId: string, row: SaveRowRequest, - { expectStatus } = { expectStatus: 200 } + expectations?: Expectations ): Promise => { - const resp = await this.request - .post(`/api/${tableId}/rows`) - .send(row) - .set(this.config.defaultHeaders()) - .expect("Content-Type", /json/) - if (resp.status !== expectStatus) { - throw new Error( - `Expected status ${expectStatus} but got ${ - resp.status - }, body: ${JSON.stringify(resp.body)}` - ) - } - return resp.body as Row + return await this._post(`/api/${tableId}/rows`, { + body: row, + expectations, + }) } validate = async ( From 58b1c2bca66f9fad4928dab2e365ce1786b5f6f3 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 1 Mar 2024 16:39:57 +0000 Subject: [PATCH 034/114] Migrate RowAPI.validate --- packages/server/src/tests/utilities/api/row.ts | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/packages/server/src/tests/utilities/api/row.ts b/packages/server/src/tests/utilities/api/row.ts index 7e3a0442f6..941ac986e2 100644 --- a/packages/server/src/tests/utilities/api/row.ts +++ b/packages/server/src/tests/utilities/api/row.ts @@ -46,15 +46,15 @@ export class RowAPI extends TestAPI { validate = async ( sourceId: string, row: SaveRowRequest, - { expectStatus } = { expectStatus: 200 } + expectations?: Expectations ): Promise => { - const resp = await this.request - .post(`/api/${sourceId}/rows/validate`) - .send(row) - .set(this.config.defaultHeaders()) - .expect("Content-Type", /json/) - .expect(expectStatus) - return resp.body as ValidateResponse + return await this._post( + `/api/${sourceId}/rows/validate`, + { + body: row, + expectations, + } + ) } patch = async ( From 4efafaeeaf6229cc20b81ce7c5d49f91e30325eb Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 1 Mar 2024 16:40:47 +0000 Subject: [PATCH 035/114] Migrate RowAPI.patch --- .../server/src/tests/utilities/api/row.ts | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/packages/server/src/tests/utilities/api/row.ts b/packages/server/src/tests/utilities/api/row.ts index 941ac986e2..b2f283bd56 100644 --- a/packages/server/src/tests/utilities/api/row.ts +++ b/packages/server/src/tests/utilities/api/row.ts @@ -60,21 +60,12 @@ export class RowAPI extends TestAPI { patch = async ( sourceId: string, row: PatchRowRequest, - { expectStatus } = { expectStatus: 200 } + expectations?: Expectations ): Promise => { - let resp = await this.request - .patch(`/api/${sourceId}/rows`) - .send(row) - .set(this.config.defaultHeaders()) - .expect("Content-Type", /json/) - if (resp.status !== expectStatus) { - throw new Error( - `Expected status ${expectStatus} but got ${ - resp.status - }, body: ${JSON.stringify(resp.body)}` - ) - } - return resp.body as Row + return await this._patch(`/api/${sourceId}/rows`, { + body: row, + expectations, + }) } delete = async ( From a98948150dca6bb114eb10283a8b4fa6ba9a6025 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 1 Mar 2024 17:03:34 +0000 Subject: [PATCH 036/114] Migrate RowAPI.delete --- .../server/src/api/routes/tests/row.spec.ts | 69 ++++++++++--------- .../server/src/tests/utilities/api/row.ts | 28 +++++--- 2 files changed, 57 insertions(+), 40 deletions(-) diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts index 69e268641a..e4e338700e 100644 --- a/packages/server/src/api/routes/tests/row.spec.ts +++ b/packages/server/src/api/routes/tests/row.spec.ts @@ -7,6 +7,7 @@ import { context, InternalTable, roles, tenancy } from "@budibase/backend-core" import { quotas } from "@budibase/pro" import { AutoFieldSubType, + DeleteRow, FieldSchema, FieldType, FieldTypeSubtypes, @@ -540,7 +541,7 @@ describe.each([ tableId: table._id!, name: 1, }, - { expectStatus: 400 } + { status: 400 } ) await assertRowUsage(rowUsage) @@ -629,8 +630,10 @@ describe.each([ const createdRow = await config.createRow() const rowUsage = await getRowUsage() - const res = await config.api.row.delete(table._id!, [createdRow]) - expect(res.body[0]._id).toEqual(createdRow._id) + const res = await config.api.row.deleteMany(table._id!, { + rows: [createdRow], + }) + expect(res[0]._id).toEqual(createdRow._id) await assertRowUsage(rowUsage - 1) }) }) @@ -679,9 +682,11 @@ describe.each([ const row2 = await config.createRow() const rowUsage = await getRowUsage() - const res = await config.api.row.delete(table._id!, [row1, row2]) + const res = await config.api.row.deleteMany(table._id!, { + rows: [row1, row2], + }) - expect(res.body.length).toEqual(2) + expect(res.length).toEqual(2) await config.api.row.get(table._id!, row1._id!, { status: 404 }) await assertRowUsage(rowUsage - 2) }) @@ -694,13 +699,11 @@ describe.each([ ]) const rowUsage = await getRowUsage() - const res = await config.api.row.delete(table._id!, [ - row1, - row2._id, - { _id: row3._id }, - ]) + const res = await config.api.row.deleteMany(table._id!, { + rows: [row1, row2._id!, { _id: row3._id }], + }) - expect(res.body.length).toEqual(3) + expect(res.length).toEqual(3) await config.api.row.get(table._id!, row1._id!, { status: 404 }) await assertRowUsage(rowUsage - 3) }) @@ -709,9 +712,9 @@ describe.each([ const row1 = await config.createRow() const rowUsage = await getRowUsage() - const res = await config.api.row.delete(table._id!, row1) + const res = await config.api.row.delete(table._id!, row1 as DeleteRow) - expect(res.body.id).toEqual(row1._id) + expect(res.id).toEqual(row1._id) await config.api.row.get(table._id!, row1._id!, { status: 404 }) await assertRowUsage(rowUsage - 1) }) @@ -719,24 +722,26 @@ describe.each([ it("Should ignore malformed/invalid delete requests", async () => { const rowUsage = await getRowUsage() - const res = await config.api.row.delete( - table._id!, - { not: "valid" }, - { expectStatus: 400 } - ) - expect(res.body.message).toEqual("Invalid delete rows request") - - const res2 = await config.api.row.delete( - table._id!, - { rows: 123 }, - { expectStatus: 400 } - ) - expect(res2.body.message).toEqual("Invalid delete rows request") - - const res3 = await config.api.row.delete(table._id!, "invalid", { - expectStatus: 400, + await config.api.row.delete(table._id!, { not: "valid" } as any, { + status: 400, + body: { + message: "Invalid delete rows request", + }, + }) + + await config.api.row.delete(table._id!, { rows: 123 } as any, { + status: 400, + body: { + message: "Invalid delete rows request", + }, + }) + + await config.api.row.delete(table._id!, "invalid" as any, { + status: 400, + body: { + message: "Invalid delete rows request", + }, }) - expect(res3.body.message).toEqual("Invalid delete rows request") await assertRowUsage(rowUsage) }) @@ -1068,7 +1073,7 @@ describe.each([ const createdRow = await config.createRow() const rowUsage = await getRowUsage() - await config.api.row.delete(view.id, [createdRow]) + await config.api.row.deleteMany(view.id, { rows: [createdRow] }) await assertRowUsage(rowUsage - 1) @@ -1094,7 +1099,7 @@ describe.each([ ]) const rowUsage = await getRowUsage() - await config.api.row.delete(view.id, [rows[0], rows[2]]) + await config.api.row.deleteMany(view.id, { rows: [rows[0], rows[2]] }) await assertRowUsage(rowUsage - 2) diff --git a/packages/server/src/tests/utilities/api/row.ts b/packages/server/src/tests/utilities/api/row.ts index b2f283bd56..2802576908 100644 --- a/packages/server/src/tests/utilities/api/row.ts +++ b/packages/server/src/tests/utilities/api/row.ts @@ -8,6 +8,9 @@ import { BulkImportResponse, SearchRowResponse, SearchParams, + DeleteRowRequest, + DeleteRows, + DeleteRow, } from "@budibase/types" import { Expectations, TestAPI } from "./base" @@ -70,15 +73,24 @@ export class RowAPI extends TestAPI { delete = async ( sourceId: string, - rows: Row | string | (Row | string)[], - { expectStatus } = { expectStatus: 200 } + row: DeleteRow, + expectations?: Expectations ) => { - return this.request - .delete(`/api/${sourceId}/rows`) - .send(Array.isArray(rows) ? { rows } : rows) - .set(this.config.defaultHeaders()) - .expect("Content-Type", /json/) - .expect(expectStatus) + return await this._delete(`/api/${sourceId}/rows`, { + body: row, + expectations, + }) + } + + deleteMany = async ( + sourceId: string, + body: DeleteRows, + expectations?: Expectations + ) => { + return await this._delete(`/api/${sourceId}/rows`, { + body, + expectations, + }) } fetch = async ( From 376bb9c105976d80676af3900b799645f9fad0ce Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 1 Mar 2024 17:04:35 +0000 Subject: [PATCH 037/114] Migrate RowAPI.fetch --- packages/server/src/tests/utilities/api/row.ts | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/packages/server/src/tests/utilities/api/row.ts b/packages/server/src/tests/utilities/api/row.ts index 2802576908..0ac05ee653 100644 --- a/packages/server/src/tests/utilities/api/row.ts +++ b/packages/server/src/tests/utilities/api/row.ts @@ -95,14 +95,11 @@ export class RowAPI extends TestAPI { fetch = async ( sourceId: string, - { expectStatus } = { expectStatus: 200 } + expectations?: Expectations ): Promise => { - const request = this.request - .get(`/api/${sourceId}/rows`) - .set(this.config.defaultHeaders()) - .expect(expectStatus) - - return (await request).body + return await this._get(`/api/${sourceId}/rows`, { + expectations, + }) } exportRows = async ( From a4e212c0d8eda3e78e23833f8f0c89bda9b8c079 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 1 Mar 2024 17:10:49 +0000 Subject: [PATCH 038/114] Migrate RowAPI.exportRows --- .../server/src/api/routes/tests/row.spec.ts | 4 ++-- .../server/src/tests/utilities/api/row.ts | 20 +++++++++++-------- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts index e4e338700e..091aec5e1f 100644 --- a/packages/server/src/api/routes/tests/row.spec.ts +++ b/packages/server/src/api/routes/tests/row.spec.ts @@ -905,7 +905,7 @@ describe.each([ const res = await config.api.row.exportRows(table._id!, { rows: [existing._id!], }) - const results = JSON.parse(res.text) + const results = JSON.parse(res) expect(results.length).toEqual(1) const row = results[0] @@ -924,7 +924,7 @@ describe.each([ rows: [existing._id!], columns: ["_id"], }) - const results = JSON.parse(res.text) + const results = JSON.parse(res) expect(results.length).toEqual(1) const row = results[0] diff --git a/packages/server/src/tests/utilities/api/row.ts b/packages/server/src/tests/utilities/api/row.ts index 0ac05ee653..7f28e7292a 100644 --- a/packages/server/src/tests/utilities/api/row.ts +++ b/packages/server/src/tests/utilities/api/row.ts @@ -11,6 +11,7 @@ import { DeleteRowRequest, DeleteRows, DeleteRow, + ExportRowsResponse, } from "@budibase/types" import { Expectations, TestAPI } from "./base" @@ -105,15 +106,18 @@ export class RowAPI extends TestAPI { exportRows = async ( tableId: string, body: ExportRowsRequest, - { expectStatus } = { expectStatus: 200 } + expectations?: Expectations ) => { - const request = this.request - .post(`/api/${tableId}/rows/exportRows?format=json`) - .set(this.config.defaultHeaders()) - .send(body) - .expect("Content-Type", /json/) - .expect(expectStatus) - return request + const response = await this._requestRaw( + "post", + `/api/${tableId}/rows/exportRows`, + { + body, + query: { format: "json" }, + expectations, + } + ) + return response.text } bulkImport = async ( From 02ac338c3f19b3852c68e741a536087ddca223a4 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 1 Mar 2024 17:11:19 +0000 Subject: [PATCH 039/114] deleteMany -> bulkDelete --- packages/server/src/api/routes/tests/row.spec.ts | 10 +++++----- packages/server/src/tests/utilities/api/row.ts | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts index 091aec5e1f..027a7fff58 100644 --- a/packages/server/src/api/routes/tests/row.spec.ts +++ b/packages/server/src/api/routes/tests/row.spec.ts @@ -630,7 +630,7 @@ describe.each([ const createdRow = await config.createRow() const rowUsage = await getRowUsage() - const res = await config.api.row.deleteMany(table._id!, { + const res = await config.api.row.bulkDelete(table._id!, { rows: [createdRow], }) expect(res[0]._id).toEqual(createdRow._id) @@ -682,7 +682,7 @@ describe.each([ const row2 = await config.createRow() const rowUsage = await getRowUsage() - const res = await config.api.row.deleteMany(table._id!, { + const res = await config.api.row.bulkDelete(table._id!, { rows: [row1, row2], }) @@ -699,7 +699,7 @@ describe.each([ ]) const rowUsage = await getRowUsage() - const res = await config.api.row.deleteMany(table._id!, { + const res = await config.api.row.bulkDelete(table._id!, { rows: [row1, row2._id!, { _id: row3._id }], }) @@ -1073,7 +1073,7 @@ describe.each([ const createdRow = await config.createRow() const rowUsage = await getRowUsage() - await config.api.row.deleteMany(view.id, { rows: [createdRow] }) + await config.api.row.bulkDelete(view.id, { rows: [createdRow] }) await assertRowUsage(rowUsage - 1) @@ -1099,7 +1099,7 @@ describe.each([ ]) const rowUsage = await getRowUsage() - await config.api.row.deleteMany(view.id, { rows: [rows[0], rows[2]] }) + await config.api.row.bulkDelete(view.id, { rows: [rows[0], rows[2]] }) await assertRowUsage(rowUsage - 2) diff --git a/packages/server/src/tests/utilities/api/row.ts b/packages/server/src/tests/utilities/api/row.ts index 7f28e7292a..af743e3c9b 100644 --- a/packages/server/src/tests/utilities/api/row.ts +++ b/packages/server/src/tests/utilities/api/row.ts @@ -83,7 +83,7 @@ export class RowAPI extends TestAPI { }) } - deleteMany = async ( + bulkDelete = async ( sourceId: string, body: DeleteRows, expectations?: Expectations From c0907d37ef890a8a117f525198f5baa8ab592aaf Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 1 Mar 2024 17:12:46 +0000 Subject: [PATCH 040/114] Migrate RowAPI.bulkImport --- packages/server/src/tests/utilities/api/row.ts | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/packages/server/src/tests/utilities/api/row.ts b/packages/server/src/tests/utilities/api/row.ts index af743e3c9b..558ef100a3 100644 --- a/packages/server/src/tests/utilities/api/row.ts +++ b/packages/server/src/tests/utilities/api/row.ts @@ -123,14 +123,12 @@ export class RowAPI extends TestAPI { bulkImport = async ( tableId: string, body: BulkImportRequest, - { expectStatus } = { expectStatus: 200 } + expectations?: Expectations ): Promise => { - let request = this.request - .post(`/api/tables/${tableId}/import`) - .send(body) - .set(this.config.defaultHeaders()) - .expect(expectStatus) - return (await request).body + return await this._post(`/api/${tableId}/rows/import`, { + body, + expectations, + }) } search = async ( From f91db6d9850e146fe0fdd1170b5eed3c2fd79187 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 1 Mar 2024 17:13:46 +0000 Subject: [PATCH 041/114] Migrate RowAPI.search --- packages/server/src/tests/utilities/api/row.ts | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/packages/server/src/tests/utilities/api/row.ts b/packages/server/src/tests/utilities/api/row.ts index 558ef100a3..58ddb7c049 100644 --- a/packages/server/src/tests/utilities/api/row.ts +++ b/packages/server/src/tests/utilities/api/row.ts @@ -134,14 +134,11 @@ export class RowAPI extends TestAPI { search = async ( sourceId: string, params?: SearchParams, - { expectStatus } = { expectStatus: 200 } + expectations?: Expectations ): Promise => { - const request = this.request - .post(`/api/${sourceId}/search`) - .send(params) - .set(this.config.defaultHeaders()) - .expect(expectStatus) - - return (await request).body + return await this._post(`/api/${sourceId}/search`, { + body: params, + expectations, + }) } } From 149d2c0b29ab594fdd2b1d3a89ab4f927c0b1a01 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 1 Mar 2024 17:14:30 +0000 Subject: [PATCH 042/114] Migrate ScreenAPI --- .../server/src/tests/utilities/api/screen.ts | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/packages/server/src/tests/utilities/api/screen.ts b/packages/server/src/tests/utilities/api/screen.ts index 9245ffe4ba..c8d3e647d8 100644 --- a/packages/server/src/tests/utilities/api/screen.ts +++ b/packages/server/src/tests/utilities/api/screen.ts @@ -1,18 +1,8 @@ -import TestConfiguration from "../TestConfiguration" import { Screen } from "@budibase/types" -import { TestAPI } from "./base" +import { Expectations, TestAPI } from "./base" export class ScreenAPI extends TestAPI { - constructor(config: TestConfiguration) { - super(config) - } - - list = async (): Promise => { - const res = await this.request - .get(`/api/screens`) - .set(this.config.defaultHeaders()) - .expect("Content-Type", /json/) - .expect(200) - return res.body as Screen[] + list = async (expectations?: Expectations): Promise => { + return await this._get(`/api/screens`, { expectations }) } } From 7fa5dbeec9ab3a39d8f609a263b0484fb6d68096 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 1 Mar 2024 17:17:38 +0000 Subject: [PATCH 043/114] Migrate UserAPI --- .../server/src/api/routes/tests/user.spec.ts | 2 +- .../server/src/tests/utilities/api/user.ts | 157 +++++------------- 2 files changed, 42 insertions(+), 117 deletions(-) diff --git a/packages/server/src/api/routes/tests/user.spec.ts b/packages/server/src/api/routes/tests/user.spec.ts index 076ee064dc..ff8c0d54b3 100644 --- a/packages/server/src/api/routes/tests/user.spec.ts +++ b/packages/server/src/api/routes/tests/user.spec.ts @@ -90,7 +90,7 @@ describe("/users", () => { }) await config.api.user.update( { ...user, roleId: roles.BUILTIN_ROLE_IDS.POWER }, - { expectStatus: 409 } + { status: 409 } ) }) }) diff --git a/packages/server/src/tests/utilities/api/user.ts b/packages/server/src/tests/utilities/api/user.ts index 2ed23c0461..bb3eae0542 100644 --- a/packages/server/src/tests/utilities/api/user.ts +++ b/packages/server/src/tests/utilities/api/user.ts @@ -4,154 +4,79 @@ import { Flags, UserMetadata, } from "@budibase/types" -import TestConfiguration from "../TestConfiguration" -import { TestAPI } from "./base" +import { Expectations, TestAPI } from "./base" import { DocumentInsertResponse } from "@budibase/nano" export class UserAPI extends TestAPI { - constructor(config: TestConfiguration) { - super(config) - } - fetch = async ( - { expectStatus } = { expectStatus: 200 } + expectations?: Expectations ): Promise => { - const res = await this.request - .get(`/api/users/metadata`) - .set(this.config.defaultHeaders()) - .expect("Content-Type", /json/) - - if (res.status !== expectStatus) { - throw new Error( - `Expected status ${expectStatus} but got ${ - res.status - } with body ${JSON.stringify(res.body)}` - ) - } - - return res.body + return await this._get("/api/users/metadata", { + expectations, + }) } find = async ( id: string, - { expectStatus } = { expectStatus: 200 } + expectations?: Expectations ): Promise => { - const res = await this.request - .get(`/api/users/metadata/${id}`) - .set(this.config.defaultHeaders()) - .expect("Content-Type", /json/) - - if (res.status !== expectStatus) { - throw new Error( - `Expected status ${expectStatus} but got ${ - res.status - } with body ${JSON.stringify(res.body)}` - ) - } - - return res.body + return await this._get( + `/api/users/metadata/${id}`, + { + expectations, + } + ) } update = async ( user: UserMetadata, - { expectStatus } = { expectStatus: 200 } + expectations?: Expectations ): Promise => { - const res = await this.request - .put(`/api/users/metadata`) - .set(this.config.defaultHeaders()) - .send(user) - .expect("Content-Type", /json/) - - if (res.status !== expectStatus) { - throw new Error( - `Expected status ${expectStatus} but got ${ - res.status - } with body ${JSON.stringify(res.body)}` - ) - } - - return res.body as DocumentInsertResponse + return await this._put("/api/users/metadata", { + body: user, + expectations, + }) } updateSelf = async ( user: UserMetadata, - { expectStatus } = { expectStatus: 200 } + expectations?: Expectations ): Promise => { - const res = await this.request - .post(`/api/users/metadata/self`) - .set(this.config.defaultHeaders()) - .send(user) - .expect("Content-Type", /json/) - - if (res.status !== expectStatus) { - throw new Error( - `Expected status ${expectStatus} but got ${ - res.status - } with body ${JSON.stringify(res.body)}` - ) - } - - return res.body as DocumentInsertResponse + return await this._post( + "/api/users/metadata/self", + { + body: user, + expectations, + } + ) } destroy = async ( id: string, - { expectStatus } = { expectStatus: 200 } + expectations?: Expectations ): Promise<{ message: string }> => { - const res = await this.request - .delete(`/api/users/metadata/${id}`) - .set(this.config.defaultHeaders()) - .expect("Content-Type", /json/) - - if (res.status !== expectStatus) { - throw new Error( - `Expected status ${expectStatus} but got ${ - res.status - } with body ${JSON.stringify(res.body)}` - ) - } - - return res.body as { message: string } + return await this._delete<{ message: string }>( + `/api/users/metadata/${id}`, + { + expectations, + } + ) } setFlag = async ( flag: string, value: any, - { expectStatus } = { expectStatus: 200 } + expectations?: Expectations ): Promise<{ message: string }> => { - const res = await this.request - .post(`/api/users/flags`) - .set(this.config.defaultHeaders()) - .send({ flag, value }) - .expect("Content-Type", /json/) - - if (res.status !== expectStatus) { - throw new Error( - `Expected status ${expectStatus} but got ${ - res.status - } with body ${JSON.stringify(res.body)}` - ) - } - - return res.body as { message: string } + return await this._post<{ message: string }>(`/api/users/flags`, { + body: { flag, value }, + expectations, + }) } - getFlags = async ( - { expectStatus } = { expectStatus: 200 } - ): Promise => { - const res = await this.request - .get(`/api/users/flags`) - .set(this.config.defaultHeaders()) - .expect("Content-Type", /json/) - - if (res.status !== expectStatus) { - throw new Error( - `Expected status ${expectStatus} but got ${ - res.status - } with body ${JSON.stringify(res.body)}` - ) - } - - return res.body as Flags + getFlags = async (expectations?: Expectations): Promise => { + return await this._get(`/api/users/flags`, { + expectations, + }) } } From 37a10857df1d979e15925c09797ed3885f98bd7a Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 1 Mar 2024 17:35:51 +0000 Subject: [PATCH 044/114] Migrate ViewV2API --- .../src/api/routes/tests/permissions.spec.ts | 18 ++-- .../server/src/api/routes/tests/row.spec.ts | 58 ++++++------ .../src/api/routes/tests/viewV2.spec.ts | 18 ++-- .../server/src/tests/utilities/api/base.ts | 7 +- .../server/src/tests/utilities/api/viewV2.ts | 89 +++++++++---------- packages/types/src/api/web/app/rows.ts | 6 +- 6 files changed, 90 insertions(+), 106 deletions(-) diff --git a/packages/server/src/api/routes/tests/permissions.spec.ts b/packages/server/src/api/routes/tests/permissions.spec.ts index b9c9ba0359..edaa73fd1a 100644 --- a/packages/server/src/api/routes/tests/permissions.spec.ts +++ b/packages/server/src/api/routes/tests/permissions.spec.ts @@ -187,9 +187,7 @@ describe("/permission", () => { // replicate changes before checking permissions await config.publish() - const res = await config.api.viewV2.search(view.id, undefined, { - usePublicUser: true, - }) + const res = await config.api.viewV2.publicSearch(view.id) expect(res.body.rows[0]._id).toEqual(row._id) }) @@ -202,10 +200,7 @@ describe("/permission", () => { // replicate changes before checking permissions await config.publish() - await config.api.viewV2.search(view.id, undefined, { - expectStatus: 403, - usePublicUser: true, - }) + await config.api.viewV2.publicSearch(view.id, undefined, { status: 403 }) }) it("should ignore the view permissions if the flag is not on", async () => { @@ -222,9 +217,8 @@ describe("/permission", () => { // replicate changes before checking permissions await config.publish() - await config.api.viewV2.search(view.id, undefined, { - expectStatus: 403, - usePublicUser: true, + await config.api.viewV2.publicSearch(view.id, undefined, { + status: 403, }) }) @@ -243,9 +237,7 @@ describe("/permission", () => { // replicate changes before checking permissions await config.publish() - const res = await config.api.viewV2.search(view.id, undefined, { - usePublicUser: true, - }) + const res = await config.api.viewV2.publicSearch(view.id) expect(res.body.rows[0]._id).toEqual(row._id) }) diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts index 027a7fff58..c02159bb42 100644 --- a/packages/server/src/api/routes/tests/row.spec.ts +++ b/packages/server/src/api/routes/tests/row.spec.ts @@ -1156,8 +1156,8 @@ describe.each([ const createViewResponse = await config.createView() const response = await config.api.viewV2.search(createViewResponse.id) - expect(response.body.rows).toHaveLength(10) - expect(response.body).toEqual({ + expect(response.rows).toHaveLength(10) + expect(response).toEqual({ rows: expect.arrayContaining( rows.map(r => ({ _viewId: createViewResponse.id, @@ -1208,8 +1208,8 @@ describe.each([ const response = await config.api.viewV2.search(createViewResponse.id) - expect(response.body.rows).toHaveLength(5) - expect(response.body).toEqual({ + expect(response.rows).toHaveLength(5) + expect(response).toEqual({ rows: expect.arrayContaining( expectedRows.map(r => ({ _viewId: createViewResponse.id, @@ -1330,8 +1330,8 @@ describe.each([ createViewResponse.id ) - expect(response.body.rows).toHaveLength(4) - expect(response.body.rows).toEqual( + expect(response.rows).toHaveLength(4) + expect(response.rows).toEqual( expected.map(name => expect.objectContaining({ name })) ) } @@ -1359,8 +1359,8 @@ describe.each([ } ) - expect(response.body.rows).toHaveLength(4) - expect(response.body.rows).toEqual( + expect(response.rows).toHaveLength(4) + expect(response.rows).toEqual( expected.map(name => expect.objectContaining({ name })) ) } @@ -1384,8 +1384,8 @@ describe.each([ }) const response = await config.api.viewV2.search(view.id) - expect(response.body.rows).toHaveLength(10) - expect(response.body.rows).toEqual( + expect(response.rows).toHaveLength(10) + expect(response.rows).toEqual( expect.arrayContaining( rows.map(r => ({ ...(isInternal @@ -1404,7 +1404,7 @@ describe.each([ const createViewResponse = await config.createView() const response = await config.api.viewV2.search(createViewResponse.id) - expect(response.body.rows).toHaveLength(0) + expect(response.rows).toHaveLength(0) }) it("respects the limit parameter", async () => { @@ -1419,7 +1419,7 @@ describe.each([ query: {}, }) - expect(response.body.rows).toHaveLength(limit) + expect(response.rows).toHaveLength(limit) }) it("can handle pagination", async () => { @@ -1428,7 +1428,7 @@ describe.each([ const createViewResponse = await config.createView() const allRows = (await config.api.viewV2.search(createViewResponse.id)) - .body.rows + .rows const firstPageResponse = await config.api.viewV2.search( createViewResponse.id, @@ -1438,7 +1438,7 @@ describe.each([ query: {}, } ) - expect(firstPageResponse.body).toEqual({ + expect(firstPageResponse).toEqual({ rows: expect.arrayContaining(allRows.slice(0, 4)), totalRows: isInternal ? 10 : undefined, hasNextPage: true, @@ -1450,12 +1450,12 @@ describe.each([ { paginate: true, limit: 4, - bookmark: firstPageResponse.body.bookmark, + bookmark: firstPageResponse.bookmark, query: {}, } ) - expect(secondPageResponse.body).toEqual({ + expect(secondPageResponse).toEqual({ rows: expect.arrayContaining(allRows.slice(4, 8)), totalRows: isInternal ? 10 : undefined, hasNextPage: true, @@ -1467,11 +1467,11 @@ describe.each([ { paginate: true, limit: 4, - bookmark: secondPageResponse.body.bookmark, + bookmark: secondPageResponse.bookmark, query: {}, } ) - expect(lastPageResponse.body).toEqual({ + expect(lastPageResponse).toEqual({ rows: expect.arrayContaining(allRows.slice(8)), totalRows: isInternal ? 10 : undefined, hasNextPage: false, @@ -1518,9 +1518,8 @@ describe.each([ it("does not allow public users to fetch by default", async () => { await config.publish() - await config.api.viewV2.search(viewId, undefined, { - expectStatus: 403, - usePublicUser: true, + await config.api.viewV2.publicSearch(viewId, undefined, { + status: 403, }) }) @@ -1532,11 +1531,9 @@ describe.each([ }) await config.publish() - const response = await config.api.viewV2.search(viewId, undefined, { - usePublicUser: true, - }) + const response = await config.api.viewV2.publicSearch(viewId) - expect(response.body.rows).toHaveLength(10) + expect(response.rows).toHaveLength(10) }) it("allow public users to fetch when permissions are inherited", async () => { @@ -1547,11 +1544,9 @@ describe.each([ }) await config.publish() - const response = await config.api.viewV2.search(viewId, undefined, { - usePublicUser: true, - }) + const response = await config.api.viewV2.publicSearch(viewId) - expect(response.body.rows).toHaveLength(10) + expect(response.rows).toHaveLength(10) }) it("respects inherited permissions, not allowing not public views from public tables", async () => { @@ -1567,9 +1562,8 @@ describe.each([ }) await config.publish() - await config.api.viewV2.search(viewId, undefined, { - usePublicUser: true, - expectStatus: 403, + await config.api.viewV2.publicSearch(viewId, undefined, { + status: 403, }) }) }) diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts index b03a73ddda..5198e63338 100644 --- a/packages/server/src/api/routes/tests/viewV2.spec.ts +++ b/packages/server/src/api/routes/tests/viewV2.spec.ts @@ -177,7 +177,7 @@ describe.each([ } await config.api.viewV2.create(newView, { - expectStatus: 201, + status: 201, }) }) }) @@ -275,7 +275,7 @@ describe.each([ const tableId = table._id! await config.api.viewV2.update( { ...view, id: generator.guid() }, - { expectStatus: 404 } + { status: 404 } ) expect(await config.api.table.get(tableId)).toEqual( @@ -304,7 +304,7 @@ describe.each([ }, ], }, - { expectStatus: 404 } + { status: 404 } ) expect(await config.api.table.get(tableId)).toEqual( @@ -326,12 +326,10 @@ describe.each([ ...viewV1, }, { - expectStatus: 400, - handleResponse: r => { - expect(r.body).toEqual({ - message: "Only views V2 can be updated", - status: 400, - }) + status: 400, + body: { + message: "Only views V2 can be updated", + status: 400, }, } ) @@ -403,7 +401,7 @@ describe.each([ } as Record, }, { - expectStatus: 200, + status: 200, } ) }) diff --git a/packages/server/src/tests/utilities/api/base.ts b/packages/server/src/tests/utilities/api/base.ts index d69a254f16..3f534fba86 100644 --- a/packages/server/src/tests/utilities/api/base.ts +++ b/packages/server/src/tests/utilities/api/base.ts @@ -38,6 +38,7 @@ export interface RequestOpts { Buffer | ReadStream | string | AttachedFile | undefined > expectations?: Expectations + publicUser?: boolean } export abstract class TestAPI { @@ -84,6 +85,7 @@ export abstract class TestAPI { fields = {}, files = {}, expectations, + publicUser = false, } = opts || {} const { status = 200 } = expectations || {} const expectHeaders = expectations?.headers || {} @@ -102,8 +104,11 @@ export abstract class TestAPI { url += `?${queryParams.join("&")}` } + const headersFn = publicUser + ? this.config.publicHeaders.bind(this.config) + : this.config.defaultHeaders.bind(this.config) let request = this.request[method](url).set( - this.config.defaultHeaders({ + headersFn({ "x-budibase-include-stacktrace": "true", }) ) diff --git a/packages/server/src/tests/utilities/api/viewV2.ts b/packages/server/src/tests/utilities/api/viewV2.ts index 92a6d394bf..dd49360aa8 100644 --- a/packages/server/src/tests/utilities/api/viewV2.ts +++ b/packages/server/src/tests/utilities/api/viewV2.ts @@ -3,21 +3,16 @@ import { UpdateViewRequest, ViewV2, SearchViewRowRequest, + PaginatedSearchRowResponse, } from "@budibase/types" -import TestConfiguration from "../TestConfiguration" -import { TestAPI } from "./base" +import { Expectations, TestAPI } from "./base" import { generator } from "@budibase/backend-core/tests" -import { Response } from "superagent" import sdk from "../../../sdk" export class ViewV2API extends TestAPI { - constructor(config: TestConfiguration) { - super(config) - } - create = async ( viewData?: Partial, - { expectStatus } = { expectStatus: 201 } + expectations?: Expectations ): Promise => { let tableId = viewData?.tableId if (!tableId && !this.config.table) { @@ -30,43 +25,27 @@ export class ViewV2API extends TestAPI { name: generator.guid(), ...viewData, } - const result = await this.request - .post(`/api/v2/views`) - .send(view) - .set(this.config.defaultHeaders()) - .expect("Content-Type", /json/) - .expect(expectStatus) - return result.body.data as ViewV2 + + const resp = await this._post<{ data: ViewV2 }>("/api/v2/views", { + body: view, + expectations, + }) + return resp.data } update = async ( view: UpdateViewRequest, - { - expectStatus, - handleResponse, - }: { - expectStatus: number - handleResponse?: (response: Response) => void - } = { expectStatus: 200 } + expectations?: Expectations ): Promise => { - const result = await this.request - .put(`/api/v2/views/${view.id}`) - .send(view) - .set(this.config.defaultHeaders()) - .expect("Content-Type", /json/) - .expect(expectStatus) - - if (handleResponse) { - handleResponse(result) - } - return result.body.data as ViewV2 + const resp = await this._put<{ data: ViewV2 }>(`/api/v2/views/${view.id}`, { + body: view, + expectations, + }) + return resp.data } - delete = async (viewId: string, { expectStatus } = { expectStatus: 204 }) => { - return this.request - .delete(`/api/v2/views/${viewId}`) - .set(this.config.defaultHeaders()) - .expect(expectStatus) + delete = async (viewId: string, expectations?: Expectations) => { + return await this._delete(`/api/v2/views/${viewId}`, { expectations }) } get = async (viewId: string) => { @@ -78,17 +57,29 @@ export class ViewV2API extends TestAPI { search = async ( viewId: string, params?: SearchViewRowRequest, - { expectStatus = 200, usePublicUser = false } = {} + expectations?: Expectations ) => { - return this.request - .post(`/api/v2/views/${viewId}/search`) - .send(params) - .set( - usePublicUser - ? this.config.publicHeaders() - : this.config.defaultHeaders() - ) - .expect("Content-Type", /json/) - .expect(expectStatus) + return await this._post( + `/api/v2/views/${viewId}/search`, + { + body: params, + expectations, + } + ) + } + + publicSearch = async ( + viewId: string, + params?: SearchViewRowRequest, + expectations?: Expectations + ) => { + return await this._post( + `/api/v2/views/${viewId}/search`, + { + body: params, + expectations, + publicUser: true, + } + ) } } diff --git a/packages/types/src/api/web/app/rows.ts b/packages/types/src/api/web/app/rows.ts index 14e28e4a01..0a43182dfd 100644 --- a/packages/types/src/api/web/app/rows.ts +++ b/packages/types/src/api/web/app/rows.ts @@ -1,6 +1,6 @@ import { SearchFilters, SearchParams } from "../../../sdk" import { Row } from "../../../documents" -import { SortOrder } from "../../../api" +import { PaginationResponse, SortOrder } from "../../../api" import { ReadStream } from "fs" export interface SaveRowRequest extends Row {} @@ -31,6 +31,10 @@ export interface SearchRowResponse { rows: any[] } +export interface PaginatedSearchRowResponse + extends SearchRowResponse, + PaginationResponse {} + export interface ExportRowsRequest { rows: string[] columns?: string[] From 90f981724d4172dcaac552e3912a663405309f5b Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 1 Mar 2024 17:38:21 +0000 Subject: [PATCH 045/114] Fix typing. --- packages/server/src/api/routes/tests/permissions.spec.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/server/src/api/routes/tests/permissions.spec.ts b/packages/server/src/api/routes/tests/permissions.spec.ts index edaa73fd1a..1eabf6edbb 100644 --- a/packages/server/src/api/routes/tests/permissions.spec.ts +++ b/packages/server/src/api/routes/tests/permissions.spec.ts @@ -188,7 +188,7 @@ describe("/permission", () => { await config.publish() const res = await config.api.viewV2.publicSearch(view.id) - expect(res.body.rows[0]._id).toEqual(row._id) + expect(res.rows[0]._id).toEqual(row._id) }) it("should not be able to access the view data when the table is not public and there are no view permissions overrides", async () => { @@ -238,7 +238,7 @@ describe("/permission", () => { await config.publish() const res = await config.api.viewV2.publicSearch(view.id) - expect(res.body.rows[0]._id).toEqual(row._id) + expect(res.rows[0]._id).toEqual(row._id) }) it("shouldn't allow writing from a public user", async () => { From 594cd5ee56bc8f6f42bff279f2ceab747014ff08 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 1 Mar 2024 17:40:10 +0000 Subject: [PATCH 046/114] Fix permissions tests. --- packages/server/src/tests/utilities/api/viewV2.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/packages/server/src/tests/utilities/api/viewV2.ts b/packages/server/src/tests/utilities/api/viewV2.ts index dd49360aa8..835431d9d1 100644 --- a/packages/server/src/tests/utilities/api/viewV2.ts +++ b/packages/server/src/tests/utilities/api/viewV2.ts @@ -26,9 +26,14 @@ export class ViewV2API extends TestAPI { ...viewData, } + const exp: Expectations = { + status: 201, + ...expectations, + } + const resp = await this._post<{ data: ViewV2 }>("/api/v2/views", { body: view, - expectations, + expectations: exp, }) return resp.data } From 89a03af92afd443a3c61d7435c0bd22d7c829ca7 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 4 Mar 2024 09:20:32 +0000 Subject: [PATCH 047/114] Fix table test. --- packages/server/src/tests/utilities/api/row.ts | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/packages/server/src/tests/utilities/api/row.ts b/packages/server/src/tests/utilities/api/row.ts index 58ddb7c049..86664574cb 100644 --- a/packages/server/src/tests/utilities/api/row.ts +++ b/packages/server/src/tests/utilities/api/row.ts @@ -125,10 +125,13 @@ export class RowAPI extends TestAPI { body: BulkImportRequest, expectations?: Expectations ): Promise => { - return await this._post(`/api/${tableId}/rows/import`, { - body, - expectations, - }) + return await this._post( + `/api/tables/${tableId}/import`, + { + body, + expectations, + } + ) } search = async ( From 143daa153cd56c88b684a48d1db05a40851527a4 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 4 Mar 2024 09:38:53 +0000 Subject: [PATCH 048/114] Fix ViewV2 tests. --- packages/server/src/tests/utilities/api/viewV2.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/server/src/tests/utilities/api/viewV2.ts b/packages/server/src/tests/utilities/api/viewV2.ts index 835431d9d1..d4539e00b1 100644 --- a/packages/server/src/tests/utilities/api/viewV2.ts +++ b/packages/server/src/tests/utilities/api/viewV2.ts @@ -50,7 +50,11 @@ export class ViewV2API extends TestAPI { } delete = async (viewId: string, expectations?: Expectations) => { - return await this._delete(`/api/v2/views/${viewId}`, { expectations }) + const exp = { + status: 204, + ...expectations, + } + return await this._delete(`/api/v2/views/${viewId}`, { expectations: exp }) } get = async (viewId: string) => { From e3c514e45aee8398c4b65b530c86536f7169dafd Mon Sep 17 00:00:00 2001 From: Peter Clement Date: Mon, 4 Mar 2024 09:48:47 +0000 Subject: [PATCH 049/114] Update test lucene builder and add more tests --- packages/shared-core/src/filters.ts | 57 ++++---- .../shared-core/src/tests/filters.test.ts | 128 +++++++++++++----- 2 files changed, 116 insertions(+), 69 deletions(-) diff --git a/packages/shared-core/src/filters.ts b/packages/shared-core/src/filters.ts index 2c4861ed60..5f975ff541 100644 --- a/packages/shared-core/src/filters.ts +++ b/packages/shared-core/src/filters.ts @@ -391,43 +391,32 @@ export const runLuceneQuery = (docs: any[], query?: SearchQuery) => { ) const docMatch = (doc: any) => { - // Determine active filters based on query object - const activeFilterKeys = Object.entries(query || {}) + const filterFunctions = { + string: stringMatch, + fuzzy: fuzzyMatch, + range: rangeMatch, + equal: equalMatch, + notEqual: notEqualMatch, + empty: emptyMatch, + notEmpty: notEmptyMatch, + oneOf: oneOf, + contains: contains, + containsAny: containsAny, + notContains: notContains, + } + const activeFilterKeys: (keyof typeof filterFunctions)[] = Object.entries( + query + ) .filter( - ([key, value]) => + ([key, value]: [string, any]) => !["allOr", "onEmptyFilter"].includes(key) && - Object.keys(value).length > 0 + Object.keys(value as Record).length > 0 ) - .map(([key]) => key) + .map(([key]) => key as keyof typeof filterFunctions) - // Apply filters dynamically based on activeFilterKeys - const results = activeFilterKeys.map(filterKey => { - switch (filterKey) { - case "string": - return stringMatch(doc) - case "fuzzy": - return fuzzyMatch(doc) - case "range": - return rangeMatch(doc) - case "equal": - return equalMatch(doc) - case "notEqual": - return notEqualMatch(doc) - case "empty": - return emptyMatch(doc) - case "notEmpty": - return notEmptyMatch(doc) - case "oneOf": - return oneOf(doc) - case "contains": - return contains(doc) - case "containsAny": - return containsAny(doc) - case "notContains": - return notContains(doc) - default: - return true // If the filter type is not recognized, default to true (assuming pass) - } + const results: boolean[] = activeFilterKeys.map(filterKey => { + const filterFunction = filterFunctions[filterKey] + return filterFunction ? filterFunction(doc) : true }) if (query!.allOr) { @@ -436,7 +425,7 @@ export const runLuceneQuery = (docs: any[], query?: SearchQuery) => { return results.every(result => result === true) } } - // Process all docs + return docs.filter(docMatch) } diff --git a/packages/shared-core/src/tests/filters.test.ts b/packages/shared-core/src/tests/filters.test.ts index 8586d58777..1e0a68de89 100644 --- a/packages/shared-core/src/tests/filters.test.ts +++ b/packages/shared-core/src/tests/filters.test.ts @@ -47,10 +47,7 @@ describe("runLuceneQuery", () => { }, ] - function buildQuery( - filterKey: string, - value: { [key: string]: any } - ): SearchQuery { + function buildQuery(filters: { [filterKey: string]: any }): SearchQuery { const query: SearchQuery = { string: {}, fuzzy: {}, @@ -63,8 +60,13 @@ describe("runLuceneQuery", () => { notContains: {}, oneOf: {}, containsAny: {}, + allOr: false, } - query[filterKey as SearchQueryOperators] = value + + for (const filterKey in filters) { + query[filterKey as SearchQueryOperators] = filters[filterKey] + } + return query } @@ -73,16 +75,17 @@ describe("runLuceneQuery", () => { }) it("should return matching rows for equal filter", () => { - const query = buildQuery("equal", { - order_status: 4, + const query = buildQuery({ + equal: { order_status: 4 }, }) expect(runLuceneQuery(docs, query).map(row => row.order_id)).toEqual([1, 2]) }) it("should return matching row for notEqual filter", () => { - const query = buildQuery("notEqual", { - order_status: 4, + const query = buildQuery({ + notEqual: { order_status: 4 }, }) + expect(runLuceneQuery(docs, query).map(row => row.order_id)).toEqual([3]) }) @@ -90,48 +93,56 @@ describe("runLuceneQuery", () => { expect( runLuceneQuery( docs, - buildQuery("fuzzy", { - description: "sm", + buildQuery({ + fuzzy: { description: "sm" }, }) ).map(row => row.description) ).toEqual(["Small box"]) expect( runLuceneQuery( docs, - buildQuery("string", { - description: "SM", + buildQuery({ + string: { description: "SM" }, }) ).map(row => row.description) ).toEqual(["Small box"]) }) it("should return rows within a range filter", () => { - const query = buildQuery("range", { - customer_id: { - low: 500, - high: 1000, + const query = buildQuery({ + range: { + customer_id: { + low: 500, + high: 1000, + }, }, }) + expect(runLuceneQuery(docs, query).map(row => row.order_id)).toEqual([3]) }) it("should return rows with numeric strings within a range filter", () => { - const query = buildQuery("range", { - customer_id: { - low: "500", - high: "1000", + const query = buildQuery({ + range: { + customer_id: { + low: "500", + high: "1000", + }, }, }) expect(runLuceneQuery(docs, query).map(row => row.order_id)).toEqual([3]) }) it("should return rows with ISO date strings within a range filter", () => { - const query = buildQuery("range", { - order_date: { - low: "2016-01-04T00:00:00.000Z", - high: "2016-01-11T00:00:00.000Z", + const query = buildQuery({ + range: { + order_date: { + low: "2016-01-04T00:00:00.000Z", + high: "2016-01-11T00:00:00.000Z", + }, }, }) + expect(runLuceneQuery(docs, query).map(row => row.order_id)).toEqual([2]) }) @@ -150,40 +161,87 @@ describe("runLuceneQuery", () => { label: "", }, ] - const query = buildQuery("range", { - order_date: { - low: "2016-01-04T00:00:00.000Z", - high: "2016-01-11T00:00:00.000Z", + + const query = buildQuery({ + range: { + order_date: { + low: "2016-01-04T00:00:00.000Z", + high: "2016-01-11T00:00:00.000Z", + }, }, }) + expect(runLuceneQuery(docs, query)).toEqual(docs) }) it("should return rows with matches on empty filter", () => { - const query = buildQuery("empty", { - label: null, + const query = buildQuery({ + empty: { + label: null, + }, }) + expect(runLuceneQuery(docs, query).map(row => row.order_id)).toEqual([1]) }) it("should return rows with matches on notEmpty filter", () => { - const query = buildQuery("notEmpty", { - label: null, + const query = buildQuery({ + notEmpty: { + label: null, + }, }) + expect(runLuceneQuery(docs, query).map(row => row.order_id)).toEqual([2, 3]) }) test.each([[523, 259], "523,259"])( "should return rows with matches on numeric oneOf filter", input => { - let query = buildQuery("oneOf", { - customer_id: input, + const query = buildQuery({ + oneOf: { + customer_id: input, + }, }) + expect(runLuceneQuery(docs, query).map(row => row.customer_id)).toEqual([ 259, 523, ]) } ) + + it("should return matching results if allOr is true and only one filter matches", () => { + const query = buildQuery({ + allOr: true, + oneOf: { staff_id: [10] }, + contains: { description: ["box"] }, + }) + + expect(runLuceneQuery(docs, query).map(row => row.order_id)).toEqual([ + 1, 2, 3, + ]) + }) + + // what should the name of this test be if it's the same test as above but with different operands + + it("should return matching results if allOr is true and only one filter matches with different operands", () => { + const query = buildQuery({ + allOr: true, + equal: { order_status: 4 }, + oneOf: { label: ["FRAGILE"] }, + }) + + expect(runLuceneQuery(docs, query).map(row => row.order_id)).toEqual([1, 2]) + }) + + it("should return nothing if allOr is false and only one filter matches", () => { + const query = buildQuery({ + allOr: false, + oneOf: { staff_id: [10] }, + contains: { description: ["box"] }, + }) + + expect(runLuceneQuery(docs, query).map(row => row.order_id)).toEqual([]) + }) }) describe("buildLuceneQuery", () => { From 5679acb86811c290dd84faf1d81b19d615680f6b Mon Sep 17 00:00:00 2001 From: Peter Clement Date: Mon, 4 Mar 2024 09:55:28 +0000 Subject: [PATCH 050/114] fix types --- packages/shared-core/src/filters.ts | 34 ++++++++++++++--------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/packages/shared-core/src/filters.ts b/packages/shared-core/src/filters.ts index 5f975ff541..6d81bbdc62 100644 --- a/packages/shared-core/src/filters.ts +++ b/packages/shared-core/src/filters.ts @@ -391,28 +391,28 @@ export const runLuceneQuery = (docs: any[], query?: SearchQuery) => { ) const docMatch = (doc: any) => { - const filterFunctions = { - string: stringMatch, - fuzzy: fuzzyMatch, - range: rangeMatch, - equal: equalMatch, - notEqual: notEqualMatch, - empty: emptyMatch, - notEmpty: notEmptyMatch, - oneOf: oneOf, - contains: contains, - containsAny: containsAny, - notContains: notContains, - } - const activeFilterKeys: (keyof typeof filterFunctions)[] = Object.entries( - query - ) + const filterFunctions: Record boolean> = + { + string: stringMatch, + fuzzy: fuzzyMatch, + range: rangeMatch, + equal: equalMatch, + notEqual: notEqualMatch, + empty: emptyMatch, + notEmpty: notEmptyMatch, + oneOf: oneOf, + contains: contains, + containsAny: containsAny, + notContains: notContains, + } + + const activeFilterKeys: SearchQueryOperators[] = Object.entries(query) .filter( ([key, value]: [string, any]) => !["allOr", "onEmptyFilter"].includes(key) && Object.keys(value as Record).length > 0 ) - .map(([key]) => key as keyof typeof filterFunctions) + .map(([key]) => key as any) const results: boolean[] = activeFilterKeys.map(filterKey => { const filterFunction = filterFunctions[filterKey] From 3d9a7e5ddf5f76236a304e65239c899e8e865cd7 Mon Sep 17 00:00:00 2001 From: Peter Clement Date: Mon, 4 Mar 2024 10:07:06 +0000 Subject: [PATCH 051/114] fix type --- packages/shared-core/src/filters.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/shared-core/src/filters.ts b/packages/shared-core/src/filters.ts index 6d81bbdc62..0a1673e558 100644 --- a/packages/shared-core/src/filters.ts +++ b/packages/shared-core/src/filters.ts @@ -406,7 +406,7 @@ export const runLuceneQuery = (docs: any[], query?: SearchQuery) => { notContains: notContains, } - const activeFilterKeys: SearchQueryOperators[] = Object.entries(query) + const activeFilterKeys: SearchQueryOperators[] = Object.entries(query || {}) .filter( ([key, value]: [string, any]) => !["allOr", "onEmptyFilter"].includes(key) && From c39053bb518514849aa801178c0423277886abbe Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 4 Mar 2024 11:06:25 +0000 Subject: [PATCH 052/114] Respond to PR feedback. --- packages/server/src/api/controllers/permission.ts | 2 +- packages/types/src/api/web/app/permission.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/server/src/api/controllers/permission.ts b/packages/server/src/api/controllers/permission.ts index e12bf3655d..cdfa6d8b1c 100644 --- a/packages/server/src/api/controllers/permission.ts +++ b/packages/server/src/api/controllers/permission.ts @@ -20,7 +20,7 @@ import { import { removeFromArray } from "../../utilities" import sdk from "../../sdk" -enum PermissionUpdateType { +const enum PermissionUpdateType { REMOVE = "remove", ADD = "add", } diff --git a/packages/types/src/api/web/app/permission.ts b/packages/types/src/api/web/app/permission.ts index ebe7a8bea3..88ff4e9d2f 100644 --- a/packages/types/src/api/web/app/permission.ts +++ b/packages/types/src/api/web/app/permission.ts @@ -16,7 +16,7 @@ export interface GetDependantResourcesResponse { } export interface AddedPermission { - _id: string + _id?: string rev?: string error?: string reason?: string From a59647e1580932f6ca278ed933e93c84582a52e5 Mon Sep 17 00:00:00 2001 From: melohagan <101575380+melohagan@users.noreply.github.com> Date: Mon, 4 Mar 2024 11:21:43 +0000 Subject: [PATCH 053/114] Rebuild table schema when adding new column to get externalType (#13165) * Rebuild table schema when adding new column to get externalType * Added MySQL integration test suite * Add test for emitting datasource on save new column * Update packages/server/src/integration-test/mysql.spec.ts Co-authored-by: Sam Rose * remove duplicate tests * Use UUID * update account portal --------- Co-authored-by: Sam Rose --- packages/account-portal | 2 +- packages/builder/src/stores/builder/tables.js | 6 + .../src/api/controllers/table/external.ts | 3 +- .../server/src/integration-test/mysql.spec.ts | 309 ++++++++++++++++++ .../src/sdk/app/tables/external/index.ts | 14 +- .../types/src/documents/app/table/table.ts | 3 +- packages/types/src/sdk/search.ts | 4 + 7 files changed, 336 insertions(+), 5 deletions(-) create mode 100644 packages/server/src/integration-test/mysql.spec.ts diff --git a/packages/account-portal b/packages/account-portal index 19f7a5829f..806b6fd5c1 160000 --- a/packages/account-portal +++ b/packages/account-portal @@ -1 +1 @@ -Subproject commit 19f7a5829f4d23cbc694136e45d94482a59a475a +Subproject commit 806b6fd5c11c284ebf4a01627d75db939f0f8152 diff --git a/packages/builder/src/stores/builder/tables.js b/packages/builder/src/stores/builder/tables.js index 51b8416eda..f86b37ab85 100644 --- a/packages/builder/src/stores/builder/tables.js +++ b/packages/builder/src/stores/builder/tables.js @@ -147,6 +147,12 @@ export function createTablesStore() { if (indexes) { draft.indexes = indexes } + // Add object to indicate if column is being added + if (draft.schema[field.name] === undefined) { + draft._add = { + name: field.name, + } + } draft.schema = { ...draft.schema, [field.name]: cloneDeep(field), diff --git a/packages/server/src/api/controllers/table/external.ts b/packages/server/src/api/controllers/table/external.ts index f035822068..f3478af83b 100644 --- a/packages/server/src/api/controllers/table/external.ts +++ b/packages/server/src/api/controllers/table/external.ts @@ -28,6 +28,7 @@ function getDatasourceId(table: Table) { export async function save(ctx: UserCtx) { const inputs = ctx.request.body const renaming = inputs?._rename + const adding = inputs?._add // can't do this right now delete inputs.rows const tableId = ctx.request.body._id @@ -40,7 +41,7 @@ export async function save(ctx: UserCtx) { const { datasource, table } = await sdk.tables.external.save( datasourceId!, inputs, - { tableId, renaming } + { tableId, renaming, adding } ) builderSocket?.emitDatasourceUpdate(ctx, datasource) return table diff --git a/packages/server/src/integration-test/mysql.spec.ts b/packages/server/src/integration-test/mysql.spec.ts new file mode 100644 index 0000000000..f7d0388c62 --- /dev/null +++ b/packages/server/src/integration-test/mysql.spec.ts @@ -0,0 +1,309 @@ +import fetch from "node-fetch" +import { + generateMakeRequest, + MakeRequestResponse, +} from "../api/routes/public/tests/utils" +import { v4 as uuidv4 } from "uuid" +import * as setup from "../api/routes/tests/utilities" +import { + Datasource, + FieldType, + Table, + TableRequest, + TableSourceType, +} from "@budibase/types" +import _ from "lodash" +import { databaseTestProviders } from "../integrations/tests/utils" +import mysql from "mysql2/promise" +import { builderSocket } from "../websockets" +// @ts-ignore +fetch.mockSearch() + +const config = setup.getConfig()! + +jest.unmock("mysql2/promise") +jest.mock("../websockets", () => ({ + clientAppSocket: jest.fn(), + gridAppSocket: jest.fn(), + initialise: jest.fn(), + builderSocket: { + emitTableUpdate: jest.fn(), + emitTableDeletion: jest.fn(), + emitDatasourceUpdate: jest.fn(), + emitDatasourceDeletion: jest.fn(), + emitScreenUpdate: jest.fn(), + emitAppMetadataUpdate: jest.fn(), + emitAppPublish: jest.fn(), + }, +})) + +describe("mysql integrations", () => { + let makeRequest: MakeRequestResponse, + mysqlDatasource: Datasource, + primaryMySqlTable: Table + + beforeAll(async () => { + await config.init() + const apiKey = await config.generateApiKey() + + makeRequest = generateMakeRequest(apiKey, true) + + mysqlDatasource = await config.api.datasource.create( + await databaseTestProviders.mysql.datasource() + ) + }) + + afterAll(async () => { + await databaseTestProviders.mysql.stop() + }) + + beforeEach(async () => { + primaryMySqlTable = await config.createTable({ + name: uuidv4(), + type: "table", + primary: ["id"], + schema: { + id: { + name: "id", + type: FieldType.AUTO, + autocolumn: true, + }, + name: { + name: "name", + type: FieldType.STRING, + }, + description: { + name: "description", + type: FieldType.STRING, + }, + value: { + name: "value", + type: FieldType.NUMBER, + }, + }, + sourceId: mysqlDatasource._id, + sourceType: TableSourceType.EXTERNAL, + }) + }) + + afterAll(config.end) + + it("validate table schema", async () => { + const res = await makeRequest( + "get", + `/api/datasources/${mysqlDatasource._id}` + ) + + expect(res.status).toBe(200) + expect(res.body).toEqual({ + config: { + database: "mysql", + host: mysqlDatasource.config!.host, + password: "--secret-value--", + port: mysqlDatasource.config!.port, + user: "root", + }, + plus: true, + source: "MYSQL", + type: "datasource_plus", + _id: expect.any(String), + _rev: expect.any(String), + createdAt: expect.any(String), + updatedAt: expect.any(String), + entities: expect.any(Object), + }) + }) + + describe("POST /api/datasources/verify", () => { + it("should be able to verify the connection", async () => { + const response = await config.api.datasource.verify({ + datasource: await databaseTestProviders.mysql.datasource(), + }) + expect(response.status).toBe(200) + expect(response.body.connected).toBe(true) + }) + + it("should state an invalid datasource cannot connect", async () => { + const dbConfig = await databaseTestProviders.mysql.datasource() + const response = await config.api.datasource.verify({ + datasource: { + ...dbConfig, + config: { + ...dbConfig.config, + password: "wrongpassword", + }, + }, + }) + + expect(response.status).toBe(200) + expect(response.body.connected).toBe(false) + expect(response.body.error).toBeDefined() + }) + }) + + describe("POST /api/datasources/info", () => { + it("should fetch information about mysql datasource", async () => { + const primaryName = primaryMySqlTable.name + const response = await makeRequest("post", "/api/datasources/info", { + datasource: mysqlDatasource, + }) + expect(response.status).toBe(200) + expect(response.body.tableNames).toBeDefined() + expect(response.body.tableNames.indexOf(primaryName)).not.toBe(-1) + }) + }) + + describe("Integration compatibility with mysql search_path", () => { + let client: mysql.Connection, pathDatasource: Datasource + const database = "test1" + const database2 = "test-2" + + beforeAll(async () => { + const dsConfig = await databaseTestProviders.mysql.datasource() + const dbConfig = dsConfig.config! + + client = await mysql.createConnection(dbConfig) + await client.query(`CREATE DATABASE \`${database}\`;`) + await client.query(`CREATE DATABASE \`${database2}\`;`) + + const pathConfig: any = { + ...dsConfig, + config: { + ...dbConfig, + database, + }, + } + pathDatasource = await config.api.datasource.create(pathConfig) + }) + + afterAll(async () => { + await client.query(`DROP DATABASE \`${database}\`;`) + await client.query(`DROP DATABASE \`${database2}\`;`) + await client.end() + }) + + it("discovers tables from any schema in search path", async () => { + await client.query( + `CREATE TABLE \`${database}\`.table1 (id1 SERIAL PRIMARY KEY);` + ) + const response = await makeRequest("post", "/api/datasources/info", { + datasource: pathDatasource, + }) + expect(response.status).toBe(200) + expect(response.body.tableNames).toBeDefined() + expect(response.body.tableNames).toEqual( + expect.arrayContaining(["table1"]) + ) + }) + + it("does not mix columns from different tables", async () => { + const repeated_table_name = "table_same_name" + await client.query( + `CREATE TABLE \`${database}\`.${repeated_table_name} (id SERIAL PRIMARY KEY, val1 TEXT);` + ) + await client.query( + `CREATE TABLE \`${database2}\`.${repeated_table_name} (id2 SERIAL PRIMARY KEY, val2 TEXT);` + ) + const response = await makeRequest( + "post", + `/api/datasources/${pathDatasource._id}/schema`, + { + tablesFilter: [repeated_table_name], + } + ) + expect(response.status).toBe(200) + expect( + response.body.datasource.entities[repeated_table_name].schema + ).toBeDefined() + const schema = + response.body.datasource.entities[repeated_table_name].schema + expect(Object.keys(schema).sort()).toEqual(["id", "val1"]) + }) + }) + + describe("POST /api/tables/", () => { + let client: mysql.Connection + const emitDatasourceUpdateMock = jest.fn() + + beforeEach(async () => { + client = await mysql.createConnection( + ( + await databaseTestProviders.mysql.datasource() + ).config! + ) + }) + + afterEach(async () => { + await client.end() + }) + + it("will emit the datasource entity schema with externalType to the front-end when adding a new column", async () => { + mysqlDatasource = ( + await makeRequest( + "post", + `/api/datasources/${mysqlDatasource._id}/schema` + ) + ).body.datasource + + const addColumnToTable: TableRequest = { + type: "table", + sourceType: TableSourceType.EXTERNAL, + name: "table", + sourceId: mysqlDatasource._id!, + primary: ["id"], + schema: { + id: { + type: FieldType.AUTO, + name: "id", + autocolumn: true, + }, + new_column: { + type: FieldType.NUMBER, + name: "new_column", + }, + }, + _add: { + name: "new_column", + }, + } + + jest + .spyOn(builderSocket!, "emitDatasourceUpdate") + .mockImplementation(emitDatasourceUpdateMock) + + await makeRequest("post", "/api/tables/", addColumnToTable) + + const expectedTable: TableRequest = { + ...addColumnToTable, + schema: { + id: { + type: FieldType.NUMBER, + name: "id", + autocolumn: true, + constraints: { + presence: false, + }, + externalType: "int unsigned", + }, + new_column: { + type: FieldType.NUMBER, + name: "new_column", + autocolumn: false, + constraints: { + presence: false, + }, + externalType: "float(8,2)", + }, + }, + created: true, + _id: `${mysqlDatasource._id}__table`, + } + delete expectedTable._add + + expect(emitDatasourceUpdateMock).toBeCalledTimes(1) + const emittedDatasource: Datasource = + emitDatasourceUpdateMock.mock.calls[0][1] + expect(emittedDatasource.entities!["table"]).toEqual(expectedTable) + }) + }) +}) diff --git a/packages/server/src/sdk/app/tables/external/index.ts b/packages/server/src/sdk/app/tables/external/index.ts index 9a2bed0da2..0ace19d00e 100644 --- a/packages/server/src/sdk/app/tables/external/index.ts +++ b/packages/server/src/sdk/app/tables/external/index.ts @@ -3,6 +3,7 @@ import { Operation, RelationshipType, RenameColumn, + AddColumn, Table, TableRequest, ViewV2, @@ -32,7 +33,7 @@ import * as viewSdk from "../../views" export async function save( datasourceId: string, update: Table, - opts?: { tableId?: string; renaming?: RenameColumn } + opts?: { tableId?: string; renaming?: RenameColumn; adding?: AddColumn } ) { let tableToSave: TableRequest = { ...update, @@ -165,8 +166,17 @@ export async function save( // remove the rename prop delete tableToSave._rename + + // if adding a new column, we need to rebuild the schema for that table to get the 'externalType' of the column + if (opts?.adding) { + datasource.entities[tableToSave.name] = ( + await datasourceSdk.buildFilteredSchema(datasource, [tableToSave.name]) + ).tables[tableToSave.name] + } else { + datasource.entities[tableToSave.name] = tableToSave + } + // store it into couch now for budibase reference - datasource.entities[tableToSave.name] = tableToSave await db.put(populateExternalTableSchemas(datasource)) // Since tables are stored inside datasources, we need to notify clients diff --git a/packages/types/src/documents/app/table/table.ts b/packages/types/src/documents/app/table/table.ts index f3b8e6df8d..3b419dd811 100644 --- a/packages/types/src/documents/app/table/table.ts +++ b/packages/types/src/documents/app/table/table.ts @@ -1,6 +1,6 @@ import { Document } from "../../document" import { View, ViewV2 } from "../view" -import { RenameColumn } from "../../../sdk" +import { AddColumn, RenameColumn } from "../../../sdk" import { TableSchema } from "./schema" export const INTERNAL_TABLE_SOURCE_ID = "bb_internal" @@ -29,5 +29,6 @@ export interface Table extends Document { export interface TableRequest extends Table { _rename?: RenameColumn + _add?: AddColumn created?: boolean } diff --git a/packages/types/src/sdk/search.ts b/packages/types/src/sdk/search.ts index 67c344d845..7a0ddaed66 100644 --- a/packages/types/src/sdk/search.ts +++ b/packages/types/src/sdk/search.ts @@ -60,6 +60,10 @@ export interface RenameColumn { updated: string } +export interface AddColumn { + name: string +} + export interface RelationshipsJson { through?: string from?: string From 5fabe14f6261e2ef7c75129ffa2b34941eae4305 Mon Sep 17 00:00:00 2001 From: melohagan <101575380+melohagan@users.noreply.github.com> Date: Mon, 4 Mar 2024 12:20:27 +0000 Subject: [PATCH 054/114] Revert "Rebuild table schema when adding new column to get externalType (#13165)" (#13184) This reverts commit a59647e1580932f6ca278ed933e93c84582a52e5. --- packages/account-portal | 2 +- packages/builder/src/stores/builder/tables.js | 6 - .../src/api/controllers/table/external.ts | 3 +- .../server/src/integration-test/mysql.spec.ts | 309 ------------------ .../src/sdk/app/tables/external/index.ts | 14 +- .../types/src/documents/app/table/table.ts | 3 +- packages/types/src/sdk/search.ts | 4 - 7 files changed, 5 insertions(+), 336 deletions(-) delete mode 100644 packages/server/src/integration-test/mysql.spec.ts diff --git a/packages/account-portal b/packages/account-portal index 806b6fd5c1..19f7a5829f 160000 --- a/packages/account-portal +++ b/packages/account-portal @@ -1 +1 @@ -Subproject commit 806b6fd5c11c284ebf4a01627d75db939f0f8152 +Subproject commit 19f7a5829f4d23cbc694136e45d94482a59a475a diff --git a/packages/builder/src/stores/builder/tables.js b/packages/builder/src/stores/builder/tables.js index f86b37ab85..51b8416eda 100644 --- a/packages/builder/src/stores/builder/tables.js +++ b/packages/builder/src/stores/builder/tables.js @@ -147,12 +147,6 @@ export function createTablesStore() { if (indexes) { draft.indexes = indexes } - // Add object to indicate if column is being added - if (draft.schema[field.name] === undefined) { - draft._add = { - name: field.name, - } - } draft.schema = { ...draft.schema, [field.name]: cloneDeep(field), diff --git a/packages/server/src/api/controllers/table/external.ts b/packages/server/src/api/controllers/table/external.ts index f3478af83b..f035822068 100644 --- a/packages/server/src/api/controllers/table/external.ts +++ b/packages/server/src/api/controllers/table/external.ts @@ -28,7 +28,6 @@ function getDatasourceId(table: Table) { export async function save(ctx: UserCtx) { const inputs = ctx.request.body const renaming = inputs?._rename - const adding = inputs?._add // can't do this right now delete inputs.rows const tableId = ctx.request.body._id @@ -41,7 +40,7 @@ export async function save(ctx: UserCtx) { const { datasource, table } = await sdk.tables.external.save( datasourceId!, inputs, - { tableId, renaming, adding } + { tableId, renaming } ) builderSocket?.emitDatasourceUpdate(ctx, datasource) return table diff --git a/packages/server/src/integration-test/mysql.spec.ts b/packages/server/src/integration-test/mysql.spec.ts deleted file mode 100644 index f7d0388c62..0000000000 --- a/packages/server/src/integration-test/mysql.spec.ts +++ /dev/null @@ -1,309 +0,0 @@ -import fetch from "node-fetch" -import { - generateMakeRequest, - MakeRequestResponse, -} from "../api/routes/public/tests/utils" -import { v4 as uuidv4 } from "uuid" -import * as setup from "../api/routes/tests/utilities" -import { - Datasource, - FieldType, - Table, - TableRequest, - TableSourceType, -} from "@budibase/types" -import _ from "lodash" -import { databaseTestProviders } from "../integrations/tests/utils" -import mysql from "mysql2/promise" -import { builderSocket } from "../websockets" -// @ts-ignore -fetch.mockSearch() - -const config = setup.getConfig()! - -jest.unmock("mysql2/promise") -jest.mock("../websockets", () => ({ - clientAppSocket: jest.fn(), - gridAppSocket: jest.fn(), - initialise: jest.fn(), - builderSocket: { - emitTableUpdate: jest.fn(), - emitTableDeletion: jest.fn(), - emitDatasourceUpdate: jest.fn(), - emitDatasourceDeletion: jest.fn(), - emitScreenUpdate: jest.fn(), - emitAppMetadataUpdate: jest.fn(), - emitAppPublish: jest.fn(), - }, -})) - -describe("mysql integrations", () => { - let makeRequest: MakeRequestResponse, - mysqlDatasource: Datasource, - primaryMySqlTable: Table - - beforeAll(async () => { - await config.init() - const apiKey = await config.generateApiKey() - - makeRequest = generateMakeRequest(apiKey, true) - - mysqlDatasource = await config.api.datasource.create( - await databaseTestProviders.mysql.datasource() - ) - }) - - afterAll(async () => { - await databaseTestProviders.mysql.stop() - }) - - beforeEach(async () => { - primaryMySqlTable = await config.createTable({ - name: uuidv4(), - type: "table", - primary: ["id"], - schema: { - id: { - name: "id", - type: FieldType.AUTO, - autocolumn: true, - }, - name: { - name: "name", - type: FieldType.STRING, - }, - description: { - name: "description", - type: FieldType.STRING, - }, - value: { - name: "value", - type: FieldType.NUMBER, - }, - }, - sourceId: mysqlDatasource._id, - sourceType: TableSourceType.EXTERNAL, - }) - }) - - afterAll(config.end) - - it("validate table schema", async () => { - const res = await makeRequest( - "get", - `/api/datasources/${mysqlDatasource._id}` - ) - - expect(res.status).toBe(200) - expect(res.body).toEqual({ - config: { - database: "mysql", - host: mysqlDatasource.config!.host, - password: "--secret-value--", - port: mysqlDatasource.config!.port, - user: "root", - }, - plus: true, - source: "MYSQL", - type: "datasource_plus", - _id: expect.any(String), - _rev: expect.any(String), - createdAt: expect.any(String), - updatedAt: expect.any(String), - entities: expect.any(Object), - }) - }) - - describe("POST /api/datasources/verify", () => { - it("should be able to verify the connection", async () => { - const response = await config.api.datasource.verify({ - datasource: await databaseTestProviders.mysql.datasource(), - }) - expect(response.status).toBe(200) - expect(response.body.connected).toBe(true) - }) - - it("should state an invalid datasource cannot connect", async () => { - const dbConfig = await databaseTestProviders.mysql.datasource() - const response = await config.api.datasource.verify({ - datasource: { - ...dbConfig, - config: { - ...dbConfig.config, - password: "wrongpassword", - }, - }, - }) - - expect(response.status).toBe(200) - expect(response.body.connected).toBe(false) - expect(response.body.error).toBeDefined() - }) - }) - - describe("POST /api/datasources/info", () => { - it("should fetch information about mysql datasource", async () => { - const primaryName = primaryMySqlTable.name - const response = await makeRequest("post", "/api/datasources/info", { - datasource: mysqlDatasource, - }) - expect(response.status).toBe(200) - expect(response.body.tableNames).toBeDefined() - expect(response.body.tableNames.indexOf(primaryName)).not.toBe(-1) - }) - }) - - describe("Integration compatibility with mysql search_path", () => { - let client: mysql.Connection, pathDatasource: Datasource - const database = "test1" - const database2 = "test-2" - - beforeAll(async () => { - const dsConfig = await databaseTestProviders.mysql.datasource() - const dbConfig = dsConfig.config! - - client = await mysql.createConnection(dbConfig) - await client.query(`CREATE DATABASE \`${database}\`;`) - await client.query(`CREATE DATABASE \`${database2}\`;`) - - const pathConfig: any = { - ...dsConfig, - config: { - ...dbConfig, - database, - }, - } - pathDatasource = await config.api.datasource.create(pathConfig) - }) - - afterAll(async () => { - await client.query(`DROP DATABASE \`${database}\`;`) - await client.query(`DROP DATABASE \`${database2}\`;`) - await client.end() - }) - - it("discovers tables from any schema in search path", async () => { - await client.query( - `CREATE TABLE \`${database}\`.table1 (id1 SERIAL PRIMARY KEY);` - ) - const response = await makeRequest("post", "/api/datasources/info", { - datasource: pathDatasource, - }) - expect(response.status).toBe(200) - expect(response.body.tableNames).toBeDefined() - expect(response.body.tableNames).toEqual( - expect.arrayContaining(["table1"]) - ) - }) - - it("does not mix columns from different tables", async () => { - const repeated_table_name = "table_same_name" - await client.query( - `CREATE TABLE \`${database}\`.${repeated_table_name} (id SERIAL PRIMARY KEY, val1 TEXT);` - ) - await client.query( - `CREATE TABLE \`${database2}\`.${repeated_table_name} (id2 SERIAL PRIMARY KEY, val2 TEXT);` - ) - const response = await makeRequest( - "post", - `/api/datasources/${pathDatasource._id}/schema`, - { - tablesFilter: [repeated_table_name], - } - ) - expect(response.status).toBe(200) - expect( - response.body.datasource.entities[repeated_table_name].schema - ).toBeDefined() - const schema = - response.body.datasource.entities[repeated_table_name].schema - expect(Object.keys(schema).sort()).toEqual(["id", "val1"]) - }) - }) - - describe("POST /api/tables/", () => { - let client: mysql.Connection - const emitDatasourceUpdateMock = jest.fn() - - beforeEach(async () => { - client = await mysql.createConnection( - ( - await databaseTestProviders.mysql.datasource() - ).config! - ) - }) - - afterEach(async () => { - await client.end() - }) - - it("will emit the datasource entity schema with externalType to the front-end when adding a new column", async () => { - mysqlDatasource = ( - await makeRequest( - "post", - `/api/datasources/${mysqlDatasource._id}/schema` - ) - ).body.datasource - - const addColumnToTable: TableRequest = { - type: "table", - sourceType: TableSourceType.EXTERNAL, - name: "table", - sourceId: mysqlDatasource._id!, - primary: ["id"], - schema: { - id: { - type: FieldType.AUTO, - name: "id", - autocolumn: true, - }, - new_column: { - type: FieldType.NUMBER, - name: "new_column", - }, - }, - _add: { - name: "new_column", - }, - } - - jest - .spyOn(builderSocket!, "emitDatasourceUpdate") - .mockImplementation(emitDatasourceUpdateMock) - - await makeRequest("post", "/api/tables/", addColumnToTable) - - const expectedTable: TableRequest = { - ...addColumnToTable, - schema: { - id: { - type: FieldType.NUMBER, - name: "id", - autocolumn: true, - constraints: { - presence: false, - }, - externalType: "int unsigned", - }, - new_column: { - type: FieldType.NUMBER, - name: "new_column", - autocolumn: false, - constraints: { - presence: false, - }, - externalType: "float(8,2)", - }, - }, - created: true, - _id: `${mysqlDatasource._id}__table`, - } - delete expectedTable._add - - expect(emitDatasourceUpdateMock).toBeCalledTimes(1) - const emittedDatasource: Datasource = - emitDatasourceUpdateMock.mock.calls[0][1] - expect(emittedDatasource.entities!["table"]).toEqual(expectedTable) - }) - }) -}) diff --git a/packages/server/src/sdk/app/tables/external/index.ts b/packages/server/src/sdk/app/tables/external/index.ts index 0ace19d00e..9a2bed0da2 100644 --- a/packages/server/src/sdk/app/tables/external/index.ts +++ b/packages/server/src/sdk/app/tables/external/index.ts @@ -3,7 +3,6 @@ import { Operation, RelationshipType, RenameColumn, - AddColumn, Table, TableRequest, ViewV2, @@ -33,7 +32,7 @@ import * as viewSdk from "../../views" export async function save( datasourceId: string, update: Table, - opts?: { tableId?: string; renaming?: RenameColumn; adding?: AddColumn } + opts?: { tableId?: string; renaming?: RenameColumn } ) { let tableToSave: TableRequest = { ...update, @@ -166,17 +165,8 @@ export async function save( // remove the rename prop delete tableToSave._rename - - // if adding a new column, we need to rebuild the schema for that table to get the 'externalType' of the column - if (opts?.adding) { - datasource.entities[tableToSave.name] = ( - await datasourceSdk.buildFilteredSchema(datasource, [tableToSave.name]) - ).tables[tableToSave.name] - } else { - datasource.entities[tableToSave.name] = tableToSave - } - // store it into couch now for budibase reference + datasource.entities[tableToSave.name] = tableToSave await db.put(populateExternalTableSchemas(datasource)) // Since tables are stored inside datasources, we need to notify clients diff --git a/packages/types/src/documents/app/table/table.ts b/packages/types/src/documents/app/table/table.ts index 3b419dd811..f3b8e6df8d 100644 --- a/packages/types/src/documents/app/table/table.ts +++ b/packages/types/src/documents/app/table/table.ts @@ -1,6 +1,6 @@ import { Document } from "../../document" import { View, ViewV2 } from "../view" -import { AddColumn, RenameColumn } from "../../../sdk" +import { RenameColumn } from "../../../sdk" import { TableSchema } from "./schema" export const INTERNAL_TABLE_SOURCE_ID = "bb_internal" @@ -29,6 +29,5 @@ export interface Table extends Document { export interface TableRequest extends Table { _rename?: RenameColumn - _add?: AddColumn created?: boolean } diff --git a/packages/types/src/sdk/search.ts b/packages/types/src/sdk/search.ts index 7a0ddaed66..67c344d845 100644 --- a/packages/types/src/sdk/search.ts +++ b/packages/types/src/sdk/search.ts @@ -60,10 +60,6 @@ export interface RenameColumn { updated: string } -export interface AddColumn { - name: string -} - export interface RelationshipsJson { through?: string from?: string From 1857383c47428e060b731a4207e61392b5bdde45 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 4 Mar 2024 13:37:41 +0000 Subject: [PATCH 055/114] Constrain query execution response slightly based on PR feedback. --- .../server/src/api/controllers/query/index.ts | 18 +++++++++++++----- packages/types/src/api/web/query.ts | 2 +- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/packages/server/src/api/controllers/query/index.ts b/packages/server/src/api/controllers/query/index.ts index 725de41c9a..3c21537484 100644 --- a/packages/server/src/api/controllers/query/index.ts +++ b/packages/server/src/api/controllers/query/index.ts @@ -297,7 +297,10 @@ export async function preview( } async function execute( - ctx: UserCtx, + ctx: UserCtx< + ExecuteQueryRequest, + ExecuteQueryResponse | Record[] + >, opts: any = { rowsOnly: false, isAutomation: false } ) { const db = context.getAppDB() @@ -352,18 +355,23 @@ async function execute( } } -export async function executeV1(ctx: UserCtx) { +export async function executeV1( + ctx: UserCtx[]> +) { return execute(ctx, { rowsOnly: true, isAutomation: false }) } export async function executeV2( - ctx: UserCtx, + ctx: UserCtx< + ExecuteQueryRequest, + ExecuteQueryResponse | Record[] + >, { isAutomation }: { isAutomation?: boolean } = {} ) { return execute(ctx, { rowsOnly: false, isAutomation }) } -const removeDynamicVariables = async (queryId: any) => { +const removeDynamicVariables = async (queryId: string) => { const db = context.getAppDB() const query = await db.get(queryId) const datasource = await sdk.datasources.get(query.datasourceId) @@ -386,7 +394,7 @@ const removeDynamicVariables = async (queryId: any) => { export async function destroy(ctx: UserCtx) { const db = context.getAppDB() - const queryId = ctx.params.queryId + const queryId = ctx.params.queryId as string await removeDynamicVariables(queryId) const query = await db.get(queryId) const datasource = await sdk.datasources.get(query.datasourceId) diff --git a/packages/types/src/api/web/query.ts b/packages/types/src/api/web/query.ts index 40f4426d3e..3959cdea19 100644 --- a/packages/types/src/api/web/query.ts +++ b/packages/types/src/api/web/query.ts @@ -16,5 +16,5 @@ export interface ExecuteQueryRequest { } export interface ExecuteQueryResponse { - data: any[] + data: Record[] } From 1e184605a1809310a355fcff9e9251252c99c334 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 4 Mar 2024 13:47:53 +0000 Subject: [PATCH 056/114] Migrate users of switchToSelfHosted to config.withEnv --- .../src/api/routes/tests/apikeys.spec.js | 4 ++-- .../src/api/routes/tests/application.spec.ts | 9 +++++++++ .../api/routes/tests/queries/query.seq.spec.ts | 2 +- .../server/src/api/routes/tests/row.spec.ts | 3 +-- .../src/api/routes/tests/utilities/index.ts | 18 ------------------ 5 files changed, 13 insertions(+), 23 deletions(-) diff --git a/packages/server/src/api/routes/tests/apikeys.spec.js b/packages/server/src/api/routes/tests/apikeys.spec.js index eb6933af7d..678da38f28 100644 --- a/packages/server/src/api/routes/tests/apikeys.spec.js +++ b/packages/server/src/api/routes/tests/apikeys.spec.js @@ -13,7 +13,7 @@ describe("/api/keys", () => { describe("fetch", () => { it("should allow fetching", async () => { - await setup.switchToSelfHosted(async () => { + await config.withEnv({ SELF_HOSTED: "true" }, async () => { const res = await request .get(`/api/keys`) .set(config.defaultHeaders()) @@ -34,7 +34,7 @@ describe("/api/keys", () => { describe("update", () => { it("should allow updating a value", async () => { - await setup.switchToSelfHosted(async () => { + await config.withEnv({ SELF_HOSTED: "true" }, async () => { const res = await request .put(`/api/keys/TEST`) .send({ diff --git a/packages/server/src/api/routes/tests/application.spec.ts b/packages/server/src/api/routes/tests/application.spec.ts index dc235dbd01..3e4ad693db 100644 --- a/packages/server/src/api/routes/tests/application.spec.ts +++ b/packages/server/src/api/routes/tests/application.spec.ts @@ -248,4 +248,13 @@ describe("/applications", () => { expect(devLogs.data.length).toBe(0) }) }) + + describe("permissions", () => { + it("should only return apps a user has access to", async () => { + const user = await config.createUser() + + const apps = await config.api.application.fetch() + expect(apps.length).toBeGreaterThan(0) + }) + }) }) diff --git a/packages/server/src/api/routes/tests/queries/query.seq.spec.ts b/packages/server/src/api/routes/tests/queries/query.seq.spec.ts index c5cb188cbc..4347ed9044 100644 --- a/packages/server/src/api/routes/tests/queries/query.seq.spec.ts +++ b/packages/server/src/api/routes/tests/queries/query.seq.spec.ts @@ -157,7 +157,7 @@ describe("/queries", () => { }) it("should find a query in cloud", async () => { - await setup.switchToSelfHosted(async () => { + await config.withEnv({ SELF_HOSTED: "true" }, async () => { const query = await config.createQuery() const res = await request .get(`/api/queries/${query._id}`) diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts index c02159bb42..de411f5397 100644 --- a/packages/server/src/api/routes/tests/row.spec.ts +++ b/packages/server/src/api/routes/tests/row.spec.ts @@ -882,8 +882,7 @@ describe.each([ ], tableId: table._id, }) - // the environment needs configured for this - await setup.switchToSelfHosted(async () => { + await config.withEnv({ SELF_HOSTED: "true" }, async () => { return context.doInAppContext(config.getAppId(), async () => { const enriched = await outputProcessing(table, [row]) expect((enriched as Row[])[0].attachment[0].url).toBe( diff --git a/packages/server/src/api/routes/tests/utilities/index.ts b/packages/server/src/api/routes/tests/utilities/index.ts index 27c178fc38..915ff5d970 100644 --- a/packages/server/src/api/routes/tests/utilities/index.ts +++ b/packages/server/src/api/routes/tests/utilities/index.ts @@ -77,21 +77,3 @@ export function getConfig() { } return config! } - -export async function switchToSelfHosted(func: any) { - // self hosted stops any attempts to Dynamo - env._set("NODE_ENV", "production") - env._set("SELF_HOSTED", true) - let error - try { - await func() - } catch (err) { - error = err - } - env._set("NODE_ENV", "jest") - env._set("SELF_HOSTED", false) - // don't throw error until after reset - if (error) { - throw error - } -} From 7b7d10d1ff35fff240726cc401d57b8ed3c802cb Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 4 Mar 2024 15:47:04 +0000 Subject: [PATCH 057/114] Fixing an issue with returning statements in MS-SQL and MySQL, when creating, updating or deleting rows the response is not retrieved correctly when aliased. --- .../server/src/integrations/tests/sqlQueryJson/createSimple.json | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 packages/server/src/integrations/tests/sqlQueryJson/createSimple.json diff --git a/packages/server/src/integrations/tests/sqlQueryJson/createSimple.json b/packages/server/src/integrations/tests/sqlQueryJson/createSimple.json new file mode 100644 index 0000000000..e69de29bb2 From 02e3b36cd80575acb9c408ab70a64d5b542e3433 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 4 Mar 2024 15:47:27 +0000 Subject: [PATCH 058/114] Adding missing files. --- .../server/src/api/controllers/row/alias.ts | 99 ++++++++++--------- packages/server/src/integrations/base/sql.ts | 34 ++++++- .../src/integrations/tests/sqlAlias.spec.ts | 23 +++++ .../tests/sqlQueryJson/createSimple.json | 65 ++++++++++++ .../types/src/documents/app/table/table.ts | 1 + 5 files changed, 172 insertions(+), 50 deletions(-) diff --git a/packages/server/src/api/controllers/row/alias.ts b/packages/server/src/api/controllers/row/alias.ts index 9658a0d638..4bd1e0a2ff 100644 --- a/packages/server/src/api/controllers/row/alias.ts +++ b/packages/server/src/api/controllers/row/alias.ts @@ -107,57 +107,62 @@ export default class AliasTables { } async queryWithAliasing(json: QueryJson): DatasourcePlusQueryResponse { - json = cloneDeep(json) - const aliasTable = (table: Table) => ({ - ...table, - name: this.getAlias(table.name), - }) - // run through the query json to update anywhere a table may be used - if (json.resource?.fields) { - json.resource.fields = json.resource.fields.map(field => - this.aliasField(field) - ) - } - if (json.filters) { - for (let [filterKey, filter] of Object.entries(json.filters)) { - if (typeof filter !== "object") { - continue - } - const aliasedFilters: typeof filter = {} - for (let key of Object.keys(filter)) { - aliasedFilters[this.aliasField(key)] = filter[key] - } - json.filters[filterKey as keyof SearchFilters] = aliasedFilters + const fieldLength = json.resource?.fields?.length + const aliasingEnabled = fieldLength && fieldLength > 0 + if (aliasingEnabled) { + json = cloneDeep(json) + const aliasTable = (table: Table) => ({ + ...table, + name: this.getAlias(table.name), + originalName: table.name, + }) + // run through the query json to update anywhere a table may be used + if (json.resource?.fields) { + json.resource.fields = json.resource.fields.map(field => + this.aliasField(field) + ) } - } - if (json.relationships) { - json.relationships = json.relationships.map(relationship => ({ - ...relationship, - aliases: this.aliasMap([ - relationship.through, - relationship.tableName, - json.endpoint.entityId, - ]), - })) - } - if (json.meta?.table) { - json.meta.table = aliasTable(json.meta.table) - } - if (json.meta?.tables) { - const aliasedTables: Record = {} - for (let [tableName, table] of Object.entries(json.meta.tables)) { - aliasedTables[this.getAlias(tableName)] = aliasTable(table) + if (json.filters) { + for (let [filterKey, filter] of Object.entries(json.filters)) { + if (typeof filter !== "object") { + continue + } + const aliasedFilters: typeof filter = {} + for (let key of Object.keys(filter)) { + aliasedFilters[this.aliasField(key)] = filter[key] + } + json.filters[filterKey as keyof SearchFilters] = aliasedFilters + } } - json.meta.tables = aliasedTables + if (json.relationships) { + json.relationships = json.relationships.map(relationship => ({ + ...relationship, + aliases: this.aliasMap([ + relationship.through, + relationship.tableName, + json.endpoint.entityId, + ]), + })) + } + if (json.meta?.table) { + json.meta.table = aliasTable(json.meta.table) + } + if (json.meta?.tables) { + const aliasedTables: Record = {} + for (let [tableName, table] of Object.entries(json.meta.tables)) { + aliasedTables[this.getAlias(tableName)] = aliasTable(table) + } + json.meta.tables = aliasedTables + } + // invert and return + const invertedTableAliases: Record = {} + for (let [key, value] of Object.entries(this.tableAliases)) { + invertedTableAliases[value] = key + } + json.tableAliases = invertedTableAliases } - // invert and return - const invertedTableAliases: Record = {} - for (let [key, value] of Object.entries(this.tableAliases)) { - invertedTableAliases[value] = key - } - json.tableAliases = invertedTableAliases const response = await getDatasourceAndQuery(json) - if (Array.isArray(response)) { + if (Array.isArray(response) && aliasingEnabled) { return this.reverse(response) } else { return response diff --git a/packages/server/src/integrations/base/sql.ts b/packages/server/src/integrations/base/sql.ts index 6605052598..5467232924 100644 --- a/packages/server/src/integrations/base/sql.ts +++ b/packages/server/src/integrations/base/sql.ts @@ -9,9 +9,12 @@ import { RelationshipsJson, SearchFilters, SortDirection, + Table, } from "@budibase/types" import environment from "../../environment" +type QueryFunction = (query: Knex.SqlNative, operation: Operation) => any + const envLimit = environment.SQL_MAX_ROWS ? parseInt(environment.SQL_MAX_ROWS) : null @@ -117,6 +120,29 @@ function generateSelectStatement( }) } +function disableAliasing(json: QueryJson) { + if (json.tableAliases) { + json.tableAliases = undefined + } + const removeTableAlias = (table: Table) => { + if (table.originalName) { + table.name = table.originalName + } + return table + } + if (json.meta?.table) { + json.meta.table = removeTableAlias(json.meta.table) + } + if (json.meta?.tables) { + for (let tableName of Object.keys(json.meta.tables)) { + json.meta.tables[tableName] = removeTableAlias( + json.meta.tables[tableName] + ) + } + } + return json +} + class InternalBuilder { private readonly client: string @@ -605,10 +631,12 @@ class SqlQueryBuilder extends SqlTableQueryBuilder { return query.toSQL().toNative() } - async getReturningRow(queryFn: Function, json: QueryJson) { + async getReturningRow(queryFn: QueryFunction, json: QueryJson) { if (!json.extra || !json.extra.idFilter) { return {} } + // disable aliasing if it is enabled + json = disableAliasing(json) const input = this._query({ endpoint: { ...json.endpoint, @@ -617,7 +645,7 @@ class SqlQueryBuilder extends SqlTableQueryBuilder { resource: { fields: [], }, - filters: json.extra.idFilter, + filters: json.extra?.idFilter, paginate: { limit: 1, }, @@ -646,7 +674,7 @@ class SqlQueryBuilder extends SqlTableQueryBuilder { // this function recreates the returning functionality of postgres async queryWithReturning( json: QueryJson, - queryFn: Function, + queryFn: QueryFunction, processFn: Function = (result: any) => result ) { const sqlClient = this.getSqlClient() diff --git a/packages/server/src/integrations/tests/sqlAlias.spec.ts b/packages/server/src/integrations/tests/sqlAlias.spec.ts index 9b3f6a1b38..c93e3916ce 100644 --- a/packages/server/src/integrations/tests/sqlAlias.spec.ts +++ b/packages/server/src/integrations/tests/sqlAlias.spec.ts @@ -4,6 +4,7 @@ import Sql from "../base/sql" import { SqlClient } from "../utils" import AliasTables from "../../api/controllers/row/alias" import { generator } from "@budibase/backend-core/tests" +import { Knex } from "knex" function multiline(sql: string) { return sql.replace(/\n/g, "").replace(/ +/g, " ") @@ -160,6 +161,28 @@ describe("Captures of real examples", () => { }) }) + describe("returning (everything bar Postgres)", () => { + it("should be able to handle row returning", () => { + const queryJson = getJson("createSimple.json") + const SQL = new Sql(SqlClient.MS_SQL, limit) + let query = SQL._query(queryJson, { disableReturning: true }) + expect(query).toEqual({ + sql: "insert into [people] ([age], [name]) values (@p0, @p1)", + bindings: [222, "awfawf"], + }) + + // now check returning + let returningQuery: Knex.SqlNative = { sql: "", bindings: [] } + SQL.getReturningRow((input: Knex.SqlNative) => { + returningQuery = input + }, queryJson) + expect(returningQuery).toEqual({ + sql: "select * from (select top (@p0) * from [people] where [people].[name] = @p1 and [people].[age] = @p2 order by [people].[name] asc) as [people]", + bindings: [1, "awfawf", 222], + }) + }) + }) + describe("check max character aliasing", () => { it("should handle over 'z' max character alias", () => { const tableNames = [] diff --git a/packages/server/src/integrations/tests/sqlQueryJson/createSimple.json b/packages/server/src/integrations/tests/sqlQueryJson/createSimple.json index e69de29bb2..42fae2325e 100644 --- a/packages/server/src/integrations/tests/sqlQueryJson/createSimple.json +++ b/packages/server/src/integrations/tests/sqlQueryJson/createSimple.json @@ -0,0 +1,65 @@ +{ + "endpoint": { + "datasourceId": "datasource_plus_0ed5835e5552496285df546030f7c4ae", + "entityId": "people", + "operation": "CREATE" + }, + "resource": { + "fields": [ + "a.name", + "a.age" + ] + }, + "filters": {}, + "relationships": [], + "body": { + "name": "awfawf", + "age": 222 + }, + "extra": { + "idFilter": { + "equal": { + "name": "awfawf", + "age": 222 + } + } + }, + "meta": { + "table": { + "_id": "datasource_plus_0ed5835e5552496285df546030f7c4ae__people", + "type": "table", + "sourceId": "datasource_plus_0ed5835e5552496285df546030f7c4ae", + "sourceType": "external", + "primary": [ + "name", + "age" + ], + "name": "a", + "schema": { + "name": { + "type": "string", + "externalType": "varchar", + "autocolumn": false, + "name": "name", + "constraints": { + "presence": true + } + }, + "age": { + "type": "number", + "externalType": "int", + "autocolumn": false, + "name": "age", + "constraints": { + "presence": false + } + } + }, + "primaryDisplay": "name", + "originalName": "people" + } + }, + "tableAliases": { + "people": "a" + } +} \ No newline at end of file diff --git a/packages/types/src/documents/app/table/table.ts b/packages/types/src/documents/app/table/table.ts index 3b419dd811..b284e9a840 100644 --- a/packages/types/src/documents/app/table/table.ts +++ b/packages/types/src/documents/app/table/table.ts @@ -15,6 +15,7 @@ export interface Table extends Document { sourceType: TableSourceType views?: { [key: string]: View | ViewV2 } name: string + originalName?: string sourceId: string primary?: string[] schema: TableSchema From ee0f0abad25d9e1bcfc499eb2e8b7d4ed3d7b38f Mon Sep 17 00:00:00 2001 From: melohagan <101575380+melohagan@users.noreply.github.com> Date: Mon, 4 Mar 2024 15:55:45 +0000 Subject: [PATCH 059/114] Fix/rename mysql column (#13186) * Rebuild table schema when adding new column to get externalType * Added MySQL integration test suite * Add test for emitting datasource on save new column * Update packages/server/src/integration-test/mysql.spec.ts Co-authored-by: Sam Rose * remove duplicate tests * Use UUID * update account portal * Remove _add for internal save * Internal DB add column unit test * rename column test * update modules * fix tests --------- Co-authored-by: Sam Rose --- packages/account-portal | 2 +- packages/builder/src/stores/builder/tables.js | 6 + packages/pro | 2 +- .../src/api/controllers/table/external.ts | 10 +- .../server/src/api/controllers/table/index.ts | 9 +- .../src/api/controllers/table/internal.ts | 14 +- .../server/src/api/routes/tests/table.spec.ts | 30 ++ .../server/src/integration-test/mysql.spec.ts | 363 ++++++++++++++++++ .../src/sdk/app/tables/external/index.ts | 14 +- .../types/src/documents/app/table/table.ts | 3 +- packages/types/src/sdk/search.ts | 4 + 11 files changed, 440 insertions(+), 17 deletions(-) create mode 100644 packages/server/src/integration-test/mysql.spec.ts diff --git a/packages/account-portal b/packages/account-portal index 19f7a5829f..0c050591c2 160000 --- a/packages/account-portal +++ b/packages/account-portal @@ -1 +1 @@ -Subproject commit 19f7a5829f4d23cbc694136e45d94482a59a475a +Subproject commit 0c050591c21d3b67dc0c9225d60cc9e2324c8dac diff --git a/packages/builder/src/stores/builder/tables.js b/packages/builder/src/stores/builder/tables.js index 51b8416eda..f86b37ab85 100644 --- a/packages/builder/src/stores/builder/tables.js +++ b/packages/builder/src/stores/builder/tables.js @@ -147,6 +147,12 @@ export function createTablesStore() { if (indexes) { draft.indexes = indexes } + // Add object to indicate if column is being added + if (draft.schema[field.name] === undefined) { + draft._add = { + name: field.name, + } + } draft.schema = { ...draft.schema, [field.name]: cloneDeep(field), diff --git a/packages/pro b/packages/pro index 183b35d3ac..22a278da72 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 183b35d3acd42433dcb2d32bcd89a36abe13afec +Subproject commit 22a278da720d92991dabdcd4cb6c96e7abe29781 diff --git a/packages/server/src/api/controllers/table/external.ts b/packages/server/src/api/controllers/table/external.ts index f035822068..c85b46a95c 100644 --- a/packages/server/src/api/controllers/table/external.ts +++ b/packages/server/src/api/controllers/table/external.ts @@ -6,6 +6,7 @@ import { BulkImportRequest, BulkImportResponse, Operation, + RenameColumn, SaveTableRequest, SaveTableResponse, Table, @@ -25,9 +26,12 @@ function getDatasourceId(table: Table) { return breakExternalTableId(table._id).datasourceId } -export async function save(ctx: UserCtx) { +export async function save( + ctx: UserCtx, + renaming?: RenameColumn +) { const inputs = ctx.request.body - const renaming = inputs?._rename + const adding = inputs?._add // can't do this right now delete inputs.rows const tableId = ctx.request.body._id @@ -40,7 +44,7 @@ export async function save(ctx: UserCtx) { const { datasource, table } = await sdk.tables.external.save( datasourceId!, inputs, - { tableId, renaming } + { tableId, renaming, adding } ) builderSocket?.emitDatasourceUpdate(ctx, datasource) return table diff --git a/packages/server/src/api/controllers/table/index.ts b/packages/server/src/api/controllers/table/index.ts index 55a896373f..69305c461e 100644 --- a/packages/server/src/api/controllers/table/index.ts +++ b/packages/server/src/api/controllers/table/index.ts @@ -74,8 +74,15 @@ export async function save(ctx: UserCtx) { const appId = ctx.appId const table = ctx.request.body const isImport = table.rows + const renaming = ctx.request.body._rename - let savedTable = await pickApi({ table }).save(ctx) + const api = pickApi({ table }) + // do not pass _rename or _add if saving to CouchDB + if (api === internal) { + delete ctx.request.body._add + delete ctx.request.body._rename + } + let savedTable = await api.save(ctx, renaming) if (!table._id) { await events.table.created(savedTable) savedTable = sdk.tables.enrichViewSchemas(savedTable) diff --git a/packages/server/src/api/controllers/table/internal.ts b/packages/server/src/api/controllers/table/internal.ts index 8e90007d88..eb5e4b6c41 100644 --- a/packages/server/src/api/controllers/table/internal.ts +++ b/packages/server/src/api/controllers/table/internal.ts @@ -12,11 +12,12 @@ import { } from "@budibase/types" import sdk from "../../../sdk" -export async function save(ctx: UserCtx) { +export async function save( + ctx: UserCtx, + renaming?: RenameColumn +) { const { rows, ...rest } = ctx.request.body - let tableToSave: Table & { - _rename?: RenameColumn - } = { + let tableToSave: Table = { _id: generateTableID(), ...rest, // Ensure these fields are populated, even if not sent in the request @@ -28,15 +29,12 @@ export async function save(ctx: UserCtx) { tableToSave.views = {} } - const renaming = tableToSave._rename - delete tableToSave._rename - try { const { table } = await sdk.tables.internal.save(tableToSave, { user: ctx.user, rowsToImport: rows, tableId: ctx.request.body._id, - renaming: renaming, + renaming, }) return table diff --git a/packages/server/src/api/routes/tests/table.spec.ts b/packages/server/src/api/routes/tests/table.spec.ts index 29465145a9..77704a0408 100644 --- a/packages/server/src/api/routes/tests/table.spec.ts +++ b/packages/server/src/api/routes/tests/table.spec.ts @@ -26,6 +26,7 @@ import { TableToBuild } from "../../../tests/utilities/TestConfiguration" tk.freeze(mocks.date.MOCK_DATE) const { basicTable } = setup.structures +const ISO_REGEX_PATTERN = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/ describe("/tables", () => { let request = setup.getRequest() @@ -285,6 +286,35 @@ describe("/tables", () => { expect(res.body.schema.roleId).toBeDefined() }) }) + + it("should add a new column for an internal DB table", async () => { + const saveTableRequest: SaveTableRequest = { + _add: { + name: "NEW_COLUMN", + }, + ...basicTable(), + } + + const response = await request + .post(`/api/tables`) + .send(saveTableRequest) + .set(config.defaultHeaders()) + .expect("Content-Type", /json/) + .expect(200) + + const expectedResponse = { + ...saveTableRequest, + _rev: expect.stringMatching(/^\d-.+/), + _id: expect.stringMatching(/^ta_.+/), + createdAt: expect.stringMatching(ISO_REGEX_PATTERN), + updatedAt: expect.stringMatching(ISO_REGEX_PATTERN), + views: {}, + } + delete expectedResponse._add + + expect(response.status).toBe(200) + expect(response.body).toEqual(expectedResponse) + }) }) describe("import", () => { diff --git a/packages/server/src/integration-test/mysql.spec.ts b/packages/server/src/integration-test/mysql.spec.ts new file mode 100644 index 0000000000..fac2bfcfeb --- /dev/null +++ b/packages/server/src/integration-test/mysql.spec.ts @@ -0,0 +1,363 @@ +import fetch from "node-fetch" +import { + generateMakeRequest, + MakeRequestResponse, +} from "../api/routes/public/tests/utils" +import { v4 as uuidv4 } from "uuid" +import * as setup from "../api/routes/tests/utilities" +import { + Datasource, + FieldType, + Table, + TableRequest, + TableSourceType, +} from "@budibase/types" +import _ from "lodash" +import { databaseTestProviders } from "../integrations/tests/utils" +import mysql from "mysql2/promise" +import { builderSocket } from "../websockets" +// @ts-ignore +fetch.mockSearch() + +const config = setup.getConfig()! + +jest.unmock("mysql2/promise") +jest.mock("../websockets", () => ({ + clientAppSocket: jest.fn(), + gridAppSocket: jest.fn(), + initialise: jest.fn(), + builderSocket: { + emitTableUpdate: jest.fn(), + emitTableDeletion: jest.fn(), + emitDatasourceUpdate: jest.fn(), + emitDatasourceDeletion: jest.fn(), + emitScreenUpdate: jest.fn(), + emitAppMetadataUpdate: jest.fn(), + emitAppPublish: jest.fn(), + }, +})) + +describe("mysql integrations", () => { + let makeRequest: MakeRequestResponse, + mysqlDatasource: Datasource, + primaryMySqlTable: Table + + beforeAll(async () => { + await config.init() + const apiKey = await config.generateApiKey() + + makeRequest = generateMakeRequest(apiKey, true) + + mysqlDatasource = await config.api.datasource.create( + await databaseTestProviders.mysql.datasource() + ) + }) + + afterAll(async () => { + await databaseTestProviders.mysql.stop() + }) + + beforeEach(async () => { + primaryMySqlTable = await config.createTable({ + name: uuidv4(), + type: "table", + primary: ["id"], + schema: { + id: { + name: "id", + type: FieldType.AUTO, + autocolumn: true, + }, + name: { + name: "name", + type: FieldType.STRING, + }, + description: { + name: "description", + type: FieldType.STRING, + }, + value: { + name: "value", + type: FieldType.NUMBER, + }, + }, + sourceId: mysqlDatasource._id, + sourceType: TableSourceType.EXTERNAL, + }) + }) + + afterAll(config.end) + + it("validate table schema", async () => { + const res = await makeRequest( + "get", + `/api/datasources/${mysqlDatasource._id}` + ) + + expect(res.status).toBe(200) + expect(res.body).toEqual({ + config: { + database: "mysql", + host: mysqlDatasource.config!.host, + password: "--secret-value--", + port: mysqlDatasource.config!.port, + user: "root", + }, + plus: true, + source: "MYSQL", + type: "datasource_plus", + _id: expect.any(String), + _rev: expect.any(String), + createdAt: expect.any(String), + updatedAt: expect.any(String), + entities: expect.any(Object), + }) + }) + + describe("POST /api/datasources/verify", () => { + it("should be able to verify the connection", async () => { + await config.api.datasource.verify( + { + datasource: await databaseTestProviders.mysql.datasource(), + }, + { + body: { + connected: true, + }, + } + ) + }) + + it("should state an invalid datasource cannot connect", async () => { + const dbConfig = await databaseTestProviders.mysql.datasource() + await config.api.datasource.verify( + { + datasource: { + ...dbConfig, + config: { + ...dbConfig.config, + password: "wrongpassword", + }, + }, + }, + { + body: { + connected: false, + error: + "Access denied for the specified user. User does not have the necessary privileges or the provided credentials are incorrect. Please verify the credentials, and ensure that the user has appropriate permissions.", + }, + } + ) + }) + }) + + describe("POST /api/datasources/info", () => { + it("should fetch information about mysql datasource", async () => { + const primaryName = primaryMySqlTable.name + const response = await makeRequest("post", "/api/datasources/info", { + datasource: mysqlDatasource, + }) + expect(response.status).toBe(200) + expect(response.body.tableNames).toBeDefined() + expect(response.body.tableNames.indexOf(primaryName)).not.toBe(-1) + }) + }) + + describe("Integration compatibility with mysql search_path", () => { + let client: mysql.Connection, pathDatasource: Datasource + const database = "test1" + const database2 = "test-2" + + beforeAll(async () => { + const dsConfig = await databaseTestProviders.mysql.datasource() + const dbConfig = dsConfig.config! + + client = await mysql.createConnection(dbConfig) + await client.query(`CREATE DATABASE \`${database}\`;`) + await client.query(`CREATE DATABASE \`${database2}\`;`) + + const pathConfig: any = { + ...dsConfig, + config: { + ...dbConfig, + database, + }, + } + pathDatasource = await config.api.datasource.create(pathConfig) + }) + + afterAll(async () => { + await client.query(`DROP DATABASE \`${database}\`;`) + await client.query(`DROP DATABASE \`${database2}\`;`) + await client.end() + }) + + it("discovers tables from any schema in search path", async () => { + await client.query( + `CREATE TABLE \`${database}\`.table1 (id1 SERIAL PRIMARY KEY);` + ) + const response = await makeRequest("post", "/api/datasources/info", { + datasource: pathDatasource, + }) + expect(response.status).toBe(200) + expect(response.body.tableNames).toBeDefined() + expect(response.body.tableNames).toEqual( + expect.arrayContaining(["table1"]) + ) + }) + + it("does not mix columns from different tables", async () => { + const repeated_table_name = "table_same_name" + await client.query( + `CREATE TABLE \`${database}\`.${repeated_table_name} (id SERIAL PRIMARY KEY, val1 TEXT);` + ) + await client.query( + `CREATE TABLE \`${database2}\`.${repeated_table_name} (id2 SERIAL PRIMARY KEY, val2 TEXT);` + ) + const response = await makeRequest( + "post", + `/api/datasources/${pathDatasource._id}/schema`, + { + tablesFilter: [repeated_table_name], + } + ) + expect(response.status).toBe(200) + expect( + response.body.datasource.entities[repeated_table_name].schema + ).toBeDefined() + const schema = + response.body.datasource.entities[repeated_table_name].schema + expect(Object.keys(schema).sort()).toEqual(["id", "val1"]) + }) + }) + + describe("POST /api/tables/", () => { + let client: mysql.Connection + const emitDatasourceUpdateMock = jest.fn() + + beforeEach(async () => { + client = await mysql.createConnection( + ( + await databaseTestProviders.mysql.datasource() + ).config! + ) + mysqlDatasource = await config.api.datasource.create( + await databaseTestProviders.mysql.datasource() + ) + }) + + afterEach(async () => { + await client.end() + }) + + it("will emit the datasource entity schema with externalType to the front-end when adding a new column", async () => { + const addColumnToTable: TableRequest = { + type: "table", + sourceType: TableSourceType.EXTERNAL, + name: "table", + sourceId: mysqlDatasource._id!, + primary: ["id"], + schema: { + id: { + type: FieldType.AUTO, + name: "id", + autocolumn: true, + }, + new_column: { + type: FieldType.NUMBER, + name: "new_column", + }, + }, + _add: { + name: "new_column", + }, + } + + jest + .spyOn(builderSocket!, "emitDatasourceUpdate") + .mockImplementation(emitDatasourceUpdateMock) + + await makeRequest("post", "/api/tables/", addColumnToTable) + + const expectedTable: TableRequest = { + ...addColumnToTable, + schema: { + id: { + type: FieldType.NUMBER, + name: "id", + autocolumn: true, + constraints: { + presence: false, + }, + externalType: "int unsigned", + }, + new_column: { + type: FieldType.NUMBER, + name: "new_column", + autocolumn: false, + constraints: { + presence: false, + }, + externalType: "float(8,2)", + }, + }, + created: true, + _id: `${mysqlDatasource._id}__table`, + } + delete expectedTable._add + + expect(emitDatasourceUpdateMock).toBeCalledTimes(1) + const emittedDatasource: Datasource = + emitDatasourceUpdateMock.mock.calls[0][1] + expect(emittedDatasource.entities!["table"]).toEqual(expectedTable) + }) + + it("will rename a column", async () => { + await makeRequest("post", "/api/tables/", primaryMySqlTable) + + let renameColumnOnTable: TableRequest = { + ...primaryMySqlTable, + schema: { + id: { + name: "id", + type: FieldType.AUTO, + autocolumn: true, + externalType: "unsigned integer", + }, + name: { + name: "name", + type: FieldType.STRING, + externalType: "text", + }, + description: { + name: "description", + type: FieldType.STRING, + externalType: "text", + }, + age: { + name: "age", + type: FieldType.NUMBER, + externalType: "float(8,2)", + }, + }, + } + + const response = await makeRequest( + "post", + "/api/tables/", + renameColumnOnTable + ) + mysqlDatasource = ( + await makeRequest( + "post", + `/api/datasources/${mysqlDatasource._id}/schema` + ) + ).body.datasource + + expect(response.status).toEqual(200) + expect( + Object.keys(mysqlDatasource.entities![primaryMySqlTable.name].schema) + ).toEqual(["id", "name", "description", "age"]) + }) + }) +}) diff --git a/packages/server/src/sdk/app/tables/external/index.ts b/packages/server/src/sdk/app/tables/external/index.ts index 9a2bed0da2..0ace19d00e 100644 --- a/packages/server/src/sdk/app/tables/external/index.ts +++ b/packages/server/src/sdk/app/tables/external/index.ts @@ -3,6 +3,7 @@ import { Operation, RelationshipType, RenameColumn, + AddColumn, Table, TableRequest, ViewV2, @@ -32,7 +33,7 @@ import * as viewSdk from "../../views" export async function save( datasourceId: string, update: Table, - opts?: { tableId?: string; renaming?: RenameColumn } + opts?: { tableId?: string; renaming?: RenameColumn; adding?: AddColumn } ) { let tableToSave: TableRequest = { ...update, @@ -165,8 +166,17 @@ export async function save( // remove the rename prop delete tableToSave._rename + + // if adding a new column, we need to rebuild the schema for that table to get the 'externalType' of the column + if (opts?.adding) { + datasource.entities[tableToSave.name] = ( + await datasourceSdk.buildFilteredSchema(datasource, [tableToSave.name]) + ).tables[tableToSave.name] + } else { + datasource.entities[tableToSave.name] = tableToSave + } + // store it into couch now for budibase reference - datasource.entities[tableToSave.name] = tableToSave await db.put(populateExternalTableSchemas(datasource)) // Since tables are stored inside datasources, we need to notify clients diff --git a/packages/types/src/documents/app/table/table.ts b/packages/types/src/documents/app/table/table.ts index f3b8e6df8d..3b419dd811 100644 --- a/packages/types/src/documents/app/table/table.ts +++ b/packages/types/src/documents/app/table/table.ts @@ -1,6 +1,6 @@ import { Document } from "../../document" import { View, ViewV2 } from "../view" -import { RenameColumn } from "../../../sdk" +import { AddColumn, RenameColumn } from "../../../sdk" import { TableSchema } from "./schema" export const INTERNAL_TABLE_SOURCE_ID = "bb_internal" @@ -29,5 +29,6 @@ export interface Table extends Document { export interface TableRequest extends Table { _rename?: RenameColumn + _add?: AddColumn created?: boolean } diff --git a/packages/types/src/sdk/search.ts b/packages/types/src/sdk/search.ts index 67c344d845..7a0ddaed66 100644 --- a/packages/types/src/sdk/search.ts +++ b/packages/types/src/sdk/search.ts @@ -60,6 +60,10 @@ export interface RenameColumn { updated: string } +export interface AddColumn { + name: string +} + export interface RelationshipsJson { through?: string from?: string From 9d8c18337dc8d438da07a90307ceaff9d9a8951b Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 4 Mar 2024 16:42:41 +0000 Subject: [PATCH 060/114] Type role controller. --- packages/backend-core/src/security/roles.ts | 57 +++++++++--------- packages/pro | 2 +- packages/server/src/api/controllers/role.ts | 58 ++++++++++--------- .../server/src/api/controllers/routing.ts | 2 +- .../src/api/routes/tests/application.spec.ts | 11 +++- .../src/api/routes/tests/utilities/index.ts | 7 +-- .../src/tests/utilities/TestConfiguration.ts | 10 ++++ packages/types/src/api/web/index.ts | 1 + packages/types/src/api/web/role.ts | 22 +++++++ 9 files changed, 102 insertions(+), 68 deletions(-) create mode 100644 packages/types/src/api/web/role.ts diff --git a/packages/backend-core/src/security/roles.ts b/packages/backend-core/src/security/roles.ts index 4f048c0a11..01473ad991 100644 --- a/packages/backend-core/src/security/roles.ts +++ b/packages/backend-core/src/security/roles.ts @@ -84,16 +84,18 @@ export function getBuiltinRoles(): { [key: string]: RoleDoc } { return cloneDeep(BUILTIN_ROLES) } -export const BUILTIN_ROLE_ID_ARRAY = Object.values(BUILTIN_ROLES).map( - role => role._id -) +export function isBuiltin(role: string) { + return getBuiltinRole(role) !== undefined +} -export const BUILTIN_ROLE_NAME_ARRAY = Object.values(BUILTIN_ROLES).map( - role => role.name -) - -export function isBuiltin(role?: string) { - return BUILTIN_ROLE_ID_ARRAY.some(builtin => role?.includes(builtin)) +export function getBuiltinRole(roleId: string): Role | undefined { + const role = Object.values(BUILTIN_ROLES).find(role => + roleId.includes(role._id) + ) + if (!role) { + return undefined + } + return cloneDeep(role) } /** @@ -123,7 +125,7 @@ export function builtinRoleToNumber(id?: string) { /** * Converts any role to a number, but has to be async to get the roles from db. */ -export async function roleToNumber(id?: string) { +export async function roleToNumber(id: string) { if (isBuiltin(id)) { return builtinRoleToNumber(id) } @@ -131,7 +133,7 @@ export async function roleToNumber(id?: string) { defaultPublic: true, })) as RoleDoc[] for (let role of hierarchy) { - if (isBuiltin(role?.inherits)) { + if (role?.inherits && isBuiltin(role.inherits)) { return builtinRoleToNumber(role.inherits) + 1 } } @@ -161,35 +163,28 @@ export function lowerBuiltinRoleID(roleId1?: string, roleId2?: string): string { * @returns The role object, which may contain an "inherits" property. */ export async function getRole( - roleId?: string, + roleId: string, opts?: { defaultPublic?: boolean } -): Promise { - if (!roleId) { - return undefined - } - let role: any = {} +): Promise { // built in roles mostly come from the in-code implementation, // but can be extended by a doc stored about them (e.g. permissions) - if (isBuiltin(roleId)) { - role = cloneDeep( - Object.values(BUILTIN_ROLES).find(role => role._id === roleId) - ) - } else { + let role: RoleDoc | undefined = getBuiltinRole(roleId) + if (!role) { // make sure has the prefix (if it has it then it won't be added) roleId = prefixRoleID(roleId) } try { const db = getAppDB() - const dbRole = await db.get(getDBRoleID(roleId)) - role = Object.assign(role, dbRole) + const dbRole = await db.get(getDBRoleID(roleId)) + role = Object.assign(role || {}, dbRole) // finalise the ID - role._id = getExternalRoleID(role._id, role.version) + role._id = getExternalRoleID(role._id!, role.version) } catch (err) { if (!isBuiltin(roleId) && opts?.defaultPublic) { return cloneDeep(BUILTIN_ROLES.PUBLIC) } // only throw an error if there is no role at all - if (Object.keys(role).length === 0) { + if (!role || Object.keys(role).length === 0) { throw err } } @@ -200,7 +195,7 @@ export async function getRole( * Simple function to get all the roles based on the top level user role ID. */ async function getAllUserRoles( - userRoleId?: string, + userRoleId: string, opts?: { defaultPublic?: boolean } ): Promise { // admins have access to all roles @@ -226,7 +221,7 @@ async function getAllUserRoles( } export async function getUserRoleIdHierarchy( - userRoleId?: string + userRoleId: string ): Promise { const roles = await getUserRoleHierarchy(userRoleId) return roles.map(role => role._id!) @@ -241,7 +236,7 @@ export async function getUserRoleIdHierarchy( * highest level of access and the last being the lowest level. */ export async function getUserRoleHierarchy( - userRoleId?: string, + userRoleId: string, opts?: { defaultPublic?: boolean } ) { // special case, if they don't have a role then they are a public user @@ -265,9 +260,9 @@ export function checkForRoleResourceArray( return rolePerms } -export async function getAllRoleIds(appId?: string) { +export async function getAllRoleIds(appId: string): Promise { const roles = await getAllRoles(appId) - return roles.map(role => role._id) + return roles.map(role => role._id!) } /** diff --git a/packages/pro b/packages/pro index 183b35d3ac..22a278da72 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 183b35d3acd42433dcb2d32bcd89a36abe13afec +Subproject commit 22a278da720d92991dabdcd4cb6c96e7abe29781 diff --git a/packages/server/src/api/controllers/role.ts b/packages/server/src/api/controllers/role.ts index ae6b89e6d4..ffc1d74209 100644 --- a/packages/server/src/api/controllers/role.ts +++ b/packages/server/src/api/controllers/role.ts @@ -7,8 +7,14 @@ import { } from "@budibase/backend-core" import { getUserMetadataParams, InternalTables } from "../../db/utils" import { + AccessibleRolesResponse, Database, + DestroyRoleResponse, + FetchRolesResponse, + FindRoleResponse, Role, + SaveRoleRequest, + SaveRoleResponse, UserCtx, UserMetadata, UserRoles, @@ -25,43 +31,35 @@ async function updateRolesOnUserTable( db: Database, roleId: string, updateOption: string, - roleVersion: string | undefined + roleVersion?: string ) { const table = await sdk.tables.getTable(InternalTables.USER_METADATA) - const schema = table.schema - const remove = updateOption === UpdateRolesOptions.REMOVED - let updated = false - for (let prop of Object.keys(schema)) { - if (prop === "roleId") { - updated = true - const constraints = schema[prop].constraints! - const updatedRoleId = - roleVersion === roles.RoleIDVersion.NAME - ? roles.getExternalRoleID(roleId, roleVersion) - : roleId - const indexOfRoleId = constraints.inclusion!.indexOf(updatedRoleId) - if (remove && indexOfRoleId !== -1) { - constraints.inclusion!.splice(indexOfRoleId, 1) - } else if (!remove && indexOfRoleId === -1) { - constraints.inclusion!.push(updatedRoleId) - } - break + const constraints = table.schema.roleId?.constraints + if (constraints) { + const updatedRoleId = + roleVersion === roles.RoleIDVersion.NAME + ? roles.getExternalRoleID(roleId, roleVersion) + : roleId + const indexOfRoleId = constraints.inclusion!.indexOf(updatedRoleId) + const remove = updateOption === UpdateRolesOptions.REMOVED + if (remove && indexOfRoleId !== -1) { + constraints.inclusion!.splice(indexOfRoleId, 1) + } else if (!remove && indexOfRoleId === -1) { + constraints.inclusion!.push(updatedRoleId) } - } - if (updated) { await db.put(table) } } -export async function fetch(ctx: UserCtx) { +export async function fetch(ctx: UserCtx) { ctx.body = await roles.getAllRoles() } -export async function find(ctx: UserCtx) { +export async function find(ctx: UserCtx) { ctx.body = await roles.getRole(ctx.params.roleId) } -export async function save(ctx: UserCtx) { +export async function save(ctx: UserCtx) { const db = context.getAppDB() let { _id, name, inherits, permissionId, version } = ctx.request.body let isCreate = false @@ -109,9 +107,9 @@ export async function save(ctx: UserCtx) { ctx.body = role } -export async function destroy(ctx: UserCtx) { +export async function destroy(ctx: UserCtx) { const db = context.getAppDB() - let roleId = ctx.params.roleId + let roleId = ctx.params.roleId as string if (roles.isBuiltin(roleId)) { ctx.throw(400, "Cannot delete builtin role.") } else { @@ -144,14 +142,18 @@ export async function destroy(ctx: UserCtx) { ctx.status = 200 } -export async function accessible(ctx: UserCtx) { +export async function accessible(ctx: UserCtx) { let roleId = ctx.user?.roleId if (!roleId) { roleId = roles.BUILTIN_ROLE_IDS.PUBLIC } if (ctx.user && sharedSdk.users.isAdminOrBuilder(ctx.user)) { const appId = context.getAppId() - ctx.body = await roles.getAllRoleIds(appId) + if (!appId) { + ctx.body = [] + } else { + ctx.body = await roles.getAllRoleIds(appId) + } } else { ctx.body = await roles.getUserRoleIdHierarchy(roleId!) } diff --git a/packages/server/src/api/controllers/routing.ts b/packages/server/src/api/controllers/routing.ts index 4154c6b597..040cda4dd0 100644 --- a/packages/server/src/api/controllers/routing.ts +++ b/packages/server/src/api/controllers/routing.ts @@ -63,7 +63,7 @@ export async function fetch(ctx: UserCtx) { export async function clientFetch(ctx: UserCtx) { const routing = await getRoutingStructure() let roleId = ctx.user?.role?._id - const roleIds = await roles.getUserRoleIdHierarchy(roleId) + const roleIds = roleId ? await roles.getUserRoleIdHierarchy(roleId) : [] for (let topLevel of Object.values(routing.routes) as any) { for (let subpathKey of Object.keys(topLevel.subpaths)) { let found = false diff --git a/packages/server/src/api/routes/tests/application.spec.ts b/packages/server/src/api/routes/tests/application.spec.ts index 3e4ad693db..5a3be462e8 100644 --- a/packages/server/src/api/routes/tests/application.spec.ts +++ b/packages/server/src/api/routes/tests/application.spec.ts @@ -251,10 +251,15 @@ describe("/applications", () => { describe("permissions", () => { it("should only return apps a user has access to", async () => { - const user = await config.createUser() + const user = await config.createUser({ + builder: { global: false }, + admin: { global: false }, + }) - const apps = await config.api.application.fetch() - expect(apps.length).toBeGreaterThan(0) + await config.withUser(user, async () => { + const apps = await config.api.application.fetch() + expect(apps).toHaveLength(0) + }) }) }) }) diff --git a/packages/server/src/api/routes/tests/utilities/index.ts b/packages/server/src/api/routes/tests/utilities/index.ts index 915ff5d970..dcb8ccd6c0 100644 --- a/packages/server/src/api/routes/tests/utilities/index.ts +++ b/packages/server/src/api/routes/tests/utilities/index.ts @@ -1,5 +1,4 @@ -import TestConfig from "../../../../tests/utilities/TestConfiguration" -import env from "../../../../environment" +import TestConfiguration from "../../../../tests/utilities/TestConfiguration" import supertest from "supertest" export * as structures from "../../../../tests/utilities/structures" @@ -47,10 +46,10 @@ export function delay(ms: number) { } let request: supertest.SuperTest | undefined | null, - config: TestConfig | null + config: TestConfiguration | null export function beforeAll() { - config = new TestConfig() + config = new TestConfiguration() request = config.getRequest() } diff --git a/packages/server/src/tests/utilities/TestConfiguration.ts b/packages/server/src/tests/utilities/TestConfiguration.ts index 35ca2982c0..2127e9d1cd 100644 --- a/packages/server/src/tests/utilities/TestConfiguration.ts +++ b/packages/server/src/tests/utilities/TestConfiguration.ts @@ -299,6 +299,16 @@ export default class TestConfiguration { } } + withUser(user: User, f: () => Promise) { + const oldUser = this.user + this.user = user + try { + return f() + } finally { + this.user = oldUser + } + } + // UTILS _req | void, Res>( diff --git a/packages/types/src/api/web/index.ts b/packages/types/src/api/web/index.ts index 9a688a17a5..8a091afdba 100644 --- a/packages/types/src/api/web/index.ts +++ b/packages/types/src/api/web/index.ts @@ -14,3 +14,4 @@ export * from "./cookies" export * from "./automation" export * from "./layout" export * from "./query" +export * from "./role" diff --git a/packages/types/src/api/web/role.ts b/packages/types/src/api/web/role.ts new file mode 100644 index 0000000000..c37dee60e0 --- /dev/null +++ b/packages/types/src/api/web/role.ts @@ -0,0 +1,22 @@ +import { Role } from "../../documents" + +export interface SaveRoleRequest { + _id?: string + _rev?: string + name: string + inherits: string + permissionId: string + version: string +} + +export interface SaveRoleResponse extends Role {} + +export interface FindRoleResponse extends Role {} + +export type FetchRolesResponse = Role[] + +export interface DestroyRoleResponse { + message: string +} + +export type AccessibleRolesResponse = string[] From a7fcd7cca16ff9d143ed9babf3b44ed34fe6c4ae Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 4 Mar 2024 16:55:10 +0000 Subject: [PATCH 061/114] Attempting to fix issue with table name needing to be aliased. --- .../scripts/integrations/mysql/init.sql | 6 ++++ .../server/src/api/controllers/row/alias.ts | 29 ++++++++--------- packages/server/src/integrations/base/sql.ts | 32 +++---------------- .../basicFetchWithRelationships.json | 2 +- .../tests/sqlQueryJson/createSimple.json | 5 ++- .../sqlQueryJson/createWithRelationships.json | 2 +- .../tests/sqlQueryJson/deleteSimple.json | 2 +- .../sqlQueryJson/enrichRelationship.json | 2 +- .../tests/sqlQueryJson/fetchManyToMany.json | 2 +- .../sqlQueryJson/filterByRelationship.json | 2 +- .../sqlQueryJson/manyRelationshipFilters.json | 2 +- .../sqlQueryJson/updateRelationship.json | 2 +- .../tests/sqlQueryJson/updateSimple.json | 2 +- 13 files changed, 35 insertions(+), 55 deletions(-) diff --git a/packages/server/scripts/integrations/mysql/init.sql b/packages/server/scripts/integrations/mysql/init.sql index ae5cd07788..e687c7c3b1 100644 --- a/packages/server/scripts/integrations/mysql/init.sql +++ b/packages/server/scripts/integrations/mysql/init.sql @@ -10,6 +10,11 @@ CREATE TABLE Persons ( City varchar(255), PRIMARY KEY (PersonID) ); +CREATE TABLE Person ( + PersonID int NOT NULL AUTO_INCREMENT, + Name varchar(255), + PRIMARY KEY (PersonID) +); CREATE TABLE Tasks ( TaskID int NOT NULL AUTO_INCREMENT, PersonID INT, @@ -27,6 +32,7 @@ CREATE TABLE Products ( ); INSERT INTO Persons (FirstName, LastName, Age, Address, City, CreatedAt) VALUES ('Mike', 'Hughes', 28.2, '123 Fake Street', 'Belfast', '2021-01-19 03:14:07'); INSERT INTO Persons (FirstName, LastName, Age, Address, City, CreatedAt) VALUES ('Dave', 'Johnson', 29, '124 Fake Street', 'Belfast', '2022-04-01 00:11:11'); +INSERT INTO Person (Name) VALUES ('Elf'); INSERT INTO Tasks (PersonID, TaskName, CreatedAt) VALUES (1, 'assembling', '2020-01-01'); INSERT INTO Tasks (PersonID, TaskName, CreatedAt) VALUES (2, 'processing', '2019-12-31'); INSERT INTO Products (name, updated) VALUES ('Meat', '11:00:22'), ('Fruit', '10:00:00'); diff --git a/packages/server/src/api/controllers/row/alias.ts b/packages/server/src/api/controllers/row/alias.ts index 4bd1e0a2ff..46b090bb97 100644 --- a/packages/server/src/api/controllers/row/alias.ts +++ b/packages/server/src/api/controllers/row/alias.ts @@ -62,7 +62,11 @@ export default class AliasTables { if (idx === -1 || idx > 1) { return } - return Math.abs(tableName.length - name.length) <= 2 + // this might look a bit mad, but the idea is if the field is wrapped, say in "", `` or [] + // then the idx of the table name will be 1, and we should allow for it ending in a closing + // character - otherwise it should be the full length if the index is zero + const allowedCharacterDiff = idx * 2 + return Math.abs(tableName.length - name.length) <= allowedCharacterDiff }) if (foundTableName) { const aliasedTableName = tableName.replace( @@ -111,11 +115,6 @@ export default class AliasTables { const aliasingEnabled = fieldLength && fieldLength > 0 if (aliasingEnabled) { json = cloneDeep(json) - const aliasTable = (table: Table) => ({ - ...table, - name: this.getAlias(table.name), - originalName: table.name, - }) // run through the query json to update anywhere a table may be used if (json.resource?.fields) { json.resource.fields = json.resource.fields.map(field => @@ -134,6 +133,14 @@ export default class AliasTables { json.filters[filterKey as keyof SearchFilters] = aliasedFilters } } + if (json.meta?.table) { + this.getAlias(json.meta.table.name) + } + if (json.meta?.tables) { + Object.keys(json.meta.tables).forEach(tableName => + this.getAlias(tableName) + ) + } if (json.relationships) { json.relationships = json.relationships.map(relationship => ({ ...relationship, @@ -144,16 +151,6 @@ export default class AliasTables { ]), })) } - if (json.meta?.table) { - json.meta.table = aliasTable(json.meta.table) - } - if (json.meta?.tables) { - const aliasedTables: Record = {} - for (let [tableName, table] of Object.entries(json.meta.tables)) { - aliasedTables[this.getAlias(tableName)] = aliasTable(table) - } - json.meta.tables = aliasedTables - } // invert and return const invertedTableAliases: Record = {} for (let [key, value] of Object.entries(this.tableAliases)) { diff --git a/packages/server/src/integrations/base/sql.ts b/packages/server/src/integrations/base/sql.ts index 5467232924..8c5066533e 100644 --- a/packages/server/src/integrations/base/sql.ts +++ b/packages/server/src/integrations/base/sql.ts @@ -120,29 +120,6 @@ function generateSelectStatement( }) } -function disableAliasing(json: QueryJson) { - if (json.tableAliases) { - json.tableAliases = undefined - } - const removeTableAlias = (table: Table) => { - if (table.originalName) { - table.name = table.originalName - } - return table - } - if (json.meta?.table) { - json.meta.table = removeTableAlias(json.meta.table) - } - if (json.meta?.tables) { - for (let tableName of Object.keys(json.meta.tables)) { - json.meta.tables[tableName] = removeTableAlias( - json.meta.tables[tableName] - ) - } - } - return json -} - class InternalBuilder { private readonly client: string @@ -348,15 +325,18 @@ class InternalBuilder { addSorting(query: Knex.QueryBuilder, json: QueryJson): Knex.QueryBuilder { let { sort, paginate } = json const table = json.meta?.table + const aliases = json.tableAliases + const aliased = + table?.name && aliases?.[table.name] ? aliases[table.name] : table?.name if (sort && Object.keys(sort || {}).length > 0) { for (let [key, value] of Object.entries(sort)) { const direction = value.direction === SortDirection.ASCENDING ? "asc" : "desc" - query = query.orderBy(`${table?.name}.${key}`, direction) + query = query.orderBy(`${aliased}.${key}`, direction) } } else if (this.client === SqlClient.MS_SQL && paginate?.limit) { // @ts-ignore - query = query.orderBy(`${table?.name}.${table?.primary[0]}`) + query = query.orderBy(`${aliased}.${table?.primary[0]}`) } return query } @@ -635,8 +615,6 @@ class SqlQueryBuilder extends SqlTableQueryBuilder { if (!json.extra || !json.extra.idFilter) { return {} } - // disable aliasing if it is enabled - json = disableAliasing(json) const input = this._query({ endpoint: { ...json.endpoint, diff --git a/packages/server/src/integrations/tests/sqlQueryJson/basicFetchWithRelationships.json b/packages/server/src/integrations/tests/sqlQueryJson/basicFetchWithRelationships.json index 3445f5fe67..ba7fa4ef9b 100644 --- a/packages/server/src/integrations/tests/sqlQueryJson/basicFetchWithRelationships.json +++ b/packages/server/src/integrations/tests/sqlQueryJson/basicFetchWithRelationships.json @@ -68,7 +68,7 @@ "primary": [ "personid" ], - "name": "a", + "name": "persons", "schema": { "year": { "type": "number", diff --git a/packages/server/src/integrations/tests/sqlQueryJson/createSimple.json b/packages/server/src/integrations/tests/sqlQueryJson/createSimple.json index 42fae2325e..5c69f270dd 100644 --- a/packages/server/src/integrations/tests/sqlQueryJson/createSimple.json +++ b/packages/server/src/integrations/tests/sqlQueryJson/createSimple.json @@ -34,7 +34,7 @@ "name", "age" ], - "name": "a", + "name": "people", "schema": { "name": { "type": "string", @@ -55,8 +55,7 @@ } } }, - "primaryDisplay": "name", - "originalName": "people" + "primaryDisplay": "name" } }, "tableAliases": { diff --git a/packages/server/src/integrations/tests/sqlQueryJson/createWithRelationships.json b/packages/server/src/integrations/tests/sqlQueryJson/createWithRelationships.json index 20331b949a..82d85c417b 100644 --- a/packages/server/src/integrations/tests/sqlQueryJson/createWithRelationships.json +++ b/packages/server/src/integrations/tests/sqlQueryJson/createWithRelationships.json @@ -58,7 +58,7 @@ "primary": [ "personid" ], - "name": "a", + "name": "persons", "schema": { "year": { "type": "number", diff --git a/packages/server/src/integrations/tests/sqlQueryJson/deleteSimple.json b/packages/server/src/integrations/tests/sqlQueryJson/deleteSimple.json index 2266b8c8be..d6e099c4b6 100644 --- a/packages/server/src/integrations/tests/sqlQueryJson/deleteSimple.json +++ b/packages/server/src/integrations/tests/sqlQueryJson/deleteSimple.json @@ -34,7 +34,7 @@ "keypartone", "keyparttwo" ], - "name": "a", + "name": "compositetable", "schema": { "keyparttwo": { "type": "string", diff --git a/packages/server/src/integrations/tests/sqlQueryJson/enrichRelationship.json b/packages/server/src/integrations/tests/sqlQueryJson/enrichRelationship.json index ee658aed18..d71f0552c6 100644 --- a/packages/server/src/integrations/tests/sqlQueryJson/enrichRelationship.json +++ b/packages/server/src/integrations/tests/sqlQueryJson/enrichRelationship.json @@ -49,7 +49,7 @@ "primary": [ "taskid" ], - "name": "a", + "name": "tasks", "schema": { "executorid": { "type": "number", diff --git a/packages/server/src/integrations/tests/sqlQueryJson/fetchManyToMany.json b/packages/server/src/integrations/tests/sqlQueryJson/fetchManyToMany.json index 682ebaab2d..cec2fdb025 100644 --- a/packages/server/src/integrations/tests/sqlQueryJson/fetchManyToMany.json +++ b/packages/server/src/integrations/tests/sqlQueryJson/fetchManyToMany.json @@ -63,7 +63,7 @@ "primary": [ "productid" ], - "name": "a", + "name": "products", "schema": { "productname": { "type": "string", diff --git a/packages/server/src/integrations/tests/sqlQueryJson/filterByRelationship.json b/packages/server/src/integrations/tests/sqlQueryJson/filterByRelationship.json index eb1025f382..399cb0f4d2 100644 --- a/packages/server/src/integrations/tests/sqlQueryJson/filterByRelationship.json +++ b/packages/server/src/integrations/tests/sqlQueryJson/filterByRelationship.json @@ -53,7 +53,7 @@ "primary": [ "productid" ], - "name": "a", + "name": "products", "schema": { "productname": { "type": "string", diff --git a/packages/server/src/integrations/tests/sqlQueryJson/manyRelationshipFilters.json b/packages/server/src/integrations/tests/sqlQueryJson/manyRelationshipFilters.json index afa0889450..2b5d156546 100644 --- a/packages/server/src/integrations/tests/sqlQueryJson/manyRelationshipFilters.json +++ b/packages/server/src/integrations/tests/sqlQueryJson/manyRelationshipFilters.json @@ -109,7 +109,7 @@ "primary": [ "taskid" ], - "name": "a", + "name": "tasks", "schema": { "executorid": { "type": "number", diff --git a/packages/server/src/integrations/tests/sqlQueryJson/updateRelationship.json b/packages/server/src/integrations/tests/sqlQueryJson/updateRelationship.json index 01e795bd6c..42c2a44335 100644 --- a/packages/server/src/integrations/tests/sqlQueryJson/updateRelationship.json +++ b/packages/server/src/integrations/tests/sqlQueryJson/updateRelationship.json @@ -66,7 +66,7 @@ "primary": [ "personid" ], - "name": "a", + "name": "persons", "schema": { "year": { "type": "number", diff --git a/packages/server/src/integrations/tests/sqlQueryJson/updateSimple.json b/packages/server/src/integrations/tests/sqlQueryJson/updateSimple.json index 01e795bd6c..42c2a44335 100644 --- a/packages/server/src/integrations/tests/sqlQueryJson/updateSimple.json +++ b/packages/server/src/integrations/tests/sqlQueryJson/updateSimple.json @@ -66,7 +66,7 @@ "primary": [ "personid" ], - "name": "a", + "name": "persons", "schema": { "year": { "type": "number", From 86aefcfe1a2d4a4b237b62ececd08ae434ea0e63 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 4 Mar 2024 17:00:34 +0000 Subject: [PATCH 062/114] Updating test values. --- packages/server/src/integrations/tests/sqlAlias.spec.ts | 4 ++-- .../src/integrations/tests/sqlQueryJson/createSimple.json | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/server/src/integrations/tests/sqlAlias.spec.ts b/packages/server/src/integrations/tests/sqlAlias.spec.ts index c93e3916ce..fe9798aaa0 100644 --- a/packages/server/src/integrations/tests/sqlAlias.spec.ts +++ b/packages/server/src/integrations/tests/sqlAlias.spec.ts @@ -168,7 +168,7 @@ describe("Captures of real examples", () => { let query = SQL._query(queryJson, { disableReturning: true }) expect(query).toEqual({ sql: "insert into [people] ([age], [name]) values (@p0, @p1)", - bindings: [222, "awfawf"], + bindings: [22, "Test"], }) // now check returning @@ -178,7 +178,7 @@ describe("Captures of real examples", () => { }, queryJson) expect(returningQuery).toEqual({ sql: "select * from (select top (@p0) * from [people] where [people].[name] = @p1 and [people].[age] = @p2 order by [people].[name] asc) as [people]", - bindings: [1, "awfawf", 222], + bindings: [1, "Test", 22], }) }) }) diff --git a/packages/server/src/integrations/tests/sqlQueryJson/createSimple.json b/packages/server/src/integrations/tests/sqlQueryJson/createSimple.json index 5c69f270dd..33a88d30e1 100644 --- a/packages/server/src/integrations/tests/sqlQueryJson/createSimple.json +++ b/packages/server/src/integrations/tests/sqlQueryJson/createSimple.json @@ -13,14 +13,14 @@ "filters": {}, "relationships": [], "body": { - "name": "awfawf", - "age": 222 + "name": "Test", + "age": 22 }, "extra": { "idFilter": { "equal": { - "name": "awfawf", - "age": 222 + "name": "Test", + "age": 22 } } }, From 45702ce91882c6c7f3005f3327ed312953f031ed Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 4 Mar 2024 17:02:52 +0000 Subject: [PATCH 063/114] Fixing typing. --- packages/server/src/integrations/base/sql.ts | 1 - packages/types/src/documents/app/table/table.ts | 1 - 2 files changed, 2 deletions(-) diff --git a/packages/server/src/integrations/base/sql.ts b/packages/server/src/integrations/base/sql.ts index 8c5066533e..c8acb606b3 100644 --- a/packages/server/src/integrations/base/sql.ts +++ b/packages/server/src/integrations/base/sql.ts @@ -9,7 +9,6 @@ import { RelationshipsJson, SearchFilters, SortDirection, - Table, } from "@budibase/types" import environment from "../../environment" diff --git a/packages/types/src/documents/app/table/table.ts b/packages/types/src/documents/app/table/table.ts index b284e9a840..3b419dd811 100644 --- a/packages/types/src/documents/app/table/table.ts +++ b/packages/types/src/documents/app/table/table.ts @@ -15,7 +15,6 @@ export interface Table extends Document { sourceType: TableSourceType views?: { [key: string]: View | ViewV2 } name: string - originalName?: string sourceId: string primary?: string[] schema: TableSchema From de0cdd25d11d539a18b3aa34947281bdd4a4667c Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Mon, 4 Mar 2024 17:37:22 +0000 Subject: [PATCH 064/114] Bump version to 2.21.1 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index c06173fe04..dc4485f1de 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.21.0", + "version": "2.21.1", "npmClient": "yarn", "packages": [ "packages/*", From 8694b8d772d0d6ac855a2167080400bc16dda3fb Mon Sep 17 00:00:00 2001 From: melohagan <101575380+melohagan@users.noreply.github.com> Date: Tue, 5 Mar 2024 09:03:19 +0000 Subject: [PATCH 065/114] Relationship picker incorrectly renders selections (#13175) * Ensure _id is decoded for external search * Fetch initial value for 'Update' type forms * test didn't run locally - might run on github workflow * Tested and appears to be as before * Null-pointer fix * undo type change * update modules * add test * update modules --- .../app/forms/RelationshipField.svelte | 27 +++- .../src/sdk/app/rows/search/external.ts | 18 ++- .../app/rows/search/tests/external.spec.ts | 115 ++++++++++-------- .../server/src/sdk/app/rows/search/utils.ts | 1 - 4 files changed, 102 insertions(+), 59 deletions(-) diff --git a/packages/client/src/components/app/forms/RelationshipField.svelte b/packages/client/src/components/app/forms/RelationshipField.svelte index 7cd571f6d9..1fbd0df522 100644 --- a/packages/client/src/components/app/forms/RelationshipField.svelte +++ b/packages/client/src/components/app/forms/RelationshipField.svelte @@ -1,7 +1,7 @@ { + const ids = breakRowIdField(row) + return ids[0] + }) + } + try { const table = await sdk.tables.getTable(tableId) options = searchInputMapping(table, options) @@ -119,9 +131,7 @@ export async function exportRows( requestQuery = { oneOf: { _id: rowIds.map((row: string) => { - const ids = JSON.parse( - decodeURI(row).replace(/'/g, `"`).replace(/%2C/g, ",") - ) + const ids = breakRowIdField(row) if (ids.length > 1) { throw new HTTPError( "Export data does not support composite keys.", diff --git a/packages/server/src/sdk/app/rows/search/tests/external.spec.ts b/packages/server/src/sdk/app/rows/search/tests/external.spec.ts index 1aaea8e258..bae84592ca 100644 --- a/packages/server/src/sdk/app/rows/search/tests/external.spec.ts +++ b/packages/server/src/sdk/app/rows/search/tests/external.spec.ts @@ -21,10 +21,11 @@ jest.unmock("mysql2/promise") jest.setTimeout(30000) -describe.skip("external", () => { +describe("external search", () => { const config = new TestConfiguration() let externalDatasource: Datasource, tableData: Table + const rows: Row[] = [] beforeAll(async () => { const container = await new GenericContainer("mysql") @@ -89,67 +90,81 @@ describe.skip("external", () => { }, }, } + + const table = await config.createExternalTable({ + ...tableData, + sourceId: externalDatasource._id, + }) + for (let i = 0; i < 10; i++) { + rows.push( + await config.createRow({ + tableId: table._id, + name: generator.first(), + surname: generator.last(), + age: generator.age(), + address: generator.address(), + }) + ) + } }) - describe("search", () => { - const rows: Row[] = [] - beforeAll(async () => { - const table = await config.createExternalTable({ - ...tableData, - sourceId: externalDatasource._id, - }) - for (let i = 0; i < 10; i++) { - rows.push( - await config.createRow({ - tableId: table._id, - name: generator.first(), - surname: generator.last(), - age: generator.age(), - address: generator.address(), - }) - ) + it("default search returns all the data", async () => { + await config.doInContext(config.appId, async () => { + const tableId = config.table!._id! + + const searchParams: SearchParams = { + tableId, + query: {}, } + const result = await search(searchParams) + + expect(result.rows).toHaveLength(10) + expect(result.rows).toEqual( + expect.arrayContaining(rows.map(r => expect.objectContaining(r))) + ) }) + }) - it("default search returns all the data", async () => { - await config.doInContext(config.appId, async () => { - const tableId = config.table!._id! + it("querying by fields will always return data attribute columns", async () => { + await config.doInContext(config.appId, async () => { + const tableId = config.table!._id! - const searchParams: SearchParams = { - tableId, - query: {}, - } - const result = await search(searchParams) + const searchParams: SearchParams = { + tableId, + query: {}, + fields: ["name", "age"], + } + const result = await search(searchParams) - expect(result.rows).toHaveLength(10) - expect(result.rows).toEqual( - expect.arrayContaining(rows.map(r => expect.objectContaining(r))) + expect(result.rows).toHaveLength(10) + expect(result.rows).toEqual( + expect.arrayContaining( + rows.map(r => ({ + ...expectAnyExternalColsAttributes, + name: r.name, + age: r.age, + })) ) - }) + ) }) + }) - it("querying by fields will always return data attribute columns", async () => { - await config.doInContext(config.appId, async () => { - const tableId = config.table!._id! + it("will decode _id in oneOf query", async () => { + await config.doInContext(config.appId, async () => { + const tableId = config.table!._id! - const searchParams: SearchParams = { - tableId, - query: {}, - fields: ["name", "age"], - } - const result = await search(searchParams) + const searchParams: SearchParams = { + tableId, + query: { + oneOf: { + _id: ["%5B1%5D", "%5B4%5D", "%5B8%5D"], + }, + }, + } + const result = await search(searchParams) - expect(result.rows).toHaveLength(10) - expect(result.rows).toEqual( - expect.arrayContaining( - rows.map(r => ({ - ...expectAnyExternalColsAttributes, - name: r.name, - age: r.age, - })) - ) - ) - }) + expect(result.rows).toHaveLength(3) + expect(result.rows.map(row => row.id)).toEqual([1, 4, 8]) }) }) }) diff --git a/packages/server/src/sdk/app/rows/search/utils.ts b/packages/server/src/sdk/app/rows/search/utils.ts index 4eee3cea41..5d93dcaca2 100644 --- a/packages/server/src/sdk/app/rows/search/utils.ts +++ b/packages/server/src/sdk/app/rows/search/utils.ts @@ -1,6 +1,5 @@ import { FieldType, - FieldTypeSubtypes, SearchParams, Table, DocumentType, From bc8fdeea6af82e8537751f3c785dbebf6e580211 Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Tue, 5 Mar 2024 09:04:26 +0000 Subject: [PATCH 066/114] Bump version to 2.21.2 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index dc4485f1de..6fb032ac77 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.21.1", + "version": "2.21.2", "npmClient": "yarn", "packages": [ "packages/*", From 13563d18dca87872dad9294c61c8018158fa191d Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 5 Mar 2024 09:20:20 +0000 Subject: [PATCH 067/114] Write a failing test. --- .../src/api/routes/tests/application.spec.ts | 46 ++++++++++++++++++- .../server/src/tests/utilities/api/index.ts | 3 ++ .../server/src/tests/utilities/api/role.ts | 41 +++++++++++++++++ 3 files changed, 89 insertions(+), 1 deletion(-) create mode 100644 packages/server/src/tests/utilities/api/role.ts diff --git a/packages/server/src/api/routes/tests/application.spec.ts b/packages/server/src/api/routes/tests/application.spec.ts index 5a3be462e8..b452e8742f 100644 --- a/packages/server/src/api/routes/tests/application.spec.ts +++ b/packages/server/src/api/routes/tests/application.spec.ts @@ -16,7 +16,13 @@ import * as setup from "./utilities" import { AppStatus } from "../../../db/utils" import { events, utils, context } from "@budibase/backend-core" import env from "../../../environment" -import type { App } from "@budibase/types" +import { + PermissionLevel, + type App, + INTERNAL_TABLE_SOURCE_ID, + TableSourceType, + FieldType, +} from "@budibase/types" import tk from "timekeeper" describe("/applications", () => { @@ -256,10 +262,48 @@ describe("/applications", () => { admin: { global: false }, }) + const table = await config.api.table.save({ + name: "table", + type: "table", + sourceId: INTERNAL_TABLE_SOURCE_ID, + sourceType: TableSourceType.INTERNAL, + schema: { + name: { + type: FieldType.STRING, + name: "name", + }, + }, + }) + await config.withUser(user, async () => { const apps = await config.api.application.fetch() expect(apps).toHaveLength(0) }) + + const role = await config.api.roles.save({ + name: "Test", + inherits: "PUBLIC", + permissionId: "read_only", + version: "name", + }) + + await config.api.user.update({ + ...user, + roles: { + [config.getAppId()]: role._id!, + }, + }) + + await config.api.permission.add({ + resourceId: table._id!, + roleId: role._id!, + level: PermissionLevel.READ, + }) + + await config.withUser(user, async () => { + const apps = await config.api.application.fetch() + expect(apps).toHaveLength(1) + }) }) }) }) diff --git a/packages/server/src/tests/utilities/api/index.ts b/packages/server/src/tests/utilities/api/index.ts index fdcec3098d..d66acd86fd 100644 --- a/packages/server/src/tests/utilities/api/index.ts +++ b/packages/server/src/tests/utilities/api/index.ts @@ -11,6 +11,7 @@ import { BackupAPI } from "./backup" import { AttachmentAPI } from "./attachment" import { UserAPI } from "./user" import { QueryAPI } from "./query" +import { RoleAPI } from "./role" export default class API { table: TableAPI @@ -25,6 +26,7 @@ export default class API { attachment: AttachmentAPI user: UserAPI query: QueryAPI + roles: RoleAPI constructor(config: TestConfiguration) { this.table = new TableAPI(config) @@ -39,5 +41,6 @@ export default class API { this.attachment = new AttachmentAPI(config) this.user = new UserAPI(config) this.query = new QueryAPI(config) + this.roles = new RoleAPI(config) } } diff --git a/packages/server/src/tests/utilities/api/role.ts b/packages/server/src/tests/utilities/api/role.ts new file mode 100644 index 0000000000..4defbc1220 --- /dev/null +++ b/packages/server/src/tests/utilities/api/role.ts @@ -0,0 +1,41 @@ +import { + AccessibleRolesResponse, + FetchRolesResponse, + FindRoleResponse, + SaveRoleRequest, + SaveRoleResponse, +} from "@budibase/types" +import { Expectations, TestAPI } from "./base" + +export class RoleAPI extends TestAPI { + fetch = async (expectations?: Expectations) => { + return await this._get(`/api/roles`, { + expectations, + }) + } + + find = async (roleId: string, expectations?: Expectations) => { + return await this._get(`/api/roles/${roleId}`, { + expectations, + }) + } + + save = async (body: SaveRoleRequest, expectations?: Expectations) => { + return await this._post(`/api/roles`, { + body, + expectations, + }) + } + + destroy = async (roleId: string, expectations?: Expectations) => { + return await this._delete(`/api/roles/${roleId}`, { + expectations, + }) + } + + accesssible = async (expectations?: Expectations) => { + return await this._get(`/api/roles/accessible`, { + expectations, + }) + } +} From fced2f369649410d9e8ea7e4ca381fef3d0afb12 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 5 Mar 2024 09:23:48 +0000 Subject: [PATCH 068/114] Respond to PR feedback. --- packages/backend-core/src/security/roles.ts | 2 +- packages/server/src/api/controllers/role.ts | 27 +++++++++++---------- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/packages/backend-core/src/security/roles.ts b/packages/backend-core/src/security/roles.ts index 01473ad991..213c65e18e 100644 --- a/packages/backend-core/src/security/roles.ts +++ b/packages/backend-core/src/security/roles.ts @@ -184,7 +184,7 @@ export async function getRole( return cloneDeep(BUILTIN_ROLES.PUBLIC) } // only throw an error if there is no role at all - if (!role || Object.keys(role).length === 0) { + if (Object.keys(role || {}).length === 0) { throw err } } diff --git a/packages/server/src/api/controllers/role.ts b/packages/server/src/api/controllers/role.ts index ffc1d74209..b3eb61a255 100644 --- a/packages/server/src/api/controllers/role.ts +++ b/packages/server/src/api/controllers/role.ts @@ -35,20 +35,21 @@ async function updateRolesOnUserTable( ) { const table = await sdk.tables.getTable(InternalTables.USER_METADATA) const constraints = table.schema.roleId?.constraints - if (constraints) { - const updatedRoleId = - roleVersion === roles.RoleIDVersion.NAME - ? roles.getExternalRoleID(roleId, roleVersion) - : roleId - const indexOfRoleId = constraints.inclusion!.indexOf(updatedRoleId) - const remove = updateOption === UpdateRolesOptions.REMOVED - if (remove && indexOfRoleId !== -1) { - constraints.inclusion!.splice(indexOfRoleId, 1) - } else if (!remove && indexOfRoleId === -1) { - constraints.inclusion!.push(updatedRoleId) - } - await db.put(table) + if (!constraints) { + return } + const updatedRoleId = + roleVersion === roles.RoleIDVersion.NAME + ? roles.getExternalRoleID(roleId, roleVersion) + : roleId + const indexOfRoleId = constraints.inclusion!.indexOf(updatedRoleId) + const remove = updateOption === UpdateRolesOptions.REMOVED + if (remove && indexOfRoleId !== -1) { + constraints.inclusion!.splice(indexOfRoleId, 1) + } else if (!remove && indexOfRoleId === -1) { + constraints.inclusion!.push(updatedRoleId) + } + await db.put(table) } export async function fetch(ctx: UserCtx) { From aa124524d4bc93e228c5ada844fccb541cb55e6e Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 5 Mar 2024 10:05:05 +0000 Subject: [PATCH 069/114] Add a simpler test. --- packages/backend-core/src/cache/user.ts | 4 +- packages/server/src/api/controllers/user.ts | 3 +- .../src/api/routes/tests/application.spec.ts | 39 ++++++++++++++++++- .../src/tests/utilities/TestConfiguration.ts | 4 +- 4 files changed, 44 insertions(+), 6 deletions(-) diff --git a/packages/backend-core/src/cache/user.ts b/packages/backend-core/src/cache/user.ts index 313b9a4d4a..ecfa20f99e 100644 --- a/packages/backend-core/src/cache/user.ts +++ b/packages/backend-core/src/cache/user.ts @@ -6,7 +6,7 @@ import env from "../environment" import * as accounts from "../accounts" import { UserDB } from "../users" import { sdk } from "@budibase/shared-core" -import { User } from "@budibase/types" +import { User, UserMetadata } from "@budibase/types" const EXPIRY_SECONDS = 3600 @@ -15,7 +15,7 @@ const EXPIRY_SECONDS = 3600 */ async function populateFromDB(userId: string, tenantId: string) { const db = tenancy.getTenantDB(tenantId) - const user = await db.get(userId) + const user = await db.get(userId) user.budibaseAccess = true if (!env.SELF_HOSTED && !env.DISABLE_ACCOUNT_PORTAL) { const account = await accounts.getAccount(user.email) diff --git a/packages/server/src/api/controllers/user.ts b/packages/server/src/api/controllers/user.ts index 108e29fd3d..d1658f9820 100644 --- a/packages/server/src/api/controllers/user.ts +++ b/packages/server/src/api/controllers/user.ts @@ -1,6 +1,6 @@ import { generateUserFlagID, InternalTables } from "../../db/utils" import { getFullUser } from "../../utilities/users" -import { context } from "@budibase/backend-core" +import { cache, context } from "@budibase/backend-core" import { ContextUserMetadata, Ctx, @@ -42,6 +42,7 @@ export async function updateMetadata( // this isn't applicable to the user delete metadata.roles ctx.body = await db.put(metadata) + await cache.user.invalidateUser(user._id!) } export async function destroyMetadata(ctx: UserCtx) { diff --git a/packages/server/src/api/routes/tests/application.spec.ts b/packages/server/src/api/routes/tests/application.spec.ts index b452e8742f..7424511200 100644 --- a/packages/server/src/api/routes/tests/application.spec.ts +++ b/packages/server/src/api/routes/tests/application.spec.ts @@ -256,7 +256,44 @@ describe("/applications", () => { }) describe("permissions", () => { - it("should only return apps a user has access to", async () => { + it.only("should only return apps a user has access to", async () => { + const user = await config.createUser({ + builder: { global: false }, + admin: { global: false }, + }) + + const table = await config.api.table.save({ + name: "table", + type: "table", + sourceId: INTERNAL_TABLE_SOURCE_ID, + sourceType: TableSourceType.INTERNAL, + schema: { + name: { + type: FieldType.STRING, + name: "name", + }, + }, + }) + + await config.withUser(user, async () => { + const apps = await config.api.application.fetch() + expect(apps).toHaveLength(0) + }) + + await config.api.user.update({ + ...user, + builder: { + [config.getAppId()]: true, + }, + }) + + await config.withUser(user, async () => { + const apps = await config.api.application.fetch() + expect(apps).toHaveLength(1) + }) + }) + + it("should only return apps a user has access to through a custom role on a group", async () => { const user = await config.createUser({ builder: { global: false }, admin: { global: false }, diff --git a/packages/server/src/tests/utilities/TestConfiguration.ts b/packages/server/src/tests/utilities/TestConfiguration.ts index 2127e9d1cd..32af88836e 100644 --- a/packages/server/src/tests/utilities/TestConfiguration.ts +++ b/packages/server/src/tests/utilities/TestConfiguration.ts @@ -299,11 +299,11 @@ export default class TestConfiguration { } } - withUser(user: User, f: () => Promise) { + async withUser(user: User, f: () => Promise) { const oldUser = this.user this.user = user try { - return f() + return await f() } finally { this.user = oldUser } From e076c0e5f53d8696d65287fca328427b52de928a Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 5 Mar 2024 15:10:51 +0100 Subject: [PATCH 070/114] Use typed redis clients --- packages/backend-core/src/redis/redis.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/backend-core/src/redis/redis.ts b/packages/backend-core/src/redis/redis.ts index d15453ba62..2280c3f6df 100644 --- a/packages/backend-core/src/redis/redis.ts +++ b/packages/backend-core/src/redis/redis.ts @@ -28,7 +28,7 @@ const DEFAULT_SELECT_DB = SelectableDatabase.DEFAULT // for testing just generate the client once let CLOSED = false -let CLIENTS: { [key: number]: any } = {} +const CLIENTS: Record = {} let CONNECTED = false // mock redis always connected @@ -36,7 +36,7 @@ if (env.MOCK_REDIS) { CONNECTED = true } -function pickClient(selectDb: number): any { +function pickClient(selectDb: number) { return CLIENTS[selectDb] } From f1decee0102c0bc6b4687fffa14f5b445e1c3689 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 5 Mar 2024 14:37:06 +0000 Subject: [PATCH 071/114] Get test passing. --- .../src/api/routes/tests/application.spec.ts | 8 +++++--- .../src/tests/utilities/TestConfiguration.ts | 18 +++++++++++------- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/packages/server/src/api/routes/tests/application.spec.ts b/packages/server/src/api/routes/tests/application.spec.ts index 7424511200..6f948d9977 100644 --- a/packages/server/src/api/routes/tests/application.spec.ts +++ b/packages/server/src/api/routes/tests/application.spec.ts @@ -25,6 +25,8 @@ import { } from "@budibase/types" import tk from "timekeeper" +jest.setTimeout(99999999) + describe("/applications", () => { let config = setup.getConfig() let app: App @@ -257,7 +259,7 @@ describe("/applications", () => { describe("permissions", () => { it.only("should only return apps a user has access to", async () => { - const user = await config.createUser({ + let user = await config.createUser({ builder: { global: false }, admin: { global: false }, }) @@ -280,10 +282,10 @@ describe("/applications", () => { expect(apps).toHaveLength(0) }) - await config.api.user.update({ + user = await config.globalUser({ ...user, builder: { - [config.getAppId()]: true, + apps: [config.getProdAppId()], }, }) diff --git a/packages/server/src/tests/utilities/TestConfiguration.ts b/packages/server/src/tests/utilities/TestConfiguration.ts index 32af88836e..cfe1bf4066 100644 --- a/packages/server/src/tests/utilities/TestConfiguration.ts +++ b/packages/server/src/tests/utilities/TestConfiguration.ts @@ -363,6 +363,7 @@ export default class TestConfiguration { _id, ...existing, ...config, + _rev: existing._rev, email, roles, tenantId, @@ -372,11 +373,12 @@ export default class TestConfiguration { admin, } await sessions.createASession(_id, { - sessionId: "sessionid", + sessionId: this.sessionIdForUser(_id), tenantId: this.getTenantId(), csrfToken: this.csrfToken, }) const resp = await db.put(user) + await cache.user.invalidateUser(_id) return { _rev: resp.rev, ...user, @@ -384,9 +386,7 @@ export default class TestConfiguration { } async createUser(user: Partial = {}): Promise { - const resp = await this.globalUser(user) - await cache.user.invalidateUser(resp._id!) - return resp + return await this.globalUser(user) } async createGroup(roleId: string = roles.BUILTIN_ROLE_IDS.BASIC) { @@ -416,6 +416,10 @@ export default class TestConfiguration { }) } + sessionIdForUser(userId: string): string { + return `sessionid-${userId}` + } + async login({ roleId, userId, @@ -442,13 +446,13 @@ export default class TestConfiguration { }) } await sessions.createASession(userId, { - sessionId: "sessionid", + sessionId: this.sessionIdForUser(userId), tenantId: this.getTenantId(), }) // have to fake this const authObj = { userId, - sessionId: "sessionid", + sessionId: this.sessionIdForUser(userId), tenantId: this.getTenantId(), } const authToken = jwt.sign(authObj, coreEnv.JWT_SECRET as Secret) @@ -470,7 +474,7 @@ export default class TestConfiguration { const user = this.getUser() const authObj: AuthToken = { userId: user._id!, - sessionId: "sessionid", + sessionId: this.sessionIdForUser(user._id!), tenantId, } const authToken = jwt.sign(authObj, coreEnv.JWT_SECRET as Secret) From 488cfea1f432c1dad91680d71afafeacdee5229b Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 5 Mar 2024 14:40:29 +0000 Subject: [PATCH 072/114] Fix typing. --- packages/backend-core/src/security/roles.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/backend-core/src/security/roles.ts b/packages/backend-core/src/security/roles.ts index 213c65e18e..01473ad991 100644 --- a/packages/backend-core/src/security/roles.ts +++ b/packages/backend-core/src/security/roles.ts @@ -184,7 +184,7 @@ export async function getRole( return cloneDeep(BUILTIN_ROLES.PUBLIC) } // only throw an error if there is no role at all - if (Object.keys(role || {}).length === 0) { + if (!role || Object.keys(role).length === 0) { throw err } } From a5d6d094e63def29122ce5d3b73470681523688f Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 5 Mar 2024 16:28:31 +0100 Subject: [PATCH 073/114] Update types --- packages/backend-core/package.json | 2 +- packages/types/package.json | 2 +- yarn.lock | 11 ++++++----- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json index 3f8c34f823..90b3316c3f 100644 --- a/packages/backend-core/package.json +++ b/packages/backend-core/package.json @@ -67,7 +67,7 @@ "@types/lodash": "4.14.200", "@types/node-fetch": "2.6.4", "@types/pouchdb": "6.4.0", - "@types/redlock": "4.0.3", + "@types/redlock": "4.0.7", "@types/semver": "7.3.7", "@types/tar-fs": "2.0.1", "@types/uuid": "8.3.4", diff --git a/packages/types/package.json b/packages/types/package.json index ce4fce95fb..558e55a632 100644 --- a/packages/types/package.json +++ b/packages/types/package.json @@ -18,7 +18,7 @@ "@budibase/nano": "10.1.5", "@types/koa": "2.13.4", "@types/pouchdb": "6.4.0", - "@types/redlock": "4.0.3", + "@types/redlock": "4.0.7", "rimraf": "3.0.2", "typescript": "5.2.2" }, diff --git a/yarn.lock b/yarn.lock index 260ae3870a..2f9f558e2c 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5408,7 +5408,7 @@ resolved "https://registry.yarnpkg.com/@types/http-errors/-/http-errors-2.0.1.tgz#20172f9578b225f6c7da63446f56d4ce108d5a65" integrity sha512-/K3ds8TRAfBvi5vfjuz8y6+GiAYBZ0x4tXv1Av6CWBWn0IlADc+ZX9pMq7oU0fNQPnBwIZl3rmeLp6SBApbxSQ== -"@types/ioredis@4.28.10": +"@types/ioredis@4.28.10", "@types/ioredis@^4.28.10": version "4.28.10" resolved "https://registry.yarnpkg.com/@types/ioredis/-/ioredis-4.28.10.tgz#40ceb157a4141088d1394bb87c98ed09a75a06ff" integrity sha512-69LyhUgrXdgcNDv7ogs1qXZomnfOEnSmrmMFqKgt1XMJxmoOSG/u3wYy13yACIfKuMJ8IhKgHafDO3sx19zVQQ== @@ -5896,12 +5896,13 @@ dependencies: "@types/node" "*" -"@types/redlock@4.0.3": - version "4.0.3" - resolved "https://registry.yarnpkg.com/@types/redlock/-/redlock-4.0.3.tgz#aeab5fe5f0d433a125f6dcf9a884372ac0cddd4b" - integrity sha512-mcvvrquwREbAqyZALNBIlf49AL9Aa324BG+J/Dv4TAP8g+nxQMBI4/APNqqS99QEY7VTNT9XvsaczCVGK8uNnQ== +"@types/redlock@4.0.7": + version "4.0.7" + resolved "https://registry.yarnpkg.com/@types/redlock/-/redlock-4.0.7.tgz#33ed56f22a38d6b2f2e6ae5ed1b3fc1875a08e6b" + integrity sha512-5D6egBv0fCfdbmnCETjEynVuiwFMEFFc3YFjh9EwhaaVTAi0YmB6UI1swq1S1rjIu+n27ppmlTFDK3D3cadJqg== dependencies: "@types/bluebird" "*" + "@types/ioredis" "^4.28.10" "@types/redis" "^2.8.0" "@types/request@^2.48.7": From 56870bed5b0bc2c4479c4107e78193e36195a795 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 5 Mar 2024 16:30:45 +0100 Subject: [PATCH 074/114] Typings --- packages/backend-core/src/redis/redis.ts | 21 ++++++++++++------- .../backend-core/src/redis/redlockImpl.ts | 2 +- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/packages/backend-core/src/redis/redis.ts b/packages/backend-core/src/redis/redis.ts index 2280c3f6df..f8f0c9f3d7 100644 --- a/packages/backend-core/src/redis/redis.ts +++ b/packages/backend-core/src/redis/redis.ts @@ -1,5 +1,5 @@ import env from "../environment" -import Redis from "ioredis" +import Redis, { Cluster } from "ioredis" // mock-redis doesn't have any typing let MockRedis: any | undefined if (env.MOCK_REDIS) { @@ -28,7 +28,7 @@ const DEFAULT_SELECT_DB = SelectableDatabase.DEFAULT // for testing just generate the client once let CLOSED = false -const CLIENTS: Record = {} +const CLIENTS: Record = {} let CONNECTED = false // mock redis always connected @@ -201,12 +201,15 @@ class RedisWrapper { key = `${db}${SEPARATOR}${key}` let stream if (CLUSTERED) { - let node = this.getClient().nodes("master") + let node = (this.getClient() as Cluster).nodes("master") stream = node[0].scanStream({ match: key + "*", count: 100 }) } else { - stream = this.getClient().scanStream({ match: key + "*", count: 100 }) + stream = (this.getClient() as Redis).scanStream({ + match: key + "*", + count: 100, + }) } - return promisifyStream(stream, this.getClient()) + return promisifyStream(stream, this.getClient() as any) } async keys(pattern: string) { @@ -221,14 +224,16 @@ class RedisWrapper { async get(key: string) { const db = this._db - let response = await this.getClient().get(addDbPrefix(db, key)) + const response = await this.getClient().get(addDbPrefix(db, key)) // overwrite the prefixed key + // @ts-ignore if (response != null && response.key) { + // @ts-ignore response.key = key } // if its not an object just return the response try { - return JSON.parse(response) + return JSON.parse(response!) } catch (err) { return response } @@ -280,7 +285,7 @@ class RedisWrapper { return this.getClient().ttl(prefixedKey) } - async setExpiry(key: string, expirySeconds: number | null) { + async setExpiry(key: string, expirySeconds: number) { const db = this._db const prefixedKey = addDbPrefix(db, key) await this.getClient().expire(prefixedKey, expirySeconds) diff --git a/packages/backend-core/src/redis/redlockImpl.ts b/packages/backend-core/src/redis/redlockImpl.ts index 7009dc6f55..adeb5b12ec 100644 --- a/packages/backend-core/src/redis/redlockImpl.ts +++ b/packages/backend-core/src/redis/redlockImpl.ts @@ -72,7 +72,7 @@ const OPTIONS: Record = { export async function newRedlock(opts: Redlock.Options = {}) { const options = { ...OPTIONS.DEFAULT, ...opts } const redisWrapper = await getLockClient() - const client = redisWrapper.getClient() + const client = redisWrapper.getClient() as any return new Redlock([client], options) } From a4288a9dd3e21dd99c3ac825effa7b1a40dadb63 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 5 Mar 2024 16:41:49 +0100 Subject: [PATCH 075/114] Basic test --- .../src/redis/tests/redis.spec.ts | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 packages/backend-core/src/redis/tests/redis.spec.ts diff --git a/packages/backend-core/src/redis/tests/redis.spec.ts b/packages/backend-core/src/redis/tests/redis.spec.ts new file mode 100644 index 0000000000..d082b6b617 --- /dev/null +++ b/packages/backend-core/src/redis/tests/redis.spec.ts @@ -0,0 +1,21 @@ +import { generator, structures } from "../../../tests" +import RedisWrapper from "../redis" + +describe("redis", () => { + const redis = new RedisWrapper(structures.db.id()) + + beforeAll(async () => { + await redis.init() + }) + + describe("store", () => { + it("a basic value can be persisted", async () => { + const key = structures.uuid() + const value = generator.word() + + await redis.store(key, value) + + expect(await redis.get(key)).toEqual(value) + }) + }) +}) From 49db47e1fd75b63b7bf06794e5418a54c266db3d Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 5 Mar 2024 16:46:33 +0100 Subject: [PATCH 076/114] Add bulk store --- packages/backend-core/src/redis/redis.ts | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/packages/backend-core/src/redis/redis.ts b/packages/backend-core/src/redis/redis.ts index f8f0c9f3d7..076f64b1ea 100644 --- a/packages/backend-core/src/redis/redis.ts +++ b/packages/backend-core/src/redis/redis.ts @@ -279,6 +279,19 @@ class RedisWrapper { } } + async bulkStore( + data: Record, + expirySeconds: number | null = null + ) { + const client = this.getClient() + + const dataToStore = Object.entries(data).reduce((acc, [key, value]) => { + acc[addDbPrefix(this._db, key)] = value + return acc + }, {} as Record) + await client.mset(dataToStore) + } + async getTTL(key: string) { const db = this._db const prefixedKey = addDbPrefix(db, key) From d9a5899b2770e7140c360f0eaf24b9e3ccb84d07 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 5 Mar 2024 16:49:45 +0100 Subject: [PATCH 077/114] Bulk store test --- .../src/redis/tests/redis.spec.ts | 24 +++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/packages/backend-core/src/redis/tests/redis.spec.ts b/packages/backend-core/src/redis/tests/redis.spec.ts index d082b6b617..6333573e6e 100644 --- a/packages/backend-core/src/redis/tests/redis.spec.ts +++ b/packages/backend-core/src/redis/tests/redis.spec.ts @@ -2,9 +2,10 @@ import { generator, structures } from "../../../tests" import RedisWrapper from "../redis" describe("redis", () => { - const redis = new RedisWrapper(structures.db.id()) + let redis: RedisWrapper - beforeAll(async () => { + beforeEach(async () => { + redis = new RedisWrapper(structures.db.id()) await redis.init() }) @@ -18,4 +19,23 @@ describe("redis", () => { expect(await redis.get(key)).toEqual(value) }) }) + + describe("bulkStore", () => { + it("a basic object can be persisted", async () => { + const data = generator + .unique(() => generator.word(), 10) + .reduce((acc, key) => { + acc[key] = generator.word() + return acc + }, {} as Record) + + await redis.bulkStore(data) + + for (const [key, value] of Object.entries(data)) { + expect(await redis.get(key)).toEqual(value) + } + + expect(await redis.keys("*")).toHaveLength(10) + }) + }) }) From 1b0a943e13fcb46cc91be18f794c3ff2a4c95684 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 5 Mar 2024 17:04:46 +0100 Subject: [PATCH 078/114] Atomic expires --- packages/backend-core/src/redis/redis.ts | 19 ++++++++++++++++++- .../src/redis/tests/redis.spec.ts | 19 +++++++++++++++++++ 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/packages/backend-core/src/redis/redis.ts b/packages/backend-core/src/redis/redis.ts index 076f64b1ea..18152aac72 100644 --- a/packages/backend-core/src/redis/redis.ts +++ b/packages/backend-core/src/redis/redis.ts @@ -289,7 +289,24 @@ class RedisWrapper { acc[addDbPrefix(this._db, key)] = value return acc }, {} as Record) - await client.mset(dataToStore) + + const luaScript = ` + for i, key in ipairs(KEYS) do + redis.call('MSET', key, ARGV[i]) + ${ + expirySeconds !== null + ? `redis.call('EXPIRE', key, ARGV[#ARGV])` + : "" + } + end + ` + const keys = Object.keys(dataToStore) + let values = Object.values(dataToStore) + if (expirySeconds !== null) { + values.push(expirySeconds) + } + + await client.eval(luaScript, keys.length, ...keys, ...values) } async getTTL(key: string) { diff --git a/packages/backend-core/src/redis/tests/redis.spec.ts b/packages/backend-core/src/redis/tests/redis.spec.ts index 6333573e6e..e3e4ae7247 100644 --- a/packages/backend-core/src/redis/tests/redis.spec.ts +++ b/packages/backend-core/src/redis/tests/redis.spec.ts @@ -37,5 +37,24 @@ describe("redis", () => { expect(await redis.keys("*")).toHaveLength(10) }) + + it("a bulk store can be persisted with TTL", async () => { + const ttl = 500 + const data = generator + .unique(() => generator.word(), 10) + .reduce((acc, key) => { + acc[key] = generator.word() + return acc + }, {} as Record) + + await redis.bulkStore(data, ttl) + + for (const [key, value] of Object.entries(data)) { + expect(await redis.get(key)).toEqual(value) + expect(await redis.getTTL(key)).toEqual(ttl) + } + + expect(await redis.keys("*")).toHaveLength(10) + }) }) }) From 3baf981d4826c68360261a1c57c9a97c1a4ed267 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 5 Mar 2024 17:08:08 +0100 Subject: [PATCH 079/114] Add TTL tests --- .../src/redis/tests/redis.spec.ts | 43 ++++++++++++++----- 1 file changed, 33 insertions(+), 10 deletions(-) diff --git a/packages/backend-core/src/redis/tests/redis.spec.ts b/packages/backend-core/src/redis/tests/redis.spec.ts index e3e4ae7247..13f2c676c9 100644 --- a/packages/backend-core/src/redis/tests/redis.spec.ts +++ b/packages/backend-core/src/redis/tests/redis.spec.ts @@ -21,13 +21,17 @@ describe("redis", () => { }) describe("bulkStore", () => { - it("a basic object can be persisted", async () => { - const data = generator - .unique(() => generator.word(), 10) + function createRandomObject(keyLength: number) { + return generator + .unique(() => generator.word(), keyLength) .reduce((acc, key) => { acc[key] = generator.word() return acc }, {} as Record) + } + + it("a basic object can be persisted", async () => { + const data = createRandomObject(10) await redis.bulkStore(data) @@ -38,14 +42,20 @@ describe("redis", () => { expect(await redis.keys("*")).toHaveLength(10) }) + it("no TTL is set by default", async () => { + const data = createRandomObject(10) + + await redis.bulkStore(data) + + for (const [key, value] of Object.entries(data)) { + expect(await redis.get(key)).toEqual(value) + expect(await redis.getTTL(key)).toEqual(-1) + } + }) + it("a bulk store can be persisted with TTL", async () => { const ttl = 500 - const data = generator - .unique(() => generator.word(), 10) - .reduce((acc, key) => { - acc[key] = generator.word() - return acc - }, {} as Record) + const data = createRandomObject(8) await redis.bulkStore(data, ttl) @@ -54,7 +64,20 @@ describe("redis", () => { expect(await redis.getTTL(key)).toEqual(ttl) } - expect(await redis.keys("*")).toHaveLength(10) + expect(await redis.keys("*")).toHaveLength(8) + }) + + it("setting a TTL of -1 will not persist the key", async () => { + const ttl = -1 + const data = createRandomObject(5) + + await redis.bulkStore(data, ttl) + + for (const [key, value] of Object.entries(data)) { + expect(await redis.get(key)).toBe(null) + } + + expect(await redis.keys("*")).toHaveLength(0) }) }) }) From a332c058ce8f1fb371c84c7beb2aff44c1590354 Mon Sep 17 00:00:00 2001 From: Michael Drury Date: Tue, 5 Mar 2024 16:19:21 +0000 Subject: [PATCH 080/114] Disabling aliasing on writes (create, update, delete) for MySQL/MS-SQL datasources. --- .../server/src/api/controllers/row/alias.ts | 48 +++++++++++++++---- packages/server/src/integrations/base/sql.ts | 10 ++-- packages/server/src/integrations/index.ts | 11 +++-- packages/server/src/sdk/app/rows/utils.ts | 20 ++++++++ 4 files changed, 74 insertions(+), 15 deletions(-) diff --git a/packages/server/src/api/controllers/row/alias.ts b/packages/server/src/api/controllers/row/alias.ts index 46b090bb97..0adcfaa582 100644 --- a/packages/server/src/api/controllers/row/alias.ts +++ b/packages/server/src/api/controllers/row/alias.ts @@ -1,12 +1,16 @@ import { - QueryJson, - SearchFilters, - Table, - Row, + Datasource, DatasourcePlusQueryResponse, + Operation, + QueryJson, + Row, + SearchFilters, } from "@budibase/types" -import { getDatasourceAndQuery } from "../../../sdk/app/rows/utils" +import { getSQLClient } from "../../../sdk/app/rows/utils" import { cloneDeep } from "lodash" +import sdk from "../../../sdk" +import { makeExternalQuery } from "../../../integrations/base/query" +import { SqlClient } from "../../../integrations/utils" class CharSequence { static alphabet = "abcdefghijklmnopqrstuvwxyz" @@ -43,6 +47,32 @@ export default class AliasTables { this.charSeq = new CharSequence() } + isAliasingEnabled(json: QueryJson, datasource: Datasource) { + const fieldLength = json.resource?.fields?.length + if (!fieldLength || fieldLength <= 0) { + return false + } + const writeOperations = [ + Operation.CREATE, + Operation.UPDATE, + Operation.DELETE, + ] + try { + const sqlClient = getSQLClient(datasource) + const isWrite = writeOperations.includes(json.endpoint.operation) + if ( + isWrite && + (sqlClient === SqlClient.MY_SQL || sqlClient === SqlClient.MS_SQL) + ) { + return false + } + } catch (err) { + // if we can't get an SQL client, we can't alias + return false + } + return true + } + getAlias(tableName: string) { if (this.aliases[tableName]) { return this.aliases[tableName] @@ -111,8 +141,10 @@ export default class AliasTables { } async queryWithAliasing(json: QueryJson): DatasourcePlusQueryResponse { - const fieldLength = json.resource?.fields?.length - const aliasingEnabled = fieldLength && fieldLength > 0 + const datasourceId = json.endpoint.datasourceId + const datasource = await sdk.datasources.get(datasourceId) + + const aliasingEnabled = this.isAliasingEnabled(json, datasource) if (aliasingEnabled) { json = cloneDeep(json) // run through the query json to update anywhere a table may be used @@ -158,7 +190,7 @@ export default class AliasTables { } json.tableAliases = invertedTableAliases } - const response = await getDatasourceAndQuery(json) + const response = await makeExternalQuery(datasource, json) if (Array.isArray(response) && aliasingEnabled) { return this.reverse(response) } else { diff --git a/packages/server/src/integrations/base/sql.ts b/packages/server/src/integrations/base/sql.ts index c8acb606b3..be1883c8ec 100644 --- a/packages/server/src/integrations/base/sql.ts +++ b/packages/server/src/integrations/base/sql.ts @@ -435,10 +435,12 @@ class InternalBuilder { aliases?: QueryJson["tableAliases"] ): Knex.QueryBuilder { const tableName = endpoint.entityId - const tableAliased = aliases?.[tableName] - ? `${tableName} as ${aliases?.[tableName]}` - : tableName - let query = knex(tableAliased) + const tableAlias = aliases?.[tableName] + let table: string | Record = tableName + if (tableAlias) { + table = { [tableAlias]: tableName } + } + let query = knex(table) if (endpoint.schema) { query = query.withSchema(endpoint.schema) } diff --git a/packages/server/src/integrations/index.ts b/packages/server/src/integrations/index.ts index ee2bb23f23..18c46b9260 100644 --- a/packages/server/src/integrations/index.ts +++ b/packages/server/src/integrations/index.ts @@ -14,13 +14,18 @@ import firebase from "./firebase" import redis from "./redis" import snowflake from "./snowflake" import oracle from "./oracle" -import { SourceName, Integration, PluginType } from "@budibase/types" +import { + SourceName, + Integration, + PluginType, + IntegrationBase, +} from "@budibase/types" import { getDatasourcePlugin } from "../utilities/fileSystem" import env from "../environment" import cloneDeep from "lodash/cloneDeep" import sdk from "../sdk" -const DEFINITIONS: Record = { +const DEFINITIONS: { [key: SourceName]: Integration | undefined } = { [SourceName.POSTGRES]: postgres.schema, [SourceName.DYNAMODB]: dynamodb.schema, [SourceName.MONGODB]: mongodb.schema, @@ -40,7 +45,7 @@ const DEFINITIONS: Record = { [SourceName.BUDIBASE]: undefined, } -const INTEGRATIONS: Record = { +const INTEGRATIONS: { [key: SourceName]: IntegrationBase | undefined } = { [SourceName.POSTGRES]: postgres.integration, [SourceName.DYNAMODB]: dynamodb.integration, [SourceName.MONGODB]: mongodb.integration, diff --git a/packages/server/src/sdk/app/rows/utils.ts b/packages/server/src/sdk/app/rows/utils.ts index a8052462a9..e090045925 100644 --- a/packages/server/src/sdk/app/rows/utils.ts +++ b/packages/server/src/sdk/app/rows/utils.ts @@ -7,11 +7,31 @@ import { Table, TableSchema, DatasourcePlusQueryResponse, + Datasource, + SourceName, } from "@budibase/types" import { makeExternalQuery } from "../../../integrations/base/query" import { Format } from "../../../api/controllers/view/exporters" import sdk from "../.." import { isRelationshipColumn } from "../../../db/utils" +import { SqlClient } from "../../../integrations/utils" + +export function getSQLClient(datasource: Datasource): SqlClient { + if (!datasource.isSQL) { + throw new Error("Cannot get SQL Client for non-SQL datasource") + } + switch (datasource.source) { + case SourceName.POSTGRES: + return SqlClient.POSTGRES + case SourceName.MYSQL: + return SqlClient.MY_SQL + case SourceName.ORACLE: + return SqlClient.ORACLE + case SourceName.SQL_SERVER: + return SqlClient.MS_SQL + } + throw new Error("Unable to find a valid SQL client") +} export async function getDatasourceAndQuery( json: QueryJson From 8f9e8b60c328af4f635c98759b475f08668d7eed Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 5 Mar 2024 17:19:26 +0100 Subject: [PATCH 081/114] Fix types --- packages/backend-core/src/redis/redis.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/backend-core/src/redis/redis.ts b/packages/backend-core/src/redis/redis.ts index 18152aac72..99613e7c32 100644 --- a/packages/backend-core/src/redis/redis.ts +++ b/packages/backend-core/src/redis/redis.ts @@ -28,7 +28,7 @@ const DEFAULT_SELECT_DB = SelectableDatabase.DEFAULT // for testing just generate the client once let CLOSED = false -const CLIENTS: Record = {} +const CLIENTS: Record = {} let CONNECTED = false // mock redis always connected @@ -201,7 +201,7 @@ class RedisWrapper { key = `${db}${SEPARATOR}${key}` let stream if (CLUSTERED) { - let node = (this.getClient() as Cluster).nodes("master") + let node = (this.getClient() as never as Cluster).nodes("master") stream = node[0].scanStream({ match: key + "*", count: 100 }) } else { stream = (this.getClient() as Redis).scanStream({ From f2330144de5bfdddc3d9b4021540e0cc9976c3ca Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 5 Mar 2024 17:57:28 +0100 Subject: [PATCH 082/114] Clean --- packages/backend-core/src/redis/redis.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/backend-core/src/redis/redis.ts b/packages/backend-core/src/redis/redis.ts index 99613e7c32..59583da366 100644 --- a/packages/backend-core/src/redis/redis.ts +++ b/packages/backend-core/src/redis/redis.ts @@ -301,7 +301,7 @@ class RedisWrapper { end ` const keys = Object.keys(dataToStore) - let values = Object.values(dataToStore) + const values = Object.values(dataToStore) if (expirySeconds !== null) { values.push(expirySeconds) } From 82ff748fd950e79e60daad95268d5c34490d25f7 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 5 Mar 2024 18:10:33 +0100 Subject: [PATCH 083/114] Add complex object tests --- .../src/redis/tests/redis.spec.ts | 31 +++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/packages/backend-core/src/redis/tests/redis.spec.ts b/packages/backend-core/src/redis/tests/redis.spec.ts index 13f2c676c9..eb32172edd 100644 --- a/packages/backend-core/src/redis/tests/redis.spec.ts +++ b/packages/backend-core/src/redis/tests/redis.spec.ts @@ -18,14 +18,26 @@ describe("redis", () => { expect(await redis.get(key)).toEqual(value) }) + + it("objects can be persisted", async () => { + const key = structures.uuid() + const value = { [generator.word()]: generator.word() } + + await redis.store(key, value) + + expect(await redis.get(key)).toEqual(value) + }) }) describe("bulkStore", () => { - function createRandomObject(keyLength: number) { + function createRandomObject( + keyLength: number, + valueGenerator: () => any = () => generator.word() + ) { return generator .unique(() => generator.word(), keyLength) .reduce((acc, key) => { - acc[key] = generator.word() + acc[key] = valueGenerator() return acc }, {} as Record) } @@ -42,6 +54,21 @@ describe("redis", () => { expect(await redis.keys("*")).toHaveLength(10) }) + it("a complex object can be persisted", async () => { + const data = { + ...createRandomObject(10, () => createRandomObject(5)), + ...createRandomObject(5), + } + + await redis.bulkStore(data) + + for (const [key, value] of Object.entries(data)) { + expect(await redis.get(key)).toEqual(value) + } + + expect(await redis.keys("*")).toHaveLength(10) + }) + it("no TTL is set by default", async () => { const data = createRandomObject(10) From de0527384aebd2703fd7eec58818366627da8b10 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 5 Mar 2024 18:10:45 +0100 Subject: [PATCH 084/114] Support complex objects --- packages/backend-core/src/redis/redis.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/backend-core/src/redis/redis.ts b/packages/backend-core/src/redis/redis.ts index 59583da366..8cfa3db5c1 100644 --- a/packages/backend-core/src/redis/redis.ts +++ b/packages/backend-core/src/redis/redis.ts @@ -286,7 +286,8 @@ class RedisWrapper { const client = this.getClient() const dataToStore = Object.entries(data).reduce((acc, [key, value]) => { - acc[addDbPrefix(this._db, key)] = value + acc[addDbPrefix(this._db, key)] = + typeof value === "object" ? JSON.stringify(value) : value return acc }, {} as Record) From a093cfca993d585ae5a83e933dcacccee35c4bf5 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 5 Mar 2024 18:11:12 +0100 Subject: [PATCH 085/114] Fix test --- packages/backend-core/src/redis/tests/redis.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/backend-core/src/redis/tests/redis.spec.ts b/packages/backend-core/src/redis/tests/redis.spec.ts index eb32172edd..1fd40acc37 100644 --- a/packages/backend-core/src/redis/tests/redis.spec.ts +++ b/packages/backend-core/src/redis/tests/redis.spec.ts @@ -66,7 +66,7 @@ describe("redis", () => { expect(await redis.get(key)).toEqual(value) } - expect(await redis.keys("*")).toHaveLength(10) + expect(await redis.keys("*")).toHaveLength(15) }) it("no TTL is set by default", async () => { From 0520c0c54083eed0900f12017dfcee2b50257bec Mon Sep 17 00:00:00 2001 From: Michael Drury Date: Tue, 5 Mar 2024 17:27:35 +0000 Subject: [PATCH 086/114] Adding tests to confirm when aliasing should be used. --- .../server/src/api/controllers/row/alias.ts | 29 +++----- .../src/integrations/tests/sqlAlias.spec.ts | 74 ++++++++++++++++++- 2 files changed, 84 insertions(+), 19 deletions(-) diff --git a/packages/server/src/api/controllers/row/alias.ts b/packages/server/src/api/controllers/row/alias.ts index 0adcfaa582..28eaa95bf8 100644 --- a/packages/server/src/api/controllers/row/alias.ts +++ b/packages/server/src/api/controllers/row/alias.ts @@ -1,17 +1,17 @@ -import { - Datasource, - DatasourcePlusQueryResponse, - Operation, - QueryJson, - Row, - SearchFilters, -} from "@budibase/types" +import { Datasource, DatasourcePlusQueryResponse, Operation, QueryJson, Row, SearchFilters } from "@budibase/types" import { getSQLClient } from "../../../sdk/app/rows/utils" import { cloneDeep } from "lodash" import sdk from "../../../sdk" import { makeExternalQuery } from "../../../integrations/base/query" import { SqlClient } from "../../../integrations/utils" +const WRITE_OPERATIONS: Operation[] = [ + Operation.CREATE, + Operation.UPDATE, + Operation.DELETE, +] +const DISABLED_WRITE_CLIENTS: SqlClient[] = [SqlClient.MY_SQL, SqlClient.MS_SQL, SqlClient.ORACLE] + class CharSequence { static alphabet = "abcdefghijklmnopqrstuvwxyz" counters: number[] @@ -52,18 +52,11 @@ export default class AliasTables { if (!fieldLength || fieldLength <= 0) { return false } - const writeOperations = [ - Operation.CREATE, - Operation.UPDATE, - Operation.DELETE, - ] try { const sqlClient = getSQLClient(datasource) - const isWrite = writeOperations.includes(json.endpoint.operation) - if ( - isWrite && - (sqlClient === SqlClient.MY_SQL || sqlClient === SqlClient.MS_SQL) - ) { + const isWrite = WRITE_OPERATIONS.includes(json.endpoint.operation) + const isDisabledClient = DISABLED_WRITE_CLIENTS.includes(sqlClient) + if (isWrite && isDisabledClient) { return false } } catch (err) { diff --git a/packages/server/src/integrations/tests/sqlAlias.spec.ts b/packages/server/src/integrations/tests/sqlAlias.spec.ts index fe9798aaa0..70dda8c335 100644 --- a/packages/server/src/integrations/tests/sqlAlias.spec.ts +++ b/packages/server/src/integrations/tests/sqlAlias.spec.ts @@ -1,4 +1,4 @@ -import { QueryJson } from "@budibase/types" +import { Datasource, Operation, QueryJson, SourceName } from "@budibase/types" import { join } from "path" import Sql from "../base/sql" import { SqlClient } from "../utils" @@ -198,6 +198,78 @@ describe("Captures of real examples", () => { }) }) + describe("check aliasing is disabled/enabled", () => { + const tables = ["tableA", "tableB"] + + function getDatasource(source: SourceName): Datasource { + return { + source, + type: "datasource", + isSQL: true, + } + } + + function getQuery(op: Operation, fields: string[] = ["a"]): QueryJson { + return { + endpoint: { datasourceId: "", entityId: "", operation: op }, + resource: { + fields, + } + } + } + + it("should check for Postgres aliased status", () => { + const aliasing = new AliasTables(tables) + const datasource = getDatasource(SourceName.POSTGRES) + expect(aliasing.isAliasingEnabled(getQuery(Operation.CREATE), datasource)).toEqual(true) + expect(aliasing.isAliasingEnabled(getQuery(Operation.READ), datasource)).toEqual(true) + expect(aliasing.isAliasingEnabled(getQuery(Operation.UPDATE), datasource)).toEqual(true) + expect(aliasing.isAliasingEnabled(getQuery(Operation.DELETE), datasource)).toEqual(true) + }) + + it("should check for MS-SQL aliased status", () => { + const aliasing = new AliasTables(tables) + const datasource = getDatasource(SourceName.SQL_SERVER) + expect(aliasing.isAliasingEnabled(getQuery(Operation.CREATE), datasource)).toEqual(false) + expect(aliasing.isAliasingEnabled(getQuery(Operation.READ), datasource)).toEqual(true) + expect(aliasing.isAliasingEnabled(getQuery(Operation.UPDATE), datasource)).toEqual(false) + expect(aliasing.isAliasingEnabled(getQuery(Operation.DELETE), datasource)).toEqual(false) + }) + + it("should check for MySQL aliased status", () => { + const aliasing = new AliasTables(tables) + const datasource = getDatasource(SourceName.MYSQL) + expect(aliasing.isAliasingEnabled(getQuery(Operation.CREATE), datasource)).toEqual(false) + expect(aliasing.isAliasingEnabled(getQuery(Operation.READ), datasource)).toEqual(true) + expect(aliasing.isAliasingEnabled(getQuery(Operation.UPDATE), datasource)).toEqual(false) + expect(aliasing.isAliasingEnabled(getQuery(Operation.DELETE), datasource)).toEqual(false) + }) + + it("should check for Oracle aliased status", () => { + const aliasing = new AliasTables(tables) + const datasource = getDatasource(SourceName.ORACLE) + expect(aliasing.isAliasingEnabled(getQuery(Operation.CREATE), datasource)).toEqual(false) + expect(aliasing.isAliasingEnabled(getQuery(Operation.READ), datasource)).toEqual(true) + expect(aliasing.isAliasingEnabled(getQuery(Operation.UPDATE), datasource)).toEqual(false) + expect(aliasing.isAliasingEnabled(getQuery(Operation.DELETE), datasource)).toEqual(false) + }) + + it("should disable aliasing for non-SQL datasources", () => { + const aliasing = new AliasTables(tables) + expect(aliasing.isAliasingEnabled(getQuery(Operation.READ), { + source: SourceName.GOOGLE_SHEETS, + type: "datasource", + isSQL: false, + })) + }) + + it("should disable when no fields", () => { + const aliasing = new AliasTables(tables) + const datasource = getDatasource(SourceName.POSTGRES) + expect(aliasing.isAliasingEnabled(getQuery(Operation.READ, []), datasource)).toEqual(false) + }) + }) + describe("check some edge cases", () => { const tableNames = ["hello", "world"] From 182a1df9606f98da9791cb50df8355fc54eb21c2 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 5 Mar 2024 17:35:04 +0000 Subject: [PATCH 087/114] Fix the bug, I think. --- packages/backend-core/src/db/Replication.ts | 36 ++++++--- packages/backend-core/src/security/roles.ts | 5 +- packages/server/src/api/controllers/role.ts | 10 +++ .../src/api/routes/tests/application.spec.ts | 81 +++++++++---------- packages/types/src/documents/app/role.ts | 1 + 5 files changed, 72 insertions(+), 61 deletions(-) diff --git a/packages/backend-core/src/db/Replication.ts b/packages/backend-core/src/db/Replication.ts index f91a37ce8f..12c11eb9e2 100644 --- a/packages/backend-core/src/db/Replication.ts +++ b/packages/backend-core/src/db/Replication.ts @@ -1,17 +1,18 @@ +import PouchDB from "pouchdb" import { getPouchDB, closePouchDB } from "./couch" import { DocumentType } from "../constants" class Replication { - source: any - target: any - replication: any + source: PouchDB.Database + target: PouchDB.Database + replication?: Promise /** * * @param source - the DB you want to replicate or rollback to * @param target - the DB you want to replicate to, or rollback from */ - constructor({ source, target }: any) { + constructor({ source, target }: { source: string; target: string }) { this.source = getPouchDB(source) this.target = getPouchDB(target) } @@ -40,7 +41,7 @@ class Replication { * Two way replication operation, intended to be promise based. * @param opts - PouchDB replication options */ - sync(opts = {}) { + sync(opts: PouchDB.Replication.SyncOptions = {}) { this.replication = this.promisify(this.source.sync, opts) return this.replication } @@ -49,18 +50,31 @@ class Replication { * One way replication operation, intended to be promise based. * @param opts - PouchDB replication options */ - replicate(opts = {}) { + replicate(opts: PouchDB.Replication.ReplicateOptions = {}) { this.replication = this.promisify(this.source.replicate.to, opts) return this.replication } - appReplicateOpts() { + appReplicateOpts( + opts: PouchDB.Replication.ReplicateOptions = {} + ): PouchDB.Replication.ReplicateOptions { + if (typeof opts.filter === "string") { + return opts + } + + const filter = opts.filter + delete opts.filter + return { - filter: (doc: any) => { + ...opts, + filter: (doc: any, params: any) => { if (doc._id && doc._id.startsWith(DocumentType.AUTOMATION_LOG)) { return false } - return doc._id !== DocumentType.APP_METADATA + if (doc._id === DocumentType.APP_METADATA) { + return false + } + return filter ? filter(doc, params) : true }, } } @@ -75,10 +89,6 @@ class Replication { // take the opportunity to remove deleted tombstones await this.replicate() } - - cancel() { - this.replication.cancel() - } } export default Replication diff --git a/packages/backend-core/src/security/roles.ts b/packages/backend-core/src/security/roles.ts index 01473ad991..a64be6b319 100644 --- a/packages/backend-core/src/security/roles.ts +++ b/packages/backend-core/src/security/roles.ts @@ -101,10 +101,7 @@ export function getBuiltinRole(roleId: string): Role | undefined { /** * Works through the inheritance ranks to see how far up the builtin stack this ID is. */ -export function builtinRoleToNumber(id?: string) { - if (!id) { - return 0 - } +export function builtinRoleToNumber(id: string) { const builtins = getBuiltinRoles() const MAX = Object.values(builtins).length + 1 if (id === BUILTIN_IDS.ADMIN || id === BUILTIN_IDS.BUILDER) { diff --git a/packages/server/src/api/controllers/role.ts b/packages/server/src/api/controllers/role.ts index b3eb61a255..fff58da86e 100644 --- a/packages/server/src/api/controllers/role.ts +++ b/packages/server/src/api/controllers/role.ts @@ -106,6 +106,16 @@ export async function save(ctx: UserCtx) { ) role._rev = result.rev ctx.body = role + + const replication = new dbCore.Replication({ + source: context.getDevAppDB().name, + target: context.getProdAppDB().name, + }) + await replication.replicate({ + filter: (doc: any, params: any) => { + return doc._id === _id + }, + }) } export async function destroy(ctx: UserCtx) { diff --git a/packages/server/src/api/routes/tests/application.spec.ts b/packages/server/src/api/routes/tests/application.spec.ts index 6f948d9977..63c9fe44b8 100644 --- a/packages/server/src/api/routes/tests/application.spec.ts +++ b/packages/server/src/api/routes/tests/application.spec.ts @@ -16,16 +16,9 @@ import * as setup from "./utilities" import { AppStatus } from "../../../db/utils" import { events, utils, context } from "@budibase/backend-core" import env from "../../../environment" -import { - PermissionLevel, - type App, - INTERNAL_TABLE_SOURCE_ID, - TableSourceType, - FieldType, -} from "@budibase/types" +import { type App } from "@budibase/types" import tk from "timekeeper" - -jest.setTimeout(99999999) +import * as uuid from "uuid" describe("/applications", () => { let config = setup.getConfig() @@ -258,25 +251,12 @@ describe("/applications", () => { }) describe("permissions", () => { - it.only("should only return apps a user has access to", async () => { + it("should only return apps a user has access to", async () => { let user = await config.createUser({ builder: { global: false }, admin: { global: false }, }) - const table = await config.api.table.save({ - name: "table", - type: "table", - sourceId: INTERNAL_TABLE_SOURCE_ID, - sourceType: TableSourceType.INTERNAL, - schema: { - name: { - type: FieldType.STRING, - name: "name", - }, - }, - }) - await config.withUser(user, async () => { const apps = await config.api.application.fetch() expect(apps).toHaveLength(0) @@ -295,25 +275,12 @@ describe("/applications", () => { }) }) - it("should only return apps a user has access to through a custom role on a group", async () => { - const user = await config.createUser({ + it("should only return apps a user has access to through a custom role", async () => { + let user = await config.createUser({ builder: { global: false }, admin: { global: false }, }) - const table = await config.api.table.save({ - name: "table", - type: "table", - sourceId: INTERNAL_TABLE_SOURCE_ID, - sourceType: TableSourceType.INTERNAL, - schema: { - name: { - type: FieldType.STRING, - name: "name", - }, - }, - }) - await config.withUser(user, async () => { const apps = await config.api.application.fetch() expect(apps).toHaveLength(0) @@ -326,17 +293,43 @@ describe("/applications", () => { version: "name", }) - await config.api.user.update({ + user = await config.globalUser({ ...user, roles: { - [config.getAppId()]: role._id!, + [config.getProdAppId()]: role.name, }, }) - await config.api.permission.add({ - resourceId: table._id!, - roleId: role._id!, - level: PermissionLevel.READ, + await config.withUser(user, async () => { + const apps = await config.api.application.fetch() + expect(apps).toHaveLength(1) + }) + }) + + it.only("should only return apps a user has access to through a custom role on a group", async () => { + let user = await config.createUser({ + builder: { global: false }, + admin: { global: false }, + }) + + await config.withUser(user, async () => { + const apps = await config.api.application.fetch() + expect(apps).toHaveLength(0) + }) + + const roleName = uuid.v4().replace(/-/g, "") + const role = await config.api.roles.save({ + name: roleName, + inherits: "PUBLIC", + permissionId: "read_only", + version: "name", + }) + + const group = await config.createGroup(role._id!) + + user = await config.globalUser({ + ...user, + userGroups: [group._id!], }) await config.withUser(user, async () => { diff --git a/packages/types/src/documents/app/role.ts b/packages/types/src/documents/app/role.ts index d126a67b16..f32ba810b0 100644 --- a/packages/types/src/documents/app/role.ts +++ b/packages/types/src/documents/app/role.ts @@ -5,4 +5,5 @@ export interface Role extends Document { inherits?: string permissions: { [key: string]: string[] } version?: string + name: string } From 11704ea983b5ec3d7426b6927afa41d1cdea81a7 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 5 Mar 2024 17:40:38 +0000 Subject: [PATCH 088/114] TODO. --- packages/server/src/api/controllers/role.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/server/src/api/controllers/role.ts b/packages/server/src/api/controllers/role.ts index fff58da86e..6b62c568e2 100644 --- a/packages/server/src/api/controllers/role.ts +++ b/packages/server/src/api/controllers/role.ts @@ -107,6 +107,8 @@ export async function save(ctx: UserCtx) { role._rev = result.rev ctx.body = role + // TODO: need to check that the prod DB actually exists, I think it won't + // if the app has never been published. const replication = new dbCore.Replication({ source: context.getDevAppDB().name, target: context.getProdAppDB().name, From edda776b14c09fe97325ded56eebca8970a1f537 Mon Sep 17 00:00:00 2001 From: Michael Drury Date: Tue, 5 Mar 2024 17:42:44 +0000 Subject: [PATCH 089/114] PR comments. --- packages/server/src/sdk/app/rows/utils.ts | 40 +++++++++++++++-------- 1 file changed, 27 insertions(+), 13 deletions(-) diff --git a/packages/server/src/sdk/app/rows/utils.ts b/packages/server/src/sdk/app/rows/utils.ts index e090045925..6e3e25364e 100644 --- a/packages/server/src/sdk/app/rows/utils.ts +++ b/packages/server/src/sdk/app/rows/utils.ts @@ -1,14 +1,14 @@ import cloneDeep from "lodash/cloneDeep" import validateJs from "validate.js" import { + Datasource, + DatasourcePlusQueryResponse, FieldType, QueryJson, Row, + SourceName, Table, TableSchema, - DatasourcePlusQueryResponse, - Datasource, - SourceName, } from "@budibase/types" import { makeExternalQuery } from "../../../integrations/base/query" import { Format } from "../../../api/controllers/view/exporters" @@ -16,21 +16,35 @@ import sdk from "../.." import { isRelationshipColumn } from "../../../db/utils" import { SqlClient } from "../../../integrations/utils" +const SQL_CLIENT_SOURCE_MAP: Record = { + [SourceName.POSTGRES]: SqlClient.POSTGRES, + [SourceName.MYSQL]: SqlClient.MY_SQL, + [SourceName.SQL_SERVER]: SqlClient.MS_SQL, + [SourceName.ORACLE]: SqlClient.ORACLE, + [SourceName.DYNAMODB]: undefined, + [SourceName.MONGODB]: undefined, + [SourceName.ELASTICSEARCH]: undefined, + [SourceName.COUCHDB]: undefined, + [SourceName.S3]: undefined, + [SourceName.AIRTABLE]: undefined, + [SourceName.ARANGODB]: undefined, + [SourceName.REST]: undefined, + [SourceName.FIRESTORE]: undefined, + [SourceName.GOOGLE_SHEETS]: undefined, + [SourceName.REDIS]: undefined, + [SourceName.SNOWFLAKE]: undefined, + [SourceName.BUDIBASE]: undefined, +} + export function getSQLClient(datasource: Datasource): SqlClient { if (!datasource.isSQL) { throw new Error("Cannot get SQL Client for non-SQL datasource") } - switch (datasource.source) { - case SourceName.POSTGRES: - return SqlClient.POSTGRES - case SourceName.MYSQL: - return SqlClient.MY_SQL - case SourceName.ORACLE: - return SqlClient.ORACLE - case SourceName.SQL_SERVER: - return SqlClient.MS_SQL + const lookup = SQL_CLIENT_SOURCE_MAP[datasource.source] + if (lookup) { + return lookup } - throw new Error("Unable to find a valid SQL client") + throw new Error("Unable to determine client for SQL datasource") } export async function getDatasourceAndQuery( From b72edc21ecde8fb9564520d94c9a6046430886d5 Mon Sep 17 00:00:00 2001 From: Michael Drury Date: Tue, 5 Mar 2024 17:46:09 +0000 Subject: [PATCH 090/114] Linting. --- .../server/src/api/controllers/row/alias.ts | 15 +++- .../src/integrations/tests/sqlAlias.spec.ts | 82 +++++++++++++------ 2 files changed, 72 insertions(+), 25 deletions(-) diff --git a/packages/server/src/api/controllers/row/alias.ts b/packages/server/src/api/controllers/row/alias.ts index 28eaa95bf8..1d586c54fd 100644 --- a/packages/server/src/api/controllers/row/alias.ts +++ b/packages/server/src/api/controllers/row/alias.ts @@ -1,4 +1,11 @@ -import { Datasource, DatasourcePlusQueryResponse, Operation, QueryJson, Row, SearchFilters } from "@budibase/types" +import { + Datasource, + DatasourcePlusQueryResponse, + Operation, + QueryJson, + Row, + SearchFilters, +} from "@budibase/types" import { getSQLClient } from "../../../sdk/app/rows/utils" import { cloneDeep } from "lodash" import sdk from "../../../sdk" @@ -10,7 +17,11 @@ const WRITE_OPERATIONS: Operation[] = [ Operation.UPDATE, Operation.DELETE, ] -const DISABLED_WRITE_CLIENTS: SqlClient[] = [SqlClient.MY_SQL, SqlClient.MS_SQL, SqlClient.ORACLE] +const DISABLED_WRITE_CLIENTS: SqlClient[] = [ + SqlClient.MY_SQL, + SqlClient.MS_SQL, + SqlClient.ORACLE, +] class CharSequence { static alphabet = "abcdefghijklmnopqrstuvwxyz" diff --git a/packages/server/src/integrations/tests/sqlAlias.spec.ts b/packages/server/src/integrations/tests/sqlAlias.spec.ts index 70dda8c335..dd82dadac0 100644 --- a/packages/server/src/integrations/tests/sqlAlias.spec.ts +++ b/packages/server/src/integrations/tests/sqlAlias.spec.ts @@ -214,59 +214,95 @@ describe("Captures of real examples", () => { endpoint: { datasourceId: "", entityId: "", operation: op }, resource: { fields, - } + }, } } it("should check for Postgres aliased status", () => { const aliasing = new AliasTables(tables) const datasource = getDatasource(SourceName.POSTGRES) - expect(aliasing.isAliasingEnabled(getQuery(Operation.CREATE), datasource)).toEqual(true) - expect(aliasing.isAliasingEnabled(getQuery(Operation.READ), datasource)).toEqual(true) - expect(aliasing.isAliasingEnabled(getQuery(Operation.UPDATE), datasource)).toEqual(true) - expect(aliasing.isAliasingEnabled(getQuery(Operation.DELETE), datasource)).toEqual(true) + expect( + aliasing.isAliasingEnabled(getQuery(Operation.CREATE), datasource) + ).toEqual(true) + expect( + aliasing.isAliasingEnabled(getQuery(Operation.READ), datasource) + ).toEqual(true) + expect( + aliasing.isAliasingEnabled(getQuery(Operation.UPDATE), datasource) + ).toEqual(true) + expect( + aliasing.isAliasingEnabled(getQuery(Operation.DELETE), datasource) + ).toEqual(true) }) it("should check for MS-SQL aliased status", () => { const aliasing = new AliasTables(tables) const datasource = getDatasource(SourceName.SQL_SERVER) - expect(aliasing.isAliasingEnabled(getQuery(Operation.CREATE), datasource)).toEqual(false) - expect(aliasing.isAliasingEnabled(getQuery(Operation.READ), datasource)).toEqual(true) - expect(aliasing.isAliasingEnabled(getQuery(Operation.UPDATE), datasource)).toEqual(false) - expect(aliasing.isAliasingEnabled(getQuery(Operation.DELETE), datasource)).toEqual(false) + expect( + aliasing.isAliasingEnabled(getQuery(Operation.CREATE), datasource) + ).toEqual(false) + expect( + aliasing.isAliasingEnabled(getQuery(Operation.READ), datasource) + ).toEqual(true) + expect( + aliasing.isAliasingEnabled(getQuery(Operation.UPDATE), datasource) + ).toEqual(false) + expect( + aliasing.isAliasingEnabled(getQuery(Operation.DELETE), datasource) + ).toEqual(false) }) it("should check for MySQL aliased status", () => { const aliasing = new AliasTables(tables) const datasource = getDatasource(SourceName.MYSQL) - expect(aliasing.isAliasingEnabled(getQuery(Operation.CREATE), datasource)).toEqual(false) - expect(aliasing.isAliasingEnabled(getQuery(Operation.READ), datasource)).toEqual(true) - expect(aliasing.isAliasingEnabled(getQuery(Operation.UPDATE), datasource)).toEqual(false) - expect(aliasing.isAliasingEnabled(getQuery(Operation.DELETE), datasource)).toEqual(false) + expect( + aliasing.isAliasingEnabled(getQuery(Operation.CREATE), datasource) + ).toEqual(false) + expect( + aliasing.isAliasingEnabled(getQuery(Operation.READ), datasource) + ).toEqual(true) + expect( + aliasing.isAliasingEnabled(getQuery(Operation.UPDATE), datasource) + ).toEqual(false) + expect( + aliasing.isAliasingEnabled(getQuery(Operation.DELETE), datasource) + ).toEqual(false) }) it("should check for Oracle aliased status", () => { const aliasing = new AliasTables(tables) const datasource = getDatasource(SourceName.ORACLE) - expect(aliasing.isAliasingEnabled(getQuery(Operation.CREATE), datasource)).toEqual(false) - expect(aliasing.isAliasingEnabled(getQuery(Operation.READ), datasource)).toEqual(true) - expect(aliasing.isAliasingEnabled(getQuery(Operation.UPDATE), datasource)).toEqual(false) - expect(aliasing.isAliasingEnabled(getQuery(Operation.DELETE), datasource)).toEqual(false) + expect( + aliasing.isAliasingEnabled(getQuery(Operation.CREATE), datasource) + ).toEqual(false) + expect( + aliasing.isAliasingEnabled(getQuery(Operation.READ), datasource) + ).toEqual(true) + expect( + aliasing.isAliasingEnabled(getQuery(Operation.UPDATE), datasource) + ).toEqual(false) + expect( + aliasing.isAliasingEnabled(getQuery(Operation.DELETE), datasource) + ).toEqual(false) }) it("should disable aliasing for non-SQL datasources", () => { const aliasing = new AliasTables(tables) - expect(aliasing.isAliasingEnabled(getQuery(Operation.READ), { - source: SourceName.GOOGLE_SHEETS, - type: "datasource", - isSQL: false, - })) + expect( + aliasing.isAliasingEnabled(getQuery(Operation.READ), { + source: SourceName.GOOGLE_SHEETS, + type: "datasource", + isSQL: false, + }) + ) }) it("should disable when no fields", () => { const aliasing = new AliasTables(tables) const datasource = getDatasource(SourceName.POSTGRES) - expect(aliasing.isAliasingEnabled(getQuery(Operation.READ, []), datasource)).toEqual(false) + expect( + aliasing.isAliasingEnabled(getQuery(Operation.READ, []), datasource) + ).toEqual(false) }) }) From 1918ec6c68f6979d41852061cf8437a7599f9511 Mon Sep 17 00:00:00 2001 From: Michael Drury Date: Tue, 5 Mar 2024 18:00:15 +0000 Subject: [PATCH 091/114] Reverting type changes. --- packages/server/src/integrations/index.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/server/src/integrations/index.ts b/packages/server/src/integrations/index.ts index 18c46b9260..92067d1918 100644 --- a/packages/server/src/integrations/index.ts +++ b/packages/server/src/integrations/index.ts @@ -25,7 +25,7 @@ import env from "../environment" import cloneDeep from "lodash/cloneDeep" import sdk from "../sdk" -const DEFINITIONS: { [key: SourceName]: Integration | undefined } = { +const DEFINITIONS: Record = { [SourceName.POSTGRES]: postgres.schema, [SourceName.DYNAMODB]: dynamodb.schema, [SourceName.MONGODB]: mongodb.schema, @@ -45,7 +45,7 @@ const DEFINITIONS: { [key: SourceName]: Integration | undefined } = { [SourceName.BUDIBASE]: undefined, } -const INTEGRATIONS: { [key: SourceName]: IntegrationBase | undefined } = { +const INTEGRATIONS: Record = { [SourceName.POSTGRES]: postgres.integration, [SourceName.DYNAMODB]: dynamodb.integration, [SourceName.MONGODB]: mongodb.integration, From b58b0d3b40320ae551b3e8c82028b18ce236f8d5 Mon Sep 17 00:00:00 2001 From: Michael Drury Date: Tue, 5 Mar 2024 18:15:19 +0000 Subject: [PATCH 092/114] Fixing integration base types. --- packages/server/src/integrations/index.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/server/src/integrations/index.ts b/packages/server/src/integrations/index.ts index 92067d1918..747a717278 100644 --- a/packages/server/src/integrations/index.ts +++ b/packages/server/src/integrations/index.ts @@ -45,7 +45,9 @@ const DEFINITIONS: Record = { [SourceName.BUDIBASE]: undefined, } -const INTEGRATIONS: Record = { +type IntegrationBaseConstructor = new (...args: any[]) => IntegrationBase + +const INTEGRATIONS: Record = { [SourceName.POSTGRES]: postgres.integration, [SourceName.DYNAMODB]: dynamodb.integration, [SourceName.MONGODB]: mongodb.integration, From de56324a4b24fd41414369435654c4833fc4dca7 Mon Sep 17 00:00:00 2001 From: Michael Drury Date: Tue, 5 Mar 2024 18:16:27 +0000 Subject: [PATCH 093/114] Linting --- packages/server/src/integrations/index.ts | 39 ++++++++++++----------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/packages/server/src/integrations/index.ts b/packages/server/src/integrations/index.ts index 747a717278..8cbc29251b 100644 --- a/packages/server/src/integrations/index.ts +++ b/packages/server/src/integrations/index.ts @@ -47,25 +47,26 @@ const DEFINITIONS: Record = { type IntegrationBaseConstructor = new (...args: any[]) => IntegrationBase -const INTEGRATIONS: Record = { - [SourceName.POSTGRES]: postgres.integration, - [SourceName.DYNAMODB]: dynamodb.integration, - [SourceName.MONGODB]: mongodb.integration, - [SourceName.ELASTICSEARCH]: elasticsearch.integration, - [SourceName.COUCHDB]: couchdb.integration, - [SourceName.SQL_SERVER]: sqlServer.integration, - [SourceName.S3]: s3.integration, - [SourceName.AIRTABLE]: airtable.integration, - [SourceName.MYSQL]: mysql.integration, - [SourceName.ARANGODB]: arangodb.integration, - [SourceName.REST]: rest.integration, - [SourceName.FIRESTORE]: firebase.integration, - [SourceName.GOOGLE_SHEETS]: googlesheets.integration, - [SourceName.REDIS]: redis.integration, - [SourceName.SNOWFLAKE]: snowflake.integration, - [SourceName.ORACLE]: undefined, - [SourceName.BUDIBASE]: undefined, -} +const INTEGRATIONS: Record = + { + [SourceName.POSTGRES]: postgres.integration, + [SourceName.DYNAMODB]: dynamodb.integration, + [SourceName.MONGODB]: mongodb.integration, + [SourceName.ELASTICSEARCH]: elasticsearch.integration, + [SourceName.COUCHDB]: couchdb.integration, + [SourceName.SQL_SERVER]: sqlServer.integration, + [SourceName.S3]: s3.integration, + [SourceName.AIRTABLE]: airtable.integration, + [SourceName.MYSQL]: mysql.integration, + [SourceName.ARANGODB]: arangodb.integration, + [SourceName.REST]: rest.integration, + [SourceName.FIRESTORE]: firebase.integration, + [SourceName.GOOGLE_SHEETS]: googlesheets.integration, + [SourceName.REDIS]: redis.integration, + [SourceName.SNOWFLAKE]: snowflake.integration, + [SourceName.ORACLE]: undefined, + [SourceName.BUDIBASE]: undefined, + } // optionally add oracle integration if the oracle binary can be installed if ( From bed813da77c2ef5c0eb7a4bdc290f21ae2fd4e4c Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Tue, 5 Mar 2024 18:29:11 +0000 Subject: [PATCH 094/114] Bump version to 2.21.3 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index 6fb032ac77..a77a16a24e 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.21.2", + "version": "2.21.3", "npmClient": "yarn", "packages": [ "packages/*", From 339ca403bfbf2a58fa1d5a61371823cd216c35e6 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 5 Mar 2024 22:43:28 +0100 Subject: [PATCH 095/114] Add redis.increment --- packages/backend-core/src/redis/redis.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/backend-core/src/redis/redis.ts b/packages/backend-core/src/redis/redis.ts index 8cfa3db5c1..7bd524b18c 100644 --- a/packages/backend-core/src/redis/redis.ts +++ b/packages/backend-core/src/redis/redis.ts @@ -331,6 +331,11 @@ class RedisWrapper { let items = await this.scan() await Promise.all(items.map((obj: any) => this.delete(obj.key))) } + + async increment(key: string) { + const result = await this.getClient().incr(addDbPrefix(this._db, key)) + return result + } } export default RedisWrapper From 837395e5e02309e3117b72e4a4316e6c16f22b1a Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 5 Mar 2024 22:43:34 +0100 Subject: [PATCH 096/114] Add tests --- .../src/redis/tests/redis.spec.ts | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/packages/backend-core/src/redis/tests/redis.spec.ts b/packages/backend-core/src/redis/tests/redis.spec.ts index 1fd40acc37..31106e1765 100644 --- a/packages/backend-core/src/redis/tests/redis.spec.ts +++ b/packages/backend-core/src/redis/tests/redis.spec.ts @@ -107,4 +107,44 @@ describe("redis", () => { expect(await redis.keys("*")).toHaveLength(0) }) }) + + describe("increment", () => { + it("can increment on a new key", async () => { + const key = structures.uuid() + const result = await redis.increment(key) + expect(result).toBe(1) + }) + + it("can increment multiple times", async () => { + const key = structures.uuid() + const results = [ + await redis.increment(key), + await redis.increment(key), + await redis.increment(key), + await redis.increment(key), + await redis.increment(key), + ] + expect(results).toEqual([1, 2, 3, 4, 5]) + }) + + it("can increment on a new key", async () => { + const key1 = structures.uuid() + const key2 = structures.uuid() + + const result1 = await redis.increment(key1) + expect(result1).toBe(1) + + const result2 = await redis.increment(key2) + expect(result2).toBe(1) + }) + + it("can increment multiple times in parallel", async () => { + const key = structures.uuid() + const results = await Promise.all( + Array.from({ length: 100 }).map(() => redis.increment(key)) + ) + expect(results).toHaveLength(100) + expect(results).toEqual(Array.from({ length: 100 }).map((_, i) => i + 1)) + }) + }) }) From 192d7deb2adc7c8bcbe7f9e08fedae1f2939695d Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 5 Mar 2024 22:52:44 +0100 Subject: [PATCH 097/114] Handle errors --- packages/backend-core/src/redis/redis.ts | 3 +++ packages/backend-core/src/redis/tests/redis.spec.ts | 11 +++++++++++ 2 files changed, 14 insertions(+) diff --git a/packages/backend-core/src/redis/redis.ts b/packages/backend-core/src/redis/redis.ts index 7bd524b18c..55d43e4f60 100644 --- a/packages/backend-core/src/redis/redis.ts +++ b/packages/backend-core/src/redis/redis.ts @@ -334,6 +334,9 @@ class RedisWrapper { async increment(key: string) { const result = await this.getClient().incr(addDbPrefix(this._db, key)) + if (isNaN(result)) { + throw new Error(`Redis ${key} does not contains a number`) + } return result } } diff --git a/packages/backend-core/src/redis/tests/redis.spec.ts b/packages/backend-core/src/redis/tests/redis.spec.ts index 31106e1765..f7038d6d5e 100644 --- a/packages/backend-core/src/redis/tests/redis.spec.ts +++ b/packages/backend-core/src/redis/tests/redis.spec.ts @@ -146,5 +146,16 @@ describe("redis", () => { expect(results).toHaveLength(100) expect(results).toEqual(Array.from({ length: 100 }).map((_, i) => i + 1)) }) + + it.each([ + generator.word(), + generator.bool(), + { [generator.word()]: generator.word() }, + ])("cannot increment if the store value is not a number", async value => { + const key = structures.uuid() + await redis.store(key, value) + + await expect(redis.increment(key)).rejects.toThrowError("") + }) }) }) From d4ffc3d5d73ec8c5df3d3b87acb3228d957715fc Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 5 Mar 2024 22:54:39 +0100 Subject: [PATCH 098/114] More tests --- packages/backend-core/src/redis/tests/redis.spec.ts | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/packages/backend-core/src/redis/tests/redis.spec.ts b/packages/backend-core/src/redis/tests/redis.spec.ts index f7038d6d5e..8cdaf3a740 100644 --- a/packages/backend-core/src/redis/tests/redis.spec.ts +++ b/packages/backend-core/src/redis/tests/redis.spec.ts @@ -147,6 +147,15 @@ describe("redis", () => { expect(results).toEqual(Array.from({ length: 100 }).map((_, i) => i + 1)) }) + it("can increment existing set keys", async () => { + const key = structures.uuid() + await redis.store(key, 70) + await redis.increment(key) + + const result = await redis.increment(key) + expect(result).toBe(72) + }) + it.each([ generator.word(), generator.bool(), @@ -155,7 +164,9 @@ describe("redis", () => { const key = structures.uuid() await redis.store(key, value) - await expect(redis.increment(key)).rejects.toThrowError("") + await expect(redis.increment(key)).rejects.toThrowError( + `Redis ${key} does not contains a number` + ) }) }) }) From b774987e615435bf68eb33d1bda477dadddb8914 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 6 Mar 2024 00:33:04 +0100 Subject: [PATCH 099/114] Install testcontainers --- packages/backend-core/package.json | 1 + packages/server/package.json | 2 +- yarn.lock | 83 ++++++++++++++++++++++-------- 3 files changed, 63 insertions(+), 23 deletions(-) diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json index 90b3316c3f..fe56780982 100644 --- a/packages/backend-core/package.json +++ b/packages/backend-core/package.json @@ -78,6 +78,7 @@ "jest-serial-runner": "1.2.1", "pino-pretty": "10.0.0", "pouchdb-adapter-memory": "7.2.2", + "testcontainers": "^10.7.2", "timekeeper": "2.2.0", "typescript": "5.2.2" }, diff --git a/packages/server/package.json b/packages/server/package.json index 45980a4be6..97de17eb58 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -149,7 +149,7 @@ "rimraf": "3.0.2", "supertest": "6.3.3", "swagger-jsdoc": "6.1.0", - "testcontainers": "10.6.0", + "testcontainers": "10.7.2", "timekeeper": "2.2.0", "ts-node": "10.8.1", "tsconfig-paths": "4.0.0", diff --git a/yarn.lock b/yarn.lock index 2f9f558e2c..4025a537a3 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5303,10 +5303,10 @@ dependencies: "@types/node" "*" -"@types/dockerode@^3.3.21": - version "3.3.23" - resolved "https://registry.yarnpkg.com/@types/dockerode/-/dockerode-3.3.23.tgz#07b2084013d01e14d5d97856446f4d9c9f27c223" - integrity sha512-Lz5J+NFgZS4cEVhquwjIGH4oQwlVn2h7LXD3boitujBnzOE5o7s9H8hchEjoDK2SlRsJTogdKnQeiJgPPKLIEw== +"@types/dockerode@^3.3.24": + version "3.3.24" + resolved "https://registry.yarnpkg.com/@types/dockerode/-/dockerode-3.3.24.tgz#bea354a4fcd0824a80fd5ea5ede3e8cda71137a7" + integrity sha512-679y69OYusf7Fr2HtdjXPUF6hnHxSA9K4EsuagsMuPno/XpJHjXxCOy2I5YL8POnWbzjsQAi0pyKIYM9HSpQog== dependencies: "@types/docker-modem" "*" "@types/node" "*" @@ -7027,7 +7027,7 @@ ast-types@0.9.6: resolved "https://registry.yarnpkg.com/ast-types/-/ast-types-0.9.6.tgz#102c9e9e9005d3e7e3829bf0c4fa24ee862ee9b9" integrity sha512-qEdtR2UH78yyHX/AUNfXmJTlM48XoFZKBdwi1nzkI1mJL21cmbu0cvjxjpkXJ5NENMq42H+hNs8VLJcqXLerBQ== -async-lock@^1.4.0: +async-lock@^1.4.1: version "1.4.1" resolved "https://registry.yarnpkg.com/async-lock/-/async-lock-1.4.1.tgz#56b8718915a9b68b10fce2f2a9a3dddf765ef53f" integrity sha512-Az2ZTpuytrtqENulXwO3GGv1Bztugx6TT37NIo7imr/Qo0gsYiGtSdBa2B6fsXhTpVZDNfu1Qn3pk531e3q+nQ== @@ -7263,6 +7263,33 @@ balanced-match@^1.0.0: resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.2.tgz#e83e3a7e3f300b34cb9d87f615fa0cbf357690ee" integrity sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw== +bare-events@^2.0.0, bare-events@^2.2.0: + version "2.2.1" + resolved "https://registry.yarnpkg.com/bare-events/-/bare-events-2.2.1.tgz#7b6d421f26a7a755e20bf580b727c84b807964c1" + integrity sha512-9GYPpsPFvrWBkelIhOhTWtkeZxVxZOdb3VnFTCzlOo3OjvmTvzLoZFUT8kNFACx0vJej6QPney1Cf9BvzCNE/A== + +bare-fs@^2.1.1: + version "2.2.1" + resolved "https://registry.yarnpkg.com/bare-fs/-/bare-fs-2.2.1.tgz#c1985d8d3e07a178956b072d3af67cb8c1fa9391" + integrity sha512-+CjmZANQDFZWy4PGbVdmALIwmt33aJg8qTkVjClU6X4WmZkTPBDxRHiBn7fpqEWEfF3AC2io++erpViAIQbSjg== + dependencies: + bare-events "^2.0.0" + bare-os "^2.0.0" + bare-path "^2.0.0" + streamx "^2.13.0" + +bare-os@^2.0.0, bare-os@^2.1.0: + version "2.2.0" + resolved "https://registry.yarnpkg.com/bare-os/-/bare-os-2.2.0.tgz#24364692984d0bd507621754781b31d7872736b2" + integrity sha512-hD0rOPfYWOMpVirTACt4/nK8mC55La12K5fY1ij8HAdfQakD62M+H4o4tpfKzVGLgRDTuk3vjA4GqGXXCeFbag== + +bare-path@^2.0.0, bare-path@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/bare-path/-/bare-path-2.1.0.tgz#830f17fd39842813ca77d211ebbabe238a88cb4c" + integrity sha512-DIIg7ts8bdRKwJRJrUMy/PICEaQZaPGZ26lsSx9MJSwIhSrcdHn7/C8W+XmnG/rKi6BaRcz+JO00CjZteybDtw== + dependencies: + bare-os "^2.1.0" + base62@^1.1.0: version "1.2.8" resolved "https://registry.yarnpkg.com/base62/-/base62-1.2.8.tgz#1264cb0fb848d875792877479dbe8bae6bae3428" @@ -9615,10 +9642,10 @@ docker-compose@^0.23.5, docker-compose@^0.23.6: dependencies: yaml "^1.10.2" -docker-compose@^0.24.2: - version "0.24.3" - resolved "https://registry.yarnpkg.com/docker-compose/-/docker-compose-0.24.3.tgz#298d7bb4aaf37b3b45d0e4ef55c7f58ccc39cca9" - integrity sha512-x3/QN3AIOMe7j2c8f/jcycizMft7dl8MluoB9OGPAYCyKHHiPUFqI9GjCcsU0kYy24vYKMCcfR6+5ZaEyQlrxg== +docker-compose@^0.24.6: + version "0.24.6" + resolved "https://registry.yarnpkg.com/docker-compose/-/docker-compose-0.24.6.tgz#d1f490a641bdb7ccc07c4d446b264f026f9a1f15" + integrity sha512-VidlUyNzXMaVsuM79sjSvwC4nfojkP2VneL+Zfs538M2XFnffZDhx6veqnz/evCNIYGyz5O+1fgL6+g0NLWTBA== dependencies: yaml "^2.2.2" @@ -20179,6 +20206,16 @@ stream-to-array@^2.3.0: dependencies: any-promise "^1.1.0" +streamx@^2.13.0: + version "2.16.1" + resolved "https://registry.yarnpkg.com/streamx/-/streamx-2.16.1.tgz#2b311bd34832f08aa6bb4d6a80297c9caef89614" + integrity sha512-m9QYj6WygWyWa3H1YY69amr4nVgy61xfjys7xO7kviL5rfIEc2naf+ewFiOA+aEJD7y0JO3h2GoiUv4TDwEGzQ== + dependencies: + fast-fifo "^1.1.0" + queue-tick "^1.0.1" + optionalDependencies: + bare-events "^2.2.0" + streamx@^2.15.0: version "2.15.6" resolved "https://registry.yarnpkg.com/streamx/-/streamx-2.15.6.tgz#28bf36997ebc7bf6c08f9eba958735231b833887" @@ -20730,14 +20767,16 @@ tar-fs@2.1.1, tar-fs@^2.0.0, tar-fs@^2.1.0: pump "^3.0.0" tar-stream "^2.1.4" -tar-fs@^3.0.4: - version "3.0.4" - resolved "https://registry.yarnpkg.com/tar-fs/-/tar-fs-3.0.4.tgz#a21dc60a2d5d9f55e0089ccd78124f1d3771dbbf" - integrity sha512-5AFQU8b9qLfZCX9zp2duONhPmZv0hGYiBPJsyUdqMjzq/mqVpy/rEUSeHk1+YitmxugaptgBh5oDGU3VsAJq4w== +tar-fs@^3.0.5: + version "3.0.5" + resolved "https://registry.yarnpkg.com/tar-fs/-/tar-fs-3.0.5.tgz#f954d77767e4e6edf973384e1eb95f8f81d64ed9" + integrity sha512-JOgGAmZyMgbqpLwct7ZV8VzkEB6pxXFBVErLtb+XCOqzc6w1xiWKI9GVd6bwk68EX7eJ4DWmfXVmq8K2ziZTGg== dependencies: - mkdirp-classic "^0.5.2" pump "^3.0.0" tar-stream "^3.1.5" + optionalDependencies: + bare-fs "^2.1.1" + bare-path "^2.1.0" tar-fs@~2.0.1: version "2.0.1" @@ -20875,25 +20914,25 @@ test-exclude@^6.0.0: glob "^7.1.4" minimatch "^3.0.4" -testcontainers@10.6.0: - version "10.6.0" - resolved "https://registry.yarnpkg.com/testcontainers/-/testcontainers-10.6.0.tgz#205ad9148e68ff5c43a6209a30b12965acbe89d8" - integrity sha512-FDJ3o3J8IMu1V7Uc6lNZ2MAD8+BV4HdpR/Vf5mHtgYHKdn6k1EbGFwtnvVNOxanJ99FCjf/EU8eA5ZQ4yjlsGA== +testcontainers@10.7.2, testcontainers@^10.7.2: + version "10.7.2" + resolved "https://registry.yarnpkg.com/testcontainers/-/testcontainers-10.7.2.tgz#619e93200dd47f174b307b40fa830cf023b74c25" + integrity sha512-7d+LVd/4YKp/cutiVMLL5cnj/8p8oYELAVRRyNUM4FyUDz1OLQuwW868nDl7Vd1ZAQxzGeCR+F86FlR9Yw9fMA== dependencies: "@balena/dockerignore" "^1.0.2" - "@types/dockerode" "^3.3.21" + "@types/dockerode" "^3.3.24" archiver "^5.3.2" - async-lock "^1.4.0" + async-lock "^1.4.1" byline "^5.0.0" debug "^4.3.4" - docker-compose "^0.24.2" + docker-compose "^0.24.6" dockerode "^3.3.5" get-port "^5.1.1" node-fetch "^2.7.0" proper-lockfile "^4.1.2" properties-reader "^2.3.0" ssh-remote-port-forward "^1.0.4" - tar-fs "^3.0.4" + tar-fs "^3.0.5" tmp "^0.2.1" testcontainers@4.7.0: From 60f82610f6ea11d56b406452b590e6a943125140 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 6 Mar 2024 00:33:24 +0100 Subject: [PATCH 100/114] Use redis testcontainer --- .../src/redis/tests/redis.spec.ts | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/packages/backend-core/src/redis/tests/redis.spec.ts b/packages/backend-core/src/redis/tests/redis.spec.ts index 8cdaf3a740..a7c0384d78 100644 --- a/packages/backend-core/src/redis/tests/redis.spec.ts +++ b/packages/backend-core/src/redis/tests/redis.spec.ts @@ -1,8 +1,26 @@ +import { GenericContainer, StartedTestContainer } from "testcontainers" import { generator, structures } from "../../../tests" import RedisWrapper from "../redis" +import { env } from "../.." describe("redis", () => { let redis: RedisWrapper + let container: StartedTestContainer + + beforeAll(async () => { + const container = await new GenericContainer("redis") + .withExposedPorts(6379) + .start() + + env._set( + "REDIS_URL", + `${container.getHost()}:${container.getMappedPort(6379)}` + ) + env._set("MOCK_REDIS", 0) + env._set("REDIS_PASSWORD", 0) + }) + + afterAll(() => container?.stop()) beforeEach(async () => { redis = new RedisWrapper(structures.db.id()) @@ -165,7 +183,7 @@ describe("redis", () => { await redis.store(key, value) await expect(redis.increment(key)).rejects.toThrowError( - `Redis ${key} does not contains a number` + "ERR value is not an integer or out of range" ) }) }) From 355dea5b05659efa5da48ed12062852aa980a4b2 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 6 Mar 2024 00:13:11 +0100 Subject: [PATCH 101/114] Add deleteIfValue --- packages/backend-core/src/redis/redis.ts | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/packages/backend-core/src/redis/redis.ts b/packages/backend-core/src/redis/redis.ts index 55d43e4f60..fd40467b6a 100644 --- a/packages/backend-core/src/redis/redis.ts +++ b/packages/backend-core/src/redis/redis.ts @@ -339,6 +339,18 @@ class RedisWrapper { } return result } + + async deleteIfValue(key: string, value: any) { + const client = this.getClient() + + const luaScript = ` + if redis.call('GET', KEYS[1]) == ARGV[1] then + redis.call('DEL', KEYS[1]) + end + ` + + await client.eval(luaScript, 1, addDbPrefix(this._db, key), value) + } } export default RedisWrapper From 8f486e7bfc31bdb1b1f3a476f600a76996cccd9e Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 6 Mar 2024 00:20:42 +0100 Subject: [PATCH 102/114] Add tests --- .../src/redis/tests/redis.spec.ts | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/packages/backend-core/src/redis/tests/redis.spec.ts b/packages/backend-core/src/redis/tests/redis.spec.ts index a7c0384d78..9ff6828cee 100644 --- a/packages/backend-core/src/redis/tests/redis.spec.ts +++ b/packages/backend-core/src/redis/tests/redis.spec.ts @@ -187,4 +187,26 @@ describe("redis", () => { ) }) }) + + describe("deleteIfValue", () => { + it("can delete if the value matches", async () => { + const key = structures.uuid() + const value = generator.word() + await redis.store(key, value) + + await redis.deleteIfValue(key, value) + + expect(await redis.get(key)).toBeNull() + }) + + it("will not delete if the value does not matches", async () => { + const key = structures.uuid() + const value = generator.word() + await redis.store(key, value) + + await redis.deleteIfValue(key, generator.word()) + + expect(await redis.get(key)).toEqual(value) + }) + }) }) From d35d382ddb54c24ceb828e691c5f96714d0eeaaf Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 6 Mar 2024 00:58:05 +0100 Subject: [PATCH 103/114] Increase timeouts --- packages/backend-core/src/redis/tests/redis.spec.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/backend-core/src/redis/tests/redis.spec.ts b/packages/backend-core/src/redis/tests/redis.spec.ts index a7c0384d78..41bbc7453a 100644 --- a/packages/backend-core/src/redis/tests/redis.spec.ts +++ b/packages/backend-core/src/redis/tests/redis.spec.ts @@ -3,6 +3,8 @@ import { generator, structures } from "../../../tests" import RedisWrapper from "../redis" import { env } from "../.." +jest.setTimeout(30000) + describe("redis", () => { let redis: RedisWrapper let container: StartedTestContainer From 2b206f2105681140a1079ba49bcc434df1e7f489 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 6 Mar 2024 10:00:02 +0000 Subject: [PATCH 104/114] Fix the TODO I left myself last night. --- packages/backend-core/src/db/Replication.ts | 41 +++++---------------- packages/server/src/api/controllers/role.ts | 25 +++++++------ 2 files changed, 23 insertions(+), 43 deletions(-) diff --git a/packages/backend-core/src/db/Replication.ts b/packages/backend-core/src/db/Replication.ts index 12c11eb9e2..9c960d76dd 100644 --- a/packages/backend-core/src/db/Replication.ts +++ b/packages/backend-core/src/db/Replication.ts @@ -5,56 +5,33 @@ import { DocumentType } from "../constants" class Replication { source: PouchDB.Database target: PouchDB.Database - replication?: Promise - /** - * - * @param source - the DB you want to replicate or rollback to - * @param target - the DB you want to replicate to, or rollback from - */ constructor({ source, target }: { source: string; target: string }) { this.source = getPouchDB(source) this.target = getPouchDB(target) } - close() { - return Promise.all([closePouchDB(this.source), closePouchDB(this.target)]) + async close() { + await Promise.all([closePouchDB(this.source), closePouchDB(this.target)]) } - promisify(operation: any, opts = {}) { - return new Promise(resolve => { - operation(this.target, opts) - .on("denied", function (err: any) { + replicate(opts: PouchDB.Replication.ReplicateOptions = {}) { + return new Promise>(resolve => { + this.source.replicate + .to(this.target, opts) + .on("denied", function (err) { // a document failed to replicate (e.g. due to permissions) throw new Error(`Denied: Document failed to replicate ${err}`) }) - .on("complete", function (info: any) { + .on("complete", function (info) { return resolve(info) }) - .on("error", function (err: any) { + .on("error", function (err) { throw new Error(`Replication Error: ${err}`) }) }) } - /** - * Two way replication operation, intended to be promise based. - * @param opts - PouchDB replication options - */ - sync(opts: PouchDB.Replication.SyncOptions = {}) { - this.replication = this.promisify(this.source.sync, opts) - return this.replication - } - - /** - * One way replication operation, intended to be promise based. - * @param opts - PouchDB replication options - */ - replicate(opts: PouchDB.Replication.ReplicateOptions = {}) { - this.replication = this.promisify(this.source.replicate.to, opts) - return this.replication - } - appReplicateOpts( opts: PouchDB.Replication.ReplicateOptions = {} ): PouchDB.Replication.ReplicateOptions { diff --git a/packages/server/src/api/controllers/role.ts b/packages/server/src/api/controllers/role.ts index 6b62c568e2..84179d8dbc 100644 --- a/packages/server/src/api/controllers/role.ts +++ b/packages/server/src/api/controllers/role.ts @@ -107,17 +107,20 @@ export async function save(ctx: UserCtx) { role._rev = result.rev ctx.body = role - // TODO: need to check that the prod DB actually exists, I think it won't - // if the app has never been published. - const replication = new dbCore.Replication({ - source: context.getDevAppDB().name, - target: context.getProdAppDB().name, - }) - await replication.replicate({ - filter: (doc: any, params: any) => { - return doc._id === _id - }, - }) + const devDb = context.getDevAppDB() + const prodDb = context.getProdAppDB() + + if (await prodDb.exists()) { + const replication = new dbCore.Replication({ + source: devDb.name, + target: prodDb.name, + }) + await replication.replicate({ + filter: (doc: any, params: any) => { + return doc._id === _id + }, + }) + } } export async function destroy(ctx: UserCtx) { From b232371efff95f7925c93960ba92862324cb1a46 Mon Sep 17 00:00:00 2001 From: Peter Clement Date: Wed, 6 Mar 2024 10:01:42 +0000 Subject: [PATCH 105/114] remove uneeded comment --- packages/shared-core/src/tests/filters.test.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/shared-core/src/tests/filters.test.ts b/packages/shared-core/src/tests/filters.test.ts index 1e0a68de89..0cf7e0e92a 100644 --- a/packages/shared-core/src/tests/filters.test.ts +++ b/packages/shared-core/src/tests/filters.test.ts @@ -221,8 +221,6 @@ describe("runLuceneQuery", () => { ]) }) - // what should the name of this test be if it's the same test as above but with different operands - it("should return matching results if allOr is true and only one filter matches with different operands", () => { const query = buildQuery({ allOr: true, From eb00ce401f9819406acde58c60018945bc95864e Mon Sep 17 00:00:00 2001 From: Peter Clement Date: Wed, 6 Mar 2024 10:10:28 +0000 Subject: [PATCH 106/114] pr comments --- packages/shared-core/src/filters.ts | 7 ++++--- packages/shared-core/src/tests/filters.test.ts | 10 ++++++++++ 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/packages/shared-core/src/filters.ts b/packages/shared-core/src/filters.ts index 0a1673e558..84b6076d56 100644 --- a/packages/shared-core/src/filters.ts +++ b/packages/shared-core/src/filters.ts @@ -12,6 +12,7 @@ import { import dayjs from "dayjs" import { OperatorOptions, SqlNumberTypeRangeMap } from "./constants" import { deepGet } from "./helpers" +import test from "node:test" const HBS_REGEX = /{{([^{].*?)}}/g @@ -359,6 +360,7 @@ export const runLuceneQuery = (docs: any[], query?: SearchQuery) => { const oneOf = match( SearchQueryOperators.ONE_OF, (docValue: any, testValue: any) => { + console.log(testValue) if (typeof testValue === "string") { testValue = testValue.split(",") if (typeof docValue === "number") { @@ -410,13 +412,13 @@ export const runLuceneQuery = (docs: any[], query?: SearchQuery) => { .filter( ([key, value]: [string, any]) => !["allOr", "onEmptyFilter"].includes(key) && + value && Object.keys(value as Record).length > 0 ) .map(([key]) => key as any) const results: boolean[] = activeFilterKeys.map(filterKey => { - const filterFunction = filterFunctions[filterKey] - return filterFunction ? filterFunction(doc) : true + return filterFunctions[filterKey]?.(doc) ?? false }) if (query!.allOr) { @@ -425,7 +427,6 @@ export const runLuceneQuery = (docs: any[], query?: SearchQuery) => { return results.every(result => result === true) } } - return docs.filter(docMatch) } diff --git a/packages/shared-core/src/tests/filters.test.ts b/packages/shared-core/src/tests/filters.test.ts index 0cf7e0e92a..1f8f534f0d 100644 --- a/packages/shared-core/src/tests/filters.test.ts +++ b/packages/shared-core/src/tests/filters.test.ts @@ -240,6 +240,16 @@ describe("runLuceneQuery", () => { expect(runLuceneQuery(docs, query).map(row => row.order_id)).toEqual([]) }) + + it("should handle when a value is null or undefined", () => { + const query = buildQuery({ + allOr: true, + equal: { order_status: null }, + oneOf: { label: ["FRAGILE"] }, + }) + + expect(runLuceneQuery(docs, query).map(row => row.order_id)).toEqual([2]) + }) }) describe("buildLuceneQuery", () => { From 30f0e3459f1ec83af56f992a14c3507813d1dfa7 Mon Sep 17 00:00:00 2001 From: Dean Date: Wed, 6 Mar 2024 10:29:43 +0000 Subject: [PATCH 107/114] PR feedback --- .../portal/onboarding/steps/NewFormSteps.svelte | 9 +++++++++ .../src/components/portal/onboarding/steps/index.js | 1 + .../builder/src/components/portal/onboarding/tours.js | 9 +++------ 3 files changed, 13 insertions(+), 6 deletions(-) create mode 100644 packages/builder/src/components/portal/onboarding/steps/NewFormSteps.svelte diff --git a/packages/builder/src/components/portal/onboarding/steps/NewFormSteps.svelte b/packages/builder/src/components/portal/onboarding/steps/NewFormSteps.svelte new file mode 100644 index 0000000000..7c3679f9e8 --- /dev/null +++ b/packages/builder/src/components/portal/onboarding/steps/NewFormSteps.svelte @@ -0,0 +1,9 @@ +
+ When faced with a sizable form, consider implementing a multi-step approach to + enhance user experience. + +

+ Breaking the form into multiple steps can significantly improve usability by + making the process more digestible for your users. +

+
diff --git a/packages/builder/src/components/portal/onboarding/steps/index.js b/packages/builder/src/components/portal/onboarding/steps/index.js index 6694ce97a7..e15d191652 100644 --- a/packages/builder/src/components/portal/onboarding/steps/index.js +++ b/packages/builder/src/components/portal/onboarding/steps/index.js @@ -2,3 +2,4 @@ export { default as OnboardingData } from "./OnboardingData.svelte" export { default as OnboardingDesign } from "./OnboardingDesign.svelte" export { default as OnboardingPublish } from "./OnboardingPublish.svelte" export { default as NewViewUpdateFormRowId } from "./NewViewUpdateFormRowId.svelte" +export { default as NewFormSteps } from "./NewFormSteps.svelte" diff --git a/packages/builder/src/components/portal/onboarding/tours.js b/packages/builder/src/components/portal/onboarding/tours.js index 5cf6735970..fab97cdd03 100644 --- a/packages/builder/src/components/portal/onboarding/tours.js +++ b/packages/builder/src/components/portal/onboarding/tours.js @@ -7,6 +7,7 @@ import { OnboardingDesign, OnboardingPublish, NewViewUpdateFormRowId, + NewFormSteps, } from "./steps" import { API } from "api" import { customPositionHandler } from "components/design/settings/controls/EditComponentPopover" @@ -176,9 +177,7 @@ const getTours = () => { { id: TOUR_STEP_KEYS.BUILDER_FORM_CREATE_STEPS, title: "Add multiple steps", - body: `When faced with a sizable form, consider implementing a multi-step - approach to enhance user experience. Breaking the form into multiple steps - can significantly improve usability by making the process more digestible for your users.`, + layout: NewFormSteps, query: "#steps-prop-control-wrap", onComplete: () => { builderStore.highlightSetting() @@ -210,9 +209,7 @@ const getTours = () => { { id: TOUR_STEP_KEYS.BUILDER_FORM_VIEW_UPDATE_STEPS, title: "Add multiple steps", - body: `When faced with a sizable form, consider implementing a multi-step - approach to enhance user experience. Breaking the form into multiple steps - can significantly improve usability by making the process more digestible for your users.`, + layout: NewFormSteps, query: "#steps-prop-control-wrap", onComplete: () => { builderStore.highlightSetting() From 4baadadaa8d2572617236ebb26d66cdbde88f611 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 6 Mar 2024 12:22:20 +0100 Subject: [PATCH 108/114] Use pipeline instead of eval --- packages/backend-core/src/redis/redis.ts | 21 +++++++-------------- 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/packages/backend-core/src/redis/redis.ts b/packages/backend-core/src/redis/redis.ts index 8cfa3db5c1..6124f5f447 100644 --- a/packages/backend-core/src/redis/redis.ts +++ b/packages/backend-core/src/redis/redis.ts @@ -291,23 +291,16 @@ class RedisWrapper { return acc }, {} as Record) - const luaScript = ` - for i, key in ipairs(KEYS) do - redis.call('MSET', key, ARGV[i]) - ${ - expirySeconds !== null - ? `redis.call('EXPIRE', key, ARGV[#ARGV])` - : "" - } - end - ` - const keys = Object.keys(dataToStore) - const values = Object.values(dataToStore) + const pipeline = client.pipeline() + pipeline.mset(dataToStore) + if (expirySeconds !== null) { - values.push(expirySeconds) + for (const key of Object.keys(dataToStore)) { + pipeline.expire(key, expirySeconds) + } } - await client.eval(luaScript, keys.length, ...keys, ...values) + await pipeline.exec() } async getTTL(key: string) { From c198a5ae03164e2dff57281c43c1e8c3ec25a894 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 6 Mar 2024 12:23:04 +0100 Subject: [PATCH 109/114] Update packages/backend-core/src/redis/redis.ts Co-authored-by: Sam Rose --- packages/backend-core/src/redis/redis.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/backend-core/src/redis/redis.ts b/packages/backend-core/src/redis/redis.ts index 55d43e4f60..a5e1597c9f 100644 --- a/packages/backend-core/src/redis/redis.ts +++ b/packages/backend-core/src/redis/redis.ts @@ -335,7 +335,7 @@ class RedisWrapper { async increment(key: string) { const result = await this.getClient().incr(addDbPrefix(this._db, key)) if (isNaN(result)) { - throw new Error(`Redis ${key} does not contains a number`) + throw new Error(`Redis ${key} does not contain a number`) } return result } From 1f107041a108aeaf677da20659819bfe2d06ec03 Mon Sep 17 00:00:00 2001 From: Peter Clement Date: Wed, 6 Mar 2024 11:57:45 +0000 Subject: [PATCH 110/114] use vitest each --- .../shared-core/src/tests/filters.test.ts | 23 +++++++------------ 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/packages/shared-core/src/tests/filters.test.ts b/packages/shared-core/src/tests/filters.test.ts index 1f8f534f0d..de969562af 100644 --- a/packages/shared-core/src/tests/filters.test.ts +++ b/packages/shared-core/src/tests/filters.test.ts @@ -209,16 +209,19 @@ describe("runLuceneQuery", () => { } ) - it("should return matching results if allOr is true and only one filter matches", () => { + test.each([ + [false, []], + [true, [1, 2, 3]], + ])("should return %s if allOr is %s ", (allOr, expectedResult) => { const query = buildQuery({ - allOr: true, + allOr, oneOf: { staff_id: [10] }, contains: { description: ["box"] }, }) - expect(runLuceneQuery(docs, query).map(row => row.order_id)).toEqual([ - 1, 2, 3, - ]) + expect(runLuceneQuery(docs, query).map(row => row.order_id)).toEqual( + expectedResult + ) }) it("should return matching results if allOr is true and only one filter matches with different operands", () => { @@ -231,16 +234,6 @@ describe("runLuceneQuery", () => { expect(runLuceneQuery(docs, query).map(row => row.order_id)).toEqual([1, 2]) }) - it("should return nothing if allOr is false and only one filter matches", () => { - const query = buildQuery({ - allOr: false, - oneOf: { staff_id: [10] }, - contains: { description: ["box"] }, - }) - - expect(runLuceneQuery(docs, query).map(row => row.order_id)).toEqual([]) - }) - it("should handle when a value is null or undefined", () => { const query = buildQuery({ allOr: true, From 632b9a26f4313216c28458db46dc9334aea7e909 Mon Sep 17 00:00:00 2001 From: Peter Clement Date: Wed, 6 Mar 2024 14:42:30 +0000 Subject: [PATCH 111/114] remove log --- packages/shared-core/src/filters.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/shared-core/src/filters.ts b/packages/shared-core/src/filters.ts index 84b6076d56..d9fe533c88 100644 --- a/packages/shared-core/src/filters.ts +++ b/packages/shared-core/src/filters.ts @@ -12,7 +12,6 @@ import { import dayjs from "dayjs" import { OperatorOptions, SqlNumberTypeRangeMap } from "./constants" import { deepGet } from "./helpers" -import test from "node:test" const HBS_REGEX = /{{([^{].*?)}}/g @@ -360,7 +359,6 @@ export const runLuceneQuery = (docs: any[], query?: SearchQuery) => { const oneOf = match( SearchQueryOperators.ONE_OF, (docValue: any, testValue: any) => { - console.log(testValue) if (typeof testValue === "string") { testValue = testValue.split(",") if (typeof docValue === "number") { From 1b387d359c669b9672ab1de10e2811bb72f26811 Mon Sep 17 00:00:00 2001 From: Conor Webb <126772285+ConorWebb96@users.noreply.github.com> Date: Wed, 6 Mar 2024 16:32:00 +0000 Subject: [PATCH 112/114] Added icon to button component, reworked icon display code. (#12624) * Added icons to buttons, removed svg code added icon component code. * Added icon functionality to button group component. * Added gap to button manifest * Added gap to button setitngs. * Added gap setting to ButtonGroup component * Added the ability to clear the selected icon. * Added enter search to icon select * Removed use:styleable as its for the button * Moved non internal props up * Fixed broken DynamicFilter component icon * Updated DynamicFilter icon to a better suited one --------- Co-authored-by: melohagan <101575380+melohagan@users.noreply.github.com> --- .../controls/IconSelect/IconSelect.svelte | 17 ++++++++-- packages/client/manifest.json | 32 +++++++++++++++++++ .../client/src/components/app/Button.svelte | 24 +++++++------- .../src/components/app/ButtonGroup.svelte | 4 ++- .../app/dynamic-filter/DynamicFilter.svelte | 4 +-- 5 files changed, 65 insertions(+), 16 deletions(-) diff --git a/packages/builder/src/components/design/settings/controls/IconSelect/IconSelect.svelte b/packages/builder/src/components/design/settings/controls/IconSelect/IconSelect.svelte index 0c68c3c3e6..a28f5cfb3b 100644 --- a/packages/builder/src/components/design/settings/controls/IconSelect/IconSelect.svelte +++ b/packages/builder/src/components/design/settings/controls/IconSelect/IconSelect.svelte @@ -139,10 +139,22 @@ {/each}
-
- +
+ { + if (event.key === "Enter") { + searchForIcon() + } + }} + thin + placeholder="Search Icon" + />
+ {#if value} + + {/if}
@@ -239,6 +251,7 @@ flex-flow: row nowrap; width: 100%; padding-right: 15px; + gap: 10px; } .input-wrapper { width: 510px; diff --git a/packages/client/manifest.json b/packages/client/manifest.json index 43b75ebe26..10f9c5f412 100644 --- a/packages/client/manifest.json +++ b/packages/client/manifest.json @@ -525,6 +525,38 @@ "barTitle": "Disable button", "key": "disabled" }, + { + "type": "icon", + "label": "Icon", + "key": "icon" + }, + { + "type": "select", + "label": "Gap", + "key": "gap", + "showInBar": true, + "barStyle": "picker", + "dependsOn": "icon", + "options": [ + { + "label": "None", + "value": "N" + }, + { + "label": "Small", + "value": "S" + }, + { + "label": "Medium", + "value": "M" + }, + { + "label": "Large", + "value": "L" + } + ], + "defaultValue": "M" + }, { "type": "event", "label": "On click", diff --git a/packages/client/src/components/app/Button.svelte b/packages/client/src/components/app/Button.svelte index 361e64a983..c43face1bb 100644 --- a/packages/client/src/components/app/Button.svelte +++ b/packages/client/src/components/app/Button.svelte @@ -13,9 +13,10 @@ export let size = "M" export let type = "cta" export let quiet = false + export let icon = null + export let gap = "M" // For internal use only for now - not defined in the manifest - export let icon = null export let active = false const handleOnClick = async () => { @@ -47,7 +48,7 @@ {#key $component.editing} @@ -92,4 +85,13 @@ .active { color: var(--spectrum-global-color-blue-600); } + .gap-S { + gap: 8px; + } + .gap-M { + gap: 16px; + } + .gap-L { + gap: 32px; + } diff --git a/packages/client/src/components/app/ButtonGroup.svelte b/packages/client/src/components/app/ButtonGroup.svelte index 3ee703e253..2cf6b3db7d 100644 --- a/packages/client/src/components/app/ButtonGroup.svelte +++ b/packages/client/src/components/app/ButtonGroup.svelte @@ -20,7 +20,7 @@ wrap: true, }} > - {#each buttons as { text, type, quiet, disabled, onClick, size }} + {#each buttons as { text, type, quiet, disabled, onClick, size, icon, gap }} diff --git a/packages/client/src/components/app/dynamic-filter/DynamicFilter.svelte b/packages/client/src/components/app/dynamic-filter/DynamicFilter.svelte index 199a6122ab..549574e89b 100644 --- a/packages/client/src/components/app/dynamic-filter/DynamicFilter.svelte +++ b/packages/client/src/components/app/dynamic-filter/DynamicFilter.svelte @@ -92,9 +92,9 @@ {#if schemaLoaded}