Merge branch 'master' into revert-12934-revert-12930-revert-12929-revert-12769-isolated-vm

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "2.17.2",
+  "version": "2.17.7",
   "npmClient": "yarn",
   "packages": [
     "packages/*",

packages/account-portal

@@ -1 +1 @@
-Subproject commit a0b13270c36dd188e2a953d026b4560a1208008e
+Subproject commit cc12291732ee902dc832bc7d93cf2086ffdf0cff

packages/builder/src/builderStore/componentUtils.js

@@ -7,6 +7,9 @@ import {
   findHBSBlocks,
 } from "@budibase/string-templates"
 import { capitalise } from "helpers"
+import { Constants } from "@budibase/frontend-core"
+
+const { ContextScopes } = Constants
 
 /**
  * Recursively searches for a specific component ID
@@ -263,11 +266,59 @@ export const getComponentName = component => {
   if (component == null) {
     return ""
   }
-
   const components = get(store)?.components || {}
   const componentDefinition = components[component._component] || {}
-  const name =
+  return componentDefinition.friendlyName || componentDefinition.name || ""
-    componentDefinition.friendlyName || componentDefinition.name || ""
+}
-
+
-  return name
+/**
+ * Recurses through the component tree and builds a tree of contexts provided
+ * by components.
+ */
+export const buildContextTree = (
+  rootComponent,
+  tree = { root: [] },
+  currentBranch = "root"
+) => {
+  // Sanity check
+  if (!rootComponent) {
+    return tree
+  }
+
+  // Process this component's contexts
+  const def = store.actions.components.getDefinition(rootComponent._component)
+  if (def?.context) {
+    tree[currentBranch].push(rootComponent._id)
+    const contexts = Array.isArray(def.context) ? def.context : [def.context]
+
+    // If we provide local context, start a new branch for our children
+    if (contexts.some(context => context.scope === ContextScopes.Local)) {
+      currentBranch = rootComponent._id
+      tree[rootComponent._id] = []
+    }
+  }
+
+  // Process children
+  if (rootComponent._children) {
+    rootComponent._children.forEach(child => {
+      buildContextTree(child, tree, currentBranch)
+    })
+  }
+
+  return tree
+}
+
+/**
+ * Generates a lookup map of which context branch all components in a component
+ * tree are inside.
+ */
+export const buildContextTreeLookupMap = rootComponent => {
+  const tree = buildContextTree(rootComponent)
+  let map = {}
+  Object.entries(tree).forEach(([branch, ids]) => {
+    ids.forEach(id => {
+      map[id] = branch
+    })
+  })
+  return map
 }

packages/builder/src/builderStore/dataBinding.js

@@ -1,6 +1,7 @@
 import { cloneDeep } from "lodash/fp"
 import { get } from "svelte/store"
 import {
+  buildContextTreeLookupMap,
   findAllComponents,
   findAllMatchingComponents,
   findComponent,
@@ -20,11 +21,13 @@ import {
   encodeJSBinding,
 } from "@budibase/string-templates"
 import { TableNames } from "../constants"
-import { JSONUtils } from "@budibase/frontend-core"
+import { JSONUtils, Constants } from "@budibase/frontend-core"
 import ActionDefinitions from "components/design/settings/controls/ButtonActionEditor/manifest.json"
 import { environment, licensing } from "stores/portal"
 import { convertOldFieldFormat } from "components/design/settings/controls/FieldConfiguration/utils"
 
+const { ContextScopes } = Constants
+
 // Regex to match all instances of template strings
 const CAPTURE_VAR_INSIDE_TEMPLATE = /{{([^}]+)}}/g
 const CAPTURE_VAR_INSIDE_JS = /\$\("([^")]+)"\)/g
@@ -214,20 +217,27 @@ export const getComponentContexts = (
     return []
   }
   let map = {}
+  const componentPath = findComponentPath(asset.props, componentId)
+  const componentPathIds = componentPath.map(component => component._id)
+  const contextTreeLookupMap = buildContextTreeLookupMap(asset.props)
 
   // Processes all contexts exposed by a component
   const processContexts = scope => component => {
-    const def = store.actions.components.getDefinition(component._component)
+    // Sanity check
+    const def = store.actions.components.getDefinition(component?._component)
     if (!def?.context) {
       return
     }
-    if (!map[component._id]) {
+
-      map[component._id] = {
+    // Filter out global contexts not in the same branch.
-        component,
+    // Global contexts are only valid if their branch root is an ancestor of
-        definition: def,
+    // this component.
-        contexts: [],
+    const branch = contextTreeLookupMap[component._id]
-      }
+    if (branch !== "root" && !componentPathIds.includes(branch)) {
+      return
     }
+
+    // Process all contexts provided by this component
     const contexts = Array.isArray(def.context) ? def.context : [def.context]
     contexts.forEach(context => {
       // Ensure type matches
@@ -235,7 +245,7 @@ export const getComponentContexts = (
         return
       }
       // Ensure scope matches
-      let contextScope = context.scope || "global"
+      let contextScope = context.scope || ContextScopes.Global
       if (contextScope !== scope) {
         return
       }
@@ -243,17 +253,23 @@ export const getComponentContexts = (
       if (!isContextCompatibleWithComponent(context, component)) {
         return
       }
+      if (!map[component._id]) {
+        map[component._id] = {
+          component,
+          definition: def,
+          contexts: [],
+        }
+      }
       map[component._id].contexts.push(context)
     })
   }
 
   // Process all global contexts
   const allComponents = findAllComponents(asset.props)
-  allComponents.forEach(processContexts("global"))
+  allComponents.forEach(processContexts(ContextScopes.Global))
 
-  // Process all local contexts
+  // Process all local contexts in the immediate tree
-  const localComponents = findComponentPath(asset.props, componentId)
+  componentPath.forEach(processContexts(ContextScopes.Local))
-  localComponents.forEach(processContexts("local"))
 
   // Exclude self if required
   if (!options?.includeSelf) {

packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte

@@ -15,7 +15,6 @@
     Icon,
     Checkbox,
     DatePicker,
-    Detail,
   } from "@budibase/bbui"
   import CreateWebhookModal from "components/automation/Shared/CreateWebhookModal.svelte"
   import { automationStore, selectedAutomation } from "builderStore"
@@ -33,6 +32,8 @@
   import Editor from "components/integration/QueryEditor.svelte"
   import ModalBindableInput from "components/common/bindings/ModalBindableInput.svelte"
   import CodeEditor from "components/common/CodeEditor/CodeEditor.svelte"
+  import BindingPicker from "components/common/bindings/BindingPicker.svelte"
+  import { BindingHelpers } from "components/common/bindings/utils"
   import {
     bindingsToCompletions,
     hbAutocomplete,
@@ -56,7 +57,7 @@
   let drawer
   let fillWidth = true
   let inputData
-  let codeBindingOpen = false
+  let insertAtPos, getCaretPosition
   $: filters = lookForFilters(schemaProperties) || []
   $: tempFilters = filters
   $: stepId = block.stepId
@@ -75,6 +76,10 @@
   $: isUpdateRow = stepId === ActionStepID.UPDATE_ROW
   $: codeMode =
     stepId === "EXECUTE_BASH" ? EditorModes.Handlebars : EditorModes.JS
+  $: bindingsHelpers = new BindingHelpers(getCaretPosition, insertAtPos, {
+    disableWrapping: true,
+  })
+  $: editingJs = codeMode === EditorModes.JS
 
   $: stepCompletions =
     codeMode === EditorModes.Handlebars
@@ -539,39 +544,51 @@
             />
           {:else if value.customType === "code"}
             <CodeEditorModal>
-              {#if codeMode == EditorModes.JS}
+              <div class:js-editor={editingJs}>
-                <ActionButton
+                <div class:js-code={editingJs} style="width: 100%">
-                  on:click={() => (codeBindingOpen = !codeBindingOpen)}
+                  <CodeEditor
-                  quiet
+                    value={inputData[key]}
-                  icon={codeBindingOpen ? "ChevronDown" : "ChevronRight"}
+                    on:change={e => {
-                >
+                      // need to pass without the value inside
-                  <Detail size="S">Bindings</Detail>
+                      onChange({ detail: e.detail }, key)
-                </ActionButton>
+                      inputData[key] = e.detail
-                {#if codeBindingOpen}
+                    }}
-                  <pre>{JSON.stringify(bindings, null, 2)}</pre>
+                    completions={stepCompletions}
-                {/if}
+                    mode={codeMode}
-              {/if}
+                    autocompleteEnabled={codeMode !== EditorModes.JS}
-              <CodeEditor
+                    bind:getCaretPosition
-                value={inputData[key]}
+                    bind:insertAtPos
-                on:change={e => {
+                    height={500}
-                  // need to pass without the value inside
+                  />
-                  onChange({ detail: e.detail }, key)
+                  <div class="messaging">
-                  inputData[key] = e.detail
+                    {#if codeMode === EditorModes.Handlebars}
-                }}
+                      <Icon name="FlashOn" />
-                completions={stepCompletions}
+                      <div class="messaging-wrap">
-                mode={codeMode}
+                        <div>
-                autocompleteEnabled={codeMode != EditorModes.JS}
+                          Add available bindings by typing <strong>
-                height={500}
+                            &#125;&#125;
-              />
+                          </strong>
-              <div class="messaging">
+                        </div>
-                {#if codeMode == EditorModes.Handlebars}
+                      </div>
-                  <Icon name="FlashOn" />
+                    {/if}
-                  <div class="messaging-wrap">
+                  </div>
-                    <div>
+                </div>
-                      Add available bindings by typing <strong>
+                {#if editingJs}
-                        &#125;&#125;
+                  <div class="js-binding-picker">
-                      </strong>
+                    <BindingPicker
-                    </div>
+                      {bindings}
+                      allowHelpers={false}
+                      addBinding={binding =>
+                        bindingsHelpers.onSelectBinding(
+                          inputData[key],
+                          binding,
+                          {
+                            js: true,
+                            dontDecode: true,
+                          }
+                        )}
+                      mode="javascript"
+                    />
                   </div>
                 {/if}
               </div>
@@ -658,4 +675,20 @@
   .test :global(.drawer) {
     width: 10000px !important;
   }
+
+  .js-editor {
+    display: flex;
+    flex-direction: row;
+    flex-grow: 1;
+    width: 100%;
+  }
+
+  .js-code {
+    flex: 7;
+  }
+
+  .js-binding-picker {
+    flex: 3;
+    margin-top: calc((var(--spacing-xl) * -1) + 1px);
+  }
 </style>

packages/builder/src/components/common/CodeEditor/CodeEditor.svelte

@@ -54,6 +54,7 @@
   export let placeholder = null
   export let autocompleteEnabled = true
   export let autofocus = false
+  export let jsBindingWrapping = true
 
   // Export a function to expose caret position
   export const getCaretPosition = () => {
@@ -187,7 +188,7 @@
       )
       complete.push(
         EditorView.inputHandler.of((view, from, to, insert) => {
-          if (insert === "$") {
+          if (jsBindingWrapping && insert === "$") {
             let { text } = view.state.doc.lineAt(from)
 
             const left = from ? text.substring(0, from) : ""

packages/builder/src/components/common/CodeEditor/index.js

@@ -286,13 +286,14 @@ export const hbInsert = (value, from, to, text) => {
   return parsedInsert
 }
 
-export function jsInsert(value, from, to, text, { helper } = {}) {
+export function jsInsert(value, from, to, text, { helper, disableWrapping }) {
   let parsedInsert = ""
 
   const left = from ? value.substring(0, from) : ""
   const right = to ? value.substring(to) : ""
-
+  if (disableWrapping) {
-  if (helper) {
+    parsedInsert = text
+  } else if (helper) {
     parsedInsert = `helpers.${text}()`
   } else if (!left.includes('$("') || !right.includes('")')) {
     parsedInsert = `$("${text}")`

packages/builder/src/components/common/bindings/BindingPanel.svelte

@@ -29,10 +29,9 @@
     hbAutocomplete,
     EditorModes,
     bindingsToCompletions,
-    hbInsert,
-    jsInsert,
   } from "../CodeEditor"
   import BindingPicker from "./BindingPicker.svelte"
+  import { BindingHelpers } from "./utils"
 
   const dispatch = createEventDispatcher()
 
@@ -60,8 +59,10 @@
   let targetMode = null
 
   $: usingJS = mode === "JavaScript"
-  $: editorMode = mode == "JavaScript" ? EditorModes.JS : EditorModes.Handlebars
+  $: editorMode =
+    mode === "JavaScript" ? EditorModes.JS : EditorModes.Handlebars
   $: bindingCompletions = bindingsToCompletions(bindings, editorMode)
+  $: bindingHelpers = new BindingHelpers(getCaretPosition, insertAtPos)
 
   const updateValue = val => {
     valid = isValid(readableToRuntimeBinding(bindings, val))
@@ -70,31 +71,13 @@
     }
   }
 
-  // Adds a JS/HBS helper to the expression
   const onSelectHelper = (helper, js) => {
-    const pos = getCaretPosition()
+    bindingHelpers.onSelectHelper(js ? jsValue : hbsValue, helper, { js })
-    const { start, end } = pos
-    if (js) {
-      let js = decodeJSBinding(jsValue)
-      const insertVal = jsInsert(js, start, end, helper.text, { helper: true })
-      insertAtPos({ start, end, value: insertVal })
-    } else {
-      const insertVal = hbInsert(hbsValue, start, end, helper.text)
-      insertAtPos({ start, end, value: insertVal })
-    }
   }
 
-  // Adds a data binding to the expression
   const onSelectBinding = (binding, { forceJS } = {}) => {
-    const { start, end } = getCaretPosition()
+    const js = usingJS || forceJS
-    if (usingJS || forceJS) {
+    bindingHelpers.onSelectBinding(js ? jsValue : hbsValue, binding, { js })
-      let js = decodeJSBinding(jsValue)
-      const insertVal = jsInsert(js, start, end, binding.readableBinding)
-      insertAtPos({ start, end, value: insertVal })
-    } else {
-      const insertVal = hbInsert(hbsValue, start, end, binding.readableBinding)
-      insertAtPos({ start, end, value: insertVal })
-    }
   }
 
   const onChangeMode = e => {

packages/builder/src/components/common/bindings/BindingPicker.svelte

@@ -9,6 +9,7 @@
   export let bindings
   export let mode
   export let allowHelpers
+  export let noPaddingTop = false
 
   let search = ""
   let popover

packages/builder/src/components/common/bindings/utils.js

@@ -1,38 +1,41 @@
-export function addHBSBinding(value, caretPos, binding) {
+import { decodeJSBinding } from "@budibase/string-templates"
-  binding = typeof binding === "string" ? binding : binding.path
+import { hbInsert, jsInsert } from "components/common/CodeEditor"
-  value = value == null ? "" : value
 
-  const left = caretPos?.start ? value.substring(0, caretPos.start) : ""
+export class BindingHelpers {
-  const right = caretPos?.end ? value.substring(caretPos.end) : ""
+  constructor(getCaretPosition, insertAtPos, { disableWrapping } = {}) {
-  if (!left.includes("{{") || !right.includes("}}")) {
+    this.getCaretPosition = getCaretPosition
-    binding = `{{ ${binding} }}`
+    this.insertAtPos = insertAtPos
+    this.disableWrapping = disableWrapping
   }
-  if (caretPos.start) {
-    value =
-      value.substring(0, caretPos.start) +
-      binding +
-      value.substring(caretPos.end, value.length)
-  } else {
-    value += binding
-  }
-  return value
-}
 
-export function addJSBinding(value, caretPos, binding, { helper } = {}) {
+  // Adds a JS/HBS helper to the expression
-  binding = typeof binding === "string" ? binding : binding.path
+  onSelectHelper(value, helper, { js, dontDecode }) {
-  value = value == null ? "" : value
+    const pos = this.getCaretPosition()
-  if (!helper) {
+    const { start, end } = pos
-    binding = `$("${binding}")`
+    if (js) {
-  } else {
+      const jsVal = dontDecode ? value : decodeJSBinding(value)
-    binding = `helpers.${binding}()`
+      const insertVal = jsInsert(jsVal, start, end, helper.text, {
+        helper: true,
+      })
+      this.insertAtPos({ start, end, value: insertVal })
+    } else {
+      const insertVal = hbInsert(value, start, end, helper.text)
+      this.insertAtPos({ start, end, value: insertVal })
+    }
   }
-  if (caretPos.start) {
+
-    value =
+  // Adds a data binding to the expression
-      value.substring(0, caretPos.start) +
+  onSelectBinding(value, binding, { js, dontDecode }) {
-      binding +
+    const { start, end } = this.getCaretPosition()
-      value.substring(caretPos.end, value.length)
+    if (js) {
-  } else {
+      const jsVal = dontDecode ? value : decodeJSBinding(value)
-    value += binding
+      const insertVal = jsInsert(jsVal, start, end, binding.readableBinding, {
+        disableWrapping: this.disableWrapping,
+      })
+      this.insertAtPos({ start, end, value: insertVal })
+    } else {
+      const insertVal = hbInsert(value, start, end, binding.readableBinding)
+      this.insertAtPos({ start, end, value: insertVal })
+    }
   }
-  return value
 }

packages/builder/src/components/integration/QueryViewer.svelte

@@ -94,7 +94,13 @@
 
       notifications.success("Query executed successfully")
     } catch (error) {
-      notifications.error(`Query Error: ${error.message}`)
+      if (typeof error.message === "string") {
+        notifications.error(`Query Error: ${error.message}`)
+      } else if (typeof error.message?.code === "string") {
+        notifications.error(`Query Error: ${error.message.code}`)
+      } else {
+        notifications.error(`Query Error: ${JSON.stringify(error.message)}`)
+      }
 
       if (!suppressErrors) {
         throw error

packages/builder/src/pages/builder/app/[application]/data/datasource/[datasourceId]/index.svelte

@@ -12,12 +12,16 @@
   import PromptQueryModal from "./_components/PromptQueryModal.svelte"
   import SettingsPanel from "./_components/panels/Settings.svelte"
   import { helpers } from "@budibase/shared-core"
+  import { admin } from "stores/portal"
+  import { IntegrationTypes } from "constants/backend"
 
   let selectedPanel = null
   let panelOptions = []
 
   $: datasource = $datasources.selected
 
+  $: isCloud = $admin.cloud
+  $: isPostgres = datasource?.source === IntegrationTypes.POSTGRES
   $: getOptions(datasource)
 
   const getOptions = datasource => {
@@ -41,7 +45,13 @@
     }
     // always the last option for SQL
     if (helpers.isSQL(datasource)) {
-      panelOptions.push("Settings")
+      if (isCloud && isPostgres) {
+        // We don't show the settings panel for Postgres on Budicloud because
+        // it requires pg_dump to work and we don't want to enable shell injection
+        // attacks.
+      } else {
+        panelOptions.push("Settings")
+      }
     }
   }
 </script>

packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Navigation/index.svelte

@@ -15,10 +15,15 @@
     Checkbox,
     notifications,
     Select,
+    Combobox,
   } from "@budibase/bbui"
   import { selectedScreen, store } from "builderStore"
   import { DefaultAppTheme } from "constants"
 
+  $: screenRouteOptions = $store.screens
+    .map(screen => screen.routing?.route)
+    .filter(x => x != null)
+
   const updateShowNavigation = async e => {
     await store.actions.screens.updateSetting(
       get(selectedScreen),
@@ -107,23 +112,6 @@
             on:change={e => update("navWidth", e.detail)}
           />
         {/if}
-        <div class="label">
-          <Label size="M">Show logo</Label>
-        </div>
-        <Checkbox
-          value={!$store.navigation.hideLogo}
-          on:change={e => update("hideLogo", !e.detail)}
-        />
-        {#if !$store.navigation.hideLogo}
-          <div class="label">
-            <Label size="M">Logo URL</Label>
-          </div>
-          <Input
-            value={$store.navigation.logoUrl}
-            on:change={e => update("logoUrl", e.detail)}
-            updateOnChange={false}
-          />
-        {/if}
         <div class="label">
           <Label size="M">Show title</Label>
         </div>
@@ -160,6 +148,47 @@
         />
       </div>
     </div>
+
+    <div class="divider" />
+    <div class="customizeSection">
+      <div class="subheading">
+        <Detail>Logo</Detail>
+      </div>
+      <div class="controls">
+        <div class="label">
+          <Label size="M">Show logo</Label>
+        </div>
+        <Checkbox
+          value={!$store.navigation.hideLogo}
+          on:change={e => update("hideLogo", !e.detail)}
+        />
+        {#if !$store.navigation.hideLogo}
+          <div class="label">
+            <Label size="M">Logo image URL</Label>
+          </div>
+          <Input
+            value={$store.navigation.logoUrl}
+            on:change={e => update("logoUrl", e.detail)}
+            updateOnChange={false}
+          />
+          <div class="label">
+            <Label size="M">Logo link URL</Label>
+          </div>
+          <Combobox
+            value={$store.navigation.logoLinkUrl}
+            on:change={e => update("logoLinkUrl", e.detail)}
+            options={screenRouteOptions}
+          />
+          <div class="label">
+            <Label size="M">New tab</Label>
+          </div>
+          <Checkbox
+            value={!!$store.navigation.openLogoLinkInNewTab}
+            on:change={e => update("openLogoLinkInNewTab", !!e.detail)}
+          />
+        {/if}
+      </div>
+    </div>
   {/if}
 </Panel>
 

packages/client/manifest.json

@@ -4720,7 +4720,8 @@
       }
     ],
     "context": {
-      "type": "schema"
+      "type": "schema",
+      "scope": "local"
     }
   },
   "daterangepicker": {
@@ -6742,6 +6743,17 @@
         "key": "disabled",
         "defaultValue": false
       },
+      {
+        "type": "boolean",
+        "label": "Read only",
+        "key": "readonly",
+        "defaultValue": false,
+        "dependsOn": {
+          "setting": "disabled",
+          "value": true,
+          "invert": true
+        }
+      },
       {
         "type": "select",
         "label": "Layout",

packages/client/src/components/app/Layout.svelte

@@ -33,6 +33,8 @@
   export let navTextColor
   export let navWidth
   export let pageWidth
+  export let logoLinkUrl
+  export let openLogoLinkInNewTab
 
   export let embedded = false
 
@@ -150,6 +152,16 @@
     }
     return style
   }
+
+  const getSanitizedUrl = (url, openInNewTab) => {
+    if (!isInternal(url)) {
+      return ensureExternal(url)
+    }
+    if (openInNewTab) {
+      return `#${url}`
+    }
+    return url
+  }
 </script>
 
 <div
@@ -192,7 +204,23 @@
               {/if}
               <div class="logo">
                 {#if !hideLogo}
-                  <img src={logoUrl || "/builder/bblogo.png"} alt={title} />
+                  {#if logoLinkUrl && isInternal(logoLinkUrl) && !openLogoLinkInNewTab}
+                    <a
+                      href={getSanitizedUrl(logoLinkUrl, openLogoLinkInNewTab)}
+                      use:linkable
+                    >
+                      <img src={logoUrl || "/builder/bblogo.png"} alt={title} />
+                    </a>
+                  {:else if logoLinkUrl}
+                    <a
+                      target={openLogoLinkInNewTab ? "_blank" : "_self"}
+                      href={getSanitizedUrl(logoLinkUrl, openLogoLinkInNewTab)}
+                    >
+                      <img src={logoUrl || "/builder/bblogo.png"} alt={title} />
+                    </a>
+                  {:else}
+                    <img src={logoUrl || "/builder/bblogo.png"} alt={title} />
+                  {/if}
                 {/if}
                 {#if !hideTitle && title}
                   <Heading size="S">{title}</Heading>

packages/client/src/components/app/Repeater.svelte

@@ -2,7 +2,9 @@
   import { getContext } from "svelte"
   import Placeholder from "./Placeholder.svelte"
   import Container from "./Container.svelte"
-  import { ContextScopes } from "constants"
+
+  const { Provider, ContextScopes } = getContext("sdk")
+  const component = getContext("component")
 
   export let dataProvider
   export let noRowsMessage
@@ -12,9 +14,6 @@
   export let gap
   export let scope = ContextScopes.Local
 
-  const { Provider } = getContext("sdk")
-  const component = getContext("component")
-
   $: rows = dataProvider?.rows ?? []
   $: loaded = dataProvider?.loaded ?? true
 </script>

packages/client/src/components/app/table/SlotRenderer.svelte

@@ -3,9 +3,9 @@
 
   export let row
 
-  const { Provider } = getContext("sdk")
+  const { Provider, ContextScopes } = getContext("sdk")
 </script>
 
-<Provider data={row}>
+<Provider data={row} scope={ContextScopes.Local}>
   <slot />
 </Provider>

packages/client/src/components/context/Provider.svelte

@@ -1,9 +1,11 @@
 <script>
   import { getContext, setContext, onDestroy } from "svelte"
   import { dataSourceStore, createContextStore } from "stores"
-  import { ActionTypes, ContextScopes } from "constants"
+  import { ActionTypes } from "constants"
   import { generate } from "shortid"
 
+  const { ContextScopes } = getContext("sdk")
+
   export let data
   export let actions
   export let key
@@ -33,7 +35,7 @@
   const provideData = newData => {
     const dataKey = JSON.stringify(newData)
     if (dataKey !== lastDataKey) {
-      context.actions.provideData(providerKey, newData, scope)
+      context.actions.provideData(providerKey, newData)
       lastDataKey = dataKey
     }
   }
@@ -43,7 +45,7 @@
     if (actionsKey !== lastActionsKey) {
       lastActionsKey = actionsKey
       newActions?.forEach(({ type, callback, metadata }) => {
-        context.actions.provideAction(providerKey, type, callback, scope)
+        context.actions.provideAction(providerKey, type, callback)
 
         // Register any "refresh datasource" actions with a singleton store
         // so we can easily refresh data at all levels for any datasource

packages/client/src/constants.js

@@ -12,10 +12,5 @@ export const ActionTypes = {
   ScrollTo: "ScrollTo",
 }
 
-export const ContextScopes = {
-  Local: "local",
-  Global: "global",
-}
-
 export const DNDPlaceholderID = "dnd-placeholder"
 export const ScreenslotType = "screenslot"

packages/client/src/sdk.js

@@ -23,12 +23,12 @@ import { getAction } from "utils/getAction"
 import Provider from "components/context/Provider.svelte"
 import Block from "components/Block.svelte"
 import BlockComponent from "components/BlockComponent.svelte"
-import { ActionTypes, ContextScopes } from "./constants"
+import { ActionTypes } from "./constants"
 import { fetchDatasourceSchema } from "./utils/schema.js"
 import { getAPIKey } from "./utils/api.js"
 import { enrichButtonActions } from "./utils/buttonActions.js"
 import { processStringSync, makePropSafe } from "@budibase/string-templates"
-import { fetchData, LuceneUtils } from "@budibase/frontend-core"
+import { fetchData, LuceneUtils, Constants } from "@budibase/frontend-core"
 
 export default {
   API,
@@ -57,7 +57,7 @@ export default {
   fetchDatasourceSchema,
   fetchData,
   LuceneUtils,
-  ContextScopes,
+  ContextScopes: Constants.ContextScopes,
   getAPIKey,
   enrichButtonActions,
   processStringSync,

packages/client/src/stores/context.js

@@ -1,5 +1,4 @@
 import { writable, derived } from "svelte/store"
-import { ContextScopes } from "constants"
 
 export const createContextStore = parentContext => {
   const context = writable({})
@@ -20,60 +19,34 @@ export const createContextStore = parentContext => {
   }
 
   // Provide some data in context
-  const provideData = (providerId, data, scope = ContextScopes.Global) => {
+  const provideData = (providerId, data) => {
     if (!providerId || data === undefined) {
       return
     }
 
-    // Proxy message up the chain if we have a parent and are providing global
-    // context
-    if (scope === ContextScopes.Global && parentContext) {
-      parentContext.actions.provideData(providerId, data, scope)
-    }
-
     // Otherwise this is either the context root, or we're providing a local
     // context override, so we need to update the local context instead
-    else {
+    context.update(state => {
-      context.update(state => {
+      state[providerId] = data
-        state[providerId] = data
+      return state
-        return state
+    })
-      })
+    broadcastChange(providerId)
-      broadcastChange(providerId)
-    }
   }
 
   // Provides some action in context
-  const provideAction = (
+  const provideAction = (providerId, actionType, callback) => {
-    providerId,
-    actionType,
-    callback,
-    scope = ContextScopes.Global
-  ) => {
     if (!providerId || !actionType) {
       return
     }
 
-    // Proxy message up the chain if we have a parent and are providing global
-    // context
-    if (scope === ContextScopes.Global && parentContext) {
-      parentContext.actions.provideAction(
-        providerId,
-        actionType,
-        callback,
-        scope
-      )
-    }
-
     // Otherwise this is either the context root, or we're providing a local
     // context override, so we need to update the local context instead
-    else {
+    const key = `${providerId}_${actionType}`
-      const key = `${providerId}_${actionType}`
+    context.update(state => {
-      context.update(state => {
+      state[key] = callback
-        state[key] = callback
+      return state
-        return state
+    })
-      })
+    broadcastChange(key)
-      broadcastChange(key)
-    }
   }
 
   const observeChanges = callback => {

packages/frontend-core/src/constants.js

@@ -106,3 +106,8 @@ export const Themes = [
 export const EventPublishType = {
   ENV_VAR_UPGRADE_PANEL_OPENED: "environment_variable_upgrade_panel_opened",
 }
+
+export const ContextScopes = {
+  Local: "local",
+  Global: "global",
+}

packages/pro

@@ -1 +1 @@
-Subproject commit 992486c10044a7495496b97bdf5f454d4020bfba
+Subproject commit aaf7101cd1493215155cc8f83124c70d53eb1be4

packages/server/src/api/controllers/query/index.ts

@@ -15,6 +15,9 @@ import {
   SessionCookie,
   QuerySchema,
   FieldType,
+  type ExecuteQueryRequest,
+  type ExecuteQueryResponse,
+  type Row,
 } from "@budibase/types"
 import { ValidQueryNameRegex } from "@budibase/shared-core"
 
@@ -223,7 +226,7 @@ export async function preview(ctx: UserCtx) {
 }
 
 async function execute(
-  ctx: UserCtx,
+  ctx: UserCtx<ExecuteQueryRequest, ExecuteQueryResponse | Row[]>,
   opts: any = { rowsOnly: false, isAutomation: false }
 ) {
   const db = context.getAppDB()

packages/server/src/api/routes/tests/queries/mongodb.spec.ts

@@ -0,0 +1,390 @@
+import { Datasource, Query } from "@budibase/types"
+import * as setup from "../utilities"
+import { databaseTestProviders } from "../../../../integrations/tests/utils"
+import { MongoClient, type Collection } from "mongodb"
+
+jest.unmock("mongodb")
+
+const collection = "test_collection"
+
+describe("/queries", () => {
+  let config = setup.getConfig()
+  let datasource: Datasource
+
+  async function createQuery(query: Partial<Query>): Promise<Query> {
+    const defaultQuery: Query = {
+      datasourceId: datasource._id!,
+      name: "New Query",
+      parameters: [],
+      fields: {},
+      schema: {},
+      queryVerb: "read",
+      transformer: "return data",
+      readable: true,
+    }
+    const combinedQuery = { ...defaultQuery, ...query }
+    if (
+      combinedQuery.fields &&
+      combinedQuery.fields.extra &&
+      !combinedQuery.fields.extra.collection
+    ) {
+      combinedQuery.fields.extra.collection = collection
+    }
+    return await config.api.query.create(combinedQuery)
+  }
+
+  async function withClient(
+    callback: (client: MongoClient) => Promise<void>
+  ): Promise<void> {
+    const ds = await databaseTestProviders.mongodb.datasource()
+    const client = new MongoClient(ds.config!.connectionString)
+    await client.connect()
+    try {
+      await callback(client)
+    } finally {
+      await client.close()
+    }
+  }
+
+  async function withCollection(
+    callback: (collection: Collection) => Promise<void>
+  ): Promise<void> {
+    await withClient(async client => {
+      const db = client.db(
+        (await databaseTestProviders.mongodb.datasource()).config!.db
+      )
+      await callback(db.collection(collection))
+    })
+  }
+
+  afterAll(async () => {
+    await databaseTestProviders.mongodb.stop()
+    setup.afterAll()
+  })
+
+  beforeAll(async () => {
+    await config.init()
+    datasource = await config.api.datasource.create(
+      await databaseTestProviders.mongodb.datasource()
+    )
+  })
+
+  beforeEach(async () => {
+    await withCollection(async collection => {
+      await collection.insertMany([
+        { name: "one" },
+        { name: "two" },
+        { name: "three" },
+        { name: "four" },
+        { name: "five" },
+      ])
+    })
+  })
+
+  afterEach(async () => {
+    await withCollection(async collection => {
+      await collection.drop()
+    })
+  })
+
+  it("should execute a count query", async () => {
+    const query = await createQuery({
+      fields: {
+        json: {},
+        extra: {
+          actionType: "count",
+        },
+      },
+    })
+
+    const result = await config.api.query.execute(query._id!)
+
+    expect(result.data).toEqual([{ value: 5 }])
+  })
+
+  it("should execute a count query with a transformer", async () => {
+    const query = await createQuery({
+      fields: {
+        json: {},
+        extra: {
+          actionType: "count",
+        },
+      },
+      transformer: "return data + 1",
+    })
+
+    const result = await config.api.query.execute(query._id!)
+
+    expect(result.data).toEqual([{ value: 6 }])
+  })
+
+  it("should execute a find query", async () => {
+    const query = await createQuery({
+      fields: {
+        json: {},
+        extra: {
+          actionType: "find",
+        },
+      },
+    })
+
+    const result = await config.api.query.execute(query._id!)
+
+    expect(result.data).toEqual([
+      { _id: expect.anything(), name: "one" },
+      { _id: expect.anything(), name: "two" },
+      { _id: expect.anything(), name: "three" },
+      { _id: expect.anything(), name: "four" },
+      { _id: expect.anything(), name: "five" },
+    ])
+  })
+
+  it("should execute a findOne query", async () => {
+    const query = await createQuery({
+      fields: {
+        json: {},
+        extra: {
+          actionType: "findOne",
+        },
+      },
+    })
+
+    const result = await config.api.query.execute(query._id!)
+
+    expect(result.data).toEqual([{ _id: expect.anything(), name: "one" }])
+  })
+
+  it("should execute a findOneAndUpdate query", async () => {
+    const query = await createQuery({
+      fields: {
+        json: {
+          filter: { name: { $eq: "one" } },
+          update: { $set: { name: "newName" } },
+        },
+        extra: {
+          actionType: "findOneAndUpdate",
+        },
+      },
+    })
+
+    const result = await config.api.query.execute(query._id!)
+
+    expect(result.data).toEqual([
+      {
+        lastErrorObject: { n: 1, updatedExisting: true },
+        ok: 1,
+        value: { _id: expect.anything(), name: "one" },
+      },
+    ])
+
+    await withCollection(async collection => {
+      expect(await collection.countDocuments()).toBe(5)
+
+      const doc = await collection.findOne({ name: { $eq: "newName" } })
+      expect(doc).toEqual({
+        _id: expect.anything(),
+        name: "newName",
+      })
+    })
+  })
+
+  it("should execute a distinct query", async () => {
+    const query = await createQuery({
+      fields: {
+        json: "name",
+        extra: {
+          actionType: "distinct",
+        },
+      },
+    })
+
+    const result = await config.api.query.execute(query._id!)
+    const values = result.data.map(o => o.value).sort()
+    expect(values).toEqual(["five", "four", "one", "three", "two"])
+  })
+
+  it("should execute a create query with parameters", async () => {
+    const query = await createQuery({
+      fields: {
+        json: { foo: "{{ foo }}" },
+        extra: {
+          actionType: "insertOne",
+        },
+      },
+      queryVerb: "create",
+      parameters: [
+        {
+          name: "foo",
+          default: "default",
+        },
+      ],
+    })
+
+    const result = await config.api.query.execute(query._id!, {
+      parameters: { foo: "bar" },
+    })
+
+    expect(result.data).toEqual([
+      {
+        acknowledged: true,
+        insertedId: expect.anything(),
+      },
+    ])
+
+    await withCollection(async collection => {
+      const doc = await collection.findOne({ foo: { $eq: "bar" } })
+      expect(doc).toEqual({
+        _id: expect.anything(),
+        foo: "bar",
+      })
+    })
+  })
+
+  it("should execute a delete query with parameters", async () => {
+    const query = await createQuery({
+      fields: {
+        json: { name: { $eq: "{{ name }}" } },
+        extra: {
+          actionType: "deleteOne",
+        },
+      },
+      queryVerb: "delete",
+      parameters: [
+        {
+          name: "name",
+          default: "",
+        },
+      ],
+    })
+
+    const result = await config.api.query.execute(query._id!, {
+      parameters: { name: "one" },
+    })
+
+    expect(result.data).toEqual([
+      {
+        acknowledged: true,
+        deletedCount: 1,
+      },
+    ])
+
+    await withCollection(async collection => {
+      const doc = await collection.findOne({ name: { $eq: "one" } })
+      expect(doc).toBeNull()
+    })
+  })
+
+  it("should execute an update query with parameters", async () => {
+    const query = await createQuery({
+      fields: {
+        json: {
+          filter: { name: { $eq: "{{ name }}" } },
+          update: { $set: { name: "{{ newName }}" } },
+        },
+        extra: {
+          actionType: "updateOne",
+        },
+      },
+      queryVerb: "update",
+      parameters: [
+        {
+          name: "name",
+          default: "",
+        },
+        {
+          name: "newName",
+          default: "",
+        },
+      ],
+    })
+
+    const result = await config.api.query.execute(query._id!, {
+      parameters: { name: "one", newName: "newOne" },
+    })
+
+    expect(result.data).toEqual([
+      {
+        acknowledged: true,
+        matchedCount: 1,
+        modifiedCount: 1,
+        upsertedCount: 0,
+        upsertedId: null,
+      },
+    ])
+
+    await withCollection(async collection => {
+      const doc = await collection.findOne({ name: { $eq: "newOne" } })
+      expect(doc).toEqual({
+        _id: expect.anything(),
+        name: "newOne",
+      })
+
+      const oldDoc = await collection.findOne({ name: { $eq: "one" } })
+      expect(oldDoc).toBeNull()
+    })
+  })
+
+  it("should be able to delete all records", async () => {
+    const query = await createQuery({
+      fields: {
+        json: {},
+        extra: {
+          actionType: "deleteMany",
+        },
+      },
+      queryVerb: "delete",
+    })
+
+    const result = await config.api.query.execute(query._id!)
+
+    expect(result.data).toEqual([
+      {
+        acknowledged: true,
+        deletedCount: 5,
+      },
+    ])
+
+    await withCollection(async collection => {
+      const docs = await collection.find().toArray()
+      expect(docs).toHaveLength(0)
+    })
+  })
+
+  it("should be able to update all documents", async () => {
+    const query = await createQuery({
+      fields: {
+        json: {
+          filter: {},
+          update: { $set: { name: "newName" } },
+        },
+        extra: {
+          actionType: "updateMany",
+        },
+      },
+      queryVerb: "update",
+    })
+
+    const result = await config.api.query.execute(query._id!)
+
+    expect(result.data).toEqual([
+      {
+        acknowledged: true,
+        matchedCount: 5,
+        modifiedCount: 5,
+        upsertedCount: 0,
+        upsertedId: null,
+      },
+    ])
+
+    await withCollection(async collection => {
+      const docs = await collection.find().toArray()
+      expect(docs).toHaveLength(5)
+      for (const doc of docs) {
+        expect(doc).toEqual({
+          _id: expect.anything(),
+          name: "newName",
+        })
+      }
+    })
+  })
+})

packages/server/src/api/routes/tests/queries/postgres.spec.ts

@@ -0,0 +1,170 @@
+import { Datasource, Query } from "@budibase/types"
+import * as setup from "../utilities"
+import { databaseTestProviders } from "../../../../integrations/tests/utils"
+import { Client } from "pg"
+
+jest.unmock("pg")
+
+const createTableSQL = `
+CREATE TABLE test_table (
+    id serial PRIMARY KEY,
+    name VARCHAR ( 50 ) NOT NULL
+);
+`
+
+const insertSQL = `
+INSERT INTO test_table (name) VALUES ('one');
+INSERT INTO test_table (name) VALUES ('two');
+INSERT INTO test_table (name) VALUES ('three');
+INSERT INTO test_table (name) VALUES ('four');
+INSERT INTO test_table (name) VALUES ('five');
+`
+
+const dropTableSQL = `
+DROP TABLE test_table;
+`
+
+describe("/queries", () => {
+  let config = setup.getConfig()
+  let datasource: Datasource
+
+  async function createQuery(query: Partial<Query>): Promise<Query> {
+    const defaultQuery: Query = {
+      datasourceId: datasource._id!,
+      name: "New Query",
+      parameters: [],
+      fields: {},
+      schema: {},
+      queryVerb: "read",
+      transformer: "return data",
+      readable: true,
+    }
+    return await config.api.query.create({ ...defaultQuery, ...query })
+  }
+
+  async function withClient(
+    callback: (client: Client) => Promise<void>
+  ): Promise<void> {
+    const ds = await databaseTestProviders.postgres.datasource()
+    const client = new Client(ds.config!)
+    await client.connect()
+    try {
+      await callback(client)
+    } finally {
+      await client.end()
+    }
+  }
+
+  afterAll(async () => {
+    await databaseTestProviders.postgres.stop()
+    setup.afterAll()
+  })
+
+  beforeAll(async () => {
+    await config.init()
+    datasource = await config.api.datasource.create(
+      await databaseTestProviders.postgres.datasource()
+    )
+  })
+
+  beforeEach(async () => {
+    await withClient(async client => {
+      await client.query(createTableSQL)
+      await client.query(insertSQL)
+    })
+  })
+
+  afterEach(async () => {
+    await withClient(async client => {
+      await client.query(dropTableSQL)
+    })
+  })
+
+  it("should execute a query", async () => {
+    const query = await createQuery({
+      fields: {
+        sql: "SELECT * FROM test_table ORDER BY id",
+      },
+    })
+
+    const result = await config.api.query.execute(query._id!)
+
+    expect(result.data).toEqual([
+      {
+        id: 1,
+        name: "one",
+      },
+      {
+        id: 2,
+        name: "two",
+      },
+      {
+        id: 3,
+        name: "three",
+      },
+      {
+        id: 4,
+        name: "four",
+      },
+      {
+        id: 5,
+        name: "five",
+      },
+    ])
+  })
+
+  it("should be able to transform a query", async () => {
+    const query = await createQuery({
+      fields: {
+        sql: "SELECT * FROM test_table WHERE id = 1",
+      },
+      transformer: `
+        data[0].id = data[0].id + 1;
+        return data;
+      `,
+    })
+
+    const result = await config.api.query.execute(query._id!)
+
+    expect(result.data).toEqual([
+      {
+        id: 2,
+        name: "one",
+      },
+    ])
+  })
+
+  it("should be able to insert with bindings", async () => {
+    const query = await createQuery({
+      fields: {
+        sql: "INSERT INTO test_table (name) VALUES ({{ foo }})",
+      },
+      parameters: [
+        {
+          name: "foo",
+          default: "bar",
+        },
+      ],
+      queryVerb: "create",
+    })
+
+    const result = await config.api.query.execute(query._id!, {
+      parameters: {
+        foo: "baz",
+      },
+    })
+
+    expect(result.data).toEqual([
+      {
+        created: true,
+      },
+    ])
+
+    await withClient(async client => {
+      const { rows } = await client.query(
+        "SELECT * FROM test_table WHERE name = 'baz'"
+      )
+      expect(rows).toHaveLength(1)
+    })
+  })
+})

packages/server/src/api/routes/tests/queries/query.seq.spec.ts

@@ -16,9 +16,9 @@ jest.mock("@budibase/backend-core", () => {
     },
   }
 })
-import * as setup from "./utilities"
+import * as setup from "../utilities"
-import { checkBuilderEndpoint } from "./utilities/TestFunctions"
+import { checkBuilderEndpoint } from "../utilities/TestFunctions"
-import { checkCacheForDynamicVariable } from "../../../threads/utils"
+import { checkCacheForDynamicVariable } from "../../../../threads/utils"
 
 const { basicQuery, basicDatasource } = setup.structures
 import { events, db as dbCore } from "@budibase/backend-core"

packages/server/src/api/routes/tests/row.spec.ts

@@ -12,7 +12,6 @@ import {
   FieldTypeSubtypes,
   FormulaType,
   INTERNAL_TABLE_SOURCE_ID,
-  MonthlyQuotaName,
   PermissionLevel,
   QuotaUsageType,
   RelationshipType,
@@ -53,7 +52,7 @@ describe.each([
 
   afterAll(async () => {
     if (dsProvider) {
-      await dsProvider.stopContainer()
+      await dsProvider.stop()
     }
     setup.afterAll()
   })
@@ -63,7 +62,7 @@ describe.each([
 
     if (dsProvider) {
       await config.createDatasource({
-        datasource: await dsProvider.getDsConfig(),
+        datasource: await dsProvider.datasource(),
       })
     }
   })
@@ -117,16 +116,6 @@ describe.each([
     return total
   }
 
-  const getQueryUsage = async () => {
-    const { total } = await config.doInContext(null, () =>
-      quotas.getCurrentUsageValues(
-        QuotaUsageType.MONTHLY,
-        MonthlyQuotaName.QUERIES
-      )
-    )
-    return total
-  }
-
   const assertRowUsage = async (expected: number) => {
     const usage = await getRowUsage()
     expect(usage).toBe(expected)
@@ -162,7 +151,6 @@ describe.each([
   describe("save, load, update", () => {
     it("returns a success message when the row is created", async () => {
       const rowUsage = await getRowUsage()
-      const queryUsage = await getQueryUsage()
 
       const res = await request
         .post(`/api/${tableId}/rows`)
@@ -180,7 +168,6 @@ describe.each([
 
     it("Increment row autoId per create row request", async () => {
       const rowUsage = await getRowUsage()
-      const queryUsage = await getQueryUsage()
 
       const tableConfig = generateTableConfig()
       const newTable = await createTable(
@@ -231,7 +218,6 @@ describe.each([
     it("updates a row successfully", async () => {
       const existing = await config.createRow()
       const rowUsage = await getRowUsage()
-      const queryUsage = await getQueryUsage()
 
       const res = await config.api.row.save(tableId, {
         _id: existing._id,
@@ -246,7 +232,6 @@ describe.each([
 
     it("should load a row", async () => {
       const existing = await config.createRow()
-      const queryUsage = await getQueryUsage()
 
       const res = await config.api.row.get(tableId, existing._id!)
 
@@ -268,7 +253,6 @@ describe.each([
       }
       const firstRow = await config.createRow({ tableId })
       await config.createRow(newRow)
-      const queryUsage = await getQueryUsage()
 
       const res = await config.api.row.fetch(tableId)
 
@@ -279,7 +263,6 @@ describe.each([
 
     it("load should return 404 when row does not exist", async () => {
       await config.createRow()
-      const queryUsage = await getQueryUsage()
 
       await config.api.row.get(tableId, "1234567", {
         expectStatus: 404,
@@ -530,7 +513,6 @@ describe.each([
       const existing = await config.createRow()
 
       const rowUsage = await getRowUsage()
-      const queryUsage = await getQueryUsage()
 
       const row = await config.api.row.patch(table._id!, {
         _id: existing._id!,
@@ -552,7 +534,6 @@ describe.each([
     it("should throw an error when given improper types", async () => {
       const existing = await config.createRow()
       const rowUsage = await getRowUsage()
-      const queryUsage = await getQueryUsage()
 
       await config.api.row.patch(
         table._id!,
@@ -650,7 +631,6 @@ describe.each([
     it("should be able to delete a row", async () => {
       const createdRow = await config.createRow()
       const rowUsage = await getRowUsage()
-      const queryUsage = await getQueryUsage()
 
       const res = await config.api.row.delete(table._id!, [createdRow])
       expect(res.body[0]._id).toEqual(createdRow._id)
@@ -666,7 +646,6 @@ describe.each([
 
     it("should return no errors on valid row", async () => {
       const rowUsage = await getRowUsage()
-      const queryUsage = await getQueryUsage()
 
       const res = await config.api.row.validate(table._id!, { name: "ivan" })
 
@@ -677,7 +656,6 @@ describe.each([
 
     it("should errors on invalid row", async () => {
       const rowUsage = await getRowUsage()
-      const queryUsage = await getQueryUsage()
 
       const res = await config.api.row.validate(table._id!, { name: 1 })
 
@@ -703,7 +681,6 @@ describe.each([
       const row1 = await config.createRow()
       const row2 = await config.createRow()
       const rowUsage = await getRowUsage()
-      const queryUsage = await getQueryUsage()
 
       const res = await config.api.row.delete(table._id!, [row1, row2])
 
@@ -719,7 +696,6 @@ describe.each([
         config.createRow(),
       ])
       const rowUsage = await getRowUsage()
-      const queryUsage = await getQueryUsage()
 
       const res = await config.api.row.delete(table._id!, [
         row1,
@@ -735,7 +711,6 @@ describe.each([
     it("should accept a valid row object and delete the row", async () => {
       const row1 = await config.createRow()
       const rowUsage = await getRowUsage()
-      const queryUsage = await getQueryUsage()
 
       const res = await config.api.row.delete(table._id!, row1)
 
@@ -746,7 +721,6 @@ describe.each([
 
     it("Should ignore malformed/invalid delete requests", async () => {
       const rowUsage = await getRowUsage()
-      const queryUsage = await getQueryUsage()
 
       const res = await config.api.row.delete(
         table._id!,
@@ -782,7 +756,6 @@ describe.each([
       it("should be able to fetch tables contents via 'view'", async () => {
         const row = await config.createRow()
         const rowUsage = await getRowUsage()
-        const queryUsage = await getQueryUsage()
 
         const res = await config.api.legacyView.get(table._id!)
         expect(res.body.length).toEqual(1)
@@ -792,7 +765,6 @@ describe.each([
 
       it("should throw an error if view doesn't exist", async () => {
         const rowUsage = await getRowUsage()
-        const queryUsage = await getQueryUsage()
 
         await config.api.legacyView.get("derp", { expectStatus: 404 })
 
@@ -808,7 +780,6 @@ describe.each([
         })
         const row = await config.createRow()
         const rowUsage = await getRowUsage()
-        const queryUsage = await getQueryUsage()
 
         const res = await config.api.legacyView.get(view.name)
         expect(res.body.length).toEqual(1)
@@ -864,7 +835,6 @@ describe.each([
         }
       )
       const rowUsage = await getRowUsage()
-      const queryUsage = await getQueryUsage()
 
       // test basic enrichment
       const resBasic = await config.api.row.get(
@@ -1100,7 +1070,6 @@ describe.each([
 
         const createdRow = await config.createRow()
         const rowUsage = await getRowUsage()
-        const queryUsage = await getQueryUsage()
 
         await config.api.row.delete(view.id, [createdRow])
 
@@ -1127,7 +1096,6 @@ describe.each([
           config.createRow(),
         ])
         const rowUsage = await getRowUsage()
-        const queryUsage = await getQueryUsage()
 
         await config.api.row.delete(view.id, [rows[0], rows[2]])
 

packages/server/src/integration-test/postgres.spec.ts

@@ -41,12 +41,12 @@ describe("postgres integrations", () => {
     makeRequest = generateMakeRequest(apiKey, true)
 
     postgresDatasource = await config.api.datasource.create(
-      await databaseTestProviders.postgres.getDsConfig()
+      await databaseTestProviders.postgres.datasource()
     )
   })
 
   afterAll(async () => {
-    await databaseTestProviders.postgres.stopContainer()
+    await databaseTestProviders.postgres.stop()
   })
 
   beforeEach(async () => {
@@ -1041,14 +1041,14 @@ describe("postgres integrations", () => {
   describe("POST /api/datasources/verify", () => {
     it("should be able to verify the connection", async () => {
       const response = await config.api.datasource.verify({
-        datasource: await databaseTestProviders.postgres.getDsConfig(),
+        datasource: await databaseTestProviders.postgres.datasource(),
       })
       expect(response.status).toBe(200)
       expect(response.body.connected).toBe(true)
     })
 
     it("should state an invalid datasource cannot connect", async () => {
-      const dbConfig = await databaseTestProviders.postgres.getDsConfig()
+      const dbConfig = await databaseTestProviders.postgres.datasource()
       const response = await config.api.datasource.verify({
         datasource: {
           ...dbConfig,
@@ -1082,7 +1082,7 @@ describe("postgres integrations", () => {
 
     beforeEach(async () => {
       client = new Client(
-        (await databaseTestProviders.postgres.getDsConfig()).config!
+        (await databaseTestProviders.postgres.datasource()).config!
       )
       await client.connect()
     })
@@ -1125,7 +1125,7 @@ describe("postgres integrations", () => {
       schema2 = "test-2"
 
     beforeAll(async () => {
-      const dsConfig = await databaseTestProviders.postgres.getDsConfig()
+      const dsConfig = await databaseTestProviders.postgres.datasource()
       const dbConfig = dsConfig.config!
 
       client = new Client(dbConfig)

packages/server/src/integrations/postgres.ts

@@ -29,6 +29,7 @@ import { Client, ClientConfig, types } from "pg"
 import { getReadableErrorMessage } from "./base/errorMapping"
 import { exec } from "child_process"
 import { storeTempFile } from "../utilities/fileSystem"
+import { env } from "@budibase/backend-core"
 
 // Return "date" and "timestamp" types as plain strings.
 // This lets us reference the original stored timezone.
@@ -202,8 +203,13 @@ class PostgresIntegration extends Sql implements DatasourcePlus {
       await this.openConnection()
       response.connected = true
     } catch (e: any) {
-      console.log(e)
+      if (typeof e.message === "string" && e.message !== "") {
-      response.error = e.message as string
+        response.error = e.message as string
+      } else if (typeof e.code === "string" && e.code !== "") {
+        response.error = e.code
+      } else {
+        response.error = "Unknown error"
+      }
     } finally {
       await this.closeConnection()
     }
@@ -428,6 +434,14 @@ class PostgresIntegration extends Sql implements DatasourcePlus {
   }
 
   async getExternalSchema() {
+    if (!env.SELF_HOSTED) {
+      // This is because it relies on shelling out to pg_dump and we don't want
+      // to enable shell injection attacks.
+      throw new Error(
+        "schema export for Postgres is not supported in Budibase Cloud"
+      )
+    }
+
     const dumpCommandParts = [
       `user=${this.config.user}`,
       `host=${this.config.host}`,

packages/server/src/integrations/tests/utils/index.ts

@@ -1,14 +1,16 @@
 jest.unmock("pg")
 
 import { Datasource } from "@budibase/types"
-import * as pg from "./postgres"
+import * as postgres from "./postgres"
+import * as mongodb from "./mongodb"
+import { StartedTestContainer } from "testcontainers"
 
 jest.setTimeout(30000)
 
-export interface DatabasePlusTestProvider {
+export interface DatabaseProvider {
-  getDsConfig(): Promise<Datasource>
+  start(): Promise<StartedTestContainer>
+  stop(): Promise<void>
+  datasource(): Promise<Datasource>
 }
 
-export const databaseTestProviders = {
+export const databaseTestProviders = { postgres, mongodb }
-  postgres: pg,
-}

packages/server/src/integrations/tests/utils/mongodb.ts

@@ -0,0 +1,41 @@
+import { Datasource, SourceName } from "@budibase/types"
+import { GenericContainer, Wait, StartedTestContainer } from "testcontainers"
+
+let container: StartedTestContainer | undefined
+
+export async function start(): Promise<StartedTestContainer> {
+  return await new GenericContainer("mongo:7.0-jammy")
+    .withExposedPorts(27017)
+    .withEnvironment({
+      MONGO_INITDB_ROOT_USERNAME: "mongo",
+      MONGO_INITDB_ROOT_PASSWORD: "password",
+    })
+    .withWaitStrategy(
+      Wait.forSuccessfulCommand(`mongosh --eval "db.version()"`)
+    )
+    .start()
+}
+
+export async function datasource(): Promise<Datasource> {
+  if (!container) {
+    container = await start()
+  }
+  const host = container.getHost()
+  const port = container.getMappedPort(27017)
+  return {
+    type: "datasource",
+    source: SourceName.MONGODB,
+    plus: false,
+    config: {
+      connectionString: `mongodb://mongo:password@${host}:${port}`,
+      db: "mongo",
+    },
+  }
+}
+
+export async function stop() {
+  if (container) {
+    await container.stop()
+    container = undefined
+  }
+}

packages/server/src/integrations/tests/utils/postgres.ts

@@ -3,45 +3,44 @@ import { GenericContainer, Wait, StartedTestContainer } from "testcontainers"
 
 let container: StartedTestContainer | undefined
 
-export async function getDsConfig(): Promise<Datasource> {
+export async function start(): Promise<StartedTestContainer> {
-  try {
+  return await new GenericContainer("postgres:16.1-bullseye")
-    if (!container) {
+    .withExposedPorts(5432)
-      container = await new GenericContainer("postgres:16.1-bullseye")
+    .withEnvironment({ POSTGRES_PASSWORD: "password" })
-        .withExposedPorts(5432)
+    .withWaitStrategy(
-        .withEnvironment({ POSTGRES_PASSWORD: "password" })
+      Wait.forSuccessfulCommand(
-        .withWaitStrategy(
+        "pg_isready -h localhost -p 5432"
-          Wait.forLogMessage(
+      ).withStartupTimeout(10000)
-            "database system is ready to accept connections",
+    )
-            2
+    .start()
-          )
+}
-        )
-        .start()
-    }
-    const host = container.getHost()
-    const port = container.getMappedPort(5432)
 
-    return {
+export async function datasource(): Promise<Datasource> {
-      type: "datasource_plus",
+  if (!container) {
-      source: SourceName.POSTGRES,
+    container = await start()
-      plus: true,
+  }
-      config: {
+  const host = container.getHost()
-        host,
+  const port = container.getMappedPort(5432)
-        port,
+
-        database: "postgres",
+  return {
-        user: "postgres",
+    type: "datasource_plus",
-        password: "password",
+    source: SourceName.POSTGRES,
-        schema: "public",
+    plus: true,
-        ssl: false,
+    config: {
-        rejectUnauthorized: false,
+      host,
-        ca: false,
+      port,
-      },
+      database: "postgres",
-    }
+      user: "postgres",
-  } catch (err) {
+      password: "password",
-    throw new Error("**UNABLE TO CREATE TO POSTGRES CONTAINER**")
+      schema: "public",
+      ssl: false,
+      rejectUnauthorized: false,
+      ca: false,
+    },
   }
 }
 
-export async function stopContainer() {
+export async function stop() {
   if (container) {
     await container.stop()
     container = undefined

packages/server/src/tests/utilities/api/index.ts

@@ -10,6 +10,7 @@ import { ApplicationAPI } from "./application"
 import { BackupAPI } from "./backup"
 import { AttachmentAPI } from "./attachment"
 import { UserAPI } from "./user"
+import { QueryAPI } from "./query"
 
 export default class API {
   table: TableAPI
@@ -23,6 +24,7 @@ export default class API {
   backup: BackupAPI
   attachment: AttachmentAPI
   user: UserAPI
+  query: QueryAPI
 
   constructor(config: TestConfiguration) {
     this.table = new TableAPI(config)
@@ -36,5 +38,6 @@ export default class API {
     this.backup = new BackupAPI(config)
     this.attachment = new AttachmentAPI(config)
     this.user = new UserAPI(config)
+    this.query = new QueryAPI(config)
   }
 }

packages/server/src/tests/utilities/api/query.ts

@@ -0,0 +1,44 @@
+import TestConfiguration from "../TestConfiguration"
+import {
+  Query,
+  type ExecuteQueryRequest,
+  type ExecuteQueryResponse,
+} from "@budibase/types"
+import { TestAPI } from "./base"
+
+export class QueryAPI extends TestAPI {
+  constructor(config: TestConfiguration) {
+    super(config)
+  }
+
+  create = async (body: Query): Promise<Query> => {
+    const res = await this.request
+      .post(`/api/queries`)
+      .set(this.config.defaultHeaders())
+      .send(body)
+      .expect("Content-Type", /json/)
+
+    if (res.status !== 200) {
+      throw new Error(JSON.stringify(res.body))
+    }
+
+    return res.body as Query
+  }
+
+  execute = async (
+    queryId: string,
+    body?: ExecuteQueryRequest
+  ): Promise<ExecuteQueryResponse> => {
+    const res = await this.request
+      .post(`/api/v2/queries/${queryId}`)
+      .set(this.config.defaultHeaders())
+      .send(body)
+      .expect("Content-Type", /json/)
+
+    if (res.status !== 200) {
+      throw new Error(JSON.stringify(res.body))
+    }
+
+    return res.body
+  }
+}

packages/server/src/utilities/workerRequests.ts

@@ -1,4 +1,4 @@
-import fetch from "node-fetch"
+import { Response, default as fetch } from "node-fetch"
 import env from "../environment"
 import { checkSlashesInUrl } from "./index"
 import {
@@ -40,25 +40,21 @@ export function request(ctx?: Ctx, request?: any) {
 }
 
 async function checkResponse(
-  response: any,
+  response: Response,
   errorMsg: string,
   { ctx }: { ctx?: Ctx } = {}
 ) {
-  if (response.status !== 200) {
+  if (response.status >= 300) {
-    let error
+    let responseErrorMessage
-    try {
+    if (response.headers.get("content-type")?.includes("json")) {
-      error = await response.json()
+      const error = await response.json()
-      if (!error.message) {
+      responseErrorMessage = error.message ?? JSON.stringify(error)
-        error = JSON.stringify(error)
+    } else {
-      }
+      responseErrorMessage = await response.text()
-    } catch (err) {
-      error = await response.text()
     }
-    const msg = `Unable to ${errorMsg} - ${
+    const msg = `Unable to ${errorMsg} - ${responseErrorMessage}`
-      error.message ? error.message : error
-    }`
     if (ctx) {
-      ctx.throw(400, msg)
+      ctx.throw(msg, response.status)
     } else {
       throw msg
     }

packages/types/src/documents/app/query.ts

@@ -1,4 +1,5 @@
 import { Document } from "../document"
+import type { Row } from "./row"
 
 export interface QuerySchema {
   name?: string
@@ -54,3 +55,12 @@ export interface PreviewQueryRequest extends Omit<Query, "parameters"> {
     urlName?: boolean
   }
 }
+
+export interface ExecuteQueryRequest {
+  parameters?: { [key: string]: string }
+  pagination?: any
+}
+
+export interface ExecuteQueryResponse {
+  data: Row[]
+}

packages/worker/package.json

@@ -63,7 +63,7 @@
     "koa-useragent": "^4.1.0",
     "lodash": "4.17.21",
     "node-fetch": "2.6.7",
-    "nodemailer": "6.7.2",
+    "nodemailer": "6.9.9",
     "passport-google-oauth": "2.0.0",
     "passport-local": "1.0.0",
     "pouchdb": "7.3.0",