Merge remote-tracking branch 'origin/master' into feature/form-screen-template

.eslintrc.json

@@ -45,6 +45,16 @@
         "no-prototype-builtins": "off",
         "local-rules/no-budibase-imports": "error"
       }
+    },
+    {
+      "files": [
+        "packages/builder/**/*",
+        "packages/client/**/*",
+        "packages/frontend-core/**/*"
+      ],
+      "rules": {
+        "no-console": ["error", { "allow": ["warn", "error", "debug"] } ]
+      }
     }
   ],
   "rules": {

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "2.16.0",
+  "version": "2.17.8",
   "npmClient": "yarn",
   "packages": [
     "packages/*",

packages/account-portal

@@ -1 +1 @@
-Subproject commit 485ec16a9eed48c548a5f1239772139f3319f028
+Subproject commit cc12291732ee902dc832bc7d93cf2086ffdf0cff

packages/backend-core/package.json

@@ -21,7 +21,7 @@
     "test:watch": "jest --watchAll"
   },
   "dependencies": {
-    "@budibase/nano": "10.1.4",
+    "@budibase/nano": "10.1.5",
     "@budibase/pouchdb-replication-stream": "1.2.10",
     "@budibase/shared-core": "0.0.0",
     "@budibase/types": "0.0.0",

packages/backend-core/src/objectStore/cloudfront.ts

@@ -23,7 +23,7 @@ const getCloudfrontSignParams = () => {
   return {
     keypairId: env.CLOUDFRONT_PUBLIC_KEY_ID!,
     privateKeyString: getPrivateKey(),
-    expireTime: new Date().getTime() + 1000 * 60 * 60, // 1 hour
+    expireTime: new Date().getTime() + 1000 * 60 * 60 * 24, // 1 day
   }
 }
 

packages/backend-core/src/objectStore/objectStore.ts

@@ -7,7 +7,7 @@ import tar from "tar-fs"
 import zlib from "zlib"
 import { promisify } from "util"
 import { join } from "path"
-import fs from "fs"
+import fs, { ReadStream } from "fs"
 import env from "../environment"
 import { budibaseTempDir } from "./utils"
 import { v4 } from "uuid"
@@ -184,7 +184,7 @@ export async function upload({
 export async function streamUpload(
   bucketName: string,
   filename: string,
-  stream: any,
+  stream: ReadStream | ReadableStream,
   extra = {}
 ) {
   const objectStore = ObjectStore(bucketName)
@@ -255,7 +255,8 @@ export async function listAllObjects(bucketName: string, path: string) {
       objects = objects.concat(response.Contents)
     }
     isTruncated = !!response.IsTruncated
-  } while (isTruncated)
+    token = response.NextContinuationToken
+  } while (isTruncated && token)
   return objects
 }
 

packages/backend-core/src/queue/queue.ts

@@ -2,7 +2,7 @@ import env from "../environment"
 import { getRedisOptions } from "../redis/utils"
 import { JobQueue } from "./constants"
 import InMemoryQueue from "./inMemoryQueue"
-import BullQueue, { QueueOptions } from "bull"
+import BullQueue, { QueueOptions, JobOptions } from "bull"
 import { addListeners, StalledFn } from "./listeners"
 import { Duration } from "../utils"
 import * as timers from "../timers"
@@ -24,17 +24,24 @@ async function cleanup() {
 
 export function createQueue<T>(
   jobQueue: JobQueue,
-  opts: { removeStalledCb?: StalledFn } = {}
+  opts: {
+    removeStalledCb?: StalledFn
+    maxStalledCount?: number
+    jobOptions?: JobOptions
+  } = {}
 ): BullQueue.Queue<T> {
   const redisOpts = getRedisOptions()
   const queueConfig: QueueOptions = {
     redis: redisOpts,
     settings: {
-      maxStalledCount: 0,
+      maxStalledCount: opts.maxStalledCount ? opts.maxStalledCount : 0,
       lockDuration: QUEUE_LOCK_MS,
       lockRenewTime: QUEUE_LOCK_RENEW_INTERNAL_MS,
     },
   }
+  if (opts.jobOptions) {
+    queueConfig.defaultJobOptions = opts.jobOptions
+  }
   let queue: any
   if (!env.isTest()) {
     queue = new BullQueue(jobQueue, queueConfig)

packages/builder/src/builderStore/componentUtils.js

@@ -7,6 +7,9 @@ import {
   findHBSBlocks,
 } from "@budibase/string-templates"
 import { capitalise } from "helpers"
+import { Constants } from "@budibase/frontend-core"
+
+const { ContextScopes } = Constants
 
 /**
  * Recursively searches for a specific component ID
@@ -263,11 +266,59 @@ export const getComponentName = component => {
   if (component == null) {
     return ""
   }
-
   const components = get(store)?.components || {}
   const componentDefinition = components[component._component] || {}
-  const name =
+  return componentDefinition.friendlyName || componentDefinition.name || ""
-    componentDefinition.friendlyName || componentDefinition.name || ""
+}
-
+
-  return name
+/**
+ * Recurses through the component tree and builds a tree of contexts provided
+ * by components.
+ */
+export const buildContextTree = (
+  rootComponent,
+  tree = { root: [] },
+  currentBranch = "root"
+) => {
+  // Sanity check
+  if (!rootComponent) {
+    return tree
+  }
+
+  // Process this component's contexts
+  const def = store.actions.components.getDefinition(rootComponent._component)
+  if (def?.context) {
+    tree[currentBranch].push(rootComponent._id)
+    const contexts = Array.isArray(def.context) ? def.context : [def.context]
+
+    // If we provide local context, start a new branch for our children
+    if (contexts.some(context => context.scope === ContextScopes.Local)) {
+      currentBranch = rootComponent._id
+      tree[rootComponent._id] = []
+    }
+  }
+
+  // Process children
+  if (rootComponent._children) {
+    rootComponent._children.forEach(child => {
+      buildContextTree(child, tree, currentBranch)
+    })
+  }
+
+  return tree
+}
+
+/**
+ * Generates a lookup map of which context branch all components in a component
+ * tree are inside.
+ */
+export const buildContextTreeLookupMap = rootComponent => {
+  const tree = buildContextTree(rootComponent)
+  let map = {}
+  Object.entries(tree).forEach(([branch, ids]) => {
+    ids.forEach(id => {
+      map[id] = branch
+    })
+  })
+  return map
 }

packages/builder/src/builderStore/dataBinding.js

@@ -1,6 +1,7 @@
 import { cloneDeep } from "lodash/fp"
 import { get } from "svelte/store"
 import {
+  buildContextTreeLookupMap,
   findAllComponents,
   findAllMatchingComponents,
   findComponent,
@@ -20,11 +21,13 @@ import {
   encodeJSBinding,
 } from "@budibase/string-templates"
 import { TableNames } from "../constants"
-import { JSONUtils } from "@budibase/frontend-core"
+import { JSONUtils, Constants } from "@budibase/frontend-core"
 import ActionDefinitions from "components/design/settings/controls/ButtonActionEditor/manifest.json"
 import { environment, licensing } from "stores/portal"
 import { convertOldFieldFormat } from "components/design/settings/controls/FieldConfiguration/utils"
 
+const { ContextScopes } = Constants
+
 // Regex to match all instances of template strings
 const CAPTURE_VAR_INSIDE_TEMPLATE = /{{([^}]+)}}/g
 const CAPTURE_VAR_INSIDE_JS = /\$\("([^")]+)"\)/g
@@ -214,20 +217,27 @@ export const getComponentContexts = (
     return []
   }
   let map = {}
+  const componentPath = findComponentPath(asset.props, componentId)
+  const componentPathIds = componentPath.map(component => component._id)
+  const contextTreeLookupMap = buildContextTreeLookupMap(asset.props)
 
   // Processes all contexts exposed by a component
   const processContexts = scope => component => {
-    const def = store.actions.components.getDefinition(component._component)
+    // Sanity check
+    const def = store.actions.components.getDefinition(component?._component)
     if (!def?.context) {
       return
     }
-    if (!map[component._id]) {
+
-      map[component._id] = {
+    // Filter out global contexts not in the same branch.
-        component,
+    // Global contexts are only valid if their branch root is an ancestor of
-        definition: def,
+    // this component.
-        contexts: [],
+    const branch = contextTreeLookupMap[component._id]
-      }
+    if (branch !== "root" && !componentPathIds.includes(branch)) {
+      return
     }
+
+    // Process all contexts provided by this component
     const contexts = Array.isArray(def.context) ? def.context : [def.context]
     contexts.forEach(context => {
       // Ensure type matches
@@ -235,7 +245,7 @@ export const getComponentContexts = (
         return
       }
       // Ensure scope matches
-      let contextScope = context.scope || "global"
+      let contextScope = context.scope || ContextScopes.Global
       if (contextScope !== scope) {
         return
       }
@@ -243,17 +253,23 @@ export const getComponentContexts = (
       if (!isContextCompatibleWithComponent(context, component)) {
         return
       }
+      if (!map[component._id]) {
+        map[component._id] = {
+          component,
+          definition: def,
+          contexts: [],
+        }
+      }
       map[component._id].contexts.push(context)
     })
   }
 
   // Process all global contexts
   const allComponents = findAllComponents(asset.props)
-  allComponents.forEach(processContexts("global"))
+  allComponents.forEach(processContexts(ContextScopes.Global))
 
-  // Process all local contexts
+  // Process all local contexts in the immediate tree
-  const localComponents = findComponentPath(asset.props, componentId)
+  componentPath.forEach(processContexts(ContextScopes.Local))
-  localComponents.forEach(processContexts("local"))
 
   // Exclude self if required
   if (!options?.includeSelf) {
@@ -364,7 +380,6 @@ const getContextBindings = (asset, componentId) => {
  * Generates a set of bindings for a given component context
  */
 const generateComponentContextBindings = (asset, componentContext) => {
-  console.log("Hello ")
   const { component, definition, contexts } = componentContext
   if (!component || !definition || !contexts?.length) {
     return []

packages/builder/src/builderStore/store/frontend.js

@@ -158,6 +158,7 @@ export const getFrontendStore = () => {
           ...INITIAL_FRONTEND_STATE.features,
           ...application.features,
         },
+        automations: application.automations || {},
         icon: application.icon || {},
         initialised: true,
       }))

packages/builder/src/builderStore/websocket.js

@@ -21,7 +21,7 @@ export const createBuilderWebsocket = appId => {
     })
   })
   socket.on("connect_error", err => {
-    console.log("Failed to connect to builder websocket:", err.message)
+    console.error("Failed to connect to builder websocket:", err.message)
   })
   socket.on("disconnect", () => {
     userStore.actions.reset()

packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte

@@ -15,7 +15,6 @@
     Icon,
     Checkbox,
     DatePicker,
-    Detail,
   } from "@budibase/bbui"
   import CreateWebhookModal from "components/automation/Shared/CreateWebhookModal.svelte"
   import { automationStore, selectedAutomation } from "builderStore"
@@ -33,6 +32,8 @@
   import Editor from "components/integration/QueryEditor.svelte"
   import ModalBindableInput from "components/common/bindings/ModalBindableInput.svelte"
   import CodeEditor from "components/common/CodeEditor/CodeEditor.svelte"
+  import BindingPicker from "components/common/bindings/BindingPicker.svelte"
+  import { BindingHelpers } from "components/common/bindings/utils"
   import {
     bindingsToCompletions,
     hbAutocomplete,
@@ -56,7 +57,7 @@
   let drawer
   let fillWidth = true
   let inputData
-  let codeBindingOpen = false
+  let insertAtPos, getCaretPosition
   $: filters = lookForFilters(schemaProperties) || []
   $: tempFilters = filters
   $: stepId = block.stepId
@@ -75,6 +76,10 @@
   $: isUpdateRow = stepId === ActionStepID.UPDATE_ROW
   $: codeMode =
     stepId === "EXECUTE_BASH" ? EditorModes.Handlebars : EditorModes.JS
+  $: bindingsHelpers = new BindingHelpers(getCaretPosition, insertAtPos, {
+    disableWrapping: true,
+  })
+  $: editingJs = codeMode === EditorModes.JS
 
   $: stepCompletions =
     codeMode === EditorModes.Handlebars
@@ -157,6 +162,7 @@
     let bindings = []
     let loopBlockCount = 0
     const addBinding = (name, value, icon, idx, isLoopBlock, bindingName) => {
+      if (!name) return
       const runtimeBinding = determineRuntimeBinding(name, idx, isLoopBlock)
       const categoryName = determineCategoryName(idx, isLoopBlock, bindingName)
 
@@ -291,7 +297,6 @@
         loopBlockCount++
         continue
       }
-
       Object.entries(schema).forEach(([name, value]) =>
         addBinding(name, value, icon, idx, isLoopBlock, bindingName)
       )
@@ -539,39 +544,51 @@
             />
           {:else if value.customType === "code"}
             <CodeEditorModal>
-              {#if codeMode == EditorModes.JS}
+              <div class:js-editor={editingJs}>
-                <ActionButton
+                <div class:js-code={editingJs} style="width: 100%">
-                  on:click={() => (codeBindingOpen = !codeBindingOpen)}
+                  <CodeEditor
-                  quiet
+                    value={inputData[key]}
-                  icon={codeBindingOpen ? "ChevronDown" : "ChevronRight"}
+                    on:change={e => {
-                >
+                      // need to pass without the value inside
-                  <Detail size="S">Bindings</Detail>
+                      onChange({ detail: e.detail }, key)
-                </ActionButton>
+                      inputData[key] = e.detail
-                {#if codeBindingOpen}
+                    }}
-                  <pre>{JSON.stringify(bindings, null, 2)}</pre>
+                    completions={stepCompletions}
-                {/if}
+                    mode={codeMode}
-              {/if}
+                    autocompleteEnabled={codeMode !== EditorModes.JS}
-              <CodeEditor
+                    bind:getCaretPosition
-                value={inputData[key]}
+                    bind:insertAtPos
-                on:change={e => {
+                    height={500}
-                  // need to pass without the value inside
+                  />
-                  onChange({ detail: e.detail }, key)
+                  <div class="messaging">
-                  inputData[key] = e.detail
+                    {#if codeMode === EditorModes.Handlebars}
-                }}
+                      <Icon name="FlashOn" />
-                completions={stepCompletions}
+                      <div class="messaging-wrap">
-                mode={codeMode}
+                        <div>
-                autocompleteEnabled={codeMode != EditorModes.JS}
+                          Add available bindings by typing <strong>
-                height={500}
+                            &#125;&#125;
-              />
+                          </strong>
-              <div class="messaging">
+                        </div>
-                {#if codeMode == EditorModes.Handlebars}
+                      </div>
-                  <Icon name="FlashOn" />
+                    {/if}
-                  <div class="messaging-wrap">
+                  </div>
-                    <div>
+                </div>
-                      Add available bindings by typing <strong>
+                {#if editingJs}
-                        &#125;&#125;
+                  <div class="js-binding-picker">
-                      </strong>
+                    <BindingPicker
-                    </div>
+                      {bindings}
+                      allowHelpers={false}
+                      addBinding={binding =>
+                        bindingsHelpers.onSelectBinding(
+                          inputData[key],
+                          binding,
+                          {
+                            js: true,
+                            dontDecode: true,
+                          }
+                        )}
+                      mode="javascript"
+                    />
                   </div>
                 {/if}
               </div>
@@ -658,4 +675,20 @@
   .test :global(.drawer) {
     width: 10000px !important;
   }
+
+  .js-editor {
+    display: flex;
+    flex-direction: row;
+    flex-grow: 1;
+    width: 100%;
+  }
+
+  .js-code {
+    flex: 7;
+  }
+
+  .js-binding-picker {
+    flex: 3;
+    margin-top: calc((var(--spacing-xl) * -1) + 1px);
+  }
 </style>

packages/builder/src/components/common/CodeEditor/CodeEditor.svelte

@@ -54,6 +54,7 @@
   export let placeholder = null
   export let autocompleteEnabled = true
   export let autofocus = false
+  export let jsBindingWrapping = true
 
   // Export a function to expose caret position
   export const getCaretPosition = () => {
@@ -187,7 +188,7 @@
       )
       complete.push(
         EditorView.inputHandler.of((view, from, to, insert) => {
-          if (insert === "$") {
+          if (jsBindingWrapping && insert === "$") {
             let { text } = view.state.doc.lineAt(from)
 
             const left = from ? text.substring(0, from) : ""

packages/builder/src/components/common/CodeEditor/index.js

@@ -286,13 +286,20 @@ export const hbInsert = (value, from, to, text) => {
   return parsedInsert
 }
 
-export function jsInsert(value, from, to, text, { helper } = {}) {
+export function jsInsert(
+  value,
+  from,
+  to,
+  text,
+  { helper, disableWrapping } = {}
+) {
   let parsedInsert = ""
 
   const left = from ? value.substring(0, from) : ""
   const right = to ? value.substring(to) : ""
-
+  if (disableWrapping) {
-  if (helper) {
+    parsedInsert = text
+  } else if (helper) {
     parsedInsert = `helpers.${text}()`
   } else if (!left.includes('$("') || !right.includes('")')) {
     parsedInsert = `$("${text}")`
@@ -312,7 +319,7 @@ export const insertBinding = (view, from, to, text, mode) => {
   } else if (mode.name == "handlebars") {
     parsedInsert = hbInsert(view.state.doc?.toString(), from, to, text)
   } else {
-    console.log("Unsupported")
+    console.warn("Unsupported")
     return
   }
 

packages/builder/src/components/common/bindings/BindingPanel.svelte

@@ -29,10 +29,9 @@
     hbAutocomplete,
     EditorModes,
     bindingsToCompletions,
-    hbInsert,
-    jsInsert,
   } from "../CodeEditor"
   import BindingPicker from "./BindingPicker.svelte"
+  import { BindingHelpers } from "./utils"
 
   const dispatch = createEventDispatcher()
 
@@ -60,8 +59,10 @@
   let targetMode = null
 
   $: usingJS = mode === "JavaScript"
-  $: editorMode = mode == "JavaScript" ? EditorModes.JS : EditorModes.Handlebars
+  $: editorMode =
+    mode === "JavaScript" ? EditorModes.JS : EditorModes.Handlebars
   $: bindingCompletions = bindingsToCompletions(bindings, editorMode)
+  $: bindingHelpers = new BindingHelpers(getCaretPosition, insertAtPos)
 
   const updateValue = val => {
     valid = isValid(readableToRuntimeBinding(bindings, val))
@@ -70,31 +71,13 @@
     }
   }
 
-  // Adds a JS/HBS helper to the expression
   const onSelectHelper = (helper, js) => {
-    const pos = getCaretPosition()
+    bindingHelpers.onSelectHelper(js ? jsValue : hbsValue, helper, { js })
-    const { start, end } = pos
-    if (js) {
-      let js = decodeJSBinding(jsValue)
-      const insertVal = jsInsert(js, start, end, helper.text, { helper: true })
-      insertAtPos({ start, end, value: insertVal })
-    } else {
-      const insertVal = hbInsert(hbsValue, start, end, helper.text)
-      insertAtPos({ start, end, value: insertVal })
-    }
   }
 
-  // Adds a data binding to the expression
   const onSelectBinding = (binding, { forceJS } = {}) => {
-    const { start, end } = getCaretPosition()
+    const js = usingJS || forceJS
-    if (usingJS || forceJS) {
+    bindingHelpers.onSelectBinding(js ? jsValue : hbsValue, binding, { js })
-      let js = decodeJSBinding(jsValue)
-      const insertVal = jsInsert(js, start, end, binding.readableBinding)
-      insertAtPos({ start, end, value: insertVal })
-    } else {
-      const insertVal = hbInsert(hbsValue, start, end, binding.readableBinding)
-      insertAtPos({ start, end, value: insertVal })
-    }
   }
 
   const onChangeMode = e => {

packages/builder/src/components/common/bindings/BindingPicker.svelte

@@ -9,6 +9,7 @@
   export let bindings
   export let mode
   export let allowHelpers
+  export let noPaddingTop = false
 
   let search = ""
   let popover
@@ -47,9 +48,10 @@
     })
   $: filteredHelpers = helpers?.filter(helper => {
     return (
-      !search ||
+      (!search ||
-      helper.label.match(searchRgx) ||
+        helper.label.match(searchRgx) ||
-      helper.description.match(searchRgx)
+        helper.description.match(searchRgx)) &&
+      (mode.name !== "javascript" || helper.allowsJs)
     )
   })
 

packages/builder/src/components/common/bindings/utils.js

@@ -1,38 +1,41 @@
-export function addHBSBinding(value, caretPos, binding) {
+import { decodeJSBinding } from "@budibase/string-templates"
-  binding = typeof binding === "string" ? binding : binding.path
+import { hbInsert, jsInsert } from "components/common/CodeEditor"
-  value = value == null ? "" : value
 
-  const left = caretPos?.start ? value.substring(0, caretPos.start) : ""
+export class BindingHelpers {
-  const right = caretPos?.end ? value.substring(caretPos.end) : ""
+  constructor(getCaretPosition, insertAtPos, { disableWrapping } = {}) {
-  if (!left.includes("{{") || !right.includes("}}")) {
+    this.getCaretPosition = getCaretPosition
-    binding = `{{ ${binding} }}`
+    this.insertAtPos = insertAtPos
+    this.disableWrapping = disableWrapping
   }
-  if (caretPos.start) {
-    value =
-      value.substring(0, caretPos.start) +
-      binding +
-      value.substring(caretPos.end, value.length)
-  } else {
-    value += binding
-  }
-  return value
-}
 
-export function addJSBinding(value, caretPos, binding, { helper } = {}) {
+  // Adds a JS/HBS helper to the expression
-  binding = typeof binding === "string" ? binding : binding.path
+  onSelectHelper(value, helper, { js, dontDecode }) {
-  value = value == null ? "" : value
+    const pos = this.getCaretPosition()
-  if (!helper) {
+    const { start, end } = pos
-    binding = `$("${binding}")`
+    if (js) {
-  } else {
+      const jsVal = dontDecode ? value : decodeJSBinding(value)
-    binding = `helpers.${binding}()`
+      const insertVal = jsInsert(jsVal, start, end, helper.text, {
+        helper: true,
+      })
+      this.insertAtPos({ start, end, value: insertVal })
+    } else {
+      const insertVal = hbInsert(value, start, end, helper.text)
+      this.insertAtPos({ start, end, value: insertVal })
+    }
   }
-  if (caretPos.start) {
+
-    value =
+  // Adds a data binding to the expression
-      value.substring(0, caretPos.start) +
+  onSelectBinding(value, binding, { js, dontDecode }) {
-      binding +
+    const { start, end } = this.getCaretPosition()
-      value.substring(caretPos.end, value.length)
+    if (js) {
-  } else {
+      const jsVal = dontDecode ? value : decodeJSBinding(value)
-    value += binding
+      const insertVal = jsInsert(jsVal, start, end, binding.readableBinding, {
+        disableWrapping: this.disableWrapping,
+      })
+      this.insertAtPos({ start, end, value: insertVal })
+    } else {
+      const insertVal = hbInsert(value, start, end, binding.readableBinding)
+      this.insertAtPos({ start, end, value: insertVal })
+    }
   }
-  return value
 }

packages/builder/src/components/integration/QueryViewer.svelte

@@ -93,7 +93,13 @@
 
       notifications.success("Query executed successfully")
     } catch (error) {
-      notifications.error(`Query Error: ${error.message}`)
+      if (typeof error.message === "string") {
+        notifications.error(`Query Error: ${error.message}`)
+      } else if (typeof error.message?.code === "string") {
+        notifications.error(`Query Error: ${error.message.code}`)
+      } else {
+        notifications.error(`Query Error: ${JSON.stringify(error.message)}`)
+      }
 
       if (!suppressErrors) {
         throw error

packages/builder/src/components/portal/onboarding/TourPopover.svelte

@@ -67,7 +67,7 @@
         }))
         navigateStep(target)
       } else {
-        console.log("Could not retrieve step")
+        console.warn("Could not retrieve step")
       }
     } else {
       if (typeof tourStep.onComplete === "function") {

packages/builder/src/components/portal/onboarding/tourHandler.js

@@ -3,11 +3,11 @@ import { get } from "svelte/store"
 
 const registerNode = async (node, tourStepKey) => {
   if (!node) {
-    console.log("Tour Handler - an anchor node is required")
+    console.warn("Tour Handler - an anchor node is required")
   }
 
   if (!get(store).tourKey) {
-    console.log("Tour Handler - No active tour ", tourStepKey, node)
+    console.error("Tour Handler - No active tour ", tourStepKey, node)
     return
   }
 

packages/builder/src/components/portal/onboarding/tours.js

@@ -45,7 +45,7 @@ const endUserOnboarding = async ({ skipped = false } = {}) => {
         onboarding: false,
       }))
     } catch (e) {
-      console.log("Onboarding failed", e)
+      console.error("Onboarding failed", e)
       return false
     }
     return true

packages/builder/src/constants/completions.js

@@ -1,4 +1,4 @@
-import { getManifest } from "@budibase/string-templates"
+import { getManifest, helpersToRemoveForJs } from "@budibase/string-templates"
 
 export function handlebarsCompletions() {
   const manifest = getManifest()
@@ -11,6 +11,9 @@ export function handlebarsCompletions() {
       label: helperName,
       displayText: helperName,
       description: helperConfig.description,
+      allowsJs:
+        !helperConfig.requiresBlock &&
+        !helpersToRemoveForJs.includes(helperName),
     }))
   )
 }

packages/builder/src/helpers/urlStateSync.js

@@ -52,7 +52,7 @@ export const syncURLToState = options => {
   let cachedPage = get(routify.page)
   let previousParamsHash = null
   let debug = false
-  const log = (...params) => debug && console.log(`[${urlParam}]`, ...params)
+  const log = (...params) => debug && console.debug(`[${urlParam}]`, ...params)
 
   // Navigate to a certain URL
   const gotoUrl = (url, params) => {

packages/builder/src/pages/builder/app/[application]/_components/BuilderSidePanel.svelte

@@ -107,7 +107,7 @@
       return
     }
     if (!prodAppId) {
-      console.log("Application id required")
+      console.error("Application id required")
       return
     }
     await usersFetch.update({

packages/builder/src/pages/builder/app/[application]/data/datasource/[datasourceId]/index.svelte

@@ -12,12 +12,16 @@
   import PromptQueryModal from "./_components/PromptQueryModal.svelte"
   import SettingsPanel from "./_components/panels/Settings.svelte"
   import { helpers } from "@budibase/shared-core"
+  import { admin } from "stores/portal"
+  import { IntegrationTypes } from "constants/backend"
 
   let selectedPanel = null
   let panelOptions = []
 
   $: datasource = $datasources.selected
 
+  $: isCloud = $admin.cloud
+  $: isPostgres = datasource?.source === IntegrationTypes.POSTGRES
   $: getOptions(datasource)
 
   const getOptions = datasource => {
@@ -41,7 +45,13 @@
     }
     // always the last option for SQL
     if (helpers.isSQL(datasource)) {
-      panelOptions.push("Settings")
+      if (isCloud && isPostgres) {
+        // We don't show the settings panel for Postgres on Budicloud because
+        // it requires pg_dump to work and we don't want to enable shell injection
+        // attacks.
+      } else {
+        panelOptions.push("Settings")
+      }
     }
   }
 </script>

packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Navigation/index.svelte

@@ -15,10 +15,15 @@
     Checkbox,
     notifications,
     Select,
+    Combobox,
   } from "@budibase/bbui"
   import { selectedScreen, store } from "builderStore"
   import { DefaultAppTheme } from "constants"
 
+  $: screenRouteOptions = $store.screens
+    .map(screen => screen.routing?.route)
+    .filter(x => x != null)
+
   const updateShowNavigation = async e => {
     await store.actions.screens.updateSetting(
       get(selectedScreen),
@@ -107,23 +112,6 @@
             on:change={e => update("navWidth", e.detail)}
           />
         {/if}
-        <div class="label">
-          <Label size="M">Show logo</Label>
-        </div>
-        <Checkbox
-          value={!$store.navigation.hideLogo}
-          on:change={e => update("hideLogo", !e.detail)}
-        />
-        {#if !$store.navigation.hideLogo}
-          <div class="label">
-            <Label size="M">Logo URL</Label>
-          </div>
-          <Input
-            value={$store.navigation.logoUrl}
-            on:change={e => update("logoUrl", e.detail)}
-            updateOnChange={false}
-          />
-        {/if}
         <div class="label">
           <Label size="M">Show title</Label>
         </div>
@@ -160,6 +148,47 @@
         />
       </div>
     </div>
+
+    <div class="divider" />
+    <div class="customizeSection">
+      <div class="subheading">
+        <Detail>Logo</Detail>
+      </div>
+      <div class="controls">
+        <div class="label">
+          <Label size="M">Show logo</Label>
+        </div>
+        <Checkbox
+          value={!$store.navigation.hideLogo}
+          on:change={e => update("hideLogo", !e.detail)}
+        />
+        {#if !$store.navigation.hideLogo}
+          <div class="label">
+            <Label size="M">Logo image URL</Label>
+          </div>
+          <Input
+            value={$store.navigation.logoUrl}
+            on:change={e => update("logoUrl", e.detail)}
+            updateOnChange={false}
+          />
+          <div class="label">
+            <Label size="M">Logo link URL</Label>
+          </div>
+          <Combobox
+            value={$store.navigation.logoLinkUrl}
+            on:change={e => update("logoLinkUrl", e.detail)}
+            options={screenRouteOptions}
+          />
+          <div class="label">
+            <Label size="M">New tab</Label>
+          </div>
+          <Checkbox
+            value={!!$store.navigation.openLogoLinkInNewTab}
+            on:change={e => update("openLogoLinkInNewTab", !!e.detail)}
+          />
+        {/if}
+      </div>
+    </div>
   {/if}
 </Panel>
 

packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Screen/GeneralPanel.svelte

@@ -66,7 +66,7 @@
     try {
       await store.actions.screens.updateSetting(get(selectedScreen), key, value)
     } catch (error) {
-      console.log(error)
+      console.error(error)
       notifications.error("Error saving screen settings")
     }
   }

packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/CreateScreenModal.svelte

@@ -78,7 +78,7 @@
       //$goto(`./${screenId}`)
       //store.actions.screens.select(screenId)
     } catch (error) {
-      console.log(error)
+      console.error(error)
       notifications.error("Error creating screens")
     }
   }

packages/builder/src/pages/builder/app/[application]/settings/automations/index.svelte

@@ -36,15 +36,12 @@
   let status = null
   let timeRange = null
   let loaded = false
-
+  $: app = $apps.find(app => $store.appId?.includes(app.appId))
-  $: app = $apps.find(app => app.devId === $store.appId?.includes(app.appId))
   $: licensePlan = $auth.user?.license?.plan
   $: page = $pageInfo.page
   $: fetchLogs(automationId, status, page, timeRange)
   $: isCloud = $admin.cloud
-
   $: chainAutomations = app?.automations?.chainAutomations ?? !isCloud
-
   const timeOptions = [
     { value: "90-d", label: "Past 90 days" },
     { value: "30-d", label: "Past 30 days" },

packages/builder/src/pages/builder/app/[application]/settings/backups/_components/ActionsRenderer.svelte

@@ -13,6 +13,7 @@
   import CreateRestoreModal from "./CreateRestoreModal.svelte"
   import { createEventDispatcher } from "svelte"
   import { isOnlyUser } from "builderStore"
+  import { BackupType } from "constants/backend/backups"
 
   export let row
 
@@ -42,12 +43,11 @@
 </script>
 
 <div class="cell">
-  <ActionMenu align="right">
+  {#if row.type !== BackupType.RESTORE}
-    <div slot="control">
+    <ActionMenu align="right">
-      <Icon size="M" hoverable name="MoreSmallList" />
+      <div slot="control">
-    </div>
+        <Icon size="M" hoverable name="MoreSmallList" />
-
+      </div>
-    {#if row.type !== "restore"}
       <AbsTooltip
         position={TooltipPosition.Left}
         text="Unavailable - another user is editing this app"
@@ -62,8 +62,8 @@
       </AbsTooltip>
       <MenuItem on:click={deleteDialog.show} icon="Delete">Delete</MenuItem>
       <MenuItem on:click={downloadExport} icon="Download">Download</MenuItem>
-    {/if}
+    </ActionMenu>
-  </ActionMenu>
+  {/if}
 </div>
 
 <Modal bind:this={restoreBackupModal}>

packages/builder/src/pages/builder/auth/login.svelte

@@ -31,7 +31,7 @@
   async function login() {
     form.validate()
     if (Object.keys(errors).length > 0) {
-      console.log("errors", errors)
+      console.error("errors", errors)
       return
     }
     try {

packages/client/manifest.json

@@ -4720,7 +4720,8 @@
       }
     ],
     "context": {
-      "type": "schema"
+      "type": "schema",
+      "scope": "local"
     }
   },
   "daterangepicker": {
@@ -6742,6 +6743,17 @@
         "key": "disabled",
         "defaultValue": false
       },
+      {
+        "type": "boolean",
+        "label": "Read only",
+        "key": "readonly",
+        "defaultValue": false,
+        "dependsOn": {
+          "setting": "disabled",
+          "value": true,
+          "invert": true
+        }
+      },
       {
         "type": "select",
         "label": "Layout",

packages/client/src/components/app/Layout.svelte

@@ -33,6 +33,8 @@
   export let navTextColor
   export let navWidth
   export let pageWidth
+  export let logoLinkUrl
+  export let openLogoLinkInNewTab
 
   export let embedded = false
 
@@ -150,6 +152,16 @@
     }
     return style
   }
+
+  const getSanitizedUrl = (url, openInNewTab) => {
+    if (!isInternal(url)) {
+      return ensureExternal(url)
+    }
+    if (openInNewTab) {
+      return `#${url}`
+    }
+    return url
+  }
 </script>
 
 <div
@@ -192,7 +204,23 @@
               {/if}
               <div class="logo">
                 {#if !hideLogo}
-                  <img src={logoUrl || "/builder/bblogo.png"} alt={title} />
+                  {#if logoLinkUrl && isInternal(logoLinkUrl) && !openLogoLinkInNewTab}
+                    <a
+                      href={getSanitizedUrl(logoLinkUrl, openLogoLinkInNewTab)}
+                      use:linkable
+                    >
+                      <img src={logoUrl || "/builder/bblogo.png"} alt={title} />
+                    </a>
+                  {:else if logoLinkUrl}
+                    <a
+                      target={openLogoLinkInNewTab ? "_blank" : "_self"}
+                      href={getSanitizedUrl(logoLinkUrl, openLogoLinkInNewTab)}
+                    >
+                      <img src={logoUrl || "/builder/bblogo.png"} alt={title} />
+                    </a>
+                  {:else}
+                    <img src={logoUrl || "/builder/bblogo.png"} alt={title} />
+                  {/if}
                 {/if}
                 {#if !hideTitle && title}
                   <Heading size="S">{title}</Heading>

packages/client/src/components/app/Repeater.svelte

@@ -2,7 +2,9 @@
   import { getContext } from "svelte"
   import Placeholder from "./Placeholder.svelte"
   import Container from "./Container.svelte"
-  import { ContextScopes } from "constants"
+
+  const { Provider, ContextScopes } = getContext("sdk")
+  const component = getContext("component")
 
   export let dataProvider
   export let noRowsMessage
@@ -12,9 +14,6 @@
   export let gap
   export let scope = ContextScopes.Local
 
-  const { Provider } = getContext("sdk")
-  const component = getContext("component")
-
   $: rows = dataProvider?.rows ?? []
   $: loaded = dataProvider?.loaded ?? true
 </script>

packages/client/src/components/app/embedded-map/EmbeddedMap.svelte

@@ -307,7 +307,7 @@
       // Reset view
       resetView()
     } catch (e) {
-      console.log("There was a problem with the map", e)
+      console.error("There was a problem with the map", e)
     }
   }
 

packages/client/src/components/app/forms/CodeScanner.svelte

@@ -61,7 +61,7 @@
           resolve({ initialised: true })
         })
         .catch(err => {
-          console.log("There was a problem scanning the image", err)
+          console.error("There was a problem scanning the image", err)
           resolve({ initialised: false })
         })
     })

packages/client/src/components/app/table/SlotRenderer.svelte

@@ -3,9 +3,9 @@
 
   export let row
 
-  const { Provider } = getContext("sdk")
+  const { Provider, ContextScopes } = getContext("sdk")
 </script>
 
-<Provider data={row}>
+<Provider data={row} scope={ContextScopes.Local}>
   <slot />
 </Provider>

packages/client/src/components/context/Provider.svelte

@@ -1,9 +1,11 @@
 <script>
   import { getContext, setContext, onDestroy } from "svelte"
   import { dataSourceStore, createContextStore } from "stores"
-  import { ActionTypes, ContextScopes } from "constants"
+  import { ActionTypes } from "constants"
   import { generate } from "shortid"
 
+  const { ContextScopes } = getContext("sdk")
+
   export let data
   export let actions
   export let key
@@ -33,7 +35,7 @@
   const provideData = newData => {
     const dataKey = JSON.stringify(newData)
     if (dataKey !== lastDataKey) {
-      context.actions.provideData(providerKey, newData, scope)
+      context.actions.provideData(providerKey, newData)
       lastDataKey = dataKey
     }
   }
@@ -43,7 +45,7 @@
     if (actionsKey !== lastActionsKey) {
       lastActionsKey = actionsKey
       newActions?.forEach(({ type, callback, metadata }) => {
-        context.actions.provideAction(providerKey, type, callback, scope)
+        context.actions.provideAction(providerKey, type, callback)
 
         // Register any "refresh datasource" actions with a singleton store
         // so we can easily refresh data at all levels for any datasource

packages/client/src/constants.js

@@ -12,10 +12,5 @@ export const ActionTypes = {
   ScrollTo: "ScrollTo",
 }
 
-export const ContextScopes = {
-  Local: "local",
-  Global: "global",
-}
-
 export const DNDPlaceholderID = "dnd-placeholder"
 export const ScreenslotType = "screenslot"

packages/client/src/sdk.js

@@ -23,11 +23,12 @@ import { getAction } from "utils/getAction"
 import Provider from "components/context/Provider.svelte"
 import Block from "components/Block.svelte"
 import BlockComponent from "components/BlockComponent.svelte"
-import { ActionTypes, ContextScopes } from "./constants"
+import { ActionTypes } from "./constants"
 import { fetchDatasourceSchema } from "./utils/schema.js"
 import { getAPIKey } from "./utils/api.js"
 import { enrichButtonActions } from "./utils/buttonActions.js"
 import { processStringSync, makePropSafe } from "@budibase/string-templates"
+import { fetchData, LuceneUtils, Constants } from "@budibase/frontend-core"
 
 export default {
   API,
@@ -54,7 +55,9 @@ export default {
   linkable,
   getAction,
   fetchDatasourceSchema,
-  ContextScopes,
+  fetchData,
+  LuceneUtils,
+  ContextScopes: Constants.ContextScopes,
   getAPIKey,
   enrichButtonActions,
   processStringSync,

packages/client/src/stores/context.js

@@ -1,5 +1,4 @@
 import { writable, derived } from "svelte/store"
-import { ContextScopes } from "constants"
 
 export const createContextStore = parentContext => {
   const context = writable({})
@@ -20,60 +19,34 @@ export const createContextStore = parentContext => {
   }
 
   // Provide some data in context
-  const provideData = (providerId, data, scope = ContextScopes.Global) => {
+  const provideData = (providerId, data) => {
     if (!providerId || data === undefined) {
       return
     }
 
-    // Proxy message up the chain if we have a parent and are providing global
-    // context
-    if (scope === ContextScopes.Global && parentContext) {
-      parentContext.actions.provideData(providerId, data, scope)
-    }
-
     // Otherwise this is either the context root, or we're providing a local
     // context override, so we need to update the local context instead
-    else {
+    context.update(state => {
-      context.update(state => {
+      state[providerId] = data
-        state[providerId] = data
+      return state
-        return state
+    })
-      })
+    broadcastChange(providerId)
-      broadcastChange(providerId)
-    }
   }
 
   // Provides some action in context
-  const provideAction = (
+  const provideAction = (providerId, actionType, callback) => {
-    providerId,
-    actionType,
-    callback,
-    scope = ContextScopes.Global
-  ) => {
     if (!providerId || !actionType) {
       return
     }
 
-    // Proxy message up the chain if we have a parent and are providing global
-    // context
-    if (scope === ContextScopes.Global && parentContext) {
-      parentContext.actions.provideAction(
-        providerId,
-        actionType,
-        callback,
-        scope
-      )
-    }
-
     // Otherwise this is either the context root, or we're providing a local
     // context override, so we need to update the local context instead
-    else {
+    const key = `${providerId}_${actionType}`
-      const key = `${providerId}_${actionType}`
+    context.update(state => {
-      context.update(state => {
+      state[key] = callback
-        state[key] = callback
+      return state
-        return state
+    })
-      })
+    broadcastChange(key)
-      broadcastChange(key)
-    }
   }
 
   const observeChanges = callback => {

packages/client/src/stores/org.js

@@ -14,7 +14,7 @@ const createOrgStore = () => {
       const settingsConfigDoc = await API.getTenantConfig(tenantId)
       set({ logoUrl: settingsConfigDoc.config.logoUrl })
     } catch (e) {
-      console.log("Could not init org ", e)
+      console.error("Could not init org ", e)
     }
   }
 

packages/client/src/utils/buttonActions.js

@@ -211,29 +211,27 @@ const deleteRowHandler = async action => {
 
 const triggerAutomationHandler = async action => {
   const { fields, notificationOverride, timeout } = action.parameters
-  if (fields) {
+  try {
-    try {
+    const result = await API.triggerAutomation({
-      const result = await API.triggerAutomation({
+      automationId: action.parameters.automationId,
-        automationId: action.parameters.automationId,
+      fields,
-        fields,
+      timeout,
-        timeout,
+    })
-      })
-
-      // Value will exist if automation is synchronous, so return it.
-      if (result.value) {
-        if (!notificationOverride) {
-          notificationStore.actions.success("Automation ran successfully")
-        }
-        return { result }
-      }
 
+    // Value will exist if automation is synchronous, so return it.
+    if (result.value) {
       if (!notificationOverride) {
-        notificationStore.actions.success("Automation triggered")
+        notificationStore.actions.success("Automation ran successfully")
       }
-    } catch (error) {
+      return { result }
-      // Abort next actions
-      return false
     }
+
+    if (!notificationOverride) {
+      notificationStore.actions.success("Automation triggered")
+    }
+  } catch (error) {
+    // Abort next actions
+    return false
   }
 }
 const navigationHandler = action => {

packages/frontend-core/src/components/grid/lib/websocket.js

@@ -29,7 +29,7 @@ export const createGridWebsocket = context => {
     connectToDatasource(get(datasource))
   })
   socket.on("connect_error", err => {
-    console.log("Failed to connect to grid websocket:", err.message)
+    console.error("Failed to connect to grid websocket:", err.message)
   })
 
   // User events

packages/frontend-core/src/constants.js

@@ -106,3 +106,8 @@ export const Themes = [
 export const EventPublishType = {
   ENV_VAR_UPGRADE_PANEL_OPENED: "environment_variable_upgrade_panel_opened",
 }
+
+export const ContextScopes = {
+  Local: "local",
+  Global: "global",
+}

packages/pro

@@ -1 +1 @@
-Subproject commit eb9565f568cfef14b336b14eee753119acfdd43b
+Subproject commit 9b14e5d5182bf5e5ee98f717997e7352e5904799

packages/server/__mocks__/aws-sdk.ts

@@ -1,7 +1,13 @@
+import fs from "fs"
+import { join } from "path"
+
 module AwsMock {
   const aws: any = {}
 
-  const response = (body: any) => () => ({ promise: () => body })
+  const response = (body: any, extra?: any) => () => ({
+    promise: () => body,
+    ...extra,
+  })
 
   function DocumentClient() {
     // @ts-ignore
@@ -73,9 +79,18 @@ module AwsMock {
 
     // @ts-ignore
     this.getObject = jest.fn(
-      response({
+      response(
-        Body: "",
+        {
-      })
+          Body: "",
+        },
+        {
+          createReadStream: jest
+            .fn()
+            .mockReturnValue(
+              fs.createReadStream(join(__dirname, "aws-sdk.ts"))
+            ),
+        }
+      )
     )
   }
 

packages/server/src/api/controllers/application.ts

@@ -445,6 +445,9 @@ export async function update(ctx: UserCtx) {
     name: app.name,
     url: app.url,
     icon: app.icon,
+    automations: {
+      chainAutomations: app.automations?.chainAutomations,
+    },
   })
 }
 

packages/server/src/api/controllers/query/index.ts

@@ -15,6 +15,9 @@ import {
   SessionCookie,
   QuerySchema,
   FieldType,
+  type ExecuteQueryRequest,
+  type ExecuteQueryResponse,
+  type Row,
 } from "@budibase/types"
 import { ValidQueryNameRegex } from "@budibase/shared-core"
 
@@ -223,7 +226,7 @@ export async function preview(ctx: UserCtx) {
 }
 
 async function execute(
-  ctx: UserCtx,
+  ctx: UserCtx<ExecuteQueryRequest, ExecuteQueryResponse | Row[]>,
   opts: any = { rowsOnly: false, isAutomation: false }
 ) {
   const db = context.getAppDB()

packages/server/src/api/routes/tests/queries/mongodb.spec.ts

@@ -0,0 +1,393 @@
+import { Datasource, Query } from "@budibase/types"
+import * as setup from "../utilities"
+import { databaseTestProviders } from "../../../../integrations/tests/utils"
+import { MongoClient, type Collection, BSON } from "mongodb"
+
+jest.unmock("mongodb")
+
+const collection = "test_collection"
+
+const expectValidId = expect.stringMatching(/^\w{24}$/)
+const expectValidBsonObjectId = expect.any(BSON.ObjectId)
+
+describe("/queries", () => {
+  let config = setup.getConfig()
+  let datasource: Datasource
+
+  async function createQuery(query: Partial<Query>): Promise<Query> {
+    const defaultQuery: Query = {
+      datasourceId: datasource._id!,
+      name: "New Query",
+      parameters: [],
+      fields: {},
+      schema: {},
+      queryVerb: "read",
+      transformer: "return data",
+      readable: true,
+    }
+    const combinedQuery = { ...defaultQuery, ...query }
+    if (
+      combinedQuery.fields &&
+      combinedQuery.fields.extra &&
+      !combinedQuery.fields.extra.collection
+    ) {
+      combinedQuery.fields.extra.collection = collection
+    }
+    return await config.api.query.create(combinedQuery)
+  }
+
+  async function withClient(
+    callback: (client: MongoClient) => Promise<void>
+  ): Promise<void> {
+    const ds = await databaseTestProviders.mongodb.datasource()
+    const client = new MongoClient(ds.config!.connectionString)
+    await client.connect()
+    try {
+      await callback(client)
+    } finally {
+      await client.close()
+    }
+  }
+
+  async function withCollection(
+    callback: (collection: Collection) => Promise<void>
+  ): Promise<void> {
+    await withClient(async client => {
+      const db = client.db(
+        (await databaseTestProviders.mongodb.datasource()).config!.db
+      )
+      await callback(db.collection(collection))
+    })
+  }
+
+  afterAll(async () => {
+    await databaseTestProviders.mongodb.stop()
+    setup.afterAll()
+  })
+
+  beforeAll(async () => {
+    await config.init()
+    datasource = await config.api.datasource.create(
+      await databaseTestProviders.mongodb.datasource()
+    )
+  })
+
+  beforeEach(async () => {
+    await withCollection(async collection => {
+      await collection.insertMany([
+        { name: "one" },
+        { name: "two" },
+        { name: "three" },
+        { name: "four" },
+        { name: "five" },
+      ])
+    })
+  })
+
+  afterEach(async () => {
+    await withCollection(async collection => {
+      await collection.drop()
+    })
+  })
+
+  it("should execute a count query", async () => {
+    const query = await createQuery({
+      fields: {
+        json: {},
+        extra: {
+          actionType: "count",
+        },
+      },
+    })
+
+    const result = await config.api.query.execute(query._id!)
+
+    expect(result.data).toEqual([{ value: 5 }])
+  })
+
+  it("should execute a count query with a transformer", async () => {
+    const query = await createQuery({
+      fields: {
+        json: {},
+        extra: {
+          actionType: "count",
+        },
+      },
+      transformer: "return data + 1",
+    })
+
+    const result = await config.api.query.execute(query._id!)
+
+    expect(result.data).toEqual([{ value: 6 }])
+  })
+
+  it("should execute a find query", async () => {
+    const query = await createQuery({
+      fields: {
+        json: {},
+        extra: {
+          actionType: "find",
+        },
+      },
+    })
+
+    const result = await config.api.query.execute(query._id!)
+
+    expect(result.data).toEqual([
+      { _id: expectValidId, name: "one" },
+      { _id: expectValidId, name: "two" },
+      { _id: expectValidId, name: "three" },
+      { _id: expectValidId, name: "four" },
+      { _id: expectValidId, name: "five" },
+    ])
+  })
+
+  it("should execute a findOne query", async () => {
+    const query = await createQuery({
+      fields: {
+        json: {},
+        extra: {
+          actionType: "findOne",
+        },
+      },
+    })
+
+    const result = await config.api.query.execute(query._id!)
+
+    expect(result.data).toEqual([{ _id: expectValidId, name: "one" }])
+  })
+
+  it("should execute a findOneAndUpdate query", async () => {
+    const query = await createQuery({
+      fields: {
+        json: {
+          filter: { name: { $eq: "one" } },
+          update: { $set: { name: "newName" } },
+        },
+        extra: {
+          actionType: "findOneAndUpdate",
+        },
+      },
+    })
+
+    const result = await config.api.query.execute(query._id!)
+
+    expect(result.data).toEqual([
+      {
+        lastErrorObject: { n: 1, updatedExisting: true },
+        ok: 1,
+        value: { _id: expectValidId, name: "one" },
+      },
+    ])
+
+    await withCollection(async collection => {
+      expect(await collection.countDocuments()).toBe(5)
+
+      const doc = await collection.findOne({ name: { $eq: "newName" } })
+      expect(doc).toEqual({
+        _id: expectValidBsonObjectId,
+        name: "newName",
+      })
+    })
+  })
+
+  it("should execute a distinct query", async () => {
+    const query = await createQuery({
+      fields: {
+        json: "name",
+        extra: {
+          actionType: "distinct",
+        },
+      },
+    })
+
+    const result = await config.api.query.execute(query._id!)
+    const values = result.data.map(o => o.value).sort()
+    expect(values).toEqual(["five", "four", "one", "three", "two"])
+  })
+
+  it("should execute a create query with parameters", async () => {
+    const query = await createQuery({
+      fields: {
+        json: { foo: "{{ foo }}" },
+        extra: {
+          actionType: "insertOne",
+        },
+      },
+      queryVerb: "create",
+      parameters: [
+        {
+          name: "foo",
+          default: "default",
+        },
+      ],
+    })
+
+    const result = await config.api.query.execute(query._id!, {
+      parameters: { foo: "bar" },
+    })
+
+    expect(result.data).toEqual([
+      {
+        acknowledged: true,
+        insertedId: expectValidId,
+      },
+    ])
+
+    await withCollection(async collection => {
+      const doc = await collection.findOne({ foo: { $eq: "bar" } })
+      expect(doc).toEqual({
+        _id: expectValidBsonObjectId,
+        foo: "bar",
+      })
+    })
+  })
+
+  it("should execute a delete query with parameters", async () => {
+    const query = await createQuery({
+      fields: {
+        json: { name: { $eq: "{{ name }}" } },
+        extra: {
+          actionType: "deleteOne",
+        },
+      },
+      queryVerb: "delete",
+      parameters: [
+        {
+          name: "name",
+          default: "",
+        },
+      ],
+    })
+
+    const result = await config.api.query.execute(query._id!, {
+      parameters: { name: "one" },
+    })
+
+    expect(result.data).toEqual([
+      {
+        acknowledged: true,
+        deletedCount: 1,
+      },
+    ])
+
+    await withCollection(async collection => {
+      const doc = await collection.findOne({ name: { $eq: "one" } })
+      expect(doc).toBeNull()
+    })
+  })
+
+  it("should execute an update query with parameters", async () => {
+    const query = await createQuery({
+      fields: {
+        json: {
+          filter: { name: { $eq: "{{ name }}" } },
+          update: { $set: { name: "{{ newName }}" } },
+        },
+        extra: {
+          actionType: "updateOne",
+        },
+      },
+      queryVerb: "update",
+      parameters: [
+        {
+          name: "name",
+          default: "",
+        },
+        {
+          name: "newName",
+          default: "",
+        },
+      ],
+    })
+
+    const result = await config.api.query.execute(query._id!, {
+      parameters: { name: "one", newName: "newOne" },
+    })
+
+    expect(result.data).toEqual([
+      {
+        acknowledged: true,
+        matchedCount: 1,
+        modifiedCount: 1,
+        upsertedCount: 0,
+        upsertedId: null,
+      },
+    ])
+
+    await withCollection(async collection => {
+      const doc = await collection.findOne({ name: { $eq: "newOne" } })
+      expect(doc).toEqual({
+        _id: expectValidBsonObjectId,
+        name: "newOne",
+      })
+
+      const oldDoc = await collection.findOne({ name: { $eq: "one" } })
+      expect(oldDoc).toBeNull()
+    })
+  })
+
+  it("should be able to delete all records", async () => {
+    const query = await createQuery({
+      fields: {
+        json: {},
+        extra: {
+          actionType: "deleteMany",
+        },
+      },
+      queryVerb: "delete",
+    })
+
+    const result = await config.api.query.execute(query._id!)
+
+    expect(result.data).toEqual([
+      {
+        acknowledged: true,
+        deletedCount: 5,
+      },
+    ])
+
+    await withCollection(async collection => {
+      const docs = await collection.find().toArray()
+      expect(docs).toHaveLength(0)
+    })
+  })
+
+  it("should be able to update all documents", async () => {
+    const query = await createQuery({
+      fields: {
+        json: {
+          filter: {},
+          update: { $set: { name: "newName" } },
+        },
+        extra: {
+          actionType: "updateMany",
+        },
+      },
+      queryVerb: "update",
+    })
+
+    const result = await config.api.query.execute(query._id!)
+
+    expect(result.data).toEqual([
+      {
+        acknowledged: true,
+        matchedCount: 5,
+        modifiedCount: 5,
+        upsertedCount: 0,
+        upsertedId: null,
+      },
+    ])
+
+    await withCollection(async collection => {
+      const docs = await collection.find().toArray()
+      expect(docs).toHaveLength(5)
+      for (const doc of docs) {
+        expect(doc).toEqual({
+          _id: expectValidBsonObjectId,
+          name: "newName",
+        })
+      }
+    })
+  })
+})

packages/server/src/api/routes/tests/queries/postgres.spec.ts

@@ -0,0 +1,170 @@
+import { Datasource, Query } from "@budibase/types"
+import * as setup from "../utilities"
+import { databaseTestProviders } from "../../../../integrations/tests/utils"
+import { Client } from "pg"
+
+jest.unmock("pg")
+
+const createTableSQL = `
+CREATE TABLE test_table (
+    id serial PRIMARY KEY,
+    name VARCHAR ( 50 ) NOT NULL
+);
+`
+
+const insertSQL = `
+INSERT INTO test_table (name) VALUES ('one');
+INSERT INTO test_table (name) VALUES ('two');
+INSERT INTO test_table (name) VALUES ('three');
+INSERT INTO test_table (name) VALUES ('four');
+INSERT INTO test_table (name) VALUES ('five');
+`
+
+const dropTableSQL = `
+DROP TABLE test_table;
+`
+
+describe("/queries", () => {
+  let config = setup.getConfig()
+  let datasource: Datasource
+
+  async function createQuery(query: Partial<Query>): Promise<Query> {
+    const defaultQuery: Query = {
+      datasourceId: datasource._id!,
+      name: "New Query",
+      parameters: [],
+      fields: {},
+      schema: {},
+      queryVerb: "read",
+      transformer: "return data",
+      readable: true,
+    }
+    return await config.api.query.create({ ...defaultQuery, ...query })
+  }
+
+  async function withClient(
+    callback: (client: Client) => Promise<void>
+  ): Promise<void> {
+    const ds = await databaseTestProviders.postgres.datasource()
+    const client = new Client(ds.config!)
+    await client.connect()
+    try {
+      await callback(client)
+    } finally {
+      await client.end()
+    }
+  }
+
+  afterAll(async () => {
+    await databaseTestProviders.postgres.stop()
+    setup.afterAll()
+  })
+
+  beforeAll(async () => {
+    await config.init()
+    datasource = await config.api.datasource.create(
+      await databaseTestProviders.postgres.datasource()
+    )
+  })
+
+  beforeEach(async () => {
+    await withClient(async client => {
+      await client.query(createTableSQL)
+      await client.query(insertSQL)
+    })
+  })
+
+  afterEach(async () => {
+    await withClient(async client => {
+      await client.query(dropTableSQL)
+    })
+  })
+
+  it("should execute a query", async () => {
+    const query = await createQuery({
+      fields: {
+        sql: "SELECT * FROM test_table ORDER BY id",
+      },
+    })
+
+    const result = await config.api.query.execute(query._id!)
+
+    expect(result.data).toEqual([
+      {
+        id: 1,
+        name: "one",
+      },
+      {
+        id: 2,
+        name: "two",
+      },
+      {
+        id: 3,
+        name: "three",
+      },
+      {
+        id: 4,
+        name: "four",
+      },
+      {
+        id: 5,
+        name: "five",
+      },
+    ])
+  })
+
+  it("should be able to transform a query", async () => {
+    const query = await createQuery({
+      fields: {
+        sql: "SELECT * FROM test_table WHERE id = 1",
+      },
+      transformer: `
+        data[0].id = data[0].id + 1;
+        return data;
+      `,
+    })
+
+    const result = await config.api.query.execute(query._id!)
+
+    expect(result.data).toEqual([
+      {
+        id: 2,
+        name: "one",
+      },
+    ])
+  })
+
+  it("should be able to insert with bindings", async () => {
+    const query = await createQuery({
+      fields: {
+        sql: "INSERT INTO test_table (name) VALUES ({{ foo }})",
+      },
+      parameters: [
+        {
+          name: "foo",
+          default: "bar",
+        },
+      ],
+      queryVerb: "create",
+    })
+
+    const result = await config.api.query.execute(query._id!, {
+      parameters: {
+        foo: "baz",
+      },
+    })
+
+    expect(result.data).toEqual([
+      {
+        created: true,
+      },
+    ])
+
+    await withClient(async client => {
+      const { rows } = await client.query(
+        "SELECT * FROM test_table WHERE name = 'baz'"
+      )
+      expect(rows).toHaveLength(1)
+    })
+  })
+})

packages/server/src/api/routes/tests/queries/query.seq.spec.ts

@@ -16,9 +16,9 @@ jest.mock("@budibase/backend-core", () => {
     },
   }
 })
-import * as setup from "./utilities"
+import * as setup from "../utilities"
-import { checkBuilderEndpoint } from "./utilities/TestFunctions"
+import { checkBuilderEndpoint } from "../utilities/TestFunctions"
-import { checkCacheForDynamicVariable } from "../../../threads/utils"
+import { checkCacheForDynamicVariable } from "../../../../threads/utils"
 
 const { basicQuery, basicDatasource } = setup.structures
 import { events, db as dbCore } from "@budibase/backend-core"

packages/server/src/api/routes/tests/row.spec.ts

@@ -12,7 +12,6 @@ import {
   FieldTypeSubtypes,
   FormulaType,
   INTERNAL_TABLE_SOURCE_ID,
-  MonthlyQuotaName,
   PermissionLevel,
   QuotaUsageType,
   RelationshipType,
@@ -53,7 +52,7 @@ describe.each([
 
   afterAll(async () => {
     if (dsProvider) {
-      await dsProvider.stopContainer()
+      await dsProvider.stop()
     }
     setup.afterAll()
   })
@@ -63,7 +62,7 @@ describe.each([
 
     if (dsProvider) {
       await config.createDatasource({
-        datasource: await dsProvider.getDsConfig(),
+        datasource: await dsProvider.datasource(),
       })
     }
   })
@@ -117,16 +116,6 @@ describe.each([
     return total
   }
 
-  const getQueryUsage = async () => {
-    const { total } = await config.doInContext(null, () =>
-      quotas.getCurrentUsageValues(
-        QuotaUsageType.MONTHLY,
-        MonthlyQuotaName.QUERIES
-      )
-    )
-    return total
-  }
-
   const assertRowUsage = async (expected: number) => {
     const usage = await getRowUsage()
     expect(usage).toBe(expected)
@@ -162,7 +151,6 @@ describe.each([
   describe("save, load, update", () => {
     it("returns a success message when the row is created", async () => {
       const rowUsage = await getRowUsage()
-      const queryUsage = await getQueryUsage()
 
       const res = await request
         .post(`/api/${tableId}/rows`)
@@ -180,7 +168,6 @@ describe.each([
 
     it("Increment row autoId per create row request", async () => {
       const rowUsage = await getRowUsage()
-      const queryUsage = await getQueryUsage()
 
       const tableConfig = generateTableConfig()
       const newTable = await createTable(
@@ -231,7 +218,6 @@ describe.each([
     it("updates a row successfully", async () => {
       const existing = await config.createRow()
       const rowUsage = await getRowUsage()
-      const queryUsage = await getQueryUsage()
 
       const res = await config.api.row.save(tableId, {
         _id: existing._id,
@@ -246,7 +232,6 @@ describe.each([
 
     it("should load a row", async () => {
       const existing = await config.createRow()
-      const queryUsage = await getQueryUsage()
 
       const res = await config.api.row.get(tableId, existing._id!)
 
@@ -268,7 +253,6 @@ describe.each([
       }
       const firstRow = await config.createRow({ tableId })
       await config.createRow(newRow)
-      const queryUsage = await getQueryUsage()
 
       const res = await config.api.row.fetch(tableId)
 
@@ -279,7 +263,6 @@ describe.each([
 
     it("load should return 404 when row does not exist", async () => {
       await config.createRow()
-      const queryUsage = await getQueryUsage()
 
       await config.api.row.get(tableId, "1234567", {
         expectStatus: 404,
@@ -530,7 +513,6 @@ describe.each([
       const existing = await config.createRow()
 
       const rowUsage = await getRowUsage()
-      const queryUsage = await getQueryUsage()
 
       const row = await config.api.row.patch(table._id!, {
         _id: existing._id!,
@@ -552,7 +534,6 @@ describe.each([
     it("should throw an error when given improper types", async () => {
       const existing = await config.createRow()
       const rowUsage = await getRowUsage()
-      const queryUsage = await getQueryUsage()
 
       await config.api.row.patch(
         table._id!,
@@ -650,7 +631,6 @@ describe.each([
     it("should be able to delete a row", async () => {
       const createdRow = await config.createRow()
       const rowUsage = await getRowUsage()
-      const queryUsage = await getQueryUsage()
 
       const res = await config.api.row.delete(table._id!, [createdRow])
       expect(res.body[0]._id).toEqual(createdRow._id)
@@ -666,7 +646,6 @@ describe.each([
 
     it("should return no errors on valid row", async () => {
       const rowUsage = await getRowUsage()
-      const queryUsage = await getQueryUsage()
 
       const res = await config.api.row.validate(table._id!, { name: "ivan" })
 
@@ -677,7 +656,6 @@ describe.each([
 
     it("should errors on invalid row", async () => {
       const rowUsage = await getRowUsage()
-      const queryUsage = await getQueryUsage()
 
       const res = await config.api.row.validate(table._id!, { name: 1 })
 
@@ -703,7 +681,6 @@ describe.each([
       const row1 = await config.createRow()
       const row2 = await config.createRow()
       const rowUsage = await getRowUsage()
-      const queryUsage = await getQueryUsage()
 
       const res = await config.api.row.delete(table._id!, [row1, row2])
 
@@ -719,7 +696,6 @@ describe.each([
         config.createRow(),
       ])
       const rowUsage = await getRowUsage()
-      const queryUsage = await getQueryUsage()
 
       const res = await config.api.row.delete(table._id!, [
         row1,
@@ -735,7 +711,6 @@ describe.each([
     it("should accept a valid row object and delete the row", async () => {
       const row1 = await config.createRow()
       const rowUsage = await getRowUsage()
-      const queryUsage = await getQueryUsage()
 
       const res = await config.api.row.delete(table._id!, row1)
 
@@ -746,7 +721,6 @@ describe.each([
 
     it("Should ignore malformed/invalid delete requests", async () => {
       const rowUsage = await getRowUsage()
-      const queryUsage = await getQueryUsage()
 
       const res = await config.api.row.delete(
         table._id!,
@@ -782,7 +756,6 @@ describe.each([
       it("should be able to fetch tables contents via 'view'", async () => {
         const row = await config.createRow()
         const rowUsage = await getRowUsage()
-        const queryUsage = await getQueryUsage()
 
         const res = await config.api.legacyView.get(table._id!)
         expect(res.body.length).toEqual(1)
@@ -792,7 +765,6 @@ describe.each([
 
       it("should throw an error if view doesn't exist", async () => {
         const rowUsage = await getRowUsage()
-        const queryUsage = await getQueryUsage()
 
         await config.api.legacyView.get("derp", { expectStatus: 404 })
 
@@ -808,7 +780,6 @@ describe.each([
         })
         const row = await config.createRow()
         const rowUsage = await getRowUsage()
-        const queryUsage = await getQueryUsage()
 
         const res = await config.api.legacyView.get(view.name)
         expect(res.body.length).toEqual(1)
@@ -864,7 +835,6 @@ describe.each([
         }
       )
       const rowUsage = await getRowUsage()
-      const queryUsage = await getQueryUsage()
 
       // test basic enrichment
       const resBasic = await config.api.row.get(
@@ -1100,7 +1070,6 @@ describe.each([
 
         const createdRow = await config.createRow()
         const rowUsage = await getRowUsage()
-        const queryUsage = await getQueryUsage()
 
         await config.api.row.delete(view.id, [createdRow])
 
@@ -1127,7 +1096,6 @@ describe.each([
           config.createRow(),
         ])
         const rowUsage = await getRowUsage()
-        const queryUsage = await getQueryUsage()
 
         await config.api.row.delete(view.id, [rows[0], rows[2]])
 

packages/server/src/integration-test/postgres.spec.ts

@@ -41,12 +41,12 @@ describe("postgres integrations", () => {
     makeRequest = generateMakeRequest(apiKey, true)
 
     postgresDatasource = await config.api.datasource.create(
-      await databaseTestProviders.postgres.getDsConfig()
+      await databaseTestProviders.postgres.datasource()
     )
   })
 
   afterAll(async () => {
-    await databaseTestProviders.postgres.stopContainer()
+    await databaseTestProviders.postgres.stop()
   })
 
   beforeEach(async () => {
@@ -1041,14 +1041,14 @@ describe("postgres integrations", () => {
   describe("POST /api/datasources/verify", () => {
     it("should be able to verify the connection", async () => {
       const response = await config.api.datasource.verify({
-        datasource: await databaseTestProviders.postgres.getDsConfig(),
+        datasource: await databaseTestProviders.postgres.datasource(),
       })
       expect(response.status).toBe(200)
       expect(response.body.connected).toBe(true)
     })
 
     it("should state an invalid datasource cannot connect", async () => {
-      const dbConfig = await databaseTestProviders.postgres.getDsConfig()
+      const dbConfig = await databaseTestProviders.postgres.datasource()
       const response = await config.api.datasource.verify({
         datasource: {
           ...dbConfig,
@@ -1082,7 +1082,7 @@ describe("postgres integrations", () => {
 
     beforeEach(async () => {
       client = new Client(
-        (await databaseTestProviders.postgres.getDsConfig()).config!
+        (await databaseTestProviders.postgres.datasource()).config!
       )
       await client.connect()
     })
@@ -1125,7 +1125,7 @@ describe("postgres integrations", () => {
       schema2 = "test-2"
 
     beforeAll(async () => {
-      const dsConfig = await databaseTestProviders.postgres.getDsConfig()
+      const dsConfig = await databaseTestProviders.postgres.datasource()
       const dbConfig = dsConfig.config!
 
       client = new Client(dbConfig)

packages/server/src/integrations/postgres.ts

@@ -29,6 +29,7 @@ import { Client, ClientConfig, types } from "pg"
 import { getReadableErrorMessage } from "./base/errorMapping"
 import { exec } from "child_process"
 import { storeTempFile } from "../utilities/fileSystem"
+import { env } from "@budibase/backend-core"
 
 // Return "date" and "timestamp" types as plain strings.
 // This lets us reference the original stored timezone.
@@ -202,8 +203,13 @@ class PostgresIntegration extends Sql implements DatasourcePlus {
       await this.openConnection()
       response.connected = true
     } catch (e: any) {
-      console.log(e)
+      if (typeof e.message === "string" && e.message !== "") {
-      response.error = e.message as string
+        response.error = e.message as string
+      } else if (typeof e.code === "string" && e.code !== "") {
+        response.error = e.code
+      } else {
+        response.error = "Unknown error"
+      }
     } finally {
       await this.closeConnection()
     }
@@ -428,6 +434,14 @@ class PostgresIntegration extends Sql implements DatasourcePlus {
   }
 
   async getExternalSchema() {
+    if (!env.SELF_HOSTED) {
+      // This is because it relies on shelling out to pg_dump and we don't want
+      // to enable shell injection attacks.
+      throw new Error(
+        "schema export for Postgres is not supported in Budibase Cloud"
+      )
+    }
+
     const dumpCommandParts = [
       `user=${this.config.user}`,
       `host=${this.config.host}`,

packages/server/src/integrations/tests/utils/index.ts

@@ -1,14 +1,16 @@
 jest.unmock("pg")
 
 import { Datasource } from "@budibase/types"
-import * as pg from "./postgres"
+import * as postgres from "./postgres"
+import * as mongodb from "./mongodb"
+import { StartedTestContainer } from "testcontainers"
 
 jest.setTimeout(30000)
 
-export interface DatabasePlusTestProvider {
+export interface DatabaseProvider {
-  getDsConfig(): Promise<Datasource>
+  start(): Promise<StartedTestContainer>
+  stop(): Promise<void>
+  datasource(): Promise<Datasource>
 }
 
-export const databaseTestProviders = {
+export const databaseTestProviders = { postgres, mongodb }
-  postgres: pg,
-}

packages/server/src/integrations/tests/utils/mongodb.ts

@@ -0,0 +1,41 @@
+import { Datasource, SourceName } from "@budibase/types"
+import { GenericContainer, Wait, StartedTestContainer } from "testcontainers"
+
+let container: StartedTestContainer | undefined
+
+export async function start(): Promise<StartedTestContainer> {
+  return await new GenericContainer("mongo:7.0-jammy")
+    .withExposedPorts(27017)
+    .withEnvironment({
+      MONGO_INITDB_ROOT_USERNAME: "mongo",
+      MONGO_INITDB_ROOT_PASSWORD: "password",
+    })
+    .withWaitStrategy(
+      Wait.forSuccessfulCommand(`mongosh --eval "db.version()"`)
+    )
+    .start()
+}
+
+export async function datasource(): Promise<Datasource> {
+  if (!container) {
+    container = await start()
+  }
+  const host = container.getHost()
+  const port = container.getMappedPort(27017)
+  return {
+    type: "datasource",
+    source: SourceName.MONGODB,
+    plus: false,
+    config: {
+      connectionString: `mongodb://mongo:password@${host}:${port}`,
+      db: "mongo",
+    },
+  }
+}
+
+export async function stop() {
+  if (container) {
+    await container.stop()
+    container = undefined
+  }
+}

packages/server/src/integrations/tests/utils/postgres.ts

@@ -3,45 +3,44 @@ import { GenericContainer, Wait, StartedTestContainer } from "testcontainers"
 
 let container: StartedTestContainer | undefined
 
-export async function getDsConfig(): Promise<Datasource> {
+export async function start(): Promise<StartedTestContainer> {
-  try {
+  return await new GenericContainer("postgres:16.1-bullseye")
-    if (!container) {
+    .withExposedPorts(5432)
-      container = await new GenericContainer("postgres:16.1-bullseye")
+    .withEnvironment({ POSTGRES_PASSWORD: "password" })
-        .withExposedPorts(5432)
+    .withWaitStrategy(
-        .withEnvironment({ POSTGRES_PASSWORD: "password" })
+      Wait.forSuccessfulCommand(
-        .withWaitStrategy(
+        "pg_isready -h localhost -p 5432"
-          Wait.forLogMessage(
+      ).withStartupTimeout(10000)
-            "database system is ready to accept connections",
+    )
-            2
+    .start()
-          )
+}
-        )
-        .start()
-    }
-    const host = container.getHost()
-    const port = container.getMappedPort(5432)
 
-    return {
+export async function datasource(): Promise<Datasource> {
-      type: "datasource_plus",
+  if (!container) {
-      source: SourceName.POSTGRES,
+    container = await start()
-      plus: true,
+  }
-      config: {
+  const host = container.getHost()
-        host,
+  const port = container.getMappedPort(5432)
-        port,
+
-        database: "postgres",
+  return {
-        user: "postgres",
+    type: "datasource_plus",
-        password: "password",
+    source: SourceName.POSTGRES,
-        schema: "public",
+    plus: true,
-        ssl: false,
+    config: {
-        rejectUnauthorized: false,
+      host,
-        ca: false,
+      port,
-      },
+      database: "postgres",
-    }
+      user: "postgres",
-  } catch (err) {
+      password: "password",
-    throw new Error("**UNABLE TO CREATE TO POSTGRES CONTAINER**")
+      schema: "public",
+      ssl: false,
+      rejectUnauthorized: false,
+      ca: false,
+    },
   }
 }
 
-export async function stopContainer() {
+export async function stop() {
   if (container) {
     await container.stop()
     container = undefined

packages/server/src/sdk/app/backups/imports.ts

@@ -165,8 +165,9 @@ export async function importApp(
   const isTar = template.file && template?.file?.type?.endsWith("gzip")
   const isDirectory =
     template.file && fs.lstatSync(template.file.path).isDirectory()
+  let tmpPath: string | undefined = undefined
   if (template.file && (isTar || isDirectory)) {
-    const tmpPath = isTar ? await untarFile(template.file) : template.file.path
+    tmpPath = isTar ? await untarFile(template.file) : template.file.path
     if (isTar && template.file.password) {
       await decryptFiles(tmpPath, template.file.password)
     }
@@ -208,5 +209,9 @@ export async function importApp(
   }
   await updateAttachmentColumns(prodAppId, db)
   await updateAutomations(prodAppId, db)
+  // clear up afterward
+  if (tmpPath) {
+    fs.rmSync(tmpPath, { recursive: true, force: true })
+  }
   return ok
 }

packages/server/src/tests/utilities/api/index.ts

@@ -10,6 +10,7 @@ import { ApplicationAPI } from "./application"
 import { BackupAPI } from "./backup"
 import { AttachmentAPI } from "./attachment"
 import { UserAPI } from "./user"
+import { QueryAPI } from "./query"
 
 export default class API {
   table: TableAPI
@@ -23,6 +24,7 @@ export default class API {
   backup: BackupAPI
   attachment: AttachmentAPI
   user: UserAPI
+  query: QueryAPI
 
   constructor(config: TestConfiguration) {
     this.table = new TableAPI(config)
@@ -36,5 +38,6 @@ export default class API {
     this.backup = new BackupAPI(config)
     this.attachment = new AttachmentAPI(config)
     this.user = new UserAPI(config)
+    this.query = new QueryAPI(config)
   }
 }

packages/server/src/tests/utilities/api/query.ts

@@ -0,0 +1,44 @@
+import TestConfiguration from "../TestConfiguration"
+import {
+  Query,
+  type ExecuteQueryRequest,
+  type ExecuteQueryResponse,
+} from "@budibase/types"
+import { TestAPI } from "./base"
+
+export class QueryAPI extends TestAPI {
+  constructor(config: TestConfiguration) {
+    super(config)
+  }
+
+  create = async (body: Query): Promise<Query> => {
+    const res = await this.request
+      .post(`/api/queries`)
+      .set(this.config.defaultHeaders())
+      .send(body)
+      .expect("Content-Type", /json/)
+
+    if (res.status !== 200) {
+      throw new Error(JSON.stringify(res.body))
+    }
+
+    return res.body as Query
+  }
+
+  execute = async (
+    queryId: string,
+    body?: ExecuteQueryRequest
+  ): Promise<ExecuteQueryResponse> => {
+    const res = await this.request
+      .post(`/api/v2/queries/${queryId}`)
+      .set(this.config.defaultHeaders())
+      .send(body)
+      .expect("Content-Type", /json/)
+
+    if (res.status !== 200) {
+      throw new Error(JSON.stringify(res.body))
+    }
+
+    return res.body
+  }
+}

packages/server/src/utilities/workerRequests.ts

@@ -1,4 +1,4 @@
-import fetch from "node-fetch"
+import { Response, default as fetch } from "node-fetch"
 import env from "../environment"
 import { checkSlashesInUrl } from "./index"
 import {
@@ -40,25 +40,21 @@ export function request(ctx?: Ctx, request?: any) {
 }
 
 async function checkResponse(
-  response: any,
+  response: Response,
   errorMsg: string,
   { ctx }: { ctx?: Ctx } = {}
 ) {
-  if (response.status !== 200) {
+  if (response.status >= 300) {
-    let error
+    let responseErrorMessage
-    try {
+    if (response.headers.get("content-type")?.includes("json")) {
-      error = await response.json()
+      const error = await response.json()
-      if (!error.message) {
+      responseErrorMessage = error.message ?? JSON.stringify(error)
-        error = JSON.stringify(error)
+    } else {
-      }
+      responseErrorMessage = await response.text()
-    } catch (err) {
-      error = await response.text()
     }
-    const msg = `Unable to ${errorMsg} - ${
+    const msg = `Unable to ${errorMsg} - ${responseErrorMessage}`
-      error.message ? error.message : error
-    }`
     if (ctx) {
-      ctx.throw(400, msg)
+      ctx.throw(msg, response.status)
     } else {
       throw msg
     }

packages/string-templates/package.json

@@ -25,7 +25,7 @@
     "manifest": "node ./scripts/gen-collection-info.js"
   },
   "dependencies": {
-    "@budibase/handlebars-helpers": "^0.13.0",
+    "@budibase/handlebars-helpers": "^0.13.1",
     "dayjs": "^1.10.8",
     "handlebars": "^4.7.6",
     "lodash.clonedeep": "^4.5.0",

packages/string-templates/scripts/gen-collection-info.js

@@ -10,8 +10,8 @@ const marked = require("marked")
  * https://github.com/budibase/handlebars-helpers
  */
 const { join } = require("path")
+const path = require("path")
 
-const DIRECTORY = join(__dirname, "..", "..", "..")
 const COLLECTIONS = [
   "math",
   "array",
@@ -115,6 +115,8 @@ function getCommentInfo(file, func) {
     docs.example = docs.example.replace("product", "multiply")
   }
   docs.description = blocks[0].trim()
+  docs.acceptsBlock = docs.tags.some(el => el.title === "block")
+  docs.acceptsInline = docs.tags.some(el => el.title === "inline")
   return docs
 }
 
@@ -127,7 +129,7 @@ function run() {
   const foundNames = []
   for (let collection of COLLECTIONS) {
     const collectionFile = fs.readFileSync(
-      `${DIRECTORY}/node_modules/${HELPER_LIBRARY}/lib/${collection}.js`,
+      `${path.dirname(require.resolve(HELPER_LIBRARY))}/lib/${collection}.js`,
       "utf8"
     )
     const collectionInfo = {}
@@ -159,6 +161,7 @@ function run() {
         numArgs: args.length,
         example: jsDocInfo.example || undefined,
         description: jsDocInfo.description,
+        requiresBlock: jsDocInfo.acceptsBlock && !jsDocInfo.acceptsInline,
       })
     }
     outputJSON[collection] = collectionInfo

packages/string-templates/src/conversion/index.js

@@ -1,4 +1,4 @@
-const { getHelperList } = require("../helpers")
+const { getJsHelperList } = require("../helpers")
 
 function getLayers(fullBlock) {
   let layers = []
@@ -109,7 +109,7 @@ module.exports.convertHBSBlock = (block, blockNumber) => {
   const layers = getLayers(block)
 
   let value = null
-  const list = getHelperList()
+  const list = getJsHelperList()
   for (let layer of layers) {
     const parts = splitBySpace(layer)
     if (value || parts.length > 1 || list[parts[0]]) {

packages/string-templates/src/helpers/date.js

@@ -115,7 +115,7 @@ module.exports.duration = (str, pattern, format) => {
   setLocale(config.str, config.pattern)
 
   const duration = dayjs.duration(config.str, config.pattern)
-  if (!isOptions(format)) {
+  if (format && !isOptions(format)) {
     return duration.format(format)
   } else {
     return duration.humanize()

packages/string-templates/src/helpers/index.js

@@ -7,7 +7,7 @@ const {
   HelperFunctionBuiltin,
   LITERAL_MARKER,
 } = require("./constants")
-const { getHelperList } = require("./list")
+const { getJsHelperList } = require("./list")
 
 const HTML_SWAPS = {
   "<": "&lt;",
@@ -97,4 +97,4 @@ module.exports.unregisterAll = handlebars => {
   externalHandlebars.unregisterAll(handlebars)
 }
 
-module.exports.getHelperList = getHelperList
+module.exports.getJsHelperList = getJsHelperList

packages/string-templates/src/helpers/javascript.js

@@ -1,7 +1,7 @@
 const { atob } = require("../utilities")
 const cloneDeep = require("lodash.clonedeep")
 const { LITERAL_MARKER } = require("../helpers/constants")
-const { getHelperList } = require("./list")
+const { getJsHelperList } = require("./list")
 
 // The method of executing JS scripts depends on the bundle being built.
 // This setter is used in the entrypoint (either index.cjs or index.mjs).
@@ -49,7 +49,7 @@ module.exports.processJS = (handlebars, context) => {
     // app context.
     const sandboxContext = {
       $: path => getContextValue(path, cloneDeep(context)),
-      helpers: getHelperList(),
+      helpers: getJsHelperList(),
     }
 
     // Create a sandbox with our context and run the JS

packages/string-templates/src/helpers/list.js

@@ -3,7 +3,10 @@ const helperList = require("@budibase/handlebars-helpers")
 
 let helpers = undefined
 
-module.exports.getHelperList = () => {
+const helpersToRemoveForJs = ["sortBy"]
+module.exports.helpersToRemoveForJs = helpersToRemoveForJs
+
+module.exports.getJsHelperList = () => {
   if (helpers) {
     return helpers
   }
@@ -15,12 +18,17 @@ module.exports.getHelperList = () => {
   }
   for (let collection of constructed) {
     for (let [key, func] of Object.entries(collection)) {
-      helpers[key] = func
+      // Handlebars injects the hbs options to the helpers by default. We are adding an empty {} as a last parameter to simulate it
+      helpers[key] = (...props) => func(...props, {})
     }
   }
   for (let key of Object.keys(externalHandlebars.addedHelpers)) {
     helpers[key] = externalHandlebars.addedHelpers[key]
   }
+
+  for (const toRemove of helpersToRemoveForJs) {
+    delete helpers[toRemove]
+  }
   Object.freeze(helpers)
   return helpers
 }

packages/string-templates/src/index.cjs

@@ -20,6 +20,7 @@ module.exports.findHBSBlocks = templates.findHBSBlocks
 module.exports.convertToJS = templates.convertToJS
 module.exports.setJSRunner = templates.setJSRunner
 module.exports.FIND_ANY_HBS_REGEX = templates.FIND_ANY_HBS_REGEX
+module.exports.helpersToRemoveForJs = templates.helpersToRemoveForJs
 
 if (!process.env.NO_JS) {
   const { VM } = require("vm2")

packages/string-templates/src/index.js

@@ -10,6 +10,7 @@ const {
 } = require("./utilities")
 const { convertHBSBlock } = require("./conversion")
 const javascript = require("./helpers/javascript")
+const { helpersToRemoveForJs } = require("./helpers/list")
 
 const hbsInstance = handlebars.create()
 registerAll(hbsInstance)
@@ -394,3 +395,4 @@ module.exports.convertToJS = hbs => {
 }
 
 module.exports.FIND_ANY_HBS_REGEX = FIND_ANY_HBS_REGEX
+module.exports.helpersToRemoveForJs = helpersToRemoveForJs

packages/string-templates/src/index.mjs

@@ -21,6 +21,7 @@ export const findHBSBlocks = templates.findHBSBlocks
 export const convertToJS = templates.convertToJS
 export const setJSRunner = templates.setJSRunner
 export const FIND_ANY_HBS_REGEX = templates.FIND_ANY_HBS_REGEX
+export const helpersToRemoveForJs = templates.helpersToRemoveForJs
 
 if (process && !process.env.NO_JS) {
   /**

packages/string-templates/test/manifest.spec.js

@@ -16,21 +16,55 @@ jest.mock("@budibase/handlebars-helpers/lib/uuid", () => {
 })
 
 const fs = require("fs")
-const { processString } = require("../src/index.cjs")
+const {
+  processString,
+  convertToJS,
+  processStringSync,
+  encodeJSBinding,
+} = require("../src/index.cjs")
 
 const tk = require("timekeeper")
+const { getJsHelperList } = require("../src/helpers")
+
 tk.freeze("2021-01-21T12:00:00")
 
+const processJS = (js, context) => {
+  return processStringSync(encodeJSBinding(js), context)
+}
+
 const manifest = JSON.parse(
   fs.readFileSync(require.resolve("../manifest.json"), "utf8")
 )
 
 const collections = Object.keys(manifest)
 const examples = collections.reduce((acc, collection) => {
-  const functions = Object.keys(manifest[collection]).filter(
+  const functions = Object.entries(manifest[collection])
-    fnc => manifest[collection][fnc].example
+    .filter(([_, details]) => details.example)
-  )
+    .map(([name, details]) => {
-  if (functions.length) {
+      const example = details.example
+      let [hbs, js] = example.split("->").map(x => x.trim())
+      if (!js) {
+        // The function has no return value
+        return
+      }
+
+      // Trim 's
+      js = js.replace(/^\'|\'$/g, "")
+      if ((parsedExpected = tryParseJson(js))) {
+        if (Array.isArray(parsedExpected)) {
+          if (typeof parsedExpected[0] === "object") {
+            js = JSON.stringify(parsedExpected)
+          } else {
+            js = parsedExpected.join(",")
+          }
+        }
+      }
+      const requiresHbsBody = details.requiresBlock
+      return [name, { hbs, js, requiresHbsBody }]
+    })
+    .filter(x => !!x)
+
+  if (Object.keys(functions).length) {
     acc[collection] = functions
   }
   return acc
@@ -55,11 +89,7 @@ function tryParseJson(str) {
 describe("manifest", () => {
   describe("examples are valid", () => {
     describe.each(Object.keys(examples))("%s", collection => {
-      it.each(examples[collection])("%s", async func => {
+      it.each(examples[collection])("%s", async (_, { hbs, js }) => {
-        const example = manifest[collection][func].example
-
-        let [hbs, js] = example.split("->").map(x => x.trim())
-
         const context = {
           double: i => i * 2,
           isString: x => typeof x === "string",
@@ -71,23 +101,40 @@ describe("manifest", () => {
           context[`array${i}`] = JSON.parse(arrayString.replace(/\'/g, '"'))
         })
 
-        if (js === undefined) {
+        let result = await processString(hbs, context)
-          // The function has no return value
+        result = result.replace(/ /g, " ")
-          return
+        expect(result).toEqual(js)
+      })
+    })
+  })
+
+  describe("can be parsed and run as js", () => {
+    const jsHelpers = getJsHelperList()
+    const jsExamples = Object.keys(examples).reduce((acc, v) => {
+      acc[v] = examples[v].filter(([key]) => jsHelpers[key])
+      return acc
+    }, {})
+
+    describe.each(Object.keys(jsExamples))("%s", collection => {
+      it.each(
+        jsExamples[collection].filter(
+          ([_, { requiresHbsBody }]) => !requiresHbsBody
+        )
+      )("%s", async (_, { hbs, js }) => {
+        const context = {
+          double: i => i * 2,
+          isString: x => typeof x === "string",
         }
 
-        let result = await processString(hbs, context)
+        const arrays = hbs.match(/\[[^/\]]+\]/)
-        // Trim 's
+        arrays?.forEach((arrayString, i) => {
-        js = js.replace(/^\'|\'$/g, "")
+          hbs = hbs.replace(new RegExp(escapeRegExp(arrayString)), `array${i}`)
-        if ((parsedExpected = tryParseJson(js))) {
+          context[`array${i}`] = JSON.parse(arrayString.replace(/\'/g, '"'))
-          if (Array.isArray(parsedExpected)) {
+        })
-            if (typeof parsedExpected[0] === "object") {
+
-              js = JSON.stringify(parsedExpected)
+        let convertedJs = convertToJS(hbs)
-            } else {
+
-              js = parsedExpected.join(",")
+        let result = processJS(convertedJs, context)
-            }
-          }
-        }
         result = result.replace(/ /g, " ")
         expect(result).toEqual(js)
       })

packages/types/package.json

@@ -15,7 +15,7 @@
   },
   "jest": {},
   "devDependencies": {
-    "@budibase/nano": "10.1.4",
+    "@budibase/nano": "10.1.5",
     "@types/koa": "2.13.4",
     "@types/pouchdb": "6.4.0",
     "@types/redlock": "4.0.3",

packages/types/src/documents/app/query.ts

@@ -1,4 +1,5 @@
 import { Document } from "../document"
+import type { Row } from "./row"
 
 export interface QuerySchema {
   name?: string
@@ -54,3 +55,12 @@ export interface PreviewQueryRequest extends Omit<Query, "parameters"> {
     urlName?: boolean
   }
 }
+
+export interface ExecuteQueryRequest {
+  parameters?: { [key: string]: string }
+  pagination?: any
+}
+
+export interface ExecuteQueryResponse {
+  data: Row[]
+}

packages/worker/package.json

@@ -63,7 +63,7 @@
     "koa-useragent": "^4.1.0",
     "lodash": "4.17.21",
     "node-fetch": "2.6.7",
-    "nodemailer": "6.7.2",
+    "nodemailer": "6.9.9",
     "passport-google-oauth": "2.0.0",
     "passport-local": "1.0.0",
     "pouchdb": "7.3.0",

yarn.lock

@@ -2031,10 +2031,10 @@
   resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
   integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==
 
-"@budibase/handlebars-helpers@^0.13.0":
+"@budibase/handlebars-helpers@^0.13.1":
-  version "0.13.0"
+  version "0.13.1"
-  resolved "https://registry.yarnpkg.com/@budibase/handlebars-helpers/-/handlebars-helpers-0.13.0.tgz#224333d14e3900b7dacf48286af1e624a9fd62ea"
+  resolved "https://registry.yarnpkg.com/@budibase/handlebars-helpers/-/handlebars-helpers-0.13.1.tgz#d02e73c0df8305cd675e70dc37f8427eb0842080"
-  integrity sha512-g8+sFrMNxsIDnK+MmdUICTVGr6ReUFtnPp9hJX0VZwz1pN3Ynolpk/Qbu6rEWAvoU1sEqY1mXr9uo/+kEfeGbQ==
+  integrity sha512-v4RbXhr3igvK3i2pj5cNltu/4NMxdPIzcUt/o0RoInhesNH1VSLRdweSFr6/Y34fsCR5jHZ6vltdcz2RgrTKgw==
   dependencies:
     get-object "^0.2.0"
     get-value "^3.0.1"
@@ -2050,10 +2050,10 @@
     to-gfm-code-block "^0.1.1"
     uuid "^9.0.1"
 
-"@budibase/nano@10.1.4":
+"@budibase/nano@10.1.5":
-  version "10.1.4"
+  version "10.1.5"
-  resolved "https://registry.yarnpkg.com/@budibase/nano/-/nano-10.1.4.tgz#5c2670d0b4c12d736ddd6581c57d47c0aa45efad"
+  resolved "https://registry.yarnpkg.com/@budibase/nano/-/nano-10.1.5.tgz#eeaded7bfc707ecabf8fde604425b865a90c06ec"
-  integrity sha512-J+IVaAljGideDvJss/AUxXA1599HEIUJo5c0LLlmc1KMA3GZWZjyX+w2fxAw3qF7hqFvX+qAStQgdcD3+/GPMA==
+  integrity sha512-q1eKIsYKo+iK17zsJYd3VBl+5ufQMPpHYLec0wVsid8wnJVrTQk7RNpBlBUn/EDgXM7t8XNNHlERqHu+CxJu8Q==
   dependencies:
     "@types/tough-cookie" "^4.0.2"
     axios "^1.1.3"
@@ -15867,6 +15867,11 @@ nodemailer@6.7.2:
   resolved "https://registry.yarnpkg.com/nodemailer/-/nodemailer-6.7.2.tgz#44b2ad5f7ed71b7067f7a21c4fedabaec62b85e0"
   integrity sha512-Dz7zVwlef4k5R71fdmxwR8Q39fiboGbu3xgswkzGwczUfjp873rVxt1O46+Fh0j1ORnAC6L9+heI8uUpO6DT7Q==
 
+nodemailer@6.9.9:
+  version "6.9.9"
+  resolved "https://registry.yarnpkg.com/nodemailer/-/nodemailer-6.9.9.tgz#4549bfbf710cc6addec5064dd0f19874d24248d9"
+  integrity sha512-dexTll8zqQoVJEZPwQAKzxxtFn0qTnjdQTchoU6Re9BUUGBJiOy3YMn/0ShTW6J5M0dfQ1NeDeRTTl4oIWgQMA==
+
 nodemon@2.0.15:
   version "2.0.15"
   resolved "https://registry.yarnpkg.com/nodemon/-/nodemon-2.0.15.tgz#504516ce3b43d9dc9a955ccd9ec57550a31a8d4e"