Merge remote-tracking branch 'origin/develop' into feature/configurable-data-export

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "2.2.12-alpha.32",
+  "version": "2.2.12-alpha.47",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

packages/backend-core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/backend-core",
-  "version": "2.2.12-alpha.32",
+  "version": "2.2.12-alpha.47",
   "description": "Budibase backend core libraries used in server and worker",
   "main": "dist/src/index.js",
   "types": "dist/src/index.d.ts",
@@ -23,7 +23,7 @@
   },
   "dependencies": {
     "@budibase/nano": "10.1.1",
-    "@budibase/types": "2.2.12-alpha.32",
+    "@budibase/types": "2.2.12-alpha.47",
     "@shopify/jest-koa-mocks": "5.0.1",
     "@techpass/passport-openidconnect": "0.3.2",
     "aws-cloudfront-sign": "2.2.0",

packages/backend-core/src/constants/db.ts

@@ -77,6 +77,7 @@ export const StaticDatabases = {
       apiKeys: "apikeys",
       usageQuota: "usage_quota",
       licenseInfo: "license_info",
+      environmentVariables: "environmentvariables",
     },
   },
   // contains information about tenancy and so on

packages/backend-core/src/context/Context.ts

@@ -1,17 +1,14 @@
 import { AsyncLocalStorage } from "async_hooks"
+import { ContextMap } from "./mainContext"
 
 export default class Context {
-  static storage = new AsyncLocalStorage<Record<string, any>>()
+  static storage = new AsyncLocalStorage<ContextMap>()
 
-  static run(context: Record<string, any>, func: any) {
+  static run(context: ContextMap, func: any) {
     return Context.storage.run(context, () => func())
   }
 
-  static get(): Record<string, any> {
+  static get(): ContextMap {
-    return Context.storage.getStore() as Record<string, any>
+    return Context.storage.getStore() as ContextMap
-  }
-
-  static set(context: Record<string, any>) {
-    Context.storage.enterWith(context)
   }
 }

packages/backend-core/src/context/mainContext.ts

@@ -16,6 +16,7 @@ export type ContextMap = {
   tenantId?: string
   appId?: string
   identity?: IdentityContext
+  environmentVariables?: Record<string, string>
 }
 
 let TEST_APP_ID: string | null = null
@@ -75,7 +76,7 @@ export function getTenantIDFromAppID(appId: string) {
   }
 }
 
-function updateContext(updates: ContextMap) {
+function updateContext(updates: ContextMap): ContextMap {
   let context: ContextMap
   try {
     context = Context.get()
@@ -120,15 +121,23 @@ export async function doInTenant(
   return newContext(updates, task)
 }
 
-export async function doInAppContext(appId: string, task: any): Promise<any> {
+export async function doInAppContext(
-  if (!appId) {
+  appId: string | null,
+  task: any
+): Promise<any> {
+  if (!appId && !env.isTest()) {
     throw new Error("appId is required")
   }
 
-  const tenantId = getTenantIDFromAppID(appId)
+  let updates: ContextMap
-  const updates: ContextMap = { appId }
+  if (!appId) {
-  if (tenantId) {
+    updates = { appId: "" }
-    updates.tenantId = tenantId
+  } else {
+    const tenantId = getTenantIDFromAppID(appId)
+    updates = { appId }
+    if (tenantId) {
+      updates.tenantId = tenantId
+    }
   }
   return newContext(updates, task)
 }
@@ -189,25 +198,25 @@ export const getProdAppId = () => {
   return conversions.getProdAppID(appId)
 }
 
-export function updateTenantId(tenantId?: string) {
+export function doInEnvironmentContext(
-  let context: ContextMap = updateContext({
+  values: Record<string, string>,
-    tenantId,
+  task: any
-  })
+) {
-  Context.set(context)
+  if (!values) {
+    throw new Error("Must supply environment variables.")
+  }
+  const updates = {
+    environmentVariables: values,
+  }
+  return newContext(updates, task)
 }
 
-export function updateAppId(appId: string) {
+export function getEnvironmentVariables() {
-  let context: ContextMap = updateContext({
+  const context = Context.get()
-    appId,
+  if (!context.environmentVariables) {
-  })
+    return null
-  try {
+  } else {
-    Context.set(context)
+    return context.environmentVariables
-  } catch (err) {
-    if (env.isTest()) {
-      TEST_APP_ID = appId
-    } else {
-      throw err
-    }
   }
 }
 

packages/backend-core/src/environment.ts

@@ -37,6 +37,7 @@ const environment = {
   },
   JS_BCRYPT: process.env.JS_BCRYPT,
   JWT_SECRET: process.env.JWT_SECRET,
+  ENCRYPTION_KEY: process.env.ENCRYPTION_KEY,
   COUCH_DB_URL: process.env.COUCH_DB_URL || "http://localhost:4005",
   COUCH_DB_USERNAME: process.env.COUCH_DB_USER,
   COUCH_DB_PASSWORD: process.env.COUCH_DB_PASSWORD,

packages/backend-core/src/events/publishers/environmentVariable.ts

@@ -0,0 +1,38 @@
+import {
+  Event,
+  EnvironmentVariableCreatedEvent,
+  EnvironmentVariableDeletedEvent,
+  EnvironmentVariableUpgradePanelOpenedEvent,
+} from "@budibase/types"
+import { publishEvent } from "../events"
+
+async function created(name: string, environments: string[]) {
+  const properties: EnvironmentVariableCreatedEvent = {
+    name,
+    environments,
+  }
+  await publishEvent(Event.ENVIRONMENT_VARIABLE_CREATED, properties)
+}
+
+async function deleted(name: string) {
+  const properties: EnvironmentVariableDeletedEvent = {
+    name,
+  }
+  await publishEvent(Event.ENVIRONMENT_VARIABLE_DELETED, properties)
+}
+
+async function upgradePanelOpened(userId: string) {
+  const properties: EnvironmentVariableUpgradePanelOpenedEvent = {
+    userId,
+  }
+  await publishEvent(
+    Event.ENVIRONMENT_VARIABLE_UPGRADE_PANEL_OPENED,
+    properties
+  )
+}
+
+export default {
+  created,
+  deleted,
+  upgradePanelOpened,
+}

packages/backend-core/src/events/publishers/index.ts

@@ -20,3 +20,4 @@ export { default as backfill } from "./backfill"
 export { default as group } from "./group"
 export { default as plugin } from "./plugin"
 export { default as backup } from "./backup"
+export { default as environmentVariable } from "./environmentVariable"

packages/backend-core/src/security/encryption.ts

@@ -2,19 +2,45 @@ import crypto from "crypto"
 import env from "../environment"
 
 const ALGO = "aes-256-ctr"
-const SECRET = env.JWT_SECRET
 const SEPARATOR = "-"
 const ITERATIONS = 10000
 const RANDOM_BYTES = 16
 const STRETCH_LENGTH = 32
 
+export enum SecretOption {
+  JWT = "jwt",
+  ENCRYPTION = "encryption",
+}
+
+function getSecret(secretOption: SecretOption): string {
+  let secret, secretName
+  switch (secretOption) {
+    case SecretOption.ENCRYPTION:
+      secret = env.ENCRYPTION_KEY
+      secretName = "ENCRYPTION_KEY"
+      break
+    case SecretOption.JWT:
+    default:
+      secret = env.JWT_SECRET
+      secretName = "JWT_SECRET"
+      break
+  }
+  if (!secret) {
+    throw new Error(`Secret "${secretName}" has not been set in environment.`)
+  }
+  return secret
+}
+
 function stretchString(string: string, salt: Buffer) {
   return crypto.pbkdf2Sync(string, salt, ITERATIONS, STRETCH_LENGTH, "sha512")
 }
 
-export function encrypt(input: string) {
+export function encrypt(
+  input: string,
+  secretOption: SecretOption = SecretOption.JWT
+) {
   const salt = crypto.randomBytes(RANDOM_BYTES)
-  const stretched = stretchString(SECRET!, salt)
+  const stretched = stretchString(getSecret(secretOption), salt)
   const cipher = crypto.createCipheriv(ALGO, stretched, salt)
   const base = cipher.update(input)
   const final = cipher.final()
@@ -22,10 +48,13 @@ export function encrypt(input: string) {
   return `${salt.toString("hex")}${SEPARATOR}${encrypted}`
 }
 
-export function decrypt(input: string) {
+export function decrypt(
+  input: string,
+  secretOption: SecretOption = SecretOption.JWT
+) {
   const [salt, encrypted] = input.split(SEPARATOR)
   const saltBuffer = Buffer.from(salt, "hex")
-  const stretched = stretchString(SECRET!, saltBuffer)
+  const stretched = stretchString(getSecret(secretOption), saltBuffer)
   const decipher = crypto.createDecipheriv(ALGO, stretched, saltBuffer)
   const base = decipher.update(Buffer.from(encrypted, "hex"))
   const final = decipher.final()

packages/bbui/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/bbui",
   "description": "A UI solution used in the different Budibase projects.",
-  "version": "2.2.12-alpha.32",
+  "version": "2.2.12-alpha.47",
   "license": "MPL-2.0",
   "svelte": "src/index.js",
   "module": "dist/bbui.es.js",
@@ -38,7 +38,8 @@
   ],
   "dependencies": {
     "@adobe/spectrum-css-workflow-icons": "1.2.1",
-    "@budibase/string-templates": "2.2.12-alpha.32",
+    "@budibase/string-templates": "2.2.12-alpha.47",
+    "@spectrum-css/accordion": "3.0.24",
     "@spectrum-css/actionbutton": "1.0.1",
     "@spectrum-css/actiongroup": "1.0.1",
     "@spectrum-css/avatar": "3.0.2",

packages/bbui/src/Accordion/Accordion.svelte

@@ -0,0 +1,58 @@
+<script>
+  import "@spectrum-css/accordion"
+
+  export let itemName
+  export let initialOpen
+  export let header
+
+  let isOpen
+
+  function getOpenClass(isOpen) {
+    if (isOpen === undefined) {
+      isOpen = initialOpen
+    }
+    return isOpen ? "is-open" : ""
+  }
+</script>
+
+<div class="spectrum-Accordion" role={itemName}>
+  <div class="spectrum-Accordion-item {getOpenClass(isOpen)}">
+    <h3 class="spectrum-Accordion-itemHeading">
+      <button
+        class="spectrum-Accordion-itemHeader"
+        type="button"
+        on:click={() => (isOpen = !isOpen)}
+      >
+        {header}
+      </button>
+      <svg
+        class="spectrum-Icon spectrum-UIIcon-ChevronRight100 spectrum-Accordion-itemIndicator"
+        focusable="false"
+        aria-hidden="true"
+      >
+        <use xlink:href="#spectrum-css-icon-Chevron100" />
+      </svg>
+    </h3>
+    <div class="spectrum-Accordion-itemContent" role={itemName}>
+      <slot />
+    </div>
+  </div>
+</div>
+
+<style>
+  .spectrum-Accordion {
+    margin-left: -20px;
+  }
+  .spectrum-Accordion-item {
+    border: none;
+  }
+  .spectrum-Accordion-itemContent {
+    width: 97%;
+    padding-left: 30px;
+  }
+  .spectrum-Accordion-itemHeader {
+    text-transform: none;
+    font-weight: bold;
+    font-size: 0.875rem;
+  }
+</style>

packages/bbui/src/ActionButton/ActionButton.svelte

@@ -88,6 +88,7 @@
   }
   .is-selected:not(.spectrum-ActionButton--emphasized) {
     background: var(--spectrum-global-color-gray-300);
+    border-color: var(--spectrum-global-color-gray-700);
   }
   .noPadding {
     padding: 0;

packages/bbui/src/Actions/click_outside.js

@@ -1,4 +1,4 @@
-const ignoredClasses = [".flatpickr-calendar"]
+const ignoredClasses = [".flatpickr-calendar", ".spectrum-Popover"]
 let clickHandlers = []
 
 /**
@@ -19,7 +19,7 @@ const handleClick = event => {
     }
 
     // Ignore clicks for modals, unless the handler is registered from a modal
-    const sourceInModal = handler.element.closest(".spectrum-Modal") != null
+    const sourceInModal = handler.anchor.closest(".spectrum-Modal") != null
     const clickInModal = event.target.closest(".spectrum-Modal") != null
     if (clickInModal && !sourceInModal) {
       return
@@ -33,10 +33,10 @@ document.documentElement.addEventListener("click", handleClick, true)
 /**
  * Adds or updates a click handler
  */
-const updateHandler = (id, element, callback) => {
+const updateHandler = (id, element, anchor, callback) => {
   let existingHandler = clickHandlers.find(x => x.id === id)
   if (!existingHandler) {
-    clickHandlers.push({ id, element, callback })
+    clickHandlers.push({ id, element, anchor, callback })
   } else {
     existingHandler.callback = callback
   }
@@ -51,12 +51,22 @@ const removeHandler = id => {
 
 /**
  * Svelte action to apply a click outside handler for a certain element
+ * opts.anchor is an optional param specifying the real root source of the
+ * component being observed. This is required for things like popovers, where
+ * the element using the clickoutside action is the popover, but the popover is
+ * rendered at the root of the DOM somewhere, whereas the popover anchor is the
+ * element we actually want to consider when determining the source component.
  */
-export default (element, callback) => {
+export default (element, opts) => {
   const id = Math.random()
-  updateHandler(id, element, callback)
+  const update = newOpts => {
+    const callback = newOpts?.callback || newOpts
+    const anchor = newOpts?.anchor || element
+    updateHandler(id, element, anchor, callback)
+  }
+  update(opts)
   return {
-    update: newCallback => updateHandler(id, element, newCallback),
+    update,
     destroy: () => removeHandler(id),
   }
 }

packages/bbui/src/FancyForm/FancyButton.svelte

@@ -0,0 +1,30 @@
+<script>
+  import Icon from "../Icon/Icon.svelte"
+  import FancyField from "./FancyField.svelte"
+
+  export let icon
+  export let disabled
+</script>
+
+<FancyField on:click clickable {disabled}>
+  {#if icon}
+    {#if icon.includes("/")}
+      <img src={icon} alt="button" />
+    {:else}
+      <Icon name={icon} />
+    {/if}
+  {/if}
+  <slot name="icon" />
+  <div>
+    <slot />
+  </div>
+</FancyField>
+
+<style>
+  img {
+    width: 22px;
+  }
+  div {
+    font-size: var(--font-size-l);
+  }
+</style>

packages/bbui/src/FancyForm/FancyButtonRadio.svelte

@@ -0,0 +1,70 @@
+<script>
+  import { createEventDispatcher } from "svelte"
+  import FancyField from "./FancyField.svelte"
+  import FancyFieldLabel from "./FancyFieldLabel.svelte"
+  import ActionButton from "../ActionButton/ActionButton.svelte"
+
+  export let label
+  export let value
+  export let disabled = false
+  export let error = null
+  export let validate = null
+  export let options = []
+  export let getOptionLabel = option => extractProperty(option, "label")
+  export let getOptionValue = option => extractProperty(option, "value")
+
+  const dispatch = createEventDispatcher()
+
+  $: placeholder = !value
+
+  const extractProperty = (value, property) => {
+    if (value && typeof value === "object") {
+      return value[property]
+    }
+    return value
+  }
+
+  const onChange = newValue => {
+    dispatch("change", newValue)
+    value = newValue
+    if (validate) {
+      error = validate(newValue)
+    }
+  }
+</script>
+
+<FancyField {error} {value} {validate} {disabled} autoHeight>
+  {#if label}
+    <FancyFieldLabel placeholder={false}>{label}</FancyFieldLabel>
+  {/if}
+
+  <div class="options">
+    {#each options as option}
+      <ActionButton
+        selected={getOptionValue(option) === value}
+        on:click={() => onChange(getOptionValue(option))}
+      >
+        {getOptionLabel(option)}
+      </ActionButton>
+    {/each}
+  </div>
+</FancyField>
+
+<style>
+  .options {
+    margin-top: 34px;
+    margin-bottom: 14px;
+    display: flex;
+    flex-direction: row;
+    justify-content: flex-start;
+    align-items: center;
+    gap: 6px;
+    flex-wrap: wrap;
+  }
+  .options :global(.spectrum-ActionButton) {
+    font-size: 15px;
+    line-height: 17px;
+    height: auto;
+    padding: 6px 10px;
+  }
+</style>

packages/bbui/src/FancyForm/FancyCheckbox.svelte

@@ -0,0 +1,53 @@
+<script>
+  import { createEventDispatcher } from "svelte"
+  import FancyField from "./FancyField.svelte"
+  import Checkbox from "../Form/Core/Checkbox.svelte"
+
+  export let value
+  export let text
+  export let disabled = false
+  export let error = null
+  export let validate = null
+
+  const dispatch = createEventDispatcher()
+
+  const onChange = () => {
+    const newValue = !value
+    dispatch("change", newValue)
+    value = newValue
+    if (validate) {
+      error = validate(newValue)
+    }
+  }
+</script>
+
+<FancyField {error} {value} {validate} {disabled} clickable on:click={onChange}>
+  <span>
+    <Checkbox {disabled} {value} />
+  </span>
+  <div class="text">
+    {#if text}
+      {text}
+    {/if}
+    <slot />
+  </div>
+</FancyField>
+
+<style>
+  span {
+    pointer-events: none;
+  }
+  .text {
+    font-size: 15px;
+    line-height: 17px;
+    overflow: hidden;
+    text-overflow: ellipsis;
+    display: -webkit-box;
+    -webkit-line-clamp: 2;
+    line-clamp: 2;
+    -webkit-box-orient: vertical;
+  }
+  .text > :global(*) {
+    font-size: inherit !important;
+  }
+</style>

packages/bbui/src/FancyForm/FancyField.svelte

@@ -0,0 +1,126 @@
+<script>
+  import Icon from "../Icon/Icon.svelte"
+  import { getContext, onMount } from "svelte"
+  import { slide } from "svelte/transition"
+
+  export let disabled = false
+  export let error = null
+  export let focused = false
+  export let clickable = false
+  export let validate
+  export let value
+  export let ref
+  export let autoHeight
+
+  const formContext = getContext("fancy-form")
+  const id = Math.random()
+  const API = {
+    validate: () => {
+      if (validate) {
+        error = validate(value)
+      }
+      return !error
+    },
+  }
+
+  onMount(() => {
+    if (formContext) {
+      formContext.registerField(id, API)
+    }
+    return () => {
+      if (formContext) {
+        formContext.unregisterField(id)
+      }
+    }
+  })
+</script>
+
+<div
+  bind:this={ref}
+  class="fancy-field"
+  class:error
+  class:disabled
+  class:focused
+  class:clickable
+  class:auto-height={autoHeight}
+>
+  <div class="content" on:click>
+    <div class="field">
+      <slot />
+    </div>
+    {#if error}
+      <div class="error-icon">
+        <Icon name="Alert" />
+      </div>
+    {/if}
+  </div>
+  {#if error}
+    <div transition:slide|local={{ duration: 130 }} class="error-message">
+      {error}
+    </div>
+  {/if}
+</div>
+
+<style>
+  .fancy-field {
+    max-width: 400px;
+    background: var(--spectrum-global-color-gray-75);
+    border: 1px solid var(--spectrum-global-color-gray-300);
+    border-radius: 4px;
+    box-sizing: border-box;
+    transition: border-color 130ms ease-out, background 130ms ease-out,
+      background 130ms ease-out;
+    color: var(--spectrum-global-color-gray-800);
+  }
+  .fancy-field:hover {
+    border-color: var(--spectrum-global-color-gray-400);
+  }
+  .fancy-field.clickable:hover {
+    background: var(--spectrum-global-color-gray-200);
+    cursor: pointer;
+  }
+  .fancy-field.focused {
+    border-color: var(--spectrum-global-color-blue-400);
+  }
+  .fancy-field.error {
+    border-color: var(--spectrum-global-color-red-400);
+  }
+  .fancy-field.disabled {
+    background: var(--spectrum-global-color-gray-200);
+    color: var(--spectrum-global-color-gray-400);
+    border: 1px solid var(--spectrum-global-color-gray-300);
+    pointer-events: none;
+  }
+  .content {
+    position: relative;
+    height: 64px;
+    padding: 0 16px;
+  }
+  .fancy-field.auto-height .content {
+    height: auto;
+  }
+  .content,
+  .field {
+    display: flex;
+    flex-direction: row;
+    justify-content: flex-start;
+    align-items: center;
+    gap: 16px;
+  }
+  .field {
+    flex: 1 1 auto;
+  }
+  .error-message {
+    background: var(--spectrum-global-color-red-400);
+    color: white;
+    font-size: 14px;
+    padding: 6px 16px;
+    font-weight: 500;
+  }
+  .error-icon {
+    flex: 0 0 auto;
+  }
+  .error-icon :global(.spectrum-Icon) {
+    fill: var(--spectrum-global-color-red-400);
+  }
+</style>

packages/bbui/src/FancyForm/FancyFieldLabel.svelte

@@ -0,0 +1,25 @@
+<script>
+  export let placeholder = true
+</script>
+
+<div class:placeholder>
+  <slot />
+</div>
+
+<style>
+  div {
+    font-size: 14px;
+    line-height: 15px;
+    font-weight: 500;
+    position: absolute;
+    top: 10px;
+    color: var(--spectrum-global-color-gray-600);
+    transition: font-size 130ms ease-out, top 130ms ease-out,
+      transform 130ms ease-out;
+  }
+  div.placeholder {
+    top: 50%;
+    font-size: 15px;
+    transform: translateY(-50%);
+  }
+</style>

packages/bbui/src/FancyForm/FancyForm.svelte

@@ -0,0 +1,40 @@
+<script>
+  import { setContext } from "svelte"
+
+  let fields = {}
+
+  setContext("fancy-form", {
+    registerField: (id, api) => {
+      fields = { ...fields, [id]: api }
+    },
+    unregisterField: id => {
+      delete fields[id]
+      fields = fields
+    },
+  })
+
+  export const validate = () => {
+    let valid = true
+    Object.values(fields).forEach(api => {
+      if (!api.validate()) {
+        valid = false
+      }
+    })
+    return valid
+  }
+</script>
+
+<div class="fancy-form">
+  <slot />
+</div>
+
+<style>
+  .fancy-form :global(.fancy-field:not(:first-of-type)) {
+    border-top-left-radius: 0;
+    border-top-right-radius: 0;
+  }
+  .fancy-form :global(.fancy-field:not(:last-of-type)) {
+    border-bottom-left-radius: 0;
+    border-bottom-right-radius: 0;
+  }
+</style>

packages/bbui/src/FancyForm/FancyInput.svelte

@@ -0,0 +1,77 @@
+<script>
+  import { createEventDispatcher } from "svelte"
+  import FancyField from "./FancyField.svelte"
+  import FancyFieldLabel from "./FancyFieldLabel.svelte"
+  import { fade } from "svelte/transition"
+
+  export let label
+  export let value
+  export let type = "text"
+  export let disabled = false
+  export let error = null
+  export let validate = null
+  export let suffix = null
+
+  const dispatch = createEventDispatcher()
+
+  let focused = false
+  $: placeholder = !focused && !value
+
+  const onChange = e => {
+    const newValue = e.target.value
+    dispatch("change", newValue)
+    value = newValue
+    if (validate) {
+      error = validate(newValue)
+    }
+  }
+</script>
+
+<FancyField {error} {value} {validate} {disabled} {focused}>
+  {#if label}
+    <FancyFieldLabel {placeholder}>{label}</FancyFieldLabel>
+  {/if}
+  <input
+    {disabled}
+    value={value || ""}
+    type={type || "text"}
+    on:input={onChange}
+    on:focus={() => (focused = true)}
+    on:blur={() => (focused = false)}
+    class:placeholder
+  />
+  {#if suffix && !placeholder}
+    <div in:fade|local={{ duration: 130 }} class="suffix">{suffix}</div>
+  {/if}
+</FancyField>
+
+<style>
+  input {
+    width: 100%;
+    transition: transform 130ms ease-out;
+    transform: translateY(9px);
+    background: transparent;
+    font-size: 15px;
+    line-height: 17px;
+    font-family: var(--font-sans);
+    color: var(--spectrum-global-color-gray-900);
+    outline: none;
+    border: none;
+    padding: 0;
+    margin: 0;
+  }
+  input.placeholder {
+    transform: translateY(0);
+    position: absolute;
+    top: 0;
+    left: 0;
+    height: 100%;
+  }
+  .suffix {
+    color: var(--spectrum-global-color-gray-600);
+    transform: translateY(9px);
+    font-size: 15px;
+    line-height: 17px;
+    font-family: var(--font-sans);
+  }
+</style>

packages/bbui/src/FancyForm/FancySelect.svelte

@@ -0,0 +1,147 @@
+<script>
+  import { createEventDispatcher } from "svelte"
+  import FancyField from "./FancyField.svelte"
+  import Icon from "../Icon/Icon.svelte"
+  import Popover from "../Popover/Popover.svelte"
+  import FancyFieldLabel from "./FancyFieldLabel.svelte"
+
+  export let label
+  export let value
+  export let disabled = false
+  export let error = null
+  export let validate = null
+  export let options = []
+  export let getOptionLabel = option => extractProperty(option, "label")
+  export let getOptionValue = option => extractProperty(option, "value")
+
+  const dispatch = createEventDispatcher()
+
+  let open = false
+  let popover
+  let wrapper
+
+  $: placeholder = !value
+  $: selectedLabel = getSelectedLabel(value)
+
+  const extractProperty = (value, property) => {
+    if (value && typeof value === "object") {
+      return value[property]
+    }
+    return value
+  }
+
+  const onChange = newValue => {
+    dispatch("change", newValue)
+    value = newValue
+    if (validate) {
+      error = validate(newValue)
+    }
+    open = false
+  }
+
+  const getSelectedLabel = value => {
+    if (!value || !options?.length) {
+      return ""
+    }
+    const selectedOption = options.find(x => getOptionValue(x) === value)
+    if (!selectedOption) {
+      return value
+    }
+    return getOptionLabel(selectedOption)
+  }
+</script>
+
+<FancyField
+  bind:ref={wrapper}
+  {error}
+  {value}
+  {validate}
+  {disabled}
+  clickable
+  on:click={() => (open = true)}
+>
+  {#if label}
+    <FancyFieldLabel {placeholder}>{label}</FancyFieldLabel>
+  {/if}
+
+  <div class="value" class:placeholder>
+    {selectedLabel || ""}
+  </div>
+
+  <div class="arrow">
+    <Icon name="ChevronDown" />
+  </div>
+</FancyField>
+
+<Popover
+  anchor={wrapper}
+  align="left"
+  portalTarget={document.documentElement}
+  bind:this={popover}
+  {open}
+  on:close={() => (open = false)}
+  useAnchorWidth={true}
+  maxWidth={null}
+>
+  <div class="popover-content">
+    {#if options.length}
+      {#each options as option, idx}
+        <div
+          class="popover-option"
+          tabindex="0"
+          on:click={() => onChange(getOptionValue(option, idx))}
+        >
+          <span class="option-text">
+            {getOptionLabel(option, idx)}
+          </span>
+          {#if value === getOptionValue(option, idx)}
+            <Icon name="Checkmark" />
+          {/if}
+        </div>
+      {/each}
+    {/if}
+  </div>
+</Popover>
+
+<style>
+  .value {
+    display: block;
+    flex: 1 1 auto;
+    font-size: 15px;
+    line-height: 17px;
+    color: var(--spectrum-global-color-gray-900);
+    transition: transform 130ms ease-out, opacity 130ms ease-out;
+    opacity: 1;
+    white-space: nowrap;
+    text-overflow: ellipsis;
+    overflow: hidden;
+    width: 0;
+    transform: translateY(9px);
+  }
+  .value.placeholder {
+    transform: translateY(0);
+    opacity: 0;
+    pointer-events: none;
+    margin-top: 0;
+  }
+  .popover-content {
+    display: flex;
+    flex-direction: column;
+    justify-content: flex-start;
+    align-items: stretch;
+    padding: 7px 0;
+  }
+  .popover-option {
+    display: flex;
+    flex-direction: row;
+    justify-content: space-between;
+    align-items: center;
+    padding: 7px 16px;
+    transition: background 130ms ease-out;
+    font-size: 15px;
+  }
+  .popover-option:hover {
+    background: var(--spectrum-global-color-gray-200);
+    cursor: pointer;
+  }
+</style>

packages/bbui/src/FancyForm/index.js

@@ -0,0 +1,6 @@
+export { default as FancyInput } from "./FancyInput.svelte"
+export { default as FancyCheckbox } from "./FancyCheckbox.svelte"
+export { default as FancySelect } from "./FancySelect.svelte"
+export { default as FancyButton } from "./FancyButton.svelte"
+export { default as FancyForm } from "./FancyForm.svelte"
+export { default as FancyButtonRadio } from "./FancyButtonRadio.svelte"

packages/bbui/src/Form/Core/EnvDropdown.svelte

@@ -0,0 +1,282 @@
+<script>
+  import "@spectrum-css/textfield/dist/index-vars.css"
+  import { createEventDispatcher, onMount } from "svelte"
+  import clickOutside from "../../Actions/click_outside"
+  import Divider from "../../Divider/Divider.svelte"
+
+  export let value = null
+  export let placeholder = null
+  export let type = "text"
+  export let disabled = false
+  export let id = null
+  export let readonly = false
+  export let updateOnChange = true
+  export let dataCy
+  export let align
+  export let autofocus = false
+  export let variables
+  export let showModal
+  export let environmentVariablesEnabled
+  export let handleUpgradePanel
+  const dispatch = createEventDispatcher()
+
+  let field
+  let focus = false
+  let iconFocused = false
+  let open = false
+
+  //eslint-disable-next-line
+  const STRIP_NAME_REGEX = /(?<=\.)(.*?)(?=\ })/g
+
+  // Strips the name out of the value which is {{ env.Variable }} resulting in an array like ["Variable"]
+  $: hbsValue = String(value)?.match(STRIP_NAME_REGEX) || []
+
+  const updateValue = newValue => {
+    if (readonly) {
+      return
+    }
+    if (type === "number") {
+      const float = parseFloat(newValue)
+      newValue = isNaN(float) ? null : float
+    }
+    dispatch("change", newValue)
+  }
+
+  const onFocus = () => {
+    if (readonly) {
+      return
+    }
+    focus = true
+  }
+
+  const onBlur = event => {
+    if (readonly) {
+      return
+    }
+    focus = false
+    updateValue(event.target.value)
+  }
+
+  const onInput = event => {
+    if (readonly || !updateOnChange) {
+      return
+    }
+    updateValue(event.target.value)
+  }
+
+  const handleOutsideClick = event => {
+    if (open) {
+      event.stopPropagation()
+      open = false
+      focus = false
+      iconFocused = false
+      dispatch("closed")
+    }
+  }
+
+  const handleVarSelect = variable => {
+    open = false
+    focus = false
+    iconFocused = false
+    updateValue(`{{ env.${variable} }}`)
+  }
+
+  onMount(() => {
+    focus = autofocus
+    if (focus) field.focus()
+  })
+
+  function removeVariable() {
+    updateValue("")
+  }
+
+  function openPopover() {
+    open = true
+    focus = true
+    iconFocused = true
+  }
+</script>
+
+<div class="spectrum-InputGroup">
+  <div
+    class:is-disabled={disabled || hbsValue.length}
+    class:is-focused={focus}
+    class="spectrum-Textfield"
+  >
+    <svg
+      class:close-color={hbsValue.length}
+      class:focused={iconFocused}
+      class="hoverable icon-position spectrum-Icon spectrum-Icon--sizeS spectrum-Textfield-validationIcon"
+      focusable="false"
+      aria-hidden="true"
+      on:click={() => {
+        hbsValue.length ? removeVariable() : openPopover()
+      }}
+    >
+      <use
+        xlink:href={`#spectrum-icon-18-${!hbsValue.length ? "Key" : "Close"}`}
+      />
+    </svg>
+
+    <input
+      bind:this={field}
+      disabled={hbsValue.length || disabled}
+      {readonly}
+      {id}
+      data-cy={dataCy}
+      value={hbsValue.length ? `{{ ${hbsValue[0]} }}` : value}
+      placeholder={placeholder || ""}
+      on:click
+      on:blur
+      on:focus
+      on:input
+      on:keyup
+      on:blur={onBlur}
+      on:focus={onFocus}
+      on:input={onInput}
+      {type}
+      style={align ? `text-align: ${align};` : ""}
+      class="spectrum-Textfield-input"
+      inputmode={type === "number" ? "decimal" : "text"}
+    />
+  </div>
+  {#if open}
+    <div
+      use:clickOutside={handleOutsideClick}
+      class="spectrum-Popover spectrum-Popover--bottom spectrum-Picker-popover is-open"
+    >
+      <ul class="spectrum-Menu" role="listbox">
+        {#if !environmentVariablesEnabled}
+          <div class="no-variables-text primary-text">
+            Upgrade your plan to get environment variables
+          </div>
+        {:else if variables.length}
+          <div style="max-height: 100px">
+            {#each variables as variable, idx}
+              <li
+                class="spectrum-Menu-item"
+                role="option"
+                aria-selected="true"
+                tabindex="0"
+                on:click={() => handleVarSelect(variable.name)}
+              >
+                <span class="spectrum-Menu-itemLabel">
+                  <div class="primary-text">
+                    {variable.name}
+                    <span />
+                  </div>
+                  <svg
+                    class="spectrum-Icon spectrum-UIIcon-Checkmark100 spectrum-Menu-checkmark spectrum-Menu-itemIcon"
+                    focusable="false"
+                    aria-hidden="true"
+                  >
+                    <use xlink:href="#spectrum-css-icon-Checkmark100" />
+                  </svg>
+                </span>
+              </li>
+            {/each}
+          </div>
+        {:else}
+          <div class="no-variables-text primary-text">
+            You don't have any environment variables yet
+          </div>
+        {/if}
+      </ul>
+      <Divider noMargin />
+      {#if environmentVariablesEnabled}
+        <div on:click={() => showModal()} class="add-variable">
+          <svg
+            class="spectrum-Icon spectrum-Icon--sizeS "
+            focusable="false"
+            aria-hidden="true"
+          >
+            <use xlink:href="#spectrum-icon-18-Add" />
+          </svg>
+          <div class="primary-text">Add Variable</div>
+        </div>
+      {:else}
+        <div on:click={() => handleUpgradePanel()} class="add-variable">
+          <svg
+            class="spectrum-Icon spectrum-Icon--sizeS "
+            focusable="false"
+            aria-hidden="true"
+          >
+            <use xlink:href="#spectrum-icon-18-ArrowUp" />
+          </svg>
+          <div class="primary-text">Upgrade plan</div>
+        </div>
+      {/if}
+    </div>
+  {/if}
+</div>
+
+<style>
+  .spectrum-Textfield {
+    width: 100%;
+  }
+
+  .icon-position {
+    position: absolute;
+    top: 25%;
+    right: 2%;
+  }
+
+  .hoverable:hover {
+    cursor: pointer;
+    color: var(--spectrum-global-color-blue-400);
+  }
+
+  .primary-text {
+    white-space: nowrap;
+    overflow: hidden;
+    text-overflow: ellipsis;
+  }
+
+  .spectrum-InputGroup {
+    min-width: 0;
+    width: 100%;
+  }
+
+  .spectrum-Popover {
+    max-height: 240px;
+    z-index: 999;
+    top: 100%;
+  }
+
+  .spectrum-Popover.spectrum-Popover--bottom.spectrum-Picker-popover.is-open {
+    width: 100%;
+  }
+
+  .no-variables-height {
+    height: 100px;
+  }
+
+  .no-variables-text {
+    padding: var(--spacing-m);
+    color: var(--spectrum-global-color-gray-600);
+  }
+
+  .add-variable {
+    display: flex;
+    padding: var(--spacing-m) 0 var(--spacing-m) var(--spacing-m);
+    align-items: center;
+    gap: var(--spacing-s);
+    cursor: pointer;
+  }
+
+  .focused {
+    color: var(--spectrum-global-color-blue-400);
+  }
+
+  .add-variable:hover {
+    background: var(--grey-1);
+  }
+
+  .close-color {
+    color: var(--spectrum-global-color-gray-900) !important;
+  }
+
+  .close-color:hover {
+    color: var(--spectrum-global-color-blue-400) !important;
+  }
+</style>

packages/bbui/src/Form/Core/Select.svelte

@@ -18,8 +18,11 @@
   export let autoWidth = false
   export let autocomplete = false
   export let sort = false
+
   const dispatch = createEventDispatcher()
+
   let open = false
+
   $: fieldText = getFieldText(value, options, placeholder)
   $: fieldIcon = getFieldAttribute(getOptionIcon, value, options)
   $: fieldColour = getFieldAttribute(getOptionColour, value, options)

packages/bbui/src/Form/EnvDropdown.svelte

@@ -0,0 +1,52 @@
+<script>
+  import Field from "./Field.svelte"
+  import EnvDropdown from "./Core/EnvDropdown.svelte"
+  import { createEventDispatcher } from "svelte"
+
+  export let value = null
+  export let label = null
+  export let labelPosition = "above"
+  export let placeholder = null
+  export let type = "text"
+  export let disabled = false
+  export let readonly = false
+  export let error = null
+  export let updateOnChange = true
+  export let quiet = false
+  export let dataCy
+  export let autofocus
+  export let variables
+  export let showModal
+  export let environmentVariablesEnabled
+  export let handleUpgradePanel
+  const dispatch = createEventDispatcher()
+  const onChange = e => {
+    value = e.detail
+    dispatch("change", e.detail)
+  }
+</script>
+
+<Field {label} {labelPosition} {error}>
+  <EnvDropdown
+    {dataCy}
+    {updateOnChange}
+    {error}
+    {disabled}
+    {readonly}
+    {value}
+    {placeholder}
+    {type}
+    {quiet}
+    {autofocus}
+    {variables}
+    {showModal}
+    {environmentVariablesEnabled}
+    {handleUpgradePanel}
+    on:change={onChange}
+    on:click
+    on:input
+    on:blur
+    on:focus
+    on:keyup
+  />
+</Field>

packages/bbui/src/Layout/Page.svelte

@@ -18,6 +18,7 @@
   <div class="main">
     <div class="content" class:wide class:noPadding class:narrow>
       <slot />
+      <div class="fix-scroll-padding" />
     </div>
   </div>
   <div
@@ -55,9 +56,14 @@
     max-width: 1080px;
     margin: 0 auto;
     flex: 1 1 auto;
-    padding: 50px;
+    padding: 50px 50px 0 50px;
     z-index: 1;
   }
+  .fix-scroll-padding {
+    content: "";
+    display: block;
+    flex: 0 0 50px;
+  }
   .content.wide {
     max-width: none;
   }

packages/bbui/src/Link/Link.svelte

@@ -1,5 +1,6 @@
 <script>
   import "@spectrum-css/link/dist/index-vars.css"
+  import { createEventDispatcher } from "svelte"
 
   export let href = "#"
   export let size = "M"
@@ -9,10 +10,12 @@
   export let overBackground = false
   export let target
   export let download
+
+  const dispatch = createEventDispatcher()
 </script>
 
 <a
-  on:click
+  on:click={e => dispatch("click") && e.stopPropagation()}
   {href}
   {target}
   {download}

packages/bbui/src/Popover/Popover.svelte

@@ -68,7 +68,10 @@
     <div
       tabindex="0"
       use:positionDropdown={{ anchor, align, maxWidth, useAnchorWidth }}
-      use:clickOutside={handleOutsideClick}
+      use:clickOutside={{
+        callback: handleOutsideClick,
+        anchor,
+      }}
       on:keydown={handleEscape}
       class={"spectrum-Popover is-open " + (tooltipClasses || "")}
       role="presentation"
@@ -88,6 +91,7 @@
   .spectrum-Popover {
     min-width: var(--spectrum-global-dimension-size-2000);
     border-color: var(--spectrum-global-color-gray-300);
+    overflow: auto;
   }
   .spectrum-Popover.is-open.spectrum-Popover--withTip {
     margin-top: var(--spacing-xs);

packages/bbui/src/index.js

@@ -27,6 +27,7 @@ export { default as RadioGroup } from "./Form/RadioGroup.svelte"
 export { default as Checkbox } from "./Form/Checkbox.svelte"
 export { default as InputDropdown } from "./Form/InputDropdown.svelte"
 export { default as PickerDropdown } from "./Form/PickerDropdown.svelte"
+export { default as EnvDropdown } from "./Form/EnvDropdown.svelte"
 export { default as DetailSummary } from "./DetailSummary/DetailSummary.svelte"
 export { default as Popover } from "./Popover/Popover.svelte"
 export { default as ProgressBar } from "./ProgressBar/ProgressBar.svelte"
@@ -75,6 +76,7 @@ export { default as ListItem } from "./List/ListItem.svelte"
 export { default as IconSideNav } from "./IconSideNav/IconSideNav.svelte"
 export { default as IconSideNavItem } from "./IconSideNav/IconSideNavItem.svelte"
 export { default as Slider } from "./Form/Slider.svelte"
+export { default as Accordion } from "./Accordion/Accordion.svelte"
 
 // Renderers
 export { default as BoldRenderer } from "./Table/BoldRenderer.svelte"
@@ -101,3 +103,6 @@ export { banner, BANNER_TYPES } from "./Stores/banner"
 
 // Helpers
 export * as Helpers from "./helpers"
+
+// Fancy form components
+export * from "./FancyForm"

packages/bbui/yarn.lock

@@ -109,6 +109,11 @@
     estree-walker "^1.0.1"
     picomatch "^2.2.2"
 
+"@spectrum-css/accordion@3.0.24":
+  version "3.0.24"
+  resolved "https://registry.yarnpkg.com/@spectrum-css/accordion/-/accordion-3.0.24.tgz#f89066c120c57b0cfc9aba66d60c39fc1cf69f74"
+  integrity sha512-jNOmUsxmiT3lRLButnN5KKHM94fd+87fjiF8L0c4uRNgJl6ZsBuxPXrM15lV4y1f8D2IACAw01/ZkGRAeaCOFA==
+
 "@spectrum-css/actionbutton@1.0.1":
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/@spectrum-css/actionbutton/-/actionbutton-1.0.1.tgz#9c75da37ea6915919fb574c74bd60dacc03b6577"

packages/builder/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/builder",
-  "version": "2.2.12-alpha.32",
+  "version": "2.2.12-alpha.47",
   "license": "GPL-3.0",
   "private": true,
   "scripts": {
@@ -71,11 +71,12 @@
     }
   },
   "dependencies": {
-    "@budibase/bbui": "2.2.12-alpha.32",
+    "@budibase/bbui": "2.2.12-alpha.47",
-    "@budibase/client": "2.2.12-alpha.32",
+    "@budibase/client": "2.2.12-alpha.47",
-    "@budibase/frontend-core": "2.2.12-alpha.32",
+    "@budibase/frontend-core": "2.2.12-alpha.47",
-    "@budibase/string-templates": "2.2.12-alpha.32",
+    "@budibase/string-templates": "2.2.12-alpha.47",
     "@sentry/browser": "5.19.1",
+    "@spectrum-css/accordion": "^3.0.24",
     "@spectrum-css/page": "^3.0.1",
     "@spectrum-css/vars": "^3.0.1",
     "codemirror": "^5.59.0",

packages/builder/src/builderStore/dataBinding.js

@@ -21,6 +21,7 @@ import {
 import { TableNames } from "../constants"
 import { JSONUtils } from "@budibase/frontend-core"
 import ActionDefinitions from "components/design/settings/controls/ButtonActionEditor/manifest.json"
+import { environment, licensing } from "stores/portal"
 
 // Regex to match all instances of template strings
 const CAPTURE_VAR_INSIDE_TEMPLATE = /{{([^}]+)}}/g
@@ -53,8 +54,13 @@ export const getBindableProperties = (asset, componentId) => {
  * Gets all rest bindable data fields
  */
 export const getRestBindings = () => {
+  const environmentVariablesEnabled = get(licensing).environmentVariablesEnabled
   const userBindings = getUserBindings()
-  return [...userBindings, ...getAuthBindings()]
+  return [
+    ...userBindings,
+    ...getAuthBindings(),
+    ...(environmentVariablesEnabled ? getEnvironmentBindings() : []),
+  ]
 }
 
 /**
@@ -89,6 +95,20 @@ export const getAuthBindings = () => {
   return bindings
 }
 
+export const getEnvironmentBindings = () => {
+  let envVars = get(environment).variables
+  return envVars.map(variable => {
+    return {
+      type: "context",
+      runtimeBinding: `env.${makePropSafe(variable.name)}`,
+      readableBinding: `env.${variable.name}`,
+      category: "Environment",
+      icon: "Key",
+      display: { type: "string", name: variable.name },
+    }
+  })
+}
+
 /**
  * Utility - convert a key/value map to an array of custom 'context' bindings
  * @param {object} valueMap Key/value pairings

packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte

@@ -18,6 +18,7 @@
 
   import { automationStore } from "builderStore"
   import { tables } from "stores/backend"
+  import { environment, licensing } from "stores/portal"
   import WebhookDisplay from "../Shared/WebhookDisplay.svelte"
   import DrawerBindableInput from "../../common/bindings/DrawerBindableInput.svelte"
   import AutomationBindingPanel from "../../common/bindings/ServerBindingPanel.svelte"
@@ -33,6 +34,7 @@
   import { Utils } from "@budibase/frontend-core"
   import { TriggerStepID, ActionStepID } from "constants/backend/automations"
   import { cloneDeep } from "lodash/fp"
+  import { onMount } from "svelte"
 
   export let block
   export let testData
@@ -166,6 +168,24 @@
       )
     }
 
+    // Environment bindings
+    if ($licensing.environmentVariablesEnabled) {
+      bindings = bindings.concat(
+        $environment.variables.map(variable => {
+          return {
+            label: `env.${variable.name}`,
+            path: `env.${variable.name}`,
+            icon: "Key",
+            category: "Environment",
+            display: {
+              type: "string",
+              name: variable.name,
+            },
+          }
+        })
+      )
+    }
+
     return bindings
   }
 
@@ -196,6 +216,14 @@
     onChange({ detail: tempFilters }, defKey)
     drawer.hide()
   }
+
+  onMount(async () => {
+    try {
+      await environment.loadVariables()
+    } catch (error) {
+      console.error(error)
+    }
+  })
 </script>
 
 <div class="fields">

packages/builder/src/components/backend/DatasourceNavigator/DatasourceNavigator.svelte

@@ -20,7 +20,8 @@
     $isActive,
     $tables,
     $queries,
-    $views
+    $views,
+    openDataSources
   )
   $: openDataSource = enrichedDataSources.find(x => x.open)
   $: {
@@ -36,7 +37,8 @@
     isActive,
     tables,
     queries,
-    views
+    views,
+    openDataSources
   ) => {
     if (!datasources?.list?.length) {
       return []

packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/IntegrationConfigForm.svelte

@@ -6,16 +6,26 @@
     Toggle,
     Button,
     TextArea,
+    Modal,
+    EnvDropdown,
+    Accordion,
+    notifications,
   } from "@budibase/bbui"
   import KeyValueBuilder from "components/integration/KeyValueBuilder.svelte"
   import { capitalise } from "helpers"
   import { IntegrationTypes } from "constants/backend"
   import { createValidationStore } from "helpers/validation/yup"
-  import { createEventDispatcher } from "svelte"
+  import { createEventDispatcher, onMount } from "svelte"
+  import { environment, licensing, auth } from "stores/portal"
+  import CreateEditVariableModal from "components/portal/environment/CreateEditVariableModal.svelte"
 
   export let datasource
   export let schema
   export let creating
+
+  let createVariableModal
+  let selectedKey
+
   const validation = createValidationStore()
   const dispatch = createEventDispatcher()
 
@@ -51,15 +61,55 @@
 
   let addButton
 
-  function getDisplayName(key) {
+  function getDisplayName(key, fieldKey) {
     let name
-    if (schema[key]?.display) {
+    if (fieldKey && schema[key]["fields"][fieldKey]?.display) {
+      name = schema[key]["fields"][fieldKey].display
+    } else if (fieldKey) {
+      name = fieldKey
+    } else if (schema[key]?.display) {
       name = schema[key].display
     } else {
       name = key
     }
     return capitalise(name)
   }
+  function getFieldGroupKeys(fieldGroup) {
+    return Object.entries(schema[fieldGroup].fields || {})
+      .filter(el => filter(el))
+      .map(([key]) => key)
+  }
+
+  async function save(data) {
+    try {
+      await environment.createVariable(data)
+      config[selectedKey] = `{{ env.${data.name} }}`
+      createVariableModal.hide()
+    } catch (err) {
+      notifications.error(`Failed to create variable: ${err.message}`)
+    }
+  }
+
+  function showModal(configKey) {
+    selectedKey = configKey
+    createVariableModal.show()
+  }
+
+  async function handleUpgradePanel() {
+    await environment.upgradePanelOpened()
+    $licensing.goToUpgradePage()
+  }
+
+  onMount(async () => {
+    try {
+      await environment.loadVariables()
+      if ($auth.user) {
+        await licensing.init()
+      }
+    } catch (err) {
+      console.error(err)
+    }
+  })
 </script>
 
 <form>
@@ -100,14 +150,39 @@
             error={$validation.errors[configKey]}
           />
         </div>
+      {:else if schema[configKey].type === "fieldGroup"}
+        <Accordion
+          itemName={configKey}
+          initialOpen={getFieldGroupKeys(configKey).some(
+            fieldKey => !!config[fieldKey]
+          )}
+          header={getDisplayName(configKey)}
+        >
+          <Layout gap="S">
+            {#each getFieldGroupKeys(configKey) as fieldKey}
+              <div class="form-row">
+                <Label>{getDisplayName(configKey, fieldKey)}</Label>
+                <Input
+                  type={schema[configKey]["fields"][fieldKey]?.type}
+                  on:change
+                  bind:value={config[fieldKey]}
+                />
+              </div>
+            {/each}
+          </Layout>
+        </Accordion>
       {:else}
         <div class="form-row">
           <Label>{getDisplayName(configKey)}</Label>
-          <Input
+          <EnvDropdown
+            showModal={() => showModal(configKey)}
+            variables={$environment.variables}
             type={schema[configKey].type}
             on:change
             bind:value={config[configKey]}
             error={$validation.errors[configKey]}
+            environmentVariablesEnabled={$licensing.environmentVariablesEnabled}
+            {handleUpgradePanel}
           />
         </div>
       {/if}
@@ -115,6 +190,10 @@
   </Layout>
 </form>
 
+<Modal bind:this={createVariableModal}>
+  <CreateEditVariableModal {save} />
+</Modal>
+
 <style>
   .form-row {
     display: grid;

packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/rest/RestExtraConfigForm.svelte

@@ -12,10 +12,12 @@
   import ViewDynamicVariables from "./variables/ViewDynamicVariables.svelte"
   import {
     getRestBindings,
+    getEnvironmentBindings,
     readableToRuntimeBinding,
     runtimeToReadableMap,
   } from "builderStore/dataBinding"
   import { cloneDeep } from "lodash/fp"
+  import { licensing } from "stores/portal"
 
   export let datasource
   export let queries
@@ -93,6 +95,9 @@
     headings
     bind:object={datasource.config.staticVariables}
     on:change
+    bindings={$licensing.environmentVariablesEnabled
+      ? getEnvironmentBindings()
+      : []}
   />
 </Layout>
 <div />

packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/rest/auth/RestAuthenticationModal.svelte

@@ -1,9 +1,23 @@
 <script>
   import { onMount } from "svelte"
-  import { ModalContent, Layout, Select, Body, Input } from "@budibase/bbui"
+  import {
+    ModalContent,
+    Layout,
+    Select,
+    Body,
+    Input,
+    EnvDropdown,
+    Modal,
+    notifications,
+  } from "@budibase/bbui"
   import { AUTH_TYPE_LABELS, AUTH_TYPES } from "./authTypes"
   import BindableCombobox from "components/common/bindings/BindableCombobox.svelte"
-  import { getAuthBindings } from "builderStore/dataBinding"
+  import {
+    getAuthBindings,
+    getEnvironmentBindings,
+  } from "builderStore/dataBinding"
+  import { environment, licensing, auth } from "stores/portal"
+  import CreateEditVariableModal from "components/portal/environment/CreateEditVariableModal.svelte"
 
   export let configs
   export let currentConfig
@@ -28,7 +42,19 @@
   let hasErrors = false
   let hasChanged = false
 
-  onMount(() => {
+  let createVariableModal
+  let formFieldkey
+
+  onMount(async () => {
+    try {
+      await environment.loadVariables()
+      if ($auth.user) {
+        await licensing.init()
+      }
+    } catch (err) {
+      console.error(err)
+    }
+
     if (currentConfig) {
       deconstructConfig()
     }
@@ -146,6 +172,16 @@
     }
   }
 
+  const save = async data => {
+    try {
+      await environment.createVariable(data)
+      form.basic[formFieldkey] = `{{ env.${data.name} }}`
+      createVariableModal.hide()
+    } catch (err) {
+      notifications.error(`Failed to create variable: ${err.message}`)
+    }
+  }
+
   const onFieldChange = () => {
     checkErrors()
     checkChanged()
@@ -154,6 +190,16 @@
   const onConfirmInternal = () => {
     onConfirm(constructConfig())
   }
+
+  async function handleUpgradePanel() {
+    await environment.upgradePanelOpened()
+    $licensing.goToUpgradePage()
+  }
+
+  function showModal(key) {
+    formFieldkey = key
+    createVariableModal.show()
+  }
 </script>
 
 <ModalContent
@@ -189,26 +235,39 @@
       error={blurred.type ? errors.type : null}
     />
     {#if form.type === AUTH_TYPES.BASIC}
-      <Input
+      <EnvDropdown
         label="Username"
         bind:value={form.basic.username}
         on:change={onFieldChange}
         on:blur={() => (blurred.basic.username = true)}
         error={blurred.basic.username ? errors.basic.username : null}
+        showModal={() => showModal("configKey")}
+        variables={$environment.variables}
+        environmentVariablesEnabled={$licensing.environmentVariablesEnabled}
+        {handleUpgradePanel}
       />
-      <Input
+      <EnvDropdown
         label="Password"
         bind:value={form.basic.password}
         on:change={onFieldChange}
         on:blur={() => (blurred.basic.password = true)}
         error={blurred.basic.password ? errors.basic.password : null}
+        showModal={() => showModal("configKey")}
+        variables={$environment.variables}
+        environmentVariablesEnabled={$licensing.environmentVariablesEnabled}
+        {handleUpgradePanel}
       />
     {/if}
     {#if form.type === AUTH_TYPES.BEARER}
       <BindableCombobox
         label="Token"
         value={form.bearer.token}
-        bindings={getAuthBindings()}
+        bindings={[
+          ...getAuthBindings(),
+          ...($licensing.environmentVariablesEnabled
+            ? getEnvironmentBindings()
+            : []),
+        ]}
         on:change={e => {
           form.bearer.token = e.detail
           onFieldChange()
@@ -226,3 +285,7 @@
     {/if}
   </Layout>
 </ModalContent>
+
+<Modal bind:this={createVariableModal}>
+  <CreateEditVariableModal {save} />
+</Modal>

packages/builder/src/components/backend/Datasources/CreateEditRelationship.svelte

@@ -19,6 +19,8 @@
   export let close
 
   const colNotSet = "Please specify a column name"
+  const relationshipAlreadyExists =
+    "A relationship between these tables already exists."
   const relationshipTypes = [
     {
       label: "One to Many",
@@ -154,6 +156,10 @@
     if (!isMany && !fromPrimary) {
       errObj.fromPrimary = "Please pick the primary key"
     }
+    if (isMany && relationshipExists()) {
+      errObj.fromTable = relationshipAlreadyExists
+      errObj.toTable = relationshipAlreadyExists
+    }
 
     // currently don't support relationships back onto the table itself, needs to relate out
     const tableError = "From/to/through tables must be different"
@@ -271,6 +277,35 @@
     toRelationship = relateTo
   }
 
+  function relationshipExists() {
+    if (
+      originalFromTable &&
+      originalToTable &&
+      originalFromTable === fromTable &&
+      originalToTable === toTable
+    ) {
+      return false
+    }
+    let fromThroughLinks = Object.values(
+      datasource.entities[fromTable.name].schema
+    ).filter(value => value.through)
+    let toThroughLinks = Object.values(
+      datasource.entities[toTable.name].schema
+    ).filter(value => value.through)
+
+    const matchAgainstUserInput = (fromTableId, toTableId) =>
+      (fromTableId === fromId && toTableId === toId) ||
+      (fromTableId === toId && toTableId === fromId)
+
+    return !!fromThroughLinks.find(from =>
+      toThroughLinks.find(
+        to =>
+          from.through === to.through &&
+          matchAgainstUserInput(from.tableId, to.tableId)
+      )
+    )
+  }
+
   function removeExistingRelationship() {
     if (originalFromTable && originalFromColumnName) {
       delete datasource.entities[originalFromTable.name].schema[
@@ -332,8 +367,13 @@
     bind:error={errors.fromTable}
     on:change={e => {
       fromColumn = tableOptions.find(opt => opt.value === e.detail)?.label || ""
+      if (errors.fromTable === relationshipAlreadyExists) {
+        errors.toColumn = null
+      }
       errors.fromTable = null
       errors.fromColumn = null
+      errors.toTable = null
+      errors.throughTable = null
     }}
   />
   {#if isManyToOne && fromTable}
@@ -352,8 +392,13 @@
     bind:error={errors.toTable}
     on:change={e => {
       toColumn = tableOptions.find(opt => opt.value === e.detail)?.label || ""
+      if (errors.toTable === relationshipAlreadyExists) {
+        errors.fromColumn = null
+      }
       errors.toTable = null
       errors.toColumn = null
+      errors.fromTable = null
+      errors.throughTable = null
     }}
   />
   {#if isManyToMany}
@@ -362,6 +407,11 @@
       options={tableOptions}
       bind:value={throughId}
       bind:error={errors.throughTable}
+      on:change={() => {
+        errors.fromTable = null
+        errors.toTable = null
+        errors.throughTable = null
+      }}
     />
     {#if fromTable && toTable && throughTable}
       <Select

packages/builder/src/components/integration/RestQueryViewer.svelte

@@ -1,6 +1,7 @@
 <script>
   import { goto, params } from "@roxi/routify"
   import { datasources, flags, integrations, queries } from "stores/backend"
+  import { environment } from "stores/portal"
   import {
     Banner,
     Body,
@@ -362,6 +363,13 @@
       notifications.error("Error getting datasources")
     }
 
+    try {
+      // load the environment variables
+      await environment.loadVariables()
+    } catch (error) {
+      notifications.error(`Error getting environment variables - ${error}`)
+    }
+
     datasource = $datasources.list.find(ds => ds._id === query?.datasourceId)
     const datasourceUrl = datasource?.config.url
     const qs = query?.fields.queryString

packages/builder/src/components/portal/environment/CreateEditVariableModal.svelte

@@ -0,0 +1,100 @@
+<script>
+  import {
+    ModalContent,
+    Button,
+    Input,
+    Checkbox,
+    Heading,
+    notifications,
+    Context,
+  } from "@budibase/bbui"
+  import { environment } from "stores/portal"
+  import ConfirmDialog from "components/common/ConfirmDialog.svelte"
+  import { getContext } from "svelte"
+  const modalContext = getContext(Context.Modal)
+
+  export let save
+  export let row
+
+  let deleteDialog
+  let name = row?.name || ""
+  let productionValue
+  let developmentValue
+  let useProductionValue = true
+
+  const deleteVariable = async name => {
+    try {
+      await environment.deleteVariable(name)
+      modalContext.hide()
+      notifications.success("Environment variable deleted")
+    } catch (err) {
+      notifications.error(err.message)
+    }
+  }
+
+  const saveVariable = async () => {
+    try {
+      await save({
+        name,
+        production: productionValue,
+        development: developmentValue,
+      })
+      notifications.success("Environment variable saved")
+    } catch (err) {
+      notifications.error(`Error saving environment variable - ${err.message}`)
+    }
+  }
+</script>
+
+<ModalContent
+  onConfirm={() => saveVariable()}
+  title={!row ? "Add new environment variable" : "Edit environment variable"}
+>
+  <Input disabled={row} label="Name" bind:value={name} />
+  <div>
+    <Heading size="XS">Production</Heading>
+    <Input
+      type="password"
+      label="Value"
+      on:change={e => {
+        productionValue = e.detail
+        if (useProductionValue) {
+          developmentValue = e.detail
+        }
+      }}
+      value={productionValue}
+    />
+  </div>
+  <div>
+    <Heading size="XS">Development</Heading>
+    <Input
+      type="password"
+      on:change={e => {
+        developmentValue = e.detail
+      }}
+      disabled={useProductionValue}
+      label="Value"
+      value={useProductionValue ? productionValue : developmentValue}
+    />
+    <Checkbox bind:value={useProductionValue} text="Use production value" />
+  </div>
+
+  <div class="footer" slot="footer">
+    {#if row}
+      <Button on:click={deleteDialog.show} warning>Delete</Button>
+    {/if}
+  </div>
+</ModalContent>
+
+<ConfirmDialog
+  bind:this={deleteDialog}
+  onOk={() => {
+    deleteVariable(row.name)
+  }}
+  okText="Delete Environment Variable"
+  title="Confirm Deletion"
+>
+  Are you sure you wish to delete the environment variable
+  <i>{row.name}?</i>
+  This action cannot be undone.
+</ConfirmDialog>

packages/builder/src/components/portal/page/SideNavItem.svelte

@@ -5,7 +5,7 @@
 </script>
 
 <a on:click href={url} class:active>
-  {text}
+  {text || ""}
 </a>
 
 <style>

packages/builder/src/components/start/ExportAppModal.svelte

@@ -1,5 +1,6 @@
 <script>
-  import { ModalContent, Toggle, Body } from "@budibase/bbui"
+  import { ModalContent, Toggle, Body, InlineAlert } from "@budibase/bbui"
+  import { licensing } from "stores/portal"
 
   export let app
   export let published
@@ -16,6 +17,11 @@
 </script>
 
 <ModalContent {title} {confirmText} onConfirm={exportApp}>
+  {#if licensing.environmentVariablesEnabled}
+    <InlineAlert
+      header="Do not share your budibase application exports publicly as they may contain sensitive information such as database credentials or secret keys."
+    />
+  {/if}
   <Body
     >Apps can be exported with or without data that is within internal tables -
     select this below.</Body

packages/builder/src/pages/builder/portal/_components/UpgradeButton.svelte

@@ -10,7 +10,7 @@
     size="S"
     on:click
     on:click={() => {
-      $goto($admin.accountPortalUrl + "/portal/upgrade")
+      window.open($admin.accountPortalUrl + "/portal/upgrade", "_blank")
     }}
   >
     Upgrade

packages/builder/src/pages/builder/portal/_layout.svelte

@@ -98,7 +98,7 @@
   /* Customise tabs appearance*/
   .nav :global(.spectrum-Tabs) {
     margin-bottom: -2px;
-    padding: 7px 0;
+    padding: 5px 0;
     flex: 1 1 auto;
   }
   .nav :global(.spectrum-Tabs-content) {

packages/builder/src/pages/builder/portal/account/_layout.svelte

@@ -1,14 +1,15 @@
 <script>
-  import { isActive } from "@roxi/routify"
+  import { goto, isActive } from "@roxi/routify"
   import { Page } from "@budibase/bbui"
   import { Content, SideNav, SideNavItem } from "components/portal/page"
   import { menu } from "stores/portal"
 
-  $: pages = $menu.find(x => x.title === "Account").subPages
+  $: pages = $menu.find(x => x.title === "Account")?.subPages || []
+  $: !pages.length && $goto("../")
 </script>
 
-<Page narrow>
+<Page>
-  <Content>
+  <Content narrow>
     <div slot="side-nav">
       <SideNav>
         {#each pages as { title, href }}

packages/builder/src/pages/builder/portal/apps/index.svelte

@@ -227,7 +227,7 @@
       {/each}
       <div class="title">
         <div class="welcome">
-          <Layout noPadding gap="S">
+          <Layout noPadding gap="XS">
             <Heading size="L">{welcomeHeader}</Heading>
             <Body size="M">
               Manage your apps and get a head start with templates

packages/builder/src/pages/builder/portal/settings/_layout.svelte

@@ -1,11 +1,12 @@
 <script>
-  import { isActive } from "@roxi/routify"
+  import { goto, isActive } from "@roxi/routify"
   import { Page } from "@budibase/bbui"
   import { Content, SideNav, SideNavItem } from "components/portal/page"
   import { menu } from "stores/portal"
 
   $: wide = $isActive("./email/:template")
-  $: pages = $menu.find(x => x.title === "Settings").subPages
+  $: pages = $menu.find(x => x.title === "Settings")?.subPages || []
+  $: !pages.length && $goto("../")
 </script>
 
 <Page>

packages/builder/src/pages/builder/portal/settings/environment/_components/EditVariableColumn.svelte

@@ -0,0 +1,35 @@
+<script>
+  import { ActionButton, Modal } from "@budibase/bbui"
+  import ConfirmDialog from "components/common/ConfirmDialog.svelte"
+  import { environment } from "stores/portal"
+  import CreateEditVariableModal from "components/portal/environment/CreateEditVariableModal.svelte"
+
+  export let row
+
+  let editVariableModal
+  let deleteDialog
+
+  const save = async data => {
+    await environment.updateVariable(data)
+    editVariableModal.hide()
+  }
+</script>
+
+<ActionButton size="S" on:click={editVariableModal.show}>Edit</ActionButton>
+
+<Modal bind:this={editVariableModal}>
+  <CreateEditVariableModal {row} {save} />
+</Modal>
+
+<ConfirmDialog
+  bind:this={deleteDialog}
+  onOk={async () => {
+    await environment.deleteVariable(row.name)
+  }}
+  okText="Delete Environment Variable"
+  title="Confirm Deletion"
+>
+  Are you sure you wish to delete the environment variable
+  <i>{row.name}?</i>
+  This action cannot be undone.
+</ConfirmDialog>

packages/builder/src/pages/builder/portal/settings/environment/index.svelte

@@ -0,0 +1,145 @@
+<script>
+  import {
+    Layout,
+    Heading,
+    Body,
+    Button,
+    Divider,
+    Modal,
+    Table,
+    Tags,
+    Tag,
+    InlineAlert,
+    notifications,
+  } from "@budibase/bbui"
+  import { environment, licensing, auth, admin } from "stores/portal"
+  import { onMount } from "svelte"
+  import CreateEditVariableModal from "components/portal/environment/CreateEditVariableModal.svelte"
+  import EditVariableColumn from "./_components/EditVariableColumn.svelte"
+
+  let modal
+
+  const customRenderers = [{ column: "edit", component: EditVariableColumn }]
+
+  $: noEncryptionKey = $environment.status?.encryptionKeyAvailable === false
+  $: schema = buildSchema(noEncryptionKey)
+
+  onMount(async () => {
+    await environment.checkStatus()
+    await environment.loadVariables()
+  })
+
+  const buildSchema = noEncryptionKey => {
+    const schema = {
+      name: {
+        width: "2fr",
+      },
+    }
+    if (!noEncryptionKey) {
+      schema.edit = {
+        width: "auto",
+        borderLeft: true,
+        displayName: "",
+      }
+    }
+    return schema
+  }
+
+  const save = async data => {
+    try {
+      await environment.createVariable(data)
+      modal.hide()
+    } catch (err) {
+      notifications.error(`Error saving variable: ${err.message}`)
+    }
+  }
+</script>
+
+<Layout noPadding>
+  <Layout gap="XS" noPadding>
+    <div class="title">
+      <Heading size="M">Environment Variables</Heading>
+      {#if !$licensing.environmentVariablesEnabled}
+        <Tags>
+          <Tag icon="LockClosed">Business plan</Tag>
+        </Tags>
+      {/if}
+    </div>
+    <Body
+      >Add and manage environment variables for development and production</Body
+    >
+  </Layout>
+  <Divider size="S" />
+
+  {#if $licensing.environmentVariablesEnabled}
+    {#if noEncryptionKey}
+      <InlineAlert
+        message="Your Budibase installation does not have a key for encryption, please update your app service's environment variables to contain an 'ENCRYPTION_KEY' value."
+        header="No encryption key found"
+        type="error"
+      />
+    {/if}
+    <div>
+      <Button on:click={modal.show} cta disabled={noEncryptionKey}
+        >Add Variable</Button
+      >
+    </div>
+
+    <Layout noPadding>
+      <Table
+        {schema}
+        data={$environment.variables}
+        allowEditColumns={false}
+        allowEditRows={false}
+        allowSelectRows={false}
+        {customRenderers}
+      />
+    </Layout>
+  {:else}
+    <div class="buttons">
+      <Button
+        primary
+        disabled={!$auth.accountPortalAccess && $admin.cloud}
+        on:click={async () => {
+          await environment.upgradePanelOpened()
+          $licensing.goToUpgradePage()
+        }}
+      >
+        Upgrade
+      </Button>
+      <!--Show the view plans button-->
+      <Button
+        secondary
+        on:click={() => {
+          window.open("https://budibase.com/pricing/", "_blank")
+        }}
+      >
+        View Plans
+      </Button>
+    </div>
+  {/if}
+</Layout>
+
+<Modal bind:this={modal}>
+  <CreateEditVariableModal {save} />
+</Modal>
+
+<style>
+  .buttons {
+    display: flex;
+    gap: var(--spacing-l);
+  }
+  .title {
+    display: flex;
+    flex-direction: row;
+    align-items: center;
+    justify-content: flex-start;
+    gap: var(--spacing-m);
+  }
+
+  .buttons {
+    display: flex;
+    flex-direction: row;
+    gap: var(--spacing-m);
+  }
+</style>

packages/builder/src/pages/builder/portal/settings/version.svelte

@@ -8,6 +8,8 @@
     Divider,
     notifications,
     Label,
+    Modal,
+    ModalContent,
   } from "@budibase/bbui"
   import { API } from "api"
   import { auth, admin } from "stores/portal"
@@ -15,6 +17,11 @@
 
   let version
   let loaded = false
+  let githubVersion
+  let githubPublishedDate
+  let githubPublishedTime
+  let needsUpdate = true
+  let updateModal
 
   // Only admins allowed here
   $: {
@@ -47,8 +54,37 @@
     }
   }
 
+  async function getLatestVersion() {
+    try {
+      //Check github API for the latest release
+      const githubCheck = await fetch(
+        "https://api.github.com/repos/Budibase/budibase/releases/latest"
+      )
+      const githubResponse = await githubCheck.json()
+
+      //Get tag and remove the v infront of the tage name e.g. v1.0.0 is 1.0.0
+      githubVersion = githubResponse.tag_name.slice(1)
+
+      //Get the release date and output it in the local time format
+      githubPublishedDate = new Date(githubResponse.published_at)
+      githubPublishedTime = githubPublishedDate.toLocaleTimeString()
+      githubPublishedDate = githubPublishedDate.toLocaleDateString()
+
+      //Does Budibase need to be updated?
+      if (githubVersion === version) {
+        needsUpdate = false
+      } else {
+        needsUpdate = true
+      }
+    } catch (error) {
+      notifications.error("Error getting the latest Budibase version")
+      githubVersion = null
+    }
+  }
+
   onMount(async () => {
     await getVersion()
+    await getLatestVersion()
     loaded = true
   })
 </script>
@@ -69,9 +105,32 @@
         <Heading size="S">
           {version || "-"}
         </Heading>
+        <Divider />
+        <Label size="L">Latest version</Label>
+        <Heading size="S">
+          {githubVersion}
+        </Heading>
+        <Label size="L"
+          >This version was released on {githubPublishedDate} at {githubPublishedTime}</Label
+        >
       </Layout>
+      <Divider />
       <div>
-        <Button cta on:click={updateBudibase}>Check for updates</Button>
+        <Button cta on:click={updateModal.show} disabled={!needsUpdate}
+          >Update Budibase</Button
+        >
+        <Modal bind:this={updateModal}>
+          <ModalContent
+            title="Update Budibase"
+            confirmText="Update"
+            onConfirm={updateBudibase}
+          >
+            <span
+              >Are you sure you want to update your budibase installation to the
+              latest version?</span
+            >
+          </ModalContent>
+        </Modal>
       </div>
     {/if}
   </Layout>

packages/builder/src/pages/builder/portal/users/_layout.svelte

@@ -1,11 +1,12 @@
 <script>
   import { Page } from "@budibase/bbui"
   import { SideNav, SideNavItem, Content } from "components/portal/page"
-  import { isActive } from "@roxi/routify"
+  import { isActive, goto } from "@roxi/routify"
   import { menu } from "stores/portal"
 
   $: wide = $isActive("./users/index") || $isActive("./groups/index")
-  $: pages = $menu.find(x => x.title === "Users").subPages
+  $: pages = $menu.find(x => x.title === "Users")?.subPages || []
+  $: !pages.length && $goto("../")
 </script>
 
 <Page>

packages/builder/src/stores/backend/datasources.js

@@ -28,7 +28,7 @@ export function createDatasourcesStore() {
     }))
   }
 
-  const updateDatasource = async response => {
+  const updateDatasource = response => {
     const { datasource, error } = response
     store.update(state => {
       const currentIdx = state.list.findIndex(ds => ds._id === datasource._id)
@@ -52,7 +52,7 @@ export function createDatasourcesStore() {
       datasourceId: datasource?._id,
       tablesFilter,
     })
-    return await updateDatasource(response)
+    return updateDatasource(response)
   }
 
   const save = async (body, fetchSchema = false) => {

packages/builder/src/stores/portal/environment.js

@@ -0,0 +1,68 @@
+import { writable } from "svelte/store"
+import { API } from "api"
+import { Constants } from "@budibase/frontend-core"
+
+export function createEnvironmentStore() {
+  const { subscribe, update } = writable({
+    variables: [],
+    status: {},
+  })
+
+  async function checkStatus() {
+    const status = await API.checkEnvironmentVariableStatus()
+    update(store => {
+      store.status = status
+      return store
+    })
+  }
+
+  async function loadVariables() {
+    const envVars = await API.fetchEnvironmentVariables()
+    const mappedVars = envVars.variables.map(name => ({ name }))
+    update(store => {
+      store.variables = mappedVars
+      return store
+    })
+  }
+
+  async function createVariable(data) {
+    await API.createEnvironmentVariable(data)
+    let mappedVar = { name: data.name }
+    update(store => {
+      store.variables = [mappedVar, ...store.variables]
+      return store
+    })
+  }
+
+  async function deleteVariable(varName) {
+    await API.deleteEnvironmentVariable(varName)
+    update(store => {
+      store.variables = store.variables.filter(
+        envVar => envVar.name !== varName
+      )
+      return store
+    })
+  }
+
+  async function updateVariable(data) {
+    await API.updateEnvironmentVariable(data)
+  }
+
+  async function upgradePanelOpened() {
+    await API.publishEvent(
+      Constants.EventPublishType.ENV_VAR_UPGRADE_PANEL_OPENED
+    )
+  }
+
+  return {
+    subscribe,
+    checkStatus,
+    loadVariables,
+    createVariable,
+    deleteVariable,
+    updateVariable,
+    upgradePanelOpened,
+  }
+}
+
+export const environment = createEnvironmentStore()

packages/builder/src/stores/portal/index.js

@@ -11,4 +11,5 @@ export { groups } from "./groups"
 export { plugins } from "./plugins"
 export { backups } from "./backups"
 export { overview } from "./overview"
+export { environment } from "./environment"
 export { menu } from "./menu"

packages/builder/src/stores/portal/licensing.js

@@ -60,6 +60,9 @@ export const createLicensingStore = () => {
       const backupsEnabled = license.features.includes(
         Constants.Features.BACKUPS
       )
+      const environmentVariablesEnabled = license.features.includes(
+        Constants.Features.ENVIRONMENT_VARIABLES
+      )
 
       store.update(state => {
         return {
@@ -68,6 +71,7 @@ export const createLicensingStore = () => {
           isFreePlan,
           groupsEnabled,
           backupsEnabled,
+          environmentVariablesEnabled,
         }
       })
     },

packages/builder/src/stores/portal/menu.js

@@ -50,6 +50,10 @@ export const menu = derived([admin, auth], ([$admin, $auth]) => {
         title: "Organisation",
         href: "/builder/portal/settings/organisation",
       },
+      {
+        title: "Environment",
+        href: "/builder/portal/settings/environment",
+      },
     ]
     if (!$admin.cloud) {
       settingsSubPages.push({

packages/builder/yarn.lock

@@ -1356,6 +1356,11 @@
   dependencies:
     "@sinonjs/commons" "^1.7.0"
 
+"@spectrum-css/accordion@^3.0.24":
+  version "3.0.24"
+  resolved "https://registry.yarnpkg.com/@spectrum-css/accordion/-/accordion-3.0.24.tgz#f89066c120c57b0cfc9aba66d60c39fc1cf69f74"
+  integrity sha512-jNOmUsxmiT3lRLButnN5KKHM94fd+87fjiF8L0c4uRNgJl6ZsBuxPXrM15lV4y1f8D2IACAw01/ZkGRAeaCOFA==
+
 "@spectrum-css/page@^3.0.1":
   version "3.0.8"
   resolved "https://registry.yarnpkg.com/@spectrum-css/page/-/page-3.0.8.tgz#001efa9e4c10095df9b2b37cf7d7d6eb60140190"

packages/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/cli",
-  "version": "2.2.12-alpha.32",
+  "version": "2.2.12-alpha.47",
   "description": "Budibase CLI, for developers, self hosting and migrations.",
   "main": "src/index.js",
   "bin": {
@@ -26,9 +26,9 @@
     "outputPath": "build"
   },
   "dependencies": {
-    "@budibase/backend-core": "2.2.12-alpha.32",
+    "@budibase/backend-core": "2.2.12-alpha.47",
-    "@budibase/string-templates": "2.2.12-alpha.32",
+    "@budibase/string-templates": "2.2.12-alpha.47",
-    "@budibase/types": "2.2.12-alpha.32",
+    "@budibase/types": "2.2.12-alpha.47",
     "axios": "0.21.2",
     "chalk": "4.1.0",
     "cli-progress": "3.11.2",

packages/client/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/client",
-  "version": "2.2.12-alpha.32",
+  "version": "2.2.12-alpha.47",
   "license": "MPL-2.0",
   "module": "dist/budibase-client.js",
   "main": "dist/budibase-client.js",
@@ -19,9 +19,9 @@
     "dev:builder": "rollup -cw"
   },
   "dependencies": {
-    "@budibase/bbui": "2.2.12-alpha.32",
+    "@budibase/bbui": "2.2.12-alpha.47",
-    "@budibase/frontend-core": "2.2.12-alpha.32",
+    "@budibase/frontend-core": "2.2.12-alpha.47",
-    "@budibase/string-templates": "2.2.12-alpha.32",
+    "@budibase/string-templates": "2.2.12-alpha.47",
     "@spectrum-css/button": "^3.0.3",
     "@spectrum-css/card": "^3.0.3",
     "@spectrum-css/divider": "^1.0.3",

packages/frontend-core/package.json

@@ -1,12 +1,12 @@
 {
   "name": "@budibase/frontend-core",
-  "version": "2.2.12-alpha.32",
+  "version": "2.2.12-alpha.47",
   "description": "Budibase frontend core libraries used in builder and client",
   "author": "Budibase",
   "license": "MPL-2.0",
   "svelte": "src/index.js",
   "dependencies": {
-    "@budibase/bbui": "2.2.12-alpha.32",
+    "@budibase/bbui": "2.2.12-alpha.47",
     "lodash": "^4.17.21",
     "svelte": "^3.46.2"
   }

packages/frontend-core/src/api/environmentVariables.js

@@ -0,0 +1,36 @@
+export const buildEnvironmentVariableEndpoints = API => ({
+  checkEnvironmentVariableStatus: async () => {
+    return await API.get({
+      url: `/api/env/variables/status`,
+    })
+  },
+
+  /**
+   * Fetches a list of environment variables
+   */
+  fetchEnvironmentVariables: async () => {
+    return await API.get({
+      url: `/api/env/variables`,
+      json: false,
+    })
+  },
+
+  createEnvironmentVariable: async data => {
+    return await API.post({
+      url: `/api/env/variables`,
+      body: data,
+    })
+  },
+  deleteEnvironmentVariable: async varName => {
+    return await API.delete({
+      url: `/api/env/variables/${varName}`,
+    })
+  },
+
+  updateEnvironmentVariable: async data => {
+    return await API.patch({
+      url: `/api/env/variables/${data.name}`,
+      body: data,
+    })
+  },
+})

packages/frontend-core/src/api/events.js

@@ -0,0 +1,13 @@
+export const buildEventEndpoints = API => ({
+  /**
+   * Publish a specific event to the backend.
+   */
+  publishEvent: async eventType => {
+    return await API.post({
+      url: `/api/global/event/publish`,
+      body: {
+        type: eventType,
+      },
+    })
+  },
+})

packages/frontend-core/src/api/index.js

@@ -26,7 +26,8 @@ import { buildLicensingEndpoints } from "./licensing"
 import { buildGroupsEndpoints } from "./groups"
 import { buildPluginEndpoints } from "./plugins"
 import { buildBackupsEndpoints } from "./backups"
-
+import { buildEnvironmentVariableEndpoints } from "./environmentVariables"
+import { buildEventEndpoints } from "./events"
 const defaultAPIClientConfig = {
   /**
    * Certain definitions can't change at runtime for client apps, such as the
@@ -247,5 +248,7 @@ export const createAPIClient = config => {
     ...buildGroupsEndpoints(API),
     ...buildPluginEndpoints(API),
     ...buildBackupsEndpoints(API),
+    ...buildEnvironmentVariableEndpoints(API),
+    ...buildEventEndpoints(API),
   }
 }

packages/frontend-core/src/components/SplitPage.svelte

@@ -0,0 +1,51 @@
+<div class="split-page">
+  <div class="left">
+    <div class="content">
+      <slot />
+    </div>
+  </div>
+  <div class="right">
+    <slot name="right" />
+  </div>
+</div>
+
+<style>
+  .split-page {
+    height: 100%;
+    display: grid;
+    grid-template-columns: max(50%, 380px) 1fr;
+    justify-content: stretch;
+    overflow: hidden;
+  }
+  .left {
+    background: var(--background);
+    display: grid;
+    place-items: center;
+    padding: 40px;
+    overflow-y: auto;
+  }
+  .right {
+    background: linear-gradient(
+      to bottom right,
+      var(--spectrum-global-color-gray-300) 0%,
+      var(--background) 100%
+    );
+  }
+  .content {
+    width: 100%;
+    max-width: 400px;
+    min-height: 480px;
+  }
+
+  @media (max-width: 740px) {
+    .split-page {
+      grid-template-columns: 1fr;
+    }
+    .left {
+      padding: 20px;
+    }
+    .right {
+      display: none;
+    }
+  }
+</style>

packages/frontend-core/src/components/TestimonialPage.svelte

@@ -0,0 +1,63 @@
+<script>
+  import SplitPage from "./SplitPage.svelte"
+  import { Layout } from "@budibase/bbui"
+</script>
+
+<SplitPage>
+  <slot />
+  <div class="wrapper" slot="right">
+    <div class="testimonial">
+      <Layout noPadding gap="S">
+        <div class="text">
+          "Here is an example of how Budibase changed my life for the better and
+          now all I do is eat, sleep, build apps, repeat."
+        </div>
+        <div class="user">
+          <img
+            alt="a-happy-budibase-user"
+            src="https://icon-library.com/images/male-user-icon/male-user-icon-24.jpg"
+          />
+          <div class="author">
+            <div class="name">No-code Enthusiast</div>
+            <div class="company">Bedroom TLD</div>
+          </div>
+        </div>
+      </Layout>
+    </div>
+  </div>
+</SplitPage>
+
+<style>
+  .wrapper {
+    width: 100%;
+    height: 100%;
+    display: grid;
+    place-items: center;
+  }
+  .testimonial {
+    width: 280px;
+    padding: 40px;
+  }
+  .text {
+    font-size: var(--font-size-l);
+    font-style: italic;
+  }
+  img {
+    width: 40px;
+  }
+  .user {
+    display: flex;
+    flex-direction: row;
+    justify-content: flex-start;
+    align-items: center;
+    gap: var(--spacing-s);
+  }
+  .name {
+    font-weight: bold;
+    color: var(--spectrum-global-color-gray-900);
+    font-size: var(--font-size-l);
+  }
+  .company {
+    color: var(--spectrum-global-color-gray-700);
+  }
+</style>

packages/frontend-core/src/components/index.js

@@ -0,0 +1,2 @@
+export { default as SplitPage } from "./SplitPage.svelte"
+export { default as TestimonialPage } from "./TestimonialPage.svelte"

packages/frontend-core/src/constants.js

@@ -114,6 +114,7 @@ export const ApiVersion = "1"
 export const Features = {
   USER_GROUPS: "userGroups",
   BACKUPS: "appBackups",
+  ENVIRONMENT_VARIABLES: "environmentVariables",
 }
 
 // Role IDs
@@ -174,3 +175,7 @@ export const Themes = [
     base: "darkest",
   },
 ]
+
+export const EventPublishType = {
+  ENV_VAR_UPGRADE_PANEL_OPENED: "environment_variable_upgrade_panel_opened",
+}

packages/frontend-core/src/index.js

@@ -3,3 +3,4 @@ export { fetchData } from "./fetch/fetchData"
 export * as Constants from "./constants"
 export * from "./stores"
 export * from "./utils"
+export * from "./components"

packages/sdk/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/sdk",
-  "version": "2.2.12-alpha.32",
+  "version": "2.2.12-alpha.47",
   "description": "Budibase Public API SDK",
   "author": "Budibase",
   "license": "MPL-2.0",

packages/sdk/yarn.lock

@@ -225,9 +225,9 @@ concat-map@0.0.1:
   integrity sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==
 
 cookiejar@^2.1.2:
-  version "2.1.3"
+  version "2.1.4"
-  resolved "https://registry.yarnpkg.com/cookiejar/-/cookiejar-2.1.3.tgz#fc7a6216e408e74414b90230050842dacda75acc"
+  resolved "https://registry.yarnpkg.com/cookiejar/-/cookiejar-2.1.4.tgz#ee669c1fea2cf42dc31585469d193fef0d65771b"
-  integrity sha512-JxbCBUdrfr6AQjOXrxoTvAMJO4HBTUIlBzslcJPAz+/KT8yk53fXun51u+RenNYvad/+Vc2DIz5o9UxlCDymFQ==
+  integrity sha512-LDx6oHrK+PhzLKJU9j5S7/Y3jM/mUHvD/DeI1WQmJn652iPC5Y4TBzC9l+5OMOXlyTTA+SmVUPm0HQUwpD5Jqw==
 
 debug@^4.1.1:
   version "4.3.4"

packages/server/__mocks__/node-fetch.ts

@@ -1,5 +1,7 @@
+// @ts-ignore
 import fs from "fs"
 module FetchMock {
+  // @ts-ignore
   const fetch = jest.requireActual("node-fetch")
   let failCount = 0
 

packages/server/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "2.2.12-alpha.32",
+  "version": "2.2.12-alpha.47",
   "description": "Budibase Web Server",
   "main": "src/index.ts",
   "repository": {
@@ -43,11 +43,11 @@
   "license": "GPL-3.0",
   "dependencies": {
     "@apidevtools/swagger-parser": "10.0.3",
-    "@budibase/backend-core": "2.2.12-alpha.32",
+    "@budibase/backend-core": "2.2.12-alpha.47",
-    "@budibase/client": "2.2.12-alpha.32",
+    "@budibase/client": "2.2.12-alpha.47",
-    "@budibase/pro": "2.2.12-alpha.32",
+    "@budibase/pro": "2.2.12-alpha.47",
-    "@budibase/string-templates": "2.2.12-alpha.32",
+    "@budibase/string-templates": "2.2.12-alpha.47",
-    "@budibase/types": "2.2.12-alpha.32",
+    "@budibase/types": "2.2.12-alpha.47",
     "@bull-board/api": "3.7.0",
     "@bull-board/koa": "3.9.4",
     "@elastic/elasticsearch": "7.10.0",

packages/server/scripts/dev/manage.js

@@ -29,6 +29,7 @@ async function init() {
       ACCOUNT_PORTAL_URL: "http://localhost:10001",
       ACCOUNT_PORTAL_API_KEY: "budibase",
       JWT_SECRET: "testsecret",
+      ENCRYPTION_KEY: "testsecret",
       REDIS_PASSWORD: "budibase",
       MINIO_ACCESS_KEY: "budibase",
       MINIO_SECRET_KEY: "budibase",

packages/server/src/api/controllers/application.ts

@@ -112,12 +112,11 @@ function checkAppName(
   }
 }
 
-async function createInstance(template: any, includeSampleData: boolean) {
+async function createInstance(
-  const tenantId = tenancy.isMultiTenant() ? tenancy.getTenantId() : null
+  appId: string,
-  const baseAppId = generateAppID(tenantId)
+  template: any,
-  const appId = generateDevAppID(baseAppId)
+  includeSampleData: boolean
-  await context.updateAppId(appId)
+) {
-
   const db = context.getAppDB()
   await db.put({
     _id: "_design/database",
@@ -250,82 +249,90 @@ async function performAppCreate(ctx: BBContext) {
     instanceConfig.file = ctx.request.files.templateFile
   }
   const includeSampleData = isQsTrue(ctx.request.body.sampleData)
-  const instance = await createInstance(instanceConfig, includeSampleData)
+  const tenantId = tenancy.isMultiTenant() ? tenancy.getTenantId() : null
-  const appId = instance._id
+  const appId = generateDevAppID(generateAppID(tenantId))
-  const db = context.getAppDB()
 
-  let newApplication: App = {
+  return await context.doInAppContext(appId, async () => {
-    _id: DocumentType.APP_METADATA,
+    const instance = await createInstance(
-    _rev: undefined,
+      appId,
-    appId,
+      instanceConfig,
-    type: "app",
+      includeSampleData
-    version: packageJson.version,
+    )
-    componentLibraries: ["@budibase/standard-components"],
+    const db = context.getAppDB()
-    name: name,
-    url: url,
-    template: templateKey,
-    instance,
-    tenantId: tenancy.getTenantId(),
-    updatedAt: new Date().toISOString(),
-    createdAt: new Date().toISOString(),
-    status: AppStatus.DEV,
-    navigation: {
-      navigation: "Top",
-      title: name,
-      navWidth: "Large",
-      navBackground: "var(--spectrum-global-color-gray-100)",
-      links: [
-        {
-          url: "/home",
-          text: "Home",
-        },
-      ],
-    },
-    theme: "spectrum--light",
-    customTheme: {
-      buttonBorderRadius: "16px",
-    },
-  }
 
-  // If we used a template or imported an app there will be an existing doc.
+    let newApplication: App = {
-  // Fetch and migrate some metadata from the existing app.
+      _id: DocumentType.APP_METADATA,
-  try {
+      _rev: undefined,
-    const existing: App = await db.get(DocumentType.APP_METADATA)
+      appId,
-    const keys: (keyof App)[] = [
+      type: "app",
-      "_rev",
+      version: packageJson.version,
-      "navigation",
+      componentLibraries: ["@budibase/standard-components"],
-      "theme",
+      name: name,
-      "customTheme",
+      url: url,
-      "icon",
+      template: templateKey,
-    ]
+      instance,
-    keys.forEach(key => {
+      tenantId: tenancy.getTenantId(),
-      if (existing[key]) {
+      updatedAt: new Date().toISOString(),
-        // @ts-ignore
+      createdAt: new Date().toISOString(),
-        newApplication[key] = existing[key]
+      status: AppStatus.DEV,
-      }
+      navigation: {
-    })
+        navigation: "Top",
-
+        title: name,
-    // Migrate navigation settings and screens if required
+        navWidth: "Large",
-    if (existing) {
+        navBackground: "var(--spectrum-global-color-gray-100)",
-      const navigation = await migrateAppNavigation()
+        links: [
-      if (navigation) {
+          {
-        newApplication.navigation = navigation
+            url: "/home",
-      }
+            text: "Home",
+          },
+        ],
+      },
+      theme: "spectrum--light",
+      customTheme: {
+        buttonBorderRadius: "16px",
+      },
     }
-  } catch (err) {
-    // Nothing to do
-  }
 
-  const response = await db.put(newApplication, { force: true })
+    // If we used a template or imported an app there will be an existing doc.
-  newApplication._rev = response.rev
+    // Fetch and migrate some metadata from the existing app.
+    try {
+      const existing: App = await db.get(DocumentType.APP_METADATA)
+      const keys: (keyof App)[] = [
+        "_rev",
+        "navigation",
+        "theme",
+        "customTheme",
+        "icon",
+      ]
+      keys.forEach(key => {
+        if (existing[key]) {
+          // @ts-ignore
+          newApplication[key] = existing[key]
+        }
+      })
 
-  /* istanbul ignore next */
+      // Migrate navigation settings and screens if required
-  if (!env.isTest()) {
+      if (existing) {
-    await createApp(appId)
+        const navigation = await migrateAppNavigation()
-  }
+        if (navigation) {
+          newApplication.navigation = navigation
+        }
+      }
+    } catch (err) {
+      // Nothing to do
+    }
 
-  await cache.app.invalidateAppMetadata(appId, newApplication)
+    const response = await db.put(newApplication, { force: true })
-  return newApplication
+    newApplication._rev = response.rev
+
+    /* istanbul ignore next */
+    if (!env.isTest()) {
+      await createApp(appId)
+    }
+
+    await cache.app.invalidateAppMetadata(appId, newApplication)
+    return newApplication
+  })
 }
 
 async function creationEvents(request: any, app: App) {

packages/server/src/api/controllers/datasource.ts

@@ -12,9 +12,11 @@ import { getIntegration } from "../../integrations"
 import { getDatasourceAndQuery } from "./row/utils"
 import { invalidateDynamicVariables } from "../../threads/utils"
 import { db as dbCore, context, events } from "@budibase/backend-core"
-import { BBContext, Datasource, Row } from "@budibase/types"
+import { UserCtx, Datasource, Row } from "@budibase/types"
+import sdk from "../../sdk"
+import { mergeConfigs } from "../../sdk/app/datasources/datasources"
 
-export async function fetch(ctx: BBContext) {
+export async function fetch(ctx: UserCtx) {
   // Get internal tables
   const db = context.getAppDB()
   const internalTables = await db.allDocs(
@@ -43,25 +45,23 @@ export async function fetch(ctx: BBContext) {
     )
   ).rows.map(row => row.doc)
 
-  const allDatasources = [bbInternalDb, ...datasources]
+  const allDatasources: Datasource[] = await sdk.datasources.removeSecrets([
+    bbInternalDb,
+    ...datasources,
+  ])
 
   for (let datasource of allDatasources) {
-    if (datasource.config && datasource.config.auth) {
-      // strip secrets from response so they don't show in the network request
-      delete datasource.config.auth
-    }
-
     if (datasource.type === dbCore.BUDIBASE_DATASOURCE_TYPE) {
-      datasource.entities = internal[datasource._id]
+      datasource.entities = internal[datasource._id!]
     }
   }
 
   ctx.body = [bbInternalDb, ...datasources]
 }
 
-export async function buildSchemaFromDb(ctx: BBContext) {
+export async function buildSchemaFromDb(ctx: UserCtx) {
   const db = context.getAppDB()
-  const datasource = await db.get(ctx.params.datasourceId)
+  const datasource = await sdk.datasources.get(ctx.params.datasourceId)
   const tablesFilter = ctx.request.body.tablesFilter
 
   let { tables, error } = await buildSchemaHelper(datasource)
@@ -146,11 +146,11 @@ async function invalidateVariables(
   await invalidateDynamicVariables(toInvalidate)
 }
 
-export async function update(ctx: BBContext) {
+export async function update(ctx: UserCtx) {
   const db = context.getAppDB()
   const datasourceId = ctx.params.datasourceId
-  let datasource = await db.get(datasourceId)
+  let datasource = await sdk.datasources.get(datasourceId)
-  const auth = datasource.config.auth
+  const auth = datasource.config?.auth
   await invalidateVariables(datasource, ctx.request.body)
 
   const isBudibaseSource = datasource.type === dbCore.BUDIBASE_DATASOURCE_TYPE
@@ -159,10 +159,13 @@ export async function update(ctx: BBContext) {
     ? { name: ctx.request.body?.name }
     : ctx.request.body
 
-  datasource = { ...datasource, ...dataSourceBody }
+  datasource = {
+    ...datasource,
+    ...sdk.datasources.mergeConfigs(dataSourceBody, datasource),
+  }
   if (auth && !ctx.request.body.auth) {
     // don't strip auth config from DB
-    datasource.config.auth = auth
+    datasource.config!.auth = auth
   }
 
   const response = await db.put(datasource)
@@ -179,10 +182,12 @@ export async function update(ctx: BBContext) {
 
   ctx.status = 200
   ctx.message = "Datasource saved successfully."
-  ctx.body = { datasource }
+  ctx.body = {
+    datasource: await sdk.datasources.removeSecretSingle(datasource),
+  }
 }
 
-export async function save(ctx: BBContext) {
+export async function save(ctx: UserCtx) {
   const db = context.getAppDB()
   const plus = ctx.request.body.datasource.plus
   const fetchSchema = ctx.request.body.fetchSchema
@@ -213,7 +218,9 @@ export async function save(ctx: BBContext) {
     }
   }
 
-  const response: any = { datasource }
+  const response: any = {
+    datasource: await sdk.datasources.removeSecretSingle(datasource),
+  }
   if (schemaError) {
     response.error = schemaError
   }
@@ -251,11 +258,11 @@ async function destroyInternalTablesBySourceId(datasourceId: string) {
   }
 }
 
-export async function destroy(ctx: BBContext) {
+export async function destroy(ctx: UserCtx) {
   const db = context.getAppDB()
   const datasourceId = ctx.params.datasourceId
 
-  const datasource = await db.get(datasourceId)
+  const datasource = await sdk.datasources.get(datasourceId)
   // Delete all queries for the datasource
 
   if (datasource.type === dbCore.BUDIBASE_DATASOURCE_TYPE) {
@@ -279,13 +286,14 @@ export async function destroy(ctx: BBContext) {
   ctx.status = 200
 }
 
-export async function find(ctx: BBContext) {
+export async function find(ctx: UserCtx) {
   const database = context.getAppDB()
-  ctx.body = await database.get(ctx.params.datasourceId)
+  const datasource = await database.get(ctx.params.datasourceId)
+  ctx.body = await sdk.datasources.removeSecretSingle(datasource)
 }
 
 // dynamic query functionality
-export async function query(ctx: BBContext) {
+export async function query(ctx: UserCtx) {
   const queryJson = ctx.request.body
   try {
     ctx.body = await getDatasourceAndQuery(queryJson)
@@ -313,7 +321,7 @@ function updateError(error: any, newError: any, tables: string[]) {
 
 async function buildSchemaHelper(datasource: Datasource) {
   const Connector = await getIntegration(datasource.source)
-
+  datasource = await sdk.datasources.enrich(datasource)
   // Connect to the DB and build the schema
   const connector = new Connector(datasource.config)
   await connector.buildSchema(datasource._id, datasource.entities)

packages/server/src/api/controllers/query/index.ts

@@ -7,6 +7,8 @@ import { invalidateDynamicVariables } from "../../../threads/utils"
 import env from "../../../environment"
 import { quotas } from "@budibase/pro"
 import { events, context, utils, constants } from "@budibase/backend-core"
+import sdk from "../../../sdk"
+import { QueryEvent } from "../../../threads/definitions"
 
 const Runner = new Thread(ThreadType.QUERY, {
   timeoutMs: env.QUERY_THREAD_TIMEOUT || 10000,
@@ -81,7 +83,7 @@ export async function save(ctx: any) {
   const db = context.getAppDB()
   const query = ctx.request.body
 
-  const datasource = await db.get(query.datasourceId)
+  const datasource = await sdk.datasources.get(query.datasourceId)
 
   let eventFn
   if (!query._id) {
@@ -126,9 +128,9 @@ function getAuthConfig(ctx: any) {
 }
 
 export async function preview(ctx: any) {
-  const db = context.getAppDB()
+  const { datasource, envVars } = await sdk.datasources.getWithEnvVars(
-
+    ctx.request.body.datasourceId
-  const datasource = await db.get(ctx.request.body.datasourceId)
+  )
   const query = ctx.request.body
   // preview may not have a queryId as it hasn't been saved, but if it does
   // this stops dynamic variables from calling the same query
@@ -137,20 +139,22 @@ export async function preview(ctx: any) {
   const authConfigCtx: any = getAuthConfig(ctx)
 
   try {
-    const runFn = () =>
+    const inputs: QueryEvent = {
-      Runner.run({
+      appId: ctx.appId,
-        appId: ctx.appId,
+      datasource,
-        datasource,
+      queryVerb,
-        queryVerb,
+      fields,
-        fields,
+      parameters,
-        parameters,
+      transformer,
-        transformer,
+      queryId,
-        queryId,
+      // have to pass down to the thread runner - can't put into context now
-        ctx: {
+      environmentVariables: envVars,
-          user: ctx.user,
+      ctx: {
-          auth: { ...authConfigCtx },
+        user: ctx.user,
-        },
+        auth: { ...authConfigCtx },
-      })
+      },
+    }
+    const runFn = () => Runner.run(inputs)
 
     const { rows, keys, info, extra } = await quotas.addQuery(runFn, {
       datasourceId: datasource._id,
@@ -201,7 +205,9 @@ async function execute(
   const db = context.getAppDB()
 
   const query = await db.get(ctx.params.queryId)
-  const datasource = await db.get(query.datasourceId)
+  const { datasource, envVars } = await sdk.datasources.getWithEnvVars(
+    query.datasourceId
+  )
 
   let authConfigCtx: any = {}
   if (!opts.isAutomation) {
@@ -219,21 +225,23 @@ async function execute(
 
   // call the relevant CRUD method on the integration class
   try {
-    const runFn = () =>
+    const inputs: QueryEvent = {
-      Runner.run({
+      appId: ctx.appId,
-        appId: ctx.appId,
+      datasource,
-        datasource,
+      queryVerb: query.queryVerb,
-        queryVerb: query.queryVerb,
+      fields: query.fields,
-        fields: query.fields,
+      pagination: ctx.request.body.pagination,
-        pagination: ctx.request.body.pagination,
+      parameters: enrichedParameters,
-        parameters: enrichedParameters,
+      transformer: query.transformer,
-        transformer: query.transformer,
+      queryId: ctx.params.queryId,
-        queryId: ctx.params.queryId,
+      // have to pass down to the thread runner - can't put into context now
-        ctx: {
+      environmentVariables: envVars,
-          user: ctx.user,
+      ctx: {
-          auth: { ...authConfigCtx },
+        user: ctx.user,
-        },
+        auth: { ...authConfigCtx },
-      })
+      },
+    }
+    const runFn = () => Runner.run(inputs)
 
     const { rows, pagination, extra } = await quotas.addQuery(runFn, {
       datasourceId: datasource._id,
@@ -266,18 +274,18 @@ export async function executeV2(
 const removeDynamicVariables = async (queryId: any) => {
   const db = context.getAppDB()
   const query = await db.get(queryId)
-  const datasource = await db.get(query.datasourceId)
+  const datasource = await sdk.datasources.get(query.datasourceId)
-  const dynamicVariables = datasource.config.dynamicVariables
+  const dynamicVariables = datasource.config?.dynamicVariables as any[]
 
   if (dynamicVariables) {
     // delete dynamic variables from the datasource
-    datasource.config.dynamicVariables = dynamicVariables.filter(
+    datasource.config!.dynamicVariables = dynamicVariables!.filter(
       (dv: any) => dv.queryId !== queryId
     )
     await db.put(datasource)
 
     // invalidate the deleted variables
-    const variablesToDelete = dynamicVariables.filter(
+    const variablesToDelete = dynamicVariables!.filter(
       (dv: any) => dv.queryId === queryId
     )
     await invalidateDynamicVariables(variablesToDelete)
@@ -289,7 +297,7 @@ export async function destroy(ctx: any) {
   const queryId = ctx.params.queryId
   await removeDynamicVariables(queryId)
   const query = await db.get(queryId)
-  const datasource = await db.get(query.datasourceId)
+  const datasource = await sdk.datasources.get(query.datasourceId)
   await db.remove(ctx.params.queryId, ctx.params.revId)
   ctx.message = `Query deleted.`
   ctx.status = 200

packages/server/src/api/controllers/row/ExternalRequest.ts

@@ -25,6 +25,7 @@ import { cloneDeep } from "lodash/fp"
 import { processFormulas, processDates } from "../../../utilities/rowProcessor"
 import { context } from "@budibase/backend-core"
 import { removeKeyNumbering } from "./utils"
+import sdk from "../../../sdk"
 
 export interface ManyRelationship {
   tableId?: string
@@ -664,8 +665,7 @@ export class ExternalRequest {
       throw "Unable to run without a table name"
     }
     if (!this.datasource) {
-      const db = context.getAppDB()
+      this.datasource = await sdk.datasources.get(datasourceId!)
-      this.datasource = await db.get(datasourceId)
       if (!this.datasource || !this.datasource.entities) {
         throw "No tables found, fetch tables before query."
       }

packages/server/src/api/controllers/row/external.ts

@@ -19,6 +19,7 @@ import {
   Table,
   Datasource,
 } from "@budibase/types"
+import sdk from "../../../sdk"
 
 const { cleanExportRows } = require("./utils")
 
@@ -101,7 +102,7 @@ export async function destroy(ctx: BBContext) {
 export async function bulkDestroy(ctx: BBContext) {
   const { rows } = ctx.request.body
   const tableId = ctx.params.tableId
-  let promises = []
+  let promises: Promise<Row[] | { row: Row; table: Table }>[] = []
   for (let row of rows) {
     promises.push(
       handleRequest(Operation.DELETE, tableId, {
@@ -180,11 +181,10 @@ export async function validate(ctx: BBContext) {
 }
 
 export async function exportRows(ctx: BBContext) {
-  const { datasourceId, tableName } = breakExternalTableId(ctx.params.tableId)
+  const { datasourceId } = breakExternalTableId(ctx.params.tableId)
-  const db = context.getAppDB()
   const format = ctx.query.format
   const { columns } = ctx.request.body
-  const datasource = await db.get(datasourceId)
+  const datasource = await sdk.datasources.get(datasourceId!)
   if (!datasource || !datasource.entities) {
     ctx.throw(400, "Datasource has not been configured for plus API.")
   }
@@ -236,8 +236,7 @@ export async function fetchEnrichedRow(ctx: BBContext) {
   const id = ctx.params.rowId
   const tableId = ctx.params.tableId
   const { datasourceId, tableName } = breakExternalTableId(tableId)
-  const db = context.getAppDB()
+  const datasource: Datasource = await sdk.datasources.get(datasourceId!)
-  const datasource: Datasource = await db.get(datasourceId)
   if (!tableName) {
     ctx.throw(400, "Unable to find table.")
   }

packages/server/src/api/controllers/row/utils.ts

@@ -9,6 +9,7 @@ const validateJs = require("validate.js")
 const { cloneDeep } = require("lodash/fp")
 import { Format } from "../view/exporters"
 import { Ctx } from "@budibase/types"
+import sdk from "../../../sdk"
 
 validateJs.extend(validateJs.validators.datetime, {
   parse: function (value: string) {
@@ -22,8 +23,7 @@ validateJs.extend(validateJs.validators.datetime, {
 
 export async function getDatasourceAndQuery(json: any) {
   const datasourceId = json.endpoint.datasourceId
-  const db = context.getAppDB()
+  const datasource = await sdk.datasources.get(datasourceId)
-  const datasource = await db.get(datasourceId)
   return makeExternalQuery(datasource, json)
 }
 

packages/server/src/api/controllers/static/index.ts

@@ -1,20 +1,21 @@
 require("svelte/register")
 
-const send = require("koa-send")
+import { resolve, join } from "../../../utilities/centralPath"
-const { resolve, join } = require("../../../utilities/centralPath")
 const uuid = require("uuid")
 import { ObjectStoreBuckets } from "../../../constants"
-const { processString } = require("@budibase/string-templates")
+import { processString } from "@budibase/string-templates"
-const {
+import {
   loadHandlebarsFile,
   NODE_MODULES_PATH,
   TOP_LEVEL_PATH,
-} = require("../../../utilities/fileSystem")
+} from "../../../utilities/fileSystem"
 import env from "../../../environment"
-const { DocumentType } = require("../../../db/utils")
+import { DocumentType } from "../../../db/utils"
-const { context, objectStore, utils } = require("@budibase/backend-core")
+import { context, objectStore, utils } from "@budibase/backend-core"
-const AWS = require("aws-sdk")
+import AWS from "aws-sdk"
-const fs = require("fs")
+import fs from "fs"
+import sdk from "../../../sdk"
+const send = require("koa-send")
 
 async function prepareUpload({ s3Key, bucket, metadata, file }: any) {
   const response = await objectStore.upload({
@@ -110,7 +111,7 @@ export const serveApp = async function (ctx: any) {
       title: appInfo.name,
       production: env.isProd(),
       appId,
-      clientLibPath: objectStore.clientLibraryUrl(appId, appInfo.version),
+      clientLibPath: objectStore.clientLibraryUrl(appId!, appInfo.version),
       usedPlugins: plugins,
     })
 
@@ -135,7 +136,7 @@ export const serveBuilderPreview = async function (ctx: any) {
     let appId = context.getAppId()
     const previewHbs = loadHandlebarsFile(`${__dirname}/templates/preview.hbs`)
     ctx.body = await processString(previewHbs, {
-      clientLibPath: objectStore.clientLibraryUrl(appId, appInfo.version),
+      clientLibPath: objectStore.clientLibraryUrl(appId!, appInfo.version),
     })
   } else {
     // just return the app info for jest to assert on
@@ -150,13 +151,11 @@ export const serveClientLibrary = async function (ctx: any) {
 }
 
 export const getSignedUploadURL = async function (ctx: any) {
-  const database = context.getAppDB()
-
   // Ensure datasource is valid
   let datasource
   try {
     const { datasourceId } = ctx.params
-    datasource = await database.get(datasourceId)
+    datasource = await sdk.datasources.get(datasourceId, { enriched: true })
     if (!datasource) {
       ctx.throw(400, "The specified datasource could not be found")
     }
@@ -172,8 +171,8 @@ export const getSignedUploadURL = async function (ctx: any) {
   // Determine type of datasource and generate signed URL
   let signedUrl
   let publicUrl
-  const awsRegion = datasource?.config?.region || "eu-west-1"
+  const awsRegion = (datasource?.config?.region || "eu-west-1") as string
-  if (datasource.source === "S3") {
+  if (datasource?.source === "S3") {
     const { bucket, key } = ctx.request.body || {}
     if (!bucket || !key) {
       ctx.throw(400, "bucket and key values are required")
@@ -182,8 +181,8 @@ export const getSignedUploadURL = async function (ctx: any) {
     try {
       const s3 = new AWS.S3({
         region: awsRegion,
-        accessKeyId: datasource?.config?.accessKeyId,
+        accessKeyId: datasource?.config?.accessKeyId as string,
-        secretAccessKey: datasource?.config?.secretAccessKey,
+        secretAccessKey: datasource?.config?.secretAccessKey as string,
         apiVersion: "2006-03-01",
         signatureVersion: "v4",
       })

packages/server/src/api/controllers/table/external.ts

@@ -219,7 +219,7 @@ export async function save(ctx: BBContext) {
   }
 
   const db = context.getAppDB()
-  const datasource = await db.get(datasourceId)
+  const datasource = await sdk.datasources.get(datasourceId)
   if (!datasource.entities) {
     datasource.entities = {}
   }
@@ -322,15 +322,17 @@ export async function destroy(ctx: BBContext) {
   const datasourceId = getDatasourceId(tableToDelete)
 
   const db = context.getAppDB()
-  const datasource = await db.get(datasourceId)
+  const datasource = await sdk.datasources.get(datasourceId!)
   const tables = datasource.entities
 
   const operation = Operation.DELETE_TABLE
-  await makeTableRequest(datasource, operation, tableToDelete, tables)
+  if (tables) {
+    await makeTableRequest(datasource, operation, tableToDelete, tables)
+    cleanupRelationships(tableToDelete, tables)
+    delete tables[tableToDelete.name]
+    datasource.entities = tables
+  }
 
-  cleanupRelationships(tableToDelete, tables)
-
-  delete datasource.entities[tableToDelete.name]
   await db.put(datasource)
 
   return tableToDelete

packages/server/src/api/controllers/webhook.ts

@@ -39,60 +39,62 @@ export async function destroy(ctx: BBContext) {
 }
 
 export async function buildSchema(ctx: BBContext) {
-  await context.updateAppId(ctx.params.instance)
+  await context.doInAppContext(ctx.params.instance, async () => {
-  const db = context.getAppDB()
+    const db = context.getAppDB()
-  const webhook = (await db.get(ctx.params.id)) as Webhook
+    const webhook = (await db.get(ctx.params.id)) as Webhook
-  webhook.bodySchema = toJsonSchema(ctx.request.body)
+    webhook.bodySchema = toJsonSchema(ctx.request.body)
-  // update the automation outputs
+    // update the automation outputs
-  if (webhook.action.type === WebhookActionType.AUTOMATION) {
+    if (webhook.action.type === WebhookActionType.AUTOMATION) {
-    let automation = (await db.get(webhook.action.target)) as Automation
+      let automation = (await db.get(webhook.action.target)) as Automation
-    const autoOutputs = automation.definition.trigger.schema.outputs
+      const autoOutputs = automation.definition.trigger.schema.outputs
-    let properties = webhook.bodySchema.properties
+      let properties = webhook.bodySchema.properties
-    // reset webhook outputs
+      // reset webhook outputs
-    autoOutputs.properties = {
+      autoOutputs.properties = {
-      body: autoOutputs.properties.body,
+        body: autoOutputs.properties.body,
-    }
-    for (let prop of Object.keys(properties)) {
-      autoOutputs.properties[prop] = {
-        type: properties[prop].type,
-        description: AUTOMATION_DESCRIPTION,
       }
+      for (let prop of Object.keys(properties)) {
+        autoOutputs.properties[prop] = {
+          type: properties[prop].type,
+          description: AUTOMATION_DESCRIPTION,
+        }
+      }
+      await db.put(automation)
     }
-    await db.put(automation)
+    ctx.body = await db.put(webhook)
-  }
+  })
-  ctx.body = await db.put(webhook)
 }
 
 export async function trigger(ctx: BBContext) {
   const prodAppId = dbCore.getProdAppID(ctx.params.instance)
-  await context.updateAppId(prodAppId)
+  await context.doInAppContext(prodAppId, async () => {
-  try {
+    try {
-    const db = context.getAppDB()
+      const db = context.getAppDB()
-    const webhook = (await db.get(ctx.params.id)) as Webhook
+      const webhook = (await db.get(ctx.params.id)) as Webhook
-    // validate against the schema
+      // validate against the schema
-    if (webhook.bodySchema) {
+      if (webhook.bodySchema) {
-      validate(ctx.request.body, webhook.bodySchema)
+        validate(ctx.request.body, webhook.bodySchema)
-    }
+      }
-    const target = await db.get(webhook.action.target)
+      const target = await db.get(webhook.action.target)
-    if (webhook.action.type === WebhookActionType.AUTOMATION) {
+      if (webhook.action.type === WebhookActionType.AUTOMATION) {
-      // trigger with both the pure request and then expand it
+        // trigger with both the pure request and then expand it
-      // incase the user has produced a schema to bind to
+        // incase the user has produced a schema to bind to
-      await triggers.externalTrigger(target, {
+        await triggers.externalTrigger(target, {
-        body: ctx.request.body,
+          body: ctx.request.body,
-        ...ctx.request.body,
+          ...ctx.request.body,
-        appId: prodAppId,
+          appId: prodAppId,
-      })
+        })
-    }
+      }
-    ctx.status = 200
-    ctx.body = {
-      message: "Webhook trigger fired successfully",
-    }
-  } catch (err: any) {
-    if (err.status === 404) {
       ctx.status = 200
       ctx.body = {
-        message: "Application not deployed yet.",
+        message: "Webhook trigger fired successfully",
+      }
+    } catch (err: any) {
+      if (err.status === 404) {
+        ctx.status = 200
+        ctx.body = {
+          message: "Application not deployed yet.",
+        }
       }
     }
-  }
+  })
 }

packages/server/src/api/routes/index.ts

@@ -33,6 +33,7 @@ export { default as publicRoutes } from "./public"
 
 const appBackupRoutes = pro.appBackups
 const scheduleRoutes = pro.schedules
+const environmentVariableRoutes = pro.environmentVariables
 
 export const mainRoutes: Router[] = [
   appBackupRoutes,
@@ -63,6 +64,7 @@ export const mainRoutes: Router[] = [
   migrationRoutes,
   pluginRoutes,
   scheduleRoutes,
+  environmentVariableRoutes,
   // these need to be handled last as they still use /api/:tableId
   // this could be breaking as koa may recognise other routes as this
   tableRoutes,

packages/server/src/api/routes/row.ts

@@ -3,7 +3,7 @@ import * as rowController from "../controllers/row"
 import authorized from "../../middleware/authorized"
 import { paramResource, paramSubResource } from "../../middleware/resourceId"
 import { permissions } from "@budibase/backend-core"
-const { internalSearchValidator } = require("./utils/validators")
+import { internalSearchValidator } from "./utils/validators"
 const { PermissionType, PermissionLevel } = permissions
 
 const router: Router = new Router()

packages/server/src/api/routes/tests/datasource.spec.ts

@@ -2,7 +2,8 @@ jest.mock("pg")
 import * as setup from "./utilities"
 import { checkBuilderEndpoint } from "./utilities/TestFunctions"
 import { checkCacheForDynamicVariable } from "../../../threads/utils"
-import { events } from "@budibase/backend-core"
+import { context, events } from "@budibase/backend-core"
+import sdk from "../../../sdk"
 
 let { basicDatasource } = setup.structures
 const pg = require("pg")
@@ -184,4 +185,37 @@ describe("/datasources", () => {
       })
     })
   })
+
+  describe("check secret replacement", () => {
+    async function makeDatasource() {
+      datasource = basicDatasource()
+      datasource.datasource.config.password = "testing"
+      const res = await request
+        .post(`/api/datasources`)
+        .send(datasource)
+        .set(config.defaultHeaders())
+        .expect("Content-Type", /json/)
+        .expect(200)
+      return res.body.datasource
+    }
+
+    it("should save a datasource with password", async () => {
+      const datasource = await makeDatasource()
+      expect(datasource.config.password).toBe("--secret-value--")
+    })
+
+    it("should not the password on update with the --secret-value--", async () => {
+      const datasource = await makeDatasource()
+      await request
+        .put(`/api/datasources/${datasource._id}`)
+        .send(datasource)
+        .set(config.defaultHeaders())
+        .expect("Content-Type", /json/)
+        .expect(200)
+      await context.doInAppContext(config.getAppId(), async () => {
+        const dbDatasource: any = await sdk.datasources.get(datasource._id)
+        expect(dbDatasource.config.password).toBe("testing")
+      })
+    })
+  })
 })

packages/server/src/automations/tests/loop.spec.ts

@@ -0,0 +1,45 @@
+import * as automation from "../index"
+import * as triggers from "../triggers"
+import { loopAutomation } from "../../tests/utilities/structures"
+import { context } from "@budibase/backend-core"
+import * as setup from "./utilities"
+
+describe("Attempt to run a basic loop automation", () => {
+  let config = setup.getConfig(),
+    table: any,
+    row: any
+
+  beforeEach(async () => {
+    await automation.init()
+    await config.init()
+    table = await config.createTable()
+    row = await config.createRow()
+  })
+
+  afterAll(setup.afterAll)
+
+  async function runLoop(loopOpts?: any) {
+    const appId = config.getAppId()
+    return await context.doInAppContext(appId, async () => {
+      const params = { fields: { appId } }
+      return await triggers.externalTrigger(
+        loopAutomation(table._id, loopOpts),
+        params,
+        { getResponses: true }
+      )
+    })
+  }
+
+  it("attempt to run a basic loop", async () => {
+    const resp = await runLoop()
+    expect(resp.steps[2].outputs.iterations).toBe(1)
+  })
+
+  it("test a loop with a string", async () => {
+    const resp = await runLoop({
+      type: "String",
+      binding: "a,b,c",
+    })
+    expect(resp.steps[2].outputs.iterations).toBe(3)
+  })
+})

packages/server/src/automations/triggers.ts

@@ -109,8 +109,13 @@ export async function externalTrigger(
     }
     params.fields = coercedFields
   }
-  const data = { automation, event: params }
+  const data: Record<string, any> = { automation, event: params }
   if (getResponses) {
+    data.event = {
+      ...data.event,
+      appId: context.getAppId(),
+      automation,
+    }
     return utils.processEvent({ data })
   } else {
     return automationQueue.add(data, JOB_OPTS)

packages/server/src/definitions/automations.ts

@@ -24,6 +24,7 @@ export interface TriggerOutput {
 
 export interface AutomationContext extends AutomationResults {
   steps: any[]
+  env?: Record<string, string>
   trigger: any
 }
 

packages/server/src/index.ts

@@ -1,6 +1,6 @@
 import { bootstrap } from "global-agent"
 const fixPath = require("fix-path")
-const { checkDevelopmentEnvironment } = require("./utilities/fileSystem")
+import { checkDevelopmentEnvironment } from "./utilities/fileSystem"
 
 function runServer() {
   // this will shutdown the system if development environment not ready

packages/server/src/integrations/base/query.ts

@@ -1,10 +1,12 @@
 import { QueryJson, Datasource } from "@budibase/types"
-const { getIntegration } = require("../index")
+import { getIntegration } from "../index"
+import sdk from "../../sdk"
 
 export async function makeExternalQuery(
   datasource: Datasource,
   json: QueryJson
 ) {
+  datasource = await sdk.datasources.enrich(datasource)
   const Integration = await getIntegration(datasource.source)
   // query is the opinionated function
   if (Integration.prototype.query) {

packages/server/src/integrations/base/sqlTable.ts

@@ -3,7 +3,7 @@ import { Operation, QueryJson, RenameColumn, Table } from "@budibase/types"
 import { breakExternalTableId } from "../utils"
 import SchemaBuilder = Knex.SchemaBuilder
 import CreateTableBuilder = Knex.CreateTableBuilder
-const { FieldTypes, RelationshipTypes } = require("../../constants")
+import { FieldTypes, RelationshipTypes } from "../../constants"
 
 function generateSchema(
   schema: CreateTableBuilder,

packages/server/src/integrations/dynamodb.ts

@@ -5,8 +5,8 @@ import {
   IntegrationBase,
 } from "@budibase/types"
 
-const AWS = require("aws-sdk")
+import AWS from "aws-sdk"
-const { AWS_REGION } = require("../db/dynamoClient")
+import { AWS_REGION } from "../db/dynamoClient"
 
 interface DynamoDBConfig {
   region: string
@@ -182,7 +182,7 @@ class DynamoDBIntegration implements IntegrationBase {
     return response
   }
 
-  async describe(query: { table: string }) {
+  async describe(query: { table: string }): Promise<any> {
     const params = {
       TableName: query.table,
     }

packages/server/src/integrations/mongodb.ts

@@ -12,11 +12,16 @@ import {
   FindOneAndUpdateOptions,
   UpdateOptions,
   OperationOptions,
+  MongoClientOptions,
 } from "mongodb"
+import environment from "../environment"
 
 interface MongoDBConfig {
   connectionString: string
   db: string
+  tlsCertificateFile: string
+  tlsCertificateKeyFile: string
+  tlsCAFile: string
 }
 
 interface MongoDBQuery {
@@ -26,292 +31,331 @@ interface MongoDBQuery {
   }
 }
 
-const SCHEMA: Integration = {
+const getSchema = () => {
-  docs: "https://github.com/mongodb/node-mongodb-native",
+  let schema = {
-  friendlyName: "MongoDB",
+    docs: "https://github.com/mongodb/node-mongodb-native",
-  type: "Non-relational",
+    friendlyName: "MongoDB",
-  description:
+    type: "Non-relational",
-    "MongoDB is a general purpose, document-based, distributed database built for modern application developers and for the cloud era.",
+    description:
-  datasource: {
+      "MongoDB is a general purpose, document-based, distributed database built for modern application developers and for the cloud era.",
-    connectionString: {
+    datasource: {
-      type: DatasourceFieldType.STRING,
+      connectionString: {
-      required: true,
+        type: DatasourceFieldType.STRING,
-      default: "mongodb://localhost:27017",
+        required: true,
-    },
+        default: "mongodb://localhost:27017",
-    db: {
+        display: "Connection string",
-      type: DatasourceFieldType.STRING,
+      },
-      required: true,
+      db: {
-    },
+        type: DatasourceFieldType.STRING,
-  },
+        required: true,
-  query: {
+        display: "DB",
-    create: {
-      type: QueryType.JSON,
-    },
-    read: {
-      type: QueryType.JSON,
-    },
-    update: {
-      type: QueryType.JSON,
-    },
-    delete: {
-      type: QueryType.JSON,
-    },
-    aggregate: {
-      type: QueryType.JSON,
-      readable: true,
-      steps: [
-        {
-          key: "$addFields",
-          template: "{\n\t\n}",
-        },
-        {
-          key: "$bucket",
-          template: `{
-  "groupBy": "",
-  "boundaries": [],
-  "default": "",
-  "output": {}
-}`,
-        },
-        {
-          key: "$bucketAuto",
-          template: `{
-  "groupBy": "",
-  "buckets": 1,
-  "output": {},
-  "granularity": "R5"
-}`,
-        },
-        {
-          key: "$changeStream",
-          template: `{
-  "allChangesForCluster": true,
-  "fullDocument": "",
-  "fullDocumentBeforeChange": "",
-  "resumeAfter": 1,
-  "showExpandedEvents": true,
-  "startAfter": {},
-  "startAtOperationTime": ""
-}`,
-        },
-        {
-          key: "$collStats",
-          template: `{
-  "latencyStats": { "histograms": true } },
-  "storageStats": { "scale": 1 } },
-  "count": {},
-  "queryExecStats": {}
-}`,
-        },
-        {
-          key: "$count",
-          template: ``,
-        },
-        {
-          key: "$densify",
-          template: `{
-  "field": "",
-  "partitionByFields": [],
-  "range": {
-    "step": 1,
-    "unit": 1,
-    "bounds": "full"
-  }
-}`,
-        },
-        {
-          key: "$documents",
-          template: `[]`,
-        },
-        {
-          key: "$facet",
-          template: `{\n\t\n}`,
-        },
-        {
-          key: "$fill",
-          template: `{
-  "partitionBy": "",
-  "partitionByFields": [],
-  "sortBy": {},
-  "output": {}
-}`,
-        },
-        {
-          key: "$geoNear",
-          template: `{
-  "near": { 
-    "type": "Point", 
-    "coordinates": [ 
-      -73.98142, 40.71782
-    ] 
-  },
-  "key": "location",
-  "distanceField": "dist.calculated",
-  "query": { "category": "Parks" }
-}`,
-        },
-        {
-          key: "$graphLookup",
-          template: `{
-  "from": "",
-  "startWith": "",
-  "connectFromField": "",
-  "connectToField": "",
-  "as": "",
-  "maxDepth": 1,
-  "depthField": "",
-  "restrictSearchWithMatch": {}
-}`,
-        },
-        {
-          key: "$group",
-          template: `{
-  "_id": ""
-}`,
-        },
-        {
-          key: "$indexStats",
-          template: "{\n\t\n}",
-        },
-        {
-          key: "$limit",
-          template: `1`,
-        },
-        {
-          key: "$listLocalSessions",
-          template: `{\n\t\n}`,
-        },
-        {
-          key: "$listSessions",
-          template: `{\n\t\n}`,
-        },
-        {
-          key: "$lookup",
-          template: `{
-  "from": "",
-  "localField": "",
-  "foreignField": "",
-  "as": ""
-}`,
-        },
-        {
-          key: "$match",
-          template: "{\n\t\n}",
-        },
-        {
-          key: "$merge",
-          template: `{
-  "into": {},
-  "on": "_id",
-  "whenMatched": "replace",
-  "whenNotMatched": "insert"
-}`,
-        },
-        {
-          key: "$out",
-          template: `{
-  "db": "",
-  "coll": ""
-}`,
-        },
-        {
-          key: "$planCacheStats",
-          template: "{\n\t\n}",
-        },
-        {
-          key: "$project",
-          template: "{\n\t\n}",
-        },
-        {
-          key: "$redact",
-          template: "",
-        },
-        {
-          key: "$replaceRoot",
-          template: `{ "newRoot": "" }`,
-        },
-        {
-          key: "$replaceWith",
-          template: ``,
-        },
-        {
-          key: "$sample",
-          template: `{ "size": 3 }`,
-        },
-        {
-          key: "$set",
-          template: "{\n\t\n}",
-        },
-        {
-          key: "$setWindowFields",
-          template: `{
-  "partitionBy": "",
-  "sortBy": {},
-  "output": {}
-}`,
-        },
-        {
-          key: "$skip",
-          template: `1`,
-        },
-        {
-          key: "$sort",
-          template: "{\n\t\n}",
-        },
-        {
-          key: "$sortByCount",
-          template: "",
-        },
-        {
-          key: "$unionWith",
-          template: `{
-  "coll": "",
-  "pipeline": []
-}`,
-        },
-        {
-          key: "$unset",
-          template: "",
-        },
-        {
-          key: "$unwind",
-          template: `{
-  "path": "",
-  "includeArrayIndex": "",
-  "preserveNullAndEmptyArrays": true
-}`,
-        },
-      ],
-    },
-  },
-  extra: {
-    collection: {
-      displayName: "Collection",
-      type: DatasourceFieldType.STRING,
-      required: true,
-    },
-    actionType: {
-      displayName: "Query Type",
-      type: DatasourceFieldType.LIST,
-      required: true,
-      data: {
-        read: ["find", "findOne", "findOneAndUpdate", "count", "distinct"],
-        create: ["insertOne", "insertMany"],
-        update: ["updateOne", "updateMany"],
-        delete: ["deleteOne", "deleteMany"],
-        aggregate: ["json", "pipeline"],
       },
     },
-  },
+    query: {
+      create: {
+        type: QueryType.JSON,
+      },
+      read: {
+        type: QueryType.JSON,
+      },
+      update: {
+        type: QueryType.JSON,
+      },
+      delete: {
+        type: QueryType.JSON,
+      },
+      aggregate: {
+        type: QueryType.JSON,
+        readable: true,
+        steps: [
+          {
+            key: "$addFields",
+            template: "{\n\t\n}",
+          },
+          {
+            key: "$bucket",
+            template: `{
+    "groupBy": "",
+    "boundaries": [],
+    "default": "",
+    "output": {}
+  }`,
+          },
+          {
+            key: "$bucketAuto",
+            template: `{
+    "groupBy": "",
+    "buckets": 1,
+    "output": {},
+    "granularity": "R5"
+  }`,
+          },
+          {
+            key: "$changeStream",
+            template: `{
+    "allChangesForCluster": true,
+    "fullDocument": "",
+    "fullDocumentBeforeChange": "",
+    "resumeAfter": 1,
+    "showExpandedEvents": true,
+    "startAfter": {},
+    "startAtOperationTime": ""
+  }`,
+          },
+          {
+            key: "$collStats",
+            template: `{
+    "latencyStats": { "histograms": true } },
+    "storageStats": { "scale": 1 } },
+    "count": {},
+    "queryExecStats": {}
+  }`,
+          },
+          {
+            key: "$count",
+            template: ``,
+          },
+          {
+            key: "$densify",
+            template: `{
+    "field": "",
+    "partitionByFields": [],
+    "range": {
+      "step": 1,
+      "unit": 1,
+      "bounds": "full"
+    }
+  }`,
+          },
+          {
+            key: "$documents",
+            template: `[]`,
+          },
+          {
+            key: "$facet",
+            template: `{\n\t\n}`,
+          },
+          {
+            key: "$fill",
+            template: `{
+    "partitionBy": "",
+    "partitionByFields": [],
+    "sortBy": {},
+    "output": {}
+  }`,
+          },
+          {
+            key: "$geoNear",
+            template: `{
+    "near": { 
+      "type": "Point", 
+      "coordinates": [ 
+        -73.98142, 40.71782
+      ] 
+    },
+    "key": "location",
+    "distanceField": "dist.calculated",
+    "query": { "category": "Parks" }
+  }`,
+          },
+          {
+            key: "$graphLookup",
+            template: `{
+    "from": "",
+    "startWith": "",
+    "connectFromField": "",
+    "connectToField": "",
+    "as": "",
+    "maxDepth": 1,
+    "depthField": "",
+    "restrictSearchWithMatch": {}
+  }`,
+          },
+          {
+            key: "$group",
+            template: `{
+    "_id": ""
+  }`,
+          },
+          {
+            key: "$indexStats",
+            template: "{\n\t\n}",
+          },
+          {
+            key: "$limit",
+            template: `1`,
+          },
+          {
+            key: "$listLocalSessions",
+            template: `{\n\t\n}`,
+          },
+          {
+            key: "$listSessions",
+            template: `{\n\t\n}`,
+          },
+          {
+            key: "$lookup",
+            template: `{
+    "from": "",
+    "localField": "",
+    "foreignField": "",
+    "as": ""
+  }`,
+          },
+          {
+            key: "$match",
+            template: "{\n\t\n}",
+          },
+          {
+            key: "$merge",
+            template: `{
+    "into": {},
+    "on": "_id",
+    "whenMatched": "replace",
+    "whenNotMatched": "insert"
+  }`,
+          },
+          {
+            key: "$out",
+            template: `{
+    "db": "",
+    "coll": ""
+  }`,
+          },
+          {
+            key: "$planCacheStats",
+            template: "{\n\t\n}",
+          },
+          {
+            key: "$project",
+            template: "{\n\t\n}",
+          },
+          {
+            key: "$redact",
+            template: "",
+          },
+          {
+            key: "$replaceRoot",
+            template: `{ "newRoot": "" }`,
+          },
+          {
+            key: "$replaceWith",
+            template: ``,
+          },
+          {
+            key: "$sample",
+            template: `{ "size": 3 }`,
+          },
+          {
+            key: "$set",
+            template: "{\n\t\n}",
+          },
+          {
+            key: "$setWindowFields",
+            template: `{
+    "partitionBy": "",
+    "sortBy": {},
+    "output": {}
+  }`,
+          },
+          {
+            key: "$skip",
+            template: `1`,
+          },
+          {
+            key: "$sort",
+            template: "{\n\t\n}",
+          },
+          {
+            key: "$sortByCount",
+            template: "",
+          },
+          {
+            key: "$unionWith",
+            template: `{
+    "coll": "",
+    "pipeline": []
+  }`,
+          },
+          {
+            key: "$unset",
+            template: "",
+          },
+          {
+            key: "$unwind",
+            template: `{
+    "path": "",
+    "includeArrayIndex": "",
+    "preserveNullAndEmptyArrays": true
+  }`,
+          },
+        ],
+      },
+    },
+    extra: {
+      collection: {
+        displayName: "Collection",
+        type: DatasourceFieldType.STRING,
+        required: true,
+      },
+      actionType: {
+        displayName: "Query Type",
+        type: DatasourceFieldType.LIST,
+        required: true,
+        data: {
+          read: ["find", "findOne", "findOneAndUpdate", "count", "distinct"],
+          create: ["insertOne", "insertMany"],
+          update: ["updateOne", "updateMany"],
+          delete: ["deleteOne", "deleteMany"],
+          aggregate: ["json", "pipeline"],
+        },
+      },
+    },
+  }
+  if (environment.SELF_HOSTED) {
+    schema.datasource = {
+      ...schema.datasource,
+      //@ts-ignore
+      tls: {
+        type: DatasourceFieldType.FIELD_GROUP,
+        display: "Configure SSL",
+        fields: {
+          tlsCertificateFile: {
+            type: DatasourceFieldType.STRING,
+            required: false,
+            display: "Certificate file path",
+          },
+          tlsCertificateKeyFile: {
+            type: DatasourceFieldType.STRING,
+            required: false,
+            display: "Certificate Key file path",
+          },
+          tlsCAFile: {
+            type: DatasourceFieldType.STRING,
+            required: false,
+            display: "CA file path",
+          },
+        },
+      },
+    }
+  }
+  return schema
 }
 
+const SCHEMA: Integration = getSchema()
+
 class MongoIntegration implements IntegrationBase {
   private config: MongoDBConfig
   private client: any
 
   constructor(config: MongoDBConfig) {
     this.config = config
-    this.client = new MongoClient(config.connectionString)
+    const options: MongoClientOptions = {
+      tlsCertificateFile: config.tlsCertificateFile || undefined,
+      tlsCertificateKeyFile: config.tlsCertificateKeyFile || undefined,
+      tlsCAFile: config.tlsCAFile || undefined,
+    }
+    this.client = new MongoClient(config.connectionString, options)
   }
 
   async connect() {

packages/server/src/integrations/mysql.ts

@@ -16,7 +16,7 @@ import {
   finaliseExternalTables,
 } from "./utils"
 import dayjs from "dayjs"
-const { NUMBER_REGEX } = require("../utilities")
+import { NUMBER_REGEX } from "../utilities"
 import Sql from "./base/sql"
 import { MySQLColumn } from "./base/types"
 

packages/server/src/integrations/queries/sql.ts

@@ -1,55 +1,10 @@
-import { findHBSBlocks, processStringSync } from "@budibase/string-templates"
+import { findHBSBlocks } from "@budibase/string-templates"
 import { DatasourcePlus } from "@budibase/types"
+import sdk from "../../sdk"
 
 const CONST_CHAR_REGEX = new RegExp("'[^']*'", "g")
 
-export function enrichQueryFields(
+export async function interpolateSQL(
-  fields: { [key: string]: any },
-  parameters = {}
-) {
-  const enrichedQuery: { [key: string]: any } = Array.isArray(fields) ? [] : {}
-  if (!fields || !parameters) {
-    return enrichedQuery
-  }
-  // enrich the fields with dynamic parameters
-  for (let key of Object.keys(fields)) {
-    if (fields[key] == null) {
-      continue
-    }
-    if (typeof fields[key] === "object") {
-      // enrich nested fields object
-      enrichedQuery[key] = enrichQueryFields(fields[key], parameters)
-    } else if (typeof fields[key] === "string") {
-      // enrich string value as normal
-      enrichedQuery[key] = processStringSync(fields[key], parameters, {
-        noEscaping: true,
-        noHelpers: true,
-        escapeNewlines: true,
-      })
-    } else {
-      enrichedQuery[key] = fields[key]
-    }
-  }
-  if (
-    enrichedQuery.json ||
-    enrichedQuery.customData ||
-    enrichedQuery.requestBody
-  ) {
-    try {
-      enrichedQuery.json = JSON.parse(
-        enrichedQuery.json ||
-          enrichedQuery.customData ||
-          enrichedQuery.requestBody
-      )
-    } catch (err) {
-      // no json found, ignore
-    }
-    delete enrichedQuery.customData
-  }
-  return enrichedQuery
-}
-
-export function interpolateSQL(
   fields: { [key: string]: any },
   parameters: { [key: string]: any },
   integration: DatasourcePlus
@@ -90,7 +45,7 @@ export function interpolateSQL(
     else if (listRegexMatch) {
       arrays.push(binding)
       // determine the length of the array
-      const value = enrichQueryFields([binding], parameters)[0]
+      const value = (await sdk.queries.enrichContext([binding], parameters))[0]
         .split(",")
         .map((val: string) => val.trim())
       // build a string like ($1, $2, $3)
@@ -109,7 +64,7 @@ export function interpolateSQL(
   }
   // replicate the knex structure
   fields.sql = sql
-  fields.bindings = enrichQueryFields(variables, parameters)
+  fields.bindings = await sdk.queries.enrichContext(variables, parameters)
   // check for arrays in the data
   let updated: string[] = []
   for (let i = 0; i < variables.length; i++) {

packages/server/src/integrations/rest.ts

@@ -16,11 +16,11 @@ import {
 import { get } from "lodash"
 import * as https from "https"
 import qs from "querystring"
-const fetch = require("node-fetch")
+import fetch from "node-fetch"
-const { formatBytes } = require("../utilities")
+import { formatBytes } from "../utilities"
-const { performance } = require("perf_hooks")
+import { performance } from "perf_hooks"
-const FormData = require("form-data")
+import FormData from "form-data"
-const { URLSearchParams } = require("url")
+import { URLSearchParams } from "url"
 
 const BodyTypes = {
   NONE: "none",
@@ -204,12 +204,12 @@ class RestIntegration implements IntegrationBase {
 
       // Append page number or cursor param if configured
       if (pageParam && paginationValues.page != null) {
-        params.append(pageParam, paginationValues.page)
+        params.append(pageParam, paginationValues.page as string)
       }
 
       // Append page size param if configured
       if (sizeParam && paginationValues.limit != null) {
-        params.append(sizeParam, paginationValues.limit)
+        params.append(sizeParam, String(paginationValues.limit))
       }
 
       // Prepend query string with pagination params
@@ -280,7 +280,7 @@ class RestIntegration implements IntegrationBase {
       case BodyTypes.ENCODED:
         const params = new URLSearchParams()
         for (let [key, value] of Object.entries(object)) {
-          params.append(key, value)
+          params.append(key, value as string)
         }
         addPaginationToBody((key: string, value: any) => {
           params.append(key, value)

packages/server/src/migrations/functions/usageQuotas/tests/syncRows.spec.ts

@@ -2,7 +2,7 @@ import TestConfig from "../../../../tests/utilities/TestConfiguration"
 import * as syncRows from "../syncRows"
 import { quotas } from "@budibase/pro"
 import { QuotaUsageType, StaticQuotaName } from "@budibase/types"
-const { db: dbCore } = require("@budibase/backend-core")
+import { db as dbCore, context } from "@budibase/backend-core"
 
 describe("syncRows", () => {
   let config = new TestConfig(false)
@@ -24,13 +24,17 @@ describe("syncRows", () => {
 
       // app 1
       const app1 = config.app
-      await config.createTable()
+      await context.doInAppContext(app1.appId, async () => {
-      await config.createRow()
+        await config.createTable()
+        await config.createRow()
+      })
       // app 2
       const app2 = await config.createApp("second-app")
-      await config.createTable()
+      await context.doInAppContext(app2.appId, async () => {
-      await config.createRow()
+        await config.createTable()
-      await config.createRow()
+        await config.createRow()
+        await config.createRow()
+      })
 
       // migrate
       await syncRows.run()

packages/server/src/sdk/app/datasources/datasources.ts

@@ -0,0 +1,98 @@
+import { context } from "@budibase/backend-core"
+import { processObjectSync, findHBSBlocks } from "@budibase/string-templates"
+import {
+  Datasource,
+  DatasourceFieldType,
+  Integration,
+  PASSWORD_REPLACEMENT,
+} from "@budibase/types"
+import { cloneDeep } from "lodash/fp"
+import { getEnvironmentVariables } from "../../utils"
+import { getDefinitions } from "../../../integrations"
+
+const ENV_VAR_PREFIX = "env."
+const USER_PREFIX = "user"
+
+async function enrichDatasourceWithValues(datasource: Datasource) {
+  const cloned = cloneDeep(datasource)
+  const env = await getEnvironmentVariables()
+  const processed = processObjectSync(cloned, { env }, { onlyFound: true })
+  return {
+    datasource: processed as Datasource,
+    envVars: env as Record<string, string>,
+  }
+}
+
+export async function enrich(datasource: Datasource) {
+  const { datasource: response } = await enrichDatasourceWithValues(datasource)
+  return response
+}
+
+export async function get(
+  datasourceId: string,
+  opts?: { enriched: boolean }
+): Promise<Datasource> {
+  const appDb = context.getAppDB()
+  const datasource = await appDb.get(datasourceId)
+  if (opts?.enriched) {
+    return (await enrichDatasourceWithValues(datasource)).datasource
+  } else {
+    return datasource
+  }
+}
+
+export async function getWithEnvVars(datasourceId: string) {
+  const appDb = context.getAppDB()
+  const datasource = await appDb.get(datasourceId)
+  return enrichDatasourceWithValues(datasource)
+}
+
+export async function removeSecrets(datasources: Datasource[]) {
+  const definitions = await getDefinitions()
+  for (let datasource of datasources) {
+    const schema = definitions[datasource.source]
+    if (datasource.config) {
+      // strip secrets from response, so they don't show in the network request
+      if (datasource.config.auth) {
+        delete datasource.config.auth
+      }
+      // remove passwords
+      for (let key of Object.keys(datasource.config)) {
+        if (typeof datasource.config[key] !== "string") {
+          continue
+        }
+        const blocks = findHBSBlocks(datasource.config[key] as string)
+        const usesEnvVars =
+          blocks.find(block => block.includes(ENV_VAR_PREFIX)) != null
+        if (
+          !usesEnvVars &&
+          schema.datasource?.[key]?.type === DatasourceFieldType.PASSWORD
+        ) {
+          datasource.config[key] = PASSWORD_REPLACEMENT
+        }
+      }
+    }
+  }
+  return datasources
+}
+
+export async function removeSecretSingle(datasource: Datasource) {
+  return (await removeSecrets([datasource]))[0]
+}
+
+export function mergeConfigs(update: Datasource, old: Datasource) {
+  if (!update.config) {
+    return update
+  }
+  for (let [key, value] of Object.entries(update.config)) {
+    if (value !== PASSWORD_REPLACEMENT) {
+      continue
+    }
+    if (old.config?.[key]) {
+      update.config[key] = old.config?.[key]
+    } else {
+      delete update.config[key]
+    }
+  }
+  return update
+}

packages/server/src/sdk/app/datasources/index.ts

@@ -0,0 +1,5 @@
+import * as datasources from "./datasources"
+
+export default {
+  ...datasources,
+}

packages/server/src/sdk/app/queries/index.ts

@@ -0,0 +1,5 @@
+import * as queries from "./queries"
+
+export default {
+  ...queries,
+}