Merge branch 'develop' of github.com:Budibase/budibase into feature/audit-logs

2023-02-13 11:38:53 +00:00 · 2023-02-13 11:38:53 +00:00 · b9c513c071
parent 73c99910b1 5747963072
commit b9c513c071
60 changed files with 1327 additions and 278 deletions
--- a/.github/ISSUE_TEMPLATE/bug_report.md
+++ b/.github/ISSUE_TEMPLATE/bug_report.md
@ -2,7 +2,7 @@
 name: Bug report
 about: Create a report to help us improve
 title: ''
-labels: ["bug", "linear"]
+labels: bug
 assignees: ''

 ---
--- a/.github/ISSUE_TEMPLATE/epic.md
+++ b/.github/ISSUE_TEMPLATE/epic.md
@ -1,24 +0,0 @@
---
-name: Epic
-about: Plan a new project
-title: ''
-labels: epic
-assignees: ''
-
---
-
-## Description
-Brief summary of what this Epic is, whether it's a larger project, goal, or user story. Describe the job to be done, which persona this Epic is mainly for, or if more multiple, break it down by user and job story.
-
-## Spec
-Link to confluence spec
-
-## Teams and Stakeholders
-Describe who needs to be kept up-to-date about this Epic, included in discussions, or updated along the way. Stakeholders can be both in Product/Engineering, as well as other teams like Customer Success who might want to keep customers updated on the Epic project.
-
-
-## Workflow
- [ ]  Spec Created and pasted above
- [ ]  Product Review
- [ ]  Designs created
- [ ]  Individual Tasks created and assigned to Epic
--- a/.github/ISSUE_TEMPLATE/feature_request.md
+++ b/.github/ISSUE_TEMPLATE/feature_request.md
@ -2,7 +2,7 @@
 name: Feature Request
 about: Request a new budibase feature or enhancement
 title: ''
-labels: ["enhancement", "linear"]
+labels: enhancement
 assignees: ''

 ---
--- a/hosting/proxy/nginx.prod.conf
+++ b/hosting/proxy/nginx.prod.conf
@ -55,7 +55,7 @@ http {
    set $csp_style "style-src 'self' 'unsafe-inline' https://cdn.jsdelivr.net https://fonts.googleapis.com https://rsms.me https://maxcdn.bootstrapcdn.com";
    set $csp_object "object-src 'none'";
    set $csp_base_uri "base-uri 'self'";
-    set $csp_connect "connect-src 'self' https://*.budibase.net https://api-iam.intercom.io https://api-iam.intercom.io  https://api-ping.intercom.io https://app.posthog.com wss://nexus-websocket-a.intercom.io  wss://nexus-websocket-b.intercom.io https://nexus-websocket-a.intercom.io https://nexus-websocket-b.intercom.io  https://uploads.intercomcdn.com  https://uploads.intercomusercontent.com https://*.s3.amazonaws.com https://*.s3.us-east-2.amazonaws.com https://*.s3.us-east-1.amazonaws.com https://*.s3.us-west-1.amazonaws.com https://*.s3.us-west-2.amazonaws.com https://*.s3.af-south-1.amazonaws.com https://*.s3.ap-east-1.amazonaws.com https://*.s3.ap-southeast-3.amazonaws.com https://*.s3.ap-south-1.amazonaws.com https://*.s3.ap-northeast-3.amazonaws.com https://*.s3.ap-northeast-2.amazonaws.com https://*.s3.ap-southeast-1.amazonaws.com https://*.s3.ap-southeast-2.amazonaws.com https://*.s3.ap-northeast-1.amazonaws.com https://*.s3.ca-central-1.amazonaws.com https://*.s3.cn-north-1.amazonaws.com https://*.s3.cn-northwest-1.amazonaws.com https://*.s3.eu-central-1.amazonaws.com https://*.s3.eu-west-1.amazonaws.com https://*.s3.eu-west-2.amazonaws.com https://*.s3.eu-south-1.amazonaws.com https://*.s3.eu-west-3.amazonaws.com https://*.s3.eu-north-1.amazonaws.com https://*.s3.sa-east-1.amazonaws.com https://*.s3.me-south-1.amazonaws.com https://*.s3.us-gov-east-1.amazonaws.com https://*.s3.us-gov-west-1.amazonaws.com";
+    set $csp_connect "connect-src 'self' https://*.budibase.net https://api-iam.intercom.io https://api-iam.intercom.io  https://api-ping.intercom.io https://app.posthog.com wss://nexus-websocket-a.intercom.io  wss://nexus-websocket-b.intercom.io https://nexus-websocket-a.intercom.io https://nexus-websocket-b.intercom.io  https://uploads.intercomcdn.com  https://uploads.intercomusercontent.com https://*.s3.amazonaws.com https://*.s3.us-east-2.amazonaws.com https://*.s3.us-east-1.amazonaws.com https://*.s3.us-west-1.amazonaws.com https://*.s3.us-west-2.amazonaws.com https://*.s3.af-south-1.amazonaws.com https://*.s3.ap-east-1.amazonaws.com https://*.s3.ap-southeast-3.amazonaws.com https://*.s3.ap-south-1.amazonaws.com https://*.s3.ap-northeast-3.amazonaws.com https://*.s3.ap-northeast-2.amazonaws.com https://*.s3.ap-southeast-1.amazonaws.com https://*.s3.ap-southeast-2.amazonaws.com https://*.s3.ap-northeast-1.amazonaws.com https://*.s3.ca-central-1.amazonaws.com https://*.s3.cn-north-1.amazonaws.com https://*.s3.cn-northwest-1.amazonaws.com https://*.s3.eu-central-1.amazonaws.com https://*.s3.eu-west-1.amazonaws.com https://*.s3.eu-west-2.amazonaws.com https://*.s3.eu-south-1.amazonaws.com https://*.s3.eu-west-3.amazonaws.com https://*.s3.eu-north-1.amazonaws.com https://*.s3.sa-east-1.amazonaws.com https://*.s3.me-south-1.amazonaws.com https://*.s3.us-gov-east-1.amazonaws.com https://*.s3.us-gov-west-1.amazonaws.com https://api.github.com";
    set $csp_font "font-src 'self' data: https://cdn.jsdelivr.net https://fonts.gstatic.com https://rsms.me https://maxcdn.bootstrapcdn.com  https://js.intercomcdn.com  https://fonts.intercomcdn.com";
    set $csp_frame "frame-src 'self' https:";
    set $csp_img "img-src http: https: data: blob:";
--- a/lerna.json
+++ b/lerna.json
@ -1,5 +1,5 @@
 {
-  "version": "2.3.2-alpha.0",
+  "version": "2.3.11-alpha.0",
  "npmClient": "yarn",
  "packages": [
    "packages/*"
--- a/packages/backend-core/package.json
+++ b/packages/backend-core/package.json
@ -1,6 +1,6 @@
 {
  "name": "@budibase/backend-core",
-  "version": "2.3.2-alpha.0",
+  "version": "2.3.11-alpha.0",
  "description": "Budibase backend core libraries used in server and worker",
  "main": "dist/src/index.js",
  "types": "dist/src/index.d.ts",
@ -24,7 +24,7 @@
  "dependencies": {
    "@budibase/nano": "10.1.1",
    "@budibase/pouchdb-replication-stream": "1.2.10",
-    "@budibase/types": "2.3.2-alpha.0",
+    "@budibase/types": "2.3.11-alpha.0",
    "@shopify/jest-koa-mocks": "5.0.1",
    "@techpass/passport-openidconnect": "0.3.2",
    "aws-cloudfront-sign": "2.2.0",
--- a/packages/backend-core/tests/utilities/mocks/licenses.ts
+++ b/packages/backend-core/tests/utilities/mocks/licenses.ts
@ -74,6 +74,10 @@ export const useGroups = () => {
  return useFeature(Feature.USER_GROUPS)
 }

+export const useEnvironmentVariables = () => {
+  return useFeature(Feature.ENVIRONMENT_VARIABLES)
+}
+
 // QUOTAS

 export const setAutomationLogsQuota = (value: number) => {
--- a/packages/backend-core/tests/utilities/testContainerUtils.ts
+++ b/packages/backend-core/tests/utilities/testContainerUtils.ts
@ -1,4 +1,7 @@
-function getTestContainerSettings(serverName: string, key: string) {
+function getTestContainerSettings(
+  serverName: string,
+  key: string
+): string | null {
  const entry = Object.entries(global).find(
    ([k]) =>
      k.includes(`_${serverName.toUpperCase()}`) &&
@ -10,20 +13,25 @@ function getTestContainerSettings(serverName: string, key: string) {
  return entry[1]
 }

-function getCouchConfig() {
-  const port = getTestContainerSettings("COUCHDB-SERVICE", "PORT_5984")
+function getContainerInfo(containerName: string, port: number) {
+  const assignedPort = getTestContainerSettings(
+    containerName.toUpperCase(),
+    `PORT_${port}`
+  )
+  const host = getTestContainerSettings(containerName.toUpperCase(), "IP")
  return {
-    port,
-    url: `http://${getTestContainerSettings("COUCHDB-SERVICE", "IP")}:${port}`,
+    port: assignedPort,
+    host,
+    url: host && assignedPort && `http://${host}:${assignedPort}`,
  }
 }

+function getCouchConfig() {
+  return getContainerInfo("couchdb-service", 5984)
+}
+
 function getMinioConfig() {
-  const port = getTestContainerSettings("MINIO-SERVICE", "PORT_9000")
-  return {
-    port,
-    url: `http://${getTestContainerSettings("MINIO-SERVICE", "IP")}:${port}`,
-  }
+  return getContainerInfo("minio-service", 9000)
 }

 export function setupEnv(...envs: any[]) {
@ -34,7 +42,7 @@ export function setupEnv(...envs: any[]) {
    { key: "MINIO_URL", value: getMinioConfig().url },
  ]

-  for (const config of configs.filter(x => x.value !== null)) {
+  for (const config of configs.filter(x => !!x.value)) {
    for (const env of envs) {
      env._set(config.key, config.value)
    }
--- a/packages/bbui/package.json
+++ b/packages/bbui/package.json
@ -1,7 +1,7 @@
 {
  "name": "@budibase/bbui",
  "description": "A UI solution used in the different Budibase projects.",
-  "version": "2.3.2-alpha.0",
+  "version": "2.3.11-alpha.0",
  "license": "MPL-2.0",
  "svelte": "src/index.js",
  "module": "dist/bbui.es.js",
@ -38,7 +38,7 @@
  ],
  "dependencies": {
    "@adobe/spectrum-css-workflow-icons": "1.2.1",
-    "@budibase/string-templates": "2.3.2-alpha.0",
+    "@budibase/string-templates": "2.3.11-alpha.0",
    "@spectrum-css/accordion": "3.0.24",
    "@spectrum-css/actionbutton": "1.0.1",
    "@spectrum-css/actiongroup": "1.0.1",
--- a/packages/bbui/src/FancyForm/FancyInput.svelte
+++ b/packages/bbui/src/FancyForm/FancyInput.svelte
@ -1,5 +1,5 @@
 <script>
-  import { createEventDispatcher } from "svelte"
+  import { createEventDispatcher, onMount } from "svelte"
  import FancyField from "./FancyField.svelte"
  import FancyFieldLabel from "./FancyFieldLabel.svelte"
  import { fade } from "svelte/transition"
@ -14,8 +14,11 @@

  const dispatch = createEventDispatcher()

+  let ref
  let focused = false
-  $: placeholder = !focused && !value
+  let autofilled = false
+
+  $: placeholder = !autofilled && !focused && !value

  const onChange = e => {
    const newValue = e.target.value
@ -25,6 +28,27 @@
      error = validate(newValue)
    }
  }
+
+  onMount(() => {
+    // Start watching for autofill every 100ms
+    const interval = setInterval(() => {
+      autofilled = ref?.matches(":-webkit-autofill")
+      if (autofilled) {
+        clearInterval(interval)
+      }
+    }, 100)
+
+    // Give up after 2 seconds and assume autofill has not been used
+    const timeout = setTimeout(() => {
+      clearInterval(interval)
+    }, 2000)
+
+    // Cleanup
+    return () => {
+      clearInterval(interval)
+      clearTimeout(timeout)
+    }
+  })
 </script>

 <FancyField {error} {value} {validate} {disabled} {focused}>
@ -39,6 +63,7 @@
    on:focus={() => (focused = true)}
    on:blur={() => (focused = false)}
    class:placeholder
+    bind:this={ref}
  />
  {#if suffix && !placeholder}
    <div in:fade|local={{ duration: 130 }} class="suffix">{suffix}</div>
@ -74,4 +99,11 @@
    line-height: 17px;
    font-family: var(--font-sans);
  }
+  input:-webkit-autofill {
+    border-radius: 2px;
+    -webkit-box-shadow: 0 0 0 100px var(--spectrum-global-color-gray-300) inset;
+    -webkit-text-fill-color: var(--spectrum-global-color-gray-900);
+    transition: -webkit-box-shadow 130ms 200ms, background-color 0s 86400s;
+    padding: 3px 8px 4px 8px;
+  }
 </style>
--- a/packages/bbui/src/Form/Core/TextField.svelte
+++ b/packages/bbui/src/Form/Core/TextField.svelte
@ -13,6 +13,7 @@
  export let quiet = false
  export let align
  export let autofocus = false
+  export let autocomplete = null

  const dispatch = createEventDispatcher()

@ -103,6 +104,7 @@
    class="spectrum-Textfield-input"
    style={align ? `text-align: ${align};` : ""}
    inputmode={type === "number" ? "decimal" : "text"}
+    {autocomplete}
  />
 </div>

--- a/packages/bbui/src/Form/Input.svelte
+++ b/packages/bbui/src/Form/Input.svelte
@ -14,6 +14,7 @@
  export let updateOnChange = true
  export let quiet = false
  export let autofocus
+  export let autocomplete

  const dispatch = createEventDispatcher()
  const onChange = e => {
@ -33,6 +34,7 @@
    {type}
    {quiet}
    {autofocus}
+    {autocomplete}
    on:change={onChange}
    on:click
    on:input
--- a/packages/builder/package.json
+++ b/packages/builder/package.json
@ -1,6 +1,6 @@
 {
  "name": "@budibase/builder",
-  "version": "2.3.2-alpha.0",
+  "version": "2.3.11-alpha.0",
  "license": "GPL-3.0",
  "private": true,
  "scripts": {
@ -58,10 +58,10 @@
    }
  },
  "dependencies": {
-    "@budibase/bbui": "2.3.2-alpha.0",
-    "@budibase/client": "2.3.2-alpha.0",
-    "@budibase/frontend-core": "2.3.2-alpha.0",
-    "@budibase/string-templates": "2.3.2-alpha.0",
+    "@budibase/bbui": "2.3.11-alpha.0",
+    "@budibase/client": "2.3.11-alpha.0",
+    "@budibase/frontend-core": "2.3.11-alpha.0",
+    "@budibase/string-templates": "2.3.11-alpha.0",
    "@fortawesome/fontawesome-svg-core": "^6.2.1",
    "@fortawesome/free-brands-svg-icons": "^6.2.1",
    "@fortawesome/free-solid-svg-icons": "^6.2.1",
--- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
+++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
@ -75,16 +75,20 @@
    editableColumn.constraints.presence = { allowEmpty: false }
  }

-  $: if (field && !savingColumn) {
-    editableColumn = cloneDeep(field)
-    originalName = editableColumn.name ? editableColumn.name + "" : null
-    linkEditDisabled = originalName != null
-    isCreating = originalName == null
-    primaryDisplay =
-      $tables.selected.primaryDisplay == null ||
-      $tables.selected.primaryDisplay === editableColumn.name
+  const initialiseField = (field, savingColumn) => {
+    if (field && !savingColumn) {
+      editableColumn = cloneDeep(field)
+      originalName = editableColumn.name ? editableColumn.name + "" : null
+      linkEditDisabled = originalName != null
+      isCreating = originalName == null
+      primaryDisplay =
+        $tables.selected.primaryDisplay == null ||
+        $tables.selected.primaryDisplay === editableColumn.name
+    }
  }

+  $: initialiseField(field, savingColumn)
+
  $: checkConstraints(editableColumn)
  $: required = !!editableColumn?.constraints?.presence || primaryDisplay
  $: uneditable =
@ -583,7 +587,12 @@
      title="Formula"
      label="Formula"
      value={editableColumn.formula}
-      on:change={e => (editableColumn.formula = e.detail)}
+      on:change={e => {
+        editableColumn = {
+          ...editableColumn,
+          formula: e.detail,
+        }
+      }}
      bindings={getBindings({ table })}
      allowJS
    />
--- a/packages/builder/src/components/backend/TableNavigator/TableDataImport.svelte
+++ b/packages/builder/src/components/backend/TableNavigator/TableDataImport.svelte
@ -94,6 +94,11 @@

    validateHash = newValidateHash
  }
+
+  const handleChange = (name, e) => {
+    schema[name].type = e.detail
+    schema[name].constraints = FIELDS[e.detail.toUpperCase()].constraints
+  }
 </script>

 <div class="dropzone">
@ -118,12 +123,12 @@
 </div>
 {#if rows.length > 0 && !error}
  <div class="schema-fields">
-    {#each Object.values(schema) as column}
+    {#each Object.entries(schema) as [name, column]}
      <div class="field">
        <span>{column.name}</span>
        <Select
          bind:value={column.type}
-          on:change={e => (column.type = e.detail)}
+          on:change={e => handleChange(name, e)}
          options={typeOptions}
          placeholder={null}
          getOptionLabel={option => option.label}
--- a/packages/builder/src/components/portal/environment/CreateEditVariableModal.svelte
+++ b/packages/builder/src/components/portal/environment/CreateEditVariableModal.svelte
@ -71,6 +71,7 @@
        }
      }}
      value={productionValue}
+      autocomplete="new-password"
    />
  </div>
  <div>
@ -83,6 +84,7 @@
      disabled={useProductionValue}
      label="Value"
      value={useProductionValue ? productionValue : developmentValue}
+      autocomplete="new-password"
    />
    <Checkbox bind:value={useProductionValue} text="Use production value" />
  </div>
--- a/packages/builder/src/components/portal/onboarding/tours.js
+++ b/packages/builder/src/components/portal/onboarding/tours.js
@ -62,6 +62,7 @@ const getTours = () => {
        id: TOUR_STEP_KEYS.BUILDER_APP_PUBLISH,
        title: "Publish",
        layout: OnboardingPublish,
+        route: "/builder/app/:application/design",
        query: ".toprightnav #builder-app-publish-button",
        onLoad: () => {
          tourEvent(TOUR_STEP_KEYS.BUILDER_APP_PUBLISH)
--- a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/components/[componentId]/_components/settings/ComponentSettingsSection.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/components/[componentId]/_components/settings/ComponentSettingsSection.svelte
@ -147,8 +147,8 @@
              options: setting.options || [],

              // Number fields
-              min: setting.min || null,
-              max: setting.max || null,
+              min: setting.min ?? null,
+              max: setting.max ?? null,
            }}
            {bindings}
            {componentBindings}
--- a/packages/builder/src/pages/builder/portal/apps/onboarding/index.svelte
+++ b/packages/builder/src/pages/builder/portal/apps/onboarding/index.svelte
@ -16,6 +16,7 @@
  import CreateTableModal from "components/backend/TableNavigator/modals/CreateTableModal.svelte"
  import createFromScratchScreen from "builderStore/store/screenTemplates/createFromScratchScreen"
  import { Roles } from "constants/backend"
+  import Spinner from "components/common/Spinner.svelte"

  let name = "My first app"
  let url = "my-first-app"
@ -25,39 +26,46 @@
  let plusIntegrations = {}
  let integrationsLoading = true
  $: getIntegrations()
+  let creationLoading = false

  let uploadModal

  const createApp = async useSampleData => {
+    creationLoading = true
    // Create form data to create app
    // This is form based and not JSON
-    let data = new FormData()
-    data.append("name", name.trim())
-    data.append("url", url.trim())
-    data.append("useTemplate", false)
+    try {
+      let data = new FormData()
+      data.append("name", name.trim())
+      data.append("url", url.trim())
+      data.append("useTemplate", false)

-    if (useSampleData) {
-      data.append("sampleData", true)
+      if (useSampleData) {
+        data.append("sampleData", true)
+      }
+
+      const createdApp = await API.createApp(data)
+
+      // Select Correct Application/DB in prep for creating user
+      const pkg = await API.fetchAppPackage(createdApp.instance._id)
+      await store.actions.initialise(pkg)
+      await automationStore.actions.fetch()
+      // Update checklist - in case first app
+      await admin.init()
+
+      // Create user
+      await auth.setInitInfo({})
+
+      let defaultScreenTemplate = createFromScratchScreen.create()
+      defaultScreenTemplate.routing.route = "/home"
+      defaultScreenTemplate.routing.roldId = Roles.BASIC
+      await store.actions.screens.save(defaultScreenTemplate)
+
+      appId = createdApp.instance._id
+    } catch (e) {
+      creationLoading = false
+      throw e
    }
-
-    const createdApp = await API.createApp(data)
-
-    // Select Correct Application/DB in prep for creating user
-    const pkg = await API.fetchAppPackage(createdApp.instance._id)
-    await store.actions.initialise(pkg)
-    await automationStore.actions.fetch()
-    // Update checklist - in case first app
-    await admin.init()
-
-    // Create user
-    await auth.setInitInfo({})
-
-    let defaultScreenTemplate = createFromScratchScreen.create()
-    defaultScreenTemplate.routing.route = "/home"
-    defaultScreenTemplate.routing.roldId = Roles.BASIC
-    await store.actions.screens.save(defaultScreenTemplate)
-
-    appId = createdApp.instance._id
  }

  const getIntegrations = async () => {
@ -102,6 +110,7 @@
      goToApp()
    } catch (e) {
      console.log(e)
+      creationLoading = false
      notifications.error("There was a problem creating your app")
    }
  }
@ -118,8 +127,10 @@
 <SplitPage>
  {#if stage === "name"}
    <NamePanel bind:name bind:url onNext={() => (stage = "data")} />
-  {:else if integrationsLoading}
-    <p>loading...</p>
+  {:else if integrationsLoading || creationLoading}
+    <div class="spinner">
+      <Spinner />
+    </div>
  {:else if stage === "data"}
    <DataPanel onBack={() => (stage = "name")}>
      <div class="dataButton">
@ -175,6 +186,13 @@
 </SplitPage>

 <style>
+  .spinner {
+    display: flex;
+    justify-content: center;
+    align-items: center;
+    min-height: 400px;
+  }
+
  .dataButton {
    margin-bottom: 12px;
  }
--- a/packages/cli/package.json
+++ b/packages/cli/package.json
@ -1,6 +1,6 @@
 {
  "name": "@budibase/cli",
-  "version": "2.3.2-alpha.0",
+  "version": "2.3.11-alpha.0",
  "description": "Budibase CLI, for developers, self hosting and migrations.",
  "main": "src/index.js",
  "bin": {
@ -26,9 +26,9 @@
    "outputPath": "build"
  },
  "dependencies": {
-    "@budibase/backend-core": "2.3.2-alpha.0",
-    "@budibase/string-templates": "2.3.2-alpha.0",
-    "@budibase/types": "2.3.2-alpha.0",
+    "@budibase/backend-core": "2.3.11-alpha.0",
+    "@budibase/string-templates": "2.3.11-alpha.0",
+    "@budibase/types": "2.3.11-alpha.0",
    "axios": "0.21.2",
    "chalk": "4.1.0",
    "cli-progress": "3.11.2",
--- a/packages/client/package.json
+++ b/packages/client/package.json
@ -1,6 +1,6 @@
 {
  "name": "@budibase/client",
-  "version": "2.3.2-alpha.0",
+  "version": "2.3.11-alpha.0",
  "license": "MPL-2.0",
  "module": "dist/budibase-client.js",
  "main": "dist/budibase-client.js",
@ -19,9 +19,9 @@
    "dev:builder": "rollup -cw"
  },
  "dependencies": {
-    "@budibase/bbui": "2.3.2-alpha.0",
-    "@budibase/frontend-core": "2.3.2-alpha.0",
-    "@budibase/string-templates": "2.3.2-alpha.0",
+    "@budibase/bbui": "2.3.11-alpha.0",
+    "@budibase/frontend-core": "2.3.11-alpha.0",
+    "@budibase/string-templates": "2.3.11-alpha.0",
    "@spectrum-css/button": "^3.0.3",
    "@spectrum-css/card": "^3.0.3",
    "@spectrum-css/divider": "^1.0.3",
--- a/packages/frontend-core/package.json
+++ b/packages/frontend-core/package.json
@ -1,12 +1,12 @@
 {
  "name": "@budibase/frontend-core",
-  "version": "2.3.2-alpha.0",
+  "version": "2.3.11-alpha.0",
  "description": "Budibase frontend core libraries used in builder and client",
  "author": "Budibase",
  "license": "MPL-2.0",
  "svelte": "src/index.js",
  "dependencies": {
-    "@budibase/bbui": "2.3.2-alpha.0",
+    "@budibase/bbui": "2.3.11-alpha.0",
    "lodash": "^4.17.21",
    "svelte": "^3.46.2"
  }
--- a/packages/sdk/package.json
+++ b/packages/sdk/package.json
@ -1,6 +1,6 @@
 {
  "name": "@budibase/sdk",
-  "version": "2.3.2-alpha.0",
+  "version": "2.3.11-alpha.0",
  "description": "Budibase Public API SDK",
  "author": "Budibase",
  "license": "MPL-2.0",
--- a/packages/server/package.json
+++ b/packages/server/package.json
@ -1,7 +1,7 @@
 {
  "name": "@budibase/server",
  "email": "hi@budibase.com",
-  "version": "2.3.2-alpha.0",
+  "version": "2.3.11-alpha.0",
  "description": "Budibase Web Server",
  "main": "src/index.ts",
  "repository": {
@ -43,11 +43,11 @@
  "license": "GPL-3.0",
  "dependencies": {
    "@apidevtools/swagger-parser": "10.0.3",
-    "@budibase/backend-core": "2.3.2-alpha.0",
-    "@budibase/client": "2.3.2-alpha.0",
-    "@budibase/pro": "2.3.2-alpha.0",
-    "@budibase/string-templates": "2.3.2-alpha.0",
-    "@budibase/types": "2.3.2-alpha.0",
+    "@budibase/backend-core": "2.3.11-alpha.0",
+    "@budibase/client": "2.3.11-alpha.0",
+    "@budibase/pro": "2.3.11-alpha.0",
+    "@budibase/string-templates": "2.3.11-alpha.0",
+    "@budibase/types": "2.3.11-alpha.0",
    "@bull-board/api": "3.7.0",
    "@bull-board/koa": "3.9.4",
    "@elastic/elasticsearch": "7.10.0",
--- a/packages/server/src/api/controllers/datasource.ts
+++ b/packages/server/src/api/controllers/datasource.ts
@ -14,7 +14,6 @@ import { invalidateDynamicVariables } from "../../threads/utils"
 import { db as dbCore, context, events } from "@budibase/backend-core"
 import { UserCtx, Datasource, Row } from "@budibase/types"
 import sdk from "../../sdk"
-import { mergeConfigs } from "../../sdk/app/datasources/datasources"

 export async function fetch(ctx: UserCtx) {
  // Get internal tables
--- a/packages/server/src/api/controllers/query/index.ts
+++ b/packages/server/src/api/controllers/query/index.ts
@ -186,6 +186,8 @@ export async function preview(ctx: any) {
        schemaFields[key] = fieldType
      }
    }
+    // remove configuration before sending event
+    delete datasource.config
    await events.query.previewed(datasource, query)
    ctx.body = {
      rows,
--- a/packages/server/src/api/controllers/row/ExternalRequest.ts
+++ b/packages/server/src/api/controllers/row/ExternalRequest.ts
@ -10,6 +10,7 @@ import {
  FieldSchema,
  Row,
  Table,
+  RelationshipTypes,
 } from "@budibase/types"
 import {
  breakRowIdField,
@ -18,7 +19,7 @@ import {
  convertRowId,
 } from "../../../integrations/utils"
 import { getDatasourceAndQuery } from "./utils"
-import { FieldTypes, RelationshipTypes } from "../../../constants"
+import { FieldTypes } from "../../../constants"
 import { breakExternalTableId, isSQL } from "../../../integrations/utils"
 import { processObjectSync } from "@budibase/string-templates"
 import { cloneDeep } from "lodash/fp"
@ -44,6 +45,7 @@ export interface RunConfig {
  row?: Row
  rows?: Row[]
  tables?: Record<string, Table>
+  includeSqlRelationships?: IncludeRelationship
 }

 function buildFilters(
@ -706,7 +708,9 @@ export class ExternalRequest {
      },
      resource: {
        // have to specify the fields to avoid column overlap (for SQL)
-        fields: isSql ? this.buildFields(table) : [],
+        fields: isSql
+          ? this.buildFields(table, config.includeSqlRelationships)
+          : [],
      },
      filters,
      sort,
--- a/packages/server/src/api/controllers/row/external.ts
+++ b/packages/server/src/api/controllers/row/external.ts
@ -18,6 +18,7 @@ import {
  PaginationJson,
  Table,
  Datasource,
+  IncludeRelationship,
 } from "@budibase/types"
 import sdk from "../../../sdk"

@ -57,6 +58,7 @@ export async function patch(ctx: BBContext) {
  return handleRequest(Operation.UPDATE, tableId, {
    id: breakRowIdField(id),
    row: inputs,
+    includeSqlRelationships: IncludeRelationship.EXCLUDE,
  })
 }

@ -65,6 +67,7 @@ export async function save(ctx: BBContext) {
  const tableId = ctx.params.tableId
  return handleRequest(Operation.CREATE, tableId, {
    row: inputs,
+    includeSqlRelationships: IncludeRelationship.EXCLUDE,
  })
 }

@ -78,7 +81,9 @@ export async function fetchView(ctx: BBContext) {

 export async function fetch(ctx: BBContext) {
  const tableId = ctx.params.tableId
-  return handleRequest(Operation.READ, tableId)
+  return handleRequest(Operation.READ, tableId, {
+    includeSqlRelationships: IncludeRelationship.INCLUDE,
+  })
 }

 export async function find(ctx: BBContext) {
@ -86,6 +91,7 @@ export async function find(ctx: BBContext) {
  const tableId = ctx.params.tableId
  const response = (await handleRequest(Operation.READ, tableId, {
    id: breakRowIdField(id),
+    includeSqlRelationships: IncludeRelationship.EXCLUDE,
  })) as Row[]
  return response ? response[0] : response
 }
@ -95,6 +101,7 @@ export async function destroy(ctx: BBContext) {
  const id = ctx.request.body._id
  const { row } = (await handleRequest(Operation.DELETE, tableId, {
    id: breakRowIdField(id),
+    includeSqlRelationships: IncludeRelationship.EXCLUDE,
  })) as { row: Row }
  return { response: { ok: true }, row }
 }
@ -107,6 +114,7 @@ export async function bulkDestroy(ctx: BBContext) {
    promises.push(
      handleRequest(Operation.DELETE, tableId, {
        id: breakRowIdField(row._id),
+        includeSqlRelationships: IncludeRelationship.EXCLUDE,
      })
    )
  }
@ -149,6 +157,7 @@ export async function search(ctx: BBContext) {
      filters: query,
      sort,
      paginate: paginateObj as PaginationJson,
+      includeSqlRelationships: IncludeRelationship.INCLUDE,
    })) as Row[]
    let hasNextPage = false
    if (paginate && rows.length === limit) {
@ -159,6 +168,7 @@ export async function search(ctx: BBContext) {
          limit: 1,
          page: bookmark * limit + 1,
        },
+        includeSqlRelationships: IncludeRelationship.INCLUDE,
      })) as Row[]
      hasNextPage = nextRows.length > 0
    }
@ -181,7 +191,7 @@ export async function validate(ctx: BBContext) {
 }

 export async function exportRows(ctx: BBContext) {
-  const { datasourceId } = breakExternalTableId(ctx.params.tableId)
+  const { datasourceId, tableName } = breakExternalTableId(ctx.params.tableId)
  const format = ctx.query.format
  const { columns } = ctx.request.body
  const datasource = await sdk.datasources.get(datasourceId!)
@ -217,7 +227,9 @@ export async function exportRows(ctx: BBContext) {
    rows = result.rows
  }

-  // @ts-ignore
+  if (!tableName) {
+    ctx.throw(400, "Could not find table name.")
+  }
  let schema = datasource.entities[tableName].schema
  let exportRows = cleanExportRows(rows, schema, format, columns)

@ -247,6 +259,7 @@ export async function fetchEnrichedRow(ctx: BBContext) {
  const response = (await handleRequest(Operation.READ, tableId, {
    id,
    datasource,
+    includeSqlRelationships: IncludeRelationship.INCLUDE,
  })) as Row[]
  const table: Table = tables[tableName]
  const row = response[0]
@ -274,6 +287,7 @@ export async function fetchEnrichedRow(ctx: BBContext) {
          [primaryLink]: linkedIds,
        },
      },
+      includeSqlRelationships: IncludeRelationship.INCLUDE,
    })
  }
  return row
--- a/packages/server/src/api/controllers/table/external.ts
+++ b/packages/server/src/api/controllers/table/external.ts
@ -8,7 +8,7 @@ import {
  foreignKeyStructure,
  hasTypeChanged,
 } from "./utils"
-import { FieldTypes, RelationshipTypes } from "../../../constants"
+import { FieldTypes } from "../../../constants"
 import { makeExternalQuery } from "../../../integrations/base/query"
 import { handleRequest } from "../row/external"
 import { events, context } from "@budibase/backend-core"
@ -22,6 +22,7 @@ import {
  FieldSchema,
  BBContext,
  TableRequest,
+  RelationshipTypes,
 } from "@budibase/types"
 import sdk from "../../../sdk"
 const { cloneDeep } = require("lodash/fp")
@ -146,7 +147,7 @@ function generateLinkSchema(
  column: FieldSchema,
  table: Table,
  relatedTable: Table,
-  type: string
+  type: RelationshipTypes
 ) {
  if (!table.primary || !relatedTable.primary) {
    throw new Error("Unable to generate link schema, no primary keys")
--- a/packages/server/src/api/controllers/table/utils.ts
+++ b/packages/server/src/api/controllers/table/utils.ts
@ -104,7 +104,6 @@ export function importToRows(data: any, table: any, user: any = {}) {
    const processed: any = inputProcessing(user, table, row, {
      noAutoRelationships: true,
    })
-    table = processed.table
    row = processed.row

    let fieldName: any
@ -113,6 +112,7 @@ export function importToRows(data: any, table: any, user: any = {}) {
      // check whether the options need to be updated for inclusion as part of the data import
      if (
        schema.type === FieldTypes.OPTIONS &&
+        row[fieldName] &&
        (!schema.constraints.inclusion ||
          schema.constraints.inclusion.indexOf(row[fieldName]) === -1)
      ) {
@ -120,6 +120,7 @@ export function importToRows(data: any, table: any, user: any = {}) {
          ...schema.constraints.inclusion,
          row[fieldName],
        ]
+        schema.constraints.inclusion.sort()
      }
    }

--- a/packages/server/src/api/routes/public/tests/compare.spec.js
+++ b/packages/server/src/api/routes/public/tests/compare.spec.js
@ -13,7 +13,7 @@ beforeAll(async () => {
  app = await config.init()
  table = await config.updateTable()
  apiKey = await config.generateApiKey()
-  makeRequest = generateMakeRequest(apiKey, setup)
+  makeRequest = generateMakeRequest(apiKey)
 })

 afterAll(setup.afterAll)
--- a/packages/server/src/api/routes/public/tests/users.spec.js
+++ b/packages/server/src/api/routes/public/tests/users.spec.js
@ -10,7 +10,7 @@ beforeAll(async () => {
  await config.init()
  globalUser = await config.globalUser()
  apiKey = await config.generateApiKey(globalUser._id)
-  makeRequest = generateMakeRequest(apiKey, setup)
+  makeRequest = generateMakeRequest(apiKey)
  workerRequests.readGlobalUser.mockReturnValue(globalUser)
 })

--- a/packages/server/src/api/routes/public/tests/utils.ts
+++ b/packages/server/src/api/routes/public/tests/utils.ts
@ -1,13 +1,27 @@
+import * as setup from "../../tests/utilities"
 import { checkSlashesInUrl } from "../../../../utilities"
+import supertest from "supertest"

-export function generateMakeRequest(apiKey: string, setup: any) {
-  const request = setup.getRequest()
-  const config = setup.getConfig()
+export type HttpMethod = "post" | "get" | "put" | "delete" | "patch"
+
+export type MakeRequestResponse = (
+  method: HttpMethod,
+  endpoint: string,
+  body?: any,
+  intAppId?: string
+) => Promise<supertest.Response>
+
+export function generateMakeRequest(
+  apiKey: string,
+  isInternal = false
+): MakeRequestResponse {
+  const request = setup.getRequest()!
+  const config = setup.getConfig()!
  return async (
-    method: string,
+    method: HttpMethod,
    endpoint: string,
    body?: any,
-    intAppId: string = config.getAppId()
+    intAppId: string | null = config.getAppId()
  ) => {
    const extraHeaders: any = {
      "x-budibase-api-key": apiKey,
@ -15,9 +29,12 @@ export function generateMakeRequest(apiKey: string, setup: any) {
    if (intAppId) {
      extraHeaders["x-budibase-app-id"] = intAppId
    }
-    const req = request[method](
-      checkSlashesInUrl(`/api/public/v1/${endpoint}`)
-    ).set(config.defaultHeaders(extraHeaders))
+
+    const url = isInternal
+      ? endpoint
+      : checkSlashesInUrl(`/api/public/v1/${endpoint}`)
+
+    const req = request[method](url).set(config.defaultHeaders(extraHeaders))
    if (body) {
      req.send(body)
    }
--- a/packages/server/src/api/routes/tests/cloud.seq.spec.ts
+++ b/packages/server/src/api/routes/tests/cloud.seq.spec.ts
@ -7,7 +7,7 @@ import * as setup from "./utilities"
 import { wipeDb } from "./utilities/TestFunctions"

 describe("/cloud", () => {
-  let request = setup.getRequest()
+  let request = setup.getRequest()!
  let config = setup.getConfig()

  afterAll(setup.afterAll)
--- a/packages/server/src/api/routes/tests/environmentVariables.spec.ts
+++ b/packages/server/src/api/routes/tests/environmentVariables.spec.ts
@ -0,0 +1,144 @@
+const pg = require("pg")
+jest.mock("pg", () => {
+  return {
+    Client: jest.fn().mockImplementation(() => ({
+      connect: jest.fn(),
+      query: jest.fn().mockImplementation(() => ({ rows: [] })),
+      end: jest.fn().mockImplementation((fn: any) => fn()),
+    })),
+    queryMock: jest.fn().mockImplementation(() => {}),
+    on: jest.fn(),
+  }
+})
+import * as setup from "./utilities"
+import { mocks } from "@budibase/backend-core/tests"
+import { env, events } from "@budibase/backend-core"
+const structures = setup.structures
+
+env._set("ENCRYPTION_KEY", "budibase")
+mocks.licenses.useEnvironmentVariables()
+
+describe("/api/env/variables", () => {
+  let request = setup.getRequest()
+  let config = setup.getConfig()
+
+  afterAll(setup.afterAll)
+
+  beforeAll(async () => {
+    await config.init()
+  })
+
+  it("should be able check the status of env var API", async () => {
+    const res = await request
+      .get(`/api/env/variables/status`)
+      .set(config.defaultHeaders())
+      .expect("Content-Type", /json/)
+      .expect(200)
+
+    expect(res.body.encryptionKeyAvailable).toEqual(true)
+  })
+
+  it("should be able to create an environment variable", async () => {
+    await request
+      .post(`/api/env/variables`)
+      .send(structures.basicEnvironmentVariable("test", "test"))
+      .set(config.defaultHeaders())
+      .expect(200)
+  })
+
+  it("should be able to fetch the 'test' variable name", async () => {
+    const res = await request
+      .get(`/api/env/variables`)
+      .set(config.defaultHeaders())
+      .expect("Content-Type", /json/)
+      .expect(200)
+    expect(res.body.variables.length).toEqual(1)
+    expect(res.body.variables[0]).toEqual("test")
+  })
+
+  it("should be able to update the environment variable 'test'", async () => {
+    const varName = "test"
+    await request
+      .patch(`/api/env/variables/${varName}`)
+      .send(structures.basicEnvironmentVariable("test", "test1"))
+      .set(config.defaultHeaders())
+      .expect(200)
+  })
+
+  it("should be able to delete the environment variable 'test'", async () => {
+    const varName = "test"
+    await request
+      .delete(`/api/env/variables/${varName}`)
+      .set(config.defaultHeaders())
+      .expect(200)
+  })
+
+  it("should create a datasource (using the environment variable) and query", async () => {
+    const datasourceBase = structures.basicDatasource()
+    await request
+      .post(`/api/env/variables`)
+      .send(structures.basicEnvironmentVariable("test", "test"))
+      .set(config.defaultHeaders())
+
+    datasourceBase.datasource.config = {
+      password: "{{ env.test }}",
+    }
+    const response = await request
+      .post(`/api/datasources`)
+      .send(datasourceBase)
+      .set(config.defaultHeaders())
+      .expect("Content-Type", /json/)
+      .expect(200)
+    expect(response.body.datasource._id).toBeDefined()
+
+    const response2 = await request
+      .post(`/api/queries`)
+      .send(structures.basicQuery(response.body.datasource._id))
+      .set(config.defaultHeaders())
+      .expect("Content-Type", /json/)
+      .expect(200)
+    expect(response2.body._id).toBeDefined()
+  })
+
+  it("should run a query preview and check the mocked results", async () => {
+    const datasourceBase = structures.basicDatasource()
+    await request
+      .post(`/api/env/variables`)
+      .send(structures.basicEnvironmentVariable("test", "test"))
+      .set(config.defaultHeaders())
+
+    datasourceBase.datasource.config = {
+      password: "{{ env.test }}",
+    }
+    const response = await request
+      .post(`/api/datasources`)
+      .send(datasourceBase)
+      .set(config.defaultHeaders())
+      .expect("Content-Type", /json/)
+      .expect(200)
+    expect(response.body.datasource._id).toBeDefined()
+
+    const query = {
+      datasourceId: response.body.datasource._id,
+      parameters: {},
+      fields: {},
+      queryVerb: "read",
+      name: response.body.datasource.name,
+    }
+    const res = await request
+      .post(`/api/queries/preview`)
+      .send(query)
+      .set(config.defaultHeaders())
+      .expect("Content-Type", /json/)
+      .expect(200)
+    expect(res.body.rows.length).toEqual(0)
+    expect(events.query.previewed).toBeCalledTimes(1)
+    // API doesn't include config in response
+    delete response.body.datasource.config
+    expect(events.query.previewed).toBeCalledWith(
+      response.body.datasource,
+      query
+    )
+    expect(pg.Client).toHaveBeenCalledWith({ password: "test", ssl: undefined })
+  })
+})
--- a/packages/server/src/api/routes/tests/query.seq.spec.js
+++ b/packages/server/src/api/routes/tests/query.seq.spec.js
@ -242,6 +242,7 @@ describe("/queries", () => {
      })
      expect(res.body.rows.length).toEqual(1)
      expect(events.query.previewed).toBeCalledTimes(1)
+      delete datasource.config
      expect(events.query.previewed).toBeCalledWith(datasource, query)
    })

--- a/packages/server/src/api/routes/tests/user.spec.js
+++ b/packages/server/src/api/routes/tests/user.spec.js
@ -1,4 +1,4 @@
-const { roles } = require("@budibase/backend-core")
+const { roles, utils } = require("@budibase/backend-core")
 const { checkPermissionsEndpoint } = require("./utilities/TestFunctions")
 const setup = require("./utilities")
 const { BUILTIN_ROLE_IDS } = roles
@ -28,8 +28,8 @@ describe("/users", () => {

  describe("fetch", () => {
    it("returns a list of users from an instance db", async () => {
-      await config.createUser("uuidx")
-      await config.createUser("uuidy")
+      await config.createUser({ id: "uuidx" })
+      await config.createUser({ id: "uuidy" })
      const res = await request
        .get(`/api/users/metadata`)
        .set(config.defaultHeaders())
@ -56,7 +56,7 @@ describe("/users", () => {

  describe("update", () => {
    it("should be able to update the user", async () => {
-      const user = await config.createUser()
+      const user = await config.createUser({ id: `us_update${Math.random()}` })
      user.roleId = BUILTIN_ROLE_IDS.BASIC
      const res = await request
        .put(`/api/users/metadata`)
@ -180,14 +180,11 @@ describe("/users", () => {
      const app1 = await config.createApp('App 1')
      const app2 = await config.createApp('App 2')

-      let user = await config.createUser(
-        undefined,
-        undefined,
-        undefined,
-        undefined,
-        false,
-        true,
-        { [app1.appId]: 'ADMIN' })
+      let user = await config.createUser({
+        builder: false,
+        admin: true,
+      roles: { [app1.appId]: 'ADMIN' }
+    })
      let res = await request
        .post(`/api/users/metadata/sync/${user._id}`)
        .set(config.defaultHeaders())
--- a/packages/server/src/constants/index.ts
+++ b/packages/server/src/constants/index.ts
@ -1,4 +1,5 @@
 import { objectStore, roles, constants } from "@budibase/backend-core"
+export { FieldType as FieldTypes, RelationshipTypes } from "@budibase/types"

 export enum FilterTypes {
  STRING = "string",
@ -22,23 +23,6 @@ export const NoEmptyFilterStrings = [
  FilterTypes.NOT_CONTAINS,
 ]

-export enum FieldTypes {
-  STRING = "string",
-  BARCODEQR = "barcodeqr",
-  LONGFORM = "longform",
-  OPTIONS = "options",
-  NUMBER = "number",
-  BOOLEAN = "boolean",
-  ARRAY = "array",
-  DATETIME = "datetime",
-  ATTACHMENT = "attachment",
-  LINK = "link",
-  FORMULA = "formula",
-  AUTO = "auto",
-  JSON = "json",
-  INTERNAL = "internal",
-}
-
 export const CanSwitchTypes = [
  [exports.FieldTypes.JSON, exports.FieldTypes.ARRAY],
  [
@ -54,12 +38,6 @@ export const SwitchableTypes = CanSwitchTypes.reduce((prev, current) =>
  prev ? prev.concat(current) : current
 )

-export enum RelationshipTypes {
-  ONE_TO_MANY = "one-to-many",
-  MANY_TO_ONE = "many-to-one",
-  MANY_TO_MANY = "many-to-many",
-}
-
 export enum FormulaTypes {
  STATIC = "static",
  DYNAMIC = "dynamic",
--- a/packages/server/src/db/defaultData/datasource_bb_default.ts
+++ b/packages/server/src/db/defaultData/datasource_bb_default.ts
@ -1,8 +1,4 @@
-import {
-  FieldTypes,
-  AutoFieldSubTypes,
-  RelationshipTypes,
-} from "../../constants"
+import { FieldTypes, AutoFieldSubTypes } from "../../constants"
 import { importToRows } from "../../api/controllers/table/utils"
 import { cloneDeep } from "lodash/fp"
 import LinkDocument from "../linkedRows/LinkDocument"
@ -11,7 +7,7 @@ import { employeeImport } from "./employeeImport"
 import { jobsImport } from "./jobsImport"
 import { expensesImport } from "./expensesImport"
 import { db as dbCore } from "@budibase/backend-core"
-import { Table, Row } from "@budibase/types"
+import { Table, Row, RelationshipTypes } from "@budibase/types"

 export const DEFAULT_JOBS_TABLE_ID = "ta_bb_jobs"
 export const DEFAULT_INVENTORY_TABLE_ID = "ta_bb_inventory"
@ -190,7 +186,7 @@ export const DEFAULT_INVENTORY_TABLE_SCHEMA: Table = {
  },
 }

-export const DEFAULT_EMPLOYEE_TABLE_SCHEMA = {
+export const DEFAULT_EMPLOYEE_TABLE_SCHEMA: Table = {
  _id: DEFAULT_EMPLOYEE_TABLE_ID,
  type: "internal",
  views: {},
@ -287,7 +283,7 @@ export const DEFAULT_EMPLOYEE_TABLE_SCHEMA = {
      sortable: false,
    },
    "Badge Photo": {
-      type: "attachment",
+      type: FieldTypes.ATTACHMENT,
      constraints: {
        type: FieldTypes.ARRAY,
        presence: false,
@ -466,7 +462,7 @@ export const DEFAULT_JOBS_TABLE_SCHEMA: Table = {
      // sortable: true,
    },
    "Works End": {
-      type: "datetime",
+      type: FieldTypes.DATETIME,
      constraints: {
        type: "string",
        length: {},
@ -480,7 +476,7 @@ export const DEFAULT_JOBS_TABLE_SCHEMA: Table = {
      ignoreTimezones: true,
    },
    "Updated Price": {
-      type: "number",
+      type: FieldTypes.NUMBER,
      constraints: {
        type: "number",
        presence: false,
--- a/packages/server/src/db/linkedRows/LinkController.ts
+++ b/packages/server/src/db/linkedRows/LinkController.ts
@ -1,13 +1,14 @@
 import { IncludeDocs, getLinkDocuments } from "./linkUtils"
 import { InternalTables, getUserMetadataParams } from "../utils"
 import Sentry from "@sentry/node"
-import { FieldTypes, RelationshipTypes } from "../../constants"
+import { FieldTypes } from "../../constants"
 import { context } from "@budibase/backend-core"
 import LinkDocument from "./LinkDocument"
 import {
  Database,
  FieldSchema,
  LinkDocumentValue,
+  RelationshipTypes,
  Row,
  Table,
 } from "@budibase/types"
--- a/packages/server/src/integration-test/postgres.spec.ts
+++ b/packages/server/src/integration-test/postgres.spec.ts
@ -0,0 +1,752 @@
+import {
+  generateMakeRequest,
+  MakeRequestResponse,
+} from "../api/routes/public/tests/utils"
+
+import * as setup from "../api/routes/tests/utilities"
+import {
+  Datasource,
+  FieldType,
+  RelationshipTypes,
+  Row,
+  SourceName,
+  Table,
+} from "@budibase/types"
+import _ from "lodash"
+import { generator } from "@budibase/backend-core/tests"
+import { utils } from "@budibase/backend-core"
+import { GenericContainer } from "testcontainers"
+
+const config = setup.getConfig()!
+
+jest.setTimeout(30000)
+
+jest.unmock("pg")
+
+describe("row api - postgres", () => {
+  let makeRequest: MakeRequestResponse,
+    postgresDatasource: Datasource,
+    primaryPostgresTable: Table,
+    auxPostgresTable: Table
+
+  let host: string
+  let port: number
+
+  beforeAll(async () => {
+    const container = await new GenericContainer("postgres")
+      .withExposedPorts(5432)
+      .withEnv("POSTGRES_PASSWORD", "password")
+      .start()
+
+    host = container.getContainerIpAddress()
+    port = container.getMappedPort(5432)
+
+    await config.init()
+    const apiKey = await config.generateApiKey()
+
+    makeRequest = generateMakeRequest(apiKey, true)
+  })
+
+  beforeEach(async () => {
+    postgresDatasource = await config.createDatasource({
+      datasource: {
+        type: "datasource",
+        source: SourceName.POSTGRES,
+        plus: true,
+        config: {
+          host,
+          port,
+          database: "postgres",
+          user: "postgres",
+          password: "password",
+          schema: "public",
+          ssl: false,
+          rejectUnauthorized: false,
+          ca: false,
+        },
+      },
+    })
+
+    auxPostgresTable = await config.createTable({
+      name: generator.word({ length: 10 }),
+      type: "external",
+      primary: ["id"],
+      schema: {
+        id: {
+          name: "id",
+          type: FieldType.AUTO,
+          constraints: {
+            presence: true,
+          },
+        },
+        title: {
+          name: "title",
+          type: FieldType.STRING,
+          constraints: {
+            presence: true,
+          },
+        },
+      },
+      sourceId: postgresDatasource._id,
+    })
+
+    primaryPostgresTable = await config.createTable({
+      name: generator.word({ length: 10 }),
+      type: "external",
+      primary: ["id"],
+      schema: {
+        id: {
+          name: "id",
+          type: FieldType.AUTO,
+          constraints: {
+            presence: true,
+          },
+        },
+        name: {
+          name: "name",
+          type: FieldType.STRING,
+          constraints: {
+            presence: true,
+          },
+        },
+        description: {
+          name: "description",
+          type: FieldType.STRING,
+        },
+        value: {
+          name: "value",
+          type: FieldType.NUMBER,
+        },
+        linkedField: {
+          type: FieldType.LINK,
+          constraints: {
+            type: "array",
+            presence: false,
+          },
+          fieldName: "foreignField",
+          name: "linkedField",
+          relationshipType: RelationshipTypes.ONE_TO_MANY,
+          tableId: auxPostgresTable._id,
+        },
+      },
+      sourceId: postgresDatasource._id,
+    })
+  })
+
+  afterAll(async () => {
+    await config.end()
+  })
+
+  function generateRandomPrimaryRowData() {
+    return {
+      name: generator.name(),
+      description: generator.paragraph(),
+      value: generator.age(),
+    }
+  }
+
+  type PrimaryRowData = {
+    name: string
+    description: string
+    value: number
+  }
+
+  async function createPrimaryRow(opts: {
+    rowData: PrimaryRowData
+    createForeignRow?: boolean
+  }) {
+    let { rowData } = opts
+    let foreignRow: Row | undefined
+    if (opts?.createForeignRow) {
+      foreignRow = await config.createRow({
+        tableId: auxPostgresTable._id,
+        title: generator.name(),
+      })
+
+      rowData = {
+        ...rowData,
+        [`fk_${auxPostgresTable.name}_foreignField`]: foreignRow.id,
+      }
+    }
+
+    const row = await config.createRow({
+      tableId: primaryPostgresTable._id,
+      ...rowData,
+    })
+
+    return { row, foreignRow }
+  }
+
+  async function createDefaultPgTable() {
+    return await config.createTable({
+      name: generator.word({ length: 10 }),
+      type: "external",
+      primary: ["id"],
+      schema: {
+        id: {
+          name: "id",
+          type: FieldType.AUTO,
+          constraints: {
+            presence: true,
+          },
+        },
+      },
+      sourceId: postgresDatasource._id,
+    })
+  }
+
+  async function populatePrimaryRows(
+    count: number,
+    opts?: {
+      createForeignRow?: boolean
+    }
+  ) {
+    return await Promise.all(
+      Array(count)
+        .fill({})
+        .map(async () => {
+          const rowData = generateRandomPrimaryRowData()
+          return {
+            rowData,
+            ...(await createPrimaryRow({
+              rowData,
+              createForeignRow: opts?.createForeignRow,
+            })),
+          }
+        })
+    )
+  }
+
+  it("validate table schema", async () => {
+    const res = await makeRequest(
+      "get",
+      `/api/datasources/${postgresDatasource._id}`
+    )
+
+    expect(res.status).toBe(200)
+    expect(res.body).toEqual({
+      config: {
+        ca: false,
+        database: "postgres",
+        host,
+        password: "--secret-value--",
+        port,
+        rejectUnauthorized: false,
+        schema: "public",
+        ssl: false,
+        user: "postgres",
+      },
+      plus: true,
+      source: "POSTGRES",
+      type: "datasource",
+      _id: expect.any(String),
+      _rev: expect.any(String),
+      createdAt: expect.any(String),
+      updatedAt: expect.any(String),
+      entities: expect.any(Object),
+    })
+  })
+
+  describe("POST /api/:tableId/rows", () => {
+    const createRow = (tableId: string | undefined, body: object) =>
+      makeRequest("post", `/api/${tableId}/rows`, body)
+
+    describe("given than no row exists", () => {
+      it("adding a new one persists it", async () => {
+        const newRow = generateRandomPrimaryRowData()
+
+        const res = await createRow(primaryPostgresTable._id, newRow)
+
+        expect(res.status).toBe(200)
+
+        const persistedRows = await config.getRows(primaryPostgresTable._id!)
+        expect(persistedRows).toHaveLength(1)
+
+        const expected = {
+          ...res.body,
+          ...newRow,
+        }
+
+        expect(persistedRows).toEqual([expect.objectContaining(expected)])
+      })
+
+      it("multiple rows can be persisted", async () => {
+        const numberOfRows = 10
+        const newRows = Array(numberOfRows).fill(generateRandomPrimaryRowData())
+
+        for (const newRow of newRows) {
+          const res = await createRow(primaryPostgresTable._id, newRow)
+          expect(res.status).toBe(200)
+        }
+
+        const persistedRows = await config.getRows(primaryPostgresTable._id!)
+        expect(persistedRows).toHaveLength(numberOfRows)
+        expect(persistedRows).toEqual(
+          expect.arrayContaining(newRows.map(expect.objectContaining))
+        )
+      })
+    })
+  })
+
+  describe("PATCH /api/:tableId/rows", () => {
+    const updateRow = (tableId: string | undefined, body: Row) =>
+      makeRequest("patch", `/api/${tableId}/rows`, body)
+
+    describe("given than a row exists", () => {
+      let row: Row
+      beforeEach(async () => {
+        let rowResponse = _.sample(await populatePrimaryRows(10))!
+        row = rowResponse.row
+      })
+
+      it("updating it persists it", async () => {
+        const newName = generator.name()
+        const newValue = generator.age()
+        const updatedRow = {
+          ...row,
+          name: newName,
+          value: newValue,
+        }
+
+        const res = await updateRow(primaryPostgresTable._id, updatedRow)
+
+        expect(res.status).toBe(200)
+        expect(res.body).toEqual(updatedRow)
+
+        const persistedRow = await config.getRow(
+          primaryPostgresTable._id!,
+          row.id
+        )
+
+        expect(persistedRow).toEqual(
+          expect.objectContaining({
+            id: row.id,
+            name: newName,
+            value: newValue,
+          })
+        )
+      })
+    })
+  })
+
+  describe("DELETE /api/:tableId/rows", () => {
+    const deleteRow = (
+      tableId: string | undefined,
+      body: Row | { rows: Row[] }
+    ) => makeRequest("delete", `/api/${tableId}/rows`, body)
+
+    describe("given than multiple row exist", () => {
+      const numberOfInitialRows = 5
+      let rows: Row[]
+      beforeEach(async () => {
+        rows = (await populatePrimaryRows(numberOfInitialRows)).map(x => x.row)
+      })
+
+      it("delete request removes it", async () => {
+        const row = _.sample(rows)!
+        const res = await deleteRow(primaryPostgresTable._id, row)
+
+        expect(res.status).toBe(200)
+
+        const persistedRows = await config.getRows(primaryPostgresTable._id!)
+        expect(persistedRows).toHaveLength(numberOfInitialRows - 1)
+
+        expect(row.id).toBeDefined()
+        expect(persistedRows).not.toContain(
+          expect.objectContaining({ _id: row.id })
+        )
+      })
+
+      it("multiple rows can be removed at once", async () => {
+        let rowsToDelete = _.sampleSize(rows, 3)!
+
+        const res = await deleteRow(primaryPostgresTable._id, {
+          rows: rowsToDelete,
+        })
+
+        expect(res.status).toBe(200)
+
+        const persistedRows = await config.getRows(primaryPostgresTable._id!)
+        expect(persistedRows).toHaveLength(numberOfInitialRows - 3)
+
+        for (const row of rowsToDelete) {
+          expect(persistedRows).not.toContain(
+            expect.objectContaining({ _id: row.id })
+          )
+        }
+      })
+    })
+  })
+
+  describe("GET /api/:tableId/rows/:rowId", () => {
+    const getRow = (tableId: string | undefined, rowId?: string | undefined) =>
+      makeRequest("get", `/api/${tableId}/rows/${rowId}`)
+
+    describe("given than a table have a single row", () => {
+      let rowData: PrimaryRowData, row: Row
+      beforeEach(async () => {
+        const [createdRow] = await populatePrimaryRows(1)
+        rowData = createdRow.rowData
+        row = createdRow.row
+      })
+
+      it("the row can be retrieved successfully", async () => {
+        const res = await getRow(primaryPostgresTable._id, row.id)
+
+        expect(res.status).toBe(200)
+
+        expect(res.body).toEqual(expect.objectContaining(rowData))
+      })
+    })
+
+    describe("given than a table have a multiple rows", () => {
+      let rows: { row: Row; rowData: PrimaryRowData }[]
+
+      beforeEach(async () => {
+        rows = await populatePrimaryRows(10)
+      })
+
+      it("a single row can be retrieved successfully", async () => {
+        const { rowData, row } = _.sample(rows)!
+
+        const res = await getRow(primaryPostgresTable._id, row.id)
+
+        expect(res.status).toBe(200)
+
+        expect(res.body).toEqual(expect.objectContaining(rowData))
+      })
+    })
+
+    describe("given a row with relation data", () => {
+      let row: Row
+      beforeEach(async () => {
+        let [createdRow] = await populatePrimaryRows(1, {
+          createForeignRow: true,
+        })
+        row = createdRow.row
+      })
+
+      it("foreign key fields are not retrieved", async () => {
+        const res = await getRow(primaryPostgresTable._id, row.id)
+
+        expect(res.status).toBe(200)
+
+        expect(res.body).toEqual({
+          ...row,
+          _id: expect.any(String),
+          _rev: expect.any(String),
+        })
+        expect(res.body.foreignField).toBeUndefined()
+      })
+    })
+  })
+
+  describe("POST /api/:tableId/search", () => {
+    const search = (tableId: string | undefined, body?: object) =>
+      makeRequest("post", `/api/${tableId}/search`, body)
+
+    describe("search without parameters", () => {
+      describe("given than a table has no rows", () => {
+        it("search without query returns empty", async () => {
+          const res = await search(primaryPostgresTable._id)
+
+          expect(res.status).toBe(200)
+
+          expect(res.body).toEqual({
+            rows: [],
+            bookmark: null,
+            hasNextPage: false,
+          })
+        })
+      })
+
+      describe("given than a table has multiple rows", () => {
+        const rowsCount = 6
+        let rows: {
+          row: Row
+          rowData: PrimaryRowData
+        }[]
+        beforeEach(async () => {
+          rows = await populatePrimaryRows(rowsCount)
+        })
+
+        it("search without query returns all of them", async () => {
+          const res = await search(primaryPostgresTable._id)
+
+          expect(res.status).toBe(200)
+
+          expect(res.body).toEqual({
+            rows: expect.arrayContaining(
+              rows.map(r => expect.objectContaining(r.rowData))
+            ),
+            bookmark: null,
+            hasNextPage: false,
+          })
+          expect(res.body.rows).toHaveLength(rowsCount)
+        })
+      })
+
+      describe("given than multiple tables have multiple rows", () => {
+        const rowsCount = 6
+        beforeEach(async () => {
+          const createRandomTableWithRows = async () =>
+            await config.createRow({
+              tableId: (await createDefaultPgTable())._id,
+              title: generator.name(),
+            })
+
+          await createRandomTableWithRows()
+          await createRandomTableWithRows()
+
+          await populatePrimaryRows(rowsCount)
+
+          await createRandomTableWithRows()
+        })
+        it("search only return the requested ones", async () => {
+          const res = await search(primaryPostgresTable._id)
+
+          expect(res.status).toBe(200)
+
+          expect(res.body.rows).toHaveLength(rowsCount)
+        })
+      })
+    })
+
+    it("Querying by a string field returns the rows with field containing or starting by that value", async () => {
+      const name = generator.name()
+      const rowsToFilter = [
+        ...Array(2).fill({
+          name,
+          description: generator.paragraph(),
+          value: generator.age(),
+        }),
+        ...Array(2).fill({
+          name: `${name}${utils.newid()}`,
+          description: generator.paragraph(),
+          value: generator.age(),
+        }),
+      ]
+
+      await populatePrimaryRows(3)
+      for (const row of rowsToFilter) {
+        await createPrimaryRow({
+          rowData: row,
+        })
+      }
+      await populatePrimaryRows(1)
+
+      const res = await search(primaryPostgresTable._id, {
+        query: {
+          string: {
+            name,
+          },
+        },
+      })
+
+      expect(res.status).toBe(200)
+
+      expect(res.body).toEqual({
+        rows: expect.arrayContaining(rowsToFilter.map(expect.objectContaining)),
+        bookmark: null,
+        hasNextPage: false,
+      })
+      expect(res.body.rows).toHaveLength(4)
+    })
+
+    it("Querying respects the limit fields", async () => {
+      await populatePrimaryRows(6)
+
+      const res = await search(primaryPostgresTable._id, {
+        limit: 2,
+      })
+
+      expect(res.status).toBe(200)
+
+      expect(res.body.rows).toHaveLength(2)
+    })
+
+    describe("sort", () => {
+      beforeEach(async () => {
+        const defaultValue = generateRandomPrimaryRowData()
+
+        await createPrimaryRow({
+          rowData: {
+            ...defaultValue,
+            name: "d",
+            value: 3,
+          },
+        })
+        await createPrimaryRow({
+          rowData: { ...defaultValue, name: "aaa", value: 40 },
+        })
+        await createPrimaryRow({
+          rowData: { ...defaultValue, name: "ccccc", value: -5 },
+        })
+        await createPrimaryRow({
+          rowData: { ...defaultValue, name: "bb", value: 0 },
+        })
+      })
+
+      it("Querying respects the sort order when sorting ascending by a string value", async () => {
+        const res = await search(primaryPostgresTable._id, {
+          sort: "name",
+          sortOrder: "ascending",
+          sortType: "string",
+        })
+
+        expect(res.status).toBe(200)
+        expect(res.body.rows).toEqual([
+          expect.objectContaining({ name: "aaa" }),
+          expect.objectContaining({ name: "bb" }),
+          expect.objectContaining({ name: "ccccc" }),
+          expect.objectContaining({ name: "d" }),
+        ])
+      })
+
+      it("Querying respects the sort order when sorting descending by a string value", async () => {
+        const res = await search(primaryPostgresTable._id, {
+          sort: "name",
+          sortOrder: "descending",
+          sortType: "string",
+        })
+
+        expect(res.status).toBe(200)
+        expect(res.body.rows).toEqual([
+          expect.objectContaining({ name: "d" }),
+          expect.objectContaining({ name: "ccccc" }),
+          expect.objectContaining({ name: "bb" }),
+          expect.objectContaining({ name: "aaa" }),
+        ])
+      })
+
+      it("Querying respects the sort order when sorting ascending by a numeric value", async () => {
+        const res = await search(primaryPostgresTable._id, {
+          sort: "value",
+          sortOrder: "ascending",
+          sortType: "number",
+        })
+
+        expect(res.status).toBe(200)
+        expect(res.body.rows).toEqual([
+          expect.objectContaining({ value: -5 }),
+          expect.objectContaining({ value: 0 }),
+          expect.objectContaining({ value: 3 }),
+          expect.objectContaining({ value: 40 }),
+        ])
+      })
+
+      it("Querying respects the sort order when sorting descending by a numeric value", async () => {
+        const res = await search(primaryPostgresTable._id, {
+          sort: "value",
+          sortOrder: "descending",
+          sortType: "number",
+        })
+
+        expect(res.status).toBe(200)
+        expect(res.body.rows).toEqual([
+          expect.objectContaining({ value: 40 }),
+          expect.objectContaining({ value: 3 }),
+          expect.objectContaining({ value: 0 }),
+          expect.objectContaining({ value: -5 }),
+        ])
+      })
+    })
+  })
+
+  describe("GET /api/:tableId/:rowId/enrich", () => {
+    const getAll = (tableId: string | undefined, rowId: string | undefined) =>
+      makeRequest("get", `/api/${tableId}/${rowId}/enrich`)
+    describe("given a row with relation data", () => {
+      let row: Row, foreignRow: Row | undefined
+
+      beforeEach(async () => {
+        const rowsInfo = await createPrimaryRow({
+          rowData: generateRandomPrimaryRowData(),
+          createForeignRow: true,
+        })
+
+        row = rowsInfo.row
+        foreignRow = rowsInfo.foreignRow
+      })
+
+      it("enrich populates the foreign field", async () => {
+        const res = await getAll(primaryPostgresTable._id, row.id)
+
+        expect(res.status).toBe(200)
+
+        expect(foreignRow).toBeDefined()
+        expect(res.body).toEqual({
+          ...row,
+          linkedField: [
+            {
+              ...foreignRow,
+            },
+          ],
+        })
+      })
+    })
+  })
+
+  describe("GET /api/:tableId/rows", () => {
+    const getAll = (tableId: string | undefined) =>
+      makeRequest("get", `/api/${tableId}/rows`)
+
+    describe("given a table with no rows", () => {
+      it("get request returns empty", async () => {
+        const res = await getAll(primaryPostgresTable._id)
+
+        expect(res.status).toBe(200)
+
+        expect(res.body).toHaveLength(0)
+      })
+    })
+    describe("given a table with multiple rows", () => {
+      const rowsCount = 6
+      let rows: {
+        row: Row
+        foreignRow: Row | undefined
+        rowData: PrimaryRowData
+      }[]
+      beforeEach(async () => {
+        rows = await populatePrimaryRows(rowsCount)
+      })
+
+      it("get request returns all of them", async () => {
+        const res = await getAll(primaryPostgresTable._id)
+
+        expect(res.status).toBe(200)
+
+        expect(res.body).toHaveLength(rowsCount)
+        expect(res.body).toEqual(
+          expect.arrayContaining(
+            rows.map(r => expect.objectContaining(r.rowData))
+          )
+        )
+      })
+    })
+
+    describe("given multiple tables with multiple rows", () => {
+      const rowsCount = 6
+
+      beforeEach(async () => {
+        const createRandomTableWithRows = async () =>
+          await config.createRow({
+            tableId: (await createDefaultPgTable())._id,
+            title: generator.name(),
+          })
+
+        await createRandomTableWithRows()
+        await populatePrimaryRows(rowsCount)
+        await createRandomTableWithRows()
+      })
+
+      it("get returns the requested ones", async () => {
+        const res = await getAll(primaryPostgresTable._id)
+
+        expect(res.status).toBe(200)
+
+        expect(res.body).toHaveLength(rowsCount)
+      })
+    })
+  })
+})
--- a/packages/server/src/integrations/base/sql.ts
+++ b/packages/server/src/integrations/base/sql.ts
@ -90,10 +90,15 @@ function parseFilters(filters: SearchFilters | undefined): SearchFilters {
 function generateSelectStatement(
  json: QueryJson,
  knex: Knex
-): (string | Knex.Raw)[] {
+): (string | Knex.Raw)[] | "*" {
  const { resource, meta } = json
+
+  if (!resource) {
+    return "*"
+  }
+
  const schema = meta?.table?.schema
-  return resource!.fields.map(field => {
+  return resource.fields.map(field => {
    const fieldNames = field.split(/\./g)
    const tableName = fieldNames[0]
    const columnName = fieldNames[1]
@ -392,11 +397,14 @@ class InternalBuilder {
        delete parsedBody[key]
      }
    }
+
    // mysql can't use returning
    if (opts.disableReturning) {
      return query.insert(parsedBody)
    } else {
-      return query.insert(parsedBody).returning("*")
+      return query
+        .insert(parsedBody)
+        .returning(generateSelectStatement(json, knex))
    }
  }

@ -481,7 +489,9 @@ class InternalBuilder {
    if (opts.disableReturning) {
      return query.update(parsedBody)
    } else {
-      return query.update(parsedBody).returning("*")
+      return query
+        .update(parsedBody)
+        .returning(generateSelectStatement(json, knex))
    }
  }

@ -496,7 +506,7 @@ class InternalBuilder {
    if (opts.disableReturning) {
      return query.delete()
    } else {
-      return query.delete().returning("*")
+      return query.delete().returning(generateSelectStatement(json, knex))
    }
  }
 }
--- a/packages/server/src/integrations/index.ts
+++ b/packages/server/src/integrations/index.ts
@ -67,6 +67,15 @@ if (
  INTEGRATIONS[SourceName.ORACLE] = oracle.integration
 }

+export async function getDefinition(source: SourceName): Promise<Integration> {
+  // check if its integrated, faster
+  if (DEFINITIONS[source]) {
+    return DEFINITIONS[source]
+  }
+  const allDefinitions = await getDefinitions()
+  return allDefinitions[source]
+}
+
 export async function getDefinitions() {
  const pluginSchemas: { [key: string]: Integration } = {}
  if (env.SELF_HOSTED) {
--- a/packages/server/src/integrations/microsoftSqlServer.ts
+++ b/packages/server/src/integrations/microsoftSqlServer.ts
@ -26,7 +26,7 @@ interface MSSQLConfig {
  user: string
  password: string
  server: string
-  port: number
+  port: number | string
  database: string
  schema: string
  encrypt?: boolean
--- a/packages/server/src/integrations/oracle.ts
+++ b/packages/server/src/integrations/oracle.ts
@ -247,7 +247,7 @@ class OracleIntegration extends Sql implements DatasourcePlus {
    )
  }

-  private internalConvertType(column: OracleColumn): { type: string } {
+  private internalConvertType(column: OracleColumn): { type: FieldTypes } {
    if (this.isBooleanType(column)) {
      return { type: FieldTypes.BOOLEAN }
    }
--- a/packages/server/src/migrations/tests/index.spec.ts
+++ b/packages/server/src/migrations/tests/index.spec.ts
@ -93,24 +93,16 @@ describe("migrations", () => {
      await clearMigrations()
      const appId = config.prodAppId
      const roles = { [appId]: "role_12345" }
-      await config.createUser(
-        undefined,
-        undefined,
-        undefined,
-        undefined,
-        false,
-        true,
-        roles
-      ) // admin only
-      await config.createUser(
-        undefined,
-        undefined,
-        undefined,
-        undefined,
-        false,
-        false,
-        roles
-      ) // non admin non builder
+      await config.createUser({
+        builder: false,
+        admin: true,
+        roles,
+      }) // admin only
+      await config.createUser({
+        builder: false,
+        admin: false,
+        roles,
+      }) // non admin non builder
      await config.createTable()
      await config.createRow()
      await config.createRow()
--- a/packages/server/src/sdk/app/datasources/datasources.ts
+++ b/packages/server/src/sdk/app/datasources/datasources.ts
@ -3,6 +3,7 @@ import { findHBSBlocks, processObjectSync } from "@budibase/string-templates"
 import {
  Datasource,
  DatasourceFieldType,
+  Integration,
  PASSWORD_REPLACEMENT,
  RestAuthConfig,
  RestAuthType,
@ -11,16 +12,38 @@ import {
 } from "@budibase/types"
 import { cloneDeep } from "lodash/fp"
 import { getEnvironmentVariables } from "../../utils"
-import { getDefinitions } from "../../../integrations"
+import { getDefinitions, getDefinition } from "../../../integrations"

 const ENV_VAR_PREFIX = "env."

+export function checkDatasourceTypes(schema: Integration, config: any) {
+  for (let key of Object.keys(config)) {
+    if (!schema.datasource[key]) {
+      continue
+    }
+    const type = schema.datasource[key].type
+    if (
+      type === DatasourceFieldType.NUMBER &&
+      typeof config[key] === "string"
+    ) {
+      config[key] = parseFloat(config[key])
+    }
+  }
+  return config
+}
+
 async function enrichDatasourceWithValues(datasource: Datasource) {
  const cloned = cloneDeep(datasource)
  const env = await getEnvironmentVariables()
-  const processed = processObjectSync(cloned, { env }, { onlyFound: true })
+  const processed = processObjectSync(
+    cloned,
+    { env },
+    { onlyFound: true }
+  ) as Datasource
+  const definition = await getDefinition(processed.source)
+  processed.config = checkDatasourceTypes(definition, processed.config)
  return {
-    datasource: processed as Datasource,
+    datasource: processed,
    envVars: env as Record<string, string>,
  }
 }
@ -65,6 +88,9 @@ export async function removeSecrets(datasources: Datasource[]) {
  const definitions = await getDefinitions()
  for (let datasource of datasources) {
    const schema = definitions[datasource.source]
+    if (!schema) {
+      continue
+    }
    if (datasource.config) {
      // strip secrets from response, so they don't show in the network request
      if (datasource.config.auth) {
--- a/packages/server/src/tests/jestSetup.ts
+++ b/packages/server/src/tests/jestSetup.ts
@ -10,7 +10,9 @@ if (!process.env.DEBUG) {
 if (!process.env.CI) {
  // set a longer timeout in dev for debugging
  // 100 seconds
-  jest.setTimeout(100000)
+  jest.setTimeout(100 * 1000)
+} else {
+  jest.setTimeout(10 * 1000)
 }

 testContainerUtils.setupEnv(env, coreEnv)
--- a/packages/server/src/tests/utilities/TestConfiguration.ts
+++ b/packages/server/src/tests/utilities/TestConfiguration.ts
@ -39,8 +39,15 @@ import { cleanup } from "../../utilities/fileSystem"
 import newid from "../../db/newid"
 import { generateUserMetadataID } from "../../db/utils"
 import { startup } from "../../startup"
-import { AuthToken, Database } from "@budibase/types"
-const supertest = require("supertest")
+import supertest from "supertest"
+import {
+  AuthToken,
+  Database,
+  Datasource,
+  Row,
+  SourceName,
+  Table,
+} from "@budibase/types"

 type DefaultUserValues = {
  globalUserId: string
@ -52,7 +59,7 @@ type DefaultUserValues = {

 class TestConfiguration {
  server: any
-  request: any
+  request: supertest.SuperTest<supertest.Test> | undefined
  started: boolean
  appId: string | null
  allApps: any[]
@ -197,7 +204,7 @@ class TestConfiguration {

  // UTILS

-  async _req(body: any, params: any, controlFunc: any) {
+  _req(body: any, params: any, controlFunc: any) {
    // create a fake request ctx
    const request: any = {}
    const appId = this.appId
@ -268,14 +275,24 @@ class TestConfiguration {
  }

  async createUser(
-    id = null,
-    firstName = this.defaultUserValues.firstName,
-    lastName = this.defaultUserValues.lastName,
-    email = this.defaultUserValues.email,
-    builder = true,
-    admin = false,
-    roles = {}
+    user: {
+      id?: string
+      firstName?: string
+      lastName?: string
+      email?: string
+      builder?: boolean
+      admin?: boolean
+      roles?: any
+    } = {}
  ) {
+    let { id, firstName, lastName, email, builder, admin, roles } = user
+    firstName = firstName || this.defaultUserValues.firstName
+    lastName = lastName || this.defaultUserValues.lastName
+    email = email || this.defaultUserValues.email
+    roles = roles || {}
+    if (builder == null) {
+      builder = true
+    }
    const globalId = !id ? `us_${Math.random()}` : `us_${id}`
    const resp = await this.globalUser({
      id: globalId,
@ -360,6 +377,7 @@ class TestConfiguration {
      [constants.Header.CSRF_TOKEN]: this.defaultUserValues.csrfToken,
      ...extras,
    }
+
    if (this.appId) {
      headers[constants.Header.APP_ID] = this.appId
    }
@ -464,13 +482,13 @@ class TestConfiguration {

  // TABLE

-  async updateTable(config?: any) {
+  async updateTable(config?: any): Promise<Table> {
    config = config || basicTable()
    this.table = await this._req(config, null, controllers.table.save)
    return this.table
  }

-  async createTable(config?: any) {
+  async createTable(config?: Table) {
    if (config != null && config._id) {
      delete config._id
    }
@ -514,7 +532,7 @@ class TestConfiguration {

  // ROW

-  async createRow(config: any = null) {
+  async createRow(config?: Row): Promise<Row> {
    if (!this.table) {
      throw "Test requires table to be configured."
    }
@ -523,7 +541,7 @@ class TestConfiguration {
    return this._req(config, { tableId }, controllers.row.save)
  }

-  async getRow(tableId: string, rowId: string) {
+  async getRow(tableId: string, rowId: string): Promise<Row> {
    return this._req(null, { tableId, rowId }, controllers.row.find)
  }

@ -605,7 +623,9 @@ class TestConfiguration {

  // DATASOURCE

-  async createDatasource(config?: any) {
+  async createDatasource(config?: {
+    datasource: Datasource
+  }): Promise<Datasource> {
    config = config || basicDatasource()
    const response = await this._req(config, null, controllers.datasource.save)
    this.datasource = response.datasource
@ -626,7 +646,7 @@ class TestConfiguration {
    return this.createDatasource({
      datasource: {
        ...basicDatasource().datasource,
-        source: "REST",
+        source: SourceName.REST,
        config: cfg || {},
      },
    })
@ -635,7 +655,7 @@ class TestConfiguration {
  async dynamicVariableDatasource() {
    let datasource = await this.restDatasource()
    const basedOnQuery = await this.createQuery({
-      ...basicQuery(datasource._id),
+      ...basicQuery(datasource._id!),
      fields: {
        path: "www.google.com",
      },
@ -663,7 +683,7 @@ class TestConfiguration {
    datasource: any,
    fields: any,
    params: any,
-    verb: string
+    verb?: string
  ) {
    return request
      .post(`/api/queries/preview`)
--- a/packages/server/src/tests/utilities/structures.ts
+++ b/packages/server/src/tests/utilities/structures.ts
@ -7,6 +7,8 @@ import {
  Automation,
  AutomationActionStepId,
  AutomationTriggerStepId,
+  Datasource,
+  SourceName,
 } from "@budibase/types"

 const { v4: uuidv4 } = require("uuid")
@ -207,12 +209,12 @@ export function basicRole() {
  }
 }

-export function basicDatasource() {
+export function basicDatasource(): { datasource: Datasource } {
  return {
    datasource: {
      type: "datasource",
      name: "Test",
-      source: "POSTGRES",
+      source: SourceName.POSTGRES,
      config: {},
    },
  }
@ -255,3 +257,15 @@ export function basicWebhook(automationId: string) {
    },
  }
 }
+
+export function basicEnvironmentVariable(
+  name: string,
+  prod: string,
+  dev?: string
+) {
+  return {
+    name,
+    production: prod,
+    development: dev || prod,
+  }
+}
--- a/packages/server/src/threads/automation.ts
+++ b/packages/server/src/threads/automation.ts
@ -464,13 +464,17 @@ export function execute(job: Job, callback: WorkerCallback) {
    throw new Error("Unable to execute, event doesn't contain app ID.")
  }
  return context.doInAppContext(appId, async () => {
-    const automationOrchestrator = new Orchestrator(job)
-    try {
-      const response = await automationOrchestrator.execute()
-      callback(null, response)
-    } catch (err) {
-      callback(err)
-    }
+    const envVars = await sdkUtils.getEnvironmentVariables()
+    // put into automation thread for whole context
+    await context.doInEnvironmentContext(envVars, async () => {
+      const automationOrchestrator = new Orchestrator(job)
+      try {
+        const response = await automationOrchestrator.execute()
+        callback(null, response)
+      } catch (err) {
+        callback(err)
+      }
+    })
  })
 }

@ -480,11 +484,7 @@ export const removeStalled = async (job: Job) => {
    throw new Error("Unable to execute, event doesn't contain app ID.")
  }
  await context.doInAppContext(appId, async () => {
-    const envVars = await sdkUtils.getEnvironmentVariables()
-    // put into automation thread for whole context
-    await context.doInEnvironmentContext(envVars, async () => {
-      const automationOrchestrator = new Orchestrator(job)
-      await automationOrchestrator.stopCron("stalled")
-    })
+    const automationOrchestrator = new Orchestrator(job)
+    await automationOrchestrator.stopCron("stalled")
  })
 }
--- a/packages/server/yarn.lock
+++ b/packages/server/yarn.lock
@ -1278,14 +1278,14 @@
  resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
  integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==

-"@budibase/backend-core@2.3.2-alpha.0":
-  version "2.3.2-alpha.0"
-  resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-2.3.2-alpha.0.tgz#1ee32fe176f6144460c7722264a09045be10f0a7"
-  integrity sha512-J5TbzhkgJt6/+ehKoVPQNoNnaTCCXD4iQhsB3Td7xbuwjDq49P1a9ZJXKtsqr+CzcD7MsiGh7DiSslie6scBFA==
+"@budibase/backend-core@2.3.11-alpha.0":
+  version "2.3.11-alpha.0"
+  resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-2.3.11-alpha.0.tgz#361e30139a1a26d023902c6bdb4fdcac2610b69f"
+  integrity sha512-hlaeTkYsSJJYIwwqL3LJ7Pxzq0tOgSjQ38+nFiBPIzjkDSaV0UPwi0rZAZq/kPvaT7AKdTEcf8tn1wiui/VkYA==
  dependencies:
    "@budibase/nano" "10.1.1"
    "@budibase/pouchdb-replication-stream" "1.2.10"
-    "@budibase/types" "2.3.2-alpha.0"
+    "@budibase/types" "2.3.11-alpha.0"
    "@shopify/jest-koa-mocks" "5.0.1"
    "@techpass/passport-openidconnect" "0.3.2"
    aws-cloudfront-sign "2.2.0"
@ -1392,13 +1392,13 @@
    pouchdb-promise "^6.0.4"
    through2 "^2.0.0"

-"@budibase/pro@2.3.2-alpha.0":
-  version "2.3.2-alpha.0"
-  resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.3.2-alpha.0.tgz#b4406ec116f5fa008c7fc7327471302d9de4f29e"
-  integrity sha512-vVI/xZ6ybFSVGbHZvfomxCNyy0KvrBpYiOdUHjexPd60590tkT+cfZAuPrh3FdTTM8g2VipsxNWuO1OEQU7dcw==
+"@budibase/pro@2.3.11-alpha.0":
+  version "2.3.11-alpha.0"
+  resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.3.11-alpha.0.tgz#b59edba2c04e2a490f3d9246cd328d8dd3213295"
+  integrity sha512-z7tLgRKYKm1psNZGdRErQT8tV+vSj9hXJkD/H/uFxsJ3IQkjWbcFP5FUBPads5RnvCtqOfYu8OpRHCJfNNBoCA==
  dependencies:
-    "@budibase/backend-core" "2.3.2-alpha.0"
-    "@budibase/types" "2.3.2-alpha.0"
+    "@budibase/backend-core" "2.3.11-alpha.0"
+    "@budibase/types" "2.3.11-alpha.0"
    "@koa/router" "8.0.8"
    bull "4.10.1"
    joi "17.6.0"
@ -1424,10 +1424,10 @@
    svelte-apexcharts "^1.0.2"
    svelte-flatpickr "^3.1.0"

-"@budibase/types@2.3.2-alpha.0":
-  version "2.3.2-alpha.0"
-  resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.3.2-alpha.0.tgz#00b1afc4b937a5a49772bd78d189443f8e4eb0e3"
-  integrity sha512-kVwXzA8ROg27tIyNZnTd7ZszRepP4D7NgrW2gZsygsUoiYZ+WO1rCIlWj1SFPBur+V/k8dShQkh1hwxpj8JPVg==
+"@budibase/types@2.3.11-alpha.0":
+  version "2.3.11-alpha.0"
+  resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.3.11-alpha.0.tgz#55cdcef6fb30c79e9d7d90ae8789e4f0cc6a2b63"
+  integrity sha512-SFW9vManFRJ45bgMW7wkDiNxiQQfAME8HOmSqCHKDC0OPzsDydygIl7BNeF/domFvCXxDD3NPu7HC2JXMlV6cw==

 "@bull-board/api@3.7.0":
  version "3.7.0"
--- a/packages/string-templates/package.json
+++ b/packages/string-templates/package.json
@ -1,6 +1,6 @@
 {
  "name": "@budibase/string-templates",
-  "version": "2.3.2-alpha.0",
+  "version": "2.3.11-alpha.0",
  "description": "Handlebars wrapper for Budibase templating.",
  "main": "src/index.cjs",
  "module": "dist/bundle.mjs",
--- a/packages/string-templates/tsconfig.json
+++ b/packages/string-templates/tsconfig.json
@ -5,6 +5,7 @@
    "allowJs": true,
    "declaration": true,
    "emitDeclarationOnly": true,
-    "outDir": "dist"
+    "outDir": "dist",
+    "esModuleInterop": true
  }
 }
--- a/packages/types/package.json
+++ b/packages/types/package.json
@ -1,6 +1,6 @@
 {
  "name": "@budibase/types",
-  "version": "2.3.2-alpha.0",
+  "version": "2.3.11-alpha.0",
  "description": "Budibase types",
  "main": "dist/index.js",
  "types": "dist/index.d.ts",
--- a/packages/types/src/documents/app/row.ts
+++ b/packages/types/src/documents/app/row.ts
@ -14,6 +14,7 @@ export enum FieldType {
  AUTO = "auto",
  JSON = "json",
  INTERNAL = "internal",
+  BARCODEQR = "barcodeqr",
 }

 export interface RowAttachment {
--- a/packages/types/src/documents/app/table.ts
+++ b/packages/types/src/documents/app/table.ts
@ -1,16 +1,22 @@
 import { Document } from "../document"
 import { View } from "./view"
 import { RenameColumn } from "../../sdk"
+import { FieldType } from "./row"
+
+export enum RelationshipTypes {
+  ONE_TO_MANY = "one-to-many",
+  MANY_TO_ONE = "many-to-one",
+  MANY_TO_MANY = "many-to-many",
+}

 export interface FieldSchema {
-  // TODO: replace with field types enum when done
-  type: string
+  type: FieldType
  externalType?: string
  fieldName?: string
  name: string
  sortable?: boolean
  tableId?: string
-  relationshipType?: string
+  relationshipType?: RelationshipTypes
  through?: string
  foreignKey?: string
  icon?: string
--- a/packages/worker/package.json
+++ b/packages/worker/package.json
@ -1,7 +1,7 @@
 {
  "name": "@budibase/worker",
  "email": "hi@budibase.com",
-  "version": "2.3.2-alpha.0",
+  "version": "2.3.11-alpha.0",
  "description": "Budibase background service",
  "main": "src/index.ts",
  "repository": {
@ -36,10 +36,10 @@
  "author": "Budibase",
  "license": "GPL-3.0",
  "dependencies": {
-    "@budibase/backend-core": "2.3.2-alpha.0",
-    "@budibase/pro": "2.3.2-alpha.0",
-    "@budibase/string-templates": "2.3.2-alpha.0",
-    "@budibase/types": "2.3.2-alpha.0",
+    "@budibase/backend-core": "2.3.11-alpha.0",
+    "@budibase/pro": "2.3.11-alpha.0",
+    "@budibase/string-templates": "2.3.11-alpha.0",
+    "@budibase/types": "2.3.11-alpha.0",
    "@koa/router": "8.0.8",
    "@sentry/node": "6.17.7",
    "@techpass/passport-openidconnect": "0.3.2",
--- a/packages/worker/src/tests/jestSetup.ts
+++ b/packages/worker/src/tests/jestSetup.ts
@ -18,7 +18,9 @@ if (!process.env.DEBUG) {
 if (!process.env.CI) {
  // set a longer timeout in dev for debugging
  // 100 seconds
-  jest.setTimeout(100000)
+  jest.setTimeout(100 * 1000)
+} else {
+  jest.setTimeout(10 * 1000)
 }

 testContainerUtils.setupEnv(env, coreEnv)
--- a/packages/worker/yarn.lock
+++ b/packages/worker/yarn.lock
@ -475,14 +475,14 @@
  resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
  integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==

-"@budibase/backend-core@2.3.2-alpha.0":
-  version "2.3.2-alpha.0"
-  resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-2.3.2-alpha.0.tgz#1ee32fe176f6144460c7722264a09045be10f0a7"
-  integrity sha512-J5TbzhkgJt6/+ehKoVPQNoNnaTCCXD4iQhsB3Td7xbuwjDq49P1a9ZJXKtsqr+CzcD7MsiGh7DiSslie6scBFA==
+"@budibase/backend-core@2.3.11-alpha.0":
+  version "2.3.11-alpha.0"
+  resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-2.3.11-alpha.0.tgz#361e30139a1a26d023902c6bdb4fdcac2610b69f"
+  integrity sha512-hlaeTkYsSJJYIwwqL3LJ7Pxzq0tOgSjQ38+nFiBPIzjkDSaV0UPwi0rZAZq/kPvaT7AKdTEcf8tn1wiui/VkYA==
  dependencies:
    "@budibase/nano" "10.1.1"
    "@budibase/pouchdb-replication-stream" "1.2.10"
-    "@budibase/types" "2.3.2-alpha.0"
+    "@budibase/types" "2.3.11-alpha.0"
    "@shopify/jest-koa-mocks" "5.0.1"
    "@techpass/passport-openidconnect" "0.3.2"
    aws-cloudfront-sign "2.2.0"
@ -539,13 +539,13 @@
    pouchdb-promise "^6.0.4"
    through2 "^2.0.0"

-"@budibase/pro@2.3.2-alpha.0":
-  version "2.3.2-alpha.0"
-  resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.3.2-alpha.0.tgz#b4406ec116f5fa008c7fc7327471302d9de4f29e"
-  integrity sha512-vVI/xZ6ybFSVGbHZvfomxCNyy0KvrBpYiOdUHjexPd60590tkT+cfZAuPrh3FdTTM8g2VipsxNWuO1OEQU7dcw==
+"@budibase/pro@2.3.11-alpha.0":
+  version "2.3.11-alpha.0"
+  resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.3.11-alpha.0.tgz#b59edba2c04e2a490f3d9246cd328d8dd3213295"
+  integrity sha512-z7tLgRKYKm1psNZGdRErQT8tV+vSj9hXJkD/H/uFxsJ3IQkjWbcFP5FUBPads5RnvCtqOfYu8OpRHCJfNNBoCA==
  dependencies:
-    "@budibase/backend-core" "2.3.2-alpha.0"
-    "@budibase/types" "2.3.2-alpha.0"
+    "@budibase/backend-core" "2.3.11-alpha.0"
+    "@budibase/types" "2.3.11-alpha.0"
    "@koa/router" "8.0.8"
    bull "4.10.1"
    joi "17.6.0"
@ -553,10 +553,10 @@
    lru-cache "^7.14.1"
    node-fetch "^2.6.1"

-"@budibase/types@2.3.2-alpha.0":
-  version "2.3.2-alpha.0"
-  resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.3.2-alpha.0.tgz#00b1afc4b937a5a49772bd78d189443f8e4eb0e3"
-  integrity sha512-kVwXzA8ROg27tIyNZnTd7ZszRepP4D7NgrW2gZsygsUoiYZ+WO1rCIlWj1SFPBur+V/k8dShQkh1hwxpj8JPVg==
+"@budibase/types@2.3.11-alpha.0":
+  version "2.3.11-alpha.0"
+  resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.3.11-alpha.0.tgz#55cdcef6fb30c79e9d7d90ae8789e4f0cc6a2b63"
+  integrity sha512-SFW9vManFRJ45bgMW7wkDiNxiQQfAME8HOmSqCHKDC0OPzsDydygIl7BNeF/domFvCXxDD3NPu7HC2JXMlV6cw==

 "@cspotcode/source-map-support@^0.8.0":
  version "0.8.1"