Merge branch 'develop' of github.com:Budibase/budibase into grid-v2

2023-05-25 10:45:35 +01:00 · 2023-05-25 10:45:35 +01:00 · d491a24d3e
parent 475c962a5f 8c6efb8ebb
commit d491a24d3e
157 changed files with 10413 additions and 1815 deletions
--- a/.github/workflows/budibase_ci.yml
+++ b/.github/workflows/budibase_ci.yml
@ -22,44 +22,64 @@ jobs:
  lint:
    runs-on: ubuntu-latest
    steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
      - name: Use Node.js 14.x
-        uses: actions/setup-node@v1
+        uses: actions/setup-node@v3
        with:
          node-version: 14.x
+          cache: "yarn"
      - run: yarn
      - run: yarn lint

  build:
    runs-on: ubuntu-latest
    steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
        with:
          submodules: true
          token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
      - name: Use Node.js 14.x
-        uses: actions/setup-node@v1
+        uses: actions/setup-node@v3
        with:
          node-version: 14.x
+          cache: "yarn"
      - run: yarn
-      - run: yarn bootstrap
-      - run: yarn build
+      - run: yarn nx run-many -t=build --configuration=production

-  test:
+  test-libraries:
    runs-on: ubuntu-latest
    steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
        with:
          submodules: true
          token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
      - name: Use Node.js 14.x
-        uses: actions/setup-node@v1
+        uses: actions/setup-node@v3
        with:
          node-version: 14.x
+          cache: "yarn"
      - run: yarn
-      - run: yarn bootstrap
-      - run: yarn build
-      - run: yarn test --ignore=@budibase/pro
+      - run: yarn test --ignore=@budibase/worker --ignore=@budibase/server --ignore=@budibase/pro
+      - uses: codecov/codecov-action@v3
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }} # not required for public repos
+          name: codecov-umbrella
+          verbose: true
+
+  test-services:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          submodules: true
+          token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
+      - name: Use Node.js 14.x
+        uses: actions/setup-node@v3
+        with:
+          node-version: 14.x
+          cache: "yarn"
+      - run: yarn
+      - run: yarn test --scope=@budibase/worker --scope=@budibase/server
      - uses: codecov/codecov-action@v3
        with:
          token: ${{ secrets.CODECOV_TOKEN }} # not required for public repos
@ -69,32 +89,34 @@ jobs:
  test-pro:
    runs-on: ubuntu-latest
    steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
        with:
          submodules: true
          token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
      - name: Use Node.js 14.x
-        uses: actions/setup-node@v1
+        uses: actions/setup-node@v3
        with:
          node-version: 14.x
+          cache: "yarn"
      - run: yarn
-      - run: yarn bootstrap
-      - run: yarn build --scope=@budibase/types --scope=@budibase/shared-core
      - run: yarn test --scope=@budibase/pro

  integration-test:
    runs-on: ubuntu-latest
    steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
        with:
          submodules: true
          token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
      - name: Use Node.js 14.x
-        uses: actions/setup-node@v1
+        uses: actions/setup-node@v3
        with:
          node-version: 14.x
-      - run: yarn && yarn bootstrap && yarn build
-      - run: |
+          cache: "yarn"
+      - run: yarn
+      - run: yarn build
+      - name: Run tests
+        run: |
          cd qa-core
          yarn setup
          yarn test:ci
@ -106,7 +128,7 @@ jobs:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
        with:
          submodules: true
          token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
--- a/.github/workflows/release-develop.yml
+++ b/.github/workflows/release-develop.yml
@ -51,7 +51,7 @@ jobs:
          node scripts/syncLocalDependencies.js $version
          echo "Syncing yarn workspace"
          yarn
-      - run: yarn build
+      - run: yarn build --configuration=production
      - run: yarn build:sdk

      - name: Publish budibase packages to NPM
--- a/.github/workflows/release-master.yml
+++ b/.github/workflows/release-master.yml
@ -57,7 +57,7 @@ jobs:
          echo "Syncing yarn workspace"
          yarn
      - run: yarn lint
-      - run: yarn build
+      - run: yarn build --configuration=production
      - run: yarn build:sdk

      - name: Publish budibase packages to NPM
--- a/.nvmrc
+++ b/.nvmrc
@ -1 +1 @@
-v14.20.1
+v14.20.1
--- a/.python-version
+++ b/.python-version
@ -1 +1 @@
-3.10.0
+3.10.0
--- a/docs/CONTRIBUTING.md
+++ b/docs/CONTRIBUTING.md
@ -144,8 +144,6 @@ The following commands can be executed to manually get Budibase up and running (

 `yarn` to install project dependencies

-`yarn bootstrap` will install all budibase modules and symlink them together using lerna.
-
 `yarn build` will build all budibase packages.

 #### 4. Running
@ -243,7 +241,7 @@ An overview of the CI pipelines can be found [here](../.github/workflows/README.

 Note that only budibase maintainers will be able to access the pro repo.

-The `yarn bootstrap` command can be used to replace the NPM supplied dependency with the local source aware version. This is achieved using the `yarn link` command. To see specifically how dependencies are linked see [scripts/link-dependencies.sh](../scripts/link-dependencies.sh). The same link script is used to link dependencies to account-portal in local dev.
+By default, NX will make sure that dependencies are replaced with local source aware version. This is achieved using the `yarn link` command. To see specifically how dependencies are linked see [scripts/link-dependencies.sh](../scripts/link-dependencies.sh). The same link script is used to link dependencies to account-portal in local dev.

 ### Troubleshooting

--- a/hosting/docker-compose.build.yaml
+++ b/hosting/docker-compose.build.yaml
@ -0,0 +1,77 @@
+version: "3"
+
+# optional ports are specified throughout for more advanced use cases.
+
+services:
+  app-service:
+    build: ../packages/server
+    container_name: build-bbapps
+    environment:
+      SELF_HOSTED: 1
+      COUCH_DB_URL: http://${COUCH_DB_USER}:${COUCH_DB_PASSWORD}@couchdb-service:5984
+      WORKER_URL: http://worker-service:4003
+      MINIO_URL: http://minio-service:9000
+      MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY}
+      MINIO_SECRET_KEY: ${MINIO_SECRET_KEY}
+      INTERNAL_API_KEY: ${INTERNAL_API_KEY}
+      BUDIBASE_ENVIRONMENT: ${BUDIBASE_ENVIRONMENT}
+      PORT: 4002
+      API_ENCRYPTION_KEY: ${API_ENCRYPTION_KEY}
+      JWT_SECRET: ${JWT_SECRET}
+      LOG_LEVEL: info
+      SENTRY_DSN: https://a34ae347621946bf8acded18e5b7d4b8@o420233.ingest.sentry.io/5338131
+      ENABLE_ANALYTICS: "true"
+      REDIS_URL: redis-service:6379
+      REDIS_PASSWORD: ${REDIS_PASSWORD}
+      BB_ADMIN_USER_EMAIL: ${BB_ADMIN_USER_EMAIL}
+      BB_ADMIN_USER_PASSWORD: ${BB_ADMIN_USER_PASSWORD}
+      PLUGINS_DIR: ${PLUGINS_DIR}
+    depends_on:
+      - worker-service
+      - redis-service
+  #    volumes:
+  #      - /some/path/to/plugins:/plugins
+
+  worker-service:
+    build: ../packages/worker
+    container_name: build-bbworker
+    environment:
+      SELF_HOSTED: 1
+      PORT: 4003
+      CLUSTER_PORT: ${MAIN_PORT}
+      API_ENCRYPTION_KEY: ${API_ENCRYPTION_KEY}
+      JWT_SECRET: ${JWT_SECRET}
+      MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY}
+      MINIO_SECRET_KEY: ${MINIO_SECRET_KEY}
+      MINIO_URL: http://minio-service:9000
+      APPS_URL: http://app-service:4002
+      COUCH_DB_USERNAME: ${COUCH_DB_USER}
+      COUCH_DB_PASSWORD: ${COUCH_DB_PASSWORD}
+      COUCH_DB_URL: http://${COUCH_DB_USER}:${COUCH_DB_PASSWORD}@couchdb-service:5984
+      SENTRY_DSN: https://a34ae347621946bf8acded18e5b7d4b8@o420233.ingest.sentry.io/5338131
+      INTERNAL_API_KEY: ${INTERNAL_API_KEY}
+      REDIS_URL: redis-service:6379
+      REDIS_PASSWORD: ${REDIS_PASSWORD}
+    depends_on:
+      - redis-service
+      - minio-service
+
+  proxy-service-docker:
+    ports:
+      - "${MAIN_PORT}:10000"
+    container_name: build-bbproxy
+    image: budibase/proxy
+    environment:
+      - PROXY_RATE_LIMIT_WEBHOOKS_PER_SECOND=10
+      - PROXY_RATE_LIMIT_API_PER_SECOND=20
+      - APPS_UPSTREAM_URL=http://app-service:4002
+      - WORKER_UPSTREAM_URL=http://worker-service:4003
+      - MINIO_UPSTREAM_URL=http://minio-service:9000
+      - COUCHDB_UPSTREAM_URL=http://couchdb-service:5984
+      - WATCHTOWER_UPSTREAM_URL=http://watchtower-service:8080
+      - RESOLVER=127.0.0.11
+    depends_on:
+      - minio-service
+      - worker-service
+      - app-service
+      - couchdb-service
--- a/hosting/single/Dockerfile
+++ b/hosting/single/Dockerfile
@ -1,22 +1,22 @@
-FROM node:14-slim as build
+FROM node:16-slim as build

 # install node-gyp dependencies
 RUN apt-get update && apt-get upgrade -y && apt-get install -y --no-install-recommends apt-utils cron g++ make python

 # add pin script
 WORKDIR /
-ADD scripts/pinVersions.js scripts/cleanup.sh ./
+ADD scripts/cleanup.sh ./
 RUN chmod +x /cleanup.sh

 # build server
 WORKDIR /app
 ADD packages/server .
-RUN node /pinVersions.js && yarn && yarn build && /cleanup.sh
+RUN yarn install --frozen-lockfile --production=true && /cleanup.sh

 # build worker
 WORKDIR /worker
 ADD packages/worker .
-RUN node /pinVersions.js && yarn && yarn build && /cleanup.sh
+RUN yarn install --frozen-lockfile --production=true && /cleanup.sh

 FROM budibase/couchdb
 ARG TARGETARCH
@ -31,9 +31,7 @@ COPY --from=build /worker /worker

 # install base dependencies
 RUN apt-get update && \
-  apt-get install -y --no-install-recommends software-properties-common nginx uuid-runtime redis-server && \
-  apt-add-repository 'deb http://security.debian.org/debian-security bullseye-security/updates main' && \
-  apt-get update
+  apt-get install -y --no-install-recommends software-properties-common nginx uuid-runtime redis-server

 # install other dependencies, nodejs, oracle requirements, jdk8, redis, nginx
 WORKDIR /nodejs
--- a/lerna.json
+++ b/lerna.json
@ -1,5 +1,5 @@
 {
-  "version": "2.6.16-alpha.0",
+  "version": "2.6.19-alpha.7",
  "npmClient": "yarn",
  "packages": [
    "packages/backend-core",
--- a/nx.json
+++ b/nx.json
@ -6,5 +6,15 @@
        "cacheableOperations": ["build", "test"]
      }
    }
+  },
+  "targetDefaults": {
+    "dev:builder": {
+      "dependsOn": [
+        {
+          "projects": ["@budibase/string-templates"],
+          "target": "build"
+        }
+      ]
+    }
  }
 }
--- a/package.json
+++ b/package.json
@ -2,17 +2,23 @@
  "name": "root",
  "private": true,
  "devDependencies": {
+    "@esbuild-plugins/node-resolve": "^0.2.2",
+    "@nx/esbuild": "16.2.1",
+    "@nx/js": "16.2.1",
    "@rollup/plugin-json": "^4.0.2",
    "@typescript-eslint/parser": "5.45.0",
    "babel-eslint": "^10.0.3",
+    "esbuild": "^0.17.18",
    "eslint": "^7.28.0",
    "eslint-plugin-cypress": "^2.11.3",
    "eslint-plugin-svelte3": "^3.2.0",
    "husky": "^8.0.3",
    "js-yaml": "^4.1.0",
    "kill-port": "^1.6.1",
-    "lerna": "^6.6.1",
+    "lerna": "7.0.0-alpha.0",
    "madge": "^6.0.0",
+    "minimist": "^1.2.8",
+    "nx": "^16.2.1",
    "prettier": "^2.3.1",
    "prettier-plugin-svelte": "^2.3.0",
    "rimraf": "^3.0.2",
@ -23,9 +29,9 @@
  },
  "scripts": {
    "preinstall": "node scripts/syncProPackage.js",
-    "setup": "git config submodule.recurse true && git submodule update && node ./hosting/scripts/setup.js && yarn && yarn bootstrap && yarn build && yarn dev",
-    "bootstrap": "./scripts/bootstrap.sh && lerna link && ./scripts/link-dependencies.sh",
-    "build": "lerna run --stream build",
+    "setup": "git config submodule.recurse true && git submodule update && node ./hosting/scripts/setup.js && yarn && yarn build && yarn dev",
+    "bootstrap": "./scripts/link-dependencies.sh && echo '***BOOTSTRAP ONLY REQUIRED FOR USE WITH ACCOUNT PORTAL***'",
+    "build": "yarn nx run-many -t=build",
    "build:dev": "lerna run --stream prebuild && yarn nx run-many --target=build --output-style=dynamic --watch --preserveWatchOutput",
    "backend:bootstrap": "./scripts/scopeBackend.sh && yarn run bootstrap",
    "backend:build": "./scripts/scopeBackend.sh 'lerna run --stream build'",
@ -41,10 +47,11 @@
    "kill-builder": "kill-port 3000",
    "kill-server": "kill-port 4001 4002",
    "kill-all": "yarn run kill-builder && yarn run kill-server",
-    "dev": "yarn run kill-all && lerna link && lerna run --stream --parallel dev:builder --concurrency 1 --stream",
-    "dev:noserver": "yarn run kill-builder && lerna link && lerna run --stream dev:stack:up && lerna run --stream --parallel dev:builder --concurrency 1 --ignore @budibase/backend-core --ignore @budibase/server --ignore @budibase/worker",
-    "dev:server": "yarn run kill-server && lerna run --stream --parallel dev:builder --concurrency 1 --scope @budibase/backend-core --scope @budibase/worker --scope @budibase/server",
+    "dev": "yarn run kill-all  && lerna run --stream --parallel dev:builder --stream",
+    "dev:noserver": "yarn run kill-builder  && lerna run --stream dev:stack:up && lerna run --stream --parallel dev:builder --ignore @budibase/backend-core --ignore @budibase/server --ignore @budibase/worker",
+    "dev:server": "yarn run kill-server && lerna run --stream --parallel dev:builder --scope @budibase/worker --scope @budibase/server",
    "dev:built": "yarn run kill-all && cd packages/server && yarn dev:stack:up && cd ../../ && lerna run --stream --parallel dev:built",
+    "dev:docker": "yarn build && docker-compose -f hosting/docker-compose.dev.yaml -f hosting/docker-compose.build.yaml up --build --scale proxy-service=0 ",
    "test": "lerna run --stream test --stream",
    "lint:eslint": "eslint packages && eslint qa-core",
    "lint:prettier": "prettier --check \"packages/**/*.{js,ts,svelte}\" && prettier --write \"examples/**/*.{js,ts,svelte}\" && prettier --check \"qa-core/**/*.{js,ts,svelte}\"",
@ -53,16 +60,16 @@
    "lint:fix:prettier": "prettier --write \"packages/**/*.{js,ts,svelte}\" && prettier --write \"examples/**/*.{js,ts,svelte}\" && prettier --write \"qa-core/**/*.{js,ts,svelte}\"",
    "lint:fix": "yarn run lint:fix:prettier && yarn run lint:fix:eslint",
    "build:specs": "lerna run --stream specs",
-    "build:docker": "lerna run --stream build:docker && npm run build:docker:proxy && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh $BUDIBASE_RELEASE_VERSION && cd -",
+    "build:docker": "lerna run --stream build:docker && yarn build:docker:proxy && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh $BUDIBASE_RELEASE_VERSION && cd -",
    "build:docker:pre": "lerna run --stream build && lerna run --stream predocker",
    "build:docker:proxy": "docker build hosting/proxy -t proxy-service",
    "build:docker:selfhost": "lerna run --stream build:docker && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh latest && cd -",
-    "build:docker:develop": "node scripts/pinVersions && lerna run --stream build:docker && npm run build:docker:proxy && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh develop && cd -",
+    "build:docker:develop": "node scripts/pinVersions && lerna run --stream build:docker && yarn build:docker:proxy && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh develop && cd -",
    "build:docker:airgap": "node hosting/scripts/airgapped/airgappedDockerBuild",
    "build:digitalocean": "cd hosting/digitalocean && ./build.sh && cd -",
    "build:docker:single:multiarch": "docker buildx build --platform linux/arm64,linux/amd64 -f hosting/single/Dockerfile -t budibase:latest .",
    "build:docker:single:image": "docker build -f hosting/single/Dockerfile -t budibase:latest .",
-    "build:docker:single": "npm run build:docker:pre && npm run build:docker:single:image",
+    "build:docker:single": "yarn build && lerna run --concurrency 1 predocker && yarn build:docker:single:image",
    "build:docker:dependencies": "docker build -f hosting/dependencies/Dockerfile -t budibase/dependencies:latest ./hosting",
    "publish:docker:couch": "docker buildx build --platform linux/arm64,linux/amd64 -f hosting/couchdb/Dockerfile -t budibase/couchdb:latest -t budibase/couchdb:v3.2.1 --push ./hosting/couchdb",
    "publish:docker:dependencies": "docker buildx build --platform linux/arm64,linux/amd64 -f hosting/dependencies/Dockerfile -t budibase/dependencies:latest -t budibase/dependencies:v3.2.1 --push ./hosting",
@ -101,5 +108,6 @@
      "packages/worker",
      "packages/pro/packages/pro"
    ]
-  }
+  },
+  "dependencies": {}
 }
--- a/packages/backend-core/package.json
+++ b/packages/backend-core/package.json
@ -88,5 +88,19 @@
    "tsconfig-paths": "4.0.0",
    "typescript": "4.7.3"
  },
+  "nx": {
+    "targets": {
+      "build": {
+        "dependsOn": [
+          {
+            "projects": [
+              "@budibase/types"
+            ],
+            "target": "build"
+          }
+        ]
+      }
+    }
+  },
  "gitHead": "d1836a898cab3f8ab80ee6d8f42be1a9eed7dcdc"
 }
--- a/packages/backend-core/src/context/mainContext.ts
+++ b/packages/backend-core/src/context/mainContext.ts
@ -104,6 +104,22 @@ async function newContext(updates: ContextMap, task: any) {
  return Context.run(context, task)
 }

+export async function doInAutomationContext(params: {
+  appId: string
+  automationId: string
+  task: any
+}): Promise<any> {
+  const tenantId = getTenantIDFromAppID(params.appId)
+  return newContext(
+    {
+      tenantId,
+      appId: params.appId,
+      automationId: params.automationId,
+    },
+    params.task
+  )
+}
+
 export async function doInContext(appId: string, task: any): Promise<any> {
  const tenantId = getTenantIDFromAppID(appId)
  return newContext(
@ -187,6 +203,11 @@ export function getTenantId(): string {
  return tenantId
 }

+export function getAutomationId(): string | undefined {
+  const context = Context.get()
+  return context?.automationId
+}
+
 export function getAppId(): string | undefined {
  const context = Context.get()
  const foundId = context?.appId
--- a/packages/backend-core/src/context/types.ts
+++ b/packages/backend-core/src/context/types.ts
@ -7,4 +7,5 @@ export type ContextMap = {
  identity?: IdentityContext
  environmentVariables?: Record<string, string>
  isScim?: boolean
+  automationId?: string
 }
--- a/packages/backend-core/src/db/couch/DatabaseImpl.ts
+++ b/packages/backend-core/src/db/couch/DatabaseImpl.ts
@ -12,7 +12,7 @@ import {
  isDocument,
 } from "@budibase/types"
 import { getCouchInfo } from "./connections"
-import { directCouchCall } from "./utils"
+import { directCouchUrlCall } from "./utils"
 import { getPouchDB } from "./pouchDB"
 import { WriteStream, ReadStream } from "fs"
 import { newid } from "../../docIds/newid"
@ -46,6 +46,8 @@ export class DatabaseImpl implements Database {
  private readonly instanceNano?: Nano.ServerScope
  private readonly pouchOpts: DatabaseOpts

+  private readonly couchInfo = getCouchInfo()
+
  constructor(dbName?: string, opts?: DatabaseOpts, connection?: string) {
    if (dbName == null) {
      throw new Error("Database name cannot be undefined.")
@ -53,8 +55,8 @@ export class DatabaseImpl implements Database {
    this.name = dbName
    this.pouchOpts = opts || {}
    if (connection) {
-      const couchInfo = getCouchInfo(connection)
-      this.instanceNano = buildNano(couchInfo)
+      this.couchInfo = getCouchInfo(connection)
+      this.instanceNano = buildNano(this.couchInfo)
    }
    if (!DatabaseImpl.nano) {
      DatabaseImpl.init()
@ -67,7 +69,11 @@ export class DatabaseImpl implements Database {
  }

  async exists() {
-    let response = await directCouchCall(`/${this.name}`, "HEAD")
+    const response = await directCouchUrlCall({
+      url: `${this.couchInfo.url}/${this.name}`,
+      method: "HEAD",
+      cookie: this.couchInfo.cookie,
+    })
    return response.status === 200
  }

--- a/packages/backend-core/src/db/couch/connections.ts
+++ b/packages/backend-core/src/db/couch/connections.ts
@ -4,21 +4,21 @@ export const getCouchInfo = (connection?: string) => {
  const urlInfo = getUrlInfo(connection)
  let username
  let password
-  if (env.COUCH_DB_USERNAME) {
-    // set from env
-    username = env.COUCH_DB_USERNAME
-  } else if (urlInfo.auth.username) {
+  if (urlInfo.auth?.username) {
    // set from url
    username = urlInfo.auth.username
+  } else if (env.COUCH_DB_USERNAME) {
+    // set from env
+    username = env.COUCH_DB_USERNAME
  } else if (!env.isTest()) {
    throw new Error("CouchDB username not set")
  }
-  if (env.COUCH_DB_PASSWORD) {
-    // set from env
-    password = env.COUCH_DB_PASSWORD
-  } else if (urlInfo.auth.password) {
+  if (urlInfo.auth?.password) {
    // set from url
    password = urlInfo.auth.password
+  } else if (env.COUCH_DB_PASSWORD) {
+    // set from env
+    password = env.COUCH_DB_PASSWORD
  } else if (!env.isTest()) {
    throw new Error("CouchDB password not set")
  }
--- a/packages/backend-core/src/db/couch/utils.ts
+++ b/packages/backend-core/src/db/couch/utils.ts
@ -9,6 +9,20 @@ export async function directCouchCall(
 ) {
  let { url, cookie } = getCouchInfo()
  const couchUrl = `${url}/${path}`
+  return await directCouchUrlCall({ url: couchUrl, cookie, method, body })
+}
+
+export async function directCouchUrlCall({
+  url,
+  cookie,
+  method,
+  body,
+}: {
+  url: string
+  cookie: string
+  method: string
+  body?: any
+}) {
  const params: any = {
    method: method,
    headers: {
@ -19,7 +33,7 @@ export async function directCouchCall(
    params.body = JSON.stringify(body)
    params.headers["Content-Type"] = "application/json"
  }
-  return await fetch(checkSlashesInUrl(encodeURI(couchUrl)), params)
+  return await fetch(checkSlashesInUrl(encodeURI(url)), params)
 }

 export async function directCouchQuery(
--- a/packages/backend-core/src/logging/pino/logger.ts
+++ b/packages/backend-core/src/logging/pino/logger.ts
@ -39,6 +39,7 @@ if (!env.DISABLE_PINO_LOGGER) {
    objects?: any[]
    tenantId?: string
    appId?: string
+    automationId?: string
    identityId?: string
    identityType?: IdentityType
    correlationId?: string
@ -86,18 +87,44 @@ if (!env.DISABLE_PINO_LOGGER) {
      contextObject = {
        tenantId: getTenantId(),
        appId: getAppId(),
+        automationId: getAutomationId(),
        identityId: identity?._id,
        identityType: identity?.type,
        correlationId: correlation.getId(),
      }
    }

-    const mergingObject = {
-      objects: objects.length ? objects : undefined,
+    const mergingObject: any = {
      err: error,
      ...contextObject,
    }

+    if (objects.length) {
+      // init generic data object for params supplied that don't have a
+      // '_logKey' field. This prints an object using argument index as the key
+      // e.g. { 0: {}, 1: {} }
+      const data: any = {}
+      let dataIndex = 0
+
+      for (let i = 0; i < objects.length; i++) {
+        const object = objects[i]
+        // the object has specified a log key
+        // use this instead of generic key
+        const logKey = object._logKey
+        if (logKey) {
+          delete object._logKey
+          mergingObject[logKey] = object
+        } else {
+          data[dataIndex] = object
+          dataIndex++
+        }
+      }
+
+      if (Object.keys(data).length) {
+        mergingObject.data = data
+      }
+    }
+
    return [mergingObject, message]
  }

@ -159,6 +186,16 @@ if (!env.DISABLE_PINO_LOGGER) {
    return appId
  }

+  const getAutomationId = () => {
+    let appId
+    try {
+      appId = context.getAutomationId()
+    } catch (e) {
+      // do nothing
+    }
+    return appId
+  }
+
  const getIdentity = () => {
    let identity
    try {
--- a/packages/backend-core/src/queue/inMemoryQueue.ts
+++ b/packages/backend-core/src/queue/inMemoryQueue.ts
@ -128,6 +128,7 @@ class InMemoryQueue {

  on() {
    // do nothing
+    return this
  }

  async waitForCompletion() {
--- a/packages/backend-core/src/queue/listeners.ts
+++ b/packages/backend-core/src/queue/listeners.ts
@ -1,5 +1,6 @@
 import { Job, JobId, Queue } from "bull"
 import { JobQueue } from "./constants"
+import * as context from "../context"

 export type StalledFn = (job: Job) => Promise<void>

@ -31,77 +32,164 @@ function handleStalled(queue: Queue, removeStalledCb?: StalledFn) {
  })
 }

-function logging(queue: Queue, jobQueue: JobQueue) {
-  let eventType: string
-  switch (jobQueue) {
-    case JobQueue.AUTOMATION:
-      eventType = "automation-event"
-      break
-    case JobQueue.APP_BACKUP:
-      eventType = "app-backup-event"
-      break
-    case JobQueue.AUDIT_LOG:
-      eventType = "audit-log-event"
-      break
-    case JobQueue.SYSTEM_EVENT_QUEUE:
-      eventType = "system-event"
-      break
+function getLogParams(
+  eventType: QueueEventType,
+  event: BullEvent,
+  opts: {
+    job?: Job
+    jobId?: JobId
+    error?: Error
+  } = {},
+  extra: any = {}
+) {
+  const message = `[BULL] ${eventType}=${event}`
+  const err = opts.error
+
+  const bullLog = {
+    _logKey: "bull",
+    eventType,
+    event,
+    job: opts.job,
+    jobId: opts.jobId || opts.job?.id,
+    ...extra,
  }
+
+  let automationLog
+  if (opts.job?.data?.automation) {
+    automationLog = {
+      _logKey: "automation",
+      trigger: opts.job
+        ? opts.job.data.automation.definition.trigger.event
+        : undefined,
+    }
+  }
+
+  return [message, err, bullLog, automationLog]
+}
+
+enum BullEvent {
+  ERROR = "error",
+  WAITING = "waiting",
+  ACTIVE = "active",
+  STALLED = "stalled",
+  PROGRESS = "progress",
+  COMPLETED = "completed",
+  FAILED = "failed",
+  PAUSED = "paused",
+  RESUMED = "resumed",
+  CLEANED = "cleaned",
+  DRAINED = "drained",
+  REMOVED = "removed",
+}
+
+enum QueueEventType {
+  AUTOMATION_EVENT = "automation-event",
+  APP_BACKUP_EVENT = "app-backup-event",
+  AUDIT_LOG_EVENT = "audit-log-event",
+  SYSTEM_EVENT = "system-event",
+}
+
+const EventTypeMap: { [key in JobQueue]: QueueEventType } = {
+  [JobQueue.AUTOMATION]: QueueEventType.AUTOMATION_EVENT,
+  [JobQueue.APP_BACKUP]: QueueEventType.APP_BACKUP_EVENT,
+  [JobQueue.AUDIT_LOG]: QueueEventType.AUDIT_LOG_EVENT,
+  [JobQueue.SYSTEM_EVENT_QUEUE]: QueueEventType.SYSTEM_EVENT,
+}
+
+function logging(queue: Queue, jobQueue: JobQueue) {
+  const eventType = EventTypeMap[jobQueue]
+
+  function doInJobContext(job: Job, task: any) {
+    // if this is an automation job try to get the app id
+    const appId = job.data.event?.appId
+    if (appId) {
+      return context.doInContext(appId, task)
+    } else {
+      task()
+    }
+  }
+
+  queue
+    .on(BullEvent.STALLED, async (job: Job) => {
+      // A job has been marked as stalled. This is useful for debugging job
+      // workers that crash or pause the event loop.
+      await doInJobContext(job, () => {
+        console.error(...getLogParams(eventType, BullEvent.STALLED, { job }))
+      })
+    })
+    .on(BullEvent.ERROR, (error: any) => {
+      // An error occurred.
+      console.error(...getLogParams(eventType, BullEvent.ERROR, { error }))
+    })
+
  if (process.env.NODE_DEBUG?.includes("bull")) {
    queue
-      .on("error", (error: any) => {
-        // An error occurred.
-        console.error(`${eventType}=error error=${JSON.stringify(error)}`)
-      })
-      .on("waiting", (jobId: JobId) => {
+      .on(BullEvent.WAITING, (jobId: JobId) => {
        // A Job is waiting to be processed as soon as a worker is idling.
-        console.log(`${eventType}=waiting jobId=${jobId}`)
+        console.info(...getLogParams(eventType, BullEvent.WAITING, { jobId }))
      })
-      .on("active", (job: Job, jobPromise: any) => {
+      .on(BullEvent.ACTIVE, async (job: Job, jobPromise: any) => {
        // A job has started. You can use `jobPromise.cancel()`` to abort it.
-        console.log(`${eventType}=active jobId=${job.id}`)
+        await doInJobContext(job, () => {
+          console.info(...getLogParams(eventType, BullEvent.ACTIVE, { job }))
+        })
      })
-      .on("stalled", (job: Job) => {
-        // A job has been marked as stalled. This is useful for debugging job
-        // workers that crash or pause the event loop.
-        console.error(
-          `${eventType}=stalled jobId=${job.id} job=${JSON.stringify(job)}`
-        )
+      .on(BullEvent.PROGRESS, async (job: Job, progress: any) => {
+        // A job's progress was updated
+        await doInJobContext(job, () => {
+          console.info(
+            ...getLogParams(
+              eventType,
+              BullEvent.PROGRESS,
+              { job },
+              { progress }
+            )
+          )
+        })
      })
-      .on("progress", (job: Job, progress: any) => {
-        // A job's progress was updated!
-        console.log(
-          `${eventType}=progress jobId=${job.id} progress=${progress}`
-        )
-      })
-      .on("completed", (job: Job, result) => {
+      .on(BullEvent.COMPLETED, async (job: Job, result) => {
        // A job successfully completed with a `result`.
-        console.log(`${eventType}=completed jobId=${job.id} result=${result}`)
+        await doInJobContext(job, () => {
+          console.info(
+            ...getLogParams(eventType, BullEvent.COMPLETED, { job }, { result })
+          )
+        })
      })
-      .on("failed", (job, err: any) => {
+      .on(BullEvent.FAILED, async (job: Job, error: any) => {
        // A job failed with reason `err`!
-        console.log(`${eventType}=failed jobId=${job.id} error=${err}`)
+        await doInJobContext(job, () => {
+          console.error(
+            ...getLogParams(eventType, BullEvent.FAILED, { job, error })
+          )
+        })
      })
-      .on("paused", () => {
+      .on(BullEvent.PAUSED, () => {
        // The queue has been paused.
-        console.log(`${eventType}=paused`)
+        console.info(...getLogParams(eventType, BullEvent.PAUSED))
      })
-      .on("resumed", (job: Job) => {
+      .on(BullEvent.RESUMED, () => {
        // The queue has been resumed.
-        console.log(`${eventType}=paused jobId=${job.id}`)
+        console.info(...getLogParams(eventType, BullEvent.RESUMED))
      })
-      .on("cleaned", (jobs: Job[], type: string) => {
+      .on(BullEvent.CLEANED, (jobs: Job[], type: string) => {
        // Old jobs have been cleaned from the queue. `jobs` is an array of cleaned
        // jobs, and `type` is the type of jobs cleaned.
-        console.log(`${eventType}=cleaned length=${jobs.length} type=${type}`)
+        console.info(
+          ...getLogParams(
+            eventType,
+            BullEvent.CLEANED,
+            {},
+            { length: jobs.length, type }
+          )
+        )
      })
-      .on("drained", () => {
+      .on(BullEvent.DRAINED, () => {
        // Emitted every time the queue has processed all the waiting jobs (even if there can be some delayed jobs not yet processed)
-        console.log(`${eventType}=drained`)
+        console.info(...getLogParams(eventType, BullEvent.DRAINED))
      })
-      .on("removed", (job: Job) => {
+      .on(BullEvent.REMOVED, (job: Job) => {
        // A job successfully removed.
-        console.log(`${eventType}=removed jobId=${job.id}`)
+        console.info(...getLogParams(eventType, BullEvent.REMOVED, { job }))
      })
  }
 }
--- a/packages/bbui/package.json
+++ b/packages/bbui/package.json
@ -84,11 +84,25 @@
    "@spectrum-css/vars": "3.0.1",
    "dayjs": "^1.10.4",
    "easymde": "^2.16.1",
-    "svelte-flatpickr": "^3.3.2",
+    "svelte-flatpickr": "3.2.3",
    "svelte-portal": "^1.0.0"
  },
  "resolutions": {
    "loader-utils": "1.4.1"
  },
+  "nx": {
+    "targets": {
+      "build": {
+        "dependsOn": [
+          {
+            "projects": [
+              "@budibase/string-templates"
+            ],
+            "target": "build"
+          }
+        ]
+      }
+    }
+  },
  "gitHead": "d1836a898cab3f8ab80ee6d8f42be1a9eed7dcdc"
 }
--- a/packages/bbui/src/ActionButton/ActionButton.svelte
+++ b/packages/bbui/src/ActionButton/ActionButton.svelte
@ -102,7 +102,9 @@
    margin-left: 0;
    transition: color ease-out 130ms;
  }
-  .is-selected:not(.spectrum-ActionButton--emphasized):not(.spectrum-ActionButton--quiet) {
+  .is-selected:not(.spectrum-ActionButton--emphasized):not(
+      .spectrum-ActionButton--quiet
+    ) {
    background: var(--spectrum-global-color-gray-300);
    border-color: var(--spectrum-global-color-gray-500);
  }
--- a/packages/bbui/src/Button/Button.svelte
+++ b/packages/bbui/src/Button/Button.svelte
@ -2,6 +2,7 @@
  import "@spectrum-css/button/dist/index-vars.css"
  import Tooltip from "../Tooltip/Tooltip.svelte"

+  export let type
  export let disabled = false
  export let size = "M"
  export let cta = false
@ -21,6 +22,7 @@

 <button
  {id}
+  {type}
  class:spectrum-Button--cta={cta}
  class:spectrum-Button--primary={primary}
  class:spectrum-Button--secondary={secondary}
@ -73,6 +75,7 @@
  button {
    position: relative;
  }
+
  .spectrum-Button-label {
    white-space: nowrap;
    overflow: hidden;
--- a/packages/bbui/src/FancyForm/ErrorMessage.svelte
+++ b/packages/bbui/src/FancyForm/ErrorMessage.svelte
@ -0,0 +1,19 @@
+<script>
+  import { slide } from "svelte/transition"
+
+  export let error = null
+</script>
+
+<div transition:slide|local={{ duration: 130 }} class="error-message">
+  {error}
+</div>
+
+<style>
+  .error-message {
+    background: var(--spectrum-global-color-red-400);
+    color: white;
+    font-size: 14px;
+    padding: 6px 16px;
+    font-weight: 500;
+  }
+</style>
--- a/packages/bbui/src/FancyForm/FancyField.svelte
+++ b/packages/bbui/src/FancyForm/FancyField.svelte
@ -1,7 +1,7 @@
 <script>
  import Icon from "../Icon/Icon.svelte"
  import { getContext, onMount } from "svelte"
-  import { slide } from "svelte/transition"
+  import ErrorMessage from "./ErrorMessage.svelte"

  export let disabled = false
  export let error = null
@ -55,9 +55,7 @@
    {/if}
  </div>
  {#if error}
-    <div transition:slide|local={{ duration: 130 }} class="error-message">
-      {error}
-    </div>
+    <ErrorMessage {error} />
  {/if}
 </div>

@ -110,13 +108,6 @@
  .field {
    flex: 1 1 auto;
  }
-  .error-message {
-    background: var(--spectrum-global-color-red-400);
-    color: white;
-    font-size: 14px;
-    padding: 6px 16px;
-    font-weight: 500;
-  }
  .error-icon {
    flex: 0 0 auto;
  }
--- a/packages/bbui/src/FancyForm/index.js
+++ b/packages/bbui/src/FancyForm/index.js
@ -4,3 +4,4 @@ export { default as FancySelect } from "./FancySelect.svelte"
 export { default as FancyButton } from "./FancyButton.svelte"
 export { default as FancyForm } from "./FancyForm.svelte"
 export { default as FancyButtonRadio } from "./FancyButtonRadio.svelte"
+export { default as ErrorMessage } from "./ErrorMessage.svelte"
--- a/packages/builder/package.json
+++ b/packages/builder/package.json
@ -9,7 +9,7 @@
    "dev:builder": "routify -c dev:vite",
    "dev:vite": "vite --host 0.0.0.0",
    "rollup": "rollup -c -w",
-    "test": "vitest"
+    "test": "vitest run"
  },
  "jest": {
    "globals": {
@ -62,6 +62,7 @@
    "@budibase/frontend-core": "0.0.1",
    "@budibase/shared-core": "0.0.1",
    "@budibase/string-templates": "0.0.1",
+    "@budibase/types": "0.0.1",
    "@fortawesome/fontawesome-svg-core": "^6.2.1",
    "@fortawesome/free-brands-svg-icons": "^6.2.1",
    "@fortawesome/free-solid-svg-icons": "^6.2.1",
@ -116,5 +117,31 @@
    "vite": "^3.0.8",
    "vitest": "^0.29.2"
  },
+  "nx": {
+    "targets": {
+      "build": {
+        "dependsOn": [
+          {
+            "projects": [
+              "@budibase/string-templates",
+              "@budibase/shared-core"
+            ],
+            "target": "build"
+          }
+        ]
+      },
+      "test": {
+        "dependsOn": [
+          {
+            "projects": [
+              "@budibase/shared-core",
+              "@budibase/string-templates"
+            ],
+            "target": "build"
+          }
+        ]
+      }
+    }
+  },
  "gitHead": "115189f72a850bfb52b65ec61d932531bf327072"
 }
--- a/packages/builder/src/components/common/CodeMirrorEditor.svelte
+++ b/packages/builder/src/components/common/CodeMirrorEditor.svelte
@ -146,15 +146,18 @@

  /* Override default active line highlight colour in dark theme */
  div
-    :global(.CodeMirror-focused.cm-s-tomorrow-night-eighties
-      .CodeMirror-activeline-background) {
+    :global(
+      .CodeMirror-focused.cm-s-tomorrow-night-eighties
+        .CodeMirror-activeline-background
+    ) {
    background: rgba(255, 255, 255, 0.075);
  }

  /* Remove active line styling when not focused */
  div
-    :global(.CodeMirror:not(.CodeMirror-focused)
-      .CodeMirror-activeline-background) {
+    :global(
+      .CodeMirror:not(.CodeMirror-focused) .CodeMirror-activeline-background
+    ) {
    background: unset;
  }

--- a/packages/builder/src/components/portal/licensing/AccountLockedModal.svelte
+++ b/packages/builder/src/components/portal/licensing/AccountLockedModal.svelte
@ -0,0 +1,31 @@
+<script>
+  import { Modal, ModalContent, Body } from "@budibase/bbui"
+
+  let modal
+
+  export let onConfirm
+
+  export function show() {
+    modal.show()
+  }
+
+  export function hide() {
+    modal.hide()
+  }
+</script>
+
+<Modal bind:this={modal} on:hide={modal}>
+  <ModalContent
+    title="Your account is currently de-activated"
+    size="S"
+    showCancelButton={true}
+    showCloseIcon={false}
+    confirmText={"View plans"}
+    {onConfirm}
+  >
+    <Body size="S"
+      >Due to the free plan user limit being exceeded, your account has been
+      de-activated. Upgrade your plan to re-activate your account.</Body
+    >
+  </ModalContent>
+</Modal>
--- a/packages/builder/src/components/portal/licensing/licensingBanners.js
+++ b/packages/builder/src/components/portal/licensing/licensingBanners.js
@ -3,7 +3,6 @@ import { temporalStore } from "builderStore"
 import { admin, auth, licensing } from "stores/portal"
 import { get } from "svelte/store"
 import { BANNER_TYPES } from "@budibase/bbui"
-import { capitalise } from "helpers"

 const oneDayInSeconds = 86400

@ -146,23 +145,19 @@ const buildUsersAboveLimitBanner = EXPIRY_KEY => {
  const userLicensing = get(licensing)
  return {
    key: EXPIRY_KEY,
-    type: BANNER_TYPES.WARNING,
+    type: BANNER_TYPES.NEGATIVE,
    onChange: () => {
      defaultCacheFn(EXPIRY_KEY)
    },
    criteria: () => {
-      return userLicensing.warnUserLimit
+      return userLicensing.errUserLimit
    },
-    message: `${capitalise(
-      userLicensing.license.plan.type
-    )} plan changes - Users will be limited to ${
-      userLicensing.userLimit
-    } users in ${userLicensing.userLimitDays}`,
+    message: "Your Budibase account is de-activated. Upgrade your plan",
    ...{
-      extraButtonText: "Find out more",
+      extraButtonText: "View plans",
      extraButtonAction: () => {
        defaultCacheFn(ExpiringKeys.LICENSING_USERS_ABOVE_LIMIT_BANNER)
-        window.location.href = "/builder/portal/users/users"
+        window.location.href = "https://budibase.com/pricing/"
      },
    },
    showCloseButton: true,
--- a/packages/builder/src/components/start/AppRow.svelte
+++ b/packages/builder/src/components/start/AppRow.svelte
@ -6,6 +6,8 @@

  export let app

+  export let lockedAction
+
  const handleDefaultClick = () => {
    if (window.innerWidth < 640) {
      goToOverview()
@ -29,7 +31,7 @@
  }
 </script>

-<div class="app-row" on:click={handleDefaultClick}>
+<div class="app-row" on:click={lockedAction || handleDefaultClick}>
  <div class="title">
    <div class="app-icon">
      <Icon size="L" name={app.icon?.name || "Apps"} color={app.icon?.color} />
@ -58,8 +60,11 @@

  <div class="app-row-actions">
    <AppLockModal {app} buttonSize="M" />
-    <Button size="S" secondary on:click={goToOverview}>Manage</Button>
-    <Button size="S" primary on:click={goToBuilder}>Edit</Button>
+    <Button size="S" secondary on:click={lockedAction || goToOverview}
+      >Manage</Button
+    >
+    <Button size="S" primary on:click={lockedAction || goToBuilder}>Edit</Button
+    >
  </div>
 </div>

--- a/packages/builder/src/pages/builder/apps/index.svelte
+++ b/packages/builder/src/pages/builder/apps/index.svelte
@ -133,7 +133,7 @@
            </Body>
          </Layout>
          <Divider />
-          {#if $licensing.usageMetrics?.dayPasses >= 100}
+          {#if $licensing.usageMetrics?.dayPasses >= 100 || $licensing.errUserLimit}
            <div>
              <Layout gap="S" justifyItems="center">
                <img class="spaceman" alt="spaceman" src={Spaceman} />
--- a/packages/builder/src/pages/builder/portal/apps/index.svelte
+++ b/packages/builder/src/pages/builder/portal/apps/index.svelte
@ -14,6 +14,7 @@
  import Spinner from "components/common/Spinner.svelte"
  import CreateAppModal from "components/start/CreateAppModal.svelte"
  import AppLimitModal from "components/portal/licensing/AppLimitModal.svelte"
+  import AccountLockedModal from "components/portal/licensing/AccountLockedModal.svelte"

  import { store, automationStore } from "builderStore"
  import { API } from "api"
@ -28,6 +29,7 @@
  let template
  let creationModal
  let appLimitModal
+  let accountLockedModal
  let creatingApp = false
  let searchTerm = ""
  let creatingFromTemplate = false
@ -48,6 +50,11 @@
        : true)
  )
  $: automationErrors = getAutomationErrors(enrichedApps)
+  $: isOwner = $auth.accountPortalAccess && $admin.cloud
+
+  const usersLimitLockAction = $licensing?.errUserLimit
+    ? () => accountLockedModal.show()
+    : null

  const enrichApps = (apps, user, sortBy) => {
    const enrichedApps = apps.map(app => ({
@ -189,6 +196,9 @@
        creatingFromTemplate = true
        createAppFromTemplateUrl(initInfo.init_template)
      }
+      if (usersLimitLockAction) {
+        usersLimitLockAction()
+      }
    } catch (error) {
      notifications.error("Error getting init info")
    }
@ -230,20 +240,30 @@
        <Layout noPadding gap="L">
          <div class="title">
            <div class="buttons">
-              <Button size="M" cta on:click={initiateAppCreation}>
+              <Button
+                size="M"
+                cta
+                on:click={usersLimitLockAction || initiateAppCreation}
+              >
                Create new app
              </Button>
              {#if $apps?.length > 0}
                <Button
                  size="M"
                  secondary
-                  on:click={$goto("/builder/portal/apps/templates")}
+                  on:click={usersLimitLockAction ||
+                    $goto("/builder/portal/apps/templates")}
                >
                  View templates
                </Button>
              {/if}
              {#if !$apps?.length}
-                <Button size="L" quiet secondary on:click={initiateAppImport}>
+                <Button
+                  size="L"
+                  quiet
+                  secondary
+                  on:click={usersLimitLockAction || initiateAppImport}
+                >
                  Import app
                </Button>
              {/if}
@ -267,7 +287,7 @@

          <div class="app-table">
            {#each filteredApps as app (app.appId)}
-              <AppRow {app} />
+              <AppRow {app} lockedAction={usersLimitLockAction} />
            {/each}
          </div>
        </Layout>
@ -294,6 +314,11 @@
 </Modal>

 <AppLimitModal bind:this={appLimitModal} />
+<AccountLockedModal
+  bind:this={accountLockedModal}
+  onConfirm={() =>
+    isOwner ? $licensing.goToUpgradePage() : $licensing.goToPricingPage()}
+/>

 <style>
  .title {
--- a/packages/builder/src/pages/builder/portal/settings/auth/scim.svelte
+++ b/packages/builder/src/pages/builder/portal/settings/auth/scim.svelte
@ -115,27 +115,6 @@
    align-items: center;
  }

-  input[type="file"] {
-    display: none;
-  }
-  .sso-link-icon {
-    padding-top: 4px;
-    margin-left: 3px;
-  }
-  .sso-link {
-    margin-top: 12px;
-    display: flex;
-    flex-direction: row;
-    align-items: center;
-  }
-  .enforce-sso-title {
-    margin-right: 10px;
-  }
-  .enforce-sso-heading-container {
-    display: flex;
-    flex-direction: row;
-    align-items: start;
-  }
  .provider-title {
    display: flex;
    flex-direction: row;
@ -143,9 +122,6 @@
    align-items: center;
    gap: var(--spacing-m);
  }
-  .provider-title span {
-    flex: 1 1 auto;
-  }
  .inputContainer {
    display: flex;
    flex-direction: row;
--- a/packages/builder/src/pages/builder/portal/users/users/_components/AddUserModal.svelte
+++ b/packages/builder/src/pages/builder/portal/users/users/_components/AddUserModal.svelte
@ -30,8 +30,8 @@
  $: hasError = userData.find(x => x.error != null)

  $: userCount = $licensing.userCount + userData.length
-  $: willReach = licensing.willReachUserLimit(userCount)
-  $: willExceed = licensing.willExceedUserLimit(userCount)
+  $: reached = licensing.usersLimitReached(userCount)
+  $: exceeded = licensing.usersLimitExceeded(userCount)

  function removeInput(idx) {
    userData = userData.filter((e, i) => i !== idx)
@ -87,7 +87,7 @@
  confirmDisabled={disabled}
  cancelText="Cancel"
  showCloseIcon={false}
-  disabled={hasError || !userData.length || willExceed}
+  disabled={hasError || !userData.length || exceeded}
 >
  <Layout noPadding gap="XS">
    <Label>Email address</Label>
@ -118,7 +118,7 @@
      </div>
    {/each}

-    {#if willReach}
+    {#if reached}
      <div class="user-notification">
        <Icon name="Info" />
        <span>
--- a/packages/builder/src/pages/builder/portal/users/users/_components/ImportUsersModal.svelte
+++ b/packages/builder/src/pages/builder/portal/users/users/_components/ImportUsersModal.svelte
@ -25,10 +25,10 @@
  $: invalidEmails = []

  $: userCount = $licensing.userCount + userEmails.length
-  $: willExceed = licensing.willExceedUserLimit(userCount)
+  $: exceed = licensing.usersLimitExceeded(userCount)

  $: importDisabled =
-    !userEmails.length || !validEmails(userEmails) || !usersRole || willExceed
+    !userEmails.length || !validEmails(userEmails) || !usersRole || exceed

  const validEmails = userEmails => {
    if ($admin.cloud && userEmails.length > MAX_USERS_UPLOAD_LIMIT) {
@ -93,7 +93,7 @@
    </label>
  </div>

-  {#if willExceed}
+  {#if exceed}
    <div class="user-notification">
      <Icon name="Info" />
      {capitalise($licensing.license.plan.type)} plan is limited to {$licensing.userLimit}
--- a/packages/builder/src/pages/builder/portal/users/users/index.svelte
+++ b/packages/builder/src/pages/builder/portal/users/users/index.svelte
@ -268,8 +268,6 @@
      notifications.error("Error fetching user group data")
    }
  })
-
-  let staticUserLimit = $licensing.license.quotas.usage.static.users.value
 </script>

 <Layout noPadding gap="M">
@ -278,7 +276,7 @@
    <Body>Add users and control who gets access to your published apps</Body>
  </Layout>
  <Divider />
-  {#if $licensing.warnUserLimit}
+  {#if $licensing.errUserLimit}
    <InlineAlert
      type="error"
      onConfirm={() => {
@ -290,13 +288,9 @@
      }}
      buttonText={isOwner ? "Upgrade" : "View plans"}
      cta
-      header={`Users will soon be limited to ${staticUserLimit}`}
-      message={`Our free plan is going to be limited to ${staticUserLimit} users in ${$licensing.userLimitDays}.
-    
-    This means any users exceeding the limit will be de-activated.
-
-    De-activated users will not able to access the builder or any published apps until you upgrade to one of our paid plans.
-    `}
+      header="Account de-activated"
+      message="Due to the free plan user limit being exceeded, your account has been de-activated.
+      Upgrade your plan to re-activate your account."
    />
  {/if}
  <div class="controls">
--- a/packages/builder/src/stores/backend/datasources.js
+++ b/packages/builder/src/stores/backend/datasources.js
@ -25,6 +25,8 @@ export function createDatasourcesStore() {
    store.update(state => ({
      ...state,
      selectedDatasourceId: id,
+      // Remove any possible schema error
+      schemaError: null,
    }))
  }

--- a/packages/builder/src/stores/portal/licensing.js
+++ b/packages/builder/src/stores/portal/licensing.js
@ -4,7 +4,7 @@ import { auth, admin } from "stores/portal"
 import { Constants } from "@budibase/frontend-core"
 import { StripeStatus } from "components/portal/licensing/constants"
 import { TENANT_FEATURE_FLAGS, isEnabled } from "helpers/featureFlags"
-import dayjs from "dayjs"
+import { PlanModel } from "@budibase/types"

 const UNLIMITED = -1

@ -12,6 +12,7 @@ export const createLicensingStore = () => {
  const DEFAULT = {
    // navigation
    goToUpgradePage: () => {},
+    goToPricingPage: () => {},
    // the top level license
    license: undefined,
    isFreePlan: true,
@ -37,29 +38,37 @@ export const createLicensingStore = () => {
    // user limits
    userCount: undefined,
    userLimit: undefined,
-    userLimitDays: undefined,
    userLimitReached: false,
-    warnUserLimit: false,
+    errUserLimit: false,
  }

  const oneDayInMilliseconds = 86400000

  const store = writable(DEFAULT)

-  function willReachUserLimit(userCount, userLimit) {
+  function usersLimitReached(userCount, userLimit) {
    if (userLimit === UNLIMITED) {
      return false
    }
    return userCount >= userLimit
  }

-  function willExceedUserLimit(userCount, userLimit) {
+  function usersLimitExceeded(userCount, userLimit) {
    if (userLimit === UNLIMITED) {
      return false
    }
    return userCount > userLimit
  }

+  async function isCloud() {
+    let adminStore = get(admin)
+    if (!adminStore.loaded) {
+      await admin.init()
+      adminStore = get(admin)
+    }
+    return adminStore.cloud
+  }
+
  const actions = {
    init: async () => {
      actions.setNavigation()
@ -71,10 +80,14 @@ export const createLicensingStore = () => {
      const goToUpgradePage = () => {
        window.location.href = upgradeUrl
      }
+      const goToPricingPage = () => {
+        window.open("https://budibase.com/pricing/", "_blank")
+      }
      store.update(state => {
        return {
          ...state,
          goToUpgradePage,
+          goToPricingPage,
        }
      })
    },
@ -128,15 +141,15 @@ export const createLicensingStore = () => {
          quotaUsage,
        }
      })
-      actions.setUsageMetrics()
+      await actions.setUsageMetrics()
    },
-    willReachUserLimit: userCount => {
-      return willReachUserLimit(userCount, get(store).userLimit)
+    usersLimitReached: userCount => {
+      return usersLimitReached(userCount, get(store).userLimit)
    },
-    willExceedUserLimit(userCount) {
-      return willExceedUserLimit(userCount, get(store).userLimit)
+    usersLimitExceeded(userCount) {
+      return usersLimitExceeded(userCount, get(store).userLimit)
    },
-    setUsageMetrics: () => {
+    setUsageMetrics: async () => {
      if (isEnabled(TENANT_FEATURE_FLAGS.LICENSING)) {
        const usage = get(store).quotaUsage
        const license = get(auth).user.license
@ -198,11 +211,13 @@ export const createLicensingStore = () => {
        const userQuota = license.quotas.usage.static.users
        const userLimit = userQuota?.value
        const userCount = usage.usageQuota.users
-        const userLimitReached = willReachUserLimit(userCount, userLimit)
-        const userLimitExceeded = willExceedUserLimit(userCount, userLimit)
-        const days = dayjs(userQuota?.startDate).diff(dayjs(), "day")
-        const userLimitDays = days > 1 ? `${days} days` : "1 day"
-        const warnUserLimit = userQuota?.startDate && userLimitExceeded
+        const userLimitReached = usersLimitReached(userCount, userLimit)
+        const userLimitExceeded = usersLimitExceeded(userCount, userLimit)
+        const isCloudAccount = await isCloud()
+        const errUserLimit =
+          isCloudAccount &&
+          license.plan.model === PlanModel.PER_USER &&
+          userLimitExceeded

        store.update(state => {
          return {
@ -217,9 +232,8 @@ export const createLicensingStore = () => {
            // user limits
            userCount,
            userLimit,
-            userLimitDays,
            userLimitReached,
-            warnUserLimit,
+            errUserLimit,
          }
        })
      }
--- a/packages/builder/tsconfig.json
+++ b/packages/builder/tsconfig.json
@ -4,12 +4,7 @@
    "composite": true,
    "declaration": true,
    "sourceMap": true,
-    "baseUrl": ".",
-    "paths": {
-      "@budibase/types": ["../types/src"],
-      "@budibase/backend-core": ["../backend-core/src"],
-      "@budibase/backend-core/*": ["../backend-core/*.js"]
-    }
+    "baseUrl": "."
  },
  "ts-node": {
    "require": ["tsconfig-paths/register"]
--- a/packages/cli/package.json
+++ b/packages/cli/package.json
@ -63,5 +63,19 @@
    "renamer": "^4.0.0",
    "ts-node": "^10.9.1",
    "typescript": "4.7.3"
+  },
+  "nx": {
+    "targets": {
+      "build": {
+        "dependsOn": [
+          {
+            "projects": [
+              "@budibase/backend-core"
+            ],
+            "target": "build"
+          }
+        ]
+      }
+    }
  }
 }
--- a/packages/client/package.json
+++ b/packages/client/package.json
@ -65,5 +65,20 @@
  "resolutions": {
    "loader-utils": "1.4.1"
  },
+  "nx": {
+    "targets": {
+      "build": {
+        "dependsOn": [
+          {
+            "projects": [
+              "@budibase/string-templates",
+              "@budibase/shared-core"
+            ],
+            "target": "build"
+          }
+        ]
+      }
+    }
+  },
  "gitHead": "d1836a898cab3f8ab80ee6d8f42be1a9eed7dcdc"
 }
--- a/packages/client/src/components/app/charts/ApexChart.svelte
+++ b/packages/client/src/components/app/charts/ApexChart.svelte
@ -54,8 +54,9 @@
    color: white;
  }
  div
-    :global(.apexcharts-theme-dark
-      .apexcharts-tooltip-series-group.apexcharts-active) {
+    :global(
+      .apexcharts-theme-dark .apexcharts-tooltip-series-group.apexcharts-active
+    ) {
    padding-bottom: 0;
  }
 </style>
--- a/packages/client/src/components/app/forms/JSONField.svelte
+++ b/packages/client/src/components/app/forms/JSONField.svelte
@ -72,9 +72,11 @@
  :global(.spectrum-Form-itemField .spectrum-Textfield--multiline) {
    min-height: calc(var(--height) - 24px);
  }
-  :global(.spectrum-Form--labelsAbove
-      .spectrum-Form-itemField
-      .spectrum-Textfield--multiline) {
+  :global(
+      .spectrum-Form--labelsAbove
+        .spectrum-Form-itemField
+        .spectrum-Textfield--multiline
+    ) {
    min-height: calc(var(--height) - 24px);
  }
 </style>
--- a/packages/frontend-core/src/components/Testimonial.svelte
+++ b/packages/frontend-core/src/components/Testimonial.svelte
@ -0,0 +1,67 @@
+<script>
+  import { Layout } from "@budibase/bbui"
+  import Bulgaria from "../../assets/bulgaria.png"
+  import Covanta from "../../assets/covanta.png"
+  import Schnellecke from "../../assets/schnellecke.png"
+
+  const testimonials = [
+    {
+      text: "Budibase was the only solution that checked all the boxes for Covanta. Covanta expects to realize $3.2MM in savings due to the elimination of redundant data entry.",
+      name: "Charles Link",
+      role: "Senior Director, Data and Analytics",
+      image: Covanta,
+      imageSize: 105,
+    },
+    {
+      text: "Budibase was mission-critical for us and went a long way in preventing what could have become a humanitarian crisis here in Bulgaria.",
+      name: "Bozhidar Bozhanov",
+      role: "Government of Bulgaria",
+      image: Bulgaria,
+      imageSize: 49,
+    },
+    {
+      text: "Centralization of authentication, quick turnaround time for requests, integration with different database systems has given it the edge and it’s now used daily for internal development for those apps that you know you need but don’t feel value in losing days of development to reinvent the wheel.",
+      name: "Davide Lenzarini",
+      role: "IT manager",
+      image: Schnellecke,
+      imageSize: 141,
+    },
+  ]
+  const testimonial = testimonials[Math.floor(Math.random() * 3)]
+</script>
+
+<div class="testimonial">
+  <Layout noPadding gap="S">
+    <img
+      width={testimonial.imageSize}
+      alt="a-happy-budibase-user"
+      src={testimonial.image}
+    />
+    <div class="text">
+      "{testimonial.text}"
+    </div>
+    <div class="author">
+      <div class="name">{testimonial.name}</div>
+      <div class="company">{testimonial.role}</div>
+    </div>
+  </Layout>
+</div>
+
+<style>
+  .testimonial {
+    width: 380px;
+    padding: 40px;
+  }
+  .text {
+    font-size: var(--font-size-l);
+    font-style: italic;
+  }
+  .name {
+    font-weight: bold;
+    color: var(--spectrum-global-color-gray-900);
+    font-size: var(--font-size-l);
+  }
+  .company {
+    color: var(--spectrum-global-color-gray-700);
+  }
+</style>
--- a/packages/frontend-core/src/components/TestimonialPage.svelte
+++ b/packages/frontend-core/src/components/TestimonialPage.svelte
@ -1,58 +1,15 @@
 <script>
  import SplitPage from "./SplitPage.svelte"
-  import { Layout } from "@budibase/bbui"
-  import Bulgaria from "../../assets/bulgaria.png"
-  import Covanta from "../../assets/covanta.png"
-  import Schnellecke from "../../assets/schnellecke.png"
+  import Testimonial from "./Testimonial.svelte"

  export let enabled = true
-
-  const testimonials = [
-    {
-      text: "Budibase was the only solution that checked all the boxes for Covanta. Covanta expects to realize $3.2MM in savings due to the elimination of redundant data entry.",
-      name: "Charles Link",
-      role: "Senior Director, Data and Analytics",
-      image: Covanta,
-      imageSize: 105,
-    },
-    {
-      text: "Budibase was mission-critical for us and went a long way in preventing what could have become a humanitarian crisis here in Bulgaria.",
-      name: "Bozhidar Bozhanov",
-      role: "Government of Bulgaria",
-      image: Bulgaria,
-      imageSize: 49,
-    },
-    {
-      text: "Centralization of authentication, quick turnaround time for requests, integration with different database systems has given it the edge and it’s now used daily for internal development for those apps that you know you need but don’t feel value in losing days of development to reinvent the wheel.",
-      name: "Davide Lenzarini",
-      role: "IT manager",
-      image: Schnellecke,
-      imageSize: 141,
-    },
-  ]
-  const testimonial = testimonials[Math.floor(Math.random() * 3)]
 </script>

 <SplitPage>
  <slot />
  <div class:wrapper={enabled} slot="right">
    {#if enabled}
-      <div class="testimonial">
-        <Layout noPadding gap="S">
-          <img
-            width={testimonial.imageSize}
-            alt="a-happy-budibase-user"
-            src={testimonial.image}
-          />
-          <div class="text">
-            "{testimonial.text}"
-          </div>
-          <div class="author">
-            <div class="name">{testimonial.name}</div>
-            <div class="company">{testimonial.role}</div>
-          </div>
-        </Layout>
-      </div>
+      <Testimonial />
    {/if}
  </div>
 </SplitPage>
@ -64,20 +21,4 @@
    display: grid;
    place-items: center;
  }
-  .testimonial {
-    width: 380px;
-    padding: 40px;
-  }
-  .text {
-    font-size: var(--font-size-l);
-    font-style: italic;
-  }
-  .name {
-    font-weight: bold;
-    color: var(--spectrum-global-color-gray-900);
-    font-size: var(--font-size-l);
-  }
-  .company {
-    color: var(--spectrum-global-color-gray-700);
-  }
 </style>
--- a/packages/frontend-core/src/components/grid/cells/DateCell.svelte
+++ b/packages/frontend-core/src/components/grid/cells/DateCell.svelte
@ -13,10 +13,10 @@
  let flatpickr
  let isOpen

-  // adding the 0- will turn a string like 00:00:00 into a valid ISO
+  // Adding the 0- will turn a string like 00:00:00 into a valid ISO
  // date, but will make actual ISO dates invalid
-  $: time = new Date(`0-${value}`)
-  $: timeOnly = !isNaN(time) || schema?.timeOnly
+  $: isTimeValue = !isNaN(new Date(`0-${value}`))
+  $: timeOnly = isTimeValue || schema?.timeOnly
  $: dateOnly = schema?.dateOnly
  $: format = timeOnly
    ? "HH:mm:ss"
@ -24,6 +24,19 @@
    ? "MMM D YYYY"
    : "MMM D YYYY, HH:mm"
  $: editable = focused && !readonly
+  $: displayValue = getDisplayValue(value, format, timeOnly, isTimeValue)
+
+  const getDisplayValue = (value, format, timeOnly, isTimeValue) => {
+    if (!value) {
+      return ""
+    }
+    // Parse full date strings
+    if (!timeOnly || !isTimeValue) {
+      return dayjs(value).format(format)
+    }
+    // Otherwise must be a time string
+    return dayjs(`0-${value}`).format(format)
+  }

  // Ensure we close flatpickr when unselected
  $: {
@ -49,7 +62,7 @@
 <div class="container">
  <div class="value">
    {#if value}
-      {dayjs(timeOnly ? time : value).format(format)}
+      {displayValue}
    {/if}
  </div>
  {#if editable}
--- a/packages/frontend-core/src/components/index.js
+++ b/packages/frontend-core/src/components/index.js
@ -1,3 +1,4 @@
 export { default as SplitPage } from "./SplitPage.svelte"
 export { default as TestimonialPage } from "./TestimonialPage.svelte"
+export { default as Testimonial } from "./Testimonial.svelte"
 export { Grid } from "./grid"
--- a/packages/pro
+++ b/packages/pro
@ -1 +1 @@
-Subproject commit 64a2025727c25d5813832c92eb360de3947b7aa6
+Subproject commit aea8a4acb0bae6a1036520bf4c6d8cae428cc7d9
--- a/packages/server/.dockerignore
+++ b/packages/server/.dockerignore
@ -1,6 +1,7 @@
-node_modules
-npm-debug.log
-Dockerfile
-.dockerignore
-.git
-.gitignore
+*
+!/dist/
+!/scripts/integrations/oracle/
+!/package.json
+!/docker_run.sh
+!/builder/
+!/client/
--- a/packages/server/Dockerfile
+++ b/packages/server/Dockerfile
@ -15,22 +15,28 @@ ENV POSTHOG_TOKEN=phc_bIjZL7oh2GEUd2vqvTBH8WvrX0fWTFQMs6H5KQxiUxU
 ENV TENANT_FEATURE_FLAGS=*:LICENSING,*:USER_GROUPS,*:ONBOARDING_TOUR
 ENV ACCOUNT_PORTAL_URL=https://account.budibase.app

-# copy files and install dependencies
-COPY . ./
+
 # handle node-gyp
 RUN apt-get update \
-    && apt-get install -y --no-install-recommends g++ make python \
-    && yarn \
-    && yarn cache clean \
-    && apt-get remove -y --purge --auto-remove g++ make python \
-    && rm -rf /tmp/* /root/.node-gyp /usr/local/lib/node_modules/npm/node_modules/node-gyp
+    && apt-get install -y --no-install-recommends g++ make python
 RUN yarn global add pm2
-RUN yarn build

 # Install client for oracle datasource
 RUN apt-get install unzip libaio1
+COPY scripts/integrations/oracle/ scripts/integrations/oracle/
 RUN /bin/bash -e scripts/integrations/oracle/instantclient/linux/x86-64/install.sh 

+COPY package.json .
+RUN yarn install --frozen-lockfile --production=true
+# Remove unneeded data from file system to reduce image size
+RUN yarn cache clean && apt-get remove -y --purge --auto-remove g++ make python \
+    && rm -rf /tmp/* /root/.node-gyp /usr/local/lib/node_modules/npm/node_modules/node-gyp
+
+COPY dist/ dist/
+COPY docker_run.sh .
+COPY builder/ builder/
+COPY client/ client/
+
 EXPOSE 4001

 # have to add node environment production after install
@ -38,4 +44,6 @@ EXPOSE 4001
 # which are actually needed to get this environment up and running
 ENV NODE_ENV=production
 ENV CLUSTER_MODE=${CLUSTER_MODE}
+ENV TOP_LEVEL_PATH=/app
+
 CMD ["./docker_run.sh"]
--- a/packages/server/jest.config.ts
+++ b/packages/server/jest.config.ts
@ -1,6 +1,7 @@
 import { Config } from "@jest/types"

 import * as fs from "fs"
+import { join } from "path"
 const preset = require("ts-jest/jest-preset")

 const baseConfig: Config.InitialProjectOptions = {
@ -49,4 +50,6 @@ const config: Config.InitialOptions = {
  coverageReporters: ["lcov", "json", "clover"],
 }

+process.env.TOP_LEVEL_PATH = join(__dirname, "..", "..")
+
 export default config
--- a/packages/server/nodemon.json
+++ b/packages/server/nodemon.json
@ -6,5 +6,5 @@
    "src/**/*.spec.js",
    "../backend-core/dist/**/*"
  ],
-  "exec": "ts-node src/index.ts"
+  "exec": "node ./scripts/build.js && node ./dist/index.js"
 }
--- a/packages/server/package.json
+++ b/packages/server/package.json
@ -10,22 +10,22 @@
  },
  "scripts": {
    "prebuild": "rimraf dist/",
-    "build": "tsc -p tsconfig.build.json && mv dist/src/* dist/ && rimraf dist/src/",
+    "build": "node ./scripts/build.js",
+    "postbuild": "copyfiles -f ../client/dist/budibase-client.js ../client/manifest.json client",
    "build:dev": "yarn prebuild && tsc --build --watch --preserveWatchOutput",
    "debug": "yarn build && node --expose-gc --inspect=9222 dist/index.js",
-    "postbuild": "copyfiles -u 1 src/**/*.svelte dist/ && copyfiles -u 1 src/**/*.hbs dist/ && copyfiles -u 1 src/**/*.json dist/",
    "test": "bash scripts/test.sh",
    "test:memory": "jest --maxWorkers=2 --logHeapUsage --forceExit",
    "test:watch": "jest --watch",
-    "predocker": "copyfiles -f ../client/dist/budibase-client.js ../client/manifest.json client",
-    "build:docker": "yarn run predocker && docker build . -t app-service --label version=$BUDIBASE_RELEASE_VERSION",
+    "predocker": "copyfiles -f ../client/dist/budibase-client.js ../client/manifest.json client && yarn build --configuration=production",
+    "build:docker": "yarn predocker && docker build . -t app-service --label version=$BUDIBASE_RELEASE_VERSION",
    "build:docs": "node ./scripts/docs/generate.js open",
    "run:docker": "node dist/index.js",
    "run:docker:cluster": "pm2-runtime start pm2.config.js",
    "dev:stack:up": "node scripts/dev/manage.js up",
    "dev:stack:down": "node scripts/dev/manage.js down",
    "dev:stack:nuke": "node scripts/dev/manage.js nuke",
-    "dev:builder": "yarn run dev:stack:up && nodemon",
+    "dev:builder": "yarn run dev:stack:up && rimraf dist/ && nodemon",
    "dev:built": "yarn run dev:stack:up && yarn run run:docker",
    "specs": "ts-node specs/generate.ts && openapi-typescript specs/openapi.yaml --output src/definitions/openapi.ts",
    "initialise": "node scripts/initialise.js",
@ -99,7 +99,7 @@
    "mysql2": "2.3.3",
    "node-fetch": "2.6.7",
    "open": "8.4.0",
-    "pg": "8.5.1",
+    "pg": "8.10.0",
    "posthog-node": "1.3.0",
    "pouchdb": "7.3.0",
    "pouchdb-adapter-memory": "7.2.2",
@ -118,8 +118,8 @@
    "validate.js": "0.13.1",
    "vm2": "3.9.17",
    "worker-farm": "1.7.0",
-    "xml2js": "0.5.0",
-    "yargs": "13.2.4"
+    "yargs": "13.2.4",
+    "xml2js": "0.5.0"
  },
  "devDependencies": {
    "@babel/core": "7.17.4",
@ -141,6 +141,7 @@
    "@types/node": "14.18.20",
    "@types/node-fetch": "2.6.1",
    "@types/oracledb": "5.2.2",
+    "@types/pg": "8.6.6",
    "@types/pouchdb": "6.4.0",
    "@types/redis": "4.0.11",
    "@types/server-destroy": "1.0.1",
@ -176,5 +177,20 @@
  "optionalDependencies": {
    "oracledb": "5.3.0"
  },
+  "nx": {
+    "targets": {
+      "test": {
+        "dependsOn": [
+          {
+            "projects": [
+              "@budibase/string-templates",
+              "@budibase/shared-core"
+            ],
+            "target": "build"
+          }
+        ]
+      }
+    }
+  },
  "gitHead": "d1836a898cab3f8ab80ee6d8f42be1a9eed7dcdc"
 }
--- a/packages/server/pm2.config.js
+++ b/packages/server/pm2.config.js
@ -1,7 +1,7 @@
 module.exports = {
  apps: [
    {
-      script: "dist/index.js",
+      script: "./dist/index.js",
      instances: "max",
      exec_mode: "cluster",
    },
--- a/packages/server/scripts/build.js
+++ b/packages/server/scripts/build.js
@ -0,0 +1,48 @@
+#!/usr/bin/node
+const { join } = require("path")
+const fs = require("fs")
+const coreBuild = require("../../../scripts/build")
+
+const dir = join(__dirname, "..")
+const entryPath = join(dir, "src")
+const outfilePath = join(dir, "dist")
+
+/**
+ * The reasoning for this is that now our built version is simple
+ * dist/index.js - any kind of threaded approach in Node.js requires
+ * a runner file to work from - I played around with a lot of
+ * different methods, but we really want to be able to use forks.
+ *
+ * Rather than trying to rewrite so that forks run the whole system,
+ * I instead went down a path of building the individual threads so
+ * that we have runner files for each of them e.g. dist/automations.js
+ * and dist/query.js - these can be ran totally independently and then
+ * the parent process can pass down data for processing to them.
+ *
+ * The ignoring is simply to remove the files which really don't need
+ * to be built - they could be built and it wouldn't cause any issues,
+ * but this just means if any further threads are added in future
+ * they will naturally work (rather than including, which would mean
+ * adjustments to the build files).
+ */
+const ignoredFiles = ["definitions", "index", "utils"]
+const threadNames = fs
+  .readdirSync(join(dir, "src", "threads"))
+  .filter(path => !ignoredFiles.find(file => path.includes(file)))
+  .map(path => path.replace(".ts", ""))
+const files = [
+  {
+    entry: join(entryPath, "index.ts"),
+    out: join(outfilePath, "index.js"),
+  },
+]
+for (let name of threadNames) {
+  files.push({
+    entry: join(entryPath, "threads", `${name}.ts`),
+    out: join(outfilePath, `${name}.js`),
+  })
+}
+
+for (let file of files) {
+  coreBuild(file.entry, file.out)
+}
--- a/packages/server/src/api/controllers/datasource.ts
+++ b/packages/server/src/api/controllers/datasource.ts
@ -18,11 +18,72 @@ import {
  Row,
  CreateDatasourceResponse,
  UpdateDatasourceResponse,
-  UpdateDatasourceRequest,
  CreateDatasourceRequest,
+  VerifyDatasourceRequest,
+  VerifyDatasourceResponse,
+  FetchDatasourceInfoResponse,
+  IntegrationBase,
+  DatasourcePlus,
 } from "@budibase/types"
 import sdk from "../../sdk"

+function getErrorTables(errors: any, errorType: string) {
+  return Object.entries(errors)
+    .filter(entry => entry[1] === errorType)
+    .map(([name]) => name)
+}
+
+function updateError(error: any, newError: any, tables: string[]) {
+  if (!error) {
+    error = ""
+  }
+  if (error.length > 0) {
+    error += "\n"
+  }
+  error += `${newError} ${tables.join(", ")}`
+  return error
+}
+
+async function getConnector(
+  datasource: Datasource
+): Promise<IntegrationBase | DatasourcePlus> {
+  const Connector = await getIntegration(datasource.source)
+  // can't enrich if it doesn't have an ID yet
+  if (datasource._id) {
+    datasource = await sdk.datasources.enrich(datasource)
+  }
+  // Connect to the DB and build the schema
+  return new Connector(datasource.config)
+}
+
+async function buildSchemaHelper(datasource: Datasource) {
+  const connector = (await getConnector(datasource)) as DatasourcePlus
+  await connector.buildSchema(datasource._id!, datasource.entities!)
+
+  const errors = connector.schemaErrors
+  let error = null
+  if (errors && Object.keys(errors).length > 0) {
+    const noKey = getErrorTables(errors, BuildSchemaErrors.NO_KEY)
+    const invalidCol = getErrorTables(errors, BuildSchemaErrors.INVALID_COLUMN)
+    if (noKey.length) {
+      error = updateError(
+        error,
+        "No primary key constraint found for the following:",
+        noKey
+      )
+    }
+    if (invalidCol.length) {
+      const invalidCols = Object.values(InvalidColumns).join(", ")
+      error = updateError(
+        error,
+        `Cannot use columns ${invalidCols} found in following:`,
+        invalidCol
+      )
+    }
+  }
+  return { tables: connector.tables, error }
+}
+
 export async function fetch(ctx: UserCtx) {
  // Get internal tables
  const db = context.getAppDB()
@ -66,6 +127,48 @@ export async function fetch(ctx: UserCtx) {
  ctx.body = [bbInternalDb, ...datasources]
 }

+export async function verify(
+  ctx: UserCtx<VerifyDatasourceRequest, VerifyDatasourceResponse>
+) {
+  const { datasource } = ctx.request.body
+  let existingDatasource: undefined | Datasource
+  if (datasource._id) {
+    existingDatasource = await sdk.datasources.get(datasource._id)
+  }
+  let enrichedDatasource = datasource
+  if (existingDatasource) {
+    enrichedDatasource = sdk.datasources.mergeConfigs(
+      datasource,
+      existingDatasource
+    )
+  }
+  const connector = await getConnector(enrichedDatasource)
+  if (!connector.testConnection) {
+    ctx.throw(400, "Connection information verification not supported")
+  }
+  const response = await connector.testConnection()
+
+  ctx.body = {
+    connected: response.connected,
+    error: response.error,
+  }
+}
+
+export async function information(
+  ctx: UserCtx<void, FetchDatasourceInfoResponse>
+) {
+  const datasourceId = ctx.params.datasourceId
+  const datasource = await sdk.datasources.get(datasourceId, { enriched: true })
+  const connector = (await getConnector(datasource)) as DatasourcePlus
+  if (!connector.getTableNames) {
+    ctx.throw(400, "Table name fetching not supported by datasource")
+  }
+  const tableNames = await connector.getTableNames()
+  ctx.body = {
+    tableNames,
+  }
+}
+
 export async function buildSchemaFromDb(ctx: UserCtx) {
  const db = context.getAppDB()
  const datasource = await sdk.datasources.get(ctx.params.datasourceId)
@ -311,51 +414,3 @@ export async function query(ctx: UserCtx) {
    ctx.throw(400, err)
  }
 }
-
-function getErrorTables(errors: any, errorType: string) {
-  return Object.entries(errors)
-    .filter(entry => entry[1] === errorType)
-    .map(([name]) => name)
-}
-
-function updateError(error: any, newError: any, tables: string[]) {
-  if (!error) {
-    error = ""
-  }
-  if (error.length > 0) {
-    error += "\n"
-  }
-  error += `${newError} ${tables.join(", ")}`
-  return error
-}
-
-async function buildSchemaHelper(datasource: Datasource) {
-  const Connector = await getIntegration(datasource.source)
-  datasource = await sdk.datasources.enrich(datasource)
-  // Connect to the DB and build the schema
-  const connector = new Connector(datasource.config)
-  await connector.buildSchema(datasource._id, datasource.entities)
-
-  const errors = connector.schemaErrors
-  let error = null
-  if (errors && Object.keys(errors).length > 0) {
-    const noKey = getErrorTables(errors, BuildSchemaErrors.NO_KEY)
-    const invalidCol = getErrorTables(errors, BuildSchemaErrors.INVALID_COLUMN)
-    if (noKey.length) {
-      error = updateError(
-        error,
-        "No primary key constraint found for the following:",
-        noKey
-      )
-    }
-    if (invalidCol.length) {
-      const invalidCols = Object.values(InvalidColumns).join(", ")
-      error = updateError(
-        error,
-        `Cannot use columns ${invalidCols} found in following:`,
-        invalidCol
-      )
-    }
-  }
-  return { tables: connector.tables, error }
-}
--- a/packages/server/src/api/controllers/integration.ts
+++ b/packages/server/src/api/controllers/integration.ts
@ -1,4 +1,4 @@
-import { getDefinitions } from "../../integrations"
+import { getDefinition, getDefinitions } from "../../integrations"
 import { BBContext } from "@budibase/types"

 export async function fetch(ctx: BBContext) {
@ -7,7 +7,7 @@ export async function fetch(ctx: BBContext) {
 }

 export async function find(ctx: BBContext) {
-  const defs = await getDefinitions()
+  const def = await getDefinition(ctx.params.type)
+  ctx.body = def
  ctx.status = 200
-  ctx.body = defs[ctx.params.type]
 }
--- a/packages/server/src/api/controllers/static/index.ts
+++ b/packages/server/src/api/controllers/static/index.ts
@ -134,7 +134,7 @@ export const serveApp = async function (ctx: any) {
            ? objectStore.getGlobalFileUrl("settings", "logoUrl")
            : "",
      })
-      const appHbs = loadHandlebarsFile(`${__dirname}/templates/app.hbs`)
+      const appHbs = loadHandlebarsFile(`${__dirname}/app.hbs`)
      ctx.body = await processString(appHbs, {
        head,
        body: html,
@ -161,7 +161,7 @@ export const serveApp = async function (ctx: any) {
            : "",
      })

-      const appHbs = loadHandlebarsFile(`${__dirname}/templates/app.hbs`)
+      const appHbs = loadHandlebarsFile(`${__dirname}/app.hbs`)
      ctx.body = await processString(appHbs, {
        head,
        body: html,
@ -177,7 +177,7 @@ export const serveBuilderPreview = async function (ctx: any) {

  if (!env.isJest()) {
    let appId = context.getAppId()
-    const previewHbs = loadHandlebarsFile(`${__dirname}/templates/preview.hbs`)
+    const previewHbs = loadHandlebarsFile(`${__dirname}/preview.hbs`)
    ctx.body = await processString(previewHbs, {
      clientLibPath: objectStore.clientLibraryUrl(appId!, appInfo.version),
    })
--- a/packages/server/src/api/routes/datasource.ts
+++ b/packages/server/src/api/routes/datasource.ts
@ -15,6 +15,16 @@ router
    authorized(permissions.BUILDER),
    datasourceController.fetch
  )
+  .post(
+    "/api/datasources/verify",
+    authorized(permissions.BUILDER),
+    datasourceController.verify
+  )
+  .get(
+    "/api/datasources/:datasourceId/info",
+    authorized(permissions.BUILDER),
+    datasourceController.information
+  )
  .get(
    "/api/datasources/:datasourceId",
    authorized(
--- a/packages/server/src/api/routes/static.ts
+++ b/packages/server/src/api/routes/static.ts
@ -5,6 +5,7 @@ import authorized from "../../middleware/authorized"
 import { permissions } from "@budibase/backend-core"
 import env from "../../environment"
 import { paramResource } from "../../middleware/resourceId"
+import { devClientLibPath } from "../../utilities/fileSystem"
 const { BUILDER, PermissionType, PermissionLevel } = permissions

 const router: Router = new Router()
@ -17,7 +18,8 @@ router.param("file", async (file: any, ctx: any, next: any) => {
  }
  // test serves from require
  if (env.isTest()) {
-    ctx.devPath = require.resolve("@budibase/client").split(ctx.file)[0]
+    const path = devClientLibPath()
+    ctx.devPath = path.split(ctx.file)[0]
  } else if (env.isDev()) {
    // Serving the client library from your local dir in dev
    ctx.devPath = budibaseTempDir()
--- a/packages/server/src/api/routes/tests/datasource.spec.ts
+++ b/packages/server/src/api/routes/tests/datasource.spec.ts
@ -87,7 +87,7 @@ describe("/datasources", () => {
        expect(contents.rows.length).toEqual(1)

        // update the datasource to remove the variables
-        datasource.config.dynamicVariables = []
+        datasource.config!.dynamicVariables = []
        const res = await request
          .put(`/api/datasources/${datasource._id}`)
          .send(datasource)
--- a/packages/server/src/automations/triggers.ts
+++ b/packages/server/src/automations/triggers.ts
@ -9,7 +9,7 @@ import { checkTestFlag } from "../utilities/redis"
 import * as utils from "./utils"
 import env from "../environment"
 import { context, db as dbCore } from "@budibase/backend-core"
-import { Automation, Row } from "@budibase/types"
+import { Automation, Row, AutomationData, AutomationJob } from "@budibase/types"

 export const TRIGGER_DEFINITIONS = definitions
 const JOB_OPTS = {
@ -109,14 +109,16 @@ export async function externalTrigger(
    }
    params.fields = coercedFields
  }
-  const data: Record<string, any> = { automation, event: params }
+
+  const data: AutomationData = { automation, event: params as any }
  if (getResponses) {
    data.event = {
      ...data.event,
      appId: context.getAppId(),
      automation,
    }
-    return utils.processEvent({ data })
+    const job = { data } as AutomationJob
+    return utils.processEvent(job)
  } else {
    return automationQueue.add(data, JOB_OPTS)
  }
--- a/packages/server/src/automations/utils.ts
+++ b/packages/server/src/automations/utils.ts
@ -8,7 +8,7 @@ import { db as dbCore, context } from "@budibase/backend-core"
 import { getAutomationMetadataParams } from "../db/utils"
 import { cloneDeep } from "lodash/fp"
 import { quotas } from "@budibase/pro"
-import { Automation, WebhookActionType } from "@budibase/types"
+import { Automation, AutomationJob, WebhookActionType } from "@budibase/types"
 import sdk from "../sdk"

 const REBOOT_CRON = "@reboot"
@ -16,27 +16,40 @@ const WH_STEP_ID = definitions.WEBHOOK.stepId
 const CRON_STEP_ID = definitions.CRON.stepId
 const Runner = new Thread(ThreadType.AUTOMATION)

-const jobMessage = (job: any, message: string) => {
-  return `app=${job.data.event.appId} automation=${job.data.automation._id} jobId=${job.id} trigger=${job.data.automation.definition.trigger.event} : ${message}`
+function loggingArgs(job: AutomationJob) {
+  return [
+    {
+      _logKey: "automation",
+      trigger: job.data.automation.definition.trigger.event,
+    },
+    {
+      _logKey: "bull",
+      jobId: job.id,
+    },
+  ]
 }

-export async function processEvent(job: any) {
-  try {
-    const automationId = job.data.automation._id
-    console.log(jobMessage(job, "running"))
-    // need to actually await these so that an error can be captured properly
-    return await context.doInContext(job.data.event.appId, async () => {
+export async function processEvent(job: AutomationJob) {
+  const appId = job.data.event.appId!
+  const automationId = job.data.automation._id!
+  const task = async () => {
+    try {
+      // need to actually await these so that an error can be captured properly
+      console.log("automation running", ...loggingArgs(job))
+
      const runFn = () => Runner.run(job)
-      return quotas.addAutomation(runFn, {
+      const result = await quotas.addAutomation(runFn, {
        automationId,
      })
-    })
-  } catch (err) {
-    const errJson = JSON.stringify(err)
-    console.error(jobMessage(job, `was unable to run - ${errJson}`))
-    console.trace(err)
-    return { err }
+      console.log("automation completed", ...loggingArgs(job))
+      return result
+    } catch (err) {
+      console.error(`automation was unable to run`, err, ...loggingArgs(job))
+      return { err }
+    }
  }
+
+  return await context.doInAutomationContext({ appId, automationId, task })
 }

 export async function updateTestHistory(
--- a/packages/server/src/constants/index.ts
+++ b/packages/server/src/constants/index.ts
@ -1,4 +1,5 @@
 import { objectStore, roles, constants } from "@budibase/backend-core"
+import { FieldType as FieldTypes } from "@budibase/types"
 export { FieldType as FieldTypes, RelationshipTypes } from "@budibase/types"

 export enum FilterTypes {
@ -24,14 +25,14 @@ export const NoEmptyFilterStrings = [
 ]

 export const CanSwitchTypes = [
-  [exports.FieldTypes.JSON, exports.FieldTypes.ARRAY],
+  [FieldTypes.JSON, FieldTypes.ARRAY],
  [
-    exports.FieldTypes.STRING,
-    exports.FieldTypes.OPTIONS,
-    exports.FieldTypes.LONGFORM,
-    exports.FieldTypes.BARCODEQR,
+    FieldTypes.STRING,
+    FieldTypes.OPTIONS,
+    FieldTypes.LONGFORM,
+    FieldTypes.BARCODEQR,
  ],
-  [exports.FieldTypes.BOOLEAN, exports.FieldTypes.NUMBER],
+  [FieldTypes.BOOLEAN, FieldTypes.NUMBER],
 ]

 export const SwitchableTypes = CanSwitchTypes.reduce((prev, current) =>
@ -77,9 +78,9 @@ export const USERS_TABLE_SCHEMA = {
  // TODO: ADMIN PANEL - when implemented this doesn't need to be carried out
  schema: {
    email: {
-      type: exports.FieldTypes.STRING,
+      type: FieldTypes.STRING,
      constraints: {
-        type: exports.FieldTypes.STRING,
+        type: FieldTypes.STRING,
        email: true,
        length: {
          maximum: "",
@ -92,27 +93,27 @@ export const USERS_TABLE_SCHEMA = {
    firstName: {
      name: "firstName",
      fieldName: "firstName",
-      type: exports.FieldTypes.STRING,
+      type: FieldTypes.STRING,
      constraints: {
-        type: exports.FieldTypes.STRING,
+        type: FieldTypes.STRING,
        presence: false,
      },
    },
    lastName: {
      name: "lastName",
      fieldName: "lastName",
-      type: exports.FieldTypes.STRING,
+      type: FieldTypes.STRING,
      constraints: {
-        type: exports.FieldTypes.STRING,
+        type: FieldTypes.STRING,
        presence: false,
      },
    },
    roleId: {
      fieldName: "roleId",
      name: "roleId",
-      type: exports.FieldTypes.OPTIONS,
+      type: FieldTypes.OPTIONS,
      constraints: {
-        type: exports.FieldTypes.STRING,
+        type: FieldTypes.STRING,
        presence: false,
        inclusion: Object.values(roles.BUILTIN_ROLE_IDS),
      },
@ -120,9 +121,9 @@ export const USERS_TABLE_SCHEMA = {
    status: {
      fieldName: "status",
      name: "status",
-      type: exports.FieldTypes.OPTIONS,
+      type: FieldTypes.OPTIONS,
      constraints: {
-        type: exports.FieldTypes.STRING,
+        type: FieldTypes.STRING,
        presence: false,
        inclusion: Object.values(constants.UserStatus),
      },
--- a/packages/server/src/db/dynamoClient.ts
+++ b/packages/server/src/db/dynamoClient.ts
@ -140,7 +140,7 @@ export function init(endpoint: string) {
  docClient = new AWS.DynamoDB.DocumentClient(docClientParams)
 }

-if (!env.isProd()) {
+if (!env.isProd() && !env.isJest()) {
  env._set("AWS_ACCESS_KEY_ID", "KEY_ID")
  env._set("AWS_SECRET_ACCESS_KEY", "SECRET_KEY")
  init("http://localhost:8333")
--- a/packages/server/src/db/inMemoryView.ts
+++ b/packages/server/src/db/inMemoryView.ts
@ -25,7 +25,9 @@ export async function runView(
      }))
    )
    let fn = (doc: Document, emit: any) => emit(doc._id)
-    eval("fn = " + view?.map?.replace("function (doc)", "function (doc, emit)"))
+    ;(0, eval)(
+      "fn = " + view?.map?.replace("function (doc)", "function (doc, emit)")
+    )
    const queryFns: any = {
      meta: view.meta,
      map: fn,
--- a/packages/server/src/definitions/automations.ts
+++ b/packages/server/src/definitions/automations.ts
@ -1,4 +1,4 @@
-import { AutomationResults, AutomationStep, Document } from "@budibase/types"
+import { AutomationResults, AutomationStep } from "@budibase/types"

 export enum LoopStepType {
  ARRAY = "Array",
@ -27,7 +27,3 @@ export interface AutomationContext extends AutomationResults {
  env?: Record<string, string>
  trigger: any
 }
-
-export interface AutomationMetadata extends Document {
-  errorCount?: number
-}
--- a/packages/server/src/environment.ts
+++ b/packages/server/src/environment.ts
@ -96,6 +96,7 @@ const environment = {
  isInThread: () => {
    return process.env.FORKED_PROCESS
  },
+  TOP_LEVEL_PATH: process.env.TOP_LEVEL_PATH,
 }

 // threading can cause memory issues with node-ts in development
--- a/packages/server/src/integration-test/postgres.spec.ts
+++ b/packages/server/src/integration-test/postgres.spec.ts
@ -19,7 +19,6 @@ import _ from "lodash"
 import { generator } from "@budibase/backend-core/tests"
 import { utils } from "@budibase/backend-core"
 import { GenericContainer } from "testcontainers"
-import { generateRowIdField } from "../integrations/utils"

 const config = setup.getConfig()!

@ -27,7 +26,7 @@ jest.setTimeout(30000)

 jest.unmock("pg")

-describe("row api - postgres", () => {
+describe("postgres integrations", () => {
  let makeRequest: MakeRequestResponse,
    postgresDatasource: Datasource,
    primaryPostgresTable: Table,
@ -53,8 +52,8 @@ describe("row api - postgres", () => {
    makeRequest = generateMakeRequest(apiKey, true)
  })

-  beforeEach(async () => {
-    postgresDatasource = await config.createDatasource({
+  function pgDatasourceConfig() {
+    return {
      datasource: {
        type: "datasource",
        source: SourceName.POSTGRES,
@ -71,7 +70,11 @@ describe("row api - postgres", () => {
          ca: false,
        },
      },
-    })
+    }
+  }
+
+  beforeEach(async () => {
+    postgresDatasource = await config.createDatasource(pgDatasourceConfig())

    async function createAuxTable(prefix: string) {
      return await config.createTable({
@ -1025,4 +1028,43 @@ describe("row api - postgres", () => {
      })
    })
  })
+
+  describe("POST /api/datasources/verify", () => {
+    it("should be able to verify the connection", async () => {
+      const config = pgDatasourceConfig()
+      const response = await makeRequest(
+        "post",
+        "/api/datasources/verify",
+        config
+      )
+      expect(response.status).toBe(200)
+      expect(response.body.connected).toBe(true)
+    })
+
+    it("should state an invalid datasource cannot connect", async () => {
+      const config = pgDatasourceConfig()
+      config.datasource.config.password = "wrongpassword"
+      const response = await makeRequest(
+        "post",
+        "/api/datasources/verify",
+        config
+      )
+      expect(response.status).toBe(200)
+      expect(response.body.connected).toBe(false)
+      expect(response.body.error).toBeDefined()
+    })
+  })
+
+  describe("GET /api/datasources/:datasourceId/info", () => {
+    it("should fetch information about postgres datasource", async () => {
+      const primaryName = primaryPostgresTable.name
+      const response = await makeRequest(
+        "get",
+        `/api/datasources/${postgresDatasource._id}/info`
+      )
+      expect(response.status).toBe(200)
+      expect(response.body.tableNames).toBeDefined()
+      expect(response.body.tableNames.indexOf(primaryName)).not.toBe(-1)
+    })
+  })
 })
--- a/packages/server/src/integrations/airtable.ts
+++ b/packages/server/src/integrations/airtable.ts
@ -1,11 +1,13 @@
 import {
-  Integration,
+  ConnectionInfo,
+  DatasourceFeature,
  DatasourceFieldType,
-  QueryType,
+  Integration,
  IntegrationBase,
+  QueryType,
 } from "@budibase/types"

-const Airtable = require("airtable")
+import Airtable from "airtable"

 interface AirtableConfig {
  apiKey: string
@ -18,6 +20,7 @@ const SCHEMA: Integration = {
    "Airtable is a spreadsheet-database hybrid, with the features of a database but applied to a spreadsheet.",
  friendlyName: "Airtable",
  type: "Spreadsheet",
+  features: [DatasourceFeature.CONNECTION_CHECKING],
  datasource: {
    apiKey: {
      type: DatasourceFieldType.PASSWORD,
@ -81,13 +84,37 @@ const SCHEMA: Integration = {

 class AirtableIntegration implements IntegrationBase {
  private config: AirtableConfig
-  private client: any
+  private client

  constructor(config: AirtableConfig) {
    this.config = config
    this.client = new Airtable(config).base(config.base)
  }

+  async testConnection(): Promise<ConnectionInfo> {
+    const mockTable = Date.now().toString()
+    try {
+      await this.client.makeRequest({
+        path: `/${mockTable}`,
+      })
+
+      return { connected: true }
+    } catch (e: any) {
+      if (
+        e.message ===
+        `Could not find table ${mockTable} in application ${this.config.base}`
+      ) {
+        // The request managed to check the application, so the credentials are valid
+        return { connected: true }
+      }
+
+      return {
+        connected: false,
+        error: e.message as string,
+      }
+    }
+  }
+
  async create(query: { table: any; json: any }) {
    const { table, json } = query

--- a/packages/server/src/integrations/arangodb.ts
+++ b/packages/server/src/integrations/arangodb.ts
@ -3,9 +3,11 @@ import {
  DatasourceFieldType,
  QueryType,
  IntegrationBase,
+  DatasourceFeature,
+  ConnectionInfo,
 } from "@budibase/types"

-const { Database, aql } = require("arangojs")
+import { Database, aql } from "arangojs"

 interface ArangodbConfig {
  url: string
@ -21,6 +23,7 @@ const SCHEMA: Integration = {
  type: "Non-relational",
  description:
    "ArangoDB is a scalable open-source multi-model database natively supporting graph, document and search. All supported data models & access patterns can be combined in queries allowing for maximal flexibility. ",
+  features: [DatasourceFeature.CONNECTION_CHECKING],
  datasource: {
    url: {
      type: DatasourceFieldType.STRING,
@ -58,7 +61,7 @@ const SCHEMA: Integration = {

 class ArangoDBIntegration implements IntegrationBase {
  private config: ArangodbConfig
-  private client: any
+  private client

  constructor(config: ArangodbConfig) {
    const newConfig = {
@ -74,6 +77,19 @@ class ArangoDBIntegration implements IntegrationBase {
    this.client = new Database(newConfig)
  }

+  async testConnection() {
+    const response: ConnectionInfo = {
+      connected: false,
+    }
+    try {
+      await this.client.get()
+      response.connected = true
+    } catch (e: any) {
+      response.error = e.message as string
+    }
+    return response
+  }
+
  async read(query: { sql: any }) {
    try {
      const result = await this.client.query(query.sql)
--- a/packages/server/src/integrations/base/sqlTable.ts
+++ b/packages/server/src/integrations/base/sqlTable.ts
@ -206,4 +206,3 @@ class SqlTableQueryBuilder {
 }

 export default SqlTableQueryBuilder
-module.exports = SqlTableQueryBuilder
--- a/packages/server/src/integrations/couchdb.ts
+++ b/packages/server/src/integrations/couchdb.ts
@ -1,4 +1,6 @@
 import {
+  ConnectionInfo,
+  DatasourceFeature,
  DatasourceFieldType,
  Document,
  Integration,
@ -18,6 +20,7 @@ const SCHEMA: Integration = {
  type: "Non-relational",
  description:
    "Apache CouchDB is an open-source document-oriented NoSQL database, implemented in Erlang.",
+  features: [DatasourceFeature.CONNECTION_CHECKING],
  datasource: {
    url: {
      type: DatasourceFieldType.STRING,
@ -61,21 +64,32 @@ const SCHEMA: Integration = {
 }

 class CouchDBIntegration implements IntegrationBase {
-  private config: CouchDBConfig
-  private readonly client: any
+  private readonly client: dbCore.DatabaseImpl

  constructor(config: CouchDBConfig) {
-    this.config = config
    this.client = dbCore.DatabaseWithConnection(config.database, config.url)
  }

+  async testConnection() {
+    const response: ConnectionInfo = {
+      connected: false,
+    }
+    try {
+      const result = await this.query("exists", "validation error", {})
+      response.connected = result === true
+    } catch (e: any) {
+      response.error = e.message as string
+    }
+    return response
+  }
+
  async query(
    command: string,
    errorMsg: string,
    query: { json?: object; id?: string }
  ) {
    try {
-      return await this.client[command](query.id || query.json)
+      return await (this.client as any)[command](query.id || query.json)
    } catch (err) {
      console.error(errorMsg, err)
      throw err
--- a/packages/server/src/integrations/dynamodb.ts
+++ b/packages/server/src/integrations/dynamodb.ts
@ -3,10 +3,13 @@ import {
  DatasourceFieldType,
  QueryType,
  IntegrationBase,
+  DatasourceFeature,
+  ConnectionInfo,
 } from "@budibase/types"

 import AWS from "aws-sdk"
 import { AWS_REGION } from "../db/dynamoClient"
+import { DocumentClient } from "aws-sdk/clients/dynamodb"

 interface DynamoDBConfig {
  region: string
@ -22,6 +25,7 @@ const SCHEMA: Integration = {
    "Amazon DynamoDB is a key-value and document database that delivers single-digit millisecond performance at any scale.",
  friendlyName: "DynamoDB",
  type: "Non-relational",
+  features: [DatasourceFeature.CONNECTION_CHECKING],
  datasource: {
    region: {
      type: DatasourceFieldType.STRING,
@ -128,7 +132,7 @@ const SCHEMA: Integration = {

 class DynamoDBIntegration implements IntegrationBase {
  private config: DynamoDBConfig
-  private client: any
+  private client

  constructor(config: DynamoDBConfig) {
    this.config = config
@ -148,7 +152,23 @@ class DynamoDBIntegration implements IntegrationBase {
    this.client = new AWS.DynamoDB.DocumentClient(this.config)
  }

-  async create(query: { table: string; json: object }) {
+  async testConnection() {
+    const response: ConnectionInfo = {
+      connected: false,
+    }
+    try {
+      const scanRes = await new AWS.DynamoDB(this.config).listTables().promise()
+      response.connected = !!scanRes.$response
+    } catch (e: any) {
+      response.error = e.message as string
+    }
+    return response
+  }
+
+  async create(query: {
+    table: string
+    json: Omit<DocumentClient.PutItemInput, "TableName">
+  }) {
    const params = {
      TableName: query.table,
      ...query.json,
@ -189,7 +209,10 @@ class DynamoDBIntegration implements IntegrationBase {
    return new AWS.DynamoDB(this.config).describeTable(params).promise()
  }

-  async get(query: { table: string; json: object }) {
+  async get(query: {
+    table: string
+    json: Omit<DocumentClient.GetItemInput, "TableName">
+  }) {
    const params = {
      TableName: query.table,
      ...query.json,
@ -197,7 +220,10 @@ class DynamoDBIntegration implements IntegrationBase {
    return this.client.get(params).promise()
  }

-  async update(query: { table: string; json: object }) {
+  async update(query: {
+    table: string
+    json: Omit<DocumentClient.UpdateItemInput, "TableName">
+  }) {
    const params = {
      TableName: query.table,
      ...query.json,
@ -205,7 +231,10 @@ class DynamoDBIntegration implements IntegrationBase {
    return this.client.update(params).promise()
  }

-  async delete(query: { table: string; json: object }) {
+  async delete(query: {
+    table: string
+    json: Omit<DocumentClient.DeleteItemInput, "TableName">
+  }) {
    const params = {
      TableName: query.table,
      ...query.json,
--- a/packages/server/src/integrations/elasticsearch.ts
+++ b/packages/server/src/integrations/elasticsearch.ts
@ -3,6 +3,8 @@ import {
  DatasourceFieldType,
  QueryType,
  IntegrationBase,
+  DatasourceFeature,
+  ConnectionInfo,
 } from "@budibase/types"

 import { Client, ClientOptions } from "@elastic/elasticsearch"
@ -20,6 +22,7 @@ const SCHEMA: Integration = {
    "Elasticsearch is a search engine based on the Lucene library. It provides a distributed, multitenant-capable full-text search engine with an HTTP web interface and schema-free JSON documents.",
  friendlyName: "ElasticSearch",
  type: "Non-relational",
+  features: [DatasourceFeature.CONNECTION_CHECKING],
  datasource: {
    url: {
      type: DatasourceFieldType.STRING,
@ -95,7 +98,7 @@ const SCHEMA: Integration = {

 class ElasticSearchIntegration implements IntegrationBase {
  private config: ElasticsearchConfig
-  private client: any
+  private client

  constructor(config: ElasticsearchConfig) {
    this.config = config
@ -114,6 +117,18 @@ class ElasticSearchIntegration implements IntegrationBase {
    this.client = new Client(clientConfig)
  }

+  async testConnection(): Promise<ConnectionInfo> {
+    try {
+      await this.client.info()
+      return { connected: true }
+    } catch (e: any) {
+      return {
+        connected: false,
+        error: e.message as string,
+      }
+    }
+  }
+
  async create(query: { index: string; json: object }) {
    const { index, json } = query

--- a/packages/server/src/integrations/firebase.ts
+++ b/packages/server/src/integrations/firebase.ts
@ -3,6 +3,8 @@ import {
  Integration,
  QueryType,
  IntegrationBase,
+  DatasourceFeature,
+  ConnectionInfo,
 } from "@budibase/types"
 import { Firestore, WhereFilterOp } from "@google-cloud/firestore"

@ -18,6 +20,7 @@ const SCHEMA: Integration = {
  type: "Non-relational",
  description:
    "Cloud Firestore is a flexible, scalable database for mobile, web, and server development from Firebase and Google Cloud.",
+  features: [DatasourceFeature.CONNECTION_CHECKING],
  datasource: {
    email: {
      type: DatasourceFieldType.STRING,
@ -99,6 +102,18 @@ class FirebaseIntegration implements IntegrationBase {
    })
  }

+  async testConnection(): Promise<ConnectionInfo> {
+    try {
+      await this.client.listCollections()
+      return { connected: true }
+    } catch (e: any) {
+      return {
+        connected: false,
+        error: e.message as string,
+      }
+    }
+  }
+
  async create(query: { json: object; extra: { [key: string]: string } }) {
    try {
      const documentReference = this.client
--- a/packages/server/src/integrations/googlesheets.ts
+++ b/packages/server/src/integrations/googlesheets.ts
@ -1,4 +1,6 @@
 import {
+  ConnectionInfo,
+  DatasourceFeature,
  DatasourceFieldType,
  DatasourcePlus,
  FieldType,
@ -61,9 +63,13 @@ const SCHEMA: Integration = {
  relationships: false,
  docs: "https://developers.google.com/sheets/api/quickstart/nodejs",
  description:
-    "Create and collaborate on online spreadsheets in real-time and from any device. ",
+    "Create and collaborate on online spreadsheets in real-time and from any device.",
  friendlyName: "Google Sheets",
  type: "Spreadsheet",
+  features: [
+    DatasourceFeature.CONNECTION_CHECKING,
+    DatasourceFeature.FETCH_TABLE_NAMES,
+  ],
  datasource: {
    spreadsheetId: {
      display: "Google Sheet URL",
@ -139,6 +145,18 @@ class GoogleSheetsIntegration implements DatasourcePlus {
    this.client = new GoogleSpreadsheet(spreadsheetId)
  }

+  async testConnection(): Promise<ConnectionInfo> {
+    try {
+      await this.connect()
+      return { connected: true }
+    } catch (e: any) {
+      return {
+        connected: false,
+        error: e.message as string,
+      }
+    }
+  }
+
  getBindingIdentifier() {
    return ""
  }
@ -224,6 +242,12 @@ class GoogleSheetsIntegration implements DatasourcePlus {
    }
  }

+  async getTableNames(): Promise<string[]> {
+    await this.connect()
+    const sheets = this.client.sheetsByIndex
+    return sheets.map(s => s.title)
+  }
+
  getTableSchema(title: string, headerValues: string[], id?: string) {
    // base table
    const table: Table = {
--- a/packages/server/src/integrations/index.ts
+++ b/packages/server/src/integrations/index.ts
@ -20,7 +20,7 @@ import env from "../environment"
 import { cloneDeep } from "lodash"
 import sdk from "../sdk"

-const DEFINITIONS: { [key: string]: Integration } = {
+const DEFINITIONS: Record<SourceName, Integration | undefined> = {
  [SourceName.POSTGRES]: postgres.schema,
  [SourceName.DYNAMODB]: dynamodb.schema,
  [SourceName.MONGODB]: mongodb.schema,
@ -36,9 +36,10 @@ const DEFINITIONS: { [key: string]: Integration } = {
  [SourceName.GOOGLE_SHEETS]: googlesheets.schema,
  [SourceName.REDIS]: redis.schema,
  [SourceName.SNOWFLAKE]: snowflake.schema,
+  [SourceName.ORACLE]: undefined,
 }

-const INTEGRATIONS: { [key: string]: any } = {
+const INTEGRATIONS: Record<SourceName, any> = {
  [SourceName.POSTGRES]: postgres.integration,
  [SourceName.DYNAMODB]: dynamodb.integration,
  [SourceName.MONGODB]: mongodb.integration,
@ -55,6 +56,7 @@ const INTEGRATIONS: { [key: string]: any } = {
  [SourceName.REDIS]: redis.integration,
  [SourceName.FIRESTORE]: firebase.integration,
  [SourceName.SNOWFLAKE]: snowflake.integration,
+  [SourceName.ORACLE]: undefined,
 }

 // optionally add oracle integration if the oracle binary can be installed
@ -67,10 +69,13 @@ if (
  INTEGRATIONS[SourceName.ORACLE] = oracle.integration
 }

-export async function getDefinition(source: SourceName): Promise<Integration> {
+export async function getDefinition(
+  source: SourceName
+): Promise<Integration | undefined> {
  // check if its integrated, faster
-  if (DEFINITIONS[source]) {
-    return DEFINITIONS[source]
+  const definition = DEFINITIONS[source]
+  if (definition) {
+    return definition
  }
  const allDefinitions = await getDefinitions()
  return allDefinitions[source]
@ -98,7 +103,7 @@ export async function getDefinitions() {
  }
 }

-export async function getIntegration(integration: string) {
+export async function getIntegration(integration: SourceName) {
  if (INTEGRATIONS[integration]) {
    return INTEGRATIONS[integration]
  }
@ -107,7 +112,7 @@ export async function getIntegration(integration: string) {
    for (let plugin of plugins) {
      if (plugin.name === integration) {
        // need to use commonJS require due to its dynamic runtime nature
-        const retrieved: any = await getDatasourcePlugin(plugin)
+        const retrieved = await getDatasourcePlugin(plugin)
        if (retrieved.integration) {
          return retrieved.integration
        } else {
--- a/packages/server/src/integrations/microsoftSqlServer.ts
+++ b/packages/server/src/integrations/microsoftSqlServer.ts
@ -8,6 +8,8 @@ import {
  QueryType,
  SqlQuery,
  DatasourcePlus,
+  DatasourceFeature,
+  ConnectionInfo,
 } from "@budibase/types"
 import {
  getSqlQuery,
@ -18,7 +20,6 @@ import {
 } from "./utils"
 import Sql from "./base/sql"
 import { MSSQLTablesResponse, MSSQLColumn } from "./base/types"
-
 const sqlServer = require("mssql")
 const DEFAULT_SCHEMA = "dbo"

@ -39,6 +40,10 @@ const SCHEMA: Integration = {
    "Microsoft SQL Server is a relational database management system developed by Microsoft. ",
  friendlyName: "MS SQL Server",
  type: "Relational",
+  features: [
+    DatasourceFeature.CONNECTION_CHECKING,
+    DatasourceFeature.FETCH_TABLE_NAMES,
+  ],
  datasource: {
    user: {
      type: DatasourceFieldType.STRING,
@ -121,6 +126,19 @@ class SqlServerIntegration extends Sql implements DatasourcePlus {
    }
  }

+  async testConnection() {
+    const response: ConnectionInfo = {
+      connected: false,
+    }
+    try {
+      await this.connect()
+      response.connected = true
+    } catch (e: any) {
+      response.error = e.message as string
+    }
+    return response
+  }
+
  getBindingIdentifier(): string {
    return `@p${this.index++}`
  }
@ -268,6 +286,20 @@ class SqlServerIntegration extends Sql implements DatasourcePlus {
    this.schemaErrors = final.errors
  }

+  async queryTableNames() {
+    let tableInfo: MSSQLTablesResponse[] = await this.runSQL(this.TABLES_SQL)
+    const schema = this.config.schema || DEFAULT_SCHEMA
+    return tableInfo
+      .filter((record: any) => record.TABLE_SCHEMA === schema)
+      .map((record: any) => record.TABLE_NAME)
+      .filter((name: string) => this.MASTER_TABLES.indexOf(name) === -1)
+  }
+
+  async getTableNames() {
+    await this.connect()
+    return this.queryTableNames()
+  }
+
  async read(query: SqlQuery | string) {
    await this.connect()
    const response = await this.internalQuery(getSqlQuery(query))
--- a/packages/server/src/integrations/mongodb.ts
+++ b/packages/server/src/integrations/mongodb.ts
@ -3,6 +3,8 @@ import {
  DatasourceFieldType,
  QueryType,
  IntegrationBase,
+  DatasourceFeature,
+  ConnectionInfo,
 } from "@budibase/types"
 import {
  MongoClient,
@ -38,6 +40,7 @@ const getSchema = () => {
    type: "Non-relational",
    description:
      "MongoDB is a general purpose, document-based, distributed database built for modern application developers and for the cloud era.",
+    features: [DatasourceFeature.CONNECTION_CHECKING],
    datasource: {
      connectionString: {
        type: DatasourceFieldType.STRING,
@ -358,6 +361,19 @@ class MongoIntegration implements IntegrationBase {
    this.client = new MongoClient(config.connectionString, options)
  }

+  async testConnection() {
+    const response: ConnectionInfo = {
+      connected: false,
+    }
+    try {
+      await this.connect()
+      response.connected = true
+    } catch (e: any) {
+      response.error = e.message as string
+    }
+    return response
+  }
+
  async connect() {
    return this.client.connect()
  }
--- a/packages/server/src/integrations/mysql.ts
+++ b/packages/server/src/integrations/mysql.ts
@ -7,6 +7,8 @@ import {
  Table,
  TableSchema,
  DatasourcePlus,
+  DatasourceFeature,
+  ConnectionInfo,
 } from "@budibase/types"
 import {
  getSqlQuery,
@ -20,18 +22,11 @@ import { NUMBER_REGEX } from "../utilities"
 import Sql from "./base/sql"
 import { MySQLColumn } from "./base/types"

-const mysql = require("mysql2/promise")
+import mysql from "mysql2/promise"

-interface MySQLConfig {
-  host: string
-  port: number
-  user: string
-  password: string
+interface MySQLConfig extends mysql.ConnectionOptions {
  database: string
-  ssl?: { [key: string]: any }
  rejectUnauthorized: boolean
-  typeCast: Function
-  multipleStatements: boolean
 }

 const SCHEMA: Integration = {
@ -41,6 +36,10 @@ const SCHEMA: Integration = {
  type: "Relational",
  description:
    "MySQL Database Service is a fully managed database service to deploy cloud-native applications. ",
+  features: [
+    DatasourceFeature.CONNECTION_CHECKING,
+    DatasourceFeature.FETCH_TABLE_NAMES,
+  ],
  datasource: {
    host: {
      type: DatasourceFieldType.STRING,
@ -92,8 +91,6 @@ const SCHEMA: Integration = {
  },
 }

-const TimezoneAwareDateTypes = ["timestamp"]
-
 function bindingTypeCoerce(bindings: any[]) {
  for (let i = 0; i < bindings.length; i++) {
    const binding = bindings[i]
@ -120,7 +117,7 @@ function bindingTypeCoerce(bindings: any[]) {

 class MySQLIntegration extends Sql implements DatasourcePlus {
  private config: MySQLConfig
-  private client: any
+  private client?: mysql.Connection
  public tables: Record<string, Table> = {}
  public schemaErrors: Record<string, string> = {}

@ -134,7 +131,8 @@ class MySQLIntegration extends Sql implements DatasourcePlus {
    if (
      config.rejectUnauthorized != null &&
      !config.rejectUnauthorized &&
-      config.ssl
+      config.ssl &&
+      typeof config.ssl !== "string"
    ) {
      config.ssl.rejectUnauthorized = config.rejectUnauthorized
    }
@ -160,6 +158,22 @@ class MySQLIntegration extends Sql implements DatasourcePlus {
    }
  }

+  async testConnection() {
+    const response: ConnectionInfo = {
+      connected: false,
+    }
+    try {
+      const [result] = await this.internalQuery(
+        { sql: "SELECT 1+1 AS checkRes" },
+        { connect: true }
+      )
+      response.connected = result?.checkRes == 2
+    } catch (e: any) {
+      response.error = e.message as string
+    }
+    return response
+  }
+
  getBindingIdentifier(): string {
    return "?"
  }
@ -173,7 +187,7 @@ class MySQLIntegration extends Sql implements DatasourcePlus {
  }

  async disconnect() {
-    await this.client.end()
+    await this.client!.end()
  }

  async internalQuery(
@ -192,10 +206,10 @@ class MySQLIntegration extends Sql implements DatasourcePlus {
        ? baseBindings
        : bindingTypeCoerce(baseBindings)
      // Node MySQL is callback based, so we must wrap our call in a promise
-      const response = await this.client.query(query.sql, bindings)
+      const response = await this.client!.query(query.sql, bindings)
      return response[0]
    } finally {
-      if (opts?.connect) {
+      if (opts?.connect && this.client) {
        await this.disconnect()
      }
    }
@ -203,20 +217,11 @@ class MySQLIntegration extends Sql implements DatasourcePlus {

  async buildSchema(datasourceId: string, entities: Record<string, Table>) {
    const tables: { [key: string]: Table } = {}
-    const database = this.config.database
    await this.connect()

    try {
      // get the tables first
-      const tablesResp: Record<string, string>[] = await this.internalQuery(
-        { sql: "SHOW TABLES;" },
-        { connect: false }
-      )
-      const tableNames: string[] = tablesResp.map(
-        (obj: any) =>
-          obj[`Tables_in_${database}`] ||
-          obj[`Tables_in_${database.toLowerCase()}`]
-      )
+      const tableNames = await this.queryTableNames()
      for (let tableName of tableNames) {
        const primaryKeys = []
        const schema: TableSchema = {}
@ -263,6 +268,28 @@ class MySQLIntegration extends Sql implements DatasourcePlus {
    this.schemaErrors = final.errors
  }

+  async queryTableNames() {
+    const database = this.config.database
+    const tablesResp: Record<string, string>[] = await this.internalQuery(
+      { sql: "SHOW TABLES;" },
+      { connect: false }
+    )
+    return tablesResp.map(
+      (obj: any) =>
+        obj[`Tables_in_${database}`] ||
+        obj[`Tables_in_${database.toLowerCase()}`]
+    )
+  }
+
+  async getTableNames() {
+    await this.connect()
+    try {
+      return this.queryTableNames()
+    } finally {
+      await this.disconnect()
+    }
+  }
+
  async create(query: SqlQuery | string) {
    const results = await this.internalQuery(getSqlQuery(query))
    return results.length ? results : [{ created: true }]
--- a/packages/server/src/integrations/oracle.ts
+++ b/packages/server/src/integrations/oracle.ts
@ -7,6 +7,8 @@ import {
  SqlQuery,
  Table,
  DatasourcePlus,
+  DatasourceFeature,
+  ConnectionInfo,
 } from "@budibase/types"
 import {
  buildExternalTableId,
@ -24,12 +26,7 @@ import {
  ExecuteOptions,
  Result,
 } from "oracledb"
-import {
-  OracleTable,
-  OracleColumn,
-  OracleColumnsResponse,
-  OracleConstraint,
-} from "./base/types"
+import { OracleTable, OracleColumn, OracleColumnsResponse } from "./base/types"
 let oracledb: any
 try {
  oracledb = require("oracledb")
@ -53,6 +50,10 @@ const SCHEMA: Integration = {
  type: "Relational",
  description:
    "Oracle Database is an object-relational database management system developed by Oracle Corporation",
+  features: [
+    DatasourceFeature.CONNECTION_CHECKING,
+    DatasourceFeature.FETCH_TABLE_NAMES,
+  ],
  datasource: {
    host: {
      type: DatasourceFieldType.STRING,
@ -325,6 +326,37 @@ class OracleIntegration extends Sql implements DatasourcePlus {
    this.schemaErrors = final.errors
  }

+  async getTableNames() {
+    const columnsResponse = await this.internalQuery<OracleColumnsResponse>({
+      sql: this.COLUMNS_SQL,
+    })
+    return (columnsResponse.rows || []).map(row => row.TABLE_NAME)
+  }
+
+  async testConnection() {
+    const response: ConnectionInfo = {
+      connected: false,
+    }
+    let connection
+    try {
+      connection = await this.getConnection()
+      response.connected = true
+    } catch (err: any) {
+      response.connected = false
+      response.error = err.message
+    } finally {
+      if (connection) {
+        try {
+          await connection.close()
+        } catch (err: any) {
+          response.connected = false
+          response.error = err.message
+        }
+      }
+    }
+    return response
+  }
+
  private async internalQuery<T>(query: SqlQuery): Promise<Result<T>> {
    let connection
    try {
--- a/packages/server/src/integrations/postgres.ts
+++ b/packages/server/src/integrations/postgres.ts
@ -6,6 +6,8 @@ import {
  SqlQuery,
  Table,
  DatasourcePlus,
+  DatasourceFeature,
+  ConnectionInfo,
 } from "@budibase/types"
 import {
  getSqlQuery,
@ -18,7 +20,7 @@ import Sql from "./base/sql"
 import { PostgresColumn } from "./base/types"
 import { escapeDangerousCharacters } from "../utilities"

-const { Client, types } = require("pg")
+import { Client, types } from "pg"

 // Return "date" and "timestamp" types as plain strings.
 // This lets us reference the original stored timezone.
@ -50,6 +52,10 @@ const SCHEMA: Integration = {
  type: "Relational",
  description:
    "PostgreSQL, also known as Postgres, is a free and open-source relational database management system emphasizing extensibility and SQL compliance.",
+  features: [
+    DatasourceFeature.CONNECTION_CHECKING,
+    DatasourceFeature.FETCH_TABLE_NAMES,
+  ],
  datasource: {
    host: {
      type: DatasourceFieldType.STRING,
@ -114,7 +120,7 @@ const SCHEMA: Integration = {
 }

 class PostgresIntegration extends Sql implements DatasourcePlus {
-  private readonly client: any
+  private readonly client: Client
  private readonly config: PostgresConfig
  private index: number = 1
  private open: boolean
@ -123,14 +129,15 @@ class PostgresIntegration extends Sql implements DatasourcePlus {

  COLUMNS_SQL!: string

-  PRIMARY_KEYS_SQL = `
-  select tc.table_schema, tc.table_name, kc.column_name as primary_key 
-  from information_schema.table_constraints tc
-  join 
-    information_schema.key_column_usage kc on kc.table_name = tc.table_name 
-    and kc.table_schema = tc.table_schema 
-    and kc.constraint_name = tc.constraint_name
-  where tc.constraint_type = 'PRIMARY KEY';
+  PRIMARY_KEYS_SQL = () => `
+  SELECT pg_namespace.nspname table_schema
+     , pg_class.relname table_name
+     , pg_attribute.attname primary_key
+  FROM pg_class
+  JOIN pg_index ON pg_class.oid = pg_index.indrelid AND pg_index.indisprimary
+  JOIN pg_attribute ON pg_attribute.attrelid = pg_class.oid AND pg_attribute.attnum = ANY(pg_index.indkey)
+  JOIN pg_namespace ON pg_namespace.oid = pg_class.relnamespace
+  WHERE pg_namespace.nspname = '${this.config.schema}';
  `

  constructor(config: PostgresConfig) {
@ -150,6 +157,21 @@ class PostgresIntegration extends Sql implements DatasourcePlus {
    this.open = false
  }

+  async testConnection() {
+    const response: ConnectionInfo = {
+      connected: false,
+    }
+    try {
+      await this.openConnection()
+      response.connected = true
+    } catch (e: any) {
+      response.error = e.message as string
+    } finally {
+      await this.closeConnection()
+    }
+    return response
+  }
+
  getBindingIdentifier(): string {
    return `$${this.index++}`
  }
@ -163,7 +185,7 @@ class PostgresIntegration extends Sql implements DatasourcePlus {
    if (!this.config.schema) {
      this.config.schema = "public"
    }
-    this.client.query(`SET search_path TO ${this.config.schema}`)
+    await this.client.query(`SET search_path TO ${this.config.schema}`)
    this.COLUMNS_SQL = `select * from information_schema.columns where table_schema = '${this.config.schema}'`
    this.open = true
  }
@ -221,7 +243,9 @@ class PostgresIntegration extends Sql implements DatasourcePlus {
    let tableKeys: { [key: string]: string[] } = {}
    await this.openConnection()
    try {
-      const primaryKeysResponse = await this.client.query(this.PRIMARY_KEYS_SQL)
+      const primaryKeysResponse = await this.client.query(
+        this.PRIMARY_KEYS_SQL()
+      )
      for (let table of primaryKeysResponse.rows) {
        const tableName = table.table_name
        if (!tableKeys[tableName]) {
@ -293,6 +317,17 @@ class PostgresIntegration extends Sql implements DatasourcePlus {
    }
  }

+  async getTableNames() {
+    try {
+      await this.openConnection()
+      const columnsResponse: { rows: PostgresColumn[] } =
+        await this.client.query(this.COLUMNS_SQL)
+      return columnsResponse.rows.map(row => row.table_name)
+    } finally {
+      await this.closeConnection()
+    }
+  }
+
  async create(query: SqlQuery | string) {
    const response = await this.internalQuery(getSqlQuery(query))
    return response.rows.length ? response.rows : [{ created: true }]
--- a/packages/server/src/integrations/redis.ts
+++ b/packages/server/src/integrations/redis.ts
@ -1,4 +1,10 @@
-import { DatasourceFieldType, Integration, QueryType } from "@budibase/types"
+import {
+  ConnectionInfo,
+  DatasourceFeature,
+  DatasourceFieldType,
+  Integration,
+  QueryType,
+} from "@budibase/types"
 import Redis from "ioredis"

 interface RedisConfig {
@ -11,9 +17,11 @@ interface RedisConfig {

 const SCHEMA: Integration = {
  docs: "https://redis.io/docs/",
-  description: "",
+  description:
+    "Redis is a caching tool, providing powerful key-value store capabilities.",
  friendlyName: "Redis",
  type: "Non-relational",
+  features: [DatasourceFeature.CONNECTION_CHECKING],
  datasource: {
    host: {
      type: "string",
@ -86,7 +94,7 @@ const SCHEMA: Integration = {

 class RedisIntegration {
  private readonly config: RedisConfig
-  private client: any
+  private client

  constructor(config: RedisConfig) {
    this.config = config
@ -99,6 +107,21 @@ class RedisIntegration {
    })
  }

+  async testConnection() {
+    const response: ConnectionInfo = {
+      connected: false,
+    }
+    try {
+      await this.client.ping()
+      response.connected = true
+    } catch (e: any) {
+      response.error = e.message as string
+    } finally {
+      await this.disconnect()
+    }
+    return response
+  }
+
  async disconnect() {
    return this.client.quit()
  }
--- a/packages/server/src/integrations/s3.ts
+++ b/packages/server/src/integrations/s3.ts
@ -3,10 +3,12 @@ import {
  QueryType,
  IntegrationBase,
  DatasourceFieldType,
+  DatasourceFeature,
+  ConnectionInfo,
 } from "@budibase/types"

-const AWS = require("aws-sdk")
-const csv = require("csvtojson")
+import AWS from "aws-sdk"
+import csv from "csvtojson"

 interface S3Config {
  region: string
@ -22,6 +24,7 @@ const SCHEMA: Integration = {
    "Amazon Simple Storage Service (Amazon S3) is an object storage service that offers industry-leading scalability, data availability, security, and performance.",
  friendlyName: "Amazon S3",
  type: "Object store",
+  features: [DatasourceFeature.CONNECTION_CHECKING],
  datasource: {
    region: {
      type: "string",
@ -152,7 +155,7 @@ const SCHEMA: Integration = {

 class S3Integration implements IntegrationBase {
  private readonly config: S3Config
-  private client: any
+  private client

  constructor(config: S3Config) {
    this.config = config
@ -165,6 +168,19 @@ class S3Integration implements IntegrationBase {
    this.client = new AWS.S3(this.config)
  }

+  async testConnection() {
+    const response: ConnectionInfo = {
+      connected: false,
+    }
+    try {
+      await this.client.listBuckets().promise()
+      response.connected = true
+    } catch (e: any) {
+      response.error = e.message as string
+    }
+    return response
+  }
+
  async create(query: {
    bucket: string
    location: string
--- a/packages/server/src/integrations/snowflake.ts
+++ b/packages/server/src/integrations/snowflake.ts
@ -1,4 +1,10 @@
-import { Integration, QueryType, SqlQuery } from "@budibase/types"
+import {
+  ConnectionInfo,
+  DatasourceFeature,
+  Integration,
+  QueryType,
+  SqlQuery,
+} from "@budibase/types"
 import { Snowflake } from "snowflake-promise"

 interface SnowflakeConfig {
@ -16,6 +22,7 @@ const SCHEMA: Integration = {
    "Snowflake is a solution for data warehousing, data lakes, data engineering, data science, data application development, and securely sharing and consuming shared data.",
  friendlyName: "Snowflake",
  type: "Relational",
+  features: [DatasourceFeature.CONNECTION_CHECKING],
  datasource: {
    account: {
      type: "string",
@ -65,6 +72,18 @@ class SnowflakeIntegration {
    this.client = new Snowflake(config)
  }

+  async testConnection(): Promise<ConnectionInfo> {
+    try {
+      await this.client.connect()
+      return { connected: true }
+    } catch (e: any) {
+      return {
+        connected: false,
+        error: e.message as string,
+      }
+    }
+  }
+
  async internalQuery(query: SqlQuery) {
    await this.client.connect()
    try {
--- a/packages/server/src/integrations/tests/googlesheets.spec.ts
+++ b/packages/server/src/integrations/tests/googlesheets.spec.ts
@ -17,14 +17,15 @@ jest.mock("google-spreadsheet")
 const { GoogleSpreadsheet } = require("google-spreadsheet")

 const sheetsByTitle: { [title: string]: GoogleSpreadsheetWorksheet } = {}
+const sheetsByIndex: GoogleSpreadsheetWorksheet[] = []
+const mockGoogleIntegration = {
+  useOAuth2Client: jest.fn(),
+  loadInfo: jest.fn(),
+  sheetsByTitle,
+  sheetsByIndex,
+}

-GoogleSpreadsheet.mockImplementation(() => {
-  return {
-    useOAuth2Client: jest.fn(),
-    loadInfo: jest.fn(),
-    sheetsByTitle,
-  }
-})
+GoogleSpreadsheet.mockImplementation(() => mockGoogleIntegration)

 import { structures } from "@budibase/backend-core/tests"
 import TestConfiguration from "../../tests/utilities/TestConfiguration"
@ -53,6 +54,8 @@ describe("Google Sheets Integration", () => {
      },
    })
    await config.init()
+
+    jest.clearAllMocks()
  })

  function createBasicTable(name: string, columns: string[]): Table {
@ -88,7 +91,7 @@ describe("Google Sheets Integration", () => {
  }

  describe("update table", () => {
-    test("adding a new field will be adding a new header row", async () => {
+    it("adding a new field will be adding a new header row", async () => {
      await config.doInContext(structures.uuid(), async () => {
        const tableColumns = ["name", "description", "new field"]
        const table = createBasicTable(structures.uuid(), tableColumns)
@ -103,7 +106,7 @@ describe("Google Sheets Integration", () => {
      })
    })

-    test("removing an existing field will remove the header from the google sheet", async () => {
+    it("removing an existing field will remove the header from the google sheet", async () => {
      const sheet = await config.doInContext(structures.uuid(), async () => {
        const tableColumns = ["name"]
        const table = createBasicTable(structures.uuid(), tableColumns)
@ -123,4 +126,33 @@ describe("Google Sheets Integration", () => {
      expect((sheet.setHeaderRow as any).mock.calls[0][0]).toHaveLength(1)
    })
  })
+
+  describe("getTableNames", () => {
+    it("can fetch table names", async () => {
+      await config.doInContext(structures.uuid(), async () => {
+        const sheetNames: string[] = []
+        for (let i = 0; i < 5; i++) {
+          const sheet = createSheet({ headerValues: [] })
+          sheetsByIndex.push(sheet)
+          sheetNames.push(sheet.title)
+        }
+
+        const res = await integration.getTableNames()
+
+        expect(mockGoogleIntegration.loadInfo).toBeCalledTimes(1)
+        expect(res).toEqual(sheetNames)
+      })
+    })
+  })
+
+  describe("testConnection", () => {
+    it("can test successful connections", async () => {
+      await config.doInContext(structures.uuid(), async () => {
+        const res = await integration.testConnection()
+
+        expect(mockGoogleIntegration.loadInfo).toBeCalledTimes(1)
+        expect(res).toEqual({ connected: true })
+      })
+    })
+  })
 })
--- a/packages/server/src/migrations/functions/backfill/app/queries.ts
+++ b/packages/server/src/migrations/functions/backfill/app/queries.ts
@ -33,7 +33,7 @@ export const backfill = async (appDb: any, timestamp: string | number) => {
        datasource = {
          type: "unknown",
          _id: query.datasourceId,
-          source: SourceName.UNKNOWN,
+          source: "unknown" as SourceName,
        }
      } else {
        throw e
--- a/packages/server/src/sdk/app/datasources/datasources.ts
+++ b/packages/server/src/sdk/app/datasources/datasources.ts
@ -13,6 +13,7 @@ import {
 import { cloneDeep } from "lodash/fp"
 import { getEnvironmentVariables } from "../../utils"
 import { getDefinitions, getDefinition } from "../../../integrations"
+import _ from "lodash"

 const ENV_VAR_PREFIX = "env."

@ -41,7 +42,7 @@ async function enrichDatasourceWithValues(datasource: Datasource) {
    { onlyFound: true }
  ) as Datasource
  const definition = await getDefinition(processed.source)
-  processed.config = checkDatasourceTypes(definition, processed.config)
+  processed.config = checkDatasourceTypes(definition!, processed.config)
  return {
    datasource: processed,
    envVars: env as Record<string, string>,
@ -147,6 +148,11 @@ export function mergeConfigs(update: Datasource, old: Datasource) {
      }
    }
  }
+
+  if (old.config?.auth) {
+    update.config = _.merge(old.config, update.config)
+  }
+
  // update back to actual passwords for everything else
  for (let [key, value] of Object.entries(update.config)) {
    if (value !== PASSWORD_REPLACEMENT) {
--- a/packages/server/src/threads/automation.ts
+++ b/packages/server/src/threads/automation.ts
@ -13,13 +13,18 @@ import { generateAutomationMetadataID, isProdAppID } from "../db/utils"
 import { definitions as triggerDefs } from "../automations/triggerInfo"
 import { AutomationErrors, MAX_AUTOMATION_RECURRING_ERRORS } from "../constants"
 import { storeLog } from "../automations/logging"
-import { Automation, AutomationStep, AutomationStatus } from "@budibase/types"
+import {
+  Automation,
+  AutomationStep,
+  AutomationStatus,
+  AutomationMetadata,
+  AutomationJob,
+} from "@budibase/types"
 import {
  LoopStep,
  LoopInput,
  TriggerOutput,
  AutomationContext,
-  AutomationMetadata,
 } from "../definitions/automations"
 import { WorkerCallback } from "./definitions"
 import { context, logging } from "@budibase/backend-core"
@ -60,11 +65,11 @@ class Orchestrator {
  _job: Job
  executionOutput: AutomationContext

-  constructor(job: Job) {
-    let automation = job.data.automation,
-      triggerOutput = job.data.event
+  constructor(job: AutomationJob) {
+    let automation = job.data.automation
+    let triggerOutput = job.data.event
    const metadata = triggerOutput.metadata
-    this._chainCount = metadata ? metadata.automationChainCount : 0
+    this._chainCount = metadata ? metadata.automationChainCount! : 0
    this._appId = triggerOutput.appId as string
    this._job = job
    const triggerStepId = automation.definition.trigger.stepId
--- a/packages/server/src/threads/definitions.ts
+++ b/packages/server/src/threads/definitions.ts
@ -1,5 +1,3 @@
-import { EnvironmentVariablesDecrypted } from "@budibase/types"
-
 export type WorkerCallback = (error: any, response?: any) => void

 export interface QueryEvent {
--- a/packages/server/src/threads/index.ts
+++ b/packages/server/src/threads/index.ts
@ -1,5 +1,7 @@
 import workerFarm from "worker-farm"
 import env from "../environment"
+import { AutomationJob } from "@budibase/types"
+import { QueryEvent } from "./definitions"

 export const ThreadType = {
  QUERY: "query",
@ -64,11 +66,11 @@ export class Thread {
    )
  }

-  run(data: any) {
+  run(job: AutomationJob | QueryEvent) {
    const timeout = this.timeoutMs
    return new Promise((resolve, reject) => {
      function fire(worker: any) {
-        worker.execute(data, (err: any, response: any) => {
+        worker.execute(job, (err: any, response: any) => {
          if (err && err.type === "TimeoutError") {
            reject(
              new Error(`Query response time exceeded ${timeout}ms timeout.`)
--- a/packages/server/src/utilities/fileSystem/app.ts
+++ b/packages/server/src/utilities/fileSystem/app.ts
@ -35,7 +35,7 @@ export const getComponentLibraryManifest = async (library: string) => {
  const filename = "manifest.json"

  if (env.isDev() || env.isTest()) {
-    const path = join(NODE_MODULES_PATH, "@budibase", "client", filename)
+    const path = join(TOP_LEVEL_PATH, "packages/client", filename)
    // always load from new so that updates are refreshed
    delete require.cache[require.resolve(path)]
    return require(path)
--- a/packages/server/src/utilities/fileSystem/clientLibrary.ts
+++ b/packages/server/src/utilities/fileSystem/clientLibrary.ts
@ -1,4 +1,4 @@
-import { join } from "path"
+import path, { join } from "path"
 import { ObjectStoreBuckets } from "../../constants"
 import fs from "fs"
 import { objectStore } from "@budibase/backend-core"
@ -6,6 +6,10 @@ import { resolve } from "../centralPath"
 import env from "../../environment"
 import { TOP_LEVEL_PATH } from "./filesystem"

+export function devClientLibPath() {
+  return require.resolve("@budibase/client")
+}
+
 /**
 * Client library paths in the object store:
 * Previously, the entire client library package was downloaded from NPM
@ -89,9 +93,10 @@ export async function updateClientLibrary(appId: string) {
  let manifest, client

  if (env.isDev()) {
+    const clientPath = devClientLibPath()
    // Load the symlinked version in dev which is always the newest
-    manifest = require.resolve("@budibase/client/manifest.json")
-    client = require.resolve("@budibase/client")
+    manifest = join(path.dirname(path.dirname(clientPath)), "manifest.json")
+    client = clientPath
  } else {
    // Load the bundled version in prod
    manifest = resolve(TOP_LEVEL_PATH, "client", "manifest.json")
--- a/packages/server/src/utilities/fileSystem/filesystem.ts
+++ b/packages/server/src/utilities/fileSystem/filesystem.ts
@ -4,9 +4,11 @@ import { budibaseTempDir } from "../budibaseDir"
 import { join } from "path"
 import env from "../../environment"
 import tar from "tar"
+import environment from "../../environment"
 const uuid = require("uuid/v4")

-export const TOP_LEVEL_PATH = join(__dirname, "..", "..", "..")
+export const TOP_LEVEL_PATH =
+  environment.TOP_LEVEL_PATH || join(__dirname, "..", "..", "..")

 /**
 * Upon first startup of instance there may not be everything we need in tmp directory, set it up.
--- a/packages/server/tsconfig.build.json
+++ b/packages/server/tsconfig.build.json
@ -10,7 +10,15 @@
    "incremental": true,
    "types": ["node", "jest"],
    "outDir": "dist/src",
-    "skipLibCheck": true
+    "skipLibCheck": true,
+    "baseUrl": ".",
+    "paths": {
+      "@budibase/types": ["../types/src"],
+      "@budibase/backend-core": ["../backend-core/src"],
+      "@budibase/backend-core/*": ["../backend-core/*"],
+      "@budibase/shared-core": ["../shared-core/src"],
+      "@budibase/pro": ["../pro/packages/pro/src"]
+    }
  },
  "include": ["src/**/*"],
  "exclude": [
--- a/packages/server/tsconfig.json
+++ b/packages/server/tsconfig.json
@ -18,12 +18,6 @@
    "require": ["tsconfig-paths/register"],
    "swc": true
  },
-  "references": [
-    { "path": "../types" },
-    { "path": "../backend-core" },
-    { "path": "../shared-core" },
-    { "path": "../../../budibase-pro/packages/pro" }
-  ],
  "include": ["src/**/*", "specs"],
  "exclude": ["node_modules", "dist"]
 }
--- a/packages/shared-core/package.json
+++ b/packages/shared-core/package.json
@ -26,5 +26,19 @@
    "concurrently": "^7.6.0",
    "rimraf": "3.0.2",
    "typescript": "4.7.3"
+  },
+  "nx": {
+    "targets": {
+      "build": {
+        "dependsOn": [
+          {
+            "projects": [
+              "@budibase/types"
+            ],
+            "target": "build"
+          }
+        ]
+      }
+    }
  }
 }
--- a/packages/shared-core/src/filters.ts
+++ b/packages/shared-core/src/filters.ts
@ -328,8 +328,8 @@ export const runLuceneQuery = (docs: any[], query?: Query) => {
      return (
        docValue == null ||
        docValue === "" ||
-        docValue < testValue.low ||
-        docValue > testValue.high
+        +docValue < testValue.low ||
+        +docValue > testValue.high
      )
    }
  )
--- a/Show More
+++ b/Show More
 @ -1 +1 @@
 .10.0
 .10.0