Merge remote-tracking branch 'origin/develop' into feature/auto-select-dataprovider-source

.eslintignore

@@ -3,6 +3,8 @@ public
 dist
 packages/server/builder
 packages/server/coverage
+packages/worker/coverage
+packages/backend-core/coverage
 packages/server/client
 packages/builder/.routify
 packages/builder/cypress/support/queryLevelTransformerFunction.js

.github/workflows/budibase_ci.yml

@@ -58,7 +58,7 @@ jobs:
     - uses: codecov/codecov-action@v1
       with:
         token: ${{ secrets.CODECOV_TOKEN }} # not required for public repos
-        files: ./packages/server/coverage/clover.xml
+        files: ./packages/server/coverage/clover.xml,./packages/worker/coverage/clover.xml,./packages/backend-core/coverage/clover.xml
         name: codecov-umbrella
         verbose: true 
 

.prettierignore

@@ -4,6 +4,8 @@ dist
 packages/builder/src/components/design/AppPreview/CurrentItemPreview.svelte
 packages/server/builder
 packages/server/coverage
+packages/worker/coverage
+packages/backend-core/coverage
 packages/server/client
 packages/server/src/definitions/openapi.ts
 packages/builder/.routify

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "2.1.22-alpha.3",
+  "version": "2.1.22-alpha.4",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

package.json

@@ -18,7 +18,7 @@
     "rimraf": "^3.0.2",
     "rollup-plugin-replace": "^2.2.0",
     "svelte": "^3.38.2",
-    "typescript": "4.5.5"
+    "typescript": "4.7.3"
   },
   "scripts": {
     "setup": "node ./hosting/scripts/setup.js && yarn && yarn bootstrap && yarn build && yarn dev",

packages/backend-core/__mocks__/aws-sdk.ts

@@ -0,0 +1,15 @@
+const mockS3 = {
+  headBucket: jest.fn().mockReturnThis(),
+  deleteObject: jest.fn().mockReturnThis(),
+  deleteObjects: jest.fn().mockReturnThis(),
+  createBucket: jest.fn().mockReturnThis(),
+  listObjects: jest.fn().mockReturnThis(),
+  promise: jest.fn().mockReturnThis(),
+  catch: jest.fn(),
+}
+
+const AWS = {
+  S3: jest.fn(() => mockS3),
+}
+
+export default AWS

packages/backend-core/__mocks__/node-fetch.ts

@@ -0,0 +1 @@
+jest.mock("node-fetch", () => jest.fn())

packages/backend-core/jest.config.ts

@@ -0,0 +1,20 @@
+import { Config } from "@jest/types"
+
+const config: Config.InitialOptions = {
+  preset: "ts-jest",
+  testEnvironment: "node",
+  setupFiles: ["./tests/jestSetup.ts"],
+  collectCoverageFrom: ["src/**/*.{js,ts}"],
+  coverageReporters: ["lcov", "json", "clover"],
+}
+
+if (!process.env.CI) {
+  // use sources when not in CI
+  config.moduleNameMapper = {
+    "@budibase/types": "<rootDir>/../types/src",
+  }
+} else {
+  console.log("Running tests with compiled dependency sources")
+}
+
+export default config

packages/backend-core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/backend-core",
-  "version": "2.1.22-alpha.3",
+  "version": "2.1.22-alpha.4",
   "description": "Budibase backend core libraries used in server and worker",
   "main": "dist/src/index.js",
   "types": "dist/src/index.d.ts",
@@ -16,11 +16,11 @@
     "prepack": "cp package.json dist",
     "build": "tsc -p tsconfig.build.json",
     "build:dev": "yarn prebuild && tsc --build --watch --preserveWatchOutput",
-    "test": "jest",
+    "test": "jest --coverage",
     "test:watch": "jest --watchAll"
   },
   "dependencies": {
-    "@budibase/types": "2.1.22-alpha.3",
+    "@budibase/types": "2.1.22-alpha.4",
     "@shopify/jest-koa-mocks": "5.0.1",
     "@techpass/passport-openidconnect": "0.3.2",
     "aws-sdk": "2.1030.0",
@@ -52,16 +52,6 @@
     "uuid": "8.3.2",
     "zlib": "1.0.5"
   },
-  "jest": {
-    "preset": "ts-jest",
-    "testEnvironment": "node",
-    "moduleNameMapper": {
-      "@budibase/types": "<rootDir>/../types/src"
-    },
-    "setupFiles": [
-      "./scripts/jestSetup.ts"
-    ]
-  },
   "devDependencies": {
     "@types/chance": "1.1.3",
     "@types/ioredis": "4.28.0",
@@ -77,12 +67,14 @@
     "@types/uuid": "8.3.4",
     "chance": "1.1.3",
     "ioredis-mock": "5.8.0",
-    "jest": "27.5.1",
+    "jest": "28.1.1",
     "koa": "2.7.0",
     "nodemon": "2.0.16",
     "pouchdb-adapter-memory": "7.2.2",
     "timekeeper": "2.2.0",
-    "ts-jest": "27.1.5",
+    "ts-jest": "28.0.4",
+    "ts-node": "10.8.1",
+    "tsconfig-paths": "4.0.0",
     "typescript": "4.7.3"
   },
   "gitHead": "d1836a898cab3f8ab80ee6d8f42be1a9eed7dcdc"

packages/backend-core/scripts/jestSetup.ts

@@ -1,12 +0,0 @@
-import env from "../src/environment"
-import { mocks } from "../tests/utilities"
-
-// mock all dates to 2020-01-01T00:00:00.000Z
-// use tk.reset() to use real dates in individual tests
-import tk from "timekeeper"
-tk.freeze(mocks.date.MOCK_DATE)
-
-env._set("SELF_HOSTED", "1")
-env._set("NODE_ENV", "jest")
-env._set("JWT_SECRET", "test-jwtsecret")
-env._set("LOG_LEVEL", "silent")

packages/backend-core/src/cache/tests/writethrough.spec.js

@@ -1,4 +1,4 @@
-require("../../../tests/utilities/TestConfiguration")
+require("../../../tests")
 const { Writethrough } = require("../writethrough")
 const { dangerousGetDB } = require("../../db")
 const tk = require("timekeeper")

packages/backend-core/src/context/tests/index.spec.ts

@@ -1,4 +1,4 @@
-import "../../../tests/utilities/TestConfiguration"
+import "../../../tests"
 import * as context from ".."
 import { DEFAULT_TENANT_ID } from "../../constants"
 import env from "../../environment"

packages/backend-core/src/db/index.ts

@@ -33,7 +33,7 @@ const checkInitialised = () => {
   }
 }
 
-export async function init(opts?: PouchOptions) {
+export function init(opts?: PouchOptions) {
   Pouch = pouch.getPouch(opts)
   initialised = true
 }

packages/backend-core/src/db/pouch.ts

@@ -1,5 +1,6 @@
 import PouchDB from "pouchdb"
 import env from "../environment"
+import { PouchOptions } from "@budibase/types"
 
 export const getUrlInfo = (url = env.COUCH_DB_URL) => {
   let cleanUrl, username, password, host
@@ -82,7 +83,7 @@ export const getCouchInfo = () => {
  * This should be rarely used outside of the main application config.
  * Exposed for exceptional cases such as in-memory views.
  */
-export const getPouch = (opts: any = {}) => {
+export const getPouch = (opts: PouchOptions = {}) => {
   let { url, cookie } = getCouchInfo()
   let POUCH_DB_DEFAULTS = {
     prefix: url,

packages/backend-core/src/db/tests/index.spec.js

@@ -1,4 +1,4 @@
-require("../../../tests/utilities/TestConfiguration")
+require("../../../tests")
 const { dangerousGetDB } = require("../")
 
 describe("db", () => { 

packages/backend-core/src/db/tests/pouch.spec.js

@@ -1,4 +1,4 @@
-require("../../../tests/utilities/TestConfiguration")
+require("../../../tests")
 const getUrlInfo = require("../pouch").getUrlInfo
 
 describe("pouch", () => {

packages/backend-core/src/db/tests/utils.spec.js

@@ -1,4 +1,4 @@
-require("../../../tests/utilities/TestConfiguration");
+require("../../../tests");
 const {
   generateAppID,
   getDevelopmentAppID,

packages/backend-core/src/events/processors/posthog/tests/PosthogProcessor.spec.ts

@@ -1,4 +1,4 @@
-import "../../../../../tests/utilities/TestConfiguration"
+import "../../../../../tests"
 import PosthogProcessor from "../PosthogProcessor"
 import { Event, IdentityType, Hosting } from "@budibase/types"
 const tk = require("timekeeper")

packages/backend-core/src/middleware/passport/tests/oidc.spec.js

@@ -1,7 +1,6 @@
 // Mock data
-
+const mockFetch = require("node-fetch")
 const { data } = require("./utilities/mock-data")
-
 const issuer = "mockIssuer"
 const sub = "mockSub"
 const profile = {
@@ -39,8 +38,6 @@ describe("oidc", () => {
     const mockStrategy = require("@techpass/passport-openidconnect").Strategy
     
     // mock the request to retrieve the oidc configuration
-    jest.mock("node-fetch")
-    const mockFetch = require("node-fetch")
     mockFetch.mockReturnValue({
       ok: true,
       json: () => oidcConfigUrlResponse

packages/backend-core/src/middleware/passport/tests/third-party-common.spec.js

@@ -1,4 +1,4 @@
-require("../../../../tests/utilities/TestConfiguration")
+require("../../../../tests")
 const { authenticateThirdParty } = require("../third-party-common")
 const { data } = require("./utilities/mock-data")
 const { DEFAULT_TENANT_ID } = require("../../../constants")

packages/backend-core/src/migrations/tests/index.spec.js

@@ -1,4 +1,4 @@
-require("../../../tests/utilities/TestConfiguration")
+require("../../../tests")
 const { runMigrations, getMigrationsDoc } = require("../index")
 const { dangerousGetDB } = require("../../db")
 const {

packages/backend-core/src/queue/inMemoryQueue.ts

@@ -29,6 +29,7 @@ class InMemoryQueue {
   _messages: any[]
   _emitter: EventEmitter
   _runCount: number
+  _addCount: number
   /**
    * The constructor the queue, exactly the same as that of Bulls.
    * @param {string} name The name of the queue which is being configured.
@@ -41,6 +42,7 @@ class InMemoryQueue {
     this._messages = []
     this._emitter = new events.EventEmitter()
     this._runCount = 0
+    this._addCount = 0
   }
 
   /**
@@ -81,6 +83,7 @@ class InMemoryQueue {
       throw "Queue only supports carrying JSON."
     }
     this._messages.push(newJob(this._name, msg))
+    this._addCount++
     this._emitter.emit("message")
   }
 
@@ -128,12 +131,9 @@ class InMemoryQueue {
   }
 
   async waitForCompletion() {
-    const currentCount = this._runCount
-    let increased = false
     do {
       await timeout(50)
-      increased = this._runCount > currentCount
-    } while (!increased)
+    } while (this._addCount < this._runCount)
   }
 }
 

packages/backend-core/src/tests/utils.spec.js

@@ -1,5 +1,4 @@
-require("../../tests/utilities/TestConfiguration")
-const { structures } = require("../../tests/utilities")
+const { structures } = require("../../tests")
 const utils = require("../utils")
 const events = require("../events")
 const { doInTenant, DEFAULT_TENANT_ID }= require("../context")

packages/backend-core/tests/jestSetup.ts

@@ -0,0 +1,23 @@
+import env from "../src/environment"
+import { mocks } from "./utilities"
+
+// mock all dates to 2020-01-01T00:00:00.000Z
+// use tk.reset() to use real dates in individual tests
+import tk from "timekeeper"
+tk.freeze(mocks.date.MOCK_DATE)
+
+env._set("SELF_HOSTED", "1")
+env._set("NODE_ENV", "jest")
+env._set("JWT_SECRET", "test-jwtsecret")
+env._set("LOG_LEVEL", "silent")
+env._set("MINIO_URL", "http://localhost")
+env._set("MINIO_ACCESS_KEY", "test")
+env._set("MINIO_SECRET_KEY", "test")
+
+global.console.log = jest.fn() // console.log are ignored in tests
+
+if (!process.env.CI) {
+  // set a longer timeout in dev for debugging
+  // 100 seconds
+  jest.setTimeout(100000)
+}

packages/backend-core/tests/utilities/TestConfiguration.js

@@ -1 +0,0 @@
-require("./db")

packages/backend-core/tests/utilities/db.js

@@ -1,6 +0,0 @@
-const core = require("../../src/index")
-const dbConfig = {
-  inMemory: true,
-  allDbs: true,
-}
-core.init({ db: dbConfig })

packages/backend-core/tests/utilities/db.ts

@@ -0,0 +1,9 @@
+import * as db from "../../src/db"
+
+const dbConfig = {
+  inMemory: true,
+}
+
+export const init = () => {
+  db.init(dbConfig)
+}

packages/backend-core/tests/utilities/index.ts

@@ -1,2 +1,5 @@
 export * as mocks from "./mocks"
 export * as structures from "./structures"
+
+import * as dbConfig from "./db"
+dbConfig.init()

packages/bbui/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/bbui",
   "description": "A UI solution used in the different Budibase projects.",
-  "version": "2.1.22-alpha.3",
+  "version": "2.1.22-alpha.4",
   "license": "MPL-2.0",
   "svelte": "src/index.js",
   "module": "dist/bbui.es.js",
@@ -38,7 +38,7 @@
   ],
   "dependencies": {
     "@adobe/spectrum-css-workflow-icons": "^1.2.1",
-    "@budibase/string-templates": "2.1.22-alpha.3",
+    "@budibase/string-templates": "2.1.22-alpha.4",
     "@spectrum-css/actionbutton": "^1.0.1",
     "@spectrum-css/actiongroup": "^1.0.1",
     "@spectrum-css/avatar": "^3.0.2",

packages/builder/cypress/integration/createAutomation.spec.js

@@ -12,7 +12,7 @@ filterTests(['smoke', 'all'], () => {
       cy.createTestTableWithData()
       cy.wait(2000)
       cy.contains("Automate").click()
-      cy.get(interact.ADD_BUTTON_SPECTRUM).click()
+      cy.get(interact.SPECTRUM_BUTTON_TEMPLATE).contains("Add automation").click({ force: true })
       cy.get(interact.MODAL_INNER_WRAPPER).within(() => {
         cy.get("input").type("Add Row")
         cy.contains("Row Created").click({ force: true })

packages/builder/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/builder",
-  "version": "2.1.22-alpha.3",
+  "version": "2.1.22-alpha.4",
   "license": "GPL-3.0",
   "private": true,
   "scripts": {
@@ -71,10 +71,10 @@
     }
   },
   "dependencies": {
-    "@budibase/bbui": "2.1.22-alpha.3",
-    "@budibase/client": "2.1.22-alpha.3",
-    "@budibase/frontend-core": "2.1.22-alpha.3",
-    "@budibase/string-templates": "2.1.22-alpha.3",
+    "@budibase/bbui": "2.1.22-alpha.4",
+    "@budibase/client": "2.1.22-alpha.4",
+    "@budibase/frontend-core": "2.1.22-alpha.4",
+    "@budibase/string-templates": "2.1.22-alpha.4",
     "@sentry/browser": "5.19.1",
     "@spectrum-css/page": "^3.0.1",
     "@spectrum-css/vars": "^3.0.1",
@@ -117,9 +117,9 @@
     "start-server-and-test": "^1.12.1",
     "svelte": "^3.48.0",
     "svelte-jester": "^1.3.2",
-    "ts-node": "^10.4.0",
+    "ts-node": "10.8.1",
     "tsconfig-paths": "4.0.0",
-    "typescript": "^4.5.5",
+    "typescript": "4.7.3",
     "vite": "^3.0.8"
   },
   "gitHead": "115189f72a850bfb52b65ec61d932531bf327072"

packages/builder/yarn.lock

@@ -923,17 +923,12 @@
     exec-sh "^0.3.2"
     minimist "^1.2.0"
 
-"@cspotcode/source-map-consumer@0.8.0":
-  version "0.8.0"
-  resolved "https://registry.yarnpkg.com/@cspotcode/source-map-consumer/-/source-map-consumer-0.8.0.tgz#33bf4b7b39c178821606f669bbc447a6a629786b"
-  integrity sha512-41qniHzTU8yAGbCp04ohlmSrZf8bkf/iJsl3V0dRGsQN/5GFfx+LbCSsCpp2gqrqjTVg/K6O8ycoV35JIwAzAg==
-
-"@cspotcode/source-map-support@0.7.0":
-  version "0.7.0"
-  resolved "https://registry.yarnpkg.com/@cspotcode/source-map-support/-/source-map-support-0.7.0.tgz#4789840aa859e46d2f3173727ab707c66bf344f5"
-  integrity sha512-X4xqRHqN8ACt2aHVe51OxeA2HjbcL4MqFqXkrmQszJ1NOUuUu5u6Vqx/0lZSVNku7velL5FC/s5uEAj1lsBMhA==
+"@cspotcode/source-map-support@^0.8.0":
+  version "0.8.1"
+  resolved "https://registry.yarnpkg.com/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz#00629c35a688e05a88b1cda684fb9d5e73f000a1"
+  integrity sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==
   dependencies:
-    "@cspotcode/source-map-consumer" "0.8.0"
+    "@jridgewell/trace-mapping" "0.3.9"
 
 "@cypress/request@^2.88.10":
   version "2.88.10"
@@ -1182,6 +1177,24 @@
     "@types/yargs" "^16.0.0"
     chalk "^4.0.0"
 
+"@jridgewell/resolve-uri@^3.0.3":
+  version "3.1.0"
+  resolved "https://registry.yarnpkg.com/@jridgewell/resolve-uri/-/resolve-uri-3.1.0.tgz#2203b118c157721addfe69d47b70465463066d78"
+  integrity sha512-F2msla3tad+Mfht5cJq7LSXcdudKTWCVYUgw6pLFOOHSTtZlj6SWNYAp+AhuqLmWdBO2X5hPrLcu8cVP8fy28w==
+
+"@jridgewell/sourcemap-codec@^1.4.10":
+  version "1.4.14"
+  resolved "https://registry.yarnpkg.com/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.14.tgz#add4c98d341472a289190b424efbdb096991bb24"
+  integrity sha512-XPSJHWmi394fuUuzDnGz1wiKqWfo1yXecHQMRf2l6hztTO+nPru658AyDngaBe7isIxEkRsPR3FZh+s7iVa4Uw==
+
+"@jridgewell/trace-mapping@0.3.9":
+  version "0.3.9"
+  resolved "https://registry.yarnpkg.com/@jridgewell/trace-mapping/-/trace-mapping-0.3.9.tgz#6534fd5933a53ba7cbf3a17615e273a0d1273ff9"
+  integrity sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ==
+  dependencies:
+    "@jridgewell/resolve-uri" "^3.0.3"
+    "@jridgewell/sourcemap-codec" "^1.4.10"
+
 "@nodelib/fs.scandir@2.1.5":
   version "2.1.5"
   resolved "https://registry.yarnpkg.com/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz#7619c2eb21b25483f6d167548b4cfd5a7488c3d5"
@@ -5935,12 +5948,12 @@ tr46@~0.0.3:
   resolved "https://registry.yarnpkg.com/tr46/-/tr46-0.0.3.tgz#8184fd347dac9cdc185992f3a6622e14b9d9ab6a"
   integrity sha1-gYT9NH2snNwYWZLzpmIuFLnZq2o=
 
-ts-node@^10.4.0:
-  version "10.4.0"
-  resolved "https://registry.yarnpkg.com/ts-node/-/ts-node-10.4.0.tgz#680f88945885f4e6cf450e7f0d6223dd404895f7"
-  integrity sha512-g0FlPvvCXSIO1JDF6S232P5jPYqBkRL9qly81ZgAOSU7rwI0stphCgd2kLiCrU9DjQCrJMWEqcNSjQL02s6d8A==
+ts-node@10.8.1:
+  version "10.8.1"
+  resolved "https://registry.yarnpkg.com/ts-node/-/ts-node-10.8.1.tgz#ea2bd3459011b52699d7e88daa55a45a1af4f066"
+  integrity sha512-Wwsnao4DQoJsN034wePSg5nZiw4YKXf56mPIAeD6wVmiv+RytNSWqc2f3fKvcUoV+Yn2+yocD71VOfQHbmVX4g==
   dependencies:
-    "@cspotcode/source-map-support" "0.7.0"
+    "@cspotcode/source-map-support" "^0.8.0"
     "@tsconfig/node10" "^1.0.7"
     "@tsconfig/node12" "^1.0.7"
     "@tsconfig/node14" "^1.0.0"
@@ -5951,6 +5964,7 @@ ts-node@^10.4.0:
     create-require "^1.1.0"
     diff "^4.0.1"
     make-error "^1.1.1"
+    v8-compile-cache-lib "^3.0.1"
     yn "3.1.1"
 
 tsconfig-paths@4.0.0:
@@ -6023,10 +6037,10 @@ typedarray-to-buffer@^3.1.5:
   dependencies:
     is-typedarray "^1.0.0"
 
-typescript@^4.5.5:
-  version "4.5.5"
-  resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.5.5.tgz#d8c953832d28924a9e3d37c73d729c846c5896f3"
-  integrity sha512-TCTIul70LyWe6IJWT8QSYeA54WQe8EjQFU4wY52Fasj5UKx88LNYKCgBEHcOMOrFF1rKGbD8v/xcNWVUq9SymA==
+typescript@4.7.3:
+  version "4.7.3"
+  resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.7.3.tgz#8364b502d5257b540f9de4c40be84c98e23a129d"
+  integrity sha512-WOkT3XYvrpXx4vMMqlD+8R8R37fZkjyLGlxavMc4iB8lrl8L0DeTcHbYgw/v0N/z9wAFsgBhcsF0ruoySS22mA==
 
 unicode-canonical-property-names-ecmascript@^2.0.0:
   version "2.0.0"
@@ -6111,6 +6125,11 @@ uuid@^8.3.0, uuid@^8.3.2:
   resolved "https://registry.yarnpkg.com/uuid/-/uuid-8.3.2.tgz#80d5b5ced271bb9af6c445f21a1a04c606cefbe2"
   integrity sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==
 
+v8-compile-cache-lib@^3.0.1:
+  version "3.0.1"
+  resolved "https://registry.yarnpkg.com/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz#6336e8d71965cb3d35a1bbb7868445a7c05264bf"
+  integrity sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg==
+
 v8-to-istanbul@^7.0.0:
   version "7.1.2"
   resolved "https://registry.yarnpkg.com/v8-to-istanbul/-/v8-to-istanbul-7.1.2.tgz#30898d1a7fa0c84d225a2c1434fb958f290883c1"

packages/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/cli",
-  "version": "2.1.22-alpha.3",
+  "version": "2.1.22-alpha.4",
   "description": "Budibase CLI, for developers, self hosting and migrations.",
   "main": "src/index.js",
   "bin": {
@@ -26,9 +26,9 @@
     "outputPath": "build"
   },
   "dependencies": {
-    "@budibase/backend-core": "2.1.22-alpha.3",
-    "@budibase/string-templates": "2.1.22-alpha.3",
-    "@budibase/types": "2.1.22-alpha.3",
+    "@budibase/backend-core": "2.1.22-alpha.4",
+    "@budibase/string-templates": "2.1.22-alpha.4",
+    "@budibase/types": "2.1.22-alpha.4",
     "axios": "0.21.2",
     "chalk": "4.1.0",
     "cli-progress": "3.11.2",

packages/client/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/client",
-  "version": "2.1.22-alpha.3",
+  "version": "2.1.22-alpha.4",
   "license": "MPL-2.0",
   "module": "dist/budibase-client.js",
   "main": "dist/budibase-client.js",
@@ -19,9 +19,9 @@
     "dev:builder": "rollup -cw"
   },
   "dependencies": {
-    "@budibase/bbui": "2.1.22-alpha.3",
-    "@budibase/frontend-core": "2.1.22-alpha.3",
-    "@budibase/string-templates": "2.1.22-alpha.3",
+    "@budibase/bbui": "2.1.22-alpha.4",
+    "@budibase/frontend-core": "2.1.22-alpha.4",
+    "@budibase/string-templates": "2.1.22-alpha.4",
     "@spectrum-css/button": "^3.0.3",
     "@spectrum-css/card": "^3.0.3",
     "@spectrum-css/divider": "^1.0.3",

packages/frontend-core/package.json

@@ -1,12 +1,12 @@
 {
   "name": "@budibase/frontend-core",
-  "version": "2.1.22-alpha.3",
+  "version": "2.1.22-alpha.4",
   "description": "Budibase frontend core libraries used in builder and client",
   "author": "Budibase",
   "license": "MPL-2.0",
   "svelte": "src/index.js",
   "dependencies": {
-    "@budibase/bbui": "2.1.22-alpha.3",
+    "@budibase/bbui": "2.1.22-alpha.4",
     "lodash": "^4.17.21",
     "svelte": "^3.46.2"
   }

packages/sdk/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/sdk",
-  "version": "2.1.22-alpha.3",
+  "version": "2.1.22-alpha.4",
   "description": "Budibase Public API SDK",
   "author": "Budibase",
   "license": "MPL-2.0",

packages/server/jest.config.ts

@@ -0,0 +1,32 @@
+import { Config } from "@jest/types"
+import * as fs from "fs"
+
+const config: Config.InitialOptions = {
+  preset: "ts-jest",
+  testEnvironment: "node",
+  setupFiles: ["./src/tests/jestSetup.ts"],
+  collectCoverageFrom: [
+    "src/**/*.{js,ts}",
+    // The use of coverage with couchdb view functions breaks tests
+    "!src/db/views/staticViews.*",
+  ],
+  coverageReporters: ["lcov", "json", "clover"],
+}
+
+if (!process.env.CI) {
+  // use sources when not in CI
+  config.moduleNameMapper = {
+    "@budibase/backend-core/(.*)": "<rootDir>/../backend-core/$1",
+    "@budibase/backend-core": "<rootDir>/../backend-core/src",
+    "@budibase/types": "<rootDir>/../types/src",
+  }
+  // add pro sources if they exist
+  if (fs.existsSync("../../../budibase-pro")) {
+    config.moduleNameMapper["@budibase/pro"] =
+      "<rootDir>/../../../budibase-pro/packages/pro/src"
+  }
+} else {
+  console.log("Running tests with compiled dependency sources")
+}
+
+export default config

packages/server/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "2.1.22-alpha.3",
+  "version": "2.1.22-alpha.4",
   "description": "Budibase Web Server",
   "main": "src/index.ts",
   "repository": {
@@ -36,40 +36,6 @@
     "env:account:enable": "node scripts/account.js enable",
     "env:account:disable": "node scripts/account.js disable"
   },
-  "jest": {
-    "preset": "ts-jest",
-    "testEnvironment": "node",
-    "moduleNameMapper": {
-      "@budibase/backend-core/(.*)": "<rootDir>/../backend-core/$1",
-      "@budibase/backend-core": "<rootDir>/../backend-core/src",
-      "@budibase/types": "<rootDir>/../types/src"
-    },
-    "setupFiles": [
-      "./scripts/jestSetup.js"
-    ],
-    "collectCoverageFrom": [
-      "src/**/*.js",
-      "!**/node_modules/**",
-      "!src/db/views/*.js",
-      "!src/api/controllers/deploy/**/*.js",
-      "!src/*.js",
-      "!src/api/controllers/static/**/*",
-      "!src/db/dynamoClient.js",
-      "!src/utilities/usageQuota.js",
-      "!src/api/routes/tests/**/*",
-      "!src/db/tests/**/*",
-      "!src/tests/**/*",
-      "!src/automations/tests/**/*",
-      "!src/utilities/fileProcessor.js",
-      "!src/utilities/fileSystem/**/*",
-      "!src/utilities/redis.js"
-    ],
-    "coverageReporters": [
-      "lcov",
-      "json",
-      "clover"
-    ]
-  },
   "keywords": [
     "budibase"
   ],
@@ -77,11 +43,11 @@
   "license": "GPL-3.0",
   "dependencies": {
     "@apidevtools/swagger-parser": "10.0.3",
-    "@budibase/backend-core": "2.1.22-alpha.3",
-    "@budibase/client": "2.1.22-alpha.3",
-    "@budibase/pro": "2.1.22-alpha.3",
-    "@budibase/string-templates": "2.1.22-alpha.3",
-    "@budibase/types": "2.1.22-alpha.3",
+    "@budibase/backend-core": "2.1.22-alpha.4",
+    "@budibase/client": "2.1.22-alpha.4",
+    "@budibase/pro": "2.1.22-alpha.4",
+    "@budibase/string-templates": "2.1.22-alpha.4",
+    "@budibase/types": "2.1.22-alpha.4",
     "@bull-board/api": "3.7.0",
     "@bull-board/koa": "3.9.4",
     "@elastic/elasticsearch": "7.10.0",
@@ -179,7 +145,7 @@
     "eslint": "6.8.0",
     "ioredis-mock": "7.2.0",
     "is-wsl": "2.2.0",
-    "jest": "27.5.1",
+    "jest": "28.1.1",
     "jest-openapi": "0.14.2",
     "nodemon": "2.0.15",
     "openapi-types": "9.3.1",
@@ -190,10 +156,10 @@
     "supertest": "4.0.2",
     "swagger-jsdoc": "6.1.0",
     "timekeeper": "2.2.0",
-    "ts-jest": "27.1.3",
-    "ts-node": "10.5.0",
+    "ts-jest": "28.0.4",
+    "ts-node": "10.8.1",
     "tsconfig-paths": "4.0.0",
-    "typescript": "4.6.2",
+    "typescript": "4.7.3",
     "update-dotenv": "1.1.1"
   },
   "optionalDependencies": {

packages/server/src/tests/jestSetup.ts

@@ -1,5 +1,5 @@
 const { tmpdir } = require("os")
-const env = require("../src/environment")
+import env from "../environment"
 
 env._set("SELF_HOSTED", "1")
 env._set("NODE_ENV", "jest")
@@ -8,8 +8,11 @@ env._set("CLIENT_ID", "test-client-id")
 env._set("BUDIBASE_DIR", tmpdir("budibase-unittests"))
 env._set("LOG_LEVEL", "silent")
 env._set("PORT", 0)
+env._set("MINIO_URL", "http://localhost")
+env._set("MINIO_ACCESS_KEY", "test")
+env._set("MINIO_SECRET_KEY", "test")
 
-const { mocks } = require("@budibase/backend-core/tests")
+import { mocks } from "@budibase/backend-core/tests"
 
 // mock all dates to 2020-01-01T00:00:00.000Z
 // use tk.reset() to use real dates in individual tests
@@ -17,3 +20,9 @@ const tk = require("timekeeper")
 tk.freeze(mocks.date.MOCK_DATE)
 
 global.console.log = jest.fn() // console.log are ignored in tests
+
+if (!process.env.CI) {
+  // set a longer timeout in dev for debugging
+  // 100 seconds
+  jest.setTimeout(100000)
+}

packages/server/src/tests/utilities/TestConfiguration.js

@@ -116,7 +116,9 @@ class TestConfiguration {
     if (this.server) {
       this.server.close()
     }
-    cleanup(this.allApps.map(app => app.appId))
+    if (this.allApps) {
+      cleanup(this.allApps.map(app => app.appId))
+    }
   }
 
   // UTILS

packages/server/tsconfig.json

@@ -8,7 +8,8 @@
     "paths": {
       "@budibase/types": ["../types/src"],
       "@budibase/backend-core": ["../backend-core/src"],
-      "@budibase/backend-core/*": ["../backend-core/*"]
+      "@budibase/backend-core/*": ["../backend-core/*"],
+      "@budibase/pro": ["../../../budibase-pro/packages/pro/src"]
     }
   },
   "ts-node": {
@@ -17,6 +18,7 @@
   "references": [
     { "path": "../types" },
     { "path": "../backend-core" },
+    { "path": "../../../budibase-pro/packages/pro" }
   ],
   "include": [
     "src/**/*",

packages/string-templates/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/string-templates",
-  "version": "2.1.22-alpha.3",
+  "version": "2.1.22-alpha.4",
   "description": "Handlebars wrapper for Budibase templating.",
   "main": "src/index.cjs",
   "module": "dist/bundle.mjs",
@@ -44,7 +44,7 @@
     "rollup-plugin-node-globals": "^1.4.0",
     "rollup-plugin-node-resolve": "^5.2.0",
     "rollup-plugin-terser": "^7.0.2",
-    "typescript": "^4.5.5"
+    "typescript": "4.7.3"
   },
   "gitHead": "d1836a898cab3f8ab80ee6d8f42be1a9eed7dcdc"
 }

packages/string-templates/yarn.lock

@@ -4337,10 +4337,10 @@ typeof-article@^0.1.1:
   dependencies:
     kind-of "^3.1.0"
 
-typescript@^4.5.5:
-  version "4.5.5"
-  resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.5.5.tgz#d8c953832d28924a9e3d37c73d729c846c5896f3"
-  integrity sha512-TCTIul70LyWe6IJWT8QSYeA54WQe8EjQFU4wY52Fasj5UKx88LNYKCgBEHcOMOrFF1rKGbD8v/xcNWVUq9SymA==
+typescript@4.7.3:
+  version "4.7.3"
+  resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.7.3.tgz#8364b502d5257b540f9de4c40be84c98e23a129d"
+  integrity sha512-WOkT3XYvrpXx4vMMqlD+8R8R37fZkjyLGlxavMc4iB8lrl8L0DeTcHbYgw/v0N/z9wAFsgBhcsF0ruoySS22mA==
 
 uglify-js@^3.1.4:
   version "3.14.3"

packages/types/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/types",
-  "version": "2.1.22-alpha.3",
+  "version": "2.1.22-alpha.4",
   "description": "Budibase types",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",

packages/types/src/documents/app/backup.ts

@@ -40,6 +40,7 @@ export interface AppBackupMetadata {
 }
 
 export interface AppBackup extends Document, AppBackupMetadata {
+  _id: string
   filename?: string
 }
 

packages/types/src/documents/pouch.ts

@@ -24,3 +24,9 @@ interface BulkDocResponse {
   id: string
   rev: string
 }
+
+export interface PutResponse {
+  ok: boolean
+  id: string
+  rev: string
+}

packages/types/src/sdk/db.ts

@@ -1,10 +1,10 @@
 import PouchDB from "pouchdb"
 
 export type PouchOptions = {
-  inMemory: boolean
-  replication: boolean
-  onDisk: boolean
-  find: boolean
+  inMemory?: boolean
+  replication?: boolean
+  onDisk?: boolean
+  find?: boolean
 }
 
 export enum SortOption {

packages/worker/__mocks__/aws-sdk.ts

@@ -0,0 +1,15 @@
+const mockS3 = {
+  headBucket: jest.fn().mockReturnThis(),
+  deleteObject: jest.fn().mockReturnThis(),
+  deleteObjects: jest.fn().mockReturnThis(),
+  createBucket: jest.fn().mockReturnThis(),
+  listObjects: jest.fn().mockReturnThis(),
+  promise: jest.fn().mockReturnThis(),
+  catch: jest.fn(),
+}
+
+const AWS = {
+  S3: jest.fn(() => mockS3),
+}
+
+export default AWS

packages/worker/__mocks__/node-fetch.ts

@@ -0,0 +1 @@
+jest.mock("node-fetch", () => jest.fn())

packages/worker/jest.config.ts

@@ -0,0 +1,28 @@
+import { Config } from "@jest/types"
+import * as fs from "fs"
+
+const config: Config.InitialOptions = {
+  preset: "ts-jest",
+  testEnvironment: "node",
+  setupFiles: ["./src/tests/jestSetup.ts"],
+  collectCoverageFrom: ["src/**/*.{js,ts}"],
+  coverageReporters: ["lcov", "json", "clover"],
+}
+
+if (!process.env.CI) {
+  // use sources when not in CI
+  config.moduleNameMapper = {
+    "@budibase/backend-core/(.*)": "<rootDir>/../backend-core/$1",
+    "@budibase/backend-core": "<rootDir>/../backend-core/src",
+    "@budibase/types": "<rootDir>/../types/src",
+  }
+  // add pro sources if they exist
+  if (fs.existsSync("../../../budibase-pro")) {
+    config.moduleNameMapper["@budibase/pro"] =
+      "<rootDir>/../../../budibase-pro/packages/pro/src"
+  }
+} else {
+  console.log("Running tests with compiled dependency sources")
+}
+
+export default config

packages/worker/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/worker",
   "email": "hi@budibase.com",
-  "version": "2.1.22-alpha.3",
+  "version": "2.1.22-alpha.4",
   "description": "Budibase background service",
   "main": "src/index.ts",
   "repository": {
@@ -22,7 +22,7 @@
     "build:docker": "docker build . -t worker-service --label version=$BUDIBASE_RELEASE_VERSION",
     "dev:stack:init": "node ./scripts/dev/manage.js init",
     "dev:builder": "npm run dev:stack:init && nodemon",
-    "test": "jest --runInBand",
+    "test": "jest --coverage --runInBand",
     "test:watch": "jest --watch",
     "env:multi:enable": "node scripts/multiTenancy.js enable",
     "env:multi:disable": "node scripts/multiTenancy.js disable",
@@ -36,10 +36,10 @@
   "author": "Budibase",
   "license": "GPL-3.0",
   "dependencies": {
-    "@budibase/backend-core": "2.1.22-alpha.3",
-    "@budibase/pro": "2.1.22-alpha.3",
-    "@budibase/string-templates": "2.1.22-alpha.3",
-    "@budibase/types": "2.1.22-alpha.3",
+    "@budibase/backend-core": "2.1.22-alpha.4",
+    "@budibase/pro": "2.1.22-alpha.4",
+    "@budibase/string-templates": "2.1.22-alpha.4",
+    "@budibase/types": "2.1.22-alpha.4",
     "@koa/router": "8.0.8",
     "@sentry/node": "6.17.7",
     "@techpass/passport-openidconnect": "0.3.2",
@@ -79,30 +79,18 @@
     "@typescript-eslint/parser": "5.12.0",
     "copyfiles": "2.4.1",
     "eslint": "6.8.0",
-    "jest": "27.4.7",
+    "jest": "28.1.1",
     "nodemon": "2.0.15",
     "pouchdb-adapter-memory": "7.2.2",
     "prettier": "2.3.1",
     "rimraf": "3.0.2",
     "supertest": "6.2.2",
     "timekeeper": "2.2.0",
-    "ts-jest": "27.1.3",
-    "ts-node": "10.4.0",
+    "ts-jest": "28.0.4",
+    "ts-node": "10.8.1",
     "tsconfig-paths": "4.0.0",
-    "typescript": "4.5.5",
+    "typescript": "4.7.3",
     "update-dotenv": "1.1.1"
   },
-  "jest": {
-    "preset": "ts-jest",
-    "testEnvironment": "node",
-    "moduleNameMapper": {
-      "@budibase/backend-core/(.*)": "<rootDir>/../backend-core/$1",
-      "@budibase/backend-core": "<rootDir>/../backend-core/src",
-      "@budibase/types": "<rootDir>/../types/src"
-    },
-    "setupFiles": [
-      "./scripts/jestSetup.js"
-    ]
-  },
   "gitHead": "d1836a898cab3f8ab80ee6d8f42be1a9eed7dcdc"
 }

packages/worker/src/db/index.js

@@ -1,11 +0,0 @@
-const core = require("@budibase/backend-core")
-const env = require("../environment")
-
-exports.init = () => {
-  const dbConfig = {}
-  if (env.isTest() && !env.COUCH_DB_URL) {
-    dbConfig.inMemory = true
-    dbConfig.allDbs = true
-  }
-  core.init({ db: dbConfig })
-}

packages/worker/src/db/index.ts

@@ -0,0 +1,10 @@
+import core from "@budibase/backend-core"
+import env from "../environment"
+
+export const init = () => {
+  const dbConfig: any = {}
+  if (env.isTest() && !env.COUCH_DB_URL) {
+    dbConfig.inMemory = true
+  }
+  core.init({ db: dbConfig })
+}

packages/worker/src/index.ts

@@ -13,7 +13,7 @@ import { Scope } from "@sentry/node"
 import { Event } from "@sentry/types/dist/event"
 import Application from "koa"
 import { bootstrap } from "global-agent"
-import db from "./db"
+import * as db from "./db"
 db.init()
 const Koa = require("koa")
 const destroyable = require("server-destroy")

packages/worker/src/tests/TestConfiguration.ts

@@ -1,5 +1,5 @@
 import "./mocks"
-import dbConfig from "../db"
+import * as dbConfig from "../db"
 dbConfig.init()
 import env from "../environment"
 import controllers from "./controllers"

packages/worker/src/tests/index.ts

@@ -1,6 +1,6 @@
+import mocks from "./mocks"
 import TestConfiguration from "./TestConfiguration"
 import structures from "./structures"
-import mocks from "./mocks"
 import API from "./api"
 
 const pkg = {

packages/worker/src/tests/jestSetup.ts

@@ -1,12 +1,15 @@
-const env = require("../src/environment")
+import env from "../environment"
 
 env._set("SELF_HOSTED", "1")
 env._set("NODE_ENV", "jest")
 env._set("JWT_SECRET", "test-jwtsecret")
 env._set("LOG_LEVEL", "silent")
 env._set("MULTI_TENANCY", true)
+env._set("MINIO_URL", "http://localhost")
+env._set("MINIO_ACCESS_KEY", "test")
+env._set("MINIO_SECRET_KEY", "test")
 
-const { mocks } = require("@budibase/backend-core/tests")
+import { mocks } from "@budibase/backend-core/tests"
 
 // mock all dates to 2020-01-01T00:00:00.000Z
 // use tk.reset() to use real dates in individual tests

packages/worker/tsconfig.build.json

@@ -1,17 +1,17 @@
 {
   "compilerOptions": {
     "target": "es6",
+    "skipLibCheck": true,
     "module": "commonjs",
     "lib": ["es2020"],
     "allowJs": true,
+    "outDir": "dist",
     "strict": true,
     "noImplicitAny": true,
     "esModuleInterop": true,
     "resolveJsonModule": true,
     "incremental": true,
-    "types": [ "node", "jest" ],
-    "outDir": "dist",
-    "skipLibCheck": true
+    "types": [ "node", "jest"],
   },
   "include": [
     "src/**/*"
@@ -20,7 +20,7 @@
     "node_modules",
     "dist",
     "src/tests",
-    "**/*.spec.ts",
-    "**/*.spec.js"
+    "**/*.spec.js",
+    "**/*.spec.ts"
   ]
 }

packages/worker/tsconfig.json

@@ -8,7 +8,8 @@
     "paths": {
       "@budibase/types": ["../types/src"],
       "@budibase/backend-core": ["../backend-core/src"],
-      "@budibase/backend-core/*": ["../backend-core/*"]
+      "@budibase/backend-core/*": ["../backend-core/*"],
+      "@budibase/pro": ["../../../budibase-pro/packages/pro/src"]
     }
   },
   "ts-node": {
@@ -17,6 +18,7 @@
   "references": [
     { "path": "../types" },
     { "path": "../backend-core" },
+    { "path": "../../../budibase-pro/packages/pro" }
   ],
   "include": [
     "src/**/*",

qa-core/package.json

@@ -41,13 +41,13 @@
     "@types/node-fetch": "2.6.2",
     "chance": "1.1.8",
     "env-cmd": "^10.1.0",
-    "jest": "28.0.2",
+    "jest": "28.1.1",
     "prettier": "2.7.1",
     "start-server-and-test": "1.14.0",
     "timekeeper": "2.2.0",
     "ts-jest": "28.0.8",
-    "ts-node": "10.9.1",
-    "tsconfig-paths": "4.1.0",
+    "ts-node": "10.8.1",
+    "tsconfig-paths": "4.0.0",
     "typescript": "4.7.3"
   },
   "dependencies": {

scripts/pro/install.sh

@@ -1,7 +1,7 @@
-#!/bin/bash 
+#!/bin/bash
 
 if [[ -z "${CI}" ]]; then
-  echo 'Cannot run insall.sh unless in CI'
+  echo 'Cannot run install.sh unless in CI'
   exit 0
 fi
 

scripts/pro/release.sh

@@ -1,5 +1,8 @@
 #!/bin/bash 
 
+# Fail when any command fails
+set -e
+
 if [[ -z "${CI}" ]]; then
   echo 'Cannot run release.sh unless in CI'
   exit 0
@@ -59,22 +62,6 @@ git push
 
 lerna publish $VERSION --yes --force-publish --dist-tag $TAG
 
-#############################################
-#             POST-PUBLISH - PRO            #
-#############################################
-
-# Revert build changes on packages/pro/package.json
-cd packages/pro
-jq '.main = "src/index.ts" | .types = "src/index.ts"' package.json > package.json.tmp && mv package.json.tmp package.json
-
-# Go back to pro repo root
-cd -
-
-# Commit and push changes
-git add packages/pro/package.json
-git commit -m "Prep next development iteration"
-git push 
-
 #############################################
 #            POST-PUBLISH - BUDIBASE        #
 #############################################

scripts/pro/test.sh

@@ -1,4 +1,8 @@
 #!/bin/bash
+
+# Fail when any command fails
+set -e
+
 cd ../
 if [[ -d "budibase-pro" ]]; then
   cd budibase-pro