Merge branch 'develop' into bump-qa-core-deps

.github/workflows/budibase_ci.yml

@@ -11,7 +11,6 @@ on:
     branches:
       - master
       - develop 
-      - release 
  workflow_dispatch:
 
 env:
@@ -20,9 +19,53 @@ env:
   PERSONAL_ACCESS_TOKEN : ${{ secrets.PERSONAL_ACCESS_TOKEN }}
 
 jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Use Node.js 14.x
+      uses: actions/setup-node@v1
+      with:
+        node-version: 14.x
+    - run: yarn
+    - run: yarn lint
+
   build:
     runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Use Node.js 14.x
+        uses: actions/setup-node@v1
+        with:
+          node-version: 14.x
+      - name: Install Pro
+        run: yarn install:pro $BRANCH $BASE_BRANCH
+      - run: yarn
+      - run: yarn bootstrap
+      - run: yarn build
 
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Use Node.js 14.x
+        uses: actions/setup-node@v1
+        with:
+          node-version: 14.x
+      - name: Install Pro
+        run: yarn install:pro $BRANCH $BASE_BRANCH
+      - run: yarn
+      - run: yarn bootstrap
+      - run: yarn test
+      - uses: codecov/codecov-action@v1
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }} # not required for public repos
+          files: ./packages/server/coverage/clover.xml,./packages/worker/coverage/clover.xml,./packages/backend-core/coverage/clover.xml
+          name: codecov-umbrella
+          verbose: true
+
+  integration-test:
+    runs-on: ubuntu-latest
     services:
       couchdb:
         image: ibmcom/couchdb3
@@ -31,39 +74,18 @@ jobs:
           COUCHDB_USER: budibase
         ports:
           - 4567:5984
-
-    strategy:
-      matrix:
-        node-version: [14.x]
-
     steps:
-    - uses: actions/checkout@v2
+      - uses: actions/checkout@v2
-
+      - name: Use Node.js 14.x
-    - name: Use Node.js ${{ matrix.node-version }}
+        uses: actions/setup-node@v1
-      uses: actions/setup-node@v1
+        with:
-      with:
+          node-version: 14.x
-        node-version: ${{ matrix.node-version }}
+      - name: Install Pro
-
+        run: yarn install:pro $BRANCH $BASE_BRANCH
-    - name: Install Pro
+      - run: yarn
-      run: yarn install:pro $BRANCH $BASE_BRANCH
+      - run: yarn bootstrap
-
+      - run: yarn build
-    - run: yarn
+      - run: |
-    - run: yarn bootstrap
+          cd qa-core
-    - run: yarn lint
+          yarn
-    - run: yarn build
+          yarn api:test:ci
-    - run: yarn test
-      env:
-        CI: true
-        name: Budibase CI
-    - uses: codecov/codecov-action@v1
-      with:
-        token: ${{ secrets.CODECOV_TOKEN }} # not required for public repos
-        files: ./packages/server/coverage/clover.xml,./packages/worker/coverage/clover.xml,./packages/backend-core/coverage/clover.xml
-        name: codecov-umbrella
-        verbose: true 
-
-    - name: QA Core Integration Tests
-      run: | 
-        cd qa-core
-        yarn
-        yarn api:test:ci

.husky/pre-commit

@@ -1,4 +1,2 @@
 #!/bin/sh
 . "$(dirname "$0")/_/husky.sh"
-
-yarn run lint

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "2.3.17-alpha.6",
+  "version": "2.3.18-alpha.0",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

packages/backend-core/jest.config.ts

@@ -9,15 +9,9 @@ const baseConfig: Config.InitialProjectOptions = {
   transform: {
     "^.+\\.ts?$": "@swc/jest",
   },
-}
+  moduleNameMapper: {
-
-if (!process.env.CI) {
-  // use sources when not in CI
-  baseConfig.moduleNameMapper = {
     "@budibase/types": "<rootDir>/../types/src",
-  }
+  },
-} else {
-  console.log("Running tests with compiled dependency sources")
 }
 
 const config: Config.InitialOptions = {

packages/backend-core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/backend-core",
-  "version": "2.3.17-alpha.6",
+  "version": "2.3.18-alpha.0",
   "description": "Budibase backend core libraries used in server and worker",
   "main": "dist/src/index.js",
   "types": "dist/src/index.d.ts",
@@ -18,13 +18,13 @@
     "build:pro": "../../scripts/pro/build.sh",
     "postbuild": "yarn run build:pro",
     "build:dev": "yarn prebuild && tsc --build --watch --preserveWatchOutput",
-    "test": "jest --coverage",
+    "test": "jest --coverage --runInBand",
     "test:watch": "jest --watchAll"
   },
   "dependencies": {
     "@budibase/nano": "10.1.1",
     "@budibase/pouchdb-replication-stream": "1.2.10",
-    "@budibase/types": "2.3.17-alpha.6",
+    "@budibase/types": "2.3.18-alpha.0",
     "@shopify/jest-koa-mocks": "5.0.1",
     "@techpass/passport-openidconnect": "0.3.2",
     "aws-cloudfront-sign": "2.2.0",

packages/backend-core/src/middleware/passport/sso/tests/google.spec.ts

@@ -19,7 +19,7 @@ describe("google", () => {
     const callbackUrl = generator.url()
 
     it("should create successfully create a google strategy", async () => {
-      await google.strategyFactory(googleConfig, callbackUrl)
+      await google.strategyFactory(googleConfig, callbackUrl, mockSaveUserFn)
 
       const expectedOptions = {
         clientID: googleConfig.clientID,

packages/backend-core/src/migrations/migrations.ts

@@ -4,7 +4,7 @@ import {
   StaticDatabases,
   getAllApps,
   getGlobalDBName,
-  doWithDB,
+  getDB,
 } from "../db"
 import environment from "../environment"
 import * as platform from "../platform"
@@ -86,66 +86,65 @@ export const runMigration = async (
     count++
     const lengthStatement = length > 1 ? `[${count}/${length}]` : ""
 
-    await doWithDB(dbName, async (db: any) => {
+    const db = getDB(dbName)
-      try {
+    try {
-        const doc = await getMigrationsDoc(db)
+      const doc = await getMigrationsDoc(db)
 
-        // the migration has already been run
+      // the migration has already been run
-        if (doc[migrationName]) {
+      if (doc[migrationName]) {
-          // check for force
+        // check for force
-          if (
+        if (
-            options.force &&
+          options.force &&
-            options.force[migrationType] &&
+          options.force[migrationType] &&
-            options.force[migrationType].includes(migrationName)
+          options.force[migrationType].includes(migrationName)
-          ) {
+        ) {
-            log(
-              `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Forcing`
-            )
-          } else {
-            // no force, exit
-            return
-          }
-        }
-
-        // check if the migration is not a no-op
-        if (!options.noOp) {
           log(
-            `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Running ${lengthStatement}`
+            `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Forcing`
-          )
-
-          if (migration.preventRetry) {
-            // eagerly set the completion date
-            // so that we never run this migration twice even upon failure
-            doc[migrationName] = Date.now()
-            const response = await db.put(doc)
-            doc._rev = response.rev
-          }
-
-          // run the migration
-          if (migrationType === MigrationType.APP) {
-            await context.doInAppContext(db.name, async () => {
-              await migration.fn(db)
-            })
-          } else {
-            await migration.fn(db)
-          }
-
-          log(
-            `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Complete`
           )
+        } else {
+          // no force, exit
+          return
         }
-
-        // mark as complete
-        doc[migrationName] = Date.now()
-        await db.put(doc)
-      } catch (err) {
-        console.error(
-          `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Error: `,
-          err
-        )
-        throw err
       }
-    })
+
+      // check if the migration is not a no-op
+      if (!options.noOp) {
+        log(
+          `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Running ${lengthStatement}`
+        )
+
+        if (migration.preventRetry) {
+          // eagerly set the completion date
+          // so that we never run this migration twice even upon failure
+          doc[migrationName] = Date.now()
+          const response = await db.put(doc)
+          doc._rev = response.rev
+        }
+
+        // run the migration
+        if (migrationType === MigrationType.APP) {
+          await context.doInAppContext(db.name, async () => {
+            await migration.fn(db)
+          })
+        } else {
+          await migration.fn(db)
+        }
+
+        log(
+          `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Complete`
+        )
+      }
+
+      // mark as complete
+      doc[migrationName] = Date.now()
+      await db.put(doc)
+    } catch (err) {
+      console.error(
+        `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Error: `,
+        err
+      )
+      throw err
+    }
   }
 }
 
@@ -185,7 +184,10 @@ export const runMigrations = async (
     // for all migrations
     for (const migration of migrations) {
       // run the migration
-      await context.doInTenant(tenantId, () => runMigration(migration, options))
+      await context.doInTenant(
+        tenantId,
+        async () => await runMigration(migration, options)
+      )
     }
   }
   console.log("Migrations complete")

packages/backend-core/src/migrations/tests/index.spec.js

@@ -1,57 +0,0 @@
-require("../../../tests")
-const { runMigrations, getMigrationsDoc } = require("../index")
-const { getGlobalDBName, getDB } = require("../../db")
-
-const { structures, testEnv } = require("../../../tests")
-testEnv.multiTenant()
-
-let db
-
-describe("migrations", () => {
-
-  const migrationFunction = jest.fn()
-
-  const MIGRATIONS = [{
-    type: "global",
-    name: "test",
-    fn: migrationFunction
-  }]
-
-  let tenantId
-
-  beforeEach(() => {
-    tenantId = structures.tenant.id()
-    db = getDB(getGlobalDBName(tenantId))
-  })
-
-  afterEach(async () => {
-    jest.clearAllMocks()
-    await db.destroy()
-  })
-
-  const migrate = () => {
-    return runMigrations(MIGRATIONS, { tenantIds: [tenantId]})
-  }
-
-  it("should run a new migration", async () => {
-    await migrate()
-    expect(migrationFunction).toHaveBeenCalled()
-    const doc = await getMigrationsDoc(db)
-    expect(doc.test).toBeDefined()
-  })
-
-  it("should match snapshot", async () => {
-    await migrate()
-    const doc = await getMigrationsDoc(db)
-    expect(doc).toMatchSnapshot()
-  })
-
-  it("should skip a previously run migration", async () => {
-    await migrate()
-    const previousMigrationTime = await getMigrationsDoc(db).test
-    await migrate()
-    const currentMigrationTime = await getMigrationsDoc(db).test
-    expect(migrationFunction).toHaveBeenCalledTimes(1)
-    expect(currentMigrationTime).toBe(previousMigrationTime)
-  })
-})

packages/backend-core/src/migrations/tests/migrations.spec.ts

@@ -0,0 +1,64 @@
+import { testEnv, DBTestConfiguration } from "../../../tests"
+import * as migrations from "../index"
+import * as context from "../../context"
+import { MigrationType } from "@budibase/types"
+
+testEnv.multiTenant()
+
+describe("migrations", () => {
+  const config = new DBTestConfiguration()
+
+  const migrationFunction = jest.fn()
+
+  const MIGRATIONS = [
+    {
+      type: MigrationType.GLOBAL,
+      name: "test" as any,
+      fn: migrationFunction,
+    },
+  ]
+
+  beforeEach(() => {
+    config.newTenant()
+  })
+
+  afterEach(async () => {
+    jest.clearAllMocks()
+  })
+
+  const migrate = () => {
+    return migrations.runMigrations(MIGRATIONS, {
+      tenantIds: [config.tenantId],
+    })
+  }
+
+  it("should run a new migration", async () => {
+    await config.doInTenant(async () => {
+      await migrate()
+      expect(migrationFunction).toHaveBeenCalled()
+      const db = context.getGlobalDB()
+      const doc = await migrations.getMigrationsDoc(db)
+      expect(doc.test).toBeDefined()
+    })
+  })
+
+  it("should match snapshot", async () => {
+    await config.doInTenant(async () => {
+      await migrate()
+      const doc = await migrations.getMigrationsDoc(context.getGlobalDB())
+      expect(doc).toMatchSnapshot()
+    })
+  })
+
+  it("should skip a previously run migration", async () => {
+    await config.doInTenant(async () => {
+      const db = context.getGlobalDB()
+      await migrate()
+      const previousDoc = await migrations.getMigrationsDoc(db)
+      await migrate()
+      const currentDoc = await migrations.getMigrationsDoc(db)
+      expect(migrationFunction).toHaveBeenCalledTimes(1)
+      expect(currentDoc.test).toBe(previousDoc.test)
+    })
+  })
+})

packages/backend-core/src/redis/init.ts

@@ -20,13 +20,17 @@ async function init() {
   ).init()
 }
 
-process.on("exit", async () => {
+export async function shutdown() {
   if (userClient) await userClient.finish()
   if (sessionClient) await sessionClient.finish()
   if (appClient) await appClient.finish()
   if (cacheClient) await cacheClient.finish()
   if (writethroughClient) await writethroughClient.finish()
   if (lockClient) await lockClient.finish()
+}
+
+process.on("exit", async () => {
+  await shutdown()
 })
 
 export async function getUserClient() {

packages/backend-core/src/redis/redis.ts

@@ -91,6 +91,11 @@ function init(selectDb = DEFAULT_SELECT_DB) {
   }
   // attach handlers
   client.on("end", (err: Error) => {
+    if (env.isTest()) {
+      // don't try to re-connect in test env
+      // allow the process to exit
+      return
+    }
     connectionError(selectDb, timeout, err)
   })
   client.on("error", (err: Error) => {

packages/backend-core/tests/utilities/DBTestConfiguration.ts

@@ -12,6 +12,10 @@ class DBTestConfiguration {
     this.tenantId = structures.tenant.id()
   }
 
+  newTenant() {
+    this.tenantId = structures.tenant.id()
+  }
+
   // TENANCY
 
   doInTenant(task: any) {

packages/bbui/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/bbui",
   "description": "A UI solution used in the different Budibase projects.",
-  "version": "2.3.17-alpha.6",
+  "version": "2.3.18-alpha.0",
   "license": "MPL-2.0",
   "svelte": "src/index.js",
   "module": "dist/bbui.es.js",
@@ -38,7 +38,7 @@
   ],
   "dependencies": {
     "@adobe/spectrum-css-workflow-icons": "1.2.1",
-    "@budibase/string-templates": "2.3.17-alpha.6",
+    "@budibase/string-templates": "2.3.18-alpha.0",
     "@spectrum-css/accordion": "3.0.24",
     "@spectrum-css/actionbutton": "1.0.1",
     "@spectrum-css/actiongroup": "1.0.1",

packages/builder/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/builder",
-  "version": "2.3.17-alpha.6",
+  "version": "2.3.18-alpha.0",
   "license": "GPL-3.0",
   "private": true,
   "scripts": {
@@ -58,10 +58,10 @@
     }
   },
   "dependencies": {
-    "@budibase/bbui": "2.3.17-alpha.6",
+    "@budibase/bbui": "2.3.18-alpha.0",
-    "@budibase/client": "2.3.17-alpha.6",
+    "@budibase/client": "2.3.18-alpha.0",
-    "@budibase/frontend-core": "2.3.17-alpha.6",
+    "@budibase/frontend-core": "2.3.18-alpha.0",
-    "@budibase/string-templates": "2.3.17-alpha.6",
+    "@budibase/string-templates": "2.3.18-alpha.0",
     "@fortawesome/fontawesome-svg-core": "^6.2.1",
     "@fortawesome/free-brands-svg-icons": "^6.2.1",
     "@fortawesome/free-solid-svg-icons": "^6.2.1",

packages/builder/src/components/common/bindings/BindingPanel.svelte

@@ -183,6 +183,7 @@
     bind:this={popover}
     anchor={popoverAnchor}
     maxWidth={300}
+    dismissible={false}
   >
     <Layout gap="S">
       <div class="helper">

packages/builder/src/components/design/settings/controls/FilterEditor/FilterDrawer.svelte

@@ -254,8 +254,8 @@
               {:else if filter.type === "datetime"}
                 <DatePicker
                   disabled={filter.noValue}
-                  enableTime={!getSchema(filter).dateOnly}
+                  enableTime={!getSchema(filter)?.dateOnly}
-                  timeOnly={getSchema(filter).timeOnly}
+                  timeOnly={getSchema(filter)?.timeOnly}
                   bind:value={filter.value}
                 />
               {:else}

packages/builder/src/components/start/CreateAppModal.svelte

@@ -26,7 +26,15 @@
 
   const values = writable({ name: "", url: null })
   const validation = createValidationStore()
-  $: validation.check($values)
+
+  $: {
+    const { name, url } = $values
+
+    validation.check({
+      name,
+      url: url?.[0] === "/" ? url.substring(1, url.length) : url,
+    })
+  }
 
   onMount(async () => {
     const lastChar = $auth.user?.firstName
@@ -87,7 +95,11 @@
     appValidation.url(validation, { apps: applications })
     appValidation.file(validation, { template })
     // init validation
-    validation.check($values)
+    const { name, url } = $values
+    validation.check({
+      name,
+      url: url?.[0] === "/" ? url.substring(1, url.length) : url,
+    })
   }
 
   async function createNewApp() {

packages/builder/src/components/start/UpdateAppModal.svelte

@@ -23,14 +23,25 @@
   })
   const validation = createValidationStore()
 
-  $: validation.check($values)
+  $: {
+    const { name, url } = $values
+
+    validation.check({
+      name,
+      url: url?.[0] === "/" ? url.substring(1, url.length) : url,
+    })
+  }
 
   const setupValidation = async () => {
     const applications = svelteGet(apps)
     appValidation.name(validation, { apps: applications, currentApp: app })
     appValidation.url(validation, { apps: applications, currentApp: app })
     // init validation
-    validation.check($values)
+    const { name, url } = $values
+    validation.check({
+      name,
+      url: url?.[0] === "/" ? url.substring(1, url.length) : url,
+    })
   }
 
   async function updateApp() {

packages/builder/src/constants/index.js

@@ -46,7 +46,7 @@ export const LAYOUT_NAMES = {
 // one or more word characters and whitespace
 export const APP_NAME_REGEX = /^[\w\s]+$/
 // zero or more non-whitespace characters
-export const APP_URL_REGEX = /^\S*$/
+export const APP_URL_REGEX = /^[0-9a-zA-Z-_]+$/
 
 export const DefaultAppTheme = {
   primaryColor: "var(--spectrum-global-color-blue-600)",

packages/builder/src/helpers/validation/yup/app.js

@@ -62,11 +62,9 @@ export const url = (validation, { apps, currentApp } = { apps: [] }) => {
         }
         // make it clear that this is a url path and cannot be a full url
         return (
-          value.startsWith("/") &&
           !value.includes("http") &&
           !value.includes("www") &&
-          !value.includes(".") &&
+          !value.includes(".")
-          value.length > 1 // just '/' is not valid
         )
       })
   )

packages/builder/src/pages/builder/portal/apps/onboarding/_components/NamePanel.svelte

@@ -1,6 +1,7 @@
 <script>
   import { Button, FancyForm, FancyInput } from "@budibase/bbui"
   import PanelHeader from "./PanelHeader.svelte"
+  import { APP_URL_REGEX } from "constants"
 
   export let name = ""
   export let url = ""
@@ -25,6 +26,10 @@
     if (url.length < 1) {
       return "URL must be provided"
     }
+
+    if (!APP_URL_REGEX.test(url)) {
+      return "Invalid URL"
+    }
   }
 </script>
 

packages/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/cli",
-  "version": "2.3.17-alpha.6",
+  "version": "2.3.18-alpha.0",
   "description": "Budibase CLI, for developers, self hosting and migrations.",
   "main": "src/index.js",
   "bin": {
@@ -26,9 +26,9 @@
     "outputPath": "build"
   },
   "dependencies": {
-    "@budibase/backend-core": "2.3.17-alpha.6",
+    "@budibase/backend-core": "2.3.18-alpha.0",
-    "@budibase/string-templates": "2.3.17-alpha.6",
+    "@budibase/string-templates": "2.3.18-alpha.0",
-    "@budibase/types": "2.3.17-alpha.6",
+    "@budibase/types": "2.3.18-alpha.0",
     "axios": "0.21.2",
     "chalk": "4.1.0",
     "cli-progress": "3.11.2",

packages/client/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/client",
-  "version": "2.3.17-alpha.6",
+  "version": "2.3.18-alpha.0",
   "license": "MPL-2.0",
   "module": "dist/budibase-client.js",
   "main": "dist/budibase-client.js",
@@ -19,9 +19,9 @@
     "dev:builder": "rollup -cw"
   },
   "dependencies": {
-    "@budibase/bbui": "2.3.17-alpha.6",
+    "@budibase/bbui": "2.3.18-alpha.0",
-    "@budibase/frontend-core": "2.3.17-alpha.6",
+    "@budibase/frontend-core": "2.3.18-alpha.0",
-    "@budibase/string-templates": "2.3.17-alpha.6",
+    "@budibase/string-templates": "2.3.18-alpha.0",
     "@spectrum-css/button": "^3.0.3",
     "@spectrum-css/card": "^3.0.3",
     "@spectrum-css/divider": "^1.0.3",

packages/frontend-core/package.json

@@ -1,12 +1,12 @@
 {
   "name": "@budibase/frontend-core",
-  "version": "2.3.17-alpha.6",
+  "version": "2.3.18-alpha.0",
   "description": "Budibase frontend core libraries used in builder and client",
   "author": "Budibase",
   "license": "MPL-2.0",
   "svelte": "src/index.js",
   "dependencies": {
-    "@budibase/bbui": "2.3.17-alpha.6",
+    "@budibase/bbui": "2.3.18-alpha.0",
     "lodash": "^4.17.21",
     "svelte": "^3.46.2"
   }

packages/sdk/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/sdk",
-  "version": "2.3.17-alpha.6",
+  "version": "2.3.18-alpha.0",
   "description": "Budibase Public API SDK",
   "author": "Budibase",
   "license": "MPL-2.0",

packages/server/jest.config.ts

@@ -11,22 +11,17 @@ const baseConfig: Config.InitialProjectOptions = {
   transform: {
     "^.+\\.ts?$": "@swc/jest",
   },
-}
+  moduleNameMapper: {
-
-if (!process.env.CI) {
-  // use sources when not in CI
-  baseConfig.moduleNameMapper = {
     "@budibase/backend-core/(.*)": "<rootDir>/../backend-core/$1",
     "@budibase/backend-core": "<rootDir>/../backend-core/src",
     "@budibase/types": "<rootDir>/../types/src",
-  }
+  },
-  // add pro sources if they exist
+}
-  if (fs.existsSync("../../../budibase-pro")) {
+
-    baseConfig.moduleNameMapper["@budibase/pro"] =
+// add pro sources if they exist
-      "<rootDir>/../../../budibase-pro/packages/pro/src"
+if (fs.existsSync("../../../budibase-pro")) {
-  }
+  baseConfig.moduleNameMapper["@budibase/pro"] =
-} else {
+    "<rootDir>/../../../budibase-pro/packages/pro/src"
-  console.log("Running tests with compiled dependency sources")
 }
 
 const config: Config.InitialOptions = {

packages/server/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "2.3.17-alpha.6",
+  "version": "2.3.18-alpha.0",
   "description": "Budibase Web Server",
   "main": "src/index.ts",
   "repository": {
@@ -14,7 +14,7 @@
     "build:dev": "yarn prebuild && tsc --build --watch --preserveWatchOutput",
     "debug": "yarn build && node --expose-gc --inspect=9222 dist/index.js",
     "postbuild": "copyfiles -u 1 src/**/*.svelte dist/ && copyfiles -u 1 src/**/*.hbs dist/ && copyfiles -u 1 src/**/*.json dist/",
-    "test": "jest --coverage --maxWorkers=2",
+    "test": "jest --coverage --runInBand",
     "test:watch": "jest --watch",
     "predocker": "copyfiles -f ../client/dist/budibase-client.js ../client/manifest.json client",
     "build:docker": "yarn run predocker && docker build . -t app-service --label version=$BUDIBASE_RELEASE_VERSION",
@@ -43,11 +43,11 @@
   "license": "GPL-3.0",
   "dependencies": {
     "@apidevtools/swagger-parser": "10.0.3",
-    "@budibase/backend-core": "2.3.17-alpha.6",
+    "@budibase/backend-core": "2.3.18-alpha.0",
-    "@budibase/client": "2.3.17-alpha.6",
+    "@budibase/client": "2.3.18-alpha.0",
-    "@budibase/pro": "2.3.17-alpha.6",
+    "@budibase/pro": "2.3.18-alpha.0",
-    "@budibase/string-templates": "2.3.17-alpha.6",
+    "@budibase/string-templates": "2.3.18-alpha.0",
-    "@budibase/types": "2.3.17-alpha.6",
+    "@budibase/types": "2.3.18-alpha.0",
     "@bull-board/api": "3.7.0",
     "@bull-board/koa": "3.9.4",
     "@elastic/elasticsearch": "7.10.0",

packages/server/specs/resources/query.js

@@ -1,6 +1,6 @@
 const Resource = require("./utils/Resource")
 const { object } = require("./utils")
-const { BaseQueryVerbs } = require("../../dist/constants")
+const { BaseQueryVerbs } = require("../../src/constants")
 
 const query = {
   _id: "query_datasource_plus_4d8be0c506b9465daf4bf84d890fdab6_454854487c574d45bc4029b1e153219e",

packages/server/specs/resources/table.js

@@ -2,7 +2,7 @@ const {
   FieldTypes,
   RelationshipTypes,
   FormulaTypes,
-} = require("../../dist/constants")
+} = require("../../src/constants")
 const { object } = require("./utils")
 const Resource = require("./utils/Resource")
 

packages/server/src/api/controllers/row/staticFormula.ts

@@ -38,7 +38,13 @@ export async function updateRelatedFormula(
         if (!relatedRows[relatedTableId]) {
           relatedRows[relatedTableId] = []
         }
-        relatedRows[relatedTableId] = relatedRows[relatedTableId].concat(field)
+        // filter down to the rows which are not already included in related
+        const currentIds = relatedRows[relatedTableId].map(row => row._id)
+        const uniqueRelatedRows = field.filter(
+          (row: Row) => !currentIds.includes(row._id)
+        )
+        relatedRows[relatedTableId] =
+          relatedRows[relatedTableId].concat(uniqueRelatedRows)
       }
     }
     for (let tableId of table.relatedFormula) {

packages/server/src/api/routes/tests/static.spec.js

@@ -13,18 +13,6 @@ describe("/static", () => {
     app = await config.init()
   })
 
-  describe("/builder", () => {
-    it("should serve the builder", async () => {
-      const res = await request
-        .get("/builder/portal")
-        .set(config.defaultHeaders())
-        .expect("Content-Type", /text\/html/)
-        .expect(200)
-
-      expect(res.text).toContain("<title>Budibase</title>")
-    })
-  })
-
   describe("/app", () => {
     beforeEach(() => {
       jest.clearAllMocks()

packages/server/src/api/routes/tests/user.spec.js

@@ -1,4 +1,4 @@
-const { roles, utils } = require("@budibase/backend-core")
+const { roles } = require("@budibase/backend-core")
 const { checkPermissionsEndpoint } = require("./utilities/TestFunctions")
 const setup = require("./utilities")
 const { BUILTIN_ROLE_IDS } = roles
@@ -21,8 +21,7 @@ describe("/users", () => {
 
   afterAll(setup.afterAll)
 
-  // For some reason this cannot be a beforeAll or the test "should be able to update the user" fail
+  beforeAll(async () => {
-  beforeEach(async () => {
     await config.init()
   })
 

packages/server/src/automations/steps/discord.ts

@@ -67,17 +67,33 @@ export async function run({ inputs }: AutomationStepInput) {
   if (!avatar_url) {
     avatar_url = DEFAULT_AVATAR_URL
   }
-  const response = await fetch(url, {
+  if (!url?.trim()?.length) {
-    method: "post",
+    return {
-    body: JSON.stringify({
+      httpStatus: 400,
-      username,
+      response: "Missing Webhook URL",
-      avatar_url,
+      success: false,
-      content,
+    }
-    }),
+  }
-    headers: {
+  let response
-      "Content-Type": "application/json",
+  try {
-    },
+    response = await fetch(url, {
-  })
+      method: "post",
+      body: JSON.stringify({
+        username,
+        avatar_url,
+        content,
+      }),
+      headers: {
+        "Content-Type": "application/json",
+      },
+    })
+  } catch (err: any) {
+    return {
+      httpStatus: 400,
+      response: err.message,
+      success: false,
+    }
+  }
 
   const { status, message } = await getFetchResponse(response)
   return {

packages/server/src/automations/steps/integromat.ts

@@ -69,19 +69,35 @@ export const definition: AutomationStepSchema = {
 export async function run({ inputs }: AutomationStepInput) {
   const { url, value1, value2, value3, value4, value5 } = inputs
 
-  const response = await fetch(url, {
+  if (!url?.trim()?.length) {
-    method: "post",
+    return {
-    body: JSON.stringify({
+      httpStatus: 400,
-      value1,
+      response: "Missing Webhook URL",
-      value2,
+      success: false,
-      value3,
+    }
-      value4,
+  }
-      value5,
+  let response
-    }),
+  try {
-    headers: {
+    response = await fetch(url, {
-      "Content-Type": "application/json",
+      method: "post",
-    },
+      body: JSON.stringify({
-  })
+        value1,
+        value2,
+        value3,
+        value4,
+        value5,
+      }),
+      headers: {
+        "Content-Type": "application/json",
+      },
+    })
+  } catch (err: any) {
+    return {
+      httpStatus: 400,
+      response: err.message,
+      success: false,
+    }
+  }
 
   const { status, message } = await getFetchResponse(response)
   return {

packages/server/src/automations/steps/slack.ts

@@ -50,15 +50,31 @@ export const definition: AutomationStepSchema = {
 
 export async function run({ inputs }: AutomationStepInput) {
   let { url, text } = inputs
-  const response = await fetch(url, {
+  if (!url?.trim()?.length) {
-    method: "post",
+    return {
-    body: JSON.stringify({
+      httpStatus: 400,
-      text,
+      response: "Missing Webhook URL",
-    }),
+      success: false,
-    headers: {
+    }
-      "Content-Type": "application/json",
+  }
-    },
+  let response
-  })
+  try {
+    response = await fetch(url, {
+      method: "post",
+      body: JSON.stringify({
+        text,
+      }),
+      headers: {
+        "Content-Type": "application/json",
+      },
+    })
+  } catch (err: any) {
+    return {
+      httpStatus: 400,
+      response: err.message,
+      success: false,
+    }
+  }
 
   const { status, message } = await getFetchResponse(response)
   return {

packages/server/src/automations/steps/zapier.ts

@@ -63,22 +63,38 @@ export const definition: AutomationStepSchema = {
 export async function run({ inputs }: AutomationStepInput) {
   const { url, value1, value2, value3, value4, value5 } = inputs
 
+  if (!url?.trim()?.length) {
+    return {
+      httpStatus: 400,
+      response: "Missing Webhook URL",
+      success: false,
+    }
+  }
   // send the platform to make sure zaps always work, even
   // if no values supplied
-  const response = await fetch(url, {
+  let response
-    method: "post",
+  try {
-    body: JSON.stringify({
+    response = await fetch(url, {
-      platform: "budibase",
+      method: "post",
-      value1,
+      body: JSON.stringify({
-      value2,
+        platform: "budibase",
-      value3,
+        value1,
-      value4,
+        value2,
-      value5,
+        value3,
-    }),
+        value4,
-    headers: {
+        value5,
-      "Content-Type": "application/json",
+      }),
-    },
+      headers: {
-  })
+        "Content-Type": "application/json",
+      },
+    })
+  } catch (err: any) {
+    return {
+      httpStatus: 400,
+      response: err.message,
+      success: false,
+    }
+  }
 
   const { status, message } = await getFetchResponse(response)
 

packages/server/src/integration-test/postgres.spec.ts

@@ -419,14 +419,16 @@ describe("row api - postgres", () => {
 
     describe("given a row with relation data", () => {
       let row: Row
+      let foreignRow: Row
       beforeEach(async () => {
         let [createdRow] = await populatePrimaryRows(1, {
           createForeignRow: true,
         })
         row = createdRow.row
+        foreignRow = createdRow.foreignRow!
       })
 
-      it("foreign key fields are not retrieved", async () => {
+      it("only foreign keys are retrieved", async () => {
         const res = await getRow(primaryPostgresTable._id, row.id)
 
         expect(res.status).toBe(200)
@@ -436,7 +438,15 @@ describe("row api - postgres", () => {
           _id: expect.any(String),
           _rev: expect.any(String),
         })
+
         expect(res.body.foreignField).toBeUndefined()
+
+        expect(
+          res.body[`fk_${auxPostgresTable.name}_foreignField`]
+        ).toBeDefined()
+        expect(res.body[`fk_${auxPostgresTable.name}_foreignField`]).toBe(
+          foreignRow.id
+        )
       })
     })
   })

packages/server/src/utilities/redis.ts

@@ -21,6 +21,8 @@ export async function shutdown() {
   if (devAppClient) await devAppClient.finish()
   if (debounceClient) await debounceClient.finish()
   if (flagClient) await flagClient.finish()
+  // shutdown core clients
+  await redis.clients.shutdown()
   console.log("Redis shutdown")
 }
 

packages/server/yarn.lock

@@ -1278,14 +1278,14 @@
   resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
   integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==
 
-"@budibase/backend-core@2.3.17-alpha.6":
+"@budibase/backend-core@2.3.18-alpha.0":
-  version "2.3.17-alpha.6"
+  version "2.3.18-alpha.0"
-  resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-2.3.17-alpha.6.tgz#b083b3899d435694105d37a1ec817ec6cdc9bb07"
+  resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-2.3.18-alpha.0.tgz#c0a64a150c1fef9cc69f95f0aece4e857d64438d"
-  integrity sha512-8ljXZnK6Db3Mexk+MyIsrQBFFu6aV3eRtiw5pwHl9BobxR+6s79YmUgLfjZCfpgzMXqnT73FnV7g8K7KPTNC3g==
+  integrity sha512-ugD+WMoFwpXm+moSLHUgaBOu4XpX0+5UhmMWcNeRtH0Yd9GpDh2QzwtoN8BtXq8k5gkVEyoNSz+6oxKfNkNVdQ==
   dependencies:
     "@budibase/nano" "10.1.1"
     "@budibase/pouchdb-replication-stream" "1.2.10"
-    "@budibase/types" "2.3.17-alpha.6"
+    "@budibase/types" "2.3.18-alpha.0"
     "@shopify/jest-koa-mocks" "5.0.1"
     "@techpass/passport-openidconnect" "0.3.2"
     aws-cloudfront-sign "2.2.0"
@@ -1392,13 +1392,13 @@
     pouchdb-promise "^6.0.4"
     through2 "^2.0.0"
 
-"@budibase/pro@2.3.17-alpha.6":
+"@budibase/pro@2.3.18-alpha.0":
-  version "2.3.17-alpha.6"
+  version "2.3.18-alpha.0"
-  resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.3.17-alpha.6.tgz#2d24afa48003ff5928f667efeff16735bed24e0c"
+  resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.3.18-alpha.0.tgz#e87a2449d9e2453766c0ea77539af359bf5a81ff"
-  integrity sha512-yVxKCHiDE4yoARDLZ3IqktUuWgNZoEVlmWo9rS8+bxPcsf5sw7Rj7TXTCMQX7F0cekMVtG+KkpH/8bFrqfR6OA==
+  integrity sha512-nKLhCdLxmBX+VY7LF6daH0/AItcHoQTmBB3tc0SP7y4OLcJZfBEYidoWqWJKCgdz6LScWWogLgbDIAC8t+LNzg==
   dependencies:
-    "@budibase/backend-core" "2.3.17-alpha.6"
+    "@budibase/backend-core" "2.3.18-alpha.0"
-    "@budibase/types" "2.3.17-alpha.6"
+    "@budibase/types" "2.3.18-alpha.0"
     "@koa/router" "8.0.8"
     bull "4.10.1"
     joi "17.6.0"
@@ -1424,10 +1424,10 @@
     svelte-apexcharts "^1.0.2"
     svelte-flatpickr "^3.1.0"
 
-"@budibase/types@2.3.17-alpha.6":
+"@budibase/types@2.3.18-alpha.0":
-  version "2.3.17-alpha.6"
+  version "2.3.18-alpha.0"
-  resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.3.17-alpha.6.tgz#85b148334312a41cfbf01b6a20417b5e20485f3f"
+  resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.3.18-alpha.0.tgz#14480e760c9e7931e884e9e0f8b1d5dd7e5d91c9"
-  integrity sha512-BgPvLdNQKJSnJmHNo1OfKSHEeVhdTwcNSr2cwHjUpJk394rJiZfsOV7it8M9dLtAtpdsNR3ns7L6biW+pfjYoQ==
+  integrity sha512-d+OcW2sNYw7VthMGrOBRY2Bz6iPQVWOnJ94XfYlBRJVIoYwBgudbYkOXPz/vQmHyjSUQFobrvs6UDeZ/3VJTaA==
 
 "@bull-board/api@3.7.0":
   version "3.7.0"

packages/string-templates/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/string-templates",
-  "version": "2.3.17-alpha.6",
+  "version": "2.3.18-alpha.0",
   "description": "Handlebars wrapper for Budibase templating.",
   "main": "src/index.cjs",
   "module": "dist/bundle.mjs",

packages/types/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/types",
-  "version": "2.3.17-alpha.6",
+  "version": "2.3.18-alpha.0",
   "description": "Budibase types",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",

packages/worker/jest.config.ts

@@ -12,24 +12,19 @@ const config: Config.InitialOptions = {
   transform: {
     "^.+\\.ts?$": "@swc/jest",
   },
-}
+  moduleNameMapper: {
-
-if (!process.env.CI) {
-  // use sources when not in CI
-  config.moduleNameMapper = {
     "@budibase/backend-core/(.*)": "<rootDir>/../backend-core/$1",
     "@budibase/backend-core": "<rootDir>/../backend-core/src",
     "@budibase/types": "<rootDir>/../types/src",
-  }
+  },
-  // add pro sources if they exist
+}
-  if (fs.existsSync("../../../budibase-pro")) {
+
-    config.moduleNameMapper["@budibase/pro/(.*)"] =
+// add pro sources if they exist
-      "<rootDir>/../../../budibase-pro/packages/pro/$1"
+if (fs.existsSync("../../../budibase-pro")) {
-    config.moduleNameMapper["@budibase/pro"] =
+  config.moduleNameMapper["@budibase/pro/(.*)"] =
-      "<rootDir>/../../../budibase-pro/packages/pro/src"
+    "<rootDir>/../../../budibase-pro/packages/pro/$1"
-  }
+  config.moduleNameMapper["@budibase/pro"] =
-} else {
+    "<rootDir>/../../../budibase-pro/packages/pro/src"
-  console.log("Running tests with compiled dependency sources")
 }
 
 export default config

packages/worker/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/worker",
   "email": "hi@budibase.com",
-  "version": "2.3.17-alpha.6",
+  "version": "2.3.18-alpha.0",
   "description": "Budibase background service",
   "main": "src/index.ts",
   "repository": {
@@ -22,7 +22,7 @@
     "build:docker": "docker build . -t worker-service --label version=$BUDIBASE_RELEASE_VERSION",
     "dev:stack:init": "node ./scripts/dev/manage.js init",
     "dev:builder": "npm run dev:stack:init && nodemon",
-    "test": "jest --coverage --maxWorkers=2",
+    "test": "jest --coverage --runInBand",
     "test:watch": "jest --watch",
     "env:multi:enable": "node scripts/multiTenancy.js enable",
     "env:multi:disable": "node scripts/multiTenancy.js disable",
@@ -36,10 +36,10 @@
   "author": "Budibase",
   "license": "GPL-3.0",
   "dependencies": {
-    "@budibase/backend-core": "2.3.17-alpha.6",
+    "@budibase/backend-core": "2.3.18-alpha.0",
-    "@budibase/pro": "2.3.17-alpha.6",
+    "@budibase/pro": "2.3.18-alpha.0",
-    "@budibase/string-templates": "2.3.17-alpha.6",
+    "@budibase/string-templates": "2.3.18-alpha.0",
-    "@budibase/types": "2.3.17-alpha.6",
+    "@budibase/types": "2.3.18-alpha.0",
     "@koa/router": "8.0.8",
     "@sentry/node": "6.17.7",
     "@techpass/passport-openidconnect": "0.3.2",

packages/worker/src/utilities/redis.ts

@@ -54,6 +54,8 @@ export async function init() {
 export async function shutdown() {
   if (pwResetClient) await pwResetClient.finish()
   if (invitationClient) await invitationClient.finish()
+  // shutdown core clients
+  await redis.clients.shutdown()
   console.log("Redis shutdown")
 }
 

packages/worker/yarn.lock

@@ -475,14 +475,14 @@
   resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
   integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==
 
-"@budibase/backend-core@2.3.17-alpha.6":
+"@budibase/backend-core@2.3.18-alpha.0":
-  version "2.3.17-alpha.6"
+  version "2.3.18-alpha.0"
-  resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-2.3.17-alpha.6.tgz#b083b3899d435694105d37a1ec817ec6cdc9bb07"
+  resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-2.3.18-alpha.0.tgz#c0a64a150c1fef9cc69f95f0aece4e857d64438d"
-  integrity sha512-8ljXZnK6Db3Mexk+MyIsrQBFFu6aV3eRtiw5pwHl9BobxR+6s79YmUgLfjZCfpgzMXqnT73FnV7g8K7KPTNC3g==
+  integrity sha512-ugD+WMoFwpXm+moSLHUgaBOu4XpX0+5UhmMWcNeRtH0Yd9GpDh2QzwtoN8BtXq8k5gkVEyoNSz+6oxKfNkNVdQ==
   dependencies:
     "@budibase/nano" "10.1.1"
     "@budibase/pouchdb-replication-stream" "1.2.10"
-    "@budibase/types" "2.3.17-alpha.6"
+    "@budibase/types" "2.3.18-alpha.0"
     "@shopify/jest-koa-mocks" "5.0.1"
     "@techpass/passport-openidconnect" "0.3.2"
     aws-cloudfront-sign "2.2.0"
@@ -539,13 +539,13 @@
     pouchdb-promise "^6.0.4"
     through2 "^2.0.0"
 
-"@budibase/pro@2.3.17-alpha.6":
+"@budibase/pro@2.3.18-alpha.0":
-  version "2.3.17-alpha.6"
+  version "2.3.18-alpha.0"
-  resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.3.17-alpha.6.tgz#2d24afa48003ff5928f667efeff16735bed24e0c"
+  resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.3.18-alpha.0.tgz#e87a2449d9e2453766c0ea77539af359bf5a81ff"
-  integrity sha512-yVxKCHiDE4yoARDLZ3IqktUuWgNZoEVlmWo9rS8+bxPcsf5sw7Rj7TXTCMQX7F0cekMVtG+KkpH/8bFrqfR6OA==
+  integrity sha512-nKLhCdLxmBX+VY7LF6daH0/AItcHoQTmBB3tc0SP7y4OLcJZfBEYidoWqWJKCgdz6LScWWogLgbDIAC8t+LNzg==
   dependencies:
-    "@budibase/backend-core" "2.3.17-alpha.6"
+    "@budibase/backend-core" "2.3.18-alpha.0"
-    "@budibase/types" "2.3.17-alpha.6"
+    "@budibase/types" "2.3.18-alpha.0"
     "@koa/router" "8.0.8"
     bull "4.10.1"
     joi "17.6.0"
@@ -553,10 +553,10 @@
     lru-cache "^7.14.1"
     node-fetch "^2.6.1"
 
-"@budibase/types@2.3.17-alpha.6":
+"@budibase/types@2.3.18-alpha.0":
-  version "2.3.17-alpha.6"
+  version "2.3.18-alpha.0"
-  resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.3.17-alpha.6.tgz#85b148334312a41cfbf01b6a20417b5e20485f3f"
+  resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.3.18-alpha.0.tgz#14480e760c9e7931e884e9e0f8b1d5dd7e5d91c9"
-  integrity sha512-BgPvLdNQKJSnJmHNo1OfKSHEeVhdTwcNSr2cwHjUpJk394rJiZfsOV7it8M9dLtAtpdsNR3ns7L6biW+pfjYoQ==
+  integrity sha512-d+OcW2sNYw7VthMGrOBRY2Bz6iPQVWOnJ94XfYlBRJVIoYwBgudbYkOXPz/vQmHyjSUQFobrvs6UDeZ/3VJTaA==
 
 "@cspotcode/source-map-support@^0.8.0":
   version "0.8.1"