Merge pull request #10120 from Budibase/fix/budi-6657

Fix for user groups adding users to app user metadata tables

.github/workflows/budibase_ci.yml

@@ -56,11 +56,11 @@ jobs:
         run: yarn install:pro $BRANCH $BASE_BRANCH
       - run: yarn
       - run: yarn bootstrap
+      - run: yarn build:client
       - run: yarn test
-      - uses: codecov/codecov-action@v1
+      - uses: codecov/codecov-action@v3
         with:
           token: ${{ secrets.CODECOV_TOKEN }} # not required for public repos
-          files: ./packages/server/coverage/clover.xml,./packages/worker/coverage/clover.xml,./packages/backend-core/coverage/clover.xml
           name: codecov-umbrella
           verbose: true
 

package.json

@@ -25,6 +25,7 @@
     "setup": "node ./hosting/scripts/setup.js && yarn && yarn bootstrap && yarn build && yarn dev",
     "bootstrap": "lerna bootstrap && lerna link && ./scripts/link-dependencies.sh",
     "build": "lerna run build",
+    "build:client": "lerna run build --ignore @budibase/backend-core --ignore @budibase/worker --ignore @budibase/server --ignore @budibase/builder --ignore @budibase/cli --ignore @budibase/sdk",
     "build:dev": "lerna run prebuild && tsc --build --watch --preserveWatchOutput",
     "build:backend": "lerna run build --ignore @budibase/client --ignore @budibase/bbui --ignore @budibase/builder --ignore @budibase/cli",
     "build:sdk": "lerna run build:sdk",

packages/backend-core/scripts/test.sh

@@ -3,10 +3,10 @@
 if [[ -n $CI ]]
 then
   # --runInBand performs better in ci where resources are limited
-  echo "jest --coverage --runInBand"
+  echo "jest --coverage --runInBand --forceExit"
-  jest --coverage --runInBand
+  jest --coverage --runInBand --forceExit
 else
   # --maxWorkers performs better in development
   echo "jest --coverage"
   jest --coverage
-fi
+fi

packages/backend-core/src/index.ts

@@ -24,6 +24,7 @@ export * as redis from "./redis"
 export * as locks from "./redis/redlockImpl"
 export * as utils from "./utils"
 export * as errors from "./errors"
+export * as timers from "./timers"
 export { default as env } from "./environment"
 export { SearchParams } from "./db"
 // Add context to tenancy for backwards compatibility

packages/backend-core/src/queue/queue.ts

@@ -4,6 +4,7 @@ import { JobQueue } from "./constants"
 import InMemoryQueue from "./inMemoryQueue"
 import BullQueue from "bull"
 import { addListeners, StalledFn } from "./listeners"
+import * as timers from "../timers"
 
 const CLEANUP_PERIOD_MS = 60 * 1000
 let QUEUES: BullQueue.Queue[] | InMemoryQueue[] = []
@@ -29,8 +30,8 @@ export function createQueue<T>(
   }
   addListeners(queue, jobQueue, opts?.removeStalledCb)
   QUEUES.push(queue)
-  if (!cleanupInterval) {
+  if (!cleanupInterval && !env.isTest()) {
-    cleanupInterval = setInterval(cleanup, CLEANUP_PERIOD_MS)
+    cleanupInterval = timers.set(cleanup, CLEANUP_PERIOD_MS)
     // fire off an initial cleanup
     cleanup().catch(err => {
       console.error(`Unable to cleanup automation queue initially - ${err}`)
@@ -41,7 +42,7 @@ export function createQueue<T>(
 
 export async function shutdown() {
   if (cleanupInterval) {
-    clearInterval(cleanupInterval)
+    timers.clear(cleanupInterval)
   }
   if (QUEUES.length) {
     for (let queue of QUEUES) {

packages/backend-core/src/redis/redis.ts

@@ -8,6 +8,7 @@ import {
   SEPARATOR,
   SelectableDatabase,
 } from "./utils"
+import * as timers from "../timers"
 
 const RETRY_PERIOD_MS = 2000
 const STARTUP_TIMEOUT_MS = 5000
@@ -117,9 +118,9 @@ function waitForConnection(selectDb: number = DEFAULT_SELECT_DB) {
       return
     }
     // check if the connection is ready
-    const interval = setInterval(() => {
+    const interval = timers.set(() => {
       if (CONNECTED) {
-        clearInterval(interval)
+        timers.clear(interval)
         resolve("")
       }
     }, 500)

packages/backend-core/src/timers/index.ts

@@ -0,0 +1 @@
+export * from "./timers"

packages/backend-core/src/timers/timers.ts

@@ -0,0 +1,22 @@
+let intervals: NodeJS.Timeout[] = []
+
+export function set(callback: () => any, period: number) {
+  const interval = setInterval(callback, period)
+  intervals.push(interval)
+  return interval
+}
+
+export function clear(interval: NodeJS.Timeout) {
+  const idx = intervals.indexOf(interval)
+  if (idx !== -1) {
+    intervals.splice(idx, 1)
+  }
+  clearInterval(interval)
+}
+
+export function cleanup() {
+  for (let interval of intervals) {
+    clearInterval(interval)
+  }
+  intervals = []
+}

packages/backend-core/tests/jestEnv.ts

@@ -4,3 +4,4 @@ process.env.NODE_ENV = "jest"
 process.env.MOCK_REDIS = "1"
 process.env.LOG_LEVEL = process.env.LOG_LEVEL || "error"
 process.env.ENABLE_4XX_HTTP_LOGGING = "0"
+process.env.REDIS_PASSWORD = "budibase"

packages/backend-core/tests/jestSetup.ts

@@ -1,5 +1,6 @@
 import "./logging"
 import env from "../src/environment"
+import { cleanup } from "../src/timers"
 import { mocks, testContainerUtils } from "./utilities"
 
 // must explicitly enable fetch mock
@@ -21,3 +22,7 @@ if (!process.env.CI) {
 }
 
 testContainerUtils.setupEnv(env)
+
+afterAll(() => {
+  cleanup()
+})

packages/server/__mocks__/node-fetch.ts

@@ -4,6 +4,7 @@ module FetchMock {
   // @ts-ignore
   const fetch = jest.requireActual("node-fetch")
   let failCount = 0
+  let mockSearch = false
 
   const func = async (url: any, opts: any) => {
     function json(body: any, status = 200) {
@@ -69,7 +70,7 @@ module FetchMock {
         },
         404
       )
-    } else if (url.includes("_search")) {
+    } else if (mockSearch && url.includes("_search")) {
       const body = opts.body
       const parts = body.split("tableId:")
       let tableId
@@ -192,5 +193,9 @@ module FetchMock {
 
   func.Headers = fetch.Headers
 
+  func.mockSearch = () => {
+    mockSearch = true
+  }
+
   module.exports = func
 }

packages/server/jest.config.ts

@@ -43,6 +43,7 @@ const config: Config.InitialOptions = {
     "../backend-core/src/**/*.{js,ts}",
     // The use of coverage with couchdb view functions breaks tests
     "!src/db/views/staticViews.*",
+    "!src/**/*.spec.{js,ts}",
   ],
   coverageReporters: ["lcov", "json", "clover"],
 }

packages/server/package.json

@@ -14,7 +14,8 @@
     "build:dev": "yarn prebuild && tsc --build --watch --preserveWatchOutput",
     "debug": "yarn build && node --expose-gc --inspect=9222 dist/index.js",
     "postbuild": "copyfiles -u 1 src/**/*.svelte dist/ && copyfiles -u 1 src/**/*.hbs dist/ && copyfiles -u 1 src/**/*.json dist/",
-    "test": "bash scripts/test.sh",
+    "test": "NODE_OPTIONS=\"--max-old-space-size=4096\" bash scripts/test.sh",
+    "test:memory": "jest --maxWorkers=2 --logHeapUsage --forceExit",
     "test:watch": "jest --watch",
     "predocker": "copyfiles -f ../client/dist/budibase-client.js ../client/manifest.json client",
     "build:docker": "yarn run predocker && docker build . -t app-service --label version=$BUDIBASE_RELEASE_VERSION",
@@ -125,7 +126,7 @@
     "@babel/core": "7.17.4",
     "@babel/preset-env": "7.16.11",
     "@budibase/standard-components": "^0.9.139",
-    "@jest/test-sequencer": "24.9.0",
+    "@jest/test-sequencer": "29.5.0",
     "@swc/core": "^1.3.25",
     "@swc/jest": "^0.2.24",
     "@trendyol/jest-testcontainers": "^2.1.1",
@@ -134,7 +135,7 @@
     "@types/global-agent": "2.1.1",
     "@types/google-spreadsheet": "3.1.5",
     "@types/ioredis": "4.28.10",
-    "@types/jest": "27.5.1",
+    "@types/jest": "29.5.0",
     "@types/koa": "2.13.4",
     "@types/koa__router": "8.0.8",
     "@types/lodash": "4.14.180",
@@ -154,7 +155,7 @@
     "eslint": "6.8.0",
     "ioredis-mock": "7.2.0",
     "is-wsl": "2.2.0",
-    "jest": "28.1.1",
+    "jest": "29.5.0",
     "jest-openapi": "0.14.2",
     "jest-serial-runner": "^1.2.1",
     "nodemon": "2.0.15",
@@ -166,7 +167,7 @@
     "supertest": "6.2.2",
     "swagger-jsdoc": "6.1.0",
     "timekeeper": "2.2.0",
-    "ts-jest": "28.0.4",
+    "ts-jest": "29.0.5",
     "ts-node": "10.8.1",
     "tsconfig-paths": "4.0.0",
     "typescript": "4.7.3",

packages/server/scripts/test.sh

@@ -3,10 +3,10 @@
 if [[ -n $CI ]]
 then
   # --runInBand performs better in ci where resources are limited
-  echo "jest --coverage --runInBand"
+  echo "jest --coverage --runInBand --forceExit"
-  jest --coverage --runInBand
+  jest --coverage --runInBand --forceExit
 else
   # --maxWorkers performs better in development
   echo "jest --coverage --maxWorkers=2"
   jest --coverage --maxWorkers=2
-fi
+fi

packages/server/src/api/controllers/application.ts

@@ -44,7 +44,6 @@ import {
   Layout,
   Screen,
   MigrationType,
-  BBContext,
   Database,
   UserCtx,
 } from "@budibase/types"
@@ -74,14 +73,14 @@ async function getScreens() {
   ).rows.map((row: any) => row.doc)
 }
 
-function getUserRoleId(ctx: BBContext) {
+function getUserRoleId(ctx: UserCtx) {
   return !ctx.user?.role || !ctx.user.role._id
     ? roles.BUILTIN_ROLE_IDS.PUBLIC
     : ctx.user.role._id
 }
 
 function checkAppUrl(
-  ctx: BBContext,
+  ctx: UserCtx,
   apps: App[],
   url: string,
   currentAppId?: string
@@ -95,7 +94,7 @@ function checkAppUrl(
 }
 
 function checkAppName(
-  ctx: BBContext,
+  ctx: UserCtx,
   apps: App[],
   name: string,
   currentAppId?: string
@@ -160,7 +159,7 @@ async function addDefaultTables(db: Database) {
   await db.bulkDocs([...defaultDbDocs])
 }
 
-export async function fetch(ctx: BBContext) {
+export async function fetch(ctx: UserCtx) {
   const dev = ctx.query && ctx.query.status === AppStatus.DEV
   const all = ctx.query && ctx.query.status === AppStatus.ALL
   const apps = (await dbCore.getAllApps({ dev, all })) as App[]
@@ -185,7 +184,7 @@ export async function fetch(ctx: BBContext) {
   ctx.body = await checkAppMetadata(apps)
 }
 
-export async function fetchAppDefinition(ctx: BBContext) {
+export async function fetchAppDefinition(ctx: UserCtx) {
   const layouts = await getLayouts()
   const userRoleId = getUserRoleId(ctx)
   const accessController = new roles.AccessController()
@@ -231,7 +230,7 @@ export async function fetchAppPackage(ctx: UserCtx) {
   }
 }
 
-async function performAppCreate(ctx: BBContext) {
+async function performAppCreate(ctx: UserCtx) {
   const apps = (await dbCore.getAllApps({ dev: true })) as App[]
   const name = ctx.request.body.name,
     possibleUrl = ctx.request.body.url
@@ -360,7 +359,7 @@ async function creationEvents(request: any, app: App) {
   }
 }
 
-async function appPostCreate(ctx: BBContext, app: App) {
+async function appPostCreate(ctx: UserCtx, app: App) {
   const tenantId = tenancy.getTenantId()
   await migrations.backPopulateMigrations({
     type: MigrationType.APP,
@@ -391,7 +390,7 @@ async function appPostCreate(ctx: BBContext, app: App) {
   }
 }
 
-export async function create(ctx: BBContext) {
+export async function create(ctx: UserCtx) {
   const newApplication = await quotas.addApp(() => performAppCreate(ctx))
   await appPostCreate(ctx, newApplication)
   await cache.bustCache(cache.CacheKey.CHECKLIST)
@@ -401,7 +400,7 @@ export async function create(ctx: BBContext) {
 
 // This endpoint currently operates as a PATCH rather than a PUT
 // Thus name and url fields are handled only if present
-export async function update(ctx: BBContext) {
+export async function update(ctx: UserCtx) {
   const apps = (await dbCore.getAllApps({ dev: true })) as App[]
   // validation
   const name = ctx.request.body.name,
@@ -421,7 +420,7 @@ export async function update(ctx: BBContext) {
   ctx.body = app
 }
 
-export async function updateClient(ctx: BBContext) {
+export async function updateClient(ctx: UserCtx) {
   // Get current app version
   const db = context.getAppDB()
   const application = await db.get(DocumentType.APP_METADATA)
@@ -445,7 +444,7 @@ export async function updateClient(ctx: BBContext) {
   ctx.body = app
 }
 
-export async function revertClient(ctx: BBContext) {
+export async function revertClient(ctx: UserCtx) {
   // Check app can be reverted
   const db = context.getAppDB()
   const application = await db.get(DocumentType.APP_METADATA)
@@ -471,7 +470,7 @@ export async function revertClient(ctx: BBContext) {
   ctx.body = app
 }
 
-const unpublishApp = async (ctx: any) => {
+async function unpublishApp(ctx: UserCtx) {
   let appId = ctx.params.appId
   appId = dbCore.getProdAppID(appId)
 
@@ -487,7 +486,7 @@ const unpublishApp = async (ctx: any) => {
   return result
 }
 
-async function destroyApp(ctx: BBContext) {
+async function destroyApp(ctx: UserCtx) {
   let appId = ctx.params.appId
   appId = dbCore.getProdAppID(appId)
   const devAppId = dbCore.getDevAppID(appId)
@@ -515,12 +514,12 @@ async function destroyApp(ctx: BBContext) {
   return result
 }
 
-async function preDestroyApp(ctx: BBContext) {
+async function preDestroyApp(ctx: UserCtx) {
   const { rows } = await getUniqueRows([ctx.params.appId])
   ctx.rowCount = rows.length
 }
 
-async function postDestroyApp(ctx: BBContext) {
+async function postDestroyApp(ctx: UserCtx) {
   const rowCount = ctx.rowCount
   await groups.cleanupApp(ctx.params.appId)
   if (rowCount) {
@@ -528,7 +527,7 @@ async function postDestroyApp(ctx: BBContext) {
   }
 }
 
-export async function destroy(ctx: BBContext) {
+export async function destroy(ctx: UserCtx) {
   await preDestroyApp(ctx)
   const result = await destroyApp(ctx)
   await postDestroyApp(ctx)
@@ -536,7 +535,7 @@ export async function destroy(ctx: BBContext) {
   ctx.body = result
 }
 
-export const unpublish = async (ctx: BBContext) => {
+export async function unpublish(ctx: UserCtx) {
   const prodAppId = dbCore.getProdAppID(ctx.params.appId)
   const dbExists = await dbCore.dbExists(prodAppId)
 
@@ -551,7 +550,7 @@ export const unpublish = async (ctx: BBContext) => {
   ctx.status = 204
 }
 
-export async function sync(ctx: BBContext) {
+export async function sync(ctx: UserCtx) {
   const appId = ctx.params.appId
   try {
     ctx.body = await sdk.applications.syncApp(appId)

packages/server/src/api/controllers/row/utils.ts

@@ -62,10 +62,11 @@ export async function validate({
   }
   const errors: any = {}
   for (let fieldName of Object.keys(fetchedTable.schema)) {
-    const constraints = cloneDeep(fetchedTable.schema[fieldName].constraints)
+    const column = fetchedTable.schema[fieldName]
-    const type = fetchedTable.schema[fieldName].type
+    const constraints = cloneDeep(column.constraints)
+    const type = column.type
     // formulas shouldn't validated, data will be deleted anyway
-    if (type === FieldTypes.FORMULA) {
+    if (type === FieldTypes.FORMULA || column.autocolumn) {
       continue
     }
     // special case for options, need to always allow unselected (null)

packages/server/src/api/controllers/view/tests/__snapshots__/viewBuilder.spec.js.snap

@@ -1,48 +1,48 @@
 // Jest Snapshot v1, https://goo.gl/fbAQLP
 
 exports[`viewBuilder Calculate and filter creates a view with the calculation statistics and filter schema 1`] = `
-Object {
+{
   "map": "function (doc) {
-      if ((doc.tableId === \\"14f1c4e94d6a47b682ce89d35d4c78b0\\" && !(
+      if ((doc.tableId === "14f1c4e94d6a47b682ce89d35d4c78b0" && !(
-      doc[\\"myField\\"] === undefined ||
+      doc["myField"] === undefined ||
-      doc[\\"myField\\"] === null ||
+      doc["myField"] === null ||
-      doc[\\"myField\\"] === \\"\\" ||
+      doc["myField"] === "" ||
-      (Array.isArray(doc[\\"myField\\"]) && doc[\\"myField\\"].length === 0)
+      (Array.isArray(doc["myField"]) && doc["myField"].length === 0)
-  )) && (doc[\\"age\\"] > 17)) {
+  )) && (doc["age"] > 17)) {
-        emit(doc[\\"_id\\"], doc[\\"myField\\"]);
+        emit(doc["_id"], doc["myField"]);
       }
     }",
-  "meta": Object {
+  "meta": {
     "calculation": "stats",
     "field": "myField",
-    "filters": Array [
+    "filters": [
-      Object {
+      {
         "condition": "MT",
         "key": "age",
         "value": 17,
       },
     ],
     "groupBy": undefined,
-    "schema": Object {
+    "schema": {
-      "avg": Object {
+      "avg": {
         "type": "number",
       },
-      "count": Object {
+      "count": {
         "type": "number",
       },
-      "field": Object {
+      "field": {
         "type": "string",
       },
-      "max": Object {
+      "max": {
         "type": "number",
       },
-      "min": Object {
+      "min": {
         "type": "number",
       },
-      "sum": Object {
+      "sum": {
         "type": "number",
       },
-      "sumsqr": Object {
+      "sumsqr": {
         "type": "number",
       },
     },
@@ -53,42 +53,42 @@ Object {
 `;
 
 exports[`viewBuilder Calculate creates a view with the calculation statistics schema 1`] = `
-Object {
+{
   "map": "function (doc) {
-      if ((doc.tableId === \\"14f1c4e94d6a47b682ce89d35d4c78b0\\" && !(
+      if ((doc.tableId === "14f1c4e94d6a47b682ce89d35d4c78b0" && !(
-      doc[\\"myField\\"] === undefined ||
+      doc["myField"] === undefined ||
-      doc[\\"myField\\"] === null ||
+      doc["myField"] === null ||
-      doc[\\"myField\\"] === \\"\\" ||
+      doc["myField"] === "" ||
-      (Array.isArray(doc[\\"myField\\"]) && doc[\\"myField\\"].length === 0)
+      (Array.isArray(doc["myField"]) && doc["myField"].length === 0)
   )) ) {
-        emit(doc[\\"_id\\"], doc[\\"myField\\"]);
+        emit(doc["_id"], doc["myField"]);
       }
     }",
-  "meta": Object {
+  "meta": {
     "calculation": "stats",
     "field": "myField",
-    "filters": Array [],
+    "filters": [],
     "groupBy": undefined,
-    "schema": Object {
+    "schema": {
-      "avg": Object {
+      "avg": {
         "type": "number",
       },
-      "count": Object {
+      "count": {
         "type": "number",
       },
-      "field": Object {
+      "field": {
         "type": "string",
       },
-      "max": Object {
+      "max": {
         "type": "number",
       },
-      "min": Object {
+      "min": {
         "type": "number",
       },
-      "sum": Object {
+      "sum": {
         "type": "number",
       },
-      "sumsqr": Object {
+      "sumsqr": {
         "type": "number",
       },
     },
@@ -99,22 +99,22 @@ Object {
 `;
 
 exports[`viewBuilder Filter creates a view with multiple filters and conjunctions 1`] = `
-Object {
+{
   "map": "function (doc) {
-      if (doc.tableId === \\"14f1c4e94d6a47b682ce89d35d4c78b0\\" && (doc[\\"Name\\"] === \\"Test\\" || doc[\\"Yes\\"] > \\"Value\\")) {
+      if (doc.tableId === "14f1c4e94d6a47b682ce89d35d4c78b0" && (doc["Name"] === "Test" || doc["Yes"] > "Value")) {
-        emit(doc[\\"_id\\"], doc[\\"undefined\\"]);
+        emit(doc["_id"], doc["undefined"]);
       }
     }",
-  "meta": Object {
+  "meta": {
     "calculation": undefined,
     "field": undefined,
-    "filters": Array [
+    "filters": [
-      Object {
+      {
         "condition": "EQUALS",
         "key": "Name",
         "value": "Test",
       },
-      Object {
+      {
         "condition": "MT",
         "conjunction": "OR",
         "key": "Yes",
@@ -129,16 +129,16 @@ Object {
 `;
 
 exports[`viewBuilder Group By creates a view emitting the group by field 1`] = `
-Object {
+{
   "map": "function (doc) {
-      if (doc.tableId === \\"14f1c4e94d6a47b682ce89d35d4c78b0\\" ) {
+      if (doc.tableId === "14f1c4e94d6a47b682ce89d35d4c78b0" ) {
-        emit(doc[\\"age\\"], doc[\\"score\\"]);
+        emit(doc["age"], doc["score"]);
       }
     }",
-  "meta": Object {
+  "meta": {
     "calculation": undefined,
     "field": "score",
-    "filters": Array [],
+    "filters": [],
     "groupBy": "age",
     "schema": null,
     "tableId": "14f1c4e94d6a47b682ce89d35d4c78b0",

packages/server/src/api/routes/tests/__snapshots__/datasource.spec.ts.snap

@@ -1,21 +1,21 @@
 // Jest Snapshot v1, https://goo.gl/fbAQLP
 
 exports[`/datasources fetch returns all the datasources from the server 1`] = `
-Array [
+[
-  Object {
+  {
-    "config": Object {},
+    "config": {},
-    "entities": Array [
+    "entities": [
-      Object {
+      {
         "_id": "ta_users",
         "_rev": "1-2375e1bc58aeec664dc1b1f04ad43e44",
         "createdAt": "2020-01-01T00:00:00.000Z",
         "name": "Users",
         "primaryDisplay": "email",
-        "schema": Object {
+        "schema": {
-          "email": Object {
+          "email": {
-            "constraints": Object {
+            "constraints": {
               "email": true,
-              "length": Object {
+              "length": {
                 "maximum": "",
               },
               "presence": true,
@@ -25,8 +25,8 @@ Array [
             "name": "email",
             "type": "string",
           },
-          "firstName": Object {
+          "firstName": {
-            "constraints": Object {
+            "constraints": {
               "presence": false,
               "type": "string",
             },
@@ -34,8 +34,8 @@ Array [
             "name": "firstName",
             "type": "string",
           },
-          "lastName": Object {
+          "lastName": {
-            "constraints": Object {
+            "constraints": {
               "presence": false,
               "type": "string",
             },
@@ -43,9 +43,9 @@ Array [
             "name": "lastName",
             "type": "string",
           },
-          "roleId": Object {
+          "roleId": {
-            "constraints": Object {
+            "constraints": {
-              "inclusion": Array [
+              "inclusion": [
                 "ADMIN",
                 "POWER",
                 "BASIC",
@@ -58,9 +58,9 @@ Array [
             "name": "roleId",
             "type": "options",
           },
-          "status": Object {
+          "status": {
-            "constraints": Object {
+            "constraints": {
-              "inclusion": Array [
+              "inclusion": [
                 "active",
                 "inactive",
               ],
@@ -74,15 +74,15 @@ Array [
         },
         "type": "table",
         "updatedAt": "2020-01-01T00:00:00.000Z",
-        "views": Object {},
+        "views": {},
       },
     ],
     "name": "Budibase DB",
     "source": "BUDIBASE",
     "type": "budibase",
   },
-  Object {
+  {
-    "config": Object {},
+    "config": {},
     "createdAt": "2020-01-01T00:00:00.000Z",
     "name": "Test",
     "source": "POSTGRES",

packages/server/src/api/routes/tests/__snapshots__/view.spec.js.snap

@@ -1,8 +1,8 @@
 // Jest Snapshot v1, https://goo.gl/fbAQLP
 
 exports[`/views query returns data for the created view 1`] = `
-Array [
+[
-  Object {
+  {
     "avg": 2333.3333333333335,
     "count": 3,
     "group": null,
@@ -15,8 +15,8 @@ Array [
 `;
 
 exports[`/views query returns data for the created view using a group by 1`] = `
-Array [
+[
-  Object {
+  {
     "avg": 1500,
     "count": 2,
     "group": "One",
@@ -25,7 +25,7 @@ Array [
     "sum": 3000,
     "sumsqr": 5000000,
   },
-  Object {
+  {
     "avg": 4000,
     "count": 1,
     "group": "Two",

packages/server/src/api/routes/tests/appSync.spec.ts

@@ -0,0 +1,31 @@
+import * as setup from "./utilities"
+import { roles, db as dbCore } from "@budibase/backend-core"
+
+describe("/api/applications/:appId/sync", () => {
+  let request = setup.getRequest()
+  let config = setup.getConfig()
+  let app
+
+  afterAll(setup.afterAll)
+
+  beforeAll(async () => {
+    app = await config.init()
+    // create some users which we will use throughout the tests
+    await config.createUser({
+      email: "sync1@test.com",
+      roles: {
+        [app._id!]: roles.BUILTIN_ROLE_IDS.BASIC,
+      },
+    })
+  })
+
+  async function getUserMetadata() {
+    const { rows } = await config.searchRows(dbCore.InternalTable.USER_METADATA)
+    return rows
+  }
+
+  it("make sure that user metadata is correctly sync'd", async () => {
+    const rows = await getUserMetadata()
+    expect(rows.length).toBe(1)
+  })
+})

packages/server/src/api/routes/tests/internalSearch.spec.js

@@ -1,6 +1,7 @@
+const fetch = require("node-fetch")
+fetch.mockSearch()
 const search = require("../../controllers/row/internalSearch")
 // this will be mocked out for _search endpoint
-const fetch = require("node-fetch")
 const PARAMS = {
   tableId: "ta_12345679abcdef",
   version: "1",
@@ -20,7 +21,7 @@ function checkLucene(resp, expected, params = PARAMS) {
     expect(json.bookmark).toBe(PARAMS.bookmark)
   }
   expect(json.include_docs).toBe(true)
-  expect(json.q).toBe(`(${expected}) AND tableId:"${params.tableId}"`)
+  expect(json.q).toBe(`${expected} AND tableId:"${params.tableId}"`)
   expect(json.limit).toBe(params.limit || 50)
 }
 
@@ -59,7 +60,7 @@ describe("internal search", () => {
         "column": "1",
       }
     }, PARAMS)
-    checkLucene(response, `column:"2" OR !column:"1"`)
+    checkLucene(response, `(column:"2" OR !column:"1")`)
   })
 
   it("test AND query", async () => {
@@ -71,7 +72,7 @@ describe("internal search", () => {
         "column": "1",
       }
     }, PARAMS)
-    checkLucene(response, `*:* AND column:"2" AND !column:"1"`)
+    checkLucene(response, `(*:* AND column:"2" AND !column:"1")`)
   })
 
   it("test pagination query", async () => {
@@ -132,7 +133,7 @@ describe("internal search", () => {
         "colArr": [1, 2, 3],
       },
     }, PARAMS)
-    checkLucene(response, `*:* AND column:a AND colArr:(1 AND 2 AND 3)`, PARAMS)
+    checkLucene(response, `(*:* AND column:a AND colArr:(1 AND 2 AND 3))`, PARAMS)
   })
 
   it("test multiple of same column", async () => {
@@ -144,7 +145,7 @@ describe("internal search", () => {
         "3:column": "c",
       },
     }, PARAMS)
-    checkLucene(response, `column:"a" OR column:"b" OR column:"c"`, PARAMS)
+    checkLucene(response, `(column:"a" OR column:"b" OR column:"c")`, PARAMS)
   })
 
   it("check a weird case for lucene building", async () => {
@@ -191,6 +192,6 @@ describe("internal search", () => {
       expect(json.bookmark).toBe(PARAMS.bookmark)
     }
     expect(json.include_docs).toBe(true)
-    expect(json.q).toBe(`(*:* AND column:"1") AND tableId:${PARAMS.tableId}`)
+    expect(json.q).toBe(`*:* AND column:"1" AND tableId:${PARAMS.tableId}`)
   })
 })

packages/server/src/app.ts

@@ -27,7 +27,7 @@ import * as api from "./api"
 import * as automations from "./automations"
 import { Thread } from "./threads"
 import * as redis from "./utilities/redis"
-import { events, logging, middleware } from "@budibase/backend-core"
+import { events, logging, middleware, timers } from "@budibase/backend-core"
 import { initialise as initialiseWebsockets } from "./websocket"
 import { startup } from "./startup"
 const Sentry = require("@sentry/node")
@@ -84,6 +84,7 @@ server.on("close", async () => {
   }
   shuttingDown = true
   console.log("Server Closed")
+  timers.cleanup()
   await automations.shutdown()
   await redis.shutdown()
   events.shutdown()

packages/server/src/automations/automationUtils.ts

@@ -23,7 +23,7 @@ import { LoopStep, LoopStepType, LoopInput } from "../definitions/automations"
  * @returns {object} The inputs object which has had all the various types supported by this function converted to their
  * primitive types.
  */
-export function cleanInputValues(inputs: Record<string, any>, schema: any) {
+export function cleanInputValues(inputs: Record<string, any>, schema?: any) {
   if (schema == null) {
     return inputs
   }

packages/server/src/automations/tests/automation.spec.js

@@ -1,84 +0,0 @@
-jest.mock("../../threads/automation")
-jest.mock("../../utilities/redis", () => ({
-  init: jest.fn(),
-  checkTestFlag: () => {
-    return false
-  },
-}))
-
-jest.spyOn(global.console, "error")
-
-require("../../environment")
-const automation = require("../index")
-const thread = require("../../threads/automation")
-const triggers = require("../triggers")
-const { basicAutomation } = require("../../tests/utilities/structures")
-const { wait } = require("../../utilities")
-const { makePartial } = require("../../tests/utilities")
-const { cleanInputValues } = require("../automationUtils")
-const setup = require("./utilities")
-
-describe("Run through some parts of the automations system", () => {
-  let config = setup.getConfig()
-
-  beforeAll(async () => {
-    await automation.init()
-    await config.init()
-  })
-
-  afterAll(setup.afterAll)
-
-  it("should be able to init in builder", async () => {
-      await triggers.externalTrigger(basicAutomation(), { a: 1, appId: config.appId })
-      await wait(100)
-      expect(thread.execute).toHaveBeenCalled()
-  })
-
-  it("should check coercion", async () => {
-    const table = await config.createTable()
-    const automation = basicAutomation()
-    automation.definition.trigger.inputs.tableId = table._id
-    automation.definition.trigger.stepId = "APP"
-    automation.definition.trigger.inputs.fields = { a: "number" }
-    await triggers.externalTrigger(automation, {
-      appId: config.getAppId(),
-      fields: {
-        a: "1"
-      }
-    })
-    await wait(100)
-    expect(thread.execute).toHaveBeenCalledWith(makePartial({
-      data: {
-        event: {
-          fields: {
-            a: 1
-          }
-        }
-      }
-    }), expect.any(Function))
-  })
-
-  it("should be able to clean inputs with the utilities", () => {
-    // can't clean without a schema
-    let output = cleanInputValues({a: "1"})
-    expect(output.a).toBe("1")
-    output = cleanInputValues({a: "1", b: "true", c: "false", d: 1, e: "help"}, {
-      properties: {
-        a: {
-          type: "number",
-        },
-        b: {
-          type: "boolean",
-        },
-        c: {
-          type: "boolean",
-        }
-      }
-    })
-    expect(output.a).toBe(1)
-    expect(output.b).toBe(true)
-    expect(output.c).toBe(false)
-    expect(output.d).toBe(1)
-    expect(output.e).toBe("help")
-  })
-})

packages/server/src/automations/tests/automation.spec.ts

@@ -0,0 +1,99 @@
+jest.mock("../../threads/automation")
+jest.mock("../../utilities/redis", () => ({
+  init: jest.fn(),
+  checkTestFlag: () => {
+    return false
+  },
+}))
+
+jest.spyOn(global.console, "error")
+
+import "../../environment"
+import * as automation from "../index"
+import * as thread from "../../threads/automation"
+import * as triggers from "../triggers"
+import { basicAutomation } from "../../tests/utilities/structures"
+import { wait } from "../../utilities"
+import { makePartial } from "../../tests/utilities"
+import { cleanInputValues } from "../automationUtils"
+import * as setup from "./utilities"
+import { Automation } from "@budibase/types"
+
+describe("Run through some parts of the automations system", () => {
+  let config = setup.getConfig()
+
+  beforeAll(async () => {
+    await automation.init()
+    await config.init()
+  })
+
+  afterAll(async () => {
+    await automation.shutdown()
+    setup.afterAll()
+  })
+
+  it("should be able to init in builder", async () => {
+    const automation: Automation = {
+      ...basicAutomation(),
+      appId: config.appId,
+    }
+    const fields: any = { a: 1, appId: config.appId }
+    await triggers.externalTrigger(automation, fields)
+    await wait(100)
+    expect(thread.execute).toHaveBeenCalled()
+  })
+
+  it("should check coercion", async () => {
+    const table = await config.createTable()
+    const automation: any = basicAutomation()
+    automation.definition.trigger.inputs.tableId = table._id
+    automation.definition.trigger.stepId = "APP"
+    automation.definition.trigger.inputs.fields = { a: "number" }
+    const fields: any = {
+      appId: config.getAppId(),
+      fields: {
+        a: "1",
+      },
+    }
+    await triggers.externalTrigger(automation, fields)
+    await wait(100)
+    expect(thread.execute).toHaveBeenCalledWith(
+      makePartial({
+        data: {
+          event: {
+            fields: {
+              a: 1,
+            },
+          },
+        },
+      }),
+      expect.any(Function)
+    )
+  })
+
+  it("should be able to clean inputs with the utilities", () => {
+    // can't clean without a schema
+    let output = cleanInputValues({ a: "1" })
+    expect(output.a).toBe("1")
+    output = cleanInputValues(
+      { a: "1", b: "true", c: "false", d: 1, e: "help" },
+      {
+        properties: {
+          a: {
+            type: "number",
+          },
+          b: {
+            type: "boolean",
+          },
+          c: {
+            type: "boolean",
+          },
+        },
+      }
+    )
+    expect(output.a).toBe(1)
+    expect(output.b).toBe(true)
+    expect(output.c).toBe(false)
+    expect(output.d).toBe(1)
+  })
+})

packages/server/src/integration-test/postgres.spec.ts

@@ -1,3 +1,6 @@
+import fetch from "node-fetch"
+// @ts-ignore
+fetch.mockSearch()
 import {
   generateMakeRequest,
   MakeRequestResponse,
@@ -16,6 +19,7 @@ import _ from "lodash"
 import { generator } from "@budibase/backend-core/tests"
 import { utils } from "@budibase/backend-core"
 import { GenericContainer } from "testcontainers"
+import { generateRowIdField } from "../integrations/utils"
 
 const config = setup.getConfig()!
 
@@ -80,16 +84,10 @@ describe("row api - postgres", () => {
             name: "id",
             type: FieldType.AUTO,
             autocolumn: true,
-            constraints: {
-              presence: true,
-            },
           },
           title: {
             name: "title",
             type: FieldType.STRING,
-            constraints: {
-              presence: true,
-            },
           },
         },
         sourceId: postgresDatasource._id,
@@ -121,16 +119,10 @@ describe("row api - postgres", () => {
           name: "id",
           type: FieldType.AUTO,
           autocolumn: true,
-          constraints: {
-            presence: true,
-          },
         },
         name: {
           name: "name",
           type: FieldType.STRING,
-          constraints: {
-            presence: true,
-          },
         },
         description: {
           name: "description",
@@ -144,7 +136,6 @@ describe("row api - postgres", () => {
           type: FieldType.LINK,
           constraints: {
             type: "array",
-            presence: false,
           },
           fieldName: oneToManyRelationshipInfo.fieldName,
           name: "oneToManyRelation",
@@ -156,7 +147,6 @@ describe("row api - postgres", () => {
           type: FieldType.LINK,
           constraints: {
             type: "array",
-            presence: false,
           },
           fieldName: manyToOneRelationshipInfo.fieldName,
           name: "manyToOneRelation",
@@ -168,7 +158,6 @@ describe("row api - postgres", () => {
           type: FieldType.LINK,
           constraints: {
             type: "array",
-            presence: false,
           },
           fieldName: manyToManyRelationshipInfo.fieldName,
           name: "manyToManyRelation",
@@ -309,9 +298,6 @@ describe("row api - postgres", () => {
         id: {
           name: "id",
           type: FieldType.AUTO,
-          constraints: {
-            presence: true,
-          },
         },
       },
       sourceId: postgresDatasource._id,
@@ -921,47 +907,55 @@ describe("row api - postgres", () => {
             foreignRows,
             x => x.relationshipType
           )
-          expect(res.body).toEqual({
+          const m2mFieldName = manyToManyRelationshipInfo.fieldName,
-            ...rowData,
+            o2mFieldName = oneToManyRelationshipInfo.fieldName,
-            [`fk_${oneToManyRelationshipInfo.table.name}_${oneToManyRelationshipInfo.fieldName}`]:
+            m2oFieldName = manyToOneRelationshipInfo.fieldName
-              foreignRowsByType[RelationshipTypes.ONE_TO_MANY][0].row.id,
+          const m2mRow1 = res.body[m2mFieldName].find(
-            [oneToManyRelationshipInfo.fieldName]: [
+            (row: Row) => row.id === 1
+          )
+          const m2mRow2 = res.body[m2mFieldName].find(
+            (row: Row) => row.id === 2
+          )
+          expect(m2mRow1).toEqual({
+            ...foreignRowsByType[RelationshipTypes.MANY_TO_MANY][0].row,
+            [m2mFieldName]: [
               {
-                ...foreignRowsByType[RelationshipTypes.ONE_TO_MANY][0].row,
+                _id: row._id,
-                _id: expect.any(String),
-                _rev: expect.any(String),
               },
             ],
-            [manyToOneRelationshipInfo.fieldName]: [
-              {
-                ...foreignRowsByType[RelationshipTypes.MANY_TO_ONE][0].row,
-                [`fk_${manyToOneRelationshipInfo.table.name}_${manyToOneRelationshipInfo.fieldName}`]:
-                  row.id,
-              },
-              {
-                ...foreignRowsByType[RelationshipTypes.MANY_TO_ONE][1].row,
-                [`fk_${manyToOneRelationshipInfo.table.name}_${manyToOneRelationshipInfo.fieldName}`]:
-                  row.id,
-              },
-              {
-                ...foreignRowsByType[RelationshipTypes.MANY_TO_ONE][2].row,
-                [`fk_${manyToOneRelationshipInfo.table.name}_${manyToOneRelationshipInfo.fieldName}`]:
-                  row.id,
-              },
-            ],
-            [manyToManyRelationshipInfo.fieldName]: [
-              {
-                ...foreignRowsByType[RelationshipTypes.MANY_TO_MANY][0].row,
-              },
-              {
-                ...foreignRowsByType[RelationshipTypes.MANY_TO_MANY][1].row,
-              },
-            ],
-            id: row.id,
-            tableId: row.tableId,
-            _id: expect.any(String),
-            _rev: expect.any(String),
           })
+          expect(m2mRow2).toEqual({
+            ...foreignRowsByType[RelationshipTypes.MANY_TO_MANY][1].row,
+            [m2mFieldName]: [
+              {
+                _id: row._id,
+              },
+            ],
+          })
+          expect(res.body[m2oFieldName]).toEqual([
+            {
+              ...foreignRowsByType[RelationshipTypes.MANY_TO_ONE][0].row,
+              [`fk_${manyToOneRelationshipInfo.table.name}_${manyToOneRelationshipInfo.fieldName}`]:
+                row.id,
+            },
+            {
+              ...foreignRowsByType[RelationshipTypes.MANY_TO_ONE][1].row,
+              [`fk_${manyToOneRelationshipInfo.table.name}_${manyToOneRelationshipInfo.fieldName}`]:
+                row.id,
+            },
+            {
+              ...foreignRowsByType[RelationshipTypes.MANY_TO_ONE][2].row,
+              [`fk_${manyToOneRelationshipInfo.table.name}_${manyToOneRelationshipInfo.fieldName}`]:
+                row.id,
+            },
+          ])
+          expect(res.body[o2mFieldName]).toEqual([
+            {
+              ...foreignRowsByType[RelationshipTypes.ONE_TO_MANY][0].row,
+              _id: expect.any(String),
+              _rev: expect.any(String),
+            },
+          ])
         })
       })
     })

packages/server/src/integrations/redis.ts

@@ -92,7 +92,7 @@ class RedisIntegration {
   }
 
   async disconnect() {
-    return this.client.disconnect()
+    return this.client.quit()
   }
 
   async redisContext(query: Function) {

packages/server/src/integrations/tests/googlesheets.spec.ts

@@ -39,6 +39,10 @@ describe("Google Sheets Integration", () => {
     config.setGoogleAuth("test")
   })
 
+  afterAll(async () => {
+    await config.end()
+  })
+
   beforeEach(async () => {
     integration = new GoogleSheetsIntegration.integration({
       spreadsheetId: "randomId",
@@ -99,8 +103,8 @@ describe("Google Sheets Integration", () => {
       })
     })
 
-    test("removing an existing field will not remove the data from the spreadsheet", async () => {
+    test("removing an existing field will remove the header from the google sheet", async () => {
-      await config.doInContext(structures.uuid(), async () => {
+      const sheet = await config.doInContext(structures.uuid(), async () => {
         const tableColumns = ["name"]
         const table = createBasicTable(structures.uuid(), tableColumns)
 
@@ -109,18 +113,14 @@ describe("Google Sheets Integration", () => {
         })
         sheetsByTitle[table.name] = sheet
         await integration.updateTable(table)
-
+        return sheet
-        expect(sheet.loadHeaderRow).toBeCalledTimes(1)
-        expect(sheet.setHeaderRow).toBeCalledTimes(1)
-        expect(sheet.setHeaderRow).toBeCalledWith([
-          "name",
-          "description",
-          "location",
-        ])
-
-        // No undefineds are sent
-        expect((sheet.setHeaderRow as any).mock.calls[0][0]).toHaveLength(3)
       })
+      expect(sheet.loadHeaderRow).toBeCalledTimes(1)
+      expect(sheet.setHeaderRow).toBeCalledTimes(1)
+      expect(sheet.setHeaderRow).toBeCalledWith(["name"])
+
+      // No undefined are sent
+      expect((sheet.setHeaderRow as any).mock.calls[0][0]).toHaveLength(1)
     })
   })
 })

packages/server/src/integrations/tests/redis.spec.ts

@@ -3,17 +3,17 @@ import { default as RedisIntegration } from "../redis"
 
 class TestConfiguration {
   integration: any
-  redis: any
 
   constructor(config: any = {}) {
     this.integration = new RedisIntegration.integration(config)
-    this.redis = new Redis({
+    // have to kill the basic integration before replacing it
+    this.integration.client.quit()
+    this.integration.client = new Redis({
       data: {
         test: "test",
         result: "1",
       },
     })
-    this.integration.client = this.redis
   }
 }
 
@@ -24,13 +24,17 @@ describe("Redis Integration", () => {
     config = new TestConfiguration()
   })
 
+  afterAll(() => {
+    config.integration.disconnect()
+  })
+
   it("calls the create method with the correct params", async () => {
     const body = {
       key: "key",
       value: "value",
     }
     await config.integration.create(body)
-    expect(await config.redis.get("key")).toEqual("value")
+    expect(await config.integration.client.get("key")).toEqual("value")
   })
 
   it("calls the read method with the correct params", async () => {
@@ -46,7 +50,7 @@ describe("Redis Integration", () => {
       key: "test",
     }
     await config.integration.delete(body)
-    expect(await config.redis.get(body.key)).toEqual(null)
+    expect(await config.integration.client.get(body.key)).toEqual(null)
   })
 
   it("calls the pipeline method with the correct params", async () => {

packages/server/src/middleware/currentapp.ts

@@ -10,9 +10,9 @@ import { generateUserMetadataID, isDevAppID } from "../db/utils"
 import { getCachedSelf } from "../utilities/global"
 import env from "../environment"
 import { isWebhookEndpoint } from "./utils"
-import { BBContext } from "@budibase/types"
+import { UserCtx } from "@budibase/types"
 
-export default async (ctx: BBContext, next: any) => {
+export default async (ctx: UserCtx, next: any) => {
   // try to get the appID from the request
   let requestAppId = await utils.getAppIdFromCtx(ctx)
   // get app cookie if it exists

packages/server/src/sdk/users/utils.ts

@@ -9,7 +9,10 @@ import { isEqual } from "lodash"
 
 export function combineMetadataAndUser(user: any, metadata: any) {
   // skip users with no access
-  if (user.roleId === rolesCore.BUILTIN_ROLE_IDS.PUBLIC) {
+  if (
+    user.roleId == null ||
+    user.roleId === rolesCore.BUILTIN_ROLE_IDS.PUBLIC
+  ) {
     return null
   }
   delete user._rev

packages/server/src/tests/jestEnv.ts

@@ -9,3 +9,4 @@ process.env.LOG_LEVEL = process.env.LOG_LEVEL || "error"
 process.env.ENABLE_4XX_HTTP_LOGGING = "0"
 process.env.MOCK_REDIS = "1"
 process.env.PLATFORM_URL = "http://localhost:10000"
+process.env.REDIS_PASSWORD = "budibase"

packages/server/src/tests/jestSetup.ts

@@ -1,6 +1,6 @@
 import "./logging"
 import env from "../environment"
-import { env as coreEnv } from "@budibase/backend-core"
+import { env as coreEnv, timers } from "@budibase/backend-core"
 import { testContainerUtils } from "@budibase/backend-core/tests"
 
 if (!process.env.DEBUG) {
@@ -17,3 +17,7 @@ if (!process.env.CI) {
 }
 
 testContainerUtils.setupEnv(env, coreEnv)
+
+afterAll(() => {
+  timers.cleanup()
+})

packages/server/src/tests/utilities/TestConfiguration.ts

@@ -46,6 +46,7 @@ import {
   Row,
   SourceName,
   Table,
+  SearchFilters,
 } from "@budibase/types"
 
 type DefaultUserValues = {
@@ -164,6 +165,8 @@ class TestConfiguration {
     }
     if (this.server) {
       this.server.close()
+    } else {
+      require("../../app").default.close()
     }
     if (this.allApps) {
       cleanup(this.allApps.map(app => app.appId))
@@ -568,6 +571,16 @@ class TestConfiguration {
     return this._req(null, { tableId }, controllers.row.fetch)
   }
 
+  async searchRows(tableId: string, searchParams: SearchFilters = {}) {
+    if (!tableId && this.table) {
+      tableId = this.table._id
+    }
+    const body = {
+      query: searchParams,
+    }
+    return this._req(body, { tableId }, controllers.row.search)
+  }
+
   // ROLE
 
   async createRole(config?: any) {

packages/server/src/tests/utilities/structures.ts

@@ -106,7 +106,7 @@ export function newAutomation({ steps, trigger }: any = {}) {
   return automation
 }
 
-export function basicAutomation() {
+export function basicAutomation(appId?: string) {
   return {
     name: "My Automation",
     screenId: "kasdkfldsafkl",
@@ -114,11 +114,23 @@ export function basicAutomation() {
     uiTree: {},
     definition: {
       trigger: {
+        stepId: AutomationTriggerStepId.APP,
+        name: "test",
+        tagline: "test",
+        icon: "test",
+        description: "test",
+        type: "trigger",
+        id: "test",
         inputs: {},
+        schema: {
+          inputs: {},
+          outputs: {},
+        },
       },
       steps: [],
     },
     type: "automation",
+    appId,
   }
 }
 

packages/server/src/utilities/global.ts

@@ -8,7 +8,7 @@ import {
 } from "@budibase/backend-core"
 import env from "../environment"
 import { groups } from "@budibase/pro"
-import { BBContext, ContextUser, User } from "@budibase/types"
+import { UserCtx, ContextUser, User, UserGroup } from "@budibase/types"
 
 export function updateAppRole(
   user: ContextUser,
@@ -43,33 +43,40 @@ export function updateAppRole(
 
 async function checkGroupRoles(
   user: ContextUser,
-  { appId }: { appId?: string } = {}
+  opts: { appId?: string; groups?: UserGroup[] } = {}
 ) {
   if (user.roleId && user.roleId !== roles.BUILTIN_ROLE_IDS.PUBLIC) {
     return user
   }
-  if (appId) {
+  if (opts.appId) {
-    user.roleId = await groups.getGroupRoleId(user as User, appId)
+    user.roleId = await groups.getGroupRoleId(user as User, opts.appId, {
+      groups: opts.groups,
+    })
+  }
+  // final fallback, simply couldn't find a role - user must be public
+  if (!user.roleId) {
+    user.roleId = roles.BUILTIN_ROLE_IDS.PUBLIC
   }
   return user
 }
 
 async function processUser(
   user: ContextUser,
-  { appId }: { appId?: string } = {}
+  opts: { appId?: string; groups?: UserGroup[] } = {}
 ) {
   if (user) {
     delete user.password
   }
-  user = await updateAppRole(user, { appId })
+  const appId = opts.appId || context.getAppId()
+  user = updateAppRole(user, { appId })
   if (!user.roleId && user?.userGroups?.length) {
-    user = await checkGroupRoles(user, { appId })
+    user = await checkGroupRoles(user, { appId, groups: opts?.groups })
   }
 
   return user
 }
 
-export async function getCachedSelf(ctx: BBContext, appId: string) {
+export async function getCachedSelf(ctx: UserCtx, appId: string) {
   // this has to be tenant aware, can't depend on the context to find it out
   // running some middlewares before the tenancy causes context to break
   const user = await cache.user.getUser(ctx.user?._id!)
@@ -90,6 +97,7 @@ export async function getGlobalUser(userId: string) {
 export async function getGlobalUsers(users?: ContextUser[]) {
   const appId = context.getAppId()
   const db = tenancy.getGlobalDB()
+  const allGroups = await groups.fetch()
   let globalUsers
   if (users) {
     const globalIds = users.map(user =>
@@ -118,7 +126,11 @@ export async function getGlobalUsers(users?: ContextUser[]) {
     return globalUsers
   }
 
-  return globalUsers.map(user => updateAppRole(user))
+  // pass in the groups, meaning we don't actually need to retrieve them for
+  // each user individually
+  return Promise.all(
+    globalUsers.map(user => processUser(user, { groups: allGroups }))
+  )
 }
 
 export async function getGlobalUsersFromMetadata(users: ContextUser[]) {

packages/worker/scripts/test.sh

@@ -3,10 +3,10 @@
 if [[ -n $CI ]]
 then
   # --runInBand performs better in ci where resources are limited
-  echo "jest --coverage --runInBand"
+  echo "jest --coverage --runInBand --forceExit"
-  jest --coverage --runInBand
+  jest --coverage --runInBand --forceExit
 else
   # --maxWorkers performs better in development
   echo "jest --coverage --maxWorkers=2"
   jest --coverage --maxWorkers=2
-fi
+fi

packages/worker/src/index.ts

@@ -21,6 +21,7 @@ import {
   middleware,
   queue,
   env as coreEnv,
+  timers,
 } from "@budibase/backend-core"
 db.init()
 import Koa from "koa"
@@ -91,6 +92,7 @@ server.on("close", async () => {
   }
   shuttingDown = true
   console.log("Server Closed")
+  timers.cleanup()
   await redis.shutdown()
   await events.shutdown()
   await queue.shutdown()

packages/worker/src/tests/TestConfiguration.ts

@@ -106,6 +106,8 @@ class TestConfiguration {
   async afterAll() {
     if (this.server) {
       await this.server.close()
+    } else {
+      await require("../index").default.close()
     }
   }
 

packages/worker/src/tests/jestEnv.ts

@@ -10,3 +10,4 @@ process.env.MINIO_SECRET_KEY = "test"
 process.env.PLATFORM_URL = "http://localhost:10000"
 process.env.INTERNAL_API_KEY = "tet"
 process.env.DISABLE_ACCOUNT_PORTAL = "0"
+process.env.REDIS_PASSWORD = "budibase"

packages/worker/src/tests/jestSetup.ts

@@ -2,7 +2,7 @@ import "./logging"
 
 import { mocks, testContainerUtils } from "@budibase/backend-core/tests"
 import env from "../environment"
-import { env as coreEnv } from "@budibase/backend-core"
+import { env as coreEnv, timers } from "@budibase/backend-core"
 
 // must explicitly enable fetch mock
 mocks.fetch.enable()
@@ -21,3 +21,7 @@ if (!process.env.CI) {
 }
 
 testContainerUtils.setupEnv(env, coreEnv)
+
+afterAll(() => {
+  timers.cleanup()
+})