From 651d50a064e12338aea8f7ef8d95898d33c316cb Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Wed, 31 May 2023 17:04:29 +0100
Subject: [PATCH 01/25] Updating datasource information endpoint to POST which
 allows sending up an unfinished/unsaved datasource for fetching information
 with. Also changing how verification and information endpoints work so that
 enrichment is used and therefore env vars can also be used.

---
 .../server/src/api/controllers/datasource.ts  | 42 +++++++++++--------
 .../src/api/controllers/plugin/index.ts       |  2 +-
 .../server/src/api/controllers/table/index.ts |  4 +-
 .../server/src/api/controllers/view/index.ts  |  4 +-
 packages/server/src/api/routes/datasource.ts  |  4 +-
 .../src/integration-test/postgres.spec.ts     | 10 ++---
 packages/server/src/sdk/plugins/plugins.ts    |  2 +-
 packages/types/src/api/web/app/datasource.ts  |  4 ++
 8 files changed, 41 insertions(+), 31 deletions(-)

diff --git a/packages/server/src/api/controllers/datasource.ts b/packages/server/src/api/controllers/datasource.ts
index 9852b01bc5..8fe0ab70da 100644
--- a/packages/server/src/api/controllers/datasource.ts
+++ b/packages/server/src/api/controllers/datasource.ts
@@ -21,6 +21,7 @@ import {
   CreateDatasourceRequest,
   VerifyDatasourceRequest,
   VerifyDatasourceResponse,
+  FetchDatasourceInfoRequest,
   FetchDatasourceInfoResponse,
   IntegrationBase,
   DatasourcePlus,
@@ -57,6 +58,21 @@ async function getConnector(
   return new Connector(datasource.config)
 }
 
+async function getAndMergeDatasource(datasource: Datasource) {
+  let existingDatasource: undefined | Datasource
+  if (datasource._id) {
+    existingDatasource = await sdk.datasources.get(datasource._id)
+  }
+  let enrichedDatasource = datasource
+  if (existingDatasource) {
+    enrichedDatasource = sdk.datasources.mergeConfigs(
+      datasource,
+      existingDatasource
+    )
+  }
+  return await sdk.datasources.enrich(enrichedDatasource)
+}
+
 async function buildSchemaHelper(datasource: Datasource) {
   const connector = (await getConnector(datasource)) as DatasourcePlus
   await connector.buildSchema(datasource._id!, datasource.entities!)
@@ -132,17 +148,7 @@ export async function verify(
   ctx: UserCtx<VerifyDatasourceRequest, VerifyDatasourceResponse>
 ) {
   const { datasource } = ctx.request.body
-  let existingDatasource: undefined | Datasource
-  if (datasource._id) {
-    existingDatasource = await sdk.datasources.get(datasource._id)
-  }
-  let enrichedDatasource = datasource
-  if (existingDatasource) {
-    enrichedDatasource = sdk.datasources.mergeConfigs(
-      datasource,
-      existingDatasource
-    )
-  }
+  const enrichedDatasource = await getAndMergeDatasource(datasource)
   const connector = await getConnector(enrichedDatasource)
   if (!connector.testConnection) {
     ctx.throw(400, "Connection information verification not supported")
@@ -156,11 +162,11 @@ export async function verify(
 }
 
 export async function information(
-  ctx: UserCtx<void, FetchDatasourceInfoResponse>
+  ctx: UserCtx<FetchDatasourceInfoRequest, FetchDatasourceInfoResponse>
 ) {
-  const datasourceId = ctx.params.datasourceId
-  const datasource = await sdk.datasources.get(datasourceId, { enriched: true })
-  const connector = (await getConnector(datasource)) as DatasourcePlus
+  const { datasource } = ctx.request.body
+  const enrichedDatasource = await getAndMergeDatasource(datasource)
+  const connector = (await getConnector(enrichedDatasource)) as DatasourcePlus
   if (!connector.getTableNames) {
     ctx.throw(400, "Table name fetching not supported by datasource")
   }
@@ -297,7 +303,7 @@ export async function update(ctx: UserCtx<any, UpdateDatasourceResponse>) {
   ctx.body = {
     datasource: await sdk.datasources.removeSecretSingle(datasource),
   }
-  builderSocket.emitDatasourceUpdate(ctx, datasource)
+  builderSocket?.emitDatasourceUpdate(ctx, datasource)
 }
 
 export async function save(
@@ -340,7 +346,7 @@ export async function save(
     response.error = schemaError
   }
   ctx.body = response
-  builderSocket.emitDatasourceUpdate(ctx, datasource)
+  builderSocket?.emitDatasourceUpdate(ctx, datasource)
 }
 
 async function destroyInternalTablesBySourceId(datasourceId: string) {
@@ -400,7 +406,7 @@ export async function destroy(ctx: UserCtx) {
 
   ctx.message = `Datasource deleted.`
   ctx.status = 200
-  builderSocket.emitDatasourceDeletion(ctx, datasourceId)
+  builderSocket?.emitDatasourceDeletion(ctx, datasourceId)
 }
 
 export async function find(ctx: UserCtx) {
diff --git a/packages/server/src/api/controllers/plugin/index.ts b/packages/server/src/api/controllers/plugin/index.ts
index 31a3468178..a8a451c98c 100644
--- a/packages/server/src/api/controllers/plugin/index.ts
+++ b/packages/server/src/api/controllers/plugin/index.ts
@@ -71,7 +71,7 @@ export async function create(ctx: any) {
 
     const doc = await pro.plugins.storePlugin(metadata, directory, source)
 
-    clientAppSocket.emit("plugins-update", { name, hash: doc.hash })
+    clientAppSocket?.emit("plugins-update", { name, hash: doc.hash })
     ctx.body = {
       message: "Plugin uploaded successfully",
       plugins: [doc],
diff --git a/packages/server/src/api/controllers/table/index.ts b/packages/server/src/api/controllers/table/index.ts
index 271f3e82fa..5ade77592a 100644
--- a/packages/server/src/api/controllers/table/index.ts
+++ b/packages/server/src/api/controllers/table/index.ts
@@ -78,7 +78,7 @@ export async function save(ctx: UserCtx) {
   ctx.eventEmitter &&
     ctx.eventEmitter.emitTable(`table:save`, appId, savedTable)
   ctx.body = savedTable
-  builderSocket.emitTableUpdate(ctx, savedTable)
+  builderSocket?.emitTableUpdate(ctx, savedTable)
 }
 
 export async function destroy(ctx: UserCtx) {
@@ -91,7 +91,7 @@ export async function destroy(ctx: UserCtx) {
   ctx.status = 200
   ctx.table = deletedTable
   ctx.body = { message: `Table ${tableId} deleted.` }
-  builderSocket.emitTableDeletion(ctx, tableId)
+  builderSocket?.emitTableDeletion(ctx, tableId)
 }
 
 export async function bulkImport(ctx: UserCtx) {
diff --git a/packages/server/src/api/controllers/view/index.ts b/packages/server/src/api/controllers/view/index.ts
index 28b0d0a81f..b194f3cb5e 100644
--- a/packages/server/src/api/controllers/view/index.ts
+++ b/packages/server/src/api/controllers/view/index.ts
@@ -58,7 +58,7 @@ export async function save(ctx: Ctx) {
   await handleViewEvents(existingTable.views[viewName], table.views[viewName])
 
   ctx.body = table.views[viewName]
-  builderSocket.emitTableUpdate(ctx, table)
+  builderSocket?.emitTableUpdate(ctx, table)
 }
 
 export async function calculationEvents(existingView: View, newView: View) {
@@ -127,7 +127,7 @@ export async function destroy(ctx: Ctx) {
   await events.view.deleted(view)
 
   ctx.body = view
-  builderSocket.emitTableUpdate(ctx, table)
+  builderSocket?.emitTableUpdate(ctx, table)
 }
 
 export async function exportView(ctx: Ctx) {
diff --git a/packages/server/src/api/routes/datasource.ts b/packages/server/src/api/routes/datasource.ts
index ac8256539c..f874d3c4aa 100644
--- a/packages/server/src/api/routes/datasource.ts
+++ b/packages/server/src/api/routes/datasource.ts
@@ -20,8 +20,8 @@ router
     authorized(permissions.BUILDER),
     datasourceController.verify
   )
-  .get(
-    "/api/datasources/:datasourceId/info",
+  .post(
+    "/api/datasources/info",
     authorized(permissions.BUILDER),
     datasourceController.information
   )
diff --git a/packages/server/src/integration-test/postgres.spec.ts b/packages/server/src/integration-test/postgres.spec.ts
index ce16ca4aba..f72cbda1ea 100644
--- a/packages/server/src/integration-test/postgres.spec.ts
+++ b/packages/server/src/integration-test/postgres.spec.ts
@@ -25,6 +25,7 @@ const config = setup.getConfig()!
 jest.setTimeout(30000)
 
 jest.unmock("pg")
+jest.mock("../websockets")
 
 describe("postgres integrations", () => {
   let makeRequest: MakeRequestResponse,
@@ -1055,13 +1056,12 @@ describe("postgres integrations", () => {
     })
   })
 
-  describe("GET /api/datasources/:datasourceId/info", () => {
+  describe("POST /api/datasources/info", () => {
     it("should fetch information about postgres datasource", async () => {
       const primaryName = primaryPostgresTable.name
-      const response = await makeRequest(
-        "get",
-        `/api/datasources/${postgresDatasource._id}/info`
-      )
+      const response = await makeRequest("post", "/api/datasources/info", {
+        datasource: postgresDatasource,
+      })
       expect(response.status).toBe(200)
       expect(response.body.tableNames).toBeDefined()
       expect(response.body.tableNames.indexOf(primaryName)).not.toBe(-1)
diff --git a/packages/server/src/sdk/plugins/plugins.ts b/packages/server/src/sdk/plugins/plugins.ts
index 02000410bf..f6c26ca339 100644
--- a/packages/server/src/sdk/plugins/plugins.ts
+++ b/packages/server/src/sdk/plugins/plugins.ts
@@ -36,6 +36,6 @@ export async function processUploaded(plugin: FileType, source?: PluginSource) {
   }
 
   const doc = await pro.plugins.storePlugin(metadata, directory, source)
-  clientAppSocket.emit("plugin-update", { name: doc.name, hash: doc.hash })
+  clientAppSocket?.emit("plugin-update", { name: doc.name, hash: doc.hash })
   return doc
 }
diff --git a/packages/types/src/api/web/app/datasource.ts b/packages/types/src/api/web/app/datasource.ts
index d15f65eb35..d692f17421 100644
--- a/packages/types/src/api/web/app/datasource.ts
+++ b/packages/types/src/api/web/app/datasource.ts
@@ -23,6 +23,10 @@ export interface VerifyDatasourceResponse {
   error?: string
 }
 
+export interface FetchDatasourceInfoRequest {
+  datasource: Datasource
+}
+
 export interface FetchDatasourceInfoResponse {
   tableNames: string[]
 }

From 249625ad1a2632a41fd53d81204511ef5576f6d4 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Wed, 31 May 2023 17:18:01 +0100
Subject: [PATCH 02/25] Updating backend tests to actually utilise the redis
 container which is spun up as part of the tests, doing away with the somewhat
 problematic ioredis-mock which was breaking some tests due to the addition of
 the changes for multi-dev collab.

---
 packages/backend-core/src/environment.ts  |  1 -
 packages/backend-core/src/index.ts        |  1 +
 packages/backend-core/src/redis/redis.ts  | 11 +----------
 packages/backend-core/src/redis/utils.ts  |  2 +-
 packages/server/package.json              |  1 -
 packages/server/src/utilities/redis.ts    |  8 ++++++--
 packages/server/src/websockets/builder.ts |  4 ++--
 packages/server/src/websockets/index.ts   | 16 ++++++++++------
 yarn.lock                                 | 14 ++------------
 9 files changed, 23 insertions(+), 35 deletions(-)

diff --git a/packages/backend-core/src/environment.ts b/packages/backend-core/src/environment.ts
index 9163dfeba6..325eac3335 100644
--- a/packages/backend-core/src/environment.ts
+++ b/packages/backend-core/src/environment.ts
@@ -97,7 +97,6 @@ const environment = {
   REDIS_URL: process.env.REDIS_URL || "localhost:6379",
   REDIS_PASSWORD: process.env.REDIS_PASSWORD,
   REDIS_CLUSTERED: process.env.REDIS_CLUSTERED,
-  MOCK_REDIS: process.env.MOCK_REDIS,
   MINIO_ACCESS_KEY: process.env.MINIO_ACCESS_KEY,
   MINIO_SECRET_KEY: process.env.MINIO_SECRET_KEY,
   AWS_REGION: process.env.AWS_REGION,
diff --git a/packages/backend-core/src/index.ts b/packages/backend-core/src/index.ts
index 40233b3827..7b98674788 100644
--- a/packages/backend-core/src/index.ts
+++ b/packages/backend-core/src/index.ts
@@ -21,6 +21,7 @@ export * as context from "./context"
 export * as cache from "./cache"
 export * as objectStore from "./objectStore"
 export * as redis from "./redis"
+export { Client as RedisClient } from "./redis"
 export * as locks from "./redis/redlockImpl"
 export * as utils from "./utils"
 export * as errors from "./errors"
diff --git a/packages/backend-core/src/redis/redis.ts b/packages/backend-core/src/redis/redis.ts
index 2d54b51a9f..60d3966e27 100644
--- a/packages/backend-core/src/redis/redis.ts
+++ b/packages/backend-core/src/redis/redis.ts
@@ -1,6 +1,6 @@
 import env from "../environment"
 // ioredis mock is all in memory
-const Redis = env.MOCK_REDIS ? require("ioredis-mock") : require("ioredis")
+import Redis from "ioredis"
 import {
   addDbPrefix,
   removeDbPrefix,
@@ -21,11 +21,6 @@ let CLIENTS: { [key: number]: any } = {}
 
 let CONNECTED = false
 
-// mock redis always connected
-if (env.MOCK_REDIS) {
-  CONNECTED = true
-}
-
 function pickClient(selectDb: number): any {
   return CLIENTS[selectDb]
 }
@@ -62,10 +57,6 @@ function init(selectDb = DEFAULT_SELECT_DB) {
   if (client && CONNECTED) {
     return
   }
-  // testing uses a single in memory client
-  if (env.MOCK_REDIS) {
-    CLIENTS[selectDb] = new Redis(getRedisOptions())
-  }
   // start the timer - only allowed 5 seconds to connect
   timeout = setTimeout(() => {
     if (!CONNECTED) {
diff --git a/packages/backend-core/src/redis/utils.ts b/packages/backend-core/src/redis/utils.ts
index f8b815824c..ab39acf215 100644
--- a/packages/backend-core/src/redis/utils.ts
+++ b/packages/backend-core/src/redis/utils.ts
@@ -95,7 +95,7 @@ export function getRedisOptions() {
     opts.port = port
     opts.password = password
   }
-  return { opts, host, port, redisProtocolUrl }
+  return { opts, host, port: parseInt(port), redisProtocolUrl }
 }
 
 export function addDbPrefix(db: string, key: string) {
diff --git a/packages/server/package.json b/packages/server/package.json
index cb53a1618e..472d221aa9 100644
--- a/packages/server/package.json
+++ b/packages/server/package.json
@@ -157,7 +157,6 @@
     "copyfiles": "2.4.1",
     "docker-compose": "0.23.17",
     "eslint": "6.8.0",
-    "ioredis-mock": "7.2.0",
     "is-wsl": "2.2.0",
     "jest": "29.5.0",
     "jest-openapi": "0.14.2",
diff --git a/packages/server/src/utilities/redis.ts b/packages/server/src/utilities/redis.ts
index ff1c863bf7..d7e9b16290 100644
--- a/packages/server/src/utilities/redis.ts
+++ b/packages/server/src/utilities/redis.ts
@@ -1,10 +1,14 @@
-import { redis } from "@budibase/backend-core"
+import { redis, RedisClient } from "@budibase/backend-core"
 import { getGlobalIDFromUserMetadataID } from "../db/utils"
 import { ContextUser } from "@budibase/types"
+import env from "../environment"
 
 const APP_DEV_LOCK_SECONDS = 600
 const AUTOMATION_TEST_FLAG_SECONDS = 60
-let devAppClient: any, debounceClient: any, flagClient: any, socketClient: any
+let devAppClient: RedisClient,
+  debounceClient: RedisClient,
+  flagClient: RedisClient,
+  socketClient: RedisClient
 
 // We need to maintain a duplicate client for socket.io pub/sub
 let socketSubClient: any
diff --git a/packages/server/src/websockets/builder.ts b/packages/server/src/websockets/builder.ts
index de8709129d..d856dd42ae 100644
--- a/packages/server/src/websockets/builder.ts
+++ b/packages/server/src/websockets/builder.ts
@@ -49,12 +49,12 @@ export default class BuilderSocket extends Socket {
 
   emitTableUpdate(ctx: any, table: Table) {
     this.io.in(ctx.appId).emit("table-change", { id: table._id, table })
-    gridSocket.emitTableUpdate(table)
+    gridSocket?.emitTableUpdate(table)
   }
 
   emitTableDeletion(ctx: any, id: string) {
     this.io.in(ctx.appId).emit("table-change", { id, table: null })
-    gridSocket.emitTableDeletion(id)
+    gridSocket?.emitTableDeletion(id)
   }
 
   emitDatasourceUpdate(ctx: any, datasource: Datasource) {
diff --git a/packages/server/src/websockets/index.ts b/packages/server/src/websockets/index.ts
index b74a8adfca..30693c2d25 100644
--- a/packages/server/src/websockets/index.ts
+++ b/packages/server/src/websockets/index.ts
@@ -3,15 +3,19 @@ import Koa from "koa"
 import ClientAppSocket from "./client"
 import GridSocket from "./grid"
 import BuilderSocket from "./builder"
+import env from "../environment"
 
-let clientAppSocket: ClientAppSocket
-let gridSocket: GridSocket
-let builderSocket: BuilderSocket
+let clientAppSocket: ClientAppSocket | undefined
+let gridSocket: GridSocket | undefined
+let builderSocket: BuilderSocket | undefined
 
 export const initialise = (app: Koa, server: http.Server) => {
-  clientAppSocket = new ClientAppSocket(app, server)
-  gridSocket = new GridSocket(app, server)
-  builderSocket = new BuilderSocket(app, server)
+  // currently we use ioredis-mock which doesn't work with the websocket system for testing
+  if (!env.isTest()) {
+    clientAppSocket = new ClientAppSocket(app, server)
+    gridSocket = new GridSocket(app, server)
+    builderSocket = new BuilderSocket(app, server)
+  }
 }
 
 export { clientAppSocket, gridSocket, builderSocket }
diff --git a/yarn.lock b/yarn.lock
index d932fa7437..9218b005b5 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -11846,7 +11846,7 @@ fecha@^4.2.0:
   resolved "https://registry.yarnpkg.com/fecha/-/fecha-4.2.3.tgz#4d9ccdbc61e8629b259fdca67e65891448d569fd"
   integrity sha512-OP2IUU6HeYKJi3i0z4A19kHMQoLVs4Hc+DPqqxI2h/DPZHTm/vjsfC6P0b4jCMy14XizLBqvndQ+UilD7707Jw==
 
-fengari-interop@^0.1.2, fengari-interop@^0.1.3:
+fengari-interop@^0.1.2:
   version "0.1.3"
   resolved "https://registry.yarnpkg.com/fengari-interop/-/fengari-interop-0.1.3.tgz#3ad37a90e7430b69b365441e9fc0ba168942a146"
   integrity sha512-EtZ+oTu3kEwVJnoymFPBVLIbQcCoy9uWCVnMA6h3M/RqHkUBsLYp29+RRHf9rKr6GwjubWREU1O7RretFIXjHw==
@@ -13893,16 +13893,6 @@ ioredis-mock@5.8.0:
     lodash "^4.17.21"
     standard-as-callback "^2.1.0"
 
-ioredis-mock@7.2.0:
-  version "7.2.0"
-  resolved "https://registry.yarnpkg.com/ioredis-mock/-/ioredis-mock-7.2.0.tgz#48f006c07ef7f1f93f75e60d8f9035fa46c4ef0a"
-  integrity sha512-xzABBG3NhfDBGxH1KX9n6vs7WGNn9lhcxMT3b+vjynVImxlUV+vOXU+tjGzSUnGmx4IYllA8RqbXN8z6ROMPVA==
-  dependencies:
-    fengari "^0.1.4"
-    fengari-interop "^0.1.3"
-    redis-commands "^1.7.0"
-    standard-as-callback "^2.1.0"
-
 ioredis@4.28.0:
   version "4.28.0"
   resolved "https://registry.yarnpkg.com/ioredis/-/ioredis-4.28.0.tgz#5a2be3f37ff2075e2332f280eaeb02ab4d9ff0d3"
@@ -21649,7 +21639,7 @@ redent@^3.0.0:
     indent-string "^4.0.0"
     strip-indent "^3.0.0"
 
-redis-commands@1.7.0, redis-commands@^1.7.0:
+redis-commands@1.7.0:
   version "1.7.0"
   resolved "https://registry.yarnpkg.com/redis-commands/-/redis-commands-1.7.0.tgz#15a6fea2d58281e27b1cd1acfb4b293e278c3a89"
   integrity sha512-nJWqw3bTFy21hX/CPKHth6sfhZbdiHP6bTawSgQBlKOVRG7EZkfHbbHwQJnrE4vsQf0CMNE+3gJ4Fmm16vdVlQ==

From 039e78379f14ccb3be8a71a2b26255e964a1ed44 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Wed, 31 May 2023 17:48:18 +0100
Subject: [PATCH 03/25] Fixing lint issue.

---
 packages/server/src/utilities/redis.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/server/src/utilities/redis.ts b/packages/server/src/utilities/redis.ts
index d8605a0e29..369c8b2e76 100644
--- a/packages/server/src/utilities/redis.ts
+++ b/packages/server/src/utilities/redis.ts
@@ -7,7 +7,7 @@ const APP_DEV_LOCK_SECONDS = 600
 const AUTOMATION_TEST_FLAG_SECONDS = 60
 let devAppClient: RedisClient,
   debounceClient: RedisClient,
-  flagClient: RedisClient,
+  flagClient: RedisClient
 
 // We need to maintain a duplicate client for socket.io pub/sub
 let socketClient: RedisClient

From 969ee52c29426547ce606cc7524d074990efe3ba Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Wed, 31 May 2023 18:08:29 +0100
Subject: [PATCH 04/25] Fixing merge conflict.

---
 packages/server/src/websockets/builder.ts | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/packages/server/src/websockets/builder.ts b/packages/server/src/websockets/builder.ts
index 6a0a401925..ff7c50ebf4 100644
--- a/packages/server/src/websockets/builder.ts
+++ b/packages/server/src/websockets/builder.ts
@@ -14,9 +14,9 @@ export default class BuilderSocket extends BaseSocket {
     super(app, server, "/socket/builder", [authorized(permissions.BUILDER)])
   }
 
-  async onConnect(socket: ?Socket) {
+  async onConnect(socket?: Socket) {
     // Initial identification of selected app
-    socket.on(BuilderSocketEvent.SelectApp, async (appId, callback) => {
+    socket?.on(BuilderSocketEvent.SelectApp, async (appId, callback) => {
       await this.joinRoom(socket, appId)
 
       // Reply with all users in current room

From 16e929bd81c72d07b55eac6cd036690e81cde04a Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Wed, 31 May 2023 18:09:15 +0100
Subject: [PATCH 05/25] Removing checks around websockets as redis properly
 available in testing now.

---
 packages/server/src/websockets/index.ts | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/packages/server/src/websockets/index.ts b/packages/server/src/websockets/index.ts
index 30693c2d25..9503041b15 100644
--- a/packages/server/src/websockets/index.ts
+++ b/packages/server/src/websockets/index.ts
@@ -10,12 +10,9 @@ let gridSocket: GridSocket | undefined
 let builderSocket: BuilderSocket | undefined
 
 export const initialise = (app: Koa, server: http.Server) => {
-  // currently we use ioredis-mock which doesn't work with the websocket system for testing
-  if (!env.isTest()) {
-    clientAppSocket = new ClientAppSocket(app, server)
-    gridSocket = new GridSocket(app, server)
-    builderSocket = new BuilderSocket(app, server)
-  }
+  clientAppSocket = new ClientAppSocket(app, server)
+  gridSocket = new GridSocket(app, server)
+  builderSocket = new BuilderSocket(app, server)
 }
 
 export { clientAppSocket, gridSocket, builderSocket }

From a3f9759a47722ceacfdcdc16ab0eb93dca386d34 Mon Sep 17 00:00:00 2001
From: Michael Drury <me@michaeldrury.co.uk>
Date: Thu, 1 Jun 2023 00:02:55 +0100
Subject: [PATCH 06/25] Reverting ioredis-mock removal for test cases - will
 come back to this in future as it currently breaks pro tests.

---
 packages/backend-core/src/environment.ts      |  1 +
 packages/backend-core/src/redis/redis.ts      | 21 +++++++++++++----
 packages/server/package.json                  |  7 +++---
 .../server/src/api/routes/tests/view.spec.js  |  7 +++---
 packages/server/src/utilities/redis.ts        |  4 +++-
 packages/server/src/websockets/index.ts       |  9 +++++---
 yarn.lock                                     | 23 ++++++++++++++++++-
 7 files changed, 56 insertions(+), 16 deletions(-)

diff --git a/packages/backend-core/src/environment.ts b/packages/backend-core/src/environment.ts
index 325eac3335..eab8cd4c45 100644
--- a/packages/backend-core/src/environment.ts
+++ b/packages/backend-core/src/environment.ts
@@ -128,6 +128,7 @@ const environment = {
   PLUGIN_BUCKET_NAME:
     process.env.PLUGIN_BUCKET_NAME || DefaultBucketName.PLUGINS,
   USE_COUCH: process.env.USE_COUCH || true,
+  MOCK_REDIS: process.env.MOCK_REDIS,
   DEFAULT_LICENSE: process.env.DEFAULT_LICENSE,
   SERVICE: process.env.SERVICE || "budibase",
   LOG_LEVEL: process.env.LOG_LEVEL || "info",
diff --git a/packages/backend-core/src/redis/redis.ts b/packages/backend-core/src/redis/redis.ts
index 48057dec0b..636544897d 100644
--- a/packages/backend-core/src/redis/redis.ts
+++ b/packages/backend-core/src/redis/redis.ts
@@ -1,6 +1,7 @@
 import env from "../environment"
-// ioredis mock is all in memory
 import Redis from "ioredis"
+// ioredis mock is all in memory
+const MockRedis = require("ioredis-mock")
 import {
   addDbPrefix,
   removeDbPrefix,
@@ -18,9 +19,14 @@ const DEFAULT_SELECT_DB = SelectableDatabase.DEFAULT
 // for testing just generate the client once
 let CLOSED = false
 let CLIENTS: { [key: number]: any } = {}
-
+0
 let CONNECTED = false
 
+// mock redis always connected
+if (env.MOCK_REDIS) {
+  CONNECTED = true
+}
+
 function pickClient(selectDb: number): any {
   return CLIENTS[selectDb]
 }
@@ -50,6 +56,7 @@ function connectionError(
  * will return the ioredis client which will be ready to use.
  */
 function init(selectDb = DEFAULT_SELECT_DB) {
+  const RedisCore = env.MOCK_REDIS ? MockRedis : Redis
   let timeout: NodeJS.Timeout
   CLOSED = false
   let client = pickClient(selectDb)
@@ -57,6 +64,10 @@ function init(selectDb = DEFAULT_SELECT_DB) {
   if (client && CONNECTED) {
     return
   }
+  // testing uses a single in memory client
+  if (env.MOCK_REDIS) {
+    CLIENTS[selectDb] = new RedisCore(getRedisOptions())
+  }
   // start the timer - only allowed 5 seconds to connect
   timeout = setTimeout(() => {
     if (!CONNECTED) {
@@ -75,11 +86,11 @@ function init(selectDb = DEFAULT_SELECT_DB) {
   const { redisProtocolUrl, opts, host, port } = getRedisOptions()
 
   if (CLUSTERED) {
-    client = new Redis.Cluster([{ host, port }], opts)
+    client = new RedisCore.Cluster([{ host, port }], opts)
   } else if (redisProtocolUrl) {
-    client = new Redis(redisProtocolUrl)
+    client = new RedisCore(redisProtocolUrl)
   } else {
-    client = new Redis(opts)
+    client = new RedisCore(opts)
   }
   // attach handlers
   client.on("end", (err: Error) => {
diff --git a/packages/server/package.json b/packages/server/package.json
index 472d221aa9..dc8401ea53 100644
--- a/packages/server/package.json
+++ b/packages/server/package.json
@@ -122,8 +122,8 @@
     "validate.js": "0.13.1",
     "vm2": "3.9.17",
     "worker-farm": "1.7.0",
-    "yargs": "13.2.4",
-    "xml2js": "0.5.0"
+    "xml2js": "0.5.0",
+    "yargs": "13.2.4"
   },
   "devDependencies": {
     "@babel/core": "7.17.4",
@@ -175,7 +175,8 @@
     "ts-node": "10.8.1",
     "tsconfig-paths": "4.0.0",
     "typescript": "4.7.3",
-    "update-dotenv": "1.1.1"
+    "update-dotenv": "1.1.1",
+    "ioredis-mock": "^8.7.0"
   },
   "optionalDependencies": {
     "oracledb": "5.3.0"
diff --git a/packages/server/src/api/routes/tests/view.spec.js b/packages/server/src/api/routes/tests/view.spec.js
index 522987ab29..df07ffa4af 100644
--- a/packages/server/src/api/routes/tests/view.spec.js
+++ b/packages/server/src/api/routes/tests/view.spec.js
@@ -114,8 +114,8 @@ describe("/views", () => {
       expect(res.body.tableId).toBe(table._id)
 
       const updatedTable = await config.getTable(table._id)
-      expect(updatedTable.views).toEqual({
-        TestView: {
+      const expectedObj = expect.objectContaining({
+        TestView: expect.objectContaining({
           field: "Price",
           calculation: "stats",
           tableId: table._id,
@@ -143,8 +143,9 @@ describe("/views", () => {
               type: "string",
             },
           },
-        },
+        }),
       })
+      expect(updatedTable.views).toEqual(expectedObj)
     })
   })
 
diff --git a/packages/server/src/utilities/redis.ts b/packages/server/src/utilities/redis.ts
index 369c8b2e76..d833ed2c10 100644
--- a/packages/server/src/utilities/redis.ts
+++ b/packages/server/src/utilities/redis.ts
@@ -25,7 +25,9 @@ export async function init() {
 
   // Duplicate the socket client for pub/sub
   socketClient = await redis.clients.getSocketClient()
-  socketSubClient = socketClient.getClient().duplicate()
+  if (!env.isTest()) {
+    socketSubClient = socketClient.getClient().duplicate()
+  }
 }
 
 export async function shutdown() {
diff --git a/packages/server/src/websockets/index.ts b/packages/server/src/websockets/index.ts
index 9503041b15..bb530b559e 100644
--- a/packages/server/src/websockets/index.ts
+++ b/packages/server/src/websockets/index.ts
@@ -10,9 +10,12 @@ let gridSocket: GridSocket | undefined
 let builderSocket: BuilderSocket | undefined
 
 export const initialise = (app: Koa, server: http.Server) => {
-  clientAppSocket = new ClientAppSocket(app, server)
-  gridSocket = new GridSocket(app, server)
-  builderSocket = new BuilderSocket(app, server)
+  // have to remove these for testing until ioredis-mock can be fully removed
+  if (!env.isTest()) {
+    clientAppSocket = new ClientAppSocket(app, server)
+    gridSocket = new GridSocket(app, server)
+    builderSocket = new BuilderSocket(app, server)
+  }
 }
 
 export { clientAppSocket, gridSocket, builderSocket }
diff --git a/yarn.lock b/yarn.lock
index 51fb4bb034..05bbe68476 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -2591,6 +2591,16 @@
   resolved "https://registry.yarnpkg.com/@hutson/parse-repository-url/-/parse-repository-url-3.0.2.tgz#98c23c950a3d9b6c8f0daed06da6c3af06981340"
   integrity sha512-H9XAx3hc0BQHY6l+IFSWHDySypcXsvsuLhgYLUGywmJ5pswRVQJUHpOsobnLYp2ZUaUlKiKDrgWWhosOwAEM8Q==
 
+"@ioredis/as-callback@^3.0.0":
+  version "3.0.0"
+  resolved "https://registry.yarnpkg.com/@ioredis/as-callback/-/as-callback-3.0.0.tgz#b96c9b05e6701e85ec6a5e62fa254071b0aec97f"
+  integrity sha512-Kqv1rZ3WbgOrS+hgzJ5xG5WQuhvzzSTRYvNeyPMLOAM78MHSnuKI20JeJGbpuAt//LCuP0vsexZcorqW7kWhJg==
+
+"@ioredis/commands@^1.2.0":
+  version "1.2.0"
+  resolved "https://registry.yarnpkg.com/@ioredis/commands/-/commands-1.2.0.tgz#6d61b3097470af1fdbbe622795b8921d42018e11"
+  integrity sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg==
+
 "@isaacs/string-locale-compare@^1.1.0":
   version "1.1.0"
   resolved "https://registry.yarnpkg.com/@isaacs/string-locale-compare/-/string-locale-compare-1.1.0.tgz#291c227e93fd407a96ecd59879a35809120e432b"
@@ -11956,7 +11966,7 @@ fecha@^4.2.0:
   resolved "https://registry.yarnpkg.com/fecha/-/fecha-4.2.3.tgz#4d9ccdbc61e8629b259fdca67e65891448d569fd"
   integrity sha512-OP2IUU6HeYKJi3i0z4A19kHMQoLVs4Hc+DPqqxI2h/DPZHTm/vjsfC6P0b4jCMy14XizLBqvndQ+UilD7707Jw==
 
-fengari-interop@^0.1.2:
+fengari-interop@^0.1.2, fengari-interop@^0.1.3:
   version "0.1.3"
   resolved "https://registry.yarnpkg.com/fengari-interop/-/fengari-interop-0.1.3.tgz#3ad37a90e7430b69b365441e9fc0ba168942a146"
   integrity sha512-EtZ+oTu3kEwVJnoymFPBVLIbQcCoy9uWCVnMA6h3M/RqHkUBsLYp29+RRHf9rKr6GwjubWREU1O7RretFIXjHw==
@@ -14003,6 +14013,17 @@ ioredis-mock@5.8.0:
     lodash "^4.17.21"
     standard-as-callback "^2.1.0"
 
+ioredis-mock@^8.7.0:
+  version "8.7.0"
+  resolved "https://registry.yarnpkg.com/ioredis-mock/-/ioredis-mock-8.7.0.tgz#9877a85e0d233e1b49123d1c6e320df01e9a1d36"
+  integrity sha512-BJcSjkR3sIMKbH93fpFzwlWi/jl1kd5I3vLvGQxnJ/W/6bD2ksrxnyQN186ljAp3Foz4p1ivViDE3rZeKEAluA==
+  dependencies:
+    "@ioredis/as-callback" "^3.0.0"
+    "@ioredis/commands" "^1.2.0"
+    fengari "^0.1.4"
+    fengari-interop "^0.1.3"
+    semver "^7.3.8"
+
 ioredis@4.28.0:
   version "4.28.0"
   resolved "https://registry.yarnpkg.com/ioredis/-/ioredis-4.28.0.tgz#5a2be3f37ff2075e2332f280eaeb02ab4d9ff0d3"

From 022016366e0ab2d1028ea345044e594c22f44215 Mon Sep 17 00:00:00 2001
From: Michael Drury <me@michaeldrury.co.uk>
Date: Thu, 1 Jun 2023 00:19:24 +0100
Subject: [PATCH 07/25] Adding ioredis-mock as an actual dependency for builds.

---
 packages/server/package.json | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/packages/server/package.json b/packages/server/package.json
index dc8401ea53..998ef768be 100644
--- a/packages/server/package.json
+++ b/packages/server/package.json
@@ -80,6 +80,7 @@
     "global-agent": "3.0.0",
     "google-auth-library": "7.12.0",
     "google-spreadsheet": "3.2.0",
+    "ioredis-mock": "^8.7.0",
     "jimp": "0.16.1",
     "joi": "17.6.0",
     "js-yaml": "4.1.0",
@@ -175,8 +176,7 @@
     "ts-node": "10.8.1",
     "tsconfig-paths": "4.0.0",
     "typescript": "4.7.3",
-    "update-dotenv": "1.1.1",
-    "ioredis-mock": "^8.7.0"
+    "update-dotenv": "1.1.1"
   },
   "optionalDependencies": {
     "oracledb": "5.3.0"

From 9cd4a49d554d4da8351537561ec2140cf100a1ab Mon Sep 17 00:00:00 2001
From: Rory Powell <rory.codes@gmail.com>
Date: Thu, 1 Jun 2023 08:49:12 +0100
Subject: [PATCH 08/25] Update pro

---
 packages/pro | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/pro b/packages/pro
index 86c32b80e0..cd06642b86 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit 86c32b80e08d2f19b57dcc2a3159667ac5a86c21
+Subproject commit cd06642b860111aa1bd3443ee10076ca3abf03c3

From fde94a262f0b695bb0cbdb80c18a56ab549cade3 Mon Sep 17 00:00:00 2001
From: Michael Drury <me@michaeldrury.co.uk>
Date: Thu, 1 Jun 2023 10:13:54 +0100
Subject: [PATCH 09/25] Moving ioredis-mock dependency back to dev dependency
 in backend-core.

---
 packages/backend-core/package.json |  2 +-
 packages/server/package.json       |  1 -
 yarn.lock                          | 12 +-----------
 3 files changed, 2 insertions(+), 13 deletions(-)

diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json
index f04437c066..5daca2796e 100644
--- a/packages/backend-core/package.json
+++ b/packages/backend-core/package.json
@@ -74,7 +74,7 @@
     "@types/tar-fs": "2.0.1",
     "@types/uuid": "8.3.4",
     "chance": "1.1.8",
-    "ioredis-mock": "5.8.0",
+    "ioredis-mock": "^8.7.0",
     "jest": "29.5.0",
     "jest-environment-node": "29.5.0",
     "jest-serial-runner": "^1.2.1",
diff --git a/packages/server/package.json b/packages/server/package.json
index 998ef768be..8e72a02b18 100644
--- a/packages/server/package.json
+++ b/packages/server/package.json
@@ -80,7 +80,6 @@
     "global-agent": "3.0.0",
     "google-auth-library": "7.12.0",
     "google-spreadsheet": "3.2.0",
-    "ioredis-mock": "^8.7.0",
     "jimp": "0.16.1",
     "joi": "17.6.0",
     "js-yaml": "4.1.0",
diff --git a/yarn.lock b/yarn.lock
index 05bbe68476..3f30468a7e 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -11966,7 +11966,7 @@ fecha@^4.2.0:
   resolved "https://registry.yarnpkg.com/fecha/-/fecha-4.2.3.tgz#4d9ccdbc61e8629b259fdca67e65891448d569fd"
   integrity sha512-OP2IUU6HeYKJi3i0z4A19kHMQoLVs4Hc+DPqqxI2h/DPZHTm/vjsfC6P0b4jCMy14XizLBqvndQ+UilD7707Jw==
 
-fengari-interop@^0.1.2, fengari-interop@^0.1.3:
+fengari-interop@^0.1.3:
   version "0.1.3"
   resolved "https://registry.yarnpkg.com/fengari-interop/-/fengari-interop-0.1.3.tgz#3ad37a90e7430b69b365441e9fc0ba168942a146"
   integrity sha512-EtZ+oTu3kEwVJnoymFPBVLIbQcCoy9uWCVnMA6h3M/RqHkUBsLYp29+RRHf9rKr6GwjubWREU1O7RretFIXjHw==
@@ -14003,16 +14003,6 @@ invert-kv@^2.0.0:
   resolved "https://registry.yarnpkg.com/invert-kv/-/invert-kv-2.0.0.tgz#7393f5afa59ec9ff5f67a27620d11c226e3eec02"
   integrity sha512-wPVv/y/QQ/Uiirj/vh3oP+1Ww+AWehmi1g5fFWGPF6IpCBCDVrhgHRMvrLfdYcwDh3QJbGXDW4JAuzxElLSqKA==
 
-ioredis-mock@5.8.0:
-  version "5.8.0"
-  resolved "https://registry.yarnpkg.com/ioredis-mock/-/ioredis-mock-5.8.0.tgz#99d3f0fe58419c2ff15c81cb4e13709aff8f2a44"
-  integrity sha512-PokVyNC/d3qaAyHAdf+Ex4HHBe7GdaSGrcn8pP7ran+pVmSrJ6Aofwm7Od5bQ5JtjDSiylEtIoZzEPYIjUKMHA==
-  dependencies:
-    fengari "^0.1.4"
-    fengari-interop "^0.1.2"
-    lodash "^4.17.21"
-    standard-as-callback "^2.1.0"
-
 ioredis-mock@^8.7.0:
   version "8.7.0"
   resolved "https://registry.yarnpkg.com/ioredis-mock/-/ioredis-mock-8.7.0.tgz#9877a85e0d233e1b49123d1c6e320df01e9a1d36"

From 9b23d3ba36618af1820b01e5836c9b97208df0c9 Mon Sep 17 00:00:00 2001
From: Michael Drury <me@michaeldrury.co.uk>
Date: Thu, 1 Jun 2023 10:14:30 +0100
Subject: [PATCH 10/25] Pinning version of ioredis-mock.

---
 packages/backend-core/package.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json
index 5daca2796e..6fc7e6fada 100644
--- a/packages/backend-core/package.json
+++ b/packages/backend-core/package.json
@@ -74,7 +74,7 @@
     "@types/tar-fs": "2.0.1",
     "@types/uuid": "8.3.4",
     "chance": "1.1.8",
-    "ioredis-mock": "^8.7.0",
+    "ioredis-mock": "8.7.0",
     "jest": "29.5.0",
     "jest-environment-node": "29.5.0",
     "jest-serial-runner": "^1.2.1",

From 13d987023a6ea389521090c34e5d2548ea9ca032 Mon Sep 17 00:00:00 2001
From: Michael Drury <me@michaeldrury.co.uk>
Date: Thu, 1 Jun 2023 10:39:16 +0100
Subject: [PATCH 11/25] Updating ioredis to latest version and removing typing
 library as no longer required.

---
 packages/backend-core/package.json |  3 +--
 yarn.lock                          | 27 +++++++++------------------
 2 files changed, 10 insertions(+), 20 deletions(-)

diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json
index 6fc7e6fada..f85687b007 100644
--- a/packages/backend-core/package.json
+++ b/packages/backend-core/package.json
@@ -33,7 +33,7 @@
     "correlation-id": "4.0.0",
     "dotenv": "16.0.1",
     "emitter-listener": "1.1.2",
-    "ioredis": "4.28.0",
+    "ioredis": "5.3.2",
     "joi": "17.6.0",
     "jsonwebtoken": "9.0.0",
     "koa-passport": "4.1.4",
@@ -62,7 +62,6 @@
     "@swc/jest": "^0.2.24",
     "@trendyol/jest-testcontainers": "^2.1.1",
     "@types/chance": "1.1.3",
-    "@types/ioredis": "4.28.0",
     "@types/jest": "29.5.0",
     "@types/koa": "2.13.4",
     "@types/lodash": "4.14.180",
diff --git a/yarn.lock b/yarn.lock
index 3f30468a7e..9a57eb4377 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -2596,7 +2596,7 @@
   resolved "https://registry.yarnpkg.com/@ioredis/as-callback/-/as-callback-3.0.0.tgz#b96c9b05e6701e85ec6a5e62fa254071b0aec97f"
   integrity sha512-Kqv1rZ3WbgOrS+hgzJ5xG5WQuhvzzSTRYvNeyPMLOAM78MHSnuKI20JeJGbpuAt//LCuP0vsexZcorqW7kWhJg==
 
-"@ioredis/commands@^1.2.0":
+"@ioredis/commands@^1.1.1", "@ioredis/commands@^1.2.0":
   version "1.2.0"
   resolved "https://registry.yarnpkg.com/@ioredis/commands/-/commands-1.2.0.tgz#6d61b3097470af1fdbbe622795b8921d42018e11"
   integrity sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg==
@@ -5659,13 +5659,6 @@
   resolved "https://registry.yarnpkg.com/@types/http-errors/-/http-errors-2.0.1.tgz#20172f9578b225f6c7da63446f56d4ce108d5a65"
   integrity sha512-/K3ds8TRAfBvi5vfjuz8y6+GiAYBZ0x4tXv1Av6CWBWn0IlADc+ZX9pMq7oU0fNQPnBwIZl3rmeLp6SBApbxSQ==
 
-"@types/ioredis@4.28.0":
-  version "4.28.0"
-  resolved "https://registry.yarnpkg.com/@types/ioredis/-/ioredis-4.28.0.tgz#609b2ea0d91231df2dd7f67dd77436bc72584911"
-  integrity sha512-HSA/JQivJgV0e+353gvgu6WVoWvGRe0HyHOnAN2AvbVIhUlJBhNnnkP8gEEokrDWrxywrBkwo8NuDZ6TVPL9XA==
-  dependencies:
-    "@types/node" "*"
-
 "@types/ioredis@4.28.10":
   version "4.28.10"
   resolved "https://registry.yarnpkg.com/@types/ioredis/-/ioredis-4.28.10.tgz#40ceb157a4141088d1394bb87c98ed09a75a06ff"
@@ -14003,7 +13996,7 @@ invert-kv@^2.0.0:
   resolved "https://registry.yarnpkg.com/invert-kv/-/invert-kv-2.0.0.tgz#7393f5afa59ec9ff5f67a27620d11c226e3eec02"
   integrity sha512-wPVv/y/QQ/Uiirj/vh3oP+1Ww+AWehmi1g5fFWGPF6IpCBCDVrhgHRMvrLfdYcwDh3QJbGXDW4JAuzxElLSqKA==
 
-ioredis-mock@^8.7.0:
+ioredis-mock@8.7.0:
   version "8.7.0"
   resolved "https://registry.yarnpkg.com/ioredis-mock/-/ioredis-mock-8.7.0.tgz#9877a85e0d233e1b49123d1c6e320df01e9a1d36"
   integrity sha512-BJcSjkR3sIMKbH93fpFzwlWi/jl1kd5I3vLvGQxnJ/W/6bD2ksrxnyQN186ljAp3Foz4p1ivViDE3rZeKEAluA==
@@ -14014,19 +14007,17 @@ ioredis-mock@^8.7.0:
     fengari-interop "^0.1.3"
     semver "^7.3.8"
 
-ioredis@4.28.0:
-  version "4.28.0"
-  resolved "https://registry.yarnpkg.com/ioredis/-/ioredis-4.28.0.tgz#5a2be3f37ff2075e2332f280eaeb02ab4d9ff0d3"
-  integrity sha512-I+zkeeWp3XFgPT2CtJKxvaF5FjGBGt4yGYljRjQecdQKteThuAsKqffeF1lgHVlYnuNeozRbPOCDNZ7tDWPeig==
+ioredis@5.3.2:
+  version "5.3.2"
+  resolved "https://registry.yarnpkg.com/ioredis/-/ioredis-5.3.2.tgz#9139f596f62fc9c72d873353ac5395bcf05709f7"
+  integrity sha512-1DKMMzlIHM02eBBVOFQ1+AolGjs6+xEcM4PDL7NqOS6szq7H9jSaEkIUH6/a5Hl241LzW6JLSiAbNvTQjUupUA==
   dependencies:
+    "@ioredis/commands" "^1.1.1"
     cluster-key-slot "^1.1.0"
-    debug "^4.3.1"
-    denque "^1.1.0"
+    debug "^4.3.4"
+    denque "^2.1.0"
     lodash.defaults "^4.2.0"
-    lodash.flatten "^4.4.0"
     lodash.isarguments "^3.1.0"
-    p-map "^2.1.0"
-    redis-commands "1.7.0"
     redis-errors "^1.2.0"
     redis-parser "^3.0.0"
     standard-as-callback "^2.1.0"

From 6276859f237a1123577e9d5df03ee86f5c39efc8 Mon Sep 17 00:00:00 2001
From: Budibase Staging Release Bot <>
Date: Thu, 1 Jun 2023 09:45:26 +0000
Subject: [PATCH 12/25] Bump version to 2.6.19-alpha.40

---
 lerna.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lerna.json b/lerna.json
index d0e9af5766..201ececb55 100644
--- a/lerna.json
+++ b/lerna.json
@@ -1,5 +1,5 @@
 {
-  "version": "2.6.19-alpha.39",
+  "version": "2.6.19-alpha.40",
   "npmClient": "yarn",
   "packages": [
     "packages/backend-core",

From eb6df79585a009426f5534185d32c365d4e9a367 Mon Sep 17 00:00:00 2001
From: Budibase Staging Release Bot <>
Date: Thu, 1 Jun 2023 10:06:35 +0000
Subject: [PATCH 13/25] Bump version to 2.6.19-alpha.41

---
 lerna.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lerna.json b/lerna.json
index 201ececb55..e0a82cfad4 100644
--- a/lerna.json
+++ b/lerna.json
@@ -1,5 +1,5 @@
 {
-  "version": "2.6.19-alpha.40",
+  "version": "2.6.19-alpha.41",
   "npmClient": "yarn",
   "packages": [
     "packages/backend-core",

From cde4dabe42ec0a645fdd79c5d80db52199d5fd00 Mon Sep 17 00:00:00 2001
From: Michael Drury <me@michaeldrury.co.uk>
Date: Thu, 1 Jun 2023 11:10:39 +0100
Subject: [PATCH 14/25] Fixing some redis type errors.

---
 packages/server/package.json              | 2 +-
 packages/server/src/integrations/redis.ts | 8 +++++++-
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/packages/server/package.json b/packages/server/package.json
index 8e72a02b18..c73bba0f16 100644
--- a/packages/server/package.json
+++ b/packages/server/package.json
@@ -80,6 +80,7 @@
     "global-agent": "3.0.0",
     "google-auth-library": "7.12.0",
     "google-spreadsheet": "3.2.0",
+    "ioredis": "5.3.2",
     "jimp": "0.16.1",
     "joi": "17.6.0",
     "js-yaml": "4.1.0",
@@ -137,7 +138,6 @@
     "@types/bson": "4.2.0",
     "@types/global-agent": "2.1.1",
     "@types/google-spreadsheet": "3.1.5",
-    "@types/ioredis": "4.28.10",
     "@types/jest": "29.5.0",
     "@types/koa": "2.13.4",
     "@types/koa__router": "8.0.8",
diff --git a/packages/server/src/integrations/redis.ts b/packages/server/src/integrations/redis.ts
index 26666ac800..a6372c8e13 100644
--- a/packages/server/src/integrations/redis.ts
+++ b/packages/server/src/integrations/redis.ts
@@ -177,7 +177,13 @@ class RedisIntegration {
       const pipeline = this.client.pipeline(pipelineCommands)
       const result = await pipeline.exec()
 
-      return result.map((output: string | string[]) => output[1])
+      return result?.map((output: any) => {
+        if (typeof output === "string") {
+          return output
+        } else if (Array.isArray(output)) {
+          return output[1]
+        }
+      })
     })
   }
 }

From 67588f1051e7a8dc6e3599f258fbc0099071e538 Mon Sep 17 00:00:00 2001
From: Michael Drury <me@michaeldrury.co.uk>
Date: Thu, 1 Jun 2023 12:14:34 +0100
Subject: [PATCH 15/25] Adding a JSON parse incase the input is actually an
 array in the form of a string (HBS).

---
 packages/server/src/threads/automation.ts | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/packages/server/src/threads/automation.ts b/packages/server/src/threads/automation.ts
index 6ed8506b85..9831a96936 100644
--- a/packages/server/src/threads/automation.ts
+++ b/packages/server/src/threads/automation.ts
@@ -44,10 +44,18 @@ function getLoopIterations(loopStep: LoopStep) {
   if (!binding) {
     return 0
   }
+  const isString = typeof binding === "string"
+  try {
+    if (isString) {
+      binding = JSON.parse(binding)
+    }
+  } catch (err) {
+    // ignore error - wasn't able to parse
+  }
   if (Array.isArray(binding)) {
     return binding.length
   }
-  if (typeof binding === "string") {
+  if (isString) {
     return automationUtils.stringSplit(binding).length
   }
   return 0

From bdc03e12cc9184e44a5b49916464b36f3bfe3fff Mon Sep 17 00:00:00 2001
From: Budibase Staging Release Bot <>
Date: Thu, 1 Jun 2023 11:34:02 +0000
Subject: [PATCH 16/25] Bump version to 2.6.19-alpha.42

---
 lerna.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lerna.json b/lerna.json
index e0a82cfad4..5c2ae9ec8b 100644
--- a/lerna.json
+++ b/lerna.json
@@ -1,5 +1,5 @@
 {
-  "version": "2.6.19-alpha.41",
+  "version": "2.6.19-alpha.42",
   "npmClient": "yarn",
   "packages": [
     "packages/backend-core",

From ec67b162b8b5c1943c634ba7650ddfc11a3add46 Mon Sep 17 00:00:00 2001
From: Michael Drury <me@michaeldrury.co.uk>
Date: Thu, 1 Jun 2023 14:10:07 +0100
Subject: [PATCH 17/25] There has been an issue with open source contributors
 being able to perform CI runs for their PRs - this is related to the addition
 of the pro checkout/submodule recursion in a recent update. Changing it to
 not use the contributors personal access token, but to instead use the github
 access token so that it can always be checked out and tested again the
 specified pro version.

---
 .github/workflows/budibase_ci.yml | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml
index 4e0bc8b3a9..d915a7f0ad 100644
--- a/.github/workflows/budibase_ci.yml
+++ b/.github/workflows/budibase_ci.yml
@@ -16,7 +16,6 @@ on:
 env:
   BRANCH: ${{ github.event.pull_request.head.ref }}
   BASE_BRANCH: ${{ github.event.pull_request.base.ref}}
-  PERSONAL_ACCESS_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
 
 jobs:
   lint:
@@ -37,7 +36,7 @@ jobs:
       - uses: actions/checkout@v3
         with:
           submodules: true
-          token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
+          token: ${{ secrets.GITHUB_TOKEN }}
       - name: Use Node.js 14.x
         uses: actions/setup-node@v3
         with:
@@ -55,7 +54,7 @@ jobs:
       - uses: actions/checkout@v3
         with:
           submodules: true
-          token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
+          token: ${{ secrets.GITHUB_TOKEN }}
       - name: Use Node.js 14.x
         uses: actions/setup-node@v3
         with:
@@ -75,7 +74,7 @@ jobs:
       - uses: actions/checkout@v3
         with:
           submodules: true
-          token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
+          token: ${{ secrets.GITHUB_TOKEN }}
       - name: Use Node.js 14.x
         uses: actions/setup-node@v3
         with:
@@ -95,7 +94,7 @@ jobs:
       - uses: actions/checkout@v3
         with:
           submodules: true
-          token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
+          token: ${{ secrets.GITHUB_TOKEN }}
       - name: Use Node.js 14.x
         uses: actions/setup-node@v3
         with:
@@ -110,7 +109,7 @@ jobs:
       - uses: actions/checkout@v3
         with:
           submodules: true
-          token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
+          token: ${{ secrets.GITHUB_TOKEN }}
       - name: Use Node.js 14.x
         uses: actions/setup-node@v3
         with:
@@ -134,7 +133,7 @@ jobs:
         uses: actions/checkout@v3
         with:
           submodules: true
-          token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
+          token: ${{ secrets.GITHUB_TOKEN }}
           fetch-depth: 0
       - name: Check submodule
         run: |

From b44a7fb2881a7a7a54bd2e966bd842d22b904a00 Mon Sep 17 00:00:00 2001
From: Dean <deanhannigan@gmail.com>
Date: Thu, 1 Jun 2023 14:44:03 +0100
Subject: [PATCH 18/25] Restored the default behaviour for tab switching when
 no switch checking fn is provided

---
 packages/bbui/src/Tabs/Tabs.svelte | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/packages/bbui/src/Tabs/Tabs.svelte b/packages/bbui/src/Tabs/Tabs.svelte
index 4cc22b226e..7cb77bc976 100644
--- a/packages/bbui/src/Tabs/Tabs.svelte
+++ b/packages/bbui/src/Tabs/Tabs.svelte
@@ -37,6 +37,10 @@
           dispatch("select", thisSelected)
         }
       }
+    } else {
+      thisSelected = $tab.title
+      selected = $tab.title
+      dispatch("select", thisSelected)
     }
     if ($tab.title !== thisSelected) {
       tab.update(state => {

From 3a70426e3eb5c21b1fc85e94875139d86deb322c Mon Sep 17 00:00:00 2001
From: Dean <deanhannigan@gmail.com>
Date: Thu, 1 Jun 2023 14:51:50 +0100
Subject: [PATCH 19/25] Tab switch default behaviour shifted to correct
 location

---
 packages/bbui/src/Tabs/Tabs.svelte | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/packages/bbui/src/Tabs/Tabs.svelte b/packages/bbui/src/Tabs/Tabs.svelte
index 7cb77bc976..9c3d25a807 100644
--- a/packages/bbui/src/Tabs/Tabs.svelte
+++ b/packages/bbui/src/Tabs/Tabs.svelte
@@ -36,11 +36,11 @@
           selected = $tab.title
           dispatch("select", thisSelected)
         }
+      } else {
+        thisSelected = $tab.title
+        selected = $tab.title
+        dispatch("select", thisSelected)
       }
-    } else {
-      thisSelected = $tab.title
-      selected = $tab.title
-      dispatch("select", thisSelected)
     }
     if ($tab.title !== thisSelected) {
       tab.update(state => {

From 1b0f8f2b9eeb5fd873e98de3c71f960cf6272fe5 Mon Sep 17 00:00:00 2001
From: Michael Drury <me@michaeldrury.co.uk>
Date: Thu, 1 Jun 2023 15:01:57 +0100
Subject: [PATCH 20/25] Revert "There has been an issue with open source
 contributors being able to perform CI runs for their PRs - this is related to
 the addition of the pro checkout/submodule recursion in a recent update.
 Changing it to not use the contributors personal access token, but to instead
 use the github access token so that it can always be checked out and tested
 again the specified pro version."

This reverts commit ec67b162b8b5c1943c634ba7650ddfc11a3add46.
---
 .github/workflows/budibase_ci.yml | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml
index d915a7f0ad..4e0bc8b3a9 100644
--- a/.github/workflows/budibase_ci.yml
+++ b/.github/workflows/budibase_ci.yml
@@ -16,6 +16,7 @@ on:
 env:
   BRANCH: ${{ github.event.pull_request.head.ref }}
   BASE_BRANCH: ${{ github.event.pull_request.base.ref}}
+  PERSONAL_ACCESS_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
 
 jobs:
   lint:
@@ -36,7 +37,7 @@ jobs:
       - uses: actions/checkout@v3
         with:
           submodules: true
-          token: ${{ secrets.GITHUB_TOKEN }}
+          token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
       - name: Use Node.js 14.x
         uses: actions/setup-node@v3
         with:
@@ -54,7 +55,7 @@ jobs:
       - uses: actions/checkout@v3
         with:
           submodules: true
-          token: ${{ secrets.GITHUB_TOKEN }}
+          token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
       - name: Use Node.js 14.x
         uses: actions/setup-node@v3
         with:
@@ -74,7 +75,7 @@ jobs:
       - uses: actions/checkout@v3
         with:
           submodules: true
-          token: ${{ secrets.GITHUB_TOKEN }}
+          token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
       - name: Use Node.js 14.x
         uses: actions/setup-node@v3
         with:
@@ -94,7 +95,7 @@ jobs:
       - uses: actions/checkout@v3
         with:
           submodules: true
-          token: ${{ secrets.GITHUB_TOKEN }}
+          token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
       - name: Use Node.js 14.x
         uses: actions/setup-node@v3
         with:
@@ -109,7 +110,7 @@ jobs:
       - uses: actions/checkout@v3
         with:
           submodules: true
-          token: ${{ secrets.GITHUB_TOKEN }}
+          token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
       - name: Use Node.js 14.x
         uses: actions/setup-node@v3
         with:
@@ -133,7 +134,7 @@ jobs:
         uses: actions/checkout@v3
         with:
           submodules: true
-          token: ${{ secrets.GITHUB_TOKEN }}
+          token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
           fetch-depth: 0
       - name: Check submodule
         run: |

From 74b4f66adc281fa12d5eb0fa0dc638d13d7f524b Mon Sep 17 00:00:00 2001
From: Michael Drury <me@michaeldrury.co.uk>
Date: Thu, 1 Jun 2023 15:04:50 +0100
Subject: [PATCH 21/25] Trying a different approach to solving token
 replacement.

---
 .github/workflows/budibase_ci.yml | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml
index 4e0bc8b3a9..3cc680a76a 100644
--- a/.github/workflows/budibase_ci.yml
+++ b/.github/workflows/budibase_ci.yml
@@ -16,7 +16,7 @@ on:
 env:
   BRANCH: ${{ github.event.pull_request.head.ref }}
   BASE_BRANCH: ${{ github.event.pull_request.base.ref}}
-  PERSONAL_ACCESS_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
+  PERSONAL_ACCESS_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }}
 
 jobs:
   lint:
@@ -37,7 +37,7 @@ jobs:
       - uses: actions/checkout@v3
         with:
           submodules: true
-          token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
+          token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }}
       - name: Use Node.js 14.x
         uses: actions/setup-node@v3
         with:
@@ -55,7 +55,7 @@ jobs:
       - uses: actions/checkout@v3
         with:
           submodules: true
-          token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
+          token: ${{ secrets.PERSONAL_ACCESS_TOKEN |github.token| github.token }}
       - name: Use Node.js 14.x
         uses: actions/setup-node@v3
         with:
@@ -75,7 +75,7 @@ jobs:
       - uses: actions/checkout@v3
         with:
           submodules: true
-          token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
+          token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }}
       - name: Use Node.js 14.x
         uses: actions/setup-node@v3
         with:
@@ -85,7 +85,7 @@ jobs:
       - run: yarn test --scope=@budibase/worker --scope=@budibase/server
       - uses: codecov/codecov-action@v3
         with:
-          token: ${{ secrets.CODECOV_TOKEN }} # not required for public repos
+          token: ${{ secrets.CODECOV_TOKEN || github.token }} # not required for public repos
           name: codecov-umbrella
           verbose: true
 
@@ -95,7 +95,7 @@ jobs:
       - uses: actions/checkout@v3
         with:
           submodules: true
-          token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
+          token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }}
       - name: Use Node.js 14.x
         uses: actions/setup-node@v3
         with:
@@ -110,7 +110,7 @@ jobs:
       - uses: actions/checkout@v3
         with:
           submodules: true
-          token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
+          token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }}
       - name: Use Node.js 14.x
         uses: actions/setup-node@v3
         with:
@@ -134,7 +134,7 @@ jobs:
         uses: actions/checkout@v3
         with:
           submodules: true
-          token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
+          token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }}
           fetch-depth: 0
       - name: Check submodule
         run: |

From 8129954f49b01dbc5635efd22d79f65982d3c49d Mon Sep 17 00:00:00 2001
From: Michael Drury <me@michaeldrury.co.uk>
Date: Thu, 1 Jun 2023 15:07:17 +0100
Subject: [PATCH 22/25] Updating env in budibase ci workflow.

---
 .github/workflows/budibase_ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml
index 3cc680a76a..c7be32bb5b 100644
--- a/.github/workflows/budibase_ci.yml
+++ b/.github/workflows/budibase_ci.yml
@@ -16,7 +16,7 @@ on:
 env:
   BRANCH: ${{ github.event.pull_request.head.ref }}
   BASE_BRANCH: ${{ github.event.pull_request.base.ref}}
-  PERSONAL_ACCESS_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }}
+  PERSONAL_ACCESS_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
 
 jobs:
   lint:

From 2526ab8a2f3005bf99718aaeb5876a63cdaa67be Mon Sep 17 00:00:00 2001
From: Michael Drury <me@michaeldrury.co.uk>
Date: Thu, 1 Jun 2023 15:15:20 +0100
Subject: [PATCH 23/25] Fixing a YAML error.

---
 .github/workflows/budibase_ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml
index c7be32bb5b..6c875f2dfe 100644
--- a/.github/workflows/budibase_ci.yml
+++ b/.github/workflows/budibase_ci.yml
@@ -55,7 +55,7 @@ jobs:
       - uses: actions/checkout@v3
         with:
           submodules: true
-          token: ${{ secrets.PERSONAL_ACCESS_TOKEN |github.token| github.token }}
+          token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }}
       - name: Use Node.js 14.x
         uses: actions/setup-node@v3
         with:

From 0d9a96b3e6b55d3a268b7495efb8ba71dcf57267 Mon Sep 17 00:00:00 2001
From: Budibase Staging Release Bot <>
Date: Thu, 1 Jun 2023 14:18:52 +0000
Subject: [PATCH 24/25] Bump version to 2.6.19-alpha.43

---
 lerna.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lerna.json b/lerna.json
index 5c2ae9ec8b..248f3c9d71 100644
--- a/lerna.json
+++ b/lerna.json
@@ -1,5 +1,5 @@
 {
-  "version": "2.6.19-alpha.42",
+  "version": "2.6.19-alpha.43",
   "npmClient": "yarn",
   "packages": [
     "packages/backend-core",

From 91f760d7c44847973545566d5733ebc6e416e7fb Mon Sep 17 00:00:00 2001
From: Budibase Staging Release Bot <>
Date: Thu, 1 Jun 2023 15:55:01 +0000
Subject: [PATCH 25/25] Bump version to 2.6.19-alpha.44

---
 lerna.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lerna.json b/lerna.json
index 248f3c9d71..86dab62dd9 100644
--- a/lerna.json
+++ b/lerna.json
@@ -1,5 +1,5 @@
 {
-  "version": "2.6.19-alpha.43",
+  "version": "2.6.19-alpha.44",
   "npmClient": "yarn",
   "packages": [
     "packages/backend-core",