From 8189952f0b7c8b2796a63c5ced4e2d6ba88ba6b7 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Fri, 19 Apr 2024 18:03:38 +0100
Subject: [PATCH 01/12] Adding implementation to DB for purge and cleanup APIs
 of SQS, to make sure the DB is cleared of any unused tables or rows.

---
 .../backend-core/src/db/couch/DatabaseImpl.ts | 50 +++++++++++++++----
 .../backend-core/src/db/instrumentation.ts    | 14 ++++++
 .../server/src/sdk/app/tables/internal/sqs.ts | 21 +++++++-
 packages/types/src/sdk/db.ts                  |  2 +
 4 files changed, 75 insertions(+), 12 deletions(-)

diff --git a/packages/backend-core/src/db/couch/DatabaseImpl.ts b/packages/backend-core/src/db/couch/DatabaseImpl.ts
index d220d0a8ac..e10d0e58c6 100644
--- a/packages/backend-core/src/db/couch/DatabaseImpl.ts
+++ b/packages/backend-core/src/db/couch/DatabaseImpl.ts
@@ -249,25 +249,53 @@ export class DatabaseImpl implements Database {
     })
   }
 
+  async _sqlQuery<T>(
+    url: string,
+    method: "POST" | "GET",
+    body?: any
+  ): Promise<T> {
+    const args: { url: string; method: string; cookie: string; body?: any } = {
+      url: `${this.couchInfo.sqlUrl}/${url}`,
+      method,
+      cookie: this.couchInfo.cookie,
+    }
+    if (body) {
+      args.body = body
+    }
+    const response = await directCouchUrlCall(body)
+    if (response.status > 300) {
+      throw new Error(await response.text())
+    }
+    return (await response.json()) as T
+  }
+
   async sql<T extends Document>(
     sql: string,
     parameters?: SqlQueryBinding
   ): Promise<T[]> {
     const dbName = this.name
     const url = `/${dbName}/${SQLITE_DESIGN_DOC_ID}`
-    const response = await directCouchUrlCall({
-      url: `${this.couchInfo.sqlUrl}/${url}`,
-      method: "POST",
-      cookie: this.couchInfo.cookie,
-      body: {
-        query: sql,
-        args: parameters,
-      },
+    return await this._sqlQuery<T[]>(url, "POST", {
+      query: sql,
+      args: parameters,
     })
-    if (response.status > 300) {
-      throw new Error(await response.text())
+  }
+
+  // checks design document is accurate (cleans up tables)
+  async sqlCleanup(): Promise<void> {
+    const dbName = this.name
+    const url = `/${dbName}/_cleanup`
+    return await this._sqlQuery<void>(url, "POST")
+  }
+
+  // removes a document from sqlite
+  async sqlPurge(docIds: string[] | string): Promise<void> {
+    if (!Array.isArray(docIds)) {
+      docIds = [docIds]
     }
-    return (await response.json()) as T[]
+    const dbName = this.name
+    const url = `/${dbName}/_purge`
+    return await this._sqlQuery<void>(url, "POST", { docs: docIds })
   }
 
   async query<T extends Document>(
diff --git a/packages/backend-core/src/db/instrumentation.ts b/packages/backend-core/src/db/instrumentation.ts
index 32ba81ebd8..715985766b 100644
--- a/packages/backend-core/src/db/instrumentation.ts
+++ b/packages/backend-core/src/db/instrumentation.ts
@@ -160,4 +160,18 @@ export class DDInstrumentedDatabase implements Database {
       return this.db.sql(sql, parameters)
     })
   }
+
+  sqlPurge(docIds: string[] | string): Promise<void> {
+    return tracer.trace("db.sqlPurge", span => {
+      span?.addTags({ db_name: this.name })
+      return this.db.sqlPurge(docIds)
+    })
+  }
+
+  sqlCleanup(): Promise<void> {
+    return tracer.trace("db.sqlCleanup", span => {
+      span?.addTags({ db_name: this.name })
+      return this.db.sqlCleanup()
+    })
+  }
 }
diff --git a/packages/server/src/sdk/app/tables/internal/sqs.ts b/packages/server/src/sdk/app/tables/internal/sqs.ts
index 0726c94962..51e3f17bca 100644
--- a/packages/server/src/sdk/app/tables/internal/sqs.ts
+++ b/packages/server/src/sdk/app/tables/internal/sqs.ts
@@ -118,7 +118,7 @@ export async function addTableToSqlite(table: Table) {
   const db = context.getAppDB()
   let definition: SQLiteDefinition
   try {
-    definition = await db.get(SQLITE_DESIGN_DOC_ID)
+    definition = await db.get<SQLiteDefinition>(SQLITE_DESIGN_DOC_ID)
   } catch (err) {
     definition = await buildBaseDefinition()
   }
@@ -128,3 +128,22 @@ export async function addTableToSqlite(table: Table) {
   }
   await db.put(definition)
 }
+
+export async function removeTableFromSqlite(table: Table) {
+  const db = context.getAppDB()
+  try {
+    const definition = await db.get<SQLiteDefinition>(SQLITE_DESIGN_DOC_ID)
+    if (definition.sql?.tables?.[table._id!]) {
+      delete definition.sql.tables[table._id!]
+      await db.put(definition)
+      // make sure SQS is cleaned up, tables removed
+      await db.sqlCleanup()
+    }
+  } catch (err: any) {
+    if (err?.status === 404) {
+      return
+    } else {
+      throw err
+    }
+  }
+}
diff --git a/packages/types/src/sdk/db.ts b/packages/types/src/sdk/db.ts
index c723f5f8d6..cd4d656a28 100644
--- a/packages/types/src/sdk/db.ts
+++ b/packages/types/src/sdk/db.ts
@@ -148,6 +148,8 @@ export interface Database {
     sql: string,
     parameters?: SqlQueryBinding
   ): Promise<T[]>
+  sqlPurge(docIds: string[] | string): Promise<void>
+  sqlCleanup(): Promise<void>
   allDocs<T extends Document | RowValue>(
     params: DatabaseQueryOpts
   ): Promise<AllDocsResponse<T>>

From 006addb9cae1d1378af5736a917a91995523c4a9 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Tue, 23 Apr 2024 13:34:50 +0100
Subject: [PATCH 02/12] Cleaning up tables when they are deleted, refactored a
 bit to make more similar to table save.

---
 .../server/src/api/controllers/table/utils.ts | 27 +++++++++++
 .../src/sdk/app/tables/internal/index.ts      | 46 +++++++------------
 yarn.lock                                     |  8 ++++
 3 files changed, 52 insertions(+), 29 deletions(-)

diff --git a/packages/server/src/api/controllers/table/utils.ts b/packages/server/src/api/controllers/table/utils.ts
index f496c686f3..cb2063abf9 100644
--- a/packages/server/src/api/controllers/table/utils.ts
+++ b/packages/server/src/api/controllers/table/utils.ts
@@ -33,6 +33,7 @@ import {
 } from "@budibase/types"
 import sdk from "../../../sdk"
 import env from "../../../environment"
+import { runStaticFormulaChecks } from "./bulkFormula"
 
 export async function clearColumns(table: Table, columnNames: string[]) {
   const db = context.getAppDB()
@@ -495,5 +496,31 @@ export function setStaticSchemas(datasource: Datasource, table: Table) {
   return table
 }
 
+export async function internalTableCleanup(table: Table, rows?: Row[]) {
+  const db = context.getAppDB()
+  const tableId = table._id!
+  // remove table search index
+  if (!env.isTest() || env.COUCH_DB_URL) {
+    const currentIndexes = await db.getIndexes()
+    const existingIndex = currentIndexes.indexes.find(
+      (existing: any) => existing.name === `search:${tableId}`
+    )
+    if (existingIndex) {
+      await db.deleteIndex(existingIndex)
+    }
+  }
+
+  // has to run after, make sure it has _id
+  await runStaticFormulaChecks(table, {
+    deletion: true,
+  })
+  if (rows) {
+    await AttachmentCleanup.tableDelete(table, rows)
+  }
+  if (env.SQS_SEARCH_ENABLE) {
+    await sdk.tables.sqs.removeTableFromSqlite(table)
+  }
+}
+
 const _TableSaveFunctions = TableSaveFunctions
 export { _TableSaveFunctions as TableSaveFunctions }
diff --git a/packages/server/src/sdk/app/tables/internal/index.ts b/packages/server/src/sdk/app/tables/internal/index.ts
index 5d9feb5fe8..caddd3ee94 100644
--- a/packages/server/src/sdk/app/tables/internal/index.ts
+++ b/packages/server/src/sdk/app/tables/internal/index.ts
@@ -10,6 +10,7 @@ import {
 import {
   hasTypeChanged,
   TableSaveFunctions,
+  internalTableCleanup,
 } from "../../../../api/controllers/table/utils"
 import { EventType, updateLinks } from "../../../../db/linkedRows"
 import { cloneDeep } from "lodash/fp"
@@ -128,16 +129,20 @@ export async function destroy(table: Table) {
   const db = context.getAppDB()
   const tableId = table._id!
 
-  // Delete all rows for that table
-  const rowsData = await db.allDocs(
-    getRowParams(tableId, null, {
-      include_docs: true,
-    })
-  )
-  await db.bulkDocs(
-    rowsData.rows.map((row: any) => ({ ...row.doc, _deleted: true }))
-  )
-  await quotas.removeRows(rowsData.rows.length, {
+  // Delete all rows for that table - we have to retrieve the full rows for
+  // attachment cleanup, this may be worth investigating if there is a better
+  // way - we could delete all rows without the `include_docs` which would be faster
+  const rows = (
+    await db.allDocs<Row>(
+      getRowParams(tableId, null, {
+        include_docs: true,
+      })
+    )
+  ).rows.map(data => data.doc!)
+  await db.bulkDocs(rows.map((row: Row) => ({ ...row, _deleted: true })))
+
+  // remove rows from quota
+  await quotas.removeRows(rows.length, {
     tableId,
   })
 
@@ -150,25 +155,8 @@ export async function destroy(table: Table) {
   // don't remove the table itself until very end
   await db.remove(tableId, table._rev)
 
-  // remove table search index
-  if (!env.isTest() || env.COUCH_DB_URL) {
-    const currentIndexes = await db.getIndexes()
-    const existingIndex = currentIndexes.indexes.find(
-      (existing: any) => existing.name === `search:${tableId}`
-    )
-    if (existingIndex) {
-      await db.deleteIndex(existingIndex)
-    }
-  }
-
-  // has to run after, make sure it has _id
-  await runStaticFormulaChecks(table, {
-    deletion: true,
-  })
-  await AttachmentCleanup.tableDelete(
-    table,
-    rowsData.rows.map((row: any) => row.doc)
-  )
+  // final cleanup, attachments, indexes, SQS
+  await internalTableCleanup(table, rows)
 
   return { table }
 }
diff --git a/yarn.lock b/yarn.lock
index 30b275c434..8580ef9657 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -5930,6 +5930,14 @@
   resolved "https://registry.yarnpkg.com/@types/range-parser/-/range-parser-1.2.4.tgz#cd667bcfdd025213aafb7ca5915a932590acdcdc"
   integrity sha512-EEhsLsD6UsDM1yFhAvy0Cjr6VwmpMWqFBCb9w07wVugF7w9nfajxLuVmngTIpgS6svCnm6Vaw+MZhoDCKnOfsw==
 
+"@types/readable-stream@^4.0.0":
+  version "4.0.11"
+  resolved "https://registry.yarnpkg.com/@types/readable-stream/-/readable-stream-4.0.11.tgz#684f1e947c90cb6a8ad3904523d650bb66cdbb84"
+  integrity sha512-R3eUMUTTKoIoaz7UpYLxvZCrOmCRPRbAmoDDHKcimTEySltaJhF8hLzj4+EzyDifiX5eK6oDQGSfmNnXjxZzYQ==
+  dependencies:
+    "@types/node" "*"
+    safe-buffer "~5.1.1"
+
 "@types/readdir-glob@*":
   version "1.1.5"
   resolved "https://registry.yarnpkg.com/@types/readdir-glob/-/readdir-glob-1.1.5.tgz#21a4a98898fc606cb568ad815f2a0eedc24d412a"

From fdeb7e57bbb952998d72b8a78706e5dcc61a3763 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Fri, 26 Apr 2024 15:56:45 +0100
Subject: [PATCH 03/12] Updating yarn lock.

---
 yarn.lock | 40 ++++------------------------------------
 1 file changed, 4 insertions(+), 36 deletions(-)

diff --git a/yarn.lock b/yarn.lock
index d833731db4..120f69cb43 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -11612,7 +11612,7 @@ fs.realpath@^1.0.0:
   resolved "https://registry.yarnpkg.com/fs.realpath/-/fs.realpath-1.0.0.tgz#1504ad2523158caa40db4a2787cb01411994ea4f"
   integrity sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==
 
-fsevents@^2.3.2, fsevents@~2.3.1, fsevents@~2.3.2:
+fsevents@^2.3.2, fsevents@~2.3.2:
   version "2.3.3"
   resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-2.3.3.tgz#cac6407785d03675a2a5e1a5305c697b347d90d6"
   integrity sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==
@@ -19761,13 +19761,6 @@ rollup-pluginutils@^2.3.1, rollup-pluginutils@^2.5.0, rollup-pluginutils@^2.8.1,
   dependencies:
     estree-walker "^0.6.1"
 
-rollup@2.45.2:
-  version "2.45.2"
-  resolved "https://registry.yarnpkg.com/rollup/-/rollup-2.45.2.tgz#8fb85917c9f35605720e92328f3ccbfba6f78b48"
-  integrity sha512-kRRU7wXzFHUzBIv0GfoFFIN3m9oteY4uAsKllIpQDId5cfnkWF2J130l+27dzDju0E6MScKiV0ZM5Bw8m4blYQ==
-  optionalDependencies:
-    fsevents "~2.3.1"
-
 rollup@^2.36.2, rollup@^2.45.2:
   version "2.79.1"
   resolved "https://registry.yarnpkg.com/rollup/-/rollup-2.79.1.tgz#bedee8faef7c9f93a2647ac0108748f497f081c7"
@@ -20747,16 +20740,7 @@ string-similarity@^4.0.4:
   resolved "https://registry.yarnpkg.com/string-similarity/-/string-similarity-4.0.4.tgz#42d01ab0b34660ea8a018da8f56a3309bb8b2a5b"
   integrity sha512-/q/8Q4Bl4ZKAPjj8WerIBJWALKkaPRfrvhfF8k/B23i4nzrlRj2/go1m90In7nG/3XDSbOo0+pu6RvCTM9RGMQ==
 
-"string-width-cjs@npm:string-width@^4.2.0":
-  version "4.2.3"
-  resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010"
-  integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==
-  dependencies:
-    emoji-regex "^8.0.0"
-    is-fullwidth-code-point "^3.0.0"
-    strip-ansi "^6.0.1"
-
-"string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.0.0, string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.2, string-width@^4.2.3:
+"string-width-cjs@npm:string-width@^4.2.0", "string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.0.0, string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.2, string-width@^4.2.3:
   version "4.2.3"
   resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010"
   integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==
@@ -20846,7 +20830,7 @@ stringify-object@^3.2.1:
     is-obj "^1.0.1"
     is-regexp "^1.0.0"
 
-"strip-ansi-cjs@npm:strip-ansi@^6.0.1":
+"strip-ansi-cjs@npm:strip-ansi@^6.0.1", strip-ansi@^6.0.0, strip-ansi@^6.0.1:
   version "6.0.1"
   resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9"
   integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==
@@ -20860,13 +20844,6 @@ strip-ansi@^5.0.0, strip-ansi@^5.1.0, strip-ansi@^5.2.0:
   dependencies:
     ansi-regex "^4.1.0"
 
-strip-ansi@^6.0.0, strip-ansi@^6.0.1:
-  version "6.0.1"
-  resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9"
-  integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==
-  dependencies:
-    ansi-regex "^5.0.1"
-
 strip-ansi@^7.0.1:
   version "7.0.1"
   resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-7.0.1.tgz#61740a08ce36b61e50e65653f07060d000975fb2"
@@ -22803,7 +22780,7 @@ worker-farm@1.7.0:
   dependencies:
     errno "~0.1.7"
 
-"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0":
+"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0", wrap-ansi@^7.0.0:
   version "7.0.0"
   resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43"
   integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==
@@ -22821,15 +22798,6 @@ wrap-ansi@^5.1.0:
     string-width "^3.0.0"
     strip-ansi "^5.0.0"
 
-wrap-ansi@^7.0.0:
-  version "7.0.0"
-  resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43"
-  integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==
-  dependencies:
-    ansi-styles "^4.0.0"
-    string-width "^4.1.0"
-    strip-ansi "^6.0.0"
-
 wrap-ansi@^8.1.0:
   version "8.1.0"
   resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-8.1.0.tgz#56dc22368ee570face1b49819975d9b9a5ead214"

From 070659c7b19d1700e05d6246c982a18a93d5939d Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Fri, 3 May 2024 18:00:43 +0100
Subject: [PATCH 04/12] Fixing an issue with typing + wrong parameter being
 passed to the direct Couch call functionality.

---
 packages/backend-core/src/db/couch/DatabaseImpl.ts | 8 +++++---
 packages/backend-core/src/db/couch/utils.ts        | 2 +-
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/packages/backend-core/src/db/couch/DatabaseImpl.ts b/packages/backend-core/src/db/couch/DatabaseImpl.ts
index e10d0e58c6..b3fce7aa29 100644
--- a/packages/backend-core/src/db/couch/DatabaseImpl.ts
+++ b/packages/backend-core/src/db/couch/DatabaseImpl.ts
@@ -21,6 +21,7 @@ import { WriteStream, ReadStream } from "fs"
 import { newid } from "../../docIds/newid"
 import { SQLITE_DESIGN_DOC_ID } from "../../constants"
 import { DDInstrumentedDatabase } from "../instrumentation"
+import { checkSlashesInUrl } from "../../helpers"
 
 const DATABASE_NOT_FOUND = "Database does not exist."
 
@@ -252,17 +253,18 @@ export class DatabaseImpl implements Database {
   async _sqlQuery<T>(
     url: string,
     method: "POST" | "GET",
-    body?: any
+    body?: Record<string, any>
   ): Promise<T> {
+    url = checkSlashesInUrl(`${this.couchInfo.sqlUrl}/${url}`)
     const args: { url: string; method: string; cookie: string; body?: any } = {
-      url: `${this.couchInfo.sqlUrl}/${url}`,
+      url,
       method,
       cookie: this.couchInfo.cookie,
     }
     if (body) {
       args.body = body
     }
-    const response = await directCouchUrlCall(body)
+    const response = await directCouchUrlCall(args)
     if (response.status > 300) {
       throw new Error(await response.text())
     }
diff --git a/packages/backend-core/src/db/couch/utils.ts b/packages/backend-core/src/db/couch/utils.ts
index 005b02a896..270d953320 100644
--- a/packages/backend-core/src/db/couch/utils.ts
+++ b/packages/backend-core/src/db/couch/utils.ts
@@ -21,7 +21,7 @@ export async function directCouchUrlCall({
   url: string
   cookie: string
   method: string
-  body?: any
+  body?: Record<string, any>
 }) {
   const params: any = {
     method: method,

From a2d4f8523c378ce2a6f610ea0e4143e942aaca43 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Tue, 7 May 2024 18:27:47 +0100
Subject: [PATCH 05/12] Some typing improvements, as well as getting
 deletion/setup working a bit better.

---
 .../backend-core/src/db/couch/DatabaseImpl.ts | 15 ++++++---
 .../backend-core/src/db/instrumentation.ts    |  8 ++++-
 packages/backend-core/src/users/db.ts         |  2 +-
 .../server/src/api/controllers/application.ts |  4 +++
 .../server/src/api/controllers/table/utils.ts |  4 +--
 .../server/src/sdk/app/rows/search/sqs.ts     |  6 +++-
 .../server/src/sdk/app/tables/internal/sqs.ts | 33 ++++++++++++++++---
 packages/types/src/documents/app/sqlite.ts    |  1 +
 packages/types/src/sdk/db.ts                  |  6 ++--
 9 files changed, 60 insertions(+), 19 deletions(-)

diff --git a/packages/backend-core/src/db/couch/DatabaseImpl.ts b/packages/backend-core/src/db/couch/DatabaseImpl.ts
index b3fce7aa29..86fc06c24c 100644
--- a/packages/backend-core/src/db/couch/DatabaseImpl.ts
+++ b/packages/backend-core/src/db/couch/DatabaseImpl.ts
@@ -264,11 +264,16 @@ export class DatabaseImpl implements Database {
     if (body) {
       args.body = body
     }
-    const response = await directCouchUrlCall(args)
-    if (response.status > 300) {
-      throw new Error(await response.text())
-    }
-    return (await response.json()) as T
+    return this.performCall(() => {
+      return async () => {
+        const response = await directCouchUrlCall(args)
+        const json = await response.json()
+        if (response.status > 300) {
+          throw json
+        }
+        return json as T
+      }
+    })
   }
 
   async sql<T extends Document>(
diff --git a/packages/backend-core/src/db/instrumentation.ts b/packages/backend-core/src/db/instrumentation.ts
index 715985766b..ef1c912e0b 100644
--- a/packages/backend-core/src/db/instrumentation.ts
+++ b/packages/backend-core/src/db/instrumentation.ts
@@ -62,7 +62,13 @@ export class DDInstrumentedDatabase implements Database {
   ): Promise<DocumentDestroyResponse> {
     return tracer.trace("db.remove", span => {
       span?.addTags({ db_name: this.name, doc_id: id })
-      return this.db.remove(id, rev)
+      if (typeof id === "object") {
+        return this.db.remove(id)
+      } else if (rev) {
+        return this.db.remove(id, rev)
+      } else {
+        throw new Error("No revision supplied for removal")
+      }
     })
   }
 
diff --git a/packages/backend-core/src/users/db.ts b/packages/backend-core/src/users/db.ts
index f77c6385ba..37547573bd 100644
--- a/packages/backend-core/src/users/db.ts
+++ b/packages/backend-core/src/users/db.ts
@@ -492,7 +492,7 @@ export class UserDB {
 
     await platform.users.removeUser(dbUser)
 
-    await db.remove(userId, dbUser._rev)
+    await db.remove(userId, dbUser._rev!)
 
     const creatorsToDelete = (await isCreator(dbUser)) ? 1 : 0
     await UserDB.quotas.removeUsers(1, creatorsToDelete)
diff --git a/packages/server/src/api/controllers/application.ts b/packages/server/src/api/controllers/application.ts
index e73058239b..6d28be9d8d 100644
--- a/packages/server/src/api/controllers/application.ts
+++ b/packages/server/src/api/controllers/application.ts
@@ -60,6 +60,7 @@ import sdk from "../../sdk"
 import { builderSocket } from "../../websockets"
 import { sdk as sharedCoreSDK } from "@budibase/shared-core"
 import * as appMigrations from "../../appMigrations"
+import { cleanupApp } from "../../sdk/app/tables/internal/sqs"
 
 // utility function, need to do away with this
 async function getLayouts() {
@@ -589,6 +590,9 @@ async function destroyApp(ctx: UserCtx) {
 }
 
 async function preDestroyApp(ctx: UserCtx) {
+  if (env.SQS_SEARCH_ENABLE) {
+    await sdk.tables.sqs.cleanupApp()
+  }
   const { rows } = await getUniqueRows([ctx.params.appId])
   ctx.rowCount = rows.length
 }
diff --git a/packages/server/src/api/controllers/table/utils.ts b/packages/server/src/api/controllers/table/utils.ts
index dfbee2e4fc..a42cfc43c3 100644
--- a/packages/server/src/api/controllers/table/utils.ts
+++ b/packages/server/src/api/controllers/table/utils.ts
@@ -325,7 +325,7 @@ class TableSaveFunctions {
       user: this.user,
     })
     if (env.SQS_SEARCH_ENABLE) {
-      await sdk.tables.sqs.addTableToSqlite(table)
+      await sdk.tables.sqs.addTable(table)
     }
     return table
   }
@@ -519,7 +519,7 @@ export async function internalTableCleanup(table: Table, rows?: Row[]) {
     await AttachmentCleanup.tableDelete(table, rows)
   }
   if (env.SQS_SEARCH_ENABLE) {
-    await sdk.tables.sqs.removeTableFromSqlite(table)
+    await sdk.tables.sqs.removeTable(table)
   }
 }
 
diff --git a/packages/server/src/sdk/app/rows/search/sqs.ts b/packages/server/src/sdk/app/rows/search/sqs.ts
index dabccc4b55..d1230614a7 100644
--- a/packages/server/src/sdk/app/rows/search/sqs.ts
+++ b/packages/server/src/sdk/app/rows/search/sqs.ts
@@ -19,7 +19,7 @@ import {
   sqlOutputProcessing,
 } from "../../../../api/controllers/row/utils"
 import sdk from "../../../index"
-import { context } from "@budibase/backend-core"
+import { context, SQLITE_DESIGN_DOC_ID } from "@budibase/backend-core"
 import {
   CONSTANT_INTERNAL_ROW_COLS,
   SQS_DATASOURCE_INTERNAL,
@@ -195,6 +195,10 @@ export async function search(
     }
   } catch (err: any) {
     const msg = typeof err === "string" ? err : err.message
+    if (err.status === 404 && err.message?.includes(SQLITE_DESIGN_DOC_ID)) {
+      await sdk.tables.sqs.syncDefinition()
+      return search(options, table)
+    }
     throw new Error(`Unable to search by SQL - ${msg}`, { cause: err })
   }
 }
diff --git a/packages/server/src/sdk/app/tables/internal/sqs.ts b/packages/server/src/sdk/app/tables/internal/sqs.ts
index d10a7d447b..3d868b446a 100644
--- a/packages/server/src/sdk/app/tables/internal/sqs.ts
+++ b/packages/server/src/sdk/app/tables/internal/sqs.ts
@@ -15,7 +15,9 @@ import {
   generateJunctionTableID,
 } from "../../../../db/utils"
 
-const BASIC_SQLITE_DOC: SQLiteDefinition = {
+type PreSaveSQLiteDefinition = Omit<SQLiteDefinition, "_rev">
+
+const BASIC_SQLITE_DOC: PreSaveSQLiteDefinition = {
   _id: SQLITE_DESIGN_DOC_ID,
   language: "sqlite",
   sql: {
@@ -102,7 +104,7 @@ function mapTable(table: Table): SQLiteTables {
 }
 
 // nothing exists, need to iterate though existing tables
-async function buildBaseDefinition(): Promise<SQLiteDefinition> {
+async function buildBaseDefinition(): Promise<PreSaveSQLiteDefinition> {
   const tables = await tablesSdk.getAllInternalTables()
   const definition = cloneDeep(BASIC_SQLITE_DOC)
   for (let table of tables) {
@@ -114,9 +116,15 @@ async function buildBaseDefinition(): Promise<SQLiteDefinition> {
   return definition
 }
 
-export async function addTableToSqlite(table: Table) {
+export async function syncDefinition(): Promise<void> {
   const db = context.getAppDB()
-  let definition: SQLiteDefinition
+  const definition = await buildBaseDefinition()
+  await db.put(definition)
+}
+
+export async function addTable(table: Table) {
+  const db = context.getAppDB()
+  let definition: PreSaveSQLiteDefinition | SQLiteDefinition
   try {
     definition = await db.get<SQLiteDefinition>(SQLITE_DESIGN_DOC_ID)
   } catch (err) {
@@ -129,7 +137,7 @@ export async function addTableToSqlite(table: Table) {
   await db.put(definition)
 }
 
-export async function removeTableFromSqlite(table: Table) {
+export async function removeTable(table: Table) {
   const db = context.getAppDB()
   try {
     const definition = await db.get<SQLiteDefinition>(SQLITE_DESIGN_DOC_ID)
@@ -147,3 +155,18 @@ export async function removeTableFromSqlite(table: Table) {
     }
   }
 }
+
+export async function cleanupApp() {
+  const db = context.getAppDB()
+  if (!(await db.exists())) {
+    throw new Error("Cleanup must be preformed before app deletion.")
+  }
+  try {
+    const definition = await db.get<SQLiteDefinition>(SQLITE_DESIGN_DOC_ID)
+    // delete the design document
+    await db.remove(SQLITE_DESIGN_DOC_ID, definition._rev)
+    await db.sqlCleanup()
+  } catch (err: any) {
+    throw new Error(`Unable to cleanup SQS files - ${err.message}`)
+  }
+}
diff --git a/packages/types/src/documents/app/sqlite.ts b/packages/types/src/documents/app/sqlite.ts
index e23a68b336..c380a2d6b4 100644
--- a/packages/types/src/documents/app/sqlite.ts
+++ b/packages/types/src/documents/app/sqlite.ts
@@ -20,6 +20,7 @@ export type SQLiteTables = Record<
 
 export interface SQLiteDefinition {
   _id: string
+  _rev: string
   language: string
   sql: {
     tables: SQLiteTables
diff --git a/packages/types/src/sdk/db.ts b/packages/types/src/sdk/db.ts
index cd4d656a28..150a9a8c21 100644
--- a/packages/types/src/sdk/db.ts
+++ b/packages/types/src/sdk/db.ts
@@ -135,10 +135,8 @@ export interface Database {
     ids: string[],
     opts?: { allowMissing?: boolean }
   ): Promise<T[]>
-  remove(
-    id: string | Document,
-    rev?: string
-  ): Promise<Nano.DocumentDestroyResponse>
+  remove(idOrDoc: Document): Promise<Nano.DocumentDestroyResponse>
+  remove(idOrDoc: string, rev: string): Promise<Nano.DocumentDestroyResponse>
   put(
     document: AnyDocument,
     opts?: DatabasePutOpts

From a3d079f8474e280cc6f9d8c44d12e357ecc7b8a9 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Thu, 9 May 2024 18:33:29 +0100
Subject: [PATCH 06/12] Fixing an issue with cleanup, making sure the correct
 app is cleaned up on deletion.

---
 packages/backend-core/src/db/Replication.ts   | 30 +++++++++++++++++--
 .../server/src/api/controllers/application.ts |  2 +-
 .../server/src/sdk/app/tables/internal/sqs.ts | 10 +++++--
 3 files changed, 36 insertions(+), 6 deletions(-)

diff --git a/packages/backend-core/src/db/Replication.ts b/packages/backend-core/src/db/Replication.ts
index 735c2fa86e..7271662f0f 100644
--- a/packages/backend-core/src/db/Replication.ts
+++ b/packages/backend-core/src/db/Replication.ts
@@ -1,14 +1,34 @@
 import PouchDB from "pouchdb"
 import { getPouchDB, closePouchDB } from "./couch"
-import { DocumentType } from "../constants"
+import { DocumentType } from "@budibase/types"
+
+enum ReplicationDirection {
+  TO_PRODUCTION = "toProduction",
+  TO_DEV = "toDev",
+  UNKNOWN = "unknown",
+}
 
 class Replication {
   source: PouchDB.Database
   target: PouchDB.Database
+  direction: ReplicationDirection
 
   constructor({ source, target }: { source: string; target: string }) {
     this.source = getPouchDB(source)
     this.target = getPouchDB(target)
+    if (
+      source.startsWith(DocumentType.APP_DEV) &&
+      target.startsWith(DocumentType.APP)
+    ) {
+      this.direction = ReplicationDirection.TO_PRODUCTION
+    } else if (
+      source.startsWith(DocumentType.APP) &&
+      target.startsWith(DocumentType.APP_DEV)
+    ) {
+      this.direction = ReplicationDirection.TO_DEV
+    } else {
+      this.direction = ReplicationDirection.UNKNOWN
+    }
   }
 
   async close() {
@@ -40,12 +60,18 @@ class Replication {
     }
 
     const filter = opts.filter
+    const direction = this.direction
+    const toDev = direction === ReplicationDirection.TO_DEV
     delete opts.filter
 
     return {
       ...opts,
       filter: (doc: any, params: any) => {
-        if (doc._id && doc._id.startsWith(DocumentType.AUTOMATION_LOG)) {
+        // don't sync design documents
+        if (toDev && doc._id?.startsWith("_design")) {
+          return false
+        }
+        if (doc._id?.startsWith(DocumentType.AUTOMATION_LOG)) {
           return false
         }
         if (doc._id === DocumentType.APP_METADATA) {
diff --git a/packages/server/src/api/controllers/application.ts b/packages/server/src/api/controllers/application.ts
index 6d28be9d8d..d0908a995e 100644
--- a/packages/server/src/api/controllers/application.ts
+++ b/packages/server/src/api/controllers/application.ts
@@ -591,7 +591,7 @@ async function destroyApp(ctx: UserCtx) {
 
 async function preDestroyApp(ctx: UserCtx) {
   if (env.SQS_SEARCH_ENABLE) {
-    await sdk.tables.sqs.cleanupApp()
+    await sdk.tables.sqs.cleanupApp(ctx.params.appId)
   }
   const { rows } = await getUniqueRows([ctx.params.appId])
   ctx.rowCount = rows.length
diff --git a/packages/server/src/sdk/app/tables/internal/sqs.ts b/packages/server/src/sdk/app/tables/internal/sqs.ts
index 3d868b446a..e6486d3339 100644
--- a/packages/server/src/sdk/app/tables/internal/sqs.ts
+++ b/packages/server/src/sdk/app/tables/internal/sqs.ts
@@ -1,4 +1,8 @@
-import { context, SQLITE_DESIGN_DOC_ID } from "@budibase/backend-core"
+import {
+  context,
+  SQLITE_DESIGN_DOC_ID,
+  db as dbCore,
+} from "@budibase/backend-core"
 import {
   FieldType,
   RelationshipFieldMetadata,
@@ -156,8 +160,8 @@ export async function removeTable(table: Table) {
   }
 }
 
-export async function cleanupApp() {
-  const db = context.getAppDB()
+export async function cleanupApp(appId: string) {
+  const db = dbCore.getDB(appId)
   if (!(await db.exists())) {
     throw new Error("Cleanup must be preformed before app deletion.")
   }

From 8c70c326222a514726a5780dede37bafa2a477df Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Fri, 10 May 2024 11:00:30 +0100
Subject: [PATCH 07/12] Cleanup.

---
 packages/server/src/api/controllers/application.ts | 1 -
 1 file changed, 1 deletion(-)

diff --git a/packages/server/src/api/controllers/application.ts b/packages/server/src/api/controllers/application.ts
index d0908a995e..33a7b28d1e 100644
--- a/packages/server/src/api/controllers/application.ts
+++ b/packages/server/src/api/controllers/application.ts
@@ -60,7 +60,6 @@ import sdk from "../../sdk"
 import { builderSocket } from "../../websockets"
 import { sdk as sharedCoreSDK } from "@budibase/shared-core"
 import * as appMigrations from "../../appMigrations"
-import { cleanupApp } from "../../sdk/app/tables/internal/sqs"
 
 // utility function, need to do away with this
 async function getLayouts() {

From aa51db20ee872709e5613f9ee3ad967995cba6ba Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Tue, 14 May 2024 15:41:48 +0100
Subject: [PATCH 08/12] Changing how cleanup works - the cleanup is now part of
 the DB deletion, making sure it cannot be missed.

---
 .../backend-core/src/db/couch/DatabaseImpl.ts | 13 ++++++++++++
 packages/backend-core/src/environment.ts      |  1 +
 .../server/src/api/controllers/application.ts |  3 ---
 .../server/src/sdk/app/tables/internal/sqs.ts | 21 +------------------
 4 files changed, 15 insertions(+), 23 deletions(-)

diff --git a/packages/backend-core/src/db/couch/DatabaseImpl.ts b/packages/backend-core/src/db/couch/DatabaseImpl.ts
index 7340d10270..711ff8b127 100644
--- a/packages/backend-core/src/db/couch/DatabaseImpl.ts
+++ b/packages/backend-core/src/db/couch/DatabaseImpl.ts
@@ -12,6 +12,7 @@ import {
   isDocument,
   RowResponse,
   RowValue,
+  SQLiteDefinition,
   SqlQueryBinding,
 } from "@budibase/types"
 import { getCouchInfo } from "./connections"
@@ -22,6 +23,7 @@ import { newid } from "../../docIds/newid"
 import { SQLITE_DESIGN_DOC_ID } from "../../constants"
 import { DDInstrumentedDatabase } from "../instrumentation"
 import { checkSlashesInUrl } from "../../helpers"
+import env from "../../environment"
 
 const DATABASE_NOT_FOUND = "Database does not exist."
 
@@ -349,6 +351,17 @@ export class DatabaseImpl implements Database {
 
   async destroy() {
     try {
+      if (env.SQS_SEARCH_ENABLE) {
+        // delete the design document, then run the cleanup operation
+        try {
+          const definition = await this.get<SQLiteDefinition>(
+            SQLITE_DESIGN_DOC_ID
+          )
+          await this.remove(SQLITE_DESIGN_DOC_ID, definition._rev)
+        } finally {
+          await this.sqlCleanup()
+        }
+      }
       return await this.nano().db.destroy(this.name)
     } catch (err: any) {
       // didn't exist, don't worry
diff --git a/packages/backend-core/src/environment.ts b/packages/backend-core/src/environment.ts
index 9ade81b9d7..20ff16739f 100644
--- a/packages/backend-core/src/environment.ts
+++ b/packages/backend-core/src/environment.ts
@@ -109,6 +109,7 @@ const environment = {
   API_ENCRYPTION_KEY: getAPIEncryptionKey(),
   COUCH_DB_URL: process.env.COUCH_DB_URL || "http://localhost:4005",
   COUCH_DB_SQL_URL: process.env.COUCH_DB_SQL_URL || "http://localhost:4006",
+  SQS_SEARCH_ENABLE: process.env.SQS_SEARCH_ENABLE,
   COUCH_DB_USERNAME: process.env.COUCH_DB_USER,
   COUCH_DB_PASSWORD: process.env.COUCH_DB_PASSWORD,
   GOOGLE_CLIENT_ID: process.env.GOOGLE_CLIENT_ID,
diff --git a/packages/server/src/api/controllers/application.ts b/packages/server/src/api/controllers/application.ts
index 33a7b28d1e..e73058239b 100644
--- a/packages/server/src/api/controllers/application.ts
+++ b/packages/server/src/api/controllers/application.ts
@@ -589,9 +589,6 @@ async function destroyApp(ctx: UserCtx) {
 }
 
 async function preDestroyApp(ctx: UserCtx) {
-  if (env.SQS_SEARCH_ENABLE) {
-    await sdk.tables.sqs.cleanupApp(ctx.params.appId)
-  }
   const { rows } = await getUniqueRows([ctx.params.appId])
   ctx.rowCount = rows.length
 }
diff --git a/packages/server/src/sdk/app/tables/internal/sqs.ts b/packages/server/src/sdk/app/tables/internal/sqs.ts
index 5554d322b4..6e326f896f 100644
--- a/packages/server/src/sdk/app/tables/internal/sqs.ts
+++ b/packages/server/src/sdk/app/tables/internal/sqs.ts
@@ -1,8 +1,4 @@
-import {
-  context,
-  SQLITE_DESIGN_DOC_ID,
-  db as dbCore,
-} from "@budibase/backend-core"
+import { context, SQLITE_DESIGN_DOC_ID } from "@budibase/backend-core"
 import {
   FieldType,
   RelationshipFieldMetadata,
@@ -160,18 +156,3 @@ export async function removeTable(table: Table) {
     }
   }
 }
-
-export async function cleanupApp(appId: string) {
-  const db = dbCore.getDB(appId)
-  if (!(await db.exists())) {
-    throw new Error("Cleanup must be preformed before app deletion.")
-  }
-  try {
-    const definition = await db.get<SQLiteDefinition>(SQLITE_DESIGN_DOC_ID)
-    // delete the design document
-    await db.remove(SQLITE_DESIGN_DOC_ID, definition._rev)
-    await db.sqlCleanup()
-  } catch (err: any) {
-    throw new Error(`Unable to cleanup SQS files - ${err.message}`)
-  }
-}

From 6b8d52def16592823878a31f1c37491e87b06ab1 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Wed, 15 May 2024 12:00:15 +0100
Subject: [PATCH 09/12] Linting and build issues.

---
 packages/server/src/sdk/app/tables/internal/index.ts | 2 --
 packages/types/src/sdk/db.ts                         | 2 +-
 2 files changed, 1 insertion(+), 3 deletions(-)

diff --git a/packages/server/src/sdk/app/tables/internal/index.ts b/packages/server/src/sdk/app/tables/internal/index.ts
index caddd3ee94..ea40d2bfe9 100644
--- a/packages/server/src/sdk/app/tables/internal/index.ts
+++ b/packages/server/src/sdk/app/tables/internal/index.ts
@@ -22,8 +22,6 @@ import { checkAutoColumns } from "./utils"
 import * as viewsSdk from "../../views"
 import { getRowParams } from "../../../../db/utils"
 import { quotas } from "@budibase/pro"
-import env from "../../../../environment"
-import { AttachmentCleanup } from "../../../../utilities/rowProcessor"
 
 export async function save(
   table: Table,
diff --git a/packages/types/src/sdk/db.ts b/packages/types/src/sdk/db.ts
index 150a9a8c21..4a4343e59f 100644
--- a/packages/types/src/sdk/db.ts
+++ b/packages/types/src/sdk/db.ts
@@ -136,7 +136,7 @@ export interface Database {
     opts?: { allowMissing?: boolean }
   ): Promise<T[]>
   remove(idOrDoc: Document): Promise<Nano.DocumentDestroyResponse>
-  remove(idOrDoc: string, rev: string): Promise<Nano.DocumentDestroyResponse>
+  remove(idOrDoc: string, rev?: string): Promise<Nano.DocumentDestroyResponse>
   put(
     document: AnyDocument,
     opts?: DatabasePutOpts

From 06d6d84b556d8070d4c524ce6b6051e23413e22f Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Thu, 16 May 2024 13:27:54 +0100
Subject: [PATCH 10/12] Addressing PR comments.

---
 packages/backend-core/src/db/Replication.ts   |  5 +---
 .../backend-core/src/db/couch/DatabaseImpl.ts |  8 +++--
 .../backend-core/src/db/instrumentation.ts    | 29 +++++++++----------
 .../server/src/sdk/app/tables/internal/sqs.ts |  2 +-
 packages/types/src/sdk/db.ts                  |  4 +--
 5 files changed, 22 insertions(+), 26 deletions(-)

diff --git a/packages/backend-core/src/db/Replication.ts b/packages/backend-core/src/db/Replication.ts
index 7271662f0f..617269df10 100644
--- a/packages/backend-core/src/db/Replication.ts
+++ b/packages/backend-core/src/db/Replication.ts
@@ -5,13 +5,12 @@ import { DocumentType } from "@budibase/types"
 enum ReplicationDirection {
   TO_PRODUCTION = "toProduction",
   TO_DEV = "toDev",
-  UNKNOWN = "unknown",
 }
 
 class Replication {
   source: PouchDB.Database
   target: PouchDB.Database
-  direction: ReplicationDirection
+  direction: ReplicationDirection | undefined
 
   constructor({ source, target }: { source: string; target: string }) {
     this.source = getPouchDB(source)
@@ -26,8 +25,6 @@ class Replication {
       target.startsWith(DocumentType.APP_DEV)
     ) {
       this.direction = ReplicationDirection.TO_DEV
-    } else {
-      this.direction = ReplicationDirection.UNKNOWN
     }
   }
 
diff --git a/packages/backend-core/src/db/couch/DatabaseImpl.ts b/packages/backend-core/src/db/couch/DatabaseImpl.ts
index 711ff8b127..8194d1aabf 100644
--- a/packages/backend-core/src/db/couch/DatabaseImpl.ts
+++ b/packages/backend-core/src/db/couch/DatabaseImpl.ts
@@ -323,14 +323,16 @@ export class DatabaseImpl implements Database {
   }
 
   // checks design document is accurate (cleans up tables)
-  async sqlCleanup(): Promise<void> {
+  // this will check the design document and remove anything from
+  // disk which is not supposed to be there
+  async sqlDiskCleanup(): Promise<void> {
     const dbName = this.name
     const url = `/${dbName}/_cleanup`
     return await this._sqlQuery<void>(url, "POST")
   }
 
   // removes a document from sqlite
-  async sqlPurge(docIds: string[] | string): Promise<void> {
+  async sqlPurgeDocument(docIds: string[] | string): Promise<void> {
     if (!Array.isArray(docIds)) {
       docIds = [docIds]
     }
@@ -359,7 +361,7 @@ export class DatabaseImpl implements Database {
           )
           await this.remove(SQLITE_DESIGN_DOC_ID, definition._rev)
         } finally {
-          await this.sqlCleanup()
+          await this.sqlDiskCleanup()
         }
       }
       return await this.nano().db.destroy(this.name)
diff --git a/packages/backend-core/src/db/instrumentation.ts b/packages/backend-core/src/db/instrumentation.ts
index ef1c912e0b..dc0fd67b37 100644
--- a/packages/backend-core/src/db/instrumentation.ts
+++ b/packages/backend-core/src/db/instrumentation.ts
@@ -56,19 +56,16 @@ export class DDInstrumentedDatabase implements Database {
     })
   }
 
+  remove(idOrDoc: Document): Promise<DocumentDestroyResponse>
+  remove(idOrDoc: string, rev?: string): Promise<DocumentDestroyResponse>
   remove(
-    id: string | Document,
-    rev?: string | undefined
+    idOrDoc: string | Document,
+    rev?: string
   ): Promise<DocumentDestroyResponse> {
     return tracer.trace("db.remove", span => {
-      span?.addTags({ db_name: this.name, doc_id: id })
-      if (typeof id === "object") {
-        return this.db.remove(id)
-      } else if (rev) {
-        return this.db.remove(id, rev)
-      } else {
-        throw new Error("No revision supplied for removal")
-      }
+      span?.addTags({ db_name: this.name, doc_id: idOrDoc })
+      const id: string = typeof idOrDoc === "object" ? idOrDoc._id! : idOrDoc
+      return this.db.remove(id, rev)
     })
   }
 
@@ -167,17 +164,17 @@ export class DDInstrumentedDatabase implements Database {
     })
   }
 
-  sqlPurge(docIds: string[] | string): Promise<void> {
-    return tracer.trace("db.sqlPurge", span => {
+  sqlPurgeDocument(docIds: string[] | string): Promise<void> {
+    return tracer.trace("db.sqlPurgeDocument", span => {
       span?.addTags({ db_name: this.name })
-      return this.db.sqlPurge(docIds)
+      return this.db.sqlPurgeDocument(docIds)
     })
   }
 
-  sqlCleanup(): Promise<void> {
-    return tracer.trace("db.sqlCleanup", span => {
+  sqlDiskCleanup(): Promise<void> {
+    return tracer.trace("db.sqlDiskCleanup", span => {
       span?.addTags({ db_name: this.name })
-      return this.db.sqlCleanup()
+      return this.db.sqlDiskCleanup()
     })
   }
 }
diff --git a/packages/server/src/sdk/app/tables/internal/sqs.ts b/packages/server/src/sdk/app/tables/internal/sqs.ts
index 6e326f896f..5ecfd9692e 100644
--- a/packages/server/src/sdk/app/tables/internal/sqs.ts
+++ b/packages/server/src/sdk/app/tables/internal/sqs.ts
@@ -146,7 +146,7 @@ export async function removeTable(table: Table) {
       delete definition.sql.tables[table._id!]
       await db.put(definition)
       // make sure SQS is cleaned up, tables removed
-      await db.sqlCleanup()
+      await db.sqlDiskCleanup()
     }
   } catch (err: any) {
     if (err?.status === 404) {
diff --git a/packages/types/src/sdk/db.ts b/packages/types/src/sdk/db.ts
index 4a4343e59f..7ad740ad05 100644
--- a/packages/types/src/sdk/db.ts
+++ b/packages/types/src/sdk/db.ts
@@ -146,8 +146,8 @@ export interface Database {
     sql: string,
     parameters?: SqlQueryBinding
   ): Promise<T[]>
-  sqlPurge(docIds: string[] | string): Promise<void>
-  sqlCleanup(): Promise<void>
+  sqlPurgeDocument(docIds: string[] | string): Promise<void>
+  sqlDiskCleanup(): Promise<void>
   allDocs<T extends Document | RowValue>(
     params: DatabaseQueryOpts
   ): Promise<AllDocsResponse<T>>

From 39ce040b1c659c82739f135343b3d22b05731c30 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Thu, 16 May 2024 13:39:16 +0100
Subject: [PATCH 11/12] Updating eslint settings to allow function overloading
 in Typescript.

---
 .eslintrc.json | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/.eslintrc.json b/.eslintrc.json
index d475bba8d1..9dab2f1a88 100644
--- a/.eslintrc.json
+++ b/.eslintrc.json
@@ -55,7 +55,9 @@
           }
         ],
         "no-redeclare": "off",
-        "@typescript-eslint/no-redeclare": "error"
+        "@typescript-eslint/no-redeclare": "error",
+        // have to turn this off to allow function overloading in typescript
+        "no-dupe-class-members": "off"
       }
     },
     {
@@ -88,7 +90,9 @@
         "jest/expect-expect": "off",
         // We do this in some tests where the behaviour of internal tables
         // differs to external, but the API is broadly the same
-        "jest/no-conditional-expect": "off"
+        "jest/no-conditional-expect": "off",
+        // have to turn this off to allow function overloading in typescript
+        "no-dupe-class-members": "off"
       }
     },
     {

From 647a8c2a74f136f78ca0adb03ce9211b14d98fb5 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Thu, 16 May 2024 13:52:38 +0100
Subject: [PATCH 12/12] Final fix for remove function.

---
 packages/backend-core/src/db/instrumentation.ts | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/packages/backend-core/src/db/instrumentation.ts b/packages/backend-core/src/db/instrumentation.ts
index dc0fd67b37..4e2b147ef3 100644
--- a/packages/backend-core/src/db/instrumentation.ts
+++ b/packages/backend-core/src/db/instrumentation.ts
@@ -64,7 +64,9 @@ export class DDInstrumentedDatabase implements Database {
   ): Promise<DocumentDestroyResponse> {
     return tracer.trace("db.remove", span => {
       span?.addTags({ db_name: this.name, doc_id: idOrDoc })
-      const id: string = typeof idOrDoc === "object" ? idOrDoc._id! : idOrDoc
+      const isDocument = typeof idOrDoc === "object"
+      const id = isDocument ? idOrDoc._id! : idOrDoc
+      rev = isDocument ? idOrDoc._rev : rev
       return this.db.remove(id, rev)
     })
   }