Account for duplicate row ids caused by app import

packages/server/src/middleware/usageQuota.js

@@ -129,8 +129,8 @@ const appPreDelete = async (ctx, usageContext) => {
 
   // store the row count to delete
   const rows = await getUniqueRows([ctx.appId])
-  if (rows.size) {
+  if (rows.length) {
-    usageContext[usageQuota.Properties.APPS] = { rowCount: rows.size }
+    usageContext[usageQuota.Properties.APPS] = { rowCount: rows.length }
   }
 }
 

packages/server/src/migrations/sync_app_and_reset_rows_quotas.js

@@ -17,7 +17,7 @@ const syncRowsQuota = async db => {
 
   // sync row count
   const usageDoc = await getUsageQuotaDoc(db)
-  usageDoc.usageQuota.rows = rows.size
+  usageDoc.usageQuota.rows = rows.length
   await db.put(usageDoc)
 }
 

packages/server/src/utilities/usageQuota/rows.js

@@ -1,27 +1,35 @@
 const { getRowParams, USER_METDATA_PREFIX } = require("../../db/utils")
 const CouchDB = require("../../db")
+const { isDevAppID, getDevelopmentAppID } = require("@budibase/backend-core/db")
 
 const ROW_EXCLUSIONS = [USER_METDATA_PREFIX]
 
-/**
+const getAppPairs = appIds => {
- * Get all rows in the given app ids.
+  // collect the app ids into dev / prod pairs
- *
+  // keyed by the dev app id
- * The returned rows may contan duplicates if there
+  const pairs = {}
- * is a production and dev app.
- */
-const getAllRows = async appIds => {
-  const allRows = []
-  let appDb
   for (let appId of appIds) {
-    try {
+    const devId = getDevelopmentAppID(appId)
-      appDb = new CouchDB(appId)
+    if (!pairs[devId]) {
+      pairs[devId] = {}
+    }
+    if (isDevAppID(appId)) {
+      pairs[devId].devId = appId
+    } else {
+      pairs[devId].prodId = appId
+    }
+  }
+  return pairs
+}
+
+const getAppRows = async appId => {
+  const appDb = new CouchDB(appId)
   const response = await appDb.allDocs(
     getRowParams(null, null, {
       include_docs: false,
     })
   )
-      allRows.push(
+  return response.rows
-        ...response.rows
     .map(r => r.id)
     .filter(id => {
       for (let exclusion of ROW_EXCLUSIONS) {
@@ -31,22 +39,35 @@ const getAllRows = async appIds => {
       }
       return true
     })
-      )
+}
+
+/**
+ * Return a set of all rows in the given app ids.
+ * The returned rows will be unique on a per dev/prod app basis.
+ * Rows duplicates may exist across apps due to data import so they are not filtered out.
+ */
+exports.getUniqueRows = async appIds => {
+  let uniqueRows = []
+  const pairs = getAppPairs(appIds)
+
+  for (let pair of Object.values(pairs)) {
+    let appRows = []
+    for (let appId of [pair.devId, pair.prodId]) {
+      if (!appId) {
+        continue
+      }
+      try {
+        appRows.push(await getAppRows(appId))
       } catch (e) {
         // don't error out if we can't count the app rows, just continue
       }
     }
 
-  return allRows
+    // ensure uniqueness on a per app pair basis
+    // this can't be done on all rows because app import results in
+    // duplicate row ids across apps
+    uniqueRows = uniqueRows.concat(...new Set(appRows))
   }
 
-/**
+  return uniqueRows
- * Get all rows in the given app ids.
- *
- * The returned rows will be unique, duplicated rows across
- * production and dev apps will be removed.
- */
-exports.getUniqueRows = async appIds => {
-  const allRows = await getAllRows(appIds)
-  return new Set(allRows)
 }