Merge pull request #5881 from Budibase/fix/5850

Fixing issues with calculation views when self hosting and using real CouchDB views

packages/backend-core/src/middleware/passport/tests/oidc.spec.js

@@ -71,7 +71,7 @@ describe("oidc", () => {
   
   describe("authenticate", () => {    
     afterEach(() => {
-      jest.clearAllMocks();
+      jest.clearAllMocks()
     });
 
     // mock third party common authentication
@@ -80,10 +80,10 @@ describe("oidc", () => {
     
     // mock the passport callback
     const mockDone = jest.fn()
+    const mockSaveUserFn = jest.fn()
 
     async function doAuthenticate() {
       const oidc = require("../oidc")
-      const mockSaveUserFn = jest.fn()
       const authenticate = await oidc.buildVerifyFn(mockSaveUserFn)
 
       await authenticate(
@@ -105,11 +105,13 @@ describe("oidc", () => {
       expect(authenticateThirdParty).toHaveBeenCalledWith(
         user,
         false, 
-        mockDone)
+        mockDone,
+        mockSaveUserFn,
+      )
     }
 
     it("delegates authentication to third party common", async () => {
-      doTest()
+      await doTest()
     })
 
     it("uses JWT email to get email", async () => {
@@ -118,7 +120,7 @@ describe("oidc", () => {
         email : "mock@budibase.com"
       }
 
-      doTest()
+      await doTest()
     })
   
     it("uses JWT username to get email", async () => {
@@ -127,7 +129,7 @@ describe("oidc", () => {
         preferred_username : "mock@budibase.com"
       }
 
-      doTest()
+      await doTest()
     })
 
     it("uses JWT invalid username to get email", async () => {

packages/server/src/api/controllers/view/tests/__snapshots__/viewBuilder.spec.js.snap

@@ -1,9 +1,66 @@
 // Jest Snapshot v1, https://goo.gl/fbAQLP
 
+exports[`viewBuilder Calculate and filter creates a view with the calculation statistics and filter schema 1`] = `
+Object {
+  "map": "function (doc) {
+      if ((doc.tableId === \\"14f1c4e94d6a47b682ce89d35d4c78b0\\" && !(
+      doc[\\"myField\\"] === undefined ||
+      doc[\\"myField\\"] === null ||
+      doc[\\"myField\\"] === \\"\\" ||
+      (Array.isArray(doc[\\"myField\\"]) && doc[\\"myField\\"].length === 0)
+  )) && (doc[\\"age\\"] > 17)) {
+        emit(doc[\\"_id\\"], doc[\\"myField\\"]);
+      }
+    }",
+  "meta": Object {
+    "calculation": "stats",
+    "field": "myField",
+    "filters": Array [
+      Object {
+        "condition": "MT",
+        "key": "age",
+        "value": 17,
+      },
+    ],
+    "groupBy": undefined,
+    "schema": Object {
+      "avg": Object {
+        "type": "number",
+      },
+      "count": Object {
+        "type": "number",
+      },
+      "field": Object {
+        "type": "string",
+      },
+      "max": Object {
+        "type": "number",
+      },
+      "min": Object {
+        "type": "number",
+      },
+      "sum": Object {
+        "type": "number",
+      },
+      "sumsqr": Object {
+        "type": "number",
+      },
+    },
+    "tableId": "14f1c4e94d6a47b682ce89d35d4c78b0",
+  },
+  "reduce": "_stats",
+}
+`;
+
 exports[`viewBuilder Calculate creates a view with the calculation statistics schema 1`] = `
 Object {
   "map": "function (doc) {
-      if (doc.tableId === \\"14f1c4e94d6a47b682ce89d35d4c78b0\\" ) {
+      if ((doc.tableId === \\"14f1c4e94d6a47b682ce89d35d4c78b0\\" && !(
+      doc[\\"myField\\"] === undefined ||
+      doc[\\"myField\\"] === null ||
+      doc[\\"myField\\"] === \\"\\" ||
+      (Array.isArray(doc[\\"myField\\"]) && doc[\\"myField\\"].length === 0)
+  )) ) {
         emit(doc[\\"_id\\"], doc[\\"myField\\"]);
       }
     }",

packages/server/src/api/controllers/view/tests/viewBuilder.spec.js

@@ -44,4 +44,22 @@ describe("viewBuilder", () => {
       })).toMatchSnapshot()
     })
   })
+
+  describe("Calculate and filter", () => {
+    it("creates a view with the calculation statistics and filter schema", () => {
+      expect(viewTemplate({
+        "name": "Calculate View",
+        "field": "myField",
+        "calculation": "stats",
+        "tableId": "14f1c4e94d6a47b682ce89d35d4c78b0",
+        "filters": [
+          {
+            "value": 17,
+            "condition": "MT",
+            "key": "age",
+          }
+        ]
+      })).toMatchSnapshot()
+    })
+  })
 });

packages/server/src/api/controllers/view/utils.js

@@ -7,6 +7,7 @@ const {
 } = require("../../../db/utils")
 const env = require("../../../environment")
 const { getAppDB } = require("@budibase/backend-core/context")
+const viewBuilder = require("./viewBuilder")
 
 exports.getView = async viewName => {
   const db = getAppDB()
@@ -114,7 +115,8 @@ exports.deleteView = async viewName => {
 exports.migrateToInMemoryView = async (db, viewName) => {
   // delete the view initially
   const designDoc = await db.get("_design/database")
-  const view = designDoc.views[viewName]
+  // run the view back through the view builder to update it
+  const view = viewBuilder(designDoc.views[viewName].meta)
   delete designDoc.views[viewName]
   await db.put(designDoc)
   await exports.saveView(db, null, viewName, view)
@@ -123,7 +125,7 @@ exports.migrateToInMemoryView = async (db, viewName) => {
 exports.migrateToDesignView = async (db, viewName) => {
   let view = await db.get(generateMemoryViewID(viewName))
   const designDoc = await db.get("_design/database")
-  designDoc.views[viewName] = view.view
+  designDoc.views[viewName] = viewBuilder(view.view.meta)
   await db.put(designDoc)
   await db.remove(view._id, view._rev)
 }

packages/server/src/api/controllers/view/viewBuilder.js

@@ -10,6 +10,12 @@ const TOKEN_MAP = {
   OR: "||",
 }
 
+const CONDITIONS = {
+  EMPTY: "EMPTY",
+  NOT_EMPTY: "NOT_EMPTY",
+  CONTAINS: "CONTAINS",
+}
+
 const isEmptyExpression = key => {
   return `(
       doc["${key}"] === undefined ||
@@ -77,13 +83,13 @@ function parseFilterExpression(filters) {
       expression.push(TOKEN_MAP[filter.conjunction])
     }
 
-    if (filter.condition === "CONTAINS") {
+    if (filter.condition === CONDITIONS.CONTAINS) {
       expression.push(
         `doc["${filter.key}"].${TOKEN_MAP[filter.condition]}("${filter.value}")`
       )
-    } else if (filter.condition === "EMPTY") {
+    } else if (filter.condition === CONDITIONS.EMPTY) {
       expression.push(isEmptyExpression(filter.key))
-    } else if (filter.condition === "NOT_EMPTY") {
+    } else if (filter.condition === CONDITIONS.NOT_EMPTY) {
       expression.push(`!${isEmptyExpression(filter.key)}`)
     } else {
       const value =
@@ -125,21 +131,36 @@ function viewTemplate({ field, tableId, groupBy, filters = [], calculation }) {
   if (filters && filters.length > 0 && filters[0].conjunction) {
     delete filters[0].conjunction
   }
-  const parsedFilters = parseFilterExpression(filters)
-  const filterExpression = parsedFilters ? `&& (${parsedFilters})` : ""
 
-  const emitExpression = parseEmitExpression(field, groupBy)
+  let schema = null,
-
+    statFilter = null
-  const reduction = field && calculation ? { reduce: `_${calculation}` } : {}
-
-  let schema = null
 
   if (calculation) {
     schema = {
       ...(groupBy ? GROUP_PROPERTY : FIELD_PROPERTY),
       ...SCHEMA_MAP[calculation],
     }
+    if (
+      !filters.find(
+        filter =>
+          filter.key === field && filter.condition === CONDITIONS.NOT_EMPTY
+      )
+    ) {
+      statFilter = parseFilterExpression([
+        { key: field, condition: CONDITIONS.NOT_EMPTY },
+      ])
     }
+  }
+
+  const parsedFilters = parseFilterExpression(filters)
+  const filterExpression = parsedFilters ? `&& (${parsedFilters})` : ""
+
+  const emitExpression = parseEmitExpression(field, groupBy)
+  const tableExpression = `doc.tableId === "${tableId}"`
+  const coreExpression = statFilter
+    ? `(${tableExpression} && ${statFilter})`
+    : tableExpression
+  const reduction = field && calculation ? { reduce: `_${calculation}` } : {}
 
   return {
     meta: {
@@ -151,7 +172,7 @@ function viewTemplate({ field, tableId, groupBy, filters = [], calculation }) {
       calculation,
     },
     map: `function (doc) {
-      if (doc.tableId === "${tableId}" ${filterExpression}) {
+      if (${coreExpression} ${filterExpression}) {
         ${emitExpression}
       }
     }`,

packages/server/yarn.lock

@@ -1014,10 +1014,10 @@
   resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
   integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==
 
-"@budibase/backend-core@1.0.154":
+"@budibase/backend-core@1.0.160":
-  version "1.0.154"
+  version "1.0.160"
-  resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.154.tgz#c310834892e7621778b07579464955487c5c9830"
+  resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.160.tgz#000e3b5a3ed91e73a542b4caa202a6f147d91294"
-  integrity sha512-mcZxt8XhGgOB4XRHKkWTvBEI4HGp2bo8qyzOJRCvDqlg56S9zqGJDl75Z0N/Wc8N3I53QRcxISerj48odX172A==
+  integrity sha512-XfAFU6sRPrCSEKlm58WeuPw8lUoJK+KwO0tcbT+bB2Nb7XCHplskryEgk/PM9ujRU6SMPDx11zKeqRebHlycbA==
   dependencies:
     "@techpass/passport-openidconnect" "^0.3.0"
     aws-sdk "^2.901.0"
@@ -1091,12 +1091,12 @@
     svelte-flatpickr "^3.2.3"
     svelte-portal "^1.0.0"
 
-"@budibase/pro@1.0.154":
+"@budibase/pro@1.0.160":
-  version "1.0.154"
+  version "1.0.160"
-  resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.154.tgz#f4e31e30376b206159b711224038141d73a1118e"
+  resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.160.tgz#921c4e3f65b866d84292644dfd8793c4d0b667c7"
-  integrity sha512-+O6bemrcgyWG4a+D5dIOoZ+LGjW4aN7tRdFeZqoaIPCc1pA6zNtLUkM1nb+Laafuwq2Aht37vEuaRy7jfzVprA==
+  integrity sha512-p+Jhnk1n98CWCJXydSQSO7a+HDpqGAHekGQbOR7aayuwuoYzyOXxTcHNLdBp+3lkXhLSZq9oIwfEGpgdrrhXPA==
   dependencies:
-    "@budibase/backend-core" "1.0.154"
+    "@budibase/backend-core" "1.0.160"
     node-fetch "^2.6.1"
 
 "@budibase/standard-components@^0.9.139":

packages/worker/yarn.lock

@@ -293,10 +293,10 @@
   resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
   integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==
 
-"@budibase/backend-core@1.0.154":
+"@budibase/backend-core@1.0.160":
-  version "1.0.154"
+  version "1.0.160"
-  resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.154.tgz#c310834892e7621778b07579464955487c5c9830"
+  resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.160.tgz#000e3b5a3ed91e73a542b4caa202a6f147d91294"
-  integrity sha512-mcZxt8XhGgOB4XRHKkWTvBEI4HGp2bo8qyzOJRCvDqlg56S9zqGJDl75Z0N/Wc8N3I53QRcxISerj48odX172A==
+  integrity sha512-XfAFU6sRPrCSEKlm58WeuPw8lUoJK+KwO0tcbT+bB2Nb7XCHplskryEgk/PM9ujRU6SMPDx11zKeqRebHlycbA==
   dependencies:
     "@techpass/passport-openidconnect" "^0.3.0"
     aws-sdk "^2.901.0"
@@ -321,12 +321,12 @@
     uuid "^8.3.2"
     zlib "^1.0.5"
 
-"@budibase/pro@1.0.154":
+"@budibase/pro@1.0.160":
-  version "1.0.154"
+  version "1.0.160"
-  resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.154.tgz#f4e31e30376b206159b711224038141d73a1118e"
+  resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.160.tgz#921c4e3f65b866d84292644dfd8793c4d0b667c7"
-  integrity sha512-+O6bemrcgyWG4a+D5dIOoZ+LGjW4aN7tRdFeZqoaIPCc1pA6zNtLUkM1nb+Laafuwq2Aht37vEuaRy7jfzVprA==
+  integrity sha512-p+Jhnk1n98CWCJXydSQSO7a+HDpqGAHekGQbOR7aayuwuoYzyOXxTcHNLdBp+3lkXhLSZq9oIwfEGpgdrrhXPA==
   dependencies:
-    "@budibase/backend-core" "1.0.154"
+    "@budibase/backend-core" "1.0.160"
     node-fetch "^2.6.1"
 
 "@cspotcode/source-map-consumer@0.8.0":