MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #5881 from Budibase/fix/5850 

Fixing issues with calculation views when self hosting and using real CouchDB views
This commit is contained in:
Martin McKeaveney 2022-05-16 17:40:42 +01:00 committed by GitHub
parent 76670f248e 7b07cff0c6
commit 931c14198f
7 changed files with 138 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
packages/backend-core/src/middleware/passport/tests/oidc.spec.js
View File

@ -71,7 +71,7 @@ describe("oidc", () => {
describe("authenticate", () => {
afterEach(() => {
jest.clearAllMocks();
jest.clearAllMocks()
});
// mock third party common authentication
@ -80,10 +80,10 @@ describe("oidc", () => {
// mock the passport callback
const mockDone = jest.fn()
const mockSaveUserFn = jest.fn()
async function doAuthenticate() {
const oidc = require("../oidc")
const mockSaveUserFn = jest.fn()
const authenticate = await oidc.buildVerifyFn(mockSaveUserFn)
await authenticate(
@ -105,11 +105,13 @@ describe("oidc", () => {
expect(authenticateThirdParty).toHaveBeenCalledWith(
user,
false,
mockDone)
mockDone,
mockSaveUserFn,
)
}
it("delegates authentication to third party common", async () => {
doTest()
await doTest()
})
it("uses JWT email to get email", async () => {
@ -118,7 +120,7 @@ describe("oidc", () => {
email : "mock@budibase.com"
}
doTest()
await doTest()
})
it("uses JWT username to get email", async () => {
@ -127,7 +129,7 @@ describe("oidc", () => {
preferred_username : "mock@budibase.com"
}
doTest()
await doTest()
})
it("uses JWT invalid username to get email", async () => {

59
packages/server/src/api/controllers/view/tests/__snapshots__/viewBuilder.spec.js.snap
View File

@ -1,9 +1,66 @@
// Jest Snapshot v1, https://goo.gl/fbAQLP
exports[`viewBuilder Calculate and filter creates a view with the calculation statistics and filter schema 1`] = `
Object {
"map": "function (doc) {
if ((doc.tableId === \\"14f1c4e94d6a47b682ce89d35d4c78b0\\" && !(
doc[\\"myField\\"] === undefined ||
doc[\\"myField\\"] === null ||
doc[\\"myField\\"] === \\"\\" ||
(Array.isArray(doc[\\"myField\\"]) && doc[\\"myField\\"].length === 0)
)) && (doc[\\"age\\"] > 17)) {
emit(doc[\\"_id\\"], doc[\\"myField\\"]);
}
}",
"meta": Object {
"calculation": "stats",
"field": "myField",
"filters": Array [
Object {
"condition": "MT",
"key": "age",
"value": 17,
},
],
"groupBy": undefined,
"schema": Object {
"avg": Object {
"type": "number",
},
"count": Object {
"type": "number",
},
"field": Object {
"type": "string",
},
"max": Object {
"type": "number",
},
"min": Object {
"type": "number",
},
"sum": Object {
"type": "number",
},
"sumsqr": Object {
"type": "number",
},
},
"tableId": "14f1c4e94d6a47b682ce89d35d4c78b0",
},
"reduce": "_stats",
}
`;
exports[`viewBuilder Calculate creates a view with the calculation statistics schema 1`] = `
Object {
"map": "function (doc) {
if (doc.tableId === \\"14f1c4e94d6a47b682ce89d35d4c78b0\\" ) {
if ((doc.tableId === \\"14f1c4e94d6a47b682ce89d35d4c78b0\\" && !(
doc[\\"myField\\"] === undefined ||
doc[\\"myField\\"] === null ||
doc[\\"myField\\"] === \\"\\" ||
(Array.isArray(doc[\\"myField\\"]) && doc[\\"myField\\"].length === 0)
)) ) {
emit(doc[\\"_id\\"], doc[\\"myField\\"]);
}
}",

18
packages/server/src/api/controllers/view/tests/viewBuilder.spec.js
View File

@ -44,4 +44,22 @@ describe("viewBuilder", () => {
})).toMatchSnapshot()
})
})
describe("Calculate and filter", () => {
it("creates a view with the calculation statistics and filter schema", () => {
expect(viewTemplate({
"name": "Calculate View",
"field": "myField",
"calculation": "stats",
"tableId": "14f1c4e94d6a47b682ce89d35d4c78b0",
"filters": [
{
"value": 17,
"condition": "MT",
"key": "age",
}
]
})).toMatchSnapshot()
})
})
});

6
packages/server/src/api/controllers/view/utils.js
View File

@ -7,6 +7,7 @@ const {
} = require("../../../db/utils")
const env = require("../../../environment")
const { getAppDB } = require("@budibase/backend-core/context")
const viewBuilder = require("./viewBuilder")
exports.getView = async viewName => {
const db = getAppDB()
@ -114,7 +115,8 @@ exports.deleteView = async viewName => {
exports.migrateToInMemoryView = async (db, viewName) => {
// delete the view initially
const designDoc = await db.get("_design/database")
const view = designDoc.views[viewName]
// run the view back through the view builder to update it
const view = viewBuilder(designDoc.views[viewName].meta)
delete designDoc.views[viewName]
await db.put(designDoc)
await exports.saveView(db, null, viewName, view)
@ -123,7 +125,7 @@ exports.migrateToInMemoryView = async (db, viewName) => {
exports.migrateToDesignView = async (db, viewName) => {
let view = await db.get(generateMemoryViewID(viewName))
const designDoc = await db.get("_design/database")
designDoc.views[viewName] = view.view
designDoc.views[viewName] = viewBuilder(view.view.meta)
await db.put(designDoc)
await db.remove(view._id, view._rev)
}

43
packages/server/src/api/controllers/view/viewBuilder.js
View File

@ -10,6 +10,12 @@ const TOKEN_MAP = {
OR: "||",
}
const CONDITIONS = {
EMPTY: "EMPTY",
NOT_EMPTY: "NOT_EMPTY",
CONTAINS: "CONTAINS",
}
const isEmptyExpression = key => {
return `(
doc["${key}"] === undefined ||
@ -77,13 +83,13 @@ function parseFilterExpression(filters) {
expression.push(TOKEN_MAP[filter.conjunction])
}
if (filter.condition === "CONTAINS") {
if (filter.condition === CONDITIONS.CONTAINS) {
expression.push(
`doc["${filter.key}"].${TOKEN_MAP[filter.condition]}("${filter.value}")`
)
} else if (filter.condition === "EMPTY") {
} else if (filter.condition === CONDITIONS.EMPTY) {
expression.push(isEmptyExpression(filter.key))
} else if (filter.condition === "NOT_EMPTY") {
} else if (filter.condition === CONDITIONS.NOT_EMPTY) {
expression.push(`!${isEmptyExpression(filter.key)}`)
} else {
const value =
@ -125,22 +131,37 @@ function viewTemplate({ field, tableId, groupBy, filters = [], calculation }) {
if (filters && filters.length > 0 && filters[0].conjunction) {
delete filters[0].conjunction
}
const parsedFilters = parseFilterExpression(filters)
const filterExpression = parsedFilters ? `&& (${parsedFilters})` : ""
const emitExpression = parseEmitExpression(field, groupBy)
const reduction = field && calculation ? { reduce: `_${calculation}` } : {}
let schema = null
let schema = null,
statFilter = null
if (calculation) {
schema = {
...(groupBy ? GROUP_PROPERTY : FIELD_PROPERTY),
...SCHEMA_MAP[calculation],
}
if (
!filters.find(
filter =>
filter.key === field && filter.condition === CONDITIONS.NOT_EMPTY
)
) {
statFilter = parseFilterExpression([
{ key: field, condition: CONDITIONS.NOT_EMPTY },
])
}
}
const parsedFilters = parseFilterExpression(filters)
const filterExpression = parsedFilters ? `&& (${parsedFilters})` : ""
const emitExpression = parseEmitExpression(field, groupBy)
const tableExpression = `doc.tableId === "${tableId}"`
const coreExpression = statFilter
? `(${tableExpression} && ${statFilter})`
: tableExpression
const reduction = field && calculation ? { reduce: `_${calculation}` } : {}
return {
meta: {
field,
@ -151,7 +172,7 @@ function viewTemplate({ field, tableId, groupBy, filters = [], calculation }) {
calculation,
},
map: `function (doc) {
if (doc.tableId === "${tableId}" ${filterExpression}) {
if (${coreExpression} ${filterExpression}) {
${emitExpression}
}
}`,

18
packages/server/yarn.lock
View File

@ -1014,10 +1014,10 @@
resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==
"@budibase/backend-core@1.0.154":
version "1.0.154"
resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.154.tgz#c310834892e7621778b07579464955487c5c9830"
integrity sha512-mcZxt8XhGgOB4XRHKkWTvBEI4HGp2bo8qyzOJRCvDqlg56S9zqGJDl75Z0N/Wc8N3I53QRcxISerj48odX172A==
"@budibase/backend-core@1.0.160":
version "1.0.160"
resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.160.tgz#000e3b5a3ed91e73a542b4caa202a6f147d91294"
integrity sha512-XfAFU6sRPrCSEKlm58WeuPw8lUoJK+KwO0tcbT+bB2Nb7XCHplskryEgk/PM9ujRU6SMPDx11zKeqRebHlycbA==
dependencies:
"@techpass/passport-openidconnect" "^0.3.0"
aws-sdk "^2.901.0"
@ -1091,12 +1091,12 @@
svelte-flatpickr "^3.2.3"
svelte-portal "^1.0.0"
"@budibase/pro@1.0.154":
version "1.0.154"
resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.154.tgz#f4e31e30376b206159b711224038141d73a1118e"
integrity sha512-+O6bemrcgyWG4a+D5dIOoZ+LGjW4aN7tRdFeZqoaIPCc1pA6zNtLUkM1nb+Laafuwq2Aht37vEuaRy7jfzVprA==
"@budibase/pro@1.0.160":
version "1.0.160"
resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.160.tgz#921c4e3f65b866d84292644dfd8793c4d0b667c7"
integrity sha512-p+Jhnk1n98CWCJXydSQSO7a+HDpqGAHekGQbOR7aayuwuoYzyOXxTcHNLdBp+3lkXhLSZq9oIwfEGpgdrrhXPA==
dependencies:
"@budibase/backend-core" "1.0.154"
"@budibase/backend-core" "1.0.160"
node-fetch "^2.6.1"
"@budibase/standard-components@^0.9.139":

18
packages/worker/yarn.lock
View File

@ -293,10 +293,10 @@
resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==
"@budibase/backend-core@1.0.154":
version "1.0.154"
resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.154.tgz#c310834892e7621778b07579464955487c5c9830"
integrity sha512-mcZxt8XhGgOB4XRHKkWTvBEI4HGp2bo8qyzOJRCvDqlg56S9zqGJDl75Z0N/Wc8N3I53QRcxISerj48odX172A==
"@budibase/backend-core@1.0.160":
version "1.0.160"
resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.160.tgz#000e3b5a3ed91e73a542b4caa202a6f147d91294"
integrity sha512-XfAFU6sRPrCSEKlm58WeuPw8lUoJK+KwO0tcbT+bB2Nb7XCHplskryEgk/PM9ujRU6SMPDx11zKeqRebHlycbA==
dependencies:
"@techpass/passport-openidconnect" "^0.3.0"
aws-sdk "^2.901.0"
@ -321,12 +321,12 @@
uuid "^8.3.2"
zlib "^1.0.5"
"@budibase/pro@1.0.154":
version "1.0.154"
resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.154.tgz#f4e31e30376b206159b711224038141d73a1118e"
integrity sha512-+O6bemrcgyWG4a+D5dIOoZ+LGjW4aN7tRdFeZqoaIPCc1pA6zNtLUkM1nb+Laafuwq2Aht37vEuaRy7jfzVprA==
"@budibase/pro@1.0.160":
version "1.0.160"
resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.160.tgz#921c4e3f65b866d84292644dfd8793c4d0b667c7"
integrity sha512-p+Jhnk1n98CWCJXydSQSO7a+HDpqGAHekGQbOR7aayuwuoYzyOXxTcHNLdBp+3lkXhLSZq9oIwfEGpgdrrhXPA==
dependencies:
"@budibase/backend-core" "1.0.154"
"@budibase/backend-core" "1.0.160"
node-fetch "^2.6.1"
"@cspotcode/source-map-consumer@0.8.0":