From a38f83d0ec3b33542b838cf6114dbf49fcfcbe85 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 20 Sep 2021 18:24:09 +0100 Subject: [PATCH] Moving views into a different location so they don't trigger tree creation and attempting to use in memory pouchDB to run views on the fly. --- .../src/api/controllers/row/internal.js | 78 ++++++++--- .../server/src/api/controllers/view/index.js | 122 ++++++++++++++---- packages/server/src/db/inMemoryView.js | 38 ++++++ packages/server/src/db/utils.js | 9 ++ packages/server/src/environment.js | 7 + 5 files changed, 207 insertions(+), 47 deletions(-) create mode 100644 packages/server/src/db/inMemoryView.js diff --git a/packages/server/src/api/controllers/row/internal.js b/packages/server/src/api/controllers/row/internal.js index 2299a20580..9cfca69fa8 100644 --- a/packages/server/src/api/controllers/row/internal.js +++ b/packages/server/src/api/controllers/row/internal.js @@ -5,6 +5,7 @@ const { generateRowID, DocumentTypes, InternalTables, + generateMemoryViewID, } = require("../../../db/utils") const userController = require("../user") const { @@ -16,6 +17,8 @@ const { isEqual } = require("lodash") const { validate, findRow } = require("./utils") const { fullSearch, paginatedSearch } = require("./internalSearch") const { getGlobalUsersFromMetadata } = require("../../../utilities/global") +const inMemoryViews = require("../../../db/inMemoryView") +const env = require("../../../environment") const CALCULATION_TYPES = { SUM: "sum", @@ -36,6 +39,40 @@ async function storeResponse(ctx, db, row, oldTable, table) { return { row, table } } +// doesn't do the outputProcessing +async function getRawTableData(ctx, db, tableId) { + let rows + if (tableId === InternalTables.USER_METADATA) { + await userController.fetchMetadata(ctx) + rows = ctx.body + } else { + const response = await db.allDocs( + getRowParams(tableId, null, { + include_docs: true, + }) + ) + rows = response.rows.map(row => row.doc) + } + return rows +} + +async function getView(db, viewName) { + let viewInfo + if (env.SELF_HOSTED) { + const designDoc = await db.get("_design/database") + viewInfo = designDoc.views[viewName] + } else { + viewInfo = await db.get(generateMemoryViewID(viewName)) + if (viewInfo) { + viewInfo = viewInfo.view + } + } + if (!viewInfo) { + throw "View does not exist." + } + return viewInfo +} + exports.patch = async ctx => { const appId = ctx.appId const db = new CouchDB(appId) @@ -139,15 +176,28 @@ exports.fetchView = async ctx => { const db = new CouchDB(appId) const { calculation, group, field } = ctx.query - const designDoc = await db.get("_design/database") - const viewInfo = designDoc.views[viewName] + const viewInfo = await getView(db, viewName) if (!viewInfo) { throw "View does not exist." } - const response = await db.query(`database/${viewName}`, { - include_docs: !calculation, - group: !!group, - }) + let response + // TODO: make sure not self hosted in Cloud + if (!env.SELF_HOSTED) { + response = await db.query(`database/${viewName}`, { + include_docs: !calculation, + group: !!group, + }) + } else { + const tableId = viewInfo.meta.tableId + const data = await getRawTableData(ctx, db, tableId) + response = await inMemoryViews.runView( + appId, + viewInfo, + calculation, + group, + data + ) + } let rows if (!calculation) { @@ -191,19 +241,9 @@ exports.fetch = async ctx => { const appId = ctx.appId const db = new CouchDB(appId) - let rows, - table = await db.get(ctx.params.tableId) - if (ctx.params.tableId === InternalTables.USER_METADATA) { - await userController.fetchMetadata(ctx) - rows = ctx.body - } else { - const response = await db.allDocs( - getRowParams(ctx.params.tableId, null, { - include_docs: true, - }) - ) - rows = response.rows.map(row => row.doc) - } + const tableId = ctx.params.tableId + let table = await db.get(tableId) + let rows = await getRawTableData(ctx, db, tableId) return outputProcessing(ctx, table, rows) } diff --git a/packages/server/src/api/controllers/view/index.js b/packages/server/src/api/controllers/view/index.js index 3d0f236fce..4c2e6ee08b 100644 --- a/packages/server/src/api/controllers/view/index.js +++ b/packages/server/src/api/controllers/view/index.js @@ -3,14 +3,27 @@ const viewTemplate = require("./viewBuilder") const { apiFileReturn } = require("../../../utilities/fileSystem") const exporters = require("./exporters") const { fetchView } = require("../row") -const { ViewNames } = require("../../../db/utils") +const { + ViewNames, + generateMemoryViewID, + getMemoryViewParams, +} = require("../../../db/utils") +const env = require("../../../environment") -const controller = { - fetch: async ctx => { - const db = new CouchDB(ctx.appId) +async function getView(db, viewName) { + if (env.SELF_HOSTED) { const designDoc = await db.get("_design/database") - const response = [] + return designDoc.views[viewName] + } else { + const viewDoc = await db.get(generateMemoryViewID(viewName)) + return viewDoc.view + } +} +async function getViews(db) { + const response = [] + if (env.SELF_HOSTED) { + const designDoc = await db.get("_design/database") for (let name of Object.keys(designDoc.views)) { // Only return custom views, not built ins if (Object.values(ViewNames).indexOf(name) !== -1) { @@ -21,30 +34,91 @@ const controller = { ...designDoc.views[name], }) } + } else { + const views = ( + await db.allDocs( + getMemoryViewParams({ + include_docs: true, + }) + ) + ).rows.map(row => row.doc) + for (let viewDoc of views) { + response.push({ + name: viewDoc.name, + ...viewDoc.view, + }) + } + } + return response +} - ctx.body = response +async function saveView(db, originalName, viewToSave, viewTemplate) { + if (env.SELF_HOSTED) { + const designDoc = await db.get("_design/database") + designDoc.views = { + ...designDoc.views, + [viewToSave.name]: viewTemplate, + } + // view has been renamed + if (originalName) { + delete designDoc.views[originalName] + } + await db.put(designDoc) + } else { + const id = generateMemoryViewID(viewToSave.name) + const originalId = originalName ? generateMemoryViewID(originalName) : null + const viewDoc = { + _id: id, + view: viewTemplate, + name: viewToSave.name, + tableId: viewTemplate.meta.tableId, + } + try { + const old = await db.get(id) + if (originalId) { + const originalDoc = await db.get(originalId) + await db.remove(originalDoc._id, originalDoc._rev) + } + if (old && old._rev) { + viewDoc._rev = old._rev + } + } catch (err) { + // didn't exist, just skip + } + await db.put(viewDoc) + } +} + +async function deleteView(db, viewName) { + if (env.SELF_HOSTED) { + const designDoc = await db.get("_design/database") + const view = designDoc.views[viewName] + delete designDoc.views[viewName] + await db.put(designDoc) + return view + } else { + const id = generateMemoryViewID(viewName) + const viewDoc = await db.get(id) + await db.remove(viewDoc._id, viewDoc._rev) + return viewDoc.view + } +} + +const controller = { + fetch: async ctx => { + const db = new CouchDB(ctx.appId) + ctx.body = await getViews(db) }, save: async ctx => { const db = new CouchDB(ctx.appId) const { originalName, ...viewToSave } = ctx.request.body - const designDoc = await db.get("_design/database") const view = viewTemplate(viewToSave) if (!viewToSave.name) { ctx.throw(400, "Cannot create view without a name") } - designDoc.views = { - ...designDoc.views, - [viewToSave.name]: view, - } - - // view has been renamed - if (originalName) { - delete designDoc.views[originalName] - } - - await db.put(designDoc) + await saveView(db, originalName, viewToSave, view) // add views to table document const table = await db.get(ctx.request.body.tableId) @@ -53,11 +127,9 @@ const controller = { view.meta.schema = table.schema } table.views[viewToSave.name] = view.meta - if (originalName) { delete table.views[originalName] } - await db.put(table) ctx.body = { @@ -67,13 +139,8 @@ const controller = { }, destroy: async ctx => { const db = new CouchDB(ctx.appId) - const designDoc = await db.get("_design/database") const viewName = decodeURI(ctx.params.viewName) - const view = designDoc.views[viewName] - delete designDoc.views[viewName] - - await db.put(designDoc) - + const view = await deleteView(db, viewName) const table = await db.get(view.meta.tableId) delete table.views[viewName] await db.put(table) @@ -82,10 +149,9 @@ const controller = { }, exportView: async ctx => { const db = new CouchDB(ctx.appId) - const designDoc = await db.get("_design/database") const viewName = decodeURI(ctx.query.view) + const view = await getView(db, viewName) - const view = designDoc.views[viewName] const format = ctx.query.format if (!format) { ctx.throw(400, "Format must be specified, either csv or json") diff --git a/packages/server/src/db/inMemoryView.js b/packages/server/src/db/inMemoryView.js new file mode 100644 index 0000000000..f82b418b7f --- /dev/null +++ b/packages/server/src/db/inMemoryView.js @@ -0,0 +1,38 @@ +const PouchDB = require("pouchdb") +const memory = require("pouchdb-adapter-memory") + +PouchDB.plugin(memory) +const Pouch = PouchDB.defaults({ + prefix: undefined, + adapter: "memory", +}) + +exports.runView = async (appId, view, calculation, group, data) => { + // appId doesn't really do anything since its all in memory + // use it just incase multiple databases at the same time + const db = new Pouch(appId) + await db.put({ + _id: "_design/database", + views: { + runner: view, + }, + }) + // write all the docs to the in memory Pouch + await db.bulkDocs(data) + const response = await db.query(`database/runner`, { + include_docs: !calculation, + group: !!group, + }) + // need to fix the revs to be totally accurate + for (let row of response.rows) { + if (!row._rev || !row._id) { + continue + } + const found = data.find(possible => possible._id === row._id) + if (found) { + row._rev = found._rev + } + } + await db.destroy() + return response +} diff --git a/packages/server/src/db/utils.js b/packages/server/src/db/utils.js index ec1c267fa2..3e20b30869 100644 --- a/packages/server/src/db/utils.js +++ b/packages/server/src/db/utils.js @@ -39,6 +39,7 @@ const DocumentTypes = { QUERY: "query", DEPLOYMENTS: "deployments", METADATA: "metadata", + MEM_VIEW: "view", } const ViewNames = { @@ -348,6 +349,14 @@ exports.getMetadataParams = (type, entityId = null, otherProps = {}) => { return getDocParams(DocumentTypes.METADATA, docId, otherProps) } +exports.generateMemoryViewID = viewName => { + return `${DocumentTypes.MEM_VIEW}${SEPARATOR}${viewName}` +} + +exports.getMemoryViewParams = (otherProps = {}) => { + return getDocParams(DocumentTypes.MEM_VIEW, null, otherProps) +} + /** * This can be used with the db.allDocs to get a list of IDs */ diff --git a/packages/server/src/environment.js b/packages/server/src/environment.js index 9e029e440a..89e015b6f5 100644 --- a/packages/server/src/environment.js +++ b/packages/server/src/environment.js @@ -66,3 +66,10 @@ module.exports = { return !isDev() }, } + +// convert any strings to numbers if required, like "0" would be true otherwise +for (let [key, value] of Object.entries(module.exports)) { + if (typeof value === "string" && !isNaN(parseInt(value))) { + module.exports[key] = parseInt(value) + } +}