MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Adding the ability to migrate from existing in db views to in memory views.

This commit is contained in:
mike12345567 2021-09-21 17:37:26 +01:00
parent 3da8adfe48
commit 838a25ea09
5 changed files with 213 additions and 188 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
packages/server/src/api/controllers/row/internal.js
View File

@ -20,6 +20,7 @@ const { fullSearch, paginatedSearch } = require("./internalSearch")
const { getGlobalUsersFromMetadata } = require("../../../utilities/global")
const inMemoryViews = require("../../../db/inMemoryView")
const env = require("../../../environment")
const { migrateToInMemoryView } = require("../view/utils")
const CALCULATION_TYPES = {
SUM: "sum",
@ -74,15 +75,33 @@ async function getRawTableData(ctx, db, tableId) {
async function getView(db, viewName) {
let viewInfo
if (env.SELF_HOSTED) {
async function getFromDesignDoc() {
const designDoc = await db.get("_design/database")
viewInfo = designDoc.views[viewName]
return viewInfo
}
let migrate = false
if (env.SELF_HOSTED) {
viewInfo = await getFromDesignDoc()
} else {
viewInfo = await db.get(generateMemoryViewID(viewName))
if (viewInfo) {
viewInfo = viewInfo.view
try {
viewInfo = await db.get(generateMemoryViewID(viewName))
if (viewInfo) {
viewInfo = viewInfo.view
}
} catch (err) {
// check if it can be retrieved from design doc (needs migrated)
if (err.status !== 404) {
viewInfo = null
} else {
viewInfo = await getFromDesignDoc()
migrate = !!viewInfo
}
}
}
if (migrate) {
await migrateToInMemoryView(db, viewName)
}
if (!viewInfo) {
throw "View does not exist."
}
@ -193,9 +212,6 @@ exports.fetchView = async ctx => {
const db = new CouchDB(appId)
const { calculation, group, field } = ctx.query
const viewInfo = await getView(db, viewName)
if (!viewInfo) {
throw "View does not exist."
}
let response
if (env.SELF_HOSTED) {
response = await db.query(`database/${viewName}`, {

258
packages/server/src/api/controllers/view/index.js
View File

@ -2,193 +2,93 @@ const CouchDB = require("../../../db")
const viewTemplate = require("./viewBuilder")
const { apiFileReturn } = require("../../../utilities/fileSystem")
const exporters = require("./exporters")
const { saveView, getView, getViews, deleteView } = require("./utils")
const { fetchView } = require("../row")
const {
ViewNames,
generateMemoryViewID,
getMemoryViewParams,
} = require("../../../db/utils")
const env = require("../../../environment")
async function getView(db, viewName) {
if (env.SELF_HOSTED) {
const designDoc = await db.get("_design/database")
return designDoc.views[viewName]
} else {
const viewDoc = await db.get(generateMemoryViewID(viewName))
return viewDoc.view
exports.fetch = async ctx => {
const db = new CouchDB(ctx.appId)
ctx.body = await getViews(db)
}
exports.save = async ctx => {
const db = new CouchDB(ctx.appId)
const { originalName, ...viewToSave } = ctx.request.body
const view = viewTemplate(viewToSave)
if (!viewToSave.name) {
ctx.throw(400, "Cannot create view without a name")
}
await saveView(db, originalName, viewToSave.name, view)
// add views to table document
const table = await db.get(ctx.request.body.tableId)
if (!table.views) table.views = {}
if (!view.meta.schema) {
view.meta.schema = table.schema
}
table.views[viewToSave.name] = view.meta
if (originalName) {
delete table.views[originalName]
}
await db.put(table)
ctx.body = {
...table.views[viewToSave.name],
name: viewToSave.name,
}
}
async function getViews(db) {
const response = []
if (env.SELF_HOSTED) {
const designDoc = await db.get("_design/database")
for (let name of Object.keys(designDoc.views)) {
// Only return custom views, not built ins
if (Object.values(ViewNames).indexOf(name) !== -1) {
continue
}
response.push({
name,
...designDoc.views[name],
})
exports.destroy = async ctx => {
const db = new CouchDB(ctx.appId)
const viewName = decodeURI(ctx.params.viewName)
const view = await deleteView(db, viewName)
const table = await db.get(view.meta.tableId)
delete table.views[viewName]
await db.put(table)
ctx.body = view
}
exports.exportView = async ctx => {
const db = new CouchDB(ctx.appId)
const viewName = decodeURI(ctx.query.view)
const view = await getView(db, viewName)
const format = ctx.query.format
if (!format) {
ctx.throw(400, "Format must be specified, either csv or json")
}
if (view) {
ctx.params.viewName = viewName
// Fetch view rows
ctx.query = {
group: view.meta.groupBy,
calculation: view.meta.calculation,
stats: !!view.meta.field,
field: view.meta.field,
}
} else {
const views = (
await db.allDocs(
getMemoryViewParams({
include_docs: true,
})
)
).rows.map(row => row.doc)
for (let viewDoc of views) {
response.push({
name: viewDoc.name,
...viewDoc.view,
})
}
// table all_ view
/* istanbul ignore next */
ctx.params.viewName = viewName
}
return response
}
async function saveView(db, originalName, viewToSave, viewTemplate) {
if (env.SELF_HOSTED) {
const designDoc = await db.get("_design/database")
designDoc.views = {
...designDoc.views,
[viewToSave.name]: viewTemplate,
}
// view has been renamed
if (originalName) {
delete designDoc.views[originalName]
}
await db.put(designDoc)
} else {
const id = generateMemoryViewID(viewToSave.name)
const originalId = originalName ? generateMemoryViewID(originalName) : null
const viewDoc = {
_id: id,
view: viewTemplate,
name: viewToSave.name,
tableId: viewTemplate.meta.tableId,
}
try {
const old = await db.get(id)
if (originalId) {
const originalDoc = await db.get(originalId)
await db.remove(originalDoc._id, originalDoc._rev)
}
if (old && old._rev) {
viewDoc._rev = old._rev
}
} catch (err) {
// didn't exist, just skip
}
await db.put(viewDoc)
await fetchView(ctx)
let schema = view && view.meta && view.meta.schema
if (!schema) {
const tableId = ctx.params.tableId || view.meta.tableId
const table = await db.get(tableId)
schema = table.schema
}
// Export part
let headers = Object.keys(schema)
const exporter = exporters[format]
const filename = `${viewName}.${format}`
// send down the file
ctx.attachment(filename)
ctx.body = apiFileReturn(exporter(headers, ctx.body))
}
async function deleteView(db, viewName) {
if (env.SELF_HOSTED) {
const designDoc = await db.get("_design/database")
const view = designDoc.views[viewName]
delete designDoc.views[viewName]
await db.put(designDoc)
return view
} else {
const id = generateMemoryViewID(viewName)
const viewDoc = await db.get(id)
await db.remove(viewDoc._id, viewDoc._rev)
return viewDoc.view
}
}
const controller = {
fetch: async ctx => {
const db = new CouchDB(ctx.appId)
ctx.body = await getViews(db)
},
save: async ctx => {
const db = new CouchDB(ctx.appId)
const { originalName, ...viewToSave } = ctx.request.body
const view = viewTemplate(viewToSave)
if (!viewToSave.name) {
ctx.throw(400, "Cannot create view without a name")
}
await saveView(db, originalName, viewToSave, view)
// add views to table document
const table = await db.get(ctx.request.body.tableId)
if (!table.views) table.views = {}
if (!view.meta.schema) {
view.meta.schema = table.schema
}
table.views[viewToSave.name] = view.meta
if (originalName) {
delete table.views[originalName]
}
await db.put(table)
ctx.body = {
...table.views[viewToSave.name],
name: viewToSave.name,
}
},
destroy: async ctx => {
const db = new CouchDB(ctx.appId)
const viewName = decodeURI(ctx.params.viewName)
const view = await deleteView(db, viewName)
const table = await db.get(view.meta.tableId)
delete table.views[viewName]
await db.put(table)
ctx.body = view
},
exportView: async ctx => {
const db = new CouchDB(ctx.appId)
const viewName = decodeURI(ctx.query.view)
const view = await getView(db, viewName)
const format = ctx.query.format
if (!format) {
ctx.throw(400, "Format must be specified, either csv or json")
}
if (view) {
ctx.params.viewName = viewName
// Fetch view rows
ctx.query = {
group: view.meta.groupBy,
calculation: view.meta.calculation,
stats: !!view.meta.field,
field: view.meta.field,
}
} else {
// table all_ view
/* istanbul ignore next */
ctx.params.viewName = viewName
}
await fetchView(ctx)
let schema = view && view.meta && view.meta.schema
if (!schema) {
const tableId = ctx.params.tableId || view.meta.tableId
const table = await db.get(tableId)
schema = table.schema
}
// Export part
let headers = Object.keys(schema)
const exporter = exporters[format]
const filename = `${viewName}.${format}`
// send down the file
ctx.attachment(filename)
ctx.body = apiFileReturn(exporter(headers, ctx.body))
},
}
module.exports = controller

109
packages/server/src/api/controllers/view/utils.js Normal file
View File

@ -0,0 +1,109 @@
const {
ViewNames,
generateMemoryViewID,
getMemoryViewParams,
} = require("../../../db/utils")
const env = require("../../../environment")
exports.getView = async (db, viewName) => {
if (env.SELF_HOSTED) {
const designDoc = await db.get("_design/database")
return designDoc.views[viewName]
} else {
const viewDoc = await db.get(generateMemoryViewID(viewName))
return viewDoc.view
}
}
exports.getViews = async db => {
const response = []
if (env.SELF_HOSTED) {
const designDoc = await db.get("_design/database")
for (let name of Object.keys(designDoc.views)) {
// Only return custom views, not built ins
if (Object.values(ViewNames).indexOf(name) !== -1) {
continue
}
response.push({
name,
...designDoc.views[name],
})
}
} else {
const views = (
await db.allDocs(
getMemoryViewParams({
include_docs: true,
})
)
).rows.map(row => row.doc)
for (let viewDoc of views) {
response.push({
name: viewDoc.name,
...viewDoc.view,
})
}
}
return response
}
exports.saveView = async (db, originalName, viewName, viewTemplate) => {
if (env.SELF_HOSTED) {
const designDoc = await db.get("_design/database")
designDoc.views = {
...designDoc.views,
[viewName]: viewTemplate,
}
// view has been renamed
if (originalName) {
delete designDoc.views[originalName]
}
await db.put(designDoc)
} else {
const id = generateMemoryViewID(viewName)
const originalId = originalName ? generateMemoryViewID(originalName) : null
const viewDoc = {
_id: id,
view: viewTemplate,
name: viewName,
tableId: viewTemplate.meta.tableId,
}
try {
const old = await db.get(id)
if (originalId) {
const originalDoc = await db.get(originalId)
await db.remove(originalDoc._id, originalDoc._rev)
}
if (old && old._rev) {
viewDoc._rev = old._rev
}
} catch (err) {
// didn't exist, just skip
}
await db.put(viewDoc)
}
}
exports.deleteView = async (db, viewName) => {
if (env.SELF_HOSTED) {
const designDoc = await db.get("_design/database")
const view = designDoc.views[viewName]
delete designDoc.views[viewName]
await db.put(designDoc)
return view
} else {
const id = generateMemoryViewID(viewName)
const viewDoc = await db.get(id)
await db.remove(viewDoc._id, viewDoc._rev)
return viewDoc.view
}
}
exports.migrateToInMemoryView = async (db, viewName) => {
// delete the view initially
const designDoc = await db.get("_design/database")
const view = designDoc.views[viewName]
delete designDoc.views[viewName]
await db.put(designDoc)
await exports.saveView(db, null, viewName, view)
}

2
packages/server/src/api/routes/tests/view.spec.js
View File

@ -205,7 +205,7 @@ describe("/views", () => {
})
describe("exportView", () => {
it("should be able to delete a view", async () => {
it("should be able to export a view", async () => {
await config.createTable(priceTable())
await config.createRow()
const view = await config.createView()

2
packages/server/src/environment.js
View File

@ -26,7 +26,7 @@ module.exports = {
COUCH_DB_URL: process.env.COUCH_DB_URL,
MINIO_URL: process.env.MINIO_URL,
WORKER_URL: process.env.WORKER_URL,
SELF_HOSTED: !!parseInt(process.env.SELF_HOSTED),
SELF_HOSTED: process.env.SELF_HOSTED,
AWS_REGION: process.env.AWS_REGION,
ENABLE_ANALYTICS: process.env.ENABLE_ANALYTICS,
MINIO_ACCESS_KEY: process.env.MINIO_ACCESS_KEY,