MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fixing up middleware to handle uploads, views, records, automation runs and users.

This commit is contained in:
mike12345567 2020-10-07 17:56:47 +01:00
parent 3a6a03403f
commit f2b7d85b6e
10 changed files with 147 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
packages/server/src/api/controllers/auth.js
View File

@ -3,7 +3,7 @@ const CouchDB = require("../../db")
const ClientDb = require("../../db/clientDb")
const bcrypt = require("../../utilities/bcrypt")
const environment = require("../../environment")
const { apiKeyTable } = require("../../db/dynamoClient")
const { getAPIKey } = require("../../utilities/usageQuota")
const { generateUserID } = require("../../db/utils")
exports.authenticate = async ctx => {
@ -55,7 +55,7 @@ exports.authenticate = async ctx => {
}
// if in cloud add the user api key
if (environment.CLOUD) {
payload.apiKey = await apiKeyTable.get({ primary: ctx.user.appId })
payload.apiKey = getAPIKey(ctx.user.appId)
}
const token = jwt.sign(payload, ctx.config.jwtSecret, {

2
packages/server/src/api/controllers/automation.js
View File

@ -33,6 +33,7 @@ function cleanAutomationInputs(automation) {
exports.create = async function(ctx) {
const db = new CouchDB(ctx.user.instanceId)
let automation = ctx.request.body
automation.appId = ctx.user.appId
automation._id = generateAutomationID()
@ -54,6 +55,7 @@ exports.create = async function(ctx) {
exports.update = async function(ctx) {
const db = new CouchDB(ctx.user.instanceId)
let automation = ctx.request.body
automation.appId = ctx.user.appId
automation = cleanAutomationInputs(automation)
const response = await db.put(automation)

4
packages/server/src/api/routes/model.js
View File

@ -1,7 +1,6 @@
const Router = require("@koa/router")
const modelController = require("../controllers/model")
const authorized = require("../../middleware/authorized")
const usage = require("../../middleware/usageQuota")
const { BUILDER, READ_MODEL } = require("../../utilities/accessLevels")
const router = Router()
@ -13,7 +12,7 @@ router
authorized(READ_MODEL, ctx => ctx.params.id),
modelController.find
)
.post("/api/models", authorized(BUILDER), usage, modelController.save)
.post("/api/models", authorized(BUILDER), modelController.save)
.post(
"/api/models/csv/validate",
authorized(BUILDER),
@ -22,7 +21,6 @@ router
.delete(
"/api/models/:modelId/:revId",
authorized(BUILDER),
usage,
modelController.destroy
)

3
packages/server/src/api/routes/static.js
View File

@ -4,6 +4,7 @@ const { budibaseTempDir } = require("../../utilities/budibaseDir")
const env = require("../../environment")
const authorized = require("../../middleware/authorized")
const { BUILDER } = require("../../utilities/accessLevels")
const usage = require("../../middleware/usageQuota")
const router = Router()
@ -28,7 +29,7 @@ router
authorized(BUILDER),
controller.performLocalFileProcessing
)
.post("/api/attachments/upload", controller.uploadFile)
.post("/api/attachments/upload", usage, controller.uploadFile)
.get("/componentlibrary", controller.serveComponentLibrary)
.get("/assets/:file*", controller.serveAppAsset)
.get("/attachments/:file*", controller.serveAttachment)

4
packages/server/src/api/routes/user.js
View File

@ -2,6 +2,7 @@ const Router = require("@koa/router")
const controller = require("../controllers/user")
const authorized = require("../../middleware/authorized")
const { USER_MANAGEMENT, LIST_USERS } = require("../../utilities/accessLevels")
const usage = require("../../middleware/usageQuota")
const router = Router()
@ -9,10 +10,11 @@ router
.get("/api/users", authorized(LIST_USERS), controller.fetch)
.get("/api/users/:username", authorized(USER_MANAGEMENT), controller.find)
.put("/api/users/", authorized(USER_MANAGEMENT), controller.update)
.post("/api/users", authorized(USER_MANAGEMENT), controller.create)
.post("/api/users", authorized(USER_MANAGEMENT), usage, controller.create)
.delete(
"/api/users/:username",
authorized(USER_MANAGEMENT),
usage,
controller.destroy
)

10
packages/server/src/api/routes/view.js
View File

@ -3,6 +3,7 @@ const viewController = require("../controllers/view")
const recordController = require("../controllers/record")
const authorized = require("../../middleware/authorized")
const { BUILDER, READ_VIEW } = require("../../utilities/accessLevels")
const usage = require("../../middleware/usageQuota")
const router = Router()
@ -13,8 +14,13 @@ router
recordController.fetchView
)
.get("/api/views", authorized(BUILDER), viewController.fetch)
.delete("/api/views/:viewName", authorized(BUILDER), viewController.destroy)
.post("/api/views", authorized(BUILDER), viewController.save)
.delete(
"/api/views/:viewName",
authorized(BUILDER),
usage,
viewController.destroy
)
.post("/api/views", authorized(BUILDER), usage, viewController.save)
.post("/api/views/export", authorized(BUILDER), viewController.exportView)
.get(
"/api/views/export/download/:fileName",

25
packages/server/src/automations/index.js
View File

@ -3,6 +3,7 @@ const actions = require("./actions")
const environment = require("../environment")
const workerFarm = require("worker-farm")
const singleThread = require("./thread")
const { getAPIKey, update, Properties } = require("../utilities/usageQuota")
let workers = workerFarm(require.resolve("./thread"))
@ -18,16 +19,32 @@ function runWorker(job) {
})
}
async function updateQuota(automation) {
const appId = automation.appId
const apiKey = await getAPIKey(appId)
// this will fail, causing automation to escape if limits reached
await update(apiKey, Properties.AUTOMATION, 1)
}
/**
* This module is built purely to kick off the worker farm and manage the inputs/outputs
*/
module.exports.init = function() {
actions.init().then(() => {
triggers.automationQueue.process(async job => {
if (environment.BUDIBASE_ENVIRONMENT === "PRODUCTION") {
await runWorker(job)
} else {
await singleThread(job)
try {
if (environment.CLOUD) {
await updateQuota(job.data.automation)
}
if (environment.BUDIBASE_ENVIRONMENT === "PRODUCTION") {
await runWorker(job)
} else {
await singleThread(job)
}
} catch (err) {
console.error(
`${job.data.automation.appId} automation ${job.data.automation._id} was unable to run - ${err}`
)
}
})
})

71
packages/server/src/middleware/usageQuota.js
View File

@ -1,9 +1,5 @@
const CouchDB = require("../db")
const environment = require("../environment")
const { apiKeyTable } = require("../db/dynamoClient")
// a normalised month in milliseconds
const QUOTA_RESET = 2592000000
const usageQuota = require("../utilities/usageQuota")
// currently only counting new writes and deletes
const METHOD_MAP = {
@ -12,36 +8,32 @@ const METHOD_MAP = {
}
const DOMAIN_MAP = {
models: "model",
records: "record",
records: usageQuota.Properties.RECORD,
upload: usageQuota.Properties.UPLOAD,
views: usageQuota.Properties.VIEW,
users: usageQuota.Properties.USER,
// this will not be updated by endpoint calls
// instead it will be updated by triggers
automationRuns: usageQuota.Properties.AUTOMATION,
}
function buildUpdateParams(key, property, usage) {
return {
primary: key,
condition: "#quota.#prop + :usage < #limits.model AND #quotaReset < :now",
expression: "ADD #quota.#prop :usage",
names: {
"#quota": "usageQuota",
"#prop": property,
"#limits": "limits",
"#quotaReset": "quotaReset",
},
values: {
":usage": usage,
":now": Date.now(),
},
function getProperty(url) {
for (let domain of Object.keys(DOMAIN_MAP)) {
if (url.indexOf(domain) !== -1) {
return DOMAIN_MAP[domain]
}
}
}
module.exports = async (ctx, next) => {
const db = new CouchDB(ctx.user.instanceId)
const usage = METHOD_MAP[ctx.req.method]
const domainParts = ctx.req.url.split("/")
const property = DOMAIN_MAP[domainParts[domainParts.length - 1]]
let usage = METHOD_MAP[ctx.req.method]
const property = getProperty(ctx.req.url)
console.log(ctx.req.url)
if (usage == null || property == null) {
return next()
}
console.log(`${usage} to ${property}`)
// post request could be a save of a pre-existing entry
if (ctx.request.body && ctx.request.body._id) {
try {
@ -52,26 +44,17 @@ module.exports = async (ctx, next) => {
return
}
}
// don't try validate in builder
if (!environment.CLOUD) {
return next()
// update usage for uploads to be the total size
if (property === usageQuota.Properties.UPLOAD) {
const files =
ctx.request.files.file.length > 1
? Array.from(ctx.request.files.file)
: [ctx.request.files.file]
usage = files.map(file => file.size).reduce((total, size) => total + size)
}
try {
await apiKeyTable.update(buildUpdateParams(ctx.apiKey, property, usage))
await usageQuota.update(ctx.apiKey, property, usage)
} catch (err) {
if (err.code !== "ConditionalCheckFailedException") {
// get the API key so we can check it
let apiKey = await apiKeyTable.get({ primary: ctx.apiKey })
// we have infact breached the reset period
if (apiKey && apiKey.quotaReset >= Date.now()) {
// update the quota reset period and reset the values for all properties
apiKey.quotaReset = Date.now() + QUOTA_RESET
for (let prop of Object.keys(apiKey.usageQuota)) {
apiKey.usageQuota[prop] = 0
}
await apiKeyTable.put({ item: apiKey })
}
ctx.throw(403, `Resource limits have been reached`)
}
ctx.throw(403, err)
}
}
}

18
packages/server/src/utilities/fileProcessor.js
View File

@ -1,5 +1,5 @@
const fs = require("fs")
const sharp = require("sharp")
// const sharp = require("sharp")
const fsPromises = fs.promises
const FORMATS = {
@ -7,14 +7,14 @@ const FORMATS = {
}
async function processImage(file) {
const imgMeta = await sharp(file.path)
.resize(300)
.toFile(file.outputPath)
return {
...file,
...imgMeta,
}
// const imgMeta = await sharp(file.path)
// .resize(300)
// .toFile(file.outputPath)
//
// return {
// ...file,
// ...imgMeta,
// }
}
async function process(file) {

72
packages/server/src/utilities/usageQuota.js Normal file
View File

@ -0,0 +1,72 @@
const environment = require("../environment")
const { apiKeyTable } = require("../db/dynamoClient")
function buildUpdateParams(key, property, usage) {
return {
primary: key,
condition: "#quota.#prop + :usage < #limits.model AND #quotaReset < :now",
expression: "ADD #quota.#prop :usage",
names: {
"#quota": "usageQuota",
"#prop": property,
"#limits": "limits",
"#quotaReset": "quotaReset",
},
values: {
":usage": usage,
":now": Date.now(),
},
}
}
// a normalised month in milliseconds
const QUOTA_RESET = 2592000000
exports.Properties = {
RECORD: "records",
UPLOAD: "storage",
VIEW: "views",
USER: "users",
AUTOMATION: "automationRuns",
}
exports.getAPIKey = async appId => {
return apiKeyTable.get({ primary: appId })
}
/**
* Given a specified API key this will add to the usage object for the specified property.
* @param {string} apiKey The API key which is to be updated.
* @param {string} property The property which is to be added to (within the nested usageQuota object).
* @param {number} usage The amount (this can be negative) to adjust the number by.
* @returns {Promise<void>} When this completes the API key will now be up to date - the quota period may have
* also been reset after this call.
*/
exports.update = async (apiKey, property, usage) => {
// don't try validate in builder
if (!environment.CLOUD) {
return
}
try {
await apiKeyTable.update(buildUpdateParams(apiKey, property, usage))
} catch (err) {
if (err.code !== "ConditionalCheckFailedException") {
// get the API key so we can check it
let apiKey = await apiKeyTable.get({ primary: apiKey })
// we have infact breached the reset period
if (apiKey && apiKey.quotaReset >= Date.now()) {
// update the quota reset period and reset the values for all properties
apiKey.quotaReset = Date.now() + QUOTA_RESET
for (let prop of Object.keys(apiKey.usageQuota)) {
if (prop === property) {
apiKey.usageQuota[prop] = usage > 0 ? usage : 0
} else {
apiKey.usageQuota[prop] = 0
}
}
await apiKeyTable.put({ item: apiKey })
}
throw "Resource limits have been reached"
}
}
}