MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Updates for API usage after testing against local Dynamo.

This commit is contained in:
mike12345567 2020-10-08 17:34:41 +01:00
parent d2ae589151
commit b400a06027
11 changed files with 127 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

55
packages/server/scripts/createApiKeyAndAppId.js Normal file
View File

@ -0,0 +1,55 @@
// THIS will create API Keys and App Ids input in a local Dynamo instance if it is running
const dynamoClient = require("../src/db/dynamoClient")
if (process.argv[2] == null || process.argv[3] == null) {
console.error(
"Inputs incorrect format, was expecting: node createApiKeyAndAppId.js <API_KEY> <APP_ID>"
)
process.exit(-1)
}
const FAKE_STRING = "fakestring"
// set fake credentials for local dynamo to actually work
process.env.AWS_ACCESS_KEY_ID = "KEY_ID"
process.env.AWS_SECRET_ACCESS_KEY = "SECRET_KEY"
dynamoClient.init("http://localhost:8333")
async function run() {
await dynamoClient.apiKeyTable.put({
item: {
pk: process.argv[2],
accountId: FAKE_STRING,
trackingId: FAKE_STRING,
quotaReset: Date.now() + 2592000000,
usageQuota: {
automationRuns: 0,
records: 0,
storage: 0,
users: 0,
views: 0,
},
usageLimits: {
automationRuns: 10,
records: 10,
storage: 1000,
users: 10,
views: 10,
},
},
})
await dynamoClient.apiKeyTable.put({
item: {
pk: process.argv[3],
apiKey: process.argv[2],
},
})
}
run()
.then(() => {
console.log("Records should have been created.")
})
.catch(err => {
console.error("Cannot create records - " + err)
})

9
packages/server/src/automations/index.js
View File

@ -21,9 +21,10 @@ function runWorker(job) {
async function updateQuota(automation) {
const appId = automation.appId
const apiKey = await getAPIKey(appId)
const apiObj = await getAPIKey(appId)
// this will fail, causing automation to escape if limits reached
await update(apiKey, Properties.AUTOMATION, 1)
await update(apiObj.apiKey, Properties.AUTOMATION, 1)
return apiObj.apiKey
}
/**
@ -33,8 +34,8 @@ module.exports.init = function() {
actions.init().then(() => {
triggers.automationQueue.process(async job => {
try {
if (environment.CLOUD) {
await updateQuota(job.data.automation)
if (environment.CLOUD && job.data.automation) {
job.data.automation.apiKey = await updateQuota(job.data.automation)
}
if (environment.BUDIBASE_ENVIRONMENT === "PRODUCTION") {
await runWorker(job)

7
packages/server/src/automations/steps/createRecord.js
View File

@ -1,5 +1,7 @@
const recordController = require("../../api/controllers/record")
const automationUtils = require("../automationUtils")
const environment = require("../../environment")
const usage = require("../../utilities/usageQuota")
module.exports.definition = {
name: "Create Row",
@ -56,7 +58,7 @@ module.exports.definition = {
},
}
module.exports.run = async function({ inputs, instanceId }) {
module.exports.run = async function({ inputs, instanceId, apiKey }) {
// TODO: better logging of when actions are missed due to missing parameters
if (inputs.record == null || inputs.record.modelId == null) {
return
@ -78,6 +80,9 @@ module.exports.run = async function({ inputs, instanceId }) {
}
try {
if (environment.CLOUD) {
await usage.update(apiKey, usage.Properties.RECORD, 1)
}
await recordController.save(ctx)
return {
record: inputs.record,

7
packages/server/src/automations/steps/createUser.js
View File

@ -1,5 +1,7 @@
const accessLevels = require("../../utilities/accessLevels")
const userController = require("../../api/controllers/user")
const environment = require("../../environment")
const usage = require("../../utilities/usageQuota")
module.exports.definition = {
description: "Create a new user",
@ -56,7 +58,7 @@ module.exports.definition = {
},
}
module.exports.run = async function({ inputs, instanceId }) {
module.exports.run = async function({ inputs, instanceId, apiKey }) {
const { username, password, accessLevelId } = inputs
const ctx = {
user: {
@ -68,6 +70,9 @@ module.exports.run = async function({ inputs, instanceId }) {
}
try {
if (environment.CLOUD) {
await usage.update(apiKey, usage.Properties.USER, 1)
}
await userController.create(ctx)
return {
response: ctx.body,

7
packages/server/src/automations/steps/deleteRecord.js
View File

@ -1,4 +1,6 @@
const recordController = require("../../api/controllers/record")
const environment = require("../../environment")
const usage = require("../../utilities/usageQuota")
module.exports.definition = {
description: "Delete a row from your database",
@ -48,7 +50,7 @@ module.exports.definition = {
},
}
module.exports.run = async function({ inputs, instanceId }) {
module.exports.run = async function({ inputs, instanceId, apiKey }) {
// TODO: better logging of when actions are missed due to missing parameters
if (inputs.id == null || inputs.revision == null) {
return
@ -63,6 +65,9 @@ module.exports.run = async function({ inputs, instanceId }) {
}
try {
if (environment.CLOUD) {
await usage.update(apiKey, usage.Properties.RECORD, -1)
}
await recordController.destroy(ctx)
return {
response: ctx.body,

1
packages/server/src/automations/thread.js
View File

@ -62,6 +62,7 @@ class Orchestrator {
const outputs = await stepFn({
inputs: step.inputs,
instanceId: this._instanceId,
apiKey: automation.apiKey,
})
if (step.stepId === FILTER_STEP_ID && !outputs.success) {
break

30
packages/server/src/db/dynamoClient.js
View File

@ -1,6 +1,8 @@
let _ = require("lodash")
let environment = require("../environment")
const AWS_REGION = environment.AWS_REGION ? environment.AWS_REGION : "eu-west-1"
const TableInfo = {
API_KEYS: {
name: "beta-api-key-table",
@ -49,6 +51,7 @@ class Table {
condition,
names,
values,
exists,
otherProps,
}) {
let params = {
@ -66,6 +69,13 @@ class Table {
if (this._sort && sort) {
params.Key[this._sort] = sort
}
if (exists) {
params.ExpressionAttributeNames["#PRIMARY"] = this._primary
if (params.ConditionExpression) {
params.ConditionExpression += " AND "
}
params.ConditionExpression += "attribute_exists(#PRIMARY)"
}
if (otherProps) {
params = _.merge(params, otherProps)
}
@ -90,15 +100,17 @@ class Table {
}
}
exports.init = () => {
if (!environment.CLOUD) {
return
}
exports.init = endpoint => {
let AWS = require("aws-sdk")
AWS.config.update({
region: AWS_REGION,
})
let docClientParams = {
correctClockSkew: true,
}
if (environment.DYNAMO_ENDPOINT) {
if (endpoint) {
docClientParams.endpoint = endpoint
} else if (environment.DYNAMO_ENDPOINT) {
docClientParams.endpoint = environment.DYNAMO_ENDPOINT
}
docClient = new AWS.DynamoDB.DocumentClient(docClientParams)
@ -106,3 +118,11 @@ exports.init = () => {
exports.apiKeyTable = new Table(TableInfo.API_KEYS)
exports.userTable = new Table(TableInfo.USERS)
if (environment.CLOUD) {
exports.init(`https://dynamodb.${AWS_REGION}.amazonaws.com`)
} else {
process.env.AWS_ACCESS_KEY_ID = "KEY_ID"
process.env.AWS_SECRET_ACCESS_KEY = "SECRET_KEY"
exports.init("http://localhost:8333")
}

12
packages/server/src/middleware/authorized.js
View File

@ -16,8 +16,7 @@ module.exports = (permName, getItemId) => async (ctx, next) => {
}
if (ctx.user.accessLevel._id === BUILDER_LEVEL_ID) {
await next()
return
return next()
}
if (permName === BUILDER) {
@ -28,8 +27,7 @@ module.exports = (permName, getItemId) => async (ctx, next) => {
const permissionId = ({ name, itemId }) => name + (itemId ? `-${itemId}` : "")
if (ctx.user.accessLevel._id === ADMIN_LEVEL_ID) {
await next()
return
return next()
}
const thisPermissionId = permissionId({
@ -42,8 +40,7 @@ module.exports = (permName, getItemId) => async (ctx, next) => {
ctx.user.accessLevel._id === POWERUSER_LEVEL_ID &&
!adminPermissions.map(permissionId).includes(thisPermissionId)
) {
await next()
return
return next()
}
if (
@ -51,8 +48,7 @@ module.exports = (permName, getItemId) => async (ctx, next) => {
.map(permissionId)
.includes(thisPermissionId)
) {
await next()
return
return next()
}
ctx.throw(403, "Not Authorized")

7
packages/server/src/middleware/usageQuota.js
View File

@ -1,5 +1,6 @@
const CouchDB = require("../db")
const usageQuota = require("../utilities/usageQuota")
const environment = require("../environment")
// currently only counting new writes and deletes
const METHOD_MAP = {
@ -29,11 +30,9 @@ module.exports = async (ctx, next) => {
const db = new CouchDB(ctx.user.instanceId)
let usage = METHOD_MAP[ctx.req.method]
const property = getProperty(ctx.req.url)
console.log(ctx.req.url)
if (usage == null || property == null) {
return next()
}
console.log(`${usage} to ${property}`)
// post request could be a save of a pre-existing entry
if (ctx.request.body && ctx.request.body._id) {
try {
@ -52,8 +51,12 @@ module.exports = async (ctx, next) => {
: [ctx.request.files.file]
usage = files.map(file => file.size).reduce((total, size) => total + size)
}
if (!environment.CLOUD) {
return next()
}
try {
await usageQuota.update(ctx.apiKey, property, usage)
return next()
} catch (err) {
ctx.throw(403, err)
}

4
packages/server/src/utilities/builder/setBuilderToken.js
View File

@ -8,7 +8,9 @@ module.exports = (ctx, appId, instanceId) => {
instanceId,
appId,
}
if (process.env.BUDIBASE_API_KEY) {
builderUser.apiKey = process.env.BUDIBASE_API_KEY
}
const token = jwt.sign(builderUser, ctx.config.jwtSecret, {
expiresIn: "30 days",
})

21
packages/server/src/utilities/usageQuota.js
View File

@ -4,12 +4,12 @@ const { apiKeyTable } = require("../db/dynamoClient")
function buildUpdateParams(key, property, usage) {
return {
primary: key,
condition: "#quota.#prop + :usage < #limits.model AND #quotaReset < :now",
condition: "#quota.#prop < #limits.#prop AND #quotaReset > :now",
expression: "ADD #quota.#prop :usage",
names: {
"#quota": "usageQuota",
"#prop": property,
"#limits": "limits",
"#limits": "usageLimits",
"#quotaReset": "quotaReset",
},
values: {
@ -50,21 +50,22 @@ exports.update = async (apiKey, property, usage) => {
try {
await apiKeyTable.update(buildUpdateParams(apiKey, property, usage))
} catch (err) {
if (err.code !== "ConditionalCheckFailedException") {
if (err.code === "ConditionalCheckFailedException") {
// get the API key so we can check it
let apiKey = await apiKeyTable.get({ primary: apiKey })
const keyObj = await apiKeyTable.get({ primary: apiKey })
// we have infact breached the reset period
if (apiKey && apiKey.quotaReset >= Date.now()) {
if (keyObj && keyObj.quotaReset <= Date.now()) {
// update the quota reset period and reset the values for all properties
apiKey.quotaReset = Date.now() + QUOTA_RESET
for (let prop of Object.keys(apiKey.usageQuota)) {
keyObj.quotaReset = Date.now() + QUOTA_RESET
for (let prop of Object.keys(keyObj.usageQuota)) {
if (prop === property) {
apiKey.usageQuota[prop] = usage > 0 ? usage : 0
keyObj.usageQuota[prop] = usage > 0 ? usage : 0
} else {
apiKey.usageQuota[prop] = 0
keyObj.usageQuota[prop] = 0
}
}
await apiKeyTable.put({ item: apiKey })
await apiKeyTable.put({ item: keyObj })
return
}
throw "Resource limits have been reached"
}