Merge branch 'develop' of github.com:Budibase/budibase into frontend-core
This commit is contained in:
Andrew Kingston
commit
9c38624d3a
|
|
@ -99,7 +99,9 @@ spec:
|
|||
- name: PLATFORM_URL
|
||||
value: {{ .Values.globals.platformUrl | quote }}
|
||||
- name: USE_QUOTAS
|
||||
value: "1"
|
||||
value: {{ .Values.globals.useQuotas | quote }}
|
||||
- name: EXCLUDE_QUOTAS_TENANTS
|
||||
value: {{ .Values.globals.excludeQuotasTenants | quote }}
|
||||
- name: ACCOUNT_PORTAL_URL
|
||||
value: {{ .Values.globals.accountPortalUrl | quote }}
|
||||
- name: ACCOUNT_PORTAL_API_KEY
|
||||
|
|
|
|||
|
|
@ -93,6 +93,8 @@ globals:
|
|||
logLevel: info
|
||||
selfHosted: "1" # set to 0 for budibase cloud environment, set to 1 for self-hosted setup
|
||||
multiTenancy: "0" # set to 0 to disable multiple orgs, set to 1 to enable multiple orgs
|
||||
useQuotas: "0"
|
||||
excludeQuotasTenants: "" # comma seperated list of tenants to exclude from quotas
|
||||
accountPortalUrl: ""
|
||||
accountPortalApiKey: ""
|
||||
cookieDomain: ""
|
||||
|
|
@ -239,7 +241,8 @@ couchdb:
|
|||
hosts:
|
||||
- chart-example.local
|
||||
path: /
|
||||
annotations: []
|
||||
annotations:
|
||||
[]
|
||||
# kubernetes.io/ingress.class: nginx
|
||||
# kubernetes.io/tls-acme: "true"
|
||||
tls:
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
{
|
||||
"version": "1.0.46-alpha.0",
|
||||
"version": "1.0.46-alpha.3",
|
||||
"npmClient": "yarn",
|
||||
"packages": [
|
||||
"packages/*"
|
||||
|
|
|
|||
|
|
@ -0,0 +1 @@
|
|||
module.exports = require("./src/migrations")
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"name": "@budibase/backend-core",
|
||||
"version": "1.0.46-alpha.0",
|
||||
"version": "1.0.46-alpha.3",
|
||||
"description": "Budibase backend core libraries used in server and worker",
|
||||
"main": "src/index.js",
|
||||
"author": "Budibase",
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ exports.StaticDatabases = {
|
|||
name: "global-db",
|
||||
docs: {
|
||||
apiKeys: "apikeys",
|
||||
usageQuota: "usage_quota",
|
||||
},
|
||||
},
|
||||
// contains information about tenancy and so on
|
||||
|
|
@ -28,7 +29,6 @@ exports.StaticDatabases = {
|
|||
name: "global-info",
|
||||
docs: {
|
||||
tenants: "tenants",
|
||||
usageQuota: "usage_quota",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
|
|
|||
|
|
@ -450,7 +450,7 @@ async function getScopedConfig(db, params) {
|
|||
|
||||
function generateNewUsageQuotaDoc() {
|
||||
return {
|
||||
_id: StaticDatabases.PLATFORM_INFO.docs.usageQuota,
|
||||
_id: StaticDatabases.GLOBAL.docs.usageQuota,
|
||||
quotaReset: Date.now() + 2592000000,
|
||||
usageQuota: {
|
||||
automationRuns: 0,
|
||||
|
|
|
|||
|
|
@ -14,4 +14,5 @@ module.exports = {
|
|||
cache: require("../cache"),
|
||||
auth: require("../auth"),
|
||||
constants: require("../constants"),
|
||||
migrations: require("../migrations"),
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
const { DocumentTypes } = require("../db/constants")
|
||||
const { getGlobalDB } = require("../tenancy")
|
||||
const { getGlobalDB, getTenantId } = require("../tenancy")
|
||||
|
||||
exports.MIGRATION_DBS = {
|
||||
GLOBAL_DB: "GLOBAL_DB",
|
||||
|
|
@ -7,11 +7,13 @@ exports.MIGRATION_DBS = {
|
|||
|
||||
exports.MIGRATIONS = {
|
||||
USER_EMAIL_VIEW_CASING: "user_email_view_casing",
|
||||
QUOTAS_1: "quotas_1",
|
||||
}
|
||||
|
||||
const DB_LOOKUP = {
|
||||
[exports.MIGRATION_DBS.GLOBAL_DB]: [
|
||||
exports.MIGRATIONS.USER_EMAIL_VIEW_CASING,
|
||||
exports.MIGRATIONS.QUOTAS_1,
|
||||
],
|
||||
}
|
||||
|
||||
|
|
@ -27,6 +29,7 @@ exports.getMigrationsDoc = async db => {
|
|||
}
|
||||
|
||||
exports.migrateIfRequired = async (migrationDb, migrationName, migrateFn) => {
|
||||
const tenantId = getTenantId()
|
||||
try {
|
||||
let db
|
||||
if (migrationDb === exports.MIGRATION_DBS.GLOBAL_DB) {
|
||||
|
|
@ -47,15 +50,18 @@ exports.migrateIfRequired = async (migrationDb, migrationName, migrateFn) => {
|
|||
return
|
||||
}
|
||||
|
||||
console.log(`Performing migration: ${migrationName}`)
|
||||
console.log(`[Tenant: ${tenantId}] Performing migration: ${migrationName}`)
|
||||
await migrateFn()
|
||||
console.log(`Migration complete: ${migrationName}`)
|
||||
console.log(`[Tenant: ${tenantId}] Migration complete: ${migrationName}`)
|
||||
|
||||
// mark as complete
|
||||
doc[migrationName] = Date.now()
|
||||
await db.put(doc)
|
||||
} catch (err) {
|
||||
console.error(`Error performing migration: ${migrationName}: `, err)
|
||||
console.error(
|
||||
`[Tenant: ${tenantId}] Error performing migration: ${migrationName}: `,
|
||||
err
|
||||
)
|
||||
throw err
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
{
|
||||
"name": "@budibase/bbui",
|
||||
"description": "A UI solution used in the different Budibase projects.",
|
||||
"version": "1.0.46-alpha.0",
|
||||
"version": "1.0.46-alpha.3",
|
||||
"license": "MPL-2.0",
|
||||
"svelte": "src/index.js",
|
||||
"module": "dist/bbui.es.js",
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"name": "@budibase/builder",
|
||||
"version": "1.0.46-alpha.0",
|
||||
"version": "1.0.46-alpha.3",
|
||||
"license": "GPL-3.0",
|
||||
"private": true,
|
||||
"scripts": {
|
||||
|
|
@ -65,11 +65,11 @@
|
|||
}
|
||||
},
|
||||
"dependencies": {
|
||||
"@budibase/bbui": "^1.0.46-alpha.0",
|
||||
"@budibase/client": "^1.0.46-alpha.0",
|
||||
"@budibase/frontend-core": "^1.0.46-alpha.0",
|
||||
"@budibase/bbui": "^1.0.46-alpha.3",
|
||||
"@budibase/client": "^1.0.46-alpha.3",
|
||||
"@budibase/frontend-core": "^1.0.46-alpha.3",
|
||||
"@budibase/colorpicker": "1.1.2",
|
||||
"@budibase/string-templates": "^1.0.46-alpha.0",
|
||||
"@budibase/string-templates": "^1.0.46-alpha.3",
|
||||
"@sentry/browser": "5.19.1",
|
||||
"@spectrum-css/page": "^3.0.1",
|
||||
"@spectrum-css/vars": "^3.0.1",
|
||||
|
|
|
|||
|
|
@ -53,7 +53,9 @@
|
|||
}
|
||||
|
||||
// Create table
|
||||
const table = await tables.save(newTable)
|
||||
let table
|
||||
try {
|
||||
table = await tables.save(newTable)
|
||||
notifications.success(`Table ${name} created successfully.`)
|
||||
analytics.captureEvent(Events.TABLE.CREATED, { name })
|
||||
|
||||
|
|
@ -63,6 +65,11 @@
|
|||
? `./table/${table._id}`
|
||||
: `../../table/${table._id}`
|
||||
$goto(path)
|
||||
} catch (e) {
|
||||
notifications.error(e)
|
||||
// reload in case the table was created
|
||||
await tables.fetch()
|
||||
}
|
||||
}
|
||||
</script>
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"name": "@budibase/cli",
|
||||
"version": "1.0.46-alpha.0",
|
||||
"version": "1.0.46-alpha.3",
|
||||
"description": "Budibase CLI, for developers, self hosting and migrations.",
|
||||
"main": "src/index.js",
|
||||
"bin": {
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"name": "@budibase/client",
|
||||
"version": "1.0.46-alpha.0",
|
||||
"version": "1.0.46-alpha.3",
|
||||
"license": "MPL-2.0",
|
||||
"module": "dist/budibase-client.js",
|
||||
"main": "dist/budibase-client.js",
|
||||
|
|
@ -19,10 +19,10 @@
|
|||
"dev:builder": "rollup -cw"
|
||||
},
|
||||
"dependencies": {
|
||||
"@budibase/bbui": "^1.0.46-alpha.0",
|
||||
"@budibase/frontend-core": "^1.0.46-alpha.0",
|
||||
"@budibase/bbui": "^1.0.46-alpha.3",
|
||||
"@budibase/frontend-core": "^1.0.46-alpha.3",
|
||||
"@budibase/standard-components": "^0.9.139",
|
||||
"@budibase/string-templates": "^1.0.46-alpha.0",
|
||||
"@budibase/string-templates": "^1.0.46-alpha.3",
|
||||
"regexparam": "^1.3.0",
|
||||
"shortid": "^2.2.15",
|
||||
"svelte-spa-router": "^3.0.5"
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"name": "@budibase/frontend-core",
|
||||
"version": "1.0.46-alpha.0",
|
||||
"version": "1.0.46-alpha.3",
|
||||
"description": "Budibase frontend core libraries used in builder and client",
|
||||
"author": "Budibase",
|
||||
"license": "GPL-3.0",
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
{
|
||||
"name": "@budibase/server",
|
||||
"email": "hi@budibase.com",
|
||||
"version": "1.0.46-alpha.0",
|
||||
"version": "1.0.46-alpha.3",
|
||||
"description": "Budibase Web Server",
|
||||
"main": "src/index.ts",
|
||||
"repository": {
|
||||
|
|
@ -70,9 +70,9 @@
|
|||
"license": "GPL-3.0",
|
||||
"dependencies": {
|
||||
"@apidevtools/swagger-parser": "^10.0.3",
|
||||
"@budibase/backend-core": "^1.0.46-alpha.0",
|
||||
"@budibase/client": "^1.0.46-alpha.0",
|
||||
"@budibase/string-templates": "^1.0.46-alpha.0",
|
||||
"@budibase/backend-core": "^1.0.46-alpha.3",
|
||||
"@budibase/client": "^1.0.46-alpha.3",
|
||||
"@budibase/string-templates": "^1.0.46-alpha.3",
|
||||
"@bull-board/api": "^3.7.0",
|
||||
"@bull-board/koa": "^3.7.0",
|
||||
"@elastic/elasticsearch": "7.10.0",
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ const {
|
|||
getTable,
|
||||
handleDataImport,
|
||||
} = require("./utils")
|
||||
const usageQuota = require("../../../utilities/usageQuota")
|
||||
|
||||
exports.save = async function (ctx) {
|
||||
const appId = ctx.appId
|
||||
|
|
@ -119,6 +120,7 @@ exports.destroy = async function (ctx) {
|
|||
})
|
||||
)
|
||||
await db.bulkDocs(rows.rows.map(row => ({ ...row.doc, _deleted: true })))
|
||||
await usageQuota.update(usageQuota.Properties.ROW, -rows.rows.length)
|
||||
|
||||
// update linked rows
|
||||
await linkRows.updateLinks({
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ const {
|
|||
} = require("../../../integrations/utils")
|
||||
const { getViews, saveView } = require("../view/utils")
|
||||
const viewTemplate = require("../view/viewBuilder")
|
||||
const usageQuota = require("../../../utilities/usageQuota")
|
||||
|
||||
exports.checkForColumnUpdates = async (db, oldTable, updatedTable) => {
|
||||
let updatedRows = []
|
||||
|
|
@ -112,7 +113,11 @@ exports.handleDataImport = async (appId, user, table, dataImport) => {
|
|||
finalData.push(row)
|
||||
}
|
||||
|
||||
await usageQuota.update(usageQuota.Properties.ROW, finalData.length, {
|
||||
dryRun: true,
|
||||
})
|
||||
await db.bulkDocs(finalData)
|
||||
await usageQuota.update(usageQuota.Properties.ROW, finalData.length)
|
||||
let response = await db.put(table)
|
||||
table._rev = response._rev
|
||||
return table
|
||||
|
|
|
|||
|
|
@ -1,6 +1,5 @@
|
|||
const rowController = require("../../api/controllers/row")
|
||||
const automationUtils = require("../automationUtils")
|
||||
const env = require("../../environment")
|
||||
const usage = require("../../utilities/usageQuota")
|
||||
const { buildCtx } = require("./utils")
|
||||
|
||||
|
|
@ -83,10 +82,9 @@ exports.run = async function ({ inputs, appId, emitter }) {
|
|||
inputs.row.tableId,
|
||||
inputs.row
|
||||
)
|
||||
if (env.USE_QUOTAS) {
|
||||
await usage.update(usage.Properties.ROW, 1)
|
||||
}
|
||||
await usage.update(usage.Properties.ROW, 1, { dryRun: true })
|
||||
await rowController.save(ctx)
|
||||
await usage.update(usage.Properties.ROW, 1)
|
||||
return {
|
||||
row: inputs.row,
|
||||
response: ctx.body,
|
||||
|
|
|
|||
|
|
@ -1,5 +1,4 @@
|
|||
const rowController = require("../../api/controllers/row")
|
||||
const env = require("../../environment")
|
||||
const usage = require("../../utilities/usageQuota")
|
||||
const { buildCtx } = require("./utils")
|
||||
const automationUtils = require("../automationUtils")
|
||||
|
|
@ -74,9 +73,7 @@ exports.run = async function ({ inputs, appId, emitter }) {
|
|||
})
|
||||
|
||||
try {
|
||||
if (env.isProd()) {
|
||||
await usage.update(usage.Properties.ROW, -1)
|
||||
}
|
||||
await rowController.destroy(ctx)
|
||||
return {
|
||||
response: ctx.body,
|
||||
|
|
|
|||
|
|
@ -38,6 +38,7 @@ module.exports = {
|
|||
MINIO_ACCESS_KEY: process.env.MINIO_ACCESS_KEY,
|
||||
MINIO_SECRET_KEY: process.env.MINIO_SECRET_KEY,
|
||||
USE_QUOTAS: process.env.USE_QUOTAS,
|
||||
EXCLUDE_QUOTAS_TENANTS: process.env.EXCLUDE_QUOTAS_TENANTS,
|
||||
REDIS_URL: process.env.REDIS_URL,
|
||||
REDIS_PASSWORD: process.env.REDIS_PASSWORD,
|
||||
INTERNAL_API_KEY: process.env.INTERNAL_API_KEY,
|
||||
|
|
|
|||
|
|
@ -1,11 +1,5 @@
|
|||
jest.mock("../../db")
|
||||
jest.mock("../../utilities/usageQuota")
|
||||
jest.mock("../../environment", () => ({
|
||||
isTest: () => true,
|
||||
isProd: () => false,
|
||||
isDev: () => true,
|
||||
_set: () => {},
|
||||
}))
|
||||
jest.mock("@budibase/backend-core/tenancy", () => ({
|
||||
getTenantId: () => "testing123"
|
||||
}))
|
||||
|
|
@ -29,9 +23,10 @@ class TestConfiguration {
|
|||
},
|
||||
req: {
|
||||
method: "POST",
|
||||
url: "/rows"
|
||||
url: "/applications"
|
||||
}
|
||||
}
|
||||
usageQuota.useQuotas = () => true
|
||||
}
|
||||
|
||||
executeMiddleware() {
|
||||
|
|
@ -113,7 +108,6 @@ describe("usageQuota middleware", () => {
|
|||
|
||||
it("calculates and persists the correct usage quota for the relevant action", async () => {
|
||||
config.setUrl("/rows")
|
||||
config.setProd(true)
|
||||
|
||||
await config.executeMiddleware()
|
||||
|
||||
|
|
@ -121,20 +115,20 @@ describe("usageQuota middleware", () => {
|
|||
expect(config.next).toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it("calculates the correct file size from a file upload call and adds it to quota", async () => {
|
||||
config.setUrl("/upload")
|
||||
config.setProd(true)
|
||||
config.setFiles([
|
||||
{
|
||||
size: 100
|
||||
},
|
||||
{
|
||||
size: 10000
|
||||
},
|
||||
])
|
||||
await config.executeMiddleware()
|
||||
// it("calculates the correct file size from a file upload call and adds it to quota", async () => {
|
||||
// config.setUrl("/upload")
|
||||
// config.setProd(true)
|
||||
// config.setFiles([
|
||||
// {
|
||||
// size: 100
|
||||
// },
|
||||
// {
|
||||
// size: 10000
|
||||
// },
|
||||
// ])
|
||||
// await config.executeMiddleware()
|
||||
|
||||
expect(usageQuota.update).toHaveBeenCalledWith("storage", 10100)
|
||||
expect(config.next).toHaveBeenCalled()
|
||||
})
|
||||
// expect(usageQuota.update).toHaveBeenCalledWith("storage", 10100)
|
||||
// expect(config.next).toHaveBeenCalled()
|
||||
// })
|
||||
})
|
||||
|
|
@ -1,14 +1,11 @@
|
|||
const CouchDB = require("../db")
|
||||
const usageQuota = require("../utilities/usageQuota")
|
||||
const env = require("../environment")
|
||||
const { getTenantId } = require("@budibase/backend-core/tenancy")
|
||||
const { getUniqueRows } = require("../utilities/usageQuota/rows")
|
||||
const {
|
||||
isExternalTable,
|
||||
isRowId: isExternalRowId,
|
||||
} = require("../integrations/utils")
|
||||
|
||||
// tenants without limits
|
||||
const EXCLUDED_TENANTS = ["bb", "default", "bbtest", "bbstaging"]
|
||||
const migration = require("../migrations/usageQuotas")
|
||||
|
||||
// currently only counting new writes and deletes
|
||||
const METHOD_MAP = {
|
||||
|
|
@ -18,13 +15,13 @@ const METHOD_MAP = {
|
|||
|
||||
const DOMAIN_MAP = {
|
||||
rows: usageQuota.Properties.ROW,
|
||||
upload: usageQuota.Properties.UPLOAD,
|
||||
views: usageQuota.Properties.VIEW,
|
||||
users: usageQuota.Properties.USER,
|
||||
// upload: usageQuota.Properties.UPLOAD, // doesn't work yet
|
||||
// views: usageQuota.Properties.VIEW, // doesn't work yet
|
||||
// users: usageQuota.Properties.USER, // doesn't work yet
|
||||
applications: usageQuota.Properties.APPS,
|
||||
// this will not be updated by endpoint calls
|
||||
// instead it will be updated by triggerInfo
|
||||
automationRuns: usageQuota.Properties.AUTOMATION,
|
||||
// automationRuns: usageQuota.Properties.AUTOMATION, // doesn't work yet
|
||||
}
|
||||
|
||||
function getProperty(url) {
|
||||
|
|
@ -36,10 +33,7 @@ function getProperty(url) {
|
|||
}
|
||||
|
||||
module.exports = async (ctx, next) => {
|
||||
const tenantId = getTenantId()
|
||||
|
||||
// if in development or a self hosted cloud usage quotas should not be executed
|
||||
if (env.isDev() || env.SELF_HOSTED || EXCLUDED_TENANTS.includes(tenantId)) {
|
||||
if (!usageQuota.useQuotas()) {
|
||||
return next()
|
||||
}
|
||||
|
||||
|
|
@ -80,9 +74,93 @@ module.exports = async (ctx, next) => {
|
|||
usage = files.map(file => file.size).reduce((total, size) => total + size)
|
||||
}
|
||||
try {
|
||||
await usageQuota.update(property, usage)
|
||||
return next()
|
||||
await migration.run()
|
||||
await performRequest(ctx, next, property, usage)
|
||||
} catch (err) {
|
||||
ctx.throw(400, err)
|
||||
}
|
||||
}
|
||||
|
||||
const performRequest = async (ctx, next, property, usage) => {
|
||||
const usageContext = {
|
||||
skipNext: false,
|
||||
skipUsage: false,
|
||||
[usageQuota.Properties.APPS]: {},
|
||||
}
|
||||
|
||||
if (usage === -1) {
|
||||
if (PRE_DELETE[property]) {
|
||||
await PRE_DELETE[property](ctx, usageContext)
|
||||
}
|
||||
} else {
|
||||
if (PRE_CREATE[property]) {
|
||||
await PRE_CREATE[property](ctx, usageContext)
|
||||
}
|
||||
}
|
||||
|
||||
// run the request
|
||||
if (!usageContext.skipNext) {
|
||||
await usageQuota.update(property, usage, { dryRun: true })
|
||||
await next()
|
||||
}
|
||||
|
||||
if (usage === -1) {
|
||||
if (POST_DELETE[property]) {
|
||||
await POST_DELETE[property](ctx, usageContext)
|
||||
}
|
||||
} else {
|
||||
if (POST_CREATE[property]) {
|
||||
await POST_CREATE[property](ctx, usageContext)
|
||||
}
|
||||
}
|
||||
|
||||
// update the usage
|
||||
if (!usageContext.skipUsage) {
|
||||
await usageQuota.update(property, usage)
|
||||
}
|
||||
}
|
||||
|
||||
const appPreDelete = async (ctx, usageContext) => {
|
||||
if (ctx.query.unpublish) {
|
||||
// don't run usage decrement for unpublish
|
||||
usageContext.skipUsage = true
|
||||
return
|
||||
}
|
||||
|
||||
// store the row count to delete
|
||||
const rows = await getUniqueRows([ctx.appId])
|
||||
if (rows.length) {
|
||||
usageContext[usageQuota.Properties.APPS] = { rowCount: rows.length }
|
||||
}
|
||||
}
|
||||
|
||||
const appPostDelete = async (ctx, usageContext) => {
|
||||
// delete the app rows from usage
|
||||
const rowCount = usageContext[usageQuota.Properties.APPS].rowCount
|
||||
if (rowCount) {
|
||||
await usageQuota.update(usageQuota.Properties.ROW, -rowCount)
|
||||
}
|
||||
}
|
||||
|
||||
const appPostCreate = async ctx => {
|
||||
// app import & template creation
|
||||
if (ctx.request.body.useTemplate === "true") {
|
||||
const rows = await getUniqueRows([ctx.response.body.appId])
|
||||
const rowCount = rows ? rows.length : 0
|
||||
await usageQuota.update(usageQuota.Properties.ROW, rowCount)
|
||||
}
|
||||
}
|
||||
|
||||
const PRE_DELETE = {
|
||||
[usageQuota.Properties.APPS]: appPreDelete,
|
||||
}
|
||||
|
||||
const POST_DELETE = {
|
||||
[usageQuota.Properties.APPS]: appPostDelete,
|
||||
}
|
||||
|
||||
const PRE_CREATE = {}
|
||||
|
||||
const POST_CREATE = {
|
||||
[usageQuota.Properties.APPS]: appPostCreate,
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,27 @@
|
|||
const env = require("../../../environment")
|
||||
const TestConfig = require("../../../tests/utilities/TestConfiguration")
|
||||
|
||||
const syncApps = jest.fn()
|
||||
const syncRows = jest.fn()
|
||||
|
||||
jest.mock("../../usageQuotas/syncApps", () => ({ run: syncApps }) )
|
||||
jest.mock("../../usageQuotas/syncRows", () => ({ run: syncRows }) )
|
||||
|
||||
const migrations = require("../../usageQuotas")
|
||||
|
||||
describe("run", () => {
|
||||
let config = new TestConfig(false)
|
||||
|
||||
beforeEach(async () => {
|
||||
await config.init()
|
||||
env._set("USE_QUOTAS", 1)
|
||||
})
|
||||
|
||||
afterAll(config.end)
|
||||
|
||||
it("runs the required migrations", async () => {
|
||||
await migrations.run()
|
||||
expect(syncApps).toHaveBeenCalledTimes(1)
|
||||
expect(syncRows).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
})
|
||||
|
|
@ -0,0 +1,37 @@
|
|||
const { getGlobalDB } = require("@budibase/backend-core/tenancy")
|
||||
const TestConfig = require("../../../tests/utilities/TestConfiguration")
|
||||
const { getUsageQuotaDoc, update, Properties } = require("../../../utilities/usageQuota")
|
||||
const syncApps = require("../../usageQuotas/syncApps")
|
||||
const env = require("../../../environment")
|
||||
|
||||
describe("syncApps", () => {
|
||||
let config = new TestConfig(false)
|
||||
|
||||
beforeEach(async () => {
|
||||
await config.init()
|
||||
env._set("USE_QUOTAS", 1)
|
||||
})
|
||||
|
||||
afterAll(config.end)
|
||||
|
||||
it("runs successfully", async () => {
|
||||
// create the usage quota doc and mock usages
|
||||
const db = getGlobalDB()
|
||||
await getUsageQuotaDoc(db)
|
||||
await update(Properties.APPS, 3)
|
||||
|
||||
let usageDoc = await getUsageQuotaDoc(db)
|
||||
expect(usageDoc.usageQuota.apps).toEqual(3)
|
||||
|
||||
// create an extra app to test the migration
|
||||
await config.createApp("quota-test")
|
||||
|
||||
// migrate
|
||||
await syncApps.run()
|
||||
|
||||
// assert the migration worked
|
||||
usageDoc = await getUsageQuotaDoc(db)
|
||||
expect(usageDoc.usageQuota.apps).toEqual(2)
|
||||
})
|
||||
})
|
||||
|
||||
|
|
@ -0,0 +1,43 @@
|
|||
const { getGlobalDB } = require("@budibase/backend-core/tenancy")
|
||||
const TestConfig = require("../../../tests/utilities/TestConfiguration")
|
||||
const { getUsageQuotaDoc, update, Properties } = require("../../../utilities/usageQuota")
|
||||
const syncRows = require("../../usageQuotas/syncRows")
|
||||
const env = require("../../../environment")
|
||||
|
||||
describe("syncRows", () => {
|
||||
let config = new TestConfig(false)
|
||||
|
||||
beforeEach(async () => {
|
||||
await config.init()
|
||||
env._set("USE_QUOTAS", 1)
|
||||
})
|
||||
|
||||
afterAll(config.end)
|
||||
|
||||
it("runs successfully", async () => {
|
||||
// create the usage quota doc and mock usages
|
||||
const db = getGlobalDB()
|
||||
await getUsageQuotaDoc(db)
|
||||
await update(Properties.ROW, 300)
|
||||
|
||||
let usageDoc = await getUsageQuotaDoc(db)
|
||||
expect(usageDoc.usageQuota.rows).toEqual(300)
|
||||
|
||||
// app 1
|
||||
await config.createTable()
|
||||
await config.createRow()
|
||||
// app 2
|
||||
await config.createApp()
|
||||
await config.createTable()
|
||||
await config.createRow()
|
||||
await config.createRow()
|
||||
|
||||
// migrate
|
||||
await syncRows.run()
|
||||
|
||||
// assert the migration worked
|
||||
usageDoc = await getUsageQuotaDoc(db)
|
||||
expect(usageDoc.usageQuota.rows).toEqual(3)
|
||||
})
|
||||
})
|
||||
|
||||
|
|
@ -0,0 +1,24 @@
|
|||
const {
|
||||
MIGRATIONS,
|
||||
MIGRATION_DBS,
|
||||
migrateIfRequired,
|
||||
} = require("@budibase/backend-core/migrations")
|
||||
const { useQuotas } = require("../../utilities/usageQuota")
|
||||
const syncApps = require("./syncApps")
|
||||
const syncRows = require("./syncRows")
|
||||
|
||||
exports.run = async () => {
|
||||
if (!useQuotas()) {
|
||||
return
|
||||
}
|
||||
|
||||
// Jan 2022
|
||||
await migrateIfRequired(
|
||||
MIGRATION_DBS.GLOBAL_DB,
|
||||
MIGRATIONS.QUOTAS_1,
|
||||
async () => {
|
||||
await syncApps.run()
|
||||
await syncRows.run()
|
||||
}
|
||||
)
|
||||
}
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
const { getGlobalDB, getTenantId } = require("@budibase/backend-core/tenancy")
|
||||
const { getAllApps } = require("@budibase/backend-core/db")
|
||||
const CouchDB = require("../../db")
|
||||
const { getUsageQuotaDoc } = require("../../utilities/usageQuota")
|
||||
|
||||
exports.run = async () => {
|
||||
const db = getGlobalDB()
|
||||
// get app count
|
||||
const devApps = await getAllApps(CouchDB, { dev: true })
|
||||
const appCount = devApps ? devApps.length : 0
|
||||
|
||||
// sync app count
|
||||
const tenantId = getTenantId()
|
||||
console.log(`[Tenant: ${tenantId}] Syncing app count: ${appCount}`)
|
||||
const usageDoc = await getUsageQuotaDoc(db)
|
||||
usageDoc.usageQuota.apps = appCount
|
||||
await db.put(usageDoc)
|
||||
}
|
||||
|
|
@ -0,0 +1,21 @@
|
|||
const { getGlobalDB, getTenantId } = require("@budibase/backend-core/tenancy")
|
||||
const { getAllApps } = require("@budibase/backend-core/db")
|
||||
const CouchDB = require("../../db")
|
||||
const { getUsageQuotaDoc } = require("../../utilities/usageQuota")
|
||||
const { getUniqueRows } = require("../../utilities/usageQuota/rows")
|
||||
|
||||
exports.run = async () => {
|
||||
const db = getGlobalDB()
|
||||
// get all rows in all apps
|
||||
const allApps = await getAllApps(CouchDB, { all: true })
|
||||
const appIds = allApps ? allApps.map(app => app.appId) : []
|
||||
const rows = await getUniqueRows(appIds)
|
||||
const rowCount = rows ? rows.length : 0
|
||||
|
||||
// sync row count
|
||||
const tenantId = getTenantId()
|
||||
console.log(`[Tenant: ${tenantId}] Syncing row count: ${rowCount}`)
|
||||
const usageDoc = await getUsageQuotaDoc(db)
|
||||
usageDoc.usageQuota.rows = rowCount
|
||||
await db.put(usageDoc)
|
||||
}
|
||||
|
|
@ -0,0 +1,72 @@
|
|||
const getTenantId = jest.fn()
|
||||
jest.mock("@budibase/backend-core/tenancy", () => ({
|
||||
getTenantId
|
||||
}))
|
||||
const usageQuota = require("../../usageQuota")
|
||||
const env = require("../../../environment")
|
||||
|
||||
class TestConfiguration {
|
||||
constructor() {
|
||||
this.enableQuotas()
|
||||
}
|
||||
|
||||
enableQuotas = () => {
|
||||
env.USE_QUOTAS = 1
|
||||
}
|
||||
|
||||
disableQuotas = () => {
|
||||
env.USE_QUOTAS = null
|
||||
}
|
||||
|
||||
setTenantId = (tenantId) => {
|
||||
getTenantId.mockReturnValue(tenantId)
|
||||
}
|
||||
|
||||
setExcludedTenants = (tenants) => {
|
||||
env.EXCLUDE_QUOTAS_TENANTS = tenants
|
||||
}
|
||||
|
||||
reset = () => {
|
||||
this.disableQuotas()
|
||||
this.setExcludedTenants(null)
|
||||
}
|
||||
}
|
||||
|
||||
describe("usageQuota", () => {
|
||||
let config
|
||||
|
||||
beforeEach(() => {
|
||||
config = new TestConfiguration()
|
||||
})
|
||||
|
||||
afterEach(() => {
|
||||
config.reset()
|
||||
})
|
||||
|
||||
describe("useQuotas", () => {
|
||||
it("works when no settings have been provided", () => {
|
||||
config.reset()
|
||||
expect(usageQuota.useQuotas()).toBe(false)
|
||||
})
|
||||
it("honours USE_QUOTAS setting", () => {
|
||||
config.disableQuotas()
|
||||
expect(usageQuota.useQuotas()).toBe(false)
|
||||
|
||||
config.enableQuotas()
|
||||
expect(usageQuota.useQuotas()).toBe(true)
|
||||
})
|
||||
it("honours EXCLUDE_QUOTAS_TENANTS setting", () => {
|
||||
config.setTenantId("test")
|
||||
|
||||
// tenantId is in the list
|
||||
config.setExcludedTenants("test, test2, test2")
|
||||
expect(usageQuota.useQuotas()).toBe(false)
|
||||
config.setExcludedTenants("test,test2,test2")
|
||||
expect(usageQuota.useQuotas()).toBe(false)
|
||||
|
||||
// tenantId is not in the list
|
||||
config.setTenantId("other")
|
||||
expect(usageQuota.useQuotas()).toBe(true)
|
||||
})
|
||||
})
|
||||
})
|
||||
|
|
@ -1,73 +0,0 @@
|
|||
const env = require("../environment")
|
||||
const { getGlobalDB } = require("@budibase/backend-core/tenancy")
|
||||
const {
|
||||
StaticDatabases,
|
||||
generateNewUsageQuotaDoc,
|
||||
} = require("@budibase/backend-core/db")
|
||||
|
||||
function getNewQuotaReset() {
|
||||
return Date.now() + 2592000000
|
||||
}
|
||||
|
||||
exports.Properties = {
|
||||
ROW: "rows",
|
||||
UPLOAD: "storage",
|
||||
VIEW: "views",
|
||||
USER: "users",
|
||||
AUTOMATION: "automationRuns",
|
||||
APPS: "apps",
|
||||
EMAILS: "emails",
|
||||
}
|
||||
|
||||
async function getUsageQuotaDoc(db) {
|
||||
let quota
|
||||
try {
|
||||
quota = await db.get(StaticDatabases.PLATFORM_INFO.docs.usageQuota)
|
||||
} catch (err) {
|
||||
// doc doesn't exist. Create it
|
||||
quota = await db.post(generateNewUsageQuotaDoc())
|
||||
}
|
||||
|
||||
return quota
|
||||
}
|
||||
|
||||
/**
|
||||
* Given a specified tenantId this will add to the usage object for the specified property.
|
||||
* @param {string} property The property which is to be added to (within the nested usageQuota object).
|
||||
* @param {number} usage The amount (this can be negative) to adjust the number by.
|
||||
* @returns {Promise<void>} When this completes the API key will now be up to date - the quota period may have
|
||||
* also been reset after this call.
|
||||
*/
|
||||
exports.update = async (property, usage) => {
|
||||
if (!env.USE_QUOTAS) {
|
||||
return
|
||||
}
|
||||
|
||||
try {
|
||||
const db = getGlobalDB()
|
||||
const quota = await getUsageQuotaDoc(db)
|
||||
|
||||
// Check if the quota needs reset
|
||||
if (Date.now() >= quota.quotaReset) {
|
||||
quota.quotaReset = getNewQuotaReset()
|
||||
for (let prop of Object.keys(quota.usageQuota)) {
|
||||
quota.usageQuota[prop] = 0
|
||||
}
|
||||
}
|
||||
|
||||
// increment the quota
|
||||
quota.usageQuota[property] += usage
|
||||
|
||||
if (quota.usageQuota[property] > quota.usageLimits[property]) {
|
||||
throw new Error(
|
||||
`You have exceeded your usage quota of ${quota.usageLimits[property]} ${property}.`
|
||||
)
|
||||
}
|
||||
|
||||
// update the usage quotas
|
||||
await db.put(quota)
|
||||
} catch (err) {
|
||||
console.error(`Error updating usage quotas for ${property}`, err)
|
||||
throw err
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,92 @@
|
|||
const env = require("../../environment")
|
||||
const { getGlobalDB, getTenantId } = require("@budibase/backend-core/tenancy")
|
||||
const {
|
||||
StaticDatabases,
|
||||
generateNewUsageQuotaDoc,
|
||||
} = require("@budibase/backend-core/db")
|
||||
|
||||
exports.useQuotas = () => {
|
||||
// check if quotas are enabled
|
||||
if (env.USE_QUOTAS) {
|
||||
// check if there are any tenants without limits
|
||||
if (env.EXCLUDE_QUOTAS_TENANTS) {
|
||||
const excludedTenants = env.EXCLUDE_QUOTAS_TENANTS.replace(
|
||||
/\s/g,
|
||||
""
|
||||
).split(",")
|
||||
const tenantId = getTenantId()
|
||||
if (excludedTenants.includes(tenantId)) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
exports.Properties = {
|
||||
ROW: "rows",
|
||||
UPLOAD: "storage", // doesn't work yet
|
||||
VIEW: "views", // doesn't work yet
|
||||
USER: "users", // doesn't work yet
|
||||
AUTOMATION: "automationRuns", // doesn't work yet
|
||||
APPS: "apps",
|
||||
EMAILS: "emails", // doesn't work yet
|
||||
}
|
||||
|
||||
exports.getUsageQuotaDoc = async db => {
|
||||
let quota
|
||||
try {
|
||||
quota = await db.get(StaticDatabases.GLOBAL.docs.usageQuota)
|
||||
} catch (err) {
|
||||
// doc doesn't exist. Create it
|
||||
quota = generateNewUsageQuotaDoc()
|
||||
const response = await db.put(quota)
|
||||
quota._rev = response.rev
|
||||
}
|
||||
|
||||
return quota
|
||||
}
|
||||
|
||||
/**
|
||||
* Given a specified tenantId this will add to the usage object for the specified property.
|
||||
* @param {string} property The property which is to be added to (within the nested usageQuota object).
|
||||
* @param {number} usage The amount (this can be negative) to adjust the number by.
|
||||
* @returns {Promise<void>} When this completes the API key will now be up to date - the quota period may have
|
||||
* also been reset after this call.
|
||||
*/
|
||||
exports.update = async (property, usage, opts = { dryRun: false }) => {
|
||||
if (!exports.useQuotas()) {
|
||||
return
|
||||
}
|
||||
|
||||
try {
|
||||
const db = getGlobalDB()
|
||||
const quota = await exports.getUsageQuotaDoc(db)
|
||||
|
||||
// increment the quota
|
||||
quota.usageQuota[property] += usage
|
||||
|
||||
if (
|
||||
quota.usageQuota[property] > quota.usageLimits[property] &&
|
||||
usage > 0 // allow for decrementing usage when the quota is already exceeded
|
||||
) {
|
||||
throw new Error(
|
||||
`You have exceeded your usage quota of ${quota.usageLimits[property]} ${property}.`
|
||||
)
|
||||
}
|
||||
|
||||
if (quota.usageQuota[property] < 0) {
|
||||
// never go negative if the quota has previously been exceeded
|
||||
quota.usageQuota[property] = 0
|
||||
}
|
||||
|
||||
// update the usage quotas
|
||||
if (!opts.dryRun) {
|
||||
await db.put(quota)
|
||||
}
|
||||
} catch (err) {
|
||||
console.error(`Error updating usage quotas for ${property}`, err)
|
||||
throw err
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,74 @@
|
|||
const { getRowParams, USER_METDATA_PREFIX } = require("../../db/utils")
|
||||
const CouchDB = require("../../db")
|
||||
const { isDevAppID, getDevelopmentAppID } = require("@budibase/backend-core/db")
|
||||
|
||||
const ROW_EXCLUSIONS = [USER_METDATA_PREFIX]
|
||||
|
||||
const getAppPairs = appIds => {
|
||||
// collect the app ids into dev / prod pairs
|
||||
// keyed by the dev app id
|
||||
const pairs = {}
|
||||
for (let appId of appIds) {
|
||||
const devId = getDevelopmentAppID(appId)
|
||||
if (!pairs[devId]) {
|
||||
pairs[devId] = {}
|
||||
}
|
||||
if (isDevAppID(appId)) {
|
||||
pairs[devId].devId = appId
|
||||
} else {
|
||||
pairs[devId].prodId = appId
|
||||
}
|
||||
}
|
||||
return pairs
|
||||
}
|
||||
|
||||
const getAppRows = async appId => {
|
||||
const appDb = new CouchDB(appId)
|
||||
const response = await appDb.allDocs(
|
||||
getRowParams(null, null, {
|
||||
include_docs: false,
|
||||
})
|
||||
)
|
||||
return response.rows
|
||||
.map(r => r.id)
|
||||
.filter(id => {
|
||||
for (let exclusion of ROW_EXCLUSIONS) {
|
||||
if (id.startsWith(exclusion)) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Return a set of all rows in the given app ids.
|
||||
* The returned rows will be unique on a per dev/prod app basis.
|
||||
* Rows duplicates may exist across apps due to data import so they are not filtered out.
|
||||
*/
|
||||
exports.getUniqueRows = async appIds => {
|
||||
let uniqueRows = []
|
||||
const pairs = getAppPairs(appIds)
|
||||
|
||||
for (let pair of Object.values(pairs)) {
|
||||
let appRows = []
|
||||
for (let appId of [pair.devId, pair.prodId]) {
|
||||
if (!appId) {
|
||||
continue
|
||||
}
|
||||
try {
|
||||
appRows.push(await getAppRows(appId))
|
||||
} catch (e) {
|
||||
console.error(e)
|
||||
// don't error out if we can't count the app rows, just continue
|
||||
}
|
||||
}
|
||||
|
||||
// ensure uniqueness on a per app pair basis
|
||||
// this can't be done on all rows because app import results in
|
||||
// duplicate row ids across apps
|
||||
uniqueRows = uniqueRows.concat(...new Set(appRows))
|
||||
}
|
||||
|
||||
return uniqueRows
|
||||
}
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
// UNUSED CODE
|
||||
// Preserved for future use
|
||||
|
||||
/* eslint-disable no-unused-vars */
|
||||
|
||||
function getNewQuotaReset() {
|
||||
return Date.now() + 2592000000
|
||||
}
|
||||
|
||||
function resetQuotasIfRequired(quota) {
|
||||
// Check if the quota needs reset
|
||||
if (Date.now() >= quota.quotaReset) {
|
||||
quota.quotaReset = getNewQuotaReset()
|
||||
for (let prop of Object.keys(quota.usageQuota)) {
|
||||
quota.usageQuota[prop] = 0
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"name": "@budibase/string-templates",
|
||||
"version": "1.0.46-alpha.0",
|
||||
"version": "1.0.46-alpha.3",
|
||||
"description": "Handlebars wrapper for Budibase templating.",
|
||||
"main": "src/index.cjs",
|
||||
"module": "dist/bundle.mjs",
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
{
|
||||
"name": "@budibase/worker",
|
||||
"email": "hi@budibase.com",
|
||||
"version": "1.0.46-alpha.0",
|
||||
"version": "1.0.46-alpha.3",
|
||||
"description": "Budibase background service",
|
||||
"main": "src/index.js",
|
||||
"repository": {
|
||||
|
|
@ -29,8 +29,8 @@
|
|||
"author": "Budibase",
|
||||
"license": "GPL-3.0",
|
||||
"dependencies": {
|
||||
"@budibase/backend-core": "^1.0.46-alpha.0",
|
||||
"@budibase/string-templates": "^1.0.46-alpha.0",
|
||||
"@budibase/backend-core": "^1.0.46-alpha.3",
|
||||
"@budibase/string-templates": "^1.0.46-alpha.3",
|
||||
"@koa/router": "^8.0.0",
|
||||
"@sentry/node": "^6.0.0",
|
||||
"@techpass/passport-openidconnect": "^0.3.0",
|
||||
|
|
|
|||
|
|
@ -73,16 +73,14 @@ exports.adminUser = async ctx => {
|
|||
if (!env.SELF_HOSTED) {
|
||||
// could be a scenario where it exists, make sure its clean
|
||||
try {
|
||||
const usageQuota = await db.get(
|
||||
StaticDatabases.PLATFORM_INFO.docs.usageQuota
|
||||
)
|
||||
const usageQuota = await db.get(StaticDatabases.GLOBAL.docs.usageQuota)
|
||||
if (usageQuota) {
|
||||
await db.remove(usageQuota._id, usageQuota._rev)
|
||||
}
|
||||
} catch (err) {
|
||||
// don't worry about errors
|
||||
}
|
||||
await db.post(generateNewUsageQuotaDoc())
|
||||
await db.put(generateNewUsageQuotaDoc())
|
||||
}
|
||||
|
||||
if (response.rows.some(row => row.doc.admin)) {
|
||||
|
|
|
|||
Loading…
Reference in New Issue