diff --git a/charts/budibase/templates/app-service-deployment.yaml b/charts/budibase/templates/app-service-deployment.yaml index 8086c0ab20..cd43631992 100644 --- a/charts/budibase/templates/app-service-deployment.yaml +++ b/charts/budibase/templates/app-service-deployment.yaml @@ -99,7 +99,9 @@ spec: - name: PLATFORM_URL value: {{ .Values.globals.platformUrl | quote }} - name: USE_QUOTAS - value: "1" + value: {{ .Values.globals.useQuotas | quote }} + - name: EXCLUDE_QUOTAS_TENANTS + value: {{ .Values.globals.excludeQuotasTenants | quote }} - name: ACCOUNT_PORTAL_URL value: {{ .Values.globals.accountPortalUrl | quote }} - name: ACCOUNT_PORTAL_API_KEY diff --git a/charts/budibase/values.yaml b/charts/budibase/values.yaml index 4666d01c70..9ea055c6c0 100644 --- a/charts/budibase/values.yaml +++ b/charts/budibase/values.yaml @@ -93,6 +93,8 @@ globals: logLevel: info selfHosted: "1" # set to 0 for budibase cloud environment, set to 1 for self-hosted setup multiTenancy: "0" # set to 0 to disable multiple orgs, set to 1 to enable multiple orgs + useQuotas: "0" + excludeQuotasTenants: "" # comma seperated list of tenants to exclude from quotas accountPortalUrl: "" accountPortalApiKey: "" cookieDomain: "" @@ -239,7 +241,8 @@ couchdb: hosts: - chart-example.local path: / - annotations: [] + annotations: + [] # kubernetes.io/ingress.class: nginx # kubernetes.io/tls-acme: "true" tls: diff --git a/lerna.json b/lerna.json index 78af255b1f..7564ea387c 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "1.0.44-alpha.9", + "version": "1.0.46-alpha.3", "npmClient": "yarn", "packages": [ "packages/*" diff --git a/packages/backend-core/migrations.js b/packages/backend-core/migrations.js new file mode 100644 index 0000000000..2de19ebf65 --- /dev/null +++ b/packages/backend-core/migrations.js @@ -0,0 +1 @@ +module.exports = require("./src/migrations") diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json index 6069396490..c4f7dd1ae8 100644 --- a/packages/backend-core/package.json +++ b/packages/backend-core/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/backend-core", - "version": "1.0.44-alpha.9", + "version": "1.0.46-alpha.3", "description": "Budibase backend core libraries used in server and worker", "main": "src/index.js", "author": "Budibase", diff --git a/packages/backend-core/src/db/constants.js b/packages/backend-core/src/db/constants.js index ecdaae5bad..2affb09c7c 100644 --- a/packages/backend-core/src/db/constants.js +++ b/packages/backend-core/src/db/constants.js @@ -21,6 +21,7 @@ exports.StaticDatabases = { name: "global-db", docs: { apiKeys: "apikeys", + usageQuota: "usage_quota", }, }, // contains information about tenancy and so on @@ -28,7 +29,6 @@ exports.StaticDatabases = { name: "global-info", docs: { tenants: "tenants", - usageQuota: "usage_quota", }, }, } diff --git a/packages/backend-core/src/db/utils.js b/packages/backend-core/src/db/utils.js index 5830de4721..2bc5462646 100644 --- a/packages/backend-core/src/db/utils.js +++ b/packages/backend-core/src/db/utils.js @@ -450,7 +450,7 @@ async function getScopedConfig(db, params) { function generateNewUsageQuotaDoc() { return { - _id: StaticDatabases.PLATFORM_INFO.docs.usageQuota, + _id: StaticDatabases.GLOBAL.docs.usageQuota, quotaReset: Date.now() + 2592000000, usageQuota: { automationRuns: 0, diff --git a/packages/backend-core/src/index.js b/packages/backend-core/src/index.js index cd3a3f5c97..b0bc524d9b 100644 --- a/packages/backend-core/src/index.js +++ b/packages/backend-core/src/index.js @@ -14,4 +14,5 @@ module.exports = { cache: require("../cache"), auth: require("../auth"), constants: require("../constants"), + migrations: require("../migrations"), } diff --git a/packages/backend-core/src/migrations/index.js b/packages/backend-core/src/migrations/index.js index 7492e94511..6b8eb3a95c 100644 --- a/packages/backend-core/src/migrations/index.js +++ b/packages/backend-core/src/migrations/index.js @@ -1,5 +1,5 @@ const { DocumentTypes } = require("../db/constants") -const { getGlobalDB } = require("../tenancy") +const { getGlobalDB, getTenantId } = require("../tenancy") exports.MIGRATION_DBS = { GLOBAL_DB: "GLOBAL_DB", @@ -7,11 +7,13 @@ exports.MIGRATION_DBS = { exports.MIGRATIONS = { USER_EMAIL_VIEW_CASING: "user_email_view_casing", + QUOTAS_1: "quotas_1", } const DB_LOOKUP = { [exports.MIGRATION_DBS.GLOBAL_DB]: [ exports.MIGRATIONS.USER_EMAIL_VIEW_CASING, + exports.MIGRATIONS.QUOTAS_1, ], } @@ -27,6 +29,7 @@ exports.getMigrationsDoc = async db => { } exports.migrateIfRequired = async (migrationDb, migrationName, migrateFn) => { + const tenantId = getTenantId() try { let db if (migrationDb === exports.MIGRATION_DBS.GLOBAL_DB) { @@ -47,15 +50,18 @@ exports.migrateIfRequired = async (migrationDb, migrationName, migrateFn) => { return } - console.log(`Performing migration: ${migrationName}`) + console.log(`[Tenant: ${tenantId}] Performing migration: ${migrationName}`) await migrateFn() - console.log(`Migration complete: ${migrationName}`) + console.log(`[Tenant: ${tenantId}] Migration complete: ${migrationName}`) // mark as complete doc[migrationName] = Date.now() await db.put(doc) } catch (err) { - console.error(`Error performing migration: ${migrationName}: `, err) + console.error( + `[Tenant: ${tenantId}] Error performing migration: ${migrationName}: `, + err + ) throw err } } diff --git a/packages/bbui/package.json b/packages/bbui/package.json index c1d3f4957f..ecc5bf5a1f 100644 --- a/packages/bbui/package.json +++ b/packages/bbui/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/bbui", "description": "A UI solution used in the different Budibase projects.", - "version": "1.0.44-alpha.9", + "version": "1.0.46-alpha.3", "license": "MPL-2.0", "svelte": "src/index.js", "module": "dist/bbui.es.js", diff --git a/packages/builder/package.json b/packages/builder/package.json index 960a320c53..474213899e 100644 --- a/packages/builder/package.json +++ b/packages/builder/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/builder", - "version": "1.0.44-alpha.9", + "version": "1.0.46-alpha.3", "license": "GPL-3.0", "private": true, "scripts": { @@ -65,10 +65,10 @@ } }, "dependencies": { - "@budibase/bbui": "^1.0.44-alpha.9", - "@budibase/client": "^1.0.44-alpha.9", + "@budibase/bbui": "^1.0.46-alpha.3", + "@budibase/client": "^1.0.46-alpha.3", "@budibase/colorpicker": "1.1.2", - "@budibase/string-templates": "^1.0.44-alpha.9", + "@budibase/string-templates": "^1.0.46-alpha.3", "@sentry/browser": "5.19.1", "@spectrum-css/page": "^3.0.1", "@spectrum-css/vars": "^3.0.1", diff --git a/packages/builder/src/builderStore/api.js b/packages/builder/src/builderStore/api.js index 897d3a74db..a5c6ceba54 100644 --- a/packages/builder/src/builderStore/api.js +++ b/packages/builder/src/builderStore/api.js @@ -6,6 +6,7 @@ const apiCall = method => async (url, body, headers = { "Content-Type": "application/json" }) => { headers["x-budibase-app-id"] = svelteGet(store).appId + headers["x-budibase-api-version"] = "1" const json = headers["Content-Type"] === "application/json" const resp = await fetch(url, { method: method, diff --git a/packages/builder/src/components/backend/TableNavigator/modals/CreateTableModal.svelte b/packages/builder/src/components/backend/TableNavigator/modals/CreateTableModal.svelte index 997864e165..a3b7ca81a6 100644 --- a/packages/builder/src/components/backend/TableNavigator/modals/CreateTableModal.svelte +++ b/packages/builder/src/components/backend/TableNavigator/modals/CreateTableModal.svelte @@ -53,16 +53,23 @@ } // Create table - const table = await tables.save(newTable) - notifications.success(`Table ${name} created successfully.`) - analytics.captureEvent(Events.TABLE.CREATED, { name }) + let table + try { + table = await tables.save(newTable) + notifications.success(`Table ${name} created successfully.`) + analytics.captureEvent(Events.TABLE.CREATED, { name }) - // Navigate to new table - const currentUrl = $url() - const path = currentUrl.endsWith("data") - ? `./table/${table._id}` - : `../../table/${table._id}` - $goto(path) + // Navigate to new table + const currentUrl = $url() + const path = currentUrl.endsWith("data") + ? `./table/${table._id}` + : `../../table/${table._id}` + $goto(path) + } catch (e) { + notifications.error(e) + // reload in case the table was created + await tables.fetch() + } } diff --git a/packages/builder/src/components/design/PropertiesPanel/PropertyControls/ButtonActionEditor/actions/ExecuteQuery.svelte b/packages/builder/src/components/design/PropertiesPanel/PropertyControls/ButtonActionEditor/actions/ExecuteQuery.svelte index 88c7e87054..8c438e4b22 100644 --- a/packages/builder/src/components/design/PropertiesPanel/PropertyControls/ButtonActionEditor/actions/ExecuteQuery.svelte +++ b/packages/builder/src/components/design/PropertiesPanel/PropertyControls/ButtonActionEditor/actions/ExecuteQuery.svelte @@ -55,8 +55,8 @@
0} {/if} @@ -49,7 +48,7 @@ {/if}
- {#each bindings as binding, idx} + {#each queryBindings as binding, idx} onBindingChange(binding.name, evt.detail)} value={runtimeToReadableBinding( - bindableOptions, + bindings, customParams?.[binding.name] )} - {bindableOptions} + {bindings} /> {:else} deleteQueryBinding(idx)} /> diff --git a/packages/builder/src/components/integration/QueryViewer.svelte b/packages/builder/src/components/integration/QueryViewer.svelte index f14d1d2b88..c6ae7c4ce8 100644 --- a/packages/builder/src/components/integration/QueryViewer.svelte +++ b/packages/builder/src/components/integration/QueryViewer.svelte @@ -120,7 +120,7 @@ config={integrationInfo.extra} /> {/if} - + {/if}
{#if shouldShowQueryConfig} diff --git a/packages/cli/package.json b/packages/cli/package.json index 9a1f80e3a7..e1df87bfd3 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/cli", - "version": "1.0.44-alpha.9", + "version": "1.0.46-alpha.3", "description": "Budibase CLI, for developers, self hosting and migrations.", "main": "src/index.js", "bin": { diff --git a/packages/client/package.json b/packages/client/package.json index e08f78dff8..5730b4a5e0 100644 --- a/packages/client/package.json +++ b/packages/client/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/client", - "version": "1.0.44-alpha.9", + "version": "1.0.46-alpha.3", "license": "MPL-2.0", "module": "dist/budibase-client.js", "main": "dist/budibase-client.js", @@ -19,9 +19,9 @@ "dev:builder": "rollup -cw" }, "dependencies": { - "@budibase/bbui": "^1.0.44-alpha.9", + "@budibase/bbui": "^1.0.46-alpha.3", "@budibase/standard-components": "^0.9.139", - "@budibase/string-templates": "^1.0.44-alpha.9", + "@budibase/string-templates": "^1.0.46-alpha.3", "regexparam": "^1.3.0", "shortid": "^2.2.15", "svelte-spa-router": "^3.0.5" diff --git a/packages/client/src/utils/fetch/DataFetch.js b/packages/client/src/utils/fetch/DataFetch.js index 884e12feb1..7fe53bb8af 100644 --- a/packages/client/src/utils/fetch/DataFetch.js +++ b/packages/client/src/utils/fetch/DataFetch.js @@ -110,12 +110,6 @@ export default class DataFetch { */ async getInitialData() { const { datasource, filter, sortColumn, paginate } = this.options - const tableId = datasource?.tableId - - // Ensure table ID exists - if (!tableId) { - return - } // Fetch datasource definition and determine feature flags const definition = await this.constructor.getDefinition(datasource) diff --git a/packages/server/package.json b/packages/server/package.json index a444c5d092..8dd3a339ae 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/server", "email": "hi@budibase.com", - "version": "1.0.44-alpha.9", + "version": "1.0.46-alpha.3", "description": "Budibase Web Server", "main": "src/index.ts", "repository": { @@ -70,9 +70,9 @@ "license": "GPL-3.0", "dependencies": { "@apidevtools/swagger-parser": "^10.0.3", - "@budibase/backend-core": "^1.0.44-alpha.9", - "@budibase/client": "^1.0.44-alpha.9", - "@budibase/string-templates": "^1.0.44-alpha.9", + "@budibase/backend-core": "^1.0.46-alpha.3", + "@budibase/client": "^1.0.46-alpha.3", + "@budibase/string-templates": "^1.0.46-alpha.3", "@bull-board/api": "^3.7.0", "@bull-board/koa": "^3.7.0", "@elastic/elasticsearch": "7.10.0", diff --git a/packages/server/src/api/controllers/table/internal.js b/packages/server/src/api/controllers/table/internal.js index 10a5c9746a..9f09e78219 100644 --- a/packages/server/src/api/controllers/table/internal.js +++ b/packages/server/src/api/controllers/table/internal.js @@ -8,6 +8,7 @@ const { getTable, handleDataImport, } = require("./utils") +const usageQuota = require("../../../utilities/usageQuota") exports.save = async function (ctx) { const appId = ctx.appId @@ -119,6 +120,7 @@ exports.destroy = async function (ctx) { }) ) await db.bulkDocs(rows.rows.map(row => ({ ...row.doc, _deleted: true }))) + await usageQuota.update(usageQuota.Properties.ROW, -rows.rows.length) // update linked rows await linkRows.updateLinks({ diff --git a/packages/server/src/api/controllers/table/utils.js b/packages/server/src/api/controllers/table/utils.js index 459566ce91..86e2837e15 100644 --- a/packages/server/src/api/controllers/table/utils.js +++ b/packages/server/src/api/controllers/table/utils.js @@ -16,6 +16,7 @@ const { } = require("../../../integrations/utils") const { getViews, saveView } = require("../view/utils") const viewTemplate = require("../view/viewBuilder") +const usageQuota = require("../../../utilities/usageQuota") exports.checkForColumnUpdates = async (db, oldTable, updatedTable) => { let updatedRows = [] @@ -112,7 +113,11 @@ exports.handleDataImport = async (appId, user, table, dataImport) => { finalData.push(row) } + await usageQuota.update(usageQuota.Properties.ROW, finalData.length, { + dryRun: true, + }) await db.bulkDocs(finalData) + await usageQuota.update(usageQuota.Properties.ROW, finalData.length) let response = await db.put(table) table._rev = response._rev return table diff --git a/packages/server/src/automations/steps/createRow.js b/packages/server/src/automations/steps/createRow.js index 8e5b44cc06..1937121062 100644 --- a/packages/server/src/automations/steps/createRow.js +++ b/packages/server/src/automations/steps/createRow.js @@ -1,6 +1,5 @@ const rowController = require("../../api/controllers/row") const automationUtils = require("../automationUtils") -const env = require("../../environment") const usage = require("../../utilities/usageQuota") const { buildCtx } = require("./utils") @@ -83,10 +82,9 @@ exports.run = async function ({ inputs, appId, emitter }) { inputs.row.tableId, inputs.row ) - if (env.USE_QUOTAS) { - await usage.update(usage.Properties.ROW, 1) - } + await usage.update(usage.Properties.ROW, 1, { dryRun: true }) await rowController.save(ctx) + await usage.update(usage.Properties.ROW, 1) return { row: inputs.row, response: ctx.body, diff --git a/packages/server/src/automations/steps/deleteRow.js b/packages/server/src/automations/steps/deleteRow.js index c7bee577a5..e41e5ad263 100644 --- a/packages/server/src/automations/steps/deleteRow.js +++ b/packages/server/src/automations/steps/deleteRow.js @@ -1,5 +1,4 @@ const rowController = require("../../api/controllers/row") -const env = require("../../environment") const usage = require("../../utilities/usageQuota") const { buildCtx } = require("./utils") const automationUtils = require("../automationUtils") @@ -74,9 +73,7 @@ exports.run = async function ({ inputs, appId, emitter }) { }) try { - if (env.isProd()) { - await usage.update(usage.Properties.ROW, -1) - } + await usage.update(usage.Properties.ROW, -1) await rowController.destroy(ctx) return { response: ctx.body, diff --git a/packages/server/src/environment.js b/packages/server/src/environment.js index a92e113851..614f41a29f 100644 --- a/packages/server/src/environment.js +++ b/packages/server/src/environment.js @@ -38,6 +38,7 @@ module.exports = { MINIO_ACCESS_KEY: process.env.MINIO_ACCESS_KEY, MINIO_SECRET_KEY: process.env.MINIO_SECRET_KEY, USE_QUOTAS: process.env.USE_QUOTAS, + EXCLUDE_QUOTAS_TENANTS: process.env.EXCLUDE_QUOTAS_TENANTS, REDIS_URL: process.env.REDIS_URL, REDIS_PASSWORD: process.env.REDIS_PASSWORD, INTERNAL_API_KEY: process.env.INTERNAL_API_KEY, diff --git a/packages/server/src/middleware/tests/usageQuota.spec.js b/packages/server/src/middleware/tests/usageQuota.spec.js index ae47d19207..1282615a50 100644 --- a/packages/server/src/middleware/tests/usageQuota.spec.js +++ b/packages/server/src/middleware/tests/usageQuota.spec.js @@ -1,11 +1,5 @@ jest.mock("../../db") jest.mock("../../utilities/usageQuota") -jest.mock("../../environment", () => ({ - isTest: () => true, - isProd: () => false, - isDev: () => true, - _set: () => {}, -})) jest.mock("@budibase/backend-core/tenancy", () => ({ getTenantId: () => "testing123" })) @@ -29,9 +23,10 @@ class TestConfiguration { }, req: { method: "POST", - url: "/rows" + url: "/applications" } } + usageQuota.useQuotas = () => true } executeMiddleware() { @@ -113,7 +108,6 @@ describe("usageQuota middleware", () => { it("calculates and persists the correct usage quota for the relevant action", async () => { config.setUrl("/rows") - config.setProd(true) await config.executeMiddleware() @@ -121,20 +115,20 @@ describe("usageQuota middleware", () => { expect(config.next).toHaveBeenCalled() }) - it("calculates the correct file size from a file upload call and adds it to quota", async () => { - config.setUrl("/upload") - config.setProd(true) - config.setFiles([ - { - size: 100 - }, - { - size: 10000 - }, - ]) - await config.executeMiddleware() + // it("calculates the correct file size from a file upload call and adds it to quota", async () => { + // config.setUrl("/upload") + // config.setProd(true) + // config.setFiles([ + // { + // size: 100 + // }, + // { + // size: 10000 + // }, + // ]) + // await config.executeMiddleware() - expect(usageQuota.update).toHaveBeenCalledWith("storage", 10100) - expect(config.next).toHaveBeenCalled() - }) + // expect(usageQuota.update).toHaveBeenCalledWith("storage", 10100) + // expect(config.next).toHaveBeenCalled() + // }) }) \ No newline at end of file diff --git a/packages/server/src/middleware/usageQuota.js b/packages/server/src/middleware/usageQuota.js index bb5fa4de3f..4bafa75132 100644 --- a/packages/server/src/middleware/usageQuota.js +++ b/packages/server/src/middleware/usageQuota.js @@ -1,14 +1,11 @@ const CouchDB = require("../db") const usageQuota = require("../utilities/usageQuota") -const env = require("../environment") -const { getTenantId } = require("@budibase/backend-core/tenancy") +const { getUniqueRows } = require("../utilities/usageQuota/rows") const { isExternalTable, isRowId: isExternalRowId, } = require("../integrations/utils") - -// tenants without limits -const EXCLUDED_TENANTS = ["bb", "default", "bbtest", "bbstaging"] +const migration = require("../migrations/usageQuotas") // currently only counting new writes and deletes const METHOD_MAP = { @@ -18,13 +15,13 @@ const METHOD_MAP = { const DOMAIN_MAP = { rows: usageQuota.Properties.ROW, - upload: usageQuota.Properties.UPLOAD, - views: usageQuota.Properties.VIEW, - users: usageQuota.Properties.USER, + // upload: usageQuota.Properties.UPLOAD, // doesn't work yet + // views: usageQuota.Properties.VIEW, // doesn't work yet + // users: usageQuota.Properties.USER, // doesn't work yet applications: usageQuota.Properties.APPS, // this will not be updated by endpoint calls // instead it will be updated by triggerInfo - automationRuns: usageQuota.Properties.AUTOMATION, + // automationRuns: usageQuota.Properties.AUTOMATION, // doesn't work yet } function getProperty(url) { @@ -36,10 +33,7 @@ function getProperty(url) { } module.exports = async (ctx, next) => { - const tenantId = getTenantId() - - // if in development or a self hosted cloud usage quotas should not be executed - if (env.isDev() || env.SELF_HOSTED || EXCLUDED_TENANTS.includes(tenantId)) { + if (!usageQuota.useQuotas()) { return next() } @@ -80,9 +74,93 @@ module.exports = async (ctx, next) => { usage = files.map(file => file.size).reduce((total, size) => total + size) } try { - await usageQuota.update(property, usage) - return next() + await migration.run() + await performRequest(ctx, next, property, usage) } catch (err) { ctx.throw(400, err) } } + +const performRequest = async (ctx, next, property, usage) => { + const usageContext = { + skipNext: false, + skipUsage: false, + [usageQuota.Properties.APPS]: {}, + } + + if (usage === -1) { + if (PRE_DELETE[property]) { + await PRE_DELETE[property](ctx, usageContext) + } + } else { + if (PRE_CREATE[property]) { + await PRE_CREATE[property](ctx, usageContext) + } + } + + // run the request + if (!usageContext.skipNext) { + await usageQuota.update(property, usage, { dryRun: true }) + await next() + } + + if (usage === -1) { + if (POST_DELETE[property]) { + await POST_DELETE[property](ctx, usageContext) + } + } else { + if (POST_CREATE[property]) { + await POST_CREATE[property](ctx, usageContext) + } + } + + // update the usage + if (!usageContext.skipUsage) { + await usageQuota.update(property, usage) + } +} + +const appPreDelete = async (ctx, usageContext) => { + if (ctx.query.unpublish) { + // don't run usage decrement for unpublish + usageContext.skipUsage = true + return + } + + // store the row count to delete + const rows = await getUniqueRows([ctx.appId]) + if (rows.length) { + usageContext[usageQuota.Properties.APPS] = { rowCount: rows.length } + } +} + +const appPostDelete = async (ctx, usageContext) => { + // delete the app rows from usage + const rowCount = usageContext[usageQuota.Properties.APPS].rowCount + if (rowCount) { + await usageQuota.update(usageQuota.Properties.ROW, -rowCount) + } +} + +const appPostCreate = async ctx => { + // app import & template creation + if (ctx.request.body.useTemplate === "true") { + const rows = await getUniqueRows([ctx.response.body.appId]) + const rowCount = rows ? rows.length : 0 + await usageQuota.update(usageQuota.Properties.ROW, rowCount) + } +} + +const PRE_DELETE = { + [usageQuota.Properties.APPS]: appPreDelete, +} + +const POST_DELETE = { + [usageQuota.Properties.APPS]: appPostDelete, +} + +const PRE_CREATE = {} + +const POST_CREATE = { + [usageQuota.Properties.APPS]: appPostCreate, +} diff --git a/packages/server/src/migrations/tests/usageQuotas/index.spec.js b/packages/server/src/migrations/tests/usageQuotas/index.spec.js new file mode 100644 index 0000000000..0c5b982909 --- /dev/null +++ b/packages/server/src/migrations/tests/usageQuotas/index.spec.js @@ -0,0 +1,27 @@ +const env = require("../../../environment") +const TestConfig = require("../../../tests/utilities/TestConfiguration") + +const syncApps = jest.fn() +const syncRows = jest.fn() + +jest.mock("../../usageQuotas/syncApps", () => ({ run: syncApps }) ) +jest.mock("../../usageQuotas/syncRows", () => ({ run: syncRows }) ) + +const migrations = require("../../usageQuotas") + +describe("run", () => { + let config = new TestConfig(false) + + beforeEach(async () => { + await config.init() + env._set("USE_QUOTAS", 1) + }) + + afterAll(config.end) + + it("runs the required migrations", async () => { + await migrations.run() + expect(syncApps).toHaveBeenCalledTimes(1) + expect(syncRows).toHaveBeenCalledTimes(1) + }) +}) diff --git a/packages/server/src/migrations/tests/usageQuotas/syncApps.spec.js b/packages/server/src/migrations/tests/usageQuotas/syncApps.spec.js new file mode 100644 index 0000000000..160319a31b --- /dev/null +++ b/packages/server/src/migrations/tests/usageQuotas/syncApps.spec.js @@ -0,0 +1,37 @@ +const { getGlobalDB } = require("@budibase/backend-core/tenancy") +const TestConfig = require("../../../tests/utilities/TestConfiguration") +const { getUsageQuotaDoc, update, Properties } = require("../../../utilities/usageQuota") +const syncApps = require("../../usageQuotas/syncApps") +const env = require("../../../environment") + +describe("syncApps", () => { + let config = new TestConfig(false) + + beforeEach(async () => { + await config.init() + env._set("USE_QUOTAS", 1) + }) + + afterAll(config.end) + + it("runs successfully", async () => { + // create the usage quota doc and mock usages + const db = getGlobalDB() + await getUsageQuotaDoc(db) + await update(Properties.APPS, 3) + + let usageDoc = await getUsageQuotaDoc(db) + expect(usageDoc.usageQuota.apps).toEqual(3) + + // create an extra app to test the migration + await config.createApp("quota-test") + + // migrate + await syncApps.run() + + // assert the migration worked + usageDoc = await getUsageQuotaDoc(db) + expect(usageDoc.usageQuota.apps).toEqual(2) + }) +}) + diff --git a/packages/server/src/migrations/tests/usageQuotas/syncRows.spec.js b/packages/server/src/migrations/tests/usageQuotas/syncRows.spec.js new file mode 100644 index 0000000000..a09bea60bd --- /dev/null +++ b/packages/server/src/migrations/tests/usageQuotas/syncRows.spec.js @@ -0,0 +1,43 @@ +const { getGlobalDB } = require("@budibase/backend-core/tenancy") +const TestConfig = require("../../../tests/utilities/TestConfiguration") +const { getUsageQuotaDoc, update, Properties } = require("../../../utilities/usageQuota") +const syncRows = require("../../usageQuotas/syncRows") +const env = require("../../../environment") + +describe("syncRows", () => { + let config = new TestConfig(false) + + beforeEach(async () => { + await config.init() + env._set("USE_QUOTAS", 1) + }) + + afterAll(config.end) + + it("runs successfully", async () => { + // create the usage quota doc and mock usages + const db = getGlobalDB() + await getUsageQuotaDoc(db) + await update(Properties.ROW, 300) + + let usageDoc = await getUsageQuotaDoc(db) + expect(usageDoc.usageQuota.rows).toEqual(300) + + // app 1 + await config.createTable() + await config.createRow() + // app 2 + await config.createApp() + await config.createTable() + await config.createRow() + await config.createRow() + + // migrate + await syncRows.run() + + // assert the migration worked + usageDoc = await getUsageQuotaDoc(db) + expect(usageDoc.usageQuota.rows).toEqual(3) + }) +}) + diff --git a/packages/server/src/migrations/usageQuotas/index.js b/packages/server/src/migrations/usageQuotas/index.js new file mode 100644 index 0000000000..39744093c2 --- /dev/null +++ b/packages/server/src/migrations/usageQuotas/index.js @@ -0,0 +1,24 @@ +const { + MIGRATIONS, + MIGRATION_DBS, + migrateIfRequired, +} = require("@budibase/backend-core/migrations") +const { useQuotas } = require("../../utilities/usageQuota") +const syncApps = require("./syncApps") +const syncRows = require("./syncRows") + +exports.run = async () => { + if (!useQuotas()) { + return + } + + // Jan 2022 + await migrateIfRequired( + MIGRATION_DBS.GLOBAL_DB, + MIGRATIONS.QUOTAS_1, + async () => { + await syncApps.run() + await syncRows.run() + } + ) +} diff --git a/packages/server/src/migrations/usageQuotas/syncApps.js b/packages/server/src/migrations/usageQuotas/syncApps.js new file mode 100644 index 0000000000..ee106129e6 --- /dev/null +++ b/packages/server/src/migrations/usageQuotas/syncApps.js @@ -0,0 +1,18 @@ +const { getGlobalDB, getTenantId } = require("@budibase/backend-core/tenancy") +const { getAllApps } = require("@budibase/backend-core/db") +const CouchDB = require("../../db") +const { getUsageQuotaDoc } = require("../../utilities/usageQuota") + +exports.run = async () => { + const db = getGlobalDB() + // get app count + const devApps = await getAllApps(CouchDB, { dev: true }) + const appCount = devApps ? devApps.length : 0 + + // sync app count + const tenantId = getTenantId() + console.log(`[Tenant: ${tenantId}] Syncing app count: ${appCount}`) + const usageDoc = await getUsageQuotaDoc(db) + usageDoc.usageQuota.apps = appCount + await db.put(usageDoc) +} diff --git a/packages/server/src/migrations/usageQuotas/syncRows.js b/packages/server/src/migrations/usageQuotas/syncRows.js new file mode 100644 index 0000000000..7990f405de --- /dev/null +++ b/packages/server/src/migrations/usageQuotas/syncRows.js @@ -0,0 +1,21 @@ +const { getGlobalDB, getTenantId } = require("@budibase/backend-core/tenancy") +const { getAllApps } = require("@budibase/backend-core/db") +const CouchDB = require("../../db") +const { getUsageQuotaDoc } = require("../../utilities/usageQuota") +const { getUniqueRows } = require("../../utilities/usageQuota/rows") + +exports.run = async () => { + const db = getGlobalDB() + // get all rows in all apps + const allApps = await getAllApps(CouchDB, { all: true }) + const appIds = allApps ? allApps.map(app => app.appId) : [] + const rows = await getUniqueRows(appIds) + const rowCount = rows ? rows.length : 0 + + // sync row count + const tenantId = getTenantId() + console.log(`[Tenant: ${tenantId}] Syncing row count: ${rowCount}`) + const usageDoc = await getUsageQuotaDoc(db) + usageDoc.usageQuota.rows = rowCount + await db.put(usageDoc) +} diff --git a/packages/server/src/utilities/tests/usageQuota/usageQuota.spec.js b/packages/server/src/utilities/tests/usageQuota/usageQuota.spec.js new file mode 100644 index 0000000000..dcd7578f59 --- /dev/null +++ b/packages/server/src/utilities/tests/usageQuota/usageQuota.spec.js @@ -0,0 +1,72 @@ +const getTenantId = jest.fn() +jest.mock("@budibase/backend-core/tenancy", () => ({ + getTenantId +})) +const usageQuota = require("../../usageQuota") +const env = require("../../../environment") + +class TestConfiguration { + constructor() { + this.enableQuotas() + } + + enableQuotas = () => { + env.USE_QUOTAS = 1 + } + + disableQuotas = () => { + env.USE_QUOTAS = null + } + + setTenantId = (tenantId) => { + getTenantId.mockReturnValue(tenantId) + } + + setExcludedTenants = (tenants) => { + env.EXCLUDE_QUOTAS_TENANTS = tenants + } + + reset = () => { + this.disableQuotas() + this.setExcludedTenants(null) + } +} + +describe("usageQuota", () => { + let config + + beforeEach(() => { + config = new TestConfiguration() + }) + + afterEach(() => { + config.reset() + }) + + describe("useQuotas", () => { + it("works when no settings have been provided", () => { + config.reset() + expect(usageQuota.useQuotas()).toBe(false) + }) + it("honours USE_QUOTAS setting", () => { + config.disableQuotas() + expect(usageQuota.useQuotas()).toBe(false) + + config.enableQuotas() + expect(usageQuota.useQuotas()).toBe(true) + }) + it("honours EXCLUDE_QUOTAS_TENANTS setting", () => { + config.setTenantId("test") + + // tenantId is in the list + config.setExcludedTenants("test, test2, test2") + expect(usageQuota.useQuotas()).toBe(false) + config.setExcludedTenants("test,test2,test2") + expect(usageQuota.useQuotas()).toBe(false) + + // tenantId is not in the list + config.setTenantId("other") + expect(usageQuota.useQuotas()).toBe(true) + }) + }) +}) \ No newline at end of file diff --git a/packages/server/src/utilities/usageQuota.js b/packages/server/src/utilities/usageQuota.js deleted file mode 100644 index 73fe83a170..0000000000 --- a/packages/server/src/utilities/usageQuota.js +++ /dev/null @@ -1,73 +0,0 @@ -const env = require("../environment") -const { getGlobalDB } = require("@budibase/backend-core/tenancy") -const { - StaticDatabases, - generateNewUsageQuotaDoc, -} = require("@budibase/backend-core/db") - -function getNewQuotaReset() { - return Date.now() + 2592000000 -} - -exports.Properties = { - ROW: "rows", - UPLOAD: "storage", - VIEW: "views", - USER: "users", - AUTOMATION: "automationRuns", - APPS: "apps", - EMAILS: "emails", -} - -async function getUsageQuotaDoc(db) { - let quota - try { - quota = await db.get(StaticDatabases.PLATFORM_INFO.docs.usageQuota) - } catch (err) { - // doc doesn't exist. Create it - quota = await db.post(generateNewUsageQuotaDoc()) - } - - return quota -} - -/** - * Given a specified tenantId this will add to the usage object for the specified property. - * @param {string} property The property which is to be added to (within the nested usageQuota object). - * @param {number} usage The amount (this can be negative) to adjust the number by. - * @returns {Promise} When this completes the API key will now be up to date - the quota period may have - * also been reset after this call. - */ -exports.update = async (property, usage) => { - if (!env.USE_QUOTAS) { - return - } - - try { - const db = getGlobalDB() - const quota = await getUsageQuotaDoc(db) - - // Check if the quota needs reset - if (Date.now() >= quota.quotaReset) { - quota.quotaReset = getNewQuotaReset() - for (let prop of Object.keys(quota.usageQuota)) { - quota.usageQuota[prop] = 0 - } - } - - // increment the quota - quota.usageQuota[property] += usage - - if (quota.usageQuota[property] > quota.usageLimits[property]) { - throw new Error( - `You have exceeded your usage quota of ${quota.usageLimits[property]} ${property}.` - ) - } - - // update the usage quotas - await db.put(quota) - } catch (err) { - console.error(`Error updating usage quotas for ${property}`, err) - throw err - } -} diff --git a/packages/server/src/utilities/usageQuota/index.js b/packages/server/src/utilities/usageQuota/index.js new file mode 100644 index 0000000000..b0ff310aa3 --- /dev/null +++ b/packages/server/src/utilities/usageQuota/index.js @@ -0,0 +1,92 @@ +const env = require("../../environment") +const { getGlobalDB, getTenantId } = require("@budibase/backend-core/tenancy") +const { + StaticDatabases, + generateNewUsageQuotaDoc, +} = require("@budibase/backend-core/db") + +exports.useQuotas = () => { + // check if quotas are enabled + if (env.USE_QUOTAS) { + // check if there are any tenants without limits + if (env.EXCLUDE_QUOTAS_TENANTS) { + const excludedTenants = env.EXCLUDE_QUOTAS_TENANTS.replace( + /\s/g, + "" + ).split(",") + const tenantId = getTenantId() + if (excludedTenants.includes(tenantId)) { + return false + } + } + return true + } + return false +} + +exports.Properties = { + ROW: "rows", + UPLOAD: "storage", // doesn't work yet + VIEW: "views", // doesn't work yet + USER: "users", // doesn't work yet + AUTOMATION: "automationRuns", // doesn't work yet + APPS: "apps", + EMAILS: "emails", // doesn't work yet +} + +exports.getUsageQuotaDoc = async db => { + let quota + try { + quota = await db.get(StaticDatabases.GLOBAL.docs.usageQuota) + } catch (err) { + // doc doesn't exist. Create it + quota = generateNewUsageQuotaDoc() + const response = await db.put(quota) + quota._rev = response.rev + } + + return quota +} + +/** + * Given a specified tenantId this will add to the usage object for the specified property. + * @param {string} property The property which is to be added to (within the nested usageQuota object). + * @param {number} usage The amount (this can be negative) to adjust the number by. + * @returns {Promise} When this completes the API key will now be up to date - the quota period may have + * also been reset after this call. + */ +exports.update = async (property, usage, opts = { dryRun: false }) => { + if (!exports.useQuotas()) { + return + } + + try { + const db = getGlobalDB() + const quota = await exports.getUsageQuotaDoc(db) + + // increment the quota + quota.usageQuota[property] += usage + + if ( + quota.usageQuota[property] > quota.usageLimits[property] && + usage > 0 // allow for decrementing usage when the quota is already exceeded + ) { + throw new Error( + `You have exceeded your usage quota of ${quota.usageLimits[property]} ${property}.` + ) + } + + if (quota.usageQuota[property] < 0) { + // never go negative if the quota has previously been exceeded + quota.usageQuota[property] = 0 + } + + // update the usage quotas + if (!opts.dryRun) { + await db.put(quota) + } + } catch (err) { + console.error(`Error updating usage quotas for ${property}`, err) + throw err + } +} diff --git a/packages/server/src/utilities/usageQuota/rows.js b/packages/server/src/utilities/usageQuota/rows.js new file mode 100644 index 0000000000..67ad07410d --- /dev/null +++ b/packages/server/src/utilities/usageQuota/rows.js @@ -0,0 +1,74 @@ +const { getRowParams, USER_METDATA_PREFIX } = require("../../db/utils") +const CouchDB = require("../../db") +const { isDevAppID, getDevelopmentAppID } = require("@budibase/backend-core/db") + +const ROW_EXCLUSIONS = [USER_METDATA_PREFIX] + +const getAppPairs = appIds => { + // collect the app ids into dev / prod pairs + // keyed by the dev app id + const pairs = {} + for (let appId of appIds) { + const devId = getDevelopmentAppID(appId) + if (!pairs[devId]) { + pairs[devId] = {} + } + if (isDevAppID(appId)) { + pairs[devId].devId = appId + } else { + pairs[devId].prodId = appId + } + } + return pairs +} + +const getAppRows = async appId => { + const appDb = new CouchDB(appId) + const response = await appDb.allDocs( + getRowParams(null, null, { + include_docs: false, + }) + ) + return response.rows + .map(r => r.id) + .filter(id => { + for (let exclusion of ROW_EXCLUSIONS) { + if (id.startsWith(exclusion)) { + return false + } + } + return true + }) +} + +/** + * Return a set of all rows in the given app ids. + * The returned rows will be unique on a per dev/prod app basis. + * Rows duplicates may exist across apps due to data import so they are not filtered out. + */ +exports.getUniqueRows = async appIds => { + let uniqueRows = [] + const pairs = getAppPairs(appIds) + + for (let pair of Object.values(pairs)) { + let appRows = [] + for (let appId of [pair.devId, pair.prodId]) { + if (!appId) { + continue + } + try { + appRows.push(await getAppRows(appId)) + } catch (e) { + console.error(e) + // don't error out if we can't count the app rows, just continue + } + } + + // ensure uniqueness on a per app pair basis + // this can't be done on all rows because app import results in + // duplicate row ids across apps + uniqueRows = uniqueRows.concat(...new Set(appRows)) + } + + return uniqueRows +} diff --git a/packages/server/src/utilities/usageQuota/usageQuoteReset.js b/packages/server/src/utilities/usageQuota/usageQuoteReset.js new file mode 100644 index 0000000000..ff5a1aa00e --- /dev/null +++ b/packages/server/src/utilities/usageQuota/usageQuoteReset.js @@ -0,0 +1,18 @@ +// UNUSED CODE +// Preserved for future use + +/* eslint-disable no-unused-vars */ + +function getNewQuotaReset() { + return Date.now() + 2592000000 +} + +function resetQuotasIfRequired(quota) { + // Check if the quota needs reset + if (Date.now() >= quota.quotaReset) { + quota.quotaReset = getNewQuotaReset() + for (let prop of Object.keys(quota.usageQuota)) { + quota.usageQuota[prop] = 0 + } + } +} diff --git a/packages/string-templates/package.json b/packages/string-templates/package.json index 278d60cabe..1a808c3f9b 100644 --- a/packages/string-templates/package.json +++ b/packages/string-templates/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/string-templates", - "version": "1.0.44-alpha.9", + "version": "1.0.46-alpha.3", "description": "Handlebars wrapper for Budibase templating.", "main": "src/index.cjs", "module": "dist/bundle.mjs", diff --git a/packages/string-templates/src/helpers/javascript.js b/packages/string-templates/src/helpers/javascript.js index 9231283e89..0173be0b54 100644 --- a/packages/string-templates/src/helpers/javascript.js +++ b/packages/string-templates/src/helpers/javascript.js @@ -1,5 +1,6 @@ const { atob } = require("../utilities") const { cloneDeep } = require("lodash/fp") +const { LITERAL_MARKER } = require("../helpers/constants") // The method of executing JS scripts depends on the bundle being built. // This setter is used in the entrypoint (either index.cjs or index.mjs). @@ -46,8 +47,9 @@ module.exports.processJS = (handlebars, context) => { $: path => getContextValue(path, cloneDeep(context)), } - // Create a sandbox with out context and run the JS - return runJS(js, sandboxContext) + // Create a sandbox with our context and run the JS + const res = { data: runJS(js, sandboxContext) } + return `{{${LITERAL_MARKER} js_result-${JSON.stringify(res)}}}` } catch (error) { return "Error while executing JS" } diff --git a/packages/string-templates/src/processors/postprocessor.js b/packages/string-templates/src/processors/postprocessor.js index 4d1c84013a..7fc3f663fe 100644 --- a/packages/string-templates/src/processors/postprocessor.js +++ b/packages/string-templates/src/processors/postprocessor.js @@ -36,6 +36,11 @@ module.exports.processors = [ return value === "true" case "object": return JSON.parse(value) + case "js_result": + // We use the literal helper to process the result of JS expressions + // as we want to be able to return any types. + // We wrap the value in an abject to be able to use undefined properly. + return JSON.parse(value).data } return value }), diff --git a/packages/string-templates/test/javascript.spec.js b/packages/string-templates/test/javascript.spec.js index 05cc80331a..5363f37e02 100644 --- a/packages/string-templates/test/javascript.spec.js +++ b/packages/string-templates/test/javascript.spec.js @@ -7,7 +7,7 @@ const processJS = (js, context) => { describe("Test the JavaScript helper", () => { it("should execute a simple expression", () => { const output = processJS(`return 1 + 2`) - expect(output).toBe("3") + expect(output).toBe(3) }) it("should be able to use primitive bindings", () => { @@ -50,6 +50,52 @@ describe("Test the JavaScript helper", () => { expect(output).toBe("shazbat") }) + it("should be able to return an object", () => { + const output = processJS(`return $("foo")`, { + foo: { + bar: { + baz: "shazbat", + }, + }, + }) + expect(output.bar.baz).toBe("shazbat") + }) + + it("should be able to return an array", () => { + const output = processJS(`return $("foo")`, { + foo: ["a", "b", "c"], + }) + expect(output[2]).toBe("c") + }) + + it("should be able to return null", () => { + const output = processJS(`return $("foo")`, { + foo: null, + }) + expect(output).toBe(null) + }) + + it("should be able to return undefined", () => { + const output = processJS(`return $("foo")`, { + foo: undefined, + }) + expect(output).toBe(undefined) + }) + + it("should be able to return 0", () => { + const output = processJS(`return $("foo")`, { + foo: 0, + }) + expect(output).toBe(0) + }) + + it("should be able to return an empty string", () => { + const output = processJS(`return $("foo")`, { + foo: "", + }) + expect(output).toBe("") + }) + it("should be able to use a deep array binding", () => { const output = processJS(`return $("foo.0.bar")`, { foo: [ diff --git a/packages/worker/package.json b/packages/worker/package.json index cbecd5d15d..93c33b648c 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/worker", "email": "hi@budibase.com", - "version": "1.0.44-alpha.9", + "version": "1.0.46-alpha.3", "description": "Budibase background service", "main": "src/index.js", "repository": { @@ -29,8 +29,8 @@ "author": "Budibase", "license": "GPL-3.0", "dependencies": { - "@budibase/backend-core": "^1.0.44-alpha.9", - "@budibase/string-templates": "^1.0.44-alpha.9", + "@budibase/backend-core": "^1.0.46-alpha.3", + "@budibase/string-templates": "^1.0.46-alpha.3", "@koa/router": "^8.0.0", "@sentry/node": "^6.0.0", "@techpass/passport-openidconnect": "^0.3.0", diff --git a/packages/worker/src/api/controllers/global/users.js b/packages/worker/src/api/controllers/global/users.js index 152bc9f5ae..676c597b84 100644 --- a/packages/worker/src/api/controllers/global/users.js +++ b/packages/worker/src/api/controllers/global/users.js @@ -73,16 +73,14 @@ exports.adminUser = async ctx => { if (!env.SELF_HOSTED) { // could be a scenario where it exists, make sure its clean try { - const usageQuota = await db.get( - StaticDatabases.PLATFORM_INFO.docs.usageQuota - ) + const usageQuota = await db.get(StaticDatabases.GLOBAL.docs.usageQuota) if (usageQuota) { await db.remove(usageQuota._id, usageQuota._rev) } } catch (err) { // don't worry about errors } - await db.post(generateNewUsageQuotaDoc()) + await db.put(generateNewUsageQuotaDoc()) } if (response.rows.some(row => row.doc.admin)) {