diff --git a/packages/backend-core/src/errors/index.js b/packages/backend-core/src/errors/index.js index 429aa68cad..4f3b4e0c41 100644 --- a/packages/backend-core/src/errors/index.js +++ b/packages/backend-core/src/errors/index.js @@ -21,7 +21,7 @@ const getPublicError = err => { type: err.type, } - if (err.code) { + if (err.code && context[err.code]) { error = { ...error, // get any additional context from this error diff --git a/packages/bbui/src/ProgressBar/ProgressBar.svelte b/packages/bbui/src/ProgressBar/ProgressBar.svelte index 16125b3e68..0bc50fb452 100644 --- a/packages/bbui/src/ProgressBar/ProgressBar.svelte +++ b/packages/bbui/src/ProgressBar/ProgressBar.svelte @@ -28,7 +28,7 @@ aria-valuenow={$progress} aria-valuemin="0" aria-valuemax="100" - style={width ? `width: ${width}px;` : ""} + style={width ? `width: ${width};` : ""} > {#if $$slots}
row.doc) + ).rows.map((row: any) => row.doc) } async function getScreens() { @@ -72,16 +75,16 @@ async function getScreens() { include_docs: true, }) ) - ).rows.map(row => row.doc) + ).rows.map((row: any) => row.doc) } -function getUserRoleId(ctx) { +function getUserRoleId(ctx: any) { return !ctx.user.role || !ctx.user.role._id ? BUILTIN_ROLE_IDS.PUBLIC : ctx.user.role._id } -exports.getAppUrl = ctx => { +export const getAppUrl = (ctx: any) => { // construct the url let url if (ctx.request.body.url) { @@ -97,29 +100,34 @@ exports.getAppUrl = ctx => { return url } -const checkAppUrl = (ctx, apps, url, currentAppId) => { +const checkAppUrl = (ctx: any, apps: any, url: any, currentAppId?: string) => { if (currentAppId) { - apps = apps.filter(app => app.appId !== currentAppId) + apps = apps.filter((app: any) => app.appId !== currentAppId) } - if (apps.some(app => app.url === url)) { + if (apps.some((app: any) => app.url === url)) { ctx.throw(400, "App URL is already in use.") } } -const checkAppName = (ctx, apps, name, currentAppId) => { +const checkAppName = ( + ctx: any, + apps: any, + name: any, + currentAppId?: string +) => { // TODO: Replace with Joi if (!name) { ctx.throw(400, "Name is required") } if (currentAppId) { - apps = apps.filter(app => app.appId !== currentAppId) + apps = apps.filter((app: any) => app.appId !== currentAppId) } - if (apps.some(app => app.name === name)) { + if (apps.some((app: any) => app.name === name)) { ctx.throw(400, "App name is already in use.") } } -async function createInstance(template) { +async function createInstance(template: any) { const tenantId = isMultiTenant() ? getTenantId() : null const baseAppId = generateAppID(tenantId) const appId = generateDevAppID(baseAppId) @@ -160,7 +168,7 @@ async function createInstance(template) { return { _id: appId } } -exports.fetch = async ctx => { +export const fetch = async (ctx: any) => { const dev = ctx.query && ctx.query.status === AppStatus.DEV const all = ctx.query && ctx.query.status === AppStatus.ALL const apps = await getAllApps({ dev, all }) @@ -172,7 +180,7 @@ exports.fetch = async ctx => { if (app.status !== "development") { continue } - const lock = locks.find(lock => lock.appId === app.appId) + const lock = locks.find((lock: any) => lock.appId === app.appId) if (lock) { app.lockedBy = lock.user } else { @@ -185,7 +193,7 @@ exports.fetch = async ctx => { ctx.body = apps } -exports.fetchAppDefinition = async ctx => { +export const fetchAppDefinition = async (ctx: any) => { const layouts = await getLayouts() const userRoleId = getUserRoleId(ctx) const accessController = new AccessController() @@ -200,7 +208,7 @@ exports.fetchAppDefinition = async ctx => { } } -exports.fetchAppPackage = async ctx => { +export const fetchAppPackage = async (ctx: any) => { const db = getAppDB() const application = await db.get(DocumentTypes.APP_METADATA) const layouts = await getLayouts() @@ -221,7 +229,7 @@ exports.fetchAppPackage = async ctx => { } } -exports.create = async ctx => { +const performAppCreate = async (ctx: any) => { const apps = await getAllApps({ dev: true }) const name = ctx.request.body.name checkAppName(ctx, apps, name) @@ -229,7 +237,7 @@ exports.create = async ctx => { checkAppUrl(ctx, apps, url) const { useTemplate, templateKey, templateString } = ctx.request.body - const instanceConfig = { + const instanceConfig: any = { useTemplate, key: templateKey, templateString, @@ -279,13 +287,41 @@ exports.create = async ctx => { } await appCache.invalidateAppMetadata(appId, newApplication) - ctx.status = 200 + return newApplication +} + +const appPostCreate = async (ctx: any, appId: string) => { + // app import & template creation + if (ctx.request.body.useTemplate === "true") { + const rows = await getUniqueRows([appId]) + const rowCount = rows ? rows.length : 0 + if (rowCount) { + try { + await quotas.addRows(rowCount) + } catch (err: any) { + if (err.code && err.code === errors.codes.USAGE_LIMIT_EXCEEDED) { + // this import resulted in row usage exceeding the quota + // delete the app + // skip pre and post steps as no rows have been added to quotas yet + ctx.params.appId = appId + await destroyApp(ctx) + } + throw err + } + } + } +} + +export const create = async (ctx: any) => { + const newApplication = await quotas.addApp(() => performAppCreate(ctx)) + await appPostCreate(ctx, newApplication.appId) ctx.body = newApplication + ctx.status = 200 } // This endpoint currently operates as a PATCH rather than a PUT // Thus name and url fields are handled only if present -exports.update = async ctx => { +export const update = async (ctx: any) => { const apps = await getAllApps({ dev: true }) // validation const name = ctx.request.body.name @@ -303,7 +339,7 @@ exports.update = async ctx => { ctx.body = data } -exports.updateClient = async ctx => { +export const updateClient = async (ctx: any) => { // Get current app version const db = getAppDB() const application = await db.get(DocumentTypes.APP_METADATA) @@ -325,7 +361,7 @@ exports.updateClient = async ctx => { ctx.body = data } -exports.revertClient = async ctx => { +export const revertClient = async (ctx: any) => { // Check app can be reverted const db = getAppDB() const application = await db.get(DocumentTypes.APP_METADATA) @@ -348,10 +384,15 @@ exports.revertClient = async ctx => { ctx.body = data } -exports.delete = async ctx => { +const destroyApp = async (ctx: any) => { const db = getAppDB() const result = await db.destroy() + if (ctx.query.unpublish) { + await quotas.removePublishedApp() + } else { + await quotas.removeApp() + } /* istanbul ignore next */ if (!env.isTest() && !ctx.query.unpublish) { await deleteApp(ctx.params.appId) @@ -362,12 +403,30 @@ exports.delete = async ctx => { // make sure the app/role doesn't stick around after the app has been deleted await removeAppFromUserRoles(ctx, ctx.params.appId) await appCache.invalidateAppMetadata(ctx.params.appId) + return result +} +const preDestroyApp = async (ctx: any) => { + const rows = await getUniqueRows([ctx.appId]) + ctx.rowCount = rows.length +} + +const postDestroyApp = async (ctx: any) => { + const rowCount = ctx.rowCount + if (rowCount) { + await quotas.removeRows(rowCount) + } +} + +export const destroy = async (ctx: any) => { + await preDestroyApp(ctx) + const result = await destroyApp(ctx) + await postDestroyApp(ctx) ctx.status = 200 ctx.body = result } -exports.sync = async (ctx, next) => { +export const sync = async (ctx: any, next: any) => { const appId = ctx.params.appId if (!isDevAppID(appId)) { ctx.throw(400, "This action cannot be performed for production apps") @@ -397,7 +456,7 @@ exports.sync = async (ctx, next) => { let error try { await replication.replicate({ - filter: function (doc) { + filter: function (doc: any) { return doc._id !== DocumentTypes.APP_METADATA }, }) @@ -417,7 +476,7 @@ exports.sync = async (ctx, next) => { } } -const updateAppPackage = async (appPackage, appId) => { +const updateAppPackage = async (appPackage: any, appId: any) => { const db = getAppDB() const application = await db.get(DocumentTypes.APP_METADATA) @@ -436,7 +495,7 @@ const updateAppPackage = async (appPackage, appId) => { return response } -const createEmptyAppPackage = async (ctx, app) => { +const createEmptyAppPackage = async (ctx: any, app: any) => { const db = getAppDB() let screensAndLayouts = [] diff --git a/packages/server/src/api/controllers/deploy/index.js b/packages/server/src/api/controllers/deploy/index.ts similarity index 73% rename from packages/server/src/api/controllers/deploy/index.js rename to packages/server/src/api/controllers/deploy/index.ts index 4186a192a4..663d4297fb 100644 --- a/packages/server/src/api/controllers/deploy/index.js +++ b/packages/server/src/api/controllers/deploy/index.ts @@ -1,20 +1,18 @@ -const Deployment = require("./Deployment") -const { +import Deployment from "./Deployment" +import { Replication, getProdAppID, getDevelopmentAppID, -} = require("@budibase/backend-core/db") -const { DocumentTypes, getAutomationParams } = require("../../../db/utils") -const { - disableAllCrons, - enableCronTrigger, -} = require("../../../automations/utils") -const { app: appCache } = require("@budibase/backend-core/cache") -const { +} from "@budibase/backend-core/db" +import { DocumentTypes, getAutomationParams } from "../../../db/utils" +import { disableAllCrons, enableCronTrigger } from "../../../automations/utils" +import { app as appCache } from "@budibase/backend-core/cache" +import { getAppId, getAppDB, getProdAppDB, -} = require("@budibase/backend-core/context") +} from "@budibase/backend-core/context" +import { quotas } from "@budibase/pro" // the max time we can wait for an invalidation to complete before considering it failed const MAX_PENDING_TIME_MS = 30 * 60000 @@ -25,9 +23,10 @@ const DeploymentStatus = { } // checks that deployments are in a good state, any pending will be updated -async function checkAllDeployments(deployments) { +async function checkAllDeployments(deployments: any) { let updated = false - for (let deployment of Object.values(deployments.history)) { + let deployment: any + for (deployment of Object.values(deployments.history)) { // check that no deployments have crashed etc and are now stuck if ( deployment.status === DeploymentStatus.PENDING && @@ -41,7 +40,7 @@ async function checkAllDeployments(deployments) { return { updated, deployments } } -async function storeDeploymentHistory(deployment) { +async function storeDeploymentHistory(deployment: any) { const deploymentJSON = deployment.getJSON() const db = getAppDB() @@ -70,7 +69,7 @@ async function storeDeploymentHistory(deployment) { return deployment } -async function initDeployedApp(prodAppId) { +async function initDeployedApp(prodAppId: any) { const db = getProdAppDB() console.log("Reading automation docs") const automations = ( @@ -79,7 +78,7 @@ async function initDeployedApp(prodAppId) { include_docs: true, }) ) - ).rows.map(row => row.doc) + ).rows.map((row: any) => row.doc) console.log("You have " + automations.length + " automations") const promises = [] console.log("Disabling prod crons..") @@ -93,16 +92,17 @@ async function initDeployedApp(prodAppId) { console.log("Enabled cron triggers for deployed app..") } -async function deployApp(deployment) { +async function deployApp(deployment: any) { try { const appId = getAppId() const devAppId = getDevelopmentAppID(appId) const productionAppId = getProdAppID(appId) - const replication = new Replication({ + const config: any = { source: devAppId, target: productionAppId, - }) + } + const replication = new Replication(config) console.log("Replication object created") @@ -119,7 +119,7 @@ async function deployApp(deployment) { console.log("Deployed app initialised, setting deployment to successful") deployment.setStatus(DeploymentStatus.SUCCESS) await storeDeploymentHistory(deployment) - } catch (err) { + } catch (err: any) { deployment.setStatus(DeploymentStatus.FAILURE, err.message) await storeDeploymentHistory(deployment) throw { @@ -129,14 +129,11 @@ async function deployApp(deployment) { } } -exports.fetchDeployments = async function (ctx) { +export async function fetchDeployments(ctx: any) { try { const db = getAppDB() const deploymentDoc = await db.get(DocumentTypes.DEPLOYMENTS) - const { updated, deployments } = await checkAllDeployments( - deploymentDoc, - ctx.user - ) + const { updated, deployments } = await checkAllDeployments(deploymentDoc) if (updated) { await db.put(deployments) } @@ -146,7 +143,7 @@ exports.fetchDeployments = async function (ctx) { } } -exports.deploymentProgress = async function (ctx) { +export async function deploymentProgress(ctx: any) { try { const db = getAppDB() const deploymentDoc = await db.get(DocumentTypes.DEPLOYMENTS) @@ -159,7 +156,20 @@ exports.deploymentProgress = async function (ctx) { } } -exports.deployApp = async function (ctx) { +const isFirstDeploy = async () => { + try { + const db = getProdAppDB() + await db.get(DocumentTypes.APP_METADATA) + } catch (e: any) { + if (e.status === 404) { + return true + } + throw e + } + return false +} + +const _deployApp = async function (ctx: any) { let deployment = new Deployment() console.log("Deployment object created") deployment.setStatus(DeploymentStatus.PENDING) @@ -168,7 +178,14 @@ exports.deployApp = async function (ctx) { console.log("Stored deployment history") console.log("Deploying app...") - await deployApp(deployment) + + if (await isFirstDeploy()) { + await quotas.addPublishedApp(() => deployApp(deployment)) + } else { + await deployApp(deployment) + } ctx.body = deployment } + +export { _deployApp as deployApp } diff --git a/packages/server/src/api/controllers/query/index.js b/packages/server/src/api/controllers/query/index.ts similarity index 66% rename from packages/server/src/api/controllers/query/index.js rename to packages/server/src/api/controllers/query/index.ts index 7a179bab35..3f9d0275b4 100644 --- a/packages/server/src/api/controllers/query/index.js +++ b/packages/server/src/api/controllers/query/index.ts @@ -1,22 +1,19 @@ -const { - generateQueryID, - getQueryParams, - isProdAppID, -} = require("../../../db/utils") -const { BaseQueryVerbs } = require("../../../constants") -const { Thread, ThreadType } = require("../../../threads") -const { save: saveDatasource } = require("../datasource") -const { RestImporter } = require("./import") -const { invalidateDynamicVariables } = require("../../../threads/utils") -const environment = require("../../../environment") -const { getAppDB } = require("@budibase/backend-core/context") +import { generateQueryID, getQueryParams, isProdAppID } from "../../../db/utils" +import { BaseQueryVerbs } from "../../../constants" +import { Thread, ThreadType } from "../../../threads" +import { save as saveDatasource } from "../datasource" +import { RestImporter } from "./import" +import { invalidateDynamicVariables } from "../../../threads/utils" +import { QUERY_THREAD_TIMEOUT } from "../../../environment" +import { getAppDB } from "@budibase/backend-core/context" +import { quotas } from "@budibase/pro" const Runner = new Thread(ThreadType.QUERY, { - timeoutMs: environment.QUERY_THREAD_TIMEOUT || 10000, + timeoutMs: QUERY_THREAD_TIMEOUT || 10000, }) // simple function to append "readable" to all read queries -function enrichQueries(input) { +function enrichQueries(input: any) { const wasArray = Array.isArray(input) const queries = wasArray ? input : [input] for (let query of queries) { @@ -27,7 +24,7 @@ function enrichQueries(input) { return wasArray ? queries : queries[0] } -exports.fetch = async function (ctx) { +export async function fetch(ctx: any) { const db = getAppDB() const body = await db.allDocs( @@ -36,10 +33,10 @@ exports.fetch = async function (ctx) { }) ) - ctx.body = enrichQueries(body.rows.map(row => row.doc)) + ctx.body = enrichQueries(body.rows.map((row: any) => row.doc)) } -exports.import = async ctx => { +const _import = async (ctx: any) => { const body = ctx.request.body const data = body.data @@ -49,7 +46,7 @@ exports.import = async ctx => { let datasourceId if (!body.datasourceId) { // construct new datasource - const info = await importer.getInfo() + const info: any = await importer.getInfo() let datasource = { type: "datasource", source: "REST", @@ -77,8 +74,9 @@ exports.import = async ctx => { } ctx.status = 200 } +export { _import as import } -exports.save = async function (ctx) { +export async function save(ctx: any) { const db = getAppDB() const query = ctx.request.body @@ -93,7 +91,7 @@ exports.save = async function (ctx) { ctx.message = `Query ${query.name} saved successfully.` } -exports.find = async function (ctx) { +export async function find(ctx: any) { const db = getAppDB() const query = enrichQueries(await db.get(ctx.params.queryId)) // remove properties that could be dangerous in real app @@ -104,7 +102,7 @@ exports.find = async function (ctx) { ctx.body = query } -exports.preview = async function (ctx) { +export async function preview(ctx: any) { const db = getAppDB() const datasource = await db.get(ctx.request.body.datasourceId) @@ -114,16 +112,18 @@ exports.preview = async function (ctx) { ctx.request.body try { - const { rows, keys, info, extra } = await Runner.run({ - appId: ctx.appId, - datasource, - queryVerb, - fields, - parameters, - transformer, - queryId, - }) + const runFn = () => + Runner.run({ + appId: ctx.appId, + datasource, + queryVerb, + fields, + parameters, + transformer, + queryId, + }) + const { rows, keys, info, extra } = await quotas.addQuery(runFn) ctx.body = { rows, schemaFields: [...new Set(keys)], @@ -135,7 +135,7 @@ exports.preview = async function (ctx) { } } -async function execute(ctx, opts = { rowsOnly: false }) { +async function execute(ctx: any, opts = { rowsOnly: false }) { const db = getAppDB() const query = await db.get(ctx.params.queryId) @@ -153,16 +153,19 @@ async function execute(ctx, opts = { rowsOnly: false }) { // call the relevant CRUD method on the integration class try { - const { rows, pagination, extra } = await Runner.run({ - appId: ctx.appId, - datasource, - queryVerb: query.queryVerb, - fields: query.fields, - pagination: ctx.request.body.pagination, - parameters: enrichedParameters, - transformer: query.transformer, - queryId: ctx.params.queryId, - }) + const runFn = () => + Runner.run({ + appId: ctx.appId, + datasource, + queryVerb: query.queryVerb, + fields: query.fields, + pagination: ctx.request.body.pagination, + parameters: enrichedParameters, + transformer: query.transformer, + queryId: ctx.params.queryId, + }) + + const { rows, pagination, extra } = await quotas.addQuery(runFn) if (opts && opts.rowsOnly) { ctx.body = rows } else { @@ -173,15 +176,15 @@ async function execute(ctx, opts = { rowsOnly: false }) { } } -exports.executeV1 = async function (ctx) { +export async function executeV1(ctx: any) { return execute(ctx, { rowsOnly: true }) } -exports.executeV2 = async function (ctx) { +export async function executeV2(ctx: any) { return execute(ctx, { rowsOnly: false }) } -const removeDynamicVariables = async queryId => { +const removeDynamicVariables = async (queryId: any) => { const db = getAppDB() const query = await db.get(queryId) const datasource = await db.get(query.datasourceId) @@ -190,19 +193,19 @@ const removeDynamicVariables = async queryId => { if (dynamicVariables) { // delete dynamic variables from the datasource datasource.config.dynamicVariables = dynamicVariables.filter( - dv => dv.queryId !== queryId + (dv: any) => dv.queryId !== queryId ) await db.put(datasource) // invalidate the deleted variables const variablesToDelete = dynamicVariables.filter( - dv => dv.queryId === queryId + (dv: any) => dv.queryId === queryId ) await invalidateDynamicVariables(variablesToDelete) } } -exports.destroy = async function (ctx) { +export async function destroy(ctx: any) { const db = getAppDB() await removeDynamicVariables(ctx.params.queryId) await db.remove(ctx.params.queryId, ctx.params.revId) diff --git a/packages/server/src/api/controllers/row/index.js b/packages/server/src/api/controllers/row/index.ts similarity index 77% rename from packages/server/src/api/controllers/row/index.js rename to packages/server/src/api/controllers/row/index.ts index 1d003ebd18..235276081c 100644 --- a/packages/server/src/api/controllers/row/index.js +++ b/packages/server/src/api/controllers/row/index.ts @@ -1,15 +1,16 @@ -const internal = require("./internal") -const external = require("./external") -const { isExternalTable } = require("../../../integrations/utils") +import { quotas } from "@budibase/pro" +import internal from "./internal" +import external from "./external" +import { isExternalTable } from "../../../integrations/utils" -function pickApi(tableId) { +function pickApi(tableId: any) { if (isExternalTable(tableId)) { return external } return internal } -function getTableId(ctx) { +function getTableId(ctx: any) { if (ctx.request.body && ctx.request.body.tableId) { return ctx.request.body.tableId } @@ -21,13 +22,13 @@ function getTableId(ctx) { } } -exports.patch = async ctx => { +export async function patch(ctx: any): Promise { const appId = ctx.appId const tableId = getTableId(ctx) const body = ctx.request.body // if it doesn't have an _id then its save if (body && !body._id) { - return exports.save(ctx) + return save(ctx) } try { const { row, table } = await pickApi(tableId).patch(ctx) @@ -41,13 +42,13 @@ exports.patch = async ctx => { } } -exports.save = async function (ctx) { +const saveRow = async (ctx: any) => { const appId = ctx.appId const tableId = getTableId(ctx) const body = ctx.request.body // if it has an ID already then its a patch if (body && body._id) { - return exports.patch(ctx) + return patch(ctx) } try { const { row, table } = await pickApi(tableId).save(ctx) @@ -60,7 +61,11 @@ exports.save = async function (ctx) { } } -exports.fetchView = async function (ctx) { +export async function save(ctx: any) { + await quotas.addRow(() => saveRow(ctx)) +} + +export async function fetchView(ctx: any) { const tableId = getTableId(ctx) try { ctx.body = await pickApi(tableId).fetchView(ctx) @@ -69,7 +74,7 @@ exports.fetchView = async function (ctx) { } } -exports.fetch = async function (ctx) { +export async function fetch(ctx: any) { const tableId = getTableId(ctx) try { ctx.body = await pickApi(tableId).fetch(ctx) @@ -78,7 +83,7 @@ exports.fetch = async function (ctx) { } } -exports.find = async function (ctx) { +export async function find(ctx: any) { const tableId = getTableId(ctx) try { ctx.body = await pickApi(tableId).find(ctx) @@ -87,19 +92,21 @@ exports.find = async function (ctx) { } } -exports.destroy = async function (ctx) { +export async function destroy(ctx: any) { const appId = ctx.appId const inputs = ctx.request.body const tableId = getTableId(ctx) let response, row if (inputs.rows) { let { rows } = await pickApi(tableId).bulkDestroy(ctx) + await quotas.removeRows(rows.length) response = rows for (let row of rows) { ctx.eventEmitter && ctx.eventEmitter.emitRow(`row:delete`, appId, row) } } else { let resp = await pickApi(tableId).destroy(ctx) + await quotas.removeRow() response = resp.response row = resp.row ctx.eventEmitter && ctx.eventEmitter.emitRow(`row:delete`, appId, row) @@ -110,7 +117,7 @@ exports.destroy = async function (ctx) { ctx.body = response } -exports.search = async ctx => { +export async function search(ctx: any) { const tableId = getTableId(ctx) try { ctx.status = 200 @@ -120,7 +127,7 @@ exports.search = async ctx => { } } -exports.validate = async function (ctx) { +export async function validate(ctx: any) { const tableId = getTableId(ctx) try { ctx.body = await pickApi(tableId).validate(ctx) @@ -129,7 +136,7 @@ exports.validate = async function (ctx) { } } -exports.fetchEnrichedRow = async function (ctx) { +export async function fetchEnrichedRow(ctx: any) { const tableId = getTableId(ctx) try { ctx.body = await pickApi(tableId).fetchEnrichedRow(ctx) diff --git a/packages/server/src/api/controllers/table/internal.ts b/packages/server/src/api/controllers/table/internal.ts index 4166066557..887f4fed0f 100644 --- a/packages/server/src/api/controllers/table/internal.ts +++ b/packages/server/src/api/controllers/table/internal.ts @@ -120,11 +120,7 @@ export async function destroy(ctx: any) { await db.bulkDocs( rows.rows.map((row: any) => ({ ...row.doc, _deleted: true })) ) - await quotas.updateUsage( - -rows.rows.length, - StaticQuotaName.ROWS, - QuotaUsageType.STATIC - ) + await quotas.removeRows(rows.rows.length) // update linked rows await updateLinks({ diff --git a/packages/server/src/api/controllers/table/utils.ts b/packages/server/src/api/controllers/table/utils.ts index 1086621202..20bd3cb090 100644 --- a/packages/server/src/api/controllers/table/utils.ts +++ b/packages/server/src/api/controllers/table/utils.ts @@ -147,12 +147,7 @@ export async function handleDataImport(user: any, table: any, dataImport: any) { finalData.push(row) } - await quotas.tryUpdateUsage( - () => db.bulkDocs(finalData), - finalData.length, - StaticQuotaName.ROWS, - QuotaUsageType.STATIC - ) + await quotas.addRows(finalData.length, () => db.bulkDocs(finalData)) let response = await db.put(table) table._rev = response._rev return table diff --git a/packages/server/src/api/routes/application.js b/packages/server/src/api/routes/application.ts similarity index 65% rename from packages/server/src/api/routes/application.js rename to packages/server/src/api/routes/application.ts index 4a7949a2bb..6ba4aa8fc7 100644 --- a/packages/server/src/api/routes/application.js +++ b/packages/server/src/api/routes/application.ts @@ -1,14 +1,13 @@ const Router = require("@koa/router") -const controller = require("../controllers/application") -const authorized = require("../../middleware/authorized") +import * as controller from "../controllers/application" +import authorized from "../../middleware/authorized" const { BUILDER } = require("@budibase/backend-core/permissions") -const usage = require("../../middleware/usageQuota") const router = Router() router .post("/api/applications/:appId/sync", authorized(BUILDER), controller.sync) - .post("/api/applications", authorized(BUILDER), usage, controller.create) + .post("/api/applications", authorized(BUILDER), controller.create) .get("/api/applications/:appId/definition", controller.fetchAppDefinition) .get("/api/applications", controller.fetch) .get("/api/applications/:appId/appPackage", controller.fetchAppPackage) @@ -23,11 +22,6 @@ router authorized(BUILDER), controller.revertClient ) - .delete( - "/api/applications/:appId", - authorized(BUILDER), - usage, - controller.delete - ) + .delete("/api/applications/:appId", authorized(BUILDER), controller.destroy) -module.exports = router +export default router diff --git a/packages/server/src/api/routes/index.js b/packages/server/src/api/routes/index.js index 8ded7104b0..5f0cf6bcc6 100644 --- a/packages/server/src/api/routes/index.js +++ b/packages/server/src/api/routes/index.js @@ -1,32 +1,33 @@ -const authRoutes = require("./auth") -const layoutRoutes = require("./layout") -const screenRoutes = require("./screen") -const userRoutes = require("./user") -const applicationRoutes = require("./application") -const tableRoutes = require("./table") -const rowRoutes = require("./row") -const viewRoutes = require("./view") -const staticRoutes = require("./static") -const componentRoutes = require("./component") -const automationRoutes = require("./automation") -const webhookRoutes = require("./webhook") -const roleRoutes = require("./role") -const deployRoutes = require("./deploy") -const apiKeysRoutes = require("./apikeys") -const templatesRoutes = require("./templates") -const analyticsRoutes = require("./analytics") -const routingRoutes = require("./routing") -const integrationRoutes = require("./integration") -const permissionRoutes = require("./permission") -const datasourceRoutes = require("./datasource") -const queryRoutes = require("./query") -const backupRoutes = require("./backup") -const metadataRoutes = require("./metadata") -const devRoutes = require("./dev") -const cloudRoutes = require("./cloud") -const migrationRoutes = require("./migrations") +import authRoutes from "./auth" +import layoutRoutes from "./layout" +import screenRoutes from "./screen" +import userRoutes from "./user" +import applicationRoutes from "./application" +import tableRoutes from "./table" +import rowRoutes from "./row" +import viewRoutes from "./view" +import componentRoutes from "./component" +import automationRoutes from "./automation" +import webhookRoutes from "./webhook" +import roleRoutes from "./role" +import deployRoutes from "./deploy" +import apiKeysRoutes from "./apikeys" +import templatesRoutes from "./templates" +import analyticsRoutes from "./analytics" +import routingRoutes from "./routing" +import integrationRoutes from "./integration" +import permissionRoutes from "./permission" +import datasourceRoutes from "./datasource" +import queryRoutes from "./query" +import backupRoutes from "./backup" +import metadataRoutes from "./metadata" +import devRoutes from "./dev" +import cloudRoutes from "./cloud" +import migrationRoutes from "./migrations" -exports.mainRoutes = [ +export { default as staticRoutes } from "./static" + +export const mainRoutes = [ authRoutes, deployRoutes, layoutRoutes, @@ -56,5 +57,3 @@ exports.mainRoutes = [ rowRoutes, migrationRoutes, ] - -exports.staticRoutes = staticRoutes diff --git a/packages/server/src/api/routes/row.js b/packages/server/src/api/routes/row.ts similarity index 97% rename from packages/server/src/api/routes/row.js rename to packages/server/src/api/routes/row.ts index aa300108f1..95112550ce 100644 --- a/packages/server/src/api/routes/row.js +++ b/packages/server/src/api/routes/row.ts @@ -1,11 +1,7 @@ const Router = require("@koa/router") -const rowController = require("../controllers/row") -const authorized = require("../../middleware/authorized") -const usage = require("../../middleware/usageQuota") -const { - paramResource, - paramSubResource, -} = require("../../middleware/resourceId") +import * as rowController from "../controllers/row" +import authorized from "../../middleware/authorized" +import { paramResource, paramSubResource } from "../../middleware/resourceId" const { PermissionLevels, PermissionTypes, @@ -180,7 +176,6 @@ router "/api/:tableId/rows", paramResource("tableId"), authorized(PermissionTypes.TABLE, PermissionLevels.WRITE), - usage, rowController.save ) /** @@ -247,8 +242,7 @@ router "/api/:tableId/rows", paramResource("tableId"), authorized(PermissionTypes.TABLE, PermissionLevels.WRITE), - usage, rowController.destroy ) -module.exports = router +export default router diff --git a/packages/server/src/api/routes/static.js b/packages/server/src/api/routes/static.js index 8a1e529a59..71efbc9d5b 100644 --- a/packages/server/src/api/routes/static.js +++ b/packages/server/src/api/routes/static.js @@ -1,14 +1,14 @@ const Router = require("@koa/router") -const controller = require("../controllers/static") -const { budibaseTempDir } = require("../../utilities/budibaseDir") -const authorized = require("../../middleware/authorized") -const { +import * as controller from "../controllers/static" +import { budibaseTempDir } from "../../utilities/budibaseDir" +import authorized from "../../middleware/authorized" +import { BUILDER, PermissionTypes, PermissionLevels, -} = require("@budibase/backend-core/permissions") -const env = require("../../environment") -const { paramResource } = require("../../middleware/resourceId") +} from "@budibase/backend-core/permissions" +import * as env from "../../environment" +import { paramResource } from "../../middleware/resourceId" const router = Router() @@ -52,4 +52,4 @@ router controller.getSignedUploadURL ) -module.exports = router +export default router diff --git a/packages/server/src/automations/steps/createRow.ts b/packages/server/src/automations/steps/createRow.ts index 6c5e5a87b2..566beb2c22 100644 --- a/packages/server/src/automations/steps/createRow.ts +++ b/packages/server/src/automations/steps/createRow.ts @@ -1,4 +1,4 @@ -import { quotas, QuotaUsageType, StaticQuotaName } from "@budibase/pro" +import { quotas } from "@budibase/pro" import { save } from "../../api/controllers/row" import { cleanUpRow, getError } from "../automationUtils" import { buildCtx } from "./utils" @@ -78,12 +78,7 @@ export async function run({ inputs, appId, emitter }: any) { try { inputs.row = await cleanUpRow(inputs.row.tableId, inputs.row) - await quotas.tryUpdateUsage( - () => save(ctx), - 1, - StaticQuotaName.ROWS, - QuotaUsageType.STATIC - ) + await quotas.addRow(() => save(ctx)) return { row: inputs.row, response: ctx.body, diff --git a/packages/server/src/automations/steps/deleteRow.ts b/packages/server/src/automations/steps/deleteRow.ts index 0261f80ea6..7b2fd10bda 100644 --- a/packages/server/src/automations/steps/deleteRow.ts +++ b/packages/server/src/automations/steps/deleteRow.ts @@ -1,7 +1,7 @@ import { destroy } from "../../api/controllers/row" import { buildCtx } from "./utils" import { getError } from "../automationUtils" -import { quotas, QuotaUsageType, StaticQuotaName } from "@budibase/pro" +import { quotas } from "@budibase/pro" export const definition = { description: "Delete a row from your database", @@ -73,8 +73,8 @@ export async function run({ inputs, appId, emitter }: any) { }) try { - await quotas.updateUsage(-1, StaticQuotaName.ROWS, QuotaUsageType.STATIC) await destroy(ctx) + await quotas.removeRow() return { response: ctx.body, row: ctx.row, diff --git a/packages/server/src/automations/utils.js b/packages/server/src/automations/utils.ts similarity index 73% rename from packages/server/src/automations/utils.js rename to packages/server/src/automations/utils.ts index 3ee1f535c7..4aff5bc929 100644 --- a/packages/server/src/automations/utils.js +++ b/packages/server/src/automations/utils.ts @@ -1,23 +1,29 @@ -const { Thread, ThreadType } = require("../threads") -const { definitions } = require("./triggerInfo") -const webhooks = require("../api/controllers/webhook") -const CouchDB = require("../db") -const { queue } = require("./bullboard") -const newid = require("../db/newid") -const { updateEntityMetadata } = require("../utilities") -const { MetadataTypes } = require("../constants") -const { getProdAppID } = require("@budibase/backend-core/db") -const { cloneDeep } = require("lodash/fp") -const { getAppDB, getAppId } = require("@budibase/backend-core/context") +import { Thread, ThreadType } from "../threads" +import { definitions } from "./triggerInfo" +import { destroy, Webhook, WebhookType, save } from "../api/controllers/webhook" +import CouchDB from "../db" +import { queue } from "./bullboard" +import newid from "../db/newid" +import { updateEntityMetadata } from "../utilities" +import { MetadataTypes } from "../constants" +import { getProdAppID } from "@budibase/backend-core/db" +import { cloneDeep } from "lodash/fp" +import { getAppDB, getAppId } from "@budibase/backend-core/context" +import { tenancy } from "@budibase/backend-core" +import { quotas } from "@budibase/pro" const WH_STEP_ID = definitions.WEBHOOK.stepId const CRON_STEP_ID = definitions.CRON.stepId const Runner = new Thread(ThreadType.AUTOMATION) -exports.processEvent = async job => { +export async function processEvent(job: any) { try { - // need to actually await these so that an error can be captured properly - return await Runner.run(job) + const tenantId = tenancy.getTenantIDFromAppID(job.data.event.appId) + return await tenancy.doInTenant(tenantId, async () => { + // need to actually await these so that an error can be captured properly + const runFn = () => Runner.run(job) + return quotas.addAutomation(runFn) + }) } catch (err) { console.error( `${job.data.automation.appId} automation ${job.data.automation._id} was unable to run - ${err}` @@ -26,11 +32,15 @@ exports.processEvent = async job => { } } -exports.updateTestHistory = async (appId, automation, history) => { +export async function updateTestHistory( + appId: any, + automation: any, + history: any +) { return updateEntityMetadata( MetadataTypes.AUTOMATION_TEST_HISTORY, automation._id, - metadata => { + (metadata: any) => { if (metadata && Array.isArray(metadata.history)) { metadata.history.push(history) } else { @@ -43,7 +53,7 @@ exports.updateTestHistory = async (appId, automation, history) => { ) } -exports.removeDeprecated = definitions => { +export function removeDeprecated(definitions: any) { const base = cloneDeep(definitions) for (let key of Object.keys(base)) { if (base[key].deprecated) { @@ -54,13 +64,15 @@ exports.removeDeprecated = definitions => { } // end the repetition and the job itself -exports.disableAllCrons = async appId => { +export async function disableAllCrons(appId: any) { const promises = [] const jobs = await queue.getRepeatableJobs() for (let job of jobs) { if (job.key.includes(`${appId}_cron`)) { promises.push(queue.removeRepeatableByKey(job.key)) - promises.push(queue.removeJobs(job.id)) + if (job.id) { + promises.push(queue.removeJobs(job.id)) + } } } return Promise.all(promises) @@ -71,9 +83,9 @@ exports.disableAllCrons = async appId => { * @param {string} appId The ID of the app in which we are checking for webhooks * @param {object|undefined} automation The automation object to be updated. */ -exports.enableCronTrigger = async (appId, automation) => { +export async function enableCronTrigger(appId: any, automation: any) { const trigger = automation ? automation.definition.trigger : null - function isCronTrigger(auto) { + function isCronTrigger(auto: any) { return ( auto && auto.definition.trigger && @@ -84,7 +96,7 @@ exports.enableCronTrigger = async (appId, automation) => { if (isCronTrigger(automation)) { // make a job id rather than letting Bull decide, makes it easier to handle on way out const jobId = `${appId}_cron_${newid()}` - const job = await queue.add( + const job: any = await queue.add( { automation, event: { appId, timestamp: Date.now() }, @@ -112,13 +124,13 @@ exports.enableCronTrigger = async (appId, automation) => { * @returns {Promise} After this is complete the new automation object may have been updated and should be * written to DB (this does not write to DB as it would be wasteful to repeat). */ -exports.checkForWebhooks = async ({ oldAuto, newAuto }) => { +export async function checkForWebhooks({ oldAuto, newAuto }: any) { const appId = getAppId() const oldTrigger = oldAuto ? oldAuto.definition.trigger : null const newTrigger = newAuto ? newAuto.definition.trigger : null const triggerChanged = oldTrigger && newTrigger && oldTrigger.id !== newTrigger.id - function isWebhookTrigger(auto) { + function isWebhookTrigger(auto: any) { return ( auto && auto.definition.trigger && @@ -144,7 +156,7 @@ exports.checkForWebhooks = async ({ oldAuto, newAuto }) => { delete newTrigger.webhookId newTrigger.inputs = {} } - await webhooks.destroy(ctx) + await destroy(ctx) } catch (err) { // don't worry about not being able to delete, if it doesn't exist all good } @@ -154,17 +166,17 @@ exports.checkForWebhooks = async ({ oldAuto, newAuto }) => { (!isWebhookTrigger(oldAuto) || triggerChanged) && isWebhookTrigger(newAuto) ) { - const ctx = { + const ctx: any = { appId, request: { - body: new webhooks.Webhook( + body: new Webhook( "Automation webhook", - webhooks.WebhookType.AUTOMATION, + WebhookType.AUTOMATION, newAuto._id ), }, } - await webhooks.save(ctx) + await save(ctx) const id = ctx.body.webhook._id newTrigger.webhookId = id // the app ID has to be development for this endpoint @@ -184,6 +196,6 @@ exports.checkForWebhooks = async ({ oldAuto, newAuto }) => { * @param appId {string} the app that is being removed. * @return {Promise} clean is complete if this succeeds. */ -exports.cleanupAutomations = async appId => { - await exports.disableAllCrons(appId) +export async function cleanupAutomations(appId: any) { + await disableAllCrons(appId) } diff --git a/packages/server/src/middleware/usageQuota.ts b/packages/server/src/middleware/usageQuota.ts deleted file mode 100644 index 7347635e78..0000000000 --- a/packages/server/src/middleware/usageQuota.ts +++ /dev/null @@ -1,172 +0,0 @@ -import { quotas, StaticQuotaName, QuotaUsageType } from "@budibase/pro" -const { getUniqueRows } = require("../utilities/usageQuota/rows") -const { - isExternalTable, - isRowId: isExternalRowId, -} = require("../integrations/utils") -const { getAppDB } = require("@budibase/backend-core/context") - -// currently only counting new writes and deletes -const METHOD_MAP: any = { - POST: 1, - DELETE: -1, -} - -const DOMAIN_MAP: any = { - rows: { - name: StaticQuotaName.ROWS, - type: QuotaUsageType.STATIC, - }, - applications: { - name: StaticQuotaName.APPS, - type: QuotaUsageType.STATIC, - }, -} - -function getQuotaInfo(url: string) { - for (let domain of Object.keys(DOMAIN_MAP)) { - if (url.indexOf(domain) !== -1) { - return DOMAIN_MAP[domain] - } - } -} - -module.exports = async (ctx: any, next: any) => { - if (!quotas.useQuotas()) { - return next() - } - - let usage = METHOD_MAP[ctx.req.method] - const quotaInfo = getQuotaInfo(ctx.req.url) - if (usage == null || quotaInfo == null) { - return next() - } - // post request could be a save of a pre-existing entry - if (ctx.request.body && ctx.request.body._id && ctx.request.body._rev) { - const usageId = ctx.request.body._id - try { - if (ctx.appId) { - const db = getAppDB() - await db.get(usageId) - } - return next() - } catch (err) { - if ( - isExternalTable(usageId) || - (ctx.request.body.tableId && - isExternalTable(ctx.request.body.tableId)) || - isExternalRowId(usageId) - ) { - return next() - } else { - ctx.throw(404, `${usageId} does not exist`) - } - } - } - try { - await performRequest(ctx, next, quotaInfo, usage) - } catch (err) { - ctx.throw(400, err) - } -} - -const performRequest = async ( - ctx: any, - next: any, - quotaInfo: any, - usage: number -) => { - const usageContext = { - skipNext: false, - skipUsage: false, - [StaticQuotaName.APPS]: {}, - } - - const quotaName = quotaInfo.name - - if (usage === -1) { - if (PRE_DELETE[quotaName]) { - await PRE_DELETE[quotaName](ctx, usageContext) - } - } else { - if (PRE_CREATE[quotaName]) { - await PRE_CREATE[quotaName](ctx, usageContext) - } - } - - // run the request - if (!usageContext.skipNext) { - await quotas.updateUsage(usage, quotaName, quotaInfo.type, { - dryRun: true, - }) - await next() - } - - if (usage === -1) { - if (POST_DELETE[quotaName]) { - await POST_DELETE[quotaName](ctx, usageContext) - } - } else { - if (POST_CREATE[quotaName]) { - await POST_CREATE[quotaName](ctx) - } - } - - // update the usage - if (!usageContext.skipUsage) { - await quotas.updateUsage(usage, quotaName, quotaInfo.type) - } -} - -const appPreDelete = async (ctx: any, usageContext: any) => { - if (ctx.query.unpublish) { - // don't run usage decrement for unpublish - usageContext.skipUsage = true - return - } - - // store the row count to delete - const rows = await getUniqueRows([ctx.appId]) - if (rows.length) { - usageContext[StaticQuotaName.APPS] = { rowCount: rows.length } - } -} - -const appPostDelete = async (ctx: any, usageContext: any) => { - // delete the app rows from usage - const rowCount = usageContext[StaticQuotaName.APPS].rowCount - if (rowCount) { - await quotas.updateUsage( - -rowCount, - StaticQuotaName.ROWS, - QuotaUsageType.STATIC - ) - } -} - -const appPostCreate = async (ctx: any) => { - // app import & template creation - if (ctx.request.body.useTemplate === "true") { - const rows = await getUniqueRows([ctx.response.body.appId]) - const rowCount = rows ? rows.length : 0 - await quotas.updateUsage( - rowCount, - StaticQuotaName.ROWS, - QuotaUsageType.STATIC - ) - } -} - -const PRE_DELETE: any = { - [StaticQuotaName.APPS]: appPreDelete, -} - -const POST_DELETE: any = { - [StaticQuotaName.APPS]: appPostDelete, -} - -const PRE_CREATE: any = {} - -const POST_CREATE: any = { - [StaticQuotaName.APPS]: appPostCreate, -} diff --git a/packages/server/src/migrations/functions/usageQuotas/index.ts b/packages/server/src/migrations/functions/usageQuotas/index.ts index 4e3fe436db..ed1a765551 100644 --- a/packages/server/src/migrations/functions/usageQuotas/index.ts +++ b/packages/server/src/migrations/functions/usageQuotas/index.ts @@ -1,8 +1,3 @@ -import { quotas } from "@budibase/pro" - export const runQuotaMigration = async (migration: Function) => { - if (!quotas.useQuotas()) { - return - } await migration() } diff --git a/packages/server/src/module.d.ts b/packages/server/src/module.d.ts index b7850efff3..eb460a74a5 100644 --- a/packages/server/src/module.d.ts +++ b/packages/server/src/module.d.ts @@ -1,3 +1,5 @@ declare module "@budibase/backend-core" declare module "@budibase/backend-core/tenancy" declare module "@budibase/backend-core/db" +declare module "@budibase/backend-core/context" +declare module "@budibase/backend-core/cache" diff --git a/packages/server/src/threads/index.js b/packages/server/src/threads/index.ts similarity index 74% rename from packages/server/src/threads/index.js rename to packages/server/src/threads/index.ts index 94571c31d1..a6a1cb1ad9 100644 --- a/packages/server/src/threads/index.js +++ b/packages/server/src/threads/index.ts @@ -1,12 +1,12 @@ -const workerFarm = require("worker-farm") -const env = require("../environment") +import workerFarm from "worker-farm" +import * as env from "../environment" -const ThreadType = { +export const ThreadType = { QUERY: "query", AUTOMATION: "automation", } -function typeToFile(type) { +function typeToFile(type: any) { let filename = null switch (type) { case ThreadType.QUERY: @@ -21,8 +21,13 @@ function typeToFile(type) { return require.resolve(filename) } -class Thread { - constructor(type, opts = { timeoutMs: null, count: 1 }) { +export class Thread { + type: any + count: any + disableThreading: any + workers: any + + constructor(type: any, opts: any = { timeoutMs: null, count: 1 }) { this.type = type this.count = opts.count ? opts.count : 1 this.disableThreading = @@ -31,7 +36,7 @@ class Thread { this.count === 0 || env.isInThread() if (!this.disableThreading) { - const workerOpts = { + const workerOpts: any = { autoStart: true, maxConcurrentWorkers: this.count, } @@ -42,7 +47,7 @@ class Thread { } } - run(data) { + run(data: any) { return new Promise((resolve, reject) => { let fncToCall // if in test then don't use threading @@ -51,7 +56,7 @@ class Thread { } else { fncToCall = this.workers } - fncToCall(data, (err, response) => { + fncToCall(data, (err: any, response: any) => { if (err) { reject(err) } else { @@ -61,6 +66,3 @@ class Thread { }) } } - -module.exports.Thread = Thread -module.exports.ThreadType = ThreadType diff --git a/packages/server/src/utilities/queue/inMemoryQueue.js b/packages/server/src/utilities/queue/inMemoryQueue.js index dbd49c43c0..aebc0ba919 100644 --- a/packages/server/src/utilities/queue/inMemoryQueue.js +++ b/packages/server/src/utilities/queue/inMemoryQueue.js @@ -66,7 +66,8 @@ class InMemoryQueue { * @param {object} msg A message to be transported over the queue, this should be * a JSON message as this is required by Bull. */ - add(msg) { + // eslint-disable-next-line no-unused-vars + add(msg, repeat) { if (typeof msg !== "object") { throw "Queue only supports carrying JSON." } @@ -90,6 +91,11 @@ class InMemoryQueue { return [] } + // eslint-disable-next-line no-unused-vars + removeJobs(pattern) { + // no-op + } + /** * Implemented for tests */ diff --git a/packages/worker/src/api/controllers/global/auth.ts b/packages/worker/src/api/controllers/global/auth.ts index 9c2047a8e3..33d8eb3d88 100644 --- a/packages/worker/src/api/controllers/global/auth.ts +++ b/packages/worker/src/api/controllers/global/auth.ts @@ -224,7 +224,7 @@ async function oidcStrategyFactory(ctx: any, configId: any) { const chosenConfig = config.configs.filter((c: any) => c.uuid === configId)[0] let callbackUrl = await exports.oidcCallbackUrl(chosenConfig) - return oidc.strategyFactory(chosenConfig, callbackUrl) + return oidc.strategyFactory(chosenConfig, callbackUrl, users.save) } /** diff --git a/packages/worker/src/api/controllers/global/users.ts b/packages/worker/src/api/controllers/global/users.ts index ddba7fd9d4..7094839477 100644 --- a/packages/worker/src/api/controllers/global/users.ts +++ b/packages/worker/src/api/controllers/global/users.ts @@ -37,7 +37,7 @@ const allUsers = async () => { export const save = async (ctx: any) => { try { - const user = await users.save(ctx.request.body, getTenantId()) + const user: any = await users.save(ctx.request.body, getTenantId()) // let server know to sync user await syncUserInApps(user._id) ctx.body = user @@ -129,6 +129,7 @@ export const destroy = async (ctx: any) => { await removeUserFromInfoDB(dbUser) await db.remove(dbUser._id, dbUser._rev) + await quotas.removeUser(dbUser) await userCache.invalidateUser(dbUser._id) await invalidateSessions(dbUser._id) // let server know to sync user