diff --git a/lerna.json b/lerna.json index c3ee9351fe..71be7bfa92 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "0.9.140-alpha.8", + "version": "0.9.143-alpha.4", "npmClient": "yarn", "packages": [ "packages/*" diff --git a/packages/auth/package.json b/packages/auth/package.json index 22f96811e8..edf2a0ad17 100644 --- a/packages/auth/package.json +++ b/packages/auth/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/auth", - "version": "0.9.140-alpha.8", + "version": "0.9.143-alpha.4", "description": "Authentication middlewares for budibase builder and apps", "main": "src/index.js", "author": "Budibase", diff --git a/packages/auth/src/db/constants.js b/packages/auth/src/db/constants.js index 77643ce4c5..ad4f6c9f66 100644 --- a/packages/auth/src/db/constants.js +++ b/packages/auth/src/db/constants.js @@ -12,6 +12,7 @@ exports.StaticDatabases = { name: "global-info", docs: { tenants: "tenants", + usageQuota: "usage_quota", }, }, } diff --git a/packages/auth/src/db/utils.js b/packages/auth/src/db/utils.js index a1a831523e..09e2ff6314 100644 --- a/packages/auth/src/db/utils.js +++ b/packages/auth/src/db/utils.js @@ -368,8 +368,33 @@ async function getScopedConfig(db, params) { return configDoc && configDoc.config ? configDoc.config : configDoc } +function generateNewUsageQuotaDoc() { + return { + _id: StaticDatabases.PLATFORM_INFO.docs.usageQuota, + quotaReset: Date.now() + 2592000000, + usageQuota: { + automationRuns: 0, + rows: 0, + storage: 0, + apps: 0, + users: 0, + views: 0, + emails: 0, + }, + usageLimits: { + automationRuns: 1000, + rows: 4000, + apps: 4, + storage: 1000, + users: 10, + emails: 50, + }, + } +} + exports.Replication = Replication exports.getScopedConfig = getScopedConfig exports.generateConfigID = generateConfigID exports.getConfigParams = getConfigParams exports.getScopedFullConfig = getScopedFullConfig +exports.generateNewUsageQuotaDoc = generateNewUsageQuotaDoc diff --git a/packages/auth/src/security/permissions.js b/packages/auth/src/security/permissions.js index 03fa5fa562..d0308d783e 100644 --- a/packages/auth/src/security/permissions.js +++ b/packages/auth/src/security/permissions.js @@ -139,8 +139,7 @@ exports.doesHaveResourcePermission = ( // set foundSub to not subResourceId, incase there is no subResource let foundMain = false, foundSub = false - for (let [resource, level] of Object.entries(permissions)) { - const levels = getAllowedLevels(level) + for (let [resource, levels] of Object.entries(permissions)) { if (resource === resourceId && levels.indexOf(permLevel) !== -1) { foundMain = true } @@ -177,10 +176,6 @@ exports.doesHaveBasePermission = (permType, permLevel, permissionIds) => { return false } -exports.higherPermission = (perm1, perm2) => { - return levelToNumber(perm1) > levelToNumber(perm2) ? perm1 : perm2 -} - exports.isPermissionLevelHigherThanRead = level => { return levelToNumber(level) > 1 } diff --git a/packages/auth/src/security/roles.js b/packages/auth/src/security/roles.js index baa8fc40dc..71fbc10132 100644 --- a/packages/auth/src/security/roles.js +++ b/packages/auth/src/security/roles.js @@ -1,6 +1,6 @@ const { getDB } = require("../db") const { cloneDeep } = require("lodash/fp") -const { BUILTIN_PERMISSION_IDS, higherPermission } = require("./permissions") +const { BUILTIN_PERMISSION_IDS } = require("./permissions") const { generateRoleID, getRoleParams, @@ -193,8 +193,17 @@ exports.getUserPermissions = async (appId, userRoleId) => { const permissions = {} for (let role of rolesHierarchy) { if (role.permissions) { - for (let [resource, level] of Object.entries(role.permissions)) { - permissions[resource] = higherPermission(permissions[resource], level) + for (let [resource, levels] of Object.entries(role.permissions)) { + if (!permissions[resource]) { + permissions[resource] = [] + } + const permsSet = new Set(permissions[resource]) + if (Array.isArray(levels)) { + levels.forEach(level => permsSet.add(level)) + } else { + permsSet.add(levels) + } + permissions[resource] = [...permsSet] } } } diff --git a/packages/bbui/package.json b/packages/bbui/package.json index 82ee4b7d31..81a7a3b077 100644 --- a/packages/bbui/package.json +++ b/packages/bbui/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/bbui", "description": "A UI solution used in the different Budibase projects.", - "version": "0.9.140-alpha.8", + "version": "0.9.143-alpha.4", "license": "AGPL-3.0", "svelte": "src/index.js", "module": "dist/bbui.es.js", diff --git a/packages/builder/package.json b/packages/builder/package.json index c4d57c9709..82bebadbf2 100644 --- a/packages/builder/package.json +++ b/packages/builder/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/builder", - "version": "0.9.140-alpha.8", + "version": "0.9.143-alpha.4", "license": "AGPL-3.0", "private": true, "scripts": { @@ -65,10 +65,10 @@ } }, "dependencies": { - "@budibase/bbui": "^0.9.140-alpha.8", - "@budibase/client": "^0.9.140-alpha.8", + "@budibase/bbui": "^0.9.143-alpha.4", + "@budibase/client": "^0.9.143-alpha.4", "@budibase/colorpicker": "1.1.2", - "@budibase/string-templates": "^0.9.140-alpha.8", + "@budibase/string-templates": "^0.9.143-alpha.4", "@sentry/browser": "5.19.1", "@spectrum-css/page": "^3.0.1", "@spectrum-css/vars": "^3.0.1", diff --git a/packages/builder/src/analytics/index.js b/packages/builder/src/analytics/index.js index b79ab67e0c..e9edf38d74 100644 --- a/packages/builder/src/analytics/index.js +++ b/packages/builder/src/analytics/index.js @@ -3,8 +3,6 @@ import PosthogClient from "./PosthogClient" import IntercomClient from "./IntercomClient" import SentryClient from "./SentryClient" import { Events } from "./constants" -import { auth } from "stores/portal" -import { get } from "svelte/store" const posthog = new PosthogClient( process.env.POSTHOG_TOKEN, @@ -19,27 +17,13 @@ class AnalyticsHub { } async activate() { - // Setting the analytics env var off in the backend overrides org/tenant settings const analyticsStatus = await api.get("/api/analytics") const json = await analyticsStatus.json() - // Multitenancy disabled on the backend + // Analytics disabled if (!json.enabled) return - const tenantId = get(auth).tenantId - - if (tenantId) { - const res = await api.get( - `/api/global/configs/public?tenantId=${tenantId}` - ) - const orgJson = await res.json() - - // analytics opted out for the tenant - if (orgJson.config?.analytics === false) return - } - this.clients.forEach(client => client.init()) - this.enabled = true } identify(id, metadata) { diff --git a/packages/builder/src/builderStore/dataBinding.js b/packages/builder/src/builderStore/dataBinding.js index d3af6799f3..0858b29bcb 100644 --- a/packages/builder/src/builderStore/dataBinding.js +++ b/packages/builder/src/builderStore/dataBinding.js @@ -443,10 +443,9 @@ function bindingReplacement(bindableProperties, textWithBindings, convertTo) { for (let from of convertFromProps) { if (shouldReplaceBinding(newBoundValue, from, convertTo)) { const binding = bindableProperties.find(el => el[convertFrom] === from) - newBoundValue = newBoundValue.replace( - new RegExp(from, "gi"), - binding[convertTo] - ) + while (newBoundValue.includes(from)) { + newBoundValue = newBoundValue.replace(from, binding[convertTo]) + } } } result = result.replace(boundValue, newBoundValue) diff --git a/packages/builder/src/components/backend/DatasourceNavigator/DatasourceNavigator.svelte b/packages/builder/src/components/backend/DatasourceNavigator/DatasourceNavigator.svelte index 19713595ce..6ba8e4042f 100644 --- a/packages/builder/src/components/backend/DatasourceNavigator/DatasourceNavigator.svelte +++ b/packages/builder/src/components/backend/DatasourceNavigator/DatasourceNavigator.svelte @@ -1,8 +1,9 @@ {#if $database?._id}
- {#each $datasources.list as datasource, idx} + {#each enrichedDataSources as datasource, idx} 0} text={datasource.name} - opened={openDataSources.includes(datasource._id)} - selected={$datasources.selected === datasource._id} + opened={datasource.open} + selected={datasource.selected} withArrow={true} on:click={() => selectDatasource(datasource)} on:iconClick={() => toggleNode(datasource)} @@ -61,7 +81,7 @@ {/if} - {#if openDataSources.includes(datasource._id)} + {#if datasource.open} {#each $queries.list.filter(query => query.datasourceId === datasource._id) as query} + import { Select, Label } from "@budibase/bbui" + import { currentAsset, store } from "builderStore" + import { getActionProviderComponents } from "builderStore/dataBinding" + + export let parameters + + $: actionProviders = getActionProviderComponents( + $currentAsset, + $store.selectedComponentId, + "RefreshDatasource" + ) + + +
+ + -
-
- - + {#if !$admin.cloud} + - Analytics - - If you would like to send analytics that help us make Budibase better, - please let us know below. - + Platform + Here you can set up general platform settings. - -
- +
+
+ + +
- + {/if} +
+ +
{/if} diff --git a/packages/builder/src/stores/backend/datasources.js b/packages/builder/src/stores/backend/datasources.js index f7e689d8ad..7c74074e03 100644 --- a/packages/builder/src/stores/backend/datasources.js +++ b/packages/builder/src/stores/backend/datasources.js @@ -1,4 +1,4 @@ -import { writable } from "svelte/store" +import { writable, get } from "svelte/store" import { queries, tables, views } from "./" import api from "../../builderStore/api" @@ -8,7 +8,8 @@ export const INITIAL_DATASOURCE_VALUES = { } export function createDatasourcesStore() { - const { subscribe, update, set } = writable(INITIAL_DATASOURCE_VALUES) + const store = writable(INITIAL_DATASOURCE_VALUES) + const { subscribe, update, set } = store return { subscribe, @@ -21,7 +22,15 @@ export function createDatasourcesStore() { fetch: async () => { const response = await api.get(`/api/datasources`) const json = await response.json() - update(state => ({ ...state, list: json, selected: null })) + + // Clear selected if it no longer exists, otherwise keep it + const selected = get(store).selected + let nextSelected = null + if (selected && json.find(source => source._id === selected)) { + nextSelected = selected + } + + update(state => ({ ...state, list: json, selected: nextSelected })) return json }, select: async datasourceId => { diff --git a/packages/cli/package.json b/packages/cli/package.json index 459df417c3..57878e2929 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/cli", - "version": "0.9.140-alpha.8", + "version": "0.9.143-alpha.4", "description": "Budibase CLI, for developers, self hosting and migrations.", "main": "src/index.js", "bin": { diff --git a/packages/client/manifest.json b/packages/client/manifest.json index 7bef9c2e4b..2e64b1fb4c 100644 --- a/packages/client/manifest.json +++ b/packages/client/manifest.json @@ -2389,6 +2389,7 @@ "icon": "Data", "illegalChildren": ["section"], "hasChildren": true, + "actions": ["RefreshDatasource"], "settings": [ { "type": "dataSource", diff --git a/packages/client/package.json b/packages/client/package.json index cc585ebac5..d4a4a24ba9 100644 --- a/packages/client/package.json +++ b/packages/client/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/client", - "version": "0.9.140-alpha.8", + "version": "0.9.143-alpha.4", "license": "MPL-2.0", "module": "dist/budibase-client.js", "main": "dist/budibase-client.js", @@ -19,9 +19,9 @@ "dev:builder": "rollup -cw" }, "dependencies": { - "@budibase/bbui": "^0.9.140-alpha.8", + "@budibase/bbui": "^0.9.143-alpha.4", "@budibase/standard-components": "^0.9.139", - "@budibase/string-templates": "^0.9.140-alpha.8", + "@budibase/string-templates": "^0.9.143-alpha.4", "regexparam": "^1.3.0", "shortid": "^2.2.15", "svelte-spa-router": "^3.0.5" diff --git a/packages/client/src/utils/buttonActions.js b/packages/client/src/utils/buttonActions.js index aeefe6163c..11aa033c1d 100644 --- a/packages/client/src/utils/buttonActions.js +++ b/packages/client/src/utils/buttonActions.js @@ -88,7 +88,7 @@ const validateFormHandler = async (action, context) => { ) } -const refreshDatasourceHandler = async (action, context) => { +const refreshDataProviderHandler = async (action, context) => { return await executeActionHandler( context, action.parameters.componentId, @@ -139,7 +139,7 @@ const handlerMap = { ["Execute Query"]: queryExecutionHandler, ["Trigger Automation"]: triggerAutomationHandler, ["Validate Form"]: validateFormHandler, - ["Refresh Datasource"]: refreshDatasourceHandler, + ["Refresh Data Provider"]: refreshDataProviderHandler, ["Log Out"]: logoutHandler, ["Clear Form"]: clearFormHandler, ["Close Screen Modal"]: closeScreenModalHandler, diff --git a/packages/server/package.json b/packages/server/package.json index e0e7a8c8f8..07f11c8161 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/server", "email": "hi@budibase.com", - "version": "0.9.140-alpha.8", + "version": "0.9.143-alpha.4", "description": "Budibase Web Server", "main": "src/index.js", "repository": { @@ -25,7 +25,9 @@ "lint:fix": "yarn run format && yarn run lint", "initialise": "node scripts/initialise.js", "multi:enable": "node scripts/multiTenancy.js enable", - "multi:disable": "node scripts/multiTenancy.js disable" + "multi:disable": "node scripts/multiTenancy.js disable", + "selfhost:enable": "node scripts/selfhost.js enable", + "selfhost:disable": "node scripts/selfhost.js disable" }, "jest": { "preset": "ts-jest", @@ -62,9 +64,9 @@ "author": "Budibase", "license": "AGPL-3.0-or-later", "dependencies": { - "@budibase/auth": "^0.9.140-alpha.8", - "@budibase/client": "^0.9.140-alpha.8", - "@budibase/string-templates": "^0.9.140-alpha.8", + "@budibase/auth": "^0.9.143-alpha.4", + "@budibase/client": "^0.9.143-alpha.4", + "@budibase/string-templates": "^0.9.143-alpha.4", "@elastic/elasticsearch": "7.10.0", "@koa/router": "8.0.0", "@sendgrid/mail": "7.1.1", diff --git a/packages/server/scripts/integrations/pg-json/docker-compose.yml b/packages/server/scripts/integrations/pg-json/docker-compose.yml new file mode 100644 index 0000000000..6bc307a86d --- /dev/null +++ b/packages/server/scripts/integrations/pg-json/docker-compose.yml @@ -0,0 +1,28 @@ +version: "3.8" +services: + db: + container_name: postgres-json + image: postgres + restart: always + environment: + POSTGRES_USER: root + POSTGRES_PASSWORD: root + POSTGRES_DB: main + ports: + - "5432:5432" + volumes: + #- pg_data:/var/lib/postgresql/data/ + - ./init.sql:/docker-entrypoint-initdb.d/init.sql + + pgadmin: + container_name: pgadmin-json + image: dpage/pgadmin4 + restart: always + environment: + PGADMIN_DEFAULT_EMAIL: root@root.com + PGADMIN_DEFAULT_PASSWORD: root + ports: + - "5050:80" + +#volumes: +# pg_data: diff --git a/packages/server/scripts/integrations/pg-json/init.sql b/packages/server/scripts/integrations/pg-json/init.sql new file mode 100644 index 0000000000..06a5b4901d --- /dev/null +++ b/packages/server/scripts/integrations/pg-json/init.sql @@ -0,0 +1,22 @@ +SELECT 'CREATE DATABASE main' +WHERE NOT EXISTS (SELECT FROM pg_database WHERE datname = 'main')\gexec +CREATE TABLE jsonTable ( + id character varying(32), + data jsonb, + text text +); + +INSERT INTO jsonTable (id, data) VALUES ('1', '{"id": 1, "age": 1, "name": "Mike", "newline": "this is text with a\n newline in it"}'); + +CREATE VIEW jsonView AS SELECT + x.id, + x.age, + x.name, + x.newline +FROM + jsonTable c, + LATERAL jsonb_to_record(c.data) x (id character varying(32), + age BIGINT, + name TEXT, + newline TEXT + ); diff --git a/packages/server/scripts/integrations/pg-json/reset.sh b/packages/server/scripts/integrations/pg-json/reset.sh new file mode 100755 index 0000000000..32778bd11f --- /dev/null +++ b/packages/server/scripts/integrations/pg-json/reset.sh @@ -0,0 +1,3 @@ +#!/bin/bash +docker-compose down +docker volume prune -f diff --git a/packages/server/scripts/integrations/postgres/docker-compose.yml b/packages/server/scripts/integrations/postgres/docker-compose.yml index e2bba9f38e..4dfcb0e1ad 100644 --- a/packages/server/scripts/integrations/postgres/docker-compose.yml +++ b/packages/server/scripts/integrations/postgres/docker-compose.yml @@ -15,7 +15,7 @@ services: - ./init.sql:/docker-entrypoint-initdb.d/init.sql pgadmin: - container_name: pgadmin + container_name: pgadmin-pg image: dpage/pgadmin4 restart: always environment: diff --git a/packages/server/src/api/controllers/application.js b/packages/server/src/api/controllers/application.js index da0014c5f8..76675116c2 100644 --- a/packages/server/src/api/controllers/application.js +++ b/packages/server/src/api/controllers/application.js @@ -230,7 +230,12 @@ exports.create = async function (ctx) { const response = await db.put(newApplication, { force: true }) newApplication._rev = response.rev - await createEmptyAppPackage(ctx, newApplication) + // Only create the default home screens and layout if we aren't importing + // an app + if (useTemplate !== "true") { + await createEmptyAppPackage(ctx, newApplication) + } + /* istanbul ignore next */ if (!env.isTest()) { await createApp(appId) diff --git a/packages/server/src/api/controllers/permission.js b/packages/server/src/api/controllers/permission.js index e269f8c41d..6c02663649 100644 --- a/packages/server/src/api/controllers/permission.js +++ b/packages/server/src/api/controllers/permission.js @@ -1,9 +1,4 @@ -const { - getBuiltinPermissions, - PermissionLevels, - isPermissionLevelHigherThanRead, - higherPermission, -} = require("@budibase/auth/permissions") +const { getBuiltinPermissions } = require("@budibase/auth/permissions") const { isBuiltin, getDBRoleID, @@ -16,6 +11,7 @@ const { CURRENTLY_SUPPORTED_LEVELS, getBasePermissions, } = require("../../utilities/security") +const { removeFromArray } = require("../../utilities") const PermissionUpdateType = { REMOVE: "remove", @@ -24,22 +20,6 @@ const PermissionUpdateType = { const SUPPORTED_LEVELS = CURRENTLY_SUPPORTED_LEVELS -// quick function to perform a bit of weird logic, make sure fetch calls -// always say a write role also has read permission -function fetchLevelPerms(permissions, level, roleId) { - if (!permissions) { - permissions = {} - } - permissions[level] = roleId - if ( - isPermissionLevelHigherThanRead(level) && - !permissions[PermissionLevels.READ] - ) { - permissions[PermissionLevels.READ] = roleId - } - return permissions -} - // utility function to stop this repetition - permissions always stored under roles async function getAllDBRoles(db) { const body = await db.allDocs( @@ -74,23 +54,31 @@ async function updatePermissionOnRole( for (let role of dbRoles) { let updated = false const rolePermissions = role.permissions ? role.permissions : {} + // make sure its an array, also handle migrating + if ( + !rolePermissions[resourceId] || + !Array.isArray(rolePermissions[resourceId]) + ) { + rolePermissions[resourceId] = + typeof rolePermissions[resourceId] === "string" + ? [rolePermissions[resourceId]] + : [] + } // handle the removal/updating the role which has this permission first // the updating (role._id !== dbRoleId) is required because a resource/level can // only be permitted in a single role (this reduces hierarchy confusion and simplifies // the general UI for this, rather than needing to show everywhere it is used) if ( (role._id !== dbRoleId || remove) && - rolePermissions[resourceId] === level + rolePermissions[resourceId].indexOf(level) !== -1 ) { - delete rolePermissions[resourceId] + removeFromArray(rolePermissions[resourceId], level) updated = true } // handle the adding, we're on the correct role, at it to this if (!remove && role._id === dbRoleId) { - rolePermissions[resourceId] = higherPermission( - rolePermissions[resourceId], - level - ) + const set = new Set(rolePermissions[resourceId]) + rolePermissions[resourceId] = [...set.add(level)] updated = true } // handle the update, add it to bulk docs to perform at end @@ -127,12 +115,11 @@ exports.fetch = async function (ctx) { continue } const roleId = getExternalRoleID(role._id) - for (let [resource, level] of Object.entries(role.permissions)) { - permissions[resource] = fetchLevelPerms( - permissions[resource], - level, - roleId - ) + for (let [resource, levelArr] of Object.entries(role.permissions)) { + const levels = Array.isArray(levelArr) ? [levelArr] : levelArr + const perms = {} + levels.forEach(level => (perms[level] = roleId)) + permissions[resource] = perms } } // apply the base permissions @@ -157,12 +144,13 @@ exports.getResourcePerms = async function (ctx) { for (let level of SUPPORTED_LEVELS) { // update the various roleIds in the resource permissions for (let role of roles) { - if (role.permissions && role.permissions[resourceId] === level) { - permissions = fetchLevelPerms( - permissions, - level, - getExternalRoleID(role._id) - ) + const rolePerms = role.permissions + if ( + rolePerms && + (rolePerms[resourceId] === level || + rolePerms[resourceId].indexOf(level) !== -1) + ) { + permissions[level] = getExternalRoleID(role._id) } } } diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts index b809e597e4..75c3e9b492 100644 --- a/packages/server/src/api/controllers/row/ExternalRequest.ts +++ b/packages/server/src/api/controllers/row/ExternalRequest.ts @@ -544,6 +544,9 @@ module External { extra: { idFilter: buildFilters(id || generateIdForRow(row, table), {}, table), }, + meta: { + table, + }, } // can't really use response right now const response = await makeExternalQuery(appId, json) diff --git a/packages/server/src/api/routes/application.js b/packages/server/src/api/routes/application.js index c1d39acbd5..4d67a0f4f4 100644 --- a/packages/server/src/api/routes/application.js +++ b/packages/server/src/api/routes/application.js @@ -2,11 +2,12 @@ const Router = require("@koa/router") const controller = require("../controllers/application") const authorized = require("../../middleware/authorized") const { BUILDER } = require("@budibase/auth/permissions") +const usage = require("../../middleware/usageQuota") const router = Router() router - .post("/api/applications", authorized(BUILDER), controller.create) + .post("/api/applications", authorized(BUILDER), usage, controller.create) .get("/api/applications/:appId/definition", controller.fetchAppDefinition) .get("/api/applications", controller.fetch) .get("/api/applications/:appId/appPackage", controller.fetchAppPackage) @@ -21,6 +22,11 @@ router authorized(BUILDER), controller.revertClient ) - .delete("/api/applications/:appId", authorized(BUILDER), controller.delete) + .delete( + "/api/applications/:appId", + authorized(BUILDER), + usage, + controller.delete + ) module.exports = router diff --git a/packages/server/src/api/routes/tests/datasource.spec.js b/packages/server/src/api/routes/tests/datasource.spec.js index 98a99717fd..b6d94f714d 100644 --- a/packages/server/src/api/routes/tests/datasource.spec.js +++ b/packages/server/src/api/routes/tests/datasource.spec.js @@ -94,7 +94,8 @@ describe("/datasources", () => { .expect(200) // this is mock data, can't test it expect(res.body).toBeDefined() - expect(pg.queryMock).toHaveBeenCalledWith(`select "users"."name" as "users.name", "users"."age" as "users.age" from "users" where "users"."name" ilike $1 limit $2`, ["John%", 5000]) + const expSql = `select "users"."name" as "users.name", "users"."age" as "users.age" from (select * from "users" where "users"."name" ilike $1 limit $2) as "users"` + expect(pg.queryMock).toHaveBeenCalledWith(expSql, ["John%", 5000]) }) }) diff --git a/packages/server/src/api/routes/tests/role.spec.js b/packages/server/src/api/routes/tests/role.spec.js index ad42ef180a..d74a84b2b2 100644 --- a/packages/server/src/api/routes/tests/role.spec.js +++ b/packages/server/src/api/routes/tests/role.spec.js @@ -72,7 +72,7 @@ describe("/roles", () => { .expect(200) expect(res.body.length).toBeGreaterThan(0) const power = res.body.find(role => role._id === BUILTIN_ROLE_IDS.POWER) - expect(power.permissions[table._id]).toEqual("read") + expect(power.permissions[table._id]).toEqual(["read"]) }) }) diff --git a/packages/server/src/api/routes/user.js b/packages/server/src/api/routes/user.js index b3b486fe45..d171870215 100644 --- a/packages/server/src/api/routes/user.js +++ b/packages/server/src/api/routes/user.js @@ -5,7 +5,6 @@ const { PermissionLevels, PermissionTypes, } = require("@budibase/auth/permissions") -const usage = require("../../middleware/usageQuota") const router = Router() @@ -28,13 +27,11 @@ router .post( "/api/users/metadata/self", authorized(PermissionTypes.USER, PermissionLevels.WRITE), - usage, controller.updateSelfMetadata ) .delete( "/api/users/metadata/:id", authorized(PermissionTypes.USER, PermissionLevels.WRITE), - usage, controller.destroyMetadata ) diff --git a/packages/server/src/api/routes/view.js b/packages/server/src/api/routes/view.js index 7d390805c6..b72fe1ac26 100644 --- a/packages/server/src/api/routes/view.js +++ b/packages/server/src/api/routes/view.js @@ -8,7 +8,6 @@ const { PermissionTypes, PermissionLevels, } = require("@budibase/auth/permissions") -const usage = require("../../middleware/usageQuota") const router = Router() @@ -25,9 +24,8 @@ router "/api/views/:viewName", paramResource("viewName"), authorized(BUILDER), - usage, viewController.destroy ) - .post("/api/views", authorized(BUILDER), usage, viewController.save) + .post("/api/views", authorized(BUILDER), viewController.save) module.exports = router diff --git a/packages/server/src/automations/steps/createRow.js b/packages/server/src/automations/steps/createRow.js index 9033004578..47d0b4eb99 100644 --- a/packages/server/src/automations/steps/createRow.js +++ b/packages/server/src/automations/steps/createRow.js @@ -60,7 +60,7 @@ exports.definition = { }, } -exports.run = async function ({ inputs, appId, apiKey, emitter }) { +exports.run = async function ({ inputs, appId, emitter }) { if (inputs.row == null || inputs.row.tableId == null) { return { success: false, @@ -84,7 +84,7 @@ exports.run = async function ({ inputs, appId, apiKey, emitter }) { inputs.row ) if (env.USE_QUOTAS) { - await usage.update(apiKey, usage.Properties.ROW, 1) + await usage.update(usage.Properties.ROW, 1) } await rowController.save(ctx) return { diff --git a/packages/server/src/automations/steps/deleteRow.js b/packages/server/src/automations/steps/deleteRow.js index 0f9648cc51..225f00c5df 100644 --- a/packages/server/src/automations/steps/deleteRow.js +++ b/packages/server/src/automations/steps/deleteRow.js @@ -52,7 +52,7 @@ exports.definition = { }, } -exports.run = async function ({ inputs, appId, apiKey, emitter }) { +exports.run = async function ({ inputs, appId, emitter }) { if (inputs.id == null || inputs.revision == null) { return { success: false, @@ -74,7 +74,7 @@ exports.run = async function ({ inputs, appId, apiKey, emitter }) { try { if (env.isProd()) { - await usage.update(apiKey, usage.Properties.ROW, -1) + await usage.update(usage.Properties.ROW, -1) } await rowController.destroy(ctx) return { diff --git a/packages/server/src/automations/steps/sendSmtpEmail.js b/packages/server/src/automations/steps/sendSmtpEmail.js index 9e4b5a6a3c..07a3059215 100644 --- a/packages/server/src/automations/steps/sendSmtpEmail.js +++ b/packages/server/src/automations/steps/sendSmtpEmail.js @@ -53,7 +53,7 @@ exports.run = async function ({ inputs }) { contents = "

No content

" } try { - let response = await sendSmtpEmail(to, from, subject, contents) + let response = await sendSmtpEmail(to, from, subject, contents, true) return { success: true, response, diff --git a/packages/server/src/automations/tests/automation.spec.js b/packages/server/src/automations/tests/automation.spec.js index 83b7b81a75..9444995ca1 100644 --- a/packages/server/src/automations/tests/automation.spec.js +++ b/packages/server/src/automations/tests/automation.spec.js @@ -13,8 +13,6 @@ const { makePartial } = require("../../tests/utilities") const { cleanInputValues } = require("../automationUtils") const setup = require("./utilities") -usageQuota.getAPIKey.mockReturnValue({ apiKey: "test" }) - describe("Run through some parts of the automations system", () => { let config = setup.getConfig() diff --git a/packages/server/src/automations/tests/createRow.spec.js b/packages/server/src/automations/tests/createRow.spec.js index 1004711d87..a04fc7aad4 100644 --- a/packages/server/src/automations/tests/createRow.spec.js +++ b/packages/server/src/automations/tests/createRow.spec.js @@ -46,7 +46,7 @@ describe("test the create row action", () => { await setup.runStep(setup.actions.CREATE_ROW.stepId, { row }) - expect(usageQuota.update).toHaveBeenCalledWith(setup.apiKey, "rows", 1) + expect(usageQuota.update).toHaveBeenCalledWith("rows", 1) }) }) diff --git a/packages/server/src/automations/tests/deleteRow.spec.js b/packages/server/src/automations/tests/deleteRow.spec.js index a3d73d3bf6..21246f22d0 100644 --- a/packages/server/src/automations/tests/deleteRow.spec.js +++ b/packages/server/src/automations/tests/deleteRow.spec.js @@ -37,7 +37,7 @@ describe("test the delete row action", () => { it("check usage quota attempts", async () => { await setup.runInProd(async () => { await setup.runStep(setup.actions.DELETE_ROW.stepId, inputs) - expect(usageQuota.update).toHaveBeenCalledWith(setup.apiKey, "rows", -1) + expect(usageQuota.update).toHaveBeenCalledWith("rows", -1) }) }) diff --git a/packages/server/src/automations/thread.js b/packages/server/src/automations/thread.js index a3e81a2274..ef12494165 100644 --- a/packages/server/src/automations/thread.js +++ b/packages/server/src/automations/thread.js @@ -4,8 +4,10 @@ const AutomationEmitter = require("../events/AutomationEmitter") const { processObject } = require("@budibase/string-templates") const { DEFAULT_TENANT_ID } = require("@budibase/auth").constants const CouchDB = require("../db") -const { DocumentTypes } = require("../db/utils") +const { DocumentTypes, isDevAppID } = require("../db/utils") const { doInTenant } = require("@budibase/auth/tenancy") +const env = require("../environment") +const usage = require("../utilities/usageQuota") const FILTER_STEP_ID = actions.ACTION_DEFINITIONS.FILTER.stepId @@ -80,7 +82,6 @@ class Orchestrator { return stepFn({ inputs: step.inputs, appId: this._appId, - apiKey: automation.apiKey, emitter: this._emitter, context: this._context, }) @@ -95,6 +96,11 @@ class Orchestrator { return err } } + + // Increment quota for automation runs + if (!env.SELF_HOSTED && !isDevAppID(this._appId)) { + usage.update(usage.Properties.AUTOMATION, 1) + } return this.executionOutput } } diff --git a/packages/server/src/definitions/datasource.ts b/packages/server/src/definitions/datasource.ts index 48fd24e1cf..2daef8eda7 100644 --- a/packages/server/src/definitions/datasource.ts +++ b/packages/server/src/definitions/datasource.ts @@ -1,3 +1,5 @@ +import { Table } from "./common" + export enum Operation { CREATE = "CREATE", READ = "READ", @@ -136,6 +138,9 @@ export interface QueryJson { sort?: SortJson paginate?: PaginationJson body?: object + meta?: { + table?: Table + } extra?: { idFilter?: SearchFilters } diff --git a/packages/server/src/integrations/base/sql.ts b/packages/server/src/integrations/base/sql.ts index b59bac5a5a..c5e9bdb0bb 100644 --- a/packages/server/src/integrations/base/sql.ts +++ b/packages/server/src/integrations/base/sql.ts @@ -1,7 +1,5 @@ import { Knex, knex } from "knex" const BASE_LIMIT = 5000 -// if requesting a single row then need to up the limit for the sake of joins -const SINGLE_ROW_LIMIT = 100 import { QueryJson, SearchFilters, @@ -146,46 +144,48 @@ function buildCreate( function buildRead(knex: Knex, json: QueryJson, limit: number): KnexQuery { let { endpoint, resource, filters, sort, paginate, relationships } = json const tableName = endpoint.entityId - let query: KnexQuery = knex(tableName) // select all if not specified if (!resource) { resource = { fields: [] } } + let selectStatement: string | string[] = "*" // handle select if (resource.fields && resource.fields.length > 0) { // select the resources as the format "table.columnName" - this is what is provided // by the resource builder further up - query = query.select(resource.fields.map(field => `${field} as ${field}`)) - } else { - query = query.select("*") + selectStatement = resource.fields.map(field => `${field} as ${field}`) + } + let foundLimit = limit || BASE_LIMIT + // handle pagination + let foundOffset: number | null = null + if (paginate && paginate.page && paginate.limit) { + // @ts-ignore + const page = paginate.page <= 1 ? 0 : paginate.page - 1 + const offset = page * paginate.limit + foundLimit = paginate.limit + foundOffset = offset + } else if (paginate && paginate.limit) { + foundLimit = paginate.limit + } + // start building the query + let query: KnexQuery = knex(tableName).limit(foundLimit) + if (foundOffset) { + query = query.offset(foundOffset) } - // handle where - query = addFilters(tableName, query, filters) - // handle join - query = addRelationships(query, tableName, relationships) - // handle sorting if (sort) { for (let [key, value] of Object.entries(sort)) { const direction = value === SortDirection.ASCENDING ? "asc" : "desc" query = query.orderBy(key, direction) } } - let foundLimit = limit || BASE_LIMIT - // handle pagination - if (paginate && paginate.page && paginate.limit) { + query = addFilters(tableName, query, filters) + // @ts-ignore + let preQuery: KnexQuery = knex({ // @ts-ignore - const page = paginate.page <= 1 ? 0 : paginate.page - 1 - const offset = page * paginate.limit - foundLimit = paginate.limit - query = query.offset(offset) - } else if (paginate && paginate.limit) { - foundLimit = paginate.limit - } - if (foundLimit === 1) { - foundLimit = SINGLE_ROW_LIMIT - } - query = query.limit(foundLimit) - return query + [tableName]: query, + }).select(selectStatement) + // handle joins + return addRelationships(preQuery, tableName, relationships) } function buildUpdate( diff --git a/packages/server/src/integrations/mysql.ts b/packages/server/src/integrations/mysql.ts index c5db35ed2a..c17cca0745 100644 --- a/packages/server/src/integrations/mysql.ts +++ b/packages/server/src/integrations/mysql.ts @@ -12,7 +12,11 @@ import { getSqlQuery } from "./utils" module MySQLModule { const mysql = require("mysql") const Sql = require("./base/sql") - const { buildExternalTableId, convertType, copyExistingPropsOver } = require("./utils") + const { + buildExternalTableId, + convertType, + copyExistingPropsOver, + } = require("./utils") const { FieldTypes } = require("../constants") interface MySQLConfig { @@ -104,7 +108,7 @@ module MySQLModule { client: any, query: SqlQuery, connect: boolean = true - ): Promise { + ): Promise { // Node MySQL is callback based, so we must wrap our call in a promise return new Promise((resolve, reject) => { if (connect) { @@ -238,6 +242,23 @@ module MySQLModule { return internalQuery(this.client, input, false) } + // when creating if an ID has been inserted need to make sure + // the id filter is enriched with it before trying to retrieve the row + checkLookupKeys(results: any, json: QueryJson) { + if (!results?.insertId || !json.meta?.table || !json.meta.table.primary) { + return json + } + const primaryKey = json.meta.table.primary?.[0] + json.extra = { + idFilter: { + equal: { + [primaryKey]: results.insertId, + }, + }, + } + return json + } + async query(json: QueryJson) { const operation = this._operation(json) this.client.connect() @@ -250,7 +271,7 @@ module MySQLModule { const results = await internalQuery(this.client, input, false) // same as delete, manage returning if (operation === Operation.CREATE || operation === Operation.UPDATE) { - row = this.getReturningRow(json) + row = this.getReturningRow(this.checkLookupKeys(results, json)) } this.client.end() if (operation !== Operation.READ) { diff --git a/packages/server/src/integrations/postgres.ts b/packages/server/src/integrations/postgres.ts index 23a8685648..db81e183d1 100644 --- a/packages/server/src/integrations/postgres.ts +++ b/packages/server/src/integrations/postgres.ts @@ -12,7 +12,14 @@ module PostgresModule { const { Pool } = require("pg") const Sql = require("./base/sql") const { FieldTypes } = require("../constants") - const { buildExternalTableId, convertType, copyExistingPropsOver } = require("./utils") + const { + buildExternalTableId, + convertType, + copyExistingPropsOver, + } = require("./utils") + const { escapeDangerousCharacters } = require("../utilities") + + const JSON_REGEX = /'{.*}'::json/s interface PostgresConfig { host: string @@ -94,6 +101,17 @@ module PostgresModule { } async function internalQuery(client: any, query: SqlQuery) { + // need to handle a specific issue with json data types in postgres, + // new lines inside the JSON data will break it + if (query && query.sql) { + const matches = query.sql.match(JSON_REGEX) + if (matches && matches.length > 0) { + for (let match of matches) { + const escaped = escapeDangerousCharacters(match) + query.sql = query.sql.replace(match, escaped) + } + } + } try { return await client.query(query.sql, query.bindings || []) } catch (err) { @@ -179,10 +197,16 @@ module PostgresModule { } const type: string = convertType(column.data_type, TYPE_MAP) - const identity = !!(column.identity_generation || column.identity_start || column.identity_increment) - const hasDefault = typeof column.column_default === "string" && + const identity = !!( + column.identity_generation || + column.identity_start || + column.identity_increment + ) + const hasDefault = + typeof column.column_default === "string" && column.column_default.startsWith("nextval") - const isGenerated = column.is_generated && column.is_generated !== "NEVER" + const isGenerated = + column.is_generated && column.is_generated !== "NEVER" const isAuto: boolean = hasDefault || identity || isGenerated tables[tableName].schema[columnName] = { autocolumn: isAuto, diff --git a/packages/server/src/integrations/tests/sql.spec.js b/packages/server/src/integrations/tests/sql.spec.js index fa8bcd1d86..64cdda215f 100644 --- a/packages/server/src/integrations/tests/sql.spec.js +++ b/packages/server/src/integrations/tests/sql.spec.js @@ -57,7 +57,7 @@ describe("SQL query builder", () => { const query = sql._query(generateReadJson()) expect(query).toEqual({ bindings: [limit], - sql: `select * from "${TABLE_NAME}" limit $1` + sql: `select * from (select * from "${TABLE_NAME}" limit $1) as "${TABLE_NAME}"` }) }) @@ -68,7 +68,7 @@ describe("SQL query builder", () => { })) expect(query).toEqual({ bindings: [limit], - sql: `select "${TABLE_NAME}"."name" as "${nameProp}", "${TABLE_NAME}"."age" as "${ageProp}" from "${TABLE_NAME}" limit $1` + sql: `select "${TABLE_NAME}"."name" as "${nameProp}", "${TABLE_NAME}"."age" as "${ageProp}" from (select * from "${TABLE_NAME}" limit $1) as "${TABLE_NAME}"` }) }) @@ -82,7 +82,7 @@ describe("SQL query builder", () => { })) expect(query).toEqual({ bindings: ["John%", limit], - sql: `select * from "${TABLE_NAME}" where "${TABLE_NAME}"."name" ilike $1 limit $2` + sql: `select * from (select * from "${TABLE_NAME}" where "${TABLE_NAME}"."name" ilike $1 limit $2) as "${TABLE_NAME}"` }) }) @@ -99,7 +99,7 @@ describe("SQL query builder", () => { })) expect(query).toEqual({ bindings: [2, 10, limit], - sql: `select * from "${TABLE_NAME}" where "${TABLE_NAME}"."age" between $1 and $2 limit $3` + sql: `select * from (select * from "${TABLE_NAME}" where "${TABLE_NAME}"."age" between $1 and $2 limit $3) as "${TABLE_NAME}"` }) }) @@ -115,7 +115,7 @@ describe("SQL query builder", () => { })) expect(query).toEqual({ bindings: [10, "John", limit], - sql: `select * from "${TABLE_NAME}" where ("${TABLE_NAME}"."age" = $1) or ("${TABLE_NAME}"."name" = $2) limit $3` + sql: `select * from (select * from "${TABLE_NAME}" where ("${TABLE_NAME}"."age" = $1) or ("${TABLE_NAME}"."name" = $2) limit $3) as "${TABLE_NAME}"` }) }) @@ -160,7 +160,7 @@ describe("SQL query builder", () => { const query = new Sql("mssql", 10)._query(generateReadJson()) expect(query).toEqual({ bindings: [10], - sql: `select top (@p0) * from [${TABLE_NAME}]` + sql: `select * from (select top (@p0) * from [${TABLE_NAME}]) as [${TABLE_NAME}]` }) }) @@ -168,7 +168,7 @@ describe("SQL query builder", () => { const query = new Sql("mysql", 10)._query(generateReadJson()) expect(query).toEqual({ bindings: [10], - sql: `select * from \`${TABLE_NAME}\` limit ?` + sql: `select * from (select * from \`${TABLE_NAME}\` limit ?) as \`${TABLE_NAME}\`` }) }) }) diff --git a/packages/server/src/integrations/utils.ts b/packages/server/src/integrations/utils.ts index 82c35bc2d9..6e3dc6f684 100644 --- a/packages/server/src/integrations/utils.ts +++ b/packages/server/src/integrations/utils.ts @@ -84,7 +84,11 @@ export function isIsoDateString(str: string) { } // add the existing relationships from the entities if they exist, to prevent them from being overridden -export function copyExistingPropsOver(tableName: string, tables: { [key: string]: any }, entities: { [key: string]: any }) { +export function copyExistingPropsOver( + tableName: string, + tables: { [key: string]: any }, + entities: { [key: string]: any } +) { if (entities && entities[tableName]) { if (entities[tableName].primaryDisplay) { tables[tableName].primaryDisplay = entities[tableName].primaryDisplay diff --git a/packages/server/src/middleware/tests/usageQuota.spec.js b/packages/server/src/middleware/tests/usageQuota.spec.js index 97d9c7794a..d828f2ca60 100644 --- a/packages/server/src/middleware/tests/usageQuota.spec.js +++ b/packages/server/src/middleware/tests/usageQuota.spec.js @@ -39,7 +39,7 @@ class TestConfiguration { if (bool) { env.isDev = () => false env.isProd = () => true - this.ctx.auth = { apiKey: "test" } + this.ctx.user = { tenantId: "test" } } else { env.isDev = () => true env.isProd = () => false @@ -114,7 +114,7 @@ describe("usageQuota middleware", () => { await config.executeMiddleware() - expect(usageQuota.update).toHaveBeenCalledWith("test", "rows", 1) + expect(usageQuota.update).toHaveBeenCalledWith("rows", 1) expect(config.next).toHaveBeenCalled() }) @@ -131,7 +131,7 @@ describe("usageQuota middleware", () => { ]) await config.executeMiddleware() - expect(usageQuota.update).toHaveBeenCalledWith("test", "storage", 10100) + expect(usageQuota.update).toHaveBeenCalledWith("storage", 10100) expect(config.next).toHaveBeenCalled() }) }) \ No newline at end of file diff --git a/packages/server/src/middleware/usageQuota.js b/packages/server/src/middleware/usageQuota.js index 4647878721..3a244ef5bc 100644 --- a/packages/server/src/middleware/usageQuota.js +++ b/packages/server/src/middleware/usageQuota.js @@ -13,6 +13,7 @@ const DOMAIN_MAP = { upload: usageQuota.Properties.UPLOAD, views: usageQuota.Properties.VIEW, users: usageQuota.Properties.USER, + applications: usageQuota.Properties.APPS, // this will not be updated by endpoint calls // instead it will be updated by triggerInfo automationRuns: usageQuota.Properties.AUTOMATION, @@ -57,9 +58,9 @@ module.exports = async (ctx, next) => { usage = files.map(file => file.size).reduce((total, size) => total + size) } try { - await usageQuota.update(ctx.auth.apiKey, property, usage) + await usageQuota.update(property, usage) return next() } catch (err) { - ctx.throw(403, err) + ctx.throw(400, err) } } diff --git a/packages/server/src/utilities/index.js b/packages/server/src/utilities/index.js index a81f9ddcf5..b16a687fe5 100644 --- a/packages/server/src/utilities/index.js +++ b/packages/server/src/utilities/index.js @@ -10,6 +10,14 @@ exports.wait = ms => new Promise(resolve => setTimeout(resolve, ms)) exports.isDev = env.isDev +exports.removeFromArray = (array, element) => { + const index = array.indexOf(element) + if (index !== -1) { + array.splice(index, 1) + } + return array +} + /** * Makes sure that a URL has the correct number of slashes, while maintaining the * http(s):// double slashes. @@ -106,3 +114,13 @@ exports.deleteEntityMetadata = async (appId, type, entityId) => { await db.remove(id, rev) } } + +exports.escapeDangerousCharacters = string => { + return string + .replace(/[\\]/g, "\\\\") + .replace(/[\b]/g, "\\b") + .replace(/[\f]/g, "\\f") + .replace(/[\n]/g, "\\n") + .replace(/[\r]/g, "\\r") + .replace(/[\t]/g, "\\t") +} diff --git a/packages/server/src/utilities/usageQuota.js b/packages/server/src/utilities/usageQuota.js index bfe71a4093..80fddb8303 100644 --- a/packages/server/src/utilities/usageQuota.js +++ b/packages/server/src/utilities/usageQuota.js @@ -1,41 +1,9 @@ const env = require("../environment") -const { apiKeyTable } = require("../db/dynamoClient") - -const DEFAULT_USAGE = { - rows: 0, - storage: 0, - views: 0, - automationRuns: 0, - users: 0, -} - -const DEFAULT_PLAN = { - rows: 1000, - // 1 GB - storage: 8589934592, - views: 10, - automationRuns: 100, - users: 10000, -} - -function buildUpdateParams(key, property, usage) { - return { - primary: key, - condition: - "attribute_exists(#quota) AND attribute_exists(#limits) AND #quota.#prop < #limits.#prop AND #quotaReset > :now", - expression: "ADD #quota.#prop :usage", - names: { - "#quota": "usageQuota", - "#prop": property, - "#limits": "usageLimits", - "#quotaReset": "quotaReset", - }, - values: { - ":usage": usage, - ":now": Date.now(), - }, - } -} +const { getGlobalDB } = require("@budibase/auth/tenancy") +const { + StaticDatabases, + generateNewUsageQuotaDoc, +} = require("@budibase/auth/db") function getNewQuotaReset() { return Date.now() + 2592000000 @@ -47,59 +15,59 @@ exports.Properties = { VIEW: "views", USER: "users", AUTOMATION: "automationRuns", + APPS: "apps", + EMAILS: "emails", } -exports.getAPIKey = async appId => { - if (!env.USE_QUOTAS) { - return { apiKey: null } +async function getUsageQuotaDoc(db) { + let quota + try { + quota = await db.get(StaticDatabases.PLATFORM_INFO.docs.usageQuota) + } catch (err) { + // doc doesn't exist. Create it + quota = await db.post(generateNewUsageQuotaDoc()) } - return apiKeyTable.get({ primary: appId }) + + return quota } /** - * Given a specified API key this will add to the usage object for the specified property. - * @param {string} apiKey The API key which is to be updated. + * Given a specified tenantId this will add to the usage object for the specified property. * @param {string} property The property which is to be added to (within the nested usageQuota object). * @param {number} usage The amount (this can be negative) to adjust the number by. * @returns {Promise} When this completes the API key will now be up to date - the quota period may have * also been reset after this call. */ -exports.update = async (apiKey, property, usage) => { +exports.update = async (property, usage) => { if (!env.USE_QUOTAS) { return } + try { - await apiKeyTable.update(buildUpdateParams(apiKey, property, usage)) - } catch (err) { - // conditional check means the condition failed, need to check why - if (err.code === "ConditionalCheckFailedException") { - // get the API key so we can check it - const keyObj = await apiKeyTable.get({ primary: apiKey }) - // the usage quota or usage limits didn't exist - if (keyObj && (keyObj.usageQuota == null || keyObj.usageLimits == null)) { - keyObj.usageQuota = - keyObj.usageQuota == null ? DEFAULT_USAGE : keyObj.usageQuota - keyObj.usageLimits = - keyObj.usageLimits == null ? DEFAULT_PLAN : keyObj.usageLimits - keyObj.quotaReset = getNewQuotaReset() - await apiKeyTable.put({ item: keyObj }) - return - } - // we have in fact breached the reset period - else if (keyObj && keyObj.quotaReset <= Date.now()) { - // update the quota reset period and reset the values for all properties - keyObj.quotaReset = getNewQuotaReset() - for (let prop of Object.keys(keyObj.usageQuota)) { - if (prop === property) { - keyObj.usageQuota[prop] = usage > 0 ? usage : 0 - } else { - keyObj.usageQuota[prop] = 0 - } - } - await apiKeyTable.put({ item: keyObj }) - return + const db = getGlobalDB() + const quota = await getUsageQuotaDoc(db) + + // Check if the quota needs reset + if (Date.now() >= quota.quotaReset) { + quota.quotaReset = getNewQuotaReset() + for (let prop of Object.keys(quota.usageQuota)) { + quota.usageQuota[prop] = 0 } } + + // increment the quota + quota.usageQuota[property] += usage + + if (quota.usageQuota[property] >= quota.usageLimits[property]) { + throw new Error( + `You have exceeded your usage quota of ${quota.usageLimits[property]} ${property}.` + ) + } + + // update the usage quotas + await db.put(quota) + } catch (err) { + console.error(`Error updating usage quotas for ${property}`, err) throw err } } diff --git a/packages/server/src/utilities/workerRequests.js b/packages/server/src/utilities/workerRequests.js index 377658084f..2ace265ca0 100644 --- a/packages/server/src/utilities/workerRequests.js +++ b/packages/server/src/utilities/workerRequests.js @@ -34,7 +34,7 @@ function request(ctx, request) { exports.request = request // have to pass in the tenant ID as this could be coming from an automation -exports.sendSmtpEmail = async (to, from, subject, contents) => { +exports.sendSmtpEmail = async (to, from, subject, contents, automation) => { // tenant ID will be set in header const response = await fetch( checkSlashesInUrl(env.WORKER_URL + `/api/global/email/send`), @@ -46,6 +46,7 @@ exports.sendSmtpEmail = async (to, from, subject, contents) => { contents, subject, purpose: "custom", + automation, }, }) ) diff --git a/packages/string-templates/package.json b/packages/string-templates/package.json index d57515dbcc..bae80e4ca6 100644 --- a/packages/string-templates/package.json +++ b/packages/string-templates/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/string-templates", - "version": "0.9.140-alpha.8", + "version": "0.9.143-alpha.4", "description": "Handlebars wrapper for Budibase templating.", "main": "src/index.cjs", "module": "dist/bundle.mjs", diff --git a/packages/worker/package.json b/packages/worker/package.json index 81b1129918..29e8137932 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/worker", "email": "hi@budibase.com", - "version": "0.9.140-alpha.8", + "version": "0.9.143-alpha.4", "description": "Budibase background service", "main": "src/index.js", "repository": { @@ -25,8 +25,8 @@ "author": "Budibase", "license": "AGPL-3.0-or-later", "dependencies": { - "@budibase/auth": "^0.9.140-alpha.8", - "@budibase/string-templates": "^0.9.140-alpha.8", + "@budibase/auth": "^0.9.143-alpha.4", + "@budibase/string-templates": "^0.9.143-alpha.4", "@koa/router": "^8.0.0", "@techpass/passport-openidconnect": "^0.3.0", "aws-sdk": "^2.811.0", diff --git a/packages/worker/src/api/controllers/global/configs.js b/packages/worker/src/api/controllers/global/configs.js index aa83fd695f..c0c300e4db 100644 --- a/packages/worker/src/api/controllers/global/configs.js +++ b/packages/worker/src/api/controllers/global/configs.js @@ -10,6 +10,7 @@ const email = require("../../../utilities/email") const { upload, ObjectStoreBuckets } = require("@budibase/auth").objectStore const CouchDB = require("../../../db") const { getGlobalDB } = require("@budibase/auth/tenancy") +const env = require("../../../environment") exports.save = async function (ctx) { const db = getGlobalDB() @@ -174,7 +175,13 @@ exports.upload = async function (ctx) { const file = ctx.request.files.file const { type, name } = ctx.params - const bucket = ObjectStoreBuckets.GLOBAL + let bucket + if (env.SELF_HOSTED) { + bucket = ObjectStoreBuckets.GLOBAL + } else { + bucket = ObjectStoreBuckets.GLOBAL_CLOUD + } + const key = `${type}/${name}` await upload({ bucket, diff --git a/packages/worker/src/api/controllers/global/email.js b/packages/worker/src/api/controllers/global/email.js index 57b78a6d7a..e194a30862 100644 --- a/packages/worker/src/api/controllers/global/email.js +++ b/packages/worker/src/api/controllers/global/email.js @@ -2,8 +2,16 @@ const { sendEmail } = require("../../../utilities/email") const { getGlobalDB } = require("@budibase/auth/tenancy") exports.sendEmail = async ctx => { - let { workspaceId, email, userId, purpose, contents, from, subject } = - ctx.request.body + let { + workspaceId, + email, + userId, + purpose, + contents, + from, + subject, + automation, + } = ctx.request.body let user if (userId) { const db = getGlobalDB() @@ -15,6 +23,7 @@ exports.sendEmail = async ctx => { contents, from, subject, + automation, }) ctx.body = { ...response, diff --git a/packages/worker/src/api/controllers/global/users.js b/packages/worker/src/api/controllers/global/users.js index 1375240f34..1d3f38698b 100644 --- a/packages/worker/src/api/controllers/global/users.js +++ b/packages/worker/src/api/controllers/global/users.js @@ -1,8 +1,8 @@ const { generateGlobalUserID, getGlobalUserParams, - StaticDatabases, + generateNewUsageQuotaDoc, } = require("@budibase/auth/db") const { hash, getGlobalUserByEmail } = require("@budibase/auth").utils const { UserStatus, EmailTemplatePurpose } = require("../../../constants") @@ -18,6 +18,7 @@ const { tryAddTenant, updateTenantId, } = require("@budibase/auth/tenancy") +const env = require("../../../environment") const PLATFORM_INFO_DB = StaticDatabases.PLATFORM_INFO.name @@ -139,6 +140,11 @@ exports.adminUser = async ctx => { }) ) + // write usage quotas for cloud + if (!env.SELF_HOSTED) { + await db.post(generateNewUsageQuotaDoc()) + } + if (response.rows.some(row => row.doc.admin)) { ctx.throw( 403, diff --git a/packages/worker/src/environment.js b/packages/worker/src/environment.js index 646536f292..28ab4e2e69 100644 --- a/packages/worker/src/environment.js +++ b/packages/worker/src/environment.js @@ -33,6 +33,12 @@ module.exports = { INTERNAL_API_KEY: process.env.INTERNAL_API_KEY, MULTI_TENANCY: process.env.MULTI_TENANCY, ACCOUNT_PORTAL_URL: process.env.ACCOUNT_PORTAL_URL, + SMTP_FALLBACK_ENABLED: process.env.SMTP_FALLBACK_ENABLED, + SMTP_USER: process.env.SMTP_USER, + SMTP_PASSWORD: process.env.SMTP_PASSWORD, + SMTP_HOST: process.env.SMTP_HOST, + SMTP_PORT: process.env.SMTP_PORT, + SMTP_FROM_ADDRESS: process.env.SMTP_FROM_ADDRESS, _set(key, value) { process.env[key] = value module.exports[key] = value diff --git a/packages/worker/src/utilities/email.js b/packages/worker/src/utilities/email.js index d22933ef36..14c836952e 100644 --- a/packages/worker/src/utilities/email.js +++ b/packages/worker/src/utilities/email.js @@ -1,4 +1,5 @@ const nodemailer = require("nodemailer") +const env = require("../environment") const { getScopedConfig } = require("@budibase/auth/db") const { EmailTemplatePurpose, TemplateTypes, Configs } = require("../constants") const { getTemplateByPurpose } = require("../constants/templates") @@ -101,16 +102,35 @@ async function buildEmail(purpose, email, context, { user, contents } = {}) { * Utility function for finding most valid SMTP configuration. * @param {object} db The CouchDB database which is to be looked up within. * @param {string|null} workspaceId If using finer grain control of configs a workspace can be used. + * @param {boolean|null} automation Whether or not the configuration is being fetched for an email automation. * @return {Promise} returns the SMTP configuration if it exists */ -async function getSmtpConfiguration(db, workspaceId = null) { +async function getSmtpConfiguration(db, workspaceId = null, automation) { const params = { type: Configs.SMTP, } if (workspaceId) { params.workspace = workspaceId } - return getScopedConfig(db, params) + + const customConfig = getScopedConfig(db, params) + + if (customConfig) { + return customConfig + } + + // Use an SMTP fallback configuration from env variables + if (!automation && env.SMTP_FALLBACK_ENABLED) { + return { + port: env.SMTP_PORT, + host: env.SMTP_HOST, + secure: false, + auth: { + user: env.SMTP_USER, + pass: env.SMTP_PASSWORD, + }, + } + } } /** @@ -118,8 +138,8 @@ async function getSmtpConfiguration(db, workspaceId = null) { * @return {Promise} returns true if there is a configuration that can be used. */ exports.isEmailConfigured = async (workspaceId = null) => { - // when "testing" simply return true - if (TEST_MODE) { + // when "testing" or smtp fallback is enabled simply return true + if (TEST_MODE || env.SMTP_FALLBACK_ENABLED) { return true } const db = getGlobalDB() @@ -138,16 +158,17 @@ exports.isEmailConfigured = async (workspaceId = null) => { * @param {string|undefined} contents If sending a custom email then can supply contents which will be added to it. * @param {string|undefined} subject A custom subject can be specified if the config one is not desired. * @param {object|undefined} info Pass in a structure of information to be stored alongside the invitation. + * @param {boolean|undefined} disableFallback Prevent email being sent from SMTP fallback to avoid spam. * @return {Promise} returns details about the attempt to send email, e.g. if it is successful; based on * nodemailer response. */ exports.sendEmail = async ( email, purpose, - { workspaceId, user, from, contents, subject, info } = {} + { workspaceId, user, from, contents, subject, info, automation } = {} ) => { const db = getGlobalDB() - let config = (await getSmtpConfiguration(db, workspaceId)) || {} + let config = (await getSmtpConfiguration(db, workspaceId, automation)) || {} if (Object.keys(config).length === 0 && !TEST_MODE) { throw "Unable to find SMTP configuration." }