Merge remote-tracking branch 'origin/develop' into feature/datasource-wizard

This commit is contained in:
Peter Clement 2021-09-28 14:26:25 +01:00
commit f45638d874
58 changed files with 505 additions and 283 deletions

View File

@ -1,5 +1,5 @@
{
"version": "0.9.140-alpha.8",
"version": "0.9.143-alpha.4",
"npmClient": "yarn",
"packages": [
"packages/*"

View File

@ -1,6 +1,6 @@
{
"name": "@budibase/auth",
"version": "0.9.140-alpha.8",
"version": "0.9.143-alpha.4",
"description": "Authentication middlewares for budibase builder and apps",
"main": "src/index.js",
"author": "Budibase",

View File

@ -12,6 +12,7 @@ exports.StaticDatabases = {
name: "global-info",
docs: {
tenants: "tenants",
usageQuota: "usage_quota",
},
},
}

View File

@ -368,8 +368,33 @@ async function getScopedConfig(db, params) {
return configDoc && configDoc.config ? configDoc.config : configDoc
}
function generateNewUsageQuotaDoc() {
return {
_id: StaticDatabases.PLATFORM_INFO.docs.usageQuota,
quotaReset: Date.now() + 2592000000,
usageQuota: {
automationRuns: 0,
rows: 0,
storage: 0,
apps: 0,
users: 0,
views: 0,
emails: 0,
},
usageLimits: {
automationRuns: 1000,
rows: 4000,
apps: 4,
storage: 1000,
users: 10,
emails: 50,
},
}
}
exports.Replication = Replication
exports.getScopedConfig = getScopedConfig
exports.generateConfigID = generateConfigID
exports.getConfigParams = getConfigParams
exports.getScopedFullConfig = getScopedFullConfig
exports.generateNewUsageQuotaDoc = generateNewUsageQuotaDoc

View File

@ -139,8 +139,7 @@ exports.doesHaveResourcePermission = (
// set foundSub to not subResourceId, incase there is no subResource
let foundMain = false,
foundSub = false
for (let [resource, level] of Object.entries(permissions)) {
const levels = getAllowedLevels(level)
for (let [resource, levels] of Object.entries(permissions)) {
if (resource === resourceId && levels.indexOf(permLevel) !== -1) {
foundMain = true
}
@ -177,10 +176,6 @@ exports.doesHaveBasePermission = (permType, permLevel, permissionIds) => {
return false
}
exports.higherPermission = (perm1, perm2) => {
return levelToNumber(perm1) > levelToNumber(perm2) ? perm1 : perm2
}
exports.isPermissionLevelHigherThanRead = level => {
return levelToNumber(level) > 1
}

View File

@ -1,6 +1,6 @@
const { getDB } = require("../db")
const { cloneDeep } = require("lodash/fp")
const { BUILTIN_PERMISSION_IDS, higherPermission } = require("./permissions")
const { BUILTIN_PERMISSION_IDS } = require("./permissions")
const {
generateRoleID,
getRoleParams,
@ -193,8 +193,17 @@ exports.getUserPermissions = async (appId, userRoleId) => {
const permissions = {}
for (let role of rolesHierarchy) {
if (role.permissions) {
for (let [resource, level] of Object.entries(role.permissions)) {
permissions[resource] = higherPermission(permissions[resource], level)
for (let [resource, levels] of Object.entries(role.permissions)) {
if (!permissions[resource]) {
permissions[resource] = []
}
const permsSet = new Set(permissions[resource])
if (Array.isArray(levels)) {
levels.forEach(level => permsSet.add(level))
} else {
permsSet.add(levels)
}
permissions[resource] = [...permsSet]
}
}
}

View File

@ -1,7 +1,7 @@
{
"name": "@budibase/bbui",
"description": "A UI solution used in the different Budibase projects.",
"version": "0.9.140-alpha.8",
"version": "0.9.143-alpha.4",
"license": "AGPL-3.0",
"svelte": "src/index.js",
"module": "dist/bbui.es.js",

View File

@ -1,6 +1,6 @@
{
"name": "@budibase/builder",
"version": "0.9.140-alpha.8",
"version": "0.9.143-alpha.4",
"license": "AGPL-3.0",
"private": true,
"scripts": {
@ -65,10 +65,10 @@
}
},
"dependencies": {
"@budibase/bbui": "^0.9.140-alpha.8",
"@budibase/client": "^0.9.140-alpha.8",
"@budibase/bbui": "^0.9.143-alpha.4",
"@budibase/client": "^0.9.143-alpha.4",
"@budibase/colorpicker": "1.1.2",
"@budibase/string-templates": "^0.9.140-alpha.8",
"@budibase/string-templates": "^0.9.143-alpha.4",
"@sentry/browser": "5.19.1",
"@spectrum-css/page": "^3.0.1",
"@spectrum-css/vars": "^3.0.1",

View File

@ -3,8 +3,6 @@ import PosthogClient from "./PosthogClient"
import IntercomClient from "./IntercomClient"
import SentryClient from "./SentryClient"
import { Events } from "./constants"
import { auth } from "stores/portal"
import { get } from "svelte/store"
const posthog = new PosthogClient(
process.env.POSTHOG_TOKEN,
@ -19,27 +17,13 @@ class AnalyticsHub {
}
async activate() {
// Setting the analytics env var off in the backend overrides org/tenant settings
const analyticsStatus = await api.get("/api/analytics")
const json = await analyticsStatus.json()
// Multitenancy disabled on the backend
// Analytics disabled
if (!json.enabled) return
const tenantId = get(auth).tenantId
if (tenantId) {
const res = await api.get(
`/api/global/configs/public?tenantId=${tenantId}`
)
const orgJson = await res.json()
// analytics opted out for the tenant
if (orgJson.config?.analytics === false) return
}
this.clients.forEach(client => client.init())
this.enabled = true
}
identify(id, metadata) {

View File

@ -443,10 +443,9 @@ function bindingReplacement(bindableProperties, textWithBindings, convertTo) {
for (let from of convertFromProps) {
if (shouldReplaceBinding(newBoundValue, from, convertTo)) {
const binding = bindableProperties.find(el => el[convertFrom] === from)
newBoundValue = newBoundValue.replace(
new RegExp(from, "gi"),
binding[convertTo]
)
while (newBoundValue.includes(from)) {
newBoundValue = newBoundValue.replace(from, binding[convertTo])
}
}
}
result = result.replace(boundValue, newBoundValue)

View File

@ -1,8 +1,9 @@
<script>
import { onMount } from "svelte"
import { get } from "svelte/store"
import { goto } from "@roxi/routify"
import { BUDIBASE_INTERNAL_DB } from "constants"
import { database, datasources, queries } from "stores/backend"
import { database, datasources, queries, tables } from "stores/backend"
import EditDatasourcePopover from "./popovers/EditDatasourcePopover.svelte"
import EditQueryPopover from "./popovers/EditQueryPopover.svelte"
import NavItem from "components/common/NavItem.svelte"
@ -10,6 +11,13 @@
import ICONS from "./icons"
let openDataSources = []
$: enrichedDataSources = $datasources.list.map(datasource => ({
...datasource,
open:
openDataSources.includes(datasource._id) ||
containsActiveTable(datasource),
selected: $datasources.selected === datasource._id,
}))
function selectDatasource(datasource) {
toggleNode(datasource)
@ -35,16 +43,28 @@
datasources.fetch()
queries.fetch()
})
const containsActiveTable = datasource => {
const activeTableId = get(tables).selected?._id
if (!datasource.entities) {
return false
}
let tableOptions = datasource.entities
if (!Array.isArray(tableOptions)) {
tableOptions = Object.values(tableOptions)
}
return tableOptions.find(x => x._id === activeTableId) != null
}
</script>
{#if $database?._id}
<div class="hierarchy-items-container">
{#each $datasources.list as datasource, idx}
{#each enrichedDataSources as datasource, idx}
<NavItem
border={idx > 0}
text={datasource.name}
opened={openDataSources.includes(datasource._id)}
selected={$datasources.selected === datasource._id}
opened={datasource.open}
selected={datasource.selected}
withArrow={true}
on:click={() => selectDatasource(datasource)}
on:iconClick={() => toggleNode(datasource)}
@ -61,7 +81,7 @@
{/if}
</NavItem>
{#if openDataSources.includes(datasource._id)}
{#if datasource.open}
<TableNavigator sourceId={datasource._id} />
{#each $queries.list.filter(query => query.datasourceId === datasource._id) as query}
<NavItem

View File

@ -0,0 +1,35 @@
<script>
import { Select, Label } from "@budibase/bbui"
import { currentAsset, store } from "builderStore"
import { getActionProviderComponents } from "builderStore/dataBinding"
export let parameters
$: actionProviders = getActionProviderComponents(
$currentAsset,
$store.selectedComponentId,
"RefreshDatasource"
)
</script>
<div class="root">
<Label small>Data Provider</Label>
<Select
bind:value={parameters.componentId}
options={actionProviders}
getOptionLabel={x => x._instanceName}
getOptionValue={x => x._id}
/>
</div>
<style>
.root {
display: grid;
column-gap: var(--spacing-l);
row-gap: var(--spacing-s);
grid-template-columns: 70px 1fr;
align-items: center;
max-width: 400px;
margin: 0 auto;
}
</style>

View File

@ -12,6 +12,7 @@ import ClearForm from "./ClearForm.svelte"
import CloseScreenModal from "./CloseScreenModal.svelte"
import ChangeFormStep from "./ChangeFormStep.svelte"
import UpdateStateStep from "./UpdateState.svelte"
import RefreshDataProvider from "./RefreshDataProvider.svelte"
// Defines which actions are available to configure in the front end.
// Unfortunately the "name" property is used as the identifier so please don't
@ -62,6 +63,10 @@ export const getAvailableActions = () => {
name: "Change Form Step",
component: ChangeFormStep,
},
{
name: "Refresh Data Provider",
component: RefreshDataProvider,
},
]
if (get(store).clientFeatures?.state) {

View File

@ -0,0 +1,7 @@
<script>
import { datasources } from "stores/backend"
datasources.select("bb_internal")
</script>
<slot />

View File

@ -7,13 +7,11 @@
Divider,
Label,
Input,
Toggle,
Dropzone,
notifications,
} from "@budibase/bbui"
import { auth, organisation } from "stores/portal"
import { auth, organisation, admin } from "stores/portal"
import { post } from "builderStore/api"
import analytics from "analytics"
import { writable } from "svelte/store"
import { redirect } from "@roxi/routify"
@ -25,7 +23,6 @@
}
const values = writable({
analytics: analytics.enabled,
company: $organisation.company,
platformUrl: $organisation.platformUrl,
logo: $organisation.logoUrl
@ -57,7 +54,6 @@
const config = {
company: $values.company ?? "",
platformUrl: $values.platformUrl ?? "",
analytics: $values.analytics,
}
// remove logo if required
if (!$values.logo) {
@ -112,34 +108,22 @@
</div>
</div>
</div>
<Divider size="S" />
<Layout gap="XS" noPadding>
<Heading size="S">Platform</Heading>
<Body size="S">Here you can set up general platform settings.</Body>
</Layout>
<div class="fields">
<div class="field">
<Label size="L">Platform URL</Label>
<Input thin bind:value={$values.platformUrl} />
</div>
</div>
<Divider size="S" />
<Layout gap="S" noPadding>
{#if !$admin.cloud}
<Divider size="S" />
<Layout gap="XS" noPadding>
<Heading size="S">Analytics</Heading>
<Body size="S">
If you would like to send analytics that help us make Budibase better,
please let us know below.
</Body>
<Heading size="S">Platform</Heading>
<Body size="S">Here you can set up general platform settings.</Body>
</Layout>
<Toggle
text="Send Analytics to Budibase"
bind:value={$values.analytics}
/>
<div>
<Button disabled={loading} on:click={saveConfig} cta>Save</Button>
<div class="fields">
<div class="field">
<Label size="L">Platform URL</Label>
<Input thin bind:value={$values.platformUrl} />
</div>
</div>
</Layout>
{/if}
<div>
<Button disabled={loading} on:click={saveConfig} cta>Save</Button>
</div>
</Layout>
{/if}

View File

@ -1,4 +1,4 @@
import { writable } from "svelte/store"
import { writable, get } from "svelte/store"
import { queries, tables, views } from "./"
import api from "../../builderStore/api"
@ -8,7 +8,8 @@ export const INITIAL_DATASOURCE_VALUES = {
}
export function createDatasourcesStore() {
const { subscribe, update, set } = writable(INITIAL_DATASOURCE_VALUES)
const store = writable(INITIAL_DATASOURCE_VALUES)
const { subscribe, update, set } = store
return {
subscribe,
@ -21,7 +22,15 @@ export function createDatasourcesStore() {
fetch: async () => {
const response = await api.get(`/api/datasources`)
const json = await response.json()
update(state => ({ ...state, list: json, selected: null }))
// Clear selected if it no longer exists, otherwise keep it
const selected = get(store).selected
let nextSelected = null
if (selected && json.find(source => source._id === selected)) {
nextSelected = selected
}
update(state => ({ ...state, list: json, selected: nextSelected }))
return json
},
select: async datasourceId => {

View File

@ -1,6 +1,6 @@
{
"name": "@budibase/cli",
"version": "0.9.140-alpha.8",
"version": "0.9.143-alpha.4",
"description": "Budibase CLI, for developers, self hosting and migrations.",
"main": "src/index.js",
"bin": {

View File

@ -2389,6 +2389,7 @@
"icon": "Data",
"illegalChildren": ["section"],
"hasChildren": true,
"actions": ["RefreshDatasource"],
"settings": [
{
"type": "dataSource",

View File

@ -1,6 +1,6 @@
{
"name": "@budibase/client",
"version": "0.9.140-alpha.8",
"version": "0.9.143-alpha.4",
"license": "MPL-2.0",
"module": "dist/budibase-client.js",
"main": "dist/budibase-client.js",
@ -19,9 +19,9 @@
"dev:builder": "rollup -cw"
},
"dependencies": {
"@budibase/bbui": "^0.9.140-alpha.8",
"@budibase/bbui": "^0.9.143-alpha.4",
"@budibase/standard-components": "^0.9.139",
"@budibase/string-templates": "^0.9.140-alpha.8",
"@budibase/string-templates": "^0.9.143-alpha.4",
"regexparam": "^1.3.0",
"shortid": "^2.2.15",
"svelte-spa-router": "^3.0.5"

View File

@ -88,7 +88,7 @@ const validateFormHandler = async (action, context) => {
)
}
const refreshDatasourceHandler = async (action, context) => {
const refreshDataProviderHandler = async (action, context) => {
return await executeActionHandler(
context,
action.parameters.componentId,
@ -139,7 +139,7 @@ const handlerMap = {
["Execute Query"]: queryExecutionHandler,
["Trigger Automation"]: triggerAutomationHandler,
["Validate Form"]: validateFormHandler,
["Refresh Datasource"]: refreshDatasourceHandler,
["Refresh Data Provider"]: refreshDataProviderHandler,
["Log Out"]: logoutHandler,
["Clear Form"]: clearFormHandler,
["Close Screen Modal"]: closeScreenModalHandler,

View File

@ -1,7 +1,7 @@
{
"name": "@budibase/server",
"email": "hi@budibase.com",
"version": "0.9.140-alpha.8",
"version": "0.9.143-alpha.4",
"description": "Budibase Web Server",
"main": "src/index.js",
"repository": {
@ -25,7 +25,9 @@
"lint:fix": "yarn run format && yarn run lint",
"initialise": "node scripts/initialise.js",
"multi:enable": "node scripts/multiTenancy.js enable",
"multi:disable": "node scripts/multiTenancy.js disable"
"multi:disable": "node scripts/multiTenancy.js disable",
"selfhost:enable": "node scripts/selfhost.js enable",
"selfhost:disable": "node scripts/selfhost.js disable"
},
"jest": {
"preset": "ts-jest",
@ -62,9 +64,9 @@
"author": "Budibase",
"license": "AGPL-3.0-or-later",
"dependencies": {
"@budibase/auth": "^0.9.140-alpha.8",
"@budibase/client": "^0.9.140-alpha.8",
"@budibase/string-templates": "^0.9.140-alpha.8",
"@budibase/auth": "^0.9.143-alpha.4",
"@budibase/client": "^0.9.143-alpha.4",
"@budibase/string-templates": "^0.9.143-alpha.4",
"@elastic/elasticsearch": "7.10.0",
"@koa/router": "8.0.0",
"@sendgrid/mail": "7.1.1",

View File

@ -0,0 +1,28 @@
version: "3.8"
services:
db:
container_name: postgres-json
image: postgres
restart: always
environment:
POSTGRES_USER: root
POSTGRES_PASSWORD: root
POSTGRES_DB: main
ports:
- "5432:5432"
volumes:
#- pg_data:/var/lib/postgresql/data/
- ./init.sql:/docker-entrypoint-initdb.d/init.sql
pgadmin:
container_name: pgadmin-json
image: dpage/pgadmin4
restart: always
environment:
PGADMIN_DEFAULT_EMAIL: root@root.com
PGADMIN_DEFAULT_PASSWORD: root
ports:
- "5050:80"
#volumes:
# pg_data:

View File

@ -0,0 +1,22 @@
SELECT 'CREATE DATABASE main'
WHERE NOT EXISTS (SELECT FROM pg_database WHERE datname = 'main')\gexec
CREATE TABLE jsonTable (
id character varying(32),
data jsonb,
text text
);
INSERT INTO jsonTable (id, data) VALUES ('1', '{"id": 1, "age": 1, "name": "Mike", "newline": "this is text with a\n newline in it"}');
CREATE VIEW jsonView AS SELECT
x.id,
x.age,
x.name,
x.newline
FROM
jsonTable c,
LATERAL jsonb_to_record(c.data) x (id character varying(32),
age BIGINT,
name TEXT,
newline TEXT
);

View File

@ -0,0 +1,3 @@
#!/bin/bash
docker-compose down
docker volume prune -f

View File

@ -15,7 +15,7 @@ services:
- ./init.sql:/docker-entrypoint-initdb.d/init.sql
pgadmin:
container_name: pgadmin
container_name: pgadmin-pg
image: dpage/pgadmin4
restart: always
environment:

View File

@ -230,7 +230,12 @@ exports.create = async function (ctx) {
const response = await db.put(newApplication, { force: true })
newApplication._rev = response.rev
await createEmptyAppPackage(ctx, newApplication)
// Only create the default home screens and layout if we aren't importing
// an app
if (useTemplate !== "true") {
await createEmptyAppPackage(ctx, newApplication)
}
/* istanbul ignore next */
if (!env.isTest()) {
await createApp(appId)

View File

@ -1,9 +1,4 @@
const {
getBuiltinPermissions,
PermissionLevels,
isPermissionLevelHigherThanRead,
higherPermission,
} = require("@budibase/auth/permissions")
const { getBuiltinPermissions } = require("@budibase/auth/permissions")
const {
isBuiltin,
getDBRoleID,
@ -16,6 +11,7 @@ const {
CURRENTLY_SUPPORTED_LEVELS,
getBasePermissions,
} = require("../../utilities/security")
const { removeFromArray } = require("../../utilities")
const PermissionUpdateType = {
REMOVE: "remove",
@ -24,22 +20,6 @@ const PermissionUpdateType = {
const SUPPORTED_LEVELS = CURRENTLY_SUPPORTED_LEVELS
// quick function to perform a bit of weird logic, make sure fetch calls
// always say a write role also has read permission
function fetchLevelPerms(permissions, level, roleId) {
if (!permissions) {
permissions = {}
}
permissions[level] = roleId
if (
isPermissionLevelHigherThanRead(level) &&
!permissions[PermissionLevels.READ]
) {
permissions[PermissionLevels.READ] = roleId
}
return permissions
}
// utility function to stop this repetition - permissions always stored under roles
async function getAllDBRoles(db) {
const body = await db.allDocs(
@ -74,23 +54,31 @@ async function updatePermissionOnRole(
for (let role of dbRoles) {
let updated = false
const rolePermissions = role.permissions ? role.permissions : {}
// make sure its an array, also handle migrating
if (
!rolePermissions[resourceId] ||
!Array.isArray(rolePermissions[resourceId])
) {
rolePermissions[resourceId] =
typeof rolePermissions[resourceId] === "string"
? [rolePermissions[resourceId]]
: []
}
// handle the removal/updating the role which has this permission first
// the updating (role._id !== dbRoleId) is required because a resource/level can
// only be permitted in a single role (this reduces hierarchy confusion and simplifies
// the general UI for this, rather than needing to show everywhere it is used)
if (
(role._id !== dbRoleId || remove) &&
rolePermissions[resourceId] === level
rolePermissions[resourceId].indexOf(level) !== -1
) {
delete rolePermissions[resourceId]
removeFromArray(rolePermissions[resourceId], level)
updated = true
}
// handle the adding, we're on the correct role, at it to this
if (!remove && role._id === dbRoleId) {
rolePermissions[resourceId] = higherPermission(
rolePermissions[resourceId],
level
)
const set = new Set(rolePermissions[resourceId])
rolePermissions[resourceId] = [...set.add(level)]
updated = true
}
// handle the update, add it to bulk docs to perform at end
@ -127,12 +115,11 @@ exports.fetch = async function (ctx) {
continue
}
const roleId = getExternalRoleID(role._id)
for (let [resource, level] of Object.entries(role.permissions)) {
permissions[resource] = fetchLevelPerms(
permissions[resource],
level,
roleId
)
for (let [resource, levelArr] of Object.entries(role.permissions)) {
const levels = Array.isArray(levelArr) ? [levelArr] : levelArr
const perms = {}
levels.forEach(level => (perms[level] = roleId))
permissions[resource] = perms
}
}
// apply the base permissions
@ -157,12 +144,13 @@ exports.getResourcePerms = async function (ctx) {
for (let level of SUPPORTED_LEVELS) {
// update the various roleIds in the resource permissions
for (let role of roles) {
if (role.permissions && role.permissions[resourceId] === level) {
permissions = fetchLevelPerms(
permissions,
level,
getExternalRoleID(role._id)
)
const rolePerms = role.permissions
if (
rolePerms &&
(rolePerms[resourceId] === level ||
rolePerms[resourceId].indexOf(level) !== -1)
) {
permissions[level] = getExternalRoleID(role._id)
}
}
}

View File

@ -544,6 +544,9 @@ module External {
extra: {
idFilter: buildFilters(id || generateIdForRow(row, table), {}, table),
},
meta: {
table,
},
}
// can't really use response right now
const response = await makeExternalQuery(appId, json)

View File

@ -2,11 +2,12 @@ const Router = require("@koa/router")
const controller = require("../controllers/application")
const authorized = require("../../middleware/authorized")
const { BUILDER } = require("@budibase/auth/permissions")
const usage = require("../../middleware/usageQuota")
const router = Router()
router
.post("/api/applications", authorized(BUILDER), controller.create)
.post("/api/applications", authorized(BUILDER), usage, controller.create)
.get("/api/applications/:appId/definition", controller.fetchAppDefinition)
.get("/api/applications", controller.fetch)
.get("/api/applications/:appId/appPackage", controller.fetchAppPackage)
@ -21,6 +22,11 @@ router
authorized(BUILDER),
controller.revertClient
)
.delete("/api/applications/:appId", authorized(BUILDER), controller.delete)
.delete(
"/api/applications/:appId",
authorized(BUILDER),
usage,
controller.delete
)
module.exports = router

View File

@ -94,7 +94,8 @@ describe("/datasources", () => {
.expect(200)
// this is mock data, can't test it
expect(res.body).toBeDefined()
expect(pg.queryMock).toHaveBeenCalledWith(`select "users"."name" as "users.name", "users"."age" as "users.age" from "users" where "users"."name" ilike $1 limit $2`, ["John%", 5000])
const expSql = `select "users"."name" as "users.name", "users"."age" as "users.age" from (select * from "users" where "users"."name" ilike $1 limit $2) as "users"`
expect(pg.queryMock).toHaveBeenCalledWith(expSql, ["John%", 5000])
})
})

View File

@ -72,7 +72,7 @@ describe("/roles", () => {
.expect(200)
expect(res.body.length).toBeGreaterThan(0)
const power = res.body.find(role => role._id === BUILTIN_ROLE_IDS.POWER)
expect(power.permissions[table._id]).toEqual("read")
expect(power.permissions[table._id]).toEqual(["read"])
})
})

View File

@ -5,7 +5,6 @@ const {
PermissionLevels,
PermissionTypes,
} = require("@budibase/auth/permissions")
const usage = require("../../middleware/usageQuota")
const router = Router()
@ -28,13 +27,11 @@ router
.post(
"/api/users/metadata/self",
authorized(PermissionTypes.USER, PermissionLevels.WRITE),
usage,
controller.updateSelfMetadata
)
.delete(
"/api/users/metadata/:id",
authorized(PermissionTypes.USER, PermissionLevels.WRITE),
usage,
controller.destroyMetadata
)

View File

@ -8,7 +8,6 @@ const {
PermissionTypes,
PermissionLevels,
} = require("@budibase/auth/permissions")
const usage = require("../../middleware/usageQuota")
const router = Router()
@ -25,9 +24,8 @@ router
"/api/views/:viewName",
paramResource("viewName"),
authorized(BUILDER),
usage,
viewController.destroy
)
.post("/api/views", authorized(BUILDER), usage, viewController.save)
.post("/api/views", authorized(BUILDER), viewController.save)
module.exports = router

View File

@ -60,7 +60,7 @@ exports.definition = {
},
}
exports.run = async function ({ inputs, appId, apiKey, emitter }) {
exports.run = async function ({ inputs, appId, emitter }) {
if (inputs.row == null || inputs.row.tableId == null) {
return {
success: false,
@ -84,7 +84,7 @@ exports.run = async function ({ inputs, appId, apiKey, emitter }) {
inputs.row
)
if (env.USE_QUOTAS) {
await usage.update(apiKey, usage.Properties.ROW, 1)
await usage.update(usage.Properties.ROW, 1)
}
await rowController.save(ctx)
return {

View File

@ -52,7 +52,7 @@ exports.definition = {
},
}
exports.run = async function ({ inputs, appId, apiKey, emitter }) {
exports.run = async function ({ inputs, appId, emitter }) {
if (inputs.id == null || inputs.revision == null) {
return {
success: false,
@ -74,7 +74,7 @@ exports.run = async function ({ inputs, appId, apiKey, emitter }) {
try {
if (env.isProd()) {
await usage.update(apiKey, usage.Properties.ROW, -1)
await usage.update(usage.Properties.ROW, -1)
}
await rowController.destroy(ctx)
return {

View File

@ -53,7 +53,7 @@ exports.run = async function ({ inputs }) {
contents = "<h1>No content</h1>"
}
try {
let response = await sendSmtpEmail(to, from, subject, contents)
let response = await sendSmtpEmail(to, from, subject, contents, true)
return {
success: true,
response,

View File

@ -13,8 +13,6 @@ const { makePartial } = require("../../tests/utilities")
const { cleanInputValues } = require("../automationUtils")
const setup = require("./utilities")
usageQuota.getAPIKey.mockReturnValue({ apiKey: "test" })
describe("Run through some parts of the automations system", () => {
let config = setup.getConfig()

View File

@ -46,7 +46,7 @@ describe("test the create row action", () => {
await setup.runStep(setup.actions.CREATE_ROW.stepId, {
row
})
expect(usageQuota.update).toHaveBeenCalledWith(setup.apiKey, "rows", 1)
expect(usageQuota.update).toHaveBeenCalledWith("rows", 1)
})
})

View File

@ -37,7 +37,7 @@ describe("test the delete row action", () => {
it("check usage quota attempts", async () => {
await setup.runInProd(async () => {
await setup.runStep(setup.actions.DELETE_ROW.stepId, inputs)
expect(usageQuota.update).toHaveBeenCalledWith(setup.apiKey, "rows", -1)
expect(usageQuota.update).toHaveBeenCalledWith("rows", -1)
})
})

View File

@ -4,8 +4,10 @@ const AutomationEmitter = require("../events/AutomationEmitter")
const { processObject } = require("@budibase/string-templates")
const { DEFAULT_TENANT_ID } = require("@budibase/auth").constants
const CouchDB = require("../db")
const { DocumentTypes } = require("../db/utils")
const { DocumentTypes, isDevAppID } = require("../db/utils")
const { doInTenant } = require("@budibase/auth/tenancy")
const env = require("../environment")
const usage = require("../utilities/usageQuota")
const FILTER_STEP_ID = actions.ACTION_DEFINITIONS.FILTER.stepId
@ -80,7 +82,6 @@ class Orchestrator {
return stepFn({
inputs: step.inputs,
appId: this._appId,
apiKey: automation.apiKey,
emitter: this._emitter,
context: this._context,
})
@ -95,6 +96,11 @@ class Orchestrator {
return err
}
}
// Increment quota for automation runs
if (!env.SELF_HOSTED && !isDevAppID(this._appId)) {
usage.update(usage.Properties.AUTOMATION, 1)
}
return this.executionOutput
}
}

View File

@ -1,3 +1,5 @@
import { Table } from "./common"
export enum Operation {
CREATE = "CREATE",
READ = "READ",
@ -136,6 +138,9 @@ export interface QueryJson {
sort?: SortJson
paginate?: PaginationJson
body?: object
meta?: {
table?: Table
}
extra?: {
idFilter?: SearchFilters
}

View File

@ -1,7 +1,5 @@
import { Knex, knex } from "knex"
const BASE_LIMIT = 5000
// if requesting a single row then need to up the limit for the sake of joins
const SINGLE_ROW_LIMIT = 100
import {
QueryJson,
SearchFilters,
@ -146,46 +144,48 @@ function buildCreate(
function buildRead(knex: Knex, json: QueryJson, limit: number): KnexQuery {
let { endpoint, resource, filters, sort, paginate, relationships } = json
const tableName = endpoint.entityId
let query: KnexQuery = knex(tableName)
// select all if not specified
if (!resource) {
resource = { fields: [] }
}
let selectStatement: string | string[] = "*"
// handle select
if (resource.fields && resource.fields.length > 0) {
// select the resources as the format "table.columnName" - this is what is provided
// by the resource builder further up
query = query.select(resource.fields.map(field => `${field} as ${field}`))
} else {
query = query.select("*")
selectStatement = resource.fields.map(field => `${field} as ${field}`)
}
let foundLimit = limit || BASE_LIMIT
// handle pagination
let foundOffset: number | null = null
if (paginate && paginate.page && paginate.limit) {
// @ts-ignore
const page = paginate.page <= 1 ? 0 : paginate.page - 1
const offset = page * paginate.limit
foundLimit = paginate.limit
foundOffset = offset
} else if (paginate && paginate.limit) {
foundLimit = paginate.limit
}
// start building the query
let query: KnexQuery = knex(tableName).limit(foundLimit)
if (foundOffset) {
query = query.offset(foundOffset)
}
// handle where
query = addFilters(tableName, query, filters)
// handle join
query = addRelationships(query, tableName, relationships)
// handle sorting
if (sort) {
for (let [key, value] of Object.entries(sort)) {
const direction = value === SortDirection.ASCENDING ? "asc" : "desc"
query = query.orderBy(key, direction)
}
}
let foundLimit = limit || BASE_LIMIT
// handle pagination
if (paginate && paginate.page && paginate.limit) {
query = addFilters(tableName, query, filters)
// @ts-ignore
let preQuery: KnexQuery = knex({
// @ts-ignore
const page = paginate.page <= 1 ? 0 : paginate.page - 1
const offset = page * paginate.limit
foundLimit = paginate.limit
query = query.offset(offset)
} else if (paginate && paginate.limit) {
foundLimit = paginate.limit
}
if (foundLimit === 1) {
foundLimit = SINGLE_ROW_LIMIT
}
query = query.limit(foundLimit)
return query
[tableName]: query,
}).select(selectStatement)
// handle joins
return addRelationships(preQuery, tableName, relationships)
}
function buildUpdate(

View File

@ -12,7 +12,11 @@ import { getSqlQuery } from "./utils"
module MySQLModule {
const mysql = require("mysql")
const Sql = require("./base/sql")
const { buildExternalTableId, convertType, copyExistingPropsOver } = require("./utils")
const {
buildExternalTableId,
convertType,
copyExistingPropsOver,
} = require("./utils")
const { FieldTypes } = require("../constants")
interface MySQLConfig {
@ -104,7 +108,7 @@ module MySQLModule {
client: any,
query: SqlQuery,
connect: boolean = true
): Promise<any[]> {
): Promise<any[] | any> {
// Node MySQL is callback based, so we must wrap our call in a promise
return new Promise((resolve, reject) => {
if (connect) {
@ -238,6 +242,23 @@ module MySQLModule {
return internalQuery(this.client, input, false)
}
// when creating if an ID has been inserted need to make sure
// the id filter is enriched with it before trying to retrieve the row
checkLookupKeys(results: any, json: QueryJson) {
if (!results?.insertId || !json.meta?.table || !json.meta.table.primary) {
return json
}
const primaryKey = json.meta.table.primary?.[0]
json.extra = {
idFilter: {
equal: {
[primaryKey]: results.insertId,
},
},
}
return json
}
async query(json: QueryJson) {
const operation = this._operation(json)
this.client.connect()
@ -250,7 +271,7 @@ module MySQLModule {
const results = await internalQuery(this.client, input, false)
// same as delete, manage returning
if (operation === Operation.CREATE || operation === Operation.UPDATE) {
row = this.getReturningRow(json)
row = this.getReturningRow(this.checkLookupKeys(results, json))
}
this.client.end()
if (operation !== Operation.READ) {

View File

@ -12,7 +12,14 @@ module PostgresModule {
const { Pool } = require("pg")
const Sql = require("./base/sql")
const { FieldTypes } = require("../constants")
const { buildExternalTableId, convertType, copyExistingPropsOver } = require("./utils")
const {
buildExternalTableId,
convertType,
copyExistingPropsOver,
} = require("./utils")
const { escapeDangerousCharacters } = require("../utilities")
const JSON_REGEX = /'{.*}'::json/s
interface PostgresConfig {
host: string
@ -94,6 +101,17 @@ module PostgresModule {
}
async function internalQuery(client: any, query: SqlQuery) {
// need to handle a specific issue with json data types in postgres,
// new lines inside the JSON data will break it
if (query && query.sql) {
const matches = query.sql.match(JSON_REGEX)
if (matches && matches.length > 0) {
for (let match of matches) {
const escaped = escapeDangerousCharacters(match)
query.sql = query.sql.replace(match, escaped)
}
}
}
try {
return await client.query(query.sql, query.bindings || [])
} catch (err) {
@ -179,10 +197,16 @@ module PostgresModule {
}
const type: string = convertType(column.data_type, TYPE_MAP)
const identity = !!(column.identity_generation || column.identity_start || column.identity_increment)
const hasDefault = typeof column.column_default === "string" &&
const identity = !!(
column.identity_generation ||
column.identity_start ||
column.identity_increment
)
const hasDefault =
typeof column.column_default === "string" &&
column.column_default.startsWith("nextval")
const isGenerated = column.is_generated && column.is_generated !== "NEVER"
const isGenerated =
column.is_generated && column.is_generated !== "NEVER"
const isAuto: boolean = hasDefault || identity || isGenerated
tables[tableName].schema[columnName] = {
autocolumn: isAuto,

View File

@ -57,7 +57,7 @@ describe("SQL query builder", () => {
const query = sql._query(generateReadJson())
expect(query).toEqual({
bindings: [limit],
sql: `select * from "${TABLE_NAME}" limit $1`
sql: `select * from (select * from "${TABLE_NAME}" limit $1) as "${TABLE_NAME}"`
})
})
@ -68,7 +68,7 @@ describe("SQL query builder", () => {
}))
expect(query).toEqual({
bindings: [limit],
sql: `select "${TABLE_NAME}"."name" as "${nameProp}", "${TABLE_NAME}"."age" as "${ageProp}" from "${TABLE_NAME}" limit $1`
sql: `select "${TABLE_NAME}"."name" as "${nameProp}", "${TABLE_NAME}"."age" as "${ageProp}" from (select * from "${TABLE_NAME}" limit $1) as "${TABLE_NAME}"`
})
})
@ -82,7 +82,7 @@ describe("SQL query builder", () => {
}))
expect(query).toEqual({
bindings: ["John%", limit],
sql: `select * from "${TABLE_NAME}" where "${TABLE_NAME}"."name" ilike $1 limit $2`
sql: `select * from (select * from "${TABLE_NAME}" where "${TABLE_NAME}"."name" ilike $1 limit $2) as "${TABLE_NAME}"`
})
})
@ -99,7 +99,7 @@ describe("SQL query builder", () => {
}))
expect(query).toEqual({
bindings: [2, 10, limit],
sql: `select * from "${TABLE_NAME}" where "${TABLE_NAME}"."age" between $1 and $2 limit $3`
sql: `select * from (select * from "${TABLE_NAME}" where "${TABLE_NAME}"."age" between $1 and $2 limit $3) as "${TABLE_NAME}"`
})
})
@ -115,7 +115,7 @@ describe("SQL query builder", () => {
}))
expect(query).toEqual({
bindings: [10, "John", limit],
sql: `select * from "${TABLE_NAME}" where ("${TABLE_NAME}"."age" = $1) or ("${TABLE_NAME}"."name" = $2) limit $3`
sql: `select * from (select * from "${TABLE_NAME}" where ("${TABLE_NAME}"."age" = $1) or ("${TABLE_NAME}"."name" = $2) limit $3) as "${TABLE_NAME}"`
})
})
@ -160,7 +160,7 @@ describe("SQL query builder", () => {
const query = new Sql("mssql", 10)._query(generateReadJson())
expect(query).toEqual({
bindings: [10],
sql: `select top (@p0) * from [${TABLE_NAME}]`
sql: `select * from (select top (@p0) * from [${TABLE_NAME}]) as [${TABLE_NAME}]`
})
})
@ -168,7 +168,7 @@ describe("SQL query builder", () => {
const query = new Sql("mysql", 10)._query(generateReadJson())
expect(query).toEqual({
bindings: [10],
sql: `select * from \`${TABLE_NAME}\` limit ?`
sql: `select * from (select * from \`${TABLE_NAME}\` limit ?) as \`${TABLE_NAME}\``
})
})
})

View File

@ -84,7 +84,11 @@ export function isIsoDateString(str: string) {
}
// add the existing relationships from the entities if they exist, to prevent them from being overridden
export function copyExistingPropsOver(tableName: string, tables: { [key: string]: any }, entities: { [key: string]: any }) {
export function copyExistingPropsOver(
tableName: string,
tables: { [key: string]: any },
entities: { [key: string]: any }
) {
if (entities && entities[tableName]) {
if (entities[tableName].primaryDisplay) {
tables[tableName].primaryDisplay = entities[tableName].primaryDisplay

View File

@ -39,7 +39,7 @@ class TestConfiguration {
if (bool) {
env.isDev = () => false
env.isProd = () => true
this.ctx.auth = { apiKey: "test" }
this.ctx.user = { tenantId: "test" }
} else {
env.isDev = () => true
env.isProd = () => false
@ -114,7 +114,7 @@ describe("usageQuota middleware", () => {
await config.executeMiddleware()
expect(usageQuota.update).toHaveBeenCalledWith("test", "rows", 1)
expect(usageQuota.update).toHaveBeenCalledWith("rows", 1)
expect(config.next).toHaveBeenCalled()
})
@ -131,7 +131,7 @@ describe("usageQuota middleware", () => {
])
await config.executeMiddleware()
expect(usageQuota.update).toHaveBeenCalledWith("test", "storage", 10100)
expect(usageQuota.update).toHaveBeenCalledWith("storage", 10100)
expect(config.next).toHaveBeenCalled()
})
})

View File

@ -13,6 +13,7 @@ const DOMAIN_MAP = {
upload: usageQuota.Properties.UPLOAD,
views: usageQuota.Properties.VIEW,
users: usageQuota.Properties.USER,
applications: usageQuota.Properties.APPS,
// this will not be updated by endpoint calls
// instead it will be updated by triggerInfo
automationRuns: usageQuota.Properties.AUTOMATION,
@ -57,9 +58,9 @@ module.exports = async (ctx, next) => {
usage = files.map(file => file.size).reduce((total, size) => total + size)
}
try {
await usageQuota.update(ctx.auth.apiKey, property, usage)
await usageQuota.update(property, usage)
return next()
} catch (err) {
ctx.throw(403, err)
ctx.throw(400, err)
}
}

View File

@ -10,6 +10,14 @@ exports.wait = ms => new Promise(resolve => setTimeout(resolve, ms))
exports.isDev = env.isDev
exports.removeFromArray = (array, element) => {
const index = array.indexOf(element)
if (index !== -1) {
array.splice(index, 1)
}
return array
}
/**
* Makes sure that a URL has the correct number of slashes, while maintaining the
* http(s):// double slashes.
@ -106,3 +114,13 @@ exports.deleteEntityMetadata = async (appId, type, entityId) => {
await db.remove(id, rev)
}
}
exports.escapeDangerousCharacters = string => {
return string
.replace(/[\\]/g, "\\\\")
.replace(/[\b]/g, "\\b")
.replace(/[\f]/g, "\\f")
.replace(/[\n]/g, "\\n")
.replace(/[\r]/g, "\\r")
.replace(/[\t]/g, "\\t")
}

View File

@ -1,41 +1,9 @@
const env = require("../environment")
const { apiKeyTable } = require("../db/dynamoClient")
const DEFAULT_USAGE = {
rows: 0,
storage: 0,
views: 0,
automationRuns: 0,
users: 0,
}
const DEFAULT_PLAN = {
rows: 1000,
// 1 GB
storage: 8589934592,
views: 10,
automationRuns: 100,
users: 10000,
}
function buildUpdateParams(key, property, usage) {
return {
primary: key,
condition:
"attribute_exists(#quota) AND attribute_exists(#limits) AND #quota.#prop < #limits.#prop AND #quotaReset > :now",
expression: "ADD #quota.#prop :usage",
names: {
"#quota": "usageQuota",
"#prop": property,
"#limits": "usageLimits",
"#quotaReset": "quotaReset",
},
values: {
":usage": usage,
":now": Date.now(),
},
}
}
const { getGlobalDB } = require("@budibase/auth/tenancy")
const {
StaticDatabases,
generateNewUsageQuotaDoc,
} = require("@budibase/auth/db")
function getNewQuotaReset() {
return Date.now() + 2592000000
@ -47,59 +15,59 @@ exports.Properties = {
VIEW: "views",
USER: "users",
AUTOMATION: "automationRuns",
APPS: "apps",
EMAILS: "emails",
}
exports.getAPIKey = async appId => {
if (!env.USE_QUOTAS) {
return { apiKey: null }
async function getUsageQuotaDoc(db) {
let quota
try {
quota = await db.get(StaticDatabases.PLATFORM_INFO.docs.usageQuota)
} catch (err) {
// doc doesn't exist. Create it
quota = await db.post(generateNewUsageQuotaDoc())
}
return apiKeyTable.get({ primary: appId })
return quota
}
/**
* Given a specified API key this will add to the usage object for the specified property.
* @param {string} apiKey The API key which is to be updated.
* Given a specified tenantId this will add to the usage object for the specified property.
* @param {string} property The property which is to be added to (within the nested usageQuota object).
* @param {number} usage The amount (this can be negative) to adjust the number by.
* @returns {Promise<void>} When this completes the API key will now be up to date - the quota period may have
* also been reset after this call.
*/
exports.update = async (apiKey, property, usage) => {
exports.update = async (property, usage) => {
if (!env.USE_QUOTAS) {
return
}
try {
await apiKeyTable.update(buildUpdateParams(apiKey, property, usage))
} catch (err) {
// conditional check means the condition failed, need to check why
if (err.code === "ConditionalCheckFailedException") {
// get the API key so we can check it
const keyObj = await apiKeyTable.get({ primary: apiKey })
// the usage quota or usage limits didn't exist
if (keyObj && (keyObj.usageQuota == null || keyObj.usageLimits == null)) {
keyObj.usageQuota =
keyObj.usageQuota == null ? DEFAULT_USAGE : keyObj.usageQuota
keyObj.usageLimits =
keyObj.usageLimits == null ? DEFAULT_PLAN : keyObj.usageLimits
keyObj.quotaReset = getNewQuotaReset()
await apiKeyTable.put({ item: keyObj })
return
}
// we have in fact breached the reset period
else if (keyObj && keyObj.quotaReset <= Date.now()) {
// update the quota reset period and reset the values for all properties
keyObj.quotaReset = getNewQuotaReset()
for (let prop of Object.keys(keyObj.usageQuota)) {
if (prop === property) {
keyObj.usageQuota[prop] = usage > 0 ? usage : 0
} else {
keyObj.usageQuota[prop] = 0
}
}
await apiKeyTable.put({ item: keyObj })
return
const db = getGlobalDB()
const quota = await getUsageQuotaDoc(db)
// Check if the quota needs reset
if (Date.now() >= quota.quotaReset) {
quota.quotaReset = getNewQuotaReset()
for (let prop of Object.keys(quota.usageQuota)) {
quota.usageQuota[prop] = 0
}
}
// increment the quota
quota.usageQuota[property] += usage
if (quota.usageQuota[property] >= quota.usageLimits[property]) {
throw new Error(
`You have exceeded your usage quota of ${quota.usageLimits[property]} ${property}.`
)
}
// update the usage quotas
await db.put(quota)
} catch (err) {
console.error(`Error updating usage quotas for ${property}`, err)
throw err
}
}

View File

@ -34,7 +34,7 @@ function request(ctx, request) {
exports.request = request
// have to pass in the tenant ID as this could be coming from an automation
exports.sendSmtpEmail = async (to, from, subject, contents) => {
exports.sendSmtpEmail = async (to, from, subject, contents, automation) => {
// tenant ID will be set in header
const response = await fetch(
checkSlashesInUrl(env.WORKER_URL + `/api/global/email/send`),
@ -46,6 +46,7 @@ exports.sendSmtpEmail = async (to, from, subject, contents) => {
contents,
subject,
purpose: "custom",
automation,
},
})
)

View File

@ -1,6 +1,6 @@
{
"name": "@budibase/string-templates",
"version": "0.9.140-alpha.8",
"version": "0.9.143-alpha.4",
"description": "Handlebars wrapper for Budibase templating.",
"main": "src/index.cjs",
"module": "dist/bundle.mjs",

View File

@ -1,7 +1,7 @@
{
"name": "@budibase/worker",
"email": "hi@budibase.com",
"version": "0.9.140-alpha.8",
"version": "0.9.143-alpha.4",
"description": "Budibase background service",
"main": "src/index.js",
"repository": {
@ -25,8 +25,8 @@
"author": "Budibase",
"license": "AGPL-3.0-or-later",
"dependencies": {
"@budibase/auth": "^0.9.140-alpha.8",
"@budibase/string-templates": "^0.9.140-alpha.8",
"@budibase/auth": "^0.9.143-alpha.4",
"@budibase/string-templates": "^0.9.143-alpha.4",
"@koa/router": "^8.0.0",
"@techpass/passport-openidconnect": "^0.3.0",
"aws-sdk": "^2.811.0",

View File

@ -10,6 +10,7 @@ const email = require("../../../utilities/email")
const { upload, ObjectStoreBuckets } = require("@budibase/auth").objectStore
const CouchDB = require("../../../db")
const { getGlobalDB } = require("@budibase/auth/tenancy")
const env = require("../../../environment")
exports.save = async function (ctx) {
const db = getGlobalDB()
@ -174,7 +175,13 @@ exports.upload = async function (ctx) {
const file = ctx.request.files.file
const { type, name } = ctx.params
const bucket = ObjectStoreBuckets.GLOBAL
let bucket
if (env.SELF_HOSTED) {
bucket = ObjectStoreBuckets.GLOBAL
} else {
bucket = ObjectStoreBuckets.GLOBAL_CLOUD
}
const key = `${type}/${name}`
await upload({
bucket,

View File

@ -2,8 +2,16 @@ const { sendEmail } = require("../../../utilities/email")
const { getGlobalDB } = require("@budibase/auth/tenancy")
exports.sendEmail = async ctx => {
let { workspaceId, email, userId, purpose, contents, from, subject } =
ctx.request.body
let {
workspaceId,
email,
userId,
purpose,
contents,
from,
subject,
automation,
} = ctx.request.body
let user
if (userId) {
const db = getGlobalDB()
@ -15,6 +23,7 @@ exports.sendEmail = async ctx => {
contents,
from,
subject,
automation,
})
ctx.body = {
...response,

View File

@ -1,8 +1,8 @@
const {
generateGlobalUserID,
getGlobalUserParams,
StaticDatabases,
generateNewUsageQuotaDoc,
} = require("@budibase/auth/db")
const { hash, getGlobalUserByEmail } = require("@budibase/auth").utils
const { UserStatus, EmailTemplatePurpose } = require("../../../constants")
@ -18,6 +18,7 @@ const {
tryAddTenant,
updateTenantId,
} = require("@budibase/auth/tenancy")
const env = require("../../../environment")
const PLATFORM_INFO_DB = StaticDatabases.PLATFORM_INFO.name
@ -139,6 +140,11 @@ exports.adminUser = async ctx => {
})
)
// write usage quotas for cloud
if (!env.SELF_HOSTED) {
await db.post(generateNewUsageQuotaDoc())
}
if (response.rows.some(row => row.doc.admin)) {
ctx.throw(
403,

View File

@ -33,6 +33,12 @@ module.exports = {
INTERNAL_API_KEY: process.env.INTERNAL_API_KEY,
MULTI_TENANCY: process.env.MULTI_TENANCY,
ACCOUNT_PORTAL_URL: process.env.ACCOUNT_PORTAL_URL,
SMTP_FALLBACK_ENABLED: process.env.SMTP_FALLBACK_ENABLED,
SMTP_USER: process.env.SMTP_USER,
SMTP_PASSWORD: process.env.SMTP_PASSWORD,
SMTP_HOST: process.env.SMTP_HOST,
SMTP_PORT: process.env.SMTP_PORT,
SMTP_FROM_ADDRESS: process.env.SMTP_FROM_ADDRESS,
_set(key, value) {
process.env[key] = value
module.exports[key] = value

View File

@ -1,4 +1,5 @@
const nodemailer = require("nodemailer")
const env = require("../environment")
const { getScopedConfig } = require("@budibase/auth/db")
const { EmailTemplatePurpose, TemplateTypes, Configs } = require("../constants")
const { getTemplateByPurpose } = require("../constants/templates")
@ -101,16 +102,35 @@ async function buildEmail(purpose, email, context, { user, contents } = {}) {
* Utility function for finding most valid SMTP configuration.
* @param {object} db The CouchDB database which is to be looked up within.
* @param {string|null} workspaceId If using finer grain control of configs a workspace can be used.
* @param {boolean|null} automation Whether or not the configuration is being fetched for an email automation.
* @return {Promise<object|null>} returns the SMTP configuration if it exists
*/
async function getSmtpConfiguration(db, workspaceId = null) {
async function getSmtpConfiguration(db, workspaceId = null, automation) {
const params = {
type: Configs.SMTP,
}
if (workspaceId) {
params.workspace = workspaceId
}
return getScopedConfig(db, params)
const customConfig = getScopedConfig(db, params)
if (customConfig) {
return customConfig
}
// Use an SMTP fallback configuration from env variables
if (!automation && env.SMTP_FALLBACK_ENABLED) {
return {
port: env.SMTP_PORT,
host: env.SMTP_HOST,
secure: false,
auth: {
user: env.SMTP_USER,
pass: env.SMTP_PASSWORD,
},
}
}
}
/**
@ -118,8 +138,8 @@ async function getSmtpConfiguration(db, workspaceId = null) {
* @return {Promise<boolean>} returns true if there is a configuration that can be used.
*/
exports.isEmailConfigured = async (workspaceId = null) => {
// when "testing" simply return true
if (TEST_MODE) {
// when "testing" or smtp fallback is enabled simply return true
if (TEST_MODE || env.SMTP_FALLBACK_ENABLED) {
return true
}
const db = getGlobalDB()
@ -138,16 +158,17 @@ exports.isEmailConfigured = async (workspaceId = null) => {
* @param {string|undefined} contents If sending a custom email then can supply contents which will be added to it.
* @param {string|undefined} subject A custom subject can be specified if the config one is not desired.
* @param {object|undefined} info Pass in a structure of information to be stored alongside the invitation.
* @param {boolean|undefined} disableFallback Prevent email being sent from SMTP fallback to avoid spam.
* @return {Promise<object>} returns details about the attempt to send email, e.g. if it is successful; based on
* nodemailer response.
*/
exports.sendEmail = async (
email,
purpose,
{ workspaceId, user, from, contents, subject, info } = {}
{ workspaceId, user, from, contents, subject, info, automation } = {}
) => {
const db = getGlobalDB()
let config = (await getSmtpConfiguration(db, workspaceId)) || {}
let config = (await getSmtpConfiguration(db, workspaceId, automation)) || {}
if (Object.keys(config).length === 0 && !TEST_MODE) {
throw "Unable to find SMTP configuration."
}