-
-
+ {#if !$admin.cloud}
+
- Analytics
-
- If you would like to send analytics that help us make Budibase better,
- please let us know below.
-
+ Platform
+ Here you can set up general platform settings.
-
-
-
+
+
+
+
+
-
+ {/if}
+
+
+
{/if}
diff --git a/packages/builder/src/stores/portal/auth.js b/packages/builder/src/stores/portal/auth.js
index e33a1f22ac..95157e3f93 100644
--- a/packages/builder/src/stores/portal/auth.js
+++ b/packages/builder/src/stores/portal/auth.js
@@ -54,15 +54,13 @@ export function createAuthStore() {
if (user) {
analytics.activate().then(() => {
analytics.identify(user._id, user)
- if (user.size === "100+" || user.size === "10000+") {
- analytics.showChat({
- email: user.email,
- created_at: user.createdAt || Date.now(),
- name: user.name,
- user_id: user._id,
- tenant: user.tenantId,
- })
- }
+ analytics.showChat({
+ email: user.email,
+ created_at: user.createdAt || Date.now(),
+ name: user.name,
+ user_id: user._id,
+ tenant: user.tenantId,
+ })
})
}
}
diff --git a/packages/cli/package.json b/packages/cli/package.json
index b71457a71a..40af06d5dd 100644
--- a/packages/cli/package.json
+++ b/packages/cli/package.json
@@ -1,6 +1,6 @@
{
"name": "@budibase/cli",
- "version": "0.9.140-alpha.11",
+ "version": "0.9.143-alpha.9",
"description": "Budibase CLI, for developers, self hosting and migrations.",
"main": "src/index.js",
"bin": {
diff --git a/packages/client/package.json b/packages/client/package.json
index 8d387eb674..e230d832a9 100644
--- a/packages/client/package.json
+++ b/packages/client/package.json
@@ -1,6 +1,6 @@
{
"name": "@budibase/client",
- "version": "0.9.140-alpha.11",
+ "version": "0.9.143-alpha.9",
"license": "MPL-2.0",
"module": "dist/budibase-client.js",
"main": "dist/budibase-client.js",
@@ -19,9 +19,9 @@
"dev:builder": "rollup -cw"
},
"dependencies": {
- "@budibase/bbui": "^0.9.140-alpha.11",
+ "@budibase/bbui": "^0.9.143-alpha.9",
"@budibase/standard-components": "^0.9.139",
- "@budibase/string-templates": "^0.9.140-alpha.11",
+ "@budibase/string-templates": "^0.9.143-alpha.9",
"regexparam": "^1.3.0",
"shortid": "^2.2.15",
"svelte-spa-router": "^3.0.5"
diff --git a/packages/server/package.json b/packages/server/package.json
index 1e2dc602ae..df0f879a56 100644
--- a/packages/server/package.json
+++ b/packages/server/package.json
@@ -1,7 +1,7 @@
{
"name": "@budibase/server",
"email": "hi@budibase.com",
- "version": "0.9.140-alpha.11",
+ "version": "0.9.143-alpha.9",
"description": "Budibase Web Server",
"main": "src/index.js",
"repository": {
@@ -25,7 +25,9 @@
"lint:fix": "yarn run format && yarn run lint",
"initialise": "node scripts/initialise.js",
"multi:enable": "node scripts/multiTenancy.js enable",
- "multi:disable": "node scripts/multiTenancy.js disable"
+ "multi:disable": "node scripts/multiTenancy.js disable",
+ "selfhost:enable": "node scripts/selfhost.js enable",
+ "selfhost:disable": "node scripts/selfhost.js disable"
},
"jest": {
"preset": "ts-jest",
@@ -62,9 +64,9 @@
"author": "Budibase",
"license": "AGPL-3.0-or-later",
"dependencies": {
- "@budibase/auth": "^0.9.140-alpha.11",
- "@budibase/client": "^0.9.140-alpha.11",
- "@budibase/string-templates": "^0.9.140-alpha.11",
+ "@budibase/auth": "^0.9.143-alpha.9",
+ "@budibase/client": "^0.9.143-alpha.9",
+ "@budibase/string-templates": "^0.9.143-alpha.9",
"@elastic/elasticsearch": "7.10.0",
"@koa/router": "8.0.0",
"@sendgrid/mail": "7.1.1",
diff --git a/packages/server/scripts/integrations/pg-json/docker-compose.yml b/packages/server/scripts/integrations/pg-json/docker-compose.yml
new file mode 100644
index 0000000000..6bc307a86d
--- /dev/null
+++ b/packages/server/scripts/integrations/pg-json/docker-compose.yml
@@ -0,0 +1,28 @@
+version: "3.8"
+services:
+ db:
+ container_name: postgres-json
+ image: postgres
+ restart: always
+ environment:
+ POSTGRES_USER: root
+ POSTGRES_PASSWORD: root
+ POSTGRES_DB: main
+ ports:
+ - "5432:5432"
+ volumes:
+ #- pg_data:/var/lib/postgresql/data/
+ - ./init.sql:/docker-entrypoint-initdb.d/init.sql
+
+ pgadmin:
+ container_name: pgadmin-json
+ image: dpage/pgadmin4
+ restart: always
+ environment:
+ PGADMIN_DEFAULT_EMAIL: root@root.com
+ PGADMIN_DEFAULT_PASSWORD: root
+ ports:
+ - "5050:80"
+
+#volumes:
+# pg_data:
diff --git a/packages/server/scripts/integrations/pg-json/init.sql b/packages/server/scripts/integrations/pg-json/init.sql
new file mode 100644
index 0000000000..06a5b4901d
--- /dev/null
+++ b/packages/server/scripts/integrations/pg-json/init.sql
@@ -0,0 +1,22 @@
+SELECT 'CREATE DATABASE main'
+WHERE NOT EXISTS (SELECT FROM pg_database WHERE datname = 'main')\gexec
+CREATE TABLE jsonTable (
+ id character varying(32),
+ data jsonb,
+ text text
+);
+
+INSERT INTO jsonTable (id, data) VALUES ('1', '{"id": 1, "age": 1, "name": "Mike", "newline": "this is text with a\n newline in it"}');
+
+CREATE VIEW jsonView AS SELECT
+ x.id,
+ x.age,
+ x.name,
+ x.newline
+FROM
+ jsonTable c,
+ LATERAL jsonb_to_record(c.data) x (id character varying(32),
+ age BIGINT,
+ name TEXT,
+ newline TEXT
+ );
diff --git a/packages/server/scripts/integrations/pg-json/reset.sh b/packages/server/scripts/integrations/pg-json/reset.sh
new file mode 100755
index 0000000000..32778bd11f
--- /dev/null
+++ b/packages/server/scripts/integrations/pg-json/reset.sh
@@ -0,0 +1,3 @@
+#!/bin/bash
+docker-compose down
+docker volume prune -f
diff --git a/packages/server/src/api/controllers/permission.js b/packages/server/src/api/controllers/permission.js
index e269f8c41d..6c02663649 100644
--- a/packages/server/src/api/controllers/permission.js
+++ b/packages/server/src/api/controllers/permission.js
@@ -1,9 +1,4 @@
-const {
- getBuiltinPermissions,
- PermissionLevels,
- isPermissionLevelHigherThanRead,
- higherPermission,
-} = require("@budibase/auth/permissions")
+const { getBuiltinPermissions } = require("@budibase/auth/permissions")
const {
isBuiltin,
getDBRoleID,
@@ -16,6 +11,7 @@ const {
CURRENTLY_SUPPORTED_LEVELS,
getBasePermissions,
} = require("../../utilities/security")
+const { removeFromArray } = require("../../utilities")
const PermissionUpdateType = {
REMOVE: "remove",
@@ -24,22 +20,6 @@ const PermissionUpdateType = {
const SUPPORTED_LEVELS = CURRENTLY_SUPPORTED_LEVELS
-// quick function to perform a bit of weird logic, make sure fetch calls
-// always say a write role also has read permission
-function fetchLevelPerms(permissions, level, roleId) {
- if (!permissions) {
- permissions = {}
- }
- permissions[level] = roleId
- if (
- isPermissionLevelHigherThanRead(level) &&
- !permissions[PermissionLevels.READ]
- ) {
- permissions[PermissionLevels.READ] = roleId
- }
- return permissions
-}
-
// utility function to stop this repetition - permissions always stored under roles
async function getAllDBRoles(db) {
const body = await db.allDocs(
@@ -74,23 +54,31 @@ async function updatePermissionOnRole(
for (let role of dbRoles) {
let updated = false
const rolePermissions = role.permissions ? role.permissions : {}
+ // make sure its an array, also handle migrating
+ if (
+ !rolePermissions[resourceId] ||
+ !Array.isArray(rolePermissions[resourceId])
+ ) {
+ rolePermissions[resourceId] =
+ typeof rolePermissions[resourceId] === "string"
+ ? [rolePermissions[resourceId]]
+ : []
+ }
// handle the removal/updating the role which has this permission first
// the updating (role._id !== dbRoleId) is required because a resource/level can
// only be permitted in a single role (this reduces hierarchy confusion and simplifies
// the general UI for this, rather than needing to show everywhere it is used)
if (
(role._id !== dbRoleId || remove) &&
- rolePermissions[resourceId] === level
+ rolePermissions[resourceId].indexOf(level) !== -1
) {
- delete rolePermissions[resourceId]
+ removeFromArray(rolePermissions[resourceId], level)
updated = true
}
// handle the adding, we're on the correct role, at it to this
if (!remove && role._id === dbRoleId) {
- rolePermissions[resourceId] = higherPermission(
- rolePermissions[resourceId],
- level
- )
+ const set = new Set(rolePermissions[resourceId])
+ rolePermissions[resourceId] = [...set.add(level)]
updated = true
}
// handle the update, add it to bulk docs to perform at end
@@ -127,12 +115,11 @@ exports.fetch = async function (ctx) {
continue
}
const roleId = getExternalRoleID(role._id)
- for (let [resource, level] of Object.entries(role.permissions)) {
- permissions[resource] = fetchLevelPerms(
- permissions[resource],
- level,
- roleId
- )
+ for (let [resource, levelArr] of Object.entries(role.permissions)) {
+ const levels = Array.isArray(levelArr) ? [levelArr] : levelArr
+ const perms = {}
+ levels.forEach(level => (perms[level] = roleId))
+ permissions[resource] = perms
}
}
// apply the base permissions
@@ -157,12 +144,13 @@ exports.getResourcePerms = async function (ctx) {
for (let level of SUPPORTED_LEVELS) {
// update the various roleIds in the resource permissions
for (let role of roles) {
- if (role.permissions && role.permissions[resourceId] === level) {
- permissions = fetchLevelPerms(
- permissions,
- level,
- getExternalRoleID(role._id)
- )
+ const rolePerms = role.permissions
+ if (
+ rolePerms &&
+ (rolePerms[resourceId] === level ||
+ rolePerms[resourceId].indexOf(level) !== -1)
+ ) {
+ permissions[level] = getExternalRoleID(role._id)
}
}
}
diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts
index 12db55efdc..75c3e9b492 100644
--- a/packages/server/src/api/controllers/row/ExternalRequest.ts
+++ b/packages/server/src/api/controllers/row/ExternalRequest.ts
@@ -546,7 +546,7 @@ module External {
},
meta: {
table,
- }
+ },
}
// can't really use response right now
const response = await makeExternalQuery(appId, json)
diff --git a/packages/server/src/api/routes/application.js b/packages/server/src/api/routes/application.js
index c1d39acbd5..4d67a0f4f4 100644
--- a/packages/server/src/api/routes/application.js
+++ b/packages/server/src/api/routes/application.js
@@ -2,11 +2,12 @@ const Router = require("@koa/router")
const controller = require("../controllers/application")
const authorized = require("../../middleware/authorized")
const { BUILDER } = require("@budibase/auth/permissions")
+const usage = require("../../middleware/usageQuota")
const router = Router()
router
- .post("/api/applications", authorized(BUILDER), controller.create)
+ .post("/api/applications", authorized(BUILDER), usage, controller.create)
.get("/api/applications/:appId/definition", controller.fetchAppDefinition)
.get("/api/applications", controller.fetch)
.get("/api/applications/:appId/appPackage", controller.fetchAppPackage)
@@ -21,6 +22,11 @@ router
authorized(BUILDER),
controller.revertClient
)
- .delete("/api/applications/:appId", authorized(BUILDER), controller.delete)
+ .delete(
+ "/api/applications/:appId",
+ authorized(BUILDER),
+ usage,
+ controller.delete
+ )
module.exports = router
diff --git a/packages/server/src/api/routes/tests/role.spec.js b/packages/server/src/api/routes/tests/role.spec.js
index ad42ef180a..d74a84b2b2 100644
--- a/packages/server/src/api/routes/tests/role.spec.js
+++ b/packages/server/src/api/routes/tests/role.spec.js
@@ -72,7 +72,7 @@ describe("/roles", () => {
.expect(200)
expect(res.body.length).toBeGreaterThan(0)
const power = res.body.find(role => role._id === BUILTIN_ROLE_IDS.POWER)
- expect(power.permissions[table._id]).toEqual("read")
+ expect(power.permissions[table._id]).toEqual(["read"])
})
})
diff --git a/packages/server/src/api/routes/user.js b/packages/server/src/api/routes/user.js
index b3b486fe45..d171870215 100644
--- a/packages/server/src/api/routes/user.js
+++ b/packages/server/src/api/routes/user.js
@@ -5,7 +5,6 @@ const {
PermissionLevels,
PermissionTypes,
} = require("@budibase/auth/permissions")
-const usage = require("../../middleware/usageQuota")
const router = Router()
@@ -28,13 +27,11 @@ router
.post(
"/api/users/metadata/self",
authorized(PermissionTypes.USER, PermissionLevels.WRITE),
- usage,
controller.updateSelfMetadata
)
.delete(
"/api/users/metadata/:id",
authorized(PermissionTypes.USER, PermissionLevels.WRITE),
- usage,
controller.destroyMetadata
)
diff --git a/packages/server/src/api/routes/view.js b/packages/server/src/api/routes/view.js
index 7d390805c6..b72fe1ac26 100644
--- a/packages/server/src/api/routes/view.js
+++ b/packages/server/src/api/routes/view.js
@@ -8,7 +8,6 @@ const {
PermissionTypes,
PermissionLevels,
} = require("@budibase/auth/permissions")
-const usage = require("../../middleware/usageQuota")
const router = Router()
@@ -25,9 +24,8 @@ router
"/api/views/:viewName",
paramResource("viewName"),
authorized(BUILDER),
- usage,
viewController.destroy
)
- .post("/api/views", authorized(BUILDER), usage, viewController.save)
+ .post("/api/views", authorized(BUILDER), viewController.save)
module.exports = router
diff --git a/packages/server/src/automations/steps/createRow.js b/packages/server/src/automations/steps/createRow.js
index 9033004578..47d0b4eb99 100644
--- a/packages/server/src/automations/steps/createRow.js
+++ b/packages/server/src/automations/steps/createRow.js
@@ -60,7 +60,7 @@ exports.definition = {
},
}
-exports.run = async function ({ inputs, appId, apiKey, emitter }) {
+exports.run = async function ({ inputs, appId, emitter }) {
if (inputs.row == null || inputs.row.tableId == null) {
return {
success: false,
@@ -84,7 +84,7 @@ exports.run = async function ({ inputs, appId, apiKey, emitter }) {
inputs.row
)
if (env.USE_QUOTAS) {
- await usage.update(apiKey, usage.Properties.ROW, 1)
+ await usage.update(usage.Properties.ROW, 1)
}
await rowController.save(ctx)
return {
diff --git a/packages/server/src/automations/steps/deleteRow.js b/packages/server/src/automations/steps/deleteRow.js
index 0f9648cc51..225f00c5df 100644
--- a/packages/server/src/automations/steps/deleteRow.js
+++ b/packages/server/src/automations/steps/deleteRow.js
@@ -52,7 +52,7 @@ exports.definition = {
},
}
-exports.run = async function ({ inputs, appId, apiKey, emitter }) {
+exports.run = async function ({ inputs, appId, emitter }) {
if (inputs.id == null || inputs.revision == null) {
return {
success: false,
@@ -74,7 +74,7 @@ exports.run = async function ({ inputs, appId, apiKey, emitter }) {
try {
if (env.isProd()) {
- await usage.update(apiKey, usage.Properties.ROW, -1)
+ await usage.update(usage.Properties.ROW, -1)
}
await rowController.destroy(ctx)
return {
diff --git a/packages/server/src/automations/steps/sendSmtpEmail.js b/packages/server/src/automations/steps/sendSmtpEmail.js
index 9e4b5a6a3c..07a3059215 100644
--- a/packages/server/src/automations/steps/sendSmtpEmail.js
+++ b/packages/server/src/automations/steps/sendSmtpEmail.js
@@ -53,7 +53,7 @@ exports.run = async function ({ inputs }) {
contents = "
No content
"
}
try {
- let response = await sendSmtpEmail(to, from, subject, contents)
+ let response = await sendSmtpEmail(to, from, subject, contents, true)
return {
success: true,
response,
diff --git a/packages/server/src/automations/tests/automation.spec.js b/packages/server/src/automations/tests/automation.spec.js
index 83b7b81a75..9444995ca1 100644
--- a/packages/server/src/automations/tests/automation.spec.js
+++ b/packages/server/src/automations/tests/automation.spec.js
@@ -13,8 +13,6 @@ const { makePartial } = require("../../tests/utilities")
const { cleanInputValues } = require("../automationUtils")
const setup = require("./utilities")
-usageQuota.getAPIKey.mockReturnValue({ apiKey: "test" })
-
describe("Run through some parts of the automations system", () => {
let config = setup.getConfig()
diff --git a/packages/server/src/automations/tests/createRow.spec.js b/packages/server/src/automations/tests/createRow.spec.js
index 1004711d87..a04fc7aad4 100644
--- a/packages/server/src/automations/tests/createRow.spec.js
+++ b/packages/server/src/automations/tests/createRow.spec.js
@@ -46,7 +46,7 @@ describe("test the create row action", () => {
await setup.runStep(setup.actions.CREATE_ROW.stepId, {
row
})
- expect(usageQuota.update).toHaveBeenCalledWith(setup.apiKey, "rows", 1)
+ expect(usageQuota.update).toHaveBeenCalledWith("rows", 1)
})
})
diff --git a/packages/server/src/automations/tests/deleteRow.spec.js b/packages/server/src/automations/tests/deleteRow.spec.js
index a3d73d3bf6..21246f22d0 100644
--- a/packages/server/src/automations/tests/deleteRow.spec.js
+++ b/packages/server/src/automations/tests/deleteRow.spec.js
@@ -37,7 +37,7 @@ describe("test the delete row action", () => {
it("check usage quota attempts", async () => {
await setup.runInProd(async () => {
await setup.runStep(setup.actions.DELETE_ROW.stepId, inputs)
- expect(usageQuota.update).toHaveBeenCalledWith(setup.apiKey, "rows", -1)
+ expect(usageQuota.update).toHaveBeenCalledWith("rows", -1)
})
})
diff --git a/packages/server/src/automations/thread.js b/packages/server/src/automations/thread.js
index a3e81a2274..ef12494165 100644
--- a/packages/server/src/automations/thread.js
+++ b/packages/server/src/automations/thread.js
@@ -4,8 +4,10 @@ const AutomationEmitter = require("../events/AutomationEmitter")
const { processObject } = require("@budibase/string-templates")
const { DEFAULT_TENANT_ID } = require("@budibase/auth").constants
const CouchDB = require("../db")
-const { DocumentTypes } = require("../db/utils")
+const { DocumentTypes, isDevAppID } = require("../db/utils")
const { doInTenant } = require("@budibase/auth/tenancy")
+const env = require("../environment")
+const usage = require("../utilities/usageQuota")
const FILTER_STEP_ID = actions.ACTION_DEFINITIONS.FILTER.stepId
@@ -80,7 +82,6 @@ class Orchestrator {
return stepFn({
inputs: step.inputs,
appId: this._appId,
- apiKey: automation.apiKey,
emitter: this._emitter,
context: this._context,
})
@@ -95,6 +96,11 @@ class Orchestrator {
return err
}
}
+
+ // Increment quota for automation runs
+ if (!env.SELF_HOSTED && !isDevAppID(this._appId)) {
+ usage.update(usage.Properties.AUTOMATION, 1)
+ }
return this.executionOutput
}
}
diff --git a/packages/server/src/definitions/datasource.ts b/packages/server/src/definitions/datasource.ts
index d7d4e77961..2daef8eda7 100644
--- a/packages/server/src/definitions/datasource.ts
+++ b/packages/server/src/definitions/datasource.ts
@@ -1,4 +1,4 @@
-import {Table} from "./common";
+import { Table } from "./common"
export enum Operation {
CREATE = "CREATE",
@@ -139,7 +139,7 @@ export interface QueryJson {
paginate?: PaginationJson
body?: object
meta?: {
- table?: Table,
+ table?: Table
}
extra?: {
idFilter?: SearchFilters
diff --git a/packages/server/src/integrations/base/sql.ts b/packages/server/src/integrations/base/sql.ts
index 91af3e1a85..c5e9bdb0bb 100644
--- a/packages/server/src/integrations/base/sql.ts
+++ b/packages/server/src/integrations/base/sql.ts
@@ -148,7 +148,7 @@ function buildRead(knex: Knex, json: QueryJson, limit: number): KnexQuery {
if (!resource) {
resource = { fields: [] }
}
- let selectStatement: string|string[] = "*"
+ let selectStatement: string | string[] = "*"
// handle select
if (resource.fields && resource.fields.length > 0) {
// select the resources as the format "table.columnName" - this is what is provided
diff --git a/packages/server/src/integrations/mysql.ts b/packages/server/src/integrations/mysql.ts
index 11220afb46..c17cca0745 100644
--- a/packages/server/src/integrations/mysql.ts
+++ b/packages/server/src/integrations/mysql.ts
@@ -12,7 +12,11 @@ import { getSqlQuery } from "./utils"
module MySQLModule {
const mysql = require("mysql")
const Sql = require("./base/sql")
- const { buildExternalTableId, convertType, copyExistingPropsOver } = require("./utils")
+ const {
+ buildExternalTableId,
+ convertType,
+ copyExistingPropsOver,
+ } = require("./utils")
const { FieldTypes } = require("../constants")
interface MySQLConfig {
@@ -104,7 +108,7 @@ module MySQLModule {
client: any,
query: SqlQuery,
connect: boolean = true
- ): Promise {
+ ): Promise {
// Node MySQL is callback based, so we must wrap our call in a promise
return new Promise((resolve, reject) => {
if (connect) {
@@ -248,9 +252,9 @@ module MySQLModule {
json.extra = {
idFilter: {
equal: {
- [primaryKey]: results.insertId
+ [primaryKey]: results.insertId,
},
- }
+ },
}
return json
}
diff --git a/packages/server/src/integrations/postgres.ts b/packages/server/src/integrations/postgres.ts
index 23a8685648..332ba8544d 100644
--- a/packages/server/src/integrations/postgres.ts
+++ b/packages/server/src/integrations/postgres.ts
@@ -12,7 +12,14 @@ module PostgresModule {
const { Pool } = require("pg")
const Sql = require("./base/sql")
const { FieldTypes } = require("../constants")
- const { buildExternalTableId, convertType, copyExistingPropsOver } = require("./utils")
+ const {
+ buildExternalTableId,
+ convertType,
+ copyExistingPropsOver,
+ } = require("./utils")
+ const { escapeDangerousCharacters } = require("../utilities")
+
+ const JSON_REGEX = /'{.*}'::json/s
interface PostgresConfig {
host: string
@@ -94,6 +101,17 @@ module PostgresModule {
}
async function internalQuery(client: any, query: SqlQuery) {
+ // need to handle a specific issue with json data types in postgres,
+ // new lines inside the JSON data will break it
+ if (query && query.sql) {
+ const matches = query.sql.match(JSON_REGEX)
+ if (matches && matches.length > 0) {
+ for (let match of matches) {
+ const escaped = escapeDangerousCharacters(match)
+ query.sql = query.sql.replace(match, escaped)
+ }
+ }
+ }
try {
return await client.query(query.sql, query.bindings || [])
} catch (err) {
@@ -108,7 +126,7 @@ module PostgresModule {
private readonly config: PostgresConfig
COLUMNS_SQL =
- "select * from information_schema.columns where table_schema = 'public'"
+ "select * from information_schema.columns where not table_schema = 'information_schema' and not table_schema = 'pg_catalog'"
PRIMARY_KEYS_SQL = `
select tc.table_schema, tc.table_name, kc.column_name as primary_key
@@ -179,10 +197,16 @@ module PostgresModule {
}
const type: string = convertType(column.data_type, TYPE_MAP)
- const identity = !!(column.identity_generation || column.identity_start || column.identity_increment)
- const hasDefault = typeof column.column_default === "string" &&
+ const identity = !!(
+ column.identity_generation ||
+ column.identity_start ||
+ column.identity_increment
+ )
+ const hasDefault =
+ typeof column.column_default === "string" &&
column.column_default.startsWith("nextval")
- const isGenerated = column.is_generated && column.is_generated !== "NEVER"
+ const isGenerated =
+ column.is_generated && column.is_generated !== "NEVER"
const isAuto: boolean = hasDefault || identity || isGenerated
tables[tableName].schema[columnName] = {
autocolumn: isAuto,
diff --git a/packages/server/src/integrations/utils.ts b/packages/server/src/integrations/utils.ts
index 82c35bc2d9..6e3dc6f684 100644
--- a/packages/server/src/integrations/utils.ts
+++ b/packages/server/src/integrations/utils.ts
@@ -84,7 +84,11 @@ export function isIsoDateString(str: string) {
}
// add the existing relationships from the entities if they exist, to prevent them from being overridden
-export function copyExistingPropsOver(tableName: string, tables: { [key: string]: any }, entities: { [key: string]: any }) {
+export function copyExistingPropsOver(
+ tableName: string,
+ tables: { [key: string]: any },
+ entities: { [key: string]: any }
+) {
if (entities && entities[tableName]) {
if (entities[tableName].primaryDisplay) {
tables[tableName].primaryDisplay = entities[tableName].primaryDisplay
diff --git a/packages/server/src/middleware/tests/usageQuota.spec.js b/packages/server/src/middleware/tests/usageQuota.spec.js
index 97d9c7794a..d828f2ca60 100644
--- a/packages/server/src/middleware/tests/usageQuota.spec.js
+++ b/packages/server/src/middleware/tests/usageQuota.spec.js
@@ -39,7 +39,7 @@ class TestConfiguration {
if (bool) {
env.isDev = () => false
env.isProd = () => true
- this.ctx.auth = { apiKey: "test" }
+ this.ctx.user = { tenantId: "test" }
} else {
env.isDev = () => true
env.isProd = () => false
@@ -114,7 +114,7 @@ describe("usageQuota middleware", () => {
await config.executeMiddleware()
- expect(usageQuota.update).toHaveBeenCalledWith("test", "rows", 1)
+ expect(usageQuota.update).toHaveBeenCalledWith("rows", 1)
expect(config.next).toHaveBeenCalled()
})
@@ -131,7 +131,7 @@ describe("usageQuota middleware", () => {
])
await config.executeMiddleware()
- expect(usageQuota.update).toHaveBeenCalledWith("test", "storage", 10100)
+ expect(usageQuota.update).toHaveBeenCalledWith("storage", 10100)
expect(config.next).toHaveBeenCalled()
})
})
\ No newline at end of file
diff --git a/packages/server/src/middleware/usageQuota.js b/packages/server/src/middleware/usageQuota.js
index 4647878721..3a244ef5bc 100644
--- a/packages/server/src/middleware/usageQuota.js
+++ b/packages/server/src/middleware/usageQuota.js
@@ -13,6 +13,7 @@ const DOMAIN_MAP = {
upload: usageQuota.Properties.UPLOAD,
views: usageQuota.Properties.VIEW,
users: usageQuota.Properties.USER,
+ applications: usageQuota.Properties.APPS,
// this will not be updated by endpoint calls
// instead it will be updated by triggerInfo
automationRuns: usageQuota.Properties.AUTOMATION,
@@ -57,9 +58,9 @@ module.exports = async (ctx, next) => {
usage = files.map(file => file.size).reduce((total, size) => total + size)
}
try {
- await usageQuota.update(ctx.auth.apiKey, property, usage)
+ await usageQuota.update(property, usage)
return next()
} catch (err) {
- ctx.throw(403, err)
+ ctx.throw(400, err)
}
}
diff --git a/packages/server/src/utilities/index.js b/packages/server/src/utilities/index.js
index a81f9ddcf5..b16a687fe5 100644
--- a/packages/server/src/utilities/index.js
+++ b/packages/server/src/utilities/index.js
@@ -10,6 +10,14 @@ exports.wait = ms => new Promise(resolve => setTimeout(resolve, ms))
exports.isDev = env.isDev
+exports.removeFromArray = (array, element) => {
+ const index = array.indexOf(element)
+ if (index !== -1) {
+ array.splice(index, 1)
+ }
+ return array
+}
+
/**
* Makes sure that a URL has the correct number of slashes, while maintaining the
* http(s):// double slashes.
@@ -106,3 +114,13 @@ exports.deleteEntityMetadata = async (appId, type, entityId) => {
await db.remove(id, rev)
}
}
+
+exports.escapeDangerousCharacters = string => {
+ return string
+ .replace(/[\\]/g, "\\\\")
+ .replace(/[\b]/g, "\\b")
+ .replace(/[\f]/g, "\\f")
+ .replace(/[\n]/g, "\\n")
+ .replace(/[\r]/g, "\\r")
+ .replace(/[\t]/g, "\\t")
+}
diff --git a/packages/server/src/utilities/usageQuota.js b/packages/server/src/utilities/usageQuota.js
index bfe71a4093..80fddb8303 100644
--- a/packages/server/src/utilities/usageQuota.js
+++ b/packages/server/src/utilities/usageQuota.js
@@ -1,41 +1,9 @@
const env = require("../environment")
-const { apiKeyTable } = require("../db/dynamoClient")
-
-const DEFAULT_USAGE = {
- rows: 0,
- storage: 0,
- views: 0,
- automationRuns: 0,
- users: 0,
-}
-
-const DEFAULT_PLAN = {
- rows: 1000,
- // 1 GB
- storage: 8589934592,
- views: 10,
- automationRuns: 100,
- users: 10000,
-}
-
-function buildUpdateParams(key, property, usage) {
- return {
- primary: key,
- condition:
- "attribute_exists(#quota) AND attribute_exists(#limits) AND #quota.#prop < #limits.#prop AND #quotaReset > :now",
- expression: "ADD #quota.#prop :usage",
- names: {
- "#quota": "usageQuota",
- "#prop": property,
- "#limits": "usageLimits",
- "#quotaReset": "quotaReset",
- },
- values: {
- ":usage": usage,
- ":now": Date.now(),
- },
- }
-}
+const { getGlobalDB } = require("@budibase/auth/tenancy")
+const {
+ StaticDatabases,
+ generateNewUsageQuotaDoc,
+} = require("@budibase/auth/db")
function getNewQuotaReset() {
return Date.now() + 2592000000
@@ -47,59 +15,59 @@ exports.Properties = {
VIEW: "views",
USER: "users",
AUTOMATION: "automationRuns",
+ APPS: "apps",
+ EMAILS: "emails",
}
-exports.getAPIKey = async appId => {
- if (!env.USE_QUOTAS) {
- return { apiKey: null }
+async function getUsageQuotaDoc(db) {
+ let quota
+ try {
+ quota = await db.get(StaticDatabases.PLATFORM_INFO.docs.usageQuota)
+ } catch (err) {
+ // doc doesn't exist. Create it
+ quota = await db.post(generateNewUsageQuotaDoc())
}
- return apiKeyTable.get({ primary: appId })
+
+ return quota
}
/**
- * Given a specified API key this will add to the usage object for the specified property.
- * @param {string} apiKey The API key which is to be updated.
+ * Given a specified tenantId this will add to the usage object for the specified property.
* @param {string} property The property which is to be added to (within the nested usageQuota object).
* @param {number} usage The amount (this can be negative) to adjust the number by.
* @returns {Promise} When this completes the API key will now be up to date - the quota period may have
* also been reset after this call.
*/
-exports.update = async (apiKey, property, usage) => {
+exports.update = async (property, usage) => {
if (!env.USE_QUOTAS) {
return
}
+
try {
- await apiKeyTable.update(buildUpdateParams(apiKey, property, usage))
- } catch (err) {
- // conditional check means the condition failed, need to check why
- if (err.code === "ConditionalCheckFailedException") {
- // get the API key so we can check it
- const keyObj = await apiKeyTable.get({ primary: apiKey })
- // the usage quota or usage limits didn't exist
- if (keyObj && (keyObj.usageQuota == null || keyObj.usageLimits == null)) {
- keyObj.usageQuota =
- keyObj.usageQuota == null ? DEFAULT_USAGE : keyObj.usageQuota
- keyObj.usageLimits =
- keyObj.usageLimits == null ? DEFAULT_PLAN : keyObj.usageLimits
- keyObj.quotaReset = getNewQuotaReset()
- await apiKeyTable.put({ item: keyObj })
- return
- }
- // we have in fact breached the reset period
- else if (keyObj && keyObj.quotaReset <= Date.now()) {
- // update the quota reset period and reset the values for all properties
- keyObj.quotaReset = getNewQuotaReset()
- for (let prop of Object.keys(keyObj.usageQuota)) {
- if (prop === property) {
- keyObj.usageQuota[prop] = usage > 0 ? usage : 0
- } else {
- keyObj.usageQuota[prop] = 0
- }
- }
- await apiKeyTable.put({ item: keyObj })
- return
+ const db = getGlobalDB()
+ const quota = await getUsageQuotaDoc(db)
+
+ // Check if the quota needs reset
+ if (Date.now() >= quota.quotaReset) {
+ quota.quotaReset = getNewQuotaReset()
+ for (let prop of Object.keys(quota.usageQuota)) {
+ quota.usageQuota[prop] = 0
}
}
+
+ // increment the quota
+ quota.usageQuota[property] += usage
+
+ if (quota.usageQuota[property] >= quota.usageLimits[property]) {
+ throw new Error(
+ `You have exceeded your usage quota of ${quota.usageLimits[property]} ${property}.`
+ )
+ }
+
+ // update the usage quotas
+ await db.put(quota)
+ } catch (err) {
+ console.error(`Error updating usage quotas for ${property}`, err)
throw err
}
}
diff --git a/packages/server/src/utilities/workerRequests.js b/packages/server/src/utilities/workerRequests.js
index 377658084f..2ace265ca0 100644
--- a/packages/server/src/utilities/workerRequests.js
+++ b/packages/server/src/utilities/workerRequests.js
@@ -34,7 +34,7 @@ function request(ctx, request) {
exports.request = request
// have to pass in the tenant ID as this could be coming from an automation
-exports.sendSmtpEmail = async (to, from, subject, contents) => {
+exports.sendSmtpEmail = async (to, from, subject, contents, automation) => {
// tenant ID will be set in header
const response = await fetch(
checkSlashesInUrl(env.WORKER_URL + `/api/global/email/send`),
@@ -46,6 +46,7 @@ exports.sendSmtpEmail = async (to, from, subject, contents) => {
contents,
subject,
purpose: "custom",
+ automation,
},
})
)
diff --git a/packages/string-templates/package.json b/packages/string-templates/package.json
index 9ea3486427..fadf50abaa 100644
--- a/packages/string-templates/package.json
+++ b/packages/string-templates/package.json
@@ -1,6 +1,6 @@
{
"name": "@budibase/string-templates",
- "version": "0.9.140-alpha.11",
+ "version": "0.9.143-alpha.9",
"description": "Handlebars wrapper for Budibase templating.",
"main": "src/index.cjs",
"module": "dist/bundle.mjs",
diff --git a/packages/worker/package.json b/packages/worker/package.json
index be9ffabce5..908bf39f11 100644
--- a/packages/worker/package.json
+++ b/packages/worker/package.json
@@ -1,7 +1,7 @@
{
"name": "@budibase/worker",
"email": "hi@budibase.com",
- "version": "0.9.140-alpha.11",
+ "version": "0.9.143-alpha.9",
"description": "Budibase background service",
"main": "src/index.js",
"repository": {
@@ -25,8 +25,8 @@
"author": "Budibase",
"license": "AGPL-3.0-or-later",
"dependencies": {
- "@budibase/auth": "^0.9.140-alpha.11",
- "@budibase/string-templates": "^0.9.140-alpha.11",
+ "@budibase/auth": "^0.9.143-alpha.9",
+ "@budibase/string-templates": "^0.9.143-alpha.9",
"@koa/router": "^8.0.0",
"@techpass/passport-openidconnect": "^0.3.0",
"aws-sdk": "^2.811.0",
diff --git a/packages/worker/src/api/controllers/global/configs.js b/packages/worker/src/api/controllers/global/configs.js
index aa83fd695f..c0c300e4db 100644
--- a/packages/worker/src/api/controllers/global/configs.js
+++ b/packages/worker/src/api/controllers/global/configs.js
@@ -10,6 +10,7 @@ const email = require("../../../utilities/email")
const { upload, ObjectStoreBuckets } = require("@budibase/auth").objectStore
const CouchDB = require("../../../db")
const { getGlobalDB } = require("@budibase/auth/tenancy")
+const env = require("../../../environment")
exports.save = async function (ctx) {
const db = getGlobalDB()
@@ -174,7 +175,13 @@ exports.upload = async function (ctx) {
const file = ctx.request.files.file
const { type, name } = ctx.params
- const bucket = ObjectStoreBuckets.GLOBAL
+ let bucket
+ if (env.SELF_HOSTED) {
+ bucket = ObjectStoreBuckets.GLOBAL
+ } else {
+ bucket = ObjectStoreBuckets.GLOBAL_CLOUD
+ }
+
const key = `${type}/${name}`
await upload({
bucket,
diff --git a/packages/worker/src/api/controllers/global/email.js b/packages/worker/src/api/controllers/global/email.js
index 57b78a6d7a..e194a30862 100644
--- a/packages/worker/src/api/controllers/global/email.js
+++ b/packages/worker/src/api/controllers/global/email.js
@@ -2,8 +2,16 @@ const { sendEmail } = require("../../../utilities/email")
const { getGlobalDB } = require("@budibase/auth/tenancy")
exports.sendEmail = async ctx => {
- let { workspaceId, email, userId, purpose, contents, from, subject } =
- ctx.request.body
+ let {
+ workspaceId,
+ email,
+ userId,
+ purpose,
+ contents,
+ from,
+ subject,
+ automation,
+ } = ctx.request.body
let user
if (userId) {
const db = getGlobalDB()
@@ -15,6 +23,7 @@ exports.sendEmail = async ctx => {
contents,
from,
subject,
+ automation,
})
ctx.body = {
...response,
diff --git a/packages/worker/src/api/controllers/global/users.js b/packages/worker/src/api/controllers/global/users.js
index 1375240f34..7753370f09 100644
--- a/packages/worker/src/api/controllers/global/users.js
+++ b/packages/worker/src/api/controllers/global/users.js
@@ -1,8 +1,8 @@
const {
generateGlobalUserID,
getGlobalUserParams,
-
StaticDatabases,
+ generateNewUsageQuotaDoc,
} = require("@budibase/auth/db")
const { hash, getGlobalUserByEmail } = require("@budibase/auth").utils
const { UserStatus, EmailTemplatePurpose } = require("../../../constants")
@@ -11,6 +11,7 @@ const { sendEmail } = require("../../../utilities/email")
const { user: userCache } = require("@budibase/auth/cache")
const { invalidateSessions } = require("@budibase/auth/sessions")
const CouchDB = require("../../../db")
+const accounts = require("@budibase/auth/accounts")
const {
getGlobalDB,
getTenantId,
@@ -18,6 +19,7 @@ const {
tryAddTenant,
updateTenantId,
} = require("@budibase/auth/tenancy")
+const env = require("../../../environment")
const PLATFORM_INFO_DB = StaticDatabases.PLATFORM_INFO.name
@@ -48,10 +50,27 @@ async function saveUser(
// make sure another user isn't using the same email
let dbUser
if (email) {
+ // check budibase users inside the tenant
dbUser = await getGlobalUserByEmail(email)
if (dbUser != null && (dbUser._id !== _id || Array.isArray(dbUser))) {
throw "Email address already in use."
}
+
+ // check budibase users in other tenants
+ if (env.MULTI_TENANCY) {
+ dbUser = await getTenantUser(email)
+ if (dbUser != null) {
+ throw "Email address already in use."
+ }
+ }
+
+ // check root account users in account portal
+ if (!env.SELF_HOSTED) {
+ const account = await accounts.getAccount(email)
+ if (account) {
+ throw "Email address already in use."
+ }
+ }
} else {
dbUser = await db.get(_id)
}
@@ -139,6 +158,11 @@ exports.adminUser = async ctx => {
})
)
+ // write usage quotas for cloud
+ if (!env.SELF_HOSTED) {
+ await db.post(generateNewUsageQuotaDoc())
+ }
+
if (response.rows.some(row => row.doc.admin)) {
ctx.throw(
403,
@@ -261,13 +285,22 @@ exports.find = async ctx => {
ctx.body = user
}
-exports.tenantUserLookup = async ctx => {
- const id = ctx.params.id
- // lookup, could be email or userId, either will return a doc
+// lookup, could be email or userId, either will return a doc
+const getTenantUser = async identifier => {
const db = new CouchDB(PLATFORM_INFO_DB)
try {
- ctx.body = await db.get(id)
+ return await db.get(identifier)
} catch (err) {
+ return null
+ }
+}
+
+exports.tenantUserLookup = async ctx => {
+ const id = ctx.params.id
+ const user = await getTenantUser(id)
+ if (user) {
+ ctx.body = user
+ } else {
ctx.throw(400, "No tenant user found.")
}
}
diff --git a/packages/worker/src/environment.js b/packages/worker/src/environment.js
index 646536f292..28ab4e2e69 100644
--- a/packages/worker/src/environment.js
+++ b/packages/worker/src/environment.js
@@ -33,6 +33,12 @@ module.exports = {
INTERNAL_API_KEY: process.env.INTERNAL_API_KEY,
MULTI_TENANCY: process.env.MULTI_TENANCY,
ACCOUNT_PORTAL_URL: process.env.ACCOUNT_PORTAL_URL,
+ SMTP_FALLBACK_ENABLED: process.env.SMTP_FALLBACK_ENABLED,
+ SMTP_USER: process.env.SMTP_USER,
+ SMTP_PASSWORD: process.env.SMTP_PASSWORD,
+ SMTP_HOST: process.env.SMTP_HOST,
+ SMTP_PORT: process.env.SMTP_PORT,
+ SMTP_FROM_ADDRESS: process.env.SMTP_FROM_ADDRESS,
_set(key, value) {
process.env[key] = value
module.exports[key] = value
diff --git a/packages/worker/src/utilities/email.js b/packages/worker/src/utilities/email.js
index d22933ef36..14c836952e 100644
--- a/packages/worker/src/utilities/email.js
+++ b/packages/worker/src/utilities/email.js
@@ -1,4 +1,5 @@
const nodemailer = require("nodemailer")
+const env = require("../environment")
const { getScopedConfig } = require("@budibase/auth/db")
const { EmailTemplatePurpose, TemplateTypes, Configs } = require("../constants")
const { getTemplateByPurpose } = require("../constants/templates")
@@ -101,16 +102,35 @@ async function buildEmail(purpose, email, context, { user, contents } = {}) {
* Utility function for finding most valid SMTP configuration.
* @param {object} db The CouchDB database which is to be looked up within.
* @param {string|null} workspaceId If using finer grain control of configs a workspace can be used.
+ * @param {boolean|null} automation Whether or not the configuration is being fetched for an email automation.
* @return {Promise} returns the SMTP configuration if it exists
*/
-async function getSmtpConfiguration(db, workspaceId = null) {
+async function getSmtpConfiguration(db, workspaceId = null, automation) {
const params = {
type: Configs.SMTP,
}
if (workspaceId) {
params.workspace = workspaceId
}
- return getScopedConfig(db, params)
+
+ const customConfig = getScopedConfig(db, params)
+
+ if (customConfig) {
+ return customConfig
+ }
+
+ // Use an SMTP fallback configuration from env variables
+ if (!automation && env.SMTP_FALLBACK_ENABLED) {
+ return {
+ port: env.SMTP_PORT,
+ host: env.SMTP_HOST,
+ secure: false,
+ auth: {
+ user: env.SMTP_USER,
+ pass: env.SMTP_PASSWORD,
+ },
+ }
+ }
}
/**
@@ -118,8 +138,8 @@ async function getSmtpConfiguration(db, workspaceId = null) {
* @return {Promise} returns true if there is a configuration that can be used.
*/
exports.isEmailConfigured = async (workspaceId = null) => {
- // when "testing" simply return true
- if (TEST_MODE) {
+ // when "testing" or smtp fallback is enabled simply return true
+ if (TEST_MODE || env.SMTP_FALLBACK_ENABLED) {
return true
}
const db = getGlobalDB()
@@ -138,16 +158,17 @@ exports.isEmailConfigured = async (workspaceId = null) => {
* @param {string|undefined} contents If sending a custom email then can supply contents which will be added to it.
* @param {string|undefined} subject A custom subject can be specified if the config one is not desired.
* @param {object|undefined} info Pass in a structure of information to be stored alongside the invitation.
+ * @param {boolean|undefined} disableFallback Prevent email being sent from SMTP fallback to avoid spam.
* @return {Promise