MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'develop' of github.com:Budibase/budibase into side-panel

This commit is contained in:
Andrew Kingston 2022-11-21 10:30:25 +00:00
parent 75ee5117ea cc3b174f87
commit c58067a589
267 changed files with 7443 additions and 8029 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.eslintignore
View File

@ -3,6 +3,8 @@ public
dist
packages/server/builder
packages/server/coverage
packages/worker/coverage
packages/backend-core/coverage
packages/server/client
packages/builder/.routify
packages/builder/cypress/support/queryLevelTransformerFunction.js

2
.github/workflows/budibase_ci.yml vendored
View File

@ -58,7 +58,7 @@ jobs:
- uses: codecov/codecov-action@v1
with:
token: ${{ secrets.CODECOV_TOKEN }} # not required for public repos
files: ./packages/server/coverage/clover.xml
files: ./packages/server/coverage/clover.xml,./packages/worker/coverage/clover.xml,./packages/backend-core/coverage/clover.xml
name: codecov-umbrella
verbose: true

2
.prettierignore
View File

@ -4,6 +4,8 @@ dist
packages/builder/src/components/design/AppPreview/CurrentItemPreview.svelte
packages/server/builder
packages/server/coverage
packages/worker/coverage
packages/backend-core/coverage
packages/server/client
packages/server/src/definitions/openapi.ts
packages/builder/.routify

39
hosting/nginx.dev.conf.hbs
View File

@ -58,12 +58,15 @@ http {
}
location ~ ^/api/(system|admin|global)/ {
proxy_pass http://worker-service;
proxy_read_timeout 120s;
proxy_connect_timeout 120s;
proxy_send_timeout 120s;
proxy_http_version 1.1;
proxy_set_header Host $host;
proxy_set_header Connection "";
proxy_pass http://worker-service;
}
location /api/backups/ {
@ -78,60 +81,78 @@ http {
location /api/ {
proxy_read_timeout 120s;
proxy_connect_timeout 120s;
proxy_send_timeout 120s;
proxy_pass http://app-service;
proxy_send_timeout 120s;
proxy_http_version 1.1;
proxy_set_header Host $host;
proxy_set_header Connection "";
proxy_pass http://app-service;
}
location = / {
proxy_pass http://app-service;
proxy_read_timeout 120s;
proxy_connect_timeout 120s;
proxy_send_timeout 120s;
proxy_http_version 1.1;
proxy_set_header Host $host;
proxy_set_header Connection "";
proxy_pass http://app-service;
}
location /app_ {
proxy_pass http://app-service;
proxy_read_timeout 120s;
proxy_connect_timeout 120s;
proxy_send_timeout 120s;
proxy_http_version 1.1;
proxy_set_header Host $host;
proxy_set_header Connection "";
proxy_pass http://app-service;
}
location /app {
proxy_pass http://app-service;
proxy_read_timeout 120s;
proxy_connect_timeout 120s;
proxy_send_timeout 120s;
proxy_http_version 1.1;
proxy_set_header Host $host;
proxy_set_header Connection "";
proxy_pass http://app-service;
}
location /builder {
proxy_pass http://builder;
proxy_read_timeout 120s;
proxy_connect_timeout 120s;
proxy_send_timeout 120s;
proxy_http_version 1.1;
proxy_set_header Host $host;
proxy_set_header Connection "";
proxy_pass http://builder;
rewrite ^/builder(.*)$ /builder/$1 break;
}
location /builder/ {
proxy_pass http://builder;
proxy_http_version 1.1;
proxy_set_header Host $host;
proxy_set_header Connection $connection_upgrade;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_read_timeout 120s;
proxy_connect_timeout 120s;
proxy_send_timeout 120s;
proxy_pass http://builder;
}
location /vite/ {

9
hosting/nginx.prod.conf.hbs
View File

@ -100,18 +100,25 @@ http {
location ~ ^/(builder|app_) {
proxy_http_version 1.1;
proxy_set_header Connection $connection_upgrade;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header Host $host;
proxy_pass http://$apps:4002;
}
location ~ ^/api/(system|admin|global)/ {
proxy_set_header Host $host;
proxy_pass http://$worker:4003;
}
location /worker/ {
proxy_set_header Host $host;
proxy_pass http://$worker:4003;
rewrite ^/worker/(.*)$ /$1 break;
}
@ -139,6 +146,7 @@ http {
proxy_set_header Upgrade $http_upgrade;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header Host $host;
proxy_pass http://$apps:4002;
}
@ -158,6 +166,7 @@ http {
proxy_set_header Upgrade $http_upgrade;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header Host $host;
proxy_pass http://$apps:4002;
}

2
lerna.json
View File

@ -1,5 +1,5 @@
{
"version": "2.1.22-alpha.2",
"version": "2.1.22-alpha.8",
"npmClient": "yarn",
"packages": [
"packages/*"

6
package.json
View File

@ -18,7 +18,7 @@
"rimraf": "^3.0.2",
"rollup-plugin-replace": "^2.2.0",
"svelte": "^3.38.2",
"typescript": "4.5.5"
"typescript": "4.7.3"
},
"scripts": {
"setup": "node ./hosting/scripts/setup.js && yarn && yarn bootstrap && yarn build && yarn dev",
@ -75,8 +75,8 @@
"env:multi:disable": "lerna run env:multi:disable",
"env:selfhost:enable": "lerna run env:selfhost:enable",
"env:selfhost:disable": "lerna run env:selfhost:disable",
"env:localdomain:enable": "lerna run env:localdomain:enable",
"env:localdomain:disable": "lerna run env:localdomain:disable",
"env:localdomain:enable": "./scripts/localdomain.sh enable",
"env:localdomain:disable": "./scripts/localdomain.sh disable",
"env:account:enable": "lerna run env:account:enable",
"env:account:disable": "lerna run env:account:disable",
"mode:self": "yarn env:selfhost:enable && yarn env:multi:disable && yarn env:account:disable",

15
packages/backend-core/__mocks__/aws-sdk.ts Normal file
View File

@ -0,0 +1,15 @@
const mockS3 = {
headBucket: jest.fn().mockReturnThis(),
deleteObject: jest.fn().mockReturnThis(),
deleteObjects: jest.fn().mockReturnThis(),
createBucket: jest.fn().mockReturnThis(),
listObjects: jest.fn().mockReturnThis(),
promise: jest.fn().mockReturnThis(),
catch: jest.fn(),
}
const AWS = {
S3: jest.fn(() => mockS3),
}
export default AWS

8
packages/backend-core/db.js
View File

@ -1,7 +1 @@
module.exports = {
...require("./src/db/utils"),
...require("./src/db/constants"),
...require("./src/db"),
...require("./src/db/views"),
...require("./src/db/pouch"),
}
module.exports = require("./src/db")

21
packages/backend-core/jest.config.ts Normal file
View File

@ -0,0 +1,21 @@
import { Config } from "@jest/types"
const config: Config.InitialOptions = {
preset: "ts-jest",
testEnvironment: "node",
setupFiles: ["./tests/jestSetup.ts"],
collectCoverageFrom: ["src/**/*.{js,ts}"],
coverageReporters: ["lcov", "json", "clover"],
}
if (!process.env.CI) {
// use sources when not in CI
config.moduleNameMapper = {
"@budibase/types": "<rootDir>/../types/src",
"^axios.*$": "<rootDir>/node_modules/axios/lib/axios.js",
}
} else {
console.log("Running tests with compiled dependency sources")
}
export default config

27
packages/backend-core/package.json
View File

@ -1,6 +1,6 @@
{
"name": "@budibase/backend-core",
"version": "2.1.22-alpha.2",
"version": "2.1.22-alpha.8",
"description": "Budibase backend core libraries used in server and worker",
"main": "dist/src/index.js",
"types": "dist/src/index.d.ts",
@ -16,11 +16,11 @@
"prepack": "cp package.json dist",
"build": "tsc -p tsconfig.build.json",
"build:dev": "yarn prebuild && tsc --build --watch --preserveWatchOutput",
"test": "jest",
"test": "jest --coverage",
"test:watch": "jest --watchAll"
},
"dependencies": {
"@budibase/types": "2.1.22-alpha.2",
"@budibase/types": "2.1.22-alpha.8",
"@shopify/jest-koa-mocks": "5.0.1",
"@techpass/passport-openidconnect": "0.3.2",
"aws-sdk": "2.1030.0",
@ -35,6 +35,7 @@
"koa-passport": "4.1.4",
"lodash": "4.17.21",
"lodash.isarguments": "3.1.0",
"nano": "^10.1.0",
"node-fetch": "2.6.7",
"passport-google-auth": "1.0.2",
"passport-google-oauth": "2.0.0",
@ -52,21 +53,11 @@
"uuid": "8.3.2",
"zlib": "1.0.5"
},
"jest": {
"preset": "ts-jest",
"testEnvironment": "node",
"moduleNameMapper": {
"@budibase/types": "<rootDir>/../types/src"
},
"setupFiles": [
"./scripts/jestSetup.ts"
]
},
"devDependencies": {
"@types/chance": "1.1.3",
"@types/ioredis": "4.28.0",
"@types/jest": "27.5.1",
"@types/koa": "2.0.52",
"@types/koa": "2.13.4",
"@types/lodash": "4.14.180",
"@types/node": "14.18.20",
"@types/node-fetch": "2.6.1",
@ -77,12 +68,14 @@
"@types/uuid": "8.3.4",
"chance": "1.1.3",
"ioredis-mock": "5.8.0",
"jest": "27.5.1",
"koa": "2.7.0",
"jest": "28.1.1",
"koa": "2.13.4",
"nodemon": "2.0.16",
"pouchdb-adapter-memory": "7.2.2",
"timekeeper": "2.2.0",
"ts-jest": "27.1.5",
"ts-jest": "28.0.4",
"ts-node": "10.8.1",
"tsconfig-paths": "4.0.0",
"typescript": "4.7.3"
},
"gitHead": "d1836a898cab3f8ab80ee6d8f42be1a9eed7dcdc"

12
packages/backend-core/scripts/jestSetup.ts
View File

@ -1,12 +0,0 @@
import env from "../src/environment"
import { mocks } from "../tests/utilities"
// mock all dates to 2020-01-01T00:00:00.000Z
// use tk.reset() to use real dates in individual tests
import tk from "timekeeper"
tk.freeze(mocks.date.MOCK_DATE)
env._set("SELF_HOSTED", "1")
env._set("NODE_ENV", "jest")
env._set("JWT_SECRET", "test-jwtsecret")
env._set("LOG_LEVEL", "silent")

8
packages/backend-core/src/auth.ts
View File

@ -3,7 +3,7 @@ const LocalStrategy = require("passport-local").Strategy
const JwtStrategy = require("passport-jwt").Strategy
import { getGlobalDB } from "./tenancy"
const refresh = require("passport-oauth2-refresh")
import { Configs } from "./constants"
import { Config } from "./constants"
import { getScopedConfig } from "./db/utils"
import {
jwt,
@ -76,7 +76,7 @@ async function refreshOIDCAccessToken(
return new Promise(resolve => {
refresh.requestNewAccessToken(
Configs.OIDC,
Config.OIDC,
refreshToken,
(err: any, accessToken: string, refreshToken: any, params: any) => {
resolve({ err, accessToken, refreshToken, params })
@ -106,7 +106,7 @@ async function refreshGoogleAccessToken(
return new Promise(resolve => {
refresh.requestNewAccessToken(
Configs.GOOGLE,
Config.GOOGLE,
refreshToken,
(err: any, accessToken: string, refreshToken: string, params: any) => {
resolve({ err, accessToken, refreshToken, params })
@ -129,7 +129,7 @@ async function refreshOAuthToken(
let chosenConfig = {}
let refreshResponse
if (configType === Configs.OIDC) {
if (configType === Config.OIDC) {
// configId - retrieved from cookie.
chosenConfig = config.configs.filter((c: any) => c.uuid === configId)[0]
if (!chosenConfig) {

8
packages/backend-core/src/cache/tests/writethrough.spec.js vendored
View File

@ -1,6 +1,6 @@
require("../../../tests/utilities/TestConfiguration")
require("../../../tests")
const { Writethrough } = require("../writethrough")
const { dangerousGetDB } = require("../../db")
const { getDB } = require("../../db")
const tk = require("timekeeper")
const START_DATE = Date.now()
@ -8,8 +8,8 @@ tk.freeze(START_DATE)
const DELAY = 5000
const db = dangerousGetDB("test")
const db2 = dangerousGetDB("test2")
const db = getDB("test")
const db2 = getDB("test2")
const writethrough = new Writethrough(db, DELAY), writethrough2 = new Writethrough(db2, DELAY)
describe("writethrough", () => {

17
packages/backend-core/src/cache/writethrough.ts vendored
View File

@ -1,7 +1,7 @@
import BaseCache from "./base"
import { getWritethroughClient } from "../redis/init"
import { logWarn } from "../logging"
import PouchDB from "pouchdb"
import { Database } from "@budibase/types"
const DEFAULT_WRITE_RATE_MS = 10000
let CACHE: BaseCache | null = null
@ -19,7 +19,7 @@ async function getCache() {
return CACHE
}
function makeCacheKey(db: PouchDB.Database, key: string) {
function makeCacheKey(db: Database, key: string) {
return db.name + key
}
@ -28,7 +28,7 @@ function makeCacheItem(doc: any, lastWrite: number | null = null): CacheItem {
}
export async function put(
db: PouchDB.Database,
db: Database,
doc: any,
writeRateMs: number = DEFAULT_WRITE_RATE_MS
) {
@ -64,7 +64,7 @@ export async function put(
return { ok: true, id: output._id, rev: output._rev }
}
export async function get(db: PouchDB.Database, id: string): Promise<any> {
export async function get(db: Database, id: string): Promise<any> {
const cache = await getCache()
const cacheKey = makeCacheKey(db, id)
let cacheItem: CacheItem = await cache.get(cacheKey)
@ -77,7 +77,7 @@ export async function get(db: PouchDB.Database, id: string): Promise<any> {
}
export async function remove(
db: PouchDB.Database,
db: Database,
docOrId: any,
rev?: any
): Promise<void> {
@ -95,13 +95,10 @@ export async function remove(
}
export class Writethrough {
db: PouchDB.Database
db: Database
writeRateMs: number
constructor(
db: PouchDB.Database,
writeRateMs: number = DEFAULT_WRITE_RATE_MS
) {
constructor(db: Database, writeRateMs: number = DEFAULT_WRITE_RATE_MS) {
this.db = db
this.writeRateMs = writeRateMs
}

8
packages/backend-core/src/cloud/accounts.ts
View File

@ -1,6 +1,6 @@
import API from "./api"
import env from "../environment"
import { Headers } from "../constants"
import { Header } from "../constants"
import { CloudAccount } from "@budibase/types"
const api = new API(env.ACCOUNT_PORTAL_URL)
@ -14,7 +14,7 @@ export const getAccount = async (
const response = await api.post(`/api/accounts/search`, {
body: payload,
headers: {
[Headers.API_KEY]: env.ACCOUNT_PORTAL_API_KEY,
[Header.API_KEY]: env.ACCOUNT_PORTAL_API_KEY,
},
})
@ -35,7 +35,7 @@ export const getAccountByTenantId = async (
const response = await api.post(`/api/accounts/search`, {
body: payload,
headers: {
[Headers.API_KEY]: env.ACCOUNT_PORTAL_API_KEY,
[Header.API_KEY]: env.ACCOUNT_PORTAL_API_KEY,
},
})
@ -50,7 +50,7 @@ export const getAccountByTenantId = async (
export const getStatus = async () => {
const response = await api.get(`/api/status`, {
headers: {
[Headers.API_KEY]: env.ACCOUNT_PORTAL_API_KEY,
[Header.API_KEY]: env.ACCOUNT_PORTAL_API_KEY,
},
})
const json = await response.json()

650
packages/backend-core/src/clshooked/index.js
View File

@ -1,650 +0,0 @@
const util = require("util")
const assert = require("assert")
const wrapEmitter = require("emitter-listener")
const async_hooks = require("async_hooks")
const CONTEXTS_SYMBOL = "cls@contexts"
const ERROR_SYMBOL = "error@context"
const DEBUG_CLS_HOOKED = process.env.DEBUG_CLS_HOOKED
let currentUid = -1
module.exports = {
getNamespace: getNamespace,
createNamespace: createNamespace,
destroyNamespace: destroyNamespace,
reset: reset,
ERROR_SYMBOL: ERROR_SYMBOL,
}
function Namespace(name) {
this.name = name
// changed in 2.7: no default context
this.active = null
this._set = []
this.id = null
this._contexts = new Map()
this._indent = 0
this._hook = null
}
Namespace.prototype.set = function set(key, value) {
if (!this.active) {
throw new Error(
"No context available. ns.run() or ns.bind() must be called first."
)
}
this.active[key] = value
if (DEBUG_CLS_HOOKED) {
const indentStr = " ".repeat(this._indent < 0 ? 0 : this._indent)
debug2(
indentStr +
"CONTEXT-SET KEY:" +
key +
"=" +
value +
" in ns:" +
this.name +
" currentUid:" +
currentUid +
" active:" +
util.inspect(this.active, { showHidden: true, depth: 2, colors: true })
)
}
return value
}
Namespace.prototype.get = function get(key) {
if (!this.active) {
if (DEBUG_CLS_HOOKED) {
const asyncHooksCurrentId = async_hooks.currentId()
const triggerId = async_hooks.triggerAsyncId()
const indentStr = " ".repeat(this._indent < 0 ? 0 : this._indent)
debug2(
`${indentStr}CONTEXT-GETTING KEY NO ACTIVE NS: (${this.name}) ${key}=undefined currentUid:${currentUid} asyncHooksCurrentId:${asyncHooksCurrentId} triggerId:${triggerId} len:${this._set.length}`
)
}
return undefined
}
if (DEBUG_CLS_HOOKED) {
const asyncHooksCurrentId = async_hooks.executionAsyncId()
const triggerId = async_hooks.triggerAsyncId()
const indentStr = " ".repeat(this._indent < 0 ? 0 : this._indent)
debug2(
indentStr +
"CONTEXT-GETTING KEY:" +
key +
"=" +
this.active[key] +
" (" +
this.name +
") currentUid:" +
currentUid +
" active:" +
util.inspect(this.active, { showHidden: true, depth: 2, colors: true })
)
debug2(
`${indentStr}CONTEXT-GETTING KEY: (${this.name}) ${key}=${
this.active[key]
} currentUid:${currentUid} asyncHooksCurrentId:${asyncHooksCurrentId} triggerId:${triggerId} len:${
this._set.length
} active:${util.inspect(this.active)}`
)
}
return this.active[key]
}
Namespace.prototype.createContext = function createContext() {
// Prototype inherit existing context if created a new child context within existing context.
let context = Object.create(this.active ? this.active : Object.prototype)
context._ns_name = this.name
context.id = currentUid
if (DEBUG_CLS_HOOKED) {
const asyncHooksCurrentId = async_hooks.executionAsyncId()
const triggerId = async_hooks.triggerAsyncId()
const indentStr = " ".repeat(this._indent < 0 ? 0 : this._indent)
debug2(
`${indentStr}CONTEXT-CREATED Context: (${
this.name
}) currentUid:${currentUid} asyncHooksCurrentId:${asyncHooksCurrentId} triggerId:${triggerId} len:${
this._set.length
} context:${util.inspect(context, {
showHidden: true,
depth: 2,
colors: true,
})}`
)
}
return context
}
Namespace.prototype.run = function run(fn) {
let context = this.createContext()
this.enter(context)
try {
if (DEBUG_CLS_HOOKED) {
const triggerId = async_hooks.triggerAsyncId()
const asyncHooksCurrentId = async_hooks.executionAsyncId()
const indentStr = " ".repeat(this._indent < 0 ? 0 : this._indent)
debug2(
`${indentStr}CONTEXT-RUN BEGIN: (${
this.name
}) currentUid:${currentUid} triggerId:${triggerId} asyncHooksCurrentId:${asyncHooksCurrentId} len:${
this._set.length
} context:${util.inspect(context)}`
)
}
fn(context)
return context
} catch (exception) {
if (exception) {
exception[ERROR_SYMBOL] = context
}
throw exception
} finally {
if (DEBUG_CLS_HOOKED) {
const triggerId = async_hooks.triggerAsyncId()
const asyncHooksCurrentId = async_hooks.executionAsyncId()
const indentStr = " ".repeat(this._indent < 0 ? 0 : this._indent)
debug2(
`${indentStr}CONTEXT-RUN END: (${
this.name
}) currentUid:${currentUid} triggerId:${triggerId} asyncHooksCurrentId:${asyncHooksCurrentId} len:${
this._set.length
} ${util.inspect(context)}`
)
}
this.exit(context)
}
}
Namespace.prototype.runAndReturn = function runAndReturn(fn) {
let value
this.run(function (context) {
value = fn(context)
})
return value
}
/**
* Uses global Promise and assumes Promise is cls friendly or wrapped already.
* @param {function} fn
* @returns {*}
*/
Namespace.prototype.runPromise = function runPromise(fn) {
let context = this.createContext()
this.enter(context)
let promise = fn(context)
if (!promise || !promise.then || !promise.catch) {
throw new Error("fn must return a promise.")
}
if (DEBUG_CLS_HOOKED) {
debug2(
"CONTEXT-runPromise BEFORE: (" +
this.name +
") currentUid:" +
currentUid +
" len:" +
this._set.length +
" " +
util.inspect(context)
)
}
return promise
.then(result => {
if (DEBUG_CLS_HOOKED) {
debug2(
"CONTEXT-runPromise AFTER then: (" +
this.name +
") currentUid:" +
currentUid +
" len:" +
this._set.length +
" " +
util.inspect(context)
)
}
this.exit(context)
return result
})
.catch(err => {
err[ERROR_SYMBOL] = context
if (DEBUG_CLS_HOOKED) {
debug2(
"CONTEXT-runPromise AFTER catch: (" +
this.name +
") currentUid:" +
currentUid +
" len:" +
this._set.length +
" " +
util.inspect(context)
)
}
this.exit(context)
throw err
})
}
Namespace.prototype.bind = function bindFactory(fn, context) {
if (!context) {
if (!this.active) {
context = this.createContext()
} else {
context = this.active
}
}
let self = this
return function clsBind() {
self.enter(context)
try {
return fn.apply(this, arguments)
} catch (exception) {
if (exception) {
exception[ERROR_SYMBOL] = context
}
throw exception
} finally {
self.exit(context)
}
}
}
Namespace.prototype.enter = function enter(context) {
assert.ok(context, "context must be provided for entering")
if (DEBUG_CLS_HOOKED) {
const asyncHooksCurrentId = async_hooks.executionAsyncId()
const triggerId = async_hooks.triggerAsyncId()
const indentStr = " ".repeat(this._indent < 0 ? 0 : this._indent)
debug2(
`${indentStr}CONTEXT-ENTER: (${
this.name
}) currentUid:${currentUid} triggerId:${triggerId} asyncHooksCurrentId:${asyncHooksCurrentId} len:${
this._set.length
} ${util.inspect(context)}`
)
}
this._set.push(this.active)
this.active = context
}
Namespace.prototype.exit = function exit(context) {
assert.ok(context, "context must be provided for exiting")
if (DEBUG_CLS_HOOKED) {
const asyncHooksCurrentId = async_hooks.executionAsyncId()
const triggerId = async_hooks.triggerAsyncId()
const indentStr = " ".repeat(this._indent < 0 ? 0 : this._indent)
debug2(
`${indentStr}CONTEXT-EXIT: (${
this.name
}) currentUid:${currentUid} triggerId:${triggerId} asyncHooksCurrentId:${asyncHooksCurrentId} len:${
this._set.length
} ${util.inspect(context)}`
)
}
// Fast path for most exits that are at the top of the stack
if (this.active === context) {
assert.ok(this._set.length, "can't remove top context")
this.active = this._set.pop()
return
}
// Fast search in the stack using lastIndexOf
let index = this._set.lastIndexOf(context)
if (index < 0) {
if (DEBUG_CLS_HOOKED) {
debug2(
"??ERROR?? context exiting but not entered - ignoring: " +
util.inspect(context)
)
}
assert.ok(
index >= 0,
"context not currently entered; can't exit. \n" +
util.inspect(this) +
"\n" +
util.inspect(context)
)
} else {
assert.ok(index, "can't remove top context")
this._set.splice(index, 1)
}
}
Namespace.prototype.bindEmitter = function bindEmitter(emitter) {
assert.ok(
emitter.on && emitter.addListener && emitter.emit,
"can only bind real EEs"
)
let namespace = this
let thisSymbol = "context@" + this.name
// Capture the context active at the time the emitter is bound.
function attach(listener) {
if (!listener) {
return
}
if (!listener[CONTEXTS_SYMBOL]) {
listener[CONTEXTS_SYMBOL] = Object.create(null)
}
listener[CONTEXTS_SYMBOL][thisSymbol] = {
namespace: namespace,
context: namespace.active,
}
}
// At emit time, bind the listener within the correct context.
function bind(unwrapped) {
if (!(unwrapped && unwrapped[CONTEXTS_SYMBOL])) {
return unwrapped
}
let wrapped = unwrapped
let unwrappedContexts = unwrapped[CONTEXTS_SYMBOL]
Object.keys(unwrappedContexts).forEach(function (name) {
let thunk = unwrappedContexts[name]
wrapped = thunk.namespace.bind(wrapped, thunk.context)
})
return wrapped
}
wrapEmitter(emitter, attach, bind)
}
/**
* If an error comes out of a namespace, it will have a context attached to it.
* This function knows how to find it.
*
* @param {Error} exception Possibly annotated error.
*/
Namespace.prototype.fromException = function fromException(exception) {
return exception[ERROR_SYMBOL]
}
function getNamespace(name) {
return process.namespaces[name]
}
function createNamespace(name) {
assert.ok(name, "namespace must be given a name.")
if (DEBUG_CLS_HOOKED) {
debug2(`NS-CREATING NAMESPACE (${name})`)
}
let namespace = new Namespace(name)
namespace.id = currentUid
const hook = async_hooks.createHook({
init(asyncId, type, triggerId, resource) {
currentUid = async_hooks.executionAsyncId()
//CHAIN Parent's Context onto child if none exists. This is needed to pass net-events.spec
// let initContext = namespace.active;
// if(!initContext && triggerId) {
// let parentContext = namespace._contexts.get(triggerId);
// if (parentContext) {
// namespace.active = parentContext;
// namespace._contexts.set(currentUid, parentContext);
// if (DEBUG_CLS_HOOKED) {
// const indentStr = ' '.repeat(namespace._indent < 0 ? 0 : namespace._indent);
// debug2(`${indentStr}INIT [${type}] (${name}) WITH PARENT CONTEXT asyncId:${asyncId} currentUid:${currentUid} triggerId:${triggerId} active:${util.inspect(namespace.active, true)} resource:${resource}`);
// }
// } else if (DEBUG_CLS_HOOKED) {
// const indentStr = ' '.repeat(namespace._indent < 0 ? 0 : namespace._indent);
// debug2(`${indentStr}INIT [${type}] (${name}) MISSING CONTEXT asyncId:${asyncId} currentUid:${currentUid} triggerId:${triggerId} active:${util.inspect(namespace.active, true)} resource:${resource}`);
// }
// }else {
// namespace._contexts.set(currentUid, namespace.active);
// if (DEBUG_CLS_HOOKED) {
// const indentStr = ' '.repeat(namespace._indent < 0 ? 0 : namespace._indent);
// debug2(`${indentStr}INIT [${type}] (${name}) asyncId:${asyncId} currentUid:${currentUid} triggerId:${triggerId} active:${util.inspect(namespace.active, true)} resource:${resource}`);
// }
// }
if (namespace.active) {
namespace._contexts.set(asyncId, namespace.active)
if (DEBUG_CLS_HOOKED) {
const indentStr = " ".repeat(
namespace._indent < 0 ? 0 : namespace._indent
)
debug2(
`${indentStr}INIT [${type}] (${name}) asyncId:${asyncId} currentUid:${currentUid} triggerId:${triggerId} active:${util.inspect(
namespace.active,
{ showHidden: true, depth: 2, colors: true }
)} resource:${resource}`
)
}
} else if (currentUid === 0) {
// CurrentId will be 0 when triggered from C++. Promise events
// https://github.com/nodejs/node/blob/master/doc/api/async_hooks.md#triggerid
const triggerId = async_hooks.triggerAsyncId()
const triggerIdContext = namespace._contexts.get(triggerId)
if (triggerIdContext) {
namespace._contexts.set(asyncId, triggerIdContext)
if (DEBUG_CLS_HOOKED) {
const indentStr = " ".repeat(
namespace._indent < 0 ? 0 : namespace._indent
)
debug2(
`${indentStr}INIT USING CONTEXT FROM TRIGGERID [${type}] (${name}) asyncId:${asyncId} currentUid:${currentUid} triggerId:${triggerId} active:${util.inspect(
namespace.active,
{ showHidden: true, depth: 2, colors: true }
)} resource:${resource}`
)
}
} else if (DEBUG_CLS_HOOKED) {
const indentStr = " ".repeat(
namespace._indent < 0 ? 0 : namespace._indent
)
debug2(
`${indentStr}INIT MISSING CONTEXT [${type}] (${name}) asyncId:${asyncId} currentUid:${currentUid} triggerId:${triggerId} active:${util.inspect(
namespace.active,
{ showHidden: true, depth: 2, colors: true }
)} resource:${resource}`
)
}
}
if (DEBUG_CLS_HOOKED && type === "PROMISE") {
debug2(util.inspect(resource, { showHidden: true }))
const parentId = resource.parentId
const indentStr = " ".repeat(
namespace._indent < 0 ? 0 : namespace._indent
)
debug2(
`${indentStr}INIT RESOURCE-PROMISE [${type}] (${name}) parentId:${parentId} asyncId:${asyncId} currentUid:${currentUid} triggerId:${triggerId} active:${util.inspect(
namespace.active,
{ showHidden: true, depth: 2, colors: true }
)} resource:${resource}`
)
}
},
before(asyncId) {
currentUid = async_hooks.executionAsyncId()
let context
/*
if(currentUid === 0){
// CurrentId will be 0 when triggered from C++. Promise events
// https://github.com/nodejs/node/blob/master/doc/api/async_hooks.md#triggerid
//const triggerId = async_hooks.triggerAsyncId();
context = namespace._contexts.get(asyncId); // || namespace._contexts.get(triggerId);
}else{
context = namespace._contexts.get(currentUid);
}
*/
//HACK to work with promises until they are fixed in node > 8.1.1
context =
namespace._contexts.get(asyncId) || namespace._contexts.get(currentUid)
if (context) {
if (DEBUG_CLS_HOOKED) {
const triggerId = async_hooks.triggerAsyncId()
const indentStr = " ".repeat(
namespace._indent < 0 ? 0 : namespace._indent
)
debug2(
`${indentStr}BEFORE (${name}) asyncId:${asyncId} currentUid:${currentUid} triggerId:${triggerId} active:${util.inspect(
namespace.active,
{ showHidden: true, depth: 2, colors: true }
)} context:${util.inspect(context)}`
)
namespace._indent += 2
}
namespace.enter(context)
} else if (DEBUG_CLS_HOOKED) {
const triggerId = async_hooks.triggerAsyncId()
const indentStr = " ".repeat(
namespace._indent < 0 ? 0 : namespace._indent
)
debug2(
`${indentStr}BEFORE MISSING CONTEXT (${name}) asyncId:${asyncId} currentUid:${currentUid} triggerId:${triggerId} active:${util.inspect(
namespace.active,
{ showHidden: true, depth: 2, colors: true }
)} namespace._contexts:${util.inspect(namespace._contexts, {
showHidden: true,
depth: 2,
colors: true,
})}`
)
namespace._indent += 2
}
},
after(asyncId) {
currentUid = async_hooks.executionAsyncId()
let context // = namespace._contexts.get(currentUid);
/*
if(currentUid === 0){
// CurrentId will be 0 when triggered from C++. Promise events
// https://github.com/nodejs/node/blob/master/doc/api/async_hooks.md#triggerid
//const triggerId = async_hooks.triggerAsyncId();
context = namespace._contexts.get(asyncId); // || namespace._contexts.get(triggerId);
}else{
context = namespace._contexts.get(currentUid);
}
*/
//HACK to work with promises until they are fixed in node > 8.1.1
context =
namespace._contexts.get(asyncId) || namespace._contexts.get(currentUid)
if (context) {
if (DEBUG_CLS_HOOKED) {
const triggerId = async_hooks.triggerAsyncId()
namespace._indent -= 2
const indentStr = " ".repeat(
namespace._indent < 0 ? 0 : namespace._indent
)
debug2(
`${indentStr}AFTER (${name}) asyncId:${asyncId} currentUid:${currentUid} triggerId:${triggerId} active:${util.inspect(
namespace.active,
{ showHidden: true, depth: 2, colors: true }
)} context:${util.inspect(context)}`
)
}
namespace.exit(context)
} else if (DEBUG_CLS_HOOKED) {
const triggerId = async_hooks.triggerAsyncId()
namespace._indent -= 2
const indentStr = " ".repeat(
namespace._indent < 0 ? 0 : namespace._indent
)
debug2(
`${indentStr}AFTER MISSING CONTEXT (${name}) asyncId:${asyncId} currentUid:${currentUid} triggerId:${triggerId} active:${util.inspect(
namespace.active,
{ showHidden: true, depth: 2, colors: true }
)} context:${util.inspect(context)}`
)
}
},
destroy(asyncId) {
currentUid = async_hooks.executionAsyncId()
if (DEBUG_CLS_HOOKED) {
const triggerId = async_hooks.triggerAsyncId()
const indentStr = " ".repeat(
namespace._indent < 0 ? 0 : namespace._indent
)
debug2(
`${indentStr}DESTROY (${name}) currentUid:${currentUid} asyncId:${asyncId} triggerId:${triggerId} active:${util.inspect(
namespace.active,
{ showHidden: true, depth: 2, colors: true }
)} context:${util.inspect(namespace._contexts.get(currentUid))}`
)
}
namespace._contexts.delete(asyncId)
},
})
hook.enable()
namespace._hook = hook
process.namespaces[name] = namespace
return namespace
}
function destroyNamespace(name) {
let namespace = getNamespace(name)
assert.ok(namespace, "can't delete nonexistent namespace! \"" + name + '"')
assert.ok(
namespace.id,
"don't assign to process.namespaces directly! " + util.inspect(namespace)
)
namespace._hook.disable()
namespace._contexts = null
process.namespaces[name] = null
}
function reset() {
// must unregister async listeners
if (process.namespaces) {
Object.keys(process.namespaces).forEach(function (name) {
destroyNamespace(name)
})
}
process.namespaces = Object.create(null)
}
process.namespaces = process.namespaces || {}
//const fs = require('fs');
function debug2(...args) {
if (DEBUG_CLS_HOOKED) {
//fs.writeSync(1, `${util.format(...args)}\n`);
process._rawDebug(`${util.format(...args)}`)
}
}
/*function getFunctionName(fn) {
if (!fn) {
return fn;
}
if (typeof fn === 'function') {
if (fn.name) {
return fn.name;
}
return (fn.toString().trim().match(/^function\s*([^\s(]+)/) || [])[1];
} else if (fn.constructor && fn.constructor.name) {
return fn.constructor.name;
}
}*/

44
packages/backend-core/src/constants.js
View File

@ -1,44 +0,0 @@
exports.UserStatus = {
ACTIVE: "active",
INACTIVE: "inactive",
}
exports.Cookies = {
CurrentApp: "budibase:currentapp",
Auth: "budibase:auth",
Init: "budibase:init",
ACCOUNT_RETURN_URL: "budibase:account:returnurl",
DatasourceAuth: "budibase:datasourceauth",
OIDC_CONFIG: "budibase:oidc:config",
}
exports.Headers = {
API_KEY: "x-budibase-api-key",
LICENSE_KEY: "x-budibase-license-key",
API_VER: "x-budibase-api-version",
APP_ID: "x-budibase-app-id",
TYPE: "x-budibase-type",
PREVIEW_ROLE: "x-budibase-role",
TENANT_ID: "x-budibase-tenant-id",
TOKEN: "x-budibase-token",
CSRF_TOKEN: "x-csrf-token",
}
exports.GlobalRoles = {
OWNER: "owner",
ADMIN: "admin",
BUILDER: "builder",
WORKSPACE_MANAGER: "workspace_manager",
}
exports.Configs = {
SETTINGS: "settings",
ACCOUNT: "account",
SMTP: "smtp",
GOOGLE: "google",
OIDC: "oidc",
OIDC_LOGOS: "logos_oidc",
}
exports.MAX_VALID_DATE = new Date(2147483647000)
exports.DEFAULT_TENANT_ID = "default"

44
packages/backend-core/src/constants.ts Normal file
View File

@ -0,0 +1,44 @@
export enum UserStatus {
ACTIVE = "active",
INACTIVE = "inactive",
}
export enum Cookie {
CurrentApp = "budibase:currentapp",
Auth = "budibase:auth",
Init = "budibase:init",
ACCOUNT_RETURN_URL = "budibase:account:returnurl",
DatasourceAuth = "budibase:datasourceauth",
OIDC_CONFIG = "budibase:oidc:config",
}
export enum Header {
API_KEY = "x-budibase-api-key",
LICENSE_KEY = "x-budibase-license-key",
API_VER = "x-budibase-api-version",
APP_ID = "x-budibase-app-id",
TYPE = "x-budibase-type",
PREVIEW_ROLE = "x-budibase-role",
TENANT_ID = "x-budibase-tenant-id",
TOKEN = "x-budibase-token",
CSRF_TOKEN = "x-csrf-token",
}
export enum GlobalRole {
OWNER = "owner",
ADMIN = "admin",
BUILDER = "builder",
WORKSPACE_MANAGER = "workspace_manager",
}
export enum Config {
SETTINGS = "settings",
ACCOUNT = "account",
SMTP = "smtp",
GOOGLE = "google",
OIDC = "oidc",
OIDC_LOGOS = "logos_oidc",
}
export const MAX_VALID_DATE = new Date(2147483647000)
export const DEFAULT_TENANT_ID = "default"

18
packages/backend-core/src/context/Context.ts Normal file
View File

@ -0,0 +1,18 @@
import { AsyncLocalStorage } from "async_hooks"
import { ContextMap } from "./constants"
export default class Context {
static storage = new AsyncLocalStorage<ContextMap>()
static run(context: ContextMap, func: any) {
return Context.storage.run(context, () => func())
}
static get(): ContextMap {
return Context.storage.getStore() as ContextMap
}
static set(context: ContextMap) {
Context.storage.enterWith(context)
}
}

47
packages/backend-core/src/context/FunctionContext.js
View File

@ -1,47 +0,0 @@
const cls = require("../clshooked")
const { newid } = require("../hashing")
const REQUEST_ID_KEY = "requestId"
const MAIN_CTX = cls.createNamespace("main")
function getContextStorage(namespace) {
if (namespace && namespace.active) {
let contextData = namespace.active
delete contextData.id
delete contextData._ns_name
return contextData
}
return {}
}
class FunctionContext {
static run(callback) {
return MAIN_CTX.runAndReturn(async () => {
const namespaceId = newid()
MAIN_CTX.set(REQUEST_ID_KEY, namespaceId)
const namespace = cls.createNamespace(namespaceId)
let response = await namespace.runAndReturn(callback)
cls.destroyNamespace(namespaceId)
return response
})
}
static setOnContext(key, value) {
const namespaceId = MAIN_CTX.get(REQUEST_ID_KEY)
const namespace = cls.getNamespace(namespaceId)
namespace.set(key, value)
}
static getFromContext(key) {
const namespaceId = MAIN_CTX.get(REQUEST_ID_KEY)
const namespace = cls.getNamespace(namespaceId)
const context = getContextStorage(namespace)
if (context) {
return context[key]
} else {
return null
}
}
}
module.exports = FunctionContext

22
packages/backend-core/src/context/constants.ts
View File

@ -1,17 +1,7 @@
export enum ContextKey {
TENANT_ID = "tenantId",
GLOBAL_DB = "globalDb",
APP_ID = "appId",
IDENTITY = "identity",
// whatever the request app DB was
CURRENT_DB = "currentDb",
// get the prod app DB from the request
PROD_DB = "prodDb",
// get the dev app DB from the request
DEV_DB = "devDb",
DB_OPTS = "dbOpts",
// check if something else is using the context, don't close DB
TENANCY_IN_USE = "tenancyInUse",
APP_IN_USE = "appInUse",
IDENTITY_IN_USE = "identityInUse",
import { IdentityContext } from "@budibase/types"
export type ContextMap = {
tenantId?: string
appId?: string
identity?: IdentityContext
}

50
packages/backend-core/src/context/deprovision.js → packages/backend-core/src/context/deprovision.ts
View File

@ -1,15 +1,19 @@
const { getGlobalUserParams, getAllApps } = require("../db/utils")
const { doWithDB } = require("../db")
const { doWithGlobalDB } = require("../tenancy")
const { StaticDatabases } = require("../db/constants")
import {
getGlobalUserParams,
getAllApps,
doWithDB,
StaticDatabases,
} from "../db"
import { doWithGlobalDB } from "../tenancy"
import { App, Tenants, User, Database } from "@budibase/types"
const TENANT_DOC = StaticDatabases.PLATFORM_INFO.docs.tenants
const PLATFORM_INFO_DB = StaticDatabases.PLATFORM_INFO.name
const removeTenantFromInfoDB = async tenantId => {
async function removeTenantFromInfoDB(tenantId: string) {
try {
await doWithDB(PLATFORM_INFO_DB, async infoDb => {
let tenants = await infoDb.get(TENANT_DOC)
await doWithDB(PLATFORM_INFO_DB, async (infoDb: Database) => {
const tenants = (await infoDb.get(TENANT_DOC)) as Tenants
tenants.tenantIds = tenants.tenantIds.filter(id => id !== tenantId)
await infoDb.put(tenants)
@ -20,14 +24,14 @@ const removeTenantFromInfoDB = async tenantId => {
}
}
exports.removeUserFromInfoDB = async dbUser => {
await doWithDB(PLATFORM_INFO_DB, async infoDb => {
const keys = [dbUser._id, dbUser.email]
export async function removeUserFromInfoDB(dbUser: User) {
await doWithDB(PLATFORM_INFO_DB, async (infoDb: Database) => {
const keys = [dbUser._id!, dbUser.email]
const userDocs = await infoDb.allDocs({
keys,
include_docs: true,
})
const toDelete = userDocs.rows.map(row => {
const toDelete = userDocs.rows.map((row: any) => {
return {
...row.doc,
_deleted: true,
@ -37,18 +41,18 @@ exports.removeUserFromInfoDB = async dbUser => {
})
}
const removeUsersFromInfoDB = async tenantId => {
return doWithGlobalDB(tenantId, async db => {
async function removeUsersFromInfoDB(tenantId: string) {
return doWithGlobalDB(tenantId, async (db: any) => {
try {
const allUsers = await db.allDocs(
getGlobalUserParams(null, {
include_docs: true,
})
)
await doWithDB(PLATFORM_INFO_DB, async infoDb => {
const allEmails = allUsers.rows.map(row => row.doc.email)
await doWithDB(PLATFORM_INFO_DB, async (infoDb: any) => {
const allEmails = allUsers.rows.map((row: any) => row.doc.email)
// get the id docs
let keys = allUsers.rows.map(row => row.id)
let keys = allUsers.rows.map((row: any) => row.id)
// and the email docs
keys = keys.concat(allEmails)
// retrieve the docs and delete them
@ -56,7 +60,7 @@ const removeUsersFromInfoDB = async tenantId => {
keys,
include_docs: true,
})
const toDelete = userDocs.rows.map(row => {
const toDelete = userDocs.rows.map((row: any) => {
return {
...row.doc,
_deleted: true,
@ -71,8 +75,8 @@ const removeUsersFromInfoDB = async tenantId => {
})
}
const removeGlobalDB = async tenantId => {
return doWithGlobalDB(tenantId, async db => {
async function removeGlobalDB(tenantId: string) {
return doWithGlobalDB(tenantId, async (db: Database) => {
try {
await db.destroy()
} catch (err) {
@ -82,11 +86,11 @@ const removeGlobalDB = async tenantId => {
})
}
const removeTenantApps = async tenantId => {
async function removeTenantApps(tenantId: string) {
try {
const apps = await getAllApps({ all: true })
const apps = (await getAllApps({ all: true })) as App[]
const destroyPromises = apps.map(app =>
doWithDB(app.appId, db => db.destroy())
doWithDB(app.appId, (db: Database) => db.destroy())
)
await Promise.allSettled(destroyPromises)
} catch (err) {
@ -96,7 +100,7 @@ const removeTenantApps = async tenantId => {
}
// can't live in tenancy package due to circular dependency on db/utils
exports.deleteTenant = async tenantId => {
export async function deleteTenant(tenantId: string) {
await removeTenantFromInfoDB(tenantId)
await removeUsersFromInfoDB(tenantId)
await removeGlobalDB(tenantId)

302
packages/backend-core/src/context/index.ts
View File

@ -1,47 +1,32 @@
import env from "../environment"
import { SEPARATOR, DocumentType } from "../db/constants"
import cls from "./FunctionContext"
import { dangerousGetDB, closeDB } from "../db"
import { baseGlobalDBName } from "../db/tenancy"
import { IdentityContext } from "@budibase/types"
import { DEFAULT_TENANT_ID as _DEFAULT_TENANT_ID } from "../constants"
import { ContextKey } from "./constants"
import PouchDB from "pouchdb"
import {
updateUsing,
closeWithUsing,
setAppTenantId,
setIdentity,
closeAppDBs,
getContextDB,
} from "./utils"
SEPARATOR,
DocumentType,
getDevelopmentAppID,
getProdAppID,
baseGlobalDBName,
getDB,
} from "../db"
import Context from "./Context"
import { IdentityContext, Database } from "@budibase/types"
import { DEFAULT_TENANT_ID as _DEFAULT_TENANT_ID } from "../constants"
import { ContextMap } from "./constants"
export const DEFAULT_TENANT_ID = _DEFAULT_TENANT_ID
// some test cases call functions directly, need to
// store an app ID to pretend there is a context
let TEST_APP_ID: string | null = null
export const closeTenancy = async () => {
try {
if (env.USE_COUCH) {
const db = getGlobalDB()
await closeDB(db)
}
} catch (err) {
// no DB found - skip closing
return
}
// clear from context now that database is closed/task is finished
cls.setOnContext(ContextKey.TENANT_ID, null)
cls.setOnContext(ContextKey.GLOBAL_DB, null)
export function isMultiTenant() {
return env.MULTI_TENANCY
}
// export const isDefaultTenant = () => {
// return getTenantId() === DEFAULT_TENANT_ID
// }
export function isTenantIdSet() {
const context = Context.get()
return !!context?.tenantId
}
export const isMultiTenant = () => {
export function isTenancyEnabled() {
return env.MULTI_TENANCY
}
@ -49,9 +34,9 @@ export const isMultiTenant = () => {
* Given an app ID this will attempt to retrieve the tenant ID from it.
* @return {null|string} The tenant ID found within the app ID.
*/
export const getTenantIDFromAppID = (appId: string) => {
export function getTenantIDFromAppID(appId: string) {
if (!appId) {
return null
return undefined
}
if (!isMultiTenant()) {
return DEFAULT_TENANT_ID
@ -59,7 +44,7 @@ export const getTenantIDFromAppID = (appId: string) => {
const split = appId.split(SEPARATOR)
const hasDev = split[1] === DocumentType.DEV
if ((hasDev && split.length === 3) || (!hasDev && split.length === 2)) {
return null
return undefined
}
if (hasDev) {
return split[2]
@ -68,127 +53,125 @@ export const getTenantIDFromAppID = (appId: string) => {
}
}
export const doInContext = async (appId: string, task: any) => {
// gets the tenant ID from the app ID
const tenantId = getTenantIDFromAppID(appId)
return doInTenant(tenantId, async () => {
return doInAppContext(appId, async () => {
return task()
})
})
function updateContext(updates: ContextMap) {
let context: ContextMap
try {
context = Context.get()
} catch (err) {
// no context, start empty
context = {}
}
context = {
...context,
...updates,
}
return context
}
export const doInTenant = (tenantId: string | null, task: any) => {
async function newContext(updates: ContextMap, task: any) {
// see if there already is a context setup
let context: ContextMap = updateContext(updates)
return Context.run(context, task)
}
export async function doInContext(appId: string, task: any): Promise<any> {
const tenantId = getTenantIDFromAppID(appId)
return newContext(
{
tenantId,
appId,
},
task
)
}
export async function doInTenant(
tenantId: string | null,
task: any
): Promise<any> {
// make sure default always selected in single tenancy
if (!env.MULTI_TENANCY) {
tenantId = tenantId || DEFAULT_TENANT_ID
}
// the internal function is so that we can re-use an existing
// context - don't want to close DB on a parent context
async function internal(opts = { existing: false }) {
// set the tenant id + global db if this is a new context
if (!opts.existing) {
updateTenantId(tenantId)
}
try {
// invoke the task
return await task()
} finally {
await closeWithUsing(ContextKey.TENANCY_IN_USE, () => {
return closeTenancy()
})
}
}
const existing = cls.getFromContext(ContextKey.TENANT_ID) === tenantId
return updateUsing(ContextKey.TENANCY_IN_USE, existing, internal)
const updates = tenantId ? { tenantId } : {}
return newContext(updates, task)
}
export const doInAppContext = (appId: string, task: any) => {
export async function doInAppContext(appId: string, task: any): Promise<any> {
if (!appId) {
throw new Error("appId is required")
}
const identity = getIdentity()
// the internal function is so that we can re-use an existing
// context - don't want to close DB on a parent context
async function internal(opts = { existing: false }) {
// set the app tenant id
if (!opts.existing) {
setAppTenantId(appId)
}
// set the app ID
cls.setOnContext(ContextKey.APP_ID, appId)
// preserve the identity
if (identity) {
setIdentity(identity)
}
try {
// invoke the task
return await task()
} finally {
await closeWithUsing(ContextKey.APP_IN_USE, async () => {
await closeAppDBs()
await closeTenancy()
})
}
const tenantId = getTenantIDFromAppID(appId)
const updates: ContextMap = { appId }
if (tenantId) {
updates.tenantId = tenantId
}
const existing = cls.getFromContext(ContextKey.APP_ID) === appId
return updateUsing(ContextKey.APP_IN_USE, existing, internal)
return newContext(updates, task)
}
export const doInIdentityContext = (identity: IdentityContext, task: any) => {
export async function doInIdentityContext(
identity: IdentityContext,
task: any
): Promise<any> {
if (!identity) {
throw new Error("identity is required")
}
async function internal(opts = { existing: false }) {
if (!opts.existing) {
cls.setOnContext(ContextKey.IDENTITY, identity)
// set the tenant so that doInTenant will preserve identity
if (identity.tenantId) {
updateTenantId(identity.tenantId)
}
}
try {
// invoke the task
return await task()
} finally {
await closeWithUsing(ContextKey.IDENTITY_IN_USE, async () => {
setIdentity(null)
await closeTenancy()
})
}
const context: ContextMap = {
identity,
}
const existing = cls.getFromContext(ContextKey.IDENTITY)
return updateUsing(ContextKey.IDENTITY_IN_USE, existing, internal)
if (identity.tenantId) {
context.tenantId = identity.tenantId
}
return newContext(context, task)
}
export const getIdentity = (): IdentityContext | undefined => {
export function getIdentity(): IdentityContext | undefined {
try {
return cls.getFromContext(ContextKey.IDENTITY)
const context = Context.get()
return context?.identity
} catch (e) {
// do nothing - identity is not in context
}
}
export const updateTenantId = (tenantId: string | null) => {
cls.setOnContext(ContextKey.TENANT_ID, tenantId)
if (env.USE_COUCH) {
setGlobalDB(tenantId)
export function getTenantId(): string {
if (!isMultiTenant()) {
return DEFAULT_TENANT_ID
}
const context = Context.get()
const tenantId = context?.tenantId
if (!tenantId) {
throw new Error("Tenant id not found")
}
return tenantId
}
export function getAppId(): string | undefined {
const context = Context.get()
const foundId = context?.appId
if (!foundId && env.isTest() && TEST_APP_ID) {
return TEST_APP_ID
} else {
return foundId
}
}
export const updateAppId = async (appId: string) => {
export function updateTenantId(tenantId?: string) {
let context: ContextMap = updateContext({
tenantId,
})
Context.set(context)
}
export function updateAppId(appId: string) {
let context: ContextMap = updateContext({
appId,
})
try {
// have to close first, before removing the databases from context
await closeAppDBs()
cls.setOnContext(ContextKey.APP_ID, appId)
Context.set(context)
} catch (err) {
if (env.isTest()) {
TEST_APP_ID = appId
@ -198,70 +181,43 @@ export const updateAppId = async (appId: string) => {
}
}
export const setGlobalDB = (tenantId: string | null) => {
const dbName = baseGlobalDBName(tenantId)
const db = dangerousGetDB(dbName)
cls.setOnContext(ContextKey.GLOBAL_DB, db)
return db
}
export const getGlobalDB = () => {
const db = cls.getFromContext(ContextKey.GLOBAL_DB)
if (!db) {
export function getGlobalDB(): Database {
const context = Context.get()
if (!context || (env.MULTI_TENANCY && !context.tenantId)) {
throw new Error("Global DB not found")
}
return db
}
export const isTenantIdSet = () => {
const tenantId = cls.getFromContext(ContextKey.TENANT_ID)
return !!tenantId
}
export const getTenantId = () => {
if (!isMultiTenant()) {
return DEFAULT_TENANT_ID
}
const tenantId = cls.getFromContext(ContextKey.TENANT_ID)
if (!tenantId) {
throw new Error("Tenant id not found")
}
return tenantId
}
export const getAppId = () => {
const foundId = cls.getFromContext(ContextKey.APP_ID)
if (!foundId && env.isTest() && TEST_APP_ID) {
return TEST_APP_ID
} else {
return foundId
}
}
export const isTenancyEnabled = () => {
return env.MULTI_TENANCY
return getDB(baseGlobalDBName(context?.tenantId))
}
/**
* Opens the app database based on whatever the request
* Gets the app database based on whatever the request
* contained, dev or prod.
*/
export const getAppDB = (opts?: any) => {
return getContextDB(ContextKey.CURRENT_DB, opts)
export function getAppDB(opts?: any): Database {
const appId = getAppId()
return getDB(appId, opts)
}
/**
* This specifically gets the prod app ID, if the request
* contained a development app ID, this will open the prod one.
* contained a development app ID, this will get the prod one.
*/
export const getProdAppDB = (opts?: any) => {
return getContextDB(ContextKey.PROD_DB, opts)
export function getProdAppDB(opts?: any): Database {
const appId = getAppId()
if (!appId) {
throw new Error("Unable to retrieve prod DB - no app ID.")
}
return getDB(getProdAppID(appId), opts)
}
/**
* This specifically gets the dev app ID, if the request
* contained a prod app ID, this will open the dev one.
* contained a prod app ID, this will get the dev one.
*/
export const getDevAppDB = (opts?: any) => {
return getContextDB(ContextKey.DEV_DB, opts)
export function getDevAppDB(opts?: any): Database {
const appId = getAppId()
if (!appId) {
throw new Error("Unable to retrieve dev DB - no app ID.")
}
return getDB(getDevelopmentAppID(appId), opts)
}

29
packages/backend-core/src/context/tests/index.spec.ts → packages/backend-core/src/context/tests/index.spec.js
View File

@ -1,18 +1,9 @@
import "../../../tests/utilities/TestConfiguration"
import * as context from ".."
import { DEFAULT_TENANT_ID } from "../../constants"
import env from "../../environment"
// must use require to spy index file exports due to known issue in jest
const dbUtils = require("../../db")
jest.spyOn(dbUtils, "closeDB")
jest.spyOn(dbUtils, "dangerousGetDB")
require("../../../tests")
const context = require("../")
const { DEFAULT_TENANT_ID } = require("../../constants")
const env = require("../../environment")
describe("context", () => {
beforeEach(() => {
jest.clearAllMocks()
})
describe("doInTenant", () => {
describe("single-tenancy", () => {
it("defaults to the default tenant", () => {
@ -25,8 +16,6 @@ describe("context", () => {
const db = context.getGlobalDB()
expect(db.name).toBe("global-db")
})
expect(dbUtils.dangerousGetDB).toHaveBeenCalledTimes(1)
expect(dbUtils.closeDB).toHaveBeenCalledTimes(1)
})
})
@ -40,7 +29,7 @@ describe("context", () => {
let error
try {
context.getTenantId()
} catch (e: any) {
} catch (e) {
error = e
}
expect(error.message).toBe("Tenant id not found")
@ -59,7 +48,7 @@ describe("context", () => {
let error
try {
context.getGlobalDB()
} catch (e: any) {
} catch (e) {
error = e
}
expect(error.message).toBe("Global DB not found")
@ -85,8 +74,6 @@ describe("context", () => {
const db = context.getGlobalDB()
expect(db.name).toBe("test_global-db")
})
expect(dbUtils.dangerousGetDB).toHaveBeenCalledTimes(1)
expect(dbUtils.closeDB).toHaveBeenCalledTimes(1)
})
it("sets the tenant id when nested with same tenant id", async () => {
@ -121,10 +108,6 @@ describe("context", () => {
})
})
})
// only 1 db is opened and closed
expect(dbUtils.dangerousGetDB).toHaveBeenCalledTimes(1)
expect(dbUtils.closeDB).toHaveBeenCalledTimes(1)
})
it("sets different tenant id inside another context", () => {

109
packages/backend-core/src/context/utils.ts
View File

@ -1,109 +0,0 @@
import {
DEFAULT_TENANT_ID,
getAppId,
getTenantIDFromAppID,
updateTenantId,
} from "./index"
import cls from "./FunctionContext"
import { IdentityContext } from "@budibase/types"
import { ContextKey } from "./constants"
import { dangerousGetDB, closeDB } from "../db"
import { isEqual } from "lodash"
import { getDevelopmentAppID, getProdAppID } from "../db/conversions"
import env from "../environment"
export async function updateUsing(
usingKey: string,
existing: boolean,
internal: (opts: { existing: boolean }) => Promise<any>
) {
const using = cls.getFromContext(usingKey)
if (using && existing) {
cls.setOnContext(usingKey, using + 1)
return internal({ existing: true })
} else {
return cls.run(async () => {
cls.setOnContext(usingKey, 1)
return internal({ existing: false })
})
}
}
export async function closeWithUsing(
usingKey: string,
closeFn: () => Promise<any>
) {
const using = cls.getFromContext(usingKey)
if (!using || using <= 1) {
await closeFn()
} else {
cls.setOnContext(usingKey, using - 1)
}
}
export const setAppTenantId = (appId: string) => {
const appTenantId = getTenantIDFromAppID(appId) || DEFAULT_TENANT_ID
updateTenantId(appTenantId)
}
export const setIdentity = (identity: IdentityContext | null) => {
cls.setOnContext(ContextKey.IDENTITY, identity)
}
// this function makes sure the PouchDB objects are closed and
// fully deleted when finished - this protects against memory leaks
export async function closeAppDBs() {
const dbKeys = [ContextKey.CURRENT_DB, ContextKey.PROD_DB, ContextKey.DEV_DB]
for (let dbKey of dbKeys) {
const db = cls.getFromContext(dbKey)
if (!db) {
continue
}
await closeDB(db)
// clear the DB from context, incase someone tries to use it again
cls.setOnContext(dbKey, null)
}
// clear the app ID now that the databases are closed
if (cls.getFromContext(ContextKey.APP_ID)) {
cls.setOnContext(ContextKey.APP_ID, null)
}
if (cls.getFromContext(ContextKey.DB_OPTS)) {
cls.setOnContext(ContextKey.DB_OPTS, null)
}
}
export function getContextDB(key: string, opts: any) {
const dbOptsKey = `${key}${ContextKey.DB_OPTS}`
let storedOpts = cls.getFromContext(dbOptsKey)
let db = cls.getFromContext(key)
if (db && isEqual(opts, storedOpts)) {
return db
}
const appId = getAppId()
let toUseAppId
switch (key) {
case ContextKey.CURRENT_DB:
toUseAppId = appId
break
case ContextKey.PROD_DB:
toUseAppId = getProdAppID(appId)
break
case ContextKey.DEV_DB:
toUseAppId = getDevelopmentAppID(appId)
break
}
db = dangerousGetDB(toUseAppId, opts)
try {
cls.setOnContext(key, db)
if (opts) {
cls.setOnContext(dbOptsKey, opts)
}
} catch (err) {
if (!env.isTest()) {
throw err
}
}
return db
}

10
packages/backend-core/src/db/Replication.ts
View File

@ -1,4 +1,4 @@
import { dangerousGetDB, closeDB } from "."
import { getPouchDB, closePouchDB } from "./couch/pouchDB"
import { DocumentType } from "./constants"
class Replication {
@ -12,12 +12,12 @@ class Replication {
* @param {String} target - the DB you want to replicate to, or rollback from
*/
constructor({ source, target }: any) {
this.source = dangerousGetDB(source)
this.target = dangerousGetDB(target)
this.source = getPouchDB(source)
this.target = getPouchDB(target)
}
close() {
return Promise.all([closeDB(this.source), closeDB(this.target)])
return Promise.all([closePouchDB(this.source), closePouchDB(this.target)])
}
promisify(operation: any, opts = {}) {
@ -68,7 +68,7 @@ class Replication {
async rollback() {
await this.target.destroy()
// Recreate the DB again
this.target = dangerousGetDB(this.target.name)
this.target = getPouchDB(this.target.name)
// take the opportunity to remove deleted tombstones
await this.replicate()
}

21
packages/backend-core/src/db/conversions.js → packages/backend-core/src/db/conversions.ts
View File

@ -1,32 +1,33 @@
import { APP_DEV_PREFIX, APP_PREFIX } from "./constants"
import { App } from "@budibase/types"
const NO_APP_ERROR = "No app provided"
const { APP_DEV_PREFIX, APP_PREFIX } = require("./constants")
exports.isDevAppID = appId => {
export function isDevAppID(appId?: string) {
if (!appId) {
throw NO_APP_ERROR
}
return appId.startsWith(APP_DEV_PREFIX)
}
exports.isProdAppID = appId => {
export function isProdAppID(appId?: string) {
if (!appId) {
throw NO_APP_ERROR
}
return appId.startsWith(APP_PREFIX) && !exports.isDevAppID(appId)
return appId.startsWith(APP_PREFIX) && !isDevAppID(appId)
}
exports.isDevApp = app => {
export function isDevApp(app: App) {
if (!app) {
throw NO_APP_ERROR
}
return exports.isDevAppID(app.appId)
return isDevAppID(app.appId)
}
/**
* Generates a development app ID from a real app ID.
* @returns {string} the dev app ID which can be used for dev database.
*/
exports.getDevelopmentAppID = appId => {
export function getDevelopmentAppID(appId: string) {
if (!appId || appId.startsWith(APP_DEV_PREFIX)) {
return appId
}
@ -36,12 +37,12 @@ exports.getDevelopmentAppID = appId => {
const rest = split.join(APP_PREFIX)
return `${APP_DEV_PREFIX}${rest}`
}
exports.getDevAppID = exports.getDevelopmentAppID
export const getDevAppID = getDevelopmentAppID
/**
* Convert a development app ID to a deployed app ID.
*/
exports.getProdAppID = appId => {
export function getProdAppID(appId: string) {
if (!appId || !appId.startsWith(APP_DEV_PREFIX)) {
return appId
}
@ -52,7 +53,7 @@ exports.getProdAppID = appId => {
return `${APP_PREFIX}${rest}`
}
exports.extractAppUUID = id => {
export function extractAppUUID(id: string) {
const split = id?.split("_") || []
return split.length ? split[split.length - 1] : null
}

179
packages/backend-core/src/db/couch/DatabaseImpl.ts Normal file
View File

@ -0,0 +1,179 @@
import Nano from "nano"
import {
AllDocsResponse,
AnyDocument,
Database,
DatabaseOpts,
DatabaseQueryOpts,
DatabasePutOpts,
} from "@budibase/types"
import { getCouchInfo } from "./connections"
import { directCouchCall } from "./utils"
import { getPouchDB } from "./pouchDB"
export class DatabaseImpl implements Database {
public readonly name: string
private static nano: Nano.ServerScope
private readonly pouchOpts: DatabaseOpts
constructor(dbName?: string, opts?: DatabaseOpts) {
if (dbName == null) {
throw new Error("Database name cannot be undefined.")
}
this.name = dbName
this.pouchOpts = opts || {}
if (!DatabaseImpl.nano) {
DatabaseImpl.init()
}
}
static init() {
const couchInfo = getCouchInfo()
DatabaseImpl.nano = Nano({
url: couchInfo.url,
requestDefaults: {
headers: {
Authorization: couchInfo.cookie,
},
},
parseUrl: false,
})
}
async exists() {
let response = await directCouchCall(`/${this.name}`, "HEAD")
return response.status === 200
}
async checkSetup() {
let shouldCreate = !this.pouchOpts?.skip_setup
// check exists in a lightweight fashion
let exists = await this.exists()
if (!shouldCreate && !exists) {
throw new Error("DB does not exist")
}
if (!exists) {
await DatabaseImpl.nano.db.create(this.name)
}
return DatabaseImpl.nano.db.use(this.name)
}
private async updateOutput(fnc: any) {
try {
return await fnc()
} catch (err: any) {
if (err.statusCode) {
err.status = err.statusCode
}
throw err
}
}
async get<T>(id?: string): Promise<T | any> {
const db = await this.checkSetup()
if (!id) {
throw new Error("Unable to get doc without a valid _id.")
}
return this.updateOutput(() => db.get(id))
}
async remove(id?: string, rev?: string) {
const db = await this.checkSetup()
if (!id || !rev) {
throw new Error("Unable to remove doc without a valid _id and _rev.")
}
return this.updateOutput(() => db.destroy(id, rev))
}
async put(document: AnyDocument, opts?: DatabasePutOpts) {
if (!document._id) {
throw new Error("Cannot store document without _id field.")
}
const db = await this.checkSetup()
if (!document.createdAt) {
document.createdAt = new Date().toISOString()
}
document.updatedAt = new Date().toISOString()
if (opts?.force && document._id) {
try {
const existing = await this.get(document._id)
if (existing) {
document._rev = existing._rev
}
} catch (err: any) {
if (err.status !== 404) {
throw err
}
}
}
return this.updateOutput(() => db.insert(document))
}
async bulkDocs(documents: AnyDocument[]) {
const db = await this.checkSetup()
return this.updateOutput(() => db.bulk({ docs: documents }))
}
async allDocs<T>(params: DatabaseQueryOpts): Promise<AllDocsResponse<T>> {
const db = await this.checkSetup()
return this.updateOutput(() => db.list(params))
}
async query<T>(
viewName: string,
params: DatabaseQueryOpts
): Promise<AllDocsResponse<T>> {
const db = await this.checkSetup()
const [database, view] = viewName.split("/")
return this.updateOutput(() => db.view(database, view, params))
}
async destroy() {
try {
await DatabaseImpl.nano.db.destroy(this.name)
} catch (err: any) {
// didn't exist, don't worry
if (err.statusCode === 404) {
return
} else {
throw { ...err, status: err.statusCode }
}
}
}
async compact() {
const db = await this.checkSetup()
return this.updateOutput(() => db.compact())
}
private doWithPouchDB(func: string) {
const dbName = this.name
return async (args: any[]) => {
const pouch = getPouchDB(dbName)
// @ts-ignore
return pouch[func](...args)
}
}
// All below functions are in-frequently called, just utilise PouchDB
// for them as it implements them better than we can
async dump(...args: any[]) {
return this.doWithPouchDB("dump")(args)
}
async load(...args: any[]) {
return this.doWithPouchDB("load")(args)
}
async createIndex(...args: any[]) {
return this.doWithPouchDB("createIndex")(args)
}
async deleteIndex(...args: any[]) {
return this.doWithPouchDB("createIndex")(args)
}
async getIndexes(...args: any[]) {
return this.doWithPouchDB("createIndex")(args)
}
}

118
packages/backend-core/src/db/pouch.ts → packages/backend-core/src/db/couch/connections.ts
View File

@ -1,5 +1,37 @@
import PouchDB from "pouchdb"
import env from "../environment"
import env from "../../environment"
export const getCouchInfo = () => {
const urlInfo = getUrlInfo()
let username
let password
if (env.COUCH_DB_USERNAME) {
// set from env
username = env.COUCH_DB_USERNAME
} else if (urlInfo.auth.username) {
// set from url
username = urlInfo.auth.username
} else if (!env.isTest()) {
throw new Error("CouchDB username not set")
}
if (env.COUCH_DB_PASSWORD) {
// set from env
password = env.COUCH_DB_PASSWORD
} else if (urlInfo.auth.password) {
// set from url
password = urlInfo.auth.password
} else if (!env.isTest()) {
throw new Error("CouchDB password not set")
}
const authCookie = Buffer.from(`${username}:${password}`).toString("base64")
return {
url: urlInfo.url!,
auth: {
username: username,
password: password,
},
cookie: `Basic ${authCookie}`,
}
}
export const getUrlInfo = (url = env.COUCH_DB_URL) => {
let cleanUrl, username, password, host
@ -43,85 +75,3 @@ export const getUrlInfo = (url = env.COUCH_DB_URL) => {
},
}
}
export const getCouchInfo = () => {
const urlInfo = getUrlInfo()
let username
let password
if (env.COUCH_DB_USERNAME) {
// set from env
username = env.COUCH_DB_USERNAME
} else if (urlInfo.auth.username) {
// set from url
username = urlInfo.auth.username
} else if (!env.isTest()) {
throw new Error("CouchDB username not set")
}
if (env.COUCH_DB_PASSWORD) {
// set from env
password = env.COUCH_DB_PASSWORD
} else if (urlInfo.auth.password) {
// set from url
password = urlInfo.auth.password
} else if (!env.isTest()) {
throw new Error("CouchDB password not set")
}
const authCookie = Buffer.from(`${username}:${password}`).toString("base64")
return {
url: urlInfo.url,
auth: {
username: username,
password: password,
},
cookie: `Basic ${authCookie}`,
}
}
/**
* Return a constructor for PouchDB.
* This should be rarely used outside of the main application config.
* Exposed for exceptional cases such as in-memory views.
*/
export const getPouch = (opts: any = {}) => {
let { url, cookie } = getCouchInfo()
let POUCH_DB_DEFAULTS = {
prefix: url,
fetch: (url: string, opts: any) => {
// use a specific authorization cookie - be very explicit about how we authenticate
opts.headers.set("Authorization", cookie)
return PouchDB.fetch(url, opts)
},
}
if (opts.inMemory) {
const inMemory = require("pouchdb-adapter-memory")
PouchDB.plugin(inMemory)
POUCH_DB_DEFAULTS = {
prefix: undefined,
// @ts-ignore
adapter: "memory",
}
}
if (opts.onDisk) {
POUCH_DB_DEFAULTS = {
prefix: undefined,
// @ts-ignore
adapter: "leveldb",
}
}
if (opts.replication) {
const replicationStream = require("pouchdb-replication-stream")
PouchDB.plugin(replicationStream.plugin)
// @ts-ignore
PouchDB.adapter("writableStream", replicationStream.adapters.writableStream)
}
if (opts.find) {
const find = require("pouchdb-find")
PouchDB.plugin(find)
}
return PouchDB.defaults(POUCH_DB_DEFAULTS)
}

4
packages/backend-core/src/db/couch/index.ts Normal file
View File

@ -0,0 +1,4 @@
export * from "./connections"
export * from "./DatabaseImpl"
export * from "./utils"
export { init, getPouch, getPouchDB, closePouchDB } from "./pouchDB"

97
packages/backend-core/src/db/couch/pouchDB.ts Normal file
View File

@ -0,0 +1,97 @@
import PouchDB from "pouchdb"
import env from "../../environment"
import { PouchOptions } from "@budibase/types"
import { getCouchInfo } from "./connections"
let Pouch: any
let initialised = false
/**
* Return a constructor for PouchDB.
* This should be rarely used outside of the main application config.
* Exposed for exceptional cases such as in-memory views.
*/
export const getPouch = (opts: PouchOptions = {}) => {
let { url, cookie } = getCouchInfo()
let POUCH_DB_DEFAULTS = {
prefix: url,
fetch: (url: string, opts: any) => {
// use a specific authorization cookie - be very explicit about how we authenticate
opts.headers.set("Authorization", cookie)
return PouchDB.fetch(url, opts)
},
}
if (opts.inMemory) {
const inMemory = require("pouchdb-adapter-memory")
PouchDB.plugin(inMemory)
POUCH_DB_DEFAULTS = {
// @ts-ignore
adapter: "memory",
}
}
if (opts.onDisk) {
POUCH_DB_DEFAULTS = {
// @ts-ignore
adapter: "leveldb",
}
}
if (opts.replication) {
const replicationStream = require("pouchdb-replication-stream")
PouchDB.plugin(replicationStream.plugin)
// @ts-ignore
PouchDB.adapter("writableStream", replicationStream.adapters.writableStream)
}
if (opts.find) {
const find = require("pouchdb-find")
PouchDB.plugin(find)
}
return PouchDB.defaults(POUCH_DB_DEFAULTS)
}
export function init(opts?: PouchOptions) {
Pouch = getPouch(opts)
initialised = true
}
const checkInitialised = () => {
if (!initialised) {
throw new Error("init has not been called")
}
}
export function getPouchDB(dbName: string, opts?: any): PouchDB.Database {
checkInitialised()
const db = new Pouch(dbName, opts)
const dbPut = db.put
db.put = async (doc: any, options = {}) => {
if (!doc.createdAt) {
doc.createdAt = new Date().toISOString()
}
doc.updatedAt = new Date().toISOString()
return dbPut(doc, options)
}
db.exists = async () => {
const info = await db.info()
return !info.error
}
return db
}
// use this function if you have called getPouchDB - close
// the databases you've opened once finished
export async function closePouchDB(db: PouchDB.Database) {
if (!db || env.isTest()) {
return
}
try {
// specifically await so that if there is an error, it can be ignored
return await db.close()
} catch (err) {
// ignore error, already closed
}
}

36
packages/backend-core/src/db/couch/utils.ts Normal file
View File

@ -0,0 +1,36 @@
import { getCouchInfo } from "./connections"
import fetch from "node-fetch"
import { checkSlashesInUrl } from "../../helpers"
export async function directCouchCall(
path: string,
method: string = "GET",
body?: any
) {
let { url, cookie } = getCouchInfo()
const couchUrl = `${url}/${path}`
const params: any = {
method: method,
headers: {
Authorization: cookie,
},
}
if (body && method !== "GET") {
params.body = JSON.stringify(body)
params.headers["Content-Type"] = "application/json"
}
return await fetch(checkSlashesInUrl(encodeURI(couchUrl)), params)
}
export async function directCouchQuery(
path: string,
method: string = "GET",
body?: any
) {
const response = await directCouchCall(path, method, body)
if (response.status < 300) {
return await response.json()
} else {
throw "Cannot connect to CouchDB instance"
}
}

46
packages/backend-core/src/db/db.ts Normal file
View File

@ -0,0 +1,46 @@
import env from "../environment"
import { directCouchQuery, getPouchDB } from "./couch"
import { CouchFindOptions, Database } from "@budibase/types"
import { DatabaseImpl } from "../db"
const dbList = new Set()
export function getDB(dbName?: string, opts?: any): Database {
// TODO: once using the test image, need to remove this
if (env.isTest()) {
dbList.add(dbName)
// @ts-ignore
return getPouchDB(dbName, opts)
}
return new DatabaseImpl(dbName, opts)
}
// we have to use a callback for this so that we can close
// the DB when we're done, without this manual requests would
// need to close the database when done with it to avoid memory leaks
export async function doWithDB(dbName: string, cb: any, opts = {}) {
const db = getDB(dbName, opts)
// need this to be async so that we can correctly close DB after all
// async operations have been completed
return await cb(db)
}
export function allDbs() {
if (!env.isTest()) {
throw new Error("Cannot be used outside test environment.")
}
return [...dbList]
}
export async function directCouchAllDbs(queryString?: string) {
let couchPath = "/_all_dbs"
if (queryString) {
couchPath += `?${queryString}`
}
return await directCouchQuery(couchPath)
}
export async function directCouchFind(dbName: string, opts: CouchFindOptions) {
const json = await directCouchQuery(`${dbName}/_find`, "POST", opts)
return { rows: json.docs, bookmark: json.bookmark }
}

140
packages/backend-core/src/db/index.ts
View File

@ -1,133 +1,7 @@
import * as pouch from "./pouch"
import env from "../environment"
import { checkSlashesInUrl } from "../helpers"
import fetch from "node-fetch"
import { PouchOptions, CouchFindOptions } from "@budibase/types"
import PouchDB from "pouchdb"
const openDbs: string[] = []
let Pouch: any
let initialised = false
const dbList = new Set()
if (env.MEMORY_LEAK_CHECK) {
setInterval(() => {
console.log("--- OPEN DBS ---")
console.log(openDbs)
}, 5000)
}
const put =
(dbPut: any) =>
async (doc: any, options = {}) => {
if (!doc.createdAt) {
doc.createdAt = new Date().toISOString()
}
doc.updatedAt = new Date().toISOString()
return dbPut(doc, options)
}
const checkInitialised = () => {
if (!initialised) {
throw new Error("init has not been called")
}
}
export async function init(opts?: PouchOptions) {
Pouch = pouch.getPouch(opts)
initialised = true
}
// NOTE: THIS IS A DANGEROUS FUNCTION - USE WITH CAUTION
// this function is prone to leaks, should only be used
// in situations that using the function doWithDB does not work
export function dangerousGetDB(dbName: string, opts?: any): PouchDB.Database {
checkInitialised()
if (env.isTest()) {
dbList.add(dbName)
}
const db = new Pouch(dbName, opts)
if (env.MEMORY_LEAK_CHECK) {
openDbs.push(db.name)
}
const dbPut = db.put
db.put = put(dbPut)
return db
}
// use this function if you have called dangerousGetDB - close
// the databases you've opened once finished
export async function closeDB(db: PouchDB.Database) {
if (!db || env.isTest()) {
return
}
if (env.MEMORY_LEAK_CHECK) {
openDbs.splice(openDbs.indexOf(db.name), 1)
}
try {
// specifically await so that if there is an error, it can be ignored
return await db.close()
} catch (err) {
// ignore error, already closed
}
}
// we have to use a callback for this so that we can close
// the DB when we're done, without this manual requests would
// need to close the database when done with it to avoid memory leaks
export async function doWithDB(dbName: string, cb: any, opts = {}) {
const db = dangerousGetDB(dbName, opts)
// need this to be async so that we can correctly close DB after all
// async operations have been completed
try {
return await cb(db)
} finally {
await closeDB(db)
}
}
export function allDbs() {
if (!env.isTest()) {
throw new Error("Cannot be used outside test environment.")
}
checkInitialised()
return [...dbList]
}
export async function directCouchQuery(
path: string,
method: string = "GET",
body?: any
) {
let { url, cookie } = pouch.getCouchInfo()
const couchUrl = `${url}/${path}`
const params: any = {
method: method,
headers: {
Authorization: cookie,
},
}
if (body && method !== "GET") {
params.body = JSON.stringify(body)
params.headers["Content-Type"] = "application/json"
}
const response = await fetch(checkSlashesInUrl(encodeURI(couchUrl)), params)
if (response.status < 300) {
return await response.json()
} else {
throw "Cannot connect to CouchDB instance"
}
}
export async function directCouchAllDbs(queryString?: string) {
let couchPath = "/_all_dbs"
if (queryString) {
couchPath += `?${queryString}`
}
return await directCouchQuery(couchPath)
}
export async function directCouchFind(dbName: string, opts: CouchFindOptions) {
const json = await directCouchQuery(`${dbName}/_find`, "POST", opts)
return { rows: json.docs, bookmark: json.bookmark }
}
export * from "./couch"
export * from "./db"
export * from "./utils"
export * from "./views"
export * from "./constants"
export * from "./conversions"
export * from "./tenancy"

8
packages/backend-core/src/db/tests/index.spec.js
View File

@ -1,11 +1,11 @@
require("../../../tests/utilities/TestConfiguration")
const { dangerousGetDB } = require("../")
require("../../../tests")
const { getDB } = require("../")
describe("db", () => {
describe("getDB", () => {
it("returns a db", async () => {
const db = dangerousGetDB("test")
const db = getDB("test")
expect(db).toBeDefined()
expect(db._adapter).toBe("memory")
expect(db.prefix).toBe("_pouch_")
@ -13,7 +13,7 @@ describe("db", () => {
})
it("uses the custom put function", async () => {
const db = dangerousGetDB("test")
const db = getDB("test")
let doc = { _id: "test" }
await db.put(doc)
doc = await db.get(doc._id)

4
packages/backend-core/src/db/tests/pouch.spec.js
View File

@ -1,5 +1,5 @@
require("../../../tests/utilities/TestConfiguration")
const getUrlInfo = require("../pouch").getUrlInfo
require("../../../tests")
const getUrlInfo = require("../couch").getUrlInfo
describe("pouch", () => {
describe("Couch DB URL parsing", () => {

12
packages/backend-core/src/db/tests/utils.spec.js
View File

@ -1,4 +1,4 @@
require("../../../tests/utilities/TestConfiguration");
require("../../../tests")
const {
generateAppID,
getDevelopmentAppID,
@ -8,8 +8,8 @@ const {
getPlatformUrl,
getScopedConfig
} = require("../utils")
const tenancy = require("../../tenancy");
const { Configs, DEFAULT_TENANT_ID } = require("../../constants");
const tenancy = require("../../tenancy")
const { Config, DEFAULT_TENANT_ID } = require("../../constants")
const env = require("../../environment")
describe("utils", () => {
@ -77,7 +77,7 @@ const setDbPlatformUrl = async () => {
const db = tenancy.getGlobalDB()
db.put({
_id: "config_settings",
type: Configs.SETTINGS,
type: Config.SETTINGS,
config: {
platformUrl: DB_URL
}
@ -178,7 +178,7 @@ describe("getScopedConfig", () => {
await tenancy.doInTenant(DEFAULT_TENANT_ID, async () => {
await setDbPlatformUrl()
const db = tenancy.getGlobalDB()
const config = await getScopedConfig(db, { type: Configs.SETTINGS })
const config = await getScopedConfig(db, { type: Config.SETTINGS })
expect(config.platformUrl).toBe(DB_URL)
})
})
@ -186,7 +186,7 @@ describe("getScopedConfig", () => {
it("returns the platform url without an existing config", async () => {
await tenancy.doInTenant(DEFAULT_TENANT_ID, async () => {
const db = tenancy.getGlobalDB()
const config = await getScopedConfig(db, { type: Configs.SETTINGS })
const config = await getScopedConfig(db, { type: Config.SETTINGS })
expect(config.platformUrl).toBe(DEFAULT_URL)
})
})

46
packages/backend-core/src/db/utils.ts
View File

@ -1,5 +1,5 @@
import { newid } from "../hashing"
import { DEFAULT_TENANT_ID, Configs } from "../constants"
import { DEFAULT_TENANT_ID, Config } from "../constants"
import env from "../environment"
import {
SEPARATOR,
@ -10,11 +10,12 @@ import {
} from "./constants"
import { getTenantId, getGlobalDB } from "../context"
import { getGlobalDBName } from "./tenancy"
import { doWithDB, allDbs, directCouchAllDbs } from "./index"
import { doWithDB, allDbs, directCouchAllDbs } from "./db"
import { getAppMetadata } from "../cache/appMetadata"
import { isDevApp, isDevAppID, getProdAppID } from "./conversions"
import { APP_PREFIX } from "./constants"
import * as events from "../events"
import { App, Database } from "@budibase/types"
export * from "./constants"
export * from "./conversions"
@ -25,7 +26,7 @@ export * from "./tenancy"
* Generates a new app ID.
* @returns {string} The new app ID which the app doc can be stored under.
*/
export const generateAppID = (tenantId = null) => {
export const generateAppID = (tenantId?: string | null) => {
let id = APP_PREFIX
if (tenantId) {
id += `${tenantId}${SEPARATOR}`
@ -250,11 +251,11 @@ export function generateRoleID(id: any) {
/**
* Gets parameters for retrieving a role, this is a utility function for the getDocParams function.
*/
export function getRoleParams(roleId = null, otherProps = {}) {
export function getRoleParams(roleId?: string | null, otherProps = {}) {
return getDocParams(DocumentType.ROLE, roleId, otherProps)
}
export function getStartEndKeyURL(baseKey: any, tenantId = null) {
export function getStartEndKeyURL(baseKey: any, tenantId?: string) {
const tenancy = tenantId ? `${SEPARATOR}${tenantId}` : ""
return `startkey="${baseKey}${tenancy}"&endkey="${baseKey}${tenancy}${UNICODE_MAX}"`
}
@ -301,7 +302,12 @@ export async function getAllDbs(opts = { efficient: false }) {
*
* @return {Promise<object[]>} returns the app information document stored in each app database.
*/
export async function getAllApps({ dev, all, idsOnly, efficient }: any = {}) {
export async function getAllApps({
dev,
all,
idsOnly,
efficient,
}: any = {}): Promise<App[] | string[]> {
let tenantId = getTenantId()
if (!env.MULTI_TENANCY && !tenantId) {
tenantId = DEFAULT_TENANT_ID
@ -373,35 +379,23 @@ export async function getAllApps({ dev, all, idsOnly, efficient }: any = {}) {
* Utility function for getAllApps but filters to production apps only.
*/
export async function getProdAppIDs() {
return (await getAllApps({ idsOnly: true })).filter(
(id: any) => !isDevAppID(id)
)
const apps = (await getAllApps({ idsOnly: true })) as string[]
return apps.filter((id: any) => !isDevAppID(id))
}
/**
* Utility function for the inverse of above.
*/
export async function getDevAppIDs() {
return (await getAllApps({ idsOnly: true })).filter((id: any) =>
isDevAppID(id)
)
const apps = (await getAllApps({ idsOnly: true })) as string[]
return apps.filter((id: any) => isDevAppID(id))
}
export async function dbExists(dbName: any) {
let exists = false
return doWithDB(
dbName,
async (db: any) => {
try {
// check if database exists
const info = await db.info()
if (info && !info.error) {
exists = true
}
} catch (err) {
exists = false
}
return exists
async (db: Database) => {
return await db.exists()
},
{ skip_setup: true }
)
@ -500,7 +494,7 @@ export const getScopedFullConfig = async function (
)[0]
// custom logic for settings doc
if (type === Configs.SETTINGS) {
if (type === Config.SETTINGS) {
if (scopedConfig && scopedConfig.doc) {
// overrides affected by environment variables
scopedConfig.doc.config.platformUrl = await getPlatformUrl({
@ -539,7 +533,7 @@ export const getPlatformUrl = async (opts = { tenantAware: true }) => {
// get the doc directly instead of with getScopedConfig to prevent loop
let settings
try {
settings = await db.get(generateConfigID({ type: Configs.SETTINGS }))
settings = await db.get(generateConfigID({ type: Config.SETTINGS }))
} catch (e: any) {
if (e.status !== 404) {
throw e

59
packages/backend-core/src/db/views.ts
View File

@ -1,8 +1,8 @@
import { DocumentType, ViewName, DeprecatedViews, SEPARATOR } from "./utils"
import { getGlobalDB } from "../context"
import PouchDB from "pouchdb"
import { StaticDatabases } from "./constants"
import { doWithDB } from "./"
import { Database, DatabaseQueryOpts } from "@budibase/types"
const DESIGN_DB = "_design/database"
@ -19,7 +19,7 @@ interface DesignDocument {
views: any
}
async function removeDeprecated(db: PouchDB.Database, viewName: ViewName) {
async function removeDeprecated(db: Database, viewName: ViewName) {
// @ts-ignore
if (!DeprecatedViews[viewName]) {
return
@ -70,16 +70,13 @@ export const createAccountEmailView = async () => {
emit(doc.email.toLowerCase(), doc._id)
}
}`
await doWithDB(
StaticDatabases.PLATFORM_INFO.name,
async (db: PouchDB.Database) => {
await createView(db, viewJs, ViewName.ACCOUNT_BY_EMAIL)
}
)
await doWithDB(StaticDatabases.PLATFORM_INFO.name, async (db: Database) => {
await createView(db, viewJs, ViewName.ACCOUNT_BY_EMAIL)
})
}
export const createUserAppView = async () => {
const db = getGlobalDB() as PouchDB.Database
const db = getGlobalDB()
const viewJs = `function(doc) {
if (doc._id.startsWith("${DocumentType.USER}${SEPARATOR}") && doc.roles) {
for (let prodAppId of Object.keys(doc.roles)) {
@ -117,12 +114,9 @@ export const createPlatformUserView = async () => {
emit(doc._id.toLowerCase(), doc._id)
}
}`
await doWithDB(
StaticDatabases.PLATFORM_INFO.name,
async (db: PouchDB.Database) => {
await createView(db, viewJs, ViewName.PLATFORM_USERS_LOWERCASE)
}
)
await doWithDB(StaticDatabases.PLATFORM_INFO.name, async (db: Database) => {
await createView(db, viewJs, ViewName.PLATFORM_USERS_LOWERCASE)
})
}
export interface QueryViewOptions {
@ -131,22 +125,24 @@ export interface QueryViewOptions {
export const queryView = async <T>(
viewName: ViewName,
params: PouchDB.Query.Options<T, T>,
db: PouchDB.Database,
params: DatabaseQueryOpts,
db: Database,
createFunc: any,
opts?: QueryViewOptions
): Promise<T[] | T | undefined> => {
try {
let response = await db.query<T, T>(`database/${viewName}`, params)
let response = await db.query<T>(`database/${viewName}`, params)
const rows = response.rows
const docs = rows.map(row => (params.include_docs ? row.doc : row.value))
const docs = rows.map((row: any) =>
params.include_docs ? row.doc : row.value
)
// if arrayResponse has been requested, always return array regardless of length
if (opts?.arrayResponse) {
return docs
return docs as T[]
} else {
// return the single document if there is only one
return docs.length <= 1 ? docs[0] : docs
return docs.length <= 1 ? (docs[0] as T) : (docs as T[])
}
} catch (err: any) {
if (err != null && err.name === "not_found") {
@ -161,7 +157,7 @@ export const queryView = async <T>(
export const queryPlatformView = async <T>(
viewName: ViewName,
params: PouchDB.Query.Options<T, T>,
params: DatabaseQueryOpts,
opts?: QueryViewOptions
): Promise<T[] | T | undefined> => {
const CreateFuncByName: any = {
@ -169,19 +165,16 @@ export const queryPlatformView = async <T>(
[ViewName.PLATFORM_USERS_LOWERCASE]: createPlatformUserView,
}
return doWithDB(
StaticDatabases.PLATFORM_INFO.name,
async (db: PouchDB.Database) => {
const createFn = CreateFuncByName[viewName]
return queryView(viewName, params, db, createFn, opts)
}
)
return doWithDB(StaticDatabases.PLATFORM_INFO.name, async (db: Database) => {
const createFn = CreateFuncByName[viewName]
return queryView(viewName, params, db, createFn, opts)
})
}
export const queryGlobalView = async <T>(
viewName: ViewName,
params: PouchDB.Query.Options<T, T>,
db?: PouchDB.Database,
params: DatabaseQueryOpts,
db?: Database,
opts?: QueryViewOptions
): Promise<T[] | T | undefined> => {
const CreateFuncByName: any = {
@ -192,8 +185,8 @@ export const queryGlobalView = async <T>(
}
// can pass DB in if working with something specific
if (!db) {
db = getGlobalDB() as PouchDB.Database
db = getGlobalDB()
}
const createFn = CreateFuncByName[viewName]
return queryView(viewName, params, db, createFn, opts)
return queryView(viewName, params, db!, createFn, opts)
}

1
packages/backend-core/src/environment.ts
View File

@ -69,7 +69,6 @@ const env = {
DISABLE_DEVELOPER_LICENSE: process.env.DISABLE_DEVELOPER_LICENSE,
DEFAULT_LICENSE: process.env.DEFAULT_LICENSE,
SERVICE: process.env.SERVICE || "budibase",
MEMORY_LEAK_CHECK: process.env.MEMORY_LEAK_CHECK || false,
LOG_LEVEL: process.env.LOG_LEVEL,
SESSION_UPDATE_PERIOD: process.env.SESSION_UPDATE_PERIOD,
DEPLOYMENT_ENVIRONMENT:

6
packages/backend-core/src/events/analytics.ts
View File

@ -1,7 +1,7 @@
import env from "../environment"
import tenancy from "../tenancy"
import * as dbUtils from "../db/utils"
import { Configs } from "../constants"
import { Config } from "../constants"
import { withCache, TTL, CacheKeys } from "../cache/generic"
export const enabled = async () => {
@ -45,9 +45,7 @@ const getSettingsDoc = async () => {
const db = tenancy.getGlobalDB()
let settings
try {
settings = await db.get(
dbUtils.generateConfigID({ type: Configs.SETTINGS })
)
settings = await db.get(dbUtils.generateConfigID({ type: Config.SETTINGS }))
} catch (e: any) {
if (e.status !== 404) {
throw e

4
packages/backend-core/src/events/identification.ts
View File

@ -19,7 +19,7 @@ import {
} from "@budibase/types"
import { processors } from "./processors"
import * as dbUtils from "../db/utils"
import { Configs } from "../constants"
import { Config } from "../constants"
import * as hashing from "../hashing"
import * as installation from "../installation"
import { withCache, TTL, CacheKeys } from "../cache/generic"
@ -273,7 +273,7 @@ const getUniqueTenantId = async (tenantId: string): Promise<string> => {
return withCache(CacheKeys.UNIQUE_TENANT_ID, TTL.ONE_DAY, async () => {
const db = context.getGlobalDB()
const config: SettingsConfig = await dbUtils.getScopedFullConfig(db, {
type: Configs.SETTINGS,
type: Config.SETTINGS,
})
let uniqueTenantId: string

2
packages/backend-core/src/events/processors/posthog/tests/PosthogProcessor.spec.ts
View File

@ -1,4 +1,4 @@
import "../../../../../tests/utilities/TestConfiguration"
import "../../../../../tests"
import PosthogProcessor from "../PosthogProcessor"
import { Event, IdentityType, Hosting } from "@budibase/types"
const tk = require("timekeeper")

6
packages/backend-core/src/index.ts
View File

@ -11,9 +11,9 @@ import env from "./environment"
import tenancy from "./tenancy"
import featureFlags from "./featureFlags"
import * as sessions from "./security/sessions"
import deprovisioning from "./context/deprovision"
import * as deprovisioning from "./context/deprovision"
import auth from "./auth"
import constants from "./constants"
import * as constants from "./constants"
import * as dbConstants from "./db/constants"
import * as logging from "./logging"
import pino from "./pino"
@ -21,9 +21,9 @@ import * as middleware from "./middleware"
import plugins from "./plugin"
import encryption from "./security/encryption"
import * as queue from "./queue"
import * as db from "./db"
// mimic the outer package exports
import * as db from "./pkg/db"
import * as objectStore from "./pkg/objectStore"
import * as utils from "./pkg/utils"
import redis from "./pkg/redis"

22
packages/backend-core/src/middleware/authenticated.ts
View File

@ -1,11 +1,9 @@
import { Cookies, Headers } from "../constants"
import { Cookie, Header } from "../constants"
import { getCookie, clearCookie, openJwt } from "../utils"
import { getUser } from "../cache/user"
import { getSession, updateSessionTTL } from "../security/sessions"
import { buildMatcherRegex, matches } from "./matchers"
import { SEPARATOR } from "../db/constants"
import { ViewName } from "../db/utils"
import { queryGlobalView } from "../db/views"
import { SEPARATOR, queryGlobalView, ViewName } from "../db"
import { getGlobalDB, doInTenant } from "../tenancy"
import { decrypt } from "../security/encryption"
const identity = require("../context/identity")
@ -74,7 +72,7 @@ export = (
const noAuthOptions = noAuthPatterns ? buildMatcherRegex(noAuthPatterns) : []
return async (ctx: any, next: any) => {
let publicEndpoint = false
const version = ctx.request.headers[Headers.API_VER]
const version = ctx.request.headers[Header.API_VER]
// the path is not authenticated
const found = matches(ctx, noAuthOptions)
if (found) {
@ -82,10 +80,10 @@ export = (
}
try {
// check the actual user is authenticated first, try header or cookie
const headerToken = ctx.request.headers[Headers.TOKEN]
const authCookie = getCookie(ctx, Cookies.Auth) || openJwt(headerToken)
const apiKey = ctx.request.headers[Headers.API_KEY]
const tenantId = ctx.request.headers[Headers.TENANT_ID]
const headerToken = ctx.request.headers[Header.TOKEN]
const authCookie = getCookie(ctx, Cookie.Auth) || openJwt(headerToken)
const apiKey = ctx.request.headers[Header.API_KEY]
const tenantId = ctx.request.headers[Header.TENANT_ID]
let authenticated = false,
user = null,
internal = false
@ -116,7 +114,7 @@ export = (
authenticated = false
console.error("Auth Error", err?.message || err)
// remove the cookie as the user does not exist anymore
clearCookie(ctx, Cookies.Auth)
clearCookie(ctx, Cookie.Auth)
}
}
// this is an internal request, no user made it
@ -140,7 +138,7 @@ export = (
delete user.password
}
// be explicit
if (authenticated !== true) {
if (!authenticated) {
authenticated = false
}
// isAuthenticated is a function, so use a variable to be able to check authed state
@ -155,7 +153,7 @@ export = (
console.error("Auth Error", err?.message || err)
// invalid token, clear the cookie
if (err && err.name === "JsonWebTokenError") {
clearCookie(ctx, Cookies.Auth)
clearCookie(ctx, Cookie.Auth)
}
// allow configuring for public access
if ((opts && opts.publicAllowed) || publicEndpoint) {

4
packages/backend-core/src/middleware/csrf.js
View File

@ -1,4 +1,4 @@
const { Headers } = require("../constants")
const { Header } = require("../constants")
const { buildMatcherRegex, matches } = require("./matchers")
/**
@ -68,7 +68,7 @@ module.exports = (opts = { noCsrfPatterns: [] }) => {
}
// reject if no token in request or mismatch
const requestToken = ctx.get(Headers.CSRF_TOKEN)
const requestToken = ctx.get(Header.CSRF_TOKEN)
if (!requestToken || requestToken !== userToken) {
ctx.throw(403, "Invalid CSRF token")
}

4
packages/backend-core/src/middleware/internalApi.js
View File

@ -1,11 +1,11 @@
const env = require("../environment")
const { Headers } = require("../constants")
const { Header } = require("../constants")
/**
* API Key only endpoint.
*/
module.exports = async (ctx, next) => {
const apiKey = ctx.request.headers[Headers.API_KEY]
const apiKey = ctx.request.headers[Header.API_KEY]
if (apiKey !== env.INTERNAL_API_KEY) {
ctx.throw(403, "Unauthorized")
}

19
packages/backend-core/src/middleware/matchers.js → packages/backend-core/src/middleware/matchers.ts
View File

@ -1,27 +1,34 @@
import { BBContext, EndpointMatcher, RegexMatcher } from "@budibase/types"
const PARAM_REGEX = /\/:(.*?)(\/.*)?$/g
exports.buildMatcherRegex = patterns => {
export const buildMatcherRegex = (
patterns: EndpointMatcher[]
): RegexMatcher[] => {
if (!patterns) {
return []
}
return patterns.map(pattern => {
const isObj = typeof pattern === "object" && pattern.route
const method = isObj ? pattern.method : "GET"
let route = pattern.route
const method = pattern.method
const strict = pattern.strict ? pattern.strict : false
let route = isObj ? pattern.route : pattern
// if there is a param in the route
// use a wildcard pattern
const matches = route.match(PARAM_REGEX)
if (matches) {
for (let match of matches) {
const pattern = "/.*" + (match.endsWith("/") ? "/" : "")
const suffix = match.endsWith("/") ? "/" : ""
const pattern = "/.*" + suffix
route = route.replace(match, pattern)
}
}
return { regex: new RegExp(route), method, strict, route }
})
}
exports.matches = (ctx, options) => {
export const matches = (ctx: BBContext, options: RegexMatcher[]) => {
return options.find(({ regex, method, strict, route }) => {
let urlMatch
if (strict) {

8
packages/backend-core/src/middleware/passport/datasource/google.js
View File

@ -1,6 +1,6 @@
const google = require("../google")
const GoogleStrategy = require("passport-google-oauth").OAuth2Strategy
const { Cookies, Configs } = require("../../../constants")
const { Cookie, Config } = require("../../../constants")
const { clearCookie, getCookie } = require("../../../utils")
const { getScopedConfig, getPlatformUrl } = require("../../../db/utils")
const { doWithDB } = require("../../../db")
@ -11,7 +11,7 @@ async function fetchGoogleCreds() {
// try and get the config from the tenant
const db = getGlobalDB()
const googleConfig = await getScopedConfig(db, {
type: Configs.GOOGLE,
type: Config.GOOGLE,
})
// or fall back to env variables
return (
@ -47,7 +47,7 @@ async function postAuth(passport, ctx, next) {
const platformUrl = await getPlatformUrl({ tenantAware: false })
let callbackUrl = `${platformUrl}/api/global/auth/datasource/google/callback`
const authStateCookie = getCookie(ctx, Cookies.DatasourceAuth)
const authStateCookie = getCookie(ctx, Cookie.DatasourceAuth)
return passport.authenticate(
new GoogleStrategy(
@ -57,7 +57,7 @@ async function postAuth(passport, ctx, next) {
callbackURL: callbackUrl,
},
(accessToken, refreshToken, profile, done) => {
clearCookie(ctx, Cookies.DatasourceAuth)
clearCookie(ctx, Cookie.DatasourceAuth)
done(null, { accessToken, refreshToken })
}
),

4
packages/backend-core/src/middleware/passport/google.js
View File

@ -1,7 +1,7 @@
const GoogleStrategy = require("passport-google-oauth").OAuth2Strategy
const { ssoCallbackUrl } = require("./utils")
const { authenticateThirdParty } = require("./third-party-common")
const { Configs } = require("../../../constants")
const { Config } = require("../../../constants")
const buildVerifyFn = saveUserFn => {
return (accessToken, refreshToken, profile, done) => {
@ -60,7 +60,7 @@ exports.strategyFactory = async function (config, callbackUrl, saveUserFn) {
}
exports.getCallbackUrl = async function (db, config) {
return ssoCallbackUrl(db, config, Configs.GOOGLE)
return ssoCallbackUrl(db, config, Config.GOOGLE)
}
// expose for testing

4
packages/backend-core/src/middleware/passport/jwt.js
View File

@ -1,11 +1,11 @@
const { Cookies } = require("../../constants")
const { Cookie } = require("../../constants")
const env = require("../../environment")
const { authError } = require("./utils")
exports.options = {
secretOrKey: env.JWT_SECRET,
jwtFromRequest: function (ctx) {
return ctx.cookies.get(Cookies.Auth)
return ctx.cookies.get(Cookie.Auth)
},
}

4
packages/backend-core/src/middleware/passport/oidc.js
View File

@ -2,7 +2,7 @@ const fetch = require("node-fetch")
const OIDCStrategy = require("@techpass/passport-openidconnect").Strategy
const { authenticateThirdParty } = require("./third-party-common")
const { ssoCallbackUrl } = require("./utils")
const { Configs } = require("../../../constants")
const { Config } = require("../../../constants")
const buildVerifyFn = saveUserFn => {
/**
@ -140,7 +140,7 @@ exports.fetchStrategyConfig = async function (enrichedConfig, callbackUrl) {
}
exports.getCallbackUrl = async function (db, config) {
return ssoCallbackUrl(db, config, Configs.OIDC)
return ssoCallbackUrl(db, config, Config.OIDC)
}
// expose for testing

5
packages/backend-core/src/middleware/passport/tests/oidc.spec.js
View File

@ -1,7 +1,6 @@
// Mock data
const mockFetch = require("node-fetch")
const { data } = require("./utilities/mock-data")
const issuer = "mockIssuer"
const sub = "mockSub"
const profile = {
@ -39,8 +38,6 @@ describe("oidc", () => {
const mockStrategy = require("@techpass/passport-openidconnect").Strategy
// mock the request to retrieve the oidc configuration
jest.mock("node-fetch")
const mockFetch = require("node-fetch")
mockFetch.mockReturnValue({
ok: true,
json: () => oidcConfigUrlResponse

2
packages/backend-core/src/middleware/passport/tests/third-party-common.spec.js
View File

@ -1,4 +1,4 @@
require("../../../../tests/utilities/TestConfiguration")
require("../../../../tests")
const { authenticateThirdParty } = require("../third-party-common")
const { data } = require("./utilities/mock-data")
const { DEFAULT_TENANT_ID } = require("../../../constants")

4
packages/backend-core/src/middleware/passport/utils.js
View File

@ -1,6 +1,6 @@
const { isMultiTenant, getTenantId } = require("../../tenancy")
const { getScopedConfig } = require("../../db/utils")
const { Configs } = require("../../constants")
const { Config } = require("../../constants")
/**
* Utility to handle authentication errors.
@ -24,7 +24,7 @@ exports.ssoCallbackUrl = async (db, config, type) => {
return config.callbackURL
}
const publicConfig = await getScopedConfig(db, {
type: Configs.SETTINGS,
type: Config.SETTINGS,
})
let callbackUrl = `/api/global/auth`

52
packages/backend-core/src/middleware/tenancy.js
View File

@ -1,52 +0,0 @@
const { doInTenant, isMultiTenant, DEFAULT_TENANT_ID } = require("../tenancy")
const { buildMatcherRegex, matches } = require("./matchers")
const { Headers } = require("../constants")
const getTenantID = (ctx, opts = { allowQs: false, allowNoTenant: false }) => {
// exit early if not multi-tenant
if (!isMultiTenant()) {
return DEFAULT_TENANT_ID
}
let tenantId
const allowQs = opts && opts.allowQs
const allowNoTenant = opts && opts.allowNoTenant
const header = ctx.request.headers[Headers.TENANT_ID]
const user = ctx.user || {}
if (allowQs) {
const query = ctx.request.query || {}
tenantId = query.tenantId
}
// override query string (if allowed) by user, or header
// URL params cannot be used in a middleware, as they are
// processed later in the chain
tenantId = user.tenantId || header || tenantId
// Set the tenantId from the subdomain
if (!tenantId) {
tenantId = ctx.subdomains && ctx.subdomains[0]
}
if (!tenantId && !allowNoTenant) {
ctx.throw(403, "Tenant id not set")
}
return tenantId
}
module.exports = (
allowQueryStringPatterns,
noTenancyPatterns,
opts = { noTenancyRequired: false }
) => {
const allowQsOptions = buildMatcherRegex(allowQueryStringPatterns)
const noTenancyOptions = buildMatcherRegex(noTenancyPatterns)
return async function (ctx, next) {
const allowNoTenant =
opts.noTenancyRequired || !!matches(ctx, noTenancyOptions)
const allowQs = !!matches(ctx, allowQsOptions)
const tenantId = getTenantID(ctx, { allowQs, allowNoTenant })
return doInTenant(tenantId, next)
}
}

37
packages/backend-core/src/middleware/tenancy.ts Normal file
View File

@ -0,0 +1,37 @@
import { doInTenant, getTenantIDFromCtx } from "../tenancy"
import { buildMatcherRegex, matches } from "./matchers"
import { Header } from "../constants"
import {
BBContext,
EndpointMatcher,
GetTenantIdOptions,
TenantResolutionStrategy,
} from "@budibase/types"
const tenancy = (
allowQueryStringPatterns: EndpointMatcher[],
noTenancyPatterns: EndpointMatcher[],
opts = { noTenancyRequired: false }
) => {
const allowQsOptions = buildMatcherRegex(allowQueryStringPatterns)
const noTenancyOptions = buildMatcherRegex(noTenancyPatterns)
return async function (ctx: BBContext, next: any) {
const allowNoTenant =
opts.noTenancyRequired || !!matches(ctx, noTenancyOptions)
const tenantOpts: GetTenantIdOptions = {
allowNoTenant,
}
const allowQs = !!matches(ctx, allowQsOptions)
if (!allowQs) {
tenantOpts.excludeStrategies = [TenantResolutionStrategy.QUERY]
}
const tenantId = getTenantIDFromCtx(ctx, tenantOpts)
ctx.set(Header.TENANT_ID, tenantId as string)
return doInTenant(tenantId, next)
}
}
export = tenancy

134
packages/backend-core/src/middleware/tests/matchers.spec.ts Normal file
View File

@ -0,0 +1,134 @@
import * as matchers from "../matchers"
import { structures } from "../../../tests"
describe("matchers", () => {
it("matches by path and method", () => {
const pattern = [
{
route: "/api/tests",
method: "POST",
},
]
const ctx = structures.koa.newContext()
ctx.request.url = "/api/tests"
ctx.request.method = "POST"
const built = matchers.buildMatcherRegex(pattern)
expect(!!matchers.matches(ctx, built)).toBe(true)
})
it("wildcards path", () => {
const pattern = [
{
route: "/api/tests",
method: "POST",
},
]
const ctx = structures.koa.newContext()
ctx.request.url = "/api/tests/id/something/else"
ctx.request.method = "POST"
const built = matchers.buildMatcherRegex(pattern)
expect(!!matchers.matches(ctx, built)).toBe(true)
})
it("doesn't wildcard path with strict", () => {
const pattern = [
{
route: "/api/tests",
method: "POST",
strict: true,
},
]
const ctx = structures.koa.newContext()
ctx.request.url = "/api/tests/id/something/else"
ctx.request.method = "POST"
const built = matchers.buildMatcherRegex(pattern)
expect(!!matchers.matches(ctx, built)).toBe(false)
})
it("matches with param", () => {
const pattern = [
{
route: "/api/tests/:testId",
method: "GET",
},
]
const ctx = structures.koa.newContext()
ctx.request.url = "/api/tests/id"
ctx.request.method = "GET"
const built = matchers.buildMatcherRegex(pattern)
expect(!!matchers.matches(ctx, built)).toBe(true)
})
// TODO: Support the below behaviour
// Strict does not work when a param is present
// it("matches with param with strict", () => {
// const pattern = [{
// route: "/api/tests/:testId",
// method: "GET",
// strict: true
// }]
// const ctx = structures.koa.newContext()
// ctx.request.url = "/api/tests/id"
// ctx.request.method = "GET"
//
// const built = matchers.buildMatcherRegex(pattern)
//
// expect(!!matchers.matches(ctx, built)).toBe(true)
// })
it("doesn't match by path", () => {
const pattern = [
{
route: "/api/tests",
method: "POST",
},
]
const ctx = structures.koa.newContext()
ctx.request.url = "/api/unknown"
ctx.request.method = "POST"
const built = matchers.buildMatcherRegex(pattern)
expect(!!matchers.matches(ctx, built)).toBe(false)
})
it("doesn't match by method", () => {
const pattern = [
{
route: "/api/tests",
method: "POST",
},
]
const ctx = structures.koa.newContext()
ctx.request.url = "/api/tests"
ctx.request.method = "GET"
const built = matchers.buildMatcherRegex(pattern)
expect(!!matchers.matches(ctx, built)).toBe(false)
})
it("matches by path and wildcard method", () => {
const pattern = [
{
route: "/api/tests",
method: "ALL",
},
]
const ctx = structures.koa.newContext()
ctx.request.url = "/api/tests"
ctx.request.method = "GET"
const built = matchers.buildMatcherRegex(pattern)
expect(!!matchers.matches(ctx, built)).toBe(true)
})
})

10
packages/backend-core/src/migrations/migrations.ts
View File

@ -12,6 +12,7 @@ import {
MigrationOptions,
MigrationType,
MigrationNoOpOptions,
App,
} from "@budibase/types"
export const getMigrationsDoc = async (db: any) => {
@ -41,7 +42,7 @@ export const runMigration = async (
options: MigrationOptions = {}
) => {
const migrationType = migration.type
let tenantId: string
let tenantId: string | undefined
if (migrationType !== MigrationType.INSTALLATION) {
tenantId = getTenantId()
}
@ -55,14 +56,17 @@ export const runMigration = async (
}
// get the db to store the migration in
let dbNames
let dbNames: string[]
if (migrationType === MigrationType.GLOBAL) {
dbNames = [getGlobalDBName()]
} else if (migrationType === MigrationType.APP) {
if (options.noOp) {
if (!options.noOp.appId) {
throw new Error("appId is required for noOp app migration")
}
dbNames = [options.noOp.appId]
} else {
const apps = await getAllApps(migration.appOpts)
const apps = (await getAllApps(migration.appOpts)) as App[]
dbNames = apps.map(app => app.appId)
}
} else if (migrationType === MigrationType.INSTALLATION) {

6
packages/backend-core/src/migrations/tests/index.spec.js
View File

@ -1,6 +1,6 @@
require("../../../tests/utilities/TestConfiguration")
require("../../../tests")
const { runMigrations, getMigrationsDoc } = require("../index")
const { dangerousGetDB } = require("../../db")
const { getDB } = require("../../db")
const {
StaticDatabases,
} = require("../../db/utils")
@ -18,7 +18,7 @@ describe("migrations", () => {
}]
beforeEach(() => {
db = dangerousGetDB(StaticDatabases.GLOBAL.name)
db = getDB(StaticDatabases.GLOBAL.name)
})
afterEach(async () => {

2
packages/backend-core/src/pkg/cache.ts
View File

@ -3,9 +3,11 @@
import * as generic from "../cache/generic"
import * as user from "../cache/user"
import * as app from "../cache/appMetadata"
import * as writethrough from "../cache/writethrough"
export = {
app,
user,
writethrough,
...generic,
}

7
packages/backend-core/src/pkg/db.ts
View File

@ -1,7 +0,0 @@
// Mimic the outer package export for usage in index.ts
// The outer exports can't be used as they now reference dist directly
export * from "../db"
export * from "../db/utils"
export * from "../db/views"
export * from "../db/pouch"
export * from "../db/constants"

8
packages/backend-core/src/queue/inMemoryQueue.ts
View File

@ -29,6 +29,7 @@ class InMemoryQueue {
_messages: any[]
_emitter: EventEmitter
_runCount: number
_addCount: number
/**
* The constructor the queue, exactly the same as that of Bulls.
* @param {string} name The name of the queue which is being configured.
@ -41,6 +42,7 @@ class InMemoryQueue {
this._messages = []
this._emitter = new events.EventEmitter()
this._runCount = 0
this._addCount = 0
}
/**
@ -81,6 +83,7 @@ class InMemoryQueue {
throw "Queue only supports carrying JSON."
}
this._messages.push(newJob(this._name, msg))
this._addCount++
this._emitter.emit("message")
}
@ -128,12 +131,9 @@ class InMemoryQueue {
}
async waitForCompletion() {
const currentCount = this._runCount
let increased = false
do {
await timeout(50)
increased = this._runCount > currentCount
} while (!increased)
} while (this._addCount < this._runCount)
}
}

164
packages/backend-core/src/security/permissions.js
View File

@ -1,164 +0,0 @@
const { flatten } = require("lodash")
const { cloneDeep } = require("lodash/fp")
const PermissionLevels = {
READ: "read",
WRITE: "write",
EXECUTE: "execute",
ADMIN: "admin",
}
// these are the global types, that govern the underlying default behaviour
const PermissionTypes = {
APP: "app",
TABLE: "table",
USER: "user",
AUTOMATION: "automation",
WEBHOOK: "webhook",
BUILDER: "builder",
VIEW: "view",
QUERY: "query",
}
function Permission(type, level) {
this.level = level
this.type = type
}
function levelToNumber(perm) {
switch (perm) {
// not everything has execute privileges
case PermissionLevels.EXECUTE:
return 0
case PermissionLevels.READ:
return 1
case PermissionLevels.WRITE:
return 2
case PermissionLevels.ADMIN:
return 3
default:
return -1
}
}
/**
* Given the specified permission level for the user return the levels they are allowed to carry out.
* @param {string} userPermLevel The permission level of the user.
* @return {string[]} All the permission levels this user is allowed to carry out.
*/
function getAllowedLevels(userPermLevel) {
switch (userPermLevel) {
case PermissionLevels.EXECUTE:
return [PermissionLevels.EXECUTE]
case PermissionLevels.READ:
return [PermissionLevels.EXECUTE, PermissionLevels.READ]
case PermissionLevels.WRITE:
case PermissionLevels.ADMIN:
return [
PermissionLevels.READ,
PermissionLevels.WRITE,
PermissionLevels.EXECUTE,
]
default:
return []
}
}
exports.BUILTIN_PERMISSION_IDS = {
PUBLIC: "public",
READ_ONLY: "read_only",
WRITE: "write",
ADMIN: "admin",
POWER: "power",
}
const BUILTIN_PERMISSIONS = {
PUBLIC: {
_id: exports.BUILTIN_PERMISSION_IDS.PUBLIC,
name: "Public",
permissions: [
new Permission(PermissionTypes.WEBHOOK, PermissionLevels.EXECUTE),
],
},
READ_ONLY: {
_id: exports.BUILTIN_PERMISSION_IDS.READ_ONLY,
name: "Read only",
permissions: [
new Permission(PermissionTypes.QUERY, PermissionLevels.READ),
new Permission(PermissionTypes.TABLE, PermissionLevels.READ),
new Permission(PermissionTypes.VIEW, PermissionLevels.READ),
],
},
WRITE: {
_id: exports.BUILTIN_PERMISSION_IDS.WRITE,
name: "Read/Write",
permissions: [
new Permission(PermissionTypes.QUERY, PermissionLevels.WRITE),
new Permission(PermissionTypes.TABLE, PermissionLevels.WRITE),
new Permission(PermissionTypes.VIEW, PermissionLevels.READ),
new Permission(PermissionTypes.AUTOMATION, PermissionLevels.EXECUTE),
],
},
POWER: {
_id: exports.BUILTIN_PERMISSION_IDS.POWER,
name: "Power",
permissions: [
new Permission(PermissionTypes.TABLE, PermissionLevels.WRITE),
new Permission(PermissionTypes.USER, PermissionLevels.READ),
new Permission(PermissionTypes.AUTOMATION, PermissionLevels.EXECUTE),
new Permission(PermissionTypes.VIEW, PermissionLevels.READ),
new Permission(PermissionTypes.WEBHOOK, PermissionLevels.READ),
],
},
ADMIN: {
_id: exports.BUILTIN_PERMISSION_IDS.ADMIN,
name: "Admin",
permissions: [
new Permission(PermissionTypes.TABLE, PermissionLevels.ADMIN),
new Permission(PermissionTypes.USER, PermissionLevels.ADMIN),
new Permission(PermissionTypes.AUTOMATION, PermissionLevels.ADMIN),
new Permission(PermissionTypes.VIEW, PermissionLevels.ADMIN),
new Permission(PermissionTypes.WEBHOOK, PermissionLevels.READ),
new Permission(PermissionTypes.QUERY, PermissionLevels.ADMIN),
],
},
}
exports.getBuiltinPermissions = () => {
return cloneDeep(BUILTIN_PERMISSIONS)
}
exports.getBuiltinPermissionByID = id => {
const perms = Object.values(BUILTIN_PERMISSIONS)
return perms.find(perm => perm._id === id)
}
exports.doesHaveBasePermission = (permType, permLevel, rolesHierarchy) => {
const basePermissions = [
...new Set(rolesHierarchy.map(role => role.permissionId)),
]
const builtins = Object.values(BUILTIN_PERMISSIONS)
let permissions = flatten(
builtins
.filter(builtin => basePermissions.indexOf(builtin._id) !== -1)
.map(builtin => builtin.permissions)
)
for (let permission of permissions) {
if (
permission.type === permType &&
getAllowedLevels(permission.level).indexOf(permLevel) !== -1
) {
return true
}
}
return false
}
exports.isPermissionLevelHigherThanRead = level => {
return levelToNumber(level) > 1
}
// utility as a lot of things need simply the builder permission
exports.BUILDER = PermissionTypes.BUILDER
exports.PermissionTypes = PermissionTypes
exports.PermissionLevels = PermissionLevels

175
packages/backend-core/src/security/permissions.ts Normal file
View File

@ -0,0 +1,175 @@
const { flatten } = require("lodash")
const { cloneDeep } = require("lodash/fp")
export type RoleHierarchy = {
permissionId: string
}[]
export enum PermissionLevel {
READ = "read",
WRITE = "write",
EXECUTE = "execute",
ADMIN = "admin",
}
// these are the global types, that govern the underlying default behaviour
export enum PermissionType {
APP = "app",
TABLE = "table",
USER = "user",
AUTOMATION = "automation",
WEBHOOK = "webhook",
BUILDER = "builder",
VIEW = "view",
QUERY = "query",
}
class Permission {
type: PermissionType
level: PermissionLevel
constructor(type: PermissionType, level: PermissionLevel) {
this.type = type
this.level = level
}
}
function levelToNumber(perm: PermissionLevel) {
switch (perm) {
// not everything has execute privileges
case PermissionLevel.EXECUTE:
return 0
case PermissionLevel.READ:
return 1
case PermissionLevel.WRITE:
return 2
case PermissionLevel.ADMIN:
return 3
default:
return -1
}
}
/**
* Given the specified permission level for the user return the levels they are allowed to carry out.
* @param {string} userPermLevel The permission level of the user.
* @return {string[]} All the permission levels this user is allowed to carry out.
*/
function getAllowedLevels(userPermLevel: PermissionLevel) {
switch (userPermLevel) {
case PermissionLevel.EXECUTE:
return [PermissionLevel.EXECUTE]
case PermissionLevel.READ:
return [PermissionLevel.EXECUTE, PermissionLevel.READ]
case PermissionLevel.WRITE:
case PermissionLevel.ADMIN:
return [
PermissionLevel.READ,
PermissionLevel.WRITE,
PermissionLevel.EXECUTE,
]
default:
return []
}
}
export enum BuiltinPermissionID {
PUBLIC = "public",
READ_ONLY = "read_only",
WRITE = "write",
ADMIN = "admin",
POWER = "power",
}
const BUILTIN_PERMISSIONS = {
PUBLIC: {
_id: BuiltinPermissionID.PUBLIC,
name: "Public",
permissions: [
new Permission(PermissionType.WEBHOOK, PermissionLevel.EXECUTE),
],
},
READ_ONLY: {
_id: BuiltinPermissionID.READ_ONLY,
name: "Read only",
permissions: [
new Permission(PermissionType.QUERY, PermissionLevel.READ),
new Permission(PermissionType.TABLE, PermissionLevel.READ),
new Permission(PermissionType.VIEW, PermissionLevel.READ),
],
},
WRITE: {
_id: BuiltinPermissionID.WRITE,
name: "Read/Write",
permissions: [
new Permission(PermissionType.QUERY, PermissionLevel.WRITE),
new Permission(PermissionType.TABLE, PermissionLevel.WRITE),
new Permission(PermissionType.VIEW, PermissionLevel.READ),
new Permission(PermissionType.AUTOMATION, PermissionLevel.EXECUTE),
],
},
POWER: {
_id: BuiltinPermissionID.POWER,
name: "Power",
permissions: [
new Permission(PermissionType.TABLE, PermissionLevel.WRITE),
new Permission(PermissionType.USER, PermissionLevel.READ),
new Permission(PermissionType.AUTOMATION, PermissionLevel.EXECUTE),
new Permission(PermissionType.VIEW, PermissionLevel.READ),
new Permission(PermissionType.WEBHOOK, PermissionLevel.READ),
],
},
ADMIN: {
_id: BuiltinPermissionID.ADMIN,
name: "Admin",
permissions: [
new Permission(PermissionType.TABLE, PermissionLevel.ADMIN),
new Permission(PermissionType.USER, PermissionLevel.ADMIN),
new Permission(PermissionType.AUTOMATION, PermissionLevel.ADMIN),
new Permission(PermissionType.VIEW, PermissionLevel.ADMIN),
new Permission(PermissionType.WEBHOOK, PermissionLevel.READ),
new Permission(PermissionType.QUERY, PermissionLevel.ADMIN),
],
},
}
export function getBuiltinPermissions() {
return cloneDeep(BUILTIN_PERMISSIONS)
}
export function getBuiltinPermissionByID(id: string) {
const perms = Object.values(BUILTIN_PERMISSIONS)
return perms.find(perm => perm._id === id)
}
export function doesHaveBasePermission(
permType: PermissionType,
permLevel: PermissionLevel,
rolesHierarchy: RoleHierarchy
) {
const basePermissions = [
...new Set(rolesHierarchy.map(role => role.permissionId)),
]
const builtins = Object.values(BUILTIN_PERMISSIONS)
let permissions = flatten(
builtins
.filter(builtin => basePermissions.indexOf(builtin._id) !== -1)
.map(builtin => builtin.permissions)
)
for (let permission of permissions) {
if (
permission.type === permType &&
getAllowedLevels(permission.level).indexOf(permLevel) !== -1
) {
return true
}
}
return false
}
export function isPermissionLevelHigherThanRead(level: PermissionLevel) {
return levelToNumber(level) > 1
}
// utility as a lot of things need simply the builder permission
export const BUILDER = PermissionType.BUILDER

24
packages/backend-core/src/security/roles.ts
View File

@ -1,4 +1,4 @@
import { BUILTIN_PERMISSION_IDS, PermissionLevels } from "./permissions"
import { BuiltinPermissionID, PermissionLevel } from "./permissions"
import {
generateRoleID,
getRoleParams,
@ -54,19 +54,19 @@ export class Role {
const BUILTIN_ROLES = {
ADMIN: new Role(BUILTIN_IDS.ADMIN, "Admin")
.addPermission(BUILTIN_PERMISSION_IDS.ADMIN)
.addPermission(BuiltinPermissionID.ADMIN)
.addInheritance(BUILTIN_IDS.POWER),
POWER: new Role(BUILTIN_IDS.POWER, "Power")
.addPermission(BUILTIN_PERMISSION_IDS.POWER)
.addPermission(BuiltinPermissionID.POWER)
.addInheritance(BUILTIN_IDS.BASIC),
BASIC: new Role(BUILTIN_IDS.BASIC, "Basic")
.addPermission(BUILTIN_PERMISSION_IDS.WRITE)
.addPermission(BuiltinPermissionID.WRITE)
.addInheritance(BUILTIN_IDS.PUBLIC),
PUBLIC: new Role(BUILTIN_IDS.PUBLIC, "Public").addPermission(
BUILTIN_PERMISSION_IDS.PUBLIC
BuiltinPermissionID.PUBLIC
),
BUILDER: new Role(BUILTIN_IDS.BUILDER, "Builder").addPermission(
BUILTIN_PERMISSION_IDS.ADMIN
BuiltinPermissionID.ADMIN
),
}
@ -147,9 +147,9 @@ export function lowerBuiltinRoleID(roleId1?: string, roleId2?: string) {
* @param {string|null} roleId The level ID to lookup.
* @returns {Promise<Role|object|null>} The role object, which may contain an "inherits" property.
*/
export async function getRole(roleId?: string) {
export async function getRole(roleId?: string): Promise<RoleDoc | undefined> {
if (!roleId) {
return null
return undefined
}
let role: any = {}
// built in roles mostly come from the in-code implementation,
@ -193,7 +193,9 @@ async function getAllUserRoles(userRoleId?: string): Promise<RoleDoc[]> {
) {
roleIds.push(currentRole.inherits)
currentRole = await getRole(currentRole.inherits)
roles.push(currentRole)
if (currentRole) {
roles.push(currentRole)
}
}
return roles
}
@ -225,8 +227,8 @@ export function checkForRoleResourceArray(
if (rolePerms && !Array.isArray(rolePerms[resourceId])) {
const permLevel = rolePerms[resourceId] as any
rolePerms[resourceId] = [permLevel]
if (permLevel === PermissionLevels.WRITE) {
rolePerms[resourceId].push(PermissionLevels.READ)
if (permLevel === PermissionLevel.WRITE) {
rolePerms[resourceId].push(PermissionLevel.READ)
}
}
return rolePerms

167
packages/backend-core/src/tenancy/tenancy.ts
View File

@ -1,20 +1,29 @@
import { doWithDB } from "../db"
import { queryPlatformView } from "../db/views"
import { StaticDatabases, ViewName } from "../db/constants"
import { getGlobalDBName } from "../db/tenancy"
import {
getTenantId,
doWithDB,
queryPlatformView,
StaticDatabases,
getGlobalDBName,
ViewName,
} from "../db"
import {
DEFAULT_TENANT_ID,
isMultiTenant,
getTenantId,
getTenantIDFromAppID,
isMultiTenant,
} from "../context"
import env from "../environment"
import { PlatformUser } from "@budibase/types"
import {
BBContext,
PlatformUser,
TenantResolutionStrategy,
GetTenantIdOptions,
} from "@budibase/types"
import { Header } from "../constants"
const TENANT_DOC = StaticDatabases.PLATFORM_INFO.docs.tenants
const PLATFORM_INFO_DB = StaticDatabases.PLATFORM_INFO.name
export const addTenantToUrl = (url: string) => {
export function addTenantToUrl(url: string) {
const tenantId = getTenantId()
if (isMultiTenant()) {
@ -25,7 +34,7 @@ export const addTenantToUrl = (url: string) => {
return url
}
export const doesTenantExist = async (tenantId: string) => {
export async function doesTenantExist(tenantId: string) {
return doWithDB(PLATFORM_INFO_DB, async (db: any) => {
let tenants
try {
@ -42,12 +51,12 @@ export const doesTenantExist = async (tenantId: string) => {
})
}
export const tryAddTenant = async (
export async function tryAddTenant(
tenantId: string,
userId: string,
email: string,
afterCreateTenant: () => Promise<void>
) => {
) {
return doWithDB(PLATFORM_INFO_DB, async (db: any) => {
const getDoc = async (id: string) => {
if (!id) {
@ -89,11 +98,11 @@ export const tryAddTenant = async (
})
}
export const doWithGlobalDB = (tenantId: string, cb: any) => {
export function doWithGlobalDB(tenantId: string, cb: any) {
return doWithDB(getGlobalDBName(tenantId), cb)
}
export const lookupTenantId = async (userId: string) => {
export async function lookupTenantId(userId: string) {
return doWithDB(StaticDatabases.PLATFORM_INFO.name, async (db: any) => {
let tenantId = env.MULTI_TENANCY ? DEFAULT_TENANT_ID : null
try {
@ -109,19 +118,26 @@ export const lookupTenantId = async (userId: string) => {
}
// lookup, could be email or userId, either will return a doc
export const getTenantUser = async (
export async function getTenantUser(
identifier: string
): Promise<PlatformUser | null> => {
): Promise<PlatformUser | undefined> {
// use the view here and allow to find anyone regardless of casing
// Use lowercase to ensure email login is case insensitive
const response = queryPlatformView(ViewName.PLATFORM_USERS_LOWERCASE, {
keys: [identifier.toLowerCase()],
include_docs: true,
}) as Promise<PlatformUser>
return response
// Use lowercase to ensure email login is case-insensitive
const users = await queryPlatformView<PlatformUser>(
ViewName.PLATFORM_USERS_LOWERCASE,
{
keys: [identifier.toLowerCase()],
include_docs: true,
}
)
if (Array.isArray(users)) {
return users[0]
} else {
return users
}
}
export const isUserInAppTenant = (appId: string, user?: any) => {
export function isUserInAppTenant(appId: string, user?: any) {
let userTenantId
if (user) {
userTenantId = user.tenantId || DEFAULT_TENANT_ID
@ -132,7 +148,7 @@ export const isUserInAppTenant = (appId: string, user?: any) => {
return tenantId === userTenantId
}
export const getTenantIds = async () => {
export async function getTenantIds() {
return doWithDB(PLATFORM_INFO_DB, async (db: any) => {
let tenants
try {
@ -144,3 +160,108 @@ export const getTenantIds = async () => {
return (tenants && tenants.tenantIds) || []
})
}
const ALL_STRATEGIES = Object.values(TenantResolutionStrategy)
export const getTenantIDFromCtx = (
ctx: BBContext,
opts: GetTenantIdOptions
): string | null => {
// exit early if not multi-tenant
if (!isMultiTenant()) {
return DEFAULT_TENANT_ID
}
// opt defaults
if (opts.allowNoTenant === undefined) {
opts.allowNoTenant = false
}
if (!opts.includeStrategies) {
opts.includeStrategies = ALL_STRATEGIES
}
if (!opts.excludeStrategies) {
opts.excludeStrategies = []
}
const isAllowed = (strategy: TenantResolutionStrategy) => {
// excluded takes precedence
if (opts.excludeStrategies?.includes(strategy)) {
return false
}
if (opts.includeStrategies?.includes(strategy)) {
return true
}
}
// always use user first
if (isAllowed(TenantResolutionStrategy.USER)) {
const userTenantId = ctx.user?.tenantId
if (userTenantId) {
return userTenantId
}
}
// header
if (isAllowed(TenantResolutionStrategy.HEADER)) {
const headerTenantId = ctx.request.headers[Header.TENANT_ID]
if (headerTenantId) {
return headerTenantId as string
}
}
// query param
if (isAllowed(TenantResolutionStrategy.QUERY)) {
const queryTenantId = ctx.request.query.tenantId
if (queryTenantId) {
return queryTenantId as string
}
}
// subdomain
if (isAllowed(TenantResolutionStrategy.SUBDOMAIN)) {
// e.g. budibase.app or local.com:10000
const platformHost = new URL(env.PLATFORM_URL).host.split(":")[0]
// e.g. tenant.budibase.app or tenant.local.com
const requestHost = ctx.host
// parse the tenant id from the difference
if (requestHost.includes(platformHost)) {
const tenantId = requestHost.substring(
0,
requestHost.indexOf(`.${platformHost}`)
)
if (tenantId) {
return tenantId
}
}
}
// path
if (isAllowed(TenantResolutionStrategy.PATH)) {
// params - have to parse manually due to koa-router not run yet
const match = ctx.matched.find(
(m: any) => !!m.paramNames.find((p: any) => p.name === "tenantId")
)
// get the raw path url - without any query params
const ctxUrl = ctx.originalUrl
let url
if (ctxUrl.includes("?")) {
url = ctxUrl.split("?")[0]
} else {
url = ctxUrl
}
if (match) {
const params = match.params(url, match.captures(url), {})
if (params.tenantId) {
return params.tenantId
}
}
}
if (!opts.allowNoTenant) {
ctx.throw(403, "Tenant id not set")
}
return null
}

3
packages/backend-core/src/tests/utils.spec.js
View File

@ -1,5 +1,4 @@
require("../../tests/utilities/TestConfiguration")
const { structures } = require("../../tests/utilities")
const { structures } = require("../../tests")
const utils = require("../utils")
const events = require("../events")
const { doInTenant, DEFAULT_TENANT_ID }= require("../context")

13
packages/backend-core/src/users.ts
View File

@ -3,15 +3,14 @@ import {
getUsersByAppParams,
getProdAppID,
generateAppUserID,
} from "./db/utils"
import { queryGlobalView } from "./db/views"
import { UNICODE_MAX } from "./db/constants"
queryGlobalView,
UNICODE_MAX,
} from "./db"
import { BulkDocsResponse, User } from "@budibase/types"
import { getGlobalDB } from "./context"
import PouchDB from "pouchdb"
export const bulkGetGlobalUsersById = async (userIds: string[]) => {
const db = getGlobalDB() as PouchDB.Database
const db = getGlobalDB()
return (
await db.allDocs({
keys: userIds,
@ -21,7 +20,7 @@ export const bulkGetGlobalUsersById = async (userIds: string[]) => {
}
export const bulkUpdateGlobalUsers = async (users: User[]) => {
const db = getGlobalDB() as PouchDB.Database
const db = getGlobalDB()
return (await db.bulkDocs(users)) as BulkDocsResponse
}
@ -69,7 +68,7 @@ export const getGlobalUserByAppPage = (appId: string, user: User) => {
if (!user) {
return
}
return generateAppUserID(getProdAppID(appId), user._id!)
return generateAppUserID(getProdAppID(appId)!, user._id!)
}
/**

100
packages/backend-core/src/utils.js → packages/backend-core/src/utils.ts
View File

@ -1,38 +1,51 @@
const { DocumentType, SEPARATOR, ViewName, getAllApps } = require("./db/utils")
import {
DocumentType,
SEPARATOR,
ViewName,
getAllApps,
queryGlobalView,
} from "./db"
import { options } from "./middleware/passport/jwt"
import { Header, Cookie, MAX_VALID_DATE } from "./constants"
import env from "./environment"
import userCache from "./cache/user"
import { getSessionsForUser, invalidateSessions } from "./security/sessions"
import * as events from "./events"
import tenancy from "./tenancy"
import {
App,
BBContext,
PlatformLogoutOpts,
TenantResolutionStrategy,
} from "@budibase/types"
import { SetOption } from "cookies"
const jwt = require("jsonwebtoken")
const { options } = require("./middleware/passport/jwt")
const { queryGlobalView } = require("./db/views")
const { Headers, Cookies, MAX_VALID_DATE } = require("./constants")
const env = require("./environment")
const userCache = require("./cache/user")
const {
getSessionsForUser,
invalidateSessions,
} = require("./security/sessions")
const events = require("./events")
const tenancy = require("./tenancy")
const APP_PREFIX = DocumentType.APP + SEPARATOR
const PROD_APP_PREFIX = "/app/"
function confirmAppId(possibleAppId) {
function confirmAppId(possibleAppId: string | undefined) {
return possibleAppId && possibleAppId.startsWith(APP_PREFIX)
? possibleAppId
: undefined
}
async function resolveAppUrl(ctx) {
async function resolveAppUrl(ctx: BBContext) {
const appUrl = ctx.path.split("/")[2]
let possibleAppUrl = `/${appUrl.toLowerCase()}`
let tenantId = tenancy.getTenantId()
if (!env.SELF_HOSTED && ctx.subdomains.length) {
// always use the tenant id from the url in cloud
tenantId = ctx.subdomains[0]
let tenantId: string | null = tenancy.getTenantId()
if (env.MULTI_TENANCY) {
// always use the tenant id from the subdomain in multi tenancy
// this ensures the logged-in user tenant id doesn't overwrite
// e.g. in the case of viewing a public app while already logged-in to another tenant
tenantId = tenancy.getTenantIDFromCtx(ctx, {
includeStrategies: [TenantResolutionStrategy.SUBDOMAIN],
})
}
// search prod apps for a url that matches
const apps = await tenancy.doInTenant(tenantId, () =>
const apps: App[] = await tenancy.doInTenant(tenantId, () =>
getAllApps({ dev: false })
)
const app = apps.filter(
@ -42,7 +55,7 @@ async function resolveAppUrl(ctx) {
return app && app.appId ? app.appId : undefined
}
exports.isServingApp = ctx => {
export function isServingApp(ctx: BBContext) {
// dev app
if (ctx.path.startsWith(`/${APP_PREFIX}`)) {
return true
@ -59,12 +72,12 @@ exports.isServingApp = ctx => {
* @param {object} ctx The main request body to look through.
* @returns {string|undefined} If an appId was found it will be returned.
*/
exports.getAppIdFromCtx = async ctx => {
export async function getAppIdFromCtx(ctx: BBContext) {
// look in headers
const options = [ctx.headers[Headers.APP_ID]]
const options = [ctx.headers[Header.APP_ID]]
let appId
for (let option of options) {
appId = confirmAppId(option)
appId = confirmAppId(option as string)
if (appId) {
break
}
@ -95,7 +108,7 @@ exports.getAppIdFromCtx = async ctx => {
* opens the contents of the specified encrypted JWT.
* @return {object} the contents of the token.
*/
exports.openJwt = token => {
export function openJwt(token: string) {
if (!token) {
return token
}
@ -107,14 +120,14 @@ exports.openJwt = token => {
* @param {object} ctx The request which is to be manipulated.
* @param {string} name The name of the cookie to get.
*/
exports.getCookie = (ctx, name) => {
export function getCookie(ctx: BBContext, name: string) {
const cookie = ctx.cookies.get(name)
if (!cookie) {
return cookie
}
return exports.openJwt(cookie)
return openJwt(cookie)
}
/**
@ -124,12 +137,17 @@ exports.getCookie = (ctx, name) => {
* @param {string|object} value The value of cookie which will be set.
* @param {object} opts options like whether to sign.
*/
exports.setCookie = (ctx, value, name = "builder", opts = { sign: true }) => {
export function setCookie(
ctx: BBContext,
value: any,
name = "builder",
opts = { sign: true }
) {
if (value && opts && opts.sign) {
value = jwt.sign(value, options.secretOrKey)
}
const config = {
const config: SetOption = {
expires: MAX_VALID_DATE,
path: "/",
httpOnly: false,
@ -146,8 +164,8 @@ exports.setCookie = (ctx, value, name = "builder", opts = { sign: true }) => {
/**
* Utility function, simply calls setCookie with an empty string for value
*/
exports.clearCookie = (ctx, name) => {
exports.setCookie(ctx, null, name)
export function clearCookie(ctx: BBContext, name: string) {
setCookie(ctx, null, name)
}
/**
@ -156,11 +174,11 @@ exports.clearCookie = (ctx, name) => {
* @param {object} ctx The koa context object to be tested.
* @return {boolean} returns true if the call is from the client lib (a built app rather than the builder).
*/
exports.isClient = ctx => {
return ctx.headers[Headers.TYPE] === "client"
export function isClient(ctx: BBContext) {
return ctx.headers[Header.TYPE] === "client"
}
const getBuilders = async () => {
async function getBuilders() {
const builders = await queryGlobalView(ViewName.USER_BY_BUILDERS, {
include_docs: false,
})
@ -176,7 +194,7 @@ const getBuilders = async () => {
}
}
exports.getBuildersCount = async () => {
export async function getBuildersCount() {
const builders = await getBuilders()
return builders.length
}
@ -184,10 +202,14 @@ exports.getBuildersCount = async () => {
/**
* Logs a user out from budibase. Re-used across account portal and builder.
*/
exports.platformLogout = async ({ ctx, userId, keepActiveSession }) => {
export async function platformLogout(opts: PlatformLogoutOpts) {
const ctx = opts.ctx
const userId = opts.userId
const keepActiveSession = opts.keepActiveSession
if (!ctx) throw new Error("Koa context must be supplied to logout.")
const currentSession = exports.getCookie(ctx, Cookies.Auth)
const currentSession = getCookie(ctx, Cookie.Auth)
let sessions = await getSessionsForUser(userId)
if (keepActiveSession) {
@ -196,8 +218,8 @@ exports.platformLogout = async ({ ctx, userId, keepActiveSession }) => {
)
} else {
// clear cookies
exports.clearCookie(ctx, Cookies.Auth)
exports.clearCookie(ctx, Cookies.CurrentApp)
clearCookie(ctx, Cookie.Auth)
clearCookie(ctx, Cookie.CurrentApp)
}
const sessionIds = sessions.map(({ sessionId }) => sessionId)
@ -206,6 +228,6 @@ exports.platformLogout = async ({ ctx, userId, keepActiveSession }) => {
await userCache.invalidateUser(userId)
}
exports.timeout = timeMs => {
export function timeout(timeMs: number) {
return new Promise(resolve => setTimeout(resolve, timeMs))
}

22
packages/worker/scripts/jestSetup.js → packages/backend-core/tests/jestSetup.ts
View File

@ -1,17 +1,21 @@
const env = require("../src/environment")
import env from "../src/environment"
import { mocks } from "./utilities"
// must explicitly enable fetch mock
mocks.fetch.enable()
// mock all dates to 2020-01-01T00:00:00.000Z
// use tk.reset() to use real dates in individual tests
import tk from "timekeeper"
tk.freeze(mocks.date.MOCK_DATE)
env._set("SELF_HOSTED", "1")
env._set("NODE_ENV", "jest")
env._set("JWT_SECRET", "test-jwtsecret")
env._set("LOG_LEVEL", "silent")
env._set("MULTI_TENANCY", true)
const { mocks } = require("@budibase/backend-core/tests")
// mock all dates to 2020-01-01T00:00:00.000Z
// use tk.reset() to use real dates in individual tests
const tk = require("timekeeper")
tk.freeze(mocks.date.MOCK_DATE)
env._set("MINIO_URL", "http://localhost")
env._set("MINIO_ACCESS_KEY", "test")
env._set("MINIO_SECRET_KEY", "test")
global.console.log = jest.fn() // console.log are ignored in tests

1
packages/backend-core/tests/utilities/TestConfiguration.js
View File

@ -1 +0,0 @@
require("./db")

6
packages/backend-core/tests/utilities/db.js
View File

@ -1,6 +0,0 @@
const core = require("../../src/index")
const dbConfig = {
inMemory: true,
allDbs: true,
}
core.init({ db: dbConfig })

9
packages/backend-core/tests/utilities/db.ts Normal file
View File

@ -0,0 +1,9 @@
import * as db from "../../src/db"
const dbConfig = {
inMemory: true,
}
export const init = () => {
db.init(dbConfig)
}

4
packages/backend-core/tests/utilities/index.ts
View File

@ -1,2 +1,6 @@
export * as mocks from "./mocks"
export * as structures from "./structures"
export { generator } from "./structures"
import * as dbConfig from "./db"
dbConfig.init()

2
packages/backend-core/tests/utilities/mocks/accounts.ts
View File

@ -1,7 +1,9 @@
export const getAccount = jest.fn()
export const getAccountByTenantId = jest.fn()
export const getStatus = jest.fn()
jest.mock("../../../src/cloud/accounts", () => ({
getAccount,
getAccountByTenantId,
getStatus,
}))

10
packages/backend-core/tests/utilities/mocks/fetch.ts Normal file
View File

@ -0,0 +1,10 @@
const mockFetch = jest.fn()
const enable = () => {
jest.mock("node-fetch", () => mockFetch)
}
export default {
...mockFetch,
enable,
}

1
packages/backend-core/tests/utilities/mocks/index.ts
View File

@ -2,3 +2,4 @@ import "./posthog"
import "./events"
export * as accounts from "./accounts"
export * as date from "./date"
export { default as fetch } from "./fetch"

22
packages/backend-core/tests/utilities/structures/accounts.ts
View File

@ -1,23 +1,29 @@
import { generator, uuid } from "."
import { AuthType, CloudAccount, Hosting } from "@budibase/types"
import * as db from "../../../src/db/utils"
import { Account, AuthType, CloudAccount, Hosting } from "@budibase/types"
export const cloudAccount = (): CloudAccount => {
export const account = (): Account => {
return {
accountId: uuid(),
tenantId: generator.word(),
email: generator.email(),
tenantName: generator.word(),
hosting: Hosting.SELF,
createdAt: Date.now(),
verified: true,
verificationSent: true,
tier: "",
email: generator.email(),
tenantId: generator.word(),
hosting: Hosting.CLOUD,
tier: "FREE", // DEPRECATED
authType: AuthType.PASSWORD,
password: generator.word(),
tenantName: generator.word(),
name: generator.name(),
size: "10+",
profession: "Software Engineer",
}
}
export const cloudAccount = (): CloudAccount => {
return {
...account(),
hosting: Hosting.CLOUD,
budibaseUserId: db.generateGlobalUserID(),
}
}

6
packages/backend-core/tests/utilities/structures/common.ts
View File

@ -1 +1,7 @@
import { v4 as uuid } from "uuid"
export { v4 as uuid } from "uuid"
export const email = () => {
return `${uuid()}@test.com`
}

15
packages/backend-core/tests/utilities/structures/koa.ts
View File

@ -1,5 +1,14 @@
import { createMockContext } from "@shopify/jest-koa-mocks"
import { createMockContext, createMockCookies } from "@shopify/jest-koa-mocks"
import { BBContext } from "@budibase/types"
export const newContext = () => {
return createMockContext()
export const newContext = (): BBContext => {
const ctx = createMockContext()
return {
...ctx,
cookies: createMockCookies(),
request: {
...ctx.request,
body: {},
},
}
}

2
packages/backend-core/tsconfig.build.json
View File

@ -12,7 +12,7 @@
"sourceMap": true,
"declaration": true,
"types": [ "node", "jest" ],
"outDir": "dist"
"outDir": "dist",
},
"include": [
"**/*.js",

1806
packages/backend-core/yarn.lock
View File

File diff suppressed because it is too large Load Diff

4
packages/bbui/package.json
View File

@ -1,7 +1,7 @@
{
"name": "@budibase/bbui",
"description": "A UI solution used in the different Budibase projects.",
"version": "2.1.22-alpha.2",
"version": "2.1.22-alpha.8",
"license": "MPL-2.0",
"svelte": "src/index.js",
"module": "dist/bbui.es.js",
@ -38,7 +38,7 @@
],
"dependencies": {
"@adobe/spectrum-css-workflow-icons": "^1.2.1",
"@budibase/string-templates": "2.1.22-alpha.2",
"@budibase/string-templates": "2.1.22-alpha.8",
"@spectrum-css/actionbutton": "^1.0.1",
"@spectrum-css/actiongroup": "^1.0.1",
"@spectrum-css/avatar": "^3.0.2",

935
packages/bbui/yarn.lock
View File

File diff suppressed because it is too large Load Diff

6
packages/builder/cypress/integration/createApp.spec.js
View File

@ -10,7 +10,7 @@ filterTests(['smoke', 'all'], () => {
})
if (!(Cypress.env("TEST_ENV"))) {
it("should show the new user UI/UX", () => {
it.skip("should show the new user UI/UX", () => {
cy.visit(`${Cypress.config().baseUrl}/builder/portal/apps/create`, { timeout: 5000 }) //added /portal/apps/create
cy.wait(1000)
cy.get(interact.CREATE_APP_BUTTON, { timeout: 10000 }).contains('Start from scratch').should("exist")
@ -83,7 +83,7 @@ filterTests(['smoke', 'all'], () => {
})
})
it("should create the first application from scratch", () => {
it.skip("should create the first application from scratch", () => {
const appName = "Cypress Tests"
cy.createApp(appName, false)
@ -93,7 +93,7 @@ filterTests(['smoke', 'all'], () => {
cy.deleteApp(appName)
})
it("should create the first application from scratch with a default name", () => {
it.skip("should create the first application from scratch with a default name", () => {
cy.updateUserInformation("", "")
cy.createApp("", false)
cy.applicationInAppTable("My app")

4
packages/builder/cypress/integration/createAutomation.spec.js
View File

@ -12,7 +12,7 @@ filterTests(['smoke', 'all'], () => {
cy.createTestTableWithData()
cy.wait(2000)
cy.contains("Automate").click()
cy.get(interact.ADD_BUTTON_SPECTRUM).click()
cy.get(interact.SPECTRUM_BUTTON_TEMPLATE).contains("Add automation").click({ force: true })
cy.get(interact.MODAL_INNER_WRAPPER).within(() => {
cy.get("input").type("Add Row")
cy.contains("Row Created").click({ force: true })
@ -24,7 +24,7 @@ filterTests(['smoke', 'all'], () => {
cy.wait(500)
cy.contains("dog").click()
// Create action
cy.get('[aria-label="AddCircle"]', { timeout: 2000 }).eq(1).click()
cy.get('[aria-label="AddCircle"]', { timeout: 2000 }).click()
cy.get(interact.MODAL_INNER_WRAPPER).within(() => {
cy.wait(1000)
cy.contains("Create Row").trigger('mouseover').click().click()

4
packages/builder/cypress/integration/createScreen.spec.js
View File

@ -9,7 +9,7 @@ filterTests(["smoke", "all"], () => {
cy.navigateToFrontend()
})
it("Should successfully create a screen", () => {
it.skip("Should successfully create a screen", () => {
cy.createScreen("test")
cy.get(interact.BODY).within(() => {
cy.contains("/test").should("exist")
@ -23,7 +23,7 @@ filterTests(["smoke", "all"], () => {
})
})
it("should delete all screens then create first screen via button", () => {
it.skip("should delete all screens then create first screen via button", () => {
cy.deleteAllScreens()
cy.contains("Create first screen").click()

2
packages/builder/cypress/integration/createTable.spec.js
View File

@ -2,7 +2,7 @@ import filterTests from "../support/filterTests"
const interact = require('../support/interact')
filterTests(["smoke", "all"], () => {
context("Create a Table", () => {
xcontext("Create a Table", () => {
before(() => {
cy.login()
cy.createTestApp()

2
packages/builder/cypress/integration/revertApp.spec.js
View File

@ -2,7 +2,7 @@ import filterTests from "../support/filterTests"
const interact = require('../support/interact')
filterTests(['smoke', 'all'], () => {
context("Revert apps", () => {
xcontext("Revert apps", () => {
before(() => {
cy.login()
cy.createTestApp()

2
packages/builder/cypress/support/commands.js
View File

@ -440,7 +440,7 @@ Cypress.Commands.add("createTable", (tableName, initialTable) => {
// Creates an internal Budibase DB table
if (!initialTable) {
cy.navigateToDataSection()
cy.get(`[data-cy="new-table"]`, { timeout: 2000 }).click()
cy.get(`[data-cy="new-datasource"]`, { timeout: 2000 }).click()
}
cy.wait(2000)
cy.get(".item", { timeout: 2000 })

14
packages/builder/package.json
View File

@ -1,6 +1,6 @@
{
"name": "@budibase/builder",
"version": "2.1.22-alpha.2",
"version": "2.1.22-alpha.8",
"license": "GPL-3.0",
"private": true,
"scripts": {
@ -71,10 +71,10 @@
}
},
"dependencies": {
"@budibase/bbui": "2.1.22-alpha.2",
"@budibase/client": "2.1.22-alpha.2",
"@budibase/frontend-core": "2.1.22-alpha.2",
"@budibase/string-templates": "2.1.22-alpha.2",
"@budibase/bbui": "2.1.22-alpha.8",
"@budibase/client": "2.1.22-alpha.8",
"@budibase/frontend-core": "2.1.22-alpha.8",
"@budibase/string-templates": "2.1.22-alpha.8",
"@sentry/browser": "5.19.1",
"@spectrum-css/page": "^3.0.1",
"@spectrum-css/vars": "^3.0.1",
@ -117,9 +117,9 @@
"start-server-and-test": "^1.12.1",
"svelte": "^3.48.0",
"svelte-jester": "^1.3.2",
"ts-node": "^10.4.0",
"ts-node": "10.8.1",
"tsconfig-paths": "4.0.0",
"typescript": "^4.5.5",
"typescript": "4.7.3",
"vite": "^3.0.8"
},
"gitHead": "115189f72a850bfb52b65ec61d932531bf327072"

76
packages/builder/src/pages/builder/portal/manage/auth/index.svelte
View File

@ -20,11 +20,12 @@
Toggle,
Tag,
Tags,
Icon,
Helpers,
} from "@budibase/bbui"
import { onMount } from "svelte"
import { API } from "api"
import { organisation, admin } from "stores/portal"
import { Helpers } from "@budibase/bbui"
const ConfigTypes = {
Google: "google",
@ -40,7 +41,9 @@
// Indicate to user that callback is based on platform url
// If there is an existing value, indicate that it may be removed to return to default behaviour
$: googleCallbackTooltip = googleCallbackReadonly
$: googleCallbackTooltip = $admin.cloud
? null
: googleCallbackReadonly
? "Vist the organisation page to update the platform URL"
: "Leave blank to use the default callback URL"
@ -54,6 +57,7 @@
readonly: googleCallbackReadonly,
tooltip: googleCallbackTooltip,
placeholder: $organisation.googleCallbackUrl,
copyButton: true,
},
],
}
@ -66,9 +70,12 @@
{
name: "callbackURL",
readonly: true,
tooltip: "Vist the organisation page to update the platform URL",
tooltip: $admin.cloud
? null
: "Vist the organisation page to update the platform URL",
label: "Callback URL",
placeholder: $organisation.oidcCallbackUrl,
copyButton: true,
},
],
}
@ -231,6 +238,11 @@
},
]
const copyToClipboard = async value => {
await Helpers.copyToClipboard(value)
notifications.success("Copied")
}
onMount(async () => {
try {
await organisation.init()
@ -336,11 +348,23 @@
{#each GoogleConfigFields.Google as field}
<div class="form-row">
<Label size="L" tooltip={field.tooltip}>{field.label}</Label>
<Input
bind:value={providers.google.config[field.name]}
readonly={field.readonly}
placeholder={field.placeholder}
/>
<div class="inputContainer">
<div class="input">
<Input
bind:value={providers.google.config[field.name]}
readonly={field.readonly}
placeholder={field.placeholder}
/>
</div>
{#if field.copyButton}
<div
class="copy"
on:click={() => copyToClipboard(field.placeholder)}
>
<Icon size="S" name="Copy" />
</div>
{/if}
</div>
</div>
{/each}
<div class="form-row">
@ -375,12 +399,24 @@
{#each OIDCConfigFields.Oidc as field}
<div class="form-row">
<Label size="L" tooltip={field.tooltip}>{field.label}</Label>
<Input
bind:value={providers.oidc.config.configs[0][field.name]}
readonly={field.readonly}
placeholder={field.placeholder}
dataCy={field.name}
/>
<div class="inputContainer">
<div class="input">
<Input
bind:value={providers.oidc.config.configs[0][field.name]}
readonly={field.readonly}
placeholder={field.placeholder}
dataCy={field.name}
/>
</div>
{#if field.copyButton}
<div
class="copy"
on:click={() => copyToClipboard(field.placeholder)}
>
<Icon size="S" name="Copy" />
</div>
{/if}
</div>
</div>
{/each}
</Layout>
@ -557,4 +593,16 @@
.provider-title span {
flex: 1 1 auto;
}
.inputContainer {
display: flex;
flex-direction: row;
}
.input {
flex: 1;
}
.copy {
display: flex;
align-items: center;
margin-left: 10px;
}
</style>

57
packages/builder/yarn.lock
View File

@ -923,17 +923,12 @@
exec-sh "^0.3.2"
minimist "^1.2.0"
"@cspotcode/source-map-consumer@0.8.0":
version "0.8.0"
resolved "https://registry.yarnpkg.com/@cspotcode/source-map-consumer/-/source-map-consumer-0.8.0.tgz#33bf4b7b39c178821606f669bbc447a6a629786b"
integrity sha512-41qniHzTU8yAGbCp04ohlmSrZf8bkf/iJsl3V0dRGsQN/5GFfx+LbCSsCpp2gqrqjTVg/K6O8ycoV35JIwAzAg==
"@cspotcode/source-map-support@0.7.0":
version "0.7.0"
resolved "https://registry.yarnpkg.com/@cspotcode/source-map-support/-/source-map-support-0.7.0.tgz#4789840aa859e46d2f3173727ab707c66bf344f5"
integrity sha512-X4xqRHqN8ACt2aHVe51OxeA2HjbcL4MqFqXkrmQszJ1NOUuUu5u6Vqx/0lZSVNku7velL5FC/s5uEAj1lsBMhA==
"@cspotcode/source-map-support@^0.8.0":
version "0.8.1"
resolved "https://registry.yarnpkg.com/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz#00629c35a688e05a88b1cda684fb9d5e73f000a1"
integrity sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==
dependencies:
"@cspotcode/source-map-consumer" "0.8.0"
"@jridgewell/trace-mapping" "0.3.9"
"@cypress/request@^2.88.10":
version "2.88.10"
@ -1182,6 +1177,24 @@
"@types/yargs" "^16.0.0"
chalk "^4.0.0"
"@jridgewell/resolve-uri@^3.0.3":
version "3.1.0"
resolved "https://registry.yarnpkg.com/@jridgewell/resolve-uri/-/resolve-uri-3.1.0.tgz#2203b118c157721addfe69d47b70465463066d78"
integrity sha512-F2msla3tad+Mfht5cJq7LSXcdudKTWCVYUgw6pLFOOHSTtZlj6SWNYAp+AhuqLmWdBO2X5hPrLcu8cVP8fy28w==
"@jridgewell/sourcemap-codec@^1.4.10":
version "1.4.14"
resolved "https://registry.yarnpkg.com/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.14.tgz#add4c98d341472a289190b424efbdb096991bb24"
integrity sha512-XPSJHWmi394fuUuzDnGz1wiKqWfo1yXecHQMRf2l6hztTO+nPru658AyDngaBe7isIxEkRsPR3FZh+s7iVa4Uw==
"@jridgewell/trace-mapping@0.3.9":
version "0.3.9"
resolved "https://registry.yarnpkg.com/@jridgewell/trace-mapping/-/trace-mapping-0.3.9.tgz#6534fd5933a53ba7cbf3a17615e273a0d1273ff9"
integrity sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ==
dependencies:
"@jridgewell/resolve-uri" "^3.0.3"
"@jridgewell/sourcemap-codec" "^1.4.10"
"@nodelib/fs.scandir@2.1.5":
version "2.1.5"
resolved "https://registry.yarnpkg.com/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz#7619c2eb21b25483f6d167548b4cfd5a7488c3d5"
@ -5935,12 +5948,12 @@ tr46@~0.0.3:
resolved "https://registry.yarnpkg.com/tr46/-/tr46-0.0.3.tgz#8184fd347dac9cdc185992f3a6622e14b9d9ab6a"
integrity sha1-gYT9NH2snNwYWZLzpmIuFLnZq2o=
ts-node@^10.4.0:
version "10.4.0"
resolved "https://registry.yarnpkg.com/ts-node/-/ts-node-10.4.0.tgz#680f88945885f4e6cf450e7f0d6223dd404895f7"
integrity sha512-g0FlPvvCXSIO1JDF6S232P5jPYqBkRL9qly81ZgAOSU7rwI0stphCgd2kLiCrU9DjQCrJMWEqcNSjQL02s6d8A==
ts-node@10.8.1:
version "10.8.1"
resolved "https://registry.yarnpkg.com/ts-node/-/ts-node-10.8.1.tgz#ea2bd3459011b52699d7e88daa55a45a1af4f066"
integrity sha512-Wwsnao4DQoJsN034wePSg5nZiw4YKXf56mPIAeD6wVmiv+RytNSWqc2f3fKvcUoV+Yn2+yocD71VOfQHbmVX4g==
dependencies:
"@cspotcode/source-map-support" "0.7.0"
"@cspotcode/source-map-support" "^0.8.0"
"@tsconfig/node10" "^1.0.7"
"@tsconfig/node12" "^1.0.7"
"@tsconfig/node14" "^1.0.0"
@ -5951,6 +5964,7 @@ ts-node@^10.4.0:
create-require "^1.1.0"
diff "^4.0.1"
make-error "^1.1.1"
v8-compile-cache-lib "^3.0.1"
yn "3.1.1"
tsconfig-paths@4.0.0:
@ -6023,10 +6037,10 @@ typedarray-to-buffer@^3.1.5:
dependencies:
is-typedarray "^1.0.0"
typescript@^4.5.5:
version "4.5.5"
resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.5.5.tgz#d8c953832d28924a9e3d37c73d729c846c5896f3"
integrity sha512-TCTIul70LyWe6IJWT8QSYeA54WQe8EjQFU4wY52Fasj5UKx88LNYKCgBEHcOMOrFF1rKGbD8v/xcNWVUq9SymA==
typescript@4.7.3:
version "4.7.3"
resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.7.3.tgz#8364b502d5257b540f9de4c40be84c98e23a129d"
integrity sha512-WOkT3XYvrpXx4vMMqlD+8R8R37fZkjyLGlxavMc4iB8lrl8L0DeTcHbYgw/v0N/z9wAFsgBhcsF0ruoySS22mA==
unicode-canonical-property-names-ecmascript@^2.0.0:
version "2.0.0"
@ -6111,6 +6125,11 @@ uuid@^8.3.0, uuid@^8.3.2:
resolved "https://registry.yarnpkg.com/uuid/-/uuid-8.3.2.tgz#80d5b5ced271bb9af6c445f21a1a04c606cefbe2"
integrity sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==
v8-compile-cache-lib@^3.0.1:
version "3.0.1"
resolved "https://registry.yarnpkg.com/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz#6336e8d71965cb3d35a1bbb7868445a7c05264bf"
integrity sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg==
v8-to-istanbul@^7.0.0:
version "7.1.2"
resolved "https://registry.yarnpkg.com/v8-to-istanbul/-/v8-to-istanbul-7.1.2.tgz#30898d1a7fa0c84d225a2c1434fb958f290883c1"

8
packages/cli/package.json
View File

@ -1,6 +1,6 @@
{
"name": "@budibase/cli",
"version": "2.1.22-alpha.2",
"version": "2.1.22-alpha.8",
"description": "Budibase CLI, for developers, self hosting and migrations.",
"main": "src/index.js",
"bin": {
@ -26,9 +26,9 @@
"outputPath": "build"
},
"dependencies": {
"@budibase/backend-core": "2.1.22-alpha.2",
"@budibase/string-templates": "2.1.22-alpha.2",
"@budibase/types": "2.1.22-alpha.2",
"@budibase/backend-core": "2.1.22-alpha.8",
"@budibase/string-templates": "2.1.22-alpha.8",
"@budibase/types": "2.1.22-alpha.8",
"axios": "0.21.2",
"chalk": "4.1.0",
"cli-progress": "3.11.2",

8
packages/client/package.json
View File

@ -1,6 +1,6 @@
{
"name": "@budibase/client",
"version": "2.1.22-alpha.2",
"version": "2.1.22-alpha.8",
"license": "MPL-2.0",
"module": "dist/budibase-client.js",
"main": "dist/budibase-client.js",
@ -19,9 +19,9 @@
"dev:builder": "rollup -cw"
},
"dependencies": {
"@budibase/bbui": "2.1.22-alpha.2",
"@budibase/frontend-core": "2.1.22-alpha.2",
"@budibase/string-templates": "2.1.22-alpha.2",
"@budibase/bbui": "2.1.22-alpha.8",
"@budibase/frontend-core": "2.1.22-alpha.8",
"@budibase/string-templates": "2.1.22-alpha.8",
"@spectrum-css/button": "^3.0.3",
"@spectrum-css/card": "^3.0.3",
"@spectrum-css/divider": "^1.0.3",

1299
packages/client/yarn.lock
View File

File diff suppressed because it is too large Load Diff

4
packages/frontend-core/package.json
View File

@ -1,12 +1,12 @@
{
"name": "@budibase/frontend-core",
"version": "2.1.22-alpha.2",
"version": "2.1.22-alpha.8",
"description": "Budibase frontend core libraries used in builder and client",
"author": "Budibase",
"license": "MPL-2.0",
"svelte": "src/index.js",
"dependencies": {
"@budibase/bbui": "2.1.22-alpha.2",
"@budibase/bbui": "2.1.22-alpha.8",
"lodash": "^4.17.21",
"svelte": "^3.46.2"
}

2
packages/sdk/package.json
View File

@ -1,6 +1,6 @@
{
"name": "@budibase/sdk",
"version": "2.1.22-alpha.2",
"version": "2.1.22-alpha.8",
"description": "Budibase Public API SDK",
"author": "Budibase",
"license": "MPL-2.0",

Some files were not shown because too many files have changed in this diff Show More