MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #9670 from Budibase/budi-6559-enable-higher-concurrency-and-resiliency 

Enable higher concurrency and resiliency in worker tests
This commit is contained in:
Rory Powell 2023-02-13 14:31:14 +00:00 committed by GitHub
parent 0f5b4abb12 ae9979929a
commit eb5aa8786d
58 changed files with 1165 additions and 1004 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
packages/backend-core/package.json
View File

@ -18,7 +18,7 @@
"build:pro": "../../scripts/pro/build.sh",
"postbuild": "yarn run build:pro",
"build:dev": "yarn prebuild && tsc --build --watch --preserveWatchOutput",
"test": "jest --coverage --maxWorkers=2",
"test": "jest --coverage",
"test:watch": "jest --watchAll"
},
"dependencies": {
@ -62,7 +62,7 @@
"@trendyol/jest-testcontainers": "^2.1.1",
"@types/chance": "1.1.3",
"@types/ioredis": "4.28.0",
"@types/jest": "27.5.1",
"@types/jest": "28.1.1",
"@types/koa": "2.13.4",
"@types/koa-pino-logger": "3.0.0",
"@types/lodash": "4.14.180",

61
packages/backend-core/src/cache/tests/writethrough.spec.js vendored
View File

@ -1,61 +0,0 @@
require("../../../tests")
const { Writethrough } = require("../writethrough")
const { getDB } = require("../../db")
const tk = require("timekeeper")
const { structures } = require("../../../tests")
const START_DATE = Date.now()
tk.freeze(START_DATE)
const DELAY = 5000
const db = getDB(structures.db.id())
const db2 = getDB(structures.db.id())
const writethrough = new Writethrough(db, DELAY), writethrough2 = new Writethrough(db2, DELAY)
describe("writethrough", () => {
describe("put", () => {
let first
it("should be able to store, will go to DB", async () => {
const response = await writethrough.put({ _id: "test", value: 1 })
const output = await db.get(response.id)
first = output
expect(output.value).toBe(1)
})
it("second put shouldn't update DB", async () => {
const response = await writethrough.put({ ...first, value: 2 })
const output = await db.get(response.id)
expect(first._rev).toBe(output._rev)
expect(output.value).toBe(1)
})
it("should put it again after delay period", async () => {
tk.freeze(START_DATE + DELAY + 1)
const response = await writethrough.put({ ...first, value: 3 })
const output = await db.get(response.id)
expect(response.rev).not.toBe(first._rev)
expect(output.value).toBe(3)
})
})
describe("get", () => {
it("should be able to retrieve", async () => {
const response = await writethrough.get("test")
expect(response.value).toBe(3)
})
})
describe("same doc, different databases (tenancy)", () => {
it("should be able to two different databases", async () => {
const resp1 = await writethrough.put({ _id: "db1", value: "first" })
const resp2 = await writethrough2.put({ _id: "db1", value: "second" })
expect(resp1.rev).toBeDefined()
expect(resp2.rev).toBeDefined()
expect((await db.get("db1")).value).toBe("first")
expect((await db2.get("db1")).value).toBe("second")
})
})
})

73
packages/backend-core/src/cache/tests/writethrough.spec.ts vendored Normal file
View File

@ -0,0 +1,73 @@
import { structures, DBTestConfiguration } from "../../../tests"
import { Writethrough } from "../writethrough"
import { getDB } from "../../db"
import tk from "timekeeper"
const START_DATE = Date.now()
tk.freeze(START_DATE)
const DELAY = 5000
describe("writethrough", () => {
const config = new DBTestConfiguration()
const db = getDB(structures.db.id())
const db2 = getDB(structures.db.id())
const writethrough = new Writethrough(db, DELAY)
const writethrough2 = new Writethrough(db2, DELAY)
describe("put", () => {
let first: any
it("should be able to store, will go to DB", async () => {
await config.doInTenant(async () => {
const response = await writethrough.put({ _id: "test", value: 1 })
const output = await db.get(response.id)
first = output
expect(output.value).toBe(1)
})
})
it("second put shouldn't update DB", async () => {
await config.doInTenant(async () => {
const response = await writethrough.put({ ...first, value: 2 })
const output = await db.get(response.id)
expect(first._rev).toBe(output._rev)
expect(output.value).toBe(1)
})
})
it("should put it again after delay period", async () => {
await config.doInTenant(async () => {
tk.freeze(START_DATE + DELAY + 1)
const response = await writethrough.put({ ...first, value: 3 })
const output = await db.get(response.id)
expect(response.rev).not.toBe(first._rev)
expect(output.value).toBe(3)
})
})
})
describe("get", () => {
it("should be able to retrieve", async () => {
await config.doInTenant(async () => {
const response = await writethrough.get("test")
expect(response.value).toBe(3)
})
})
})
describe("same doc, different databases (tenancy)", () => {
it("should be able to two different databases", async () => {
await config.doInTenant(async () => {
const resp1 = await writethrough.put({ _id: "db1", value: "first" })
const resp2 = await writethrough2.put({ _id: "db1", value: "second" })
expect(resp1.rev).toBeDefined()
expect(resp2.rev).toBeDefined()
expect((await db.get("db1")).value).toBe("first")
expect((await db2.get("db1")).value).toBe("second")
})
})
})
})

14
packages/backend-core/src/cache/user.ts vendored
View File

@ -1,8 +1,9 @@
import * as redis from "../redis/init"
import { getTenantId, lookupTenantId, doWithGlobalDB } from "../tenancy"
import * as tenancy from "../tenancy"
import * as context from "../context"
import * as platform from "../platform"
import env from "../environment"
import * as accounts from "../cloud/accounts"
import { Database } from "@budibase/types"
import * as accounts from "../accounts"
const EXPIRY_SECONDS = 3600
@ -10,7 +11,8 @@ const EXPIRY_SECONDS = 3600
* The default populate user function
*/
async function populateFromDB(userId: string, tenantId: string) {
const user = await doWithGlobalDB(tenantId, (db: Database) => db.get(userId))
const db = tenancy.getTenantDB(tenantId)
const user = await db.get(userId)
user.budibaseAccess = true
if (!env.SELF_HOSTED && !env.DISABLE_ACCOUNT_PORTAL) {
const account = await accounts.getAccount(user.email)
@ -42,9 +44,9 @@ export async function getUser(
}
if (!tenantId) {
try {
tenantId = getTenantId()
tenantId = context.getTenantId()
} catch (err) {
tenantId = await lookupTenantId(userId)
tenantId = await platform.users.lookupTenantId(userId)
}
}
const client = await redis.getUserClient()

108
packages/backend-core/src/context/deprovision.ts
View File

@ -1,108 +0,0 @@
import {
getGlobalUserParams,
getAllApps,
doWithDB,
StaticDatabases,
} from "../db"
import { doWithGlobalDB } from "../tenancy"
import { App, Tenants, User, Database } from "@budibase/types"
const TENANT_DOC = StaticDatabases.PLATFORM_INFO.docs.tenants
const PLATFORM_INFO_DB = StaticDatabases.PLATFORM_INFO.name
async function removeTenantFromInfoDB(tenantId: string) {
try {
await doWithDB(PLATFORM_INFO_DB, async (infoDb: Database) => {
const tenants = (await infoDb.get(TENANT_DOC)) as Tenants
tenants.tenantIds = tenants.tenantIds.filter(id => id !== tenantId)
await infoDb.put(tenants)
})
} catch (err) {
console.error(`Error removing tenant ${tenantId} from info db`, err)
throw err
}
}
export async function removeUserFromInfoDB(dbUser: User) {
await doWithDB(PLATFORM_INFO_DB, async (infoDb: Database) => {
const keys = [dbUser._id!, dbUser.email]
const userDocs = await infoDb.allDocs({
keys,
include_docs: true,
})
const toDelete = userDocs.rows.map((row: any) => {
return {
...row.doc,
_deleted: true,
}
})
await infoDb.bulkDocs(toDelete)
})
}
async function removeUsersFromInfoDB(tenantId: string) {
return doWithGlobalDB(tenantId, async (db: any) => {
try {
const allUsers = await db.allDocs(
getGlobalUserParams(null, {
include_docs: true,
})
)
await doWithDB(PLATFORM_INFO_DB, async (infoDb: any) => {
const allEmails = allUsers.rows.map((row: any) => row.doc.email)
// get the id docs
let keys = allUsers.rows.map((row: any) => row.id)
// and the email docs
keys = keys.concat(allEmails)
// retrieve the docs and delete them
const userDocs = await infoDb.allDocs({
keys,
include_docs: true,
})
const toDelete = userDocs.rows.map((row: any) => {
return {
...row.doc,
_deleted: true,
}
})
await infoDb.bulkDocs(toDelete)
})
} catch (err) {
console.error(`Error removing tenant ${tenantId} users from info db`, err)
throw err
}
})
}
async function removeGlobalDB(tenantId: string) {
return doWithGlobalDB(tenantId, async (db: Database) => {
try {
await db.destroy()
} catch (err) {
console.error(`Error removing tenant ${tenantId} users from info db`, err)
throw err
}
})
}
async function removeTenantApps(tenantId: string) {
try {
const apps = (await getAllApps({ all: true })) as App[]
const destroyPromises = apps.map(app =>
doWithDB(app.appId, (db: Database) => db.destroy())
)
await Promise.allSettled(destroyPromises)
} catch (err) {
console.error(`Error removing tenant ${tenantId} apps`, err)
throw err
}
}
// can't live in tenancy package due to circular dependency on db/utils
export async function deleteTenant(tenantId: string) {
await removeTenantFromInfoDB(tenantId)
await removeUsersFromInfoDB(tenantId)
await removeGlobalDB(tenantId)
await removeTenantApps(tenantId)
}

11
packages/backend-core/src/context/tests/index.spec.ts
View File

@ -1,11 +1,14 @@
require("../../../tests")
import { testEnv } from "../../../tests"
const context = require("../")
const { DEFAULT_TENANT_ID } = require("../../constants")
import env from "../../environment"
describe("context", () => {
describe("doInTenant", () => {
describe("single-tenancy", () => {
beforeAll(() => {
testEnv.singleTenant()
})
it("defaults to the default tenant", () => {
const tenantId = context.getTenantId()
expect(tenantId).toBe(DEFAULT_TENANT_ID)
@ -20,8 +23,8 @@ describe("context", () => {
})
describe("multi-tenancy", () => {
beforeEach(() => {
env._set("MULTI_TENANCY", 1)
beforeAll(() => {
testEnv.multiTenant()
})
it("fails when no tenant id is set", () => {

3
packages/backend-core/src/db/db.ts
View File

@ -1,7 +1,6 @@
import env from "../environment"
import { directCouchQuery, getPouchDB } from "./couch"
import { directCouchQuery, DatabaseImpl } from "./couch"
import { CouchFindOptions, Database } from "@budibase/types"
import { DatabaseImpl } from "../db"
const dbList = new Set()

190
packages/backend-core/src/db/tests/utils.seq.spec.ts
View File

@ -1,190 +0,0 @@
require("../../../tests")
const {
getDevelopmentAppID,
getProdAppID,
isDevAppID,
isProdAppID,
} = require("../conversions")
const { generateAppID, getPlatformUrl, getScopedConfig } = require("../utils")
const tenancy = require("../../tenancy")
const { Config, DEFAULT_TENANT_ID } = require("../../constants")
import { generator } from "../../../tests"
import env from "../../environment"
describe("utils", () => {
describe("app ID manipulation", () => {
function getID() {
const appId = generateAppID()
const split = appId.split("_")
const uuid = split[split.length - 1]
const devAppId = `app_dev_${uuid}`
return { appId, devAppId, split, uuid }
}
it("should be able to generate a new app ID", () => {
expect(generateAppID().startsWith("app_")).toEqual(true)
})
it("should be able to convert a production app ID to development", () => {
const { appId, uuid } = getID()
expect(getDevelopmentAppID(appId)).toEqual(`app_dev_${uuid}`)
})
it("should be able to convert a development app ID to development", () => {
const { devAppId, uuid } = getID()
expect(getDevelopmentAppID(devAppId)).toEqual(`app_dev_${uuid}`)
})
it("should be able to convert a development ID to a production", () => {
const { devAppId, uuid } = getID()
expect(getProdAppID(devAppId)).toEqual(`app_${uuid}`)
})
it("should be able to convert a production ID to production", () => {
const { appId, uuid } = getID()
expect(getProdAppID(appId)).toEqual(`app_${uuid}`)
})
it("should be able to confirm dev app ID is development", () => {
const { devAppId } = getID()
expect(isDevAppID(devAppId)).toEqual(true)
})
it("should be able to confirm prod app ID is not development", () => {
const { appId } = getID()
expect(isDevAppID(appId)).toEqual(false)
})
it("should be able to confirm prod app ID is prod", () => {
const { appId } = getID()
expect(isProdAppID(appId)).toEqual(true)
})
it("should be able to confirm dev app ID is not prod", () => {
const { devAppId } = getID()
expect(isProdAppID(devAppId)).toEqual(false)
})
})
})
const DEFAULT_URL = "http://localhost:10000"
const ENV_URL = "http://env.com"
const setDbPlatformUrl = async (dbUrl: string) => {
const db = tenancy.getGlobalDB()
await db.put({
_id: "config_settings",
type: Config.SETTINGS,
config: {
platformUrl: dbUrl,
},
})
}
const clearSettingsConfig = async () => {
await tenancy.doInTenant(DEFAULT_TENANT_ID, async () => {
const db = tenancy.getGlobalDB()
try {
const config = await db.get("config_settings")
await db.remove("config_settings", config._rev)
} catch (e: any) {
if (e.status !== 404) {
throw e
}
}
})
}
describe("getPlatformUrl", () => {
describe("self host", () => {
beforeEach(async () => {
env._set("SELF_HOST", 1)
await clearSettingsConfig()
})
it("gets the default url", async () => {
await tenancy.doInTenant(null, async () => {
const url = await getPlatformUrl()
expect(url).toBe(DEFAULT_URL)
})
})
it("gets the platform url from the environment", async () => {
await tenancy.doInTenant(null, async () => {
env._set("PLATFORM_URL", ENV_URL)
const url = await getPlatformUrl()
expect(url).toBe(ENV_URL)
})
})
it("gets the platform url from the database", async () => {
await tenancy.doInTenant(null, async () => {
const dbUrl = generator.url()
await setDbPlatformUrl(dbUrl)
const url = await getPlatformUrl()
expect(url).toBe(dbUrl)
})
})
})
describe("cloud", () => {
const TENANT_AWARE_URL = "http://default.env.com"
beforeEach(async () => {
env._set("SELF_HOSTED", 0)
env._set("MULTI_TENANCY", 1)
env._set("PLATFORM_URL", ENV_URL)
await clearSettingsConfig()
})
it("gets the platform url from the environment without tenancy", async () => {
await tenancy.doInTenant(DEFAULT_TENANT_ID, async () => {
const url = await getPlatformUrl({ tenantAware: false })
expect(url).toBe(ENV_URL)
})
})
it("gets the platform url from the environment with tenancy", async () => {
await tenancy.doInTenant(DEFAULT_TENANT_ID, async () => {
const url = await getPlatformUrl()
expect(url).toBe(TENANT_AWARE_URL)
})
})
it("never gets the platform url from the database", async () => {
await tenancy.doInTenant(DEFAULT_TENANT_ID, async () => {
await setDbPlatformUrl(generator.url())
const url = await getPlatformUrl()
expect(url).toBe(TENANT_AWARE_URL)
})
})
})
})
describe("getScopedConfig", () => {
describe("settings config", () => {
beforeEach(async () => {
env._set("SELF_HOSTED", 1)
env._set("PLATFORM_URL", "")
await clearSettingsConfig()
})
it("returns the platform url with an existing config", async () => {
await tenancy.doInTenant(DEFAULT_TENANT_ID, async () => {
const dbUrl = generator.url()
await setDbPlatformUrl(dbUrl)
const db = tenancy.getGlobalDB()
const config = await getScopedConfig(db, { type: Config.SETTINGS })
expect(config.platformUrl).toBe(dbUrl)
})
})
it("returns the platform url without an existing config", async () => {
await tenancy.doInTenant(DEFAULT_TENANT_ID, async () => {
const db = tenancy.getGlobalDB()
const config = await getScopedConfig(db, { type: Config.SETTINGS })
expect(config.platformUrl).toBe(DEFAULT_URL)
})
})
})
})

192
packages/backend-core/src/db/tests/utils.spec.ts Normal file
View File

@ -0,0 +1,192 @@
import { generator, DBTestConfiguration, testEnv } from "../../../tests"
import {
getDevelopmentAppID,
getProdAppID,
isDevAppID,
isProdAppID,
} from "../conversions"
import { generateAppID, getPlatformUrl, getScopedConfig } from "../utils"
import * as context from "../../context"
import { Config } from "../../constants"
import env from "../../environment"
describe("utils", () => {
const config = new DBTestConfiguration()
describe("app ID manipulation", () => {
function getID() {
const appId = generateAppID()
const split = appId.split("_")
const uuid = split[split.length - 1]
const devAppId = `app_dev_${uuid}`
return { appId, devAppId, split, uuid }
}
it("should be able to generate a new app ID", () => {
expect(generateAppID().startsWith("app_")).toEqual(true)
})
it("should be able to convert a production app ID to development", () => {
const { appId, uuid } = getID()
expect(getDevelopmentAppID(appId)).toEqual(`app_dev_${uuid}`)
})
it("should be able to convert a development app ID to development", () => {
const { devAppId, uuid } = getID()
expect(getDevelopmentAppID(devAppId)).toEqual(`app_dev_${uuid}`)
})
it("should be able to convert a development ID to a production", () => {
const { devAppId, uuid } = getID()
expect(getProdAppID(devAppId)).toEqual(`app_${uuid}`)
})
it("should be able to convert a production ID to production", () => {
const { appId, uuid } = getID()
expect(getProdAppID(appId)).toEqual(`app_${uuid}`)
})
it("should be able to confirm dev app ID is development", () => {
const { devAppId } = getID()
expect(isDevAppID(devAppId)).toEqual(true)
})
it("should be able to confirm prod app ID is not development", () => {
const { appId } = getID()
expect(isDevAppID(appId)).toEqual(false)
})
it("should be able to confirm prod app ID is prod", () => {
const { appId } = getID()
expect(isProdAppID(appId)).toEqual(true)
})
it("should be able to confirm dev app ID is not prod", () => {
const { devAppId } = getID()
expect(isProdAppID(devAppId)).toEqual(false)
})
})
const DEFAULT_URL = "http://localhost:10000"
const ENV_URL = "http://env.com"
const setDbPlatformUrl = async (dbUrl: string) => {
const db = context.getGlobalDB()
await db.put({
_id: "config_settings",
type: Config.SETTINGS,
config: {
platformUrl: dbUrl,
},
})
}
const clearSettingsConfig = async () => {
await config.doInTenant(async () => {
const db = context.getGlobalDB()
try {
const config = await db.get("config_settings")
await db.remove("config_settings", config._rev)
} catch (e: any) {
if (e.status !== 404) {
throw e
}
}
})
}
describe("getPlatformUrl", () => {
describe("self host", () => {
beforeEach(async () => {
testEnv.selfHosted()
await clearSettingsConfig()
})
it("gets the default url", async () => {
await config.doInTenant(async () => {
const url = await getPlatformUrl()
expect(url).toBe(DEFAULT_URL)
})
})
it("gets the platform url from the environment", async () => {
await config.doInTenant(async () => {
env._set("PLATFORM_URL", ENV_URL)
const url = await getPlatformUrl()
expect(url).toBe(ENV_URL)
})
})
it("gets the platform url from the database", async () => {
await config.doInTenant(async () => {
const dbUrl = generator.url()
await setDbPlatformUrl(dbUrl)
const url = await getPlatformUrl()
expect(url).toBe(dbUrl)
})
})
})
describe("cloud", () => {
const TENANT_AWARE_URL = `http://${config.tenantId}.env.com`
beforeEach(async () => {
testEnv.cloudHosted()
testEnv.multiTenant()
env._set("PLATFORM_URL", ENV_URL)
await clearSettingsConfig()
})
it("gets the platform url from the environment without tenancy", async () => {
await config.doInTenant(async () => {
const url = await getPlatformUrl({ tenantAware: false })
expect(url).toBe(ENV_URL)
})
})
it("gets the platform url from the environment with tenancy", async () => {
await config.doInTenant(async () => {
const url = await getPlatformUrl()
expect(url).toBe(TENANT_AWARE_URL)
})
})
it("never gets the platform url from the database", async () => {
await config.doInTenant(async () => {
await setDbPlatformUrl(generator.url())
const url = await getPlatformUrl()
expect(url).toBe(TENANT_AWARE_URL)
})
})
})
})
describe("getScopedConfig", () => {
describe("settings config", () => {
beforeEach(async () => {
env._set("SELF_HOSTED", 1)
env._set("PLATFORM_URL", "")
await clearSettingsConfig()
})
it("returns the platform url with an existing config", async () => {
await config.doInTenant(async () => {
const dbUrl = generator.url()
await setDbPlatformUrl(dbUrl)
const db = context.getGlobalDB()
const config = await getScopedConfig(db, { type: Config.SETTINGS })
expect(config.platformUrl).toBe(dbUrl)
})
})
it("returns the platform url without an existing config", async () => {
await config.doInTenant(async () => {
const db = context.getGlobalDB()
const config = await getScopedConfig(db, { type: Config.SETTINGS })
expect(config.platformUrl).toBe(DEFAULT_URL)
})
})
})
})
})

64
packages/backend-core/src/db/views.ts
View File

@ -1,13 +1,14 @@
import {
DocumentType,
ViewName,
DeprecatedViews,
DocumentType,
SEPARATOR,
StaticDatabases,
ViewName,
} from "../constants"
import { getGlobalDB } from "../context"
import { doWithDB } from "./"
import { Database, DatabaseQueryOpts } from "@budibase/types"
import env from "../environment"
const DESIGN_DB = "_design/database"
@ -69,17 +70,6 @@ export const createNewUserEmailView = async () => {
await createView(db, viewJs, ViewName.USER_BY_EMAIL)
}
export const createAccountEmailView = async () => {
const viewJs = `function(doc) {
if (doc._id.startsWith("${DocumentType.ACCOUNT_METADATA}${SEPARATOR}")) {
emit(doc.email.toLowerCase(), doc._id)
}
}`
await doWithDB(StaticDatabases.PLATFORM_INFO.name, async (db: Database) => {
await createView(db, viewJs, ViewName.ACCOUNT_BY_EMAIL)
})
}
export const createUserAppView = async () => {
const db = getGlobalDB()
const viewJs = `function(doc) {
@ -113,17 +103,6 @@ export const createUserBuildersView = async () => {
await createView(db, viewJs, ViewName.USER_BY_BUILDERS)
}
export const createPlatformUserView = async () => {
const viewJs = `function(doc) {
if (doc.tenantId) {
emit(doc._id.toLowerCase(), doc._id)
}
}`
await doWithDB(StaticDatabases.PLATFORM_INFO.name, async (db: Database) => {
await createView(db, viewJs, ViewName.PLATFORM_USERS_LOWERCASE)
})
}
export interface QueryViewOptions {
arrayResponse?: boolean
}
@ -162,13 +141,48 @@ export const queryView = async <T>(
}
}
// PLATFORM
async function createPlatformView(viewJs: string, viewName: ViewName) {
try {
await doWithDB(StaticDatabases.PLATFORM_INFO.name, async (db: Database) => {
await createView(db, viewJs, viewName)
})
} catch (e: any) {
if (e.status === 409 && env.isTest()) {
// multiple tests can try to initialise platforms views
// at once - safe to exit on conflict
return
}
throw e
}
}
export const createPlatformAccountEmailView = async () => {
const viewJs = `function(doc) {
if (doc._id.startsWith("${DocumentType.ACCOUNT_METADATA}${SEPARATOR}")) {
emit(doc.email.toLowerCase(), doc._id)
}
}`
await createPlatformView(viewJs, ViewName.ACCOUNT_BY_EMAIL)
}
export const createPlatformUserView = async () => {
const viewJs = `function(doc) {
if (doc.tenantId) {
emit(doc._id.toLowerCase(), doc._id)
}
}`
await createPlatformView(viewJs, ViewName.PLATFORM_USERS_LOWERCASE)
}
export const queryPlatformView = async <T>(
viewName: ViewName,
params: DatabaseQueryOpts,
opts?: QueryViewOptions
): Promise<T[] | T | undefined> => {
const CreateFuncByName: any = {
[ViewName.ACCOUNT_BY_EMAIL]: createAccountEmailView,
[ViewName.ACCOUNT_BY_EMAIL]: createPlatformAccountEmailView,
[ViewName.PLATFORM_USERS_LOWERCASE]: createPlatformUserView,
}

4
packages/backend-core/src/events/analytics.ts
View File

@ -1,5 +1,5 @@
import env from "../environment"
import * as tenancy from "../tenancy"
import * as context from "../context"
import * as dbUtils from "../db/utils"
import { Config } from "../constants"
import { withCache, TTL, CacheKey } from "../cache"
@ -42,7 +42,7 @@ export const enabled = async () => {
}
const getSettingsDoc = async () => {
const db = tenancy.getGlobalDB()
const db = context.getGlobalDB()
let settings
try {
settings = await db.get(dbUtils.generateConfigID({ type: Config.SETTINGS }))

6
packages/backend-core/src/events/processors/posthog/tests/PosthogProcessor.spec.ts
View File

@ -1,4 +1,4 @@
import "../../../../../tests"
import { testEnv } from "../../../../../tests"
import PosthogProcessor from "../PosthogProcessor"
import { Event, IdentityType, Hosting } from "@budibase/types"
const tk = require("timekeeper")
@ -16,6 +16,10 @@ const newIdentity = () => {
}
describe("PosthogProcessor", () => {
beforeAll(() => {
testEnv.singleTenant()
})
beforeEach(async () => {
jest.clearAllMocks()
await cache.bustCache(

4
packages/backend-core/src/featureFlags/index.ts
View File

@ -1,5 +1,5 @@
import env from "../environment"
import * as tenancy from "../tenancy"
import * as context from "../context"
/**
* Read the TENANT_FEATURE_FLAGS env var and return an array of features flags for each tenant.
@ -28,7 +28,7 @@ export function buildFeatureFlags() {
}
export function isEnabled(featureFlag: string) {
const tenantId = tenancy.getTenantId()
const tenantId = context.getTenantId()
const flags = getTenantFeatureFlags(tenantId)
return flags.includes(featureFlag)
}

20
packages/backend-core/src/index.ts
View File

@ -3,12 +3,11 @@ export * as migrations from "./migrations"
export * as users from "./users"
export * as roles from "./security/roles"
export * as permissions from "./security/permissions"
export * as accounts from "./cloud/accounts"
export * as accounts from "./accounts"
export * as installation from "./installation"
export * as tenancy from "./tenancy"
export * as featureFlags from "./featureFlags"
export * as sessions from "./security/sessions"
export * as deprovisioning from "./context/deprovision"
export * as platform from "./platform"
export * as auth from "./auth"
export * as constants from "./constants"
export * as logging from "./logging"
@ -21,20 +20,27 @@ export * as context from "./context"
export * as cache from "./cache"
export * as objectStore from "./objectStore"
export * as redis from "./redis"
export * as locks from "./redis/redlock"
export * as utils from "./utils"
export * as errors from "./errors"
export { default as env } from "./environment"
// Add context to tenancy for backwards compatibility
// only do this for external usages to prevent internal
// circular dependencies
import * as context from "./context"
import * as _tenancy from "./tenancy"
export const tenancy = {
..._tenancy,
...context,
}
// expose error classes directly
export * from "./errors"
// expose constants directly
export * from "./constants"
// expose inner locks from redis directly
import * as redis from "./redis"
export const locks = redis.redlock
// expose package init function
import * as db from "./db"
export const init = (opts: any = {}) => {

2
packages/backend-core/src/middleware/authenticated.ts
View File

@ -4,7 +4,7 @@ import { getUser } from "../cache/user"
import { getSession, updateSessionTTL } from "../security/sessions"
import { buildMatcherRegex, matches } from "./matchers"
import { SEPARATOR, queryGlobalView, ViewName } from "../db"
import { getGlobalDB, doInTenant } from "../tenancy"
import { getGlobalDB, doInTenant } from "../context"
import { decrypt } from "../security/encryption"
import * as identity from "../context/identity"
import env from "../environment"

4
packages/backend-core/src/middleware/passport/utils.ts
View File

@ -1,6 +1,6 @@
import { isMultiTenant, getTenantId } from "../../tenancy"
import { isMultiTenant, getTenantId } from "../../context"
import { getScopedConfig } from "../../db"
import { ConfigType, Database, Config } from "@budibase/types"
import { ConfigType, Database } from "@budibase/types"
/**
* Utility to handle authentication errors.

3
packages/backend-core/src/middleware/tenancy.ts
View File

@ -1,4 +1,5 @@
import { doInTenant, getTenantIDFromCtx } from "../tenancy"
import { doInTenant } from "../context"
import { getTenantIDFromCtx } from "../tenancy"
import { buildMatcherRegex, matches } from "./matchers"
import { Header } from "../constants"
import {

8
packages/backend-core/src/migrations/migrations.ts
View File

@ -7,7 +7,7 @@ import {
doWithDB,
} from "../db"
import environment from "../environment"
import { doInTenant, getTenantIds, getTenantId } from "../tenancy"
import * as platform from "../platform"
import * as context from "../context"
import { DEFINITIONS } from "."
import {
@ -47,7 +47,7 @@ export const runMigration = async (
const migrationType = migration.type
let tenantId: string | undefined
if (migrationType !== MigrationType.INSTALLATION) {
tenantId = getTenantId()
tenantId = context.getTenantId()
}
const migrationName = migration.name
const silent = migration.silent
@ -160,7 +160,7 @@ export const runMigrations = async (
tenantIds = [options.noOp.tenantId]
} else if (!options.tenantIds || !options.tenantIds.length) {
// run for all tenants
tenantIds = await getTenantIds()
tenantIds = await platform.tenants.getTenantIds()
} else {
tenantIds = options.tenantIds
}
@ -185,7 +185,7 @@ export const runMigrations = async (
// for all migrations
for (const migration of migrations) {
// run the migration
await doInTenant(tenantId, () => runMigration(migration, options))
await context.doInTenant(tenantId, () => runMigration(migration, options))
}
}
console.log("Migrations complete")

4
packages/backend-core/src/objectStore/buckets/global.ts
View File

@ -1,5 +1,5 @@
import env from "../../environment"
import * as tenancy from "../../tenancy"
import * as context from "../../context"
import * as objectStore from "../objectStore"
import * as cloudfront from "../cloudfront"
@ -22,7 +22,7 @@ export const getGlobalFileUrl = (type: string, name: string, etag?: string) => {
export const getGlobalFileS3Key = (type: string, name: string) => {
let file = `${type}/${name}`
if (env.MULTI_TENANCY) {
const tenantId = tenancy.getTenantId()
const tenantId = context.getTenantId()
file = `${tenantId}/${file}`
}
return file

4
packages/backend-core/src/objectStore/buckets/plugins.ts
View File

@ -1,6 +1,6 @@
import env from "../../environment"
import * as objectStore from "../objectStore"
import * as tenancy from "../../tenancy"
import * as context from "../../context"
import * as cloudfront from "../cloudfront"
import { Plugin } from "@budibase/types"
@ -61,7 +61,7 @@ const getPluginS3Key = (plugin: Plugin, fileName: string) => {
export const getPluginS3Dir = (pluginName: string) => {
let s3Key = `${pluginName}`
if (env.MULTI_TENANCY) {
const tenantId = tenancy.getTenantId()
const tenantId = context.getTenantId()
s3Key = `${tenantId}/${s3Key}`
}
if (env.CLOUDFRONT_CDN) {

3
packages/backend-core/src/platform/index.ts Normal file
View File

@ -0,0 +1,3 @@
export * as users from "./users"
export * as tenants from "./tenants"
export * from "./platformDb"

6
packages/backend-core/src/platform/platformDb.ts Normal file
View File

@ -0,0 +1,6 @@
import { StaticDatabases } from "../constants"
import { getDB } from "../db/db"
export function getPlatformDB() {
return getDB(StaticDatabases.PLATFORM_INFO.name)
}

101
packages/backend-core/src/platform/tenants.ts Normal file
View File

@ -0,0 +1,101 @@
import { StaticDatabases } from "../constants"
import { getPlatformDB } from "./platformDb"
import { LockName, LockOptions, LockType, Tenants } from "@budibase/types"
import * as locks from "../redis/redlock"
const TENANT_DOC = StaticDatabases.PLATFORM_INFO.docs.tenants
export const tenacyLockOptions: LockOptions = {
type: LockType.DEFAULT,
name: LockName.UPDATE_TENANTS_DOC,
ttl: 10 * 1000, // auto expire after 10 seconds
systemLock: true,
}
// READ
export async function getTenantIds(): Promise<string[]> {
const tenants = await getTenants()
return tenants.tenantIds
}
async function getTenants(): Promise<Tenants> {
const db = getPlatformDB()
let tenants: Tenants
try {
tenants = await db.get(TENANT_DOC)
} catch (e: any) {
// doesn't exist yet - create
if (e.status === 404) {
tenants = await createTenantsDoc()
} else {
throw e
}
}
return tenants
}
export async function exists(tenantId: string) {
const tenants = await getTenants()
return tenants.tenantIds.indexOf(tenantId) !== -1
}
// CREATE / UPDATE
function newTenantsDoc(): Tenants {
return {
_id: TENANT_DOC,
tenantIds: [],
}
}
async function createTenantsDoc(): Promise<Tenants> {
const db = getPlatformDB()
let tenants = newTenantsDoc()
try {
const response = await db.put(tenants)
tenants._rev = response.rev
} catch (e: any) {
// don't throw 409 is doc has already been created
if (e.status === 409) {
return db.get(TENANT_DOC)
}
throw e
}
return tenants
}
export async function addTenant(tenantId: string) {
const db = getPlatformDB()
// use a lock as tenant creation is conflict prone
await locks.doWithLock(tenacyLockOptions, async () => {
const tenants = await getTenants()
// write the new tenant if it doesn't already exist
if (tenants.tenantIds.indexOf(tenantId) === -1) {
tenants.tenantIds.push(tenantId)
await db.put(tenants)
}
})
}
// DELETE
export async function removeTenant(tenantId: string) {
try {
await locks.doWithLock(tenacyLockOptions, async () => {
const db = getPlatformDB()
const tenants = await getTenants()
tenants.tenantIds = tenants.tenantIds.filter(id => id !== tenantId)
await db.put(tenants)
})
} catch (err) {
console.error(`Error removing tenant ${tenantId} from info db`, err)
throw err
}
}

25
packages/backend-core/src/platform/tests/tenants.spec.ts Normal file
View File

@ -0,0 +1,25 @@
import { DBTestConfiguration, structures } from "../../../tests"
import * as tenants from "../tenants"
describe("tenants", () => {
const config = new DBTestConfiguration()
describe("addTenant", () => {
it("concurrently adds multiple tenants safely", async () => {
const tenant1 = structures.tenant.id()
const tenant2 = structures.tenant.id()
const tenant3 = structures.tenant.id()
await Promise.all([
tenants.addTenant(tenant1),
tenants.addTenant(tenant2),
tenants.addTenant(tenant3),
])
const tenantIds = await tenants.getTenantIds()
expect(tenantIds.includes(tenant1)).toBe(true)
expect(tenantIds.includes(tenant2)).toBe(true)
expect(tenantIds.includes(tenant3)).toBe(true)
})
})
})

90
packages/backend-core/src/platform/users.ts Normal file
View File

@ -0,0 +1,90 @@
import { getPlatformDB } from "./platformDb"
import { DEFAULT_TENANT_ID } from "../constants"
import env from "../environment"
import {
PlatformUser,
PlatformUserByEmail,
PlatformUserById,
User,
} from "@budibase/types"
// READ
export async function lookupTenantId(userId: string) {
if (!env.MULTI_TENANCY) {
return DEFAULT_TENANT_ID
}
const user = await getUserDoc(userId)
return user.tenantId
}
async function getUserDoc(emailOrId: string): Promise<PlatformUser> {
const db = getPlatformDB()
return db.get(emailOrId)
}
// CREATE
function newUserIdDoc(id: string, tenantId: string): PlatformUserById {
return {
_id: id,
tenantId,
}
}
function newUserEmailDoc(
userId: string,
email: string,
tenantId: string
): PlatformUserByEmail {
return {
_id: email,
userId,
tenantId,
}
}
/**
* Add a new user id or email doc if it doesn't exist.
*/
async function addUserDoc(emailOrId: string, newDocFn: () => PlatformUser) {
const db = getPlatformDB()
let user: PlatformUser
try {
await db.get(emailOrId)
} catch (e: any) {
if (e.status === 404) {
user = newDocFn()
await db.put(user)
} else {
throw e
}
}
}
export async function addUser(tenantId: string, userId: string, email: string) {
await Promise.all([
addUserDoc(userId, () => newUserIdDoc(userId, tenantId)),
addUserDoc(email, () => newUserEmailDoc(userId, email, tenantId)),
])
}
// DELETE
export async function removeUser(user: User) {
const db = getPlatformDB()
const keys = [user._id!, user.email]
const userDocs = await db.allDocs({
keys,
include_docs: true,
})
const toDelete = userDocs.rows.map((row: any) => {
return {
...row.doc,
_deleted: true,
}
})
await db.bulkDocs(toDelete)
}

2
packages/backend-core/src/redis/index.ts
View File

@ -3,4 +3,4 @@
export { default as Client } from "./redis"
export * as utils from "./utils"
export * as clients from "./init"
export * as redlock from "./redlock"
export * as locks from "./redlock"

39
packages/backend-core/src/redis/redlock.ts
View File

@ -1,29 +1,22 @@
import Redlock, { Options } from "redlock"
import { getLockClient } from "./init"
import { LockOptions, LockType } from "@budibase/types"
import * as tenancy from "../tenancy"
let noRetryRedlock: Redlock | undefined
import * as context from "../context"
import env from "../environment"
const getClient = async (type: LockType): Promise<Redlock> => {
if (env.isTest() && type !== LockType.TRY_ONCE) {
return newRedlock(OPTIONS.TEST)
}
switch (type) {
case LockType.TRY_ONCE: {
if (!noRetryRedlock) {
noRetryRedlock = await newRedlock(OPTIONS.TRY_ONCE)
}
return noRetryRedlock
return newRedlock(OPTIONS.TRY_ONCE)
}
case LockType.DEFAULT: {
if (!noRetryRedlock) {
noRetryRedlock = await newRedlock(OPTIONS.DEFAULT)
}
return noRetryRedlock
return newRedlock(OPTIONS.DEFAULT)
}
case LockType.DELAY_500: {
if (!noRetryRedlock) {
noRetryRedlock = await newRedlock(OPTIONS.DELAY_500)
}
return noRetryRedlock
return newRedlock(OPTIONS.DELAY_500)
}
default: {
throw new Error(`Could not get redlock client: ${type}`)
@ -36,6 +29,11 @@ export const OPTIONS = {
// immediately throws an error if the lock is already held
retryCount: 0,
},
TEST: {
// higher retry count in unit tests
// due to high contention.
retryCount: 100,
},
DEFAULT: {
// the expected clock drift; for more details
// see http://redis.io/topics/distlock
@ -69,12 +67,19 @@ export const doWithLock = async (opts: LockOptions, task: any) => {
const redlock = await getClient(opts.type)
let lock
try {
// aquire lock
let name: string = `lock:${tenancy.getTenantId()}_${opts.name}`
// determine lock name
// by default use the tenantId for uniqueness, unless using a system lock
const prefix = opts.systemLock ? "system" : context.getTenantId()
let name: string = `lock:${prefix}_${opts.name}`
// add additional unique name if required
if (opts.nameSuffix) {
name = name + `_${opts.nameSuffix}`
}
// create the lock
lock = await redlock.lock(name, opts.ttl)
// perform locked task
// need to await to ensure completion before unlocking
const result = await task()

6
packages/backend-core/src/tenancy/db.ts Normal file
View File

@ -0,0 +1,6 @@
import { getDB } from "../db/db"
import { getGlobalDBName } from "../context"
export function getTenantDB(tenantId: string) {
return getDB(getGlobalDBName(tenantId))
}

2
packages/backend-core/src/tenancy/index.ts
View File

@ -1,2 +1,2 @@
export * from "../context"
export * from "./db"
export * from "./tenancy"

102
packages/backend-core/src/tenancy/tenancy.ts
View File

@ -1,4 +1,3 @@
import { doWithDB, getGlobalDBName } from "../db"
import {
DEFAULT_TENANT_ID,
getTenantId,
@ -11,10 +10,7 @@ import {
TenantResolutionStrategy,
GetTenantIdOptions,
} from "@budibase/types"
import { Header, StaticDatabases } from "../constants"
const TENANT_DOC = StaticDatabases.PLATFORM_INFO.docs.tenants
const PLATFORM_INFO_DB = StaticDatabases.PLATFORM_INFO.name
import { Header } from "../constants"
export function addTenantToUrl(url: string) {
const tenantId = getTenantId()
@ -27,89 +23,6 @@ export function addTenantToUrl(url: string) {
return url
}
export async function doesTenantExist(tenantId: string) {
return doWithDB(PLATFORM_INFO_DB, async (db: any) => {
let tenants
try {
tenants = await db.get(TENANT_DOC)
} catch (err) {
// if theres an error the doc doesn't exist, no tenants exist
return false
}
return (
tenants &&
Array.isArray(tenants.tenantIds) &&
tenants.tenantIds.indexOf(tenantId) !== -1
)
})
}
export async function tryAddTenant(
tenantId: string,
userId: string,
email: string,
afterCreateTenant: () => Promise<void>
) {
return doWithDB(PLATFORM_INFO_DB, async (db: any) => {
const getDoc = async (id: string) => {
if (!id) {
return null
}
try {
return await db.get(id)
} catch (err) {
return { _id: id }
}
}
let [tenants, userIdDoc, emailDoc] = await Promise.all([
getDoc(TENANT_DOC),
getDoc(userId),
getDoc(email),
])
if (!Array.isArray(tenants.tenantIds)) {
tenants = {
_id: TENANT_DOC,
tenantIds: [],
}
}
let promises = []
if (userIdDoc) {
userIdDoc.tenantId = tenantId
promises.push(db.put(userIdDoc))
}
if (emailDoc) {
emailDoc.tenantId = tenantId
emailDoc.userId = userId
promises.push(db.put(emailDoc))
}
if (tenants.tenantIds.indexOf(tenantId) === -1) {
tenants.tenantIds.push(tenantId)
promises.push(db.put(tenants))
await afterCreateTenant()
}
await Promise.all(promises)
})
}
export function doWithGlobalDB(tenantId: string, cb: any) {
return doWithDB(getGlobalDBName(tenantId), cb)
}
export async function lookupTenantId(userId: string) {
return doWithDB(StaticDatabases.PLATFORM_INFO.name, async (db: any) => {
let tenantId = env.MULTI_TENANCY ? DEFAULT_TENANT_ID : null
try {
const doc = await db.get(userId)
if (doc && doc.tenantId) {
tenantId = doc.tenantId
}
} catch (err) {
// just return the default
}
return tenantId
})
}
export const isUserInAppTenant = (appId: string, user?: any) => {
let userTenantId
if (user) {
@ -121,19 +34,6 @@ export const isUserInAppTenant = (appId: string, user?: any) => {
return tenantId === userTenantId
}
export async function getTenantIds() {
return doWithDB(PLATFORM_INFO_DB, async (db: any) => {
let tenants
try {
tenants = await db.get(TENANT_DOC)
} catch (err) {
// if theres an error the doc doesn't exist, no tenants exist
return []
}
return (tenants && tenants.tenantIds) || []
})
}
const ALL_STRATEGIES = Object.values(TenantResolutionStrategy)
export const getTenantIDFromCtx = (

46
packages/backend-core/src/utils/tests/utils.spec.ts
View File

@ -1,21 +1,12 @@
import { structures } from "../../../tests"
import { structures, DBTestConfiguration } from "../../../tests"
import * as utils from "../../utils"
import * as events from "../../events"
import * as db from "../../db"
import { Header } from "../../constants"
import { doInTenant } from "../../context"
import { newid } from "../../utils"
import env from "../../environment"
describe("utils", () => {
describe("platformLogout", () => {
it("should call platform logout", async () => {
await doInTenant(structures.tenant.id(), async () => {
const ctx = structures.koa.newContext()
await utils.platformLogout({ ctx, userId: "test" })
expect(events.auth.logout).toBeCalledTimes(1)
})
})
})
const config = new DBTestConfiguration()
describe("getAppIdFromCtx", () => {
it("gets appId from header", async () => {
@ -50,21 +41,28 @@ describe("utils", () => {
})
it("gets appId from url", async () => {
const ctx = structures.koa.newContext()
const expected = db.generateAppID()
const app = structures.apps.app(expected)
await config.doInTenant(async () => {
const url = "http://test.com"
env._set("PLATFORM_URL", url)
// set custom url
const appUrl = newid()
app.url = `/${appUrl}`
ctx.path = `/app/${appUrl}`
const ctx = structures.koa.newContext()
ctx.host = `${config.tenantId}.test.com`
// save the app
const database = db.getDB(expected)
await database.put(app)
const expected = db.generateAppID(config.tenantId)
const app = structures.apps.app(expected)
const actual = await utils.getAppIdFromCtx(ctx)
expect(actual).toBe(expected)
// set custom url
const appUrl = newid()
app.url = `/${appUrl}`
ctx.path = `/app/${appUrl}`
// save the app
const database = db.getDB(expected)
await database.put(app)
const actual = await utils.getAppIdFromCtx(ctx)
expect(actual).toBe(expected)
})
})
it("doesn't get appId from url when previewing", async () => {

32
packages/backend-core/tests/utilities/DBTestConfiguration.ts Normal file
View File

@ -0,0 +1,32 @@
import "./mocks"
import * as structures from "./structures"
import * as testEnv from "./testEnv"
import * as context from "../../src/context"
class DBTestConfiguration {
tenantId: string
constructor() {
// db tests need to be multi tenant to prevent conflicts
testEnv.multiTenant()
this.tenantId = structures.tenant.id()
}
// TENANCY
doInTenant(task: any) {
return context.doInTenant(this.tenantId, () => {
return task()
})
}
getTenantId() {
try {
return context.getTenantId()
} catch (e) {
return this.tenantId!
}
}
}
export default DBTestConfiguration

9
packages/backend-core/tests/utilities/db.ts
View File

@ -1,9 +0,0 @@
import * as db from "../../src/db"
const dbConfig = {
inMemory: true,
}
export const init = () => {
db.init(dbConfig)
}

3
packages/backend-core/tests/utilities/index.ts
View File

@ -4,5 +4,4 @@ export { generator } from "./structures"
export * as testEnv from "./testEnv"
export * as testContainerUtils from "./testContainerUtils"
import * as dbConfig from "./db"
dbConfig.init()
export { default as DBTestConfiguration } from "./DBTestConfiguration"

16
packages/backend-core/tests/utilities/testEnv.ts
View File

@ -1,12 +1,12 @@
import env from "../../src/environment"
import * as tenancy from "../../src/tenancy"
import { newid } from "../../src/utils"
import * as context from "../../src/context"
import * as structures from "./structures"
// TENANCY
export async function withTenant(task: (tenantId: string) => any) {
const tenantId = newid()
return tenancy.doInTenant(tenantId, async () => {
const tenantId = structures.tenant.id()
return context.doInTenant(tenantId, async () => {
await task(tenantId)
})
}
@ -19,6 +19,14 @@ export function multiTenant() {
env._set("MULTI_TENANCY", 1)
}
export function selfHosted() {
env._set("SELF_HOSTED", 1)
}
export function cloudHosted() {
env._set("SELF_HOSTED", 0)
}
// NODE
export function nodeDev() {

8
packages/backend-core/yarn.lock
View File

@ -1197,10 +1197,10 @@
dependencies:
"@types/istanbul-lib-report" "*"
"@types/jest@27.5.1":
version "27.5.1"
resolved "https://registry.yarnpkg.com/@types/jest/-/jest-27.5.1.tgz#2c8b6dc6ff85c33bcd07d0b62cb3d19ddfdb3ab9"
integrity sha512-fUy7YRpT+rHXto1YlL+J9rs0uLGyiqVt3ZOTQR+4ROc47yNl8WLdVLgUloBRhOxP1PZvguHl44T3H0wAWxahYQ==
"@types/jest@28.1.1":
version "28.1.1"
resolved "https://registry.yarnpkg.com/@types/jest/-/jest-28.1.1.tgz#8c9ba63702a11f8c386ee211280e8b68cb093cd1"
integrity sha512-C2p7yqleUKtCkVjlOur9BWVA4HgUQmEj/HWCt5WzZ5mLXrWnyIfl0wGuArc+kBXsy0ZZfLp+7dywB4HtSVYGVA==
dependencies:
jest-matcher-utils "^27.0.0"
pretty-format "^27.0.0"

3
packages/server/package.json
View File

@ -141,6 +141,7 @@
"@types/pouchdb": "6.4.0",
"@types/redis": "4.0.11",
"@types/server-destroy": "1.0.1",
"@types/supertest": "2.0.12",
"@types/tar": "6.1.3",
"@typescript-eslint/parser": "5.45.0",
"apidoc": "0.50.4",
@ -159,7 +160,7 @@
"path-to-regexp": "6.2.0",
"prettier": "2.5.1",
"rimraf": "3.0.2",
"supertest": "4.0.2",
"supertest": "6.2.2",
"swagger-jsdoc": "6.1.0",
"timekeeper": "2.2.0",
"ts-jest": "28.0.4",

1
packages/server/src/api/routes/tests/backup.spec.ts
View File

@ -19,7 +19,6 @@ describe("/backups", () => {
.get(`/api/backups/export?appId=${config.getAppId()}&appname=test`)
.set(config.defaultHeaders())
.expect(200)
expect(res.text).toBeDefined()
expect(res.headers["content-type"]).toEqual("application/gzip")
expect(events.app.exported).toBeCalledTimes(1)
})

96
packages/server/src/tests/utilities/TestConfiguration.ts
View File

@ -236,42 +236,41 @@ class TestConfiguration {
email = this.defaultUserValues.email,
roles,
}: any = {}) {
return tenancy.doWithGlobalDB(this.getTenantId(), async (db: Database) => {
let existing
try {
existing = await db.get(id)
} catch (err) {
existing = { email }
}
const user = {
_id: id,
...existing,
roles: roles || {},
tenantId: this.getTenantId(),
firstName,
lastName,
}
await sessions.createASession(id, {
sessionId: "sessionid",
tenantId: this.getTenantId(),
csrfToken: this.defaultUserValues.csrfToken,
})
if (builder) {
user.builder = { global: true }
} else {
user.builder = { global: false }
}
if (admin) {
user.admin = { global: true }
} else {
user.admin = { global: false }
}
const resp = await db.put(user)
return {
_rev: resp.rev,
...user,
}
const db = tenancy.getTenantDB(this.getTenantId())
let existing
try {
existing = await db.get(id)
} catch (err) {
existing = { email }
}
const user = {
_id: id,
...existing,
roles: roles || {},
tenantId: this.getTenantId(),
firstName,
lastName,
}
await sessions.createASession(id, {
sessionId: "sessionid",
tenantId: this.getTenantId(),
csrfToken: this.defaultUserValues.csrfToken,
})
if (builder) {
user.builder = { global: true }
} else {
user.builder = { global: false }
}
if (admin) {
user.admin = { global: true }
} else {
user.admin = { global: false }
}
const resp = await db.put(user)
return {
_rev: resp.rev,
...user,
}
}
async createUser(
@ -407,20 +406,19 @@ class TestConfiguration {
// API
async generateApiKey(userId = this.defaultUserValues.globalUserId) {
return tenancy.doWithGlobalDB(this.getTenantId(), async (db: any) => {
const id = dbCore.generateDevInfoID(userId)
let devInfo
try {
devInfo = await db.get(id)
} catch (err) {
devInfo = { _id: id, userId }
}
devInfo.apiKey = encryption.encrypt(
`${this.getTenantId()}${dbCore.SEPARATOR}${newid()}`
)
await db.put(devInfo)
return devInfo.apiKey
})
const db = tenancy.getTenantDB(this.getTenantId())
const id = dbCore.generateDevInfoID(userId)
let devInfo
try {
devInfo = await db.get(id)
} catch (err) {
devInfo = { _id: id, userId }
}
devInfo.apiKey = encryption.encrypt(
`${this.getTenantId()}${dbCore.SEPARATOR}${newid()}`
)
await db.put(devInfo)
return devInfo.apiKey
}
// APP

109
packages/server/yarn.lock
View File

@ -3547,6 +3547,14 @@
resolved "https://registry.yarnpkg.com/@types/stack-utils/-/stack-utils-2.0.1.tgz#20f18294f797f2209b5f65c8e3b5c8e8261d127c"
integrity sha512-Hl219/BT5fLAaz6NDkSuhzasy49dwQS/DSdu4MdggFB8zcXv7vflBI3xp7FEmkmdDkBUI2bPUNeMttp2knYdxw==
"@types/superagent@*":
version "4.1.16"
resolved "https://registry.yarnpkg.com/@types/superagent/-/superagent-4.1.16.tgz#12c9c16f232f9d89beab91d69368f96ce8e2d881"
integrity sha512-tLfnlJf6A5mB6ddqF159GqcDizfzbMUB1/DeT59/wBNqzRTNNKsaw79A/1TZ84X+f/EwWH8FeuSkjlCLyqS/zQ==
dependencies:
"@types/cookiejar" "*"
"@types/node" "*"
"@types/superagent@^4.1.12":
version "4.1.15"
resolved "https://registry.yarnpkg.com/@types/superagent/-/superagent-4.1.15.tgz#63297de457eba5e2bc502a7609426c4cceab434a"
@ -3555,6 +3563,13 @@
"@types/cookiejar" "*"
"@types/node" "*"
"@types/supertest@2.0.12":
version "2.0.12"
resolved "https://registry.yarnpkg.com/@types/supertest/-/supertest-2.0.12.tgz#ddb4a0568597c9aadff8dbec5b2e8fddbe8692fc"
integrity sha512-X3HPWTwXRerBZS7Mo1k6vMVR1Z6zmJcDVn5O/31whe0tnjE4te6ZJSJGq1RiqHPjzPdMTfjCFogDJmwng9xHaQ==
dependencies:
"@types/superagent" "*"
"@types/tar@6.1.3":
version "6.1.3"
resolved "https://registry.yarnpkg.com/@types/tar/-/tar-6.1.3.tgz#46a2ce7617950c4852dfd7e9cd41aa8161b9d750"
@ -4241,7 +4256,7 @@ arrify@^2.0.0:
resolved "https://registry.yarnpkg.com/arrify/-/arrify-2.0.1.tgz#c9655e9331e0abcd588d2a7cad7e9956f66701fa"
integrity sha512-3duEwti880xqi4eAMN8AyR4a0ByT90zoYdLlevfrvU43vb0YZwZVfxOgxWrLXXXpyugL0hNZc9G6BiB5B3nUug==
asap@^2.0.3:
asap@^2.0.0, asap@^2.0.3:
version "2.0.6"
resolved "https://registry.yarnpkg.com/asap/-/asap-2.0.6.tgz#e50347611d7e690943208bbdafebcbc2fb866d46"
integrity sha512-BSHWgDSAiKs50o2Re8ppvp3seVHXSRM44cdSsT9FfNEUUZLOGWVCsiWaRPWM1Znn+mqZ1OfVZ3z3DWEzSp7hRA==
@ -5416,7 +5431,7 @@ commoner@^0.10.1:
q "^1.1.2"
recast "^0.11.17"
component-emitter@^1.2.0, component-emitter@^1.2.1:
component-emitter@^1.2.1, component-emitter@^1.3.0:
version "1.3.0"
resolved "https://registry.yarnpkg.com/component-emitter/-/component-emitter-1.3.0.tgz#16e4070fba8ae29b679f2215853ee181ab2eabc0"
integrity sha512-Rd3se6QB+sO1TwqZjscQrurpEPIfO0/yYnSin6Q/rD3mOutHvUrCAhJub3r90uNb+SESBuE0QYoB90YdfatsRg==
@ -5518,7 +5533,7 @@ cookie@^0.5.0:
resolved "https://registry.yarnpkg.com/cookie/-/cookie-0.5.0.tgz#d1f5d71adec6558c58f389987c366aa47e994f8b"
integrity sha512-YZ3GUyn/o8gfKJlnlX7g7xq4gyO6OSuhGPKaaGssGB2qgDUS0gPgtTvoyZLTt9Ab6dC4hfc9dV5arkvc/OCmrw==
cookiejar@^2.1.0:
cookiejar@^2.1.3:
version "2.1.4"
resolved "https://registry.yarnpkg.com/cookiejar/-/cookiejar-2.1.4.tgz#ee669c1fea2cf42dc31585469d193fef0d65771b"
integrity sha512-LDx6oHrK+PhzLKJU9j5S7/Y3jM/mUHvD/DeI1WQmJn652iPC5Y4TBzC9l+5OMOXlyTTA+SmVUPm0HQUwpD5Jqw==
@ -5967,6 +5982,14 @@ detective@^4.3.1:
acorn "^5.2.1"
defined "^1.0.0"
dezalgo@^1.0.4:
version "1.0.4"
resolved "https://registry.yarnpkg.com/dezalgo/-/dezalgo-1.0.4.tgz#751235260469084c132157dfa857f386d4c33d81"
integrity sha512-rXSP0bf+5n0Qonsb+SVVfNfIsimO4HEtmnIpPHY8Q1UCzKlQrDMfdobr8nJOOsRgWCyMRqeSBQzmWUMq7zvVig==
dependencies:
asap "^2.0.0"
wrappy "1"
diff-match-patch@^1.0.5:
version "1.0.5"
resolved "https://registry.yarnpkg.com/diff-match-patch/-/diff-match-patch-1.0.5.tgz#abb584d5f10cd1196dfc55aa03701592ae3f7b37"
@ -6061,11 +6084,6 @@ dotenv@16.0.1:
resolved "https://registry.yarnpkg.com/dotenv/-/dotenv-16.0.1.tgz#8f8f9d94876c35dac989876a5d3a82a267fdce1d"
integrity sha512-1K6hR6wtk2FviQ4kEiSjFiH5rpzEVi8WW0x96aztHVMhEspNpc4DVOUTEHtEva5VThQ8IaBX1Pe4gSzpVVUsKQ==
dotenv@8.2.0:
version "8.2.0"
resolved "https://registry.yarnpkg.com/dotenv/-/dotenv-8.2.0.tgz#97e619259ada750eea3e4ea3e26bceea5424b16a"
integrity sha512-8sJ78ElpbDJBHNeBzUbUVLsqKdccaa/BXF1uPTw3GrvQTBgrQrtObr2mUrE38vzYd8cEv+m/JBfDLioYcfXoaw==
dotenv@^8.2.0:
version "8.6.0"
resolved "https://registry.yarnpkg.com/dotenv/-/dotenv-8.6.0.tgz#061af664d19f7f4d8fc6e4ff9b584ce237adcb8b"
@ -6888,7 +6906,7 @@ extend-shallow@^3.0.0, extend-shallow@^3.0.2:
assign-symbols "^1.0.0"
is-extendable "^1.0.1"
extend@^3.0.0, extend@^3.0.2, extend@~3.0.2:
extend@^3.0.2, extend@~3.0.2:
version "3.0.2"
resolved "https://registry.yarnpkg.com/extend/-/extend-3.0.2.tgz#f8b1136b4071fbd8eb140aff858b1019ec2915fa"
integrity sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==
@ -6977,7 +6995,7 @@ fast-redact@^3.0.0:
resolved "https://registry.yarnpkg.com/fast-redact/-/fast-redact-3.1.1.tgz#790fcff8f808c2e12fabbfb2be5cb2deda448fa0"
integrity sha512-odVmjC8x8jNeMZ3C+rPMESzXVSEU8tSWSHv9HFxP2mm89G/1WwqhrerJDQm9Zus8X6aoRgQDThKqptdNA6bt+A==
fast-safe-stringify@^2.0.7, fast-safe-stringify@^2.0.8:
fast-safe-stringify@^2.0.7, fast-safe-stringify@^2.0.8, fast-safe-stringify@^2.1.1:
version "2.1.1"
resolved "https://registry.yarnpkg.com/fast-safe-stringify/-/fast-safe-stringify-2.1.1.tgz#c406a83b6e70d9e35ce3b30a81141df30aeba884"
integrity sha512-W+KJc2dmILlPplD/H4K9l9LcAHAfPtP6BY84uVLXQ6Evcz9Lcg33Y2z1IVblT6xdY54PXYVHEv+0Wpq8Io6zkA==
@ -7264,7 +7282,7 @@ form-data@4.0.0, form-data@^4.0.0:
combined-stream "^1.0.8"
mime-types "^2.1.12"
form-data@^2.3.1, form-data@^2.5.0:
form-data@^2.5.0:
version "2.5.1"
resolved "https://registry.yarnpkg.com/form-data/-/form-data-2.5.1.tgz#f2cbec57b5e59e23716e128fe44d4e5dd23895f4"
integrity sha512-m21N3WOmEEURgk6B9GLOE4RuWOFf28Lhh9qGYeNlGq4VDXUlJy2th2slBNU8Gp8EzloYZOibZJ7t5ecIrFSjVA==
@ -7291,11 +7309,21 @@ form-data@~2.3.2:
combined-stream "^1.0.6"
mime-types "^2.1.12"
formidable@^1.1.1, formidable@^1.2.0:
formidable@^1.1.1:
version "1.2.6"
resolved "https://registry.yarnpkg.com/formidable/-/formidable-1.2.6.tgz#d2a51d60162bbc9b4a055d8457a7c75315d1a168"
integrity sha512-KcpbcpuLNOwrEjnbpMC0gS+X8ciDoZE1kkqzat4a8vrprf+s9pKNQ/QIwWfbfs4ltgmFl3MD177SNTkve3BwGQ==
formidable@^2.0.1:
version "2.1.1"
resolved "https://registry.yarnpkg.com/formidable/-/formidable-2.1.1.tgz#81269cbea1a613240049f5f61a9d97731517414f"
integrity sha512-0EcS9wCFEzLvfiks7omJ+SiYJAiD+TzK4Pcw1UlUoGnhUxDcMKjt0P7x8wEb0u6OHu8Nb98WG3nxtlF5C7bvUQ==
dependencies:
dezalgo "^1.0.4"
hexoid "^1.0.0"
once "^1.4.0"
qs "^6.11.0"
forwarded-parse@^2.1.0:
version "2.1.2"
resolved "https://registry.yarnpkg.com/forwarded-parse/-/forwarded-parse-2.1.2.tgz#08511eddaaa2ddfd56ba11138eee7df117a09325"
@ -7947,6 +7975,11 @@ has@^1.0.3:
dependencies:
function-bind "^1.1.1"
hexoid@^1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/hexoid/-/hexoid-1.0.0.tgz#ad10c6573fb907de23d9ec63a711267d9dc9bc18"
integrity sha512-QFLV0taWQOZtvIRIAdBChesmogZrtuXvVWsFHZTk2SU+anspqZ2vMnoLg7IE1+Uk16N19APic1BuF8bC8c2m5g==
homedir-polyfill@^1.0.0, homedir-polyfill@^1.0.1:
version "1.0.3"
resolved "https://registry.yarnpkg.com/homedir-polyfill/-/homedir-polyfill-1.0.3.tgz#743298cef4e5af3e194161fbadcc2151d3a058e8"
@ -10703,7 +10736,7 @@ merge2@^1.3.0, merge2@^1.4.1:
resolved "https://registry.yarnpkg.com/merge2/-/merge2-1.4.1.tgz#4368892f885e907455a6fd7dc55c0c9d404990ae"
integrity sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==
methods@^1.1.1, methods@^1.1.2:
methods@^1.1.2:
version "1.1.2"
resolved "https://registry.yarnpkg.com/methods/-/methods-1.1.2.tgz#5529a4d67654134edcc5266656835b0f851afcee"
integrity sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w==
@ -10755,7 +10788,12 @@ mime-types@^2.1.12, mime-types@^2.1.18, mime-types@^2.1.24, mime-types@^2.1.27,
dependencies:
mime-db "1.52.0"
mime@^1.3.4, mime@^1.4.1:
mime@2.6.0:
version "2.6.0"
resolved "https://registry.yarnpkg.com/mime/-/mime-2.6.0.tgz#a2a682a95cd4d0cb1d6257e28f83da7e35800367"
integrity sha512-USPkMeET31rOMiarsBNIHZKLGgvKc/LrjofAnBlOttf5ajRvqiRA8QsenbcooctK6d6Ts6aqZXBA+XbkKthiQg==
mime@^1.3.4:
version "1.6.0"
resolved "https://registry.yarnpkg.com/mime/-/mime-1.6.0.tgz#32cd9e5c64553bd58d19a568af452acff04981b1"
integrity sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==
@ -12502,14 +12540,14 @@ q@^1.1.2:
resolved "https://registry.yarnpkg.com/q/-/q-1.5.1.tgz#7e32f75b41381291d04611f1bf14109ac00651d7"
integrity sha512-kV/CThkXo6xyFEZUugw/+pIOywXcDbFYgSct5cT3gqlbkBE1SJdwy6UQoZvodiWF/ckQLZyDE/Bu1M6gVu5lVw==
qs@^6.11.0:
qs@^6.10.3, qs@^6.11.0:
version "6.11.0"
resolved "https://registry.yarnpkg.com/qs/-/qs-6.11.0.tgz#fd0d963446f7a65e1367e01abd85429453f0c37a"
integrity sha512-MvjoMCJwEarSbUYk5O+nmoSzSutSsTwF85zcHPQ9OrlFoZOYIjaqBAJIqIXjptyD5vThxGq52Xu/MaJzRkIk4Q==
dependencies:
side-channel "^1.0.4"
qs@^6.4.0, qs@^6.5.1:
qs@^6.4.0:
version "6.10.5"
resolved "https://registry.yarnpkg.com/qs/-/qs-6.10.5.tgz#974715920a80ff6a262264acd2c7e6c2a53282b4"
integrity sha512-O5RlPh0VFtR78y79rgcgKK4wbAI0C5zGVLztOIdpWX6ep368q5Hv6XRxDvXuZ9q3C6v+e3n8UfZZJw7IIG27eQ==
@ -13990,29 +14028,30 @@ sublevel-pouchdb@7.2.2:
ltgt "2.2.1"
readable-stream "1.1.14"
superagent@^3.8.3:
version "3.8.3"
resolved "https://registry.yarnpkg.com/superagent/-/superagent-3.8.3.tgz#460ea0dbdb7d5b11bc4f78deba565f86a178e128"
integrity sha512-GLQtLMCoEIK4eDv6OGtkOoSMt3D+oq0y3dsxMuYuDvaNUvuT8eFBuLmfR0iYYzHC1e8hpzC6ZsxbuP6DIalMFA==
superagent@^7.1.0:
version "7.1.6"
resolved "https://registry.yarnpkg.com/superagent/-/superagent-7.1.6.tgz#64f303ed4e4aba1e9da319f134107a54cacdc9c6"
integrity sha512-gZkVCQR1gy/oUXr+kxJMLDjla434KmSOKbx5iGD30Ql+AkJQ/YlPKECJy2nhqOsHLjGHzoDTXNSjhnvWhzKk7g==
dependencies:
component-emitter "^1.2.0"
cookiejar "^2.1.0"
debug "^3.1.0"
extend "^3.0.0"
form-data "^2.3.1"
formidable "^1.2.0"
methods "^1.1.1"
mime "^1.4.1"
qs "^6.5.1"
readable-stream "^2.3.5"
component-emitter "^1.3.0"
cookiejar "^2.1.3"
debug "^4.3.4"
fast-safe-stringify "^2.1.1"
form-data "^4.0.0"
formidable "^2.0.1"
methods "^1.1.2"
mime "2.6.0"
qs "^6.10.3"
readable-stream "^3.6.0"
semver "^7.3.7"
supertest@4.0.2:
version "4.0.2"
resolved "https://registry.yarnpkg.com/supertest/-/supertest-4.0.2.tgz#c2234dbdd6dc79b6f15b99c8d6577b90e4ce3f36"
integrity sha512-1BAbvrOZsGA3YTCWqbmh14L0YEq0EGICX/nBnfkfVJn7SrxQV1I3pMYjSzG9y/7ZU2V9dWqyqk2POwxlb09duQ==
supertest@6.2.2:
version "6.2.2"
resolved "https://registry.yarnpkg.com/supertest/-/supertest-6.2.2.tgz#04a5998fd3efaff187cb69f07a169755d655b001"
integrity sha512-wCw9WhAtKJsBvh07RaS+/By91NNE0Wh0DN19/hWPlBOU8tAfOtbZoVSV4xXeoKoxgPx0rx2y+y+8660XtE7jzg==
dependencies:
methods "^1.1.2"
superagent "^3.8.3"
superagent "^7.1.0"
supports-color@^5.3.0, supports-color@^5.5.0:
version "5.5.0"

1
packages/types/src/sdk/locks.ts
View File

@ -12,6 +12,7 @@ export enum LockName {
MIGRATIONS = "migrations",
TRIGGER_QUOTA = "trigger_quota",
SYNC_ACCOUNT_LICENSE = "sync_account_license",
UPDATE_TENANTS_DOC = "update_tenants_doc",
}
export interface LockOptions {

5
packages/worker/package.json
View File

@ -22,7 +22,7 @@
"build:docker": "docker build . -t worker-service --label version=$BUDIBASE_RELEASE_VERSION",
"dev:stack:init": "node ./scripts/dev/manage.js init",
"dev:builder": "npm run dev:stack:init && nodemon",
"test": "jest --coverage --maxWorkers=2",
"test": "jest --coverage",
"test:watch": "jest --watch",
"env:multi:enable": "node scripts/multiTenancy.js enable",
"env:multi:disable": "node scripts/multiTenancy.js disable",
@ -73,7 +73,7 @@
"@swc/core": "^1.3.25",
"@swc/jest": "^0.2.24",
"@trendyol/jest-testcontainers": "^2.1.1",
"@types/jest": "26.0.23",
"@types/jest": "28.1.1",
"@types/jsonwebtoken": "8.5.1",
"@types/koa": "2.13.4",
"@types/koa__router": "8.0.8",
@ -81,6 +81,7 @@
"@types/node-fetch": "2.6.1",
"@types/pouchdb": "6.4.0",
"@types/server-destroy": "1.0.1",
"@types/supertest": "2.0.12",
"@types/uuid": "8.3.4",
"@typescript-eslint/parser": "5.45.0",
"copyfiles": "2.4.1",

49
packages/worker/src/api/controllers/global/users.ts
View File

@ -1,5 +1,5 @@
import { checkInviteCode } from "../../../utilities/redis"
import sdk from "../../../sdk"
import * as userSdk from "../../../sdk/users"
import env from "../../../environment"
import {
BulkUserRequest,
@ -8,6 +8,7 @@ import {
CreateAdminUserRequest,
InviteUserRequest,
InviteUsersRequest,
MigrationType,
SearchUsersRequest,
User,
} from "@budibase/types"
@ -16,7 +17,9 @@ import {
cache,
errors,
events,
migrations,
tenancy,
platform,
} from "@budibase/backend-core"
import { checkAnyUserExists } from "../../../utilities/users"
@ -25,7 +28,7 @@ const MAX_USERS_UPLOAD_LIMIT = 1000
export const save = async (ctx: any) => {
try {
const currentUserId = ctx.user._id
ctx.body = await sdk.users.save(ctx.request.body, { currentUserId })
ctx.body = await userSdk.save(ctx.request.body, { currentUserId })
} catch (err: any) {
ctx.throw(err.status || 400, err)
}
@ -35,7 +38,7 @@ const bulkDelete = async (userIds: string[], currentUserId: string) => {
if (userIds?.indexOf(currentUserId) !== -1) {
throw new Error("Unable to delete self.")
}
return await sdk.users.bulkDelete(userIds)
return await userSdk.bulkDelete(userIds)
}
const bulkCreate = async (users: User[], groupIds: string[]) => {
@ -44,7 +47,7 @@ const bulkCreate = async (users: User[], groupIds: string[]) => {
"Max limit for upload is 1000 users. Please reduce file size and try again."
)
}
return await sdk.users.bulkCreate(users, groupIds)
return await userSdk.bulkCreate(users, groupIds)
}
export const bulkUpdate = async (ctx: any) => {
@ -71,16 +74,26 @@ const parseBooleanParam = (param: any) => {
export const adminUser = async (ctx: any) => {
const { email, password, tenantId } = ctx.request
.body as CreateAdminUserRequest
if (await platform.tenants.exists(tenantId)) {
ctx.throw(403, "Organisation already exists.")
}
if (env.MULTI_TENANCY) {
// store the new tenant record in the platform db
await platform.tenants.addTenant(tenantId)
await migrations.backPopulateMigrations({
type: MigrationType.GLOBAL,
tenantId,
})
}
await tenancy.doInTenant(tenantId, async () => {
// account portal sends a pre-hashed password - honour param to prevent double hashing
const hashPassword = parseBooleanParam(ctx.request.query.hashPassword)
// account portal sends no password for SSO users
const requirePassword = parseBooleanParam(ctx.request.query.requirePassword)
if (await tenancy.doesTenantExist(tenantId)) {
ctx.throw(403, "Organisation already exists.")
}
const userExists = await checkAnyUserExists()
if (userExists) {
ctx.throw(
@ -106,7 +119,7 @@ export const adminUser = async (ctx: any) => {
// always bust checklist beforehand, if an error occurs but can proceed, don't get
// stuck in a cycle
await cache.bustCache(cache.CacheKey.CHECKLIST)
const finalUser = await sdk.users.save(user, {
const finalUser = await userSdk.save(user, {
hashPassword,
requirePassword,
})
@ -128,7 +141,7 @@ export const adminUser = async (ctx: any) => {
export const countByApp = async (ctx: any) => {
const appId = ctx.params.appId
try {
ctx.body = await sdk.users.countUsersByApp(appId)
ctx.body = await userSdk.countUsersByApp(appId)
} catch (err: any) {
ctx.throw(err.status || 400, err)
}
@ -140,7 +153,7 @@ export const destroy = async (ctx: any) => {
ctx.throw(400, "Unable to delete self.")
}
await sdk.users.destroy(id, ctx.user)
await userSdk.destroy(id, ctx.user)
ctx.body = {
message: `User ${id} deleted.`,
@ -149,7 +162,7 @@ export const destroy = async (ctx: any) => {
export const search = async (ctx: any) => {
const body = ctx.request.body as SearchUsersRequest
const paginated = await sdk.users.paginatedUsers(body)
const paginated = await userSdk.paginatedUsers(body)
// user hashed password shouldn't ever be returned
for (let user of paginated.data) {
if (user) {
@ -161,7 +174,7 @@ export const search = async (ctx: any) => {
// called internally by app server user fetch
export const fetch = async (ctx: any) => {
const all = await sdk.users.allUsers()
const all = await userSdk.allUsers()
// user hashed password shouldn't ever be returned
for (let user of all) {
if (user) {
@ -173,12 +186,12 @@ export const fetch = async (ctx: any) => {
// called internally by app server user find
export const find = async (ctx: any) => {
ctx.body = await sdk.users.getUser(ctx.params.id)
ctx.body = await userSdk.getUser(ctx.params.id)
}
export const tenantUserLookup = async (ctx: any) => {
const id = ctx.params.id
const user = await sdk.users.getPlatformUser(id)
const user = await userSdk.getPlatformUser(id)
if (user) {
ctx.body = user
} else {
@ -188,7 +201,7 @@ export const tenantUserLookup = async (ctx: any) => {
export const invite = async (ctx: any) => {
const request = ctx.request.body as InviteUserRequest
const response = await sdk.users.invite([request])
const response = await userSdk.invite([request])
// explicitly throw for single user invite
if (response.unsuccessful.length) {
@ -207,7 +220,7 @@ export const invite = async (ctx: any) => {
export const inviteMultiple = async (ctx: any) => {
const request = ctx.request.body as InviteUsersRequest
ctx.body = await sdk.users.invite(request)
ctx.body = await userSdk.invite(request)
}
export const checkInvite = async (ctx: any) => {
@ -229,7 +242,7 @@ export const inviteAccept = async (ctx: any) => {
// info is an extension of the user object that was stored by global
const { email, info }: any = await checkInviteCode(inviteCode)
ctx.body = await tenancy.doInTenant(info.tenantId, async () => {
const saved = await sdk.users.save({
const saved = await userSdk.save({
firstName,
lastName,
password,

12
packages/worker/src/api/controllers/system/tenants.ts
View File

@ -1,8 +1,7 @@
import { BBContext } from "@budibase/types"
import { deprovisioning } from "@budibase/backend-core"
import { quotas } from "@budibase/pro"
import { UserCtx } from "@budibase/types"
import * as tenantSdk from "../../../sdk/tenants"
const _delete = async (ctx: BBContext) => {
export async function destroy(ctx: UserCtx) {
const user = ctx.user!
const tenantId = ctx.params.tenantId
@ -11,13 +10,10 @@ const _delete = async (ctx: BBContext) => {
}
try {
await quotas.bustCache()
await deprovisioning.deleteTenant(tenantId)
await tenantSdk.deleteTenant(tenantId)
ctx.status = 204
} catch (err) {
ctx.log.error(err)
throw err
}
}
export { _delete as delete }

198
packages/worker/src/api/routes/global/tests/configs.spec.ts
View File

@ -2,7 +2,7 @@
jest.mock("nodemailer")
import { TestConfiguration, structures, mocks } from "../../../../tests"
mocks.email.mock()
import { Config, context, events } from "@budibase/backend-core"
import { Config, events } from "@budibase/backend-core"
describe("configs", () => {
const config = new TestConfiguration()
@ -113,64 +113,56 @@ describe("configs", () => {
describe("create", () => {
it("should create activated OIDC config", async () => {
await context.doInTenant(config.tenant1User!.tenantId, async () => {
await saveOIDCConfig()
expect(events.auth.SSOCreated).toBeCalledTimes(1)
expect(events.auth.SSOCreated).toBeCalledWith(Config.OIDC)
expect(events.auth.SSODeactivated).not.toBeCalled()
expect(events.auth.SSOActivated).toBeCalledTimes(1)
expect(events.auth.SSOActivated).toBeCalledWith(Config.OIDC)
await config.deleteConfig(Config.OIDC)
})
await saveOIDCConfig()
expect(events.auth.SSOCreated).toBeCalledTimes(1)
expect(events.auth.SSOCreated).toBeCalledWith(Config.OIDC)
expect(events.auth.SSODeactivated).not.toBeCalled()
expect(events.auth.SSOActivated).toBeCalledTimes(1)
expect(events.auth.SSOActivated).toBeCalledWith(Config.OIDC)
await config.deleteConfig(Config.OIDC)
})
it("should create deactivated OIDC config", async () => {
await context.doInTenant(config.tenant1User!.tenantId, async () => {
await saveOIDCConfig({ activated: false })
expect(events.auth.SSOCreated).toBeCalledTimes(1)
expect(events.auth.SSOCreated).toBeCalledWith(Config.OIDC)
expect(events.auth.SSOActivated).not.toBeCalled()
expect(events.auth.SSODeactivated).not.toBeCalled()
await config.deleteConfig(Config.OIDC)
})
await saveOIDCConfig({ activated: false })
expect(events.auth.SSOCreated).toBeCalledTimes(1)
expect(events.auth.SSOCreated).toBeCalledWith(Config.OIDC)
expect(events.auth.SSOActivated).not.toBeCalled()
expect(events.auth.SSODeactivated).not.toBeCalled()
await config.deleteConfig(Config.OIDC)
})
})
describe("update", () => {
it("should update OIDC config to deactivated", async () => {
await context.doInTenant(config.tenant1User!.tenantId, async () => {
const oidcConf = await saveOIDCConfig()
jest.clearAllMocks()
await saveOIDCConfig(
{ ...oidcConf.config.configs[0], activated: false },
oidcConf._id,
oidcConf._rev
)
expect(events.auth.SSOUpdated).toBeCalledTimes(1)
expect(events.auth.SSOUpdated).toBeCalledWith(Config.OIDC)
expect(events.auth.SSOActivated).not.toBeCalled()
expect(events.auth.SSODeactivated).toBeCalledTimes(1)
expect(events.auth.SSODeactivated).toBeCalledWith(Config.OIDC)
await config.deleteConfig(Config.OIDC)
})
const oidcConf = await saveOIDCConfig()
jest.clearAllMocks()
await saveOIDCConfig(
{ ...oidcConf.config.configs[0], activated: false },
oidcConf._id,
oidcConf._rev
)
expect(events.auth.SSOUpdated).toBeCalledTimes(1)
expect(events.auth.SSOUpdated).toBeCalledWith(Config.OIDC)
expect(events.auth.SSOActivated).not.toBeCalled()
expect(events.auth.SSODeactivated).toBeCalledTimes(1)
expect(events.auth.SSODeactivated).toBeCalledWith(Config.OIDC)
await config.deleteConfig(Config.OIDC)
})
it("should update OIDC config to activated", async () => {
await context.doInTenant(config.tenant1User!.tenantId, async () => {
const oidcConf = await saveOIDCConfig({ activated: false })
jest.clearAllMocks()
await saveOIDCConfig(
{ ...oidcConf.config.configs[0], activated: true },
oidcConf._id,
oidcConf._rev
)
expect(events.auth.SSOUpdated).toBeCalledTimes(1)
expect(events.auth.SSOUpdated).toBeCalledWith(Config.OIDC)
expect(events.auth.SSODeactivated).not.toBeCalled()
expect(events.auth.SSOActivated).toBeCalledTimes(1)
expect(events.auth.SSOActivated).toBeCalledWith(Config.OIDC)
await config.deleteConfig(Config.OIDC)
})
const oidcConf = await saveOIDCConfig({ activated: false })
jest.clearAllMocks()
await saveOIDCConfig(
{ ...oidcConf.config.configs[0], activated: true },
oidcConf._id,
oidcConf._rev
)
expect(events.auth.SSOUpdated).toBeCalledTimes(1)
expect(events.auth.SSOUpdated).toBeCalledWith(Config.OIDC)
expect(events.auth.SSODeactivated).not.toBeCalled()
expect(events.auth.SSOActivated).toBeCalledTimes(1)
expect(events.auth.SSOActivated).toBeCalledWith(Config.OIDC)
await config.deleteConfig(Config.OIDC)
})
})
})
@ -187,26 +179,22 @@ describe("configs", () => {
describe("create", () => {
it("should create SMTP config", async () => {
await context.doInTenant(config.tenant1User!.tenantId, async () => {
await config.deleteConfig(Config.SMTP)
await saveSMTPConfig()
expect(events.email.SMTPUpdated).not.toBeCalled()
expect(events.email.SMTPCreated).toBeCalledTimes(1)
await config.deleteConfig(Config.SMTP)
})
await config.deleteConfig(Config.SMTP)
await saveSMTPConfig()
expect(events.email.SMTPUpdated).not.toBeCalled()
expect(events.email.SMTPCreated).toBeCalledTimes(1)
await config.deleteConfig(Config.SMTP)
})
})
describe("update", () => {
it("should update SMTP config", async () => {
await context.doInTenant(config.tenant1User!.tenantId, async () => {
const smtpConf = await saveSMTPConfig()
jest.clearAllMocks()
await saveSMTPConfig(smtpConf.config, smtpConf._id, smtpConf._rev)
expect(events.email.SMTPCreated).not.toBeCalled()
expect(events.email.SMTPUpdated).toBeCalledTimes(1)
await config.deleteConfig(Config.SMTP)
})
const smtpConf = await saveSMTPConfig()
jest.clearAllMocks()
await saveSMTPConfig(smtpConf.config, smtpConf._id, smtpConf._rev)
expect(events.email.SMTPCreated).not.toBeCalled()
expect(events.email.SMTPUpdated).toBeCalledTimes(1)
await config.deleteConfig(Config.SMTP)
})
})
})
@ -223,73 +211,65 @@ describe("configs", () => {
describe("create", () => {
it("should create settings config with default settings", async () => {
await context.doInTenant(config.tenant1User!.tenantId, async () => {
await config.deleteConfig(Config.SETTINGS)
await config.deleteConfig(Config.SETTINGS)
await saveSettingsConfig()
await saveSettingsConfig()
expect(events.org.nameUpdated).not.toBeCalled()
expect(events.org.logoUpdated).not.toBeCalled()
expect(events.org.platformURLUpdated).not.toBeCalled()
})
expect(events.org.nameUpdated).not.toBeCalled()
expect(events.org.logoUpdated).not.toBeCalled()
expect(events.org.platformURLUpdated).not.toBeCalled()
})
it("should create settings config with non-default settings", async () => {
await context.doInTenant(config.tenant1User!.tenantId, async () => {
config.modeSelf()
await config.deleteConfig(Config.SETTINGS)
const conf = {
company: "acme",
logoUrl: "http://example.com",
platformUrl: "http://example.com",
}
config.selfHosted()
await config.deleteConfig(Config.SETTINGS)
const conf = {
company: "acme",
logoUrl: "http://example.com",
platformUrl: "http://example.com",
}
await saveSettingsConfig(conf)
await saveSettingsConfig(conf)
expect(events.org.nameUpdated).toBeCalledTimes(1)
expect(events.org.logoUpdated).toBeCalledTimes(1)
expect(events.org.platformURLUpdated).toBeCalledTimes(1)
config.modeCloud()
})
expect(events.org.nameUpdated).toBeCalledTimes(1)
expect(events.org.logoUpdated).toBeCalledTimes(1)
expect(events.org.platformURLUpdated).toBeCalledTimes(1)
config.cloudHosted()
})
})
describe("update", () => {
it("should update settings config", async () => {
await context.doInTenant(config.tenant1User!.tenantId, async () => {
config.modeSelf()
await config.deleteConfig(Config.SETTINGS)
const settingsConfig = await saveSettingsConfig()
settingsConfig.config.company = "acme"
settingsConfig.config.logoUrl = "http://example.com"
settingsConfig.config.platformUrl = "http://example.com"
config.selfHosted()
await config.deleteConfig(Config.SETTINGS)
const settingsConfig = await saveSettingsConfig()
settingsConfig.config.company = "acme"
settingsConfig.config.logoUrl = "http://example.com"
settingsConfig.config.platformUrl = "http://example.com"
await saveSettingsConfig(
settingsConfig.config,
settingsConfig._id,
settingsConfig._rev
)
await saveSettingsConfig(
settingsConfig.config,
settingsConfig._id,
settingsConfig._rev
)
expect(events.org.nameUpdated).toBeCalledTimes(1)
expect(events.org.logoUpdated).toBeCalledTimes(1)
expect(events.org.platformURLUpdated).toBeCalledTimes(1)
config.modeCloud()
})
expect(events.org.nameUpdated).toBeCalledTimes(1)
expect(events.org.logoUpdated).toBeCalledTimes(1)
expect(events.org.platformURLUpdated).toBeCalledTimes(1)
config.cloudHosted()
})
})
})
})
it("should return the correct checklist status based on the state of the budibase installation", async () => {
await context.doInTenant(config.tenant1User!.tenantId, async () => {
await config.saveSmtpConfig()
await config.saveSmtpConfig()
const res = await config.api.configs.getConfigChecklist()
const checklist = res.body
const res = await config.api.configs.getConfigChecklist()
const checklist = res.body
expect(checklist.apps.checked).toBeFalsy()
expect(checklist.smtp.checked).toBeTruthy()
expect(checklist.adminUser.checked).toBeTruthy()
})
expect(checklist.apps.checked).toBeFalsy()
expect(checklist.smtp.checked).toBeTruthy()
expect(checklist.adminUser.checked).toBeTruthy()
})
})

5
packages/worker/src/api/routes/global/tests/roles.spec.ts
View File

@ -32,6 +32,7 @@ async function addAppMetadata() {
describe("/api/global/roles", () => {
const config = new TestConfiguration()
const role = new roles.Role(
db.generateRoleID("newRole"),
roles.BUILTIN_ROLE_IDS.BASIC,
@ -43,13 +44,13 @@ describe("/api/global/roles", () => {
})
beforeEach(async () => {
appId = db.generateAppID()
appId = db.generateAppID(config.tenantId)
appDb = db.getDB(appId)
const mockAppDB = context.getAppDB as Mock
mockAppDB.mockReturnValue(appDb)
await addAppMetadata()
appDb.put(role)
await appDb.put(role)
})
afterAll(async () => {

84
packages/worker/src/api/routes/global/tests/users.spec.ts
View File

@ -3,7 +3,9 @@ import { InviteUsersResponse, User } from "@budibase/types"
jest.mock("nodemailer")
import { TestConfiguration, mocks, structures } from "../../../../tests"
const sendMailMock = mocks.email.mock()
import { context, events, tenancy } from "@budibase/backend-core"
import { events, tenancy, accounts as _accounts } from "@budibase/backend-core"
const accounts = jest.mocked(_accounts)
describe("/api/global/users", () => {
const config = new TestConfiguration()
@ -20,26 +22,24 @@ describe("/api/global/users", () => {
jest.clearAllMocks()
})
describe("invite", () => {
describe("POST /api/global/users/invite", () => {
it("should be able to generate an invitation", async () => {
await context.doInTenant(config.tenant1User!.tenantId, async () => {
const email = structures.users.newEmail()
const { code, res } = await config.api.users.sendUserInvite(
sendMailMock,
email
)
const email = structures.users.newEmail()
const { code, res } = await config.api.users.sendUserInvite(
sendMailMock,
email
)
expect(res.body).toEqual({ message: "Invitation has been sent." })
expect(sendMailMock).toHaveBeenCalled()
expect(code).toBeDefined()
expect(events.user.invited).toBeCalledTimes(1)
})
expect(res.body).toEqual({ message: "Invitation has been sent." })
expect(sendMailMock).toHaveBeenCalled()
expect(code).toBeDefined()
expect(events.user.invited).toBeCalledTimes(1)
})
it("should not be able to generate an invitation for existing user", async () => {
const { code, res } = await config.api.users.sendUserInvite(
sendMailMock,
config.defaultUser!.email,
config.user!.email,
400
)
@ -50,26 +50,24 @@ describe("/api/global/users", () => {
})
it("should be able to create new user from invite", async () => {
await context.doInTenant(config.tenant1User!.tenantId, async () => {
const email = structures.users.newEmail()
const { code } = await config.api.users.sendUserInvite(
sendMailMock,
email
)
const email = structures.users.newEmail()
const { code } = await config.api.users.sendUserInvite(
sendMailMock,
email
)
const res = await config.api.users.acceptInvite(code)
const res = await config.api.users.acceptInvite(code)
expect(res.body._id).toBeDefined()
const user = await config.getUser(email)
expect(user).toBeDefined()
expect(user._id).toEqual(res.body._id)
expect(events.user.inviteAccepted).toBeCalledTimes(1)
expect(events.user.inviteAccepted).toBeCalledWith(user)
})
expect(res.body._id).toBeDefined()
const user = await config.getUser(email)
expect(user).toBeDefined()
expect(user._id).toEqual(res.body._id)
expect(events.user.inviteAccepted).toBeCalledTimes(1)
expect(events.user.inviteAccepted).toBeCalledWith(user)
})
})
describe("inviteMultiple", () => {
describe("POST /api/global/users/multi/invite", () => {
it("should be able to generate an invitation", async () => {
const newUserInvite = () => ({
email: structures.users.newEmail(),
@ -87,7 +85,7 @@ describe("/api/global/users", () => {
})
it("should not be able to generate an invitation for existing user", async () => {
const request = [{ email: config.defaultUser!.email, userInfo: {} }]
const request = [{ email: config.user!.email, userInfo: {} }]
const res = await config.api.users.sendMultiUserInvite(request)
@ -100,7 +98,7 @@ describe("/api/global/users", () => {
})
})
describe("bulk (create)", () => {
describe("POST /api/global/users/bulk", () => {
it("should ignore users existing in the same tenant", async () => {
const user = await config.createUser()
jest.clearAllMocks()
@ -163,7 +161,7 @@ describe("/api/global/users", () => {
})
})
describe("create", () => {
describe("POST /api/global/users", () => {
it("should be able to create a basic user", async () => {
const user = structures.users.user()
@ -242,7 +240,7 @@ describe("/api/global/users", () => {
it("should not be able to create user with the same email as an account", async () => {
const user = structures.users.user()
const account = structures.accounts.cloudAccount()
mocks.accounts.getAccount.mockReturnValueOnce(account)
accounts.getAccount.mockReturnValueOnce(Promise.resolve(account))
const response = await config.api.users.saveUser(user, 400)
@ -283,7 +281,7 @@ describe("/api/global/users", () => {
})
})
describe("update", () => {
describe("POST /api/global/users (update)", () => {
it("should be able to update a basic user", async () => {
const user = await config.createUser()
jest.clearAllMocks()
@ -298,7 +296,7 @@ describe("/api/global/users", () => {
})
it("should not allow a user to update their own admin/builder status", async () => {
const user = (await config.api.users.getUser(config.defaultUser?._id!))
const user = (await config.api.users.getUser(config.user?._id!))
.body as User
await config.api.users.saveUser({
...user,
@ -468,9 +466,9 @@ describe("/api/global/users", () => {
})
})
describe("bulk (delete)", () => {
describe("POST /api/global/users/bulk (delete)", () => {
it("should not be able to bulk delete current user", async () => {
const user = await config.defaultUser!
const user = await config.user!
const response = await config.api.users.bulkDeleteUsers([user._id!], 400)
@ -482,7 +480,7 @@ describe("/api/global/users", () => {
const user = await config.createUser()
const account = structures.accounts.cloudAccount()
account.budibaseUserId = user._id!
mocks.accounts.getAccountByTenantId.mockReturnValue(account)
accounts.getAccountByTenantId.mockReturnValue(Promise.resolve(account))
const response = await config.api.users.bulkDeleteUsers([user._id!])
@ -497,7 +495,7 @@ describe("/api/global/users", () => {
it("should be able to bulk delete users", async () => {
const account = structures.accounts.cloudAccount()
mocks.accounts.getAccountByTenantId.mockReturnValue(account)
accounts.getAccountByTenantId.mockReturnValue(Promise.resolve(account))
const builder = structures.users.builderUser()
const admin = structures.users.adminUser()
@ -521,7 +519,7 @@ describe("/api/global/users", () => {
})
})
describe("destroy", () => {
describe("DELETE /api/global/users/:userId", () => {
it("should be able to destroy a basic user", async () => {
const user = await config.createUser()
jest.clearAllMocks()
@ -558,7 +556,7 @@ describe("/api/global/users", () => {
it("should not be able to destroy account owner", async () => {
const user = await config.createUser()
const account = structures.accounts.cloudAccount()
mocks.accounts.getAccount.mockReturnValueOnce(account)
accounts.getAccount.mockReturnValueOnce(Promise.resolve(account))
const response = await config.api.users.deleteUser(user._id!, 400)
@ -566,10 +564,10 @@ describe("/api/global/users", () => {
})
it("should not be able to destroy account owner as account owner", async () => {
const user = await config.defaultUser!
const user = await config.user!
const account = structures.accounts.cloudAccount()
account.email = user.email
mocks.accounts.getAccount.mockReturnValueOnce(account)
accounts.getAccount.mockReturnValueOnce(Promise.resolve(account))
const response = await config.api.users.deleteUser(user._id!, 400)

2
packages/worker/src/api/routes/system/tenants.ts
View File

@ -7,7 +7,7 @@ const router: Router = new Router()
router.delete(
"/api/system/tenants/:tenantId",
middleware.adminOnly,
controller.delete
controller.destroy
)
export default router

4
packages/worker/src/api/routes/system/tests/restore.spec.ts
View File

@ -25,12 +25,12 @@ describe("/api/system/restore", () => {
})
it("restores in self host", async () => {
config.modeSelf()
config.selfHosted()
const res = await config.api.restore.restored()
expect(res.body).toEqual({
message: "System prepared after restore.",
})
config.modeCloud()
config.cloudHosted()
})
})
})

10
packages/worker/src/api/routes/system/tests/status.spec.ts
View File

@ -1,6 +1,6 @@
import { TestConfiguration } from "../../../../tests"
import { accounts } from "@budibase/backend-core"
import { mocks } from "@budibase/backend-core/tests"
import { accounts as _accounts } from "@budibase/backend-core"
const accounts = jest.mocked(_accounts)
describe("/api/system/status", () => {
const config = new TestConfiguration()
@ -19,7 +19,7 @@ describe("/api/system/status", () => {
describe("GET /api/system/status", () => {
it("returns status in self host", async () => {
config.modeSelf()
config.selfHosted()
const res = await config.api.status.getStatus()
expect(res.body).toEqual({
health: {
@ -27,7 +27,7 @@ describe("/api/system/status", () => {
},
})
expect(accounts.getStatus).toBeCalledTimes(0)
config.modeCloud()
config.cloudHosted()
})
it("returns status in cloud", async () => {
@ -37,7 +37,7 @@ describe("/api/system/status", () => {
},
}
mocks.accounts.getStatus.mockReturnValueOnce(value)
accounts.getStatus.mockReturnValueOnce(Promise.resolve(value))
const res = await config.api.status.getStatus()

7
packages/worker/src/migrations/functions/globalInfoSyncUsers.ts
View File

@ -1,5 +1,6 @@
import { User } from "@budibase/types"
import sdk from "../../sdk"
import * as usersSdk from "../../sdk/users"
import { platform } from "@budibase/backend-core"
/**
* Date:
@ -9,11 +10,11 @@ import sdk from "../../sdk"
* Re-sync the global-db users to the global-info db users
*/
export const run = async (globalDb: any) => {
const users = (await sdk.users.allUsers()) as User[]
const users = (await usersSdk.allUsers()) as User[]
const promises = []
for (let user of users) {
promises.push(
sdk.users.addTenant(user.tenantId, user._id as string, user.email)
platform.users.addUser(user.tenantId, user._id as string, user.email)
)
}
await Promise.all(promises)

1
packages/worker/src/sdk/tenants/index.ts Normal file
View File

@ -0,0 +1 @@
export * from "./tenants"

76
packages/worker/src/sdk/tenants/tenants.ts Normal file
View File

@ -0,0 +1,76 @@
import { App } from "@budibase/types"
import { tenancy, db as dbCore, platform } from "@budibase/backend-core"
import { quotas } from "@budibase/pro"
export async function deleteTenant(tenantId: string) {
await quotas.bustCache()
await platform.tenants.removeTenant(tenantId)
await removeGlobalDB(tenantId)
await removeTenantUsers(tenantId)
await removeTenantApps(tenantId)
}
async function removeGlobalDB(tenantId: string) {
try {
const db = tenancy.getTenantDB(tenantId)
await db.destroy()
} catch (err) {
console.error(`Error removing tenant ${tenantId} users from info db`, err)
throw err
}
}
async function removeTenantApps(tenantId: string) {
try {
const apps = (await dbCore.getAllApps({ all: true })) as App[]
const destroyPromises = apps.map(app => {
const db = dbCore.getDB(app.appId)
return db.destroy()
})
await Promise.allSettled(destroyPromises)
} catch (err) {
console.error(`Error removing tenant ${tenantId} apps`, err)
throw err
}
}
function getTenantUsers(tenantId: string) {
const db = tenancy.getTenantDB(tenantId)
return db.allDocs(
dbCore.getGlobalUserParams(null, {
include_docs: true,
})
)
}
async function removeTenantUsers(tenantId: string) {
try {
const allUsers = await getTenantUsers(tenantId)
const allEmails = allUsers.rows.map((row: any) => row.doc.email)
// get the id and email doc ids
let keys = allUsers.rows.map((row: any) => row.id)
keys = keys.concat(allEmails)
const platformDb = platform.getPlatformDB()
// retrieve the docs
const userDocs = await platformDb.allDocs({
keys,
include_docs: true,
})
// delete the docs
const toDelete = userDocs.rows.map((row: any) => {
return {
...row.doc,
_deleted: true,
}
})
await platformDb.bulkDocs(toDelete)
} catch (err) {
console.error(`Error removing tenant ${tenantId} users from info db`, err)
throw err
}
}

2
packages/worker/src/sdk/users/users.ts
View File

@ -549,7 +549,7 @@ export const bulkDelete = async (
export const destroy = async (id: string, currentUser: any) => {
const db = tenancy.getGlobalDB()
const dbUser = await db.get(id)
const dbUser = (await db.get(id)) as User
const userId = dbUser._id as string
if (!env.SELF_HOSTED && !env.DISABLE_ACCOUNT_PORTAL) {

121
packages/worker/src/tests/TestConfiguration.ts
View File

@ -15,61 +15,29 @@ const supertest = require("supertest")
import { Config } from "../constants"
import {
users,
tenancy,
context,
sessions,
auth,
constants,
env as coreEnv,
DEFAULT_TENANT_ID,
} from "@budibase/backend-core"
import structures, { TENANT_ID, CSRF_TOKEN } from "./structures"
import structures, { CSRF_TOKEN } from "./structures"
import { CreateUserResponse, User, AuthToken } from "@budibase/types"
import API from "./api"
import sdk from "../sdk"
enum Mode {
CLOUD = "cloud",
SELF = "self",
}
async function retry<T extends (...arg0: any[]) => any>(
fn: T,
maxTry: number = 5,
retryCount = 1
): Promise<Awaited<ReturnType<T>>> {
const currRetry = typeof retryCount === "number" ? retryCount : 1
try {
const result = await fn()
return result
} catch (e) {
console.log(`Retry ${currRetry} failed.`)
if (currRetry > maxTry) {
console.log(`All ${maxTry} retry attempts exhausted`)
throw e
}
return retry(fn, maxTry, currRetry + 1)
}
}
class TestConfiguration {
server: any
request: any
api: API
defaultUser?: User
tenant1User?: User
#tenantId?: string
tenantId: string
user?: User
userPassword = "test"
constructor(
opts: { openServer: boolean; mode: Mode } = {
openServer: true,
mode: Mode.CLOUD,
}
) {
if (opts.mode === Mode.CLOUD) {
this.modeCloud()
} else if (opts.mode === Mode.SELF) {
this.modeSelf()
}
constructor(opts: { openServer: boolean } = { openServer: true }) {
// default to cloud hosting
this.cloudHosted()
this.tenantId = structures.tenant.id()
if (opts.openServer) {
env.PORT = "0" // random port
@ -85,26 +53,19 @@ class TestConfiguration {
return this.request
}
// MODES
setMultiTenancy = (value: boolean) => {
env._set("MULTI_TENANCY", value)
coreEnv._set("MULTI_TENANCY", value)
}
// HOSTING
setSelfHosted = (value: boolean) => {
env._set("SELF_HOSTED", value)
coreEnv._set("SELF_HOSTED", value)
}
modeCloud = () => {
cloudHosted = () => {
this.setSelfHosted(false)
this.setMultiTenancy(true)
}
modeSelf = () => {
selfHosted = () => {
this.setSelfHosted(true)
this.setMultiTenancy(false)
}
// UTILS
@ -125,7 +86,7 @@ class TestConfiguration {
if (params) {
request.params = params
}
await tenancy.doInTenant(this.getTenantId(), () => {
await context.doInTenant(this.getTenantId(), () => {
return controlFunc(request)
})
return request.body
@ -135,18 +96,10 @@ class TestConfiguration {
async beforeAll() {
try {
this.#tenantId = structures.tenant.id()
// Running tests in parallel causes issues creating the globaldb twice. This ensures the db is properly created before starting
await retry(async () => await this.createDefaultUser())
await this.createSession(this.defaultUser!)
await tenancy.doInTenant(this.#tenantId, async () => {
await this.createTenant1User()
await this.createSession(this.tenant1User!)
})
await this.createDefaultUser()
await this.createSession(this.user!)
} catch (e: any) {
console.log(e)
console.error(e)
throw new Error(e.message)
}
}
@ -159,12 +112,16 @@ class TestConfiguration {
// TENANCY
doInTenant(task: any) {
return context.doInTenant(this.tenantId, () => {
return task()
})
}
createTenant = async (): Promise<User> => {
// create user / new tenant
const res = await this.api.users.createAdminUser()
await sdk.users.addTenant(res.tenantId, res.userId, res.email)
// return the created user
const userRes = await this.api.users.getUser(res.userId, {
headers: {
@ -182,9 +139,9 @@ class TestConfiguration {
getTenantId() {
try {
return tenancy.getTenantId()
} catch (e: any) {
return DEFAULT_TENANT_ID
return context.getTenantId()
} catch (e) {
return this.tenantId!
}
}
@ -232,14 +189,11 @@ class TestConfiguration {
}
defaultHeaders() {
const tenantId = this.getTenantId()
if (tenantId === TENANT_ID) {
return this.authHeaders(this.defaultUser!)
} else if (tenantId === this.getTenantId()) {
return this.authHeaders(this.tenant1User!)
} else {
throw new Error("could not determine auth headers to use")
}
return this.authHeaders(this.user!)
}
tenantIdHeaders() {
return { [constants.Header.TENANT_ID]: this.tenantId }
}
internalAPIHeaders() {
@ -254,20 +208,15 @@ class TestConfiguration {
async createDefaultUser() {
const user = structures.users.adminUser({
password: "test",
password: this.userPassword,
})
this.defaultUser = await this.createUser(user)
}
async createTenant1User() {
const user = structures.users.adminUser({
password: "test",
await context.doInTenant(this.tenantId!, async () => {
this.user = await this.createUser(user)
})
this.tenant1User = await this.createUser(user)
}
async getUser(email: string): Promise<User> {
return tenancy.doInTenant(this.getTenantId(), () => {
return context.doInTenant(this.getTenantId(), () => {
return users.getGlobalUserByEmail(email)
})
}

3
packages/worker/src/tests/api/base.ts
View File

@ -1,4 +1,5 @@
import TestConfiguration from "../TestConfiguration"
import { SuperTest, Test } from "supertest"
export interface TestAPIOpts {
headers?: any
@ -7,7 +8,7 @@ export interface TestAPIOpts {
export abstract class TestAPI {
config: TestConfiguration
request: any
request: SuperTest<Test>
protected constructor(config: TestConfiguration) {
this.config = config

1
packages/worker/src/tests/api/restore.ts
View File

@ -9,6 +9,7 @@ export class RestoreAPI extends TestAPI {
restored = (opts?: TestAPIOpts) => {
return this.request
.post(`/api/system/restored`)
.set(this.config.tenantIdHeaders())
.expect(opts?.status ? opts.status : 200)
}
}

110
packages/worker/yarn.lock
View File

@ -800,17 +800,6 @@
slash "^3.0.0"
write-file-atomic "^4.0.1"
"@jest/types@^26.6.2":
version "26.6.2"
resolved "https://registry.yarnpkg.com/@jest/types/-/types-26.6.2.tgz#bef5a532030e1d88a2f5a6d933f84e97226ed48e"
integrity sha512-fC6QCp7Sc5sX6g8Tvbmj4XUTbyrik0akgRy03yjXbQaBWWNWGE7SGtJk98m0N8nzegD/7SggrUlivxo5ax4KWQ==
dependencies:
"@types/istanbul-lib-coverage" "^2.0.0"
"@types/istanbul-reports" "^3.0.0"
"@types/node" "*"
"@types/yargs" "^15.0.0"
chalk "^4.0.0"
"@jest/types@^27.5.1":
version "27.5.1"
resolved "https://registry.yarnpkg.com/@jest/types/-/types-27.5.1.tgz#3c79ec4a8ba61c170bf937bcf9e98a9df175ec80"
@ -1317,6 +1306,11 @@
resolved "https://registry.yarnpkg.com/@types/content-disposition/-/content-disposition-0.5.5.tgz#650820e95de346e1f84e30667d168c8fd25aa6e3"
integrity sha512-v6LCdKfK6BwcqMo+wYW05rLS12S0ZO0Fl4w1h4aaZMD7bqT3gVUns6FvLJKGZHQmYn3SX55JWGpziwJRwVgutA==
"@types/cookiejar@*":
version "2.1.2"
resolved "https://registry.yarnpkg.com/@types/cookiejar/-/cookiejar-2.1.2.tgz#66ad9331f63fe8a3d3d9d8c6e3906dd10f6446e8"
integrity sha512-t73xJJrvdTjXrn4jLS9VSGRbz0nUY3cl2DMGDU48lKl+HR9dbbjW2A9r3g40VA++mQpy6uuHg33gy7du2BKpog==
"@types/cookies@*":
version "0.7.7"
resolved "https://registry.yarnpkg.com/@types/cookies/-/cookies-0.7.7.tgz#7a92453d1d16389c05a5301eef566f34946cfd81"
@ -1413,13 +1407,13 @@
dependencies:
"@types/istanbul-lib-report" "*"
"@types/jest@26.0.23":
version "26.0.23"
resolved "https://registry.yarnpkg.com/@types/jest/-/jest-26.0.23.tgz#a1b7eab3c503b80451d019efb588ec63522ee4e7"
integrity sha512-ZHLmWMJ9jJ9PTiT58juykZpL7KjwJywFN3Rr2pTSkyQfydf/rk22yS7W8p5DaVUMQ2BQC7oYiU3FjbTM/mYrOA==
"@types/jest@28.1.1":
version "28.1.1"
resolved "https://registry.yarnpkg.com/@types/jest/-/jest-28.1.1.tgz#8c9ba63702a11f8c386ee211280e8b68cb093cd1"
integrity sha512-C2p7yqleUKtCkVjlOur9BWVA4HgUQmEj/HWCt5WzZ5mLXrWnyIfl0wGuArc+kBXsy0ZZfLp+7dywB4HtSVYGVA==
dependencies:
jest-diff "^26.0.0"
pretty-format "^26.0.0"
jest-matcher-utils "^27.0.0"
pretty-format "^27.0.0"
"@types/json-buffer@~3.0.0":
version "3.0.0"
@ -1689,6 +1683,21 @@
resolved "https://registry.yarnpkg.com/@types/stack-utils/-/stack-utils-2.0.1.tgz#20f18294f797f2209b5f65c8e3b5c8e8261d127c"
integrity sha512-Hl219/BT5fLAaz6NDkSuhzasy49dwQS/DSdu4MdggFB8zcXv7vflBI3xp7FEmkmdDkBUI2bPUNeMttp2knYdxw==
"@types/superagent@*":
version "4.1.16"
resolved "https://registry.yarnpkg.com/@types/superagent/-/superagent-4.1.16.tgz#12c9c16f232f9d89beab91d69368f96ce8e2d881"
integrity sha512-tLfnlJf6A5mB6ddqF159GqcDizfzbMUB1/DeT59/wBNqzRTNNKsaw79A/1TZ84X+f/EwWH8FeuSkjlCLyqS/zQ==
dependencies:
"@types/cookiejar" "*"
"@types/node" "*"
"@types/supertest@2.0.12":
version "2.0.12"
resolved "https://registry.yarnpkg.com/@types/supertest/-/supertest-2.0.12.tgz#ddb4a0568597c9aadff8dbec5b2e8fddbe8692fc"
integrity sha512-X3HPWTwXRerBZS7Mo1k6vMVR1Z6zmJcDVn5O/31whe0tnjE4te6ZJSJGq1RiqHPjzPdMTfjCFogDJmwng9xHaQ==
dependencies:
"@types/superagent" "*"
"@types/tough-cookie@^4.0.2":
version "4.0.2"
resolved "https://registry.yarnpkg.com/@types/tough-cookie/-/tough-cookie-4.0.2.tgz#6286b4c7228d58ab7866d19716f3696e03a09397"
@ -1704,13 +1713,6 @@
resolved "https://registry.yarnpkg.com/@types/yargs-parser/-/yargs-parser-21.0.0.tgz#0c60e537fa790f5f9472ed2776c2b71ec117351b"
integrity sha512-iO9ZQHkZxHn4mSakYV0vFHAVDyEOIJQrV2uZ06HxEPcx+mt8swXoZHIbaaJ2crJYFfErySgktuTZ3BeLz+XmFA==
"@types/yargs@^15.0.0":
version "15.0.14"
resolved "https://registry.yarnpkg.com/@types/yargs/-/yargs-15.0.14.tgz#26d821ddb89e70492160b66d10a0eb6df8f6fb06"
integrity sha512-yEJzHoxf6SyQGhBhIYGXQDSCkJjB6HohDShto7m8vaKg9Yp0Yn8+71J9eakh2bnPg6BfsH9PRMhiRTZnd4eXGQ==
dependencies:
"@types/yargs-parser" "*"
"@types/yargs@^16.0.0":
version "16.0.5"
resolved "https://registry.yarnpkg.com/@types/yargs/-/yargs-16.0.5.tgz#12cc86393985735a283e387936398c2f9e5f88e3"
@ -1898,7 +1900,7 @@ ansi-regex@^4.1.0:
resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-4.1.1.tgz#164daac87ab2d6f6db3a29875e2d1766582dabed"
integrity sha512-ILlv4k/3f6vfQ4OoP2AGvirOktlQ98ZEL1k9FaQjxa3L1abBgbuTDAdPOpvbGncC0BTVQrl+OM8xZGK6tWXt7g==
ansi-regex@^5.0.0, ansi-regex@^5.0.1:
ansi-regex@^5.0.1:
version "5.0.1"
resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-5.0.1.tgz#082cb2c89c9fe8659a311a53bd6a4dc5301db304"
integrity sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==
@ -2999,10 +3001,10 @@ dezalgo@1.0.3:
asap "^2.0.0"
wrappy "1"
diff-sequences@^26.6.2:
version "26.6.2"
resolved "https://registry.yarnpkg.com/diff-sequences/-/diff-sequences-26.6.2.tgz#48ba99157de1923412eed41db6b6d4aa9ca7c0b1"
integrity sha512-Mv/TDa3nZ9sbc5soK+OoA74BsS3mL37yixCvUAQkiuA4Wz6YtwP/K47n2rv2ovzHZvoiQeA5FTQOschKkEwB0Q==
diff-sequences@^27.5.1:
version "27.5.1"
resolved "https://registry.yarnpkg.com/diff-sequences/-/diff-sequences-27.5.1.tgz#eaecc0d327fd68c8d9672a1e64ab8dccb2ef5327"
integrity sha512-k1gCAXAsNgLwEL+Y8Wvl+M6oEFj5bgazfZULpS5CneoPPXRaCCW7dm+q21Ky2VEE5X+VeRDBVg1Pcvvsr4TtNQ==
diff-sequences@^28.1.1:
version "28.1.1"
@ -3066,11 +3068,6 @@ dotenv@16.0.1:
resolved "https://registry.yarnpkg.com/dotenv/-/dotenv-16.0.1.tgz#8f8f9d94876c35dac989876a5d3a82a267fdce1d"
integrity sha512-1K6hR6wtk2FviQ4kEiSjFiH5rpzEVi8WW0x96aztHVMhEspNpc4DVOUTEHtEva5VThQ8IaBX1Pe4gSzpVVUsKQ==
dotenv@8.6.0:
version "8.6.0"
resolved "https://registry.yarnpkg.com/dotenv/-/dotenv-8.6.0.tgz#061af664d19f7f4d8fc6e4ff9b584ce237adcb8b"
integrity sha512-IrPdXQsk2BbzvCBGBOTmmSH5SodmqZNt4ERAZDmW4CT+tL8VtvinqywuANaFu4bOMWki16nqf0e4oC0QIaDr/g==
double-ended-queue@2.1.0-0:
version "2.1.0-0"
resolved "https://registry.yarnpkg.com/double-ended-queue/-/double-ended-queue-2.1.0-0.tgz#103d3527fd31528f40188130c841efdd78264e5c"
@ -4727,15 +4724,15 @@ jest-config@^28.1.3:
slash "^3.0.0"
strip-json-comments "^3.1.1"
jest-diff@^26.0.0:
version "26.6.2"
resolved "https://registry.yarnpkg.com/jest-diff/-/jest-diff-26.6.2.tgz#1aa7468b52c3a68d7d5c5fdcdfcd5e49bd164394"
integrity sha512-6m+9Z3Gv9wN0WFVasqjCL/06+EFCMTqDEUl/b87HYK2rAPTyfz4ZIuSlPhY51PIQRWx5TaxeF1qmXKe9gfN3sA==
jest-diff@^27.5.1:
version "27.5.1"
resolved "https://registry.yarnpkg.com/jest-diff/-/jest-diff-27.5.1.tgz#a07f5011ac9e6643cf8a95a462b7b1ecf6680def"
integrity sha512-m0NvkX55LDt9T4mctTEgnZk3fmEg3NRYutvMPWM/0iPnkFj2wIeF45O1718cMSOFO1vINkqmxqD8vE37uTEbqw==
dependencies:
chalk "^4.0.0"
diff-sequences "^26.6.2"
jest-get-type "^26.3.0"
pretty-format "^26.6.2"
diff-sequences "^27.5.1"
jest-get-type "^27.5.1"
pretty-format "^27.5.1"
jest-diff@^28.1.3:
version "28.1.3"
@ -4777,10 +4774,10 @@ jest-environment-node@^28.1.3:
jest-mock "^28.1.3"
jest-util "^28.1.3"
jest-get-type@^26.3.0:
version "26.3.0"
resolved "https://registry.yarnpkg.com/jest-get-type/-/jest-get-type-26.3.0.tgz#e97dc3c3f53c2b406ca7afaed4493b1d099199e0"
integrity sha512-TpfaviN1R2pQWkIihlfEanwOXK0zcxrKEE4MlU6Tn7keoXdN6/3gK/xl0yEh8DOunn5pOVGKf8hB4R9gVh04ig==
jest-get-type@^27.5.1:
version "27.5.1"
resolved "https://registry.yarnpkg.com/jest-get-type/-/jest-get-type-27.5.1.tgz#3cd613c507b0f7ace013df407a1c1cd578bcb4f1"
integrity sha512-2KY95ksYSaK7DMBWQn6dQz3kqAf3BB64y2udeG+hv4KfSOb9qwcYQstTJc1KCbsix+wLZWZYN8t7nwX3GOBLRw==
jest-get-type@^28.0.2:
version "28.0.2"
@ -4814,6 +4811,16 @@ jest-leak-detector@^28.1.3:
jest-get-type "^28.0.2"
pretty-format "^28.1.3"
jest-matcher-utils@^27.0.0:
version "27.5.1"
resolved "https://registry.yarnpkg.com/jest-matcher-utils/-/jest-matcher-utils-27.5.1.tgz#9c0cdbda8245bc22d2331729d1091308b40cf8ab"
integrity sha512-z2uTx/T6LBaCoNWNFWwChLBKYxTMcGBRjAt+2SbP929/Fflb9aa5LGma654Rz8z9HLxsrUaYzxE9T/EFIL/PAw==
dependencies:
chalk "^4.0.0"
jest-diff "^27.5.1"
jest-get-type "^27.5.1"
pretty-format "^27.5.1"
jest-matcher-utils@^28.1.3:
version "28.1.3"
resolved "https://registry.yarnpkg.com/jest-matcher-utils/-/jest-matcher-utils-28.1.3.tgz#5a77f1c129dd5ba3b4d7fc20728806c78893146e"
@ -6624,14 +6631,13 @@ prettier@2.3.1:
resolved "https://registry.yarnpkg.com/prettier/-/prettier-2.3.1.tgz#76903c3f8c4449bc9ac597acefa24dc5ad4cbea6"
integrity sha512-p+vNbgpLjif/+D+DwAZAbndtRrR0md0MwfmOVN9N+2RgyACMT+7tfaRnT+WDPkqnuVwleyuBIG2XBxKDme3hPA==
pretty-format@^26.0.0, pretty-format@^26.6.2:
version "26.6.2"
resolved "https://registry.yarnpkg.com/pretty-format/-/pretty-format-26.6.2.tgz#e35c2705f14cb7fe2fe94fa078345b444120fc93"
integrity sha512-7AeGuCYNGmycyQbCqd/3PWH4eOoX/OiCa0uphp57NVTeAGdJGaAliecxwBDHYQCIvrW7aDBZCYeNTP/WX69mkg==
pretty-format@^27.0.0, pretty-format@^27.5.1:
version "27.5.1"
resolved "https://registry.yarnpkg.com/pretty-format/-/pretty-format-27.5.1.tgz#2181879fdea51a7a5851fb39d920faa63f01d88e"
integrity sha512-Qb1gy5OrP5+zDf2Bvnzdl3jsTf1qXVMazbvCoKhtKqVs4/YK4ozX4gKQJJVyNe+cajNPn0KoC0MC3FUmaHWEmQ==
dependencies:
"@jest/types" "^26.6.2"
ansi-regex "^5.0.0"
ansi-styles "^4.0.0"
ansi-regex "^5.0.1"
ansi-styles "^5.0.0"
react-is "^17.0.1"
pretty-format@^28.1.3: