MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #6705 from Budibase/fix/memory-leak 

Memory leak fix
This commit is contained in:
Michael Drury 2022-07-14 19:22:26 +01:00 committed by GitHub
parent 21f6197df0 fcfe1341a1
commit 328f6951d8
19 changed files with 3191 additions and 2767 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
packages/backend-core/package.json
View File

@ -70,6 +70,7 @@
"@types/semver": "7.3.7", "@types/semver": "7.3.7",
"@types/tar-fs": "2.0.1", "@types/tar-fs": "2.0.1",
"@types/uuid": "8.3.4", "@types/uuid": "8.3.4",
"@types/lodash": "4.14.180",
"ioredis-mock": "5.8.0", "ioredis-mock": "5.8.0",
"jest": "27.5.1", "jest": "27.5.1",
"koa": "2.7.0", "koa": "2.7.0",

17
packages/backend-core/src/context/constants.ts Normal file
View File

@ -0,0 +1,17 @@
export enum ContextKeys {
TENANT_ID = "tenantId",
GLOBAL_DB = "globalDb",
APP_ID = "appId",
IDENTITY = "identity",
// whatever the request app DB was
CURRENT_DB = "currentDb",
// get the prod app DB from the request
PROD_DB = "prodDb",
// get the dev app DB from the request
DEV_DB = "devDb",
DB_OPTS = "dbOpts",
// check if something else is using the context, don't close DB
TENANCY_IN_USE = "tenancyInUse",
APP_IN_USE = "appInUse",
IDENTITY_IN_USE = "identityInUse",
}

354
packages/backend-core/src/context/index.js
View File

@ -1,354 +0,0 @@
const env = require("../environment")
const { SEPARATOR, DocumentTypes } = require("../db/constants")
const { DEFAULT_TENANT_ID } = require("../constants")
const cls = require("./FunctionContext")
const { dangerousGetDB, closeDB } = require("../db")
const { getProdAppID, getDevelopmentAppID } = require("../db/conversions")
const { baseGlobalDBName } = require("../tenancy/utils")
const { isEqual } = require("lodash")
// some test cases call functions directly, need to
// store an app ID to pretend there is a context
let TEST_APP_ID = null
const ContextKeys = {
TENANT_ID: "tenantId",
GLOBAL_DB: "globalDb",
APP_ID: "appId",
IDENTITY: "identity",
// whatever the request app DB was
CURRENT_DB: "currentDb",
// get the prod app DB from the request
PROD_DB: "prodDb",
// get the dev app DB from the request
DEV_DB: "devDb",
DB_OPTS: "dbOpts",
// check if something else is using the context, don't close DB
IN_USE: "inUse",
}
exports.DEFAULT_TENANT_ID = DEFAULT_TENANT_ID
// this function makes sure the PouchDB objects are closed and
// fully deleted when finished - this protects against memory leaks
async function closeAppDBs() {
const dbKeys = [
ContextKeys.CURRENT_DB,
ContextKeys.PROD_DB,
ContextKeys.DEV_DB,
]
for (let dbKey of dbKeys) {
const db = cls.getFromContext(dbKey)
if (!db) {
continue
}
await closeDB(db)
// clear the DB from context, incase someone tries to use it again
cls.setOnContext(dbKey, null)
}
// clear the app ID now that the databases are closed
if (cls.getFromContext(ContextKeys.APP_ID)) {
cls.setOnContext(ContextKeys.APP_ID, null)
}
if (cls.getFromContext(ContextKeys.DB_OPTS)) {
cls.setOnContext(ContextKeys.DB_OPTS, null)
}
}
exports.closeTenancy = async () => {
if (env.USE_COUCH) {
await closeDB(exports.getGlobalDB())
}
// clear from context now that database is closed/task is finished
cls.setOnContext(ContextKeys.TENANT_ID, null)
cls.setOnContext(ContextKeys.GLOBAL_DB, null)
}
exports.isDefaultTenant = () => {
return exports.getTenantId() === exports.DEFAULT_TENANT_ID
}
exports.isMultiTenant = () => {
return env.MULTI_TENANCY
}
// used for automations, API endpoints should always be in context already
exports.doInTenant = (tenantId, task, { forceNew } = {}) => {
// the internal function is so that we can re-use an existing
// context - don't want to close DB on a parent context
async function internal(opts = { existing: false }) {
// set the tenant id
if (!opts.existing) {
exports.updateTenantId(tenantId)
}
try {
// invoke the task
return await task()
} finally {
const using = cls.getFromContext(ContextKeys.IN_USE)
if (!using || using <= 1) {
await exports.closeTenancy()
} else {
cls.setOnContext(using - 1)
}
}
}
const using = cls.getFromContext(ContextKeys.IN_USE)
if (
!forceNew &&
using &&
cls.getFromContext(ContextKeys.TENANT_ID) === tenantId
) {
cls.setOnContext(ContextKeys.IN_USE, using + 1)
return internal({ existing: true })
} else {
return cls.run(async () => {
cls.setOnContext(ContextKeys.IN_USE, 1)
return internal()
})
}
}
/**
* Given an app ID this will attempt to retrieve the tenant ID from it.
* @return {null|string} The tenant ID found within the app ID.
*/
exports.getTenantIDFromAppID = appId => {
if (!appId) {
return null
}
const split = appId.split(SEPARATOR)
const hasDev = split[1] === DocumentTypes.DEV
if ((hasDev && split.length === 3) || (!hasDev && split.length === 2)) {
return null
}
if (hasDev) {
return split[2]
} else {
return split[1]
}
}
const setAppTenantId = appId => {
const appTenantId =
exports.getTenantIDFromAppID(appId) || exports.DEFAULT_TENANT_ID
exports.updateTenantId(appTenantId)
}
exports.doInAppContext = (appId, task, { forceNew } = {}) => {
if (!appId) {
throw new Error("appId is required")
}
const identity = exports.getIdentity()
// the internal function is so that we can re-use an existing
// context - don't want to close DB on a parent context
async function internal(opts = { existing: false }) {
// set the app tenant id
if (!opts.existing) {
setAppTenantId(appId)
}
// set the app ID
cls.setOnContext(ContextKeys.APP_ID, appId)
// preserve the identity
exports.setIdentity(identity)
try {
// invoke the task
return await task()
} finally {
const using = cls.getFromContext(ContextKeys.IN_USE)
if (!using || using <= 1) {
await closeAppDBs()
} else {
cls.setOnContext(using - 1)
}
}
}
const using = cls.getFromContext(ContextKeys.IN_USE)
if (!forceNew && using && cls.getFromContext(ContextKeys.APP_ID) === appId) {
cls.setOnContext(ContextKeys.IN_USE, using + 1)
return internal({ existing: true })
} else {
return cls.run(async () => {
cls.setOnContext(ContextKeys.IN_USE, 1)
return internal()
})
}
}
exports.doInIdentityContext = (identity, task) => {
if (!identity) {
throw new Error("identity is required")
}
async function internal(opts = { existing: false }) {
if (!opts.existing) {
cls.setOnContext(ContextKeys.IDENTITY, identity)
// set the tenant so that doInTenant will preserve identity
if (identity.tenantId) {
exports.updateTenantId(identity.tenantId)
}
}
try {
// invoke the task
return await task()
} finally {
const using = cls.getFromContext(ContextKeys.IN_USE)
if (!using || using <= 1) {
exports.setIdentity(null)
} else {
cls.setOnContext(using - 1)
}
}
}
const existing = cls.getFromContext(ContextKeys.IDENTITY)
const using = cls.getFromContext(ContextKeys.IN_USE)
if (using && existing && existing._id === identity._id) {
cls.setOnContext(ContextKeys.IN_USE, using + 1)
return internal({ existing: true })
} else {
return cls.run(async () => {
cls.setOnContext(ContextKeys.IN_USE, 1)
return internal({ existing: false })
})
}
}
exports.setIdentity = identity => {
cls.setOnContext(ContextKeys.IDENTITY, identity)
}
exports.getIdentity = () => {
try {
return cls.getFromContext(ContextKeys.IDENTITY)
} catch (e) {
// do nothing - identity is not in context
}
}
exports.updateTenantId = tenantId => {
cls.setOnContext(ContextKeys.TENANT_ID, tenantId)
if (env.USE_COUCH) {
exports.setGlobalDB(tenantId)
}
}
exports.updateAppId = async appId => {
try {
// have to close first, before removing the databases from context
await closeAppDBs()
cls.setOnContext(ContextKeys.APP_ID, appId)
} catch (err) {
if (env.isTest()) {
TEST_APP_ID = appId
} else {
throw err
}
}
}
exports.setGlobalDB = tenantId => {
const dbName = baseGlobalDBName(tenantId)
const db = dangerousGetDB(dbName)
cls.setOnContext(ContextKeys.GLOBAL_DB, db)
return db
}
exports.getGlobalDB = () => {
const db = cls.getFromContext(ContextKeys.GLOBAL_DB)
if (!db) {
throw new Error("Global DB not found")
}
return db
}
exports.isTenantIdSet = () => {
const tenantId = cls.getFromContext(ContextKeys.TENANT_ID)
return !!tenantId
}
exports.getTenantId = () => {
if (!exports.isMultiTenant()) {
return exports.DEFAULT_TENANT_ID
}
const tenantId = cls.getFromContext(ContextKeys.TENANT_ID)
if (!tenantId) {
throw new Error("Tenant id not found")
}
return tenantId
}
exports.getAppId = () => {
const foundId = cls.getFromContext(ContextKeys.APP_ID)
if (!foundId && env.isTest() && TEST_APP_ID) {
return TEST_APP_ID
} else {
return foundId
}
}
function getContextDB(key, opts) {
const dbOptsKey = `${key}${ContextKeys.DB_OPTS}`
let storedOpts = cls.getFromContext(dbOptsKey)
let db = cls.getFromContext(key)
if (db && isEqual(opts, storedOpts)) {
return db
}
const appId = exports.getAppId()
let toUseAppId
switch (key) {
case ContextKeys.CURRENT_DB:
toUseAppId = appId
break
case ContextKeys.PROD_DB:
toUseAppId = getProdAppID(appId)
break
case ContextKeys.DEV_DB:
toUseAppId = getDevelopmentAppID(appId)
break
}
db = dangerousGetDB(toUseAppId, opts)
try {
cls.setOnContext(key, db)
if (opts) {
cls.setOnContext(dbOptsKey, opts)
}
} catch (err) {
if (!env.isTest()) {
throw err
}
}
return db
}
/**
* Opens the app database based on whatever the request
* contained, dev or prod.
*/
exports.getAppDB = (opts = null) => {
return getContextDB(ContextKeys.CURRENT_DB, opts)
}
/**
* This specifically gets the prod app ID, if the request
* contained a development app ID, this will open the prod one.
*/
exports.getProdAppDB = (opts = null) => {
return getContextDB(ContextKeys.PROD_DB, opts)
}
/**
* This specifically gets the dev app ID, if the request
* contained a prod app ID, this will open the dev one.
*/
exports.getDevAppDB = (opts = null) => {
return getContextDB(ContextKeys.DEV_DB, opts)
}

247
packages/backend-core/src/context/index.ts Normal file
View File

@ -0,0 +1,247 @@
import env from "../environment"
import { SEPARATOR, DocumentTypes } from "../db/constants"
import cls from "./FunctionContext"
import { dangerousGetDB, closeDB } from "../db"
import { baseGlobalDBName } from "../tenancy/utils"
import { IdentityContext } from "@budibase/types"
import { DEFAULT_TENANT_ID as _DEFAULT_TENANT_ID } from "../constants"
import { ContextKeys } from "./constants"
import {
updateUsing,
closeWithUsing,
setAppTenantId,
setIdentity,
closeAppDBs,
getContextDB,
} from "./utils"
export const DEFAULT_TENANT_ID = _DEFAULT_TENANT_ID
// some test cases call functions directly, need to
// store an app ID to pretend there is a context
let TEST_APP_ID: string | null = null
export const closeTenancy = async () => {
let db
try {
if (env.USE_COUCH) {
db = getGlobalDB()
}
} catch (err) {
// no DB found - skip closing
return
}
await closeDB(db)
// clear from context now that database is closed/task is finished
cls.setOnContext(ContextKeys.TENANT_ID, null)
cls.setOnContext(ContextKeys.GLOBAL_DB, null)
}
// export const isDefaultTenant = () => {
// return getTenantId() === DEFAULT_TENANT_ID
// }
export const isMultiTenant = () => {
return env.MULTI_TENANCY
}
/**
* Given an app ID this will attempt to retrieve the tenant ID from it.
* @return {null|string} The tenant ID found within the app ID.
*/
export const getTenantIDFromAppID = (appId: string) => {
if (!appId) {
return null
}
const split = appId.split(SEPARATOR)
const hasDev = split[1] === DocumentTypes.DEV
if ((hasDev && split.length === 3) || (!hasDev && split.length === 2)) {
return null
}
if (hasDev) {
return split[2]
} else {
return split[1]
}
}
// used for automations, API endpoints should always be in context already
export const doInTenant = (tenantId: string | null, task: any) => {
// the internal function is so that we can re-use an existing
// context - don't want to close DB on a parent context
async function internal(opts = { existing: false }) {
// set the tenant id + global db if this is a new context
if (!opts.existing) {
updateTenantId(tenantId)
}
try {
// invoke the task
return await task()
} finally {
await closeWithUsing(ContextKeys.TENANCY_IN_USE, () => {
return closeTenancy()
})
}
}
const existing = cls.getFromContext(ContextKeys.TENANT_ID) === tenantId
return updateUsing(ContextKeys.TENANCY_IN_USE, existing, internal)
}
export const doInAppContext = (appId: string, task: any) => {
if (!appId) {
throw new Error("appId is required")
}
const identity = getIdentity()
// the internal function is so that we can re-use an existing
// context - don't want to close DB on a parent context
async function internal(opts = { existing: false }) {
// set the app tenant id
if (!opts.existing) {
setAppTenantId(appId)
}
// set the app ID
cls.setOnContext(ContextKeys.APP_ID, appId)
// preserve the identity
if (identity) {
setIdentity(identity)
}
try {
// invoke the task
return await task()
} finally {
await closeWithUsing(ContextKeys.APP_IN_USE, async () => {
await closeAppDBs()
await closeTenancy()
})
}
}
const existing = cls.getFromContext(ContextKeys.APP_ID) === appId
return updateUsing(ContextKeys.APP_IN_USE, existing, internal)
}
export const doInIdentityContext = (identity: IdentityContext, task: any) => {
if (!identity) {
throw new Error("identity is required")
}
async function internal(opts = { existing: false }) {
if (!opts.existing) {
cls.setOnContext(ContextKeys.IDENTITY, identity)
// set the tenant so that doInTenant will preserve identity
if (identity.tenantId) {
updateTenantId(identity.tenantId)
}
}
try {
// invoke the task
return await task()
} finally {
await closeWithUsing(ContextKeys.IDENTITY_IN_USE, async () => {
setIdentity(null)
await closeTenancy()
})
}
}
const existing = cls.getFromContext(ContextKeys.IDENTITY)
return updateUsing(ContextKeys.IDENTITY_IN_USE, existing, internal)
}
export const getIdentity = (): IdentityContext | undefined => {
try {
return cls.getFromContext(ContextKeys.IDENTITY)
} catch (e) {
// do nothing - identity is not in context
}
}
export const updateTenantId = (tenantId: string | null) => {
cls.setOnContext(ContextKeys.TENANT_ID, tenantId)
if (env.USE_COUCH) {
setGlobalDB(tenantId)
}
}
export const updateAppId = async (appId: string) => {
try {
// have to close first, before removing the databases from context
await closeAppDBs()
cls.setOnContext(ContextKeys.APP_ID, appId)
} catch (err) {
if (env.isTest()) {
TEST_APP_ID = appId
} else {
throw err
}
}
}
export const setGlobalDB = (tenantId: string | null) => {
const dbName = baseGlobalDBName(tenantId)
const db = dangerousGetDB(dbName)
cls.setOnContext(ContextKeys.GLOBAL_DB, db)
return db
}
export const getGlobalDB = () => {
const db = cls.getFromContext(ContextKeys.GLOBAL_DB)
if (!db) {
throw new Error("Global DB not found")
}
return db
}
export const isTenantIdSet = () => {
const tenantId = cls.getFromContext(ContextKeys.TENANT_ID)
return !!tenantId
}
export const getTenantId = () => {
if (!isMultiTenant()) {
return DEFAULT_TENANT_ID
}
const tenantId = cls.getFromContext(ContextKeys.TENANT_ID)
if (!tenantId) {
throw new Error("Tenant id not found")
}
return tenantId
}
export const getAppId = () => {
const foundId = cls.getFromContext(ContextKeys.APP_ID)
if (!foundId && env.isTest() && TEST_APP_ID) {
return TEST_APP_ID
} else {
return foundId
}
}
/**
* Opens the app database based on whatever the request
* contained, dev or prod.
*/
export const getAppDB = (opts?: any) => {
return getContextDB(ContextKeys.CURRENT_DB, opts)
}
/**
* This specifically gets the prod app ID, if the request
* contained a development app ID, this will open the prod one.
*/
export const getProdAppDB = (opts?: any) => {
return getContextDB(ContextKeys.PROD_DB, opts)
}
/**
* This specifically gets the dev app ID, if the request
* contained a prod app ID, this will open the dev one.
*/
export const getDevAppDB = (opts?: any) => {
return getContextDB(ContextKeys.DEV_DB, opts)
}

148
packages/backend-core/src/context/tests/index.spec.ts Normal file
View File

@ -0,0 +1,148 @@
import "../../../tests/utilities/TestConfiguration"
import * as context from ".."
import { DEFAULT_TENANT_ID } from "../../constants"
import env from "../../environment"
// must use require to spy index file exports due to known issue in jest
const dbUtils = require("../../db")
jest.spyOn(dbUtils, "closeDB")
jest.spyOn(dbUtils, "dangerousGetDB")
describe("context", () => {
beforeEach(() => {
jest.clearAllMocks()
})
describe("doInTenant", () => {
describe("single-tenancy", () => {
it("defaults to the default tenant", () => {
const tenantId = context.getTenantId()
expect(tenantId).toBe(DEFAULT_TENANT_ID)
})
it("defaults to the default tenant db", async () => {
await context.doInTenant(DEFAULT_TENANT_ID, () => {
const db = context.getGlobalDB()
expect(db.name).toBe("global-db")
})
expect(dbUtils.dangerousGetDB).toHaveBeenCalledTimes(1)
expect(dbUtils.closeDB).toHaveBeenCalledTimes(1)
})
})
describe("multi-tenancy", () => {
beforeEach(() => {
env._set("MULTI_TENANCY", 1)
})
it("fails when no tenant id is set", () => {
const test = () => {
let error
try {
context.getTenantId()
} catch (e: any) {
error = e
}
expect(error.message).toBe("Tenant id not found")
}
// test under no tenancy
test()
// test after tenancy has been accessed to ensure cleanup
context.doInTenant("test", () => {})
test()
})
it("fails when no tenant db is set", () => {
const test = () => {
let error
try {
context.getGlobalDB()
} catch (e: any) {
error = e
}
expect(error.message).toBe("Global DB not found")
}
// test under no tenancy
test()
// test after tenancy has been accessed to ensure cleanup
context.doInTenant("test", () => {})
test()
})
it("sets tenant id", () => {
context.doInTenant("test", () => {
const tenantId = context.getTenantId()
expect(tenantId).toBe("test")
})
})
it("initialises the tenant db", async () => {
await context.doInTenant("test", () => {
const db = context.getGlobalDB()
expect(db.name).toBe("test_global-db")
})
expect(dbUtils.dangerousGetDB).toHaveBeenCalledTimes(1)
expect(dbUtils.closeDB).toHaveBeenCalledTimes(1)
})
it("sets the tenant id when nested with same tenant id", async () => {
await context.doInTenant("test", async () => {
const tenantId = context.getTenantId()
expect(tenantId).toBe("test")
await context.doInTenant("test", async () => {
const tenantId = context.getTenantId()
expect(tenantId).toBe("test")
await context.doInTenant("test", () => {
const tenantId = context.getTenantId()
expect(tenantId).toBe("test")
})
})
})
})
it("initialises the tenant db when nested with same tenant id", async () => {
await context.doInTenant("test", async () => {
const db = context.getGlobalDB()
expect(db.name).toBe("test_global-db")
await context.doInTenant("test", async () => {
const db = context.getGlobalDB()
expect(db.name).toBe("test_global-db")
await context.doInTenant("test", () => {
const db = context.getGlobalDB()
expect(db.name).toBe("test_global-db")
})
})
})
// only 1 db is opened and closed
expect(dbUtils.dangerousGetDB).toHaveBeenCalledTimes(1)
expect(dbUtils.closeDB).toHaveBeenCalledTimes(1)
})
it("sets different tenant id inside another context", () => {
context.doInTenant("test", () => {
const tenantId = context.getTenantId()
expect(tenantId).toBe("test")
context.doInTenant("nested", () => {
const tenantId = context.getTenantId()
expect(tenantId).toBe("nested")
context.doInTenant("double-nested", () => {
const tenantId = context.getTenantId()
expect(tenantId).toBe("double-nested")
})
})
})
})
})
})
})

113
packages/backend-core/src/context/utils.ts Normal file
View File

@ -0,0 +1,113 @@
import {
DEFAULT_TENANT_ID,
getAppId,
getTenantIDFromAppID,
updateTenantId,
} from "./index"
import cls from "./FunctionContext"
import { IdentityContext } from "@budibase/types"
import { ContextKeys } from "./constants"
import { dangerousGetDB, closeDB } from "../db"
import { isEqual } from "lodash"
import { getDevelopmentAppID, getProdAppID } from "../db/conversions"
import env from "../environment"
export async function updateUsing(
usingKey: string,
existing: boolean,
internal: (opts: { existing: boolean }) => Promise<any>
) {
const using = cls.getFromContext(usingKey)
if (using && existing) {
cls.setOnContext(usingKey, using + 1)
return internal({ existing: true })
} else {
return cls.run(async () => {
cls.setOnContext(usingKey, 1)
return internal({ existing: false })
})
}
}
export async function closeWithUsing(
usingKey: string,
closeFn: () => Promise<any>
) {
const using = cls.getFromContext(usingKey)
if (!using || using <= 1) {
await closeFn()
} else {
cls.setOnContext(usingKey, using - 1)
}
}
export const setAppTenantId = (appId: string) => {
const appTenantId = getTenantIDFromAppID(appId) || DEFAULT_TENANT_ID
updateTenantId(appTenantId)
}
export const setIdentity = (identity: IdentityContext | null) => {
cls.setOnContext(ContextKeys.IDENTITY, identity)
}
// this function makes sure the PouchDB objects are closed and
// fully deleted when finished - this protects against memory leaks
export async function closeAppDBs() {
const dbKeys = [
ContextKeys.CURRENT_DB,
ContextKeys.PROD_DB,
ContextKeys.DEV_DB,
]
for (let dbKey of dbKeys) {
const db = cls.getFromContext(dbKey)
if (!db) {
continue
}
await closeDB(db)
// clear the DB from context, incase someone tries to use it again
cls.setOnContext(dbKey, null)
}
// clear the app ID now that the databases are closed
if (cls.getFromContext(ContextKeys.APP_ID)) {
cls.setOnContext(ContextKeys.APP_ID, null)
}
if (cls.getFromContext(ContextKeys.DB_OPTS)) {
cls.setOnContext(ContextKeys.DB_OPTS, null)
}
}
export function getContextDB(key: string, opts: any) {
const dbOptsKey = `${key}${ContextKeys.DB_OPTS}`
let storedOpts = cls.getFromContext(dbOptsKey)
let db = cls.getFromContext(key)
if (db && isEqual(opts, storedOpts)) {
return db
}
const appId = getAppId()
let toUseAppId
switch (key) {
case ContextKeys.CURRENT_DB:
toUseAppId = appId
break
case ContextKeys.PROD_DB:
toUseAppId = getProdAppID(appId)
break
case ContextKeys.DEV_DB:
toUseAppId = getDevelopmentAppID(appId)
break
}
db = dangerousGetDB(toUseAppId, opts)
try {
cls.setOnContext(key, db)
if (opts) {
cls.setOnContext(dbOptsKey, opts)
}
} catch (err) {
if (!env.isTest()) {
throw err
}
}
return db
}

14
packages/backend-core/src/db/index.js
View File

@ -1,10 +1,18 @@
const pouch = require("./pouch") const pouch = require("./pouch")
const env = require("../environment") const env = require("../environment")
const openDbs = []
let PouchDB let PouchDB
let initialised = false let initialised = false
const dbList = new Set() const dbList = new Set()
if (env.MEMORY_LEAK_CHECK) {
setInterval(() => {
console.log("--- OPEN DBS ---")
console.log(openDbs)
}, 5000)
}
const put = const put =
dbPut => dbPut =>
async (doc, options = {}) => { async (doc, options = {}) => {
@ -35,6 +43,9 @@ exports.dangerousGetDB = (dbName, opts) => {
dbList.add(dbName) dbList.add(dbName)
} }
const db = new PouchDB(dbName, opts) const db = new PouchDB(dbName, opts)
if (env.MEMORY_LEAK_CHECK) {
openDbs.push(db.name)
}
const dbPut = db.put const dbPut = db.put
db.put = put(dbPut) db.put = put(dbPut)
return db return db
@ -46,6 +57,9 @@ exports.closeDB = async db => {
if (!db || env.isTest()) { if (!db || env.isTest()) {
return return
} }
if (env.MEMORY_LEAK_CHECK) {
openDbs.splice(openDbs.indexOf(db.name), 1)
}
try { try {
// specifically await so that if there is an error, it can be ignored // specifically await so that if there is an error, it can be ignored
return await db.close() return await db.close()

1
packages/backend-core/src/environment.ts
View File

@ -54,6 +54,7 @@ const env = {
DISABLE_DEVELOPER_LICENSE: process.env.DISABLE_DEVELOPER_LICENSE, DISABLE_DEVELOPER_LICENSE: process.env.DISABLE_DEVELOPER_LICENSE,
DEFAULT_LICENSE: process.env.DEFAULT_LICENSE, DEFAULT_LICENSE: process.env.DEFAULT_LICENSE,
SERVICE: process.env.SERVICE || "budibase", SERVICE: process.env.SERVICE || "budibase",
MEMORY_LEAK_CHECK: process.env.MEMORY_LEAK_CHECK || false,
DEPLOYMENT_ENVIRONMENT: DEPLOYMENT_ENVIRONMENT:
process.env.DEPLOYMENT_ENVIRONMENT || "docker-compose", process.env.DEPLOYMENT_ENVIRONMENT || "docker-compose",
_set(key: any, value: any) { _set(key: any, value: any) {

2
packages/backend-core/src/events/processors/PosthogProcessor.ts
View File

@ -2,7 +2,7 @@ import PostHog from "posthog-node"
import { Event, Identity, Group, BaseEvent } from "@budibase/types" import { Event, Identity, Group, BaseEvent } from "@budibase/types"
import { EventProcessor } from "./types" import { EventProcessor } from "./types"
import env from "../../environment" import env from "../../environment"
import context from "../../context" import * as context from "../../context"
const pkg = require("../../../package.json") const pkg = require("../../../package.json")
export default class PosthogProcessor implements EventProcessor { export default class PosthogProcessor implements EventProcessor {

2
packages/backend-core/src/migrations/migrations.ts
View File

@ -9,7 +9,7 @@ import {
getGlobalDBName, getGlobalDBName,
getTenantId, getTenantId,
} from "../tenancy" } from "../tenancy"
import context from "../context" import * as context from "../context"
import { DEFINITIONS } from "." import { DEFINITIONS } from "."
import { import {
Migration, Migration,

5
packages/backend-core/yarn.lock
View File

@ -764,6 +764,11 @@
"@types/koa-compose" "*" "@types/koa-compose" "*"
"@types/node" "*" "@types/node" "*"
"@types/lodash@4.14.180":
version "4.14.180"
resolved "https://registry.yarnpkg.com/@types/lodash/-/lodash-4.14.180.tgz#4ab7c9ddfc92ec4a887886483bc14c79fb380670"
integrity sha512-XOKXa1KIxtNXgASAnwj7cnttJxS4fksBRywK/9LzRV5YxrF80BXZIGeQSuoESQ/VkUj30Ae0+YcuHc15wJCB2g==
"@types/mime@^1": "@types/mime@^1":
version "1.3.2" version "1.3.2"
resolved "https://registry.yarnpkg.com/@types/mime/-/mime-1.3.2.tgz#93e25bf9ee75fe0fd80b594bc4feb0e862111b5a" resolved "https://registry.yarnpkg.com/@types/mime/-/mime-1.3.2.tgz#93e25bf9ee75fe0fd80b594bc4feb0e862111b5a"

34
packages/server/src/api/controllers/application.ts
View File

@ -22,9 +22,6 @@ const {
BUILTIN_ROLE_IDS, BUILTIN_ROLE_IDS,
AccessController, AccessController,
} = require("@budibase/backend-core/roles") } = require("@budibase/backend-core/roles")
import { BASE_LAYOUTS } from "../../constants/layouts"
import { cloneDeep } from "lodash/fp"
const { processObject } = require("@budibase/string-templates")
const { CacheKeys, bustCache } = require("@budibase/backend-core/cache") const { CacheKeys, bustCache } = require("@budibase/backend-core/cache")
const { const {
getAllApps, getAllApps,
@ -45,13 +42,8 @@ const { getTenantId, isMultiTenant } = require("@budibase/backend-core/tenancy")
import { syncGlobalUsers } from "./user" import { syncGlobalUsers } from "./user"
const { app: appCache } = require("@budibase/backend-core/cache") const { app: appCache } = require("@budibase/backend-core/cache")
import { cleanupAutomations } from "../../automations/utils" import { cleanupAutomations } from "../../automations/utils"
import { context } from "@budibase/backend-core"
import { checkAppMetadata } from "../../automations/logging" import { checkAppMetadata } from "../../automations/logging"
const {
getAppDB,
getProdAppDB,
updateAppId,
doInAppContext,
} = require("@budibase/backend-core/context")
import { getUniqueRows } from "../../utilities/usageQuota/rows" import { getUniqueRows } from "../../utilities/usageQuota/rows"
import { quotas } from "@budibase/pro" import { quotas } from "@budibase/pro"
import { errors, events, migrations } from "@budibase/backend-core" import { errors, events, migrations } from "@budibase/backend-core"
@ -61,7 +53,7 @@ const URL_REGEX_SLASH = /\/|\\/g
// utility function, need to do away with this // utility function, need to do away with this
async function getLayouts() { async function getLayouts() {
const db = getAppDB() const db = context.getAppDB()
return ( return (
await db.allDocs( await db.allDocs(
getLayoutParams(null, { getLayoutParams(null, {
@ -72,7 +64,7 @@ async function getLayouts() {
} }
async function getScreens() { async function getScreens() {
const db = getAppDB() const db = context.getAppDB()
return ( return (
await db.allDocs( await db.allDocs(
getScreenParams(null, { getScreenParams(null, {
@ -135,9 +127,9 @@ async function createInstance(template: any) {
const tenantId = isMultiTenant() ? getTenantId() : null const tenantId = isMultiTenant() ? getTenantId() : null
const baseAppId = generateAppID(tenantId) const baseAppId = generateAppID(tenantId)
const appId = generateDevAppID(baseAppId) const appId = generateDevAppID(baseAppId)
await updateAppId(appId) await context.updateAppId(appId)
const db = getAppDB() const db = context.getAppDB()
await db.put({ await db.put({
_id: "_design/database", _id: "_design/database",
// view collation information, read before writing any complex views: // view collation information, read before writing any complex views:
@ -213,7 +205,7 @@ export const fetchAppDefinition = async (ctx: any) => {
} }
export const fetchAppPackage = async (ctx: any) => { export const fetchAppPackage = async (ctx: any) => {
const db = getAppDB() const db = context.getAppDB()
const application = await db.get(DocumentTypes.APP_METADATA) const application = await db.get(DocumentTypes.APP_METADATA)
const layouts = await getLayouts() const layouts = await getLayouts()
let screens = await getScreens() let screens = await getScreens()
@ -252,7 +244,7 @@ const performAppCreate = async (ctx: any) => {
const instance = await createInstance(instanceConfig) const instance = await createInstance(instanceConfig)
const appId = instance._id const appId = instance._id
const db = getAppDB() const db = context.getAppDB()
let _rev let _rev
try { try {
// if template there will be an existing doc // if template there will be an existing doc
@ -390,7 +382,7 @@ export const update = async (ctx: any) => {
export const updateClient = async (ctx: any) => { export const updateClient = async (ctx: any) => {
// Get current app version // Get current app version
const db = getAppDB() const db = context.getAppDB()
const application = await db.get(DocumentTypes.APP_METADATA) const application = await db.get(DocumentTypes.APP_METADATA)
const currentVersion = application.version const currentVersion = application.version
@ -414,7 +406,7 @@ export const updateClient = async (ctx: any) => {
export const revertClient = async (ctx: any) => { export const revertClient = async (ctx: any) => {
// Check app can be reverted // Check app can be reverted
const db = getAppDB() const db = context.getAppDB()
const application = await db.get(DocumentTypes.APP_METADATA) const application = await db.get(DocumentTypes.APP_METADATA)
if (!application.revertableVersion) { if (!application.revertableVersion) {
ctx.throw(400, "There is no version to revert to") ctx.throw(400, "There is no version to revert to")
@ -446,7 +438,7 @@ const destroyApp = async (ctx: any) => {
appId = getProdAppID(appId) appId = getProdAppID(appId)
} }
const db = isUnpublish ? getProdAppDB() : getAppDB() const db = isUnpublish ? context.getProdAppDB() : context.getAppDB()
const app = await db.get(DocumentTypes.APP_METADATA) const app = await db.get(DocumentTypes.APP_METADATA)
const result = await db.destroy() const result = await db.destroy()
@ -514,7 +506,7 @@ export const sync = async (ctx: any, next: any) => {
try { try {
// specific case, want to make sure setup is skipped // specific case, want to make sure setup is skipped
const prodDb = getProdAppDB({ skip_setup: true }) const prodDb = context.getProdAppDB({ skip_setup: true })
const info = await prodDb.info() const info = await prodDb.info()
if (info.error) throw info.error if (info.error) throw info.error
} catch (err) { } catch (err) {
@ -556,8 +548,8 @@ export const sync = async (ctx: any, next: any) => {
} }
const updateAppPackage = async (appPackage: any, appId: any) => { const updateAppPackage = async (appPackage: any, appId: any) => {
return doInAppContext(appId, async () => { return context.doInAppContext(appId, async () => {
const db = getAppDB() const db = context.getAppDB()
const application = await db.get(DocumentTypes.APP_METADATA) const application = await db.get(DocumentTypes.APP_METADATA)
const newAppPackage = { ...application, ...appPackage } const newAppPackage = { ...application, ...appPackage }

36
packages/server/src/api/routes/tests/row.spec.js
View File

@ -46,26 +46,26 @@ describe("/rows", () => {
describe("save, load, update", () => { describe("save, load, update", () => {
it("returns a success message when the row is created", async () => { it("returns a success message when the row is created", async () => {
const rowUsage = await getRowUsage() // const rowUsage = await getRowUsage()
const queryUsage = await getQueryUsage() // const queryUsage = await getQueryUsage()
//
const res = await request // const res = await request
.post(`/api/${row.tableId}/rows`) // .post(`/api/${row.tableId}/rows`)
.send(row) // .send(row)
.set(config.defaultHeaders()) // .set(config.defaultHeaders())
.expect('Content-Type', /json/) // .expect('Content-Type', /json/)
.expect(200) // .expect(200)
expect(res.res.statusMessage).toEqual(`${table.name} saved successfully`) // expect(res.res.statusMessage).toEqual(`${table.name} saved successfully`)
expect(res.body.name).toEqual("Test Contact") // expect(res.body.name).toEqual("Test Contact")
expect(res.body._rev).toBeDefined() // expect(res.body._rev).toBeDefined()
await assertRowUsage(rowUsage + 1) // await assertRowUsage(rowUsage + 1)
await assertQueryUsage(queryUsage + 1) // await assertQueryUsage(queryUsage + 1)
}) })
it("updates a row successfully", async () => { it("updates a row successfully", async () => {
const existing = await config.createRow() const existing = await config.createRow()
const rowUsage = await getRowUsage() // const rowUsage = await getRowUsage()
const queryUsage = await getQueryUsage() // const queryUsage = await getQueryUsage()
const res = await request const res = await request
.post(`/api/${table._id}/rows`) .post(`/api/${table._id}/rows`)
@ -81,8 +81,8 @@ describe("/rows", () => {
expect(res.res.statusMessage).toEqual(`${table.name} updated successfully.`) expect(res.res.statusMessage).toEqual(`${table.name} updated successfully.`)
expect(res.body.name).toEqual("Updated Name") expect(res.body.name).toEqual("Updated Name")
await assertRowUsage(rowUsage) // await assertRowUsage(rowUsage)
await assertQueryUsage(queryUsage + 1) // await assertQueryUsage(queryUsage + 1)
}) })
it("should load a row", async () => { it("should load a row", async () => {

1
packages/server/src/app.ts
View File

@ -20,7 +20,6 @@ import redis from "./utilities/redis"
import * as migrations from "./migrations" import * as migrations from "./migrations"
import { events, installation, tenancy } from "@budibase/backend-core" import { events, installation, tenancy } from "@budibase/backend-core"
import { createAdminUser, getChecklist } from "./utilities/workerRequests" import { createAdminUser, getChecklist } from "./utilities/workerRequests"
import { tenantSucceeded } from "@budibase/backend-core/dist/src/events/publishers/backfill"
const app = new Koa() const app = new Koa()

25
packages/server/src/tests/utilities/TestConfiguration.js
View File

@ -106,21 +106,31 @@ class TestConfiguration {
// UTILS // UTILS
async _req(config, params, controlFunc) { async _req(body, params, controlFunc, opts = { prodApp: false }) {
// create a fake request ctx
const request = {} const request = {}
// set the app id
let appId
if (opts.prodApp) {
appId = this.prodAppId
} else {
appId = this.appId
}
request.appId = appId
// fake cookies, we don't need them // fake cookies, we don't need them
request.cookies = { set: () => {}, get: () => {} } request.cookies = { set: () => {}, get: () => {} }
request.config = { jwtSecret: env.JWT_SECRET } request.config = { jwtSecret: env.JWT_SECRET }
request.appId = this.appId request.user = { appId, tenantId: TENANT_ID }
request.user = { appId: this.appId, tenantId: TENANT_ID }
request.query = {} request.query = {}
request.request = { request.request = {
body: config, body,
} }
return this.doInContext(this.appId, async () => {
if (params) { if (params) {
request.params = params request.params = params
} }
return this.doInContext(appId, async () => {
await controlFunc(request) await controlFunc(request)
return request.body return request.body
}) })
@ -323,7 +333,6 @@ class TestConfiguration {
// create production app // create production app
this.prodApp = await this.deploy() this.prodApp = await this.deploy()
this.prodAppId = this.prodApp.appId
this.allApps.push(this.prodApp) this.allApps.push(this.prodApp)
this.allApps.push(this.app) this.allApps.push(this.app)
@ -334,11 +343,13 @@ class TestConfiguration {
async deploy() { async deploy() {
await this._req(null, null, controllers.deploy.deployApp) await this._req(null, null, controllers.deploy.deployApp)
const prodAppId = this.getAppId().replace("_dev", "") const prodAppId = this.getAppId().replace("_dev", "")
this.prodAppId = prodAppId
return context.doInAppContext(prodAppId, async () => { return context.doInAppContext(prodAppId, async () => {
const appPackage = await this._req( const appPackage = await this._req(
null, null,
{ appId: prodAppId }, { appId: prodAppId },
controllers.app.fetchAppPackage controllers.app.fetchAppPackage,
{ prodApp: true }
) )
return appPackage.application return appPackage.application
}) })

3025
packages/server/yarn.lock
View File

File diff suppressed because it is too large Load Diff

2
packages/types/src/sdk/context.ts
View File

@ -4,6 +4,7 @@ import { IdentityType } from "./events/identification"
export interface BaseContext { export interface BaseContext {
_id: string _id: string
type: IdentityType type: IdentityType
tenantId?: string
} }
export interface AccountUserContext extends BaseContext { export interface AccountUserContext extends BaseContext {
@ -13,6 +14,7 @@ export interface AccountUserContext extends BaseContext {
export interface UserContext extends BaseContext, User { export interface UserContext extends BaseContext, User {
_id: string _id: string
tenantId: string
account?: Account account?: Account
} }

1
packages/worker/package.json
View File

@ -17,6 +17,7 @@
"postbuild": "copyfiles -u 1 src/**/*.hbs dist/", "postbuild": "copyfiles -u 1 src/**/*.hbs dist/",
"build:dev": "yarn prebuild && tsc --build --watch --preserveWatchOutput", "build:dev": "yarn prebuild && tsc --build --watch --preserveWatchOutput",
"run:docker": "node dist/index.js", "run:docker": "node dist/index.js",
"debug": "yarn build && node --expose-gc --inspect=9223 dist/index.js",
"run:docker:cluster": "pm2-runtime start pm2.config.js", "run:docker:cluster": "pm2-runtime start pm2.config.js",
"build:docker": "docker build . -t worker-service --label version=$BUDIBASE_RELEASE_VERSION", "build:docker": "docker build . -t worker-service --label version=$BUDIBASE_RELEASE_VERSION",
"dev:stack:init": "node ./scripts/dev/manage.js init", "dev:stack:init": "node ./scripts/dev/manage.js init",

1918
packages/worker/yarn.lock
View File

File diff suppressed because it is too large Load Diff