MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Major update - removing the use of context for PouchDB instances, swapping knowledge of PouchDB to the PouchLike structure that replaces it.

This commit is contained in:
mike12345567 2022-11-09 16:53:42 +00:00
parent a5624142a8
commit c744d23832
25 changed files with 211 additions and 419 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
packages/backend-core/src/cache/tests/writethrough.spec.js vendored
View File

@ -1,6 +1,6 @@
require("../../../tests/utilities/TestConfiguration")
const { Writethrough } = require("../writethrough")
const { dangerousGetDB } = require("../../db")
const { getDB } = require("../../db")
const tk = require("timekeeper")
const START_DATE = Date.now()
@ -8,8 +8,8 @@ tk.freeze(START_DATE)
const DELAY = 5000
const db = dangerousGetDB("test")
const db2 = dangerousGetDB("test2")
const db = getDB("test")
const db2 = getDB("test2")
const writethrough = new Writethrough(db, DELAY), writethrough2 = new Writethrough(db2, DELAY)
describe("writethrough", () => {

17
packages/backend-core/src/cache/writethrough.ts vendored
View File

@ -1,7 +1,7 @@
import BaseCache from "./base"
import { getWritethroughClient } from "../redis/init"
import { logWarn } from "../logging"
import PouchDB from "pouchdb"
import { PouchLike } from "../couch"
const DEFAULT_WRITE_RATE_MS = 10000
let CACHE: BaseCache | null = null
@ -19,7 +19,7 @@ async function getCache() {
return CACHE
}
function makeCacheKey(db: PouchDB.Database, key: string) {
function makeCacheKey(db: PouchLike, key: string) {
return db.name + key
}
@ -28,7 +28,7 @@ function makeCacheItem(doc: any, lastWrite: number | null = null): CacheItem {
}
export async function put(
db: PouchDB.Database,
db: PouchLike,
doc: any,
writeRateMs: number = DEFAULT_WRITE_RATE_MS
) {
@ -64,7 +64,7 @@ export async function put(
return { ok: true, id: output._id, rev: output._rev }
}
export async function get(db: PouchDB.Database, id: string): Promise<any> {
export async function get(db: PouchLike, id: string): Promise<any> {
const cache = await getCache()
const cacheKey = makeCacheKey(db, id)
let cacheItem: CacheItem = await cache.get(cacheKey)
@ -77,7 +77,7 @@ export async function get(db: PouchDB.Database, id: string): Promise<any> {
}
export async function remove(
db: PouchDB.Database,
db: PouchLike,
docOrId: any,
rev?: any
): Promise<void> {
@ -95,13 +95,10 @@ export async function remove(
}
export class Writethrough {
db: PouchDB.Database
db: PouchLike
writeRateMs: number
constructor(
db: PouchDB.Database,
writeRateMs: number = DEFAULT_WRITE_RATE_MS
) {
constructor(db: PouchLike, writeRateMs: number = DEFAULT_WRITE_RATE_MS) {
this.db = db
this.writeRateMs = writeRateMs
}

12
packages/backend-core/src/context/constants.ts
View File

@ -1,17 +1,5 @@
export enum ContextKey {
TENANT_ID = "tenantId",
GLOBAL_DB = "globalDb",
APP_ID = "appId",
IDENTITY = "identity",
// whatever the request app DB was
CURRENT_DB = "currentDb",
// get the prod app DB from the request
PROD_DB = "prodDb",
// get the dev app DB from the request
DEV_DB = "devDb",
DB_OPTS = "dbOpts",
// check if something else is using the context, don't close DB
TENANCY_IN_USE = "tenancyInUse",
APP_IN_USE = "appInUse",
IDENTITY_IN_USE = "identityInUse",
}

160
packages/backend-core/src/context/index.ts
View File

@ -1,20 +1,12 @@
import env from "../environment"
import { SEPARATOR, DocumentType } from "../db/constants"
import cls from "./FunctionContext"
import { dangerousGetDB, closeDB } from "../db"
import { baseGlobalDBName } from "../db/tenancy"
import { IdentityContext } from "@budibase/types"
import { DEFAULT_TENANT_ID as _DEFAULT_TENANT_ID } from "../constants"
import { ContextKey } from "./constants"
import PouchDB from "pouchdb"
import {
updateUsing,
closeWithUsing,
setAppTenantId,
setIdentity,
closeAppDBs,
getContextDB,
} from "./utils"
import { PouchLike } from "../couch"
import { getDevelopmentAppID, getProdAppID } from "../db/conversions"
export const DEFAULT_TENANT_ID = _DEFAULT_TENANT_ID
@ -22,29 +14,19 @@ export const DEFAULT_TENANT_ID = _DEFAULT_TENANT_ID
// store an app ID to pretend there is a context
let TEST_APP_ID: string | null = null
export const closeTenancy = async () => {
try {
if (env.USE_COUCH) {
const db = getGlobalDB()
await closeDB(db)
}
} catch (err) {
// no DB found - skip closing
return
}
// clear from context now that database is closed/task is finished
cls.setOnContext(ContextKey.TENANT_ID, null)
cls.setOnContext(ContextKey.GLOBAL_DB, null)
}
// export const isDefaultTenant = () => {
// return getTenantId() === DEFAULT_TENANT_ID
// }
export const isMultiTenant = () => {
return env.MULTI_TENANCY
}
const setAppTenantId = (appId: string) => {
const appTenantId = getTenantIDFromAppID(appId) || DEFAULT_TENANT_ID
updateTenantId(appTenantId)
}
const setIdentity = (identity: IdentityContext | null) => {
cls.setOnContext(ContextKey.IDENTITY, identity)
}
/**
* Given an app ID this will attempt to retrieve the tenant ID from it.
* @return {null|string} The tenant ID found within the app ID.
@ -78,47 +60,28 @@ export const doInContext = async (appId: string, task: any) => {
})
}
export const doInTenant = (tenantId: string | null, task: any) => {
export const doInTenant = (tenantId: string | null, task: any): any => {
// make sure default always selected in single tenancy
if (!env.MULTI_TENANCY) {
tenantId = tenantId || DEFAULT_TENANT_ID
}
// the internal function is so that we can re-use an existing
// context - don't want to close DB on a parent context
async function internal(opts = { existing: false }) {
// set the tenant id + global db if this is a new context
if (!opts.existing) {
updateTenantId(tenantId)
}
try {
// invoke the task
return await task()
} finally {
await closeWithUsing(ContextKey.TENANCY_IN_USE, () => {
return closeTenancy()
})
}
}
const existing = cls.getFromContext(ContextKey.TENANT_ID) === tenantId
return updateUsing(ContextKey.TENANCY_IN_USE, existing, internal)
return cls.run(async () => {
updateTenantId(tenantId)
return await task()
})
}
export const doInAppContext = (appId: string, task: any) => {
export const doInAppContext = (appId: string, task: any): any => {
if (!appId) {
throw new Error("appId is required")
}
const identity = getIdentity()
// the internal function is so that we can re-use an existing
// context - don't want to close DB on a parent context
async function internal(opts = { existing: false }) {
return cls.run(async () => {
// set the app tenant id
if (!opts.existing) {
setAppTenantId(appId)
}
setAppTenantId(appId)
// set the app ID
cls.setOnContext(ContextKey.APP_ID, appId)
@ -126,47 +89,28 @@ export const doInAppContext = (appId: string, task: any) => {
if (identity) {
setIdentity(identity)
}
try {
// invoke the task
return await task()
} finally {
await closeWithUsing(ContextKey.APP_IN_USE, async () => {
await closeAppDBs()
await closeTenancy()
})
}
}
const existing = cls.getFromContext(ContextKey.APP_ID) === appId
return updateUsing(ContextKey.APP_IN_USE, existing, internal)
// invoke the task
return await task()
})
}
export const doInIdentityContext = (identity: IdentityContext, task: any) => {
export const doInIdentityContext = (
identity: IdentityContext,
task: any
): any => {
if (!identity) {
throw new Error("identity is required")
}
async function internal(opts = { existing: false }) {
if (!opts.existing) {
cls.setOnContext(ContextKey.IDENTITY, identity)
// set the tenant so that doInTenant will preserve identity
if (identity.tenantId) {
updateTenantId(identity.tenantId)
}
return cls.run(async () => {
cls.setOnContext(ContextKey.IDENTITY, identity)
// set the tenant so that doInTenant will preserve identity
if (identity.tenantId) {
updateTenantId(identity.tenantId)
}
try {
// invoke the task
return await task()
} finally {
await closeWithUsing(ContextKey.IDENTITY_IN_USE, async () => {
setIdentity(null)
await closeTenancy()
})
}
}
const existing = cls.getFromContext(ContextKey.IDENTITY)
return updateUsing(ContextKey.IDENTITY_IN_USE, existing, internal)
// invoke the task
return await task()
})
}
export const getIdentity = (): IdentityContext | undefined => {
@ -179,15 +123,10 @@ export const getIdentity = (): IdentityContext | undefined => {
export const updateTenantId = (tenantId: string | null) => {
cls.setOnContext(ContextKey.TENANT_ID, tenantId)
if (env.USE_COUCH) {
setGlobalDB(tenantId)
}
}
export const updateAppId = async (appId: string) => {
try {
// have to close first, before removing the databases from context
await closeAppDBs()
cls.setOnContext(ContextKey.APP_ID, appId)
} catch (err) {
if (env.isTest()) {
@ -198,19 +137,9 @@ export const updateAppId = async (appId: string) => {
}
}
export const setGlobalDB = (tenantId: string | null) => {
const dbName = baseGlobalDBName(tenantId)
const db = dangerousGetDB(dbName)
cls.setOnContext(ContextKey.GLOBAL_DB, db)
return db
}
export const getGlobalDB = () => {
const db = cls.getFromContext(ContextKey.GLOBAL_DB)
if (!db) {
throw new Error("Global DB not found")
}
return db
export const getGlobalDB = (): PouchLike => {
const tenantId = cls.getFromContext(ContextKey.TENANT_ID)
return new PouchLike(baseGlobalDBName(tenantId))
}
export const isTenantIdSet = () => {
@ -246,22 +175,25 @@ export const isTenancyEnabled = () => {
* Opens the app database based on whatever the request
* contained, dev or prod.
*/
export const getAppDB = (opts?: any) => {
return getContextDB(ContextKey.CURRENT_DB, opts)
export const getAppDB = (opts?: any): PouchLike => {
const appId = getAppId()
return new PouchLike(appId, opts)
}
/**
* This specifically gets the prod app ID, if the request
* contained a development app ID, this will open the prod one.
*/
export const getProdAppDB = (opts?: any) => {
return getContextDB(ContextKey.PROD_DB, opts)
export const getProdAppDB = (opts?: any): PouchLike => {
const appId = getAppId()
return new PouchLike(getProdAppID(appId), opts)
}
/**
* This specifically gets the dev app ID, if the request
* contained a prod app ID, this will open the dev one.
*/
export const getDevAppDB = (opts?: any) => {
return getContextDB(ContextKey.DEV_DB, opts)
export const getDevAppDB = (opts?: any): PouchLike => {
const appId = getAppId()
return new PouchLike(getDevelopmentAppID(appId), opts)
}

16
packages/backend-core/src/context/tests/index.spec.ts
View File

@ -5,8 +5,8 @@ import env from "../../environment"
// must use require to spy index file exports due to known issue in jest
const dbUtils = require("../../db")
jest.spyOn(dbUtils, "closeDB")
jest.spyOn(dbUtils, "dangerousGetDB")
jest.spyOn(dbUtils, "closePouchDB")
jest.spyOn(dbUtils, "getDB")
describe("context", () => {
beforeEach(() => {
@ -25,8 +25,8 @@ describe("context", () => {
const db = context.getGlobalDB()
expect(db.name).toBe("global-db")
})
expect(dbUtils.dangerousGetDB).toHaveBeenCalledTimes(1)
expect(dbUtils.closeDB).toHaveBeenCalledTimes(1)
expect(dbUtils.getDB).toHaveBeenCalledTimes(1)
expect(dbUtils.closePouchDB).toHaveBeenCalledTimes(1)
})
})
@ -85,8 +85,8 @@ describe("context", () => {
const db = context.getGlobalDB()
expect(db.name).toBe("test_global-db")
})
expect(dbUtils.dangerousGetDB).toHaveBeenCalledTimes(1)
expect(dbUtils.closeDB).toHaveBeenCalledTimes(1)
expect(dbUtils.getDB).toHaveBeenCalledTimes(1)
expect(dbUtils.closePouchDB).toHaveBeenCalledTimes(1)
})
it("sets the tenant id when nested with same tenant id", async () => {
@ -123,8 +123,8 @@ describe("context", () => {
})
// only 1 db is opened and closed
expect(dbUtils.dangerousGetDB).toHaveBeenCalledTimes(1)
expect(dbUtils.closeDB).toHaveBeenCalledTimes(1)
expect(dbUtils.getDB).toHaveBeenCalledTimes(1)
expect(dbUtils.closePouchDB).toHaveBeenCalledTimes(1)
})
it("sets different tenant id inside another context", () => {

109
packages/backend-core/src/context/utils.ts
View File

@ -1,109 +0,0 @@
import {
DEFAULT_TENANT_ID,
getAppId,
getTenantIDFromAppID,
updateTenantId,
} from "./index"
import cls from "./FunctionContext"
import { IdentityContext } from "@budibase/types"
import { ContextKey } from "./constants"
import { dangerousGetDB, closeDB } from "../db"
import { isEqual } from "lodash"
import { getDevelopmentAppID, getProdAppID } from "../db/conversions"
import env from "../environment"
export async function updateUsing(
usingKey: string,
existing: boolean,
internal: (opts: { existing: boolean }) => Promise<any>
) {
const using = cls.getFromContext(usingKey)
if (using && existing) {
cls.setOnContext(usingKey, using + 1)
return internal({ existing: true })
} else {
return cls.run(async () => {
cls.setOnContext(usingKey, 1)
return internal({ existing: false })
})
}
}
export async function closeWithUsing(
usingKey: string,
closeFn: () => Promise<any>
) {
const using = cls.getFromContext(usingKey)
if (!using || using <= 1) {
await closeFn()
} else {
cls.setOnContext(usingKey, using - 1)
}
}
export const setAppTenantId = (appId: string) => {
const appTenantId = getTenantIDFromAppID(appId) || DEFAULT_TENANT_ID
updateTenantId(appTenantId)
}
export const setIdentity = (identity: IdentityContext | null) => {
cls.setOnContext(ContextKey.IDENTITY, identity)
}
// this function makes sure the PouchDB objects are closed and
// fully deleted when finished - this protects against memory leaks
export async function closeAppDBs() {
const dbKeys = [ContextKey.CURRENT_DB, ContextKey.PROD_DB, ContextKey.DEV_DB]
for (let dbKey of dbKeys) {
const db = cls.getFromContext(dbKey)
if (!db) {
continue
}
await closeDB(db)
// clear the DB from context, incase someone tries to use it again
cls.setOnContext(dbKey, null)
}
// clear the app ID now that the databases are closed
if (cls.getFromContext(ContextKey.APP_ID)) {
cls.setOnContext(ContextKey.APP_ID, null)
}
if (cls.getFromContext(ContextKey.DB_OPTS)) {
cls.setOnContext(ContextKey.DB_OPTS, null)
}
}
export function getContextDB(key: string, opts: any) {
const dbOptsKey = `${key}${ContextKey.DB_OPTS}`
let storedOpts = cls.getFromContext(dbOptsKey)
let db = cls.getFromContext(key)
if (db && isEqual(opts, storedOpts)) {
return db
}
const appId = getAppId()
let toUseAppId
switch (key) {
case ContextKey.CURRENT_DB:
toUseAppId = appId
break
case ContextKey.PROD_DB:
toUseAppId = getProdAppID(appId)
break
case ContextKey.DEV_DB:
toUseAppId = getDevelopmentAppID(appId)
break
}
db = dangerousGetDB(toUseAppId, opts)
try {
cls.setOnContext(key, db)
if (opts) {
cls.setOnContext(dbOptsKey, opts)
}
} catch (err) {
if (!env.isTest()) {
throw err
}
}
return db
}

1
packages/backend-core/src/couch/index.ts
View File

@ -1,3 +1,4 @@
export * from "./couch"
export * from "./pouchLike"
export * from "./utils"
export { init } from "./pouchDB"

37
packages/backend-core/src/couch/pouchDB.ts Normal file
View File

@ -0,0 +1,37 @@
import PouchDB from "pouchdb"
import env from "../environment"
import { PouchOptions } from "@budibase/types"
import * as pouch from "../db/pouch"
let Pouch: any
let initialised = false
export async function init(opts?: PouchOptions) {
Pouch = pouch.getPouch(opts)
initialised = true
}
const checkInitialised = () => {
if (!initialised) {
throw new Error("init has not been called")
}
}
export function getPouchDB(dbName: string, opts?: any): PouchDB.Database {
checkInitialised()
return new Pouch(dbName, opts)
}
// use this function if you have called getPouchDB - close
// the databases you've opened once finished
export async function closePouchDB(db: PouchDB.Database) {
if (!db || env.isTest()) {
return
}
try {
// specifically await so that if there is an error, it can be ignored
return await db.close()
} catch (err) {
// ignore error, already closed
}
}

55
packages/backend-core/src/couch/pouchLike.ts
View File

@ -2,12 +2,16 @@ import Nano from "nano"
import { AnyDocument } from "@budibase/types"
import { getCouchInfo } from "./couch"
import { directCouchCall } from "./utils"
import { getPouchDB } from "../db"
import { getPouchDB } from "./pouchDB"
export type PouchLikeOpts = {
skip_setup?: boolean
}
export type PutOpts = {
force?: boolean
}
export type QueryOpts = {
include_docs?: boolean
startkey?: string
@ -19,6 +23,10 @@ export type QueryOpts = {
keys?: string[]
}
type QueryResp<T> = Promise<{
rows: { doc?: T | any; value?: any }[]
}>
export class PouchLike {
public readonly name: string
private static nano: Nano.ServerScope
@ -45,11 +53,15 @@ export class PouchLike {
})
}
async exists() {
let response = await directCouchCall(`/${this.name}`, "HEAD")
return response.status === 200
}
async checkSetup() {
let shouldCreate = !this.pouchOpts?.skip_setup
// check exists in a lightweight fashion
let response = await directCouchCall(`/${this.name}`, "HEAD")
let exists = response.status === 200
let exists = await this.exists()
if (!shouldCreate && !exists) {
throw new Error("DB does not exist")
}
@ -70,26 +82,43 @@ export class PouchLike {
}
}
async info() {
const db = PouchLike.nano.db.use(this.name)
return db.info()
}
async get(id: string) {
async get<T>(id?: string): Promise<T | any> {
const db = await this.checkSetup()
if (!id) {
throw new Error("Unable to get doc without a valid _id.")
}
return this.updateOutput(() => db.get(id))
}
async remove(id: string, rev: string) {
async remove(id?: string, rev?: string) {
const db = await this.checkSetup()
if (!id || !rev) {
throw new Error("Unable to remove doc without a valid _id and _rev.")
}
return this.updateOutput(() => db.destroy(id, rev))
}
async put(document: AnyDocument) {
async put(document: AnyDocument, opts?: PutOpts) {
if (!document._id) {
throw new Error("Cannot store document without _id field.")
}
const db = await this.checkSetup()
if (!document.createdAt) {
document.createdAt = new Date().toISOString()
}
document.updatedAt = new Date().toISOString()
if (opts?.force && document._id) {
try {
const existing = await this.get(document._id)
if (existing) {
document._rev = existing._rev
}
} catch (err: any) {
if (err.status !== 404) {
throw err
}
}
}
return this.updateOutput(() => db.insert(document))
}
@ -98,12 +127,12 @@ export class PouchLike {
return this.updateOutput(() => db.bulk({ docs: documents }))
}
async allDocs(params: QueryOpts) {
async allDocs<T>(params: QueryOpts): QueryResp<T> {
const db = await this.checkSetup()
return this.updateOutput(() => db.list(params))
}
async query(viewName: string, params: QueryOpts) {
async query<T>(viewName: string, params: QueryOpts): QueryResp<T> {
const db = await this.checkSetup()
const [database, view] = viewName.split("/")
return this.updateOutput(() => db.view(database, view, params))

10
packages/backend-core/src/db/Replication.ts
View File

@ -1,4 +1,4 @@
import { dangerousGetDB, closeDB } from "."
import { getPouchDB, closePouchDB } from "../couch/pouchDB"
import { DocumentType } from "./constants"
class Replication {
@ -12,12 +12,12 @@ class Replication {
* @param {String} target - the DB you want to replicate to, or rollback from
*/
constructor({ source, target }: any) {
this.source = dangerousGetDB(source)
this.target = dangerousGetDB(target)
this.source = getPouchDB(source)
this.target = getPouchDB(target)
}
close() {
return Promise.all([closeDB(this.source), closeDB(this.target)])
return Promise.all([closePouchDB(this.source), closePouchDB(this.target)])
}
promisify(operation: any, opts = {}) {
@ -68,7 +68,7 @@ class Replication {
async rollback() {
await this.target.destroy()
// Recreate the DB again
this.target = dangerousGetDB(this.target.name)
this.target = getPouchDB(this.target.name)
// take the opportunity to remove deleted tombstones
await this.replicate()
}

65
packages/backend-core/src/db/index.ts
View File

@ -1,87 +1,30 @@
import * as pouch from "./pouch"
import env from "../environment"
import { PouchOptions, CouchFindOptions } from "@budibase/types"
import PouchDB from "pouchdb"
import { CouchFindOptions } from "@budibase/types"
import { PouchLike } from "../couch"
import { directCouchQuery } from "../couch"
export { directCouchQuery } from "../couch"
export { init, PouchLike } from "../couch"
const openDbs: string[] = []
let Pouch: any
let initialised = false
const dbList = new Set()
if (env.MEMORY_LEAK_CHECK) {
setInterval(() => {
console.log("--- OPEN DBS ---")
console.log(openDbs)
}, 5000)
}
const put =
(dbPut: any) =>
async (doc: any, options = {}) => {
if (!doc.createdAt) {
doc.createdAt = new Date().toISOString()
}
doc.updatedAt = new Date().toISOString()
return dbPut(doc, options)
}
const checkInitialised = () => {
if (!initialised) {
throw new Error("init has not been called")
}
}
export async function init(opts?: PouchOptions) {
Pouch = pouch.getPouch(opts)
initialised = true
}
export function getPouchDB(dbName: string, opts?: any): PouchDB.Database {
checkInitialised()
export function getDB(dbName: string, opts?: any): PouchLike {
if (env.isTest()) {
dbList.add(dbName)
}
const db = new Pouch(dbName, opts)
if (env.MEMORY_LEAK_CHECK) {
openDbs.push(db.name)
}
const dbPut = db.put
db.put = put(dbPut)
return db
}
// NOTE: THIS IS A DANGEROUS FUNCTION - USE WITH CAUTION
// this function is prone to leaks, should only be used
// in situations that using the function doWithDB does not work
export function dangerousGetDB(dbName: string, opts?: any): PouchLike {
return new PouchLike(dbName, opts)
}
// use this function if you have called dangerousGetDB - close
// the databases you've opened once finished
export async function closeDB(db: PouchDB.Database) {
if (!db || env.isTest()) {
return
}
if (env.MEMORY_LEAK_CHECK) {
openDbs.splice(openDbs.indexOf(db.name), 1)
}
try {
// specifically await so that if there is an error, it can be ignored
return await db.close()
} catch (err) {
// ignore error, already closed
}
}
// we have to use a callback for this so that we can close
// the DB when we're done, without this manual requests would
// need to close the database when done with it to avoid memory leaks
export async function doWithDB(dbName: string, cb: any, opts = {}) {
const db = dangerousGetDB(dbName, opts)
const db = getDB(dbName, opts)
// need this to be async so that we can correctly close DB after all
// async operations have been completed
return await cb(db)

6
packages/backend-core/src/db/tests/index.spec.js
View File

@ -1,11 +1,11 @@
require("../../../tests/utilities/TestConfiguration")
const { dangerousGetDB } = require("../")
const { getDB } = require("../")
describe("db", () => {
describe("getDB", () => {
it("returns a db", async () => {
const db = dangerousGetDB("test")
const db = getDB("test")
expect(db).toBeDefined()
expect(db._adapter).toBe("memory")
expect(db.prefix).toBe("_pouch_")
@ -13,7 +13,7 @@ describe("db", () => {
})
it("uses the custom put function", async () => {
const db = dangerousGetDB("test")
const db = getDB("test")
let doc = { _id: "test" }
await db.put(doc)
doc = await db.get(doc._id)

49
packages/backend-core/src/db/views.ts
View File

@ -1,6 +1,6 @@
import { DocumentType, ViewName, DeprecatedViews, SEPARATOR } from "./utils"
import { getGlobalDB } from "../context"
import PouchDB from "pouchdb"
import { PouchLike, QueryOpts } from "../couch"
import { StaticDatabases } from "./constants"
import { doWithDB } from "./"
@ -19,7 +19,7 @@ interface DesignDocument {
views: any
}
async function removeDeprecated(db: PouchDB.Database, viewName: ViewName) {
async function removeDeprecated(db: PouchLike, viewName: ViewName) {
// @ts-ignore
if (!DeprecatedViews[viewName]) {
return
@ -70,16 +70,13 @@ export const createAccountEmailView = async () => {
emit(doc.email.toLowerCase(), doc._id)
}
}`
await doWithDB(
StaticDatabases.PLATFORM_INFO.name,
async (db: PouchDB.Database) => {
await createView(db, viewJs, ViewName.ACCOUNT_BY_EMAIL)
}
)
await doWithDB(StaticDatabases.PLATFORM_INFO.name, async (db: PouchLike) => {
await createView(db, viewJs, ViewName.ACCOUNT_BY_EMAIL)
})
}
export const createUserAppView = async () => {
const db = getGlobalDB() as PouchDB.Database
const db = getGlobalDB()
const viewJs = `function(doc) {
if (doc._id.startsWith("${DocumentType.USER}${SEPARATOR}") && doc.roles) {
for (let prodAppId of Object.keys(doc.roles)) {
@ -117,12 +114,9 @@ export const createPlatformUserView = async () => {
emit(doc._id.toLowerCase(), doc._id)
}
}`
await doWithDB(
StaticDatabases.PLATFORM_INFO.name,
async (db: PouchDB.Database) => {
await createView(db, viewJs, ViewName.PLATFORM_USERS_LOWERCASE)
}
)
await doWithDB(StaticDatabases.PLATFORM_INFO.name, async (db: PouchLike) => {
await createView(db, viewJs, ViewName.PLATFORM_USERS_LOWERCASE)
})
}
export interface QueryViewOptions {
@ -131,13 +125,13 @@ export interface QueryViewOptions {
export const queryView = async <T>(
viewName: ViewName,
params: PouchDB.Query.Options<T, T>,
db: PouchDB.Database,
params: QueryOpts,
db: PouchLike,
createFunc: any,
opts?: QueryViewOptions
): Promise<T[] | T | undefined> => {
try {
let response = await db.query<T, T>(`database/${viewName}`, params)
let response = await db.query(`database/${viewName}`, params)
const rows = response.rows
const docs = rows.map(row => (params.include_docs ? row.doc : row.value))
@ -161,7 +155,7 @@ export const queryView = async <T>(
export const queryPlatformView = async <T>(
viewName: ViewName,
params: PouchDB.Query.Options<T, T>,
params: QueryOpts,
opts?: QueryViewOptions
): Promise<T[] | T | undefined> => {
const CreateFuncByName: any = {
@ -169,19 +163,16 @@ export const queryPlatformView = async <T>(
[ViewName.PLATFORM_USERS_LOWERCASE]: createPlatformUserView,
}
return doWithDB(
StaticDatabases.PLATFORM_INFO.name,
async (db: PouchDB.Database) => {
const createFn = CreateFuncByName[viewName]
return queryView(viewName, params, db, createFn, opts)
}
)
return doWithDB(StaticDatabases.PLATFORM_INFO.name, async (db: PouchLike) => {
const createFn = CreateFuncByName[viewName]
return queryView(viewName, params, db, createFn, opts)
})
}
export const queryGlobalView = async <T>(
viewName: ViewName,
params: PouchDB.Query.Options<T, T>,
db?: PouchDB.Database,
params: QueryOpts,
db?: PouchLike,
opts?: QueryViewOptions
): Promise<T[] | T | undefined> => {
const CreateFuncByName: any = {
@ -192,7 +183,7 @@ export const queryGlobalView = async <T>(
}
// can pass DB in if working with something specific
if (!db) {
db = getGlobalDB() as PouchDB.Database
db = getGlobalDB()
}
const createFn = CreateFuncByName[viewName]
return queryView(viewName, params, db, createFn, opts)

1
packages/backend-core/src/environment.ts
View File

@ -69,7 +69,6 @@ const env = {
DISABLE_DEVELOPER_LICENSE: process.env.DISABLE_DEVELOPER_LICENSE,
DEFAULT_LICENSE: process.env.DEFAULT_LICENSE,
SERVICE: process.env.SERVICE || "budibase",
MEMORY_LEAK_CHECK: process.env.MEMORY_LEAK_CHECK || false,
LOG_LEVEL: process.env.LOG_LEVEL,
SESSION_UPDATE_PERIOD: process.env.SESSION_UPDATE_PERIOD,
DEPLOYMENT_ENVIRONMENT:

2
packages/backend-core/src/index.ts
View File

@ -21,6 +21,7 @@ import * as middleware from "./middleware"
import plugins from "./plugin"
import encryption from "./security/encryption"
import * as queue from "./queue"
import * as types from "./types"
// mimic the outer package exports
import * as db from "./pkg/db"
@ -67,6 +68,7 @@ const core = {
encryption,
queue,
permissions,
...types,
}
export = core

4
packages/backend-core/src/migrations/tests/index.spec.js
View File

@ -1,6 +1,6 @@
require("../../../tests/utilities/TestConfiguration")
const { runMigrations, getMigrationsDoc } = require("../index")
const { dangerousGetDB } = require("../../db")
const { getDB } = require("../../db")
const {
StaticDatabases,
} = require("../../db/utils")
@ -18,7 +18,7 @@ describe("migrations", () => {
}]
beforeEach(() => {
db = dangerousGetDB(StaticDatabases.GLOBAL.name)
db = getDB(StaticDatabases.GLOBAL.name)
})
afterEach(async () => {

1
packages/backend-core/src/types.ts Normal file
View File

@ -0,0 +1 @@
export { PouchLike } from "./couch"

5
packages/backend-core/src/users.ts
View File

@ -8,10 +8,9 @@ import { queryGlobalView } from "./db/views"
import { UNICODE_MAX } from "./db/constants"
import { BulkDocsResponse, User } from "@budibase/types"
import { getGlobalDB } from "./context"
import PouchDB from "pouchdb"
export const bulkGetGlobalUsersById = async (userIds: string[]) => {
const db = getGlobalDB() as PouchDB.Database
const db = getGlobalDB()
return (
await db.allDocs({
keys: userIds,
@ -21,7 +20,7 @@ export const bulkGetGlobalUsersById = async (userIds: string[]) => {
}
export const bulkUpdateGlobalUsers = async (users: User[]) => {
const db = getGlobalDB() as PouchDB.Database
const db = getGlobalDB()
return (await db.bulkDocs(users)) as BulkDocsResponse
}

10
packages/server/src/api/controllers/application.ts
View File

@ -524,12 +524,10 @@ export const sync = async (ctx: any, next: any) => {
// replicate prod to dev
const prodAppId = getProdAppID(appId)
try {
// specific case, want to make sure setup is skipped
const prodDb = context.getProdAppDB({ skip_setup: true })
const info = await prodDb.info()
if (info.error) throw info.error
} catch (err) {
// specific case, want to make sure setup is skipped
const prodDb = context.getProdAppDB({ skip_setup: true })
const exists = await prodDb.exists()
if (!exists) {
// the database doesn't exist. Don't replicate
ctx.status = 200
ctx.body = {

4
packages/server/src/api/controllers/row/internal.js
View File

@ -6,7 +6,7 @@ const {
DocumentType,
InternalTables,
} = require("../../../db/utils")
const { dangerousGetDB } = require("@budibase/backend-core/db")
const { getDB } = require("@budibase/backend-core/db")
const userController = require("../user")
const {
inputProcessing,
@ -251,7 +251,7 @@ exports.fetch = async ctx => {
}
exports.find = async ctx => {
const db = dangerousGetDB(ctx.appId)
const db = getDB(ctx.appId)
const table = await db.get(ctx.params.tableId)
let row = await findRow(ctx, ctx.params.tableId, ctx.params.rowId)
row = await outputProcessing(table, row)

4
packages/server/src/db/inMemoryView.js
View File

@ -2,7 +2,7 @@ const newid = require("./newid")
// bypass the main application db config
// use in memory pouchdb directly
const { getPouch, closeDB } = require("@budibase/backend-core/db")
const { getPouch, closePouchDB } = require("@budibase/backend-core/db")
const Pouch = getPouch({ inMemory: true })
exports.runView = async (view, calculation, group, data) => {
@ -44,6 +44,6 @@ exports.runView = async (view, calculation, group, data) => {
return response
} finally {
await db.destroy()
await closeDB(db)
await closePouchDB(db)
}
}

5
packages/server/src/migrations/functions/backfill/app/tables.ts
View File

@ -1,9 +1,8 @@
import { events } from "@budibase/backend-core"
import { events, PouchLike } from "@budibase/backend-core"
import sdk from "../../../../sdk"
import PouchDB from "pouchdb"
export const backfill = async (
appDb: PouchDB.Database,
appDb: PouchLike,
timestamp: string | number
) => {
const tables = await sdk.tables.getAllInternalTables(appDb)

12
packages/server/src/sdk/app/backups/imports.ts
View File

@ -1,4 +1,4 @@
import { db as dbCore } from "@budibase/backend-core"
import { db as dbCore, PouchLike } from "@budibase/backend-core"
import { getAutomationParams, TABLE_ROW_PREFIX } from "../../../db/utils"
import { budibaseTempDir } from "../../../utilities/budibaseDir"
import { DB_EXPORT_FILE, GLOBAL_DB_EXPORT_FILE } from "./constants"
@ -17,7 +17,6 @@ import {
CouchFindOptions,
RowAttachment,
} from "@budibase/types"
import PouchDB from "pouchdb"
const uuid = require("uuid/v4")
const tar = require("tar")
@ -29,10 +28,7 @@ type TemplateType = {
key?: string
}
async function updateAttachmentColumns(
prodAppId: string,
db: PouchDB.Database
) {
async function updateAttachmentColumns(prodAppId: string, db: PouchLike) {
// iterate through attachment documents and update them
const tables = await sdk.tables.getAllInternalTables(db)
for (let table of tables) {
@ -86,7 +82,7 @@ async function updateAttachmentColumns(
}
}
async function updateAutomations(prodAppId: string, db: PouchDB.Database) {
async function updateAutomations(prodAppId: string, db: PouchLike) {
const automations = (
await db.allDocs(
getAutomationParams(null, {
@ -154,7 +150,7 @@ export function getListOfAppsInMulti(tmpPath: string) {
export async function importApp(
appId: string,
db: PouchDB.Database,
db: PouchLike,
template: TemplateType
) {
let prodAppId = dbCore.getProdAppID(appId)

28
packages/server/src/sdk/app/backups/statistics.ts
View File

@ -1,13 +1,12 @@
import { context, db as dbCore } from "@budibase/backend-core"
import { context, db as dbCore, PouchLike } from "@budibase/backend-core"
import {
getDatasourceParams,
getTableParams,
getAutomationParams,
getScreenParams,
} from "../../../db/utils"
import PouchDB from "pouchdb"
async function runInContext(appId: string, cb: any, db?: PouchDB.Database) {
async function runInContext(appId: string, cb: any, db?: PouchLike) {
if (db) {
return cb(db)
} else {
@ -19,13 +18,10 @@ async function runInContext(appId: string, cb: any, db?: PouchDB.Database) {
}
}
export async function calculateDatasourceCount(
appId: string,
db?: PouchDB.Database
) {
export async function calculateDatasourceCount(appId: string, db?: PouchLike) {
return runInContext(
appId,
async (db: PouchDB.Database) => {
async (db: PouchLike) => {
const datasourceList = await db.allDocs(getDatasourceParams())
const tableList = await db.allDocs(getTableParams())
return datasourceList.rows.length + tableList.rows.length
@ -34,13 +30,10 @@ export async function calculateDatasourceCount(
)
}
export async function calculateAutomationCount(
appId: string,
db?: PouchDB.Database
) {
export async function calculateAutomationCount(appId: string, db?: PouchLike) {
return runInContext(
appId,
async (db: PouchDB.Database) => {
async (db: PouchLike) => {
const automationList = await db.allDocs(getAutomationParams())
return automationList.rows.length
},
@ -48,13 +41,10 @@ export async function calculateAutomationCount(
)
}
export async function calculateScreenCount(
appId: string,
db?: PouchDB.Database
) {
export async function calculateScreenCount(appId: string, db?: PouchLike) {
return runInContext(
appId,
async (db: PouchDB.Database) => {
async (db: PouchLike) => {
const screenList = await db.allDocs(getScreenParams())
return screenList.rows.length
},
@ -63,7 +53,7 @@ export async function calculateScreenCount(
}
export async function calculateBackupStats(appId: string) {
return runInContext(appId, async (db: PouchDB.Database) => {
return runInContext(appId, async (db: PouchLike) => {
const promises = []
promises.push(calculateDatasourceCount(appId, db))
promises.push(calculateAutomationCount(appId, db))

11
packages/server/src/sdk/app/tables/index.ts
View File

@ -1,4 +1,4 @@
import { getAppDB } from "@budibase/backend-core/context"
import { context, PouchLike } from "@budibase/backend-core"
import { BudibaseInternalDB, getTableParams } from "../../../db/utils"
import {
breakExternalTableId,
@ -6,11 +6,10 @@ import {
isSQL,
} from "../../../integrations/utils"
import { Table } from "@budibase/types"
import PouchDB from "pouchdb"
async function getAllInternalTables(db?: PouchDB.Database): Promise<Table[]> {
async function getAllInternalTables(db?: PouchLike): Promise<Table[]> {
if (!db) {
db = getAppDB() as PouchDB.Database
db = context.getAppDB()
}
const internalTables = await db.allDocs(
getTableParams(null, {
@ -25,7 +24,7 @@ async function getAllInternalTables(db?: PouchDB.Database): Promise<Table[]> {
}
async function getAllExternalTables(datasourceId: any): Promise<Table[]> {
const db = getAppDB()
const db = context.getAppDB()
const datasource = await db.get(datasourceId)
if (!datasource || !datasource.entities) {
throw "Datasource is not configured fully."
@ -42,7 +41,7 @@ async function getExternalTable(
}
async function getTable(tableId: any): Promise<Table> {
const db = getAppDB()
const db = context.getAppDB()
if (isExternalTable(tableId)) {
let { datasourceId, tableName } = breakExternalTableId(tableId)
const datasource = await db.get(datasourceId)