diff --git a/.github/workflows/deploy-single-image.yml b/.github/workflows/deploy-single-image.yml index 4a04bf3f98..0bd5c71a40 100644 --- a/.github/workflows/deploy-single-image.yml +++ b/.github/workflows/deploy-single-image.yml @@ -1,11 +1,8 @@ name: Deploy Budibase Single Container Image to DockerHub + on: - push: - branches: - - "omnibus-action" - - "develop" - - "master" - - "main" + workflow_dispatch: + env: BASE_BRANCH: ${{ github.event.pull_request.base.ref}} BRANCH: ${{ github.event.pull_request.head.ref }} @@ -40,7 +37,7 @@ jobs: - name: Runt Yarn Lint run: yarn lint - name: Run Yarn Build - run: yarn build + run: yarn build:docker:pre - name: Login to Docker Hub uses: docker/login-action@v2 with: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6c05c4f684..348b600f90 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -68,7 +68,7 @@ jobs: - name: Publish budibase packages to NPM env: NPM_TOKEN: ${{ secrets.NPM_TOKEN }} - RELEASE_VERSION_TYPE: ${{ github.event.inputs.version }} + RELEASE_VERSION_TYPE: ${{ github.event.inputs.versioning }} run: | # setup the username and email. I tend to use 'GitHub Actions Bot' with no email by default git config --global user.name "Budibase Release Bot" diff --git a/lerna.json b/lerna.json index 0858860bdd..ee80fb1271 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "1.0.220-alpha.4", + "version": "1.1.10-alpha.3", "npmClient": "yarn", "packages": [ "packages/*" diff --git a/package.json b/package.json index b787b4aee5..0c7d3989a2 100644 --- a/package.json +++ b/package.json @@ -54,6 +54,7 @@ "test:e2e:ci:notify": "lerna run cy:ci:notify", "build:specs": "lerna run specs", "build:docker": "lerna run build:docker && npm run build:docker:proxy:compose && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh $BUDIBASE_RELEASE_VERSION && cd -", + "build:docker:pre": "lerna run build && lerna run predocker", "build:docker:proxy": "docker build hosting/proxy -t proxy-service", "build:docker:proxy:compose": "node scripts/proxy/generateProxyConfig compose && npm run build:docker:proxy", "build:docker:proxy:preprod": "node scripts/proxy/generateProxyConfig preprod && npm run build:docker:proxy", @@ -65,7 +66,7 @@ "build:digitalocean": "cd hosting/digitalocean && ./build.sh && cd -", "build:docker:single:multiarch": "docker buildx build --platform linux/arm64,linux/amd64 -f hosting/single/Dockerfile -t budibase:latest .", "build:docker:single:image": "docker build -f hosting/single/Dockerfile -t budibase:latest .", - "build:docker:single": "lerna run build && lerna run predocker && npm run build:docker:single:image", + "build:docker:single": "npm run build:docker:pre && npm run build:docker:single:image", "build:docs": "lerna run build:docs", "release:helm": "node scripts/releaseHelmChart", "env:multi:enable": "lerna run env:multi:enable", @@ -84,4 +85,4 @@ "install:pro": "bash scripts/pro/install.sh", "dep:clean": "yarn clean && yarn bootstrap" } -} \ No newline at end of file +} diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json index 9b25ab7c5b..8156a4d0c0 100644 --- a/packages/backend-core/package.json +++ b/packages/backend-core/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/backend-core", - "version": "1.0.220-alpha.4", + "version": "1.1.10-alpha.3", "description": "Budibase backend core libraries used in server and worker", "main": "dist/src/index.js", "types": "dist/src/index.d.ts", @@ -20,7 +20,7 @@ "test:watch": "jest --watchAll" }, "dependencies": { - "@budibase/types": "^1.0.220-alpha.4", + "@budibase/types": "^1.1.10-alpha.3", "@techpass/passport-openidconnect": "0.3.2", "aws-sdk": "2.1030.0", "bcrypt": "5.0.1", @@ -59,10 +59,10 @@ ] }, "devDependencies": { - "@budibase/types": "^1.0.219", "@shopify/jest-koa-mocks": "3.1.5", "@types/jest": "27.5.1", "@types/koa": "2.0.52", + "@types/lodash": "4.14.180", "@types/node": "14.18.20", "@types/node-fetch": "2.6.1", "@types/pouchdb": "6.4.0", diff --git a/packages/backend-core/src/context/constants.ts b/packages/backend-core/src/context/constants.ts new file mode 100644 index 0000000000..ef8dcd7821 --- /dev/null +++ b/packages/backend-core/src/context/constants.ts @@ -0,0 +1,17 @@ +export enum ContextKeys { + TENANT_ID = "tenantId", + GLOBAL_DB = "globalDb", + APP_ID = "appId", + IDENTITY = "identity", + // whatever the request app DB was + CURRENT_DB = "currentDb", + // get the prod app DB from the request + PROD_DB = "prodDb", + // get the dev app DB from the request + DEV_DB = "devDb", + DB_OPTS = "dbOpts", + // check if something else is using the context, don't close DB + TENANCY_IN_USE = "tenancyInUse", + APP_IN_USE = "appInUse", + IDENTITY_IN_USE = "identityInUse", +} diff --git a/packages/backend-core/src/context/index.js b/packages/backend-core/src/context/index.js deleted file mode 100644 index bd4d857ef2..0000000000 --- a/packages/backend-core/src/context/index.js +++ /dev/null @@ -1,354 +0,0 @@ -const env = require("../environment") -const { SEPARATOR, DocumentTypes } = require("../db/constants") -const { DEFAULT_TENANT_ID } = require("../constants") -const cls = require("./FunctionContext") -const { dangerousGetDB, closeDB } = require("../db") -const { getProdAppID, getDevelopmentAppID } = require("../db/conversions") -const { baseGlobalDBName } = require("../tenancy/utils") -const { isEqual } = require("lodash") - -// some test cases call functions directly, need to -// store an app ID to pretend there is a context -let TEST_APP_ID = null - -const ContextKeys = { - TENANT_ID: "tenantId", - GLOBAL_DB: "globalDb", - APP_ID: "appId", - IDENTITY: "identity", - // whatever the request app DB was - CURRENT_DB: "currentDb", - // get the prod app DB from the request - PROD_DB: "prodDb", - // get the dev app DB from the request - DEV_DB: "devDb", - DB_OPTS: "dbOpts", - // check if something else is using the context, don't close DB - IN_USE: "inUse", -} - -exports.DEFAULT_TENANT_ID = DEFAULT_TENANT_ID - -// this function makes sure the PouchDB objects are closed and -// fully deleted when finished - this protects against memory leaks -async function closeAppDBs() { - const dbKeys = [ - ContextKeys.CURRENT_DB, - ContextKeys.PROD_DB, - ContextKeys.DEV_DB, - ] - for (let dbKey of dbKeys) { - const db = cls.getFromContext(dbKey) - if (!db) { - continue - } - await closeDB(db) - // clear the DB from context, incase someone tries to use it again - cls.setOnContext(dbKey, null) - } - // clear the app ID now that the databases are closed - if (cls.getFromContext(ContextKeys.APP_ID)) { - cls.setOnContext(ContextKeys.APP_ID, null) - } - if (cls.getFromContext(ContextKeys.DB_OPTS)) { - cls.setOnContext(ContextKeys.DB_OPTS, null) - } -} - -exports.closeTenancy = async () => { - if (env.USE_COUCH) { - await closeDB(exports.getGlobalDB()) - } - // clear from context now that database is closed/task is finished - cls.setOnContext(ContextKeys.TENANT_ID, null) - cls.setOnContext(ContextKeys.GLOBAL_DB, null) -} - -exports.isDefaultTenant = () => { - return exports.getTenantId() === exports.DEFAULT_TENANT_ID -} - -exports.isMultiTenant = () => { - return env.MULTI_TENANCY -} - -// used for automations, API endpoints should always be in context already -exports.doInTenant = (tenantId, task, { forceNew } = {}) => { - // the internal function is so that we can re-use an existing - // context - don't want to close DB on a parent context - async function internal(opts = { existing: false }) { - // set the tenant id - if (!opts.existing) { - exports.updateTenantId(tenantId) - } - - try { - // invoke the task - return await task() - } finally { - const using = cls.getFromContext(ContextKeys.IN_USE) - if (!using || using <= 1) { - await exports.closeTenancy() - } else { - cls.setOnContext(using - 1) - } - } - } - - const using = cls.getFromContext(ContextKeys.IN_USE) - if ( - !forceNew && - using && - cls.getFromContext(ContextKeys.TENANT_ID) === tenantId - ) { - cls.setOnContext(ContextKeys.IN_USE, using + 1) - return internal({ existing: true }) - } else { - return cls.run(async () => { - cls.setOnContext(ContextKeys.IN_USE, 1) - return internal() - }) - } -} - -/** - * Given an app ID this will attempt to retrieve the tenant ID from it. - * @return {null|string} The tenant ID found within the app ID. - */ -exports.getTenantIDFromAppID = appId => { - if (!appId) { - return null - } - const split = appId.split(SEPARATOR) - const hasDev = split[1] === DocumentTypes.DEV - if ((hasDev && split.length === 3) || (!hasDev && split.length === 2)) { - return null - } - if (hasDev) { - return split[2] - } else { - return split[1] - } -} - -const setAppTenantId = appId => { - const appTenantId = - exports.getTenantIDFromAppID(appId) || exports.DEFAULT_TENANT_ID - exports.updateTenantId(appTenantId) -} - -exports.doInAppContext = (appId, task, { forceNew } = {}) => { - if (!appId) { - throw new Error("appId is required") - } - - const identity = exports.getIdentity() - - // the internal function is so that we can re-use an existing - // context - don't want to close DB on a parent context - async function internal(opts = { existing: false }) { - // set the app tenant id - if (!opts.existing) { - setAppTenantId(appId) - } - // set the app ID - cls.setOnContext(ContextKeys.APP_ID, appId) - // preserve the identity - exports.setIdentity(identity) - try { - // invoke the task - return await task() - } finally { - const using = cls.getFromContext(ContextKeys.IN_USE) - if (!using || using <= 1) { - await closeAppDBs() - } else { - cls.setOnContext(using - 1) - } - } - } - const using = cls.getFromContext(ContextKeys.IN_USE) - if (!forceNew && using && cls.getFromContext(ContextKeys.APP_ID) === appId) { - cls.setOnContext(ContextKeys.IN_USE, using + 1) - return internal({ existing: true }) - } else { - return cls.run(async () => { - cls.setOnContext(ContextKeys.IN_USE, 1) - return internal() - }) - } -} - -exports.doInIdentityContext = (identity, task) => { - if (!identity) { - throw new Error("identity is required") - } - - async function internal(opts = { existing: false }) { - if (!opts.existing) { - cls.setOnContext(ContextKeys.IDENTITY, identity) - // set the tenant so that doInTenant will preserve identity - if (identity.tenantId) { - exports.updateTenantId(identity.tenantId) - } - } - - try { - // invoke the task - return await task() - } finally { - const using = cls.getFromContext(ContextKeys.IN_USE) - if (!using || using <= 1) { - exports.setIdentity(null) - } else { - cls.setOnContext(using - 1) - } - } - } - - const existing = cls.getFromContext(ContextKeys.IDENTITY) - const using = cls.getFromContext(ContextKeys.IN_USE) - if (using && existing && existing._id === identity._id) { - cls.setOnContext(ContextKeys.IN_USE, using + 1) - return internal({ existing: true }) - } else { - return cls.run(async () => { - cls.setOnContext(ContextKeys.IN_USE, 1) - return internal({ existing: false }) - }) - } -} - -exports.setIdentity = identity => { - cls.setOnContext(ContextKeys.IDENTITY, identity) -} - -exports.getIdentity = () => { - try { - return cls.getFromContext(ContextKeys.IDENTITY) - } catch (e) { - // do nothing - identity is not in context - } -} - -exports.updateTenantId = tenantId => { - cls.setOnContext(ContextKeys.TENANT_ID, tenantId) - if (env.USE_COUCH) { - exports.setGlobalDB(tenantId) - } -} - -exports.updateAppId = async appId => { - try { - // have to close first, before removing the databases from context - await closeAppDBs() - cls.setOnContext(ContextKeys.APP_ID, appId) - } catch (err) { - if (env.isTest()) { - TEST_APP_ID = appId - } else { - throw err - } - } -} - -exports.setGlobalDB = tenantId => { - const dbName = baseGlobalDBName(tenantId) - const db = dangerousGetDB(dbName) - cls.setOnContext(ContextKeys.GLOBAL_DB, db) - return db -} - -exports.getGlobalDB = () => { - const db = cls.getFromContext(ContextKeys.GLOBAL_DB) - if (!db) { - throw new Error("Global DB not found") - } - return db -} - -exports.isTenantIdSet = () => { - const tenantId = cls.getFromContext(ContextKeys.TENANT_ID) - return !!tenantId -} - -exports.getTenantId = () => { - if (!exports.isMultiTenant()) { - return exports.DEFAULT_TENANT_ID - } - const tenantId = cls.getFromContext(ContextKeys.TENANT_ID) - if (!tenantId) { - throw new Error("Tenant id not found") - } - return tenantId -} - -exports.getAppId = () => { - const foundId = cls.getFromContext(ContextKeys.APP_ID) - if (!foundId && env.isTest() && TEST_APP_ID) { - return TEST_APP_ID - } else { - return foundId - } -} - -function getContextDB(key, opts) { - const dbOptsKey = `${key}${ContextKeys.DB_OPTS}` - let storedOpts = cls.getFromContext(dbOptsKey) - let db = cls.getFromContext(key) - if (db && isEqual(opts, storedOpts)) { - return db - } - - const appId = exports.getAppId() - let toUseAppId - - switch (key) { - case ContextKeys.CURRENT_DB: - toUseAppId = appId - break - case ContextKeys.PROD_DB: - toUseAppId = getProdAppID(appId) - break - case ContextKeys.DEV_DB: - toUseAppId = getDevelopmentAppID(appId) - break - } - - db = dangerousGetDB(toUseAppId, opts) - try { - cls.setOnContext(key, db) - if (opts) { - cls.setOnContext(dbOptsKey, opts) - } - } catch (err) { - if (!env.isTest()) { - throw err - } - } - return db -} - -/** - * Opens the app database based on whatever the request - * contained, dev or prod. - */ -exports.getAppDB = (opts = null) => { - return getContextDB(ContextKeys.CURRENT_DB, opts) -} - -/** - * This specifically gets the prod app ID, if the request - * contained a development app ID, this will open the prod one. - */ -exports.getProdAppDB = (opts = null) => { - return getContextDB(ContextKeys.PROD_DB, opts) -} - -/** - * This specifically gets the dev app ID, if the request - * contained a prod app ID, this will open the dev one. - */ -exports.getDevAppDB = (opts = null) => { - return getContextDB(ContextKeys.DEV_DB, opts) -} diff --git a/packages/backend-core/src/context/index.ts b/packages/backend-core/src/context/index.ts new file mode 100644 index 0000000000..e0db18dde6 --- /dev/null +++ b/packages/backend-core/src/context/index.ts @@ -0,0 +1,247 @@ +import env from "../environment" +import { SEPARATOR, DocumentTypes } from "../db/constants" +import cls from "./FunctionContext" +import { dangerousGetDB, closeDB } from "../db" +import { baseGlobalDBName } from "../tenancy/utils" +import { IdentityContext } from "@budibase/types" +import { DEFAULT_TENANT_ID as _DEFAULT_TENANT_ID } from "../constants" +import { ContextKeys } from "./constants" +import { + updateUsing, + closeWithUsing, + setAppTenantId, + setIdentity, + closeAppDBs, + getContextDB, +} from "./utils" + +export const DEFAULT_TENANT_ID = _DEFAULT_TENANT_ID + +// some test cases call functions directly, need to +// store an app ID to pretend there is a context +let TEST_APP_ID: string | null = null + +export const closeTenancy = async () => { + let db + try { + if (env.USE_COUCH) { + db = getGlobalDB() + } + } catch (err) { + // no DB found - skip closing + return + } + await closeDB(db) + // clear from context now that database is closed/task is finished + cls.setOnContext(ContextKeys.TENANT_ID, null) + cls.setOnContext(ContextKeys.GLOBAL_DB, null) +} + +// export const isDefaultTenant = () => { +// return getTenantId() === DEFAULT_TENANT_ID +// } + +export const isMultiTenant = () => { + return env.MULTI_TENANCY +} + +/** + * Given an app ID this will attempt to retrieve the tenant ID from it. + * @return {null|string} The tenant ID found within the app ID. + */ +export const getTenantIDFromAppID = (appId: string) => { + if (!appId) { + return null + } + const split = appId.split(SEPARATOR) + const hasDev = split[1] === DocumentTypes.DEV + if ((hasDev && split.length === 3) || (!hasDev && split.length === 2)) { + return null + } + if (hasDev) { + return split[2] + } else { + return split[1] + } +} + +// used for automations, API endpoints should always be in context already +export const doInTenant = (tenantId: string | null, task: any) => { + // the internal function is so that we can re-use an existing + // context - don't want to close DB on a parent context + async function internal(opts = { existing: false }) { + // set the tenant id + global db if this is a new context + if (!opts.existing) { + updateTenantId(tenantId) + } + + try { + // invoke the task + return await task() + } finally { + await closeWithUsing(ContextKeys.TENANCY_IN_USE, () => { + return closeTenancy() + }) + } + } + + const existing = cls.getFromContext(ContextKeys.TENANT_ID) === tenantId + return updateUsing(ContextKeys.TENANCY_IN_USE, existing, internal) +} + +export const doInAppContext = (appId: string, task: any) => { + if (!appId) { + throw new Error("appId is required") + } + + const identity = getIdentity() + + // the internal function is so that we can re-use an existing + // context - don't want to close DB on a parent context + async function internal(opts = { existing: false }) { + // set the app tenant id + if (!opts.existing) { + setAppTenantId(appId) + } + // set the app ID + cls.setOnContext(ContextKeys.APP_ID, appId) + + // preserve the identity + if (identity) { + setIdentity(identity) + } + try { + // invoke the task + return await task() + } finally { + await closeWithUsing(ContextKeys.APP_IN_USE, async () => { + await closeAppDBs() + await closeTenancy() + }) + } + } + const existing = cls.getFromContext(ContextKeys.APP_ID) === appId + return updateUsing(ContextKeys.APP_IN_USE, existing, internal) +} + +export const doInIdentityContext = (identity: IdentityContext, task: any) => { + if (!identity) { + throw new Error("identity is required") + } + + async function internal(opts = { existing: false }) { + if (!opts.existing) { + cls.setOnContext(ContextKeys.IDENTITY, identity) + // set the tenant so that doInTenant will preserve identity + if (identity.tenantId) { + updateTenantId(identity.tenantId) + } + } + + try { + // invoke the task + return await task() + } finally { + await closeWithUsing(ContextKeys.IDENTITY_IN_USE, async () => { + setIdentity(null) + await closeTenancy() + }) + } + } + + const existing = cls.getFromContext(ContextKeys.IDENTITY) + return updateUsing(ContextKeys.IDENTITY_IN_USE, existing, internal) +} + +export const getIdentity = (): IdentityContext | undefined => { + try { + return cls.getFromContext(ContextKeys.IDENTITY) + } catch (e) { + // do nothing - identity is not in context + } +} + +export const updateTenantId = (tenantId: string | null) => { + cls.setOnContext(ContextKeys.TENANT_ID, tenantId) + if (env.USE_COUCH) { + setGlobalDB(tenantId) + } +} + +export const updateAppId = async (appId: string) => { + try { + // have to close first, before removing the databases from context + await closeAppDBs() + cls.setOnContext(ContextKeys.APP_ID, appId) + } catch (err) { + if (env.isTest()) { + TEST_APP_ID = appId + } else { + throw err + } + } +} + +export const setGlobalDB = (tenantId: string | null) => { + const dbName = baseGlobalDBName(tenantId) + const db = dangerousGetDB(dbName) + cls.setOnContext(ContextKeys.GLOBAL_DB, db) + return db +} + +export const getGlobalDB = () => { + const db = cls.getFromContext(ContextKeys.GLOBAL_DB) + if (!db) { + throw new Error("Global DB not found") + } + return db +} + +export const isTenantIdSet = () => { + const tenantId = cls.getFromContext(ContextKeys.TENANT_ID) + return !!tenantId +} + +export const getTenantId = () => { + if (!isMultiTenant()) { + return DEFAULT_TENANT_ID + } + const tenantId = cls.getFromContext(ContextKeys.TENANT_ID) + if (!tenantId) { + throw new Error("Tenant id not found") + } + return tenantId +} + +export const getAppId = () => { + const foundId = cls.getFromContext(ContextKeys.APP_ID) + if (!foundId && env.isTest() && TEST_APP_ID) { + return TEST_APP_ID + } else { + return foundId + } +} + +/** + * Opens the app database based on whatever the request + * contained, dev or prod. + */ +export const getAppDB = (opts?: any) => { + return getContextDB(ContextKeys.CURRENT_DB, opts) +} + +/** + * This specifically gets the prod app ID, if the request + * contained a development app ID, this will open the prod one. + */ +export const getProdAppDB = (opts?: any) => { + return getContextDB(ContextKeys.PROD_DB, opts) +} + +/** + * This specifically gets the dev app ID, if the request + * contained a prod app ID, this will open the dev one. + */ +export const getDevAppDB = (opts?: any) => { + return getContextDB(ContextKeys.DEV_DB, opts) +} diff --git a/packages/backend-core/src/context/tests/index.spec.ts b/packages/backend-core/src/context/tests/index.spec.ts new file mode 100644 index 0000000000..55ecd333a3 --- /dev/null +++ b/packages/backend-core/src/context/tests/index.spec.ts @@ -0,0 +1,148 @@ +import "../../../tests/utilities/TestConfiguration" +import * as context from ".." +import { DEFAULT_TENANT_ID } from "../../constants" +import env from "../../environment" + +// must use require to spy index file exports due to known issue in jest +const dbUtils = require("../../db") +jest.spyOn(dbUtils, "closeDB") +jest.spyOn(dbUtils, "dangerousGetDB") + +describe("context", () => { + beforeEach(() => { + jest.clearAllMocks() + }) + + describe("doInTenant", () => { + describe("single-tenancy", () => { + it("defaults to the default tenant", () => { + const tenantId = context.getTenantId() + expect(tenantId).toBe(DEFAULT_TENANT_ID) + }) + + it("defaults to the default tenant db", async () => { + await context.doInTenant(DEFAULT_TENANT_ID, () => { + const db = context.getGlobalDB() + expect(db.name).toBe("global-db") + }) + expect(dbUtils.dangerousGetDB).toHaveBeenCalledTimes(1) + expect(dbUtils.closeDB).toHaveBeenCalledTimes(1) + }) + }) + + describe("multi-tenancy", () => { + beforeEach(() => { + env._set("MULTI_TENANCY", 1) + }) + + it("fails when no tenant id is set", () => { + const test = () => { + let error + try { + context.getTenantId() + } catch (e: any) { + error = e + } + expect(error.message).toBe("Tenant id not found") + } + + // test under no tenancy + test() + + // test after tenancy has been accessed to ensure cleanup + context.doInTenant("test", () => {}) + test() + }) + + it("fails when no tenant db is set", () => { + const test = () => { + let error + try { + context.getGlobalDB() + } catch (e: any) { + error = e + } + expect(error.message).toBe("Global DB not found") + } + + // test under no tenancy + test() + + // test after tenancy has been accessed to ensure cleanup + context.doInTenant("test", () => {}) + test() + }) + + it("sets tenant id", () => { + context.doInTenant("test", () => { + const tenantId = context.getTenantId() + expect(tenantId).toBe("test") + }) + }) + + it("initialises the tenant db", async () => { + await context.doInTenant("test", () => { + const db = context.getGlobalDB() + expect(db.name).toBe("test_global-db") + }) + expect(dbUtils.dangerousGetDB).toHaveBeenCalledTimes(1) + expect(dbUtils.closeDB).toHaveBeenCalledTimes(1) + }) + + it("sets the tenant id when nested with same tenant id", async () => { + await context.doInTenant("test", async () => { + const tenantId = context.getTenantId() + expect(tenantId).toBe("test") + + await context.doInTenant("test", async () => { + const tenantId = context.getTenantId() + expect(tenantId).toBe("test") + + await context.doInTenant("test", () => { + const tenantId = context.getTenantId() + expect(tenantId).toBe("test") + }) + }) + }) + }) + + it("initialises the tenant db when nested with same tenant id", async () => { + await context.doInTenant("test", async () => { + const db = context.getGlobalDB() + expect(db.name).toBe("test_global-db") + + await context.doInTenant("test", async () => { + const db = context.getGlobalDB() + expect(db.name).toBe("test_global-db") + + await context.doInTenant("test", () => { + const db = context.getGlobalDB() + expect(db.name).toBe("test_global-db") + }) + }) + }) + + // only 1 db is opened and closed + expect(dbUtils.dangerousGetDB).toHaveBeenCalledTimes(1) + expect(dbUtils.closeDB).toHaveBeenCalledTimes(1) + }) + + it("sets different tenant id inside another context", () => { + context.doInTenant("test", () => { + const tenantId = context.getTenantId() + expect(tenantId).toBe("test") + + context.doInTenant("nested", () => { + const tenantId = context.getTenantId() + expect(tenantId).toBe("nested") + + context.doInTenant("double-nested", () => { + const tenantId = context.getTenantId() + expect(tenantId).toBe("double-nested") + }) + }) + }) + }) + }) + }) +}) diff --git a/packages/backend-core/src/context/utils.ts b/packages/backend-core/src/context/utils.ts new file mode 100644 index 0000000000..62693f18e8 --- /dev/null +++ b/packages/backend-core/src/context/utils.ts @@ -0,0 +1,113 @@ +import { + DEFAULT_TENANT_ID, + getAppId, + getTenantIDFromAppID, + updateTenantId, +} from "./index" +import cls from "./FunctionContext" +import { IdentityContext } from "@budibase/types" +import { ContextKeys } from "./constants" +import { dangerousGetDB, closeDB } from "../db" +import { isEqual } from "lodash" +import { getDevelopmentAppID, getProdAppID } from "../db/conversions" +import env from "../environment" + +export async function updateUsing( + usingKey: string, + existing: boolean, + internal: (opts: { existing: boolean }) => Promise +) { + const using = cls.getFromContext(usingKey) + if (using && existing) { + cls.setOnContext(usingKey, using + 1) + return internal({ existing: true }) + } else { + return cls.run(async () => { + cls.setOnContext(usingKey, 1) + return internal({ existing: false }) + }) + } +} + +export async function closeWithUsing( + usingKey: string, + closeFn: () => Promise +) { + const using = cls.getFromContext(usingKey) + if (!using || using <= 1) { + await closeFn() + } else { + cls.setOnContext(usingKey, using - 1) + } +} + +export const setAppTenantId = (appId: string) => { + const appTenantId = getTenantIDFromAppID(appId) || DEFAULT_TENANT_ID + updateTenantId(appTenantId) +} + +export const setIdentity = (identity: IdentityContext | null) => { + cls.setOnContext(ContextKeys.IDENTITY, identity) +} + +// this function makes sure the PouchDB objects are closed and +// fully deleted when finished - this protects against memory leaks +export async function closeAppDBs() { + const dbKeys = [ + ContextKeys.CURRENT_DB, + ContextKeys.PROD_DB, + ContextKeys.DEV_DB, + ] + for (let dbKey of dbKeys) { + const db = cls.getFromContext(dbKey) + if (!db) { + continue + } + await closeDB(db) + // clear the DB from context, incase someone tries to use it again + cls.setOnContext(dbKey, null) + } + // clear the app ID now that the databases are closed + if (cls.getFromContext(ContextKeys.APP_ID)) { + cls.setOnContext(ContextKeys.APP_ID, null) + } + if (cls.getFromContext(ContextKeys.DB_OPTS)) { + cls.setOnContext(ContextKeys.DB_OPTS, null) + } +} + +export function getContextDB(key: string, opts: any) { + const dbOptsKey = `${key}${ContextKeys.DB_OPTS}` + let storedOpts = cls.getFromContext(dbOptsKey) + let db = cls.getFromContext(key) + if (db && isEqual(opts, storedOpts)) { + return db + } + + const appId = getAppId() + let toUseAppId + + switch (key) { + case ContextKeys.CURRENT_DB: + toUseAppId = appId + break + case ContextKeys.PROD_DB: + toUseAppId = getProdAppID(appId) + break + case ContextKeys.DEV_DB: + toUseAppId = getDevelopmentAppID(appId) + break + } + db = dangerousGetDB(toUseAppId, opts) + try { + cls.setOnContext(key, db) + if (opts) { + cls.setOnContext(dbOptsKey, opts) + } + } catch (err) { + if (!env.isTest()) { + throw err + } + } + return db +} diff --git a/packages/backend-core/src/db/constants.ts b/packages/backend-core/src/db/constants.ts index be0e824e61..716762dd45 100644 --- a/packages/backend-core/src/db/constants.ts +++ b/packages/backend-core/src/db/constants.ts @@ -11,7 +11,7 @@ export enum AutomationViewModes { } export enum ViewNames { - USER_BY_EMAIL = "by_email", + USER_BY_EMAIL = "by_email2", BY_API_KEY = "by_api_key", USER_BY_BUILDERS = "by_builders", LINK = "by_link", @@ -19,6 +19,13 @@ export enum ViewNames { AUTOMATION_LOGS = "automation_logs", } +export const DeprecatedViews = { + [ViewNames.USER_BY_EMAIL]: [ + // removed due to inaccuracy in view doc filter logic + "by_email", + ], +} + export enum DocumentTypes { USER = "us", WORKSPACE = "workspace", diff --git a/packages/backend-core/src/db/index.js b/packages/backend-core/src/db/index.js index 8124be979e..aa6f7ebc2c 100644 --- a/packages/backend-core/src/db/index.js +++ b/packages/backend-core/src/db/index.js @@ -1,10 +1,18 @@ const pouch = require("./pouch") const env = require("../environment") +const openDbs = [] let PouchDB let initialised = false const dbList = new Set() +if (env.MEMORY_LEAK_CHECK) { + setInterval(() => { + console.log("--- OPEN DBS ---") + console.log(openDbs) + }, 5000) +} + const put = dbPut => async (doc, options = {}) => { @@ -35,6 +43,9 @@ exports.dangerousGetDB = (dbName, opts) => { dbList.add(dbName) } const db = new PouchDB(dbName, opts) + if (env.MEMORY_LEAK_CHECK) { + openDbs.push(db.name) + } const dbPut = db.put db.put = put(dbPut) return db @@ -46,6 +57,9 @@ exports.closeDB = async db => { if (!db || env.isTest()) { return } + if (env.MEMORY_LEAK_CHECK) { + openDbs.splice(openDbs.indexOf(db.name), 1) + } try { // specifically await so that if there is an error, it can be ignored return await db.close() diff --git a/packages/backend-core/src/db/views.js b/packages/backend-core/src/db/views.js index e0281c6584..1e8dd7ee77 100644 --- a/packages/backend-core/src/db/views.js +++ b/packages/backend-core/src/db/views.js @@ -1,20 +1,42 @@ -const { DocumentTypes, ViewNames } = require("./utils") +const { + DocumentTypes, + ViewNames, + DeprecatedViews, + SEPARATOR, +} = require("./utils") const { getGlobalDB } = require("../tenancy") +const DESIGN_DB = "_design/database" + function DesignDoc() { return { - _id: "_design/database", + _id: DESIGN_DB, // view collation information, read before writing any complex views: // https://docs.couchdb.org/en/master/ddocs/views/collation.html#collation-specification views: {}, } } -exports.createUserEmailView = async () => { +async function removeDeprecated(db, viewName) { + if (!DeprecatedViews[viewName]) { + return + } + try { + const designDoc = await db.get(DESIGN_DB) + for (let deprecatedNames of DeprecatedViews[viewName]) { + delete designDoc.views[deprecatedNames] + } + await db.put(designDoc) + } catch (err) { + // doesn't exist, ignore + } +} + +exports.createNewUserEmailView = async () => { const db = getGlobalDB() let designDoc try { - designDoc = await db.get("_design/database") + designDoc = await db.get(DESIGN_DB) } catch (err) { // no design doc, make one designDoc = DesignDoc() @@ -22,7 +44,7 @@ exports.createUserEmailView = async () => { const view = { // if using variables in a map function need to inject them before use map: `function(doc) { - if (doc._id.startsWith("${DocumentTypes.USER}")) { + if (doc._id.startsWith("${DocumentTypes.USER}${SEPARATOR}")) { emit(doc.email.toLowerCase(), doc._id) } }`, @@ -81,7 +103,7 @@ exports.createUserBuildersView = async () => { exports.queryGlobalView = async (viewName, params, db = null) => { const CreateFuncByName = { - [ViewNames.USER_BY_EMAIL]: exports.createUserEmailView, + [ViewNames.USER_BY_EMAIL]: exports.createNewUserEmailView, [ViewNames.BY_API_KEY]: exports.createApiKeyView, [ViewNames.USER_BY_BUILDERS]: exports.createUserBuildersView, } @@ -98,6 +120,7 @@ exports.queryGlobalView = async (viewName, params, db = null) => { } catch (err) { if (err != null && err.name === "not_found") { const createFunc = CreateFuncByName[viewName] + await removeDeprecated(db, viewName) await createFunc() return exports.queryGlobalView(viewName, params) } else { diff --git a/packages/backend-core/src/environment.ts b/packages/backend-core/src/environment.ts index 845504fdc9..37804b31a6 100644 --- a/packages/backend-core/src/environment.ts +++ b/packages/backend-core/src/environment.ts @@ -54,6 +54,7 @@ const env = { DISABLE_DEVELOPER_LICENSE: process.env.DISABLE_DEVELOPER_LICENSE, DEFAULT_LICENSE: process.env.DEFAULT_LICENSE, SERVICE: process.env.SERVICE || "budibase", + MEMORY_LEAK_CHECK: process.env.MEMORY_LEAK_CHECK || false, DEPLOYMENT_ENVIRONMENT: process.env.DEPLOYMENT_ENVIRONMENT || "docker-compose", _set(key: any, value: any) { diff --git a/packages/backend-core/src/events/processors/PosthogProcessor.ts b/packages/backend-core/src/events/processors/PosthogProcessor.ts index 67407fdd5c..eb12db1dc4 100644 --- a/packages/backend-core/src/events/processors/PosthogProcessor.ts +++ b/packages/backend-core/src/events/processors/PosthogProcessor.ts @@ -2,7 +2,7 @@ import PostHog from "posthog-node" import { Event, Identity, Group, BaseEvent } from "@budibase/types" import { EventProcessor } from "./types" import env from "../../environment" -import context from "../../context" +import * as context from "../../context" const pkg = require("../../../package.json") export default class PosthogProcessor implements EventProcessor { diff --git a/packages/backend-core/src/migrations/migrations.ts b/packages/backend-core/src/migrations/migrations.ts index b926334317..2e4ef0da76 100644 --- a/packages/backend-core/src/migrations/migrations.ts +++ b/packages/backend-core/src/migrations/migrations.ts @@ -9,7 +9,7 @@ import { getGlobalDBName, getTenantId, } from "../tenancy" -import context from "../context" +import * as context from "../context" import { DEFINITIONS } from "." import { Migration, diff --git a/packages/backend-core/yarn.lock b/packages/backend-core/yarn.lock index 77dbc61425..e1f38a798f 100644 --- a/packages/backend-core/yarn.lock +++ b/packages/backend-core/yarn.lock @@ -764,6 +764,11 @@ "@types/koa-compose" "*" "@types/node" "*" +"@types/lodash@4.14.180": + version "4.14.180" + resolved "https://registry.yarnpkg.com/@types/lodash/-/lodash-4.14.180.tgz#4ab7c9ddfc92ec4a887886483bc14c79fb380670" + integrity sha512-XOKXa1KIxtNXgASAnwj7cnttJxS4fksBRywK/9LzRV5YxrF80BXZIGeQSuoESQ/VkUj30Ae0+YcuHc15wJCB2g== + "@types/mime@^1": version "1.3.2" resolved "https://registry.yarnpkg.com/@types/mime/-/mime-1.3.2.tgz#93e25bf9ee75fe0fd80b594bc4feb0e862111b5a" diff --git a/packages/bbui/package.json b/packages/bbui/package.json index f1f84b1cf9..4b986d28dc 100644 --- a/packages/bbui/package.json +++ b/packages/bbui/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/bbui", "description": "A UI solution used in the different Budibase projects.", - "version": "1.0.220-alpha.4", + "version": "1.1.10-alpha.3", "license": "MPL-2.0", "svelte": "src/index.js", "module": "dist/bbui.es.js", @@ -38,7 +38,7 @@ ], "dependencies": { "@adobe/spectrum-css-workflow-icons": "^1.2.1", - "@budibase/string-templates": "^1.0.220-alpha.4", + "@budibase/string-templates": "^1.1.10-alpha.3", "@spectrum-css/actionbutton": "^1.0.1", "@spectrum-css/actiongroup": "^1.0.1", "@spectrum-css/avatar": "^3.0.2", diff --git a/packages/builder/package.json b/packages/builder/package.json index e22d58f25c..6d81b42b8b 100644 --- a/packages/builder/package.json +++ b/packages/builder/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/builder", - "version": "1.0.220-alpha.4", + "version": "1.1.10-alpha.3", "license": "GPL-3.0", "private": true, "scripts": { @@ -69,10 +69,10 @@ } }, "dependencies": { - "@budibase/bbui": "^1.0.220-alpha.4", - "@budibase/client": "^1.0.220-alpha.4", - "@budibase/frontend-core": "^1.0.220-alpha.4", - "@budibase/string-templates": "^1.0.220-alpha.4", + "@budibase/bbui": "^1.1.10-alpha.3", + "@budibase/client": "^1.1.10-alpha.3", + "@budibase/frontend-core": "^1.1.10-alpha.3", + "@budibase/string-templates": "^1.1.10-alpha.3", "@sentry/browser": "5.19.1", "@spectrum-css/page": "^3.0.1", "@spectrum-css/vars": "^3.0.1", diff --git a/packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowItemHeader.svelte b/packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowItemHeader.svelte index e8b61d7402..99c6f251f9 100644 --- a/packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowItemHeader.svelte +++ b/packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowItemHeader.svelte @@ -96,7 +96,7 @@ onSelect(block) }} > - + diff --git a/packages/builder/src/components/automation/AutomationBuilder/TestDisplay.svelte b/packages/builder/src/components/automation/AutomationBuilder/TestDisplay.svelte index a2eb904c94..c6585b0bce 100644 --- a/packages/builder/src/components/automation/AutomationBuilder/TestDisplay.svelte +++ b/packages/builder/src/components/automation/AutomationBuilder/TestDisplay.svelte @@ -67,27 +67,20 @@ {/if}
- + -
-