From 2426123686c95a80d671c898cbcdaccf7f1c1592 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 7 Nov 2022 18:27:39 +0000 Subject: [PATCH 01/24] First version of pouch-like system, a wrapper for Couch nano. --- packages/backend-core/package.json | 1 + packages/backend-core/src/couch/couch.ts | 35 +++++ packages/backend-core/src/couch/index.ts | 3 + packages/backend-core/src/couch/pouchLike.ts | 133 ++++++++++++++++++ packages/backend-core/src/couch/utils.ts | 36 +++++ packages/backend-core/src/db/index.ts | 29 +--- packages/backend-core/src/db/pouch.ts | 35 +---- packages/backend-core/yarn.lock | 137 ++++++++++++++++++- packages/types/src/documents/document.ts | 4 + 9 files changed, 351 insertions(+), 62 deletions(-) create mode 100644 packages/backend-core/src/couch/couch.ts create mode 100644 packages/backend-core/src/couch/index.ts create mode 100644 packages/backend-core/src/couch/pouchLike.ts create mode 100644 packages/backend-core/src/couch/utils.ts diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json index 43e34142dc..d8575dcad3 100644 --- a/packages/backend-core/package.json +++ b/packages/backend-core/package.json @@ -35,6 +35,7 @@ "koa-passport": "4.1.4", "lodash": "4.17.21", "lodash.isarguments": "3.1.0", + "nano": "^10.1.0", "node-fetch": "2.6.7", "passport-google-auth": "1.0.2", "passport-google-oauth": "2.0.0", diff --git a/packages/backend-core/src/couch/couch.ts b/packages/backend-core/src/couch/couch.ts new file mode 100644 index 0000000000..41fb83cb86 --- /dev/null +++ b/packages/backend-core/src/couch/couch.ts @@ -0,0 +1,35 @@ +import env from "../environment" +import { getUrlInfo } from "../db/pouch" + +export const getCouchInfo = () => { + const urlInfo = getUrlInfo() + let username + let password + if (env.COUCH_DB_USERNAME) { + // set from env + username = env.COUCH_DB_USERNAME + } else if (urlInfo.auth.username) { + // set from url + username = urlInfo.auth.username + } else if (!env.isTest()) { + throw new Error("CouchDB username not set") + } + if (env.COUCH_DB_PASSWORD) { + // set from env + password = env.COUCH_DB_PASSWORD + } else if (urlInfo.auth.password) { + // set from url + password = urlInfo.auth.password + } else if (!env.isTest()) { + throw new Error("CouchDB password not set") + } + const authCookie = Buffer.from(`${username}:${password}`).toString("base64") + return { + url: urlInfo.url!, + auth: { + username: username, + password: password, + }, + cookie: `Basic ${authCookie}`, + } +} diff --git a/packages/backend-core/src/couch/index.ts b/packages/backend-core/src/couch/index.ts new file mode 100644 index 0000000000..daba8f74a9 --- /dev/null +++ b/packages/backend-core/src/couch/index.ts @@ -0,0 +1,3 @@ +export * from "./couch" +export * from "./pouchLike" +export * from "./utils" diff --git a/packages/backend-core/src/couch/pouchLike.ts b/packages/backend-core/src/couch/pouchLike.ts new file mode 100644 index 0000000000..80ebe1d363 --- /dev/null +++ b/packages/backend-core/src/couch/pouchLike.ts @@ -0,0 +1,133 @@ +import Nano from "nano" +import { AnyDocument } from "@budibase/types" +import { getCouchInfo } from "./couch" +import { directCouchCall } from "./utils" +import { ReadStream, WriteStream } from "fs" +import { dangerousGetDB } from "../db" + +export type PouchLikeOpts = { + skip_setup?: boolean +} + +export type QueryOpts = { + include_docs?: boolean + startkey?: string + endkey?: string + limit?: number + skip?: number + descending?: boolean + key?: string + keys?: string[] +} + +export type DumpOpts = { + filter?: (doc: AnyDocument) => boolean +} + +export class PouchLike { + public readonly name: string + private static nano: Nano.ServerScope + private readonly pouchOpts: PouchLikeOpts + + constructor(dbName: string, opts?: PouchLikeOpts) { + this.name = dbName + this.pouchOpts = opts || {} + } + + static init() { + const couchInfo = getCouchInfo() + this.nano = Nano({ + url: couchInfo.url, + cookie: couchInfo.cookie, + }) + } + + async checkSetup() { + let shouldCreate = !this.pouchOpts?.skip_setup + // check exists in a lightweight fashion + let response = await directCouchCall(`/${this.name}`, "HEAD") + let exists = response.status === 200 + if (!shouldCreate && !exists) { + throw new Error("DB does not exist") + } + if (!exists) { + await PouchLike.nano.db.create(this.name) + } + return PouchLike.nano.db.use(this.name) + } + + async info() {} + + async get(id: string) { + const db = await this.checkSetup() + return await db.get(id) + } + + async remove(id: string, rev: string) { + const db = await this.checkSetup() + return await db.destroy(id, rev) + } + + async put(document: AnyDocument) { + if (!document._id) { + throw new Error("Cannot store document without _id field.") + } + const db = await this.checkSetup() + return await db.insert(document) + } + + async bulkDocs(documents: AnyDocument[]) { + const db = await this.checkSetup() + return await db.bulk({ docs: documents }) + } + + async allDocs(params: QueryOpts) { + const db = await this.checkSetup() + return await db.fetch({ keys: [] }, params) + } + + async query(viewName: string, params: QueryOpts) { + const db = await this.checkSetup() + const [database, view] = viewName.split("/") + return await db.view(database, view, params) + } + + async destroy() { + try { + await PouchLike.nano.db.destroy(this.name) + } catch (err: any) { + // didn't exist, don't worry + if (err.status === 404) { + return + } else { + throw err + } + } + } + + async compact() { + const db = await this.checkSetup() + return await db.compact() + } + + // utilise PouchDB for this + async dump(stream: WriteStream, params: DumpOpts) { + const pouch = dangerousGetDB(this.name) + // @ts-ignore + return pouch.dump(stream, params) + } + + // utilise PouchDB for this + async load(stream: ReadStream) { + const pouch = dangerousGetDB(this.name) + // @ts-ignore + return pouch.load(stream) + } + + // pouch specific functions - indexes come from the pouchdb-find library + async createIndex() {} + + async deleteIndex() {} + + async getIndexes() {} +} diff --git a/packages/backend-core/src/couch/utils.ts b/packages/backend-core/src/couch/utils.ts new file mode 100644 index 0000000000..25a4643d54 --- /dev/null +++ b/packages/backend-core/src/couch/utils.ts @@ -0,0 +1,36 @@ +import { getCouchInfo } from "./couch" +import fetch from "node-fetch" +import { checkSlashesInUrl } from "../helpers" + +export async function directCouchCall( + path: string, + method: string = "GET", + body?: any +) { + let { url, cookie } = getCouchInfo() + const couchUrl = `${url}/${path}` + const params: any = { + method: method, + headers: { + Authorization: cookie, + }, + } + if (body && method !== "GET") { + params.body = JSON.stringify(body) + params.headers["Content-Type"] = "application/json" + } + return await fetch(checkSlashesInUrl(encodeURI(couchUrl)), params) +} + +export async function directCouchQuery( + path: string, + method: string = "GET", + body?: any +) { + const response = await directCouchCall(path, method, body) + if (response.status < 300) { + return await response.json() + } else { + throw "Cannot connect to CouchDB instance" + } +} diff --git a/packages/backend-core/src/db/index.ts b/packages/backend-core/src/db/index.ts index 429cd61fc1..9f6c097f0d 100644 --- a/packages/backend-core/src/db/index.ts +++ b/packages/backend-core/src/db/index.ts @@ -1,9 +1,9 @@ import * as pouch from "./pouch" import env from "../environment" -import { checkSlashesInUrl } from "../helpers" -import fetch from "node-fetch" import { PouchOptions, CouchFindOptions } from "@budibase/types" import PouchDB from "pouchdb" +import { directCouchQuery } from "../couch" +export { directCouchQuery } from "../couch" const openDbs: string[] = [] let Pouch: any @@ -94,31 +94,6 @@ export function allDbs() { return [...dbList] } -export async function directCouchQuery( - path: string, - method: string = "GET", - body?: any -) { - let { url, cookie } = pouch.getCouchInfo() - const couchUrl = `${url}/${path}` - const params: any = { - method: method, - headers: { - Authorization: cookie, - }, - } - if (body && method !== "GET") { - params.body = JSON.stringify(body) - params.headers["Content-Type"] = "application/json" - } - const response = await fetch(checkSlashesInUrl(encodeURI(couchUrl)), params) - if (response.status < 300) { - return await response.json() - } else { - throw "Cannot connect to CouchDB instance" - } -} - export async function directCouchAllDbs(queryString?: string) { let couchPath = "/_all_dbs" if (queryString) { diff --git a/packages/backend-core/src/db/pouch.ts b/packages/backend-core/src/db/pouch.ts index 1e37da9240..6eca07e0cd 100644 --- a/packages/backend-core/src/db/pouch.ts +++ b/packages/backend-core/src/db/pouch.ts @@ -1,5 +1,7 @@ import PouchDB from "pouchdb" import env from "../environment" +import { getCouchInfo } from "../couch" +export { getCouchInfo } from "../couch" export const getUrlInfo = (url = env.COUCH_DB_URL) => { let cleanUrl, username, password, host @@ -44,39 +46,6 @@ export const getUrlInfo = (url = env.COUCH_DB_URL) => { } } -export const getCouchInfo = () => { - const urlInfo = getUrlInfo() - let username - let password - if (env.COUCH_DB_USERNAME) { - // set from env - username = env.COUCH_DB_USERNAME - } else if (urlInfo.auth.username) { - // set from url - username = urlInfo.auth.username - } else if (!env.isTest()) { - throw new Error("CouchDB username not set") - } - if (env.COUCH_DB_PASSWORD) { - // set from env - password = env.COUCH_DB_PASSWORD - } else if (urlInfo.auth.password) { - // set from url - password = urlInfo.auth.password - } else if (!env.isTest()) { - throw new Error("CouchDB password not set") - } - const authCookie = Buffer.from(`${username}:${password}`).toString("base64") - return { - url: urlInfo.url, - auth: { - username: username, - password: password, - }, - cookie: `Basic ${authCookie}`, - } -} - /** * Return a constructor for PouchDB. * This should be rarely used outside of the main application config. diff --git a/packages/backend-core/yarn.lock b/packages/backend-core/yarn.lock index d301526ba1..c7a66cfce1 100644 --- a/packages/backend-core/yarn.lock +++ b/packages/backend-core/yarn.lock @@ -291,6 +291,11 @@ resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw== +"@budibase/types@2.1.14-alpha.2": + version "2.1.14-alpha.2" + resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.1.14-alpha.2.tgz#a537796012504e59afe06595f094c5ee2075de9b" + integrity sha512-6lrxQDBozX+yOWXBYN2K2Usg3liWXjmWtZ/F4Pky01dsFdD9M0GdYEUI0+Efhw78wEFaDeA2H9iXvuswKT/I6g== + "@hapi/hoek@^9.0.0": version "9.3.0" resolved "https://registry.yarnpkg.com/@hapi/hoek/-/hoek-9.3.0.tgz#8368869dcb735be2e7f5cb7647de78e167a251fb" @@ -1077,6 +1082,11 @@ dependencies: "@types/node" "*" +"@types/tough-cookie@^4.0.2": + version "4.0.2" + resolved "https://registry.yarnpkg.com/@types/tough-cookie/-/tough-cookie-4.0.2.tgz#6286b4c7228d58ab7866d19716f3696e03a09397" + integrity sha512-Q5vtl1W5ue16D+nIaW8JWebSSraJVlK+EthKn7e7UcD4KWsaSJ8BqGPXNaPghgtcn/fhvrN17Tv8ksUsQpiplw== + "@types/uuid@8.3.4": version "8.3.4" resolved "https://registry.yarnpkg.com/@types/uuid/-/uuid-8.3.4.tgz#bd86a43617df0594787d38b735f55c805becf1bc" @@ -1171,7 +1181,7 @@ acorn@^8.2.4: resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.7.1.tgz#0197122c843d1bf6d0a5e83220a788f278f63c30" integrity sha512-Xx54uLJQZ19lKygFXOWsscKUbsBZW0CPykPhVQdhIeIwrbPmJzqeASDInc8nKBnp/JT6igTs82qPXz069H8I/A== -agent-base@6: +agent-base@6, agent-base@^6.0.2: version "6.0.2" resolved "https://registry.yarnpkg.com/agent-base/-/agent-base-6.0.2.tgz#49fff58577cfee3f37176feab4c22e00f86d7f77" integrity sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ== @@ -1328,6 +1338,15 @@ axios@0.24.0: dependencies: follow-redirects "^1.14.4" +axios@^1.1.3: + version "1.1.3" + resolved "https://registry.yarnpkg.com/axios/-/axios-1.1.3.tgz#8274250dada2edf53814ed7db644b9c2866c1e35" + integrity sha512-00tXVRwKx/FZr/IDVFt4C+f9FYairX517WoGCL6dpOntqLkZofjhu43F/Xl44UOpqa+9sLFDrG/XAnFsUYgkDA== + dependencies: + follow-redirects "^1.15.0" + form-data "^4.0.0" + proxy-from-env "^1.1.0" + babel-jest@^27.5.1: version "27.5.1" resolved "https://registry.yarnpkg.com/babel-jest/-/babel-jest-27.5.1.tgz#a1bf8d61928edfefd21da27eb86a695bfd691444" @@ -1570,6 +1589,14 @@ cacheable-request@^6.0.0: normalize-url "^4.1.0" responselike "^1.0.2" +call-bind@^1.0.0: + version "1.0.2" + resolved "https://registry.yarnpkg.com/call-bind/-/call-bind-1.0.2.tgz#b1d4e89e688119c3c9a903ad30abb2f6a919be3c" + integrity sha512-7O+FbCihrB5WGbFYesctwmTKae6rOiIzmz1icreWJ+0aA7LJfuqhEso2T9ncpcFtzMQtzXf2QGGueWJGTYsqrA== + dependencies: + function-bind "^1.1.1" + get-intrinsic "^1.0.2" + callsites@^3.0.0: version "3.1.0" resolved "https://registry.yarnpkg.com/callsites/-/callsites-3.1.0.tgz#b3630abd8943432f54b3f0519238e33cd7df2f73" @@ -2292,6 +2319,11 @@ follow-redirects@^1.14.4: resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.1.tgz#0ca6a452306c9b276e4d3127483e29575e207ad5" integrity sha512-yLAMQs+k0b2m7cVxpS1VKJVvoz7SS9Td1zss3XRwXj+ZDH00RJgnuLx7E44wx02kQLrdM3aOOy+FpzS7+8OizA== +follow-redirects@^1.15.0: + version "1.15.2" + resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.2.tgz#b460864144ba63f2681096f274c4e57026da2c13" + integrity sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA== + forever-agent@~0.6.1: version "0.6.1" resolved "https://registry.yarnpkg.com/forever-agent/-/forever-agent-0.6.1.tgz#fbc71f0c41adeb37f96c577ad1ed42d8fdacca91" @@ -2306,6 +2338,15 @@ form-data@^3.0.0: combined-stream "^1.0.8" mime-types "^2.1.12" +form-data@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/form-data/-/form-data-4.0.0.tgz#93919daeaf361ee529584b9b31664dc12c9fa452" + integrity sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww== + dependencies: + asynckit "^0.4.0" + combined-stream "^1.0.8" + mime-types "^2.1.12" + form-data@~2.3.2: version "2.3.3" resolved "https://registry.yarnpkg.com/form-data/-/form-data-2.3.3.tgz#dcce52c05f644f298c6a7ab936bd724ceffbf3a6" @@ -2377,6 +2418,15 @@ get-caller-file@^2.0.5: resolved "https://registry.yarnpkg.com/get-caller-file/-/get-caller-file-2.0.5.tgz#4f94412a82db32f36e3b0b9741f8a97feb031f7e" integrity sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg== +get-intrinsic@^1.0.2: + version "1.1.3" + resolved "https://registry.yarnpkg.com/get-intrinsic/-/get-intrinsic-1.1.3.tgz#063c84329ad93e83893c7f4f243ef63ffa351385" + integrity sha512-QJVz1Tj7MS099PevUG5jvnt9tSkXN8K14dxQlikJuPt4uD9hHAHjLyLBiLR5zELelBdD9QNRAXZzsJx0WaDL9A== + dependencies: + function-bind "^1.1.1" + has "^1.0.3" + has-symbols "^1.0.3" + get-package-type@^0.1.0: version "0.1.0" resolved "https://registry.yarnpkg.com/get-package-type/-/get-package-type-0.1.0.tgz#8de2d803cff44df3bc6c456e6668b36c3926e11a" @@ -2525,7 +2575,7 @@ has-flag@^4.0.0: resolved "https://registry.yarnpkg.com/has-flag/-/has-flag-4.0.0.tgz#944771fd9c81c81265c4d6941860da06bb59479b" integrity sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ== -has-symbols@^1.0.2: +has-symbols@^1.0.2, has-symbols@^1.0.3: version "1.0.3" resolved "https://registry.yarnpkg.com/has-symbols/-/has-symbols-1.0.3.tgz#bb7b2c4349251dce87b125f7bdf874aa7c8b39f8" integrity sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A== @@ -2579,6 +2629,13 @@ http-cache-semantics@^4.0.0: resolved "https://registry.yarnpkg.com/http-cache-semantics/-/http-cache-semantics-4.1.0.tgz#49e91c5cbf36c9b94bcfcd71c23d5249ec74e390" integrity sha512-carPklcUh7ROWRK7Cv27RPtdhYhUsela/ue5/jKzjegVvXDqM2ILE9Q2BGn9JZJh1g87cp56su/FgQSzcWS8cQ== +http-cookie-agent@^4.0.2: + version "4.0.2" + resolved "https://registry.yarnpkg.com/http-cookie-agent/-/http-cookie-agent-4.0.2.tgz#dcdaae18ed1f7452d81ae4d5cd80b227d6831b69" + integrity sha512-noTmxdH5CuytTnLj/Qv3Z84e/YFq8yLXAw3pqIYZ25Edhb9pQErIAC+ednw40Cic6Le/h9ryph5/TqsvkOaUCw== + dependencies: + agent-base "^6.0.2" + http-errors@^1.6.3, http-errors@~1.8.0: version "1.8.1" resolved "https://registry.yarnpkg.com/http-errors/-/http-errors-1.8.1.tgz#7c3f28577cbc8a207388455dbd62295ed07bd68c" @@ -3984,6 +4041,18 @@ msgpackr@^1.5.2: optionalDependencies: msgpackr-extract "^2.1.2" +nano@^10.1.0: + version "10.1.0" + resolved "https://registry.yarnpkg.com/nano/-/nano-10.1.0.tgz#afdd5a7440e62f09a8e23f41fcea328d27383922" + integrity sha512-COeN2TpLcHuSN44QLnPmfZCoCsKAg8/aelPOVqqm/2/MvRHDEA11/Kld5C4sLzDlWlhFZ3SO2WGJGevCsvcEzQ== + dependencies: + "@types/tough-cookie" "^4.0.2" + axios "^1.1.3" + http-cookie-agent "^4.0.2" + node-abort-controller "^3.0.1" + qs "^6.11.0" + tough-cookie "^4.1.2" + napi-macros@~2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/napi-macros/-/napi-macros-2.0.0.tgz#2b6bae421e7b96eb687aa6c77a7858640670001b" @@ -4009,6 +4078,11 @@ negotiator@0.6.3: resolved "https://registry.yarnpkg.com/negotiator/-/negotiator-0.6.3.tgz#58e323a72fedc0d6f9cd4d31fe49f51479590ccd" integrity sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg== +node-abort-controller@^3.0.1: + version "3.0.1" + resolved "https://registry.yarnpkg.com/node-abort-controller/-/node-abort-controller-3.0.1.tgz#f91fa50b1dee3f909afabb7e261b1e1d6b0cb74e" + integrity sha512-/ujIVxthRs+7q6hsdjHMaj8hRG9NuWmwrz+JdRwZ14jdFoKSkm+vDsCbF9PLpnSqjaWQJuTmVtcWHNLr+vrOFw== + node-addon-api@^3.1.0: version "3.2.1" resolved "https://registry.yarnpkg.com/node-addon-api/-/node-addon-api-3.2.1.tgz#81325e0a2117789c0128dab65e7e38f07ceba161" @@ -4144,6 +4218,11 @@ object-assign@^4.1.1: resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-4.1.1.tgz#2109adc7965887cfc05cbbd442cac8bfbb360863" integrity sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg== +object-inspect@^1.9.0: + version "1.12.2" + resolved "https://registry.yarnpkg.com/object-inspect/-/object-inspect-1.12.2.tgz#c0641f26394532f28ab8d796ab954e43c009a8ea" + integrity sha512-z+cPxW0QGUp0mcqcsgQyLVRDoXFQbXOwBaqyF7VIgI4TWNQsDHrBpUQslRmIfAoYWdYzs6UlKJtB2XJpTaNSpQ== + on-finished@^2.3.0: version "2.4.1" resolved "https://registry.yarnpkg.com/on-finished/-/on-finished-2.4.1.tgz#58c8c44116e54845ad57f14ab10b03533184ac3f" @@ -4646,6 +4725,11 @@ prompts@^2.0.1: kleur "^3.0.3" sisteransi "^1.0.5" +proxy-from-env@^1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/proxy-from-env/-/proxy-from-env-1.1.0.tgz#e102f16ca355424865755d2c9e8ea4f24d58c3e2" + integrity sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg== + prr@~1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/prr/-/prr-1.0.1.tgz#d3fc114ba06995a45ec6893f484ceb1d78f5f476" @@ -4686,6 +4770,13 @@ pupa@^2.1.1: dependencies: escape-goat "^2.0.0" +qs@^6.11.0: + version "6.11.0" + resolved "https://registry.yarnpkg.com/qs/-/qs-6.11.0.tgz#fd0d963446f7a65e1367e01abd85429453f0c37a" + integrity sha512-MvjoMCJwEarSbUYk5O+nmoSzSutSsTwF85zcHPQ9OrlFoZOYIjaqBAJIqIXjptyD5vThxGq52Xu/MaJzRkIk4Q== + dependencies: + side-channel "^1.0.4" + qs@~6.5.2: version "6.5.3" resolved "https://registry.yarnpkg.com/qs/-/qs-6.5.3.tgz#3aeeffc91967ef6e35c0e488ef46fb296ab76aad" @@ -4696,6 +4787,11 @@ querystring@0.2.0: resolved "https://registry.yarnpkg.com/querystring/-/querystring-0.2.0.tgz#b209849203bb25df820da756e747005878521620" integrity sha512-X/xY82scca2tau62i9mDyU9K+I+djTMUsvwf7xnUX5GLvVzgJybOJf4Y6o9Zx3oJK/LSXg5tTZBjwzqVPaPO2g== +querystringify@^2.1.1: + version "2.2.0" + resolved "https://registry.yarnpkg.com/querystringify/-/querystringify-2.2.0.tgz#3345941b4153cb9d082d8eee4cda2016a9aef7f6" + integrity sha512-FIqgj2EUvTa7R50u0rGsyTftzjYmv/a3hO345bZNrqabNqjtgiDMgmo4mkUjd+nzU5oF3dClKqFIPUKybUyqoQ== + range-parser@^1.2.0: version "1.2.1" resolved "https://registry.yarnpkg.com/range-parser/-/range-parser-1.2.1.tgz#3cf37023d199e1c24d1a55b84800c2f3e6468031" @@ -4844,6 +4940,11 @@ require-directory@^2.1.1: resolved "https://registry.yarnpkg.com/require-directory/-/require-directory-2.1.1.tgz#8c64ad5fd30dab1c976e2344ffe7f792a6a6df42" integrity sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q== +requires-port@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/requires-port/-/requires-port-1.0.0.tgz#925d2601d39ac485e091cf0da5c6e694dc3dcaff" + integrity sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ== + resolve-cwd@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/resolve-cwd/-/resolve-cwd-3.0.0.tgz#0f0075f1bb2544766cf73ba6a6e2adfebcb13f2d" @@ -4972,6 +5073,15 @@ shimmer@^1.2.0: resolved "https://registry.yarnpkg.com/shimmer/-/shimmer-1.2.1.tgz#610859f7de327b587efebf501fb43117f9aff337" integrity sha512-sQTKC1Re/rM6XyFM6fIAGHRPVGvyXfgzIDvzoq608vM+jeyVD0Tu1E6Np0Kc2zAIFWIj963V2800iF/9LPieQw== +side-channel@^1.0.4: + version "1.0.4" + resolved "https://registry.yarnpkg.com/side-channel/-/side-channel-1.0.4.tgz#efce5c8fdc104ee751b25c58d4290011fa5ea2cf" + integrity sha512-q5XPytqFEIKHkGdiMIrY10mvLRvnQh42/+GoBlFW3b2LXLE2xxJpZFdm94we0BaoV3RwJyGqg5wS7epxTv0Zvw== + dependencies: + call-bind "^1.0.0" + get-intrinsic "^1.0.2" + object-inspect "^1.9.0" + signal-exit@^3.0.0, signal-exit@^3.0.2, signal-exit@^3.0.3: version "3.0.7" resolved "https://registry.yarnpkg.com/signal-exit/-/signal-exit-3.0.7.tgz#a9a1767f8af84155114eaabd73f99273c8f59ad9" @@ -5312,6 +5422,16 @@ touch@^3.1.0: punycode "^2.1.1" universalify "^0.1.2" +tough-cookie@^4.1.2: + version "4.1.2" + resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-4.1.2.tgz#e53e84b85f24e0b65dd526f46628db6c85f6b874" + integrity sha512-G9fqXWoYFZgTc2z8Q5zaHy/vJMjm+WV0AkAeHxVCQiEB1b+dGvWzFW6QV07cY5jQ5gRkeid2qIkzkxUnmoQZUQ== + dependencies: + psl "^1.1.33" + punycode "^2.1.1" + universalify "^0.2.0" + url-parse "^1.5.3" + tough-cookie@~2.5.0: version "2.5.0" resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-2.5.0.tgz#cd9fb2a0aa1d5a12b473bd9fb96fa3dcff65ade2" @@ -5427,6 +5547,11 @@ universalify@^0.1.2: resolved "https://registry.yarnpkg.com/universalify/-/universalify-0.1.2.tgz#b646f69be3942dabcecc9d6639c80dc105efaa66" integrity sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg== +universalify@^0.2.0: + version "0.2.0" + resolved "https://registry.yarnpkg.com/universalify/-/universalify-0.2.0.tgz#6451760566fa857534745ab1dde952d1b1761be0" + integrity sha512-CJ1QgKmNg3CwvAv/kOFmtnEN05f0D/cn9QntgNOQlQF9dgvVTHj3t+8JPdjqawCHk7V/KA+fbUqzZ9XWhcqPUg== + update-notifier@^5.1.0: version "5.1.0" resolved "https://registry.yarnpkg.com/update-notifier/-/update-notifier-5.1.0.tgz#4ab0d7c7f36a231dd7316cf7729313f0214d9ad9" @@ -5461,6 +5586,14 @@ url-parse-lax@^3.0.0: dependencies: prepend-http "^2.0.0" +url-parse@^1.5.3: + version "1.5.10" + resolved "https://registry.yarnpkg.com/url-parse/-/url-parse-1.5.10.tgz#9d3c2f736c1d75dd3bd2be507dcc111f1e2ea9c1" + integrity sha512-WypcfiRhfeUP9vvF0j6rw0J3hrWrw6iZv3+22h6iRMJ/8z1Tj6XfLP4DsUix5MhMPnXpiHDoKyoZ/bdCkwBCiQ== + dependencies: + querystringify "^2.1.1" + requires-port "^1.0.0" + url@0.10.3: version "0.10.3" resolved "https://registry.yarnpkg.com/url/-/url-0.10.3.tgz#021e4d9c7705f21bbf37d03ceb58767402774c64" diff --git a/packages/types/src/documents/document.ts b/packages/types/src/documents/document.ts index fea05918d5..ac05214b82 100644 --- a/packages/types/src/documents/document.ts +++ b/packages/types/src/documents/document.ts @@ -4,3 +4,7 @@ export interface Document { createdAt?: string | number updatedAt?: string } + +export interface AnyDocument extends Document { + [key: string]: any +} From 9e7ac26aa08354972038add5b202fd12726fa039 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 8 Nov 2022 16:32:13 +0000 Subject: [PATCH 02/24] Getting basic pouch replacement working. --- packages/backend-core/src/couch/pouchLike.ts | 91 +++++++++++++------- packages/backend-core/src/db/index.ts | 19 ++-- packages/backend-core/src/db/pouch.ts | 2 - 3 files changed, 69 insertions(+), 43 deletions(-) diff --git a/packages/backend-core/src/couch/pouchLike.ts b/packages/backend-core/src/couch/pouchLike.ts index 80ebe1d363..2591380ba0 100644 --- a/packages/backend-core/src/couch/pouchLike.ts +++ b/packages/backend-core/src/couch/pouchLike.ts @@ -2,8 +2,7 @@ import Nano from "nano" import { AnyDocument } from "@budibase/types" import { getCouchInfo } from "./couch" import { directCouchCall } from "./utils" -import { ReadStream, WriteStream } from "fs" -import { dangerousGetDB } from "../db" +import { getPouchDB } from "../db" export type PouchLikeOpts = { skip_setup?: boolean @@ -20,10 +19,6 @@ export type QueryOpts = { keys?: string[] } -export type DumpOpts = { - filter?: (doc: AnyDocument) => boolean -} - export class PouchLike { public readonly name: string private static nano: Nano.ServerScope @@ -32,13 +27,21 @@ export class PouchLike { constructor(dbName: string, opts?: PouchLikeOpts) { this.name = dbName this.pouchOpts = opts || {} + if (!PouchLike.nano) { + PouchLike.init() + } } static init() { const couchInfo = getCouchInfo() - this.nano = Nano({ + PouchLike.nano = Nano({ url: couchInfo.url, - cookie: couchInfo.cookie, + requestDefaults: { + headers: { + Authorization: couchInfo.cookie, + }, + }, + parseUrl: false, }) } @@ -56,16 +59,30 @@ export class PouchLike { return PouchLike.nano.db.use(this.name) } - async info() {} + private async updateOutput(fnc: any) { + try { + return await fnc() + } catch (err: any) { + if (err.statusCode) { + err.status = err.statusCode + } + throw err + } + } + + async info() { + const db = PouchLike.nano.db.use(this.name) + return db.info() + } async get(id: string) { const db = await this.checkSetup() - return await db.get(id) + return this.updateOutput(() => db.get(id)) } async remove(id: string, rev: string) { const db = await this.checkSetup() - return await db.destroy(id, rev) + return this.updateOutput(() => db.destroy(id, rev)) } async put(document: AnyDocument) { @@ -73,23 +90,23 @@ export class PouchLike { throw new Error("Cannot store document without _id field.") } const db = await this.checkSetup() - return await db.insert(document) + return this.updateOutput(() => db.insert(document)) } async bulkDocs(documents: AnyDocument[]) { const db = await this.checkSetup() - return await db.bulk({ docs: documents }) + return this.updateOutput(() => db.bulk({ docs: documents })) } async allDocs(params: QueryOpts) { const db = await this.checkSetup() - return await db.fetch({ keys: [] }, params) + return this.updateOutput(() => db.list(params)) } async query(viewName: string, params: QueryOpts) { const db = await this.checkSetup() const [database, view] = viewName.split("/") - return await db.view(database, view, params) + return this.updateOutput(() => db.view(database, view, params)) } async destroy() { @@ -97,37 +114,47 @@ export class PouchLike { await PouchLike.nano.db.destroy(this.name) } catch (err: any) { // didn't exist, don't worry - if (err.status === 404) { + if (err.statusCode === 404) { return } else { - throw err + throw { ...err, status: err.statusCode } } } } async compact() { const db = await this.checkSetup() - return await db.compact() + return this.updateOutput(() => db.compact()) } - // utilise PouchDB for this - async dump(stream: WriteStream, params: DumpOpts) { - const pouch = dangerousGetDB(this.name) - // @ts-ignore - return pouch.dump(stream, params) + private doWithPouchDB(func: string) { + const dbName = this.name + return async (args: any[]) => { + const pouch = getPouchDB(dbName) + // @ts-ignore + return pouch[func](...args) + } } - // utilise PouchDB for this - async load(stream: ReadStream) { - const pouch = dangerousGetDB(this.name) - // @ts-ignore - return pouch.load(stream) + // All below functions are in-frequently called, just utilise PouchDB + // for them as it implements them better than we can + async dump(...args: any[]) { + return this.doWithPouchDB("dump")(args) } - // pouch specific functions - indexes come from the pouchdb-find library - async createIndex() {} + async load(...args: any[]) { + return this.doWithPouchDB("load")(args) + } - async deleteIndex() {} + async createIndex(...args: any[]) { + return this.doWithPouchDB("createIndex")(args) + } - async getIndexes() {} + async deleteIndex(...args: any[]) { + return this.doWithPouchDB("createIndex")(args) + } + + async getIndexes(...args: any[]) { + return this.doWithPouchDB("createIndex")(args) + } } diff --git a/packages/backend-core/src/db/index.ts b/packages/backend-core/src/db/index.ts index 9f6c097f0d..73d1327af3 100644 --- a/packages/backend-core/src/db/index.ts +++ b/packages/backend-core/src/db/index.ts @@ -2,6 +2,7 @@ import * as pouch from "./pouch" import env from "../environment" import { PouchOptions, CouchFindOptions } from "@budibase/types" import PouchDB from "pouchdb" +import { PouchLike } from "../couch" import { directCouchQuery } from "../couch" export { directCouchQuery } from "../couch" @@ -38,10 +39,7 @@ export async function init(opts?: PouchOptions) { initialised = true } -// NOTE: THIS IS A DANGEROUS FUNCTION - USE WITH CAUTION -// this function is prone to leaks, should only be used -// in situations that using the function doWithDB does not work -export function dangerousGetDB(dbName: string, opts?: any): PouchDB.Database { +export function getPouchDB(dbName: string, opts?: any): PouchDB.Database { checkInitialised() if (env.isTest()) { dbList.add(dbName) @@ -55,6 +53,13 @@ export function dangerousGetDB(dbName: string, opts?: any): PouchDB.Database { return db } +// NOTE: THIS IS A DANGEROUS FUNCTION - USE WITH CAUTION +// this function is prone to leaks, should only be used +// in situations that using the function doWithDB does not work +export function dangerousGetDB(dbName: string, opts?: any): PouchLike { + return new PouchLike(dbName, opts) +} + // use this function if you have called dangerousGetDB - close // the databases you've opened once finished export async function closeDB(db: PouchDB.Database) { @@ -79,11 +84,7 @@ export async function doWithDB(dbName: string, cb: any, opts = {}) { const db = dangerousGetDB(dbName, opts) // need this to be async so that we can correctly close DB after all // async operations have been completed - try { - return await cb(db) - } finally { - await closeDB(db) - } + return await cb(db) } export function allDbs() { diff --git a/packages/backend-core/src/db/pouch.ts b/packages/backend-core/src/db/pouch.ts index 6eca07e0cd..fdb051060a 100644 --- a/packages/backend-core/src/db/pouch.ts +++ b/packages/backend-core/src/db/pouch.ts @@ -66,7 +66,6 @@ export const getPouch = (opts: any = {}) => { const inMemory = require("pouchdb-adapter-memory") PouchDB.plugin(inMemory) POUCH_DB_DEFAULTS = { - prefix: undefined, // @ts-ignore adapter: "memory", } @@ -74,7 +73,6 @@ export const getPouch = (opts: any = {}) => { if (opts.onDisk) { POUCH_DB_DEFAULTS = { - prefix: undefined, // @ts-ignore adapter: "leveldb", } From c744d23832413f3df04eec7c03ccb97a2c41ac2a Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 9 Nov 2022 16:53:42 +0000 Subject: [PATCH 03/24] Major update - removing the use of context for PouchDB instances, swapping knowledge of PouchDB to the PouchLike structure that replaces it. --- .../src/cache/tests/writethrough.spec.js | 6 +- .../backend-core/src/cache/writethrough.ts | 17 +- .../backend-core/src/context/constants.ts | 12 -- packages/backend-core/src/context/index.ts | 160 +++++------------- .../src/context/tests/index.spec.ts | 16 +- packages/backend-core/src/context/utils.ts | 109 ------------ packages/backend-core/src/couch/index.ts | 1 + packages/backend-core/src/couch/pouchDB.ts | 37 ++++ packages/backend-core/src/couch/pouchLike.ts | 55 ++++-- packages/backend-core/src/db/Replication.ts | 10 +- packages/backend-core/src/db/index.ts | 65 +------ .../backend-core/src/db/tests/index.spec.js | 6 +- packages/backend-core/src/db/views.ts | 49 +++--- packages/backend-core/src/environment.ts | 1 - packages/backend-core/src/index.ts | 2 + .../src/migrations/tests/index.spec.js | 4 +- packages/backend-core/src/types.ts | 1 + packages/backend-core/src/users.ts | 5 +- .../server/src/api/controllers/application.ts | 10 +- .../src/api/controllers/row/internal.js | 4 +- packages/server/src/db/inMemoryView.js | 4 +- .../functions/backfill/app/tables.ts | 5 +- .../server/src/sdk/app/backups/imports.ts | 12 +- .../server/src/sdk/app/backups/statistics.ts | 28 +-- packages/server/src/sdk/app/tables/index.ts | 11 +- 25 files changed, 211 insertions(+), 419 deletions(-) delete mode 100644 packages/backend-core/src/context/utils.ts create mode 100644 packages/backend-core/src/couch/pouchDB.ts create mode 100644 packages/backend-core/src/types.ts diff --git a/packages/backend-core/src/cache/tests/writethrough.spec.js b/packages/backend-core/src/cache/tests/writethrough.spec.js index 68db24b325..ad29cb2aea 100644 --- a/packages/backend-core/src/cache/tests/writethrough.spec.js +++ b/packages/backend-core/src/cache/tests/writethrough.spec.js @@ -1,6 +1,6 @@ require("../../../tests/utilities/TestConfiguration") const { Writethrough } = require("../writethrough") -const { dangerousGetDB } = require("../../db") +const { getDB } = require("../../db") const tk = require("timekeeper") const START_DATE = Date.now() @@ -8,8 +8,8 @@ tk.freeze(START_DATE) const DELAY = 5000 -const db = dangerousGetDB("test") -const db2 = dangerousGetDB("test2") +const db = getDB("test") +const db2 = getDB("test2") const writethrough = new Writethrough(db, DELAY), writethrough2 = new Writethrough(db2, DELAY) describe("writethrough", () => { diff --git a/packages/backend-core/src/cache/writethrough.ts b/packages/backend-core/src/cache/writethrough.ts index 495ba58590..11dad81239 100644 --- a/packages/backend-core/src/cache/writethrough.ts +++ b/packages/backend-core/src/cache/writethrough.ts @@ -1,7 +1,7 @@ import BaseCache from "./base" import { getWritethroughClient } from "../redis/init" import { logWarn } from "../logging" -import PouchDB from "pouchdb" +import { PouchLike } from "../couch" const DEFAULT_WRITE_RATE_MS = 10000 let CACHE: BaseCache | null = null @@ -19,7 +19,7 @@ async function getCache() { return CACHE } -function makeCacheKey(db: PouchDB.Database, key: string) { +function makeCacheKey(db: PouchLike, key: string) { return db.name + key } @@ -28,7 +28,7 @@ function makeCacheItem(doc: any, lastWrite: number | null = null): CacheItem { } export async function put( - db: PouchDB.Database, + db: PouchLike, doc: any, writeRateMs: number = DEFAULT_WRITE_RATE_MS ) { @@ -64,7 +64,7 @@ export async function put( return { ok: true, id: output._id, rev: output._rev } } -export async function get(db: PouchDB.Database, id: string): Promise { +export async function get(db: PouchLike, id: string): Promise { const cache = await getCache() const cacheKey = makeCacheKey(db, id) let cacheItem: CacheItem = await cache.get(cacheKey) @@ -77,7 +77,7 @@ export async function get(db: PouchDB.Database, id: string): Promise { } export async function remove( - db: PouchDB.Database, + db: PouchLike, docOrId: any, rev?: any ): Promise { @@ -95,13 +95,10 @@ export async function remove( } export class Writethrough { - db: PouchDB.Database + db: PouchLike writeRateMs: number - constructor( - db: PouchDB.Database, - writeRateMs: number = DEFAULT_WRITE_RATE_MS - ) { + constructor(db: PouchLike, writeRateMs: number = DEFAULT_WRITE_RATE_MS) { this.db = db this.writeRateMs = writeRateMs } diff --git a/packages/backend-core/src/context/constants.ts b/packages/backend-core/src/context/constants.ts index 937ad8f248..af30b1d241 100644 --- a/packages/backend-core/src/context/constants.ts +++ b/packages/backend-core/src/context/constants.ts @@ -1,17 +1,5 @@ export enum ContextKey { TENANT_ID = "tenantId", - GLOBAL_DB = "globalDb", APP_ID = "appId", IDENTITY = "identity", - // whatever the request app DB was - CURRENT_DB = "currentDb", - // get the prod app DB from the request - PROD_DB = "prodDb", - // get the dev app DB from the request - DEV_DB = "devDb", - DB_OPTS = "dbOpts", - // check if something else is using the context, don't close DB - TENANCY_IN_USE = "tenancyInUse", - APP_IN_USE = "appInUse", - IDENTITY_IN_USE = "identityInUse", } diff --git a/packages/backend-core/src/context/index.ts b/packages/backend-core/src/context/index.ts index c3955c71d9..30a53f1587 100644 --- a/packages/backend-core/src/context/index.ts +++ b/packages/backend-core/src/context/index.ts @@ -1,20 +1,12 @@ import env from "../environment" import { SEPARATOR, DocumentType } from "../db/constants" import cls from "./FunctionContext" -import { dangerousGetDB, closeDB } from "../db" import { baseGlobalDBName } from "../db/tenancy" import { IdentityContext } from "@budibase/types" import { DEFAULT_TENANT_ID as _DEFAULT_TENANT_ID } from "../constants" import { ContextKey } from "./constants" -import PouchDB from "pouchdb" -import { - updateUsing, - closeWithUsing, - setAppTenantId, - setIdentity, - closeAppDBs, - getContextDB, -} from "./utils" +import { PouchLike } from "../couch" +import { getDevelopmentAppID, getProdAppID } from "../db/conversions" export const DEFAULT_TENANT_ID = _DEFAULT_TENANT_ID @@ -22,29 +14,19 @@ export const DEFAULT_TENANT_ID = _DEFAULT_TENANT_ID // store an app ID to pretend there is a context let TEST_APP_ID: string | null = null -export const closeTenancy = async () => { - try { - if (env.USE_COUCH) { - const db = getGlobalDB() - await closeDB(db) - } - } catch (err) { - // no DB found - skip closing - return - } - // clear from context now that database is closed/task is finished - cls.setOnContext(ContextKey.TENANT_ID, null) - cls.setOnContext(ContextKey.GLOBAL_DB, null) -} - -// export const isDefaultTenant = () => { -// return getTenantId() === DEFAULT_TENANT_ID -// } - export const isMultiTenant = () => { return env.MULTI_TENANCY } +const setAppTenantId = (appId: string) => { + const appTenantId = getTenantIDFromAppID(appId) || DEFAULT_TENANT_ID + updateTenantId(appTenantId) +} + +const setIdentity = (identity: IdentityContext | null) => { + cls.setOnContext(ContextKey.IDENTITY, identity) +} + /** * Given an app ID this will attempt to retrieve the tenant ID from it. * @return {null|string} The tenant ID found within the app ID. @@ -78,47 +60,28 @@ export const doInContext = async (appId: string, task: any) => { }) } -export const doInTenant = (tenantId: string | null, task: any) => { +export const doInTenant = (tenantId: string | null, task: any): any => { // make sure default always selected in single tenancy if (!env.MULTI_TENANCY) { tenantId = tenantId || DEFAULT_TENANT_ID } - // the internal function is so that we can re-use an existing - // context - don't want to close DB on a parent context - async function internal(opts = { existing: false }) { - // set the tenant id + global db if this is a new context - if (!opts.existing) { - updateTenantId(tenantId) - } - try { - // invoke the task - return await task() - } finally { - await closeWithUsing(ContextKey.TENANCY_IN_USE, () => { - return closeTenancy() - }) - } - } - - const existing = cls.getFromContext(ContextKey.TENANT_ID) === tenantId - return updateUsing(ContextKey.TENANCY_IN_USE, existing, internal) + return cls.run(async () => { + updateTenantId(tenantId) + return await task() + }) } -export const doInAppContext = (appId: string, task: any) => { +export const doInAppContext = (appId: string, task: any): any => { if (!appId) { throw new Error("appId is required") } const identity = getIdentity() - // the internal function is so that we can re-use an existing - // context - don't want to close DB on a parent context - async function internal(opts = { existing: false }) { + return cls.run(async () => { // set the app tenant id - if (!opts.existing) { - setAppTenantId(appId) - } + setAppTenantId(appId) // set the app ID cls.setOnContext(ContextKey.APP_ID, appId) @@ -126,47 +89,28 @@ export const doInAppContext = (appId: string, task: any) => { if (identity) { setIdentity(identity) } - try { - // invoke the task - return await task() - } finally { - await closeWithUsing(ContextKey.APP_IN_USE, async () => { - await closeAppDBs() - await closeTenancy() - }) - } - } - const existing = cls.getFromContext(ContextKey.APP_ID) === appId - return updateUsing(ContextKey.APP_IN_USE, existing, internal) + // invoke the task + return await task() + }) } -export const doInIdentityContext = (identity: IdentityContext, task: any) => { +export const doInIdentityContext = ( + identity: IdentityContext, + task: any +): any => { if (!identity) { throw new Error("identity is required") } - async function internal(opts = { existing: false }) { - if (!opts.existing) { - cls.setOnContext(ContextKey.IDENTITY, identity) - // set the tenant so that doInTenant will preserve identity - if (identity.tenantId) { - updateTenantId(identity.tenantId) - } + return cls.run(async () => { + cls.setOnContext(ContextKey.IDENTITY, identity) + // set the tenant so that doInTenant will preserve identity + if (identity.tenantId) { + updateTenantId(identity.tenantId) } - - try { - // invoke the task - return await task() - } finally { - await closeWithUsing(ContextKey.IDENTITY_IN_USE, async () => { - setIdentity(null) - await closeTenancy() - }) - } - } - - const existing = cls.getFromContext(ContextKey.IDENTITY) - return updateUsing(ContextKey.IDENTITY_IN_USE, existing, internal) + // invoke the task + return await task() + }) } export const getIdentity = (): IdentityContext | undefined => { @@ -179,15 +123,10 @@ export const getIdentity = (): IdentityContext | undefined => { export const updateTenantId = (tenantId: string | null) => { cls.setOnContext(ContextKey.TENANT_ID, tenantId) - if (env.USE_COUCH) { - setGlobalDB(tenantId) - } } export const updateAppId = async (appId: string) => { try { - // have to close first, before removing the databases from context - await closeAppDBs() cls.setOnContext(ContextKey.APP_ID, appId) } catch (err) { if (env.isTest()) { @@ -198,19 +137,9 @@ export const updateAppId = async (appId: string) => { } } -export const setGlobalDB = (tenantId: string | null) => { - const dbName = baseGlobalDBName(tenantId) - const db = dangerousGetDB(dbName) - cls.setOnContext(ContextKey.GLOBAL_DB, db) - return db -} - -export const getGlobalDB = () => { - const db = cls.getFromContext(ContextKey.GLOBAL_DB) - if (!db) { - throw new Error("Global DB not found") - } - return db +export const getGlobalDB = (): PouchLike => { + const tenantId = cls.getFromContext(ContextKey.TENANT_ID) + return new PouchLike(baseGlobalDBName(tenantId)) } export const isTenantIdSet = () => { @@ -246,22 +175,25 @@ export const isTenancyEnabled = () => { * Opens the app database based on whatever the request * contained, dev or prod. */ -export const getAppDB = (opts?: any) => { - return getContextDB(ContextKey.CURRENT_DB, opts) +export const getAppDB = (opts?: any): PouchLike => { + const appId = getAppId() + return new PouchLike(appId, opts) } /** * This specifically gets the prod app ID, if the request * contained a development app ID, this will open the prod one. */ -export const getProdAppDB = (opts?: any) => { - return getContextDB(ContextKey.PROD_DB, opts) +export const getProdAppDB = (opts?: any): PouchLike => { + const appId = getAppId() + return new PouchLike(getProdAppID(appId), opts) } /** * This specifically gets the dev app ID, if the request * contained a prod app ID, this will open the dev one. */ -export const getDevAppDB = (opts?: any) => { - return getContextDB(ContextKey.DEV_DB, opts) +export const getDevAppDB = (opts?: any): PouchLike => { + const appId = getAppId() + return new PouchLike(getDevelopmentAppID(appId), opts) } diff --git a/packages/backend-core/src/context/tests/index.spec.ts b/packages/backend-core/src/context/tests/index.spec.ts index 55ecd333a3..749231c3e5 100644 --- a/packages/backend-core/src/context/tests/index.spec.ts +++ b/packages/backend-core/src/context/tests/index.spec.ts @@ -5,8 +5,8 @@ import env from "../../environment" // must use require to spy index file exports due to known issue in jest const dbUtils = require("../../db") -jest.spyOn(dbUtils, "closeDB") -jest.spyOn(dbUtils, "dangerousGetDB") +jest.spyOn(dbUtils, "closePouchDB") +jest.spyOn(dbUtils, "getDB") describe("context", () => { beforeEach(() => { @@ -25,8 +25,8 @@ describe("context", () => { const db = context.getGlobalDB() expect(db.name).toBe("global-db") }) - expect(dbUtils.dangerousGetDB).toHaveBeenCalledTimes(1) - expect(dbUtils.closeDB).toHaveBeenCalledTimes(1) + expect(dbUtils.getDB).toHaveBeenCalledTimes(1) + expect(dbUtils.closePouchDB).toHaveBeenCalledTimes(1) }) }) @@ -85,8 +85,8 @@ describe("context", () => { const db = context.getGlobalDB() expect(db.name).toBe("test_global-db") }) - expect(dbUtils.dangerousGetDB).toHaveBeenCalledTimes(1) - expect(dbUtils.closeDB).toHaveBeenCalledTimes(1) + expect(dbUtils.getDB).toHaveBeenCalledTimes(1) + expect(dbUtils.closePouchDB).toHaveBeenCalledTimes(1) }) it("sets the tenant id when nested with same tenant id", async () => { @@ -123,8 +123,8 @@ describe("context", () => { }) // only 1 db is opened and closed - expect(dbUtils.dangerousGetDB).toHaveBeenCalledTimes(1) - expect(dbUtils.closeDB).toHaveBeenCalledTimes(1) + expect(dbUtils.getDB).toHaveBeenCalledTimes(1) + expect(dbUtils.closePouchDB).toHaveBeenCalledTimes(1) }) it("sets different tenant id inside another context", () => { diff --git a/packages/backend-core/src/context/utils.ts b/packages/backend-core/src/context/utils.ts deleted file mode 100644 index 6e7100b594..0000000000 --- a/packages/backend-core/src/context/utils.ts +++ /dev/null @@ -1,109 +0,0 @@ -import { - DEFAULT_TENANT_ID, - getAppId, - getTenantIDFromAppID, - updateTenantId, -} from "./index" -import cls from "./FunctionContext" -import { IdentityContext } from "@budibase/types" -import { ContextKey } from "./constants" -import { dangerousGetDB, closeDB } from "../db" -import { isEqual } from "lodash" -import { getDevelopmentAppID, getProdAppID } from "../db/conversions" -import env from "../environment" - -export async function updateUsing( - usingKey: string, - existing: boolean, - internal: (opts: { existing: boolean }) => Promise -) { - const using = cls.getFromContext(usingKey) - if (using && existing) { - cls.setOnContext(usingKey, using + 1) - return internal({ existing: true }) - } else { - return cls.run(async () => { - cls.setOnContext(usingKey, 1) - return internal({ existing: false }) - }) - } -} - -export async function closeWithUsing( - usingKey: string, - closeFn: () => Promise -) { - const using = cls.getFromContext(usingKey) - if (!using || using <= 1) { - await closeFn() - } else { - cls.setOnContext(usingKey, using - 1) - } -} - -export const setAppTenantId = (appId: string) => { - const appTenantId = getTenantIDFromAppID(appId) || DEFAULT_TENANT_ID - updateTenantId(appTenantId) -} - -export const setIdentity = (identity: IdentityContext | null) => { - cls.setOnContext(ContextKey.IDENTITY, identity) -} - -// this function makes sure the PouchDB objects are closed and -// fully deleted when finished - this protects against memory leaks -export async function closeAppDBs() { - const dbKeys = [ContextKey.CURRENT_DB, ContextKey.PROD_DB, ContextKey.DEV_DB] - for (let dbKey of dbKeys) { - const db = cls.getFromContext(dbKey) - if (!db) { - continue - } - await closeDB(db) - // clear the DB from context, incase someone tries to use it again - cls.setOnContext(dbKey, null) - } - // clear the app ID now that the databases are closed - if (cls.getFromContext(ContextKey.APP_ID)) { - cls.setOnContext(ContextKey.APP_ID, null) - } - if (cls.getFromContext(ContextKey.DB_OPTS)) { - cls.setOnContext(ContextKey.DB_OPTS, null) - } -} - -export function getContextDB(key: string, opts: any) { - const dbOptsKey = `${key}${ContextKey.DB_OPTS}` - let storedOpts = cls.getFromContext(dbOptsKey) - let db = cls.getFromContext(key) - if (db && isEqual(opts, storedOpts)) { - return db - } - - const appId = getAppId() - let toUseAppId - - switch (key) { - case ContextKey.CURRENT_DB: - toUseAppId = appId - break - case ContextKey.PROD_DB: - toUseAppId = getProdAppID(appId) - break - case ContextKey.DEV_DB: - toUseAppId = getDevelopmentAppID(appId) - break - } - db = dangerousGetDB(toUseAppId, opts) - try { - cls.setOnContext(key, db) - if (opts) { - cls.setOnContext(dbOptsKey, opts) - } - } catch (err) { - if (!env.isTest()) { - throw err - } - } - return db -} diff --git a/packages/backend-core/src/couch/index.ts b/packages/backend-core/src/couch/index.ts index daba8f74a9..7a4ac20486 100644 --- a/packages/backend-core/src/couch/index.ts +++ b/packages/backend-core/src/couch/index.ts @@ -1,3 +1,4 @@ export * from "./couch" export * from "./pouchLike" export * from "./utils" +export { init } from "./pouchDB" diff --git a/packages/backend-core/src/couch/pouchDB.ts b/packages/backend-core/src/couch/pouchDB.ts new file mode 100644 index 0000000000..74ad8a3dc6 --- /dev/null +++ b/packages/backend-core/src/couch/pouchDB.ts @@ -0,0 +1,37 @@ +import PouchDB from "pouchdb" +import env from "../environment" +import { PouchOptions } from "@budibase/types" +import * as pouch from "../db/pouch" + +let Pouch: any +let initialised = false + +export async function init(opts?: PouchOptions) { + Pouch = pouch.getPouch(opts) + initialised = true +} + +const checkInitialised = () => { + if (!initialised) { + throw new Error("init has not been called") + } +} + +export function getPouchDB(dbName: string, opts?: any): PouchDB.Database { + checkInitialised() + return new Pouch(dbName, opts) +} + +// use this function if you have called getPouchDB - close +// the databases you've opened once finished +export async function closePouchDB(db: PouchDB.Database) { + if (!db || env.isTest()) { + return + } + try { + // specifically await so that if there is an error, it can be ignored + return await db.close() + } catch (err) { + // ignore error, already closed + } +} diff --git a/packages/backend-core/src/couch/pouchLike.ts b/packages/backend-core/src/couch/pouchLike.ts index 2591380ba0..d605ec3d04 100644 --- a/packages/backend-core/src/couch/pouchLike.ts +++ b/packages/backend-core/src/couch/pouchLike.ts @@ -2,12 +2,16 @@ import Nano from "nano" import { AnyDocument } from "@budibase/types" import { getCouchInfo } from "./couch" import { directCouchCall } from "./utils" -import { getPouchDB } from "../db" +import { getPouchDB } from "./pouchDB" export type PouchLikeOpts = { skip_setup?: boolean } +export type PutOpts = { + force?: boolean +} + export type QueryOpts = { include_docs?: boolean startkey?: string @@ -19,6 +23,10 @@ export type QueryOpts = { keys?: string[] } +type QueryResp = Promise<{ + rows: { doc?: T | any; value?: any }[] +}> + export class PouchLike { public readonly name: string private static nano: Nano.ServerScope @@ -45,11 +53,15 @@ export class PouchLike { }) } + async exists() { + let response = await directCouchCall(`/${this.name}`, "HEAD") + return response.status === 200 + } + async checkSetup() { let shouldCreate = !this.pouchOpts?.skip_setup // check exists in a lightweight fashion - let response = await directCouchCall(`/${this.name}`, "HEAD") - let exists = response.status === 200 + let exists = await this.exists() if (!shouldCreate && !exists) { throw new Error("DB does not exist") } @@ -70,26 +82,43 @@ export class PouchLike { } } - async info() { - const db = PouchLike.nano.db.use(this.name) - return db.info() - } - - async get(id: string) { + async get(id?: string): Promise { const db = await this.checkSetup() + if (!id) { + throw new Error("Unable to get doc without a valid _id.") + } return this.updateOutput(() => db.get(id)) } - async remove(id: string, rev: string) { + async remove(id?: string, rev?: string) { const db = await this.checkSetup() + if (!id || !rev) { + throw new Error("Unable to remove doc without a valid _id and _rev.") + } return this.updateOutput(() => db.destroy(id, rev)) } - async put(document: AnyDocument) { + async put(document: AnyDocument, opts?: PutOpts) { if (!document._id) { throw new Error("Cannot store document without _id field.") } const db = await this.checkSetup() + if (!document.createdAt) { + document.createdAt = new Date().toISOString() + } + document.updatedAt = new Date().toISOString() + if (opts?.force && document._id) { + try { + const existing = await this.get(document._id) + if (existing) { + document._rev = existing._rev + } + } catch (err: any) { + if (err.status !== 404) { + throw err + } + } + } return this.updateOutput(() => db.insert(document)) } @@ -98,12 +127,12 @@ export class PouchLike { return this.updateOutput(() => db.bulk({ docs: documents })) } - async allDocs(params: QueryOpts) { + async allDocs(params: QueryOpts): QueryResp { const db = await this.checkSetup() return this.updateOutput(() => db.list(params)) } - async query(viewName: string, params: QueryOpts) { + async query(viewName: string, params: QueryOpts): QueryResp { const db = await this.checkSetup() const [database, view] = viewName.split("/") return this.updateOutput(() => db.view(database, view, params)) diff --git a/packages/backend-core/src/db/Replication.ts b/packages/backend-core/src/db/Replication.ts index e0bd3c7a43..c6eea8db5e 100644 --- a/packages/backend-core/src/db/Replication.ts +++ b/packages/backend-core/src/db/Replication.ts @@ -1,4 +1,4 @@ -import { dangerousGetDB, closeDB } from "." +import { getPouchDB, closePouchDB } from "../couch/pouchDB" import { DocumentType } from "./constants" class Replication { @@ -12,12 +12,12 @@ class Replication { * @param {String} target - the DB you want to replicate to, or rollback from */ constructor({ source, target }: any) { - this.source = dangerousGetDB(source) - this.target = dangerousGetDB(target) + this.source = getPouchDB(source) + this.target = getPouchDB(target) } close() { - return Promise.all([closeDB(this.source), closeDB(this.target)]) + return Promise.all([closePouchDB(this.source), closePouchDB(this.target)]) } promisify(operation: any, opts = {}) { @@ -68,7 +68,7 @@ class Replication { async rollback() { await this.target.destroy() // Recreate the DB again - this.target = dangerousGetDB(this.target.name) + this.target = getPouchDB(this.target.name) // take the opportunity to remove deleted tombstones await this.replicate() } diff --git a/packages/backend-core/src/db/index.ts b/packages/backend-core/src/db/index.ts index 73d1327af3..e732b3a2fb 100644 --- a/packages/backend-core/src/db/index.ts +++ b/packages/backend-core/src/db/index.ts @@ -1,87 +1,30 @@ -import * as pouch from "./pouch" import env from "../environment" -import { PouchOptions, CouchFindOptions } from "@budibase/types" -import PouchDB from "pouchdb" +import { CouchFindOptions } from "@budibase/types" import { PouchLike } from "../couch" import { directCouchQuery } from "../couch" -export { directCouchQuery } from "../couch" +export { init, PouchLike } from "../couch" -const openDbs: string[] = [] -let Pouch: any let initialised = false const dbList = new Set() -if (env.MEMORY_LEAK_CHECK) { - setInterval(() => { - console.log("--- OPEN DBS ---") - console.log(openDbs) - }, 5000) -} - -const put = - (dbPut: any) => - async (doc: any, options = {}) => { - if (!doc.createdAt) { - doc.createdAt = new Date().toISOString() - } - doc.updatedAt = new Date().toISOString() - return dbPut(doc, options) - } - const checkInitialised = () => { if (!initialised) { throw new Error("init has not been called") } } -export async function init(opts?: PouchOptions) { - Pouch = pouch.getPouch(opts) - initialised = true -} - -export function getPouchDB(dbName: string, opts?: any): PouchDB.Database { - checkInitialised() +export function getDB(dbName: string, opts?: any): PouchLike { if (env.isTest()) { dbList.add(dbName) } - const db = new Pouch(dbName, opts) - if (env.MEMORY_LEAK_CHECK) { - openDbs.push(db.name) - } - const dbPut = db.put - db.put = put(dbPut) - return db -} - -// NOTE: THIS IS A DANGEROUS FUNCTION - USE WITH CAUTION -// this function is prone to leaks, should only be used -// in situations that using the function doWithDB does not work -export function dangerousGetDB(dbName: string, opts?: any): PouchLike { return new PouchLike(dbName, opts) } -// use this function if you have called dangerousGetDB - close -// the databases you've opened once finished -export async function closeDB(db: PouchDB.Database) { - if (!db || env.isTest()) { - return - } - if (env.MEMORY_LEAK_CHECK) { - openDbs.splice(openDbs.indexOf(db.name), 1) - } - try { - // specifically await so that if there is an error, it can be ignored - return await db.close() - } catch (err) { - // ignore error, already closed - } -} - // we have to use a callback for this so that we can close // the DB when we're done, without this manual requests would // need to close the database when done with it to avoid memory leaks export async function doWithDB(dbName: string, cb: any, opts = {}) { - const db = dangerousGetDB(dbName, opts) + const db = getDB(dbName, opts) // need this to be async so that we can correctly close DB after all // async operations have been completed return await cb(db) diff --git a/packages/backend-core/src/db/tests/index.spec.js b/packages/backend-core/src/db/tests/index.spec.js index bc0c638126..8f81e6b8dc 100644 --- a/packages/backend-core/src/db/tests/index.spec.js +++ b/packages/backend-core/src/db/tests/index.spec.js @@ -1,11 +1,11 @@ require("../../../tests/utilities/TestConfiguration") -const { dangerousGetDB } = require("../") +const { getDB } = require("../") describe("db", () => { describe("getDB", () => { it("returns a db", async () => { - const db = dangerousGetDB("test") + const db = getDB("test") expect(db).toBeDefined() expect(db._adapter).toBe("memory") expect(db.prefix).toBe("_pouch_") @@ -13,7 +13,7 @@ describe("db", () => { }) it("uses the custom put function", async () => { - const db = dangerousGetDB("test") + const db = getDB("test") let doc = { _id: "test" } await db.put(doc) doc = await db.get(doc._id) diff --git a/packages/backend-core/src/db/views.ts b/packages/backend-core/src/db/views.ts index f0fff918fc..06919fd188 100644 --- a/packages/backend-core/src/db/views.ts +++ b/packages/backend-core/src/db/views.ts @@ -1,6 +1,6 @@ import { DocumentType, ViewName, DeprecatedViews, SEPARATOR } from "./utils" import { getGlobalDB } from "../context" -import PouchDB from "pouchdb" +import { PouchLike, QueryOpts } from "../couch" import { StaticDatabases } from "./constants" import { doWithDB } from "./" @@ -19,7 +19,7 @@ interface DesignDocument { views: any } -async function removeDeprecated(db: PouchDB.Database, viewName: ViewName) { +async function removeDeprecated(db: PouchLike, viewName: ViewName) { // @ts-ignore if (!DeprecatedViews[viewName]) { return @@ -70,16 +70,13 @@ export const createAccountEmailView = async () => { emit(doc.email.toLowerCase(), doc._id) } }` - await doWithDB( - StaticDatabases.PLATFORM_INFO.name, - async (db: PouchDB.Database) => { - await createView(db, viewJs, ViewName.ACCOUNT_BY_EMAIL) - } - ) + await doWithDB(StaticDatabases.PLATFORM_INFO.name, async (db: PouchLike) => { + await createView(db, viewJs, ViewName.ACCOUNT_BY_EMAIL) + }) } export const createUserAppView = async () => { - const db = getGlobalDB() as PouchDB.Database + const db = getGlobalDB() const viewJs = `function(doc) { if (doc._id.startsWith("${DocumentType.USER}${SEPARATOR}") && doc.roles) { for (let prodAppId of Object.keys(doc.roles)) { @@ -117,12 +114,9 @@ export const createPlatformUserView = async () => { emit(doc._id.toLowerCase(), doc._id) } }` - await doWithDB( - StaticDatabases.PLATFORM_INFO.name, - async (db: PouchDB.Database) => { - await createView(db, viewJs, ViewName.PLATFORM_USERS_LOWERCASE) - } - ) + await doWithDB(StaticDatabases.PLATFORM_INFO.name, async (db: PouchLike) => { + await createView(db, viewJs, ViewName.PLATFORM_USERS_LOWERCASE) + }) } export interface QueryViewOptions { @@ -131,13 +125,13 @@ export interface QueryViewOptions { export const queryView = async ( viewName: ViewName, - params: PouchDB.Query.Options, - db: PouchDB.Database, + params: QueryOpts, + db: PouchLike, createFunc: any, opts?: QueryViewOptions ): Promise => { try { - let response = await db.query(`database/${viewName}`, params) + let response = await db.query(`database/${viewName}`, params) const rows = response.rows const docs = rows.map(row => (params.include_docs ? row.doc : row.value)) @@ -161,7 +155,7 @@ export const queryView = async ( export const queryPlatformView = async ( viewName: ViewName, - params: PouchDB.Query.Options, + params: QueryOpts, opts?: QueryViewOptions ): Promise => { const CreateFuncByName: any = { @@ -169,19 +163,16 @@ export const queryPlatformView = async ( [ViewName.PLATFORM_USERS_LOWERCASE]: createPlatformUserView, } - return doWithDB( - StaticDatabases.PLATFORM_INFO.name, - async (db: PouchDB.Database) => { - const createFn = CreateFuncByName[viewName] - return queryView(viewName, params, db, createFn, opts) - } - ) + return doWithDB(StaticDatabases.PLATFORM_INFO.name, async (db: PouchLike) => { + const createFn = CreateFuncByName[viewName] + return queryView(viewName, params, db, createFn, opts) + }) } export const queryGlobalView = async ( viewName: ViewName, - params: PouchDB.Query.Options, - db?: PouchDB.Database, + params: QueryOpts, + db?: PouchLike, opts?: QueryViewOptions ): Promise => { const CreateFuncByName: any = { @@ -192,7 +183,7 @@ export const queryGlobalView = async ( } // can pass DB in if working with something specific if (!db) { - db = getGlobalDB() as PouchDB.Database + db = getGlobalDB() } const createFn = CreateFuncByName[viewName] return queryView(viewName, params, db, createFn, opts) diff --git a/packages/backend-core/src/environment.ts b/packages/backend-core/src/environment.ts index 6e2ac94be9..2443287d5a 100644 --- a/packages/backend-core/src/environment.ts +++ b/packages/backend-core/src/environment.ts @@ -69,7 +69,6 @@ const env = { DISABLE_DEVELOPER_LICENSE: process.env.DISABLE_DEVELOPER_LICENSE, DEFAULT_LICENSE: process.env.DEFAULT_LICENSE, SERVICE: process.env.SERVICE || "budibase", - MEMORY_LEAK_CHECK: process.env.MEMORY_LEAK_CHECK || false, LOG_LEVEL: process.env.LOG_LEVEL, SESSION_UPDATE_PERIOD: process.env.SESSION_UPDATE_PERIOD, DEPLOYMENT_ENVIRONMENT: diff --git a/packages/backend-core/src/index.ts b/packages/backend-core/src/index.ts index 17393b8ac3..06997ced90 100644 --- a/packages/backend-core/src/index.ts +++ b/packages/backend-core/src/index.ts @@ -21,6 +21,7 @@ import * as middleware from "./middleware" import plugins from "./plugin" import encryption from "./security/encryption" import * as queue from "./queue" +import * as types from "./types" // mimic the outer package exports import * as db from "./pkg/db" @@ -67,6 +68,7 @@ const core = { encryption, queue, permissions, + ...types, } export = core diff --git a/packages/backend-core/src/migrations/tests/index.spec.js b/packages/backend-core/src/migrations/tests/index.spec.js index c5ec143143..653ef453c5 100644 --- a/packages/backend-core/src/migrations/tests/index.spec.js +++ b/packages/backend-core/src/migrations/tests/index.spec.js @@ -1,6 +1,6 @@ require("../../../tests/utilities/TestConfiguration") const { runMigrations, getMigrationsDoc } = require("../index") -const { dangerousGetDB } = require("../../db") +const { getDB } = require("../../db") const { StaticDatabases, } = require("../../db/utils") @@ -18,7 +18,7 @@ describe("migrations", () => { }] beforeEach(() => { - db = dangerousGetDB(StaticDatabases.GLOBAL.name) + db = getDB(StaticDatabases.GLOBAL.name) }) afterEach(async () => { diff --git a/packages/backend-core/src/types.ts b/packages/backend-core/src/types.ts new file mode 100644 index 0000000000..5eb3109b27 --- /dev/null +++ b/packages/backend-core/src/types.ts @@ -0,0 +1 @@ +export { PouchLike } from "./couch" diff --git a/packages/backend-core/src/users.ts b/packages/backend-core/src/users.ts index 44f04749c9..a38debfc19 100644 --- a/packages/backend-core/src/users.ts +++ b/packages/backend-core/src/users.ts @@ -8,10 +8,9 @@ import { queryGlobalView } from "./db/views" import { UNICODE_MAX } from "./db/constants" import { BulkDocsResponse, User } from "@budibase/types" import { getGlobalDB } from "./context" -import PouchDB from "pouchdb" export const bulkGetGlobalUsersById = async (userIds: string[]) => { - const db = getGlobalDB() as PouchDB.Database + const db = getGlobalDB() return ( await db.allDocs({ keys: userIds, @@ -21,7 +20,7 @@ export const bulkGetGlobalUsersById = async (userIds: string[]) => { } export const bulkUpdateGlobalUsers = async (users: User[]) => { - const db = getGlobalDB() as PouchDB.Database + const db = getGlobalDB() return (await db.bulkDocs(users)) as BulkDocsResponse } diff --git a/packages/server/src/api/controllers/application.ts b/packages/server/src/api/controllers/application.ts index f3dca51f72..b97e980a5e 100644 --- a/packages/server/src/api/controllers/application.ts +++ b/packages/server/src/api/controllers/application.ts @@ -524,12 +524,10 @@ export const sync = async (ctx: any, next: any) => { // replicate prod to dev const prodAppId = getProdAppID(appId) - try { - // specific case, want to make sure setup is skipped - const prodDb = context.getProdAppDB({ skip_setup: true }) - const info = await prodDb.info() - if (info.error) throw info.error - } catch (err) { + // specific case, want to make sure setup is skipped + const prodDb = context.getProdAppDB({ skip_setup: true }) + const exists = await prodDb.exists() + if (!exists) { // the database doesn't exist. Don't replicate ctx.status = 200 ctx.body = { diff --git a/packages/server/src/api/controllers/row/internal.js b/packages/server/src/api/controllers/row/internal.js index 52f8a548fe..0f1324f10e 100644 --- a/packages/server/src/api/controllers/row/internal.js +++ b/packages/server/src/api/controllers/row/internal.js @@ -6,7 +6,7 @@ const { DocumentType, InternalTables, } = require("../../../db/utils") -const { dangerousGetDB } = require("@budibase/backend-core/db") +const { getDB } = require("@budibase/backend-core/db") const userController = require("../user") const { inputProcessing, @@ -251,7 +251,7 @@ exports.fetch = async ctx => { } exports.find = async ctx => { - const db = dangerousGetDB(ctx.appId) + const db = getDB(ctx.appId) const table = await db.get(ctx.params.tableId) let row = await findRow(ctx, ctx.params.tableId, ctx.params.rowId) row = await outputProcessing(table, row) diff --git a/packages/server/src/db/inMemoryView.js b/packages/server/src/db/inMemoryView.js index ec99b4738c..57ea89071c 100644 --- a/packages/server/src/db/inMemoryView.js +++ b/packages/server/src/db/inMemoryView.js @@ -2,7 +2,7 @@ const newid = require("./newid") // bypass the main application db config // use in memory pouchdb directly -const { getPouch, closeDB } = require("@budibase/backend-core/db") +const { getPouch, closePouchDB } = require("@budibase/backend-core/db") const Pouch = getPouch({ inMemory: true }) exports.runView = async (view, calculation, group, data) => { @@ -44,6 +44,6 @@ exports.runView = async (view, calculation, group, data) => { return response } finally { await db.destroy() - await closeDB(db) + await closePouchDB(db) } } diff --git a/packages/server/src/migrations/functions/backfill/app/tables.ts b/packages/server/src/migrations/functions/backfill/app/tables.ts index 6663c3c43b..c9d1e5c794 100644 --- a/packages/server/src/migrations/functions/backfill/app/tables.ts +++ b/packages/server/src/migrations/functions/backfill/app/tables.ts @@ -1,9 +1,8 @@ -import { events } from "@budibase/backend-core" +import { events, PouchLike } from "@budibase/backend-core" import sdk from "../../../../sdk" -import PouchDB from "pouchdb" export const backfill = async ( - appDb: PouchDB.Database, + appDb: PouchLike, timestamp: string | number ) => { const tables = await sdk.tables.getAllInternalTables(appDb) diff --git a/packages/server/src/sdk/app/backups/imports.ts b/packages/server/src/sdk/app/backups/imports.ts index 7cc9e0b0e6..634f507220 100644 --- a/packages/server/src/sdk/app/backups/imports.ts +++ b/packages/server/src/sdk/app/backups/imports.ts @@ -1,4 +1,4 @@ -import { db as dbCore } from "@budibase/backend-core" +import { db as dbCore, PouchLike } from "@budibase/backend-core" import { getAutomationParams, TABLE_ROW_PREFIX } from "../../../db/utils" import { budibaseTempDir } from "../../../utilities/budibaseDir" import { DB_EXPORT_FILE, GLOBAL_DB_EXPORT_FILE } from "./constants" @@ -17,7 +17,6 @@ import { CouchFindOptions, RowAttachment, } from "@budibase/types" -import PouchDB from "pouchdb" const uuid = require("uuid/v4") const tar = require("tar") @@ -29,10 +28,7 @@ type TemplateType = { key?: string } -async function updateAttachmentColumns( - prodAppId: string, - db: PouchDB.Database -) { +async function updateAttachmentColumns(prodAppId: string, db: PouchLike) { // iterate through attachment documents and update them const tables = await sdk.tables.getAllInternalTables(db) for (let table of tables) { @@ -86,7 +82,7 @@ async function updateAttachmentColumns( } } -async function updateAutomations(prodAppId: string, db: PouchDB.Database) { +async function updateAutomations(prodAppId: string, db: PouchLike) { const automations = ( await db.allDocs( getAutomationParams(null, { @@ -154,7 +150,7 @@ export function getListOfAppsInMulti(tmpPath: string) { export async function importApp( appId: string, - db: PouchDB.Database, + db: PouchLike, template: TemplateType ) { let prodAppId = dbCore.getProdAppID(appId) diff --git a/packages/server/src/sdk/app/backups/statistics.ts b/packages/server/src/sdk/app/backups/statistics.ts index 7a8e24dc58..9fe1a04d21 100644 --- a/packages/server/src/sdk/app/backups/statistics.ts +++ b/packages/server/src/sdk/app/backups/statistics.ts @@ -1,13 +1,12 @@ -import { context, db as dbCore } from "@budibase/backend-core" +import { context, db as dbCore, PouchLike } from "@budibase/backend-core" import { getDatasourceParams, getTableParams, getAutomationParams, getScreenParams, } from "../../../db/utils" -import PouchDB from "pouchdb" -async function runInContext(appId: string, cb: any, db?: PouchDB.Database) { +async function runInContext(appId: string, cb: any, db?: PouchLike) { if (db) { return cb(db) } else { @@ -19,13 +18,10 @@ async function runInContext(appId: string, cb: any, db?: PouchDB.Database) { } } -export async function calculateDatasourceCount( - appId: string, - db?: PouchDB.Database -) { +export async function calculateDatasourceCount(appId: string, db?: PouchLike) { return runInContext( appId, - async (db: PouchDB.Database) => { + async (db: PouchLike) => { const datasourceList = await db.allDocs(getDatasourceParams()) const tableList = await db.allDocs(getTableParams()) return datasourceList.rows.length + tableList.rows.length @@ -34,13 +30,10 @@ export async function calculateDatasourceCount( ) } -export async function calculateAutomationCount( - appId: string, - db?: PouchDB.Database -) { +export async function calculateAutomationCount(appId: string, db?: PouchLike) { return runInContext( appId, - async (db: PouchDB.Database) => { + async (db: PouchLike) => { const automationList = await db.allDocs(getAutomationParams()) return automationList.rows.length }, @@ -48,13 +41,10 @@ export async function calculateAutomationCount( ) } -export async function calculateScreenCount( - appId: string, - db?: PouchDB.Database -) { +export async function calculateScreenCount(appId: string, db?: PouchLike) { return runInContext( appId, - async (db: PouchDB.Database) => { + async (db: PouchLike) => { const screenList = await db.allDocs(getScreenParams()) return screenList.rows.length }, @@ -63,7 +53,7 @@ export async function calculateScreenCount( } export async function calculateBackupStats(appId: string) { - return runInContext(appId, async (db: PouchDB.Database) => { + return runInContext(appId, async (db: PouchLike) => { const promises = [] promises.push(calculateDatasourceCount(appId, db)) promises.push(calculateAutomationCount(appId, db)) diff --git a/packages/server/src/sdk/app/tables/index.ts b/packages/server/src/sdk/app/tables/index.ts index 5af92404a1..ef41630d6b 100644 --- a/packages/server/src/sdk/app/tables/index.ts +++ b/packages/server/src/sdk/app/tables/index.ts @@ -1,4 +1,4 @@ -import { getAppDB } from "@budibase/backend-core/context" +import { context, PouchLike } from "@budibase/backend-core" import { BudibaseInternalDB, getTableParams } from "../../../db/utils" import { breakExternalTableId, @@ -6,11 +6,10 @@ import { isSQL, } from "../../../integrations/utils" import { Table } from "@budibase/types" -import PouchDB from "pouchdb" -async function getAllInternalTables(db?: PouchDB.Database): Promise { +async function getAllInternalTables(db?: PouchLike): Promise { if (!db) { - db = getAppDB() as PouchDB.Database + db = context.getAppDB() } const internalTables = await db.allDocs( getTableParams(null, { @@ -25,7 +24,7 @@ async function getAllInternalTables(db?: PouchDB.Database): Promise { } async function getAllExternalTables(datasourceId: any): Promise { - const db = getAppDB() + const db = context.getAppDB() const datasource = await db.get(datasourceId) if (!datasource || !datasource.entities) { throw "Datasource is not configured fully." @@ -42,7 +41,7 @@ async function getExternalTable( } async function getTable(tableId: any): Promise { - const db = getAppDB() + const db = context.getAppDB() if (isExternalTable(tableId)) { let { datasourceId, tableName } = breakExternalTableId(tableId) const datasource = await db.get(datasourceId) From 9e01a9d1be447cd2c3eab047f5220bc6e8ea4e58 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 9 Nov 2022 17:53:35 +0000 Subject: [PATCH 04/24] Some minor updates based on getting the worker using the new pouchlike impl. --- packages/backend-core/src/couch/pouchLike.ts | 13 ++++++------- packages/backend-core/src/db/views.ts | 6 +++--- packages/types/src/documents/pouch.ts | 2 +- packages/worker/src/sdk/users/users.ts | 2 +- 4 files changed, 11 insertions(+), 12 deletions(-) diff --git a/packages/backend-core/src/couch/pouchLike.ts b/packages/backend-core/src/couch/pouchLike.ts index d605ec3d04..73555b1b7c 100644 --- a/packages/backend-core/src/couch/pouchLike.ts +++ b/packages/backend-core/src/couch/pouchLike.ts @@ -1,5 +1,5 @@ import Nano from "nano" -import { AnyDocument } from "@budibase/types" +import { AllDocsResponse, AnyDocument } from "@budibase/types" import { getCouchInfo } from "./couch" import { directCouchCall } from "./utils" import { getPouchDB } from "./pouchDB" @@ -23,10 +23,6 @@ export type QueryOpts = { keys?: string[] } -type QueryResp = Promise<{ - rows: { doc?: T | any; value?: any }[] -}> - export class PouchLike { public readonly name: string private static nano: Nano.ServerScope @@ -127,12 +123,15 @@ export class PouchLike { return this.updateOutput(() => db.bulk({ docs: documents })) } - async allDocs(params: QueryOpts): QueryResp { + async allDocs(params: QueryOpts): Promise> { const db = await this.checkSetup() return this.updateOutput(() => db.list(params)) } - async query(viewName: string, params: QueryOpts): QueryResp { + async query( + viewName: string, + params: QueryOpts + ): Promise> { const db = await this.checkSetup() const [database, view] = viewName.split("/") return this.updateOutput(() => db.view(database, view, params)) diff --git a/packages/backend-core/src/db/views.ts b/packages/backend-core/src/db/views.ts index 06919fd188..8ba508050f 100644 --- a/packages/backend-core/src/db/views.ts +++ b/packages/backend-core/src/db/views.ts @@ -131,16 +131,16 @@ export const queryView = async ( opts?: QueryViewOptions ): Promise => { try { - let response = await db.query(`database/${viewName}`, params) + let response = await db.query(`database/${viewName}`, params) const rows = response.rows const docs = rows.map(row => (params.include_docs ? row.doc : row.value)) // if arrayResponse has been requested, always return array regardless of length if (opts?.arrayResponse) { - return docs + return docs as T[] } else { // return the single document if there is only one - return docs.length <= 1 ? docs[0] : docs + return docs.length <= 1 ? (docs[0] as T) : (docs as T[]) } } catch (err: any) { if (err != null && err.name === "not_found") { diff --git a/packages/types/src/documents/pouch.ts b/packages/types/src/documents/pouch.ts index f9ed43b32f..fa66bfa57f 100644 --- a/packages/types/src/documents/pouch.ts +++ b/packages/types/src/documents/pouch.ts @@ -8,7 +8,7 @@ export interface RowResponse { key: string error: string value: RowValue - doc: T + doc?: T } export interface AllDocsResponse { diff --git a/packages/worker/src/sdk/users/users.ts b/packages/worker/src/sdk/users/users.ts index 539ac21300..e2215cee80 100644 --- a/packages/worker/src/sdk/users/users.ts +++ b/packages/worker/src/sdk/users/users.ts @@ -479,7 +479,7 @@ export const bulkDelete = async ( (user: RowResponse) => { return user.doc } - ) + ) as User[] // Delete from DB const toDelete = usersToDelete.map(user => ({ From 45e7ef61ef82c8a1d91e722ca6ee392847359b65 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 10 Nov 2022 16:38:32 +0000 Subject: [PATCH 05/24] Re-building the context module to use a single object, meaning we can create new context frames and copy over whatever exists, then update. --- .../backend-core/src/context/constants.ts | 4 + packages/backend-core/src/context/index.ts | 205 ++++++++++-------- packages/backend-core/src/couch/pouchLike.ts | 5 +- packages/backend-core/src/db/utils.ts | 17 +- .../backend-core/src/migrations/migrations.ts | 2 +- .../src/api/controllers/{dev.js => dev.ts} | 68 +++--- packages/server/src/middleware/currentapp.js | 3 +- 7 files changed, 159 insertions(+), 145 deletions(-) rename packages/server/src/api/controllers/{dev.js => dev.ts} (60%) diff --git a/packages/backend-core/src/context/constants.ts b/packages/backend-core/src/context/constants.ts index af30b1d241..14ab9a531c 100644 --- a/packages/backend-core/src/context/constants.ts +++ b/packages/backend-core/src/context/constants.ts @@ -1,4 +1,8 @@ export enum ContextKey { + MAIN = "main", +} + +export enum ContextElement { TENANT_ID = "tenantId", APP_ID = "appId", IDENTITY = "identity", diff --git a/packages/backend-core/src/context/index.ts b/packages/backend-core/src/context/index.ts index 30a53f1587..f88fe012a9 100644 --- a/packages/backend-core/src/context/index.ts +++ b/packages/backend-core/src/context/index.ts @@ -4,34 +4,36 @@ import cls from "./FunctionContext" import { baseGlobalDBName } from "../db/tenancy" import { IdentityContext } from "@budibase/types" import { DEFAULT_TENANT_ID as _DEFAULT_TENANT_ID } from "../constants" -import { ContextKey } from "./constants" +import { ContextElement, ContextKey } from "./constants" import { PouchLike } from "../couch" import { getDevelopmentAppID, getProdAppID } from "../db/conversions" +type ContextMap = { [key in ContextElement]?: any } + export const DEFAULT_TENANT_ID = _DEFAULT_TENANT_ID // some test cases call functions directly, need to // store an app ID to pretend there is a context let TEST_APP_ID: string | null = null -export const isMultiTenant = () => { +export function isMultiTenant() { return env.MULTI_TENANCY } -const setAppTenantId = (appId: string) => { - const appTenantId = getTenantIDFromAppID(appId) || DEFAULT_TENANT_ID - updateTenantId(appTenantId) +export function isTenantIdSet() { + const context = cls.getFromContext(ContextKey.MAIN) as ContextMap + return !!context?.[ContextElement.TENANT_ID] } -const setIdentity = (identity: IdentityContext | null) => { - cls.setOnContext(ContextKey.IDENTITY, identity) +export function isTenancyEnabled() { + return env.MULTI_TENANCY } /** * Given an app ID this will attempt to retrieve the tenant ID from it. * @return {null|string} The tenant ID found within the app ID. */ -export const getTenantIDFromAppID = (appId: string) => { +export function getTenantIDFromAppID(appId: string) { if (!appId) { return null } @@ -50,84 +52,134 @@ export const getTenantIDFromAppID = (appId: string) => { } } -export const doInContext = async (appId: string, task: any) => { - // gets the tenant ID from the app ID - const tenantId = getTenantIDFromAppID(appId) - return doInTenant(tenantId, async () => { - return doInAppContext(appId, async () => { - return task() - }) +function updateContext(updates: ContextMap) { + let context: ContextMap + try { + context = cls.getFromContext(ContextKey.MAIN) + } catch (err) { + // no context, start empty + context = {} + } + context = { + ...context, + ...updates, + } + return context +} + +async function newContext(updates: ContextMap, task: any) { + // see if there already is a context setup + let context: ContextMap = updateContext(updates) + return cls.run(async () => { + cls.setOnContext(ContextKey.MAIN, context) + return await task() }) } -export const doInTenant = (tenantId: string | null, task: any): any => { +export async function doInContext(appId: string, task: any): Promise { + const tenantId = getTenantIDFromAppID(appId) + return newContext( + { + [ContextElement.TENANT_ID]: tenantId, + [ContextElement.APP_ID]: appId, + }, + task + ) +} + +export async function doInTenant( + tenantId: string | null, + task: any +): Promise { // make sure default always selected in single tenancy if (!env.MULTI_TENANCY) { tenantId = tenantId || DEFAULT_TENANT_ID } - return cls.run(async () => { - updateTenantId(tenantId) - return await task() - }) + return newContext( + { + [ContextElement.TENANT_ID]: tenantId, + }, + task + ) } -export const doInAppContext = (appId: string, task: any): any => { +export async function doInAppContext(appId: string, task: any): Promise { if (!appId) { throw new Error("appId is required") } - const identity = getIdentity() - - return cls.run(async () => { - // set the app tenant id - setAppTenantId(appId) - // set the app ID - cls.setOnContext(ContextKey.APP_ID, appId) - - // preserve the identity - if (identity) { - setIdentity(identity) - } - // invoke the task - return await task() - }) + const tenantId = getTenantIDFromAppID(appId) + return newContext( + { + [ContextElement.TENANT_ID]: tenantId, + [ContextElement.APP_ID]: appId, + }, + task + ) } -export const doInIdentityContext = ( +export async function doInIdentityContext( identity: IdentityContext, task: any -): any => { +): Promise { if (!identity) { throw new Error("identity is required") } - return cls.run(async () => { - cls.setOnContext(ContextKey.IDENTITY, identity) - // set the tenant so that doInTenant will preserve identity - if (identity.tenantId) { - updateTenantId(identity.tenantId) - } - // invoke the task - return await task() - }) + const context: ContextMap = { + [ContextElement.IDENTITY]: identity, + } + if (identity.tenantId) { + context[ContextElement.TENANT_ID] = identity.tenantId + } + return newContext(context, task) } -export const getIdentity = (): IdentityContext | undefined => { +export function getIdentity(): IdentityContext | undefined { try { - return cls.getFromContext(ContextKey.IDENTITY) + const context = cls.getFromContext(ContextKey.MAIN) as ContextMap + return context?.[ContextElement.IDENTITY] } catch (e) { // do nothing - identity is not in context } } -export const updateTenantId = (tenantId: string | null) => { - cls.setOnContext(ContextKey.TENANT_ID, tenantId) +export function getTenantId(): string { + if (!isMultiTenant()) { + return DEFAULT_TENANT_ID + } + const context = cls.getFromContext(ContextKey.MAIN) as ContextMap + const tenantId = context?.[ContextElement.TENANT_ID] + if (!tenantId) { + throw new Error("Tenant id not found") + } + return tenantId } -export const updateAppId = async (appId: string) => { +export function getAppId(): string | undefined { + const context = cls.getFromContext(ContextKey.MAIN) as ContextMap + const foundId = context?.[ContextElement.APP_ID] + if (!foundId && env.isTest() && TEST_APP_ID) { + return TEST_APP_ID + } else { + return foundId + } +} + +export function updateTenantId(tenantId: string | null) { + let context: ContextMap = updateContext({ + [ContextElement.TENANT_ID]: tenantId, + }) + cls.setOnContext(ContextKey.MAIN, context) +} + +export function updateAppId(appId: string) { + let context: ContextMap = updateContext({ + [ContextElement.APP_ID]: appId, + }) try { - cls.setOnContext(ContextKey.APP_ID, appId) + cls.setOnContext(ContextKey.MAIN, context) } catch (err) { if (env.isTest()) { TEST_APP_ID = appId @@ -137,63 +189,34 @@ export const updateAppId = async (appId: string) => { } } -export const getGlobalDB = (): PouchLike => { - const tenantId = cls.getFromContext(ContextKey.TENANT_ID) - return new PouchLike(baseGlobalDBName(tenantId)) -} - -export const isTenantIdSet = () => { - const tenantId = cls.getFromContext(ContextKey.TENANT_ID) - return !!tenantId -} - -export const getTenantId = () => { - if (!isMultiTenant()) { - return DEFAULT_TENANT_ID - } - const tenantId = cls.getFromContext(ContextKey.TENANT_ID) - if (!tenantId) { - throw new Error("Tenant id not found") - } - return tenantId -} - -export const getAppId = () => { - const foundId = cls.getFromContext(ContextKey.APP_ID) - if (!foundId && env.isTest() && TEST_APP_ID) { - return TEST_APP_ID - } else { - return foundId - } -} - -export const isTenancyEnabled = () => { - return env.MULTI_TENANCY +export function getGlobalDB(): PouchLike { + const context = cls.getFromContext(ContextKey.MAIN) as ContextMap + return new PouchLike(baseGlobalDBName(context?.[ContextElement.TENANT_ID])) } /** - * Opens the app database based on whatever the request + * Gets the app database based on whatever the request * contained, dev or prod. */ -export const getAppDB = (opts?: any): PouchLike => { +export function getAppDB(opts?: any): PouchLike { const appId = getAppId() return new PouchLike(appId, opts) } /** * This specifically gets the prod app ID, if the request - * contained a development app ID, this will open the prod one. + * contained a development app ID, this will get the prod one. */ -export const getProdAppDB = (opts?: any): PouchLike => { +export function getProdAppDB(opts?: any): PouchLike { const appId = getAppId() return new PouchLike(getProdAppID(appId), opts) } /** * This specifically gets the dev app ID, if the request - * contained a prod app ID, this will open the dev one. + * contained a prod app ID, this will get the dev one. */ -export const getDevAppDB = (opts?: any): PouchLike => { +export function getDevAppDB(opts?: any): PouchLike { const appId = getAppId() return new PouchLike(getDevelopmentAppID(appId), opts) } diff --git a/packages/backend-core/src/couch/pouchLike.ts b/packages/backend-core/src/couch/pouchLike.ts index 73555b1b7c..fd5a94e1af 100644 --- a/packages/backend-core/src/couch/pouchLike.ts +++ b/packages/backend-core/src/couch/pouchLike.ts @@ -28,7 +28,10 @@ export class PouchLike { private static nano: Nano.ServerScope private readonly pouchOpts: PouchLikeOpts - constructor(dbName: string, opts?: PouchLikeOpts) { + constructor(dbName?: string, opts?: PouchLikeOpts) { + if (dbName == null) { + throw new Error("Database name cannot be undefined.") + } this.name = dbName this.pouchOpts = opts || {} if (!PouchLike.nano) { diff --git a/packages/backend-core/src/db/utils.ts b/packages/backend-core/src/db/utils.ts index c04da5da4f..a59b2ffe05 100644 --- a/packages/backend-core/src/db/utils.ts +++ b/packages/backend-core/src/db/utils.ts @@ -15,6 +15,7 @@ import { getAppMetadata } from "../cache/appMetadata" import { isDevApp, isDevAppID, getProdAppID } from "./conversions" import { APP_PREFIX } from "./constants" import * as events from "../events" +import { PouchLike } from "../couch" export * from "./constants" export * from "./conversions" @@ -254,7 +255,7 @@ export function getRoleParams(roleId = null, otherProps = {}) { return getDocParams(DocumentType.ROLE, roleId, otherProps) } -export function getStartEndKeyURL(baseKey: any, tenantId = null) { +export function getStartEndKeyURL(baseKey: any, tenantId?: string) { const tenancy = tenantId ? `${SEPARATOR}${tenantId}` : "" return `startkey="${baseKey}${tenancy}"&endkey="${baseKey}${tenancy}${UNICODE_MAX}"` } @@ -388,20 +389,10 @@ export async function getDevAppIDs() { } export async function dbExists(dbName: any) { - let exists = false return doWithDB( dbName, - async (db: any) => { - try { - // check if database exists - const info = await db.info() - if (info && !info.error) { - exists = true - } - } catch (err) { - exists = false - } - return exists + async (db: PouchLike) => { + return await db.exists() }, { skip_setup: true } ) diff --git a/packages/backend-core/src/migrations/migrations.ts b/packages/backend-core/src/migrations/migrations.ts index 90a12acec2..2170c5983a 100644 --- a/packages/backend-core/src/migrations/migrations.ts +++ b/packages/backend-core/src/migrations/migrations.ts @@ -41,7 +41,7 @@ export const runMigration = async ( options: MigrationOptions = {} ) => { const migrationType = migration.type - let tenantId: string + let tenantId: string | undefined if (migrationType !== MigrationType.INSTALLATION) { tenantId = getTenantId() } diff --git a/packages/server/src/api/controllers/dev.js b/packages/server/src/api/controllers/dev.ts similarity index 60% rename from packages/server/src/api/controllers/dev.js rename to packages/server/src/api/controllers/dev.ts index c8f134756b..9dbbe90555 100644 --- a/packages/server/src/api/controllers/dev.js +++ b/packages/server/src/api/controllers/dev.ts @@ -1,29 +1,23 @@ -const fetch = require("node-fetch") -const env = require("../../environment") -const { checkSlashesInUrl } = require("../../utilities") -const { request } = require("../../utilities/workerRequests") -const { clearLock } = require("../../utilities/redis") -const { Replication, getProdAppID } = require("@budibase/backend-core/db") -const { DocumentType } = require("../../db/utils") -const { app: appCache } = require("@budibase/backend-core/cache") -const { getProdAppDB, getAppDB } = require("@budibase/backend-core/context") -const { events } = require("@budibase/backend-core") +import fetch from "node-fetch" +import env from "../../environment" +import { checkSlashesInUrl } from "../../utilities" +import { request } from "../../utilities/workerRequests" +import { clearLock as redisClearLock } from "../../utilities/redis" +import { DocumentType } from "../../db/utils" +import { context } from "@budibase/backend-core" +import { events, db as dbCore, cache } from "@budibase/backend-core" -async function redirect(ctx, method, path = "global") { +async function redirect(ctx: any, method: string, path: string = "global") { const { devPath } = ctx.params const queryString = ctx.originalUrl.split("?")[1] || "" const response = await fetch( checkSlashesInUrl( `${env.WORKER_URL}/api/${path}/${devPath}?${queryString}` ), - request( - ctx, - { - method, - body: ctx.request.body, - }, - true - ) + request(ctx, { + method, + body: ctx.request.body, + }) ) if (response.status !== 200) { const err = await response.text() @@ -46,28 +40,28 @@ async function redirect(ctx, method, path = "global") { ctx.cookies } -exports.buildRedirectGet = path => { - return async ctx => { +export function buildRedirectGet(path: string) { + return async (ctx: any) => { await redirect(ctx, "GET", path) } } -exports.buildRedirectPost = path => { - return async ctx => { +export function buildRedirectPost(path: string) { + return async (ctx: any) => { await redirect(ctx, "POST", path) } } -exports.buildRedirectDelete = path => { - return async ctx => { +export function buildRedirectDelete(path: string) { + return async (ctx: any) => { await redirect(ctx, "DELETE", path) } } -exports.clearLock = async ctx => { +export async function clearLock(ctx: any) { const { appId } = ctx.params try { - await clearLock(appId, ctx.user) + await redisClearLock(appId, ctx.user) } catch (err) { ctx.throw(400, `Unable to remove lock. ${err}`) } @@ -76,16 +70,16 @@ exports.clearLock = async ctx => { } } -exports.revert = async ctx => { +export async function revert(ctx: any) { const { appId } = ctx.params - const productionAppId = getProdAppID(appId) + const productionAppId = dbCore.getProdAppID(appId) // App must have been deployed first try { - const db = getProdAppDB({ skip_setup: true }) - const info = await db.info() - if (info.error) { - throw info.error + const db = context.getProdAppDB({ skip_setup: true }) + const exists = await db.exists() + if (!exists) { + throw new Error("App must be deployed to be reverted.") } const deploymentDoc = await db.get(DocumentType.DEPLOYMENTS) if ( @@ -98,7 +92,7 @@ exports.revert = async ctx => { return ctx.throw(400, "App has not yet been deployed") } - const replication = new Replication({ + const replication = new dbCore.Replication({ source: productionAppId, target: appId, }) @@ -109,12 +103,12 @@ exports.revert = async ctx => { } // update appID in reverted app to be dev version again - const db = getAppDB() + const db = context.getAppDB() const appDoc = await db.get(DocumentType.APP_METADATA) appDoc.appId = appId appDoc.instance._id = appId await db.put(appDoc) - await appCache.invalidateAppMetadata(appId) + await cache.app.invalidateAppMetadata(appId) ctx.body = { message: "Reverted changes successfully.", } @@ -126,7 +120,7 @@ exports.revert = async ctx => { } } -exports.getBudibaseVersion = async ctx => { +export async function getBudibaseVersion(ctx: any) { const version = require("../../../package.json").version ctx.body = { version, diff --git a/packages/server/src/middleware/currentapp.js b/packages/server/src/middleware/currentapp.js index 8d55175d27..f9c4ae40db 100644 --- a/packages/server/src/middleware/currentapp.js +++ b/packages/server/src/middleware/currentapp.js @@ -95,8 +95,7 @@ module.exports = async (ctx, next) => { // need to judge this only based on the request app ID, if ( env.MULTI_TENANCY && - ctx.user && - requestAppId && + ctx.user & requestAppId && !isUserInAppTenant(requestAppId, ctx.user) ) { // don't error, simply remove the users rights (they are a public user) From c63c3b48c55f832ad2730ae128c18a7820b0c4fc Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 10 Nov 2022 17:38:26 +0000 Subject: [PATCH 06/24] Typescript conversions, as well as updating context to just use an object map. --- .../backend-core/src/context/constants.ts | 10 ++- packages/backend-core/src/context/index.ts | 44 +++++----- packages/backend-core/src/security/roles.ts | 8 +- packages/backend-core/src/tenancy/tenancy.ts | 43 ++++++---- .../{currentapp.js => currentapp.ts} | 84 ++++++++++--------- .../src/middleware/tests/currentapp.spec.js | 2 +- packages/types/src/sdk/koa.ts | 8 +- 7 files changed, 106 insertions(+), 93 deletions(-) rename packages/server/src/middleware/{currentapp.js => currentapp.ts} (58%) diff --git a/packages/backend-core/src/context/constants.ts b/packages/backend-core/src/context/constants.ts index 14ab9a531c..eb156c357b 100644 --- a/packages/backend-core/src/context/constants.ts +++ b/packages/backend-core/src/context/constants.ts @@ -1,9 +1,11 @@ +import { IdentityContext } from "@budibase/types" + export enum ContextKey { MAIN = "main", } -export enum ContextElement { - TENANT_ID = "tenantId", - APP_ID = "appId", - IDENTITY = "identity", +export type ContextMap = { + tenantId?: string + appId?: string + identity?: IdentityContext } diff --git a/packages/backend-core/src/context/index.ts b/packages/backend-core/src/context/index.ts index f88fe012a9..e8813c5a2f 100644 --- a/packages/backend-core/src/context/index.ts +++ b/packages/backend-core/src/context/index.ts @@ -4,12 +4,10 @@ import cls from "./FunctionContext" import { baseGlobalDBName } from "../db/tenancy" import { IdentityContext } from "@budibase/types" import { DEFAULT_TENANT_ID as _DEFAULT_TENANT_ID } from "../constants" -import { ContextElement, ContextKey } from "./constants" +import { ContextMap, ContextKey } from "./constants" import { PouchLike } from "../couch" import { getDevelopmentAppID, getProdAppID } from "../db/conversions" -type ContextMap = { [key in ContextElement]?: any } - export const DEFAULT_TENANT_ID = _DEFAULT_TENANT_ID // some test cases call functions directly, need to @@ -22,7 +20,7 @@ export function isMultiTenant() { export function isTenantIdSet() { const context = cls.getFromContext(ContextKey.MAIN) as ContextMap - return !!context?.[ContextElement.TENANT_ID] + return !!context?.tenantId } export function isTenancyEnabled() { @@ -35,7 +33,7 @@ export function isTenancyEnabled() { */ export function getTenantIDFromAppID(appId: string) { if (!appId) { - return null + return undefined } if (!isMultiTenant()) { return DEFAULT_TENANT_ID @@ -43,7 +41,7 @@ export function getTenantIDFromAppID(appId: string) { const split = appId.split(SEPARATOR) const hasDev = split[1] === DocumentType.DEV if ((hasDev && split.length === 3) || (!hasDev && split.length === 2)) { - return null + return undefined } if (hasDev) { return split[2] @@ -80,8 +78,8 @@ export async function doInContext(appId: string, task: any): Promise { const tenantId = getTenantIDFromAppID(appId) return newContext( { - [ContextElement.TENANT_ID]: tenantId, - [ContextElement.APP_ID]: appId, + tenantId, + appId, }, task ) @@ -96,12 +94,8 @@ export async function doInTenant( tenantId = tenantId || DEFAULT_TENANT_ID } - return newContext( - { - [ContextElement.TENANT_ID]: tenantId, - }, - task - ) + const updates = tenantId ? { tenantId } : {} + return newContext(updates, task) } export async function doInAppContext(appId: string, task: any): Promise { @@ -112,8 +106,8 @@ export async function doInAppContext(appId: string, task: any): Promise { const tenantId = getTenantIDFromAppID(appId) return newContext( { - [ContextElement.TENANT_ID]: tenantId, - [ContextElement.APP_ID]: appId, + tenantId, + appId, }, task ) @@ -128,10 +122,10 @@ export async function doInIdentityContext( } const context: ContextMap = { - [ContextElement.IDENTITY]: identity, + identity, } if (identity.tenantId) { - context[ContextElement.TENANT_ID] = identity.tenantId + context.tenantId = identity.tenantId } return newContext(context, task) } @@ -139,7 +133,7 @@ export async function doInIdentityContext( export function getIdentity(): IdentityContext | undefined { try { const context = cls.getFromContext(ContextKey.MAIN) as ContextMap - return context?.[ContextElement.IDENTITY] + return context?.identity } catch (e) { // do nothing - identity is not in context } @@ -150,7 +144,7 @@ export function getTenantId(): string { return DEFAULT_TENANT_ID } const context = cls.getFromContext(ContextKey.MAIN) as ContextMap - const tenantId = context?.[ContextElement.TENANT_ID] + const tenantId = context?.tenantId if (!tenantId) { throw new Error("Tenant id not found") } @@ -159,7 +153,7 @@ export function getTenantId(): string { export function getAppId(): string | undefined { const context = cls.getFromContext(ContextKey.MAIN) as ContextMap - const foundId = context?.[ContextElement.APP_ID] + const foundId = context?.appId if (!foundId && env.isTest() && TEST_APP_ID) { return TEST_APP_ID } else { @@ -167,16 +161,16 @@ export function getAppId(): string | undefined { } } -export function updateTenantId(tenantId: string | null) { +export function updateTenantId(tenantId?: string) { let context: ContextMap = updateContext({ - [ContextElement.TENANT_ID]: tenantId, + tenantId, }) cls.setOnContext(ContextKey.MAIN, context) } export function updateAppId(appId: string) { let context: ContextMap = updateContext({ - [ContextElement.APP_ID]: appId, + appId, }) try { cls.setOnContext(ContextKey.MAIN, context) @@ -191,7 +185,7 @@ export function updateAppId(appId: string) { export function getGlobalDB(): PouchLike { const context = cls.getFromContext(ContextKey.MAIN) as ContextMap - return new PouchLike(baseGlobalDBName(context?.[ContextElement.TENANT_ID])) + return new PouchLike(baseGlobalDBName(context?.tenantId)) } /** diff --git a/packages/backend-core/src/security/roles.ts b/packages/backend-core/src/security/roles.ts index 1064936fd7..cba88d9751 100644 --- a/packages/backend-core/src/security/roles.ts +++ b/packages/backend-core/src/security/roles.ts @@ -147,9 +147,9 @@ export function lowerBuiltinRoleID(roleId1?: string, roleId2?: string) { * @param {string|null} roleId The level ID to lookup. * @returns {Promise} The role object, which may contain an "inherits" property. */ -export async function getRole(roleId?: string) { +export async function getRole(roleId?: string): Promise { if (!roleId) { - return null + return undefined } let role: any = {} // built in roles mostly come from the in-code implementation, @@ -193,7 +193,9 @@ async function getAllUserRoles(userRoleId?: string): Promise { ) { roleIds.push(currentRole.inherits) currentRole = await getRole(currentRole.inherits) - roles.push(currentRole) + if (currentRole) { + roles.push(currentRole) + } } return roles } diff --git a/packages/backend-core/src/tenancy/tenancy.ts b/packages/backend-core/src/tenancy/tenancy.ts index ad5c6b5287..55ac66b95c 100644 --- a/packages/backend-core/src/tenancy/tenancy.ts +++ b/packages/backend-core/src/tenancy/tenancy.ts @@ -3,10 +3,10 @@ import { queryPlatformView } from "../db/views" import { StaticDatabases, ViewName } from "../db/constants" import { getGlobalDBName } from "../db/tenancy" import { - getTenantId, DEFAULT_TENANT_ID, - isMultiTenant, + getTenantId, getTenantIDFromAppID, + isMultiTenant, } from "../context" import env from "../environment" import { PlatformUser } from "@budibase/types" @@ -14,7 +14,7 @@ import { PlatformUser } from "@budibase/types" const TENANT_DOC = StaticDatabases.PLATFORM_INFO.docs.tenants const PLATFORM_INFO_DB = StaticDatabases.PLATFORM_INFO.name -export const addTenantToUrl = (url: string) => { +export function addTenantToUrl(url: string) { const tenantId = getTenantId() if (isMultiTenant()) { @@ -25,7 +25,7 @@ export const addTenantToUrl = (url: string) => { return url } -export const doesTenantExist = async (tenantId: string) => { +export async function doesTenantExist(tenantId: string) { return doWithDB(PLATFORM_INFO_DB, async (db: any) => { let tenants try { @@ -42,12 +42,12 @@ export const doesTenantExist = async (tenantId: string) => { }) } -export const tryAddTenant = async ( +export async function tryAddTenant( tenantId: string, userId: string, email: string, afterCreateTenant: () => Promise -) => { +) { return doWithDB(PLATFORM_INFO_DB, async (db: any) => { const getDoc = async (id: string) => { if (!id) { @@ -89,11 +89,11 @@ export const tryAddTenant = async ( }) } -export const doWithGlobalDB = (tenantId: string, cb: any) => { +export function doWithGlobalDB(tenantId: string, cb: any) { return doWithDB(getGlobalDBName(tenantId), cb) } -export const lookupTenantId = async (userId: string) => { +export async function lookupTenantId(userId: string) { return doWithDB(StaticDatabases.PLATFORM_INFO.name, async (db: any) => { let tenantId = env.MULTI_TENANCY ? DEFAULT_TENANT_ID : null try { @@ -109,19 +109,26 @@ export const lookupTenantId = async (userId: string) => { } // lookup, could be email or userId, either will return a doc -export const getTenantUser = async ( +export async function getTenantUser( identifier: string -): Promise => { +): Promise { // use the view here and allow to find anyone regardless of casing - // Use lowercase to ensure email login is case insensitive - const response = queryPlatformView(ViewName.PLATFORM_USERS_LOWERCASE, { - keys: [identifier.toLowerCase()], - include_docs: true, - }) as Promise - return response + // Use lowercase to ensure email login is case-insensitive + const users = await queryPlatformView( + ViewName.PLATFORM_USERS_LOWERCASE, + { + keys: [identifier.toLowerCase()], + include_docs: true, + } + ) + if (Array.isArray(users)) { + return users[0] + } else { + return users + } } -export const isUserInAppTenant = (appId: string, user?: any) => { +export function isUserInAppTenant(appId: string, user?: any) { let userTenantId if (user) { userTenantId = user.tenantId || DEFAULT_TENANT_ID @@ -132,7 +139,7 @@ export const isUserInAppTenant = (appId: string, user?: any) => { return tenantId === userTenantId } -export const getTenantIds = async () => { +export async function getTenantIds() { return doWithDB(PLATFORM_INFO_DB, async (db: any) => { let tenants try { diff --git a/packages/server/src/middleware/currentapp.js b/packages/server/src/middleware/currentapp.ts similarity index 58% rename from packages/server/src/middleware/currentapp.js rename to packages/server/src/middleware/currentapp.ts index f9c4ae40db..9496ddd0a0 100644 --- a/packages/server/src/middleware/currentapp.js +++ b/packages/server/src/middleware/currentapp.ts @@ -1,29 +1,26 @@ -const { - getAppIdFromCtx, - setCookie, - getCookie, - clearCookie, -} = require("@budibase/backend-core/utils") -const { Cookies, Headers } = require("@budibase/backend-core/constants") -const { getRole } = require("@budibase/backend-core/roles") -const { BUILTIN_ROLE_IDS } = require("@budibase/backend-core/roles") -const { generateUserMetadataID, isDevAppID } = require("../db/utils") -const { dbExists } = require("@budibase/backend-core/db") -const { isUserInAppTenant } = require("@budibase/backend-core/tenancy") -const { getCachedSelf } = require("../utilities/global") -const env = require("../environment") -const { isWebhookEndpoint } = require("./utils") -const { doInAppContext } = require("@budibase/backend-core/context") +import { + utils, + constants, + roles, + db as dbCore, + tenancy, + context, +} from "@budibase/backend-core" +import { generateUserMetadataID, isDevAppID } from "../db/utils" +import { getCachedSelf } from "../utilities/global" +import env from "../environment" +import { isWebhookEndpoint } from "./utils" +import { BBContext } from "@budibase/types" -module.exports = async (ctx, next) => { +export = async (ctx: BBContext, next: any) => { // try to get the appID from the request - let requestAppId = await getAppIdFromCtx(ctx) + let requestAppId = await utils.getAppIdFromCtx(ctx) // get app cookie if it exists - let appCookie = null + let appCookie: { appId?: string } | undefined try { - appCookie = getCookie(ctx, Cookies.CurrentApp) + appCookie = utils.getCookie(ctx, constants.Cookies.CurrentApp) } catch (err) { - clearCookie(ctx, Cookies.CurrentApp) + utils.clearCookie(ctx, constants.Cookies.CurrentApp) } if (!appCookie && !requestAppId) { return next() @@ -31,9 +28,9 @@ module.exports = async (ctx, next) => { // check the app exists referenced in cookie if (appCookie) { const appId = appCookie.appId - const exists = await dbExists(appId) + const exists = await dbCore.dbExists(appId) if (!exists) { - clearCookie(ctx, Cookies.CurrentApp) + utils.clearCookie(ctx, constants.Cookies.CurrentApp) return next() } // if the request app ID wasn't set, update it with the cookie @@ -47,13 +44,13 @@ module.exports = async (ctx, next) => { !isWebhookEndpoint(ctx) && (!ctx.user || !ctx.user.builder || !ctx.user.builder.global) ) { - clearCookie(ctx, Cookies.CurrentApp) + utils.clearCookie(ctx, constants.Cookies.CurrentApp) return ctx.redirect("/") } } - let appId, - roleId = BUILTIN_ROLE_IDS.PUBLIC + let appId: string | undefined, + roleId = roles.BUILTIN_ROLE_IDS.PUBLIC if (!ctx.user) { // not logged in, try to set a cookie for public apps appId = requestAppId @@ -68,16 +65,20 @@ module.exports = async (ctx, next) => { const isBuilder = globalUser && globalUser.builder && globalUser.builder.global const isDevApp = appId && isDevAppID(appId) - const roleHeader = ctx.request && ctx.request.headers[Headers.PREVIEW_ROLE] + const roleHeader = + ctx.request && + (ctx.request.headers[constants.Headers.PREVIEW_ROLE] as string) if (isBuilder && isDevApp && roleHeader) { // Ensure the role is valid by ensuring a definition exists try { - await getRole(roleHeader) - roleId = roleHeader + if (roleHeader) { + await roles.getRole(roleHeader) + roleId = roleHeader - // Delete admin and builder flags so that the specified role is honoured - delete ctx.user.builder - delete ctx.user.admin + // Delete admin and builder flags so that the specified role is honoured + delete ctx.user.builder + delete ctx.user.admin + } } catch (error) { // Swallow error and do nothing } @@ -89,21 +90,22 @@ module.exports = async (ctx, next) => { return next() } - return doInAppContext(appId, async () => { + return context.doInAppContext(appId, async () => { let skipCookie = false // if the user not in the right tenant then make sure they have no permissions // need to judge this only based on the request app ID, if ( env.MULTI_TENANCY && - ctx.user & requestAppId && - !isUserInAppTenant(requestAppId, ctx.user) + ctx.user && + requestAppId && + !tenancy.isUserInAppTenant(requestAppId, ctx.user) ) { // don't error, simply remove the users rights (they are a public user) delete ctx.user.builder delete ctx.user.admin delete ctx.user.roles ctx.isAuthenticated = false - roleId = BUILTIN_ROLE_IDS.PUBLIC + roleId = roles.BUILTIN_ROLE_IDS.PUBLIC skipCookie = true } @@ -111,15 +113,17 @@ module.exports = async (ctx, next) => { if (roleId) { ctx.roleId = roleId const globalId = ctx.user ? ctx.user._id : undefined - const userId = ctx.user ? generateUserMetadataID(ctx.user._id) : null + const userId = ctx.user + ? generateUserMetadataID(ctx.user._id!) + : undefined ctx.user = { - ...ctx.user, + ...ctx.user!, // override userID with metadata one _id: userId, userId, globalId, roleId, - role: await getRole(roleId), + role: await roles.getRole(roleId), } } if ( @@ -128,7 +132,7 @@ module.exports = async (ctx, next) => { appCookie.appId !== requestAppId) && !skipCookie ) { - setCookie(ctx, { appId }, Cookies.CurrentApp) + utils.setCookie(ctx, { appId }, constants.Cookies.CurrentApp) } return next() diff --git a/packages/server/src/middleware/tests/currentapp.spec.js b/packages/server/src/middleware/tests/currentapp.spec.js index 57c21b2107..638abcbba4 100644 --- a/packages/server/src/middleware/tests/currentapp.spec.js +++ b/packages/server/src/middleware/tests/currentapp.spec.js @@ -7,7 +7,7 @@ jest.mock("@budibase/backend-core/db", () => { coreDb.init() return { ...coreDb, - dbExists: () => true, + dbExists: async () => true, } }) diff --git a/packages/types/src/sdk/koa.ts b/packages/types/src/sdk/koa.ts index 8004ba72ae..b8831dcbee 100644 --- a/packages/types/src/sdk/koa.ts +++ b/packages/types/src/sdk/koa.ts @@ -1,10 +1,14 @@ import { Context, Request } from "koa" -import { User } from "../documents" +import { User, Role, UserRoles } from "../documents" import { License } from "../sdk" -export interface ContextUser extends User { +export interface ContextUser extends Omit { globalId?: string license: License + userId?: string + roleId?: string + role?: Role + roles?: UserRoles } export interface BBRequest extends Request { From bc94f20794428d58ba4ddca3dfaa7702df318e1f Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 11 Nov 2022 11:57:50 +0000 Subject: [PATCH 07/24] Typescript conversions - trying to get all of context/db layer into TS. --- ...{FunctionContext.js => FunctionContext.ts} | 14 ++-- .../{deprovision.js => deprovision.ts} | 68 ++++++++++--------- packages/backend-core/src/context/index.ts | 6 ++ .../src/db/{conversions.js => conversions.ts} | 21 +++--- packages/backend-core/src/index.ts | 2 +- packages/backend-core/src/users.ts | 2 +- .../src/api/controllers/deploy/index.ts | 38 ++++------- packages/server/src/automations/utils.ts | 21 +++--- packages/types/src/documents/pouch.ts | 2 +- 9 files changed, 85 insertions(+), 89 deletions(-) rename packages/backend-core/src/context/{FunctionContext.js => FunctionContext.ts} (79%) rename packages/backend-core/src/context/{deprovision.js => deprovision.ts} (64%) rename packages/backend-core/src/db/{conversions.js => conversions.ts} (68%) diff --git a/packages/backend-core/src/context/FunctionContext.js b/packages/backend-core/src/context/FunctionContext.ts similarity index 79% rename from packages/backend-core/src/context/FunctionContext.js rename to packages/backend-core/src/context/FunctionContext.ts index c0ed34fe78..1010f585ef 100644 --- a/packages/backend-core/src/context/FunctionContext.js +++ b/packages/backend-core/src/context/FunctionContext.ts @@ -1,10 +1,10 @@ -const cls = require("../clshooked") -const { newid } = require("../hashing") +import cls from "../clshooked" +import { newid } from "../hashing" const REQUEST_ID_KEY = "requestId" const MAIN_CTX = cls.createNamespace("main") -function getContextStorage(namespace) { +function getContextStorage(namespace: any) { if (namespace && namespace.active) { let contextData = namespace.active delete contextData.id @@ -15,7 +15,7 @@ function getContextStorage(namespace) { } class FunctionContext { - static run(callback) { + static run(callback: any) { return MAIN_CTX.runAndReturn(async () => { const namespaceId = newid() MAIN_CTX.set(REQUEST_ID_KEY, namespaceId) @@ -26,13 +26,13 @@ class FunctionContext { }) } - static setOnContext(key, value) { + static setOnContext(key: string, value: any) { const namespaceId = MAIN_CTX.get(REQUEST_ID_KEY) const namespace = cls.getNamespace(namespaceId) namespace.set(key, value) } - static getFromContext(key) { + static getFromContext(key: string) { const namespaceId = MAIN_CTX.get(REQUEST_ID_KEY) const namespace = cls.getNamespace(namespaceId) const context = getContextStorage(namespace) @@ -44,4 +44,4 @@ class FunctionContext { } } -module.exports = FunctionContext +export = FunctionContext diff --git a/packages/backend-core/src/context/deprovision.js b/packages/backend-core/src/context/deprovision.ts similarity index 64% rename from packages/backend-core/src/context/deprovision.js rename to packages/backend-core/src/context/deprovision.ts index ba3c2d8449..179c7f407a 100644 --- a/packages/backend-core/src/context/deprovision.js +++ b/packages/backend-core/src/context/deprovision.ts @@ -1,16 +1,20 @@ -const { getGlobalUserParams, getAllApps } = require("../db/utils") -const { doWithDB } = require("../db") -const { doWithGlobalDB } = require("../tenancy") -const { StaticDatabases } = require("../db/constants") +import { getGlobalUserParams, getAllApps } from "../db/utils" +import { doWithDB } from "../db" +import { doWithGlobalDB } from "../tenancy" +import { StaticDatabases } from "../db/constants" +import { PouchLike } from "../couch" +import { User } from "@budibase/types" const TENANT_DOC = StaticDatabases.PLATFORM_INFO.docs.tenants const PLATFORM_INFO_DB = StaticDatabases.PLATFORM_INFO.name -const removeTenantFromInfoDB = async tenantId => { +async function removeTenantFromInfoDB(tenantId: string) { try { - await doWithDB(PLATFORM_INFO_DB, async infoDb => { + await doWithDB(PLATFORM_INFO_DB, async (infoDb: PouchLike) => { let tenants = await infoDb.get(TENANT_DOC) - tenants.tenantIds = tenants.tenantIds.filter(id => id !== tenantId) + tenants.tenantIds = tenants.tenantIds.filter( + (id: string) => id !== tenantId + ) await infoDb.put(tenants) }) @@ -20,32 +24,15 @@ const removeTenantFromInfoDB = async tenantId => { } } -exports.removeUserFromInfoDB = async dbUser => { - await doWithDB(PLATFORM_INFO_DB, async infoDb => { - const keys = [dbUser._id, dbUser.email] - const userDocs = await infoDb.allDocs({ - keys, - include_docs: true, - }) - const toDelete = userDocs.rows.map(row => { - return { - ...row.doc, - _deleted: true, - } - }) - await infoDb.bulkDocs(toDelete) - }) -} - -const removeUsersFromInfoDB = async tenantId => { - return doWithGlobalDB(tenantId, async db => { +async function removeUsersFromInfoDB(tenantId: string) { + return doWithGlobalDB(tenantId, async (db: PouchLike) => { try { const allUsers = await db.allDocs( getGlobalUserParams(null, { include_docs: true, }) ) - await doWithDB(PLATFORM_INFO_DB, async infoDb => { + await doWithDB(PLATFORM_INFO_DB, async (infoDb: PouchLike) => { const allEmails = allUsers.rows.map(row => row.doc.email) // get the id docs let keys = allUsers.rows.map(row => row.id) @@ -71,8 +58,8 @@ const removeUsersFromInfoDB = async tenantId => { }) } -const removeGlobalDB = async tenantId => { - return doWithGlobalDB(tenantId, async db => { +async function removeGlobalDB(tenantId: string) { + return doWithGlobalDB(tenantId, async (db: PouchLike) => { try { await db.destroy() } catch (err) { @@ -82,11 +69,11 @@ const removeGlobalDB = async tenantId => { }) } -const removeTenantApps = async tenantId => { +async function removeTenantApps(tenantId: string) { try { const apps = await getAllApps({ all: true }) const destroyPromises = apps.map(app => - doWithDB(app.appId, db => db.destroy()) + doWithDB(app.appId, (db: PouchLike) => db.destroy()) ) await Promise.allSettled(destroyPromises) } catch (err) { @@ -95,8 +82,25 @@ const removeTenantApps = async tenantId => { } } +export async function removeUserFromInfoDB(dbUser: User) { + await doWithDB(PLATFORM_INFO_DB, async (infoDb: PouchLike) => { + const keys = [dbUser._id!, dbUser.email] + const userDocs = await infoDb.allDocs({ + keys, + include_docs: true, + }) + const toDelete = userDocs.rows.map(row => { + return { + ...row.doc, + _deleted: true, + } + }) + await infoDb.bulkDocs(toDelete) + }) +} + // can't live in tenancy package due to circular dependency on db/utils -exports.deleteTenant = async tenantId => { +export async function deleteTenant(tenantId: string) { await removeTenantFromInfoDB(tenantId) await removeUsersFromInfoDB(tenantId) await removeGlobalDB(tenantId) diff --git a/packages/backend-core/src/context/index.ts b/packages/backend-core/src/context/index.ts index e8813c5a2f..63b57a1b6e 100644 --- a/packages/backend-core/src/context/index.ts +++ b/packages/backend-core/src/context/index.ts @@ -203,6 +203,9 @@ export function getAppDB(opts?: any): PouchLike { */ export function getProdAppDB(opts?: any): PouchLike { const appId = getAppId() + if (!appId) { + throw new Error("Unable to retrieve prod DB - no app ID.") + } return new PouchLike(getProdAppID(appId), opts) } @@ -212,5 +215,8 @@ export function getProdAppDB(opts?: any): PouchLike { */ export function getDevAppDB(opts?: any): PouchLike { const appId = getAppId() + if (!appId) { + throw new Error("Unable to retrieve dev DB - no app ID.") + } return new PouchLike(getDevelopmentAppID(appId), opts) } diff --git a/packages/backend-core/src/db/conversions.js b/packages/backend-core/src/db/conversions.ts similarity index 68% rename from packages/backend-core/src/db/conversions.js rename to packages/backend-core/src/db/conversions.ts index 5b1a785ecc..48eaf31844 100644 --- a/packages/backend-core/src/db/conversions.js +++ b/packages/backend-core/src/db/conversions.ts @@ -1,32 +1,33 @@ +import { APP_DEV_PREFIX, APP_PREFIX } from "./constants" +import { App } from "@budibase/types" const NO_APP_ERROR = "No app provided" -const { APP_DEV_PREFIX, APP_PREFIX } = require("./constants") -exports.isDevAppID = appId => { +export function isDevAppID(appId?: string) { if (!appId) { throw NO_APP_ERROR } return appId.startsWith(APP_DEV_PREFIX) } -exports.isProdAppID = appId => { +export function isProdAppID(appId?: string) { if (!appId) { throw NO_APP_ERROR } - return appId.startsWith(APP_PREFIX) && !exports.isDevAppID(appId) + return appId.startsWith(APP_PREFIX) && !isDevAppID(appId) } -exports.isDevApp = app => { +export function isDevApp(app: App) { if (!app) { throw NO_APP_ERROR } - return exports.isDevAppID(app.appId) + return isDevAppID(app.appId) } /** * Generates a development app ID from a real app ID. * @returns {string} the dev app ID which can be used for dev database. */ -exports.getDevelopmentAppID = appId => { +export function getDevelopmentAppID(appId: string) { if (!appId || appId.startsWith(APP_DEV_PREFIX)) { return appId } @@ -36,12 +37,12 @@ exports.getDevelopmentAppID = appId => { const rest = split.join(APP_PREFIX) return `${APP_DEV_PREFIX}${rest}` } -exports.getDevAppID = exports.getDevelopmentAppID +export const getDevAppID = getDevelopmentAppID /** * Convert a development app ID to a deployed app ID. */ -exports.getProdAppID = appId => { +export function getProdAppID(appId: string) { if (!appId || !appId.startsWith(APP_DEV_PREFIX)) { return appId } @@ -52,7 +53,7 @@ exports.getProdAppID = appId => { return `${APP_PREFIX}${rest}` } -exports.extractAppUUID = id => { +export function extractAppUUID(id: string) { const split = id?.split("_") || [] return split.length ? split[split.length - 1] : null } diff --git a/packages/backend-core/src/index.ts b/packages/backend-core/src/index.ts index 06997ced90..b45248abc2 100644 --- a/packages/backend-core/src/index.ts +++ b/packages/backend-core/src/index.ts @@ -11,7 +11,7 @@ import env from "./environment" import tenancy from "./tenancy" import featureFlags from "./featureFlags" import * as sessions from "./security/sessions" -import deprovisioning from "./context/deprovision" +import * as deprovisioning from "./context/deprovision" import auth from "./auth" import constants from "./constants" import * as dbConstants from "./db/constants" diff --git a/packages/backend-core/src/users.ts b/packages/backend-core/src/users.ts index a38debfc19..13e8f8d36e 100644 --- a/packages/backend-core/src/users.ts +++ b/packages/backend-core/src/users.ts @@ -68,7 +68,7 @@ export const getGlobalUserByAppPage = (appId: string, user: User) => { if (!user) { return } - return generateAppUserID(getProdAppID(appId), user._id!) + return generateAppUserID(getProdAppID(appId)!, user._id!) } /** diff --git a/packages/server/src/api/controllers/deploy/index.ts b/packages/server/src/api/controllers/deploy/index.ts index cb4534a1a5..a87e7fbd7d 100644 --- a/packages/server/src/api/controllers/deploy/index.ts +++ b/packages/server/src/api/controllers/deploy/index.ts @@ -1,23 +1,11 @@ import Deployment from "./Deployment" -import { - getDevelopmentAppID, - getProdAppID, - Replication, -} from "@budibase/backend-core/db" +import { context, db as dbCore, events, cache } from "@budibase/backend-core" import { DocumentType, getAutomationParams } from "../../../db/utils" import { clearMetadata, disableAllCrons, enableCronTrigger, } from "../../../automations/utils" -import { app as appCache } from "@budibase/backend-core/cache" -import { - getAppDB, - getAppId, - getDevAppDB, - getProdAppDB, -} from "@budibase/backend-core/context" -import { events } from "@budibase/backend-core" import { backups } from "@budibase/pro" import { AppBackupTrigger } from "@budibase/types" @@ -49,7 +37,7 @@ async function checkAllDeployments(deployments: any) { async function storeDeploymentHistory(deployment: any) { const deploymentJSON = deployment.getJSON() - const db = getAppDB() + const db = context.getAppDB() let deploymentDoc try { @@ -77,7 +65,7 @@ async function storeDeploymentHistory(deployment: any) { } async function initDeployedApp(prodAppId: any) { - const db = getProdAppDB() + const db = context.getProdAppDB() console.log("Reading automation docs") const automations = ( await db.allDocs( @@ -103,9 +91,9 @@ async function initDeployedApp(prodAppId: any) { async function deployApp(deployment: any, userId: string) { let replication try { - const appId = getAppId() - const devAppId = getDevelopmentAppID(appId) - const productionAppId = getProdAppID(appId) + const appId = context.getAppId()! + const devAppId = dbCore.getDevelopmentAppID(appId) + const productionAppId = dbCore.getProdAppID(appId) // don't try this if feature isn't allowed, will error if (await backups.isEnabled()) { @@ -122,8 +110,8 @@ async function deployApp(deployment: any, userId: string) { source: devAppId, target: productionAppId, } - replication = new Replication(config) - const devDb = getDevAppDB() + replication = new dbCore.Replication(config) + const devDb = context.getDevAppDB() console.log("Compacting development DB") await devDb.compact() console.log("Replication object created") @@ -131,7 +119,7 @@ async function deployApp(deployment: any, userId: string) { console.log("replication complete.. replacing app meta doc") // app metadata is excluded as it is likely to be in conflict // replicate the app metadata document manually - const db = getProdAppDB() + const db = context.getProdAppDB() const appDoc = await devDb.get(DocumentType.APP_METADATA) try { const prodAppDoc = await db.get(DocumentType.APP_METADATA) @@ -147,7 +135,7 @@ async function deployApp(deployment: any, userId: string) { // remove automation errors if they exist delete appDoc.automationErrors await db.put(appDoc) - await appCache.invalidateAppMetadata(productionAppId) + await cache.app.invalidateAppMetadata(productionAppId) console.log("New app doc written successfully.") await initDeployedApp(productionAppId) console.log("Deployed app initialised, setting deployment to successful") @@ -170,7 +158,7 @@ async function deployApp(deployment: any, userId: string) { export async function fetchDeployments(ctx: any) { try { - const db = getAppDB() + const db = context.getAppDB() const deploymentDoc = await db.get(DocumentType.DEPLOYMENTS) const { updated, deployments } = await checkAllDeployments(deploymentDoc) if (updated) { @@ -184,7 +172,7 @@ export async function fetchDeployments(ctx: any) { export async function deploymentProgress(ctx: any) { try { - const db = getAppDB() + const db = context.getAppDB() const deploymentDoc = await db.get(DocumentType.DEPLOYMENTS) ctx.body = deploymentDoc[ctx.params.deploymentId] } catch (err) { @@ -197,7 +185,7 @@ export async function deploymentProgress(ctx: any) { const isFirstDeploy = async () => { try { - const db = getProdAppDB() + const db = context.getProdAppDB() await db.get(DocumentType.APP_METADATA) } catch (e: any) { if (e.status === 404) { diff --git a/packages/server/src/automations/utils.ts b/packages/server/src/automations/utils.ts index af4bb8d3af..5296a0fa50 100644 --- a/packages/server/src/automations/utils.ts +++ b/packages/server/src/automations/utils.ts @@ -4,15 +4,9 @@ import { automationQueue } from "./bullboard" import newid from "../db/newid" import { updateEntityMetadata } from "../utilities" import { MetadataTypes } from "../constants" -import { getProdAppID, doWithDB } from "@budibase/backend-core/db" +import { db as dbCore, context } from "@budibase/backend-core" import { getAutomationMetadataParams } from "../db/utils" import { cloneDeep } from "lodash/fp" -import { - getAppDB, - getAppId, - getProdAppDB, -} from "@budibase/backend-core/context" -import { context } from "@budibase/backend-core" import { quotas } from "@budibase/pro" import { Automation, WebhookActionType } from "@budibase/types" import sdk from "../sdk" @@ -102,7 +96,7 @@ export async function disableCronById(jobId: number | string) { } export async function clearMetadata() { - const db = getProdAppDB() + const db = context.getProdAppDB() const automationMetadata = ( await db.allDocs( getAutomationMetadataParams({ @@ -157,7 +151,7 @@ export async function enableCronTrigger(appId: any, automation: Automation) { // can't use getAppDB here as this is likely to be called from dev app, // but this call could be for dev app or prod app, need to just use what // was passed in - await doWithDB(appId, async (db: any) => { + await dbCore.doWithDB(appId, async (db: any) => { const response = await db.put(automation) automation._id = response.id automation._rev = response.rev @@ -175,7 +169,10 @@ export async function enableCronTrigger(appId: any, automation: Automation) { * written to DB (this does not write to DB as it would be wasteful to repeat). */ export async function checkForWebhooks({ oldAuto, newAuto }: any) { - const appId = getAppId() + const appId = context.getAppId() + if (!appId) { + throw new Error("Unable to check webhooks - no app ID in context.") + } const oldTrigger = oldAuto ? oldAuto.definition.trigger : null const newTrigger = newAuto ? newAuto.definition.trigger : null const triggerChanged = @@ -194,7 +191,7 @@ export async function checkForWebhooks({ oldAuto, newAuto }: any) { oldTrigger.webhookId ) { try { - let db = getAppDB() + let db = context.getAppDB() // need to get the webhook to get the rev const webhook = await db.get(oldTrigger.webhookId) // might be updating - reset the inputs to remove the URLs @@ -224,7 +221,7 @@ export async function checkForWebhooks({ oldAuto, newAuto }: any) { // the app ID has to be development for this endpoint // it can only be used when building the app // but the trigger endpoint will always be used in production - const prodAppId = getProdAppID(appId) + const prodAppId = dbCore.getProdAppID(appId) newTrigger.inputs = { schemaUrl: `api/webhooks/schema/${appId}/${id}`, triggerUrl: `api/webhooks/trigger/${prodAppId}/${id}`, diff --git a/packages/types/src/documents/pouch.ts b/packages/types/src/documents/pouch.ts index fa66bfa57f..269d8cbbae 100644 --- a/packages/types/src/documents/pouch.ts +++ b/packages/types/src/documents/pouch.ts @@ -8,7 +8,7 @@ export interface RowResponse { key: string error: string value: RowValue - doc?: T + doc?: T | any } export interface AllDocsResponse { From b4b471e054552ef13d1ba1f4e314f566da82f088 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 11 Nov 2022 12:46:32 +0000 Subject: [PATCH 08/24] Re-jigging things a bit, hiding as much of the couch/pouch stuff away. --- packages/backend-core/db.js | 1 - packages/backend-core/package.json | 3 +- .../backend-core/src/cache/writethrough.ts | 2 +- .../backend-core/src/context/deprovision.ts | 3 +- packages/backend-core/src/context/index.ts | 2 +- packages/backend-core/src/couch/couch.ts | 35 -------- packages/backend-core/src/couch/index.ts | 4 - packages/backend-core/src/couch/pouchDB.ts | 37 --------- packages/backend-core/src/db/Replication.ts | 2 +- .../backend-core/src/db/couch/connections.ts | 77 ++++++++++++++++++ packages/backend-core/src/db/couch/index.ts | 4 + .../src/db/{pouch.ts => couch/pouchDB.ts} | 80 ++++++++----------- .../src/{ => db}/couch/pouchLike.ts | 2 +- .../backend-core/src/{ => db}/couch/utils.ts | 4 +- packages/backend-core/src/db/index.ts | 5 +- packages/backend-core/src/db/utils.ts | 2 +- packages/backend-core/src/db/views.ts | 8 +- packages/backend-core/src/pkg/db.ts | 1 - packages/backend-core/src/types.ts | 2 +- packages/server/package.json | 3 +- packages/worker/package.json | 3 +- 21 files changed, 138 insertions(+), 142 deletions(-) delete mode 100644 packages/backend-core/src/couch/couch.ts delete mode 100644 packages/backend-core/src/couch/index.ts delete mode 100644 packages/backend-core/src/couch/pouchDB.ts create mode 100644 packages/backend-core/src/db/couch/connections.ts create mode 100644 packages/backend-core/src/db/couch/index.ts rename packages/backend-core/src/db/{pouch.ts => couch/pouchDB.ts} (52%) rename packages/backend-core/src/{ => db}/couch/pouchLike.ts (99%) rename packages/backend-core/src/{ => db}/couch/utils.ts (89%) diff --git a/packages/backend-core/db.js b/packages/backend-core/db.js index 0d2869d9f1..d2adf6c092 100644 --- a/packages/backend-core/db.js +++ b/packages/backend-core/db.js @@ -3,5 +3,4 @@ module.exports = { ...require("./src/db/constants"), ...require("./src/db"), ...require("./src/db/views"), - ...require("./src/db/pouch"), } diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json index 42a5f2bc53..2fbc087214 100644 --- a/packages/backend-core/package.json +++ b/packages/backend-core/package.json @@ -57,7 +57,8 @@ "preset": "ts-jest", "testEnvironment": "node", "moduleNameMapper": { - "@budibase/types": "/../types/src" + "@budibase/types": "/../types/src", + "axios": "axios/dist/node/axios.cjs" }, "setupFiles": [ "./scripts/jestSetup.ts" diff --git a/packages/backend-core/src/cache/writethrough.ts b/packages/backend-core/src/cache/writethrough.ts index 11dad81239..e13cc8cc0d 100644 --- a/packages/backend-core/src/cache/writethrough.ts +++ b/packages/backend-core/src/cache/writethrough.ts @@ -1,7 +1,7 @@ import BaseCache from "./base" import { getWritethroughClient } from "../redis/init" import { logWarn } from "../logging" -import { PouchLike } from "../couch" +import { PouchLike } from "../db" const DEFAULT_WRITE_RATE_MS = 10000 let CACHE: BaseCache | null = null diff --git a/packages/backend-core/src/context/deprovision.ts b/packages/backend-core/src/context/deprovision.ts index 179c7f407a..befcd48a15 100644 --- a/packages/backend-core/src/context/deprovision.ts +++ b/packages/backend-core/src/context/deprovision.ts @@ -1,8 +1,7 @@ import { getGlobalUserParams, getAllApps } from "../db/utils" -import { doWithDB } from "../db" +import { doWithDB, PouchLike } from "../db" import { doWithGlobalDB } from "../tenancy" import { StaticDatabases } from "../db/constants" -import { PouchLike } from "../couch" import { User } from "@budibase/types" const TENANT_DOC = StaticDatabases.PLATFORM_INFO.docs.tenants diff --git a/packages/backend-core/src/context/index.ts b/packages/backend-core/src/context/index.ts index 63b57a1b6e..34e6603ef7 100644 --- a/packages/backend-core/src/context/index.ts +++ b/packages/backend-core/src/context/index.ts @@ -5,7 +5,7 @@ import { baseGlobalDBName } from "../db/tenancy" import { IdentityContext } from "@budibase/types" import { DEFAULT_TENANT_ID as _DEFAULT_TENANT_ID } from "../constants" import { ContextMap, ContextKey } from "./constants" -import { PouchLike } from "../couch" +import { PouchLike } from "../db" import { getDevelopmentAppID, getProdAppID } from "../db/conversions" export const DEFAULT_TENANT_ID = _DEFAULT_TENANT_ID diff --git a/packages/backend-core/src/couch/couch.ts b/packages/backend-core/src/couch/couch.ts deleted file mode 100644 index 41fb83cb86..0000000000 --- a/packages/backend-core/src/couch/couch.ts +++ /dev/null @@ -1,35 +0,0 @@ -import env from "../environment" -import { getUrlInfo } from "../db/pouch" - -export const getCouchInfo = () => { - const urlInfo = getUrlInfo() - let username - let password - if (env.COUCH_DB_USERNAME) { - // set from env - username = env.COUCH_DB_USERNAME - } else if (urlInfo.auth.username) { - // set from url - username = urlInfo.auth.username - } else if (!env.isTest()) { - throw new Error("CouchDB username not set") - } - if (env.COUCH_DB_PASSWORD) { - // set from env - password = env.COUCH_DB_PASSWORD - } else if (urlInfo.auth.password) { - // set from url - password = urlInfo.auth.password - } else if (!env.isTest()) { - throw new Error("CouchDB password not set") - } - const authCookie = Buffer.from(`${username}:${password}`).toString("base64") - return { - url: urlInfo.url!, - auth: { - username: username, - password: password, - }, - cookie: `Basic ${authCookie}`, - } -} diff --git a/packages/backend-core/src/couch/index.ts b/packages/backend-core/src/couch/index.ts deleted file mode 100644 index 7a4ac20486..0000000000 --- a/packages/backend-core/src/couch/index.ts +++ /dev/null @@ -1,4 +0,0 @@ -export * from "./couch" -export * from "./pouchLike" -export * from "./utils" -export { init } from "./pouchDB" diff --git a/packages/backend-core/src/couch/pouchDB.ts b/packages/backend-core/src/couch/pouchDB.ts deleted file mode 100644 index 74ad8a3dc6..0000000000 --- a/packages/backend-core/src/couch/pouchDB.ts +++ /dev/null @@ -1,37 +0,0 @@ -import PouchDB from "pouchdb" -import env from "../environment" -import { PouchOptions } from "@budibase/types" -import * as pouch from "../db/pouch" - -let Pouch: any -let initialised = false - -export async function init(opts?: PouchOptions) { - Pouch = pouch.getPouch(opts) - initialised = true -} - -const checkInitialised = () => { - if (!initialised) { - throw new Error("init has not been called") - } -} - -export function getPouchDB(dbName: string, opts?: any): PouchDB.Database { - checkInitialised() - return new Pouch(dbName, opts) -} - -// use this function if you have called getPouchDB - close -// the databases you've opened once finished -export async function closePouchDB(db: PouchDB.Database) { - if (!db || env.isTest()) { - return - } - try { - // specifically await so that if there is an error, it can be ignored - return await db.close() - } catch (err) { - // ignore error, already closed - } -} diff --git a/packages/backend-core/src/db/Replication.ts b/packages/backend-core/src/db/Replication.ts index c6eea8db5e..12f6001a70 100644 --- a/packages/backend-core/src/db/Replication.ts +++ b/packages/backend-core/src/db/Replication.ts @@ -1,4 +1,4 @@ -import { getPouchDB, closePouchDB } from "../couch/pouchDB" +import { getPouchDB, closePouchDB } from "./couch/pouchDB" import { DocumentType } from "./constants" class Replication { diff --git a/packages/backend-core/src/db/couch/connections.ts b/packages/backend-core/src/db/couch/connections.ts new file mode 100644 index 0000000000..a2206de634 --- /dev/null +++ b/packages/backend-core/src/db/couch/connections.ts @@ -0,0 +1,77 @@ +import env from "../../environment" + +export const getCouchInfo = () => { + const urlInfo = getUrlInfo() + let username + let password + if (env.COUCH_DB_USERNAME) { + // set from env + username = env.COUCH_DB_USERNAME + } else if (urlInfo.auth.username) { + // set from url + username = urlInfo.auth.username + } else if (!env.isTest()) { + throw new Error("CouchDB username not set") + } + if (env.COUCH_DB_PASSWORD) { + // set from env + password = env.COUCH_DB_PASSWORD + } else if (urlInfo.auth.password) { + // set from url + password = urlInfo.auth.password + } else if (!env.isTest()) { + throw new Error("CouchDB password not set") + } + const authCookie = Buffer.from(`${username}:${password}`).toString("base64") + return { + url: urlInfo.url!, + auth: { + username: username, + password: password, + }, + cookie: `Basic ${authCookie}`, + } +} + +export const getUrlInfo = (url = env.COUCH_DB_URL) => { + let cleanUrl, username, password, host + if (url) { + // Ensure the URL starts with a protocol + const protoRegex = /^https?:\/\//i + if (!protoRegex.test(url)) { + url = `http://${url}` + } + + // Split into protocol and remainder + const split = url.split("://") + const protocol = split[0] + const rest = split.slice(1).join("://") + + // Extract auth if specified + if (url.includes("@")) { + // Split into host and remainder + let parts = rest.split("@") + host = parts[parts.length - 1] + let auth = parts.slice(0, -1).join("@") + + // Split auth into username and password + if (auth.includes(":")) { + const authParts = auth.split(":") + username = authParts[0] + password = authParts.slice(1).join(":") + } else { + username = auth + } + } else { + host = rest + } + cleanUrl = `${protocol}://${host}` + } + return { + url: cleanUrl, + auth: { + username, + password, + }, + } +} diff --git a/packages/backend-core/src/db/couch/index.ts b/packages/backend-core/src/db/couch/index.ts new file mode 100644 index 0000000000..3730a1e248 --- /dev/null +++ b/packages/backend-core/src/db/couch/index.ts @@ -0,0 +1,4 @@ +export * from "./connections" +export * from "./pouchLike" +export * from "./utils" +export { init, getPouch, getPouchDB } from "./pouchDB" diff --git a/packages/backend-core/src/db/pouch.ts b/packages/backend-core/src/db/couch/pouchDB.ts similarity index 52% rename from packages/backend-core/src/db/pouch.ts rename to packages/backend-core/src/db/couch/pouchDB.ts index fdb051060a..eead0e4d6f 100644 --- a/packages/backend-core/src/db/pouch.ts +++ b/packages/backend-core/src/db/couch/pouchDB.ts @@ -1,50 +1,10 @@ import PouchDB from "pouchdb" -import env from "../environment" -import { getCouchInfo } from "../couch" -export { getCouchInfo } from "../couch" +import env from "../../environment" +import { PouchOptions } from "@budibase/types" +import { getCouchInfo } from "./connections" -export const getUrlInfo = (url = env.COUCH_DB_URL) => { - let cleanUrl, username, password, host - if (url) { - // Ensure the URL starts with a protocol - const protoRegex = /^https?:\/\//i - if (!protoRegex.test(url)) { - url = `http://${url}` - } - - // Split into protocol and remainder - const split = url.split("://") - const protocol = split[0] - const rest = split.slice(1).join("://") - - // Extract auth if specified - if (url.includes("@")) { - // Split into host and remainder - let parts = rest.split("@") - host = parts[parts.length - 1] - let auth = parts.slice(0, -1).join("@") - - // Split auth into username and password - if (auth.includes(":")) { - const authParts = auth.split(":") - username = authParts[0] - password = authParts.slice(1).join(":") - } else { - username = auth - } - } else { - host = rest - } - cleanUrl = `${protocol}://${host}` - } - return { - url: cleanUrl, - auth: { - username, - password, - }, - } -} +let Pouch: any +let initialised = false /** * Return a constructor for PouchDB. @@ -92,3 +52,33 @@ export const getPouch = (opts: any = {}) => { return PouchDB.defaults(POUCH_DB_DEFAULTS) } + +export async function init(opts?: PouchOptions) { + Pouch = getPouch(opts) + initialised = true +} + +const checkInitialised = () => { + if (!initialised) { + throw new Error("init has not been called") + } +} + +export function getPouchDB(dbName: string, opts?: any): PouchDB.Database { + checkInitialised() + return new Pouch(dbName, opts) +} + +// use this function if you have called getPouchDB - close +// the databases you've opened once finished +export async function closePouchDB(db: PouchDB.Database) { + if (!db || env.isTest()) { + return + } + try { + // specifically await so that if there is an error, it can be ignored + return await db.close() + } catch (err) { + // ignore error, already closed + } +} diff --git a/packages/backend-core/src/couch/pouchLike.ts b/packages/backend-core/src/db/couch/pouchLike.ts similarity index 99% rename from packages/backend-core/src/couch/pouchLike.ts rename to packages/backend-core/src/db/couch/pouchLike.ts index fd5a94e1af..975bbd96c4 100644 --- a/packages/backend-core/src/couch/pouchLike.ts +++ b/packages/backend-core/src/db/couch/pouchLike.ts @@ -1,6 +1,6 @@ import Nano from "nano" import { AllDocsResponse, AnyDocument } from "@budibase/types" -import { getCouchInfo } from "./couch" +import { getCouchInfo } from "./connections" import { directCouchCall } from "./utils" import { getPouchDB } from "./pouchDB" diff --git a/packages/backend-core/src/couch/utils.ts b/packages/backend-core/src/db/couch/utils.ts similarity index 89% rename from packages/backend-core/src/couch/utils.ts rename to packages/backend-core/src/db/couch/utils.ts index 25a4643d54..426bf92158 100644 --- a/packages/backend-core/src/couch/utils.ts +++ b/packages/backend-core/src/db/couch/utils.ts @@ -1,6 +1,6 @@ -import { getCouchInfo } from "./couch" +import { getCouchInfo } from "./connections" import fetch from "node-fetch" -import { checkSlashesInUrl } from "../helpers" +import { checkSlashesInUrl } from "../../helpers" export async function directCouchCall( path: string, diff --git a/packages/backend-core/src/db/index.ts b/packages/backend-core/src/db/index.ts index e732b3a2fb..7cc49eae40 100644 --- a/packages/backend-core/src/db/index.ts +++ b/packages/backend-core/src/db/index.ts @@ -1,8 +1,7 @@ import env from "../environment" import { CouchFindOptions } from "@budibase/types" -import { PouchLike } from "../couch" -import { directCouchQuery } from "../couch" -export { init, PouchLike } from "../couch" +import { directCouchQuery, PouchLike } from "./couch" +export { init, PouchLike, getPouch, getPouchDB } from "./couch" let initialised = false const dbList = new Set() diff --git a/packages/backend-core/src/db/utils.ts b/packages/backend-core/src/db/utils.ts index a59b2ffe05..9a4ed98418 100644 --- a/packages/backend-core/src/db/utils.ts +++ b/packages/backend-core/src/db/utils.ts @@ -15,7 +15,7 @@ import { getAppMetadata } from "../cache/appMetadata" import { isDevApp, isDevAppID, getProdAppID } from "./conversions" import { APP_PREFIX } from "./constants" import * as events from "../events" -import { PouchLike } from "../couch" +import { PouchLike } from "./couch" export * from "./constants" export * from "./conversions" diff --git a/packages/backend-core/src/db/views.ts b/packages/backend-core/src/db/views.ts index 8ba508050f..23771d3f2d 100644 --- a/packages/backend-core/src/db/views.ts +++ b/packages/backend-core/src/db/views.ts @@ -1,6 +1,6 @@ import { DocumentType, ViewName, DeprecatedViews, SEPARATOR } from "./utils" import { getGlobalDB } from "../context" -import { PouchLike, QueryOpts } from "../couch" +import { PouchLike, QueryOpts } from "./couch" import { StaticDatabases } from "./constants" import { doWithDB } from "./" @@ -133,7 +133,9 @@ export const queryView = async ( try { let response = await db.query(`database/${viewName}`, params) const rows = response.rows - const docs = rows.map(row => (params.include_docs ? row.doc : row.value)) + const docs = rows.map((row: any) => + params.include_docs ? row.doc : row.value + ) // if arrayResponse has been requested, always return array regardless of length if (opts?.arrayResponse) { @@ -186,5 +188,5 @@ export const queryGlobalView = async ( db = getGlobalDB() } const createFn = CreateFuncByName[viewName] - return queryView(viewName, params, db, createFn, opts) + return queryView(viewName, params, db!, createFn, opts) } diff --git a/packages/backend-core/src/pkg/db.ts b/packages/backend-core/src/pkg/db.ts index 0254adddd5..d2bc474786 100644 --- a/packages/backend-core/src/pkg/db.ts +++ b/packages/backend-core/src/pkg/db.ts @@ -3,5 +3,4 @@ export * from "../db" export * from "../db/utils" export * from "../db/views" -export * from "../db/pouch" export * from "../db/constants" diff --git a/packages/backend-core/src/types.ts b/packages/backend-core/src/types.ts index 5eb3109b27..eeb784b0c1 100644 --- a/packages/backend-core/src/types.ts +++ b/packages/backend-core/src/types.ts @@ -1 +1 @@ -export { PouchLike } from "./couch" +export { PouchLike } from "./db" diff --git a/packages/server/package.json b/packages/server/package.json index 2c4394fc64..6d75877c79 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -42,7 +42,8 @@ "moduleNameMapper": { "@budibase/backend-core/(.*)": "/../backend-core/$1", "@budibase/backend-core": "/../backend-core/src", - "@budibase/types": "/../types/src" + "@budibase/types": "/../types/src", + "axios": "axios/dist/node/axios.cjs" }, "setupFiles": [ "./scripts/jestSetup.js" diff --git a/packages/worker/package.json b/packages/worker/package.json index c3b90811ac..844448b569 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -98,7 +98,8 @@ "moduleNameMapper": { "@budibase/backend-core/(.*)": "/../backend-core/$1", "@budibase/backend-core": "/../backend-core/src", - "@budibase/types": "/../types/src" + "@budibase/types": "/../types/src", + "axios": "axios/dist/node/axios.cjs" }, "setupFiles": [ "./scripts/jestSetup.js" From 3f6cdd787ce09fd884eef39e28672b516a33b2b6 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 14 Nov 2022 18:00:20 +0000 Subject: [PATCH 09/24] Replacing CLS with local-storage, simplified usage which should remove the memory leak permenantly. --- packages/backend-core/db.js | 7 +- packages/backend-core/src/clshooked/index.js | 650 ------------------ .../src/context/FunctionContext.ts | 47 -- .../backend-core/src/context/constants.ts | 4 - packages/backend-core/src/context/index.ts | 38 +- .../backend-core/src/context/localStorage.ts | 18 + packages/backend-core/src/db/couch/pouchDB.ts | 2 +- packages/backend-core/src/db/db.ts | 50 ++ packages/backend-core/src/db/index.ts | 58 +- packages/backend-core/src/db/utils.ts | 2 +- packages/backend-core/src/index.ts | 2 +- packages/backend-core/src/pkg/db.ts | 6 - 12 files changed, 98 insertions(+), 786 deletions(-) delete mode 100644 packages/backend-core/src/clshooked/index.js delete mode 100644 packages/backend-core/src/context/FunctionContext.ts create mode 100644 packages/backend-core/src/context/localStorage.ts create mode 100644 packages/backend-core/src/db/db.ts delete mode 100644 packages/backend-core/src/pkg/db.ts diff --git a/packages/backend-core/db.js b/packages/backend-core/db.js index d2adf6c092..f7004972d5 100644 --- a/packages/backend-core/db.js +++ b/packages/backend-core/db.js @@ -1,6 +1 @@ -module.exports = { - ...require("./src/db/utils"), - ...require("./src/db/constants"), - ...require("./src/db"), - ...require("./src/db/views"), -} +module.exports = require("./src/db") diff --git a/packages/backend-core/src/clshooked/index.js b/packages/backend-core/src/clshooked/index.js deleted file mode 100644 index d69ffdd914..0000000000 --- a/packages/backend-core/src/clshooked/index.js +++ /dev/null @@ -1,650 +0,0 @@ -const util = require("util") -const assert = require("assert") -const wrapEmitter = require("emitter-listener") -const async_hooks = require("async_hooks") - -const CONTEXTS_SYMBOL = "cls@contexts" -const ERROR_SYMBOL = "error@context" - -const DEBUG_CLS_HOOKED = process.env.DEBUG_CLS_HOOKED - -let currentUid = -1 - -module.exports = { - getNamespace: getNamespace, - createNamespace: createNamespace, - destroyNamespace: destroyNamespace, - reset: reset, - ERROR_SYMBOL: ERROR_SYMBOL, -} - -function Namespace(name) { - this.name = name - // changed in 2.7: no default context - this.active = null - this._set = [] - this.id = null - this._contexts = new Map() - this._indent = 0 - this._hook = null -} - -Namespace.prototype.set = function set(key, value) { - if (!this.active) { - throw new Error( - "No context available. ns.run() or ns.bind() must be called first." - ) - } - - this.active[key] = value - - if (DEBUG_CLS_HOOKED) { - const indentStr = " ".repeat(this._indent < 0 ? 0 : this._indent) - debug2( - indentStr + - "CONTEXT-SET KEY:" + - key + - "=" + - value + - " in ns:" + - this.name + - " currentUid:" + - currentUid + - " active:" + - util.inspect(this.active, { showHidden: true, depth: 2, colors: true }) - ) - } - - return value -} - -Namespace.prototype.get = function get(key) { - if (!this.active) { - if (DEBUG_CLS_HOOKED) { - const asyncHooksCurrentId = async_hooks.currentId() - const triggerId = async_hooks.triggerAsyncId() - const indentStr = " ".repeat(this._indent < 0 ? 0 : this._indent) - debug2( - `${indentStr}CONTEXT-GETTING KEY NO ACTIVE NS: (${this.name}) ${key}=undefined currentUid:${currentUid} asyncHooksCurrentId:${asyncHooksCurrentId} triggerId:${triggerId} len:${this._set.length}` - ) - } - return undefined - } - if (DEBUG_CLS_HOOKED) { - const asyncHooksCurrentId = async_hooks.executionAsyncId() - const triggerId = async_hooks.triggerAsyncId() - const indentStr = " ".repeat(this._indent < 0 ? 0 : this._indent) - debug2( - indentStr + - "CONTEXT-GETTING KEY:" + - key + - "=" + - this.active[key] + - " (" + - this.name + - ") currentUid:" + - currentUid + - " active:" + - util.inspect(this.active, { showHidden: true, depth: 2, colors: true }) - ) - debug2( - `${indentStr}CONTEXT-GETTING KEY: (${this.name}) ${key}=${ - this.active[key] - } currentUid:${currentUid} asyncHooksCurrentId:${asyncHooksCurrentId} triggerId:${triggerId} len:${ - this._set.length - } active:${util.inspect(this.active)}` - ) - } - return this.active[key] -} - -Namespace.prototype.createContext = function createContext() { - // Prototype inherit existing context if created a new child context within existing context. - let context = Object.create(this.active ? this.active : Object.prototype) - context._ns_name = this.name - context.id = currentUid - - if (DEBUG_CLS_HOOKED) { - const asyncHooksCurrentId = async_hooks.executionAsyncId() - const triggerId = async_hooks.triggerAsyncId() - const indentStr = " ".repeat(this._indent < 0 ? 0 : this._indent) - debug2( - `${indentStr}CONTEXT-CREATED Context: (${ - this.name - }) currentUid:${currentUid} asyncHooksCurrentId:${asyncHooksCurrentId} triggerId:${triggerId} len:${ - this._set.length - } context:${util.inspect(context, { - showHidden: true, - depth: 2, - colors: true, - })}` - ) - } - - return context -} - -Namespace.prototype.run = function run(fn) { - let context = this.createContext() - this.enter(context) - - try { - if (DEBUG_CLS_HOOKED) { - const triggerId = async_hooks.triggerAsyncId() - const asyncHooksCurrentId = async_hooks.executionAsyncId() - const indentStr = " ".repeat(this._indent < 0 ? 0 : this._indent) - debug2( - `${indentStr}CONTEXT-RUN BEGIN: (${ - this.name - }) currentUid:${currentUid} triggerId:${triggerId} asyncHooksCurrentId:${asyncHooksCurrentId} len:${ - this._set.length - } context:${util.inspect(context)}` - ) - } - fn(context) - return context - } catch (exception) { - if (exception) { - exception[ERROR_SYMBOL] = context - } - throw exception - } finally { - if (DEBUG_CLS_HOOKED) { - const triggerId = async_hooks.triggerAsyncId() - const asyncHooksCurrentId = async_hooks.executionAsyncId() - const indentStr = " ".repeat(this._indent < 0 ? 0 : this._indent) - debug2( - `${indentStr}CONTEXT-RUN END: (${ - this.name - }) currentUid:${currentUid} triggerId:${triggerId} asyncHooksCurrentId:${asyncHooksCurrentId} len:${ - this._set.length - } ${util.inspect(context)}` - ) - } - this.exit(context) - } -} - -Namespace.prototype.runAndReturn = function runAndReturn(fn) { - let value - this.run(function (context) { - value = fn(context) - }) - return value -} - -/** - * Uses global Promise and assumes Promise is cls friendly or wrapped already. - * @param {function} fn - * @returns {*} - */ -Namespace.prototype.runPromise = function runPromise(fn) { - let context = this.createContext() - this.enter(context) - - let promise = fn(context) - if (!promise || !promise.then || !promise.catch) { - throw new Error("fn must return a promise.") - } - - if (DEBUG_CLS_HOOKED) { - debug2( - "CONTEXT-runPromise BEFORE: (" + - this.name + - ") currentUid:" + - currentUid + - " len:" + - this._set.length + - " " + - util.inspect(context) - ) - } - - return promise - .then(result => { - if (DEBUG_CLS_HOOKED) { - debug2( - "CONTEXT-runPromise AFTER then: (" + - this.name + - ") currentUid:" + - currentUid + - " len:" + - this._set.length + - " " + - util.inspect(context) - ) - } - this.exit(context) - return result - }) - .catch(err => { - err[ERROR_SYMBOL] = context - if (DEBUG_CLS_HOOKED) { - debug2( - "CONTEXT-runPromise AFTER catch: (" + - this.name + - ") currentUid:" + - currentUid + - " len:" + - this._set.length + - " " + - util.inspect(context) - ) - } - this.exit(context) - throw err - }) -} - -Namespace.prototype.bind = function bindFactory(fn, context) { - if (!context) { - if (!this.active) { - context = this.createContext() - } else { - context = this.active - } - } - - let self = this - return function clsBind() { - self.enter(context) - try { - return fn.apply(this, arguments) - } catch (exception) { - if (exception) { - exception[ERROR_SYMBOL] = context - } - throw exception - } finally { - self.exit(context) - } - } -} - -Namespace.prototype.enter = function enter(context) { - assert.ok(context, "context must be provided for entering") - if (DEBUG_CLS_HOOKED) { - const asyncHooksCurrentId = async_hooks.executionAsyncId() - const triggerId = async_hooks.triggerAsyncId() - const indentStr = " ".repeat(this._indent < 0 ? 0 : this._indent) - debug2( - `${indentStr}CONTEXT-ENTER: (${ - this.name - }) currentUid:${currentUid} triggerId:${triggerId} asyncHooksCurrentId:${asyncHooksCurrentId} len:${ - this._set.length - } ${util.inspect(context)}` - ) - } - - this._set.push(this.active) - this.active = context -} - -Namespace.prototype.exit = function exit(context) { - assert.ok(context, "context must be provided for exiting") - if (DEBUG_CLS_HOOKED) { - const asyncHooksCurrentId = async_hooks.executionAsyncId() - const triggerId = async_hooks.triggerAsyncId() - const indentStr = " ".repeat(this._indent < 0 ? 0 : this._indent) - debug2( - `${indentStr}CONTEXT-EXIT: (${ - this.name - }) currentUid:${currentUid} triggerId:${triggerId} asyncHooksCurrentId:${asyncHooksCurrentId} len:${ - this._set.length - } ${util.inspect(context)}` - ) - } - - // Fast path for most exits that are at the top of the stack - if (this.active === context) { - assert.ok(this._set.length, "can't remove top context") - this.active = this._set.pop() - return - } - - // Fast search in the stack using lastIndexOf - let index = this._set.lastIndexOf(context) - - if (index < 0) { - if (DEBUG_CLS_HOOKED) { - debug2( - "??ERROR?? context exiting but not entered - ignoring: " + - util.inspect(context) - ) - } - assert.ok( - index >= 0, - "context not currently entered; can't exit. \n" + - util.inspect(this) + - "\n" + - util.inspect(context) - ) - } else { - assert.ok(index, "can't remove top context") - this._set.splice(index, 1) - } -} - -Namespace.prototype.bindEmitter = function bindEmitter(emitter) { - assert.ok( - emitter.on && emitter.addListener && emitter.emit, - "can only bind real EEs" - ) - - let namespace = this - let thisSymbol = "context@" + this.name - - // Capture the context active at the time the emitter is bound. - function attach(listener) { - if (!listener) { - return - } - if (!listener[CONTEXTS_SYMBOL]) { - listener[CONTEXTS_SYMBOL] = Object.create(null) - } - - listener[CONTEXTS_SYMBOL][thisSymbol] = { - namespace: namespace, - context: namespace.active, - } - } - - // At emit time, bind the listener within the correct context. - function bind(unwrapped) { - if (!(unwrapped && unwrapped[CONTEXTS_SYMBOL])) { - return unwrapped - } - - let wrapped = unwrapped - let unwrappedContexts = unwrapped[CONTEXTS_SYMBOL] - Object.keys(unwrappedContexts).forEach(function (name) { - let thunk = unwrappedContexts[name] - wrapped = thunk.namespace.bind(wrapped, thunk.context) - }) - return wrapped - } - - wrapEmitter(emitter, attach, bind) -} - -/** - * If an error comes out of a namespace, it will have a context attached to it. - * This function knows how to find it. - * - * @param {Error} exception Possibly annotated error. - */ -Namespace.prototype.fromException = function fromException(exception) { - return exception[ERROR_SYMBOL] -} - -function getNamespace(name) { - return process.namespaces[name] -} - -function createNamespace(name) { - assert.ok(name, "namespace must be given a name.") - - if (DEBUG_CLS_HOOKED) { - debug2(`NS-CREATING NAMESPACE (${name})`) - } - let namespace = new Namespace(name) - namespace.id = currentUid - - const hook = async_hooks.createHook({ - init(asyncId, type, triggerId, resource) { - currentUid = async_hooks.executionAsyncId() - - //CHAIN Parent's Context onto child if none exists. This is needed to pass net-events.spec - // let initContext = namespace.active; - // if(!initContext && triggerId) { - // let parentContext = namespace._contexts.get(triggerId); - // if (parentContext) { - // namespace.active = parentContext; - // namespace._contexts.set(currentUid, parentContext); - // if (DEBUG_CLS_HOOKED) { - // const indentStr = ' '.repeat(namespace._indent < 0 ? 0 : namespace._indent); - // debug2(`${indentStr}INIT [${type}] (${name}) WITH PARENT CONTEXT asyncId:${asyncId} currentUid:${currentUid} triggerId:${triggerId} active:${util.inspect(namespace.active, true)} resource:${resource}`); - // } - // } else if (DEBUG_CLS_HOOKED) { - // const indentStr = ' '.repeat(namespace._indent < 0 ? 0 : namespace._indent); - // debug2(`${indentStr}INIT [${type}] (${name}) MISSING CONTEXT asyncId:${asyncId} currentUid:${currentUid} triggerId:${triggerId} active:${util.inspect(namespace.active, true)} resource:${resource}`); - // } - // }else { - // namespace._contexts.set(currentUid, namespace.active); - // if (DEBUG_CLS_HOOKED) { - // const indentStr = ' '.repeat(namespace._indent < 0 ? 0 : namespace._indent); - // debug2(`${indentStr}INIT [${type}] (${name}) asyncId:${asyncId} currentUid:${currentUid} triggerId:${triggerId} active:${util.inspect(namespace.active, true)} resource:${resource}`); - // } - // } - if (namespace.active) { - namespace._contexts.set(asyncId, namespace.active) - - if (DEBUG_CLS_HOOKED) { - const indentStr = " ".repeat( - namespace._indent < 0 ? 0 : namespace._indent - ) - debug2( - `${indentStr}INIT [${type}] (${name}) asyncId:${asyncId} currentUid:${currentUid} triggerId:${triggerId} active:${util.inspect( - namespace.active, - { showHidden: true, depth: 2, colors: true } - )} resource:${resource}` - ) - } - } else if (currentUid === 0) { - // CurrentId will be 0 when triggered from C++. Promise events - // https://github.com/nodejs/node/blob/master/doc/api/async_hooks.md#triggerid - const triggerId = async_hooks.triggerAsyncId() - const triggerIdContext = namespace._contexts.get(triggerId) - if (triggerIdContext) { - namespace._contexts.set(asyncId, triggerIdContext) - if (DEBUG_CLS_HOOKED) { - const indentStr = " ".repeat( - namespace._indent < 0 ? 0 : namespace._indent - ) - debug2( - `${indentStr}INIT USING CONTEXT FROM TRIGGERID [${type}] (${name}) asyncId:${asyncId} currentUid:${currentUid} triggerId:${triggerId} active:${util.inspect( - namespace.active, - { showHidden: true, depth: 2, colors: true } - )} resource:${resource}` - ) - } - } else if (DEBUG_CLS_HOOKED) { - const indentStr = " ".repeat( - namespace._indent < 0 ? 0 : namespace._indent - ) - debug2( - `${indentStr}INIT MISSING CONTEXT [${type}] (${name}) asyncId:${asyncId} currentUid:${currentUid} triggerId:${triggerId} active:${util.inspect( - namespace.active, - { showHidden: true, depth: 2, colors: true } - )} resource:${resource}` - ) - } - } - - if (DEBUG_CLS_HOOKED && type === "PROMISE") { - debug2(util.inspect(resource, { showHidden: true })) - const parentId = resource.parentId - const indentStr = " ".repeat( - namespace._indent < 0 ? 0 : namespace._indent - ) - debug2( - `${indentStr}INIT RESOURCE-PROMISE [${type}] (${name}) parentId:${parentId} asyncId:${asyncId} currentUid:${currentUid} triggerId:${triggerId} active:${util.inspect( - namespace.active, - { showHidden: true, depth: 2, colors: true } - )} resource:${resource}` - ) - } - }, - before(asyncId) { - currentUid = async_hooks.executionAsyncId() - let context - - /* - if(currentUid === 0){ - // CurrentId will be 0 when triggered from C++. Promise events - // https://github.com/nodejs/node/blob/master/doc/api/async_hooks.md#triggerid - //const triggerId = async_hooks.triggerAsyncId(); - context = namespace._contexts.get(asyncId); // || namespace._contexts.get(triggerId); - }else{ - context = namespace._contexts.get(currentUid); - } - */ - - //HACK to work with promises until they are fixed in node > 8.1.1 - context = - namespace._contexts.get(asyncId) || namespace._contexts.get(currentUid) - - if (context) { - if (DEBUG_CLS_HOOKED) { - const triggerId = async_hooks.triggerAsyncId() - const indentStr = " ".repeat( - namespace._indent < 0 ? 0 : namespace._indent - ) - debug2( - `${indentStr}BEFORE (${name}) asyncId:${asyncId} currentUid:${currentUid} triggerId:${triggerId} active:${util.inspect( - namespace.active, - { showHidden: true, depth: 2, colors: true } - )} context:${util.inspect(context)}` - ) - namespace._indent += 2 - } - - namespace.enter(context) - } else if (DEBUG_CLS_HOOKED) { - const triggerId = async_hooks.triggerAsyncId() - const indentStr = " ".repeat( - namespace._indent < 0 ? 0 : namespace._indent - ) - debug2( - `${indentStr}BEFORE MISSING CONTEXT (${name}) asyncId:${asyncId} currentUid:${currentUid} triggerId:${triggerId} active:${util.inspect( - namespace.active, - { showHidden: true, depth: 2, colors: true } - )} namespace._contexts:${util.inspect(namespace._contexts, { - showHidden: true, - depth: 2, - colors: true, - })}` - ) - namespace._indent += 2 - } - }, - after(asyncId) { - currentUid = async_hooks.executionAsyncId() - let context // = namespace._contexts.get(currentUid); - /* - if(currentUid === 0){ - // CurrentId will be 0 when triggered from C++. Promise events - // https://github.com/nodejs/node/blob/master/doc/api/async_hooks.md#triggerid - //const triggerId = async_hooks.triggerAsyncId(); - context = namespace._contexts.get(asyncId); // || namespace._contexts.get(triggerId); - }else{ - context = namespace._contexts.get(currentUid); - } - */ - //HACK to work with promises until they are fixed in node > 8.1.1 - context = - namespace._contexts.get(asyncId) || namespace._contexts.get(currentUid) - - if (context) { - if (DEBUG_CLS_HOOKED) { - const triggerId = async_hooks.triggerAsyncId() - namespace._indent -= 2 - const indentStr = " ".repeat( - namespace._indent < 0 ? 0 : namespace._indent - ) - debug2( - `${indentStr}AFTER (${name}) asyncId:${asyncId} currentUid:${currentUid} triggerId:${triggerId} active:${util.inspect( - namespace.active, - { showHidden: true, depth: 2, colors: true } - )} context:${util.inspect(context)}` - ) - } - - namespace.exit(context) - } else if (DEBUG_CLS_HOOKED) { - const triggerId = async_hooks.triggerAsyncId() - namespace._indent -= 2 - const indentStr = " ".repeat( - namespace._indent < 0 ? 0 : namespace._indent - ) - debug2( - `${indentStr}AFTER MISSING CONTEXT (${name}) asyncId:${asyncId} currentUid:${currentUid} triggerId:${triggerId} active:${util.inspect( - namespace.active, - { showHidden: true, depth: 2, colors: true } - )} context:${util.inspect(context)}` - ) - } - }, - destroy(asyncId) { - currentUid = async_hooks.executionAsyncId() - if (DEBUG_CLS_HOOKED) { - const triggerId = async_hooks.triggerAsyncId() - const indentStr = " ".repeat( - namespace._indent < 0 ? 0 : namespace._indent - ) - debug2( - `${indentStr}DESTROY (${name}) currentUid:${currentUid} asyncId:${asyncId} triggerId:${triggerId} active:${util.inspect( - namespace.active, - { showHidden: true, depth: 2, colors: true } - )} context:${util.inspect(namespace._contexts.get(currentUid))}` - ) - } - - namespace._contexts.delete(asyncId) - }, - }) - - hook.enable() - namespace._hook = hook - - process.namespaces[name] = namespace - return namespace -} - -function destroyNamespace(name) { - let namespace = getNamespace(name) - - assert.ok(namespace, "can't delete nonexistent namespace! \"" + name + '"') - assert.ok( - namespace.id, - "don't assign to process.namespaces directly! " + util.inspect(namespace) - ) - - namespace._hook.disable() - namespace._contexts = null - process.namespaces[name] = null -} - -function reset() { - // must unregister async listeners - if (process.namespaces) { - Object.keys(process.namespaces).forEach(function (name) { - destroyNamespace(name) - }) - } - process.namespaces = Object.create(null) -} - -process.namespaces = process.namespaces || {} - -//const fs = require('fs'); -function debug2(...args) { - if (DEBUG_CLS_HOOKED) { - //fs.writeSync(1, `${util.format(...args)}\n`); - process._rawDebug(`${util.format(...args)}`) - } -} - -/*function getFunctionName(fn) { - if (!fn) { - return fn; - } - if (typeof fn === 'function') { - if (fn.name) { - return fn.name; - } - return (fn.toString().trim().match(/^function\s*([^\s(]+)/) || [])[1]; - } else if (fn.constructor && fn.constructor.name) { - return fn.constructor.name; - } -}*/ diff --git a/packages/backend-core/src/context/FunctionContext.ts b/packages/backend-core/src/context/FunctionContext.ts deleted file mode 100644 index 1010f585ef..0000000000 --- a/packages/backend-core/src/context/FunctionContext.ts +++ /dev/null @@ -1,47 +0,0 @@ -import cls from "../clshooked" -import { newid } from "../hashing" - -const REQUEST_ID_KEY = "requestId" -const MAIN_CTX = cls.createNamespace("main") - -function getContextStorage(namespace: any) { - if (namespace && namespace.active) { - let contextData = namespace.active - delete contextData.id - delete contextData._ns_name - return contextData - } - return {} -} - -class FunctionContext { - static run(callback: any) { - return MAIN_CTX.runAndReturn(async () => { - const namespaceId = newid() - MAIN_CTX.set(REQUEST_ID_KEY, namespaceId) - const namespace = cls.createNamespace(namespaceId) - let response = await namespace.runAndReturn(callback) - cls.destroyNamespace(namespaceId) - return response - }) - } - - static setOnContext(key: string, value: any) { - const namespaceId = MAIN_CTX.get(REQUEST_ID_KEY) - const namespace = cls.getNamespace(namespaceId) - namespace.set(key, value) - } - - static getFromContext(key: string) { - const namespaceId = MAIN_CTX.get(REQUEST_ID_KEY) - const namespace = cls.getNamespace(namespaceId) - const context = getContextStorage(namespace) - if (context) { - return context[key] - } else { - return null - } - } -} - -export = FunctionContext diff --git a/packages/backend-core/src/context/constants.ts b/packages/backend-core/src/context/constants.ts index eb156c357b..64fdb45dec 100644 --- a/packages/backend-core/src/context/constants.ts +++ b/packages/backend-core/src/context/constants.ts @@ -1,9 +1,5 @@ import { IdentityContext } from "@budibase/types" -export enum ContextKey { - MAIN = "main", -} - export type ContextMap = { tenantId?: string appId?: string diff --git a/packages/backend-core/src/context/index.ts b/packages/backend-core/src/context/index.ts index 34e6603ef7..910bd192b4 100644 --- a/packages/backend-core/src/context/index.ts +++ b/packages/backend-core/src/context/index.ts @@ -1,13 +1,16 @@ import env from "../environment" -import { SEPARATOR, DocumentType } from "../db/constants" -import cls from "./FunctionContext" -import { baseGlobalDBName } from "../db/tenancy" +import { + SEPARATOR, + DocumentType, + getDevelopmentAppID, + getProdAppID, + baseGlobalDBName, + PouchLike, +} from "../db" +import { Context } from "./localStorage" import { IdentityContext } from "@budibase/types" import { DEFAULT_TENANT_ID as _DEFAULT_TENANT_ID } from "../constants" -import { ContextMap, ContextKey } from "./constants" -import { PouchLike } from "../db" -import { getDevelopmentAppID, getProdAppID } from "../db/conversions" - +import { ContextMap } from "./constants" export const DEFAULT_TENANT_ID = _DEFAULT_TENANT_ID // some test cases call functions directly, need to @@ -19,7 +22,7 @@ export function isMultiTenant() { } export function isTenantIdSet() { - const context = cls.getFromContext(ContextKey.MAIN) as ContextMap + const context = Context.get() return !!context?.tenantId } @@ -53,7 +56,7 @@ export function getTenantIDFromAppID(appId: string) { function updateContext(updates: ContextMap) { let context: ContextMap try { - context = cls.getFromContext(ContextKey.MAIN) + context = Context.get() } catch (err) { // no context, start empty context = {} @@ -68,10 +71,7 @@ function updateContext(updates: ContextMap) { async function newContext(updates: ContextMap, task: any) { // see if there already is a context setup let context: ContextMap = updateContext(updates) - return cls.run(async () => { - cls.setOnContext(ContextKey.MAIN, context) - return await task() - }) + return Context.run(context, task) } export async function doInContext(appId: string, task: any): Promise { @@ -132,7 +132,7 @@ export async function doInIdentityContext( export function getIdentity(): IdentityContext | undefined { try { - const context = cls.getFromContext(ContextKey.MAIN) as ContextMap + const context = Context.get() return context?.identity } catch (e) { // do nothing - identity is not in context @@ -143,7 +143,7 @@ export function getTenantId(): string { if (!isMultiTenant()) { return DEFAULT_TENANT_ID } - const context = cls.getFromContext(ContextKey.MAIN) as ContextMap + const context = Context.get() const tenantId = context?.tenantId if (!tenantId) { throw new Error("Tenant id not found") @@ -152,7 +152,7 @@ export function getTenantId(): string { } export function getAppId(): string | undefined { - const context = cls.getFromContext(ContextKey.MAIN) as ContextMap + const context = Context.get() const foundId = context?.appId if (!foundId && env.isTest() && TEST_APP_ID) { return TEST_APP_ID @@ -165,7 +165,7 @@ export function updateTenantId(tenantId?: string) { let context: ContextMap = updateContext({ tenantId, }) - cls.setOnContext(ContextKey.MAIN, context) + Context.set(context) } export function updateAppId(appId: string) { @@ -173,7 +173,7 @@ export function updateAppId(appId: string) { appId, }) try { - cls.setOnContext(ContextKey.MAIN, context) + Context.set(context) } catch (err) { if (env.isTest()) { TEST_APP_ID = appId @@ -184,7 +184,7 @@ export function updateAppId(appId: string) { } export function getGlobalDB(): PouchLike { - const context = cls.getFromContext(ContextKey.MAIN) as ContextMap + const context = Context.get() return new PouchLike(baseGlobalDBName(context?.tenantId)) } diff --git a/packages/backend-core/src/context/localStorage.ts b/packages/backend-core/src/context/localStorage.ts new file mode 100644 index 0000000000..fd761d10bc --- /dev/null +++ b/packages/backend-core/src/context/localStorage.ts @@ -0,0 +1,18 @@ +import { AsyncLocalStorage } from "async_hooks" +import { ContextMap } from "./constants" + +export class Context { + static storage = new AsyncLocalStorage() + + static run(context: ContextMap, func: any) { + return Context.storage.run(context, () => func()) + } + + static get(): ContextMap { + return Context.storage.getStore() as ContextMap + } + + static set(context: ContextMap) { + Context.storage.enterWith(context) + } +} diff --git a/packages/backend-core/src/db/couch/pouchDB.ts b/packages/backend-core/src/db/couch/pouchDB.ts index eead0e4d6f..07f7b6c90d 100644 --- a/packages/backend-core/src/db/couch/pouchDB.ts +++ b/packages/backend-core/src/db/couch/pouchDB.ts @@ -53,7 +53,7 @@ export const getPouch = (opts: any = {}) => { return PouchDB.defaults(POUCH_DB_DEFAULTS) } -export async function init(opts?: PouchOptions) { +export function init(opts?: PouchOptions) { Pouch = getPouch(opts) initialised = true } diff --git a/packages/backend-core/src/db/db.ts b/packages/backend-core/src/db/db.ts new file mode 100644 index 0000000000..497b747b80 --- /dev/null +++ b/packages/backend-core/src/db/db.ts @@ -0,0 +1,50 @@ +import env from "../environment" +import { directCouchQuery, PouchLike } from "./couch" +import { CouchFindOptions } from "@budibase/types" + +let initialised = false +const dbList = new Set() + +const checkInitialised = () => { + if (!initialised) { + throw new Error("init has not been called") + } +} + +export function getDB(dbName: string, opts?: any): PouchLike { + if (env.isTest()) { + dbList.add(dbName) + } + return new PouchLike(dbName, opts) +} + +// we have to use a callback for this so that we can close +// the DB when we're done, without this manual requests would +// need to close the database when done with it to avoid memory leaks +export async function doWithDB(dbName: string, cb: any, opts = {}) { + const db = getDB(dbName, opts) + // need this to be async so that we can correctly close DB after all + // async operations have been completed + return await cb(db) +} + +export function allDbs() { + if (!env.isTest()) { + throw new Error("Cannot be used outside test environment.") + } + checkInitialised() + return [...dbList] +} + +export async function directCouchAllDbs(queryString?: string) { + let couchPath = "/_all_dbs" + if (queryString) { + couchPath += `?${queryString}` + } + return await directCouchQuery(couchPath) +} + +export async function directCouchFind(dbName: string, opts: CouchFindOptions) { + const json = await directCouchQuery(`${dbName}/_find`, "POST", opts) + return { rows: json.docs, bookmark: json.bookmark } +} diff --git a/packages/backend-core/src/db/index.ts b/packages/backend-core/src/db/index.ts index 7cc49eae40..7269aa8f92 100644 --- a/packages/backend-core/src/db/index.ts +++ b/packages/backend-core/src/db/index.ts @@ -1,51 +1,7 @@ -import env from "../environment" -import { CouchFindOptions } from "@budibase/types" -import { directCouchQuery, PouchLike } from "./couch" -export { init, PouchLike, getPouch, getPouchDB } from "./couch" - -let initialised = false -const dbList = new Set() - -const checkInitialised = () => { - if (!initialised) { - throw new Error("init has not been called") - } -} - -export function getDB(dbName: string, opts?: any): PouchLike { - if (env.isTest()) { - dbList.add(dbName) - } - return new PouchLike(dbName, opts) -} - -// we have to use a callback for this so that we can close -// the DB when we're done, without this manual requests would -// need to close the database when done with it to avoid memory leaks -export async function doWithDB(dbName: string, cb: any, opts = {}) { - const db = getDB(dbName, opts) - // need this to be async so that we can correctly close DB after all - // async operations have been completed - return await cb(db) -} - -export function allDbs() { - if (!env.isTest()) { - throw new Error("Cannot be used outside test environment.") - } - checkInitialised() - return [...dbList] -} - -export async function directCouchAllDbs(queryString?: string) { - let couchPath = "/_all_dbs" - if (queryString) { - couchPath += `?${queryString}` - } - return await directCouchQuery(couchPath) -} - -export async function directCouchFind(dbName: string, opts: CouchFindOptions) { - const json = await directCouchQuery(`${dbName}/_find`, "POST", opts) - return { rows: json.docs, bookmark: json.bookmark } -} +export * from "./couch" +export * from "./db" +export * from "./utils" +export * from "./views" +export * from "./constants" +export * from "./conversions" +export * from "./tenancy" diff --git a/packages/backend-core/src/db/utils.ts b/packages/backend-core/src/db/utils.ts index 9a4ed98418..fc67a2c49b 100644 --- a/packages/backend-core/src/db/utils.ts +++ b/packages/backend-core/src/db/utils.ts @@ -10,7 +10,7 @@ import { } from "./constants" import { getTenantId, getGlobalDB } from "../context" import { getGlobalDBName } from "./tenancy" -import { doWithDB, allDbs, directCouchAllDbs } from "./index" +import { doWithDB, allDbs, directCouchAllDbs } from "./db" import { getAppMetadata } from "../cache/appMetadata" import { isDevApp, isDevAppID, getProdAppID } from "./conversions" import { APP_PREFIX } from "./constants" diff --git a/packages/backend-core/src/index.ts b/packages/backend-core/src/index.ts index b45248abc2..a2cfaaa9c6 100644 --- a/packages/backend-core/src/index.ts +++ b/packages/backend-core/src/index.ts @@ -24,7 +24,7 @@ import * as queue from "./queue" import * as types from "./types" // mimic the outer package exports -import * as db from "./pkg/db" +import * as db from "./db" import * as objectStore from "./pkg/objectStore" import * as utils from "./pkg/utils" import redis from "./pkg/redis" diff --git a/packages/backend-core/src/pkg/db.ts b/packages/backend-core/src/pkg/db.ts deleted file mode 100644 index d2bc474786..0000000000 --- a/packages/backend-core/src/pkg/db.ts +++ /dev/null @@ -1,6 +0,0 @@ -// Mimic the outer package export for usage in index.ts -// The outer exports can't be used as they now reference dist directly -export * from "../db" -export * from "../db/utils" -export * from "../db/views" -export * from "../db/constants" From 3f0e53f90c3f5e37238d02df2eceb2bcf9886080 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 14 Nov 2022 18:29:15 +0000 Subject: [PATCH 10/24] Updating context test, some minor adjustments based on getting the test working again. --- packages/backend-core/src/context/index.ts | 12 +++++--- .../tests/{index.spec.ts => index.spec.js} | 30 +++++-------------- packages/backend-core/src/db/couch/index.ts | 2 +- packages/backend-core/src/db/db.ts | 2 +- 4 files changed, 17 insertions(+), 29 deletions(-) rename packages/backend-core/src/context/tests/{index.spec.ts => index.spec.js} (80%) diff --git a/packages/backend-core/src/context/index.ts b/packages/backend-core/src/context/index.ts index 910bd192b4..15eed8d0ba 100644 --- a/packages/backend-core/src/context/index.ts +++ b/packages/backend-core/src/context/index.ts @@ -5,6 +5,7 @@ import { getDevelopmentAppID, getProdAppID, baseGlobalDBName, + getDB, PouchLike, } from "../db" import { Context } from "./localStorage" @@ -185,7 +186,10 @@ export function updateAppId(appId: string) { export function getGlobalDB(): PouchLike { const context = Context.get() - return new PouchLike(baseGlobalDBName(context?.tenantId)) + if (!context || (env.MULTI_TENANCY && !context.tenantId)) { + throw new Error("Global DB not found") + } + return getDB(baseGlobalDBName(context?.tenantId)) } /** @@ -194,7 +198,7 @@ export function getGlobalDB(): PouchLike { */ export function getAppDB(opts?: any): PouchLike { const appId = getAppId() - return new PouchLike(appId, opts) + return getDB(appId, opts) } /** @@ -206,7 +210,7 @@ export function getProdAppDB(opts?: any): PouchLike { if (!appId) { throw new Error("Unable to retrieve prod DB - no app ID.") } - return new PouchLike(getProdAppID(appId), opts) + return getDB(getProdAppID(appId), opts) } /** @@ -218,5 +222,5 @@ export function getDevAppDB(opts?: any): PouchLike { if (!appId) { throw new Error("Unable to retrieve dev DB - no app ID.") } - return new PouchLike(getDevelopmentAppID(appId), opts) + return getDB(getDevelopmentAppID(appId), opts) } diff --git a/packages/backend-core/src/context/tests/index.spec.ts b/packages/backend-core/src/context/tests/index.spec.js similarity index 80% rename from packages/backend-core/src/context/tests/index.spec.ts rename to packages/backend-core/src/context/tests/index.spec.js index 749231c3e5..6d09446059 100644 --- a/packages/backend-core/src/context/tests/index.spec.ts +++ b/packages/backend-core/src/context/tests/index.spec.js @@ -1,18 +1,10 @@ -import "../../../tests/utilities/TestConfiguration" -import * as context from ".." -import { DEFAULT_TENANT_ID } from "../../constants" -import env from "../../environment" - -// must use require to spy index file exports due to known issue in jest -const dbUtils = require("../../db") -jest.spyOn(dbUtils, "closePouchDB") -jest.spyOn(dbUtils, "getDB") +require("../../../tests/utilities/TestConfiguration") +const context = require("../") +const { DEFAULT_TENANT_ID } = require("../../constants") +const env = require("../../environment") +const dbCore = require("../../db") describe("context", () => { - beforeEach(() => { - jest.clearAllMocks() - }) - describe("doInTenant", () => { describe("single-tenancy", () => { it("defaults to the default tenant", () => { @@ -25,8 +17,6 @@ describe("context", () => { const db = context.getGlobalDB() expect(db.name).toBe("global-db") }) - expect(dbUtils.getDB).toHaveBeenCalledTimes(1) - expect(dbUtils.closePouchDB).toHaveBeenCalledTimes(1) }) }) @@ -40,7 +30,7 @@ describe("context", () => { let error try { context.getTenantId() - } catch (e: any) { + } catch (e) { error = e } expect(error.message).toBe("Tenant id not found") @@ -59,7 +49,7 @@ describe("context", () => { let error try { context.getGlobalDB() - } catch (e: any) { + } catch (e) { error = e } expect(error.message).toBe("Global DB not found") @@ -85,8 +75,6 @@ describe("context", () => { const db = context.getGlobalDB() expect(db.name).toBe("test_global-db") }) - expect(dbUtils.getDB).toHaveBeenCalledTimes(1) - expect(dbUtils.closePouchDB).toHaveBeenCalledTimes(1) }) it("sets the tenant id when nested with same tenant id", async () => { @@ -121,10 +109,6 @@ describe("context", () => { }) }) }) - - // only 1 db is opened and closed - expect(dbUtils.getDB).toHaveBeenCalledTimes(1) - expect(dbUtils.closePouchDB).toHaveBeenCalledTimes(1) }) it("sets different tenant id inside another context", () => { diff --git a/packages/backend-core/src/db/couch/index.ts b/packages/backend-core/src/db/couch/index.ts index 3730a1e248..b8cb0ab79f 100644 --- a/packages/backend-core/src/db/couch/index.ts +++ b/packages/backend-core/src/db/couch/index.ts @@ -1,4 +1,4 @@ export * from "./connections" export * from "./pouchLike" export * from "./utils" -export { init, getPouch, getPouchDB } from "./pouchDB" +export { init, getPouch, getPouchDB, closePouchDB } from "./pouchDB" diff --git a/packages/backend-core/src/db/db.ts b/packages/backend-core/src/db/db.ts index 497b747b80..909d286d32 100644 --- a/packages/backend-core/src/db/db.ts +++ b/packages/backend-core/src/db/db.ts @@ -11,7 +11,7 @@ const checkInitialised = () => { } } -export function getDB(dbName: string, opts?: any): PouchLike { +export function getDB(dbName?: string, opts?: any): PouchLike { if (env.isTest()) { dbList.add(dbName) } From 72c9aadae083f0c9d7bc005da0ade75ff9ebe824 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 14 Nov 2022 20:25:05 +0000 Subject: [PATCH 11/24] Updating test cases - trying to get everything in order for merge. --- packages/backend-core/package.json | 2 +- packages/backend-core/src/db/couch/pouchDB.ts | 15 +- packages/backend-core/src/db/db.ts | 13 +- .../backend-core/src/db/tests/pouch.spec.js | 2 +- .../backend-core/src/db/tests/utils.spec.js | 6 +- packages/server/__mocks__/node-fetch.ts | 6 +- packages/server/package.json | 2 +- .../server/src/db/tests/linkTests.spec.js | 31 ++-- .../src/middleware/tests/currentapp.spec.js | 150 ++++++++++-------- packages/worker/package.json | 2 +- 10 files changed, 134 insertions(+), 95 deletions(-) diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json index 2fbc087214..63e0d9ff9b 100644 --- a/packages/backend-core/package.json +++ b/packages/backend-core/package.json @@ -58,7 +58,7 @@ "testEnvironment": "node", "moduleNameMapper": { "@budibase/types": "/../types/src", - "axios": "axios/dist/node/axios.cjs" + "^axios.*$": "/node_modules/axios/lib/axios.js" }, "setupFiles": [ "./scripts/jestSetup.ts" diff --git a/packages/backend-core/src/db/couch/pouchDB.ts b/packages/backend-core/src/db/couch/pouchDB.ts index 07f7b6c90d..e7efbf6dcd 100644 --- a/packages/backend-core/src/db/couch/pouchDB.ts +++ b/packages/backend-core/src/db/couch/pouchDB.ts @@ -66,7 +66,20 @@ const checkInitialised = () => { export function getPouchDB(dbName: string, opts?: any): PouchDB.Database { checkInitialised() - return new Pouch(dbName, opts) + const db = new Pouch(dbName, opts) + const dbPut = db.put + db.put = async (doc: any, options = {}) => { + if (!doc.createdAt) { + doc.createdAt = new Date().toISOString() + } + doc.updatedAt = new Date().toISOString() + return dbPut(doc, options) + } + db.exists = async () => { + const info = await db.info() + return !info.error + } + return db } // use this function if you have called getPouchDB - close diff --git a/packages/backend-core/src/db/db.ts b/packages/backend-core/src/db/db.ts index 909d286d32..beaea3d42c 100644 --- a/packages/backend-core/src/db/db.ts +++ b/packages/backend-core/src/db/db.ts @@ -1,19 +1,15 @@ import env from "../environment" -import { directCouchQuery, PouchLike } from "./couch" +import { directCouchQuery, PouchLike, getPouchDB } from "./couch" import { CouchFindOptions } from "@budibase/types" -let initialised = false const dbList = new Set() -const checkInitialised = () => { - if (!initialised) { - throw new Error("init has not been called") - } -} - export function getDB(dbName?: string, opts?: any): PouchLike { + // TODO: once using the test image, need to remove this if (env.isTest()) { dbList.add(dbName) + // @ts-ignore + return getPouchDB(dbName, opts) } return new PouchLike(dbName, opts) } @@ -32,7 +28,6 @@ export function allDbs() { if (!env.isTest()) { throw new Error("Cannot be used outside test environment.") } - checkInitialised() return [...dbList] } diff --git a/packages/backend-core/src/db/tests/pouch.spec.js b/packages/backend-core/src/db/tests/pouch.spec.js index 30cdd0f5ec..ce27ab7d22 100644 --- a/packages/backend-core/src/db/tests/pouch.spec.js +++ b/packages/backend-core/src/db/tests/pouch.spec.js @@ -1,5 +1,5 @@ require("../../../tests/utilities/TestConfiguration") -const getUrlInfo = require("../pouch").getUrlInfo +const getUrlInfo = require("../couch").getUrlInfo describe("pouch", () => { describe("Couch DB URL parsing", () => { diff --git a/packages/backend-core/src/db/tests/utils.spec.js b/packages/backend-core/src/db/tests/utils.spec.js index 5f9a224e7a..1700621334 100644 --- a/packages/backend-core/src/db/tests/utils.spec.js +++ b/packages/backend-core/src/db/tests/utils.spec.js @@ -1,4 +1,4 @@ -require("../../../tests/utilities/TestConfiguration"); +require("../../../tests/utilities/TestConfiguration") const { generateAppID, getDevelopmentAppID, @@ -8,8 +8,8 @@ const { getPlatformUrl, getScopedConfig } = require("../utils") -const tenancy = require("../../tenancy"); -const { Configs, DEFAULT_TENANT_ID } = require("../../constants"); +const tenancy = require("../../tenancy") +const { Configs, DEFAULT_TENANT_ID } = require("../../constants") const env = require("../../environment") describe("utils", () => { diff --git a/packages/server/__mocks__/node-fetch.ts b/packages/server/__mocks__/node-fetch.ts index 0e32c39edd..c3d6b623dd 100644 --- a/packages/server/__mocks__/node-fetch.ts +++ b/packages/server/__mocks__/node-fetch.ts @@ -2,7 +2,7 @@ module FetchMock { const fetch = jest.requireActual("node-fetch") let failCount = 0 - module.exports = async (url: any, opts: any) => { + const func = async (url: any, opts: any) => { function json(body: any, status = 200) { return { status, @@ -106,4 +106,8 @@ module FetchMock { } return fetch(url, opts) } + + func.Headers = fetch.Headers + + module.exports = func } diff --git a/packages/server/package.json b/packages/server/package.json index 6d75877c79..ad8f62739d 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -43,7 +43,7 @@ "@budibase/backend-core/(.*)": "/../backend-core/$1", "@budibase/backend-core": "/../backend-core/src", "@budibase/types": "/../types/src", - "axios": "axios/dist/node/axios.cjs" + "^axios.*$": "/node_modules/axios/lib/axios.js" }, "setupFiles": [ "./scripts/jestSetup.js" diff --git a/packages/server/src/db/tests/linkTests.spec.js b/packages/server/src/db/tests/linkTests.spec.js index 8bc26cde2a..0684255ea1 100644 --- a/packages/server/src/db/tests/linkTests.spec.js +++ b/packages/server/src/db/tests/linkTests.spec.js @@ -1,30 +1,34 @@ const TestConfig = require("../../tests/utilities/TestConfiguration") const { basicTable } = require("../../tests/utilities/structures") const linkUtils = require("../linkedRows/linkUtils") -const { getAppDB } = require("@budibase/backend-core/context") -const { doWithDB } = require("@budibase/backend-core/db") +const { context } = require("@budibase/backend-core") describe("test link functionality", () => { const config = new TestConfig(false) + let appId describe("getLinkedTable", () => { - let db, table + let table beforeEach(async () => { - await config.init() - db = getAppDB() + const app = await config.init() + appId = app.appId table = await config.createTable() }) it("should be able to retrieve a linked table from a list", async () => { - const retrieved = await linkUtils.getLinkedTable(table._id, [table]) - expect(retrieved._id).toBe(table._id) + await context.doInAppContext(appId, async () => { + const retrieved = await linkUtils.getLinkedTable(table._id, [table]) + expect(retrieved._id).toBe(table._id) + }) }) it("should be able to retrieve a table from DB and update list", async () => { const tables = [] - const retrieved = await linkUtils.getLinkedTable(table._id, tables) - expect(retrieved._id).toBe(table._id) - expect(tables[0]).toBeDefined() + await context.doInAppContext(appId, async () => { + const retrieved = await linkUtils.getLinkedTable(table._id, tables) + expect(retrieved._id).toBe(table._id) + expect(tables[0]).toBeDefined() + }) }) }) @@ -48,15 +52,14 @@ describe("test link functionality", () => { describe("getLinkDocuments", () => { it("should create the link view when it doesn't exist", async () => { // create the DB and a very basic app design DB - const output = await doWithDB("test", async db => { - await db.put({ _id: "_design/database", views: {} }) - return await linkUtils.getLinkDocuments({ + await context.doInAppContext(appId, async () => { + const output = await linkUtils.getLinkDocuments({ tableId: "test", rowId: "test", includeDocs: false, }) + expect(Array.isArray(output)).toBe(true) }) - expect(Array.isArray(output)).toBe(true) }) }) }) \ No newline at end of file diff --git a/packages/server/src/middleware/tests/currentapp.spec.js b/packages/server/src/middleware/tests/currentapp.spec.js index 638abcbba4..12f68bedfc 100644 --- a/packages/server/src/middleware/tests/currentapp.spec.js +++ b/packages/server/src/middleware/tests/currentapp.spec.js @@ -1,16 +1,8 @@ +require("../../db").init() mockAuthWithNoCookie() mockWorker() mockUserGroups() -jest.mock("@budibase/backend-core/db", () => { - const coreDb = jest.requireActual("@budibase/backend-core/db") - coreDb.init() - return { - ...coreDb, - dbExists: async () => true, - } -}) - function mockWorker() { jest.mock("../../utilities/workerRequests", () => ({ getGlobalSelf: () => { @@ -43,42 +35,51 @@ function mockUserGroups() { function mockAuthWithNoCookie() { jest.resetModules() mockWorker() - jest.mock("@budibase/backend-core/cache", () => ({ - user: { - getUser: () => { - return { - _id: "us_uuid1", - } + jest.mock("@budibase/backend-core", () => { + const core = jest.requireActual("@budibase/backend-core") + return { + ...core, + db: { + dbExists: () => true, }, - }, - })) - jest.mock("@budibase/backend-core/utils", () => ({ - getAppIdFromCtx: jest.fn(), - setCookie: jest.fn(), - getCookie: jest.fn(), - })) - jest.mock("@budibase/backend-core/constants", () => ({ - Cookies: {}, - })) + cache: { + user: { + getUser: () => { + return { + _id: "us_uuid1", + } + }, + }, + }, + utils: { + getAppIdFromCtx: jest.fn(), + setCookie: jest.fn(), + getCookie: jest.fn(), + }, + } + }) } function mockAuthWithCookie() { jest.resetModules() mockWorker() - jest.mock("@budibase/backend-core/utils", () => ({ - getAppIdFromCtx: () => { - return "app_test" - }, - setCookie: jest.fn(), - clearCookie: jest.fn(), - getCookie: () => ({appId: "app_different", roleId: "PUBLIC"}), - })) - jest.mock("@budibase/backend-core/constants", () => ({ - Cookies: { - Auth: "auth", - CurrentApp: "currentapp", - }, - })) + jest.mock("@budibase/backend-core", () => { + const core = jest.requireActual("@budibase/backend-core") + return { + ...core, + db: { + dbExists: () => true, + }, + utils: { + getAppIdFromCtx: () => { + return "app_test" + }, + setCookie: jest.fn(), + clearCookie: jest.fn(), + getCookie: () => ({ appId: "app_different", roleId: "PUBLIC" }), + }, + } + }) } class TestConfiguration { @@ -88,7 +89,16 @@ class TestConfiguration { this.ctx = { next: this.next, - throw: this.throw + throw: this.throw, + request: { + body: {}, + headers: {}, + }, + headers: {}, + path: "", + cookies: { + set: jest.fn(), + } } } @@ -101,6 +111,8 @@ class TestConfiguration { executeMiddleware() { // import as late as possible for mocks + jest.resetModules() + require("../../db").init() const currentAppMiddleware = require("../currentapp") return currentAppMiddleware(this.ctx, this.next) } @@ -138,11 +150,11 @@ describe("Current app middleware", () => { async function checkExpected(setCookie) { config.setUser() await config.executeMiddleware() - let { setCookie: cookieFn } = require("@budibase/backend-core/utils") + let { utils } = require("@budibase/backend-core") if (setCookie) { - expect(cookieFn).toHaveBeenCalled() + expect(utils.setCookie).toHaveBeenCalled() } else { - expect(cookieFn).not.toHaveBeenCalled() + expect(utils.setCookie).not.toHaveBeenCalled() } expect(config.ctx.roleId).toEqual("PUBLIC") expect(config.ctx.user.role._id).toEqual("PUBLIC") @@ -157,31 +169,43 @@ describe("Current app middleware", () => { it("should perform correct when no cookie exists", async () => { mockReset() - jest.mock("@budibase/backend-core/utils", () => ({ - getAppIdFromCtx: () => { - return "app_test" - }, - setCookie: jest.fn(), - getCookie: jest.fn(), - })) - jest.mock("@budibase/backend-core/constants", () => ({ - Cookies: {}, - })) + jest.mock("@budibase/backend-core", () => { + const core = jest.requireActual("@budibase/backend-core") + return { + ...core, + db: { + dbExists: () => true, + }, + utils: { + getAppIdFromCtx: () => { + return "app_test" + }, + setCookie: jest.fn(), + getCookie: jest.fn(), + }, + } + }) await checkExpected(true) }) it("lastly check what occurs when cookie doesn't need updated", async () => { mockReset() - jest.mock("@budibase/backend-core/utils", () => ({ - getAppIdFromCtx: () => { - return "app_test" - }, - setCookie: jest.fn(), - getCookie: () => ({appId: "app_test", roleId: "PUBLIC"}), - })) - jest.mock("@budibase/backend-core/constants", () => ({ - Cookies: {}, - })) + jest.mock("@budibase/backend-core", () => { + const core = jest.requireActual("@budibase/backend-core") + return { + ...core, + db: { + dbExists: () => true, + }, + utils: { + getAppIdFromCtx: () => { + return "app_test" + }, + setCookie: jest.fn(), + getCookie: () => ({ appId: "app_test", roleId: "PUBLIC" }), + }, + } + }) await checkExpected(false) }) }) diff --git a/packages/worker/package.json b/packages/worker/package.json index 844448b569..1e694adfc9 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -99,7 +99,7 @@ "@budibase/backend-core/(.*)": "/../backend-core/$1", "@budibase/backend-core": "/../backend-core/src", "@budibase/types": "/../types/src", - "axios": "axios/dist/node/axios.cjs" + "^axios.*$": "/node_modules/axios/lib/axios.js" }, "setupFiles": [ "./scripts/jestSetup.js" From b5472f882be279bc4161989e500272f219b3d7f3 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 15 Nov 2022 17:35:17 +0000 Subject: [PATCH 12/24] Some updates for currentapp.spec.js test case. --- packages/backend-core/yarn.lock | 22 +-- .../src/middleware/tests/currentapp.spec.js | 29 +++- packages/server/src/utilities/global.js | 127 --------------- packages/server/src/utilities/global.ts | 136 ++++++++++++++++ packages/types/src/sdk/koa.ts | 2 +- .../TestConfiguration/applications.ts | 9 +- .../internal-api/TestConfiguration/tables.ts | 4 +- .../internal-api/fixtures/applications.ts | 1 - .../applications/applications.spec.ts | 29 +--- .../internal-api/screens/screens.spec.ts | 5 +- .../tests/internal-api/tables/tables.spec.ts | 150 +++++++++--------- 11 files changed, 253 insertions(+), 261 deletions(-) delete mode 100644 packages/server/src/utilities/global.js create mode 100644 packages/server/src/utilities/global.ts diff --git a/packages/backend-core/yarn.lock b/packages/backend-core/yarn.lock index 38302ee108..42614b4b3b 100644 --- a/packages/backend-core/yarn.lock +++ b/packages/backend-core/yarn.lock @@ -470,19 +470,12 @@ resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw== -<<<<<<< HEAD -"@budibase/types@2.1.14-alpha.2": - version "2.1.14-alpha.2" - resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.1.14-alpha.2.tgz#a537796012504e59afe06595f094c5ee2075de9b" - integrity sha512-6lrxQDBozX+yOWXBYN2K2Usg3liWXjmWtZ/F4Pky01dsFdD9M0GdYEUI0+Efhw78wEFaDeA2H9iXvuswKT/I6g== -======= "@cspotcode/source-map-support@^0.8.0": version "0.8.1" resolved "https://registry.yarnpkg.com/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz#00629c35a688e05a88b1cda684fb9d5e73f000a1" integrity sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw== dependencies: "@jridgewell/trace-mapping" "0.3.9" ->>>>>>> 32163fce702c09aa56597b2467d750d218406fc1 "@hapi/hoek@^9.0.0": version "9.3.0" @@ -1597,7 +1590,6 @@ axios@0.24.0: dependencies: follow-redirects "^1.14.4" -<<<<<<< HEAD axios@^1.1.3: version "1.1.3" resolved "https://registry.yarnpkg.com/axios/-/axios-1.1.3.tgz#8274250dada2edf53814ed7db644b9c2866c1e35" @@ -1607,16 +1599,10 @@ axios@^1.1.3: form-data "^4.0.0" proxy-from-env "^1.1.0" -babel-jest@^27.5.1: - version "27.5.1" - resolved "https://registry.yarnpkg.com/babel-jest/-/babel-jest-27.5.1.tgz#a1bf8d61928edfefd21da27eb86a695bfd691444" - integrity sha512-cdQ5dXjGRd0IBRATiQ4mZGlGlRE8kJpjPOixdNRdT+m3UcNqmYWN6rK6nvtXYfY3D76cb8s/O1Ss8ea24PIwcg== -======= babel-jest@^28.1.3: version "28.1.3" resolved "https://registry.yarnpkg.com/babel-jest/-/babel-jest-28.1.3.tgz#c1187258197c099072156a0a121c11ee1e3917d5" integrity sha512-epUaPOEWMk3cWX0M/sPvCHHCe9fMFAa/9hXEgKP8nFfNl/jlGkE9ucq9NqkZGXLDduCJYS0UvSlPUwC0S+rH6Q== ->>>>>>> 32163fce702c09aa56597b2467d750d218406fc1 dependencies: "@jest/transform" "^28.1.3" "@types/babel__core" "^7.1.14" @@ -5206,7 +5192,6 @@ shimmer@^1.2.0: resolved "https://registry.yarnpkg.com/shimmer/-/shimmer-1.2.1.tgz#610859f7de327b587efebf501fb43117f9aff337" integrity sha512-sQTKC1Re/rM6XyFM6fIAGHRPVGvyXfgzIDvzoq608vM+jeyVD0Tu1E6Np0Kc2zAIFWIj963V2800iF/9LPieQw== -<<<<<<< HEAD side-channel@^1.0.4: version "1.0.4" resolved "https://registry.yarnpkg.com/side-channel/-/side-channel-1.0.4.tgz#efce5c8fdc104ee751b25c58d4290011fa5ea2cf" @@ -5216,10 +5201,7 @@ side-channel@^1.0.4: get-intrinsic "^1.0.2" object-inspect "^1.9.0" -signal-exit@^3.0.0, signal-exit@^3.0.2, signal-exit@^3.0.3: -======= signal-exit@^3.0.0, signal-exit@^3.0.2, signal-exit@^3.0.3, signal-exit@^3.0.7: ->>>>>>> 32163fce702c09aa56597b2467d750d218406fc1 version "3.0.7" resolved "https://registry.yarnpkg.com/signal-exit/-/signal-exit-3.0.7.tgz#a9a1767f8af84155114eaabd73f99273c8f59ad9" integrity sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ== @@ -5688,12 +5670,11 @@ universalify@^0.1.2: resolved "https://registry.yarnpkg.com/universalify/-/universalify-0.1.2.tgz#b646f69be3942dabcecc9d6639c80dc105efaa66" integrity sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg== -<<<<<<< HEAD universalify@^0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/universalify/-/universalify-0.2.0.tgz#6451760566fa857534745ab1dde952d1b1761be0" integrity sha512-CJ1QgKmNg3CwvAv/kOFmtnEN05f0D/cn9QntgNOQlQF9dgvVTHj3t+8JPdjqawCHk7V/KA+fbUqzZ9XWhcqPUg== -======= + update-browserslist-db@^1.0.9: version "1.0.10" resolved "https://registry.yarnpkg.com/update-browserslist-db/-/update-browserslist-db-1.0.10.tgz#0f54b876545726f17d00cd9a2561e6dade943ff3" @@ -5701,7 +5682,6 @@ update-browserslist-db@^1.0.9: dependencies: escalade "^3.1.1" picocolors "^1.0.0" ->>>>>>> 32163fce702c09aa56597b2467d750d218406fc1 update-notifier@^5.1.0: version "5.1.0" diff --git a/packages/server/src/middleware/tests/currentapp.spec.js b/packages/server/src/middleware/tests/currentapp.spec.js index 12f68bedfc..16beb14f83 100644 --- a/packages/server/src/middleware/tests/currentapp.spec.js +++ b/packages/server/src/middleware/tests/currentapp.spec.js @@ -44,7 +44,7 @@ function mockAuthWithNoCookie() { }, cache: { user: { - getUser: () => { + getUser: async id => { return { _id: "us_uuid1", } @@ -78,6 +78,15 @@ function mockAuthWithCookie() { clearCookie: jest.fn(), getCookie: () => ({ appId: "app_different", roleId: "PUBLIC" }), }, + cache: { + user: { + getUser: async id => { + return { + _id: "us_uuid1", + } + }, + }, + }, } }) } @@ -183,6 +192,15 @@ describe("Current app middleware", () => { setCookie: jest.fn(), getCookie: jest.fn(), }, + cache: { + user: { + getUser: async id => { + return { + _id: "us_uuid1", + } + }, + }, + }, } }) await checkExpected(true) @@ -204,6 +222,15 @@ describe("Current app middleware", () => { setCookie: jest.fn(), getCookie: () => ({ appId: "app_test", roleId: "PUBLIC" }), }, + cache: { + user: { + getUser: async id => { + return { + _id: "us_uuid1", + } + }, + }, + }, } }) await checkExpected(false) diff --git a/packages/server/src/utilities/global.js b/packages/server/src/utilities/global.js deleted file mode 100644 index 6d82f79ce2..0000000000 --- a/packages/server/src/utilities/global.js +++ /dev/null @@ -1,127 +0,0 @@ -const { - getMultiIDParams, - getGlobalIDFromUserMetadataID, -} = require("../db/utils") -const { BUILTIN_ROLE_IDS } = require("@budibase/backend-core/roles") -const { getProdAppID } = require("@budibase/backend-core/db") -const { getGlobalUserParams } = require("@budibase/backend-core/db") -const { user: userCache } = require("@budibase/backend-core/cache") -const { - getGlobalDB, - isUserInAppTenant, -} = require("@budibase/backend-core/tenancy") -const env = require("../environment") -const { getAppId } = require("@budibase/backend-core/context") -const { groups } = require("@budibase/pro") - -exports.updateAppRole = (user, { appId } = {}) => { - appId = appId || getAppId() - - if (!user || !user.roles) { - return user - } - // if in an multi-tenancy environment make sure roles are never updated - if (env.MULTI_TENANCY && !isUserInAppTenant(appId, user)) { - delete user.builder - delete user.admin - user.roleId = BUILTIN_ROLE_IDS.PUBLIC - return user - } - // always use the deployed app - user.roleId = user.roles[getProdAppID(appId)] - // if a role wasn't found then either set as admin (builder) or public (everyone else) - if (!user.roleId && user.builder && user.builder.global) { - user.roleId = BUILTIN_ROLE_IDS.ADMIN - } else if (!user.roleId && !user?.userGroups?.length) { - user.roleId = BUILTIN_ROLE_IDS.PUBLIC - } else if (user?.userGroups?.length) { - user.roleId = null - } - - delete user.roles - return user -} - -async function checkGroupRoles(user, { appId } = {}) { - if (user.roleId && user.roleId !== BUILTIN_ROLE_IDS.PUBLIC) { - return user - } - user.roleId = await groups.getGroupRoleId(user, appId) - return user -} - -async function processUser(user, { appId } = {}) { - if (user) { - delete user.password - } - user = await exports.updateAppRole(user, { appId }) - if (!user.roleId && user?.userGroups?.length) { - user = await checkGroupRoles(user, { appId }) - } - - return user -} - -exports.getCachedSelf = async (ctx, appId) => { - // this has to be tenant aware, can't depend on the context to find it out - // running some middlewares before the tenancy causes context to break - const user = await userCache.getUser(ctx.user._id) - return processUser(user, { appId }) -} - -exports.getRawGlobalUser = async userId => { - const db = getGlobalDB() - return db.get(getGlobalIDFromUserMetadataID(userId)) -} - -exports.getGlobalUser = async userId => { - const appId = getAppId() - let user = await exports.getRawGlobalUser(userId) - return processUser(user, { appId }) -} - -exports.getGlobalUsers = async (users = null) => { - const appId = getAppId() - const db = getGlobalDB() - let globalUsers - if (users) { - const globalIds = users.map(user => getGlobalIDFromUserMetadataID(user._id)) - globalUsers = (await db.allDocs(getMultiIDParams(globalIds))).rows.map( - row => row.doc - ) - } else { - globalUsers = ( - await db.allDocs( - getGlobalUserParams(null, { - include_docs: true, - }) - ) - ).rows.map(row => row.doc) - } - globalUsers = globalUsers - .filter(user => user != null) - .map(user => { - delete user.password - delete user.forceResetPassword - return user - }) - if (!appId) { - return globalUsers - } - - return globalUsers.map(user => exports.updateAppRole(user)) -} - -exports.getGlobalUsersFromMetadata = async users => { - const globalUsers = await exports.getGlobalUsers(users) - return users.map(user => { - const globalUser = globalUsers.find( - globalUser => globalUser && user._id.includes(globalUser._id) - ) - return { - ...globalUser, - // doing user second overwrites the id and rev (always metadata) - ...user, - } - }) -} diff --git a/packages/server/src/utilities/global.ts b/packages/server/src/utilities/global.ts new file mode 100644 index 0000000000..f9f6b376ce --- /dev/null +++ b/packages/server/src/utilities/global.ts @@ -0,0 +1,136 @@ +import { getMultiIDParams, getGlobalIDFromUserMetadataID } from "../db/utils" +import { + roles, + db as dbCore, + cache, + tenancy, + context, +} from "@budibase/backend-core" +import env from "../environment" +import { groups } from "@budibase/pro" +import { BBContext, ContextUser, User } from "@budibase/types" + +export function updateAppRole( + user: ContextUser, + { appId }: { appId?: string } = {} +) { + appId = appId || context.getAppId() + + if (!user || !user.roles) { + return user + } + // if in an multi-tenancy environment make sure roles are never updated + if (env.MULTI_TENANCY && appId && !tenancy.isUserInAppTenant(appId, user)) { + delete user.builder + delete user.admin + user.roleId = roles.BUILTIN_ROLE_IDS.PUBLIC + return user + } + // always use the deployed app + if (appId) { + user.roleId = user.roles[dbCore.getProdAppID(appId)] + } + // if a role wasn't found then either set as admin (builder) or public (everyone else) + if (!user.roleId && user.builder && user.builder.global) { + user.roleId = roles.BUILTIN_ROLE_IDS.ADMIN + } else if (!user.roleId && !user?.userGroups?.length) { + user.roleId = roles.BUILTIN_ROLE_IDS.PUBLIC + } else if (user?.userGroups?.length) { + user.roleId = undefined + } + + delete user.roles + return user +} + +async function checkGroupRoles( + user: ContextUser, + { appId }: { appId?: string } = {} +) { + if (user.roleId && user.roleId !== roles.BUILTIN_ROLE_IDS.PUBLIC) { + return user + } + if (appId) { + user.roleId = await groups.getGroupRoleId(user as User, appId) + } + return user +} + +async function processUser( + user: ContextUser, + { appId }: { appId?: string } = {} +) { + if (user) { + delete user.password + } + user = await updateAppRole(user, { appId }) + if (!user.roleId && user?.userGroups?.length) { + user = await checkGroupRoles(user, { appId }) + } + + return user +} + +export async function getCachedSelf(ctx: BBContext, appId: string) { + // this has to be tenant aware, can't depend on the context to find it out + // running some middlewares before the tenancy causes context to break + const user = await cache.user.getUser(ctx.user?._id!) + return processUser(user, { appId }) +} + +export async function getRawGlobalUser(userId: string) { + const db = tenancy.getGlobalDB() + return db.get(getGlobalIDFromUserMetadataID(userId)) +} + +export async function getGlobalUser(userId: string) { + const appId = context.getAppId() + let user = await getRawGlobalUser(userId) + return processUser(user, { appId }) +} + +export async function getGlobalUsers(users?: ContextUser[]) { + const appId = context.getAppId() + const db = tenancy.getGlobalDB() + let globalUsers + if (users) { + const globalIds = users.map(user => getGlobalIDFromUserMetadataID(user._id)) + globalUsers = (await db.allDocs(getMultiIDParams(globalIds))).rows.map( + row => row.doc + ) + } else { + globalUsers = ( + await db.allDocs( + dbCore.getGlobalUserParams(null, { + include_docs: true, + }) + ) + ).rows.map(row => row.doc) + } + globalUsers = globalUsers + .filter(user => user != null) + .map(user => { + delete user.password + delete user.forceResetPassword + return user + }) + if (!appId) { + return globalUsers + } + + return globalUsers.map(user => updateAppRole(user)) +} + +export async function getGlobalUsersFromMetadata(users: ContextUser[]) { + const globalUsers = await getGlobalUsers(users) + return users.map(user => { + const globalUser = globalUsers.find( + globalUser => globalUser && user._id?.includes(globalUser._id) + ) + return { + ...globalUser, + // doing user second overwrites the id and rev (always metadata) + ...user, + } + }) +} diff --git a/packages/types/src/sdk/koa.ts b/packages/types/src/sdk/koa.ts index b8831dcbee..cd171bfe9e 100644 --- a/packages/types/src/sdk/koa.ts +++ b/packages/types/src/sdk/koa.ts @@ -6,7 +6,7 @@ export interface ContextUser extends Omit { globalId?: string license: License userId?: string - roleId?: string + roleId?: string | null role?: Role roles?: UserRoles } diff --git a/qa-core/src/config/internal-api/TestConfiguration/applications.ts b/qa-core/src/config/internal-api/TestConfiguration/applications.ts index cb0558222e..13d0969854 100644 --- a/qa-core/src/config/internal-api/TestConfiguration/applications.ts +++ b/qa-core/src/config/internal-api/TestConfiguration/applications.ts @@ -114,8 +114,6 @@ export default class AppApi { return [response, json] } - - async delete(appId: string): Promise<[Response, any]> { const response = await this.api.del(`/applications/${appId}`) const json = await response.json() @@ -123,7 +121,11 @@ export default class AppApi { return [response, json] } - async update(appId: string, oldName: string, body: any): Promise<[Response, Application]> { + async update( + appId: string, + oldName: string, + body: any + ): Promise<[Response, Application]> { const response = await this.api.put(`/applications/${appId}`, { body }) const json = await response.json() expect(response).toHaveStatusCode(200) @@ -142,7 +144,6 @@ export default class AppApi { const json = await response.json() expect(response).toHaveStatusCode(200) if (screenExists) { - expect(json.routes["/test"]).toBeTruthy() } else { expect(json.routes["/test"]).toBeUndefined() diff --git a/qa-core/src/config/internal-api/TestConfiguration/tables.ts b/qa-core/src/config/internal-api/TestConfiguration/tables.ts index ed0ab78cad..5b7e1648a0 100644 --- a/qa-core/src/config/internal-api/TestConfiguration/tables.ts +++ b/qa-core/src/config/internal-api/TestConfiguration/tables.ts @@ -46,9 +46,7 @@ export default class TablesApi { const response = await this.api.del(`/tables/${id}/${revId}`) const json = await response.json() expect(response).toHaveStatusCode(200) - expect(json.message).toEqual( - `Table ${id} deleted.` - ) + expect(json.message).toEqual(`Table ${id} deleted.`) return [response, json] } } diff --git a/qa-core/src/config/internal-api/fixtures/applications.ts b/qa-core/src/config/internal-api/fixtures/applications.ts index abdd674577..200aa9abff 100644 --- a/qa-core/src/config/internal-api/fixtures/applications.ts +++ b/qa-core/src/config/internal-api/fixtures/applications.ts @@ -1,7 +1,6 @@ import generator from "../../generator" import { Application } from "@budibase/server/api/controllers/public/mapping/types" - const generate = ( overrides: Partial = {} ): Partial => ({ diff --git a/qa-core/src/tests/internal-api/applications/applications.spec.ts b/qa-core/src/tests/internal-api/applications/applications.spec.ts index 4b9b66ec65..4b3208ee10 100644 --- a/qa-core/src/tests/internal-api/applications/applications.spec.ts +++ b/qa-core/src/tests/internal-api/applications/applications.spec.ts @@ -48,7 +48,8 @@ describe("Internal API - Application creation, update, publish and delete", () = }) config.applications.api.appId = app.appId - const [appPackageResponse, appPackageJson] = await config.applications.getAppPackage(app.appId) + const [appPackageResponse, appPackageJson] = + await config.applications.getAppPackage(app.appId) expect(appPackageJson.application.name).toEqual(app.name) expect(appPackageJson.application.version).toEqual(app.version) expect(appPackageJson.application.url).toEqual(app.url) @@ -72,7 +73,6 @@ describe("Internal API - Application creation, update, publish and delete", () = config.applications.api.appId = db.getProdAppID(app.appId) await config.applications.canRender() - // unpublish app await config.applications.unpublish(app.appId) }) @@ -109,22 +109,16 @@ describe("Internal API - Application creation, update, publish and delete", () = config.applications.api.appId = app.appId - await config.applications.update( - app.appId, - app.name, - { - name: generator.word(), - } - ) + await config.applications.update(app.appId, app.name, { + name: generator.word(), + }) }) it("POST - Revert Changes without changes", async () => { const app = await config.applications.create(generateApp()) config.applications.api.appId = app.appId - await config.applications.revertUnpublished( - app.appId - ) + await config.applications.revertUnpublished(app.appId) }) it("POST - Revert Changes", async () => { @@ -134,20 +128,14 @@ describe("Internal API - Application creation, update, publish and delete", () = // publish app await config.applications.publish(app.url) - // Change/add component to the app - await config.screen.create( - generateScreen("BASIC") - ) + await config.screen.create(generateScreen("BASIC")) // // Revert the app to published state - await config.applications.revertPublished( - app.appId - ) + await config.applications.revertPublished(app.appId) // Check screen is removed await config.applications.getRoutes() - }) it("DELETE - Delete an application", async () => { @@ -155,5 +143,4 @@ describe("Internal API - Application creation, update, publish and delete", () = await config.applications.delete(app.appId) }) - }) diff --git a/qa-core/src/tests/internal-api/screens/screens.spec.ts b/qa-core/src/tests/internal-api/screens/screens.spec.ts index 2dc7962914..218d71cb0d 100644 --- a/qa-core/src/tests/internal-api/screens/screens.spec.ts +++ b/qa-core/src/tests/internal-api/screens/screens.spec.ts @@ -38,9 +38,7 @@ describe("Internal API - /screens endpoints", () => { // Create Screen appConfig.applications.api.appId = app.appId - await config.screen.create( - generateScreen("BASIC") - ) + await config.screen.create(generateScreen("BASIC")) // Check screen exists await appConfig.applications.getRoutes(true) @@ -58,6 +56,5 @@ describe("Internal API - /screens endpoints", () => { // Delete Screen await config.screen.delete(screen._id!, screen._rev!) - }) }) diff --git a/qa-core/src/tests/internal-api/tables/tables.spec.ts b/qa-core/src/tests/internal-api/tables/tables.spec.ts index 69ad0fed7b..4f9e4299cf 100644 --- a/qa-core/src/tests/internal-api/tables/tables.spec.ts +++ b/qa-core/src/tests/internal-api/tables/tables.spec.ts @@ -3,93 +3,87 @@ import { Application } from "@budibase/server/api/controllers/public/mapping/typ import InternalAPIClient from "../../../config/internal-api/TestConfiguration/InternalAPIClient" import generator from "../../../config/generator" import { - generateTable, - generateNewColumnForTable, + generateTable, + generateNewColumnForTable, } from "../../../config/internal-api/fixtures/table" import { generateNewRowForTable } from "../../../config/internal-api/fixtures/rows" describe("Internal API - Application creation, update, publish and delete", () => { - const api = new InternalAPIClient() - const config = new TestConfiguration(api) + const api = new InternalAPIClient() + const config = new TestConfiguration(api) - beforeAll(async () => { - await config.beforeAll() + beforeAll(async () => { + await config.beforeAll() + }) + + afterAll(async () => { + await config.afterAll() + }) + + async function createAppFromTemplate() { + return config.applications.create({ + name: generator.word(), + url: `/${generator.word()}`, + useTemplate: "true", + templateName: "Near Miss Register", + templateKey: "app/near-miss-register", + templateFile: undefined, }) + } - afterAll(async () => { - await config.afterAll() - }) + it("Operations on Tables", async () => { + // create the app + const appName = generator.word() + const app = await createAppFromTemplate() + config.applications.api.appId = app.appId - async function createAppFromTemplate() { - return config.applications.create({ - name: generator.word(), - url: `/${generator.word()}`, - useTemplate: "true", - templateName: "Near Miss Register", - templateKey: "app/near-miss-register", - templateFile: undefined, - }) + // Get current tables: expect 2 in this template + await config.tables.getAll(2) + + // Add new table + const [createdTableResponse, createdTableData] = await config.tables.save( + generateTable() + ) + + //Table was added + await config.tables.getAll(3) + + //Get information about the table + await config.tables.getTableById(createdTableData._id) + + //Add Column to table + const newColumn = generateNewColumnForTable(createdTableData) + const [addColumnResponse, addColumnData] = await config.tables.save( + newColumn, + true + ) + + //Add Row to table + const newRow = generateNewRowForTable(addColumnData._id) + await config.rows.add(addColumnData._id, newRow) + + //Get Row from table + const [getRowResponse, getRowData] = await config.rows.getAll( + addColumnData._id + ) + + //Delete Row from table + const rowToDelete = { + rows: [getRowData[0]], } + const [deleteRowResponse, deleteRowData] = await config.rows.delete( + addColumnData._id, + rowToDelete + ) + expect(deleteRowData[0]._id).toEqual(getRowData[0]._id) - it("Operations on Tables", async () => { - // create the app - const appName = generator.word() - const app = await createAppFromTemplate() - config.applications.api.appId = app.appId + //Delete the table + const [deleteTableResponse, deleteTable] = await config.tables.delete( + addColumnData._id, + addColumnData._rev + ) - // Get current tables: expect 2 in this template - await config.tables.getAll(2) - - // Add new table - const [createdTableResponse, createdTableData] = await config.tables.save( - generateTable() - ) - - //Table was added - await config.tables.getAll(3) - - //Get information about the table - await config.tables.getTableById( - createdTableData._id - ) - - //Add Column to table - const newColumn = generateNewColumnForTable(createdTableData) - const [addColumnResponse, addColumnData] = await config.tables.save( - newColumn, - true - ) - - //Add Row to table - const newRow = generateNewRowForTable(addColumnData._id) - await config.rows.add( - addColumnData._id, - newRow - ) - - //Get Row from table - const [getRowResponse, getRowData] = await config.rows.getAll( - addColumnData._id - ) - - //Delete Row from table - const rowToDelete = { - rows: [getRowData[0]], - } - const [deleteRowResponse, deleteRowData] = await config.rows.delete( - addColumnData._id, - rowToDelete - ) - expect(deleteRowData[0]._id).toEqual(getRowData[0]._id) - - //Delete the table - const [deleteTableResponse, deleteTable] = await config.tables.delete( - addColumnData._id, - addColumnData._rev - ) - - - //Table was deleted - await config.tables.getAll(2) - }) + //Table was deleted + await config.tables.getAll(2) + }) }) From 5f658a039809dab273bbd1191048b6712a651ccd Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 15 Nov 2022 18:33:24 +0000 Subject: [PATCH 13/24] Converting further DB layers to TS - fixing mocks. --- packages/server/src/api/controllers/auth.ts | 5 +- .../server/src/api/controllers/query/index.ts | 25 ++- .../server/src/api/routes/tests/query.spec.js | 25 +-- .../server/src/api/routes/tests/table.spec.js | 73 ++++---- packages/server/src/db/{utils.js => utils.ts} | 164 ++++++++---------- packages/server/src/threads/automation.ts | 2 +- packages/server/src/utilities/global.ts | 4 +- packages/types/src/sdk/koa.ts | 1 + 8 files changed, 146 insertions(+), 153 deletions(-) rename packages/server/src/db/{utils.js => utils.ts} (61%) diff --git a/packages/server/src/api/controllers/auth.ts b/packages/server/src/api/controllers/auth.ts index ef2cb29385..88dc581020 100644 --- a/packages/server/src/api/controllers/auth.ts +++ b/packages/server/src/api/controllers/auth.ts @@ -3,6 +3,7 @@ import { InternalTables } from "../../db/utils" import { getFullUser } from "../../utilities/users" import { roles, context } from "@budibase/backend-core" import { groups } from "@budibase/pro" +import { ContextUser, User } from "@budibase/types" const PUBLIC_ROLE = roles.BUILTIN_ROLE_IDS.PUBLIC @@ -24,7 +25,7 @@ export async function fetchSelf(ctx: any) { } const appId = context.getAppId() - const user = await getFullUser(ctx, userId) + const user: ContextUser = await getFullUser(ctx, userId) // this shouldn't be returned by the app self delete user.roles // forward the csrf token from the session @@ -34,7 +35,7 @@ export async function fetchSelf(ctx: any) { const db = context.getAppDB() // check for group permissions if (!user.roleId || user.roleId === PUBLIC_ROLE) { - const groupRoleId = await groups.getGroupRoleId(user, appId) + const groupRoleId = await groups.getGroupRoleId(user as User, appId) user.roleId = groupRoleId || user.roleId } // remove the full roles structure diff --git a/packages/server/src/api/controllers/query/index.ts b/packages/server/src/api/controllers/query/index.ts index c92f942986..fa40320e91 100644 --- a/packages/server/src/api/controllers/query/index.ts +++ b/packages/server/src/api/controllers/query/index.ts @@ -5,11 +5,8 @@ import { save as saveDatasource } from "../datasource" import { RestImporter } from "./import" import { invalidateDynamicVariables } from "../../../threads/utils" import { QUERY_THREAD_TIMEOUT } from "../../../environment" -import { getAppDB } from "@budibase/backend-core/context" import { quotas } from "@budibase/pro" -import { events } from "@budibase/backend-core" -import { getCookie } from "@budibase/backend-core/utils" -import { Cookies, Configs } from "@budibase/backend-core/constants" +import { events, context, utils, constants } from "@budibase/backend-core" const Runner = new Thread(ThreadType.QUERY, { timeoutMs: QUERY_THREAD_TIMEOUT || 10000, @@ -28,7 +25,7 @@ function enrichQueries(input: any) { } export async function fetch(ctx: any) { - const db = getAppDB() + const db = context.getAppDB() const body = await db.allDocs( getQueryParams(null, { @@ -81,7 +78,7 @@ const _import = async (ctx: any) => { export { _import as import } export async function save(ctx: any) { - const db = getAppDB() + const db = context.getAppDB() const query = ctx.request.body const datasource = await db.get(query.datasourceId) @@ -103,7 +100,7 @@ export async function save(ctx: any) { } export async function find(ctx: any) { - const db = getAppDB() + const db = context.getAppDB() const query = enrichQueries(await db.get(ctx.params.queryId)) // remove properties that could be dangerous in real app if (isProdAppID(ctx.appId)) { @@ -115,13 +112,13 @@ export async function find(ctx: any) { //Required to discern between OIDC OAuth config entries function getOAuthConfigCookieId(ctx: any) { - if (ctx.user.providerType === Configs.OIDC) { - return getCookie(ctx, Cookies.OIDC_CONFIG) + if (ctx.user.providerType === constants.Configs.OIDC) { + return utils.getCookie(ctx, constants.Cookies.OIDC_CONFIG) } } function getAuthConfig(ctx: any) { - const authCookie = getCookie(ctx, Cookies.Auth) + const authCookie = utils.getCookie(ctx, constants.Cookies.Auth) let authConfigCtx: any = {} authConfigCtx["configId"] = getOAuthConfigCookieId(ctx) authConfigCtx["sessionId"] = authCookie ? authCookie.sessionId : null @@ -129,7 +126,7 @@ function getAuthConfig(ctx: any) { } export async function preview(ctx: any) { - const db = getAppDB() + const db = context.getAppDB() const datasource = await db.get(ctx.request.body.datasourceId) const query = ctx.request.body @@ -201,7 +198,7 @@ async function execute( ctx: any, opts: any = { rowsOnly: false, isAutomation: false } ) { - const db = getAppDB() + const db = context.getAppDB() const query = await db.get(ctx.params.queryId) const datasource = await db.get(query.datasourceId) @@ -267,7 +264,7 @@ export async function executeV2( } const removeDynamicVariables = async (queryId: any) => { - const db = getAppDB() + const db = context.getAppDB() const query = await db.get(queryId) const datasource = await db.get(query.datasourceId) const dynamicVariables = datasource.config.dynamicVariables @@ -288,7 +285,7 @@ const removeDynamicVariables = async (queryId: any) => { } export async function destroy(ctx: any) { - const db = getAppDB() + const db = context.getAppDB() const queryId = ctx.params.queryId await removeDynamicVariables(queryId) const query = await db.get(queryId) diff --git a/packages/server/src/api/routes/tests/query.spec.js b/packages/server/src/api/routes/tests/query.spec.js index 897659e13f..f9f2fa8347 100644 --- a/packages/server/src/api/routes/tests/query.spec.js +++ b/packages/server/src/api/routes/tests/query.spec.js @@ -4,16 +4,21 @@ jest.mock("node-fetch") // Mock isProdAppID to we can later mock the implementation and pretend we are // using prod app IDs -const authDb = require("@budibase/backend-core/db") -const { isProdAppID } = authDb -const mockIsProdAppID = jest.fn(isProdAppID) -authDb.isProdAppID = mockIsProdAppID - +jest.mock("@budibase/backend-core", () => { + const core = jest.requireActual("@budibase/backend-core") + return { + ...core, + db: { + ...core.db, + isProdAppID: jest.fn(), + } + } +}) const setup = require("./utilities") const { checkBuilderEndpoint } = require("./utilities/TestFunctions") const { checkCacheForDynamicVariable } = require("../../../threads/utils") const { basicQuery, basicDatasource } = setup.structures -const { events } = require("@budibase/backend-core") +const { events, db: dbCore } = require("@budibase/backend-core") describe("/queries", () => { let request = setup.getRequest() @@ -152,8 +157,8 @@ describe("/queries", () => { it("should remove sensitive info for prod apps", async () => { // Mock isProdAppID to pretend we are using a prod app - mockIsProdAppID.mockClear() - mockIsProdAppID.mockImplementation(() => true) + dbCore.isProdAppID.mockClear() + dbCore.isProdAppID.mockImplementation(() => true) const query = await config.createQuery() const res = await request @@ -167,8 +172,8 @@ describe("/queries", () => { expect(res.body.schema).toBeDefined() // Reset isProdAppID mock - expect(mockIsProdAppID).toHaveBeenCalledTimes(1) - mockIsProdAppID.mockImplementation(isProdAppID) + expect(dbCore.isProdAppID).toHaveBeenCalledTimes(1) + dbCore.isProdAppID.mockImplementation(() => false) }) }) diff --git a/packages/server/src/api/routes/tests/table.spec.js b/packages/server/src/api/routes/tests/table.spec.js index ca1a067a4d..e42aaf07e4 100644 --- a/packages/server/src/api/routes/tests/table.spec.js +++ b/packages/server/src/api/routes/tests/table.spec.js @@ -1,17 +1,18 @@ const { checkBuilderEndpoint } = require("./utilities/TestFunctions") -const { getAppDB } = require("@budibase/backend-core/context") const setup = require("./utilities") const { basicTable } = setup.structures -const { events } = require("@budibase/backend-core") +const { events, context } = require("@budibase/backend-core") describe("/tables", () => { let request = setup.getRequest() let config = setup.getConfig() + let appId afterAll(setup.afterAll) beforeEach(async () => { - await config.init() + const app = await config.init() + appId = app.appId }) describe("create", () => { @@ -199,38 +200,6 @@ describe("/tables", () => { }) }) - describe("indexing", () => { - it("should be able to create a table with indexes", async () => { - const db = getAppDB(config) - const indexCount = (await db.getIndexes()).total_rows - const table = basicTable() - table.indexes = ["name"] - const res = await request - .post(`/api/tables`) - .send(table) - .set(config.defaultHeaders()) - .expect('Content-Type', /json/) - .expect(200) - expect(res.body._id).toBeDefined() - expect(res.body._rev).toBeDefined() - expect((await db.getIndexes()).total_rows).toEqual(indexCount + 1) - // update index to see what happens - table.indexes = ["name", "description"] - await request - .post(`/api/tables`) - .send({ - ...table, - _id: res.body._id, - _rev: res.body._rev, - }) - .set(config.defaultHeaders()) - .expect('Content-Type', /json/) - .expect(200) - // shouldn't have created a new index - expect((await db.getIndexes()).total_rows).toEqual(indexCount + 1) - }) - }) - describe("validate csv", () => { it("should be able to validate a CSV layout", async () => { const res = await request @@ -249,6 +218,40 @@ describe("/tables", () => { }) }) + describe("indexing", () => { + it("should be able to create a table with indexes", async () => { + await context.doInAppContext(appId, async () => { + const db = context.getAppDB() + const indexCount = (await db.getIndexes()).total_rows + const table = basicTable() + table.indexes = ["name"] + const res = await request + .post(`/api/tables`) + .send(table) + .set(config.defaultHeaders()) + .expect('Content-Type', /json/) + .expect(200) + expect(res.body._id).toBeDefined() + expect(res.body._rev).toBeDefined() + expect((await db.getIndexes()).total_rows).toEqual(indexCount + 1) + // update index to see what happens + table.indexes = ["name", "description"] + await request + .post(`/api/tables`) + .send({ + ...table, + _id: res.body._id, + _rev: res.body._rev, + }) + .set(config.defaultHeaders()) + .expect('Content-Type', /json/) + .expect(200) + // shouldn't have created a new index + expect((await db.getIndexes()).total_rows).toEqual(indexCount + 1) + }) + }) + }) + describe("destroy", () => { let testTable diff --git a/packages/server/src/db/utils.js b/packages/server/src/db/utils.ts similarity index 61% rename from packages/server/src/db/utils.js rename to packages/server/src/db/utils.ts index acd72cbf66..58ea76c5ae 100644 --- a/packages/server/src/db/utils.js +++ b/packages/server/src/db/utils.ts @@ -1,44 +1,19 @@ -const newid = require("./newid") -const { - DocumentType: CoreDocType, - InternalTable, - getRoleParams, - generateRoleID, - APP_DEV_PREFIX, - APP_PREFIX, - SEPARATOR, - StaticDatabases, - isDevAppID, - isProdAppID, - getDevelopmentAppID, - generateAppID, - getQueryIndex, - ViewName, - getDocParams, - getRowParams, - generateRowID, - getUserMetadataParams, - generateUserMetadataID, - getGlobalIDFromUserMetadataID, -} = require("@budibase/backend-core/db") +import newid from "./newid" +import { db as dbCore } from "@budibase/backend-core" -const UNICODE_MAX = "\ufff0" +type Optional = string | null -const AppStatus = { +export const AppStatus = { DEV: "development", ALL: "all", DEPLOYED: "published", } -const DocumentType = CoreDocType - -const SearchIndexes = { +export const SearchIndexes = { ROWS: "rows", } -exports.StaticDatabases = StaticDatabases - -const BudibaseInternalDB = { +export const BudibaseInternalDB = { _id: "bb_internal", type: "budibase", name: "Budibase DB", @@ -46,37 +21,36 @@ const BudibaseInternalDB = { config: {}, } -exports.APP_PREFIX = APP_PREFIX -exports.APP_DEV_PREFIX = APP_DEV_PREFIX -exports.isDevAppID = isDevAppID -exports.isProdAppID = isProdAppID -exports.USER_METDATA_PREFIX = `${DocumentType.ROW}${SEPARATOR}${InternalTable.USER_METADATA}${SEPARATOR}` -exports.LINK_USER_METADATA_PREFIX = `${DocumentType.LINK}${SEPARATOR}${InternalTable.USER_METADATA}${SEPARATOR}` -exports.TABLE_ROW_PREFIX = `${DocumentType.ROW}${SEPARATOR}${DocumentType.TABLE}` -exports.ViewName = ViewName -exports.InternalTables = InternalTable -exports.DocumentType = DocumentType -exports.SEPARATOR = SEPARATOR -exports.UNICODE_MAX = UNICODE_MAX -exports.SearchIndexes = SearchIndexes -exports.AppStatus = AppStatus -exports.BudibaseInternalDB = BudibaseInternalDB -exports.generateAppID = generateAppID -exports.generateDevAppID = getDevelopmentAppID -exports.generateRoleID = generateRoleID -exports.getRoleParams = getRoleParams -exports.getQueryIndex = getQueryIndex -exports.getDocParams = getDocParams -exports.getRowParams = getRowParams -exports.generateRowID = generateRowID -exports.getUserMetadataParams = getUserMetadataParams -exports.generateUserMetadataID = generateUserMetadataID -exports.getGlobalIDFromUserMetadataID = getGlobalIDFromUserMetadataID +export const SEPARATOR = dbCore.SEPARATOR +export const StaticDatabases = dbCore.StaticDatabases +export const DocumentType = dbCore.DocumentType +export const APP_PREFIX = dbCore.APP_PREFIX +export const APP_DEV_PREFIX = dbCore.APP_DEV_PREFIX +export const isDevAppID = dbCore.isDevAppID +export const isProdAppID = dbCore.isProdAppID +export const USER_METDATA_PREFIX = `${DocumentType.ROW}${SEPARATOR}${dbCore.InternalTable.USER_METADATA}${SEPARATOR}` +export const LINK_USER_METADATA_PREFIX = `${DocumentType.LINK}${SEPARATOR}${dbCore.InternalTable.USER_METADATA}${SEPARATOR}` +export const TABLE_ROW_PREFIX = `${DocumentType.ROW}${SEPARATOR}${DocumentType.TABLE}` +export const ViewName = dbCore.ViewName +export const InternalTables = dbCore.InternalTable +export const UNICODE_MAX = dbCore.UNICODE_MAX +export const generateAppID = dbCore.generateAppID +export const generateDevAppID = dbCore.getDevelopmentAppID +export const generateRoleID = dbCore.generateRoleID +export const getRoleParams = dbCore.getRoleParams +export const getQueryIndex = dbCore.getQueryIndex +export const getDocParams = dbCore.getDocParams +export const getRowParams = dbCore.getRowParams +export const generateRowID = dbCore.generateRowID +export const getUserMetadataParams = dbCore.getUserMetadataParams +export const generateUserMetadataID = dbCore.generateUserMetadataID +export const getGlobalIDFromUserMetadataID = + dbCore.getGlobalIDFromUserMetadataID /** * Gets parameters for retrieving tables, this is a utility function for the getDocParams function. */ -exports.getTableParams = (tableId = null, otherProps = {}) => { +export function getTableParams(tableId?: Optional, otherProps = {}) { return getDocParams(DocumentType.TABLE, tableId, otherProps) } @@ -84,7 +58,7 @@ exports.getTableParams = (tableId = null, otherProps = {}) => { * Generates a new table ID. * @returns {string} The new table ID which the table doc can be stored under. */ -exports.generateTableID = () => { +export function generateTableID() { return `${DocumentType.TABLE}${SEPARATOR}${newid()}` } @@ -93,7 +67,7 @@ exports.generateTableID = () => { * @param {string} rowId The ID of the row. * @returns {string} The table ID. */ -exports.getTableIDFromRowID = rowId => { +export function getTableIDFromRowID(rowId: string) { const components = rowId .split(DocumentType.TABLE + SEPARATOR)[1] .split(SEPARATOR) @@ -103,7 +77,10 @@ exports.getTableIDFromRowID = rowId => { /** * Gets parameters for retrieving automations, this is a utility function for the getDocParams function. */ -exports.getAutomationParams = (automationId = null, otherProps = {}) => { +export function getAutomationParams( + automationId?: Optional, + otherProps: any = {} +) { return getDocParams(DocumentType.AUTOMATION, automationId, otherProps) } @@ -111,7 +88,7 @@ exports.getAutomationParams = (automationId = null, otherProps = {}) => { * Generates a new automation ID. * @returns {string} The new automation ID which the automation doc can be stored under. */ -exports.generateAutomationID = () => { +export function generateAutomationID() { return `${DocumentType.AUTOMATION}${SEPARATOR}${newid()}` } @@ -126,14 +103,14 @@ exports.generateAutomationID = () => { * @param {string} fieldName2 the name of the field in the linked row. * @returns {string} The new link doc ID which the automation doc can be stored under. */ -exports.generateLinkID = ( - tableId1, - tableId2, - rowId1, - rowId2, - fieldName1, - fieldName2 -) => { +export function generateLinkID( + tableId1: string, + tableId2: string, + rowId1: string, + rowId2: string, + fieldName1: string, + fieldName2: string +) { const tables = `${SEPARATOR}${tableId1}${SEPARATOR}${tableId2}` const rows = `${SEPARATOR}${rowId1}${SEPARATOR}${rowId2}` const fields = `${SEPARATOR}${fieldName1}${SEPARATOR}${fieldName2}` @@ -143,7 +120,7 @@ exports.generateLinkID = ( /** * Gets parameters for retrieving link docs, this is a utility function for the getDocParams function. */ -exports.getLinkParams = (otherProps = {}) => { +export function getLinkParams(otherProps: any = {}) { return getDocParams(DocumentType.LINK, null, otherProps) } @@ -151,14 +128,14 @@ exports.getLinkParams = (otherProps = {}) => { * Generates a new layout ID. * @returns {string} The new layout ID which the layout doc can be stored under. */ -exports.generateLayoutID = id => { +export function generateLayoutID(id: string) { return `${DocumentType.LAYOUT}${SEPARATOR}${id || newid()}` } /** * Gets parameters for retrieving layout, this is a utility function for the getDocParams function. */ -exports.getLayoutParams = (layoutId = null, otherProps = {}) => { +export function getLayoutParams(layoutId?: Optional, otherProps: any = {}) { return getDocParams(DocumentType.LAYOUT, layoutId, otherProps) } @@ -166,14 +143,14 @@ exports.getLayoutParams = (layoutId = null, otherProps = {}) => { * Generates a new screen ID. * @returns {string} The new screen ID which the screen doc can be stored under. */ -exports.generateScreenID = () => { +export function generateScreenID() { return `${DocumentType.SCREEN}${SEPARATOR}${newid()}` } /** * Gets parameters for retrieving screens, this is a utility function for the getDocParams function. */ -exports.getScreenParams = (screenId = null, otherProps = {}) => { +export function getScreenParams(screenId?: Optional, otherProps: any = {}) { return getDocParams(DocumentType.SCREEN, screenId, otherProps) } @@ -181,14 +158,14 @@ exports.getScreenParams = (screenId = null, otherProps = {}) => { * Generates a new webhook ID. * @returns {string} The new webhook ID which the webhook doc can be stored under. */ -exports.generateWebhookID = () => { +export function generateWebhookID() { return `${DocumentType.WEBHOOK}${SEPARATOR}${newid()}` } /** * Gets parameters for retrieving a webhook, this is a utility function for the getDocParams function. */ -exports.getWebhookParams = (webhookId = null, otherProps = {}) => { +export function getWebhookParams(webhookId?: Optional, otherProps: any = {}) { return getDocParams(DocumentType.WEBHOOK, webhookId, otherProps) } @@ -196,7 +173,7 @@ exports.getWebhookParams = (webhookId = null, otherProps = {}) => { * Generates a new datasource ID. * @returns {string} The new datasource ID which the webhook doc can be stored under. */ -exports.generateDatasourceID = ({ plus = false } = {}) => { +export function generateDatasourceID({ plus = false } = {}) { return `${ plus ? DocumentType.DATASOURCE_PLUS : DocumentType.DATASOURCE }${SEPARATOR}${newid()}` @@ -205,7 +182,10 @@ exports.generateDatasourceID = ({ plus = false } = {}) => { /** * Gets parameters for retrieving a datasource, this is a utility function for the getDocParams function. */ -exports.getDatasourceParams = (datasourceId = null, otherProps = {}) => { +export function getDatasourceParams( + datasourceId?: Optional, + otherProps: any = {} +) { return getDocParams(DocumentType.DATASOURCE, datasourceId, otherProps) } @@ -213,7 +193,7 @@ exports.getDatasourceParams = (datasourceId = null, otherProps = {}) => { * Generates a new query ID. * @returns {string} The new query ID which the query doc can be stored under. */ -exports.generateQueryID = datasourceId => { +export function generateQueryID(datasourceId: string) { return `${ DocumentType.QUERY }${SEPARATOR}${datasourceId}${SEPARATOR}${newid()}` @@ -223,21 +203,21 @@ exports.generateQueryID = datasourceId => { * Generates a metadata ID for automations, used to track errors in recurring * automations etc. */ -exports.generateAutomationMetadataID = automationId => { +export function generateAutomationMetadataID(automationId: string) { return `${DocumentType.AUTOMATION_METADATA}${SEPARATOR}${automationId}` } /** * Retrieve all automation metadata in an app database. */ -exports.getAutomationMetadataParams = (otherProps = {}) => { +export function getAutomationMetadataParams(otherProps: any = {}) { return getDocParams(DocumentType.AUTOMATION_METADATA, null, otherProps) } /** * Gets parameters for retrieving a query, this is a utility function for the getDocParams function. */ -exports.getQueryParams = (datasourceId = null, otherProps = {}) => { +export function getQueryParams(datasourceId?: Optional, otherProps: any = {}) { if (datasourceId == null) { return getDocParams(DocumentType.QUERY, null, otherProps) } @@ -253,15 +233,19 @@ exports.getQueryParams = (datasourceId = null, otherProps = {}) => { * Generates a new flag document ID. * @returns {string} The ID of the flag document that was generated. */ -exports.generateUserFlagID = userId => { +export function generateUserFlagID(userId: string) { return `${DocumentType.USER_FLAG}${SEPARATOR}${userId}` } -exports.generateMetadataID = (type, entityId) => { +export function generateMetadataID(type: string, entityId: string) { return `${DocumentType.METADATA}${SEPARATOR}${type}${SEPARATOR}${entityId}` } -exports.getMetadataParams = (type, entityId = null, otherProps = {}) => { +export function getMetadataParams( + type: string, + entityId?: Optional, + otherProps: any = {} +) { let docId = `${type}${SEPARATOR}` if (entityId != null) { docId += entityId @@ -269,22 +253,22 @@ exports.getMetadataParams = (type, entityId = null, otherProps = {}) => { return getDocParams(DocumentType.METADATA, docId, otherProps) } -exports.generateMemoryViewID = viewName => { +export function generateMemoryViewID(viewName: string) { return `${DocumentType.MEM_VIEW}${SEPARATOR}${viewName}` } -exports.getMemoryViewParams = (otherProps = {}) => { +export function getMemoryViewParams(otherProps: any = {}) { return getDocParams(DocumentType.MEM_VIEW, null, otherProps) } -exports.generatePluginID = name => { +export function generatePluginID(name: string) { return `${DocumentType.PLUGIN}${SEPARATOR}${name}` } /** * This can be used with the db.allDocs to get a list of IDs */ -exports.getMultiIDParams = ids => { +export function getMultiIDParams(ids: string[]) { return { keys: ids, include_docs: true, diff --git a/packages/server/src/threads/automation.ts b/packages/server/src/threads/automation.ts index b4b290462e..3552ffa0f3 100644 --- a/packages/server/src/threads/automation.ts +++ b/packages/server/src/threads/automation.ts @@ -120,7 +120,7 @@ class Orchestrator { } async getMetadata(): Promise { - const metadataId = generateAutomationMetadataID(this._automation._id) + const metadataId = generateAutomationMetadataID(this._automation._id!) const db = getAppDB() let metadata: AutomationMetadata try { diff --git a/packages/server/src/utilities/global.ts b/packages/server/src/utilities/global.ts index f9f6b376ce..c9161aa426 100644 --- a/packages/server/src/utilities/global.ts +++ b/packages/server/src/utilities/global.ts @@ -94,7 +94,9 @@ export async function getGlobalUsers(users?: ContextUser[]) { const db = tenancy.getGlobalDB() let globalUsers if (users) { - const globalIds = users.map(user => getGlobalIDFromUserMetadataID(user._id)) + const globalIds = users.map(user => + getGlobalIDFromUserMetadataID(user._id!) + ) globalUsers = (await db.allDocs(getMultiIDParams(globalIds))).rows.map( row => row.doc ) diff --git a/packages/types/src/sdk/koa.ts b/packages/types/src/sdk/koa.ts index cd171bfe9e..ffbb1cf053 100644 --- a/packages/types/src/sdk/koa.ts +++ b/packages/types/src/sdk/koa.ts @@ -9,6 +9,7 @@ export interface ContextUser extends Omit { roleId?: string | null role?: Role roles?: UserRoles + csrfToken?: string } export interface BBRequest extends Request { From c6366c573a15717e8f44e77d039acc0e8fe2a0c9 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 15 Nov 2022 19:04:39 +0000 Subject: [PATCH 14/24] Final server test fixes, after all updates to context/removal of context faking. --- .../src/automations/tests/updateRow.spec.js | 1 - .../src/automations/tests/utilities/index.js | 14 +- .../src/db/tests/linkController.spec.js | 130 ++++++++++-------- .../src/middleware/tests/currentapp.spec.js | 4 + 4 files changed, 87 insertions(+), 62 deletions(-) diff --git a/packages/server/src/automations/tests/updateRow.spec.js b/packages/server/src/automations/tests/updateRow.spec.js index 79c998459b..36b3b1748f 100644 --- a/packages/server/src/automations/tests/updateRow.spec.js +++ b/packages/server/src/automations/tests/updateRow.spec.js @@ -1,4 +1,3 @@ -const env = require("../../environment") const setup = require("./utilities") describe("test the update row action", () => { diff --git a/packages/server/src/automations/tests/utilities/index.js b/packages/server/src/automations/tests/utilities/index.js index b2e1566687..1e705182dc 100644 --- a/packages/server/src/automations/tests/utilities/index.js +++ b/packages/server/src/automations/tests/utilities/index.js @@ -1,6 +1,5 @@ const TestConfig = require("../../../tests/utilities/TestConfiguration") -const { TENANT_ID } = require("../../../tests/utilities/structures") -const { doInTenant } = require("@budibase/backend-core/tenancy") +const { context } = require("@budibase/backend-core") const actions = require("../../actions") const emitter = require("../../../events/index") const env = require("../../../environment") @@ -33,7 +32,7 @@ exports.runInProd = async fn => { } exports.runStep = async function runStep(stepId, inputs) { - return doInTenant(TENANT_ID, async () => { + async function run() { let step = await actions.getAction(stepId) expect(step).toBeDefined() return step({ @@ -43,7 +42,14 @@ exports.runStep = async function runStep(stepId, inputs) { apiKey: exports.apiKey, emitter, }) - }) + } + if (config?.appId) { + return context.doInContext(config?.appId, async () => { + return run() + }) + } else { + return run() + } } exports.apiKey = "test" diff --git a/packages/server/src/db/tests/linkController.spec.js b/packages/server/src/db/tests/linkController.spec.js index 180cc2b3a0..730ec3a8b3 100644 --- a/packages/server/src/db/tests/linkController.spec.js +++ b/packages/server/src/db/tests/linkController.spec.js @@ -1,15 +1,17 @@ const TestConfig = require("../../tests/utilities/TestConfiguration") const { basicRow, basicLinkedRow, basicTable } = require("../../tests/utilities/structures") const LinkController = require("../linkedRows/LinkController") +const { context } = require("@budibase/backend-core") const { RelationshipTypes } = require("../../constants") const { cloneDeep } = require("lodash/fp") describe("test the link controller", () => { let config = new TestConfig(false) - let table1, table2 + let table1, table2, appId beforeEach(async () => { - await config.init() + const app = await config.init() + appId = app.appId const { _id } = await config.createTable() table2 = await config.createLinkedTable(RelationshipTypes.MANY_TO_MANY, ["link", "link2"]) // update table after creating link @@ -18,18 +20,20 @@ describe("test the link controller", () => { afterAll(config.end) - function createLinkController(table, row = null, oldTable = null) { - const linkConfig = { - tableId: table._id, - table, - } - if (row) { - linkConfig.row = row - } - if (oldTable) { - linkConfig.oldTable = oldTable - } - return new LinkController(linkConfig) + async function createLinkController(table, row = null, oldTable = null) { + return context.doInAppContext(appId, () => { + const linkConfig = { + tableId: table._id, + table, + } + if (row) { + linkConfig.row = row + } + if (oldTable) { + linkConfig.oldTable = oldTable + } + return new LinkController(linkConfig) + }) } async function createLinkedRow(linkField = "link", t1 = table1, t2 = table2) { @@ -38,16 +42,16 @@ describe("test the link controller", () => { return config.getRow(t1._id, _id) } - it("should be able to confirm if two table schemas are equal", () => { - const controller = createLinkController(table1) + it("should be able to confirm if two table schemas are equal", async () => { + const controller = await createLinkController(table1) let equal = controller.areLinkSchemasEqual(table2.schema.link, table2.schema.link) expect(equal).toEqual(true) equal = controller.areLinkSchemasEqual(table1.schema.link, table2.schema.link) expect(equal).toEqual(false) }) - it("should be able to check the relationship types across two fields", () => { - const controller = createLinkController(table1) + it("should be able to check the relationship types across two fields", async () => { + const controller = await createLinkController(table1) // empty case let output = controller.handleRelationshipType({}, {}) expect(output.linkedField.relationshipType).toEqual(RelationshipTypes.MANY_TO_MANY) @@ -65,29 +69,33 @@ describe("test the link controller", () => { it("should be able to delete a row", async () => { const row = await createLinkedRow() - const controller = createLinkController(table1, row) - // get initial count - const beforeLinks = await controller.getRowLinkDocs(row._id) - await controller.rowDeleted() - let afterLinks = await controller.getRowLinkDocs(row._id) - expect(beforeLinks.length).toEqual(1) - expect(afterLinks.length).toEqual(0) + const controller = await createLinkController(table1, row) + await context.doInAppContext(appId, async () => { + // get initial count + const beforeLinks = await controller.getRowLinkDocs(row._id) + await controller.rowDeleted() + let afterLinks = await controller.getRowLinkDocs(row._id) + expect(beforeLinks.length).toEqual(1) + expect(afterLinks.length).toEqual(0) + }) }) it("shouldn't throw an error when deleting a row with no links", async () => { const row = await config.createRow(basicRow(table1._id)) - const controller = createLinkController(table1, row) - let error - try { - await controller.rowDeleted() - } catch (err) { - error = err - } - expect(error).toBeUndefined() + const controller = await createLinkController(table1, row) + await context.doInAppContext(appId, async () => { + let error + try { + await controller.rowDeleted() + } catch (err) { + error = err + } + expect(error).toBeUndefined() + }) }) - it("should throw an error when validating a table which is invalid", () => { - const controller = createLinkController(table1) + it("should throw an error when validating a table which is invalid", async () => { + const controller = await createLinkController(table1) const copyTable = { ...table1 } @@ -110,32 +118,38 @@ describe("test the link controller", () => { const row = await createLinkedRow() // remove the link from the row row.link = [] - const controller = createLinkController(table1, row) - await controller.rowSaved() - let links = await controller.getRowLinkDocs(row._id) - expect(links.length).toEqual(0) + const controller = await createLinkController(table1, row) + await context.doInAppContext(appId, async () => { + await controller.rowSaved() + let links = await controller.getRowLinkDocs(row._id) + expect(links.length).toEqual(0) + }) }) it("should be able to delete a table and have links deleted", async () => { await createLinkedRow() - const controller = createLinkController(table1) - let before = await controller.getTableLinkDocs() - await controller.tableDeleted() - let after = await controller.getTableLinkDocs() - expect(before.length).toEqual(1) - expect(after.length).toEqual(0) + const controller = await createLinkController(table1) + await context.doInAppContext(appId, async () => { + let before = await controller.getTableLinkDocs() + await controller.tableDeleted() + let after = await controller.getTableLinkDocs() + expect(before.length).toEqual(1) + expect(after.length).toEqual(0) + }) }) it("should be able to remove a linked field from a table", async () => { await createLinkedRow() await createLinkedRow("link2") - const controller = createLinkController(table1, null, table1) - let before = await controller.getTableLinkDocs() - await controller.removeFieldFromTable("link") - let after = await controller.getTableLinkDocs() - expect(before.length).toEqual(2) - // shouldn't delete the other field - expect(after.length).toEqual(1) + const controller = await createLinkController(table1, null, table1) + await context.doInAppContext(appId, async () => { + let before = await controller.getTableLinkDocs() + await controller.removeFieldFromTable("link") + let after = await controller.getTableLinkDocs() + expect(before.length).toEqual(2) + // shouldn't delete the other field + expect(after.length).toEqual(1) + }) }) it("should throw an error when overwriting a link column", async () => { @@ -143,7 +157,7 @@ describe("test the link controller", () => { update.schema.link.relationshipType = RelationshipTypes.MANY_TO_ONE let error try { - const controller = createLinkController(update) + const controller = await createLinkController(update) await controller.tableSaved() } catch (err) { error = err @@ -156,10 +170,12 @@ describe("test the link controller", () => { await createLinkedRow() const newTable = cloneDeep(table1) delete newTable.schema.link - const controller = createLinkController(newTable, null, table1) - await controller.tableUpdated() - const links = await controller.getTableLinkDocs() - expect(links.length).toEqual(0) + const controller = await createLinkController(newTable, null, table1) + await context.doInAppContext(appId, async () => { + await controller.tableUpdated() + const links = await controller.getTableLinkDocs() + expect(links.length).toEqual(0) + }) }) it("shouldn't allow one to many having many relationships against it", async () => { diff --git a/packages/server/src/middleware/tests/currentapp.spec.js b/packages/server/src/middleware/tests/currentapp.spec.js index 16beb14f83..6534c8ef2d 100644 --- a/packages/server/src/middleware/tests/currentapp.spec.js +++ b/packages/server/src/middleware/tests/currentapp.spec.js @@ -40,6 +40,7 @@ function mockAuthWithNoCookie() { return { ...core, db: { + ...core.db, dbExists: () => true, }, cache: { @@ -68,6 +69,7 @@ function mockAuthWithCookie() { return { ...core, db: { + ...core.db, dbExists: () => true, }, utils: { @@ -183,6 +185,7 @@ describe("Current app middleware", () => { return { ...core, db: { + ...core.db, dbExists: () => true, }, utils: { @@ -213,6 +216,7 @@ describe("Current app middleware", () => { return { ...core, db: { + ...core.db, dbExists: () => true, }, utils: { From 535fab79973aee10998e3b6316ae600d87eebf7f Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 16 Nov 2022 17:23:12 +0000 Subject: [PATCH 15/24] Some major reworks towards higher levels of typescript. --- packages/backend-core/src/auth.ts | 8 +-- packages/backend-core/src/cloud/accounts.ts | 8 +-- packages/backend-core/src/constants.js | 44 -------------- packages/backend-core/src/constants.ts | 44 ++++++++++++++ packages/backend-core/src/context/index.ts | 9 +-- .../src/context/tests/index.spec.js | 3 +- .../backend-core/src/db/tests/utils.spec.js | 8 +-- packages/backend-core/src/db/utils.ts | 6 +- packages/backend-core/src/events/analytics.ts | 4 +- .../backend-core/src/events/identification.ts | 4 +- packages/backend-core/src/index.ts | 2 +- .../src/middleware/authenticated.ts | 22 ++++--- packages/backend-core/src/middleware/csrf.js | 4 +- .../src/middleware/internalApi.js | 4 +- .../middleware/passport/datasource/google.js | 8 +-- .../src/middleware/passport/google.js | 4 +- .../src/middleware/passport/jwt.js | 4 +- .../src/middleware/passport/oidc.js | 4 +- .../src/middleware/passport/utils.js | 4 +- .../backend-core/src/middleware/tenancy.js | 4 +- packages/backend-core/src/pkg/cache.ts | 2 + packages/backend-core/src/utils.js | 12 ++-- .../server/src/api/controllers/query/index.ts | 6 +- .../src/api/routes/tests/analytics.spec.js | 2 +- .../src/api/routes/tests/static.spec.js | 4 +- .../server/src/integrations/googlesheets.ts | 11 ++-- .../src/middleware/{appInfo.js => appInfo.ts} | 0 .../src/middleware/{builder.js => builder.ts} | 0 packages/server/src/middleware/currentapp.ts | 12 ++-- .../{joi-validator.js => joi-validator.ts} | 0 .../middleware/{publicApi.js => publicApi.ts} | 8 +-- .../{resourceId.js => resourceId.ts} | 0 .../middleware/{selfhost.js => selfhost.ts} | 0 .../src/middleware/{utils.js => utils.ts} | 0 .../src/tests/utilities/TestConfiguration.js | 18 +++--- .../server/src/utilities/workerRequests.js | 4 +- .../worker/src/api/controllers/global/auth.ts | 32 +++++----- .../src/api/controllers/global/configs.js | 34 +++++------ .../worker/src/api/controllers/global/self.ts | 4 +- .../worker/src/api/routes/global/configs.js | 18 +++--- .../api/routes/global/tests/configs.spec.ts | 58 +++++++++---------- packages/worker/src/constants/index.js | 4 +- .../worker/src/middleware/cloudRestricted.js | 2 +- .../worker/src/tests/TestConfiguration.ts | 22 +++---- .../worker/src/tests/structures/configs.js | 12 ++-- packages/worker/src/utilities/appService.js | 4 +- packages/worker/src/utilities/email.js | 4 +- packages/worker/src/utilities/templates.js | 4 +- 48 files changed, 236 insertions(+), 239 deletions(-) delete mode 100644 packages/backend-core/src/constants.js create mode 100644 packages/backend-core/src/constants.ts rename packages/server/src/middleware/{appInfo.js => appInfo.ts} (100%) rename packages/server/src/middleware/{builder.js => builder.ts} (100%) rename packages/server/src/middleware/{joi-validator.js => joi-validator.ts} (100%) rename packages/server/src/middleware/{publicApi.js => publicApi.ts} (56%) rename packages/server/src/middleware/{resourceId.js => resourceId.ts} (100%) rename packages/server/src/middleware/{selfhost.js => selfhost.ts} (100%) rename packages/server/src/middleware/{utils.js => utils.ts} (100%) diff --git a/packages/backend-core/src/auth.ts b/packages/backend-core/src/auth.ts index 98bf17beef..5e1959e0c8 100644 --- a/packages/backend-core/src/auth.ts +++ b/packages/backend-core/src/auth.ts @@ -3,7 +3,7 @@ const LocalStrategy = require("passport-local").Strategy const JwtStrategy = require("passport-jwt").Strategy import { getGlobalDB } from "./tenancy" const refresh = require("passport-oauth2-refresh") -import { Configs } from "./constants" +import { Config } from "./constants" import { getScopedConfig } from "./db/utils" import { jwt, @@ -76,7 +76,7 @@ async function refreshOIDCAccessToken( return new Promise(resolve => { refresh.requestNewAccessToken( - Configs.OIDC, + Config.OIDC, refreshToken, (err: any, accessToken: string, refreshToken: any, params: any) => { resolve({ err, accessToken, refreshToken, params }) @@ -106,7 +106,7 @@ async function refreshGoogleAccessToken( return new Promise(resolve => { refresh.requestNewAccessToken( - Configs.GOOGLE, + Config.GOOGLE, refreshToken, (err: any, accessToken: string, refreshToken: string, params: any) => { resolve({ err, accessToken, refreshToken, params }) @@ -129,7 +129,7 @@ async function refreshOAuthToken( let chosenConfig = {} let refreshResponse - if (configType === Configs.OIDC) { + if (configType === Config.OIDC) { // configId - retrieved from cookie. chosenConfig = config.configs.filter((c: any) => c.uuid === configId)[0] if (!chosenConfig) { diff --git a/packages/backend-core/src/cloud/accounts.ts b/packages/backend-core/src/cloud/accounts.ts index cca7469060..90fa7ab824 100644 --- a/packages/backend-core/src/cloud/accounts.ts +++ b/packages/backend-core/src/cloud/accounts.ts @@ -1,6 +1,6 @@ import API from "./api" import env from "../environment" -import { Headers } from "../constants" +import { Header } from "../constants" import { CloudAccount } from "@budibase/types" const api = new API(env.ACCOUNT_PORTAL_URL) @@ -14,7 +14,7 @@ export const getAccount = async ( const response = await api.post(`/api/accounts/search`, { body: payload, headers: { - [Headers.API_KEY]: env.ACCOUNT_PORTAL_API_KEY, + [Header.API_KEY]: env.ACCOUNT_PORTAL_API_KEY, }, }) @@ -35,7 +35,7 @@ export const getAccountByTenantId = async ( const response = await api.post(`/api/accounts/search`, { body: payload, headers: { - [Headers.API_KEY]: env.ACCOUNT_PORTAL_API_KEY, + [Header.API_KEY]: env.ACCOUNT_PORTAL_API_KEY, }, }) @@ -50,7 +50,7 @@ export const getAccountByTenantId = async ( export const getStatus = async () => { const response = await api.get(`/api/status`, { headers: { - [Headers.API_KEY]: env.ACCOUNT_PORTAL_API_KEY, + [Header.API_KEY]: env.ACCOUNT_PORTAL_API_KEY, }, }) const json = await response.json() diff --git a/packages/backend-core/src/constants.js b/packages/backend-core/src/constants.js deleted file mode 100644 index 44c271a4f8..0000000000 --- a/packages/backend-core/src/constants.js +++ /dev/null @@ -1,44 +0,0 @@ -exports.UserStatus = { - ACTIVE: "active", - INACTIVE: "inactive", -} - -exports.Cookies = { - CurrentApp: "budibase:currentapp", - Auth: "budibase:auth", - Init: "budibase:init", - ACCOUNT_RETURN_URL: "budibase:account:returnurl", - DatasourceAuth: "budibase:datasourceauth", - OIDC_CONFIG: "budibase:oidc:config", -} - -exports.Headers = { - API_KEY: "x-budibase-api-key", - LICENSE_KEY: "x-budibase-license-key", - API_VER: "x-budibase-api-version", - APP_ID: "x-budibase-app-id", - TYPE: "x-budibase-type", - PREVIEW_ROLE: "x-budibase-role", - TENANT_ID: "x-budibase-tenant-id", - TOKEN: "x-budibase-token", - CSRF_TOKEN: "x-csrf-token", -} - -exports.GlobalRoles = { - OWNER: "owner", - ADMIN: "admin", - BUILDER: "builder", - WORKSPACE_MANAGER: "workspace_manager", -} - -exports.Configs = { - SETTINGS: "settings", - ACCOUNT: "account", - SMTP: "smtp", - GOOGLE: "google", - OIDC: "oidc", - OIDC_LOGOS: "logos_oidc", -} - -exports.MAX_VALID_DATE = new Date(2147483647000) -exports.DEFAULT_TENANT_ID = "default" diff --git a/packages/backend-core/src/constants.ts b/packages/backend-core/src/constants.ts new file mode 100644 index 0000000000..61b3cea1f6 --- /dev/null +++ b/packages/backend-core/src/constants.ts @@ -0,0 +1,44 @@ +export enum UserStatus { + ACTIVE = "active", + INACTIVE = "inactive", +} + +export enum Cookie { + CurrentApp = "budibase:currentapp", + Auth = "budibase:auth", + Init = "budibase:init", + ACCOUNT_RETURN_URL = "budibase:account:returnurl", + DatasourceAuth = "budibase:datasourceauth", + OIDC_CONFIG = "budibase:oidc:config", +} + +export enum Header { + API_KEY = "x-budibase-api-key", + LICENSE_KEY = "x-budibase-license-key", + API_VER = "x-budibase-api-version", + APP_ID = "x-budibase-app-id", + TYPE = "x-budibase-type", + PREVIEW_ROLE = "x-budibase-role", + TENANT_ID = "x-budibase-tenant-id", + TOKEN = "x-budibase-token", + CSRF_TOKEN = "x-csrf-token", +} + +export enum GlobalRole { + OWNER = "owner", + ADMIN = "admin", + BUILDER = "builder", + WORKSPACE_MANAGER = "workspace_manager", +} + +export enum Config { + SETTINGS = "settings", + ACCOUNT = "account", + SMTP = "smtp", + GOOGLE = "google", + OIDC = "oidc", + OIDC_LOGOS = "logos_oidc", +} + +export const MAX_VALID_DATE = new Date(2147483647000) +export const DEFAULT_TENANT_ID = "default" diff --git a/packages/backend-core/src/context/index.ts b/packages/backend-core/src/context/index.ts index 15eed8d0ba..aabc5059d1 100644 --- a/packages/backend-core/src/context/index.ts +++ b/packages/backend-core/src/context/index.ts @@ -105,11 +105,12 @@ export async function doInAppContext(appId: string, task: any): Promise { } const tenantId = getTenantIDFromAppID(appId) + const updates: ContextMap = { appId } + if (tenantId) { + updates.tenantId = tenantId + } return newContext( - { - tenantId, - appId, - }, + updates, task ) } diff --git a/packages/backend-core/src/context/tests/index.spec.js b/packages/backend-core/src/context/tests/index.spec.js index 6d09446059..ea60806d21 100644 --- a/packages/backend-core/src/context/tests/index.spec.js +++ b/packages/backend-core/src/context/tests/index.spec.js @@ -1,8 +1,7 @@ -require("../../../tests/utilities/TestConfiguration") +require("../../../tests") const context = require("../") const { DEFAULT_TENANT_ID } = require("../../constants") const env = require("../../environment") -const dbCore = require("../../db") describe("context", () => { describe("doInTenant", () => { diff --git a/packages/backend-core/src/db/tests/utils.spec.js b/packages/backend-core/src/db/tests/utils.spec.js index ff6737627d..0d16e2dec2 100644 --- a/packages/backend-core/src/db/tests/utils.spec.js +++ b/packages/backend-core/src/db/tests/utils.spec.js @@ -9,7 +9,7 @@ const { getScopedConfig } = require("../utils") const tenancy = require("../../tenancy") -const { Configs, DEFAULT_TENANT_ID } = require("../../constants") +const { Config, DEFAULT_TENANT_ID } = require("../../constants") const env = require("../../environment") describe("utils", () => { @@ -77,7 +77,7 @@ const setDbPlatformUrl = async () => { const db = tenancy.getGlobalDB() db.put({ _id: "config_settings", - type: Configs.SETTINGS, + type: Config.SETTINGS, config: { platformUrl: DB_URL } @@ -178,7 +178,7 @@ describe("getScopedConfig", () => { await tenancy.doInTenant(DEFAULT_TENANT_ID, async () => { await setDbPlatformUrl() const db = tenancy.getGlobalDB() - const config = await getScopedConfig(db, { type: Configs.SETTINGS }) + const config = await getScopedConfig(db, { type: Config.SETTINGS }) expect(config.platformUrl).toBe(DB_URL) }) }) @@ -186,7 +186,7 @@ describe("getScopedConfig", () => { it("returns the platform url without an existing config", async () => { await tenancy.doInTenant(DEFAULT_TENANT_ID, async () => { const db = tenancy.getGlobalDB() - const config = await getScopedConfig(db, { type: Configs.SETTINGS }) + const config = await getScopedConfig(db, { type: Config.SETTINGS }) expect(config.platformUrl).toBe(DEFAULT_URL) }) }) diff --git a/packages/backend-core/src/db/utils.ts b/packages/backend-core/src/db/utils.ts index fc67a2c49b..61997901d7 100644 --- a/packages/backend-core/src/db/utils.ts +++ b/packages/backend-core/src/db/utils.ts @@ -1,5 +1,5 @@ import { newid } from "../hashing" -import { DEFAULT_TENANT_ID, Configs } from "../constants" +import { DEFAULT_TENANT_ID, Config } from "../constants" import env from "../environment" import { SEPARATOR, @@ -491,7 +491,7 @@ export const getScopedFullConfig = async function ( )[0] // custom logic for settings doc - if (type === Configs.SETTINGS) { + if (type === Config.SETTINGS) { if (scopedConfig && scopedConfig.doc) { // overrides affected by environment variables scopedConfig.doc.config.platformUrl = await getPlatformUrl({ @@ -530,7 +530,7 @@ export const getPlatformUrl = async (opts = { tenantAware: true }) => { // get the doc directly instead of with getScopedConfig to prevent loop let settings try { - settings = await db.get(generateConfigID({ type: Configs.SETTINGS })) + settings = await db.get(generateConfigID({ type: Config.SETTINGS })) } catch (e: any) { if (e.status !== 404) { throw e diff --git a/packages/backend-core/src/events/analytics.ts b/packages/backend-core/src/events/analytics.ts index 802b6d6314..eb121bdeec 100644 --- a/packages/backend-core/src/events/analytics.ts +++ b/packages/backend-core/src/events/analytics.ts @@ -1,7 +1,7 @@ import env from "../environment" import tenancy from "../tenancy" import * as dbUtils from "../db/utils" -import { Configs } from "../constants" +import { Config } from "../constants" import { withCache, TTL, CacheKeys } from "../cache/generic" export const enabled = async () => { @@ -46,7 +46,7 @@ const getSettingsDoc = async () => { let settings try { settings = await db.get( - dbUtils.generateConfigID({ type: Configs.SETTINGS }) + dbUtils.generateConfigID({ type: Config.SETTINGS }) ) } catch (e: any) { if (e.status !== 404) { diff --git a/packages/backend-core/src/events/identification.ts b/packages/backend-core/src/events/identification.ts index a29a6821cd..0b4b043837 100644 --- a/packages/backend-core/src/events/identification.ts +++ b/packages/backend-core/src/events/identification.ts @@ -19,7 +19,7 @@ import { } from "@budibase/types" import { processors } from "./processors" import * as dbUtils from "../db/utils" -import { Configs } from "../constants" +import { Config } from "../constants" import * as hashing from "../hashing" import * as installation from "../installation" import { withCache, TTL, CacheKeys } from "../cache/generic" @@ -273,7 +273,7 @@ const getUniqueTenantId = async (tenantId: string): Promise => { return withCache(CacheKeys.UNIQUE_TENANT_ID, TTL.ONE_DAY, async () => { const db = context.getGlobalDB() const config: SettingsConfig = await dbUtils.getScopedFullConfig(db, { - type: Configs.SETTINGS, + type: Config.SETTINGS, }) let uniqueTenantId: string diff --git a/packages/backend-core/src/index.ts b/packages/backend-core/src/index.ts index a2cfaaa9c6..24207a7a91 100644 --- a/packages/backend-core/src/index.ts +++ b/packages/backend-core/src/index.ts @@ -13,7 +13,7 @@ import featureFlags from "./featureFlags" import * as sessions from "./security/sessions" import * as deprovisioning from "./context/deprovision" import auth from "./auth" -import constants from "./constants" +import * as constants from "./constants" import * as dbConstants from "./db/constants" import * as logging from "./logging" import pino from "./pino" diff --git a/packages/backend-core/src/middleware/authenticated.ts b/packages/backend-core/src/middleware/authenticated.ts index d8f2b33399..8a1e52f414 100644 --- a/packages/backend-core/src/middleware/authenticated.ts +++ b/packages/backend-core/src/middleware/authenticated.ts @@ -1,11 +1,9 @@ -import { Cookies, Headers } from "../constants" +import { Cookie, Header } from "../constants" import { getCookie, clearCookie, openJwt } from "../utils" import { getUser } from "../cache/user" import { getSession, updateSessionTTL } from "../security/sessions" import { buildMatcherRegex, matches } from "./matchers" -import { SEPARATOR } from "../db/constants" -import { ViewName } from "../db/utils" -import { queryGlobalView } from "../db/views" +import { SEPARATOR, queryGlobalView, ViewName } from "../db" import { getGlobalDB, doInTenant } from "../tenancy" import { decrypt } from "../security/encryption" const identity = require("../context/identity") @@ -74,7 +72,7 @@ export = ( const noAuthOptions = noAuthPatterns ? buildMatcherRegex(noAuthPatterns) : [] return async (ctx: any, next: any) => { let publicEndpoint = false - const version = ctx.request.headers[Headers.API_VER] + const version = ctx.request.headers[Header.API_VER] // the path is not authenticated const found = matches(ctx, noAuthOptions) if (found) { @@ -82,10 +80,10 @@ export = ( } try { // check the actual user is authenticated first, try header or cookie - const headerToken = ctx.request.headers[Headers.TOKEN] - const authCookie = getCookie(ctx, Cookies.Auth) || openJwt(headerToken) - const apiKey = ctx.request.headers[Headers.API_KEY] - const tenantId = ctx.request.headers[Headers.TENANT_ID] + const headerToken = ctx.request.headers[Header.TOKEN] + const authCookie = getCookie(ctx, Cookie.Auth) || openJwt(headerToken) + const apiKey = ctx.request.headers[Header.API_KEY] + const tenantId = ctx.request.headers[Header.TENANT_ID] let authenticated = false, user = null, internal = false @@ -116,7 +114,7 @@ export = ( authenticated = false console.error("Auth Error", err?.message || err) // remove the cookie as the user does not exist anymore - clearCookie(ctx, Cookies.Auth) + clearCookie(ctx, Cookie.Auth) } } // this is an internal request, no user made it @@ -140,7 +138,7 @@ export = ( delete user.password } // be explicit - if (authenticated !== true) { + if (!authenticated) { authenticated = false } // isAuthenticated is a function, so use a variable to be able to check authed state @@ -155,7 +153,7 @@ export = ( console.error("Auth Error", err?.message || err) // invalid token, clear the cookie if (err && err.name === "JsonWebTokenError") { - clearCookie(ctx, Cookies.Auth) + clearCookie(ctx, Cookie.Auth) } // allow configuring for public access if ((opts && opts.publicAllowed) || publicEndpoint) { diff --git a/packages/backend-core/src/middleware/csrf.js b/packages/backend-core/src/middleware/csrf.js index 12bd9473e6..1557740cd6 100644 --- a/packages/backend-core/src/middleware/csrf.js +++ b/packages/backend-core/src/middleware/csrf.js @@ -1,4 +1,4 @@ -const { Headers } = require("../constants") +const { Header } = require("../constants") const { buildMatcherRegex, matches } = require("./matchers") /** @@ -68,7 +68,7 @@ module.exports = (opts = { noCsrfPatterns: [] }) => { } // reject if no token in request or mismatch - const requestToken = ctx.get(Headers.CSRF_TOKEN) + const requestToken = ctx.get(Header.CSRF_TOKEN) if (!requestToken || requestToken !== userToken) { ctx.throw(403, "Invalid CSRF token") } diff --git a/packages/backend-core/src/middleware/internalApi.js b/packages/backend-core/src/middleware/internalApi.js index 275d559a9e..05833842ce 100644 --- a/packages/backend-core/src/middleware/internalApi.js +++ b/packages/backend-core/src/middleware/internalApi.js @@ -1,11 +1,11 @@ const env = require("../environment") -const { Headers } = require("../constants") +const { Header } = require("../constants") /** * API Key only endpoint. */ module.exports = async (ctx, next) => { - const apiKey = ctx.request.headers[Headers.API_KEY] + const apiKey = ctx.request.headers[Header.API_KEY] if (apiKey !== env.INTERNAL_API_KEY) { ctx.throw(403, "Unauthorized") } diff --git a/packages/backend-core/src/middleware/passport/datasource/google.js b/packages/backend-core/src/middleware/passport/datasource/google.js index 8f2022c2d7..7cfd7f55f6 100644 --- a/packages/backend-core/src/middleware/passport/datasource/google.js +++ b/packages/backend-core/src/middleware/passport/datasource/google.js @@ -1,6 +1,6 @@ const google = require("../google") const GoogleStrategy = require("passport-google-oauth").OAuth2Strategy -const { Cookies, Configs } = require("../../../constants") +const { Cookie, Config } = require("../../../constants") const { clearCookie, getCookie } = require("../../../utils") const { getScopedConfig, getPlatformUrl } = require("../../../db/utils") const { doWithDB } = require("../../../db") @@ -11,7 +11,7 @@ async function fetchGoogleCreds() { // try and get the config from the tenant const db = getGlobalDB() const googleConfig = await getScopedConfig(db, { - type: Configs.GOOGLE, + type: Config.GOOGLE, }) // or fall back to env variables return ( @@ -47,7 +47,7 @@ async function postAuth(passport, ctx, next) { const platformUrl = await getPlatformUrl({ tenantAware: false }) let callbackUrl = `${platformUrl}/api/global/auth/datasource/google/callback` - const authStateCookie = getCookie(ctx, Cookies.DatasourceAuth) + const authStateCookie = getCookie(ctx, Cookie.DatasourceAuth) return passport.authenticate( new GoogleStrategy( @@ -57,7 +57,7 @@ async function postAuth(passport, ctx, next) { callbackURL: callbackUrl, }, (accessToken, refreshToken, profile, done) => { - clearCookie(ctx, Cookies.DatasourceAuth) + clearCookie(ctx, Cookie.DatasourceAuth) done(null, { accessToken, refreshToken }) } ), diff --git a/packages/backend-core/src/middleware/passport/google.js b/packages/backend-core/src/middleware/passport/google.js index 7419974cd7..7eb1215c1f 100644 --- a/packages/backend-core/src/middleware/passport/google.js +++ b/packages/backend-core/src/middleware/passport/google.js @@ -1,7 +1,7 @@ const GoogleStrategy = require("passport-google-oauth").OAuth2Strategy const { ssoCallbackUrl } = require("./utils") const { authenticateThirdParty } = require("./third-party-common") -const { Configs } = require("../../../constants") +const { Config } = require("../../../constants") const buildVerifyFn = saveUserFn => { return (accessToken, refreshToken, profile, done) => { @@ -60,7 +60,7 @@ exports.strategyFactory = async function (config, callbackUrl, saveUserFn) { } exports.getCallbackUrl = async function (db, config) { - return ssoCallbackUrl(db, config, Configs.GOOGLE) + return ssoCallbackUrl(db, config, Config.GOOGLE) } // expose for testing diff --git a/packages/backend-core/src/middleware/passport/jwt.js b/packages/backend-core/src/middleware/passport/jwt.js index 690c2ac8a1..36316264b0 100644 --- a/packages/backend-core/src/middleware/passport/jwt.js +++ b/packages/backend-core/src/middleware/passport/jwt.js @@ -1,11 +1,11 @@ -const { Cookies } = require("../../constants") +const { Cookie } = require("../../constants") const env = require("../../environment") const { authError } = require("./utils") exports.options = { secretOrKey: env.JWT_SECRET, jwtFromRequest: function (ctx) { - return ctx.cookies.get(Cookies.Auth) + return ctx.cookies.get(Cookie.Auth) }, } diff --git a/packages/backend-core/src/middleware/passport/oidc.js b/packages/backend-core/src/middleware/passport/oidc.js index 20dbd4669b..55a7033e40 100644 --- a/packages/backend-core/src/middleware/passport/oidc.js +++ b/packages/backend-core/src/middleware/passport/oidc.js @@ -2,7 +2,7 @@ const fetch = require("node-fetch") const OIDCStrategy = require("@techpass/passport-openidconnect").Strategy const { authenticateThirdParty } = require("./third-party-common") const { ssoCallbackUrl } = require("./utils") -const { Configs } = require("../../../constants") +const { Config } = require("../../../constants") const buildVerifyFn = saveUserFn => { /** @@ -140,7 +140,7 @@ exports.fetchStrategyConfig = async function (enrichedConfig, callbackUrl) { } exports.getCallbackUrl = async function (db, config) { - return ssoCallbackUrl(db, config, Configs.OIDC) + return ssoCallbackUrl(db, config, Config.OIDC) } // expose for testing diff --git a/packages/backend-core/src/middleware/passport/utils.js b/packages/backend-core/src/middleware/passport/utils.js index 217130cd6d..ab199b9f2f 100644 --- a/packages/backend-core/src/middleware/passport/utils.js +++ b/packages/backend-core/src/middleware/passport/utils.js @@ -1,6 +1,6 @@ const { isMultiTenant, getTenantId } = require("../../tenancy") const { getScopedConfig } = require("../../db/utils") -const { Configs } = require("../../constants") +const { Config } = require("../../constants") /** * Utility to handle authentication errors. @@ -24,7 +24,7 @@ exports.ssoCallbackUrl = async (db, config, type) => { return config.callbackURL } const publicConfig = await getScopedConfig(db, { - type: Configs.SETTINGS, + type: Config.SETTINGS, }) let callbackUrl = `/api/global/auth` diff --git a/packages/backend-core/src/middleware/tenancy.js b/packages/backend-core/src/middleware/tenancy.js index 8083322b29..b2a96cd5e2 100644 --- a/packages/backend-core/src/middleware/tenancy.js +++ b/packages/backend-core/src/middleware/tenancy.js @@ -1,6 +1,6 @@ const { doInTenant, isMultiTenant, DEFAULT_TENANT_ID } = require("../tenancy") const { buildMatcherRegex, matches } = require("./matchers") -const { Headers } = require("../constants") +const { Header } = require("../constants") const getTenantID = (ctx, opts = { allowQs: false, allowNoTenant: false }) => { // exit early if not multi-tenant @@ -11,7 +11,7 @@ const getTenantID = (ctx, opts = { allowQs: false, allowNoTenant: false }) => { let tenantId const allowQs = opts && opts.allowQs const allowNoTenant = opts && opts.allowNoTenant - const header = ctx.request.headers[Headers.TENANT_ID] + const header = ctx.request.headers[Header.TENANT_ID] const user = ctx.user || {} if (allowQs) { const query = ctx.request.query || {} diff --git a/packages/backend-core/src/pkg/cache.ts b/packages/backend-core/src/pkg/cache.ts index 1aaa40370d..c40a686260 100644 --- a/packages/backend-core/src/pkg/cache.ts +++ b/packages/backend-core/src/pkg/cache.ts @@ -3,9 +3,11 @@ import * as generic from "../cache/generic" import * as user from "../cache/user" import * as app from "../cache/appMetadata" +import * as writethrough from "../cache/writethrough" export = { app, user, + writethrough, ...generic, } diff --git a/packages/backend-core/src/utils.js b/packages/backend-core/src/utils.js index 6b59c7cb72..ed8f21d5c2 100644 --- a/packages/backend-core/src/utils.js +++ b/packages/backend-core/src/utils.js @@ -2,7 +2,7 @@ const { DocumentType, SEPARATOR, ViewName, getAllApps } = require("./db/utils") const jwt = require("jsonwebtoken") const { options } = require("./middleware/passport/jwt") const { queryGlobalView } = require("./db/views") -const { Headers, Cookies, MAX_VALID_DATE } = require("./constants") +const { Header, Cookie, MAX_VALID_DATE } = require("./constants") const env = require("./environment") const userCache = require("./cache/user") const { @@ -61,7 +61,7 @@ exports.isServingApp = ctx => { */ exports.getAppIdFromCtx = async ctx => { // look in headers - const options = [ctx.headers[Headers.APP_ID]] + const options = [ctx.headers[Header.APP_ID]] let appId for (let option of options) { appId = confirmAppId(option) @@ -157,7 +157,7 @@ exports.clearCookie = (ctx, name) => { * @return {boolean} returns true if the call is from the client lib (a built app rather than the builder). */ exports.isClient = ctx => { - return ctx.headers[Headers.TYPE] === "client" + return ctx.headers[Header.TYPE] === "client" } const getBuilders = async () => { @@ -187,7 +187,7 @@ exports.getBuildersCount = async () => { exports.platformLogout = async ({ ctx, userId, keepActiveSession }) => { if (!ctx) throw new Error("Koa context must be supplied to logout.") - const currentSession = exports.getCookie(ctx, Cookies.Auth) + const currentSession = exports.getCookie(ctx, Cookie.Auth) let sessions = await getSessionsForUser(userId) if (keepActiveSession) { @@ -196,8 +196,8 @@ exports.platformLogout = async ({ ctx, userId, keepActiveSession }) => { ) } else { // clear cookies - exports.clearCookie(ctx, Cookies.Auth) - exports.clearCookie(ctx, Cookies.CurrentApp) + exports.clearCookie(ctx, Cookie.Auth) + exports.clearCookie(ctx, Cookie.CurrentApp) } const sessionIds = sessions.map(({ sessionId }) => sessionId) diff --git a/packages/server/src/api/controllers/query/index.ts b/packages/server/src/api/controllers/query/index.ts index fa40320e91..eade7ddab6 100644 --- a/packages/server/src/api/controllers/query/index.ts +++ b/packages/server/src/api/controllers/query/index.ts @@ -112,13 +112,13 @@ export async function find(ctx: any) { //Required to discern between OIDC OAuth config entries function getOAuthConfigCookieId(ctx: any) { - if (ctx.user.providerType === constants.Configs.OIDC) { - return utils.getCookie(ctx, constants.Cookies.OIDC_CONFIG) + if (ctx.user.providerType === constants.Config.OIDC) { + return utils.getCookie(ctx, constants.Cookie.OIDC_CONFIG) } } function getAuthConfig(ctx: any) { - const authCookie = utils.getCookie(ctx, constants.Cookies.Auth) + const authCookie = utils.getCookie(ctx, constants.Cookie.Auth) let authConfigCtx: any = {} authConfigCtx["configId"] = getOAuthConfigCookieId(ctx) authConfigCtx["sessionId"] = authCookie ? authCookie.sessionId : null diff --git a/packages/server/src/api/routes/tests/analytics.spec.js b/packages/server/src/api/routes/tests/analytics.spec.js index 73d5810d7f..51196943e1 100644 --- a/packages/server/src/api/routes/tests/analytics.spec.js +++ b/packages/server/src/api/routes/tests/analytics.spec.js @@ -43,7 +43,7 @@ describe("/static", () => { it("should ping from app", async () => { const headers = config.defaultHeaders() - headers[constants.Headers.APP_ID] = config.prodAppId + headers[constants.Header.APP_ID] = config.prodAppId await request .post("/api/bbtel/ping") diff --git a/packages/server/src/api/routes/tests/static.spec.js b/packages/server/src/api/routes/tests/static.spec.js index 205f0beabe..dba8cd76f9 100644 --- a/packages/server/src/api/routes/tests/static.spec.js +++ b/packages/server/src/api/routes/tests/static.spec.js @@ -46,7 +46,7 @@ describe("/static", () => { it("should serve the app by id", async () => { const headers = config.defaultHeaders() - delete headers[constants.Headers.APP_ID] + delete headers[constants.Header.APP_ID] const res = await request .get(`/${config.prodAppId}`) @@ -58,7 +58,7 @@ describe("/static", () => { it("should serve the app by url", async () => { const headers = config.defaultHeaders() - delete headers[constants.Headers.APP_ID] + delete headers[constants.Header.APP_ID] const res = await request .get(`/app${config.prodApp.url}`) diff --git a/packages/server/src/integrations/googlesheets.ts b/packages/server/src/integrations/googlesheets.ts index 2e25d9683d..3e17df4076 100644 --- a/packages/server/src/integrations/googlesheets.ts +++ b/packages/server/src/integrations/googlesheets.ts @@ -12,10 +12,7 @@ import { buildExternalTableId } from "./utils" import { DataSourceOperation, FieldTypes } from "../constants" import { GoogleSpreadsheet } from "google-spreadsheet" import env from "../environment" - -const { getGlobalDB } = require("@budibase/backend-core/tenancy") -const { getScopedConfig } = require("@budibase/backend-core/db") -const { Configs } = require("@budibase/backend-core/constants") +import { tenancy, db as dbCore, constants } from "@budibase/backend-core" const fetch = require("node-fetch") interface GoogleSheetsConfig { @@ -176,9 +173,9 @@ class GoogleSheetsIntegration implements DatasourcePlus { async connect() { try { // Initialise oAuth client - const db = getGlobalDB() - let googleConfig = await getScopedConfig(db, { - type: Configs.GOOGLE, + const db = tenancy.getGlobalDB() + let googleConfig = await dbCore.getScopedConfig(db, { + type: constants.Config.GOOGLE, }) if (!googleConfig) { diff --git a/packages/server/src/middleware/appInfo.js b/packages/server/src/middleware/appInfo.ts similarity index 100% rename from packages/server/src/middleware/appInfo.js rename to packages/server/src/middleware/appInfo.ts diff --git a/packages/server/src/middleware/builder.js b/packages/server/src/middleware/builder.ts similarity index 100% rename from packages/server/src/middleware/builder.js rename to packages/server/src/middleware/builder.ts diff --git a/packages/server/src/middleware/currentapp.ts b/packages/server/src/middleware/currentapp.ts index 9496ddd0a0..3b50d62405 100644 --- a/packages/server/src/middleware/currentapp.ts +++ b/packages/server/src/middleware/currentapp.ts @@ -18,9 +18,9 @@ export = async (ctx: BBContext, next: any) => { // get app cookie if it exists let appCookie: { appId?: string } | undefined try { - appCookie = utils.getCookie(ctx, constants.Cookies.CurrentApp) + appCookie = utils.getCookie(ctx, constants.Cookie.CurrentApp) } catch (err) { - utils.clearCookie(ctx, constants.Cookies.CurrentApp) + utils.clearCookie(ctx, constants.Cookie.CurrentApp) } if (!appCookie && !requestAppId) { return next() @@ -30,7 +30,7 @@ export = async (ctx: BBContext, next: any) => { const appId = appCookie.appId const exists = await dbCore.dbExists(appId) if (!exists) { - utils.clearCookie(ctx, constants.Cookies.CurrentApp) + utils.clearCookie(ctx, constants.Cookie.CurrentApp) return next() } // if the request app ID wasn't set, update it with the cookie @@ -44,7 +44,7 @@ export = async (ctx: BBContext, next: any) => { !isWebhookEndpoint(ctx) && (!ctx.user || !ctx.user.builder || !ctx.user.builder.global) ) { - utils.clearCookie(ctx, constants.Cookies.CurrentApp) + utils.clearCookie(ctx, constants.Cookie.CurrentApp) return ctx.redirect("/") } } @@ -67,7 +67,7 @@ export = async (ctx: BBContext, next: any) => { const isDevApp = appId && isDevAppID(appId) const roleHeader = ctx.request && - (ctx.request.headers[constants.Headers.PREVIEW_ROLE] as string) + (ctx.request.headers[constants.Header.PREVIEW_ROLE] as string) if (isBuilder && isDevApp && roleHeader) { // Ensure the role is valid by ensuring a definition exists try { @@ -132,7 +132,7 @@ export = async (ctx: BBContext, next: any) => { appCookie.appId !== requestAppId) && !skipCookie ) { - utils.setCookie(ctx, { appId }, constants.Cookies.CurrentApp) + utils.setCookie(ctx, { appId }, constants.Cookie.CurrentApp) } return next() diff --git a/packages/server/src/middleware/joi-validator.js b/packages/server/src/middleware/joi-validator.ts similarity index 100% rename from packages/server/src/middleware/joi-validator.js rename to packages/server/src/middleware/joi-validator.ts diff --git a/packages/server/src/middleware/publicApi.js b/packages/server/src/middleware/publicApi.ts similarity index 56% rename from packages/server/src/middleware/publicApi.js rename to packages/server/src/middleware/publicApi.ts index 241873f5f9..76dbfe8214 100644 --- a/packages/server/src/middleware/publicApi.js +++ b/packages/server/src/middleware/publicApi.ts @@ -1,4 +1,4 @@ -const { Headers } = require("@budibase/backend-core/constants") +const { Header } = require("@budibase/backend-core/constants") const { getAppIdFromCtx } = require("@budibase/backend-core/utils") module.exports = function ({ requiresAppId } = {}) { @@ -7,13 +7,13 @@ module.exports = function ({ requiresAppId } = {}) { if (requiresAppId && !appId) { ctx.throw( 400, - `Invalid app ID provided, please check the ${Headers.APP_ID} header.` + `Invalid app ID provided, please check the ${Header.APP_ID} header.` ) } - if (!ctx.headers[Headers.API_KEY]) { + if (!ctx.headers[Header.API_KEY]) { ctx.throw( 400, - `Invalid API key provided, please check the ${Headers.API_KEY} header.` + `Invalid API key provided, please check the ${Header.API_KEY} header.` ) } return next() diff --git a/packages/server/src/middleware/resourceId.js b/packages/server/src/middleware/resourceId.ts similarity index 100% rename from packages/server/src/middleware/resourceId.js rename to packages/server/src/middleware/resourceId.ts diff --git a/packages/server/src/middleware/selfhost.js b/packages/server/src/middleware/selfhost.ts similarity index 100% rename from packages/server/src/middleware/selfhost.js rename to packages/server/src/middleware/selfhost.ts diff --git a/packages/server/src/middleware/utils.js b/packages/server/src/middleware/utils.ts similarity index 100% rename from packages/server/src/middleware/utils.js rename to packages/server/src/middleware/utils.ts diff --git a/packages/server/src/tests/utilities/TestConfiguration.js b/packages/server/src/tests/utilities/TestConfiguration.js index 1a5cf8ca85..40722513a0 100644 --- a/packages/server/src/tests/utilities/TestConfiguration.js +++ b/packages/server/src/tests/utilities/TestConfiguration.js @@ -16,7 +16,7 @@ const { const controllers = require("./controllers") const supertest = require("supertest") const { cleanup } = require("../../utilities/fileSystem") -const { Cookies, Headers } = require("@budibase/backend-core/constants") +const { Cookie, Headers } = require("@budibase/backend-core/constants") const { jwt } = require("@budibase/backend-core/auth") const { doInTenant, doWithGlobalDB } = require("@budibase/backend-core/tenancy") const { createASession } = require("@budibase/backend-core/sessions") @@ -256,10 +256,10 @@ class TestConfiguration { return { Accept: "application/json", Cookie: [ - `${Cookies.Auth}=${authToken}`, - `${Cookies.CurrentApp}=${appToken}`, + `${Cookie.Auth}=${authToken}`, + `${Cookie.CurrentApp}=${appToken}`, ], - [Headers.APP_ID]: appId, + [Header.APP_ID]: appId, } }) } @@ -279,14 +279,14 @@ class TestConfiguration { const headers = { Accept: "application/json", Cookie: [ - `${Cookies.Auth}=${authToken}`, - `${Cookies.CurrentApp}=${appToken}`, + `${Cookie.Auth}=${authToken}`, + `${Cookie.CurrentApp}=${appToken}`, ], - [Headers.CSRF_TOKEN]: CSRF_TOKEN, + [Header.CSRF_TOKEN]: CSRF_TOKEN, ...extras, } if (this.appId) { - headers[Headers.APP_ID] = this.appId + headers[Header.APP_ID] = this.appId } return headers } @@ -298,7 +298,7 @@ class TestConfiguration { Accept: "application/json", } if (appId) { - headers[Headers.APP_ID] = appId + headers[Header.APP_ID] = appId } return headers } diff --git a/packages/server/src/utilities/workerRequests.js b/packages/server/src/utilities/workerRequests.js index 53f13b6e02..89fb886033 100644 --- a/packages/server/src/utilities/workerRequests.js +++ b/packages/server/src/utilities/workerRequests.js @@ -11,9 +11,9 @@ function request(ctx, request) { request.headers = {} } if (!ctx) { - request.headers[Headers.API_KEY] = env.INTERNAL_API_KEY + request.headers[Header.API_KEY] = env.INTERNAL_API_KEY if (isTenantIdSet()) { - request.headers[Headers.TENANT_ID] = getTenantId() + request.headers[Header.TENANT_ID] = getTenantId() } } if (request.body && Object.keys(request.body).length > 0) { diff --git a/packages/worker/src/api/controllers/global/auth.ts b/packages/worker/src/api/controllers/global/auth.ts index c27fe17ee7..2e97ea6be8 100644 --- a/packages/worker/src/api/controllers/global/auth.ts +++ b/packages/worker/src/api/controllers/global/auth.ts @@ -1,8 +1,8 @@ const core = require("@budibase/backend-core") -const { Configs, EmailTemplatePurpose } = require("../../../constants") +const { Config, EmailTemplatePurpose } = require("../../../constants") const { sendEmail, isEmailConfigured } = require("../../../utilities/email") const { setCookie, getCookie, clearCookie, hash, platformLogout } = core.utils -const { Cookies, Headers } = core.constants +const { Cookie, Headers } = core.constants const { passport, ssoCallbackUrl, google, oidc } = core.auth const { checkResetPasswordCode } = require("../../../utilities/redis") const { getGlobalDB } = require("@budibase/backend-core/tenancy") @@ -30,13 +30,13 @@ async function authInternal(ctx: any, user: any, err = null, info = null) { } // set a cookie for browser access - setCookie(ctx, user.token, Cookies.Auth, { sign: false }) + setCookie(ctx, user.token, Cookie.Auth, { sign: false }) // set the token in a header as well for APIs - ctx.set(Headers.TOKEN, user.token) + ctx.set(Header.TOKEN, user.token) // get rid of any app cookies on login // have to check test because this breaks cypress if (!env.isTest()) { - clearCookie(ctx, Cookies.CurrentApp) + clearCookie(ctx, Cookie.CurrentApp) } } @@ -55,15 +55,15 @@ export const authenticate = async (ctx: any, next: any) => { export const setInitInfo = (ctx: any) => { const initInfo = ctx.request.body - setCookie(ctx, initInfo, Cookies.Init) + setCookie(ctx, initInfo, Cookie.Init) ctx.status = 200 } export const getInitInfo = (ctx: any) => { try { - ctx.body = getCookie(ctx, Cookies.Init) || {} + ctx.body = getCookie(ctx, Cookie.Init) || {} } catch (err) { - clearCookie(ctx, Cookies.Init) + clearCookie(ctx, Cookie.Init) ctx.body = {} } } @@ -141,14 +141,14 @@ export const datasourcePreAuth = async (ctx: any, next: any) => { appId: ctx.query.appId, datasourceId: ctx.query.datasourceId, }, - Cookies.DatasourceAuth + Cookie.DatasourceAuth ) return handler.preAuth(passport, ctx, next) } export const datasourceAuth = async (ctx: any, next: any) => { - const authStateCookie = getCookie(ctx, Cookies.DatasourceAuth) + const authStateCookie = getCookie(ctx, Cookie.DatasourceAuth) const provider = authStateCookie.provider const middleware = require(`@budibase/backend-core/middleware`) const handler = middleware.datasource[provider] @@ -163,7 +163,7 @@ export const googlePreAuth = async (ctx: any, next: any) => { const db = getGlobalDB() const config = await core.db.getScopedConfig(db, { - type: Configs.GOOGLE, + type: Config.GOOGLE, workspace: ctx.query.workspace, }) let callbackUrl = await exports.googleCallbackUrl(config) @@ -184,7 +184,7 @@ export const googleAuth = async (ctx: any, next: any) => { const db = getGlobalDB() const config = await core.db.getScopedConfig(db, { - type: Configs.GOOGLE, + type: Config.GOOGLE, workspace: ctx.query.workspace, }) const callbackUrl = await exports.googleCallbackUrl(config) @@ -210,7 +210,7 @@ export const googleAuth = async (ctx: any, next: any) => { export const oidcStrategyFactory = async (ctx: any, configId: any) => { const db = getGlobalDB() const config = await core.db.getScopedConfig(db, { - type: Configs.OIDC, + type: Config.OIDC, group: ctx.query.group, }) @@ -233,11 +233,11 @@ export const oidcPreAuth = async (ctx: any, next: any) => { const { configId } = ctx.params const strategy = await oidcStrategyFactory(ctx, configId) - setCookie(ctx, configId, Cookies.OIDC_CONFIG) + setCookie(ctx, configId, Cookie.OIDC_CONFIG) const db = getGlobalDB() const config = await core.db.getScopedConfig(db, { - type: Configs.OIDC, + type: Config.OIDC, group: ctx.query.group, }) @@ -255,7 +255,7 @@ export const oidcPreAuth = async (ctx: any, next: any) => { } export const oidcAuth = async (ctx: any, next: any) => { - const configId = getCookie(ctx, Cookies.OIDC_CONFIG) + const configId = getCookie(ctx, Cookie.OIDC_CONFIG) const strategy = await oidcStrategyFactory(ctx, configId) return passport.authenticate( diff --git a/packages/worker/src/api/controllers/global/configs.js b/packages/worker/src/api/controllers/global/configs.js index 5be6141c3f..cb8dc48c25 100644 --- a/packages/worker/src/api/controllers/global/configs.js +++ b/packages/worker/src/api/controllers/global/configs.js @@ -4,7 +4,7 @@ const { getScopedFullConfig, getAllApps, } = require("@budibase/backend-core/db") -const { Configs } = require("../../../constants") +const { Config } = require("../../../constants") const email = require("../../../utilities/email") const { upload, @@ -33,25 +33,25 @@ const getEventFns = async (db, config) => { if (!existing) { switch (config.type) { - case Configs.SMTP: { + case Config.SMTP: { fns.push(events.email.SMTPCreated) break } - case Configs.GOOGLE: { + case Config.GOOGLE: { fns.push(() => events.auth.SSOCreated(type)) if (config.config.activated) { fns.push(() => events.auth.SSOActivated(type)) } break } - case Configs.OIDC: { + case Config.OIDC: { fns.push(() => events.auth.SSOCreated(type)) if (config.config.configs[0].activated) { fns.push(() => events.auth.SSOActivated(type)) } break } - case Configs.SETTINGS: { + case Config.SETTINGS: { // company const company = config.config.company if (company && company !== "Budibase") { @@ -78,11 +78,11 @@ const getEventFns = async (db, config) => { } } else { switch (config.type) { - case Configs.SMTP: { + case Config.SMTP: { fns.push(events.email.SMTPUpdated) break } - case Configs.GOOGLE: { + case Config.GOOGLE: { fns.push(() => events.auth.SSOUpdated(type)) if (!existing.config.activated && config.config.activated) { fns.push(() => events.auth.SSOActivated(type)) @@ -91,7 +91,7 @@ const getEventFns = async (db, config) => { } break } - case Configs.OIDC: { + case Config.OIDC: { fns.push(() => events.auth.SSOUpdated(type)) if ( !existing.config.configs[0].activated && @@ -106,7 +106,7 @@ const getEventFns = async (db, config) => { } break } - case Configs.SETTINGS: { + case Config.SETTINGS: { // company const existingCompany = existing.config.company const company = config.config.company @@ -155,7 +155,7 @@ exports.save = async function (ctx) { try { // verify the configuration switch (type) { - case Configs.SMTP: + case Config.SMTP: await email.verifyConfig(config) break } @@ -237,7 +237,7 @@ exports.publicOidc = async function (ctx) { try { // Find the config with the most granular scope based on context const oidcConfig = await getScopedFullConfig(db, { - type: Configs.OIDC, + type: Config.OIDC, }) if (!oidcConfig) { @@ -260,15 +260,15 @@ exports.publicSettings = async function (ctx) { try { // Find the config with the most granular scope based on context const publicConfig = await getScopedFullConfig(db, { - type: Configs.SETTINGS, + type: Config.SETTINGS, }) const googleConfig = await getScopedFullConfig(db, { - type: Configs.GOOGLE, + type: Config.GOOGLE, }) const oidcConfig = await getScopedFullConfig(db, { - type: Configs.OIDC, + type: Config.OIDC, }) let config @@ -390,17 +390,17 @@ exports.configChecklist = async function (ctx) { // They have set up SMTP const smtpConfig = await getScopedFullConfig(db, { - type: Configs.SMTP, + type: Config.SMTP, }) // They have set up Google Auth const googleConfig = await getScopedFullConfig(db, { - type: Configs.GOOGLE, + type: Config.GOOGLE, }) // They have set up OIDC const oidcConfig = await getScopedFullConfig(db, { - type: Configs.OIDC, + type: Config.OIDC, }) // They have set up an global user diff --git a/packages/worker/src/api/controllers/global/self.ts b/packages/worker/src/api/controllers/global/self.ts index 685e2c8243..06906f1e8e 100644 --- a/packages/worker/src/api/controllers/global/self.ts +++ b/packages/worker/src/api/controllers/global/self.ts @@ -73,12 +73,12 @@ export async function fetchAPIKey(ctx: any) { } const checkCurrentApp = (ctx: any) => { - const appCookie = getCookie(ctx, constants.Cookies.CurrentApp) + const appCookie = getCookie(ctx, constants.Cookie.CurrentApp) if (appCookie && !tenancy.isUserInAppTenant(appCookie.appId)) { // there is a currentapp cookie from another tenant // remove the cookie as this is incompatible with the builder // due to builder and admin permissions being removed - clearCookie(ctx, constants.Cookies.CurrentApp) + clearCookie(ctx, constants.Cookie.CurrentApp) } } diff --git a/packages/worker/src/api/routes/global/configs.js b/packages/worker/src/api/routes/global/configs.js index a7cd1a38e8..bcbaa5caa2 100644 --- a/packages/worker/src/api/routes/global/configs.js +++ b/packages/worker/src/api/routes/global/configs.js @@ -3,7 +3,7 @@ const controller = require("../../controllers/global/configs") const { joiValidator } = require("@budibase/backend-core/auth") const { adminOnly } = require("@budibase/backend-core/auth") const Joi = require("joi") -const { Configs } = require("../../../constants") +const { Config } = require("../../../constants") const router = new Router() @@ -65,17 +65,17 @@ function buildConfigSaveValidation() { _id: Joi.string().optional(), _rev: Joi.string().optional(), workspace: Joi.string().optional(), - type: Joi.string().valid(...Object.values(Configs)).required(), + type: Joi.string().valid(...Object.values(Config)).required(), createdAt: Joi.string().optional(), updatedAt: Joi.string().optional(), config: Joi.alternatives() .conditional("type", { switch: [ - { is: Configs.SMTP, then: smtpValidation() }, - { is: Configs.SETTINGS, then: settingValidation() }, - { is: Configs.ACCOUNT, then: Joi.object().unknown(true) }, - { is: Configs.GOOGLE, then: googleValidation() }, - { is: Configs.OIDC, then: oidcValidation() } + { is: Config.SMTP, then: smtpValidation() }, + { is: Config.SETTINGS, then: settingValidation() }, + { is: Config.ACCOUNT, then: Joi.object().unknown(true) }, + { is: Config.GOOGLE, then: googleValidation() }, + { is: Config.OIDC, then: oidcValidation() } ], }), }).required().unknown(true), @@ -85,7 +85,7 @@ function buildConfigSaveValidation() { function buildUploadValidation() { // prettier-ignore return joiValidator.params(Joi.object({ - type: Joi.string().valid(...Object.values(Configs)).required(), + type: Joi.string().valid(...Object.values(Config)).required(), name: Joi.string().required(), }).required().unknown(true)) } @@ -93,7 +93,7 @@ function buildUploadValidation() { function buildConfigGetValidation() { // prettier-ignore return joiValidator.params(Joi.object({ - type: Joi.string().valid(...Object.values(Configs)).required() + type: Joi.string().valid(...Object.values(Config)).required() }).required().unknown(true)) } diff --git a/packages/worker/src/api/routes/global/tests/configs.spec.ts b/packages/worker/src/api/routes/global/tests/configs.spec.ts index 31510c03dd..82e80f4c90 100644 --- a/packages/worker/src/api/routes/global/tests/configs.spec.ts +++ b/packages/worker/src/api/routes/global/tests/configs.spec.ts @@ -2,7 +2,7 @@ jest.mock("nodemailer") import { TestConfiguration, structures, mocks, API } from "../../../../tests" mocks.email.mock() -import { Configs, events } from "@budibase/backend-core" +import { Config, events } from "@budibase/backend-core" describe("configs", () => { const config = new TestConfiguration() @@ -50,20 +50,20 @@ describe("configs", () => { it("should create activated google config", async () => { await saveGoogleConfig() expect(events.auth.SSOCreated).toBeCalledTimes(1) - expect(events.auth.SSOCreated).toBeCalledWith(Configs.GOOGLE) + expect(events.auth.SSOCreated).toBeCalledWith(Config.GOOGLE) expect(events.auth.SSODeactivated).not.toBeCalled() expect(events.auth.SSOActivated).toBeCalledTimes(1) - expect(events.auth.SSOActivated).toBeCalledWith(Configs.GOOGLE) - await config.deleteConfig(Configs.GOOGLE) + expect(events.auth.SSOActivated).toBeCalledWith(Config.GOOGLE) + await config.deleteConfig(Config.GOOGLE) }) it("should create deactivated google config", async () => { await saveGoogleConfig({ activated: false }) expect(events.auth.SSOCreated).toBeCalledTimes(1) - expect(events.auth.SSOCreated).toBeCalledWith(Configs.GOOGLE) + expect(events.auth.SSOCreated).toBeCalledWith(Config.GOOGLE) expect(events.auth.SSOActivated).not.toBeCalled() expect(events.auth.SSODeactivated).not.toBeCalled() - await config.deleteConfig(Configs.GOOGLE) + await config.deleteConfig(Config.GOOGLE) }) }) @@ -77,11 +77,11 @@ describe("configs", () => { googleConf._rev ) expect(events.auth.SSOUpdated).toBeCalledTimes(1) - expect(events.auth.SSOUpdated).toBeCalledWith(Configs.GOOGLE) + expect(events.auth.SSOUpdated).toBeCalledWith(Config.GOOGLE) expect(events.auth.SSOActivated).not.toBeCalled() expect(events.auth.SSODeactivated).toBeCalledTimes(1) - expect(events.auth.SSODeactivated).toBeCalledWith(Configs.GOOGLE) - await config.deleteConfig(Configs.GOOGLE) + expect(events.auth.SSODeactivated).toBeCalledWith(Config.GOOGLE) + await config.deleteConfig(Config.GOOGLE) }) it("should update google config to activated", async () => { @@ -93,11 +93,11 @@ describe("configs", () => { googleConf._rev ) expect(events.auth.SSOUpdated).toBeCalledTimes(1) - expect(events.auth.SSOUpdated).toBeCalledWith(Configs.GOOGLE) + expect(events.auth.SSOUpdated).toBeCalledWith(Config.GOOGLE) expect(events.auth.SSODeactivated).not.toBeCalled() expect(events.auth.SSOActivated).toBeCalledTimes(1) - expect(events.auth.SSOActivated).toBeCalledWith(Configs.GOOGLE) - await config.deleteConfig(Configs.GOOGLE) + expect(events.auth.SSOActivated).toBeCalledWith(Config.GOOGLE) + await config.deleteConfig(Config.GOOGLE) }) }) }) @@ -116,20 +116,20 @@ describe("configs", () => { it("should create activated OIDC config", async () => { await saveOIDCConfig() expect(events.auth.SSOCreated).toBeCalledTimes(1) - expect(events.auth.SSOCreated).toBeCalledWith(Configs.OIDC) + expect(events.auth.SSOCreated).toBeCalledWith(Config.OIDC) expect(events.auth.SSODeactivated).not.toBeCalled() expect(events.auth.SSOActivated).toBeCalledTimes(1) - expect(events.auth.SSOActivated).toBeCalledWith(Configs.OIDC) - await config.deleteConfig(Configs.OIDC) + expect(events.auth.SSOActivated).toBeCalledWith(Config.OIDC) + await config.deleteConfig(Config.OIDC) }) it("should create deactivated OIDC config", async () => { await saveOIDCConfig({ activated: false }) expect(events.auth.SSOCreated).toBeCalledTimes(1) - expect(events.auth.SSOCreated).toBeCalledWith(Configs.OIDC) + expect(events.auth.SSOCreated).toBeCalledWith(Config.OIDC) expect(events.auth.SSOActivated).not.toBeCalled() expect(events.auth.SSODeactivated).not.toBeCalled() - await config.deleteConfig(Configs.OIDC) + await config.deleteConfig(Config.OIDC) }) }) @@ -143,11 +143,11 @@ describe("configs", () => { oidcConf._rev ) expect(events.auth.SSOUpdated).toBeCalledTimes(1) - expect(events.auth.SSOUpdated).toBeCalledWith(Configs.OIDC) + expect(events.auth.SSOUpdated).toBeCalledWith(Config.OIDC) expect(events.auth.SSOActivated).not.toBeCalled() expect(events.auth.SSODeactivated).toBeCalledTimes(1) - expect(events.auth.SSODeactivated).toBeCalledWith(Configs.OIDC) - await config.deleteConfig(Configs.OIDC) + expect(events.auth.SSODeactivated).toBeCalledWith(Config.OIDC) + await config.deleteConfig(Config.OIDC) }) it("should update OIDC config to activated", async () => { @@ -159,11 +159,11 @@ describe("configs", () => { oidcConf._rev ) expect(events.auth.SSOUpdated).toBeCalledTimes(1) - expect(events.auth.SSOUpdated).toBeCalledWith(Configs.OIDC) + expect(events.auth.SSOUpdated).toBeCalledWith(Config.OIDC) expect(events.auth.SSODeactivated).not.toBeCalled() expect(events.auth.SSOActivated).toBeCalledTimes(1) - expect(events.auth.SSOActivated).toBeCalledWith(Configs.OIDC) - await config.deleteConfig(Configs.OIDC) + expect(events.auth.SSOActivated).toBeCalledWith(Config.OIDC) + await config.deleteConfig(Config.OIDC) }) }) }) @@ -180,11 +180,11 @@ describe("configs", () => { describe("create", () => { it("should create SMTP config", async () => { - await config.deleteConfig(Configs.SMTP) + await config.deleteConfig(Config.SMTP) await saveSMTPConfig() expect(events.email.SMTPUpdated).not.toBeCalled() expect(events.email.SMTPCreated).toBeCalledTimes(1) - await config.deleteConfig(Configs.SMTP) + await config.deleteConfig(Config.SMTP) }) }) @@ -195,7 +195,7 @@ describe("configs", () => { await saveSMTPConfig(smtpConf.config, smtpConf._id, smtpConf._rev) expect(events.email.SMTPCreated).not.toBeCalled() expect(events.email.SMTPUpdated).toBeCalledTimes(1) - await config.deleteConfig(Configs.SMTP) + await config.deleteConfig(Config.SMTP) }) }) }) @@ -212,7 +212,7 @@ describe("configs", () => { describe("create", () => { it("should create settings config with default settings", async () => { - await config.deleteConfig(Configs.SETTINGS) + await config.deleteConfig(Config.SETTINGS) await saveSettingsConfig() @@ -223,7 +223,7 @@ describe("configs", () => { it("should create settings config with non-default settings", async () => { config.modeSelf() - await config.deleteConfig(Configs.SETTINGS) + await config.deleteConfig(Config.SETTINGS) const conf = { company: "acme", logoUrl: "http://example.com", @@ -242,7 +242,7 @@ describe("configs", () => { describe("update", () => { it("should update settings config", async () => { config.modeSelf() - await config.deleteConfig(Configs.SETTINGS) + await config.deleteConfig(Config.SETTINGS) const settingsConfig = await saveSettingsConfig() settingsConfig.config.company = "acme" settingsConfig.config.logoUrl = "http://example.com" diff --git a/packages/worker/src/constants/index.js b/packages/worker/src/constants/index.js index 5affaaccb3..822864350e 100644 --- a/packages/worker/src/constants/index.js +++ b/packages/worker/src/constants/index.js @@ -1,4 +1,4 @@ -const { Configs } = require("@budibase/backend-core/constants") +const { Config } = require("@budibase/backend-core/constants") exports.LOGO_URL = "https://d33wubrfki0l68.cloudfront.net/aac32159d7207b5085e74a7ef67afbb7027786c5/2b1fd/img/logo/bb-emblem.svg" @@ -8,7 +8,7 @@ exports.UserStatus = { INACTIVE: "inactive", } -exports.Configs = Configs +exports.Config = Config exports.ConfigUploads = { LOGO: "logo", diff --git a/packages/worker/src/middleware/cloudRestricted.js b/packages/worker/src/middleware/cloudRestricted.js index b42ce4f3a4..29aa40662b 100644 --- a/packages/worker/src/middleware/cloudRestricted.js +++ b/packages/worker/src/middleware/cloudRestricted.js @@ -7,7 +7,7 @@ const { Headers } = require("@budibase/backend-core/constants") */ module.exports = async (ctx, next) => { if (!env.SELF_HOSTED && !env.DISABLE_ACCOUNT_PORTAL) { - const apiKey = ctx.request.headers[Headers.API_KEY] + const apiKey = ctx.request.headers[Header.API_KEY] if (apiKey !== env.INTERNAL_API_KEY) { ctx.throw(403, "Unauthorized") } diff --git a/packages/worker/src/tests/TestConfiguration.ts b/packages/worker/src/tests/TestConfiguration.ts index 1becb7d5c8..f52a852ebf 100644 --- a/packages/worker/src/tests/TestConfiguration.ts +++ b/packages/worker/src/tests/TestConfiguration.ts @@ -4,12 +4,12 @@ dbConfig.init() import env from "../environment" import controllers from "./controllers" const supertest = require("supertest") -import { Configs } from "../constants" +import { Config } from "../constants" import { users, tenancy, - Cookies, - Headers, + Cookie, + Header, sessions, auth, } from "@budibase/backend-core" @@ -163,8 +163,8 @@ class TestConfiguration { const authCookie = auth.jwt.sign(authToken, env.JWT_SECRET) return { Accept: "application/json", - ...this.cookieHeader([`${Cookies.Auth}=${authCookie}`]), - [Headers.CSRF_TOKEN]: CSRF_TOKEN, + ...this.cookieHeader([`${Cookie.Auth}=${authCookie}`]), + [Header.CSRF_TOKEN]: CSRF_TOKEN, } } @@ -223,7 +223,7 @@ class TestConfiguration { // CONFIGS - SETTINGS async saveSettingsConfig() { - await this.deleteConfig(Configs.SETTINGS) + await this.deleteConfig(Config.SETTINGS) await this._req( structures.configs.settings(), null, @@ -234,7 +234,7 @@ class TestConfiguration { // CONFIGS - GOOGLE async saveGoogleConfig() { - await this.deleteConfig(Configs.GOOGLE) + await this.deleteConfig(Config.GOOGLE) await this._req(structures.configs.google(), null, controllers.config.save) } @@ -242,11 +242,11 @@ class TestConfiguration { getOIDConfigCookie(configId: string) { const token = auth.jwt.sign(configId, env.JWT_SECRET) - return this.cookieHeader([[`${Cookies.OIDC_CONFIG}=${token}`]]) + return this.cookieHeader([[`${Cookie.OIDC_CONFIG}=${token}`]]) } async saveOIDCConfig() { - await this.deleteConfig(Configs.OIDC) + await this.deleteConfig(Config.OIDC) const config = structures.configs.oidc() await this._req(config, null, controllers.config.save) @@ -256,12 +256,12 @@ class TestConfiguration { // CONFIGS - SMTP async saveSmtpConfig() { - await this.deleteConfig(Configs.SMTP) + await this.deleteConfig(Config.SMTP) await this._req(structures.configs.smtp(), null, controllers.config.save) } async saveEtherealSmtpConfig() { - await this.deleteConfig(Configs.SMTP) + await this.deleteConfig(Config.SMTP) await this._req( structures.configs.smtpEthereal(), null, diff --git a/packages/worker/src/tests/structures/configs.js b/packages/worker/src/tests/structures/configs.js index 41ba283eaf..fbd82de0a7 100644 --- a/packages/worker/src/tests/structures/configs.js +++ b/packages/worker/src/tests/structures/configs.js @@ -1,9 +1,9 @@ -const { Configs } = require("../../constants") +const { Config } = require("../../constants") const { utils } = require("@budibase/backend-core") exports.oidc = conf => { return { - type: Configs.OIDC, + type: Config.OIDC, config: { configs: [ { @@ -23,7 +23,7 @@ exports.oidc = conf => { exports.google = conf => { return { - type: Configs.GOOGLE, + type: Config.GOOGLE, config: { clientID: "clientId", clientSecret: "clientSecret", @@ -35,7 +35,7 @@ exports.google = conf => { exports.smtp = conf => { return { - type: Configs.SMTP, + type: Config.SMTP, config: { port: 12345, host: "smtptesthost.com", @@ -49,7 +49,7 @@ exports.smtp = conf => { exports.smtpEthereal = () => { return { - type: Configs.SMTP, + type: Config.SMTP, config: { port: 587, host: "smtp.ethereal.email", @@ -65,7 +65,7 @@ exports.smtpEthereal = () => { exports.settings = conf => { return { - type: Configs.SETTINGS, + type: Config.SETTINGS, config: { platformUrl: "http://localhost:10000", logoUrl: "", diff --git a/packages/worker/src/utilities/appService.js b/packages/worker/src/utilities/appService.js index 36785198d7..acf0c359a0 100644 --- a/packages/worker/src/utilities/appService.js +++ b/packages/worker/src/utilities/appService.js @@ -9,9 +9,9 @@ async function makeAppRequest(url, method, body) { return } const request = { headers: {} } - request.headers[Headers.API_KEY] = env.INTERNAL_API_KEY + request.headers[Header.API_KEY] = env.INTERNAL_API_KEY if (isTenantIdSet()) { - request.headers[Headers.TENANT_ID] = getTenantId() + request.headers[Header.TENANT_ID] = getTenantId() } if (body) { request.headers["Content-Type"] = "application/json" diff --git a/packages/worker/src/utilities/email.js b/packages/worker/src/utilities/email.js index 66f78bb543..dbf5abea93 100644 --- a/packages/worker/src/utilities/email.js +++ b/packages/worker/src/utilities/email.js @@ -1,7 +1,7 @@ const nodemailer = require("nodemailer") const env = require("../environment") const { getScopedConfig } = require("@budibase/backend-core/db") -const { EmailTemplatePurpose, TemplateTypes, Configs } = require("../constants") +const { EmailTemplatePurpose, TemplateTypes, Config } = require("../constants") const { getTemplateByPurpose } = require("../constants/templates") const { getSettingsTemplateContext } = require("./templates") const { processString } = require("@budibase/string-templates") @@ -115,7 +115,7 @@ async function buildEmail(purpose, email, context, { user, contents } = {}) { */ async function getSmtpConfiguration(db, workspaceId = null, automation) { const params = { - type: Configs.SMTP, + type: Config.SMTP, } if (workspaceId) { params.workspace = workspaceId diff --git a/packages/worker/src/utilities/templates.js b/packages/worker/src/utilities/templates.js index 6e559dec65..935495d428 100644 --- a/packages/worker/src/utilities/templates.js +++ b/packages/worker/src/utilities/templates.js @@ -1,6 +1,6 @@ const { getScopedConfig } = require("@budibase/backend-core/db") const { - Configs, + Config, InternalTemplateBindings, LOGO_URL, EmailTemplatePurpose, @@ -15,7 +15,7 @@ const BASE_COMPANY = "Budibase" exports.getSettingsTemplateContext = async (purpose, code = null) => { const db = getGlobalDB() // TODO: use more granular settings in the future if required - let settings = (await getScopedConfig(db, { type: Configs.SETTINGS })) || {} + let settings = (await getScopedConfig(db, { type: Config.SETTINGS })) || {} const URL = settings.platformUrl const context = { [InternalTemplateBindings.LOGO_URL]: From 4328ed1b67c7b17d12223479aba6ee3c450d33c5 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 16 Nov 2022 17:24:13 +0000 Subject: [PATCH 16/24] Updating middlewares to Typescript, as well as some fixes based on running tests. --- packages/backend-core/src/context/index.ts | 5 +-- packages/backend-core/src/events/analytics.ts | 4 +-- .../api/routes/public/tests/compare.spec.js | 2 +- packages/server/src/middleware/appInfo.ts | 19 ++++++----- packages/server/src/middleware/builder.ts | 33 ++++++++++--------- .../server/src/middleware/joi-validator.ts | 17 +++++----- packages/server/src/middleware/publicApi.ts | 16 ++++----- packages/server/src/middleware/resourceId.ts | 27 ++++++++------- packages/server/src/middleware/selfhost.ts | 6 ++-- packages/server/src/middleware/utils.ts | 4 ++- .../src/tests/utilities/TestConfiguration.js | 2 +- .../server/src/utilities/workerRequests.js | 2 +- .../worker/src/api/controllers/global/auth.ts | 2 +- .../worker/src/middleware/cloudRestricted.js | 2 +- packages/worker/src/utilities/appService.js | 2 +- 15 files changed, 76 insertions(+), 67 deletions(-) diff --git a/packages/backend-core/src/context/index.ts b/packages/backend-core/src/context/index.ts index aabc5059d1..076d9af65a 100644 --- a/packages/backend-core/src/context/index.ts +++ b/packages/backend-core/src/context/index.ts @@ -109,10 +109,7 @@ export async function doInAppContext(appId: string, task: any): Promise { if (tenantId) { updates.tenantId = tenantId } - return newContext( - updates, - task - ) + return newContext(updates, task) } export async function doInIdentityContext( diff --git a/packages/backend-core/src/events/analytics.ts b/packages/backend-core/src/events/analytics.ts index eb121bdeec..228805ef82 100644 --- a/packages/backend-core/src/events/analytics.ts +++ b/packages/backend-core/src/events/analytics.ts @@ -45,9 +45,7 @@ const getSettingsDoc = async () => { const db = tenancy.getGlobalDB() let settings try { - settings = await db.get( - dbUtils.generateConfigID({ type: Config.SETTINGS }) - ) + settings = await db.get(dbUtils.generateConfigID({ type: Config.SETTINGS })) } catch (e: any) { if (e.status !== 404) { throw e diff --git a/packages/server/src/api/routes/public/tests/compare.spec.js b/packages/server/src/api/routes/public/tests/compare.spec.js index c42898fe40..860907b69d 100644 --- a/packages/server/src/api/routes/public/tests/compare.spec.js +++ b/packages/server/src/api/routes/public/tests/compare.spec.js @@ -31,7 +31,7 @@ async function makeRequest(method, endpoint, body, appId = config.getAppId()) { if (body) { req.send(body) } - const res = await req.expect("Content-Type", /json/).expect(200) + const res = await req expect(res.body).toBeDefined() return res } diff --git a/packages/server/src/middleware/appInfo.ts b/packages/server/src/middleware/appInfo.ts index ee3655c6cc..75e40ed473 100644 --- a/packages/server/src/middleware/appInfo.ts +++ b/packages/server/src/middleware/appInfo.ts @@ -1,19 +1,20 @@ -const { isDevAppID, isProdAppID } = require("../db/utils") +import { isDevAppID, isProdAppID } from "../db/utils" +import { BBContext } from "@budibase/types" -exports.AppType = { - DEV: "dev", - PROD: "prod", +export enum AppType { + DEV = "dev", + PROD = "prod", } -exports.middleware = - ({ appType } = {}) => - (ctx, next) => { +export function middleware({ appType }: { appType?: AppType } = {}) { + return (ctx: BBContext, next: any) => { const appId = ctx.appId - if (appType === exports.AppType.DEV && appId && !isDevAppID(appId)) { + if (appType === AppType.DEV && appId && !isDevAppID(appId)) { ctx.throw(400, "Only apps in development support this endpoint") } - if (appType === exports.AppType.PROD && appId && !isProdAppID(appId)) { + if (appType === AppType.PROD && appId && !isProdAppID(appId)) { ctx.throw(400, "Only apps in production support this endpoint") } return next() } +} diff --git a/packages/server/src/middleware/builder.ts b/packages/server/src/middleware/builder.ts index 12c4b93c75..403597ff1a 100644 --- a/packages/server/src/middleware/builder.ts +++ b/packages/server/src/middleware/builder.ts @@ -1,14 +1,17 @@ -const { APP_DEV_PREFIX } = require("../db/utils") -const { +import { + APP_DEV_PREFIX, + DocumentType, + getGlobalIDFromUserMetadataID, +} from "../db/utils" +import { doesUserHaveLock, updateLock, checkDebounce, setDebounce, -} = require("../utilities/redis") -const { doWithDB } = require("@budibase/backend-core/db") -const { DocumentType, getGlobalIDFromUserMetadataID } = require("../db/utils") -const { PermissionTypes } = require("@budibase/backend-core/permissions") -const { app: appCache } = require("@budibase/backend-core/cache") +} from "../utilities/redis" +import { db as dbCore, cache, permissions } from "@budibase/backend-core" +import { BBContext } from "@budibase/types" +import { PouchLike } from "@budibase/backend-core/src/db" const DEBOUNCE_TIME_SEC = 30 @@ -21,11 +24,11 @@ const DEBOUNCE_TIME_SEC = 30 * through the authorized middleware * ****************************************************/ -async function checkDevAppLocks(ctx) { +async function checkDevAppLocks(ctx: BBContext) { const appId = ctx.appId // if any public usage, don't proceed - if (!ctx.user._id && !ctx.user.userId) { + if (!ctx.user?._id && !ctx.user?.userId) { return } @@ -41,34 +44,34 @@ async function checkDevAppLocks(ctx) { await updateLock(appId, ctx.user) } -async function updateAppUpdatedAt(ctx) { +async function updateAppUpdatedAt(ctx: BBContext) { const appId = ctx.appId // if debouncing skip this update // get methods also aren't updating if (ctx.method === "GET" || (await checkDebounce(appId))) { return } - await doWithDB(appId, async db => { + await dbCore.doWithDB(appId, async (db: PouchLike) => { const metadata = await db.get(DocumentType.APP_METADATA) metadata.updatedAt = new Date().toISOString() - metadata.updatedBy = getGlobalIDFromUserMetadataID(ctx.user.userId) + metadata.updatedBy = getGlobalIDFromUserMetadataID(ctx.user?.userId!) const response = await db.put(metadata) metadata._rev = response.rev - await appCache.invalidateAppMetadata(appId, metadata) + await cache.app.invalidateAppMetadata(appId, metadata) // set a new debounce record with a short TTL await setDebounce(appId, DEBOUNCE_TIME_SEC) }) } -module.exports = async (ctx, permType) => { +export = async function builder(ctx: BBContext, permType: string) { const appId = ctx.appId // this only functions within an app context if (!appId) { return } - const isBuilderApi = permType === PermissionTypes.BUILDER + const isBuilderApi = permType === permissions.PermissionTypes.BUILDER const referer = ctx.headers["referer"] const overviewPath = "/builder/portal/overview/" diff --git a/packages/server/src/middleware/joi-validator.ts b/packages/server/src/middleware/joi-validator.ts index 6812dbdd54..5d783acc80 100644 --- a/packages/server/src/middleware/joi-validator.ts +++ b/packages/server/src/middleware/joi-validator.ts @@ -1,20 +1,21 @@ -const Joi = require("joi") +import Joi from "joi" +import { BBContext } from "@budibase/types" -function validate(schema, property) { +function validate(schema: Joi.Schema, property: string) { // Return a Koa middleware function - return (ctx, next) => { + return (ctx: BBContext, next: any) => { if (!schema) { return next() } let params = null if (ctx[property] != null) { params = ctx[property] - } else if (ctx.request[property] != null) { - params = ctx.request[property] + } else if (ctx.request.get(property) != null) { + params = ctx.request.get(property) } // not all schemas have the append property e.g. array schemas - if (schema.append) { + if ("append" in schema && schema.append) { schema = schema.append({ createdAt: Joi.any().optional(), updatedAt: Joi.any().optional(), @@ -30,10 +31,10 @@ function validate(schema, property) { } } -module.exports.body = schema => { +export function body(schema: Joi.Schema) { return validate(schema, "body") } -module.exports.params = schema => { +export function params(schema: Joi.Schema) { return validate(schema, "params") } diff --git a/packages/server/src/middleware/publicApi.ts b/packages/server/src/middleware/publicApi.ts index 76dbfe8214..9b9a34df46 100644 --- a/packages/server/src/middleware/publicApi.ts +++ b/packages/server/src/middleware/publicApi.ts @@ -1,19 +1,19 @@ -const { Header } = require("@budibase/backend-core/constants") -const { getAppIdFromCtx } = require("@budibase/backend-core/utils") +import { constants, utils } from "@budibase/backend-core" +import { BBContext } from "@budibase/types" -module.exports = function ({ requiresAppId } = {}) { - return async (ctx, next) => { - const appId = await getAppIdFromCtx(ctx) +export = function ({ requiresAppId }: { requiresAppId?: boolean } = {}) { + return async (ctx: BBContext, next: any) => { + const appId = await utils.getAppIdFromCtx(ctx) if (requiresAppId && !appId) { ctx.throw( 400, - `Invalid app ID provided, please check the ${Header.APP_ID} header.` + `Invalid app ID provided, please check the ${constants.Header.APP_ID} header.` ) } - if (!ctx.headers[Header.API_KEY]) { + if (!ctx.headers[constants.Header.API_KEY]) { ctx.throw( 400, - `Invalid API key provided, please check the ${Header.API_KEY} header.` + `Invalid API key provided, please check the ${constants.Header.API_KEY} header.` ) } return next() diff --git a/packages/server/src/middleware/resourceId.ts b/packages/server/src/middleware/resourceId.ts index 4216131119..0917941061 100644 --- a/packages/server/src/middleware/resourceId.ts +++ b/packages/server/src/middleware/resourceId.ts @@ -1,17 +1,23 @@ -class ResourceIdGetter { - constructor(ctxProperty) { +import { BBContext } from "@budibase/types" + +export class ResourceIdGetter { + parameter: string + main: string | null + sub: string | null + + constructor(ctxProperty: string) { this.parameter = ctxProperty this.main = null this.sub = null return this } - mainResource(field) { + mainResource(field: string) { this.main = field return this } - subResource(field) { + subResource(field: string) { this.sub = field return this } @@ -20,7 +26,8 @@ class ResourceIdGetter { const parameter = this.parameter, main = this.main, sub = this.sub - return (ctx, next) => { + return (ctx: BBContext, next: any) => { + // @ts-ignore const request = ctx.request[parameter] || ctx[parameter] if (request == null) { return next() @@ -36,24 +43,22 @@ class ResourceIdGetter { } } -module.exports.ResourceIdGetter = ResourceIdGetter - -module.exports.paramResource = main => { +export function paramResource(main: string) { return new ResourceIdGetter("params").mainResource(main).build() } -module.exports.paramSubResource = (main, sub) => { +export function paramSubResource(main: string, sub: string) { return new ResourceIdGetter("params") .mainResource(main) .subResource(sub) .build() } -module.exports.bodyResource = main => { +export function bodyResource(main: string) { return new ResourceIdGetter("body").mainResource(main).build() } -module.exports.bodySubResource = (main, sub) => { +export function bodySubResource(main: string, sub: string) { return new ResourceIdGetter("body") .mainResource(main) .subResource(sub) diff --git a/packages/server/src/middleware/selfhost.ts b/packages/server/src/middleware/selfhost.ts index 1e7117c83d..180d80c8ea 100644 --- a/packages/server/src/middleware/selfhost.ts +++ b/packages/server/src/middleware/selfhost.ts @@ -1,7 +1,9 @@ -const env = require("../environment") +import env from "../environment" +import { BBContext } from "@budibase/types" + // if added as a middleware will stop requests unless builder is in self host mode // or cloud is in self host -module.exports = async (ctx, next) => { +module.exports = async (ctx: BBContext, next: any) => { if (env.SELF_HOSTED) { await next() return diff --git a/packages/server/src/middleware/utils.ts b/packages/server/src/middleware/utils.ts index b1eea8cd66..714df12f38 100644 --- a/packages/server/src/middleware/utils.ts +++ b/packages/server/src/middleware/utils.ts @@ -1,7 +1,9 @@ +import { BBContext } from "@budibase/types" + const WEBHOOK_ENDPOINTS = new RegExp( ["webhooks/trigger", "webhooks/schema"].join("|") ) -exports.isWebhookEndpoint = ctx => { +export function isWebhookEndpoint(ctx: BBContext) { return WEBHOOK_ENDPOINTS.test(ctx.request.url) } diff --git a/packages/server/src/tests/utilities/TestConfiguration.js b/packages/server/src/tests/utilities/TestConfiguration.js index 40722513a0..044ad4bbf7 100644 --- a/packages/server/src/tests/utilities/TestConfiguration.js +++ b/packages/server/src/tests/utilities/TestConfiguration.js @@ -16,7 +16,7 @@ const { const controllers = require("./controllers") const supertest = require("supertest") const { cleanup } = require("../../utilities/fileSystem") -const { Cookie, Headers } = require("@budibase/backend-core/constants") +const { Cookie, Header } = require("@budibase/backend-core/constants") const { jwt } = require("@budibase/backend-core/auth") const { doInTenant, doWithGlobalDB } = require("@budibase/backend-core/tenancy") const { createASession } = require("@budibase/backend-core/sessions") diff --git a/packages/server/src/utilities/workerRequests.js b/packages/server/src/utilities/workerRequests.js index 89fb886033..e606ba9fa6 100644 --- a/packages/server/src/utilities/workerRequests.js +++ b/packages/server/src/utilities/workerRequests.js @@ -3,7 +3,7 @@ const env = require("../environment") const { checkSlashesInUrl } = require("./index") const { getProdAppID } = require("@budibase/backend-core/db") const { updateAppRole } = require("./global") -const { Headers } = require("@budibase/backend-core/constants") +const { Header } = require("@budibase/backend-core/constants") const { getTenantId, isTenantIdSet } = require("@budibase/backend-core/tenancy") function request(ctx, request) { diff --git a/packages/worker/src/api/controllers/global/auth.ts b/packages/worker/src/api/controllers/global/auth.ts index 2e97ea6be8..3df77f024d 100644 --- a/packages/worker/src/api/controllers/global/auth.ts +++ b/packages/worker/src/api/controllers/global/auth.ts @@ -2,7 +2,7 @@ const core = require("@budibase/backend-core") const { Config, EmailTemplatePurpose } = require("../../../constants") const { sendEmail, isEmailConfigured } = require("../../../utilities/email") const { setCookie, getCookie, clearCookie, hash, platformLogout } = core.utils -const { Cookie, Headers } = core.constants +const { Cookie, Header } = core.constants const { passport, ssoCallbackUrl, google, oidc } = core.auth const { checkResetPasswordCode } = require("../../../utilities/redis") const { getGlobalDB } = require("@budibase/backend-core/tenancy") diff --git a/packages/worker/src/middleware/cloudRestricted.js b/packages/worker/src/middleware/cloudRestricted.js index 29aa40662b..5b9d64b92f 100644 --- a/packages/worker/src/middleware/cloudRestricted.js +++ b/packages/worker/src/middleware/cloudRestricted.js @@ -1,5 +1,5 @@ const env = require("../environment") -const { Headers } = require("@budibase/backend-core/constants") +const { Header } = require("@budibase/backend-core/constants") /** * This is a restricted endpoint in the cloud. diff --git a/packages/worker/src/utilities/appService.js b/packages/worker/src/utilities/appService.js index acf0c359a0..ae895b831c 100644 --- a/packages/worker/src/utilities/appService.js +++ b/packages/worker/src/utilities/appService.js @@ -1,5 +1,5 @@ const fetch = require("node-fetch") -const { Headers } = require("@budibase/backend-core/constants") +const { Header } = require("@budibase/backend-core/constants") const { getTenantId, isTenantIdSet } = require("@budibase/backend-core/tenancy") const { checkSlashesInUrl } = require("../utilities") const env = require("../environment") From cdc25d703236028513440b772f2f6775a1af4c41 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 16 Nov 2022 18:13:34 +0000 Subject: [PATCH 17/24] Some worker typescript conversions. --- packages/worker/src/constants/index.ts | 121 +++++++++--------- .../worker/src/constants/templates/index.js | 4 +- packages/worker/src/utilities/email.ts | 121 +++++++++++------- packages/worker/src/utilities/redis.ts | 53 ++++---- packages/worker/src/utilities/templates.ts | 54 ++++---- 5 files changed, 192 insertions(+), 161 deletions(-) diff --git a/packages/worker/src/constants/index.ts b/packages/worker/src/constants/index.ts index 822864350e..4ed2c99714 100644 --- a/packages/worker/src/constants/index.ts +++ b/packages/worker/src/constants/index.ts @@ -1,97 +1,97 @@ -const { Config } = require("@budibase/backend-core/constants") +import { constants } from "@budibase/backend-core" -exports.LOGO_URL = +export const LOGO_URL = "https://d33wubrfki0l68.cloudfront.net/aac32159d7207b5085e74a7ef67afbb7027786c5/2b1fd/img/logo/bb-emblem.svg" -exports.UserStatus = { - ACTIVE: "active", - INACTIVE: "inactive", +export enum UserStatus { + ACTIVE = "active", + INACTIVE = "inactive", } -exports.Config = Config +export const Config = constants.Config -exports.ConfigUploads = { - LOGO: "logo", - OIDC_LOGO: "oidc_logo", +export enum ConfigUpload { + LOGO = "logo", + OIDC_LOGO = "oidc_logo", } -const TemplateTypes = { - EMAIL: "email", +export enum TemplateType { + EMAIL = "email", } -const EmailTemplatePurpose = { - BASE: "base", - PASSWORD_RECOVERY: "password_recovery", - INVITATION: "invitation", - WELCOME: "welcome", - CUSTOM: "custom", +export enum EmailTemplatePurpose { + BASE = "base", + PASSWORD_RECOVERY = "password_recovery", + INVITATION = "invitation", + WELCOME = "welcome", + CUSTOM = "custom", } -const InternalTemplateBindings = { - PLATFORM_URL: "platformUrl", - COMPANY: "company", - LOGO_URL: "logoUrl", - EMAIL: "email", - USER: "user", - REQUEST: "request", - DOCS_URL: "docsUrl", - LOGIN_URL: "loginUrl", - CURRENT_YEAR: "currentYear", - CURRENT_DATE: "currentDate", - BODY: "body", - STYLES: "styles", - RESET_URL: "resetUrl", - RESET_CODE: "resetCode", - INVITE_URL: "inviteUrl", - INVITE_CODE: "inviteUrl", - CONTENTS: "contents", +export enum InternalTemplateBinding { + PLATFORM_URL = "platformUrl", + COMPANY = "company", + LOGO_URL = "logoUrl", + EMAIL = "email", + USER = "user", + REQUEST = "request", + DOCS_URL = "docsUrl", + LOGIN_URL = "loginUrl", + CURRENT_YEAR = "currentYear", + CURRENT_DATE = "currentDate", + BODY = "body", + STYLES = "styles", + RESET_URL = "resetUrl", + RESET_CODE = "resetCode", + INVITE_URL = "inviteUrl", + INVITE_CODE = "inviteUrl", + CONTENTS = "contents", } -const TemplateBindings = { +export const TemplateBindings = { PLATFORM_URL: { - name: InternalTemplateBindings.PLATFORM_URL, + name: InternalTemplateBinding.PLATFORM_URL, description: "The URL used to access the budibase platform", }, COMPANY: { - name: InternalTemplateBindings.COMPANY, + name: InternalTemplateBinding.COMPANY, description: "The name of your organization", }, LOGO_URL: { - name: InternalTemplateBindings.LOGO_URL, + name: InternalTemplateBinding.LOGO_URL, description: "The URL of your organizations logo.", }, EMAIL: { - name: InternalTemplateBindings.EMAIL, + name: InternalTemplateBinding.EMAIL, description: "The recipients email address.", }, USER: { - name: InternalTemplateBindings.USER, + name: InternalTemplateBinding.USER, description: "The recipients user object.", }, REQUEST: { - name: InternalTemplateBindings.REQUEST, + name: InternalTemplateBinding.REQUEST, description: "Additional request metadata.", }, DOCS_URL: { - name: InternalTemplateBindings.DOCS_URL, + name: InternalTemplateBinding.DOCS_URL, description: "Organization documentation URL.", }, LOGIN_URL: { - name: InternalTemplateBindings.LOGIN_URL, + name: InternalTemplateBinding.LOGIN_URL, description: "The URL used to log into the organization budibase instance.", }, CURRENT_YEAR: { - name: InternalTemplateBindings.CURRENT_YEAR, + name: InternalTemplateBinding.CURRENT_YEAR, description: "The current year.", }, CURRENT_DATE: { - name: InternalTemplateBindings.CURRENT_DATE, + name: InternalTemplateBinding.CURRENT_DATE, description: "The current date.", }, } -const TemplateMetadata = { - [TemplateTypes.EMAIL]: [ +export const TemplateMetadata = { + [TemplateType.EMAIL]: [ { name: "Base format", description: @@ -100,11 +100,11 @@ const TemplateMetadata = { purpose: EmailTemplatePurpose.BASE, bindings: [ { - name: InternalTemplateBindings.BODY, + name: InternalTemplateBinding.BODY, description: "The main body of another email template.", }, { - name: InternalTemplateBindings.STYLES, + name: InternalTemplateBinding.STYLES, description: "The contents of the Styling email template.", }, ], @@ -117,12 +117,12 @@ const TemplateMetadata = { purpose: EmailTemplatePurpose.PASSWORD_RECOVERY, bindings: [ { - name: InternalTemplateBindings.RESET_URL, + name: InternalTemplateBinding.RESET_URL, description: "The URL the recipient must click to reset their password.", }, { - name: InternalTemplateBindings.RESET_CODE, + name: InternalTemplateBinding.RESET_CODE, description: "The temporary password reset code used in the recipients password reset URL.", }, @@ -144,12 +144,12 @@ const TemplateMetadata = { purpose: EmailTemplatePurpose.INVITATION, bindings: [ { - name: InternalTemplateBindings.INVITE_URL, + name: InternalTemplateBinding.INVITE_URL, description: "The URL the recipient must click to accept the invitation and activate their account.", }, { - name: InternalTemplateBindings.INVITE_CODE, + name: InternalTemplateBinding.INVITE_CODE, description: "The temporary invite code used in the recipients invitation URL.", }, @@ -163,7 +163,7 @@ const TemplateMetadata = { purpose: EmailTemplatePurpose.CUSTOM, bindings: [ { - name: InternalTemplateBindings.CONTENTS, + name: InternalTemplateBinding.CONTENTS, description: "Custom content body.", }, ], @@ -172,12 +172,5 @@ const TemplateMetadata = { } // all purpose combined -exports.TemplatePurpose = { - ...EmailTemplatePurpose, -} -exports.TemplateTypes = TemplateTypes -exports.EmailTemplatePurpose = EmailTemplatePurpose -exports.TemplateMetadata = TemplateMetadata -exports.TemplateBindings = TemplateBindings -exports.InternalTemplateBindings = InternalTemplateBindings -exports.GLOBAL_OWNER = "global" +export { EmailTemplatePurpose as TemplatePurpose } +export const GLOBAL_OWNER = "global" diff --git a/packages/worker/src/constants/templates/index.js b/packages/worker/src/constants/templates/index.js index cfca00f471..0631df7011 100644 --- a/packages/worker/src/constants/templates/index.js +++ b/packages/worker/src/constants/templates/index.js @@ -1,7 +1,7 @@ const { readStaticFile } = require("../../utilities/fileSystem") const { EmailTemplatePurpose, - TemplateTypes, + TemplateType, TemplatePurpose, GLOBAL_OWNER, } = require("../index") @@ -26,7 +26,7 @@ exports.EmailTemplates = { exports.addBaseTemplates = (templates, type = null) => { let purposeList switch (type) { - case TemplateTypes.EMAIL: + case TemplateType.EMAIL: purposeList = Object.values(EmailTemplatePurpose) break default: diff --git a/packages/worker/src/utilities/email.ts b/packages/worker/src/utilities/email.ts index dbf5abea93..c07604827d 100644 --- a/packages/worker/src/utilities/email.ts +++ b/packages/worker/src/utilities/email.ts @@ -1,15 +1,33 @@ +import env from "../environment" +import { EmailTemplatePurpose, TemplateType, Config } from "../constants" +import { getTemplateByPurpose } from "../constants/templates" +import { getSettingsTemplateContext } from "./templates" +import { processString } from "@budibase/string-templates" +import { getResetPasswordCode, getInviteCode } from "./redis" +import { User } from "@budibase/types" +import { tenancy, db as dbCore, PouchLike } from "@budibase/backend-core" const nodemailer = require("nodemailer") -const env = require("../environment") -const { getScopedConfig } = require("@budibase/backend-core/db") -const { EmailTemplatePurpose, TemplateTypes, Config } = require("../constants") -const { getTemplateByPurpose } = require("../constants/templates") -const { getSettingsTemplateContext } = require("./templates") -const { processString } = require("@budibase/string-templates") -const { getResetPasswordCode, getInviteCode } = require("../utilities/redis") -const { getGlobalDB } = require("@budibase/backend-core/tenancy") + +type SendEmailOpts = { + // workspaceId If finer grain controls being used then this will lookup config for workspace. + workspaceId?: string + // user If sending to an existing user the object can be provided, this is used in the context. + user: User + // from If sending from an address that is not what is configured in the SMTP config. + from?: string + // contents If sending a custom email then can supply contents which will be added to it. + contents?: string + // subject A custom subject can be specified if the config one is not desired. + subject?: string + // info Pass in a structure of information to be stored alongside the invitation. + info?: any + cc?: boolean + bcc?: boolean + automation?: boolean +} const TEST_MODE = false -const TYPE = TemplateTypes.EMAIL +const TYPE = TemplateType.EMAIL const FULL_EMAIL_PURPOSES = [ EmailTemplatePurpose.INVITATION, @@ -18,8 +36,8 @@ const FULL_EMAIL_PURPOSES = [ EmailTemplatePurpose.CUSTOM, ] -function createSMTPTransport(config) { - let options +function createSMTPTransport(config: any) { + let options: any let secure = config.secure // default it if not specified if (secure == null) { @@ -52,10 +70,15 @@ function createSMTPTransport(config) { return nodemailer.createTransport(options) } -async function getLinkCode(purpose, email, user, info = null) { +async function getLinkCode( + purpose: EmailTemplatePurpose, + email: string, + user: User, + info: any = null +) { switch (purpose) { case EmailTemplatePurpose.PASSWORD_RECOVERY: - return getResetPasswordCode(user._id, info) + return getResetPasswordCode(user._id!, info) case EmailTemplatePurpose.INVITATION: return getInviteCode(email, info) default: @@ -72,7 +95,12 @@ async function getLinkCode(purpose, email, user, info = null) { * @param {string|null} contents if using a custom template can supply contents for context. * @return {Promise} returns the built email HTML if all provided parameters were valid. */ -async function buildEmail(purpose, email, context, { user, contents } = {}) { +async function buildEmail( + purpose: EmailTemplatePurpose, + email: string, + context: any, + { user, contents }: any = {} +) { // this isn't a full email if (FULL_EMAIL_PURPOSES.indexOf(purpose) === -1) { throw `Unable to build an email of type ${purpose}` @@ -113,15 +141,19 @@ async function buildEmail(purpose, email, context, { user, contents } = {}) { * @param {boolean|null} automation Whether or not the configuration is being fetched for an email automation. * @return {Promise} returns the SMTP configuration if it exists */ -async function getSmtpConfiguration(db, workspaceId = null, automation) { - const params = { +async function getSmtpConfiguration( + db: PouchLike, + workspaceId?: string, + automation?: boolean +) { + const params: any = { type: Config.SMTP, } if (workspaceId) { params.workspace = workspaceId } - const customConfig = await getScopedConfig(db, params) + const customConfig = await dbCore.getScopedConfig(db, params) if (customConfig) { return customConfig @@ -146,12 +178,12 @@ async function getSmtpConfiguration(db, workspaceId = null, automation) { * Checks if a SMTP config exists based on passed in parameters. * @return {Promise} returns true if there is a configuration that can be used. */ -exports.isEmailConfigured = async (workspaceId = null) => { +export async function isEmailConfigured(workspaceId?: string) { // when "testing" or smtp fallback is enabled simply return true if (TEST_MODE || env.SMTP_FALLBACK_ENABLED) { return true } - const db = getGlobalDB() + const db = tenancy.getGlobalDB() const config = await getSmtpConfiguration(db, workspaceId) return config != null } @@ -161,48 +193,49 @@ exports.isEmailConfigured = async (workspaceId = null) => { * send an email using it. * @param {string} email The email address to send to. * @param {string} purpose The purpose of the email being sent (e.g. reset password). - * @param {string|undefined} workspaceId If finer grain controls being used then this will lookup config for workspace. - * @param {object|undefined} user If sending to an existing user the object can be provided, this is used in the context. - * @param {string|undefined} from If sending from an address that is not what is configured in the SMTP config. - * @param {string|undefined} contents If sending a custom email then can supply contents which will be added to it. - * @param {string|undefined} subject A custom subject can be specified if the config one is not desired. - * @param {object|undefined} info Pass in a structure of information to be stored alongside the invitation. - * @param {boolean|undefined} disableFallback Prevent email being sent from SMTP fallback to avoid spam. + * @param {object} opts The options for sending the email. * @return {Promise} returns details about the attempt to send email, e.g. if it is successful; based on * nodemailer response. */ -exports.sendEmail = async ( - email, - purpose, - { workspaceId, user, from, contents, subject, info, cc, bcc, automation } = {} -) => { - const db = getGlobalDB() - let config = (await getSmtpConfiguration(db, workspaceId, automation)) || {} +export async function sendEmail( + email: string, + purpose: EmailTemplatePurpose, + opts: SendEmailOpts +) { + const db = tenancy.getGlobalDB() + let config = + (await getSmtpConfiguration(db, opts?.workspaceId, opts?.automation)) || {} if (Object.keys(config).length === 0 && !TEST_MODE) { throw "Unable to find SMTP configuration." } const transport = createSMTPTransport(config) // if there is a link code needed this will retrieve it - const code = await getLinkCode(purpose, email, user, info) - const context = await getSettingsTemplateContext(purpose, code) + const code = await getLinkCode(purpose, email, opts.user, opts?.info) + let context + if (code) { + context = await getSettingsTemplateContext(purpose, code) + } - let message = { - from: from || config.from, + let message: any = { + from: opts?.from || config.from, html: await buildEmail(purpose, email, context, { - user, - contents, + user: opts?.user, + contents: opts?.contents, }), } message = { ...message, to: email, - cc: cc, - bcc: bcc, + cc: opts?.cc, + bcc: opts?.bcc, } - if (subject || config.subject) { - message.subject = await processString(subject || config.subject, context) + if (opts?.subject || config.subject) { + message.subject = await processString( + opts?.subject || config.subject, + context + ) } const response = await transport.sendMail(message) if (TEST_MODE) { @@ -216,7 +249,7 @@ exports.sendEmail = async ( * @param {object} config an SMTP configuration - this is based on the nodemailer API. * @return {Promise} returns true if the configuration is valid. */ -exports.verifyConfig = async config => { +export async function verifyConfig(config: any) { const transport = createSMTPTransport(config) await transport.verify() } diff --git a/packages/worker/src/utilities/redis.ts b/packages/worker/src/utilities/redis.ts index 7e474b2c28..893ec9f0a8 100644 --- a/packages/worker/src/utilities/redis.ts +++ b/packages/worker/src/utilities/redis.ts @@ -1,36 +1,35 @@ -const { Client, utils } = require("@budibase/backend-core/redis") -const { newid } = require("@budibase/backend-core/utils") +import { redis, utils } from "@budibase/backend-core" -function getExpirySecondsForDB(db) { +function getExpirySecondsForDB(db: string) { switch (db) { - case utils.Databases.PW_RESETS: + case redis.utils.Databases.PW_RESETS: // a hour return 3600 - case utils.Databases.INVITATIONS: + case redis.utils.Databases.INVITATIONS: // a day return 86400 } } -let pwResetClient, invitationClient +let pwResetClient: any, invitationClient: any -function getClient(db) { +function getClient(db: string) { switch (db) { - case utils.Databases.PW_RESETS: + case redis.utils.Databases.PW_RESETS: return pwResetClient - case utils.Databases.INVITATIONS: + case redis.utils.Databases.INVITATIONS: return invitationClient } } -async function writeACode(db, value) { +async function writeACode(db: string, value: any) { const client = await getClient(db) - const code = newid() + const code = utils.newid() await client.store(code, value, getExpirySecondsForDB(db)) return code } -async function getACode(db, code, deleteCode = true) { +async function getACode(db: string, code: string, deleteCode = true) { const client = await getClient(db) const value = await client.get(code) if (!value) { @@ -42,9 +41,9 @@ async function getACode(db, code, deleteCode = true) { return value } -exports.init = async () => { - pwResetClient = new Client(utils.Databases.PW_RESETS) - invitationClient = new Client(utils.Databases.INVITATIONS) +export async function init() { + pwResetClient = new redis.Client(redis.utils.Databases.PW_RESETS) + invitationClient = new redis.Client(redis.utils.Databases.INVITATIONS) await pwResetClient.init() await invitationClient.init() } @@ -52,7 +51,7 @@ exports.init = async () => { /** * make sure redis connection is closed. */ -exports.shutdown = async () => { +export async function shutdown() { if (pwResetClient) await pwResetClient.finish() if (invitationClient) await invitationClient.finish() console.log("Redis shutdown") @@ -65,8 +64,8 @@ exports.shutdown = async () => { * @param {object} info Info about the user/the reset process. * @return {Promise} returns the code that was stored to redis. */ -exports.getResetPasswordCode = async (userId, info) => { - return writeACode(utils.Databases.PW_RESETS, { userId, info }) +export async function getResetPasswordCode(userId: string, info: any) { + return writeACode(redis.utils.Databases.PW_RESETS, { userId, info }) } /** @@ -75,9 +74,12 @@ exports.getResetPasswordCode = async (userId, info) => { * @param {boolean} deleteCode If the code is used/finished with this will delete it - defaults to true. * @return {Promise} returns the user ID if it is found */ -exports.checkResetPasswordCode = async (resetCode, deleteCode = true) => { +export async function checkResetPasswordCode( + resetCode: string, + deleteCode = true +) { try { - return getACode(utils.Databases.PW_RESETS, resetCode, deleteCode) + return getACode(redis.utils.Databases.PW_RESETS, resetCode, deleteCode) } catch (err) { throw "Provided information is not valid, cannot reset password - please try again." } @@ -89,8 +91,8 @@ exports.checkResetPasswordCode = async (resetCode, deleteCode = true) => { * @param {object|null} info Information to be carried along with the invitation. * @return {Promise} returns the code that was stored to redis. */ -exports.getInviteCode = async (email, info) => { - return writeACode(utils.Databases.INVITATIONS, { email, info }) +export async function getInviteCode(email: string, info: any) { + return writeACode(redis.utils.Databases.INVITATIONS, { email, info }) } /** @@ -99,9 +101,12 @@ exports.getInviteCode = async (email, info) => { * @param {boolean} deleteCode whether or not the code should be deleted after retrieval - defaults to true. * @return {Promise} If the code is valid then an email address will be returned. */ -exports.checkInviteCode = async (inviteCode, deleteCode = true) => { +export async function checkInviteCode( + inviteCode: string, + deleteCode: boolean = true +) { try { - return getACode(utils.Databases.INVITATIONS, inviteCode, deleteCode) + return getACode(redis.utils.Databases.INVITATIONS, inviteCode, deleteCode) } catch (err) { throw "Invitation is not valid or has expired, please request a new one." } diff --git a/packages/worker/src/utilities/templates.ts b/packages/worker/src/utilities/templates.ts index 935495d428..ede95dbe4a 100644 --- a/packages/worker/src/utilities/templates.ts +++ b/packages/worker/src/utilities/templates.ts @@ -1,47 +1,47 @@ -const { getScopedConfig } = require("@budibase/backend-core/db") -const { +import { db as dbCore, tenancy } from "@budibase/backend-core" +import { Config, - InternalTemplateBindings, + InternalTemplateBinding, LOGO_URL, EmailTemplatePurpose, -} = require("../constants") -const { checkSlashesInUrl } = require("./index") -const { - getGlobalDB, - addTenantToUrl, -} = require("@budibase/backend-core/tenancy") +} from "../constants" +import { checkSlashesInUrl } from "./index" const BASE_COMPANY = "Budibase" -exports.getSettingsTemplateContext = async (purpose, code = null) => { - const db = getGlobalDB() +export async function getSettingsTemplateContext( + purpose: EmailTemplatePurpose, + code?: string +) { + const db = tenancy.getGlobalDB() // TODO: use more granular settings in the future if required - let settings = (await getScopedConfig(db, { type: Config.SETTINGS })) || {} + let settings = + (await dbCore.getScopedConfig(db, { type: Config.SETTINGS })) || {} const URL = settings.platformUrl - const context = { - [InternalTemplateBindings.LOGO_URL]: + const context: any = { + [InternalTemplateBinding.LOGO_URL]: checkSlashesInUrl(`${URL}/${settings.logoUrl}`) || LOGO_URL, - [InternalTemplateBindings.PLATFORM_URL]: URL, - [InternalTemplateBindings.COMPANY]: settings.company || BASE_COMPANY, - [InternalTemplateBindings.DOCS_URL]: + [InternalTemplateBinding.PLATFORM_URL]: URL, + [InternalTemplateBinding.COMPANY]: settings.company || BASE_COMPANY, + [InternalTemplateBinding.DOCS_URL]: settings.docsUrl || "https://docs.budibase.com/", - [InternalTemplateBindings.LOGIN_URL]: checkSlashesInUrl( - addTenantToUrl(`${URL}/login`) + [InternalTemplateBinding.LOGIN_URL]: checkSlashesInUrl( + tenancy.addTenantToUrl(`${URL}/login`) ), - [InternalTemplateBindings.CURRENT_DATE]: new Date().toISOString(), - [InternalTemplateBindings.CURRENT_YEAR]: new Date().getFullYear(), + [InternalTemplateBinding.CURRENT_DATE]: new Date().toISOString(), + [InternalTemplateBinding.CURRENT_YEAR]: new Date().getFullYear(), } // attach purpose specific context switch (purpose) { case EmailTemplatePurpose.PASSWORD_RECOVERY: - context[InternalTemplateBindings.RESET_CODE] = code - context[InternalTemplateBindings.RESET_URL] = checkSlashesInUrl( - addTenantToUrl(`${URL}/builder/auth/reset?code=${code}`) + context[InternalTemplateBinding.RESET_CODE] = code + context[InternalTemplateBinding.RESET_URL] = checkSlashesInUrl( + tenancy.addTenantToUrl(`${URL}/builder/auth/reset?code=${code}`) ) break case EmailTemplatePurpose.INVITATION: - context[InternalTemplateBindings.INVITE_CODE] = code - context[InternalTemplateBindings.INVITE_URL] = checkSlashesInUrl( - addTenantToUrl(`${URL}/builder/invite?code=${code}`) + context[InternalTemplateBinding.INVITE_CODE] = code + context[InternalTemplateBinding.INVITE_URL] = checkSlashesInUrl( + tenancy.addTenantToUrl(`${URL}/builder/invite?code=${code}`) ) break } From f5760b6601c64525a187a0d3d94d862e74e97672 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 16 Nov 2022 18:28:45 +0000 Subject: [PATCH 18/24] Fixes post merge. --- packages/backend-core/src/middleware/tenancy.ts | 4 ++-- packages/backend-core/src/tenancy/tenancy.ts | 15 +++++++++------ .../worker/src/middleware/tests/tenancy.spec.ts | 14 +++++++------- 3 files changed, 18 insertions(+), 15 deletions(-) diff --git a/packages/backend-core/src/middleware/tenancy.ts b/packages/backend-core/src/middleware/tenancy.ts index 03dd9d11e6..0aaacef139 100644 --- a/packages/backend-core/src/middleware/tenancy.ts +++ b/packages/backend-core/src/middleware/tenancy.ts @@ -1,6 +1,6 @@ import { doInTenant, getTenantIDFromCtx } from "../tenancy" import { buildMatcherRegex, matches } from "./matchers" -import { Headers } from "../constants" +import { Header } from "../constants" import { BBContext, EndpointMatcher, @@ -29,7 +29,7 @@ const tenancy = ( } const tenantId = getTenantIDFromCtx(ctx, tenantOpts) - ctx.set(Headers.TENANT_ID, tenantId as string) + ctx.set(Header.TENANT_ID, tenantId as string) return doInTenant(tenantId, next) } } diff --git a/packages/backend-core/src/tenancy/tenancy.ts b/packages/backend-core/src/tenancy/tenancy.ts index 0bbd423fe5..cc1088ab08 100644 --- a/packages/backend-core/src/tenancy/tenancy.ts +++ b/packages/backend-core/src/tenancy/tenancy.ts @@ -1,7 +1,10 @@ -import { doWithDB } from "../db" -import { queryPlatformView } from "../db/views" -import { StaticDatabases, ViewName } from "../db/constants" -import { getGlobalDBName } from "../db/tenancy" +import { + doWithDB, + queryPlatformView, + StaticDatabases, + getGlobalDBName, + ViewName, +} from "../db" import { DEFAULT_TENANT_ID, getTenantId, @@ -15,7 +18,7 @@ import { TenantResolutionStrategy, GetTenantIdOptions, } from "@budibase/types" -import { Headers } from "../constants" +import { Header } from "../constants" const TENANT_DOC = StaticDatabases.PLATFORM_INFO.docs.tenants const PLATFORM_INFO_DB = StaticDatabases.PLATFORM_INFO.name @@ -200,7 +203,7 @@ export const getTenantIDFromCtx = ( // header if (isAllowed(TenantResolutionStrategy.HEADER)) { - const headerTenantId = ctx.request.headers[Headers.TENANT_ID] + const headerTenantId = ctx.request.headers[Header.TENANT_ID] if (headerTenantId) { return headerTenantId as string } diff --git a/packages/worker/src/middleware/tests/tenancy.spec.ts b/packages/worker/src/middleware/tests/tenancy.spec.ts index 72c00fb6fb..a8b7a50e55 100644 --- a/packages/worker/src/middleware/tests/tenancy.spec.ts +++ b/packages/worker/src/middleware/tests/tenancy.spec.ts @@ -20,18 +20,18 @@ describe("tenancy middleware", () => { const user = await config.createTenant() await config.createSession(user) const res = await config.api.self.getSelf(user) - expect(res.headers[constants.Headers.TENANT_ID]).toBe(user.tenantId) + expect(res.headers[constants.Header.TENANT_ID]).toBe(user.tenantId) }) it("should get tenant id from header", async () => { const tenantId = structures.uuid() const headers = { - [constants.Headers.TENANT_ID]: tenantId, + [constants.Header.TENANT_ID]: tenantId, } const res = await config.request .get(`/api/global/configs/checklist`) .set(headers) - expect(res.headers[constants.Headers.TENANT_ID]).toBe(tenantId) + expect(res.headers[constants.Header.TENANT_ID]).toBe(tenantId) }) it("should get tenant id from query param", async () => { @@ -39,7 +39,7 @@ describe("tenancy middleware", () => { const res = await config.request.get( `/api/global/configs/checklist?tenantId=${tenantId}` ) - expect(res.headers[constants.Headers.TENANT_ID]).toBe(tenantId) + expect(res.headers[constants.Header.TENANT_ID]).toBe(tenantId) }) it("should get tenant id from subdomain", async () => { @@ -50,7 +50,7 @@ describe("tenancy middleware", () => { const res = await config.request .get(`/api/global/configs/checklist`) .set(headers) - expect(res.headers[constants.Headers.TENANT_ID]).toBe(tenantId) + expect(res.headers[constants.Header.TENANT_ID]).toBe(tenantId) }) it("should get tenant id from path variable", async () => { @@ -61,13 +61,13 @@ describe("tenancy middleware", () => { username: user.email, password: user.password, }) - expect(res.headers[constants.Headers.TENANT_ID]).toBe(user.tenantId) + expect(res.headers[constants.Header.TENANT_ID]).toBe(user.tenantId) }) it("should throw when no tenant id is found", async () => { const res = await config.request.get(`/api/global/configs/checklist`) expect(res.status).toBe(403) expect(res.text).toBe("Tenant id not set") - expect(res.headers[constants.Headers.TENANT_ID]).toBe(undefined) + expect(res.headers[constants.Header.TENANT_ID]).toBe(undefined) }) }) From bc857d7bb8ef71acaf0fdd62df2292ea1aa3ac4e Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 17 Nov 2022 13:31:54 +0000 Subject: [PATCH 19/24] PR comments - still experiencing build issues. --- .../src/context/{localStorage.ts => Context.ts} | 2 +- packages/backend-core/src/context/index.ts | 2 +- .../src/db/couch/{pouchLike.ts => PouchLike.ts} | 0 packages/backend-core/src/db/couch/index.ts | 2 +- packages/backend-core/src/db/index.ts | 1 + packages/backend-core/src/index.ts | 4 +--- packages/backend-core/src/types.ts | 1 - packages/backend-core/src/users.ts | 6 +++--- packages/backend-core/tsconfig.build.json | 2 +- packages/server/src/middleware/builder.ts | 8 ++++++-- packages/server/src/middleware/selfhost.ts | 2 +- packages/worker/src/sdk/users/users.ts | 2 +- packages/worker/src/utilities/email.ts | 2 +- packages/worker/tsconfig.build.json | 6 +++--- 14 files changed, 21 insertions(+), 19 deletions(-) rename packages/backend-core/src/context/{localStorage.ts => Context.ts} (93%) rename packages/backend-core/src/db/couch/{pouchLike.ts => PouchLike.ts} (100%) delete mode 100644 packages/backend-core/src/types.ts diff --git a/packages/backend-core/src/context/localStorage.ts b/packages/backend-core/src/context/Context.ts similarity index 93% rename from packages/backend-core/src/context/localStorage.ts rename to packages/backend-core/src/context/Context.ts index fd761d10bc..6ffb57e44e 100644 --- a/packages/backend-core/src/context/localStorage.ts +++ b/packages/backend-core/src/context/Context.ts @@ -1,7 +1,7 @@ import { AsyncLocalStorage } from "async_hooks" import { ContextMap } from "./constants" -export class Context { +export default class Context { static storage = new AsyncLocalStorage() static run(context: ContextMap, func: any) { diff --git a/packages/backend-core/src/context/index.ts b/packages/backend-core/src/context/index.ts index 076d9af65a..3499082f6a 100644 --- a/packages/backend-core/src/context/index.ts +++ b/packages/backend-core/src/context/index.ts @@ -8,7 +8,7 @@ import { getDB, PouchLike, } from "../db" -import { Context } from "./localStorage" +import Context from "./Context" import { IdentityContext } from "@budibase/types" import { DEFAULT_TENANT_ID as _DEFAULT_TENANT_ID } from "../constants" import { ContextMap } from "./constants" diff --git a/packages/backend-core/src/db/couch/pouchLike.ts b/packages/backend-core/src/db/couch/PouchLike.ts similarity index 100% rename from packages/backend-core/src/db/couch/pouchLike.ts rename to packages/backend-core/src/db/couch/PouchLike.ts diff --git a/packages/backend-core/src/db/couch/index.ts b/packages/backend-core/src/db/couch/index.ts index b8cb0ab79f..ea47022c35 100644 --- a/packages/backend-core/src/db/couch/index.ts +++ b/packages/backend-core/src/db/couch/index.ts @@ -1,4 +1,4 @@ export * from "./connections" -export * from "./pouchLike" +export * from "./PouchLike" export * from "./utils" export { init, getPouch, getPouchDB, closePouchDB } from "./pouchDB" diff --git a/packages/backend-core/src/db/index.ts b/packages/backend-core/src/db/index.ts index 7269aa8f92..3db29a905f 100644 --- a/packages/backend-core/src/db/index.ts +++ b/packages/backend-core/src/db/index.ts @@ -5,3 +5,4 @@ export * from "./views" export * from "./constants" export * from "./conversions" export * from "./tenancy" +export { PouchLike } from "./couch" diff --git a/packages/backend-core/src/index.ts b/packages/backend-core/src/index.ts index 24207a7a91..c68c8f0927 100644 --- a/packages/backend-core/src/index.ts +++ b/packages/backend-core/src/index.ts @@ -21,10 +21,9 @@ import * as middleware from "./middleware" import plugins from "./plugin" import encryption from "./security/encryption" import * as queue from "./queue" -import * as types from "./types" +import * as db from "./db" // mimic the outer package exports -import * as db from "./db" import * as objectStore from "./pkg/objectStore" import * as utils from "./pkg/utils" import redis from "./pkg/redis" @@ -68,7 +67,6 @@ const core = { encryption, queue, permissions, - ...types, } export = core diff --git a/packages/backend-core/src/types.ts b/packages/backend-core/src/types.ts deleted file mode 100644 index eeb784b0c1..0000000000 --- a/packages/backend-core/src/types.ts +++ /dev/null @@ -1 +0,0 @@ -export { PouchLike } from "./db" diff --git a/packages/backend-core/src/users.ts b/packages/backend-core/src/users.ts index 13e8f8d36e..1720a79a83 100644 --- a/packages/backend-core/src/users.ts +++ b/packages/backend-core/src/users.ts @@ -3,9 +3,9 @@ import { getUsersByAppParams, getProdAppID, generateAppUserID, -} from "./db/utils" -import { queryGlobalView } from "./db/views" -import { UNICODE_MAX } from "./db/constants" + queryGlobalView, + UNICODE_MAX, +} from "./db" import { BulkDocsResponse, User } from "@budibase/types" import { getGlobalDB } from "./context" diff --git a/packages/backend-core/tsconfig.build.json b/packages/backend-core/tsconfig.build.json index 40ffe6b827..f5b16eda1a 100644 --- a/packages/backend-core/tsconfig.build.json +++ b/packages/backend-core/tsconfig.build.json @@ -12,7 +12,7 @@ "sourceMap": true, "declaration": true, "types": [ "node", "jest" ], - "outDir": "dist" + "outDir": "dist", }, "include": [ "**/*.js", diff --git a/packages/server/src/middleware/builder.ts b/packages/server/src/middleware/builder.ts index 403597ff1a..719a4dc060 100644 --- a/packages/server/src/middleware/builder.ts +++ b/packages/server/src/middleware/builder.ts @@ -9,9 +9,13 @@ import { checkDebounce, setDebounce, } from "../utilities/redis" -import { db as dbCore, cache, permissions } from "@budibase/backend-core" +import { + db as dbCore, + cache, + permissions, + PouchLike, +} from "@budibase/backend-core" import { BBContext } from "@budibase/types" -import { PouchLike } from "@budibase/backend-core/src/db" const DEBOUNCE_TIME_SEC = 30 diff --git a/packages/server/src/middleware/selfhost.ts b/packages/server/src/middleware/selfhost.ts index 180d80c8ea..54b757d29c 100644 --- a/packages/server/src/middleware/selfhost.ts +++ b/packages/server/src/middleware/selfhost.ts @@ -3,7 +3,7 @@ import { BBContext } from "@budibase/types" // if added as a middleware will stop requests unless builder is in self host mode // or cloud is in self host -module.exports = async (ctx: BBContext, next: any) => { +export = async (ctx: BBContext, next: any) => { if (env.SELF_HOSTED) { await next() return diff --git a/packages/worker/src/sdk/users/users.ts b/packages/worker/src/sdk/users/users.ts index e2215cee80..539ac21300 100644 --- a/packages/worker/src/sdk/users/users.ts +++ b/packages/worker/src/sdk/users/users.ts @@ -479,7 +479,7 @@ export const bulkDelete = async ( (user: RowResponse) => { return user.doc } - ) as User[] + ) // Delete from DB const toDelete = usersToDelete.map(user => ({ diff --git a/packages/worker/src/utilities/email.ts b/packages/worker/src/utilities/email.ts index c07604827d..e71109c0b8 100644 --- a/packages/worker/src/utilities/email.ts +++ b/packages/worker/src/utilities/email.ts @@ -142,7 +142,7 @@ async function buildEmail( * @return {Promise} returns the SMTP configuration if it exists */ async function getSmtpConfiguration( - db: PouchLike, + db: dbCore.PouchLike, workspaceId?: string, automation?: boolean ) { diff --git a/packages/worker/tsconfig.build.json b/packages/worker/tsconfig.build.json index 36622d33eb..8071b4ad00 100644 --- a/packages/worker/tsconfig.build.json +++ b/packages/worker/tsconfig.build.json @@ -1,17 +1,17 @@ { "compilerOptions": { "target": "es6", - "skipLibCheck": true, "module": "commonjs", "lib": ["es2020"], "allowJs": true, - "outDir": "dist", "strict": true, "noImplicitAny": true, "esModuleInterop": true, "resolveJsonModule": true, "incremental": true, - "types": [ "node", "jest"], + "types": [ "node", "jest" ], + "outDir": "dist", + "skipLibCheck": true }, "include": [ "src/**/*" From 58e59bfacc0d9d3305a660e5fa228032f0dbc2f0 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 17 Nov 2022 14:35:03 +0000 Subject: [PATCH 20/24] Updating PouchLike to be an interface in types, with its impl stored in backend-core, now called Database. --- .../backend-core/src/cache/writethrough.ts | 14 +- .../backend-core/src/context/deprovision.ts | 11 +- packages/backend-core/src/context/index.ts | 11 +- .../couch/{PouchLike.ts => DatabaseImpl.ts} | 52 ++--- packages/backend-core/src/db/couch/index.ts | 2 +- packages/backend-core/src/db/db.ts | 9 +- packages/backend-core/src/db/index.ts | 1 - packages/backend-core/src/db/utils.ts | 5 +- packages/backend-core/src/db/views.ts | 20 +- packages/server/src/middleware/builder.ts | 11 +- .../functions/backfill/app/tables.ts | 8 +- .../server/src/sdk/app/backups/imports.ts | 9 +- .../server/src/sdk/app/backups/statistics.ts | 19 +- packages/server/src/sdk/app/tables/index.ts | 6 +- packages/types/package.json | 3 +- packages/types/src/sdk/db.ts | 49 +++++ packages/types/yarn.lock | 203 ++++++++++++++++++ packages/worker/src/utilities/email.ts | 6 +- 18 files changed, 336 insertions(+), 103 deletions(-) rename packages/backend-core/src/db/couch/{PouchLike.ts => DatabaseImpl.ts} (82%) diff --git a/packages/backend-core/src/cache/writethrough.ts b/packages/backend-core/src/cache/writethrough.ts index e13cc8cc0d..dc889d5b18 100644 --- a/packages/backend-core/src/cache/writethrough.ts +++ b/packages/backend-core/src/cache/writethrough.ts @@ -1,7 +1,7 @@ import BaseCache from "./base" import { getWritethroughClient } from "../redis/init" import { logWarn } from "../logging" -import { PouchLike } from "../db" +import { Database } from "@budibase/types" const DEFAULT_WRITE_RATE_MS = 10000 let CACHE: BaseCache | null = null @@ -19,7 +19,7 @@ async function getCache() { return CACHE } -function makeCacheKey(db: PouchLike, key: string) { +function makeCacheKey(db: Database, key: string) { return db.name + key } @@ -28,7 +28,7 @@ function makeCacheItem(doc: any, lastWrite: number | null = null): CacheItem { } export async function put( - db: PouchLike, + db: Database, doc: any, writeRateMs: number = DEFAULT_WRITE_RATE_MS ) { @@ -64,7 +64,7 @@ export async function put( return { ok: true, id: output._id, rev: output._rev } } -export async function get(db: PouchLike, id: string): Promise { +export async function get(db: Database, id: string): Promise { const cache = await getCache() const cacheKey = makeCacheKey(db, id) let cacheItem: CacheItem = await cache.get(cacheKey) @@ -77,7 +77,7 @@ export async function get(db: PouchLike, id: string): Promise { } export async function remove( - db: PouchLike, + db: Database, docOrId: any, rev?: any ): Promise { @@ -95,10 +95,10 @@ export async function remove( } export class Writethrough { - db: PouchLike + db: Database writeRateMs: number - constructor(db: PouchLike, writeRateMs: number = DEFAULT_WRITE_RATE_MS) { + constructor(db: Database, writeRateMs: number = DEFAULT_WRITE_RATE_MS) { this.db = db this.writeRateMs = writeRateMs } diff --git a/packages/backend-core/src/context/deprovision.ts b/packages/backend-core/src/context/deprovision.ts index 6f397f60d0..81f03096dc 100644 --- a/packages/backend-core/src/context/deprovision.ts +++ b/packages/backend-core/src/context/deprovision.ts @@ -3,17 +3,16 @@ import { getAllApps, doWithDB, StaticDatabases, - PouchLike, } from "../db" import { doWithGlobalDB } from "../tenancy" -import { App, Tenants, User } from "@budibase/types" +import { App, Tenants, User, Database } from "@budibase/types" const TENANT_DOC = StaticDatabases.PLATFORM_INFO.docs.tenants const PLATFORM_INFO_DB = StaticDatabases.PLATFORM_INFO.name async function removeTenantFromInfoDB(tenantId: string) { try { - await doWithDB(PLATFORM_INFO_DB, async (infoDb: PouchLike) => { + await doWithDB(PLATFORM_INFO_DB, async (infoDb: Database) => { const tenants = (await infoDb.get(TENANT_DOC)) as Tenants tenants.tenantIds = tenants.tenantIds.filter(id => id !== tenantId) @@ -26,7 +25,7 @@ async function removeTenantFromInfoDB(tenantId: string) { } export async function removeUserFromInfoDB(dbUser: User) { - await doWithDB(PLATFORM_INFO_DB, async (infoDb: PouchLike) => { + await doWithDB(PLATFORM_INFO_DB, async (infoDb: Database) => { const keys = [dbUser._id!, dbUser.email] const userDocs = await infoDb.allDocs({ keys, @@ -77,7 +76,7 @@ async function removeUsersFromInfoDB(tenantId: string) { } async function removeGlobalDB(tenantId: string) { - return doWithGlobalDB(tenantId, async (db: PouchLike) => { + return doWithGlobalDB(tenantId, async (db: Database) => { try { await db.destroy() } catch (err) { @@ -91,7 +90,7 @@ async function removeTenantApps(tenantId: string) { try { const apps = (await getAllApps({ all: true })) as App[] const destroyPromises = apps.map(app => - doWithDB(app.appId, (db: PouchLike) => db.destroy()) + doWithDB(app.appId, (db: Database) => db.destroy()) ) await Promise.allSettled(destroyPromises) } catch (err) { diff --git a/packages/backend-core/src/context/index.ts b/packages/backend-core/src/context/index.ts index 3499082f6a..ce37d4f0b4 100644 --- a/packages/backend-core/src/context/index.ts +++ b/packages/backend-core/src/context/index.ts @@ -6,10 +6,9 @@ import { getProdAppID, baseGlobalDBName, getDB, - PouchLike, } from "../db" import Context from "./Context" -import { IdentityContext } from "@budibase/types" +import { IdentityContext, Database } from "@budibase/types" import { DEFAULT_TENANT_ID as _DEFAULT_TENANT_ID } from "../constants" import { ContextMap } from "./constants" export const DEFAULT_TENANT_ID = _DEFAULT_TENANT_ID @@ -182,7 +181,7 @@ export function updateAppId(appId: string) { } } -export function getGlobalDB(): PouchLike { +export function getGlobalDB(): Database { const context = Context.get() if (!context || (env.MULTI_TENANCY && !context.tenantId)) { throw new Error("Global DB not found") @@ -194,7 +193,7 @@ export function getGlobalDB(): PouchLike { * Gets the app database based on whatever the request * contained, dev or prod. */ -export function getAppDB(opts?: any): PouchLike { +export function getAppDB(opts?: any): Database { const appId = getAppId() return getDB(appId, opts) } @@ -203,7 +202,7 @@ export function getAppDB(opts?: any): PouchLike { * This specifically gets the prod app ID, if the request * contained a development app ID, this will get the prod one. */ -export function getProdAppDB(opts?: any): PouchLike { +export function getProdAppDB(opts?: any): Database { const appId = getAppId() if (!appId) { throw new Error("Unable to retrieve prod DB - no app ID.") @@ -215,7 +214,7 @@ export function getProdAppDB(opts?: any): PouchLike { * This specifically gets the dev app ID, if the request * contained a prod app ID, this will get the dev one. */ -export function getDevAppDB(opts?: any): PouchLike { +export function getDevAppDB(opts?: any): Database { const appId = getAppId() if (!appId) { throw new Error("Unable to retrieve dev DB - no app ID.") diff --git a/packages/backend-core/src/db/couch/PouchLike.ts b/packages/backend-core/src/db/couch/DatabaseImpl.ts similarity index 82% rename from packages/backend-core/src/db/couch/PouchLike.ts rename to packages/backend-core/src/db/couch/DatabaseImpl.ts index 975bbd96c4..d7220e845c 100644 --- a/packages/backend-core/src/db/couch/PouchLike.ts +++ b/packages/backend-core/src/db/couch/DatabaseImpl.ts @@ -1,47 +1,35 @@ import Nano from "nano" -import { AllDocsResponse, AnyDocument } from "@budibase/types" +import { + AllDocsResponse, + AnyDocument, + Database, + DatabaseOpts, + DatabaseQueryOpts, + DatabasePutOpts, +} from "@budibase/types" import { getCouchInfo } from "./connections" import { directCouchCall } from "./utils" import { getPouchDB } from "./pouchDB" -export type PouchLikeOpts = { - skip_setup?: boolean -} - -export type PutOpts = { - force?: boolean -} - -export type QueryOpts = { - include_docs?: boolean - startkey?: string - endkey?: string - limit?: number - skip?: number - descending?: boolean - key?: string - keys?: string[] -} - -export class PouchLike { +export class DatabaseImpl implements Database { public readonly name: string private static nano: Nano.ServerScope - private readonly pouchOpts: PouchLikeOpts + private readonly pouchOpts: DatabaseOpts - constructor(dbName?: string, opts?: PouchLikeOpts) { + constructor(dbName?: string, opts?: DatabaseOpts) { if (dbName == null) { throw new Error("Database name cannot be undefined.") } this.name = dbName this.pouchOpts = opts || {} - if (!PouchLike.nano) { - PouchLike.init() + if (!DatabaseImpl.nano) { + DatabaseImpl.init() } } static init() { const couchInfo = getCouchInfo() - PouchLike.nano = Nano({ + DatabaseImpl.nano = Nano({ url: couchInfo.url, requestDefaults: { headers: { @@ -65,9 +53,9 @@ export class PouchLike { throw new Error("DB does not exist") } if (!exists) { - await PouchLike.nano.db.create(this.name) + await DatabaseImpl.nano.db.create(this.name) } - return PouchLike.nano.db.use(this.name) + return DatabaseImpl.nano.db.use(this.name) } private async updateOutput(fnc: any) { @@ -97,7 +85,7 @@ export class PouchLike { return this.updateOutput(() => db.destroy(id, rev)) } - async put(document: AnyDocument, opts?: PutOpts) { + async put(document: AnyDocument, opts?: DatabasePutOpts) { if (!document._id) { throw new Error("Cannot store document without _id field.") } @@ -126,14 +114,14 @@ export class PouchLike { return this.updateOutput(() => db.bulk({ docs: documents })) } - async allDocs(params: QueryOpts): Promise> { + async allDocs(params: DatabaseQueryOpts): Promise> { const db = await this.checkSetup() return this.updateOutput(() => db.list(params)) } async query( viewName: string, - params: QueryOpts + params: DatabaseQueryOpts ): Promise> { const db = await this.checkSetup() const [database, view] = viewName.split("/") @@ -142,7 +130,7 @@ export class PouchLike { async destroy() { try { - await PouchLike.nano.db.destroy(this.name) + await DatabaseImpl.nano.db.destroy(this.name) } catch (err: any) { // didn't exist, don't worry if (err.statusCode === 404) { diff --git a/packages/backend-core/src/db/couch/index.ts b/packages/backend-core/src/db/couch/index.ts index ea47022c35..c731d20d6c 100644 --- a/packages/backend-core/src/db/couch/index.ts +++ b/packages/backend-core/src/db/couch/index.ts @@ -1,4 +1,4 @@ export * from "./connections" -export * from "./PouchLike" +export * from "./DatabaseImpl" export * from "./utils" export { init, getPouch, getPouchDB, closePouchDB } from "./pouchDB" diff --git a/packages/backend-core/src/db/db.ts b/packages/backend-core/src/db/db.ts index beaea3d42c..3887e8b09f 100644 --- a/packages/backend-core/src/db/db.ts +++ b/packages/backend-core/src/db/db.ts @@ -1,17 +1,18 @@ import env from "../environment" -import { directCouchQuery, PouchLike, getPouchDB } from "./couch" -import { CouchFindOptions } from "@budibase/types" +import { directCouchQuery, getPouchDB } from "./couch" +import { CouchFindOptions, Database } from "@budibase/types" +import { DatabaseImpl } from "../db" const dbList = new Set() -export function getDB(dbName?: string, opts?: any): PouchLike { +export function getDB(dbName?: string, opts?: any): Database { // TODO: once using the test image, need to remove this if (env.isTest()) { dbList.add(dbName) // @ts-ignore return getPouchDB(dbName, opts) } - return new PouchLike(dbName, opts) + return new DatabaseImpl(dbName, opts) } // we have to use a callback for this so that we can close diff --git a/packages/backend-core/src/db/index.ts b/packages/backend-core/src/db/index.ts index 3db29a905f..7269aa8f92 100644 --- a/packages/backend-core/src/db/index.ts +++ b/packages/backend-core/src/db/index.ts @@ -5,4 +5,3 @@ export * from "./views" export * from "./constants" export * from "./conversions" export * from "./tenancy" -export { PouchLike } from "./couch" diff --git a/packages/backend-core/src/db/utils.ts b/packages/backend-core/src/db/utils.ts index 76e011735e..076552beec 100644 --- a/packages/backend-core/src/db/utils.ts +++ b/packages/backend-core/src/db/utils.ts @@ -15,8 +15,7 @@ import { getAppMetadata } from "../cache/appMetadata" import { isDevApp, isDevAppID, getProdAppID } from "./conversions" import { APP_PREFIX } from "./constants" import * as events from "../events" -import { PouchLike } from "./couch" -import { App } from "@budibase/types" +import { App, Database } from "@budibase/types" export * from "./constants" export * from "./conversions" @@ -395,7 +394,7 @@ export async function getDevAppIDs() { export async function dbExists(dbName: any) { return doWithDB( dbName, - async (db: PouchLike) => { + async (db: Database) => { return await db.exists() }, { skip_setup: true } diff --git a/packages/backend-core/src/db/views.ts b/packages/backend-core/src/db/views.ts index 23771d3f2d..5c63ce4689 100644 --- a/packages/backend-core/src/db/views.ts +++ b/packages/backend-core/src/db/views.ts @@ -1,8 +1,8 @@ import { DocumentType, ViewName, DeprecatedViews, SEPARATOR } from "./utils" import { getGlobalDB } from "../context" -import { PouchLike, QueryOpts } from "./couch" import { StaticDatabases } from "./constants" import { doWithDB } from "./" +import { Database, DatabaseQueryOpts } from "@budibase/types" const DESIGN_DB = "_design/database" @@ -19,7 +19,7 @@ interface DesignDocument { views: any } -async function removeDeprecated(db: PouchLike, viewName: ViewName) { +async function removeDeprecated(db: Database, viewName: ViewName) { // @ts-ignore if (!DeprecatedViews[viewName]) { return @@ -70,7 +70,7 @@ export const createAccountEmailView = async () => { emit(doc.email.toLowerCase(), doc._id) } }` - await doWithDB(StaticDatabases.PLATFORM_INFO.name, async (db: PouchLike) => { + await doWithDB(StaticDatabases.PLATFORM_INFO.name, async (db: Database) => { await createView(db, viewJs, ViewName.ACCOUNT_BY_EMAIL) }) } @@ -114,7 +114,7 @@ export const createPlatformUserView = async () => { emit(doc._id.toLowerCase(), doc._id) } }` - await doWithDB(StaticDatabases.PLATFORM_INFO.name, async (db: PouchLike) => { + await doWithDB(StaticDatabases.PLATFORM_INFO.name, async (db: Database) => { await createView(db, viewJs, ViewName.PLATFORM_USERS_LOWERCASE) }) } @@ -125,8 +125,8 @@ export interface QueryViewOptions { export const queryView = async ( viewName: ViewName, - params: QueryOpts, - db: PouchLike, + params: DatabaseQueryOpts, + db: Database, createFunc: any, opts?: QueryViewOptions ): Promise => { @@ -157,7 +157,7 @@ export const queryView = async ( export const queryPlatformView = async ( viewName: ViewName, - params: QueryOpts, + params: DatabaseQueryOpts, opts?: QueryViewOptions ): Promise => { const CreateFuncByName: any = { @@ -165,7 +165,7 @@ export const queryPlatformView = async ( [ViewName.PLATFORM_USERS_LOWERCASE]: createPlatformUserView, } - return doWithDB(StaticDatabases.PLATFORM_INFO.name, async (db: PouchLike) => { + return doWithDB(StaticDatabases.PLATFORM_INFO.name, async (db: Database) => { const createFn = CreateFuncByName[viewName] return queryView(viewName, params, db, createFn, opts) }) @@ -173,8 +173,8 @@ export const queryPlatformView = async ( export const queryGlobalView = async ( viewName: ViewName, - params: QueryOpts, - db?: PouchLike, + params: DatabaseQueryOpts, + db?: Database, opts?: QueryViewOptions ): Promise => { const CreateFuncByName: any = { diff --git a/packages/server/src/middleware/builder.ts b/packages/server/src/middleware/builder.ts index 719a4dc060..b53f2903b7 100644 --- a/packages/server/src/middleware/builder.ts +++ b/packages/server/src/middleware/builder.ts @@ -9,13 +9,8 @@ import { checkDebounce, setDebounce, } from "../utilities/redis" -import { - db as dbCore, - cache, - permissions, - PouchLike, -} from "@budibase/backend-core" -import { BBContext } from "@budibase/types" +import { db as dbCore, cache, permissions } from "@budibase/backend-core" +import { BBContext, Database } from "@budibase/types" const DEBOUNCE_TIME_SEC = 30 @@ -55,7 +50,7 @@ async function updateAppUpdatedAt(ctx: BBContext) { if (ctx.method === "GET" || (await checkDebounce(appId))) { return } - await dbCore.doWithDB(appId, async (db: PouchLike) => { + await dbCore.doWithDB(appId, async (db: Database) => { const metadata = await db.get(DocumentType.APP_METADATA) metadata.updatedAt = new Date().toISOString() diff --git a/packages/server/src/migrations/functions/backfill/app/tables.ts b/packages/server/src/migrations/functions/backfill/app/tables.ts index c9d1e5c794..51b0de5d29 100644 --- a/packages/server/src/migrations/functions/backfill/app/tables.ts +++ b/packages/server/src/migrations/functions/backfill/app/tables.ts @@ -1,10 +1,8 @@ -import { events, PouchLike } from "@budibase/backend-core" +import { events } from "@budibase/backend-core" +import { Database } from "@budibase/types" import sdk from "../../../../sdk" -export const backfill = async ( - appDb: PouchLike, - timestamp: string | number -) => { +export const backfill = async (appDb: Database, timestamp: string | number) => { const tables = await sdk.tables.getAllInternalTables(appDb) for (const table of tables) { diff --git a/packages/server/src/sdk/app/backups/imports.ts b/packages/server/src/sdk/app/backups/imports.ts index 634f507220..998ffe9241 100644 --- a/packages/server/src/sdk/app/backups/imports.ts +++ b/packages/server/src/sdk/app/backups/imports.ts @@ -1,4 +1,5 @@ -import { db as dbCore, PouchLike } from "@budibase/backend-core" +import { db as dbCore } from "@budibase/backend-core" +import { Database } from "@budibase/types" import { getAutomationParams, TABLE_ROW_PREFIX } from "../../../db/utils" import { budibaseTempDir } from "../../../utilities/budibaseDir" import { DB_EXPORT_FILE, GLOBAL_DB_EXPORT_FILE } from "./constants" @@ -28,7 +29,7 @@ type TemplateType = { key?: string } -async function updateAttachmentColumns(prodAppId: string, db: PouchLike) { +async function updateAttachmentColumns(prodAppId: string, db: Database) { // iterate through attachment documents and update them const tables = await sdk.tables.getAllInternalTables(db) for (let table of tables) { @@ -82,7 +83,7 @@ async function updateAttachmentColumns(prodAppId: string, db: PouchLike) { } } -async function updateAutomations(prodAppId: string, db: PouchLike) { +async function updateAutomations(prodAppId: string, db: Database) { const automations = ( await db.allDocs( getAutomationParams(null, { @@ -150,7 +151,7 @@ export function getListOfAppsInMulti(tmpPath: string) { export async function importApp( appId: string, - db: PouchLike, + db: Database, template: TemplateType ) { let prodAppId = dbCore.getProdAppID(appId) diff --git a/packages/server/src/sdk/app/backups/statistics.ts b/packages/server/src/sdk/app/backups/statistics.ts index 9fe1a04d21..aecb3de423 100644 --- a/packages/server/src/sdk/app/backups/statistics.ts +++ b/packages/server/src/sdk/app/backups/statistics.ts @@ -1,4 +1,5 @@ -import { context, db as dbCore, PouchLike } from "@budibase/backend-core" +import { context, db as dbCore } from "@budibase/backend-core" +import { Database } from "@budibase/types" import { getDatasourceParams, getTableParams, @@ -6,7 +7,7 @@ import { getScreenParams, } from "../../../db/utils" -async function runInContext(appId: string, cb: any, db?: PouchLike) { +async function runInContext(appId: string, cb: any, db?: Database) { if (db) { return cb(db) } else { @@ -18,10 +19,10 @@ async function runInContext(appId: string, cb: any, db?: PouchLike) { } } -export async function calculateDatasourceCount(appId: string, db?: PouchLike) { +export async function calculateDatasourceCount(appId: string, db?: Database) { return runInContext( appId, - async (db: PouchLike) => { + async (db: Database) => { const datasourceList = await db.allDocs(getDatasourceParams()) const tableList = await db.allDocs(getTableParams()) return datasourceList.rows.length + tableList.rows.length @@ -30,10 +31,10 @@ export async function calculateDatasourceCount(appId: string, db?: PouchLike) { ) } -export async function calculateAutomationCount(appId: string, db?: PouchLike) { +export async function calculateAutomationCount(appId: string, db?: Database) { return runInContext( appId, - async (db: PouchLike) => { + async (db: Database) => { const automationList = await db.allDocs(getAutomationParams()) return automationList.rows.length }, @@ -41,10 +42,10 @@ export async function calculateAutomationCount(appId: string, db?: PouchLike) { ) } -export async function calculateScreenCount(appId: string, db?: PouchLike) { +export async function calculateScreenCount(appId: string, db?: Database) { return runInContext( appId, - async (db: PouchLike) => { + async (db: Database) => { const screenList = await db.allDocs(getScreenParams()) return screenList.rows.length }, @@ -53,7 +54,7 @@ export async function calculateScreenCount(appId: string, db?: PouchLike) { } export async function calculateBackupStats(appId: string) { - return runInContext(appId, async (db: PouchLike) => { + return runInContext(appId, async (db: Database) => { const promises = [] promises.push(calculateDatasourceCount(appId, db)) promises.push(calculateAutomationCount(appId, db)) diff --git a/packages/server/src/sdk/app/tables/index.ts b/packages/server/src/sdk/app/tables/index.ts index ef41630d6b..15bf02cf62 100644 --- a/packages/server/src/sdk/app/tables/index.ts +++ b/packages/server/src/sdk/app/tables/index.ts @@ -1,13 +1,13 @@ -import { context, PouchLike } from "@budibase/backend-core" +import { context } from "@budibase/backend-core" import { BudibaseInternalDB, getTableParams } from "../../../db/utils" import { breakExternalTableId, isExternalTable, isSQL, } from "../../../integrations/utils" -import { Table } from "@budibase/types" +import { Table, Database } from "@budibase/types" -async function getAllInternalTables(db?: PouchLike): Promise { +async function getAllInternalTables(db?: Database): Promise { if (!db) { db = context.getAppDB() } diff --git a/packages/types/package.json b/packages/types/package.json index 2978e8b6c1..6d67753d7f 100644 --- a/packages/types/package.json +++ b/packages/types/package.json @@ -17,6 +17,7 @@ "@types/node": "14.18.20", "@types/pouchdb": "6.4.0", "rimraf": "3.0.2", - "typescript": "4.7.3" + "typescript": "4.7.3", + "nano": "10.1.0" } } diff --git a/packages/types/src/sdk/db.ts b/packages/types/src/sdk/db.ts index b3e51d2dff..9d2ed9c21c 100644 --- a/packages/types/src/sdk/db.ts +++ b/packages/types/src/sdk/db.ts @@ -1,4 +1,6 @@ import PouchDB from "pouchdb" +import Nano from "nano" +import { AllDocsResponse, AnyDocument } from "../" export type PouchOptions = { inMemory?: boolean @@ -22,3 +24,50 @@ export type CouchFindOptions = { skip?: number bookmark?: string } + +export type DatabaseOpts = { + skip_setup?: boolean +} + +export type DatabasePutOpts = { + force?: boolean +} + +export type DatabaseQueryOpts = { + include_docs?: boolean + startkey?: string + endkey?: string + limit?: number + skip?: number + descending?: boolean + key?: string + keys?: string[] +} + +export interface Database { + name: string + + exists(): Promise + checkSetup(): Promise> + get(id?: string): Promise + remove(id?: string, rev?: string): Promise + put( + document: AnyDocument, + opts?: DatabasePutOpts + ): Promise + bulkDocs(documents: AnyDocument[]): Promise + allDocs(params: DatabaseQueryOpts): Promise> + query( + viewName: string, + params: DatabaseQueryOpts + ): Promise> + destroy(): Promise + compact(): Promise + // these are all PouchDB related functions that are rarely used - in future + // should be replaced by better typed/non-pouch implemented methods + dump(...args: any[]): Promise + load(...args: any[]): Promise + createIndex(...args: any[]): Promise + deleteIndex(...args: any[]): Promise + getIndexes(...args: any[]): Promise +} diff --git a/packages/types/yarn.lock b/packages/types/yarn.lock index f225ffc442..98e5845121 100644 --- a/packages/types/yarn.lock +++ b/packages/types/yarn.lock @@ -299,6 +299,32 @@ "@types/mime" "^1" "@types/node" "*" +"@types/tough-cookie@^4.0.2": + version "4.0.2" + resolved "https://registry.yarnpkg.com/@types/tough-cookie/-/tough-cookie-4.0.2.tgz#6286b4c7228d58ab7866d19716f3696e03a09397" + integrity sha512-Q5vtl1W5ue16D+nIaW8JWebSSraJVlK+EthKn7e7UcD4KWsaSJ8BqGPXNaPghgtcn/fhvrN17Tv8ksUsQpiplw== + +agent-base@^6.0.2: + version "6.0.2" + resolved "https://registry.yarnpkg.com/agent-base/-/agent-base-6.0.2.tgz#49fff58577cfee3f37176feab4c22e00f86d7f77" + integrity sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ== + dependencies: + debug "4" + +asynckit@^0.4.0: + version "0.4.0" + resolved "https://registry.yarnpkg.com/asynckit/-/asynckit-0.4.0.tgz#c79ed97f7f34cb8f2ba1bc9790bcc366474b4b79" + integrity sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q== + +axios@^1.1.3: + version "1.1.3" + resolved "https://registry.yarnpkg.com/axios/-/axios-1.1.3.tgz#8274250dada2edf53814ed7db644b9c2866c1e35" + integrity sha512-00tXVRwKx/FZr/IDVFt4C+f9FYairX517WoGCL6dpOntqLkZofjhu43F/Xl44UOpqa+9sLFDrG/XAnFsUYgkDA== + dependencies: + follow-redirects "^1.15.0" + form-data "^4.0.0" + proxy-from-env "^1.1.0" + balanced-match@^1.0.0: version "1.0.2" resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.2.tgz#e83e3a7e3f300b34cb9d87f615fa0cbf357690ee" @@ -312,16 +338,71 @@ brace-expansion@^1.1.7: balanced-match "^1.0.0" concat-map "0.0.1" +call-bind@^1.0.0: + version "1.0.2" + resolved "https://registry.yarnpkg.com/call-bind/-/call-bind-1.0.2.tgz#b1d4e89e688119c3c9a903ad30abb2f6a919be3c" + integrity sha512-7O+FbCihrB5WGbFYesctwmTKae6rOiIzmz1icreWJ+0aA7LJfuqhEso2T9ncpcFtzMQtzXf2QGGueWJGTYsqrA== + dependencies: + function-bind "^1.1.1" + get-intrinsic "^1.0.2" + +combined-stream@^1.0.8: + version "1.0.8" + resolved "https://registry.yarnpkg.com/combined-stream/-/combined-stream-1.0.8.tgz#c3d45a8b34fd730631a110a8a2520682b31d5a7f" + integrity sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg== + dependencies: + delayed-stream "~1.0.0" + concat-map@0.0.1: version "0.0.1" resolved "https://registry.yarnpkg.com/concat-map/-/concat-map-0.0.1.tgz#d8a96bd77fd68df7793a73036a3ba0d5405d477b" integrity sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg== +debug@4: + version "4.3.4" + resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.4.tgz#1319f6579357f2338d3337d2cdd4914bb5dcc865" + integrity sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ== + dependencies: + ms "2.1.2" + +delayed-stream@~1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/delayed-stream/-/delayed-stream-1.0.0.tgz#df3ae199acadfb7d440aaae0b29e2272b24ec619" + integrity sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ== + +follow-redirects@^1.15.0: + version "1.15.2" + resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.2.tgz#b460864144ba63f2681096f274c4e57026da2c13" + integrity sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA== + +form-data@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/form-data/-/form-data-4.0.0.tgz#93919daeaf361ee529584b9b31664dc12c9fa452" + integrity sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww== + dependencies: + asynckit "^0.4.0" + combined-stream "^1.0.8" + mime-types "^2.1.12" + fs.realpath@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/fs.realpath/-/fs.realpath-1.0.0.tgz#1504ad2523158caa40db4a2787cb01411994ea4f" integrity sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw== +function-bind@^1.1.1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/function-bind/-/function-bind-1.1.1.tgz#a56899d3ea3c9bab874bb9773b7c5ede92f4895d" + integrity sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A== + +get-intrinsic@^1.0.2: + version "1.1.3" + resolved "https://registry.yarnpkg.com/get-intrinsic/-/get-intrinsic-1.1.3.tgz#063c84329ad93e83893c7f4f243ef63ffa351385" + integrity sha512-QJVz1Tj7MS099PevUG5jvnt9tSkXN8K14dxQlikJuPt4uD9hHAHjLyLBiLR5zELelBdD9QNRAXZzsJx0WaDL9A== + dependencies: + function-bind "^1.1.1" + has "^1.0.3" + has-symbols "^1.0.3" + glob@^7.1.3: version "7.2.3" resolved "https://registry.yarnpkg.com/glob/-/glob-7.2.3.tgz#b8df0fb802bbfa8e89bd1d938b4e16578ed44f2b" @@ -334,6 +415,25 @@ glob@^7.1.3: once "^1.3.0" path-is-absolute "^1.0.0" +has-symbols@^1.0.3: + version "1.0.3" + resolved "https://registry.yarnpkg.com/has-symbols/-/has-symbols-1.0.3.tgz#bb7b2c4349251dce87b125f7bdf874aa7c8b39f8" + integrity sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A== + +has@^1.0.3: + version "1.0.3" + resolved "https://registry.yarnpkg.com/has/-/has-1.0.3.tgz#722d7cbfc1f6aa8241f16dd814e011e1f41e8796" + integrity sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw== + dependencies: + function-bind "^1.1.1" + +http-cookie-agent@^4.0.2: + version "4.0.2" + resolved "https://registry.yarnpkg.com/http-cookie-agent/-/http-cookie-agent-4.0.2.tgz#dcdaae18ed1f7452d81ae4d5cd80b227d6831b69" + integrity sha512-noTmxdH5CuytTnLj/Qv3Z84e/YFq8yLXAw3pqIYZ25Edhb9pQErIAC+ednw40Cic6Le/h9ryph5/TqsvkOaUCw== + dependencies: + agent-base "^6.0.2" + inflight@^1.0.4: version "1.0.6" resolved "https://registry.yarnpkg.com/inflight/-/inflight-1.0.6.tgz#49bd6331d7d02d0c09bc910a1075ba8165b56df9" @@ -347,6 +447,18 @@ inherits@2: resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.4.tgz#0fa2c64f932917c3433a0ded55363aae37416b7c" integrity sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ== +mime-db@1.52.0: + version "1.52.0" + resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.52.0.tgz#bbabcdc02859f4987301c856e3387ce5ec43bf70" + integrity sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg== + +mime-types@^2.1.12: + version "2.1.35" + resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.35.tgz#381a871b62a734450660ae3deee44813f70d959a" + integrity sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw== + dependencies: + mime-db "1.52.0" + minimatch@^3.1.1: version "3.1.2" resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.1.2.tgz#19cd194bfd3e428f049a70817c038d89ab4be35b" @@ -354,6 +466,33 @@ minimatch@^3.1.1: dependencies: brace-expansion "^1.1.7" +ms@2.1.2: + version "2.1.2" + resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.2.tgz#d09d1f357b443f493382a8eb3ccd183872ae6009" + integrity sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w== + +nano@^10.1.0: + version "10.1.0" + resolved "https://registry.yarnpkg.com/nano/-/nano-10.1.0.tgz#afdd5a7440e62f09a8e23f41fcea328d27383922" + integrity sha512-COeN2TpLcHuSN44QLnPmfZCoCsKAg8/aelPOVqqm/2/MvRHDEA11/Kld5C4sLzDlWlhFZ3SO2WGJGevCsvcEzQ== + dependencies: + "@types/tough-cookie" "^4.0.2" + axios "^1.1.3" + http-cookie-agent "^4.0.2" + node-abort-controller "^3.0.1" + qs "^6.11.0" + tough-cookie "^4.1.2" + +node-abort-controller@^3.0.1: + version "3.0.1" + resolved "https://registry.yarnpkg.com/node-abort-controller/-/node-abort-controller-3.0.1.tgz#f91fa50b1dee3f909afabb7e261b1e1d6b0cb74e" + integrity sha512-/ujIVxthRs+7q6hsdjHMaj8hRG9NuWmwrz+JdRwZ14jdFoKSkm+vDsCbF9PLpnSqjaWQJuTmVtcWHNLr+vrOFw== + +object-inspect@^1.9.0: + version "1.12.2" + resolved "https://registry.yarnpkg.com/object-inspect/-/object-inspect-1.12.2.tgz#c0641f26394532f28ab8d796ab954e43c009a8ea" + integrity sha512-z+cPxW0QGUp0mcqcsgQyLVRDoXFQbXOwBaqyF7VIgI4TWNQsDHrBpUQslRmIfAoYWdYzs6UlKJtB2XJpTaNSpQ== + once@^1.3.0: version "1.4.0" resolved "https://registry.yarnpkg.com/once/-/once-1.4.0.tgz#583b1aa775961d4b113ac17d9c50baef9dd76bd1" @@ -366,6 +505,38 @@ path-is-absolute@^1.0.0: resolved "https://registry.yarnpkg.com/path-is-absolute/-/path-is-absolute-1.0.1.tgz#174b9268735534ffbc7ace6bf53a5a9e1b5c5f5f" integrity sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg== +proxy-from-env@^1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/proxy-from-env/-/proxy-from-env-1.1.0.tgz#e102f16ca355424865755d2c9e8ea4f24d58c3e2" + integrity sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg== + +psl@^1.1.33: + version "1.9.0" + resolved "https://registry.yarnpkg.com/psl/-/psl-1.9.0.tgz#d0df2a137f00794565fcaf3b2c00cd09f8d5a5a7" + integrity sha512-E/ZsdU4HLs/68gYzgGTkMicWTLPdAftJLfJFlLUAAKZGkStNU72sZjT66SnMDVOfOWY/YAoiD7Jxa9iHvngcag== + +punycode@^2.1.1: + version "2.1.1" + resolved "https://registry.yarnpkg.com/punycode/-/punycode-2.1.1.tgz#b58b010ac40c22c5657616c8d2c2c02c7bf479ec" + integrity sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A== + +qs@^6.11.0: + version "6.11.0" + resolved "https://registry.yarnpkg.com/qs/-/qs-6.11.0.tgz#fd0d963446f7a65e1367e01abd85429453f0c37a" + integrity sha512-MvjoMCJwEarSbUYk5O+nmoSzSutSsTwF85zcHPQ9OrlFoZOYIjaqBAJIqIXjptyD5vThxGq52Xu/MaJzRkIk4Q== + dependencies: + side-channel "^1.0.4" + +querystringify@^2.1.1: + version "2.2.0" + resolved "https://registry.yarnpkg.com/querystringify/-/querystringify-2.2.0.tgz#3345941b4153cb9d082d8eee4cda2016a9aef7f6" + integrity sha512-FIqgj2EUvTa7R50u0rGsyTftzjYmv/a3hO345bZNrqabNqjtgiDMgmo4mkUjd+nzU5oF3dClKqFIPUKybUyqoQ== + +requires-port@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/requires-port/-/requires-port-1.0.0.tgz#925d2601d39ac485e091cf0da5c6e694dc3dcaff" + integrity sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ== + rimraf@3.0.2: version "3.0.2" resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-3.0.2.tgz#f1a5402ba6220ad52cc1282bac1ae3aa49fd061a" @@ -373,11 +544,43 @@ rimraf@3.0.2: dependencies: glob "^7.1.3" +side-channel@^1.0.4: + version "1.0.4" + resolved "https://registry.yarnpkg.com/side-channel/-/side-channel-1.0.4.tgz#efce5c8fdc104ee751b25c58d4290011fa5ea2cf" + integrity sha512-q5XPytqFEIKHkGdiMIrY10mvLRvnQh42/+GoBlFW3b2LXLE2xxJpZFdm94we0BaoV3RwJyGqg5wS7epxTv0Zvw== + dependencies: + call-bind "^1.0.0" + get-intrinsic "^1.0.2" + object-inspect "^1.9.0" + +tough-cookie@^4.1.2: + version "4.1.2" + resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-4.1.2.tgz#e53e84b85f24e0b65dd526f46628db6c85f6b874" + integrity sha512-G9fqXWoYFZgTc2z8Q5zaHy/vJMjm+WV0AkAeHxVCQiEB1b+dGvWzFW6QV07cY5jQ5gRkeid2qIkzkxUnmoQZUQ== + dependencies: + psl "^1.1.33" + punycode "^2.1.1" + universalify "^0.2.0" + url-parse "^1.5.3" + typescript@4.7.3: version "4.7.3" resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.7.3.tgz#8364b502d5257b540f9de4c40be84c98e23a129d" integrity sha512-WOkT3XYvrpXx4vMMqlD+8R8R37fZkjyLGlxavMc4iB8lrl8L0DeTcHbYgw/v0N/z9wAFsgBhcsF0ruoySS22mA== +universalify@^0.2.0: + version "0.2.0" + resolved "https://registry.yarnpkg.com/universalify/-/universalify-0.2.0.tgz#6451760566fa857534745ab1dde952d1b1761be0" + integrity sha512-CJ1QgKmNg3CwvAv/kOFmtnEN05f0D/cn9QntgNOQlQF9dgvVTHj3t+8JPdjqawCHk7V/KA+fbUqzZ9XWhcqPUg== + +url-parse@^1.5.3: + version "1.5.10" + resolved "https://registry.yarnpkg.com/url-parse/-/url-parse-1.5.10.tgz#9d3c2f736c1d75dd3bd2be507dcc111f1e2ea9c1" + integrity sha512-WypcfiRhfeUP9vvF0j6rw0J3hrWrw6iZv3+22h6iRMJ/8z1Tj6XfLP4DsUix5MhMPnXpiHDoKyoZ/bdCkwBCiQ== + dependencies: + querystringify "^2.1.1" + requires-port "^1.0.0" + wrappy@1: version "1.0.2" resolved "https://registry.yarnpkg.com/wrappy/-/wrappy-1.0.2.tgz#b5243d8f3ec1aa35f1364605bc0d1036e30ab69f" diff --git a/packages/worker/src/utilities/email.ts b/packages/worker/src/utilities/email.ts index e71109c0b8..7ec3447707 100644 --- a/packages/worker/src/utilities/email.ts +++ b/packages/worker/src/utilities/email.ts @@ -4,8 +4,8 @@ import { getTemplateByPurpose } from "../constants/templates" import { getSettingsTemplateContext } from "./templates" import { processString } from "@budibase/string-templates" import { getResetPasswordCode, getInviteCode } from "./redis" -import { User } from "@budibase/types" -import { tenancy, db as dbCore, PouchLike } from "@budibase/backend-core" +import { User, Database } from "@budibase/types" +import { tenancy, db as dbCore } from "@budibase/backend-core" const nodemailer = require("nodemailer") type SendEmailOpts = { @@ -142,7 +142,7 @@ async function buildEmail( * @return {Promise} returns the SMTP configuration if it exists */ async function getSmtpConfiguration( - db: dbCore.PouchLike, + db: Database, workspaceId?: string, automation?: boolean ) { From 56e46dbfb01877361873ed890be79d6a2d2e766f Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 17 Nov 2022 14:38:02 +0000 Subject: [PATCH 21/24] Minor adjustment to interface type. --- packages/types/src/sdk/db.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/types/src/sdk/db.ts b/packages/types/src/sdk/db.ts index 9d2ed9c21c..44b4735882 100644 --- a/packages/types/src/sdk/db.ts +++ b/packages/types/src/sdk/db.ts @@ -55,7 +55,7 @@ export interface Database { document: AnyDocument, opts?: DatabasePutOpts ): Promise - bulkDocs(documents: AnyDocument[]): Promise + bulkDocs(documents: AnyDocument[]): Promise allDocs(params: DatabaseQueryOpts): Promise> query( viewName: string, From 055721914099f15a45ee768fa0f72dc9b73fb2bb Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 17 Nov 2022 14:47:52 +0000 Subject: [PATCH 22/24] Fixing permission types. --- .../{permissions.js => permissions.ts} | 85 +++++++++++-------- 1 file changed, 48 insertions(+), 37 deletions(-) rename packages/backend-core/src/security/{permissions.js => permissions.ts} (72%) diff --git a/packages/backend-core/src/security/permissions.js b/packages/backend-core/src/security/permissions.ts similarity index 72% rename from packages/backend-core/src/security/permissions.js rename to packages/backend-core/src/security/permissions.ts index 2ecb8a9f1e..dda041924f 100644 --- a/packages/backend-core/src/security/permissions.js +++ b/packages/backend-core/src/security/permissions.ts @@ -1,31 +1,40 @@ const { flatten } = require("lodash") const { cloneDeep } = require("lodash/fp") -const PermissionLevels = { - READ: "read", - WRITE: "write", - EXECUTE: "execute", - ADMIN: "admin", +export type RoleHierarchy = { + permissionId: string +}[] + +export enum PermissionLevels { + READ = "read", + WRITE = "write", + EXECUTE = "execute", + ADMIN = "admin", } // these are the global types, that govern the underlying default behaviour -const PermissionTypes = { - APP: "app", - TABLE: "table", - USER: "user", - AUTOMATION: "automation", - WEBHOOK: "webhook", - BUILDER: "builder", - VIEW: "view", - QUERY: "query", +export enum PermissionTypes { + APP = "app", + TABLE = "table", + USER = "user", + AUTOMATION = "automation", + WEBHOOK = "webhook", + BUILDER = "builder", + VIEW = "view", + QUERY = "query", } -function Permission(type, level) { - this.level = level - this.type = type +class Permission { + type: PermissionTypes + level: PermissionLevels + + constructor(type: PermissionTypes, level: PermissionLevels) { + this.type = type + this.level = level + } } -function levelToNumber(perm) { +function levelToNumber(perm: PermissionLevels) { switch (perm) { // not everything has execute privileges case PermissionLevels.EXECUTE: @@ -46,7 +55,7 @@ function levelToNumber(perm) { * @param {string} userPermLevel The permission level of the user. * @return {string[]} All the permission levels this user is allowed to carry out. */ -function getAllowedLevels(userPermLevel) { +function getAllowedLevels(userPermLevel: PermissionLevels) { switch (userPermLevel) { case PermissionLevels.EXECUTE: return [PermissionLevels.EXECUTE] @@ -64,24 +73,24 @@ function getAllowedLevels(userPermLevel) { } } -exports.BUILTIN_PERMISSION_IDS = { - PUBLIC: "public", - READ_ONLY: "read_only", - WRITE: "write", - ADMIN: "admin", - POWER: "power", +export enum BUILTIN_PERMISSION_IDS { + PUBLIC = "public", + READ_ONLY = "read_only", + WRITE = "write", + ADMIN = "admin", + POWER = "power", } const BUILTIN_PERMISSIONS = { PUBLIC: { - _id: exports.BUILTIN_PERMISSION_IDS.PUBLIC, + _id: BUILTIN_PERMISSION_IDS.PUBLIC, name: "Public", permissions: [ new Permission(PermissionTypes.WEBHOOK, PermissionLevels.EXECUTE), ], }, READ_ONLY: { - _id: exports.BUILTIN_PERMISSION_IDS.READ_ONLY, + _id: BUILTIN_PERMISSION_IDS.READ_ONLY, name: "Read only", permissions: [ new Permission(PermissionTypes.QUERY, PermissionLevels.READ), @@ -90,7 +99,7 @@ const BUILTIN_PERMISSIONS = { ], }, WRITE: { - _id: exports.BUILTIN_PERMISSION_IDS.WRITE, + _id: BUILTIN_PERMISSION_IDS.WRITE, name: "Read/Write", permissions: [ new Permission(PermissionTypes.QUERY, PermissionLevels.WRITE), @@ -100,7 +109,7 @@ const BUILTIN_PERMISSIONS = { ], }, POWER: { - _id: exports.BUILTIN_PERMISSION_IDS.POWER, + _id: BUILTIN_PERMISSION_IDS.POWER, name: "Power", permissions: [ new Permission(PermissionTypes.TABLE, PermissionLevels.WRITE), @@ -111,7 +120,7 @@ const BUILTIN_PERMISSIONS = { ], }, ADMIN: { - _id: exports.BUILTIN_PERMISSION_IDS.ADMIN, + _id: BUILTIN_PERMISSION_IDS.ADMIN, name: "Admin", permissions: [ new Permission(PermissionTypes.TABLE, PermissionLevels.ADMIN), @@ -124,16 +133,20 @@ const BUILTIN_PERMISSIONS = { }, } -exports.getBuiltinPermissions = () => { +export function getBuiltinPermissions() { return cloneDeep(BUILTIN_PERMISSIONS) } -exports.getBuiltinPermissionByID = id => { +export function getBuiltinPermissionByID(id: string) { const perms = Object.values(BUILTIN_PERMISSIONS) return perms.find(perm => perm._id === id) } -exports.doesHaveBasePermission = (permType, permLevel, rolesHierarchy) => { +export function doesHaveBasePermission( + permType: PermissionTypes, + permLevel: PermissionLevels, + rolesHierarchy: RoleHierarchy +) { const basePermissions = [ ...new Set(rolesHierarchy.map(role => role.permissionId)), ] @@ -154,11 +167,9 @@ exports.doesHaveBasePermission = (permType, permLevel, rolesHierarchy) => { return false } -exports.isPermissionLevelHigherThanRead = level => { +export function isPermissionLevelHigherThanRead(level: PermissionLevels) { return levelToNumber(level) > 1 } // utility as a lot of things need simply the builder permission -exports.BUILDER = PermissionTypes.BUILDER -exports.PermissionTypes = PermissionTypes -exports.PermissionLevels = PermissionLevels +export const BUILDER = PermissionTypes.BUILDER From 31c198888aafbebcb3ad9f39ea8a992cf9fb4572 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 17 Nov 2022 14:59:18 +0000 Subject: [PATCH 23/24] Updating some enums, plural to single. --- .../backend-core/src/security/permissions.ts | 98 +++++++++---------- packages/backend-core/src/security/roles.ts | 16 +-- packages/server/src/api/routes/automation.js | 8 +- packages/server/src/api/routes/datasource.js | 10 +- .../server/src/api/routes/public/index.ts | 20 ++-- packages/server/src/api/routes/query.js | 10 +- packages/server/src/api/routes/row.ts | 24 ++--- packages/server/src/api/routes/static.ts | 10 +- packages/server/src/api/routes/table.js | 6 +- .../server/src/api/routes/tests/role.spec.js | 10 +- packages/server/src/api/routes/user.js | 20 ++-- .../server/src/api/routes/utils/validators.js | 10 +- packages/server/src/api/routes/view.js | 6 +- packages/server/src/middleware/authorized.ts | 12 +-- packages/server/src/middleware/builder.ts | 2 +- .../src/middleware/tests/authorized.spec.js | 10 +- .../server/src/tests/utilities/structures.js | 4 +- packages/server/src/utilities/security.js | 24 ++--- 18 files changed, 150 insertions(+), 150 deletions(-) diff --git a/packages/backend-core/src/security/permissions.ts b/packages/backend-core/src/security/permissions.ts index dda041924f..42189bba0c 100644 --- a/packages/backend-core/src/security/permissions.ts +++ b/packages/backend-core/src/security/permissions.ts @@ -5,7 +5,7 @@ export type RoleHierarchy = { permissionId: string }[] -export enum PermissionLevels { +export enum PermissionLevel { READ = "read", WRITE = "write", EXECUTE = "execute", @@ -13,7 +13,7 @@ export enum PermissionLevels { } // these are the global types, that govern the underlying default behaviour -export enum PermissionTypes { +export enum PermissionType { APP = "app", TABLE = "table", USER = "user", @@ -25,25 +25,25 @@ export enum PermissionTypes { } class Permission { - type: PermissionTypes - level: PermissionLevels + type: PermissionType + level: PermissionLevel - constructor(type: PermissionTypes, level: PermissionLevels) { + constructor(type: PermissionType, level: PermissionLevel) { this.type = type this.level = level } } -function levelToNumber(perm: PermissionLevels) { +function levelToNumber(perm: PermissionLevel) { switch (perm) { // not everything has execute privileges - case PermissionLevels.EXECUTE: + case PermissionLevel.EXECUTE: return 0 - case PermissionLevels.READ: + case PermissionLevel.READ: return 1 - case PermissionLevels.WRITE: + case PermissionLevel.WRITE: return 2 - case PermissionLevels.ADMIN: + case PermissionLevel.ADMIN: return 3 default: return -1 @@ -55,25 +55,25 @@ function levelToNumber(perm: PermissionLevels) { * @param {string} userPermLevel The permission level of the user. * @return {string[]} All the permission levels this user is allowed to carry out. */ -function getAllowedLevels(userPermLevel: PermissionLevels) { +function getAllowedLevels(userPermLevel: PermissionLevel) { switch (userPermLevel) { - case PermissionLevels.EXECUTE: - return [PermissionLevels.EXECUTE] - case PermissionLevels.READ: - return [PermissionLevels.EXECUTE, PermissionLevels.READ] - case PermissionLevels.WRITE: - case PermissionLevels.ADMIN: + case PermissionLevel.EXECUTE: + return [PermissionLevel.EXECUTE] + case PermissionLevel.READ: + return [PermissionLevel.EXECUTE, PermissionLevel.READ] + case PermissionLevel.WRITE: + case PermissionLevel.ADMIN: return [ - PermissionLevels.READ, - PermissionLevels.WRITE, - PermissionLevels.EXECUTE, + PermissionLevel.READ, + PermissionLevel.WRITE, + PermissionLevel.EXECUTE, ] default: return [] } } -export enum BUILTIN_PERMISSION_IDS { +export enum BuiltinPermissionID { PUBLIC = "public", READ_ONLY = "read_only", WRITE = "write", @@ -83,52 +83,52 @@ export enum BUILTIN_PERMISSION_IDS { const BUILTIN_PERMISSIONS = { PUBLIC: { - _id: BUILTIN_PERMISSION_IDS.PUBLIC, + _id: BuiltinPermissionID.PUBLIC, name: "Public", permissions: [ - new Permission(PermissionTypes.WEBHOOK, PermissionLevels.EXECUTE), + new Permission(PermissionType.WEBHOOK, PermissionLevel.EXECUTE), ], }, READ_ONLY: { - _id: BUILTIN_PERMISSION_IDS.READ_ONLY, + _id: BuiltinPermissionID.READ_ONLY, name: "Read only", permissions: [ - new Permission(PermissionTypes.QUERY, PermissionLevels.READ), - new Permission(PermissionTypes.TABLE, PermissionLevels.READ), - new Permission(PermissionTypes.VIEW, PermissionLevels.READ), + new Permission(PermissionType.QUERY, PermissionLevel.READ), + new Permission(PermissionType.TABLE, PermissionLevel.READ), + new Permission(PermissionType.VIEW, PermissionLevel.READ), ], }, WRITE: { - _id: BUILTIN_PERMISSION_IDS.WRITE, + _id: BuiltinPermissionID.WRITE, name: "Read/Write", permissions: [ - new Permission(PermissionTypes.QUERY, PermissionLevels.WRITE), - new Permission(PermissionTypes.TABLE, PermissionLevels.WRITE), - new Permission(PermissionTypes.VIEW, PermissionLevels.READ), - new Permission(PermissionTypes.AUTOMATION, PermissionLevels.EXECUTE), + new Permission(PermissionType.QUERY, PermissionLevel.WRITE), + new Permission(PermissionType.TABLE, PermissionLevel.WRITE), + new Permission(PermissionType.VIEW, PermissionLevel.READ), + new Permission(PermissionType.AUTOMATION, PermissionLevel.EXECUTE), ], }, POWER: { - _id: BUILTIN_PERMISSION_IDS.POWER, + _id: BuiltinPermissionID.POWER, name: "Power", permissions: [ - new Permission(PermissionTypes.TABLE, PermissionLevels.WRITE), - new Permission(PermissionTypes.USER, PermissionLevels.READ), - new Permission(PermissionTypes.AUTOMATION, PermissionLevels.EXECUTE), - new Permission(PermissionTypes.VIEW, PermissionLevels.READ), - new Permission(PermissionTypes.WEBHOOK, PermissionLevels.READ), + new Permission(PermissionType.TABLE, PermissionLevel.WRITE), + new Permission(PermissionType.USER, PermissionLevel.READ), + new Permission(PermissionType.AUTOMATION, PermissionLevel.EXECUTE), + new Permission(PermissionType.VIEW, PermissionLevel.READ), + new Permission(PermissionType.WEBHOOK, PermissionLevel.READ), ], }, ADMIN: { - _id: BUILTIN_PERMISSION_IDS.ADMIN, + _id: BuiltinPermissionID.ADMIN, name: "Admin", permissions: [ - new Permission(PermissionTypes.TABLE, PermissionLevels.ADMIN), - new Permission(PermissionTypes.USER, PermissionLevels.ADMIN), - new Permission(PermissionTypes.AUTOMATION, PermissionLevels.ADMIN), - new Permission(PermissionTypes.VIEW, PermissionLevels.ADMIN), - new Permission(PermissionTypes.WEBHOOK, PermissionLevels.READ), - new Permission(PermissionTypes.QUERY, PermissionLevels.ADMIN), + new Permission(PermissionType.TABLE, PermissionLevel.ADMIN), + new Permission(PermissionType.USER, PermissionLevel.ADMIN), + new Permission(PermissionType.AUTOMATION, PermissionLevel.ADMIN), + new Permission(PermissionType.VIEW, PermissionLevel.ADMIN), + new Permission(PermissionType.WEBHOOK, PermissionLevel.READ), + new Permission(PermissionType.QUERY, PermissionLevel.ADMIN), ], }, } @@ -143,8 +143,8 @@ export function getBuiltinPermissionByID(id: string) { } export function doesHaveBasePermission( - permType: PermissionTypes, - permLevel: PermissionLevels, + permType: PermissionType, + permLevel: PermissionLevel, rolesHierarchy: RoleHierarchy ) { const basePermissions = [ @@ -167,9 +167,9 @@ export function doesHaveBasePermission( return false } -export function isPermissionLevelHigherThanRead(level: PermissionLevels) { +export function isPermissionLevelHigherThanRead(level: PermissionLevel) { return levelToNumber(level) > 1 } // utility as a lot of things need simply the builder permission -export const BUILDER = PermissionTypes.BUILDER +export const BUILDER = PermissionType.BUILDER diff --git a/packages/backend-core/src/security/roles.ts b/packages/backend-core/src/security/roles.ts index cba88d9751..da475322a7 100644 --- a/packages/backend-core/src/security/roles.ts +++ b/packages/backend-core/src/security/roles.ts @@ -1,4 +1,4 @@ -import { BUILTIN_PERMISSION_IDS, PermissionLevels } from "./permissions" +import { BuiltinPermissionID, PermissionLevel } from "./permissions" import { generateRoleID, getRoleParams, @@ -54,19 +54,19 @@ export class Role { const BUILTIN_ROLES = { ADMIN: new Role(BUILTIN_IDS.ADMIN, "Admin") - .addPermission(BUILTIN_PERMISSION_IDS.ADMIN) + .addPermission(BuiltinPermissionID.ADMIN) .addInheritance(BUILTIN_IDS.POWER), POWER: new Role(BUILTIN_IDS.POWER, "Power") - .addPermission(BUILTIN_PERMISSION_IDS.POWER) + .addPermission(BuiltinPermissionID.POWER) .addInheritance(BUILTIN_IDS.BASIC), BASIC: new Role(BUILTIN_IDS.BASIC, "Basic") - .addPermission(BUILTIN_PERMISSION_IDS.WRITE) + .addPermission(BuiltinPermissionID.WRITE) .addInheritance(BUILTIN_IDS.PUBLIC), PUBLIC: new Role(BUILTIN_IDS.PUBLIC, "Public").addPermission( - BUILTIN_PERMISSION_IDS.PUBLIC + BuiltinPermissionID.PUBLIC ), BUILDER: new Role(BUILTIN_IDS.BUILDER, "Builder").addPermission( - BUILTIN_PERMISSION_IDS.ADMIN + BuiltinPermissionID.ADMIN ), } @@ -227,8 +227,8 @@ export function checkForRoleResourceArray( if (rolePerms && !Array.isArray(rolePerms[resourceId])) { const permLevel = rolePerms[resourceId] as any rolePerms[resourceId] = [permLevel] - if (permLevel === PermissionLevels.WRITE) { - rolePerms[resourceId].push(PermissionLevels.READ) + if (permLevel === PermissionLevel.WRITE) { + rolePerms[resourceId].push(PermissionLevel.READ) } } return rolePerms diff --git a/packages/server/src/api/routes/automation.js b/packages/server/src/api/routes/automation.js index e30a0c1113..544d94abb2 100644 --- a/packages/server/src/api/routes/automation.js +++ b/packages/server/src/api/routes/automation.js @@ -3,8 +3,8 @@ const controller = require("../controllers/automation") const authorized = require("../../middleware/authorized") const { BUILDER, - PermissionLevels, - PermissionTypes, + PermissionLevel, + PermissionType, } = require("@budibase/backend-core/permissions") const { bodyResource, paramResource } = require("../../middleware/resourceId") const { @@ -71,14 +71,14 @@ router "/api/automations/:id/trigger", appInfoMiddleware({ appType: AppType.PROD }), paramResource("id"), - authorized(PermissionTypes.AUTOMATION, PermissionLevels.EXECUTE), + authorized(PermissionType.AUTOMATION, PermissionLevel.EXECUTE), controller.trigger ) .post( "/api/automations/:id/test", appInfoMiddleware({ appType: AppType.DEV }), paramResource("id"), - authorized(PermissionTypes.AUTOMATION, PermissionLevels.EXECUTE), + authorized(PermissionType.AUTOMATION, PermissionLevel.EXECUTE), controller.test ) diff --git a/packages/server/src/api/routes/datasource.js b/packages/server/src/api/routes/datasource.js index 23a3ea9fb0..402e464e16 100644 --- a/packages/server/src/api/routes/datasource.js +++ b/packages/server/src/api/routes/datasource.js @@ -3,8 +3,8 @@ const datasourceController = require("../controllers/datasource") const authorized = require("../../middleware/authorized") const { BUILDER, - PermissionLevels, - PermissionTypes, + PermissionLevel, + PermissionType, } = require("@budibase/backend-core/permissions") const { datasourceValidator, @@ -17,17 +17,17 @@ router .get("/api/datasources", authorized(BUILDER), datasourceController.fetch) .get( "/api/datasources/:datasourceId", - authorized(PermissionTypes.TABLE, PermissionLevels.READ), + authorized(PermissionType.TABLE, PermissionLevel.READ), datasourceController.find ) .put( "/api/datasources/:datasourceId", - authorized(PermissionTypes.TABLE, PermissionLevels.READ), + authorized(PermissionType.TABLE, PermissionLevel.READ), datasourceController.update ) .post( "/api/datasources/query", - authorized(PermissionTypes.TABLE, PermissionLevels.READ), + authorized(PermissionType.TABLE, PermissionLevel.READ), datasourceQueryValidator(), datasourceController.query ) diff --git a/packages/server/src/api/routes/public/index.ts b/packages/server/src/api/routes/public/index.ts index ca49a1a7d6..198ba2d2b2 100644 --- a/packages/server/src/api/routes/public/index.ts +++ b/packages/server/src/api/routes/public/index.ts @@ -13,8 +13,8 @@ import env from "../../../environment" const Router = require("@koa/router") const { RateLimit, Stores } = require("koa2-ratelimit") const { - PermissionLevels, - PermissionTypes, + PermissionLevel, + PermissionType, } = require("@budibase/backend-core/permissions") const { getRedisOptions } = require("@budibase/backend-core/redis").utils @@ -105,7 +105,7 @@ function applyRoutes( : paramResource(resource) const publicApiMiddleware = publicApi({ requiresAppId: - permType !== PermissionTypes.APP && permType !== PermissionTypes.USER, + permType !== PermissionType.APP && permType !== PermissionType.USER, }) addMiddleware(endpoints.read, publicApiMiddleware) addMiddleware(endpoints.write, publicApiMiddleware) @@ -113,8 +113,8 @@ function applyRoutes( addMiddleware(endpoints.read, paramMiddleware) addMiddleware(endpoints.write, paramMiddleware) // add the authorization middleware, using the correct perm type - addMiddleware(endpoints.read, authorized(permType, PermissionLevels.READ)) - addMiddleware(endpoints.write, authorized(permType, PermissionLevels.WRITE)) + addMiddleware(endpoints.read, authorized(permType, PermissionLevel.READ)) + addMiddleware(endpoints.write, authorized(permType, PermissionLevel.WRITE)) // add the output mapper middleware addMiddleware(endpoints.read, mapperMiddleware, { output: true }) addMiddleware(endpoints.write, mapperMiddleware, { output: true }) @@ -122,12 +122,12 @@ function applyRoutes( addToRouter(endpoints.write) } -applyRoutes(appEndpoints, PermissionTypes.APP, "appId") -applyRoutes(tableEndpoints, PermissionTypes.TABLE, "tableId") -applyRoutes(userEndpoints, PermissionTypes.USER, "userId") -applyRoutes(queryEndpoints, PermissionTypes.QUERY, "queryId") +applyRoutes(appEndpoints, PermissionType.APP, "appId") +applyRoutes(tableEndpoints, PermissionType.TABLE, "tableId") +applyRoutes(userEndpoints, PermissionType.USER, "userId") +applyRoutes(queryEndpoints, PermissionType.QUERY, "queryId") // needs to be applied last for routing purposes, don't override other endpoints -applyRoutes(rowEndpoints, PermissionTypes.TABLE, "tableId", "rowId") +applyRoutes(rowEndpoints, PermissionType.TABLE, "tableId", "rowId") export default publicRouter diff --git a/packages/server/src/api/routes/query.js b/packages/server/src/api/routes/query.js index 14434a45c7..1195e55113 100644 --- a/packages/server/src/api/routes/query.js +++ b/packages/server/src/api/routes/query.js @@ -2,8 +2,8 @@ const Router = require("@koa/router") const queryController = require("../controllers/query") const authorized = require("../../middleware/authorized") const { - PermissionLevels, - PermissionTypes, + PermissionLevel, + PermissionType, BUILDER, } = require("@budibase/backend-core/permissions") const { @@ -38,20 +38,20 @@ router .get( "/api/queries/:queryId", paramResource("queryId"), - authorized(PermissionTypes.QUERY, PermissionLevels.READ), + authorized(PermissionType.QUERY, PermissionLevel.READ), queryController.find ) // DEPRECATED - use new query endpoint for future work .post( "/api/queries/:queryId", paramResource("queryId"), - authorized(PermissionTypes.QUERY, PermissionLevels.WRITE), + authorized(PermissionType.QUERY, PermissionLevel.WRITE), queryController.executeV1 ) .post( "/api/v2/queries/:queryId", paramResource("queryId"), - authorized(PermissionTypes.QUERY, PermissionLevels.WRITE), + authorized(PermissionType.QUERY, PermissionLevel.WRITE), queryController.executeV2 ) .delete( diff --git a/packages/server/src/api/routes/row.ts b/packages/server/src/api/routes/row.ts index cb342adebb..72189a2482 100644 --- a/packages/server/src/api/routes/row.ts +++ b/packages/server/src/api/routes/row.ts @@ -3,8 +3,8 @@ import * as rowController from "../controllers/row" import authorized from "../../middleware/authorized" import { paramResource, paramSubResource } from "../../middleware/resourceId" const { - PermissionLevels, - PermissionTypes, + PermissionLevel, + PermissionType, } = require("@budibase/backend-core/permissions") const { internalSearchValidator } = require("./utils/validators") @@ -28,7 +28,7 @@ router .get( "/api/:tableId/:rowId/enrich", paramSubResource("tableId", "rowId"), - authorized(PermissionTypes.TABLE, PermissionLevels.READ), + authorized(PermissionType.TABLE, PermissionLevel.READ), rowController.fetchEnrichedRow ) /** @@ -48,7 +48,7 @@ router .get( "/api/:tableId/rows", paramResource("tableId"), - authorized(PermissionTypes.TABLE, PermissionLevels.READ), + authorized(PermissionType.TABLE, PermissionLevel.READ), rowController.fetch ) /** @@ -67,7 +67,7 @@ router .get( "/api/:tableId/rows/:rowId", paramSubResource("tableId", "rowId"), - authorized(PermissionTypes.TABLE, PermissionLevels.READ), + authorized(PermissionType.TABLE, PermissionLevel.READ), rowController.find ) /** @@ -137,7 +137,7 @@ router "/api/:tableId/search", internalSearchValidator(), paramResource("tableId"), - authorized(PermissionTypes.TABLE, PermissionLevels.READ), + authorized(PermissionType.TABLE, PermissionLevel.READ), rowController.search ) // DEPRECATED - this is an old API, but for backwards compat it needs to be @@ -145,7 +145,7 @@ router .post( "/api/search/:tableId/rows", paramResource("tableId"), - authorized(PermissionTypes.TABLE, PermissionLevels.READ), + authorized(PermissionType.TABLE, PermissionLevel.READ), rowController.search ) /** @@ -175,7 +175,7 @@ router .post( "/api/:tableId/rows", paramResource("tableId"), - authorized(PermissionTypes.TABLE, PermissionLevels.WRITE), + authorized(PermissionType.TABLE, PermissionLevel.WRITE), rowController.save ) /** @@ -189,7 +189,7 @@ router .patch( "/api/:tableId/rows", paramResource("tableId"), - authorized(PermissionTypes.TABLE, PermissionLevels.WRITE), + authorized(PermissionType.TABLE, PermissionLevel.WRITE), rowController.patch ) /** @@ -215,7 +215,7 @@ router .post( "/api/:tableId/rows/validate", paramResource("tableId"), - authorized(PermissionTypes.TABLE, PermissionLevels.WRITE), + authorized(PermissionType.TABLE, PermissionLevel.WRITE), rowController.validate ) /** @@ -241,7 +241,7 @@ router .delete( "/api/:tableId/rows", paramResource("tableId"), - authorized(PermissionTypes.TABLE, PermissionLevels.WRITE), + authorized(PermissionType.TABLE, PermissionLevel.WRITE), rowController.destroy ) @@ -261,7 +261,7 @@ router .post( "/api/:tableId/rows/exportRows", paramResource("tableId"), - authorized(PermissionTypes.TABLE, PermissionLevels.WRITE), + authorized(PermissionType.TABLE, PermissionLevel.WRITE), rowController.exportRows ) diff --git a/packages/server/src/api/routes/static.ts b/packages/server/src/api/routes/static.ts index 8ca8a1cc2b..ccfec6fd8c 100644 --- a/packages/server/src/api/routes/static.ts +++ b/packages/server/src/api/routes/static.ts @@ -4,8 +4,8 @@ import { budibaseTempDir } from "../../utilities/budibaseDir" import authorized from "../../middleware/authorized" import { BUILDER, - PermissionTypes, - PermissionLevels, + PermissionType, + PermissionLevel, } from "@budibase/backend-core/permissions" import * as env from "../../environment" import { paramResource } from "../../middleware/resourceId" @@ -47,13 +47,13 @@ router .post( "/api/attachments/:tableId/upload", paramResource("tableId"), - authorized(PermissionTypes.TABLE, PermissionLevels.WRITE), + authorized(PermissionType.TABLE, PermissionLevel.WRITE), controller.uploadFile ) .post( "/api/attachments/:tableId/delete", paramResource("tableId"), - authorized(PermissionTypes.TABLE, PermissionLevels.WRITE), + authorized(PermissionType.TABLE, PermissionLevel.WRITE), controller.deleteObjects ) .get("/app/preview", authorized(BUILDER), controller.serveBuilderPreview) @@ -61,7 +61,7 @@ router .get("/app/:appUrl/:path*", controller.serveApp) .post( "/api/attachments/:datasourceId/url", - authorized(PermissionTypes.TABLE, PermissionLevels.READ), + authorized(PermissionType.TABLE, PermissionLevel.READ), controller.getSignedUploadURL ) diff --git a/packages/server/src/api/routes/table.js b/packages/server/src/api/routes/table.js index 711312149a..8d280e06d3 100644 --- a/packages/server/src/api/routes/table.js +++ b/packages/server/src/api/routes/table.js @@ -4,8 +4,8 @@ const authorized = require("../../middleware/authorized") const { paramResource, bodyResource } = require("../../middleware/resourceId") const { BUILDER, - PermissionLevels, - PermissionTypes, + PermissionLevel, + PermissionType, } = require("@budibase/backend-core/permissions") const { tableValidator } = require("./utils/validators") @@ -40,7 +40,7 @@ router .get( "/api/tables/:tableId", paramResource("tableId"), - authorized(PermissionTypes.TABLE, PermissionLevels.READ, { schema: true }), + authorized(PermissionType.TABLE, PermissionLevel.READ, { schema: true }), tableController.find ) /** diff --git a/packages/server/src/api/routes/tests/role.spec.js b/packages/server/src/api/routes/tests/role.spec.js index 9f44cbd136..56a3f1e9d9 100644 --- a/packages/server/src/api/routes/tests/role.spec.js +++ b/packages/server/src/api/routes/tests/role.spec.js @@ -1,6 +1,6 @@ const { BUILTIN_ROLE_IDS } = require("@budibase/backend-core/roles") const { - BUILTIN_PERMISSION_IDS, + BuiltinPermissionID, } = require("@budibase/backend-core/permissions") const setup = require("./utilities") const { basicRole } = setup.structures @@ -76,18 +76,18 @@ describe("/roles", () => { const adminRole = res.body.find(r => r._id === BUILTIN_ROLE_IDS.ADMIN) expect(adminRole).toBeDefined() expect(adminRole.inherits).toEqual(BUILTIN_ROLE_IDS.POWER) - expect(adminRole.permissionId).toEqual(BUILTIN_PERMISSION_IDS.ADMIN) + expect(adminRole.permissionId).toEqual(BuiltinPermissionID.ADMIN) const powerUserRole = res.body.find(r => r._id === BUILTIN_ROLE_IDS.POWER) expect(powerUserRole).toBeDefined() expect(powerUserRole.inherits).toEqual(BUILTIN_ROLE_IDS.BASIC) - expect(powerUserRole.permissionId).toEqual(BUILTIN_PERMISSION_IDS.POWER) + expect(powerUserRole.permissionId).toEqual(BuiltinPermissionID.POWER) const customRoleFetched = res.body.find(r => r._id === customRole._id) expect(customRoleFetched).toBeDefined() expect(customRoleFetched.inherits).toEqual(BUILTIN_ROLE_IDS.BASIC) expect(customRoleFetched.permissionId).toEqual( - BUILTIN_PERMISSION_IDS.READ_ONLY + BuiltinPermissionID.READ_ONLY ) }) @@ -109,7 +109,7 @@ describe("/roles", () => { it("should delete custom roles", async () => { const customRole = await config.createRole({ name: "user", - permissionId: BUILTIN_PERMISSION_IDS.READ_ONLY, + permissionId: BuiltinPermissionID.READ_ONLY, inherits: BUILTIN_ROLE_IDS.BASIC, }) delete customRole._rev_tree diff --git a/packages/server/src/api/routes/user.js b/packages/server/src/api/routes/user.js index a0eaf26ec6..a290ced829 100644 --- a/packages/server/src/api/routes/user.js +++ b/packages/server/src/api/routes/user.js @@ -2,8 +2,8 @@ const Router = require("@koa/router") const controller = require("../controllers/user") const authorized = require("../../middleware/authorized") const { - PermissionLevels, - PermissionTypes, + PermissionLevel, + PermissionType, } = require("@budibase/backend-core/permissions") const router = new Router() @@ -11,42 +11,42 @@ const router = new Router() router .get( "/api/users/metadata", - authorized(PermissionTypes.USER, PermissionLevels.READ), + authorized(PermissionType.USER, PermissionLevel.READ), controller.fetchMetadata ) .get( "/api/users/metadata/:id", - authorized(PermissionTypes.USER, PermissionLevels.READ), + authorized(PermissionType.USER, PermissionLevel.READ), controller.findMetadata ) .put( "/api/users/metadata", - authorized(PermissionTypes.USER, PermissionLevels.WRITE), + authorized(PermissionType.USER, PermissionLevel.WRITE), controller.updateMetadata ) .post( "/api/users/metadata/self", - authorized(PermissionTypes.USER, PermissionLevels.WRITE), + authorized(PermissionType.USER, PermissionLevel.WRITE), controller.updateSelfMetadata ) .delete( "/api/users/metadata/:id", - authorized(PermissionTypes.USER, PermissionLevels.WRITE), + authorized(PermissionType.USER, PermissionLevel.WRITE), controller.destroyMetadata ) .post( "/api/users/metadata/sync/:id", - authorized(PermissionTypes.USER, PermissionLevels.WRITE), + authorized(PermissionType.USER, PermissionLevel.WRITE), controller.syncUser ) .post( "/api/users/flags", - authorized(PermissionTypes.USER, PermissionLevels.WRITE), + authorized(PermissionType.USER, PermissionLevel.WRITE), controller.setFlag ) .get( "/api/users/flags", - authorized(PermissionTypes.USER, PermissionLevels.READ), + authorized(PermissionType.USER, PermissionLevel.READ), controller.getFlags ) diff --git a/packages/server/src/api/routes/utils/validators.js b/packages/server/src/api/routes/utils/validators.js index f1d8871805..b44cce5771 100644 --- a/packages/server/src/api/routes/utils/validators.js +++ b/packages/server/src/api/routes/utils/validators.js @@ -1,8 +1,8 @@ const { joiValidator } = require("@budibase/backend-core/auth") const { DataSourceOperation } = require("../../../constants") const { - BUILTIN_PERMISSION_IDS, - PermissionLevels, + BuiltinPermissionID, + PermissionLevel, } = require("@budibase/backend-core/permissions") const { WebhookActionType } = require("@budibase/types") const Joi = require("joi") @@ -133,14 +133,14 @@ exports.webhookValidator = () => { } exports.roleValidator = () => { - const permLevelArray = Object.values(PermissionLevels) + const permLevelArray = Object.values(PermissionLevel) // prettier-ignore return joiValidator.body(Joi.object({ _id: OPTIONAL_STRING, _rev: OPTIONAL_STRING, name: Joi.string().required(), // this is the base permission ID (for now a built in) - permissionId: Joi.string().valid(...Object.values(BUILTIN_PERMISSION_IDS)).required(), + permissionId: Joi.string().valid(...Object.values(BuiltinPermissionID)).required(), permissions: Joi.object() .pattern(/.*/, [Joi.string().valid(...permLevelArray)]) .optional(), @@ -149,7 +149,7 @@ exports.roleValidator = () => { } exports.permissionValidator = () => { - const permLevelArray = Object.values(PermissionLevels) + const permLevelArray = Object.values(PermissionLevel) // prettier-ignore return joiValidator.params(Joi.object({ level: Joi.string().valid(...permLevelArray).required(), diff --git a/packages/server/src/api/routes/view.js b/packages/server/src/api/routes/view.js index a7045f0814..0a8bd7d4ce 100644 --- a/packages/server/src/api/routes/view.js +++ b/packages/server/src/api/routes/view.js @@ -5,8 +5,8 @@ const authorized = require("../../middleware/authorized") const { paramResource } = require("../../middleware/resourceId") const { BUILDER, - PermissionTypes, - PermissionLevels, + PermissionType, + PermissionLevel, } = require("@budibase/backend-core/permissions") const router = new Router() @@ -16,7 +16,7 @@ router .get( "/api/views/:viewName", paramResource("viewName"), - authorized(PermissionTypes.VIEW, PermissionLevels.READ), + authorized(PermissionType.VIEW, PermissionLevel.READ), rowController.fetchView ) .get("/api/views", authorized(BUILDER), viewController.fetch) diff --git a/packages/server/src/middleware/authorized.ts b/packages/server/src/middleware/authorized.ts index 1fa983a72a..9c870208a7 100644 --- a/packages/server/src/middleware/authorized.ts +++ b/packages/server/src/middleware/authorized.ts @@ -4,8 +4,8 @@ import { BUILTIN_ROLE_IDS, } from "@budibase/backend-core/roles" const { - PermissionTypes, - PermissionLevels, + PermissionType, + PermissionLevel, doesHaveBasePermission, } = require("@budibase/backend-core/permissions") const builderMiddleware = require("./builder") @@ -33,7 +33,7 @@ const checkAuthorized = async ( ) => { // check if this is a builder api and the user is not a builder const isBuilder = ctx.user && ctx.user.builder && ctx.user.builder.global - const isBuilderApi = permType === PermissionTypes.BUILDER + const isBuilderApi = permType === PermissionType.BUILDER if (isBuilderApi && !isBuilder) { return ctx.throw(403, "Not Authorized") } @@ -91,9 +91,9 @@ export = (permType: any, permLevel: any = null, opts = { schema: false }) => let resourceRoles: any = [] let otherLevelRoles: any = [] const otherLevel = - permLevel === PermissionLevels.READ - ? PermissionLevels.WRITE - : PermissionLevels.READ + permLevel === PermissionLevel.READ + ? PermissionLevel.WRITE + : PermissionLevel.READ const appId = getAppId() if (appId && hasResource(ctx)) { resourceRoles = await getRequiredResourceRole(permLevel, ctx) diff --git a/packages/server/src/middleware/builder.ts b/packages/server/src/middleware/builder.ts index b53f2903b7..529818a916 100644 --- a/packages/server/src/middleware/builder.ts +++ b/packages/server/src/middleware/builder.ts @@ -70,7 +70,7 @@ export = async function builder(ctx: BBContext, permType: string) { if (!appId) { return } - const isBuilderApi = permType === permissions.PermissionTypes.BUILDER + const isBuilderApi = permType === permissions.PermissionType.BUILDER const referer = ctx.headers["referer"] const overviewPath = "/builder/portal/overview/" diff --git a/packages/server/src/middleware/tests/authorized.spec.js b/packages/server/src/middleware/tests/authorized.spec.js index f23eb6206b..c64f758749 100644 --- a/packages/server/src/middleware/tests/authorized.spec.js +++ b/packages/server/src/middleware/tests/authorized.spec.js @@ -9,7 +9,7 @@ jest.mock("../../environment", () => ({ ) const authorizedMiddleware = require("../authorized") const env = require("../../environment") -const { PermissionTypes, PermissionLevels } = require("@budibase/backend-core/permissions") +const { PermissionType, PermissionLevel } = require("@budibase/backend-core/permissions") const { doInAppContext } = require("@budibase/backend-core/context") const APP_ID = "" @@ -113,7 +113,7 @@ describe("Authorization middleware", () => { it("throws if the user does not have builder permissions", async () => { config.setEnvironment(false) - config.setMiddlewareRequiredPermission(PermissionTypes.BUILDER) + config.setMiddlewareRequiredPermission(PermissionType.BUILDER) config.setUser({ role: { _id: "" @@ -125,13 +125,13 @@ describe("Authorization middleware", () => { }) it("passes on to next() middleware if the user has resource permission", async () => { - config.setResourceId(PermissionTypes.QUERY) + config.setResourceId(PermissionType.QUERY) config.setUser({ role: { _id: "" } }) - config.setMiddlewareRequiredPermission(PermissionTypes.QUERY) + config.setMiddlewareRequiredPermission(PermissionType.QUERY) await config.executeMiddleware() expect(config.next).toHaveBeenCalled() @@ -155,7 +155,7 @@ describe("Authorization middleware", () => { _id: "" }, }) - config.setMiddlewareRequiredPermission(PermissionTypes.ADMIN, PermissionLevels.BASIC) + config.setMiddlewareRequiredPermission(PermissionType.ADMIN, PermissionLevel.BASIC) await config.executeMiddleware() expect(config.throw).toHaveBeenCalledWith(403, "User does not have permission") diff --git a/packages/server/src/tests/utilities/structures.js b/packages/server/src/tests/utilities/structures.js index c4bd6fc774..39361c5d32 100644 --- a/packages/server/src/tests/utilities/structures.js +++ b/packages/server/src/tests/utilities/structures.js @@ -1,5 +1,5 @@ const { BUILTIN_ROLE_IDS } = require("@budibase/backend-core/roles") -const { BUILTIN_PERMISSION_IDS } = require("@budibase/backend-core/permissions") +const { BuiltinPermissionID } = require("@budibase/backend-core/permissions") const { createHomeScreen } = require("../../constants/screens") const { EMPTY_LAYOUT } = require("../../constants/layouts") const { cloneDeep } = require("lodash/fp") @@ -135,7 +135,7 @@ exports.basicRole = () => { return { name: "NewRole", inherits: BUILTIN_ROLE_IDS.BASIC, - permissionId: BUILTIN_PERMISSION_IDS.READ_ONLY, + permissionId: BuiltinPermissionID.READ_ONLY, } } diff --git a/packages/server/src/utilities/security.js b/packages/server/src/utilities/security.js index d8133e585b..34d31ce8d0 100644 --- a/packages/server/src/utilities/security.js +++ b/packages/server/src/utilities/security.js @@ -1,6 +1,6 @@ const { - PermissionLevels, - PermissionTypes, + PermissionLevel, + PermissionType, getBuiltinPermissionByID, isPermissionLevelHigherThanRead, } = require("@budibase/backend-core/permissions") @@ -11,9 +11,9 @@ const { const { DocumentType } = require("../db/utils") const CURRENTLY_SUPPORTED_LEVELS = [ - PermissionLevels.WRITE, - PermissionLevels.READ, - PermissionLevels.EXECUTE, + PermissionLevel.WRITE, + PermissionLevel.READ, + PermissionLevel.EXECUTE, ] exports.getPermissionType = resourceId => { @@ -23,17 +23,17 @@ exports.getPermissionType = resourceId => { switch (docType) { case DocumentType.TABLE: case DocumentType.ROW: - return PermissionTypes.TABLE + return PermissionType.TABLE case DocumentType.AUTOMATION: - return PermissionTypes.AUTOMATION + return PermissionType.AUTOMATION case DocumentType.WEBHOOK: - return PermissionTypes.WEBHOOK + return PermissionType.WEBHOOK case DocumentType.QUERY: case DocumentType.DATASOURCE: - return PermissionTypes.QUERY + return PermissionType.QUERY default: // views don't have an ID, will end up here - return PermissionTypes.VIEW + return PermissionType.VIEW } } @@ -58,8 +58,8 @@ exports.getBasePermissions = resourceId => { const level = typedPermission.level permissions[level] = lowerBuiltinRoleID(permissions[level], roleId) if (isPermissionLevelHigherThanRead(level)) { - permissions[PermissionLevels.READ] = lowerBuiltinRoleID( - permissions[PermissionLevels.READ], + permissions[PermissionLevel.READ] = lowerBuiltinRoleID( + permissions[PermissionLevel.READ], roleId ) } From 0d13a77409642893589742b80213b6299a25135d Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 17 Nov 2022 16:30:38 +0000 Subject: [PATCH 24/24] Some minor updates after running qa-core tests, found some issues that likely exists before context update. --- packages/backend-core/src/db/utils.ts | 4 +- .../server/src/api/controllers/application.ts | 63 +++++++++---------- .../api/controllers/public/applications.ts | 24 ++++--- packages/types/src/documents/app/app.ts | 3 +- .../applications/applications.spec.ts | 2 +- qa-core/yarn.lock | 35 ++++++----- 6 files changed, 68 insertions(+), 63 deletions(-) diff --git a/packages/backend-core/src/db/utils.ts b/packages/backend-core/src/db/utils.ts index 076552beec..9920be7e55 100644 --- a/packages/backend-core/src/db/utils.ts +++ b/packages/backend-core/src/db/utils.ts @@ -26,7 +26,7 @@ export * from "./tenancy" * Generates a new app ID. * @returns {string} The new app ID which the app doc can be stored under. */ -export const generateAppID = (tenantId = null) => { +export const generateAppID = (tenantId?: string | null) => { let id = APP_PREFIX if (tenantId) { id += `${tenantId}${SEPARATOR}` @@ -251,7 +251,7 @@ export function generateRoleID(id: any) { /** * Gets parameters for retrieving a role, this is a utility function for the getDocParams function. */ -export function getRoleParams(roleId = null, otherProps = {}) { +export function getRoleParams(roleId?: string | null, otherProps = {}) { return getDocParams(DocumentType.ROLE, roleId, otherProps) } diff --git a/packages/server/src/api/controllers/application.ts b/packages/server/src/api/controllers/application.ts index b97e980a5e..dcd092d06a 100644 --- a/packages/server/src/api/controllers/application.ts +++ b/packages/server/src/api/controllers/application.ts @@ -14,17 +14,16 @@ import { DocumentType, AppStatus, } from "../../db/utils" -const { - BUILTIN_ROLE_IDS, - AccessController, -} = require("@budibase/backend-core/roles") -const { CacheKeys, bustCache } = require("@budibase/backend-core/cache") -const { - getAllApps, - isDevAppID, - getProdAppID, - Replication, -} = require("@budibase/backend-core/db") +import { + db as dbCore, + roles, + cache, + tenancy, + context, + errors, + events, + migrations, +} from "@budibase/backend-core" import { USERS_TABLE_SCHEMA } from "../../constants" import { removeAppFromUserRoles } from "../../utilities/workerRequests" import { clientLibraryPath, stringToReadStream } from "../../utilities" @@ -34,15 +33,11 @@ import { backupClientLibrary, revertClientLibrary, } from "../../utilities/fileSystem/clientLibrary" -const { getTenantId, isMultiTenant } = require("@budibase/backend-core/tenancy") import { syncGlobalUsers } from "./user" -const { app: appCache } = require("@budibase/backend-core/cache") import { cleanupAutomations } from "../../automations/utils" -import { context } from "@budibase/backend-core" import { checkAppMetadata } from "../../automations/logging" import { getUniqueRows } from "../../utilities/usageQuota/rows" import { quotas, groups } from "@budibase/pro" -import { errors, events, migrations } from "@budibase/backend-core" import { App, Layout, Screen, MigrationType } from "@budibase/types" import { BASE_LAYOUT_PROP_IDS } from "../../constants/layouts" import { enrichPluginURLs } from "../../utilities/plugins" @@ -75,7 +70,7 @@ async function getScreens() { function getUserRoleId(ctx: any) { return !ctx.user.role || !ctx.user.role._id - ? BUILTIN_ROLE_IDS.PUBLIC + ? roles.BUILTIN_ROLE_IDS.PUBLIC : ctx.user.role._id } @@ -123,7 +118,7 @@ const checkAppName = ( } async function createInstance(template: any) { - const tenantId = isMultiTenant() ? getTenantId() : null + const tenantId = tenancy.isMultiTenant() ? tenancy.getTenantId() : null const baseAppId = generateAppID(tenantId) const appId = generateDevAppID(baseAppId) await context.updateAppId(appId) @@ -162,7 +157,7 @@ async function createInstance(template: any) { export const fetch = async (ctx: any) => { const dev = ctx.query && ctx.query.status === AppStatus.DEV const all = ctx.query && ctx.query.status === AppStatus.ALL - const apps = await getAllApps({ dev, all }) + const apps = (await dbCore.getAllApps({ dev, all })) as App[] const appIds = apps .filter((app: any) => app.status === "development") @@ -187,7 +182,7 @@ export const fetch = async (ctx: any) => { export const fetchAppDefinition = async (ctx: any) => { const layouts = await getLayouts() const userRoleId = getUserRoleId(ctx) - const accessController = new AccessController() + const accessController = new roles.AccessController() const screens = await accessController.checkScreensAccess( await getScreens(), userRoleId @@ -211,7 +206,7 @@ export const fetchAppPackage = async (ctx: any) => { // Only filter screens if the user is not a builder if (!(ctx.user.builder && ctx.user.builder.global)) { const userRoleId = getUserRoleId(ctx) - const accessController = new AccessController() + const accessController = new roles.AccessController() screens = await accessController.checkScreensAccess(screens, userRoleId) } @@ -224,7 +219,7 @@ export const fetchAppPackage = async (ctx: any) => { } const performAppCreate = async (ctx: any) => { - const apps = await getAllApps({ dev: true }) + const apps = await dbCore.getAllApps({ dev: true }) const name = ctx.request.body.name checkAppName(ctx, apps, name) const url = getAppUrl(ctx) @@ -254,7 +249,7 @@ const performAppCreate = async (ctx: any) => { url: url, template: templateKey, instance, - tenantId: getTenantId(), + tenantId: tenancy.getTenantId(), updatedAt: new Date().toISOString(), createdAt: new Date().toISOString(), status: AppStatus.DEV, @@ -313,7 +308,7 @@ const performAppCreate = async (ctx: any) => { await createApp(appId) } - await appCache.invalidateAppMetadata(appId, newApplication) + await cache.app.invalidateAppMetadata(appId, newApplication) return newApplication } @@ -343,7 +338,7 @@ const creationEvents = async (request: any, app: App) => { } const appPostCreate = async (ctx: any, app: App) => { - const tenantId = getTenantId() + const tenantId = tenancy.getTenantId() await migrations.backPopulateMigrations({ type: MigrationType.APP, tenantId, @@ -356,7 +351,9 @@ const appPostCreate = async (ctx: any, app: App) => { const rowCount = rows ? rows.length : 0 if (rowCount) { try { - await quotas.addRows(rowCount) + await context.doInAppContext(app.appId, () => { + return quotas.addRows(rowCount) + }) } catch (err: any) { if (err.code && err.code === errors.codes.USAGE_LIMIT_EXCEEDED) { // this import resulted in row usage exceeding the quota @@ -374,7 +371,7 @@ const appPostCreate = async (ctx: any, app: App) => { export const create = async (ctx: any) => { const newApplication = await quotas.addApp(() => performAppCreate(ctx)) await appPostCreate(ctx, newApplication) - await bustCache(CacheKeys.CHECKLIST) + await cache.bustCache(cache.CacheKeys.CHECKLIST) ctx.body = newApplication ctx.status = 200 } @@ -382,7 +379,7 @@ export const create = async (ctx: any) => { // This endpoint currently operates as a PATCH rather than a PUT // Thus name and url fields are handled only if present export const update = async (ctx: any) => { - const apps = await getAllApps({ dev: true }) + const apps = await dbCore.getAllApps({ dev: true }) // validation const name = ctx.request.body.name if (name) { @@ -455,7 +452,7 @@ const destroyApp = async (ctx: any) => { let isUnpublish = ctx.query && ctx.query.unpublish if (isUnpublish) { - appId = getProdAppID(appId) + appId = dbCore.getProdAppID(appId) } const db = isUnpublish ? context.getProdAppDB() : context.getAppDB() @@ -481,7 +478,7 @@ const destroyApp = async (ctx: any) => { else { await removeAppFromUserRoles(ctx, appId) } - await appCache.invalidateAppMetadata(appId) + await cache.app.invalidateAppMetadata(appId) return result } @@ -517,12 +514,12 @@ export const sync = async (ctx: any, next: any) => { } const appId = ctx.params.appId - if (!isDevAppID(appId)) { + if (!dbCore.isDevAppID(appId)) { ctx.throw(400, "This action cannot be performed for production apps") } // replicate prod to dev - const prodAppId = getProdAppID(appId) + const prodAppId = dbCore.getProdAppID(appId) // specific case, want to make sure setup is skipped const prodDb = context.getProdAppDB({ skip_setup: true }) @@ -536,7 +533,7 @@ export const sync = async (ctx: any, next: any) => { return next() } - const replication = new Replication({ + const replication = new dbCore.Replication({ source: prodAppId, target: appId, }) @@ -577,7 +574,7 @@ export const updateAppPackage = async (appPackage: any, appId: any) => { await db.put(newAppPackage) // remove any cached metadata, so that it will be updated - await appCache.invalidateAppMetadata(appId) + await cache.app.invalidateAppMetadata(appId) return newAppPackage }) } diff --git a/packages/server/src/api/controllers/public/applications.ts b/packages/server/src/api/controllers/public/applications.ts index c2c62ffc28..c4d41fe5fb 100644 --- a/packages/server/src/api/controllers/public/applications.ts +++ b/packages/server/src/api/controllers/public/applications.ts @@ -1,5 +1,4 @@ -const { getAllApps } = require("@budibase/backend-core/db") -const { doInAppContext } = require("@budibase/backend-core/context") +import { db as dbCore, context } from "@budibase/backend-core" import { search as stringSearch, addRev } from "./utils" import * as controller from "../application" import { Application } from "../../../definitions/common" @@ -15,15 +14,22 @@ function fixAppID(app: Application, params: any) { } async function setResponseApp(ctx: any) { - if (ctx.body && ctx.body.appId && (!ctx.params || !ctx.params.appId)) { - ctx.params = { appId: ctx.body.appId } + const appId = ctx.body?.appId + if (appId && (!ctx.params || !ctx.params.appId)) { + ctx.params = { appId } + } + if (appId) { + await context.doInContext(appId, () => { + return controller.fetchAppPackage(ctx) + }) + } else { + return controller.fetchAppPackage(ctx) } - await controller.fetchAppPackage(ctx) } export async function search(ctx: any, next: any) { const { name } = ctx.request.body - const apps = await getAllApps({ all: true }) + const apps = await dbCore.getAllApps({ all: true }) ctx.body = stringSearch(apps, name) await next() } @@ -41,7 +47,7 @@ export async function create(ctx: any, next: any) { } export async function read(ctx: any, next: any) { - await doInAppContext(ctx.params.appId, async () => { + await context.doInAppContext(ctx.params.appId, async () => { await setResponseApp(ctx) await next() }) @@ -49,7 +55,7 @@ export async function read(ctx: any, next: any) { export async function update(ctx: any, next: any) { ctx.request.body = await addRev(fixAppID(ctx.request.body, ctx.params)) - await doInAppContext(ctx.params.appId, async () => { + await context.doInAppContext(ctx.params.appId, async () => { await controller.update(ctx) await setResponseApp(ctx) await next() @@ -57,7 +63,7 @@ export async function update(ctx: any, next: any) { } export async function destroy(ctx: any, next: any) { - await doInAppContext(ctx.params.appId, async () => { + await context.doInAppContext(ctx.params.appId, async () => { // get the app before deleting it await setResponseApp(ctx) const body = ctx.body diff --git a/packages/types/src/documents/app/app.ts b/packages/types/src/documents/app/app.ts index a03875aa50..c91d575714 100644 --- a/packages/types/src/documents/app/app.ts +++ b/packages/types/src/documents/app/app.ts @@ -1,4 +1,4 @@ -import { Document } from "../document" +import { User, Document } from "../" export type AppMetadataErrors = { [key: string]: string[] } @@ -16,6 +16,7 @@ export interface App extends Document { theme?: string customTheme?: AppCustomTheme revertableVersion?: string + lockedBy?: User navigation?: AppNavigation automationErrors?: AppMetadataErrors icon?: AppIcon diff --git a/qa-core/src/tests/internal-api/applications/applications.spec.ts b/qa-core/src/tests/internal-api/applications/applications.spec.ts index 4b3208ee10..98895a6f28 100644 --- a/qa-core/src/tests/internal-api/applications/applications.spec.ts +++ b/qa-core/src/tests/internal-api/applications/applications.spec.ts @@ -70,7 +70,7 @@ describe("Internal API - Application creation, update, publish and delete", () = await config.applications.publish(app.url) // check published app renders - config.applications.api.appId = db.getProdAppID(app.appId) + config.applications.api.appId = db.getProdAppID(app.appId!) await config.applications.canRender() // unpublish app diff --git a/qa-core/yarn.lock b/qa-core/yarn.lock index 5b86c6084f..b090ff872f 100644 --- a/qa-core/yarn.lock +++ b/qa-core/yarn.lock @@ -390,7 +390,7 @@ jest-util "^28.1.3" slash "^3.0.0" -"@jest/core@^28.0.2", "@jest/core@^28.1.3": +"@jest/core@^28.1.1", "@jest/core@^28.1.3": version "28.1.3" resolved "https://registry.yarnpkg.com/@jest/core/-/core-28.1.3.tgz#0ebf2bd39840f1233cd5f2d1e6fc8b71bd5a1ac7" integrity sha512-CIKBrlaKOzA7YG19BEqCw3SLIsEwjZkeJzf5bdooVnW4bH5cktqe3JX+G2YV1aK5vP8N9na1IGWFzYaTp6k6NA== @@ -573,7 +573,7 @@ slash "^3.0.0" write-file-atomic "^4.0.1" -"@jest/types@^28.1.3": +"@jest/types@^28.1.1", "@jest/types@^28.1.3": version "28.1.3" resolved "https://registry.yarnpkg.com/@jest/types/-/types-28.1.3.tgz#b05de80996ff12512bc5ceb1d208285a7d11748b" integrity sha512-RyjiyMUZrKz/c+zlMFO1pm70DcIlST8AeWTkoUdZevew44wcNZQHsEVOiCVtgVnlFFD82FPaXycys58cf2muVQ== @@ -2299,7 +2299,7 @@ jest-circus@^28.1.3: slash "^3.0.0" stack-utils "^2.0.3" -jest-cli@^28.0.2: +jest-cli@^28.1.1: version "28.1.3" resolved "https://registry.yarnpkg.com/jest-cli/-/jest-cli-28.1.3.tgz#558b33c577d06de55087b8448d373b9f654e46b2" integrity sha512-roY3kvrv57Azn1yPgdTebPAXvdR2xfezaKKYzVxZ6It/5NCxzJym6tUI5P1zkdWhfUYkxEI9uZWcQdaFLo8mJQ== @@ -2666,14 +2666,15 @@ jest-worker@^28.1.3: merge-stream "^2.0.0" supports-color "^8.0.0" -jest@28.0.2: - version "28.0.2" - resolved "https://registry.yarnpkg.com/jest/-/jest-28.0.2.tgz#41385ca21d009708bb9fc65e08663110e08e2cc0" - integrity sha512-COUtjybolW4koQvO7kCfq5kgbeeU5WbSJfVZprz4zbS8AL32+RAZZTUjBEyRRdpsXqss/pHIvSL7/P+LyMYHXg== +jest@28.1.1: + version "28.1.1" + resolved "https://registry.yarnpkg.com/jest/-/jest-28.1.1.tgz#3c39a3a09791e16e9ef283597d24ab19a0df701e" + integrity sha512-qw9YHBnjt6TCbIDMPMpJZqf9E12rh6869iZaN08/vpOGgHJSAaLLUn6H8W3IAEuy34Ls3rct064mZLETkxJ2XA== dependencies: - "@jest/core" "^28.0.2" + "@jest/core" "^28.1.1" + "@jest/types" "^28.1.1" import-local "^3.0.2" - jest-cli "^28.0.2" + jest-cli "^28.1.1" jmespath@0.15.0: version "0.15.0" @@ -4351,10 +4352,10 @@ ts-jest@28.0.8: semver "7.x" yargs-parser "^21.0.1" -ts-node@10.9.1: - version "10.9.1" - resolved "https://registry.yarnpkg.com/ts-node/-/ts-node-10.9.1.tgz#e73de9102958af9e1f0b168a6ff320e25adcff4b" - integrity sha512-NtVysVPkxxrwFGUUxGYhfux8k78pQB3JqYBXlLRZgdGUqTO5wU/UyHop5p70iEbGhB7q5KmiZiU0Y3KlJrScEw== +ts-node@10.8.1: + version "10.8.1" + resolved "https://registry.yarnpkg.com/ts-node/-/ts-node-10.8.1.tgz#ea2bd3459011b52699d7e88daa55a45a1af4f066" + integrity sha512-Wwsnao4DQoJsN034wePSg5nZiw4YKXf56mPIAeD6wVmiv+RytNSWqc2f3fKvcUoV+Yn2+yocD71VOfQHbmVX4g== dependencies: "@cspotcode/source-map-support" "^0.8.0" "@tsconfig/node10" "^1.0.7" @@ -4370,10 +4371,10 @@ ts-node@10.9.1: v8-compile-cache-lib "^3.0.1" yn "3.1.1" -tsconfig-paths@4.1.0: - version "4.1.0" - resolved "https://registry.yarnpkg.com/tsconfig-paths/-/tsconfig-paths-4.1.0.tgz#f8ef7d467f08ae3a695335bf1ece088c5538d2c1" - integrity sha512-AHx4Euop/dXFC+Vx589alFba8QItjF+8hf8LtmuiCwHyI4rHXQtOOENaM8kvYf5fR0dRChy3wzWIZ9WbB7FWow== +tsconfig-paths@4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/tsconfig-paths/-/tsconfig-paths-4.0.0.tgz#1082f5d99fd127b72397eef4809e4dd06d229b64" + integrity sha512-SLBg2GBKlR6bVtMgJJlud/o3waplKtL7skmLkExomIiaAtLGtVsoXIqP3SYdjbcH9lq/KVv7pMZeCBpLYOit6Q== dependencies: json5 "^2.2.1" minimist "^1.2.6"