From 2426123686c95a80d671c898cbcdaccf7f1c1592 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 7 Nov 2022 18:27:39 +0000 Subject: [PATCH 01/91] First version of pouch-like system, a wrapper for Couch nano. --- packages/backend-core/package.json | 1 + packages/backend-core/src/couch/couch.ts | 35 +++++ packages/backend-core/src/couch/index.ts | 3 + packages/backend-core/src/couch/pouchLike.ts | 133 ++++++++++++++++++ packages/backend-core/src/couch/utils.ts | 36 +++++ packages/backend-core/src/db/index.ts | 29 +--- packages/backend-core/src/db/pouch.ts | 35 +---- packages/backend-core/yarn.lock | 137 ++++++++++++++++++- packages/types/src/documents/document.ts | 4 + 9 files changed, 351 insertions(+), 62 deletions(-) create mode 100644 packages/backend-core/src/couch/couch.ts create mode 100644 packages/backend-core/src/couch/index.ts create mode 100644 packages/backend-core/src/couch/pouchLike.ts create mode 100644 packages/backend-core/src/couch/utils.ts diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json index 43e34142dc..d8575dcad3 100644 --- a/packages/backend-core/package.json +++ b/packages/backend-core/package.json @@ -35,6 +35,7 @@ "koa-passport": "4.1.4", "lodash": "4.17.21", "lodash.isarguments": "3.1.0", + "nano": "^10.1.0", "node-fetch": "2.6.7", "passport-google-auth": "1.0.2", "passport-google-oauth": "2.0.0", diff --git a/packages/backend-core/src/couch/couch.ts b/packages/backend-core/src/couch/couch.ts new file mode 100644 index 0000000000..41fb83cb86 --- /dev/null +++ b/packages/backend-core/src/couch/couch.ts @@ -0,0 +1,35 @@ +import env from "../environment" +import { getUrlInfo } from "../db/pouch" + +export const getCouchInfo = () => { + const urlInfo = getUrlInfo() + let username + let password + if (env.COUCH_DB_USERNAME) { + // set from env + username = env.COUCH_DB_USERNAME + } else if (urlInfo.auth.username) { + // set from url + username = urlInfo.auth.username + } else if (!env.isTest()) { + throw new Error("CouchDB username not set") + } + if (env.COUCH_DB_PASSWORD) { + // set from env + password = env.COUCH_DB_PASSWORD + } else if (urlInfo.auth.password) { + // set from url + password = urlInfo.auth.password + } else if (!env.isTest()) { + throw new Error("CouchDB password not set") + } + const authCookie = Buffer.from(`${username}:${password}`).toString("base64") + return { + url: urlInfo.url!, + auth: { + username: username, + password: password, + }, + cookie: `Basic ${authCookie}`, + } +} diff --git a/packages/backend-core/src/couch/index.ts b/packages/backend-core/src/couch/index.ts new file mode 100644 index 0000000000..daba8f74a9 --- /dev/null +++ b/packages/backend-core/src/couch/index.ts @@ -0,0 +1,3 @@ +export * from "./couch" +export * from "./pouchLike" +export * from "./utils" diff --git a/packages/backend-core/src/couch/pouchLike.ts b/packages/backend-core/src/couch/pouchLike.ts new file mode 100644 index 0000000000..80ebe1d363 --- /dev/null +++ b/packages/backend-core/src/couch/pouchLike.ts @@ -0,0 +1,133 @@ +import Nano from "nano" +import { AnyDocument } from "@budibase/types" +import { getCouchInfo } from "./couch" +import { directCouchCall } from "./utils" +import { ReadStream, WriteStream } from "fs" +import { dangerousGetDB } from "../db" + +export type PouchLikeOpts = { + skip_setup?: boolean +} + +export type QueryOpts = { + include_docs?: boolean + startkey?: string + endkey?: string + limit?: number + skip?: number + descending?: boolean + key?: string + keys?: string[] +} + +export type DumpOpts = { + filter?: (doc: AnyDocument) => boolean +} + +export class PouchLike { + public readonly name: string + private static nano: Nano.ServerScope + private readonly pouchOpts: PouchLikeOpts + + constructor(dbName: string, opts?: PouchLikeOpts) { + this.name = dbName + this.pouchOpts = opts || {} + } + + static init() { + const couchInfo = getCouchInfo() + this.nano = Nano({ + url: couchInfo.url, + cookie: couchInfo.cookie, + }) + } + + async checkSetup() { + let shouldCreate = !this.pouchOpts?.skip_setup + // check exists in a lightweight fashion + let response = await directCouchCall(`/${this.name}`, "HEAD") + let exists = response.status === 200 + if (!shouldCreate && !exists) { + throw new Error("DB does not exist") + } + if (!exists) { + await PouchLike.nano.db.create(this.name) + } + return PouchLike.nano.db.use(this.name) + } + + async info() {} + + async get(id: string) { + const db = await this.checkSetup() + return await db.get(id) + } + + async remove(id: string, rev: string) { + const db = await this.checkSetup() + return await db.destroy(id, rev) + } + + async put(document: AnyDocument) { + if (!document._id) { + throw new Error("Cannot store document without _id field.") + } + const db = await this.checkSetup() + return await db.insert(document) + } + + async bulkDocs(documents: AnyDocument[]) { + const db = await this.checkSetup() + return await db.bulk({ docs: documents }) + } + + async allDocs(params: QueryOpts) { + const db = await this.checkSetup() + return await db.fetch({ keys: [] }, params) + } + + async query(viewName: string, params: QueryOpts) { + const db = await this.checkSetup() + const [database, view] = viewName.split("/") + return await db.view(database, view, params) + } + + async destroy() { + try { + await PouchLike.nano.db.destroy(this.name) + } catch (err: any) { + // didn't exist, don't worry + if (err.status === 404) { + return + } else { + throw err + } + } + } + + async compact() { + const db = await this.checkSetup() + return await db.compact() + } + + // utilise PouchDB for this + async dump(stream: WriteStream, params: DumpOpts) { + const pouch = dangerousGetDB(this.name) + // @ts-ignore + return pouch.dump(stream, params) + } + + // utilise PouchDB for this + async load(stream: ReadStream) { + const pouch = dangerousGetDB(this.name) + // @ts-ignore + return pouch.load(stream) + } + + // pouch specific functions - indexes come from the pouchdb-find library + async createIndex() {} + + async deleteIndex() {} + + async getIndexes() {} +} diff --git a/packages/backend-core/src/couch/utils.ts b/packages/backend-core/src/couch/utils.ts new file mode 100644 index 0000000000..25a4643d54 --- /dev/null +++ b/packages/backend-core/src/couch/utils.ts @@ -0,0 +1,36 @@ +import { getCouchInfo } from "./couch" +import fetch from "node-fetch" +import { checkSlashesInUrl } from "../helpers" + +export async function directCouchCall( + path: string, + method: string = "GET", + body?: any +) { + let { url, cookie } = getCouchInfo() + const couchUrl = `${url}/${path}` + const params: any = { + method: method, + headers: { + Authorization: cookie, + }, + } + if (body && method !== "GET") { + params.body = JSON.stringify(body) + params.headers["Content-Type"] = "application/json" + } + return await fetch(checkSlashesInUrl(encodeURI(couchUrl)), params) +} + +export async function directCouchQuery( + path: string, + method: string = "GET", + body?: any +) { + const response = await directCouchCall(path, method, body) + if (response.status < 300) { + return await response.json() + } else { + throw "Cannot connect to CouchDB instance" + } +} diff --git a/packages/backend-core/src/db/index.ts b/packages/backend-core/src/db/index.ts index 429cd61fc1..9f6c097f0d 100644 --- a/packages/backend-core/src/db/index.ts +++ b/packages/backend-core/src/db/index.ts @@ -1,9 +1,9 @@ import * as pouch from "./pouch" import env from "../environment" -import { checkSlashesInUrl } from "../helpers" -import fetch from "node-fetch" import { PouchOptions, CouchFindOptions } from "@budibase/types" import PouchDB from "pouchdb" +import { directCouchQuery } from "../couch" +export { directCouchQuery } from "../couch" const openDbs: string[] = [] let Pouch: any @@ -94,31 +94,6 @@ export function allDbs() { return [...dbList] } -export async function directCouchQuery( - path: string, - method: string = "GET", - body?: any -) { - let { url, cookie } = pouch.getCouchInfo() - const couchUrl = `${url}/${path}` - const params: any = { - method: method, - headers: { - Authorization: cookie, - }, - } - if (body && method !== "GET") { - params.body = JSON.stringify(body) - params.headers["Content-Type"] = "application/json" - } - const response = await fetch(checkSlashesInUrl(encodeURI(couchUrl)), params) - if (response.status < 300) { - return await response.json() - } else { - throw "Cannot connect to CouchDB instance" - } -} - export async function directCouchAllDbs(queryString?: string) { let couchPath = "/_all_dbs" if (queryString) { diff --git a/packages/backend-core/src/db/pouch.ts b/packages/backend-core/src/db/pouch.ts index 1e37da9240..6eca07e0cd 100644 --- a/packages/backend-core/src/db/pouch.ts +++ b/packages/backend-core/src/db/pouch.ts @@ -1,5 +1,7 @@ import PouchDB from "pouchdb" import env from "../environment" +import { getCouchInfo } from "../couch" +export { getCouchInfo } from "../couch" export const getUrlInfo = (url = env.COUCH_DB_URL) => { let cleanUrl, username, password, host @@ -44,39 +46,6 @@ export const getUrlInfo = (url = env.COUCH_DB_URL) => { } } -export const getCouchInfo = () => { - const urlInfo = getUrlInfo() - let username - let password - if (env.COUCH_DB_USERNAME) { - // set from env - username = env.COUCH_DB_USERNAME - } else if (urlInfo.auth.username) { - // set from url - username = urlInfo.auth.username - } else if (!env.isTest()) { - throw new Error("CouchDB username not set") - } - if (env.COUCH_DB_PASSWORD) { - // set from env - password = env.COUCH_DB_PASSWORD - } else if (urlInfo.auth.password) { - // set from url - password = urlInfo.auth.password - } else if (!env.isTest()) { - throw new Error("CouchDB password not set") - } - const authCookie = Buffer.from(`${username}:${password}`).toString("base64") - return { - url: urlInfo.url, - auth: { - username: username, - password: password, - }, - cookie: `Basic ${authCookie}`, - } -} - /** * Return a constructor for PouchDB. * This should be rarely used outside of the main application config. diff --git a/packages/backend-core/yarn.lock b/packages/backend-core/yarn.lock index d301526ba1..c7a66cfce1 100644 --- a/packages/backend-core/yarn.lock +++ b/packages/backend-core/yarn.lock @@ -291,6 +291,11 @@ resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw== +"@budibase/types@2.1.14-alpha.2": + version "2.1.14-alpha.2" + resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.1.14-alpha.2.tgz#a537796012504e59afe06595f094c5ee2075de9b" + integrity sha512-6lrxQDBozX+yOWXBYN2K2Usg3liWXjmWtZ/F4Pky01dsFdD9M0GdYEUI0+Efhw78wEFaDeA2H9iXvuswKT/I6g== + "@hapi/hoek@^9.0.0": version "9.3.0" resolved "https://registry.yarnpkg.com/@hapi/hoek/-/hoek-9.3.0.tgz#8368869dcb735be2e7f5cb7647de78e167a251fb" @@ -1077,6 +1082,11 @@ dependencies: "@types/node" "*" +"@types/tough-cookie@^4.0.2": + version "4.0.2" + resolved "https://registry.yarnpkg.com/@types/tough-cookie/-/tough-cookie-4.0.2.tgz#6286b4c7228d58ab7866d19716f3696e03a09397" + integrity sha512-Q5vtl1W5ue16D+nIaW8JWebSSraJVlK+EthKn7e7UcD4KWsaSJ8BqGPXNaPghgtcn/fhvrN17Tv8ksUsQpiplw== + "@types/uuid@8.3.4": version "8.3.4" resolved "https://registry.yarnpkg.com/@types/uuid/-/uuid-8.3.4.tgz#bd86a43617df0594787d38b735f55c805becf1bc" @@ -1171,7 +1181,7 @@ acorn@^8.2.4: resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.7.1.tgz#0197122c843d1bf6d0a5e83220a788f278f63c30" integrity sha512-Xx54uLJQZ19lKygFXOWsscKUbsBZW0CPykPhVQdhIeIwrbPmJzqeASDInc8nKBnp/JT6igTs82qPXz069H8I/A== -agent-base@6: +agent-base@6, agent-base@^6.0.2: version "6.0.2" resolved "https://registry.yarnpkg.com/agent-base/-/agent-base-6.0.2.tgz#49fff58577cfee3f37176feab4c22e00f86d7f77" integrity sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ== @@ -1328,6 +1338,15 @@ axios@0.24.0: dependencies: follow-redirects "^1.14.4" +axios@^1.1.3: + version "1.1.3" + resolved "https://registry.yarnpkg.com/axios/-/axios-1.1.3.tgz#8274250dada2edf53814ed7db644b9c2866c1e35" + integrity sha512-00tXVRwKx/FZr/IDVFt4C+f9FYairX517WoGCL6dpOntqLkZofjhu43F/Xl44UOpqa+9sLFDrG/XAnFsUYgkDA== + dependencies: + follow-redirects "^1.15.0" + form-data "^4.0.0" + proxy-from-env "^1.1.0" + babel-jest@^27.5.1: version "27.5.1" resolved "https://registry.yarnpkg.com/babel-jest/-/babel-jest-27.5.1.tgz#a1bf8d61928edfefd21da27eb86a695bfd691444" @@ -1570,6 +1589,14 @@ cacheable-request@^6.0.0: normalize-url "^4.1.0" responselike "^1.0.2" +call-bind@^1.0.0: + version "1.0.2" + resolved "https://registry.yarnpkg.com/call-bind/-/call-bind-1.0.2.tgz#b1d4e89e688119c3c9a903ad30abb2f6a919be3c" + integrity sha512-7O+FbCihrB5WGbFYesctwmTKae6rOiIzmz1icreWJ+0aA7LJfuqhEso2T9ncpcFtzMQtzXf2QGGueWJGTYsqrA== + dependencies: + function-bind "^1.1.1" + get-intrinsic "^1.0.2" + callsites@^3.0.0: version "3.1.0" resolved "https://registry.yarnpkg.com/callsites/-/callsites-3.1.0.tgz#b3630abd8943432f54b3f0519238e33cd7df2f73" @@ -2292,6 +2319,11 @@ follow-redirects@^1.14.4: resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.1.tgz#0ca6a452306c9b276e4d3127483e29575e207ad5" integrity sha512-yLAMQs+k0b2m7cVxpS1VKJVvoz7SS9Td1zss3XRwXj+ZDH00RJgnuLx7E44wx02kQLrdM3aOOy+FpzS7+8OizA== +follow-redirects@^1.15.0: + version "1.15.2" + resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.2.tgz#b460864144ba63f2681096f274c4e57026da2c13" + integrity sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA== + forever-agent@~0.6.1: version "0.6.1" resolved "https://registry.yarnpkg.com/forever-agent/-/forever-agent-0.6.1.tgz#fbc71f0c41adeb37f96c577ad1ed42d8fdacca91" @@ -2306,6 +2338,15 @@ form-data@^3.0.0: combined-stream "^1.0.8" mime-types "^2.1.12" +form-data@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/form-data/-/form-data-4.0.0.tgz#93919daeaf361ee529584b9b31664dc12c9fa452" + integrity sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww== + dependencies: + asynckit "^0.4.0" + combined-stream "^1.0.8" + mime-types "^2.1.12" + form-data@~2.3.2: version "2.3.3" resolved "https://registry.yarnpkg.com/form-data/-/form-data-2.3.3.tgz#dcce52c05f644f298c6a7ab936bd724ceffbf3a6" @@ -2377,6 +2418,15 @@ get-caller-file@^2.0.5: resolved "https://registry.yarnpkg.com/get-caller-file/-/get-caller-file-2.0.5.tgz#4f94412a82db32f36e3b0b9741f8a97feb031f7e" integrity sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg== +get-intrinsic@^1.0.2: + version "1.1.3" + resolved "https://registry.yarnpkg.com/get-intrinsic/-/get-intrinsic-1.1.3.tgz#063c84329ad93e83893c7f4f243ef63ffa351385" + integrity sha512-QJVz1Tj7MS099PevUG5jvnt9tSkXN8K14dxQlikJuPt4uD9hHAHjLyLBiLR5zELelBdD9QNRAXZzsJx0WaDL9A== + dependencies: + function-bind "^1.1.1" + has "^1.0.3" + has-symbols "^1.0.3" + get-package-type@^0.1.0: version "0.1.0" resolved "https://registry.yarnpkg.com/get-package-type/-/get-package-type-0.1.0.tgz#8de2d803cff44df3bc6c456e6668b36c3926e11a" @@ -2525,7 +2575,7 @@ has-flag@^4.0.0: resolved "https://registry.yarnpkg.com/has-flag/-/has-flag-4.0.0.tgz#944771fd9c81c81265c4d6941860da06bb59479b" integrity sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ== -has-symbols@^1.0.2: +has-symbols@^1.0.2, has-symbols@^1.0.3: version "1.0.3" resolved "https://registry.yarnpkg.com/has-symbols/-/has-symbols-1.0.3.tgz#bb7b2c4349251dce87b125f7bdf874aa7c8b39f8" integrity sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A== @@ -2579,6 +2629,13 @@ http-cache-semantics@^4.0.0: resolved "https://registry.yarnpkg.com/http-cache-semantics/-/http-cache-semantics-4.1.0.tgz#49e91c5cbf36c9b94bcfcd71c23d5249ec74e390" integrity sha512-carPklcUh7ROWRK7Cv27RPtdhYhUsela/ue5/jKzjegVvXDqM2ILE9Q2BGn9JZJh1g87cp56su/FgQSzcWS8cQ== +http-cookie-agent@^4.0.2: + version "4.0.2" + resolved "https://registry.yarnpkg.com/http-cookie-agent/-/http-cookie-agent-4.0.2.tgz#dcdaae18ed1f7452d81ae4d5cd80b227d6831b69" + integrity sha512-noTmxdH5CuytTnLj/Qv3Z84e/YFq8yLXAw3pqIYZ25Edhb9pQErIAC+ednw40Cic6Le/h9ryph5/TqsvkOaUCw== + dependencies: + agent-base "^6.0.2" + http-errors@^1.6.3, http-errors@~1.8.0: version "1.8.1" resolved "https://registry.yarnpkg.com/http-errors/-/http-errors-1.8.1.tgz#7c3f28577cbc8a207388455dbd62295ed07bd68c" @@ -3984,6 +4041,18 @@ msgpackr@^1.5.2: optionalDependencies: msgpackr-extract "^2.1.2" +nano@^10.1.0: + version "10.1.0" + resolved "https://registry.yarnpkg.com/nano/-/nano-10.1.0.tgz#afdd5a7440e62f09a8e23f41fcea328d27383922" + integrity sha512-COeN2TpLcHuSN44QLnPmfZCoCsKAg8/aelPOVqqm/2/MvRHDEA11/Kld5C4sLzDlWlhFZ3SO2WGJGevCsvcEzQ== + dependencies: + "@types/tough-cookie" "^4.0.2" + axios "^1.1.3" + http-cookie-agent "^4.0.2" + node-abort-controller "^3.0.1" + qs "^6.11.0" + tough-cookie "^4.1.2" + napi-macros@~2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/napi-macros/-/napi-macros-2.0.0.tgz#2b6bae421e7b96eb687aa6c77a7858640670001b" @@ -4009,6 +4078,11 @@ negotiator@0.6.3: resolved "https://registry.yarnpkg.com/negotiator/-/negotiator-0.6.3.tgz#58e323a72fedc0d6f9cd4d31fe49f51479590ccd" integrity sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg== +node-abort-controller@^3.0.1: + version "3.0.1" + resolved "https://registry.yarnpkg.com/node-abort-controller/-/node-abort-controller-3.0.1.tgz#f91fa50b1dee3f909afabb7e261b1e1d6b0cb74e" + integrity sha512-/ujIVxthRs+7q6hsdjHMaj8hRG9NuWmwrz+JdRwZ14jdFoKSkm+vDsCbF9PLpnSqjaWQJuTmVtcWHNLr+vrOFw== + node-addon-api@^3.1.0: version "3.2.1" resolved "https://registry.yarnpkg.com/node-addon-api/-/node-addon-api-3.2.1.tgz#81325e0a2117789c0128dab65e7e38f07ceba161" @@ -4144,6 +4218,11 @@ object-assign@^4.1.1: resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-4.1.1.tgz#2109adc7965887cfc05cbbd442cac8bfbb360863" integrity sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg== +object-inspect@^1.9.0: + version "1.12.2" + resolved "https://registry.yarnpkg.com/object-inspect/-/object-inspect-1.12.2.tgz#c0641f26394532f28ab8d796ab954e43c009a8ea" + integrity sha512-z+cPxW0QGUp0mcqcsgQyLVRDoXFQbXOwBaqyF7VIgI4TWNQsDHrBpUQslRmIfAoYWdYzs6UlKJtB2XJpTaNSpQ== + on-finished@^2.3.0: version "2.4.1" resolved "https://registry.yarnpkg.com/on-finished/-/on-finished-2.4.1.tgz#58c8c44116e54845ad57f14ab10b03533184ac3f" @@ -4646,6 +4725,11 @@ prompts@^2.0.1: kleur "^3.0.3" sisteransi "^1.0.5" +proxy-from-env@^1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/proxy-from-env/-/proxy-from-env-1.1.0.tgz#e102f16ca355424865755d2c9e8ea4f24d58c3e2" + integrity sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg== + prr@~1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/prr/-/prr-1.0.1.tgz#d3fc114ba06995a45ec6893f484ceb1d78f5f476" @@ -4686,6 +4770,13 @@ pupa@^2.1.1: dependencies: escape-goat "^2.0.0" +qs@^6.11.0: + version "6.11.0" + resolved "https://registry.yarnpkg.com/qs/-/qs-6.11.0.tgz#fd0d963446f7a65e1367e01abd85429453f0c37a" + integrity sha512-MvjoMCJwEarSbUYk5O+nmoSzSutSsTwF85zcHPQ9OrlFoZOYIjaqBAJIqIXjptyD5vThxGq52Xu/MaJzRkIk4Q== + dependencies: + side-channel "^1.0.4" + qs@~6.5.2: version "6.5.3" resolved "https://registry.yarnpkg.com/qs/-/qs-6.5.3.tgz#3aeeffc91967ef6e35c0e488ef46fb296ab76aad" @@ -4696,6 +4787,11 @@ querystring@0.2.0: resolved "https://registry.yarnpkg.com/querystring/-/querystring-0.2.0.tgz#b209849203bb25df820da756e747005878521620" integrity sha512-X/xY82scca2tau62i9mDyU9K+I+djTMUsvwf7xnUX5GLvVzgJybOJf4Y6o9Zx3oJK/LSXg5tTZBjwzqVPaPO2g== +querystringify@^2.1.1: + version "2.2.0" + resolved "https://registry.yarnpkg.com/querystringify/-/querystringify-2.2.0.tgz#3345941b4153cb9d082d8eee4cda2016a9aef7f6" + integrity sha512-FIqgj2EUvTa7R50u0rGsyTftzjYmv/a3hO345bZNrqabNqjtgiDMgmo4mkUjd+nzU5oF3dClKqFIPUKybUyqoQ== + range-parser@^1.2.0: version "1.2.1" resolved "https://registry.yarnpkg.com/range-parser/-/range-parser-1.2.1.tgz#3cf37023d199e1c24d1a55b84800c2f3e6468031" @@ -4844,6 +4940,11 @@ require-directory@^2.1.1: resolved "https://registry.yarnpkg.com/require-directory/-/require-directory-2.1.1.tgz#8c64ad5fd30dab1c976e2344ffe7f792a6a6df42" integrity sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q== +requires-port@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/requires-port/-/requires-port-1.0.0.tgz#925d2601d39ac485e091cf0da5c6e694dc3dcaff" + integrity sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ== + resolve-cwd@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/resolve-cwd/-/resolve-cwd-3.0.0.tgz#0f0075f1bb2544766cf73ba6a6e2adfebcb13f2d" @@ -4972,6 +5073,15 @@ shimmer@^1.2.0: resolved "https://registry.yarnpkg.com/shimmer/-/shimmer-1.2.1.tgz#610859f7de327b587efebf501fb43117f9aff337" integrity sha512-sQTKC1Re/rM6XyFM6fIAGHRPVGvyXfgzIDvzoq608vM+jeyVD0Tu1E6Np0Kc2zAIFWIj963V2800iF/9LPieQw== +side-channel@^1.0.4: + version "1.0.4" + resolved "https://registry.yarnpkg.com/side-channel/-/side-channel-1.0.4.tgz#efce5c8fdc104ee751b25c58d4290011fa5ea2cf" + integrity sha512-q5XPytqFEIKHkGdiMIrY10mvLRvnQh42/+GoBlFW3b2LXLE2xxJpZFdm94we0BaoV3RwJyGqg5wS7epxTv0Zvw== + dependencies: + call-bind "^1.0.0" + get-intrinsic "^1.0.2" + object-inspect "^1.9.0" + signal-exit@^3.0.0, signal-exit@^3.0.2, signal-exit@^3.0.3: version "3.0.7" resolved "https://registry.yarnpkg.com/signal-exit/-/signal-exit-3.0.7.tgz#a9a1767f8af84155114eaabd73f99273c8f59ad9" @@ -5312,6 +5422,16 @@ touch@^3.1.0: punycode "^2.1.1" universalify "^0.1.2" +tough-cookie@^4.1.2: + version "4.1.2" + resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-4.1.2.tgz#e53e84b85f24e0b65dd526f46628db6c85f6b874" + integrity sha512-G9fqXWoYFZgTc2z8Q5zaHy/vJMjm+WV0AkAeHxVCQiEB1b+dGvWzFW6QV07cY5jQ5gRkeid2qIkzkxUnmoQZUQ== + dependencies: + psl "^1.1.33" + punycode "^2.1.1" + universalify "^0.2.0" + url-parse "^1.5.3" + tough-cookie@~2.5.0: version "2.5.0" resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-2.5.0.tgz#cd9fb2a0aa1d5a12b473bd9fb96fa3dcff65ade2" @@ -5427,6 +5547,11 @@ universalify@^0.1.2: resolved "https://registry.yarnpkg.com/universalify/-/universalify-0.1.2.tgz#b646f69be3942dabcecc9d6639c80dc105efaa66" integrity sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg== +universalify@^0.2.0: + version "0.2.0" + resolved "https://registry.yarnpkg.com/universalify/-/universalify-0.2.0.tgz#6451760566fa857534745ab1dde952d1b1761be0" + integrity sha512-CJ1QgKmNg3CwvAv/kOFmtnEN05f0D/cn9QntgNOQlQF9dgvVTHj3t+8JPdjqawCHk7V/KA+fbUqzZ9XWhcqPUg== + update-notifier@^5.1.0: version "5.1.0" resolved "https://registry.yarnpkg.com/update-notifier/-/update-notifier-5.1.0.tgz#4ab0d7c7f36a231dd7316cf7729313f0214d9ad9" @@ -5461,6 +5586,14 @@ url-parse-lax@^3.0.0: dependencies: prepend-http "^2.0.0" +url-parse@^1.5.3: + version "1.5.10" + resolved "https://registry.yarnpkg.com/url-parse/-/url-parse-1.5.10.tgz#9d3c2f736c1d75dd3bd2be507dcc111f1e2ea9c1" + integrity sha512-WypcfiRhfeUP9vvF0j6rw0J3hrWrw6iZv3+22h6iRMJ/8z1Tj6XfLP4DsUix5MhMPnXpiHDoKyoZ/bdCkwBCiQ== + dependencies: + querystringify "^2.1.1" + requires-port "^1.0.0" + url@0.10.3: version "0.10.3" resolved "https://registry.yarnpkg.com/url/-/url-0.10.3.tgz#021e4d9c7705f21bbf37d03ceb58767402774c64" diff --git a/packages/types/src/documents/document.ts b/packages/types/src/documents/document.ts index fea05918d5..ac05214b82 100644 --- a/packages/types/src/documents/document.ts +++ b/packages/types/src/documents/document.ts @@ -4,3 +4,7 @@ export interface Document { createdAt?: string | number updatedAt?: string } + +export interface AnyDocument extends Document { + [key: string]: any +} From 5aba8e4a65e28a7b4ced0af39d9103f513fbcc89 Mon Sep 17 00:00:00 2001 From: Mel O'Hagan Date: Tue, 8 Nov 2022 14:31:36 +0000 Subject: [PATCH 02/91] Added readable prop to aggregate function --- packages/server/src/integrations/mongodb.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/server/src/integrations/mongodb.ts b/packages/server/src/integrations/mongodb.ts index d7709a08c7..9ccefbda58 100644 --- a/packages/server/src/integrations/mongodb.ts +++ b/packages/server/src/integrations/mongodb.ts @@ -58,6 +58,7 @@ const SCHEMA: Integration = { }, aggregate: { type: QueryType.JSON, + readable: true, steps: [ { key: "$addFields", From 9e7ac26aa08354972038add5b202fd12726fa039 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 8 Nov 2022 16:32:13 +0000 Subject: [PATCH 03/91] Getting basic pouch replacement working. --- packages/backend-core/src/couch/pouchLike.ts | 91 +++++++++++++------- packages/backend-core/src/db/index.ts | 19 ++-- packages/backend-core/src/db/pouch.ts | 2 - 3 files changed, 69 insertions(+), 43 deletions(-) diff --git a/packages/backend-core/src/couch/pouchLike.ts b/packages/backend-core/src/couch/pouchLike.ts index 80ebe1d363..2591380ba0 100644 --- a/packages/backend-core/src/couch/pouchLike.ts +++ b/packages/backend-core/src/couch/pouchLike.ts @@ -2,8 +2,7 @@ import Nano from "nano" import { AnyDocument } from "@budibase/types" import { getCouchInfo } from "./couch" import { directCouchCall } from "./utils" -import { ReadStream, WriteStream } from "fs" -import { dangerousGetDB } from "../db" +import { getPouchDB } from "../db" export type PouchLikeOpts = { skip_setup?: boolean @@ -20,10 +19,6 @@ export type QueryOpts = { keys?: string[] } -export type DumpOpts = { - filter?: (doc: AnyDocument) => boolean -} - export class PouchLike { public readonly name: string private static nano: Nano.ServerScope @@ -32,13 +27,21 @@ export class PouchLike { constructor(dbName: string, opts?: PouchLikeOpts) { this.name = dbName this.pouchOpts = opts || {} + if (!PouchLike.nano) { + PouchLike.init() + } } static init() { const couchInfo = getCouchInfo() - this.nano = Nano({ + PouchLike.nano = Nano({ url: couchInfo.url, - cookie: couchInfo.cookie, + requestDefaults: { + headers: { + Authorization: couchInfo.cookie, + }, + }, + parseUrl: false, }) } @@ -56,16 +59,30 @@ export class PouchLike { return PouchLike.nano.db.use(this.name) } - async info() {} + private async updateOutput(fnc: any) { + try { + return await fnc() + } catch (err: any) { + if (err.statusCode) { + err.status = err.statusCode + } + throw err + } + } + + async info() { + const db = PouchLike.nano.db.use(this.name) + return db.info() + } async get(id: string) { const db = await this.checkSetup() - return await db.get(id) + return this.updateOutput(() => db.get(id)) } async remove(id: string, rev: string) { const db = await this.checkSetup() - return await db.destroy(id, rev) + return this.updateOutput(() => db.destroy(id, rev)) } async put(document: AnyDocument) { @@ -73,23 +90,23 @@ export class PouchLike { throw new Error("Cannot store document without _id field.") } const db = await this.checkSetup() - return await db.insert(document) + return this.updateOutput(() => db.insert(document)) } async bulkDocs(documents: AnyDocument[]) { const db = await this.checkSetup() - return await db.bulk({ docs: documents }) + return this.updateOutput(() => db.bulk({ docs: documents })) } async allDocs(params: QueryOpts) { const db = await this.checkSetup() - return await db.fetch({ keys: [] }, params) + return this.updateOutput(() => db.list(params)) } async query(viewName: string, params: QueryOpts) { const db = await this.checkSetup() const [database, view] = viewName.split("/") - return await db.view(database, view, params) + return this.updateOutput(() => db.view(database, view, params)) } async destroy() { @@ -97,37 +114,47 @@ export class PouchLike { await PouchLike.nano.db.destroy(this.name) } catch (err: any) { // didn't exist, don't worry - if (err.status === 404) { + if (err.statusCode === 404) { return } else { - throw err + throw { ...err, status: err.statusCode } } } } async compact() { const db = await this.checkSetup() - return await db.compact() + return this.updateOutput(() => db.compact()) } - // utilise PouchDB for this - async dump(stream: WriteStream, params: DumpOpts) { - const pouch = dangerousGetDB(this.name) - // @ts-ignore - return pouch.dump(stream, params) + private doWithPouchDB(func: string) { + const dbName = this.name + return async (args: any[]) => { + const pouch = getPouchDB(dbName) + // @ts-ignore + return pouch[func](...args) + } } - // utilise PouchDB for this - async load(stream: ReadStream) { - const pouch = dangerousGetDB(this.name) - // @ts-ignore - return pouch.load(stream) + // All below functions are in-frequently called, just utilise PouchDB + // for them as it implements them better than we can + async dump(...args: any[]) { + return this.doWithPouchDB("dump")(args) } - // pouch specific functions - indexes come from the pouchdb-find library - async createIndex() {} + async load(...args: any[]) { + return this.doWithPouchDB("load")(args) + } - async deleteIndex() {} + async createIndex(...args: any[]) { + return this.doWithPouchDB("createIndex")(args) + } - async getIndexes() {} + async deleteIndex(...args: any[]) { + return this.doWithPouchDB("createIndex")(args) + } + + async getIndexes(...args: any[]) { + return this.doWithPouchDB("createIndex")(args) + } } diff --git a/packages/backend-core/src/db/index.ts b/packages/backend-core/src/db/index.ts index 9f6c097f0d..73d1327af3 100644 --- a/packages/backend-core/src/db/index.ts +++ b/packages/backend-core/src/db/index.ts @@ -2,6 +2,7 @@ import * as pouch from "./pouch" import env from "../environment" import { PouchOptions, CouchFindOptions } from "@budibase/types" import PouchDB from "pouchdb" +import { PouchLike } from "../couch" import { directCouchQuery } from "../couch" export { directCouchQuery } from "../couch" @@ -38,10 +39,7 @@ export async function init(opts?: PouchOptions) { initialised = true } -// NOTE: THIS IS A DANGEROUS FUNCTION - USE WITH CAUTION -// this function is prone to leaks, should only be used -// in situations that using the function doWithDB does not work -export function dangerousGetDB(dbName: string, opts?: any): PouchDB.Database { +export function getPouchDB(dbName: string, opts?: any): PouchDB.Database { checkInitialised() if (env.isTest()) { dbList.add(dbName) @@ -55,6 +53,13 @@ export function dangerousGetDB(dbName: string, opts?: any): PouchDB.Database { return db } +// NOTE: THIS IS A DANGEROUS FUNCTION - USE WITH CAUTION +// this function is prone to leaks, should only be used +// in situations that using the function doWithDB does not work +export function dangerousGetDB(dbName: string, opts?: any): PouchLike { + return new PouchLike(dbName, opts) +} + // use this function if you have called dangerousGetDB - close // the databases you've opened once finished export async function closeDB(db: PouchDB.Database) { @@ -79,11 +84,7 @@ export async function doWithDB(dbName: string, cb: any, opts = {}) { const db = dangerousGetDB(dbName, opts) // need this to be async so that we can correctly close DB after all // async operations have been completed - try { - return await cb(db) - } finally { - await closeDB(db) - } + return await cb(db) } export function allDbs() { diff --git a/packages/backend-core/src/db/pouch.ts b/packages/backend-core/src/db/pouch.ts index 6eca07e0cd..fdb051060a 100644 --- a/packages/backend-core/src/db/pouch.ts +++ b/packages/backend-core/src/db/pouch.ts @@ -66,7 +66,6 @@ export const getPouch = (opts: any = {}) => { const inMemory = require("pouchdb-adapter-memory") PouchDB.plugin(inMemory) POUCH_DB_DEFAULTS = { - prefix: undefined, // @ts-ignore adapter: "memory", } @@ -74,7 +73,6 @@ export const getPouch = (opts: any = {}) => { if (opts.onDisk) { POUCH_DB_DEFAULTS = { - prefix: undefined, // @ts-ignore adapter: "leveldb", } From 572e31fb29ac6a7304aaf2e7a804aab9c32b2d1a Mon Sep 17 00:00:00 2001 From: Mel O'Hagan Date: Tue, 8 Nov 2022 17:02:31 +0000 Subject: [PATCH 04/91] Block lucene special characters from new column --- .../components/backend/DataTable/modals/CreateEditColumn.svelte | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte index dce6df6d0d..a2400f1836 100644 --- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte +++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte @@ -304,6 +304,8 @@ const newError = {} if (!external && fieldInfo.name?.startsWith("_")) { newError.name = `Column name cannot start with an underscore.` + } else if (fieldInfo.name?.match(/[\-!*+?^"{}()~\/[\]\\]/g)) { + newError.name = `Illegal character; cannot be: + - ! ( ) { } [ ] ^ " ~ * ? : \\ /` } else if (PROHIBITED_COLUMN_NAMES.some(name => fieldInfo.name === name)) { newError.name = `${PROHIBITED_COLUMN_NAMES.join( ", " From 847fc9f83a6d7d1d9fa2dc4887513c238b3b3d58 Mon Sep 17 00:00:00 2001 From: Mel O'Hagan Date: Tue, 8 Nov 2022 17:03:13 +0000 Subject: [PATCH 05/91] Remove escaped characters --- .../components/backend/DataTable/modals/CreateEditColumn.svelte | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte index a2400f1836..279a901ce9 100644 --- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte +++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte @@ -304,7 +304,7 @@ const newError = {} if (!external && fieldInfo.name?.startsWith("_")) { newError.name = `Column name cannot start with an underscore.` - } else if (fieldInfo.name?.match(/[\-!*+?^"{}()~\/[\]\\]/g)) { + } else if (fieldInfo.name?.match(/[-!*+?^"{}()~/[\]\\]/g)) { newError.name = `Illegal character; cannot be: + - ! ( ) { } [ ] ^ " ~ * ? : \\ /` } else if (PROHIBITED_COLUMN_NAMES.some(name => fieldInfo.name === name)) { newError.name = `${PROHIBITED_COLUMN_NAMES.join( From 4f1ecf6dc13a18302c27e2a0f6ddbe3165a1ee30 Mon Sep 17 00:00:00 2001 From: Mel O'Hagan Date: Tue, 8 Nov 2022 17:07:25 +0000 Subject: [PATCH 06/91] Missed colon --- .../components/backend/DataTable/modals/CreateEditColumn.svelte | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte index 279a901ce9..4ae1d31edb 100644 --- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte +++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte @@ -304,7 +304,7 @@ const newError = {} if (!external && fieldInfo.name?.startsWith("_")) { newError.name = `Column name cannot start with an underscore.` - } else if (fieldInfo.name?.match(/[-!*+?^"{}()~/[\]\\]/g)) { + } else if (fieldInfo.name?.match(/[-!*+?:^"{}()~/[\]\\]/g)) { newError.name = `Illegal character; cannot be: + - ! ( ) { } [ ] ^ " ~ * ? : \\ /` } else if (PROHIBITED_COLUMN_NAMES.some(name => fieldInfo.name === name)) { newError.name = `${PROHIBITED_COLUMN_NAMES.join( From ca2f85b6f911b215c8de74b1b8c354b32d415980 Mon Sep 17 00:00:00 2001 From: Mel O'Hagan Date: Tue, 8 Nov 2022 17:16:35 +0000 Subject: [PATCH 07/91] Use whitelist instead --- .../backend/DataTable/modals/CreateEditColumn.svelte | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte index 4ae1d31edb..84ce61222c 100644 --- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte +++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte @@ -304,8 +304,8 @@ const newError = {} if (!external && fieldInfo.name?.startsWith("_")) { newError.name = `Column name cannot start with an underscore.` - } else if (fieldInfo.name?.match(/[-!*+?:^"{}()~/[\]\\]/g)) { - newError.name = `Illegal character; cannot be: + - ! ( ) { } [ ] ^ " ~ * ? : \\ /` + } else if (fieldInfo.name && !fieldInfo.name.match(/^[a-zA-Z0-9\s]*$/g)) { + newError.name = `Illegal character; must be alpha-numeric.` } else if (PROHIBITED_COLUMN_NAMES.some(name => fieldInfo.name === name)) { newError.name = `${PROHIBITED_COLUMN_NAMES.join( ", " From 43b5c120b5225f78cfb68febe4572a177dafed89 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Wed, 9 Nov 2022 10:28:15 +0000 Subject: [PATCH 08/91] Refactor form block to use a different structure depending on type, and fix issue with bindings breaking and forms not appearing --- .../components/app/blocks/FormBlock.svelte | 242 ------------------ .../app/blocks/form/FormBlock.svelte | 101 ++++++++ .../app/blocks/form/InnerFormBlock.svelte | 192 ++++++++++++++ .../client/src/components/app/blocks/index.js | 2 +- 4 files changed, 294 insertions(+), 243 deletions(-) delete mode 100644 packages/client/src/components/app/blocks/FormBlock.svelte create mode 100644 packages/client/src/components/app/blocks/form/FormBlock.svelte create mode 100644 packages/client/src/components/app/blocks/form/InnerFormBlock.svelte diff --git a/packages/client/src/components/app/blocks/FormBlock.svelte b/packages/client/src/components/app/blocks/FormBlock.svelte deleted file mode 100644 index 3311ddd7bb..0000000000 --- a/packages/client/src/components/app/blocks/FormBlock.svelte +++ /dev/null @@ -1,242 +0,0 @@ - - - - {#if fields?.length} - - - - - {#if renderHeader} - - - {#if renderButtons} - - {#if renderDeleteButton} - - {/if} - {#if renderSaveButton} - - {/if} - - {/if} - - {/if} - - {#each fields as field, idx} - {#if getComponentForField(field)} - - {/if} - {/each} - - - - - - {:else} - - {/if} - diff --git a/packages/client/src/components/app/blocks/form/FormBlock.svelte b/packages/client/src/components/app/blocks/form/FormBlock.svelte new file mode 100644 index 0000000000..9ec587519f --- /dev/null +++ b/packages/client/src/components/app/blocks/form/FormBlock.svelte @@ -0,0 +1,101 @@ + + + + {#if actionType === "Create"} + + + + {:else} + + + + + + {/if} + diff --git a/packages/client/src/components/app/blocks/form/InnerFormBlock.svelte b/packages/client/src/components/app/blocks/form/InnerFormBlock.svelte new file mode 100644 index 0000000000..948584120b --- /dev/null +++ b/packages/client/src/components/app/blocks/form/InnerFormBlock.svelte @@ -0,0 +1,192 @@ + + +{#if fields?.length} + + + {#if renderHeader} + + + {#if renderButtons} + + {#if renderDeleteButton} + + {/if} + {#if renderSaveButton} + + {/if} + + {/if} + + {/if} + + {#each fields as field, idx} + {#if getComponentForField(field)} + + {/if} + {/each} + + + +{:else} + +{/if} diff --git a/packages/client/src/components/app/blocks/index.js b/packages/client/src/components/app/blocks/index.js index 734bff2c0f..0f05890fef 100644 --- a/packages/client/src/components/app/blocks/index.js +++ b/packages/client/src/components/app/blocks/index.js @@ -1,5 +1,5 @@ export { default as tableblock } from "./TableBlock.svelte" export { default as cardsblock } from "./CardsBlock.svelte" export { default as repeaterblock } from "./RepeaterBlock.svelte" -export { default as formblock } from "./FormBlock.svelte" +export { default as formblock } from "./form/FormBlock.svelte" export { default as chartblock } from "./ChartBlock.svelte" From 217743e6fd98830345756b56d1724f749c70fbe6 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Wed, 9 Nov 2022 13:57:22 +0000 Subject: [PATCH 09/91] patching sub dep for loader-utils to patch security vulnerability --- packages/bbui/package.json | 5 ++++- packages/client/package.json | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/packages/bbui/package.json b/packages/bbui/package.json index cb0a06aece..2c7aa7c19a 100644 --- a/packages/bbui/package.json +++ b/packages/bbui/package.json @@ -85,5 +85,8 @@ "svelte-flatpickr": "^3.2.3", "svelte-portal": "^1.0.0" }, + "resolutions": { + "loader-utils": "1.4.1" + }, "gitHead": "d1836a898cab3f8ab80ee6d8f42be1a9eed7dcdc" -} +} \ No newline at end of file diff --git a/packages/client/package.json b/packages/client/package.json index 0c51890dff..bfce293df3 100644 --- a/packages/client/package.json +++ b/packages/client/package.json @@ -59,5 +59,8 @@ "rollup-plugin-terser": "^7.0.2", "rollup-plugin-visualizer": "^5.5.4" }, + "resolutions": { + "loader-utils": "1.4.1" + }, "gitHead": "d1836a898cab3f8ab80ee6d8f42be1a9eed7dcdc" -} +} \ No newline at end of file From 44fbdec0a22998c53be6d8a54e75378969ea2c8e Mon Sep 17 00:00:00 2001 From: Budibase Release Bot <> Date: Wed, 9 Nov 2022 14:27:49 +0000 Subject: [PATCH 10/91] v2.1.22 --- lerna.json | 2 +- packages/backend-core/package.json | 4 ++-- packages/bbui/package.json | 6 +++--- packages/builder/package.json | 10 +++++----- packages/cli/package.json | 8 ++++---- packages/client/package.json | 10 +++++----- packages/frontend-core/package.json | 4 ++-- packages/sdk/package.json | 2 +- packages/server/package.json | 10 +++++----- packages/string-templates/package.json | 2 +- packages/types/package.json | 2 +- packages/worker/package.json | 8 ++++---- 12 files changed, 34 insertions(+), 34 deletions(-) diff --git a/lerna.json b/lerna.json index 6275d78747..49766f3ab7 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.1.21", + "version": "2.1.22", "npmClient": "yarn", "packages": [ "packages/*" diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json index 8ef3588b7f..584db9db53 100644 --- a/packages/backend-core/package.json +++ b/packages/backend-core/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/backend-core", - "version": "2.1.21", + "version": "2.1.22", "description": "Budibase backend core libraries used in server and worker", "main": "dist/src/index.js", "types": "dist/src/index.d.ts", @@ -20,7 +20,7 @@ "test:watch": "jest --watchAll" }, "dependencies": { - "@budibase/types": "^2.1.21", + "@budibase/types": "^2.1.22", "@shopify/jest-koa-mocks": "5.0.1", "@techpass/passport-openidconnect": "0.3.2", "aws-sdk": "2.1030.0", diff --git a/packages/bbui/package.json b/packages/bbui/package.json index 2c7aa7c19a..786defcec5 100644 --- a/packages/bbui/package.json +++ b/packages/bbui/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/bbui", "description": "A UI solution used in the different Budibase projects.", - "version": "2.1.21", + "version": "2.1.22", "license": "MPL-2.0", "svelte": "src/index.js", "module": "dist/bbui.es.js", @@ -38,7 +38,7 @@ ], "dependencies": { "@adobe/spectrum-css-workflow-icons": "^1.2.1", - "@budibase/string-templates": "^2.1.21", + "@budibase/string-templates": "^2.1.22", "@spectrum-css/actionbutton": "^1.0.1", "@spectrum-css/actiongroup": "^1.0.1", "@spectrum-css/avatar": "^3.0.2", @@ -89,4 +89,4 @@ "loader-utils": "1.4.1" }, "gitHead": "d1836a898cab3f8ab80ee6d8f42be1a9eed7dcdc" -} \ No newline at end of file +} diff --git a/packages/builder/package.json b/packages/builder/package.json index 8b55d1e955..dafb7ef7f7 100644 --- a/packages/builder/package.json +++ b/packages/builder/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/builder", - "version": "2.1.21", + "version": "2.1.22", "license": "GPL-3.0", "private": true, "scripts": { @@ -71,10 +71,10 @@ } }, "dependencies": { - "@budibase/bbui": "^2.1.21", - "@budibase/client": "^2.1.21", - "@budibase/frontend-core": "^2.1.21", - "@budibase/string-templates": "^2.1.21", + "@budibase/bbui": "^2.1.22", + "@budibase/client": "^2.1.22", + "@budibase/frontend-core": "^2.1.22", + "@budibase/string-templates": "^2.1.22", "@sentry/browser": "5.19.1", "@spectrum-css/page": "^3.0.1", "@spectrum-css/vars": "^3.0.1", diff --git a/packages/cli/package.json b/packages/cli/package.json index a390614cd2..fbc2e97845 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/cli", - "version": "2.1.21", + "version": "2.1.22", "description": "Budibase CLI, for developers, self hosting and migrations.", "main": "src/index.js", "bin": { @@ -26,9 +26,9 @@ "outputPath": "build" }, "dependencies": { - "@budibase/backend-core": "^2.1.21", - "@budibase/string-templates": "^2.1.21", - "@budibase/types": "^2.1.21", + "@budibase/backend-core": "^2.1.22", + "@budibase/string-templates": "^2.1.22", + "@budibase/types": "^2.1.22", "axios": "0.21.2", "chalk": "4.1.0", "cli-progress": "3.11.2", diff --git a/packages/client/package.json b/packages/client/package.json index bfce293df3..f834ebfb5a 100644 --- a/packages/client/package.json +++ b/packages/client/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/client", - "version": "2.1.21", + "version": "2.1.22", "license": "MPL-2.0", "module": "dist/budibase-client.js", "main": "dist/budibase-client.js", @@ -19,9 +19,9 @@ "dev:builder": "rollup -cw" }, "dependencies": { - "@budibase/bbui": "^2.1.21", - "@budibase/frontend-core": "^2.1.21", - "@budibase/string-templates": "^2.1.21", + "@budibase/bbui": "^2.1.22", + "@budibase/frontend-core": "^2.1.22", + "@budibase/string-templates": "^2.1.22", "@spectrum-css/button": "^3.0.3", "@spectrum-css/card": "^3.0.3", "@spectrum-css/divider": "^1.0.3", @@ -63,4 +63,4 @@ "loader-utils": "1.4.1" }, "gitHead": "d1836a898cab3f8ab80ee6d8f42be1a9eed7dcdc" -} \ No newline at end of file +} diff --git a/packages/frontend-core/package.json b/packages/frontend-core/package.json index 7b87b2f5ac..80b3df6306 100644 --- a/packages/frontend-core/package.json +++ b/packages/frontend-core/package.json @@ -1,12 +1,12 @@ { "name": "@budibase/frontend-core", - "version": "2.1.21", + "version": "2.1.22", "description": "Budibase frontend core libraries used in builder and client", "author": "Budibase", "license": "MPL-2.0", "svelte": "src/index.js", "dependencies": { - "@budibase/bbui": "^2.1.21", + "@budibase/bbui": "^2.1.22", "lodash": "^4.17.21", "svelte": "^3.46.2" } diff --git a/packages/sdk/package.json b/packages/sdk/package.json index 1475a862c8..4b63618094 100644 --- a/packages/sdk/package.json +++ b/packages/sdk/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/sdk", - "version": "2.1.21", + "version": "2.1.22", "description": "Budibase Public API SDK", "author": "Budibase", "license": "MPL-2.0", diff --git a/packages/server/package.json b/packages/server/package.json index 577aac9c9c..a0df6eacce 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/server", "email": "hi@budibase.com", - "version": "2.1.21", + "version": "2.1.22", "description": "Budibase Web Server", "main": "src/index.ts", "repository": { @@ -77,11 +77,11 @@ "license": "GPL-3.0", "dependencies": { "@apidevtools/swagger-parser": "10.0.3", - "@budibase/backend-core": "^2.1.21", - "@budibase/client": "^2.1.21", + "@budibase/backend-core": "^2.1.22", + "@budibase/client": "^2.1.22", "@budibase/pro": "2.1.21", - "@budibase/string-templates": "^2.1.21", - "@budibase/types": "^2.1.21", + "@budibase/string-templates": "^2.1.22", + "@budibase/types": "^2.1.22", "@bull-board/api": "3.7.0", "@bull-board/koa": "3.9.4", "@elastic/elasticsearch": "7.10.0", diff --git a/packages/string-templates/package.json b/packages/string-templates/package.json index 58e0354181..aeee568ced 100644 --- a/packages/string-templates/package.json +++ b/packages/string-templates/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/string-templates", - "version": "2.1.21", + "version": "2.1.22", "description": "Handlebars wrapper for Budibase templating.", "main": "src/index.cjs", "module": "dist/bundle.mjs", diff --git a/packages/types/package.json b/packages/types/package.json index 66dfd92698..7c26504406 100644 --- a/packages/types/package.json +++ b/packages/types/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/types", - "version": "2.1.21", + "version": "2.1.22", "description": "Budibase types", "main": "dist/index.js", "types": "dist/index.d.ts", diff --git a/packages/worker/package.json b/packages/worker/package.json index 979551c3fc..025208cd2b 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/worker", "email": "hi@budibase.com", - "version": "2.1.21", + "version": "2.1.22", "description": "Budibase background service", "main": "src/index.ts", "repository": { @@ -36,10 +36,10 @@ "author": "Budibase", "license": "GPL-3.0", "dependencies": { - "@budibase/backend-core": "^2.1.21", + "@budibase/backend-core": "^2.1.22", "@budibase/pro": "2.1.21", - "@budibase/string-templates": "^2.1.21", - "@budibase/types": "^2.1.21", + "@budibase/string-templates": "^2.1.22", + "@budibase/types": "^2.1.22", "@koa/router": "8.0.8", "@sentry/node": "6.17.7", "@techpass/passport-openidconnect": "0.3.2", From d6ffeb1b7350d417f0c53f9c229be1268baef42f Mon Sep 17 00:00:00 2001 From: Budibase Release Bot <> Date: Wed, 9 Nov 2022 14:31:12 +0000 Subject: [PATCH 11/91] Update pro version to 2.1.22 --- packages/server/package.json | 2 +- packages/server/yarn.lock | 30 +++++++++++++++--------------- packages/worker/package.json | 2 +- packages/worker/yarn.lock | 30 +++++++++++++++--------------- 4 files changed, 32 insertions(+), 32 deletions(-) diff --git a/packages/server/package.json b/packages/server/package.json index a0df6eacce..f4c5c85004 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -79,7 +79,7 @@ "@apidevtools/swagger-parser": "10.0.3", "@budibase/backend-core": "^2.1.22", "@budibase/client": "^2.1.22", - "@budibase/pro": "2.1.21", + "@budibase/pro": "2.1.22", "@budibase/string-templates": "^2.1.22", "@budibase/types": "^2.1.22", "@bull-board/api": "3.7.0", diff --git a/packages/server/yarn.lock b/packages/server/yarn.lock index 452e0afb39..73deee3b20 100644 --- a/packages/server/yarn.lock +++ b/packages/server/yarn.lock @@ -1094,12 +1094,12 @@ resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw== -"@budibase/backend-core@2.1.21": - version "2.1.21" - resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-2.1.21.tgz#6e473385dd6329229e54a3299a261b44568721ee" - integrity sha512-ktfXBNzRuRUeAhmZ0EnEyTNteHcMGsY6AK8+j+nEJzdc6zP/6DMSISXJcGG8SkXT77fWQkem69ao21/XN5r+ZQ== +"@budibase/backend-core@2.1.22": + version "2.1.22" + resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-2.1.22.tgz#5986f1ff2189dd789fe6184806d327ff2813d7b7" + integrity sha512-ReuWU75gkNJqHEcZHO15qCOW6M8YpADe7B4fyXDMl6eZOw20xm71X/y7vsegS9VXDSxvh8LJpLZTCo7EOaFwRg== dependencies: - "@budibase/types" "^2.1.21" + "@budibase/types" "^2.1.22" "@shopify/jest-koa-mocks" "5.0.1" "@techpass/passport-openidconnect" "0.3.2" aws-sdk "2.1030.0" @@ -1181,13 +1181,13 @@ svelte-flatpickr "^3.2.3" svelte-portal "^1.0.0" -"@budibase/pro@2.1.21": - version "2.1.21" - resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.1.21.tgz#f5a73c47f2e8eebdf0b86bcfe9d1bf30c7be5f6b" - integrity sha512-nNouTv7RwRc0XtxNWimrkaoYyqFO+9ZIgJsEaFhzDv/p4D6i0tggyMkAC3TDSZNuPPzxMq/3pThjrdt7LBW/DQ== +"@budibase/pro@2.1.22": + version "2.1.22" + resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.1.22.tgz#9df787c65375a556b84f314eea1866b7d3275c56" + integrity sha512-X+R6AqLtw5MJy4fosBlsNBI98Zp8oItRGK26M8LVlCCcjZTePSeGGwY1o1+r/eXYDHHilmOQl9s+EUQhSEQ2XA== dependencies: - "@budibase/backend-core" "2.1.21" - "@budibase/types" "2.1.21" + "@budibase/backend-core" "2.1.22" + "@budibase/types" "2.1.22" "@koa/router" "8.0.8" bull "4.10.1" joi "17.6.0" @@ -1211,10 +1211,10 @@ svelte-apexcharts "^1.0.2" svelte-flatpickr "^3.1.0" -"@budibase/types@2.1.21", "@budibase/types@^2.1.21": - version "2.1.21" - resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.1.21.tgz#ff607f0979e710f68469b0a586b3dc835ceaf236" - integrity sha512-KSvvUj2DQECVsZ2xbh0jsPz6u/WfoIQbkzsJa7wdRcC9goQ5fDdjLtHZrLDKKBRV9ceSVh0021BxFQ705+CaEA== +"@budibase/types@2.1.22", "@budibase/types@^2.1.22": + version "2.1.22" + resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.1.22.tgz#7280072af0fedd9a24b715391481e7f6b475dc27" + integrity sha512-v2/w9XMP8hf6NsdpvS5aTfbNKhug08ZqpW0S9vtJwM/Afk/ZWKCj43r7O0Vs1IUODm2Oxev5lujugMF2HqPhVQ== "@bull-board/api@3.7.0": version "3.7.0" diff --git a/packages/worker/package.json b/packages/worker/package.json index 025208cd2b..a9cb2becb9 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -37,7 +37,7 @@ "license": "GPL-3.0", "dependencies": { "@budibase/backend-core": "^2.1.22", - "@budibase/pro": "2.1.21", + "@budibase/pro": "2.1.22", "@budibase/string-templates": "^2.1.22", "@budibase/types": "^2.1.22", "@koa/router": "8.0.8", diff --git a/packages/worker/yarn.lock b/packages/worker/yarn.lock index 4647252bcc..ff3f85db2e 100644 --- a/packages/worker/yarn.lock +++ b/packages/worker/yarn.lock @@ -291,12 +291,12 @@ resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw== -"@budibase/backend-core@2.1.21": - version "2.1.21" - resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-2.1.21.tgz#6e473385dd6329229e54a3299a261b44568721ee" - integrity sha512-ktfXBNzRuRUeAhmZ0EnEyTNteHcMGsY6AK8+j+nEJzdc6zP/6DMSISXJcGG8SkXT77fWQkem69ao21/XN5r+ZQ== +"@budibase/backend-core@2.1.22": + version "2.1.22" + resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-2.1.22.tgz#5986f1ff2189dd789fe6184806d327ff2813d7b7" + integrity sha512-ReuWU75gkNJqHEcZHO15qCOW6M8YpADe7B4fyXDMl6eZOw20xm71X/y7vsegS9VXDSxvh8LJpLZTCo7EOaFwRg== dependencies: - "@budibase/types" "^2.1.21" + "@budibase/types" "^2.1.22" "@shopify/jest-koa-mocks" "5.0.1" "@techpass/passport-openidconnect" "0.3.2" aws-sdk "2.1030.0" @@ -328,22 +328,22 @@ uuid "8.3.2" zlib "1.0.5" -"@budibase/pro@2.1.21": - version "2.1.21" - resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.1.21.tgz#f5a73c47f2e8eebdf0b86bcfe9d1bf30c7be5f6b" - integrity sha512-nNouTv7RwRc0XtxNWimrkaoYyqFO+9ZIgJsEaFhzDv/p4D6i0tggyMkAC3TDSZNuPPzxMq/3pThjrdt7LBW/DQ== +"@budibase/pro@2.1.22": + version "2.1.22" + resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.1.22.tgz#9df787c65375a556b84f314eea1866b7d3275c56" + integrity sha512-X+R6AqLtw5MJy4fosBlsNBI98Zp8oItRGK26M8LVlCCcjZTePSeGGwY1o1+r/eXYDHHilmOQl9s+EUQhSEQ2XA== dependencies: - "@budibase/backend-core" "2.1.21" - "@budibase/types" "2.1.21" + "@budibase/backend-core" "2.1.22" + "@budibase/types" "2.1.22" "@koa/router" "8.0.8" bull "4.10.1" joi "17.6.0" node-fetch "^2.6.1" -"@budibase/types@2.1.21", "@budibase/types@^2.1.21": - version "2.1.21" - resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.1.21.tgz#ff607f0979e710f68469b0a586b3dc835ceaf236" - integrity sha512-KSvvUj2DQECVsZ2xbh0jsPz6u/WfoIQbkzsJa7wdRcC9goQ5fDdjLtHZrLDKKBRV9ceSVh0021BxFQ705+CaEA== +"@budibase/types@2.1.22", "@budibase/types@^2.1.22": + version "2.1.22" + resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.1.22.tgz#7280072af0fedd9a24b715391481e7f6b475dc27" + integrity sha512-v2/w9XMP8hf6NsdpvS5aTfbNKhug08ZqpW0S9vtJwM/Afk/ZWKCj43r7O0Vs1IUODm2Oxev5lujugMF2HqPhVQ== "@cspotcode/source-map-consumer@0.8.0": version "0.8.0" From c744d23832413f3df04eec7c03ccb97a2c41ac2a Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 9 Nov 2022 16:53:42 +0000 Subject: [PATCH 12/91] Major update - removing the use of context for PouchDB instances, swapping knowledge of PouchDB to the PouchLike structure that replaces it. --- .../src/cache/tests/writethrough.spec.js | 6 +- .../backend-core/src/cache/writethrough.ts | 17 +- .../backend-core/src/context/constants.ts | 12 -- packages/backend-core/src/context/index.ts | 160 +++++------------- .../src/context/tests/index.spec.ts | 16 +- packages/backend-core/src/context/utils.ts | 109 ------------ packages/backend-core/src/couch/index.ts | 1 + packages/backend-core/src/couch/pouchDB.ts | 37 ++++ packages/backend-core/src/couch/pouchLike.ts | 55 ++++-- packages/backend-core/src/db/Replication.ts | 10 +- packages/backend-core/src/db/index.ts | 65 +------ .../backend-core/src/db/tests/index.spec.js | 6 +- packages/backend-core/src/db/views.ts | 49 +++--- packages/backend-core/src/environment.ts | 1 - packages/backend-core/src/index.ts | 2 + .../src/migrations/tests/index.spec.js | 4 +- packages/backend-core/src/types.ts | 1 + packages/backend-core/src/users.ts | 5 +- .../server/src/api/controllers/application.ts | 10 +- .../src/api/controllers/row/internal.js | 4 +- packages/server/src/db/inMemoryView.js | 4 +- .../functions/backfill/app/tables.ts | 5 +- .../server/src/sdk/app/backups/imports.ts | 12 +- .../server/src/sdk/app/backups/statistics.ts | 28 +-- packages/server/src/sdk/app/tables/index.ts | 11 +- 25 files changed, 211 insertions(+), 419 deletions(-) delete mode 100644 packages/backend-core/src/context/utils.ts create mode 100644 packages/backend-core/src/couch/pouchDB.ts create mode 100644 packages/backend-core/src/types.ts diff --git a/packages/backend-core/src/cache/tests/writethrough.spec.js b/packages/backend-core/src/cache/tests/writethrough.spec.js index 68db24b325..ad29cb2aea 100644 --- a/packages/backend-core/src/cache/tests/writethrough.spec.js +++ b/packages/backend-core/src/cache/tests/writethrough.spec.js @@ -1,6 +1,6 @@ require("../../../tests/utilities/TestConfiguration") const { Writethrough } = require("../writethrough") -const { dangerousGetDB } = require("../../db") +const { getDB } = require("../../db") const tk = require("timekeeper") const START_DATE = Date.now() @@ -8,8 +8,8 @@ tk.freeze(START_DATE) const DELAY = 5000 -const db = dangerousGetDB("test") -const db2 = dangerousGetDB("test2") +const db = getDB("test") +const db2 = getDB("test2") const writethrough = new Writethrough(db, DELAY), writethrough2 = new Writethrough(db2, DELAY) describe("writethrough", () => { diff --git a/packages/backend-core/src/cache/writethrough.ts b/packages/backend-core/src/cache/writethrough.ts index 495ba58590..11dad81239 100644 --- a/packages/backend-core/src/cache/writethrough.ts +++ b/packages/backend-core/src/cache/writethrough.ts @@ -1,7 +1,7 @@ import BaseCache from "./base" import { getWritethroughClient } from "../redis/init" import { logWarn } from "../logging" -import PouchDB from "pouchdb" +import { PouchLike } from "../couch" const DEFAULT_WRITE_RATE_MS = 10000 let CACHE: BaseCache | null = null @@ -19,7 +19,7 @@ async function getCache() { return CACHE } -function makeCacheKey(db: PouchDB.Database, key: string) { +function makeCacheKey(db: PouchLike, key: string) { return db.name + key } @@ -28,7 +28,7 @@ function makeCacheItem(doc: any, lastWrite: number | null = null): CacheItem { } export async function put( - db: PouchDB.Database, + db: PouchLike, doc: any, writeRateMs: number = DEFAULT_WRITE_RATE_MS ) { @@ -64,7 +64,7 @@ export async function put( return { ok: true, id: output._id, rev: output._rev } } -export async function get(db: PouchDB.Database, id: string): Promise { +export async function get(db: PouchLike, id: string): Promise { const cache = await getCache() const cacheKey = makeCacheKey(db, id) let cacheItem: CacheItem = await cache.get(cacheKey) @@ -77,7 +77,7 @@ export async function get(db: PouchDB.Database, id: string): Promise { } export async function remove( - db: PouchDB.Database, + db: PouchLike, docOrId: any, rev?: any ): Promise { @@ -95,13 +95,10 @@ export async function remove( } export class Writethrough { - db: PouchDB.Database + db: PouchLike writeRateMs: number - constructor( - db: PouchDB.Database, - writeRateMs: number = DEFAULT_WRITE_RATE_MS - ) { + constructor(db: PouchLike, writeRateMs: number = DEFAULT_WRITE_RATE_MS) { this.db = db this.writeRateMs = writeRateMs } diff --git a/packages/backend-core/src/context/constants.ts b/packages/backend-core/src/context/constants.ts index 937ad8f248..af30b1d241 100644 --- a/packages/backend-core/src/context/constants.ts +++ b/packages/backend-core/src/context/constants.ts @@ -1,17 +1,5 @@ export enum ContextKey { TENANT_ID = "tenantId", - GLOBAL_DB = "globalDb", APP_ID = "appId", IDENTITY = "identity", - // whatever the request app DB was - CURRENT_DB = "currentDb", - // get the prod app DB from the request - PROD_DB = "prodDb", - // get the dev app DB from the request - DEV_DB = "devDb", - DB_OPTS = "dbOpts", - // check if something else is using the context, don't close DB - TENANCY_IN_USE = "tenancyInUse", - APP_IN_USE = "appInUse", - IDENTITY_IN_USE = "identityInUse", } diff --git a/packages/backend-core/src/context/index.ts b/packages/backend-core/src/context/index.ts index c3955c71d9..30a53f1587 100644 --- a/packages/backend-core/src/context/index.ts +++ b/packages/backend-core/src/context/index.ts @@ -1,20 +1,12 @@ import env from "../environment" import { SEPARATOR, DocumentType } from "../db/constants" import cls from "./FunctionContext" -import { dangerousGetDB, closeDB } from "../db" import { baseGlobalDBName } from "../db/tenancy" import { IdentityContext } from "@budibase/types" import { DEFAULT_TENANT_ID as _DEFAULT_TENANT_ID } from "../constants" import { ContextKey } from "./constants" -import PouchDB from "pouchdb" -import { - updateUsing, - closeWithUsing, - setAppTenantId, - setIdentity, - closeAppDBs, - getContextDB, -} from "./utils" +import { PouchLike } from "../couch" +import { getDevelopmentAppID, getProdAppID } from "../db/conversions" export const DEFAULT_TENANT_ID = _DEFAULT_TENANT_ID @@ -22,29 +14,19 @@ export const DEFAULT_TENANT_ID = _DEFAULT_TENANT_ID // store an app ID to pretend there is a context let TEST_APP_ID: string | null = null -export const closeTenancy = async () => { - try { - if (env.USE_COUCH) { - const db = getGlobalDB() - await closeDB(db) - } - } catch (err) { - // no DB found - skip closing - return - } - // clear from context now that database is closed/task is finished - cls.setOnContext(ContextKey.TENANT_ID, null) - cls.setOnContext(ContextKey.GLOBAL_DB, null) -} - -// export const isDefaultTenant = () => { -// return getTenantId() === DEFAULT_TENANT_ID -// } - export const isMultiTenant = () => { return env.MULTI_TENANCY } +const setAppTenantId = (appId: string) => { + const appTenantId = getTenantIDFromAppID(appId) || DEFAULT_TENANT_ID + updateTenantId(appTenantId) +} + +const setIdentity = (identity: IdentityContext | null) => { + cls.setOnContext(ContextKey.IDENTITY, identity) +} + /** * Given an app ID this will attempt to retrieve the tenant ID from it. * @return {null|string} The tenant ID found within the app ID. @@ -78,47 +60,28 @@ export const doInContext = async (appId: string, task: any) => { }) } -export const doInTenant = (tenantId: string | null, task: any) => { +export const doInTenant = (tenantId: string | null, task: any): any => { // make sure default always selected in single tenancy if (!env.MULTI_TENANCY) { tenantId = tenantId || DEFAULT_TENANT_ID } - // the internal function is so that we can re-use an existing - // context - don't want to close DB on a parent context - async function internal(opts = { existing: false }) { - // set the tenant id + global db if this is a new context - if (!opts.existing) { - updateTenantId(tenantId) - } - try { - // invoke the task - return await task() - } finally { - await closeWithUsing(ContextKey.TENANCY_IN_USE, () => { - return closeTenancy() - }) - } - } - - const existing = cls.getFromContext(ContextKey.TENANT_ID) === tenantId - return updateUsing(ContextKey.TENANCY_IN_USE, existing, internal) + return cls.run(async () => { + updateTenantId(tenantId) + return await task() + }) } -export const doInAppContext = (appId: string, task: any) => { +export const doInAppContext = (appId: string, task: any): any => { if (!appId) { throw new Error("appId is required") } const identity = getIdentity() - // the internal function is so that we can re-use an existing - // context - don't want to close DB on a parent context - async function internal(opts = { existing: false }) { + return cls.run(async () => { // set the app tenant id - if (!opts.existing) { - setAppTenantId(appId) - } + setAppTenantId(appId) // set the app ID cls.setOnContext(ContextKey.APP_ID, appId) @@ -126,47 +89,28 @@ export const doInAppContext = (appId: string, task: any) => { if (identity) { setIdentity(identity) } - try { - // invoke the task - return await task() - } finally { - await closeWithUsing(ContextKey.APP_IN_USE, async () => { - await closeAppDBs() - await closeTenancy() - }) - } - } - const existing = cls.getFromContext(ContextKey.APP_ID) === appId - return updateUsing(ContextKey.APP_IN_USE, existing, internal) + // invoke the task + return await task() + }) } -export const doInIdentityContext = (identity: IdentityContext, task: any) => { +export const doInIdentityContext = ( + identity: IdentityContext, + task: any +): any => { if (!identity) { throw new Error("identity is required") } - async function internal(opts = { existing: false }) { - if (!opts.existing) { - cls.setOnContext(ContextKey.IDENTITY, identity) - // set the tenant so that doInTenant will preserve identity - if (identity.tenantId) { - updateTenantId(identity.tenantId) - } + return cls.run(async () => { + cls.setOnContext(ContextKey.IDENTITY, identity) + // set the tenant so that doInTenant will preserve identity + if (identity.tenantId) { + updateTenantId(identity.tenantId) } - - try { - // invoke the task - return await task() - } finally { - await closeWithUsing(ContextKey.IDENTITY_IN_USE, async () => { - setIdentity(null) - await closeTenancy() - }) - } - } - - const existing = cls.getFromContext(ContextKey.IDENTITY) - return updateUsing(ContextKey.IDENTITY_IN_USE, existing, internal) + // invoke the task + return await task() + }) } export const getIdentity = (): IdentityContext | undefined => { @@ -179,15 +123,10 @@ export const getIdentity = (): IdentityContext | undefined => { export const updateTenantId = (tenantId: string | null) => { cls.setOnContext(ContextKey.TENANT_ID, tenantId) - if (env.USE_COUCH) { - setGlobalDB(tenantId) - } } export const updateAppId = async (appId: string) => { try { - // have to close first, before removing the databases from context - await closeAppDBs() cls.setOnContext(ContextKey.APP_ID, appId) } catch (err) { if (env.isTest()) { @@ -198,19 +137,9 @@ export const updateAppId = async (appId: string) => { } } -export const setGlobalDB = (tenantId: string | null) => { - const dbName = baseGlobalDBName(tenantId) - const db = dangerousGetDB(dbName) - cls.setOnContext(ContextKey.GLOBAL_DB, db) - return db -} - -export const getGlobalDB = () => { - const db = cls.getFromContext(ContextKey.GLOBAL_DB) - if (!db) { - throw new Error("Global DB not found") - } - return db +export const getGlobalDB = (): PouchLike => { + const tenantId = cls.getFromContext(ContextKey.TENANT_ID) + return new PouchLike(baseGlobalDBName(tenantId)) } export const isTenantIdSet = () => { @@ -246,22 +175,25 @@ export const isTenancyEnabled = () => { * Opens the app database based on whatever the request * contained, dev or prod. */ -export const getAppDB = (opts?: any) => { - return getContextDB(ContextKey.CURRENT_DB, opts) +export const getAppDB = (opts?: any): PouchLike => { + const appId = getAppId() + return new PouchLike(appId, opts) } /** * This specifically gets the prod app ID, if the request * contained a development app ID, this will open the prod one. */ -export const getProdAppDB = (opts?: any) => { - return getContextDB(ContextKey.PROD_DB, opts) +export const getProdAppDB = (opts?: any): PouchLike => { + const appId = getAppId() + return new PouchLike(getProdAppID(appId), opts) } /** * This specifically gets the dev app ID, if the request * contained a prod app ID, this will open the dev one. */ -export const getDevAppDB = (opts?: any) => { - return getContextDB(ContextKey.DEV_DB, opts) +export const getDevAppDB = (opts?: any): PouchLike => { + const appId = getAppId() + return new PouchLike(getDevelopmentAppID(appId), opts) } diff --git a/packages/backend-core/src/context/tests/index.spec.ts b/packages/backend-core/src/context/tests/index.spec.ts index 55ecd333a3..749231c3e5 100644 --- a/packages/backend-core/src/context/tests/index.spec.ts +++ b/packages/backend-core/src/context/tests/index.spec.ts @@ -5,8 +5,8 @@ import env from "../../environment" // must use require to spy index file exports due to known issue in jest const dbUtils = require("../../db") -jest.spyOn(dbUtils, "closeDB") -jest.spyOn(dbUtils, "dangerousGetDB") +jest.spyOn(dbUtils, "closePouchDB") +jest.spyOn(dbUtils, "getDB") describe("context", () => { beforeEach(() => { @@ -25,8 +25,8 @@ describe("context", () => { const db = context.getGlobalDB() expect(db.name).toBe("global-db") }) - expect(dbUtils.dangerousGetDB).toHaveBeenCalledTimes(1) - expect(dbUtils.closeDB).toHaveBeenCalledTimes(1) + expect(dbUtils.getDB).toHaveBeenCalledTimes(1) + expect(dbUtils.closePouchDB).toHaveBeenCalledTimes(1) }) }) @@ -85,8 +85,8 @@ describe("context", () => { const db = context.getGlobalDB() expect(db.name).toBe("test_global-db") }) - expect(dbUtils.dangerousGetDB).toHaveBeenCalledTimes(1) - expect(dbUtils.closeDB).toHaveBeenCalledTimes(1) + expect(dbUtils.getDB).toHaveBeenCalledTimes(1) + expect(dbUtils.closePouchDB).toHaveBeenCalledTimes(1) }) it("sets the tenant id when nested with same tenant id", async () => { @@ -123,8 +123,8 @@ describe("context", () => { }) // only 1 db is opened and closed - expect(dbUtils.dangerousGetDB).toHaveBeenCalledTimes(1) - expect(dbUtils.closeDB).toHaveBeenCalledTimes(1) + expect(dbUtils.getDB).toHaveBeenCalledTimes(1) + expect(dbUtils.closePouchDB).toHaveBeenCalledTimes(1) }) it("sets different tenant id inside another context", () => { diff --git a/packages/backend-core/src/context/utils.ts b/packages/backend-core/src/context/utils.ts deleted file mode 100644 index 6e7100b594..0000000000 --- a/packages/backend-core/src/context/utils.ts +++ /dev/null @@ -1,109 +0,0 @@ -import { - DEFAULT_TENANT_ID, - getAppId, - getTenantIDFromAppID, - updateTenantId, -} from "./index" -import cls from "./FunctionContext" -import { IdentityContext } from "@budibase/types" -import { ContextKey } from "./constants" -import { dangerousGetDB, closeDB } from "../db" -import { isEqual } from "lodash" -import { getDevelopmentAppID, getProdAppID } from "../db/conversions" -import env from "../environment" - -export async function updateUsing( - usingKey: string, - existing: boolean, - internal: (opts: { existing: boolean }) => Promise -) { - const using = cls.getFromContext(usingKey) - if (using && existing) { - cls.setOnContext(usingKey, using + 1) - return internal({ existing: true }) - } else { - return cls.run(async () => { - cls.setOnContext(usingKey, 1) - return internal({ existing: false }) - }) - } -} - -export async function closeWithUsing( - usingKey: string, - closeFn: () => Promise -) { - const using = cls.getFromContext(usingKey) - if (!using || using <= 1) { - await closeFn() - } else { - cls.setOnContext(usingKey, using - 1) - } -} - -export const setAppTenantId = (appId: string) => { - const appTenantId = getTenantIDFromAppID(appId) || DEFAULT_TENANT_ID - updateTenantId(appTenantId) -} - -export const setIdentity = (identity: IdentityContext | null) => { - cls.setOnContext(ContextKey.IDENTITY, identity) -} - -// this function makes sure the PouchDB objects are closed and -// fully deleted when finished - this protects against memory leaks -export async function closeAppDBs() { - const dbKeys = [ContextKey.CURRENT_DB, ContextKey.PROD_DB, ContextKey.DEV_DB] - for (let dbKey of dbKeys) { - const db = cls.getFromContext(dbKey) - if (!db) { - continue - } - await closeDB(db) - // clear the DB from context, incase someone tries to use it again - cls.setOnContext(dbKey, null) - } - // clear the app ID now that the databases are closed - if (cls.getFromContext(ContextKey.APP_ID)) { - cls.setOnContext(ContextKey.APP_ID, null) - } - if (cls.getFromContext(ContextKey.DB_OPTS)) { - cls.setOnContext(ContextKey.DB_OPTS, null) - } -} - -export function getContextDB(key: string, opts: any) { - const dbOptsKey = `${key}${ContextKey.DB_OPTS}` - let storedOpts = cls.getFromContext(dbOptsKey) - let db = cls.getFromContext(key) - if (db && isEqual(opts, storedOpts)) { - return db - } - - const appId = getAppId() - let toUseAppId - - switch (key) { - case ContextKey.CURRENT_DB: - toUseAppId = appId - break - case ContextKey.PROD_DB: - toUseAppId = getProdAppID(appId) - break - case ContextKey.DEV_DB: - toUseAppId = getDevelopmentAppID(appId) - break - } - db = dangerousGetDB(toUseAppId, opts) - try { - cls.setOnContext(key, db) - if (opts) { - cls.setOnContext(dbOptsKey, opts) - } - } catch (err) { - if (!env.isTest()) { - throw err - } - } - return db -} diff --git a/packages/backend-core/src/couch/index.ts b/packages/backend-core/src/couch/index.ts index daba8f74a9..7a4ac20486 100644 --- a/packages/backend-core/src/couch/index.ts +++ b/packages/backend-core/src/couch/index.ts @@ -1,3 +1,4 @@ export * from "./couch" export * from "./pouchLike" export * from "./utils" +export { init } from "./pouchDB" diff --git a/packages/backend-core/src/couch/pouchDB.ts b/packages/backend-core/src/couch/pouchDB.ts new file mode 100644 index 0000000000..74ad8a3dc6 --- /dev/null +++ b/packages/backend-core/src/couch/pouchDB.ts @@ -0,0 +1,37 @@ +import PouchDB from "pouchdb" +import env from "../environment" +import { PouchOptions } from "@budibase/types" +import * as pouch from "../db/pouch" + +let Pouch: any +let initialised = false + +export async function init(opts?: PouchOptions) { + Pouch = pouch.getPouch(opts) + initialised = true +} + +const checkInitialised = () => { + if (!initialised) { + throw new Error("init has not been called") + } +} + +export function getPouchDB(dbName: string, opts?: any): PouchDB.Database { + checkInitialised() + return new Pouch(dbName, opts) +} + +// use this function if you have called getPouchDB - close +// the databases you've opened once finished +export async function closePouchDB(db: PouchDB.Database) { + if (!db || env.isTest()) { + return + } + try { + // specifically await so that if there is an error, it can be ignored + return await db.close() + } catch (err) { + // ignore error, already closed + } +} diff --git a/packages/backend-core/src/couch/pouchLike.ts b/packages/backend-core/src/couch/pouchLike.ts index 2591380ba0..d605ec3d04 100644 --- a/packages/backend-core/src/couch/pouchLike.ts +++ b/packages/backend-core/src/couch/pouchLike.ts @@ -2,12 +2,16 @@ import Nano from "nano" import { AnyDocument } from "@budibase/types" import { getCouchInfo } from "./couch" import { directCouchCall } from "./utils" -import { getPouchDB } from "../db" +import { getPouchDB } from "./pouchDB" export type PouchLikeOpts = { skip_setup?: boolean } +export type PutOpts = { + force?: boolean +} + export type QueryOpts = { include_docs?: boolean startkey?: string @@ -19,6 +23,10 @@ export type QueryOpts = { keys?: string[] } +type QueryResp = Promise<{ + rows: { doc?: T | any; value?: any }[] +}> + export class PouchLike { public readonly name: string private static nano: Nano.ServerScope @@ -45,11 +53,15 @@ export class PouchLike { }) } + async exists() { + let response = await directCouchCall(`/${this.name}`, "HEAD") + return response.status === 200 + } + async checkSetup() { let shouldCreate = !this.pouchOpts?.skip_setup // check exists in a lightweight fashion - let response = await directCouchCall(`/${this.name}`, "HEAD") - let exists = response.status === 200 + let exists = await this.exists() if (!shouldCreate && !exists) { throw new Error("DB does not exist") } @@ -70,26 +82,43 @@ export class PouchLike { } } - async info() { - const db = PouchLike.nano.db.use(this.name) - return db.info() - } - - async get(id: string) { + async get(id?: string): Promise { const db = await this.checkSetup() + if (!id) { + throw new Error("Unable to get doc without a valid _id.") + } return this.updateOutput(() => db.get(id)) } - async remove(id: string, rev: string) { + async remove(id?: string, rev?: string) { const db = await this.checkSetup() + if (!id || !rev) { + throw new Error("Unable to remove doc without a valid _id and _rev.") + } return this.updateOutput(() => db.destroy(id, rev)) } - async put(document: AnyDocument) { + async put(document: AnyDocument, opts?: PutOpts) { if (!document._id) { throw new Error("Cannot store document without _id field.") } const db = await this.checkSetup() + if (!document.createdAt) { + document.createdAt = new Date().toISOString() + } + document.updatedAt = new Date().toISOString() + if (opts?.force && document._id) { + try { + const existing = await this.get(document._id) + if (existing) { + document._rev = existing._rev + } + } catch (err: any) { + if (err.status !== 404) { + throw err + } + } + } return this.updateOutput(() => db.insert(document)) } @@ -98,12 +127,12 @@ export class PouchLike { return this.updateOutput(() => db.bulk({ docs: documents })) } - async allDocs(params: QueryOpts) { + async allDocs(params: QueryOpts): QueryResp { const db = await this.checkSetup() return this.updateOutput(() => db.list(params)) } - async query(viewName: string, params: QueryOpts) { + async query(viewName: string, params: QueryOpts): QueryResp { const db = await this.checkSetup() const [database, view] = viewName.split("/") return this.updateOutput(() => db.view(database, view, params)) diff --git a/packages/backend-core/src/db/Replication.ts b/packages/backend-core/src/db/Replication.ts index e0bd3c7a43..c6eea8db5e 100644 --- a/packages/backend-core/src/db/Replication.ts +++ b/packages/backend-core/src/db/Replication.ts @@ -1,4 +1,4 @@ -import { dangerousGetDB, closeDB } from "." +import { getPouchDB, closePouchDB } from "../couch/pouchDB" import { DocumentType } from "./constants" class Replication { @@ -12,12 +12,12 @@ class Replication { * @param {String} target - the DB you want to replicate to, or rollback from */ constructor({ source, target }: any) { - this.source = dangerousGetDB(source) - this.target = dangerousGetDB(target) + this.source = getPouchDB(source) + this.target = getPouchDB(target) } close() { - return Promise.all([closeDB(this.source), closeDB(this.target)]) + return Promise.all([closePouchDB(this.source), closePouchDB(this.target)]) } promisify(operation: any, opts = {}) { @@ -68,7 +68,7 @@ class Replication { async rollback() { await this.target.destroy() // Recreate the DB again - this.target = dangerousGetDB(this.target.name) + this.target = getPouchDB(this.target.name) // take the opportunity to remove deleted tombstones await this.replicate() } diff --git a/packages/backend-core/src/db/index.ts b/packages/backend-core/src/db/index.ts index 73d1327af3..e732b3a2fb 100644 --- a/packages/backend-core/src/db/index.ts +++ b/packages/backend-core/src/db/index.ts @@ -1,87 +1,30 @@ -import * as pouch from "./pouch" import env from "../environment" -import { PouchOptions, CouchFindOptions } from "@budibase/types" -import PouchDB from "pouchdb" +import { CouchFindOptions } from "@budibase/types" import { PouchLike } from "../couch" import { directCouchQuery } from "../couch" -export { directCouchQuery } from "../couch" +export { init, PouchLike } from "../couch" -const openDbs: string[] = [] -let Pouch: any let initialised = false const dbList = new Set() -if (env.MEMORY_LEAK_CHECK) { - setInterval(() => { - console.log("--- OPEN DBS ---") - console.log(openDbs) - }, 5000) -} - -const put = - (dbPut: any) => - async (doc: any, options = {}) => { - if (!doc.createdAt) { - doc.createdAt = new Date().toISOString() - } - doc.updatedAt = new Date().toISOString() - return dbPut(doc, options) - } - const checkInitialised = () => { if (!initialised) { throw new Error("init has not been called") } } -export async function init(opts?: PouchOptions) { - Pouch = pouch.getPouch(opts) - initialised = true -} - -export function getPouchDB(dbName: string, opts?: any): PouchDB.Database { - checkInitialised() +export function getDB(dbName: string, opts?: any): PouchLike { if (env.isTest()) { dbList.add(dbName) } - const db = new Pouch(dbName, opts) - if (env.MEMORY_LEAK_CHECK) { - openDbs.push(db.name) - } - const dbPut = db.put - db.put = put(dbPut) - return db -} - -// NOTE: THIS IS A DANGEROUS FUNCTION - USE WITH CAUTION -// this function is prone to leaks, should only be used -// in situations that using the function doWithDB does not work -export function dangerousGetDB(dbName: string, opts?: any): PouchLike { return new PouchLike(dbName, opts) } -// use this function if you have called dangerousGetDB - close -// the databases you've opened once finished -export async function closeDB(db: PouchDB.Database) { - if (!db || env.isTest()) { - return - } - if (env.MEMORY_LEAK_CHECK) { - openDbs.splice(openDbs.indexOf(db.name), 1) - } - try { - // specifically await so that if there is an error, it can be ignored - return await db.close() - } catch (err) { - // ignore error, already closed - } -} - // we have to use a callback for this so that we can close // the DB when we're done, without this manual requests would // need to close the database when done with it to avoid memory leaks export async function doWithDB(dbName: string, cb: any, opts = {}) { - const db = dangerousGetDB(dbName, opts) + const db = getDB(dbName, opts) // need this to be async so that we can correctly close DB after all // async operations have been completed return await cb(db) diff --git a/packages/backend-core/src/db/tests/index.spec.js b/packages/backend-core/src/db/tests/index.spec.js index bc0c638126..8f81e6b8dc 100644 --- a/packages/backend-core/src/db/tests/index.spec.js +++ b/packages/backend-core/src/db/tests/index.spec.js @@ -1,11 +1,11 @@ require("../../../tests/utilities/TestConfiguration") -const { dangerousGetDB } = require("../") +const { getDB } = require("../") describe("db", () => { describe("getDB", () => { it("returns a db", async () => { - const db = dangerousGetDB("test") + const db = getDB("test") expect(db).toBeDefined() expect(db._adapter).toBe("memory") expect(db.prefix).toBe("_pouch_") @@ -13,7 +13,7 @@ describe("db", () => { }) it("uses the custom put function", async () => { - const db = dangerousGetDB("test") + const db = getDB("test") let doc = { _id: "test" } await db.put(doc) doc = await db.get(doc._id) diff --git a/packages/backend-core/src/db/views.ts b/packages/backend-core/src/db/views.ts index f0fff918fc..06919fd188 100644 --- a/packages/backend-core/src/db/views.ts +++ b/packages/backend-core/src/db/views.ts @@ -1,6 +1,6 @@ import { DocumentType, ViewName, DeprecatedViews, SEPARATOR } from "./utils" import { getGlobalDB } from "../context" -import PouchDB from "pouchdb" +import { PouchLike, QueryOpts } from "../couch" import { StaticDatabases } from "./constants" import { doWithDB } from "./" @@ -19,7 +19,7 @@ interface DesignDocument { views: any } -async function removeDeprecated(db: PouchDB.Database, viewName: ViewName) { +async function removeDeprecated(db: PouchLike, viewName: ViewName) { // @ts-ignore if (!DeprecatedViews[viewName]) { return @@ -70,16 +70,13 @@ export const createAccountEmailView = async () => { emit(doc.email.toLowerCase(), doc._id) } }` - await doWithDB( - StaticDatabases.PLATFORM_INFO.name, - async (db: PouchDB.Database) => { - await createView(db, viewJs, ViewName.ACCOUNT_BY_EMAIL) - } - ) + await doWithDB(StaticDatabases.PLATFORM_INFO.name, async (db: PouchLike) => { + await createView(db, viewJs, ViewName.ACCOUNT_BY_EMAIL) + }) } export const createUserAppView = async () => { - const db = getGlobalDB() as PouchDB.Database + const db = getGlobalDB() const viewJs = `function(doc) { if (doc._id.startsWith("${DocumentType.USER}${SEPARATOR}") && doc.roles) { for (let prodAppId of Object.keys(doc.roles)) { @@ -117,12 +114,9 @@ export const createPlatformUserView = async () => { emit(doc._id.toLowerCase(), doc._id) } }` - await doWithDB( - StaticDatabases.PLATFORM_INFO.name, - async (db: PouchDB.Database) => { - await createView(db, viewJs, ViewName.PLATFORM_USERS_LOWERCASE) - } - ) + await doWithDB(StaticDatabases.PLATFORM_INFO.name, async (db: PouchLike) => { + await createView(db, viewJs, ViewName.PLATFORM_USERS_LOWERCASE) + }) } export interface QueryViewOptions { @@ -131,13 +125,13 @@ export interface QueryViewOptions { export const queryView = async ( viewName: ViewName, - params: PouchDB.Query.Options, - db: PouchDB.Database, + params: QueryOpts, + db: PouchLike, createFunc: any, opts?: QueryViewOptions ): Promise => { try { - let response = await db.query(`database/${viewName}`, params) + let response = await db.query(`database/${viewName}`, params) const rows = response.rows const docs = rows.map(row => (params.include_docs ? row.doc : row.value)) @@ -161,7 +155,7 @@ export const queryView = async ( export const queryPlatformView = async ( viewName: ViewName, - params: PouchDB.Query.Options, + params: QueryOpts, opts?: QueryViewOptions ): Promise => { const CreateFuncByName: any = { @@ -169,19 +163,16 @@ export const queryPlatformView = async ( [ViewName.PLATFORM_USERS_LOWERCASE]: createPlatformUserView, } - return doWithDB( - StaticDatabases.PLATFORM_INFO.name, - async (db: PouchDB.Database) => { - const createFn = CreateFuncByName[viewName] - return queryView(viewName, params, db, createFn, opts) - } - ) + return doWithDB(StaticDatabases.PLATFORM_INFO.name, async (db: PouchLike) => { + const createFn = CreateFuncByName[viewName] + return queryView(viewName, params, db, createFn, opts) + }) } export const queryGlobalView = async ( viewName: ViewName, - params: PouchDB.Query.Options, - db?: PouchDB.Database, + params: QueryOpts, + db?: PouchLike, opts?: QueryViewOptions ): Promise => { const CreateFuncByName: any = { @@ -192,7 +183,7 @@ export const queryGlobalView = async ( } // can pass DB in if working with something specific if (!db) { - db = getGlobalDB() as PouchDB.Database + db = getGlobalDB() } const createFn = CreateFuncByName[viewName] return queryView(viewName, params, db, createFn, opts) diff --git a/packages/backend-core/src/environment.ts b/packages/backend-core/src/environment.ts index 6e2ac94be9..2443287d5a 100644 --- a/packages/backend-core/src/environment.ts +++ b/packages/backend-core/src/environment.ts @@ -69,7 +69,6 @@ const env = { DISABLE_DEVELOPER_LICENSE: process.env.DISABLE_DEVELOPER_LICENSE, DEFAULT_LICENSE: process.env.DEFAULT_LICENSE, SERVICE: process.env.SERVICE || "budibase", - MEMORY_LEAK_CHECK: process.env.MEMORY_LEAK_CHECK || false, LOG_LEVEL: process.env.LOG_LEVEL, SESSION_UPDATE_PERIOD: process.env.SESSION_UPDATE_PERIOD, DEPLOYMENT_ENVIRONMENT: diff --git a/packages/backend-core/src/index.ts b/packages/backend-core/src/index.ts index 17393b8ac3..06997ced90 100644 --- a/packages/backend-core/src/index.ts +++ b/packages/backend-core/src/index.ts @@ -21,6 +21,7 @@ import * as middleware from "./middleware" import plugins from "./plugin" import encryption from "./security/encryption" import * as queue from "./queue" +import * as types from "./types" // mimic the outer package exports import * as db from "./pkg/db" @@ -67,6 +68,7 @@ const core = { encryption, queue, permissions, + ...types, } export = core diff --git a/packages/backend-core/src/migrations/tests/index.spec.js b/packages/backend-core/src/migrations/tests/index.spec.js index c5ec143143..653ef453c5 100644 --- a/packages/backend-core/src/migrations/tests/index.spec.js +++ b/packages/backend-core/src/migrations/tests/index.spec.js @@ -1,6 +1,6 @@ require("../../../tests/utilities/TestConfiguration") const { runMigrations, getMigrationsDoc } = require("../index") -const { dangerousGetDB } = require("../../db") +const { getDB } = require("../../db") const { StaticDatabases, } = require("../../db/utils") @@ -18,7 +18,7 @@ describe("migrations", () => { }] beforeEach(() => { - db = dangerousGetDB(StaticDatabases.GLOBAL.name) + db = getDB(StaticDatabases.GLOBAL.name) }) afterEach(async () => { diff --git a/packages/backend-core/src/types.ts b/packages/backend-core/src/types.ts new file mode 100644 index 0000000000..5eb3109b27 --- /dev/null +++ b/packages/backend-core/src/types.ts @@ -0,0 +1 @@ +export { PouchLike } from "./couch" diff --git a/packages/backend-core/src/users.ts b/packages/backend-core/src/users.ts index 44f04749c9..a38debfc19 100644 --- a/packages/backend-core/src/users.ts +++ b/packages/backend-core/src/users.ts @@ -8,10 +8,9 @@ import { queryGlobalView } from "./db/views" import { UNICODE_MAX } from "./db/constants" import { BulkDocsResponse, User } from "@budibase/types" import { getGlobalDB } from "./context" -import PouchDB from "pouchdb" export const bulkGetGlobalUsersById = async (userIds: string[]) => { - const db = getGlobalDB() as PouchDB.Database + const db = getGlobalDB() return ( await db.allDocs({ keys: userIds, @@ -21,7 +20,7 @@ export const bulkGetGlobalUsersById = async (userIds: string[]) => { } export const bulkUpdateGlobalUsers = async (users: User[]) => { - const db = getGlobalDB() as PouchDB.Database + const db = getGlobalDB() return (await db.bulkDocs(users)) as BulkDocsResponse } diff --git a/packages/server/src/api/controllers/application.ts b/packages/server/src/api/controllers/application.ts index f3dca51f72..b97e980a5e 100644 --- a/packages/server/src/api/controllers/application.ts +++ b/packages/server/src/api/controllers/application.ts @@ -524,12 +524,10 @@ export const sync = async (ctx: any, next: any) => { // replicate prod to dev const prodAppId = getProdAppID(appId) - try { - // specific case, want to make sure setup is skipped - const prodDb = context.getProdAppDB({ skip_setup: true }) - const info = await prodDb.info() - if (info.error) throw info.error - } catch (err) { + // specific case, want to make sure setup is skipped + const prodDb = context.getProdAppDB({ skip_setup: true }) + const exists = await prodDb.exists() + if (!exists) { // the database doesn't exist. Don't replicate ctx.status = 200 ctx.body = { diff --git a/packages/server/src/api/controllers/row/internal.js b/packages/server/src/api/controllers/row/internal.js index 52f8a548fe..0f1324f10e 100644 --- a/packages/server/src/api/controllers/row/internal.js +++ b/packages/server/src/api/controllers/row/internal.js @@ -6,7 +6,7 @@ const { DocumentType, InternalTables, } = require("../../../db/utils") -const { dangerousGetDB } = require("@budibase/backend-core/db") +const { getDB } = require("@budibase/backend-core/db") const userController = require("../user") const { inputProcessing, @@ -251,7 +251,7 @@ exports.fetch = async ctx => { } exports.find = async ctx => { - const db = dangerousGetDB(ctx.appId) + const db = getDB(ctx.appId) const table = await db.get(ctx.params.tableId) let row = await findRow(ctx, ctx.params.tableId, ctx.params.rowId) row = await outputProcessing(table, row) diff --git a/packages/server/src/db/inMemoryView.js b/packages/server/src/db/inMemoryView.js index ec99b4738c..57ea89071c 100644 --- a/packages/server/src/db/inMemoryView.js +++ b/packages/server/src/db/inMemoryView.js @@ -2,7 +2,7 @@ const newid = require("./newid") // bypass the main application db config // use in memory pouchdb directly -const { getPouch, closeDB } = require("@budibase/backend-core/db") +const { getPouch, closePouchDB } = require("@budibase/backend-core/db") const Pouch = getPouch({ inMemory: true }) exports.runView = async (view, calculation, group, data) => { @@ -44,6 +44,6 @@ exports.runView = async (view, calculation, group, data) => { return response } finally { await db.destroy() - await closeDB(db) + await closePouchDB(db) } } diff --git a/packages/server/src/migrations/functions/backfill/app/tables.ts b/packages/server/src/migrations/functions/backfill/app/tables.ts index 6663c3c43b..c9d1e5c794 100644 --- a/packages/server/src/migrations/functions/backfill/app/tables.ts +++ b/packages/server/src/migrations/functions/backfill/app/tables.ts @@ -1,9 +1,8 @@ -import { events } from "@budibase/backend-core" +import { events, PouchLike } from "@budibase/backend-core" import sdk from "../../../../sdk" -import PouchDB from "pouchdb" export const backfill = async ( - appDb: PouchDB.Database, + appDb: PouchLike, timestamp: string | number ) => { const tables = await sdk.tables.getAllInternalTables(appDb) diff --git a/packages/server/src/sdk/app/backups/imports.ts b/packages/server/src/sdk/app/backups/imports.ts index 7cc9e0b0e6..634f507220 100644 --- a/packages/server/src/sdk/app/backups/imports.ts +++ b/packages/server/src/sdk/app/backups/imports.ts @@ -1,4 +1,4 @@ -import { db as dbCore } from "@budibase/backend-core" +import { db as dbCore, PouchLike } from "@budibase/backend-core" import { getAutomationParams, TABLE_ROW_PREFIX } from "../../../db/utils" import { budibaseTempDir } from "../../../utilities/budibaseDir" import { DB_EXPORT_FILE, GLOBAL_DB_EXPORT_FILE } from "./constants" @@ -17,7 +17,6 @@ import { CouchFindOptions, RowAttachment, } from "@budibase/types" -import PouchDB from "pouchdb" const uuid = require("uuid/v4") const tar = require("tar") @@ -29,10 +28,7 @@ type TemplateType = { key?: string } -async function updateAttachmentColumns( - prodAppId: string, - db: PouchDB.Database -) { +async function updateAttachmentColumns(prodAppId: string, db: PouchLike) { // iterate through attachment documents and update them const tables = await sdk.tables.getAllInternalTables(db) for (let table of tables) { @@ -86,7 +82,7 @@ async function updateAttachmentColumns( } } -async function updateAutomations(prodAppId: string, db: PouchDB.Database) { +async function updateAutomations(prodAppId: string, db: PouchLike) { const automations = ( await db.allDocs( getAutomationParams(null, { @@ -154,7 +150,7 @@ export function getListOfAppsInMulti(tmpPath: string) { export async function importApp( appId: string, - db: PouchDB.Database, + db: PouchLike, template: TemplateType ) { let prodAppId = dbCore.getProdAppID(appId) diff --git a/packages/server/src/sdk/app/backups/statistics.ts b/packages/server/src/sdk/app/backups/statistics.ts index 7a8e24dc58..9fe1a04d21 100644 --- a/packages/server/src/sdk/app/backups/statistics.ts +++ b/packages/server/src/sdk/app/backups/statistics.ts @@ -1,13 +1,12 @@ -import { context, db as dbCore } from "@budibase/backend-core" +import { context, db as dbCore, PouchLike } from "@budibase/backend-core" import { getDatasourceParams, getTableParams, getAutomationParams, getScreenParams, } from "../../../db/utils" -import PouchDB from "pouchdb" -async function runInContext(appId: string, cb: any, db?: PouchDB.Database) { +async function runInContext(appId: string, cb: any, db?: PouchLike) { if (db) { return cb(db) } else { @@ -19,13 +18,10 @@ async function runInContext(appId: string, cb: any, db?: PouchDB.Database) { } } -export async function calculateDatasourceCount( - appId: string, - db?: PouchDB.Database -) { +export async function calculateDatasourceCount(appId: string, db?: PouchLike) { return runInContext( appId, - async (db: PouchDB.Database) => { + async (db: PouchLike) => { const datasourceList = await db.allDocs(getDatasourceParams()) const tableList = await db.allDocs(getTableParams()) return datasourceList.rows.length + tableList.rows.length @@ -34,13 +30,10 @@ export async function calculateDatasourceCount( ) } -export async function calculateAutomationCount( - appId: string, - db?: PouchDB.Database -) { +export async function calculateAutomationCount(appId: string, db?: PouchLike) { return runInContext( appId, - async (db: PouchDB.Database) => { + async (db: PouchLike) => { const automationList = await db.allDocs(getAutomationParams()) return automationList.rows.length }, @@ -48,13 +41,10 @@ export async function calculateAutomationCount( ) } -export async function calculateScreenCount( - appId: string, - db?: PouchDB.Database -) { +export async function calculateScreenCount(appId: string, db?: PouchLike) { return runInContext( appId, - async (db: PouchDB.Database) => { + async (db: PouchLike) => { const screenList = await db.allDocs(getScreenParams()) return screenList.rows.length }, @@ -63,7 +53,7 @@ export async function calculateScreenCount( } export async function calculateBackupStats(appId: string) { - return runInContext(appId, async (db: PouchDB.Database) => { + return runInContext(appId, async (db: PouchLike) => { const promises = [] promises.push(calculateDatasourceCount(appId, db)) promises.push(calculateAutomationCount(appId, db)) diff --git a/packages/server/src/sdk/app/tables/index.ts b/packages/server/src/sdk/app/tables/index.ts index 5af92404a1..ef41630d6b 100644 --- a/packages/server/src/sdk/app/tables/index.ts +++ b/packages/server/src/sdk/app/tables/index.ts @@ -1,4 +1,4 @@ -import { getAppDB } from "@budibase/backend-core/context" +import { context, PouchLike } from "@budibase/backend-core" import { BudibaseInternalDB, getTableParams } from "../../../db/utils" import { breakExternalTableId, @@ -6,11 +6,10 @@ import { isSQL, } from "../../../integrations/utils" import { Table } from "@budibase/types" -import PouchDB from "pouchdb" -async function getAllInternalTables(db?: PouchDB.Database): Promise { +async function getAllInternalTables(db?: PouchLike): Promise { if (!db) { - db = getAppDB() as PouchDB.Database + db = context.getAppDB() } const internalTables = await db.allDocs( getTableParams(null, { @@ -25,7 +24,7 @@ async function getAllInternalTables(db?: PouchDB.Database): Promise { } async function getAllExternalTables(datasourceId: any): Promise { - const db = getAppDB() + const db = context.getAppDB() const datasource = await db.get(datasourceId) if (!datasource || !datasource.entities) { throw "Datasource is not configured fully." @@ -42,7 +41,7 @@ async function getExternalTable( } async function getTable(tableId: any): Promise { - const db = getAppDB() + const db = context.getAppDB() if (isExternalTable(tableId)) { let { datasourceId, tableName } = breakExternalTableId(tableId) const datasource = await db.get(datasourceId) From 9e01a9d1be447cd2c3eab047f5220bc6e8ea4e58 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 9 Nov 2022 17:53:35 +0000 Subject: [PATCH 13/91] Some minor updates based on getting the worker using the new pouchlike impl. --- packages/backend-core/src/couch/pouchLike.ts | 13 ++++++------- packages/backend-core/src/db/views.ts | 6 +++--- packages/types/src/documents/pouch.ts | 2 +- packages/worker/src/sdk/users/users.ts | 2 +- 4 files changed, 11 insertions(+), 12 deletions(-) diff --git a/packages/backend-core/src/couch/pouchLike.ts b/packages/backend-core/src/couch/pouchLike.ts index d605ec3d04..73555b1b7c 100644 --- a/packages/backend-core/src/couch/pouchLike.ts +++ b/packages/backend-core/src/couch/pouchLike.ts @@ -1,5 +1,5 @@ import Nano from "nano" -import { AnyDocument } from "@budibase/types" +import { AllDocsResponse, AnyDocument } from "@budibase/types" import { getCouchInfo } from "./couch" import { directCouchCall } from "./utils" import { getPouchDB } from "./pouchDB" @@ -23,10 +23,6 @@ export type QueryOpts = { keys?: string[] } -type QueryResp = Promise<{ - rows: { doc?: T | any; value?: any }[] -}> - export class PouchLike { public readonly name: string private static nano: Nano.ServerScope @@ -127,12 +123,15 @@ export class PouchLike { return this.updateOutput(() => db.bulk({ docs: documents })) } - async allDocs(params: QueryOpts): QueryResp { + async allDocs(params: QueryOpts): Promise> { const db = await this.checkSetup() return this.updateOutput(() => db.list(params)) } - async query(viewName: string, params: QueryOpts): QueryResp { + async query( + viewName: string, + params: QueryOpts + ): Promise> { const db = await this.checkSetup() const [database, view] = viewName.split("/") return this.updateOutput(() => db.view(database, view, params)) diff --git a/packages/backend-core/src/db/views.ts b/packages/backend-core/src/db/views.ts index 06919fd188..8ba508050f 100644 --- a/packages/backend-core/src/db/views.ts +++ b/packages/backend-core/src/db/views.ts @@ -131,16 +131,16 @@ export const queryView = async ( opts?: QueryViewOptions ): Promise => { try { - let response = await db.query(`database/${viewName}`, params) + let response = await db.query(`database/${viewName}`, params) const rows = response.rows const docs = rows.map(row => (params.include_docs ? row.doc : row.value)) // if arrayResponse has been requested, always return array regardless of length if (opts?.arrayResponse) { - return docs + return docs as T[] } else { // return the single document if there is only one - return docs.length <= 1 ? docs[0] : docs + return docs.length <= 1 ? (docs[0] as T) : (docs as T[]) } } catch (err: any) { if (err != null && err.name === "not_found") { diff --git a/packages/types/src/documents/pouch.ts b/packages/types/src/documents/pouch.ts index f9ed43b32f..fa66bfa57f 100644 --- a/packages/types/src/documents/pouch.ts +++ b/packages/types/src/documents/pouch.ts @@ -8,7 +8,7 @@ export interface RowResponse { key: string error: string value: RowValue - doc: T + doc?: T } export interface AllDocsResponse { diff --git a/packages/worker/src/sdk/users/users.ts b/packages/worker/src/sdk/users/users.ts index 539ac21300..e2215cee80 100644 --- a/packages/worker/src/sdk/users/users.ts +++ b/packages/worker/src/sdk/users/users.ts @@ -479,7 +479,7 @@ export const bulkDelete = async ( (user: RowResponse) => { return user.doc } - ) + ) as User[] // Delete from DB const toDelete = usersToDelete.map(user => ({ From 291514f57be9c4bdcc74da52ac00dffc1d7e9fcb Mon Sep 17 00:00:00 2001 From: Budibase Release Bot <> Date: Thu, 10 Nov 2022 10:58:45 +0000 Subject: [PATCH 14/91] v2.1.23 --- lerna.json | 2 +- packages/backend-core/package.json | 4 ++-- packages/bbui/package.json | 4 ++-- packages/builder/package.json | 10 +++++----- packages/cli/package.json | 8 ++++---- packages/client/package.json | 8 ++++---- packages/frontend-core/package.json | 4 ++-- packages/sdk/package.json | 2 +- packages/server/package.json | 10 +++++----- packages/string-templates/package.json | 2 +- packages/types/package.json | 2 +- packages/worker/package.json | 8 ++++---- 12 files changed, 32 insertions(+), 32 deletions(-) diff --git a/lerna.json b/lerna.json index 49766f3ab7..7b37c8f2ce 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.1.22", + "version": "2.1.23", "npmClient": "yarn", "packages": [ "packages/*" diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json index 584db9db53..ac74b7dbdd 100644 --- a/packages/backend-core/package.json +++ b/packages/backend-core/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/backend-core", - "version": "2.1.22", + "version": "2.1.23", "description": "Budibase backend core libraries used in server and worker", "main": "dist/src/index.js", "types": "dist/src/index.d.ts", @@ -20,7 +20,7 @@ "test:watch": "jest --watchAll" }, "dependencies": { - "@budibase/types": "^2.1.22", + "@budibase/types": "^2.1.23", "@shopify/jest-koa-mocks": "5.0.1", "@techpass/passport-openidconnect": "0.3.2", "aws-sdk": "2.1030.0", diff --git a/packages/bbui/package.json b/packages/bbui/package.json index 786defcec5..48243d51c4 100644 --- a/packages/bbui/package.json +++ b/packages/bbui/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/bbui", "description": "A UI solution used in the different Budibase projects.", - "version": "2.1.22", + "version": "2.1.23", "license": "MPL-2.0", "svelte": "src/index.js", "module": "dist/bbui.es.js", @@ -38,7 +38,7 @@ ], "dependencies": { "@adobe/spectrum-css-workflow-icons": "^1.2.1", - "@budibase/string-templates": "^2.1.22", + "@budibase/string-templates": "^2.1.23", "@spectrum-css/actionbutton": "^1.0.1", "@spectrum-css/actiongroup": "^1.0.1", "@spectrum-css/avatar": "^3.0.2", diff --git a/packages/builder/package.json b/packages/builder/package.json index dafb7ef7f7..abff9e0d36 100644 --- a/packages/builder/package.json +++ b/packages/builder/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/builder", - "version": "2.1.22", + "version": "2.1.23", "license": "GPL-3.0", "private": true, "scripts": { @@ -71,10 +71,10 @@ } }, "dependencies": { - "@budibase/bbui": "^2.1.22", - "@budibase/client": "^2.1.22", - "@budibase/frontend-core": "^2.1.22", - "@budibase/string-templates": "^2.1.22", + "@budibase/bbui": "^2.1.23", + "@budibase/client": "^2.1.23", + "@budibase/frontend-core": "^2.1.23", + "@budibase/string-templates": "^2.1.23", "@sentry/browser": "5.19.1", "@spectrum-css/page": "^3.0.1", "@spectrum-css/vars": "^3.0.1", diff --git a/packages/cli/package.json b/packages/cli/package.json index fbc2e97845..134b1e2bee 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/cli", - "version": "2.1.22", + "version": "2.1.23", "description": "Budibase CLI, for developers, self hosting and migrations.", "main": "src/index.js", "bin": { @@ -26,9 +26,9 @@ "outputPath": "build" }, "dependencies": { - "@budibase/backend-core": "^2.1.22", - "@budibase/string-templates": "^2.1.22", - "@budibase/types": "^2.1.22", + "@budibase/backend-core": "^2.1.23", + "@budibase/string-templates": "^2.1.23", + "@budibase/types": "^2.1.23", "axios": "0.21.2", "chalk": "4.1.0", "cli-progress": "3.11.2", diff --git a/packages/client/package.json b/packages/client/package.json index f834ebfb5a..2437080573 100644 --- a/packages/client/package.json +++ b/packages/client/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/client", - "version": "2.1.22", + "version": "2.1.23", "license": "MPL-2.0", "module": "dist/budibase-client.js", "main": "dist/budibase-client.js", @@ -19,9 +19,9 @@ "dev:builder": "rollup -cw" }, "dependencies": { - "@budibase/bbui": "^2.1.22", - "@budibase/frontend-core": "^2.1.22", - "@budibase/string-templates": "^2.1.22", + "@budibase/bbui": "^2.1.23", + "@budibase/frontend-core": "^2.1.23", + "@budibase/string-templates": "^2.1.23", "@spectrum-css/button": "^3.0.3", "@spectrum-css/card": "^3.0.3", "@spectrum-css/divider": "^1.0.3", diff --git a/packages/frontend-core/package.json b/packages/frontend-core/package.json index 80b3df6306..6be2f6de05 100644 --- a/packages/frontend-core/package.json +++ b/packages/frontend-core/package.json @@ -1,12 +1,12 @@ { "name": "@budibase/frontend-core", - "version": "2.1.22", + "version": "2.1.23", "description": "Budibase frontend core libraries used in builder and client", "author": "Budibase", "license": "MPL-2.0", "svelte": "src/index.js", "dependencies": { - "@budibase/bbui": "^2.1.22", + "@budibase/bbui": "^2.1.23", "lodash": "^4.17.21", "svelte": "^3.46.2" } diff --git a/packages/sdk/package.json b/packages/sdk/package.json index 4b63618094..15710f8e56 100644 --- a/packages/sdk/package.json +++ b/packages/sdk/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/sdk", - "version": "2.1.22", + "version": "2.1.23", "description": "Budibase Public API SDK", "author": "Budibase", "license": "MPL-2.0", diff --git a/packages/server/package.json b/packages/server/package.json index f4c5c85004..219da84e3c 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/server", "email": "hi@budibase.com", - "version": "2.1.22", + "version": "2.1.23", "description": "Budibase Web Server", "main": "src/index.ts", "repository": { @@ -77,11 +77,11 @@ "license": "GPL-3.0", "dependencies": { "@apidevtools/swagger-parser": "10.0.3", - "@budibase/backend-core": "^2.1.22", - "@budibase/client": "^2.1.22", + "@budibase/backend-core": "^2.1.23", + "@budibase/client": "^2.1.23", "@budibase/pro": "2.1.22", - "@budibase/string-templates": "^2.1.22", - "@budibase/types": "^2.1.22", + "@budibase/string-templates": "^2.1.23", + "@budibase/types": "^2.1.23", "@bull-board/api": "3.7.0", "@bull-board/koa": "3.9.4", "@elastic/elasticsearch": "7.10.0", diff --git a/packages/string-templates/package.json b/packages/string-templates/package.json index aeee568ced..5c04fb6c13 100644 --- a/packages/string-templates/package.json +++ b/packages/string-templates/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/string-templates", - "version": "2.1.22", + "version": "2.1.23", "description": "Handlebars wrapper for Budibase templating.", "main": "src/index.cjs", "module": "dist/bundle.mjs", diff --git a/packages/types/package.json b/packages/types/package.json index 7c26504406..ba17c37516 100644 --- a/packages/types/package.json +++ b/packages/types/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/types", - "version": "2.1.22", + "version": "2.1.23", "description": "Budibase types", "main": "dist/index.js", "types": "dist/index.d.ts", diff --git a/packages/worker/package.json b/packages/worker/package.json index a9cb2becb9..4cf3ae1b22 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/worker", "email": "hi@budibase.com", - "version": "2.1.22", + "version": "2.1.23", "description": "Budibase background service", "main": "src/index.ts", "repository": { @@ -36,10 +36,10 @@ "author": "Budibase", "license": "GPL-3.0", "dependencies": { - "@budibase/backend-core": "^2.1.22", + "@budibase/backend-core": "^2.1.23", "@budibase/pro": "2.1.22", - "@budibase/string-templates": "^2.1.22", - "@budibase/types": "^2.1.22", + "@budibase/string-templates": "^2.1.23", + "@budibase/types": "^2.1.23", "@koa/router": "8.0.8", "@sentry/node": "6.17.7", "@techpass/passport-openidconnect": "0.3.2", From bda23107606f1364dfebf3124b5b16e09b319c4b Mon Sep 17 00:00:00 2001 From: Budibase Release Bot <> Date: Thu, 10 Nov 2022 11:02:04 +0000 Subject: [PATCH 15/91] Update pro version to 2.1.23 --- packages/server/package.json | 2 +- packages/server/yarn.lock | 30 +++++++++++++++--------------- packages/worker/package.json | 2 +- packages/worker/yarn.lock | 30 +++++++++++++++--------------- 4 files changed, 32 insertions(+), 32 deletions(-) diff --git a/packages/server/package.json b/packages/server/package.json index 219da84e3c..47b00931b8 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -79,7 +79,7 @@ "@apidevtools/swagger-parser": "10.0.3", "@budibase/backend-core": "^2.1.23", "@budibase/client": "^2.1.23", - "@budibase/pro": "2.1.22", + "@budibase/pro": "2.1.23", "@budibase/string-templates": "^2.1.23", "@budibase/types": "^2.1.23", "@bull-board/api": "3.7.0", diff --git a/packages/server/yarn.lock b/packages/server/yarn.lock index 73deee3b20..a3ae885644 100644 --- a/packages/server/yarn.lock +++ b/packages/server/yarn.lock @@ -1094,12 +1094,12 @@ resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw== -"@budibase/backend-core@2.1.22": - version "2.1.22" - resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-2.1.22.tgz#5986f1ff2189dd789fe6184806d327ff2813d7b7" - integrity sha512-ReuWU75gkNJqHEcZHO15qCOW6M8YpADe7B4fyXDMl6eZOw20xm71X/y7vsegS9VXDSxvh8LJpLZTCo7EOaFwRg== +"@budibase/backend-core@2.1.23": + version "2.1.23" + resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-2.1.23.tgz#622e10844c64a0cc189d02d18f0ce2872d23110d" + integrity sha512-DvSd8XxQxsoGG8UrI7fxL7gzhgxUxXxT0k9kiF4sEi+MbC8fthFhoEhTfsxoPBBBfiHa9nQOZVrTC+3tmJBAPg== dependencies: - "@budibase/types" "^2.1.22" + "@budibase/types" "^2.1.23" "@shopify/jest-koa-mocks" "5.0.1" "@techpass/passport-openidconnect" "0.3.2" aws-sdk "2.1030.0" @@ -1181,13 +1181,13 @@ svelte-flatpickr "^3.2.3" svelte-portal "^1.0.0" -"@budibase/pro@2.1.22": - version "2.1.22" - resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.1.22.tgz#9df787c65375a556b84f314eea1866b7d3275c56" - integrity sha512-X+R6AqLtw5MJy4fosBlsNBI98Zp8oItRGK26M8LVlCCcjZTePSeGGwY1o1+r/eXYDHHilmOQl9s+EUQhSEQ2XA== +"@budibase/pro@2.1.23": + version "2.1.23" + resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.1.23.tgz#156a538fb40b3c8d67f39bf9c91c92bd45af011c" + integrity sha512-+o6Ie/z0WHrJdPWuq9YgyKylZatkn/lZSRCt2Vb89g20xRtWOsatQmUF5TdHrVzHGFh83Sp9NZE0QoemkvzZaA== dependencies: - "@budibase/backend-core" "2.1.22" - "@budibase/types" "2.1.22" + "@budibase/backend-core" "2.1.23" + "@budibase/types" "2.1.23" "@koa/router" "8.0.8" bull "4.10.1" joi "17.6.0" @@ -1211,10 +1211,10 @@ svelte-apexcharts "^1.0.2" svelte-flatpickr "^3.1.0" -"@budibase/types@2.1.22", "@budibase/types@^2.1.22": - version "2.1.22" - resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.1.22.tgz#7280072af0fedd9a24b715391481e7f6b475dc27" - integrity sha512-v2/w9XMP8hf6NsdpvS5aTfbNKhug08ZqpW0S9vtJwM/Afk/ZWKCj43r7O0Vs1IUODm2Oxev5lujugMF2HqPhVQ== +"@budibase/types@2.1.23", "@budibase/types@^2.1.23": + version "2.1.23" + resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.1.23.tgz#73495b52fa67aca67b3e728b7b84f190ee21b33e" + integrity sha512-VvdmYNTbL9KEL/kZeeCcMkCUjcXHu0YFNmqZgJIoIz0ssyAJf5kmqkNeGD9I4ORw4AMAUOcPVyObZXCyRSFBAg== "@bull-board/api@3.7.0": version "3.7.0" diff --git a/packages/worker/package.json b/packages/worker/package.json index 4cf3ae1b22..861f415dce 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -37,7 +37,7 @@ "license": "GPL-3.0", "dependencies": { "@budibase/backend-core": "^2.1.23", - "@budibase/pro": "2.1.22", + "@budibase/pro": "2.1.23", "@budibase/string-templates": "^2.1.23", "@budibase/types": "^2.1.23", "@koa/router": "8.0.8", diff --git a/packages/worker/yarn.lock b/packages/worker/yarn.lock index ff3f85db2e..0d3664f3e2 100644 --- a/packages/worker/yarn.lock +++ b/packages/worker/yarn.lock @@ -291,12 +291,12 @@ resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw== -"@budibase/backend-core@2.1.22": - version "2.1.22" - resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-2.1.22.tgz#5986f1ff2189dd789fe6184806d327ff2813d7b7" - integrity sha512-ReuWU75gkNJqHEcZHO15qCOW6M8YpADe7B4fyXDMl6eZOw20xm71X/y7vsegS9VXDSxvh8LJpLZTCo7EOaFwRg== +"@budibase/backend-core@2.1.23": + version "2.1.23" + resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-2.1.23.tgz#622e10844c64a0cc189d02d18f0ce2872d23110d" + integrity sha512-DvSd8XxQxsoGG8UrI7fxL7gzhgxUxXxT0k9kiF4sEi+MbC8fthFhoEhTfsxoPBBBfiHa9nQOZVrTC+3tmJBAPg== dependencies: - "@budibase/types" "^2.1.22" + "@budibase/types" "^2.1.23" "@shopify/jest-koa-mocks" "5.0.1" "@techpass/passport-openidconnect" "0.3.2" aws-sdk "2.1030.0" @@ -328,22 +328,22 @@ uuid "8.3.2" zlib "1.0.5" -"@budibase/pro@2.1.22": - version "2.1.22" - resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.1.22.tgz#9df787c65375a556b84f314eea1866b7d3275c56" - integrity sha512-X+R6AqLtw5MJy4fosBlsNBI98Zp8oItRGK26M8LVlCCcjZTePSeGGwY1o1+r/eXYDHHilmOQl9s+EUQhSEQ2XA== +"@budibase/pro@2.1.23": + version "2.1.23" + resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.1.23.tgz#156a538fb40b3c8d67f39bf9c91c92bd45af011c" + integrity sha512-+o6Ie/z0WHrJdPWuq9YgyKylZatkn/lZSRCt2Vb89g20xRtWOsatQmUF5TdHrVzHGFh83Sp9NZE0QoemkvzZaA== dependencies: - "@budibase/backend-core" "2.1.22" - "@budibase/types" "2.1.22" + "@budibase/backend-core" "2.1.23" + "@budibase/types" "2.1.23" "@koa/router" "8.0.8" bull "4.10.1" joi "17.6.0" node-fetch "^2.6.1" -"@budibase/types@2.1.22", "@budibase/types@^2.1.22": - version "2.1.22" - resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.1.22.tgz#7280072af0fedd9a24b715391481e7f6b475dc27" - integrity sha512-v2/w9XMP8hf6NsdpvS5aTfbNKhug08ZqpW0S9vtJwM/Afk/ZWKCj43r7O0Vs1IUODm2Oxev5lujugMF2HqPhVQ== +"@budibase/types@2.1.23", "@budibase/types@^2.1.23": + version "2.1.23" + resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.1.23.tgz#73495b52fa67aca67b3e728b7b84f190ee21b33e" + integrity sha512-VvdmYNTbL9KEL/kZeeCcMkCUjcXHu0YFNmqZgJIoIz0ssyAJf5kmqkNeGD9I4ORw4AMAUOcPVyObZXCyRSFBAg== "@cspotcode/source-map-consumer@0.8.0": version "0.8.0" From be32ef1cb6819aae1606aabd504a8019dd7e26e6 Mon Sep 17 00:00:00 2001 From: Mel O'Hagan Date: Thu, 10 Nov 2022 14:23:22 +0000 Subject: [PATCH 16/91] Add readable flag to datasource validation --- packages/backend-core/src/plugin/utils.js | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/backend-core/src/plugin/utils.js b/packages/backend-core/src/plugin/utils.js index 60a40f3a76..b943747483 100644 --- a/packages/backend-core/src/plugin/utils.js +++ b/packages/backend-core/src/plugin/utils.js @@ -51,6 +51,7 @@ function validateDatasource(schema) { const queryValidator = joi .object({ type: joi.string().allow(...Object.values(QueryType)), + readable: joi.boolean(), fields: joi.object().pattern(joi.string(), fieldValidator), }) .required() From 6d1449431ead6761ec667ed0724dec07c682aa75 Mon Sep 17 00:00:00 2001 From: Mel O'Hagan Date: Thu, 10 Nov 2022 15:22:05 +0000 Subject: [PATCH 17/91] Null pointer fix --- packages/server/src/sdk/app/backups/imports.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/sdk/app/backups/imports.ts b/packages/server/src/sdk/app/backups/imports.ts index 7cc9e0b0e6..6332716710 100644 --- a/packages/server/src/sdk/app/backups/imports.ts +++ b/packages/server/src/sdk/app/backups/imports.ts @@ -100,7 +100,7 @@ async function updateAutomations(prodAppId: string, db: PouchDB.Database) { const oldDevAppId = automation.appId, oldProdAppId = dbCore.getProdAppID(automation.appId) if ( - automation.definition.trigger.stepId === AutomationTriggerStepId.WEBHOOK + automation.definition.trigger?.stepId === AutomationTriggerStepId.WEBHOOK ) { const old = automation.definition.trigger.inputs automation.definition.trigger.inputs = { From 870bee4de53eaec6d356e66c7511ea112f6a8901 Mon Sep 17 00:00:00 2001 From: Budibase Release Bot <> Date: Thu, 10 Nov 2022 15:53:01 +0000 Subject: [PATCH 18/91] v2.1.24 --- lerna.json | 2 +- packages/backend-core/package.json | 4 ++-- packages/bbui/package.json | 4 ++-- packages/builder/package.json | 10 +++++----- packages/cli/package.json | 8 ++++---- packages/client/package.json | 8 ++++---- packages/frontend-core/package.json | 4 ++-- packages/sdk/package.json | 2 +- packages/server/package.json | 10 +++++----- packages/string-templates/package.json | 2 +- packages/types/package.json | 2 +- packages/worker/package.json | 8 ++++---- 12 files changed, 32 insertions(+), 32 deletions(-) diff --git a/lerna.json b/lerna.json index 7b37c8f2ce..becfc10031 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.1.23", + "version": "2.1.24", "npmClient": "yarn", "packages": [ "packages/*" diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json index ac74b7dbdd..40e0ff00f7 100644 --- a/packages/backend-core/package.json +++ b/packages/backend-core/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/backend-core", - "version": "2.1.23", + "version": "2.1.24", "description": "Budibase backend core libraries used in server and worker", "main": "dist/src/index.js", "types": "dist/src/index.d.ts", @@ -20,7 +20,7 @@ "test:watch": "jest --watchAll" }, "dependencies": { - "@budibase/types": "^2.1.23", + "@budibase/types": "^2.1.24", "@shopify/jest-koa-mocks": "5.0.1", "@techpass/passport-openidconnect": "0.3.2", "aws-sdk": "2.1030.0", diff --git a/packages/bbui/package.json b/packages/bbui/package.json index 48243d51c4..f1f81105e0 100644 --- a/packages/bbui/package.json +++ b/packages/bbui/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/bbui", "description": "A UI solution used in the different Budibase projects.", - "version": "2.1.23", + "version": "2.1.24", "license": "MPL-2.0", "svelte": "src/index.js", "module": "dist/bbui.es.js", @@ -38,7 +38,7 @@ ], "dependencies": { "@adobe/spectrum-css-workflow-icons": "^1.2.1", - "@budibase/string-templates": "^2.1.23", + "@budibase/string-templates": "^2.1.24", "@spectrum-css/actionbutton": "^1.0.1", "@spectrum-css/actiongroup": "^1.0.1", "@spectrum-css/avatar": "^3.0.2", diff --git a/packages/builder/package.json b/packages/builder/package.json index abff9e0d36..db3b6cc6bf 100644 --- a/packages/builder/package.json +++ b/packages/builder/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/builder", - "version": "2.1.23", + "version": "2.1.24", "license": "GPL-3.0", "private": true, "scripts": { @@ -71,10 +71,10 @@ } }, "dependencies": { - "@budibase/bbui": "^2.1.23", - "@budibase/client": "^2.1.23", - "@budibase/frontend-core": "^2.1.23", - "@budibase/string-templates": "^2.1.23", + "@budibase/bbui": "^2.1.24", + "@budibase/client": "^2.1.24", + "@budibase/frontend-core": "^2.1.24", + "@budibase/string-templates": "^2.1.24", "@sentry/browser": "5.19.1", "@spectrum-css/page": "^3.0.1", "@spectrum-css/vars": "^3.0.1", diff --git a/packages/cli/package.json b/packages/cli/package.json index 134b1e2bee..4aadb050b3 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/cli", - "version": "2.1.23", + "version": "2.1.24", "description": "Budibase CLI, for developers, self hosting and migrations.", "main": "src/index.js", "bin": { @@ -26,9 +26,9 @@ "outputPath": "build" }, "dependencies": { - "@budibase/backend-core": "^2.1.23", - "@budibase/string-templates": "^2.1.23", - "@budibase/types": "^2.1.23", + "@budibase/backend-core": "^2.1.24", + "@budibase/string-templates": "^2.1.24", + "@budibase/types": "^2.1.24", "axios": "0.21.2", "chalk": "4.1.0", "cli-progress": "3.11.2", diff --git a/packages/client/package.json b/packages/client/package.json index 2437080573..97c25473c1 100644 --- a/packages/client/package.json +++ b/packages/client/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/client", - "version": "2.1.23", + "version": "2.1.24", "license": "MPL-2.0", "module": "dist/budibase-client.js", "main": "dist/budibase-client.js", @@ -19,9 +19,9 @@ "dev:builder": "rollup -cw" }, "dependencies": { - "@budibase/bbui": "^2.1.23", - "@budibase/frontend-core": "^2.1.23", - "@budibase/string-templates": "^2.1.23", + "@budibase/bbui": "^2.1.24", + "@budibase/frontend-core": "^2.1.24", + "@budibase/string-templates": "^2.1.24", "@spectrum-css/button": "^3.0.3", "@spectrum-css/card": "^3.0.3", "@spectrum-css/divider": "^1.0.3", diff --git a/packages/frontend-core/package.json b/packages/frontend-core/package.json index 6be2f6de05..9723355dd4 100644 --- a/packages/frontend-core/package.json +++ b/packages/frontend-core/package.json @@ -1,12 +1,12 @@ { "name": "@budibase/frontend-core", - "version": "2.1.23", + "version": "2.1.24", "description": "Budibase frontend core libraries used in builder and client", "author": "Budibase", "license": "MPL-2.0", "svelte": "src/index.js", "dependencies": { - "@budibase/bbui": "^2.1.23", + "@budibase/bbui": "^2.1.24", "lodash": "^4.17.21", "svelte": "^3.46.2" } diff --git a/packages/sdk/package.json b/packages/sdk/package.json index 15710f8e56..97d401aee9 100644 --- a/packages/sdk/package.json +++ b/packages/sdk/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/sdk", - "version": "2.1.23", + "version": "2.1.24", "description": "Budibase Public API SDK", "author": "Budibase", "license": "MPL-2.0", diff --git a/packages/server/package.json b/packages/server/package.json index 47b00931b8..a92f17a73e 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/server", "email": "hi@budibase.com", - "version": "2.1.23", + "version": "2.1.24", "description": "Budibase Web Server", "main": "src/index.ts", "repository": { @@ -77,11 +77,11 @@ "license": "GPL-3.0", "dependencies": { "@apidevtools/swagger-parser": "10.0.3", - "@budibase/backend-core": "^2.1.23", - "@budibase/client": "^2.1.23", + "@budibase/backend-core": "^2.1.24", + "@budibase/client": "^2.1.24", "@budibase/pro": "2.1.23", - "@budibase/string-templates": "^2.1.23", - "@budibase/types": "^2.1.23", + "@budibase/string-templates": "^2.1.24", + "@budibase/types": "^2.1.24", "@bull-board/api": "3.7.0", "@bull-board/koa": "3.9.4", "@elastic/elasticsearch": "7.10.0", diff --git a/packages/string-templates/package.json b/packages/string-templates/package.json index 5c04fb6c13..36a8bd95f1 100644 --- a/packages/string-templates/package.json +++ b/packages/string-templates/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/string-templates", - "version": "2.1.23", + "version": "2.1.24", "description": "Handlebars wrapper for Budibase templating.", "main": "src/index.cjs", "module": "dist/bundle.mjs", diff --git a/packages/types/package.json b/packages/types/package.json index ba17c37516..9334dd6074 100644 --- a/packages/types/package.json +++ b/packages/types/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/types", - "version": "2.1.23", + "version": "2.1.24", "description": "Budibase types", "main": "dist/index.js", "types": "dist/index.d.ts", diff --git a/packages/worker/package.json b/packages/worker/package.json index 861f415dce..b70ee5e2f9 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/worker", "email": "hi@budibase.com", - "version": "2.1.23", + "version": "2.1.24", "description": "Budibase background service", "main": "src/index.ts", "repository": { @@ -36,10 +36,10 @@ "author": "Budibase", "license": "GPL-3.0", "dependencies": { - "@budibase/backend-core": "^2.1.23", + "@budibase/backend-core": "^2.1.24", "@budibase/pro": "2.1.23", - "@budibase/string-templates": "^2.1.23", - "@budibase/types": "^2.1.23", + "@budibase/string-templates": "^2.1.24", + "@budibase/types": "^2.1.24", "@koa/router": "8.0.8", "@sentry/node": "6.17.7", "@techpass/passport-openidconnect": "0.3.2", From 816382f63817da162223a9edc9eb2bad6f84e7b3 Mon Sep 17 00:00:00 2001 From: Budibase Release Bot <> Date: Thu, 10 Nov 2022 15:56:42 +0000 Subject: [PATCH 19/91] Update pro version to 2.1.24 --- packages/server/package.json | 2 +- packages/server/yarn.lock | 30 +++++++++++++++--------------- packages/worker/package.json | 2 +- packages/worker/yarn.lock | 30 +++++++++++++++--------------- 4 files changed, 32 insertions(+), 32 deletions(-) diff --git a/packages/server/package.json b/packages/server/package.json index a92f17a73e..5ed82b10a1 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -79,7 +79,7 @@ "@apidevtools/swagger-parser": "10.0.3", "@budibase/backend-core": "^2.1.24", "@budibase/client": "^2.1.24", - "@budibase/pro": "2.1.23", + "@budibase/pro": "2.1.24", "@budibase/string-templates": "^2.1.24", "@budibase/types": "^2.1.24", "@bull-board/api": "3.7.0", diff --git a/packages/server/yarn.lock b/packages/server/yarn.lock index a3ae885644..652056a0a8 100644 --- a/packages/server/yarn.lock +++ b/packages/server/yarn.lock @@ -1094,12 +1094,12 @@ resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw== -"@budibase/backend-core@2.1.23": - version "2.1.23" - resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-2.1.23.tgz#622e10844c64a0cc189d02d18f0ce2872d23110d" - integrity sha512-DvSd8XxQxsoGG8UrI7fxL7gzhgxUxXxT0k9kiF4sEi+MbC8fthFhoEhTfsxoPBBBfiHa9nQOZVrTC+3tmJBAPg== +"@budibase/backend-core@2.1.24": + version "2.1.24" + resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-2.1.24.tgz#4c0172359150cac624788c725cd6842a9a1b42e7" + integrity sha512-QjQ5bRNRWuiGB6J5MGaR1Uv6s6+jcCgrar4e2NCJn1Z+Ta7ZaBPI+6lH5d1vMLYjIUW9b/CavLtxpGazkfEPKQ== dependencies: - "@budibase/types" "^2.1.23" + "@budibase/types" "^2.1.24" "@shopify/jest-koa-mocks" "5.0.1" "@techpass/passport-openidconnect" "0.3.2" aws-sdk "2.1030.0" @@ -1181,13 +1181,13 @@ svelte-flatpickr "^3.2.3" svelte-portal "^1.0.0" -"@budibase/pro@2.1.23": - version "2.1.23" - resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.1.23.tgz#156a538fb40b3c8d67f39bf9c91c92bd45af011c" - integrity sha512-+o6Ie/z0WHrJdPWuq9YgyKylZatkn/lZSRCt2Vb89g20xRtWOsatQmUF5TdHrVzHGFh83Sp9NZE0QoemkvzZaA== +"@budibase/pro@2.1.24": + version "2.1.24" + resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.1.24.tgz#bec9461bde8f57f9b7648b82e381b763fc0cff30" + integrity sha512-FttB/wZVljPp8nKEnOK7B0gOa4ZJLkfq2jEA/7GyoI9kvht/QJx1sOqVaafHVh24cev6NsfCsMtxCiaiM42cOw== dependencies: - "@budibase/backend-core" "2.1.23" - "@budibase/types" "2.1.23" + "@budibase/backend-core" "2.1.24" + "@budibase/types" "2.1.24" "@koa/router" "8.0.8" bull "4.10.1" joi "17.6.0" @@ -1211,10 +1211,10 @@ svelte-apexcharts "^1.0.2" svelte-flatpickr "^3.1.0" -"@budibase/types@2.1.23", "@budibase/types@^2.1.23": - version "2.1.23" - resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.1.23.tgz#73495b52fa67aca67b3e728b7b84f190ee21b33e" - integrity sha512-VvdmYNTbL9KEL/kZeeCcMkCUjcXHu0YFNmqZgJIoIz0ssyAJf5kmqkNeGD9I4ORw4AMAUOcPVyObZXCyRSFBAg== +"@budibase/types@2.1.24", "@budibase/types@^2.1.24": + version "2.1.24" + resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.1.24.tgz#bb502e331eeecb3cf174e5222e174d45d952fb9d" + integrity sha512-Vbn70peWwRCGhgKSpfEz6hCHjF3tzyfMzToXHCagDKh96/qhHaH5pAX5ioIQ2lkWU97ymnS2bKL5fmq9e7xdwg== "@bull-board/api@3.7.0": version "3.7.0" diff --git a/packages/worker/package.json b/packages/worker/package.json index b70ee5e2f9..c0b3f99171 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -37,7 +37,7 @@ "license": "GPL-3.0", "dependencies": { "@budibase/backend-core": "^2.1.24", - "@budibase/pro": "2.1.23", + "@budibase/pro": "2.1.24", "@budibase/string-templates": "^2.1.24", "@budibase/types": "^2.1.24", "@koa/router": "8.0.8", diff --git a/packages/worker/yarn.lock b/packages/worker/yarn.lock index 0d3664f3e2..301332f393 100644 --- a/packages/worker/yarn.lock +++ b/packages/worker/yarn.lock @@ -291,12 +291,12 @@ resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw== -"@budibase/backend-core@2.1.23": - version "2.1.23" - resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-2.1.23.tgz#622e10844c64a0cc189d02d18f0ce2872d23110d" - integrity sha512-DvSd8XxQxsoGG8UrI7fxL7gzhgxUxXxT0k9kiF4sEi+MbC8fthFhoEhTfsxoPBBBfiHa9nQOZVrTC+3tmJBAPg== +"@budibase/backend-core@2.1.24": + version "2.1.24" + resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-2.1.24.tgz#4c0172359150cac624788c725cd6842a9a1b42e7" + integrity sha512-QjQ5bRNRWuiGB6J5MGaR1Uv6s6+jcCgrar4e2NCJn1Z+Ta7ZaBPI+6lH5d1vMLYjIUW9b/CavLtxpGazkfEPKQ== dependencies: - "@budibase/types" "^2.1.23" + "@budibase/types" "^2.1.24" "@shopify/jest-koa-mocks" "5.0.1" "@techpass/passport-openidconnect" "0.3.2" aws-sdk "2.1030.0" @@ -328,22 +328,22 @@ uuid "8.3.2" zlib "1.0.5" -"@budibase/pro@2.1.23": - version "2.1.23" - resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.1.23.tgz#156a538fb40b3c8d67f39bf9c91c92bd45af011c" - integrity sha512-+o6Ie/z0WHrJdPWuq9YgyKylZatkn/lZSRCt2Vb89g20xRtWOsatQmUF5TdHrVzHGFh83Sp9NZE0QoemkvzZaA== +"@budibase/pro@2.1.24": + version "2.1.24" + resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.1.24.tgz#bec9461bde8f57f9b7648b82e381b763fc0cff30" + integrity sha512-FttB/wZVljPp8nKEnOK7B0gOa4ZJLkfq2jEA/7GyoI9kvht/QJx1sOqVaafHVh24cev6NsfCsMtxCiaiM42cOw== dependencies: - "@budibase/backend-core" "2.1.23" - "@budibase/types" "2.1.23" + "@budibase/backend-core" "2.1.24" + "@budibase/types" "2.1.24" "@koa/router" "8.0.8" bull "4.10.1" joi "17.6.0" node-fetch "^2.6.1" -"@budibase/types@2.1.23", "@budibase/types@^2.1.23": - version "2.1.23" - resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.1.23.tgz#73495b52fa67aca67b3e728b7b84f190ee21b33e" - integrity sha512-VvdmYNTbL9KEL/kZeeCcMkCUjcXHu0YFNmqZgJIoIz0ssyAJf5kmqkNeGD9I4ORw4AMAUOcPVyObZXCyRSFBAg== +"@budibase/types@2.1.24", "@budibase/types@^2.1.24": + version "2.1.24" + resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.1.24.tgz#bb502e331eeecb3cf174e5222e174d45d952fb9d" + integrity sha512-Vbn70peWwRCGhgKSpfEz6hCHjF3tzyfMzToXHCagDKh96/qhHaH5pAX5ioIQ2lkWU97ymnS2bKL5fmq9e7xdwg== "@cspotcode/source-map-consumer@0.8.0": version "0.8.0" From 45e7ef61ef82c8a1d91e722ca6ee392847359b65 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 10 Nov 2022 16:38:32 +0000 Subject: [PATCH 20/91] Re-building the context module to use a single object, meaning we can create new context frames and copy over whatever exists, then update. --- .../backend-core/src/context/constants.ts | 4 + packages/backend-core/src/context/index.ts | 205 ++++++++++-------- packages/backend-core/src/couch/pouchLike.ts | 5 +- packages/backend-core/src/db/utils.ts | 17 +- .../backend-core/src/migrations/migrations.ts | 2 +- .../src/api/controllers/{dev.js => dev.ts} | 68 +++--- packages/server/src/middleware/currentapp.js | 3 +- 7 files changed, 159 insertions(+), 145 deletions(-) rename packages/server/src/api/controllers/{dev.js => dev.ts} (60%) diff --git a/packages/backend-core/src/context/constants.ts b/packages/backend-core/src/context/constants.ts index af30b1d241..14ab9a531c 100644 --- a/packages/backend-core/src/context/constants.ts +++ b/packages/backend-core/src/context/constants.ts @@ -1,4 +1,8 @@ export enum ContextKey { + MAIN = "main", +} + +export enum ContextElement { TENANT_ID = "tenantId", APP_ID = "appId", IDENTITY = "identity", diff --git a/packages/backend-core/src/context/index.ts b/packages/backend-core/src/context/index.ts index 30a53f1587..f88fe012a9 100644 --- a/packages/backend-core/src/context/index.ts +++ b/packages/backend-core/src/context/index.ts @@ -4,34 +4,36 @@ import cls from "./FunctionContext" import { baseGlobalDBName } from "../db/tenancy" import { IdentityContext } from "@budibase/types" import { DEFAULT_TENANT_ID as _DEFAULT_TENANT_ID } from "../constants" -import { ContextKey } from "./constants" +import { ContextElement, ContextKey } from "./constants" import { PouchLike } from "../couch" import { getDevelopmentAppID, getProdAppID } from "../db/conversions" +type ContextMap = { [key in ContextElement]?: any } + export const DEFAULT_TENANT_ID = _DEFAULT_TENANT_ID // some test cases call functions directly, need to // store an app ID to pretend there is a context let TEST_APP_ID: string | null = null -export const isMultiTenant = () => { +export function isMultiTenant() { return env.MULTI_TENANCY } -const setAppTenantId = (appId: string) => { - const appTenantId = getTenantIDFromAppID(appId) || DEFAULT_TENANT_ID - updateTenantId(appTenantId) +export function isTenantIdSet() { + const context = cls.getFromContext(ContextKey.MAIN) as ContextMap + return !!context?.[ContextElement.TENANT_ID] } -const setIdentity = (identity: IdentityContext | null) => { - cls.setOnContext(ContextKey.IDENTITY, identity) +export function isTenancyEnabled() { + return env.MULTI_TENANCY } /** * Given an app ID this will attempt to retrieve the tenant ID from it. * @return {null|string} The tenant ID found within the app ID. */ -export const getTenantIDFromAppID = (appId: string) => { +export function getTenantIDFromAppID(appId: string) { if (!appId) { return null } @@ -50,84 +52,134 @@ export const getTenantIDFromAppID = (appId: string) => { } } -export const doInContext = async (appId: string, task: any) => { - // gets the tenant ID from the app ID - const tenantId = getTenantIDFromAppID(appId) - return doInTenant(tenantId, async () => { - return doInAppContext(appId, async () => { - return task() - }) +function updateContext(updates: ContextMap) { + let context: ContextMap + try { + context = cls.getFromContext(ContextKey.MAIN) + } catch (err) { + // no context, start empty + context = {} + } + context = { + ...context, + ...updates, + } + return context +} + +async function newContext(updates: ContextMap, task: any) { + // see if there already is a context setup + let context: ContextMap = updateContext(updates) + return cls.run(async () => { + cls.setOnContext(ContextKey.MAIN, context) + return await task() }) } -export const doInTenant = (tenantId: string | null, task: any): any => { +export async function doInContext(appId: string, task: any): Promise { + const tenantId = getTenantIDFromAppID(appId) + return newContext( + { + [ContextElement.TENANT_ID]: tenantId, + [ContextElement.APP_ID]: appId, + }, + task + ) +} + +export async function doInTenant( + tenantId: string | null, + task: any +): Promise { // make sure default always selected in single tenancy if (!env.MULTI_TENANCY) { tenantId = tenantId || DEFAULT_TENANT_ID } - return cls.run(async () => { - updateTenantId(tenantId) - return await task() - }) + return newContext( + { + [ContextElement.TENANT_ID]: tenantId, + }, + task + ) } -export const doInAppContext = (appId: string, task: any): any => { +export async function doInAppContext(appId: string, task: any): Promise { if (!appId) { throw new Error("appId is required") } - const identity = getIdentity() - - return cls.run(async () => { - // set the app tenant id - setAppTenantId(appId) - // set the app ID - cls.setOnContext(ContextKey.APP_ID, appId) - - // preserve the identity - if (identity) { - setIdentity(identity) - } - // invoke the task - return await task() - }) + const tenantId = getTenantIDFromAppID(appId) + return newContext( + { + [ContextElement.TENANT_ID]: tenantId, + [ContextElement.APP_ID]: appId, + }, + task + ) } -export const doInIdentityContext = ( +export async function doInIdentityContext( identity: IdentityContext, task: any -): any => { +): Promise { if (!identity) { throw new Error("identity is required") } - return cls.run(async () => { - cls.setOnContext(ContextKey.IDENTITY, identity) - // set the tenant so that doInTenant will preserve identity - if (identity.tenantId) { - updateTenantId(identity.tenantId) - } - // invoke the task - return await task() - }) + const context: ContextMap = { + [ContextElement.IDENTITY]: identity, + } + if (identity.tenantId) { + context[ContextElement.TENANT_ID] = identity.tenantId + } + return newContext(context, task) } -export const getIdentity = (): IdentityContext | undefined => { +export function getIdentity(): IdentityContext | undefined { try { - return cls.getFromContext(ContextKey.IDENTITY) + const context = cls.getFromContext(ContextKey.MAIN) as ContextMap + return context?.[ContextElement.IDENTITY] } catch (e) { // do nothing - identity is not in context } } -export const updateTenantId = (tenantId: string | null) => { - cls.setOnContext(ContextKey.TENANT_ID, tenantId) +export function getTenantId(): string { + if (!isMultiTenant()) { + return DEFAULT_TENANT_ID + } + const context = cls.getFromContext(ContextKey.MAIN) as ContextMap + const tenantId = context?.[ContextElement.TENANT_ID] + if (!tenantId) { + throw new Error("Tenant id not found") + } + return tenantId } -export const updateAppId = async (appId: string) => { +export function getAppId(): string | undefined { + const context = cls.getFromContext(ContextKey.MAIN) as ContextMap + const foundId = context?.[ContextElement.APP_ID] + if (!foundId && env.isTest() && TEST_APP_ID) { + return TEST_APP_ID + } else { + return foundId + } +} + +export function updateTenantId(tenantId: string | null) { + let context: ContextMap = updateContext({ + [ContextElement.TENANT_ID]: tenantId, + }) + cls.setOnContext(ContextKey.MAIN, context) +} + +export function updateAppId(appId: string) { + let context: ContextMap = updateContext({ + [ContextElement.APP_ID]: appId, + }) try { - cls.setOnContext(ContextKey.APP_ID, appId) + cls.setOnContext(ContextKey.MAIN, context) } catch (err) { if (env.isTest()) { TEST_APP_ID = appId @@ -137,63 +189,34 @@ export const updateAppId = async (appId: string) => { } } -export const getGlobalDB = (): PouchLike => { - const tenantId = cls.getFromContext(ContextKey.TENANT_ID) - return new PouchLike(baseGlobalDBName(tenantId)) -} - -export const isTenantIdSet = () => { - const tenantId = cls.getFromContext(ContextKey.TENANT_ID) - return !!tenantId -} - -export const getTenantId = () => { - if (!isMultiTenant()) { - return DEFAULT_TENANT_ID - } - const tenantId = cls.getFromContext(ContextKey.TENANT_ID) - if (!tenantId) { - throw new Error("Tenant id not found") - } - return tenantId -} - -export const getAppId = () => { - const foundId = cls.getFromContext(ContextKey.APP_ID) - if (!foundId && env.isTest() && TEST_APP_ID) { - return TEST_APP_ID - } else { - return foundId - } -} - -export const isTenancyEnabled = () => { - return env.MULTI_TENANCY +export function getGlobalDB(): PouchLike { + const context = cls.getFromContext(ContextKey.MAIN) as ContextMap + return new PouchLike(baseGlobalDBName(context?.[ContextElement.TENANT_ID])) } /** - * Opens the app database based on whatever the request + * Gets the app database based on whatever the request * contained, dev or prod. */ -export const getAppDB = (opts?: any): PouchLike => { +export function getAppDB(opts?: any): PouchLike { const appId = getAppId() return new PouchLike(appId, opts) } /** * This specifically gets the prod app ID, if the request - * contained a development app ID, this will open the prod one. + * contained a development app ID, this will get the prod one. */ -export const getProdAppDB = (opts?: any): PouchLike => { +export function getProdAppDB(opts?: any): PouchLike { const appId = getAppId() return new PouchLike(getProdAppID(appId), opts) } /** * This specifically gets the dev app ID, if the request - * contained a prod app ID, this will open the dev one. + * contained a prod app ID, this will get the dev one. */ -export const getDevAppDB = (opts?: any): PouchLike => { +export function getDevAppDB(opts?: any): PouchLike { const appId = getAppId() return new PouchLike(getDevelopmentAppID(appId), opts) } diff --git a/packages/backend-core/src/couch/pouchLike.ts b/packages/backend-core/src/couch/pouchLike.ts index 73555b1b7c..fd5a94e1af 100644 --- a/packages/backend-core/src/couch/pouchLike.ts +++ b/packages/backend-core/src/couch/pouchLike.ts @@ -28,7 +28,10 @@ export class PouchLike { private static nano: Nano.ServerScope private readonly pouchOpts: PouchLikeOpts - constructor(dbName: string, opts?: PouchLikeOpts) { + constructor(dbName?: string, opts?: PouchLikeOpts) { + if (dbName == null) { + throw new Error("Database name cannot be undefined.") + } this.name = dbName this.pouchOpts = opts || {} if (!PouchLike.nano) { diff --git a/packages/backend-core/src/db/utils.ts b/packages/backend-core/src/db/utils.ts index c04da5da4f..a59b2ffe05 100644 --- a/packages/backend-core/src/db/utils.ts +++ b/packages/backend-core/src/db/utils.ts @@ -15,6 +15,7 @@ import { getAppMetadata } from "../cache/appMetadata" import { isDevApp, isDevAppID, getProdAppID } from "./conversions" import { APP_PREFIX } from "./constants" import * as events from "../events" +import { PouchLike } from "../couch" export * from "./constants" export * from "./conversions" @@ -254,7 +255,7 @@ export function getRoleParams(roleId = null, otherProps = {}) { return getDocParams(DocumentType.ROLE, roleId, otherProps) } -export function getStartEndKeyURL(baseKey: any, tenantId = null) { +export function getStartEndKeyURL(baseKey: any, tenantId?: string) { const tenancy = tenantId ? `${SEPARATOR}${tenantId}` : "" return `startkey="${baseKey}${tenancy}"&endkey="${baseKey}${tenancy}${UNICODE_MAX}"` } @@ -388,20 +389,10 @@ export async function getDevAppIDs() { } export async function dbExists(dbName: any) { - let exists = false return doWithDB( dbName, - async (db: any) => { - try { - // check if database exists - const info = await db.info() - if (info && !info.error) { - exists = true - } - } catch (err) { - exists = false - } - return exists + async (db: PouchLike) => { + return await db.exists() }, { skip_setup: true } ) diff --git a/packages/backend-core/src/migrations/migrations.ts b/packages/backend-core/src/migrations/migrations.ts index 90a12acec2..2170c5983a 100644 --- a/packages/backend-core/src/migrations/migrations.ts +++ b/packages/backend-core/src/migrations/migrations.ts @@ -41,7 +41,7 @@ export const runMigration = async ( options: MigrationOptions = {} ) => { const migrationType = migration.type - let tenantId: string + let tenantId: string | undefined if (migrationType !== MigrationType.INSTALLATION) { tenantId = getTenantId() } diff --git a/packages/server/src/api/controllers/dev.js b/packages/server/src/api/controllers/dev.ts similarity index 60% rename from packages/server/src/api/controllers/dev.js rename to packages/server/src/api/controllers/dev.ts index c8f134756b..9dbbe90555 100644 --- a/packages/server/src/api/controllers/dev.js +++ b/packages/server/src/api/controllers/dev.ts @@ -1,29 +1,23 @@ -const fetch = require("node-fetch") -const env = require("../../environment") -const { checkSlashesInUrl } = require("../../utilities") -const { request } = require("../../utilities/workerRequests") -const { clearLock } = require("../../utilities/redis") -const { Replication, getProdAppID } = require("@budibase/backend-core/db") -const { DocumentType } = require("../../db/utils") -const { app: appCache } = require("@budibase/backend-core/cache") -const { getProdAppDB, getAppDB } = require("@budibase/backend-core/context") -const { events } = require("@budibase/backend-core") +import fetch from "node-fetch" +import env from "../../environment" +import { checkSlashesInUrl } from "../../utilities" +import { request } from "../../utilities/workerRequests" +import { clearLock as redisClearLock } from "../../utilities/redis" +import { DocumentType } from "../../db/utils" +import { context } from "@budibase/backend-core" +import { events, db as dbCore, cache } from "@budibase/backend-core" -async function redirect(ctx, method, path = "global") { +async function redirect(ctx: any, method: string, path: string = "global") { const { devPath } = ctx.params const queryString = ctx.originalUrl.split("?")[1] || "" const response = await fetch( checkSlashesInUrl( `${env.WORKER_URL}/api/${path}/${devPath}?${queryString}` ), - request( - ctx, - { - method, - body: ctx.request.body, - }, - true - ) + request(ctx, { + method, + body: ctx.request.body, + }) ) if (response.status !== 200) { const err = await response.text() @@ -46,28 +40,28 @@ async function redirect(ctx, method, path = "global") { ctx.cookies } -exports.buildRedirectGet = path => { - return async ctx => { +export function buildRedirectGet(path: string) { + return async (ctx: any) => { await redirect(ctx, "GET", path) } } -exports.buildRedirectPost = path => { - return async ctx => { +export function buildRedirectPost(path: string) { + return async (ctx: any) => { await redirect(ctx, "POST", path) } } -exports.buildRedirectDelete = path => { - return async ctx => { +export function buildRedirectDelete(path: string) { + return async (ctx: any) => { await redirect(ctx, "DELETE", path) } } -exports.clearLock = async ctx => { +export async function clearLock(ctx: any) { const { appId } = ctx.params try { - await clearLock(appId, ctx.user) + await redisClearLock(appId, ctx.user) } catch (err) { ctx.throw(400, `Unable to remove lock. ${err}`) } @@ -76,16 +70,16 @@ exports.clearLock = async ctx => { } } -exports.revert = async ctx => { +export async function revert(ctx: any) { const { appId } = ctx.params - const productionAppId = getProdAppID(appId) + const productionAppId = dbCore.getProdAppID(appId) // App must have been deployed first try { - const db = getProdAppDB({ skip_setup: true }) - const info = await db.info() - if (info.error) { - throw info.error + const db = context.getProdAppDB({ skip_setup: true }) + const exists = await db.exists() + if (!exists) { + throw new Error("App must be deployed to be reverted.") } const deploymentDoc = await db.get(DocumentType.DEPLOYMENTS) if ( @@ -98,7 +92,7 @@ exports.revert = async ctx => { return ctx.throw(400, "App has not yet been deployed") } - const replication = new Replication({ + const replication = new dbCore.Replication({ source: productionAppId, target: appId, }) @@ -109,12 +103,12 @@ exports.revert = async ctx => { } // update appID in reverted app to be dev version again - const db = getAppDB() + const db = context.getAppDB() const appDoc = await db.get(DocumentType.APP_METADATA) appDoc.appId = appId appDoc.instance._id = appId await db.put(appDoc) - await appCache.invalidateAppMetadata(appId) + await cache.app.invalidateAppMetadata(appId) ctx.body = { message: "Reverted changes successfully.", } @@ -126,7 +120,7 @@ exports.revert = async ctx => { } } -exports.getBudibaseVersion = async ctx => { +export async function getBudibaseVersion(ctx: any) { const version = require("../../../package.json").version ctx.body = { version, diff --git a/packages/server/src/middleware/currentapp.js b/packages/server/src/middleware/currentapp.js index 8d55175d27..f9c4ae40db 100644 --- a/packages/server/src/middleware/currentapp.js +++ b/packages/server/src/middleware/currentapp.js @@ -95,8 +95,7 @@ module.exports = async (ctx, next) => { // need to judge this only based on the request app ID, if ( env.MULTI_TENANCY && - ctx.user && - requestAppId && + ctx.user & requestAppId && !isUserInAppTenant(requestAppId, ctx.user) ) { // don't error, simply remove the users rights (they are a public user) From c63c3b48c55f832ad2730ae128c18a7820b0c4fc Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 10 Nov 2022 17:38:26 +0000 Subject: [PATCH 21/91] Typescript conversions, as well as updating context to just use an object map. --- .../backend-core/src/context/constants.ts | 10 ++- packages/backend-core/src/context/index.ts | 44 +++++----- packages/backend-core/src/security/roles.ts | 8 +- packages/backend-core/src/tenancy/tenancy.ts | 43 ++++++---- .../{currentapp.js => currentapp.ts} | 84 ++++++++++--------- .../src/middleware/tests/currentapp.spec.js | 2 +- packages/types/src/sdk/koa.ts | 8 +- 7 files changed, 106 insertions(+), 93 deletions(-) rename packages/server/src/middleware/{currentapp.js => currentapp.ts} (58%) diff --git a/packages/backend-core/src/context/constants.ts b/packages/backend-core/src/context/constants.ts index 14ab9a531c..eb156c357b 100644 --- a/packages/backend-core/src/context/constants.ts +++ b/packages/backend-core/src/context/constants.ts @@ -1,9 +1,11 @@ +import { IdentityContext } from "@budibase/types" + export enum ContextKey { MAIN = "main", } -export enum ContextElement { - TENANT_ID = "tenantId", - APP_ID = "appId", - IDENTITY = "identity", +export type ContextMap = { + tenantId?: string + appId?: string + identity?: IdentityContext } diff --git a/packages/backend-core/src/context/index.ts b/packages/backend-core/src/context/index.ts index f88fe012a9..e8813c5a2f 100644 --- a/packages/backend-core/src/context/index.ts +++ b/packages/backend-core/src/context/index.ts @@ -4,12 +4,10 @@ import cls from "./FunctionContext" import { baseGlobalDBName } from "../db/tenancy" import { IdentityContext } from "@budibase/types" import { DEFAULT_TENANT_ID as _DEFAULT_TENANT_ID } from "../constants" -import { ContextElement, ContextKey } from "./constants" +import { ContextMap, ContextKey } from "./constants" import { PouchLike } from "../couch" import { getDevelopmentAppID, getProdAppID } from "../db/conversions" -type ContextMap = { [key in ContextElement]?: any } - export const DEFAULT_TENANT_ID = _DEFAULT_TENANT_ID // some test cases call functions directly, need to @@ -22,7 +20,7 @@ export function isMultiTenant() { export function isTenantIdSet() { const context = cls.getFromContext(ContextKey.MAIN) as ContextMap - return !!context?.[ContextElement.TENANT_ID] + return !!context?.tenantId } export function isTenancyEnabled() { @@ -35,7 +33,7 @@ export function isTenancyEnabled() { */ export function getTenantIDFromAppID(appId: string) { if (!appId) { - return null + return undefined } if (!isMultiTenant()) { return DEFAULT_TENANT_ID @@ -43,7 +41,7 @@ export function getTenantIDFromAppID(appId: string) { const split = appId.split(SEPARATOR) const hasDev = split[1] === DocumentType.DEV if ((hasDev && split.length === 3) || (!hasDev && split.length === 2)) { - return null + return undefined } if (hasDev) { return split[2] @@ -80,8 +78,8 @@ export async function doInContext(appId: string, task: any): Promise { const tenantId = getTenantIDFromAppID(appId) return newContext( { - [ContextElement.TENANT_ID]: tenantId, - [ContextElement.APP_ID]: appId, + tenantId, + appId, }, task ) @@ -96,12 +94,8 @@ export async function doInTenant( tenantId = tenantId || DEFAULT_TENANT_ID } - return newContext( - { - [ContextElement.TENANT_ID]: tenantId, - }, - task - ) + const updates = tenantId ? { tenantId } : {} + return newContext(updates, task) } export async function doInAppContext(appId: string, task: any): Promise { @@ -112,8 +106,8 @@ export async function doInAppContext(appId: string, task: any): Promise { const tenantId = getTenantIDFromAppID(appId) return newContext( { - [ContextElement.TENANT_ID]: tenantId, - [ContextElement.APP_ID]: appId, + tenantId, + appId, }, task ) @@ -128,10 +122,10 @@ export async function doInIdentityContext( } const context: ContextMap = { - [ContextElement.IDENTITY]: identity, + identity, } if (identity.tenantId) { - context[ContextElement.TENANT_ID] = identity.tenantId + context.tenantId = identity.tenantId } return newContext(context, task) } @@ -139,7 +133,7 @@ export async function doInIdentityContext( export function getIdentity(): IdentityContext | undefined { try { const context = cls.getFromContext(ContextKey.MAIN) as ContextMap - return context?.[ContextElement.IDENTITY] + return context?.identity } catch (e) { // do nothing - identity is not in context } @@ -150,7 +144,7 @@ export function getTenantId(): string { return DEFAULT_TENANT_ID } const context = cls.getFromContext(ContextKey.MAIN) as ContextMap - const tenantId = context?.[ContextElement.TENANT_ID] + const tenantId = context?.tenantId if (!tenantId) { throw new Error("Tenant id not found") } @@ -159,7 +153,7 @@ export function getTenantId(): string { export function getAppId(): string | undefined { const context = cls.getFromContext(ContextKey.MAIN) as ContextMap - const foundId = context?.[ContextElement.APP_ID] + const foundId = context?.appId if (!foundId && env.isTest() && TEST_APP_ID) { return TEST_APP_ID } else { @@ -167,16 +161,16 @@ export function getAppId(): string | undefined { } } -export function updateTenantId(tenantId: string | null) { +export function updateTenantId(tenantId?: string) { let context: ContextMap = updateContext({ - [ContextElement.TENANT_ID]: tenantId, + tenantId, }) cls.setOnContext(ContextKey.MAIN, context) } export function updateAppId(appId: string) { let context: ContextMap = updateContext({ - [ContextElement.APP_ID]: appId, + appId, }) try { cls.setOnContext(ContextKey.MAIN, context) @@ -191,7 +185,7 @@ export function updateAppId(appId: string) { export function getGlobalDB(): PouchLike { const context = cls.getFromContext(ContextKey.MAIN) as ContextMap - return new PouchLike(baseGlobalDBName(context?.[ContextElement.TENANT_ID])) + return new PouchLike(baseGlobalDBName(context?.tenantId)) } /** diff --git a/packages/backend-core/src/security/roles.ts b/packages/backend-core/src/security/roles.ts index 1064936fd7..cba88d9751 100644 --- a/packages/backend-core/src/security/roles.ts +++ b/packages/backend-core/src/security/roles.ts @@ -147,9 +147,9 @@ export function lowerBuiltinRoleID(roleId1?: string, roleId2?: string) { * @param {string|null} roleId The level ID to lookup. * @returns {Promise} The role object, which may contain an "inherits" property. */ -export async function getRole(roleId?: string) { +export async function getRole(roleId?: string): Promise { if (!roleId) { - return null + return undefined } let role: any = {} // built in roles mostly come from the in-code implementation, @@ -193,7 +193,9 @@ async function getAllUserRoles(userRoleId?: string): Promise { ) { roleIds.push(currentRole.inherits) currentRole = await getRole(currentRole.inherits) - roles.push(currentRole) + if (currentRole) { + roles.push(currentRole) + } } return roles } diff --git a/packages/backend-core/src/tenancy/tenancy.ts b/packages/backend-core/src/tenancy/tenancy.ts index ad5c6b5287..55ac66b95c 100644 --- a/packages/backend-core/src/tenancy/tenancy.ts +++ b/packages/backend-core/src/tenancy/tenancy.ts @@ -3,10 +3,10 @@ import { queryPlatformView } from "../db/views" import { StaticDatabases, ViewName } from "../db/constants" import { getGlobalDBName } from "../db/tenancy" import { - getTenantId, DEFAULT_TENANT_ID, - isMultiTenant, + getTenantId, getTenantIDFromAppID, + isMultiTenant, } from "../context" import env from "../environment" import { PlatformUser } from "@budibase/types" @@ -14,7 +14,7 @@ import { PlatformUser } from "@budibase/types" const TENANT_DOC = StaticDatabases.PLATFORM_INFO.docs.tenants const PLATFORM_INFO_DB = StaticDatabases.PLATFORM_INFO.name -export const addTenantToUrl = (url: string) => { +export function addTenantToUrl(url: string) { const tenantId = getTenantId() if (isMultiTenant()) { @@ -25,7 +25,7 @@ export const addTenantToUrl = (url: string) => { return url } -export const doesTenantExist = async (tenantId: string) => { +export async function doesTenantExist(tenantId: string) { return doWithDB(PLATFORM_INFO_DB, async (db: any) => { let tenants try { @@ -42,12 +42,12 @@ export const doesTenantExist = async (tenantId: string) => { }) } -export const tryAddTenant = async ( +export async function tryAddTenant( tenantId: string, userId: string, email: string, afterCreateTenant: () => Promise -) => { +) { return doWithDB(PLATFORM_INFO_DB, async (db: any) => { const getDoc = async (id: string) => { if (!id) { @@ -89,11 +89,11 @@ export const tryAddTenant = async ( }) } -export const doWithGlobalDB = (tenantId: string, cb: any) => { +export function doWithGlobalDB(tenantId: string, cb: any) { return doWithDB(getGlobalDBName(tenantId), cb) } -export const lookupTenantId = async (userId: string) => { +export async function lookupTenantId(userId: string) { return doWithDB(StaticDatabases.PLATFORM_INFO.name, async (db: any) => { let tenantId = env.MULTI_TENANCY ? DEFAULT_TENANT_ID : null try { @@ -109,19 +109,26 @@ export const lookupTenantId = async (userId: string) => { } // lookup, could be email or userId, either will return a doc -export const getTenantUser = async ( +export async function getTenantUser( identifier: string -): Promise => { +): Promise { // use the view here and allow to find anyone regardless of casing - // Use lowercase to ensure email login is case insensitive - const response = queryPlatformView(ViewName.PLATFORM_USERS_LOWERCASE, { - keys: [identifier.toLowerCase()], - include_docs: true, - }) as Promise - return response + // Use lowercase to ensure email login is case-insensitive + const users = await queryPlatformView( + ViewName.PLATFORM_USERS_LOWERCASE, + { + keys: [identifier.toLowerCase()], + include_docs: true, + } + ) + if (Array.isArray(users)) { + return users[0] + } else { + return users + } } -export const isUserInAppTenant = (appId: string, user?: any) => { +export function isUserInAppTenant(appId: string, user?: any) { let userTenantId if (user) { userTenantId = user.tenantId || DEFAULT_TENANT_ID @@ -132,7 +139,7 @@ export const isUserInAppTenant = (appId: string, user?: any) => { return tenantId === userTenantId } -export const getTenantIds = async () => { +export async function getTenantIds() { return doWithDB(PLATFORM_INFO_DB, async (db: any) => { let tenants try { diff --git a/packages/server/src/middleware/currentapp.js b/packages/server/src/middleware/currentapp.ts similarity index 58% rename from packages/server/src/middleware/currentapp.js rename to packages/server/src/middleware/currentapp.ts index f9c4ae40db..9496ddd0a0 100644 --- a/packages/server/src/middleware/currentapp.js +++ b/packages/server/src/middleware/currentapp.ts @@ -1,29 +1,26 @@ -const { - getAppIdFromCtx, - setCookie, - getCookie, - clearCookie, -} = require("@budibase/backend-core/utils") -const { Cookies, Headers } = require("@budibase/backend-core/constants") -const { getRole } = require("@budibase/backend-core/roles") -const { BUILTIN_ROLE_IDS } = require("@budibase/backend-core/roles") -const { generateUserMetadataID, isDevAppID } = require("../db/utils") -const { dbExists } = require("@budibase/backend-core/db") -const { isUserInAppTenant } = require("@budibase/backend-core/tenancy") -const { getCachedSelf } = require("../utilities/global") -const env = require("../environment") -const { isWebhookEndpoint } = require("./utils") -const { doInAppContext } = require("@budibase/backend-core/context") +import { + utils, + constants, + roles, + db as dbCore, + tenancy, + context, +} from "@budibase/backend-core" +import { generateUserMetadataID, isDevAppID } from "../db/utils" +import { getCachedSelf } from "../utilities/global" +import env from "../environment" +import { isWebhookEndpoint } from "./utils" +import { BBContext } from "@budibase/types" -module.exports = async (ctx, next) => { +export = async (ctx: BBContext, next: any) => { // try to get the appID from the request - let requestAppId = await getAppIdFromCtx(ctx) + let requestAppId = await utils.getAppIdFromCtx(ctx) // get app cookie if it exists - let appCookie = null + let appCookie: { appId?: string } | undefined try { - appCookie = getCookie(ctx, Cookies.CurrentApp) + appCookie = utils.getCookie(ctx, constants.Cookies.CurrentApp) } catch (err) { - clearCookie(ctx, Cookies.CurrentApp) + utils.clearCookie(ctx, constants.Cookies.CurrentApp) } if (!appCookie && !requestAppId) { return next() @@ -31,9 +28,9 @@ module.exports = async (ctx, next) => { // check the app exists referenced in cookie if (appCookie) { const appId = appCookie.appId - const exists = await dbExists(appId) + const exists = await dbCore.dbExists(appId) if (!exists) { - clearCookie(ctx, Cookies.CurrentApp) + utils.clearCookie(ctx, constants.Cookies.CurrentApp) return next() } // if the request app ID wasn't set, update it with the cookie @@ -47,13 +44,13 @@ module.exports = async (ctx, next) => { !isWebhookEndpoint(ctx) && (!ctx.user || !ctx.user.builder || !ctx.user.builder.global) ) { - clearCookie(ctx, Cookies.CurrentApp) + utils.clearCookie(ctx, constants.Cookies.CurrentApp) return ctx.redirect("/") } } - let appId, - roleId = BUILTIN_ROLE_IDS.PUBLIC + let appId: string | undefined, + roleId = roles.BUILTIN_ROLE_IDS.PUBLIC if (!ctx.user) { // not logged in, try to set a cookie for public apps appId = requestAppId @@ -68,16 +65,20 @@ module.exports = async (ctx, next) => { const isBuilder = globalUser && globalUser.builder && globalUser.builder.global const isDevApp = appId && isDevAppID(appId) - const roleHeader = ctx.request && ctx.request.headers[Headers.PREVIEW_ROLE] + const roleHeader = + ctx.request && + (ctx.request.headers[constants.Headers.PREVIEW_ROLE] as string) if (isBuilder && isDevApp && roleHeader) { // Ensure the role is valid by ensuring a definition exists try { - await getRole(roleHeader) - roleId = roleHeader + if (roleHeader) { + await roles.getRole(roleHeader) + roleId = roleHeader - // Delete admin and builder flags so that the specified role is honoured - delete ctx.user.builder - delete ctx.user.admin + // Delete admin and builder flags so that the specified role is honoured + delete ctx.user.builder + delete ctx.user.admin + } } catch (error) { // Swallow error and do nothing } @@ -89,21 +90,22 @@ module.exports = async (ctx, next) => { return next() } - return doInAppContext(appId, async () => { + return context.doInAppContext(appId, async () => { let skipCookie = false // if the user not in the right tenant then make sure they have no permissions // need to judge this only based on the request app ID, if ( env.MULTI_TENANCY && - ctx.user & requestAppId && - !isUserInAppTenant(requestAppId, ctx.user) + ctx.user && + requestAppId && + !tenancy.isUserInAppTenant(requestAppId, ctx.user) ) { // don't error, simply remove the users rights (they are a public user) delete ctx.user.builder delete ctx.user.admin delete ctx.user.roles ctx.isAuthenticated = false - roleId = BUILTIN_ROLE_IDS.PUBLIC + roleId = roles.BUILTIN_ROLE_IDS.PUBLIC skipCookie = true } @@ -111,15 +113,17 @@ module.exports = async (ctx, next) => { if (roleId) { ctx.roleId = roleId const globalId = ctx.user ? ctx.user._id : undefined - const userId = ctx.user ? generateUserMetadataID(ctx.user._id) : null + const userId = ctx.user + ? generateUserMetadataID(ctx.user._id!) + : undefined ctx.user = { - ...ctx.user, + ...ctx.user!, // override userID with metadata one _id: userId, userId, globalId, roleId, - role: await getRole(roleId), + role: await roles.getRole(roleId), } } if ( @@ -128,7 +132,7 @@ module.exports = async (ctx, next) => { appCookie.appId !== requestAppId) && !skipCookie ) { - setCookie(ctx, { appId }, Cookies.CurrentApp) + utils.setCookie(ctx, { appId }, constants.Cookies.CurrentApp) } return next() diff --git a/packages/server/src/middleware/tests/currentapp.spec.js b/packages/server/src/middleware/tests/currentapp.spec.js index 57c21b2107..638abcbba4 100644 --- a/packages/server/src/middleware/tests/currentapp.spec.js +++ b/packages/server/src/middleware/tests/currentapp.spec.js @@ -7,7 +7,7 @@ jest.mock("@budibase/backend-core/db", () => { coreDb.init() return { ...coreDb, - dbExists: () => true, + dbExists: async () => true, } }) diff --git a/packages/types/src/sdk/koa.ts b/packages/types/src/sdk/koa.ts index 8004ba72ae..b8831dcbee 100644 --- a/packages/types/src/sdk/koa.ts +++ b/packages/types/src/sdk/koa.ts @@ -1,10 +1,14 @@ import { Context, Request } from "koa" -import { User } from "../documents" +import { User, Role, UserRoles } from "../documents" import { License } from "../sdk" -export interface ContextUser extends User { +export interface ContextUser extends Omit { globalId?: string license: License + userId?: string + roleId?: string + role?: Role + roles?: UserRoles } export interface BBRequest extends Request { From a5a8b96be1315801e3ec60e9496f78895f034a6f Mon Sep 17 00:00:00 2001 From: Rory Powell Date: Fri, 11 Nov 2022 11:26:36 +0000 Subject: [PATCH 22/91] Fix logout race condition --- packages/builder/src/stores/portal/auth.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/builder/src/stores/portal/auth.js b/packages/builder/src/stores/portal/auth.js index 31b4533738..b10cd05e00 100644 --- a/packages/builder/src/stores/portal/auth.js +++ b/packages/builder/src/stores/portal/auth.js @@ -155,9 +155,9 @@ export function createAuthStore() { await actions.getSelf() }, logout: async () => { - setUser(null) - setPostLogout() await API.logOut() + setPostLogout() + setUser(null) await setInitInfo({}) }, updateSelf: async fields => { From bc94f20794428d58ba4ddca3dfaa7702df318e1f Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 11 Nov 2022 11:57:50 +0000 Subject: [PATCH 23/91] Typescript conversions - trying to get all of context/db layer into TS. --- ...{FunctionContext.js => FunctionContext.ts} | 14 ++-- .../{deprovision.js => deprovision.ts} | 68 ++++++++++--------- packages/backend-core/src/context/index.ts | 6 ++ .../src/db/{conversions.js => conversions.ts} | 21 +++--- packages/backend-core/src/index.ts | 2 +- packages/backend-core/src/users.ts | 2 +- .../src/api/controllers/deploy/index.ts | 38 ++++------- packages/server/src/automations/utils.ts | 21 +++--- packages/types/src/documents/pouch.ts | 2 +- 9 files changed, 85 insertions(+), 89 deletions(-) rename packages/backend-core/src/context/{FunctionContext.js => FunctionContext.ts} (79%) rename packages/backend-core/src/context/{deprovision.js => deprovision.ts} (64%) rename packages/backend-core/src/db/{conversions.js => conversions.ts} (68%) diff --git a/packages/backend-core/src/context/FunctionContext.js b/packages/backend-core/src/context/FunctionContext.ts similarity index 79% rename from packages/backend-core/src/context/FunctionContext.js rename to packages/backend-core/src/context/FunctionContext.ts index c0ed34fe78..1010f585ef 100644 --- a/packages/backend-core/src/context/FunctionContext.js +++ b/packages/backend-core/src/context/FunctionContext.ts @@ -1,10 +1,10 @@ -const cls = require("../clshooked") -const { newid } = require("../hashing") +import cls from "../clshooked" +import { newid } from "../hashing" const REQUEST_ID_KEY = "requestId" const MAIN_CTX = cls.createNamespace("main") -function getContextStorage(namespace) { +function getContextStorage(namespace: any) { if (namespace && namespace.active) { let contextData = namespace.active delete contextData.id @@ -15,7 +15,7 @@ function getContextStorage(namespace) { } class FunctionContext { - static run(callback) { + static run(callback: any) { return MAIN_CTX.runAndReturn(async () => { const namespaceId = newid() MAIN_CTX.set(REQUEST_ID_KEY, namespaceId) @@ -26,13 +26,13 @@ class FunctionContext { }) } - static setOnContext(key, value) { + static setOnContext(key: string, value: any) { const namespaceId = MAIN_CTX.get(REQUEST_ID_KEY) const namespace = cls.getNamespace(namespaceId) namespace.set(key, value) } - static getFromContext(key) { + static getFromContext(key: string) { const namespaceId = MAIN_CTX.get(REQUEST_ID_KEY) const namespace = cls.getNamespace(namespaceId) const context = getContextStorage(namespace) @@ -44,4 +44,4 @@ class FunctionContext { } } -module.exports = FunctionContext +export = FunctionContext diff --git a/packages/backend-core/src/context/deprovision.js b/packages/backend-core/src/context/deprovision.ts similarity index 64% rename from packages/backend-core/src/context/deprovision.js rename to packages/backend-core/src/context/deprovision.ts index ba3c2d8449..179c7f407a 100644 --- a/packages/backend-core/src/context/deprovision.js +++ b/packages/backend-core/src/context/deprovision.ts @@ -1,16 +1,20 @@ -const { getGlobalUserParams, getAllApps } = require("../db/utils") -const { doWithDB } = require("../db") -const { doWithGlobalDB } = require("../tenancy") -const { StaticDatabases } = require("../db/constants") +import { getGlobalUserParams, getAllApps } from "../db/utils" +import { doWithDB } from "../db" +import { doWithGlobalDB } from "../tenancy" +import { StaticDatabases } from "../db/constants" +import { PouchLike } from "../couch" +import { User } from "@budibase/types" const TENANT_DOC = StaticDatabases.PLATFORM_INFO.docs.tenants const PLATFORM_INFO_DB = StaticDatabases.PLATFORM_INFO.name -const removeTenantFromInfoDB = async tenantId => { +async function removeTenantFromInfoDB(tenantId: string) { try { - await doWithDB(PLATFORM_INFO_DB, async infoDb => { + await doWithDB(PLATFORM_INFO_DB, async (infoDb: PouchLike) => { let tenants = await infoDb.get(TENANT_DOC) - tenants.tenantIds = tenants.tenantIds.filter(id => id !== tenantId) + tenants.tenantIds = tenants.tenantIds.filter( + (id: string) => id !== tenantId + ) await infoDb.put(tenants) }) @@ -20,32 +24,15 @@ const removeTenantFromInfoDB = async tenantId => { } } -exports.removeUserFromInfoDB = async dbUser => { - await doWithDB(PLATFORM_INFO_DB, async infoDb => { - const keys = [dbUser._id, dbUser.email] - const userDocs = await infoDb.allDocs({ - keys, - include_docs: true, - }) - const toDelete = userDocs.rows.map(row => { - return { - ...row.doc, - _deleted: true, - } - }) - await infoDb.bulkDocs(toDelete) - }) -} - -const removeUsersFromInfoDB = async tenantId => { - return doWithGlobalDB(tenantId, async db => { +async function removeUsersFromInfoDB(tenantId: string) { + return doWithGlobalDB(tenantId, async (db: PouchLike) => { try { const allUsers = await db.allDocs( getGlobalUserParams(null, { include_docs: true, }) ) - await doWithDB(PLATFORM_INFO_DB, async infoDb => { + await doWithDB(PLATFORM_INFO_DB, async (infoDb: PouchLike) => { const allEmails = allUsers.rows.map(row => row.doc.email) // get the id docs let keys = allUsers.rows.map(row => row.id) @@ -71,8 +58,8 @@ const removeUsersFromInfoDB = async tenantId => { }) } -const removeGlobalDB = async tenantId => { - return doWithGlobalDB(tenantId, async db => { +async function removeGlobalDB(tenantId: string) { + return doWithGlobalDB(tenantId, async (db: PouchLike) => { try { await db.destroy() } catch (err) { @@ -82,11 +69,11 @@ const removeGlobalDB = async tenantId => { }) } -const removeTenantApps = async tenantId => { +async function removeTenantApps(tenantId: string) { try { const apps = await getAllApps({ all: true }) const destroyPromises = apps.map(app => - doWithDB(app.appId, db => db.destroy()) + doWithDB(app.appId, (db: PouchLike) => db.destroy()) ) await Promise.allSettled(destroyPromises) } catch (err) { @@ -95,8 +82,25 @@ const removeTenantApps = async tenantId => { } } +export async function removeUserFromInfoDB(dbUser: User) { + await doWithDB(PLATFORM_INFO_DB, async (infoDb: PouchLike) => { + const keys = [dbUser._id!, dbUser.email] + const userDocs = await infoDb.allDocs({ + keys, + include_docs: true, + }) + const toDelete = userDocs.rows.map(row => { + return { + ...row.doc, + _deleted: true, + } + }) + await infoDb.bulkDocs(toDelete) + }) +} + // can't live in tenancy package due to circular dependency on db/utils -exports.deleteTenant = async tenantId => { +export async function deleteTenant(tenantId: string) { await removeTenantFromInfoDB(tenantId) await removeUsersFromInfoDB(tenantId) await removeGlobalDB(tenantId) diff --git a/packages/backend-core/src/context/index.ts b/packages/backend-core/src/context/index.ts index e8813c5a2f..63b57a1b6e 100644 --- a/packages/backend-core/src/context/index.ts +++ b/packages/backend-core/src/context/index.ts @@ -203,6 +203,9 @@ export function getAppDB(opts?: any): PouchLike { */ export function getProdAppDB(opts?: any): PouchLike { const appId = getAppId() + if (!appId) { + throw new Error("Unable to retrieve prod DB - no app ID.") + } return new PouchLike(getProdAppID(appId), opts) } @@ -212,5 +215,8 @@ export function getProdAppDB(opts?: any): PouchLike { */ export function getDevAppDB(opts?: any): PouchLike { const appId = getAppId() + if (!appId) { + throw new Error("Unable to retrieve dev DB - no app ID.") + } return new PouchLike(getDevelopmentAppID(appId), opts) } diff --git a/packages/backend-core/src/db/conversions.js b/packages/backend-core/src/db/conversions.ts similarity index 68% rename from packages/backend-core/src/db/conversions.js rename to packages/backend-core/src/db/conversions.ts index 5b1a785ecc..48eaf31844 100644 --- a/packages/backend-core/src/db/conversions.js +++ b/packages/backend-core/src/db/conversions.ts @@ -1,32 +1,33 @@ +import { APP_DEV_PREFIX, APP_PREFIX } from "./constants" +import { App } from "@budibase/types" const NO_APP_ERROR = "No app provided" -const { APP_DEV_PREFIX, APP_PREFIX } = require("./constants") -exports.isDevAppID = appId => { +export function isDevAppID(appId?: string) { if (!appId) { throw NO_APP_ERROR } return appId.startsWith(APP_DEV_PREFIX) } -exports.isProdAppID = appId => { +export function isProdAppID(appId?: string) { if (!appId) { throw NO_APP_ERROR } - return appId.startsWith(APP_PREFIX) && !exports.isDevAppID(appId) + return appId.startsWith(APP_PREFIX) && !isDevAppID(appId) } -exports.isDevApp = app => { +export function isDevApp(app: App) { if (!app) { throw NO_APP_ERROR } - return exports.isDevAppID(app.appId) + return isDevAppID(app.appId) } /** * Generates a development app ID from a real app ID. * @returns {string} the dev app ID which can be used for dev database. */ -exports.getDevelopmentAppID = appId => { +export function getDevelopmentAppID(appId: string) { if (!appId || appId.startsWith(APP_DEV_PREFIX)) { return appId } @@ -36,12 +37,12 @@ exports.getDevelopmentAppID = appId => { const rest = split.join(APP_PREFIX) return `${APP_DEV_PREFIX}${rest}` } -exports.getDevAppID = exports.getDevelopmentAppID +export const getDevAppID = getDevelopmentAppID /** * Convert a development app ID to a deployed app ID. */ -exports.getProdAppID = appId => { +export function getProdAppID(appId: string) { if (!appId || !appId.startsWith(APP_DEV_PREFIX)) { return appId } @@ -52,7 +53,7 @@ exports.getProdAppID = appId => { return `${APP_PREFIX}${rest}` } -exports.extractAppUUID = id => { +export function extractAppUUID(id: string) { const split = id?.split("_") || [] return split.length ? split[split.length - 1] : null } diff --git a/packages/backend-core/src/index.ts b/packages/backend-core/src/index.ts index 06997ced90..b45248abc2 100644 --- a/packages/backend-core/src/index.ts +++ b/packages/backend-core/src/index.ts @@ -11,7 +11,7 @@ import env from "./environment" import tenancy from "./tenancy" import featureFlags from "./featureFlags" import * as sessions from "./security/sessions" -import deprovisioning from "./context/deprovision" +import * as deprovisioning from "./context/deprovision" import auth from "./auth" import constants from "./constants" import * as dbConstants from "./db/constants" diff --git a/packages/backend-core/src/users.ts b/packages/backend-core/src/users.ts index a38debfc19..13e8f8d36e 100644 --- a/packages/backend-core/src/users.ts +++ b/packages/backend-core/src/users.ts @@ -68,7 +68,7 @@ export const getGlobalUserByAppPage = (appId: string, user: User) => { if (!user) { return } - return generateAppUserID(getProdAppID(appId), user._id!) + return generateAppUserID(getProdAppID(appId)!, user._id!) } /** diff --git a/packages/server/src/api/controllers/deploy/index.ts b/packages/server/src/api/controllers/deploy/index.ts index cb4534a1a5..a87e7fbd7d 100644 --- a/packages/server/src/api/controllers/deploy/index.ts +++ b/packages/server/src/api/controllers/deploy/index.ts @@ -1,23 +1,11 @@ import Deployment from "./Deployment" -import { - getDevelopmentAppID, - getProdAppID, - Replication, -} from "@budibase/backend-core/db" +import { context, db as dbCore, events, cache } from "@budibase/backend-core" import { DocumentType, getAutomationParams } from "../../../db/utils" import { clearMetadata, disableAllCrons, enableCronTrigger, } from "../../../automations/utils" -import { app as appCache } from "@budibase/backend-core/cache" -import { - getAppDB, - getAppId, - getDevAppDB, - getProdAppDB, -} from "@budibase/backend-core/context" -import { events } from "@budibase/backend-core" import { backups } from "@budibase/pro" import { AppBackupTrigger } from "@budibase/types" @@ -49,7 +37,7 @@ async function checkAllDeployments(deployments: any) { async function storeDeploymentHistory(deployment: any) { const deploymentJSON = deployment.getJSON() - const db = getAppDB() + const db = context.getAppDB() let deploymentDoc try { @@ -77,7 +65,7 @@ async function storeDeploymentHistory(deployment: any) { } async function initDeployedApp(prodAppId: any) { - const db = getProdAppDB() + const db = context.getProdAppDB() console.log("Reading automation docs") const automations = ( await db.allDocs( @@ -103,9 +91,9 @@ async function initDeployedApp(prodAppId: any) { async function deployApp(deployment: any, userId: string) { let replication try { - const appId = getAppId() - const devAppId = getDevelopmentAppID(appId) - const productionAppId = getProdAppID(appId) + const appId = context.getAppId()! + const devAppId = dbCore.getDevelopmentAppID(appId) + const productionAppId = dbCore.getProdAppID(appId) // don't try this if feature isn't allowed, will error if (await backups.isEnabled()) { @@ -122,8 +110,8 @@ async function deployApp(deployment: any, userId: string) { source: devAppId, target: productionAppId, } - replication = new Replication(config) - const devDb = getDevAppDB() + replication = new dbCore.Replication(config) + const devDb = context.getDevAppDB() console.log("Compacting development DB") await devDb.compact() console.log("Replication object created") @@ -131,7 +119,7 @@ async function deployApp(deployment: any, userId: string) { console.log("replication complete.. replacing app meta doc") // app metadata is excluded as it is likely to be in conflict // replicate the app metadata document manually - const db = getProdAppDB() + const db = context.getProdAppDB() const appDoc = await devDb.get(DocumentType.APP_METADATA) try { const prodAppDoc = await db.get(DocumentType.APP_METADATA) @@ -147,7 +135,7 @@ async function deployApp(deployment: any, userId: string) { // remove automation errors if they exist delete appDoc.automationErrors await db.put(appDoc) - await appCache.invalidateAppMetadata(productionAppId) + await cache.app.invalidateAppMetadata(productionAppId) console.log("New app doc written successfully.") await initDeployedApp(productionAppId) console.log("Deployed app initialised, setting deployment to successful") @@ -170,7 +158,7 @@ async function deployApp(deployment: any, userId: string) { export async function fetchDeployments(ctx: any) { try { - const db = getAppDB() + const db = context.getAppDB() const deploymentDoc = await db.get(DocumentType.DEPLOYMENTS) const { updated, deployments } = await checkAllDeployments(deploymentDoc) if (updated) { @@ -184,7 +172,7 @@ export async function fetchDeployments(ctx: any) { export async function deploymentProgress(ctx: any) { try { - const db = getAppDB() + const db = context.getAppDB() const deploymentDoc = await db.get(DocumentType.DEPLOYMENTS) ctx.body = deploymentDoc[ctx.params.deploymentId] } catch (err) { @@ -197,7 +185,7 @@ export async function deploymentProgress(ctx: any) { const isFirstDeploy = async () => { try { - const db = getProdAppDB() + const db = context.getProdAppDB() await db.get(DocumentType.APP_METADATA) } catch (e: any) { if (e.status === 404) { diff --git a/packages/server/src/automations/utils.ts b/packages/server/src/automations/utils.ts index af4bb8d3af..5296a0fa50 100644 --- a/packages/server/src/automations/utils.ts +++ b/packages/server/src/automations/utils.ts @@ -4,15 +4,9 @@ import { automationQueue } from "./bullboard" import newid from "../db/newid" import { updateEntityMetadata } from "../utilities" import { MetadataTypes } from "../constants" -import { getProdAppID, doWithDB } from "@budibase/backend-core/db" +import { db as dbCore, context } from "@budibase/backend-core" import { getAutomationMetadataParams } from "../db/utils" import { cloneDeep } from "lodash/fp" -import { - getAppDB, - getAppId, - getProdAppDB, -} from "@budibase/backend-core/context" -import { context } from "@budibase/backend-core" import { quotas } from "@budibase/pro" import { Automation, WebhookActionType } from "@budibase/types" import sdk from "../sdk" @@ -102,7 +96,7 @@ export async function disableCronById(jobId: number | string) { } export async function clearMetadata() { - const db = getProdAppDB() + const db = context.getProdAppDB() const automationMetadata = ( await db.allDocs( getAutomationMetadataParams({ @@ -157,7 +151,7 @@ export async function enableCronTrigger(appId: any, automation: Automation) { // can't use getAppDB here as this is likely to be called from dev app, // but this call could be for dev app or prod app, need to just use what // was passed in - await doWithDB(appId, async (db: any) => { + await dbCore.doWithDB(appId, async (db: any) => { const response = await db.put(automation) automation._id = response.id automation._rev = response.rev @@ -175,7 +169,10 @@ export async function enableCronTrigger(appId: any, automation: Automation) { * written to DB (this does not write to DB as it would be wasteful to repeat). */ export async function checkForWebhooks({ oldAuto, newAuto }: any) { - const appId = getAppId() + const appId = context.getAppId() + if (!appId) { + throw new Error("Unable to check webhooks - no app ID in context.") + } const oldTrigger = oldAuto ? oldAuto.definition.trigger : null const newTrigger = newAuto ? newAuto.definition.trigger : null const triggerChanged = @@ -194,7 +191,7 @@ export async function checkForWebhooks({ oldAuto, newAuto }: any) { oldTrigger.webhookId ) { try { - let db = getAppDB() + let db = context.getAppDB() // need to get the webhook to get the rev const webhook = await db.get(oldTrigger.webhookId) // might be updating - reset the inputs to remove the URLs @@ -224,7 +221,7 @@ export async function checkForWebhooks({ oldAuto, newAuto }: any) { // the app ID has to be development for this endpoint // it can only be used when building the app // but the trigger endpoint will always be used in production - const prodAppId = getProdAppID(appId) + const prodAppId = dbCore.getProdAppID(appId) newTrigger.inputs = { schemaUrl: `api/webhooks/schema/${appId}/${id}`, triggerUrl: `api/webhooks/trigger/${prodAppId}/${id}`, diff --git a/packages/types/src/documents/pouch.ts b/packages/types/src/documents/pouch.ts index fa66bfa57f..269d8cbbae 100644 --- a/packages/types/src/documents/pouch.ts +++ b/packages/types/src/documents/pouch.ts @@ -8,7 +8,7 @@ export interface RowResponse { key: string error: string value: RowValue - doc?: T + doc?: T | any } export interface AllDocsResponse { From b4b471e054552ef13d1ba1f4e314f566da82f088 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 11 Nov 2022 12:46:32 +0000 Subject: [PATCH 24/91] Re-jigging things a bit, hiding as much of the couch/pouch stuff away. --- packages/backend-core/db.js | 1 - packages/backend-core/package.json | 3 +- .../backend-core/src/cache/writethrough.ts | 2 +- .../backend-core/src/context/deprovision.ts | 3 +- packages/backend-core/src/context/index.ts | 2 +- packages/backend-core/src/couch/couch.ts | 35 -------- packages/backend-core/src/couch/index.ts | 4 - packages/backend-core/src/couch/pouchDB.ts | 37 --------- packages/backend-core/src/db/Replication.ts | 2 +- .../backend-core/src/db/couch/connections.ts | 77 ++++++++++++++++++ packages/backend-core/src/db/couch/index.ts | 4 + .../src/db/{pouch.ts => couch/pouchDB.ts} | 80 ++++++++----------- .../src/{ => db}/couch/pouchLike.ts | 2 +- .../backend-core/src/{ => db}/couch/utils.ts | 4 +- packages/backend-core/src/db/index.ts | 5 +- packages/backend-core/src/db/utils.ts | 2 +- packages/backend-core/src/db/views.ts | 8 +- packages/backend-core/src/pkg/db.ts | 1 - packages/backend-core/src/types.ts | 2 +- packages/server/package.json | 3 +- packages/worker/package.json | 3 +- 21 files changed, 138 insertions(+), 142 deletions(-) delete mode 100644 packages/backend-core/src/couch/couch.ts delete mode 100644 packages/backend-core/src/couch/index.ts delete mode 100644 packages/backend-core/src/couch/pouchDB.ts create mode 100644 packages/backend-core/src/db/couch/connections.ts create mode 100644 packages/backend-core/src/db/couch/index.ts rename packages/backend-core/src/db/{pouch.ts => couch/pouchDB.ts} (52%) rename packages/backend-core/src/{ => db}/couch/pouchLike.ts (99%) rename packages/backend-core/src/{ => db}/couch/utils.ts (89%) diff --git a/packages/backend-core/db.js b/packages/backend-core/db.js index 0d2869d9f1..d2adf6c092 100644 --- a/packages/backend-core/db.js +++ b/packages/backend-core/db.js @@ -3,5 +3,4 @@ module.exports = { ...require("./src/db/constants"), ...require("./src/db"), ...require("./src/db/views"), - ...require("./src/db/pouch"), } diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json index 42a5f2bc53..2fbc087214 100644 --- a/packages/backend-core/package.json +++ b/packages/backend-core/package.json @@ -57,7 +57,8 @@ "preset": "ts-jest", "testEnvironment": "node", "moduleNameMapper": { - "@budibase/types": "/../types/src" + "@budibase/types": "/../types/src", + "axios": "axios/dist/node/axios.cjs" }, "setupFiles": [ "./scripts/jestSetup.ts" diff --git a/packages/backend-core/src/cache/writethrough.ts b/packages/backend-core/src/cache/writethrough.ts index 11dad81239..e13cc8cc0d 100644 --- a/packages/backend-core/src/cache/writethrough.ts +++ b/packages/backend-core/src/cache/writethrough.ts @@ -1,7 +1,7 @@ import BaseCache from "./base" import { getWritethroughClient } from "../redis/init" import { logWarn } from "../logging" -import { PouchLike } from "../couch" +import { PouchLike } from "../db" const DEFAULT_WRITE_RATE_MS = 10000 let CACHE: BaseCache | null = null diff --git a/packages/backend-core/src/context/deprovision.ts b/packages/backend-core/src/context/deprovision.ts index 179c7f407a..befcd48a15 100644 --- a/packages/backend-core/src/context/deprovision.ts +++ b/packages/backend-core/src/context/deprovision.ts @@ -1,8 +1,7 @@ import { getGlobalUserParams, getAllApps } from "../db/utils" -import { doWithDB } from "../db" +import { doWithDB, PouchLike } from "../db" import { doWithGlobalDB } from "../tenancy" import { StaticDatabases } from "../db/constants" -import { PouchLike } from "../couch" import { User } from "@budibase/types" const TENANT_DOC = StaticDatabases.PLATFORM_INFO.docs.tenants diff --git a/packages/backend-core/src/context/index.ts b/packages/backend-core/src/context/index.ts index 63b57a1b6e..34e6603ef7 100644 --- a/packages/backend-core/src/context/index.ts +++ b/packages/backend-core/src/context/index.ts @@ -5,7 +5,7 @@ import { baseGlobalDBName } from "../db/tenancy" import { IdentityContext } from "@budibase/types" import { DEFAULT_TENANT_ID as _DEFAULT_TENANT_ID } from "../constants" import { ContextMap, ContextKey } from "./constants" -import { PouchLike } from "../couch" +import { PouchLike } from "../db" import { getDevelopmentAppID, getProdAppID } from "../db/conversions" export const DEFAULT_TENANT_ID = _DEFAULT_TENANT_ID diff --git a/packages/backend-core/src/couch/couch.ts b/packages/backend-core/src/couch/couch.ts deleted file mode 100644 index 41fb83cb86..0000000000 --- a/packages/backend-core/src/couch/couch.ts +++ /dev/null @@ -1,35 +0,0 @@ -import env from "../environment" -import { getUrlInfo } from "../db/pouch" - -export const getCouchInfo = () => { - const urlInfo = getUrlInfo() - let username - let password - if (env.COUCH_DB_USERNAME) { - // set from env - username = env.COUCH_DB_USERNAME - } else if (urlInfo.auth.username) { - // set from url - username = urlInfo.auth.username - } else if (!env.isTest()) { - throw new Error("CouchDB username not set") - } - if (env.COUCH_DB_PASSWORD) { - // set from env - password = env.COUCH_DB_PASSWORD - } else if (urlInfo.auth.password) { - // set from url - password = urlInfo.auth.password - } else if (!env.isTest()) { - throw new Error("CouchDB password not set") - } - const authCookie = Buffer.from(`${username}:${password}`).toString("base64") - return { - url: urlInfo.url!, - auth: { - username: username, - password: password, - }, - cookie: `Basic ${authCookie}`, - } -} diff --git a/packages/backend-core/src/couch/index.ts b/packages/backend-core/src/couch/index.ts deleted file mode 100644 index 7a4ac20486..0000000000 --- a/packages/backend-core/src/couch/index.ts +++ /dev/null @@ -1,4 +0,0 @@ -export * from "./couch" -export * from "./pouchLike" -export * from "./utils" -export { init } from "./pouchDB" diff --git a/packages/backend-core/src/couch/pouchDB.ts b/packages/backend-core/src/couch/pouchDB.ts deleted file mode 100644 index 74ad8a3dc6..0000000000 --- a/packages/backend-core/src/couch/pouchDB.ts +++ /dev/null @@ -1,37 +0,0 @@ -import PouchDB from "pouchdb" -import env from "../environment" -import { PouchOptions } from "@budibase/types" -import * as pouch from "../db/pouch" - -let Pouch: any -let initialised = false - -export async function init(opts?: PouchOptions) { - Pouch = pouch.getPouch(opts) - initialised = true -} - -const checkInitialised = () => { - if (!initialised) { - throw new Error("init has not been called") - } -} - -export function getPouchDB(dbName: string, opts?: any): PouchDB.Database { - checkInitialised() - return new Pouch(dbName, opts) -} - -// use this function if you have called getPouchDB - close -// the databases you've opened once finished -export async function closePouchDB(db: PouchDB.Database) { - if (!db || env.isTest()) { - return - } - try { - // specifically await so that if there is an error, it can be ignored - return await db.close() - } catch (err) { - // ignore error, already closed - } -} diff --git a/packages/backend-core/src/db/Replication.ts b/packages/backend-core/src/db/Replication.ts index c6eea8db5e..12f6001a70 100644 --- a/packages/backend-core/src/db/Replication.ts +++ b/packages/backend-core/src/db/Replication.ts @@ -1,4 +1,4 @@ -import { getPouchDB, closePouchDB } from "../couch/pouchDB" +import { getPouchDB, closePouchDB } from "./couch/pouchDB" import { DocumentType } from "./constants" class Replication { diff --git a/packages/backend-core/src/db/couch/connections.ts b/packages/backend-core/src/db/couch/connections.ts new file mode 100644 index 0000000000..a2206de634 --- /dev/null +++ b/packages/backend-core/src/db/couch/connections.ts @@ -0,0 +1,77 @@ +import env from "../../environment" + +export const getCouchInfo = () => { + const urlInfo = getUrlInfo() + let username + let password + if (env.COUCH_DB_USERNAME) { + // set from env + username = env.COUCH_DB_USERNAME + } else if (urlInfo.auth.username) { + // set from url + username = urlInfo.auth.username + } else if (!env.isTest()) { + throw new Error("CouchDB username not set") + } + if (env.COUCH_DB_PASSWORD) { + // set from env + password = env.COUCH_DB_PASSWORD + } else if (urlInfo.auth.password) { + // set from url + password = urlInfo.auth.password + } else if (!env.isTest()) { + throw new Error("CouchDB password not set") + } + const authCookie = Buffer.from(`${username}:${password}`).toString("base64") + return { + url: urlInfo.url!, + auth: { + username: username, + password: password, + }, + cookie: `Basic ${authCookie}`, + } +} + +export const getUrlInfo = (url = env.COUCH_DB_URL) => { + let cleanUrl, username, password, host + if (url) { + // Ensure the URL starts with a protocol + const protoRegex = /^https?:\/\//i + if (!protoRegex.test(url)) { + url = `http://${url}` + } + + // Split into protocol and remainder + const split = url.split("://") + const protocol = split[0] + const rest = split.slice(1).join("://") + + // Extract auth if specified + if (url.includes("@")) { + // Split into host and remainder + let parts = rest.split("@") + host = parts[parts.length - 1] + let auth = parts.slice(0, -1).join("@") + + // Split auth into username and password + if (auth.includes(":")) { + const authParts = auth.split(":") + username = authParts[0] + password = authParts.slice(1).join(":") + } else { + username = auth + } + } else { + host = rest + } + cleanUrl = `${protocol}://${host}` + } + return { + url: cleanUrl, + auth: { + username, + password, + }, + } +} diff --git a/packages/backend-core/src/db/couch/index.ts b/packages/backend-core/src/db/couch/index.ts new file mode 100644 index 0000000000..3730a1e248 --- /dev/null +++ b/packages/backend-core/src/db/couch/index.ts @@ -0,0 +1,4 @@ +export * from "./connections" +export * from "./pouchLike" +export * from "./utils" +export { init, getPouch, getPouchDB } from "./pouchDB" diff --git a/packages/backend-core/src/db/pouch.ts b/packages/backend-core/src/db/couch/pouchDB.ts similarity index 52% rename from packages/backend-core/src/db/pouch.ts rename to packages/backend-core/src/db/couch/pouchDB.ts index fdb051060a..eead0e4d6f 100644 --- a/packages/backend-core/src/db/pouch.ts +++ b/packages/backend-core/src/db/couch/pouchDB.ts @@ -1,50 +1,10 @@ import PouchDB from "pouchdb" -import env from "../environment" -import { getCouchInfo } from "../couch" -export { getCouchInfo } from "../couch" +import env from "../../environment" +import { PouchOptions } from "@budibase/types" +import { getCouchInfo } from "./connections" -export const getUrlInfo = (url = env.COUCH_DB_URL) => { - let cleanUrl, username, password, host - if (url) { - // Ensure the URL starts with a protocol - const protoRegex = /^https?:\/\//i - if (!protoRegex.test(url)) { - url = `http://${url}` - } - - // Split into protocol and remainder - const split = url.split("://") - const protocol = split[0] - const rest = split.slice(1).join("://") - - // Extract auth if specified - if (url.includes("@")) { - // Split into host and remainder - let parts = rest.split("@") - host = parts[parts.length - 1] - let auth = parts.slice(0, -1).join("@") - - // Split auth into username and password - if (auth.includes(":")) { - const authParts = auth.split(":") - username = authParts[0] - password = authParts.slice(1).join(":") - } else { - username = auth - } - } else { - host = rest - } - cleanUrl = `${protocol}://${host}` - } - return { - url: cleanUrl, - auth: { - username, - password, - }, - } -} +let Pouch: any +let initialised = false /** * Return a constructor for PouchDB. @@ -92,3 +52,33 @@ export const getPouch = (opts: any = {}) => { return PouchDB.defaults(POUCH_DB_DEFAULTS) } + +export async function init(opts?: PouchOptions) { + Pouch = getPouch(opts) + initialised = true +} + +const checkInitialised = () => { + if (!initialised) { + throw new Error("init has not been called") + } +} + +export function getPouchDB(dbName: string, opts?: any): PouchDB.Database { + checkInitialised() + return new Pouch(dbName, opts) +} + +// use this function if you have called getPouchDB - close +// the databases you've opened once finished +export async function closePouchDB(db: PouchDB.Database) { + if (!db || env.isTest()) { + return + } + try { + // specifically await so that if there is an error, it can be ignored + return await db.close() + } catch (err) { + // ignore error, already closed + } +} diff --git a/packages/backend-core/src/couch/pouchLike.ts b/packages/backend-core/src/db/couch/pouchLike.ts similarity index 99% rename from packages/backend-core/src/couch/pouchLike.ts rename to packages/backend-core/src/db/couch/pouchLike.ts index fd5a94e1af..975bbd96c4 100644 --- a/packages/backend-core/src/couch/pouchLike.ts +++ b/packages/backend-core/src/db/couch/pouchLike.ts @@ -1,6 +1,6 @@ import Nano from "nano" import { AllDocsResponse, AnyDocument } from "@budibase/types" -import { getCouchInfo } from "./couch" +import { getCouchInfo } from "./connections" import { directCouchCall } from "./utils" import { getPouchDB } from "./pouchDB" diff --git a/packages/backend-core/src/couch/utils.ts b/packages/backend-core/src/db/couch/utils.ts similarity index 89% rename from packages/backend-core/src/couch/utils.ts rename to packages/backend-core/src/db/couch/utils.ts index 25a4643d54..426bf92158 100644 --- a/packages/backend-core/src/couch/utils.ts +++ b/packages/backend-core/src/db/couch/utils.ts @@ -1,6 +1,6 @@ -import { getCouchInfo } from "./couch" +import { getCouchInfo } from "./connections" import fetch from "node-fetch" -import { checkSlashesInUrl } from "../helpers" +import { checkSlashesInUrl } from "../../helpers" export async function directCouchCall( path: string, diff --git a/packages/backend-core/src/db/index.ts b/packages/backend-core/src/db/index.ts index e732b3a2fb..7cc49eae40 100644 --- a/packages/backend-core/src/db/index.ts +++ b/packages/backend-core/src/db/index.ts @@ -1,8 +1,7 @@ import env from "../environment" import { CouchFindOptions } from "@budibase/types" -import { PouchLike } from "../couch" -import { directCouchQuery } from "../couch" -export { init, PouchLike } from "../couch" +import { directCouchQuery, PouchLike } from "./couch" +export { init, PouchLike, getPouch, getPouchDB } from "./couch" let initialised = false const dbList = new Set() diff --git a/packages/backend-core/src/db/utils.ts b/packages/backend-core/src/db/utils.ts index a59b2ffe05..9a4ed98418 100644 --- a/packages/backend-core/src/db/utils.ts +++ b/packages/backend-core/src/db/utils.ts @@ -15,7 +15,7 @@ import { getAppMetadata } from "../cache/appMetadata" import { isDevApp, isDevAppID, getProdAppID } from "./conversions" import { APP_PREFIX } from "./constants" import * as events from "../events" -import { PouchLike } from "../couch" +import { PouchLike } from "./couch" export * from "./constants" export * from "./conversions" diff --git a/packages/backend-core/src/db/views.ts b/packages/backend-core/src/db/views.ts index 8ba508050f..23771d3f2d 100644 --- a/packages/backend-core/src/db/views.ts +++ b/packages/backend-core/src/db/views.ts @@ -1,6 +1,6 @@ import { DocumentType, ViewName, DeprecatedViews, SEPARATOR } from "./utils" import { getGlobalDB } from "../context" -import { PouchLike, QueryOpts } from "../couch" +import { PouchLike, QueryOpts } from "./couch" import { StaticDatabases } from "./constants" import { doWithDB } from "./" @@ -133,7 +133,9 @@ export const queryView = async ( try { let response = await db.query(`database/${viewName}`, params) const rows = response.rows - const docs = rows.map(row => (params.include_docs ? row.doc : row.value)) + const docs = rows.map((row: any) => + params.include_docs ? row.doc : row.value + ) // if arrayResponse has been requested, always return array regardless of length if (opts?.arrayResponse) { @@ -186,5 +188,5 @@ export const queryGlobalView = async ( db = getGlobalDB() } const createFn = CreateFuncByName[viewName] - return queryView(viewName, params, db, createFn, opts) + return queryView(viewName, params, db!, createFn, opts) } diff --git a/packages/backend-core/src/pkg/db.ts b/packages/backend-core/src/pkg/db.ts index 0254adddd5..d2bc474786 100644 --- a/packages/backend-core/src/pkg/db.ts +++ b/packages/backend-core/src/pkg/db.ts @@ -3,5 +3,4 @@ export * from "../db" export * from "../db/utils" export * from "../db/views" -export * from "../db/pouch" export * from "../db/constants" diff --git a/packages/backend-core/src/types.ts b/packages/backend-core/src/types.ts index 5eb3109b27..eeb784b0c1 100644 --- a/packages/backend-core/src/types.ts +++ b/packages/backend-core/src/types.ts @@ -1 +1 @@ -export { PouchLike } from "./couch" +export { PouchLike } from "./db" diff --git a/packages/server/package.json b/packages/server/package.json index 2c4394fc64..6d75877c79 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -42,7 +42,8 @@ "moduleNameMapper": { "@budibase/backend-core/(.*)": "/../backend-core/$1", "@budibase/backend-core": "/../backend-core/src", - "@budibase/types": "/../types/src" + "@budibase/types": "/../types/src", + "axios": "axios/dist/node/axios.cjs" }, "setupFiles": [ "./scripts/jestSetup.js" diff --git a/packages/worker/package.json b/packages/worker/package.json index c3b90811ac..844448b569 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -98,7 +98,8 @@ "moduleNameMapper": { "@budibase/backend-core/(.*)": "/../backend-core/$1", "@budibase/backend-core": "/../backend-core/src", - "@budibase/types": "/../types/src" + "@budibase/types": "/../types/src", + "axios": "axios/dist/node/axios.cjs" }, "setupFiles": [ "./scripts/jestSetup.js" From 08766b3c228493b1c68d0a5ad4a0e1b565c546f6 Mon Sep 17 00:00:00 2001 From: Budibase Release Bot <> Date: Fri, 11 Nov 2022 12:51:08 +0000 Subject: [PATCH 25/91] v2.1.25 --- lerna.json | 2 +- packages/backend-core/package.json | 4 ++-- packages/bbui/package.json | 4 ++-- packages/builder/package.json | 10 +++++----- packages/cli/package.json | 8 ++++---- packages/client/package.json | 8 ++++---- packages/frontend-core/package.json | 4 ++-- packages/sdk/package.json | 2 +- packages/server/package.json | 10 +++++----- packages/string-templates/package.json | 2 +- packages/types/package.json | 2 +- packages/worker/package.json | 8 ++++---- 12 files changed, 32 insertions(+), 32 deletions(-) diff --git a/lerna.json b/lerna.json index becfc10031..d325a57f37 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.1.24", + "version": "2.1.25", "npmClient": "yarn", "packages": [ "packages/*" diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json index 40e0ff00f7..459219aca3 100644 --- a/packages/backend-core/package.json +++ b/packages/backend-core/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/backend-core", - "version": "2.1.24", + "version": "2.1.25", "description": "Budibase backend core libraries used in server and worker", "main": "dist/src/index.js", "types": "dist/src/index.d.ts", @@ -20,7 +20,7 @@ "test:watch": "jest --watchAll" }, "dependencies": { - "@budibase/types": "^2.1.24", + "@budibase/types": "^2.1.25", "@shopify/jest-koa-mocks": "5.0.1", "@techpass/passport-openidconnect": "0.3.2", "aws-sdk": "2.1030.0", diff --git a/packages/bbui/package.json b/packages/bbui/package.json index f1f81105e0..a2e6f7da7e 100644 --- a/packages/bbui/package.json +++ b/packages/bbui/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/bbui", "description": "A UI solution used in the different Budibase projects.", - "version": "2.1.24", + "version": "2.1.25", "license": "MPL-2.0", "svelte": "src/index.js", "module": "dist/bbui.es.js", @@ -38,7 +38,7 @@ ], "dependencies": { "@adobe/spectrum-css-workflow-icons": "^1.2.1", - "@budibase/string-templates": "^2.1.24", + "@budibase/string-templates": "^2.1.25", "@spectrum-css/actionbutton": "^1.0.1", "@spectrum-css/actiongroup": "^1.0.1", "@spectrum-css/avatar": "^3.0.2", diff --git a/packages/builder/package.json b/packages/builder/package.json index db3b6cc6bf..4eb7bbc4f0 100644 --- a/packages/builder/package.json +++ b/packages/builder/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/builder", - "version": "2.1.24", + "version": "2.1.25", "license": "GPL-3.0", "private": true, "scripts": { @@ -71,10 +71,10 @@ } }, "dependencies": { - "@budibase/bbui": "^2.1.24", - "@budibase/client": "^2.1.24", - "@budibase/frontend-core": "^2.1.24", - "@budibase/string-templates": "^2.1.24", + "@budibase/bbui": "^2.1.25", + "@budibase/client": "^2.1.25", + "@budibase/frontend-core": "^2.1.25", + "@budibase/string-templates": "^2.1.25", "@sentry/browser": "5.19.1", "@spectrum-css/page": "^3.0.1", "@spectrum-css/vars": "^3.0.1", diff --git a/packages/cli/package.json b/packages/cli/package.json index 4aadb050b3..0cb29dd5fc 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/cli", - "version": "2.1.24", + "version": "2.1.25", "description": "Budibase CLI, for developers, self hosting and migrations.", "main": "src/index.js", "bin": { @@ -26,9 +26,9 @@ "outputPath": "build" }, "dependencies": { - "@budibase/backend-core": "^2.1.24", - "@budibase/string-templates": "^2.1.24", - "@budibase/types": "^2.1.24", + "@budibase/backend-core": "^2.1.25", + "@budibase/string-templates": "^2.1.25", + "@budibase/types": "^2.1.25", "axios": "0.21.2", "chalk": "4.1.0", "cli-progress": "3.11.2", diff --git a/packages/client/package.json b/packages/client/package.json index 97c25473c1..3b4a5df0e7 100644 --- a/packages/client/package.json +++ b/packages/client/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/client", - "version": "2.1.24", + "version": "2.1.25", "license": "MPL-2.0", "module": "dist/budibase-client.js", "main": "dist/budibase-client.js", @@ -19,9 +19,9 @@ "dev:builder": "rollup -cw" }, "dependencies": { - "@budibase/bbui": "^2.1.24", - "@budibase/frontend-core": "^2.1.24", - "@budibase/string-templates": "^2.1.24", + "@budibase/bbui": "^2.1.25", + "@budibase/frontend-core": "^2.1.25", + "@budibase/string-templates": "^2.1.25", "@spectrum-css/button": "^3.0.3", "@spectrum-css/card": "^3.0.3", "@spectrum-css/divider": "^1.0.3", diff --git a/packages/frontend-core/package.json b/packages/frontend-core/package.json index 9723355dd4..01af86d099 100644 --- a/packages/frontend-core/package.json +++ b/packages/frontend-core/package.json @@ -1,12 +1,12 @@ { "name": "@budibase/frontend-core", - "version": "2.1.24", + "version": "2.1.25", "description": "Budibase frontend core libraries used in builder and client", "author": "Budibase", "license": "MPL-2.0", "svelte": "src/index.js", "dependencies": { - "@budibase/bbui": "^2.1.24", + "@budibase/bbui": "^2.1.25", "lodash": "^4.17.21", "svelte": "^3.46.2" } diff --git a/packages/sdk/package.json b/packages/sdk/package.json index 97d401aee9..7410038dcd 100644 --- a/packages/sdk/package.json +++ b/packages/sdk/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/sdk", - "version": "2.1.24", + "version": "2.1.25", "description": "Budibase Public API SDK", "author": "Budibase", "license": "MPL-2.0", diff --git a/packages/server/package.json b/packages/server/package.json index 5ed82b10a1..d5efdf2f0a 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/server", "email": "hi@budibase.com", - "version": "2.1.24", + "version": "2.1.25", "description": "Budibase Web Server", "main": "src/index.ts", "repository": { @@ -77,11 +77,11 @@ "license": "GPL-3.0", "dependencies": { "@apidevtools/swagger-parser": "10.0.3", - "@budibase/backend-core": "^2.1.24", - "@budibase/client": "^2.1.24", + "@budibase/backend-core": "^2.1.25", + "@budibase/client": "^2.1.25", "@budibase/pro": "2.1.24", - "@budibase/string-templates": "^2.1.24", - "@budibase/types": "^2.1.24", + "@budibase/string-templates": "^2.1.25", + "@budibase/types": "^2.1.25", "@bull-board/api": "3.7.0", "@bull-board/koa": "3.9.4", "@elastic/elasticsearch": "7.10.0", diff --git a/packages/string-templates/package.json b/packages/string-templates/package.json index 36a8bd95f1..13a194c596 100644 --- a/packages/string-templates/package.json +++ b/packages/string-templates/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/string-templates", - "version": "2.1.24", + "version": "2.1.25", "description": "Handlebars wrapper for Budibase templating.", "main": "src/index.cjs", "module": "dist/bundle.mjs", diff --git a/packages/types/package.json b/packages/types/package.json index 9334dd6074..8f2baab743 100644 --- a/packages/types/package.json +++ b/packages/types/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/types", - "version": "2.1.24", + "version": "2.1.25", "description": "Budibase types", "main": "dist/index.js", "types": "dist/index.d.ts", diff --git a/packages/worker/package.json b/packages/worker/package.json index c0b3f99171..32f3c59ece 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/worker", "email": "hi@budibase.com", - "version": "2.1.24", + "version": "2.1.25", "description": "Budibase background service", "main": "src/index.ts", "repository": { @@ -36,10 +36,10 @@ "author": "Budibase", "license": "GPL-3.0", "dependencies": { - "@budibase/backend-core": "^2.1.24", + "@budibase/backend-core": "^2.1.25", "@budibase/pro": "2.1.24", - "@budibase/string-templates": "^2.1.24", - "@budibase/types": "^2.1.24", + "@budibase/string-templates": "^2.1.25", + "@budibase/types": "^2.1.25", "@koa/router": "8.0.8", "@sentry/node": "6.17.7", "@techpass/passport-openidconnect": "0.3.2", From 0cf55b933e5724a97c9c6a1c3f9618f086bc7c78 Mon Sep 17 00:00:00 2001 From: Budibase Release Bot <> Date: Fri, 11 Nov 2022 12:54:20 +0000 Subject: [PATCH 26/91] Update pro version to 2.1.25 --- packages/server/package.json | 2 +- packages/server/yarn.lock | 30 +++++++++++++++--------------- packages/worker/package.json | 2 +- packages/worker/yarn.lock | 30 +++++++++++++++--------------- 4 files changed, 32 insertions(+), 32 deletions(-) diff --git a/packages/server/package.json b/packages/server/package.json index d5efdf2f0a..fab93cda87 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -79,7 +79,7 @@ "@apidevtools/swagger-parser": "10.0.3", "@budibase/backend-core": "^2.1.25", "@budibase/client": "^2.1.25", - "@budibase/pro": "2.1.24", + "@budibase/pro": "2.1.25", "@budibase/string-templates": "^2.1.25", "@budibase/types": "^2.1.25", "@bull-board/api": "3.7.0", diff --git a/packages/server/yarn.lock b/packages/server/yarn.lock index 652056a0a8..1bf160010b 100644 --- a/packages/server/yarn.lock +++ b/packages/server/yarn.lock @@ -1094,12 +1094,12 @@ resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw== -"@budibase/backend-core@2.1.24": - version "2.1.24" - resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-2.1.24.tgz#4c0172359150cac624788c725cd6842a9a1b42e7" - integrity sha512-QjQ5bRNRWuiGB6J5MGaR1Uv6s6+jcCgrar4e2NCJn1Z+Ta7ZaBPI+6lH5d1vMLYjIUW9b/CavLtxpGazkfEPKQ== +"@budibase/backend-core@2.1.25": + version "2.1.25" + resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-2.1.25.tgz#2cb8291f7a08ff8b8988f8c5ced232195e6f4986" + integrity sha512-xoVwfNbt+1PAJGxRqZFtJFwTv+vVNe4mJcLSUz/lAVOcwY6pKE1L5EjsA1z7czEHuDxoQs3S9Ls1QiQQwZh9gw== dependencies: - "@budibase/types" "^2.1.24" + "@budibase/types" "^2.1.25" "@shopify/jest-koa-mocks" "5.0.1" "@techpass/passport-openidconnect" "0.3.2" aws-sdk "2.1030.0" @@ -1181,13 +1181,13 @@ svelte-flatpickr "^3.2.3" svelte-portal "^1.0.0" -"@budibase/pro@2.1.24": - version "2.1.24" - resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.1.24.tgz#bec9461bde8f57f9b7648b82e381b763fc0cff30" - integrity sha512-FttB/wZVljPp8nKEnOK7B0gOa4ZJLkfq2jEA/7GyoI9kvht/QJx1sOqVaafHVh24cev6NsfCsMtxCiaiM42cOw== +"@budibase/pro@2.1.25": + version "2.1.25" + resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.1.25.tgz#d5a5a65b8e447ee96988e8d96e7caad665d75ed3" + integrity sha512-aqFI6WaSwEhMrL9fMr0CPF+DHtukI0pQHArW1xZKD5+7TIR6MA82gTIpEn/YLVLiUIc4hjfsgpSHTSxZ495T3w== dependencies: - "@budibase/backend-core" "2.1.24" - "@budibase/types" "2.1.24" + "@budibase/backend-core" "2.1.25" + "@budibase/types" "2.1.25" "@koa/router" "8.0.8" bull "4.10.1" joi "17.6.0" @@ -1211,10 +1211,10 @@ svelte-apexcharts "^1.0.2" svelte-flatpickr "^3.1.0" -"@budibase/types@2.1.24", "@budibase/types@^2.1.24": - version "2.1.24" - resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.1.24.tgz#bb502e331eeecb3cf174e5222e174d45d952fb9d" - integrity sha512-Vbn70peWwRCGhgKSpfEz6hCHjF3tzyfMzToXHCagDKh96/qhHaH5pAX5ioIQ2lkWU97ymnS2bKL5fmq9e7xdwg== +"@budibase/types@2.1.25", "@budibase/types@^2.1.25": + version "2.1.25" + resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.1.25.tgz#17166655921e977b128025f6e10971f081eeb1c1" + integrity sha512-kWs3PKbCo9jHejzA9gaNmjNBwVHrR2wzhBNZrfILUXj8Nphgf8GRl2sj3u6xdO3jg7NcO/OawAQdDXpYOdR8PQ== "@bull-board/api@3.7.0": version "3.7.0" diff --git a/packages/worker/package.json b/packages/worker/package.json index 32f3c59ece..bf32dc3eb3 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -37,7 +37,7 @@ "license": "GPL-3.0", "dependencies": { "@budibase/backend-core": "^2.1.25", - "@budibase/pro": "2.1.24", + "@budibase/pro": "2.1.25", "@budibase/string-templates": "^2.1.25", "@budibase/types": "^2.1.25", "@koa/router": "8.0.8", diff --git a/packages/worker/yarn.lock b/packages/worker/yarn.lock index 301332f393..427d449251 100644 --- a/packages/worker/yarn.lock +++ b/packages/worker/yarn.lock @@ -291,12 +291,12 @@ resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw== -"@budibase/backend-core@2.1.24": - version "2.1.24" - resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-2.1.24.tgz#4c0172359150cac624788c725cd6842a9a1b42e7" - integrity sha512-QjQ5bRNRWuiGB6J5MGaR1Uv6s6+jcCgrar4e2NCJn1Z+Ta7ZaBPI+6lH5d1vMLYjIUW9b/CavLtxpGazkfEPKQ== +"@budibase/backend-core@2.1.25": + version "2.1.25" + resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-2.1.25.tgz#2cb8291f7a08ff8b8988f8c5ced232195e6f4986" + integrity sha512-xoVwfNbt+1PAJGxRqZFtJFwTv+vVNe4mJcLSUz/lAVOcwY6pKE1L5EjsA1z7czEHuDxoQs3S9Ls1QiQQwZh9gw== dependencies: - "@budibase/types" "^2.1.24" + "@budibase/types" "^2.1.25" "@shopify/jest-koa-mocks" "5.0.1" "@techpass/passport-openidconnect" "0.3.2" aws-sdk "2.1030.0" @@ -328,22 +328,22 @@ uuid "8.3.2" zlib "1.0.5" -"@budibase/pro@2.1.24": - version "2.1.24" - resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.1.24.tgz#bec9461bde8f57f9b7648b82e381b763fc0cff30" - integrity sha512-FttB/wZVljPp8nKEnOK7B0gOa4ZJLkfq2jEA/7GyoI9kvht/QJx1sOqVaafHVh24cev6NsfCsMtxCiaiM42cOw== +"@budibase/pro@2.1.25": + version "2.1.25" + resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.1.25.tgz#d5a5a65b8e447ee96988e8d96e7caad665d75ed3" + integrity sha512-aqFI6WaSwEhMrL9fMr0CPF+DHtukI0pQHArW1xZKD5+7TIR6MA82gTIpEn/YLVLiUIc4hjfsgpSHTSxZ495T3w== dependencies: - "@budibase/backend-core" "2.1.24" - "@budibase/types" "2.1.24" + "@budibase/backend-core" "2.1.25" + "@budibase/types" "2.1.25" "@koa/router" "8.0.8" bull "4.10.1" joi "17.6.0" node-fetch "^2.6.1" -"@budibase/types@2.1.24", "@budibase/types@^2.1.24": - version "2.1.24" - resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.1.24.tgz#bb502e331eeecb3cf174e5222e174d45d952fb9d" - integrity sha512-Vbn70peWwRCGhgKSpfEz6hCHjF3tzyfMzToXHCagDKh96/qhHaH5pAX5ioIQ2lkWU97ymnS2bKL5fmq9e7xdwg== +"@budibase/types@2.1.25", "@budibase/types@^2.1.25": + version "2.1.25" + resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.1.25.tgz#17166655921e977b128025f6e10971f081eeb1c1" + integrity sha512-kWs3PKbCo9jHejzA9gaNmjNBwVHrR2wzhBNZrfILUXj8Nphgf8GRl2sj3u6xdO3jg7NcO/OawAQdDXpYOdR8PQ== "@cspotcode/source-map-consumer@0.8.0": version "0.8.0" From c71f9ccecf71bda4d9d06dfbc4589c1c08837242 Mon Sep 17 00:00:00 2001 From: Jonny McCullagh Date: Sun, 13 Nov 2022 17:53:32 +0000 Subject: [PATCH 27/91] single image NFS & more logging --- hosting/single/runner.sh | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/hosting/single/runner.sh b/hosting/single/runner.sh index a95c21a98f..ea825131db 100644 --- a/hosting/single/runner.sh +++ b/hosting/single/runner.sh @@ -27,12 +27,14 @@ if [[ "${TARGETBUILD}" = "aas" ]]; then else DATA_DIR=${DATA_DIR:-/data} fi - +mkdir -p ${DATA_DIR} # Mount NFS or GCP Filestore if env vars exist for it -if [[ -z ${FILESHARE_IP} && -z ${FILESHARE_NAME} ]]; then +if [[ ! -z ${FILESHARE_IP} && ! -z ${FILESHARE_NAME} ]]; then + echo "Mounting NFS share" + apt update && apt install -y nfs-common nfs-kernel-server echo "Mount file share ${FILESHARE_IP}:/${FILESHARE_NAME} to ${DATA_DIR}" mount -o nolock ${FILESHARE_IP}:/${FILESHARE_NAME} ${DATA_DIR} - echo "Mounting completed." + echo "Mounting result: $?" fi if [ -f "${DATA_DIR}/.env" ]; then @@ -74,9 +76,9 @@ mkdir -p ${DATA_DIR}/couch/{dbs,views} mkdir -p ${DATA_DIR}/minio mkdir -p ${DATA_DIR}/search chown -R couchdb:couchdb ${DATA_DIR}/couch -redis-server --requirepass $REDIS_PASSWORD & -/opt/clouseau/bin/clouseau & -/minio/minio server ${DATA_DIR}/minio & +redis-server --requirepass $REDIS_PASSWORD > /dev/stdout 2>&1 & +/opt/clouseau/bin/clouseau > /dev/stdout 2>&1 & +/minio/minio server ${DATA_DIR}/minio > /dev/stdout 2>&1 & /docker-entrypoint.sh /opt/couchdb/bin/couchdb & /etc/init.d/nginx restart if [[ ! -z "${CUSTOM_DOMAIN}" ]]; then @@ -85,16 +87,18 @@ if [[ ! -z "${CUSTOM_DOMAIN}" ]]; then chmod +x /etc/cron.d/certificate-renew # Request the certbot certificate /app/letsencrypt/certificate-request.sh ${CUSTOM_DOMAIN} + /etc/init.d/nginx restart fi -/etc/init.d/nginx restart pushd app -pm2 start --name app "yarn run:docker" +pm2 start -l /dev/stdout --name app "yarn run:docker" popd pushd worker -pm2 start --name worker "yarn run:docker" +pm2 start -l /dev/stdout --name worker "yarn run:docker" popd sleep 10 +echo "curl to couchdb endpoints" curl -X PUT ${COUCH_DB_URL}/_users curl -X PUT ${COUCH_DB_URL}/_replicator +echo "end of runner.sh, sleeping ..." sleep infinity From 8f058e222b4085b51835bf4a99c809ee3fa64c46 Mon Sep 17 00:00:00 2001 From: Dean Date: Sun, 13 Nov 2022 18:08:48 +0000 Subject: [PATCH 28/91] Fix for attachment cleanup on internal row delete. --- packages/server/src/api/controllers/table/internal.ts | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/packages/server/src/api/controllers/table/internal.ts b/packages/server/src/api/controllers/table/internal.ts index a50009b1f6..d569fd4547 100644 --- a/packages/server/src/api/controllers/table/internal.ts +++ b/packages/server/src/api/controllers/table/internal.ts @@ -142,15 +142,15 @@ export async function destroy(ctx: any) { const tableToDelete = await db.get(ctx.params.tableId) // Delete all rows for that table - const rows = await db.allDocs( + const rowsData = await db.allDocs( getRowParams(ctx.params.tableId, null, { include_docs: true, }) ) await db.bulkDocs( - rows.rows.map((row: any) => ({ ...row.doc, _deleted: true })) + rowsData.rows.map((row: any) => ({ ...row.doc, _deleted: true })) ) - await quotas.removeRows(rows.rows.length, { + await quotas.removeRows(rowsData.rows.length, { tableId: ctx.params.tableId, }) @@ -179,7 +179,9 @@ export async function destroy(ctx: any) { oldTable: null, deletion: true, }) - await cleanupAttachments(tableToDelete, { rows }) + await cleanupAttachments(tableToDelete, { + rows: rowsData.rows.map((row: any) => row.doc), + }) return tableToDelete } From 0619fcd281de21164e9a02d09d9a9ae04e51e6d3 Mon Sep 17 00:00:00 2001 From: Dean Date: Sun, 13 Nov 2022 18:16:04 +0000 Subject: [PATCH 29/91] Added multiselect/array options support to the view contains filter --- .../src/components/backend/DataTable/modals/FilterModal.svelte | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/builder/src/components/backend/DataTable/modals/FilterModal.svelte b/packages/builder/src/components/backend/DataTable/modals/FilterModal.svelte index a9eff7f957..b125e18b31 100644 --- a/packages/builder/src/components/backend/DataTable/modals/FilterModal.svelte +++ b/packages/builder/src/components/backend/DataTable/modals/FilterModal.svelte @@ -97,7 +97,7 @@ } function fieldOptions(field) { - return schema[field]?.type === "options" + return schema[field]?.type === "options" || schema[field]?.type === "array" ? schema[field]?.constraints.inclusion : [true, false] } From 2eeac325f31abe2a479230454da80b6566286c35 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Mon, 14 Nov 2022 09:25:45 +0000 Subject: [PATCH 30/91] Adjust styles so placeholder component is left aligned --- packages/client/src/components/app/blocks/form/FormBlock.svelte | 2 +- .../client/src/components/app/blocks/form/InnerFormBlock.svelte | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/client/src/components/app/blocks/form/FormBlock.svelte b/packages/client/src/components/app/blocks/form/FormBlock.svelte index 9ec587519f..8964475673 100644 --- a/packages/client/src/components/app/blocks/form/FormBlock.svelte +++ b/packages/client/src/components/app/blocks/form/FormBlock.svelte @@ -65,7 +65,7 @@ type="container" props={{ direction: "column", - hAlign: "center", + hAlign: "left", vAlign: "stretch", }} > diff --git a/packages/client/src/components/app/blocks/form/InnerFormBlock.svelte b/packages/client/src/components/app/blocks/form/InnerFormBlock.svelte index 948584120b..c4e9b0941c 100644 --- a/packages/client/src/components/app/blocks/form/InnerFormBlock.svelte +++ b/packages/client/src/components/app/blocks/form/InnerFormBlock.svelte @@ -97,6 +97,8 @@ styles={{ normal: { width: "600px", + "margin-left": "auto", + "margin-right": "auto", }, }} context="form" From 4c366114f02b205d39b59af458a944a5bb3c1e6d Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Mon, 14 Nov 2022 09:33:01 +0000 Subject: [PATCH 31/91] Fix date time field inconsistency with showing time --- packages/client/src/components/app/forms/DateTimeField.svelte | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/client/src/components/app/forms/DateTimeField.svelte b/packages/client/src/components/app/forms/DateTimeField.svelte index 19ff49c9ba..6bcd20d250 100644 --- a/packages/client/src/components/app/forms/DateTimeField.svelte +++ b/packages/client/src/components/app/forms/DateTimeField.svelte @@ -6,7 +6,7 @@ export let label export let placeholder export let disabled = false - export let enableTime = false + export let enableTime = true export let timeOnly = false export let time24hr = false export let ignoreTimezones = false From 80e223465054a978cd5edcaded9af981d8e99850 Mon Sep 17 00:00:00 2001 From: Budibase Release Bot <> Date: Mon, 14 Nov 2022 14:32:46 +0000 Subject: [PATCH 32/91] v2.1.26 --- lerna.json | 2 +- packages/backend-core/package.json | 4 ++-- packages/bbui/package.json | 4 ++-- packages/builder/package.json | 10 +++++----- packages/cli/package.json | 8 ++++---- packages/client/package.json | 8 ++++---- packages/frontend-core/package.json | 4 ++-- packages/sdk/package.json | 2 +- packages/server/package.json | 10 +++++----- packages/string-templates/package.json | 2 +- packages/types/package.json | 2 +- packages/worker/package.json | 8 ++++---- 12 files changed, 32 insertions(+), 32 deletions(-) diff --git a/lerna.json b/lerna.json index d325a57f37..f53cf65bde 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.1.25", + "version": "2.1.26", "npmClient": "yarn", "packages": [ "packages/*" diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json index 459219aca3..d7d55a5e1e 100644 --- a/packages/backend-core/package.json +++ b/packages/backend-core/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/backend-core", - "version": "2.1.25", + "version": "2.1.26", "description": "Budibase backend core libraries used in server and worker", "main": "dist/src/index.js", "types": "dist/src/index.d.ts", @@ -20,7 +20,7 @@ "test:watch": "jest --watchAll" }, "dependencies": { - "@budibase/types": "^2.1.25", + "@budibase/types": "^2.1.26", "@shopify/jest-koa-mocks": "5.0.1", "@techpass/passport-openidconnect": "0.3.2", "aws-sdk": "2.1030.0", diff --git a/packages/bbui/package.json b/packages/bbui/package.json index a2e6f7da7e..9d9b4ce295 100644 --- a/packages/bbui/package.json +++ b/packages/bbui/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/bbui", "description": "A UI solution used in the different Budibase projects.", - "version": "2.1.25", + "version": "2.1.26", "license": "MPL-2.0", "svelte": "src/index.js", "module": "dist/bbui.es.js", @@ -38,7 +38,7 @@ ], "dependencies": { "@adobe/spectrum-css-workflow-icons": "^1.2.1", - "@budibase/string-templates": "^2.1.25", + "@budibase/string-templates": "^2.1.26", "@spectrum-css/actionbutton": "^1.0.1", "@spectrum-css/actiongroup": "^1.0.1", "@spectrum-css/avatar": "^3.0.2", diff --git a/packages/builder/package.json b/packages/builder/package.json index 4eb7bbc4f0..102dd4ccf8 100644 --- a/packages/builder/package.json +++ b/packages/builder/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/builder", - "version": "2.1.25", + "version": "2.1.26", "license": "GPL-3.0", "private": true, "scripts": { @@ -71,10 +71,10 @@ } }, "dependencies": { - "@budibase/bbui": "^2.1.25", - "@budibase/client": "^2.1.25", - "@budibase/frontend-core": "^2.1.25", - "@budibase/string-templates": "^2.1.25", + "@budibase/bbui": "^2.1.26", + "@budibase/client": "^2.1.26", + "@budibase/frontend-core": "^2.1.26", + "@budibase/string-templates": "^2.1.26", "@sentry/browser": "5.19.1", "@spectrum-css/page": "^3.0.1", "@spectrum-css/vars": "^3.0.1", diff --git a/packages/cli/package.json b/packages/cli/package.json index 0cb29dd5fc..cc0e4bf9d1 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/cli", - "version": "2.1.25", + "version": "2.1.26", "description": "Budibase CLI, for developers, self hosting and migrations.", "main": "src/index.js", "bin": { @@ -26,9 +26,9 @@ "outputPath": "build" }, "dependencies": { - "@budibase/backend-core": "^2.1.25", - "@budibase/string-templates": "^2.1.25", - "@budibase/types": "^2.1.25", + "@budibase/backend-core": "^2.1.26", + "@budibase/string-templates": "^2.1.26", + "@budibase/types": "^2.1.26", "axios": "0.21.2", "chalk": "4.1.0", "cli-progress": "3.11.2", diff --git a/packages/client/package.json b/packages/client/package.json index 3b4a5df0e7..b41679ad6e 100644 --- a/packages/client/package.json +++ b/packages/client/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/client", - "version": "2.1.25", + "version": "2.1.26", "license": "MPL-2.0", "module": "dist/budibase-client.js", "main": "dist/budibase-client.js", @@ -19,9 +19,9 @@ "dev:builder": "rollup -cw" }, "dependencies": { - "@budibase/bbui": "^2.1.25", - "@budibase/frontend-core": "^2.1.25", - "@budibase/string-templates": "^2.1.25", + "@budibase/bbui": "^2.1.26", + "@budibase/frontend-core": "^2.1.26", + "@budibase/string-templates": "^2.1.26", "@spectrum-css/button": "^3.0.3", "@spectrum-css/card": "^3.0.3", "@spectrum-css/divider": "^1.0.3", diff --git a/packages/frontend-core/package.json b/packages/frontend-core/package.json index 01af86d099..6cafe9910f 100644 --- a/packages/frontend-core/package.json +++ b/packages/frontend-core/package.json @@ -1,12 +1,12 @@ { "name": "@budibase/frontend-core", - "version": "2.1.25", + "version": "2.1.26", "description": "Budibase frontend core libraries used in builder and client", "author": "Budibase", "license": "MPL-2.0", "svelte": "src/index.js", "dependencies": { - "@budibase/bbui": "^2.1.25", + "@budibase/bbui": "^2.1.26", "lodash": "^4.17.21", "svelte": "^3.46.2" } diff --git a/packages/sdk/package.json b/packages/sdk/package.json index 7410038dcd..917083748d 100644 --- a/packages/sdk/package.json +++ b/packages/sdk/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/sdk", - "version": "2.1.25", + "version": "2.1.26", "description": "Budibase Public API SDK", "author": "Budibase", "license": "MPL-2.0", diff --git a/packages/server/package.json b/packages/server/package.json index fab93cda87..54c290a21b 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/server", "email": "hi@budibase.com", - "version": "2.1.25", + "version": "2.1.26", "description": "Budibase Web Server", "main": "src/index.ts", "repository": { @@ -77,11 +77,11 @@ "license": "GPL-3.0", "dependencies": { "@apidevtools/swagger-parser": "10.0.3", - "@budibase/backend-core": "^2.1.25", - "@budibase/client": "^2.1.25", + "@budibase/backend-core": "^2.1.26", + "@budibase/client": "^2.1.26", "@budibase/pro": "2.1.25", - "@budibase/string-templates": "^2.1.25", - "@budibase/types": "^2.1.25", + "@budibase/string-templates": "^2.1.26", + "@budibase/types": "^2.1.26", "@bull-board/api": "3.7.0", "@bull-board/koa": "3.9.4", "@elastic/elasticsearch": "7.10.0", diff --git a/packages/string-templates/package.json b/packages/string-templates/package.json index 13a194c596..6dce3eb62c 100644 --- a/packages/string-templates/package.json +++ b/packages/string-templates/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/string-templates", - "version": "2.1.25", + "version": "2.1.26", "description": "Handlebars wrapper for Budibase templating.", "main": "src/index.cjs", "module": "dist/bundle.mjs", diff --git a/packages/types/package.json b/packages/types/package.json index 8f2baab743..e25a40857f 100644 --- a/packages/types/package.json +++ b/packages/types/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/types", - "version": "2.1.25", + "version": "2.1.26", "description": "Budibase types", "main": "dist/index.js", "types": "dist/index.d.ts", diff --git a/packages/worker/package.json b/packages/worker/package.json index bf32dc3eb3..b2ef15c456 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/worker", "email": "hi@budibase.com", - "version": "2.1.25", + "version": "2.1.26", "description": "Budibase background service", "main": "src/index.ts", "repository": { @@ -36,10 +36,10 @@ "author": "Budibase", "license": "GPL-3.0", "dependencies": { - "@budibase/backend-core": "^2.1.25", + "@budibase/backend-core": "^2.1.26", "@budibase/pro": "2.1.25", - "@budibase/string-templates": "^2.1.25", - "@budibase/types": "^2.1.25", + "@budibase/string-templates": "^2.1.26", + "@budibase/types": "^2.1.26", "@koa/router": "8.0.8", "@sentry/node": "6.17.7", "@techpass/passport-openidconnect": "0.3.2", From d55998940fc671dbb13c0f9d64a8caf0e7c01992 Mon Sep 17 00:00:00 2001 From: Budibase Release Bot <> Date: Mon, 14 Nov 2022 14:36:05 +0000 Subject: [PATCH 33/91] Update pro version to 2.1.26 --- packages/server/package.json | 2 +- packages/server/yarn.lock | 30 +++++++++++++++--------------- packages/worker/package.json | 2 +- packages/worker/yarn.lock | 30 +++++++++++++++--------------- 4 files changed, 32 insertions(+), 32 deletions(-) diff --git a/packages/server/package.json b/packages/server/package.json index 54c290a21b..baedd1ad92 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -79,7 +79,7 @@ "@apidevtools/swagger-parser": "10.0.3", "@budibase/backend-core": "^2.1.26", "@budibase/client": "^2.1.26", - "@budibase/pro": "2.1.25", + "@budibase/pro": "2.1.26", "@budibase/string-templates": "^2.1.26", "@budibase/types": "^2.1.26", "@bull-board/api": "3.7.0", diff --git a/packages/server/yarn.lock b/packages/server/yarn.lock index 1bf160010b..7ad7e7092f 100644 --- a/packages/server/yarn.lock +++ b/packages/server/yarn.lock @@ -1094,12 +1094,12 @@ resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw== -"@budibase/backend-core@2.1.25": - version "2.1.25" - resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-2.1.25.tgz#2cb8291f7a08ff8b8988f8c5ced232195e6f4986" - integrity sha512-xoVwfNbt+1PAJGxRqZFtJFwTv+vVNe4mJcLSUz/lAVOcwY6pKE1L5EjsA1z7czEHuDxoQs3S9Ls1QiQQwZh9gw== +"@budibase/backend-core@2.1.26": + version "2.1.26" + resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-2.1.26.tgz#e663ccb83e027c46b4a0a58e701ed75d2ef79ce8" + integrity sha512-ZXzeX+ifI0I3PVjyQCPJhn0RRXZptzfluziaSDrfAZQ85pvH6lluWcbUTPd1dx9WdSCS4N0rxvPsfL9FHb/TtA== dependencies: - "@budibase/types" "^2.1.25" + "@budibase/types" "^2.1.26" "@shopify/jest-koa-mocks" "5.0.1" "@techpass/passport-openidconnect" "0.3.2" aws-sdk "2.1030.0" @@ -1181,13 +1181,13 @@ svelte-flatpickr "^3.2.3" svelte-portal "^1.0.0" -"@budibase/pro@2.1.25": - version "2.1.25" - resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.1.25.tgz#d5a5a65b8e447ee96988e8d96e7caad665d75ed3" - integrity sha512-aqFI6WaSwEhMrL9fMr0CPF+DHtukI0pQHArW1xZKD5+7TIR6MA82gTIpEn/YLVLiUIc4hjfsgpSHTSxZ495T3w== +"@budibase/pro@2.1.26": + version "2.1.26" + resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.1.26.tgz#335ca9343ae3f84c7a99224c6863ecf09573221f" + integrity sha512-1bdpQO+v//vbhQa1/+Ic1tB2PlU3hCAwgYNm4C+MhNicMc9InpOE4bMNn22s6eoOI5XcuHiSZld+Au8J9/xuRA== dependencies: - "@budibase/backend-core" "2.1.25" - "@budibase/types" "2.1.25" + "@budibase/backend-core" "2.1.26" + "@budibase/types" "2.1.26" "@koa/router" "8.0.8" bull "4.10.1" joi "17.6.0" @@ -1211,10 +1211,10 @@ svelte-apexcharts "^1.0.2" svelte-flatpickr "^3.1.0" -"@budibase/types@2.1.25", "@budibase/types@^2.1.25": - version "2.1.25" - resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.1.25.tgz#17166655921e977b128025f6e10971f081eeb1c1" - integrity sha512-kWs3PKbCo9jHejzA9gaNmjNBwVHrR2wzhBNZrfILUXj8Nphgf8GRl2sj3u6xdO3jg7NcO/OawAQdDXpYOdR8PQ== +"@budibase/types@2.1.26", "@budibase/types@^2.1.26": + version "2.1.26" + resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.1.26.tgz#15422f49fc096a7b0419b4104a86df97198ebfde" + integrity sha512-gykmR8jSLVNAvbg7/senimW0HGs/lC3W1TlfoAVvkhzS2AAiuyH3CVbEYmgdd0SnZbY2Hpaas+Eqdp4fkqFu8A== "@bull-board/api@3.7.0": version "3.7.0" diff --git a/packages/worker/package.json b/packages/worker/package.json index b2ef15c456..fbed9ddc0e 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -37,7 +37,7 @@ "license": "GPL-3.0", "dependencies": { "@budibase/backend-core": "^2.1.26", - "@budibase/pro": "2.1.25", + "@budibase/pro": "2.1.26", "@budibase/string-templates": "^2.1.26", "@budibase/types": "^2.1.26", "@koa/router": "8.0.8", diff --git a/packages/worker/yarn.lock b/packages/worker/yarn.lock index 427d449251..22609a331c 100644 --- a/packages/worker/yarn.lock +++ b/packages/worker/yarn.lock @@ -291,12 +291,12 @@ resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw== -"@budibase/backend-core@2.1.25": - version "2.1.25" - resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-2.1.25.tgz#2cb8291f7a08ff8b8988f8c5ced232195e6f4986" - integrity sha512-xoVwfNbt+1PAJGxRqZFtJFwTv+vVNe4mJcLSUz/lAVOcwY6pKE1L5EjsA1z7czEHuDxoQs3S9Ls1QiQQwZh9gw== +"@budibase/backend-core@2.1.26": + version "2.1.26" + resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-2.1.26.tgz#e663ccb83e027c46b4a0a58e701ed75d2ef79ce8" + integrity sha512-ZXzeX+ifI0I3PVjyQCPJhn0RRXZptzfluziaSDrfAZQ85pvH6lluWcbUTPd1dx9WdSCS4N0rxvPsfL9FHb/TtA== dependencies: - "@budibase/types" "^2.1.25" + "@budibase/types" "^2.1.26" "@shopify/jest-koa-mocks" "5.0.1" "@techpass/passport-openidconnect" "0.3.2" aws-sdk "2.1030.0" @@ -328,22 +328,22 @@ uuid "8.3.2" zlib "1.0.5" -"@budibase/pro@2.1.25": - version "2.1.25" - resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.1.25.tgz#d5a5a65b8e447ee96988e8d96e7caad665d75ed3" - integrity sha512-aqFI6WaSwEhMrL9fMr0CPF+DHtukI0pQHArW1xZKD5+7TIR6MA82gTIpEn/YLVLiUIc4hjfsgpSHTSxZ495T3w== +"@budibase/pro@2.1.26": + version "2.1.26" + resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.1.26.tgz#335ca9343ae3f84c7a99224c6863ecf09573221f" + integrity sha512-1bdpQO+v//vbhQa1/+Ic1tB2PlU3hCAwgYNm4C+MhNicMc9InpOE4bMNn22s6eoOI5XcuHiSZld+Au8J9/xuRA== dependencies: - "@budibase/backend-core" "2.1.25" - "@budibase/types" "2.1.25" + "@budibase/backend-core" "2.1.26" + "@budibase/types" "2.1.26" "@koa/router" "8.0.8" bull "4.10.1" joi "17.6.0" node-fetch "^2.6.1" -"@budibase/types@2.1.25", "@budibase/types@^2.1.25": - version "2.1.25" - resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.1.25.tgz#17166655921e977b128025f6e10971f081eeb1c1" - integrity sha512-kWs3PKbCo9jHejzA9gaNmjNBwVHrR2wzhBNZrfILUXj8Nphgf8GRl2sj3u6xdO3jg7NcO/OawAQdDXpYOdR8PQ== +"@budibase/types@2.1.26", "@budibase/types@^2.1.26": + version "2.1.26" + resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.1.26.tgz#15422f49fc096a7b0419b4104a86df97198ebfde" + integrity sha512-gykmR8jSLVNAvbg7/senimW0HGs/lC3W1TlfoAVvkhzS2AAiuyH3CVbEYmgdd0SnZbY2Hpaas+Eqdp4fkqFu8A== "@cspotcode/source-map-consumer@0.8.0": version "0.8.0" From 3f6cdd787ce09fd884eef39e28672b516a33b2b6 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 14 Nov 2022 18:00:20 +0000 Subject: [PATCH 34/91] Replacing CLS with local-storage, simplified usage which should remove the memory leak permenantly. --- packages/backend-core/db.js | 7 +- packages/backend-core/src/clshooked/index.js | 650 ------------------ .../src/context/FunctionContext.ts | 47 -- .../backend-core/src/context/constants.ts | 4 - packages/backend-core/src/context/index.ts | 38 +- .../backend-core/src/context/localStorage.ts | 18 + packages/backend-core/src/db/couch/pouchDB.ts | 2 +- packages/backend-core/src/db/db.ts | 50 ++ packages/backend-core/src/db/index.ts | 58 +- packages/backend-core/src/db/utils.ts | 2 +- packages/backend-core/src/index.ts | 2 +- packages/backend-core/src/pkg/db.ts | 6 - 12 files changed, 98 insertions(+), 786 deletions(-) delete mode 100644 packages/backend-core/src/clshooked/index.js delete mode 100644 packages/backend-core/src/context/FunctionContext.ts create mode 100644 packages/backend-core/src/context/localStorage.ts create mode 100644 packages/backend-core/src/db/db.ts delete mode 100644 packages/backend-core/src/pkg/db.ts diff --git a/packages/backend-core/db.js b/packages/backend-core/db.js index d2adf6c092..f7004972d5 100644 --- a/packages/backend-core/db.js +++ b/packages/backend-core/db.js @@ -1,6 +1 @@ -module.exports = { - ...require("./src/db/utils"), - ...require("./src/db/constants"), - ...require("./src/db"), - ...require("./src/db/views"), -} +module.exports = require("./src/db") diff --git a/packages/backend-core/src/clshooked/index.js b/packages/backend-core/src/clshooked/index.js deleted file mode 100644 index d69ffdd914..0000000000 --- a/packages/backend-core/src/clshooked/index.js +++ /dev/null @@ -1,650 +0,0 @@ -const util = require("util") -const assert = require("assert") -const wrapEmitter = require("emitter-listener") -const async_hooks = require("async_hooks") - -const CONTEXTS_SYMBOL = "cls@contexts" -const ERROR_SYMBOL = "error@context" - -const DEBUG_CLS_HOOKED = process.env.DEBUG_CLS_HOOKED - -let currentUid = -1 - -module.exports = { - getNamespace: getNamespace, - createNamespace: createNamespace, - destroyNamespace: destroyNamespace, - reset: reset, - ERROR_SYMBOL: ERROR_SYMBOL, -} - -function Namespace(name) { - this.name = name - // changed in 2.7: no default context - this.active = null - this._set = [] - this.id = null - this._contexts = new Map() - this._indent = 0 - this._hook = null -} - -Namespace.prototype.set = function set(key, value) { - if (!this.active) { - throw new Error( - "No context available. ns.run() or ns.bind() must be called first." - ) - } - - this.active[key] = value - - if (DEBUG_CLS_HOOKED) { - const indentStr = " ".repeat(this._indent < 0 ? 0 : this._indent) - debug2( - indentStr + - "CONTEXT-SET KEY:" + - key + - "=" + - value + - " in ns:" + - this.name + - " currentUid:" + - currentUid + - " active:" + - util.inspect(this.active, { showHidden: true, depth: 2, colors: true }) - ) - } - - return value -} - -Namespace.prototype.get = function get(key) { - if (!this.active) { - if (DEBUG_CLS_HOOKED) { - const asyncHooksCurrentId = async_hooks.currentId() - const triggerId = async_hooks.triggerAsyncId() - const indentStr = " ".repeat(this._indent < 0 ? 0 : this._indent) - debug2( - `${indentStr}CONTEXT-GETTING KEY NO ACTIVE NS: (${this.name}) ${key}=undefined currentUid:${currentUid} asyncHooksCurrentId:${asyncHooksCurrentId} triggerId:${triggerId} len:${this._set.length}` - ) - } - return undefined - } - if (DEBUG_CLS_HOOKED) { - const asyncHooksCurrentId = async_hooks.executionAsyncId() - const triggerId = async_hooks.triggerAsyncId() - const indentStr = " ".repeat(this._indent < 0 ? 0 : this._indent) - debug2( - indentStr + - "CONTEXT-GETTING KEY:" + - key + - "=" + - this.active[key] + - " (" + - this.name + - ") currentUid:" + - currentUid + - " active:" + - util.inspect(this.active, { showHidden: true, depth: 2, colors: true }) - ) - debug2( - `${indentStr}CONTEXT-GETTING KEY: (${this.name}) ${key}=${ - this.active[key] - } currentUid:${currentUid} asyncHooksCurrentId:${asyncHooksCurrentId} triggerId:${triggerId} len:${ - this._set.length - } active:${util.inspect(this.active)}` - ) - } - return this.active[key] -} - -Namespace.prototype.createContext = function createContext() { - // Prototype inherit existing context if created a new child context within existing context. - let context = Object.create(this.active ? this.active : Object.prototype) - context._ns_name = this.name - context.id = currentUid - - if (DEBUG_CLS_HOOKED) { - const asyncHooksCurrentId = async_hooks.executionAsyncId() - const triggerId = async_hooks.triggerAsyncId() - const indentStr = " ".repeat(this._indent < 0 ? 0 : this._indent) - debug2( - `${indentStr}CONTEXT-CREATED Context: (${ - this.name - }) currentUid:${currentUid} asyncHooksCurrentId:${asyncHooksCurrentId} triggerId:${triggerId} len:${ - this._set.length - } context:${util.inspect(context, { - showHidden: true, - depth: 2, - colors: true, - })}` - ) - } - - return context -} - -Namespace.prototype.run = function run(fn) { - let context = this.createContext() - this.enter(context) - - try { - if (DEBUG_CLS_HOOKED) { - const triggerId = async_hooks.triggerAsyncId() - const asyncHooksCurrentId = async_hooks.executionAsyncId() - const indentStr = " ".repeat(this._indent < 0 ? 0 : this._indent) - debug2( - `${indentStr}CONTEXT-RUN BEGIN: (${ - this.name - }) currentUid:${currentUid} triggerId:${triggerId} asyncHooksCurrentId:${asyncHooksCurrentId} len:${ - this._set.length - } context:${util.inspect(context)}` - ) - } - fn(context) - return context - } catch (exception) { - if (exception) { - exception[ERROR_SYMBOL] = context - } - throw exception - } finally { - if (DEBUG_CLS_HOOKED) { - const triggerId = async_hooks.triggerAsyncId() - const asyncHooksCurrentId = async_hooks.executionAsyncId() - const indentStr = " ".repeat(this._indent < 0 ? 0 : this._indent) - debug2( - `${indentStr}CONTEXT-RUN END: (${ - this.name - }) currentUid:${currentUid} triggerId:${triggerId} asyncHooksCurrentId:${asyncHooksCurrentId} len:${ - this._set.length - } ${util.inspect(context)}` - ) - } - this.exit(context) - } -} - -Namespace.prototype.runAndReturn = function runAndReturn(fn) { - let value - this.run(function (context) { - value = fn(context) - }) - return value -} - -/** - * Uses global Promise and assumes Promise is cls friendly or wrapped already. - * @param {function} fn - * @returns {*} - */ -Namespace.prototype.runPromise = function runPromise(fn) { - let context = this.createContext() - this.enter(context) - - let promise = fn(context) - if (!promise || !promise.then || !promise.catch) { - throw new Error("fn must return a promise.") - } - - if (DEBUG_CLS_HOOKED) { - debug2( - "CONTEXT-runPromise BEFORE: (" + - this.name + - ") currentUid:" + - currentUid + - " len:" + - this._set.length + - " " + - util.inspect(context) - ) - } - - return promise - .then(result => { - if (DEBUG_CLS_HOOKED) { - debug2( - "CONTEXT-runPromise AFTER then: (" + - this.name + - ") currentUid:" + - currentUid + - " len:" + - this._set.length + - " " + - util.inspect(context) - ) - } - this.exit(context) - return result - }) - .catch(err => { - err[ERROR_SYMBOL] = context - if (DEBUG_CLS_HOOKED) { - debug2( - "CONTEXT-runPromise AFTER catch: (" + - this.name + - ") currentUid:" + - currentUid + - " len:" + - this._set.length + - " " + - util.inspect(context) - ) - } - this.exit(context) - throw err - }) -} - -Namespace.prototype.bind = function bindFactory(fn, context) { - if (!context) { - if (!this.active) { - context = this.createContext() - } else { - context = this.active - } - } - - let self = this - return function clsBind() { - self.enter(context) - try { - return fn.apply(this, arguments) - } catch (exception) { - if (exception) { - exception[ERROR_SYMBOL] = context - } - throw exception - } finally { - self.exit(context) - } - } -} - -Namespace.prototype.enter = function enter(context) { - assert.ok(context, "context must be provided for entering") - if (DEBUG_CLS_HOOKED) { - const asyncHooksCurrentId = async_hooks.executionAsyncId() - const triggerId = async_hooks.triggerAsyncId() - const indentStr = " ".repeat(this._indent < 0 ? 0 : this._indent) - debug2( - `${indentStr}CONTEXT-ENTER: (${ - this.name - }) currentUid:${currentUid} triggerId:${triggerId} asyncHooksCurrentId:${asyncHooksCurrentId} len:${ - this._set.length - } ${util.inspect(context)}` - ) - } - - this._set.push(this.active) - this.active = context -} - -Namespace.prototype.exit = function exit(context) { - assert.ok(context, "context must be provided for exiting") - if (DEBUG_CLS_HOOKED) { - const asyncHooksCurrentId = async_hooks.executionAsyncId() - const triggerId = async_hooks.triggerAsyncId() - const indentStr = " ".repeat(this._indent < 0 ? 0 : this._indent) - debug2( - `${indentStr}CONTEXT-EXIT: (${ - this.name - }) currentUid:${currentUid} triggerId:${triggerId} asyncHooksCurrentId:${asyncHooksCurrentId} len:${ - this._set.length - } ${util.inspect(context)}` - ) - } - - // Fast path for most exits that are at the top of the stack - if (this.active === context) { - assert.ok(this._set.length, "can't remove top context") - this.active = this._set.pop() - return - } - - // Fast search in the stack using lastIndexOf - let index = this._set.lastIndexOf(context) - - if (index < 0) { - if (DEBUG_CLS_HOOKED) { - debug2( - "??ERROR?? context exiting but not entered - ignoring: " + - util.inspect(context) - ) - } - assert.ok( - index >= 0, - "context not currently entered; can't exit. \n" + - util.inspect(this) + - "\n" + - util.inspect(context) - ) - } else { - assert.ok(index, "can't remove top context") - this._set.splice(index, 1) - } -} - -Namespace.prototype.bindEmitter = function bindEmitter(emitter) { - assert.ok( - emitter.on && emitter.addListener && emitter.emit, - "can only bind real EEs" - ) - - let namespace = this - let thisSymbol = "context@" + this.name - - // Capture the context active at the time the emitter is bound. - function attach(listener) { - if (!listener) { - return - } - if (!listener[CONTEXTS_SYMBOL]) { - listener[CONTEXTS_SYMBOL] = Object.create(null) - } - - listener[CONTEXTS_SYMBOL][thisSymbol] = { - namespace: namespace, - context: namespace.active, - } - } - - // At emit time, bind the listener within the correct context. - function bind(unwrapped) { - if (!(unwrapped && unwrapped[CONTEXTS_SYMBOL])) { - return unwrapped - } - - let wrapped = unwrapped - let unwrappedContexts = unwrapped[CONTEXTS_SYMBOL] - Object.keys(unwrappedContexts).forEach(function (name) { - let thunk = unwrappedContexts[name] - wrapped = thunk.namespace.bind(wrapped, thunk.context) - }) - return wrapped - } - - wrapEmitter(emitter, attach, bind) -} - -/** - * If an error comes out of a namespace, it will have a context attached to it. - * This function knows how to find it. - * - * @param {Error} exception Possibly annotated error. - */ -Namespace.prototype.fromException = function fromException(exception) { - return exception[ERROR_SYMBOL] -} - -function getNamespace(name) { - return process.namespaces[name] -} - -function createNamespace(name) { - assert.ok(name, "namespace must be given a name.") - - if (DEBUG_CLS_HOOKED) { - debug2(`NS-CREATING NAMESPACE (${name})`) - } - let namespace = new Namespace(name) - namespace.id = currentUid - - const hook = async_hooks.createHook({ - init(asyncId, type, triggerId, resource) { - currentUid = async_hooks.executionAsyncId() - - //CHAIN Parent's Context onto child if none exists. This is needed to pass net-events.spec - // let initContext = namespace.active; - // if(!initContext && triggerId) { - // let parentContext = namespace._contexts.get(triggerId); - // if (parentContext) { - // namespace.active = parentContext; - // namespace._contexts.set(currentUid, parentContext); - // if (DEBUG_CLS_HOOKED) { - // const indentStr = ' '.repeat(namespace._indent < 0 ? 0 : namespace._indent); - // debug2(`${indentStr}INIT [${type}] (${name}) WITH PARENT CONTEXT asyncId:${asyncId} currentUid:${currentUid} triggerId:${triggerId} active:${util.inspect(namespace.active, true)} resource:${resource}`); - // } - // } else if (DEBUG_CLS_HOOKED) { - // const indentStr = ' '.repeat(namespace._indent < 0 ? 0 : namespace._indent); - // debug2(`${indentStr}INIT [${type}] (${name}) MISSING CONTEXT asyncId:${asyncId} currentUid:${currentUid} triggerId:${triggerId} active:${util.inspect(namespace.active, true)} resource:${resource}`); - // } - // }else { - // namespace._contexts.set(currentUid, namespace.active); - // if (DEBUG_CLS_HOOKED) { - // const indentStr = ' '.repeat(namespace._indent < 0 ? 0 : namespace._indent); - // debug2(`${indentStr}INIT [${type}] (${name}) asyncId:${asyncId} currentUid:${currentUid} triggerId:${triggerId} active:${util.inspect(namespace.active, true)} resource:${resource}`); - // } - // } - if (namespace.active) { - namespace._contexts.set(asyncId, namespace.active) - - if (DEBUG_CLS_HOOKED) { - const indentStr = " ".repeat( - namespace._indent < 0 ? 0 : namespace._indent - ) - debug2( - `${indentStr}INIT [${type}] (${name}) asyncId:${asyncId} currentUid:${currentUid} triggerId:${triggerId} active:${util.inspect( - namespace.active, - { showHidden: true, depth: 2, colors: true } - )} resource:${resource}` - ) - } - } else if (currentUid === 0) { - // CurrentId will be 0 when triggered from C++. Promise events - // https://github.com/nodejs/node/blob/master/doc/api/async_hooks.md#triggerid - const triggerId = async_hooks.triggerAsyncId() - const triggerIdContext = namespace._contexts.get(triggerId) - if (triggerIdContext) { - namespace._contexts.set(asyncId, triggerIdContext) - if (DEBUG_CLS_HOOKED) { - const indentStr = " ".repeat( - namespace._indent < 0 ? 0 : namespace._indent - ) - debug2( - `${indentStr}INIT USING CONTEXT FROM TRIGGERID [${type}] (${name}) asyncId:${asyncId} currentUid:${currentUid} triggerId:${triggerId} active:${util.inspect( - namespace.active, - { showHidden: true, depth: 2, colors: true } - )} resource:${resource}` - ) - } - } else if (DEBUG_CLS_HOOKED) { - const indentStr = " ".repeat( - namespace._indent < 0 ? 0 : namespace._indent - ) - debug2( - `${indentStr}INIT MISSING CONTEXT [${type}] (${name}) asyncId:${asyncId} currentUid:${currentUid} triggerId:${triggerId} active:${util.inspect( - namespace.active, - { showHidden: true, depth: 2, colors: true } - )} resource:${resource}` - ) - } - } - - if (DEBUG_CLS_HOOKED && type === "PROMISE") { - debug2(util.inspect(resource, { showHidden: true })) - const parentId = resource.parentId - const indentStr = " ".repeat( - namespace._indent < 0 ? 0 : namespace._indent - ) - debug2( - `${indentStr}INIT RESOURCE-PROMISE [${type}] (${name}) parentId:${parentId} asyncId:${asyncId} currentUid:${currentUid} triggerId:${triggerId} active:${util.inspect( - namespace.active, - { showHidden: true, depth: 2, colors: true } - )} resource:${resource}` - ) - } - }, - before(asyncId) { - currentUid = async_hooks.executionAsyncId() - let context - - /* - if(currentUid === 0){ - // CurrentId will be 0 when triggered from C++. Promise events - // https://github.com/nodejs/node/blob/master/doc/api/async_hooks.md#triggerid - //const triggerId = async_hooks.triggerAsyncId(); - context = namespace._contexts.get(asyncId); // || namespace._contexts.get(triggerId); - }else{ - context = namespace._contexts.get(currentUid); - } - */ - - //HACK to work with promises until they are fixed in node > 8.1.1 - context = - namespace._contexts.get(asyncId) || namespace._contexts.get(currentUid) - - if (context) { - if (DEBUG_CLS_HOOKED) { - const triggerId = async_hooks.triggerAsyncId() - const indentStr = " ".repeat( - namespace._indent < 0 ? 0 : namespace._indent - ) - debug2( - `${indentStr}BEFORE (${name}) asyncId:${asyncId} currentUid:${currentUid} triggerId:${triggerId} active:${util.inspect( - namespace.active, - { showHidden: true, depth: 2, colors: true } - )} context:${util.inspect(context)}` - ) - namespace._indent += 2 - } - - namespace.enter(context) - } else if (DEBUG_CLS_HOOKED) { - const triggerId = async_hooks.triggerAsyncId() - const indentStr = " ".repeat( - namespace._indent < 0 ? 0 : namespace._indent - ) - debug2( - `${indentStr}BEFORE MISSING CONTEXT (${name}) asyncId:${asyncId} currentUid:${currentUid} triggerId:${triggerId} active:${util.inspect( - namespace.active, - { showHidden: true, depth: 2, colors: true } - )} namespace._contexts:${util.inspect(namespace._contexts, { - showHidden: true, - depth: 2, - colors: true, - })}` - ) - namespace._indent += 2 - } - }, - after(asyncId) { - currentUid = async_hooks.executionAsyncId() - let context // = namespace._contexts.get(currentUid); - /* - if(currentUid === 0){ - // CurrentId will be 0 when triggered from C++. Promise events - // https://github.com/nodejs/node/blob/master/doc/api/async_hooks.md#triggerid - //const triggerId = async_hooks.triggerAsyncId(); - context = namespace._contexts.get(asyncId); // || namespace._contexts.get(triggerId); - }else{ - context = namespace._contexts.get(currentUid); - } - */ - //HACK to work with promises until they are fixed in node > 8.1.1 - context = - namespace._contexts.get(asyncId) || namespace._contexts.get(currentUid) - - if (context) { - if (DEBUG_CLS_HOOKED) { - const triggerId = async_hooks.triggerAsyncId() - namespace._indent -= 2 - const indentStr = " ".repeat( - namespace._indent < 0 ? 0 : namespace._indent - ) - debug2( - `${indentStr}AFTER (${name}) asyncId:${asyncId} currentUid:${currentUid} triggerId:${triggerId} active:${util.inspect( - namespace.active, - { showHidden: true, depth: 2, colors: true } - )} context:${util.inspect(context)}` - ) - } - - namespace.exit(context) - } else if (DEBUG_CLS_HOOKED) { - const triggerId = async_hooks.triggerAsyncId() - namespace._indent -= 2 - const indentStr = " ".repeat( - namespace._indent < 0 ? 0 : namespace._indent - ) - debug2( - `${indentStr}AFTER MISSING CONTEXT (${name}) asyncId:${asyncId} currentUid:${currentUid} triggerId:${triggerId} active:${util.inspect( - namespace.active, - { showHidden: true, depth: 2, colors: true } - )} context:${util.inspect(context)}` - ) - } - }, - destroy(asyncId) { - currentUid = async_hooks.executionAsyncId() - if (DEBUG_CLS_HOOKED) { - const triggerId = async_hooks.triggerAsyncId() - const indentStr = " ".repeat( - namespace._indent < 0 ? 0 : namespace._indent - ) - debug2( - `${indentStr}DESTROY (${name}) currentUid:${currentUid} asyncId:${asyncId} triggerId:${triggerId} active:${util.inspect( - namespace.active, - { showHidden: true, depth: 2, colors: true } - )} context:${util.inspect(namespace._contexts.get(currentUid))}` - ) - } - - namespace._contexts.delete(asyncId) - }, - }) - - hook.enable() - namespace._hook = hook - - process.namespaces[name] = namespace - return namespace -} - -function destroyNamespace(name) { - let namespace = getNamespace(name) - - assert.ok(namespace, "can't delete nonexistent namespace! \"" + name + '"') - assert.ok( - namespace.id, - "don't assign to process.namespaces directly! " + util.inspect(namespace) - ) - - namespace._hook.disable() - namespace._contexts = null - process.namespaces[name] = null -} - -function reset() { - // must unregister async listeners - if (process.namespaces) { - Object.keys(process.namespaces).forEach(function (name) { - destroyNamespace(name) - }) - } - process.namespaces = Object.create(null) -} - -process.namespaces = process.namespaces || {} - -//const fs = require('fs'); -function debug2(...args) { - if (DEBUG_CLS_HOOKED) { - //fs.writeSync(1, `${util.format(...args)}\n`); - process._rawDebug(`${util.format(...args)}`) - } -} - -/*function getFunctionName(fn) { - if (!fn) { - return fn; - } - if (typeof fn === 'function') { - if (fn.name) { - return fn.name; - } - return (fn.toString().trim().match(/^function\s*([^\s(]+)/) || [])[1]; - } else if (fn.constructor && fn.constructor.name) { - return fn.constructor.name; - } -}*/ diff --git a/packages/backend-core/src/context/FunctionContext.ts b/packages/backend-core/src/context/FunctionContext.ts deleted file mode 100644 index 1010f585ef..0000000000 --- a/packages/backend-core/src/context/FunctionContext.ts +++ /dev/null @@ -1,47 +0,0 @@ -import cls from "../clshooked" -import { newid } from "../hashing" - -const REQUEST_ID_KEY = "requestId" -const MAIN_CTX = cls.createNamespace("main") - -function getContextStorage(namespace: any) { - if (namespace && namespace.active) { - let contextData = namespace.active - delete contextData.id - delete contextData._ns_name - return contextData - } - return {} -} - -class FunctionContext { - static run(callback: any) { - return MAIN_CTX.runAndReturn(async () => { - const namespaceId = newid() - MAIN_CTX.set(REQUEST_ID_KEY, namespaceId) - const namespace = cls.createNamespace(namespaceId) - let response = await namespace.runAndReturn(callback) - cls.destroyNamespace(namespaceId) - return response - }) - } - - static setOnContext(key: string, value: any) { - const namespaceId = MAIN_CTX.get(REQUEST_ID_KEY) - const namespace = cls.getNamespace(namespaceId) - namespace.set(key, value) - } - - static getFromContext(key: string) { - const namespaceId = MAIN_CTX.get(REQUEST_ID_KEY) - const namespace = cls.getNamespace(namespaceId) - const context = getContextStorage(namespace) - if (context) { - return context[key] - } else { - return null - } - } -} - -export = FunctionContext diff --git a/packages/backend-core/src/context/constants.ts b/packages/backend-core/src/context/constants.ts index eb156c357b..64fdb45dec 100644 --- a/packages/backend-core/src/context/constants.ts +++ b/packages/backend-core/src/context/constants.ts @@ -1,9 +1,5 @@ import { IdentityContext } from "@budibase/types" -export enum ContextKey { - MAIN = "main", -} - export type ContextMap = { tenantId?: string appId?: string diff --git a/packages/backend-core/src/context/index.ts b/packages/backend-core/src/context/index.ts index 34e6603ef7..910bd192b4 100644 --- a/packages/backend-core/src/context/index.ts +++ b/packages/backend-core/src/context/index.ts @@ -1,13 +1,16 @@ import env from "../environment" -import { SEPARATOR, DocumentType } from "../db/constants" -import cls from "./FunctionContext" -import { baseGlobalDBName } from "../db/tenancy" +import { + SEPARATOR, + DocumentType, + getDevelopmentAppID, + getProdAppID, + baseGlobalDBName, + PouchLike, +} from "../db" +import { Context } from "./localStorage" import { IdentityContext } from "@budibase/types" import { DEFAULT_TENANT_ID as _DEFAULT_TENANT_ID } from "../constants" -import { ContextMap, ContextKey } from "./constants" -import { PouchLike } from "../db" -import { getDevelopmentAppID, getProdAppID } from "../db/conversions" - +import { ContextMap } from "./constants" export const DEFAULT_TENANT_ID = _DEFAULT_TENANT_ID // some test cases call functions directly, need to @@ -19,7 +22,7 @@ export function isMultiTenant() { } export function isTenantIdSet() { - const context = cls.getFromContext(ContextKey.MAIN) as ContextMap + const context = Context.get() return !!context?.tenantId } @@ -53,7 +56,7 @@ export function getTenantIDFromAppID(appId: string) { function updateContext(updates: ContextMap) { let context: ContextMap try { - context = cls.getFromContext(ContextKey.MAIN) + context = Context.get() } catch (err) { // no context, start empty context = {} @@ -68,10 +71,7 @@ function updateContext(updates: ContextMap) { async function newContext(updates: ContextMap, task: any) { // see if there already is a context setup let context: ContextMap = updateContext(updates) - return cls.run(async () => { - cls.setOnContext(ContextKey.MAIN, context) - return await task() - }) + return Context.run(context, task) } export async function doInContext(appId: string, task: any): Promise { @@ -132,7 +132,7 @@ export async function doInIdentityContext( export function getIdentity(): IdentityContext | undefined { try { - const context = cls.getFromContext(ContextKey.MAIN) as ContextMap + const context = Context.get() return context?.identity } catch (e) { // do nothing - identity is not in context @@ -143,7 +143,7 @@ export function getTenantId(): string { if (!isMultiTenant()) { return DEFAULT_TENANT_ID } - const context = cls.getFromContext(ContextKey.MAIN) as ContextMap + const context = Context.get() const tenantId = context?.tenantId if (!tenantId) { throw new Error("Tenant id not found") @@ -152,7 +152,7 @@ export function getTenantId(): string { } export function getAppId(): string | undefined { - const context = cls.getFromContext(ContextKey.MAIN) as ContextMap + const context = Context.get() const foundId = context?.appId if (!foundId && env.isTest() && TEST_APP_ID) { return TEST_APP_ID @@ -165,7 +165,7 @@ export function updateTenantId(tenantId?: string) { let context: ContextMap = updateContext({ tenantId, }) - cls.setOnContext(ContextKey.MAIN, context) + Context.set(context) } export function updateAppId(appId: string) { @@ -173,7 +173,7 @@ export function updateAppId(appId: string) { appId, }) try { - cls.setOnContext(ContextKey.MAIN, context) + Context.set(context) } catch (err) { if (env.isTest()) { TEST_APP_ID = appId @@ -184,7 +184,7 @@ export function updateAppId(appId: string) { } export function getGlobalDB(): PouchLike { - const context = cls.getFromContext(ContextKey.MAIN) as ContextMap + const context = Context.get() return new PouchLike(baseGlobalDBName(context?.tenantId)) } diff --git a/packages/backend-core/src/context/localStorage.ts b/packages/backend-core/src/context/localStorage.ts new file mode 100644 index 0000000000..fd761d10bc --- /dev/null +++ b/packages/backend-core/src/context/localStorage.ts @@ -0,0 +1,18 @@ +import { AsyncLocalStorage } from "async_hooks" +import { ContextMap } from "./constants" + +export class Context { + static storage = new AsyncLocalStorage() + + static run(context: ContextMap, func: any) { + return Context.storage.run(context, () => func()) + } + + static get(): ContextMap { + return Context.storage.getStore() as ContextMap + } + + static set(context: ContextMap) { + Context.storage.enterWith(context) + } +} diff --git a/packages/backend-core/src/db/couch/pouchDB.ts b/packages/backend-core/src/db/couch/pouchDB.ts index eead0e4d6f..07f7b6c90d 100644 --- a/packages/backend-core/src/db/couch/pouchDB.ts +++ b/packages/backend-core/src/db/couch/pouchDB.ts @@ -53,7 +53,7 @@ export const getPouch = (opts: any = {}) => { return PouchDB.defaults(POUCH_DB_DEFAULTS) } -export async function init(opts?: PouchOptions) { +export function init(opts?: PouchOptions) { Pouch = getPouch(opts) initialised = true } diff --git a/packages/backend-core/src/db/db.ts b/packages/backend-core/src/db/db.ts new file mode 100644 index 0000000000..497b747b80 --- /dev/null +++ b/packages/backend-core/src/db/db.ts @@ -0,0 +1,50 @@ +import env from "../environment" +import { directCouchQuery, PouchLike } from "./couch" +import { CouchFindOptions } from "@budibase/types" + +let initialised = false +const dbList = new Set() + +const checkInitialised = () => { + if (!initialised) { + throw new Error("init has not been called") + } +} + +export function getDB(dbName: string, opts?: any): PouchLike { + if (env.isTest()) { + dbList.add(dbName) + } + return new PouchLike(dbName, opts) +} + +// we have to use a callback for this so that we can close +// the DB when we're done, without this manual requests would +// need to close the database when done with it to avoid memory leaks +export async function doWithDB(dbName: string, cb: any, opts = {}) { + const db = getDB(dbName, opts) + // need this to be async so that we can correctly close DB after all + // async operations have been completed + return await cb(db) +} + +export function allDbs() { + if (!env.isTest()) { + throw new Error("Cannot be used outside test environment.") + } + checkInitialised() + return [...dbList] +} + +export async function directCouchAllDbs(queryString?: string) { + let couchPath = "/_all_dbs" + if (queryString) { + couchPath += `?${queryString}` + } + return await directCouchQuery(couchPath) +} + +export async function directCouchFind(dbName: string, opts: CouchFindOptions) { + const json = await directCouchQuery(`${dbName}/_find`, "POST", opts) + return { rows: json.docs, bookmark: json.bookmark } +} diff --git a/packages/backend-core/src/db/index.ts b/packages/backend-core/src/db/index.ts index 7cc49eae40..7269aa8f92 100644 --- a/packages/backend-core/src/db/index.ts +++ b/packages/backend-core/src/db/index.ts @@ -1,51 +1,7 @@ -import env from "../environment" -import { CouchFindOptions } from "@budibase/types" -import { directCouchQuery, PouchLike } from "./couch" -export { init, PouchLike, getPouch, getPouchDB } from "./couch" - -let initialised = false -const dbList = new Set() - -const checkInitialised = () => { - if (!initialised) { - throw new Error("init has not been called") - } -} - -export function getDB(dbName: string, opts?: any): PouchLike { - if (env.isTest()) { - dbList.add(dbName) - } - return new PouchLike(dbName, opts) -} - -// we have to use a callback for this so that we can close -// the DB when we're done, without this manual requests would -// need to close the database when done with it to avoid memory leaks -export async function doWithDB(dbName: string, cb: any, opts = {}) { - const db = getDB(dbName, opts) - // need this to be async so that we can correctly close DB after all - // async operations have been completed - return await cb(db) -} - -export function allDbs() { - if (!env.isTest()) { - throw new Error("Cannot be used outside test environment.") - } - checkInitialised() - return [...dbList] -} - -export async function directCouchAllDbs(queryString?: string) { - let couchPath = "/_all_dbs" - if (queryString) { - couchPath += `?${queryString}` - } - return await directCouchQuery(couchPath) -} - -export async function directCouchFind(dbName: string, opts: CouchFindOptions) { - const json = await directCouchQuery(`${dbName}/_find`, "POST", opts) - return { rows: json.docs, bookmark: json.bookmark } -} +export * from "./couch" +export * from "./db" +export * from "./utils" +export * from "./views" +export * from "./constants" +export * from "./conversions" +export * from "./tenancy" diff --git a/packages/backend-core/src/db/utils.ts b/packages/backend-core/src/db/utils.ts index 9a4ed98418..fc67a2c49b 100644 --- a/packages/backend-core/src/db/utils.ts +++ b/packages/backend-core/src/db/utils.ts @@ -10,7 +10,7 @@ import { } from "./constants" import { getTenantId, getGlobalDB } from "../context" import { getGlobalDBName } from "./tenancy" -import { doWithDB, allDbs, directCouchAllDbs } from "./index" +import { doWithDB, allDbs, directCouchAllDbs } from "./db" import { getAppMetadata } from "../cache/appMetadata" import { isDevApp, isDevAppID, getProdAppID } from "./conversions" import { APP_PREFIX } from "./constants" diff --git a/packages/backend-core/src/index.ts b/packages/backend-core/src/index.ts index b45248abc2..a2cfaaa9c6 100644 --- a/packages/backend-core/src/index.ts +++ b/packages/backend-core/src/index.ts @@ -24,7 +24,7 @@ import * as queue from "./queue" import * as types from "./types" // mimic the outer package exports -import * as db from "./pkg/db" +import * as db from "./db" import * as objectStore from "./pkg/objectStore" import * as utils from "./pkg/utils" import redis from "./pkg/redis" diff --git a/packages/backend-core/src/pkg/db.ts b/packages/backend-core/src/pkg/db.ts deleted file mode 100644 index d2bc474786..0000000000 --- a/packages/backend-core/src/pkg/db.ts +++ /dev/null @@ -1,6 +0,0 @@ -// Mimic the outer package export for usage in index.ts -// The outer exports can't be used as they now reference dist directly -export * from "../db" -export * from "../db/utils" -export * from "../db/views" -export * from "../db/constants" From 3f0e53f90c3f5e37238d02df2eceb2bcf9886080 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 14 Nov 2022 18:29:15 +0000 Subject: [PATCH 35/91] Updating context test, some minor adjustments based on getting the test working again. --- packages/backend-core/src/context/index.ts | 12 +++++--- .../tests/{index.spec.ts => index.spec.js} | 30 +++++-------------- packages/backend-core/src/db/couch/index.ts | 2 +- packages/backend-core/src/db/db.ts | 2 +- 4 files changed, 17 insertions(+), 29 deletions(-) rename packages/backend-core/src/context/tests/{index.spec.ts => index.spec.js} (80%) diff --git a/packages/backend-core/src/context/index.ts b/packages/backend-core/src/context/index.ts index 910bd192b4..15eed8d0ba 100644 --- a/packages/backend-core/src/context/index.ts +++ b/packages/backend-core/src/context/index.ts @@ -5,6 +5,7 @@ import { getDevelopmentAppID, getProdAppID, baseGlobalDBName, + getDB, PouchLike, } from "../db" import { Context } from "./localStorage" @@ -185,7 +186,10 @@ export function updateAppId(appId: string) { export function getGlobalDB(): PouchLike { const context = Context.get() - return new PouchLike(baseGlobalDBName(context?.tenantId)) + if (!context || (env.MULTI_TENANCY && !context.tenantId)) { + throw new Error("Global DB not found") + } + return getDB(baseGlobalDBName(context?.tenantId)) } /** @@ -194,7 +198,7 @@ export function getGlobalDB(): PouchLike { */ export function getAppDB(opts?: any): PouchLike { const appId = getAppId() - return new PouchLike(appId, opts) + return getDB(appId, opts) } /** @@ -206,7 +210,7 @@ export function getProdAppDB(opts?: any): PouchLike { if (!appId) { throw new Error("Unable to retrieve prod DB - no app ID.") } - return new PouchLike(getProdAppID(appId), opts) + return getDB(getProdAppID(appId), opts) } /** @@ -218,5 +222,5 @@ export function getDevAppDB(opts?: any): PouchLike { if (!appId) { throw new Error("Unable to retrieve dev DB - no app ID.") } - return new PouchLike(getDevelopmentAppID(appId), opts) + return getDB(getDevelopmentAppID(appId), opts) } diff --git a/packages/backend-core/src/context/tests/index.spec.ts b/packages/backend-core/src/context/tests/index.spec.js similarity index 80% rename from packages/backend-core/src/context/tests/index.spec.ts rename to packages/backend-core/src/context/tests/index.spec.js index 749231c3e5..6d09446059 100644 --- a/packages/backend-core/src/context/tests/index.spec.ts +++ b/packages/backend-core/src/context/tests/index.spec.js @@ -1,18 +1,10 @@ -import "../../../tests/utilities/TestConfiguration" -import * as context from ".." -import { DEFAULT_TENANT_ID } from "../../constants" -import env from "../../environment" - -// must use require to spy index file exports due to known issue in jest -const dbUtils = require("../../db") -jest.spyOn(dbUtils, "closePouchDB") -jest.spyOn(dbUtils, "getDB") +require("../../../tests/utilities/TestConfiguration") +const context = require("../") +const { DEFAULT_TENANT_ID } = require("../../constants") +const env = require("../../environment") +const dbCore = require("../../db") describe("context", () => { - beforeEach(() => { - jest.clearAllMocks() - }) - describe("doInTenant", () => { describe("single-tenancy", () => { it("defaults to the default tenant", () => { @@ -25,8 +17,6 @@ describe("context", () => { const db = context.getGlobalDB() expect(db.name).toBe("global-db") }) - expect(dbUtils.getDB).toHaveBeenCalledTimes(1) - expect(dbUtils.closePouchDB).toHaveBeenCalledTimes(1) }) }) @@ -40,7 +30,7 @@ describe("context", () => { let error try { context.getTenantId() - } catch (e: any) { + } catch (e) { error = e } expect(error.message).toBe("Tenant id not found") @@ -59,7 +49,7 @@ describe("context", () => { let error try { context.getGlobalDB() - } catch (e: any) { + } catch (e) { error = e } expect(error.message).toBe("Global DB not found") @@ -85,8 +75,6 @@ describe("context", () => { const db = context.getGlobalDB() expect(db.name).toBe("test_global-db") }) - expect(dbUtils.getDB).toHaveBeenCalledTimes(1) - expect(dbUtils.closePouchDB).toHaveBeenCalledTimes(1) }) it("sets the tenant id when nested with same tenant id", async () => { @@ -121,10 +109,6 @@ describe("context", () => { }) }) }) - - // only 1 db is opened and closed - expect(dbUtils.getDB).toHaveBeenCalledTimes(1) - expect(dbUtils.closePouchDB).toHaveBeenCalledTimes(1) }) it("sets different tenant id inside another context", () => { diff --git a/packages/backend-core/src/db/couch/index.ts b/packages/backend-core/src/db/couch/index.ts index 3730a1e248..b8cb0ab79f 100644 --- a/packages/backend-core/src/db/couch/index.ts +++ b/packages/backend-core/src/db/couch/index.ts @@ -1,4 +1,4 @@ export * from "./connections" export * from "./pouchLike" export * from "./utils" -export { init, getPouch, getPouchDB } from "./pouchDB" +export { init, getPouch, getPouchDB, closePouchDB } from "./pouchDB" diff --git a/packages/backend-core/src/db/db.ts b/packages/backend-core/src/db/db.ts index 497b747b80..909d286d32 100644 --- a/packages/backend-core/src/db/db.ts +++ b/packages/backend-core/src/db/db.ts @@ -11,7 +11,7 @@ const checkInitialised = () => { } } -export function getDB(dbName: string, opts?: any): PouchLike { +export function getDB(dbName?: string, opts?: any): PouchLike { if (env.isTest()) { dbList.add(dbName) } From 72c9aadae083f0c9d7bc005da0ade75ff9ebe824 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 14 Nov 2022 20:25:05 +0000 Subject: [PATCH 36/91] Updating test cases - trying to get everything in order for merge. --- packages/backend-core/package.json | 2 +- packages/backend-core/src/db/couch/pouchDB.ts | 15 +- packages/backend-core/src/db/db.ts | 13 +- .../backend-core/src/db/tests/pouch.spec.js | 2 +- .../backend-core/src/db/tests/utils.spec.js | 6 +- packages/server/__mocks__/node-fetch.ts | 6 +- packages/server/package.json | 2 +- .../server/src/db/tests/linkTests.spec.js | 31 ++-- .../src/middleware/tests/currentapp.spec.js | 150 ++++++++++-------- packages/worker/package.json | 2 +- 10 files changed, 134 insertions(+), 95 deletions(-) diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json index 2fbc087214..63e0d9ff9b 100644 --- a/packages/backend-core/package.json +++ b/packages/backend-core/package.json @@ -58,7 +58,7 @@ "testEnvironment": "node", "moduleNameMapper": { "@budibase/types": "/../types/src", - "axios": "axios/dist/node/axios.cjs" + "^axios.*$": "/node_modules/axios/lib/axios.js" }, "setupFiles": [ "./scripts/jestSetup.ts" diff --git a/packages/backend-core/src/db/couch/pouchDB.ts b/packages/backend-core/src/db/couch/pouchDB.ts index 07f7b6c90d..e7efbf6dcd 100644 --- a/packages/backend-core/src/db/couch/pouchDB.ts +++ b/packages/backend-core/src/db/couch/pouchDB.ts @@ -66,7 +66,20 @@ const checkInitialised = () => { export function getPouchDB(dbName: string, opts?: any): PouchDB.Database { checkInitialised() - return new Pouch(dbName, opts) + const db = new Pouch(dbName, opts) + const dbPut = db.put + db.put = async (doc: any, options = {}) => { + if (!doc.createdAt) { + doc.createdAt = new Date().toISOString() + } + doc.updatedAt = new Date().toISOString() + return dbPut(doc, options) + } + db.exists = async () => { + const info = await db.info() + return !info.error + } + return db } // use this function if you have called getPouchDB - close diff --git a/packages/backend-core/src/db/db.ts b/packages/backend-core/src/db/db.ts index 909d286d32..beaea3d42c 100644 --- a/packages/backend-core/src/db/db.ts +++ b/packages/backend-core/src/db/db.ts @@ -1,19 +1,15 @@ import env from "../environment" -import { directCouchQuery, PouchLike } from "./couch" +import { directCouchQuery, PouchLike, getPouchDB } from "./couch" import { CouchFindOptions } from "@budibase/types" -let initialised = false const dbList = new Set() -const checkInitialised = () => { - if (!initialised) { - throw new Error("init has not been called") - } -} - export function getDB(dbName?: string, opts?: any): PouchLike { + // TODO: once using the test image, need to remove this if (env.isTest()) { dbList.add(dbName) + // @ts-ignore + return getPouchDB(dbName, opts) } return new PouchLike(dbName, opts) } @@ -32,7 +28,6 @@ export function allDbs() { if (!env.isTest()) { throw new Error("Cannot be used outside test environment.") } - checkInitialised() return [...dbList] } diff --git a/packages/backend-core/src/db/tests/pouch.spec.js b/packages/backend-core/src/db/tests/pouch.spec.js index 30cdd0f5ec..ce27ab7d22 100644 --- a/packages/backend-core/src/db/tests/pouch.spec.js +++ b/packages/backend-core/src/db/tests/pouch.spec.js @@ -1,5 +1,5 @@ require("../../../tests/utilities/TestConfiguration") -const getUrlInfo = require("../pouch").getUrlInfo +const getUrlInfo = require("../couch").getUrlInfo describe("pouch", () => { describe("Couch DB URL parsing", () => { diff --git a/packages/backend-core/src/db/tests/utils.spec.js b/packages/backend-core/src/db/tests/utils.spec.js index 5f9a224e7a..1700621334 100644 --- a/packages/backend-core/src/db/tests/utils.spec.js +++ b/packages/backend-core/src/db/tests/utils.spec.js @@ -1,4 +1,4 @@ -require("../../../tests/utilities/TestConfiguration"); +require("../../../tests/utilities/TestConfiguration") const { generateAppID, getDevelopmentAppID, @@ -8,8 +8,8 @@ const { getPlatformUrl, getScopedConfig } = require("../utils") -const tenancy = require("../../tenancy"); -const { Configs, DEFAULT_TENANT_ID } = require("../../constants"); +const tenancy = require("../../tenancy") +const { Configs, DEFAULT_TENANT_ID } = require("../../constants") const env = require("../../environment") describe("utils", () => { diff --git a/packages/server/__mocks__/node-fetch.ts b/packages/server/__mocks__/node-fetch.ts index 0e32c39edd..c3d6b623dd 100644 --- a/packages/server/__mocks__/node-fetch.ts +++ b/packages/server/__mocks__/node-fetch.ts @@ -2,7 +2,7 @@ module FetchMock { const fetch = jest.requireActual("node-fetch") let failCount = 0 - module.exports = async (url: any, opts: any) => { + const func = async (url: any, opts: any) => { function json(body: any, status = 200) { return { status, @@ -106,4 +106,8 @@ module FetchMock { } return fetch(url, opts) } + + func.Headers = fetch.Headers + + module.exports = func } diff --git a/packages/server/package.json b/packages/server/package.json index 6d75877c79..ad8f62739d 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -43,7 +43,7 @@ "@budibase/backend-core/(.*)": "/../backend-core/$1", "@budibase/backend-core": "/../backend-core/src", "@budibase/types": "/../types/src", - "axios": "axios/dist/node/axios.cjs" + "^axios.*$": "/node_modules/axios/lib/axios.js" }, "setupFiles": [ "./scripts/jestSetup.js" diff --git a/packages/server/src/db/tests/linkTests.spec.js b/packages/server/src/db/tests/linkTests.spec.js index 8bc26cde2a..0684255ea1 100644 --- a/packages/server/src/db/tests/linkTests.spec.js +++ b/packages/server/src/db/tests/linkTests.spec.js @@ -1,30 +1,34 @@ const TestConfig = require("../../tests/utilities/TestConfiguration") const { basicTable } = require("../../tests/utilities/structures") const linkUtils = require("../linkedRows/linkUtils") -const { getAppDB } = require("@budibase/backend-core/context") -const { doWithDB } = require("@budibase/backend-core/db") +const { context } = require("@budibase/backend-core") describe("test link functionality", () => { const config = new TestConfig(false) + let appId describe("getLinkedTable", () => { - let db, table + let table beforeEach(async () => { - await config.init() - db = getAppDB() + const app = await config.init() + appId = app.appId table = await config.createTable() }) it("should be able to retrieve a linked table from a list", async () => { - const retrieved = await linkUtils.getLinkedTable(table._id, [table]) - expect(retrieved._id).toBe(table._id) + await context.doInAppContext(appId, async () => { + const retrieved = await linkUtils.getLinkedTable(table._id, [table]) + expect(retrieved._id).toBe(table._id) + }) }) it("should be able to retrieve a table from DB and update list", async () => { const tables = [] - const retrieved = await linkUtils.getLinkedTable(table._id, tables) - expect(retrieved._id).toBe(table._id) - expect(tables[0]).toBeDefined() + await context.doInAppContext(appId, async () => { + const retrieved = await linkUtils.getLinkedTable(table._id, tables) + expect(retrieved._id).toBe(table._id) + expect(tables[0]).toBeDefined() + }) }) }) @@ -48,15 +52,14 @@ describe("test link functionality", () => { describe("getLinkDocuments", () => { it("should create the link view when it doesn't exist", async () => { // create the DB and a very basic app design DB - const output = await doWithDB("test", async db => { - await db.put({ _id: "_design/database", views: {} }) - return await linkUtils.getLinkDocuments({ + await context.doInAppContext(appId, async () => { + const output = await linkUtils.getLinkDocuments({ tableId: "test", rowId: "test", includeDocs: false, }) + expect(Array.isArray(output)).toBe(true) }) - expect(Array.isArray(output)).toBe(true) }) }) }) \ No newline at end of file diff --git a/packages/server/src/middleware/tests/currentapp.spec.js b/packages/server/src/middleware/tests/currentapp.spec.js index 638abcbba4..12f68bedfc 100644 --- a/packages/server/src/middleware/tests/currentapp.spec.js +++ b/packages/server/src/middleware/tests/currentapp.spec.js @@ -1,16 +1,8 @@ +require("../../db").init() mockAuthWithNoCookie() mockWorker() mockUserGroups() -jest.mock("@budibase/backend-core/db", () => { - const coreDb = jest.requireActual("@budibase/backend-core/db") - coreDb.init() - return { - ...coreDb, - dbExists: async () => true, - } -}) - function mockWorker() { jest.mock("../../utilities/workerRequests", () => ({ getGlobalSelf: () => { @@ -43,42 +35,51 @@ function mockUserGroups() { function mockAuthWithNoCookie() { jest.resetModules() mockWorker() - jest.mock("@budibase/backend-core/cache", () => ({ - user: { - getUser: () => { - return { - _id: "us_uuid1", - } + jest.mock("@budibase/backend-core", () => { + const core = jest.requireActual("@budibase/backend-core") + return { + ...core, + db: { + dbExists: () => true, }, - }, - })) - jest.mock("@budibase/backend-core/utils", () => ({ - getAppIdFromCtx: jest.fn(), - setCookie: jest.fn(), - getCookie: jest.fn(), - })) - jest.mock("@budibase/backend-core/constants", () => ({ - Cookies: {}, - })) + cache: { + user: { + getUser: () => { + return { + _id: "us_uuid1", + } + }, + }, + }, + utils: { + getAppIdFromCtx: jest.fn(), + setCookie: jest.fn(), + getCookie: jest.fn(), + }, + } + }) } function mockAuthWithCookie() { jest.resetModules() mockWorker() - jest.mock("@budibase/backend-core/utils", () => ({ - getAppIdFromCtx: () => { - return "app_test" - }, - setCookie: jest.fn(), - clearCookie: jest.fn(), - getCookie: () => ({appId: "app_different", roleId: "PUBLIC"}), - })) - jest.mock("@budibase/backend-core/constants", () => ({ - Cookies: { - Auth: "auth", - CurrentApp: "currentapp", - }, - })) + jest.mock("@budibase/backend-core", () => { + const core = jest.requireActual("@budibase/backend-core") + return { + ...core, + db: { + dbExists: () => true, + }, + utils: { + getAppIdFromCtx: () => { + return "app_test" + }, + setCookie: jest.fn(), + clearCookie: jest.fn(), + getCookie: () => ({ appId: "app_different", roleId: "PUBLIC" }), + }, + } + }) } class TestConfiguration { @@ -88,7 +89,16 @@ class TestConfiguration { this.ctx = { next: this.next, - throw: this.throw + throw: this.throw, + request: { + body: {}, + headers: {}, + }, + headers: {}, + path: "", + cookies: { + set: jest.fn(), + } } } @@ -101,6 +111,8 @@ class TestConfiguration { executeMiddleware() { // import as late as possible for mocks + jest.resetModules() + require("../../db").init() const currentAppMiddleware = require("../currentapp") return currentAppMiddleware(this.ctx, this.next) } @@ -138,11 +150,11 @@ describe("Current app middleware", () => { async function checkExpected(setCookie) { config.setUser() await config.executeMiddleware() - let { setCookie: cookieFn } = require("@budibase/backend-core/utils") + let { utils } = require("@budibase/backend-core") if (setCookie) { - expect(cookieFn).toHaveBeenCalled() + expect(utils.setCookie).toHaveBeenCalled() } else { - expect(cookieFn).not.toHaveBeenCalled() + expect(utils.setCookie).not.toHaveBeenCalled() } expect(config.ctx.roleId).toEqual("PUBLIC") expect(config.ctx.user.role._id).toEqual("PUBLIC") @@ -157,31 +169,43 @@ describe("Current app middleware", () => { it("should perform correct when no cookie exists", async () => { mockReset() - jest.mock("@budibase/backend-core/utils", () => ({ - getAppIdFromCtx: () => { - return "app_test" - }, - setCookie: jest.fn(), - getCookie: jest.fn(), - })) - jest.mock("@budibase/backend-core/constants", () => ({ - Cookies: {}, - })) + jest.mock("@budibase/backend-core", () => { + const core = jest.requireActual("@budibase/backend-core") + return { + ...core, + db: { + dbExists: () => true, + }, + utils: { + getAppIdFromCtx: () => { + return "app_test" + }, + setCookie: jest.fn(), + getCookie: jest.fn(), + }, + } + }) await checkExpected(true) }) it("lastly check what occurs when cookie doesn't need updated", async () => { mockReset() - jest.mock("@budibase/backend-core/utils", () => ({ - getAppIdFromCtx: () => { - return "app_test" - }, - setCookie: jest.fn(), - getCookie: () => ({appId: "app_test", roleId: "PUBLIC"}), - })) - jest.mock("@budibase/backend-core/constants", () => ({ - Cookies: {}, - })) + jest.mock("@budibase/backend-core", () => { + const core = jest.requireActual("@budibase/backend-core") + return { + ...core, + db: { + dbExists: () => true, + }, + utils: { + getAppIdFromCtx: () => { + return "app_test" + }, + setCookie: jest.fn(), + getCookie: () => ({ appId: "app_test", roleId: "PUBLIC" }), + }, + } + }) await checkExpected(false) }) }) diff --git a/packages/worker/package.json b/packages/worker/package.json index 844448b569..1e694adfc9 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -99,7 +99,7 @@ "@budibase/backend-core/(.*)": "/../backend-core/$1", "@budibase/backend-core": "/../backend-core/src", "@budibase/types": "/../types/src", - "axios": "axios/dist/node/axios.cjs" + "^axios.*$": "/node_modules/axios/lib/axios.js" }, "setupFiles": [ "./scripts/jestSetup.js" From d2f8a59b6540a6d8986f9643ea1fa878aaa1d0d9 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 15 Nov 2022 10:49:17 +0000 Subject: [PATCH 37/91] Updating proxy configuration for docker-compose, k8s and single image to allow longer timeout for app export requests. Also fixing an issue that blocked requests from getting to app-service. --- hosting/nginx.prod.conf.hbs | 13 +++++++++++-- hosting/single/nginx/nginx-default-site.conf | 18 ++++++++++++++++++ 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/hosting/nginx.prod.conf.hbs b/hosting/nginx.prod.conf.hbs index fe6f2529d3..64f6cbedd2 100644 --- a/hosting/nginx.prod.conf.hbs +++ b/hosting/nginx.prod.conf.hbs @@ -117,12 +117,21 @@ http { } location /api/backups/ { + # calls to export apps are limited + limit_req zone=ratelimit burst=20 nodelay; + + # 1800s timeout for app export requests proxy_read_timeout 1800s; proxy_connect_timeout 1800s; proxy_send_timeout 1800s; - proxy_pass http://app-service; + proxy_http_version 1.1; - proxy_set_header Connection ""; + proxy_set_header Connection $connection_upgrade; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + + proxy_pass http://$apps:4002; } location /api/ { diff --git a/hosting/single/nginx/nginx-default-site.conf b/hosting/single/nginx/nginx-default-site.conf index bd89e21251..9a5ec91c1f 100644 --- a/hosting/single/nginx/nginx-default-site.conf +++ b/hosting/single/nginx/nginx-default-site.conf @@ -43,6 +43,24 @@ server { rewrite ^/worker/(.*)$ /$1 break; } + location /api/backups/ { + # calls to export apps are limited + limit_req zone=ratelimit burst=20 nodelay; + + # 1800s timeout for app export requests + proxy_read_timeout 1800s; + proxy_connect_timeout 1800s; + proxy_send_timeout 1800s; + + proxy_http_version 1.1; + proxy_set_header Connection $connection_upgrade; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + + proxy_pass http://127.0.0.1:4001; + } + location /api/ { # calls to the API are rate limited with bursting limit_req zone=ratelimit burst=20 nodelay; From e7061647da6e2989166a201bc30a1526febe817c Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Tue, 15 Nov 2022 12:52:06 +0000 Subject: [PATCH 38/91] Ensure action parameters can never be null and fix display of close screen modal action --- .../ButtonActionDrawer.svelte | 22 +++++++++---------- .../actions/CloseScreenModal.svelte | 9 ++++++-- 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/packages/builder/src/components/design/settings/controls/ButtonActionEditor/ButtonActionDrawer.svelte b/packages/builder/src/components/design/settings/controls/ButtonActionEditor/ButtonActionDrawer.svelte index ef7c81233b..1d18fa3a92 100644 --- a/packages/builder/src/components/design/settings/controls/ButtonActionEditor/ButtonActionDrawer.svelte +++ b/packages/builder/src/components/design/settings/controls/ButtonActionEditor/ButtonActionDrawer.svelte @@ -23,14 +23,18 @@ export let bindings = [] export let nested - $: showAvailableActions = !actions?.length - let actionQuery - $: parsedQuery = - typeof actionQuery === "string" ? actionQuery.toLowerCase().trim() : "" - let selectedAction = actions?.length ? actions[0] : null + $: { + // Ensure parameters object is never null + if (selectedAction && !selectedAction.parameters) { + selectedAction.parameters = {} + } + } + $: parsedQuery = + typeof actionQuery === "string" ? actionQuery.toLowerCase().trim() : "" + $: showAvailableActions = !actions?.length $: mappedActionTypes = actionTypes.reduce((acc, action) => { let parsedName = action.name.toLowerCase().trim() if (parsedQuery.length && parsedName.indexOf(parsedQuery) < 0) { @@ -40,7 +44,6 @@ acc[action.type].push(action) return acc }, {}) - // These are ephemeral bindings which only exist while executing actions $: eventContexBindings = getEventContextBindings( $currentAsset, @@ -50,9 +53,8 @@ selectedAction?.id ) $: allBindings = eventContexBindings.concat(bindings) - - // Assign a unique ID to each action $: { + // Ensure each action has a unique ID if (actions) { actions.forEach(action => { if (!action.id) { @@ -61,13 +63,11 @@ }) } } - $: selectedActionComponent = selectedAction && actionTypes.find(t => t.name === selectedAction[EVENT_TYPE_KEY])?.component - - // Select the first action if we delete an action $: { + // Select the first action if we delete an action if (selectedAction && !actions?.includes(selectedAction)) { selectedAction = actions?.[0] } diff --git a/packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/CloseScreenModal.svelte b/packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/CloseScreenModal.svelte index 5f3b3ef639..d01a085a23 100644 --- a/packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/CloseScreenModal.svelte +++ b/packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/CloseScreenModal.svelte @@ -6,9 +6,11 @@ export let bindings = [] -Navigate To screen, or leave blank. -
+ + You can optionally navigate to another screen after closing the screen + modal. +