diff --git a/.eslintignore b/.eslintignore index 91f5433596..54824be5c7 100644 --- a/.eslintignore +++ b/.eslintignore @@ -6,4 +6,5 @@ packages/server/coverage packages/server/client packages/builder/.routify packages/builder/cypress/support/queryLevelTransformerFunction.js -packages/builder/cypress/support/queryLevelTransformerFunctionWithData.js \ No newline at end of file +packages/builder/cypress/support/queryLevelTransformerFunctionWithData.js +packages/builder/cypress/reports \ No newline at end of file diff --git a/.github/workflows/release-develop.yml b/.github/workflows/release-develop.yml index 7ee02ccdd1..d4050ab40e 100644 --- a/.github/workflows/release-develop.yml +++ b/.github/workflows/release-develop.yml @@ -1,4 +1,5 @@ name: Budibase Release Staging +concurrency: release-develop on: push: diff --git a/.github/workflows/release-selfhost.yml b/.github/workflows/release-selfhost.yml index 9f42a9cc5d..fc2b7b0cca 100644 --- a/.github/workflows/release-selfhost.yml +++ b/.github/workflows/release-selfhost.yml @@ -87,3 +87,10 @@ jobs: packages/cli/build/cli-macos packages/server/specs/openapi.yaml packages/server/specs/openapi.json + + - name: Discord Webhook Action + uses: tsickert/discord-webhook@v4.0.0 + with: + webhook-url: ${{ secrets.PROD_DEPLOY_WEBHOOK_URL }} + content: "Self Host Deployment Complete: ${{ env.RELEASE_VERSION }} deployed to Self Host." + embed-title: ${{ env.RELEASE_VERSION }} \ No newline at end of file diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 359ad4467b..fa3aaf28e9 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,4 +1,5 @@ name: Budibase Release +concurrency: release on: push: diff --git a/.github/workflows/smoke_test.yaml b/.github/workflows/smoke_test.yaml index 79c97b69af..7002c8335b 100644 --- a/.github/workflows/smoke_test.yaml +++ b/.github/workflows/smoke_test.yaml @@ -31,24 +31,22 @@ jobs: continue-on-error: true uses: cypress-io/github-action@v2 with: + record: true install: false + tag: nightly command: yarn test:e2e:ci:record env: CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }} - # TODO: upload recordings to s3 - # - name: Configure AWS Credentials - # uses: aws-actions/configure-aws-credentials@v1 - # with: - # aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - # aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - # aws-region: eu-west-1 - - - name: Discord Webhook Action - uses: tsickert/discord-webhook@v4.0.0 + - uses: actions/upload-artifact@v3 with: - webhook-url: ${{ secrets.BUDI_QA_WEBHOOK }} - content: "Smoke test run completed with ${{ steps.cypress.outcome }}. See results at ${{ steps.cypress.dashboardUrl }}" - embed-title: ${{ steps.cypress.outcome }} - embed-color: ${{ steps.cypress.outcome == 'success' && '3066993' || '15548997' }} + name: Test Reports + path: packages/builder/cypress/reports/testReport.html + - name: Cypress Discord Notify + run: yarn test:e2e:ci:notify + env: + CYPRESS_WEBHOOK_URL: ${{ secrets.BUDI_QA_WEBHOOK }} + CYPRESS_OUTCOME: ${{ steps.cypress.outcome }} + CYPRESS_DASHBOARD_URL: ${{ steps.cypress.outputs.dashboardUrl }} + GITHUB_RUN_URL: $GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID diff --git a/.gitignore b/.gitignore index 7d09f0a2ba..03d77c5477 100644 --- a/.gitignore +++ b/.gitignore @@ -97,5 +97,7 @@ hosting/proxy/.generated-nginx.prod.conf bin/ hosting/.generated* -packages/builder/cypress.env.json -stats.html +packages/builder/cypress.env.json +packages/builder/cypress/reports +stats.html + diff --git a/charts/budibase/templates/app-service-deployment.yaml b/charts/budibase/templates/app-service-deployment.yaml index 3bbe718d73..98a949418c 100644 --- a/charts/budibase/templates/app-service-deployment.yaml +++ b/charts/budibase/templates/app-service-deployment.yaml @@ -114,8 +114,8 @@ spec: value: {{ .Values.globals.google.secret | quote }} - name: AUTOMATION_MAX_ITERATIONS value: {{ .Values.globals.automationMaxIterations | quote }} - - name: EXCLUDE_QUOTAS_TENANTS - value: {{ .Values.globals.excludeQuotasTenants | quote }} + - name: TENANT_FEATURE_FLAGS + value: {{ .Values.globals.tenantFeatureFlags | quote }} image: budibase/apps:{{ .Values.globals.appVersion }} imagePullPolicy: Always diff --git a/hosting/nginx.prod.conf.hbs b/hosting/nginx.prod.conf.hbs index cd5f7b4d48..a127dfbd5c 100644 --- a/hosting/nginx.prod.conf.hbs +++ b/hosting/nginx.prod.conf.hbs @@ -30,7 +30,7 @@ http { log_format main '$remote_addr - $remote_user [$time_local] "$request" ' '$status $body_bytes_sent "$http_referer" ' '"$http_user_agent" "$http_x_forwarded_for"'; - + map $http_upgrade $connection_upgrade { default "upgrade"; } @@ -42,13 +42,13 @@ http { client_max_body_size 1000m; ignore_invalid_headers off; proxy_buffering off; - + set $csp_default "default-src 'self'"; set $csp_script "script-src 'self' 'unsafe-inline' 'unsafe-eval' https://cdn.budi.live https://js.intercomcdn.com https://widget.intercom.io"; set $csp_style "style-src 'self' 'unsafe-inline' https://cdn.jsdelivr.net https://fonts.googleapis.com https://rsms.me https://maxcdn.bootstrapcdn.com"; set $csp_object "object-src 'none'"; set $csp_base_uri "base-uri 'self'"; - set $csp_connect "connect-src 'self' https://api-iam.intercom.io https://api-iam.intercom.io https://api-ping.intercom.io https://app.posthog.com wss://nexus-websocket-a.intercom.io wss://nexus-websocket-b.intercom.io https://nexus-websocket-a.intercom.io https://nexus-websocket-b.intercom.io https://uploads.intercomcdn.com https://uploads.intercomusercontent.com https://*.s3.*.amazonaws.com"; + set $csp_connect "connect-src 'self' https://api-iam.intercom.io https://api-iam.intercom.io https://api-ping.intercom.io https://app.posthog.com wss://nexus-websocket-a.intercom.io wss://nexus-websocket-b.intercom.io https://nexus-websocket-a.intercom.io https://nexus-websocket-b.intercom.io https://uploads.intercomcdn.com https://uploads.intercomusercontent.com https://*.s3.us-east-2.amazonaws.com https://*.s3.us-east-1.amazonaws.com https://*.s3.us-west-1.amazonaws.com https://*.s3.us-west-2.amazonaws.com https://*.s3.af-south-1.amazonaws.com https://*.s3.ap-east-1.amazonaws.com https://*.s3.ap-southeast-3.amazonaws.com https://*.s3.ap-south-1.amazonaws.com https://*.s3.ap-northeast-3.amazonaws.com https://*.s3.ap-northeast-2.amazonaws.com https://*.s3.ap-southeast-1.amazonaws.com https://*.s3.ap-southeast-2.amazonaws.com https://*.s3.ap-northeast-1.amazonaws.com https://*.s3.ca-central-1.amazonaws.com https://*.s3.cn-north-1.amazonaws.com https://*.s3.cn-northwest-1.amazonaws.com https://*.s3.eu-central-1.amazonaws.com https://*.s3.eu-west-1.amazonaws.com https://*.s3.eu-west-2.amazonaws.com https://*.s3.eu-south-1.amazonaws.com https://*.s3.eu-west-3.amazonaws.com https://*.s3.eu-north-1.amazonaws.com https://*.s3.sa-east-1.amazonaws.com https://*.s3.me-south-1.amazonaws.com https://*.s3.us-gov-east-1.amazonaws.com https://*.s3.us-gov-west-1.amazonaws.com"; set $csp_font "font-src 'self' data: https://cdn.jsdelivr.net https://fonts.gstatic.com https://rsms.me https://maxcdn.bootstrapcdn.com https://js.intercomcdn.com https://fonts.intercomcdn.com"; set $csp_frame "frame-src 'self' https:"; set $csp_img "img-src http: https: data: blob:"; @@ -58,7 +58,7 @@ http { error_page 502 503 504 /error.html; location = /error.html { - root /usr/share/nginx/html; + root /usr/share/nginx/html; internal; } @@ -154,4 +154,4 @@ http { gzip_comp_level 6; gzip_types text/plain text/css text/xml application/json application/javascript application/rss+xml application/atom+xml image/svg+xml; } -} \ No newline at end of file +} diff --git a/lerna.json b/lerna.json index 708450c773..8f68c88cd1 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "1.0.142", + "version": "1.0.167-alpha.8", "npmClient": "yarn", "packages": [ "packages/*" diff --git a/package.json b/package.json index fb6d9da990..bd71dd6bc7 100644 --- a/package.json +++ b/package.json @@ -3,6 +3,8 @@ "private": true, "devDependencies": { "@rollup/plugin-json": "^4.0.2", + "@types/mongodb": "3.6.3", + "@typescript-eslint/parser": "4.28.0", "babel-eslint": "^10.0.3", "eslint": "^7.28.0", "eslint-plugin-cypress": "^2.11.3", @@ -16,7 +18,6 @@ "rimraf": "^3.0.2", "rollup-plugin-replace": "^2.2.0", "svelte": "^3.38.2", - "@typescript-eslint/parser": "4.28.0", "typescript": "4.5.5" }, "scripts": { @@ -48,6 +49,7 @@ "test:e2e": "lerna run cy:test --stream", "test:e2e:ci": "lerna run cy:ci --stream", "test:e2e:ci:record": "lerna run cy:ci:record --stream", + "test:e2e:ci:notify": "lerna run cy:ci:notify", "build:specs": "lerna run specs", "build:docker": "lerna run build:docker && npm run build:docker:proxy:compose && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh $BUDIBASE_RELEASE_VERSION && cd -", "build:docker:proxy": "docker build hosting/proxy -t proxy-service", diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json index e4a74eacc8..a6c8290fdf 100644 --- a/packages/backend-core/package.json +++ b/packages/backend-core/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/backend-core", - "version": "1.0.142", + "version": "1.0.167-alpha.8", "description": "Budibase backend core libraries used in server and worker", "main": "src/index.js", "types": "dist/src/index.d.ts", @@ -15,7 +15,8 @@ "@techpass/passport-openidconnect": "^0.3.0", "aws-sdk": "^2.901.0", "bcryptjs": "^2.4.3", - "cls-hooked": "^4.2.2", + "dotenv": "^16.0.1", + "emitter-listener": "^1.1.2", "ioredis": "^4.27.1", "jsonwebtoken": "^8.5.1", "koa-passport": "^4.1.4", diff --git a/packages/backend-core/src/clshooked/index.js b/packages/backend-core/src/clshooked/index.js new file mode 100644 index 0000000000..d69ffdd914 --- /dev/null +++ b/packages/backend-core/src/clshooked/index.js @@ -0,0 +1,650 @@ +const util = require("util") +const assert = require("assert") +const wrapEmitter = require("emitter-listener") +const async_hooks = require("async_hooks") + +const CONTEXTS_SYMBOL = "cls@contexts" +const ERROR_SYMBOL = "error@context" + +const DEBUG_CLS_HOOKED = process.env.DEBUG_CLS_HOOKED + +let currentUid = -1 + +module.exports = { + getNamespace: getNamespace, + createNamespace: createNamespace, + destroyNamespace: destroyNamespace, + reset: reset, + ERROR_SYMBOL: ERROR_SYMBOL, +} + +function Namespace(name) { + this.name = name + // changed in 2.7: no default context + this.active = null + this._set = [] + this.id = null + this._contexts = new Map() + this._indent = 0 + this._hook = null +} + +Namespace.prototype.set = function set(key, value) { + if (!this.active) { + throw new Error( + "No context available. ns.run() or ns.bind() must be called first." + ) + } + + this.active[key] = value + + if (DEBUG_CLS_HOOKED) { + const indentStr = " ".repeat(this._indent < 0 ? 0 : this._indent) + debug2( + indentStr + + "CONTEXT-SET KEY:" + + key + + "=" + + value + + " in ns:" + + this.name + + " currentUid:" + + currentUid + + " active:" + + util.inspect(this.active, { showHidden: true, depth: 2, colors: true }) + ) + } + + return value +} + +Namespace.prototype.get = function get(key) { + if (!this.active) { + if (DEBUG_CLS_HOOKED) { + const asyncHooksCurrentId = async_hooks.currentId() + const triggerId = async_hooks.triggerAsyncId() + const indentStr = " ".repeat(this._indent < 0 ? 0 : this._indent) + debug2( + `${indentStr}CONTEXT-GETTING KEY NO ACTIVE NS: (${this.name}) ${key}=undefined currentUid:${currentUid} asyncHooksCurrentId:${asyncHooksCurrentId} triggerId:${triggerId} len:${this._set.length}` + ) + } + return undefined + } + if (DEBUG_CLS_HOOKED) { + const asyncHooksCurrentId = async_hooks.executionAsyncId() + const triggerId = async_hooks.triggerAsyncId() + const indentStr = " ".repeat(this._indent < 0 ? 0 : this._indent) + debug2( + indentStr + + "CONTEXT-GETTING KEY:" + + key + + "=" + + this.active[key] + + " (" + + this.name + + ") currentUid:" + + currentUid + + " active:" + + util.inspect(this.active, { showHidden: true, depth: 2, colors: true }) + ) + debug2( + `${indentStr}CONTEXT-GETTING KEY: (${this.name}) ${key}=${ + this.active[key] + } currentUid:${currentUid} asyncHooksCurrentId:${asyncHooksCurrentId} triggerId:${triggerId} len:${ + this._set.length + } active:${util.inspect(this.active)}` + ) + } + return this.active[key] +} + +Namespace.prototype.createContext = function createContext() { + // Prototype inherit existing context if created a new child context within existing context. + let context = Object.create(this.active ? this.active : Object.prototype) + context._ns_name = this.name + context.id = currentUid + + if (DEBUG_CLS_HOOKED) { + const asyncHooksCurrentId = async_hooks.executionAsyncId() + const triggerId = async_hooks.triggerAsyncId() + const indentStr = " ".repeat(this._indent < 0 ? 0 : this._indent) + debug2( + `${indentStr}CONTEXT-CREATED Context: (${ + this.name + }) currentUid:${currentUid} asyncHooksCurrentId:${asyncHooksCurrentId} triggerId:${triggerId} len:${ + this._set.length + } context:${util.inspect(context, { + showHidden: true, + depth: 2, + colors: true, + })}` + ) + } + + return context +} + +Namespace.prototype.run = function run(fn) { + let context = this.createContext() + this.enter(context) + + try { + if (DEBUG_CLS_HOOKED) { + const triggerId = async_hooks.triggerAsyncId() + const asyncHooksCurrentId = async_hooks.executionAsyncId() + const indentStr = " ".repeat(this._indent < 0 ? 0 : this._indent) + debug2( + `${indentStr}CONTEXT-RUN BEGIN: (${ + this.name + }) currentUid:${currentUid} triggerId:${triggerId} asyncHooksCurrentId:${asyncHooksCurrentId} len:${ + this._set.length + } context:${util.inspect(context)}` + ) + } + fn(context) + return context + } catch (exception) { + if (exception) { + exception[ERROR_SYMBOL] = context + } + throw exception + } finally { + if (DEBUG_CLS_HOOKED) { + const triggerId = async_hooks.triggerAsyncId() + const asyncHooksCurrentId = async_hooks.executionAsyncId() + const indentStr = " ".repeat(this._indent < 0 ? 0 : this._indent) + debug2( + `${indentStr}CONTEXT-RUN END: (${ + this.name + }) currentUid:${currentUid} triggerId:${triggerId} asyncHooksCurrentId:${asyncHooksCurrentId} len:${ + this._set.length + } ${util.inspect(context)}` + ) + } + this.exit(context) + } +} + +Namespace.prototype.runAndReturn = function runAndReturn(fn) { + let value + this.run(function (context) { + value = fn(context) + }) + return value +} + +/** + * Uses global Promise and assumes Promise is cls friendly or wrapped already. + * @param {function} fn + * @returns {*} + */ +Namespace.prototype.runPromise = function runPromise(fn) { + let context = this.createContext() + this.enter(context) + + let promise = fn(context) + if (!promise || !promise.then || !promise.catch) { + throw new Error("fn must return a promise.") + } + + if (DEBUG_CLS_HOOKED) { + debug2( + "CONTEXT-runPromise BEFORE: (" + + this.name + + ") currentUid:" + + currentUid + + " len:" + + this._set.length + + " " + + util.inspect(context) + ) + } + + return promise + .then(result => { + if (DEBUG_CLS_HOOKED) { + debug2( + "CONTEXT-runPromise AFTER then: (" + + this.name + + ") currentUid:" + + currentUid + + " len:" + + this._set.length + + " " + + util.inspect(context) + ) + } + this.exit(context) + return result + }) + .catch(err => { + err[ERROR_SYMBOL] = context + if (DEBUG_CLS_HOOKED) { + debug2( + "CONTEXT-runPromise AFTER catch: (" + + this.name + + ") currentUid:" + + currentUid + + " len:" + + this._set.length + + " " + + util.inspect(context) + ) + } + this.exit(context) + throw err + }) +} + +Namespace.prototype.bind = function bindFactory(fn, context) { + if (!context) { + if (!this.active) { + context = this.createContext() + } else { + context = this.active + } + } + + let self = this + return function clsBind() { + self.enter(context) + try { + return fn.apply(this, arguments) + } catch (exception) { + if (exception) { + exception[ERROR_SYMBOL] = context + } + throw exception + } finally { + self.exit(context) + } + } +} + +Namespace.prototype.enter = function enter(context) { + assert.ok(context, "context must be provided for entering") + if (DEBUG_CLS_HOOKED) { + const asyncHooksCurrentId = async_hooks.executionAsyncId() + const triggerId = async_hooks.triggerAsyncId() + const indentStr = " ".repeat(this._indent < 0 ? 0 : this._indent) + debug2( + `${indentStr}CONTEXT-ENTER: (${ + this.name + }) currentUid:${currentUid} triggerId:${triggerId} asyncHooksCurrentId:${asyncHooksCurrentId} len:${ + this._set.length + } ${util.inspect(context)}` + ) + } + + this._set.push(this.active) + this.active = context +} + +Namespace.prototype.exit = function exit(context) { + assert.ok(context, "context must be provided for exiting") + if (DEBUG_CLS_HOOKED) { + const asyncHooksCurrentId = async_hooks.executionAsyncId() + const triggerId = async_hooks.triggerAsyncId() + const indentStr = " ".repeat(this._indent < 0 ? 0 : this._indent) + debug2( + `${indentStr}CONTEXT-EXIT: (${ + this.name + }) currentUid:${currentUid} triggerId:${triggerId} asyncHooksCurrentId:${asyncHooksCurrentId} len:${ + this._set.length + } ${util.inspect(context)}` + ) + } + + // Fast path for most exits that are at the top of the stack + if (this.active === context) { + assert.ok(this._set.length, "can't remove top context") + this.active = this._set.pop() + return + } + + // Fast search in the stack using lastIndexOf + let index = this._set.lastIndexOf(context) + + if (index < 0) { + if (DEBUG_CLS_HOOKED) { + debug2( + "??ERROR?? context exiting but not entered - ignoring: " + + util.inspect(context) + ) + } + assert.ok( + index >= 0, + "context not currently entered; can't exit. \n" + + util.inspect(this) + + "\n" + + util.inspect(context) + ) + } else { + assert.ok(index, "can't remove top context") + this._set.splice(index, 1) + } +} + +Namespace.prototype.bindEmitter = function bindEmitter(emitter) { + assert.ok( + emitter.on && emitter.addListener && emitter.emit, + "can only bind real EEs" + ) + + let namespace = this + let thisSymbol = "context@" + this.name + + // Capture the context active at the time the emitter is bound. + function attach(listener) { + if (!listener) { + return + } + if (!listener[CONTEXTS_SYMBOL]) { + listener[CONTEXTS_SYMBOL] = Object.create(null) + } + + listener[CONTEXTS_SYMBOL][thisSymbol] = { + namespace: namespace, + context: namespace.active, + } + } + + // At emit time, bind the listener within the correct context. + function bind(unwrapped) { + if (!(unwrapped && unwrapped[CONTEXTS_SYMBOL])) { + return unwrapped + } + + let wrapped = unwrapped + let unwrappedContexts = unwrapped[CONTEXTS_SYMBOL] + Object.keys(unwrappedContexts).forEach(function (name) { + let thunk = unwrappedContexts[name] + wrapped = thunk.namespace.bind(wrapped, thunk.context) + }) + return wrapped + } + + wrapEmitter(emitter, attach, bind) +} + +/** + * If an error comes out of a namespace, it will have a context attached to it. + * This function knows how to find it. + * + * @param {Error} exception Possibly annotated error. + */ +Namespace.prototype.fromException = function fromException(exception) { + return exception[ERROR_SYMBOL] +} + +function getNamespace(name) { + return process.namespaces[name] +} + +function createNamespace(name) { + assert.ok(name, "namespace must be given a name.") + + if (DEBUG_CLS_HOOKED) { + debug2(`NS-CREATING NAMESPACE (${name})`) + } + let namespace = new Namespace(name) + namespace.id = currentUid + + const hook = async_hooks.createHook({ + init(asyncId, type, triggerId, resource) { + currentUid = async_hooks.executionAsyncId() + + //CHAIN Parent's Context onto child if none exists. This is needed to pass net-events.spec + // let initContext = namespace.active; + // if(!initContext && triggerId) { + // let parentContext = namespace._contexts.get(triggerId); + // if (parentContext) { + // namespace.active = parentContext; + // namespace._contexts.set(currentUid, parentContext); + // if (DEBUG_CLS_HOOKED) { + // const indentStr = ' '.repeat(namespace._indent < 0 ? 0 : namespace._indent); + // debug2(`${indentStr}INIT [${type}] (${name}) WITH PARENT CONTEXT asyncId:${asyncId} currentUid:${currentUid} triggerId:${triggerId} active:${util.inspect(namespace.active, true)} resource:${resource}`); + // } + // } else if (DEBUG_CLS_HOOKED) { + // const indentStr = ' '.repeat(namespace._indent < 0 ? 0 : namespace._indent); + // debug2(`${indentStr}INIT [${type}] (${name}) MISSING CONTEXT asyncId:${asyncId} currentUid:${currentUid} triggerId:${triggerId} active:${util.inspect(namespace.active, true)} resource:${resource}`); + // } + // }else { + // namespace._contexts.set(currentUid, namespace.active); + // if (DEBUG_CLS_HOOKED) { + // const indentStr = ' '.repeat(namespace._indent < 0 ? 0 : namespace._indent); + // debug2(`${indentStr}INIT [${type}] (${name}) asyncId:${asyncId} currentUid:${currentUid} triggerId:${triggerId} active:${util.inspect(namespace.active, true)} resource:${resource}`); + // } + // } + if (namespace.active) { + namespace._contexts.set(asyncId, namespace.active) + + if (DEBUG_CLS_HOOKED) { + const indentStr = " ".repeat( + namespace._indent < 0 ? 0 : namespace._indent + ) + debug2( + `${indentStr}INIT [${type}] (${name}) asyncId:${asyncId} currentUid:${currentUid} triggerId:${triggerId} active:${util.inspect( + namespace.active, + { showHidden: true, depth: 2, colors: true } + )} resource:${resource}` + ) + } + } else if (currentUid === 0) { + // CurrentId will be 0 when triggered from C++. Promise events + // https://github.com/nodejs/node/blob/master/doc/api/async_hooks.md#triggerid + const triggerId = async_hooks.triggerAsyncId() + const triggerIdContext = namespace._contexts.get(triggerId) + if (triggerIdContext) { + namespace._contexts.set(asyncId, triggerIdContext) + if (DEBUG_CLS_HOOKED) { + const indentStr = " ".repeat( + namespace._indent < 0 ? 0 : namespace._indent + ) + debug2( + `${indentStr}INIT USING CONTEXT FROM TRIGGERID [${type}] (${name}) asyncId:${asyncId} currentUid:${currentUid} triggerId:${triggerId} active:${util.inspect( + namespace.active, + { showHidden: true, depth: 2, colors: true } + )} resource:${resource}` + ) + } + } else if (DEBUG_CLS_HOOKED) { + const indentStr = " ".repeat( + namespace._indent < 0 ? 0 : namespace._indent + ) + debug2( + `${indentStr}INIT MISSING CONTEXT [${type}] (${name}) asyncId:${asyncId} currentUid:${currentUid} triggerId:${triggerId} active:${util.inspect( + namespace.active, + { showHidden: true, depth: 2, colors: true } + )} resource:${resource}` + ) + } + } + + if (DEBUG_CLS_HOOKED && type === "PROMISE") { + debug2(util.inspect(resource, { showHidden: true })) + const parentId = resource.parentId + const indentStr = " ".repeat( + namespace._indent < 0 ? 0 : namespace._indent + ) + debug2( + `${indentStr}INIT RESOURCE-PROMISE [${type}] (${name}) parentId:${parentId} asyncId:${asyncId} currentUid:${currentUid} triggerId:${triggerId} active:${util.inspect( + namespace.active, + { showHidden: true, depth: 2, colors: true } + )} resource:${resource}` + ) + } + }, + before(asyncId) { + currentUid = async_hooks.executionAsyncId() + let context + + /* + if(currentUid === 0){ + // CurrentId will be 0 when triggered from C++. Promise events + // https://github.com/nodejs/node/blob/master/doc/api/async_hooks.md#triggerid + //const triggerId = async_hooks.triggerAsyncId(); + context = namespace._contexts.get(asyncId); // || namespace._contexts.get(triggerId); + }else{ + context = namespace._contexts.get(currentUid); + } + */ + + //HACK to work with promises until they are fixed in node > 8.1.1 + context = + namespace._contexts.get(asyncId) || namespace._contexts.get(currentUid) + + if (context) { + if (DEBUG_CLS_HOOKED) { + const triggerId = async_hooks.triggerAsyncId() + const indentStr = " ".repeat( + namespace._indent < 0 ? 0 : namespace._indent + ) + debug2( + `${indentStr}BEFORE (${name}) asyncId:${asyncId} currentUid:${currentUid} triggerId:${triggerId} active:${util.inspect( + namespace.active, + { showHidden: true, depth: 2, colors: true } + )} context:${util.inspect(context)}` + ) + namespace._indent += 2 + } + + namespace.enter(context) + } else if (DEBUG_CLS_HOOKED) { + const triggerId = async_hooks.triggerAsyncId() + const indentStr = " ".repeat( + namespace._indent < 0 ? 0 : namespace._indent + ) + debug2( + `${indentStr}BEFORE MISSING CONTEXT (${name}) asyncId:${asyncId} currentUid:${currentUid} triggerId:${triggerId} active:${util.inspect( + namespace.active, + { showHidden: true, depth: 2, colors: true } + )} namespace._contexts:${util.inspect(namespace._contexts, { + showHidden: true, + depth: 2, + colors: true, + })}` + ) + namespace._indent += 2 + } + }, + after(asyncId) { + currentUid = async_hooks.executionAsyncId() + let context // = namespace._contexts.get(currentUid); + /* + if(currentUid === 0){ + // CurrentId will be 0 when triggered from C++. Promise events + // https://github.com/nodejs/node/blob/master/doc/api/async_hooks.md#triggerid + //const triggerId = async_hooks.triggerAsyncId(); + context = namespace._contexts.get(asyncId); // || namespace._contexts.get(triggerId); + }else{ + context = namespace._contexts.get(currentUid); + } + */ + //HACK to work with promises until they are fixed in node > 8.1.1 + context = + namespace._contexts.get(asyncId) || namespace._contexts.get(currentUid) + + if (context) { + if (DEBUG_CLS_HOOKED) { + const triggerId = async_hooks.triggerAsyncId() + namespace._indent -= 2 + const indentStr = " ".repeat( + namespace._indent < 0 ? 0 : namespace._indent + ) + debug2( + `${indentStr}AFTER (${name}) asyncId:${asyncId} currentUid:${currentUid} triggerId:${triggerId} active:${util.inspect( + namespace.active, + { showHidden: true, depth: 2, colors: true } + )} context:${util.inspect(context)}` + ) + } + + namespace.exit(context) + } else if (DEBUG_CLS_HOOKED) { + const triggerId = async_hooks.triggerAsyncId() + namespace._indent -= 2 + const indentStr = " ".repeat( + namespace._indent < 0 ? 0 : namespace._indent + ) + debug2( + `${indentStr}AFTER MISSING CONTEXT (${name}) asyncId:${asyncId} currentUid:${currentUid} triggerId:${triggerId} active:${util.inspect( + namespace.active, + { showHidden: true, depth: 2, colors: true } + )} context:${util.inspect(context)}` + ) + } + }, + destroy(asyncId) { + currentUid = async_hooks.executionAsyncId() + if (DEBUG_CLS_HOOKED) { + const triggerId = async_hooks.triggerAsyncId() + const indentStr = " ".repeat( + namespace._indent < 0 ? 0 : namespace._indent + ) + debug2( + `${indentStr}DESTROY (${name}) currentUid:${currentUid} asyncId:${asyncId} triggerId:${triggerId} active:${util.inspect( + namespace.active, + { showHidden: true, depth: 2, colors: true } + )} context:${util.inspect(namespace._contexts.get(currentUid))}` + ) + } + + namespace._contexts.delete(asyncId) + }, + }) + + hook.enable() + namespace._hook = hook + + process.namespaces[name] = namespace + return namespace +} + +function destroyNamespace(name) { + let namespace = getNamespace(name) + + assert.ok(namespace, "can't delete nonexistent namespace! \"" + name + '"') + assert.ok( + namespace.id, + "don't assign to process.namespaces directly! " + util.inspect(namespace) + ) + + namespace._hook.disable() + namespace._contexts = null + process.namespaces[name] = null +} + +function reset() { + // must unregister async listeners + if (process.namespaces) { + Object.keys(process.namespaces).forEach(function (name) { + destroyNamespace(name) + }) + } + process.namespaces = Object.create(null) +} + +process.namespaces = process.namespaces || {} + +//const fs = require('fs'); +function debug2(...args) { + if (DEBUG_CLS_HOOKED) { + //fs.writeSync(1, `${util.format(...args)}\n`); + process._rawDebug(`${util.format(...args)}`) + } +} + +/*function getFunctionName(fn) { + if (!fn) { + return fn; + } + if (typeof fn === 'function') { + if (fn.name) { + return fn.name; + } + return (fn.toString().trim().match(/^function\s*([^\s(]+)/) || [])[1]; + } else if (fn.constructor && fn.constructor.name) { + return fn.constructor.name; + } +}*/ diff --git a/packages/backend-core/src/context/FunctionContext.js b/packages/backend-core/src/context/FunctionContext.js index 34d39492f9..c0ed34fe78 100644 --- a/packages/backend-core/src/context/FunctionContext.js +++ b/packages/backend-core/src/context/FunctionContext.js @@ -1,84 +1,47 @@ -const cls = require("cls-hooked") +const cls = require("../clshooked") const { newid } = require("../hashing") const REQUEST_ID_KEY = "requestId" +const MAIN_CTX = cls.createNamespace("main") + +function getContextStorage(namespace) { + if (namespace && namespace.active) { + let contextData = namespace.active + delete contextData.id + delete contextData._ns_name + return contextData + } + return {} +} class FunctionContext { - static getMiddleware( - updateCtxFn = null, - destroyFn = null, - contextName = "session" - ) { - const namespace = this.createNamespace(contextName) - - return async function (ctx, next) { - await new Promise( - namespace.bind(function (resolve, reject) { - // store a contextual request ID that can be used anywhere (audit logs) - namespace.set(REQUEST_ID_KEY, newid()) - namespace.bindEmitter(ctx.req) - namespace.bindEmitter(ctx.res) - - if (updateCtxFn) { - updateCtxFn(ctx) - } - next() - .then(resolve) - .catch(reject) - .finally(() => { - if (destroyFn) { - return destroyFn(ctx) - } - }) - }) - ) - } + static run(callback) { + return MAIN_CTX.runAndReturn(async () => { + const namespaceId = newid() + MAIN_CTX.set(REQUEST_ID_KEY, namespaceId) + const namespace = cls.createNamespace(namespaceId) + let response = await namespace.runAndReturn(callback) + cls.destroyNamespace(namespaceId) + return response + }) } - static run(callback, contextName = "session") { - const namespace = this.createNamespace(contextName) - - return namespace.runAndReturn(callback) - } - - static setOnContext(key, value, contextName = "session") { - const namespace = this.createNamespace(contextName) + static setOnContext(key, value) { + const namespaceId = MAIN_CTX.get(REQUEST_ID_KEY) + const namespace = cls.getNamespace(namespaceId) namespace.set(key, value) } - static getContextStorage() { - if (this._namespace && this._namespace.active) { - let contextData = this._namespace.active - delete contextData.id - delete contextData._ns_name - return contextData - } - - return {} - } - static getFromContext(key) { - const context = this.getContextStorage() + const namespaceId = MAIN_CTX.get(REQUEST_ID_KEY) + const namespace = cls.getNamespace(namespaceId) + const context = getContextStorage(namespace) if (context) { return context[key] } else { return null } } - - static destroyNamespace(name = "session") { - if (this._namespace) { - cls.destroyNamespace(name) - this._namespace = null - } - } - - static createNamespace(name = "session") { - if (!this._namespace) { - this._namespace = cls.createNamespace(name) - } - return this._namespace - } } module.exports = FunctionContext diff --git a/packages/backend-core/src/context/index.js b/packages/backend-core/src/context/index.js index b6b6f2380c..20e5e26693 100644 --- a/packages/backend-core/src/context/index.js +++ b/packages/backend-core/src/context/index.js @@ -55,6 +55,15 @@ async function closeAppDBs() { } } +exports.closeTenancy = async () => { + if (env.USE_COUCH) { + await closeDB(exports.getGlobalDB()) + } + // clear from context now that database is closed/task is finished + cls.setOnContext(ContextKeys.TENANT_ID, null) + cls.setOnContext(ContextKeys.GLOBAL_DB, null) +} + exports.isDefaultTenant = () => { return exports.getTenantId() === exports.DEFAULT_TENANT_ID } @@ -82,12 +91,7 @@ exports.doInTenant = (tenantId, task) => { } finally { const using = cls.getFromContext(ContextKeys.IN_USE) if (!using || using <= 1) { - if (env.USE_COUCH) { - await closeDB(exports.getGlobalDB()) - } - // clear from context now that database is closed/task is finished - cls.setOnContext(ContextKeys.TENANT_ID, null) - cls.setOnContext(ContextKeys.GLOBAL_DB, null) + await exports.closeTenancy() } else { cls.setOnContext(using - 1) } diff --git a/packages/backend-core/src/db/conversions.js b/packages/backend-core/src/db/conversions.js index 50d896322f..455cc712d8 100644 --- a/packages/backend-core/src/db/conversions.js +++ b/packages/backend-core/src/db/conversions.js @@ -23,24 +23,30 @@ exports.isDevApp = app => { } /** - * Convert a development app ID to a deployed app ID. + * Generates a development app ID from a real app ID. + * @returns {string} the dev app ID which can be used for dev database. */ -exports.getProdAppID = appId => { - // if dev, convert it - if (appId.startsWith(APP_DEV_PREFIX)) { - const id = appId.split(APP_DEV_PREFIX)[1] - return `${APP_PREFIX}${id}` +exports.getDevelopmentAppID = appId => { + if (!appId || appId.startsWith(APP_DEV_PREFIX)) { + return appId } - return appId + // split to take off the app_ element, then join it together incase any other app_ exist + const split = appId.split(APP_PREFIX) + split.shift() + const rest = split.join(APP_PREFIX) + return `${APP_DEV_PREFIX}${rest}` } /** - * Convert a deployed app ID to a development app ID. + * Convert a development app ID to a deployed app ID. */ -exports.getDevelopmentAppID = appId => { - if (!appId.startsWith(APP_DEV_PREFIX)) { - const id = appId.split(APP_PREFIX)[1] - return `${APP_DEV_PREFIX}${id}` +exports.getProdAppID = appId => { + if (!appId || !appId.startsWith(APP_DEV_PREFIX)) { + return appId } - return appId + // split to take off the app_dev element, then join it together incase any other app_ exist + const split = appId.split(APP_DEV_PREFIX) + split.shift() + const rest = split.join(APP_DEV_PREFIX) + return `${APP_PREFIX}${rest}` } diff --git a/packages/backend-core/src/db/tests/utils.spec.js b/packages/backend-core/src/db/tests/utils.spec.js new file mode 100644 index 0000000000..f8b9549d46 --- /dev/null +++ b/packages/backend-core/src/db/tests/utils.spec.js @@ -0,0 +1,194 @@ +require("../../tests/utilities/dbConfig"); +const { + generateAppID, + getDevelopmentAppID, + getProdAppID, + isDevAppID, + isProdAppID, + getPlatformUrl, + getScopedConfig +} = require("../utils") +const tenancy = require("../../tenancy"); +const { Configs, DEFAULT_TENANT_ID } = require("../../constants"); +const env = require("../../environment") + +describe("utils", () => { + describe("app ID manipulation", () => { + + function getID() { + const appId = generateAppID() + const split = appId.split("_") + const uuid = split[split.length - 1] + const devAppId = `app_dev_${uuid}` + return { appId, devAppId, split, uuid } + } + + it("should be able to generate a new app ID", () => { + expect(generateAppID().startsWith("app_")).toEqual(true) + }) + + it("should be able to convert a production app ID to development", () => { + const { appId, uuid } = getID() + expect(getDevelopmentAppID(appId)).toEqual(`app_dev_${uuid}`) + }) + + it("should be able to convert a development app ID to development", () => { + const { devAppId, uuid } = getID() + expect(getDevelopmentAppID(devAppId)).toEqual(`app_dev_${uuid}`) + }) + + it("should be able to convert a development ID to a production", () => { + const { devAppId, uuid } = getID() + expect(getProdAppID(devAppId)).toEqual(`app_${uuid}`) + }) + + it("should be able to convert a production ID to production", () => { + const { appId, uuid } = getID() + expect(getProdAppID(appId)).toEqual(`app_${uuid}`) + }) + + it("should be able to confirm dev app ID is development", () => { + const { devAppId } = getID() + expect(isDevAppID(devAppId)).toEqual(true) + }) + + it("should be able to confirm prod app ID is not development", () => { + const { appId } = getID() + expect(isDevAppID(appId)).toEqual(false) + }) + + it("should be able to confirm prod app ID is prod", () => { + const { appId } = getID() + expect(isProdAppID(appId)).toEqual(true) + }) + + it("should be able to confirm dev app ID is not prod", () => { + const { devAppId } = getID() + expect(isProdAppID(devAppId)).toEqual(false) + }) + }) +}) + +const DB_URL = "http://dburl.com" +const DEFAULT_URL = "http://localhost:10000" +const ENV_URL = "http://env.com" + +const setDbPlatformUrl = async () => { + const db = tenancy.getGlobalDB() + db.put({ + _id: "config_settings", + type: Configs.SETTINGS, + config: { + platformUrl: DB_URL + } + }) +} + +const clearSettingsConfig = async () => { + await tenancy.doInTenant(DEFAULT_TENANT_ID, async () => { + const db = tenancy.getGlobalDB() + try { + const config = await db.get("config_settings") + await db.remove("config_settings", config._rev) + } catch (e) { + if (e.status !== 404) { + throw e + } + } + }) +} + +describe("getPlatformUrl", () => { + describe("self host", () => { + + beforeEach(async () => { + env._set("SELF_HOST", 1) + await clearSettingsConfig() + }) + + it("gets the default url", async () => { + await tenancy.doInTenant(null, async () => { + const url = await getPlatformUrl() + expect(url).toBe(DEFAULT_URL) + }) + }) + + it("gets the platform url from the environment", async () => { + await tenancy.doInTenant(null, async () => { + env._set("PLATFORM_URL", ENV_URL) + const url = await getPlatformUrl() + expect(url).toBe(ENV_URL) + }) + }) + + it("gets the platform url from the database", async () => { + await tenancy.doInTenant(null, async () => { + await setDbPlatformUrl() + const url = await getPlatformUrl() + expect(url).toBe(DB_URL) + }) + }) + }) + + + describe("cloud", () => { + const TENANT_AWARE_URL = "http://default.env.com" + + beforeEach(async () => { + env._set("SELF_HOSTED", 0) + env._set("MULTI_TENANCY", 1) + env._set("PLATFORM_URL", ENV_URL) + await clearSettingsConfig() + }) + + it("gets the platform url from the environment without tenancy", async () => { + await tenancy.doInTenant(DEFAULT_TENANT_ID, async () => { + const url = await getPlatformUrl({ tenantAware: false }) + expect(url).toBe(ENV_URL) + }) + }) + + it("gets the platform url from the environment with tenancy", async () => { + await tenancy.doInTenant(DEFAULT_TENANT_ID, async () => { + const url = await getPlatformUrl() + expect(url).toBe(TENANT_AWARE_URL) + }) + }) + + it("never gets the platform url from the database", async () => { + await tenancy.doInTenant(DEFAULT_TENANT_ID, async () => { + await setDbPlatformUrl() + const url = await getPlatformUrl() + expect(url).toBe(TENANT_AWARE_URL) + }) + }) + }) +}) + +describe("getScopedConfig", () => { + describe("settings config", () => { + + beforeEach(async () => { + env._set("SELF_HOSTED", 1) + env._set("PLATFORM_URL", "") + await clearSettingsConfig() + }) + + it("returns the platform url with an existing config", async () => { + await tenancy.doInTenant(DEFAULT_TENANT_ID, async () => { + await setDbPlatformUrl() + const db = tenancy.getGlobalDB() + const config = await getScopedConfig(db, { type: Configs.SETTINGS }) + expect(config.platformUrl).toBe(DB_URL) + }) + }) + + it("returns the platform url without an existing config", async () => { + await tenancy.doInTenant(DEFAULT_TENANT_ID, async () => { + const db = tenancy.getGlobalDB() + const config = await getScopedConfig(db, { type: Configs.SETTINGS }) + expect(config.platformUrl).toBe(DEFAULT_URL) + }) + }) + }) +}) diff --git a/packages/backend-core/src/db/utils.ts b/packages/backend-core/src/db/utils.ts index 9223d38189..e7ec6a5f03 100644 --- a/packages/backend-core/src/db/utils.ts +++ b/packages/backend-core/src/db/utils.ts @@ -2,13 +2,14 @@ import { newid } from "../hashing" import { DEFAULT_TENANT_ID, Configs } from "../constants" import * as env from "../environment" import { SEPARATOR, DocumentTypes } from "./constants" -import { getTenantId, getGlobalDBName } from "../tenancy" +import { getTenantId, getGlobalDBName, getGlobalDB } from "../tenancy" import fetch from "node-fetch" import { doWithDB, allDbs } from "./index" import { getCouchInfo } from "./pouch" import { getAppMetadata } from "../cache/appMetadata" import { checkSlashesInUrl } from "../helpers" import { isDevApp, isDevAppID } from "./conversions" +import { APP_PREFIX } from "./constants" const UNICODE_MAX = "\ufff0" @@ -22,6 +23,18 @@ export * from "./constants" export * from "./conversions" export { default as Replication } from "./Replication" +/** + * Generates a new app ID. + * @returns {string} The new app ID which the app doc can be stored under. + */ +export const generateAppID = (tenantId = null) => { + let id = APP_PREFIX + if (tenantId) { + id += `${tenantId}${SEPARATOR}` + } + return `${id}${newid()}` +} + /** * If creating DB allDocs/query params with only a single top level ID this can be used, this * is usually the case as most of our docs are top level e.g. tables, automations, users and so on. @@ -370,9 +383,7 @@ export const getScopedFullConfig = async function ( // always provide the platform URL if (type === Configs.SETTINGS) { if (scopedConfig && scopedConfig.doc) { - scopedConfig.doc.config.platformUrl = await getPlatformUrl( - scopedConfig.doc.config - ) + scopedConfig.doc.config.platformUrl = await getPlatformUrl() } else { scopedConfig = { doc: { @@ -387,19 +398,30 @@ export const getScopedFullConfig = async function ( return scopedConfig && scopedConfig.doc } -export const getPlatformUrl = async (settings?: any) => { +const getPlatformUrl = async (opts = { tenantAware: true }) => { let platformUrl = env.PLATFORM_URL || "http://localhost:10000" - if (!env.SELF_HOSTED && env.MULTI_TENANCY) { + if (!env.SELF_HOSTED && env.MULTI_TENANCY && opts.tenantAware) { // cloud and multi tenant - add the tenant to the default platform url const tenantId = getTenantId() if (!platformUrl.includes("localhost:")) { platformUrl = platformUrl.replace("://", `://${tenantId}.`) } - } else { + } else if (env.SELF_HOSTED) { + const db = getGlobalDB() + // get the doc directly instead of with getScopedConfig to prevent loop + let settings + try { + settings = await db.get(generateConfigID({ type: Configs.SETTINGS })) + } catch (e) { + if (e.status !== 404) { + throw e + } + } + // self hosted - check for platform url override - if (settings && settings.platformUrl) { - platformUrl = settings.platformUrl + if (settings && settings.config && settings.config.platformUrl) { + platformUrl = settings.config.platformUrl } } diff --git a/packages/backend-core/src/environment.js b/packages/backend-core/src/environment.js index e54bbaf875..5d355d97c0 100644 --- a/packages/backend-core/src/environment.js +++ b/packages/backend-core/src/environment.js @@ -10,7 +10,15 @@ function isDev() { return process.env.NODE_ENV !== "production" } +let LOADED = false +if (!LOADED && isDev() && !isTest()) { + require("dotenv").config() + LOADED = true +} + module.exports = { + isTest, + isDev, JWT_SECRET: process.env.JWT_SECRET, COUCH_DB_URL: process.env.COUCH_DB_URL || "http://localhost:4005", COUCH_DB_USERNAME: process.env.COUCH_DB_USER, @@ -36,9 +44,14 @@ module.exports = { POSTHOG_TOKEN: process.env.POSTHOG_TOKEN, ENABLE_ANALYTICS: process.env.ENABLE_ANALYTICS, TENANT_FEATURE_FLAGS: process.env.TENANT_FEATURE_FLAGS, + BACKUPS_BUCKET_NAME: process.env.BACKUPS_BUCKET_NAME || "backups", + APPS_BUCKET_NAME: process.env.APPS_BUCKET_NAME || "prod-budi-app-assets", + TEMPLATES_BUCKET_NAME: process.env.TEMPLATES_BUCKET_NAME || "templates", + GLOBAL_BUCKET_NAME: process.env.GLOBAL_BUCKET_NAME || "global", + GLOBAL_CLOUD_BUCKET_NAME: + process.env.GLOBAL_CLOUD_BUCKET_NAME || "prod-budi-tenant-uploads", USE_COUCH: process.env.USE_COUCH || true, - isTest, - isDev, + DISABLE_DEVELOPER_LICENSE: process.env.DISABLE_DEVELOPER_LICENSE, _set(key, value) { process.env[key] = value module.exports[key] = value diff --git a/packages/backend-core/src/featureFlags/index.js b/packages/backend-core/src/featureFlags/index.js index 6d3d86978a..c050cbdfef 100644 --- a/packages/backend-core/src/featureFlags/index.js +++ b/packages/backend-core/src/featureFlags/index.js @@ -49,4 +49,5 @@ exports.getTenantFeatureFlags = tenantId => { exports.FeatureFlag = { LICENSING: "LICENSING", + GOOGLE_SHEETS: "GOOGLE_SHEETS", } diff --git a/packages/backend-core/src/index.js b/packages/backend-core/src/index.js index 2752312c45..77eebd2d7d 100644 --- a/packages/backend-core/src/index.js +++ b/packages/backend-core/src/index.js @@ -23,6 +23,7 @@ module.exports = { env: require("./environment"), accounts: require("./cloud/accounts"), tenancy: require("./tenancy"), + context: require("../context"), featureFlags: require("./featureFlags"), events, analytics: require("./analytics"), diff --git a/packages/backend-core/src/middleware/passport/datasource/google.js b/packages/backend-core/src/middleware/passport/datasource/google.js index 96c7f99953..53719b8350 100644 --- a/packages/backend-core/src/middleware/passport/datasource/google.js +++ b/packages/backend-core/src/middleware/passport/datasource/google.js @@ -21,20 +21,12 @@ async function fetchGoogleCreds() { ) } -async function platformUrl() { - const db = getGlobalDB() - const publicConfig = await getScopedConfig(db, { - type: Configs.SETTINGS, - }) - return getPlatformUrl(publicConfig) -} - async function preAuth(passport, ctx, next) { // get the relevant config const googleConfig = await fetchGoogleCreds() - const platUrl = await platformUrl() + const platformUrl = await getPlatformUrl({ tenantAware: false }) - let callbackUrl = `${platUrl}/api/global/auth/datasource/google/callback` + let callbackUrl = `${platformUrl}/api/global/auth/datasource/google/callback` const strategy = await google.strategyFactory(googleConfig, callbackUrl) if (!ctx.query.appId || !ctx.query.datasourceId) { @@ -51,9 +43,9 @@ async function preAuth(passport, ctx, next) { async function postAuth(passport, ctx, next) { // get the relevant config const config = await fetchGoogleCreds() - const platUrl = await platformUrl() + const platformUrl = await getPlatformUrl({ tenantAware: false }) - let callbackUrl = `${platUrl}/api/global/auth/datasource/google/callback` + let callbackUrl = `${platformUrl}/api/global/auth/datasource/google/callback` const strategy = await google.strategyFactory( config, callbackUrl, diff --git a/packages/backend-core/src/middleware/passport/tests/oidc.spec.js b/packages/backend-core/src/middleware/passport/tests/oidc.spec.js index bfe9f97dc0..c5e9fe0034 100644 --- a/packages/backend-core/src/middleware/passport/tests/oidc.spec.js +++ b/packages/backend-core/src/middleware/passport/tests/oidc.spec.js @@ -71,7 +71,7 @@ describe("oidc", () => { describe("authenticate", () => { afterEach(() => { - jest.clearAllMocks(); + jest.clearAllMocks() }); // mock third party common authentication @@ -80,10 +80,10 @@ describe("oidc", () => { // mock the passport callback const mockDone = jest.fn() + const mockSaveUserFn = jest.fn() async function doAuthenticate() { const oidc = require("../oidc") - const mockSaveUserFn = jest.fn() const authenticate = await oidc.buildVerifyFn(mockSaveUserFn) await authenticate( @@ -105,11 +105,13 @@ describe("oidc", () => { expect(authenticateThirdParty).toHaveBeenCalledWith( user, false, - mockDone) + mockDone, + mockSaveUserFn, + ) } it("delegates authentication to third party common", async () => { - doTest() + await doTest() }) it("uses JWT email to get email", async () => { @@ -118,7 +120,7 @@ describe("oidc", () => { email : "mock@budibase.com" } - doTest() + await doTest() }) it("uses JWT username to get email", async () => { @@ -127,7 +129,7 @@ describe("oidc", () => { preferred_username : "mock@budibase.com" } - doTest() + await doTest() }) it("uses JWT invalid username to get email", async () => { diff --git a/packages/backend-core/src/middleware/tenancy.js b/packages/backend-core/src/middleware/tenancy.js index f4053d1f5b..9a0cb8a0c6 100644 --- a/packages/backend-core/src/middleware/tenancy.js +++ b/packages/backend-core/src/middleware/tenancy.js @@ -1,6 +1,5 @@ -const { setTenantId, setGlobalDB, getGlobalDB } = require("../tenancy") -const { closeDB } = require("../db") -const ContextFactory = require("../context/FunctionContext") +const { setTenantId, setGlobalDB, closeTenancy } = require("../tenancy") +const cls = require("../context/FunctionContext") const { buildMatcherRegex, matches } = require("./matchers") module.exports = ( @@ -11,17 +10,16 @@ module.exports = ( const allowQsOptions = buildMatcherRegex(allowQueryStringPatterns) const noTenancyOptions = buildMatcherRegex(noTenancyPatterns) - const updateCtxFn = ctx => { - const allowNoTenant = - opts.noTenancyRequired || !!matches(ctx, noTenancyOptions) - const allowQs = !!matches(ctx, allowQsOptions) - const tenantId = setTenantId(ctx, { allowQs, allowNoTenant }) - setGlobalDB(tenantId) + return async function (ctx, next) { + return cls.run(async () => { + const allowNoTenant = + opts.noTenancyRequired || !!matches(ctx, noTenancyOptions) + const allowQs = !!matches(ctx, allowQsOptions) + const tenantId = setTenantId(ctx, { allowQs, allowNoTenant }) + setGlobalDB(tenantId) + const res = await next() + await closeTenancy() + return res + }) } - const destroyFn = async () => { - const db = getGlobalDB() - await closeDB(db) - } - - return ContextFactory.getMiddleware(updateCtxFn, destroyFn) } diff --git a/packages/backend-core/src/objectStore/utils.js b/packages/backend-core/src/objectStore/utils.js index 1634a24981..a243553df8 100644 --- a/packages/backend-core/src/objectStore/utils.js +++ b/packages/backend-core/src/objectStore/utils.js @@ -1,12 +1,13 @@ const { join } = require("path") const { tmpdir } = require("os") +const env = require("../environment") exports.ObjectStoreBuckets = { - BACKUPS: "backups", - APPS: "prod-budi-app-assets", - TEMPLATES: "templates", - GLOBAL: "global", - GLOBAL_CLOUD: "prod-budi-tenant-uploads", + BACKUPS: env.BACKUPS_BUCKET_NAME, + APPS: env.APPS_BUCKET_NAME, + TEMPLATES: env.TEMPLATES_BUCKET_NAME, + GLOBAL: env.GLOBAL_BUCKET_NAME, + GLOBAL_CLOUD: env.GLOBAL_CLOUD_BUCKET_NAME, } exports.budibaseTempDir = function () { diff --git a/packages/backend-core/src/security/permissions.js b/packages/backend-core/src/security/permissions.js index 28044a5129..2ecb8a9f1e 100644 --- a/packages/backend-core/src/security/permissions.js +++ b/packages/backend-core/src/security/permissions.js @@ -96,6 +96,7 @@ const BUILTIN_PERMISSIONS = { new Permission(PermissionTypes.QUERY, PermissionLevels.WRITE), new Permission(PermissionTypes.TABLE, PermissionLevels.WRITE), new Permission(PermissionTypes.VIEW, PermissionLevels.READ), + new Permission(PermissionTypes.AUTOMATION, PermissionLevels.EXECUTE), ], }, POWER: { diff --git a/packages/backend-core/yarn.lock b/packages/backend-core/yarn.lock index 0971ca25f0..1711f16d55 100644 --- a/packages/backend-core/yarn.lock +++ b/packages/backend-core/yarn.lock @@ -847,13 +847,6 @@ ast-types@0.9.6: resolved "https://registry.yarnpkg.com/ast-types/-/ast-types-0.9.6.tgz#102c9e9e9005d3e7e3829bf0c4fa24ee862ee9b9" integrity sha1-ECyenpAF0+fjgpvwxPok7oYu6bk= -async-hook-jl@^1.7.6: - version "1.7.6" - resolved "https://registry.yarnpkg.com/async-hook-jl/-/async-hook-jl-1.7.6.tgz#4fd25c2f864dbaf279c610d73bf97b1b28595e68" - integrity sha512-gFaHkFfSxTjvoxDMYqDuGHlcRyUuamF8s+ZTtJdDzqjws4mCt7v0vuV79/E2Wr2/riMQgtG4/yUtXWs1gZ7JMg== - dependencies: - stack-chain "^1.3.7" - async@~2.1.4: version "2.1.5" resolved "https://registry.yarnpkg.com/async/-/async-2.1.5.tgz#e587c68580994ac67fc56ff86d3ac56bdbe810bc" @@ -1255,15 +1248,6 @@ clone-buffer@1.0.0: resolved "https://registry.yarnpkg.com/clone-buffer/-/clone-buffer-1.0.0.tgz#e3e25b207ac4e701af721e2cb5a16792cac3dc58" integrity sha1-4+JbIHrE5wGvch4staFnksrD3Fg= -cls-hooked@^4.2.2: - version "4.2.2" - resolved "https://registry.yarnpkg.com/cls-hooked/-/cls-hooked-4.2.2.tgz#ad2e9a4092680cdaffeb2d3551da0e225eae1908" - integrity sha512-J4Xj5f5wq/4jAvcdgoGsL3G103BtWpZrMo8NEinRltN+xpTZdI+M38pyQqhuFU/P792xkMFvnKSf+Lm81U1bxw== - dependencies: - async-hook-jl "^1.7.6" - emitter-listener "^1.0.1" - semver "^5.4.1" - cluster-key-slot@^1.1.0: version "1.1.0" resolved "https://registry.yarnpkg.com/cluster-key-slot/-/cluster-key-slot-1.1.0.tgz#30474b2a981fb12172695833052bc0d01336d10d" @@ -1608,6 +1592,11 @@ domexception@^2.0.1: dependencies: webidl-conversions "^5.0.0" +dotenv@^16.0.1: + version "16.0.1" + resolved "https://registry.yarnpkg.com/dotenv/-/dotenv-16.0.1.tgz#8f8f9d94876c35dac989876a5d3a82a267fdce1d" + integrity sha512-1K6hR6wtk2FviQ4kEiSjFiH5rpzEVi8WW0x96aztHVMhEspNpc4DVOUTEHtEva5VThQ8IaBX1Pe4gSzpVVUsKQ== + double-ended-queue@2.1.0-0: version "2.1.0-0" resolved "https://registry.yarnpkg.com/double-ended-queue/-/double-ended-queue-2.1.0-0.tgz#103d3527fd31528f40188130c841efdd78264e5c" @@ -1645,7 +1634,7 @@ electron-to-chromium@^1.3.896: resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.3.900.tgz#5be2c5818a2a012c511b4b43e87b6ab7a296d4f5" integrity sha512-SuXbQD8D4EjsaBaJJxySHbC+zq8JrFfxtb4GIr4E9n1BcROyMcRrJCYQNpJ9N+Wjf5mFp7Wp0OHykd14JNEzzQ== -emitter-listener@^1.0.1: +emitter-listener@^1.1.2: version "1.1.2" resolved "https://registry.yarnpkg.com/emitter-listener/-/emitter-listener-1.1.2.tgz#56b140e8f6992375b3d7cb2cab1cc7432d9632e8" integrity sha512-Bt1sBAGFHY9DKY+4/2cV6izcKJUf5T7/gkdmkxzX/qv9CcGH8xSwVRW5mtX03SWJtRTWSOpzCuWN9rBFYZepZQ== @@ -4789,7 +4778,7 @@ saxes@^5.0.1: dependencies: xmlchars "^2.2.0" -"semver@2 || 3 || 4 || 5", semver@^5.4.1, semver@^5.5.0, semver@^5.6.0: +"semver@2 || 3 || 4 || 5", semver@^5.5.0, semver@^5.6.0: version "5.7.1" resolved "https://registry.yarnpkg.com/semver/-/semver-5.7.1.tgz#a954f931aeba508d307bbf069eff0c01c96116f7" integrity sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ== @@ -5034,11 +5023,6 @@ sshpk@^1.7.0: jsbn "~0.1.0" tweetnacl "~0.14.0" -stack-chain@^1.3.7: - version "1.3.7" - resolved "https://registry.yarnpkg.com/stack-chain/-/stack-chain-1.3.7.tgz#d192c9ff4ea6a22c94c4dd459171e3f00cea1285" - integrity sha1-0ZLJ/06moiyUxN1FkXHj8AzqEoU= - stack-utils@^2.0.2: version "2.0.5" resolved "https://registry.yarnpkg.com/stack-utils/-/stack-utils-2.0.5.tgz#d25265fca995154659dbbfba3b49254778d2fdd5" diff --git a/packages/bbui/package.json b/packages/bbui/package.json index 6dfaaf6975..86c1258e69 100644 --- a/packages/bbui/package.json +++ b/packages/bbui/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/bbui", "description": "A UI solution used in the different Budibase projects.", - "version": "1.0.142", + "version": "1.0.167-alpha.8", "license": "MPL-2.0", "svelte": "src/index.js", "module": "dist/bbui.es.js", @@ -38,7 +38,7 @@ ], "dependencies": { "@adobe/spectrum-css-workflow-icons": "^1.2.1", - "@budibase/string-templates": "^1.0.142", + "@budibase/string-templates": "^1.0.167-alpha.8", "@spectrum-css/actionbutton": "^1.0.1", "@spectrum-css/actiongroup": "^1.0.1", "@spectrum-css/avatar": "^3.0.2", diff --git a/packages/bbui/src/Form/Core/CheckboxGroup.svelte b/packages/bbui/src/Form/Core/CheckboxGroup.svelte new file mode 100644 index 0000000000..640d5d99cd --- /dev/null +++ b/packages/bbui/src/Form/Core/CheckboxGroup.svelte @@ -0,0 +1,68 @@ + + +