diff --git a/.github/workflows/release-develop.yml b/.github/workflows/release-develop.yml index 7ee02ccdd1..d4050ab40e 100644 --- a/.github/workflows/release-develop.yml +++ b/.github/workflows/release-develop.yml @@ -1,4 +1,5 @@ name: Budibase Release Staging +concurrency: release-develop on: push: diff --git a/.github/workflows/release-selfhost.yml b/.github/workflows/release-selfhost.yml index 9f42a9cc5d..fc2b7b0cca 100644 --- a/.github/workflows/release-selfhost.yml +++ b/.github/workflows/release-selfhost.yml @@ -87,3 +87,10 @@ jobs: packages/cli/build/cli-macos packages/server/specs/openapi.yaml packages/server/specs/openapi.json + + - name: Discord Webhook Action + uses: tsickert/discord-webhook@v4.0.0 + with: + webhook-url: ${{ secrets.PROD_DEPLOY_WEBHOOK_URL }} + content: "Self Host Deployment Complete: ${{ env.RELEASE_VERSION }} deployed to Self Host." + embed-title: ${{ env.RELEASE_VERSION }} \ No newline at end of file diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 359ad4467b..fa3aaf28e9 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,4 +1,5 @@ name: Budibase Release +concurrency: release on: push: diff --git a/lerna.json b/lerna.json index dd50fdf909..e767606692 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "1.0.151-alpha.2", + "version": "1.0.159-alpha.1", "npmClient": "yarn", "packages": [ "packages/*" diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json index 858588c0ee..3f215537b3 100644 --- a/packages/backend-core/package.json +++ b/packages/backend-core/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/backend-core", - "version": "1.0.151-alpha.2", + "version": "1.0.159-alpha.1", "description": "Budibase backend core libraries used in server and worker", "main": "src/index.js", "author": "Budibase", @@ -13,7 +13,7 @@ "@techpass/passport-openidconnect": "^0.3.0", "aws-sdk": "^2.901.0", "bcryptjs": "^2.4.3", - "cls-hooked": "^4.2.2", + "emitter-listener": "^1.1.2", "ioredis": "^4.27.1", "jsonwebtoken": "^8.5.1", "koa-passport": "^4.1.4", diff --git a/packages/backend-core/src/clshooked/index.js b/packages/backend-core/src/clshooked/index.js new file mode 100644 index 0000000000..d69ffdd914 --- /dev/null +++ b/packages/backend-core/src/clshooked/index.js @@ -0,0 +1,650 @@ +const util = require("util") +const assert = require("assert") +const wrapEmitter = require("emitter-listener") +const async_hooks = require("async_hooks") + +const CONTEXTS_SYMBOL = "cls@contexts" +const ERROR_SYMBOL = "error@context" + +const DEBUG_CLS_HOOKED = process.env.DEBUG_CLS_HOOKED + +let currentUid = -1 + +module.exports = { + getNamespace: getNamespace, + createNamespace: createNamespace, + destroyNamespace: destroyNamespace, + reset: reset, + ERROR_SYMBOL: ERROR_SYMBOL, +} + +function Namespace(name) { + this.name = name + // changed in 2.7: no default context + this.active = null + this._set = [] + this.id = null + this._contexts = new Map() + this._indent = 0 + this._hook = null +} + +Namespace.prototype.set = function set(key, value) { + if (!this.active) { + throw new Error( + "No context available. ns.run() or ns.bind() must be called first." + ) + } + + this.active[key] = value + + if (DEBUG_CLS_HOOKED) { + const indentStr = " ".repeat(this._indent < 0 ? 0 : this._indent) + debug2( + indentStr + + "CONTEXT-SET KEY:" + + key + + "=" + + value + + " in ns:" + + this.name + + " currentUid:" + + currentUid + + " active:" + + util.inspect(this.active, { showHidden: true, depth: 2, colors: true }) + ) + } + + return value +} + +Namespace.prototype.get = function get(key) { + if (!this.active) { + if (DEBUG_CLS_HOOKED) { + const asyncHooksCurrentId = async_hooks.currentId() + const triggerId = async_hooks.triggerAsyncId() + const indentStr = " ".repeat(this._indent < 0 ? 0 : this._indent) + debug2( + `${indentStr}CONTEXT-GETTING KEY NO ACTIVE NS: (${this.name}) ${key}=undefined currentUid:${currentUid} asyncHooksCurrentId:${asyncHooksCurrentId} triggerId:${triggerId} len:${this._set.length}` + ) + } + return undefined + } + if (DEBUG_CLS_HOOKED) { + const asyncHooksCurrentId = async_hooks.executionAsyncId() + const triggerId = async_hooks.triggerAsyncId() + const indentStr = " ".repeat(this._indent < 0 ? 0 : this._indent) + debug2( + indentStr + + "CONTEXT-GETTING KEY:" + + key + + "=" + + this.active[key] + + " (" + + this.name + + ") currentUid:" + + currentUid + + " active:" + + util.inspect(this.active, { showHidden: true, depth: 2, colors: true }) + ) + debug2( + `${indentStr}CONTEXT-GETTING KEY: (${this.name}) ${key}=${ + this.active[key] + } currentUid:${currentUid} asyncHooksCurrentId:${asyncHooksCurrentId} triggerId:${triggerId} len:${ + this._set.length + } active:${util.inspect(this.active)}` + ) + } + return this.active[key] +} + +Namespace.prototype.createContext = function createContext() { + // Prototype inherit existing context if created a new child context within existing context. + let context = Object.create(this.active ? this.active : Object.prototype) + context._ns_name = this.name + context.id = currentUid + + if (DEBUG_CLS_HOOKED) { + const asyncHooksCurrentId = async_hooks.executionAsyncId() + const triggerId = async_hooks.triggerAsyncId() + const indentStr = " ".repeat(this._indent < 0 ? 0 : this._indent) + debug2( + `${indentStr}CONTEXT-CREATED Context: (${ + this.name + }) currentUid:${currentUid} asyncHooksCurrentId:${asyncHooksCurrentId} triggerId:${triggerId} len:${ + this._set.length + } context:${util.inspect(context, { + showHidden: true, + depth: 2, + colors: true, + })}` + ) + } + + return context +} + +Namespace.prototype.run = function run(fn) { + let context = this.createContext() + this.enter(context) + + try { + if (DEBUG_CLS_HOOKED) { + const triggerId = async_hooks.triggerAsyncId() + const asyncHooksCurrentId = async_hooks.executionAsyncId() + const indentStr = " ".repeat(this._indent < 0 ? 0 : this._indent) + debug2( + `${indentStr}CONTEXT-RUN BEGIN: (${ + this.name + }) currentUid:${currentUid} triggerId:${triggerId} asyncHooksCurrentId:${asyncHooksCurrentId} len:${ + this._set.length + } context:${util.inspect(context)}` + ) + } + fn(context) + return context + } catch (exception) { + if (exception) { + exception[ERROR_SYMBOL] = context + } + throw exception + } finally { + if (DEBUG_CLS_HOOKED) { + const triggerId = async_hooks.triggerAsyncId() + const asyncHooksCurrentId = async_hooks.executionAsyncId() + const indentStr = " ".repeat(this._indent < 0 ? 0 : this._indent) + debug2( + `${indentStr}CONTEXT-RUN END: (${ + this.name + }) currentUid:${currentUid} triggerId:${triggerId} asyncHooksCurrentId:${asyncHooksCurrentId} len:${ + this._set.length + } ${util.inspect(context)}` + ) + } + this.exit(context) + } +} + +Namespace.prototype.runAndReturn = function runAndReturn(fn) { + let value + this.run(function (context) { + value = fn(context) + }) + return value +} + +/** + * Uses global Promise and assumes Promise is cls friendly or wrapped already. + * @param {function} fn + * @returns {*} + */ +Namespace.prototype.runPromise = function runPromise(fn) { + let context = this.createContext() + this.enter(context) + + let promise = fn(context) + if (!promise || !promise.then || !promise.catch) { + throw new Error("fn must return a promise.") + } + + if (DEBUG_CLS_HOOKED) { + debug2( + "CONTEXT-runPromise BEFORE: (" + + this.name + + ") currentUid:" + + currentUid + + " len:" + + this._set.length + + " " + + util.inspect(context) + ) + } + + return promise + .then(result => { + if (DEBUG_CLS_HOOKED) { + debug2( + "CONTEXT-runPromise AFTER then: (" + + this.name + + ") currentUid:" + + currentUid + + " len:" + + this._set.length + + " " + + util.inspect(context) + ) + } + this.exit(context) + return result + }) + .catch(err => { + err[ERROR_SYMBOL] = context + if (DEBUG_CLS_HOOKED) { + debug2( + "CONTEXT-runPromise AFTER catch: (" + + this.name + + ") currentUid:" + + currentUid + + " len:" + + this._set.length + + " " + + util.inspect(context) + ) + } + this.exit(context) + throw err + }) +} + +Namespace.prototype.bind = function bindFactory(fn, context) { + if (!context) { + if (!this.active) { + context = this.createContext() + } else { + context = this.active + } + } + + let self = this + return function clsBind() { + self.enter(context) + try { + return fn.apply(this, arguments) + } catch (exception) { + if (exception) { + exception[ERROR_SYMBOL] = context + } + throw exception + } finally { + self.exit(context) + } + } +} + +Namespace.prototype.enter = function enter(context) { + assert.ok(context, "context must be provided for entering") + if (DEBUG_CLS_HOOKED) { + const asyncHooksCurrentId = async_hooks.executionAsyncId() + const triggerId = async_hooks.triggerAsyncId() + const indentStr = " ".repeat(this._indent < 0 ? 0 : this._indent) + debug2( + `${indentStr}CONTEXT-ENTER: (${ + this.name + }) currentUid:${currentUid} triggerId:${triggerId} asyncHooksCurrentId:${asyncHooksCurrentId} len:${ + this._set.length + } ${util.inspect(context)}` + ) + } + + this._set.push(this.active) + this.active = context +} + +Namespace.prototype.exit = function exit(context) { + assert.ok(context, "context must be provided for exiting") + if (DEBUG_CLS_HOOKED) { + const asyncHooksCurrentId = async_hooks.executionAsyncId() + const triggerId = async_hooks.triggerAsyncId() + const indentStr = " ".repeat(this._indent < 0 ? 0 : this._indent) + debug2( + `${indentStr}CONTEXT-EXIT: (${ + this.name + }) currentUid:${currentUid} triggerId:${triggerId} asyncHooksCurrentId:${asyncHooksCurrentId} len:${ + this._set.length + } ${util.inspect(context)}` + ) + } + + // Fast path for most exits that are at the top of the stack + if (this.active === context) { + assert.ok(this._set.length, "can't remove top context") + this.active = this._set.pop() + return + } + + // Fast search in the stack using lastIndexOf + let index = this._set.lastIndexOf(context) + + if (index < 0) { + if (DEBUG_CLS_HOOKED) { + debug2( + "??ERROR?? context exiting but not entered - ignoring: " + + util.inspect(context) + ) + } + assert.ok( + index >= 0, + "context not currently entered; can't exit. \n" + + util.inspect(this) + + "\n" + + util.inspect(context) + ) + } else { + assert.ok(index, "can't remove top context") + this._set.splice(index, 1) + } +} + +Namespace.prototype.bindEmitter = function bindEmitter(emitter) { + assert.ok( + emitter.on && emitter.addListener && emitter.emit, + "can only bind real EEs" + ) + + let namespace = this + let thisSymbol = "context@" + this.name + + // Capture the context active at the time the emitter is bound. + function attach(listener) { + if (!listener) { + return + } + if (!listener[CONTEXTS_SYMBOL]) { + listener[CONTEXTS_SYMBOL] = Object.create(null) + } + + listener[CONTEXTS_SYMBOL][thisSymbol] = { + namespace: namespace, + context: namespace.active, + } + } + + // At emit time, bind the listener within the correct context. + function bind(unwrapped) { + if (!(unwrapped && unwrapped[CONTEXTS_SYMBOL])) { + return unwrapped + } + + let wrapped = unwrapped + let unwrappedContexts = unwrapped[CONTEXTS_SYMBOL] + Object.keys(unwrappedContexts).forEach(function (name) { + let thunk = unwrappedContexts[name] + wrapped = thunk.namespace.bind(wrapped, thunk.context) + }) + return wrapped + } + + wrapEmitter(emitter, attach, bind) +} + +/** + * If an error comes out of a namespace, it will have a context attached to it. + * This function knows how to find it. + * + * @param {Error} exception Possibly annotated error. + */ +Namespace.prototype.fromException = function fromException(exception) { + return exception[ERROR_SYMBOL] +} + +function getNamespace(name) { + return process.namespaces[name] +} + +function createNamespace(name) { + assert.ok(name, "namespace must be given a name.") + + if (DEBUG_CLS_HOOKED) { + debug2(`NS-CREATING NAMESPACE (${name})`) + } + let namespace = new Namespace(name) + namespace.id = currentUid + + const hook = async_hooks.createHook({ + init(asyncId, type, triggerId, resource) { + currentUid = async_hooks.executionAsyncId() + + //CHAIN Parent's Context onto child if none exists. This is needed to pass net-events.spec + // let initContext = namespace.active; + // if(!initContext && triggerId) { + // let parentContext = namespace._contexts.get(triggerId); + // if (parentContext) { + // namespace.active = parentContext; + // namespace._contexts.set(currentUid, parentContext); + // if (DEBUG_CLS_HOOKED) { + // const indentStr = ' '.repeat(namespace._indent < 0 ? 0 : namespace._indent); + // debug2(`${indentStr}INIT [${type}] (${name}) WITH PARENT CONTEXT asyncId:${asyncId} currentUid:${currentUid} triggerId:${triggerId} active:${util.inspect(namespace.active, true)} resource:${resource}`); + // } + // } else if (DEBUG_CLS_HOOKED) { + // const indentStr = ' '.repeat(namespace._indent < 0 ? 0 : namespace._indent); + // debug2(`${indentStr}INIT [${type}] (${name}) MISSING CONTEXT asyncId:${asyncId} currentUid:${currentUid} triggerId:${triggerId} active:${util.inspect(namespace.active, true)} resource:${resource}`); + // } + // }else { + // namespace._contexts.set(currentUid, namespace.active); + // if (DEBUG_CLS_HOOKED) { + // const indentStr = ' '.repeat(namespace._indent < 0 ? 0 : namespace._indent); + // debug2(`${indentStr}INIT [${type}] (${name}) asyncId:${asyncId} currentUid:${currentUid} triggerId:${triggerId} active:${util.inspect(namespace.active, true)} resource:${resource}`); + // } + // } + if (namespace.active) { + namespace._contexts.set(asyncId, namespace.active) + + if (DEBUG_CLS_HOOKED) { + const indentStr = " ".repeat( + namespace._indent < 0 ? 0 : namespace._indent + ) + debug2( + `${indentStr}INIT [${type}] (${name}) asyncId:${asyncId} currentUid:${currentUid} triggerId:${triggerId} active:${util.inspect( + namespace.active, + { showHidden: true, depth: 2, colors: true } + )} resource:${resource}` + ) + } + } else if (currentUid === 0) { + // CurrentId will be 0 when triggered from C++. Promise events + // https://github.com/nodejs/node/blob/master/doc/api/async_hooks.md#triggerid + const triggerId = async_hooks.triggerAsyncId() + const triggerIdContext = namespace._contexts.get(triggerId) + if (triggerIdContext) { + namespace._contexts.set(asyncId, triggerIdContext) + if (DEBUG_CLS_HOOKED) { + const indentStr = " ".repeat( + namespace._indent < 0 ? 0 : namespace._indent + ) + debug2( + `${indentStr}INIT USING CONTEXT FROM TRIGGERID [${type}] (${name}) asyncId:${asyncId} currentUid:${currentUid} triggerId:${triggerId} active:${util.inspect( + namespace.active, + { showHidden: true, depth: 2, colors: true } + )} resource:${resource}` + ) + } + } else if (DEBUG_CLS_HOOKED) { + const indentStr = " ".repeat( + namespace._indent < 0 ? 0 : namespace._indent + ) + debug2( + `${indentStr}INIT MISSING CONTEXT [${type}] (${name}) asyncId:${asyncId} currentUid:${currentUid} triggerId:${triggerId} active:${util.inspect( + namespace.active, + { showHidden: true, depth: 2, colors: true } + )} resource:${resource}` + ) + } + } + + if (DEBUG_CLS_HOOKED && type === "PROMISE") { + debug2(util.inspect(resource, { showHidden: true })) + const parentId = resource.parentId + const indentStr = " ".repeat( + namespace._indent < 0 ? 0 : namespace._indent + ) + debug2( + `${indentStr}INIT RESOURCE-PROMISE [${type}] (${name}) parentId:${parentId} asyncId:${asyncId} currentUid:${currentUid} triggerId:${triggerId} active:${util.inspect( + namespace.active, + { showHidden: true, depth: 2, colors: true } + )} resource:${resource}` + ) + } + }, + before(asyncId) { + currentUid = async_hooks.executionAsyncId() + let context + + /* + if(currentUid === 0){ + // CurrentId will be 0 when triggered from C++. Promise events + // https://github.com/nodejs/node/blob/master/doc/api/async_hooks.md#triggerid + //const triggerId = async_hooks.triggerAsyncId(); + context = namespace._contexts.get(asyncId); // || namespace._contexts.get(triggerId); + }else{ + context = namespace._contexts.get(currentUid); + } + */ + + //HACK to work with promises until they are fixed in node > 8.1.1 + context = + namespace._contexts.get(asyncId) || namespace._contexts.get(currentUid) + + if (context) { + if (DEBUG_CLS_HOOKED) { + const triggerId = async_hooks.triggerAsyncId() + const indentStr = " ".repeat( + namespace._indent < 0 ? 0 : namespace._indent + ) + debug2( + `${indentStr}BEFORE (${name}) asyncId:${asyncId} currentUid:${currentUid} triggerId:${triggerId} active:${util.inspect( + namespace.active, + { showHidden: true, depth: 2, colors: true } + )} context:${util.inspect(context)}` + ) + namespace._indent += 2 + } + + namespace.enter(context) + } else if (DEBUG_CLS_HOOKED) { + const triggerId = async_hooks.triggerAsyncId() + const indentStr = " ".repeat( + namespace._indent < 0 ? 0 : namespace._indent + ) + debug2( + `${indentStr}BEFORE MISSING CONTEXT (${name}) asyncId:${asyncId} currentUid:${currentUid} triggerId:${triggerId} active:${util.inspect( + namespace.active, + { showHidden: true, depth: 2, colors: true } + )} namespace._contexts:${util.inspect(namespace._contexts, { + showHidden: true, + depth: 2, + colors: true, + })}` + ) + namespace._indent += 2 + } + }, + after(asyncId) { + currentUid = async_hooks.executionAsyncId() + let context // = namespace._contexts.get(currentUid); + /* + if(currentUid === 0){ + // CurrentId will be 0 when triggered from C++. Promise events + // https://github.com/nodejs/node/blob/master/doc/api/async_hooks.md#triggerid + //const triggerId = async_hooks.triggerAsyncId(); + context = namespace._contexts.get(asyncId); // || namespace._contexts.get(triggerId); + }else{ + context = namespace._contexts.get(currentUid); + } + */ + //HACK to work with promises until they are fixed in node > 8.1.1 + context = + namespace._contexts.get(asyncId) || namespace._contexts.get(currentUid) + + if (context) { + if (DEBUG_CLS_HOOKED) { + const triggerId = async_hooks.triggerAsyncId() + namespace._indent -= 2 + const indentStr = " ".repeat( + namespace._indent < 0 ? 0 : namespace._indent + ) + debug2( + `${indentStr}AFTER (${name}) asyncId:${asyncId} currentUid:${currentUid} triggerId:${triggerId} active:${util.inspect( + namespace.active, + { showHidden: true, depth: 2, colors: true } + )} context:${util.inspect(context)}` + ) + } + + namespace.exit(context) + } else if (DEBUG_CLS_HOOKED) { + const triggerId = async_hooks.triggerAsyncId() + namespace._indent -= 2 + const indentStr = " ".repeat( + namespace._indent < 0 ? 0 : namespace._indent + ) + debug2( + `${indentStr}AFTER MISSING CONTEXT (${name}) asyncId:${asyncId} currentUid:${currentUid} triggerId:${triggerId} active:${util.inspect( + namespace.active, + { showHidden: true, depth: 2, colors: true } + )} context:${util.inspect(context)}` + ) + } + }, + destroy(asyncId) { + currentUid = async_hooks.executionAsyncId() + if (DEBUG_CLS_HOOKED) { + const triggerId = async_hooks.triggerAsyncId() + const indentStr = " ".repeat( + namespace._indent < 0 ? 0 : namespace._indent + ) + debug2( + `${indentStr}DESTROY (${name}) currentUid:${currentUid} asyncId:${asyncId} triggerId:${triggerId} active:${util.inspect( + namespace.active, + { showHidden: true, depth: 2, colors: true } + )} context:${util.inspect(namespace._contexts.get(currentUid))}` + ) + } + + namespace._contexts.delete(asyncId) + }, + }) + + hook.enable() + namespace._hook = hook + + process.namespaces[name] = namespace + return namespace +} + +function destroyNamespace(name) { + let namespace = getNamespace(name) + + assert.ok(namespace, "can't delete nonexistent namespace! \"" + name + '"') + assert.ok( + namespace.id, + "don't assign to process.namespaces directly! " + util.inspect(namespace) + ) + + namespace._hook.disable() + namespace._contexts = null + process.namespaces[name] = null +} + +function reset() { + // must unregister async listeners + if (process.namespaces) { + Object.keys(process.namespaces).forEach(function (name) { + destroyNamespace(name) + }) + } + process.namespaces = Object.create(null) +} + +process.namespaces = process.namespaces || {} + +//const fs = require('fs'); +function debug2(...args) { + if (DEBUG_CLS_HOOKED) { + //fs.writeSync(1, `${util.format(...args)}\n`); + process._rawDebug(`${util.format(...args)}`) + } +} + +/*function getFunctionName(fn) { + if (!fn) { + return fn; + } + if (typeof fn === 'function') { + if (fn.name) { + return fn.name; + } + return (fn.toString().trim().match(/^function\s*([^\s(]+)/) || [])[1]; + } else if (fn.constructor && fn.constructor.name) { + return fn.constructor.name; + } +}*/ diff --git a/packages/backend-core/src/context/FunctionContext.js b/packages/backend-core/src/context/FunctionContext.js index 34d39492f9..c0ed34fe78 100644 --- a/packages/backend-core/src/context/FunctionContext.js +++ b/packages/backend-core/src/context/FunctionContext.js @@ -1,84 +1,47 @@ -const cls = require("cls-hooked") +const cls = require("../clshooked") const { newid } = require("../hashing") const REQUEST_ID_KEY = "requestId" +const MAIN_CTX = cls.createNamespace("main") + +function getContextStorage(namespace) { + if (namespace && namespace.active) { + let contextData = namespace.active + delete contextData.id + delete contextData._ns_name + return contextData + } + return {} +} class FunctionContext { - static getMiddleware( - updateCtxFn = null, - destroyFn = null, - contextName = "session" - ) { - const namespace = this.createNamespace(contextName) - - return async function (ctx, next) { - await new Promise( - namespace.bind(function (resolve, reject) { - // store a contextual request ID that can be used anywhere (audit logs) - namespace.set(REQUEST_ID_KEY, newid()) - namespace.bindEmitter(ctx.req) - namespace.bindEmitter(ctx.res) - - if (updateCtxFn) { - updateCtxFn(ctx) - } - next() - .then(resolve) - .catch(reject) - .finally(() => { - if (destroyFn) { - return destroyFn(ctx) - } - }) - }) - ) - } + static run(callback) { + return MAIN_CTX.runAndReturn(async () => { + const namespaceId = newid() + MAIN_CTX.set(REQUEST_ID_KEY, namespaceId) + const namespace = cls.createNamespace(namespaceId) + let response = await namespace.runAndReturn(callback) + cls.destroyNamespace(namespaceId) + return response + }) } - static run(callback, contextName = "session") { - const namespace = this.createNamespace(contextName) - - return namespace.runAndReturn(callback) - } - - static setOnContext(key, value, contextName = "session") { - const namespace = this.createNamespace(contextName) + static setOnContext(key, value) { + const namespaceId = MAIN_CTX.get(REQUEST_ID_KEY) + const namespace = cls.getNamespace(namespaceId) namespace.set(key, value) } - static getContextStorage() { - if (this._namespace && this._namespace.active) { - let contextData = this._namespace.active - delete contextData.id - delete contextData._ns_name - return contextData - } - - return {} - } - static getFromContext(key) { - const context = this.getContextStorage() + const namespaceId = MAIN_CTX.get(REQUEST_ID_KEY) + const namespace = cls.getNamespace(namespaceId) + const context = getContextStorage(namespace) if (context) { return context[key] } else { return null } } - - static destroyNamespace(name = "session") { - if (this._namespace) { - cls.destroyNamespace(name) - this._namespace = null - } - } - - static createNamespace(name = "session") { - if (!this._namespace) { - this._namespace = cls.createNamespace(name) - } - return this._namespace - } } module.exports = FunctionContext diff --git a/packages/backend-core/src/context/index.js b/packages/backend-core/src/context/index.js index b6b6f2380c..20e5e26693 100644 --- a/packages/backend-core/src/context/index.js +++ b/packages/backend-core/src/context/index.js @@ -55,6 +55,15 @@ async function closeAppDBs() { } } +exports.closeTenancy = async () => { + if (env.USE_COUCH) { + await closeDB(exports.getGlobalDB()) + } + // clear from context now that database is closed/task is finished + cls.setOnContext(ContextKeys.TENANT_ID, null) + cls.setOnContext(ContextKeys.GLOBAL_DB, null) +} + exports.isDefaultTenant = () => { return exports.getTenantId() === exports.DEFAULT_TENANT_ID } @@ -82,12 +91,7 @@ exports.doInTenant = (tenantId, task) => { } finally { const using = cls.getFromContext(ContextKeys.IN_USE) if (!using || using <= 1) { - if (env.USE_COUCH) { - await closeDB(exports.getGlobalDB()) - } - // clear from context now that database is closed/task is finished - cls.setOnContext(ContextKeys.TENANT_ID, null) - cls.setOnContext(ContextKeys.GLOBAL_DB, null) + await exports.closeTenancy() } else { cls.setOnContext(using - 1) } diff --git a/packages/backend-core/src/middleware/tenancy.js b/packages/backend-core/src/middleware/tenancy.js index f4053d1f5b..9a0cb8a0c6 100644 --- a/packages/backend-core/src/middleware/tenancy.js +++ b/packages/backend-core/src/middleware/tenancy.js @@ -1,6 +1,5 @@ -const { setTenantId, setGlobalDB, getGlobalDB } = require("../tenancy") -const { closeDB } = require("../db") -const ContextFactory = require("../context/FunctionContext") +const { setTenantId, setGlobalDB, closeTenancy } = require("../tenancy") +const cls = require("../context/FunctionContext") const { buildMatcherRegex, matches } = require("./matchers") module.exports = ( @@ -11,17 +10,16 @@ module.exports = ( const allowQsOptions = buildMatcherRegex(allowQueryStringPatterns) const noTenancyOptions = buildMatcherRegex(noTenancyPatterns) - const updateCtxFn = ctx => { - const allowNoTenant = - opts.noTenancyRequired || !!matches(ctx, noTenancyOptions) - const allowQs = !!matches(ctx, allowQsOptions) - const tenantId = setTenantId(ctx, { allowQs, allowNoTenant }) - setGlobalDB(tenantId) + return async function (ctx, next) { + return cls.run(async () => { + const allowNoTenant = + opts.noTenancyRequired || !!matches(ctx, noTenancyOptions) + const allowQs = !!matches(ctx, allowQsOptions) + const tenantId = setTenantId(ctx, { allowQs, allowNoTenant }) + setGlobalDB(tenantId) + const res = await next() + await closeTenancy() + return res + }) } - const destroyFn = async () => { - const db = getGlobalDB() - await closeDB(db) - } - - return ContextFactory.getMiddleware(updateCtxFn, destroyFn) } diff --git a/packages/backend-core/src/security/permissions.js b/packages/backend-core/src/security/permissions.js index 28044a5129..2ecb8a9f1e 100644 --- a/packages/backend-core/src/security/permissions.js +++ b/packages/backend-core/src/security/permissions.js @@ -96,6 +96,7 @@ const BUILTIN_PERMISSIONS = { new Permission(PermissionTypes.QUERY, PermissionLevels.WRITE), new Permission(PermissionTypes.TABLE, PermissionLevels.WRITE), new Permission(PermissionTypes.VIEW, PermissionLevels.READ), + new Permission(PermissionTypes.AUTOMATION, PermissionLevels.EXECUTE), ], }, POWER: { diff --git a/packages/backend-core/yarn.lock b/packages/backend-core/yarn.lock index 87db3761bc..7dfa64810e 100644 --- a/packages/backend-core/yarn.lock +++ b/packages/backend-core/yarn.lock @@ -805,13 +805,6 @@ ast-types@0.9.6: resolved "https://registry.yarnpkg.com/ast-types/-/ast-types-0.9.6.tgz#102c9e9e9005d3e7e3829bf0c4fa24ee862ee9b9" integrity sha1-ECyenpAF0+fjgpvwxPok7oYu6bk= -async-hook-jl@^1.7.6: - version "1.7.6" - resolved "https://registry.yarnpkg.com/async-hook-jl/-/async-hook-jl-1.7.6.tgz#4fd25c2f864dbaf279c610d73bf97b1b28595e68" - integrity sha512-gFaHkFfSxTjvoxDMYqDuGHlcRyUuamF8s+ZTtJdDzqjws4mCt7v0vuV79/E2Wr2/riMQgtG4/yUtXWs1gZ7JMg== - dependencies: - stack-chain "^1.3.7" - async@~2.1.4: version "2.1.5" resolved "https://registry.yarnpkg.com/async/-/async-2.1.5.tgz#e587c68580994ac67fc56ff86d3ac56bdbe810bc" @@ -1205,15 +1198,6 @@ clone-buffer@1.0.0: resolved "https://registry.yarnpkg.com/clone-buffer/-/clone-buffer-1.0.0.tgz#e3e25b207ac4e701af721e2cb5a16792cac3dc58" integrity sha1-4+JbIHrE5wGvch4staFnksrD3Fg= -cls-hooked@^4.2.2: - version "4.2.2" - resolved "https://registry.yarnpkg.com/cls-hooked/-/cls-hooked-4.2.2.tgz#ad2e9a4092680cdaffeb2d3551da0e225eae1908" - integrity sha512-J4Xj5f5wq/4jAvcdgoGsL3G103BtWpZrMo8NEinRltN+xpTZdI+M38pyQqhuFU/P792xkMFvnKSf+Lm81U1bxw== - dependencies: - async-hook-jl "^1.7.6" - emitter-listener "^1.0.1" - semver "^5.4.1" - cluster-key-slot@^1.1.0: version "1.1.0" resolved "https://registry.yarnpkg.com/cluster-key-slot/-/cluster-key-slot-1.1.0.tgz#30474b2a981fb12172695833052bc0d01336d10d" @@ -1533,7 +1517,7 @@ electron-to-chromium@^1.3.896: resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.3.900.tgz#5be2c5818a2a012c511b4b43e87b6ab7a296d4f5" integrity sha512-SuXbQD8D4EjsaBaJJxySHbC+zq8JrFfxtb4GIr4E9n1BcROyMcRrJCYQNpJ9N+Wjf5mFp7Wp0OHykd14JNEzzQ== -emitter-listener@^1.0.1: +emitter-listener@^1.1.2: version "1.1.2" resolved "https://registry.yarnpkg.com/emitter-listener/-/emitter-listener-1.1.2.tgz#56b140e8f6992375b3d7cb2cab1cc7432d9632e8" integrity sha512-Bt1sBAGFHY9DKY+4/2cV6izcKJUf5T7/gkdmkxzX/qv9CcGH8xSwVRW5mtX03SWJtRTWSOpzCuWN9rBFYZepZQ== @@ -4466,7 +4450,7 @@ saxes@^5.0.1: dependencies: xmlchars "^2.2.0" -"semver@2 || 3 || 4 || 5", semver@^5.4.1, semver@^5.5.0, semver@^5.6.0: +"semver@2 || 3 || 4 || 5", semver@^5.5.0, semver@^5.6.0: version "5.7.1" resolved "https://registry.yarnpkg.com/semver/-/semver-5.7.1.tgz#a954f931aeba508d307bbf069eff0c01c96116f7" integrity sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ== @@ -4706,11 +4690,6 @@ sshpk@^1.7.0: jsbn "~0.1.0" tweetnacl "~0.14.0" -stack-chain@^1.3.7: - version "1.3.7" - resolved "https://registry.yarnpkg.com/stack-chain/-/stack-chain-1.3.7.tgz#d192c9ff4ea6a22c94c4dd459171e3f00cea1285" - integrity sha1-0ZLJ/06moiyUxN1FkXHj8AzqEoU= - stack-utils@^2.0.2: version "2.0.5" resolved "https://registry.yarnpkg.com/stack-utils/-/stack-utils-2.0.5.tgz#d25265fca995154659dbbfba3b49254778d2fdd5" diff --git a/packages/bbui/package.json b/packages/bbui/package.json index 9cd9f2cbba..8bc0329509 100644 --- a/packages/bbui/package.json +++ b/packages/bbui/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/bbui", "description": "A UI solution used in the different Budibase projects.", - "version": "1.0.151-alpha.2", + "version": "1.0.159-alpha.1", "license": "MPL-2.0", "svelte": "src/index.js", "module": "dist/bbui.es.js", @@ -38,7 +38,7 @@ ], "dependencies": { "@adobe/spectrum-css-workflow-icons": "^1.2.1", - "@budibase/string-templates": "^1.0.151-alpha.2", + "@budibase/string-templates": "^1.0.159-alpha.1", "@spectrum-css/actionbutton": "^1.0.1", "@spectrum-css/actiongroup": "^1.0.1", "@spectrum-css/avatar": "^3.0.2", diff --git a/packages/bbui/src/Form/Core/CheckboxGroup.svelte b/packages/bbui/src/Form/Core/CheckboxGroup.svelte new file mode 100644 index 0000000000..640d5d99cd --- /dev/null +++ b/packages/bbui/src/Form/Core/CheckboxGroup.svelte @@ -0,0 +1,68 @@ + + +
+ {#if options && Array.isArray(options)} + {#each options as option} +
+ +
+ {/each} + {/if} +
+ + diff --git a/packages/bbui/src/Form/Core/DatePicker.svelte b/packages/bbui/src/Form/Core/DatePicker.svelte index fd67fa41bb..1d0d32b03c 100644 --- a/packages/bbui/src/Form/Core/DatePicker.svelte +++ b/packages/bbui/src/Form/Core/DatePicker.svelte @@ -156,8 +156,8 @@ @@ -212,4 +212,7 @@ :global(.flatpickr-calendar) { font-family: "Source Sans Pro", sans-serif; } + .is-disabled { + pointer-events: none !important; + } diff --git a/packages/bbui/src/Form/Core/Picker.svelte b/packages/bbui/src/Form/Core/Picker.svelte index 143536a60a..2585f11939 100644 --- a/packages/bbui/src/Form/Core/Picker.svelte +++ b/packages/bbui/src/Form/Core/Picker.svelte @@ -43,7 +43,7 @@ return } searchTerm = null - open = true + open = !open } const getSortedOptions = (options, getLabel, sort) => { @@ -71,105 +71,73 @@ } - -{#if open} -
(open = false)} - transition:fly|local={{ y: -20, duration: 200 }} - class="spectrum-Popover spectrum-Popover--bottom spectrum-Picker-popover is-open" - class:auto-width={autoWidth} - > - {#if autocomplete} - (searchTerm = event.detail)} - {disabled} - placeholder="Search" - /> - {/if} - +
+ {/if} + diff --git a/packages/client/src/components/devtools/DevToolsHeader.svelte b/packages/client/src/components/devtools/DevToolsHeader.svelte index 3ea528be5b..4dfaae610f 100644 --- a/packages/client/src/components/devtools/DevToolsHeader.svelte +++ b/packages/client/src/components/devtools/DevToolsHeader.svelte @@ -71,4 +71,9 @@ .dev-preview-header :global(.spectrum-Picker-label) { color: white !important; } + @media print { + .dev-preview-header { + display: none; + } + } diff --git a/packages/frontend-core/package.json b/packages/frontend-core/package.json index 904597a7d9..5da1605daf 100644 --- a/packages/frontend-core/package.json +++ b/packages/frontend-core/package.json @@ -1,12 +1,12 @@ { "name": "@budibase/frontend-core", - "version": "1.0.151-alpha.2", + "version": "1.0.159-alpha.1", "description": "Budibase frontend core libraries used in builder and client", "author": "Budibase", "license": "MPL-2.0", "svelte": "src/index.js", "dependencies": { - "@budibase/bbui": "^1.0.151-alpha.2", + "@budibase/bbui": "^1.0.159-alpha.1", "lodash": "^4.17.21", "svelte": "^3.46.2" } diff --git a/packages/server/package.json b/packages/server/package.json index 0e7910d31d..7843b8d256 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/server", "email": "hi@budibase.com", - "version": "1.0.151-alpha.2", + "version": "1.0.159-alpha.1", "description": "Budibase Web Server", "main": "src/index.ts", "repository": { @@ -10,6 +10,7 @@ }, "scripts": { "build": "rimraf dist/ && tsc -p tsconfig.build.json && mv dist/src/* dist/ && rimraf dist/src/ && yarn postbuild", + "debug": "yarn build && node --expose-gc --inspect=9222 dist/index.js", "postbuild": "copyfiles -u 1 src/**/*.svelte dist/ && copyfiles -u 1 src/**/*.hbs dist/ && copyfiles -u 1 src/**/*.json dist/", "test": "jest --coverage --maxWorkers=2", "test:watch": "jest --watch", @@ -68,10 +69,10 @@ "license": "GPL-3.0", "dependencies": { "@apidevtools/swagger-parser": "^10.0.3", - "@budibase/backend-core": "^1.0.151-alpha.2", - "@budibase/client": "^1.0.151-alpha.2", - "@budibase/pro": "1.0.151-alpha.2", - "@budibase/string-templates": "^1.0.151-alpha.2", + "@budibase/backend-core": "^1.0.159-alpha.1", + "@budibase/client": "^1.0.159-alpha.1", + "@budibase/pro": "1.0.159-alpha.1", + "@budibase/string-templates": "^1.0.159-alpha.1", "@bull-board/api": "^3.7.0", "@bull-board/koa": "^3.7.0", "@elastic/elasticsearch": "7.10.0", diff --git a/packages/server/src/api/controllers/row/index.ts b/packages/server/src/api/controllers/row/index.ts index c50fef496e..27810008d3 100644 --- a/packages/server/src/api/controllers/row/index.ts +++ b/packages/server/src/api/controllers/row/index.ts @@ -31,7 +31,9 @@ export async function patch(ctx: any): Promise { return save(ctx) } try { - const { row, table } = await pickApi(tableId).patch(ctx) + const { row, table } = await quotas.addQuery(() => + pickApi(tableId).patch(ctx) + ) ctx.status = 200 ctx.eventEmitter && ctx.eventEmitter.emitRow(`row:update`, appId, row, table) @@ -42,7 +44,7 @@ export async function patch(ctx: any): Promise { } } -const saveRow = async (ctx: any) => { +export const save = async (ctx: any) => { const appId = ctx.appId const tableId = getTableId(ctx) const body = ctx.request.body @@ -51,7 +53,9 @@ const saveRow = async (ctx: any) => { return patch(ctx) } try { - const { row, table } = await pickApi(tableId).save(ctx) + const { row, table } = await quotas.addRow(() => + quotas.addQuery(() => pickApi(tableId).save(ctx)) + ) ctx.status = 200 ctx.eventEmitter && ctx.eventEmitter.emitRow(`row:save`, appId, row, table) ctx.message = `${table.name} saved successfully` @@ -61,14 +65,10 @@ const saveRow = async (ctx: any) => { } } -export async function save(ctx: any) { - await quotas.addRow(() => saveRow(ctx)) -} - export async function fetchView(ctx: any) { const tableId = getTableId(ctx) try { - ctx.body = await pickApi(tableId).fetchView(ctx) + ctx.body = await quotas.addQuery(() => pickApi(tableId).fetchView(ctx)) } catch (err) { ctx.throw(400, err) } @@ -77,7 +77,7 @@ export async function fetchView(ctx: any) { export async function fetch(ctx: any) { const tableId = getTableId(ctx) try { - ctx.body = await pickApi(tableId).fetch(ctx) + ctx.body = await quotas.addQuery(() => pickApi(tableId).fetch(ctx)) } catch (err) { ctx.throw(400, err) } @@ -86,7 +86,7 @@ export async function fetch(ctx: any) { export async function find(ctx: any) { const tableId = getTableId(ctx) try { - ctx.body = await pickApi(tableId).find(ctx) + ctx.body = await quotas.addQuery(() => pickApi(tableId).find(ctx)) } catch (err) { ctx.throw(400, err) } @@ -98,14 +98,16 @@ export async function destroy(ctx: any) { const tableId = getTableId(ctx) let response, row if (inputs.rows) { - let { rows } = await pickApi(tableId).bulkDestroy(ctx) + let { rows } = await quotas.addQuery(() => + pickApi(tableId).bulkDestroy(ctx) + ) await quotas.removeRows(rows.length) response = rows for (let row of rows) { ctx.eventEmitter && ctx.eventEmitter.emitRow(`row:delete`, appId, row) } } else { - let resp = await pickApi(tableId).destroy(ctx) + let resp = await quotas.addQuery(() => pickApi(tableId).destroy(ctx)) await quotas.removeRow() response = resp.response row = resp.row @@ -121,7 +123,7 @@ export async function search(ctx: any) { const tableId = getTableId(ctx) try { ctx.status = 200 - ctx.body = await pickApi(tableId).search(ctx) + ctx.body = await quotas.addQuery(() => pickApi(tableId).search(ctx)) } catch (err) { ctx.throw(400, err) } @@ -139,7 +141,9 @@ export async function validate(ctx: any) { export async function fetchEnrichedRow(ctx: any) { const tableId = getTableId(ctx) try { - ctx.body = await pickApi(tableId).fetchEnrichedRow(ctx) + ctx.body = await quotas.addQuery(() => + pickApi(tableId).fetchEnrichedRow(ctx) + ) } catch (err) { ctx.throw(400, err) } @@ -148,7 +152,7 @@ export async function fetchEnrichedRow(ctx: any) { export const exportRows = async (ctx: any) => { const tableId = getTableId(ctx) try { - ctx.body = await pickApi(tableId).exportRows(ctx) + ctx.body = await quotas.addQuery(() => pickApi(tableId).exportRows(ctx)) } catch (err) { ctx.throw(400, err) } diff --git a/packages/server/src/api/routes/tests/row.spec.js b/packages/server/src/api/routes/tests/row.spec.js index d7ec995edb..6b06b93872 100644 --- a/packages/server/src/api/routes/tests/row.spec.js +++ b/packages/server/src/api/routes/tests/row.spec.js @@ -3,6 +3,7 @@ const setup = require("./utilities") const { basicRow } = setup.structures const { doInAppContext } = require("@budibase/backend-core/context") const { doInTenant } = require("@budibase/backend-core/tenancy") +const { quotas, QuotaUsageType, StaticQuotaName, MonthlyQuotaName } = require("@budibase/pro") // mock the fetch for the search system jest.mock("node-fetch") @@ -28,9 +29,29 @@ describe("/rows", () => { .expect('Content-Type', /json/) .expect(status) + const getRowUsage = async () => { + return config.doInContext(null, () => quotas.getCurrentUsageValue(QuotaUsageType.STATIC, StaticQuotaName.ROWS)) + } + + const getQueryUsage = async () => { + return config.doInContext(null, () => quotas.getCurrentUsageValue(QuotaUsageType.MONTHLY, MonthlyQuotaName.QUERIES)) + } + + const assertRowUsage = async (expected) => { + const usage = await getRowUsage() + expect(usage).toBe(expected) + } + + const assertQueryUsage = async (expected) => { + const usage = await getQueryUsage() + expect(usage).toBe(expected) + } describe("save, load, update", () => { it("returns a success message when the row is created", async () => { + const rowUsage = await getRowUsage() + const queryUsage = await getQueryUsage() + const res = await request .post(`/api/${row.tableId}/rows`) .send(row) @@ -40,10 +61,14 @@ describe("/rows", () => { expect(res.res.statusMessage).toEqual(`${table.name} saved successfully`) expect(res.body.name).toEqual("Test Contact") expect(res.body._rev).toBeDefined() + await assertRowUsage(rowUsage + 1) + await assertQueryUsage(queryUsage + 1) }) it("updates a row successfully", async () => { const existing = await config.createRow() + const rowUsage = await getRowUsage() + const queryUsage = await getQueryUsage() const res = await request .post(`/api/${table._id}/rows`) @@ -59,10 +84,13 @@ describe("/rows", () => { expect(res.res.statusMessage).toEqual(`${table.name} updated successfully.`) expect(res.body.name).toEqual("Updated Name") + await assertRowUsage(rowUsage) + await assertQueryUsage(queryUsage + 1) }) it("should load a row", async () => { const existing = await config.createRow() + const queryUsage = await getQueryUsage() const res = await request .get(`/api/${table._id}/rows/${existing._id}`) @@ -76,6 +104,7 @@ describe("/rows", () => { _rev: existing._rev, type: "row", }) + await assertQueryUsage(queryUsage + 1) }) it("should list all rows for given tableId", async () => { @@ -86,6 +115,7 @@ describe("/rows", () => { } await config.createRow() await config.createRow(newRow) + const queryUsage = await getQueryUsage() const res = await request .get(`/api/${table._id}/rows`) @@ -96,15 +126,19 @@ describe("/rows", () => { expect(res.body.length).toBe(2) expect(res.body.find(r => r.name === newRow.name)).toBeDefined() expect(res.body.find(r => r.name === row.name)).toBeDefined() + await assertQueryUsage(queryUsage + 1) }) it("load should return 404 when row does not exist", async () => { await config.createRow() + const queryUsage = await getQueryUsage() + await request .get(`/api/${table._id}/rows/not-a-valid-id`) .set(config.defaultHeaders()) .expect('Content-Type', /json/) .expect(404) + await assertQueryUsage(queryUsage) // no change }) it("row values are coerced", async () => { @@ -202,6 +236,9 @@ describe("/rows", () => { it("should update only the fields that are supplied", async () => { const existing = await config.createRow() + const rowUsage = await getRowUsage() + const queryUsage = await getQueryUsage() + const res = await request .patch(`/api/${table._id}/rows`) .send({ @@ -222,10 +259,15 @@ describe("/rows", () => { expect(savedRow.body.description).toEqual(existing.description) expect(savedRow.body.name).toEqual("Updated Name") + await assertRowUsage(rowUsage) + await assertQueryUsage(queryUsage + 2) // account for the second load }) it("should throw an error when given improper types", async () => { const existing = await config.createRow() + const rowUsage = await getRowUsage() + const queryUsage = await getQueryUsage() + await request .patch(`/api/${table._id}/rows`) .send({ @@ -236,12 +278,18 @@ describe("/rows", () => { }) .set(config.defaultHeaders()) .expect(400) + + await assertRowUsage(rowUsage) + await assertQueryUsage(queryUsage) }) }) describe("destroy", () => { it("should be able to delete a row", async () => { const createdRow = await config.createRow(row) + const rowUsage = await getRowUsage() + const queryUsage = await getQueryUsage() + const res = await request .delete(`/api/${table._id}/rows`) .send({ @@ -253,11 +301,16 @@ describe("/rows", () => { .expect('Content-Type', /json/) .expect(200) expect(res.body[0]._id).toEqual(createdRow._id) + await assertRowUsage(rowUsage -1) + await assertQueryUsage(queryUsage +1) }) }) describe("validate", () => { it("should return no errors on valid row", async () => { + const rowUsage = await getRowUsage() + const queryUsage = await getQueryUsage() + const res = await request .post(`/api/${table._id}/rows/validate`) .send({ name: "ivan" }) @@ -267,9 +320,14 @@ describe("/rows", () => { expect(res.body.valid).toBe(true) expect(Object.keys(res.body.errors)).toEqual([]) + await assertRowUsage(rowUsage) + await assertQueryUsage(queryUsage) }) it("should errors on invalid row", async () => { + const rowUsage = await getRowUsage() + const queryUsage = await getQueryUsage() + const res = await request .post(`/api/${table._id}/rows/validate`) .send({ name: 1 }) @@ -279,7 +337,8 @@ describe("/rows", () => { expect(res.body.valid).toBe(false) expect(Object.keys(res.body.errors)).toEqual(["name"]) - + await assertRowUsage(rowUsage) + await assertQueryUsage(queryUsage) }) }) @@ -287,6 +346,9 @@ describe("/rows", () => { it("should be able to delete a bulk set of rows", async () => { const row1 = await config.createRow() const row2 = await config.createRow() + const rowUsage = await getRowUsage() + const queryUsage = await getQueryUsage() + const res = await request .delete(`/api/${table._id}/rows`) .send({ @@ -298,14 +360,20 @@ describe("/rows", () => { .set(config.defaultHeaders()) .expect('Content-Type', /json/) .expect(200) + expect(res.body.length).toEqual(2) await loadRow(row1._id, 404) + await assertRowUsage(rowUsage - 2) + await assertQueryUsage(queryUsage +1) }) }) describe("fetchView", () => { it("should be able to fetch tables contents via 'view'", async () => { const row = await config.createRow() + const rowUsage = await getRowUsage() + const queryUsage = await getQueryUsage() + const res = await request .get(`/api/views/${table._id}`) .set(config.defaultHeaders()) @@ -313,18 +381,29 @@ describe("/rows", () => { .expect(200) expect(res.body.length).toEqual(1) expect(res.body[0]._id).toEqual(row._id) + await assertRowUsage(rowUsage) + await assertQueryUsage(queryUsage +1) }) it("should throw an error if view doesn't exist", async () => { + const rowUsage = await getRowUsage() + const queryUsage = await getQueryUsage() + await request .get(`/api/views/derp`) .set(config.defaultHeaders()) .expect(404) + + await assertRowUsage(rowUsage) + await assertQueryUsage(queryUsage) }) it("should be able to run on a view", async () => { const view = await config.createView() const row = await config.createRow() + const rowUsage = await getRowUsage() + const queryUsage = await getQueryUsage() + const res = await request .get(`/api/views/${view.name}`) .set(config.defaultHeaders()) @@ -332,13 +411,12 @@ describe("/rows", () => { .expect(200) expect(res.body.length).toEqual(1) expect(res.body[0]._id).toEqual(row._id) + + await assertRowUsage(rowUsage) + await assertQueryUsage(queryUsage + 1) }) }) - describe("user testing", () => { - - }) - describe("fetchEnrichedRows", () => { it("should allow enriching some linked rows", async () => { const { table, firstRow, secondRow } = await doInTenant(setup.structures.TENANT_ID, async () => { @@ -356,6 +434,8 @@ describe("/rows", () => { }) return { table, firstRow, secondRow } }) + const rowUsage = await getRowUsage() + const queryUsage = await getQueryUsage() // test basic enrichment const resBasic = await request @@ -376,6 +456,8 @@ describe("/rows", () => { expect(resEnriched.body.link[0]._id).toBe(firstRow._id) expect(resEnriched.body.link[0].name).toBe("Test Contact") expect(resEnriched.body.link[0].description).toBe("original description") + await assertRowUsage(rowUsage) + await assertQueryUsage(queryUsage +2) }) }) diff --git a/packages/server/src/api/routes/utils/validators.js b/packages/server/src/api/routes/utils/validators.js index dac936911b..7d5de340b2 100644 --- a/packages/server/src/api/routes/utils/validators.js +++ b/packages/server/src/api/routes/utils/validators.js @@ -167,7 +167,6 @@ exports.screenValidator = () => { _id: Joi.string().required(), _component: Joi.string().required(), _children: Joi.array().required(), - _instanceName: Joi.string().required(), _styles: Joi.object().required(), type: OPTIONAL_STRING, table: OPTIONAL_STRING, diff --git a/packages/server/src/constants/index.js b/packages/server/src/constants/index.js index 60f3c981d6..3aa5b2fb7b 100644 --- a/packages/server/src/constants/index.js +++ b/packages/server/src/constants/index.js @@ -208,5 +208,10 @@ exports.AutomationErrors = { FAILURE_CONDITION: "FAILURE_CONDITION_MET", } +exports.LoopStepTypes = { + ARRAY: "Array", + STRING: "String", +} + // pass through the list from the auth/core lib exports.ObjectStoreBuckets = ObjectStoreBuckets diff --git a/packages/server/src/integrations/base/sql.ts b/packages/server/src/integrations/base/sql.ts index 2e14eae870..782f61e49e 100644 --- a/packages/server/src/integrations/base/sql.ts +++ b/packages/server/src/integrations/base/sql.ts @@ -21,6 +21,31 @@ type KnexQuery = Knex.QueryBuilder | Knex const MIN_ISO_DATE = "0000-00-00T00:00:00.000Z" const MAX_ISO_DATE = "9999-00-00T00:00:00.000Z" +function likeKey(client: string, key: string): string { + if (!key.includes(" ")) { + return key + } + let start: string, end: string + switch (client) { + case SqlClients.MY_SQL: + start = end = "`" + break + case SqlClients.ORACLE: + case SqlClients.POSTGRES: + start = end = '"' + break + case SqlClients.MS_SQL: + start = "[" + end = "]" + break + default: + throw "Unknown client" + } + const parts = key.split(".") + key = parts.map(part => `${start}${part}${end}`).join(".") + return key +} + function parse(input: any) { if (Array.isArray(input)) { return JSON.stringify(input) @@ -125,7 +150,9 @@ class InternalBuilder { } else { const rawFnc = `${fnc}Raw` // @ts-ignore - query = query[rawFnc](`LOWER(${key}) LIKE ?`, [`%${value}%`]) + query = query[rawFnc](`LOWER(${likeKey(this.client, key)}) LIKE ?`, [ + `%${value}%`, + ]) } }) } diff --git a/packages/server/src/threads/automation.js b/packages/server/src/threads/automation.js index 98c45e4af3..4ca490affd 100644 --- a/packages/server/src/threads/automation.js +++ b/packages/server/src/threads/automation.js @@ -8,7 +8,7 @@ const { DocumentTypes } = require("../db/utils") const { doInTenant } = require("@budibase/backend-core/tenancy") const { definitions: triggerDefs } = require("../automations/triggerInfo") const { doInAppContext, getAppDB } = require("@budibase/backend-core/context") -const { AutomationErrors } = require("../constants") +const { AutomationErrors, LoopStepTypes } = require("../constants") const FILTER_STEP_ID = actions.ACTION_DEFINITIONS.FILTER.stepId const LOOP_STEP_ID = actions.ACTION_DEFINITIONS.LOOP.stepId @@ -17,6 +17,41 @@ const STOPPED_STATUS = { success: false, status: "STOPPED" } const { cloneDeep } = require("lodash/fp") const env = require("../environment") +function typecastForLooping(loopStep, input) { + if (!input || !input.binding) { + return null + } + const isArray = Array.isArray(input.binding), + isString = typeof input.binding === "string" + try { + switch (loopStep.inputs.option) { + case LoopStepTypes.ARRAY: + if (isString) { + return JSON.parse(input.binding) + } + break + case LoopStepTypes.STRING: + if (isArray) { + return input.binding.join(",") + } + break + } + } catch (err) { + throw new Error("Unable to cast to correct type") + } + return input.binding +} + +function getLoopIterations(loopStep, input) { + const binding = typecastForLooping(loopStep, input) + if (!loopStep || !binding) { + return 1 + } + return Array.isArray(binding) + ? binding.length + : automationUtils.stringSplit(binding).length +} + /** * The automation orchestrator is a class responsible for executing automations. * It handles the context of the automation and makes sure each step gets the correct @@ -107,7 +142,9 @@ class Orchestrator { let loopSteps = [] for (let step of automation.definition.steps) { stepCount++ - let input + let input, + iterations = 1, + iterationCount = 0 if (step.stepId === LOOP_STEP_ID) { loopStep = step loopStepNumber = stepCount @@ -116,13 +153,9 @@ class Orchestrator { if (loopStep) { input = await processObject(loopStep.inputs, this._context) + iterations = getLoopIterations(loopStep, input) } - let iterations = loopStep - ? Array.isArray(input.binding) - ? input.binding.length - : automationUtils.stringSplit(input.binding).length - : 1 - let iterationCount = 0 + for (let index = 0; index < iterations; index++) { let originalStepInput = cloneDeep(step.inputs) @@ -132,18 +165,11 @@ class Orchestrator { loopStep.inputs, cloneDeep(this._context) ) - newInput = automationUtils.cleanInputValues( - newInput, - loopStep.schema.inputs - ) let tempOutput = { items: loopSteps, iterations: iterationCount } - if ( - (loopStep.inputs.option === "Array" && - !Array.isArray(newInput.binding)) || - (loopStep.inputs.option === "String" && - typeof newInput.binding !== "string") - ) { + try { + newInput.binding = typecastForLooping(loopStep, newInput) + } catch (err) { this.updateContextAndOutput(loopStepNumber, step, tempOutput, { status: AutomationErrors.INCORRECT_TYPE, success: false, @@ -205,21 +231,13 @@ class Orchestrator { } let isFailure = false - if ( - typeof this._context.steps[loopStepNumber]?.currentItem === "object" - ) { - isFailure = Object.keys( - this._context.steps[loopStepNumber].currentItem - ).some(value => { - return ( - this._context.steps[loopStepNumber].currentItem[value] === - loopStep.inputs.failure - ) + const currentItem = this._context.steps[loopStepNumber]?.currentItem + if (currentItem && typeof currentItem === "object") { + isFailure = Object.keys(currentItem).some(value => { + return currentItem[value] === loopStep.inputs.failure }) } else { - isFailure = - this._context.steps[loopStepNumber]?.currentItem === - loopStep.inputs.failure + isFailure = currentItem && currentItem === loopStep.inputs.failure } if (isFailure) { diff --git a/packages/string-templates/package.json b/packages/string-templates/package.json index fccfec87e2..33b7b6d321 100644 --- a/packages/string-templates/package.json +++ b/packages/string-templates/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/string-templates", - "version": "1.0.151-alpha.2", + "version": "1.0.159-alpha.1", "description": "Handlebars wrapper for Budibase templating.", "main": "src/index.cjs", "module": "dist/bundle.mjs", diff --git a/packages/string-templates/src/helpers/index.js b/packages/string-templates/src/helpers/index.js index ad4082e3a4..76a4c5d2ca 100644 --- a/packages/string-templates/src/helpers/index.js +++ b/packages/string-templates/src/helpers/index.js @@ -13,6 +13,16 @@ const HTML_SWAPS = { ">": ">", } +function isObject(value) { + if (value == null || typeof value !== "object") { + return false + } + return ( + value.toString() === "[object Object]" || + (value.length > 0 && typeof value[0] === "object") + ) +} + const HELPERS = [ // external helpers new Helper(HelperFunctionNames.OBJECT, value => { @@ -22,11 +32,7 @@ const HELPERS = [ new Helper(HelperFunctionNames.JS, processJS, false), // this help is applied to all statements new Helper(HelperFunctionNames.ALL, (value, { __opts }) => { - if ( - value != null && - typeof value === "object" && - value.toString() === "[object Object]" - ) { + if (isObject(value)) { return new SafeString(JSON.stringify(value)) } // null/undefined values produce bad results diff --git a/packages/string-templates/src/processors/preprocessor.js b/packages/string-templates/src/processors/preprocessor.js index 4b296d0fc7..185a3ab38a 100644 --- a/packages/string-templates/src/processors/preprocessor.js +++ b/packages/string-templates/src/processors/preprocessor.js @@ -64,9 +64,10 @@ module.exports.processors = [ return statement } } + const testHelper = possibleHelper.trim().toLowerCase() if ( !noHelpers && - HelperNames().some(option => option.includes(possibleHelper)) + HelperNames().some(option => testHelper === option.toLowerCase()) ) { insideStatement = `(${insideStatement})` } diff --git a/packages/string-templates/test/basic.spec.js b/packages/string-templates/test/basic.spec.js index 6c85aa5fa1..8dd1aeb394 100644 --- a/packages/string-templates/test/basic.spec.js +++ b/packages/string-templates/test/basic.spec.js @@ -106,6 +106,16 @@ describe("Test that the object processing works correctly", () => { }) }) +describe("check returning objects", () => { + it("should handle an array of objects", async () => { + const json = [{a: 1},{a: 2}] + const output = await processString("{{ testing }}", { + testing: json + }) + expect(output).toEqual(JSON.stringify(json)) + }) +}) + describe("check the utility functions", () => { it("should return false for an invalid template string", () => { const valid = isValid("{{ table1.thing prop }}") diff --git a/packages/string-templates/test/escapes.spec.js b/packages/string-templates/test/escapes.spec.js index b845fddec9..a14e78d2f2 100644 --- a/packages/string-templates/test/escapes.spec.js +++ b/packages/string-templates/test/escapes.spec.js @@ -30,6 +30,11 @@ describe("Handling context properties with spaces in their name", () => { }) expect(output).toBe("testcase 1") }) + + it("should allow the use of a", async () => { + const output = await processString("{{ a }}", { a: 1 }) + expect(output).toEqual("1") + }) }) describe("attempt some complex problems", () => { diff --git a/packages/worker/package.json b/packages/worker/package.json index 75a464f1c3..fcc87e5944 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/worker", "email": "hi@budibase.com", - "version": "1.0.151-alpha.2", + "version": "1.0.159-alpha.1", "description": "Budibase background service", "main": "src/index.ts", "repository": { @@ -31,9 +31,9 @@ "author": "Budibase", "license": "GPL-3.0", "dependencies": { - "@budibase/backend-core": "^1.0.151-alpha.2", - "@budibase/pro": "1.0.151-alpha.2", - "@budibase/string-templates": "^1.0.151-alpha.2", + "@budibase/backend-core": "^1.0.159-alpha.1", + "@budibase/pro": "1.0.159-alpha.1", + "@budibase/string-templates": "^1.0.159-alpha.1", "@koa/router": "^8.0.0", "@sentry/node": "6.17.7", "@techpass/passport-openidconnect": "^0.3.0",