diff --git a/.github/workflows/smoke_test.yaml b/.github/workflows/smoke_test.yaml index 745fed1306..52374b3960 100644 --- a/.github/workflows/smoke_test.yaml +++ b/.github/workflows/smoke_test.yaml @@ -2,6 +2,9 @@ name: Budibase Smoke Test on: workflow_dispatch: + schedule: + - cron: "0 5 * * *" # every day at 5AM + jobs: release: @@ -23,10 +26,13 @@ jobs: -o packages/builder/cypress.env.json \ -L https://api.github.com/repos/budibase/budibase-infra/contents/test/cypress.env.json wc -l packages/builder/cypress.env.json - - run: yarn test:e2e:ci - env: - CI: true - name: Budibase CI + + - name: Cypress run + id: cypress + uses: cypress-io/github-action@v2 + with: + install: false + command: yarn test:e2e:ci # TODO: upload recordings to s3 # - name: Configure AWS Credentials @@ -36,11 +42,11 @@ jobs: # aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} # aws-region: eu-west-1 - # TODO look at cypress reporters - # - name: Discord Webhook Action - # uses: tsickert/discord-webhook@v4.0.0 - # with: - # webhook-url: ${{ secrets.PROD_DEPLOY_WEBHOOK_URL }} - # content: "Production Deployment Complete: ${{ env.RELEASE_VERSION }} deployed to Budibase Cloud." - # embed-title: ${{ env.RELEASE_VERSION }} + - name: Discord Webhook Action + uses: tsickert/discord-webhook@v4.0.0 + with: + webhook-url: ${{ secrets.BUDI_QA_WEBHOOK }} + content: "Smoke test run completed with ${{ steps.cypress.outcome }}. See results at ${{ steps.cypress.dashboardUrl }}" + embed-title: ${{ steps.cypress.outcome }} + embed-color: ${{ steps.cypress.outcome == 'success' && '3066993' || '15548997' }} diff --git a/hosting/nginx.prod.conf.hbs b/hosting/nginx.prod.conf.hbs index 0ab7ed2c7e..8560ad0198 100644 --- a/hosting/nginx.prod.conf.hbs +++ b/hosting/nginx.prod.conf.hbs @@ -48,7 +48,7 @@ http { add_header X-Frame-Options SAMEORIGIN always; add_header X-Content-Type-Options nosniff always; add_header X-XSS-Protection "1; mode=block" always; - add_header Content-Security-Policy "default-src 'self'; script-src 'self' 'unsafe-inline' 'unsafe-eval' https://cdn.budi.live https://js.intercomcdn.com https://widget.intercom.io; style-src 'self' 'unsafe-inline' https://cdn.jsdelivr.net https://fonts.googleapis.com https://rsms.me https://maxcdn.bootstrapcdn.com; object-src 'none'; base-uri 'self'; connect-src 'self' https://api-iam.intercom.io https://app.posthog.com wss://nexus-websocket-a.intercom.io ; font-src 'self' data https://cdn.jsdelivr.net https://fonts.gstatic.com https://rsms.me https://maxcdn.bootstrapcdn.com; frame-src 'self' https:; img-src http: https: data; manifest-src 'self'; media-src 'self'; worker-src 'none';" always; + add_header Content-Security-Policy "default-src 'self'; script-src 'self' 'unsafe-inline' 'unsafe-eval' https://cdn.budi.live https://js.intercomcdn.com https://widget.intercom.io; style-src 'self' 'unsafe-inline' https://cdn.jsdelivr.net https://fonts.googleapis.com https://rsms.me https://maxcdn.bootstrapcdn.com; object-src 'none'; base-uri 'self'; connect-src 'self' https://api-iam.intercom.io https://app.posthog.com wss://nexus-websocket-a.intercom.io ; font-src 'self' data: https://cdn.jsdelivr.net https://fonts.gstatic.com https://rsms.me https://maxcdn.bootstrapcdn.com; frame-src 'self' https:; img-src http: https: data:; manifest-src 'self'; media-src 'self'; worker-src 'none';" always; # upstreams set $apps {{ apps }}; diff --git a/packages/backend-core/src/security/roles.js b/packages/backend-core/src/security/roles.js index 11abc70bdd..8535cdc716 100644 --- a/packages/backend-core/src/security/roles.js +++ b/packages/backend-core/src/security/roles.js @@ -1,5 +1,5 @@ const { cloneDeep } = require("lodash/fp") -const { BUILTIN_PERMISSION_IDS } = require("./permissions") +const { BUILTIN_PERMISSION_IDS, PermissionLevels } = require("./permissions") const { generateRoleID, getRoleParams, @@ -180,6 +180,20 @@ exports.getUserRoleHierarchy = async (userRoleId, opts = { idOnly: true }) => { return opts.idOnly ? roles.map(role => role._id) : roles } +// this function checks that the provided permissions are in an array format +// some templates/older apps will use a simple string instead of array for roles +// convert the string to an array using the theory that write is higher than read +exports.checkForRoleResourceArray = (rolePerms, resourceId) => { + if (rolePerms && !Array.isArray(rolePerms[resourceId])) { + const permLevel = rolePerms[resourceId] + rolePerms[resourceId] = [permLevel] + if (permLevel === PermissionLevels.WRITE) { + rolePerms[resourceId].push(PermissionLevels.READ) + } + } + return rolePerms +} + /** * Given an app ID this will retrieve all of the roles that are currently within that app. * @return {Promise} An array of the role objects that were found. @@ -209,15 +223,27 @@ exports.getAllRoles = async appId => { roles.push(Object.assign(builtinRole, dbBuiltin)) } } + // check permissions + for (let role of roles) { + if (!role.permissions) { + continue + } + for (let resourceId of Object.keys(role.permissions)) { + role.permissions = exports.checkForRoleResourceArray( + role.permissions, + resourceId + ) + } + } return roles } /** - * This retrieves the required role - * @param permLevel - * @param resourceId - * @param subResourceId - * @return {Promise<{permissions}|Object>} + * This retrieves the required role for a resource + * @param permLevel The level of request + * @param resourceId The resource being requested + * @param subResourceId The sub resource being requested + * @return {Promise<{permissions}|Object>} returns the permissions required to access. */ exports.getRequiredResourceRole = async ( permLevel, diff --git a/packages/server/specs/openapi.json b/packages/server/specs/openapi.json index 6ca1a5b9fb..30ef0a8c6f 100644 --- a/packages/server/specs/openapi.json +++ b/packages/server/specs/openapi.json @@ -1260,10 +1260,30 @@ ] }, "executeQuery": { - "description": "The query body must contain the required parameters for the query, this depends on query type, setup and bindings.", + "description": "The parameters required for executing a query.", "type": "object", - "additionalProperties": { - "description": "Key value properties of any type, depending on the query output schema." + "properties": { + "parameters": { + "type": "object", + "description": "This contains the required parameters for the query, this depends on query type, setup and bindings.", + "additionalProperties": { + "description": "Key value properties of any type, depending on the query output schema." + } + }, + "pagination": { + "type": "object", + "description": "For supported query types (currently on REST) pagination can be performed using these properties.", + "properties": { + "page": { + "type": "string", + "description": "The page which has been returned from a previous query." + }, + "limit": { + "type": "number", + "description": "The number of rows to return per page." + } + } + } } }, "executeQueryOutput": { diff --git a/packages/server/specs/openapi.yaml b/packages/server/specs/openapi.yaml index c4d9808c86..ed55df953a 100644 --- a/packages/server/specs/openapi.yaml +++ b/packages/server/specs/openapi.yaml @@ -951,11 +951,27 @@ components: required: - data executeQuery: - description: The query body must contain the required parameters for the query, - this depends on query type, setup and bindings. + description: The parameters required for executing a query. type: object - additionalProperties: - description: Key value properties of any type, depending on the query output schema. + properties: + parameters: + type: object + description: This contains the required parameters for the query, this depends + on query type, setup and bindings. + additionalProperties: + description: Key value properties of any type, depending on the query output + schema. + pagination: + type: object + description: For supported query types (currently on REST) pagination can be + performed using these properties. + properties: + page: + type: string + description: The page which has been returned from a previous query. + limit: + type: number + description: The number of rows to return per page. executeQueryOutput: type: object properties: diff --git a/packages/server/specs/resources/query.js b/packages/server/specs/resources/query.js index df532c9a3a..d4a4882fb2 100644 --- a/packages/server/specs/resources/query.js +++ b/packages/server/specs/resources/query.js @@ -124,12 +124,34 @@ const querySchema = object( ) const executeQuerySchema = { - description: - "The query body must contain the required parameters for the query, this depends on query type, setup and bindings.", + description: "The parameters required for executing a query.", type: "object", - additionalProperties: { - description: - "Key value properties of any type, depending on the query output schema.", + properties: { + parameters: { + type: "object", + description: + "This contains the required parameters for the query, this depends on query type, setup and bindings.", + additionalProperties: { + description: + "Key value properties of any type, depending on the query output schema.", + }, + }, + pagination: { + type: "object", + description: + "For supported query types (currently on REST) pagination can be performed using these properties.", + properties: { + page: { + type: "string", + description: + "The page which has been returned from a previous query.", + }, + limit: { + type: "number", + description: "The number of rows to return per page.", + }, + }, + }, }, } diff --git a/packages/server/src/api/controllers/permission.js b/packages/server/src/api/controllers/permission.js index 0e37a3e7d3..e1547eb597 100644 --- a/packages/server/src/api/controllers/permission.js +++ b/packages/server/src/api/controllers/permission.js @@ -4,6 +4,7 @@ const { getDBRoleID, getExternalRoleID, getBuiltinRoles, + checkForRoleResourceArray, } = require("@budibase/backend-core/roles") const { getRoleParams } = require("../../db/utils") const { @@ -144,12 +145,11 @@ exports.getResourcePerms = async function (ctx) { for (let level of SUPPORTED_LEVELS) { // update the various roleIds in the resource permissions for (let role of roles) { - const rolePerms = role.permissions + const rolePerms = checkForRoleResourceArray(role.permissions, resourceId) if ( rolePerms && rolePerms[resourceId] && - (rolePerms[resourceId] === level || - rolePerms[resourceId].indexOf(level) !== -1) + rolePerms[resourceId].indexOf(level) !== -1 ) { permissions[level] = getExternalRoleID(role._id) } diff --git a/packages/server/src/api/routes/table.js b/packages/server/src/api/routes/table.js index 4d20b98962..5d2378710d 100644 --- a/packages/server/src/api/routes/table.js +++ b/packages/server/src/api/routes/table.js @@ -40,7 +40,7 @@ router .get( "/api/tables/:tableId", paramResource("tableId"), - authorized(PermissionTypes.TABLE, PermissionLevels.READ), + authorized(PermissionTypes.TABLE, PermissionLevels.READ, { schema: true }), tableController.find ) /** diff --git a/packages/server/src/automations/utils.js b/packages/server/src/automations/utils.js index 64007f28b4..425ccec9de 100644 --- a/packages/server/src/automations/utils.js +++ b/packages/server/src/automations/utils.js @@ -17,10 +17,14 @@ const Runner = new Thread(ThreadType.AUTOMATION) exports.processEvent = async job => { try { // need to actually await these so that an error can be captured properly + console.log( + `${job.data.automation.appId} automation ${job.data.automation._id} running` + ) return await Runner.run(job) } catch (err) { + const errJson = JSON.stringify(err) console.error( - `${job.data.automation.appId} automation ${job.data.automation._id} was unable to run - ${err}` + `${job.data.automation.appId} automation ${job.data.automation._id} was unable to run - ${errJson}` ) console.trace(err) return { err } diff --git a/packages/server/src/definitions/datasource.ts b/packages/server/src/definitions/datasource.ts index 88115237a0..4df08683f0 100644 --- a/packages/server/src/definitions/datasource.ts +++ b/packages/server/src/definitions/datasource.ts @@ -182,11 +182,7 @@ export interface QueryJson { export interface SqlQuery { sql: string - bindings?: - | string[] - | { - [key: string]: any - } + bindings?: string[] } export interface QueryOptions { diff --git a/packages/server/src/definitions/openapi.ts b/packages/server/src/definitions/openapi.ts index 532869069b..0fef60f4a0 100644 --- a/packages/server/src/definitions/openapi.ts +++ b/packages/server/src/definitions/openapi.ts @@ -932,11 +932,21 @@ export interface components { } } /** @description The ID of the table. */ - _id: string - }[] - } - /** @description The query body must contain the required parameters for the query, this depends on query type, setup and bindings. */ - executeQuery: { [key: string]: unknown } + _id: string; + }[]; + }; + /** @description The parameters required for executing a query. */ + executeQuery: { + /** @description This contains the required parameters for the query, this depends on query type, setup and bindings. */ + parameters?: { [key: string]: unknown }; + /** @description For supported query types (currently on REST) pagination can be performed using these properties. */ + pagination?: { + /** @description The page which has been returned from a previous query. */ + page?: string; + /** @description The number of rows to return per page. */ + limit?: number; + }; + }; executeQueryOutput: { /** @description The data response from the query. */ data: { [key: string]: unknown }[] diff --git a/packages/server/src/integrations/mysql.ts b/packages/server/src/integrations/mysql.ts index 8b2c9ac944..6f009bbd4a 100644 --- a/packages/server/src/integrations/mysql.ts +++ b/packages/server/src/integrations/mysql.ts @@ -80,6 +80,20 @@ module MySQLModule { }, } + function bindingTypeCoerce(bindings: any[]) { + for (let i = 0; i < bindings.length; i++) { + const binding = bindings[i] + if (typeof binding !== "string") { + continue + } + const matches = binding.match(/^\d*/g) + if (matches && matches[0] !== "" && !isNaN(Number(matches[0]))) { + bindings[i] = parseFloat(binding) + } + } + return bindings + } + class MySQLIntegration extends Sql implements DatasourcePlus { private config: MySQLConfig private client: any @@ -122,7 +136,7 @@ module MySQLModule { // Node MySQL is callback based, so we must wrap our call in a promise const response = await this.client.query( query.sql, - query.bindings || [] + bindingTypeCoerce(query.bindings || []) ) return response[0] } finally { diff --git a/packages/server/src/middleware/authorized.js b/packages/server/src/middleware/authorized.js index c8d6497ca3..d6f904290a 100644 --- a/packages/server/src/middleware/authorized.js +++ b/packages/server/src/middleware/authorized.js @@ -5,6 +5,7 @@ const { } = require("@budibase/backend-core/roles") const { PermissionTypes, + PermissionLevels, doesHaveBasePermission, } = require("@budibase/backend-core/permissions") const builderMiddleware = require("./builder") @@ -64,7 +65,7 @@ const checkAuthorizedResource = async ( } module.exports = - (permType, permLevel = null) => + (permType, permLevel = null, opts = { schema: false }) => async (ctx, next) => { // webhooks don't need authentication, each webhook unique // also internal requests (between services) don't need authorized @@ -81,15 +82,25 @@ module.exports = await builderMiddleware(ctx, permType) // get the resource roles - let resourceRoles = [] + let resourceRoles = [], + otherLevelRoles + const otherLevel = + permLevel === PermissionLevels.READ + ? PermissionLevels.WRITE + : PermissionLevels.READ const appId = getAppId() if (appId && hasResource(ctx)) { resourceRoles = await getRequiredResourceRole(permLevel, ctx) + if (opts && opts.schema) { + otherLevelRoles = await getRequiredResourceRole(otherLevel, ctx) + } } // if the resource is public, proceed - const isPublicResource = resourceRoles.includes(BUILTIN_ROLE_IDS.PUBLIC) - if (isPublicResource) { + if ( + resourceRoles.includes(BUILTIN_ROLE_IDS.PUBLIC) || + (otherLevelRoles && otherLevelRoles.includes(BUILTIN_ROLE_IDS.PUBLIC)) + ) { return next() } @@ -98,8 +109,17 @@ module.exports = return ctx.throw(403, "Session not authenticated") } - // check authorized - await checkAuthorized(ctx, resourceRoles, permType, permLevel) + try { + // check authorized + await checkAuthorized(ctx, resourceRoles, permType, permLevel) + } catch (err) { + // this is a schema, check if + if (opts && opts.schema && permLevel) { + await checkAuthorized(ctx, otherLevelRoles, permType, otherLevel) + } else { + throw err + } + } // csrf protection return csrf(ctx, next) diff --git a/packages/worker/src/api/routes/tests/realEmail.spec.js b/packages/worker/src/api/routes/tests/realEmail.spec.js index d0cfd24010..28d0151284 100644 --- a/packages/worker/src/api/routes/tests/realEmail.spec.js +++ b/packages/worker/src/api/routes/tests/realEmail.spec.js @@ -3,6 +3,9 @@ const { EmailTemplatePurpose } = require("../../../constants") const nodemailer = require("nodemailer") const fetch = require("node-fetch") +// for the real email tests give them a long time to try complete/fail +jest.setTimeout(30000) + describe("/api/global/email", () => { let request = setup.getRequest() let config = setup.getConfig() @@ -27,6 +30,7 @@ describe("/api/global/email", () => { userId: user._id, }) .set(config.defaultHeaders()) + .timeout(20000) // ethereal hiccup, can't test right now if (res.status >= 300) { return @@ -39,7 +43,7 @@ describe("/api/global/email", () => { text = await response.text() } catch (err) { // ethereal hiccup, can't test right now - if (parseInt(err.status) >= 300) { + if (parseInt(err.status) >= 300 || (err && err.errno === "ETIME")) { return } else { throw err