merge with master

.github/workflows/smoke_test.yaml

@@ -2,6 +2,9 @@ name: Budibase Smoke Test
 
 on:
  workflow_dispatch:
+ schedule:
+  - cron: "0 5 * * *" # every day at 5AM
+
 
 jobs:
   release:
@@ -23,10 +26,13 @@ jobs:
           -o packages/builder/cypress.env.json \
           -L https://api.github.com/repos/budibase/budibase-infra/contents/test/cypress.env.json
           wc -l packages/builder/cypress.env.json
-      - run: yarn test:e2e:ci
+
-        env:
+      - name: Cypress run
-          CI: true
+        id: cypress
-          name: Budibase CI
+        uses: cypress-io/github-action@v2
+        with:
+          install: false
+          command: yarn test:e2e:ci
       
       # TODO: upload recordings to s3
       # - name: Configure AWS Credentials
@@ -36,11 +42,11 @@ jobs:
       #     aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
       #     aws-region: eu-west-1
         
-      # TODO look at cypress reporters
+      - name: Discord Webhook Action
-      # - name: Discord Webhook Action
+        uses: tsickert/discord-webhook@v4.0.0
-      #   uses: tsickert/discord-webhook@v4.0.0
+        with:
-      #   with:
+          webhook-url: ${{ secrets.BUDI_QA_WEBHOOK }}
-      #     webhook-url: ${{ secrets.PROD_DEPLOY_WEBHOOK_URL }}
+          content: "Smoke test run completed with ${{ steps.cypress.outcome }}. See results at ${{ steps.cypress.dashboardUrl }}"
-      #     content: "Production Deployment Complete: ${{ env.RELEASE_VERSION }} deployed to Budibase Cloud."
+          embed-title: ${{ steps.cypress.outcome }}
-      #     embed-title: ${{ env.RELEASE_VERSION }}
+          embed-color: ${{ steps.cypress.outcome == 'success' && '3066993' || '15548997' }}
 

hosting/nginx.prod.conf.hbs

@@ -48,7 +48,7 @@ http {
     add_header X-Frame-Options SAMEORIGIN always;
     add_header X-Content-Type-Options nosniff always;
     add_header X-XSS-Protection "1; mode=block" always;
-    add_header Content-Security-Policy  "default-src 'self'; script-src 'self' 'unsafe-inline' 'unsafe-eval' https://cdn.budi.live https://js.intercomcdn.com https://widget.intercom.io; style-src 'self' 'unsafe-inline' https://cdn.jsdelivr.net https://fonts.googleapis.com https://rsms.me https://maxcdn.bootstrapcdn.com; object-src 'none'; base-uri 'self'; connect-src 'self' https://api-iam.intercom.io https://app.posthog.com wss://nexus-websocket-a.intercom.io ; font-src 'self' data https://cdn.jsdelivr.net https://fonts.gstatic.com https://rsms.me https://maxcdn.bootstrapcdn.com; frame-src 'self' https:; img-src http: https: data; manifest-src 'self'; media-src 'self'; worker-src 'none';" always;
+    add_header Content-Security-Policy  "default-src 'self'; script-src 'self' 'unsafe-inline' 'unsafe-eval' https://cdn.budi.live https://js.intercomcdn.com https://widget.intercom.io; style-src 'self' 'unsafe-inline' https://cdn.jsdelivr.net https://fonts.googleapis.com https://rsms.me https://maxcdn.bootstrapcdn.com; object-src 'none'; base-uri 'self'; connect-src 'self' https://api-iam.intercom.io https://app.posthog.com wss://nexus-websocket-a.intercom.io ; font-src 'self' data: https://cdn.jsdelivr.net https://fonts.gstatic.com https://rsms.me https://maxcdn.bootstrapcdn.com; frame-src 'self' https:; img-src http: https: data:; manifest-src 'self'; media-src 'self'; worker-src 'none';" always;
 
     # upstreams
     set $apps {{ apps }};

packages/backend-core/src/security/roles.js

@@ -1,5 +1,5 @@
 const { cloneDeep } = require("lodash/fp")
-const { BUILTIN_PERMISSION_IDS } = require("./permissions")
+const { BUILTIN_PERMISSION_IDS, PermissionLevels } = require("./permissions")
 const {
   generateRoleID,
   getRoleParams,
@@ -180,6 +180,20 @@ exports.getUserRoleHierarchy = async (userRoleId, opts = { idOnly: true }) => {
   return opts.idOnly ? roles.map(role => role._id) : roles
 }
 
+// this function checks that the provided permissions are in an array format
+// some templates/older apps will use a simple string instead of array for roles
+// convert the string to an array using the theory that write is higher than read
+exports.checkForRoleResourceArray = (rolePerms, resourceId) => {
+  if (rolePerms && !Array.isArray(rolePerms[resourceId])) {
+    const permLevel = rolePerms[resourceId]
+    rolePerms[resourceId] = [permLevel]
+    if (permLevel === PermissionLevels.WRITE) {
+      rolePerms[resourceId].push(PermissionLevels.READ)
+    }
+  }
+  return rolePerms
+}
+
 /**
  * Given an app ID this will retrieve all of the roles that are currently within that app.
  * @return {Promise<object[]>} An array of the role objects that were found.
@@ -209,15 +223,27 @@ exports.getAllRoles = async appId => {
       roles.push(Object.assign(builtinRole, dbBuiltin))
     }
   }
+  // check permissions
+  for (let role of roles) {
+    if (!role.permissions) {
+      continue
+    }
+    for (let resourceId of Object.keys(role.permissions)) {
+      role.permissions = exports.checkForRoleResourceArray(
+        role.permissions,
+        resourceId
+      )
+    }
+  }
   return roles
 }
 
 /**
- * This retrieves the required role
+ * This retrieves the required role for a resource
- * @param permLevel
+ * @param permLevel The level of request
- * @param resourceId
+ * @param resourceId The resource being requested
- * @param subResourceId
+ * @param subResourceId The sub resource being requested
- * @return {Promise<{permissions}|Object>}
+ * @return {Promise<{permissions}|Object>} returns the permissions required to access.
  */
 exports.getRequiredResourceRole = async (
   permLevel,

packages/server/specs/openapi.json

@@ -1260,12 +1260,32 @@
         ]
       },
       "executeQuery": {
-        "description": "The query body must contain the required parameters for the query, this depends on query type, setup and bindings.",
+        "description": "The parameters required for executing a query.",
         "type": "object",
+        "properties": {
+          "parameters": {
+            "type": "object",
+            "description": "This contains the required parameters for the query, this depends on query type, setup and bindings.",
             "additionalProperties": {
               "description": "Key value properties of any type, depending on the query output schema."
             }
           },
+          "pagination": {
+            "type": "object",
+            "description": "For supported query types (currently on REST) pagination can be performed using these properties.",
+            "properties": {
+              "page": {
+                "type": "string",
+                "description": "The page which has been returned from a previous query."
+              },
+              "limit": {
+                "type": "number",
+                "description": "The number of rows to return per page."
+              }
+            }
+          }
+        }
+      },
       "executeQueryOutput": {
         "type": "object",
         "properties": {

packages/server/specs/openapi.yaml

@@ -951,11 +951,27 @@ components:
       required:
         - data
     executeQuery:
-      description: The query body must contain the required parameters for the query,
+      description: The parameters required for executing a query.
-        this depends on query type, setup and bindings.
       type: object
+      properties:
+        parameters:
+          type: object
+          description: This contains the required parameters for the query, this depends
+            on query type, setup and bindings.
           additionalProperties:
-        description: Key value properties of any type, depending on the query output schema.
+            description: Key value properties of any type, depending on the query output
+              schema.
+        pagination:
+          type: object
+          description: For supported query types (currently on REST) pagination can be
+            performed using these properties.
+          properties:
+            page:
+              type: string
+              description: The page which has been returned from a previous query.
+            limit:
+              type: number
+              description: The number of rows to return per page.
     executeQueryOutput:
       type: object
       properties:

packages/server/specs/resources/query.js

@@ -124,13 +124,35 @@ const querySchema = object(
 )
 
 const executeQuerySchema = {
-  description:
+  description: "The parameters required for executing a query.",
-    "The query body must contain the required parameters for the query, this depends on query type, setup and bindings.",
   type: "object",
+  properties: {
+    parameters: {
+      type: "object",
+      description:
+        "This contains the required parameters for the query, this depends on query type, setup and bindings.",
       additionalProperties: {
         description:
           "Key value properties of any type, depending on the query output schema.",
       },
+    },
+    pagination: {
+      type: "object",
+      description:
+        "For supported query types (currently on REST) pagination can be performed using these properties.",
+      properties: {
+        page: {
+          type: "string",
+          description:
+            "The page which has been returned from a previous query.",
+        },
+        limit: {
+          type: "number",
+          description: "The number of rows to return per page.",
+        },
+      },
+    },
+  },
 }
 
 const executeQueryOutputSchema = object(

packages/server/src/api/controllers/permission.js

@@ -4,6 +4,7 @@ const {
   getDBRoleID,
   getExternalRoleID,
   getBuiltinRoles,
+  checkForRoleResourceArray,
 } = require("@budibase/backend-core/roles")
 const { getRoleParams } = require("../../db/utils")
 const {
@@ -144,12 +145,11 @@ exports.getResourcePerms = async function (ctx) {
   for (let level of SUPPORTED_LEVELS) {
     // update the various roleIds in the resource permissions
     for (let role of roles) {
-      const rolePerms = role.permissions
+      const rolePerms = checkForRoleResourceArray(role.permissions, resourceId)
       if (
         rolePerms &&
         rolePerms[resourceId] &&
-        (rolePerms[resourceId] === level ||
+        rolePerms[resourceId].indexOf(level) !== -1
-          rolePerms[resourceId].indexOf(level) !== -1)
       ) {
         permissions[level] = getExternalRoleID(role._id)
       }

packages/server/src/api/routes/table.js

@@ -40,7 +40,7 @@ router
   .get(
     "/api/tables/:tableId",
     paramResource("tableId"),
-    authorized(PermissionTypes.TABLE, PermissionLevels.READ),
+    authorized(PermissionTypes.TABLE, PermissionLevels.READ, { schema: true }),
     tableController.find
   )
   /**

packages/server/src/automations/utils.js

@@ -17,10 +17,14 @@ const Runner = new Thread(ThreadType.AUTOMATION)
 exports.processEvent = async job => {
   try {
     // need to actually await these so that an error can be captured properly
+    console.log(
+      `${job.data.automation.appId} automation ${job.data.automation._id} running`
+    )
     return await Runner.run(job)
   } catch (err) {
+    const errJson = JSON.stringify(err)
     console.error(
-      `${job.data.automation.appId} automation ${job.data.automation._id} was unable to run - ${err}`
+      `${job.data.automation.appId} automation ${job.data.automation._id} was unable to run - ${errJson}`
     )
     console.trace(err)
     return { err }

packages/server/src/definitions/datasource.ts

@@ -182,11 +182,7 @@ export interface QueryJson {
 
 export interface SqlQuery {
   sql: string
-  bindings?:
+  bindings?: string[]
-    | string[]
-    | {
-        [key: string]: any
-      }
 }
 
 export interface QueryOptions {

packages/server/src/definitions/openapi.ts

@@ -932,11 +932,21 @@ export interface components {
               }
         }
         /** @description The ID of the table. */
-        _id: string
+        _id: string;
-      }[]
+      }[];
-    }
+    };
-    /** @description The query body must contain the required parameters for the query, this depends on query type, setup and bindings. */
+    /** @description The parameters required for executing a query. */
-    executeQuery: { [key: string]: unknown }
+    executeQuery: {
+      /** @description This contains the required parameters for the query, this depends on query type, setup and bindings. */
+      parameters?: { [key: string]: unknown };
+      /** @description For supported query types (currently on REST) pagination can be performed using these properties. */
+      pagination?: {
+        /** @description The page which has been returned from a previous query. */
+        page?: string;
+        /** @description The number of rows to return per page. */
+        limit?: number;
+      };
+    };
     executeQueryOutput: {
       /** @description The data response from the query. */
       data: { [key: string]: unknown }[]

packages/server/src/integrations/mysql.ts

@@ -80,6 +80,20 @@ module MySQLModule {
     },
   }
 
+  function bindingTypeCoerce(bindings: any[]) {
+    for (let i = 0; i < bindings.length; i++) {
+      const binding = bindings[i]
+      if (typeof binding !== "string") {
+        continue
+      }
+      const matches = binding.match(/^\d*/g)
+      if (matches && matches[0] !== "" && !isNaN(Number(matches[0]))) {
+        bindings[i] = parseFloat(binding)
+      }
+    }
+    return bindings
+  }
+
   class MySQLIntegration extends Sql implements DatasourcePlus {
     private config: MySQLConfig
     private client: any
@@ -122,7 +136,7 @@ module MySQLModule {
         // Node MySQL is callback based, so we must wrap our call in a promise
         const response = await this.client.query(
           query.sql,
-          query.bindings || []
+          bindingTypeCoerce(query.bindings || [])
         )
         return response[0]
       } finally {

packages/server/src/middleware/authorized.js

@@ -5,6 +5,7 @@ const {
 } = require("@budibase/backend-core/roles")
 const {
   PermissionTypes,
+  PermissionLevels,
   doesHaveBasePermission,
 } = require("@budibase/backend-core/permissions")
 const builderMiddleware = require("./builder")
@@ -64,7 +65,7 @@ const checkAuthorizedResource = async (
 }
 
 module.exports =
-  (permType, permLevel = null) =>
+  (permType, permLevel = null, opts = { schema: false }) =>
   async (ctx, next) => {
     // webhooks don't need authentication, each webhook unique
     // also internal requests (between services) don't need authorized
@@ -81,15 +82,25 @@ module.exports =
     await builderMiddleware(ctx, permType)
 
     // get the resource roles
-    let resourceRoles = []
+    let resourceRoles = [],
+      otherLevelRoles
+    const otherLevel =
+      permLevel === PermissionLevels.READ
+        ? PermissionLevels.WRITE
+        : PermissionLevels.READ
     const appId = getAppId()
     if (appId && hasResource(ctx)) {
       resourceRoles = await getRequiredResourceRole(permLevel, ctx)
+      if (opts && opts.schema) {
+        otherLevelRoles = await getRequiredResourceRole(otherLevel, ctx)
+      }
     }
 
     // if the resource is public, proceed
-    const isPublicResource = resourceRoles.includes(BUILTIN_ROLE_IDS.PUBLIC)
+    if (
-    if (isPublicResource) {
+      resourceRoles.includes(BUILTIN_ROLE_IDS.PUBLIC) ||
+      (otherLevelRoles && otherLevelRoles.includes(BUILTIN_ROLE_IDS.PUBLIC))
+    ) {
       return next()
     }
 
@@ -98,8 +109,17 @@ module.exports =
       return ctx.throw(403, "Session not authenticated")
     }
 
+    try {
       // check authorized
       await checkAuthorized(ctx, resourceRoles, permType, permLevel)
+    } catch (err) {
+      // this is a schema, check if
+      if (opts && opts.schema && permLevel) {
+        await checkAuthorized(ctx, otherLevelRoles, permType, otherLevel)
+      } else {
+        throw err
+      }
+    }
 
     // csrf protection
     return csrf(ctx, next)

packages/worker/src/api/routes/tests/realEmail.spec.js

@@ -3,6 +3,9 @@ const { EmailTemplatePurpose } = require("../../../constants")
 const nodemailer = require("nodemailer")
 const fetch = require("node-fetch")
 
+// for the real email tests give them a long time to try complete/fail
+jest.setTimeout(30000)
+
 describe("/api/global/email", () => {
   let request = setup.getRequest()
   let config = setup.getConfig()
@@ -27,6 +30,7 @@ describe("/api/global/email", () => {
           userId: user._id,
         })
         .set(config.defaultHeaders())
+        .timeout(20000)
       // ethereal hiccup, can't test right now
       if (res.status >= 300) {
         return
@@ -39,7 +43,7 @@ describe("/api/global/email", () => {
       text = await response.text()
     } catch (err) {
       // ethereal hiccup, can't test right now
-      if (parseInt(err.status) >= 300) {
+      if (parseInt(err.status) >= 300 || (err && err.errno === "ETIME")) {
         return
       } else {
         throw err