Merge branch 'develop' into feature/remove-analytics-platform-url

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "0.9.140-alpha.11",
+  "version": "0.9.140-alpha.12",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

packages/auth/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/auth",
-  "version": "0.9.140-alpha.11",
+  "version": "0.9.140-alpha.12",
   "description": "Authentication middlewares for budibase builder and apps",
   "main": "src/index.js",
   "author": "Budibase",

packages/auth/src/security/permissions.js

@@ -139,8 +139,7 @@ exports.doesHaveResourcePermission = (
   // set foundSub to not subResourceId, incase there is no subResource
   let foundMain = false,
     foundSub = false
-  for (let [resource, level] of Object.entries(permissions)) {
+  for (let [resource, levels] of Object.entries(permissions)) {
-    const levels = getAllowedLevels(level)
     if (resource === resourceId && levels.indexOf(permLevel) !== -1) {
       foundMain = true
     }
@@ -177,10 +176,6 @@ exports.doesHaveBasePermission = (permType, permLevel, permissionIds) => {
   return false
 }
 
-exports.higherPermission = (perm1, perm2) => {
-  return levelToNumber(perm1) > levelToNumber(perm2) ? perm1 : perm2
-}
-
 exports.isPermissionLevelHigherThanRead = level => {
   return levelToNumber(level) > 1
 }

packages/auth/src/security/roles.js

@@ -1,6 +1,6 @@
 const { getDB } = require("../db")
 const { cloneDeep } = require("lodash/fp")
-const { BUILTIN_PERMISSION_IDS, higherPermission } = require("./permissions")
+const { BUILTIN_PERMISSION_IDS } = require("./permissions")
 const {
   generateRoleID,
   getRoleParams,
@@ -193,8 +193,17 @@ exports.getUserPermissions = async (appId, userRoleId) => {
   const permissions = {}
   for (let role of rolesHierarchy) {
     if (role.permissions) {
-      for (let [resource, level] of Object.entries(role.permissions)) {
+      for (let [resource, levels] of Object.entries(role.permissions)) {
-        permissions[resource] = higherPermission(permissions[resource], level)
+        if (!permissions[resource]) {
+          permissions[resource] = []
+        }
+        const permsSet = new Set(permissions[resource])
+        if (Array.isArray(levels)) {
+          levels.forEach(level => permsSet.add(level))
+        } else {
+          permsSet.add(levels)
+        }
+        permissions[resource] = [...permsSet]
       }
     }
   }

packages/bbui/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/bbui",
   "description": "A UI solution used in the different Budibase projects.",
-  "version": "0.9.140-alpha.11",
+  "version": "0.9.140-alpha.12",
   "license": "AGPL-3.0",
   "svelte": "src/index.js",
   "module": "dist/bbui.es.js",

packages/builder/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/builder",
-  "version": "0.9.140-alpha.11",
+  "version": "0.9.140-alpha.12",
   "license": "AGPL-3.0",
   "private": true,
   "scripts": {
@@ -65,10 +65,10 @@
     }
   },
   "dependencies": {
-    "@budibase/bbui": "^0.9.140-alpha.11",
+    "@budibase/bbui": "^0.9.140-alpha.12",
-    "@budibase/client": "^0.9.140-alpha.11",
+    "@budibase/client": "^0.9.140-alpha.12",
     "@budibase/colorpicker": "1.1.2",
-    "@budibase/string-templates": "^0.9.140-alpha.11",
+    "@budibase/string-templates": "^0.9.140-alpha.12",
     "@sentry/browser": "5.19.1",
     "@spectrum-css/page": "^3.0.1",
     "@spectrum-css/vars": "^3.0.1",

packages/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/cli",
-  "version": "0.9.140-alpha.11",
+  "version": "0.9.140-alpha.12",
   "description": "Budibase CLI, for developers, self hosting and migrations.",
   "main": "src/index.js",
   "bin": {

packages/client/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/client",
-  "version": "0.9.140-alpha.11",
+  "version": "0.9.140-alpha.12",
   "license": "MPL-2.0",
   "module": "dist/budibase-client.js",
   "main": "dist/budibase-client.js",
@@ -19,9 +19,9 @@
     "dev:builder": "rollup -cw"
   },
   "dependencies": {
-    "@budibase/bbui": "^0.9.140-alpha.11",
+    "@budibase/bbui": "^0.9.140-alpha.12",
     "@budibase/standard-components": "^0.9.139",
-    "@budibase/string-templates": "^0.9.140-alpha.11",
+    "@budibase/string-templates": "^0.9.140-alpha.12",
     "regexparam": "^1.3.0",
     "shortid": "^2.2.15",
     "svelte-spa-router": "^3.0.5"

packages/server/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "0.9.140-alpha.11",
+  "version": "0.9.140-alpha.12",
   "description": "Budibase Web Server",
   "main": "src/index.js",
   "repository": {
@@ -64,9 +64,9 @@
   "author": "Budibase",
   "license": "AGPL-3.0-or-later",
   "dependencies": {
-    "@budibase/auth": "^0.9.140-alpha.11",
+    "@budibase/auth": "^0.9.140-alpha.12",
-    "@budibase/client": "^0.9.140-alpha.11",
+    "@budibase/client": "^0.9.140-alpha.12",
-    "@budibase/string-templates": "^0.9.140-alpha.11",
+    "@budibase/string-templates": "^0.9.140-alpha.12",
     "@elastic/elasticsearch": "7.10.0",
     "@koa/router": "8.0.0",
     "@sendgrid/mail": "7.1.1",

packages/server/scripts/integrations/pg-json/docker-compose.yml

@@ -0,0 +1,28 @@
+version: "3.8"
+services:
+  db:
+    container_name: postgres-json
+    image: postgres
+    restart: always
+    environment:
+      POSTGRES_USER: root
+      POSTGRES_PASSWORD: root
+      POSTGRES_DB: main
+    ports:
+      - "5432:5432"
+    volumes:
+      #- pg_data:/var/lib/postgresql/data/
+      - ./init.sql:/docker-entrypoint-initdb.d/init.sql
+
+  pgadmin:
+    container_name: pgadmin-json
+    image: dpage/pgadmin4
+    restart: always
+    environment:
+      PGADMIN_DEFAULT_EMAIL: root@root.com
+      PGADMIN_DEFAULT_PASSWORD: root
+    ports:
+      - "5050:80"
+
+#volumes:
+#  pg_data:

packages/server/scripts/integrations/pg-json/init.sql

@@ -0,0 +1,22 @@
+SELECT 'CREATE DATABASE main'
+WHERE NOT EXISTS (SELECT FROM pg_database WHERE datname = 'main')\gexec
+CREATE TABLE jsonTable (
+    id character varying(32),
+    data jsonb,
+    text text
+);
+
+INSERT INTO jsonTable (id, data) VALUES ('1', '{"id": 1, "age": 1, "name": "Mike", "newline": "this is text with a\n newline in it"}');
+
+CREATE VIEW jsonView AS SELECT
+	x.id,
+	x.age,
+	x.name,
+	x.newline
+FROM
+	jsonTable c,
+	LATERAL jsonb_to_record(c.data) x (id character varying(32),
+		age BIGINT,
+		name TEXT,
+		newline TEXT
+		);

packages/server/scripts/integrations/pg-json/reset.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+docker-compose down
+docker volume prune -f

packages/server/src/api/controllers/permission.js

@@ -1,9 +1,4 @@
-const {
+const { getBuiltinPermissions } = require("@budibase/auth/permissions")
-  getBuiltinPermissions,
-  PermissionLevels,
-  isPermissionLevelHigherThanRead,
-  higherPermission,
-} = require("@budibase/auth/permissions")
 const {
   isBuiltin,
   getDBRoleID,
@@ -16,6 +11,7 @@ const {
   CURRENTLY_SUPPORTED_LEVELS,
   getBasePermissions,
 } = require("../../utilities/security")
+const { removeFromArray } = require("../../utilities")
 
 const PermissionUpdateType = {
   REMOVE: "remove",
@@ -24,22 +20,6 @@ const PermissionUpdateType = {
 
 const SUPPORTED_LEVELS = CURRENTLY_SUPPORTED_LEVELS
 
-// quick function to perform a bit of weird logic, make sure fetch calls
-// always say a write role also has read permission
-function fetchLevelPerms(permissions, level, roleId) {
-  if (!permissions) {
-    permissions = {}
-  }
-  permissions[level] = roleId
-  if (
-    isPermissionLevelHigherThanRead(level) &&
-    !permissions[PermissionLevels.READ]
-  ) {
-    permissions[PermissionLevels.READ] = roleId
-  }
-  return permissions
-}
-
 // utility function to stop this repetition - permissions always stored under roles
 async function getAllDBRoles(db) {
   const body = await db.allDocs(
@@ -74,23 +54,31 @@ async function updatePermissionOnRole(
   for (let role of dbRoles) {
     let updated = false
     const rolePermissions = role.permissions ? role.permissions : {}
+    // make sure its an array, also handle migrating
+    if (
+      !rolePermissions[resourceId] ||
+      !Array.isArray(rolePermissions[resourceId])
+    ) {
+      rolePermissions[resourceId] =
+        typeof rolePermissions[resourceId] === "string"
+          ? [rolePermissions[resourceId]]
+          : []
+    }
     // handle the removal/updating the role which has this permission first
     // the updating (role._id !== dbRoleId) is required because a resource/level can
     // only be permitted in a single role (this reduces hierarchy confusion and simplifies
     // the general UI for this, rather than needing to show everywhere it is used)
     if (
       (role._id !== dbRoleId || remove) &&
-      rolePermissions[resourceId] === level
+      rolePermissions[resourceId].indexOf(level) !== -1
     ) {
-      delete rolePermissions[resourceId]
+      removeFromArray(rolePermissions[resourceId], level)
       updated = true
     }
     // handle the adding, we're on the correct role, at it to this
     if (!remove && role._id === dbRoleId) {
-      rolePermissions[resourceId] = higherPermission(
+      const set = new Set(rolePermissions[resourceId])
-        rolePermissions[resourceId],
+      rolePermissions[resourceId] = [...set.add(level)]
-        level
-      )
       updated = true
     }
     // handle the update, add it to bulk docs to perform at end
@@ -127,12 +115,11 @@ exports.fetch = async function (ctx) {
       continue
     }
     const roleId = getExternalRoleID(role._id)
-    for (let [resource, level] of Object.entries(role.permissions)) {
+    for (let [resource, levelArr] of Object.entries(role.permissions)) {
-      permissions[resource] = fetchLevelPerms(
+      const levels = Array.isArray(levelArr) ? [levelArr] : levelArr
-        permissions[resource],
+      const perms = {}
-        level,
+      levels.forEach(level => (perms[level] = roleId))
-        roleId
+      permissions[resource] = perms
-      )
     }
   }
   // apply the base permissions
@@ -157,12 +144,13 @@ exports.getResourcePerms = async function (ctx) {
   for (let level of SUPPORTED_LEVELS) {
     // update the various roleIds in the resource permissions
     for (let role of roles) {
-      if (role.permissions && role.permissions[resourceId] === level) {
+      const rolePerms = role.permissions
-        permissions = fetchLevelPerms(
+      if (
-          permissions,
+        rolePerms &&
-          level,
+        (rolePerms[resourceId] === level ||
-          getExternalRoleID(role._id)
+          rolePerms[resourceId].indexOf(level) !== -1)
-        )
+      ) {
+        permissions[level] = getExternalRoleID(role._id)
       }
     }
   }

packages/server/src/api/routes/tests/role.spec.js

@@ -72,7 +72,7 @@ describe("/roles", () => {
         .expect(200)
       expect(res.body.length).toBeGreaterThan(0)
       const power = res.body.find(role => role._id === BUILTIN_ROLE_IDS.POWER)
-      expect(power.permissions[table._id]).toEqual("read")
+      expect(power.permissions[table._id]).toEqual(["read"])
     })
   })
 

packages/server/src/integrations/postgres.ts

@@ -13,6 +13,9 @@ module PostgresModule {
   const Sql = require("./base/sql")
   const { FieldTypes } = require("../constants")
   const { buildExternalTableId, convertType, copyExistingPropsOver } = require("./utils")
+  const { escapeDangerousCharacters } = require("../utilities")
+
+  const JSON_REGEX = /'{.*}'::json/s
 
   interface PostgresConfig {
     host: string
@@ -94,6 +97,17 @@ module PostgresModule {
   }
 
   async function internalQuery(client: any, query: SqlQuery) {
+    // need to handle a specific issue with json data types in postgres,
+    // new lines inside the JSON data will break it
+    if (query && query.sql) {
+      const matches = query.sql.match(JSON_REGEX)
+      if (matches && matches.length > 0) {
+        for (let match of matches) {
+          const escaped = escapeDangerousCharacters(match)
+          query.sql = query.sql.replace(match, escaped)
+        }
+      }
+    }
     try {
       return await client.query(query.sql, query.bindings || [])
     } catch (err) {

packages/server/src/utilities/index.js

@@ -10,6 +10,14 @@ exports.wait = ms => new Promise(resolve => setTimeout(resolve, ms))
 
 exports.isDev = env.isDev
 
+exports.removeFromArray = (array, element) => {
+  const index = array.indexOf(element)
+  if (index !== -1) {
+    array.splice(index, 1)
+  }
+  return array
+}
+
 /**
  * Makes sure that a URL has the correct number of slashes, while maintaining the
  * http(s):// double slashes.
@@ -106,3 +114,13 @@ exports.deleteEntityMetadata = async (appId, type, entityId) => {
     await db.remove(id, rev)
   }
 }
+
+exports.escapeDangerousCharacters = string => {
+  return string
+    .replace(/[\\]/g, "\\\\")
+    .replace(/[\b]/g, "\\b")
+    .replace(/[\f]/g, "\\f")
+    .replace(/[\n]/g, "\\n")
+    .replace(/[\r]/g, "\\r")
+    .replace(/[\t]/g, "\\t")
+}

packages/string-templates/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/string-templates",
-  "version": "0.9.140-alpha.11",
+  "version": "0.9.140-alpha.12",
   "description": "Handlebars wrapper for Budibase templating.",
   "main": "src/index.cjs",
   "module": "dist/bundle.mjs",

packages/worker/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/worker",
   "email": "hi@budibase.com",
-  "version": "0.9.140-alpha.11",
+  "version": "0.9.140-alpha.12",
   "description": "Budibase background service",
   "main": "src/index.js",
   "repository": {
@@ -25,8 +25,8 @@
   "author": "Budibase",
   "license": "AGPL-3.0-or-later",
   "dependencies": {
-    "@budibase/auth": "^0.9.140-alpha.11",
+    "@budibase/auth": "^0.9.140-alpha.12",
-    "@budibase/string-templates": "^0.9.140-alpha.11",
+    "@budibase/string-templates": "^0.9.140-alpha.12",
     "@koa/router": "^8.0.0",
     "@techpass/passport-openidconnect": "^0.3.0",
     "aws-sdk": "^2.811.0",