From 334b4b1696cb04e1722c3b6db8bc5638c6890546 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Fri, 9 Oct 2020 09:47:37 +0100
Subject: [PATCH 1/2] Update to IDs as has been discussed, to change them from
 colons (:) to underscores (_) as this is more URL, S3 and file system safe.
 Also shortening most prefixes down to two characters.

---
 packages/server/src/api/controllers/record.js |  9 +++-
 packages/server/src/db/utils.js               | 41 +++++++++++--------
 .../server/src/utilities/sanitisedPath.js     | 12 ++++++
 3 files changed, 42 insertions(+), 20 deletions(-)

diff --git a/packages/server/src/api/controllers/record.js b/packages/server/src/api/controllers/record.js
index e7718665e2..473bb4f413 100644
--- a/packages/server/src/api/controllers/record.js
+++ b/packages/server/src/api/controllers/record.js
@@ -1,10 +1,15 @@
 const CouchDB = require("../../db")
 const validateJs = require("validate.js")
 const linkRecords = require("../../db/linkedRecords")
-const { getRecordParams, generateRecordID } = require("../../db/utils")
+const {
+  getRecordParams,
+  generateRecordID,
+  DocumentTypes,
+  SEPARATOR,
+} = require("../../db/utils")
 const { cloneDeep } = require("lodash")
 
-const MODEL_VIEW_BEGINS_WITH = "all_model:"
+const MODEL_VIEW_BEGINS_WITH = `all${SEPARATOR}${DocumentTypes.MODEL}${SEPARATOR}`
 
 validateJs.extend(validateJs.validators.datetime, {
   parse: function(value) {
diff --git a/packages/server/src/db/utils.js b/packages/server/src/db/utils.js
index e66eb7624f..7c55da43ff 100644
--- a/packages/server/src/db/utils.js
+++ b/packages/server/src/db/utils.js
@@ -1,18 +1,20 @@
 const newid = require("./newid")
 
+const UNICODE_MAX = "\ufff0"
+const SEPARATOR = "_"
+
 const DocumentTypes = {
-  MODEL: "model",
-  RECORD: "record",
-  USER: "user",
-  AUTOMATION: "automation",
-  LINK: "link",
+  MODEL: "mo",
+  RECORD: "re",
+  USER: "us",
+  AUTOMATION: "au",
+  LINK: "li",
   APP: "app",
-  ACCESS_LEVEL: "accesslevel",
+  ACCESS_LEVEL: "ac",
 }
 
 exports.DocumentTypes = DocumentTypes
-
-const UNICODE_MAX = "\ufff0"
+exports.SEPARATOR = SEPARATOR
 
 /**
  * If creating DB allDocs/query params with only a single top level ID this can be used, this
@@ -32,8 +34,8 @@ function getDocParams(docType, docId = null, otherProps = {}) {
   }
   return {
     ...otherProps,
-    startkey: `${docType}:${docId}`,
-    endkey: `${docType}:${docId}${UNICODE_MAX}`,
+    startkey: `${docType}${SEPARATOR}${docId}`,
+    endkey: `${docType}${SEPARATOR}${docId}${UNICODE_MAX}`,
   }
 }
 
@@ -49,7 +51,7 @@ exports.getModelParams = (modelId = null, otherProps = {}) => {
  * @returns {string} The new model ID which the model doc can be stored under.
  */
 exports.generateModelID = () => {
-  return `${DocumentTypes.MODEL}:${newid()}`
+  return `${DocumentTypes.MODEL}${SEPARATOR}${newid()}`
 }
 
 /**
@@ -64,7 +66,10 @@ exports.getRecordParams = (modelId, recordId = null, otherProps = {}) => {
   if (modelId == null) {
     throw "Cannot build params for records without a model ID"
   }
-  const endOfKey = recordId == null ? `${modelId}:` : `${modelId}:${recordId}`
+  const endOfKey =
+    recordId == null
+      ? `${modelId}${SEPARATOR}`
+      : `${modelId}${SEPARATOR}${recordId}`
   return getDocParams(DocumentTypes.RECORD, endOfKey, otherProps)
 }
 
@@ -74,7 +79,7 @@ exports.getRecordParams = (modelId, recordId = null, otherProps = {}) => {
  * @returns {string} The new ID which a record doc can be stored under.
  */
 exports.generateRecordID = modelId => {
-  return `${DocumentTypes.RECORD}:${modelId}:${newid()}`
+  return `${DocumentTypes.RECORD}${SEPARATOR}${modelId}${SEPARATOR}${newid()}`
 }
 
 /**
@@ -90,7 +95,7 @@ exports.getUserParams = (username = null, otherProps = {}) => {
  * @returns {string} The new user ID which the user doc can be stored under.
  */
 exports.generateUserID = username => {
-  return `${DocumentTypes.USER}:${username}`
+  return `${DocumentTypes.USER}${SEPARATOR}${username}`
 }
 
 /**
@@ -105,7 +110,7 @@ exports.getAutomationParams = (automationId = null, otherProps = {}) => {
  * @returns {string} The new automation ID which the automation doc can be stored under.
  */
 exports.generateAutomationID = () => {
-  return `${DocumentTypes.AUTOMATION}:${newid()}`
+  return `${DocumentTypes.AUTOMATION}${SEPARATOR}${newid()}`
 }
 
 /**
@@ -118,7 +123,7 @@ exports.generateAutomationID = () => {
  * @returns {string} The new link doc ID which the automation doc can be stored under.
  */
 exports.generateLinkID = (modelId1, modelId2, recordId1, recordId2) => {
-  return `${DocumentTypes.AUTOMATION}:${modelId1}:${modelId2}:${recordId1}:${recordId2}`
+  return `${DocumentTypes.AUTOMATION}${SEPARATOR}${modelId1}${SEPARATOR}${modelId2}${SEPARATOR}${recordId1}${SEPARATOR}${recordId2}`
 }
 
 /**
@@ -126,7 +131,7 @@ exports.generateLinkID = (modelId1, modelId2, recordId1, recordId2) => {
  * @returns {string} The new app ID which the app doc can be stored under.
  */
 exports.generateAppID = () => {
-  return `${DocumentTypes.APP}:${newid()}`
+  return `${DocumentTypes.APP}${SEPARATOR}${newid()}`
 }
 
 /**
@@ -141,7 +146,7 @@ exports.getAppParams = (appId = null, otherProps = {}) => {
  * @returns {string} The new access level ID which the access level doc can be stored under.
  */
 exports.generateAccessLevelID = () => {
-  return `${DocumentTypes.ACCESS_LEVEL}:${newid()}`
+  return `${DocumentTypes.ACCESS_LEVEL}${SEPARATOR}${newid()}`
 }
 
 /**
diff --git a/packages/server/src/utilities/sanitisedPath.js b/packages/server/src/utilities/sanitisedPath.js
index c2c304aad9..840fa134f2 100644
--- a/packages/server/src/utilities/sanitisedPath.js
+++ b/packages/server/src/utilities/sanitisedPath.js
@@ -1,11 +1,23 @@
 const path = require("path")
 
 const regex = new RegExp(/:(?![\\/])/g)
+// set a limit on path depth, just incase recursion is occurring
+const MAX_ARGS = 50
 
 function sanitiseArgs(args) {
   let sanitised = []
+  let count = 0
   for (let arg of args) {
+    // if a known string is found don't continue, can't operate on it
+    if (typeof arg !== "string") {
+      throw "Sanitisation of paths can only occur on strings"
+    }
+    // maximum number of path args have been iterated on
+    if (count > MAX_ARGS) {
+      break
+    }
     sanitised.push(arg.replace(regex, ""))
+    count++
   }
   return sanitised
 }

From ba758905cc94f24a08d36fc7ee15ea9d9d8573b9 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Fri, 9 Oct 2020 10:00:57 +0100
Subject: [PATCH 2/2] Removing sanisation, instead just using it as a central
 path system.

---
 .../server/src/api/controllers/apikeys.js     |  2 +-
 .../server/src/api/controllers/application.js |  2 +-
 .../server/src/api/controllers/component.js   |  2 +-
 .../server/src/api/controllers/deploy/aws.js  |  2 +-
 .../server/src/api/controllers/instance.js    |  2 +-
 packages/server/src/api/controllers/static.js |  2 +-
 .../server/src/api/controllers/view/index.js  |  2 +-
 packages/server/src/automations/actions.js    |  2 +-
 packages/server/src/db/client.js              |  9 +---
 packages/server/src/electron.js               |  2 +-
 packages/server/src/index.js                  |  2 +-
 packages/server/src/utilities/budibaseDir.js  |  2 +-
 .../server/src/utilities/builder/buildPage.js |  2 +-
 .../utilities/builder/convertCssToFiles.js    |  2 +-
 .../server/src/utilities/builder/getPages.js  |  2 +-
 .../server/src/utilities/builder/index.js     |  2 +-
 .../src/utilities/builder/listScreens.js      |  2 +-
 .../src/utilities/builder/publicPath.js       |  2 +-
 packages/server/src/utilities/centralPath.js  | 22 ++++++++
 .../server/src/utilities/createAppPackage.js  |  2 +-
 .../src/utilities/initialiseBudibase.js       |  2 +-
 .../server/src/utilities/sanitisedPath.js     | 50 -------------------
 packages/server/src/utilities/templates.js    |  2 +-
 23 files changed, 43 insertions(+), 78 deletions(-)
 create mode 100644 packages/server/src/utilities/centralPath.js
 delete mode 100644 packages/server/src/utilities/sanitisedPath.js

diff --git a/packages/server/src/api/controllers/apikeys.js b/packages/server/src/api/controllers/apikeys.js
index 14642ac83b..11cf1493da 100644
--- a/packages/server/src/api/controllers/apikeys.js
+++ b/packages/server/src/api/controllers/apikeys.js
@@ -1,5 +1,5 @@
 const fs = require("fs")
-const { join } = require("../../utilities/sanitisedPath")
+const { join } = require("../../utilities/centralPath")
 const readline = require("readline")
 const { budibaseAppsDir } = require("../../utilities/budibaseDir")
 const ENV_FILE_PATH = "/.env"
diff --git a/packages/server/src/api/controllers/application.js b/packages/server/src/api/controllers/application.js
index 67b005aace..53b8792f49 100644
--- a/packages/server/src/api/controllers/application.js
+++ b/packages/server/src/api/controllers/application.js
@@ -8,7 +8,7 @@ const { budibaseAppsDir } = require("../../utilities/budibaseDir")
 const sqrl = require("squirrelly")
 const setBuilderToken = require("../../utilities/builder/setBuilderToken")
 const fs = require("fs-extra")
-const { join, resolve } = require("../../utilities/sanitisedPath")
+const { join, resolve } = require("../../utilities/centralPath")
 const { promisify } = require("util")
 const chmodr = require("chmodr")
 const { generateAppID, getAppParams } = require("../../db/utils")
diff --git a/packages/server/src/api/controllers/component.js b/packages/server/src/api/controllers/component.js
index 96da08a2df..9c5e0df151 100644
--- a/packages/server/src/api/controllers/component.js
+++ b/packages/server/src/api/controllers/component.js
@@ -1,6 +1,6 @@
 const CouchDB = require("../../db")
 const ClientDb = require("../../db/clientDb")
-const { resolve, join } = require("../../utilities/sanitisedPath")
+const { resolve, join } = require("../../utilities/centralPath")
 const {
   budibaseTempDir,
   budibaseAppsDir,
diff --git a/packages/server/src/api/controllers/deploy/aws.js b/packages/server/src/api/controllers/deploy/aws.js
index c95cd5ab55..476f6c230f 100644
--- a/packages/server/src/api/controllers/deploy/aws.js
+++ b/packages/server/src/api/controllers/deploy/aws.js
@@ -1,5 +1,5 @@
 const fs = require("fs")
-const { join } = require("../../../utilities/sanitisedPath")
+const { join } = require("../../../utilities/centralPath")
 const AWS = require("aws-sdk")
 const fetch = require("node-fetch")
 const { budibaseAppsDir } = require("../../../utilities/budibaseDir")
diff --git a/packages/server/src/api/controllers/instance.js b/packages/server/src/api/controllers/instance.js
index 2c11713c91..fb81988fa2 100644
--- a/packages/server/src/api/controllers/instance.js
+++ b/packages/server/src/api/controllers/instance.js
@@ -3,7 +3,7 @@ const CouchDB = require("../../db")
 const client = require("../../db/clientDb")
 const newid = require("../../db/newid")
 const { createLinkView } = require("../../db/linkedRecords")
-const { join } = require("../../utilities/sanitisedPath")
+const { join } = require("../../utilities/centralPath")
 const { downloadTemplate } = require("../../utilities/templates")
 
 exports.create = async function(ctx) {
diff --git a/packages/server/src/api/controllers/static.js b/packages/server/src/api/controllers/static.js
index 3b0c378e50..df0f9a8867 100644
--- a/packages/server/src/api/controllers/static.js
+++ b/packages/server/src/api/controllers/static.js
@@ -1,5 +1,5 @@
 const send = require("koa-send")
-const { resolve, join } = require("../../utilities/sanitisedPath")
+const { resolve, join } = require("../../utilities/centralPath")
 const jwt = require("jsonwebtoken")
 const fetch = require("node-fetch")
 const fs = require("fs-extra")
diff --git a/packages/server/src/api/controllers/view/index.js b/packages/server/src/api/controllers/view/index.js
index 140bd4bf40..1c98466b7a 100644
--- a/packages/server/src/api/controllers/view/index.js
+++ b/packages/server/src/api/controllers/view/index.js
@@ -1,7 +1,7 @@
 const CouchDB = require("../../../db")
 const viewTemplate = require("./viewBuilder")
 const fs = require("fs")
-const { join } = require("../../../utilities/sanitisedPath")
+const { join } = require("../../../utilities/centralPath")
 const os = require("os")
 const exporters = require("./exporters")
 const { fetchView } = require("../record")
diff --git a/packages/server/src/automations/actions.js b/packages/server/src/automations/actions.js
index 47dd3aa8bb..2f86978e72 100644
--- a/packages/server/src/automations/actions.js
+++ b/packages/server/src/automations/actions.js
@@ -6,7 +6,7 @@ const createUser = require("./steps/createUser")
 const environment = require("../environment")
 const download = require("download")
 const fetch = require("node-fetch")
-const { join } = require("../utilities/sanitisedPath")
+const { join } = require("../utilities/centralPath")
 const os = require("os")
 const fs = require("fs")
 const Sentry = require("@sentry/node")
diff --git a/packages/server/src/db/client.js b/packages/server/src/db/client.js
index 6bef09740a..a3051eea7f 100644
--- a/packages/server/src/db/client.js
+++ b/packages/server/src/db/client.js
@@ -2,7 +2,6 @@ const PouchDB = require("pouchdb")
 const replicationStream = require("pouchdb-replication-stream")
 const allDbs = require("pouchdb-all-dbs")
 const { budibaseAppsDir } = require("../utilities/budibaseDir")
-const { sanitise } = require("../utilities/sanitisedPath")
 const env = require("../environment")
 
 const COUCH_DB_URL = env.COUCH_DB_URL || `leveldb://${budibaseAppsDir()}/.data/`
@@ -27,10 +26,4 @@ const Pouch = PouchDB.defaults(POUCH_DB_DEFAULTS)
 
 allDbs(Pouch)
 
-function PouchWrapper(instance) {
-  Pouch.apply(this, [sanitise(instance)])
-}
-
-PouchWrapper.prototype = Object.create(Pouch.prototype)
-
-module.exports = PouchWrapper
+module.exports = Pouch
diff --git a/packages/server/src/electron.js b/packages/server/src/electron.js
index f4055d8b6b..755bb7fde7 100644
--- a/packages/server/src/electron.js
+++ b/packages/server/src/electron.js
@@ -1,5 +1,5 @@
 const { app, BrowserWindow, shell, dialog } = require("electron")
-const { join } = require("./utilities/sanitisedPath")
+const { join } = require("./utilities/centralPath")
 const isDev = require("electron-is-dev")
 const { autoUpdater } = require("electron-updater")
 const unhandled = require("electron-unhandled")
diff --git a/packages/server/src/index.js b/packages/server/src/index.js
index e69e37f6b0..afe2b09e3b 100644
--- a/packages/server/src/index.js
+++ b/packages/server/src/index.js
@@ -1,4 +1,4 @@
-const { resolve, join } = require("./utilities/sanitisedPath")
+const { resolve, join } = require("./utilities/centralPath")
 const { homedir } = require("os")
 const { app } = require("electron")
 const fixPath = require("fix-path")
diff --git a/packages/server/src/utilities/budibaseDir.js b/packages/server/src/utilities/budibaseDir.js
index 22dc0d7efd..e27c45b9e0 100644
--- a/packages/server/src/utilities/budibaseDir.js
+++ b/packages/server/src/utilities/budibaseDir.js
@@ -1,4 +1,4 @@
-const { join } = require("./sanitisedPath")
+const { join } = require("./centralPath")
 const { homedir, tmpdir } = require("os")
 const env = require("../environment")
 
diff --git a/packages/server/src/utilities/builder/buildPage.js b/packages/server/src/utilities/builder/buildPage.js
index 7792b50848..7db1299fa2 100644
--- a/packages/server/src/utilities/builder/buildPage.js
+++ b/packages/server/src/utilities/builder/buildPage.js
@@ -6,7 +6,7 @@ const {
   readFile,
   writeJSON,
 } = require("fs-extra")
-const { join, resolve } = require("../sanitisedPath")
+const { join, resolve } = require("../centralPath")
 const sqrl = require("squirrelly")
 const { convertCssToFiles } = require("./convertCssToFiles")
 const publicPath = require("./publicPath")
diff --git a/packages/server/src/utilities/builder/convertCssToFiles.js b/packages/server/src/utilities/builder/convertCssToFiles.js
index 0105df9cdf..c81f9497ec 100644
--- a/packages/server/src/utilities/builder/convertCssToFiles.js
+++ b/packages/server/src/utilities/builder/convertCssToFiles.js
@@ -1,6 +1,6 @@
 const crypto = require("crypto")
 const { ensureDir, emptyDir, writeFile } = require("fs-extra")
-const { join } = require("../sanitisedPath")
+const { join } = require("../centralPath")
 
 module.exports.convertCssToFiles = async (publicPagePath, pkg) => {
   const cssDir = join(publicPagePath, "css")
diff --git a/packages/server/src/utilities/builder/getPages.js b/packages/server/src/utilities/builder/getPages.js
index 7a124d1716..2485bed419 100644
--- a/packages/server/src/utilities/builder/getPages.js
+++ b/packages/server/src/utilities/builder/getPages.js
@@ -1,5 +1,5 @@
 const { readJSON, readdir } = require("fs-extra")
-const { join } = require("../sanitisedPath")
+const { join } = require("../centralPath")
 
 module.exports = async appPath => {
   const pages = {}
diff --git a/packages/server/src/utilities/builder/index.js b/packages/server/src/utilities/builder/index.js
index 2e87d35f9b..a74d5cd0b2 100644
--- a/packages/server/src/utilities/builder/index.js
+++ b/packages/server/src/utilities/builder/index.js
@@ -8,7 +8,7 @@ const {
   unlink,
   rmdir,
 } = require("fs-extra")
-const { join, resolve } = require("../sanitisedPath")
+const { join, resolve } = require("../centralPath")
 const { dirname } = require("path")
 const env = require("../../environment")
 
diff --git a/packages/server/src/utilities/builder/listScreens.js b/packages/server/src/utilities/builder/listScreens.js
index 9a9c7cb2d9..8964ac6cec 100644
--- a/packages/server/src/utilities/builder/listScreens.js
+++ b/packages/server/src/utilities/builder/listScreens.js
@@ -1,6 +1,6 @@
 const { appPackageFolder } = require("../createAppPackage")
 const { readJSON, readdir, stat } = require("fs-extra")
-const { join } = require("../sanitisedPath")
+const { join } = require("../centralPath")
 const { keyBy } = require("lodash/fp")
 
 module.exports = async (config, appname, pagename) => {
diff --git a/packages/server/src/utilities/builder/publicPath.js b/packages/server/src/utilities/builder/publicPath.js
index b9408f026f..03917d4370 100644
--- a/packages/server/src/utilities/builder/publicPath.js
+++ b/packages/server/src/utilities/builder/publicPath.js
@@ -1,3 +1,3 @@
-const { join } = require("../sanitisedPath")
+const { join } = require("../centralPath")
 
 module.exports = (appPath, pageName) => join(appPath, "public", pageName)
diff --git a/packages/server/src/utilities/centralPath.js b/packages/server/src/utilities/centralPath.js
new file mode 100644
index 0000000000..030ba55fc5
--- /dev/null
+++ b/packages/server/src/utilities/centralPath.js
@@ -0,0 +1,22 @@
+const path = require("path")
+
+// this simply runs all of our path join and resolve functions through
+// a central location incase we need to add some protection to file paths
+
+/**
+ * Exactly the same as path.join
+ * @param args Any number of string arguments to add to a path
+ * @returns {string} The final path ready to use
+ */
+exports.join = function(...args) {
+  return path.join(...args)
+}
+
+/**
+ * Exactly the same as path.resolve
+ * @param args Any number of string arguments to add to a path
+ * @returns {string} The final path ready to use
+ */
+exports.resolve = function(...args) {
+  return path.resolve(...args)
+}
diff --git a/packages/server/src/utilities/createAppPackage.js b/packages/server/src/utilities/createAppPackage.js
index 39647e4db4..d2a1840cb2 100644
--- a/packages/server/src/utilities/createAppPackage.js
+++ b/packages/server/src/utilities/createAppPackage.js
@@ -1,4 +1,4 @@
-const { resolve } = require("./sanitisedPath")
+const { resolve } = require("./centralPath")
 const { cwd } = require("process")
 const stream = require("stream")
 const fetch = require("node-fetch")
diff --git a/packages/server/src/utilities/initialiseBudibase.js b/packages/server/src/utilities/initialiseBudibase.js
index 56a2c059d6..c86b6083e9 100644
--- a/packages/server/src/utilities/initialiseBudibase.js
+++ b/packages/server/src/utilities/initialiseBudibase.js
@@ -1,5 +1,5 @@
 const { exists, readFile, writeFile, ensureDir } = require("fs-extra")
-const { join, resolve } = require("./sanitisedPath")
+const { join, resolve } = require("./centralPath")
 const Sqrl = require("squirrelly")
 const uuid = require("uuid")
 
diff --git a/packages/server/src/utilities/sanitisedPath.js b/packages/server/src/utilities/sanitisedPath.js
deleted file mode 100644
index 840fa134f2..0000000000
--- a/packages/server/src/utilities/sanitisedPath.js
+++ /dev/null
@@ -1,50 +0,0 @@
-const path = require("path")
-
-const regex = new RegExp(/:(?![\\/])/g)
-// set a limit on path depth, just incase recursion is occurring
-const MAX_ARGS = 50
-
-function sanitiseArgs(args) {
-  let sanitised = []
-  let count = 0
-  for (let arg of args) {
-    // if a known string is found don't continue, can't operate on it
-    if (typeof arg !== "string") {
-      throw "Sanitisation of paths can only occur on strings"
-    }
-    // maximum number of path args have been iterated on
-    if (count > MAX_ARGS) {
-      break
-    }
-    sanitised.push(arg.replace(regex, ""))
-    count++
-  }
-  return sanitised
-}
-
-/**
- * Exactly the same as path.join but creates a sanitised path.
- * @param args Any number of string arguments to add to a path
- * @returns {string} The final path ready to use
- */
-exports.join = function(...args) {
-  return path.join(...sanitiseArgs(args))
-}
-
-/**
- * Exactly the same as path.resolve but creates a sanitised path.
- * @param args Any number of string arguments to add to a path
- * @returns {string} The final path ready to use
- */
-exports.resolve = function(...args) {
-  return path.resolve(...sanitiseArgs(args))
-}
-
-/**
- * Sanitise a single string
- * @param string input string to sanitise
- * @returns {string} the final sanitised string
- */
-exports.sanitise = function(string) {
-  return sanitiseArgs([string])[0]
-}
diff --git a/packages/server/src/utilities/templates.js b/packages/server/src/utilities/templates.js
index 39ae97f839..626f1d003b 100644
--- a/packages/server/src/utilities/templates.js
+++ b/packages/server/src/utilities/templates.js
@@ -1,5 +1,5 @@
 const fs = require("fs-extra")
-const { join } = require("./sanitisedPath")
+const { join } = require("./centralPath")
 const os = require("os")
 const fetch = require("node-fetch")
 const stream = require("stream")