Merge pull request #662 from Budibase/remove-views

Replace the use of views with that of allDocs

packages/builder/cypress/integration/createAutomation.spec.js

@@ -28,7 +28,7 @@ context("Create a automation", () => {
     })
 
     // Create action
-    cy.get("[data-cy=SAVE_RECORD]").click()
+    cy.get("[data-cy=CREATE_RECORD]").click()
     cy.get("[data-cy=automation-block-setup]").within(() => {
       cy.get("select")
         .first()

packages/server/src/api/controllers/accesslevel.js

@@ -1,18 +1,22 @@
 const CouchDB = require("../../db")
-const newid = require("../../db/newid")
 const {
   generateAdminPermissions,
   generatePowerUserPermissions,
   POWERUSER_LEVEL_ID,
   ADMIN_LEVEL_ID,
 } = require("../../utilities/accessLevels")
+const {
+  generateAccessLevelID,
+  getAccessLevelParams,
+} = require("../../db/utils")
 
 exports.fetch = async function(ctx) {
   const db = new CouchDB(ctx.user.instanceId)
-  const body = await db.query("database/by_type", {
+  const body = await db.allDocs(
+    getAccessLevelParams(null, {
       include_docs: true,
-    key: ["accesslevel"],
     })
+  )
   const customAccessLevels = body.rows.map(row => row.doc)
 
   const staticAccessLevels = [
@@ -90,7 +94,7 @@ exports.create = async function(ctx) {
     name: ctx.request.body.name,
     _rev: ctx.request.body._rev,
     permissions: ctx.request.body.permissions || [],
-    _id: newid(),
+    _id: generateAccessLevelID(),
     type: "accesslevel",
   }
 

packages/server/src/api/controllers/application.js

@@ -1,7 +1,6 @@
 const CouchDB = require("../../db")
 const ClientDb = require("../../db/clientDb")
 const { getPackageForBuilder, buildPage } = require("../../utilities/builder")
-const newid = require("../../db/newid")
 const env = require("../../environment")
 const instanceController = require("./instance")
 const { resolve, join } = require("path")
@@ -12,17 +11,18 @@ const setBuilderToken = require("../../utilities/builder/setBuilderToken")
 const fs = require("fs-extra")
 const { promisify } = require("util")
 const chmodr = require("chmodr")
+const { generateAppID, getAppParams } = require("../../db/utils")
 const {
   downloadExtractComponentLibraries,
 } = require("../../utilities/createAppPackage")
 
 exports.fetch = async function(ctx) {
   const db = new CouchDB(ClientDb.name(getClientId(ctx)))
-  const body = await db.query("client/by_type", {
+  const body = await db.allDocs(
+    getAppParams(null, {
       include_docs: true,
-    key: ["app"],
     })
-
+  )
   ctx.body = body.rows.map(row => row.doc)
 }
 
@@ -48,7 +48,7 @@ exports.create = async function(ctx) {
   if (!clientId) {
     ctx.throw(400, "ClientId not suplied")
   }
-  const appId = newid()
+  const appId = generateAppID()
   // insert an appId -> clientId lookup
   const masterDb = new CouchDB("client_app_lookup")
 

packages/server/src/api/controllers/auth.js

@@ -2,6 +2,7 @@ const jwt = require("jsonwebtoken")
 const CouchDB = require("../../db")
 const ClientDb = require("../../db/clientDb")
 const bcrypt = require("../../utilities/bcrypt")
+const { generateUserID } = require("../../db/utils")
 
 exports.authenticate = async ctx => {
   if (!ctx.user.appId) ctx.throw(400, "No appId")
@@ -35,7 +36,7 @@ exports.authenticate = async ctx => {
 
   let dbUser
   try {
-    dbUser = await instanceDb.get(`user_${username}`)
+    dbUser = await instanceDb.get(generateUserID(username))
   } catch (_) {
     // do not want to throw a 404 - as this could be
     // used to dtermine valid usernames

packages/server/src/api/controllers/automation.js

@@ -1,8 +1,8 @@
 const CouchDB = require("../../db")
-const newid = require("../../db/newid")
 const actions = require("../../automations/actions")
 const logic = require("../../automations/logic")
 const triggers = require("../../automations/triggers")
+const { getAutomationParams, generateAutomationID } = require("../../db/utils")
 
 /*************************
  *                       *
@@ -34,7 +34,7 @@ exports.create = async function(ctx) {
   const db = new CouchDB(ctx.user.instanceId)
   let automation = ctx.request.body
 
-  automation._id = newid()
+  automation._id = generateAutomationID()
 
   automation.type = "automation"
   automation = cleanAutomationInputs(automation)
@@ -72,10 +72,11 @@ exports.update = async function(ctx) {
 
 exports.fetch = async function(ctx) {
   const db = new CouchDB(ctx.user.instanceId)
-  const response = await db.query(`database/by_type`, {
+  const response = await db.allDocs(
-    key: ["automation"],
+    getAutomationParams(null, {
       include_docs: true,
     })
+  )
   ctx.body = response.rows.map(row => row.doc)
 }
 

packages/server/src/api/controllers/instance.js

@@ -21,30 +21,7 @@ exports.create = async function(ctx) {
       clientId,
       applicationId: appId,
     },
-    views: {
+    views: {},
-      by_username: {
-        map: function(doc) {
-          if (doc.type === "user") {
-            emit([doc.username], doc._id)
-          }
-        }.toString(),
-      },
-      by_type: {
-        map: function(doc) {
-          emit([doc.type], doc._id)
-        }.toString(),
-      },
-      by_automation_trigger: {
-        map: function(doc) {
-          if (doc.type === "automation") {
-            const trigger = doc.definition.trigger
-            if (trigger) {
-              emit([trigger.event], trigger)
-            }
-          }
-        }.toString(),
-      },
-    },
   })
 
   // Add the new instance under the app clientDB

packages/server/src/api/controllers/model.js

@@ -1,26 +1,30 @@
 const CouchDB = require("../../db")
-const newid = require("../../db/newid")
+const {
+  getRecordParams,
+  getModelParams,
+  generateModelID,
+} = require("../../db/utils")
 
 exports.fetch = async function(ctx) {
   const db = new CouchDB(ctx.user.instanceId)
-  const body = await db.query("database/by_type", {
+  const body = await db.allDocs(
+    getModelParams(null, {
       include_docs: true,
-    key: ["model"],
     })
+  )
   ctx.body = body.rows.map(row => row.doc)
 }
 
 exports.find = async function(ctx) {
   const db = new CouchDB(ctx.user.instanceId)
-  const model = await db.get(ctx.params.id)
+  ctx.body = await db.get(ctx.params.id)
-  ctx.body = model
 }
 
 exports.save = async function(ctx) {
   const db = new CouchDB(ctx.user.instanceId)
   const modelToSave = {
     type: "model",
-    _id: newid(),
+    _id: generateModelID(),
     views: {},
     ...ctx.request.body,
   }
@@ -28,9 +32,11 @@ exports.save = async function(ctx) {
   // rename record fields when table column is renamed
   const { _rename } = modelToSave
   if (_rename) {
-    const records = await db.query(`database/all_${modelToSave._id}`, {
+    const records = await db.allDocs(
+      getRecordParams(modelToSave._id, null, {
         include_docs: true,
       })
+    )
     const docs = records.rows.map(({ doc }) => {
       doc[_rename.updated] = doc[_rename.old]
       delete doc[_rename.old]
@@ -54,7 +60,7 @@ exports.save = async function(ctx) {
   modelToSave._rev = result.rev
 
   const { schema } = ctx.request.body
-  for (let key in schema) {
+  for (let key of Object.keys(schema)) {
     // model has a linked record
     if (schema[key].type === "link") {
       // create the link field in the other model
@@ -71,19 +77,6 @@ exports.save = async function(ctx) {
     }
   }
 
-  const designDoc = await db.get("_design/database")
-  designDoc.views = {
-    ...designDoc.views,
-    [`all_${modelToSave._id}`]: {
-      map: `function(doc) {
-        if (doc.modelId === "${modelToSave._id}") {
-          emit(doc._id); 
-        }
-      }`,
-    },
-  }
-  await db.put(designDoc)
-
   ctx.status = 200
   ctx.message = `Model ${ctx.request.body.name} saved successfully.`
   ctx.body = modelToSave
@@ -96,16 +89,18 @@ exports.destroy = async function(ctx) {
 
   await db.remove(modelToDelete)
 
-  const modelViewId = `all_${ctx.params.modelId}`
-
   // Delete all records for that model
-  const records = await db.query(`database/${modelViewId}`)
+  const records = await db.allDocs(
+    getRecordParams(ctx.params.modelId, null, {
+      include_docs: true,
+    })
+  )
   await db.bulkDocs(
     records.rows.map(record => ({ _id: record.id, _deleted: true }))
   )
 
   // Delete linked record fields in dependent models
-  for (let key in modelToDelete.schema) {
+  for (let key of Object.keys(modelToDelete.schema)) {
     const { type, modelId } = modelToDelete.schema[key]
     if (type === "link") {
       const linkedModel = await db.get(modelId)
@@ -114,11 +109,6 @@ exports.destroy = async function(ctx) {
     }
   }
 
-  // delete the "all" view
-  const designDoc = await db.get("_design/database")
-  delete designDoc.views[modelViewId]
-  await db.put(designDoc)
-
   ctx.status = 200
   ctx.message = `Model ${ctx.params.modelId} deleted.`
 }

packages/server/src/api/controllers/record.js

@@ -1,6 +1,8 @@
 const CouchDB = require("../../db")
 const validateJs = require("validate.js")
-const newid = require("../../db/newid")
+const { getRecordParams, generateRecordID } = require("../../db/utils")
+
+const MODEL_VIEW_BEGINS_WITH = "all_model:"
 
 function emitEvent(eventType, ctx, record) {
   let event = {
@@ -66,7 +68,7 @@ exports.save = async function(ctx) {
   record.modelId = ctx.params.modelId
 
   if (!record._rev && !record._id) {
-    record._id = newid()
+    record._id = generateRecordID(record.modelId)
   }
 
   const model = await db.get(record.modelId)
@@ -133,7 +135,16 @@ exports.save = async function(ctx) {
 exports.fetchView = async function(ctx) {
   const db = new CouchDB(ctx.user.instanceId)
   const { stats, group, field } = ctx.query
-  const response = await db.query(`database/${ctx.params.viewName}`, {
+  const viewName = ctx.params.viewName
+
+  // if this is a model view being looked for just transfer to that
+  if (viewName.indexOf(MODEL_VIEW_BEGINS_WITH) === 0) {
+    ctx.params.modelId = viewName.substring(4)
+    await exports.fetchModelRecords(ctx)
+    return
+  }
+
+  const response = await db.query(`database/${viewName}`, {
     include_docs: !stats,
     group,
   })
@@ -154,9 +165,11 @@ exports.fetchView = async function(ctx) {
 
 exports.fetchModelRecords = async function(ctx) {
   const db = new CouchDB(ctx.user.instanceId)
-  const response = await db.query(`database/all_${ctx.params.modelId}`, {
+  const response = await db.allDocs(
+    getRecordParams(ctx.params.modelId, null, {
       include_docs: true,
     })
+  )
   ctx.body = response.rows.map(row => row.doc)
 }
 

packages/server/src/api/controllers/user.js

@@ -1,7 +1,7 @@
 const CouchDB = require("../../db")
 const clientDb = require("../../db/clientDb")
 const bcrypt = require("../../utilities/bcrypt")
-const getUserId = userName => `user_${userName}`
+const { generateUserID, getUserParams } = require("../../db/utils")
 const {
   POWERUSER_LEVEL_ID,
   ADMIN_LEVEL_ID,
@@ -9,11 +9,11 @@ const {
 
 exports.fetch = async function(ctx) {
   const database = new CouchDB(ctx.user.instanceId)
-  const data = await database.query("database/by_type", {
+  const data = await database.allDocs(
+    getUserParams(null, {
       include_docs: true,
-    key: ["user"],
     })
-
+  )
   ctx.body = data.rows.map(row => row.doc)
 }
 
@@ -31,7 +31,7 @@ exports.create = async function(ctx) {
   if (!accessLevel) ctx.throw(400, "Invalid Access Level")
 
   const user = {
-    _id: getUserId(username),
+    _id: generateUserID(username),
     username,
     password: await bcrypt.hash(password),
     name: name || username,
@@ -80,14 +80,14 @@ exports.update = async function(ctx) {
 
 exports.destroy = async function(ctx) {
   const database = new CouchDB(ctx.user.instanceId)
-  await database.destroy(getUserId(ctx.params.username))
+  await database.destroy(generateUserID(ctx.params.username))
   ctx.message = `User ${ctx.params.username} deleted.`
   ctx.status = 200
 }
 
 exports.find = async function(ctx) {
   const database = new CouchDB(ctx.user.instanceId)
-  const user = await database.get(getUserId(ctx.params.username))
+  const user = await database.get(generateUserID(ctx.params.username))
   ctx.body = {
     username: user.username,
     name: user.name,

packages/server/src/api/controllers/view/index.js

@@ -11,19 +11,12 @@ const controller = {
     const designDoc = await db.get("_design/database")
     const response = []
 
-    for (let name in designDoc.views) {
+    for (let name of Object.keys(designDoc.views)) {
-      if (
-        !name.startsWith("all") &&
-        name !== "by_type" &&
-        name !== "by_username" &&
-        name !== "by_automation_trigger"
-      ) {
       response.push({
         name,
         ...designDoc.views[name],
       })
     }
-    }
 
     ctx.body = response
   },

packages/server/src/api/routes/tests/automation.spec.js

@@ -10,12 +10,14 @@ const {
   destroyDocument,
   builderEndpointShouldBlockNormalUsers
 } = require("./couchTestUtils")
+let { generateAutomationID } = require("../../../db/utils")
 
 const { delay } = require("./testUtils")
 
 const MAX_RETRIES = 4
+const AUTOMATION_ID = generateAutomationID()
 const TEST_AUTOMATION = {
-  _id: "Test Automation",
+  _id: AUTOMATION_ID,
   name: "My Automation",
   pageId: "123123123",
   screenId: "kasdkfldsafkl",
@@ -126,14 +128,14 @@ describe("/automations", () => {
     it("should setup the automation fully", () => {
       let trigger = TRIGGER_DEFINITIONS["RECORD_SAVED"]
       trigger.id = "wadiawdo34"
-      let saveAction = ACTION_DEFINITIONS["SAVE_RECORD"]
+      let createAction = ACTION_DEFINITIONS["CREATE_RECORD"]
-      saveAction.inputs.record = {
+      createAction.inputs.record = {
         name: "{{trigger.name}}",
         description: "{{trigger.description}}"
       }
-      saveAction.id = "awde444wk"
+      createAction.id = "awde444wk"
 
-      TEST_AUTOMATION.definition.steps.push(saveAction)
+      TEST_AUTOMATION.definition.steps.push(createAction)
       TEST_AUTOMATION.definition.trigger = trigger
     })
 
@@ -206,7 +208,7 @@ describe("/automations", () => {
         .expect('Content-Type', /json/)
         .expect(200)
 
-        expect(res.body.message).toEqual("Automation Test Automation updated successfully.")
+        expect(res.body.message).toEqual(`Automation ${AUTOMATION_ID} updated successfully.`)
         expect(res.body.automation.name).toEqual("Updated Name")
     })
   })

packages/server/src/automations/actions.js

@@ -1,5 +1,5 @@
 const sendEmail = require("./steps/sendEmail")
-const saveRecord = require("./steps/saveRecord")
+const createRecord = require("./steps/createRecord")
 const updateRecord = require("./steps/updateRecord")
 const deleteRecord = require("./steps/deleteRecord")
 const createUser = require("./steps/createUser")
@@ -17,14 +17,14 @@ const DEFAULT_DIRECTORY = ".budibase-automations"
 const AUTOMATION_MANIFEST = "manifest.json"
 const BUILTIN_ACTIONS = {
   SEND_EMAIL: sendEmail.run,
-  SAVE_RECORD: saveRecord.run,
+  CREATE_RECORD: createRecord.run,
   UPDATE_RECORD: updateRecord.run,
   DELETE_RECORD: deleteRecord.run,
   CREATE_USER: createUser.run,
 }
 const BUILTIN_DEFINITIONS = {
   SEND_EMAIL: sendEmail.definition,
-  SAVE_RECORD: saveRecord.definition,
+  CREATE_RECORD: createRecord.definition,
   UPDATE_RECORD: updateRecord.definition,
   DELETE_RECORD: deleteRecord.definition,
   CREATE_USER: createUser.definition,

packages/server/src/automations/steps/createRecord.js

@@ -2,12 +2,12 @@ const recordController = require("../../api/controllers/record")
 const automationUtils = require("../automationUtils")
 
 module.exports.definition = {
-  name: "Save Record",
+  name: "Create Record",
-  tagline: "Save a {{inputs.enriched.model.name}} record",
+  tagline: "Create a {{inputs.enriched.model.name}} record",
   icon: "ri-save-3-fill",
-  description: "Save a record to your database",
+  description: "Create a record to your database",
   type: "ACTION",
-  stepId: "SAVE_RECORD",
+  stepId: "CREATE_RECORD",
   inputs: {},
   schema: {
     inputs: {

packages/server/src/automations/steps/filter.js

@@ -55,7 +55,15 @@ module.exports.definition = {
 }
 
 module.exports.run = async function filter({ inputs }) {
-  const { field, condition, value } = inputs
+  let { field, condition, value } = inputs
+  // coerce types so that we can use them
+  if (!isNaN(value) && !isNaN(field)) {
+    value = parseFloat(value)
+    field = parseFloat(field)
+  } else if (!isNaN(Date.parse(value)) && !isNaN(Date.parse(field))) {
+    value = Date.parse(value)
+    field = Date.parse(field)
+  }
   let success
   if (typeof field !== "object" && typeof value !== "object") {
     switch (condition) {

packages/server/src/automations/triggers.js

@@ -1,6 +1,7 @@
 const CouchDB = require("../db")
 const emitter = require("../events/index")
 const InMemoryQueue = require("./queue/inMemoryQueue")
+const { getAutomationParams } = require("../db/utils")
 
 let automationQueue = new InMemoryQueue()
 
@@ -89,15 +90,18 @@ async function queueRelevantRecordAutomations(event, eventType) {
     throw `No instanceId specified for ${eventType} - check event emitters.`
   }
   const db = new CouchDB(event.instanceId)
-  const automationsToTrigger = await db.query(
+  let automations = await db.allDocs(
-    "database/by_automation_trigger",
+    getAutomationParams(null, { include_docs: true })
-    {
-      key: [eventType],
-      include_docs: true,
-    }
   )
 
-  const automations = automationsToTrigger.rows.map(wf => wf.doc)
+  // filter down to the correct event type
+  automations = automations.rows
+    .map(automation => automation.doc)
+    .filter(automation => {
+      const trigger = automation.definition.trigger
+      return trigger && trigger.event === eventType
+    })
+
   for (let automation of automations) {
     let automationDef = automation.definition
     let automationTrigger = automationDef ? automationDef.trigger : {}

packages/server/src/db/utils.js

@@ -0,0 +1,152 @@
+const newid = require("./newid")
+
+const DocumentTypes = {
+  MODEL: "model",
+  RECORD: "record",
+  USER: "user",
+  AUTOMATION: "automation",
+  LINK: "link",
+  APP: "app",
+  ACCESS_LEVEL: "accesslevel",
+}
+
+exports.DocumentTypes = DocumentTypes
+
+const UNICODE_MAX = "\ufff0"
+
+/**
+ * If creating DB allDocs/query params with only a single top level ID this can be used, this
+ * is usually the case as most of our docs are top level e.g. models, automations, users and so on.
+ * More complex cases such as link docs and records which have multiple levels of IDs that their
+ * ID consists of need their own functions to build the allDocs parameters.
+ * @param {string} docType The type of document which input params are being built for, e.g. user,
+ * link, app, model and so on.
+ * @param {string|null} docId The ID of the document minus its type - this is only needed if looking
+ * for a singular document.
+ * @param {object} otherProps Add any other properties onto the request, e.g. include_docs.
+ * @returns {object} Parameters which can then be used with an allDocs request.
+ */
+function getDocParams(docType, docId = null, otherProps = {}) {
+  if (docId == null) {
+    docId = ""
+  }
+  return {
+    ...otherProps,
+    startkey: `${docType}:${docId}`,
+    endkey: `${docType}:${docId}${UNICODE_MAX}`,
+  }
+}
+
+/**
+ * Gets parameters for retrieving models, this is a utility function for the getDocParams function.
+ */
+exports.getModelParams = (modelId = null, otherProps = {}) => {
+  return getDocParams(DocumentTypes.MODEL, modelId, otherProps)
+}
+
+/**
+ * Generates a new model ID.
+ * @returns {string} The new model ID which the model doc can be stored under.
+ */
+exports.generateModelID = () => {
+  return `${DocumentTypes.MODEL}:${newid()}`
+}
+
+/**
+ * Gets the DB allDocs/query params for retrieving a record.
+ * @param {string} modelId The model in which the records have been stored.
+ * @param {string|null} recordId The ID of the record which is being specifically queried for. This can be
+ * left null to get all the records in the model.
+ * @param {object} otherProps Any other properties to add to the request.
+ * @returns {object} Parameters which can then be used with an allDocs request.
+ */
+exports.getRecordParams = (modelId, recordId = null, otherProps = {}) => {
+  if (modelId == null) {
+    throw "Cannot build params for records without a model ID"
+  }
+  const endOfKey = recordId == null ? `${modelId}:` : `${modelId}:${recordId}`
+  return getDocParams(DocumentTypes.RECORD, endOfKey, otherProps)
+}
+
+/**
+ * Gets a new record ID for the specified model.
+ * @param {string} modelId The model which the record is being created for.
+ * @returns {string} The new ID which a record doc can be stored under.
+ */
+exports.generateRecordID = modelId => {
+  return `${DocumentTypes.RECORD}:${modelId}:${newid()}`
+}
+
+/**
+ * Gets parameters for retrieving users, this is a utility function for the getDocParams function.
+ */
+exports.getUserParams = (username = null, otherProps = {}) => {
+  return getDocParams(DocumentTypes.USER, username, otherProps)
+}
+
+/**
+ * Generates a new user ID based on the passed in username.
+ * @param {string} username The username which the ID is going to be built up of.
+ * @returns {string} The new user ID which the user doc can be stored under.
+ */
+exports.generateUserID = username => {
+  return `${DocumentTypes.USER}:${username}`
+}
+
+/**
+ * Gets parameters for retrieving automations, this is a utility function for the getDocParams function.
+ */
+exports.getAutomationParams = (automationId = null, otherProps = {}) => {
+  return getDocParams(DocumentTypes.AUTOMATION, automationId, otherProps)
+}
+
+/**
+ * Generates a new automation ID.
+ * @returns {string} The new automation ID which the automation doc can be stored under.
+ */
+exports.generateAutomationID = () => {
+  return `${DocumentTypes.AUTOMATION}:${newid()}`
+}
+
+/**
+ * Generates a new link doc ID. This is currently not usable with the alldocs call,
+ * instead a view is built to make walking to tree easier.
+ * @param {string} modelId1 The ID of the linker model.
+ * @param {string} modelId2 The ID of the linked model.
+ * @param {string} recordId1 The ID of the linker record.
+ * @param {string} recordId2 The ID of the linked record.
+ * @returns {string} The new link doc ID which the automation doc can be stored under.
+ */
+exports.generateLinkID = (modelId1, modelId2, recordId1, recordId2) => {
+  return `${DocumentTypes.AUTOMATION}:${modelId1}:${modelId2}:${recordId1}:${recordId2}`
+}
+
+/**
+ * Generates a new app ID.
+ * @returns {string} The new app ID which the app doc can be stored under.
+ */
+exports.generateAppID = () => {
+  return `${DocumentTypes.APP}:${newid()}`
+}
+
+/**
+ * Gets parameters for retrieving apps, this is a utility function for the getDocParams function.
+ */
+exports.getAppParams = (appId = null, otherProps = {}) => {
+  return getDocParams(DocumentTypes.APP, appId, otherProps)
+}
+
+/**
+ * Generates a new access level ID.
+ * @returns {string} The new access level ID which the access level doc can be stored under.
+ */
+exports.generateAccessLevelID = () => {
+  return `${DocumentTypes.ACCESS_LEVEL}:${newid()}`
+}
+
+/**
+ * Gets parameters for retrieving an access level, this is a utility function for the getDocParams function.
+ */
+exports.getAccessLevelParams = (accessLevelId = null, otherProps = {}) => {
+  return getDocParams(DocumentTypes.ACCESS_LEVEL, accessLevelId, otherProps)
+}