Progress towards using couchdb for automation logs, after testing redis and realising it won't work.

packages/backend-core/src/redis/utils.js

@@ -19,6 +19,7 @@ exports.Databases = {
   QUERY_VARS: "queryVars",
   LICENSES: "license",
   GENERIC_CACHE: "data_cache",
+  AUTOMATION_LOGS: "autoLogs",
 }
 
 exports.SEPARATOR = SEPARATOR

packages/builder/src/components/automation/AutomationBuilder/TestDisplay.svelte

@@ -13,6 +13,13 @@
     return results.steps.filter(x => x.stepId !== "LOOP" || [])
   }
 
+  function textArea(results, message) {
+    if (!results) {
+      return message
+    }
+    return JSON.stringify(results, null, 2)
+  }
+
   $: filteredResults = prepTestResults(testResults)
 
   $: {
@@ -66,11 +73,7 @@
                   <TextArea
                     minHeight="80px"
                     disabled
-                    value={JSON.stringify(
-                      filteredResults?.[idx]?.inputs,
-                      null,
-                      2
-                    )}
+                    value={textArea(filteredResults?.[idx]?.inputs, "No input")}
                   />
                 </div></Tab
               >
@@ -79,10 +82,9 @@
                   <TextArea
                     minHeight="100px"
                     disabled
-                    value={JSON.stringify(
+                    value={textArea(
                       filteredResults?.[idx]?.outputs,
-                      null,
-                      2
+                      "No output"
                     )}
                   />
                 </div>

packages/builder/src/components/portal/overview/HistoryTab.svelte

@@ -56,7 +56,7 @@
     let definitions = await automationStore.actions.definitions()
     runHistory = enrichHistory(definitions, [
       {
-        status: "Error",
+        status: "Success",
         timestamp: "2022-05-11T16:06:14.438Z",
         name: "automation name",
         steps: [
@@ -88,7 +88,7 @@
         ],
       },
       {
-        status: "Success",
+        status: "Error",
         timestamp: "2022-05-11T16:03:14.438Z",
         name: "automation name",
         steps: [
@@ -151,7 +151,7 @@
   }
 
   .panelOpen {
-    grid-template-columns: auto 360px;
+    grid-template-columns: auto 420px;
   }
 
   .search {
@@ -178,7 +178,7 @@
     position: absolute;
     right: 0;
     height: 100%;
-    width: 360px;
+    width: 420px;
     overflow: hidden;
     background-color: var(--background);
   }

packages/server/src/automations/history/index.ts

@@ -0,0 +1,67 @@
+import { AutomationResults } from "../../definitions/automation"
+import { getAppDB } from "@budibase/backend-core/context"
+import {
+  generateAutomationLogID,
+  getAutomationLogParams,
+  // getQueryIndex,
+  // ViewNames,
+} from "../../db/utils"
+// import { createLogByAutomationView } from "../../db/views/staticViews"
+
+const FREE_EXPIRY_SEC = 86400
+// const PRO_EXPIRY_SEC = FREE_EXPIRY_SEC * 30
+
+async function clearOldHistory() {
+  const db = getAppDB()
+  // TODO: handle license lookup for deletion
+  const time = new Date(new Date().getTime() - FREE_EXPIRY_SEC * 1000)
+  const queryParams: any = {
+    endIso: time,
+  }
+  const results = await db.allDocs(getAutomationLogParams(queryParams))
+  const toDelete = results.map((doc: any) => ({
+    _id: doc.id,
+    _rev: doc.rev,
+    _deleted: true,
+  }))
+  await db.bulkDocs(toDelete)
+}
+
+// async function getByAutomationID(automationId: string) {
+//   const db = getAppDB()
+//   try {
+//     const queryParams: any = {
+//       automationId,
+//     }
+//     return (
+//       await db.query(
+//         getQueryIndex(ViewNames.LOGS_BY_AUTOMATION),
+//         getAutomationLogParams(queryParams, { include_docs: true })
+//       )
+//     ).rows.map((row: any) => row.doc)
+//   } catch (err: any) {
+//     if (err != null && err.name === "not_found") {
+//       await createLogByAutomationView()
+//       return getByAutomationID(automationId)
+//     }
+//   }
+// }
+
+export async function storeHistory(
+  automationId: string,
+  results: AutomationResults
+) {
+  const db = getAppDB()
+  const isoDate = new Date().toISOString()
+  const id = generateAutomationLogID(isoDate, automationId)
+  await db.put({
+    // results contain automationId and status for view
+    ...results,
+    createdAt: isoDate,
+    _id: id,
+  })
+  // clear up old history for app
+  await clearOldHistory()
+}
+
+export async function retrieve() {}

packages/server/src/db/utils.js

@@ -31,6 +31,7 @@ const DocumentTypes = {
   ROW: "ro",
   USER: "us",
   AUTOMATION: "au",
+  AUTOMATION_LOG: "log_au",
   LINK: "li",
   WEBHOOK: "wh",
   INSTANCE: "inst",
@@ -48,6 +49,7 @@ const DocumentTypes = {
 const ViewNames = {
   LINK: "by_link",
   ROUTING: "screen_routes",
+  LOGS_BY_AUTOMATION: "log_by_auto",
 }
 
 const InternalTables = {
@@ -362,6 +364,29 @@ exports.getMemoryViewParams = (otherProps = {}) => {
   return getDocParams(DocumentTypes.MEM_VIEW, null, otherProps)
 }
 
+exports.generateAutomationLogID = (automationId, isoDate) => {
+  return `${DocumentTypes.AUTOMATION_LOG}${SEPARATOR}${isoDate}${SEPARATOR}${automationId}`
+}
+
+exports.getAutomationLogParams = (
+  { startIso, endIso, automationId } = {},
+  otherProps = {}
+) => {
+  const base = `${DocumentTypes.AUTOMATION_LOG}${SEPARATOR}`
+  let start = startIso || "",
+    end = endIso || ""
+  // reverse for view
+  if (automationId) {
+    start = `${automationId}${SEPARATOR}${start}`
+    end = `${automationId}${SEPARATOR}${end}`
+  }
+  return {
+    ...otherProps,
+    startkey: `${base}${start}`,
+    endkey: `${base}${end}${UNICODE_MAX}`,
+  }
+}
+
 /**
  * This can be used with the db.allDocs to get a list of IDs
  */

packages/server/src/db/views/staticViews.js

@@ -6,6 +6,7 @@ const {
   SearchIndexes,
 } = require("../utils")
 const SCREEN_PREFIX = DocumentTypes.SCREEN + SEPARATOR
+const LOG_PREFIX = DocumentTypes.AUTOMATION_LOG + SEPARATOR
 
 /**************************************************
  *                  INFORMATION                   *
@@ -58,6 +59,28 @@ exports.createLinkView = async () => {
   await db.put(designDoc)
 }
 
+/**
+ * A separate view that allows us to perform queries by the automation ID and time series, while the
+ * main all_docs allows access to time series only
+ */
+exports.createLogByAutomationView = async () => {
+  const db = getAppDB()
+  const designDoc = await db.get("_design/database")
+  const view = {
+    map: `function(doc) {
+      if (doc._id.startsWith("${LOG_PREFIX}")) {
+        let key = doc.automationId + ${SEPARATOR} + doc.createdAt
+        emit(key, doc._id)
+      }
+    }`,
+  }
+  designDoc.views = {
+    ...designDoc.views,
+    [ViewNames.LOGS_BY_AUTOMATION]: view,
+  }
+  await db.put(designDoc)
+}
+
 exports.createRoutingView = async () => {
   const db = getAppDB()
   const designDoc = await db.get("_design/database")

packages/server/src/definitions/automation.ts

@@ -0,0 +1,13 @@
+export interface AutomationResults {
+  automationId: string
+  status: string
+  steps: {
+    stepId: string
+    inputs: {
+      [key: string]: any
+    }
+    outputs: {
+      [key: string]: any
+    }
+  }[]
+}

packages/server/src/module.d.ts

@@ -9,3 +9,4 @@ declare module "@budibase/backend-core/constants"
 declare module "@budibase/backend-core/auth"
 declare module "@budibase/backend-core/sessions"
 declare module "@budibase/backend-core/encryption"
+declare module "@budibase/backend-core/redis"

packages/server/src/threads/index.ts

@@ -77,7 +77,7 @@ export class Thread {
     })
   }
 
-  static shutdown() {
+  static stopThreads() {
     return new Promise<void>(resolve => {
       if (Thread.workerRefs.length === 0) {
         resolve()
@@ -95,4 +95,8 @@ export class Thread {
       Thread.workerRefs = []
     })
   }
+
+  static async shutdown() {
+    await Thread.stopThreads()
+  }
 }