Adding pagination control to the API and to the frontend, as well as getting view working as expected, emitting different key combinations to be able to search by any pattern.

packages/builder/src/builderStore/store/automation/index.js

@@ -122,10 +122,12 @@ const automationActions = store => ({
       return state
     })
   },
-  getLogs: async (automationId, startDate) => {
+  getLogs: async ({ automationId, startDate, status, page } = {}) => {
     return await API.getAutomationLogs({
       automationId,
       startDate,
+      status,
+      page,
     })
   },
   addTestDataToAutomation: data => {

packages/builder/src/components/portal/overview/HistoryTab.svelte

@@ -1,5 +1,5 @@
 <script>
-  import { Layout, Table, Select } from "@budibase/bbui"
+  import { Layout, Table, Select, Pagination } from "@budibase/bbui"
   import DateTimeRenderer from "components/common/renderers/DateTimeRenderer.svelte"
   import StatusRenderer from "./StatusRenderer.svelte"
   import HistoryDetailsPanel from "./HistoryDetailsPanel.svelte"
@@ -8,9 +8,24 @@
 
   export let appId
 
+  let runHistory = []
   let showPanel = false
   let selectedHistory = null
-  let runHistory = []
+  let automationOptions = []
+  let automationId = null
+  let status = null
+  let prevPage,
+    nextPage,
+    page,
+    hasNextPage,
+    pageNumber = 1
+
+  $: fetchLogs(automationId, status, page)
+
+  const statusOptions = [
+    { value: "success", label: "Success" },
+    { value: "error", label: "Error" },
+  ]
 
   const runHistorySchema = {
     status: { displayName: "Status" },
@@ -23,7 +38,33 @@
     { column: "status", component: StatusRenderer },
   ]
 
+  async function fetchLogs(automationId, status, page) {
+    const response = await automationStore.actions.getLogs({
+      automationId,
+      status,
+      page,
+    })
+    nextPage = response.nextPage
+    hasNextPage = response.hasNextPage
+    runHistory = enrichHistory($automationStore.blockDefinitions, response.data)
+  }
+
+  function goToNextPage() {
+    pageNumber++
+    prevPage = page
+    page = nextPage
+  }
+
+  function goToPrevPage() {
+    pageNumber--
+    nextPage = page
+    page = prevPage
+  }
+
   function enrichHistory(definitions, runHistory) {
+    if (!definitions) {
+      return []
+    }
     const finalHistory = []
     for (let history of runHistory) {
       if (!history.steps) {
@@ -31,8 +72,8 @@
       }
       let notFound = false
       for (let step of history.steps) {
-        const trigger = definitions.trigger[step.stepId],
-          action = definitions.action[step.stepId]
+        const trigger = definitions.TRIGGER[step.stepId],
+          action = definitions.ACTION[step.stepId]
         if (!trigger && !action) {
           notFound = true
           break
@@ -53,11 +94,12 @@
   }
 
   onMount(async () => {
-    let definitions = await automationStore.actions.definitions()
-    runHistory = enrichHistory(
-      definitions,
-      await automationStore.actions.getLogs()
-    )
+    await automationStore.actions.fetch()
+    await fetchLogs()
+    automationOptions = []
+    for (let automation of $automationStore.automations) {
+      automationOptions.push({ value: automation._id, label: automation.name })
+    }
   })
 </script>
 
@@ -65,13 +107,23 @@
   <Layout paddingX="XL" gap="S" alignContent="start">
     <div class="search">
       <div class="select">
-        <Select placeholder="All automations" label="Automation" />
+        <Select
+          placeholder="All automations"
+          label="Automation"
+          bind:value={automationId}
+          options={automationOptions}
+        />
       </div>
       <div class="select">
         <Select placeholder="Past 30 days" label="Date range" />
       </div>
       <div class="select">
-        <Select placeholder="All status" label="Status" />
+        <Select
+          placeholder="All status"
+          label="Status"
+          bind:value={status}
+          options={statusOptions}
+        />
       </div>
     </div>
     {#if runHistory}
@@ -95,6 +147,15 @@
     />
   </div>
 </div>
+<div class="pagination">
+  <Pagination
+    page={pageNumber}
+    hasPrevPage={pageNumber > 1}
+    {hasNextPage}
+    {goToPrevPage}
+    {goToNextPage}
+  />
+</div>
 
 <style>
   .root {
@@ -118,12 +179,11 @@
     flex-basis: 150px;
   }
 
-  .separator {
-    flex-grow: 1;
-  }
-
-  .searchInput {
-    margin-top: auto;
+  .pagination {
+    position: absolute;
+    bottom: 0;
+    margin-bottom: var(--spacing-xl);
+    margin-left: var(--spacing-l);
   }
 
   .panel {

packages/builder/src/components/portal/overview/StatusRenderer.svelte

@@ -10,7 +10,9 @@
 
 <div class="cell">
   <Icon {color} name={isError ? "Alert" : "CheckmarkCircle"} />
-  <div class:green={!isError} class:red={isError}>{value}</div>
+  <div class:green={!isError} class:red={isError}>
+    {isError ? "Error" : "Success"}
+  </div>
 </div>
 
 <style>

packages/builder/src/pages/builder/app/[application]/automate/_layout.svelte

@@ -5,6 +5,7 @@
   import CreateAutomationModal from "components/automation/AutomationPanel/CreateAutomationModal.svelte"
   import CreateWebhookModal from "components/automation/Shared/CreateWebhookModal.svelte"
   import TestPanel from "components/automation/AutomationBuilder/TestPanel.svelte"
+  import { onMount } from "svelte"
 
   $: automation =
     $automationStore.selectedAutomation?.automation ||
@@ -12,6 +13,9 @@
 
   let modal
   let webhookModal
+  onMount(() => {
+    $automationStore.showTestPanel = false
+  })
 </script>
 
 <!-- routify:options index=3 -->

packages/frontend-core/src/api/automations.js

@@ -79,12 +79,14 @@ export const buildAutomationEndpoints = API => ({
    * @param automationId The ID of the automation to get logs for.
    * @param startDate An ISO date string to state the start of the date range.
    */
-  getAutomationLogs: async ({ automationId, startDate }) => {
+  getAutomationLogs: async ({ automationId, startDate, status, page }) => {
     return await API.post({
       url: "/api/automations/logs/search",
       body: {
         automationId,
         startDate,
+        status,
+        page,
       },
     })
   },

packages/server/src/api/controllers/automation.js

@@ -1,6 +1,6 @@
 const actions = require("../../automations/actions")
 const triggers = require("../../automations/triggers")
-const { getLogs, oneDayAgo } = require("../../automations/history")
+const { getLogs, oneDayAgo } = require("../../automations/logging")
 const { getAutomationParams, generateAutomationID } = require("../../db/utils")
 const {
   checkForWebhooks,
@@ -152,11 +152,11 @@ exports.destroy = async function (ctx) {
 }
 
 exports.logSearch = async function (ctx) {
-  const { automationId } = ctx.request.body
+  const { automationId, status, page } = ctx.request.body
   // TODO: check if there is a date range in the search params
   // also check the date range vs their license, see if it is allowed
   const startDate = oneDayAgo()
-  ctx.body = await getLogs(startDate, automationId)
+  ctx.body = await getLogs(startDate, status, automationId, page)
 }
 
 exports.getActionList = async function (ctx) {

packages/server/src/automations/history/index.ts

@@ -1,140 +0,0 @@
-import {
-  AutomationLog,
-  AutomationResults,
-  AutomationStatus,
-} from "../../definitions/automation"
-import { getAppDB } from "@budibase/backend-core/context"
-import {
-  generateAutomationLogID,
-  getAutomationLogParams,
-  getQueryIndex,
-  ViewNames,
-} from "../../db/utils"
-import { createLogByAutomationView } from "../../db/views/staticViews"
-import { Automation } from "../../definitions/common"
-
-const EARLIEST_DATE = new Date(0).toISOString()
-const FREE_EXPIRY_SEC = 86400
-const PRO_EXPIRY_SEC = FREE_EXPIRY_SEC * 30
-
-function getStatus(results: AutomationResults) {
-  let status = AutomationStatus.SUCCESS
-  let first = true
-  for (let step of results.steps) {
-    // skip the trigger, its always successful if automation ran
-    if (first) {
-      first = false
-      continue
-    }
-    if (!step.outputs?.success) {
-      status = AutomationStatus.ERROR
-    }
-  }
-  return status
-}
-
-// export function oneMonthAgo() {
-//   return new Date(
-//     new Date().getTime() - PRO_EXPIRY_SEC * 1000
-//   ).toISOString()
-// }
-
-export function oneDayAgo() {
-  return new Date(new Date().getTime() - FREE_EXPIRY_SEC * 1000).toISOString()
-}
-
-async function clearOldHistory() {
-  const db = getAppDB()
-  // TODO: handle license lookup for deletion
-  const expiredEnd = oneDayAgo()
-  const results = await getAllLogs(EARLIEST_DATE, expiredEnd, {
-    include_docs: false,
-  })
-  const toDelete = results.map((doc: any) => ({
-    _id: doc.id,
-    _rev: doc.rev,
-    _deleted: true,
-  }))
-  await db.bulkDocs(toDelete)
-}
-
-async function getAllLogs(
-  startDate: string,
-  endDate: string,
-  opts: any = { include_docs: true }
-) {
-  const db = getAppDB()
-  const queryParams: any = {
-    endDate,
-    startDate,
-  }
-  let response = (await db.allDocs(getAutomationLogParams(queryParams, opts)))
-    .rows
-  if (opts?.include_docs) {
-    response = response.map((row: any) => row.doc)
-  }
-  return response
-}
-
-async function getLogsByAutomationID(
-  automationId: string,
-  opts: { startDate?: string; endDate?: string } = {}
-): Promise<AutomationLog[]> {
-  const db = getAppDB()
-  try {
-    const queryParams = {
-      startDate: opts?.startDate,
-      endDate: opts?.startDate,
-      automationId,
-    }
-    return (
-      await db.query(
-        getQueryIndex(ViewNames.LOGS_BY_AUTOMATION),
-        getAutomationLogParams(queryParams, { include_docs: true })
-      )
-    ).rows.map((row: any) => row.doc)
-  } catch (err: any) {
-    if (err != null && err.name === "not_found") {
-      await createLogByAutomationView()
-      return getLogsByAutomationID(automationId, opts)
-    }
-  }
-  return []
-}
-
-export async function storeLog(
-  automation: Automation,
-  results: AutomationResults
-) {
-  const automationId = automation._id
-  const name = automation.name
-  const db = getAppDB()
-  const isoDate = new Date().toISOString()
-  const id = generateAutomationLogID(automationId, isoDate)
-
-  await db.put({
-    // results contain automationId and status for view
-    ...results,
-    automationId,
-    automationName: name,
-    status: getStatus(results),
-    createdAt: isoDate,
-    _id: id,
-  })
-  // clear up old history for app
-  await clearOldHistory()
-}
-
-export async function getLogs(startDate: string, automationId?: string) {
-  let logs: AutomationLog[]
-  let endDate = new Date().toISOString()
-  if (automationId) {
-    logs = await getLogsByAutomationID(automationId, {
-      startDate,
-      endDate,
-    })
-  } else {
-    logs = await getAllLogs(startDate, endDate)
-  }
-  return logs
-}

packages/server/src/automations/logging/index.ts

@@ -0,0 +1,180 @@
+import {
+  AutomationLog,
+  AutomationLogPage,
+  AutomationResults,
+  AutomationStatus,
+} from "../../definitions/automation"
+import { getAppDB } from "@budibase/backend-core/context"
+import {
+  generateAutomationLogID,
+  getAutomationLogParams,
+  getQueryIndex,
+  ViewNames,
+} from "../../db/utils"
+import { createLogByAutomationView } from "../../db/views/staticViews"
+import { Automation } from "../../definitions/common"
+
+const PAGE_SIZE = 9
+const EARLIEST_DATE = new Date(0).toISOString()
+const FREE_EXPIRY_SEC = 86400
+// const PRO_EXPIRY_SEC = FREE_EXPIRY_SEC * 30
+
+function getStatus(results: AutomationResults) {
+  let status = AutomationStatus.SUCCESS
+  let first = true
+  for (let step of results.steps) {
+    // skip the trigger, its always successful if automation ran
+    if (first) {
+      first = false
+      continue
+    }
+    if (!step.outputs?.success) {
+      status = AutomationStatus.ERROR
+    }
+  }
+  return status
+}
+
+// export function oneMonthAgo() {
+//   return new Date(
+//     new Date().getTime() - PRO_EXPIRY_SEC * 1000
+//   ).toISOString()
+// }
+
+export function oneDayAgo() {
+  return new Date(new Date().getTime() - FREE_EXPIRY_SEC * 1000).toISOString()
+}
+
+async function clearOldHistory() {
+  const db = getAppDB()
+  // TODO: handle license lookup for deletion
+  const expiredEnd = oneDayAgo()
+  const results = await getAllLogs(EARLIEST_DATE, expiredEnd, {
+    docs: false,
+  })
+  const toDelete = results.data.map((doc: any) => ({
+    _id: doc.id,
+    _rev: doc.rev,
+    _deleted: true,
+  }))
+  await db.bulkDocs(toDelete)
+}
+
+function pagination(
+  response: any,
+  paginate: boolean = true
+): AutomationLogPage {
+  const data = response.rows.map((row: any) => {
+    return row.doc ? row.doc : row
+  })
+  if (!paginate) {
+    return { data, hasNextPage: false }
+  }
+  const hasNextPage = data.length > PAGE_SIZE
+  return {
+    data: data.slice(0, PAGE_SIZE),
+    hasNextPage,
+    nextPage: hasNextPage ? data[PAGE_SIZE]?._id : undefined,
+  }
+}
+
+async function getAllLogs(
+  startDate: string,
+  endDate: string,
+  opts: {
+    docs: boolean
+    status?: string
+    paginate?: boolean
+    page?: string
+  } = { docs: true }
+): Promise<AutomationLogPage> {
+  const db = getAppDB()
+  let optional: any = { status: opts.status }
+  const params = getAutomationLogParams(startDate, endDate, optional, {
+    include_docs: opts.docs,
+    limit: opts?.paginate ? PAGE_SIZE + 1 : undefined,
+  })
+  if (opts?.page) {
+    params.startkey = opts.page
+  }
+  let response = await db.allDocs(params)
+  return pagination(response, opts?.paginate)
+}
+
+async function getLogsByView(
+  startDate: string,
+  endDate: string,
+  viewParams: { automationId?: string; status?: string; page?: string } = {}
+): Promise<AutomationLogPage> {
+  const db = getAppDB()
+  let response
+  try {
+    let optional = {
+      automationId: viewParams?.automationId,
+      status: viewParams?.status,
+    }
+    const params = getAutomationLogParams(startDate, endDate, optional, {
+      include_docs: true,
+      limit: PAGE_SIZE,
+    })
+    if (viewParams?.page) {
+      params.startkey = viewParams.page
+    }
+    response = await db.query(getQueryIndex(ViewNames.AUTO_LOGS), params)
+  } catch (err: any) {
+    if (err != null && err.name === "not_found") {
+      await createLogByAutomationView()
+      return getLogsByView(startDate, endDate, viewParams)
+    }
+  }
+  return pagination(response)
+}
+
+export async function storeLog(
+  automation: Automation,
+  results: AutomationResults
+) {
+  const db = getAppDB()
+  const automationId = automation._id
+  const name = automation.name
+  const status = getStatus(results)
+  const isoDate = new Date().toISOString()
+  const id = generateAutomationLogID(isoDate, status, automationId)
+
+  await db.put({
+    // results contain automationId and status for view
+    ...results,
+    automationId,
+    status,
+    automationName: name,
+    createdAt: isoDate,
+    _id: id,
+  })
+  // clear up old logging for app
+  await clearOldHistory()
+}
+
+export async function getLogs(
+  startDate: string,
+  status?: string,
+  automationId?: string,
+  page?: string
+): Promise<AutomationLogPage> {
+  let response: AutomationLogPage
+  let endDate = new Date().toISOString()
+  if (automationId || status) {
+    response = await getLogsByView(startDate, endDate, {
+      automationId,
+      status,
+      page,
+    })
+  } else {
+    response = await getAllLogs(startDate, endDate, {
+      status,
+      page,
+      docs: true,
+      paginate: true,
+    })
+  }
+  return response
+}

packages/server/src/db/utils.js

@@ -49,7 +49,13 @@ const DocumentTypes = {
 const ViewNames = {
   LINK: "by_link",
   ROUTING: "screen_routes",
-  LOGS_BY_AUTOMATION: "log_by_auto",
+  AUTO_LOGS: "auto_log",
+}
+
+const ViewModes = {
+  ALL: "all",
+  AUTOMATION: "auto",
+  STATUS: "status",
 }
 
 const InternalTables = {
@@ -77,6 +83,7 @@ exports.isProdAppID = isProdAppID
 exports.USER_METDATA_PREFIX = `${DocumentTypes.ROW}${SEPARATOR}${InternalTables.USER_METADATA}${SEPARATOR}`
 exports.LINK_USER_METADATA_PREFIX = `${DocumentTypes.LINK}${SEPARATOR}${InternalTables.USER_METADATA}${SEPARATOR}`
 exports.ViewNames = ViewNames
+exports.ViewModes = ViewModes
 exports.InternalTables = InternalTables
 exports.DocumentTypes = DocumentTypes
 exports.SEPARATOR = SEPARATOR
@@ -364,26 +371,32 @@ exports.getMemoryViewParams = (otherProps = {}) => {
   return getDocParams(DocumentTypes.MEM_VIEW, null, otherProps)
 }
 
-exports.generateAutomationLogID = (automationId, isoDate) => {
+exports.generateAutomationLogID = (isoDate, status, automationId) => {
   return `${DocumentTypes.AUTOMATION_LOG}${SEPARATOR}${isoDate}${SEPARATOR}${automationId}`
 }
 
 exports.getAutomationLogParams = (
-  { startDate, endDate, automationId } = {},
+  startDate,
+  endDate,
+  { status, automationId } = {},
   otherProps = {}
 ) => {
-  const base = `${DocumentTypes.AUTOMATION_LOG}${SEPARATOR}`
-  let start = startDate || "",
-    end = endDate || ""
-  // reverse for view
-  if (automationId) {
-    start = `${automationId}${SEPARATOR}${start}`
-    end = `${automationId}${SEPARATOR}${end}`
+  const automationBase = automationId ? `${automationId}${SEPARATOR}` : ""
+  const statusBase = status ? `${status}${SEPARATOR}` : ""
+  let base
+  if (status && automationId) {
+    base = `${ViewModes.ALL}${SEPARATOR}${statusBase}${automationBase}`
+  } else if (status) {
+    base = `${ViewModes.STATUS}${SEPARATOR}${statusBase}`
+  } else if (automationId) {
+    base = `${ViewModes.AUTOMATION}${SEPARATOR}${automationBase}`
+  } else {
+    base = `${DocumentTypes.AUTOMATION_LOG}${SEPARATOR}`
   }
   return {
     ...otherProps,
-    startkey: `${base}${start}`,
-    endkey: `${base}${end}${UNICODE_MAX}`,
+    startkey: `${base}${startDate}`,
+    endkey: `${base}${endDate}${UNICODE_MAX}`,
   }
 }
 

packages/server/src/db/views/staticViews.js

@@ -3,6 +3,7 @@ const {
   DocumentTypes,
   SEPARATOR,
   ViewNames,
+  ViewModes,
   SearchIndexes,
 } = require("../utils")
 const SCREEN_PREFIX = DocumentTypes.SCREEN + SEPARATOR
@@ -69,14 +70,20 @@ exports.createLogByAutomationView = async () => {
   const view = {
     map: `function(doc) {
       if (doc._id.startsWith("${LOG_PREFIX}")) {
-        let key = doc.automationId + ${SEPARATOR} + doc.createdAt
-        emit(key, doc._id)
+        let autoId = doc.automationId + "${SEPARATOR}"
+        let status = doc.status + "${SEPARATOR}"
+        let autoKey = "${ViewModes.AUTOMATION}${SEPARATOR}" + autoId + doc.createdAt
+        let statusKey = "${ViewModes.STATUS}${SEPARATOR}" + status + doc.createdAt
+        let allKey = "${ViewModes.ALL}${SEPARATOR}" + status + autoId + doc.createdAt
+        emit(statusKey)
+        emit(autoKey)
+        emit(allKey)
       }
     }`,
   }
   designDoc.views = {
     ...designDoc.views,
-    [ViewNames.LOGS_BY_AUTOMATION]: view,
+    [ViewNames.AUTO_LOGS]: view,
   }
   await db.put(designDoc)
 }

packages/server/src/definitions/automation.ts

@@ -23,3 +23,9 @@ export interface AutomationLog extends AutomationResults {
   _id: string
   _rev: string
 }
+
+export interface AutomationLogPage {
+  data: AutomationLog[]
+  hasNextPage: boolean
+  nextPage?: string
+}

packages/server/src/threads/automation.js

@@ -9,7 +9,7 @@ const { doInTenant } = require("@budibase/backend-core/tenancy")
 const { definitions: triggerDefs } = require("../automations/triggerInfo")
 const { doInAppContext, getAppDB } = require("@budibase/backend-core/context")
 const { AutomationErrors, LoopStepTypes } = require("../constants")
-const { storeLog } = require("../automations/history")
+const { storeLog } = require("../automations/logging")
 const FILTER_STEP_ID = actions.ACTION_DEFINITIONS.FILTER.stepId
 const LOOP_STEP_ID = actions.ACTION_DEFINITIONS.LOOP.stepId
 
@@ -326,7 +326,7 @@ class Orchestrator {
       }
     }
 
-    // store the history for the automation run
+    // store the logs for the automation run
     await storeLog(this._automation, this.executionOutput)
     return this.executionOutput
   }