Moving majority of automation logging functionality to pro.

packages/backend-core/src/index.ts

@@ -13,6 +13,7 @@ import deprovisioning from "./context/deprovision"
 import auth from "./auth"
 import constants from "./constants"
 import * as dbConstants from "./db/constants"
+import logging from "./logging"
 
 // mimic the outer package exports
 import * as db from "./pkg/db"
@@ -49,6 +50,7 @@ const core = {
   deprovisioning,
   installation,
   errors,
+  logging,
   ...errorClasses,
 }
 

packages/builder/src/components/portal/overview/automation/HistoryDetailsPanel.svelte

@@ -1,6 +1,6 @@
 <script>
   import { Layout, Icon, ActionButton } from "@budibase/bbui"
-  import StatusRenderer from "components/portal/overview/StatusRenderer.svelte"
+  import StatusRenderer from "./StatusRenderer.svelte"
   import DateTimeRenderer from "components/common/renderers/DateTimeRenderer.svelte"
   import TestDisplay from "components/automation/AutomationBuilder/TestDisplay.svelte"
   import { goto } from "@roxi/routify"

packages/server/src/api/controllers/application.ts

@@ -45,6 +45,7 @@ const { getTenantId, isMultiTenant } = require("@budibase/backend-core/tenancy")
 import { syncGlobalUsers } from "./user"
 const { app: appCache } = require("@budibase/backend-core/cache")
 import { cleanupAutomations } from "../../automations/utils"
+import { checkAppMetadata } from "../../automations/logging"
 const {
   getAppDB,
   getProdAppDB,
@@ -192,7 +193,7 @@ export const fetch = async (ctx: any) => {
     }
   }
 
-  ctx.body = apps
+  ctx.body = await checkAppMetadata(apps)
 }
 
 export const fetchAppDefinition = async (ctx: any) => {

packages/server/src/api/controllers/automation.js

@@ -21,7 +21,6 @@ const {
 const { events } = require("@budibase/backend-core")
 const { app } = require("@budibase/backend-core/cache")
 const { automations } = require("@budibase/pro")
-const { clearOldHistory } = require("../../automations/logging")
 
 const ACTION_DEFS = removeDeprecated(actions.ACTION_DEFINITIONS)
 const TRIGGER_DEFS = removeDeprecated(triggers.TRIGGER_DEFINITIONS)
@@ -195,15 +194,7 @@ exports.destroy = async function (ctx) {
 }
 
 exports.logSearch = async function (ctx) {
-  let { automationId, status, page, startDate } = ctx.request.body
+  ctx.body = await automations.logs.logSearch(ctx.request.body)
-  // before querying logs, make sure old logs are cleared out
-  await clearOldHistory()
-  ctx.body = await automations.logs.getLogs(
-    startDate,
-    status,
-    automationId,
-    page
-  )
 }
 
 exports.clearLogError = async function (ctx) {

packages/server/src/automations/logging/index.ts

@@ -1,132 +1,35 @@
-import { getAppId, getProdAppDB } from "@budibase/backend-core/context"
-import {
-  DocumentTypes,
-  generateAutomationLogID,
-  isProdAppID,
-  SEPARATOR,
-} from "../../db/utils"
-import { Automation } from "../../definitions/common"
-import { app } from "@budibase/backend-core/cache"
-import { backOff } from "../../utilities"
 import * as env from "../../environment"
+import { AutomationResults, Automation, App } from "@budibase/types"
 import { automations } from "@budibase/pro"
-import { AppMetadataErrors } from "@budibase/types"
+import { db as dbUtils } from "@budibase/backend-core"
-import { AutomationResults, AutomationStatus } from "@budibase/types"
-
-const { logAlert } = require("@budibase/backend-core/logging")
-
-function getStatus(results: AutomationResults) {
-  let status = AutomationStatus.SUCCESS
-  let first = true
-  for (let step of results.steps) {
-    // skip the trigger, its always successful if automation ran
-    if (first) {
-      first = false
-      continue
-    }
-    if (!step.outputs?.success) {
-      status = AutomationStatus.ERROR
-      break
-    } else if (step.outputs?.status?.toLowerCase() === "stopped") {
-      status = AutomationStatus.STOPPED
-      break
-    }
-  }
-  return status
-}
-
-export async function clearOldHistory() {
-  const db = getProdAppDB()
-  try {
-    const expired = await automations.logs.getExpiredLogs()
-    const toDelete = expired.data.map((doc: any) => ({
-      _id: doc.id,
-      _rev: doc.value.rev,
-      _deleted: true,
-    }))
-    const errorLogIds = expired.data
-      .filter((doc: any) => {
-        const parts = doc.id.split(SEPARATOR)
-        const status = parts[parts.length - 1]
-        return status === AutomationStatus.ERROR
-      })
-      .map((doc: any) => doc.id)
-    await db.bulkDocs(toDelete)
-    if (errorLogIds.length) {
-      await updateAppMetadataWithErrors(errorLogIds, { clearing: true })
-    }
-  } catch (err) {
-    logAlert(`Failed to cleanup automation log history - Database "${db.name}"`)
-  }
-}
-
-async function updateAppMetadataWithErrors(
-  logIds: string[],
-  { clearing } = { clearing: false }
-) {
-  const db = getProdAppDB()
-  // this will try multiple times with a delay between to update the metadata
-  await backOff(async () => {
-    const metadata = await db.get(DocumentTypes.APP_METADATA)
-    for (let logId of logIds) {
-      const parts = logId.split(SEPARATOR)
-      const autoId = `${parts[parts.length - 3]}${SEPARATOR}${
-        parts[parts.length - 2]
-      }`
-      let errors: AppMetadataErrors = {}
-      if (metadata.automationErrors) {
-        errors = metadata.automationErrors as AppMetadataErrors
-      }
-      if (!Array.isArray(errors[autoId])) {
-        errors[autoId] = []
-      }
-      const idx = errors[autoId].indexOf(logId)
-      if (clearing && idx !== -1) {
-        errors[autoId].splice(idx, 1)
-      } else {
-        errors[autoId].push(logId)
-      }
-      // if clearing and reach zero, this will pass and will remove the element
-      if (errors[autoId].length === 0) {
-        delete errors[autoId]
-      }
-      metadata.automationErrors = errors
-    }
-    await db.put(metadata)
-    // don't update cache until after DB put, make sure it has been stored successfully
-    await app.invalidateAppMetadata(metadata.appId, metadata)
-  }, "Failed to update app metadata with automation log error")
-}
 
 export async function storeLog(
   automation: Automation,
   results: AutomationResults
 ) {
   // can disable this if un-needed in self-host, also only do this for prod apps
-  if (env.DISABLE_AUTOMATION_LOGS || !isProdAppID(getAppId())) {
+  if (env.DISABLE_AUTOMATION_LOGS) {
     return
   }
-  const db = getProdAppDB()
+  await automations.logs.storeLog(automation, results)
-  const automationId = automation._id
+}
-  const name = automation.name
+
-  const status = getStatus(results)
+export async function checkAppMetadata(apps: App[]) {
-  const isoDate = new Date().toISOString()
+  const maxStartDate = await automations.logs.oldestLogDate()
-  const id = generateAutomationLogID(isoDate, status, automationId)
+  for (let metadata of apps) {
-  await db.put({
+    if (!metadata.automationErrors) {
-    // results contain automationId and status for view
+      continue
-    ...results,
+    }
-    automationId,
+    for (let [key, errors] of Object.entries(metadata.automationErrors)) {
-    status,
+      const updated = []
-    automationName: name,
+      for (let error of errors) {
-    createdAt: isoDate,
+        const startDate = error.split(dbUtils.SEPARATOR)[2]
-    _id: id,
+        if (startDate > maxStartDate) {
-  })
+          updated.push(error)
-
+        }
-  // need to note on the app metadata that there is an error, store what the error is
+      }
-  if (status === AutomationStatus.ERROR) {
+      metadata.automationErrors[key] = updated
-    await updateAppMetadataWithErrors([id])
+    }
-  }
+  }
-
+  return apps
-  // clear up old logging for app
-  await clearOldHistory()
 }

packages/server/src/db/utils.js

@@ -354,10 +354,6 @@ exports.getMemoryViewParams = (otherProps = {}) => {
   return getDocParams(DocumentTypes.MEM_VIEW, null, otherProps)
 }
 
-exports.generateAutomationLogID = (isoDate, status, automationId) => {
-  return `${DocumentTypes.AUTOMATION_LOG}${SEPARATOR}${isoDate}${SEPARATOR}${automationId}${SEPARATOR}${status}`
-}
-
 /**
  * This can be used with the db.allDocs to get a list of IDs
  */

packages/server/src/utilities/index.js

@@ -4,7 +4,6 @@ const { sanitizeKey } = require("@budibase/backend-core/objectStore")
 const { generateMetadataID } = require("../db/utils")
 const Readable = require("stream").Readable
 const { getAppDB } = require("@budibase/backend-core/context")
-const { logAlert } = require("@budibase/backend-core/logging")
 
 const BB_CDN = "https://cdn.budi.live"
 
@@ -14,42 +13,6 @@ exports.isDev = env.isDev
 
 exports.NUMBER_REGEX = /^[+-]?([0-9]*[.])?[0-9]+$/g
 
-exports.randomDelay = fn => {
-  return new Promise((resolve, reject) => {
-    setTimeout(async () => {
-      try {
-        resolve(await fn())
-      } catch (err) {
-        reject(err)
-      }
-    }, Math.floor(Math.random() * 1000))
-  })
-}
-
-exports.backOff = async (fn, errMsg) => {
-  let attempts = 5,
-    success = false,
-    response,
-    first = true
-  for (; attempts > 0; attempts--) {
-    try {
-      if (first) {
-        response = await fn()
-      } else {
-        response = await exports.randomDelay(fn)
-      }
-      success = true
-      break
-    } catch (err) {
-      // ignore error here
-    }
-  }
-  if (!success) {
-    logAlert("Failed to backoff - ", errMsg)
-  }
-  return response
-}
-
 exports.removeFromArray = (array, element) => {
   const index = array.indexOf(element)
   if (index !== -1) {

packages/types/src/documents/app/automation.ts

@@ -6,6 +6,7 @@ export interface Automation extends Document {
     trigger: AutomationTrigger
   }
   appId: string
+  name: string
 }
 
 export interface AutomationStep {
@@ -40,7 +41,8 @@ export interface AutomationResults {
 }
 
 export interface AutomationLog extends AutomationResults, Document {
-  _rev: string
+  automationName: string
+  _rev?: string
 }
 
 export interface AutomationLogPage {

scripts/link-dependencies.sh

@@ -29,6 +29,9 @@ if [ -d "../budibase-pro" ]; then
   echo "Linking backend-core to pro"
   yarn link '@budibase/backend-core'
 
+  echo "Linking types to pro"
+  yarn link '@budibase/types'
+
   cd ../../../budibase
 
   echo "Linking pro to worker"