MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Moving majority of automation logging functionality to pro.

This commit is contained in:
mike12345567 2022-07-04 15:44:47 +01:00
parent a6ce6ef924
commit d390bb9c20
9 changed files with 36 additions and 175 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
packages/backend-core/src/index.ts
View File

@ -13,6 +13,7 @@ import deprovisioning from "./context/deprovision"
import auth from "./auth"
import constants from "./constants"
import * as dbConstants from "./db/constants"
import logging from "./logging"
// mimic the outer package exports
import * as db from "./pkg/db"
@ -49,6 +50,7 @@ const core = {
deprovisioning,
installation,
errors,
logging,
...errorClasses,
}

2
packages/builder/src/components/portal/overview/automation/HistoryDetailsPanel.svelte
View File

@ -1,6 +1,6 @@
<script>
import { Layout, Icon, ActionButton } from "@budibase/bbui"
import StatusRenderer from "components/portal/overview/StatusRenderer.svelte"
import StatusRenderer from "./StatusRenderer.svelte"
import DateTimeRenderer from "components/common/renderers/DateTimeRenderer.svelte"
import TestDisplay from "components/automation/AutomationBuilder/TestDisplay.svelte"
import { goto } from "@roxi/routify"

3
packages/server/src/api/controllers/application.ts
View File

@ -45,6 +45,7 @@ const { getTenantId, isMultiTenant } = require("@budibase/backend-core/tenancy")
import { syncGlobalUsers } from "./user"
const { app: appCache } = require("@budibase/backend-core/cache")
import { cleanupAutomations } from "../../automations/utils"
import { checkAppMetadata } from "../../automations/logging"
const {
getAppDB,
getProdAppDB,
@ -192,7 +193,7 @@ export const fetch = async (ctx: any) => {
}
}
ctx.body = apps
ctx.body = await checkAppMetadata(apps)
}
export const fetchAppDefinition = async (ctx: any) => {

11
packages/server/src/api/controllers/automation.js
View File

@ -21,7 +21,6 @@ const {
const { events } = require("@budibase/backend-core")
const { app } = require("@budibase/backend-core/cache")
const { automations } = require("@budibase/pro")
const { clearOldHistory } = require("../../automations/logging")
const ACTION_DEFS = removeDeprecated(actions.ACTION_DEFINITIONS)
const TRIGGER_DEFS = removeDeprecated(triggers.TRIGGER_DEFINITIONS)
@ -195,15 +194,7 @@ exports.destroy = async function (ctx) {
}
exports.logSearch = async function (ctx) {
let { automationId, status, page, startDate } = ctx.request.body
// before querying logs, make sure old logs are cleared out
await clearOldHistory()
ctx.body = await automations.logs.getLogs(
startDate,
status,
automationId,
page
)
ctx.body = await automations.logs.logSearch(ctx.request.body)
}
exports.clearLogError = async function (ctx) {

145
packages/server/src/automations/logging/index.ts
View File

@ -1,132 +1,35 @@
import { getAppId, getProdAppDB } from "@budibase/backend-core/context"
import {
DocumentTypes,
generateAutomationLogID,
isProdAppID,
SEPARATOR,
} from "../../db/utils"
import { Automation } from "../../definitions/common"
import { app } from "@budibase/backend-core/cache"
import { backOff } from "../../utilities"
import * as env from "../../environment"
import { AutomationResults, Automation, App } from "@budibase/types"
import { automations } from "@budibase/pro"
import { AppMetadataErrors } from "@budibase/types"
import { AutomationResults, AutomationStatus } from "@budibase/types"
const { logAlert } = require("@budibase/backend-core/logging")
function getStatus(results: AutomationResults) {
let status = AutomationStatus.SUCCESS
let first = true
for (let step of results.steps) {
// skip the trigger, its always successful if automation ran
if (first) {
first = false
continue
}
if (!step.outputs?.success) {
status = AutomationStatus.ERROR
break
} else if (step.outputs?.status?.toLowerCase() === "stopped") {
status = AutomationStatus.STOPPED
break
}
}
return status
}
export async function clearOldHistory() {
const db = getProdAppDB()
try {
const expired = await automations.logs.getExpiredLogs()
const toDelete = expired.data.map((doc: any) => ({
_id: doc.id,
_rev: doc.value.rev,
_deleted: true,
}))
const errorLogIds = expired.data
.filter((doc: any) => {
const parts = doc.id.split(SEPARATOR)
const status = parts[parts.length - 1]
return status === AutomationStatus.ERROR
})
.map((doc: any) => doc.id)
await db.bulkDocs(toDelete)
if (errorLogIds.length) {
await updateAppMetadataWithErrors(errorLogIds, { clearing: true })
}
} catch (err) {
logAlert(`Failed to cleanup automation log history - Database "${db.name}"`)
}
}
async function updateAppMetadataWithErrors(
logIds: string[],
{ clearing } = { clearing: false }
) {
const db = getProdAppDB()
// this will try multiple times with a delay between to update the metadata
await backOff(async () => {
const metadata = await db.get(DocumentTypes.APP_METADATA)
for (let logId of logIds) {
const parts = logId.split(SEPARATOR)
const autoId = `${parts[parts.length - 3]}${SEPARATOR}${
parts[parts.length - 2]
}`
let errors: AppMetadataErrors = {}
if (metadata.automationErrors) {
errors = metadata.automationErrors as AppMetadataErrors
}
if (!Array.isArray(errors[autoId])) {
errors[autoId] = []
}
const idx = errors[autoId].indexOf(logId)
if (clearing && idx !== -1) {
errors[autoId].splice(idx, 1)
} else {
errors[autoId].push(logId)
}
// if clearing and reach zero, this will pass and will remove the element
if (errors[autoId].length === 0) {
delete errors[autoId]
}
metadata.automationErrors = errors
}
await db.put(metadata)
// don't update cache until after DB put, make sure it has been stored successfully
await app.invalidateAppMetadata(metadata.appId, metadata)
}, "Failed to update app metadata with automation log error")
}
import { db as dbUtils } from "@budibase/backend-core"
export async function storeLog(
automation: Automation,
results: AutomationResults
) {
// can disable this if un-needed in self-host, also only do this for prod apps
if (env.DISABLE_AUTOMATION_LOGS || !isProdAppID(getAppId())) {
if (env.DISABLE_AUTOMATION_LOGS) {
return
}
const db = getProdAppDB()
const automationId = automation._id
const name = automation.name
const status = getStatus(results)
const isoDate = new Date().toISOString()
const id = generateAutomationLogID(isoDate, status, automationId)
await db.put({
// results contain automationId and status for view
...results,
automationId,
status,
automationName: name,
createdAt: isoDate,
_id: id,
})
// need to note on the app metadata that there is an error, store what the error is
if (status === AutomationStatus.ERROR) {
await updateAppMetadataWithErrors([id])
}
// clear up old logging for app
await clearOldHistory()
await automations.logs.storeLog(automation, results)
}
export async function checkAppMetadata(apps: App[]) {
const maxStartDate = await automations.logs.oldestLogDate()
for (let metadata of apps) {
if (!metadata.automationErrors) {
continue
}
for (let [key, errors] of Object.entries(metadata.automationErrors)) {
const updated = []
for (let error of errors) {
const startDate = error.split(dbUtils.SEPARATOR)[2]
if (startDate > maxStartDate) {
updated.push(error)
}
}
metadata.automationErrors[key] = updated
}
}
return apps
}

4
packages/server/src/db/utils.js
View File

@ -354,10 +354,6 @@ exports.getMemoryViewParams = (otherProps = {}) => {
return getDocParams(DocumentTypes.MEM_VIEW, null, otherProps)
}
exports.generateAutomationLogID = (isoDate, status, automationId) => {
return `${DocumentTypes.AUTOMATION_LOG}${SEPARATOR}${isoDate}${SEPARATOR}${automationId}${SEPARATOR}${status}`
}
/**
* This can be used with the db.allDocs to get a list of IDs
*/

37
packages/server/src/utilities/index.js
View File

@ -4,7 +4,6 @@ const { sanitizeKey } = require("@budibase/backend-core/objectStore")
const { generateMetadataID } = require("../db/utils")
const Readable = require("stream").Readable
const { getAppDB } = require("@budibase/backend-core/context")
const { logAlert } = require("@budibase/backend-core/logging")
const BB_CDN = "https://cdn.budi.live"
@ -14,42 +13,6 @@ exports.isDev = env.isDev
exports.NUMBER_REGEX = /^[+-]?([0-9]*[.])?[0-9]+$/g
exports.randomDelay = fn => {
return new Promise((resolve, reject) => {
setTimeout(async () => {
try {
resolve(await fn())
} catch (err) {
reject(err)
}
}, Math.floor(Math.random() * 1000))
})
}
exports.backOff = async (fn, errMsg) => {
let attempts = 5,
success = false,
response,
first = true
for (; attempts > 0; attempts--) {
try {
if (first) {
response = await fn()
} else {
response = await exports.randomDelay(fn)
}
success = true
break
} catch (err) {
// ignore error here
}
}
if (!success) {
logAlert("Failed to backoff - ", errMsg)
}
return response
}
exports.removeFromArray = (array, element) => {
const index = array.indexOf(element)
if (index !== -1) {

4
packages/types/src/documents/app/automation.ts
View File

@ -6,6 +6,7 @@ export interface Automation extends Document {
trigger: AutomationTrigger
}
appId: string
name: string
}
export interface AutomationStep {
@ -40,7 +41,8 @@ export interface AutomationResults {
}
export interface AutomationLog extends AutomationResults, Document {
_rev: string
automationName: string
_rev?: string
}
export interface AutomationLogPage {

3
scripts/link-dependencies.sh
View File

@ -29,6 +29,9 @@ if [ -d "../budibase-pro" ]; then
echo "Linking backend-core to pro"
yarn link '@budibase/backend-core'
echo "Linking types to pro"
yarn link '@budibase/types'
cd ../../../budibase
echo "Linking pro to worker"