From 5914b0c560c55f99486827bea56f136a0e3070fb Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 1 Jun 2022 16:01:06 +0100 Subject: [PATCH] Adding real history to the history tab, getting storage to couch and retrieval + api working. --- .../builderStore/store/automation/index.js | 7 + .../FlowChart/FlowChart.svelte | 2 +- .../FlowChart/TestDataModal.svelte | 2 +- .../AutomationBuilder/TestDisplay.svelte | 2 +- .../AutomationBuilder/TestPanel.svelte | 2 +- .../portal/overview/HistoryTab.svelte | 61 +------- .../portal/overview/StatusRenderer.svelte | 2 +- .../app/[application]/automate/_layout.svelte | 2 +- packages/frontend-core/src/api/automations.js | 15 ++ .../server/src/api/controllers/automation.js | 9 ++ packages/server/src/api/routes/automation.js | 5 + .../server/src/automations/history/index.ts | 143 +++++++++++++----- packages/server/src/db/utils.js | 6 +- packages/server/src/definitions/automation.ts | 12 ++ packages/server/src/threads/automation.js | 3 + 15 files changed, 175 insertions(+), 98 deletions(-) diff --git a/packages/builder/src/builderStore/store/automation/index.js b/packages/builder/src/builderStore/store/automation/index.js index 5a2019b5c1..73e3d83832 100644 --- a/packages/builder/src/builderStore/store/automation/index.js +++ b/packages/builder/src/builderStore/store/automation/index.js @@ -6,6 +6,7 @@ import analytics, { Events } from "analytics" const initialAutomationState = { automations: [], + showTestPanel: false, blockDefinitions: { TRIGGER: [], ACTION: [], @@ -121,6 +122,12 @@ const automationActions = store => ({ return state }) }, + getLogs: async (automationId, startDate) => { + return await API.getAutomationLogs({ + automationId, + startDate, + }) + }, addTestDataToAutomation: data => { store.update(state => { state.selectedAutomation.addTestData(data) diff --git a/packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowChart.svelte b/packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowChart.svelte index 3e58b25ff6..9c987c89d8 100644 --- a/packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowChart.svelte +++ b/packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowChart.svelte @@ -65,7 +65,7 @@ { - $automationStore.selectedAutomation.automation.showTestPanel = true + $automationStore.showTestPanel = true }} size="M">Test Details diff --git a/packages/builder/src/components/automation/AutomationBuilder/FlowChart/TestDataModal.svelte b/packages/builder/src/components/automation/AutomationBuilder/FlowChart/TestDataModal.svelte index fecd0fcc7e..b86cffb1f9 100644 --- a/packages/builder/src/components/automation/AutomationBuilder/FlowChart/TestDataModal.svelte +++ b/packages/builder/src/components/automation/AutomationBuilder/FlowChart/TestDataModal.svelte @@ -51,7 +51,7 @@ $automationStore.selectedAutomation?.automation, testData ) - $automationStore.selectedAutomation.automation.showTestPanel = true + $automationStore.showTestPanel = true } catch (error) { notifications.error("Error testing notification") } diff --git a/packages/builder/src/components/automation/AutomationBuilder/TestDisplay.svelte b/packages/builder/src/components/automation/AutomationBuilder/TestDisplay.svelte index e8bcc2e349..6d4c83de11 100644 --- a/packages/builder/src/components/automation/AutomationBuilder/TestDisplay.svelte +++ b/packages/builder/src/components/automation/AutomationBuilder/TestDisplay.svelte @@ -10,7 +10,7 @@ let blocks function prepTestResults(results) { - return results.steps.filter(x => x.stepId !== "LOOP" || []) + return results?.steps.filter(x => x.stepId !== "LOOP" || []) } function textArea(results, message) { diff --git a/packages/builder/src/components/automation/AutomationBuilder/TestPanel.svelte b/packages/builder/src/components/automation/AutomationBuilder/TestPanel.svelte index 2fb4994a76..9e84350cd0 100644 --- a/packages/builder/src/components/automation/AutomationBuilder/TestPanel.svelte +++ b/packages/builder/src/components/automation/AutomationBuilder/TestPanel.svelte @@ -36,7 +36,7 @@
{ - $automationStore.selectedAutomation.automation.showTestPanel = false + $automationStore.showTestPanel = false }} hoverable name="Close" diff --git a/packages/builder/src/components/portal/overview/HistoryTab.svelte b/packages/builder/src/components/portal/overview/HistoryTab.svelte index f38becece8..7925419091 100644 --- a/packages/builder/src/components/portal/overview/HistoryTab.svelte +++ b/packages/builder/src/components/portal/overview/HistoryTab.svelte @@ -14,12 +14,12 @@ const runHistorySchema = { status: { displayName: "Status" }, - timestamp: { displayName: "Time" }, - name: { displayName: "Automation" }, + createdAt: { displayName: "Time" }, + automationName: { displayName: "Automation" }, } const customRenderers = [ - { column: "time", component: DateTimeRenderer }, + { column: "createdAt", component: DateTimeRenderer }, { column: "status", component: StatusRenderer }, ] @@ -54,57 +54,10 @@ onMount(async () => { let definitions = await automationStore.actions.definitions() - runHistory = enrichHistory(definitions, [ - { - status: "Success", - timestamp: "2022-05-11T16:06:14.438Z", - name: "automation name", - steps: [ - { - stepId: "ROW_SAVED", - inputs: null, - outputs: { - id: "awd", - revision: "awd", - row: { - tableId: "ta_240cfde36405479fa814b8a2c46655b5", - name: "", - suppliers: [], - "supplier name": "", - _id: "awd", - _rev: "awd", - }, - }, - }, - { - stepId: "SERVER_LOG", - inputs: { - text: "awdawdawd", - }, - outputs: { - success: true, - }, - }, - ], - }, - { - status: "Error", - timestamp: "2022-05-11T16:03:14.438Z", - name: "automation name", - steps: [ - { - stepId: "ROW_SAVED", - inputs: {}, - outputs: {}, - }, - { - stepId: "SEND_EMAIL_SMTP", - inputs: {}, - outputs: {}, - }, - ], - }, - ]) + runHistory = enrichHistory( + definitions, + await automationStore.actions.getLogs() + ) }) diff --git a/packages/builder/src/components/portal/overview/StatusRenderer.svelte b/packages/builder/src/components/portal/overview/StatusRenderer.svelte index 7abfebb2f2..8cbcd6c10e 100644 --- a/packages/builder/src/components/portal/overview/StatusRenderer.svelte +++ b/packages/builder/src/components/portal/overview/StatusRenderer.svelte @@ -2,7 +2,7 @@ import { Icon } from "@budibase/bbui" export let value - $: isError = value === "Error" + $: isError = !value || value.toLowerCase() === "error" $: color = isError ? "var(--spectrum-semantic-negative-color-background)" : "var(--green)" diff --git a/packages/builder/src/pages/builder/app/[application]/automate/_layout.svelte b/packages/builder/src/pages/builder/app/[application]/automate/_layout.svelte index a713067bbe..447441b2fc 100644 --- a/packages/builder/src/pages/builder/app/[application]/automate/_layout.svelte +++ b/packages/builder/src/pages/builder/app/[application]/automate/_layout.svelte @@ -45,7 +45,7 @@ {/if}
- {#if automation?.showTestPanel} + {#if $automationStore.showTestPanel}
diff --git a/packages/frontend-core/src/api/automations.js b/packages/frontend-core/src/api/automations.js index 3b60b6dbac..2072337073 100644 --- a/packages/frontend-core/src/api/automations.js +++ b/packages/frontend-core/src/api/automations.js @@ -73,4 +73,19 @@ export const buildAutomationEndpoints = API => ({ url: `/api/automations/${automationId}/${automationRev}`, }) }, + + /** + * Get the logs for the app, or by automation ID. + * @param automationId The ID of the automation to get logs for. + * @param startDate An ISO date string to state the start of the date range. + */ + getAutomationLogs: async ({ automationId, startDate }) => { + return await API.post({ + url: "/api/automations/logs/search", + body: { + automationId, + startDate, + }, + }) + }, }) diff --git a/packages/server/src/api/controllers/automation.js b/packages/server/src/api/controllers/automation.js index 74942dad40..c3c4f4da98 100644 --- a/packages/server/src/api/controllers/automation.js +++ b/packages/server/src/api/controllers/automation.js @@ -1,5 +1,6 @@ const actions = require("../../automations/actions") const triggers = require("../../automations/triggers") +const { getLogs, oneDayAgo } = require("../../automations/history") const { getAutomationParams, generateAutomationID } = require("../../db/utils") const { checkForWebhooks, @@ -150,6 +151,14 @@ exports.destroy = async function (ctx) { ctx.body = await db.remove(automationId, ctx.params.rev) } +exports.logSearch = async function (ctx) { + const { automationId } = ctx.request.body + // TODO: check if there is a date range in the search params + // also check the date range vs their license, see if it is allowed + const startDate = oneDayAgo() + ctx.body = await getLogs(startDate, automationId) +} + exports.getActionList = async function (ctx) { ctx.body = ACTION_DEFS } diff --git a/packages/server/src/api/routes/automation.js b/packages/server/src/api/routes/automation.js index be1af1f786..d25f7bc328 100644 --- a/packages/server/src/api/routes/automation.js +++ b/packages/server/src/api/routes/automation.js @@ -57,6 +57,11 @@ router authorized(BUILDER), controller.destroy ) + .post( + "/api/automations/logs/search", + authorized(BUILDER), + controller.logSearch + ) .post( "/api/automations/:id/trigger", appInfoMiddleware({ appType: AppType.PROD }), diff --git a/packages/server/src/automations/history/index.ts b/packages/server/src/automations/history/index.ts index b1fd761291..65ebb84ccd 100644 --- a/packages/server/src/automations/history/index.ts +++ b/packages/server/src/automations/history/index.ts @@ -1,24 +1,55 @@ -import { AutomationResults } from "../../definitions/automation" +import { + AutomationLog, + AutomationResults, + AutomationStatus, +} from "../../definitions/automation" import { getAppDB } from "@budibase/backend-core/context" import { generateAutomationLogID, getAutomationLogParams, - // getQueryIndex, - // ViewNames, + getQueryIndex, + ViewNames, } from "../../db/utils" -// import { createLogByAutomationView } from "../../db/views/staticViews" +import { createLogByAutomationView } from "../../db/views/staticViews" +import { Automation } from "../../definitions/common" +const EARLIEST_DATE = new Date(0).toISOString() const FREE_EXPIRY_SEC = 86400 -// const PRO_EXPIRY_SEC = FREE_EXPIRY_SEC * 30 +const PRO_EXPIRY_SEC = FREE_EXPIRY_SEC * 30 + +function getStatus(results: AutomationResults) { + let status = AutomationStatus.SUCCESS + let first = true + for (let step of results.steps) { + // skip the trigger, its always successful if automation ran + if (first) { + first = false + continue + } + if (!step.outputs?.success) { + status = AutomationStatus.ERROR + } + } + return status +} + +// export function oneMonthAgo() { +// return new Date( +// new Date().getTime() - PRO_EXPIRY_SEC * 1000 +// ).toISOString() +// } + +export function oneDayAgo() { + return new Date(new Date().getTime() - FREE_EXPIRY_SEC * 1000).toISOString() +} async function clearOldHistory() { const db = getAppDB() // TODO: handle license lookup for deletion - const time = new Date(new Date().getTime() - FREE_EXPIRY_SEC * 1000) - const queryParams: any = { - endIso: time, - } - const results = await db.allDocs(getAutomationLogParams(queryParams)) + const expiredEnd = oneDayAgo() + const results = await getAllLogs(EARLIEST_DATE, expiredEnd, { + include_docs: false, + }) const toDelete = results.map((doc: any) => ({ _id: doc.id, _rev: doc.rev, @@ -27,36 +58,66 @@ async function clearOldHistory() { await db.bulkDocs(toDelete) } -// async function getByAutomationID(automationId: string) { -// const db = getAppDB() -// try { -// const queryParams: any = { -// automationId, -// } -// return ( -// await db.query( -// getQueryIndex(ViewNames.LOGS_BY_AUTOMATION), -// getAutomationLogParams(queryParams, { include_docs: true }) -// ) -// ).rows.map((row: any) => row.doc) -// } catch (err: any) { -// if (err != null && err.name === "not_found") { -// await createLogByAutomationView() -// return getByAutomationID(automationId) -// } -// } -// } - -export async function storeHistory( - automationId: string, - results: AutomationResults +async function getAllLogs( + startDate: string, + endDate: string, + opts: any = { include_docs: true } ) { + const db = getAppDB() + const queryParams: any = { + endDate, + startDate, + } + let response = (await db.allDocs(getAutomationLogParams(queryParams, opts))) + .rows + if (opts?.include_docs) { + response = response.map((row: any) => row.doc) + } + return response +} + +async function getLogsByAutomationID( + automationId: string, + opts: { startDate?: string; endDate?: string } = {} +): Promise { + const db = getAppDB() + try { + const queryParams = { + startDate: opts?.startDate, + endDate: opts?.startDate, + automationId, + } + return ( + await db.query( + getQueryIndex(ViewNames.LOGS_BY_AUTOMATION), + getAutomationLogParams(queryParams, { include_docs: true }) + ) + ).rows.map((row: any) => row.doc) + } catch (err: any) { + if (err != null && err.name === "not_found") { + await createLogByAutomationView() + return getLogsByAutomationID(automationId, opts) + } + } + return [] +} + +export async function storeLog( + automation: Automation, + results: AutomationResults +) { + const automationId = automation._id + const name = automation.name const db = getAppDB() const isoDate = new Date().toISOString() - const id = generateAutomationLogID(isoDate, automationId) + const id = generateAutomationLogID(automationId, isoDate) + await db.put({ // results contain automationId and status for view ...results, + automationId, + automationName: name, + status: getStatus(results), createdAt: isoDate, _id: id, }) @@ -64,4 +125,16 @@ export async function storeHistory( await clearOldHistory() } -export async function retrieve() {} +export async function getLogs(startDate: string, automationId?: string) { + let logs: AutomationLog[] + let endDate = new Date().toISOString() + if (automationId) { + logs = await getLogsByAutomationID(automationId, { + startDate, + endDate, + }) + } else { + logs = await getAllLogs(startDate, endDate) + } + return logs +} diff --git a/packages/server/src/db/utils.js b/packages/server/src/db/utils.js index 1cf7b54bbc..daf6bac82e 100644 --- a/packages/server/src/db/utils.js +++ b/packages/server/src/db/utils.js @@ -369,12 +369,12 @@ exports.generateAutomationLogID = (automationId, isoDate) => { } exports.getAutomationLogParams = ( - { startIso, endIso, automationId } = {}, + { startDate, endDate, automationId } = {}, otherProps = {} ) => { const base = `${DocumentTypes.AUTOMATION_LOG}${SEPARATOR}` - let start = startIso || "", - end = endIso || "" + let start = startDate || "", + end = endDate || "" // reverse for view if (automationId) { start = `${automationId}${SEPARATOR}${start}` diff --git a/packages/server/src/definitions/automation.ts b/packages/server/src/definitions/automation.ts index f48f3f370a..5ec2415a3f 100644 --- a/packages/server/src/definitions/automation.ts +++ b/packages/server/src/definitions/automation.ts @@ -1,6 +1,12 @@ +export enum AutomationStatus { + SUCCESS = "success", + ERROR = "error", +} + export interface AutomationResults { automationId: string status: string + trigger?: any steps: { stepId: string inputs: { @@ -11,3 +17,9 @@ export interface AutomationResults { } }[] } + +export interface AutomationLog extends AutomationResults { + createdAt: string + _id: string + _rev: string +} diff --git a/packages/server/src/threads/automation.js b/packages/server/src/threads/automation.js index 7c5c96fc39..55f4ffd772 100644 --- a/packages/server/src/threads/automation.js +++ b/packages/server/src/threads/automation.js @@ -9,6 +9,7 @@ const { doInTenant } = require("@budibase/backend-core/tenancy") const { definitions: triggerDefs } = require("../automations/triggerInfo") const { doInAppContext, getAppDB } = require("@budibase/backend-core/context") const { AutomationErrors, LoopStepTypes } = require("../constants") +const { storeLog } = require("../automations/history") const FILTER_STEP_ID = actions.ACTION_DEFINITIONS.FILTER.stepId const LOOP_STEP_ID = actions.ACTION_DEFINITIONS.LOOP.stepId @@ -325,6 +326,8 @@ class Orchestrator { } } + // store the history for the automation run + await storeLog(this._automation, this.executionOutput) return this.executionOutput } }