From da2303803c44b972d0eb1f5c58745a96223e3d8c Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 27 Jun 2022 18:00:48 +0100 Subject: [PATCH] Moving get of logs to pro. --- packages/backend-core/src/db/constants.js | 20 +++ packages/backend-core/src/db/utils.ts | 17 +- .../server/src/api/controllers/automation.js | 13 +- .../server/src/automations/logging/index.ts | 160 +++--------------- packages/server/src/db/utils.js | 19 +-- packages/server/src/db/views/staticViews.js | 30 ---- packages/server/yarn.lock | 26 +-- packages/worker/yarn.lock | 24 ++- 8 files changed, 94 insertions(+), 215 deletions(-) diff --git a/packages/backend-core/src/db/constants.js b/packages/backend-core/src/db/constants.js index 10c6e174d7..8bdf0f56de 100644 --- a/packages/backend-core/src/db/constants.js +++ b/packages/backend-core/src/db/constants.js @@ -1,8 +1,27 @@ exports.SEPARATOR = "_" +exports.UNICODE_MAX = "\ufff0" const PRE_APP = "app" const PRE_DEV = "dev" +/** + * Can be used to create a few different forms of querying a view. + */ +exports.ViewModes = { + ALL: "all", + AUTOMATION: "auto", + STATUS: "status", +} + +exports.ViewNames = { + USER_BY_EMAIL: "by_email", + BY_API_KEY: "by_api_key", + USER_BY_BUILDERS: "by_builders", + LINK: "by_link", + ROUTING: "screen_routes", + AUTO_LOGS: "auto_log", +} + exports.DocumentTypes = { USER: "us", WORKSPACE: "workspace", @@ -15,6 +34,7 @@ exports.DocumentTypes = { ROLE: "role", MIGRATIONS: "migrations", DEV_INFO: "devinfo", + AUTOMATION_LOG: "log_au", } exports.StaticDatabases = { diff --git a/packages/backend-core/src/db/utils.ts b/packages/backend-core/src/db/utils.ts index dc7a0454c3..125d7a81da 100644 --- a/packages/backend-core/src/db/utils.ts +++ b/packages/backend-core/src/db/utils.ts @@ -1,7 +1,7 @@ import { newid } from "../hashing" import { DEFAULT_TENANT_ID, Configs } from "../constants" import env from "../environment" -import { SEPARATOR, DocumentTypes } from "./constants" +import { SEPARATOR, DocumentTypes, UNICODE_MAX } from "./constants" import { getTenantId, getGlobalDBName, getGlobalDB } from "../tenancy" import fetch from "node-fetch" import { doWithDB, allDbs } from "./index" @@ -12,14 +12,6 @@ import { isDevApp, isDevAppID } from "./conversions" import { APP_PREFIX } from "./constants" import * as events from "../events" -const UNICODE_MAX = "\ufff0" - -export const ViewNames = { - USER_BY_EMAIL: "by_email", - BY_API_KEY: "by_api_key", - USER_BY_BUILDERS: "by_builders", -} - export * from "./constants" export * from "./conversions" export { default as Replication } from "./Replication" @@ -63,6 +55,13 @@ export function getDocParams( } } +/** + * Retrieve the correct index for a view based on default design DB. + */ +export function getQueryIndex(viewName: string) { + return `database/${viewName}` +} + /** * Generates a new workspace ID. * @returns {string} The new workspace ID which the workspace doc can be stored under. diff --git a/packages/server/src/api/controllers/automation.js b/packages/server/src/api/controllers/automation.js index b0d33d2352..4b273fe49d 100644 --- a/packages/server/src/api/controllers/automation.js +++ b/packages/server/src/api/controllers/automation.js @@ -1,6 +1,5 @@ const actions = require("../../automations/actions") const triggers = require("../../automations/triggers") -const { getLogs, oneMonthAgo } = require("../../automations/logging") const { getAutomationParams, generateAutomationID, @@ -21,6 +20,7 @@ const { } = require("@budibase/backend-core/context") const { events } = require("@budibase/backend-core") const { app } = require("@budibase/backend-core/cache") +const { logs } = require("@budibase/pro") const ACTION_DEFS = removeDeprecated(actions.ACTION_DEFINITIONS) const TRIGGER_DEFS = removeDeprecated(triggers.TRIGGER_DEFINITIONS) @@ -195,11 +195,12 @@ exports.destroy = async function (ctx) { exports.logSearch = async function (ctx) { let { automationId, status, page, startDate } = ctx.request.body - // TODO: need to check maximum allowed in license - if (!startDate) { - startDate = oneMonthAgo() - } - ctx.body = await getLogs(startDate, status, automationId, page) + ctx.body = await logs.automations.getLogs( + startDate, + status, + automationId, + page + ) } exports.clearLogError = async function (ctx) { diff --git a/packages/server/src/automations/logging/index.ts b/packages/server/src/automations/logging/index.ts index a61bf31eb3..3372726918 100644 --- a/packages/server/src/automations/logging/index.ts +++ b/packages/server/src/automations/logging/index.ts @@ -1,28 +1,20 @@ -import { - AutomationLogPage, - AutomationResults, - AutomationStatus, -} from "../../definitions/automation" import { getAppId, getProdAppDB } from "@budibase/backend-core/context" import { DocumentTypes, generateAutomationLogID, - getAutomationLogParams, - getQueryIndex, - ViewNames, SEPARATOR, isProdAppID, } from "../../db/utils" -import { createLogByAutomationView } from "../../db/views/staticViews" import { Automation, MetadataErrors } from "../../definitions/common" import { app } from "@budibase/backend-core/cache" import { backOff } from "../../utilities" import * as env from "../../environment" - -const PAGE_SIZE = 9 -const EARLIEST_DATE = new Date(0).toISOString() -const FREE_EXPIRY_SEC = 86400 -const PRO_EXPIRY_SEC = FREE_EXPIRY_SEC * 30 +import { logs } from "@budibase/pro" +import { + AutomationResults, + AutomationStatus, +} from "../../definitions/automation" +const { logAlert } = require("@budibase/backend-core/logging") function getStatus(results: AutomationResults) { let status = AutomationStatus.SUCCESS @@ -40,108 +32,29 @@ function getStatus(results: AutomationResults) { return status } -export function oneMonthAgo() { - return new Date(new Date().getTime() - PRO_EXPIRY_SEC * 1000).toISOString() -} - -export function oneDayAgo() { - return new Date(new Date().getTime() - FREE_EXPIRY_SEC * 1000).toISOString() -} - async function clearOldHistory() { const db = getProdAppDB() - // TODO: handle license lookup for deletion - const expiredEnd = oneDayAgo() - const results = await getAllLogs(EARLIEST_DATE, expiredEnd, { - docs: false, - paginate: false, - }) - const toDelete = results.data.map((doc: any) => ({ - _id: doc.id, - _rev: doc.value.rev, - _deleted: true, - })) - const errorLogIds = results.data - .filter((doc: any) => { - const parts = doc.id.split(SEPARATOR) - const status = parts[parts.length - 1] - return status === AutomationStatus.ERROR - }) - .map((doc: any) => doc.id) - await db.bulkDocs(toDelete) - if (errorLogIds.length) { - await updateAppMetadataWithErrors(errorLogIds, { clearing: true }) - } -} - -function pagination( - response: any, - paginate: boolean = true -): AutomationLogPage { - const data = response.rows.map((row: any) => { - return row.doc ? row.doc : row - }) - if (!paginate) { - return { data, hasNextPage: false } - } - const hasNextPage = data.length > PAGE_SIZE - return { - data: data.slice(0, PAGE_SIZE), - hasNextPage, - nextPage: hasNextPage ? data[PAGE_SIZE]?._id : undefined, - } -} - -async function getAllLogs( - startDate: string, - endDate: string, - opts: { - docs: boolean - status?: string - paginate?: boolean - page?: string - } = { docs: true } -): Promise { - const db = getProdAppDB() - let optional: any = { status: opts.status } - const params = getAutomationLogParams(startDate, endDate, optional, { - include_docs: opts.docs, - limit: opts?.paginate ? PAGE_SIZE + 1 : undefined, - }) - if (opts?.page) { - params.startkey = opts.page - } - let response = await db.allDocs(params) - return pagination(response, opts?.paginate) -} - -async function getLogsByView( - startDate: string, - endDate: string, - viewParams: { automationId?: string; status?: string; page?: string } = {} -): Promise { - const db = getProdAppDB() - let response try { - let optional = { - automationId: viewParams?.automationId, - status: viewParams?.status, - } - const params = getAutomationLogParams(startDate, endDate, optional, { - include_docs: true, - limit: PAGE_SIZE, - }) - if (viewParams?.page) { - params.startkey = viewParams.page - } - response = await db.query(getQueryIndex(ViewNames.AUTO_LOGS), params) - } catch (err: any) { - if (err != null && err.name === "not_found") { - await createLogByAutomationView() - return getLogsByView(startDate, endDate, viewParams) + const expired = await logs.automations.getExpiredLogs() + const toDelete = expired.data.map((doc: any) => ({ + _id: doc.id, + _rev: doc.value.rev, + _deleted: true, + })) + const errorLogIds = expired.data + .filter((doc: any) => { + const parts = doc.id.split(SEPARATOR) + const status = parts[parts.length - 1] + return status === AutomationStatus.ERROR + }) + .map((doc: any) => doc.id) + await db.bulkDocs(toDelete) + if (errorLogIds.length) { + await updateAppMetadataWithErrors(errorLogIds, { clearing: true }) } + } catch (err) { + logAlert(`Failed to cleanup automation log history - Database "${db.name}"`) } - return pagination(response) } async function updateAppMetadataWithErrors( @@ -214,28 +127,3 @@ export async function storeLog( // clear up old logging for app await clearOldHistory() } - -export async function getLogs( - startDate: string, - status?: string, - automationId?: string, - page?: string -): Promise { - let response: AutomationLogPage - let endDate = new Date().toISOString() - if (automationId || status) { - response = await getLogsByView(startDate, endDate, { - automationId, - status, - page, - }) - } else { - response = await getAllLogs(startDate, endDate, { - status, - page, - docs: true, - paginate: true, - }) - } - return response -} diff --git a/packages/server/src/db/utils.js b/packages/server/src/db/utils.js index 10c1c98f38..154f765315 100644 --- a/packages/server/src/db/utils.js +++ b/packages/server/src/db/utils.js @@ -11,6 +11,8 @@ const { isProdAppID, getDevelopmentAppID, generateAppID, + getQueryIndex, + ViewNames, } = require("@budibase/backend-core/db") const UNICODE_MAX = "\ufff0" @@ -22,16 +24,11 @@ const AppStatus = { } const DocumentTypes = { - APP: CoreDocTypes.APP, - DEV: CoreDocTypes.DEV, - APP_DEV: CoreDocTypes.APP_DEV, - APP_METADATA: CoreDocTypes.APP_METADATA, - ROLE: CoreDocTypes.ROLE, + ...CoreDocTypes, TABLE: "ta", ROW: "ro", USER: "us", AUTOMATION: "au", - AUTOMATION_LOG: "log_au", LINK: "li", WEBHOOK: "wh", INSTANCE: "inst", @@ -46,12 +43,6 @@ const DocumentTypes = { USER_FLAG: "flag", } -const ViewNames = { - LINK: "by_link", - ROUTING: "screen_routes", - AUTO_LOGS: "auto_log", -} - const ViewModes = { ALL: "all", AUTOMATION: "auto", @@ -98,9 +89,7 @@ exports.generateDevAppID = getDevelopmentAppID exports.generateRoleID = generateRoleID exports.getRoleParams = getRoleParams -exports.getQueryIndex = viewName => { - return `database/${viewName}` -} +exports.getQueryIndex = getQueryIndex /** * If creating DB allDocs/query params with only a single top level ID this can be used, this diff --git a/packages/server/src/db/views/staticViews.js b/packages/server/src/db/views/staticViews.js index 5f9c65ce01..5cfae746df 100644 --- a/packages/server/src/db/views/staticViews.js +++ b/packages/server/src/db/views/staticViews.js @@ -3,11 +3,9 @@ const { DocumentTypes, SEPARATOR, ViewNames, - ViewModes, SearchIndexes, } = require("../utils") const SCREEN_PREFIX = DocumentTypes.SCREEN + SEPARATOR -const LOG_PREFIX = DocumentTypes.AUTOMATION_LOG + SEPARATOR /************************************************** * INFORMATION * @@ -60,34 +58,6 @@ exports.createLinkView = async () => { await db.put(designDoc) } -/** - * A separate view that allows us to perform queries by the automation ID and time series, while the - * main all_docs allows access to time series only - */ -exports.createLogByAutomationView = async () => { - const db = getAppDB() - const designDoc = await db.get("_design/database") - const view = { - map: `function(doc) { - if (doc._id.startsWith("${LOG_PREFIX}")) { - let autoId = doc.automationId + "${SEPARATOR}" - let status = doc.status + "${SEPARATOR}" - let autoKey = "${ViewModes.AUTOMATION}${SEPARATOR}" + autoId + doc.createdAt - let statusKey = "${ViewModes.STATUS}${SEPARATOR}" + status + doc.createdAt - let allKey = "${ViewModes.ALL}${SEPARATOR}" + status + autoId + doc.createdAt - emit(statusKey) - emit(autoKey) - emit(allKey) - } - }`, - } - designDoc.views = { - ...designDoc.views, - [ViewNames.AUTO_LOGS]: view, - } - await db.put(designDoc) -} - exports.createRoutingView = async () => { const db = getAppDB() const designDoc = await db.get("_design/database") diff --git a/packages/server/yarn.lock b/packages/server/yarn.lock index cccf69abc0..6de05a1d67 100644 --- a/packages/server/yarn.lock +++ b/packages/server/yarn.lock @@ -1080,11 +1080,12 @@ resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw== -"@budibase/backend-core@1.0.211": - version "1.0.211" - resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.211.tgz#579bcc97acf1df2510e302bb70b43245e5ab0e37" - integrity sha512-ham+Qk6WXQi37Lgnz1Gh/+ItcvCo+dnPZDcVvPESrVxqQkVVIejlxQinZSueIrsgOYHE1j3TUH5zER5M2t6RWw== +"@budibase/backend-core@1.0.212-alpha.0": + version "1.0.212-alpha.0" + resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.212-alpha.0.tgz#6ac3448c8272e918f1af1fff0cf8c5773ae61219" + integrity sha512-hFvbQQEbF3w2u9fe/S+RhNw5HUETS6rhu9q5KDTDQ57k05D4YMPcpMBGSh7SPMqmVyEwUDgcL36mkFOc3AgjYQ== dependencies: + "@budibase/types" "^1.0.212-alpha.0" "@techpass/passport-openidconnect" "0.3.2" aws-sdk "2.1030.0" bcrypt "5.0.1" @@ -1160,12 +1161,12 @@ svelte-flatpickr "^3.2.3" svelte-portal "^1.0.0" -"@budibase/pro@1.0.211": - version "1.0.211" - resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.211.tgz#d362e9af8c15f6ed386f27b7cca95cc096a91344" - integrity sha512-dfFByJhlTIURT3sXei5mVXq5rczFMM/ij2Scze0uqPZNpmIlWVqiYdGX7/HEcmIFSS0+UfcdBxjCJJlTGiK4/w== +"@budibase/pro@1.0.212-alpha.0": + version "1.0.212-alpha.0" + resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.212-alpha.0.tgz#843f8291fcc0a2fbcb4d857a750bc116cdaee293" + integrity sha512-4nhWxjMcxSQBPXRy/U+37IaVLYOr4/RVe79/fUvnXrr5qAeecbEk/QbkJJd3dU1WaNxB2eGhNtH3uBUPQvcT9A== dependencies: - "@budibase/backend-core" "1.0.211" + "@budibase/backend-core" "1.0.212-alpha.0" node-fetch "^2.6.1" "@budibase/standard-components@^0.9.139": @@ -1186,6 +1187,11 @@ svelte-apexcharts "^1.0.2" svelte-flatpickr "^3.1.0" +"@budibase/types@^1.0.212-alpha.0": + version "1.0.212" + resolved "https://registry.yarnpkg.com/@budibase/types/-/types-1.0.212.tgz#e66a15b711544b4fab7767261fd5f2f1dd7f40d7" + integrity sha512-DhGyw6snwJQZQlx7havVYnqPZfZERueKZfmVCBySzwInZZt0+sXZaBl1BVjGjYuwpaUQBMDBf7geBgHXp6DIKg== + "@bull-board/api@3.7.0": version "3.7.0" resolved "https://registry.yarnpkg.com/@bull-board/api/-/api-3.7.0.tgz#231f687187c0cb34e0b97f463917b6aaeb4ef6af" @@ -8832,7 +8838,7 @@ keyv@^3.0.0: kind-of@^3.0.2, kind-of@^3.0.3, kind-of@^3.2.0: version "3.2.2" resolved "https://registry.yarnpkg.com/kind-of/-/kind-of-3.2.2.tgz#31ea21a734bab9bbb0f32466d893aea51e4a3c64" - integrity sha1-MeohpzS6ubuw8yRm2JOupR5KPGQ= + integrity sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ== dependencies: is-buffer "^1.1.5" diff --git a/packages/worker/yarn.lock b/packages/worker/yarn.lock index 75707bfdb3..bda94a6b27 100644 --- a/packages/worker/yarn.lock +++ b/packages/worker/yarn.lock @@ -293,11 +293,12 @@ resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw== -"@budibase/backend-core@1.0.207-alpha.3": - version "1.0.207-alpha.3" - resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.207-alpha.3.tgz#98bced0575ec4e2b158239a8c73b39ca2d816719" - integrity sha512-DU4X6jJ+DfhzOv4TTa1w4Dk5ZEdlK/z1joCTruT+SGM5qI75bXrGeol5OX2OaEbNKtXFKJ1zeVTmBCYcu7OFUg== +"@budibase/backend-core@1.0.212-alpha.0": + version "1.0.212-alpha.0" + resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.212-alpha.0.tgz#6ac3448c8272e918f1af1fff0cf8c5773ae61219" + integrity sha512-hFvbQQEbF3w2u9fe/S+RhNw5HUETS6rhu9q5KDTDQ57k05D4YMPcpMBGSh7SPMqmVyEwUDgcL36mkFOc3AgjYQ== dependencies: + "@budibase/types" "^1.0.212-alpha.0" "@techpass/passport-openidconnect" "0.3.2" aws-sdk "2.1030.0" bcrypt "5.0.1" @@ -324,14 +325,19 @@ uuid "8.3.2" zlib "1.0.5" -"@budibase/pro@1.0.207-alpha.3": - version "1.0.207-alpha.3" - resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.207-alpha.3.tgz#9bde845ceb685f1b43286a124620c21fdf891a01" - integrity sha512-WFEMujpKTVAMvAgLBnMdw8ou9PxsbM4Oa9Dq+DAUsWpPACsMWOProyHLsdRxJyvHlgGfwVjo5MEusvStjI4j6g== +"@budibase/pro@1.0.212-alpha.0": + version "1.0.212-alpha.0" + resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.212-alpha.0.tgz#843f8291fcc0a2fbcb4d857a750bc116cdaee293" + integrity sha512-4nhWxjMcxSQBPXRy/U+37IaVLYOr4/RVe79/fUvnXrr5qAeecbEk/QbkJJd3dU1WaNxB2eGhNtH3uBUPQvcT9A== dependencies: - "@budibase/backend-core" "1.0.207-alpha.3" + "@budibase/backend-core" "1.0.212-alpha.0" node-fetch "^2.6.1" +"@budibase/types@^1.0.212-alpha.0": + version "1.0.212" + resolved "https://registry.yarnpkg.com/@budibase/types/-/types-1.0.212.tgz#e66a15b711544b4fab7767261fd5f2f1dd7f40d7" + integrity sha512-DhGyw6snwJQZQlx7havVYnqPZfZERueKZfmVCBySzwInZZt0+sXZaBl1BVjGjYuwpaUQBMDBf7geBgHXp6DIKg== + "@cspotcode/source-map-consumer@0.8.0": version "0.8.0" resolved "https://registry.yarnpkg.com/@cspotcode/source-map-consumer/-/source-map-consumer-0.8.0.tgz#33bf4b7b39c178821606f669bbc447a6a629786b"