Moving get of logs to pro.

This commit is contained in:
mike12345567 2022-06-27 18:00:48 +01:00
parent efdbd1dc8f
commit da2303803c
8 changed files with 94 additions and 215 deletions

View File

@ -1,8 +1,27 @@
exports.SEPARATOR = "_" exports.SEPARATOR = "_"
exports.UNICODE_MAX = "\ufff0"
const PRE_APP = "app" const PRE_APP = "app"
const PRE_DEV = "dev" const PRE_DEV = "dev"
/**
* Can be used to create a few different forms of querying a view.
*/
exports.ViewModes = {
ALL: "all",
AUTOMATION: "auto",
STATUS: "status",
}
exports.ViewNames = {
USER_BY_EMAIL: "by_email",
BY_API_KEY: "by_api_key",
USER_BY_BUILDERS: "by_builders",
LINK: "by_link",
ROUTING: "screen_routes",
AUTO_LOGS: "auto_log",
}
exports.DocumentTypes = { exports.DocumentTypes = {
USER: "us", USER: "us",
WORKSPACE: "workspace", WORKSPACE: "workspace",
@ -15,6 +34,7 @@ exports.DocumentTypes = {
ROLE: "role", ROLE: "role",
MIGRATIONS: "migrations", MIGRATIONS: "migrations",
DEV_INFO: "devinfo", DEV_INFO: "devinfo",
AUTOMATION_LOG: "log_au",
} }
exports.StaticDatabases = { exports.StaticDatabases = {

View File

@ -1,7 +1,7 @@
import { newid } from "../hashing" import { newid } from "../hashing"
import { DEFAULT_TENANT_ID, Configs } from "../constants" import { DEFAULT_TENANT_ID, Configs } from "../constants"
import env from "../environment" import env from "../environment"
import { SEPARATOR, DocumentTypes } from "./constants" import { SEPARATOR, DocumentTypes, UNICODE_MAX } from "./constants"
import { getTenantId, getGlobalDBName, getGlobalDB } from "../tenancy" import { getTenantId, getGlobalDBName, getGlobalDB } from "../tenancy"
import fetch from "node-fetch" import fetch from "node-fetch"
import { doWithDB, allDbs } from "./index" import { doWithDB, allDbs } from "./index"
@ -12,14 +12,6 @@ import { isDevApp, isDevAppID } from "./conversions"
import { APP_PREFIX } from "./constants" import { APP_PREFIX } from "./constants"
import * as events from "../events" import * as events from "../events"
const UNICODE_MAX = "\ufff0"
export const ViewNames = {
USER_BY_EMAIL: "by_email",
BY_API_KEY: "by_api_key",
USER_BY_BUILDERS: "by_builders",
}
export * from "./constants" export * from "./constants"
export * from "./conversions" export * from "./conversions"
export { default as Replication } from "./Replication" export { default as Replication } from "./Replication"
@ -63,6 +55,13 @@ export function getDocParams(
} }
} }
/**
* Retrieve the correct index for a view based on default design DB.
*/
export function getQueryIndex(viewName: string) {
return `database/${viewName}`
}
/** /**
* Generates a new workspace ID. * Generates a new workspace ID.
* @returns {string} The new workspace ID which the workspace doc can be stored under. * @returns {string} The new workspace ID which the workspace doc can be stored under.

View File

@ -1,6 +1,5 @@
const actions = require("../../automations/actions") const actions = require("../../automations/actions")
const triggers = require("../../automations/triggers") const triggers = require("../../automations/triggers")
const { getLogs, oneMonthAgo } = require("../../automations/logging")
const { const {
getAutomationParams, getAutomationParams,
generateAutomationID, generateAutomationID,
@ -21,6 +20,7 @@ const {
} = require("@budibase/backend-core/context") } = require("@budibase/backend-core/context")
const { events } = require("@budibase/backend-core") const { events } = require("@budibase/backend-core")
const { app } = require("@budibase/backend-core/cache") const { app } = require("@budibase/backend-core/cache")
const { logs } = require("@budibase/pro")
const ACTION_DEFS = removeDeprecated(actions.ACTION_DEFINITIONS) const ACTION_DEFS = removeDeprecated(actions.ACTION_DEFINITIONS)
const TRIGGER_DEFS = removeDeprecated(triggers.TRIGGER_DEFINITIONS) const TRIGGER_DEFS = removeDeprecated(triggers.TRIGGER_DEFINITIONS)
@ -195,11 +195,12 @@ exports.destroy = async function (ctx) {
exports.logSearch = async function (ctx) { exports.logSearch = async function (ctx) {
let { automationId, status, page, startDate } = ctx.request.body let { automationId, status, page, startDate } = ctx.request.body
// TODO: need to check maximum allowed in license ctx.body = await logs.automations.getLogs(
if (!startDate) { startDate,
startDate = oneMonthAgo() status,
} automationId,
ctx.body = await getLogs(startDate, status, automationId, page) page
)
} }
exports.clearLogError = async function (ctx) { exports.clearLogError = async function (ctx) {

View File

@ -1,28 +1,20 @@
import {
AutomationLogPage,
AutomationResults,
AutomationStatus,
} from "../../definitions/automation"
import { getAppId, getProdAppDB } from "@budibase/backend-core/context" import { getAppId, getProdAppDB } from "@budibase/backend-core/context"
import { import {
DocumentTypes, DocumentTypes,
generateAutomationLogID, generateAutomationLogID,
getAutomationLogParams,
getQueryIndex,
ViewNames,
SEPARATOR, SEPARATOR,
isProdAppID, isProdAppID,
} from "../../db/utils" } from "../../db/utils"
import { createLogByAutomationView } from "../../db/views/staticViews"
import { Automation, MetadataErrors } from "../../definitions/common" import { Automation, MetadataErrors } from "../../definitions/common"
import { app } from "@budibase/backend-core/cache" import { app } from "@budibase/backend-core/cache"
import { backOff } from "../../utilities" import { backOff } from "../../utilities"
import * as env from "../../environment" import * as env from "../../environment"
import { logs } from "@budibase/pro"
const PAGE_SIZE = 9 import {
const EARLIEST_DATE = new Date(0).toISOString() AutomationResults,
const FREE_EXPIRY_SEC = 86400 AutomationStatus,
const PRO_EXPIRY_SEC = FREE_EXPIRY_SEC * 30 } from "../../definitions/automation"
const { logAlert } = require("@budibase/backend-core/logging")
function getStatus(results: AutomationResults) { function getStatus(results: AutomationResults) {
let status = AutomationStatus.SUCCESS let status = AutomationStatus.SUCCESS
@ -40,28 +32,16 @@ function getStatus(results: AutomationResults) {
return status return status
} }
export function oneMonthAgo() {
return new Date(new Date().getTime() - PRO_EXPIRY_SEC * 1000).toISOString()
}
export function oneDayAgo() {
return new Date(new Date().getTime() - FREE_EXPIRY_SEC * 1000).toISOString()
}
async function clearOldHistory() { async function clearOldHistory() {
const db = getProdAppDB() const db = getProdAppDB()
// TODO: handle license lookup for deletion try {
const expiredEnd = oneDayAgo() const expired = await logs.automations.getExpiredLogs()
const results = await getAllLogs(EARLIEST_DATE, expiredEnd, { const toDelete = expired.data.map((doc: any) => ({
docs: false,
paginate: false,
})
const toDelete = results.data.map((doc: any) => ({
_id: doc.id, _id: doc.id,
_rev: doc.value.rev, _rev: doc.value.rev,
_deleted: true, _deleted: true,
})) }))
const errorLogIds = results.data const errorLogIds = expired.data
.filter((doc: any) => { .filter((doc: any) => {
const parts = doc.id.split(SEPARATOR) const parts = doc.id.split(SEPARATOR)
const status = parts[parts.length - 1] const status = parts[parts.length - 1]
@ -72,76 +52,9 @@ async function clearOldHistory() {
if (errorLogIds.length) { if (errorLogIds.length) {
await updateAppMetadataWithErrors(errorLogIds, { clearing: true }) await updateAppMetadataWithErrors(errorLogIds, { clearing: true })
} }
} } catch (err) {
logAlert(`Failed to cleanup automation log history - Database "${db.name}"`)
function pagination(
response: any,
paginate: boolean = true
): AutomationLogPage {
const data = response.rows.map((row: any) => {
return row.doc ? row.doc : row
})
if (!paginate) {
return { data, hasNextPage: false }
} }
const hasNextPage = data.length > PAGE_SIZE
return {
data: data.slice(0, PAGE_SIZE),
hasNextPage,
nextPage: hasNextPage ? data[PAGE_SIZE]?._id : undefined,
}
}
async function getAllLogs(
startDate: string,
endDate: string,
opts: {
docs: boolean
status?: string
paginate?: boolean
page?: string
} = { docs: true }
): Promise<AutomationLogPage> {
const db = getProdAppDB()
let optional: any = { status: opts.status }
const params = getAutomationLogParams(startDate, endDate, optional, {
include_docs: opts.docs,
limit: opts?.paginate ? PAGE_SIZE + 1 : undefined,
})
if (opts?.page) {
params.startkey = opts.page
}
let response = await db.allDocs(params)
return pagination(response, opts?.paginate)
}
async function getLogsByView(
startDate: string,
endDate: string,
viewParams: { automationId?: string; status?: string; page?: string } = {}
): Promise<AutomationLogPage> {
const db = getProdAppDB()
let response
try {
let optional = {
automationId: viewParams?.automationId,
status: viewParams?.status,
}
const params = getAutomationLogParams(startDate, endDate, optional, {
include_docs: true,
limit: PAGE_SIZE,
})
if (viewParams?.page) {
params.startkey = viewParams.page
}
response = await db.query(getQueryIndex(ViewNames.AUTO_LOGS), params)
} catch (err: any) {
if (err != null && err.name === "not_found") {
await createLogByAutomationView()
return getLogsByView(startDate, endDate, viewParams)
}
}
return pagination(response)
} }
async function updateAppMetadataWithErrors( async function updateAppMetadataWithErrors(
@ -214,28 +127,3 @@ export async function storeLog(
// clear up old logging for app // clear up old logging for app
await clearOldHistory() await clearOldHistory()
} }
export async function getLogs(
startDate: string,
status?: string,
automationId?: string,
page?: string
): Promise<AutomationLogPage> {
let response: AutomationLogPage
let endDate = new Date().toISOString()
if (automationId || status) {
response = await getLogsByView(startDate, endDate, {
automationId,
status,
page,
})
} else {
response = await getAllLogs(startDate, endDate, {
status,
page,
docs: true,
paginate: true,
})
}
return response
}

View File

@ -11,6 +11,8 @@ const {
isProdAppID, isProdAppID,
getDevelopmentAppID, getDevelopmentAppID,
generateAppID, generateAppID,
getQueryIndex,
ViewNames,
} = require("@budibase/backend-core/db") } = require("@budibase/backend-core/db")
const UNICODE_MAX = "\ufff0" const UNICODE_MAX = "\ufff0"
@ -22,16 +24,11 @@ const AppStatus = {
} }
const DocumentTypes = { const DocumentTypes = {
APP: CoreDocTypes.APP, ...CoreDocTypes,
DEV: CoreDocTypes.DEV,
APP_DEV: CoreDocTypes.APP_DEV,
APP_METADATA: CoreDocTypes.APP_METADATA,
ROLE: CoreDocTypes.ROLE,
TABLE: "ta", TABLE: "ta",
ROW: "ro", ROW: "ro",
USER: "us", USER: "us",
AUTOMATION: "au", AUTOMATION: "au",
AUTOMATION_LOG: "log_au",
LINK: "li", LINK: "li",
WEBHOOK: "wh", WEBHOOK: "wh",
INSTANCE: "inst", INSTANCE: "inst",
@ -46,12 +43,6 @@ const DocumentTypes = {
USER_FLAG: "flag", USER_FLAG: "flag",
} }
const ViewNames = {
LINK: "by_link",
ROUTING: "screen_routes",
AUTO_LOGS: "auto_log",
}
const ViewModes = { const ViewModes = {
ALL: "all", ALL: "all",
AUTOMATION: "auto", AUTOMATION: "auto",
@ -98,9 +89,7 @@ exports.generateDevAppID = getDevelopmentAppID
exports.generateRoleID = generateRoleID exports.generateRoleID = generateRoleID
exports.getRoleParams = getRoleParams exports.getRoleParams = getRoleParams
exports.getQueryIndex = viewName => { exports.getQueryIndex = getQueryIndex
return `database/${viewName}`
}
/** /**
* If creating DB allDocs/query params with only a single top level ID this can be used, this * If creating DB allDocs/query params with only a single top level ID this can be used, this

View File

@ -3,11 +3,9 @@ const {
DocumentTypes, DocumentTypes,
SEPARATOR, SEPARATOR,
ViewNames, ViewNames,
ViewModes,
SearchIndexes, SearchIndexes,
} = require("../utils") } = require("../utils")
const SCREEN_PREFIX = DocumentTypes.SCREEN + SEPARATOR const SCREEN_PREFIX = DocumentTypes.SCREEN + SEPARATOR
const LOG_PREFIX = DocumentTypes.AUTOMATION_LOG + SEPARATOR
/************************************************** /**************************************************
* INFORMATION * * INFORMATION *
@ -60,34 +58,6 @@ exports.createLinkView = async () => {
await db.put(designDoc) await db.put(designDoc)
} }
/**
* A separate view that allows us to perform queries by the automation ID and time series, while the
* main all_docs allows access to time series only
*/
exports.createLogByAutomationView = async () => {
const db = getAppDB()
const designDoc = await db.get("_design/database")
const view = {
map: `function(doc) {
if (doc._id.startsWith("${LOG_PREFIX}")) {
let autoId = doc.automationId + "${SEPARATOR}"
let status = doc.status + "${SEPARATOR}"
let autoKey = "${ViewModes.AUTOMATION}${SEPARATOR}" + autoId + doc.createdAt
let statusKey = "${ViewModes.STATUS}${SEPARATOR}" + status + doc.createdAt
let allKey = "${ViewModes.ALL}${SEPARATOR}" + status + autoId + doc.createdAt
emit(statusKey)
emit(autoKey)
emit(allKey)
}
}`,
}
designDoc.views = {
...designDoc.views,
[ViewNames.AUTO_LOGS]: view,
}
await db.put(designDoc)
}
exports.createRoutingView = async () => { exports.createRoutingView = async () => {
const db = getAppDB() const db = getAppDB()
const designDoc = await db.get("_design/database") const designDoc = await db.get("_design/database")

View File

@ -1080,11 +1080,12 @@
resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw== integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==
"@budibase/backend-core@1.0.211": "@budibase/backend-core@1.0.212-alpha.0":
version "1.0.211" version "1.0.212-alpha.0"
resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.211.tgz#579bcc97acf1df2510e302bb70b43245e5ab0e37" resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.212-alpha.0.tgz#6ac3448c8272e918f1af1fff0cf8c5773ae61219"
integrity sha512-ham+Qk6WXQi37Lgnz1Gh/+ItcvCo+dnPZDcVvPESrVxqQkVVIejlxQinZSueIrsgOYHE1j3TUH5zER5M2t6RWw== integrity sha512-hFvbQQEbF3w2u9fe/S+RhNw5HUETS6rhu9q5KDTDQ57k05D4YMPcpMBGSh7SPMqmVyEwUDgcL36mkFOc3AgjYQ==
dependencies: dependencies:
"@budibase/types" "^1.0.212-alpha.0"
"@techpass/passport-openidconnect" "0.3.2" "@techpass/passport-openidconnect" "0.3.2"
aws-sdk "2.1030.0" aws-sdk "2.1030.0"
bcrypt "5.0.1" bcrypt "5.0.1"
@ -1160,12 +1161,12 @@
svelte-flatpickr "^3.2.3" svelte-flatpickr "^3.2.3"
svelte-portal "^1.0.0" svelte-portal "^1.0.0"
"@budibase/pro@1.0.211": "@budibase/pro@1.0.212-alpha.0":
version "1.0.211" version "1.0.212-alpha.0"
resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.211.tgz#d362e9af8c15f6ed386f27b7cca95cc096a91344" resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.212-alpha.0.tgz#843f8291fcc0a2fbcb4d857a750bc116cdaee293"
integrity sha512-dfFByJhlTIURT3sXei5mVXq5rczFMM/ij2Scze0uqPZNpmIlWVqiYdGX7/HEcmIFSS0+UfcdBxjCJJlTGiK4/w== integrity sha512-4nhWxjMcxSQBPXRy/U+37IaVLYOr4/RVe79/fUvnXrr5qAeecbEk/QbkJJd3dU1WaNxB2eGhNtH3uBUPQvcT9A==
dependencies: dependencies:
"@budibase/backend-core" "1.0.211" "@budibase/backend-core" "1.0.212-alpha.0"
node-fetch "^2.6.1" node-fetch "^2.6.1"
"@budibase/standard-components@^0.9.139": "@budibase/standard-components@^0.9.139":
@ -1186,6 +1187,11 @@
svelte-apexcharts "^1.0.2" svelte-apexcharts "^1.0.2"
svelte-flatpickr "^3.1.0" svelte-flatpickr "^3.1.0"
"@budibase/types@^1.0.212-alpha.0":
version "1.0.212"
resolved "https://registry.yarnpkg.com/@budibase/types/-/types-1.0.212.tgz#e66a15b711544b4fab7767261fd5f2f1dd7f40d7"
integrity sha512-DhGyw6snwJQZQlx7havVYnqPZfZERueKZfmVCBySzwInZZt0+sXZaBl1BVjGjYuwpaUQBMDBf7geBgHXp6DIKg==
"@bull-board/api@3.7.0": "@bull-board/api@3.7.0":
version "3.7.0" version "3.7.0"
resolved "https://registry.yarnpkg.com/@bull-board/api/-/api-3.7.0.tgz#231f687187c0cb34e0b97f463917b6aaeb4ef6af" resolved "https://registry.yarnpkg.com/@bull-board/api/-/api-3.7.0.tgz#231f687187c0cb34e0b97f463917b6aaeb4ef6af"
@ -8832,7 +8838,7 @@ keyv@^3.0.0:
kind-of@^3.0.2, kind-of@^3.0.3, kind-of@^3.2.0: kind-of@^3.0.2, kind-of@^3.0.3, kind-of@^3.2.0:
version "3.2.2" version "3.2.2"
resolved "https://registry.yarnpkg.com/kind-of/-/kind-of-3.2.2.tgz#31ea21a734bab9bbb0f32466d893aea51e4a3c64" resolved "https://registry.yarnpkg.com/kind-of/-/kind-of-3.2.2.tgz#31ea21a734bab9bbb0f32466d893aea51e4a3c64"
integrity sha1-MeohpzS6ubuw8yRm2JOupR5KPGQ= integrity sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ==
dependencies: dependencies:
is-buffer "^1.1.5" is-buffer "^1.1.5"

View File

@ -293,11 +293,12 @@
resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw== integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==
"@budibase/backend-core@1.0.207-alpha.3": "@budibase/backend-core@1.0.212-alpha.0":
version "1.0.207-alpha.3" version "1.0.212-alpha.0"
resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.207-alpha.3.tgz#98bced0575ec4e2b158239a8c73b39ca2d816719" resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.212-alpha.0.tgz#6ac3448c8272e918f1af1fff0cf8c5773ae61219"
integrity sha512-DU4X6jJ+DfhzOv4TTa1w4Dk5ZEdlK/z1joCTruT+SGM5qI75bXrGeol5OX2OaEbNKtXFKJ1zeVTmBCYcu7OFUg== integrity sha512-hFvbQQEbF3w2u9fe/S+RhNw5HUETS6rhu9q5KDTDQ57k05D4YMPcpMBGSh7SPMqmVyEwUDgcL36mkFOc3AgjYQ==
dependencies: dependencies:
"@budibase/types" "^1.0.212-alpha.0"
"@techpass/passport-openidconnect" "0.3.2" "@techpass/passport-openidconnect" "0.3.2"
aws-sdk "2.1030.0" aws-sdk "2.1030.0"
bcrypt "5.0.1" bcrypt "5.0.1"
@ -324,14 +325,19 @@
uuid "8.3.2" uuid "8.3.2"
zlib "1.0.5" zlib "1.0.5"
"@budibase/pro@1.0.207-alpha.3": "@budibase/pro@1.0.212-alpha.0":
version "1.0.207-alpha.3" version "1.0.212-alpha.0"
resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.207-alpha.3.tgz#9bde845ceb685f1b43286a124620c21fdf891a01" resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.212-alpha.0.tgz#843f8291fcc0a2fbcb4d857a750bc116cdaee293"
integrity sha512-WFEMujpKTVAMvAgLBnMdw8ou9PxsbM4Oa9Dq+DAUsWpPACsMWOProyHLsdRxJyvHlgGfwVjo5MEusvStjI4j6g== integrity sha512-4nhWxjMcxSQBPXRy/U+37IaVLYOr4/RVe79/fUvnXrr5qAeecbEk/QbkJJd3dU1WaNxB2eGhNtH3uBUPQvcT9A==
dependencies: dependencies:
"@budibase/backend-core" "1.0.207-alpha.3" "@budibase/backend-core" "1.0.212-alpha.0"
node-fetch "^2.6.1" node-fetch "^2.6.1"
"@budibase/types@^1.0.212-alpha.0":
version "1.0.212"
resolved "https://registry.yarnpkg.com/@budibase/types/-/types-1.0.212.tgz#e66a15b711544b4fab7767261fd5f2f1dd7f40d7"
integrity sha512-DhGyw6snwJQZQlx7havVYnqPZfZERueKZfmVCBySzwInZZt0+sXZaBl1BVjGjYuwpaUQBMDBf7geBgHXp6DIKg==
"@cspotcode/source-map-consumer@0.8.0": "@cspotcode/source-map-consumer@0.8.0":
version "0.8.0" version "0.8.0"
resolved "https://registry.yarnpkg.com/@cspotcode/source-map-consumer/-/source-map-consumer-0.8.0.tgz#33bf4b7b39c178821606f669bbc447a6a629786b" resolved "https://registry.yarnpkg.com/@cspotcode/source-map-consumer/-/source-map-consumer-0.8.0.tgz#33bf4b7b39c178821606f669bbc447a6a629786b"