MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Moving get of logs to pro.

This commit is contained in:
mike12345567 2022-06-27 18:00:48 +01:00
parent 99f5d3123d
commit 5100d86752
8 changed files with 94 additions and 215 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
packages/backend-core/src/db/constants.js
View File

@ -1,8 +1,27 @@
exports.SEPARATOR = "_"
exports.UNICODE_MAX = "\ufff0"
const PRE_APP = "app"
const PRE_DEV = "dev"
/**
* Can be used to create a few different forms of querying a view.
*/
exports.ViewModes = {
ALL: "all",
AUTOMATION: "auto",
STATUS: "status",
}
exports.ViewNames = {
USER_BY_EMAIL: "by_email",
BY_API_KEY: "by_api_key",
USER_BY_BUILDERS: "by_builders",
LINK: "by_link",
ROUTING: "screen_routes",
AUTO_LOGS: "auto_log",
}
exports.DocumentTypes = {
USER: "us",
WORKSPACE: "workspace",
@ -15,6 +34,7 @@ exports.DocumentTypes = {
ROLE: "role",
MIGRATIONS: "migrations",
DEV_INFO: "devinfo",
AUTOMATION_LOG: "log_au",
}
exports.StaticDatabases = {

17
packages/backend-core/src/db/utils.ts
View File

@ -1,7 +1,7 @@
import { newid } from "../hashing"
import { DEFAULT_TENANT_ID, Configs } from "../constants"
import env from "../environment"
import { SEPARATOR, DocumentTypes } from "./constants"
import { SEPARATOR, DocumentTypes, UNICODE_MAX } from "./constants"
import { getTenantId, getGlobalDBName, getGlobalDB } from "../tenancy"
import fetch from "node-fetch"
import { doWithDB, allDbs } from "./index"
@ -12,14 +12,6 @@ import { isDevApp, isDevAppID } from "./conversions"
import { APP_PREFIX } from "./constants"
import * as events from "../events"
const UNICODE_MAX = "\ufff0"
export const ViewNames = {
USER_BY_EMAIL: "by_email",
BY_API_KEY: "by_api_key",
USER_BY_BUILDERS: "by_builders",
}
export * from "./constants"
export * from "./conversions"
export { default as Replication } from "./Replication"
@ -63,6 +55,13 @@ export function getDocParams(
}
}
/**
* Retrieve the correct index for a view based on default design DB.
*/
export function getQueryIndex(viewName: string) {
return `database/${viewName}`
}
/**
* Generates a new workspace ID.
* @returns {string} The new workspace ID which the workspace doc can be stored under.

13
packages/server/src/api/controllers/automation.js
View File

@ -1,6 +1,5 @@
const actions = require("../../automations/actions")
const triggers = require("../../automations/triggers")
const { getLogs, oneMonthAgo } = require("../../automations/logging")
const {
getAutomationParams,
generateAutomationID,
@ -21,6 +20,7 @@ const {
} = require("@budibase/backend-core/context")
const { events } = require("@budibase/backend-core")
const { app } = require("@budibase/backend-core/cache")
const { logs } = require("@budibase/pro")
const ACTION_DEFS = removeDeprecated(actions.ACTION_DEFINITIONS)
const TRIGGER_DEFS = removeDeprecated(triggers.TRIGGER_DEFINITIONS)
@ -195,11 +195,12 @@ exports.destroy = async function (ctx) {
exports.logSearch = async function (ctx) {
let { automationId, status, page, startDate } = ctx.request.body
// TODO: need to check maximum allowed in license
if (!startDate) {
startDate = oneMonthAgo()
}
ctx.body = await getLogs(startDate, status, automationId, page)
ctx.body = await logs.automations.getLogs(
startDate,
status,
automationId,
page
)
}
exports.clearLogError = async function (ctx) {

160
packages/server/src/automations/logging/index.ts
View File

@ -1,28 +1,20 @@
import {
AutomationLogPage,
AutomationResults,
AutomationStatus,
} from "../../definitions/automation"
import { getAppId, getProdAppDB } from "@budibase/backend-core/context"
import {
DocumentTypes,
generateAutomationLogID,
getAutomationLogParams,
getQueryIndex,
ViewNames,
SEPARATOR,
isProdAppID,
} from "../../db/utils"
import { createLogByAutomationView } from "../../db/views/staticViews"
import { Automation, MetadataErrors } from "../../definitions/common"
import { app } from "@budibase/backend-core/cache"
import { backOff } from "../../utilities"
import * as env from "../../environment"
const PAGE_SIZE = 9
const EARLIEST_DATE = new Date(0).toISOString()
const FREE_EXPIRY_SEC = 86400
const PRO_EXPIRY_SEC = FREE_EXPIRY_SEC * 30
import { logs } from "@budibase/pro"
import {
AutomationResults,
AutomationStatus,
} from "../../definitions/automation"
const { logAlert } = require("@budibase/backend-core/logging")
function getStatus(results: AutomationResults) {
let status = AutomationStatus.SUCCESS
@ -40,108 +32,29 @@ function getStatus(results: AutomationResults) {
return status
}
export function oneMonthAgo() {
return new Date(new Date().getTime() - PRO_EXPIRY_SEC * 1000).toISOString()
}
export function oneDayAgo() {
return new Date(new Date().getTime() - FREE_EXPIRY_SEC * 1000).toISOString()
}
async function clearOldHistory() {
const db = getProdAppDB()
// TODO: handle license lookup for deletion
const expiredEnd = oneDayAgo()
const results = await getAllLogs(EARLIEST_DATE, expiredEnd, {
docs: false,
paginate: false,
})
const toDelete = results.data.map((doc: any) => ({
_id: doc.id,
_rev: doc.value.rev,
_deleted: true,
}))
const errorLogIds = results.data
.filter((doc: any) => {
const parts = doc.id.split(SEPARATOR)
const status = parts[parts.length - 1]
return status === AutomationStatus.ERROR
})
.map((doc: any) => doc.id)
await db.bulkDocs(toDelete)
if (errorLogIds.length) {
await updateAppMetadataWithErrors(errorLogIds, { clearing: true })
}
}
function pagination(
response: any,
paginate: boolean = true
): AutomationLogPage {
const data = response.rows.map((row: any) => {
return row.doc ? row.doc : row
})
if (!paginate) {
return { data, hasNextPage: false }
}
const hasNextPage = data.length > PAGE_SIZE
return {
data: data.slice(0, PAGE_SIZE),
hasNextPage,
nextPage: hasNextPage ? data[PAGE_SIZE]?._id : undefined,
}
}
async function getAllLogs(
startDate: string,
endDate: string,
opts: {
docs: boolean
status?: string
paginate?: boolean
page?: string
} = { docs: true }
): Promise<AutomationLogPage> {
const db = getProdAppDB()
let optional: any = { status: opts.status }
const params = getAutomationLogParams(startDate, endDate, optional, {
include_docs: opts.docs,
limit: opts?.paginate ? PAGE_SIZE + 1 : undefined,
})
if (opts?.page) {
params.startkey = opts.page
}
let response = await db.allDocs(params)
return pagination(response, opts?.paginate)
}
async function getLogsByView(
startDate: string,
endDate: string,
viewParams: { automationId?: string; status?: string; page?: string } = {}
): Promise<AutomationLogPage> {
const db = getProdAppDB()
let response
try {
let optional = {
automationId: viewParams?.automationId,
status: viewParams?.status,
}
const params = getAutomationLogParams(startDate, endDate, optional, {
include_docs: true,
limit: PAGE_SIZE,
})
if (viewParams?.page) {
params.startkey = viewParams.page
}
response = await db.query(getQueryIndex(ViewNames.AUTO_LOGS), params)
} catch (err: any) {
if (err != null && err.name === "not_found") {
await createLogByAutomationView()
return getLogsByView(startDate, endDate, viewParams)
const expired = await logs.automations.getExpiredLogs()
const toDelete = expired.data.map((doc: any) => ({
_id: doc.id,
_rev: doc.value.rev,
_deleted: true,
}))
const errorLogIds = expired.data
.filter((doc: any) => {
const parts = doc.id.split(SEPARATOR)
const status = parts[parts.length - 1]
return status === AutomationStatus.ERROR
})
.map((doc: any) => doc.id)
await db.bulkDocs(toDelete)
if (errorLogIds.length) {
await updateAppMetadataWithErrors(errorLogIds, { clearing: true })
}
} catch (err) {
logAlert(`Failed to cleanup automation log history - Database "${db.name}"`)
}
return pagination(response)
}
async function updateAppMetadataWithErrors(
@ -214,28 +127,3 @@ export async function storeLog(
// clear up old logging for app
await clearOldHistory()
}
export async function getLogs(
startDate: string,
status?: string,
automationId?: string,
page?: string
): Promise<AutomationLogPage> {
let response: AutomationLogPage
let endDate = new Date().toISOString()
if (automationId || status) {
response = await getLogsByView(startDate, endDate, {
automationId,
status,
page,
})
} else {
response = await getAllLogs(startDate, endDate, {
status,
page,
docs: true,
paginate: true,
})
}
return response
}

19
packages/server/src/db/utils.js
View File

@ -11,6 +11,8 @@ const {
isProdAppID,
getDevelopmentAppID,
generateAppID,
getQueryIndex,
ViewNames,
} = require("@budibase/backend-core/db")
const UNICODE_MAX = "\ufff0"
@ -22,16 +24,11 @@ const AppStatus = {
}
const DocumentTypes = {
APP: CoreDocTypes.APP,
DEV: CoreDocTypes.DEV,
APP_DEV: CoreDocTypes.APP_DEV,
APP_METADATA: CoreDocTypes.APP_METADATA,
ROLE: CoreDocTypes.ROLE,
...CoreDocTypes,
TABLE: "ta",
ROW: "ro",
USER: "us",
AUTOMATION: "au",
AUTOMATION_LOG: "log_au",
LINK: "li",
WEBHOOK: "wh",
INSTANCE: "inst",
@ -46,12 +43,6 @@ const DocumentTypes = {
USER_FLAG: "flag",
}
const ViewNames = {
LINK: "by_link",
ROUTING: "screen_routes",
AUTO_LOGS: "auto_log",
}
const ViewModes = {
ALL: "all",
AUTOMATION: "auto",
@ -98,9 +89,7 @@ exports.generateDevAppID = getDevelopmentAppID
exports.generateRoleID = generateRoleID
exports.getRoleParams = getRoleParams
exports.getQueryIndex = viewName => {
return `database/${viewName}`
}
exports.getQueryIndex = getQueryIndex
/**
* If creating DB allDocs/query params with only a single top level ID this can be used, this

30
packages/server/src/db/views/staticViews.js
View File

@ -3,11 +3,9 @@ const {
DocumentTypes,
SEPARATOR,
ViewNames,
ViewModes,
SearchIndexes,
} = require("../utils")
const SCREEN_PREFIX = DocumentTypes.SCREEN + SEPARATOR
const LOG_PREFIX = DocumentTypes.AUTOMATION_LOG + SEPARATOR
/**************************************************
* INFORMATION *
@ -60,34 +58,6 @@ exports.createLinkView = async () => {
await db.put(designDoc)
}
/**
* A separate view that allows us to perform queries by the automation ID and time series, while the
* main all_docs allows access to time series only
*/
exports.createLogByAutomationView = async () => {
const db = getAppDB()
const designDoc = await db.get("_design/database")
const view = {
map: `function(doc) {
if (doc._id.startsWith("${LOG_PREFIX}")) {
let autoId = doc.automationId + "${SEPARATOR}"
let status = doc.status + "${SEPARATOR}"
let autoKey = "${ViewModes.AUTOMATION}${SEPARATOR}" + autoId + doc.createdAt
let statusKey = "${ViewModes.STATUS}${SEPARATOR}" + status + doc.createdAt
let allKey = "${ViewModes.ALL}${SEPARATOR}" + status + autoId + doc.createdAt
emit(statusKey)
emit(autoKey)
emit(allKey)
}
}`,
}
designDoc.views = {
...designDoc.views,
[ViewNames.AUTO_LOGS]: view,
}
await db.put(designDoc)
}
exports.createRoutingView = async () => {
const db = getAppDB()
const designDoc = await db.get("_design/database")

26
packages/server/yarn.lock
View File

@ -1080,11 +1080,12 @@
resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==
"@budibase/backend-core@1.0.211":
version "1.0.211"
resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.211.tgz#579bcc97acf1df2510e302bb70b43245e5ab0e37"
integrity sha512-ham+Qk6WXQi37Lgnz1Gh/+ItcvCo+dnPZDcVvPESrVxqQkVVIejlxQinZSueIrsgOYHE1j3TUH5zER5M2t6RWw==
"@budibase/backend-core@1.0.212-alpha.0":
version "1.0.212-alpha.0"
resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.212-alpha.0.tgz#6ac3448c8272e918f1af1fff0cf8c5773ae61219"
integrity sha512-hFvbQQEbF3w2u9fe/S+RhNw5HUETS6rhu9q5KDTDQ57k05D4YMPcpMBGSh7SPMqmVyEwUDgcL36mkFOc3AgjYQ==
dependencies:
"@budibase/types" "^1.0.212-alpha.0"
"@techpass/passport-openidconnect" "0.3.2"
aws-sdk "2.1030.0"
bcrypt "5.0.1"
@ -1160,12 +1161,12 @@
svelte-flatpickr "^3.2.3"
svelte-portal "^1.0.0"
"@budibase/pro@1.0.211":
version "1.0.211"
resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.211.tgz#d362e9af8c15f6ed386f27b7cca95cc096a91344"
integrity sha512-dfFByJhlTIURT3sXei5mVXq5rczFMM/ij2Scze0uqPZNpmIlWVqiYdGX7/HEcmIFSS0+UfcdBxjCJJlTGiK4/w==
"@budibase/pro@1.0.212-alpha.0":
version "1.0.212-alpha.0"
resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.212-alpha.0.tgz#843f8291fcc0a2fbcb4d857a750bc116cdaee293"
integrity sha512-4nhWxjMcxSQBPXRy/U+37IaVLYOr4/RVe79/fUvnXrr5qAeecbEk/QbkJJd3dU1WaNxB2eGhNtH3uBUPQvcT9A==
dependencies:
"@budibase/backend-core" "1.0.211"
"@budibase/backend-core" "1.0.212-alpha.0"
node-fetch "^2.6.1"
"@budibase/standard-components@^0.9.139":
@ -1186,6 +1187,11 @@
svelte-apexcharts "^1.0.2"
svelte-flatpickr "^3.1.0"
"@budibase/types@^1.0.212-alpha.0":
version "1.0.212"
resolved "https://registry.yarnpkg.com/@budibase/types/-/types-1.0.212.tgz#e66a15b711544b4fab7767261fd5f2f1dd7f40d7"
integrity sha512-DhGyw6snwJQZQlx7havVYnqPZfZERueKZfmVCBySzwInZZt0+sXZaBl1BVjGjYuwpaUQBMDBf7geBgHXp6DIKg==
"@bull-board/api@3.7.0":
version "3.7.0"
resolved "https://registry.yarnpkg.com/@bull-board/api/-/api-3.7.0.tgz#231f687187c0cb34e0b97f463917b6aaeb4ef6af"
@ -8832,7 +8838,7 @@ keyv@^3.0.0:
kind-of@^3.0.2, kind-of@^3.0.3, kind-of@^3.2.0:
version "3.2.2"
resolved "https://registry.yarnpkg.com/kind-of/-/kind-of-3.2.2.tgz#31ea21a734bab9bbb0f32466d893aea51e4a3c64"
integrity sha1-MeohpzS6ubuw8yRm2JOupR5KPGQ=
integrity sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ==
dependencies:
is-buffer "^1.1.5"

24
packages/worker/yarn.lock
View File

@ -293,11 +293,12 @@
resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==
"@budibase/backend-core@1.0.207-alpha.3":
version "1.0.207-alpha.3"
resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.207-alpha.3.tgz#98bced0575ec4e2b158239a8c73b39ca2d816719"
integrity sha512-DU4X6jJ+DfhzOv4TTa1w4Dk5ZEdlK/z1joCTruT+SGM5qI75bXrGeol5OX2OaEbNKtXFKJ1zeVTmBCYcu7OFUg==
"@budibase/backend-core@1.0.212-alpha.0":
version "1.0.212-alpha.0"
resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.212-alpha.0.tgz#6ac3448c8272e918f1af1fff0cf8c5773ae61219"
integrity sha512-hFvbQQEbF3w2u9fe/S+RhNw5HUETS6rhu9q5KDTDQ57k05D4YMPcpMBGSh7SPMqmVyEwUDgcL36mkFOc3AgjYQ==
dependencies:
"@budibase/types" "^1.0.212-alpha.0"
"@techpass/passport-openidconnect" "0.3.2"
aws-sdk "2.1030.0"
bcrypt "5.0.1"
@ -324,14 +325,19 @@
uuid "8.3.2"
zlib "1.0.5"
"@budibase/pro@1.0.207-alpha.3":
version "1.0.207-alpha.3"
resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.207-alpha.3.tgz#9bde845ceb685f1b43286a124620c21fdf891a01"
integrity sha512-WFEMujpKTVAMvAgLBnMdw8ou9PxsbM4Oa9Dq+DAUsWpPACsMWOProyHLsdRxJyvHlgGfwVjo5MEusvStjI4j6g==
"@budibase/pro@1.0.212-alpha.0":
version "1.0.212-alpha.0"
resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.212-alpha.0.tgz#843f8291fcc0a2fbcb4d857a750bc116cdaee293"
integrity sha512-4nhWxjMcxSQBPXRy/U+37IaVLYOr4/RVe79/fUvnXrr5qAeecbEk/QbkJJd3dU1WaNxB2eGhNtH3uBUPQvcT9A==
dependencies:
"@budibase/backend-core" "1.0.207-alpha.3"
"@budibase/backend-core" "1.0.212-alpha.0"
node-fetch "^2.6.1"
"@budibase/types@^1.0.212-alpha.0":
version "1.0.212"
resolved "https://registry.yarnpkg.com/@budibase/types/-/types-1.0.212.tgz#e66a15b711544b4fab7767261fd5f2f1dd7f40d7"
integrity sha512-DhGyw6snwJQZQlx7havVYnqPZfZERueKZfmVCBySzwInZZt0+sXZaBl1BVjGjYuwpaUQBMDBf7geBgHXp6DIKg==
"@cspotcode/source-map-consumer@0.8.0":
version "0.8.0"
resolved "https://registry.yarnpkg.com/@cspotcode/source-map-consumer/-/source-map-consumer-0.8.0.tgz#33bf4b7b39c178821606f669bbc447a6a629786b"