MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Removing automation logs from app exports as they are not useful - they take up space and the imported app won't have ran any automations yet. I have added it as a flag so that this does not affect backups (as you would want to maintain logs for backups).

This commit is contained in:
mike12345567 2023-04-27 11:36:52 +01:00
parent 1b815c363a
commit 5363693ac3
3 changed files with 24 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
packages/server/src/db/utils.ts
View File

@ -27,6 +27,7 @@ export const isProdAppID = dbCore.isProdAppID
export const USER_METDATA_PREFIX = `${DocumentType.ROW}${SEPARATOR}${dbCore.InternalTable.USER_METADATA}${SEPARATOR}` export const USER_METDATA_PREFIX = `${DocumentType.ROW}${SEPARATOR}${dbCore.InternalTable.USER_METADATA}${SEPARATOR}`
export const LINK_USER_METADATA_PREFIX = `${DocumentType.LINK}${SEPARATOR}${dbCore.InternalTable.USER_METADATA}${SEPARATOR}` export const LINK_USER_METADATA_PREFIX = `${DocumentType.LINK}${SEPARATOR}${dbCore.InternalTable.USER_METADATA}${SEPARATOR}`
export const TABLE_ROW_PREFIX = `${DocumentType.ROW}${SEPARATOR}${DocumentType.TABLE}` export const TABLE_ROW_PREFIX = `${DocumentType.ROW}${SEPARATOR}${DocumentType.TABLE}`
export const AUTOMATION_LOG_PREFIX = `${DocumentType.AUTOMATION_LOG}${SEPARATOR}`
export const ViewName = dbCore.ViewName export const ViewName = dbCore.ViewName
export const InternalTables = dbCore.InternalTable export const InternalTables = dbCore.InternalTable
export const UNICODE_MAX = dbCore.UNICODE_MAX export const UNICODE_MAX = dbCore.UNICODE_MAX

23
packages/server/src/sdk/app/backups/exports.ts
View File

@ -3,6 +3,7 @@ import { budibaseTempDir } from "../../../utilities/budibaseDir"
import { streamFile, createTempFolder } from "../../../utilities/fileSystem" import { streamFile, createTempFolder } from "../../../utilities/fileSystem"
import { ObjectStoreBuckets } from "../../../constants" import { ObjectStoreBuckets } from "../../../constants"
import { import {
AUTOMATION_LOG_PREFIX,
LINK_USER_METADATA_PREFIX, LINK_USER_METADATA_PREFIX,
TABLE_ROW_PREFIX, TABLE_ROW_PREFIX,
USER_METDATA_PREFIX, USER_METDATA_PREFIX,
@ -20,11 +21,15 @@ const uuid = require("uuid/v4")
const tar = require("tar") const tar = require("tar")
const MemoryStream = require("memorystream") const MemoryStream = require("memorystream")
type ExportOpts = { interface DBDumpOpts {
filter?: any filter?: any
exportPath?: string exportPath?: string
}
interface ExportOpts extends DBDumpOpts {
tar?: boolean tar?: boolean
excludeRows?: boolean excludeRows?: boolean
excludeLogs?: boolean
} }
function tarFilesToTmp(tmpDir: string, files: string[]) { function tarFilesToTmp(tmpDir: string, files: string[]) {
@ -49,7 +54,7 @@ function tarFilesToTmp(tmpDir: string, files: string[]) {
* a filter function or the name of the export. * a filter function or the name of the export.
* @return {*} either a readable stream or a string * @return {*} either a readable stream or a string
*/ */
export async function exportDB(dbName: string, opts: ExportOpts = {}) { export async function exportDB(dbName: string, opts: DBDumpOpts = {}) {
const exportOpts = { const exportOpts = {
filter: opts?.filter, filter: opts?.filter,
batch_size: 1000, batch_size: 1000,
@ -76,11 +81,14 @@ export async function exportDB(dbName: string, opts: ExportOpts = {}) {
}) })
} }
function defineFilter(excludeRows?: boolean) { function defineFilter(excludeRows?: boolean, excludeLogs?: boolean) {
const ids = [USER_METDATA_PREFIX, LINK_USER_METADATA_PREFIX] const ids = [USER_METDATA_PREFIX, LINK_USER_METADATA_PREFIX]
if (excludeRows) { if (excludeRows) {
ids.push(TABLE_ROW_PREFIX) ids.push(TABLE_ROW_PREFIX)
} }
if (excludeLogs) {
ids.push(AUTOMATION_LOG_PREFIX)
}
return (doc: any) => return (doc: any) =>
!ids.map(key => doc._id.includes(key)).reduce((prev, curr) => prev || curr) !ids.map(key => doc._id.includes(key)).reduce((prev, curr) => prev || curr)
} }
@ -130,8 +138,7 @@ export async function exportApp(appId: string, config?: ExportOpts) {
// enforce an export of app DB to the tmp path // enforce an export of app DB to the tmp path
const dbPath = join(tmpPath, DB_EXPORT_FILE) const dbPath = join(tmpPath, DB_EXPORT_FILE)
await exportDB(appId, { await exportDB(appId, {
...config, filter: defineFilter(config?.excludeRows, config?.excludeLogs),
filter: defineFilter(config?.excludeRows),
exportPath: dbPath, exportPath: dbPath,
}) })
// if tar requested, return where the tarball is // if tar requested, return where the tarball is
@ -155,6 +162,10 @@ export async function exportApp(appId: string, config?: ExportOpts) {
* @returns {*} a readable stream of the backup which is written in real time * @returns {*} a readable stream of the backup which is written in real time
*/ */
export async function streamExportApp(appId: string, excludeRows: boolean) { export async function streamExportApp(appId: string, excludeRows: boolean) {
const tmpPath = await exportApp(appId, { excludeRows, tar: true }) const tmpPath = await exportApp(appId, {
excludeRows,
excludeLogs: true,
tar: true,
})
return streamFile(tmpPath) return streamFile(tmpPath)
} }

12
yarn.lock
View File

@ -1486,15 +1486,15 @@
pouchdb-promise "^6.0.4" pouchdb-promise "^6.0.4"
through2 "^2.0.0" through2 "^2.0.0"
"@budibase/pro@2.5.6-alpha.35": "@budibase/pro@2.5.6-alpha.36":
version "2.5.6-alpha.35" version "2.5.6-alpha.36"
resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.5.6-alpha.35.tgz#b065c39f23829db191ced3a583285bcecf7c1dde" resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.5.6-alpha.36.tgz#361afe64b0881ee436a5ef294fb315c05ea94ce6"
integrity sha512-YxKCRSkLprOK9ipu+R9vg9LIumas4zM0ZTTPyL+0NRDhJRwINRQziJIkakNaIHPLEPadVQFNrBY33HOS3hzdZg== integrity sha512-uX1wgOk47aVGl/yIJZiZS8x31sTS6wGDEFv0AMZ2h6rwIp6GwHDGq2/QT6a8hRMsAM4sqr8R2GkyyAG+dm0DGQ==
dependencies: dependencies:
"@budibase/backend-core" "2.5.6-alpha.35" "@budibase/backend-core" "2.5.6-alpha.36"
"@budibase/shared-core" "2.4.44-alpha.1" "@budibase/shared-core" "2.4.44-alpha.1"
"@budibase/string-templates" "2.4.44-alpha.1" "@budibase/string-templates" "2.4.44-alpha.1"
"@budibase/types" "2.5.6-alpha.35" "@budibase/types" "2.5.6-alpha.36"
"@koa/router" "8.0.8" "@koa/router" "8.0.8"
bull "4.10.1" bull "4.10.1"
joi "17.6.0" joi "17.6.0"