MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Adding main functionality for exporting CouchDB to disk, just need to add MinIO export.

This commit is contained in:
mike12345567 2022-06-30 19:20:50 +01:00
parent 3c70bac76c
commit 11df05a7a0
9 changed files with 1352 additions and 82 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
packages/backend-core/src/db/pouch.js
View File

@ -102,6 +102,13 @@ exports.getPouch = (opts = {}) => {
} }
} }
if (opts.onDisk) {
POUCH_DB_DEFAULTS = {
prefix: undefined,
adapter: "leveldb",
}
}
if (opts.replication) { if (opts.replication) {
const replicationStream = require("pouchdb-replication-stream") const replicationStream = require("pouchdb-replication-stream")
PouchDB.plugin(replicationStream.plugin) PouchDB.plugin(replicationStream.plugin)

1
packages/backend-core/src/db/utils.ts
View File

@ -185,6 +185,7 @@ export async function getAllDbs(opts = { efficient: false }) {
} }
} }
let couchUrl = `${url}/_all_dbs` let couchUrl = `${url}/_all_dbs`
console.log(couchUrl)
let tenantId = getTenantId() let tenantId = getTenantId()
if (!env.MULTI_TENANCY || (!efficient && tenantId === DEFAULT_TENANT_ID)) { if (!env.MULTI_TENANCY || (!efficient && tenantId === DEFAULT_TENANT_ID)) {
// just get all DBs when: // just get all DBs when:

6
packages/cli/package.json
View File

@ -20,8 +20,10 @@
"outputPath": "build" "outputPath": "build"
}, },
"dependencies": { "dependencies": {
"@budibase/backend-core": "^1.0.212-alpha.7",
"axios": "^0.21.1", "axios": "^0.21.1",
"chalk": "^4.1.0", "chalk": "^4.1.0",
"cli-progress": "^3.11.2",
"commander": "^7.1.0", "commander": "^7.1.0",
"docker-compose": "^0.23.6", "docker-compose": "^0.23.6",
"dotenv": "^16.0.1", "dotenv": "^16.0.1",
@ -29,8 +31,8 @@
"lookpath": "^1.1.0", "lookpath": "^1.1.0",
"pkg": "^5.3.0", "pkg": "^5.3.0",
"posthog-node": "1.0.7", "posthog-node": "1.0.7",
"pouchdb": "^7.3.0", "randomstring": "^1.1.5",
"randomstring": "^1.1.5" "tar": "^6.1.11"
}, },
"devDependencies": { "devDependencies": {
"eslint": "^7.20.0" "eslint": "^7.20.0"

157
packages/cli/src/backups/index.js
View File

@ -1,41 +1,48 @@
const Command = require("../structures/Command") const Command = require("../structures/Command")
const { CommandWords } = require("../constants") const { CommandWords } = require("../constants")
//const pouchdb = require("pouchdb")
const dotenv = require("dotenv") const dotenv = require("dotenv")
const fs = require("fs") const fs = require("fs")
const { join } = require("path")
const { string } = require("../questions") const { string } = require("../questions")
const { env } = require("@budibase/backend-core")
const { getPouch, getAllDbs } = require("@budibase/backend-core/db")
const tar = require("tar")
const { progressBar } = require("../utils")
const DEFAULT_COUCH = "http://budibase:budibase@localhost:10000/db/"
const DEFAULT_MINIO = "http://localhost:10000/"
const TEMP_DIR = ".temp"
const REQUIRED = [ const REQUIRED = [
{ value: "MAIN_PORT", key: "Budibase Port", default: "10000" }, { value: "MAIN_PORT", default: "10000" },
{ { value: "COUCH_DB_URL", default: DEFAULT_COUCH },
value: "COUCH_DB_URL", { value: "MINIO_URL", default: DEFAULT_MINIO },
key: "CouchDB URL", { value: "MINIO_ACCESS_KEY" },
default: "http://budibase:budibase@localhost:10000/db/", { value: "MINIO_SECRET_KEY" },
},
{ value: "MINIO_URL", key: "MinIO URL", default: "http://localhost:10000/" },
{ value: "MINIO_ACCESS_KEY", key: "MinIO Access Key" },
{ value: "MINIO_SECRET_KEY", key: "MinIO Secret Key" },
] ]
function checkCouchURL(config) { function checkURLs(config) {
if (config["COUCH_DB_URL"]) {
return config
}
const mainPort = config["MAIN_PORT"], const mainPort = config["MAIN_PORT"],
username = config["COUCH_DB_USERNAME"], username = config["COUCH_DB_USER"],
password = config["COUCH_DB_PASSWORD"] password = config["COUCH_DB_PASSWORD"]
if (mainPort && username && password) { if (!config["COUCH_DB_URL"] && mainPort && username && password) {
config[ config[
"COUCH_DB_URL" "COUCH_DB_URL"
] = `http://${username}:${password}@localhost:${mainPort}/db/` ] = `http://${username}:${password}@localhost:${mainPort}/db/`
} }
if (!config["MINIO_URL"]) {
config["MINIO_URL"] = DEFAULT_MINIO
}
return config return config
} }
async function askQuestions() { async function askQuestions() {
console.log(
"*** NOTE: use a .env file to load these parameters repeatedly ***"
)
let config = {} let config = {}
for (let property of REQUIRED) { for (let property of REQUIRED) {
config[property.value] = await string(property.key, property.default) config[property.value] = await string(property.value, property.default)
} }
return config return config
} }
@ -45,7 +52,7 @@ function loadEnvironment(path) {
throw "Unable to file specified .env file" throw "Unable to file specified .env file"
} }
const env = fs.readFileSync(path, "utf8") const env = fs.readFileSync(path, "utf8")
const config = checkCouchURL(dotenv.parse(env)) const config = checkURLs(dotenv.parse(env))
for (let required of REQUIRED) { for (let required of REQUIRED) {
if (!config[required.value]) { if (!config[required.value]) {
throw `Cannot find "${required.value}" property in .env file` throw `Cannot find "${required.value}" property in .env file`
@ -56,21 +63,110 @@ function loadEnvironment(path) {
// true is the default value passed by commander // true is the default value passed by commander
async function getConfig(envFile = true) { async function getConfig(envFile = true) {
let config
if (envFile !== true) { if (envFile !== true) {
return loadEnvironment(envFile) config = loadEnvironment(envFile)
} else { } else {
return askQuestions() config = askQuestions()
} }
for (let required of REQUIRED) {
env._set(required.value, config[required.value])
}
return config
} }
async function exportBackup(envFile) { function replication(from, to) {
const config = await getConfig(envFile) return new Promise((resolve, reject) => {
console.log(config) from.replicate
.to(to)
.on("complete", () => {
resolve()
})
.on("error", err => {
reject(err)
})
})
} }
async function importBackup(envFile) { function getPouches() {
const config = await getConfig(envFile) const Remote = getPouch({ replication: true })
console.log(config) const Local = getPouch({ onDisk: true, directory: TEMP_DIR })
return { Remote, Local }
}
async function exportBackup(opts) {
const envFile = opts.env || undefined
await getConfig(envFile)
let filename = opts["export"] || opts
if (typeof filename !== "string") {
filename = `backup-${new Date().toISOString()}.tar.gz`
}
await getConfig(envFile)
const dbList = await getAllDbs()
const { Remote, Local } = getPouches()
if (fs.existsSync(TEMP_DIR)) {
fs.rmSync(TEMP_DIR, { recursive: true })
}
const couchDir = join(TEMP_DIR, "couchdb")
fs.mkdirSync(TEMP_DIR)
fs.mkdirSync(couchDir)
const bar = progressBar(dbList.length)
let count = 0
for (let db of dbList) {
bar.update(++count)
const remote = new Remote(db)
const local = new Local(join(TEMP_DIR, "couchdb", db))
await replication(remote, local)
}
bar.stop()
tar.create(
{
sync: true,
gzip: true,
file: filename,
cwd: join(TEMP_DIR),
},
["couchdb"]
)
fs.rmSync(TEMP_DIR, { recursive: true })
console.log(`Generated export file - ${filename}`)
}
async function importBackup(opts) {
const envFile = opts.env || undefined
const filename = opts["import"] || opts
await getConfig(envFile)
if (!filename || !fs.existsSync(filename)) {
console.error("Cannot import without specifying a valid file to import")
process.exit(-1)
}
fs.mkdirSync(TEMP_DIR)
tar.extract({
sync: true,
cwd: join(TEMP_DIR),
file: filename,
})
const { Remote, Local } = getPouches()
const dbList = fs.readdirSync(join(TEMP_DIR, "couchdb"))
const bar = progressBar(dbList.length)
let count = 0
for (let db of dbList) {
bar.update(++count)
const remote = new Remote(db)
const local = new Local(join(TEMP_DIR, "couchdb", db))
await replication(local, remote)
}
bar.stop()
console.log("Import complete")
fs.rmSync(TEMP_DIR, { recursive: true })
}
async function pickOne(opts) {
if (opts["import"]) {
return importBackup(opts)
} else if (opts["export"]) {
return exportBackup(opts)
}
} }
const command = new Command(`${CommandWords.BACKUPS}`) const command = new Command(`${CommandWords.BACKUPS}`)
@ -78,14 +174,19 @@ const command = new Command(`${CommandWords.BACKUPS}`)
"Allows building backups of Budibase, as well as importing a backup to a new instance." "Allows building backups of Budibase, as well as importing a backup to a new instance."
) )
.addSubOption( .addSubOption(
"--export [envFile]", "--export [filename]",
"Export a backup from an existing Budibase installation.", "Export a backup from an existing Budibase installation.",
exportBackup exportBackup
) )
.addSubOption( .addSubOption(
"--import [envFile]", "--import [filename]",
"Import a backup to a new Budibase installation.", "Import a backup to a new Budibase installation.",
importBackup importBackup
) )
.addSubOption(
"--env [envFile]",
"Provide an environment variable file to configure the CLI.",
pickOne
)
exports.command = command exports.command = command

6
packages/cli/src/structures/Command.js
View File

@ -39,8 +39,10 @@ class Command {
let executed = false let executed = false
for (let opt of thisCmd.opts) { for (let opt of thisCmd.opts) {
const lookup = opt.command.split(" ")[0].replace("--", "") const lookup = opt.command.split(" ")[0].replace("--", "")
if (options[lookup]) { if (!executed && options[lookup]) {
await opt.func(options[lookup]) const input =
Object.keys(options).length > 1 ? options : options[lookup]
await opt.func(input)
executed = true executed = true
} }
} }

7
packages/cli/src/utils.js
View File

@ -2,6 +2,7 @@ const chalk = require("chalk")
const fs = require("fs") const fs = require("fs")
const axios = require("axios") const axios = require("axios")
const path = require("path") const path = require("path")
const progress = require("cli-progress")
exports.downloadFile = async (url, filePath) => { exports.downloadFile = async (url, filePath) => {
filePath = path.resolve(filePath) filePath = path.resolve(filePath)
@ -56,3 +57,9 @@ exports.parseEnv = env => {
} }
return result return result
} }
exports.progressBar = total => {
const bar = new progress.SingleBar({}, progress.Presets.shades_classic)
bar.start(total, 0)
return bar
}

1194
packages/cli/yarn.lock
View File

File diff suppressed because it is too large Load Diff

28
packages/server/yarn.lock
View File

@ -1094,12 +1094,12 @@
resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw== integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==
"@budibase/backend-core@1.0.212-alpha.0": "@budibase/backend-core@1.0.212-alpha.7":
version "1.0.212-alpha.0" version "1.0.212-alpha.7"
resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.212-alpha.0.tgz#6ac3448c8272e918f1af1fff0cf8c5773ae61219" resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.212-alpha.7.tgz#33502474c79f1b417a0456ced68ea437eb6badd3"
integrity sha512-hFvbQQEbF3w2u9fe/S+RhNw5HUETS6rhu9q5KDTDQ57k05D4YMPcpMBGSh7SPMqmVyEwUDgcL36mkFOc3AgjYQ== integrity sha512-aQWQbRasLwk0+lVGUp7Yzc4Yui/v9J02Al8SLhv8NDF81JNLmEgGKD/uo8hY+eseuNZe4rLCAoHkoP+0PeHNVQ==
dependencies: dependencies:
"@budibase/types" "^1.0.212-alpha.0" "@budibase/types" "^1.0.212-alpha.7"
"@techpass/passport-openidconnect" "0.3.2" "@techpass/passport-openidconnect" "0.3.2"
aws-sdk "2.1030.0" aws-sdk "2.1030.0"
bcrypt "5.0.1" bcrypt "5.0.1"
@ -1176,12 +1176,12 @@
svelte-flatpickr "^3.2.3" svelte-flatpickr "^3.2.3"
svelte-portal "^1.0.0" svelte-portal "^1.0.0"
"@budibase/pro@1.0.212-alpha.0": "@budibase/pro@1.0.212-alpha.7":
version "1.0.212-alpha.0" version "1.0.212-alpha.7"
resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.212-alpha.0.tgz#843f8291fcc0a2fbcb4d857a750bc116cdaee293" resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.212-alpha.7.tgz#c2bdddcd3823f361126a47943c3207dbf213377e"
integrity sha512-4nhWxjMcxSQBPXRy/U+37IaVLYOr4/RVe79/fUvnXrr5qAeecbEk/QbkJJd3dU1WaNxB2eGhNtH3uBUPQvcT9A== integrity sha512-EMF3a9KTuNJ506EKeJpOLQug0KkuryN952SIJ/+Q22cKaBAvo4NDal7w4rRvld9iQVLBhKhqHwuJ+l948WWffA==
dependencies: dependencies:
"@budibase/backend-core" "1.0.212-alpha.0" "@budibase/backend-core" "1.0.212-alpha.7"
node-fetch "^2.6.1" node-fetch "^2.6.1"
"@budibase/standard-components@^0.9.139": "@budibase/standard-components@^0.9.139":
@ -1202,10 +1202,10 @@
svelte-apexcharts "^1.0.2" svelte-apexcharts "^1.0.2"
svelte-flatpickr "^3.1.0" svelte-flatpickr "^3.1.0"
"@budibase/types@^1.0.212-alpha.0": "@budibase/types@^1.0.212-alpha.7":
version "1.0.212" version "1.0.214"
resolved "https://registry.yarnpkg.com/@budibase/types/-/types-1.0.212.tgz#e66a15b711544b4fab7767261fd5f2f1dd7f40d7" resolved "https://registry.yarnpkg.com/@budibase/types/-/types-1.0.214.tgz#400724fb23a1ce4b80bd04ed472a376d59126069"
integrity sha512-DhGyw6snwJQZQlx7havVYnqPZfZERueKZfmVCBySzwInZZt0+sXZaBl1BVjGjYuwpaUQBMDBf7geBgHXp6DIKg== integrity sha512-KD9XkCdL+R4Nal/X2IX0Bv+AxsylGzAAmo7M6tuoM7YAegskmsX5Kfk5yasnbMnhOdICyvbJ1D6Hz7WKwdApzw==
"@bull-board/api@3.7.0": "@bull-board/api@3.7.0":
version "3.7.0" version "3.7.0"

28
packages/worker/yarn.lock
View File

@ -291,12 +291,12 @@
resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw== integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==
"@budibase/backend-core@1.0.212-alpha.0": "@budibase/backend-core@1.0.212-alpha.7":
version "1.0.212-alpha.0" version "1.0.212-alpha.7"
resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.212-alpha.0.tgz#6ac3448c8272e918f1af1fff0cf8c5773ae61219" resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.212-alpha.7.tgz#33502474c79f1b417a0456ced68ea437eb6badd3"
integrity sha512-hFvbQQEbF3w2u9fe/S+RhNw5HUETS6rhu9q5KDTDQ57k05D4YMPcpMBGSh7SPMqmVyEwUDgcL36mkFOc3AgjYQ== integrity sha512-aQWQbRasLwk0+lVGUp7Yzc4Yui/v9J02Al8SLhv8NDF81JNLmEgGKD/uo8hY+eseuNZe4rLCAoHkoP+0PeHNVQ==
dependencies: dependencies:
"@budibase/types" "^1.0.212-alpha.0" "@budibase/types" "^1.0.212-alpha.7"
"@techpass/passport-openidconnect" "0.3.2" "@techpass/passport-openidconnect" "0.3.2"
aws-sdk "2.1030.0" aws-sdk "2.1030.0"
bcrypt "5.0.1" bcrypt "5.0.1"
@ -323,18 +323,18 @@
uuid "8.3.2" uuid "8.3.2"
zlib "1.0.5" zlib "1.0.5"
"@budibase/pro@1.0.212-alpha.0": "@budibase/pro@1.0.212-alpha.7":
version "1.0.212-alpha.0" version "1.0.212-alpha.7"
resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.212-alpha.0.tgz#843f8291fcc0a2fbcb4d857a750bc116cdaee293" resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.212-alpha.7.tgz#c2bdddcd3823f361126a47943c3207dbf213377e"
integrity sha512-4nhWxjMcxSQBPXRy/U+37IaVLYOr4/RVe79/fUvnXrr5qAeecbEk/QbkJJd3dU1WaNxB2eGhNtH3uBUPQvcT9A== integrity sha512-EMF3a9KTuNJ506EKeJpOLQug0KkuryN952SIJ/+Q22cKaBAvo4NDal7w4rRvld9iQVLBhKhqHwuJ+l948WWffA==
dependencies: dependencies:
"@budibase/backend-core" "1.0.212-alpha.0" "@budibase/backend-core" "1.0.212-alpha.7"
node-fetch "^2.6.1" node-fetch "^2.6.1"
"@budibase/types@^1.0.212-alpha.0": "@budibase/types@^1.0.212-alpha.7":
version "1.0.212" version "1.0.214"
resolved "https://registry.yarnpkg.com/@budibase/types/-/types-1.0.212.tgz#e66a15b711544b4fab7767261fd5f2f1dd7f40d7" resolved "https://registry.yarnpkg.com/@budibase/types/-/types-1.0.214.tgz#400724fb23a1ce4b80bd04ed472a376d59126069"
integrity sha512-DhGyw6snwJQZQlx7havVYnqPZfZERueKZfmVCBySzwInZZt0+sXZaBl1BVjGjYuwpaUQBMDBf7geBgHXp6DIKg== integrity sha512-KD9XkCdL+R4Nal/X2IX0Bv+AxsylGzAAmo7M6tuoM7YAegskmsX5Kfk5yasnbMnhOdICyvbJ1D6Hz7WKwdApzw==
"@cspotcode/source-map-consumer@0.8.0": "@cspotcode/source-map-consumer@0.8.0":
version "0.8.0" version "0.8.0"