From 57c3b97f0ac24f48ab6dbe777d915cf12deff731 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Fri, 6 Aug 2021 14:59:39 +0100 Subject: [PATCH 1/3] scarf support --- hosting/docker-compose.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hosting/docker-compose.yaml b/hosting/docker-compose.yaml index 92765c29bf..9cdf2b2114 100644 --- a/hosting/docker-compose.yaml +++ b/hosting/docker-compose.yaml @@ -5,7 +5,7 @@ version: "3" services: app-service: restart: always - image: budibase/apps + image: budibase.docker.scarf.sh/budibase/apps container_name: bbapps ports: - "${APP_PORT}:4002" @@ -33,7 +33,7 @@ services: worker-service: restart: always - image: budibase/worker + image: budibase.docker.scarf.sh/budibase/worker container_name: bbworker ports: - "${WORKER_PORT}:4003" From c4f8d17a6ef12742852d339b3f7b61646830dc25 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 6 Aug 2021 16:38:07 +0100 Subject: [PATCH 2/3] Fixes for a lot of issues generated by the use of the pouchdb allDbs call, which is not designed for multi-client DB setups like ours, using CouchDB method instead. --- packages/auth/src/db/utils.js | 24 +++++++++++++++++-- .../middleware/passport/tests/utilities/db.js | 3 --- .../components/start/CreateAppModal.svelte | 3 +++ .../pages/builder/portal/apps/index.svelte | 4 +++- .../builder/src/stores/portal/organisation.js | 2 +- packages/server/scripts/replicateApp.js | 5 ++-- packages/server/src/db/client.js | 16 +------------ .../src/api/controllers/admin/configs.js | 7 ++++-- packages/worker/src/api/controllers/app.js | 9 ++++--- packages/worker/src/db/index.js | 1 + 10 files changed, 43 insertions(+), 31 deletions(-) diff --git a/packages/auth/src/db/utils.js b/packages/auth/src/db/utils.js index 100dc005c8..ae1609c2cb 100644 --- a/packages/auth/src/db/utils.js +++ b/packages/auth/src/db/utils.js @@ -1,5 +1,8 @@ const { newid } = require("../hashing") const Replication = require("./Replication") +const env = require("../environment") +const fetch = require("node-fetch") +const { getCouch } = require("./index") const UNICODE_MAX = "\ufff0" const SEPARATOR = "_" @@ -156,6 +159,23 @@ exports.getDeployedAppID = appId => { return appId } +/** + * if in production this will use the CouchDB _all_dbs call to retrieve a list of databases. If testing + * when using Pouch it will use the pouchdb-all-dbs package. + */ +exports.getAllDbs = async () => { + // specifically for testing we use the pouch package for this + if (env.isTest()) { + return getCouch().allDbs() + } + const response = await fetch(`${env.COUCH_DB_URL}/_all_dbs`) + if (response.status === 200) { + return response.json() + } else { + throw "Cannot connect to CouchDB instance" + } +} + /** * Lots of different points in the system need to find the full list of apps, this will * enumerate the entire CouchDB cluster and get the list of databases (every app). @@ -164,8 +184,8 @@ exports.getDeployedAppID = appId => { * @return {Promise} returns the app information document stored in each app database. */ exports.getAllApps = async ({ CouchDB, dev, all } = {}) => { - let allDbs = await CouchDB.allDbs() - const appDbNames = allDbs.filter(dbName => + let dbs = await exports.getAllDbs() + const appDbNames = dbs.filter(dbName => dbName.startsWith(exports.APP_PREFIX) ) const appPromises = appDbNames.map(db => diff --git a/packages/auth/src/middleware/passport/tests/utilities/db.js b/packages/auth/src/middleware/passport/tests/utilities/db.js index 3a792e3272..e83784471b 100644 --- a/packages/auth/src/middleware/passport/tests/utilities/db.js +++ b/packages/auth/src/middleware/passport/tests/utilities/db.js @@ -1,5 +1,4 @@ const PouchDB = require("pouchdb") -const allDbs = require("pouchdb-all-dbs") const env = require("../../../../environment") let POUCH_DB_DEFAULTS @@ -15,6 +14,4 @@ if (env.isTest()) { const Pouch = PouchDB.defaults(POUCH_DB_DEFAULTS) -allDbs(Pouch) - module.exports = Pouch diff --git a/packages/builder/src/components/start/CreateAppModal.svelte b/packages/builder/src/components/start/CreateAppModal.svelte index 8d2ec56655..93eeec77bb 100644 --- a/packages/builder/src/components/start/CreateAppModal.svelte +++ b/packages/builder/src/components/start/CreateAppModal.svelte @@ -9,6 +9,7 @@ Checkbox, } from "@budibase/bbui" import { store, automationStore, hostingStore } from "builderStore" + import { admin } from "stores/portal" import { string, mixed, object } from "yup" import api, { get, post } from "builderStore/api" import analytics from "analytics" @@ -102,6 +103,8 @@ if (applicationPkg.ok) { await store.actions.initialise(pkg) await automationStore.actions.fetch() + // update checklist - incase first app + await admin.init() } else { throw new Error(pkg) } diff --git a/packages/builder/src/pages/builder/portal/apps/index.svelte b/packages/builder/src/pages/builder/portal/apps/index.svelte index aafe9b60c3..50b49de9da 100644 --- a/packages/builder/src/pages/builder/portal/apps/index.svelte +++ b/packages/builder/src/pages/builder/portal/apps/index.svelte @@ -18,7 +18,7 @@ import api, { del } from "builderStore/api" import analytics from "analytics" import { onMount } from "svelte" - import { apps, auth } from "stores/portal" + import { apps, auth, admin } from "stores/portal" import download from "downloadjs" import { goto } from "@roxi/routify" import ConfirmDialog from "components/common/ConfirmDialog.svelte" @@ -159,6 +159,8 @@ throw json.message } await apps.load() + // get checklist, just in case that was the last app + await admin.init() notifications.success("App deleted successfully") } catch (err) { notifications.error(`Error deleting app: ${err}`) diff --git a/packages/builder/src/stores/portal/organisation.js b/packages/builder/src/stores/portal/organisation.js index 71c0be4b4d..df950b92a0 100644 --- a/packages/builder/src/stores/portal/organisation.js +++ b/packages/builder/src/stores/portal/organisation.js @@ -2,7 +2,7 @@ import { writable, get } from "svelte/store" import api from "builderStore/api" const DEFAULT_CONFIG = { - platformUrl: "http://localhost:1000", + platformUrl: "http://localhost:10000", logoUrl: undefined, docsUrl: undefined, company: "Budibase", diff --git a/packages/server/scripts/replicateApp.js b/packages/server/scripts/replicateApp.js index a27f77b9a0..8b363309ab 100644 --- a/packages/server/scripts/replicateApp.js +++ b/packages/server/scripts/replicateApp.js @@ -7,6 +7,7 @@ const CouchDB = require("../src/db") const { DocumentTypes } = require("../src/db/utils") +const { getAllDbs } = require("@budibase/auth/db") const appName = process.argv[2].toLowerCase() const remoteUrl = process.argv[3] @@ -14,8 +15,8 @@ const remoteUrl = process.argv[3] console.log(`Replicating from ${appName} to ${remoteUrl}/${appName}`) const run = async () => { - const allDbs = await CouchDB.allDbs() - const appDbNames = allDbs.filter(dbName => dbName.startsWith("inst_app")) + const dbs = await getAllDbs() + const appDbNames = dbs.filter(dbName => dbName.startsWith("inst_app")) let apps = [] for (let dbName of appDbNames) { const db = new CouchDB(dbName) diff --git a/packages/server/src/db/client.js b/packages/server/src/db/client.js index fbc25986d5..d50d72b18d 100644 --- a/packages/server/src/db/client.js +++ b/packages/server/src/db/client.js @@ -24,21 +24,7 @@ if (env.isTest()) { const Pouch = PouchDB.defaults(POUCH_DB_DEFAULTS) +// have to still have pouch alldbs for testing allDbs(Pouch) -// replicate your local levelDB pouch to a running HTTP compliant couch or pouchdb server. -/* istanbul ignore next */ -// eslint-disable-next-line no-unused-vars -function replicateLocal() { - Pouch.allDbs().then(dbs => { - for (let db of dbs) { - new Pouch(db).sync( - new PouchDB(`http://127.0.0.1:5984/${db}`, { live: true }) - ) - } - }) -} - -// replicateLocal() - module.exports = Pouch diff --git a/packages/worker/src/api/controllers/admin/configs.js b/packages/worker/src/api/controllers/admin/configs.js index 78caa817b2..956a9e012b 100644 --- a/packages/worker/src/api/controllers/admin/configs.js +++ b/packages/worker/src/api/controllers/admin/configs.js @@ -5,10 +5,13 @@ const { getConfigParams, getGlobalUserParams, getScopedFullConfig, + getAllDbs, } = require("@budibase/auth").db const { Configs } = require("../../../constants") const email = require("../../../utilities/email") const { upload, ObjectStoreBuckets } = require("@budibase/auth").objectStore +const fetch = require("node-fetch") +const env = require("../../../environment") const APP_PREFIX = "app_" @@ -229,8 +232,8 @@ exports.configChecklist = async function (ctx) { // TODO: Watch get started video // Apps exist - let allDbs = await CouchDB.allDbs() - const appDbNames = allDbs.filter(dbName => dbName.startsWith(APP_PREFIX)) + let dbs = await getAllDbs() + const appDbNames = dbs.filter(dbName => dbName.startsWith(APP_PREFIX)) // They have set up SMTP const smtpConfig = await getScopedFullConfig(db, { diff --git a/packages/worker/src/api/controllers/app.js b/packages/worker/src/api/controllers/app.js index ff9692a5ec..e3dbaf5f1a 100644 --- a/packages/worker/src/api/controllers/app.js +++ b/packages/worker/src/api/controllers/app.js @@ -1,15 +1,14 @@ -const { DocumentTypes } = require("@budibase/auth").db +const { DocumentTypes, getAllDbs } = require("@budibase/auth").db const CouchDB = require("../../db") const APP_PREFIX = "app_" const URL_REGEX_SLASH = /\/|\\/g exports.getApps = async ctx => { - // allDbs call of CouchDB is very inaccurate in production - const allDbs = await CouchDB.allDbs() - const appDbNames = allDbs.filter(dbName => dbName.startsWith(APP_PREFIX)) + const dbs = await getAllDbs() + const appDbNames = dbs.filter(dbName => dbName.startsWith(APP_PREFIX)) const appPromises = appDbNames.map(db => - new CouchDB(db).get(DocumentTypes.APP_METADATA) + new CouchDB(db, { skip_setup: true }).get(DocumentTypes.APP_METADATA) ) const apps = await Promise.allSettled(appPromises) diff --git a/packages/worker/src/db/index.js b/packages/worker/src/db/index.js index 1ade1cd6db..d6d035cff1 100644 --- a/packages/worker/src/db/index.js +++ b/packages/worker/src/db/index.js @@ -19,6 +19,7 @@ if (env.isTest()) { const Pouch = PouchDB.defaults(POUCH_DB_DEFAULTS) +// have to still have pouch alldbs for testing allDbs(Pouch) module.exports = Pouch From 5f997ab8361388211daed75f6008c9d13bdb2272 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 6 Aug 2021 16:39:30 +0100 Subject: [PATCH 3/3] Linting. --- packages/auth/src/db/utils.js | 4 +--- packages/worker/src/api/controllers/admin/configs.js | 2 -- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/packages/auth/src/db/utils.js b/packages/auth/src/db/utils.js index ae1609c2cb..1fc78f4182 100644 --- a/packages/auth/src/db/utils.js +++ b/packages/auth/src/db/utils.js @@ -185,9 +185,7 @@ exports.getAllDbs = async () => { */ exports.getAllApps = async ({ CouchDB, dev, all } = {}) => { let dbs = await exports.getAllDbs() - const appDbNames = dbs.filter(dbName => - dbName.startsWith(exports.APP_PREFIX) - ) + const appDbNames = dbs.filter(dbName => dbName.startsWith(exports.APP_PREFIX)) const appPromises = appDbNames.map(db => // skip setup otherwise databases could be re-created new CouchDB(db, { skip_setup: true }).get(DocumentTypes.APP_METADATA) diff --git a/packages/worker/src/api/controllers/admin/configs.js b/packages/worker/src/api/controllers/admin/configs.js index 956a9e012b..0b90d8fb5a 100644 --- a/packages/worker/src/api/controllers/admin/configs.js +++ b/packages/worker/src/api/controllers/admin/configs.js @@ -10,8 +10,6 @@ const { const { Configs } = require("../../../constants") const email = require("../../../utilities/email") const { upload, ObjectStoreBuckets } = require("@budibase/auth").objectStore -const fetch = require("node-fetch") -const env = require("../../../environment") const APP_PREFIX = "app_"