Merge pull request #2815 from Budibase/feature/cloud-export

Cloud export -> self host import

packages/auth/src/cache/user.js

@@ -12,7 +12,7 @@ const populateFromDB = async (userId, tenantId) => {
   const user = await getGlobalDB(tenantId).get(userId)
   user.budibaseAccess = true
 
-  if (!env.SELF_HOSTED) {
+  if (!env.SELF_HOSTED && !env.DISABLE_ACCOUNT_PORTAL) {
     const account = await accounts.getAccount(user.email)
     if (account) {
       user.account = account

packages/auth/src/environment.js

@@ -21,6 +21,7 @@ module.exports = {
   INTERNAL_API_KEY: process.env.INTERNAL_API_KEY,
   MULTI_TENANCY: process.env.MULTI_TENANCY,
   ACCOUNT_PORTAL_URL: process.env.ACCOUNT_PORTAL_URL,
+  DISABLE_ACCOUNT_PORTAL: process.env.DISABLE_ACCOUNT_PORTAL,
   SELF_HOSTED: !!parseInt(process.env.SELF_HOSTED),
   COOKIE_DOMAIN: process.env.COOKIE_DOMAIN,
   isTest,

packages/auth/src/tenancy/tenancy.js

@@ -73,7 +73,7 @@ exports.tryAddTenant = async (tenantId, userId, email) => {
   await Promise.all(promises)
 }
 
-exports.getGlobalDB = (tenantId = null) => {
+exports.getGlobalDBName = (tenantId = null) => {
   // tenant ID can be set externally, for example user API where
   // new tenants are being created, this may be the case
   if (!tenantId) {
@@ -81,13 +81,16 @@ exports.getGlobalDB = (tenantId = null) => {
   }
 
   let dbName
-
   if (tenantId === DEFAULT_TENANT_ID) {
     dbName = StaticDatabases.GLOBAL.name
   } else {
     dbName = `${tenantId}${SEPARATOR}${StaticDatabases.GLOBAL.name}`
   }
+  return dbName
+}
 
+exports.getGlobalDB = (tenantId = null) => {
+  const dbName = exports.getGlobalDBName(tenantId)
   return getDB(dbName)
 }
 

packages/builder/src/pages/builder/_layout.svelte

@@ -4,6 +4,9 @@
   import { onMount } from "svelte"
 
   let loaded = false
+  // don't react to these
+  let cloud = $admin.cloud
+  let shouldRedirect = !cloud || $admin.disableAccountPortal
 
   $: multiTenancyEnabled = $admin.multiTenancy
   $: hasAdminUser = $admin?.checklist?.adminUser?.checked
@@ -39,30 +42,35 @@
 
   $: {
     // We should never see the org or admin user creation screens in the cloud
-    if (!cloud) {
-      const apiReady = $admin.loaded && $auth.loaded
-      // if tenant is not set go to it
-      if (loaded && apiReady && multiTenancyEnabled && !tenantSet) {
-        $redirect("./auth/org")
-      }
-      // Force creation of an admin user if one doesn't exist
-      else if (loaded && apiReady && !hasAdminUser) {
-        $redirect("./admin")
-      }
-    }
-  }
-  // Redirect to log in at any time if the user isn't authenticated
-  $: {
+    const apiReady = $admin.loaded && $auth.loaded
+    // if tenant is not set go to it
     if (
+      loaded &&
+      shouldRedirect &&
+      apiReady &&
+      multiTenancyEnabled &&
+      !tenantSet
+    ) {
+      $redirect("./auth/org")
+    }
+    // Force creation of an admin user if one doesn't exist
+    else if (loaded && shouldRedirect && apiReady && !hasAdminUser) {
+      $redirect("./admin")
+    }
+    // Redirect to log in at any time if the user isn't authenticated
+    else if (
       loaded &&
       (hasAdminUser || cloud) &&
       !$auth.user &&
       !$isActive("./auth") &&
-      !$isActive("./invite")
+      !$isActive("./invite") &&
+      !$isActive("./admin")
     ) {
       const returnUrl = encodeURIComponent(window.location.pathname)
       $redirect("./auth?", { returnUrl })
-    } else if ($auth?.user?.forceResetPassword) {
+    }
+    // check if password reset required for user
+    else if ($auth.user?.forceResetPassword) {
       $redirect("./auth/reset")
     }
   }

packages/builder/src/pages/builder/admin/_components/ImportAppsModal.svelte

@@ -0,0 +1,50 @@
+<script>
+  import { notifications, ModalContent, Dropzone, Body } from "@budibase/bbui"
+  import { post } from "builderStore/api"
+
+  let submitting = false
+
+  $: value = { file: null }
+
+  async function importApps() {
+    submitting = true
+
+    try {
+      // Create form data to create app
+      let data = new FormData()
+      data.append("importFile", value.file)
+
+      // Create App
+      const importResp = await post("/api/cloud/import", data, {})
+      const importJson = await importResp.json()
+      if (!importResp.ok) {
+        throw new Error(importJson.message)
+      }
+      // now reload to get to login
+      window.location.reload()
+    } catch (error) {
+      notifications.error(error)
+      submitting = false
+    }
+  }
+</script>
+
+<ModalContent
+  title="Import apps"
+  confirmText="Import apps"
+  onConfirm={importApps}
+  disabled={!value.file}
+>
+  <Body
+    >Please upload the file that was exported from your Cloud environment to get
+    started</Body
+  >
+  <Dropzone
+    gallery={false}
+    label="File to import"
+    value={[value.file]}
+    on:change={e => {
+      value.file = e.detail?.[0]
+    }}
+  />
+</ModalContent>

packages/builder/src/pages/builder/admin/index.svelte

@@ -7,18 +7,22 @@
     Input,
     Body,
     ActionButton,
+    Modal,
   } from "@budibase/bbui"
   import { goto } from "@roxi/routify"
   import api from "builderStore/api"
   import { admin, auth } from "stores/portal"
   import PasswordRepeatInput from "components/common/users/PasswordRepeatInput.svelte"
+  import ImportAppsModal from "./_components/ImportAppsModal.svelte"
   import Logo from "assets/bb-emblem.svg"
 
   let adminUser = {}
   let error
+  let modal
 
   $: tenantId = $auth.tenantId
   $: multiTenancyEnabled = $admin.multiTenancy
+  $: cloud = $admin.cloud
 
   async function save() {
     try {
@@ -38,6 +42,9 @@
   }
 </script>
 
+<Modal bind:this={modal} padding={false} width="600px">
+  <ImportAppsModal />
+</Modal>
 <section>
   <div class="container">
     <Layout>
@@ -66,6 +73,15 @@
           >
             Change organisation
           </ActionButton>
+        {:else if !cloud}
+          <ActionButton
+            quiet
+            on:click={() => {
+              modal.show()
+            }}
+          >
+            Import from cloud
+          </ActionButton>
         {/if}
       </Layout>
     </Layout>

packages/builder/src/pages/builder/auth/_layout.svelte

@@ -9,10 +9,10 @@
     $redirect("../")
   }
 
-  // redirect to account portal for authentication in the cloud
   if (
     !$auth.user &&
     $admin.cloud &&
+    !$admin.disableAccountPortal &&
     $admin.accountPortalUrl &&
     !$admin?.checklist?.sso?.checked
   ) {

packages/builder/src/pages/builder/auth/org.svelte

@@ -9,6 +9,7 @@
   let tenantId = get(auth).tenantSet ? get(auth).tenantId : ""
   $: multiTenancyEnabled = $admin.multiTenancy
   $: cloud = $admin.cloud
+  $: disableAccountPortal = $admin.disableAccountPortal
 
   async function setOrg() {
     if (tenantId == null || tenantId === "") {
@@ -26,7 +27,7 @@
 
   onMount(async () => {
     await auth.checkQueryString()
-    if (!multiTenancyEnabled || cloud) {
+    if (!multiTenancyEnabled || (cloud && !disableAccountPortal)) {
       $goto("../")
     } else {
       admin.unload()

packages/builder/src/pages/builder/index.svelte

@@ -5,11 +5,9 @@
   auth.checkQueryString()
 
   $: {
-    if (!$auth.user) {
-      $redirect(`./auth`)
-    } else if ($auth.user.builder?.global) {
+    if ($auth.user?.builder?.global) {
       $redirect(`./portal`)
-    } else {
+    } else if ($auth.user) {
       $redirect(`./apps`)
     }
   }

packages/builder/src/pages/builder/portal/apps/index.svelte

@@ -35,6 +35,7 @@
   let unpublishModal
   let creatingApp = false
   let loaded = false
+  let cloud = $admin.cloud
 
   $: enrichedApps = enrichApps($apps, $auth.user, sortBy)
 
@@ -70,6 +71,15 @@
     creatingApp = true
   }
 
+  const initiateAppsExport = () => {
+    try {
+      download(`/api/cloud/export`)
+      notifications.success("Apps exported successfully")
+    } catch (err) {
+      notifications.error(`Error exporting apps: ${err}`)
+    }
+  }
+
   const initiateAppImport = () => {
     template = { fromFile: true }
     creationModal.show()
@@ -190,6 +200,9 @@
       <div class="title">
         <Heading>Apps</Heading>
         <ButtonGroup>
+          {#if cloud}
+            <Button secondary on:click={initiateAppsExport}>Export apps</Button>
+          {/if}
           <Button secondary on:click={initiateAppImport}>Import app</Button>
           <Button cta on:click={initiateAppCreation}>Create app</Button>
         </ButtonGroup>

packages/builder/src/stores/portal/admin.js

@@ -7,6 +7,7 @@ export function createAdminStore() {
     loaded: false,
     multiTenancy: false,
     cloud: false,
+    disableAccountPortal: false,
     accountPortalUrl: "",
     onboardingProgress: 0,
     checklist: {
@@ -47,12 +48,14 @@ export function createAdminStore() {
   async function getEnvironment() {
     let multiTenancyEnabled = false
     let cloud = false
+    let disableAccountPortal = false
     let accountPortalUrl = ""
     try {
       const response = await api.get(`/api/system/environment`)
       const json = await response.json()
       multiTenancyEnabled = json.multiTenancy
       cloud = json.cloud
+      disableAccountPortal = json.disableAccountPortal
       accountPortalUrl = json.accountPortalUrl
     } catch (err) {
       // just let it stay disabled
@@ -60,6 +63,7 @@ export function createAdminStore() {
     admin.update(store => {
       store.multiTenancy = multiTenancyEnabled
       store.cloud = cloud
+      store.disableAccountPortal = disableAccountPortal
       store.accountPortalUrl = accountPortalUrl
       return store
     })

packages/server/package.json

@@ -96,6 +96,7 @@
     "koa-session": "5.12.0",
     "koa-static": "5.0.0",
     "lodash": "4.17.21",
+    "memorystream": "^0.3.1",
     "mongodb": "3.6.3",
     "mssql": "6.2.3",
     "mysql": "2.18.1",

packages/server/scripts/dev/manage.js

@@ -37,7 +37,7 @@ async function init() {
     const envFileJson = {
       PORT: 4001,
       MINIO_URL: "http://localhost:10000/",
-      COUCH_DB_URL: "http://@localhost:10000/db/",
+      COUCH_DB_URL: "http://budibase:budibase@localhost:10000/db/",
       REDIS_URL: "localhost:6379",
       WORKER_URL: "http://localhost:4002",
       INTERNAL_API_KEY: "budibase",
@@ -48,6 +48,7 @@ async function init() {
       COUCH_DB_PASSWORD: "budibase",
       COUCH_DB_USER: "budibase",
       SELF_HOSTED: 1,
+      DISABLE_ACCOUNT_PORTAL: "",
       MULTI_TENANCY: "",
     }
     let envFile = ""

packages/server/src/api/controllers/application.js

@@ -31,7 +31,7 @@ const {
   getDeployedApps,
   removeAppFromUserRoles,
 } = require("../../utilities/workerRequests")
-const { clientLibraryPath } = require("../../utilities")
+const { clientLibraryPath, stringToReadStream } = require("../../utilities")
 const { getAllLocks } = require("../../utilities/redis")
 const {
   updateClientLibrary,
@@ -114,8 +114,13 @@ async function createInstance(template) {
 
   // replicate the template data to the instance DB
   // this is currently very hard to test, downloading and importing template files
-  /* istanbul ignore next */
-  if (template && template.useTemplate === "true") {
+  if (template && template.templateString) {
+    const { ok } = await db.load(stringToReadStream(template.templateString))
+    if (!ok) {
+      throw "Error loading database dump from memory."
+    }
+  } else if (template && template.useTemplate === "true") {
+    /* istanbul ignore next */
     const { ok } = await db.load(await getTemplateStream(template))
     if (!ok) {
       throw "Error loading database dump from template."
@@ -191,10 +196,11 @@ exports.fetchAppPackage = async function (ctx) {
 }
 
 exports.create = async function (ctx) {
-  const { useTemplate, templateKey } = ctx.request.body
+  const { useTemplate, templateKey, templateString } = ctx.request.body
   const instanceConfig = {
     useTemplate,
     key: templateKey,
+    templateString,
   }
   if (ctx.request.files && ctx.request.files.templateFile) {
     instanceConfig.file = ctx.request.files.templateFile

packages/server/src/api/controllers/cloud.js

@@ -0,0 +1,92 @@
+const env = require("../../environment")
+const { getAllApps } = require("@budibase/auth/db")
+const CouchDB = require("../../db")
+const {
+  exportDB,
+  sendTempFile,
+  readFileSync,
+} = require("../../utilities/fileSystem")
+const { stringToReadStream } = require("../../utilities")
+const { getGlobalDBName, getGlobalDB } = require("@budibase/auth/tenancy")
+const { create } = require("./application")
+const { getDocParams, DocumentTypes, isDevAppID } = require("../../db/utils")
+
+async function createApp(appName, appImport) {
+  const ctx = {
+    request: {
+      body: {
+        templateString: appImport,
+        name: appName,
+      },
+    },
+  }
+  return create(ctx)
+}
+
+exports.exportApps = async ctx => {
+  if (env.SELF_HOSTED || !env.MULTI_TENANCY) {
+    ctx.throw(400, "Exporting only allowed in multi-tenant cloud environments.")
+  }
+  const apps = await getAllApps(CouchDB, { all: true })
+  const globalDBString = await exportDB(getGlobalDBName())
+  let allDBs = {
+    global: globalDBString,
+  }
+  for (let app of apps) {
+    // only export the dev apps as they will be the latest, the user can republish the apps
+    // in their self hosted environment
+    if (isDevAppID(app._id)) {
+      allDBs[app.name] = await exportDB(app._id)
+    }
+  }
+  const filename = `cloud-export-${new Date().getTime()}.txt`
+  ctx.attachment(filename)
+  ctx.body = sendTempFile(JSON.stringify(allDBs))
+}
+
+async function getAllDocType(db, docType) {
+  const response = await db.allDocs(
+    getDocParams(docType, null, {
+      include_docs: true,
+    })
+  )
+  return response.rows.map(row => row.doc)
+}
+
+exports.importApps = async ctx => {
+  if (!env.SELF_HOSTED || env.MULTI_TENANCY) {
+    ctx.throw(400, "Importing only allowed in self hosted environments.")
+  }
+  const apps = await getAllApps(CouchDB, { all: true })
+  if (
+    apps.length !== 0 ||
+    !ctx.request.files ||
+    !ctx.request.files.importFile
+  ) {
+    ctx.throw(
+      400,
+      "Import file is required and environment must be fresh to import apps."
+    )
+  }
+  const importFile = ctx.request.files.importFile
+  const importString = readFileSync(importFile.path)
+  const dbs = JSON.parse(importString)
+  const globalDbImport = dbs.global
+  // remove from the list of apps
+  delete dbs.global
+  const globalDb = getGlobalDB()
+  // load the global db first
+  await globalDb.load(stringToReadStream(globalDbImport))
+  for (let [appName, appImport] of Object.entries(dbs)) {
+    await createApp(appName, appImport)
+  }
+  // once apps are created clean up the global db
+  let users = await getAllDocType(globalDb, DocumentTypes.USER)
+  for (let user of users) {
+    delete user.tenantId
+  }
+  await globalDb.bulkDocs(users)
+  ctx.body = {
+    message: "Apps successfully imported.",
+  }
+}

packages/server/src/api/controllers/row/internal.js

@@ -5,7 +5,6 @@ const {
   generateRowID,
   DocumentTypes,
   InternalTables,
-  generateMemoryViewID,
 } = require("../../../db/utils")
 const userController = require("../user")
 const {
@@ -20,7 +19,12 @@ const { fullSearch, paginatedSearch } = require("./internalSearch")
 const { getGlobalUsersFromMetadata } = require("../../../utilities/global")
 const inMemoryViews = require("../../../db/inMemoryView")
 const env = require("../../../environment")
-const { migrateToInMemoryView } = require("../view/utils")
+const {
+  migrateToInMemoryView,
+  migrateToDesignView,
+  getFromDesignDoc,
+  getFromMemoryDoc,
+} = require("../view/utils")
 
 const CALCULATION_TYPES = {
   SUM: "sum",
@@ -74,33 +78,24 @@ async function getRawTableData(ctx, db, tableId) {
 }
 
 async function getView(db, viewName) {
-  let viewInfo
-  async function getFromDesignDoc() {
-    const designDoc = await db.get("_design/database")
-    viewInfo = designDoc.views[viewName]
-    return viewInfo
-  }
-  let migrate = false
-  if (env.SELF_HOSTED) {
-    viewInfo = await getFromDesignDoc()
-  } else {
-    try {
-      viewInfo = await db.get(generateMemoryViewID(viewName))
-      if (viewInfo) {
-        viewInfo = viewInfo.view
-      }
-    } catch (err) {
-      // check if it can be retrieved from design doc (needs migrated)
-      if (err.status !== 404) {
-        viewInfo = null
-      } else {
-        viewInfo = await getFromDesignDoc()
-        migrate = !!viewInfo
-      }
+  let mainGetter = env.SELF_HOSTED ? getFromDesignDoc : getFromMemoryDoc
+  let secondaryGetter = env.SELF_HOSTED ? getFromMemoryDoc : getFromDesignDoc
+  let migration = env.SELF_HOSTED ? migrateToDesignView : migrateToInMemoryView
+  let viewInfo,
+    migrate = false
+  try {
+    viewInfo = await mainGetter(db, viewName)
+  } catch (err) {
+    // check if it can be retrieved from design doc (needs migrated)
+    if (err.status !== 404) {
+      viewInfo = null
+    } else {
+      viewInfo = await secondaryGetter(db, viewName)
+      migrate = !!viewInfo
     }
   }
   if (migrate) {
-    await migrateToInMemoryView(db, viewName)
+    await migration(db, viewName)
   }
   if (!viewInfo) {
     throw "View does not exist."

packages/server/src/api/controllers/view/utils.js

@@ -107,3 +107,30 @@ exports.migrateToInMemoryView = async (db, viewName) => {
   await db.put(designDoc)
   await exports.saveView(db, null, viewName, view)
 }
+
+exports.migrateToDesignView = async (db, viewName) => {
+  let view = await db.get(generateMemoryViewID(viewName))
+  const designDoc = await db.get("_design/database")
+  designDoc.views[viewName] = view.view
+  await db.put(designDoc)
+  await db.remove(view._id, view._rev)
+}
+
+exports.getFromDesignDoc = async (db, viewName) => {
+  const designDoc = await db.get("_design/database")
+  let view = designDoc.views[viewName]
+  if (view == null) {
+    throw { status: 404, message: "Unable to get view" }
+  }
+  return view
+}
+
+exports.getFromMemoryDoc = async (db, viewName) => {
+  let view = await db.get(generateMemoryViewID(viewName))
+  if (view) {
+    view = view.view
+  } else {
+    throw { status: 404, message: "Unable to get view" }
+  }
+  return view
+}

packages/server/src/api/routes/cloud.js

@@ -0,0 +1,13 @@
+const Router = require("@koa/router")
+const controller = require("../controllers/cloud")
+const authorized = require("../../middleware/authorized")
+const { BUILDER } = require("@budibase/auth/permissions")
+
+const router = Router()
+
+router
+  .get("/api/cloud/export", authorized(BUILDER), controller.exportApps)
+  // has to be public, only run if apps don't exist
+  .post("/api/cloud/import", controller.importApps)
+
+module.exports = router

packages/server/src/api/routes/index.js

@@ -24,6 +24,7 @@ const hostingRoutes = require("./hosting")
 const backupRoutes = require("./backup")
 const metadataRoutes = require("./metadata")
 const devRoutes = require("./dev")
+const cloudRoutes = require("./cloud")
 
 exports.mainRoutes = [
   authRoutes,
@@ -49,6 +50,7 @@ exports.mainRoutes = [
   backupRoutes,
   metadataRoutes,
   devRoutes,
+  cloudRoutes,
   // these need to be handled last as they still use /api/:tableId
   // this could be breaking as koa may recognise other routes as this
   tableRoutes,

packages/server/src/api/routes/tests/row.spec.js

@@ -317,7 +317,7 @@ describe("/rows", () => {
       await request
         .get(`/api/views/derp`)
         .set(config.defaultHeaders())
-        .expect(400)
+        .expect(404)
     })
 
     it("should be able to run on a view", async () => {
@@ -394,4 +394,4 @@ describe("/rows", () => {
       })
     })
   })
-})
+})

packages/server/src/db/utils.js

@@ -110,6 +110,8 @@ function getDocParams(docType, docId = null, otherProps = {}) {
   }
 }
 
+exports.getDocParams = getDocParams
+
 /**
  * Gets parameters for retrieving tables, this is a utility function for the getDocParams function.
  */

packages/server/src/environment.js

@@ -44,6 +44,7 @@ module.exports = {
   NODE_ENV: process.env.NODE_ENV,
   JEST_WORKER_ID: process.env.JEST_WORKER_ID,
   BUDIBASE_ENVIRONMENT: process.env.BUDIBASE_ENVIRONMENT,
+  DISABLE_ACCOUNT_PORTAL: process.env.DISABLE_ACCOUNT_PORTAL,
   // minor
   SALT_ROUNDS: process.env.SALT_ROUNDS,
   LOGGER: process.env.LOGGER,

packages/server/src/utilities/fileSystem/index.js

@@ -19,6 +19,7 @@ const {
   USER_METDATA_PREFIX,
   LINK_USER_METADATA_PREFIX,
 } = require("../../db/utils")
+const MemoryStream = require("memorystream")
 
 const TOP_LEVEL_PATH = join(__dirname, "..", "..", "..")
 const NODE_MODULES_PATH = join(TOP_LEVEL_PATH, "node_modules")
@@ -111,29 +112,88 @@ exports.apiFileReturn = contents => {
  * to the temporary backup file (to return via API if required).
  */
 exports.performBackup = async (appId, backupName) => {
-  const path = join(budibaseTempDir(), backupName)
-  const writeStream = fs.createWriteStream(path)
-  // perform couch dump
-  const instanceDb = new CouchDB(appId)
-  await instanceDb.dump(writeStream, {
-    // filter out anything that has a user metadata structure in its ID
+  return exports.exportDB(appId, {
+    exportName: backupName,
     filter: doc =>
       !(
         doc._id.includes(USER_METDATA_PREFIX) ||
         doc.includes(LINK_USER_METADATA_PREFIX)
       ),
   })
+}
+
+/**
+ * exports a DB to either file or a variable (memory).
+ * @param {string} dbName the DB which is to be exported.
+ * @param {string} exportName optional - the file name to export to, if not in memory.
+ * @param {function} filter optional - a filter function to clear out any un-wanted docs.
+ * @return Either the file stream or the variable (if no export name provided).
+ */
+exports.exportDB = async (
+  dbName,
+  { exportName, filter } = { exportName: undefined, filter: undefined }
+) => {
+  let stream,
+    appString = "",
+    path = null
+  if (exportName) {
+    path = join(budibaseTempDir(), exportName)
+    stream = fs.createWriteStream(path)
+  } else {
+    stream = new MemoryStream()
+    stream.on("data", chunk => {
+      appString += chunk.toString()
+    })
+  }
+  // perform couch dump
+  const instanceDb = new CouchDB(dbName)
+  await instanceDb.dump(stream, {
+    filter,
+  })
+  // just in memory, return the final string
+  if (!exportName) {
+    return appString
+  }
   // write the file to the object store
   if (env.SELF_HOSTED) {
     await streamUpload(
       ObjectStoreBuckets.BACKUPS,
-      join(appId, backupName),
+      join(dbName, exportName),
       fs.createReadStream(path)
     )
   }
   return fs.createReadStream(path)
 }
 
+/**
+ * Writes the provided contents to a temporary file, which can be used briefly.
+ * @param {string} fileContents contents which will be written to a temp file.
+ * @return {string} the path to the temp file.
+ */
+exports.storeTempFile = fileContents => {
+  const path = join(budibaseTempDir(), uuid())
+  fs.writeFileSync(path, fileContents)
+  return path
+}
+
+/**
+ * Utility function for getting a file read stream - a simple in memory buffered read
+ * stream doesn't work for pouchdb.
+ */
+exports.stringToFileStream = contents => {
+  const path = exports.storeTempFile(contents)
+  return fs.createReadStream(path)
+}
+
+/**
+ * Creates a temp file and returns it from the API.
+ * @param {string} fileContents the contents to be returned in file.
+ */
+exports.sendTempFile = fileContents => {
+  const path = exports.storeTempFile(fileContents)
+  return fs.createReadStream(path)
+}
+
 /**
  * Uploads the latest client library to the object store.
  * @param {string} appId The ID of the app which is being created.

packages/server/src/utilities/index.js

@@ -3,6 +3,7 @@ const { OBJ_STORE_DIRECTORY } = require("../constants")
 const { sanitizeKey } = require("@budibase/auth/src/objectStore")
 const CouchDB = require("../db")
 const { generateMetadataID } = require("../db/utils")
+const Readable = require("stream").Readable
 
 const BB_CDN = "https://cdn.budi.live"
 
@@ -124,3 +125,12 @@ exports.escapeDangerousCharacters = string => {
     .replace(/[\r]/g, "\\r")
     .replace(/[\t]/g, "\\t")
 }
+
+exports.stringToReadStream = string => {
+  return new Readable({
+    read() {
+      this.push(string)
+      this.push(null)
+    },
+  })
+}

packages/worker/scripts/dev/manage.js

@@ -21,6 +21,7 @@ async function init() {
       COUCH_DB_PASSWORD: "budibase",
       // empty string is false
       MULTI_TENANCY: "",
+      DISABLE_ACCOUNT_PORTAL: "",
       ACCOUNT_PORTAL_URL: "http://localhost:10001",
     }
     let envFile = ""

packages/worker/src/api/controllers/global/users.js

@@ -66,7 +66,7 @@ async function saveUser(
     }
 
     // check root account users in account portal
-    if (!env.SELF_HOSTED) {
+    if (!env.SELF_HOSTED && !env.DISABLE_ACCOUNT_PORTAL) {
       const account = await accounts.getAccount(email)
       if (account && account.verified && account.tenantId !== tenantId) {
         throw `Email address ${email} already in use.`
@@ -133,7 +133,7 @@ exports.save = async ctx => {
 }
 
 const parseBooleanParam = param => {
-  if (param && param == "false") {
+  if (param && param === "false") {
     return false
   } else {
     return true
@@ -161,6 +161,17 @@ exports.adminUser = async ctx => {
 
   // write usage quotas for cloud
   if (!env.SELF_HOSTED) {
+    // could be a scenario where it exists, make sure its clean
+    try {
+      const usageQuota = await db.get(
+        StaticDatabases.PLATFORM_INFO.docs.usageQuota
+      )
+      if (usageQuota) {
+        await db.remove(usageQuota._id, usageQuota._rev)
+      }
+    } catch (err) {
+      // don't worry about errors
+    }
     await db.post(generateNewUsageQuotaDoc())
   }
 

packages/worker/src/api/controllers/system/environment.js

@@ -5,5 +5,6 @@ exports.fetch = async ctx => {
     multiTenancy: !!env.MULTI_TENANCY,
     cloud: !env.SELF_HOSTED,
     accountPortalUrl: env.ACCOUNT_PORTAL_URL,
+    disableAccountPortal: env.DISABLE_ACCOUNT_PORTAL,
   }
 }

packages/worker/src/environment.js

@@ -32,6 +32,7 @@ module.exports = {
   REDIS_PASSWORD: process.env.REDIS_PASSWORD,
   INTERNAL_API_KEY: process.env.INTERNAL_API_KEY,
   MULTI_TENANCY: process.env.MULTI_TENANCY,
+  DISABLE_ACCOUNT_PORTAL: process.env.DISABLE_ACCOUNT_PORTAL,
   ACCOUNT_PORTAL_URL: process.env.ACCOUNT_PORTAL_URL,
   SMTP_FALLBACK_ENABLED: process.env.SMTP_FALLBACK_ENABLED,
   SMTP_USER: process.env.SMTP_USER,