merge

hosting/docker-compose.yaml

@@ -11,13 +11,18 @@ services:
       - "${APP_PORT}:4002"
     environment:
       SELF_HOSTED: 1
-      CLOUD: 1
       COUCH_DB_URL: http://${COUCH_DB_USER}:${COUCH_DB_PASSWORD}@couchdb-service:5984
       WORKER_URL: http://worker-service:4003
+      MINIO_URL: http://minio-service:9000
+      MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY}
+      MINIO_SECRET_KEY: ${MINIO_SECRET_KEY}
       HOSTING_KEY: ${HOSTING_KEY}
       BUDIBASE_ENVIRONMENT: ${BUDIBASE_ENVIRONMENT}
       PORT: 4002
       JWT_SECRET: ${JWT_SECRET}
+      LOG_LEVEL: info
+      SENTRY_DSN: https://a34ae347621946bf8acded18e5b7d4b8@o420233.ingest.sentry.io/5338131
+      ENABLE_ANALYTICS: true
     depends_on:
       - worker-service
 
@@ -28,7 +33,7 @@ services:
     ports:
       - "${WORKER_PORT}:4003"
     environment:
-      SELF_HOSTED: 1,
+      SELF_HOSTED: 1
       PORT: 4003
       MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY}
       MINIO_SECRET_KEY: ${MINIO_SECRET_KEY}

packages/builder/src/components/backend/DatasourceNavigator/popovers/EditDatasourcePopover.svelte

@@ -27,7 +27,7 @@
     notifier.success("Datasource deleted")
     // navigate to first index page if the source you are deleting is selected
     if (wasSelectedSource === datasource._id) {
-      $goto('./datasource')
+      $goto("./datasource")
     }
     hideEditor()
   }

packages/builder/src/components/backend/TableNavigator/popovers/EditTablePopover.svelte

@@ -37,13 +37,13 @@
   }
 
   async function deleteTable() {
-    const wasSelectedTable = $backendUiStore.selectedTable 
+    const wasSelectedTable = $backendUiStore.selectedTable
     await backendUiStore.actions.tables.delete(table)
     store.actions.screens.delete(templateScreens)
     await backendUiStore.actions.tables.fetch()
     notifier.success("Table deleted")
     if (wasSelectedTable._id === table._id) {
-      $goto('./table')
+      $goto("./table")
     }
     hideEditor()
   }

packages/server/.env.template

@@ -1,17 +0,0 @@
-# url of couch db, including username and password
-# http://admin:password@localhost:5984
-COUCH_DB_URL={{couchDbUrl}}
-
-# identifies a client database - i.e. group of apps
-CLIENT_ID={{clientId}}
-
-# used to create cookie hashes
-JWT_SECRET={{cookieKey1}}
-
-# error level for koa-pino
-LOG_LEVEL=info
-
-DEPLOYMENT_CREDENTIALS_URL="https://dt4mpwwap8.execute-api.eu-west-1.amazonaws.com/prod/"
-DEPLOYMENT_DB_URL="https://couchdb.budi.live:5984"
-SENTRY_DSN=https://a34ae347621946bf8acded18e5b7d4b8@o420233.ingest.sentry.io/5338131
-ENABLE_ANALYTICS="true"

packages/server/.gitignore

@@ -1,7 +1,6 @@
 node_modules/
 myapps/
 .env
-dev.env
 /builder/*
 !/builder/assets/
 !/builder/pickr.min.js

packages/server/Dockerfile

@@ -2,7 +2,6 @@ FROM node:12-alpine
 
 WORKDIR /app
 
-ENV CLOUD=1
 ENV PORT=4001
 ENV COUCH_DB_URL=https://couchdb.budi.live:5984
 ENV BUDIBASE_ENVIRONMENT=PRODUCTION

packages/server/package.json

@@ -39,7 +39,7 @@
     "dev:stack:up": "node scripts/dev/manage.js up",
     "dev:stack:down": "node scripts/dev/manage.js down",
     "dev:stack:nuke": "node scripts/dev/manage.js nuke",
-    "dev:builder": "npm run dev:stack:up && env-cmd -f dev.env nodemon src/index.js",
+    "dev:builder": "npm run dev:stack:up && nodemon src/index.js",
     "electron": "electron src/electron.js",
     "build:electron": "electron-builder --dir",
     "publish:electron": "electron-builder -mwl --publish always",
@@ -65,7 +65,8 @@
       "!src/db/tests/**/*",
       "!src/tests/**/*",
       "!src/automations/tests/**/*",
-      "!src/utilities/fileProcessor.js"
+      "!src/utilities/fileProcessor.js",
+      "!src/utilities/fileSystem/**/*"
     ],
     "coverageReporters": [
       "lcov",
@@ -140,7 +141,6 @@
     "electron": "10.1.3",
     "electron-builder": "^22.9.1",
     "electron-builder-notarize": "^1.1.2",
-    "env-cmd": "^10.1.0",
     "eslint": "^6.8.0",
     "jest": "^24.8.0",
     "nodemon": "^2.0.4",

packages/server/scripts/dev/manage.js

@@ -1,6 +1,7 @@
 #!/usr/bin/env node
 const compose = require("docker-compose")
 const path = require("path")
+const fs = require("fs")
 
 // This script wraps docker-compose allowing you to manage your dev infrastructure with simple commands.
 const CONFIG = {
@@ -15,8 +16,33 @@ const Commands = {
   Nuke: "nuke",
 }
 
+async function init() {
+  const envFilePath = path.join(process.cwd(), ".env")
+  if (fs.existsSync(envFilePath)) {
+    return
+  }
+  const envFileJson = {
+    PORT: 4001,
+    MINIO_URL: "http://localhost:10000/",
+    COUCH_DB_URL: "http://budibase:budibase@localhost:10000/db/",
+    WORKER_URL: "http://localhost:4002",
+    JWT_SECRET: "testsecret",
+    MINIO_ACCESS_KEY: "budibase",
+    MINIO_SECRET_KEY: "budibase",
+    COUCH_DB_PASSWORD: "budibase",
+    COUCH_DB_USER: "budibase",
+    SELF_HOSTED: 1,
+  }
+  let envFile = ""
+  Object.keys(envFileJson).forEach(key => {
+    envFile += `${key}=${envFileJson[key]}\n`
+  })
+  fs.writeFileSync(envFilePath, envFile)
+}
+
 async function up() {
   console.log("Spinning up your budibase dev environment... 🔧✨")
+  await init()
   await compose.upAll(CONFIG)
 }
 

packages/server/scripts/exportAppTemplate.js

@@ -1,6 +1,9 @@
 #!/usr/bin/env node
-const { exportTemplateFromApp } = require("../src/utilities/templates")
 const yargs = require("yargs")
+const fs = require("fs")
+const { join } = require("path")
+const CouchDB = require("../src/db")
+const { budibaseAppsDir } = require("../src/utilities/budibaseDir")
 
 // Script to export a chosen budibase app into a package
 // Usage: ./scripts/exportAppTemplate.js export --name=Funky --appId=appId
@@ -22,18 +25,22 @@ yargs
       },
     },
     async args => {
+      const name = args.name,
+        appId = args.appId
       console.log("Exporting app..")
-      if (args.name == null || args.appId == null) {
+      if (name == null || appId == null) {
         console.error(
           "Unable to export without a name and app ID being specified, check help for more info."
         )
         return
       }
-      const exportPath = await exportTemplateFromApp({
+      const exportPath = join(budibaseAppsDir(), "templates", "app", name, "db")
-        templateName: args.name,
+      fs.ensureDirSync(exportPath)
-        appId: args.appId,
+      const writeStream = fs.createWriteStream(join(exportPath, "dump.text"))
-      })
+      // perform couch dump
-      console.log(`Template ${args.name} exported to ${exportPath}`)
+      const instanceDb = new CouchDB(appId)
+      await instanceDb.dump(writeStream, {})
+      console.log(`Template ${name} exported to ${exportPath}`)
     }
   )
   .help()

packages/server/src/api/controllers/apikeys.js

@@ -1,56 +1,32 @@
-const fs = require("fs")
+const builderDB = require("../../db/builder")
-const { join } = require("../../utilities/centralPath")
-const readline = require("readline")
-const { budibaseAppsDir } = require("../../utilities/budibaseDir")
-const env = require("../../environment")
-const ENV_FILE_PATH = "/.env"
 
 exports.fetch = async function(ctx) {
-  ctx.status = 200
+  try {
-  ctx.body = {
+    const mainDoc = await builderDB.getBuilderMainDoc()
-    budibase: env.BUDIBASE_API_KEY,
+    ctx.body = mainDoc.apiKeys ? mainDoc.apiKeys : {}
-    userId: env.USERID_API_KEY,
+  } catch (err) {
+    /* istanbul ignore next */
+    ctx.throw(400, err)
   }
 }
 
 exports.update = async function(ctx) {
-  const key = `${ctx.params.key.toUpperCase()}_API_KEY`
+  const key = ctx.params.key
   const value = ctx.request.body.value
 
-  // set environment variables
+  try {
-  env._set(key, value)
+    const mainDoc = await builderDB.getBuilderMainDoc()
-
+    if (mainDoc.apiKeys == null) {
-  // Write to file
+      mainDoc.apiKeys = {}
-  await updateValues([key, value])
-
-  ctx.status = 200
-  ctx.message = `Updated ${ctx.params.key} API key succesfully.`
-  ctx.body = { [ctx.params.key]: ctx.request.body.value }
-}
-
-async function updateValues([key, value]) {
-  let newContent = ""
-  let keyExists = false
-  let envPath = join(budibaseAppsDir(), ENV_FILE_PATH)
-  const readInterface = readline.createInterface({
-    input: fs.createReadStream(envPath),
-    output: process.stdout,
-    console: false,
-  })
-  readInterface.on("line", function(line) {
-    // Mutate lines and change API Key
-    if (line.startsWith(key)) {
-      line = `${key}=${value}`
-      keyExists = true
     }
-    newContent = `${newContent}\n${line}`
+    mainDoc.apiKeys[key] = value
-  })
+    const resp = await builderDB.setBuilderMainDoc(mainDoc)
-  readInterface.on("close", function() {
+    ctx.body = {
-    // Write file here
+      _id: resp.id,
-    if (!keyExists) {
+      _rev: resp.rev,
-      // Add API Key if it doesn't exist in the file at all
-      newContent = `${newContent}\n${key}=${value}`
     }
-    fs.writeFileSync(envPath, newContent)
+  } catch (err) {
-  })
+    /* istanbul ignore next */
+    ctx.throw(400, err)
+  }
 }

packages/server/src/api/controllers/application.js

@@ -1,15 +1,14 @@
 const CouchDB = require("../../db")
-const compileStaticAssets = require("../../utilities/builder/compileStaticAssets")
 const env = require("../../environment")
-const { existsSync } = require("fs-extra")
-const { budibaseAppsDir } = require("../../utilities/budibaseDir")
 const setBuilderToken = require("../../utilities/builder/setBuilderToken")
-const fs = require("fs-extra")
-const { join, resolve } = require("../../utilities/centralPath")
 const packageJson = require("../../../package.json")
 const { createLinkView } = require("../../db/linkedRows")
 const { createRoutingView } = require("../../utilities/routing")
-const { getTemplateStream } = require("../../utilities/fileSystem")
+const {
+  getTemplateStream,
+  createApp,
+  deleteApp,
+} = require("../../utilities/fileSystem")
 const {
   generateAppID,
   getLayoutParams,
@@ -20,9 +19,6 @@ const {
   BUILTIN_ROLE_IDS,
   AccessController,
 } = require("../../utilities/security/roles")
-const {
-  downloadExtractComponentLibraries,
-} = require("../../utilities/createAppPackage")
 const { BASE_LAYOUTS } = require("../../constants/layouts")
 const {
   createHomeScreen,
@@ -32,11 +28,7 @@ const { cloneDeep } = require("lodash/fp")
 const { processObject } = require("@budibase/string-templates")
 const { getAllApps } = require("../../utilities")
 const { USERS_TABLE_SCHEMA } = require("../../constants")
-const {
+const { getDeployedApps } = require("../../utilities/builder/hosting")
-  getDeployedApps,
-  getHostingInfo,
-  HostingTypes,
-} = require("../../utilities/builder/hosting")
 
 const URL_REGEX_SLASH = /\/|\\/g
 
@@ -75,8 +67,7 @@ async function getAppUrlIfNotInUse(ctx) {
     url = encodeURI(`${ctx.request.body.name}`)
   }
   url = `/${url.replace(URL_REGEX_SLASH, "")}`.toLowerCase()
-  const hostingInfo = await getHostingInfo()
+  if (!env.SELF_HOSTED) {
-  if (hostingInfo.type === HostingTypes.CLOUD) {
     return url
   }
   const deployedApps = await getDeployedApps()
@@ -181,10 +172,10 @@ exports.create = async function(ctx) {
   const instanceDb = new CouchDB(appId)
   await instanceDb.put(newApplication)
 
-  const newAppFolder = await createEmptyAppPackage(ctx, newApplication)
+  await createEmptyAppPackage(ctx, newApplication)
   /* istanbul ignore next */
   if (env.NODE_ENV !== "jest") {
-    await downloadExtractComponentLibraries(newAppFolder)
+    await createApp(appId)
   }
 
   await setBuilderToken(ctx, appId, version)
@@ -214,10 +205,9 @@ exports.delete = async function(ctx) {
   const app = await db.get(ctx.params.appId)
   const result = await db.destroy()
 
-  // remove top level directory
+  if (env.NODE_ENV !== "jest") {
-  await fs.rmdir(join(budibaseAppsDir(), ctx.params.appId), {
+    await deleteApp(ctx.params.appId)
-    recursive: true,
+  }
-  })
 
   ctx.status = 200
   ctx.message = `Application ${app.name} deleted successfully.`
@@ -225,17 +215,8 @@ exports.delete = async function(ctx) {
 }
 
 const createEmptyAppPackage = async (ctx, app) => {
-  const appsFolder = budibaseAppsDir()
-  const newAppFolder = resolve(appsFolder, app._id)
-
   const db = new CouchDB(app._id)
 
-  if (existsSync(newAppFolder)) {
-    ctx.throw(400, "App folder already exists for this application")
-  }
-
-  fs.mkdirpSync(newAppFolder)
-
   let screensAndLayouts = []
   for (let layout of BASE_LAYOUTS) {
     const cloned = cloneDeep(layout)
@@ -251,6 +232,4 @@ const createEmptyAppPackage = async (ctx, app) => {
   screensAndLayouts.push(loginScreen)
 
   await db.bulkDocs(screensAndLayouts)
-  await compileStaticAssets(app._id)
-  return newAppFolder
 }

packages/server/src/api/controllers/auth.js

@@ -45,9 +45,9 @@ exports.authenticate = async ctx => {
       roleId: dbUser.roleId,
       version: app.version,
     }
-    // if in cloud add the user api key, unless self hosted
+    // if in prod add the user api key, unless self hosted
     /* istanbul ignore next */
-    if (env.CLOUD && !env.SELF_HOSTED) {
+    if (env.isProd() && !env.SELF_HOSTED) {
       const { apiKey } = await getAPIKey(ctx.user.appId)
       payload.apiKey = apiKey
     }

packages/server/src/api/controllers/backup.js

@@ -1,28 +1,10 @@
-const { performDump } = require("../../utilities/templates")
+const { performBackup } = require("../../utilities/fileSystem")
-const path = require("path")
-const os = require("os")
-const fs = require("fs-extra")
 
 exports.exportAppDump = async function(ctx) {
   const { appId } = ctx.query
-
   const appname = decodeURI(ctx.query.appname)
-
-  const backupsDir = path.join(os.homedir(), ".budibase", "backups")
-  fs.ensureDirSync(backupsDir)
-
   const backupIdentifier = `${appname}Backup${new Date().getTime()}.txt`
 
-  await performDump({
-    dir: backupsDir,
-    appId,
-    name: backupIdentifier,
-  })
-
-  ctx.status = 200
-
-  const backupFile = path.join(backupsDir, backupIdentifier)
-
   ctx.attachment(backupIdentifier)
-  ctx.body = fs.createReadStream(backupFile)
+  ctx.body = await performBackup(appId, backupIdentifier)
 }

packages/server/src/api/controllers/component.js

@@ -1,44 +1,37 @@
 const CouchDB = require("../../db")
-const { resolve, join } = require("../../utilities/centralPath")
+const { join } = require("../../utilities/centralPath")
-const {
+const { budibaseTempDir } = require("../../utilities/budibaseDir")
-  budibaseTempDir,
+const fileSystem = require("../../utilities/fileSystem")
-  budibaseAppsDir,
+const env = require("../../environment")
-} = require("../../utilities/budibaseDir")
 
 exports.fetchAppComponentDefinitions = async function(ctx) {
   const appId = ctx.params.appId || ctx.appId
   const db = new CouchDB(appId)
   const app = await db.get(appId)
 
-  ctx.body = app.componentLibraries.reduce((acc, componentLibrary) => {
+  let componentManifests = await Promise.all(
-    let appDirectory = resolve(budibaseAppsDir(), appId, "node_modules")
+    app.componentLibraries.map(async library => {
-
+      let manifest
-    if (ctx.isDev) {
+      if (env.isDev()) {
-      appDirectory = budibaseTempDir()
+        manifest = require(join(budibaseTempDir(), library, "manifest.json"))
-    }
+      } else {
-
+        manifest = await fileSystem.getComponentLibraryManifest(appId, library)
-    const componentJson = require(join(
+      }
-      appDirectory,
+      return {
-      componentLibrary,
+        manifest,
-      ctx.isDev ? "" : "package",
+        library,
-      "manifest.json"
+      }
-    ))
+    })
-
+  )
-    const result = {}
+  const definitions = {}
-
+  for (let { manifest, library } of componentManifests) {
-    // map over the components.json and add the library identifier as a key
+    for (let key of Object.keys(manifest)) {
-    // button -> @budibase/standard-components/button
+      const fullComponentName = `${library}/${key}`.toLowerCase()
-    for (let key of Object.keys(componentJson)) {
+      definitions[fullComponentName] = {
-      const fullComponentName = `${componentLibrary}/${key}`.toLowerCase()
-      result[fullComponentName] = {
         component: fullComponentName,
-        ...componentJson[key],
+        ...manifest[key],
       }
     }
-
+  }
-    return {
+  ctx.body = definitions
-      ...acc,
-      ...result,
-    }
-  }, {})
 }

packages/server/src/api/controllers/deploy/awsDeploy.js

@@ -66,12 +66,7 @@ exports.deploy = async function(deployment) {
   const appId = deployment.getAppId()
   const { bucket, accountId } = deployment.getVerification()
   const metadata = { accountId }
-  const s3Client = new AWS.S3({
+  await deployToObjectStore(appId, bucket, metadata)
-    params: {
-      Bucket: bucket,
-    },
-  })
-  await deployToObjectStore(appId, s3Client, metadata)
 }
 
 exports.replicateDb = async function(deployment) {

packages/server/src/api/controllers/deploy/selfDeploy.js

@@ -7,7 +7,6 @@ const {
 const {
   getWorkerUrl,
   getCouchUrl,
-  getMinioUrl,
   getSelfHostKey,
 } = require("../../../utilities/builder/hosting")
 
@@ -45,17 +44,9 @@ exports.postDeployment = async function() {
 exports.deploy = async function(deployment) {
   const appId = deployment.getAppId()
   const verification = deployment.getVerification()
-  const objClient = new AWS.S3({
-    endpoint: await getMinioUrl(),
-    s3ForcePathStyle: true, // needed with minio?
-    signatureVersion: "v4",
-    params: {
-      Bucket: verification.bucket,
-    },
-  })
   // no metadata, aws has account ID in metadata
   const metadata = {}
-  await deployToObjectStore(appId, objClient, metadata)
+  await deployToObjectStore(appId, verification.bucket, metadata)
 }
 
 exports.replicateDb = async function(deployment) {

packages/server/src/api/controllers/deploy/utils.js

@@ -1,16 +1,24 @@
-const fs = require("fs")
-const sanitize = require("sanitize-s3-objectkey")
-const { walkDir } = require("../../../utilities")
 const { join } = require("../../../utilities/centralPath")
+const fs = require("fs")
 const { budibaseAppsDir } = require("../../../utilities/budibaseDir")
 const fetch = require("node-fetch")
 const PouchDB = require("../../../db")
 const CouchDB = require("pouchdb")
+const { upload } = require("../../../utilities/fileSystem")
 
-const CONTENT_TYPE_MAP = {
+// TODO: everything in this file is to be removed
-  html: "text/html",
+
-  css: "text/css",
+function walkDir(dirPath, callback) {
-  js: "application/javascript",
+  for (let filename of fs.readdirSync(dirPath)) {
+    const filePath = `${dirPath}/${filename}`
+    const stat = fs.lstatSync(filePath)
+
+    if (stat.isFile()) {
+      callback(filePath)
+    } else {
+      walkDir(filePath, callback)
+    }
+  }
 }
 
 exports.fetchCredentials = async function(url, body) {
@@ -34,30 +42,25 @@ exports.fetchCredentials = async function(url, body) {
   return json
 }
 
-exports.prepareUpload = async function({ s3Key, metadata, client, file }) {
+exports.prepareUpload = async function({ s3Key, bucket, metadata, file }) {
-  const extension = [...file.name.split(".")].pop()
+  const response = await upload({
-  const fileBytes = fs.readFileSync(file.path)
+    bucket,
-
+    metadata,
-  const upload = await client
+    filename: s3Key,
-    .upload({
+    path: file.path,
-      // windows file paths need to be converted to forward slashes for s3
+    type: file.type,
-      Key: sanitize(s3Key).replace(/\\/g, "/"),
+  })
-      Body: fileBytes,
-      ContentType: file.type || CONTENT_TYPE_MAP[extension.toLowerCase()],
-      Metadata: metadata,
-    })
-    .promise()
 
   return {
     size: file.size,
     name: file.name,
-    extension,
+    extension: [...file.name.split(".")].pop(),
-    url: upload.Location,
+    url: response.Location,
-    key: upload.Key,
+    key: response.Key,
   }
 }
 
-exports.deployToObjectStore = async function(appId, objectClient, metadata) {
+exports.deployToObjectStore = async function(appId, bucket, metadata) {
   const appAssetsPath = join(budibaseAppsDir(), appId, "public")
 
   let uploads = []
@@ -66,12 +69,12 @@ exports.deployToObjectStore = async function(appId, objectClient, metadata) {
   walkDir(appAssetsPath, function(filePath) {
     const filePathParts = filePath.split("/")
     const appAssetUpload = exports.prepareUpload({
+      bucket,
       file: {
         path: filePath,
         name: filePathParts.pop(),
       },
       s3Key: filePath.replace(appAssetsPath, `assets/${appId}`),
-      client: objectClient,
       metadata,
     })
     uploads.push(appAssetUpload)
@@ -92,7 +95,7 @@ exports.deployToObjectStore = async function(appId, objectClient, metadata) {
     const attachmentUpload = exports.prepareUpload({
       file,
       s3Key: `assets/${appId}/attachments/${file.processedFileName}`,
-      client: objectClient,
+      bucket,
       metadata,
     })
 

packages/server/src/api/controllers/hosting.js

@@ -1,11 +1,11 @@
 const CouchDB = require("../../db")
-const { BUILDER_CONFIG_DB, HOSTING_DOC } = require("../../constants")
 const {
   getHostingInfo,
   getDeployedApps,
   HostingTypes,
   getAppUrl,
 } = require("../../utilities/builder/hosting")
+const { StaticDatabases } = require("../../db/utils")
 
 exports.fetchInfo = async ctx => {
   ctx.body = {
@@ -14,17 +14,17 @@ exports.fetchInfo = async ctx => {
 }
 
 exports.save = async ctx => {
-  const db = new CouchDB(BUILDER_CONFIG_DB)
+  const db = new CouchDB(StaticDatabases.BUILDER_HOSTING.name)
   const { type } = ctx.request.body
   if (type === HostingTypes.CLOUD && ctx.request.body._rev) {
     ctx.body = await db.remove({
       ...ctx.request.body,
-      _id: HOSTING_DOC,
+      _id: StaticDatabases.BUILDER_HOSTING.baseDoc,
     })
   } else {
     ctx.body = await db.put({
       ...ctx.request.body,
-      _id: HOSTING_DOC,
+      _id: StaticDatabases.BUILDER_HOSTING.baseDoc,
     })
   }
 }

packages/server/src/api/controllers/query.js

@@ -93,7 +93,7 @@ exports.find = async function(ctx) {
   const db = new CouchDB(ctx.user.appId)
   const query = enrichQueries(await db.get(ctx.params.queryId))
   // remove properties that could be dangerous in real app
-  if (env.CLOUD) {
+  if (env.isProd()) {
     delete query.fields
     delete query.parameters
     delete query.schema

packages/server/src/api/controllers/static/index.js

@@ -3,9 +3,7 @@ require("svelte/register")
 const send = require("koa-send")
 const { resolve, join } = require("../../../utilities/centralPath")
 const fetch = require("node-fetch")
-const fs = require("fs-extra")
 const uuid = require("uuid")
-const AWS = require("aws-sdk")
 const { prepareUpload } = require("../deploy/utils")
 const { processString } = require("@budibase/string-templates")
 const {
@@ -15,9 +13,10 @@ const {
 const { getDeployedApps } = require("../../../utilities/builder/hosting")
 const CouchDB = require("../../../db")
 const setBuilderToken = require("../../../utilities/builder/setBuilderToken")
-const fileProcessor = require("../../../utilities/fileProcessor")
+const { loadHandlebarsFile } = require("../../../utilities/fileSystem")
 const env = require("../../../environment")
 const { OBJ_STORE_DIRECTORY } = require("../../../constants")
+const fileProcessor = require("../../../utilities/fileSystem/processor")
 
 function objectStoreUrl() {
   if (env.SELF_HOSTED) {
@@ -51,94 +50,30 @@ exports.serveBuilder = async function(ctx) {
 }
 
 exports.uploadFile = async function(ctx) {
-  let files
+  let files =
-  files =
     ctx.request.files.file.length > 1
       ? Array.from(ctx.request.files.file)
       : [ctx.request.files.file]
 
-  const attachmentsPath = resolve(
+  const uploads = files.map(async file => {
-    budibaseAppsDir(),
-    ctx.user.appId,
-    "attachments"
-  )
-
-  if (env.CLOUD) {
-    // remote upload
-    const s3 = new AWS.S3({
-      params: {
-        Bucket: "prod-budi-app-assets",
-      },
-    })
-
-    const uploads = files.map(file => {
-      const fileExtension = [...file.name.split(".")].pop()
-      const processedFileName = `${uuid.v4()}.${fileExtension}`
-
-      return prepareUpload({
-        file,
-        s3Key: `assets/${ctx.user.appId}/attachments/${processedFileName}`,
-        s3,
-      })
-    })
-
-    ctx.body = await Promise.all(uploads)
-    return
-  }
-
-  ctx.body = await processLocalFileUploads({
-    files,
-    outputPath: attachmentsPath,
-    appId: ctx.user.appId,
-  })
-}
-
-async function processLocalFileUploads({ files, outputPath, appId }) {
-  // create attachments dir if it doesnt exist
-  !fs.existsSync(outputPath) && fs.mkdirSync(outputPath, { recursive: true })
-
-  const filesToProcess = files.map(file => {
     const fileExtension = [...file.name.split(".")].pop()
     // filenames converted to UUIDs so they are unique
     const processedFileName = `${uuid.v4()}.${fileExtension}`
 
-    return {
+    // need to handle image processing
-      name: file.name,
+    await fileProcessor.process({
-      path: file.path,
+      ...file,
-      size: file.size,
-      type: file.type,
-      processedFileName,
       extension: fileExtension,
-      outputPath: join(outputPath, processedFileName),
+    })
-      url: join("/attachments", processedFileName),
+
-    }
+    return prepareUpload({
+      file,
+      s3Key: `assets/${ctx.user.appId}/attachments/${processedFileName}`,
+      bucket: "prod-budi-app-assets",
+    })
   })
 
-  const fileProcessOperations = filesToProcess.map(fileProcessor.process)
+  ctx.body = await Promise.all(uploads)
-
-  const processedFiles = await Promise.all(fileProcessOperations)
-
-  let pendingFileUploads
-  // local document used to track which files need to be uploaded
-  // db.get throws an error if the document doesn't exist
-  // need to use a promise to default
-  const db = new CouchDB(appId)
-  await db
-    .get("_local/fileuploads")
-    .then(data => {
-      pendingFileUploads = data
-    })
-    .catch(() => {
-      pendingFileUploads = { _id: "_local/fileuploads", uploads: [] }
-    })
-
-  pendingFileUploads.uploads = [
-    ...processedFiles,
-    ...pendingFileUploads.uploads,
-  ]
-  await db.put(pendingFileUploads)
-
-  return processedFiles
 }
 
 exports.serveApp = async function(ctx) {
@@ -152,12 +87,12 @@ exports.serveApp = async function(ctx) {
 
   const { head, html, css } = App.render({
     title: appInfo.name,
-    production: env.CLOUD,
+    production: env.isProd(),
     appId,
     objectStoreUrl: objectStoreUrl(),
   })
 
-  const appHbs = fs.readFileSync(`${__dirname}/templates/app.hbs`, "utf8")
+  const appHbs = loadHandlebarsFile(`${__dirname}/templates/app.hbs`)
   ctx.body = await processString(appHbs, {
     head,
     body: html,
@@ -171,7 +106,7 @@ exports.serveAttachment = async function(ctx) {
   const attachmentsPath = resolve(budibaseAppsDir(), appId, "attachments")
 
   // Serve from object store
-  if (env.CLOUD) {
+  if (env.isProd()) {
     const S3_URL = join(objectStoreUrl(), appId, "attachments", ctx.file)
     const response = await fetch(S3_URL)
     const body = await response.text()
@@ -202,15 +137,13 @@ exports.serveComponentLibrary = async function(ctx) {
     "dist"
   )
 
-  if (ctx.isDev) {
+  if (env.isDev()) {
     componentLibraryPath = join(
       budibaseTempDir(),
       decodeURI(ctx.query.library),
       "dist"
     )
-  }
+  } else {
-
-  if (env.CLOUD) {
     let componentLib = "componentlibrary"
     if (ctx.user.version) {
       componentLib += `-${ctx.user.version}`

packages/server/src/api/controllers/templates.js

@@ -1,10 +1,5 @@
 const fetch = require("node-fetch")
-const {
+const { downloadTemplate } = require("../../utilities/fileSystem")
-  downloadTemplate,
-  exportTemplateFromApp,
-  getLocalTemplates,
-} = require("../../utilities/templates")
-const env = require("../../environment")
 
 // development flag, can be used to test against templates exported locally
 const DEFAULT_TEMPLATES_BUCKET =
@@ -12,16 +7,11 @@ const DEFAULT_TEMPLATES_BUCKET =
 
 exports.fetch = async function(ctx) {
   const { type = "app" } = ctx.query
-
+  const response = await fetch(
-  if (env.LOCAL_TEMPLATES) {
+    `https://${DEFAULT_TEMPLATES_BUCKET}/manifest.json`
-    ctx.body = Object.values(getLocalTemplates()[type])
+  )
-  } else {
+  const json = await response.json()
-    const response = await fetch(
+  ctx.body = Object.values(json.templates[type])
-      `https://${DEFAULT_TEMPLATES_BUCKET}/manifest.json`
-    )
-    const json = await response.json()
-    ctx.body = Object.values(json.templates[type])
-  }
 }
 
 // can't currently test this, have to ignore from coverage
@@ -29,26 +19,9 @@ exports.fetch = async function(ctx) {
 exports.downloadTemplate = async function(ctx) {
   const { type, name } = ctx.params
 
-  if (!env.LOCAL_TEMPLATES) {
+  await downloadTemplate(type, name)
-    await downloadTemplate(type, name)
-  }
 
   ctx.body = {
     message: `template ${type}:${name} downloaded successfully.`,
   }
 }
-
-exports.exportTemplateFromApp = async function(ctx) {
-  const { appId } = ctx.user
-  const { templateName } = ctx.request.body
-
-  await exportTemplateFromApp({
-    appId,
-    templateName,
-  })
-
-  ctx.status = 200
-  ctx.body = {
-    message: `Created template: ${templateName}`,
-  }
-}

packages/server/src/api/controllers/view/index.js

@@ -1,8 +1,6 @@
 const CouchDB = require("../../../db")
 const viewTemplate = require("./viewBuilder")
-const fs = require("fs")
+const { apiFileReturn } = require("../../../utilities/fileSystem")
-const { join } = require("../../../utilities/centralPath")
-const os = require("os")
 const exporters = require("./exporters")
 const { fetchView } = require("../row")
 const { ViewNames } = require("../../../db/utils")
@@ -120,12 +118,10 @@ const controller = {
     // Export part
     let headers = Object.keys(schema)
     const exporter = exporters[format]
-    const exportedFile = exporter(headers, ctx.body)
     const filename = `${viewName}.${format}`
-    fs.writeFileSync(join(os.tmpdir(), filename), exportedFile)
+    // send down the file
-
     ctx.attachment(filename)
-    ctx.body = fs.createReadStream(join(os.tmpdir(), filename))
+    ctx.body = apiFileReturn(exporter(headers, ctx.body))
   },
 }
 

packages/server/src/api/index.js

@@ -3,7 +3,6 @@ const authenticated = require("../middleware/authenticated")
 const compress = require("koa-compress")
 const zlib = require("zlib")
 const { budibaseAppsDir } = require("../utilities/budibaseDir")
-const { isDev } = require("../utilities")
 const { mainRoutes, authRoutes, staticRoutes } = require("./routes")
 const pkg = require("../../package.json")
 
@@ -29,7 +28,6 @@ router
       jwtSecret: env.JWT_SECRET,
       useAppRootPath: true,
     }
-    ctx.isDev = isDev()
     await next()
   })
   .use("/health", ctx => (ctx.status = 200))
@@ -68,8 +66,6 @@ for (let route of mainRoutes) {
 router.use(staticRoutes.routes())
 router.use(staticRoutes.allowedMethods())
 
-if (!env.SELF_HOSTED && !env.CLOUD) {
+router.redirect("/", "/_builder")
-  router.redirect("/", "/_builder")
-}
 
 module.exports = router

packages/server/src/api/routes/static.js

@@ -4,6 +4,7 @@ const { budibaseTempDir } = require("../../utilities/budibaseDir")
 const authorized = require("../../middleware/authorized")
 const { BUILDER } = require("../../utilities/security/permissions")
 const usage = require("../../middleware/usageQuota")
+const env = require("../../environment")
 
 const router = Router()
 
@@ -12,7 +13,7 @@ router.param("file", async (file, ctx, next) => {
   ctx.file = file && file.includes(".") ? file : "index.html"
 
   // Serving the client library from your local dir in dev
-  if (ctx.isDev && ctx.file.startsWith("budibase-client")) {
+  if (env.isDev() && ctx.file.startsWith("budibase-client")) {
     ctx.devPath = budibaseTempDir()
   }
 

packages/server/src/api/routes/templates.js

@@ -12,6 +12,5 @@ router
     authorized(BUILDER),
     controller.downloadTemplate
   )
-  .post("/api/templates", authorized(BUILDER), controller.exportTemplateFromApp)
 
 module.exports = router

packages/server/src/api/routes/tests/apikeys.spec.js

@@ -1,8 +1,5 @@
 const setup = require("./utilities")
 const { checkBuilderEndpoint } = require("./utilities/TestFunctions")
-const { budibaseAppsDir } = require("../../../utilities/budibaseDir")
-const fs = require("fs")
-const path = require("path")
 
 describe("/api/keys", () => {
   let request = setup.getRequest()
@@ -16,12 +13,14 @@ describe("/api/keys", () => {
 
   describe("fetch", () => {
     it("should allow fetching", async () => {
-      const res = await request
+      await setup.switchToSelfHosted(async () => {
-        .get(`/api/keys`)
+        const res = await request
-        .set(config.defaultHeaders())
+          .get(`/api/keys`)
-        .expect("Content-Type", /json/)
+          .set(config.defaultHeaders())
-        .expect(200)
+          .expect("Content-Type", /json/)
-      expect(res.body).toBeDefined()
+          .expect(200)
+        expect(res.body).toBeDefined()
+      })
     })
 
     it("should check authorization for builder", async () => {
@@ -35,17 +34,18 @@ describe("/api/keys", () => {
 
   describe("update", () => {
     it("should allow updating a value", async () => {
-      fs.writeFileSync(path.join(budibaseAppsDir(), ".env"), "TEST_API_KEY=thing")
+      await setup.switchToSelfHosted(async () => {
-      const res = await request
+        const res = await request
-        .put(`/api/keys/TEST`)
+          .put(`/api/keys/TEST`)
-        .send({
+          .send({
-          value: "test"
+            value: "test"
-        })
+          })
-        .set(config.defaultHeaders())
+          .set(config.defaultHeaders())
-        .expect("Content-Type", /json/)
+          .expect("Content-Type", /json/)
-        .expect(200)
+          .expect(200)
-      expect(res.body["TEST"]).toEqual("test")
+        expect(res.body._id).toBeDefined()
-      expect(process.env.TEST_API_KEY).toEqual("test")
+        expect(res.body._rev).toBeDefined()
+      })
     })
 
     it("should check authorization for builder", async () => {

packages/server/src/api/routes/tests/auth.spec.js

@@ -1,4 +1,3 @@
-const { checkBuilderEndpoint } = require("./utilities/TestFunctions")
 const setup = require("./utilities")
 
 describe("/authenticate", () => {

packages/server/src/api/routes/tests/backup.spec.js

@@ -1,6 +1,8 @@
 const { checkBuilderEndpoint } = require("./utilities/TestFunctions")
 const setup = require("./utilities")
 
+jest.mock("../../../utilities/fileSystem/utilities")
+
 describe("/backups", () => {
   let request = setup.getRequest()
   let config = setup.getConfig()
@@ -14,7 +16,7 @@ describe("/backups", () => {
   describe("exportAppDump", () => {
     it("should be able to export app", async () => {
       const res = await request
-        .get(`/api/backups/export?appId=${config.getAppId()}`)
+        .get(`/api/backups/export?appId=${config.getAppId()}&appname=test`)
         .set(config.defaultHeaders())
         .expect(200)
       expect(res.text).toBeDefined()

packages/server/src/api/routes/tests/cloud.spec.js

@@ -1,16 +0,0 @@
-const setup = require("./utilities")
-
-describe("test things in the Cloud/Self hosted", () => {
-  describe("test self hosted static page", () => {
-    it("should be able to load the static page", async () => {
-      await setup.switchToCloudForFunction(async () => {
-        let request = setup.getRequest()
-        let config = setup.getConfig()
-        await config.init()
-        const res = await request.get(`/`).expect(200)
-        expect(res.text.includes("<title>Budibase self hosting️</title>")).toEqual(true)
-        setup.afterAll()
-      })
-    })
-  })
-})

packages/server/src/api/routes/tests/component.spec.js

@@ -1,8 +1,15 @@
 const { checkBuilderEndpoint } = require("./utilities/TestFunctions")
 const setup = require("./utilities")
-const fs = require("fs")
+
-const { resolve, join } = require("path")
+jest.mock("../../../utilities/fileSystem/utilities", () => ({
-const { budibaseAppsDir } = require("../../../utilities/budibaseDir")
+  ...jest.requireActual("../../../utilities/fileSystem/utilities"),
+  retrieve: () => {
+    const { join } = require("path")
+    const library = join("@budibase", "standard-components")
+    const path = require.resolve(library).split(join("dist", "index.js"))[0] + "manifest.json"
+    return JSON.stringify(require(path))
+  }
+}))
 
 describe("/component", () => {
   let request = setup.getRequest()
@@ -14,23 +21,8 @@ describe("/component", () => {
     await config.init()
   })
 
-  function mock() {
-    const manifestFile = "manifest.json"
-    const appId = config.getAppId()
-    const libraries = [join("@budibase", "standard-components")]
-    for (let library of libraries) {
-      let appDirectory = resolve(budibaseAppsDir(), appId, "node_modules", library, "package")
-      fs.mkdirSync(appDirectory, { recursive: true })
-
-      const file = require.resolve(library).split(join("dist", "index.js"))[0] + manifestFile
-      fs.copyFileSync(file, join(appDirectory, manifestFile))
-    }
-  }
-
   describe("fetch definitions", () => {
     it("should be able to fetch definitions", async () => {
-      // have to "mock" the files required
-      mock()
       const res = await request
         .get(`/${config.getAppId()}/components/definitions`)
         .set(config.defaultHeaders())

packages/server/src/api/routes/tests/hosting.spec.js

@@ -107,17 +107,16 @@ describe("/hosting", () => {
   })
 
   describe("getDeployedApps", () => {
-    it("should get apps when in builder", async () => {
+    it("should fail when not self hosted", async () => {
-      const res = await request
+      await request
         .get(`/api/hosting/apps`)
         .set(config.defaultHeaders())
         .expect("Content-Type", /json/)
-        .expect(200)
+        .expect(400)
-      expect(res.body.app1).toEqual({url: "/app1"})
     })
 
     it("should get apps when in cloud", async () => {
-      await setup.switchToCloudForFunction(async () => {
+      await setup.switchToSelfHosted(async () => {
         const res = await request
           .get(`/api/hosting/apps`)
           .set(config.defaultHeaders())

packages/server/src/api/routes/tests/query.spec.js

@@ -89,7 +89,7 @@ describe("/queries", () => {
     })
 
     it("should find a query in cloud", async () => {
-      await setup.switchToCloudForFunction(async () => {
+      await setup.switchToSelfHosted(async () => {
         const query = await config.createQuery()
         const res = await request
           .get(`/api/queries/${query._id}`)

packages/server/src/api/routes/tests/row.spec.js

@@ -410,7 +410,7 @@ describe("/rows", () => {
         tableId: table._id,
       })
       // the environment needs configured for this
-      await setup.switchToCloudForFunction(async () => {
+      await setup.switchToSelfHosted(async () => {
         const enriched = await outputProcessing(config.getAppId(), table, [row])
         expect(enriched[0].attachment[0].url).toBe(`/app-assets/assets/${config.getAppId()}/test/thing`)
       })

packages/server/src/api/routes/tests/templates.spec.js

@@ -1,7 +1,4 @@
 const setup = require("./utilities")
-const { budibaseAppsDir } = require("../../../utilities/budibaseDir")
-const fs = require("fs")
-const { join } = require("path")
 
 describe("/templates", () => {
   let request = setup.getRequest()
@@ -24,26 +21,4 @@ describe("/templates", () => {
       expect(Array.isArray(res.body)).toEqual(true)
     })
   })
-
-  describe("export", () => {
-    it("should be able to export the basic app", async () => {
-      const res = await request
-        .post(`/api/templates`)
-        .send({
-          templateName: "test",
-        })
-        .set(config.defaultHeaders())
-        .expect("Content-Type", /json/)
-        .expect(200)
-      expect(res.body.message).toEqual("Created template: test")
-      const dir = join(
-        budibaseAppsDir(),
-        "templates",
-        "app",
-        "test",
-        "db"
-      )
-      expect(fs.existsSync(dir)).toEqual(true)
-    })
-  })
 })

packages/server/src/api/routes/tests/utilities/index.js

@@ -35,18 +35,18 @@ exports.getConfig = () => {
   return config
 }
 
-exports.switchToCloudForFunction = async func => {
+exports.switchToSelfHosted = async func => {
   // self hosted stops any attempts to Dynamo
-  env.CLOUD = true
+  env._set("NODE_ENV", "production")
-  env.SELF_HOSTED = true
+  env._set("SELF_HOSTED", true)
   let error
   try {
     await func()
   } catch (err) {
     error = err
   }
-  env.CLOUD = false
+  env._set("NODE_ENV", "jest")
-  env.SELF_HOSTED = false
+  env._set("SELF_HOSTED", false)
   // don't throw error until after reset
   if (error) {
     throw error

packages/server/src/app.js

@@ -9,6 +9,7 @@ const env = require("./environment")
 const eventEmitter = require("./events")
 const automations = require("./automations/index")
 const Sentry = require("@sentry/node")
+const fileSystem = require("./utilities/fileSystem")
 
 const app = new Koa()
 
@@ -65,6 +66,7 @@ module.exports = server.listen(env.PORT || 0, async () => {
   console.log(`Budibase running on ${JSON.stringify(server.address())}`)
   env._set("PORT", server.address().port)
   eventEmitter.emitPort(env.PORT)
+  fileSystem.init()
   await automations.init()
 })
 

packages/server/src/automations/actions.js

@@ -5,17 +5,12 @@ const deleteRow = require("./steps/deleteRow")
 const createUser = require("./steps/createUser")
 const outgoingWebhook = require("./steps/outgoingWebhook")
 const env = require("../environment")
-const download = require("download")
-const fetch = require("node-fetch")
-const { join } = require("../utilities/centralPath")
-const os = require("os")
-const fs = require("fs")
 const Sentry = require("@sentry/node")
+const {
+  automationInit,
+  getExternalAutomationStep,
+} = require("../utilities/fileSystem")
 
-const DEFAULT_BUCKET =
-  "https://prod-budi-automations.s3-eu-west-1.amazonaws.com"
-const DEFAULT_DIRECTORY = ".budibase-automations"
-const AUTOMATION_MANIFEST = "manifest.json"
 const BUILTIN_ACTIONS = {
   SEND_EMAIL: sendEmail.run,
   CREATE_ROW: createRow.run,
@@ -33,8 +28,6 @@ const BUILTIN_DEFINITIONS = {
   OUTGOING_WEBHOOK: outgoingWebhook.definition,
 }
 
-let AUTOMATION_BUCKET = env.AUTOMATION_BUCKET
-let AUTOMATION_DIRECTORY = env.AUTOMATION_DIRECTORY
 let MANIFEST = null
 
 /* istanbul ignore next */
@@ -42,22 +35,13 @@ function buildBundleName(pkgName, version) {
   return `${pkgName}@${version}.min.js`
 }
 
-/* istanbul ignore next */
-async function downloadPackage(name, version, bundleName) {
-  await download(
-    `${AUTOMATION_BUCKET}/${name}/${version}/${bundleName}`,
-    AUTOMATION_DIRECTORY
-  )
-  return require(join(AUTOMATION_DIRECTORY, bundleName))
-}
-
 /* istanbul ignore next */
 module.exports.getAction = async function(actionName) {
   if (BUILTIN_ACTIONS[actionName] != null) {
     return BUILTIN_ACTIONS[actionName]
   }
   // worker pools means that a worker may not have manifest
-  if (env.CLOUD && MANIFEST == null) {
+  if (env.isProd() && MANIFEST == null) {
     MANIFEST = await module.exports.init()
   }
   // env setup to get async packages
@@ -66,28 +50,12 @@ module.exports.getAction = async function(actionName) {
   }
   const pkg = MANIFEST.packages[actionName]
   const bundleName = buildBundleName(pkg.stepId, pkg.version)
-  try {
+  return getExternalAutomationStep(pkg.stepId, pkg.version, bundleName)
-    return require(join(AUTOMATION_DIRECTORY, bundleName))
-  } catch (err) {
-    return downloadPackage(pkg.stepId, pkg.version, bundleName)
-  }
 }
 
 module.exports.init = async function() {
-  // set defaults
-  if (!AUTOMATION_DIRECTORY) {
-    AUTOMATION_DIRECTORY = join(os.homedir(), DEFAULT_DIRECTORY)
-  }
-  if (!AUTOMATION_BUCKET) {
-    AUTOMATION_BUCKET = DEFAULT_BUCKET
-  }
-  if (!fs.existsSync(AUTOMATION_DIRECTORY)) {
-    fs.mkdirSync(AUTOMATION_DIRECTORY, { recursive: true })
-  }
-  // env setup to get async packages
   try {
-    let response = await fetch(`${AUTOMATION_BUCKET}/${AUTOMATION_MANIFEST}`)
+    MANIFEST = await automationInit()
-    MANIFEST = await response.json()
     module.exports.DEFINITIONS =
       MANIFEST && MANIFEST.packages
         ? Object.assign(MANIFEST.packages, BUILTIN_DEFINITIONS)

packages/server/src/automations/index.js

@@ -34,10 +34,10 @@ module.exports.init = async function() {
   await actions.init()
   triggers.automationQueue.process(async job => {
     try {
-      if (env.CLOUD && job.data.automation && !env.SELF_HOSTED) {
+      if (env.USE_QUOTAS) {
         job.data.automation.apiKey = await updateQuota(job.data.automation)
       }
-      if (env.BUDIBASE_ENVIRONMENT === "PRODUCTION") {
+      if (env.isProd()) {
         await runWorker(job)
       } else {
         await singleThread(job)

packages/server/src/automations/steps/createRow.js

@@ -85,7 +85,7 @@ module.exports.run = async function({ inputs, appId, apiKey, emitter }) {
       inputs.row.tableId,
       inputs.row
     )
-    if (env.CLOUD) {
+    if (env.isProd()) {
       await usage.update(apiKey, usage.Properties.ROW, 1)
     }
     await rowController.save(ctx)

packages/server/src/automations/steps/createUser.js

@@ -72,7 +72,7 @@ module.exports.run = async function({ inputs, appId, apiKey, emitter }) {
   }
 
   try {
-    if (env.CLOUD) {
+    if (env.isProd()) {
       await usage.update(apiKey, usage.Properties.USER, 1)
     }
     await userController.create(ctx)

packages/server/src/automations/steps/deleteRow.js

@@ -70,7 +70,7 @@ module.exports.run = async function({ inputs, appId, apiKey, emitter }) {
   }
 
   try {
-    if (env.CLOUD) {
+    if (env.isProd()) {
       await usage.update(apiKey, usage.Properties.ROW, -1)
     }
     await rowController.destroy(ctx)

packages/server/src/automations/tests/automation.spec.js

@@ -47,27 +47,23 @@ describe("Run through some parts of the automations system", () => {
     expect(thread).toHaveBeenCalled()
   })
 
-  it("should be able to init in cloud", async () => {
+  it("should be able to init in prod", async () => {
-    env.CLOUD = true
+    await setup.runInProd(async () => {
-    env.BUDIBASE_ENVIRONMENT = "PRODUCTION"
+      await triggers.externalTrigger(basicAutomation(), { a: 1 })
-    await triggers.externalTrigger(basicAutomation(), { a: 1 })
+      await wait(100)
-    await wait(100)
+      // haven't added a mock implementation so getAPIKey of usageQuota just returns undefined
-    // haven't added a mock implementation so getAPIKey of usageQuota just returns undefined
+      expect(usageQuota.update).toHaveBeenCalledWith("test", "automationRuns", 1)
-    expect(usageQuota.update).toHaveBeenCalledWith("test", "automationRuns", 1)
+      expect(workerJob).toBeDefined()
-    expect(workerJob).toBeDefined()
+    })
-    env.BUDIBASE_ENVIRONMENT = "JEST"
-    env.CLOUD = false
   })
 
   it("try error scenario", async () => {
-    env.CLOUD = true
+    await setup.runInProd(async () => {
-    env.BUDIBASE_ENVIRONMENT = "PRODUCTION"
+      // the second call will throw an error
-    // the second call will throw an error
+      await triggers.externalTrigger(basicAutomation(), { a: 1 })
-    await triggers.externalTrigger(basicAutomation(), { a: 1 })
+      await wait(100)
-    await wait(100)
+      expect(console.error).toHaveBeenCalled()
-    expect(console.error).toHaveBeenCalled()
+    })
-    env.BUDIBASE_ENVIRONMENT = "JEST"
-    env.CLOUD = false
   })
 
   it("should be able to check triggering row filling", async () => {

packages/server/src/automations/tests/createRow.spec.js

@@ -42,12 +42,12 @@ describe("test the create row action", () => {
   })
 
   it("check usage quota attempts", async () => {
-    env.CLOUD = true
+    await setup.runInProd(async () => {
-    await setup.runStep(setup.actions.CREATE_ROW.stepId, {
+      await setup.runStep(setup.actions.CREATE_ROW.stepId, {
-      row
+        row
+      })
+      expect(usageQuota.update).toHaveBeenCalledWith(setup.apiKey, "rows", 1)
     })
-    expect(usageQuota.update).toHaveBeenCalledWith(setup.apiKey, "rows", 1)
-    env.CLOUD = false
   })
 
   it("should check invalid inputs return an error", async () => {

packages/server/src/automations/tests/createUser.spec.js

@@ -35,9 +35,9 @@ describe("test the create user action", () => {
   })
 
   it("check usage quota attempts", async () => {
-    env.CLOUD = true
+    await setup.runInProd(async () => {
-    await setup.runStep(setup.actions.CREATE_USER.stepId, user)
+      await setup.runStep(setup.actions.CREATE_USER.stepId, user)
-    expect(usageQuota.update).toHaveBeenCalledWith(setup.apiKey, "users", 1)
+      expect(usageQuota.update).toHaveBeenCalledWith(setup.apiKey, "users", 1)
-    env.CLOUD = false
+    })
   })
 })

packages/server/src/automations/tests/deleteRow.spec.js

@@ -36,10 +36,10 @@ describe("test the delete row action", () => {
   })
 
   it("check usage quota attempts", async () => {
-    env.CLOUD = true
+    await setup.runInProd(async () => {
-    await setup.runStep(setup.actions.DELETE_ROW.stepId, inputs)
+      await setup.runStep(setup.actions.DELETE_ROW.stepId, inputs)
-    expect(usageQuota.update).toHaveBeenCalledWith(setup.apiKey, "rows", -1)
+      expect(usageQuota.update).toHaveBeenCalledWith(setup.apiKey, "rows", -1)
-    env.CLOUD = false
+    })
   })
 
   it("should check invalid inputs return an error", async () => {

packages/server/src/automations/tests/utilities/index.js

@@ -2,6 +2,7 @@ const TestConfig = require("../../../tests/utilities/TestConfiguration")
 const actions = require("../../actions")
 const logic = require("../../logic")
 const emitter = require("../../../events/index")
+const env = require("../../../environment")
 
 let config
 
@@ -16,6 +17,22 @@ exports.afterAll = () => {
   config.end()
 }
 
+exports.runInProd = async fn => {
+  env._set("NODE_ENV", "production")
+  env._set("USE_QUOTAS", 1)
+  let error
+  try {
+    await fn()
+  } catch (err) {
+    error = err
+  }
+  env._set("NODE_ENV", "jest")
+  env._set("USE_QUOTAS", null)
+  if (error) {
+    throw error
+  }
+}
+
 exports.runStep = async function runStep(stepId, inputs) {
   let step
   if (

packages/server/src/constants/index.js

@@ -80,8 +80,6 @@ exports.AutoFieldSubTypes = {
   AUTO_ID: "autoID",
 }
 
-exports.BUILDER_CONFIG_DB = "builder-config-db"
-exports.HOSTING_DOC = "hosting-doc"
 exports.OBJ_STORE_DIRECTORY = "/app-assets/assets"
 exports.BaseQueryVerbs = {
   CREATE: "create",
@@ -89,3 +87,9 @@ exports.BaseQueryVerbs = {
   UPDATE: "update",
   DELETE: "delete",
 }
+
+exports.ObjectStoreBuckets = {
+  BACKUPS: "backups",
+  APPS: "prod-budi-app-assets",
+  TEMPLATES: "templates",
+}

packages/server/src/db/builder.js

@@ -0,0 +1,38 @@
+const CouchDB = require("./index")
+const { StaticDatabases } = require("./utils")
+const env = require("../environment")
+
+const SELF_HOST_ERR = "Unable to access builder DB/doc - not self hosted."
+const BUILDER_DB = StaticDatabases.BUILDER
+
+/**
+ * This is the builder database, right now this is a single, static database
+ * that is present across the whole system and determines some core functionality
+ * for the builder (e.g. storage of API keys). This has been limited to self hosting
+ * as it doesn't make as much sense against the currently design Cloud system.
+ */
+
+exports.getBuilderMainDoc = async () => {
+  if (!env.SELF_HOSTED) {
+    throw SELF_HOST_ERR
+  }
+  const db = new CouchDB(BUILDER_DB.name)
+  try {
+    return await db.get(BUILDER_DB.baseDoc)
+  } catch (err) {
+    // doesn't exist yet, nothing to get
+    return {
+      _id: BUILDER_DB.baseDoc,
+    }
+  }
+}
+
+exports.setBuilderMainDoc = async doc => {
+  if (!env.SELF_HOSTED) {
+    throw SELF_HOST_ERR
+  }
+  // make sure to override the ID
+  doc._id = BUILDER_DB.baseDoc
+  const db = new CouchDB(BUILDER_DB.name)
+  return db.put(doc)
+}

packages/server/src/db/client.js

@@ -5,7 +5,6 @@ const find = require("pouchdb-find")
 const env = require("../environment")
 
 const COUCH_DB_URL = env.COUCH_DB_URL || "http://localhost:10000/db/"
-const isInMemory = env.NODE_ENV === "jest"
 
 PouchDB.plugin(replicationStream.plugin)
 PouchDB.plugin(find)
@@ -13,10 +12,10 @@ PouchDB.adapter("writableStream", replicationStream.adapters.writableStream)
 
 let POUCH_DB_DEFAULTS = {
   prefix: COUCH_DB_URL,
-  skip_setup: !!env.CLOUD,
+  skip_setup: env.isProd(),
 }
 
-if (isInMemory) {
+if (env.isTest()) {
   PouchDB.plugin(require("pouchdb-adapter-memory"))
   POUCH_DB_DEFAULTS = {
     prefix: undefined,

packages/server/src/db/dynamoClient.js

@@ -1,4 +1,4 @@
-let _ = require("lodash")
+let { merge } = require("lodash")
 let env = require("../environment")
 
 const AWS_REGION = env.AWS_REGION ? env.AWS_REGION : "eu-west-1"
@@ -38,7 +38,7 @@ class Table {
       params.Key[this._sort] = sort
     }
     if (otherProps) {
-      params = _.merge(params, otherProps)
+      params = merge(params, otherProps)
     }
     let response = await docClient.get(params).promise()
     return response.Item
@@ -77,7 +77,7 @@ class Table {
       params.ConditionExpression += "attribute_exists(#PRIMARY)"
     }
     if (otherProps) {
-      params = _.merge(params, otherProps)
+      params = merge(params, otherProps)
     }
     return docClient.update(params).promise()
   }
@@ -94,7 +94,7 @@ class Table {
       Item: item,
     }
     if (otherProps) {
-      params = _.merge(params, otherProps)
+      params = merge(params, otherProps)
     }
     return docClient.put(params).promise()
   }
@@ -119,7 +119,7 @@ exports.init = endpoint => {
 exports.apiKeyTable = new Table(TableInfo.API_KEYS)
 exports.userTable = new Table(TableInfo.USERS)
 
-if (env.CLOUD) {
+if (env.isProd()) {
   exports.init(`https://dynamodb.${AWS_REGION}.amazonaws.com`)
 } else {
   env._set("AWS_ACCESS_KEY_ID", "KEY_ID")

packages/server/src/db/utils.js

@@ -3,6 +3,18 @@ const newid = require("./newid")
 const UNICODE_MAX = "\ufff0"
 const SEPARATOR = "_"
 
+const StaticDatabases = {
+  BUILDER: {
+    name: "builder-db",
+    baseDoc: "builder-doc",
+  },
+  // TODO: needs removed
+  BUILDER_HOSTING: {
+    name: "builder-config-db",
+    baseDoc: "hosting-doc",
+  },
+}
+
 const DocumentTypes = {
   TABLE: "ta",
   ROW: "ro",
@@ -25,6 +37,7 @@ const ViewNames = {
   USERS: "ta_users",
 }
 
+exports.StaticDatabases = StaticDatabases
 exports.ViewNames = ViewNames
 exports.DocumentTypes = DocumentTypes
 exports.SEPARATOR = SEPARATOR

packages/server/src/environment.js

@@ -1,45 +1,64 @@
-const { resolve, join } = require("./utilities/centralPath")
+function isTest() {
-const { homedir } = require("os")
+  return (
-const { app } = require("electron")
+    process.env.NODE_ENV === "jest" ||
+    process.env.NODE_ENV === "cypress" ||
+    process.env.JEST_WORKER_ID != null
+  )
+}
+
+function isDev() {
+  return (
+    process.env.NODE_ENV !== "production" &&
+    process.env.BUDIBASE_ENVIRONMENT !== "production"
+  )
+}
 
 let LOADED = false
-
+if (!LOADED && isDev() && !isTest()) {
-if (!LOADED) {
+  require("dotenv").config()
-  const homeDir = app ? app.getPath("home") : homedir()
-  const budibaseDir = join(homeDir, ".budibase")
-  process.env.BUDIBASE_DIR = budibaseDir
-  require("dotenv").config({ path: resolve(budibaseDir, ".env") })
   LOADED = true
 }
 
 module.exports = {
-  CLIENT_ID: process.env.CLIENT_ID,
+  // important
-  NODE_ENV: process.env.NODE_ENV,
-  JWT_SECRET: process.env.JWT_SECRET,
-  BUDIBASE_DIR: process.env.BUDIBASE_DIR,
   PORT: process.env.PORT,
+  JWT_SECRET: process.env.JWT_SECRET,
   COUCH_DB_URL: process.env.COUCH_DB_URL,
+  MINIO_URL: process.env.MINIO_URL,
+  WORKER_URL: process.env.WORKER_URL,
+  SELF_HOSTED: process.env.SELF_HOSTED,
+  AWS_REGION: process.env.AWS_REGION,
+  ENABLE_ANALYTICS: process.env.ENABLE_ANALYTICS,
+  MINIO_ACCESS_KEY: process.env.MINIO_ACCESS_KEY,
+  MINIO_SECRET_KEY: process.env.MINIO_SECRET_KEY,
+  USE_QUOTAS: process.env.USE_QUOTAS,
+  // environment
+  NODE_ENV: process.env.NODE_ENV,
+  JEST_WORKER_ID: process.env.JEST_WORKER_ID,
+  BUDIBASE_ENVIRONMENT: process.env.BUDIBASE_ENVIRONMENT,
+  // minor
   SALT_ROUNDS: process.env.SALT_ROUNDS,
   LOGGER: process.env.LOGGER,
   LOG_LEVEL: process.env.LOG_LEVEL,
   AUTOMATION_DIRECTORY: process.env.AUTOMATION_DIRECTORY,
   AUTOMATION_BUCKET: process.env.AUTOMATION_BUCKET,
-  BUDIBASE_ENVIRONMENT: process.env.BUDIBASE_ENVIRONMENT,
   SENDGRID_API_KEY: process.env.SENDGRID_API_KEY,
-  CLOUD: process.env.CLOUD,
-  SELF_HOSTED: process.env.SELF_HOSTED,
-  WORKER_URL: process.env.WORKER_URL,
-  HOSTING_KEY: process.env.HOSTING_KEY,
   DYNAMO_ENDPOINT: process.env.DYNAMO_ENDPOINT,
-  AWS_REGION: process.env.AWS_REGION,
+  // old - to remove
-  DEPLOYMENT_CREDENTIALS_URL: process.env.DEPLOYMENT_CREDENTIALS_URL,
+  CLIENT_ID: process.env.CLIENT_ID,
+  BUDIBASE_DIR: process.env.BUDIBASE_DIR,
+  DEPLOYMENT_DB_URL: process.env.DEPLOYMENT_DB_URL,
   BUDIBASE_API_KEY: process.env.BUDIBASE_API_KEY,
   USERID_API_KEY: process.env.USERID_API_KEY,
-  ENABLE_ANALYTICS: process.env.ENABLE_ANALYTICS,
+  DEPLOYMENT_CREDENTIALS_URL: process.env.DEPLOYMENT_CREDENTIALS_URL,
-  DEPLOYMENT_DB_URL: process.env.DEPLOYMENT_DB_URL,
+  HOSTING_KEY: process.env.HOSTING_KEY,
-  LOCAL_TEMPLATES: process.env.LOCAL_TEMPLATES,
   _set(key, value) {
     process.env[key] = value
     module.exports[key] = value
   },
+  isTest,
+  isDev,
+  isProd: () => {
+    return !isDev()
+  },
 }

packages/server/src/middleware/authorized.js

@@ -13,19 +13,12 @@ const { AuthTypes } = require("../constants")
 
 const ADMIN_ROLES = [BUILTIN_ROLE_IDS.ADMIN, BUILTIN_ROLE_IDS.BUILDER]
 
-const LOCAL_PASS = new RegExp(["webhooks/trigger"].join("|"))
-
 function hasResource(ctx) {
   return ctx.resourceId != null
 }
 
 module.exports = (permType, permLevel = null) => async (ctx, next) => {
-  // webhooks can pass locally
+  if (env.isProd() && ctx.headers["x-api-key"] && ctx.headers["x-instanceid"]) {
-  if (!env.CLOUD && LOCAL_PASS.test(ctx.request.url)) {
-    return next()
-  }
-
-  if (env.CLOUD && ctx.headers["x-api-key"] && ctx.headers["x-instanceid"]) {
     // api key header passed by external webhook
     if (await isAPIKeyValid(ctx.headers["x-api-key"])) {
       ctx.auth = {
@@ -41,20 +34,23 @@ module.exports = (permType, permLevel = null) => async (ctx, next) => {
     return ctx.throw(403, "API key invalid")
   }
 
-  // don't expose builder endpoints in the cloud
-  if (env.CLOUD && permType === PermissionTypes.BUILDER) return
-
   if (!ctx.user) {
     return ctx.throw(403, "No user info found")
   }
 
   const role = ctx.user.role
+  const isBuilder = role._id === BUILTIN_ROLE_IDS.BUILDER
+  const isAdmin = ADMIN_ROLES.includes(role._id)
+  const isAuthed = ctx.auth.authenticated
+
+  if (permType === PermissionTypes.BUILDER && isBuilder) {
+    return next()
+  }
+
   const { basePermissions, permissions } = await getUserPermissions(
     ctx.appId,
     role._id
   )
-  const isAdmin = ADMIN_ROLES.includes(role._id)
-  const isAuthed = ctx.auth.authenticated
 
   // this may need to change in the future, right now only admins
   // can have access to builder features, this is hard coded into

packages/server/src/middleware/selfhost.js

@@ -1,14 +1,8 @@
 const env = require("../environment")
-const hosting = require("../utilities/builder/hosting")
 // if added as a middleware will stop requests unless builder is in self host mode
 // or cloud is in self host
 module.exports = async (ctx, next) => {
-  if (env.CLOUD && env.SELF_HOSTED) {
+  if (env.SELF_HOSTED) {
-    await next()
-    return
-  }
-  const hostingInfo = await hosting.getHostingInfo()
-  if (hostingInfo.type === hosting.HostingTypes.SELF) {
     await next()
     return
   }

packages/server/src/middleware/tests/authorized.spec.js

@@ -3,8 +3,15 @@ const env = require("../../environment")
 const apiKey = require("../../utilities/security/apikey")
 const { AuthTypes } = require("../../constants")
 const { PermissionTypes, PermissionLevels } = require("../../utilities/security/permissions")
-const { Test } = require("supertest")
+jest.mock("../../environment", () => ({
-jest.mock("../../environment")
+    prod: false,
+    isTest: () => true,
+    isProd: () => this.prod,
+    _set: (key, value) => {
+      this.prod = value === "production"
+    }
+  })
+)
 jest.mock("../../utilities/security/apikey")
 
 class TestConfiguration {
@@ -47,8 +54,8 @@ class TestConfiguration {
     this.ctx.request.url = url
   }
 
-  setCloudEnv(isCloud) {
+  setEnvironment(isProd) {
-    env.CLOUD = isCloud
+    env._set("NODE_ENV", isProd ? "production" : "jest")
   }
 
   setRequestHeaders(headers) {
@@ -71,12 +78,6 @@ describe("Authorization middleware", () => {
 
   beforeEach(() => {
     config = new TestConfiguration()
-  }) 
-
-  it("passes the middleware for local webhooks", async () => {
-    config.setRequestUrl("https://something/webhooks/trigger")
-    await config.executeMiddleware()
-    expect(config.next).toHaveBeenCalled()
   })
 
   describe("external web hook call", () => {
@@ -85,7 +86,7 @@ describe("Authorization middleware", () => {
 
     beforeEach(() => {
       config = new TestConfiguration()
-      config.setCloudEnv(true)
+      config.setEnvironment(true)
       config.setRequestHeaders({
         "x-api-key": "abc123",
         "x-instanceid": "instance123",
@@ -121,7 +122,7 @@ describe("Authorization middleware", () => {
 
     beforeEach(() => {
       config = new TestConfiguration()
-      config.setCloudEnv(true)
+      config.setEnvironment(true)
       config.setAuthenticated(true)
     })
 
@@ -144,7 +145,7 @@ describe("Authorization middleware", () => {
     })
 
     it("throws if the user has only builder permissions", async () => {
-      config.setCloudEnv(false)
+      config.setEnvironment(false)
       config.setMiddlewareRequiredPermission(PermissionTypes.BUILDER)
       config.setUser({
         role: {

packages/server/src/middleware/tests/selfhost.spec.js

@@ -1,6 +1,5 @@
-const selfHostMiddleware = require("../selfhost");
+const selfHostMiddleware = require("../selfhost")
 const env = require("../../environment")
-const hosting = require("../../utilities/builder/hosting");
 jest.mock("../../environment") 
 jest.mock("../../utilities/builder/hosting") 
 
@@ -20,16 +19,6 @@ class TestConfiguration {
     return this.middleware(this.ctx, this.next)
   }
 
-  setCloudHosted() {
-    env.CLOUD = 1 
-    env.SELF_HOSTED = 0
-  }
-
-  setSelfHosted() {
-    env.CLOUD = 0
-    env.SELF_HOSTED = 1
-  }
-
   afterEach() {
     jest.clearAllMocks()
   }
@@ -46,30 +35,10 @@ describe("Self host middleware", () => {
     config.afterEach()
   })
 
-  it("calls next() when CLOUD and SELF_HOSTED env vars are set", async () => {
+  it("calls next() when SELF_HOSTED env var is set", async () => {
-    env.CLOUD = 1 
     env.SELF_HOSTED = 1 
 
     await config.executeMiddleware()
     expect(config.next).toHaveBeenCalled()
   })
-
-  it("throws when hostingInfo type is cloud", async () => {
-    config.setSelfHosted()
-
-    hosting.getHostingInfo.mockImplementationOnce(() => ({ type: hosting.HostingTypes.CLOUD }))
-
-    await config.executeMiddleware()
-    expect(config.throw).toHaveBeenCalledWith(400, "Endpoint unavailable in cloud hosting.")
-    expect(config.next).not.toHaveBeenCalled()
-  })
-
-  it("calls the self hosting middleware to pass through to next() when the hostingInfo type is self", async () => {
-    config.setSelfHosted()
-
-    hosting.getHostingInfo.mockImplementationOnce(() => ({ type: hosting.HostingTypes.SELF }))
-
-    await config.executeMiddleware()
-    expect(config.next).toHaveBeenCalled()
-  })
 })

packages/server/src/middleware/tests/usageQuota.spec.js

@@ -5,7 +5,12 @@ const env = require("../../environment")
 
 jest.mock("../../db")
 jest.mock("../../utilities/usageQuota")
-jest.mock("../../environment")
+jest.mock("../../environment", () => ({
+  isTest: () => true,
+  isProd: () => false,
+  isDev: () => true,
+  _set: () => {},
+}))
 
 class TestConfiguration {
   constructor() {
@@ -32,12 +37,14 @@ class TestConfiguration {
     return this.middleware(this.ctx, this.next)
   }
 
-  cloudHosted(bool) {
+  setProd(bool) {
     if (bool) {
-      env.CLOUD = 1
+      env.isDev = () => false
+      env.isProd = () => true
       this.ctx.auth = { apiKey: "test" }
     } else {
-      env.CLOUD = 0
+      env.isDev = () => true
+      env.isProd = () => false
     }
   }
 
@@ -102,7 +109,7 @@ describe("usageQuota middleware", () => {
 
   it("calculates and persists the correct usage quota for the relevant action", async () => {
     config.setUrl("/rows")
-    config.cloudHosted(true)
+    config.setProd(true)
 
     await config.executeMiddleware()
 
@@ -112,7 +119,7 @@ describe("usageQuota middleware", () => {
 
   it("calculates the correct file size from a file upload call and adds it to quota", async () => {
     config.setUrl("/upload")
-    config.cloudHosted(true)
+    config.setProd(true)
     config.setFiles([
       {
         size: 100

packages/server/src/middleware/usageQuota.js

@@ -44,8 +44,8 @@ module.exports = async (ctx, next) => {
     }
   }
 
-  // if running in builder or a self hosted cloud usage quotas should not be executed
+  // if in development or a self hosted cloud usage quotas should not be executed
-  if (!env.CLOUD || env.SELF_HOSTED) {
+  if (env.isDev() || env.SELF_HOSTED) {
     return next()
   }
   // update usage for uploads to be the total size

packages/server/src/tests/utilities/TestConfiguration.js

@@ -14,9 +14,6 @@ const {
 } = require("./structures")
 const controllers = require("./controllers")
 const supertest = require("supertest")
-const fs = require("fs")
-const { budibaseAppsDir } = require("../../utilities/budibaseDir")
-const { join } = require("path")
 
 const EMAIL = "babs@babs.com"
 const PASSWORD = "babs_password"
@@ -66,13 +63,6 @@ class TestConfiguration {
     if (this.server) {
       this.server.close()
     }
-    const appDir = budibaseAppsDir()
-    const files = fs.readdirSync(appDir)
-    for (let file of files) {
-      if (this.allApps.some(app => file.includes(app._id))) {
-        fs.rmdirSync(join(appDir, file), { recursive: true })
-      }
-    }
   }
 
   defaultHeaders() {
@@ -81,9 +71,11 @@ class TestConfiguration {
       roleId: BUILTIN_ROLE_IDS.BUILDER,
     }
     const builderToken = jwt.sign(builderUser, env.JWT_SECRET)
+    // can be "production" for test case
+    const type = env.isProd() ? "cloud" : "local"
     const headers = {
       Accept: "application/json",
-      Cookie: [`budibase:builder:local=${builderToken}`],
+      Cookie: [`budibase:builder:${type}=${builderToken}`],
     }
     if (this.appId) {
       headers["x-budibase-app-id"] = this.appId

packages/server/src/utilities/builder/compileStaticAssets.js

@@ -1,35 +0,0 @@
-const { ensureDir, constants, copyFile } = require("fs-extra")
-const { join } = require("../centralPath")
-const { budibaseAppsDir } = require("../budibaseDir")
-
-/**
- * Compile all the non-db static web assets that are required for the running of
- * a budibase application. This includes the JSON structure of the DOM and
- * the client library, a script responsible for reading the JSON structure
- * and rendering the application.
- * @param {string} appId id of the application we want to compile static assets for
- */
-module.exports = async appId => {
-  const publicPath = join(budibaseAppsDir(), appId, "public")
-  await ensureDir(publicPath)
-  await copyClientLib(publicPath)
-}
-
-/**
- * Copy the budibase client library and sourcemap from NPM to <appId>/public/.
- * The client library is then served as a static asset when the budibase application
- * is running in preview or prod
- * @param {String} publicPath - path to write the client library to
- */
-const copyClientLib = async publicPath => {
-  const sourcepath = require.resolve("@budibase/client")
-  const destPath = join(publicPath, "budibase-client.js")
-
-  await copyFile(sourcepath, destPath, constants.COPYFILE_FICLONE)
-
-  await copyFile(
-    sourcepath + ".map",
-    destPath + ".map",
-    constants.COPYFILE_FICLONE
-  )
-}

packages/server/src/utilities/builder/hosting.js

@@ -1,5 +1,5 @@
 const CouchDB = require("../../db")
-const { BUILDER_CONFIG_DB, HOSTING_DOC } = require("../../constants")
+const { StaticDatabases } = require("../../db/utils")
 const fetch = require("node-fetch")
 const env = require("../../environment")
 
@@ -23,16 +23,16 @@ exports.HostingTypes = {
 }
 
 exports.getHostingInfo = async () => {
-  const db = new CouchDB(BUILDER_CONFIG_DB)
+  const db = new CouchDB(StaticDatabases.BUILDER_HOSTING.name)
   let doc
   try {
-    doc = await db.get(HOSTING_DOC)
+    doc = await db.get(StaticDatabases.BUILDER_HOSTING.baseDoc)
   } catch (err) {
     // don't write this doc, want to be able to update these default props
     // for our servers with a new release without needing to worry about state of
     // PouchDB in peoples installations
     doc = {
-      _id: HOSTING_DOC,
+      _id: StaticDatabases.BUILDER_HOSTING.baseDoc,
       type: exports.HostingTypes.CLOUD,
       hostingUrl: PROD_HOSTING_URL,
       selfHostKey: "",
@@ -85,15 +85,11 @@ exports.getTemplatesUrl = async (appId, type, name) => {
 }
 
 exports.getDeployedApps = async () => {
-  const hostingInfo = await exports.getHostingInfo()
+  if (!env.SELF_HOSTED) {
-  if (
-    (!env.CLOUD && hostingInfo.type === exports.HostingTypes.CLOUD) ||
-    (env.CLOUD && !env.SELF_HOSTED)
-  ) {
     throw "Can only check apps for self hosted environments"
   }
-  const workerUrl = !env.CLOUD ? await exports.getWorkerUrl() : env.WORKER_URL
+  const workerUrl = env.WORKER_URL
-  const hostingKey = !env.CLOUD ? hostingInfo.selfHostKey : env.HOSTING_KEY
+  const hostingKey = env.HOSTING_KEY
   try {
     const response = await fetch(`${workerUrl}/api/apps`, {
       method: "GET",

packages/server/src/utilities/createAppPackage.js

@@ -1,29 +0,0 @@
-const stream = require("stream")
-const fetch = require("node-fetch")
-const tar = require("tar-fs")
-const zlib = require("zlib")
-const { promisify } = require("util")
-const packageJson = require("../../package.json")
-
-const streamPipeline = promisify(stream.pipeline)
-
-// can't really test this due to the downloading nature of it, wouldn't be a great test case
-/* istanbul ignore next */
-exports.downloadExtractComponentLibraries = async appFolder => {
-  const LIBRARIES = ["standard-components"]
-
-  // Need to download tarballs directly from NPM as our users may not have node on their machine
-  for (let lib of LIBRARIES) {
-    // download tarball
-    const registryUrl = `https://registry.npmjs.org/@budibase/${lib}/-/${lib}-${packageJson.version}.tgz`
-    const response = await fetch(registryUrl)
-    if (!response.ok)
-      throw new Error(`unexpected response ${response.statusText}`)
-
-    await streamPipeline(
-      response.body,
-      zlib.Unzip(),
-      tar.extract(`${appFolder}/node_modules/@budibase/${lib}`)
-    )
-  }
-}

packages/server/src/utilities/fileSystem.js

@@ -1,40 +0,0 @@
-const { budibaseTempDir } = require("./budibaseDir")
-const { isDev } = require("./index")
-const fs = require("fs")
-const { join } = require("path")
-const { downloadTemplate } = require("./templates")
-
-/**
- * The single stack system (Cloud and Builder) should not make use of the file system where possible,
- * this file handles all of the file access for the system with the intention of limiting it all to one
- * place. Keeping all of this logic in one place means that when we need to do file system access (like
- * downloading a package or opening a temporary file) in can be done in way that we can confirm it shouldn't
- * be done through an object store instead.
- */
-
-/**
- * Checks if the system is currently in development mode and if it is makes sure
- * everything required to function is ready.
- */
-exports.checkDevelopmentEnvironment = () => {
-  if (isDev() && !fs.existsSync(budibaseTempDir())) {
-    console.error(
-      "Please run a build before attempting to run server independently to fill 'tmp' directory."
-    )
-    process.exit(-1)
-  }
-}
-
-/**
- * This function manages temporary template files which are stored by Koa.
- * @param {Object} template The template object retrieved from the Koa context object.
- * @returns {Object} Returns an fs read stream which can be loaded into the database.
- */
-exports.getTemplateStream = async template => {
-  if (template.file) {
-    return fs.createReadStream(template.file.path)
-  } else {
-    const templatePath = await downloadTemplate(...template.key.split("/"))
-    return fs.createReadStream(join(templatePath, "db", "dump.txt"))
-  }
-}

packages/server/src/utilities/fileSystem/index.js

@@ -0,0 +1,204 @@
+const { budibaseTempDir } = require("../budibaseDir")
+const { isDev } = require("../index")
+const fs = require("fs")
+const { join } = require("path")
+const uuid = require("uuid/v4")
+const CouchDB = require("../../db")
+const { ObjectStoreBuckets } = require("../../constants")
+const {
+  upload,
+  retrieve,
+  streamUpload,
+  deleteFolder,
+  downloadTarball,
+} = require("./utilities")
+const { downloadLibraries, newAppPublicPath } = require("./newApp")
+const download = require("download")
+const env = require("../../environment")
+const { homedir } = require("os")
+const fetch = require("node-fetch")
+
+const DEFAULT_AUTOMATION_BUCKET =
+  "https://prod-budi-automations.s3-eu-west-1.amazonaws.com"
+const DEFAULT_AUTOMATION_DIRECTORY = ".budibase-automations"
+
+/**
+ * The single stack system (Cloud and Builder) should not make use of the file system where possible,
+ * this file handles all of the file access for the system with the intention of limiting it all to one
+ * place. Keeping all of this logic in one place means that when we need to do file system access (like
+ * downloading a package or opening a temporary file) in can be done in way that we can confirm it shouldn't
+ * be done through an object store instead.
+ */
+
+/**
+ * Upon first startup of instance there may not be everything we need in tmp directory, set it up.
+ */
+exports.init = () => {
+  const tempDir = budibaseTempDir()
+  if (!fs.existsSync(tempDir)) {
+    fs.mkdirSync(tempDir)
+  }
+}
+
+/**
+ * Checks if the system is currently in development mode and if it is makes sure
+ * everything required to function is ready.
+ */
+exports.checkDevelopmentEnvironment = () => {
+  if (!isDev()) {
+    return
+  }
+  let error
+  if (!fs.existsSync(budibaseTempDir())) {
+    error =
+      "Please run a build before attempting to run server independently to fill 'tmp' directory."
+  }
+  if (!fs.existsSync(join(process.cwd(), ".env"))) {
+    error = "Must run via yarn once to generate environment."
+  }
+  if (error) {
+    console.error(error)
+    process.exit(-1)
+  }
+}
+
+/**
+ * This function manages temporary template files which are stored by Koa.
+ * @param {Object} template The template object retrieved from the Koa context object.
+ * @returns {Object} Returns an fs read stream which can be loaded into the database.
+ */
+exports.getTemplateStream = async template => {
+  if (template.file) {
+    return fs.createReadStream(template.file.path)
+  } else {
+    const tmpPath = await exports.downloadTemplate(...template.key.split("/"))
+    return fs.createReadStream(join(tmpPath, "db", "dump.txt"))
+  }
+}
+
+/**
+ * Used to retrieve a handlebars file from the system which will be used as a template.
+ * This is allowable as the template handlebars files should be static and identical across
+ * the cluster.
+ * @param {string} path The path to the handlebars file which is to be loaded.
+ * @returns {string} The loaded handlebars file as a string - loaded as utf8.
+ */
+exports.loadHandlebarsFile = path => {
+  return fs.readFileSync(path, "utf8")
+}
+
+/**
+ * When return a file from the API need to write the file to the system temporarily so we
+ * can create a read stream to send.
+ * @param {string} contents the contents of the file which is to be returned from the API.
+ * @return {Object} the read stream which can be put into the koa context body.
+ */
+exports.apiFileReturn = contents => {
+  const path = join(budibaseTempDir(), uuid())
+  fs.writeFileSync(path, contents)
+  return fs.createReadStream(path)
+}
+
+/**
+ * Takes a copy of the database state for an app to the object store.
+ * @param {string} appId The ID of the app which is to be backed up.
+ * @param {string} backupName The name of the backup located in the object store.
+ * @return The backup has been completed when this promise completes and returns a file stream
+ * to the temporary backup file (to return via API if required).
+ */
+exports.performBackup = async (appId, backupName) => {
+  const path = join(budibaseTempDir(), backupName)
+  const writeStream = fs.createWriteStream(path)
+  // perform couch dump
+  const instanceDb = new CouchDB(appId)
+  await instanceDb.dump(writeStream, {})
+  // write the file to the object store
+  await streamUpload(
+    ObjectStoreBuckets.BACKUPS,
+    join(appId, backupName),
+    fs.createReadStream(path)
+  )
+  return fs.createReadStream(path)
+}
+
+/**
+ * Downloads required libraries and creates a new path in the object store.
+ * @param {string} appId The ID of the app which is being created.
+ * @return {Promise<void>} once promise completes app resources should be ready in object store.
+ */
+exports.createApp = async appId => {
+  await downloadLibraries(appId)
+  await newAppPublicPath(appId)
+}
+
+/**
+ * Removes all of the assets created for an app in the object store.
+ * @param {string} appId The ID of the app which is being deleted.
+ * @return {Promise<void>} once promise completes the app resources will be removed from object store.
+ */
+exports.deleteApp = async appId => {
+  await deleteFolder(ObjectStoreBuckets.APPS, `${appId}/`)
+}
+
+/**
+ * Retrieves a template and pipes it to minio as well as making it available temporarily.
+ * @param {string} type The type of template which is to be retrieved.
+ * @param name
+ * @return {Promise<*>}
+ */
+exports.downloadTemplate = async (type, name) => {
+  const DEFAULT_TEMPLATES_BUCKET =
+    "prod-budi-templates.s3-eu-west-1.amazonaws.com"
+  const templateUrl = `https://${DEFAULT_TEMPLATES_BUCKET}/templates/${type}/${name}.tar.gz`
+  return downloadTarball(templateUrl, ObjectStoreBuckets.TEMPLATES, type)
+}
+
+/**
+ * Retrieves component libraries from object store (or tmp symlink if in local)
+ */
+exports.getComponentLibraryManifest = async (appId, library) => {
+  const path = join(appId, "node_modules", library, "package", "manifest.json")
+  let resp = await retrieve(ObjectStoreBuckets.APPS, path)
+  if (typeof resp !== "string") {
+    resp = resp.toString("utf8")
+  }
+  return JSON.parse(resp)
+}
+
+exports.automationInit = async () => {
+  const directory =
+    env.AUTOMATION_DIRECTORY || join(homedir(), DEFAULT_AUTOMATION_DIRECTORY)
+  const bucket = env.AUTOMATION_BUCKET || DEFAULT_AUTOMATION_BUCKET
+  if (!fs.existsSync(directory)) {
+    fs.mkdirSync(directory, { recursive: true })
+  }
+  // env setup to get async packages
+  let response = await fetch(`${bucket}/manifest.json`)
+  return response.json()
+}
+
+exports.getExternalAutomationStep = async (name, version, bundleName) => {
+  const directory =
+    env.AUTOMATION_DIRECTORY || join(homedir(), DEFAULT_AUTOMATION_DIRECTORY)
+  const bucket = env.AUTOMATION_BUCKET || DEFAULT_AUTOMATION_BUCKET
+  try {
+    return require(join(directory, bundleName))
+  } catch (err) {
+    await download(`${bucket}/${name}/${version}/${bundleName}`, directory)
+    return require(join(directory, bundleName))
+  }
+}
+
+/**
+ * All file reads come through here just to make sure all of them make sense
+ * allows a centralised location to check logic is all good.
+ */
+exports.readFileSync = (filepath, options = "utf8") => {
+  return fs.readFileSync(filepath, options)
+}
+
+/**
+ * Full function definition for below can be found in the utilities.
+ */
+exports.upload = upload
+exports.retrieve = retrieve

packages/server/src/utilities/fileSystem/newApp.js

@@ -0,0 +1,34 @@
+const packageJson = require("../../../package.json")
+const { join } = require("path")
+const { ObjectStoreBuckets } = require("../../constants")
+const { streamUpload, downloadTarball } = require("./utilities")
+const fs = require("fs")
+
+const BUCKET_NAME = ObjectStoreBuckets.APPS
+
+// can't really test this due to the downloading nature of it, wouldn't be a great test case
+/* istanbul ignore next */
+exports.downloadLibraries = async appId => {
+  const LIBRARIES = ["standard-components"]
+
+  // Need to download tarballs directly from NPM as our users may not have node on their machine
+  for (let lib of LIBRARIES) {
+    // download tarball
+    const registryUrl = `https://registry.npmjs.org/@budibase/${lib}/-/${lib}-${packageJson.version}.tgz`
+    const path = join(appId, "node_modules", "@budibase", lib)
+    await downloadTarball(registryUrl, BUCKET_NAME, path)
+  }
+}
+
+exports.newAppPublicPath = async appId => {
+  const path = join(appId, "public")
+  const sourcepath = require.resolve("@budibase/client")
+  const destPath = join(path, "budibase-client.js")
+
+  await streamUpload(BUCKET_NAME, destPath, fs.createReadStream(sourcepath))
+  await streamUpload(
+    BUCKET_NAME,
+    destPath + ".map",
+    fs.createReadStream(sourcepath + ".map")
+  )
+}

packages/server/src/utilities/fileSystem/processor.js

@@ -1,25 +1,20 @@
-const fs = require("fs")
 const jimp = require("jimp")
-const fsPromises = fs.promises
 
 const FORMATS = {
   IMAGES: ["png", "jpg", "jpeg", "gif", "bmp", "tiff"],
 }
 
 function processImage(file) {
+  // this will overwrite the temp file
   return jimp.read(file.path).then(img => {
-    return img.resize(300, jimp.AUTO).write(file.outputPath)
+    return img.resize(300, jimp.AUTO).write(file.path)
   })
 }
 
 async function process(file) {
   if (FORMATS.IMAGES.includes(file.extension.toLowerCase())) {
     await processImage(file)
-    return file
   }
-
-  // No processing required
-  await fsPromises.copyFile(file.path, file.outputPath)
   return file
 }
 

packages/server/src/utilities/fileSystem/utilities.js

@@ -0,0 +1,221 @@
+const sanitize = require("sanitize-s3-objectkey")
+const AWS = require("aws-sdk")
+const stream = require("stream")
+const fetch = require("node-fetch")
+const tar = require("tar-fs")
+const zlib = require("zlib")
+const { promisify } = require("util")
+const { join } = require("path")
+const fs = require("fs")
+const { budibaseTempDir } = require("../budibaseDir")
+const env = require("../../environment")
+const { ObjectStoreBuckets } = require("../../constants")
+
+const streamPipeline = promisify(stream.pipeline)
+
+const CONTENT_TYPE_MAP = {
+  html: "text/html",
+  css: "text/css",
+  js: "application/javascript",
+}
+const STRING_CONTENT_TYPES = [
+  CONTENT_TYPE_MAP.html,
+  CONTENT_TYPE_MAP.css,
+  CONTENT_TYPE_MAP.js,
+]
+
+function publicPolicy(bucketName) {
+  return {
+    Version: "2012-10-17",
+    Statement: [
+      {
+        Effect: "Allow",
+        Principal: {
+          AWS: ["*"],
+        },
+        Action: "s3:GetObject",
+        Resource: [`arn:aws:s3:::${bucketName}/*`],
+      },
+    ],
+  }
+}
+
+const PUBLIC_BUCKETS = [ObjectStoreBuckets.APPS]
+
+/**
+ * Gets a connection to the object store using the S3 SDK.
+ * @param {string} bucket the name of the bucket which blobs will be uploaded/retrieved from.
+ * @return {Object} an S3 object store object, check S3 Nodejs SDK for usage.
+ * @constructor
+ */
+exports.ObjectStore = bucket => {
+  if (env.SELF_HOSTED) {
+    AWS.config.update({
+      accessKeyId: env.MINIO_ACCESS_KEY,
+      secretAccessKey: env.MINIO_SECRET_KEY,
+    })
+  }
+  const config = {
+    s3ForcePathStyle: true,
+    signatureVersion: "v4",
+    params: {
+      Bucket: bucket,
+    },
+  }
+  if (env.MINIO_URL) {
+    config.endpoint = env.MINIO_URL
+  }
+  return new AWS.S3(config)
+}
+
+/**
+ * Given an object store and a bucket name this will make sure the bucket exists,
+ * if it does not exist then it will create it.
+ */
+exports.makeSureBucketExists = async (client, bucketName) => {
+  try {
+    await client
+      .headBucket({
+        Bucket: bucketName,
+      })
+      .promise()
+  } catch (err) {
+    // bucket doesn't exist create it
+    if (err.statusCode === 404) {
+      await client
+        .createBucket({
+          Bucket: bucketName,
+        })
+        .promise()
+      // public buckets are quite hidden in the system, make sure
+      // no bucket is set accidentally
+      if (PUBLIC_BUCKETS.includes(bucketName)) {
+        await client
+          .putBucketPolicy({
+            Bucket: bucketName,
+            Policy: JSON.stringify(publicPolicy(bucketName)),
+          })
+          .promise()
+      }
+    } else {
+      throw err
+    }
+  }
+}
+
+/**
+ * Uploads the contents of a file given the required parameters, useful when
+ * temp files in use (for example file uploaded as an attachment).
+ */
+exports.upload = async ({ bucket, filename, path, type, metadata }) => {
+  const extension = [...filename.split(".")].pop()
+  const fileBytes = fs.readFileSync(path)
+
+  const objectStore = exports.ObjectStore(bucket)
+  await exports.makeSureBucketExists(objectStore, bucket)
+
+  const config = {
+    // windows file paths need to be converted to forward slashes for s3
+    Key: sanitize(filename).replace(/\\/g, "/"),
+    Body: fileBytes,
+    ContentType: type || CONTENT_TYPE_MAP[extension.toLowerCase()],
+  }
+  if (metadata) {
+    config.Metadata = metadata
+  }
+  return objectStore.upload(config).promise()
+}
+
+/**
+ * Similar to the upload function but can be used to send a file stream
+ * through to the object store.
+ */
+exports.streamUpload = async (bucket, filename, stream) => {
+  const objectStore = exports.ObjectStore(bucket)
+  await exports.makeSureBucketExists(objectStore, bucket)
+
+  const params = {
+    Bucket: bucket,
+    Key: sanitize(filename).replace(/\\/g, "/"),
+    Body: stream,
+  }
+  return objectStore.upload(params).promise()
+}
+
+/**
+ * retrieves the contents of a file from the object store, if it is a known content type it
+ * will be converted, otherwise it will be returned as a buffer stream.
+ */
+exports.retrieve = async (bucket, filename) => {
+  const objectStore = exports.ObjectStore(bucket)
+  const params = {
+    Bucket: bucket,
+    Key: sanitize(filename).replace(/\\/g, "/"),
+  }
+  const response = await objectStore.getObject(params).promise()
+  // currently these are all strings
+  if (STRING_CONTENT_TYPES.includes(response.ContentType)) {
+    return response.Body.toString("utf8")
+  } else {
+    return response.Body
+  }
+}
+
+exports.deleteFolder = async (bucket, folder) => {
+  const client = exports.ObjectStore(bucket)
+  const listParams = {
+    Bucket: bucket,
+    Prefix: folder,
+  }
+
+  let response = await client.listObjects(listParams).promise()
+  if (response.Contents.length === 0) {
+    return
+  }
+  const deleteParams = {
+    Bucket: bucket,
+    Delete: {
+      Objects: [],
+    },
+  }
+
+  response.Contents.forEach(content => {
+    deleteParams.Delete.Objects.push({ Key: content.Key })
+  })
+
+  response = await client.deleteObjects(deleteParams).promise()
+  // can only empty 1000 items at once
+  if (response.Deleted.length === 1000) {
+    return exports.deleteFolder(bucket, folder)
+  }
+}
+
+exports.uploadDirectory = async (bucket, localPath, bucketPath) => {
+  let uploads = []
+  const files = fs.readdirSync(localPath, { withFileTypes: true })
+  for (let file of files) {
+    const path = join(bucketPath, file.name)
+    const local = join(localPath, file.name)
+    if (file.isDirectory()) {
+      uploads.push(exports.uploadDirectory(bucket, local, path))
+    } else {
+      uploads.push(
+        exports.streamUpload(bucket, path, fs.createReadStream(local))
+      )
+    }
+  }
+  await Promise.all(uploads)
+}
+
+exports.downloadTarball = async (url, bucket, path) => {
+  const response = await fetch(url)
+  if (!response.ok) {
+    throw new Error(`unexpected response ${response.statusText}`)
+  }
+
+  const tmpPath = join(budibaseTempDir(), path)
+  await streamPipeline(response.body, zlib.Unzip(), tar.extract(tmpPath))
+  await exports.uploadDirectory(bucket, tmpPath, path)
+  // return the temporary path incase there is a use for it
+  return tmpPath
+}

packages/server/src/utilities/index.js

@@ -1,6 +1,5 @@
 const env = require("../environment")
 const { DocumentTypes, SEPARATOR } = require("../db/utils")
-const fs = require("fs")
 const CouchDB = require("../db")
 
 const APP_PREFIX = DocumentTypes.APP + SEPARATOR
@@ -13,14 +12,7 @@ function confirmAppId(possibleAppId) {
 
 exports.wait = ms => new Promise(resolve => setTimeout(resolve, ms))
 
-exports.isDev = () => {
+exports.isDev = env.isDev
-  return (
-    !env.CLOUD &&
-    env.NODE_ENV !== "production" &&
-    env.NODE_ENV !== "jest" &&
-    env.NODE_ENV !== "cypress"
-  )
-}
 
 /**
  * Given a request tries to find the appId, which can be located in various places
@@ -28,10 +20,18 @@ exports.isDev = () => {
  * @returns {string|undefined} If an appId was found it will be returned.
  */
 exports.getAppId = ctx => {
-  let appId = confirmAppId(ctx.headers["x-budibase-app-id"])
+  const options = [ctx.headers["x-budibase-app-id"], ctx.params.appId]
-  if (!appId) {
+  if (ctx.subdomains) {
-    appId = confirmAppId(env.CLOUD ? ctx.subdomains[1] : ctx.params.appId)
+    options.push(ctx.subdomains[1])
   }
+  let appId
+  for (let option of options) {
+    appId = confirmAppId(option)
+    if (appId) {
+      break
+    }
+  }
+
   // look in body if can't find it in subdomain
   if (!appId && ctx.request.body && ctx.request.body.appId) {
     appId = confirmAppId(ctx.request.body.appId)
@@ -51,7 +51,7 @@ exports.getAppId = ctx => {
  * @returns {string} The name of the token trying to find
  */
 exports.getCookieName = (name = "builder") => {
-  let environment = env.CLOUD ? "cloud" : "local"
+  let environment = env.isProd() ? "cloud" : "local"
   return `budibase:${name}:${environment}`
 }
 
@@ -89,24 +89,6 @@ exports.isClient = ctx => {
   return ctx.headers["x-budibase-type"] === "client"
 }
 
-/**
- * Recursively walk a directory tree and execute a callback on all files.
- * @param {String} dirPath - Directory to traverse
- * @param {Function} callback - callback to execute on files
- */
-exports.walkDir = (dirPath, callback) => {
-  for (let filename of fs.readdirSync(dirPath)) {
-    const filePath = `${dirPath}/${filename}`
-    const stat = fs.lstatSync(filePath)
-
-    if (stat.isFile()) {
-      callback(filePath)
-    } else {
-      exports.walkDir(filePath, callback)
-    }
-  }
-}
-
 exports.getLogoUrl = () => {
   return "https://d33wubrfki0l68.cloudfront.net/aac32159d7207b5085e74a7ef67afbb7027786c5/2b1fd/img/logo/bb-emblem.svg"
 }

packages/server/src/utilities/rowProcessor.js

@@ -180,7 +180,7 @@ exports.outputProcessing = async (appId, table, rows) => {
     rows
   )
   // update the attachments URL depending on hosting
-  if (env.CLOUD && env.SELF_HOSTED) {
+  if (env.isProd() && env.SELF_HOSTED) {
     for (let [property, column] of Object.entries(table.schema)) {
       if (column.type === FieldTypes.ATTACHMENT) {
         for (let row of outputRows) {

packages/server/src/utilities/templates.js

@@ -1,81 +0,0 @@
-const fs = require("fs-extra")
-const { join } = require("./centralPath")
-const os = require("os")
-const fetch = require("node-fetch")
-const stream = require("stream")
-const tar = require("tar-fs")
-const zlib = require("zlib")
-const { promisify } = require("util")
-const streamPipeline = promisify(stream.pipeline)
-const { budibaseAppsDir } = require("./budibaseDir")
-const env = require("../environment")
-const CouchDB = require("../db")
-
-const DEFAULT_TEMPLATES_BUCKET =
-  "prod-budi-templates.s3-eu-west-1.amazonaws.com"
-
-exports.getLocalTemplates = function() {
-  const templatesDir = join(os.homedir(), ".budibase", "templates", "app")
-  const templateObj = { app: {} }
-  fs.ensureDirSync(templatesDir)
-  const templateNames = fs.readdirSync(templatesDir)
-  for (let name of templateNames) {
-    templateObj.app[name] = {
-      name,
-      category: "local",
-      description: "local template",
-      type: "app",
-      key: `app/${name}`,
-    }
-  }
-  return templateObj
-}
-
-// can't really test this, downloading is just not something we should do in a behavioural test
-/* istanbul ignore next */
-exports.downloadTemplate = async function(type, name) {
-  const dirName = join(budibaseAppsDir(), "templates", type, name)
-  if (env.LOCAL_TEMPLATES) {
-    return dirName
-  }
-  const templateUrl = `https://${DEFAULT_TEMPLATES_BUCKET}/templates/${type}/${name}.tar.gz`
-  const response = await fetch(templateUrl)
-
-  if (!response.ok) {
-    throw new Error(
-      `Error downloading template ${type}:${name}: ${response.statusText}`
-    )
-  }
-
-  // stream the response, unzip and extract
-  await streamPipeline(
-    response.body,
-    zlib.Unzip(),
-    tar.extract(join(budibaseAppsDir(), "templates", type))
-  )
-
-  return dirName
-}
-
-async function performDump({ dir, appId, name = "dump.txt" }) {
-  const writeStream = fs.createWriteStream(join(dir, name))
-  // perform couch dump
-  const instanceDb = new CouchDB(appId)
-  await instanceDb.dump(writeStream, {})
-}
-
-exports.performDump = performDump
-
-exports.exportTemplateFromApp = async function({ templateName, appId }) {
-  // Copy frontend files
-  const templatesDir = join(
-    budibaseAppsDir(),
-    "templates",
-    "app",
-    templateName,
-    "db"
-  )
-  fs.ensureDirSync(templatesDir)
-  await performDump({ dir: templatesDir, appId })
-  return templatesDir
-}

packages/server/src/utilities/tests/csvParser.spec.js

@@ -1,4 +1,4 @@
-const fs = require("fs")
+const { readFileSync } = require("../fileSystem")
 const csvParser = require("../csvParser")
 
 const CSV_PATH = __dirname + "/test.csv"
@@ -33,7 +33,7 @@ const SCHEMAS = {
 }
 
 describe("CSV Parser", () => {
-  const csvString = fs.readFileSync(CSV_PATH, "utf8")
+  const csvString = readFileSync(CSV_PATH, "utf8")
 
   describe("parsing", () => {
     it("returns status and types for a valid CSV transformation", async () => {

packages/server/src/utilities/usageQuota.js

@@ -50,7 +50,7 @@ exports.Properties = {
 }
 
 exports.getAPIKey = async appId => {
-  if (env.SELF_HOSTED) {
+  if (!env.USE_QUOTAS) {
     return { apiKey: null }
   }
   return apiKeyTable.get({ primary: appId })
@@ -65,8 +65,7 @@ exports.getAPIKey = async appId => {
  * also been reset after this call.
  */
 exports.update = async (apiKey, property, usage) => {
-  // don't try validate in builder
+  if (!env.USE_QUOTAS) {
-  if (!env.CLOUD || env.SELF_HOSTED) {
     return
   }
   try {