From 95ee61500655c7ab08f0a58d894b5cf28f4c9857 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Fri, 19 Mar 2021 19:07:47 +0000
Subject: [PATCH] Work in progress, have refactored the templating system to
 remove local templates, application and template system now work through
 minio with no file access.

---
 packages/server/scripts/exportAppTemplate.js  |  21 ++--
 .../server/src/api/controllers/application.js |  34 ++---
 packages/server/src/api/controllers/backup.js |  22 +---
 .../src/api/controllers/static/index.js       |  91 ++------------
 .../server/src/api/controllers/templates.js   |  41 ++-----
 .../server/src/api/controllers/view/index.js  |  10 +-
 packages/server/src/api/routes/templates.js   |   1 -
 .../src/api/routes/tests/templates.spec.js    |  22 ----
 packages/server/src/constants/index.js        |   6 +
 .../utilities/builder/compileStaticAssets.js  |  35 ------
 .../server/src/utilities/createAppPackage.js  |  29 -----
 packages/server/src/utilities/fileSystem.js   |  40 ------
 .../server/src/utilities/fileSystem/index.js  |  98 +++++++++++++++
 .../server/src/utilities/fileSystem/newApp.js |  34 +++++
 .../src/utilities/fileSystem/utilities.js     | 116 ++++++++++++++++++
 packages/server/src/utilities/templates.js    |  68 +---------
 16 files changed, 303 insertions(+), 365 deletions(-)
 delete mode 100644 packages/server/src/utilities/builder/compileStaticAssets.js
 delete mode 100644 packages/server/src/utilities/createAppPackage.js
 delete mode 100644 packages/server/src/utilities/fileSystem.js
 create mode 100644 packages/server/src/utilities/fileSystem/index.js
 create mode 100644 packages/server/src/utilities/fileSystem/newApp.js
 create mode 100644 packages/server/src/utilities/fileSystem/utilities.js

diff --git a/packages/server/scripts/exportAppTemplate.js b/packages/server/scripts/exportAppTemplate.js
index e896917d5b..c6d738225c 100755
--- a/packages/server/scripts/exportAppTemplate.js
+++ b/packages/server/scripts/exportAppTemplate.js
@@ -1,6 +1,9 @@
 #!/usr/bin/env node
-const { exportTemplateFromApp } = require("../src/utilities/templates")
 const yargs = require("yargs")
+const fs = require("fs")
+const { join } = require("path")
+const CouchDB = require("../src/db")
+const { budibaseAppsDir } = require("../src/utilities/budibaseDir")
 
 // Script to export a chosen budibase app into a package
 // Usage: ./scripts/exportAppTemplate.js export --name=Funky --appId=appId
@@ -22,18 +25,22 @@ yargs
       },
     },
     async args => {
+      const name = args.name,
+        appId = args.appId
       console.log("Exporting app..")
-      if (args.name == null || args.appId == null) {
+      if (name == null || appId == null) {
         console.error(
           "Unable to export without a name and app ID being specified, check help for more info."
         )
         return
       }
-      const exportPath = await exportTemplateFromApp({
-        templateName: args.name,
-        appId: args.appId,
-      })
-      console.log(`Template ${args.name} exported to ${exportPath}`)
+      const exportPath = join(budibaseAppsDir(), "templates", "app", name, "db")
+      fs.ensureDirSync(exportPath)
+      const writeStream = fs.createWriteStream(join(exportPath, "dump.text"))
+      // perform couch dump
+      const instanceDb = new CouchDB(appId)
+      await instanceDb.dump(writeStream, {})
+      console.log(`Template ${name} exported to ${exportPath}`)
     }
   )
   .help()
diff --git a/packages/server/src/api/controllers/application.js b/packages/server/src/api/controllers/application.js
index 89e233ac5c..e2bb15dbd3 100644
--- a/packages/server/src/api/controllers/application.js
+++ b/packages/server/src/api/controllers/application.js
@@ -1,15 +1,14 @@
 const CouchDB = require("../../db")
-const compileStaticAssets = require("../../utilities/builder/compileStaticAssets")
 const env = require("../../environment")
-const { existsSync } = require("fs-extra")
-const { budibaseAppsDir } = require("../../utilities/budibaseDir")
 const setBuilderToken = require("../../utilities/builder/setBuilderToken")
-const fs = require("fs-extra")
-const { join, resolve } = require("../../utilities/centralPath")
 const packageJson = require("../../../package.json")
 const { createLinkView } = require("../../db/linkedRows")
 const { createRoutingView } = require("../../utilities/routing")
-const { getTemplateStream } = require("../../utilities/fileSystem")
+const {
+  getTemplateStream,
+  createApp,
+  deleteApp,
+} = require("../../utilities/fileSystem")
 const {
   generateAppID,
   getLayoutParams,
@@ -20,9 +19,6 @@ const {
   BUILTIN_ROLE_IDS,
   AccessController,
 } = require("../../utilities/security/roles")
-const {
-  downloadExtractComponentLibraries,
-} = require("../../utilities/createAppPackage")
 const { BASE_LAYOUTS } = require("../../constants/layouts")
 const {
   createHomeScreen,
@@ -181,10 +177,10 @@ exports.create = async function(ctx) {
   const instanceDb = new CouchDB(appId)
   await instanceDb.put(newApplication)
 
-  const newAppFolder = await createEmptyAppPackage(ctx, newApplication)
+  await createEmptyAppPackage(ctx, newApplication)
   /* istanbul ignore next */
   if (env.NODE_ENV !== "jest") {
-    await downloadExtractComponentLibraries(newAppFolder)
+    await createApp(appId)
   }
 
   await setBuilderToken(ctx, appId, version)
@@ -214,10 +210,7 @@ exports.delete = async function(ctx) {
   const app = await db.get(ctx.params.appId)
   const result = await db.destroy()
 
-  // remove top level directory
-  await fs.rmdir(join(budibaseAppsDir(), ctx.params.appId), {
-    recursive: true,
-  })
+  await deleteApp(ctx.params.appId)
 
   ctx.status = 200
   ctx.message = `Application ${app.name} deleted successfully.`
@@ -225,17 +218,8 @@ exports.delete = async function(ctx) {
 }
 
 const createEmptyAppPackage = async (ctx, app) => {
-  const appsFolder = budibaseAppsDir()
-  const newAppFolder = resolve(appsFolder, app._id)
-
   const db = new CouchDB(app._id)
 
-  if (existsSync(newAppFolder)) {
-    ctx.throw(400, "App folder already exists for this application")
-  }
-
-  fs.mkdirpSync(newAppFolder)
-
   let screensAndLayouts = []
   for (let layout of BASE_LAYOUTS) {
     const cloned = cloneDeep(layout)
@@ -251,6 +235,4 @@ const createEmptyAppPackage = async (ctx, app) => {
   screensAndLayouts.push(loginScreen)
 
   await db.bulkDocs(screensAndLayouts)
-  await compileStaticAssets(app._id)
-  return newAppFolder
 }
diff --git a/packages/server/src/api/controllers/backup.js b/packages/server/src/api/controllers/backup.js
index a83f96165b..02be10bbec 100644
--- a/packages/server/src/api/controllers/backup.js
+++ b/packages/server/src/api/controllers/backup.js
@@ -1,28 +1,10 @@
-const { performDump } = require("../../utilities/templates")
-const path = require("path")
-const os = require("os")
-const fs = require("fs-extra")
+const { performBackup } = require("../../utilities/fileSystem")
 
 exports.exportAppDump = async function(ctx) {
   const { appId } = ctx.query
-
   const appname = decodeURI(ctx.query.appname)
-
-  const backupsDir = path.join(os.homedir(), ".budibase", "backups")
-  fs.ensureDirSync(backupsDir)
-
   const backupIdentifier = `${appname}Backup${new Date().getTime()}.txt`
 
-  await performDump({
-    dir: backupsDir,
-    appId,
-    name: backupIdentifier,
-  })
-
-  ctx.status = 200
-
-  const backupFile = path.join(backupsDir, backupIdentifier)
-
   ctx.attachment(backupIdentifier)
-  ctx.body = fs.createReadStream(backupFile)
+  ctx.body = await performBackup(appId, backupIdentifier)
 }
diff --git a/packages/server/src/api/controllers/static/index.js b/packages/server/src/api/controllers/static/index.js
index 24cee67146..466e680a7d 100644
--- a/packages/server/src/api/controllers/static/index.js
+++ b/packages/server/src/api/controllers/static/index.js
@@ -3,7 +3,6 @@ require("svelte/register")
 const send = require("koa-send")
 const { resolve, join } = require("../../../utilities/centralPath")
 const fetch = require("node-fetch")
-const fs = require("fs-extra")
 const uuid = require("uuid")
 const AWS = require("aws-sdk")
 const { prepareUpload } = require("../deploy/utils")
@@ -15,7 +14,7 @@ const {
 const { getDeployedApps } = require("../../../utilities/builder/hosting")
 const CouchDB = require("../../../db")
 const setBuilderToken = require("../../../utilities/builder/setBuilderToken")
-const fileProcessor = require("../../../utilities/fileProcessor")
+const { loadHandlebarsFile } = require("../../../utilities/fileSystem")
 const env = require("../../../environment")
 const { OBJ_STORE_DIRECTORY } = require("../../../constants")
 
@@ -57,88 +56,24 @@ exports.uploadFile = async function(ctx) {
       ? Array.from(ctx.request.files.file)
       : [ctx.request.files.file]
 
-  const attachmentsPath = resolve(
-    budibaseAppsDir(),
-    ctx.user.appId,
-    "attachments"
-  )
-
-  if (env.CLOUD) {
-    // remote upload
-    const s3 = new AWS.S3({
-      params: {
-        Bucket: "prod-budi-app-assets",
-      },
-    })
-
-    const uploads = files.map(file => {
-      const fileExtension = [...file.name.split(".")].pop()
-      const processedFileName = `${uuid.v4()}.${fileExtension}`
-
-      return prepareUpload({
-        file,
-        s3Key: `assets/${ctx.user.appId}/attachments/${processedFileName}`,
-        s3,
-      })
-    })
-
-    ctx.body = await Promise.all(uploads)
-    return
-  }
-
-  ctx.body = await processLocalFileUploads({
-    files,
-    outputPath: attachmentsPath,
-    appId: ctx.user.appId,
+  const s3 = new AWS.S3({
+    params: {
+      Bucket: "prod-budi-app-assets",
+    },
   })
-}
 
-async function processLocalFileUploads({ files, outputPath, appId }) {
-  // create attachments dir if it doesnt exist
-  !fs.existsSync(outputPath) && fs.mkdirSync(outputPath, { recursive: true })
-
-  const filesToProcess = files.map(file => {
+  const uploads = files.map(file => {
     const fileExtension = [...file.name.split(".")].pop()
-    // filenames converted to UUIDs so they are unique
     const processedFileName = `${uuid.v4()}.${fileExtension}`
 
-    return {
-      name: file.name,
-      path: file.path,
-      size: file.size,
-      type: file.type,
-      processedFileName,
-      extension: fileExtension,
-      outputPath: join(outputPath, processedFileName),
-      url: join("/attachments", processedFileName),
-    }
+    return prepareUpload({
+      file,
+      s3Key: `assets/${ctx.user.appId}/attachments/${processedFileName}`,
+      s3,
+    })
   })
 
-  const fileProcessOperations = filesToProcess.map(fileProcessor.process)
-
-  const processedFiles = await Promise.all(fileProcessOperations)
-
-  let pendingFileUploads
-  // local document used to track which files need to be uploaded
-  // db.get throws an error if the document doesn't exist
-  // need to use a promise to default
-  const db = new CouchDB(appId)
-  await db
-    .get("_local/fileuploads")
-    .then(data => {
-      pendingFileUploads = data
-    })
-    .catch(() => {
-      pendingFileUploads = { _id: "_local/fileuploads", uploads: [] }
-    })
-
-  pendingFileUploads.uploads = [
-    ...processedFiles,
-    ...pendingFileUploads.uploads,
-  ]
-  await db.put(pendingFileUploads)
-
-  return processedFiles
+  ctx.body = await Promise.all(uploads)
 }
 
 exports.serveApp = async function(ctx) {
@@ -157,7 +92,7 @@ exports.serveApp = async function(ctx) {
     objectStoreUrl: objectStoreUrl(),
   })
 
-  const appHbs = fs.readFileSync(`${__dirname}/templates/app.hbs`, "utf8")
+  const appHbs = loadHandlebarsFile(`${__dirname}/templates/app.hbs`)
   ctx.body = await processString(appHbs, {
     head,
     body: html,
diff --git a/packages/server/src/api/controllers/templates.js b/packages/server/src/api/controllers/templates.js
index c3cfa28706..4d55bc5957 100644
--- a/packages/server/src/api/controllers/templates.js
+++ b/packages/server/src/api/controllers/templates.js
@@ -1,10 +1,5 @@
 const fetch = require("node-fetch")
-const {
-  downloadTemplate,
-  exportTemplateFromApp,
-  getLocalTemplates,
-} = require("../../utilities/templates")
-const env = require("../../environment")
+const { downloadTemplate } = require("../../utilities/fileSystem")
 
 // development flag, can be used to test against templates exported locally
 const DEFAULT_TEMPLATES_BUCKET =
@@ -12,16 +7,11 @@ const DEFAULT_TEMPLATES_BUCKET =
 
 exports.fetch = async function(ctx) {
   const { type = "app" } = ctx.query
-
-  if (env.LOCAL_TEMPLATES) {
-    ctx.body = Object.values(getLocalTemplates()[type])
-  } else {
-    const response = await fetch(
-      `https://${DEFAULT_TEMPLATES_BUCKET}/manifest.json`
-    )
-    const json = await response.json()
-    ctx.body = Object.values(json.templates[type])
-  }
+  const response = await fetch(
+    `https://${DEFAULT_TEMPLATES_BUCKET}/manifest.json`
+  )
+  const json = await response.json()
+  ctx.body = Object.values(json.templates[type])
 }
 
 // can't currently test this, have to ignore from coverage
@@ -29,26 +19,9 @@ exports.fetch = async function(ctx) {
 exports.downloadTemplate = async function(ctx) {
   const { type, name } = ctx.params
 
-  if (!env.LOCAL_TEMPLATES) {
-    await downloadTemplate(type, name)
-  }
+  await downloadTemplate(type, name)
 
   ctx.body = {
     message: `template ${type}:${name} downloaded successfully.`,
   }
 }
-
-exports.exportTemplateFromApp = async function(ctx) {
-  const { appId } = ctx.user
-  const { templateName } = ctx.request.body
-
-  await exportTemplateFromApp({
-    appId,
-    templateName,
-  })
-
-  ctx.status = 200
-  ctx.body = {
-    message: `Created template: ${templateName}`,
-  }
-}
diff --git a/packages/server/src/api/controllers/view/index.js b/packages/server/src/api/controllers/view/index.js
index f482f3f2a6..0f6f008a1b 100644
--- a/packages/server/src/api/controllers/view/index.js
+++ b/packages/server/src/api/controllers/view/index.js
@@ -1,8 +1,6 @@
 const CouchDB = require("../../../db")
 const viewTemplate = require("./viewBuilder")
-const fs = require("fs")
-const { join } = require("../../../utilities/centralPath")
-const os = require("os")
+const { apiFileReturn } = require("../../../utilities/fileSystem")
 const exporters = require("./exporters")
 const { fetchView } = require("../row")
 const { ViewNames } = require("../../../db/utils")
@@ -120,12 +118,10 @@ const controller = {
     // Export part
     let headers = Object.keys(schema)
     const exporter = exporters[format]
-    const exportedFile = exporter(headers, ctx.body)
     const filename = `${viewName}.${format}`
-    fs.writeFileSync(join(os.tmpdir(), filename), exportedFile)
-
+    // send down the file
     ctx.attachment(filename)
-    ctx.body = fs.createReadStream(join(os.tmpdir(), filename))
+    ctx.body = apiFileReturn(exporter(headers, ctx.body))
   },
 }
 
diff --git a/packages/server/src/api/routes/templates.js b/packages/server/src/api/routes/templates.js
index 05882a22ea..6a427e8383 100644
--- a/packages/server/src/api/routes/templates.js
+++ b/packages/server/src/api/routes/templates.js
@@ -12,6 +12,5 @@ router
     authorized(BUILDER),
     controller.downloadTemplate
   )
-  .post("/api/templates", authorized(BUILDER), controller.exportTemplateFromApp)
 
 module.exports = router
diff --git a/packages/server/src/api/routes/tests/templates.spec.js b/packages/server/src/api/routes/tests/templates.spec.js
index f0d26bc7db..1b98d9f7a9 100644
--- a/packages/server/src/api/routes/tests/templates.spec.js
+++ b/packages/server/src/api/routes/tests/templates.spec.js
@@ -24,26 +24,4 @@ describe("/templates", () => {
       expect(Array.isArray(res.body)).toEqual(true)
     })
   })
-
-  describe("export", () => {
-    it("should be able to export the basic app", async () => {
-      const res = await request
-        .post(`/api/templates`)
-        .send({
-          templateName: "test",
-        })
-        .set(config.defaultHeaders())
-        .expect("Content-Type", /json/)
-        .expect(200)
-      expect(res.body.message).toEqual("Created template: test")
-      const dir = join(
-        budibaseAppsDir(),
-        "templates",
-        "app",
-        "test",
-        "db"
-      )
-      expect(fs.existsSync(dir)).toEqual(true)
-    })
-  })
 })
\ No newline at end of file
diff --git a/packages/server/src/constants/index.js b/packages/server/src/constants/index.js
index 46fb5cb649..f4e8c1cf20 100644
--- a/packages/server/src/constants/index.js
+++ b/packages/server/src/constants/index.js
@@ -89,3 +89,9 @@ exports.BaseQueryVerbs = {
   UPDATE: "update",
   DELETE: "delete",
 }
+
+exports.ObjectStoreBuckets = {
+  BACKUPS: "backups",
+  APPS: "apps",
+  TEMPLATES: "templates",
+}
diff --git a/packages/server/src/utilities/builder/compileStaticAssets.js b/packages/server/src/utilities/builder/compileStaticAssets.js
deleted file mode 100644
index 0389c920ee..0000000000
--- a/packages/server/src/utilities/builder/compileStaticAssets.js
+++ /dev/null
@@ -1,35 +0,0 @@
-const { ensureDir, constants, copyFile } = require("fs-extra")
-const { join } = require("../centralPath")
-const { budibaseAppsDir } = require("../budibaseDir")
-
-/**
- * Compile all the non-db static web assets that are required for the running of
- * a budibase application. This includes the JSON structure of the DOM and
- * the client library, a script responsible for reading the JSON structure
- * and rendering the application.
- * @param {string} appId id of the application we want to compile static assets for
- */
-module.exports = async appId => {
-  const publicPath = join(budibaseAppsDir(), appId, "public")
-  await ensureDir(publicPath)
-  await copyClientLib(publicPath)
-}
-
-/**
- * Copy the budibase client library and sourcemap from NPM to <appId>/public/.
- * The client library is then served as a static asset when the budibase application
- * is running in preview or prod
- * @param {String} publicPath - path to write the client library to
- */
-const copyClientLib = async publicPath => {
-  const sourcepath = require.resolve("@budibase/client")
-  const destPath = join(publicPath, "budibase-client.js")
-
-  await copyFile(sourcepath, destPath, constants.COPYFILE_FICLONE)
-
-  await copyFile(
-    sourcepath + ".map",
-    destPath + ".map",
-    constants.COPYFILE_FICLONE
-  )
-}
diff --git a/packages/server/src/utilities/createAppPackage.js b/packages/server/src/utilities/createAppPackage.js
deleted file mode 100644
index 9500554227..0000000000
--- a/packages/server/src/utilities/createAppPackage.js
+++ /dev/null
@@ -1,29 +0,0 @@
-const stream = require("stream")
-const fetch = require("node-fetch")
-const tar = require("tar-fs")
-const zlib = require("zlib")
-const { promisify } = require("util")
-const packageJson = require("../../package.json")
-
-const streamPipeline = promisify(stream.pipeline)
-
-// can't really test this due to the downloading nature of it, wouldn't be a great test case
-/* istanbul ignore next */
-exports.downloadExtractComponentLibraries = async appFolder => {
-  const LIBRARIES = ["standard-components"]
-
-  // Need to download tarballs directly from NPM as our users may not have node on their machine
-  for (let lib of LIBRARIES) {
-    // download tarball
-    const registryUrl = `https://registry.npmjs.org/@budibase/${lib}/-/${lib}-${packageJson.version}.tgz`
-    const response = await fetch(registryUrl)
-    if (!response.ok)
-      throw new Error(`unexpected response ${response.statusText}`)
-
-    await streamPipeline(
-      response.body,
-      zlib.Unzip(),
-      tar.extract(`${appFolder}/node_modules/@budibase/${lib}`)
-    )
-  }
-}
diff --git a/packages/server/src/utilities/fileSystem.js b/packages/server/src/utilities/fileSystem.js
deleted file mode 100644
index b55c4110fd..0000000000
--- a/packages/server/src/utilities/fileSystem.js
+++ /dev/null
@@ -1,40 +0,0 @@
-const { budibaseTempDir } = require("./budibaseDir")
-const { isDev } = require("./index")
-const fs = require("fs")
-const { join } = require("path")
-const { downloadTemplate } = require("./templates")
-
-/**
- * The single stack system (Cloud and Builder) should not make use of the file system where possible,
- * this file handles all of the file access for the system with the intention of limiting it all to one
- * place. Keeping all of this logic in one place means that when we need to do file system access (like
- * downloading a package or opening a temporary file) in can be done in way that we can confirm it shouldn't
- * be done through an object store instead.
- */
-
-/**
- * Checks if the system is currently in development mode and if it is makes sure
- * everything required to function is ready.
- */
-exports.checkDevelopmentEnvironment = () => {
-  if (isDev() && !fs.existsSync(budibaseTempDir())) {
-    console.error(
-      "Please run a build before attempting to run server independently to fill 'tmp' directory."
-    )
-    process.exit(-1)
-  }
-}
-
-/**
- * This function manages temporary template files which are stored by Koa.
- * @param {Object} template The template object retrieved from the Koa context object.
- * @returns {Object} Returns an fs read stream which can be loaded into the database.
- */
-exports.getTemplateStream = async template => {
-  if (template.file) {
-    return fs.createReadStream(template.file.path)
-  } else {
-    const templatePath = await downloadTemplate(...template.key.split("/"))
-    return fs.createReadStream(join(templatePath, "db", "dump.txt"))
-  }
-}
diff --git a/packages/server/src/utilities/fileSystem/index.js b/packages/server/src/utilities/fileSystem/index.js
new file mode 100644
index 0000000000..88346da110
--- /dev/null
+++ b/packages/server/src/utilities/fileSystem/index.js
@@ -0,0 +1,98 @@
+const { budibaseTempDir } = require("../budibaseDir")
+const { isDev } = require("../index")
+const fs = require("fs")
+const { join } = require("path")
+const uuid = require("uuid/v4")
+const CouchDB = require("../../db")
+const { ObjectStoreBuckets } = require("../../constants")
+const { streamUpload, deleteFolder, downloadTarball } = require("./utilities")
+const { downloadLibraries, newAppPublicPath } = require("./newApp")
+
+/**
+ * The single stack system (Cloud and Builder) should not make use of the file system where possible,
+ * this file handles all of the file access for the system with the intention of limiting it all to one
+ * place. Keeping all of this logic in one place means that when we need to do file system access (like
+ * downloading a package or opening a temporary file) in can be done in way that we can confirm it shouldn't
+ * be done through an object store instead.
+ */
+
+/**
+ * Checks if the system is currently in development mode and if it is makes sure
+ * everything required to function is ready.
+ */
+exports.checkDevelopmentEnvironment = () => {
+  if (isDev() && !fs.existsSync(budibaseTempDir())) {
+    console.error(
+      "Please run a build before attempting to run server independently to fill 'tmp' directory."
+    )
+    process.exit(-1)
+  }
+}
+
+/**
+ * This function manages temporary template files which are stored by Koa.
+ * @param {Object} template The template object retrieved from the Koa context object.
+ * @returns {Object} Returns an fs read stream which can be loaded into the database.
+ */
+exports.getTemplateStream = async template => {
+  if (template.file) {
+    return fs.createReadStream(template.file.path)
+  } else {
+    const tmpPath = await exports.downloadTemplate(...template.key.split("/"))
+    return fs.createReadStream(join(tmpPath, "db", "dump.txt"))
+  }
+}
+
+/**
+ * Used to retrieve a handlebars file from the system which will be used as a template.
+ * This is allowable as the template handlebars files should be static and identical across
+ * the cluster.
+ * @param {string} path The path to the handlebars file which is to be loaded.
+ * @returns {string} The loaded handlebars file as a string - loaded as utf8.
+ */
+exports.loadHandlebarsFile = path => {
+  return fs.readFileSync(path, "utf8")
+}
+
+/**
+ * When return a file from the API need to write the file to the system temporarily so we
+ * can create a read stream to send.
+ * @param {string} contents the contents of the file which is to be returned from the API.
+ * @return {Object} the read stream which can be put into the koa context body.
+ */
+exports.apiFileReturn = contents => {
+  const path = join(budibaseTempDir(), uuid())
+  fs.writeFileSync(path, contents)
+  return fs.createReadStream(path)
+}
+
+exports.performBackup = async (appId, backupName) => {
+  const path = join(budibaseTempDir(), backupName)
+  const writeStream = fs.createWriteStream(path)
+  // perform couch dump
+  const instanceDb = new CouchDB(appId)
+  await instanceDb.dump(writeStream, {})
+  // write the file to the object store
+  await streamUpload(
+    ObjectStoreBuckets.BACKUPS,
+    join(appId, backupName),
+    fs.createReadStream(path)
+  )
+  return fs.createReadStream(path)
+}
+
+exports.createApp = async appId => {
+  await downloadLibraries(appId)
+  await newAppPublicPath(appId)
+}
+
+exports.deleteApp = async appId => {
+  await deleteFolder(ObjectStoreBuckets.APPS, `${appId}/`)
+}
+
+exports.downloadTemplate = async (type, name) => {
+  const DEFAULT_TEMPLATES_BUCKET =
+    "prod-budi-templates.s3-eu-west-1.amazonaws.com"
+  const templateUrl = `https://${DEFAULT_TEMPLATES_BUCKET}/templates/${type}/${name}.tar.gz`
+  return downloadTarball(templateUrl, ObjectStoreBuckets.TEMPLATES, type)
+}
diff --git a/packages/server/src/utilities/fileSystem/newApp.js b/packages/server/src/utilities/fileSystem/newApp.js
new file mode 100644
index 0000000000..ba3d13afed
--- /dev/null
+++ b/packages/server/src/utilities/fileSystem/newApp.js
@@ -0,0 +1,34 @@
+const packageJson = require("../../../package.json")
+const { join } = require("path")
+const { ObjectStoreBuckets } = require("../../constants")
+const { streamUpload, downloadTarball } = require("./utilities")
+const fs = require("fs")
+
+const BUCKET_NAME = ObjectStoreBuckets.APPS
+
+// can't really test this due to the downloading nature of it, wouldn't be a great test case
+/* istanbul ignore next */
+exports.downloadLibraries = async appId => {
+  const LIBRARIES = ["standard-components"]
+
+  // Need to download tarballs directly from NPM as our users may not have node on their machine
+  for (let lib of LIBRARIES) {
+    // download tarball
+    const registryUrl = `https://registry.npmjs.org/@budibase/${lib}/-/${lib}-${packageJson.version}.tgz`
+    const path = join(appId, "node_modules", "@budibase", lib)
+    await downloadTarball(registryUrl, BUCKET_NAME, path)
+  }
+}
+
+exports.newAppPublicPath = async appId => {
+  const path = join(appId, "public")
+  const sourcepath = require.resolve("@budibase/client")
+  const destPath = join(path, "budibase-client.js")
+
+  await streamUpload(BUCKET_NAME, destPath, fs.createReadStream(sourcepath))
+  await streamUpload(
+    BUCKET_NAME,
+    destPath + ".map",
+    fs.createReadStream(sourcepath + ".map")
+  )
+}
diff --git a/packages/server/src/utilities/fileSystem/utilities.js b/packages/server/src/utilities/fileSystem/utilities.js
new file mode 100644
index 0000000000..01d956dac6
--- /dev/null
+++ b/packages/server/src/utilities/fileSystem/utilities.js
@@ -0,0 +1,116 @@
+const sanitize = require("sanitize-s3-objectkey")
+const AWS = require("aws-sdk")
+const stream = require("stream")
+const fetch = require("node-fetch")
+const tar = require("tar-fs")
+const zlib = require("zlib")
+const { promisify } = require("util")
+const { join } = require("path")
+const { streamUpload } = require("./utilities")
+const fs = require("fs")
+const { budibaseTempDir } = require("../budibaseDir")
+
+const streamPipeline = promisify(stream.pipeline)
+
+/**
+ * Gets a connection to the object store using the S3 SDK.
+ * @param {string} bucket the name of the bucket which blobs will be uploaded/retrieved from.
+ * @return {Object} an S3 object store object, check S3 Nodejs SDK for usage.
+ * @constructor
+ */
+exports.ObjectStore = bucket => {
+  return new AWS.S3({
+    params: {
+      Bucket: bucket,
+    },
+  })
+}
+
+exports.makeSureBucketExists = async (client, bucketName) => {
+  try {
+    await client
+      .headBucket({
+        Bucket: bucketName,
+      })
+      .promise()
+  } catch (err) {
+    // bucket doesn't exist create it
+    if (err.statusCode === 404) {
+      await client
+        .createBucket({
+          Bucket: bucketName,
+        })
+        .promise()
+    } else {
+      throw err
+    }
+  }
+}
+
+exports.streamUpload = async (bucket, filename, stream) => {
+  const objectStore = exports.ObjectStore(bucket)
+  await exports.makeSureBucketExists(objectStore, bucket)
+
+  const params = {
+    Bucket: bucket,
+    Key: sanitize(filename).replace(/\\/g, "/"),
+    Body: stream,
+  }
+  return objectStore.upload(params).promise()
+}
+
+exports.deleteFolder = async (bucket, folder) => {
+  const client = exports.ObjectStore(bucket)
+  const listParams = {
+    Bucket: bucket,
+    Prefix: folder,
+  }
+
+  const data = await client.listObjects(listParams).promise()
+  if (data.Contents.length > 0) {
+    const deleteParams = {
+      Bucket: bucket,
+      Delete: {
+        Objects: [],
+      },
+    }
+
+    data.Contents.forEach(content => {
+      deleteParams.Delete.Objects.push({ Key: content.Key })
+    })
+
+    const data = await client.deleteObjects(deleteParams).promise()
+    // can only empty 1000 items at once
+    if (data.Contents.length === 1000) {
+      return exports.deleteFolder(bucket, folder)
+    }
+  }
+}
+
+exports.uploadDirectory = async (bucket, localPath, bucketPath) => {
+  let uploads = []
+  const files = fs.readdirSync(localPath, { withFileTypes: true })
+  for (let file of files) {
+    const path = join(bucketPath, file.name)
+    const local = join(localPath, file.name)
+    if (file.isDirectory()) {
+      uploads.push(exports.uploadDirectory(bucket, local, path))
+    } else {
+      uploads.push(streamUpload(bucket, path, fs.createReadStream(local)))
+    }
+  }
+  await Promise.all(uploads)
+}
+
+exports.downloadTarball = async (url, bucket, path) => {
+  const response = await fetch(url)
+  if (!response.ok) {
+    throw new Error(`unexpected response ${response.statusText}`)
+  }
+
+  const tmpPath = join(budibaseTempDir(), path)
+  await streamPipeline(response.body, zlib.Unzip(), tar.extract(tmpPath))
+  await exports.uploadDirectory(bucket, tmpPath, path)
+  // return the temporary path incase there is a use for it
+  return tmpPath
+}
diff --git a/packages/server/src/utilities/templates.js b/packages/server/src/utilities/templates.js
index c3d89477df..ccad4a9d73 100644
--- a/packages/server/src/utilities/templates.js
+++ b/packages/server/src/utilities/templates.js
@@ -9,73 +9,9 @@ const { promisify } = require("util")
 const streamPipeline = promisify(stream.pipeline)
 const { budibaseAppsDir } = require("./budibaseDir")
 const env = require("../environment")
-const CouchDB = require("../db")
+const { downloadTemplate } = require("./fileSystem")
 
-const DEFAULT_TEMPLATES_BUCKET =
-  "prod-budi-templates.s3-eu-west-1.amazonaws.com"
-
-exports.getLocalTemplates = function() {
-  const templatesDir = join(os.homedir(), ".budibase", "templates", "app")
-  const templateObj = { app: {} }
-  fs.ensureDirSync(templatesDir)
-  const templateNames = fs.readdirSync(templatesDir)
-  for (let name of templateNames) {
-    templateObj.app[name] = {
-      name,
-      category: "local",
-      description: "local template",
-      type: "app",
-      key: `app/${name}`,
-    }
-  }
-  return templateObj
-}
 
 // can't really test this, downloading is just not something we should do in a behavioural test
 /* istanbul ignore next */
-exports.downloadTemplate = async function(type, name) {
-  const dirName = join(budibaseAppsDir(), "templates", type, name)
-  if (env.LOCAL_TEMPLATES) {
-    return dirName
-  }
-  const templateUrl = `https://${DEFAULT_TEMPLATES_BUCKET}/templates/${type}/${name}.tar.gz`
-  const response = await fetch(templateUrl)
-
-  if (!response.ok) {
-    throw new Error(
-      `Error downloading template ${type}:${name}: ${response.statusText}`
-    )
-  }
-
-  // stream the response, unzip and extract
-  await streamPipeline(
-    response.body,
-    zlib.Unzip(),
-    tar.extract(join(budibaseAppsDir(), "templates", type))
-  )
-
-  return dirName
-}
-
-async function performDump({ dir, appId, name = "dump.txt" }) {
-  const writeStream = fs.createWriteStream(join(dir, name))
-  // perform couch dump
-  const instanceDb = new CouchDB(appId)
-  await instanceDb.dump(writeStream, {})
-}
-
-exports.performDump = performDump
-
-exports.exportTemplateFromApp = async function({ templateName, appId }) {
-  // Copy frontend files
-  const templatesDir = join(
-    budibaseAppsDir(),
-    "templates",
-    "app",
-    templateName,
-    "db"
-  )
-  fs.ensureDirSync(templatesDir)
-  await performDump({ dir: templatesDir, appId })
-  return templatesDir
-}
+exports.downloadTemplate = downloadTemplate