merge next

.github/workflows/release.yml

@@ -52,8 +52,8 @@ jobs:
 
           mac_certs: ${{ secrets.mac_certs }}
           mac_certs_password: ${{ secrets.mac_certs_password }}
-          # windows_certs: ${{ secrets.windows_certs }}
+          windows_certs: ${{ secrets.windows_certs }}
-          # windows_certs_password: ${{ secrets.windows_certs_password }}
+          windows_certs_password: ${{ secrets.windows_certs_password }}
 
           # release the app after building
           release: ${{ startsWith(github.ref, 'refs/tags/v') }}

.gitignore

@@ -84,3 +84,6 @@ typings/
 
 # Nova Editor
 .nova
+
+# swap files (linux)
+*.swp

hosting/build/docker-compose.yaml

@@ -1,16 +0,0 @@
-version: "3"
-
-services:
-  app-service:
-    build: ./server
-    volumes: 
-      - ./server:/app
-    environment:
-      SELF_HOSTED: 1 
-      PORT: 4002
-
-  worker-service:
-    build: ./worker
-    environment:
-      SELF_HOSTED: 1,
-      PORT: 4003

hosting/build/server

@@ -1 +0,0 @@
-../../packages/server/

hosting/build/worker

@@ -1 +0,0 @@
-../../packages/worker/

hosting/docker-compose.dev.yaml

@@ -0,0 +1,73 @@
+version: "3"
+
+# optional ports are specified throughout for more advanced use cases.
+
+services:
+  minio-service:
+    container_name: budi-minio-dev
+    restart: always
+    image: minio/minio
+    volumes:
+      - minio_data:/data
+    ports:
+      - "${MINIO_PORT}:9000"
+    environment:
+      MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY}
+      MINIO_SECRET_KEY: ${MINIO_SECRET_KEY}
+      MINIO_BROWSER: "off"
+    command: server /data
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
+      interval: 30s
+      timeout: 20s
+      retries: 3
+
+  proxy-service:
+    container_name: budi-envoy-dev
+    restart: always
+    image: envoyproxy/envoy:v1.16-latest
+    volumes:
+      - ./envoy.dev.yaml:/etc/envoy/envoy.yaml
+    ports:
+      - "${MAIN_PORT}:10000"
+    depends_on:
+      - minio-service
+      - couchdb-service
+
+  couchdb-service:
+    container_name: budi-couchdb-dev
+    restart: always
+    image: ibmcom/couchdb3
+    environment:
+      - COUCHDB_PASSWORD=${COUCH_DB_PASSWORD}
+      - COUCHDB_USER=${COUCH_DB_USER}
+    ports:
+      - "${COUCH_DB_PORT}:5984"
+    volumes:
+      - couchdb3_data:/opt/couchdb/data
+  
+  couch-init:
+    container_name: budi-couchdb-init-dev
+    image: curlimages/curl
+    environment:
+      PUT_CALL: "curl -u ${COUCH_DB_USER}:${COUCH_DB_PASSWORD} -X PUT couchdb-service:5984"
+    depends_on:
+      - couchdb-service
+    command: ["sh","-c","sleep 10 && $${PUT_CALL}/_users && $${PUT_CALL}/_replicator; fg;"]
+
+  redis-service:
+    container_name: budi-redis-dev
+    restart: always
+    image: redis
+    ports:
+      - "${REDIS_PORT}:6379"
+    volumes:
+      - redis_data:/data
+
+volumes:
+  couchdb3_data:
+    driver: local
+  minio_data:
+    driver: local
+  redis_data:
+    driver: local

hosting/docker-compose.yaml

@@ -11,13 +11,18 @@ services:
       - "${APP_PORT}:4002"
     environment:
       SELF_HOSTED: 1
-      CLOUD: 1
       COUCH_DB_URL: http://${COUCH_DB_USER}:${COUCH_DB_PASSWORD}@couchdb-service:5984
       WORKER_URL: http://worker-service:4003
+      MINIO_URL: http://minio-service:9000
+      MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY}
+      MINIO_SECRET_KEY: ${MINIO_SECRET_KEY}
       HOSTING_KEY: ${HOSTING_KEY}
       BUDIBASE_ENVIRONMENT: ${BUDIBASE_ENVIRONMENT}
       PORT: 4002
       JWT_SECRET: ${JWT_SECRET}
+      LOG_LEVEL: info
+      SENTRY_DSN: https://a34ae347621946bf8acded18e5b7d4b8@o420233.ingest.sentry.io/5338131
+      ENABLE_ANALYTICS: true
     depends_on:
       - worker-service
 
@@ -28,7 +33,7 @@ services:
     ports:
       - "${WORKER_PORT}:4003"
     environment:
-      SELF_HOSTED: 1,
+      SELF_HOSTED: 1
       PORT: 4003
       MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY}
       MINIO_SECRET_KEY: ${MINIO_SECRET_KEY}
@@ -66,7 +71,6 @@ services:
       - ./envoy.yaml:/etc/envoy/envoy.yaml
     ports:
       - "${MAIN_PORT}:10000"
-      #- "9901:9901"
     depends_on:
       - minio-service
       - worker-service
@@ -75,16 +79,14 @@ services:
 
   couchdb-service:
     restart: always
-    image: apache/couchdb:3.0
+    image: ibmcom/couchdb3
     environment:
       - COUCHDB_PASSWORD=${COUCH_DB_PASSWORD}
       - COUCHDB_USER=${COUCH_DB_USER}
     ports:
       - "${COUCH_DB_PORT}:5984"
-      #- "4369:4369"
-      #- "9100:9100"
     volumes:
-      - couchdb_data:/opt/couchdb/data
+      - couchdb3_data:/opt/couchdb/data
 
   couch-init:
     image: curlimages/curl
@@ -94,8 +96,18 @@ services:
       - couchdb-service
     command: ["sh","-c","sleep 10 && $${PUT_CALL}/_users && $${PUT_CALL}/_replicator; fg;"]
   
+  redis-service:
+    restart: always
+    image: redis
+    ports:
+      - "${REDIS_PORT}:6379"
     volumes:
-  couchdb_data:
+      - redis_data:/data
+
+volumes:
+  couchdb3_data:
     driver: local
   minio_data:
     driver: local
+  redis_data:
+    driver: local

hosting/envoy.dev.yaml

@@ -0,0 +1,79 @@
+static_resources:
+  listeners:
+  - name: main_listener
+    address:
+      socket_address: { address: 0.0.0.0, port_value: 10000 }
+    filter_chains:
+    - filters:
+      - name: envoy.filters.network.http_connection_manager
+        typed_config:
+          "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager
+          stat_prefix: ingress
+          codec_type: auto
+          route_config:
+            name: local_route
+            virtual_hosts:
+            - name: local_services
+              domains: ["*"]
+              routes:
+              - match: { prefix: "/db/" }
+                route:
+                  cluster: couchdb-service
+                  prefix_rewrite: "/"
+
+              - match: { prefix: "/cache/" }
+                route:
+                  cluster: redis-service
+                  prefix_rewrite: "/"
+
+              # minio is on the default route because this works
+              # best, minio + AWS SDK doesn't handle path proxy
+              - match: { prefix: "/" }
+                route:
+                  cluster: minio-service
+
+          http_filters:
+          - name: envoy.filters.http.router
+
+  clusters:
+  - name: minio-service
+    connect_timeout: 0.25s
+    type: strict_dns
+    lb_policy: round_robin
+    load_assignment:
+      cluster_name: minio-service
+      endpoints:
+      - lb_endpoints:
+        - endpoint:
+            address:
+              socket_address:
+                address: minio-service
+                port_value: 9000
+
+  - name: couchdb-service
+    connect_timeout: 0.25s
+    type: strict_dns
+    lb_policy: round_robin
+    load_assignment:
+      cluster_name: couchdb-service
+      endpoints:
+      - lb_endpoints:
+        - endpoint:
+            address:
+              socket_address:
+                address: couchdb-service
+                port_value: 5984
+
+  - name: redis-service
+    connect_timeout: 0.25s
+    type: strict_dns
+    lb_policy: round_robin
+    load_assignment:
+      cluster_name: redis-service
+      endpoints:
+      - lb_endpoints:
+        - endpoint:
+            address:
+              socket_address:
+                address: redis-service
+                port_value: 6379

hosting/envoy.yaml

@@ -36,6 +36,11 @@ static_resources:
                   cluster: worker-service
                   prefix_rewrite: "/"
 
+              - match: { prefix: "/cache/" }
+                route:
+                  cluster: redis-service
+                  prefix_rewrite: "/"
+
               - match: { prefix: "/db/" }
                 route:
                   cluster: couchdb-service
@@ -107,3 +112,18 @@ static_resources:
                 address: couchdb-service
                 port_value: 5984
 
+  - name: redis-service
+    connect_timeout: 0.25s
+    type: strict_dns
+    lb_policy: round_robin
+    load_assignment:
+      cluster_name: redis-service
+      endpoints:
+      - lb_endpoints:
+        - endpoint:
+            address:
+              socket_address:
+                address: redis-service
+                port_value: 6379
+
+

hosting/hosting.properties

@@ -18,4 +18,5 @@ APP_PORT=4002
 WORKER_PORT=4003
 MINIO_PORT=4004
 COUCH_DB_PORT=4005
+REDIS_PORT=6379
 BUDIBASE_ENVIRONMENT=PRODUCTION

packages/builder/cypress.json

@@ -3,6 +3,7 @@
   "video": true,
   "projectId": "bmbemn",
   "env": {
-    "PORT": "4001"
+    "PORT": "4001",
+    "JWT_SECRET": "test"
   }
 }

packages/builder/cypress/setup.js

@@ -1,25 +1,26 @@
-// What this script does:
-// 1. Removes the old test folder if it exists (.budibase)
-// 2. Initialises using `.budibase`
-// 3. Runs the server using said folder
-
-const { join, resolve } = require("path")
-const initialiseBudibase = require("../../server/src/utilities/initialiseBudibase")
 const cypressConfig = require("../cypress.json")
+const path = require("path")
 
-const homedir = join(require("os").homedir(), ".budibase")
+const tmpdir = path.join(require("os").tmpdir(), ".budibase")
 
 process.env.BUDIBASE_API_KEY = "6BE826CB-6B30-4AEC-8777-2E90464633DE"
 process.env.NODE_ENV = "cypress"
 process.env.ENABLE_ANALYTICS = "false"
 process.env.PORT = cypressConfig.env.PORT
+process.env.JWT_SECRET = cypressConfig.env.JWT_SECRET
+process.env.COUCH_URL = `leveldb://${tmpdir}/.data/`
+process.env.SELF_HOSTED = 1
+process.env.MINIO_URL = "http://localhost:10000/"
+process.env.MINIO_ACCESS_KEY = "budibase"
+process.env.MINIO_SECRET_KEY = "budibase"
+process.env.COUCH_DB_USER = "budibase"
+process.env.COUCH_DB_PASSWORD = "budibase"
 
 // Stop info logs polluting test outputs
 process.env.LOG_LEVEL = "error"
 
-async function run(dir) {
+async function run() {
-  process.env.BUDIBASE_DIR = resolve(dir)
+  // require("dotenv").config({ path: resolve(dir, ".env") })
-  require("dotenv").config({ path: resolve(dir, ".env") })
 
   // dont make this a variable or top level require
   // it will cause environment module to be loaded prematurely
@@ -27,12 +28,15 @@ async function run(dir) {
   server.on("close", () => console.log("Server Closed"))
 }
 
-initialiseBudibase({ dir: homedir, clientId: "cypress-test" })
+run()
-  .then(() => {
+
-    delete require.cache[require.resolve("../../server/src/environment")]
+// TODO: ensure that this still works
-    const xPlatHomeDir = homedir.startsWith("~")
+// initialiseBudibase({ dir: homedir, clientId: "cypress-test" })
-      ? join(homedir(), homedir.substring(1))
+// .then(() => {
-      : homedir
+//   delete require.cache[require.resolve("../../server/src/environment")]
-    run(xPlatHomeDir)
+//   const xPlatHomeDir = homedir.startsWith("~")
-  })
+//     ? join(homedir(), homedir.substring(1))
-  .catch(e => console.error(e))
+//     : homedir
+//   run(xPlatHomeDir)
+// })
+// .catch(e => console.error(e))

packages/builder/src/builderStore/store/backend.js

@@ -0,0 +1,362 @@
+import { writable, get } from "svelte/store"
+import { cloneDeep } from "lodash/fp"
+import api from "../api"
+
+const INITIAL_BACKEND_UI_STATE = {
+  tables: [],
+  views: [],
+  users: [],
+  roles: [],
+  datasources: [],
+  queries: [],
+  integrations: {},
+  selectedDatabase: {},
+  selectedTable: {},
+  draftTable: {},
+}
+
+export const getBackendUiStore = () => {
+  const store = writable({ ...INITIAL_BACKEND_UI_STATE })
+
+  store.actions = {
+    reset: () => store.set({ ...INITIAL_BACKEND_UI_STATE }),
+    database: {
+      select: async db => {
+        const tablesResponse = await api.get(`/api/tables`)
+        const tables = await tablesResponse.json()
+        const datasourcesResponse = await api.get(`/api/datasources`)
+        const datasources = await datasourcesResponse.json()
+        const queriesResponse = await api.get(`/api/queries`)
+        const queries = await queriesResponse.json()
+        const integrationsResponse = await api.get("/api/integrations")
+        const integrations = await integrationsResponse.json()
+        const permissionLevels = await store.actions.permissions.fetchLevels()
+
+        store.update(state => {
+          state.selectedDatabase = db
+          state.tables = tables
+          state.datasources = datasources
+          state.queries = queries
+          state.integrations = integrations
+          state.permissionLevels = permissionLevels
+          return state
+        })
+      },
+    },
+    rows: {
+      save: () =>
+        store.update(state => {
+          state.selectedView = state.selectedView
+          return state
+        }),
+      delete: () =>
+        store.update(state => {
+          state.selectedView = state.selectedView
+          return state
+        }),
+      select: row =>
+        store.update(state => {
+          state.selectedRow = row
+          return state
+        }),
+    },
+    datasources: {
+      fetch: async () => {
+        const response = await api.get(`/api/datasources`)
+        const json = await response.json()
+        store.update(state => {
+          state.datasources = json
+          return state
+        })
+        return json
+      },
+      select: async datasourceId => {
+        store.update(state => {
+          state.selectedDatasourceId = datasourceId
+          state.selectedQueryId = null
+          return state
+        })
+      },
+      save: async datasource => {
+        const response = await api.post("/api/datasources", datasource)
+        const json = await response.json()
+        store.update(state => {
+          const currentIdx = state.datasources.findIndex(
+            ds => ds._id === json._id
+          )
+
+          if (currentIdx >= 0) {
+            state.datasources.splice(currentIdx, 1, json)
+          } else {
+            state.datasources.push(json)
+          }
+
+          state.datasources = state.datasources
+          state.selectedDatasourceId = json._id
+          return state
+        })
+        return json
+      },
+      delete: async datasource => {
+        await api.delete(
+          `/api/datasources/${datasource._id}/${datasource._rev}`
+        )
+        store.update(state => {
+          state.datasources = state.datasources.filter(
+            existing => existing._id !== datasource._id
+          )
+          if (datasource._id === state.selectedDatasourceId) {
+            state.selectedDatasourceId = null
+          }
+          return state
+        })
+      },
+    },
+    queries: {
+      fetch: async () => {
+        const response = await api.get(`/api/queries`)
+        const json = await response.json()
+        store.update(state => {
+          state.queries = json
+          return state
+        })
+        return json
+      },
+      save: async (datasourceId, query) => {
+        const integrations = get(store).integrations
+        const dataSource = get(store).datasources.filter(
+          ds => ds._id === datasourceId
+        )
+        // check if readable attribute is found
+        if (dataSource.length !== 0) {
+          const integration = integrations[dataSource[0].source]
+          const readable = integration.query[query.queryVerb].readable
+          if (readable) {
+            query.readable = readable
+          }
+        }
+        query.datasourceId = datasourceId
+        const response = await api.post(`/api/queries`, query)
+        if (response.status !== 200) {
+          throw new Error("Failed saving query.")
+        }
+        const json = await response.json()
+        store.update(state => {
+          const currentIdx = state.queries.findIndex(
+            query => query._id === json._id
+          )
+
+          if (currentIdx >= 0) {
+            state.queries.splice(currentIdx, 1, json)
+          } else {
+            state.queries.push(json)
+          }
+
+          state.queries = state.queries
+          state.selectedQueryId = json._id
+          return state
+        })
+        return json
+      },
+      select: query =>
+        store.update(state => {
+          state.selectedDatasourceId = query.datasourceId
+          state.selectedQueryId = query._id
+          return state
+        }),
+      delete: async query => {
+        await api.delete(`/api/queries/${query._id}/${query._rev}`)
+        store.update(state => {
+          state.queries = state.queries.filter(
+            existing => existing._id !== query._id
+          )
+          if (state.selectedQueryId === query._id) {
+            state.selectedQueryId = null
+          }
+
+          return state
+        })
+      },
+    },
+    tables: {
+      fetch: async () => {
+        const tablesResponse = await api.get(`/api/tables`)
+        const tables = await tablesResponse.json()
+        store.update(state => {
+          state.tables = tables
+          return state
+        })
+      },
+      select: table =>
+        store.update(state => {
+          state.selectedTable = table
+          state.draftTable = cloneDeep(table)
+          state.selectedView = { name: `all_${table._id}` }
+          return state
+        }),
+      save: async table => {
+        const updatedTable = cloneDeep(table)
+        const oldTable = get(store).tables.filter(t => t._id === table._id)[0]
+
+        const fieldNames = []
+        // update any renamed schema keys to reflect their names
+        for (let key of Object.keys(updatedTable.schema)) {
+          // if field name has been seen before remove it
+          if (fieldNames.indexOf(key.toLowerCase()) !== -1) {
+            delete updatedTable.schema[key]
+            continue
+          }
+          const field = updatedTable.schema[key]
+          const oldField = oldTable?.schema[key]
+          // if the type has changed then revert back to the old field
+          if (oldField != null && oldField.type !== field.type) {
+            updatedTable.schema[key] = oldField
+          }
+          // field has been renamed
+          if (field.name && field.name !== key) {
+            updatedTable.schema[field.name] = field
+            updatedTable._rename = { old: key, updated: field.name }
+            delete updatedTable.schema[key]
+          }
+          // finally record this field has been used
+          fieldNames.push(key.toLowerCase())
+        }
+
+        const SAVE_TABLE_URL = `/api/tables`
+        const response = await api.post(SAVE_TABLE_URL, updatedTable)
+        const savedTable = await response.json()
+        await store.actions.tables.fetch()
+        store.actions.tables.select(savedTable)
+        return savedTable
+      },
+      delete: async table => {
+        await api.delete(`/api/tables/${table._id}/${table._rev}`)
+        store.update(state => {
+          state.tables = state.tables.filter(
+            existing => existing._id !== table._id
+          )
+          if (table._id === state.selectedTable._id) {
+            state.selectedTable = {}
+          }
+          return state
+        })
+      },
+      saveField: ({ originalName, field, primaryDisplay = false, indexes }) => {
+        store.update(state => {
+          // delete the original if renaming
+          // need to handle if the column had no name, empty string
+          if (originalName || originalName === "") {
+            delete state.draftTable.schema[originalName]
+            state.draftTable._rename = {
+              old: originalName,
+              updated: field.name,
+            }
+          }
+
+          // Optionally set display column
+          if (primaryDisplay) {
+            state.draftTable.primaryDisplay = field.name
+          }
+
+          if (indexes) {
+            state.draftTable.indexes = indexes
+          }
+
+          state.draftTable.schema[field.name] = cloneDeep(field)
+          store.actions.tables.save(state.draftTable)
+          return state
+        })
+      },
+      deleteField: field => {
+        store.update(state => {
+          delete state.draftTable.schema[field.name]
+          store.actions.tables.save(state.draftTable)
+          return state
+        })
+      },
+    },
+    views: {
+      select: view =>
+        store.update(state => {
+          state.selectedView = view
+          state.selectedTable = {}
+          return state
+        }),
+      delete: async view => {
+        await api.delete(`/api/views/${view}`)
+        await store.actions.tables.fetch()
+      },
+      save: async view => {
+        const response = await api.post(`/api/views`, view)
+        const json = await response.json()
+
+        const viewMeta = {
+          name: view.name,
+          ...json,
+        }
+
+        store.update(state => {
+          const viewTable = state.tables.find(
+            table => table._id === view.tableId
+          )
+
+          if (view.originalName) delete viewTable.views[view.originalName]
+          viewTable.views[view.name] = viewMeta
+
+          state.tables = state.tables
+          state.selectedView = viewMeta
+          return state
+        })
+      },
+    },
+    users: {
+      create: user =>
+        store.update(state => {
+          state.users.push(user)
+          state.users = state.users
+          return state
+        }),
+    },
+    roles: {
+      fetch: async () => {
+        const response = await api.get("/api/roles")
+        const roles = await response.json()
+        store.update(state => {
+          state.roles = roles
+          return state
+        })
+      },
+      delete: async role => {
+        const response = await api.delete(`/api/roles/${role._id}/${role._rev}`)
+        await store.actions.roles.fetch()
+        return response
+      },
+      save: async role => {
+        const response = await api.post("/api/roles", role)
+        await store.actions.roles.fetch()
+        return response
+      },
+    },
+    permissions: {
+      fetchLevels: async () => {
+        const response = await api.get("/api/permission/levels")
+        const json = await response.json()
+        return json
+      },
+      forResource: async resourceId => {
+        const response = await api.get(`/api/permission/${resourceId}`)
+        const json = await response.json()
+        return json
+      },
+      save: async ({ role, resource, level }) => {
+        const response = await api.post(
+          `/api/permission/${role}/${resource}/${level}`
+        )
+        const json = await response.json()
+        return json
+      },
+    },
+  }
+
+  return store
+}

packages/builder/src/components/automation/AutomationPanel/EditAutomationPopover.svelte

@@ -25,7 +25,7 @@
       automation,
     })
     notifier.success("Automation deleted.")
-    $goto('../automate')
+    $goto("../automate")
   }
 </script>
 

packages/builder/src/components/backend/DatasourceNavigator/popovers/EditDatasourcePopover.svelte

@@ -22,9 +22,13 @@
   }
 
   async function deleteDatasource() {
+    const wasSelectedSource = $datasources.selected
     await datasources.delete(datasource)
     notifier.success("Datasource deleted")
-    $goto('./datasource')
+    // navigate to first index page if the source you are deleting is selected
+    if (wasSelectedSource === datasource._id) {
+      $goto("./datasource")
+    }
     hideEditor()
   }
 </script>

packages/builder/src/components/backend/TableNavigator/popovers/EditTablePopover.svelte

@@ -1,4 +1,5 @@
 <script>
+  import { goto } from '@sveltech/routify'
   import { store, allScreens } from "builderStore"
   import { tables } from 'stores/backend/'
   import { notifier } from "builderStore/store/notifications"
@@ -37,10 +38,14 @@
   }
 
   async function deleteTable() {
+    const wasSelectedTable = $tables.selected
     await tables.delete(table)
     store.actions.screens.delete(templateScreens)
     await tables.fetch()
     notifier.success("Table deleted")
+    if (wasSelectedTable._id === table._id) {
+      $goto("./table")
+    }
     hideEditor()
   }
 

packages/builder/src/components/deploy/DeploymentHistory.svelte

@@ -119,7 +119,9 @@
             {#if deployment.status.toLowerCase() === 'pending'}
               <Spinner size="10" />
             {/if}
-            <div on:click={() => showErrorReasonModal(deployment.err)} class={`deployment-status ${deployment.status}`}>
+            <div
+              on:click={() => showErrorReasonModal(deployment.err)}
+              class={`deployment-status ${deployment.status}`}>
               <span>
                 {deployment.status}
                 {#if deployment.status === DeploymentStatus.FAILURE}

packages/builder/src/pages/[application]/automate/[automation]/index.svelte

@@ -3,4 +3,3 @@
 </script>
 
 <AutomationBuilder />
-  

packages/builder/src/pages/[application]/automate/index.svelte

@@ -1,6 +1,6 @@
 <script>
   import { goto, leftover } from "@sveltech/routify"
-  import { onMount } from 'svelte'
+  import { onMount } from "svelte"
   import { automationStore } from "builderStore"
 
   onMount(async () => {
@@ -8,7 +8,8 @@
     if (
       !$leftover &&
       $automationStore.automations.length > 0 &&
-        (!$automationStore.selectedAutomation || !$automationStore.selectedAutomation?.automation?._id)
+      (!$automationStore.selectedAutomation ||
+        !$automationStore.selectedAutomation?.automation?._id)
     ) {
       $goto(`../${$automationStore.automations[0]._id}`)
     }

packages/builder/src/pages/[application]/data/_layout.svelte

@@ -17,11 +17,11 @@
     },
   ]
 
-  let tab = $isActive('./datasource') ? "datasource" : "table"
+  let tab = $isActive("./datasource") ? "datasource" : "table"
 
   function selectFirstTableOrSource({ detail }) {
     const type = detail.heading.key
-    if (type === 'datasource') {
+    if (type === "datasource") {
       $goto("./datasource")
     } else {
       $goto("./table")
@@ -34,7 +34,10 @@
 <!-- routify:options index=0 -->
 <div class="root">
   <div class="nav">
-    <Switcher headings={tabs} bind:value={tab} on:change={selectFirstTableOrSource}>
+    <Switcher
+      headings={tabs}
+      bind:value={tab}
+      on:change={selectFirstTableOrSource}>
       <div class="title">
         <i
           data-cy={`new-${tab}`}

packages/builder/src/stores/backend/tests/views.spec.js

@@ -15,7 +15,8 @@ describe("Tables Store", () => {
     await store.init()
   })
 
-  it("Initialises correctly", async () => {
+  it("Selects the correct view", async () => {
+      
     expect(get(store)).toEqual({ list: SOME_TABLES, selected: {}, draft: {}})
   })
 })

packages/builder/src/stores/backend/views.js

@@ -29,7 +29,6 @@ export function createViewsStore() {
         name: view.name,
         ...json,
       }
-      console.log('JSON: ', json)
 
       update(state => {
         const viewTable = get(tables).list.find(

packages/builder/yarn.lock

@@ -988,64 +988,11 @@
     svelte-portal "^1.0.0"
     turndown "^7.0.0"
 
-"@budibase/client@^0.8.9":
-  version "0.8.12"
-  resolved "https://registry.yarnpkg.com/@budibase/client/-/client-0.8.12.tgz#1186d95509e28bb660f0c801b37b3cc52b0f3864"
-  integrity sha512-68eh1QvF443OgMsjpDEgSY9D/MUutnaKFruP98PHJgrRkVhQDY7F2EKn2FWgC5NJuCY3aJ1emRPdXUq4MapWuA==
-  dependencies:
-    "@budibase/string-templates" "^0.8.12"
-    regexparam "^1.3.0"
-    shortid "^2.2.15"
-    svelte-spa-router "^3.0.5"
-
 "@budibase/colorpicker@1.1.2":
   version "1.1.2"
   resolved "https://registry.yarnpkg.com/@budibase/colorpicker/-/colorpicker-1.1.2.tgz#f7436924ee746d7be9b2009c2fa193e710c30f89"
   integrity sha512-2PlZBVkATDqDC4b4Ri8Xi8X3OxhuHOGfmZwtXbZL38lNIeofaQT3Qyc1ECzEY5N+HrdGrWhY9EnliF6QM+LIuA==
 
-"@budibase/handlebars-helpers@^0.11.3":
-  version "0.11.3"
-  resolved "https://registry.yarnpkg.com/@budibase/handlebars-helpers/-/handlebars-helpers-0.11.3.tgz#b6e5c91b83e8906e7d7ff10ddde277a3d561016e"
-  integrity sha512-MS1ptZEYq8o9J3tNLM7cZ2RGSSJIer4GiMIUHtbBI3sC9UKqZebao1JYNfmZKpNjntuqhZKgjqc5GfnVIEjsYQ==
-  dependencies:
-    arr-flatten "^1.1.0"
-    array-sort "^0.1.4"
-    define-property "^1.0.0"
-    extend-shallow "^3.0.2"
-    "falsey" "^0.3.2"
-    for-in "^1.0.2"
-    for-own "^1.0.0"
-    get-object "^0.2.0"
-    get-value "^2.0.6"
-    handlebars "^4.0.11"
-    handlebars-utils "^1.0.6"
-    has-value "^1.0.0"
-    helper-date "^1.0.1"
-    helper-markdown "^1.0.0"
-    helper-md "^0.2.2"
-    html-tag "^2.0.0"
-    is-even "^1.0.0"
-    is-glob "^4.0.0"
-    is-number "^4.0.0"
-    kind-of "^6.0.0"
-    logging-helpers "^1.0.0"
-    micromatch "^3.1.4"
-    relative "^3.0.2"
-    striptags "^3.1.0"
-    to-gfm-code-block "^0.1.1"
-    year "^0.2.1"
-
-"@budibase/string-templates@^0.8.12", "@budibase/string-templates@^0.8.9":
-  version "0.8.12"
-  resolved "https://registry.yarnpkg.com/@budibase/string-templates/-/string-templates-0.8.12.tgz#bc08d441ac90b2f71c89b4b15c40c7c5e934a6fb"
-  integrity sha512-FkTm3XRBozhrJqJA4YdDxbLbcY3me7sAbJ+dgO7mZkTybRcrdbxRwL1XjYfRw9DL7+8pucN2Y6OW8gOQgyaTtw==
-  dependencies:
-    "@budibase/handlebars-helpers" "^0.11.3"
-    dayjs "^1.10.4"
-    handlebars "^4.7.6"
-    handlebars-utils "^1.0.6"
-    lodash "^4.17.20"
-
 "@budibase/svelte-ag-grid@^1.0.4":
   version "1.0.4"
   resolved "https://registry.yarnpkg.com/@budibase/svelte-ag-grid/-/svelte-ag-grid-1.0.4.tgz#41cceec4bde2c4aea8b9da8f610fe36055c7709f"
@@ -1756,130 +1703,6 @@ ajv@^6.12.3:
     json-schema-traverse "^0.4.1"
     uri-js "^4.2.2"
 
-ansi-bgblack@^0.1.1:
-  version "0.1.1"
-  resolved "https://registry.yarnpkg.com/ansi-bgblack/-/ansi-bgblack-0.1.1.tgz#a68ba5007887701b6aafbe3fa0dadfdfa8ee3ca2"
-  integrity sha1-poulAHiHcBtqr74/oNrf36juPKI=
-  dependencies:
-    ansi-wrap "0.1.0"
-
-ansi-bgblue@^0.1.1:
-  version "0.1.1"
-  resolved "https://registry.yarnpkg.com/ansi-bgblue/-/ansi-bgblue-0.1.1.tgz#67bdc04edc9b9b5278969da196dea3d75c8c3613"
-  integrity sha1-Z73ATtybm1J4lp2hlt6j11yMNhM=
-  dependencies:
-    ansi-wrap "0.1.0"
-
-ansi-bgcyan@^0.1.1:
-  version "0.1.1"
-  resolved "https://registry.yarnpkg.com/ansi-bgcyan/-/ansi-bgcyan-0.1.1.tgz#58489425600bde9f5507068dd969ebfdb50fe768"
-  integrity sha1-WEiUJWAL3p9VBwaN2Wnr/bUP52g=
-  dependencies:
-    ansi-wrap "0.1.0"
-
-ansi-bggreen@^0.1.1:
-  version "0.1.1"
-  resolved "https://registry.yarnpkg.com/ansi-bggreen/-/ansi-bggreen-0.1.1.tgz#4e3191248529943f4321e96bf131d1c13816af49"
-  integrity sha1-TjGRJIUplD9DIelr8THRwTgWr0k=
-  dependencies:
-    ansi-wrap "0.1.0"
-
-ansi-bgmagenta@^0.1.1:
-  version "0.1.1"
-  resolved "https://registry.yarnpkg.com/ansi-bgmagenta/-/ansi-bgmagenta-0.1.1.tgz#9b28432c076eaa999418672a3efbe19391c2c7a1"
-  integrity sha1-myhDLAduqpmUGGcqPvvhk5HCx6E=
-  dependencies:
-    ansi-wrap "0.1.0"
-
-ansi-bgred@^0.1.1:
-  version "0.1.1"
-  resolved "https://registry.yarnpkg.com/ansi-bgred/-/ansi-bgred-0.1.1.tgz#a76f92838382ba43290a6c1778424f984d6f1041"
-  integrity sha1-p2+Sg4OCukMpCmwXeEJPmE1vEEE=
-  dependencies:
-    ansi-wrap "0.1.0"
-
-ansi-bgwhite@^0.1.1:
-  version "0.1.1"
-  resolved "https://registry.yarnpkg.com/ansi-bgwhite/-/ansi-bgwhite-0.1.1.tgz#6504651377a58a6ececd0331994e480258e11ba8"
-  integrity sha1-ZQRlE3elim7OzQMxmU5IAljhG6g=
-  dependencies:
-    ansi-wrap "0.1.0"
-
-ansi-bgyellow@^0.1.1:
-  version "0.1.1"
-  resolved "https://registry.yarnpkg.com/ansi-bgyellow/-/ansi-bgyellow-0.1.1.tgz#c3fe2eb08cd476648029e6874d15a0b38f61d44f"
-  integrity sha1-w/4usIzUdmSAKeaHTRWgs49h1E8=
-  dependencies:
-    ansi-wrap "0.1.0"
-
-ansi-black@^0.1.1:
-  version "0.1.1"
-  resolved "https://registry.yarnpkg.com/ansi-black/-/ansi-black-0.1.1.tgz#f6185e889360b2545a1ec50c0bf063fc43032453"
-  integrity sha1-9hheiJNgslRaHsUMC/Bj/EMDJFM=
-  dependencies:
-    ansi-wrap "0.1.0"
-
-ansi-blue@^0.1.1:
-  version "0.1.1"
-  resolved "https://registry.yarnpkg.com/ansi-blue/-/ansi-blue-0.1.1.tgz#15b804990e92fc9ca8c5476ce8f699777c21edbf"
-  integrity sha1-FbgEmQ6S/JyoxUds6PaZd3wh7b8=
-  dependencies:
-    ansi-wrap "0.1.0"
-
-ansi-bold@^0.1.1:
-  version "0.1.1"
-  resolved "https://registry.yarnpkg.com/ansi-bold/-/ansi-bold-0.1.1.tgz#3e63950af5acc2ae2e670e6f67deb115d1a5f505"
-  integrity sha1-PmOVCvWswq4uZw5vZ96xFdGl9QU=
-  dependencies:
-    ansi-wrap "0.1.0"
-
-ansi-colors@^0.2.0:
-  version "0.2.0"
-  resolved "https://registry.yarnpkg.com/ansi-colors/-/ansi-colors-0.2.0.tgz#72c31de2a0d9a2ccd0cac30cc9823eeb2f6434b5"
-  integrity sha1-csMd4qDZoszQysMMyYI+6y9kNLU=
-  dependencies:
-    ansi-bgblack "^0.1.1"
-    ansi-bgblue "^0.1.1"
-    ansi-bgcyan "^0.1.1"
-    ansi-bggreen "^0.1.1"
-    ansi-bgmagenta "^0.1.1"
-    ansi-bgred "^0.1.1"
-    ansi-bgwhite "^0.1.1"
-    ansi-bgyellow "^0.1.1"
-    ansi-black "^0.1.1"
-    ansi-blue "^0.1.1"
-    ansi-bold "^0.1.1"
-    ansi-cyan "^0.1.1"
-    ansi-dim "^0.1.1"
-    ansi-gray "^0.1.1"
-    ansi-green "^0.1.1"
-    ansi-grey "^0.1.1"
-    ansi-hidden "^0.1.1"
-    ansi-inverse "^0.1.1"
-    ansi-italic "^0.1.1"
-    ansi-magenta "^0.1.1"
-    ansi-red "^0.1.1"
-    ansi-reset "^0.1.1"
-    ansi-strikethrough "^0.1.1"
-    ansi-underline "^0.1.1"
-    ansi-white "^0.1.1"
-    ansi-yellow "^0.1.1"
-    lazy-cache "^2.0.1"
-
-ansi-cyan@^0.1.1:
-  version "0.1.1"
-  resolved "https://registry.yarnpkg.com/ansi-cyan/-/ansi-cyan-0.1.1.tgz#538ae528af8982f28ae30d86f2f17456d2609873"
-  integrity sha1-U4rlKK+JgvKK4w2G8vF0VtJgmHM=
-  dependencies:
-    ansi-wrap "0.1.0"
-
-ansi-dim@^0.1.1:
-  version "0.1.1"
-  resolved "https://registry.yarnpkg.com/ansi-dim/-/ansi-dim-0.1.1.tgz#40de4c603aa8086d8e7a86b8ff998d5c36eefd6c"
-  integrity sha1-QN5MYDqoCG2Oeoa4/5mNXDbu/Ww=
-  dependencies:
-    ansi-wrap "0.1.0"
-
 ansi-escapes@^3.0.0:
   version "3.2.0"
   resolved "https://registry.yarnpkg.com/ansi-escapes/-/ansi-escapes-3.2.0.tgz#8780b98ff9dbf5638152d1f1fe5c1d7b4442976b"
@@ -1892,62 +1715,6 @@ ansi-escapes@^4.2.1:
   dependencies:
     type-fest "^0.21.3"
 
-ansi-gray@^0.1.1:
-  version "0.1.1"
-  resolved "https://registry.yarnpkg.com/ansi-gray/-/ansi-gray-0.1.1.tgz#2962cf54ec9792c48510a3deb524436861ef7251"
-  integrity sha1-KWLPVOyXksSFEKPetSRDaGHvclE=
-  dependencies:
-    ansi-wrap "0.1.0"
-
-ansi-green@^0.1.1:
-  version "0.1.1"
-  resolved "https://registry.yarnpkg.com/ansi-green/-/ansi-green-0.1.1.tgz#8a5d9a979e458d57c40e33580b37390b8e10d0f7"
-  integrity sha1-il2al55FjVfEDjNYCzc5C44Q0Pc=
-  dependencies:
-    ansi-wrap "0.1.0"
-
-ansi-grey@^0.1.1:
-  version "0.1.1"
-  resolved "https://registry.yarnpkg.com/ansi-grey/-/ansi-grey-0.1.1.tgz#59d98b6ac2ba19f8a51798e9853fba78339a33c1"
-  integrity sha1-WdmLasK6GfilF5jphT+6eDOaM8E=
-  dependencies:
-    ansi-wrap "0.1.0"
-
-ansi-hidden@^0.1.1:
-  version "0.1.1"
-  resolved "https://registry.yarnpkg.com/ansi-hidden/-/ansi-hidden-0.1.1.tgz#ed6a4c498d2bb7cbb289dbf2a8d1dcc8567fae0f"
-  integrity sha1-7WpMSY0rt8uyidvyqNHcyFZ/rg8=
-  dependencies:
-    ansi-wrap "0.1.0"
-
-ansi-inverse@^0.1.1:
-  version "0.1.1"
-  resolved "https://registry.yarnpkg.com/ansi-inverse/-/ansi-inverse-0.1.1.tgz#b6af45826fe826bfb528a6c79885794355ccd269"
-  integrity sha1-tq9Fgm/oJr+1KKbHmIV5Q1XM0mk=
-  dependencies:
-    ansi-wrap "0.1.0"
-
-ansi-italic@^0.1.1:
-  version "0.1.1"
-  resolved "https://registry.yarnpkg.com/ansi-italic/-/ansi-italic-0.1.1.tgz#104743463f625c142a036739cf85eda688986f23"
-  integrity sha1-EEdDRj9iXBQqA2c5z4XtpoiYbyM=
-  dependencies:
-    ansi-wrap "0.1.0"
-
-ansi-magenta@^0.1.1:
-  version "0.1.1"
-  resolved "https://registry.yarnpkg.com/ansi-magenta/-/ansi-magenta-0.1.1.tgz#063b5ba16fb3f23e1cfda2b07c0a89de11e430ae"
-  integrity sha1-BjtboW+z8j4c/aKwfAqJ3hHkMK4=
-  dependencies:
-    ansi-wrap "0.1.0"
-
-ansi-red@^0.1.1:
-  version "0.1.1"
-  resolved "https://registry.yarnpkg.com/ansi-red/-/ansi-red-0.1.1.tgz#8c638f9d1080800a353c9c28c8a81ca4705d946c"
-  integrity sha1-jGOPnRCAgAo1PJwoyKgcpHBdlGw=
-  dependencies:
-    ansi-wrap "0.1.0"
-
 ansi-regex@^2.0.0:
   version "2.1.1"
   resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-2.1.1.tgz#c3b33ab5ee360d86e0e628f0468ae7ef27d654df"
@@ -1963,20 +1730,6 @@ ansi-regex@^5.0.0:
   resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-5.0.0.tgz#388539f55179bf39339c81af30a654d69f87cb75"
   integrity sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==
 
-ansi-reset@^0.1.1:
-  version "0.1.1"
-  resolved "https://registry.yarnpkg.com/ansi-reset/-/ansi-reset-0.1.1.tgz#e7e71292c3c7ddcd4d62ef4a6c7c05980911c3b7"
-  integrity sha1-5+cSksPH3c1NYu9KbHwFmAkRw7c=
-  dependencies:
-    ansi-wrap "0.1.0"
-
-ansi-strikethrough@^0.1.1:
-  version "0.1.1"
-  resolved "https://registry.yarnpkg.com/ansi-strikethrough/-/ansi-strikethrough-0.1.1.tgz#d84877140b2cff07d1c93ebce69904f68885e568"
-  integrity sha1-2Eh3FAss/wfRyT685pkE9oiF5Wg=
-  dependencies:
-    ansi-wrap "0.1.0"
-
 ansi-styles@^2.2.1:
   version "2.2.1"
   resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-2.2.1.tgz#b432dd3358b634cf75e1e4664368240533c1ddbe"
@@ -1996,32 +1749,6 @@ ansi-styles@^4.0.0, ansi-styles@^4.1.0:
   dependencies:
     color-convert "^2.0.1"
 
-ansi-underline@^0.1.1:
-  version "0.1.1"
-  resolved "https://registry.yarnpkg.com/ansi-underline/-/ansi-underline-0.1.1.tgz#dfc920f4c97b5977ea162df8ffb988308aaa71a4"
-  integrity sha1-38kg9Ml7WXfqFi34/7mIMIqqcaQ=
-  dependencies:
-    ansi-wrap "0.1.0"
-
-ansi-white@^0.1.1:
-  version "0.1.1"
-  resolved "https://registry.yarnpkg.com/ansi-white/-/ansi-white-0.1.1.tgz#9c77b7c193c5ee992e6011d36ec4c921b4578944"
-  integrity sha1-nHe3wZPF7pkuYBHTbsTJIbRXiUQ=
-  dependencies:
-    ansi-wrap "0.1.0"
-
-ansi-wrap@0.1.0:
-  version "0.1.0"
-  resolved "https://registry.yarnpkg.com/ansi-wrap/-/ansi-wrap-0.1.0.tgz#a82250ddb0015e9a27ca82e82ea603bbfa45efaf"
-  integrity sha1-qCJQ3bABXponyoLoLqYDu/pF768=
-
-ansi-yellow@^0.1.1:
-  version "0.1.1"
-  resolved "https://registry.yarnpkg.com/ansi-yellow/-/ansi-yellow-0.1.1.tgz#cb9356f2f46c732f0e3199e6102955a77da83c1d"
-  integrity sha1-y5NW8vRscy8OMZnmEClVp32oPB0=
-  dependencies:
-    ansi-wrap "0.1.0"
-
 any-observable@^0.3.0:
   version "0.3.0"
   resolved "https://registry.yarnpkg.com/any-observable/-/any-observable-0.3.0.tgz#af933475e5806a67d0d7df090dd5e8bef65d119b"
@@ -2048,7 +1775,7 @@ arch@^2.1.2:
   resolved "https://registry.yarnpkg.com/arch/-/arch-2.2.0.tgz#1bc47818f305764f23ab3306b0bfc086c5a29d11"
   integrity sha512-Of/R0wqp83cgHozfIYLbBMnej79U/SVGOOyuB3VVFv1NRM/PSFMK12x9KVtiYzJqmnU5WR2qp0Z5rHb7sWGnFQ==
 
-argparse@^1.0.10, argparse@^1.0.7:
+argparse@^1.0.7:
   version "1.0.10"
   resolved "https://registry.yarnpkg.com/argparse/-/argparse-1.0.10.tgz#bcd6791ea5ae09725e17e5ad988134cd40b3d911"
   integrity sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==
@@ -2083,15 +1810,6 @@ arr-union@^3.1.0:
   resolved "https://registry.yarnpkg.com/arr-union/-/arr-union-3.1.0.tgz#e39b09aea9def866a8f206e288af63919bae39c4"
   integrity sha1-45sJrqne+Gao8gbiiK9jkZuuOcQ=
 
-array-sort@^0.1.4:
-  version "0.1.4"
-  resolved "https://registry.yarnpkg.com/array-sort/-/array-sort-0.1.4.tgz#662855eaeb671b4188df4451b2f24a0753992b23"
-  integrity sha512-BNcM+RXxndPxiZ2rd76k6nyQLRZr2/B/sdi8pQ+Joafr5AH279L40dfokSUTp8O+AaqYjXWhblBWa2st2nc4fQ==
-  dependencies:
-    default-compare "^1.0.0"
-    get-value "^2.0.6"
-    kind-of "^5.0.2"
-
 array-union@^2.1.0:
   version "2.1.0"
   resolved "https://registry.yarnpkg.com/array-union/-/array-union-2.1.0.tgz#b798420adbeb1de828d84acd8a2e23d3efe85e8d"
@@ -2154,13 +1872,6 @@ atob@^2.1.2:
   resolved "https://registry.yarnpkg.com/atob/-/atob-2.1.2.tgz#6d9517eb9e030d2436666651e86bd9f6f13533c9"
   integrity sha512-Wm6ukoaOGJi/73p/cl2GvLjTI5JM1k/O14isD73YML8StrH/7/lRFgmg8nICZgD3bZZvjwCGxtMOD3wWNAu8cg==
 
-autolinker@~0.28.0:
-  version "0.28.1"
-  resolved "https://registry.yarnpkg.com/autolinker/-/autolinker-0.28.1.tgz#0652b491881879f0775dace0cdca3233942a4e47"
-  integrity sha1-BlK0kYgYefB3XazgzcoyM5QqTkc=
-  dependencies:
-    gulp-header "^1.7.1"
-
 aws-sign2@~0.7.0:
   version "0.7.0"
   resolved "https://registry.yarnpkg.com/aws-sign2/-/aws-sign2-0.7.0.tgz#b46e890934a9591f2d2f6f86d7e6a9f1b3fe76a8"
@@ -2800,13 +2511,6 @@ concat-stream@^1.4.4, concat-stream@^1.6.2:
     readable-stream "^2.2.2"
     typedarray "^0.0.6"
 
-concat-with-sourcemaps@*:
-  version "1.1.0"
-  resolved "https://registry.yarnpkg.com/concat-with-sourcemaps/-/concat-with-sourcemaps-1.1.0.tgz#d4ea93f05ae25790951b99e7b3b09e3908a4082e"
-  integrity sha512-4gEjHJFT9e+2W/77h/DS5SGUgwDaOwprX8L/gl5+3ixnzkVJJsZWDSelmN3Oilw3LNDZjZV0yqH1hLG3k6nghg==
-  dependencies:
-    source-map "^0.6.1"
-
 console-clear@^1.1.0:
   version "1.1.1"
   resolved "https://registry.yarnpkg.com/console-clear/-/console-clear-1.1.1.tgz#995e20cbfbf14dd792b672cde387bd128d674bf7"
@@ -3030,18 +2734,6 @@ date-fns@^1.27.2:
   resolved "https://registry.yarnpkg.com/date-fns/-/date-fns-1.30.1.tgz#2e71bf0b119153dbb4cc4e88d9ea5acfb50dc05c"
   integrity sha512-hBSVCvSmWC+QypYObzwGOd9wqdDpOt+0wl0KbU+R+uuZBS1jN8VsD1ss3irQDknRj5NvxiTF6oj/nDRnN/UQNw==
 
-date.js@^0.3.1:
-  version "0.3.3"
-  resolved "https://registry.yarnpkg.com/date.js/-/date.js-0.3.3.tgz#ef1e92332f507a638795dbb985e951882e50bbda"
-  integrity sha512-HgigOS3h3k6HnW011nAb43c5xx5rBXk8P2v/WIT9Zv4koIaVXiH2BURguI78VVp+5Qc076T7OR378JViCnZtBw==
-  dependencies:
-    debug "~3.1.0"
-
-dayjs@^1.10.4:
-  version "1.10.4"
-  resolved "https://registry.yarnpkg.com/dayjs/-/dayjs-1.10.4.tgz#8e544a9b8683f61783f570980a8a80eaf54ab1e2"
-  integrity sha512-RI/Hh4kqRc1UKLOAf/T5zdMMX5DQIlDxwUe3wSyMMnEbGunnpENCdbUgM+dW7kXidZqCttBrmw7BhN4TMddkCw==
-
 debug@4.3.1, debug@^4.1.0, debug@^4.1.1:
   version "4.3.1"
   resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.1.tgz#f0d229c505e0c6d8c49ac553d1b13dc183f6b2ee"
@@ -3063,13 +2755,6 @@ debug@^3.1.0:
   dependencies:
     ms "^2.1.1"
 
-debug@~3.1.0:
-  version "3.1.0"
-  resolved "https://registry.yarnpkg.com/debug/-/debug-3.1.0.tgz#5bb5a0672628b64149566ba16819e61518c67261"
-  integrity sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==
-  dependencies:
-    ms "2.0.0"
-
 decamelize@^1.2.0:
   version "1.2.0"
   resolved "https://registry.yarnpkg.com/decamelize/-/decamelize-1.2.0.tgz#f6534d15148269b20352e7bee26f501f9a191290"
@@ -3107,13 +2792,6 @@ deepmerge@^4.2.2:
   resolved "https://registry.yarnpkg.com/deepmerge/-/deepmerge-4.2.2.tgz#44d2ea3679b8f4d4ffba33f03d865fc1e7bf4955"
   integrity sha512-FJ3UgI4gIl+PHZm53knsuSFpE+nESMr7M4v9QcgB7S63Kj/6WqMiFQJpBBYz1Pt+66bZpP3Q7Lye0Oo9MPKEdg==
 
-default-compare@^1.0.0:
-  version "1.0.0"
-  resolved "https://registry.yarnpkg.com/default-compare/-/default-compare-1.0.0.tgz#cb61131844ad84d84788fb68fd01681ca7781a2f"
-  integrity sha512-QWfXlM0EkAbqOCbD/6HjdwT19j7WCkMyiRhWilc4H9/5h/RzTF9gv5LYh1+CmDV5d1rki6KAWLtQale0xt20eQ==
-  dependencies:
-    kind-of "^5.0.2"
-
 deferred-leveldown@~0.2.0:
   version "0.2.0"
   resolved "https://registry.yarnpkg.com/deferred-leveldown/-/deferred-leveldown-0.2.0.tgz#2cef1f111e1c57870d8bbb8af2650e587cd2f5b4"
@@ -3269,11 +2947,6 @@ end-of-stream@^1.1.0:
   dependencies:
     once "^1.4.0"
 
-ent@^2.2.0:
-  version "2.2.0"
-  resolved "https://registry.yarnpkg.com/ent/-/ent-2.2.0.tgz#e964219325a21d05f44466a2f686ed6ce5f5dd1d"
-  integrity sha1-6WQhkyWiHQX0RGai9obtbOX13R0=
-
 entities@~2.1.0:
   version "2.1.0"
   resolved "https://registry.yarnpkg.com/entities/-/entities-2.1.0.tgz#992d3129cf7df6870b96c57858c249a120f8b8b5"
@@ -3293,11 +2966,6 @@ error-ex@^1.3.1:
   dependencies:
     is-arrayish "^0.2.1"
 
-error-symbol@^0.1.0:
-  version "0.1.0"
-  resolved "https://registry.yarnpkg.com/error-symbol/-/error-symbol-0.1.0.tgz#0a4dae37d600d15a29ba453d8ef920f1844333f6"
-  integrity sha1-Ck2uN9YA0VopukU9jvkg8YRDM/Y=
-
 escalade@^3.1.1:
   version "3.1.1"
   resolved "https://registry.yarnpkg.com/escalade/-/escalade-3.1.1.tgz#d8cfdc7000965c5a0174b4a82eaa5c0552742e40"
@@ -3565,13 +3233,6 @@ extsprintf@^1.2.0:
   resolved "https://registry.yarnpkg.com/extsprintf/-/extsprintf-1.4.0.tgz#e2689f8f356fad62cca65a3a91c5df5f9551692f"
   integrity sha1-4mifjzVvrWLMplo6kcXfX5VRaS8=
 
-"falsey@^0.3.2":
-  version "0.3.2"
-  resolved "https://registry.yarnpkg.com/falsey/-/falsey-0.3.2.tgz#b21c90c5c34660fc192bf909575db95b6880d597"
-  integrity sha512-lxEuefF5MBIVDmE6XeqCdM4BWk1+vYmGZtkbKZ/VFcg6uBBw6fXNEbWmxCjDdQlFc9hy450nkiWwM3VAW6G1qg==
-  dependencies:
-    kind-of "^5.0.2"
-
 fast-deep-equal@^3.1.1:
   version "3.1.3"
   resolved "https://registry.yarnpkg.com/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz#3a7d56b559d6cbc3eb512325244e619a65c6c525"
@@ -3685,18 +3346,11 @@ follow-redirects@^1.10.0:
   resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.13.2.tgz#dd73c8effc12728ba5cf4259d760ea5fb83e3147"
   integrity sha512-6mPTgLxYm3r6Bkkg0vNM0HTjfGrOEtsfbhagQvbxDEsEkpNhw582upBaoRZylzen6krEmxXJgt9Ju6HiI4O7BA==
 
-for-in@^1.0.1, for-in@^1.0.2:
+for-in@^1.0.2:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/for-in/-/for-in-1.0.2.tgz#81068d295a8142ec0ac726c6e2200c30fb6d5e80"
   integrity sha1-gQaNKVqBQuwKxybG4iAMMPttXoA=
 
-for-own@^1.0.0:
-  version "1.0.0"
-  resolved "https://registry.yarnpkg.com/for-own/-/for-own-1.0.0.tgz#c63332f415cedc4b04dbfe70cf836494c53cb44b"
-  integrity sha1-xjMy9BXO3EsE2/5wz4NklMU8tEs=
-  dependencies:
-    for-in "^1.0.1"
-
 foreach@~2.0.1:
   version "2.0.5"
   resolved "https://registry.yarnpkg.com/foreach/-/foreach-2.0.5.tgz#0bee005018aeb260d0a3af3ae658dd0136ec1b99"
@@ -3728,11 +3382,6 @@ from@~0:
   resolved "https://registry.yarnpkg.com/from/-/from-0.1.7.tgz#83c60afc58b9c56997007ed1a768b3ab303a44fe"
   integrity sha1-g8YK/Fi5xWmXAH7Rp2izqzA6RP4=
 
-fs-exists-sync@^0.1.0:
-  version "0.1.0"
-  resolved "https://registry.yarnpkg.com/fs-exists-sync/-/fs-exists-sync-0.1.0.tgz#982d6893af918e72d08dec9e8673ff2b5a8d6add"
-  integrity sha1-mC1ok6+RjnLQjeyehnP/K1qNat0=
-
 fs-extra@^8.1.0:
   version "8.1.0"
   resolved "https://registry.yarnpkg.com/fs-extra/-/fs-extra-8.1.0.tgz#49d43c45a88cd9677668cb7be1b46efdb8d2e1c0"
@@ -3793,14 +3442,6 @@ get-intrinsic@^1.0.2:
     has "^1.0.3"
     has-symbols "^1.0.1"
 
-get-object@^0.2.0:
-  version "0.2.0"
-  resolved "https://registry.yarnpkg.com/get-object/-/get-object-0.2.0.tgz#d92ff7d5190c64530cda0543dac63a3d47fe8c0c"
-  integrity sha1-2S/31RkMZFMM2gVD2sY6PUf+jAw=
-  dependencies:
-    is-number "^2.0.2"
-    isobject "^0.2.0"
-
 get-package-type@^0.1.0:
   version "0.1.0"
   resolved "https://registry.yarnpkg.com/get-package-type/-/get-package-type-0.1.0.tgz#8de2d803cff44df3bc6c456e6668b36c3926e11a"
@@ -3899,35 +3540,6 @@ growly@^1.3.0:
   resolved "https://registry.yarnpkg.com/growly/-/growly-1.3.0.tgz#f10748cbe76af964b7c96c93c6bcc28af120c081"
   integrity sha1-8QdIy+dq+WS3yWyTxrzCivEgwIE=
 
-gulp-header@^1.7.1:
-  version "1.8.12"
-  resolved "https://registry.yarnpkg.com/gulp-header/-/gulp-header-1.8.12.tgz#ad306be0066599127281c4f8786660e705080a84"
-  integrity sha512-lh9HLdb53sC7XIZOYzTXM4lFuXElv3EVkSDhsd7DoJBj7hm+Ni7D3qYbb+Rr8DuM8nRanBvkVO9d7askreXGnQ==
-  dependencies:
-    concat-with-sourcemaps "*"
-    lodash.template "^4.4.0"
-    through2 "^2.0.0"
-
-handlebars-utils@^1.0.2, handlebars-utils@^1.0.4, handlebars-utils@^1.0.6:
-  version "1.0.6"
-  resolved "https://registry.yarnpkg.com/handlebars-utils/-/handlebars-utils-1.0.6.tgz#cb9db43362479054782d86ffe10f47abc76357f9"
-  integrity sha512-d5mmoQXdeEqSKMtQQZ9WkiUcO1E3tPbWxluCK9hVgIDPzQa9WsKo3Lbe/sGflTe7TomHEeZaOgwIkyIr1kfzkw==
-  dependencies:
-    kind-of "^6.0.0"
-    typeof-article "^0.1.1"
-
-handlebars@^4.0.11, handlebars@^4.7.6:
-  version "4.7.7"
-  resolved "https://registry.yarnpkg.com/handlebars/-/handlebars-4.7.7.tgz#9ce33416aad02dbd6c8fafa8240d5d98004945a1"
-  integrity sha512-aAcXm5OAfE/8IXkcZvCepKU3VzW1/39Fb5ZuqMtgI/hT8X2YgoMvBY5dLhq/cpOvw7Lk1nK/UF71aLG/ZnVYRA==
-  dependencies:
-    minimist "^1.2.5"
-    neo-async "^2.6.0"
-    source-map "^0.6.1"
-    wordwrap "^1.0.0"
-  optionalDependencies:
-    uglify-js "^3.1.4"
-
 har-schema@^2.0.0:
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/har-schema/-/har-schema-2.0.0.tgz#a94c2224ebcac04782a0d9035521f24735b7ec92"
@@ -4028,39 +3640,6 @@ he@1.2.x:
   resolved "https://registry.yarnpkg.com/he/-/he-1.2.0.tgz#84ae65fa7eafb165fddb61566ae14baf05664f0f"
   integrity sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==
 
-helper-date@^1.0.1:
-  version "1.0.1"
-  resolved "https://registry.yarnpkg.com/helper-date/-/helper-date-1.0.1.tgz#12fedea3ad8e44a7ca4c4efb0ff4104a5120cffb"
-  integrity sha512-wU3VOwwTJvGr/w5rZr3cprPHO+hIhlblTJHD6aFBrKLuNbf4lAmkawd2iK3c6NbJEvY7HAmDpqjOFSI5/+Ey2w==
-  dependencies:
-    date.js "^0.3.1"
-    handlebars-utils "^1.0.4"
-    moment "^2.18.1"
-
-helper-markdown@^1.0.0:
-  version "1.0.0"
-  resolved "https://registry.yarnpkg.com/helper-markdown/-/helper-markdown-1.0.0.tgz#ee7e9fc554675007d37eb90f7853b13ce74f3e10"
-  integrity sha512-AnDqMS4ejkQK0MXze7pA9TM3pu01ZY+XXsES6gEE0RmCGk5/NIfvTn0NmItfyDOjRAzyo9z6X7YHbHX4PzIvOA==
-  dependencies:
-    handlebars-utils "^1.0.2"
-    highlight.js "^9.12.0"
-    remarkable "^1.7.1"
-
-helper-md@^0.2.2:
-  version "0.2.2"
-  resolved "https://registry.yarnpkg.com/helper-md/-/helper-md-0.2.2.tgz#c1f59d7e55bbae23362fd8a0e971607aec69d41f"
-  integrity sha1-wfWdflW7riM2L9ig6XFgeuxp1B8=
-  dependencies:
-    ent "^2.2.0"
-    extend-shallow "^2.0.1"
-    fs-exists-sync "^0.1.0"
-    remarkable "^1.6.2"
-
-highlight.js@^9.12.0:
-  version "9.18.5"
-  resolved "https://registry.yarnpkg.com/highlight.js/-/highlight.js-9.18.5.tgz#d18a359867f378c138d6819edfc2a8acd5f29825"
-  integrity sha512-a5bFyofd/BHCX52/8i8uJkjr9DYwXIPnM/plwI6W7ezItLGqzt7X2G2nXuYSfsIJdkwwj/g9DG1LkcGJI/dDoA==
-
 hmac-drbg@^1.0.1:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/hmac-drbg/-/hmac-drbg-1.0.1.tgz#d2745701025a6c775a6c545793ed502fc0c649a1"
@@ -4100,14 +3679,6 @@ html-minifier@^3.0.2:
     relateurl "0.2.x"
     uglify-js "3.4.x"
 
-html-tag@^2.0.0:
-  version "2.0.0"
-  resolved "https://registry.yarnpkg.com/html-tag/-/html-tag-2.0.0.tgz#36c3bc8d816fd30b570d5764a497a641640c2fed"
-  integrity sha512-XxzooSo6oBoxBEUazgjdXj7VwTn/iSTSZzTYKzYY6I916tkaYzypHxy+pbVU1h+0UQ9JlVf5XkNQyxOAiiQO1g==
-  dependencies:
-    is-self-closing "^1.0.1"
-    kind-of "^6.0.0"
-
 http-signature@~1.2.0:
   version "1.2.0"
   resolved "https://registry.yarnpkg.com/http-signature/-/http-signature-1.2.0.tgz#9aecd925114772f3d95b65a60abb8f7c18fbace1"
@@ -4190,11 +3761,6 @@ inflight@^1.0.4:
     once "^1.3.0"
     wrappy "1"
 
-info-symbol@^0.1.0:
-  version "0.1.0"
-  resolved "https://registry.yarnpkg.com/info-symbol/-/info-symbol-0.1.0.tgz#27841d72867ddb4242cd612d79c10633881c6a78"
-  integrity sha1-J4QdcoZ920JCzWEtecEGM4gcang=
-
 inherits@2, inherits@^2.0.1, inherits@^2.0.3, inherits@^2.0.4, inherits@~2.0.1, inherits@~2.0.3:
   version "2.0.4"
   resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.4.tgz#0fa2c64f932917c3433a0ded55363aae37416b7c"
@@ -4304,13 +3870,6 @@ is-docker@^2.0.0:
   resolved "https://registry.yarnpkg.com/is-docker/-/is-docker-2.1.1.tgz#4125a88e44e450d384e09047ede71adc2d144156"
   integrity sha512-ZOoqiXfEwtGknTiuDEy8pN2CfE3TxMHprvNer1mXiqwkOT77Rw3YVrUQ52EqAOU3QAWDQ+bQdx7HJzrv7LS2Hw==
 
-is-even@^1.0.0:
-  version "1.0.0"
-  resolved "https://registry.yarnpkg.com/is-even/-/is-even-1.0.0.tgz#76b5055fbad8d294a86b6a949015e1c97b717c06"
-  integrity sha1-drUFX7rY0pSoa2qUkBXhyXtxfAY=
-  dependencies:
-    is-odd "^0.1.2"
-
 is-extendable@^0.1.0, is-extendable@^0.1.1:
   version "0.1.1"
   resolved "https://registry.yarnpkg.com/is-extendable/-/is-extendable-0.1.1.tgz#62b110e289a471418e3ec36a617d472e301dfc89"
@@ -4350,7 +3909,7 @@ is-generator-fn@^2.0.0:
   resolved "https://registry.yarnpkg.com/is-generator-fn/-/is-generator-fn-2.1.0.tgz#7d140adc389aaf3011a8f2a2a4cfa6faadffb118"
   integrity sha512-cTIB4yPYL/Grw0EaSzASzg6bBy9gqCofvWN8okThAYIxKJZC+udlRAmGbM0XLeniEJSs8uEgHPGuHSe1XsOLSQ==
 
-is-glob@^4.0.0, is-glob@^4.0.1, is-glob@~4.0.1:
+is-glob@^4.0.1, is-glob@~4.0.1:
   version "4.0.1"
   resolved "https://registry.yarnpkg.com/is-glob/-/is-glob-4.0.1.tgz#7567dbe9f2f5e2467bc77ab83c4a29482407a5dc"
   integrity sha512-5G0tKtBTFImOqDnLB2hG6Bp2qcKEFduo4tZu9MT/H6NQv/ghhy30o55ufafxJ/LdH79LLs2Kfrn85TLKyA7BUg==
@@ -4370,13 +3929,6 @@ is-module@^1.0.0:
   resolved "https://registry.yarnpkg.com/is-module/-/is-module-1.0.0.tgz#3258fb69f78c14d5b815d664336b4cffb6441591"
   integrity sha1-Mlj7afeMFNW4FdZkM2tM/7ZEFZE=
 
-is-number@^2.0.2:
-  version "2.1.0"
-  resolved "https://registry.yarnpkg.com/is-number/-/is-number-2.1.0.tgz#01fcbbb393463a548f2f466cce16dece49db908f"
-  integrity sha1-Afy7s5NGOlSPL0ZszhbezknbkI8=
-  dependencies:
-    kind-of "^3.0.2"
-
 is-number@^3.0.0:
   version "3.0.0"
   resolved "https://registry.yarnpkg.com/is-number/-/is-number-3.0.0.tgz#24fd6201a4782cf50561c810276afc7d12d71195"
@@ -4384,11 +3936,6 @@ is-number@^3.0.0:
   dependencies:
     kind-of "^3.0.2"
 
-is-number@^4.0.0:
-  version "4.0.0"
-  resolved "https://registry.yarnpkg.com/is-number/-/is-number-4.0.0.tgz#0026e37f5454d73e356dfe6564699867c6a7f0ff"
-  integrity sha512-rSklcAIlf1OmFdyAqbnWTLVelsQ58uvZ66S/ZyawjWqIviTWCjg2PzVGw8WUA+nNuPTqb4wgA+NszrJ+08LlgQ==
-
 is-number@^7.0.0:
   version "7.0.0"
   resolved "https://registry.yarnpkg.com/is-number/-/is-number-7.0.0.tgz#7535345b896734d5f80c4d06c50955527a14f12b"
@@ -4406,13 +3953,6 @@ is-observable@^1.1.0:
   dependencies:
     symbol-observable "^1.1.0"
 
-is-odd@^0.1.2:
-  version "0.1.2"
-  resolved "https://registry.yarnpkg.com/is-odd/-/is-odd-0.1.2.tgz#bc573b5ce371ef2aad6e6f49799b72bef13978a7"
-  integrity sha1-vFc7XONx7yqtbm9JeZtyvvE5eKc=
-  dependencies:
-    is-number "^3.0.0"
-
 is-path-inside@^3.0.1:
   version "3.0.2"
   resolved "https://registry.yarnpkg.com/is-path-inside/-/is-path-inside-3.0.2.tgz#f5220fc82a3e233757291dddc9c5877f2a1f3017"
@@ -4455,13 +3995,6 @@ is-regex@^1.0.4:
     call-bind "^1.0.2"
     has-symbols "^1.0.1"
 
-is-self-closing@^1.0.1:
-  version "1.0.1"
-  resolved "https://registry.yarnpkg.com/is-self-closing/-/is-self-closing-1.0.1.tgz#5f406b527c7b12610176320338af0fa3896416e4"
-  integrity sha512-E+60FomW7Blv5GXTlYee2KDrnG6srxF7Xt1SjrhWUGUEsTFIqY/nq2y3DaftCsgUMdh89V07IVfhY9KIJhLezg==
-  dependencies:
-    self-closing-tags "^1.0.1"
-
 is-stream@^1.1.0:
   version "1.1.0"
   resolved "https://registry.yarnpkg.com/is-stream/-/is-stream-1.1.0.tgz#12d4a3dd4e68e0b79ceb8dbc84173ae80d91ca44"
@@ -4514,11 +4047,6 @@ isexe@^2.0.0:
   resolved "https://registry.yarnpkg.com/isexe/-/isexe-2.0.0.tgz#e8fbf374dc556ff8947a10dcb0572d633f2cfa10"
   integrity sha1-6PvzdNxVb/iUehDcsFctYz8s+hA=
 
-isobject@^0.2.0:
-  version "0.2.0"
-  resolved "https://registry.yarnpkg.com/isobject/-/isobject-0.2.0.tgz#a3432192f39b910b5f02cc989487836ec70aa85e"
-  integrity sha1-o0MhkvObkQtfAsyYlIeDbscKqF4=
-
 isobject@^2.0.0:
   version "2.1.0"
   resolved "https://registry.yarnpkg.com/isobject/-/isobject-2.1.0.tgz#f065561096a3f1da2ef46272f815c840d87e0c89"
@@ -5106,7 +4634,7 @@ jsprim@^1.2.2:
     json-schema "0.2.3"
     verror "1.10.0"
 
-kind-of@^3.0.2, kind-of@^3.0.3, kind-of@^3.1.0, kind-of@^3.2.0:
+kind-of@^3.0.2, kind-of@^3.0.3, kind-of@^3.2.0:
   version "3.2.2"
   resolved "https://registry.yarnpkg.com/kind-of/-/kind-of-3.2.2.tgz#31ea21a734bab9bbb0f32466d893aea51e4a3c64"
   integrity sha1-MeohpzS6ubuw8yRm2JOupR5KPGQ=
@@ -5120,7 +4648,7 @@ kind-of@^4.0.0:
   dependencies:
     is-buffer "^1.1.5"
 
-kind-of@^5.0.0, kind-of@^5.0.2:
+kind-of@^5.0.0:
   version "5.1.0"
   resolved "https://registry.yarnpkg.com/kind-of/-/kind-of-5.1.0.tgz#729c91e2d857b7a419a1f9aa65685c4c33f5845d"
   integrity sha512-NGEErnH6F2vUuXDh+OlbcKW7/wOcfdRHaZ7VWtqCztfHri/++YKmP51OdWeGPuqCOba6kk2OTe5d02VmTB80Pw==
@@ -5140,13 +4668,6 @@ lazy-ass@1.6.0, lazy-ass@^1.6.0:
   resolved "https://registry.yarnpkg.com/lazy-ass/-/lazy-ass-1.6.0.tgz#7999655e8646c17f089fdd187d150d3324d54513"
   integrity sha1-eZllXoZGwX8In90YfRUNMyTVRRM=
 
-lazy-cache@^2.0.1:
-  version "2.0.2"
-  resolved "https://registry.yarnpkg.com/lazy-cache/-/lazy-cache-2.0.2.tgz#b9190a4f913354694840859f8a8f7084d8822264"
-  integrity sha1-uRkKT5EzVGlIQIWfio9whNiCImQ=
-  dependencies:
-    set-getter "^0.1.0"
-
 level-blobs@^0.1.7:
   version "0.1.7"
   resolved "https://registry.yarnpkg.com/level-blobs/-/level-blobs-0.1.7.tgz#9ab9b97bb99f1edbf9f78a3433e21ed56386bdaf"
@@ -5333,11 +4854,6 @@ lodash-es@^4.17.11:
   resolved "https://registry.yarnpkg.com/lodash-es/-/lodash-es-4.17.20.tgz#29f6332eefc60e849f869c264bc71126ad61e8f7"
   integrity sha512-JD1COMZsq8maT6mnuz1UMV0jvYD0E0aUsSOdrr1/nAG3dhqQXwRRgeW0cSqH1U43INKcqxaiVIQNOUDld7gRDA==
 
-lodash._reinterpolate@^3.0.0:
-  version "3.0.0"
-  resolved "https://registry.yarnpkg.com/lodash._reinterpolate/-/lodash._reinterpolate-3.0.0.tgz#0ccf2d89166af03b3663c796538b75ac6e114d9d"
-  integrity sha1-DM8tiRZq8Ds2Y8eWU4t1rG4RTZ0=
-
 lodash.once@^4.1.1:
   version "4.1.1"
   resolved "https://registry.yarnpkg.com/lodash.once/-/lodash.once-4.1.1.tgz#0dd3971213c7c56df880977d504c88fb471a97ac"
@@ -5348,21 +4864,6 @@ lodash.sortby@^4.7.0:
   resolved "https://registry.yarnpkg.com/lodash.sortby/-/lodash.sortby-4.7.0.tgz#edd14c824e2cc9c1e0b0a1b42bb5210516a42438"
   integrity sha1-7dFMgk4sycHgsKG0K7UhBRakJDg=
 
-lodash.template@^4.4.0:
-  version "4.5.0"
-  resolved "https://registry.yarnpkg.com/lodash.template/-/lodash.template-4.5.0.tgz#f976195cf3f347d0d5f52483569fe8031ccce8ab"
-  integrity sha512-84vYFxIkmidUiFxidA/KjjH9pAycqW+h980j7Fuz5qxRtO9pgB7MDFTdys1N7A5mcucRiDyEq4fusljItR1T/A==
-  dependencies:
-    lodash._reinterpolate "^3.0.0"
-    lodash.templatesettings "^4.0.0"
-
-lodash.templatesettings@^4.0.0:
-  version "4.2.0"
-  resolved "https://registry.yarnpkg.com/lodash.templatesettings/-/lodash.templatesettings-4.2.0.tgz#e481310f049d3cf6d47e912ad09313b154f0fb33"
-  integrity sha512-stgLz+i3Aa9mZgnjr/O+v9ruKZsPsndy7qPZOchbqk2cnTU1ZaldKK+v7m54WoKIyxiuMZTKT2H81F8BeAc3ZQ==
-  dependencies:
-    lodash._reinterpolate "^3.0.0"
-
 lodash@4.17.13:
   version "4.17.13"
   resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.13.tgz#0bdc3a6adc873d2f4e0c4bac285df91b64fc7b93"
@@ -5373,14 +4874,6 @@ lodash@^4.17.15, lodash@^4.17.19, lodash@^4.17.20:
   resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.20.tgz#b44a9b6297bcb698f1c51a3545a2b3b368d59c52"
   integrity sha512-PlhdFcillOINfeV7Ni6oF1TAEayyZBoZ8bcshTHqOYJYlrqzRK5hagpagky5o4HfCzzd1TRkXPMFq6cKk9rGmA==
 
-log-ok@^0.1.1:
-  version "0.1.1"
-  resolved "https://registry.yarnpkg.com/log-ok/-/log-ok-0.1.1.tgz#bea3dd36acd0b8a7240d78736b5b97c65444a334"
-  integrity sha1-vqPdNqzQuKckDXhza1uXxlREozQ=
-  dependencies:
-    ansi-green "^0.1.1"
-    success-symbol "^0.1.0"
-
 log-symbols@^1.0.2:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/log-symbols/-/log-symbols-1.0.2.tgz#376ff7b58ea3086a0f09facc74617eca501e1a18"
@@ -5411,27 +4904,6 @@ log-update@^2.3.0:
     cli-cursor "^2.0.0"
     wrap-ansi "^3.0.1"
 
-log-utils@^0.2.1:
-  version "0.2.1"
-  resolved "https://registry.yarnpkg.com/log-utils/-/log-utils-0.2.1.tgz#a4c217a0dd9a50515d9b920206091ab3d4e031cf"
-  integrity sha1-pMIXoN2aUFFdm5ICBgkas9TgMc8=
-  dependencies:
-    ansi-colors "^0.2.0"
-    error-symbol "^0.1.0"
-    info-symbol "^0.1.0"
-    log-ok "^0.1.1"
-    success-symbol "^0.1.0"
-    time-stamp "^1.0.1"
-    warning-symbol "^0.1.0"
-
-logging-helpers@^1.0.0:
-  version "1.0.0"
-  resolved "https://registry.yarnpkg.com/logging-helpers/-/logging-helpers-1.0.0.tgz#b5a37b32ad53eb0137c58c7898a47b175ddb7c36"
-  integrity sha512-qyIh2goLt1sOgQQrrIWuwkRjUx4NUcEqEGAcYqD8VOnOC6ItwkrVE8/tA4smGpjzyp4Svhc6RodDp9IO5ghpyA==
-  dependencies:
-    isobject "^3.0.0"
-    log-utils "^0.2.1"
-
 lower-case@^1.1.1:
   version "1.1.4"
   resolved "https://registry.yarnpkg.com/lower-case/-/lower-case-1.1.4.tgz#9a2cabd1b9e8e0ae993a4bf7d5875c39c42e8eac"
@@ -5643,7 +5115,7 @@ mkdirp@^0.5.1, mkdirp@^0.5.4:
   dependencies:
     minimist "^1.2.5"
 
-moment@^2.18.1, moment@^2.27.0:
+moment@^2.27.0:
   version "2.29.1"
   resolved "https://registry.yarnpkg.com/moment/-/moment-2.29.1.tgz#b2be769fa31940be9eeea6469c075e35006fa3d3"
   integrity sha512-kHmoybcPV8Sqy59DwNDY3Jefr64lK/by/da0ViFcuA4DH0vQg5Q6Ze5VimxkfQNSC+Mls/Kx53s7TjP1RhFEDQ==
@@ -5700,11 +5172,6 @@ ncp@^2.0.0:
   resolved "https://registry.yarnpkg.com/ncp/-/ncp-2.0.0.tgz#195a21d6c46e361d2fb1281ba38b91e9df7bdbb3"
   integrity sha1-GVoh1sRuNh0vsSgbo4uR6d9727M=
 
-neo-async@^2.6.0:
-  version "2.6.2"
-  resolved "https://registry.yarnpkg.com/neo-async/-/neo-async-2.6.2.tgz#b4aafb93e3aeb2d8174ca53cf163ab7d7308305f"
-  integrity sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw==
-
 nice-try@^1.0.4:
   version "1.0.5"
   resolved "https://registry.yarnpkg.com/nice-try/-/nice-try-1.0.5.tgz#a3378a7696ce7d223e88fc9b764bd7ef1089e366"
@@ -6294,7 +5761,7 @@ readable-stream@^1.0.26-4:
     isarray "0.0.1"
     string_decoder "~0.10.x"
 
-readable-stream@^2.2.2, readable-stream@~2.3.6:
+readable-stream@^2.2.2:
   version "2.3.7"
   resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.3.7.tgz#1eca1cf711aef814c04f62252a36a62f6cb23b57"
   integrity sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==
@@ -6381,11 +5848,6 @@ regexp.prototype.flags@^1.2.0:
     call-bind "^1.0.2"
     define-properties "^1.1.3"
 
-regexparam@1.3.0, regexparam@^1.3.0:
-  version "1.3.0"
-  resolved "https://registry.yarnpkg.com/regexparam/-/regexparam-1.3.0.tgz#2fe42c93e32a40eff6235d635e0ffa344b92965f"
-  integrity sha512-6IQpFBv6e5vz1QAqI+V4k8P2e/3gRrqfCJ9FI+O1FLQTO+Uz6RXZEZOPmTJ6hlGj7gkERzY5BRCv09whKP96/g==
-
 regexpu-core@^4.7.1:
   version "4.7.1"
   resolved "https://registry.yarnpkg.com/regexpu-core/-/regexpu-core-4.7.1.tgz#2dea5a9a07233298fbf0db91fa9abc4c6e0f8ad6"
@@ -6415,21 +5877,6 @@ relateurl@0.2.x:
   resolved "https://registry.yarnpkg.com/relateurl/-/relateurl-0.2.7.tgz#54dbf377e51440aca90a4cd274600d3ff2d888a9"
   integrity sha1-VNvzd+UUQKypCkzSdGANP/LYiKk=
 
-relative@^3.0.2:
-  version "3.0.2"
-  resolved "https://registry.yarnpkg.com/relative/-/relative-3.0.2.tgz#0dcd8ec54a5d35a3c15e104503d65375b5a5367f"
-  integrity sha1-Dc2OxUpdNaPBXhBFA9ZTdbWlNn8=
-  dependencies:
-    isobject "^2.0.0"
-
-remarkable@^1.6.2, remarkable@^1.7.1:
-  version "1.7.4"
-  resolved "https://registry.yarnpkg.com/remarkable/-/remarkable-1.7.4.tgz#19073cb960398c87a7d6546eaa5e50d2022fcd00"
-  integrity sha512-e6NKUXgX95whv7IgddywbeN/ItCkWbISmc2DiqHJb0wTrqZIexqdco5b8Z3XZoo/48IdNVKM9ZCvTPJ4F5uvhg==
-  dependencies:
-    argparse "^1.0.10"
-    autolinker "~0.28.0"
-
 remixicon@2.5.0:
   version "2.5.0"
   resolved "https://registry.yarnpkg.com/remixicon/-/remixicon-2.5.0.tgz#b5e245894a1550aa23793f95daceadbf96ad1a41"
@@ -6779,11 +6226,6 @@ saxes@^5.0.0, saxes@^5.0.1:
   dependencies:
     xmlchars "^2.2.0"
 
-self-closing-tags@^1.0.1:
-  version "1.0.1"
-  resolved "https://registry.yarnpkg.com/self-closing-tags/-/self-closing-tags-1.0.1.tgz#6c5fa497994bb826b484216916371accee490a5d"
-  integrity sha512-7t6hNbYMxM+VHXTgJmxwgZgLGktuXtVVD5AivWzNTdJBM4DBjnDKDzkf2SrNjihaArpeJYNjxkELBu1evI4lQA==
-
 "semver@2 || 3 || 4 || 5", semver@^5.4.1, semver@^5.5.0, semver@^5.5.1:
   version "5.7.1"
   resolved "https://registry.yarnpkg.com/semver/-/semver-5.7.1.tgz#a954f931aeba508d307bbf069eff0c01c96116f7"
@@ -6823,13 +6265,6 @@ set-blocking@^2.0.0:
   resolved "https://registry.yarnpkg.com/set-blocking/-/set-blocking-2.0.0.tgz#045f9782d011ae9a6803ddd382b24392b3d890f7"
   integrity sha1-BF+XgtARrppoA93TgrJDkrPYkPc=
 
-set-getter@^0.1.0:
-  version "0.1.0"
-  resolved "https://registry.yarnpkg.com/set-getter/-/set-getter-0.1.0.tgz#d769c182c9d5a51f409145f2fba82e5e86e80376"
-  integrity sha1-12nBgsnVpR9AkUXy+6guXoboA3Y=
-  dependencies:
-    to-object-path "^0.3.0"
-
 set-value@^2.0.0, set-value@^2.0.1:
   version "2.0.1"
   resolved "https://registry.yarnpkg.com/set-value/-/set-value-2.0.1.tgz#a18d40530e6f07de4228c7defe4227af8cad005b"
@@ -6884,13 +6319,6 @@ shortid@2.2.15:
   dependencies:
     nanoid "^2.1.0"
 
-shortid@^2.2.15:
-  version "2.2.16"
-  resolved "https://registry.yarnpkg.com/shortid/-/shortid-2.2.16.tgz#b742b8f0cb96406fd391c76bfc18a67a57fe5608"
-  integrity sha512-Ugt+GIZqvGXCIItnsL+lvFJOiN7RYqlGy7QE41O3YC1xbNSeDGIRO7xg2JJXIAj1cAGnOeC1r7/T9pgrtQbv4g==
-  dependencies:
-    nanoid "^2.1.0"
-
 signal-exit@^3.0.0, signal-exit@^3.0.2:
   version "3.0.3"
   resolved "https://registry.yarnpkg.com/signal-exit/-/signal-exit-3.0.3.tgz#a1410c2edd8f077b08b4e253c8eacfcaf057461c"
@@ -7224,16 +6652,6 @@ strip-indent@^3.0.0:
   dependencies:
     min-indent "^1.0.0"
 
-striptags@^3.1.0:
-  version "3.1.1"
-  resolved "https://registry.yarnpkg.com/striptags/-/striptags-3.1.1.tgz#c8c3e7fdd6fb4bb3a32a3b752e5b5e3e38093ebd"
-  integrity sha1-yMPn/db7S7OjKjt1LltePjgJPr0=
-
-success-symbol@^0.1.0:
-  version "0.1.0"
-  resolved "https://registry.yarnpkg.com/success-symbol/-/success-symbol-0.1.0.tgz#24022e486f3bf1cdca094283b769c472d3b72897"
-  integrity sha1-JAIuSG878c3KCUKDt2nEctO3KJc=
-
 supports-color@^2.0.0:
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-2.0.0.tgz#535d045ce6b6363fa40117084629995e9df324c7"
@@ -7295,13 +6713,6 @@ svelte-portal@^1.0.0:
   resolved "https://registry.yarnpkg.com/svelte-portal/-/svelte-portal-1.0.0.tgz#36a47c5578b1a4d9b4dc60fa32a904640ec4cdd3"
   integrity sha512-nHf+DS/jZ6jjnZSleBMSaZua9JlG5rZv9lOGKgJuaZStfevtjIlUJrkLc3vbV8QdBvPPVmvcjTlazAzfKu0v3Q==
 
-svelte-spa-router@^3.0.5:
-  version "3.1.0"
-  resolved "https://registry.yarnpkg.com/svelte-spa-router/-/svelte-spa-router-3.1.0.tgz#a929f0def7e12c41f32bc356f91685aeadcd75bf"
-  integrity sha512-jlM/xwjn57mylr+pzHYCOOy+IPQauT46gOucNGTBu6jHcFXu3F+oaojN4PXC1LYizRGxFB6QA0qnYbZnRfX7Sg==
-  dependencies:
-    regexparam "1.3.0"
-
 svelte@^3.30.0:
   version "3.32.3"
   resolved "https://registry.yarnpkg.com/svelte/-/svelte-3.32.3.tgz#db0c50c65573ecffe4e2f4924e4862d8f9feda74"
@@ -7358,24 +6769,11 @@ throttleit@^1.0.0:
   resolved "https://registry.yarnpkg.com/throttleit/-/throttleit-1.0.0.tgz#9e785836daf46743145a5984b6268d828528ac6c"
   integrity sha1-nnhYNtr0Z0MUWlmEtiaNgoUorGw=
 
-through2@^2.0.0:
-  version "2.0.5"
-  resolved "https://registry.yarnpkg.com/through2/-/through2-2.0.5.tgz#01c1e39eb31d07cb7d03a96a70823260b23132cd"
-  integrity sha512-/mrRod8xqpA+IHSLyGCQ2s8SPHiCDEeQJSep1jqLYeEUClOFG2Qsh+4FU6G9VeqpZnGW/Su8LQGc4YKni5rYSQ==
-  dependencies:
-    readable-stream "~2.3.6"
-    xtend "~4.0.1"
-
 through@2, through@~2.3, through@~2.3.1:
   version "2.3.8"
   resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5"
   integrity sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU=
 
-time-stamp@^1.0.1:
-  version "1.1.0"
-  resolved "https://registry.yarnpkg.com/time-stamp/-/time-stamp-1.1.0.tgz#764a5a11af50561921b133f3b44e618687e0f5c3"
-  integrity sha1-dkpaEa9QVhkhsTPztE5hhofg9cM=
-
 tinydate@^1.0.0:
   version "1.3.0"
   resolved "https://registry.yarnpkg.com/tinydate/-/tinydate-1.3.0.tgz#e6ca8e5a22b51bb4ea1c3a2a4fd1352dbd4c57fb"
@@ -7398,11 +6796,6 @@ to-fast-properties@^2.0.0:
   resolved "https://registry.yarnpkg.com/to-fast-properties/-/to-fast-properties-2.0.0.tgz#dc5e698cbd079265bc73e0377681a4e4e83f616e"
   integrity sha1-3F5pjL0HkmW8c+A3doGk5Og/YW4=
 
-to-gfm-code-block@^0.1.1:
-  version "0.1.1"
-  resolved "https://registry.yarnpkg.com/to-gfm-code-block/-/to-gfm-code-block-0.1.1.tgz#25d045a5fae553189e9637b590900da732d8aa82"
-  integrity sha1-JdBFpfrlUxielje1kJANpzLYqoI=
-
 to-object-path@^0.3.0:
   version "0.3.0"
   resolved "https://registry.yarnpkg.com/to-object-path/-/to-object-path-0.3.0.tgz#297588b7b0e7e0ac08e04e672f85c1f4999e17af"
@@ -7541,13 +6934,6 @@ typedarray@^0.0.6:
   resolved "https://registry.yarnpkg.com/typedarray/-/typedarray-0.0.6.tgz#867ac74e3864187b1d3d47d996a78ec5c8830777"
   integrity sha1-hnrHTjhkGHsdPUfZlqeOxciDB3c=
 
-typeof-article@^0.1.1:
-  version "0.1.1"
-  resolved "https://registry.yarnpkg.com/typeof-article/-/typeof-article-0.1.1.tgz#9f07e733c3fbb646ffa9e61c08debacd460e06af"
-  integrity sha1-nwfnM8P7tkb/qeYcCN66zUYOBq8=
-  dependencies:
-    kind-of "^3.1.0"
-
 uc.micro@^1.0.1, uc.micro@^1.0.5:
   version "1.0.6"
   resolved "https://registry.yarnpkg.com/uc.micro/-/uc.micro-1.0.6.tgz#9c411a802a409a91fc6cf74081baba34b24499ac"
@@ -7561,11 +6947,6 @@ uglify-js@3.4.x:
     commander "~2.19.0"
     source-map "~0.6.1"
 
-uglify-js@^3.1.4:
-  version "3.13.2"
-  resolved "https://registry.yarnpkg.com/uglify-js/-/uglify-js-3.13.2.tgz#fe10319861bccc8682bfe2e8151fbdd8aa921c44"
-  integrity sha512-SbMu4D2Vo95LMC/MetNaso1194M1htEA+JrqE9Hk+G2DhI+itfS9TRu9ZKeCahLDNa/J3n4MqUJ/fOHMzQpRWw==
-
 unicode-canonical-property-names-ecmascript@^1.0.4:
   version "1.0.4"
   resolved "https://registry.yarnpkg.com/unicode-canonical-property-names-ecmascript/-/unicode-canonical-property-names-ecmascript-1.0.4.tgz#2619800c4c825800efdd8343af7dd9933cbe2818"
@@ -7742,11 +7123,6 @@ walker@^1.0.7, walker@~1.0.5:
   dependencies:
     makeerror "1.0.x"
 
-warning-symbol@^0.1.0:
-  version "0.1.0"
-  resolved "https://registry.yarnpkg.com/warning-symbol/-/warning-symbol-0.1.0.tgz#bb31dd11b7a0f9d67ab2ed95f457b65825bbad21"
-  integrity sha1-uzHdEbeg+dZ6su2V9Fe2WCW7rSE=
-
 webidl-conversions@^5.0.0:
   version "5.0.0"
   resolved "https://registry.yarnpkg.com/webidl-conversions/-/webidl-conversions-5.0.0.tgz#ae59c8a00b121543a2acc65c0434f57b0fc11aff"
@@ -7802,11 +7178,6 @@ word-wrap@~1.2.3:
   resolved "https://registry.yarnpkg.com/word-wrap/-/word-wrap-1.2.3.tgz#610636f6b1f703891bd34771ccb17fb93b47079c"
   integrity sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ==
 
-wordwrap@^1.0.0:
-  version "1.0.0"
-  resolved "https://registry.yarnpkg.com/wordwrap/-/wordwrap-1.0.0.tgz#27584810891456a4171c8d0226441ade90cbcaeb"
-  integrity sha1-J1hIEIkUVqQXHI0CJkQa3pDLyus=
-
 wrap-ansi@^3.0.1:
   version "3.0.1"
   resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-3.0.1.tgz#288a04d87eda5c286e060dfe8f135ce8d007f8ba"
@@ -7891,11 +7262,6 @@ xtend@~3.0.0:
   resolved "https://registry.yarnpkg.com/xtend/-/xtend-3.0.0.tgz#5cce7407baf642cba7becda568111c493f59665a"
   integrity sha1-XM50B7r2Qsunvs2laBEcST9ZZlo=
 
-xtend@~4.0.1:
-  version "4.0.2"
-  resolved "https://registry.yarnpkg.com/xtend/-/xtend-4.0.2.tgz#bb72779f5fa465186b1f438f674fa347fdb5db54"
-  integrity sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==
-
 y18n@^4.0.0:
   version "4.0.1"
   resolved "https://registry.yarnpkg.com/y18n/-/y18n-4.0.1.tgz#8db2b83c31c5d75099bb890b23f3094891e247d4"
@@ -7944,11 +7310,6 @@ yauzl@^2.10.0:
     buffer-crc32 "~0.2.3"
     fd-slicer "~1.1.0"
 
-year@^0.2.1:
-  version "0.2.1"
-  resolved "https://registry.yarnpkg.com/year/-/year-0.2.1.tgz#4083ae520a318b23ec86037f3000cb892bdf9bb0"
-  integrity sha1-QIOuUgoxiyPshgN/MADLiSvfm7A=
-
 yup@0.29.2:
   version "0.29.2"
   resolved "https://registry.yarnpkg.com/yup/-/yup-0.29.2.tgz#5302abd9024cca335b987793f8df868e410b7b67"

packages/cli/package.json

@@ -1,9 +1,11 @@
 {
-  "name": "cli",
+  "name": "@budibase/cli",
-  "version": "0.8.7",
+  "version": "0.8.10",
   "description": "Budibase CLI, for developers, self hosting and migrations.",
   "main": "src/index.js",
-  "bin": "src/index.js",
+  "bin": {
+    "budi": "src/index.js"
+  },
   "author": "Budibase",
   "license": "AGPL-3.0-or-later",
   "scripts": {

packages/cli/src/constants.js

@@ -2,3 +2,8 @@ exports.CommandWords = {
   HOSTING: "hosting",
   HELP: "help",
 }
+
+exports.InitTypes = {
+  QUICK: "quick",
+  DIGITAL_OCEAN: "do",
+}

packages/cli/src/hosting/index.js

@@ -1,11 +1,18 @@
 const Command = require("../structures/Command")
-const { CommandWords } = require("../constants")
+const { CommandWords, InitTypes } = require("../constants")
 const { lookpath } = require("lookpath")
-const { downloadFile, logErrorToFile, success, info } = require("../utils")
+const {
+  downloadFile,
+  logErrorToFile,
+  success,
+  info,
+  parseEnv,
+} = require("../utils")
 const { confirmation } = require("../questions")
 const fs = require("fs")
 const compose = require("docker-compose")
-const envFile = require("./makeEnv")
+const makeEnv = require("./makeEnv")
+const axios = require("axios")
 
 const BUDIBASE_SERVICES = ["app-service", "worker-service"]
 const ERROR_FILE = "docker-error.log"
@@ -13,6 +20,7 @@ const FILE_URLS = [
   "https://raw.githubusercontent.com/Budibase/budibase/master/hosting/docker-compose.yaml",
   "https://raw.githubusercontent.com/Budibase/budibase/master/hosting/envoy.yaml",
 ]
+const DO_USER_DATA_URL = "http://169.254.169.254/metadata/v1/user-data"
 
 async function downloadFiles() {
   const promises = []
@@ -34,7 +42,7 @@ async function checkDockerConfigured() {
 }
 
 function checkInitComplete() {
-  if (!fs.existsSync(envFile.filePath)) {
+  if (!fs.existsSync(makeEnv.filePath)) {
     throw "Please run the hosting --init command before any other hosting command."
   }
 }
@@ -50,8 +58,10 @@ async function handleError(func) {
   }
 }
 
-async function init() {
+async function init(type) {
+  const isQuick = type === InitTypes.QUICK || type === InitTypes.DIGITAL_OCEAN
   await checkDockerConfigured()
+  if (!isQuick) {
     const shouldContinue = await confirmation(
       "This will create multiple files in current directory, should continue?"
     )
@@ -59,15 +69,30 @@ async function init() {
       console.log("Stopping.")
       return
     }
+  }
   await downloadFiles()
-  await envFile.make()
+  const config = isQuick ? makeEnv.QUICK_CONFIG : {}
+  if (type === InitTypes.DIGITAL_OCEAN) {
+    try {
+      const output = await axios.get(DO_USER_DATA_URL)
+      const response = parseEnv(output.data)
+      for (let [key, value] of Object.entries(makeEnv.ConfigMap)) {
+        if (response[key]) {
+          config[value] = response[key]
+        }
+      }
+    } catch (err) {
+      // don't need to handle error, just don't do anything
+    }
+  }
+  await makeEnv.make(config)
 }
 
 async function start() {
   await checkDockerConfigured()
   checkInitComplete()
   console.log(info("Starting services, this may take a moment."))
-  const port = envFile.get("MAIN_PORT")
+  const port = makeEnv.get("MAIN_PORT")
   await handleError(async () => {
     await compose.upAll({ cwd: "./", log: false })
   })
@@ -128,8 +153,8 @@ async function update() {
 const command = new Command(`${CommandWords.HOSTING}`)
   .addHelp("Controls self hosting on the Budibase platform.")
   .addSubOption(
-    "--init",
+    "--init [type]",
-    "Configure a self hosted platform in current directory.",
+    "Configure a self hosted platform in current directory, type can be unspecified or 'quick'.",
     init
   )
   .addSubOption(

packages/cli/src/hosting/makeEnv.js

@@ -26,19 +26,32 @@ APP_PORT=4002
 WORKER_PORT=4003
 MINIO_PORT=4004
 COUCH_DB_PORT=4005
+REDIS_PORT=6379
 BUDIBASE_ENVIRONMENT=PRODUCTION`
 }
 
 module.exports.filePath = FILE_PATH
+module.exports.ConfigMap = {
+  HOSTING_KEY: "key",
+  MAIN_PORT: "port",
+}
+module.exports.QUICK_CONFIG = {
+  key: "budibase",
+  port: 10000,
+}
 
-module.exports.make = async () => {
+module.exports.make = async (inputs = {}) => {
-  const hostingKey = await string(
+  const hostingKey =
+    inputs.key ||
+    (await string(
       "Please input the password you'd like to use as your hosting key: "
-  )
+    ))
-  const hostingPort = await number(
+  const hostingPort =
+    inputs.port ||
+    (await number(
       "Please enter the port on which you want your installation to run: ",
       10000
-  )
+    ))
   const fileContents = getContents(hostingPort, hostingKey)
   fs.writeFileSync(FILE_PATH, fileContents)
   console.log(

packages/cli/src/index.js

@@ -1,3 +1,4 @@
+#!/usr/bin/env node
 const { getCommands } = require("./options")
 const { Command } = require("commander")
 const { getHelpDescription } = require("./utils")

packages/cli/src/structures/Command.js

@@ -13,8 +13,8 @@ class Command {
     return this
   }
 
-  addSubOption(command, help, func) {
+  addSubOption(command, help, func, extras = []) {
-    this.opts.push({ command, help, func })
+    this.opts.push({ command, help, func, extras })
     return this
   }
 
@@ -37,13 +37,10 @@ class Command {
     command.action(async options => {
       try {
         let executed = false
-        if (thisCmd.func) {
-          await thisCmd.func(options)
-          executed = true
-        }
         for (let opt of thisCmd.opts) {
-          if (options[opt.command.replace("--", "")]) {
+          const lookup = opt.command.split(" ")[0].replace("--", "")
-            await opt.func(options)
+          if (options[lookup]) {
+            await opt.func(options[lookup])
             executed = true
           }
         }

packages/cli/src/utils.js

@@ -44,3 +44,15 @@ exports.info = info => {
 exports.logErrorToFile = (file, error) => {
   fs.writeFileSync(path.resolve(`./${file}`), `Budiase Error\n${error}`)
 }
+
+exports.parseEnv = env => {
+  const lines = env.toString().split("\n")
+  let result = {}
+  for (const line of lines) {
+    const match = line.match(/^([^=:#]+?)[=:](.*)/)
+    if (match) {
+      result[match[1].trim()] = match[2].trim()
+    }
+  }
+  return result
+}

packages/cli/yarn.lock

@@ -15,30 +15,30 @@
   integrity sha512-np/lG3uARFybkoHokJUmf1QfEvRVCPbmQeUQpKow5cQ3xWrV9i3rUHodKDJPQfTVX61qKi+UdYk8kik84n7XOw==
 
 "@babel/highlight@^7.10.4":
-  version "7.12.13"
+  version "7.13.10"
-  resolved "https://registry.yarnpkg.com/@babel/highlight/-/highlight-7.12.13.tgz#8ab538393e00370b26271b01fa08f7f27f2e795c"
+  resolved "https://registry.yarnpkg.com/@babel/highlight/-/highlight-7.13.10.tgz#a8b2a66148f5b27d666b15d81774347a731d52d1"
-  integrity sha512-kocDQvIbgMKlWxXe9fof3TQ+gkIPOUSEYhJjqUjvKMez3krV7vbzYCDq39Oj11UAVK7JqPVGQPlgE85dPNlQww==
+  integrity sha512-5aPpe5XQPzflQrFwL1/QoeHkP2MsA4JCntcXHRhEsdsfPVkvPi2w7Qix4iV7t5S/oC9OodGrggd8aco1g3SZFg==
   dependencies:
     "@babel/helper-validator-identifier" "^7.12.11"
     chalk "^2.0.0"
     js-tokens "^4.0.0"
 
 "@babel/parser@^7.9.4":
-  version "7.13.4"
+  version "7.13.11"
-  resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.13.4.tgz#340211b0da94a351a6f10e63671fa727333d13ab"
+  resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.13.11.tgz#f93ebfc99d21c1772afbbaa153f47e7ce2f50b88"
-  integrity sha512-uvoOulWHhI+0+1f9L4BoozY7U5cIkZ9PgJqvb041d6vypgUmtVPG4vmGm4pSggjl8BELzvHyUeJSUyEMY6b+qA==
+  integrity sha512-PhuoqeHoO9fc4ffMEVk4qb/w/s2iOSWohvbHxLtxui0eBg3Lg5gN1U8wp1V1u61hOWkPQJJyJzGH6Y+grwkq8Q==
 
 "@babel/runtime@^7.9.2":
-  version "7.13.7"
+  version "7.13.10"
-  resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.13.7.tgz#d494e39d198ee9ca04f4dcb76d25d9d7a1dc961a"
+  resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.13.10.tgz#47d42a57b6095f4468da440388fdbad8bebf0d7d"
-  integrity sha512-h+ilqoX998mRVM5FtB5ijRuHUDVt5l3yfoOi2uh18Z/O3hvyaHQ39NpxVkCIG5yFs+mLq/ewFp8Bss6zmWv6ZA==
+  integrity sha512-4QPkjJq6Ns3V/RgpEahRk+AGfL0eO6RHHtTWoNNr5mO49G6B5+X6d6THgWEAvTrznU5xYpbAlVKRYcsCgh/Akw==
   dependencies:
     regenerator-runtime "^0.13.4"
 
-"@eslint/eslintrc@^0.3.0":
+"@eslint/eslintrc@^0.4.0":
-  version "0.3.0"
+  version "0.4.0"
-  resolved "https://registry.yarnpkg.com/@eslint/eslintrc/-/eslintrc-0.3.0.tgz#d736d6963d7003b6514e6324bec9c602ac340318"
+  resolved "https://registry.yarnpkg.com/@eslint/eslintrc/-/eslintrc-0.4.0.tgz#99cc0a0584d72f1df38b900fb062ba995f395547"
-  integrity sha512-1JTKgrOKAHVivSvOYw+sJOunkBjUOvjqWk1DPja7ZFhIS2mX/4EgTT8M7eTK9jrKhL/FvXXEbQwIs3pg1xp3dg==
+  integrity sha512-2ZPCc+uNbjV5ERJr+aKSPRwZgKd2z11x0EgLvb1PURmUrn9QNRXFqje0Ldq454PfAVyaJYyrDvvIKSFP4NnBog==
   dependencies:
     ajv "^6.12.4"
     debug "^4.1.1"
@@ -47,7 +47,6 @@
     ignore "^4.0.6"
     import-fresh "^3.2.1"
     js-yaml "^3.13.1"
-    lodash "^4.17.20"
     minimatch "^3.0.4"
     strip-json-comments "^3.1.1"
 
@@ -93,9 +92,9 @@ ajv@^6.10.0, ajv@^6.12.3, ajv@^6.12.4:
     uri-js "^4.2.2"
 
 ajv@^7.0.2:
-  version "7.1.1"
+  version "7.2.1"
-  resolved "https://registry.yarnpkg.com/ajv/-/ajv-7.1.1.tgz#1e6b37a454021fa9941713f38b952fc1c8d32a84"
+  resolved "https://registry.yarnpkg.com/ajv/-/ajv-7.2.1.tgz#a5ac226171912447683524fa2f1248fcf8bac83d"
-  integrity sha512-ga/aqDYnUy/o7vbsRTFhhTsNeXiYb5JWDIcRIeZfwRNCefwjNTVYCGdGSUrEmiu3yDK3vFvNbgJxvrQW4JXrYQ==
+  integrity sha512-+nu0HDv7kNSOua9apAVc979qd932rrZeb3WOvoiD31A/p1mIE5/9bN2027pE2rOPYEdS3UHzsvof4hY+lM9/WQ==
   dependencies:
     fast-deep-equal "^3.1.1"
     json-schema-traverse "^1.0.0"
@@ -434,12 +433,12 @@ eslint-visitor-keys@^2.0.0:
   integrity sha512-QudtT6av5WXels9WjIM7qz1XD1cWGvX4gGXvp/zBn9nXG02D0utdU3Em2m/QjTnrsk6bBjmCygl3rmj118msQQ==
 
 eslint@^7.20.0:
-  version "7.20.0"
+  version "7.22.0"
-  resolved "https://registry.yarnpkg.com/eslint/-/eslint-7.20.0.tgz#db07c4ca4eda2e2316e7aa57ac7fc91ec550bdc7"
+  resolved "https://registry.yarnpkg.com/eslint/-/eslint-7.22.0.tgz#07ecc61052fec63661a2cab6bd507127c07adc6f"
-  integrity sha512-qGi0CTcOGP2OtCQBgWZlQjcTuP0XkIpYFj25XtRTQSHC+umNnp7UMshr2G8SLsRFYDdAPFeHOsiteadmMH02Yw==
+  integrity sha512-3VawOtjSJUQiiqac8MQc+w457iGLfuNGLFn8JmF051tTKbh5/x/0vlcEj8OgDCaw7Ysa2Jn8paGshV7x2abKXg==
   dependencies:
     "@babel/code-frame" "7.12.11"
-    "@eslint/eslintrc" "^0.3.0"
+    "@eslint/eslintrc" "^0.4.0"
     ajv "^6.10.0"
     chalk "^4.0.0"
     cross-spawn "^7.0.2"
@@ -452,10 +451,10 @@ eslint@^7.20.0:
     espree "^7.3.1"
     esquery "^1.4.0"
     esutils "^2.0.2"
-    file-entry-cache "^6.0.0"
+    file-entry-cache "^6.0.1"
     functional-red-black-tree "^1.0.1"
     glob-parent "^5.0.0"
-    globals "^12.1.0"
+    globals "^13.6.0"
     ignore "^4.0.6"
     import-fresh "^3.0.0"
     imurmurhash "^0.1.4"
@@ -463,7 +462,7 @@ eslint@^7.20.0:
     js-yaml "^3.13.1"
     json-stable-stringify-without-jsonify "^1.0.1"
     levn "^0.4.1"
-    lodash "^4.17.20"
+    lodash "^4.17.21"
     minimatch "^3.0.4"
     natural-compare "^1.4.0"
     optionator "^0.9.1"
@@ -589,7 +588,7 @@ figures@^3.0.0:
   dependencies:
     escape-string-regexp "^1.0.5"
 
-file-entry-cache@^6.0.0:
+file-entry-cache@^6.0.1:
   version "6.0.1"
   resolved "https://registry.yarnpkg.com/file-entry-cache/-/file-entry-cache-6.0.1.tgz#211b2dd9659cb0394b073e7323ac3c933d522027"
   integrity sha512-7Gps/XWymbLk2QLYK4NzpMOrYjMhdIxXuIvy2QBsLE6ljuodKvdkWs/cpyJJ3CVIVpH0Oi1Hvg1ovbMzLdFBBg==
@@ -617,9 +616,9 @@ flatted@^3.1.0:
   integrity sha512-zAoAQiudy+r5SvnSw3KJy5os/oRJYHzrzja/tBDqrZtNhUw8bt6y8OBzMWcjWr+8liV8Eb6yOhw8WZ7VFZ5ZzA==
 
 follow-redirects@^1.10.0:
-  version "1.13.2"
+  version "1.13.3"
-  resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.13.2.tgz#dd73c8effc12728ba5cf4259d760ea5fb83e3147"
+  resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.13.3.tgz#e5598ad50174c1bc4e872301e82ac2cd97f90267"
-  integrity sha512-6mPTgLxYm3r6Bkkg0vNM0HTjfGrOEtsfbhagQvbxDEsEkpNhw582upBaoRZylzen6krEmxXJgt9Ju6HiI4O7BA==
+  integrity sha512-DUgl6+HDzB0iEptNQEXLx/KhTmDb8tZUHSeLqpnjpknR70H0nC2t9N73BK6fN4hOvJ84pKlIQVQ4k5FFlBedKA==
 
 forever-agent@~0.6.1:
   version "0.6.1"
@@ -675,9 +674,9 @@ getpass@^0.1.1:
     assert-plus "^1.0.0"
 
 glob-parent@^5.0.0, glob-parent@^5.1.0:
-  version "5.1.1"
+  version "5.1.2"
-  resolved "https://registry.yarnpkg.com/glob-parent/-/glob-parent-5.1.1.tgz#b6c1ef417c4e5663ea498f1c45afac6916bbc229"
+  resolved "https://registry.yarnpkg.com/glob-parent/-/glob-parent-5.1.2.tgz#869832c58034fe68a4093c17dc15e8340d8401c4"
-  integrity sha512-FnI+VGOpnlGHWZxthPGR+QhR78fuiK0sNLkHQv+bL9fQi57lNNdquIbna/WrfROrolq8GK5Ek6BiMwqL/voRYQ==
+  integrity sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==
   dependencies:
     is-glob "^4.0.1"
 
@@ -700,6 +699,13 @@ globals@^12.1.0:
   dependencies:
     type-fest "^0.8.1"
 
+globals@^13.6.0:
+  version "13.7.0"
+  resolved "https://registry.yarnpkg.com/globals/-/globals-13.7.0.tgz#aed3bcefd80ad3ec0f0be2cf0c895110c0591795"
+  integrity sha512-Aipsz6ZKRxa/xQkZhNg0qIWXT6x6rD46f6x/PCnBomlttdIyAPak4YD9jTmKpZ72uROSMU87qJtcgpgHaVchiA==
+  dependencies:
+    type-fest "^0.20.2"
+
 globby@^11.0.0:
   version "11.0.2"
   resolved "https://registry.yarnpkg.com/globby/-/globby-11.0.2.tgz#1af538b766a3b540ebfb58a32b2e2d5897321d83"
@@ -957,9 +963,9 @@ lodash@^4.17.20, lodash@^4.17.21:
   integrity sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==
 
 lookpath@^1.1.0:
-  version "1.1.0"
+  version "1.2.0"
-  resolved "https://registry.yarnpkg.com/lookpath/-/lookpath-1.1.0.tgz#932d68371a2f0b4a5644f03d6a2b4728edba96d2"
+  resolved "https://registry.yarnpkg.com/lookpath/-/lookpath-1.2.0.tgz#5fccf91497acec085e66d98cb12446c21fe665ae"
-  integrity sha512-B9NM7XpVfkyWqfOBI/UW0kVhGw7pJztsduch+1wkbYDi90mYK6/InFul3lG0hYko/VEcVMARVBJ5daFRc5aKCw==
+  integrity sha512-cUl+R2bGJcSJiHLVKzGHRTYTBhudbHIgd7s63gfGHteaz0BBKEEz2yw2rgbxZAFze92KlbkiWzL1ylYOmqIPVA==
 
 lru-cache@^6.0.0:
   version "6.0.0"
@@ -1417,9 +1423,9 @@ stream-meter@^1.0.4:
     readable-stream "^2.1.4"
 
 string-width@^4.1.0, string-width@^4.2.0:
-  version "4.2.1"
+  version "4.2.2"
-  resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.1.tgz#1933ce1f470973d224368009bd1316cad81d5f4f"
+  resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.2.tgz#dafd4f9559a7585cfba529c6a0a4f73488ebd4c5"
-  integrity sha512-LL0OLyN6AnfV9xqGQpDBwedT2Rt63737LxvsRxbcwpa2aIeynBApG2Sm//F3TaLHIR1aJBN52DWklc06b94o5Q==
+  integrity sha512-XBJbT3N4JhVumXE0eoLU9DCjcaF92KLNqTmFCnG1pf8duUxFGwtP6AD6nkjw9a3IdiRtL3E2w3JDiE/xi3vOeA==
   dependencies:
     emoji-regex "^8.0.0"
     is-fullwidth-code-point "^3.0.0"
@@ -1541,6 +1547,11 @@ type-fest@^0.11.0:
   resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-0.11.0.tgz#97abf0872310fed88a5c466b25681576145e33f1"
   integrity sha512-OdjXJxnCN1AvyLSzeKIgXTXxV+99ZuXl3Hpo9XpJAv9MBcHrrJOQ5kV7ypXOuQie+AmWG25hLbiKdwYTifzcfQ==
 
+type-fest@^0.20.2:
+  version "0.20.2"
+  resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-0.20.2.tgz#1bf207f4b28f91583666cb5fbd327887301cd5f4"
+  integrity sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ==
+
 type-fest@^0.8.1:
   version "0.8.1"
   resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-0.8.1.tgz#09e249ebde851d3b1e48d27c105444667f17b83d"
@@ -1583,9 +1594,9 @@ uuid@^3.3.2:
   integrity sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==
 
 v8-compile-cache@^2.0.3:
-  version "2.2.0"
+  version "2.3.0"
-  resolved "https://registry.yarnpkg.com/v8-compile-cache/-/v8-compile-cache-2.2.0.tgz#9471efa3ef9128d2f7c6a7ca39c4dd6b5055b132"
+  resolved "https://registry.yarnpkg.com/v8-compile-cache/-/v8-compile-cache-2.3.0.tgz#2de19618c66dc247dcfb6f99338035d8245a2cee"
-  integrity sha512-gTpR5XQNKFwOd4clxfnhaqvfqMpqEwr4tOtCyz4MtYZX2JYhfr1JvBFKdS+7K/9rfpZR3VLX+YWBbKoxCgS43Q==
+  integrity sha512-l8lCEmLcLYZh4nbunNZvQCJc5pv7+RCwa8q/LdUx8u7lsWvPDKmpodJAJNwkAhJC//dFY48KuIEmjtd4RViDrA==
 
 verror@1.10.0:
   version "1.10.0"

packages/client/src/api/tables.js

@@ -21,14 +21,16 @@ export const fetchTableData = async tableId => {
  * Perform a mango query against an internal table
  * @param {String} tableId - id of the table to search
  * @param {Object} search - Mango Compliant search object
+ * @param {Object} pagination - the pagination controls
  */
 export const searchTableData = async ({ tableId, search, pagination }) => {
-  const rows = await API.post({
+  const output = await API.post({
     url: `/api/${tableId}/rows/search`,
     body: {
       query: search,
       pagination,
     },
   })
-  return await enrichRows(rows, tableId)
+  output.rows = await enrichRows(output.rows, tableId)
+  return output
 }

packages/server/.env.template

@@ -1,17 +0,0 @@
-# url of couch db, including username and password
-# http://admin:password@localhost:5984
-COUCH_DB_URL={{couchDbUrl}}
-
-# identifies a client database - i.e. group of apps
-CLIENT_ID={{clientId}}
-
-# used to create cookie hashes
-JWT_SECRET={{cookieKey1}}
-
-# error level for koa-pino
-LOG_LEVEL=info
-
-DEPLOYMENT_CREDENTIALS_URL="https://dt4mpwwap8.execute-api.eu-west-1.amazonaws.com/prod/"
-DEPLOYMENT_DB_URL="https://couchdb.budi.live:5984"
-SENTRY_DSN=https://a34ae347621946bf8acded18e5b7d4b8@o420233.ingest.sentry.io/5338131
-ENABLE_ANALYTICS="true"

packages/server/Dockerfile

@@ -2,7 +2,6 @@ FROM node:12-alpine
 
 WORKDIR /app
 
-ENV CLOUD=1
 ENV PORT=4001
 ENV COUCH_DB_URL=https://couchdb.budi.live:5984
 ENV BUDIBASE_ENVIRONMENT=PRODUCTION

packages/server/__mocks__/node-fetch.js

@@ -30,6 +30,17 @@ module.exports = async (url, opts) => {
       },
       404
     )
+  } else if (url.includes("_search")) {
+    return json({
+      rows: [
+        {
+          doc: {
+            _id: "test",
+          },
+        },
+      ],
+      bookmark: "test",
+    })
   }
   return fetch(url, opts)
 }

packages/server/package.json

@@ -36,7 +36,10 @@
     "test:integration": "jest --coverage --detectOpenHandles",
     "test:watch": "jest --watch",
     "run:docker": "node src/index",
-    "dev:builder": "cross-env PORT=4001 nodemon src/index.js",
+    "dev:stack:up": "node scripts/dev/manage.js up",
+    "dev:stack:down": "node scripts/dev/manage.js down",
+    "dev:stack:nuke": "node scripts/dev/manage.js nuke",
+    "dev:builder": "npm run dev:stack:up && nodemon src/index.js",
     "electron": "electron src/electron.js",
     "build:electron": "electron-builder --dir",
     "publish:electron": "electron-builder -mwl --publish always",
@@ -63,7 +66,7 @@
       "!src/tests/**/*",
       "!src/automations/tests/**/*",
       "!src/utilities/fileProcessor.js",
-      "!src/utilities/initialiseBudibase.js"
+      "!src/utilities/fileSystem/**/*"
     ],
     "coverageReporters": [
       "lcov",
@@ -134,7 +137,7 @@
   "devDependencies": {
     "@budibase/standard-components": "^0.8.9",
     "@jest/test-sequencer": "^24.8.0",
-    "cross-env": "^7.0.3",
+    "docker-compose": "^0.23.6",
     "electron": "10.1.3",
     "electron-builder": "^22.9.1",
     "electron-builder-notarize": "^1.1.2",

packages/server/scripts/dev/manage.js

@@ -0,0 +1,96 @@
+#!/usr/bin/env node
+const compose = require("docker-compose")
+const path = require("path")
+const fs = require("fs")
+
+// This script wraps docker-compose allowing you to manage your dev infrastructure with simple commands.
+const CONFIG = {
+  cwd: path.resolve(process.cwd(), "../../hosting"),
+  config: "docker-compose.dev.yaml",
+  log: true,
+}
+
+const Commands = {
+  Up: "up",
+  Down: "down",
+  Nuke: "nuke",
+}
+
+async function init() {
+  const envFilePath = path.join(process.cwd(), ".env")
+  if (fs.existsSync(envFilePath)) {
+    return
+  }
+  const envFileJson = {
+    PORT: 4001,
+    MINIO_URL: "http://localhost:10000/",
+    COUCH_DB_URL: "http://budibase:budibase@localhost:10000/db/",
+    WORKER_URL: "http://localhost:4002",
+    JWT_SECRET: "testsecret",
+    MINIO_ACCESS_KEY: "budibase",
+    MINIO_SECRET_KEY: "budibase",
+    COUCH_DB_PASSWORD: "budibase",
+    COUCH_DB_USER: "budibase",
+    SELF_HOSTED: 1,
+  }
+  let envFile = ""
+  Object.keys(envFileJson).forEach(key => {
+    envFile += `${key}=${envFileJson[key]}\n`
+  })
+  fs.writeFileSync(envFilePath, envFile)
+}
+
+async function up() {
+  console.log("Spinning up your budibase dev environment... 🔧✨")
+  await init()
+  await compose.upAll(CONFIG)
+}
+
+async function down() {
+  console.log("Spinning down your budibase dev environment... 🌇")
+  await compose.stop(CONFIG)
+}
+
+async function nuke() {
+  console.log(
+    "Clearing down your budibase dev environment, including all containers and volumes... 💥"
+  )
+  await compose.down(CONFIG)
+}
+
+const managementCommand = process.argv.slice(2)[0]
+
+if (
+  !managementCommand ||
+  !Object.values(Commands).some(command => managementCommand === command)
+) {
+  throw new Error(
+    "You must supply either an 'up', 'down' or 'nuke' commmand to manage the budibase development environment."
+  )
+}
+
+let command
+switch (managementCommand) {
+  case Commands.Up:
+    command = up
+    break
+  case Commands.Down:
+    command = down
+    break
+  case Commands.Nuke:
+    command = nuke
+    break
+  default:
+    command = up
+}
+
+command()
+  .then(() => {
+    console.log("Done! 🎉")
+  })
+  .catch(err => {
+    console.error(
+      "Something went wrong while managing budibase dev environment:",
+      err.message
+    )
+  })

packages/server/scripts/exportAppTemplate.js

@@ -1,6 +1,10 @@
 #!/usr/bin/env node
-const { exportTemplateFromApp } = require("../src/utilities/templates")
 const yargs = require("yargs")
+const fs = require("fs")
+const { join } = require("path")
+const CouchDB = require("../src/db")
+// load environment
+const env = require("../src/environment")
 
 // Script to export a chosen budibase app into a package
 // Usage: ./scripts/exportAppTemplate.js export --name=Funky --appId=appId
@@ -22,18 +26,26 @@ yargs
       },
     },
     async args => {
+      if (!env.isDev()) {
+        throw "Only works in dev"
+      }
+      const name = args.name,
+        appId = args.appId
       console.log("Exporting app..")
-      if (args.name == null || args.appId == null) {
+      if (name == null || appId == null) {
         console.error(
           "Unable to export without a name and app ID being specified, check help for more info."
         )
         return
       }
-      const exportPath = await exportTemplateFromApp({
+      const exportPath = join(process.cwd(), name, "db")
-        templateName: args.name,
+      fs.ensureDirSync(exportPath)
-        appId: args.appId,
+      const writeStream = fs.createWriteStream(join(exportPath, "dump.text"))
-      })
+      // perform couch dump
-      console.log(`Template ${args.name} exported to ${exportPath}`)
+
+      const instanceDb = new CouchDB(appId)
+      await instanceDb.dump(writeStream, {})
+      console.log(`Template ${name} exported to ${exportPath}`)
     }
   )
   .help()

packages/server/scripts/initialise.js

@@ -1,17 +0,0 @@
-const { join } = require("path")
-const { homedir } = require("os")
-
-const initialiseBudibase = require("../src/utilities/initialiseBudibase")
-const DIRECTORY = "~/.budibase"
-
-function run() {
-  let opts = {}
-  let dir = DIRECTORY
-  opts.quiet = true
-  opts.dir = dir.startsWith("~") ? join(homedir(), dir.substring(1)) : dir
-  return initialiseBudibase(opts)
-}
-
-run().then(() => {
-  console.log("Init complete.")
-})

packages/server/src/api/controllers/apikeys.js

@@ -1,56 +1,32 @@
-const fs = require("fs")
+const builderDB = require("../../db/builder")
-const { join } = require("../../utilities/centralPath")
-const readline = require("readline")
-const { budibaseAppsDir } = require("../../utilities/budibaseDir")
-const env = require("../../environment")
-const ENV_FILE_PATH = "/.env"
 
 exports.fetch = async function(ctx) {
-  ctx.status = 200
+  try {
-  ctx.body = {
+    const mainDoc = await builderDB.getBuilderMainDoc()
-    budibase: env.BUDIBASE_API_KEY,
+    ctx.body = mainDoc.apiKeys ? mainDoc.apiKeys : {}
-    userId: env.USERID_API_KEY,
+  } catch (err) {
+    /* istanbul ignore next */
+    ctx.throw(400, err)
   }
 }
 
 exports.update = async function(ctx) {
-  const key = `${ctx.params.key.toUpperCase()}_API_KEY`
+  const key = ctx.params.key
   const value = ctx.request.body.value
 
-  // set environment variables
+  try {
-  env._set(key, value)
+    const mainDoc = await builderDB.getBuilderMainDoc()
-
+    if (mainDoc.apiKeys == null) {
-  // Write to file
+      mainDoc.apiKeys = {}
-  await updateValues([key, value])
-
-  ctx.status = 200
-  ctx.message = `Updated ${ctx.params.key} API key succesfully.`
-  ctx.body = { [ctx.params.key]: ctx.request.body.value }
     }
-
+    mainDoc.apiKeys[key] = value
-async function updateValues([key, value]) {
+    const resp = await builderDB.setBuilderMainDoc(mainDoc)
-  let newContent = ""
+    ctx.body = {
-  let keyExists = false
+      _id: resp.id,
-  let envPath = join(budibaseAppsDir(), ENV_FILE_PATH)
+      _rev: resp.rev,
-  const readInterface = readline.createInterface({
-    input: fs.createReadStream(envPath),
-    output: process.stdout,
-    console: false,
-  })
-  readInterface.on("line", function(line) {
-    // Mutate lines and change API Key
-    if (line.startsWith(key)) {
-      line = `${key}=${value}`
-      keyExists = true
     }
-    newContent = `${newContent}\n${line}`
+  } catch (err) {
-  })
+    /* istanbul ignore next */
-  readInterface.on("close", function() {
+    ctx.throw(400, err)
-    // Write file here
-    if (!keyExists) {
-      // Add API Key if it doesn't exist in the file at all
-      newContent = `${newContent}\n${key}=${value}`
   }
-    fs.writeFileSync(envPath, newContent)
-  })
 }

packages/server/src/api/controllers/application.js

@@ -1,15 +1,17 @@
 const CouchDB = require("../../db")
-const compileStaticAssets = require("../../utilities/builder/compileStaticAssets")
 const env = require("../../environment")
-const { existsSync } = require("fs-extra")
-const { budibaseAppsDir } = require("../../utilities/budibaseDir")
 const setBuilderToken = require("../../utilities/builder/setBuilderToken")
-const fs = require("fs-extra")
-const { join, resolve } = require("../../utilities/centralPath")
 const packageJson = require("../../../package.json")
-const { createLinkView } = require("../../db/linkedRows")
+const {
-const { createRoutingView } = require("../../utilities/routing")
+  createLinkView,
-const { downloadTemplate } = require("../../utilities/templates")
+  createRoutingView,
+  createAllSearchIndex,
+} = require("../../db/views/staticViews")
+const {
+  getTemplateStream,
+  createApp,
+  deleteApp,
+} = require("../../utilities/fileSystem")
 const {
   generateAppID,
   getLayoutParams,
@@ -20,9 +22,6 @@ const {
   BUILTIN_ROLE_IDS,
   AccessController,
 } = require("../../utilities/security/roles")
-const {
-  downloadExtractComponentLibraries,
-} = require("../../utilities/createAppPackage")
 const { BASE_LAYOUTS } = require("../../constants/layouts")
 const {
   createHomeScreen,
@@ -32,11 +31,7 @@ const { cloneDeep } = require("lodash/fp")
 const { processObject } = require("@budibase/string-templates")
 const { getAllApps } = require("../../utilities")
 const { USERS_TABLE_SCHEMA } = require("../../constants")
-const {
+const { getDeployedApps } = require("../../utilities/builder/hosting")
-  getDeployedApps,
-  getHostingInfo,
-  HostingTypes,
-} = require("../../utilities/builder/hosting")
 
 const URL_REGEX_SLASH = /\/|\\/g
 
@@ -75,8 +70,7 @@ async function getAppUrlIfNotInUse(ctx) {
     url = encodeURI(`${ctx.request.body.name}`)
   }
   url = `/${url.replace(URL_REGEX_SLASH, "")}`.toLowerCase()
-  const hostingInfo = await getHostingInfo()
+  if (!env.SELF_HOSTED) {
-  if (hostingInfo.type === HostingTypes.CLOUD) {
     return url
   }
   const deployedApps = await getDeployedApps()
@@ -101,21 +95,13 @@ async function createInstance(template) {
   // add view for linked rows
   await createLinkView(appId)
   await createRoutingView(appId)
+  await createAllSearchIndex(appId)
 
   // replicate the template data to the instance DB
   // this is currently very hard to test, downloading and importing template files
   /* istanbul ignore next */
   if (template && template.useTemplate === "true") {
-    let dbDumpReadStream
+    const { ok } = await db.load(await getTemplateStream(template))
-    if (template.file) {
-      dbDumpReadStream = fs.createReadStream(template.file.path)
-    } else {
-      const templatePath = await downloadTemplate(...template.key.split("/"))
-      dbDumpReadStream = fs.createReadStream(
-        join(templatePath, "db", "dump.txt")
-      )
-    }
-    const { ok } = await db.load(dbDumpReadStream)
     if (!ok) {
       throw "Error loading database dump from template."
     }
@@ -190,10 +176,10 @@ exports.create = async function(ctx) {
   const instanceDb = new CouchDB(appId)
   await instanceDb.put(newApplication)
 
-  const newAppFolder = await createEmptyAppPackage(ctx, newApplication)
+  await createEmptyAppPackage(ctx, newApplication)
   /* istanbul ignore next */
-  if (env.NODE_ENV !== "jest") {
+  if (!env.isTest()) {
-    await downloadExtractComponentLibraries(newAppFolder)
+    await createApp(appId)
   }
 
   await setBuilderToken(ctx, appId, version)
@@ -222,11 +208,10 @@ exports.delete = async function(ctx) {
   const db = new CouchDB(ctx.params.appId)
   const app = await db.get(ctx.params.appId)
   const result = await db.destroy()
-
+  /* istanbul ignore next */
-  // remove top level directory
+  if (!env.isTest()) {
-  await fs.rmdir(join(budibaseAppsDir(), ctx.params.appId), {
+    await deleteApp(ctx.params.appId)
-    recursive: true,
+  }
-  })
 
   ctx.status = 200
   ctx.message = `Application ${app.name} deleted successfully.`
@@ -234,17 +219,8 @@ exports.delete = async function(ctx) {
 }
 
 const createEmptyAppPackage = async (ctx, app) => {
-  const appsFolder = budibaseAppsDir()
-  const newAppFolder = resolve(appsFolder, app._id)
-
   const db = new CouchDB(app._id)
 
-  if (existsSync(newAppFolder)) {
-    ctx.throw(400, "App folder already exists for this application")
-  }
-
-  fs.mkdirpSync(newAppFolder)
-
   let screensAndLayouts = []
   for (let layout of BASE_LAYOUTS) {
     const cloned = cloneDeep(layout)
@@ -260,6 +236,4 @@ const createEmptyAppPackage = async (ctx, app) => {
   screensAndLayouts.push(loginScreen)
 
   await db.bulkDocs(screensAndLayouts)
-  await compileStaticAssets(app._id)
-  return newAppFolder
 }

packages/server/src/api/controllers/auth.js

@@ -45,9 +45,9 @@ exports.authenticate = async ctx => {
       roleId: dbUser.roleId,
       version: app.version,
     }
-    // if in cloud add the user api key, unless self hosted
+    // if in prod add the user api key, unless self hosted
     /* istanbul ignore next */
-    if (env.CLOUD && !env.SELF_HOSTED) {
+    if (env.isProd() && !env.SELF_HOSTED) {
       const { apiKey } = await getAPIKey(ctx.user.appId)
       payload.apiKey = apiKey
     }

packages/server/src/api/controllers/backup.js

@@ -1,28 +1,10 @@
-const { performDump } = require("../../utilities/templates")
+const { performBackup } = require("../../utilities/fileSystem")
-const path = require("path")
-const os = require("os")
-const fs = require("fs-extra")
 
 exports.exportAppDump = async function(ctx) {
   const { appId } = ctx.query
-
   const appname = decodeURI(ctx.query.appname)
-
-  const backupsDir = path.join(os.homedir(), ".budibase", "backups")
-  fs.ensureDirSync(backupsDir)
-
   const backupIdentifier = `${appname}Backup${new Date().getTime()}.txt`
 
-  await performDump({
-    dir: backupsDir,
-    appId,
-    name: backupIdentifier,
-  })
-
-  ctx.status = 200
-
-  const backupFile = path.join(backupsDir, backupIdentifier)
-
   ctx.attachment(backupIdentifier)
-  ctx.body = fs.createReadStream(backupFile)
+  ctx.body = await performBackup(appId, backupIdentifier)
 }

packages/server/src/api/controllers/component.js

@@ -1,44 +1,30 @@
 const CouchDB = require("../../db")
-const { resolve, join } = require("../../utilities/centralPath")
+const { getComponentLibraryManifest } = require("../../utilities/fileSystem")
-const {
-  budibaseTempDir,
-  budibaseAppsDir,
-} = require("../../utilities/budibaseDir")
 
 exports.fetchAppComponentDefinitions = async function(ctx) {
   const appId = ctx.params.appId || ctx.appId
   const db = new CouchDB(appId)
   const app = await db.get(appId)
 
-  ctx.body = app.componentLibraries.reduce((acc, componentLibrary) => {
+  let componentManifests = await Promise.all(
-    let appDirectory = resolve(budibaseAppsDir(), appId, "node_modules")
+    app.componentLibraries.map(async library => {
-
+      let manifest = await getComponentLibraryManifest(appId, library)
-    if (ctx.isDev) {
-      appDirectory = budibaseTempDir()
-    }
-
-    const componentJson = require(join(
-      appDirectory,
-      componentLibrary,
-      ctx.isDev ? "" : "package",
-      "manifest.json"
-    ))
-
-    const result = {}
-
-    // map over the components.json and add the library identifier as a key
-    // button -> @budibase/standard-components/button
-    for (let key of Object.keys(componentJson)) {
-      const fullComponentName = `${componentLibrary}/${key}`.toLowerCase()
-      result[fullComponentName] = {
-        component: fullComponentName,
-        ...componentJson[key],
-      }
-    }
 
       return {
-      ...acc,
+        manifest,
-      ...result,
+        library,
       }
-  }, {})
+    })
+  )
+  const definitions = {}
+  for (let { manifest, library } of componentManifests) {
+    for (let key of Object.keys(manifest)) {
+      const fullComponentName = `${library}/${key}`.toLowerCase()
+      definitions[fullComponentName] = {
+        component: fullComponentName,
+        ...manifest[key],
+      }
+    }
+  }
+  ctx.body = definitions
 }

packages/server/src/api/controllers/deploy/awsDeploy.js

@@ -66,12 +66,7 @@ exports.deploy = async function(deployment) {
   const appId = deployment.getAppId()
   const { bucket, accountId } = deployment.getVerification()
   const metadata = { accountId }
-  const s3Client = new AWS.S3({
+  await deployToObjectStore(appId, bucket, metadata)
-    params: {
-      Bucket: bucket,
-    },
-  })
-  await deployToObjectStore(appId, s3Client, metadata)
 }
 
 exports.replicateDb = async function(deployment) {

packages/server/src/api/controllers/deploy/selfDeploy.js

@@ -7,7 +7,6 @@ const {
 const {
   getWorkerUrl,
   getCouchUrl,
-  getMinioUrl,
   getSelfHostKey,
 } = require("../../../utilities/builder/hosting")
 
@@ -45,17 +44,9 @@ exports.postDeployment = async function() {
 exports.deploy = async function(deployment) {
   const appId = deployment.getAppId()
   const verification = deployment.getVerification()
-  const objClient = new AWS.S3({
-    endpoint: await getMinioUrl(),
-    s3ForcePathStyle: true, // needed with minio?
-    signatureVersion: "v4",
-    params: {
-      Bucket: verification.bucket,
-    },
-  })
   // no metadata, aws has account ID in metadata
   const metadata = {}
-  await deployToObjectStore(appId, objClient, metadata)
+  await deployToObjectStore(appId, verification.bucket, metadata)
 }
 
 exports.replicateDb = async function(deployment) {

packages/server/src/api/controllers/deploy/utils.js

@@ -1,16 +1,24 @@
-const fs = require("fs")
-const sanitize = require("sanitize-s3-objectkey")
-const { walkDir } = require("../../../utilities")
 const { join } = require("../../../utilities/centralPath")
+const fs = require("fs")
 const { budibaseAppsDir } = require("../../../utilities/budibaseDir")
 const fetch = require("node-fetch")
 const PouchDB = require("../../../db")
 const CouchDB = require("pouchdb")
+const { upload } = require("../../../utilities/fileSystem")
 
-const CONTENT_TYPE_MAP = {
+// TODO: everything in this file is to be removed
-  html: "text/html",
+
-  css: "text/css",
+function walkDir(dirPath, callback) {
-  js: "application/javascript",
+  for (let filename of fs.readdirSync(dirPath)) {
+    const filePath = `${dirPath}/${filename}`
+    const stat = fs.lstatSync(filePath)
+
+    if (stat.isFile()) {
+      callback(filePath)
+    } else {
+      walkDir(filePath, callback)
+    }
+  }
 }
 
 exports.fetchCredentials = async function(url, body) {
@@ -34,30 +42,25 @@ exports.fetchCredentials = async function(url, body) {
   return json
 }
 
-exports.prepareUpload = async function({ s3Key, metadata, client, file }) {
+exports.prepareUpload = async function({ s3Key, bucket, metadata, file }) {
-  const extension = [...file.name.split(".")].pop()
+  const response = await upload({
-  const fileBytes = fs.readFileSync(file.path)
+    bucket,
-
+    metadata,
-  const upload = await client
+    filename: s3Key,
-    .upload({
+    path: file.path,
-      // windows file paths need to be converted to forward slashes for s3
+    type: file.type,
-      Key: sanitize(s3Key).replace(/\\/g, "/"),
-      Body: fileBytes,
-      ContentType: file.type || CONTENT_TYPE_MAP[extension.toLowerCase()],
-      Metadata: metadata,
   })
-    .promise()
 
   return {
     size: file.size,
     name: file.name,
-    extension,
+    extension: [...file.name.split(".")].pop(),
-    url: upload.Location,
+    url: response.Location,
-    key: upload.Key,
+    key: response.Key,
   }
 }
 
-exports.deployToObjectStore = async function(appId, objectClient, metadata) {
+exports.deployToObjectStore = async function(appId, bucket, metadata) {
   const appAssetsPath = join(budibaseAppsDir(), appId, "public")
 
   let uploads = []
@@ -66,12 +69,12 @@ exports.deployToObjectStore = async function(appId, objectClient, metadata) {
   walkDir(appAssetsPath, function(filePath) {
     const filePathParts = filePath.split("/")
     const appAssetUpload = exports.prepareUpload({
+      bucket,
       file: {
         path: filePath,
         name: filePathParts.pop(),
       },
       s3Key: filePath.replace(appAssetsPath, `assets/${appId}`),
-      client: objectClient,
       metadata,
     })
     uploads.push(appAssetUpload)
@@ -92,7 +95,7 @@ exports.deployToObjectStore = async function(appId, objectClient, metadata) {
     const attachmentUpload = exports.prepareUpload({
       file,
       s3Key: `assets/${appId}/attachments/${file.processedFileName}`,
-      client: objectClient,
+      bucket,
       metadata,
     })
 

packages/server/src/api/controllers/hosting.js

@@ -1,11 +1,11 @@
 const CouchDB = require("../../db")
-const { BUILDER_CONFIG_DB, HOSTING_DOC } = require("../../constants")
 const {
   getHostingInfo,
   getDeployedApps,
   HostingTypes,
   getAppUrl,
 } = require("../../utilities/builder/hosting")
+const { StaticDatabases } = require("../../db/utils")
 
 exports.fetchInfo = async ctx => {
   ctx.body = {
@@ -14,17 +14,17 @@ exports.fetchInfo = async ctx => {
 }
 
 exports.save = async ctx => {
-  const db = new CouchDB(BUILDER_CONFIG_DB)
+  const db = new CouchDB(StaticDatabases.BUILDER_HOSTING.name)
   const { type } = ctx.request.body
   if (type === HostingTypes.CLOUD && ctx.request.body._rev) {
     ctx.body = await db.remove({
       ...ctx.request.body,
-      _id: HOSTING_DOC,
+      _id: StaticDatabases.BUILDER_HOSTING.baseDoc,
     })
   } else {
     ctx.body = await db.put({
       ...ctx.request.body,
-      _id: HOSTING_DOC,
+      _id: StaticDatabases.BUILDER_HOSTING.baseDoc,
     })
   }
 }

packages/server/src/api/controllers/query.js

@@ -93,7 +93,7 @@ exports.find = async function(ctx) {
   const db = new CouchDB(ctx.user.appId)
   const query = enrichQueries(await db.get(ctx.params.queryId))
   // remove properties that could be dangerous in real app
-  if (env.CLOUD) {
+  if (env.isProd()) {
     delete query.fields
     delete query.parameters
     delete query.schema

packages/server/src/api/controllers/row.js

@@ -17,6 +17,7 @@ const {
 const { FieldTypes } = require("../../constants")
 const { isEqual } = require("lodash")
 const { cloneDeep } = require("lodash/fp")
+const { QueryBuilder, search } = require("./search/utils")
 
 const TABLE_VIEW_BEGINS_WITH = `all${SEPARATOR}${DocumentTypes.TABLE}${SEPARATOR}`
 
@@ -259,39 +260,46 @@ exports.search = async function(ctx) {
   const db = new CouchDB(appId)
   const {
     query,
-    pagination: { pageSize = 10, page },
+    pagination: { pageSize = 10, bookmark },
   } = ctx.request.body
+  const tableId = ctx.params.tableId
 
+  const queryBuilder = new QueryBuilder(appId)
+    .setLimit(pageSize)
+    .addTable(tableId)
+  if (bookmark) {
+    queryBuilder.setBookmark(bookmark)
+  }
+
+  let searchString
+  if (ctx.query && ctx.query.raw && ctx.query.raw !== "") {
+    searchString = queryBuilder.complete(query["RAW"])
+  } else {
     // make all strings a starts with operation rather than pure equality
     for (const [key, queryVal] of Object.entries(query)) {
       if (typeof queryVal === "string") {
-      query[key] = {
+        queryBuilder.addString(key, queryVal)
-        $gt: queryVal,
+      } else {
-        $lt: `${queryVal}\uffff`,
+        queryBuilder.addEqual(key, queryVal)
       }
     }
+    searchString = queryBuilder.complete()
   }
 
-  // pure equality for table
+  const response = await search(searchString)
-  query.tableId = ctx.params.tableId
-  const response = await db.find({
-    selector: query,
-    limit: pageSize,
-    skip: pageSize * page,
-  })
-
-  const rows = response.docs
 
   // delete passwords from users
-  if (query.tableId === ViewNames.USERS) {
+  if (tableId === ViewNames.USERS) {
-    for (let row of rows) {
+    for (let row of response.rows) {
       delete row.password
     }
   }
 
-  const table = await db.get(ctx.params.tableId)
+  const table = await db.get(tableId)
-
+  ctx.body = {
-  ctx.body = await outputProcessing(appId, table, rows)
+    rows: await outputProcessing(appId, table, response.rows),
+    bookmark: response.bookmark,
+  }
 }
 
 exports.fetchTableRows = async function(ctx) {

packages/server/src/api/controllers/search/index.js

@@ -0,0 +1,18 @@
+const { QueryBuilder, buildSearchUrl, search } = require("./utils")
+
+exports.rowSearch = async ctx => {
+  // this can't be done through pouch, have to reach for trusty node-fetch
+  const appId = ctx.user.appId
+  const bookmark = ctx.params.bookmark
+  let url
+  if (ctx.params.query) {
+    url = new QueryBuilder(appId, ctx.params.query, bookmark).complete()
+  } else if (ctx.params.raw) {
+    url = buildSearchUrl({
+      appId,
+      query: ctx.params.raw,
+      bookmark,
+    })
+  }
+  ctx.body = await search(url)
+}

packages/server/src/api/controllers/search/utils.js

@@ -0,0 +1,137 @@
+const { SearchIndexes } = require("../../../db/utils")
+const { checkSlashesInUrl } = require("../../../utilities")
+const env = require("../../../environment")
+const fetch = require("node-fetch")
+
+/**
+ * Given a set of inputs this will generate the URL which is to be sent to the search proxy in CouchDB.
+ * @param {string} appId The ID of the app which we will be searching within.
+ * @param {string} query The lucene query string which is to be used for searching.
+ * @param {string|null} bookmark If there were more than the limit specified can send the bookmark that was
+ * returned with query for next set of search results.
+ * @param {number} limit The number of entries to return per query.
+ * @param {boolean} excludeDocs By default full rows are returned, if required this can be disabled.
+ * @return {string} The URL which a GET can be performed on to receive results.
+ */
+function buildSearchUrl({ appId, query, bookmark, excludeDocs, limit = 50 }) {
+  let url = `${env.COUCH_DB_URL}/${appId}/_design/database/_search`
+  url += `/${SearchIndexes.ROWS}?q=${query}`
+  url += `&limit=${limit}`
+  if (!excludeDocs) {
+    url += "&include_docs=true"
+  }
+  if (bookmark) {
+    url += `&bookmark=${bookmark}`
+  }
+  return checkSlashesInUrl(url)
+}
+
+class QueryBuilder {
+  constructor(appId, base) {
+    this.appId = appId
+    this.query = {
+      string: {},
+      fuzzy: {},
+      range: {},
+      equal: {},
+      ...base,
+    }
+    this.limit = 50
+    this.bookmark = null
+  }
+
+  setLimit(limit) {
+    this.limit = limit
+    return this
+  }
+
+  setBookmark(bookmark) {
+    this.bookmark = bookmark
+    return this
+  }
+
+  addString(key, partial) {
+    this.query.string[key] = partial
+    return this
+  }
+
+  addFuzzy(key, fuzzy) {
+    this.query.fuzzy[key] = fuzzy
+    return this
+  }
+
+  addRange(key, low, high) {
+    this.query.range = {
+      low,
+      high,
+    }
+    return this
+  }
+
+  addEqual(key, value) {
+    this.query.equal[key] = value
+    return this
+  }
+
+  addTable(tableId) {
+    this.query.equal.tableId = tableId
+    return this
+  }
+
+  complete(rawQuery = null) {
+    let output = ""
+    function build(structure, queryFn) {
+      for (let [key, value] of Object.entries(structure)) {
+        if (output.length !== 0) {
+          output += " AND "
+        }
+        output += queryFn(key, value)
+      }
+    }
+
+    if (this.query.string) {
+      build(this.query.string, (key, value) => `${key}:${value}*`)
+    }
+    if (this.query.range) {
+      build(
+        this.query.range,
+        (key, value) => `${key}:[${value.low} TO ${value.high}]`
+      )
+    }
+    if (this.query.fuzzy) {
+      build(this.query.fuzzy, (key, value) => `${key}:${value}~`)
+    }
+    if (this.query.equal) {
+      build(this.query.equal, (key, value) => `${key}:${value}`)
+    }
+    if (rawQuery) {
+      output = output.length === 0 ? rawQuery : `&${rawQuery}`
+    }
+    return buildSearchUrl({
+      appId: this.appId,
+      query: output,
+      bookmark: this.bookmark,
+      limit: this.limit,
+    })
+  }
+}
+
+exports.search = async query => {
+  const response = await fetch(query, {
+    method: "GET",
+  })
+  const json = await response.json()
+  let output = {
+    rows: [],
+  }
+  if (json.rows != null && json.rows.length > 0) {
+    output.rows = json.rows.map(row => row.doc)
+  }
+  if (json.bookmark) {
+    output.bookmark = json.bookmark
+  }
+  return output
+}
+
+exports.QueryBuilder = QueryBuilder
+exports.buildSearchUrl = buildSearchUrl

packages/server/src/api/controllers/static/index.js

@@ -2,32 +2,47 @@ require("svelte/register")
 
 const send = require("koa-send")
 const { resolve, join } = require("../../../utilities/centralPath")
+const { checkSlashesInUrl } = require("../../../utilities")
 const fetch = require("node-fetch")
-const fs = require("fs-extra")
 const uuid = require("uuid")
-const AWS = require("aws-sdk")
 const { prepareUpload } = require("../deploy/utils")
 const { processString } = require("@budibase/string-templates")
-const {
+const { budibaseTempDir } = require("../../../utilities/budibaseDir")
-  budibaseAppsDir,
-  budibaseTempDir,
-} = require("../../../utilities/budibaseDir")
 const { getDeployedApps } = require("../../../utilities/builder/hosting")
 const CouchDB = require("../../../db")
 const setBuilderToken = require("../../../utilities/builder/setBuilderToken")
-const fileProcessor = require("../../../utilities/fileProcessor")
+const { loadHandlebarsFile } = require("../../../utilities/fileSystem")
 const env = require("../../../environment")
 const { OBJ_STORE_DIRECTORY } = require("../../../constants")
+const fileProcessor = require("../../../utilities/fileSystem/processor")
+
+const BB_CDN = "https://cdn.app.budi.live/assets"
 
 function objectStoreUrl() {
   if (env.SELF_HOSTED) {
     // can use a relative url for this as all goes through the proxy (this is hosted in minio)
     return OBJ_STORE_DIRECTORY
   } else {
-    return "https://cdn.app.budi.live/assets"
+    return BB_CDN
   }
 }
 
+function internalObjectStoreUrl() {
+  if (env.SELF_HOSTED) {
+    return checkSlashesInUrl(env.MINIO_URL + OBJ_STORE_DIRECTORY)
+  } else {
+    return BB_CDN
+  }
+}
+
+async function returnObjectStoreFile(ctx, path) {
+  const S3_URL = `${internalObjectStoreUrl()}/${path}`
+  const response = await fetch(S3_URL)
+  const body = await response.text()
+  ctx.set("Content-Type", response.headers.get("Content-Type"))
+  ctx.body = body
+}
+
 async function checkForSelfHostedURL(ctx) {
   // the "appId" component of the URL may actually be a specific self hosted URL
   let possibleAppUrl = `/${encodeURI(ctx.params.appId).toLowerCase()}`
@@ -50,106 +65,31 @@ exports.serveBuilder = async function(ctx) {
   await send(ctx, ctx.file, { root: ctx.devPath || builderPath })
 }
 
-exports.serveSelfHostPage = async function(ctx) {
-  const logo = fs.readFileSync(resolve(__dirname, "selfhost/logo.svg"), "utf8")
-  const hostingHbs = fs.readFileSync(
-    resolve(__dirname, "selfhost/index.hbs"),
-    "utf8"
-  )
-  ctx.body = await processString(hostingHbs, {
-    logo,
-  })
-}
-
 exports.uploadFile = async function(ctx) {
-  let files
+  let files =
-  files =
     ctx.request.files.file.length > 1
       ? Array.from(ctx.request.files.file)
       : [ctx.request.files.file]
 
-  const attachmentsPath = resolve(
+  const uploads = files.map(async file => {
-    budibaseAppsDir(),
-    ctx.user.appId,
-    "attachments"
-  )
-
-  if (env.CLOUD) {
-    // remote upload
-    const s3 = new AWS.S3({
-      params: {
-        Bucket: "prod-budi-app-assets",
-      },
-    })
-
-    const uploads = files.map(file => {
-      const fileExtension = [...file.name.split(".")].pop()
-      const processedFileName = `${uuid.v4()}.${fileExtension}`
-
-      return prepareUpload({
-        file,
-        s3Key: `assets/${ctx.user.appId}/attachments/${processedFileName}`,
-        s3,
-      })
-    })
-
-    ctx.body = await Promise.all(uploads)
-    return
-  }
-
-  ctx.body = await processLocalFileUploads({
-    files,
-    outputPath: attachmentsPath,
-    appId: ctx.user.appId,
-  })
-}
-
-async function processLocalFileUploads({ files, outputPath, appId }) {
-  // create attachments dir if it doesnt exist
-  !fs.existsSync(outputPath) && fs.mkdirSync(outputPath, { recursive: true })
-
-  const filesToProcess = files.map(file => {
     const fileExtension = [...file.name.split(".")].pop()
     // filenames converted to UUIDs so they are unique
     const processedFileName = `${uuid.v4()}.${fileExtension}`
 
-    return {
+    // need to handle image processing
-      name: file.name,
+    await fileProcessor.process({
-      path: file.path,
+      ...file,
-      size: file.size,
-      type: file.type,
-      processedFileName,
       extension: fileExtension,
-      outputPath: join(outputPath, processedFileName),
-      url: join("/attachments", processedFileName),
-    }
     })
 
-  const fileProcessOperations = filesToProcess.map(fileProcessor.process)
+    return prepareUpload({
-
+      file,
-  const processedFiles = await Promise.all(fileProcessOperations)
+      s3Key: `assets/${ctx.user.appId}/attachments/${processedFileName}`,
-
+      bucket: "prod-budi-app-assets",
-  let pendingFileUploads
-  // local document used to track which files need to be uploaded
-  // db.get throws an error if the document doesn't exist
-  // need to use a promise to default
-  const db = new CouchDB(appId)
-  await db
-    .get("_local/fileuploads")
-    .then(data => {
-      pendingFileUploads = data
     })
-    .catch(() => {
-      pendingFileUploads = { _id: "_local/fileuploads", uploads: [] }
   })
 
-  pendingFileUploads.uploads = [
+  ctx.body = await Promise.all(uploads)
-    ...processedFiles,
-    ...pendingFileUploads.uploads,
-  ]
-  await db.put(pendingFileUploads)
-
-  return processedFiles
 }
 
 exports.serveApp = async function(ctx) {
@@ -163,12 +103,12 @@ exports.serveApp = async function(ctx) {
 
   const { head, html, css } = App.render({
     title: appInfo.name,
-    production: env.CLOUD,
+    production: env.isProd(),
     appId,
     objectStoreUrl: objectStoreUrl(),
   })
 
-  const appHbs = fs.readFileSync(`${__dirname}/templates/app.hbs`, "utf8")
+  const appHbs = loadHandlebarsFile(`${__dirname}/templates/app.hbs`)
   ctx.body = await processString(appHbs, {
     head,
     body: html,
@@ -178,50 +118,30 @@ exports.serveApp = async function(ctx) {
 }
 
 exports.serveAttachment = async function(ctx) {
-  const appId = ctx.user.appId
+  await returnObjectStoreFile(
-  const attachmentsPath = resolve(budibaseAppsDir(), appId, "attachments")
+    ctx,
-
+    join(ctx.user.appId, "attachments", ctx.file)
-  // Serve from object store
+  )
-  if (env.CLOUD) {
-    const S3_URL = join(objectStoreUrl(), appId, "attachments", ctx.file)
-    const response = await fetch(S3_URL)
-    const body = await response.text()
-    ctx.set("Content-Type", response.headers.get("Content-Type"))
-    ctx.body = body
-    return
-  }
-
-  await send(ctx, ctx.file, { root: attachmentsPath })
 }
 
 exports.serveAppAsset = async function(ctx) {
-  // default to homedir
+  if (env.isDev() || env.isTest()) {
-  const appPath = resolve(budibaseAppsDir(), ctx.user.appId, "public")
+    return send(ctx, ctx.file, { root: budibaseTempDir() })
-
+  }
-  await send(ctx, ctx.file, { root: ctx.devPath || appPath })
+  await returnObjectStoreFile(ctx, join(ctx.user.appId, "public", ctx.file))
 }
 
 exports.serveComponentLibrary = async function(ctx) {
   const appId = ctx.query.appId || ctx.appId
-  // default to homedir
-  let componentLibraryPath = resolve(
-    budibaseAppsDir(),
-    appId,
-    "node_modules",
-    decodeURI(ctx.query.library),
-    "package",
-    "dist"
-  )
 
-  if (ctx.isDev) {
+  if (env.isDev() || env.isTest()) {
-    componentLibraryPath = join(
+    const componentLibraryPath = join(
       budibaseTempDir(),
       decodeURI(ctx.query.library),
       "dist"
     )
+    return send(ctx, "/awsDeploy.js", { root: componentLibraryPath })
   }
-
-  if (env.CLOUD) {
   let componentLib = "componentlibrary"
   if (ctx.user.version) {
     componentLib += `-${ctx.user.version}`
@@ -241,8 +161,4 @@ exports.serveComponentLibrary = async function(ctx) {
   const body = await response.text()
   ctx.type = "application/javascript"
   ctx.body = body
-    return
-  }
-
-  await send(ctx, "/awsDeploy.js", { root: componentLibraryPath })
 }

packages/server/src/api/controllers/static/selfhost/index.hbs

@@ -1,173 +0,0 @@
-<!DOCTYPE html>
-<html lang="en">
-<head>
-  <meta charset="UTF-8">
-  <title>Budibase self hosting️</title>
-  <style>
-    body {
-      font-family: Inter, -apple-system, BlinkMacSystemFont, Roboto, Helvetica, Arial, sans-serif, "Apple Color Emoji", "Segoe UI Emoji", "Segoe UI Symbol";
-      height: 100%;
-      width: 100%;
-      margin: 0;
-      padding: 0;
-      background: #fafafa;
-    }
-
-    .main {
-      padding: 0 20px;
-      margin: 30px auto;
-      width: 60%;
-    }
-
-    h2 {
-      font-size: clamp(24px, 1.5vw, 30px);
-      text-align: center;
-      line-height: 1.3;
-      font-weight: bold;
-    }
-
-    .card-grid {
-      display: grid;
-      grid-template-columns: 1fr 1fr;
-      gap: 3rem;
-    }
-
-    .card {
-      display: grid;
-      background-color: #222222;
-      grid-template-columns: 1fr;
-      align-items: center;
-      padding: 2.5rem 1.75rem;
-      border-radius: 12px;
-      color: white;
-    }
-
-    .card h3 {
-      margin: 0;
-      font-size: 24px;
-      font-family: sans-serif;
-      font-weight: 600;
-    }
-
-    .card h3 b {
-      text-wrap: normal;
-      font-size: 36px;
-      padding-right: 14px;
-    }
-
-    .card p {
-      color: #ffffff;
-      opacity: 0.8;
-      font-size: 18px;
-      text-align: left;
-      line-height: 1.3;
-      margin-top: 1rem;
-    }
-
-    .logo {
-      width: 60px;
-      height: 60px;
-      margin: auto;
-    }
-
-    .top-text {
-      text-align: center;
-      color: #707070;
-      margin: 0 0 1.5rem 0;
-    }
-
-    .button {
-      cursor: pointer;
-      display: block;
-      background: #4285f4;
-      color: white;
-      padding: 12px 16px;
-      font-size: 16px;
-      font-weight: 600;
-      border-radius: 6px;
-      max-width: 120px;
-      text-align: center;
-      transition: 200ms background ease;
-      text-decoration: none;
-    }
-
-    .info {
-      background: #f5f5f5;
-      padding: 1rem 1rem 1rem 1rem;
-      border: #ccc 1px solid;
-      border-radius: 6px;
-      margin-top: 40px;
-      display: flex;
-      align-items: center;
-    }
-
-    .info p {
-      margin-left: 20px;
-      color: #222222;
-      font-family: sans-serif;
-    }
-
-    .info p {
-      margin-right: 20px;
-    }
-
-    .info svg {
-      margin-left: 20px;
-    }
-
-    .info a {
-      color: #4285f4;
-    }
-  </style>
-</head>
-<body>
-  <div class="main">
-    <div class="logo">
-      {{logo}}
-    </div>
-    <h2>Get started with Budibase Self Hosting</h2>
-    <p class="top-text">Use the address <b id="url"></b> in your Builder</p>
-    <div class="card-grid">
-      <div class="card">
-        <h3><b>📚</b>Documentation</h3>
-        <p>
-          Find out more about your self hosted platform.
-        </p>
-        <a class="button"
-           href="https://docs.budibase.com/self-hosting/introduction-to-self-hosting">
-          Documentation
-        </a>
-      </div>
-      <div class="card">
-        <h3><b>💻</b>Next steps</h3>
-        <p>
-          Find out how to make use of your self hosted Budibase platform.
-        </p>
-        <a class="button"
-           href="https://docs.budibase.com/self-hosting/builder-settings">
-          Next steps
-        </a>
-      </div>
-    </div>
-    <div class="info">
-      <svg preserveAspectRatio="xMidYMid meet" height="28px" width="28px" fill="currentColor" viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"
-           xmlns:xlink="http://www.w3.org/1999/xlink" stroke="none" class="icon-7f6730be--text-3f89f380">
-        <g>
-          <path d="M12.2 8.98c.06-.01.12-.03.18-.06.06-.02.12-.05.18-.09l.15-.12c.18-.19.29-.45.29-.71 0-.06-.01-.13-.02-.19a.603.603 0 0 0-.06-.19.757.757 0 0 0-.09-.18c-.03-.05-.08-.1-.12-.15-.28-.27-.72-.37-1.09-.21-.13.05-.23.12-.33.21-.04.05-.09.1-.12.15-.04.06-.07.12-.09.18-.03.06-.05.12-.06.19-.01.06-.02.13-.02.19 0 .26.11.52.29.71.1.09.2.16.33.21.12.05.25.08.38.08.06 0 .13-.01.2-.02M13 16v-4a1 1 0 1 0-2 0v4a1 1 0 1 0 2 0M12 3c-4.962 0-9 4.038-9 9 0 4.963 4.038 9 9 9 4.963 0 9-4.037 9-9 0-4.962-4.037-9-9-9m0 20C5.935 23 1 18.065 1 12S5.935 1 12 1c6.066 0 11 4.935 11 11s-4.934 11-11 11" fill-rule="evenodd">
-          </path>
-        </g>
-      </svg>
-      <p>A <b>Hosting Key</b> will also be required, this can be found in your hosting properties, info found <a href="https://docs.budibase.com/self-hosting/hosting-settings">here</a>.</p>
-    </div>
-  </div>
-  <script>
-    window.addEventListener("load", () => {
-      let url = document.URL.split("//")[1]
-      if (url.substring(url.length - 1) === "/") {
-        url = url.substring(0, url.length - 1)
-      }
-      document.getElementById("url").innerHTML = url
-    })
-  </script>
-</body>
-</html>

packages/server/src/api/controllers/static/selfhost/logo.svg

@@ -1,17 +0,0 @@
-<svg version="1.1" id="Layer_1" xmlns="http://www.w3.org/2000/svg" x="0" y="0" viewBox="0 0 48 48" xml:space="preserve">
-  <style>
-    .st0{fill:#393c44}.st1{fill:#fff}
-  </style>
-    <path class="st0" d="M-152.17-24.17H-56V72h-96.17z"/>
-    <path class="st1" d="M-83.19 48h-41.79c-1.76 0-3.19-1.43-3.19-3.19V3.02c0-1.76 1.43-3.19 3.19-3.19h41.79c1.76 0 3.19 1.43 3.19 3.19v41.79c0 1.76-1.43 3.19-3.19 3.19z"/>
-    <path class="st0" d="M-99.62 12.57v9.94c1.15-1.21 2.59-1.81 4.32-1.81 1.03 0 1.97.19 2.82.58.86.39 1.59.91 2.19 1.57.6.66 1.08 1.43 1.42 2.32.34.89.51 1.84.51 2.85 0 1.03-.18 1.99-.53 2.89-.35.9-.84 1.68-1.47 2.35-.63.67-1.37 1.19-2.23 1.58-.86.39-1.78.58-2.77.58-1.8 0-3.22-.66-4.27-1.97V35h-4.89V12.57h4.9zm6.16 15.54c0-.43-.08-.84-.24-1.23-.16-.39-.39-.72-.68-1.01-.29-.29-.62-.52-1-.69-.38-.17-.79-.26-1.24-.26-.43 0-.84.08-1.22.24-.38.16-.71.39-.99.68-.28.29-.5.63-.68 1.01-.17.39-.26.8-.26 1.23 0 .43.08.84.24 1.22.16.38.39.71.68.99.29.28.63.5 1.01.68.39.17.8.26 1.23.26.43 0 .84-.08 1.22-.24.38-.16.71-.39.99-.68.28-.29.5-.62.68-1 .17-.39.26-.79.26-1.2z"/>
-    <path class="st0" d="M-114.76 12.57v9.94c1.15-1.21 2.59-1.81 4.32-1.81 1.03 0 1.97.19 2.82.58.86.39 1.59.91 2.19 1.57.6.66 1.08 1.43 1.42 2.32.34.89.51 1.84.51 2.85 0 1.03-.18 1.99-.53 2.89-.35.9-.84 1.68-1.47 2.35-.63.67-1.37 1.19-2.23 1.58-.86.39-1.78.58-2.77.58-1.8 0-3.22-.66-4.27-1.97V35h-4.89V12.57h4.9zm6.16 15.54c0-.43-.08-.84-.24-1.23-.16-.39-.39-.72-.68-1.01-.29-.29-.62-.52-1-.69-.38-.17-.79-.26-1.24-.26-.43 0-.84.08-1.22.24-.38.16-.71.39-.99.68-.28.29-.5.63-.68 1.01-.17.39-.26.8-.26 1.23 0 .43.08.84.24 1.22.16.38.39.71.68.99.29.28.63.5 1.01.68.39.17.8.26 1.23.26.43 0 .84-.08 1.22-.24.38-.16.71-.39.99-.68.28-.29.5-.62.68-1 .18-.39.26-.79.26-1.2z"/>
-    <path d="M44.81 159H3.02c-1.76 0-3.19-1.43-3.19-3.19v-41.79c0-1.76 1.43-3.19 3.19-3.19h41.79c1.76 0 3.19 1.43 3.19 3.19v41.79c0 1.76-1.43 3.19-3.19 3.19z" fill="#4285f4"/>
-    <path class="st1" d="M28.38 123.57v9.94c1.15-1.21 2.59-1.81 4.32-1.81 1.03 0 1.97.19 2.82.58.86.39 1.59.91 2.19 1.57.6.66 1.08 1.43 1.42 2.32.34.89.51 1.84.51 2.85 0 1.03-.18 1.99-.53 2.89-.35.9-.84 1.68-1.47 2.35-.63.67-1.37 1.19-2.23 1.58-.86.39-1.78.58-2.77.58-1.8 0-3.22-.66-4.27-1.97V146h-4.89v-22.43h4.9zm6.16 15.54c0-.43-.08-.84-.24-1.23-.16-.39-.39-.72-.68-1.01-.29-.29-.62-.52-1-.69-.38-.17-.79-.26-1.24-.26-.43 0-.84.08-1.22.24-.38.16-.71.39-.99.68-.28.29-.5.63-.68 1.01-.17.39-.26.8-.26 1.23 0 .43.08.84.24 1.22.16.38.39.71.68.99.29.28.63.5 1.01.68.39.17.8.26 1.23.26.43 0 .84-.08 1.22-.24.38-.16.71-.39.99-.68.28-.29.5-.62.68-1 .17-.39.26-.79.26-1.2z"/>
-    <path class="st1" d="M13.24 123.57v9.94c1.15-1.21 2.59-1.81 4.32-1.81 1.03 0 1.97.19 2.82.58.86.39 1.59.91 2.19 1.57.6.66 1.08 1.43 1.42 2.32.34.89.51 1.84.51 2.85 0 1.03-.18 1.99-.53 2.89-.35.9-.84 1.68-1.47 2.35-.63.67-1.37 1.19-2.23 1.58-.86.39-1.78.58-2.77.58-1.8 0-3.22-.66-4.27-1.97V146H8.35v-22.43h4.89zm6.16 15.54c0-.43-.08-.84-.24-1.23-.16-.39-.39-.72-.68-1.01-.29-.29-.62-.52-1-.69-.38-.17-.79-.26-1.24-.26-.43 0-.84.08-1.22.24-.38.16-.71.39-.99.68-.28.29-.5.63-.68 1.01-.17.39-.26.8-.26 1.23 0 .43.08.84.24 1.22.16.38.39.71.68.99.29.28.63.5 1.01.68.39.17.8.26 1.23.26.43 0 .84-.08 1.22-.24.38-.16.71-.39.99-.68.28-.29.5-.62.68-1 .18-.39.26-.79.26-1.2z"/>
-    <g>
-    <path class="st0" d="M44 48H4c-2.21 0-4-1.79-4-4V4c0-2.21 1.79-4 4-4h40c2.21 0 4 1.79 4 4v40c0 2.21-1.79 4-4 4z"/>
-        <path class="st1" d="M28.48 12v10.44c1.18-1.27 2.65-1.9 4.42-1.9 1.05 0 2.01.2 2.89.61.87.41 1.62.96 2.24 1.65.62.69 1.1 1.5 1.45 2.44.35.94.52 1.93.52 2.99 0 1.08-.18 2.09-.54 3.04-.36.95-.86 1.77-1.51 2.47-.64.7-1.4 1.25-2.28 1.66-.87.4-1.81.6-2.83.6-1.84 0-3.3-.69-4.37-2.07v1.62h-5V12h5.01zm6.3 16.31c0-.45-.08-.88-.25-1.29-.17-.41-.4-.76-.69-1.06-.3-.3-.64-.54-1.02-.72-.39-.18-.81-.27-1.27-.27-.44 0-.86.09-1.24.26-.39.17-.72.41-1.01.71-.29.3-.52.66-.69 1.06-.18.41-.26.84-.26 1.29s.08.88.25 1.28c.17.4.4.74.69 1.04.29.29.64.53 1.04.71.4.18.82.27 1.26.27.44 0 .86-.09 1.24-.26.39-.17.72-.41 1.01-.71.29-.3.52-.65.69-1.05.16-.41.25-.82.25-1.26z"/>
-        <path class="st1" d="M13 12v10.44c1.18-1.27 2.65-1.9 4.42-1.9 1.05 0 2.01.2 2.89.61.87.41 1.62.96 2.24 1.65.62.69 1.1 1.5 1.45 2.44.35.94.52 1.93.52 2.99 0 1.08-.18 2.09-.54 3.04-.36.95-.86 1.77-1.51 2.47-.64.7-1.4 1.25-2.28 1.66-.87.4-1.81.6-2.82.6-1.84 0-3.3-.69-4.37-2.07v1.62H8V12h5zm6.3 16.31c0-.45-.08-.88-.25-1.29-.17-.41-.4-.76-.69-1.06-.3-.3-.64-.54-1.02-.72-.39-.18-.81-.27-1.27-.27-.44 0-.86.09-1.24.26-.39.17-.72.41-1.01.71-.29.3-.52.66-.69 1.06-.18.41-.26.84-.26 1.29s.08.88.25 1.28c.17.4.4.74.69 1.04.29.29.64.53 1.04.71.4.18.82.27 1.26.27.44 0 .86-.09 1.24-.26.39-.17.72-.41 1.01-.71.29-.3.52-.65.69-1.05.16-.41.25-.82.25-1.26z"/>
-  </g>
-</svg>

packages/server/src/api/controllers/templates.js

@@ -1,10 +1,5 @@
 const fetch = require("node-fetch")
-const {
+const { downloadTemplate } = require("../../utilities/fileSystem")
-  downloadTemplate,
-  exportTemplateFromApp,
-  getLocalTemplates,
-} = require("../../utilities/templates")
-const env = require("../../environment")
 
 // development flag, can be used to test against templates exported locally
 const DEFAULT_TEMPLATES_BUCKET =
@@ -12,43 +7,21 @@ const DEFAULT_TEMPLATES_BUCKET =
 
 exports.fetch = async function(ctx) {
   const { type = "app" } = ctx.query
-
-  if (env.LOCAL_TEMPLATES) {
-    ctx.body = Object.values(getLocalTemplates()[type])
-  } else {
   const response = await fetch(
     `https://${DEFAULT_TEMPLATES_BUCKET}/manifest.json`
   )
   const json = await response.json()
   ctx.body = Object.values(json.templates[type])
 }
-}
 
 // can't currently test this, have to ignore from coverage
 /* istanbul ignore next */
 exports.downloadTemplate = async function(ctx) {
   const { type, name } = ctx.params
 
-  if (!env.LOCAL_TEMPLATES) {
   await downloadTemplate(type, name)
-  }
 
   ctx.body = {
     message: `template ${type}:${name} downloaded successfully.`,
   }
 }
-
-exports.exportTemplateFromApp = async function(ctx) {
-  const { appId } = ctx.user
-  const { templateName } = ctx.request.body
-
-  await exportTemplateFromApp({
-    appId,
-    templateName,
-  })
-
-  ctx.status = 200
-  ctx.body = {
-    message: `Created template: ${templateName}`,
-  }
-}

packages/server/src/api/controllers/view/index.js

@@ -1,8 +1,6 @@
 const CouchDB = require("../../../db")
 const viewTemplate = require("./viewBuilder")
-const fs = require("fs")
+const { apiFileReturn } = require("../../../utilities/fileSystem")
-const { join } = require("../../../utilities/centralPath")
-const os = require("os")
 const exporters = require("./exporters")
 const { fetchView } = require("../row")
 const { ViewNames } = require("../../../db/utils")
@@ -120,12 +118,10 @@ const controller = {
     // Export part
     let headers = Object.keys(schema)
     const exporter = exporters[format]
-    const exportedFile = exporter(headers, ctx.body)
     const filename = `${viewName}.${format}`
-    fs.writeFileSync(join(os.tmpdir(), filename), exportedFile)
+    // send down the file
-
     ctx.attachment(filename)
-    ctx.body = fs.createReadStream(join(os.tmpdir(), filename))
+    ctx.body = apiFileReturn(exporter(headers, ctx.body))
   },
 }
 

packages/server/src/api/index.js

@@ -2,8 +2,6 @@ const Router = require("@koa/router")
 const authenticated = require("../middleware/authenticated")
 const compress = require("koa-compress")
 const zlib = require("zlib")
-const { budibaseAppsDir } = require("../utilities/budibaseDir")
-const { isDev } = require("../utilities")
 const { mainRoutes, authRoutes, staticRoutes } = require("./routes")
 const pkg = require("../../package.json")
 
@@ -25,11 +23,9 @@ router
   )
   .use(async (ctx, next) => {
     ctx.config = {
-      latestPackagesFolder: budibaseAppsDir(),
       jwtSecret: env.JWT_SECRET,
       useAppRootPath: true,
     }
-    ctx.isDev = isDev()
     await next()
   })
   .use("/health", ctx => (ctx.status = 200))
@@ -68,8 +64,6 @@ for (let route of mainRoutes) {
 router.use(staticRoutes.routes())
 router.use(staticRoutes.allowedMethods())
 
-if (!env.SELF_HOSTED && !env.CLOUD) {
 router.redirect("/", "/_builder")
-}
 
 module.exports = router

packages/server/src/api/routes/search.js

@@ -0,0 +1,8 @@
+const Router = require("@koa/router")
+const controller = require("../controllers/search")
+
+const router = Router()
+
+router.get("/api/search/rows", controller.rowSearch)
+
+module.exports = router

packages/server/src/api/routes/static.js

@@ -1,34 +1,32 @@
 const Router = require("@koa/router")
 const controller = require("../controllers/static")
 const { budibaseTempDir } = require("../../utilities/budibaseDir")
-const env = require("../../environment")
 const authorized = require("../../middleware/authorized")
 const { BUILDER } = require("../../utilities/security/permissions")
 const usage = require("../../middleware/usageQuota")
+const env = require("../../environment")
 
 const router = Router()
 
 /* istanbul ignore next */
 router.param("file", async (file, ctx, next) => {
   ctx.file = file && file.includes(".") ? file : "index.html"
-
+  if (!ctx.file.startsWith("budibase-client")) {
+    return next()
+  }
+  // test serves from require
+  if (env.isTest()) {
+    ctx.devPath = require.resolve("@budibase/client").split(ctx.file)[0]
+  } else if (env.isDev()) {
     // Serving the client library from your local dir in dev
-  if (ctx.isDev && ctx.file.startsWith("budibase-client")) {
     ctx.devPath = budibaseTempDir()
   }
-
+  return next()
-  await next()
 })
 
-if (env.NODE_ENV !== "production") {
-  router.get("/_builder/:file*", controller.serveBuilder)
-}
-
-if (env.SELF_HOSTED) {
-  router.get("/", controller.serveSelfHostPage)
-}
-
 router
+  // TODO: for now this _builder endpoint is not authorized/secured, will need to be
+  .get("/_builder/:file*", controller.serveBuilder)
   .post("/api/attachments/process", authorized(BUILDER), controller.uploadFile)
   .post("/api/attachments/upload", usage, controller.uploadFile)
   .get("/componentlibrary", controller.serveComponentLibrary)

packages/server/src/api/routes/templates.js

@@ -12,6 +12,5 @@ router
     authorized(BUILDER),
     controller.downloadTemplate
   )
-  .post("/api/templates", authorized(BUILDER), controller.exportTemplateFromApp)
 
 module.exports = router

packages/server/src/api/routes/tests/apikeys.spec.js

@@ -1,8 +1,5 @@
 const setup = require("./utilities")
 const { checkBuilderEndpoint } = require("./utilities/TestFunctions")
-const { budibaseAppsDir } = require("../../../utilities/budibaseDir")
-const fs = require("fs")
-const path = require("path")
 
 describe("/api/keys", () => {
   let request = setup.getRequest()
@@ -16,6 +13,7 @@ describe("/api/keys", () => {
 
   describe("fetch", () => {
     it("should allow fetching", async () => {
+      await setup.switchToSelfHosted(async () => {
         const res = await request
           .get(`/api/keys`)
           .set(config.defaultHeaders())
@@ -23,6 +21,7 @@ describe("/api/keys", () => {
           .expect(200)
         expect(res.body).toBeDefined()
       })
+    })
 
     it("should check authorization for builder", async () => {
       await checkBuilderEndpoint({
@@ -35,7 +34,7 @@ describe("/api/keys", () => {
 
   describe("update", () => {
     it("should allow updating a value", async () => {
-      fs.writeFileSync(path.join(budibaseAppsDir(), ".env"), "TEST_API_KEY=thing")
+      await setup.switchToSelfHosted(async () => {
         const res = await request
           .put(`/api/keys/TEST`)
           .send({
@@ -44,8 +43,9 @@ describe("/api/keys", () => {
           .set(config.defaultHeaders())
           .expect("Content-Type", /json/)
           .expect(200)
-      expect(res.body["TEST"]).toEqual("test")
+        expect(res.body._id).toBeDefined()
-      expect(process.env.TEST_API_KEY).toEqual("test")
+        expect(res.body._rev).toBeDefined()
+      })
     })
 
     it("should check authorization for builder", async () => {

packages/server/src/api/routes/tests/auth.spec.js

@@ -1,4 +1,3 @@
-const { checkBuilderEndpoint } = require("./utilities/TestFunctions")
 const setup = require("./utilities")
 
 describe("/authenticate", () => {

packages/server/src/api/routes/tests/backup.spec.js

@@ -1,6 +1,8 @@
 const { checkBuilderEndpoint } = require("./utilities/TestFunctions")
 const setup = require("./utilities")
 
+jest.mock("../../../utilities/fileSystem/utilities")
+
 describe("/backups", () => {
   let request = setup.getRequest()
   let config = setup.getConfig()
@@ -14,7 +16,7 @@ describe("/backups", () => {
   describe("exportAppDump", () => {
     it("should be able to export app", async () => {
       const res = await request
-        .get(`/api/backups/export?appId=${config.getAppId()}`)
+        .get(`/api/backups/export?appId=${config.getAppId()}&appname=test`)
         .set(config.defaultHeaders())
         .expect(200)
       expect(res.text).toBeDefined()

packages/server/src/api/routes/tests/cloud.spec.js

@@ -1,16 +0,0 @@
-const setup = require("./utilities")
-
-describe("test things in the Cloud/Self hosted", () => {
-  describe("test self hosted static page", () => {
-    it("should be able to load the static page", async () => {
-      await setup.switchToCloudForFunction(async () => {
-        let request = setup.getRequest()
-        let config = setup.getConfig()
-        await config.init()
-        const res = await request.get(`/`).expect(200)
-        expect(res.text.includes("<title>Budibase self hosting️</title>")).toEqual(true)
-        setup.afterAll()
-      })
-    })
-  })
-})

packages/server/src/api/routes/tests/component.spec.js

@@ -1,8 +1,5 @@
 const { checkBuilderEndpoint } = require("./utilities/TestFunctions")
 const setup = require("./utilities")
-const fs = require("fs")
-const { resolve, join } = require("path")
-const { budibaseAppsDir } = require("../../../utilities/budibaseDir")
 
 describe("/component", () => {
   let request = setup.getRequest()
@@ -14,23 +11,8 @@ describe("/component", () => {
     await config.init()
   })
 
-  function mock() {
-    const manifestFile = "manifest.json"
-    const appId = config.getAppId()
-    const libraries = [join("@budibase", "standard-components")]
-    for (let library of libraries) {
-      let appDirectory = resolve(budibaseAppsDir(), appId, "node_modules", library, "package")
-      fs.mkdirSync(appDirectory, { recursive: true })
-
-      const file = require.resolve(library).split(join("dist", "index.js"))[0] + manifestFile
-      fs.copyFileSync(file, join(appDirectory, manifestFile))
-    }
-  }
-
   describe("fetch definitions", () => {
     it("should be able to fetch definitions", async () => {
-      // have to "mock" the files required
-      mock()
       const res = await request
         .get(`/${config.getAppId()}/components/definitions`)
         .set(config.defaultHeaders())

packages/server/src/api/routes/tests/hosting.spec.js

@@ -107,17 +107,16 @@ describe("/hosting", () => {
   })
 
   describe("getDeployedApps", () => {
-    it("should get apps when in builder", async () => {
+    it("should fail when not self hosted", async () => {
-      const res = await request
+      await request
         .get(`/api/hosting/apps`)
         .set(config.defaultHeaders())
         .expect("Content-Type", /json/)
-        .expect(200)
+        .expect(400)
-      expect(res.body.app1).toEqual({url: "/app1"})
     })
 
     it("should get apps when in cloud", async () => {
-      await setup.switchToCloudForFunction(async () => {
+      await setup.switchToSelfHosted(async () => {
         const res = await request
           .get(`/api/hosting/apps`)
           .set(config.defaultHeaders())

packages/server/src/api/routes/tests/query.spec.js

@@ -89,7 +89,7 @@ describe("/queries", () => {
     })
 
     it("should find a query in cloud", async () => {
-      await setup.switchToCloudForFunction(async () => {
+      await setup.switchToSelfHosted(async () => {
         const query = await config.createQuery()
         const res = await request
           .get(`/api/queries/${query._id}`)

packages/server/src/api/routes/tests/row.spec.js

@@ -2,6 +2,9 @@ const { outputProcessing } = require("../../../utilities/rowProcessor")
 const setup = require("./utilities")
 const { basicRow } = setup.structures
 
+// mock the fetch for the search system
+jest.mock("node-fetch")
+
 describe("/rows", () => {
   let request = setup.getRequest()
   let config = setup.getConfig()
@@ -303,25 +306,19 @@ describe("/rows", () => {
 
   describe("search", () => {
     it("should run a search on the table", async () => {
-      const row = await config.createRow()
-      // add another row that shouldn't be found
-      await config.createRow({
-        ...basicRow(),
-        name: "Other Contact",
-      })
       const res = await request
         .post(`/api/${table._id}/rows/search`)
         .send({
           query: {
             name: "Test",
           },
-          pagination: { pageSize: 25, page: 0 }
+          pagination: { pageSize: 25 }
         })
         .set(config.defaultHeaders())
         .expect('Content-Type', /json/)
         .expect(200)
-      expect(res.body.length).toEqual(1)
+      expect(res.body.rows.length).toEqual(1)
-      expect(res.body[0]._id).toEqual(row._id)
+      expect(res.body.bookmark).toBeDefined()
     })
   })
 
@@ -410,7 +407,7 @@ describe("/rows", () => {
         tableId: table._id,
       })
       // the environment needs configured for this
-      await setup.switchToCloudForFunction(async () => {
+      await setup.switchToSelfHosted(async () => {
         const enriched = await outputProcessing(config.getAppId(), table, [row])
         expect(enriched[0].attachment[0].url).toBe(`/app-assets/assets/${config.getAppId()}/test/thing`)
       })

packages/server/src/api/routes/tests/templates.spec.js

@@ -1,7 +1,4 @@
 const setup = require("./utilities")
-const { budibaseAppsDir } = require("../../../utilities/budibaseDir")
-const fs = require("fs")
-const { join } = require("path")
 
 describe("/templates", () => {
   let request = setup.getRequest()
@@ -24,26 +21,4 @@ describe("/templates", () => {
       expect(Array.isArray(res.body)).toEqual(true)
     })
   })
-
-  describe("export", () => {
-    it("should be able to export the basic app", async () => {
-      const res = await request
-        .post(`/api/templates`)
-        .send({
-          templateName: "test",
-        })
-        .set(config.defaultHeaders())
-        .expect("Content-Type", /json/)
-        .expect(200)
-      expect(res.body.message).toEqual("Created template: test")
-      const dir = join(
-        budibaseAppsDir(),
-        "templates",
-        "app",
-        "test",
-        "db"
-      )
-      expect(fs.existsSync(dir)).toEqual(true)
-    })
-  })
 })

packages/server/src/api/routes/tests/utilities/index.js

@@ -35,18 +35,18 @@ exports.getConfig = () => {
   return config
 }
 
-exports.switchToCloudForFunction = async func => {
+exports.switchToSelfHosted = async func => {
   // self hosted stops any attempts to Dynamo
-  env.CLOUD = true
+  env._set("NODE_ENV", "production")
-  env.SELF_HOSTED = true
+  env._set("SELF_HOSTED", true)
   let error
   try {
     await func()
   } catch (err) {
     error = err
   }
-  env.CLOUD = false
+  env._set("NODE_ENV", "jest")
-  env.SELF_HOSTED = false
+  env._set("SELF_HOSTED", false)
   // don't throw error until after reset
   if (error) {
     throw error

packages/server/src/app.js

@@ -9,6 +9,7 @@ const env = require("./environment")
 const eventEmitter = require("./events")
 const automations = require("./automations/index")
 const Sentry = require("@sentry/node")
+const fileSystem = require("./utilities/fileSystem")
 
 const app = new Koa()
 
@@ -65,6 +66,7 @@ module.exports = server.listen(env.PORT || 0, async () => {
   console.log(`Budibase running on ${JSON.stringify(server.address())}`)
   env._set("PORT", server.address().port)
   eventEmitter.emitPort(env.PORT)
+  fileSystem.init()
   await automations.init()
 })
 

packages/server/src/automations/actions.js

@@ -5,17 +5,12 @@ const deleteRow = require("./steps/deleteRow")
 const createUser = require("./steps/createUser")
 const outgoingWebhook = require("./steps/outgoingWebhook")
 const env = require("../environment")
-const download = require("download")
-const fetch = require("node-fetch")
-const { join } = require("../utilities/centralPath")
-const os = require("os")
-const fs = require("fs")
 const Sentry = require("@sentry/node")
+const {
+  automationInit,
+  getExternalAutomationStep,
+} = require("../utilities/fileSystem")
 
-const DEFAULT_BUCKET =
-  "https://prod-budi-automations.s3-eu-west-1.amazonaws.com"
-const DEFAULT_DIRECTORY = ".budibase-automations"
-const AUTOMATION_MANIFEST = "manifest.json"
 const BUILTIN_ACTIONS = {
   SEND_EMAIL: sendEmail.run,
   CREATE_ROW: createRow.run,
@@ -33,8 +28,6 @@ const BUILTIN_DEFINITIONS = {
   OUTGOING_WEBHOOK: outgoingWebhook.definition,
 }
 
-let AUTOMATION_BUCKET = env.AUTOMATION_BUCKET
-let AUTOMATION_DIRECTORY = env.AUTOMATION_DIRECTORY
 let MANIFEST = null
 
 /* istanbul ignore next */
@@ -42,22 +35,13 @@ function buildBundleName(pkgName, version) {
   return `${pkgName}@${version}.min.js`
 }
 
-/* istanbul ignore next */
-async function downloadPackage(name, version, bundleName) {
-  await download(
-    `${AUTOMATION_BUCKET}/${name}/${version}/${bundleName}`,
-    AUTOMATION_DIRECTORY
-  )
-  return require(join(AUTOMATION_DIRECTORY, bundleName))
-}
-
 /* istanbul ignore next */
 module.exports.getAction = async function(actionName) {
   if (BUILTIN_ACTIONS[actionName] != null) {
     return BUILTIN_ACTIONS[actionName]
   }
   // worker pools means that a worker may not have manifest
-  if (env.CLOUD && MANIFEST == null) {
+  if (env.isProd() && MANIFEST == null) {
     MANIFEST = await module.exports.init()
   }
   // env setup to get async packages
@@ -66,28 +50,12 @@ module.exports.getAction = async function(actionName) {
   }
   const pkg = MANIFEST.packages[actionName]
   const bundleName = buildBundleName(pkg.stepId, pkg.version)
-  try {
+  return getExternalAutomationStep(pkg.stepId, pkg.version, bundleName)
-    return require(join(AUTOMATION_DIRECTORY, bundleName))
-  } catch (err) {
-    return downloadPackage(pkg.stepId, pkg.version, bundleName)
-  }
 }
 
 module.exports.init = async function() {
-  // set defaults
-  if (!AUTOMATION_DIRECTORY) {
-    AUTOMATION_DIRECTORY = join(os.homedir(), DEFAULT_DIRECTORY)
-  }
-  if (!AUTOMATION_BUCKET) {
-    AUTOMATION_BUCKET = DEFAULT_BUCKET
-  }
-  if (!fs.existsSync(AUTOMATION_DIRECTORY)) {
-    fs.mkdirSync(AUTOMATION_DIRECTORY, { recursive: true })
-  }
-  // env setup to get async packages
   try {
-    let response = await fetch(`${AUTOMATION_BUCKET}/${AUTOMATION_MANIFEST}`)
+    MANIFEST = await automationInit()
-    MANIFEST = await response.json()
     module.exports.DEFINITIONS =
       MANIFEST && MANIFEST.packages
         ? Object.assign(MANIFEST.packages, BUILTIN_DEFINITIONS)

packages/server/src/automations/index.js

@@ -34,10 +34,10 @@ module.exports.init = async function() {
   await actions.init()
   triggers.automationQueue.process(async job => {
     try {
-      if (env.CLOUD && job.data.automation && !env.SELF_HOSTED) {
+      if (env.USE_QUOTAS) {
         job.data.automation.apiKey = await updateQuota(job.data.automation)
       }
-      if (env.BUDIBASE_ENVIRONMENT === "PRODUCTION") {
+      if (env.isProd()) {
         await runWorker(job)
       } else {
         await singleThread(job)

packages/server/src/automations/steps/createRow.js

@@ -85,7 +85,7 @@ module.exports.run = async function({ inputs, appId, apiKey, emitter }) {
       inputs.row.tableId,
       inputs.row
     )
-    if (env.CLOUD) {
+    if (env.isProd()) {
       await usage.update(apiKey, usage.Properties.ROW, 1)
     }
     await rowController.save(ctx)

packages/server/src/automations/steps/createUser.js

@@ -72,7 +72,7 @@ module.exports.run = async function({ inputs, appId, apiKey, emitter }) {
   }
 
   try {
-    if (env.CLOUD) {
+    if (env.isProd()) {
       await usage.update(apiKey, usage.Properties.USER, 1)
     }
     await userController.create(ctx)

packages/server/src/automations/steps/deleteRow.js

@@ -70,7 +70,7 @@ module.exports.run = async function({ inputs, appId, apiKey, emitter }) {
   }
 
   try {
-    if (env.CLOUD) {
+    if (env.isProd()) {
       await usage.update(apiKey, usage.Properties.ROW, -1)
     }
     await rowController.destroy(ctx)

packages/server/src/automations/tests/automation.spec.js

@@ -47,27 +47,23 @@ describe("Run through some parts of the automations system", () => {
     expect(thread).toHaveBeenCalled()
   })
 
-  it("should be able to init in cloud", async () => {
+  it("should be able to init in prod", async () => {
-    env.CLOUD = true
+    await setup.runInProd(async () => {
-    env.BUDIBASE_ENVIRONMENT = "PRODUCTION"
       await triggers.externalTrigger(basicAutomation(), { a: 1 })
       await wait(100)
       // haven't added a mock implementation so getAPIKey of usageQuota just returns undefined
       expect(usageQuota.update).toHaveBeenCalledWith("test", "automationRuns", 1)
       expect(workerJob).toBeDefined()
-    env.BUDIBASE_ENVIRONMENT = "JEST"
+    })
-    env.CLOUD = false
   })
 
   it("try error scenario", async () => {
-    env.CLOUD = true
+    await setup.runInProd(async () => {
-    env.BUDIBASE_ENVIRONMENT = "PRODUCTION"
       // the second call will throw an error
       await triggers.externalTrigger(basicAutomation(), { a: 1 })
       await wait(100)
       expect(console.error).toHaveBeenCalled()
-    env.BUDIBASE_ENVIRONMENT = "JEST"
+    })
-    env.CLOUD = false
   })
 
   it("should be able to check triggering row filling", async () => {

packages/server/src/automations/tests/createRow.spec.js

@@ -42,12 +42,12 @@ describe("test the create row action", () => {
   })
 
   it("check usage quota attempts", async () => {
-    env.CLOUD = true
+    await setup.runInProd(async () => {
       await setup.runStep(setup.actions.CREATE_ROW.stepId, {
         row
       })
       expect(usageQuota.update).toHaveBeenCalledWith(setup.apiKey, "rows", 1)
-    env.CLOUD = false
+    })
   })
 
   it("should check invalid inputs return an error", async () => {

packages/server/src/automations/tests/createUser.spec.js

@@ -35,9 +35,9 @@ describe("test the create user action", () => {
   })
 
   it("check usage quota attempts", async () => {
-    env.CLOUD = true
+    await setup.runInProd(async () => {
       await setup.runStep(setup.actions.CREATE_USER.stepId, user)
       expect(usageQuota.update).toHaveBeenCalledWith(setup.apiKey, "users", 1)
-    env.CLOUD = false
+    })
   })
 })

packages/server/src/automations/tests/deleteRow.spec.js

@@ -36,10 +36,10 @@ describe("test the delete row action", () => {
   })
 
   it("check usage quota attempts", async () => {
-    env.CLOUD = true
+    await setup.runInProd(async () => {
       await setup.runStep(setup.actions.DELETE_ROW.stepId, inputs)
       expect(usageQuota.update).toHaveBeenCalledWith(setup.apiKey, "rows", -1)
-    env.CLOUD = false
+    })
   })
 
   it("should check invalid inputs return an error", async () => {

packages/server/src/automations/tests/utilities/index.js

@@ -2,6 +2,7 @@ const TestConfig = require("../../../tests/utilities/TestConfiguration")
 const actions = require("../../actions")
 const logic = require("../../logic")
 const emitter = require("../../../events/index")
+const env = require("../../../environment")
 
 let config
 
@@ -16,6 +17,22 @@ exports.afterAll = () => {
   config.end()
 }
 
+exports.runInProd = async fn => {
+  env._set("NODE_ENV", "production")
+  env._set("USE_QUOTAS", 1)
+  let error
+  try {
+    await fn()
+  } catch (err) {
+    error = err
+  }
+  env._set("NODE_ENV", "jest")
+  env._set("USE_QUOTAS", null)
+  if (error) {
+    throw error
+  }
+}
+
 exports.runStep = async function runStep(stepId, inputs) {
   let step
   if (

packages/server/src/constants/index.js

@@ -80,8 +80,6 @@ exports.AutoFieldSubTypes = {
   AUTO_ID: "autoID",
 }
 
-exports.BUILDER_CONFIG_DB = "builder-config-db"
-exports.HOSTING_DOC = "hosting-doc"
 exports.OBJ_STORE_DIRECTORY = "/app-assets/assets"
 exports.BaseQueryVerbs = {
   CREATE: "create",
@@ -89,3 +87,9 @@ exports.BaseQueryVerbs = {
   UPDATE: "update",
   DELETE: "delete",
 }
+
+exports.ObjectStoreBuckets = {
+  BACKUPS: "backups",
+  APPS: "prod-budi-app-assets",
+  TEMPLATES: "templates",
+}

packages/server/src/db/builder.js

@@ -0,0 +1,38 @@
+const CouchDB = require("./index")
+const { StaticDatabases } = require("./utils")
+const env = require("../environment")
+
+const SELF_HOST_ERR = "Unable to access builder DB/doc - not self hosted."
+const BUILDER_DB = StaticDatabases.BUILDER
+
+/**
+ * This is the builder database, right now this is a single, static database
+ * that is present across the whole system and determines some core functionality
+ * for the builder (e.g. storage of API keys). This has been limited to self hosting
+ * as it doesn't make as much sense against the currently design Cloud system.
+ */
+
+exports.getBuilderMainDoc = async () => {
+  if (!env.SELF_HOSTED) {
+    throw SELF_HOST_ERR
+  }
+  const db = new CouchDB(BUILDER_DB.name)
+  try {
+    return await db.get(BUILDER_DB.baseDoc)
+  } catch (err) {
+    // doesn't exist yet, nothing to get
+    return {
+      _id: BUILDER_DB.baseDoc,
+    }
+  }
+}
+
+exports.setBuilderMainDoc = async doc => {
+  if (!env.SELF_HOSTED) {
+    throw SELF_HOST_ERR
+  }
+  // make sure to override the ID
+  doc._id = BUILDER_DB.baseDoc
+  const db = new CouchDB(BUILDER_DB.name)
+  return db.put(doc)
+}

packages/server/src/db/client.js

@@ -1,12 +1,10 @@
 const PouchDB = require("pouchdb")
 const replicationStream = require("pouchdb-replication-stream")
 const allDbs = require("pouchdb-all-dbs")
-const { budibaseAppsDir } = require("../utilities/budibaseDir")
 const find = require("pouchdb-find")
 const env = require("../environment")
 
-const COUCH_DB_URL = env.COUCH_DB_URL || `leveldb://${budibaseAppsDir()}/.data/`
+const COUCH_DB_URL = env.COUCH_DB_URL || "http://localhost:10000/db/"
-const isInMemory = env.NODE_ENV === "jest"
 
 PouchDB.plugin(replicationStream.plugin)
 PouchDB.plugin(find)
@@ -14,10 +12,10 @@ PouchDB.adapter("writableStream", replicationStream.adapters.writableStream)
 
 let POUCH_DB_DEFAULTS = {
   prefix: COUCH_DB_URL,
-  skip_setup: !!env.CLOUD,
+  skip_setup: env.isProd(),
 }
 
-if (isInMemory) {
+if (env.isTest()) {
   PouchDB.plugin(require("pouchdb-adapter-memory"))
   POUCH_DB_DEFAULTS = {
     prefix: undefined,

packages/server/src/db/dynamoClient.js

@@ -1,4 +1,4 @@
-let _ = require("lodash")
+let { merge } = require("lodash")
 let env = require("../environment")
 
 const AWS_REGION = env.AWS_REGION ? env.AWS_REGION : "eu-west-1"
@@ -38,7 +38,7 @@ class Table {
       params.Key[this._sort] = sort
     }
     if (otherProps) {
-      params = _.merge(params, otherProps)
+      params = merge(params, otherProps)
     }
     let response = await docClient.get(params).promise()
     return response.Item
@@ -77,7 +77,7 @@ class Table {
       params.ConditionExpression += "attribute_exists(#PRIMARY)"
     }
     if (otherProps) {
-      params = _.merge(params, otherProps)
+      params = merge(params, otherProps)
     }
     return docClient.update(params).promise()
   }
@@ -94,7 +94,7 @@ class Table {
       Item: item,
     }
     if (otherProps) {
-      params = _.merge(params, otherProps)
+      params = merge(params, otherProps)
     }
     return docClient.put(params).promise()
   }
@@ -119,7 +119,7 @@ exports.init = endpoint => {
 exports.apiKeyTable = new Table(TableInfo.API_KEYS)
 exports.userTable = new Table(TableInfo.USERS)
 
-if (env.CLOUD) {
+if (env.isProd()) {
   exports.init(`https://dynamodb.${AWS_REGION}.amazonaws.com`)
 } else {
   env._set("AWS_ACCESS_KEY_ID", "KEY_ID")

packages/server/src/db/linkedRows/index.js

@@ -27,7 +27,7 @@ const EventType = {
 }
 
 exports.EventType = EventType
-// re-export utils here for ease of use
+// re-export search here for ease of use
 exports.IncludeDocs = IncludeDocs
 exports.getLinkDocuments = getLinkDocuments
 exports.createLinkView = createLinkView

packages/server/src/db/utils.js

@@ -3,6 +3,18 @@ const newid = require("./newid")
 const UNICODE_MAX = "\ufff0"
 const SEPARATOR = "_"
 
+const StaticDatabases = {
+  BUILDER: {
+    name: "builder-db",
+    baseDoc: "builder-doc",
+  },
+  // TODO: needs removed
+  BUILDER_HOSTING: {
+    name: "builder-config-db",
+    baseDoc: "hosting-doc",
+  },
+}
+
 const DocumentTypes = {
   TABLE: "ta",
   ROW: "ro",
@@ -25,10 +37,16 @@ const ViewNames = {
   USERS: "ta_users",
 }
 
+const SearchIndexes = {
+  ROWS: "rows",
+}
+
+exports.StaticDatabases = StaticDatabases
 exports.ViewNames = ViewNames
 exports.DocumentTypes = DocumentTypes
 exports.SEPARATOR = SEPARATOR
 exports.UNICODE_MAX = UNICODE_MAX
+exports.SearchIndexes = SearchIndexes
 
 exports.getQueryIndex = viewName => {
   return `database/${viewName}`

packages/server/src/db/views/staticViews.js

@@ -1,5 +1,10 @@
 const CouchDB = require("../index")
-const { DocumentTypes, SEPARATOR, ViewNames } = require("../utils")
+const {
+  DocumentTypes,
+  SEPARATOR,
+  ViewNames,
+  SearchIndexes,
+} = require("../utils")
 const SCREEN_PREFIX = DocumentTypes.SCREEN + SEPARATOR
 
 /**************************************************
@@ -72,3 +77,42 @@ exports.createRoutingView = async appId => {
   }
   await db.put(designDoc)
 }
+
+async function searchIndex(appId, indexName, fnString) {
+  const db = new CouchDB(appId)
+  const designDoc = await db.get("_design/database")
+  designDoc.indexes = {
+    [indexName]: {
+      index: fnString,
+    },
+  }
+  await db.put(designDoc)
+}
+
+exports.createAllSearchIndex = async appId => {
+  await searchIndex(
+    appId,
+    SearchIndexes.ROWS,
+    function(doc) {
+      function idx(input, prev) {
+        for (let key of Object.keys(input)) {
+          const idxKey = prev != null ? `${prev}.${key}` : key
+          if (key === "_id" || key === "_rev") {
+            continue
+          }
+          if (typeof input[key] !== "object") {
+            // eslint-disable-next-line no-undef
+            index(idxKey, input[key], { store: true })
+          } else {
+            idx(input[key], idxKey)
+          }
+        }
+      }
+      if (doc._id.startsWith("ro_")) {
+        // eslint-disable-next-line no-undef
+        index("default", doc._id)
+        idx(doc)
+      }
+    }.toString()
+  )
+}

packages/server/src/electron.js

@@ -1,112 +1,112 @@
-const { app, BrowserWindow, shell, dialog } = require("electron")
+// const { app, BrowserWindow, shell, dialog } = require("electron")
-const { join } = require("./utilities/centralPath")
+// const { join } = require("./utilities/centralPath")
-const isDev = require("electron-is-dev")
+// const isDev = require("electron-is-dev")
-const { autoUpdater } = require("electron-updater")
+// const { autoUpdater } = require("electron-updater")
-const unhandled = require("electron-unhandled")
+// const unhandled = require("electron-unhandled")
-const { existsSync } = require("fs-extra")
+// const { existsSync } = require("fs-extra")
-const initialiseBudibase = require("./utilities/initialiseBudibase")
+// const initialiseBudibase = require("./utilities/initialiseBudibase")
-const { budibaseAppsDir } = require("./utilities/budibaseDir")
+// const { budibaseAppsDir } = require("./utilities/budibaseDir")
-const { openNewGitHubIssue, debugInfo } = require("electron-util")
+// const { openNewGitHubIssue, debugInfo } = require("electron-util")
-const eventEmitter = require("./events")
+// const eventEmitter = require("./events")
 
-const budibaseDir = budibaseAppsDir()
+// const budibaseDir = budibaseAppsDir()
-const envFile = join(budibaseDir, ".env")
+// const envFile = join(budibaseDir, ".env")
 
-async function startApp() {
+// async function startApp() {
-  if (!existsSync(envFile)) {
+//   if (!existsSync(envFile)) {
-    await initialiseBudibase({ dir: budibaseDir })
+//     await initialiseBudibase({ dir: budibaseDir })
-  }
+//   }
-  // evict environment from cache, so it reloads when next asked
+//   // evict environment from cache, so it reloads when next asked
-  delete require.cache[require.resolve("./environment")]
+//   delete require.cache[require.resolve("./environment")]
-  // store the port incase its going to get overridden
+//   // store the port incase its going to get overridden
-  const port = process.env.PORT
+//   const port = process.env.PORT
-  require("dotenv").config({ path: envFile })
+//   require("dotenv").config({ path: envFile })
-  // overwrite the port - don't want to use dotenv for the port
+//   // overwrite the port - don't want to use dotenv for the port
-  require("./environment")._set("PORT", port)
+//   require("./environment")._set("PORT", port)
 
-  unhandled({
+//   unhandled({
-    showDialog: true,
+//     showDialog: true,
-    reportButton: error => {
+//     reportButton: error => {
-      openNewGitHubIssue({
+//       openNewGitHubIssue({
-        title: error.message,
+//         title: error.message,
-        user: "Budibase",
+//         user: "Budibase",
-        labels: ["error-report"],
+//         labels: ["error-report"],
-        repo: "budibase",
+//         repo: "budibase",
-        body: `### Error that occurred when using the budibase builder:\n\`\`\`\n${
+//         body: `### Error that occurred when using the budibase builder:\n\`\`\`\n${
-          error.stack
+//           error.stack
-        }\n\`\`\`\n### Operating System Information:\n---\n\n${debugInfo()}`,
+//         }\n\`\`\`\n### Operating System Information:\n---\n\n${debugInfo()}`,
-      })
+//       })
-    },
+//     },
-  })
+//   })
 
-  let win
+//   let win
 
-  function handleRedirect(e, url) {
+//   function handleRedirect(e, url) {
-    e.preventDefault()
+//     e.preventDefault()
-    shell.openExternal(url)
+//     shell.openExternal(url)
-  }
+//   }
 
-  async function createWindow() {
+//   async function createWindow() {
-    app.server = require("./app")
+//     app.server = require("./app")
-    eventEmitter.on("internal:port", port => {
+//     eventEmitter.on("internal:port", port => {
-      const APP_URL = `http://localhost:${port}/_builder`
+//       const APP_URL = `http://localhost:${port}/_builder`
-      const APP_TITLE = "Budibase Builder"
+//       const APP_TITLE = "Budibase Builder"
-      win = new BrowserWindow({
+//       win = new BrowserWindow({
-        width: 1920,
+//         width: 1920,
-        height: 1080,
+//         height: 1080,
-        icon: join(__dirname, "..", "build", "icons", "512x512.png"),
+//         icon: join(__dirname, "..", "build", "icons", "512x512.png"),
-      })
+//       })
-      win.setTitle(APP_TITLE)
+//       win.setTitle(APP_TITLE)
-      win.loadURL(APP_URL)
+//       win.loadURL(APP_URL)
-      if (isDev) {
+//       if (isDev) {
-        win.webContents.openDevTools()
+//         win.webContents.openDevTools()
-      } else {
+//       } else {
-        autoUpdater.checkForUpdatesAndNotify()
+//         autoUpdater.checkForUpdatesAndNotify()
-      }
+//       }
 
-      // open _blank in default browser
+//       // open _blank in default browser
-      win.webContents.on("new-window", handleRedirect)
+//       win.webContents.on("new-window", handleRedirect)
-      win.webContents.on("will-navigate", handleRedirect)
+//       win.webContents.on("will-navigate", handleRedirect)
-    })
+//     })
-  }
+//   }
 
-  app.whenReady().then(createWindow)
+//   app.whenReady().then(createWindow)
 
-  // Quit when all windows are closed.
+//   // Quit when all windows are closed.
-  app.on("window-all-closed", () => {
+//   app.on("window-all-closed", () => {
-    // On macOS it is common for applications and their menu bar
+//     // On macOS it is common for applications and their menu bar
-    // to stay active until the user quits explicitly with Cmd + Q
+//     // to stay active until the user quits explicitly with Cmd + Q
-    if (process.platform !== "darwin") {
+//     if (process.platform !== "darwin") {
-      app.server.close()
+//       app.server.close()
-      app.quit()
+//       app.quit()
-    }
+//     }
-  })
+//   })
 
-  app.on("activate", () => {
+//   app.on("activate", () => {
-    // On macOS it's common to re-create a window in the app when the
+//     // On macOS it's common to re-create a window in the app when the
-    // dock icon is clicked and there are no other windows open.
+//     // dock icon is clicked and there are no other windows open.
-    if (win === null) createWindow()
+//     if (win === null) createWindow()
-  })
+//   })
-}
+// }
 
-autoUpdater.on("update-downloaded", (event, releaseNotes, releaseName) => {
+// autoUpdater.on("update-downloaded", (event, releaseNotes, releaseName) => {
-  const dialogOpts = {
+//   const dialogOpts = {
-    type: "info",
+//     type: "info",
-    buttons: ["Restart", "Later"],
+//     buttons: ["Restart", "Later"],
-    title: "Budibase Update Available",
+//     title: "Budibase Update Available",
-    message: process.platform === "win32" ? releaseNotes : releaseName,
+//     message: process.platform === "win32" ? releaseNotes : releaseName,
-    detail:
+//     detail:
-      "A new version of the budibase builder has been downloaded. Restart the application to apply the updates.",
+//       "A new version of the budibase builder has been downloaded. Restart the application to apply the updates.",
-  }
+//   }
 
-  dialog.showMessageBox(dialogOpts).then(returnValue => {
+//   dialog.showMessageBox(dialogOpts).then(returnValue => {
-    if (returnValue.response === 0) autoUpdater.quitAndInstall()
+//     if (returnValue.response === 0) autoUpdater.quitAndInstall()
-  })
+//   })
-})
+// })
 
-autoUpdater.on("error", message => {
+// autoUpdater.on("error", message => {
-  console.error("There was a problem updating the application")
+//   console.error("There was a problem updating the application")
-  console.error(message)
+//   console.error(message)
-})
+// })
 
-startApp()
+// startApp()

packages/server/src/environment.js

@@ -1,45 +1,64 @@
-const { resolve, join } = require("./utilities/centralPath")
+function isTest() {
-const { homedir } = require("os")
+  return (
-const { app } = require("electron")
+    process.env.NODE_ENV === "jest" ||
+    process.env.NODE_ENV === "cypress" ||
+    process.env.JEST_WORKER_ID != null
+  )
+}
+
+function isDev() {
+  return (
+    process.env.NODE_ENV !== "production" &&
+    process.env.BUDIBASE_ENVIRONMENT !== "production"
+  )
+}
 
 let LOADED = false
-
+if (!LOADED && isDev() && !isTest()) {
-if (!LOADED) {
+  require("dotenv").config()
-  const homeDir = app ? app.getPath("home") : homedir()
-  const budibaseDir = join(homeDir, ".budibase")
-  process.env.BUDIBASE_DIR = budibaseDir
-  require("dotenv").config({ path: resolve(budibaseDir, ".env") })
   LOADED = true
 }
 
 module.exports = {
-  CLIENT_ID: process.env.CLIENT_ID,
+  // important
-  NODE_ENV: process.env.NODE_ENV,
-  JWT_SECRET: process.env.JWT_SECRET,
-  BUDIBASE_DIR: process.env.BUDIBASE_DIR,
   PORT: process.env.PORT,
+  JWT_SECRET: process.env.JWT_SECRET,
   COUCH_DB_URL: process.env.COUCH_DB_URL,
+  MINIO_URL: process.env.MINIO_URL,
+  WORKER_URL: process.env.WORKER_URL,
+  SELF_HOSTED: process.env.SELF_HOSTED,
+  AWS_REGION: process.env.AWS_REGION,
+  ENABLE_ANALYTICS: process.env.ENABLE_ANALYTICS,
+  MINIO_ACCESS_KEY: process.env.MINIO_ACCESS_KEY,
+  MINIO_SECRET_KEY: process.env.MINIO_SECRET_KEY,
+  USE_QUOTAS: process.env.USE_QUOTAS,
+  // environment
+  NODE_ENV: process.env.NODE_ENV,
+  JEST_WORKER_ID: process.env.JEST_WORKER_ID,
+  BUDIBASE_ENVIRONMENT: process.env.BUDIBASE_ENVIRONMENT,
+  // minor
   SALT_ROUNDS: process.env.SALT_ROUNDS,
   LOGGER: process.env.LOGGER,
   LOG_LEVEL: process.env.LOG_LEVEL,
   AUTOMATION_DIRECTORY: process.env.AUTOMATION_DIRECTORY,
   AUTOMATION_BUCKET: process.env.AUTOMATION_BUCKET,
-  BUDIBASE_ENVIRONMENT: process.env.BUDIBASE_ENVIRONMENT,
   SENDGRID_API_KEY: process.env.SENDGRID_API_KEY,
-  CLOUD: process.env.CLOUD,
-  SELF_HOSTED: process.env.SELF_HOSTED,
-  WORKER_URL: process.env.WORKER_URL,
-  HOSTING_KEY: process.env.HOSTING_KEY,
   DYNAMO_ENDPOINT: process.env.DYNAMO_ENDPOINT,
-  AWS_REGION: process.env.AWS_REGION,
+  // old - to remove
-  DEPLOYMENT_CREDENTIALS_URL: process.env.DEPLOYMENT_CREDENTIALS_URL,
+  CLIENT_ID: process.env.CLIENT_ID,
+  BUDIBASE_DIR: process.env.BUDIBASE_DIR,
+  DEPLOYMENT_DB_URL: process.env.DEPLOYMENT_DB_URL,
   BUDIBASE_API_KEY: process.env.BUDIBASE_API_KEY,
   USERID_API_KEY: process.env.USERID_API_KEY,
-  ENABLE_ANALYTICS: process.env.ENABLE_ANALYTICS,
+  DEPLOYMENT_CREDENTIALS_URL: process.env.DEPLOYMENT_CREDENTIALS_URL,
-  DEPLOYMENT_DB_URL: process.env.DEPLOYMENT_DB_URL,
+  HOSTING_KEY: process.env.HOSTING_KEY,
-  LOCAL_TEMPLATES: process.env.LOCAL_TEMPLATES,
   _set(key, value) {
     process.env[key] = value
     module.exports[key] = value
   },
+  isTest,
+  isDev,
+  isProd: () => {
+    return !isDev()
+  },
 }

packages/server/src/index.js

@@ -1,17 +1,10 @@
-const { budibaseTempDir } = require("./utilities/budibaseDir")
-const { isDev } = require("./utilities")
-
 const fixPath = require("fix-path")
-const fs = require("fs")
+const { checkDevelopmentEnvironment } = require("./utilities/fileSystem")
-
-async function runServer() {
-  if (isDev() && !fs.existsSync(budibaseTempDir())) {
-    console.error(
-      "Please run a build before attempting to run server independently to fill 'tmp' directory."
-    )
-    process.exit(-1)
-  }
 
+function runServer() {
+  // this will shutdown the system if development environment not ready
+  // will print an error explaining what to do
+  checkDevelopmentEnvironment()
   fixPath()
   require("./app")
 }

packages/server/src/middleware/authorized.js

@@ -13,19 +13,12 @@ const { AuthTypes } = require("../constants")
 
 const ADMIN_ROLES = [BUILTIN_ROLE_IDS.ADMIN, BUILTIN_ROLE_IDS.BUILDER]
 
-const LOCAL_PASS = new RegExp(["webhooks/trigger"].join("|"))
-
 function hasResource(ctx) {
   return ctx.resourceId != null
 }
 
 module.exports = (permType, permLevel = null) => async (ctx, next) => {
-  // webhooks can pass locally
+  if (env.isProd() && ctx.headers["x-api-key"] && ctx.headers["x-instanceid"]) {
-  if (!env.CLOUD && LOCAL_PASS.test(ctx.request.url)) {
-    return next()
-  }
-
-  if (env.CLOUD && ctx.headers["x-api-key"] && ctx.headers["x-instanceid"]) {
     // api key header passed by external webhook
     if (await isAPIKeyValid(ctx.headers["x-api-key"])) {
       ctx.auth = {
@@ -41,20 +34,23 @@ module.exports = (permType, permLevel = null) => async (ctx, next) => {
     return ctx.throw(403, "API key invalid")
   }
 
-  // don't expose builder endpoints in the cloud
-  if (env.CLOUD && permType === PermissionTypes.BUILDER) return
-
   if (!ctx.user) {
     return ctx.throw(403, "No user info found")
   }
 
   const role = ctx.user.role
+  const isBuilder = role._id === BUILTIN_ROLE_IDS.BUILDER
+  const isAdmin = ADMIN_ROLES.includes(role._id)
+  const isAuthed = ctx.auth.authenticated
+
+  if (permType === PermissionTypes.BUILDER && isBuilder) {
+    return next()
+  }
+
   const { basePermissions, permissions } = await getUserPermissions(
     ctx.appId,
     role._id
   )
-  const isAdmin = ADMIN_ROLES.includes(role._id)
-  const isAuthed = ctx.auth.authenticated
 
   // this may need to change in the future, right now only admins
   // can have access to builder features, this is hard coded into

packages/server/src/middleware/selfhost.js

@@ -1,14 +1,8 @@
 const env = require("../environment")
-const hosting = require("../utilities/builder/hosting")
 // if added as a middleware will stop requests unless builder is in self host mode
 // or cloud is in self host
 module.exports = async (ctx, next) => {
-  if (env.CLOUD && env.SELF_HOSTED) {
+  if (env.SELF_HOSTED) {
-    await next()
-    return
-  }
-  const hostingInfo = await hosting.getHostingInfo()
-  if (hostingInfo.type === hosting.HostingTypes.SELF) {
     await next()
     return
   }

packages/server/src/middleware/tests/authorized.spec.js

@@ -3,8 +3,15 @@ const env = require("../../environment")
 const apiKey = require("../../utilities/security/apikey")
 const { AuthTypes } = require("../../constants")
 const { PermissionTypes, PermissionLevels } = require("../../utilities/security/permissions")
-const { Test } = require("supertest")
+jest.mock("../../environment", () => ({
-jest.mock("../../environment")
+    prod: false,
+    isTest: () => true,
+    isProd: () => this.prod,
+    _set: (key, value) => {
+      this.prod = value === "production"
+    }
+  })
+)
 jest.mock("../../utilities/security/apikey")
 
 class TestConfiguration {
@@ -47,8 +54,8 @@ class TestConfiguration {
     this.ctx.request.url = url
   }
 
-  setCloudEnv(isCloud) {
+  setEnvironment(isProd) {
-    env.CLOUD = isCloud
+    env._set("NODE_ENV", isProd ? "production" : "jest")
   }
 
   setRequestHeaders(headers) {
@@ -73,19 +80,13 @@ describe("Authorization middleware", () => {
     config = new TestConfiguration()
   })
 
-  it("passes the middleware for local webhooks", async () => {
-    config.setRequestUrl("https://something/webhooks/trigger")
-    await config.executeMiddleware()
-    expect(config.next).toHaveBeenCalled()
-  })
-
   describe("external web hook call", () => {
     let ctx = {}
     let middleware
 
     beforeEach(() => {
       config = new TestConfiguration()
-      config.setCloudEnv(true)
+      config.setEnvironment(true)
       config.setRequestHeaders({
         "x-api-key": "abc123",
         "x-instanceid": "instance123",
@@ -121,7 +122,7 @@ describe("Authorization middleware", () => {
 
     beforeEach(() => {
       config = new TestConfiguration()
-      config.setCloudEnv(true)
+      config.setEnvironment(true)
       config.setAuthenticated(true)
     })
 
@@ -144,7 +145,7 @@ describe("Authorization middleware", () => {
     })
 
     it("throws if the user has only builder permissions", async () => {
-      config.setCloudEnv(false)
+      config.setEnvironment(false)
       config.setMiddlewareRequiredPermission(PermissionTypes.BUILDER)
       config.setUser({
         role: {

packages/server/src/middleware/tests/selfhost.spec.js

@@ -1,6 +1,5 @@
-const selfHostMiddleware = require("../selfhost");
+const selfHostMiddleware = require("../selfhost")
 const env = require("../../environment")
-const hosting = require("../../utilities/builder/hosting");
 jest.mock("../../environment") 
 jest.mock("../../utilities/builder/hosting") 
 
@@ -20,16 +19,6 @@ class TestConfiguration {
     return this.middleware(this.ctx, this.next)
   }
 
-  setCloudHosted() {
-    env.CLOUD = 1 
-    env.SELF_HOSTED = 0
-  }
-
-  setSelfHosted() {
-    env.CLOUD = 0
-    env.SELF_HOSTED = 1
-  }
-
   afterEach() {
     jest.clearAllMocks()
   }
@@ -46,30 +35,10 @@ describe("Self host middleware", () => {
     config.afterEach()
   })
 
-  it("calls next() when CLOUD and SELF_HOSTED env vars are set", async () => {
+  it("calls next() when SELF_HOSTED env var is set", async () => {
-    env.CLOUD = 1 
     env.SELF_HOSTED = 1 
 
     await config.executeMiddleware()
     expect(config.next).toHaveBeenCalled()
   })
-
-  it("throws when hostingInfo type is cloud", async () => {
-    config.setSelfHosted()
-
-    hosting.getHostingInfo.mockImplementationOnce(() => ({ type: hosting.HostingTypes.CLOUD }))
-
-    await config.executeMiddleware()
-    expect(config.throw).toHaveBeenCalledWith(400, "Endpoint unavailable in cloud hosting.")
-    expect(config.next).not.toHaveBeenCalled()
-  })
-
-  it("calls the self hosting middleware to pass through to next() when the hostingInfo type is self", async () => {
-    config.setSelfHosted()
-
-    hosting.getHostingInfo.mockImplementationOnce(() => ({ type: hosting.HostingTypes.SELF }))
-
-    await config.executeMiddleware()
-    expect(config.next).toHaveBeenCalled()
-  })
 })

packages/server/src/middleware/tests/usageQuota.spec.js

@@ -5,7 +5,12 @@ const env = require("../../environment")
 
 jest.mock("../../db")
 jest.mock("../../utilities/usageQuota")
-jest.mock("../../environment")
+jest.mock("../../environment", () => ({
+  isTest: () => true,
+  isProd: () => false,
+  isDev: () => true,
+  _set: () => {},
+}))
 
 class TestConfiguration {
   constructor() {
@@ -32,12 +37,14 @@ class TestConfiguration {
     return this.middleware(this.ctx, this.next)
   }
 
-  cloudHosted(bool) {
+  setProd(bool) {
     if (bool) {
-      env.CLOUD = 1
+      env.isDev = () => false
+      env.isProd = () => true
       this.ctx.auth = { apiKey: "test" }
     } else {
-      env.CLOUD = 0
+      env.isDev = () => true
+      env.isProd = () => false
     }
   }
 
@@ -102,7 +109,7 @@ describe("usageQuota middleware", () => {
 
   it("calculates and persists the correct usage quota for the relevant action", async () => {
     config.setUrl("/rows")
-    config.cloudHosted(true)
+    config.setProd(true)
 
     await config.executeMiddleware()
 
@@ -112,7 +119,7 @@ describe("usageQuota middleware", () => {
 
   it("calculates the correct file size from a file upload call and adds it to quota", async () => {
     config.setUrl("/upload")
-    config.cloudHosted(true)
+    config.setProd(true)
     config.setFiles([
       {
         size: 100

packages/server/src/middleware/usageQuota.js

@@ -44,8 +44,8 @@ module.exports = async (ctx, next) => {
     }
   }
 
-  // if running in builder or a self hosted cloud usage quotas should not be executed
+  // if in development or a self hosted cloud usage quotas should not be executed
-  if (!env.CLOUD || env.SELF_HOSTED) {
+  if (env.isDev() || env.SELF_HOSTED) {
     return next()
   }
   // update usage for uploads to be the total size

packages/server/src/tests/utilities/TestConfiguration.js

@@ -14,9 +14,7 @@ const {
 } = require("./structures")
 const controllers = require("./controllers")
 const supertest = require("supertest")
-const fs = require("fs")
+const { cleanup } = require("../../utilities/fileSystem")
-const { budibaseAppsDir } = require("../../utilities/budibaseDir")
-const { join } = require("path")
 
 const EMAIL = "babs@babs.com"
 const PASSWORD = "babs_password"
@@ -66,13 +64,7 @@ class TestConfiguration {
     if (this.server) {
       this.server.close()
     }
-    const appDir = budibaseAppsDir()
+    cleanup(this.allApps.map(app => app._id))
-    const files = fs.readdirSync(appDir)
-    for (let file of files) {
-      if (this.allApps.some(app => file.includes(app._id))) {
-        fs.rmdirSync(join(appDir, file), { recursive: true })
-      }
-    }
   }
 
   defaultHeaders() {
@@ -81,9 +73,11 @@ class TestConfiguration {
       roleId: BUILTIN_ROLE_IDS.BUILDER,
     }
     const builderToken = jwt.sign(builderUser, env.JWT_SECRET)
+    // can be "production" for test case
+    const type = env.isProd() ? "cloud" : "local"
     const headers = {
       Accept: "application/json",
-      Cookie: [`budibase:builder:local=${builderToken}`],
+      Cookie: [`budibase:builder:${type}=${builderToken}`],
     }
     if (this.appId) {
       headers["x-budibase-app-id"] = this.appId

packages/server/src/utilities/builder/compileStaticAssets.js

@@ -1,35 +0,0 @@
-const { ensureDir, constants, copyFile } = require("fs-extra")
-const { join } = require("../centralPath")
-const { budibaseAppsDir } = require("../budibaseDir")
-
-/**
- * Compile all the non-db static web assets that are required for the running of
- * a budibase application. This includes the JSON structure of the DOM and
- * the client library, a script responsible for reading the JSON structure
- * and rendering the application.
- * @param {string} appId id of the application we want to compile static assets for
- */
-module.exports = async appId => {
-  const publicPath = join(budibaseAppsDir(), appId, "public")
-  await ensureDir(publicPath)
-  await copyClientLib(publicPath)
-}
-
-/**
- * Copy the budibase client library and sourcemap from NPM to <appId>/public/.
- * The client library is then served as a static asset when the budibase application
- * is running in preview or prod
- * @param {String} publicPath - path to write the client library to
- */
-const copyClientLib = async publicPath => {
-  const sourcepath = require.resolve("@budibase/client")
-  const destPath = join(publicPath, "budibase-client.js")
-
-  await copyFile(sourcepath, destPath, constants.COPYFILE_FICLONE)
-
-  await copyFile(
-    sourcepath + ".map",
-    destPath + ".map",
-    constants.COPYFILE_FICLONE
-  )
-}