Updating API keys and changing over system to allow use of builder endpoints when running in cloud.

packages/server/src/api/controllers/apikeys.js

@@ -1,56 +1,32 @@
-const fs = require("fs")
+const builderDB = require("../../db/builder")
-const { join } = require("../../utilities/centralPath")
-const readline = require("readline")
-const { budibaseAppsDir } = require("../../utilities/budibaseDir")
-const env = require("../../environment")
-const ENV_FILE_PATH = "/.env"
 
 exports.fetch = async function(ctx) {
-  ctx.status = 200
+  try {
-  ctx.body = {
+    const mainDoc = await builderDB.getBuilderMainDoc()
-    budibase: env.BUDIBASE_API_KEY,
+    ctx.body = mainDoc.apiKeys ? mainDoc.apiKeys : {}
-    userId: env.USERID_API_KEY,
+  } catch (err) {
+    /* istanbul ignore next */
+    ctx.throw(400, err)
   }
 }
 
 exports.update = async function(ctx) {
-  const key = `${ctx.params.key.toUpperCase()}_API_KEY`
+  const key = ctx.params.key
   const value = ctx.request.body.value
 
-  // set environment variables
+  try {
-  env._set(key, value)
+    const mainDoc = await builderDB.getBuilderMainDoc()
-
+    if (mainDoc.apiKeys == null) {
-  // Write to file
+      mainDoc.apiKeys = {}
-  await updateValues([key, value])
-
-  ctx.status = 200
-  ctx.message = `Updated ${ctx.params.key} API key succesfully.`
-  ctx.body = { [ctx.params.key]: ctx.request.body.value }
-}
-
-async function updateValues([key, value]) {
-  let newContent = ""
-  let keyExists = false
-  let envPath = join(budibaseAppsDir(), ENV_FILE_PATH)
-  const readInterface = readline.createInterface({
-    input: fs.createReadStream(envPath),
-    output: process.stdout,
-    console: false,
-  })
-  readInterface.on("line", function(line) {
-    // Mutate lines and change API Key
-    if (line.startsWith(key)) {
-      line = `${key}=${value}`
-      keyExists = true
     }
-    newContent = `${newContent}\n${line}`
+    mainDoc.apiKeys[key] = value
-  })
+    const resp = await builderDB.setBuilderMainDoc(mainDoc)
-  readInterface.on("close", function() {
+    ctx.body = {
-    // Write file here
+      _id: resp.id,
-    if (!keyExists) {
+      _rev: resp.rev,
-      // Add API Key if it doesn't exist in the file at all
-      newContent = `${newContent}\n${key}=${value}`
     }
-    fs.writeFileSync(envPath, newContent)
+  } catch (err) {
-  })
+    /* istanbul ignore next */
+    ctx.throw(400, err)
+  }
 }

packages/server/src/api/controllers/hosting.js

@@ -1,11 +1,11 @@
 const CouchDB = require("../../db")
-const { BUILDER_CONFIG_DB, HOSTING_DOC } = require("../../constants")
 const {
   getHostingInfo,
   getDeployedApps,
   HostingTypes,
   getAppUrl,
 } = require("../../utilities/builder/hosting")
+const { StaticDatabases } = require("../../db/utils")
 
 exports.fetchInfo = async ctx => {
   ctx.body = {
@@ -14,17 +14,17 @@ exports.fetchInfo = async ctx => {
 }
 
 exports.save = async ctx => {
-  const db = new CouchDB(BUILDER_CONFIG_DB)
+  const db = new CouchDB(StaticDatabases.BUILDER_HOSTING.name)
   const { type } = ctx.request.body
   if (type === HostingTypes.CLOUD && ctx.request.body._rev) {
     ctx.body = await db.remove({
       ...ctx.request.body,
-      _id: HOSTING_DOC,
+      _id: StaticDatabases.BUILDER_HOSTING.baseDoc,
     })
   } else {
     ctx.body = await db.put({
       ...ctx.request.body,
-      _id: HOSTING_DOC,
+      _id: StaticDatabases.BUILDER_HOSTING.baseDoc,
     })
   }
 }

packages/server/src/api/routes/tests/apikeys.spec.js

@@ -1,8 +1,5 @@
 const setup = require("./utilities")
 const { checkBuilderEndpoint } = require("./utilities/TestFunctions")
-const { budibaseAppsDir } = require("../../../utilities/budibaseDir")
-const fs = require("fs")
-const path = require("path")
 
 describe("/api/keys", () => {
   let request = setup.getRequest()
@@ -16,12 +13,14 @@ describe("/api/keys", () => {
 
   describe("fetch", () => {
     it("should allow fetching", async () => {
-      const res = await request
+      await setup.switchToCloudForFunction(async () => {
-        .get(`/api/keys`)
+        const res = await request
-        .set(config.defaultHeaders())
+          .get(`/api/keys`)
-        .expect("Content-Type", /json/)
+          .set(config.defaultHeaders())
-        .expect(200)
+          .expect("Content-Type", /json/)
-      expect(res.body).toBeDefined()
+          .expect(200)
+        expect(res.body).toBeDefined()
+      })
     })
 
     it("should check authorization for builder", async () => {
@@ -35,17 +34,18 @@ describe("/api/keys", () => {
 
   describe("update", () => {
     it("should allow updating a value", async () => {
-      fs.writeFileSync(path.join(budibaseAppsDir(), ".env"), "TEST_API_KEY=thing")
+      await setup.switchToCloudForFunction(async () => {
-      const res = await request
+        const res = await request
-        .put(`/api/keys/TEST`)
+          .put(`/api/keys/TEST`)
-        .send({
+          .send({
-          value: "test"
+            value: "test"
-        })
+          })
-        .set(config.defaultHeaders())
+          .set(config.defaultHeaders())
-        .expect("Content-Type", /json/)
+          .expect("Content-Type", /json/)
-        .expect(200)
+          .expect(200)
-      expect(res.body["TEST"]).toEqual("test")
+        expect(res.body._id).toBeDefined()
-      expect(process.env.TEST_API_KEY).toEqual("test")
+        expect(res.body._rev).toBeDefined()
+      })
     })
 
     it("should check authorization for builder", async () => {

packages/server/src/constants/index.js

@@ -80,8 +80,6 @@ exports.AutoFieldSubTypes = {
   AUTO_ID: "autoID",
 }
 
-exports.BUILDER_CONFIG_DB = "builder-config-db"
-exports.HOSTING_DOC = "hosting-doc"
 exports.OBJ_STORE_DIRECTORY = "/app-assets/assets"
 exports.BaseQueryVerbs = {
   CREATE: "create",

packages/server/src/db/builder.js

@@ -0,0 +1,38 @@
+const CouchDB = require("./index")
+const { StaticDatabases } = require("./utils")
+const env = require("../environment")
+
+const SELF_HOST_ERR = "Unable to access builder DB/doc - not self hosted."
+const BUILDER_DB = StaticDatabases.BUILDER
+
+/**
+ * This is the builder database, right now this is a single, static database
+ * that is present across the whole system and determines some core functionality
+ * for the builder (e.g. storage of API keys). This has been limited to self hosting
+ * as it doesn't make as much sense against the currently design Cloud system.
+ */
+
+exports.getBuilderMainDoc = async () => {
+  if (!env.SELF_HOSTED) {
+    throw SELF_HOST_ERR
+  }
+  const db = new CouchDB(BUILDER_DB.name)
+  try {
+    return await db.get(BUILDER_DB.baseDoc)
+  } catch (err) {
+    // doesn't exist yet, nothing to get
+    return {
+      _id: BUILDER_DB.baseDoc,
+    }
+  }
+}
+
+exports.setBuilderMainDoc = async doc => {
+  if (!env.SELF_HOSTED) {
+    throw SELF_HOST_ERR
+  }
+  // make sure to override the ID
+  doc._id = BUILDER_DB.baseDoc
+  const db = new CouchDB(BUILDER_DB.name)
+  return db.put(doc)
+}

packages/server/src/db/utils.js

@@ -3,6 +3,18 @@ const newid = require("./newid")
 const UNICODE_MAX = "\ufff0"
 const SEPARATOR = "_"
 
+const StaticDatabases = {
+  BUILDER: {
+    name: "builder-db",
+    baseDoc: "builder-doc",
+  },
+  // TODO: needs removed
+  BUILDER_HOSTING: {
+    name: "builder-config-db",
+    baseDoc: "hosting-doc",
+  },
+}
+
 const DocumentTypes = {
   TABLE: "ta",
   ROW: "ro",
@@ -25,6 +37,7 @@ const ViewNames = {
   USERS: "ta_users",
 }
 
+exports.StaticDatabases = StaticDatabases
 exports.ViewNames = ViewNames
 exports.DocumentTypes = DocumentTypes
 exports.SEPARATOR = SEPARATOR

packages/server/src/environment.js

@@ -29,15 +29,15 @@ module.exports = {
   CLOUD: process.env.CLOUD,
   SELF_HOSTED: process.env.SELF_HOSTED,
   WORKER_URL: process.env.WORKER_URL,
-  HOSTING_KEY: process.env.HOSTING_KEY,
   DYNAMO_ENDPOINT: process.env.DYNAMO_ENDPOINT,
   AWS_REGION: process.env.AWS_REGION,
-  DEPLOYMENT_CREDENTIALS_URL: process.env.DEPLOYMENT_CREDENTIALS_URL,
+  ENABLE_ANALYTICS: process.env.ENABLE_ANALYTICS,
+  // TODO: remove all below - single stack conversion
+  DEPLOYMENT_DB_URL: process.env.DEPLOYMENT_DB_URL,
   BUDIBASE_API_KEY: process.env.BUDIBASE_API_KEY,
   USERID_API_KEY: process.env.USERID_API_KEY,
-  ENABLE_ANALYTICS: process.env.ENABLE_ANALYTICS,
+  DEPLOYMENT_CREDENTIALS_URL: process.env.DEPLOYMENT_CREDENTIALS_URL,
-  DEPLOYMENT_DB_URL: process.env.DEPLOYMENT_DB_URL,
+  HOSTING_KEY: process.env.HOSTING_KEY,
-  LOCAL_TEMPLATES: process.env.LOCAL_TEMPLATES,
   _set(key, value) {
     process.env[key] = value
     module.exports[key] = value

packages/server/src/middleware/authorized.js

@@ -13,18 +13,11 @@ const { AuthTypes } = require("../constants")
 
 const ADMIN_ROLES = [BUILTIN_ROLE_IDS.ADMIN, BUILTIN_ROLE_IDS.BUILDER]
 
-const LOCAL_PASS = new RegExp(["webhooks/trigger"].join("|"))
-
 function hasResource(ctx) {
   return ctx.resourceId != null
 }
 
 module.exports = (permType, permLevel = null) => async (ctx, next) => {
-  // webhooks can pass locally
-  if (!env.CLOUD && LOCAL_PASS.test(ctx.request.url)) {
-    return next()
-  }
-
   if (env.CLOUD && ctx.headers["x-api-key"] && ctx.headers["x-instanceid"]) {
     // api key header passed by external webhook
     if (await isAPIKeyValid(ctx.headers["x-api-key"])) {
@@ -41,20 +34,23 @@ module.exports = (permType, permLevel = null) => async (ctx, next) => {
     return ctx.throw(403, "API key invalid")
   }
 
-  // don't expose builder endpoints in the cloud
-  if (env.CLOUD && permType === PermissionTypes.BUILDER) return
-
   if (!ctx.user) {
     return ctx.throw(403, "No user info found")
   }
 
   const role = ctx.user.role
+  const isBuilder = role._id === BUILTIN_ROLE_IDS.BUILDER
+  const isAdmin = ADMIN_ROLES.includes(role._id)
+  const isAuthed = ctx.auth.authenticated
+
+  if (permType === PermissionTypes.BUILDER && isBuilder) {
+    return next()
+  }
+
   const { basePermissions, permissions } = await getUserPermissions(
     ctx.appId,
     role._id
   )
-  const isAdmin = ADMIN_ROLES.includes(role._id)
-  const isAuthed = ctx.auth.authenticated
 
   // this may need to change in the future, right now only admins
   // can have access to builder features, this is hard coded into

packages/server/src/tests/utilities/TestConfiguration.js

@@ -81,9 +81,10 @@ class TestConfiguration {
       roleId: BUILTIN_ROLE_IDS.BUILDER,
     }
     const builderToken = jwt.sign(builderUser, env.JWT_SECRET)
+    const type = env.CLOUD ? "cloud" : "local"
     const headers = {
       Accept: "application/json",
-      Cookie: [`budibase:builder:local=${builderToken}`],
+      Cookie: [`budibase:builder:${type}=${builderToken}`],
     }
     if (this.appId) {
       headers["x-budibase-app-id"] = this.appId

packages/server/src/utilities/builder/hosting.js

@@ -1,5 +1,5 @@
 const CouchDB = require("../../db")
-const { BUILDER_CONFIG_DB, HOSTING_DOC } = require("../../constants")
+const { StaticDatabases } = require("../../db/utils")
 const fetch = require("node-fetch")
 const env = require("../../environment")
 
@@ -23,16 +23,16 @@ exports.HostingTypes = {
 }
 
 exports.getHostingInfo = async () => {
-  const db = new CouchDB(BUILDER_CONFIG_DB)
+  const db = new CouchDB(StaticDatabases.BUILDER_HOSTING.name)
   let doc
   try {
-    doc = await db.get(HOSTING_DOC)
+    doc = await db.get(StaticDatabases.BUILDER_HOSTING.baseDoc)
   } catch (err) {
     // don't write this doc, want to be able to update these default props
     // for our servers with a new release without needing to worry about state of
     // PouchDB in peoples installations
     doc = {
-      _id: HOSTING_DOC,
+      _id: StaticDatabases.BUILDER_HOSTING.baseDoc,
       type: exports.HostingTypes.CLOUD,
       hostingUrl: PROD_HOSTING_URL,
       selfHostKey: "",