Merge pull request #646 from mjashanks/master

Analytics

packages/builder/cypress/setup.js

@@ -14,6 +14,7 @@ rimraf.sync(homedir)
 
 process.env.BUDIBASE_API_KEY = "6BE826CB-6B30-4AEC-8777-2E90464633DE"
 process.env.NODE_ENV = "cypress"
+process.env.ENABLE_ANALYTICS = "false"
 
 initialiseBudibase({ dir: homedir, clientId: "cypress-test" })
   .then(() => {

packages/builder/package.json

@@ -75,7 +75,7 @@
     "fast-sort": "^2.2.0",
     "lodash": "^4.17.13",
     "mustache": "^4.0.1",
-    "posthog-js": "1.3.1",
+    "posthog-js": "1.4.5",
     "shortid": "^2.2.15",
     "svelte-loading-spinners": "^0.1.1",
     "svelte-portal": "^0.1.0",

packages/builder/rollup.config.js

@@ -158,6 +158,10 @@ export default {
           find: "constants",
           replacement: path.resolve(projectRootDir, "src/constants"),
         },
+        {
+          find: "analytics",
+          replacement: path.resolve(projectRootDir, "src/analytics"),
+        },
       ],
       customResolver,
     }),

packages/builder/src/analytics.js

@@ -1,25 +1,68 @@
 import * as Sentry from "@sentry/browser"
 import posthog from "posthog-js"
+import api from "builderStore/api"
 
-function activate() {
+let analyticsEnabled
+
+async function activate() {
+  if (analyticsEnabled === undefined) {
+    // only the server knows the true NODE_ENV
+    // this was an issue as NODE_ENV = 'cypress' on the server,
+    // but 'production' on the client
+    const response = await api.get("/api/analytics")
+    analyticsEnabled = (await response.json()) === true
+  }
+  if (!analyticsEnabled) return
   Sentry.init({ dsn: process.env.SENTRY_DSN })
   if (!process.env.POSTHOG_TOKEN) return
   posthog.init(process.env.POSTHOG_TOKEN, {
     api_host: process.env.POSTHOG_URL,
   })
+  posthog.set_config({ persistence: "cookie" })
+}
+
+function identify(id) {
+  if (!analyticsEnabled) return
+  if (!id) return
+  posthog.identify(id)
+  Sentry.configureScope(scope => {
+    scope.setUser({ id: id })
+  })
+}
+
+async function identifyByApiKey(apiKey) {
+  if (!analyticsEnabled) return true
+  const response = await fetch(
+    `https://03gaine137.execute-api.eu-west-1.amazonaws.com/prod/account/id?api_key=${apiKey.trim()}`
+  )
+
+  if (response.status === 200) {
+    const id = await response.json()
+
+    await api.put("/api/keys/userId", { value: id })
+    identify(id)
+    return true
+  }
+
+  return false
 }
 
 function captureException(err) {
+  if (!analyticsEnabled) return
   Sentry.captureException(err)
+  captureEvent("Error", { error: err.message ? err.message : err })
 }
 
-function captureEvent(event) {
+function captureEvent(eventName, props = {}) {
-  if (!process.env.POSTHOG_TOKEN) return
+  if (!analyticsEnabled || !process.env.POSTHOG_TOKEN) return
-  posthog.capture(event)
+  props.sourceApp = "builder"
+  posthog.capture(eventName, props)
 }
 
 export default {
   activate,
+  identify,
+  identifyByApiKey,
   captureException,
   captureEvent,
 }

packages/builder/src/builderStore/index.js

@@ -1,7 +1,7 @@
 import { getStore } from "./store"
 import { getBackendUiStore } from "./store/backend"
 import { getAutomationStore } from "./store/automation/"
-import analytics from "../analytics"
+import analytics from "analytics"
 
 export const store = getStore()
 export const backendUiStore = getBackendUiStore()
@@ -9,9 +9,8 @@ export const automationStore = getAutomationStore()
 
 export const initialise = async () => {
   try {
-    if (process.env.NODE_ENV === "production") {
+    analytics.activate()
-      analytics.activate()
+    analytics.captureEvent("Builder Started")
-    }
   } catch (err) {
     console.log(err)
   }

packages/builder/src/builderStore/store/index.js

@@ -14,6 +14,7 @@ import { fetchComponentLibDefinitions } from "../loadComponentLibraries"
 import { buildCodeForScreens } from "../buildCodeForScreens"
 import { generate_screen_css } from "../generate_css"
 import { insertCodeMetadata } from "../insertCodeMetadata"
+import analytics from "analytics"
 import { uuid } from "../uuid"
 import {
   selectComponent as _selectComponent,
@@ -308,7 +309,9 @@ const addChildComponent = store => (componentToAdd, presetProps = {}) => {
 
     state.currentView = "component"
     state.currentComponentInfo = newComponent.props
-
+    analytics.captureEvent("Added Component", {
+      name: newComponent.props._component,
+    })
     return state
   })
 }

packages/builder/src/components/automation/AutomationPanel/AutomationList/CreateAutomationModal.svelte

@@ -3,6 +3,7 @@
   import { notifier } from "builderStore/store/notifications"
   import ActionButton from "components/common/ActionButton.svelte"
   import { Input } from "@budibase/bbui"
+  import analytics from "analytics"
 
   export let onClosed
 
@@ -19,6 +20,7 @@
     })
     onClosed()
     notifier.success(`Automation ${name} created.`)
+    analytics.captureEvent("Automation Created", { name })
   }
 </script>
 

packages/builder/src/components/automation/AutomationPanel/BlockList/AutomationBlock.svelte

@@ -1,5 +1,6 @@
 <script>
   import { automationStore } from "builderStore"
+  import analytics from "analytics"
 
   export let blockDefinition
   export let stepId
@@ -12,6 +13,9 @@
       stepId,
       type: blockType,
     })
+    analytics.captureEvent("Added Automation Block", {
+      name: blockDefinition.name,
+    })
   }
 </script>
 

packages/builder/src/components/database/DataTable/popovers/Calculate.svelte

@@ -10,6 +10,7 @@
   import { backendUiStore } from "builderStore"
   import { notifier } from "builderStore/store/notifications"
   import CreateEditRecord from "../modals/CreateEditRecord.svelte"
+  import analytics from "analytics"
 
   const CALCULATIONS = [
     {
@@ -35,6 +36,7 @@
   function saveView() {
     backendUiStore.actions.views.save(view)
     notifier.success(`View ${view.name} saved.`)
+    analytics.captureEvent("Added View Calculate", { field: view.field })
     dropdown.hide()
   }
 </script>

packages/builder/src/components/database/DataTable/popovers/Filter.svelte

@@ -10,6 +10,7 @@
   import { backendUiStore } from "builderStore"
   import { notifier } from "builderStore/store/notifications"
   import CreateEditRecord from "../modals/CreateEditRecord.svelte"
+  import analytics from "analytics"
 
   const CONDITIONS = [
     {
@@ -63,6 +64,9 @@
     backendUiStore.actions.views.save(view)
     notifier.success(`View ${view.name} saved.`)
     dropdown.hide()
+    analytics.captureEvent("Added View Filter", {
+      filters: JSON.stringify(view.filters),
+    })
   }
 
   function removeFilter(idx) {

packages/builder/src/components/database/DataTable/popovers/View.svelte

@@ -11,6 +11,7 @@
   import { backendUiStore } from "builderStore"
   import { notifier } from "builderStore/store/notifications"
   import CreateEditRecord from "../modals/CreateEditRecord.svelte"
+  import analytics from "analytics"
 
   let anchor
   let dropdown
@@ -37,6 +38,7 @@
     })
     notifier.success(`View ${name} created`)
     dropdown.hide()
+    analytics.captureEvent("View Created", { name })
     $goto(`../../../view/${name}`)
   }
 </script>

packages/builder/src/components/nav/ModelNavigator/CreateTable.svelte

@@ -3,6 +3,7 @@
   import { backendUiStore } from "builderStore"
   import { notifier } from "builderStore/store/notifications"
   import { DropdownMenu, Button, Icon, Input, Select } from "@budibase/bbui"
+  import analytics from "analytics"
 
   export let table
 
@@ -19,6 +20,7 @@
     $goto(`./model/${model._id}`)
     name = ""
     dropdown.hide()
+    analytics.captureEvent("Table Created", { name })
   }
 
   const onClosed = () => {

packages/builder/src/components/settings/tabs/APIKeys.svelte

@@ -3,13 +3,21 @@
   import { store } from "builderStore"
   import api from "builderStore/api"
   import posthog from "posthog-js"
+  import analytics from "analytics"
 
   let keys = { budibase: "", sendGrid: "" }
 
   async function updateKey([key, value]) {
+    if (key === "budibase") {
+      const isValid = await analytics.identifyByApiKey(value)
+      if (!isValid) {
+        // TODO: add validation message
+        keys = { ...keys }
+        return
+      }
+    }
     const response = await api.put(`/api/keys/${key}`, { value })
     const res = await response.json()
-    if (key === "budibase") posthog.identify(value)
     keys = { ...keys, ...res }
   }
 
@@ -17,6 +25,8 @@
   async function fetchKeys() {
     const response = await api.get(`/api/keys/`)
     const res = await response.json()
+    // dont want this to ever be editable, as its fetched based on Api Key
+    if (res.userId) delete res.userId
     keys = res
   }
 

packages/builder/src/components/start/CreateAppModal.svelte

@@ -14,7 +14,7 @@
   import { getContext } from "svelte"
   import { fade } from "svelte/transition"
   import { post } from "builderStore/api"
-  import analytics from "../../analytics"
+  import analytics from "analytics"
 
   const { open, close } = getContext("simple-modal")
   //Move this to context="module" once svelte-forms is updated so that it can bind to stores correctly
@@ -22,12 +22,34 @@
 
   export let hasKey
 
+  let isApiKeyValid
+  let lastApiKey
+  let fetchApiKeyPromise
+  const validateApiKey = async apiKey => {
+    if (!apiKey) return false
+
+    // make sure we only fetch once, unless API Key is changed
+    if (isApiKeyValid === undefined || apiKey !== lastApiKey) {
+      lastApiKey = apiKey
+      // svelte reactivity was causing a requst to get fired mutiple times
+      // so, we make everything await the same promise, if one exists
+      if (!fetchApiKeyPromise) {
+        fetchApiKeyPromise = analytics.identifyByApiKey(apiKey)
+      }
+      isApiKeyValid = await fetchApiKeyPromise
+      fetchApiKeyPromise = undefined
+    }
+    return isApiKeyValid
+  }
+
   let submitting = false
   let errors = {}
   let validationErrors = {}
   let validationSchemas = [
     {
-      apiKey: string().required("Please enter your API key."),
+      apiKey: string()
+        .required("Please enter your API key.")
+        .test("valid-apikey", "This API key is invalid", validateApiKey),
     },
     {
       applicationName: string().required("Your application must have a name."),
@@ -122,7 +144,7 @@
         name: $createAppStore.values.applicationName,
       })
       const appJson = await appResp.json()
-      analytics.captureEvent("web_app_created", {
+      analytics.captureEvent("App Created", {
         name,
         appId: appJson._id,
       })
@@ -160,6 +182,7 @@
   }
 
   function extractErrors({ inner }) {
+    if (!inner) return {}
     return inner.reduce((acc, err) => {
       return { ...acc, [err.path]: err.message }
     }, {})

packages/builder/src/pages/[application]/deploy/index.svelte

@@ -4,7 +4,7 @@
   import { notifier } from "builderStore/store/notifications"
   import api from "builderStore/api"
   import Spinner from "components/common/Spinner.svelte"
-  import analytics from "../../../analytics"
+  import analytics from "analytics"
 
   let deployed = false
   let loading = false
@@ -26,10 +26,13 @@
       notifier.success(`Your Deployment is Complete.`)
       deployed = true
       loading = false
-      analytics.captureEvent("web_app_deployment", {
+      analytics.captureEvent("Deployed App", {
         appId,
       })
     } catch (err) {
+      analytics.captureEvent("Deploy App Failed", {
+        appId,
+      })
       analytics.captureException(err)
       notifier.danger("Deployment unsuccessful. Please try again later.")
       loading = false

packages/builder/src/pages/index.svelte

@@ -9,6 +9,7 @@
   import Spinner from "components/common/Spinner.svelte"
   import CreateAppModal from "components/start/CreateAppModal.svelte"
   import { Button } from "@budibase/bbui"
+  import analytics from "analytics"
 
   let promise = getApps()
 
@@ -27,15 +28,15 @@
 
   async function fetchKeys() {
     const response = await api.get(`/api/keys/`)
-    const res = await response.json()
+    return await response.json()
-    return res.budibase
   }
 
   async function checkIfKeysAndApps() {
-    const key = await fetchKeys()
+    const keys = await fetchKeys()
     const apps = await getApps()
-    if (key) {
+    if (keys.userId) {
       hasKey = true
+      analytics.identify(keys.userId)
     } else {
       showCreateAppModal()
     }

packages/builder/yarn.lock

@@ -4847,9 +4847,10 @@ posix-character-classes@^0.1.0:
   version "0.1.1"
   resolved "https://registry.yarnpkg.com/posix-character-classes/-/posix-character-classes-0.1.1.tgz#01eac0fe3b5af71a2a6c02feabb8c1fef7e00eab"
 
-posthog-js@1.3.1:
+posthog-js@1.4.5:
-  version "1.3.1"
+  version "1.4.5"
-  resolved "https://registry.yarnpkg.com/posthog-js/-/posthog-js-1.3.1.tgz#970acec1423eaa5dba0d2603410c9c70294e16da"
+  resolved "https://registry.yarnpkg.com/posthog-js/-/posthog-js-1.4.5.tgz#b16235afe47938bd71eaed4ede3790c8b910ed71"
+  integrity sha512-Rzc5/DpuX55BqwNEbZB0tLav1gEinnr5H+82cbLiMtXLADlxmCwZiEaVXcC3XOqW0x8bcAEehicx1TbpfBamzA==
 
 prelude-ls@~1.1.2:
   version "1.1.2"

packages/server/.env.template

@@ -17,3 +17,4 @@ LOG_LEVEL=error
 DEPLOYMENT_CREDENTIALS_URL="https://dt4mpwwap8.execute-api.eu-west-1.amazonaws.com/prod/"
 DEPLOYMENT_DB_URL="https://couchdb.budi.live:5984"
 SENTRY_DSN=https://a34ae347621946bf8acded18e5b7d4b8@o420233.ingest.sentry.io/5338131
+ENABLE_ANALYTICS="true"

packages/server/src/api/controllers/analytics.js

@@ -0,0 +1,3 @@
+exports.isEnabled = async function(ctx) {
+  ctx.body = JSON.stringify(process.env.ENABLE_ANALYTICS === "true")
+}

packages/server/src/api/controllers/apikeys.js

@@ -8,6 +8,7 @@ exports.fetch = async function(ctx) {
   ctx.body = {
     budibase: process.env.BUDIBASE_API_KEY,
     sendgrid: process.env.SENDGRID_API_KEY,
+    userId: process.env.USERID_API_KEY,
   }
 }
 

packages/server/src/api/index.js

@@ -19,6 +19,7 @@ const {
   automationRoutes,
   accesslevelRoutes,
   apiKeysRoutes,
+  analyticsRoutes,
 } = require("./routes")
 
 const router = new Router()
@@ -109,6 +110,9 @@ router.use(accesslevelRoutes.allowedMethods())
 router.use(apiKeysRoutes.routes())
 router.use(apiKeysRoutes.allowedMethods())
 
+router.use(analyticsRoutes.routes())
+router.use(analyticsRoutes.allowedMethods())
+
 router.use(staticRoutes.routes())
 router.use(staticRoutes.allowedMethods())
 

packages/server/src/api/routes/analytics.js

@@ -0,0 +1,10 @@
+const Router = require("@koa/router")
+const authorized = require("../../middleware/authorized")
+const { BUILDER } = require("../../utilities/accessLevels")
+const controller = require("../controllers/analytics")
+
+const router = Router()
+
+router.get("/api/analytics", authorized(BUILDER), controller.isEnabled)
+
+module.exports = router

packages/server/src/api/routes/index.js

@@ -13,6 +13,7 @@ const automationRoutes = require("./automation")
 const accesslevelRoutes = require("./accesslevel")
 const deployRoutes = require("./deploy")
 const apiKeysRoutes = require("./apikeys")
+const analyticsRoutes = require("./analytics")
 
 module.exports = {
   deployRoutes,
@@ -30,4 +31,5 @@ module.exports = {
   automationRoutes,
   accesslevelRoutes,
   apiKeysRoutes,
+  analyticsRoutes,
 }