Merge pull request #15772 from Budibase/BUDI-9127/track-usage

Track usage

packages/backend-core/src/cache/docWritethrough.ts

@@ -1,4 +1,4 @@
-import { AnyDocument, Database } from "@budibase/types"
+import { AnyDocument, Database, Document } from "@budibase/types"
 
 import { JobQueue, Queue, createQueue } from "../queue"
 import * as dbUtils from "../db"
@@ -70,7 +70,7 @@ export class DocWritethroughProcessor {
   }
 }
 
-export class DocWritethrough {
+export class DocWritethrough<T extends Document = Document> {
   private db: Database
   private _docId: string
 
@@ -83,7 +83,7 @@ export class DocWritethrough {
     return this._docId
   }
 
-  async patch(data: Record<string, any>) {
+  async patch(data: Partial<T>) {
     await DocWritethroughProcessor.queue.add({
       dbName: this.db.name,
       docId: this.docId,

packages/backend-core/src/docIds/ids.ts

@@ -124,3 +124,7 @@ export const generateDevInfoID = (userId: string) => {
 export const generatePluginID = (name: string) => {
   return `${DocumentType.PLUGIN}${SEPARATOR}${name}`
 }
+
+export const generateOAuth2LogID = (id: string) => {
+  return `${DocumentType.OAUTH2_CONFIG_LOG}${SEPARATOR}${id}`
+}

packages/backend-core/src/queue/inMemoryQueue.ts

@@ -1,5 +1,5 @@
 import events from "events"
-import { newid } from "../utils"
+import { newid, timeout } from "../utils"
 import { Queue, QueueOptions, JobOptions } from "./queue"
 import { helpers } from "@budibase/shared-core"
 import { Job, JobId, JobInformation } from "bull"
@@ -109,6 +109,12 @@ export class InMemoryQueue<T = any> implements Partial<Queue<T>> {
         try {
           await retryFunc(resp)
           this._emitter.emit("completed", message as Job<T>)
+
+          const indexToRemove = this._messages.indexOf(message)
+          if (indexToRemove === -1) {
+            throw "Failed deleting a processed message"
+          }
+          this._messages.splice(indexToRemove, 1)
         } catch (e: any) {
           console.error(e)
         }
@@ -248,6 +254,16 @@ export class InMemoryQueue<T = any> implements Partial<Queue<T>> {
       .filter(job => job.opts?.repeat != null)
       .map(job => jobToJobInformation(job as Job))
   }
+
+  async whenCurrentJobsFinished() {
+    do {
+      await timeout(50)
+    } while (this.hasRunningJobs())
+  }
+
+  private hasRunningJobs() {
+    return this._addCount > this._runCount
+  }
 }
 
 export default InMemoryQueue

packages/builder/src/helpers/date.ts

@@ -0,0 +1,7 @@
+import { processStringSync } from "@budibase/string-templates"
+
+export function durationFromNow(isoDate: string) {
+  return processStringSync("{{ durationFromNow time 'millisecond' }} ago", {
+    time: isoDate,
+  })
+}

packages/builder/src/helpers/index.ts

@@ -12,3 +12,4 @@ export {
 export * as featureFlag from "./featureFlags"
 export * as bindings from "./bindings"
 export * from "./confirm"
+export * from "./date"

packages/builder/src/pages/builder/app/[application]/settings/oauth2/index.svelte

@@ -4,6 +4,7 @@
   import AddButton from "./AddButton.svelte"
   import { onMount } from "svelte"
   import MoreMenuRenderer from "./MoreMenuRenderer.svelte"
+  import { capitalise, durationFromNow } from "@/helpers"
 
   const schema = {
     name: {
@@ -24,10 +25,14 @@
     oauth2.fetch()
   })
 
-  $: configs = $oauth2.configs.map(c => ({
+  $: configs = $oauth2.configs.map(c => {
-    lastUsed: "Never used",
+    return {
       ...c,
-  }))
+      lastUsed: c.lastUsage
+        ? `${capitalise(durationFromNow(c.lastUsage))}`
+        : "Never used",
+    }
+  })
 </script>
 
 <Layout noPadding>

packages/builder/src/stores/builder/oauth2.ts

@@ -38,6 +38,7 @@ export class OAuth2Store extends BudiStore<OAuth2StoreState> {
           clientId: c.clientId,
           clientSecret: c.clientSecret,
           method: c.method,
+          lastUsage: c.lastUsage,
         })),
         loading: false,
       }))

packages/builder/src/types/oauth2.ts

@@ -1,3 +1,3 @@
-import { OAuth2ConfigResponse } from "@budibase/types"
+import { FetchOAuth2ConfigsResponse } from "@budibase/types"
 
-export interface OAuth2Config extends OAuth2ConfigResponse {}
+export type OAuth2Config = FetchOAuth2ConfigsResponse["configs"][0]

packages/frontend-core/src/api/oauth2.ts

@@ -2,7 +2,6 @@ import {
   FetchOAuth2ConfigsResponse,
   InsertOAuth2ConfigRequest,
   InsertOAuth2ConfigResponse,
-  OAuth2ConfigResponse,
   UpdateOAuth2ConfigRequest,
   UpdateOAuth2ConfigResponse,
   ValidateConfigRequest,
@@ -11,7 +10,7 @@ import {
 import { BaseAPIClient } from "./types"
 
 export interface OAuth2Endpoints {
-  fetch: () => Promise<OAuth2ConfigResponse[]>
+  fetch: () => Promise<FetchOAuth2ConfigsResponse["configs"]>
   create: (
     config: InsertOAuth2ConfigRequest
   ) => Promise<InsertOAuth2ConfigResponse>

packages/server/src/api/controllers/oauth2.ts

@@ -30,9 +30,17 @@ function toFetchOAuth2ConfigsResponse(
 export async function fetch(ctx: Ctx<void, FetchOAuth2ConfigsResponse>) {
   const configs = await sdk.oauth2.fetch()
 
+  const timestamps = await sdk.oauth2.getLastUsages(configs.map(c => c._id))
+
   const response: FetchOAuth2ConfigsResponse = {
-    configs: (configs || []).map(toFetchOAuth2ConfigsResponse),
+    configs: (configs || []).map(c => ({
+      ...toFetchOAuth2ConfigsResponse(c),
+      lastUsage: timestamps[c._id]
+        ? new Date(timestamps[c._id]).toISOString()
+        : undefined,
+    })),
   }
+
   ctx.body = response
 }
 

packages/server/src/integrations/rest.ts

@@ -386,7 +386,7 @@ export class RestIntegration implements IntegrationBase {
     }
 
     if (authConfigType === RestAuthType.OAUTH2) {
-      return { Authorization: await sdk.oauth2.generateToken(authConfigId) }
+      return { Authorization: await sdk.oauth2.getToken(authConfigId) }
     }
 
     if (!this.config.authConfigs) {

packages/server/src/sdk/app/oauth2/crud.ts

@@ -89,4 +89,9 @@ export async function remove(configId: string, _rev: string): Promise<void> {
     }
     throw e
   }
+
+  const usageLog = await db.tryGet(docIds.generateOAuth2LogID(configId))
+  if (usageLog) {
+    await db.remove(usageLog)
+  }
 }

packages/server/src/sdk/app/oauth2/tests/utils.spec.ts

@@ -1,12 +1,14 @@
-import { generator } from "@budibase/backend-core/tests"
+import { generator, utils as testUtils } from "@budibase/backend-core/tests"
 import { GenericContainer, Wait } from "testcontainers"
 import sdk from "../../.."
 import TestConfiguration from "../../../../tests/utilities/TestConfiguration"
-import { generateToken } from "../utils"
+import { getToken } from "../utils"
 import path from "path"
 import { KEYCLOAK_IMAGE } from "../../../../integrations/tests/utils/images"
 import { startContainer } from "../../../../integrations/tests/utils"
 import { OAuth2CredentialsMethod } from "@budibase/types"
+import { cache } from "@budibase/backend-core"
+import tk from "timekeeper"
 
 const config = new TestConfiguration()
 
@@ -43,7 +45,7 @@ describe("oauth2 utils", () => {
   })
 
   describe.each(Object.values(OAuth2CredentialsMethod))(
-    "generateToken (in %s)",
+    "getToken (in %s)",
     method => {
       it("successfully generates tokens", async () => {
         const response = await config.doInContext(config.appId, async () => {
@@ -55,7 +57,7 @@ describe("oauth2 utils", () => {
             method,
           })
 
-          const response = await generateToken(oauthConfig._id)
+          const response = await getToken(oauthConfig._id)
           return response
         })
 
@@ -73,7 +75,7 @@ describe("oauth2 utils", () => {
               method,
             })
 
-            await generateToken(oauthConfig._id)
+            await getToken(oauthConfig._id)
           })
         ).rejects.toThrow("Error fetching oauth2 token: Not Found")
       })
@@ -89,7 +91,7 @@ describe("oauth2 utils", () => {
               method,
             })
 
-            await generateToken(oauthConfig._id)
+            await getToken(oauthConfig._id)
           })
         ).rejects.toThrow(
           "Error fetching oauth2 token: Invalid client or Invalid client credentials"
@@ -107,12 +109,104 @@ describe("oauth2 utils", () => {
               method,
             })
 
-            await generateToken(oauthConfig._id)
+            await getToken(oauthConfig._id)
           })
         ).rejects.toThrow(
           "Error fetching oauth2 token: Invalid client or Invalid client credentials"
         )
       })
+
+      describe("track usages", () => {
+        beforeAll(() => {
+          tk.freeze(Date.now())
+        })
+
+        it("tracks usages on generation", async () => {
+          const oauthConfig = await config.doInContext(config.appId, () =>
+            sdk.oauth2.create({
+              name: generator.guid(),
+              url: `${keycloakUrl}/realms/myrealm/protocol/openid-connect/token`,
+              clientId: "my-client",
+              clientSecret: "my-secret",
+              method,
+            })
+          )
+
+          await config.doInContext(config.appId, () =>
+            getToken(oauthConfig._id)
+          )
+          await testUtils.queue.processMessages(
+            cache.docWritethrough.DocWritethroughProcessor.queue
+          )
+
+          const usageLog = await config.doInContext(config.appId, () =>
+            sdk.oauth2.getLastUsages([oauthConfig._id])
+          )
+
+          expect(usageLog[oauthConfig._id]).toEqual(Date.now())
+        })
+
+        it("does not track on failed usages", async () => {
+          const oauthConfig = await config.doInContext(config.appId, () =>
+            sdk.oauth2.create({
+              name: generator.guid(),
+              url: `${keycloakUrl}/realms/myrealm/protocol/openid-connect/token`,
+              clientId: "wrong-client",
+              clientSecret: "my-secret",
+              method,
+            })
+          )
+
+          await expect(
+            config.doInContext(config.appId, () => getToken(oauthConfig._id))
+          ).rejects.toThrow()
+          await testUtils.queue.processMessages(
+            cache.docWritethrough.DocWritethroughProcessor.queue
+          )
+
+          const usageLog = await config.doInContext(config.appId, () =>
+            sdk.oauth2.getLastUsages([oauthConfig._id])
+          )
+
+          expect(usageLog[oauthConfig._id]).toBeUndefined()
+        })
+
+        it("tracks usages between prod and dev, keeping always the latest", async () => {
+          const oauthConfig = await config.doInContext(config.appId, () =>
+            sdk.oauth2.create({
+              name: generator.guid(),
+              url: `${keycloakUrl}/realms/myrealm/protocol/openid-connect/token`,
+              clientId: "my-client",
+              clientSecret: "my-secret",
+              method,
+            })
+          )
+
+          await config.doInContext(config.appId, () =>
+            getToken(oauthConfig._id)
+          )
+
+          await config.publish()
+
+          tk.travel(Date.now() + 100)
+          await config.doInContext(config.prodAppId, () =>
+            getToken(oauthConfig._id)
+          )
+          await testUtils.queue.processMessages(
+            cache.docWritethrough.DocWritethroughProcessor.queue
+          )
+
+          for (const appId of [config.appId, config.prodAppId]) {
+            const usageLog = await config.doInContext(appId, () =>
+              sdk.oauth2.getLastUsages([oauthConfig._id])
+            )
+
+            expect(usageLog).toEqual({
+              [oauthConfig._id]: Date.now(),
+            })
+          }
+        })
+      })
     }
   )
 })

packages/server/src/sdk/app/oauth2/utils.ts

@@ -1,7 +1,14 @@
 import fetch, { RequestInit } from "node-fetch"
 import { HttpError } from "koa"
 import { get } from "../oauth2"
-import { OAuth2CredentialsMethod } from "@budibase/types"
+import { Document, OAuth2CredentialsMethod } from "@budibase/types"
+import { cache, context, docIds } from "@budibase/backend-core"
+
+interface OAuth2LogDocument extends Document {
+  lastUsage: number
+}
+
+const { DocWritethrough } = cache.docWritethrough
 
 async function fetchToken(config: {
   url: string
@@ -40,8 +47,18 @@ async function fetchToken(config: {
   return resp
 }
 
+const trackUsage = async (id: string) => {
+  const writethrough = new DocWritethrough<OAuth2LogDocument>(
+    context.getAppDB(),
+    docIds.generateOAuth2LogID(id)
+  )
+  await writethrough.patch({
+    lastUsage: Date.now(),
+  })
+}
+
 // TODO: check if caching is worth
-export async function generateToken(id: string) {
+export async function getToken(id: string) {
   const config = await get(id)
   if (!config) {
     throw new HttpError(`oAuth config ${id} count not be found`)
@@ -56,6 +73,7 @@ export async function generateToken(id: string) {
     throw new Error(`Error fetching oauth2 token: ${message}`)
   }
 
+  await trackUsage(id)
   return `${jsonResponse.token_type} ${jsonResponse.access_token}`
 }
 
@@ -79,3 +97,31 @@ export async function validateConfig(config: {
     return { valid: false, message: e.message }
   }
 }
+
+export async function getLastUsages(ids: string[]) {
+  const devDocs = await context
+    .getAppDB()
+    .getMultiple<OAuth2LogDocument>(ids.map(docIds.generateOAuth2LogID), {
+      allowMissing: true,
+    })
+
+  const prodDocs = await context
+    .getProdAppDB()
+    .getMultiple<OAuth2LogDocument>(ids.map(docIds.generateOAuth2LogID), {
+      allowMissing: true,
+    })
+
+  const result = ids.reduce<Record<string, number>>((acc, id) => {
+    const devDoc = devDocs.find(d => d._id === docIds.generateOAuth2LogID(id))
+    if (devDoc) {
+      acc[id] = devDoc.lastUsage
+    }
+
+    const prodDoc = prodDocs.find(d => d._id === docIds.generateOAuth2LogID(id))
+    if (prodDoc && (!acc[id] || acc[id] < prodDoc.lastUsage)) {
+      acc[id] = prodDoc.lastUsage
+    }
+    return acc
+  }, {})
+  return result
+}

packages/types/src/api/web/app/oauth2.ts

@@ -11,7 +11,7 @@ export interface OAuth2ConfigResponse {
 }
 
 export interface FetchOAuth2ConfigsResponse {
-  configs: OAuth2ConfigResponse[]
+  configs: (OAuth2ConfigResponse & { lastUsage?: string })[]
 }
 
 export interface InsertOAuth2ConfigRequest {

packages/types/src/documents/document.ts

@@ -41,6 +41,7 @@ export enum DocumentType {
   SCIM_LOG = "scimlog",
   ROW_ACTIONS = "ra",
   OAUTH2_CONFIG = "oauth2",
+  OAUTH2_CONFIG_LOG = "oauth2log",
 }
 
 // Because DocumentTypes can overlap, we need to make sure that we search