Merge pull request #15733 from Budibase/BUDI-9127/use-oauth2-on-rest

Use OAuth2 on rest

.github/workflows/budibase_ci.yml

@@ -186,7 +186,7 @@ jobs:
         id: dotenv
         uses: falti/dotenv-action@v1.1.3
         with:
-          path: ./packages/server/datasource-sha.env
+          path: ./packages/server/images-sha.env
 
       - name: Pull testcontainers images
         run: |
@@ -213,6 +213,7 @@ jobs:
           docker pull redis &
           docker pull testcontainers/ryuk:0.5.1 &
           docker pull budibase/couchdb:v3.3.3-sqs-v2.1.1 &
+          docker pull ${{ steps.dotenv.outputs.KEYCLOAK_IMAGE }} &
 
           wait $(jobs -p)
 

packages/server/images-sha.env

@@ -4,4 +4,5 @@ POSTGRES_SHA=sha256:bd0d8e485d1aca439d39e5ea99b931160bd28d862e74c786f7508e9d0053
 MONGODB_SHA=sha256:afa36bca12295b5f9dae68a493c706113922bdab520e901bd5d6c9d7247a1d8d
 MARIADB_SHA=sha256:e59ba8783bf7bc02a4779f103bb0d8751ac0e10f9471089709608377eded7aa8
 ELASTICSEARCH_SHA=sha256:9a6443f55243f6acbfeb4a112d15eb3b9aac74bf25e0e39fa19b3ddd3a6879d0
-DYNAMODB_SHA=sha256:cf8cebd061f988628c02daff10fdb950a54478feff9c52f6ddf84710fe3c3906
+DYNAMODB_SHA=sha256:cf8cebd061f988628c02daff10fdb950a54478feff9c52f6ddf84710fe3c3906
+KEYCLOAK_IMAGE=keycloak/keycloak@sha256:044a457e04987e1fff756be3d2fa325a4ef420fa356b7034ecc9f1b693c32761

packages/server/src/api/controllers/oauth2.ts

@@ -1,5 +1,6 @@
 import {
   CreateOAuth2ConfigRequest,
+  CreateOAuth2ConfigResponse,
   Ctx,
   FetchOAuth2ConfigsResponse,
   OAuth2Config,
@@ -12,17 +13,26 @@ export async function fetch(ctx: Ctx<void, FetchOAuth2ConfigsResponse>) {
 
   const response: FetchOAuth2ConfigsResponse = {
     configs: (configs || []).map(c => ({
+      id: c.id,
       name: c.name,
+      url: c.url,
     })),
   }
   ctx.body = response
 }
 
-export async function create(ctx: Ctx<CreateOAuth2ConfigRequest, void>) {
-  const newConfig: RequiredKeys<OAuth2Config> = {
-    name: ctx.request.body.name,
+export async function create(
+  ctx: Ctx<CreateOAuth2ConfigRequest, CreateOAuth2ConfigResponse>
+) {
+  const { body } = ctx.request
+  const newConfig: RequiredKeys<Omit<OAuth2Config, "id">> = {
+    name: body.name,
+    url: body.url,
+    clientId: body.clientId,
+    clientSecret: body.clientSecret,
   }
 
-  await sdk.oauth2.create(newConfig)
+  const config = await sdk.oauth2.create(newConfig)
   ctx.status = 201
+  ctx.body = { config }
 }

packages/server/src/api/routes/tests/oauth2.spec.ts

@@ -1,4 +1,4 @@
-import { CreateOAuth2ConfigRequest } from "@budibase/types"
+import { CreateOAuth2ConfigRequest, VirtualDocumentType } from "@budibase/types"
 import * as setup from "./utilities"
 import { generator } from "@budibase/backend-core/tests"
 
@@ -8,6 +8,9 @@ describe("/oauth2", () => {
   function makeOAuth2Config(): CreateOAuth2ConfigRequest {
     return {
       name: generator.guid(),
+      url: generator.url(),
+      clientId: generator.guid(),
+      clientSecret: generator.hash(),
     }
   }
 
@@ -15,6 +18,10 @@ describe("/oauth2", () => {
 
   beforeEach(async () => await config.newTenant())
 
+  const expectOAuth2ConfigId = expect.stringMatching(
+    `^${VirtualDocumentType.OAUTH2_CONFIG}_.+$`
+  )
+
   describe("fetch", () => {
     it("returns empty when no oauth are created", async () => {
       const response = await config.api.oauth2.fetch()
@@ -33,7 +40,9 @@ describe("/oauth2", () => {
       expect(response).toEqual({
         configs: [
           {
+            id: expectOAuth2ConfigId,
             name: oauth2Config.name,
+            url: oauth2Config.url,
           },
         ],
       })
@@ -48,12 +57,17 @@ describe("/oauth2", () => {
       const response = await config.api.oauth2.fetch()
       expect(response.configs).toEqual([
         {
+          id: expectOAuth2ConfigId,
           name: oauth2Config.name,
+          url: oauth2Config.url,
         },
         {
+          id: expectOAuth2ConfigId,
           name: oauth2Config2.name,
+          url: oauth2Config2.url,
         },
       ])
+      expect(response.configs[0].id).not.toEqual(response.configs[1].id)
     })
 
     it("cannot create configurations with already existing names", async () => {
@@ -71,7 +85,9 @@ describe("/oauth2", () => {
       const response = await config.api.oauth2.fetch()
       expect(response.configs).toEqual([
         {
+          id: expectOAuth2ConfigId,
           name: oauth2Config.name,
+          url: oauth2Config.url,
         },
       ])
     })

packages/server/src/integrations/rest.ts

@@ -8,8 +8,6 @@ import {
   PaginationValues,
   QueryType,
   RestAuthType,
-  RestBasicAuthConfig,
-  RestBearerAuthConfig,
   RestConfig,
   RestQueryFields as RestQuery,
 } from "@budibase/types"
@@ -28,6 +26,8 @@ import { parse } from "content-disposition"
 import path from "path"
 import { Builder as XmlBuilder } from "xml2js"
 import { getAttachmentHeaders } from "./utils/restUtils"
+import { utils } from "@budibase/shared-core"
+import sdk from "../sdk"
 
 const coreFields = {
   path: {
@@ -377,29 +377,41 @@ export class RestIntegration implements IntegrationBase {
     return input
   }
 
-  getAuthHeaders(authConfigId?: string): { [key: string]: any } {
-    let headers: any = {}
+  async getAuthHeaders(
+    authConfigId?: string,
+    authConfigType?: RestAuthType
+  ): Promise<{ [key: string]: any }> {
+    if (!authConfigId) {
+      return {}
+    }
 
-    if (this.config.authConfigs && authConfigId) {
-      const authConfig = this.config.authConfigs.filter(
-        c => c._id === authConfigId
-      )[0]
-      // check the config still exists before proceeding
-      // if not - do nothing
-      if (authConfig) {
-        let config
-        switch (authConfig.type) {
-          case RestAuthType.BASIC:
-            config = authConfig.config as RestBasicAuthConfig
-            headers.Authorization = `Basic ${Buffer.from(
-              `${config.username}:${config.password}`
-            ).toString("base64")}`
-            break
-          case RestAuthType.BEARER:
-            config = authConfig.config as RestBearerAuthConfig
-            headers.Authorization = `Bearer ${config.token}`
-            break
-        }
+    if (authConfigType === RestAuthType.OAUTH2) {
+      return { Authorization: await sdk.oauth2.generateToken(authConfigId) }
+    }
+
+    if (!this.config.authConfigs) {
+      return {}
+    }
+
+    let headers: any = {}
+    const authConfig = this.config.authConfigs.filter(
+      c => c._id === authConfigId
+    )[0]
+    // check the config still exists before proceeding
+    // if not - do nothing
+    if (authConfig) {
+      const { type, config } = authConfig
+      switch (type) {
+        case RestAuthType.BASIC:
+          headers.Authorization = `Basic ${Buffer.from(
+            `${config.username}:${config.password}`
+          ).toString("base64")}`
+          break
+        case RestAuthType.BEARER:
+          headers.Authorization = `Bearer ${config.token}`
+          break
+        default:
+          throw utils.unreachable(type)
       }
     }
 
@@ -416,10 +428,11 @@ export class RestIntegration implements IntegrationBase {
       bodyType = BodyType.NONE,
       requestBody,
       authConfigId,
+      authConfigType,
       pagination,
       paginationValues,
     } = query
-    const authHeaders = this.getAuthHeaders(authConfigId)
+    const authHeaders = await this.getAuthHeaders(authConfigId, authConfigType)
 
     this.headers = {
       ...(this.config.defaultHeaders || {}),

packages/server/src/integrations/tests/rest.spec.ts

@@ -1,10 +1,16 @@
 import nock from "nock"
-import { RestIntegration } from "../rest"
-import { BodyType, RestAuthType } from "@budibase/types"
-import { Response } from "node-fetch"
 import TestConfiguration from "../../../src/tests/utilities/TestConfiguration"
+import { RestIntegration } from "../rest"
+import {
+  BasicRestAuthConfig,
+  BearerRestAuthConfig,
+  BodyType,
+  RestAuthType,
+} from "@budibase/types"
+import { Response } from "node-fetch"
 import { createServer } from "http"
 import { AddressInfo } from "net"
+import { generator } from "@budibase/backend-core/tests"
 
 const UUID_REGEX =
   "[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}"
@@ -224,7 +230,7 @@ describe("REST Integration", () => {
   })
 
   describe("authentication", () => {
-    const basicAuth = {
+    const basicAuth: BasicRestAuthConfig = {
       _id: "c59c14bd1898a43baa08da68959b24686",
       name: "basic-1",
       type: RestAuthType.BASIC,
@@ -234,7 +240,7 @@ describe("REST Integration", () => {
       },
     }
 
-    const bearerAuth = {
+    const bearerAuth: BearerRestAuthConfig = {
       _id: "0d91d732f34e4befabeff50b392a8ff3",
       name: "bearer-1",
       type: RestAuthType.BEARER,
@@ -269,6 +275,38 @@ describe("REST Integration", () => {
       const { data } = await integration.read({ authConfigId: bearerAuth._id })
       expect(data).toEqual({ foo: "bar" })
     })
+
+    it("adds OAuth2 auth", async () => {
+      const oauth2Url = generator.url()
+      const { config: oauthConfig } = await config.api.oauth2.create({
+        name: generator.guid(),
+        url: oauth2Url,
+        clientId: generator.guid(),
+        clientSecret: generator.hash(),
+      })
+
+      const token = generator.guid()
+
+      const url = new URL(oauth2Url)
+      nock(url.origin)
+        .post(url.pathname)
+        .reply(200, { token_type: "Bearer", access_token: token })
+
+      nock("https://example.com", {
+        reqheaders: { Authorization: `Bearer ${token}` },
+      })
+        .get("/")
+        .reply(200, { foo: "bar" })
+      const { data } = await config.doInContext(
+        config.appId,
+        async () =>
+          await integration.read({
+            authConfigId: oauthConfig.id,
+            authConfigType: RestAuthType.OAUTH2,
+          })
+      )
+      expect(data).toEqual({ foo: "bar" })
+    })
   })
 
   describe("page based pagination", () => {

packages/server/src/integrations/tests/utils/images.ts

@@ -1,7 +1,7 @@
 import dotenv from "dotenv"
 import { join } from "path"
 
-const path = join(__dirname, "..", "..", "..", "..", "datasource-sha.env")
+const path = join(__dirname, "..", "..", "..", "..", "images-sha.env")
 dotenv.config({
   path,
 })
@@ -14,3 +14,4 @@ export const MONGODB_IMAGE = `mongo@${process.env.MONGODB_SHA}`
 export const MARIADB_IMAGE = `mariadb@${process.env.MARIADB_SHA}`
 export const ELASTICSEARCH_IMAGE = `elasticsearch@${process.env.ELASTICSEARCH_SHA}`
 export const DYNAMODB_IMAGE = `amazon/dynamodb-local@${process.env.DYNAMODB_SHA}`
+export const KEYCLOAK_IMAGE = process.env.KEYCLOAK_IMAGE || ""

packages/server/src/sdk/app/oauth2.ts

@@ -1,28 +0,0 @@
-import { context, HTTPError } from "@budibase/backend-core"
-import { DocumentType, OAuth2Config, OAuth2Configs } from "@budibase/types"
-
-export async function fetch(): Promise<OAuth2Config[]> {
-  const db = context.getAppDB()
-  const result = await db.tryGet<OAuth2Configs>(DocumentType.OAUTH2_CONFIG)
-  if (!result) {
-    return []
-  }
-  return Object.values(result.configs)
-}
-
-export async function create(config: OAuth2Config) {
-  const db = context.getAppDB()
-  const doc: OAuth2Configs = (await db.tryGet<OAuth2Configs>(
-    DocumentType.OAUTH2_CONFIG
-  )) ?? {
-    _id: DocumentType.OAUTH2_CONFIG,
-    configs: {},
-  }
-
-  if (doc.configs[config.name]) {
-    throw new HTTPError("Name already used", 400)
-  }
-
-  doc.configs[config.name] = config
-  await db.put(doc)
-}

packages/server/src/sdk/app/oauth2/crud.ts

@@ -0,0 +1,50 @@
+import { context, HTTPError, utils } from "@budibase/backend-core"
+import {
+  Database,
+  DocumentType,
+  OAuth2Config,
+  OAuth2Configs,
+  SEPARATOR,
+  VirtualDocumentType,
+} from "@budibase/types"
+
+async function getDocument(db: Database = context.getAppDB()) {
+  const result = await db.tryGet<OAuth2Configs>(DocumentType.OAUTH2_CONFIG)
+  return result
+}
+
+export async function fetch(): Promise<OAuth2Config[]> {
+  const result = await getDocument()
+  if (!result) {
+    return []
+  }
+  return Object.values(result.configs)
+}
+
+export async function create(
+  config: Omit<OAuth2Config, "id">
+): Promise<OAuth2Config> {
+  const db = context.getAppDB()
+  const doc: OAuth2Configs = (await getDocument(db)) ?? {
+    _id: DocumentType.OAUTH2_CONFIG,
+    configs: {},
+  }
+
+  if (Object.values(doc.configs).find(c => c.name === config.name)) {
+    throw new HTTPError("Name already used", 400)
+  }
+
+  const id = `${VirtualDocumentType.OAUTH2_CONFIG}${SEPARATOR}${utils.newid()}`
+  doc.configs[id] = {
+    id,
+    ...config,
+  }
+
+  await db.put(doc)
+  return doc.configs[id]
+}
+
+export async function get(id: string): Promise<OAuth2Config | undefined> {
+  const doc = await getDocument()
+  return doc?.configs?.[id]
+}

packages/server/src/sdk/app/oauth2/index.ts

@@ -0,0 +1,2 @@
+export * from "./crud"
+export * from "./utils"

packages/server/src/sdk/app/oauth2/tests/docker-volume/realm.json

@@ -0,0 +1,13 @@
+{
+  "id": "myrealm",
+  "realm": "myrealm",
+  "enabled": true,
+  "clients": [
+    {
+      "clientId": "my-client",
+      "secret": "my-secret",
+      "directAccessGrantsEnabled": true,
+      "serviceAccountsEnabled": true
+    }
+  ]
+}

packages/server/src/sdk/app/oauth2/tests/utils.spec.ts

@@ -0,0 +1,110 @@
+import { generator } from "@budibase/backend-core/tests"
+import { GenericContainer, Wait } from "testcontainers"
+import sdk from "../../.."
+import TestConfiguration from "../../../../tests/utilities/TestConfiguration"
+import { generateToken } from "../utils"
+import path from "path"
+import { KEYCLOAK_IMAGE } from "../../../../integrations/tests/utils/images"
+import { startContainer } from "../../../../integrations/tests/utils"
+
+const config = new TestConfiguration()
+
+const volumePath = path.resolve(__dirname, "docker-volume")
+
+jest.setTimeout(60000)
+
+describe("oauth2 utils", () => {
+  let keycloakUrl: string
+
+  beforeAll(async () => {
+    await config.init()
+
+    const ports = await startContainer(
+      new GenericContainer(KEYCLOAK_IMAGE)
+        .withName("keycloak_testcontainer")
+        .withExposedPorts(8080)
+        .withBindMounts([
+          { source: volumePath, target: "/opt/keycloak/data/import/" },
+        ])
+        .withCommand(["start-dev", "--import-realm"])
+        .withWaitStrategy(
+          Wait.forLogMessage("Listening on: http://0.0.0.0:8080")
+        )
+        .withStartupTimeout(60000)
+    )
+
+    const port = ports.find(x => x.container === 8080)?.host
+    if (!port) {
+      throw new Error("Keycloak port not found")
+    }
+
+    keycloakUrl = `http://127.0.0.1:${port}`
+  })
+
+  describe("generateToken", () => {
+    it("successfully generates tokens", async () => {
+      const response = await config.doInContext(config.appId, async () => {
+        const oauthConfig = await sdk.oauth2.create({
+          name: generator.guid(),
+          url: `${keycloakUrl}/realms/myrealm/protocol/openid-connect/token`,
+          clientId: "my-client",
+          clientSecret: "my-secret",
+        })
+
+        const response = await generateToken(oauthConfig.id)
+        return response
+      })
+
+      expect(response).toEqual(expect.stringMatching(/^Bearer .+/))
+    })
+
+    it("handles wrong urls", async () => {
+      await expect(
+        config.doInContext(config.appId, async () => {
+          const oauthConfig = await sdk.oauth2.create({
+            name: generator.guid(),
+            url: `${keycloakUrl}/realms/wrong/protocol/openid-connect/token`,
+            clientId: "my-client",
+            clientSecret: "my-secret",
+          })
+
+          await generateToken(oauthConfig.id)
+        })
+      ).rejects.toThrow("Error fetching oauth2 token: Not Found")
+    })
+
+    it("handles wrong client ids", async () => {
+      await expect(
+        config.doInContext(config.appId, async () => {
+          const oauthConfig = await sdk.oauth2.create({
+            name: generator.guid(),
+            url: `${keycloakUrl}/realms/myrealm/protocol/openid-connect/token`,
+            clientId: "wrong-client-id",
+            clientSecret: "my-secret",
+          })
+
+          await generateToken(oauthConfig.id)
+        })
+      ).rejects.toThrow(
+        "Error fetching oauth2 token: Invalid client or Invalid client credentials"
+      )
+    })
+
+    it("handles wrong secrets", async () => {
+      await expect(
+        config.doInContext(config.appId, async () => {
+          const oauthConfig = await sdk.oauth2.create({
+            name: generator.guid(),
+            url: `${keycloakUrl}/realms/myrealm/protocol/openid-connect/token`,
+            clientId: "my-client",
+            clientSecret: "wrong-secret",
+          })
+
+          await generateToken(oauthConfig.id)
+        })
+      ).rejects.toThrow(
+        "Error fetching oauth2 token: Invalid client or Invalid client credentials"
+      )
+    })
+  })
+})

packages/server/src/sdk/app/oauth2/utils.ts

@@ -0,0 +1,33 @@
+import fetch from "node-fetch"
+import { HttpError } from "koa"
+import { get } from "../oauth2"
+
+// TODO: check if caching is worth
+export async function generateToken(id: string) {
+  const config = await get(id)
+  if (!config) {
+    throw new HttpError(`oAuth config ${id} count not be found`)
+  }
+
+  const resp = await fetch(config.url, {
+    method: "POST",
+    headers: {
+      "Content-Type": "application/x-www-form-urlencoded",
+    },
+    body: new URLSearchParams({
+      grant_type: "client_credentials",
+      client_id: config.clientId,
+      client_secret: config.clientSecret,
+    }),
+    redirect: "follow",
+  })
+
+  const jsonResponse = await resp.json()
+  if (!resp.ok) {
+    const message = jsonResponse.error_description ?? resp.statusText
+
+    throw new Error(`Error fetching oauth2 token: ${message}`)
+  }
+
+  return `${jsonResponse.token_type} ${jsonResponse.access_token}`
+}

packages/server/src/tests/utilities/api/oauth2.ts

@@ -1,5 +1,6 @@
 import {
   CreateOAuth2ConfigRequest,
+  CreateOAuth2ConfigResponse,
   FetchOAuth2ConfigsResponse,
 } from "@budibase/types"
 import { Expectations, TestAPI } from "./base"
@@ -15,9 +16,12 @@ export class OAuth2API extends TestAPI {
     body: CreateOAuth2ConfigRequest,
     expectations?: Expectations
   ) => {
-    return await this._post<CreateOAuth2ConfigRequest>("/api/oauth2", {
+    return await this._post<CreateOAuth2ConfigResponse>("/api/oauth2", {
       body,
-      expectations,
+      expectations: {
+        status: expectations?.status ?? 201,
+        ...expectations,
+      },
     })
   }
 }

packages/types/src/api/web/app/oauth2.ts

@@ -1,9 +1,19 @@
-interface OAuth2Config {
+interface OAuth2ConfigResponse {
+  id: string
   name: string
 }
 
 export interface FetchOAuth2ConfigsResponse {
-  configs: OAuth2Config[]
+  configs: OAuth2ConfigResponse[]
 }
 
-export interface CreateOAuth2ConfigRequest extends OAuth2Config {}
+export interface CreateOAuth2ConfigRequest {
+  name: string
+  url: string
+  clientId: string
+  clientSecret: string
+}
+
+export interface CreateOAuth2ConfigResponse {
+  config: OAuth2ConfigResponse
+}

packages/types/src/documents/app/datasource.ts

@@ -19,6 +19,7 @@ export interface Datasource extends Document {
 export enum RestAuthType {
   BASIC = "basic",
   BEARER = "bearer",
+  OAUTH2 = "oauth2",
 }
 
 export interface RestBasicAuthConfig {
@@ -30,13 +31,22 @@ export interface RestBearerAuthConfig {
   token: string
 }
 
-export interface RestAuthConfig {
+export interface BasicRestAuthConfig {
   _id: string
   name: string
-  type: RestAuthType
-  config: RestBasicAuthConfig | RestBearerAuthConfig
+  type: RestAuthType.BASIC
+  config: RestBasicAuthConfig
 }
 
+export interface BearerRestAuthConfig {
+  _id: string
+  name: string
+  type: RestAuthType.BEARER
+  config: RestBearerAuthConfig
+}
+
+export type RestAuthConfig = BasicRestAuthConfig | BearerRestAuthConfig
+
 export interface DynamicVariable {
   name: string
   queryId: string

packages/types/src/documents/app/oauth2.ts

@@ -1,7 +1,11 @@
 import { Document } from "../document"
 
 export interface OAuth2Config {
+  id: string
   name: string
+  url: string
+  clientId: string
+  clientSecret: string
 }
 
 export interface OAuth2Configs extends Document {

packages/types/src/documents/app/query.ts

@@ -1,4 +1,5 @@
 import { Document } from "../document"
+import { RestAuthType } from "./datasource"
 import { Row } from "./row"
 
 export interface QuerySchema {
@@ -56,6 +57,7 @@ export interface RestQueryFields {
   bodyType?: BodyType
   method?: string
   authConfigId?: string
+  authConfigType?: RestAuthType
   pagination?: PaginationConfig
   paginationValues?: PaginationValues
 }

packages/types/src/documents/document.ts

@@ -82,6 +82,7 @@ export enum InternalTable {
 export enum VirtualDocumentType {
   VIEW = "view",
   ROW_ACTION = "row_action",
+  OAUTH2_CONFIG = "oauth2",
 }
 
 // Because VirtualDocumentTypes can overlap, we need to make sure that we search