Merge branch 'master' into table-width-setting

packages/backend-core/src/context/types.ts

@@ -1,4 +1,6 @@
 import { IdentityContext, Snippet, VM } from "@budibase/types"
+import { OAuth2Client } from "google-auth-library"
+import { GoogleSpreadsheet } from "google-spreadsheet"
 
 // keep this out of Budibase types, don't want to expose context info
 export type ContextMap = {
@@ -12,4 +14,8 @@ export type ContextMap = {
   vm?: VM
   cleanup?: (() => void | Promise<void>)[]
   snippets?: Snippet[]
+  googleSheets?: {
+    oauthClient: OAuth2Client
+    clients: Record<string, GoogleSpreadsheet>
+  }
 }

packages/builder/src/components/backend/Datasources/ConfigEditor/stores/validatedConfig.js

@@ -86,8 +86,9 @@ export const createValidatedConfigStore = (integration, config) => {
     ([$configStore, $errorsStore, $selectedValidatorsStore]) => {
       const validatedConfig = []
 
+      const allowedRestKeys = ["rejectUnauthorized", "downloadImages"]
       Object.entries(integration.datasource).forEach(([key, properties]) => {
-        if (integration.name === "REST" && key !== "rejectUnauthorized") {
+        if (integration.name === "REST" && !allowedRestKeys.includes(key)) {
           return
         }
 

packages/server/package.json

@@ -79,7 +79,7 @@
     "dotenv": "8.2.0",
     "form-data": "4.0.0",
     "global-agent": "3.0.0",
-    "google-spreadsheet": "4.1.2",
+    "google-spreadsheet": "npm:@budibase/google-spreadsheet@4.1.2",
     "ioredis": "5.3.2",
     "isolated-vm": "^4.7.2",
     "jimp": "0.22.10",

packages/server/src/integrations/googlesheets.ts

@@ -228,32 +228,53 @@ class GoogleSheetsIntegration implements DatasourcePlus {
 
   private async connect() {
     try {
-      await setupCreationAuth(this.config)
+      const bbCtx = context.getCurrentContext()
+      let oauthClient = bbCtx?.googleSheets?.oauthClient
 
-      // Initialise oAuth client
+      if (!oauthClient) {
-      const googleConfig = await configs.getGoogleDatasourceConfig()
+        await setupCreationAuth(this.config)
-      if (!googleConfig) {
+
-        throw new HTTPError("Google config not found", 400)
+        // Initialise oAuth client
+        const googleConfig = await configs.getGoogleDatasourceConfig()
+        if (!googleConfig) {
+          throw new HTTPError("Google config not found", 400)
+        }
+
+        oauthClient = new OAuth2Client({
+          clientId: googleConfig.clientID,
+          clientSecret: googleConfig.clientSecret,
+        })
+
+        const tokenResponse = await this.fetchAccessToken({
+          client_id: googleConfig.clientID,
+          client_secret: googleConfig.clientSecret,
+          refresh_token: this.config.auth.refreshToken,
+        })
+
+        oauthClient.setCredentials({
+          refresh_token: this.config.auth.refreshToken,
+          access_token: tokenResponse.access_token,
+        })
+        if (bbCtx && !bbCtx.googleSheets) {
+          bbCtx.googleSheets = {
+            oauthClient,
+            clients: {},
+          }
+          bbCtx.cleanup = bbCtx.cleanup || []
+        }
       }
 
-      const oauthClient = new OAuth2Client({
+      let client = bbCtx?.googleSheets?.clients[this.spreadsheetId]
-        clientId: googleConfig.clientID,
+      if (!client) {
-        clientSecret: googleConfig.clientSecret,
+        client = new GoogleSpreadsheet(this.spreadsheetId, oauthClient)
-      })
+        await client.loadInfo()
 
-      const tokenResponse = await this.fetchAccessToken({
+        if (bbCtx?.googleSheets?.clients) {
-        client_id: googleConfig.clientID,
+          bbCtx.googleSheets.clients[this.spreadsheetId] = client
-        client_secret: googleConfig.clientSecret,
+        }
-        refresh_token: this.config.auth.refreshToken,
+      }
-      })
 
-      oauthClient.setCredentials({
+      this.client = client
-        refresh_token: this.config.auth.refreshToken,
-        access_token: tokenResponse.access_token,
-      })
-
-      this.client = new GoogleSpreadsheet(this.spreadsheetId, oauthClient)
-      await this.client.loadInfo()
     } catch (err: any) {
       // this happens for xlsx imports
       if (err.message?.includes("operation is not supported")) {

packages/server/src/integrations/rest.ts

@@ -1,22 +1,22 @@
 import {
-  Integration,
   DatasourceFieldType,
-  QueryType,
+  HttpMethod,
-  PaginationConfig,
+  Integration,
   IntegrationBase,
+  PaginationConfig,
   PaginationValues,
-  RestQueryFields as RestQuery,
+  QueryType,
-  RestConfig,
   RestAuthType,
   RestBasicAuthConfig,
   RestBearerAuthConfig,
-  HttpMethod,
+  RestConfig,
+  RestQueryFields as RestQuery,
 } from "@budibase/types"
 import get from "lodash/get"
 import * as https from "https"
 import qs from "querystring"
-import fetch from "node-fetch"
 import type { Response } from "node-fetch"
+import fetch from "node-fetch"
 import { formatBytes } from "../utilities"
 import { performance } from "perf_hooks"
 import FormData from "form-data"
@@ -87,6 +87,12 @@ const SCHEMA: Integration = {
       default: true,
       required: false,
     },
+    downloadImages: {
+      display: "Download images",
+      type: DatasourceFieldType.BOOLEAN,
+      default: true,
+      required: false,
+    },
   },
   query: {
     create: {
@@ -139,7 +145,8 @@ class RestIntegration implements IntegrationBase {
       filename: string | undefined
 
     const { contentType, contentDisposition } = getAttachmentHeaders(
-      response.headers
+      response.headers,
+      { downloadImages: this.config.downloadImages }
     )
     if (
       contentDisposition.includes("filename") ||

packages/server/src/integrations/tests/restUtils.spec.ts

@@ -1,13 +1,13 @@
 import { getAttachmentHeaders } from "../utils/restUtils"
 import type { Headers } from "node-fetch"
 
-function headers(dispositionValue: string) {
+function headers(dispositionValue: string, contentType?: string) {
   return {
     get: (name: string) => {
       if (name.toLowerCase() === "content-disposition") {
         return dispositionValue
       } else {
-        return "application/pdf"
+        return contentType || "application/pdf"
       }
     },
     set: () => {},
@@ -35,4 +35,14 @@ describe("getAttachmentHeaders", () => {
     )
     expect(contentDisposition).toBe(`inline; filename="report.pdf"`)
   })
+
+  it("should handle an image", () => {
+    const { contentDisposition } = getAttachmentHeaders(
+      headers("", "image/png"),
+      {
+        downloadImages: true,
+      }
+    )
+    expect(contentDisposition).toBe(`attachment; filename="image.png"`)
+  })
 })

packages/server/src/integrations/utils/restUtils.ts

@@ -1,6 +1,9 @@
 import type { Headers } from "node-fetch"
 
-export function getAttachmentHeaders(headers: Headers) {
+export function getAttachmentHeaders(
+  headers: Headers,
+  opts?: { downloadImages?: boolean }
+) {
   const contentType = headers.get("content-type") || ""
   let contentDisposition = headers.get("content-disposition") || ""
 
@@ -23,6 +26,15 @@ export function getAttachmentHeaders(headers: Headers) {
       }
     }
   }
+  // for images which don't supply a content disposition, make one up, as binary
+  // data for images in REST responses isn't really useful, we should always download them
+  else if (opts?.downloadImages && contentType.startsWith("image/")) {
+    const format = contentType.split("/")[1]
+    return {
+      contentDisposition: `attachment; filename="image.${format}"`,
+      contentType,
+    }
+  }
 
   return { contentDisposition, contentType }
 }

packages/types/src/documents/app/datasource.ts

@@ -46,6 +46,7 @@ export interface DynamicVariable {
 export interface RestConfig {
   url: string
   rejectUnauthorized: boolean
+  downloadImages?: boolean
   defaultHeaders: {
     [key: string]: any
   }

yarn.lock

@@ -11540,10 +11540,10 @@ google-p12-pem@^4.0.0:
   dependencies:
     node-forge "^1.3.1"
 
-google-spreadsheet@4.1.2:
+"google-spreadsheet@npm:@budibase/google-spreadsheet@4.1.2":
   version "4.1.2"
-  resolved "https://registry.yarnpkg.com/google-spreadsheet/-/google-spreadsheet-4.1.2.tgz#92e30fdba7e0d78c55d50731528df7835d58bfee"
+  resolved "https://registry.yarnpkg.com/@budibase/google-spreadsheet/-/google-spreadsheet-4.1.2.tgz#90548ccba2284b3042b08d2974ef3caeaf772ad9"
-  integrity sha512-HFBweDAkOcyC2qO9kmWESKbNuOcn+R7UzZN/tj5LLNxVv8FHmg113u0Ow+yaKwwIOt/NnDtPLuptAhaxTs0FYw==
+  integrity sha512-dxoY3rQGGnuNeZiXhNc9oYPduzU8xnIjWujFwNvaRRv3zWeUV7mj6HE2o/OJOeekPGt7o44B+w6DfkiaoteZgg==
   dependencies:
     axios "^1.4.0"
     lodash "^4.17.21"