budibase/packages/server/src/integrations/rest.ts

import {
  BodyType,
  DatasourceFieldType,
  HttpMethod,
  Integration,
  IntegrationBase,
  PaginationConfig,
  PaginationValues,
  QueryType,
  RestAuthType,
  RestBasicAuthConfig,
  RestBearerAuthConfig,
  RestConfig,
  RestQueryFields as RestQuery,
} from "@budibase/types"
import get from "lodash/get"
import * as https from "https"
import qs from "querystring"
import type { Response, RequestInit } from "node-fetch"
import fetch from "node-fetch"
import { formatBytes } from "../utilities"
import { performance } from "perf_hooks"
import FormData from "form-data"
import { URLSearchParams } from "url"
import { blacklist } from "@budibase/backend-core"
import { handleFileResponse, handleXml } from "./utils"
import { parse } from "content-disposition"
import path from "path"
import { Builder as XmlBuilder } from "xml2js"
import { getAttachmentHeaders } from "./utils/restUtils"

const coreFields = {
  path: {
    type: DatasourceFieldType.STRING,
    display: "URL",
  },
  queryString: {
    type: DatasourceFieldType.STRING,
  },
  headers: {
    type: DatasourceFieldType.OBJECT,
  },
  enabledHeaders: {
    type: DatasourceFieldType.OBJECT,
  },
  requestBody: {
    type: DatasourceFieldType.JSON,
  },
  bodyType: {
    type: DatasourceFieldType.STRING,
    enum: Object.values(BodyType),
  },
  pagination: {
    type: DatasourceFieldType.OBJECT,
  },
}

const SCHEMA: Integration = {
  docs: "https://github.com/node-fetch/node-fetch",
  description:
    "With the REST API datasource, you can connect, query and pull data from multiple REST APIs. You can then use the retrieved data to build apps.",
  friendlyName: "REST API",
  type: "API",
  datasource: {
    url: {
      type: DatasourceFieldType.STRING,
      default: "",
      required: false,
      deprecated: true,
    },
    defaultHeaders: {
      type: DatasourceFieldType.OBJECT,
      required: false,
      default: {},
    },
    rejectUnauthorized: {
      display: "Reject Unauthorized",
      type: DatasourceFieldType.BOOLEAN,
      default: true,
      required: false,
    },
    downloadImages: {
      display: "Download images",
      type: DatasourceFieldType.BOOLEAN,
      default: true,
      required: false,
    },
  },
  query: {
    create: {
      readable: true,
      displayName: HttpMethod.POST,
      type: QueryType.FIELDS,
      fields: coreFields,
    },
    read: {
      displayName: HttpMethod.GET,
      readable: true,
      type: QueryType.FIELDS,
      fields: coreFields,
    },
    update: {
      displayName: HttpMethod.PUT,
      readable: true,
      type: QueryType.FIELDS,
      fields: coreFields,
    },
    patch: {
      displayName: HttpMethod.PATCH,
      readable: true,
      type: QueryType.FIELDS,
      fields: coreFields,
    },
    delete: {
      displayName: HttpMethod.DELETE,
      type: QueryType.FIELDS,
      fields: coreFields,
    },
  },
}

interface ParsedResponse {
  data: any
  info: {
    code: number
    size: string
    time: string
  }
  extra?: {
    raw: string | undefined
    headers: Record<string, string[] | string>
  }
  pagination?: {
    cursor: any
  }
}

export class RestIntegration implements IntegrationBase {
  private config: RestConfig
  private headers: {
    [key: string]: string
  } = {}
  private startTimeMs: number = performance.now()

  constructor(config: RestConfig) {
    this.config = config
  }

  async parseResponse(
    response: Response,
    pagination?: PaginationConfig
  ): Promise<ParsedResponse> {
    let data: any[] | string | undefined,
      raw: string | undefined,
      headers: Record<string, string[] | string> = {},
      filename: string | undefined

    const { contentType, contentDisposition } = getAttachmentHeaders(
      response.headers,
      { downloadImages: this.config.downloadImages }
    )
    let contentLength = response.headers.get("content-length")
    let isSuccess = response.status >= 200 && response.status < 300
    if (
      (contentDisposition.includes("filename") ||
        contentDisposition.includes("attachment") ||
        contentDisposition.includes("form-data")) &&
      isSuccess
    ) {
      filename =
        path.basename(parse(contentDisposition).parameters?.filename) || ""
    }

    let triedParsing: boolean = false,
      responseTxt: string | undefined
    try {
      if (filename) {
        return handleFileResponse(response, filename, this.startTimeMs)
      } else {
        responseTxt = response.text ? await response.text() : ""
        if (!contentLength && responseTxt) {
          contentLength = Buffer.byteLength(responseTxt, "utf8").toString()
        }
        const hasContent =
          (contentLength && parseInt(contentLength) > 0) ||
          responseTxt.length > 0
        if (response.status === 204) {
          data = []
          raw = ""
        } else if (hasContent && contentType.includes("application/json")) {
          triedParsing = true
          data = JSON.parse(responseTxt)
          raw = responseTxt
        } else if (
          (hasContent && contentType.includes("text/xml")) ||
          contentType.includes("application/xml")
        ) {
          triedParsing = true
          let xmlResponse = await handleXml(responseTxt)
          data = xmlResponse.data
          raw = xmlResponse.rawXml
        } else {
          data = responseTxt
          raw = data as string
        }
      }
    } catch (err) {
      if (triedParsing) {
        data = responseTxt
        raw = data as string
      } else {
        throw new Error(`Failed to parse response body: ${err}`)
      }
    }

    const size = formatBytes(contentLength || "0")
    const time = `${Math.round(performance.now() - this.startTimeMs)}ms`
    headers = response.headers.raw()
    for (let [key, value] of Object.entries(headers)) {
      headers[key] = Array.isArray(value) ? value[0] : value
    }

    // Check if a pagination cursor exists in the response
    let nextCursor = null
    if (pagination?.responseParam) {
      nextCursor = get(data, pagination.responseParam)
    }

    return {
      data,
      info: {
        code: response.status,
        size,
        time,
      },
      extra: {
        raw,
        headers,
      },
      pagination: {
        cursor: nextCursor,
      },
    }
  }

  getUrl(
    path: string,
    queryString: string,
    pagination?: PaginationConfig,
    paginationValues?: PaginationValues
  ): string {
    // Add pagination params to query string if required
    if (pagination?.location === "query" && paginationValues) {
      const { pageParam, sizeParam } = pagination
      const params = new URLSearchParams()

      // Append page number or cursor param if configured
      if (pageParam && paginationValues.page != null) {
        params.append(pageParam, paginationValues.page as string)
      }

      // Append page size param if configured
      if (sizeParam && paginationValues.limit != null) {
        params.append(sizeParam, String(paginationValues.limit))
      }

      // Prepend query string with pagination params
      let paginationString = params.toString()
      if (paginationString) {
        queryString = `${paginationString}&${queryString}`
      }
    }

    if (queryString) {
      // make sure the query string is fully encoded
      queryString = "?" + qs.encode(qs.decode(queryString))
    }
    const main = `${path}${queryString}`

    let complete = main
    if (this.config.url && !main.startsWith("http")) {
      complete = !this.config.url ? main : `${this.config.url}/${main}`
    }
    if (!complete.startsWith("http")) {
      complete = `http://${complete}`
    }
    return complete
  }

  addBody(
    bodyType: string,
    body: string | any,
    input: RequestInit,
    pagination?: PaginationConfig,
    paginationValues?: PaginationValues
  ): RequestInit {
    if (!input.headers) {
      input.headers = {}
    }
    if (bodyType === BodyType.NONE) {
      return input
    }
    let error,
      object: any = {},
      string = ""
    try {
      if (body) {
        string = typeof body !== "string" ? JSON.stringify(body) : body
        object = typeof body === "object" ? body : JSON.parse(body)
      }
    } catch (err) {
      error = err
    }

    // Util to add pagination values to a certain body type
    const addPaginationToBody = (insertFn: Function) => {
      if (pagination?.location === "body") {
        if (pagination?.pageParam && paginationValues?.page != null) {
          insertFn(pagination.pageParam, paginationValues.page)
        }
        if (pagination?.sizeParam && paginationValues?.limit != null) {
          insertFn(pagination.sizeParam, paginationValues.limit)
        }
      }
    }

    switch (bodyType) {
      case BodyType.TEXT:
        // content type defaults to plaintext
        input.body = string
        break
      case BodyType.ENCODED: {
        const params = new URLSearchParams()
        for (let [key, value] of Object.entries(object)) {
          params.append(key, value as string)
        }
        addPaginationToBody((key: string, value: any) => {
          params.append(key, value)
        })
        input.body = params
        break
      }
      case BodyType.FORM_DATA: {
        const form = new FormData()
        for (let [key, value] of Object.entries(object)) {
          form.append(key, value)
        }
        addPaginationToBody((key: string, value: any) => {
          form.append(key, value)
        })
        input.body = form
        break
      }
      case BodyType.XML:
        if (object != null && Object.keys(object).length) {
          string = new XmlBuilder().buildObject(object)
        }
        input.body = string
        // @ts-ignore
        input.headers["Content-Type"] = "application/xml"
        break
      case BodyType.JSON:
        // if JSON error, throw it
        if (error) {
          throw "Invalid JSON for request body"
        }
        addPaginationToBody((key: string, value: any) => {
          object[key] = value
        })
        input.body = JSON.stringify(object)
        // @ts-ignore
        input.headers["Content-Type"] = "application/json"
        break
    }
    return input
  }

  getAuthHeaders(authConfigId?: string): { [key: string]: any } {
    let headers: any = {}

    if (this.config.authConfigs && authConfigId) {
      const authConfig = this.config.authConfigs.filter(
        c => c._id === authConfigId
      )[0]
      // check the config still exists before proceeding
      // if not - do nothing
      if (authConfig) {
        let config
        switch (authConfig.type) {
          case RestAuthType.BASIC:
            config = authConfig.config as RestBasicAuthConfig
            headers.Authorization = `Basic ${Buffer.from(
              `${config.username}:${config.password}`
            ).toString("base64")}`
            break
          case RestAuthType.BEARER:
            config = authConfig.config as RestBearerAuthConfig
            headers.Authorization = `Bearer ${config.token}`
            break
        }
      }
    }

    return headers
  }

  async _req(query: RestQuery) {
    const {
      path = "",
      queryString = "",
      headers = {},
      method = HttpMethod.GET,
      disabledHeaders,
      bodyType = BodyType.NONE,
      requestBody,
      authConfigId,
      pagination,
      paginationValues,
    } = query
    const authHeaders = this.getAuthHeaders(authConfigId)

    this.headers = {
      ...(this.config.defaultHeaders || {}),
      ...headers,
      ...authHeaders,
    }

    if (disabledHeaders) {
      for (let headerKey of Object.keys(this.headers)) {
        if (disabledHeaders[headerKey]) {
          delete this.headers[headerKey]
        }
      }
    }

    let input: RequestInit = { method, headers: this.headers }
    input = this.addBody(
      bodyType,
      requestBody,
      input,
      pagination,
      paginationValues
    )

    if (this.config.rejectUnauthorized == false) {
      input.agent = new https.Agent({
        rejectUnauthorized: false,
      })
    }

    // Deprecated by rejectUnauthorized
    if (this.config.legacyHttpParser) {
      // NOTE(samwho): it seems like this code doesn't actually work because it requires
      // node-fetch >=3, and we're not on that because upgrading to it produces errors to
      // do with ESM that are above my pay grade.

      // https://github.com/nodejs/node/issues/43798
      // @ts-ignore
      input.extraHttpOptions = { insecureHTTPParser: true }
    }

    this.startTimeMs = performance.now()
    const url = this.getUrl(path, queryString, pagination, paginationValues)
    if (await blacklist.isBlacklisted(url)) {
      throw new Error("Cannot connect to URL.")
    }
    const response = await fetch(url, input)
    return await this.parseResponse(response, pagination)
  }

  async create(opts: RestQuery) {
    return this._req({ ...opts, method: HttpMethod.POST })
  }

  async read(opts: RestQuery) {
    return this._req({ ...opts, method: HttpMethod.GET })
  }

  async update(opts: RestQuery) {
    return this._req({ ...opts, method: HttpMethod.PUT })
  }

  async patch(opts: RestQuery) {
    return this._req({ ...opts, method: HttpMethod.PATCH })
  }

  async delete(opts: RestQuery) {
    return this._req({ ...opts, method: HttpMethod.DELETE })
  }
}

export default {
  schema: SCHEMA,
  integration: RestIntegration,
}