410 lines
11 KiB
TypeScript
410 lines
11 KiB
TypeScript
import {
|
|
Integration,
|
|
DatasourceFieldTypes,
|
|
QueryTypes,
|
|
RestConfig,
|
|
RestQueryFields as RestQuery,
|
|
PaginationConfig,
|
|
AuthType,
|
|
BasicAuthConfig,
|
|
BearerAuthConfig,
|
|
PaginationValues,
|
|
} from "../definitions/datasource"
|
|
import { IntegrationBase } from "./base/IntegrationBase"
|
|
import { get } from "lodash"
|
|
|
|
const BodyTypes = {
|
|
NONE: "none",
|
|
FORM_DATA: "form",
|
|
XML: "xml",
|
|
ENCODED: "encoded",
|
|
JSON: "json",
|
|
TEXT: "text",
|
|
}
|
|
|
|
const coreFields = {
|
|
path: {
|
|
type: DatasourceFieldTypes.STRING,
|
|
display: "URL",
|
|
},
|
|
queryString: {
|
|
type: DatasourceFieldTypes.STRING,
|
|
},
|
|
headers: {
|
|
type: DatasourceFieldTypes.OBJECT,
|
|
},
|
|
enabledHeaders: {
|
|
type: DatasourceFieldTypes.OBJECT,
|
|
},
|
|
requestBody: {
|
|
type: DatasourceFieldTypes.JSON,
|
|
},
|
|
bodyType: {
|
|
type: DatasourceFieldTypes.STRING,
|
|
enum: Object.values(BodyTypes),
|
|
},
|
|
pagination: {
|
|
type: DatasourceFieldTypes.OBJECT,
|
|
},
|
|
}
|
|
|
|
module RestModule {
|
|
const fetch = require("node-fetch")
|
|
const { formatBytes } = require("../utilities")
|
|
const { performance } = require("perf_hooks")
|
|
const FormData = require("form-data")
|
|
const { URLSearchParams } = require("url")
|
|
const {
|
|
parseStringPromise: xmlParser,
|
|
Builder: XmlBuilder,
|
|
} = require("xml2js")
|
|
|
|
const SCHEMA: Integration = {
|
|
docs: "https://github.com/node-fetch/node-fetch",
|
|
description:
|
|
"With the REST API datasource, you can connect, query and pull data from multiple REST APIs. You can then use the retrieved data to build apps.",
|
|
friendlyName: "REST API",
|
|
datasource: {
|
|
url: {
|
|
type: DatasourceFieldTypes.STRING,
|
|
default: "",
|
|
required: false,
|
|
deprecated: true,
|
|
},
|
|
defaultHeaders: {
|
|
type: DatasourceFieldTypes.OBJECT,
|
|
required: false,
|
|
default: {},
|
|
},
|
|
},
|
|
query: {
|
|
create: {
|
|
readable: true,
|
|
displayName: "POST",
|
|
type: QueryTypes.FIELDS,
|
|
fields: coreFields,
|
|
},
|
|
read: {
|
|
displayName: "GET",
|
|
readable: true,
|
|
type: QueryTypes.FIELDS,
|
|
fields: coreFields,
|
|
},
|
|
update: {
|
|
displayName: "PUT",
|
|
readable: true,
|
|
type: QueryTypes.FIELDS,
|
|
fields: coreFields,
|
|
},
|
|
patch: {
|
|
displayName: "PATCH",
|
|
readable: true,
|
|
type: QueryTypes.FIELDS,
|
|
fields: coreFields,
|
|
},
|
|
delete: {
|
|
displayName: "DELETE",
|
|
type: QueryTypes.FIELDS,
|
|
fields: coreFields,
|
|
},
|
|
},
|
|
}
|
|
|
|
class RestIntegration implements IntegrationBase {
|
|
private config: RestConfig
|
|
private headers: {
|
|
[key: string]: string
|
|
} = {}
|
|
private startTimeMs: number = performance.now()
|
|
|
|
constructor(config: RestConfig) {
|
|
this.config = config
|
|
}
|
|
|
|
async parseResponse(response: any, pagination: PaginationConfig | null) {
|
|
let data, raw, headers
|
|
const contentType = response.headers.get("content-type") || ""
|
|
try {
|
|
if (contentType.includes("application/json")) {
|
|
data = await response.json()
|
|
raw = JSON.stringify(data)
|
|
} else if (
|
|
contentType.includes("text/xml") ||
|
|
contentType.includes("application/xml")
|
|
) {
|
|
const rawXml = await response.text()
|
|
data =
|
|
(await xmlParser(rawXml, {
|
|
explicitArray: false,
|
|
trim: true,
|
|
explicitRoot: false,
|
|
})) || {}
|
|
// there is only one structure, its an array, return the array so it appears as rows
|
|
const keys = Object.keys(data)
|
|
if (keys.length === 1 && Array.isArray(data[keys[0]])) {
|
|
data = data[keys[0]]
|
|
}
|
|
raw = rawXml
|
|
} else {
|
|
data = await response.text()
|
|
raw = data
|
|
}
|
|
} catch (err) {
|
|
throw "Failed to parse response body."
|
|
}
|
|
const size = formatBytes(
|
|
response.headers.get("content-length") || Buffer.byteLength(raw, "utf8")
|
|
)
|
|
const time = `${Math.round(performance.now() - this.startTimeMs)}ms`
|
|
headers = response.headers.raw()
|
|
for (let [key, value] of Object.entries(headers)) {
|
|
headers[key] = Array.isArray(value) ? value[0] : value
|
|
}
|
|
|
|
// Check if a pagination cursor exists in the response
|
|
let nextCursor = null
|
|
if (pagination?.responseParam) {
|
|
nextCursor = get(data, pagination.responseParam)
|
|
}
|
|
|
|
return {
|
|
data,
|
|
info: {
|
|
code: response.status,
|
|
size,
|
|
time,
|
|
},
|
|
extra: {
|
|
raw,
|
|
headers,
|
|
},
|
|
pagination: {
|
|
cursor: nextCursor,
|
|
},
|
|
}
|
|
}
|
|
|
|
getUrl(
|
|
path: string,
|
|
queryString: string,
|
|
pagination: PaginationConfig | null,
|
|
paginationValues: PaginationValues | null
|
|
): string {
|
|
// Add pagination params to query string if required
|
|
if (pagination?.location === "query" && paginationValues) {
|
|
const { pageParam, sizeParam } = pagination
|
|
const params = new URLSearchParams()
|
|
|
|
// Append page number or cursor param if configured
|
|
if (pageParam && paginationValues.page != null) {
|
|
params.append(pageParam, paginationValues.page)
|
|
}
|
|
|
|
// Append page size param if configured
|
|
if (sizeParam && paginationValues.limit != null) {
|
|
params.append(sizeParam, paginationValues.limit)
|
|
}
|
|
|
|
// Prepend query string with pagination params
|
|
let paginationString = params.toString()
|
|
if (paginationString) {
|
|
queryString = `${paginationString}&${queryString}`
|
|
}
|
|
}
|
|
|
|
const main = `${path}?${queryString}`
|
|
let complete = main
|
|
if (this.config.url && !main.startsWith("http")) {
|
|
complete = !this.config.url ? main : `${this.config.url}/${main}`
|
|
}
|
|
if (!complete.startsWith("http")) {
|
|
complete = `http://${complete}`
|
|
}
|
|
return complete
|
|
}
|
|
|
|
addBody(
|
|
bodyType: string,
|
|
body: string | any,
|
|
input: any,
|
|
pagination: PaginationConfig | null,
|
|
paginationValues: PaginationValues | null
|
|
) {
|
|
if (!input.headers) {
|
|
input.headers = {}
|
|
}
|
|
if (bodyType === BodyTypes.NONE) {
|
|
return input
|
|
}
|
|
let error,
|
|
object: any = {},
|
|
string = ""
|
|
try {
|
|
if (body) {
|
|
string = typeof body !== "string" ? JSON.stringify(body) : body
|
|
object = typeof body === "object" ? body : JSON.parse(body)
|
|
}
|
|
} catch (err) {
|
|
error = err
|
|
}
|
|
|
|
// Util to add pagination values to a certain body type
|
|
const addPaginationToBody = (insertFn: Function) => {
|
|
if (pagination?.location === "body") {
|
|
if (pagination?.pageParam && paginationValues?.page != null) {
|
|
insertFn(pagination.pageParam, paginationValues.page)
|
|
}
|
|
if (pagination?.sizeParam && paginationValues?.limit != null) {
|
|
insertFn(pagination.sizeParam, paginationValues.limit)
|
|
}
|
|
}
|
|
}
|
|
|
|
switch (bodyType) {
|
|
case BodyTypes.TEXT:
|
|
// content type defaults to plaintext
|
|
input.body = string
|
|
break
|
|
case BodyTypes.ENCODED:
|
|
const params = new URLSearchParams()
|
|
for (let [key, value] of Object.entries(object)) {
|
|
params.append(key, value)
|
|
}
|
|
addPaginationToBody((key: string, value: any) => {
|
|
params.append(key, value)
|
|
})
|
|
input.body = params
|
|
break
|
|
case BodyTypes.FORM_DATA:
|
|
const form = new FormData()
|
|
for (let [key, value] of Object.entries(object)) {
|
|
form.append(key, value)
|
|
}
|
|
addPaginationToBody((key: string, value: any) => {
|
|
form.append(key, value)
|
|
})
|
|
input.body = form
|
|
break
|
|
case BodyTypes.XML:
|
|
if (object != null) {
|
|
string = new XmlBuilder().buildObject(object)
|
|
}
|
|
input.body = string
|
|
input.headers["Content-Type"] = "application/xml"
|
|
break
|
|
case BodyTypes.JSON:
|
|
// if JSON error, throw it
|
|
if (error) {
|
|
throw "Invalid JSON for request body"
|
|
}
|
|
addPaginationToBody((key: string, value: any) => {
|
|
object[key] = value
|
|
})
|
|
input.body = JSON.stringify(object)
|
|
input.headers["Content-Type"] = "application/json"
|
|
break
|
|
}
|
|
return input
|
|
}
|
|
|
|
getAuthHeaders(authConfigId: string): { [key: string]: any } {
|
|
let headers: any = {}
|
|
|
|
if (this.config.authConfigs && authConfigId) {
|
|
const authConfig = this.config.authConfigs.filter(
|
|
c => c._id === authConfigId
|
|
)[0]
|
|
// check the config still exists before proceeding
|
|
// if not - do nothing
|
|
if (authConfig) {
|
|
let config
|
|
switch (authConfig.type) {
|
|
case AuthType.BASIC:
|
|
config = authConfig.config as BasicAuthConfig
|
|
headers.Authorization = `Basic ${Buffer.from(
|
|
`${config.username}:${config.password}`
|
|
).toString("base64")}`
|
|
break
|
|
case AuthType.BEARER:
|
|
config = authConfig.config as BearerAuthConfig
|
|
headers.Authorization = `Bearer ${config.token}`
|
|
break
|
|
}
|
|
}
|
|
}
|
|
|
|
return headers
|
|
}
|
|
|
|
async _req(query: RestQuery) {
|
|
const {
|
|
path = "",
|
|
queryString = "",
|
|
headers = {},
|
|
method = "GET",
|
|
disabledHeaders,
|
|
bodyType,
|
|
requestBody,
|
|
authConfigId,
|
|
pagination,
|
|
paginationValues,
|
|
} = query
|
|
const authHeaders = this.getAuthHeaders(authConfigId)
|
|
|
|
this.headers = {
|
|
...this.config.defaultHeaders,
|
|
...headers,
|
|
...authHeaders,
|
|
}
|
|
|
|
if (disabledHeaders) {
|
|
for (let headerKey of Object.keys(this.headers)) {
|
|
if (disabledHeaders[headerKey]) {
|
|
delete this.headers[headerKey]
|
|
}
|
|
}
|
|
}
|
|
|
|
let input: any = { method, headers: this.headers }
|
|
input = this.addBody(
|
|
bodyType,
|
|
requestBody,
|
|
input,
|
|
pagination,
|
|
paginationValues
|
|
)
|
|
|
|
this.startTimeMs = performance.now()
|
|
const url = this.getUrl(path, queryString, pagination, paginationValues)
|
|
const response = await fetch(url, input)
|
|
return await this.parseResponse(response, pagination)
|
|
}
|
|
|
|
async create(opts: RestQuery) {
|
|
return this._req({ ...opts, method: "POST" })
|
|
}
|
|
|
|
async read(opts: RestQuery) {
|
|
return this._req({ ...opts, method: "GET" })
|
|
}
|
|
|
|
async update(opts: RestQuery) {
|
|
return this._req({ ...opts, method: "PUT" })
|
|
}
|
|
|
|
async patch(opts: RestQuery) {
|
|
return this._req({ ...opts, method: "PATCH" })
|
|
}
|
|
|
|
async delete(opts: RestQuery) {
|
|
return this._req({ ...opts, method: "DELETE" })
|
|
}
|
|
}
|
|
|
|
module.exports = {
|
|
schema: SCHEMA,
|
|
integration: RestIntegration,
|
|
AuthType,
|
|
}
|
|
}
|