OAPI2 (swagger) complete + tests

This commit is contained in:
Rory Powell 2021-12-02 11:55:13 +00:00
parent fd1b762410
commit a5b86afa60
12 changed files with 694 additions and 974 deletions

View File

@ -92,6 +92,7 @@
"fs-extra": "8.1.0",
"jimp": "0.16.1",
"joi": "17.2.1",
"js-yaml": "^4.1.0",
"jsonschema": "1.4.0",
"knex": "^0.95.6",
"koa": "2.7.0",

View File

@ -3,11 +3,11 @@ import { queryValidation } from "../validation"
import { generateQueryID } from "../../../../db/utils"
import { Query, ImportInfo, ImportSource } from "./sources/base"
import { OpenAPI2 } from "./sources/openapi2"
import { OpenAPI3 } from "./sources/openapi3"
import { Curl } from "./sources/curl"
interface ImportResult {
errorQueries: Query[]
queries: Query[]
}
export class RestImporter {
@ -17,7 +17,7 @@ export class RestImporter {
constructor(data: string) {
this.data = data
this.sources = [new OpenAPI2(), new OpenAPI3(), new Curl()]
this.sources = [new OpenAPI2(), new Curl()]
}
init = async () => {
@ -37,12 +37,11 @@ export class RestImporter {
appId: string,
datasourceId: string,
): Promise<ImportResult> => {
// constuct the queries
let queries = await this.source.getQueries(datasourceId)
// validate queries
const errorQueries = []
const errorQueries: Query[] = []
const schema = queryValidation()
queries = queries
.filter(query => {
@ -57,19 +56,30 @@ export class RestImporter {
query._id = generateQueryID(query.datasourceId)
return query
})
// persist queries
const db = new CouchDB(appId)
for (const query of queries) {
try {
await db.put(query)
} catch (error) {
errorQueries.push(query)
const response = await db.bulkDocs(queries)
// create index to seperate queries and errors
const queryIndex = queries.reduce((acc, query) => {
if (query._id) {
acc[query._id] = query
}
}
return acc
}, ({} as { [key: string]: Query; }))
// check for failed writes
response.forEach((query: any) => {
if (!query.ok) {
errorQueries.push(queryIndex[query.id])
delete queryIndex[query.id]
}
});
return {
errorQueries,
queries: Object.values(queryIndex)
}
}

View File

@ -17,7 +17,7 @@ export interface Query {
headers: object
queryString: string | null
path: string
requestBody?: object
requestBody: string | undefined
}
transformer: string | null
schema: any
@ -47,7 +47,7 @@ export abstract class ImportSource {
queryString: string,
headers: object = {},
parameters: QueryParameter[] = [],
requestBody: object | undefined = undefined,
body: object | undefined = undefined,
): Query => {
const readable = true
const queryVerb = this.verbFromMethod(method)
@ -55,6 +55,7 @@ export abstract class ImportSource {
const schema = {}
path = this.processPath(path)
queryString = this.processQuery(queryString)
const requestBody = JSON.stringify(body, null, 2)
const query: Query = {
datasourceId,
@ -85,9 +86,13 @@ export abstract class ImportSource {
processPath = (path: string): string => {
if (path?.startsWith("/")) {
return path.substring(1)
path = path.substring(1)
}
// add extra braces around params for binding
path = path.replace(/[{]/g, "{{");
path = path.replace(/[}]/g, "}}");
return path
}

View File

@ -2,11 +2,25 @@
import { ImportSource } from "."
import SwaggerParser from "@apidevtools/swagger-parser";
import { OpenAPI } from "openapi-types";
const yaml = require('js-yaml');
export abstract class OpenAPISource extends ImportSource {
parseData = async (data: string): Promise<OpenAPI.Document> => {
const json = JSON.parse(data)
let json: OpenAPI.Document;
try {
json = JSON.parse(data)
} catch (jsonErr) {
// couldn't parse json
// try to convert yaml -> json
try {
json = yaml.load(data)
} catch (yamlErr) {
// couldn't parse yaml
throw new Error("Could not parse JSON or YAML")
}
}
return SwaggerParser.validate(json, {})
}

View File

@ -2,13 +2,8 @@ import { ImportInfo, QueryParameter, Query } from "./base"
import { OpenAPIV2 } from "openapi-types"
import { OpenAPISource } from "./base/openapi";
const isBodyParameter = (param: OpenAPIV2.Parameter): param is OpenAPIV2.InBodyParameterObject => {
return param.in === "body"
}
const isParameter = (param: OpenAPIV2.Parameter | OpenAPIV2.ReferenceObject): param is OpenAPIV2.Parameter => {
// we can guarantee this is not a reference object
// due to the deferencing done by the parser library
const parameterNotRef = (param: OpenAPIV2.Parameter | OpenAPIV2.ReferenceObject): param is OpenAPIV2.Parameter => {
// all refs are deferenced by parser library
return true
}
@ -20,6 +15,16 @@ const isOpenAPI2 = (document: any): document is OpenAPIV2.Document => {
}
}
const methods: string[] = Object.values(OpenAPIV2.HttpMethods)
const isOperation = (key: string, pathItem: any): pathItem is OpenAPIV2.OperationObject => {
return methods.includes(key)
}
const isParameter = (key: string, pathItem: any): pathItem is OpenAPIV2.Parameter => {
return !isOperation(key, pathItem)
}
/**
* OpenAPI Version 2.0 - aka "Swagger"
* https://github.com/OAI/OpenAPI-Specification/blob/main/versions/2.0.md
@ -57,30 +62,70 @@ export class OpenAPI2 extends OpenAPISource {
getQueries = async (datasourceId: string): Promise<Query[]> => {
const queries = []
let pathName: string
let path: OpenAPIV2.PathItemObject
for (let [path, pathItem] of Object.entries(this.document.paths)) {
// parameters that apply to every operation in the path
let pathParams: OpenAPIV2.Parameter[] = []
for (let [key, opOrParams] of Object.entries(pathItem)) {
if (isParameter(key, opOrParams)) {
const pathParameters = opOrParams as OpenAPIV2.Parameter[]
pathParams.push(...pathParameters)
continue
}
// can not be a parameter, must be an operation
const operation = opOrParams as OpenAPIV2.OperationObject
for ([pathName, path] of Object.entries(this.document.paths)) {
for (let [methodName, op] of Object.entries(path)) {
let operation = op as OpenAPIV2.OperationObject
const name = operation.operationId || pathName
const queryString = ""
const headers = {}
const methodName = key
const name = operation.operationId || path
let queryString = ""
const headers: any = {}
let requestBody = undefined
const parameters: QueryParameter[] = []
if (operation.parameters) {
for (let param of operation.parameters) {
if (isParameter(param)) {
if (isBodyParameter(param)) {
requestBody = {}
} else {
parameters.push({
name: param.name,
default: "",
})
}
if (operation.consumes) {
headers["Content-Type"] = operation.consumes[0]
}
// combine the path parameters with the operation parameters
const operationParams = operation.parameters || []
const allParams = [...pathParams, ...operationParams]
for (let param of allParams) {
if (parameterNotRef(param)) {
switch (param.in) {
case "query":
let prefix = ""
if (queryString) {
prefix = "&"
}
queryString = `${queryString}${prefix}${param.name}={{${param.name}}}`
break
case "header":
headers[param.name] = `{{${param.name}}}`
break
case "path":
// do nothing: param is already in the path
break
case "formData":
// future enhancement
break
case "body":
// set the request body to the example provided
// future enhancement: generate an example from the schema
let bodyParam: OpenAPIV2.InBodyParameterObject = param as OpenAPIV2.InBodyParameterObject
if (param.schema.example) {
const schema = bodyParam.schema as OpenAPIV2.SchemaObject
requestBody = schema.example
}
break;
}
// add the parameter if it can be bound in our config
if (['query', 'header', 'path'].includes(param.in)) {
parameters.push({
name: param.name,
default: param.default || "",
})
}
}
}
@ -89,7 +134,7 @@ export class OpenAPI2 extends OpenAPISource {
datasourceId,
name,
methodName,
pathName,
path,
queryString,
headers,
parameters,

View File

@ -1,40 +0,0 @@
import { ImportInfo, Query } from "./base"
import { OpenAPISource } from "./base/openapi"
import { OpenAPIV3 } from "openapi-types"
const isOpenAPI3 = (document: any): document is OpenAPIV3.Document => {
return document.openapi === "3.0.0"
}
/**
* OpenAPI Version 3.0.0
* https://github.com/OAI/OpenAPI-Specification/blob/main/versions/3.0.0.md
*/
export class OpenAPI3 extends OpenAPISource {
document!: OpenAPIV3.Document
isSupported = async (data: string): Promise<boolean> => {
try {
const document: any = await this.parseData(data)
if (isOpenAPI3(document)) {
this.document = document
return true
} else {
return false
}
} catch (err) {
return false
}
}
getInfo = async (): Promise<ImportInfo> => {
return {
url: "http://localhost:3000",
name: "swagger",
}
}
getQueries = async (datasourceId: string): Promise<Query[]> => {
return []
}
}

View File

@ -90,9 +90,9 @@ describe("Curl Import", () => {
await testQuery("query", "q1=v1&q1=v2")
})
const testBody = async (file, queryString) => {
const testBody = async (file, body) => {
const queries = await getQueries(file)
expect(queries[0].fields.requestBody).toStrictEqual(queryString)
expect(queries[0].fields.requestBody).toStrictEqual(JSON.stringify(body, null, 2))
}
it("populates body", async () => {

View File

@ -0,0 +1,160 @@
---
swagger: "2.0"
info:
description: A basic swagger file
version: 1.0.0
title: CRUD
host: example.com
tags:
- name: entity
schemes:
- http
paths:
"/entities":
post:
tags:
- entity
operationId: createEntity
consumes:
- application/json
produces:
- application/json
parameters:
- "$ref": "#/parameters/CreateEntity"
responses:
"200":
description: successful operation
schema:
"$ref": "#/definitions/Entity"
get:
tags:
- entity
operationId: getEntities
produces:
- application/json
parameters:
- "$ref": "#/parameters/Page"
- "$ref": "#/parameters/Size"
responses:
"200":
description: successful operation
schema:
"$ref": "#/definitions/Entities"
"/entities/{entityId}":
parameters:
- "$ref": "#/parameters/EntityId"
get:
tags:
- entity
operationId: getEntity
produces:
- application/json
responses:
"200":
description: successful operation
schema:
"$ref": "#/definitions/Entity"
put:
tags:
- entity
operationId: updateEntity
consumes:
- application/json
produces:
- application/json
parameters:
- "$ref": "#/parameters/Entity"
responses:
"200":
description: successful operation
schema:
"$ref": "#/definitions/Entity"
patch:
tags:
- entity
operationId: patchEntity
consumes:
- application/json
produces:
- application/json
parameters:
- "$ref": "#/parameters/Entity"
responses:
"200":
description: successful operation
schema:
"$ref": "#/definitions/Entity"
delete:
tags:
- entity
parameters:
- "$ref": "#/parameters/APIKey"
operationId: deleteEntity
produces:
- application/json
responses:
"204":
description: successful operation
parameters:
EntityId:
type: integer
format: int64
name: entityId
in: path
required: true
CreateEntity:
name: entity
in: body
required: true
schema:
"$ref": "#/definitions/CreateEntity"
Entity:
name: entity
in: body
required: true
schema:
"$ref": "#/definitions/Entity"
Page:
type: integer
format: int32
name: page
in: query
required: false
Size:
type: integer
format: int32
name: size
in: query
required: false
APIKey:
type: string
name: x-api-key
in: header
required: false
definitions:
CreateEntity:
type: object
properties:
name:
type: string
type:
type: string
example:
name: name
type: type
Entity:
allOf:
- type: object
properties:
id:
type: integer
format: int64
- "$ref": "#/definitions/CreateEntity"
example:
id: 1
name: name
type: type
Entities:
type: array
items:
"$ref": "#/definitions/Entity"

View File

@ -32,9 +32,9 @@ describe("OpenAPI2 Import", () => {
await openapi2.isSupported(getData(file, extension))
}
const runTests = async (filename, test) => {
const runTests = async (filename, test, assertions) => {
for (let extension of ["json", "yaml"]) {
await test(filename, extension)
await test(filename, extension, assertions)
}
}
@ -50,64 +50,190 @@ describe("OpenAPI2 Import", () => {
})
describe("Returns queries", () => {
const getQueries = async (file) => {
await init(file)
const queries = await openapi2.getQueries()
expect(queries.length).toBe(1)
return queries
const indexQueries = (queries) => {
return queries.reduce((acc, query) => {
acc[query.name] = query
return acc
}, {})
}
const testVerb = async (file, verb) => {
const queries = await getQueries(file)
expect(queries[0].queryVerb).toBe(verb)
const getQueries = async (file, extension) => {
await init(file, extension)
const queries = await openapi2.getQueries()
expect(queries.length).toBe(6)
return indexQueries(queries)
}
const testVerb = async (file, extension, assertions) => {
const queries = await getQueries(file, extension)
for (let [operationId, method] of Object.entries(assertions)) {
expect(queries[operationId].queryVerb).toBe(method)
}
}
it("populates verb", async () => {
await testVerb("get", "read")
await testVerb("post", "create")
await testVerb("put", "update")
await testVerb("delete", "delete")
await testVerb("patch", "patch")
const assertions = {
"createEntity" : "create",
"getEntities" : "read",
"getEntity" : "read",
"updateEntity" : "update",
"patchEntity" : "patch",
"deleteEntity" : "delete"
}
await runTests("crud", testVerb, assertions)
})
const testPath = async (file, urlPath) => {
const queries = await getQueries(file)
expect(queries[0].fields.path).toBe(urlPath)
const testPath = async (file, extension, assertions) => {
const queries = await getQueries(file, extension)
for (let [operationId, urlPath] of Object.entries(assertions)) {
expect(queries[operationId].fields.path).toBe(urlPath)
}
}
it("populates path", async () => {
await testPath("get", "")
await testPath("path", "paths/abc")
const assertions = {
"createEntity" : "entities",
"getEntities" : "entities",
"getEntity" : "entities/{{entityId}}",
"updateEntity" : "entities/{{entityId}}",
"patchEntity" : "entities/{{entityId}}",
"deleteEntity" : "entities/{{entityId}}"
}
await runTests("crud", testPath, assertions)
})
const testHeaders = async (file, headers) => {
const queries = await getQueries(file)
expect(queries[0].fields.headers).toStrictEqual(headers)
const testHeaders = async (file, extension, assertions) => {
const queries = await getQueries(file, extension)
for (let [operationId, headers] of Object.entries(assertions)) {
expect(queries[operationId].fields.headers).toStrictEqual(headers)
}
}
const contentTypeHeader = {
"Content-Type" : "application/json",
}
it("populates headers", async () => {
await testHeaders("get", {})
await testHeaders("headers", { "x-bb-header-1" : "123", "x-bb-header-2" : "456"} )
const assertions = {
"createEntity" : {
...contentTypeHeader
},
"getEntities" : {
},
"getEntity" : {
},
"updateEntity" : {
...contentTypeHeader
},
"patchEntity" : {
...contentTypeHeader
},
"deleteEntity" : {
"x-api-key" : "{{x-api-key}}",
}
}
await runTests("crud", testHeaders, assertions)
})
const testQuery = async (file, queryString) => {
const queries = await getQueries(file)
expect(queries[0].fields.queryString).toBe(queryString)
const testQuery = async (file, extension, assertions) => {
const queries = await getQueries(file, extension)
for (let [operationId, queryString] of Object.entries(assertions)) {
expect(queries[operationId].fields.queryString).toStrictEqual(queryString)
}
}
it("populates query", async () => {
await testQuery("get", "")
await testQuery("query", "q1=v1&q1=v2")
const assertions = {
"createEntity" : "",
"getEntities" : "page={{page}}&size={{size}}",
"getEntity" : "",
"updateEntity" : "",
"patchEntity" : "",
"deleteEntity" : ""
}
await runTests("crud", testQuery, assertions)
})
const testBody = async (file, queryString) => {
const queries = await getQueries(file)
expect(queries[0].fields.requestBody).toStrictEqual(queryString)
const testParameters = async (file, extension, assertions) => {
const queries = await getQueries(file, extension)
for (let [operationId, parameters] of Object.entries(assertions)) {
expect(queries[operationId].parameters).toStrictEqual(parameters)
}
}
it("populates parameters", async () => {
const assertions = {
"createEntity" : [],
"getEntities" : [
{
"name" : "page",
"default" : "",
},
{
"name" : "size",
"default" : "",
}
],
"getEntity" : [
{
"name" : "entityId",
"default" : "",
}
],
"updateEntity" : [
{
"name" : "entityId",
"default" : "",
}
],
"patchEntity" : [
{
"name" : "entityId",
"default" : "",
}
],
"deleteEntity" : [
{
"name" : "entityId",
"default" : "",
},
{
"name" : "x-api-key",
"default" : "",
}
]
}
await runTests("crud", testParameters, assertions)
})
const testBody = async (file, extension, assertions) => {
const queries = await getQueries(file, extension)
for (let [operationId, body] of Object.entries(assertions)) {
expect(queries[operationId].fields.requestBody).toStrictEqual(JSON.stringify(body, null, 2))
}
}
it("populates body", async () => {
await testBody("get", undefined)
await testBody("post", { "key" : "val" })
await testBody("empty-body", {})
const assertions = {
"createEntity" : {
"name" : "name",
"type" : "type",
},
"getEntities" : undefined,
"getEntity" : undefined,
"updateEntity" : {
"id": 1,
"name" : "name",
"type" : "type",
},
"patchEntity" : {
"id": 1,
"name" : "name",
"type" : "type",
},
"deleteEntity" : undefined
}
await runTests("crud", testBody, assertions)
})
})
})

View File

@ -0,0 +1,115 @@
const bulkDocs = jest.fn()
const db = jest.fn(() => {
return {
bulkDocs
}
})
jest.mock("../../../../../db", () => db)
const { RestImporter } = require("../index")
const fs = require("fs")
const path = require('path')
const getData = (file) => {
return fs.readFileSync(path.join(__dirname, `../sources/tests/${file}`), "utf8")
}
// openapi2 (swagger)
const oapi2CrudJson = getData("openapi2/data/crud/crud.json")
const oapi2CrudYaml = getData("openapi2/data/crud/crud.json")
const oapi2PetstoreJson = getData("openapi2/data/petstore/petstore.json")
const oapi2PetstoreYaml = getData("openapi2/data/petstore/petstore.json")
// curl
const curl = getData("curl/data/post.txt")
const datasets = {
oapi2CrudJson,
oapi2CrudYaml,
oapi2PetstoreJson,
oapi2PetstoreYaml,
curl
}
describe("Rest Importer", () => {
let restImporter
const init = async (data) => {
restImporter = new RestImporter(data)
await restImporter.init()
}
const runTest = async (test, assertions) => {
for (let [key, data] of Object.entries(datasets)) {
await test(key, data, assertions)
}
}
const testGetInfo = async (key, data, assertions) => {
await init(data)
const info = await restImporter.getInfo()
expect(info.name).toBe(assertions[key].name)
expect(info.url).toBe(assertions[key].url)
}
it("gets info", async () => {
const assertions = {
"oapi2CrudJson" : {
name: "CRUD",
url: "http://example.com"
},
"oapi2CrudYaml" : {
name: "CRUD",
url: "http://example.com"
},
"oapi2PetstoreJson" : {
name: "Swagger Petstore",
url: "https://petstore.swagger.io/v2"
},
"oapi2PetstoreYaml" :{
name: "Swagger Petstore",
url: "https://petstore.swagger.io/v2"
},
"curl": {
name: "example.com",
url: "http://example.com"
}
}
await runTest(testGetInfo, assertions)
})
const testImportQueries = async (key, data, assertions) => {
await init(data)
bulkDocs.mockReturnValue([])
const importResult = await restImporter.importQueries("appId", "datasourceId")
expect(importResult.errorQueries.length).toBe(0)
expect(importResult.queries.length).toBe(assertions[key].count)
expect(bulkDocs).toHaveBeenCalledTimes(1)
jest.clearAllMocks()
}
it("imports queries", async () => {
// simple sanity assertions that the whole dataset
// makes it through the importer
const assertions = {
"oapi2CrudJson" : {
count: 6,
},
"oapi2CrudYaml" :{
count: 6,
},
"oapi2PetstoreJson" : {
count: 20,
},
"oapi2PetstoreYaml" :{
count: 20,
},
"curl": {
count: 1
}
}
await runTest(testImportQueries, assertions)
})
})