OAPI2 (swagger) complete + tests

packages/server/package.json

@@ -92,6 +92,7 @@
     "fs-extra": "8.1.0",
     "jimp": "0.16.1",
     "joi": "17.2.1",
+    "js-yaml": "^4.1.0",
     "jsonschema": "1.4.0",
     "knex": "^0.95.6",
     "koa": "2.7.0",

packages/server/src/api/controllers/query/import/index.ts

@@ -3,11 +3,11 @@ import { queryValidation } from "../validation"
 import { generateQueryID } from "../../../../db/utils"
 import { Query, ImportInfo, ImportSource } from "./sources/base"
 import { OpenAPI2 } from "./sources/openapi2"
-import { OpenAPI3 } from "./sources/openapi3"
 import { Curl } from "./sources/curl"
 
 interface ImportResult {
   errorQueries: Query[]
+  queries: Query[]
 }
 
 export class RestImporter {
@@ -17,7 +17,7 @@ export class RestImporter {
 
   constructor(data: string) {
     this.data = data
-    this.sources = [new OpenAPI2(), new OpenAPI3(), new Curl()]
+    this.sources = [new OpenAPI2(), new Curl()]
   }
 
   init = async () => {
@@ -37,12 +37,11 @@ export class RestImporter {
     appId: string,
     datasourceId: string,
   ): Promise<ImportResult> => {
-  
     // constuct the queries
     let queries = await this.source.getQueries(datasourceId)
   
     // validate queries
-    const errorQueries = []
+    const errorQueries: Query[] = []
     const schema = queryValidation()
     queries = queries
       .filter(query => {
@@ -60,16 +59,27 @@ export class RestImporter {
 
     // persist queries
     const db = new CouchDB(appId)
-    for (const query of queries) {
+    const response = await db.bulkDocs(queries) 
-      try {
+
-        await db.put(query)
+    // create index to seperate queries and errors
-      } catch (error) {
+    const queryIndex = queries.reduce((acc, query) => {
-        errorQueries.push(query)
+      if (query._id) {
+        acc[query._id] = query
       }
+      return acc
+    }, ({} as { [key: string]: Query; }))
+
+    // check for failed writes
+    response.forEach((query: any) => {
+      if (!query.ok) {
+        errorQueries.push(queryIndex[query.id])
+        delete queryIndex[query.id]
       } 
+    });
   
     return {
       errorQueries,
+      queries: Object.values(queryIndex)
     }
   }
 

packages/server/src/api/controllers/query/import/sources/base/index.ts

@@ -17,7 +17,7 @@ export interface Query {
     headers: object
     queryString: string | null
     path: string
-    requestBody?: object
+    requestBody: string | undefined
   }
   transformer: string | null
   schema: any
@@ -47,7 +47,7 @@ export abstract class ImportSource {
     queryString: string,
     headers: object = {},
     parameters: QueryParameter[] = [],
-    requestBody: object | undefined = undefined,
+    body: object | undefined = undefined,
   ): Query => {
     const readable = true
     const queryVerb = this.verbFromMethod(method)
@@ -55,6 +55,7 @@ export abstract class ImportSource {
     const schema = {}
     path = this.processPath(path)
     queryString = this.processQuery(queryString)
+    const requestBody = JSON.stringify(body, null, 2)
   
     const query: Query = {
       datasourceId,
@@ -85,9 +86,13 @@ export abstract class ImportSource {
 
   processPath = (path: string): string => {
     if (path?.startsWith("/")) {
-      return path.substring(1)
+      path = path.substring(1)
     }
 
+    // add extra braces around params for binding
+    path = path.replace(/[{]/g, "{{");
+    path = path.replace(/[}]/g, "}}");
+    
     return path
   }
 

packages/server/src/api/controllers/query/import/sources/base/openapi.ts

@@ -2,11 +2,25 @@
 import { ImportSource } from "."
 import SwaggerParser from "@apidevtools/swagger-parser";
 import { OpenAPI } from "openapi-types";
+const yaml = require('js-yaml');
 
 export abstract class OpenAPISource extends ImportSource {
 
   parseData = async (data: string): Promise<OpenAPI.Document> => {
-    const json = JSON.parse(data)
+    let json: OpenAPI.Document;
+    try {
+      json = JSON.parse(data)
+    } catch (jsonErr) {
+      // couldn't parse json
+      // try to convert yaml -> json
+      try {
+        json = yaml.load(data)
+      } catch (yamlErr) {
+        // couldn't parse yaml
+        throw new Error("Could not parse JSON or YAML")
+      }
+    }
+    
     return SwaggerParser.validate(json, {})
   }
 

packages/server/src/api/controllers/query/import/sources/openapi2.ts

@@ -2,13 +2,8 @@ import { ImportInfo, QueryParameter, Query } from "./base"
 import { OpenAPIV2 } from "openapi-types"
 import { OpenAPISource } from "./base/openapi";
 
-const isBodyParameter = (param: OpenAPIV2.Parameter): param is OpenAPIV2.InBodyParameterObject => {
+const parameterNotRef = (param: OpenAPIV2.Parameter | OpenAPIV2.ReferenceObject): param is OpenAPIV2.Parameter => {
-  return param.in === "body"
+  // all refs are deferenced by parser library
-}
-
-const isParameter = (param: OpenAPIV2.Parameter | OpenAPIV2.ReferenceObject): param is OpenAPIV2.Parameter => {
-  // we can guarantee this is not a reference object
-  // due to the deferencing done by the parser library
   return true
 }
 
@@ -20,6 +15,16 @@ const isOpenAPI2 = (document: any): document is OpenAPIV2.Document => {
   }
 }
 
+const methods: string[] = Object.values(OpenAPIV2.HttpMethods)
+
+const isOperation = (key: string, pathItem: any): pathItem is OpenAPIV2.OperationObject => {
+  return methods.includes(key)
+}
+
+const isParameter = (key: string, pathItem: any): pathItem is OpenAPIV2.Parameter => {
+  return !isOperation(key, pathItem)
+}
+
 /**
  * OpenAPI Version 2.0 - aka "Swagger"
  * https://github.com/OAI/OpenAPI-Specification/blob/main/versions/2.0.md
@@ -57,39 +62,79 @@ export class OpenAPI2 extends OpenAPISource {
   getQueries = async (datasourceId: string): Promise<Query[]> => {
     const queries = []
   
-    let pathName: string
+    for (let [path, pathItem] of Object.entries(this.document.paths)) {
-    let path: OpenAPIV2.PathItemObject
+      // parameters that apply to every operation in the path
+      let pathParams: OpenAPIV2.Parameter[] = []
 
-    for ([pathName, path] of Object.entries(this.document.paths)) {
+      for (let [key, opOrParams] of Object.entries(pathItem)) {
-      for (let [methodName, op] of Object.entries(path)) {
+        if (isParameter(key, opOrParams)) {
-        let operation = op as OpenAPIV2.OperationObject
+          const pathParameters = opOrParams as OpenAPIV2.Parameter[]
+          pathParams.push(...pathParameters)
+          continue
+        }
+        // can not be a parameter, must be an operation
+        const operation = opOrParams as OpenAPIV2.OperationObject
   
-        const name = operation.operationId || pathName
+        const methodName = key
-        const queryString = ""
+        const name = operation.operationId || path
-        const headers = {}
+        let queryString = ""
+        const headers: any = {}
         let requestBody = undefined
         const parameters: QueryParameter[] = []
 
-        if (operation.parameters) {
+        if (operation.consumes) {
-          for (let param of operation.parameters) { 
+          headers["Content-Type"] = operation.consumes[0]
-            if (isParameter(param)) {
+        }
-              if (isBodyParameter(param)) {
+
-                requestBody = {}
+        // combine the path parameters with the operation parameters
-              } else {
+        const operationParams = operation.parameters || []
+        const allParams = [...pathParams, ...operationParams]
+
+        for (let param of allParams) {
+          if (parameterNotRef(param)) {
+            switch (param.in) {
+              case "query":
+                let prefix = ""
+                if (queryString) {
+                  prefix = "&"
+                }
+                queryString = `${queryString}${prefix}${param.name}={{${param.name}}}`
+                break
+              case "header":
+                headers[param.name] = `{{${param.name}}}`
+                break
+              case "path":
+                // do nothing: param is already in the path
+                break
+              case "formData":
+                // future enhancement
+                break
+              case "body":
+                // set the request body to the example provided
+                // future enhancement: generate an example from the schema
+                let bodyParam: OpenAPIV2.InBodyParameterObject = param as OpenAPIV2.InBodyParameterObject
+                if (param.schema.example) {
+                  const schema = bodyParam.schema as OpenAPIV2.SchemaObject
+                  requestBody = schema.example
+                }
+                break;
+            }
+
+            // add the parameter if it can be bound in our config
+            if (['query', 'header', 'path'].includes(param.in)) {
               parameters.push({
                 name: param.name,
-                  default: "",
+                default: param.default || "",
               })  
             }
           }
         }
-        }
   
         const query = this.constructQuery(
           datasourceId,
           name,
           methodName,
-          pathName,
+          path,
           queryString,
           headers,
           parameters,

packages/server/src/api/controllers/query/import/sources/openapi3.ts

@@ -1,40 +0,0 @@
-import { ImportInfo, Query } from "./base"
-import { OpenAPISource } from "./base/openapi"
-import { OpenAPIV3 } from "openapi-types"
-
-const isOpenAPI3 = (document: any): document is OpenAPIV3.Document => {
-  return document.openapi === "3.0.0" 
-}
-
-/**
- * OpenAPI Version 3.0.0
- * https://github.com/OAI/OpenAPI-Specification/blob/main/versions/3.0.0.md
- */
-export class OpenAPI3 extends OpenAPISource {
-  document!: OpenAPIV3.Document
-
-  isSupported = async (data: string): Promise<boolean> => {
-    try {
-      const document: any = await this.parseData(data)
-      if (isOpenAPI3(document)) {
-        this.document = document
-        return true
-      } else {
-        return false
-      }
-    } catch (err) {
-      return false
-    }
-  }
-
-  getInfo = async (): Promise<ImportInfo> => {
-    return {
-      url: "http://localhost:3000",
-      name: "swagger",
-    }
-  }
-
-  getQueries = async (datasourceId: string): Promise<Query[]> => {
-    return []
-  }
-}

packages/server/src/api/controllers/query/import/sources/tests/curl/curl.spec.js

@@ -90,9 +90,9 @@ describe("Curl Import", () => {
       await testQuery("query", "q1=v1&q1=v2")
     })
 
-    const testBody = async (file, queryString) => {
+    const testBody = async (file, body) => {
       const queries = await getQueries(file)
-      expect(queries[0].fields.requestBody).toStrictEqual(queryString)
+      expect(queries[0].fields.requestBody).toStrictEqual(JSON.stringify(body, null, 2))
     }
 
     it("populates body", async () => {

packages/server/src/api/controllers/query/import/sources/tests/openapi2/data/crud/crud.yaml

@@ -0,0 +1,160 @@
+---
+swagger: "2.0"
+info:
+  description: A basic swagger file
+  version: 1.0.0
+  title: CRUD
+host: example.com
+tags:
+  - name: entity
+schemes:
+  - http
+paths:
+  "/entities":
+    post:
+      tags:
+        - entity
+      operationId: createEntity
+      consumes:
+        - application/json
+      produces:
+        - application/json
+      parameters:
+        - "$ref": "#/parameters/CreateEntity"
+      responses:
+        "200":
+          description: successful operation
+          schema:
+            "$ref": "#/definitions/Entity"
+    get:
+      tags:
+        - entity
+      operationId: getEntities
+      produces:
+        - application/json
+      parameters:
+        - "$ref": "#/parameters/Page"
+        - "$ref": "#/parameters/Size"
+      responses:
+        "200":
+          description: successful operation
+          schema:
+            "$ref": "#/definitions/Entities"
+  "/entities/{entityId}":
+    parameters:
+      - "$ref": "#/parameters/EntityId"
+    get:
+      tags:
+        - entity
+      operationId: getEntity
+      produces:
+        - application/json
+      responses:
+        "200":
+          description: successful operation
+          schema:
+            "$ref": "#/definitions/Entity"
+    put:
+      tags:
+        - entity
+      operationId: updateEntity
+      consumes:
+        - application/json
+      produces:
+        - application/json
+      parameters:
+        - "$ref": "#/parameters/Entity"
+      responses:
+        "200":
+          description: successful operation
+          schema:
+            "$ref": "#/definitions/Entity"
+    patch:
+      tags:
+        - entity
+      operationId: patchEntity
+      consumes:
+        - application/json
+      produces:
+        - application/json
+      parameters:
+        - "$ref": "#/parameters/Entity"
+      responses:
+        "200":
+          description: successful operation
+          schema:
+            "$ref": "#/definitions/Entity"
+    delete:
+      tags:
+        - entity
+      parameters:
+        - "$ref": "#/parameters/APIKey"
+      operationId: deleteEntity
+      produces:
+        - application/json
+      responses:
+        "204":
+          description: successful operation
+parameters:
+  EntityId:
+    type: integer
+    format: int64
+    name: entityId
+    in: path
+    required: true
+  CreateEntity:
+    name: entity
+    in: body
+    required: true
+    schema:
+      "$ref": "#/definitions/CreateEntity"
+  Entity:
+    name: entity
+    in: body
+    required: true
+    schema:
+      "$ref": "#/definitions/Entity"
+  Page:
+    type: integer
+    format: int32
+    name: page
+    in: query
+    required: false
+  Size:
+    type: integer
+    format: int32
+    name: size
+    in: query
+    required: false
+  APIKey:
+    type: string
+    name: x-api-key
+    in: header
+    required: false
+definitions:
+  CreateEntity:
+    type: object
+    properties:
+      name:
+        type: string
+      type:
+        type: string
+    example:
+      name: name
+      type: type
+  Entity:
+    allOf:
+      - type: object
+        properties:
+          id:
+            type: integer
+            format: int64
+      - "$ref": "#/definitions/CreateEntity"
+    example:
+      id: 1
+      name: name
+      type: type
+  Entities:
+    type: array
+    items:
+      "$ref": "#/definitions/Entity"

packages/server/src/api/controllers/query/import/sources/tests/openapi2/openapi2.spec.js

@@ -32,9 +32,9 @@ describe("OpenAPI2 Import", () => {
     await openapi2.isSupported(getData(file, extension))
   }
 
-  const runTests = async (filename, test) => {
+  const runTests = async (filename, test, assertions) => {
     for (let extension of ["json", "yaml"]) {
-      await test(filename, extension)
+      await test(filename, extension, assertions)
     }
   }
 
@@ -50,64 +50,190 @@ describe("OpenAPI2 Import", () => {
   })
   
   describe("Returns queries", () => {
-    const getQueries = async (file) => {
+    const indexQueries = (queries) => {
-      await init(file)
+      return queries.reduce((acc, query) => {
-      const queries = await openapi2.getQueries()
+        acc[query.name] = query
-      expect(queries.length).toBe(1)
+        return acc
-      return queries
+      }, {})
     }
 
-    const testVerb = async (file, verb) => {
+    const getQueries = async (file, extension) => {
-      const queries = await getQueries(file)
+      await init(file, extension)
-      expect(queries[0].queryVerb).toBe(verb)
+      const queries = await openapi2.getQueries()
+      expect(queries.length).toBe(6)
+      return indexQueries(queries)
+    }
+
+    const testVerb = async (file, extension, assertions) => {
+      const queries = await getQueries(file, extension)
+      for (let [operationId, method] of Object.entries(assertions)) {
+        expect(queries[operationId].queryVerb).toBe(method)
+      }
     }
 
     it("populates verb", async () => {
-      await testVerb("get", "read")
+      const assertions = {
-      await testVerb("post", "create")
+        "createEntity" : "create",
-      await testVerb("put", "update")
+        "getEntities" : "read",
-      await testVerb("delete", "delete")
+        "getEntity" : "read",
-      await testVerb("patch", "patch")
+        "updateEntity" : "update",
+        "patchEntity" : "patch",
+        "deleteEntity" : "delete"
+      }
+      await runTests("crud", testVerb, assertions)
     }) 
 
-    const testPath = async (file, urlPath) => {
+    const testPath = async (file, extension, assertions) => {
-      const queries = await getQueries(file)
+      const queries = await getQueries(file, extension)
-      expect(queries[0].fields.path).toBe(urlPath)
+      for (let [operationId, urlPath] of Object.entries(assertions)) {
+        expect(queries[operationId].fields.path).toBe(urlPath)
+      }
     }
 
     it("populates path", async () => {
-        await testPath("get", "")
+      const assertions = {
-        await testPath("path", "paths/abc")
+        "createEntity" : "entities",
+        "getEntities" : "entities",
+        "getEntity" : "entities/{{entityId}}",
+        "updateEntity" : "entities/{{entityId}}",
+        "patchEntity" : "entities/{{entityId}}",
+        "deleteEntity" : "entities/{{entityId}}"
+      }
+      await runTests("crud", testPath, assertions)
     })
 
-    const testHeaders = async (file, headers) => {
+    const testHeaders = async (file, extension, assertions) => {
-      const queries = await getQueries(file)
+      const queries = await getQueries(file, extension)
-      expect(queries[0].fields.headers).toStrictEqual(headers)
+      for (let [operationId, headers] of Object.entries(assertions)) {
+        expect(queries[operationId].fields.headers).toStrictEqual(headers)
+      }
+    }
+
+    const contentTypeHeader = {
+      "Content-Type" : "application/json",
     }
 
     it("populates headers", async () => {
-      await testHeaders("get", {})
+      const assertions = {
-      await testHeaders("headers", { "x-bb-header-1" : "123", "x-bb-header-2" : "456"} )
+        "createEntity" : {
+         ...contentTypeHeader
+        },
+        "getEntities" : {
+        },
+        "getEntity" : {
+        },
+        "updateEntity" : {
+          ...contentTypeHeader
+        },
+        "patchEntity" : {
+          ...contentTypeHeader
+        },
+        "deleteEntity" : {
+            "x-api-key" : "{{x-api-key}}",
+        }
+      }
+
+      await runTests("crud", testHeaders, assertions)
     })
 
-    const testQuery = async (file, queryString) => {
+    const testQuery = async (file, extension, assertions) => {
-      const queries = await getQueries(file)
+      const queries = await getQueries(file, extension)
-      expect(queries[0].fields.queryString).toBe(queryString)
+      for (let [operationId, queryString] of Object.entries(assertions)) {
+        expect(queries[operationId].fields.queryString).toStrictEqual(queryString)
       }
+    }
+
     it("populates query", async () => {
-      await testQuery("get", "")
+      const assertions = {
-      await testQuery("query", "q1=v1&q1=v2")
+        "createEntity" : "",
+        "getEntities" : "page={{page}}&size={{size}}",
+        "getEntity" : "",
+        "updateEntity" : "",
+        "patchEntity" : "",
+        "deleteEntity" : ""
+      }
+      await runTests("crud", testQuery, assertions)
     })
 
-    const testBody = async (file, queryString) => {
+    const testParameters = async (file, extension, assertions) => {
-      const queries = await getQueries(file)
+      const queries = await getQueries(file, extension)
-      expect(queries[0].fields.requestBody).toStrictEqual(queryString)
+      for (let [operationId, parameters] of Object.entries(assertions)) {
+        expect(queries[operationId].parameters).toStrictEqual(parameters)
+      }
     }
 
+    it("populates parameters", async () => {
+      const assertions = {
+        "createEntity" : [],
+        "getEntities" : [
+          {
+            "name" : "page",
+            "default" : "",
+          },
+          {
+            "name" : "size",
+            "default" : "",
+          }
+        ],
+        "getEntity" : [
+          {
+            "name" : "entityId",
+            "default" : "",
+          }
+        ],
+        "updateEntity" : [
+          {
+            "name" : "entityId",
+            "default" : "",
+          }
+        ],
+        "patchEntity" : [
+          {
+            "name" : "entityId",
+            "default" : "",
+          }
+        ],
+        "deleteEntity" : [
+          {
+            "name" : "entityId",
+            "default" : "",
+          },
+          {
+            "name" : "x-api-key",
+            "default" : "",
+          }
+        ]
+      }
+      await runTests("crud", testParameters, assertions)
+    })
+
+    const testBody = async (file, extension, assertions) => {
+      const queries = await getQueries(file, extension)
+      for (let [operationId, body] of Object.entries(assertions)) {
+        expect(queries[operationId].fields.requestBody).toStrictEqual(JSON.stringify(body, null, 2))
+      }
+    }
     it("populates body", async () => {
-      await testBody("get", undefined)
+      const assertions = {
-      await testBody("post", { "key" : "val" })
+        "createEntity" : {
-      await testBody("empty-body", {})
+          "name" : "name",
+          "type" : "type",
+        },
+        "getEntities" : undefined,
+        "getEntity" : undefined,
+        "updateEntity" : {
+          "id": 1,
+          "name" : "name",
+          "type" : "type",
+        },
+        "patchEntity" : {
+          "id": 1,
+          "name" : "name",
+          "type" : "type",
+        },
+        "deleteEntity" : undefined
+      }
+      await runTests("crud", testBody, assertions)
     })
   })
 })

packages/server/src/api/controllers/query/import/tests/index.spec.js

@@ -0,0 +1,115 @@
+
+const bulkDocs = jest.fn()
+const db = jest.fn(() => {
+  return {
+    bulkDocs
+  }
+})
+jest.mock("../../../../../db", () => db)
+
+const { RestImporter } = require("../index")
+
+const fs = require("fs")
+const path = require('path')
+
+const getData = (file) => {
+  return fs.readFileSync(path.join(__dirname, `../sources/tests/${file}`), "utf8")
+}
+
+// openapi2 (swagger)
+const oapi2CrudJson = getData("openapi2/data/crud/crud.json")
+const oapi2CrudYaml = getData("openapi2/data/crud/crud.json")
+const oapi2PetstoreJson = getData("openapi2/data/petstore/petstore.json")
+const oapi2PetstoreYaml = getData("openapi2/data/petstore/petstore.json")
+
+// curl
+const curl = getData("curl/data/post.txt")
+
+const datasets = {
+  oapi2CrudJson,
+  oapi2CrudYaml,
+  oapi2PetstoreJson,
+  oapi2PetstoreYaml,
+  curl
+}
+
+describe("Rest Importer", () => {
+  let restImporter 
+
+  const init = async (data) => {
+    restImporter = new RestImporter(data)
+    await restImporter.init()
+  }
+
+  const runTest = async (test, assertions) => {
+    for (let [key, data] of Object.entries(datasets)) {
+      await test(key, data, assertions)
+    }
+  }
+
+  const testGetInfo = async (key, data, assertions) => {
+    await init(data)
+    const info = await restImporter.getInfo()
+    expect(info.name).toBe(assertions[key].name)
+    expect(info.url).toBe(assertions[key].url)
+  }
+
+  it("gets info", async () => {
+    const assertions = {
+      "oapi2CrudJson" : {
+        name: "CRUD",
+        url: "http://example.com"
+      },
+      "oapi2CrudYaml" : {
+        name: "CRUD",
+        url: "http://example.com"
+      },
+      "oapi2PetstoreJson" : {
+        name: "Swagger Petstore",
+        url: "https://petstore.swagger.io/v2"
+      },
+      "oapi2PetstoreYaml" :{
+        name: "Swagger Petstore",
+        url: "https://petstore.swagger.io/v2"
+      },
+      "curl": {
+        name: "example.com",
+        url: "http://example.com"
+      }
+    }
+    await runTest(testGetInfo, assertions)
+  })
+
+  const testImportQueries = async (key, data, assertions) => {
+    await init(data)
+    bulkDocs.mockReturnValue([])
+    const importResult = await restImporter.importQueries("appId", "datasourceId")
+    expect(importResult.errorQueries.length).toBe(0)
+    expect(importResult.queries.length).toBe(assertions[key].count)
+    expect(bulkDocs).toHaveBeenCalledTimes(1)
+    jest.clearAllMocks()
+  }
+
+  it("imports queries", async () => {
+    // simple sanity assertions that the whole dataset 
+    // makes it through the importer
+    const assertions = {
+      "oapi2CrudJson" : {
+        count: 6,
+      },
+      "oapi2CrudYaml" :{
+        count: 6,
+      },
+      "oapi2PetstoreJson" : {
+        count: 20,
+      },
+      "oapi2PetstoreYaml" :{
+        count: 20,
+      },
+      "curl": {
+        count: 1
+      }
+    }
+    await runTest(testImportQueries, assertions)
+  })
+})