Update backend to be extensible for different import sources

packages/builder/src/analytics/constants.js

@@ -9,6 +9,9 @@ export const Events = {
     CREATED: "Datasource Created",
     UPDATED: "Datasource Updated",
   },
+  QUERIES: {
+    REST: "REST Queries Imported",
+  },
   TABLE: {
     CREATED: "Table Created",
   },

packages/builder/src/components/backend/DatasourceNavigator/modals/CreateDatasourceModal.svelte

@@ -6,7 +6,7 @@
   import { IntegrationNames } from "constants"
   import CreateTableModal from "components/backend/TableNavigator/modals/CreateTableModal.svelte"
   import DatasourceConfigModal from "components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte"
-  import ImportRestDatasourceModal from "./ImportRestDatasourceModal.svelte"
+  import ImportRestQueriesModal from "./ImportRestQueriesModal.svelte"
 
   export let modal
   let integrations = []
@@ -80,7 +80,7 @@
 
 <Modal bind:this={importModal}>
   {#if integration.type === "REST"}
-    <ImportRestDatasourceModal {integration} {modal} />
+    <ImportRestQueriesModal {integration} {modal} createDatasource={true} />
   {/if}
 </Modal>
 

packages/builder/src/components/backend/DatasourceNavigator/modals/ImportRestQueriesModal.svelte

@@ -16,6 +16,8 @@
   import { datasources, queries } from "stores/backend"
 
   export let modal
+  export let datasourceId
+  export let createDatasource = false
 
   let data = {
     url: "",
@@ -25,47 +27,60 @@
 
   let lastTouched = "url"
 
-  const getPayload = async () => {
+  const getData = async () => {
-    let payloadData
+    let dataString
-    let type
 
     // parse the file into memory and send as string
     if (lastTouched === "file") {
-      type = "raw"
+      dataString = await data.file[0].text()
-      payloadData = await data.file[0].text()
+    } else if (lastTouched === "url") {
-    } else {
+      const response = await fetch(data.url)
-      type = lastTouched
+      dataString = await response.text()
-      payloadData = data[lastTouched]
+    } else if (lastTouched === "raw") {
+      dataString = data.raw
     }
 
-    return {
+    return dataString
-      type: type,
-      data: payloadData,
-    }
   }
 
-  async function importDatasource() {
+  async function importQueries() {
     try {
-      const resp = await datasources.import(await getPayload())
+      const dataString = await getData()
 
+      if (!datasourceId && !createDatasource) {
+        throw new Error("No datasource id")
+      }
+
+      const body = {
+        data: dataString,
+        datasourceId,
+      }
+
+      const resp = await queries.import(body)
+      datasourceId = resp.datasourceId
+
+      // reload
+      await datasources.fetch()
       await queries.fetch()
-      await datasources.select(resp._id)
+      await datasources.select(datasourceId)
-      $goto(`./datasource/${resp._id}`)
+
-      notifications.success(`Datasource imported successfully.`)
+      $goto(`./datasource/${datasourceId}`)
-      analytics.captureEvent(Events.DATASOURCE.IMPORTED, {
+      notifications.success(`Imported successfully.`)
-        name: resp.name,
+      analytics.captureEvent(Events.QUERIES.REST.IMPORTED, {
-        source: resp.source,
+        importType: lastTouched,
+        newDatasource: createDatasource,
       })
+
       return true
     } catch (err) {
-      notifications.error(`Error importing datasource: ${err}`)
+      notifications.error(`Error importing: ${err}`)
       return false
     }
   }
 </script>
 
 <ModalContent
-  onConfirm={() => importDatasource()}
+  onConfirm={() => importQueries()}
   onCancel={() => modal.show()}
   confirmText={"Import"}
   cancelText="Back"

packages/builder/src/stores/backend/datasources.js

@@ -84,11 +84,6 @@ export function createDatasourcesStore() {
 
       return updateDatasource(response)
     },
-    import: async body => {
-      let response
-      response = await api.post(`/api/queries/import/swagger2`, body)
-      return updateDatasource(response)
-    },
     delete: async datasource => {
       const response = await api.delete(
         `/api/datasources/${datasource._id}/${datasource._rev}`

packages/builder/src/stores/backend/queries.js

@@ -53,6 +53,10 @@ export function createQueriesStore() {
       })
       return json
     },
+    import: async body => {
+      const response = await api.post(`/api/queries/import`, body)
+      return response.json()
+    },
     select: query => {
       update(state => ({ ...state, selected: query._id }))
       views.unselect()

packages/server/package.json

@@ -77,6 +77,7 @@
     "@koa/router": "8.0.0",
     "@sendgrid/mail": "7.1.1",
     "@sentry/node": "^6.0.0",
+    "@types/swagger-schema-official": "^2.0.22",
     "airtable": "0.10.1",
     "arangojs": "7.2.0",
     "aws-sdk": "^2.767.0",

packages/server/src/api/controllers/query/import.ts

@@ -0,0 +1,275 @@
+import CouchDB from "../../../db"
+import { queryValidation } from "./validation"
+import { generateQueryID } from "../../../db/utils"
+import { Spec as Swagger2, Operation } from "swagger-schema-official"
+
+// {
+//   "_id": "query_datasource_d62738f2d72a466997ffbf46f4952404_e7258ad382cd4c37961b81730633ff2d",
+//   "_rev": "1-e702a18eaa96c7cb4be1b402c34eaa59",
+//   "datasourceId": "datasource_d62738f2d72a466997ffbf46f4952404",
+//   "parameters": [
+//   {
+//     "name": "paramtest",
+//     "default": "defaultValue"
+//   }
+// ],
+//   "fields": {
+//     "headers": {
+//       "headertest": "test"
+//     },
+//     "queryString": "query=test",
+//     "path": "/path/test"
+//   },
+//   "queryVerb": "read",
+//   "transformer": "return data.test",
+//   "schema": {},
+//   "name": "name",
+//   "readable": true
+// }
+
+// return joiValidator.body(Joi.object({
+//   _id: Joi.string(),
+//   _rev: Joi.string(),
+//   name: Joi.string().required(),
+//   fields: Joi.object().required(),
+//   datasourceId: Joi.string().required(),
+//   readable: Joi.boolean(),
+//   parameters: Joi.array().items(Joi.object({
+//     name: Joi.string(),
+//     default: Joi.string().allow(""),
+//   })),
+//   queryVerb: Joi.string().allow().required(),
+//   extra: Joi.object().optional(),
+//   schema: Joi.object({}).required().unknown(true),
+//   transformer: Joi.string().optional(),
+// }))
+
+interface Parameter {
+  name: string
+  default: string
+}
+
+interface Query {
+  _id?: string
+  datasourceId: string
+  name: string
+  parameters: Parameter[]
+  fields: {
+    headers: any
+    queryString: string
+    path: string
+  }
+  transformer: string | null
+  schema: any
+  readable: boolean
+  queryVerb: string
+}
+
+enum Strategy {
+  SWAGGER2,
+  OPENAPI3,
+  CURL,
+}
+
+enum MethodToVerb {
+  get = "read",
+  post = "create",
+  put = "update",
+  patch = "patch",
+  delete = "delete",
+}
+
+interface ImportResult {
+  errorQueries: Query[]
+}
+
+interface DatasourceInfo {
+  url: string
+  name: string
+  defaultHeaders: any[]
+}
+
+const parseImportStrategy = (data: string): Strategy => {
+  return Strategy.SWAGGER2
+}
+
+// SWAGGER
+
+const parseSwagger2Info = (swagger2: Swagger2): DatasourceInfo => {
+  return {
+    url: "http://localhost:3000",
+    name: "swagger",
+    defaultHeaders: [],
+  }
+}
+
+const parseSwagger2Queries = (
+  datasourceId: string,
+  swagger2: Swagger2
+): Query[] => {
+  const queries = []
+
+  for (let [pathName, path] of Object.entries(swagger2.paths)) {
+    for (let [methodName, op] of Object.entries(path)) {
+      let operation = op as Operation
+
+      const name = operation.operationId || pathName
+      const queryString = ""
+      const headers = {}
+      const parameters: Parameter[] = []
+
+      const query = constructQuery(
+        datasourceId,
+        name,
+        methodName,
+        pathName,
+        queryString,
+        headers,
+        parameters
+      )
+      queries.push(query)
+    }
+  }
+
+  return queries
+}
+
+// OPEN API
+
+const parseOpenAPI3Info = (data: any): DatasourceInfo => {
+  return {
+    url: "http://localhost:3000",
+    name: "swagger",
+    defaultHeaders: [],
+  }
+}
+
+const parseOpenAPI3Queries = (datasourceId: string, data: string): Query[] => {
+  return []
+}
+
+// CURL
+
+const parseCurlDatasourceInfo = (data: any): DatasourceInfo => {
+  return {
+    url: "http://localhost:3000",
+    name: "swagger",
+    defaultHeaders: [],
+  }
+}
+
+const parseCurlQueries = (datasourceId: string, data: string): Query[] => {
+  return []
+}
+
+const verbFromMethod = (method: string) => {
+  const verb = (<any>MethodToVerb)[method]
+  if (!verb) {
+    throw new Error(`Unsupported method: ${method}`)
+  }
+  return verb
+}
+
+const constructQuery = (
+  datasourceId: string,
+  name: string,
+  method: string,
+  path: string,
+  queryString: string,
+  headers: any = {},
+  parameters: Parameter[] = []
+): Query => {
+  const readable = true
+  const queryVerb = verbFromMethod(method)
+  const transformer = "return data"
+  const schema = {}
+
+  const query: Query = {
+    datasourceId,
+    name,
+    parameters,
+    fields: {
+      headers,
+      queryString,
+      path,
+    },
+    transformer,
+    schema,
+    readable,
+    queryVerb,
+  }
+
+  return query
+}
+
+export const getDatasourceInfo = (data: string): DatasourceInfo => {
+  const strategy = parseImportStrategy(data)
+
+  let info: DatasourceInfo
+  switch (strategy) {
+    case Strategy.SWAGGER2:
+      info = parseSwagger2Info(JSON.parse(data))
+      break
+    case Strategy.OPENAPI3:
+      info = parseOpenAPI3Info(JSON.parse(data))
+      break
+    case Strategy.CURL:
+      info = parseCurlDatasourceInfo(data)
+      break
+  }
+
+  return info
+}
+
+export const importQueries = async (
+  appId: string,
+  datasourceId: string,
+  data: string
+): Promise<ImportResult> => {
+  const strategy = parseImportStrategy(data)
+
+  // constuct the queries
+  let queries: Query[]
+  switch (strategy) {
+    case Strategy.SWAGGER2:
+      queries = parseSwagger2Queries(datasourceId, JSON.parse(data))
+      break
+    case Strategy.OPENAPI3:
+      queries = parseOpenAPI3Queries(datasourceId, JSON.parse(data))
+      break
+    case Strategy.CURL:
+      queries = parseCurlQueries(datasourceId, data)
+      break
+  }
+
+  // validate queries
+  const errorQueries = []
+  const schema = queryValidation()
+  queries = queries
+    .filter(query => {
+      const validation = schema.validate(query)
+      if (validation.error) {
+        errorQueries.push(query)
+        return false
+      }
+      return true
+    })
+    .map(query => {
+      query._id = generateQueryID(query.datasourceId)
+      return query
+    })
+
+  // persist queries
+  const db = new CouchDB(appId)
+  for (const query of queries) {
+    try {
+      await db.put(query)
+    } catch (error) {
+      errorQueries.push(query)
+    }
+  }
+
+  return {
+    errorQueries,
+  }
+}

packages/server/src/api/controllers/query/index.js

@@ -1,13 +1,11 @@
 const { processString } = require("@budibase/string-templates")
-const CouchDB = require("../../db")
+const CouchDB = require("../../../db")
-const { generateQueryID, getQueryParams } = require("../../db/utils")
+const { generateQueryID, getQueryParams } = require("../../../db/utils")
-const { BaseQueryVerbs } = require("../../constants")
+const { BaseQueryVerbs } = require("../../../constants")
-const env = require("../../environment")
+const env = require("../../../environment")
-const { Thread, ThreadType } = require("../../threads")
+const { Thread, ThreadType } = require("../../../threads")
-
+const { importQueries, getDatasourceInfo } = require("./import")
-const fetch = require("node-fetch")
+const { save: saveDatasource } = require("../datasource")
-const Joi = require("joi")
-const { save: saveDatasource } = require("./datasource")
 
 const Runner = new Thread(ThreadType.QUERY, { timeoutMs: 10000 })
 
@@ -31,134 +29,43 @@ exports.fetch = async function (ctx) {
       include_docs: true,
     })
   )
+
   ctx.body = enrichQueries(body.rows.map(row => row.doc))
 }
 
-// const query = {
+exports.import = async ctx => {
-//   datasourceId: "datasource_b9a474302a174d1295e4c273cd72bde9",
+  const body = ctx.request.body
-//   name: "available pets (import)",
+  const data = body.data
-//   parameters: [],
-//   fields: {
-//     headers: {},
-//     queryString: "status=available",
-//     path: "v2/pet/findByStatus",
-//   },
-//   queryVerb: "read",
-//   transformer: "return data",
-//   schema: {},
-//   readable: true
-// }
 
-function generateQueryValidation() {
+  let datasourceId
-  // prettier-ignore
+  if (!body.datasourceId) {
-  return Joi.object({
+    // construct new datasource
-    _id: Joi.string(),
+    const info = getDatasourceInfo(data)
-    _rev: Joi.string(),
+    let datasource = {
-    name: Joi.string().required(),
+      type: "datasource",
-    fields: Joi.object().required(),
+      source: "REST",
-    datasourceId: Joi.string().required(),
+      config: {
-    readable: Joi.boolean(),
+        url: info.url,
-    parameters: Joi.array().items(Joi.object({
+        defaultHeaders: info.defaultHeaders,
-      name: Joi.string(),
+      },
-      default: Joi.string().allow(""),
+      name: info.name,
-    })),
-    queryVerb: Joi.string().allow().required(),
-    extra: Joi.object().optional(),
-    schema: Joi.object({}).required().unknown(true),
-    transformer: Joi.string().optional(),
-  })
-}
-
-const verbs = {
-  get: "read",
-  post: "create",
-  put: "update",
-  patch: "patch",
-  delete: "delete",
-}
-
-const constructQuery = (datasource, swagger, path, method, config) => {
-  const query = {
-    datasourceId: datasource._id,
-  }
-  query.name = config.operationId || path
-  query.parameters = []
-  query.fields = {
-    headers: {},
-    // queryString: "status=available",
-    path: path,
-  }
-  query.transformer = "return data"
-  query.schema = {}
-  query.readable = true
-  query.queryVerb = verbs[method]
-
-  return query
-}
-
-// {
-//   "type": "url",
-//   "data": "www.url.com/swagger.json"
-// }
-
-exports.import = async function (ctx) {
-  const importConfig = ctx.request.body
-
-  let data
-
-  if (importConfig.type === "url") {
-    data = await fetch(importConfig.data).then(res => res.json())
-  } else if (importConfig.type === "raw") {
-    data = JSON.parse(importConfig.data)
-  } else {
-    throw new Error("Invalid data type")
-  }
-
-  const db = new CouchDB(ctx.appId)
-  const schema = generateQueryValidation()
-
-  // create datasource
-  const scheme = data.schemes.includes("https") ? "https" : "http"
-  const url = `${scheme}://${data.host}${data.basePath}`
-  const name = data.info.title
-
-  // TODO: Refactor datasource creation into shared function
-  const datasourceCtx = {
-    ...ctx,
-  }
-  datasourceCtx.request.body.datasource = {
-    type: "datasource",
-    source: "REST",
-    config: {
-      url: url,
-      defaultHeaders: {},
-    },
-    name: name,
-  }
-  await saveDatasource(datasourceCtx)
-  const datasource = datasourceCtx.body.datasource
-
-  // create query
-
-  for (const [path, method] of Object.entries(data.paths)) {
-    for (const [methodName, config] of Object.entries(method)) {
-      const query = constructQuery(datasource, data, path, methodName, config)
-
-      // validate query
-      const { error } = schema.validate(query)
-      if (error) {
-        ctx.throw(400, `Invalid - ${error.message}`)
-        return
-      }
-
-      // persist query
-      query._id = generateQueryID(query.datasourceId)
-      await db.put(query)
     }
+    // save the datasource
+    const datasourceCtx = { ...ctx }
+    datasourceCtx.request.body.datasource = datasource
+    await saveDatasource(datasourceCtx)
+    datasourceId = datasourceCtx.body.datasource._id
+  } else {
+    // use existing datasource
+    datasourceId = body.datasourceId
   }
 
-  // return the datasource
+  const importResult = await importQueries(ctx.appId, datasourceId, data)
-  ctx.body = { datasource }
+
+  ctx.body = {
+    ...importResult,
+    datasourceId,
+  }
   ctx.status = 200
 }
 

packages/server/src/api/controllers/query/validation.js

@@ -0,0 +1,40 @@
+const joiValidator = require("../../../middleware/joi-validator")
+const Joi = require("joi")
+
+exports.queryValidation = () => {
+  return Joi.object({
+    _id: Joi.string(),
+    _rev: Joi.string(),
+    name: Joi.string().required(),
+    fields: Joi.object().required(),
+    datasourceId: Joi.string().required(),
+    readable: Joi.boolean(),
+    parameters: Joi.array().items(
+      Joi.object({
+        name: Joi.string(),
+        default: Joi.string().allow(""),
+      })
+    ),
+    queryVerb: Joi.string().allow().required(),
+    extra: Joi.object().optional(),
+    schema: Joi.object({}).required().unknown(true),
+    transformer: Joi.string().optional(),
+  })
+}
+
+exports.generateQueryValidation = () => {
+  // prettier-ignore
+  return joiValidator.body(exports.queryValidation())
+}
+
+exports.generateQueryPreviewValidation = () => {
+  // prettier-ignore
+  return joiValidator.body(Joi.object({
+    fields: Joi.object().required(),
+    queryVerb: Joi.string().allow().required(),
+    extra: Joi.object().optional(),
+    datasourceId: Joi.string().required(),
+    transformer: Joi.string().optional(),
+    parameters: Joi.object({}).required().unknown(true)
+  }))
+}

packages/server/src/api/routes/query.js

@@ -1,53 +1,23 @@
 const Router = require("@koa/router")
 const queryController = require("../controllers/query")
 const authorized = require("../../middleware/authorized")
-const Joi = require("joi")
 const {
   PermissionLevels,
   PermissionTypes,
   BUILDER,
 } = require("@budibase/auth/permissions")
-const joiValidator = require("../../middleware/joi-validator")
 const {
   bodyResource,
   bodySubResource,
   paramResource,
 } = require("../../middleware/resourceId")
+const {
+  generateQueryPreviewValidation,
+  generateQueryValidation,
+} = require("../controllers/query/validation")
 
 const router = Router()
 
-function generateQueryValidation() {
-  // prettier-ignore
-  return joiValidator.body(Joi.object({
-    _id: Joi.string(),
-    _rev: Joi.string(),
-    name: Joi.string().required(),
-    fields: Joi.object().required(),
-    datasourceId: Joi.string().required(),
-    readable: Joi.boolean(),
-    parameters: Joi.array().items(Joi.object({
-      name: Joi.string(),
-      default: Joi.string().allow(""),
-    })),
-    queryVerb: Joi.string().allow().required(),
-    extra: Joi.object().optional(),
-    schema: Joi.object({}).required().unknown(true),
-    transformer: Joi.string().optional(),
-  }))
-}
-
-function generateQueryPreviewValidation() {
-  // prettier-ignore
-  return joiValidator.body(Joi.object({
-    fields: Joi.object().required(),
-    queryVerb: Joi.string().allow().required(),
-    extra: Joi.object().optional(),
-    datasourceId: Joi.string().required(),
-    transformer: Joi.string().optional(),
-    parameters: Joi.object({}).required().unknown(true)
-  }))
-}
-
 router
   .get("/api/queries", authorized(BUILDER), queryController.fetch)
   .post(
@@ -57,11 +27,7 @@ router
     generateQueryValidation(),
     queryController.save
   )
-  .post(
+  .post("/api/queries/import", authorized(BUILDER), queryController.import)
-    "/api/queries/import/swagger2",
-    authorized(BUILDER),
-    queryController.import
-  )
   .post(
     "/api/queries/preview",
     bodyResource("datasourceId"),