From 71ba024974fa5681865a42d88cf0114c2184d3a0 Mon Sep 17 00:00:00 2001
From: Rory Powell
Date: Thu, 25 Nov 2021 17:14:07 +0000
Subject: [PATCH] WIP: Rest API import
---
packages/bbui/src/Form/Core/Dropzone.svelte | 27 ++++
packages/bbui/src/Form/Dropzone.svelte | 2 +
packages/bbui/src/Modal/ModalContent.svelte | 25 ++++
.../modals/CreateDatasourceModal.svelte | 24 ++++
.../modals/ImportRestDatasourceModal.svelte | 103 ++++++++++++++
.../builder/src/stores/backend/datasources.js | 5 +
packages/builder/yarn.lock | 28 ++--
packages/server/src/api/controllers/query.js | 130 ++++++++++++++++++
packages/server/src/api/routes/query.js | 5 +
packages/server/src/integrations/index.ts | 4 +-
packages/server/src/integrations/oracle.ts | 29 ++--
packages/server/src/integrations/utils.ts | 8 +-
12 files changed, 364 insertions(+), 26 deletions(-)
create mode 100644 packages/builder/src/components/backend/DatasourceNavigator/modals/ImportRestDatasourceModal.svelte
diff --git a/packages/bbui/src/Form/Core/Dropzone.svelte b/packages/bbui/src/Form/Core/Dropzone.svelte
index a543dea1d4..1595a2ea92 100644
--- a/packages/bbui/src/Form/Core/Dropzone.svelte
+++ b/packages/bbui/src/Form/Core/Dropzone.svelte
@@ -6,6 +6,8 @@
import { generateID } from "../../utils/helpers"
import Icon from "../../Icon/Icon.svelte"
import Link from "../../Link/Link.svelte"
+ import Tag from "../../Tags/Tag.svelte"
+ import Tags from "../../Tags/Tags.svelte"
const BYTES_IN_KB = 1000
const BYTES_IN_MB = 1000000
@@ -18,6 +20,7 @@
export let handleFileTooLarge = null
export let gallery = true
export let error = null
+ export let fileTags = []
const dispatch = createEventDispatcher()
const imageExtensions = [
@@ -278,6 +281,19 @@
from your computer
+ {#if fileTags.length}
+
+
+
+ {/if}
{/if}
@@ -390,4 +406,15 @@
.disabled .spectrum-Heading--sizeL {
color: var(--spectrum-alias-text-color-disabled);
}
+
+ .tags {
+ margin-top: 20px;
+ display: flex;
+ flex-wrap: wrap;
+ justify-content: center;
+ }
+
+ .tag {
+ margin-top: 8px;
+ }
diff --git a/packages/bbui/src/Form/Dropzone.svelte b/packages/bbui/src/Form/Dropzone.svelte
index f8facaada9..a8a160010d 100644
--- a/packages/bbui/src/Form/Dropzone.svelte
+++ b/packages/bbui/src/Form/Dropzone.svelte
@@ -12,6 +12,7 @@
export let processFiles = undefined
export let handleFileTooLarge = undefined
export let gallery = true
+ export let fileTags = []
const dispatch = createEventDispatcher()
const onChange = e => {
@@ -29,6 +30,7 @@
{processFiles}
{handleFileTooLarge}
{gallery}
+ {fileTags}
on:change={onChange}
/>
diff --git a/packages/bbui/src/Modal/ModalContent.svelte b/packages/bbui/src/Modal/ModalContent.svelte
index 09cc4f6c52..7f2d7fbdb9 100644
--- a/packages/bbui/src/Modal/ModalContent.svelte
+++ b/packages/bbui/src/Modal/ModalContent.svelte
@@ -18,10 +18,22 @@
export let disabled = false
export let showDivider = true
+ export let showSecondaryButton = false
+ export let secondaryButtonText = undefined
+ export let secondaryAction = undefined
+
const { hide, cancel } = getContext(Context.Modal)
let loading = false
$: confirmDisabled = disabled || loading
+ async function secondary() {
+ loading = true
+ if (!secondaryAction || (await secondaryAction()) !== false) {
+ hide()
+ }
+ loading = false
+ }
+
async function confirm() {
loading = true
if (!onConfirm || (await onConfirm()) !== false) {
@@ -73,6 +85,15 @@
class="spectrum-ButtonGroup spectrum-Dialog-buttonGroup spectrum-Dialog-buttonGroup--noFooter"
>
+
+ {#if showSecondaryButton && secondaryButtonText && secondaryAction}
+
+
+
+ {/if}
+
{#if showCancelButton}
{/if}
@@ -136,4 +157,8 @@
display: flex;
justify-content: space-between;
}
+
+ .secondary-action {
+ margin-right: auto;
+ }
diff --git a/packages/builder/src/components/backend/DatasourceNavigator/modals/CreateDatasourceModal.svelte b/packages/builder/src/components/backend/DatasourceNavigator/modals/CreateDatasourceModal.svelte
index 1a433785dc..1fc3937ba0 100644
--- a/packages/builder/src/components/backend/DatasourceNavigator/modals/CreateDatasourceModal.svelte
+++ b/packages/builder/src/components/backend/DatasourceNavigator/modals/CreateDatasourceModal.svelte
@@ -6,12 +6,18 @@
import { IntegrationNames } from "constants"
import CreateTableModal from "components/backend/TableNavigator/modals/CreateTableModal.svelte"
import DatasourceConfigModal from "components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte"
+ import ImportRestDatasourceModal from "./ImportRestDatasourceModal.svelte"
export let modal
let integrations = []
let integration = {}
let internalTableModal
let externalDatasourceModal
+ let importModal
+
+ $: showImportButton = false
+
+ checkShowImport()
const INTERNAL = "BUDIBASE"
@@ -33,6 +39,15 @@
config,
schema: selected.datasource,
}
+ checkShowImport()
+ }
+
+ function checkShowImport() {
+ showImportButton = integration.type === "REST"
+ }
+
+ function showImportModal() {
+ importModal.show()
}
function chooseNextModal() {
@@ -63,11 +78,20 @@
+
+ {#if integration.type === "REST"}
+
+ {/if}
+
+
showImportModal()}
showCancelButton={false}
size="M"
onConfirm={() => {
diff --git a/packages/builder/src/components/backend/DatasourceNavigator/modals/ImportRestDatasourceModal.svelte b/packages/builder/src/components/backend/DatasourceNavigator/modals/ImportRestDatasourceModal.svelte
new file mode 100644
index 0000000000..dd25880491
--- /dev/null
+++ b/packages/builder/src/components/backend/DatasourceNavigator/modals/ImportRestDatasourceModal.svelte
@@ -0,0 +1,103 @@
+
+
+ importDatasource()}
+ onCancel={() => modal.show()}
+ confirmText={"Import"}
+ cancelText="Back"
+ size="L"
+>
+
+ Import
+ Import your rest collection using one of the options below
+
+
+ (lastTouched = "url")}
+ label="Enter a URL"
+ placeholder="e.g. https://petstore.swagger.io/v2/swagger.json"
+ />
+
+
+ (lastTouched = "file")}
+ fileTags={["OpenAPI", "Swagger 2.0"]}
+ />
+
+
+
+
+
+
+
+
diff --git a/packages/builder/src/stores/backend/datasources.js b/packages/builder/src/stores/backend/datasources.js
index 7810c3a950..2d4fbdfb0b 100644
--- a/packages/builder/src/stores/backend/datasources.js
+++ b/packages/builder/src/stores/backend/datasources.js
@@ -84,6 +84,11 @@ export function createDatasourcesStore() {
return updateDatasource(response)
},
+ import: async body => {
+ let response
+ response = await api.post(`/api/queries/import/swagger2`, body)
+ return updateDatasource(response)
+ },
delete: async datasource => {
const response = await api.delete(
`/api/datasources/${datasource._id}/${datasource._rev}`
diff --git a/packages/builder/yarn.lock b/packages/builder/yarn.lock
index f3d3362bd3..b76300b9ea 100644
--- a/packages/builder/yarn.lock
+++ b/packages/builder/yarn.lock
@@ -969,10 +969,10 @@
svelte-flatpickr "^3.2.3"
svelte-portal "^1.0.0"
-"@budibase/bbui@^0.9.185-alpha.12", "@budibase/bbui@^0.9.188":
- version "0.9.188"
- resolved "https://registry.yarnpkg.com/@budibase/bbui/-/bbui-0.9.188.tgz#82c108172fbf81a84378e0ef4ca7cba61ea8d0ba"
- integrity sha512-KevJxHdASITX9RzLvm+b2K3VMwqYFTumvrlpStAP6UIoyPkls0xaAc2KiJJ7Kkq48UkkBtAbOYaMxsFbAaTsbQ==
+"@budibase/bbui@^0.9.185-alpha.21", "@budibase/bbui@^0.9.189":
+ version "0.9.189"
+ resolved "https://registry.yarnpkg.com/@budibase/bbui/-/bbui-0.9.189.tgz#d5802e9b6aabccdef4205f0edfa7ed5616ac1aff"
+ integrity sha512-YqM21mtrg8yTN9mqG4CnFfvoOelmhy3V69LyoITdQT6aGiwt/efHzknSlaUH3/0yLH9MuzwkHDzUmbe7QrsqEA==
dependencies:
"@adobe/spectrum-css-workflow-icons" "^1.2.1"
"@spectrum-css/actionbutton" "^1.0.1"
@@ -1018,14 +1018,14 @@
svelte-flatpickr "^3.2.3"
svelte-portal "^1.0.0"
-"@budibase/client@^0.9.185-alpha.12":
- version "0.9.188"
- resolved "https://registry.yarnpkg.com/@budibase/client/-/client-0.9.188.tgz#c5bf6f3bccdb370b236b9e69e0118334ad3eccfd"
- integrity sha512-yP2WLWb2yQAwPBxVzpNGjSHpATMZMzcxl2gK6vw662F7YC8xGFHNfZZqEwPvrVwnx+d7LFZR/kJxJOvvk7YCVw==
+"@budibase/client@^0.9.185-alpha.21":
+ version "0.9.189"
+ resolved "https://registry.yarnpkg.com/@budibase/client/-/client-0.9.189.tgz#8f96b607f36bbb7390fd53b04360851f0c12aaac"
+ integrity sha512-L2i3CaQt4aFL7JKkRrEWWx8NemHTEOKLXvXq7LGM4u3GlcFIIkcL113EkXQT1bIZcf6AuuC2CfNsmZKioOPh2A==
dependencies:
- "@budibase/bbui" "^0.9.188"
+ "@budibase/bbui" "^0.9.189"
"@budibase/standard-components" "^0.9.139"
- "@budibase/string-templates" "^0.9.188"
+ "@budibase/string-templates" "^0.9.189"
regexparam "^1.3.0"
shortid "^2.2.15"
svelte-spa-router "^3.0.5"
@@ -1080,10 +1080,10 @@
svelte-apexcharts "^1.0.2"
svelte-flatpickr "^3.1.0"
-"@budibase/string-templates@^0.9.185-alpha.12", "@budibase/string-templates@^0.9.188":
- version "0.9.188"
- resolved "https://registry.yarnpkg.com/@budibase/string-templates/-/string-templates-0.9.188.tgz#f37836ed23dbd2217cb157030ada7cd59f6a2165"
- integrity sha512-O/bL0I5OJO9W2OizIe9vBHowCLwwASPBrsGiAIB8L0x6AivYMq8j1mvNRwLXZjpHTjv86bU/LyG/3CP837oDsg==
+"@budibase/string-templates@^0.9.185-alpha.21", "@budibase/string-templates@^0.9.189":
+ version "0.9.189"
+ resolved "https://registry.yarnpkg.com/@budibase/string-templates/-/string-templates-0.9.189.tgz#515a4ce85da550ce19d78c4c592ab5839a8ebe3a"
+ integrity sha512-JmnpuPx1CItNIFCMUxBz+4DVpYu96QxteU2Vi17pjWb0B7qsWwHkmcMmYbd+iTW4oxgOufbP8slfyfbu2XhbzA==
dependencies:
"@budibase/handlebars-helpers" "^0.11.7"
dayjs "^1.10.4"
diff --git a/packages/server/src/api/controllers/query.js b/packages/server/src/api/controllers/query.js
index 502ef5e67b..13b14a6343 100644
--- a/packages/server/src/api/controllers/query.js
+++ b/packages/server/src/api/controllers/query.js
@@ -5,6 +5,10 @@ const { BaseQueryVerbs } = require("../../constants")
const env = require("../../environment")
const { Thread, ThreadType } = require("../../threads")
+const fetch = require("node-fetch")
+const Joi = require("joi")
+const { save: saveDatasource } = require("./datasource")
+
const Runner = new Thread(ThreadType.QUERY, { timeoutMs: 10000 })
// simple function to append "readable" to all read queries
@@ -30,6 +34,132 @@ exports.fetch = async function (ctx) {
ctx.body = enrichQueries(body.rows.map(row => row.doc))
}
+// const query = {
+// datasourceId: "datasource_b9a474302a174d1295e4c273cd72bde9",
+// name: "available pets (import)",
+// parameters: [],
+// fields: {
+// headers: {},
+// queryString: "status=available",
+// path: "v2/pet/findByStatus",
+// },
+// queryVerb: "read",
+// transformer: "return data",
+// schema: {},
+// readable: true
+// }
+
+function generateQueryValidation() {
+ // prettier-ignore
+ return Joi.object({
+ _id: Joi.string(),
+ _rev: Joi.string(),
+ name: Joi.string().required(),
+ fields: Joi.object().required(),
+ datasourceId: Joi.string().required(),
+ readable: Joi.boolean(),
+ parameters: Joi.array().items(Joi.object({
+ name: Joi.string(),
+ default: Joi.string().allow(""),
+ })),
+ queryVerb: Joi.string().allow().required(),
+ extra: Joi.object().optional(),
+ schema: Joi.object({}).required().unknown(true),
+ transformer: Joi.string().optional(),
+ })
+}
+
+const verbs = {
+ get: "read",
+ post: "create",
+ put: "update",
+ patch: "patch",
+ delete: "delete",
+}
+
+const constructQuery = (datasource, swagger, path, method, config) => {
+ const query = {
+ datasourceId: datasource._id,
+ }
+ query.name = config.operationId || path
+ query.parameters = []
+ query.fields = {
+ headers: {},
+ // queryString: "status=available",
+ path: path,
+ }
+ query.transformer = "return data"
+ query.schema = {}
+ query.readable = true
+ query.queryVerb = verbs[method]
+
+ return query
+}
+
+// {
+// "type": "url",
+// "data": "www.url.com/swagger.json"
+// }
+
+exports.import = async function (ctx) {
+ const importConfig = ctx.request.body
+
+ let data
+
+ if (importConfig.type === "url") {
+ data = await fetch(importConfig.data).then(res => res.json())
+ } else if (importConfig.type === "raw") {
+ data = JSON.parse(importConfig.data)
+ }
+
+ const db = new CouchDB(ctx.appId)
+ const schema = generateQueryValidation()
+
+ // create datasource
+ const scheme = data.schemes.includes("https") ? "https" : "http"
+ const url = `${scheme}://${data.host}${data.basePath}`
+ const name = data.info.title
+
+ // TODO: Refactor datasource creation into shared function
+ const datasourceCtx = {
+ ...ctx,
+ }
+ datasourceCtx.request.body.datasource = {
+ type: "datasource",
+ source: "REST",
+ config: {
+ url: url,
+ defaultHeaders: {},
+ },
+ name: name,
+ }
+ await saveDatasource(datasourceCtx)
+ const datasource = datasourceCtx.body.datasource
+
+ // create query
+
+ for (const [path, method] of Object.entries(data.paths)) {
+ for (const [methodName, config] of Object.entries(method)) {
+ const query = constructQuery(datasource, data, path, methodName, config)
+
+ // validate query
+ const { error } = schema.validate(query)
+ if (error) {
+ ctx.throw(400, `Invalid - ${error.message}`)
+ return
+ }
+
+ // persist query
+ query._id = generateQueryID(query.datasourceId)
+ await db.put(query)
+ }
+ }
+
+ // return the datasource
+ ctx.body = { datasource }
+ ctx.status = 200
+}
+
exports.save = async function (ctx) {
const db = new CouchDB(ctx.appId)
const query = ctx.request.body
diff --git a/packages/server/src/api/routes/query.js b/packages/server/src/api/routes/query.js
index 3fca0c1e0f..bc1c2204de 100644
--- a/packages/server/src/api/routes/query.js
+++ b/packages/server/src/api/routes/query.js
@@ -57,6 +57,11 @@ router
generateQueryValidation(),
queryController.save
)
+ .post(
+ "/api/queries/import/swagger2",
+ authorized(BUILDER),
+ queryController.import
+ )
.post(
"/api/queries/preview",
bodyResource("datasourceId"),
diff --git a/packages/server/src/integrations/index.ts b/packages/server/src/integrations/index.ts
index eff499f978..8f2f083fc5 100644
--- a/packages/server/src/integrations/index.ts
+++ b/packages/server/src/integrations/index.ts
@@ -40,9 +40,9 @@ const INTEGRATIONS = {
}
// optionally add oracle integration if the oracle binary can be installed
-if (!(process.arch === 'arm64' && process.platform === 'darwin')) {
+if (!(process.arch === "arm64" && process.platform === "darwin")) {
const oracle = require("./oracle")
- DEFINITIONS[SourceNames.ORACLE] = oracle.schema
+ DEFINITIONS[SourceNames.ORACLE] = oracle.schema
INTEGRATIONS[SourceNames.ORACLE] = oracle.integration
}
diff --git a/packages/server/src/integrations/oracle.ts b/packages/server/src/integrations/oracle.ts
index c74977bece..13658399db 100644
--- a/packages/server/src/integrations/oracle.ts
+++ b/packages/server/src/integrations/oracle.ts
@@ -352,14 +352,23 @@ module OracleModule {
* Knex default returning behaviour does not work with oracle
* Manually add the behaviour for the return column
*/
- private addReturning(query: SqlQuery, bindings: BindParameters, returnColumn: string) {
+ private addReturning(
+ query: SqlQuery,
+ bindings: BindParameters,
+ returnColumn: string
+ ) {
if (bindings instanceof Array) {
bindings.push({ dir: oracledb.BIND_OUT })
- query.sql = query.sql + ` returning \"${returnColumn}\" into :${bindings.length}`
+ query.sql =
+ query.sql + ` returning \"${returnColumn}\" into :${bindings.length}`
}
}
- private async internalQuery(query: SqlQuery, returnColum?: string, operation?: string): Promise> {
+ private async internalQuery(
+ query: SqlQuery,
+ returnColum?: string,
+ operation?: string
+ ): Promise> {
let connection
try {
connection = await this.getConnection()
@@ -367,7 +376,10 @@ module OracleModule {
const options: ExecuteOptions = { autoCommit: true }
const bindings: BindParameters = query.bindings || []
- if (returnColum && (operation === Operation.CREATE || operation === Operation.UPDATE)) {
+ if (
+ returnColum &&
+ (operation === Operation.CREATE || operation === Operation.UPDATE)
+ ) {
this.addReturning(query, bindings, returnColum)
}
@@ -414,14 +426,14 @@ module OracleModule {
return response.rows ? response.rows : []
}
- async update(query: SqlQuery | string): Promise {
+ async update(query: SqlQuery | string): Promise {
const response = await this.internalQuery(getSqlQuery(query))
return response.rows && response.rows.length
? response.rows
: [{ updated: true }]
}
- async delete(query: SqlQuery | string): Promise {
+ async delete(query: SqlQuery | string): Promise {
const response = await this.internalQuery(getSqlQuery(query))
return response.rows && response.rows.length
? response.rows
@@ -431,8 +443,9 @@ module OracleModule {
async query(json: QueryJson) {
const primaryKeys = json.meta!.table!.primary
const primaryKey = primaryKeys ? primaryKeys[0] : undefined
- const queryFn = (query: any, operation: string) => this.internalQuery(query, primaryKey, operation)
- const processFn = (response: any) => response.rows ? response.rows : []
+ const queryFn = (query: any, operation: string) =>
+ this.internalQuery(query, primaryKey, operation)
+ const processFn = (response: any) => (response.rows ? response.rows : [])
const output = await this.queryWithReturning(json, queryFn, processFn)
return output
}
diff --git a/packages/server/src/integrations/utils.ts b/packages/server/src/integrations/utils.ts
index 07cf0505bf..97380b1b5b 100644
--- a/packages/server/src/integrations/utils.ts
+++ b/packages/server/src/integrations/utils.ts
@@ -2,7 +2,11 @@ import { SqlQuery } from "../definitions/datasource"
import { Datasource, Table } from "../definitions/common"
import { SourceNames } from "../definitions/datasource"
const { DocumentTypes, SEPARATOR } = require("../db/utils")
-const { FieldTypes, BuildSchemaErrors, InvalidColumns } = require("../constants")
+const {
+ FieldTypes,
+ BuildSchemaErrors,
+ InvalidColumns,
+} = require("../constants")
const DOUBLE_SEPARATOR = `${SEPARATOR}${SEPARATOR}`
const ROW_ID_REGEX = /^\[.*]$/g
@@ -42,7 +46,7 @@ export enum SqlClients {
MS_SQL = "mssql",
POSTGRES = "pg",
MY_SQL = "mysql",
- ORACLE = "oracledb"
+ ORACLE = "oracledb",
}
export function isExternalTable(tableId: string) {