From 7a6e677b8927587ec771f3c9bfb18a1b51cd1f99 Mon Sep 17 00:00:00 2001 From: sovlookup <805408477@qq.com> Date: Thu, 28 Jan 2021 21:42:40 +0800 Subject: [PATCH 01/16] add arangodb integration --- .../DatasourceNavigator/icons/ArangoDB.svelte | 22 +++++ .../DatasourceNavigator/icons/index.js | 2 + packages/server/package.json | 1 + packages/server/src/integrations/arangodb.js | 87 +++++++++++++++++++ packages/server/src/integrations/index.js | 3 + 5 files changed, 115 insertions(+) create mode 100644 packages/builder/src/components/backend/DatasourceNavigator/icons/ArangoDB.svelte create mode 100644 packages/server/src/integrations/arangodb.js diff --git a/packages/builder/src/components/backend/DatasourceNavigator/icons/ArangoDB.svelte b/packages/builder/src/components/backend/DatasourceNavigator/icons/ArangoDB.svelte new file mode 100644 index 0000000000..0be4ca2663 --- /dev/null +++ b/packages/builder/src/components/backend/DatasourceNavigator/icons/ArangoDB.svelte @@ -0,0 +1,22 @@ + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/packages/builder/src/components/backend/DatasourceNavigator/icons/index.js b/packages/builder/src/components/backend/DatasourceNavigator/icons/index.js index 4f8d79dbf4..ff38782288 100644 --- a/packages/builder/src/components/backend/DatasourceNavigator/icons/index.js +++ b/packages/builder/src/components/backend/DatasourceNavigator/icons/index.js @@ -7,6 +7,7 @@ import S3 from "./S3.svelte" import Airtable from "./Airtable.svelte" import SqlServer from "./SQLServer.svelte" import MySQL from "./MySQL.svelte" +import ArangoDB from "./ArangoDB.svelte" export default { POSTGRES: Postgres, @@ -18,4 +19,5 @@ export default { S3: S3, AIRTABLE: Airtable, MYSQL: MySQL, + ARANGODB: ArangoDB, } diff --git a/packages/server/package.json b/packages/server/package.json index fd8d5196a0..2ee2c79e22 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -56,6 +56,7 @@ "@sendgrid/mail": "^7.1.1", "@sentry/node": "^5.19.2", "airtable": "^0.10.1", + "arangojs": "^7.2.0", "aws-sdk": "^2.767.0", "bcryptjs": "^2.4.3", "chmodr": "^1.2.0", diff --git a/packages/server/src/integrations/arangodb.js b/packages/server/src/integrations/arangodb.js new file mode 100644 index 0000000000..eefd656cbc --- /dev/null +++ b/packages/server/src/integrations/arangodb.js @@ -0,0 +1,87 @@ +const arangojs = require("arangojs") +const { FIELD_TYPES, QUERY_TYPES } = require("./Integration") +// const arangodb = arangojs.Database +const aql = arangojs.aql + +const SCHEMA = { + docs: "https://github.com/arangodb/arangojs", + datasource: { + url: { + type: FIELD_TYPES.STRING, + default: "http://localhost:8529", + required: true, + }, + username: { + type: FIELD_TYPES.STRING, + default: "root", + required: true, + }, + password: { + type: FIELD_TYPES.PASSWORD, + required: true, + }, + databaseName: { + type: FIELD_TYPES.STRING, + default: "_system", + required: true, + }, + collection: { + type: FIELD_TYPES.STRING, + required: true, + }, + }, + query: { + read: { + type: QUERY_TYPES.SQL, + }, + create: { + type: QUERY_TYPES.JSON, + }, + }, +} + +class ArangoDBIntegration { + constructor(config) { + config.auth = { + username: config.username, + password: config.password, + } + + this.config = config + this.client = arangojs.arangojs(config) + } + + async read(query) { + try { + const result = await this.client.query(query.sql) + let rl = [] + await result.forEach((r)=> rl.push(r)) + return rl + } catch (err) { + console.error("Error querying arangodb",err.message) + throw err + } finally { + this.client.close() + } + } + + async create(query) { + const clc = this.client.collection(this.config.collection) + try { + const result = await this.client.query(aql`INSERT ${query.json} INTO ${clc} LET n = NEW RETURN NEW`) + let rl = [] + await result.forEach((r)=> rl.push(r)) + return rl + } catch (err) { + console.error("Error querying arangodb",err.message) + throw err + } finally { + this.client.close() + } + } +} + +module.exports = { + schema: SCHEMA, + integration: ArangoDBIntegration, +} \ No newline at end of file diff --git a/packages/server/src/integrations/index.js b/packages/server/src/integrations/index.js index acb81992a9..8ff89f4900 100644 --- a/packages/server/src/integrations/index.js +++ b/packages/server/src/integrations/index.js @@ -7,6 +7,7 @@ const sqlServer = require("./microsoftSqlServer") const s3 = require("./s3") const airtable = require("./airtable") const mysql = require("./mysql") +const arangodb = require("./arangodb") const DEFINITIONS = { POSTGRES: postgres.schema, @@ -18,6 +19,7 @@ const DEFINITIONS = { S3: s3.schema, AIRTABLE: airtable.schema, MYSQL: mysql.schema, + ARANGODB: arangodb.schema, } const INTEGRATIONS = { @@ -30,6 +32,7 @@ const INTEGRATIONS = { SQL_SERVER: sqlServer.integration, AIRTABLE: airtable.integration, MYSQL: mysql.integration, + ARANGODB:　arangodb.integration, } module.exports = { From 1b6eb426c68e3d29a1057b13aeb1c6ac656b2d0f Mon Sep 17 00:00:00 2001 From: sovlookup <805408477@qq.com> Date: Thu, 28 Jan 2021 21:48:29 +0800 Subject: [PATCH 02/16] add arangodb integration --- packages/server/src/integrations/arangodb.js | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/packages/server/src/integrations/arangodb.js b/packages/server/src/integrations/arangodb.js index eefd656cbc..f2aa54667e 100644 --- a/packages/server/src/integrations/arangodb.js +++ b/packages/server/src/integrations/arangodb.js @@ -1,7 +1,5 @@ -const arangojs = require("arangojs") +const { Database, aql } = require("arangojs") const { FIELD_TYPES, QUERY_TYPES } = require("./Integration") -// const arangodb = arangojs.Database -const aql = arangojs.aql const SCHEMA = { docs: "https://github.com/arangodb/arangojs", @@ -48,7 +46,7 @@ class ArangoDBIntegration { } this.config = config - this.client = arangojs.arangojs(config) + this.client = new Database(config) } async read(query) { From 28ddffb8620d18612357ced105ed1527fa3e0c02 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 28 Jan 2021 14:03:19 +0000 Subject: [PATCH 03/16] Fixing an issue that aptkingston found with relationships were links on the way out would be multiplied (due to the bi-directional nature). --- packages/server/src/db/linkedRows/linkUtils.js | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/packages/server/src/db/linkedRows/linkUtils.js b/packages/server/src/db/linkedRows/linkUtils.js index 3a9aff6c33..5f9dca4088 100644 --- a/packages/server/src/db/linkedRows/linkUtils.js +++ b/packages/server/src/db/linkedRows/linkUtils.js @@ -77,11 +77,21 @@ exports.getLinkDocuments = async function({ } params.include_docs = !!includeDocs try { - const response = await db.query(getQueryIndex(ViewNames.LINK), params) + let linkRows = (await db.query(getQueryIndex(ViewNames.LINK), params)).rows + // filter to get unique entries + const foundIds = [] + linkRows = linkRows.filter(link => { + const unique = foundIds.indexOf(link.id) === -1 + if (unique) { + foundIds.push(link.id) + } + return unique + }) + if (includeDocs) { - return response.rows.map(row => row.doc) + return linkRows.map(row => row.doc) } else { - return response.rows.map(row => row.value) + return linkRows.map(row => row.value) } } catch (err) { // check if the view doesn't exist, it should for all new instances From 8996200d8deba244a67932c3607fd71dda16afa2 Mon Sep 17 00:00:00 2001 From: sovlookup <805408477@qq.com> Date: Thu, 28 Jan 2021 22:17:39 +0800 Subject: [PATCH 04/16] add arangodb integration --- packages/server/src/integrations/arangodb.js | 136 ++++++++++--------- 1 file changed, 69 insertions(+), 67 deletions(-) diff --git a/packages/server/src/integrations/arangodb.js b/packages/server/src/integrations/arangodb.js index f2aa54667e..2b3ee1b11b 100644 --- a/packages/server/src/integrations/arangodb.js +++ b/packages/server/src/integrations/arangodb.js @@ -2,84 +2,86 @@ const { Database, aql } = require("arangojs") const { FIELD_TYPES, QUERY_TYPES } = require("./Integration") const SCHEMA = { - docs: "https://github.com/arangodb/arangojs", - datasource: { - url: { - type: FIELD_TYPES.STRING, - default: "http://localhost:8529", - required: true, - }, - username: { - type: FIELD_TYPES.STRING, - default: "root", - required: true, - }, - password: { - type: FIELD_TYPES.PASSWORD, - required: true, - }, - databaseName: { - type: FIELD_TYPES.STRING, - default: "_system", - required: true, - }, - collection: { - type: FIELD_TYPES.STRING, - required: true, - }, + docs: "https://github.com/arangodb/arangojs", + datasource: { + url: { + type: FIELD_TYPES.STRING, + default: "http://localhost:8529", + required: true, }, - query: { - read: { - type: QUERY_TYPES.SQL, - }, - create: { - type: QUERY_TYPES.JSON, - }, + username: { + type: FIELD_TYPES.STRING, + default: "root", + required: true, }, + password: { + type: FIELD_TYPES.PASSWORD, + required: true, + }, + databaseName: { + type: FIELD_TYPES.STRING, + default: "_system", + required: true, + }, + collection: { + type: FIELD_TYPES.STRING, + required: true, + }, + }, + query: { + read: { + type: QUERY_TYPES.SQL, + }, + create: { + type: QUERY_TYPES.JSON, + }, + }, } class ArangoDBIntegration { - constructor(config) { - config.auth = { - username: config.username, - password: config.password, - } - - this.config = config - this.client = new Database(config) + constructor(config) { + config.auth = { + username: config.username, + password: config.password, } - async read(query) { - try { - const result = await this.client.query(query.sql) - let rl = [] - await result.forEach((r)=> rl.push(r)) - return rl - } catch (err) { - console.error("Error querying arangodb",err.message) - throw err - } finally { - this.client.close() - } - } + this.config = config + this.client = new Database(config) + } - async create(query) { - const clc = this.client.collection(this.config.collection) - try { - const result = await this.client.query(aql`INSERT ${query.json} INTO ${clc} LET n = NEW RETURN NEW`) - let rl = [] - await result.forEach((r)=> rl.push(r)) - return rl - } catch (err) { - console.error("Error querying arangodb",err.message) - throw err - } finally { - this.client.close() - } + async read(query) { + try { + const result = await this.client.query(query.sql) + let rl = [] + await result.forEach(r => rl.push(r)) + return rl + } catch (err) { + console.error("Error querying arangodb", err.message) + throw err + } finally { + this.client.close() } + } + + async create(query) { + const clc = this.client.collection(this.config.collection) + try { + const result = await this.client.query( + aql`INSERT ${query.json} INTO ${clc} LET n = NEW RETURN NEW` + ) + let rl = [] + await result.forEach(r => rl.push(r)) + return rl + } catch (err) { + console.error("Error querying arangodb", err.message) + throw err + } finally { + this.client.close() + } + } } module.exports = { schema: SCHEMA, integration: ArangoDBIntegration, -} \ No newline at end of file +} From 0b8987f6a1411e61d1c95f8e52480d938bd745bf Mon Sep 17 00:00:00 2001 From: sovlookup <805408477@qq.com> Date: Thu, 28 Jan 2021 22:20:15 +0800 Subject: [PATCH 05/16] add arangodb integration --- packages/server/src/integrations/arangodb.js | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/packages/server/src/integrations/arangodb.js b/packages/server/src/integrations/arangodb.js index 2b3ee1b11b..c886b6b918 100644 --- a/packages/server/src/integrations/arangodb.js +++ b/packages/server/src/integrations/arangodb.js @@ -52,9 +52,7 @@ class ArangoDBIntegration { async read(query) { try { const result = await this.client.query(query.sql) - let rl = [] - await result.forEach(r => rl.push(r)) - return rl + return result.forEach(r => rl.push(r)) } catch (err) { console.error("Error querying arangodb", err.message) throw err @@ -69,9 +67,7 @@ class ArangoDBIntegration { const result = await this.client.query( aql`INSERT ${query.json} INTO ${clc} LET n = NEW RETURN NEW` ) - let rl = [] - await result.forEach(r => rl.push(r)) - return rl + return result.forEach(r => rl.push(r)) } catch (err) { console.error("Error querying arangodb", err.message) throw err From 9684def1ae97c1b115be0581e650bf54e156a2e2 Mon Sep 17 00:00:00 2001 From: sovlookup <805408477@qq.com> Date: Thu, 28 Jan 2021 22:23:17 +0800 Subject: [PATCH 06/16] fix a little space --- packages/server/src/integrations/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/integrations/index.js b/packages/server/src/integrations/index.js index 8ff89f4900..9f9d5a1530 100644 --- a/packages/server/src/integrations/index.js +++ b/packages/server/src/integrations/index.js @@ -32,7 +32,7 @@ const INTEGRATIONS = { SQL_SERVER: sqlServer.integration, AIRTABLE: airtable.integration, MYSQL: mysql.integration, - ARANGODB:　arangodb.integration, + ARANGODB: arangodb.integration, } module.exports = { From aeee8af0d8b653854d070b7a733c91680780edfc Mon Sep 17 00:00:00 2001 From: sovlookup <805408477@qq.com> Date: Thu, 28 Jan 2021 22:26:53 +0800 Subject: [PATCH 07/16] this commit is ok --- packages/server/src/integrations/arangodb.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/server/src/integrations/arangodb.js b/packages/server/src/integrations/arangodb.js index c886b6b918..f4feed8246 100644 --- a/packages/server/src/integrations/arangodb.js +++ b/packages/server/src/integrations/arangodb.js @@ -52,6 +52,7 @@ class ArangoDBIntegration { async read(query) { try { const result = await this.client.query(query.sql) + let rl = [] return result.forEach(r => rl.push(r)) } catch (err) { console.error("Error querying arangodb", err.message) @@ -67,6 +68,7 @@ class ArangoDBIntegration { const result = await this.client.query( aql`INSERT ${query.json} INTO ${clc} LET n = NEW RETURN NEW` ) + let rl = [] return result.forEach(r => rl.push(r)) } catch (err) { console.error("Error querying arangodb", err.message) From 2168991591e10867330f1b00bbc6bef87cd92b70 Mon Sep 17 00:00:00 2001 From: sovlookup <805408477@qq.com> Date: Thu, 28 Jan 2021 22:35:41 +0800 Subject: [PATCH 08/16] cannot return result directly fix it --- packages/server/src/integrations/arangodb.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/server/src/integrations/arangodb.js b/packages/server/src/integrations/arangodb.js index f4feed8246..2b3ee1b11b 100644 --- a/packages/server/src/integrations/arangodb.js +++ b/packages/server/src/integrations/arangodb.js @@ -53,7 +53,8 @@ class ArangoDBIntegration { try { const result = await this.client.query(query.sql) let rl = [] - return result.forEach(r => rl.push(r)) + await result.forEach(r => rl.push(r)) + return rl } catch (err) { console.error("Error querying arangodb", err.message) throw err @@ -69,7 +70,8 @@ class ArangoDBIntegration { aql`INSERT ${query.json} INTO ${clc} LET n = NEW RETURN NEW` ) let rl = [] - return result.forEach(r => rl.push(r)) + await result.forEach(r => rl.push(r)) + return rl } catch (err) { console.error("Error querying arangodb", err.message) throw err From 31925ce3c030712630883bb7eb1c8574f67297fe Mon Sep 17 00:00:00 2001 From: sovlookup <805408477@qq.com> Date: Thu, 28 Jan 2021 23:08:53 +0800 Subject: [PATCH 09/16] prettify code --- packages/server/src/integrations/arangodb.js | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/packages/server/src/integrations/arangodb.js b/packages/server/src/integrations/arangodb.js index 2b3ee1b11b..10449fc967 100644 --- a/packages/server/src/integrations/arangodb.js +++ b/packages/server/src/integrations/arangodb.js @@ -52,9 +52,7 @@ class ArangoDBIntegration { async read(query) { try { const result = await this.client.query(query.sql) - let rl = [] - await result.forEach(r => rl.push(r)) - return rl + return result.all() } catch (err) { console.error("Error querying arangodb", err.message) throw err @@ -67,11 +65,9 @@ class ArangoDBIntegration { const clc = this.client.collection(this.config.collection) try { const result = await this.client.query( - aql`INSERT ${query.json} INTO ${clc} LET n = NEW RETURN NEW` + aql`INSERT ${query.json} INTO ${clc} RETURN NEW` ) - let rl = [] - await result.forEach(r => rl.push(r)) - return rl + return result.all() } catch (err) { console.error("Error querying arangodb", err.message) throw err From f9190c1ed779ac5b7041b24d7d3de248de5f8685 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Thu, 28 Jan 2021 19:32:13 +0000 Subject: [PATCH 10/16] privacy features + allow builder usage without cloud API key --- packages/builder/src/analytics.js | 13 +++ .../DatasourceNavigator/icons/ArangoDB.svelte | 53 ++++++++---- .../components/settings/tabs/APIKeys.svelte | 10 ++- .../start/BuilderSettingsModal.svelte | 21 +++++ .../components/start/CreateAppModal.svelte | 42 +--------- .../pages/[application]/deploy/index.svelte | 12 +++ packages/builder/src/pages/index.svelte | 5 +- packages/server/yarn.lock | 81 ++++++++++++++++++- 8 files changed, 172 insertions(+), 65 deletions(-) diff --git a/packages/builder/src/analytics.js b/packages/builder/src/analytics.js index f59d71ac30..c68f228c12 100644 --- a/packages/builder/src/analytics.js +++ b/packages/builder/src/analytics.js @@ -106,6 +106,16 @@ function highlightFeedbackIcon() { return isFeedbackTimeElapsed(firstRunStr) } +// Opt In/Out +const ifAnalyticsEnabled = func => () => { + if (analyticsEnabled && process.env.POSTHOG_TOKEN) { + return func() + } +} +const disabled = () => posthog.has_opted_out_capturing() +const optIn = () => posthog.opt_in_capturing() +const optOut = () => posthog.opt_out_capturing() + export default { activate, identify, @@ -115,4 +125,7 @@ export default { requestFeedbackOnDeploy, submitFeedback, highlightFeedbackIcon, + disabled: ifAnalyticsEnabled(disabled), + optIn: ifAnalyticsEnabled(optIn), + optOut: ifAnalyticsEnabled(optOut), } diff --git a/packages/builder/src/components/backend/DatasourceNavigator/icons/ArangoDB.svelte b/packages/builder/src/components/backend/DatasourceNavigator/icons/ArangoDB.svelte index 0be4ca2663..d305c08af0 100644 --- a/packages/builder/src/components/backend/DatasourceNavigator/icons/ArangoDB.svelte +++ b/packages/builder/src/components/backend/DatasourceNavigator/icons/ArangoDB.svelte @@ -3,20 +3,39 @@ export let height = "100" - - - - - - - - - - - - - - - - - \ No newline at end of file + + + + + + + + + + + + + + + + diff --git a/packages/builder/src/components/settings/tabs/APIKeys.svelte b/packages/builder/src/components/settings/tabs/APIKeys.svelte index 8e569596a4..82e66a12fd 100644 --- a/packages/builder/src/components/settings/tabs/APIKeys.svelte +++ b/packages/builder/src/components/settings/tabs/APIKeys.svelte @@ -1,6 +1,7 @@