From e9405a18027191719862e9e239ddeb844051c1fd Mon Sep 17 00:00:00 2001 From: Maurits Lourens Date: Tue, 8 Mar 2022 17:31:36 +0100 Subject: [PATCH 1/5] initial setup for google firebase integration --- .../DatasourceNavigator/icons/Firebase.svelte | 54 ++++++ .../DatasourceNavigator/icons/firebase.svg | 91 ++++++++++ .../DatasourceNavigator/icons/index.js | 2 + .../builder/src/constants/backend/index.js | 2 + packages/server/package.json | 1 + .../server/src/api/routes/public/index.ts | 4 +- packages/server/src/definitions/datasource.ts | 1 + packages/server/src/integrations/firebase.ts | 161 ++++++++++++++++++ packages/server/src/integrations/index.ts | 3 + 9 files changed, 317 insertions(+), 2 deletions(-) create mode 100644 packages/builder/src/components/backend/DatasourceNavigator/icons/Firebase.svelte create mode 100644 packages/builder/src/components/backend/DatasourceNavigator/icons/firebase.svg create mode 100644 packages/server/src/integrations/firebase.ts diff --git a/packages/builder/src/components/backend/DatasourceNavigator/icons/Firebase.svelte b/packages/builder/src/components/backend/DatasourceNavigator/icons/Firebase.svelte new file mode 100644 index 0000000000..3a776a9217 --- /dev/null +++ b/packages/builder/src/components/backend/DatasourceNavigator/icons/Firebase.svelte @@ -0,0 +1,54 @@ + + + + + + + + + + + + + + + + + + + + diff --git a/packages/builder/src/components/backend/DatasourceNavigator/icons/firebase.svg b/packages/builder/src/components/backend/DatasourceNavigator/icons/firebase.svg new file mode 100644 index 0000000000..dc569606ad --- /dev/null +++ b/packages/builder/src/components/backend/DatasourceNavigator/icons/firebase.svg @@ -0,0 +1,91 @@ + + + + + + + + + + + + + + + + + + + + diff --git a/packages/builder/src/components/backend/DatasourceNavigator/icons/index.js b/packages/builder/src/components/backend/DatasourceNavigator/icons/index.js index 350fccf73f..515f20a93b 100644 --- a/packages/builder/src/components/backend/DatasourceNavigator/icons/index.js +++ b/packages/builder/src/components/backend/DatasourceNavigator/icons/index.js @@ -12,6 +12,7 @@ import Rest from "./Rest.svelte" import Budibase from "./Budibase.svelte" import Oracle from "./Oracle.svelte" import GoogleSheets from "./GoogleSheets.svelte" +import Firebase from "./Firebase.svelte" export default { BUDIBASE: Budibase, @@ -28,4 +29,5 @@ export default { REST: Rest, ORACLE: Oracle, GOOGLE_SHEETS: GoogleSheets, + FIREBASE: Firebase, } diff --git a/packages/builder/src/constants/backend/index.js b/packages/builder/src/constants/backend/index.js index 4d6a7e3884..9a623241ca 100644 --- a/packages/builder/src/constants/backend/index.js +++ b/packages/builder/src/constants/backend/index.js @@ -178,6 +178,7 @@ export const IntegrationTypes = { ORACLE: "ORACLE", INTERNAL: "INTERNAL", GOOGLE_SHEETS: "GOOGLE_SHEETS", + FIREBASE: "FIREBASE", } export const IntegrationNames = { @@ -195,6 +196,7 @@ export const IntegrationNames = { [IntegrationTypes.ORACLE]: "Oracle", [IntegrationTypes.INTERNAL]: "Internal", [IntegrationTypes.GOOGLE_SHEETS]: "Google Sheets", + [IntegrationTypes.FIREBASE]: "Firebase", } export const SchemaTypeOptions = [ diff --git a/packages/server/package.json b/packages/server/package.json index 30e21a7170..b80b99ab69 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -77,6 +77,7 @@ "@bull-board/api": "^3.7.0", "@bull-board/koa": "^3.7.0", "@elastic/elasticsearch": "7.10.0", + "@google-cloud/firestore": "^5.0.2", "@koa/router": "8.0.0", "@sendgrid/mail": "7.1.1", "@sentry/node": "^6.0.0", diff --git a/packages/server/src/api/routes/public/index.ts b/packages/server/src/api/routes/public/index.ts index 800eae6101..63a6946606 100644 --- a/packages/server/src/api/routes/public/index.ts +++ b/packages/server/src/api/routes/public/index.ts @@ -29,7 +29,7 @@ function getApiLimitPerSecond(): number { return parseInt(env.API_REQ_LIMIT_PER_SEC) } -if (!env.isTest()) { +/*if (!env.isTest()) { const REDIS_OPTS = getRedisOptions() RateLimit.defaultOptions({ store: new Stores.Redis({ @@ -42,7 +42,7 @@ if (!env.isTest()) { database: 1, }), }) -} +}*/ // rate limiting, allows for 2 requests per second const limiter = RateLimit.middleware({ interval: { sec: 1 }, diff --git a/packages/server/src/definitions/datasource.ts b/packages/server/src/definitions/datasource.ts index 6c8c8dc07d..208b2dc0a3 100644 --- a/packages/server/src/definitions/datasource.ts +++ b/packages/server/src/definitions/datasource.ts @@ -48,6 +48,7 @@ export enum SourceNames { REST = "REST", ORACLE = "ORACLE", GOOGLE_SHEETS = "GOOGLE_SHEETS", + FIREBASE = "FIREBASE", } export enum IncludeRelationships { diff --git a/packages/server/src/integrations/firebase.ts b/packages/server/src/integrations/firebase.ts new file mode 100644 index 0000000000..8bcf198b7d --- /dev/null +++ b/packages/server/src/integrations/firebase.ts @@ -0,0 +1,161 @@ +import { + DatasourceFieldTypes, + Integration, + QueryTypes, +} from "../definitions/datasource" +import { IntegrationBase } from "./base/IntegrationBase" +import { Firestore, WhereFilterOp } from "@google-cloud/firestore" + +module Firebase { + interface FirebaseConfig { + email: string + privateKey: string + projectId: string + } + + const SCHEMA: Integration = { + docs: "https://firebase.google.com/docs/firestore/quickstart", + friendlyName: "Firestore", + description: + "Cloud Firestore is a flexible, scalable database for mobile, web, and server development from Firebase and Google Cloud.", + datasource: { + email: { + type: DatasourceFieldTypes.STRING, + required: true, + }, + privateKey: { + type: DatasourceFieldTypes.STRING, + required: true, + }, + projectId: { + type: DatasourceFieldTypes.STRING, + required: true, + }, + }, + query: { + create: { + type: QueryTypes.JSON, + }, + read: { + type: QueryTypes.JSON, + }, + update: { + type: QueryTypes.JSON, + }, + delete: { + type: QueryTypes.JSON, + }, + }, + extra: { + collection: { + displayName: "Collection", + type: DatasourceFieldTypes.STRING, + required: true, + }, + filter: { + displayName: "Filter query", + type: DatasourceFieldTypes.LIST, + required: false, + data: { + read: [ + "==", + "<", + "<=", + "==", + "!=", + ">=", + ">", + "array-contains", + "in", + "not-in", + "array-contains-any", + ], + }, + }, + queryValue: { + displayName: "Query value", + type: DatasourceFieldTypes.STRING, + required: false, + }, + }, + } + + class FirebaseIntegration implements IntegrationBase { + private config: FirebaseConfig + private db: Firestore + + constructor(config: FirebaseConfig) { + this.config = config + this.db = new Firestore({ + projectId: config.projectId, + credential: { + clientEmail: config.email, + privateKey: config.privateKey, + }, + }) + } + + async create(query: { json: object; extra: { [key: string]: string } }) { + try { + return await this.db.collection(query.extra.collection).add(query.json) + } catch (err) { + console.error("Error writing to Firestore", err) + throw err + } + } + + async read(query: { + field: string + opStr: WhereFilterOp + value: any + extra: { [key: string]: string } + }) { + try { + const snapshot = await this.db + .collection(query.extra.collection) + .where(query.field, query.opStr, query.value) + .get() + const result: any[] = [] + snapshot.forEach(doc => result.push(doc.data())) + + return result + } catch (err) { + console.error("Error querying Firestore", err) + throw err + } + } + + async update(query: { + id: string + json: object + extra: { [key: string]: string } + }) { + try { + return await this.db + .collection(query.extra.collection) + .doc(query.id) + .update(query.json) + } catch (err) { + console.error("Error writing to mongodb", err) + throw err + } + } + + async delete(query: { id: string; extra: { [key: string]: string } }) { + try { + return await this.db + .collection(query.extra.collection) + .doc(query.id) + .delete() + } catch (err) { + console.error("Error writing to mongodb", err) + throw err + } + } + } + + module.exports = { + schema: SCHEMA, + integration: FirebaseIntegration, + } +} diff --git a/packages/server/src/integrations/index.ts b/packages/server/src/integrations/index.ts index 00b00c25fb..07f3211fcb 100644 --- a/packages/server/src/integrations/index.ts +++ b/packages/server/src/integrations/index.ts @@ -10,6 +10,7 @@ const mysql = require("./mysql") const arangodb = require("./arangodb") const rest = require("./rest") const googlesheets = require("./googlesheets") +const firebase = require("./firebase") const { SourceNames } = require("../definitions/datasource") const environment = require("../environment") @@ -25,6 +26,7 @@ const DEFINITIONS = { [SourceNames.MYSQL]: mysql.schema, [SourceNames.ARANGODB]: arangodb.schema, [SourceNames.REST]: rest.schema, + [SourceNames.FIREBASE]: firebase.schema, } const INTEGRATIONS = { @@ -39,6 +41,7 @@ const INTEGRATIONS = { [SourceNames.MYSQL]: mysql.integration, [SourceNames.ARANGODB]: arangodb.integration, [SourceNames.REST]: rest.integration, + [SourceNames.FIREBASE]: firebase.integration, } // optionally add oracle integration if the oracle binary can be installed From bdb5b127dd7e6511a060eeab86ff6dda22507558 Mon Sep 17 00:00:00 2001 From: Maurits Lourens Date: Wed, 9 Mar 2022 17:46:25 +0100 Subject: [PATCH 2/5] fix connection to firebase using service account --- packages/server/src/integrations/firebase.ts | 54 ++++++++++++++------ 1 file changed, 37 insertions(+), 17 deletions(-) diff --git a/packages/server/src/integrations/firebase.ts b/packages/server/src/integrations/firebase.ts index 8bcf198b7d..00a8e227eb 100644 --- a/packages/server/src/integrations/firebase.ts +++ b/packages/server/src/integrations/firebase.ts @@ -11,6 +11,7 @@ module Firebase { email: string privateKey: string projectId: string + serviceAccount?: string } const SCHEMA: Integration = { @@ -31,6 +32,10 @@ module Firebase { type: DatasourceFieldTypes.STRING, required: true, }, + serviceAccount: { + type: DatasourceFieldTypes.JSON, + required: false, + }, }, query: { create: { @@ -86,13 +91,24 @@ module Firebase { constructor(config: FirebaseConfig) { this.config = config - this.db = new Firestore({ - projectId: config.projectId, - credential: { - clientEmail: config.email, - privateKey: config.privateKey, - }, - }) + if (config.serviceAccount) { + const serviceAccount = JSON.parse(config.serviceAccount) + this.db = new Firestore({ + projectId: serviceAccount.project_id, + credentials: { + client_email: serviceAccount.client_email, + private_key: serviceAccount.private_key, + }, + }) + } else { + this.db = new Firestore({ + projectId: config.projectId, + credentials: { + client_email: config.email, + private_key: config.privateKey, + }, + }) + } } async create(query: { json: object; extra: { [key: string]: string } }) { @@ -104,17 +120,21 @@ module Firebase { } } - async read(query: { - field: string - opStr: WhereFilterOp - value: any - extra: { [key: string]: string } - }) { + async read(query: { json: object; extra: { [key: string]: string } }) { try { - const snapshot = await this.db - .collection(query.extra.collection) - .where(query.field, query.opStr, query.value) - .get() + let snapshot + const collectionRef = this.db.collection(query.extra.collection) + if (query.extra.field && query.extra.opStr && query.extra.queryValue) { + snapshot = await collectionRef + .where( + query.extra.field, + query.extra.opStr as WhereFilterOp, + query.extra.value + ) + .get() + } else { + snapshot = await collectionRef.get() + } const result: any[] = [] snapshot.forEach(doc => result.push(doc.data())) From 70c5574e75f2530d47065b7c48e1e1e7b3469e79 Mon Sep 17 00:00:00 2001 From: Maurits Lourens Date: Thu, 10 Mar 2022 00:23:36 +0100 Subject: [PATCH 3/5] finish up Firebase implementation --- packages/server/src/integrations/firebase.ts | 56 ++++++++++++++------ 1 file changed, 40 insertions(+), 16 deletions(-) diff --git a/packages/server/src/integrations/firebase.ts b/packages/server/src/integrations/firebase.ts index 00a8e227eb..503dae5c95 100644 --- a/packages/server/src/integrations/firebase.ts +++ b/packages/server/src/integrations/firebase.ts @@ -57,8 +57,13 @@ module Firebase { type: DatasourceFieldTypes.STRING, required: true, }, + filterField: { + displayName: "Filter field", + type: DatasourceFieldTypes.STRING, + required: false, + }, filter: { - displayName: "Filter query", + displayName: "Filter comparison", type: DatasourceFieldTypes.LIST, required: false, data: { @@ -77,8 +82,8 @@ module Firebase { ], }, }, - queryValue: { - displayName: "Query value", + filterValue: { + displayName: "Filter value", type: DatasourceFieldTypes.STRING, required: false, }, @@ -113,7 +118,12 @@ module Firebase { async create(query: { json: object; extra: { [key: string]: string } }) { try { - return await this.db.collection(query.extra.collection).add(query.json) + const documentReference = this.db + .collection(query.extra.collection) + .doc() + await documentReference.set({ ...query.json, id: documentReference.id }) + const snapshot = await documentReference.get() + return snapshot.data() } catch (err) { console.error("Error writing to Firestore", err) throw err @@ -124,12 +134,16 @@ module Firebase { try { let snapshot const collectionRef = this.db.collection(query.extra.collection) - if (query.extra.field && query.extra.opStr && query.extra.queryValue) { + if ( + query.extra.filterField && + query.extra.filter && + query.extra.filterValue + ) { snapshot = await collectionRef .where( - query.extra.field, - query.extra.opStr as WhereFilterOp, - query.extra.value + query.extra.filterField, + query.extra.filter as WhereFilterOp, + query.extra.filterValue ) .get() } else { @@ -146,27 +160,37 @@ module Firebase { } async update(query: { - id: string - json: object + json: Record extra: { [key: string]: string } }) { try { - return await this.db + await this.db .collection(query.extra.collection) - .doc(query.id) + .doc(query.json.id) .update(query.json) + + return ( + await this.db + .collection(query.extra.collection) + .doc(query.json.id) + .get() + ).data() } catch (err) { - console.error("Error writing to mongodb", err) + console.error("Error writing to firebase", err) throw err } } - async delete(query: { id: string; extra: { [key: string]: string } }) { + async delete(query: { + json: { id: string } + extra: { [key: string]: string } + }) { try { - return await this.db + await this.db .collection(query.extra.collection) - .doc(query.id) + .doc(query.json.id) .delete() + return true } catch (err) { console.error("Error writing to mongodb", err) throw err From 99e78c75787b74db98b012c4e1433a5f1992f21c Mon Sep 17 00:00:00 2001 From: Maurits Lourens Date: Thu, 10 Mar 2022 00:34:08 +0100 Subject: [PATCH 4/5] remove inkscape related data from svg --- .../DatasourceNavigator/icons/firebase.svg | 24 ------------------- 1 file changed, 24 deletions(-) diff --git a/packages/builder/src/components/backend/DatasourceNavigator/icons/firebase.svg b/packages/builder/src/components/backend/DatasourceNavigator/icons/firebase.svg index dc569606ad..583b90f5c3 100644 --- a/packages/builder/src/components/backend/DatasourceNavigator/icons/firebase.svg +++ b/packages/builder/src/components/backend/DatasourceNavigator/icons/firebase.svg @@ -3,32 +3,8 @@ viewBox="23 6 469 132" width="100" height="100" - version="1.1" - id="svg216" - sodipodi:docname="firebase.svg" - inkscape:version="1.1.2 (b8e25be8, 2022-02-05)" - xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape" - xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd" xmlns="http://www.w3.org/2000/svg" xmlns:svg="http://www.w3.org/2000/svg"> - Date: Wed, 16 Mar 2022 13:43:09 +0100 Subject: [PATCH 5/5] revert uncommenting code --- packages/server/src/api/routes/public/index.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/server/src/api/routes/public/index.ts b/packages/server/src/api/routes/public/index.ts index 3680496a84..57436def1d 100644 --- a/packages/server/src/api/routes/public/index.ts +++ b/packages/server/src/api/routes/public/index.ts @@ -30,7 +30,7 @@ function getApiLimitPerSecond(): number { return parseInt(env.API_REQ_LIMIT_PER_SEC) } -/*if (!env.isTest()) { +if (!env.isTest()) { const REDIS_OPTS = getRedisOptions() let options if (REDIS_OPTS.redisProtocolUrl) { @@ -51,7 +51,7 @@ function getApiLimitPerSecond(): number { RateLimit.defaultOptions({ store: new Stores.Redis(options), }) -}*/ +} // rate limiting, allows for 2 requests per second const limiter = RateLimit.middleware({ interval: { sec: 1 },