diff --git a/lerna.json b/lerna.json index 50e7e77b52..74dc9d096f 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "1.0.46-alpha.5", + "version": "1.0.46-alpha.6", "npmClient": "yarn", "packages": [ "packages/*" diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json index 31133af249..4ea9f22797 100644 --- a/packages/backend-core/package.json +++ b/packages/backend-core/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/backend-core", - "version": "1.0.46-alpha.5", + "version": "1.0.46-alpha.6", "description": "Budibase backend core libraries used in server and worker", "main": "src/index.js", "author": "Budibase", diff --git a/packages/backend-core/src/constants.js b/packages/backend-core/src/constants.js index 28b9ced49b..091e4337cf 100644 --- a/packages/backend-core/src/constants.js +++ b/packages/backend-core/src/constants.js @@ -7,6 +7,7 @@ exports.Cookies = { CurrentApp: "budibase:currentapp", Auth: "budibase:auth", Init: "budibase:init", + DatasourceAuth: "budibase:datasourceauth", OIDC_CONFIG: "budibase:oidc:config", } diff --git a/packages/backend-core/src/middleware/index.js b/packages/backend-core/src/middleware/index.js index cf8676a2bc..7ea07a21ce 100644 --- a/packages/backend-core/src/middleware/index.js +++ b/packages/backend-core/src/middleware/index.js @@ -7,6 +7,7 @@ const authenticated = require("./authenticated") const auditLog = require("./auditLog") const tenancy = require("./tenancy") const appTenancy = require("./appTenancy") +const datasourceGoogle = require("./passport/datasource/google") module.exports = { google, @@ -18,4 +19,7 @@ module.exports = { tenancy, appTenancy, authError, + datasource: { + google: datasourceGoogle, + }, } diff --git a/packages/backend-core/src/middleware/passport/datasource/google.js b/packages/backend-core/src/middleware/passport/datasource/google.js new file mode 100644 index 0000000000..bfc2e4a61e --- /dev/null +++ b/packages/backend-core/src/middleware/passport/datasource/google.js @@ -0,0 +1,76 @@ +const { getScopedConfig } = require("../../../db/utils") +const { getGlobalDB } = require("../../../tenancy") +const google = require("../google") +const { Configs, Cookies } = require("../../../constants") +const { clearCookie, getCookie } = require("../../../utils") +const { getDB } = require("../../../db") + +async function preAuth(passport, ctx, next) { + const db = getGlobalDB() + // get the relevant config + const config = await getScopedConfig(db, { + type: Configs.GOOGLE, + workspace: ctx.query.workspace, + }) + const publicConfig = await getScopedConfig(db, { + type: Configs.SETTINGS, + }) + let callbackUrl = `${publicConfig.platformUrl}/api/global/auth/datasource/google/callback` + const strategy = await google.strategyFactory(config, callbackUrl) + + if (!ctx.query.appId || !ctx.query.datasourceId) { + ctx.throw(400, "appId and datasourceId query params not present.") + } + + return passport.authenticate(strategy, { + scope: ["profile", "email", "https://www.googleapis.com/auth/spreadsheets"], + accessType: "offline", + prompt: "consent", + })(ctx, next) +} + +async function postAuth(passport, ctx, next) { + const db = getGlobalDB() + + const config = await getScopedConfig(db, { + type: Configs.GOOGLE, + workspace: ctx.query.workspace, + }) + + const publicConfig = await getScopedConfig(db, { + type: Configs.SETTINGS, + }) + + let callbackUrl = `${publicConfig.platformUrl}/api/global/auth/datasource/google/callback` + const strategy = await google.strategyFactory( + config, + callbackUrl, + (accessToken, refreshToken, profile, done) => { + clearCookie(ctx, Cookies.DatasourceAuth) + done(null, { accessToken, refreshToken }) + } + ) + + const authStateCookie = getCookie(ctx, Cookies.DatasourceAuth) + + return passport.authenticate( + strategy, + { successRedirect: "/", failureRedirect: "/error" }, + async (err, tokens) => { + // update the DB for the datasource with all the user info + const db = getDB(authStateCookie.appId) + const datasource = await db.get(authStateCookie.datasourceId) + if (!datasource.config) { + datasource.config = {} + } + datasource.config.auth = { type: "google", ...tokens } + await db.put(datasource) + ctx.redirect( + `/builder/app/${authStateCookie.appId}/data/datasource/${authStateCookie.datasourceId}` + ) + } + )(ctx, next) +} + +exports.preAuth = preAuth +exports.postAuth = postAuth diff --git a/packages/bbui/package.json b/packages/bbui/package.json index 14b5afbfca..074d73c939 100644 --- a/packages/bbui/package.json +++ b/packages/bbui/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/bbui", "description": "A UI solution used in the different Budibase projects.", - "version": "1.0.46-alpha.5", + "version": "1.0.46-alpha.6", "license": "MPL-2.0", "svelte": "src/index.js", "module": "dist/bbui.es.js", diff --git a/packages/builder/package.json b/packages/builder/package.json index d7b20b0864..87a9f88d4a 100644 --- a/packages/builder/package.json +++ b/packages/builder/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/builder", - "version": "1.0.46-alpha.5", + "version": "1.0.46-alpha.6", "license": "GPL-3.0", "private": true, "scripts": { @@ -65,11 +65,11 @@ } }, "dependencies": { - "@budibase/bbui": "^1.0.46-alpha.5", - "@budibase/client": "^1.0.46-alpha.5", - "@budibase/frontend-core": "^1.0.46-alpha.5", + "@budibase/bbui": "^1.0.46-alpha.6", + "@budibase/client": "^1.0.46-alpha.6", + "@budibase/frontend-core": "^1.0.46-alpha.6", "@budibase/colorpicker": "1.1.2", - "@budibase/string-templates": "^1.0.46-alpha.5", + "@budibase/string-templates": "^1.0.46-alpha.6", "@sentry/browser": "5.19.1", "@spectrum-css/page": "^3.0.1", "@spectrum-css/vars": "^3.0.1", diff --git a/packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/PlusConfigForm.svelte b/packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/PlusConfigForm.svelte index 1148695712..8c6857d2b9 100644 --- a/packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/PlusConfigForm.svelte +++ b/packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/PlusConfigForm.svelte @@ -188,7 +188,7 @@ {:else} No tables found. {/if} -{#if plusTables?.length !== 0} +{#if plusTables?.length !== 0 && integration.relationships}
Relationships diff --git a/packages/builder/src/components/backend/DatasourceNavigator/_components/GoogleButton.svelte b/packages/builder/src/components/backend/DatasourceNavigator/_components/GoogleButton.svelte new file mode 100644 index 0000000000..091e332832 --- /dev/null +++ b/packages/builder/src/components/backend/DatasourceNavigator/_components/GoogleButton.svelte @@ -0,0 +1,47 @@ + + + { + let ds = datasource + if (!ds) { + ds = await preAuthStep() + } + window.open( + `/api/global/auth/${tenantId}/datasource/google?datasourceId=${datasource._id}&appId=${$store.appId}`, + "_blank" + ) + }} +> +
+ google icon +

Sign in with Google

+
+ + + diff --git a/packages/builder/src/components/backend/DatasourceNavigator/icons/GoogleSheets.svelte b/packages/builder/src/components/backend/DatasourceNavigator/icons/GoogleSheets.svelte new file mode 100644 index 0000000000..0d376e4400 --- /dev/null +++ b/packages/builder/src/components/backend/DatasourceNavigator/icons/GoogleSheets.svelte @@ -0,0 +1,184 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/packages/builder/src/components/backend/DatasourceNavigator/icons/index.js b/packages/builder/src/components/backend/DatasourceNavigator/icons/index.js index 56ae03dcc3..350fccf73f 100644 --- a/packages/builder/src/components/backend/DatasourceNavigator/icons/index.js +++ b/packages/builder/src/components/backend/DatasourceNavigator/icons/index.js @@ -11,6 +11,7 @@ import ArangoDB from "./ArangoDB.svelte" import Rest from "./Rest.svelte" import Budibase from "./Budibase.svelte" import Oracle from "./Oracle.svelte" +import GoogleSheets from "./GoogleSheets.svelte" export default { BUDIBASE: Budibase, @@ -26,4 +27,5 @@ export default { ARANGODB: ArangoDB, REST: Rest, ORACLE: Oracle, + GOOGLE_SHEETS: GoogleSheets, } diff --git a/packages/builder/src/components/backend/DatasourceNavigator/modals/CreateDatasourceModal.svelte b/packages/builder/src/components/backend/DatasourceNavigator/modals/CreateDatasourceModal.svelte index 0564565012..14adbfbd02 100644 --- a/packages/builder/src/components/backend/DatasourceNavigator/modals/CreateDatasourceModal.svelte +++ b/packages/builder/src/components/backend/DatasourceNavigator/modals/CreateDatasourceModal.svelte @@ -13,6 +13,7 @@ import { IntegrationNames, IntegrationTypes } from "constants/backend" import CreateTableModal from "components/backend/TableNavigator/modals/CreateTableModal.svelte" import DatasourceConfigModal from "components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte" + import GoogleDatasourceConfigModal from "components/backend/DatasourceNavigator/modals/GoogleDatasourceConfigModal.svelte" import { createRestDatasource } from "builderStore/datasource" import { goto } from "@roxi/routify" import ImportRestQueriesModal from "./ImportRestQueriesModal.svelte" @@ -45,6 +46,7 @@ plus: selected.plus, config, schema: selected.datasource, + auth: selected.auth, } checkShowImport() } @@ -96,7 +98,11 @@ - + {#if integration?.auth?.type === "google"} + + {:else} + + {/if} diff --git a/packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte b/packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte index 06e00ff5d0..8f1218f087 100644 --- a/packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte +++ b/packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte @@ -51,13 +51,9 @@ >Connect your database to Budibase using the config below. - - - diff --git a/packages/builder/src/components/backend/DatasourceNavigator/modals/GoogleDatasourceConfigModal.svelte b/packages/builder/src/components/backend/DatasourceNavigator/modals/GoogleDatasourceConfigModal.svelte new file mode 100644 index 0000000000..7d03dafeb9 --- /dev/null +++ b/packages/builder/src/components/backend/DatasourceNavigator/modals/GoogleDatasourceConfigModal.svelte @@ -0,0 +1,29 @@ + + + modal.show()} + cancelText="Back" + size="L" +> + + Authenticate with your google account to use the {IntegrationNames[ + datasource.type + ]} integration. + + save(datasource, true)} /> + diff --git a/packages/builder/src/components/integration/QueryBindingBuilder.svelte b/packages/builder/src/components/integration/QueryBindingBuilder.svelte index 34d4d4219b..9bcf9d36f8 100644 --- a/packages/builder/src/components/integration/QueryBindingBuilder.svelte +++ b/packages/builder/src/components/integration/QueryBindingBuilder.svelte @@ -15,8 +15,6 @@ queryBindings = [...queryBindings, {}] } - $: console.log(bindings) - function deleteQueryBinding(idx) { queryBindings.splice(idx, 1) queryBindings = queryBindings diff --git a/packages/builder/src/constants/backend/index.js b/packages/builder/src/constants/backend/index.js index c1eea5b0ef..50413fcb22 100644 --- a/packages/builder/src/constants/backend/index.js +++ b/packages/builder/src/constants/backend/index.js @@ -177,6 +177,7 @@ export const IntegrationTypes = { ARANGODB: "ARANGODB", ORACLE: "ORACLE", INTERNAL: "INTERNAL", + GOOGLE_SHEETS: "GOOGLE_SHEETS", } export const IntegrationNames = { @@ -193,6 +194,7 @@ export const IntegrationNames = { [IntegrationTypes.ARANGODB]: "ArangoDB", [IntegrationTypes.ORACLE]: "Oracle", [IntegrationTypes.INTERNAL]: "Internal", + [IntegrationTypes.GOOGLE_SHEETS]: "Google Sheets", } export const SchemaTypeOptions = [ diff --git a/packages/builder/src/constants/index.js b/packages/builder/src/constants/index.js index 6b7aafdfa9..abeaadc718 100644 --- a/packages/builder/src/constants/index.js +++ b/packages/builder/src/constants/index.js @@ -15,6 +15,22 @@ export const AppStatus = { DEPLOYED: "published", } +export const IntegrationNames = { + POSTGRES: "PostgreSQL", + MONGODB: "MongoDB", + COUCHDB: "CouchDB", + S3: "S3", + MYSQL: "MySQL", + REST: "REST", + DYNAMODB: "DynamoDB", + ELASTICSEARCH: "ElasticSearch", + SQL_SERVER: "SQL Server", + AIRTABLE: "Airtable", + ARANGODB: "ArangoDB", + ORACLE: "Oracle", + GOOGLE_SHEETS: "Google Sheets", +} + // fields on the user table that cannot be edited export const UNEDITABLE_USER_FIELDS = [ "email", diff --git a/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/index.svelte index 98b7859305..bb4cc6e1fb 100644 --- a/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/index.svelte +++ b/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/index.svelte @@ -19,8 +19,8 @@ import { IntegrationTypes } from "constants/backend" import { isEqual } from "lodash" import { cloneDeep } from "lodash/fp" - import ImportRestQueriesModal from "components/backend/DatasourceNavigator/modals/ImportRestQueriesModal.svelte" + let importQueriesModal let changed diff --git a/packages/cli/package.json b/packages/cli/package.json index fa4026e729..e98bf37d9c 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/cli", - "version": "1.0.46-alpha.5", + "version": "1.0.46-alpha.6", "description": "Budibase CLI, for developers, self hosting and migrations.", "main": "src/index.js", "bin": { diff --git a/packages/client/package.json b/packages/client/package.json index 05073d5aba..acdc39d7a6 100644 --- a/packages/client/package.json +++ b/packages/client/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/client", - "version": "1.0.46-alpha.5", + "version": "1.0.46-alpha.6", "license": "MPL-2.0", "module": "dist/budibase-client.js", "main": "dist/budibase-client.js", @@ -19,10 +19,10 @@ "dev:builder": "rollup -cw" }, "dependencies": { - "@budibase/bbui": "^1.0.46-alpha.5", - "@budibase/frontend-core": "^1.0.46-alpha.5", + "@budibase/bbui": "^1.0.46-alpha.6", + "@budibase/frontend-core": "^1.0.46-alpha.6", "@budibase/standard-components": "^0.9.139", - "@budibase/string-templates": "^1.0.46-alpha.5", + "@budibase/string-templates": "^1.0.46-alpha.6", "regexparam": "^1.3.0", "rollup-plugin-polyfill-node": "^0.8.0", "shortid": "^2.2.15", diff --git a/packages/frontend-core/package.json b/packages/frontend-core/package.json index c98263e7fb..64c869d522 100644 --- a/packages/frontend-core/package.json +++ b/packages/frontend-core/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/frontend-core", - "version": "1.0.46-alpha.5", + "version": "1.0.46-alpha.6", "description": "Budibase frontend core libraries used in builder and client", "author": "Budibase", "license": "MPL-2.0", diff --git a/packages/server/package.json b/packages/server/package.json index fc1ebec9fc..ad3618dd55 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/server", "email": "hi@budibase.com", - "version": "1.0.46-alpha.5", + "version": "1.0.46-alpha.6", "description": "Budibase Web Server", "main": "src/index.ts", "repository": { @@ -70,9 +70,9 @@ "license": "GPL-3.0", "dependencies": { "@apidevtools/swagger-parser": "^10.0.3", - "@budibase/backend-core": "^1.0.46-alpha.5", - "@budibase/client": "^1.0.46-alpha.5", - "@budibase/string-templates": "^1.0.46-alpha.5", + "@budibase/backend-core": "^1.0.46-alpha.6", + "@budibase/client": "^1.0.46-alpha.6", + "@budibase/string-templates": "^1.0.46-alpha.6", "@bull-board/api": "^3.7.0", "@bull-board/koa": "^3.7.0", "@elastic/elasticsearch": "7.10.0", @@ -92,6 +92,8 @@ "fix-path": "3.0.0", "form-data": "^4.0.0", "fs-extra": "8.1.0", + "google-auth-library": "^7.11.0", + "google-spreadsheet": "^3.2.0", "jimp": "0.16.1", "joi": "17.2.1", "js-yaml": "^4.1.0", @@ -139,6 +141,7 @@ "@jest/test-sequencer": "^24.8.0", "@types/apidoc": "^0.50.0", "@types/bull": "^3.15.1", + "@types/google-spreadsheet": "^3.1.5", "@types/jest": "^26.0.23", "@types/koa": "^2.13.3", "@types/koa-router": "^7.4.2", diff --git a/packages/server/src/api/controllers/datasource.js b/packages/server/src/api/controllers/datasource.js index f08b622c3e..5ab3c0a865 100644 --- a/packages/server/src/api/controllers/datasource.js +++ b/packages/server/src/api/controllers/datasource.js @@ -38,6 +38,13 @@ exports.fetch = async function (ctx) { ) ).rows.map(row => row.doc) + for (let datasource of datasources) { + if (datasource.config && datasource.config.auth) { + // strip secrets from response so they don't show in the network request + delete datasource.config.auth + } + } + ctx.body = [bbInternalDb, ...datasources] } @@ -94,8 +101,13 @@ exports.update = async function (ctx) { const db = new CouchDB(ctx.appId) const datasourceId = ctx.params.datasourceId let datasource = await db.get(datasourceId) + const auth = datasource.config.auth await invalidateVariables(datasource, ctx.request.body) datasource = { ...datasource, ...ctx.request.body } + if (auth && !ctx.request.body.auth) { + // don't strip auth config from DB + datasource.config.auth = auth + } const response = await db.put(datasource) datasource._rev = response.rev diff --git a/packages/server/src/api/controllers/query/index.js b/packages/server/src/api/controllers/query/index.js index 21db1eebbf..9cf7612e8a 100644 --- a/packages/server/src/api/controllers/query/index.js +++ b/packages/server/src/api/controllers/query/index.js @@ -141,6 +141,16 @@ async function execute(ctx, opts = { rowsOnly: false }) { const query = await db.get(ctx.params.queryId) const datasource = await db.get(query.datasourceId) + const enrichedParameters = ctx.request.body.parameters || {} + // make sure parameters are fully enriched with defaults + if (query && query.parameters) { + for (let parameter of query.parameters) { + if (!enrichedParameters[parameter.name]) { + enrichedParameters[parameter.name] = parameter.default + } + } + } + // call the relevant CRUD method on the integration class try { const { rows, pagination, extra } = await Runner.run({ @@ -149,7 +159,7 @@ async function execute(ctx, opts = { rowsOnly: false }) { queryVerb: query.queryVerb, fields: query.fields, pagination: ctx.request.body.pagination, - parameters: ctx.request.body.parameters, + parameters: enrichedParameters, transformer: query.transformer, queryId: ctx.params.queryId, }) @@ -178,8 +188,9 @@ const removeDynamicVariables = async (db, queryId) => { if (dynamicVariables) { // delete dynamic variables from the datasource - const newVariables = dynamicVariables.filter(dv => dv.queryId !== queryId) - datasource.config.dynamicVariables = newVariables + datasource.config.dynamicVariables = dynamicVariables.filter( + dv => dv.queryId !== queryId + ) await db.put(datasource) // invalidate the deleted variables diff --git a/packages/server/src/api/controllers/row/utils.js b/packages/server/src/api/controllers/row/utils.js index 3fb0cc78d8..51bc03eba4 100644 --- a/packages/server/src/api/controllers/row/utils.js +++ b/packages/server/src/api/controllers/row/utils.js @@ -52,10 +52,7 @@ exports.validate = async ({ appId, tableId, row, table }) => { const constraints = cloneDeep(table.schema[fieldName].constraints) const type = table.schema[fieldName].type // special case for options, need to always allow unselected (null) - if ( - (type === FieldTypes.OPTIONS || type === FieldTypes.ARRAY) && - constraints.inclusion - ) { + if (type === FieldTypes.OPTIONS && constraints.inclusion) { constraints.inclusion.push(null) } let res diff --git a/packages/server/src/constants/index.js b/packages/server/src/constants/index.js index b63b71d2c2..cf5e15a9fb 100644 --- a/packages/server/src/constants/index.js +++ b/packages/server/src/constants/index.js @@ -75,6 +75,10 @@ exports.DataSourceOperation = { DELETE_TABLE: "DELETE_TABLE", } +exports.DatasourceAuthTypes = { + GOOGLE: "google", +} + exports.SortDirection = { ASCENDING: "ASCENDING", DESCENDING: "DESCENDING", diff --git a/packages/server/src/definitions/datasource.ts b/packages/server/src/definitions/datasource.ts index ee71a7b08c..efac92334e 100644 --- a/packages/server/src/definitions/datasource.ts +++ b/packages/server/src/definitions/datasource.ts @@ -47,6 +47,7 @@ export enum SourceNames { ARANGODB = "ARANGODB", REST = "REST", ORACLE = "ORACLE", + GOOGLE_SHEETS = "GOOGLE_SHEETS", } export enum IncludeRelationships { @@ -86,6 +87,8 @@ export interface ExtraQueryConfig { export interface Integration { docs: string plus?: boolean + auth?: { type: string } + relationships?: boolean description: string friendlyName: string datasource: {} diff --git a/packages/server/src/integrations/googlesheets.ts b/packages/server/src/integrations/googlesheets.ts new file mode 100644 index 0000000000..5f76e5b548 --- /dev/null +++ b/packages/server/src/integrations/googlesheets.ts @@ -0,0 +1,353 @@ +import { + DatasourceFieldTypes, + Integration, + QueryJson, + QueryTypes, +} from "../definitions/datasource" +import { OAuth2Client } from "google-auth-library" +import { DatasourcePlus } from "./base/datasourcePlus" +import { Row, Table, TableSchema } from "../definitions/common" +import { buildExternalTableId } from "./utils" +import { DataSourceOperation, FieldTypes } from "../constants" +import { GoogleSpreadsheet } from "google-spreadsheet" +import { table } from "console" + +module GoogleSheetsModule { + const { getGlobalDB } = require("@budibase/backend-core/tenancy") + const { getScopedConfig } = require("@budibase/backend-core/db") + const { Configs } = require("@budibase/backend-core/constants") + + interface GoogleSheetsConfig { + spreadsheetId: string + auth: OAuthClientConfig + } + + interface OAuthClientConfig { + appId: string + accessToken: string + refreshToken: string + } + + const SCHEMA: Integration = { + plus: true, + auth: { + type: "google", + }, + relationships: false, + docs: "https://developers.google.com/sheets/api/quickstart/nodejs", + description: + "Create and collaborate on online spreadsheets in real-time and from any device. ", + friendlyName: "Google Sheets", + datasource: { + spreadsheetId: { + type: DatasourceFieldTypes.STRING, + required: true, + }, + }, + query: { + create: { + type: QueryTypes.FIELDS, + fields: { + sheet: { + type: DatasourceFieldTypes.STRING, + required: true, + }, + row: { + type: QueryTypes.JSON, + required: true, + }, + }, + }, + read: { + type: QueryTypes.FIELDS, + fields: { + sheet: { + type: DatasourceFieldTypes.STRING, + required: true, + }, + }, + }, + update: { + type: QueryTypes.FIELDS, + fields: { + sheet: { + type: DatasourceFieldTypes.STRING, + required: true, + }, + rowIndex: { + type: DatasourceFieldTypes.STRING, + required: true, + }, + row: { + type: QueryTypes.JSON, + required: true, + }, + }, + }, + delete: { + type: QueryTypes.FIELDS, + fields: { + sheet: { + type: DatasourceFieldTypes.STRING, + required: true, + }, + rowIndex: { + type: DatasourceFieldTypes.NUMBER, + required: true, + }, + }, + }, + }, + } + + class GoogleSheetsIntegration implements DatasourcePlus { + private readonly config: GoogleSheetsConfig + private client: any + public tables: Record = {} + public schemaErrors: Record = {} + + constructor(config: GoogleSheetsConfig) { + this.config = config + const spreadsheetId = this.cleanSpreadsheetUrl(this.config.spreadsheetId) + this.client = new GoogleSpreadsheet(spreadsheetId) + } + + /** + * Pull the spreadsheet ID out from a valid google sheets URL + * @param spreadsheetId - the URL or standard spreadsheetId of the google sheet + * @returns spreadsheet Id of the google sheet + */ + cleanSpreadsheetUrl(spreadsheetId: string) { + if (!spreadsheetId) { + throw new Error( + "You must set a spreadsheet ID in your configuration to fetch tables." + ) + } + const parts = spreadsheetId.split("/") + return parts.length > 5 ? parts[5] : spreadsheetId + } + + async connect() { + try { + // Initialise oAuth client + const db = getGlobalDB() + const googleConfig = await getScopedConfig(db, { + type: Configs.GOOGLE, + }) + const oauthClient = new OAuth2Client({ + clientId: googleConfig.clientID, + clientSecret: googleConfig.clientSecret, + }) + oauthClient.credentials.access_token = this.config.auth.accessToken + oauthClient.credentials.refresh_token = this.config.auth.refreshToken + this.client.useOAuth2Client(oauthClient) + await this.client.loadInfo() + } catch (err) { + console.error("Error connecting to google sheets", err) + throw err + } + } + + async buildSchema(datasourceId: string) { + await this.connect() + const sheets = await this.client.sheetsByIndex + const tables: Record = {} + for (let sheet of sheets) { + // must fetch rows to determine schema + await sheet.getRows() + // build schema + const schema: TableSchema = {} + + // build schema from headers + for (let header of sheet.headerValues) { + schema[header] = { + name: header, + type: FieldTypes.STRING, + } + } + + // create tables + tables[sheet.title] = { + _id: buildExternalTableId(datasourceId, sheet.title), + name: sheet.title, + primary: ["rowNumber"], + schema, + } + } + + this.tables = tables + } + + async query(json: QueryJson) { + const sheet = json.endpoint.entityId + + const handlers = { + [DataSourceOperation.CREATE]: () => + this.create({ sheet, row: json.body }), + [DataSourceOperation.READ]: () => this.read({ sheet }), + [DataSourceOperation.UPDATE]: () => + this.update({ + // exclude the header row and zero index + rowIndex: json.extra?.idFilter?.equal?.rowNumber - 2, + sheet, + row: json.body, + }), + [DataSourceOperation.DELETE]: () => + this.delete({ + // exclude the header row and zero index + rowIndex: json.extra?.idFilter?.equal?.rowNumber - 2, + sheet, + }), + [DataSourceOperation.CREATE_TABLE]: () => + this.createTable(json?.table?.name), + [DataSourceOperation.UPDATE_TABLE]: () => this.updateTable(json.table), + [DataSourceOperation.DELETE_TABLE]: () => + this.deleteTable(json?.table?.name), + } + + const internalQueryMethod = handlers[json.endpoint.operation] + + return await internalQueryMethod() + } + + buildRowObject(headers: string[], values: string[], rowNumber: number) { + const rowObject: { rowNumber: number; [key: string]: any } = { rowNumber } + for (let i = 0; i < headers.length; i++) { + rowObject._id = rowNumber + rowObject[headers[i]] = values[i] + } + return rowObject + } + + async createTable(name?: string) { + try { + await this.connect() + const sheet = await this.client.addSheet({ title: name }) + return sheet + } catch (err) { + console.error("Error creating new table in google sheets", err) + throw err + } + } + + async updateTable(table?: any) { + try { + await this.connect() + const sheet = await this.client.sheetsByTitle[table.name] + await sheet.loadHeaderRow() + + if (table._rename) { + const headers = [] + for (let header of sheet.headerValues) { + if (header === table._rename.old) { + headers.push(table._rename.updated) + } else { + headers.push(header) + } + } + await sheet.setHeaderRow(headers) + } else { + let newField = Object.keys(table.schema).find( + key => !sheet.headerValues.includes(key) + ) + await sheet.setHeaderRow([...sheet.headerValues, newField]) + } + } catch (err) { + console.error("Error updating table in google sheets", err) + throw err + } + } + + async deleteTable(sheet: any) { + try { + await this.connect() + const sheetToDelete = await this.client.sheetsByTitle[sheet] + return await sheetToDelete.delete() + } catch (err) { + console.error("Error deleting table in google sheets", err) + throw err + } + } + + async create(query: { sheet: string; row: any }) { + try { + await this.connect() + const sheet = await this.client.sheetsByTitle[query.sheet] + const rowToInsert = + typeof query.row === "string" ? JSON.parse(query.row) : query.row + const row = await sheet.addRow(rowToInsert) + return [ + this.buildRowObject(sheet.headerValues, row._rawData, row._rowNumber), + ] + } catch (err) { + console.error("Error writing to google sheets", err) + throw err + } + } + + async read(query: { sheet: string }) { + try { + await this.connect() + const sheet = await this.client.sheetsByTitle[query.sheet] + const rows = await sheet.getRows() + const headerValues = sheet.headerValues + const response = [] + for (let row of rows) { + response.push( + this.buildRowObject(headerValues, row._rawData, row._rowNumber) + ) + } + return response + } catch (err) { + console.error("Error reading from google sheets", err) + throw err + } + } + + async update(query: { sheet: string; rowIndex: number; row: any }) { + try { + await this.connect() + const sheet = await this.client.sheetsByTitle[query.sheet] + const rows = await sheet.getRows() + const row = rows[query.rowIndex] + if (row) { + const updateValues = query.row + for (let key in updateValues) { + row[key] = updateValues[key] + } + await row.save() + return [ + this.buildRowObject( + sheet.headerValues, + row._rawData, + row._rowNumber + ), + ] + } else { + throw new Error("Row does not exist.") + } + } catch (err) { + console.error("Error reading from google sheets", err) + throw err + } + } + + async delete(query: { sheet: string; rowIndex: number }) { + await this.connect() + const sheet = await this.client.sheetsByTitle[query.sheet] + const rows = await sheet.getRows() + const row = rows[query.rowIndex] + if (row) { + await row.delete() + return [{ deleted: query.rowIndex }] + } else { + throw new Error("Row does not exist.") + } + } + } + + module.exports = { + schema: SCHEMA, + integration: GoogleSheetsIntegration, + } +} diff --git a/packages/server/src/integrations/index.ts b/packages/server/src/integrations/index.ts index 8f2f083fc5..4679e658b6 100644 --- a/packages/server/src/integrations/index.ts +++ b/packages/server/src/integrations/index.ts @@ -9,6 +9,7 @@ const airtable = require("./airtable") const mysql = require("./mysql") const arangodb = require("./arangodb") const rest = require("./rest") +const googlesheets = require("./googlesheets") const { SourceNames } = require("../definitions/datasource") const DEFINITIONS = { @@ -23,6 +24,7 @@ const DEFINITIONS = { [SourceNames.MYSQL]: mysql.schema, [SourceNames.ARANGODB]: arangodb.schema, [SourceNames.REST]: rest.schema, + [SourceNames.GOOGLE_SHEETS]: googlesheets.schema, } const INTEGRATIONS = { @@ -37,6 +39,7 @@ const INTEGRATIONS = { [SourceNames.MYSQL]: mysql.integration, [SourceNames.ARANGODB]: arangodb.integration, [SourceNames.REST]: rest.integration, + [SourceNames.GOOGLE_SHEETS]: googlesheets.integration, } // optionally add oracle integration if the oracle binary can be installed diff --git a/packages/server/src/integrations/rest.ts b/packages/server/src/integrations/rest.ts index 3199ce3bde..ea40dfb609 100644 --- a/packages/server/src/integrations/rest.ts +++ b/packages/server/src/integrations/rest.ts @@ -43,8 +43,8 @@ const coreFields = { enum: Object.values(BodyTypes), }, pagination: { - type: DatasourceFieldTypes.OBJECT - } + type: DatasourceFieldTypes.OBJECT, + }, } module RestModule { @@ -178,12 +178,17 @@ module RestModule { headers, }, pagination: { - cursor: nextCursor - } + cursor: nextCursor, + }, } } - getUrl(path: string, queryString: string, pagination: PaginationConfig | null, paginationValues: PaginationValues | null): string { + getUrl( + path: string, + queryString: string, + pagination: PaginationConfig | null, + paginationValues: PaginationValues | null + ): string { // Add pagination params to query string if required if (pagination?.location === "query" && paginationValues) { const { pageParam, sizeParam } = pagination @@ -217,14 +222,22 @@ module RestModule { return complete } - addBody(bodyType: string, body: string | any, input: any, pagination: PaginationConfig | null, paginationValues: PaginationValues | null) { + addBody( + bodyType: string, + body: string | any, + input: any, + pagination: PaginationConfig | null, + paginationValues: PaginationValues | null + ) { if (!input.headers) { input.headers = {} } if (bodyType === BodyTypes.NONE) { return input } - let error, object: any = {}, string = "" + let error, + object: any = {}, + string = "" try { if (body) { string = typeof body !== "string" ? JSON.stringify(body) : body @@ -333,7 +346,7 @@ module RestModule { requestBody, authConfigId, pagination, - paginationValues + paginationValues, } = query const authHeaders = this.getAuthHeaders(authConfigId) @@ -352,7 +365,13 @@ module RestModule { } let input: any = { method, headers: this.headers } - input = this.addBody(bodyType, requestBody, input, pagination, paginationValues) + input = this.addBody( + bodyType, + requestBody, + input, + pagination, + paginationValues + ) this.startTimeMs = performance.now() const url = this.getUrl(path, queryString, pagination, paginationValues) diff --git a/packages/server/src/integrations/s3.ts b/packages/server/src/integrations/s3.ts index 25b439fd58..273f221575 100644 --- a/packages/server/src/integrations/s3.ts +++ b/packages/server/src/integrations/s3.ts @@ -38,7 +38,7 @@ module S3Module { signatureVersion: { type: "string", required: false, - default: "v4" + default: "v4", }, }, query: { diff --git a/packages/server/yarn.lock b/packages/server/yarn.lock index fd632e8df4..df4e50c48f 100644 --- a/packages/server/yarn.lock +++ b/packages/server/yarn.lock @@ -983,10 +983,10 @@ resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw== -"@budibase/backend-core@^1.0.46-alpha.3": - version "1.0.46" - resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.46.tgz#795e80038e11c054bb1aa313c16716a7035f3000" - integrity sha512-vXDjTOMlTaGx1Vm6ste7D7ZXwC+NgLzzu+8Ji7T0Pz2WXj+05vWpPha6L5CkNxRYTwUGoU1BAOvMYrChbGOftQ== +"@budibase/backend-core@^1.0.46-alpha.5": + version "1.0.47" + resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.47.tgz#af1e501e20f8a648a40fe7d336b89e65f058c803" + integrity sha512-nj+MC2j6WEH+6LEJhs+zMbnm4BRGCaX7kXvlyq7EXA9h6QOxrNkB/PNFqEumkMJGjorkZAQ/qe8MUEjcE26QBw== dependencies: "@techpass/passport-openidconnect" "^0.3.0" aws-sdk "^2.901.0" @@ -1056,10 +1056,10 @@ svelte-flatpickr "^3.2.3" svelte-portal "^1.0.0" -"@budibase/bbui@^1.0.46": - version "1.0.46" - resolved "https://registry.yarnpkg.com/@budibase/bbui/-/bbui-1.0.46.tgz#7306d4eda7f2c827577a4affa1fd314b38ba1198" - integrity sha512-padm0qq2SBNIslXEQW+HIv32pkIHFzloR93FDzSXh0sO43Q+/d2gbAhjI9ZUSAVncx9JNc46dolL1CwrvHFElg== +"@budibase/bbui@^1.0.47": + version "1.0.47" + resolved "https://registry.yarnpkg.com/@budibase/bbui/-/bbui-1.0.47.tgz#df2848b89f881fe603e7156855d6a6c31d4f58bf" + integrity sha512-RRm/BgK5aSx2/vGjMGljw240/48Ksc3/h4yB1nhQj8Xx3fKhlGnWDvWNy+sakvA6+fJvEXuti8RoxHtQ6lXmqA== dependencies: "@adobe/spectrum-css-workflow-icons" "^1.2.1" "@spectrum-css/actionbutton" "^1.0.1" @@ -1106,14 +1106,14 @@ svelte-flatpickr "^3.2.3" svelte-portal "^1.0.0" -"@budibase/client@^1.0.46-alpha.3": - version "1.0.46" - resolved "https://registry.yarnpkg.com/@budibase/client/-/client-1.0.46.tgz#e6ef8945b9d7046b6e6d6761628aa1d85387acca" - integrity sha512-jI3z1G/EsfJNCQCvrqzsR4vR1zLoVefzCXCEASIPg9BPzdiAFSwuUJVLijLFIIKfuDVeveUll94fgu7XNY8U2w== +"@budibase/client@^1.0.46-alpha.5": + version "1.0.47" + resolved "https://registry.yarnpkg.com/@budibase/client/-/client-1.0.47.tgz#ce9e2fbd300e5dc389ea29a3a3347897f096c824" + integrity sha512-jB/al8v+nY/VLc6sH5Jt9JzWONVo+24/cI95iXlZSV5xwiKIVGj4+2F5QjKZ0c9Gm7SrrfP2T571N+4XaXNCGg== dependencies: - "@budibase/bbui" "^1.0.46" + "@budibase/bbui" "^1.0.47" "@budibase/standard-components" "^0.9.139" - "@budibase/string-templates" "^1.0.46" + "@budibase/string-templates" "^1.0.47" regexparam "^1.3.0" shortid "^2.2.15" svelte-spa-router "^3.0.5" @@ -1163,10 +1163,10 @@ svelte-apexcharts "^1.0.2" svelte-flatpickr "^3.1.0" -"@budibase/string-templates@^1.0.46", "@budibase/string-templates@^1.0.46-alpha.3": - version "1.0.46" - resolved "https://registry.yarnpkg.com/@budibase/string-templates/-/string-templates-1.0.46.tgz#5beef1687b451e4512a465b4e143c8ab46234006" - integrity sha512-t4ZAUkSz2XatjAN0faex5ovmD3mFz672lV/aBk7tfLFzZiKlWjngqdwpLLQNnsqeGvYo75JP2J06j86SX6O83w== +"@budibase/string-templates@^1.0.46-alpha.5", "@budibase/string-templates@^1.0.47": + version "1.0.47" + resolved "https://registry.yarnpkg.com/@budibase/string-templates/-/string-templates-1.0.47.tgz#626b9fc4542c7b36a0ae24e820d25a704c527bec" + integrity sha512-87BUfOPr8FGKH8Pt88jhKNGT9PcOmkLRCeen4xi1dI113pAQznBO9vgV+cXOChUBBEQka9Rrt85LMJXidiwVgg== dependencies: "@budibase/handlebars-helpers" "^0.11.7" dayjs "^1.10.4" @@ -2417,6 +2417,11 @@ dependencies: "@types/node" "*" +"@types/google-spreadsheet@^3.1.5": + version "3.1.5" + resolved "https://registry.yarnpkg.com/@types/google-spreadsheet/-/google-spreadsheet-3.1.5.tgz#2bdc6f9f5372551e0506cb6ef3f562adcf44fc2e" + integrity sha512-7N+mDtZ1pmya2RRFPPl4KYc2TRgiqCNBLUZfyrKfER+u751JgCO+C24/LzF70UmUm/zhHUbzRZ5mtfaxekQ1ZQ== + "@types/graceful-fs@^4.1.2": version "4.1.5" resolved "https://registry.yarnpkg.com/@types/graceful-fs/-/graceful-fs-4.1.5.tgz#21ffba0d98da4350db64891f92a9e5db3cdb4e15" @@ -3193,6 +3198,11 @@ array-unique@^0.3.2: resolved "https://registry.yarnpkg.com/array-unique/-/array-unique-0.3.2.tgz#a894b75d4bc4f6cd679ef3244a9fd8f46ae2d428" integrity sha1-qJS3XUvE9s1nnvMkSp/Y9Gri1Cg= +arrify@^2.0.0: + version "2.0.1" + resolved "https://registry.yarnpkg.com/arrify/-/arrify-2.0.1.tgz#c9655e9331e0abcd588d2a7cad7e9956f66701fa" + integrity sha512-3duEwti880xqi4eAMN8AyR4a0ByT90zoYdLlevfrvU43vb0YZwZVfxOgxWrLXXXpyugL0hNZc9G6BiB5B3nUug== + asap@^2.0.3: version "2.0.6" resolved "https://registry.yarnpkg.com/asap/-/asap-2.0.6.tgz#e50347611d7e690943208bbdafebcbc2fb866d46" @@ -3495,7 +3505,7 @@ base62@^1.1.0: resolved "https://registry.yarnpkg.com/base62/-/base62-1.2.8.tgz#1264cb0fb848d875792877479dbe8bae6bae3428" integrity sha512-V6YHUbjLxN1ymqNLb1DPHoU1CpfdL7d2YTIp5W3U4hhoG4hhxNmsFDs66M9EXxBiSEke5Bt5dwdfMwwZF70iLA== -base64-js@^1.0.2, base64-js@^1.3.1: +base64-js@^1.0.2, base64-js@^1.3.0, base64-js@^1.3.1: version "1.5.1" resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.5.1.tgz#1b1b440160a5bf7ad40b650f095963481903930a" integrity sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA== @@ -3535,6 +3545,11 @@ big.js@^5.2.2: resolved "https://registry.yarnpkg.com/big.js/-/big.js-5.2.2.tgz#65f0af382f578bcdc742bd9c281e9cb2d7768328" integrity sha512-vyL2OymJxmarO8gxMr0mhChsO9QGwhynfuu4+MHTAW6czfq9humCB7rKpUjDd9YUiDPU4mzpyupFSvOClAwbmQ== +bignumber.js@^9.0.0: + version "9.0.1" + resolved "https://registry.yarnpkg.com/bignumber.js/-/bignumber.js-9.0.1.tgz#8d7ba124c882bfd8e43260c67475518d0689e4e5" + integrity sha512-IdZR9mh6ahOBv/hYGiXyVuyCetmGJhtYkqLBpTStdhEGjegpPlUawydyaF3pbIOFynJTpllEs+NP+CS9jKFLjA== + binary-extensions@^2.0.0: version "2.2.0" resolved "https://registry.yarnpkg.com/binary-extensions/-/binary-extensions-2.2.0.tgz#75f502eeaf9ffde42fc98829645be4ea76bd9e2d" @@ -4826,7 +4841,7 @@ ecc-jsbn@~0.1.1: jsbn "~0.1.0" safer-buffer "^2.1.0" -ecdsa-sig-formatter@1.0.11: +ecdsa-sig-formatter@1.0.11, ecdsa-sig-formatter@^1.0.11: version "1.0.11" resolved "https://registry.yarnpkg.com/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz#ae0f0fa2d85045ef14a817daa3ce9acd0489e5bf" integrity sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ== @@ -5490,7 +5505,7 @@ extend-shallow@^3.0.0, extend-shallow@^3.0.2: assign-symbols "^1.0.0" is-extendable "^1.0.1" -extend@^3.0.0, extend@~3.0.2: +extend@^3.0.0, extend@^3.0.2, extend@~3.0.2: version "3.0.2" resolved "https://registry.yarnpkg.com/extend/-/extend-3.0.2.tgz#f8b1136b4071fbd8eb140aff858b1019ec2915fa" integrity sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g== @@ -5569,6 +5584,11 @@ fast-safe-stringify@^2.0.7, fast-safe-stringify@^2.0.8: resolved "https://registry.yarnpkg.com/fast-safe-stringify/-/fast-safe-stringify-2.1.1.tgz#c406a83b6e70d9e35ce3b30a81141df30aeba884" integrity sha512-W+KJc2dmILlPplD/H4K9l9LcAHAfPtP6BY84uVLXQ6Evcz9Lcg33Y2z1IVblT6xdY54PXYVHEv+0Wpq8Io6zkA== +fast-text-encoding@^1.0.0: + version "1.0.3" + resolved "https://registry.yarnpkg.com/fast-text-encoding/-/fast-text-encoding-1.0.3.tgz#ec02ac8e01ab8a319af182dae2681213cfe9ce53" + integrity sha512-dtm4QZH9nZtcDt8qJiOH9fcQd1NAgi+K1O2DbE6GG1PPCK/BWfOH3idCTRQ4ImXRUOyopDEgDEnVEE7Y/2Wrig== + fast-url-parser@^1.1.3: version "1.1.3" resolved "https://registry.yarnpkg.com/fast-url-parser/-/fast-url-parser-1.1.3.tgz#f4af3ea9f34d8a271cf58ad2b3759f431f0b318d" @@ -5919,6 +5939,25 @@ functional-red-black-tree@^1.0.1: resolved "https://registry.yarnpkg.com/functional-red-black-tree/-/functional-red-black-tree-1.0.1.tgz#1b0ab3bd553b2a0d6399d29c0e3ea0b252078327" integrity sha1-GwqzvVU7Kg1jmdKcDj6gslIHgyc= +gaxios@^4.0.0: + version "4.3.2" + resolved "https://registry.yarnpkg.com/gaxios/-/gaxios-4.3.2.tgz#845827c2dc25a0213c8ab4155c7a28910f5be83f" + integrity sha512-T+ap6GM6UZ0c4E6yb1y/hy2UB6hTrqhglp3XfmU9qbLCGRYhLVV5aRPpC4EmoG8N8zOnkYCgoBz+ScvGAARY6Q== + dependencies: + abort-controller "^3.0.0" + extend "^3.0.2" + https-proxy-agent "^5.0.0" + is-stream "^2.0.0" + node-fetch "^2.6.1" + +gcp-metadata@^4.2.0: + version "4.3.1" + resolved "https://registry.yarnpkg.com/gcp-metadata/-/gcp-metadata-4.3.1.tgz#fb205fe6a90fef2fd9c85e6ba06e5559ee1eefa9" + integrity sha512-x850LS5N7V1F3UcV7PoupzGsyD6iVwTVvsh3tbXfkctZnBnjW5yu5z1/3k3SehF7TyoTIe78rJs02GMMy+LF+A== + dependencies: + gaxios "^4.0.0" + json-bigint "^1.0.0" + generate-function@^2.3.1: version "2.3.1" resolved "https://registry.yarnpkg.com/generate-function/-/generate-function-2.3.1.tgz#f069617690c10c868e73b8465746764f97c3479f" @@ -6116,6 +6155,36 @@ globby@^11.0.3: merge2 "^1.3.0" slash "^3.0.0" +google-auth-library@^6.1.3: + version "6.1.6" + resolved "https://registry.yarnpkg.com/google-auth-library/-/google-auth-library-6.1.6.tgz#deacdcdb883d9ed6bac78bb5d79a078877fdf572" + integrity sha512-Q+ZjUEvLQj/lrVHF/IQwRo6p3s8Nc44Zk/DALsN+ac3T4HY/g/3rrufkgtl+nZ1TW7DNAw5cTChdVp4apUXVgQ== + dependencies: + arrify "^2.0.0" + base64-js "^1.3.0" + ecdsa-sig-formatter "^1.0.11" + fast-text-encoding "^1.0.0" + gaxios "^4.0.0" + gcp-metadata "^4.2.0" + gtoken "^5.0.4" + jws "^4.0.0" + lru-cache "^6.0.0" + +google-auth-library@^7.11.0: + version "7.11.0" + resolved "https://registry.yarnpkg.com/google-auth-library/-/google-auth-library-7.11.0.tgz#b63699c65037310a424128a854ba7e736704cbdb" + integrity sha512-3S5jn2quRumvh9F/Ubf7GFrIq71HZ5a6vqosgdIu105kkk0WtSqc2jGCRqtWWOLRS8SX3AHACMOEDxhyWAQIcg== + dependencies: + arrify "^2.0.0" + base64-js "^1.3.0" + ecdsa-sig-formatter "^1.0.11" + fast-text-encoding "^1.0.0" + gaxios "^4.0.0" + gcp-metadata "^4.2.0" + gtoken "^5.0.4" + jws "^4.0.0" + lru-cache "^6.0.0" + google-auth-library@~0.10.0: version "0.10.0" resolved "https://registry.yarnpkg.com/google-auth-library/-/google-auth-library-0.10.0.tgz#6e15babee85fd1dd14d8d128a295b6838d52136e" @@ -6133,6 +6202,22 @@ google-p12-pem@^0.1.0: dependencies: node-forge "^0.7.1" +google-p12-pem@^3.0.3: + version "3.1.2" + resolved "https://registry.yarnpkg.com/google-p12-pem/-/google-p12-pem-3.1.2.tgz#c3d61c2da8e10843ff830fdb0d2059046238c1d4" + integrity sha512-tjf3IQIt7tWCDsa0ofDQ1qqSCNzahXDxdAGJDbruWqu3eCg5CKLYKN+hi0s6lfvzYZ1GDVr+oDF9OOWlDSdf0A== + dependencies: + node-forge "^0.10.0" + +google-spreadsheet@^3.2.0: + version "3.2.0" + resolved "https://registry.yarnpkg.com/google-spreadsheet/-/google-spreadsheet-3.2.0.tgz#ce8aa75c15705aa950ad52b091a6fc4d33dcb329" + integrity sha512-z7XMaqb+26rdo8p51r5O03u8aPLAPzn5YhOXYJPcf2hdMVr0dUbIARgdkRdmGiBeoV/QoU/7VNhq1MMCLZv3kQ== + dependencies: + axios "^0.21.4" + google-auth-library "^6.1.3" + lodash "^4.17.21" + googleapis@^16.0.0: version "16.1.0" resolved "https://registry.yarnpkg.com/googleapis/-/googleapis-16.1.0.tgz#0f19f2d70572d918881a0f626e3b1a2fa8629576" @@ -6197,6 +6282,15 @@ gtoken@^1.2.1: mime "^1.4.1" request "^2.72.0" +gtoken@^5.0.4: + version "5.3.1" + resolved "https://registry.yarnpkg.com/gtoken/-/gtoken-5.3.1.tgz#c1c2598a826f2b5df7c6bb53d7be6cf6d50c3c78" + integrity sha512-yqOREjzLHcbzz1UrQoxhBtpk8KjrVhuqPE7od1K2uhyxG2BHjKZetlbLw/SPZak/QqTIQW+addS+EcjqQsZbwQ== + dependencies: + gaxios "^4.0.0" + google-p12-pem "^3.0.3" + jws "^4.0.0" + gulp-header@^1.7.1: version "1.8.12" resolved "https://registry.yarnpkg.com/gulp-header/-/gulp-header-1.8.12.tgz#ad306be0066599127281c4f8786660e705080a84" @@ -8084,6 +8178,13 @@ jsesc@~0.5.0: resolved "https://registry.yarnpkg.com/jsesc/-/jsesc-0.5.0.tgz#e7dee66e35d6fc16f710fe91d5cf69f70f08911d" integrity sha1-597mbjXW/Bb3EP6R1c9p9w8IkR0= +json-bigint@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/json-bigint/-/json-bigint-1.0.0.tgz#ae547823ac0cad8398667f8cd9ef4730f5b01ff1" + integrity sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ== + dependencies: + bignumber.js "^9.0.0" + json-buffer@3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/json-buffer/-/json-buffer-3.0.0.tgz#5b1f397afc75d677bde8bcfc0e47e1f9a3d9a898" @@ -8197,6 +8298,15 @@ jwa@^1.4.1: ecdsa-sig-formatter "1.0.11" safe-buffer "^5.0.1" +jwa@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/jwa/-/jwa-2.0.0.tgz#a7e9c3f29dae94027ebcaf49975c9345593410fc" + integrity sha512-jrZ2Qx916EA+fq9cEAeCROWPTfCwi1IVHqT2tapuqLEVVDKFDENFw1oL+MwrTvH6msKxsd1YTDVw6uKEcsrLEA== + dependencies: + buffer-equal-constant-time "1.0.1" + ecdsa-sig-formatter "1.0.11" + safe-buffer "^5.0.1" + jws@3.x.x, jws@^3.0.0, jws@^3.1.4, jws@^3.2.2: version "3.2.2" resolved "https://registry.yarnpkg.com/jws/-/jws-3.2.2.tgz#001099f3639468c9414000e99995fa52fb478304" @@ -8205,6 +8315,14 @@ jws@3.x.x, jws@^3.0.0, jws@^3.1.4, jws@^3.2.2: jwa "^1.4.1" safe-buffer "^5.0.1" +jws@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/jws/-/jws-4.0.0.tgz#2d4e8cf6a318ffaa12615e9dec7e86e6c97310f4" + integrity sha512-KDncfTmOZoOMTFG4mBlG0qUIOlc03fmzH+ru6RgYVZhPkyiy/92Owlt/8UEN+a4TXR1FQetfIpJE8ApdvdVxTg== + dependencies: + jwa "^2.0.0" + safe-buffer "^5.0.1" + keygrip@~1.0.3: version "1.0.3" resolved "https://registry.yarnpkg.com/keygrip/-/keygrip-1.0.3.tgz#399d709f0aed2bab0a059e0cdd3a5023a053e1dc" @@ -9325,6 +9443,11 @@ node-fetch@2.6.7, node-fetch@^2.6.0, node-fetch@^2.6.1: dependencies: whatwg-url "^5.0.0" +node-forge@^0.10.0: + version "0.10.0" + resolved "https://registry.yarnpkg.com/node-forge/-/node-forge-0.10.0.tgz#32dea2afb3e9926f02ee5ce8794902691a676bf3" + integrity sha512-PPmu8eEeG9saEUvI97fm4OYxXVB6bFvyNTyiUOBichBpFG8A1Ljw3bY62+5oOjDEMHRnd0Y7HQ+x7uzxOzC6JA== + node-forge@^0.7.1: version "0.7.6" resolved "https://registry.yarnpkg.com/node-forge/-/node-forge-0.7.6.tgz#fdf3b418aee1f94f0ef642cd63486c77ca9724ac" diff --git a/packages/string-templates/package.json b/packages/string-templates/package.json index 012ebf8a06..868553f68d 100644 --- a/packages/string-templates/package.json +++ b/packages/string-templates/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/string-templates", - "version": "1.0.46-alpha.5", + "version": "1.0.46-alpha.6", "description": "Handlebars wrapper for Budibase templating.", "main": "src/index.cjs", "module": "dist/bundle.mjs", diff --git a/packages/string-templates/src/index.js b/packages/string-templates/src/index.js index 820b8da290..d824d5f1db 100644 --- a/packages/string-templates/src/index.js +++ b/packages/string-templates/src/index.js @@ -112,9 +112,10 @@ module.exports.processStringSync = (string, context, opts) => { const template = instance.compile(string, { strict: false, }) + const now = Math.floor(Date.now() / 1000) * 1000 return processors.postprocess( template({ - now: new Date().toISOString(), + now: new Date(now).toISOString(), ...context, }) ) diff --git a/packages/worker/package.json b/packages/worker/package.json index d4012a02dc..d3a7a12dc1 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/worker", "email": "hi@budibase.com", - "version": "1.0.46-alpha.5", + "version": "1.0.46-alpha.6", "description": "Budibase background service", "main": "src/index.js", "repository": { @@ -29,8 +29,8 @@ "author": "Budibase", "license": "GPL-3.0", "dependencies": { - "@budibase/backend-core": "^1.0.46-alpha.5", - "@budibase/string-templates": "^1.0.46-alpha.5", + "@budibase/backend-core": "^1.0.46-alpha.6", + "@budibase/string-templates": "^1.0.46-alpha.6", "@koa/router": "^8.0.0", "@sentry/node": "^6.0.0", "@techpass/passport-openidconnect": "^0.3.0", diff --git a/packages/worker/src/api/controllers/global/auth.js b/packages/worker/src/api/controllers/global/auth.js index 44ee99aee7..b39e8745e9 100644 --- a/packages/worker/src/api/controllers/global/auth.js +++ b/packages/worker/src/api/controllers/global/auth.js @@ -147,6 +147,32 @@ exports.logout = async ctx => { ctx.body = { message: "User logged out." } } +exports.datasourcePreAuth = async (ctx, next) => { + const provider = ctx.params.provider + const middleware = require(`@budibase/backend-core/middleware`) + const handler = middleware.datasource[provider] + + setCookie( + ctx, + { + provider, + appId: ctx.query.appId, + datasourceId: ctx.query.datasourceId, + }, + Cookies.DatasourceAuth + ) + + return handler.preAuth(passport, ctx, next) +} + +exports.datasourceAuth = async (ctx, next) => { + const authStateCookie = getCookie(ctx, Cookies.DatasourceAuth) + const provider = authStateCookie.provider + const middleware = require(`@budibase/backend-core/middleware`) + const handler = middleware.datasource[provider] + return handler.postAuth(passport, ctx, next) +} + /** * The initial call that google authentication makes to take you to the google login screen. * On a successful login, you will be redirected to the googleAuth callback route. diff --git a/packages/worker/src/api/routes/global/auth.js b/packages/worker/src/api/routes/global/auth.js index 20c615b85a..373bf5736a 100644 --- a/packages/worker/src/api/routes/global/auth.js +++ b/packages/worker/src/api/routes/global/auth.js @@ -63,8 +63,17 @@ router updateTenant, authController.googlePreAuth ) + .get( + "/api/global/auth/:tenantId/datasource/:provider", + updateTenant, + authController.datasourcePreAuth + ) // single tenancy endpoint .get("/api/global/auth/google/callback", authController.googleAuth) + .get( + "/api/global/auth/datasource/:provider/callback", + authController.datasourceAuth + ) // multi-tenancy endpoint .get( "/api/global/auth/:tenantId/google/callback",