692 lines
19 KiB
TypeScript
692 lines
19 KiB
TypeScript
import {
|
|
ConnectionInfo,
|
|
DatasourceFeature,
|
|
DatasourceFieldType,
|
|
DatasourcePlus,
|
|
FieldType,
|
|
Integration,
|
|
Operation,
|
|
PaginationJson,
|
|
QueryJson,
|
|
QueryType,
|
|
Row,
|
|
Schema,
|
|
SearchFilters,
|
|
SortJson,
|
|
Table,
|
|
TableRequest,
|
|
TableSourceType,
|
|
DatasourcePlusQueryResponse,
|
|
BBReferenceFieldSubType,
|
|
} from "@budibase/types"
|
|
import { OAuth2Client } from "google-auth-library"
|
|
import {
|
|
buildExternalTableId,
|
|
checkExternalTables,
|
|
finaliseExternalTables,
|
|
} from "./utils"
|
|
import { GoogleSpreadsheet, GoogleSpreadsheetRow } from "google-spreadsheet"
|
|
import fetch from "node-fetch"
|
|
import { cache, configs, context, HTTPError } from "@budibase/backend-core"
|
|
import { dataFilters, utils } from "@budibase/shared-core"
|
|
import { GOOGLE_SHEETS_PRIMARY_KEY } from "../constants"
|
|
|
|
interface GoogleSheetsConfig {
|
|
spreadsheetId: string
|
|
auth: OAuthClientConfig
|
|
continueSetupId?: string
|
|
}
|
|
|
|
interface OAuthClientConfig {
|
|
appId: string
|
|
accessToken: string
|
|
refreshToken: string
|
|
}
|
|
|
|
interface AuthTokenRequest {
|
|
client_id: string
|
|
client_secret: string
|
|
refresh_token: string
|
|
}
|
|
|
|
interface AuthTokenResponse {
|
|
access_token: string
|
|
}
|
|
|
|
const isTypeAllowed: Record<FieldType, boolean> = {
|
|
[FieldType.STRING]: true,
|
|
[FieldType.FORMULA]: true,
|
|
[FieldType.NUMBER]: true,
|
|
[FieldType.LONGFORM]: true,
|
|
[FieldType.DATETIME]: true,
|
|
[FieldType.OPTIONS]: true,
|
|
[FieldType.BOOLEAN]: true,
|
|
[FieldType.BARCODEQR]: true,
|
|
[FieldType.BB_REFERENCE]: true,
|
|
[FieldType.BB_REFERENCE_SINGLE]: true,
|
|
[FieldType.ARRAY]: false,
|
|
[FieldType.ATTACHMENTS]: false,
|
|
[FieldType.ATTACHMENT_SINGLE]: false,
|
|
[FieldType.LINK]: false,
|
|
[FieldType.AUTO]: false,
|
|
[FieldType.JSON]: false,
|
|
[FieldType.INTERNAL]: false,
|
|
[FieldType.BIGINT]: false,
|
|
[FieldType.SIGNATURE_SINGLE]: false,
|
|
}
|
|
|
|
const ALLOWED_TYPES = Object.entries(isTypeAllowed)
|
|
.filter(([_, allowed]) => allowed)
|
|
.map(([type]) => type as FieldType)
|
|
|
|
const SCHEMA: Integration = {
|
|
plus: true,
|
|
auth: {
|
|
type: "google",
|
|
},
|
|
relationships: false,
|
|
docs: "https://developers.google.com/sheets/api/quickstart/nodejs",
|
|
description:
|
|
"Create and collaborate on online spreadsheets in real-time and from any device.",
|
|
friendlyName: "Google Sheets",
|
|
type: "Spreadsheet",
|
|
features: {
|
|
[DatasourceFeature.CONNECTION_CHECKING]: true,
|
|
[DatasourceFeature.FETCH_TABLE_NAMES]: true,
|
|
},
|
|
datasource: {
|
|
spreadsheetId: {
|
|
display: "Spreadsheet URL",
|
|
type: DatasourceFieldType.STRING,
|
|
required: true,
|
|
},
|
|
},
|
|
query: {
|
|
create: {
|
|
type: QueryType.FIELDS,
|
|
fields: {
|
|
sheet: {
|
|
type: DatasourceFieldType.STRING,
|
|
required: true,
|
|
},
|
|
row: {
|
|
type: QueryType.JSON,
|
|
required: true,
|
|
},
|
|
},
|
|
},
|
|
read: {
|
|
type: QueryType.FIELDS,
|
|
fields: {
|
|
sheet: {
|
|
type: DatasourceFieldType.STRING,
|
|
required: true,
|
|
},
|
|
},
|
|
},
|
|
update: {
|
|
type: QueryType.FIELDS,
|
|
fields: {
|
|
sheet: {
|
|
type: DatasourceFieldType.STRING,
|
|
required: true,
|
|
},
|
|
rowIndex: {
|
|
type: DatasourceFieldType.STRING,
|
|
required: true,
|
|
},
|
|
row: {
|
|
type: QueryType.JSON,
|
|
required: true,
|
|
},
|
|
},
|
|
},
|
|
delete: {
|
|
type: QueryType.FIELDS,
|
|
fields: {
|
|
sheet: {
|
|
type: DatasourceFieldType.STRING,
|
|
required: true,
|
|
},
|
|
rowIndex: {
|
|
type: DatasourceFieldType.NUMBER,
|
|
required: true,
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
class GoogleSheetsIntegration implements DatasourcePlus {
|
|
private readonly config: GoogleSheetsConfig
|
|
private readonly spreadsheetId: string
|
|
private client: GoogleSpreadsheet = undefined!
|
|
|
|
constructor(config: GoogleSheetsConfig) {
|
|
this.config = config
|
|
this.spreadsheetId = this.cleanSpreadsheetUrl(this.config.spreadsheetId)
|
|
}
|
|
|
|
async testConnection(): Promise<ConnectionInfo> {
|
|
try {
|
|
await this.connect()
|
|
return { connected: true }
|
|
} catch (e: any) {
|
|
return {
|
|
connected: false,
|
|
error: e.message as string,
|
|
}
|
|
}
|
|
}
|
|
|
|
getBindingIdentifier() {
|
|
return ""
|
|
}
|
|
|
|
getStringConcat(_parts: string[]) {
|
|
return ""
|
|
}
|
|
|
|
/**
|
|
* Pull the spreadsheet ID out from a valid google sheets URL
|
|
* @param spreadsheetId - the URL or standard spreadsheetId of the google sheet
|
|
* @returns spreadsheet Id of the google sheet
|
|
*/
|
|
private cleanSpreadsheetUrl(spreadsheetId: string) {
|
|
if (!spreadsheetId) {
|
|
throw new Error(
|
|
"You must set a spreadsheet ID in your configuration to fetch tables."
|
|
)
|
|
}
|
|
const parts = spreadsheetId.split("/")
|
|
return parts.length > 5 ? parts[5] : spreadsheetId
|
|
}
|
|
|
|
private async fetchAccessToken(
|
|
payload: AuthTokenRequest
|
|
): Promise<AuthTokenResponse> {
|
|
const response = await fetch("https://www.googleapis.com/oauth2/v4/token", {
|
|
method: "POST",
|
|
body: JSON.stringify({
|
|
...payload,
|
|
grant_type: "refresh_token",
|
|
}),
|
|
headers: {
|
|
"Content-Type": "application/json",
|
|
},
|
|
})
|
|
|
|
const json = await response.json()
|
|
|
|
if (response.status !== 200) {
|
|
throw new Error(
|
|
`Error authenticating with google sheets. ${json.error_description}`
|
|
)
|
|
}
|
|
|
|
return json
|
|
}
|
|
|
|
private async connect() {
|
|
try {
|
|
const bbCtx = context.getCurrentContext()
|
|
let oauthClient = bbCtx?.googleSheets?.oauthClient
|
|
|
|
if (!oauthClient) {
|
|
await setupCreationAuth(this.config)
|
|
|
|
// Initialise oAuth client
|
|
const googleConfig = await configs.getGoogleDatasourceConfig()
|
|
if (!googleConfig) {
|
|
throw new HTTPError("Google config not found", 400)
|
|
}
|
|
|
|
oauthClient = new OAuth2Client({
|
|
clientId: googleConfig.clientID,
|
|
clientSecret: googleConfig.clientSecret,
|
|
})
|
|
|
|
const tokenResponse = await this.fetchAccessToken({
|
|
client_id: googleConfig.clientID,
|
|
client_secret: googleConfig.clientSecret,
|
|
refresh_token: this.config.auth.refreshToken,
|
|
})
|
|
|
|
oauthClient.setCredentials({
|
|
refresh_token: this.config.auth.refreshToken,
|
|
access_token: tokenResponse.access_token,
|
|
})
|
|
if (bbCtx && !bbCtx.googleSheets) {
|
|
bbCtx.googleSheets = {
|
|
oauthClient,
|
|
clients: {},
|
|
}
|
|
bbCtx.cleanup = bbCtx.cleanup || []
|
|
}
|
|
}
|
|
|
|
let client = bbCtx?.googleSheets?.clients[this.spreadsheetId]
|
|
if (!client) {
|
|
client = new GoogleSpreadsheet(this.spreadsheetId, oauthClient)
|
|
await client.loadInfo()
|
|
|
|
if (bbCtx?.googleSheets?.clients) {
|
|
bbCtx.googleSheets.clients[this.spreadsheetId] = client
|
|
}
|
|
}
|
|
|
|
this.client = client
|
|
} catch (err: any) {
|
|
// this happens for xlsx imports
|
|
if (err.message?.includes("operation is not supported")) {
|
|
err.message =
|
|
"This operation is not supported - XLSX sheets must be converted."
|
|
}
|
|
console.error("Error connecting to google sheets", err)
|
|
throw err
|
|
}
|
|
}
|
|
|
|
async getTableNames(): Promise<string[]> {
|
|
await this.connect()
|
|
const sheets = this.client.sheetsByIndex
|
|
return sheets.map(s => s.title)
|
|
}
|
|
|
|
private getTableSchema(
|
|
title: string,
|
|
headerValues: string[],
|
|
datasourceId: string,
|
|
id?: string
|
|
) {
|
|
// base table
|
|
const table: Table = {
|
|
type: "table",
|
|
name: title,
|
|
primary: [GOOGLE_SHEETS_PRIMARY_KEY],
|
|
schema: {},
|
|
sourceId: datasourceId,
|
|
sourceType: TableSourceType.EXTERNAL,
|
|
}
|
|
if (id) {
|
|
table._id = id
|
|
}
|
|
// build schema from headers
|
|
for (let header of headerValues) {
|
|
table.schema[header] = {
|
|
name: header,
|
|
type: FieldType.STRING,
|
|
}
|
|
}
|
|
return table
|
|
}
|
|
|
|
async buildSchema(
|
|
datasourceId: string,
|
|
entities: Record<string, Table>
|
|
): Promise<Schema> {
|
|
// not fully configured yet
|
|
if (!this.config.auth) {
|
|
return { tables: {}, errors: {} }
|
|
}
|
|
await this.connect()
|
|
const sheets = this.client.sheetsByIndex
|
|
const tables: Record<string, Table> = {}
|
|
let errors: Record<string, string> = {}
|
|
await utils.parallelForeach(
|
|
sheets,
|
|
async sheet => {
|
|
// must fetch rows to determine schema
|
|
try {
|
|
await sheet.getRows()
|
|
} catch (err) {
|
|
// We expect this to always be an Error so if it's not, rethrow it to
|
|
// make sure we don't fail quietly.
|
|
if (!(err instanceof Error)) {
|
|
throw err
|
|
}
|
|
|
|
if (err.message.startsWith("No values in the header row")) {
|
|
errors[sheet.title] = err.message
|
|
} else {
|
|
// If we get an error we don't expect, rethrow to avoid failing
|
|
// quietly.
|
|
throw err
|
|
}
|
|
return
|
|
}
|
|
|
|
const id = buildExternalTableId(datasourceId, sheet.title)
|
|
tables[sheet.title] = this.getTableSchema(
|
|
sheet.title,
|
|
sheet.headerValues,
|
|
datasourceId,
|
|
id
|
|
)
|
|
},
|
|
10
|
|
)
|
|
let externalTables = finaliseExternalTables(tables, entities)
|
|
errors = { ...errors, ...checkExternalTables(externalTables) }
|
|
return { tables: externalTables, errors }
|
|
}
|
|
|
|
async query(json: QueryJson): Promise<DatasourcePlusQueryResponse> {
|
|
const sheet = json.endpoint.entityId
|
|
switch (json.endpoint.operation) {
|
|
case Operation.CREATE:
|
|
return this.create({ sheet, row: json.body as Row })
|
|
case Operation.BULK_CREATE:
|
|
return this.createBulk({ sheet, rows: json.body as Row[] })
|
|
case Operation.READ:
|
|
return this.read({ ...json, sheet })
|
|
case Operation.UPDATE:
|
|
return this.update({
|
|
// exclude the header row and zero index
|
|
rowIndex: json.extra?.idFilter?.equal?.rowNumber,
|
|
sheet,
|
|
row: json.body,
|
|
table: json.meta.table,
|
|
})
|
|
case Operation.DELETE:
|
|
return this.delete({
|
|
// exclude the header row and zero index
|
|
rowIndex: json.extra?.idFilter?.equal?.rowNumber,
|
|
sheet,
|
|
})
|
|
case Operation.CREATE_TABLE:
|
|
return this.createTable(json?.table?.name)
|
|
case Operation.UPDATE_TABLE:
|
|
return this.updateTable(json.table!)
|
|
case Operation.DELETE_TABLE:
|
|
return this.deleteTable(json?.table?.name)
|
|
default:
|
|
throw new Error(
|
|
`GSheets integration does not support "${json.endpoint.operation}".`
|
|
)
|
|
}
|
|
}
|
|
|
|
private buildRowObject(
|
|
headers: string[],
|
|
values: Record<string, string>,
|
|
rowNumber: number
|
|
) {
|
|
const rowObject: { rowNumber: number } & Row = {
|
|
rowNumber,
|
|
_id: rowNumber.toString(),
|
|
}
|
|
for (let i = 0; i < headers.length; i++) {
|
|
rowObject[headers[i]] = values[headers[i]]
|
|
}
|
|
return rowObject
|
|
}
|
|
|
|
private async createTable(name?: string) {
|
|
if (!name) {
|
|
throw new Error("Must provide name for new sheet.")
|
|
}
|
|
try {
|
|
await this.connect()
|
|
await this.client.addSheet({ title: name, headerValues: [name] })
|
|
} catch (err) {
|
|
console.error("Error creating new table in google sheets", err)
|
|
throw err
|
|
}
|
|
}
|
|
|
|
private async updateTable(table: TableRequest) {
|
|
await this.connect()
|
|
const sheet = this.client.sheetsByTitle[table.name]
|
|
await sheet.loadHeaderRow()
|
|
|
|
if (table._rename) {
|
|
const headers = []
|
|
for (let header of sheet.headerValues) {
|
|
if (header === table._rename.old) {
|
|
headers.push(table._rename.updated)
|
|
} else {
|
|
headers.push(header)
|
|
}
|
|
}
|
|
try {
|
|
await sheet.setHeaderRow(headers)
|
|
} catch (err) {
|
|
console.error("Error updating column name in google sheets", err)
|
|
throw err
|
|
}
|
|
} else {
|
|
const updatedHeaderValues = [...sheet.headerValues]
|
|
|
|
// add new column - doesn't currently exist
|
|
for (let [key, column] of Object.entries(table.schema)) {
|
|
if (!ALLOWED_TYPES.includes(column.type)) {
|
|
throw new Error(
|
|
`Column type: ${column.type} not allowed for GSheets integration.`
|
|
)
|
|
}
|
|
if (
|
|
!sheet.headerValues.includes(key) &&
|
|
column.type !== FieldType.FORMULA
|
|
) {
|
|
updatedHeaderValues.push(key)
|
|
}
|
|
}
|
|
|
|
try {
|
|
await sheet.setHeaderRow(updatedHeaderValues)
|
|
} catch (err) {
|
|
console.error("Error updating table in google sheets", err)
|
|
throw err
|
|
}
|
|
}
|
|
}
|
|
|
|
private async deleteTable(sheet: any) {
|
|
try {
|
|
await this.connect()
|
|
const sheetToDelete = this.client.sheetsByTitle[sheet]
|
|
await sheetToDelete.delete()
|
|
} catch (err) {
|
|
console.error("Error deleting table in google sheets", err)
|
|
throw err
|
|
}
|
|
}
|
|
|
|
async create(query: { sheet: string; row: Row }) {
|
|
try {
|
|
await this.connect()
|
|
const sheet = this.client.sheetsByTitle[query.sheet]
|
|
const rowToInsert =
|
|
typeof query.row === "string" ? JSON.parse(query.row) : query.row
|
|
const row = await sheet.addRow(rowToInsert)
|
|
return [
|
|
this.buildRowObject(sheet.headerValues, row.toObject(), row.rowNumber),
|
|
]
|
|
} catch (err) {
|
|
console.error("Error writing to google sheets", err)
|
|
throw err
|
|
}
|
|
}
|
|
|
|
private async createBulk(query: { sheet: string; rows: Row[] }) {
|
|
try {
|
|
await this.connect()
|
|
const sheet = this.client.sheetsByTitle[query.sheet]
|
|
let rowsToInsert = []
|
|
for (let row of query.rows) {
|
|
rowsToInsert.push(typeof row === "string" ? JSON.parse(row) : row)
|
|
}
|
|
const rows = await sheet.addRows(rowsToInsert)
|
|
return rows.map(row =>
|
|
this.buildRowObject(sheet.headerValues, row.toObject(), row.rowNumber)
|
|
)
|
|
} catch (err) {
|
|
console.error("Error bulk writing to google sheets", err)
|
|
throw err
|
|
}
|
|
}
|
|
|
|
async read(query: {
|
|
sheet: string
|
|
filters?: SearchFilters
|
|
sort?: SortJson
|
|
paginate?: PaginationJson
|
|
}) {
|
|
try {
|
|
await this.connect()
|
|
const hasFilters = dataFilters.hasFilters(query.filters)
|
|
const limit = query.paginate?.limit || 100
|
|
const page: number =
|
|
typeof query.paginate?.page === "number"
|
|
? query.paginate.page
|
|
: parseInt(query.paginate?.page || "1")
|
|
const offset = (page - 1) * limit
|
|
const sheet = this.client.sheetsByTitle[query.sheet]
|
|
let rows: GoogleSpreadsheetRow[] = []
|
|
if (query.paginate && !hasFilters) {
|
|
rows = await sheet.getRows({
|
|
limit,
|
|
offset,
|
|
})
|
|
} else {
|
|
rows = await sheet.getRows()
|
|
}
|
|
// this is a special case - need to handle the _id, it doesn't exist
|
|
// we cannot edit the returned structure from google, it does not have
|
|
// setter functions and is immutable, easier to update the filters
|
|
// to look for the _rowNumber property rather than rowNumber
|
|
if (query.filters?.equal) {
|
|
const idFilterKeys = Object.keys(query.filters.equal).filter(filter =>
|
|
filter.includes(GOOGLE_SHEETS_PRIMARY_KEY)
|
|
)
|
|
for (let idFilterKey of idFilterKeys) {
|
|
const id = query.filters.equal[idFilterKey]
|
|
delete query.filters.equal[idFilterKey]
|
|
query.filters.equal[`_${GOOGLE_SHEETS_PRIMARY_KEY}`] = id
|
|
}
|
|
}
|
|
let filtered = dataFilters.runLuceneQuery(rows, query.filters)
|
|
if (hasFilters && query.paginate) {
|
|
filtered = filtered.slice(offset, offset + limit)
|
|
}
|
|
const headerValues = sheet.headerValues
|
|
let response = []
|
|
for (let row of filtered) {
|
|
response.push(
|
|
this.buildRowObject(headerValues, row.toObject(), row._rowNumber)
|
|
)
|
|
}
|
|
|
|
if (query.sort) {
|
|
if (Object.keys(query.sort).length !== 1) {
|
|
console.warn("Googlesheets does not support multiple sorting", {
|
|
sortInfo: query.sort,
|
|
})
|
|
}
|
|
const [sortField, sortInfo] = Object.entries(query.sort)[0]
|
|
response = dataFilters.luceneSort(
|
|
response,
|
|
sortField,
|
|
sortInfo.direction,
|
|
sortInfo.type
|
|
)
|
|
}
|
|
|
|
return response
|
|
} catch (err) {
|
|
console.error("Error reading from google sheets", err)
|
|
throw err
|
|
}
|
|
}
|
|
|
|
private async getRowByIndex(sheetTitle: string, rowIndex: number) {
|
|
const sheet = this.client.sheetsByTitle[sheetTitle]
|
|
const rows = await sheet.getRows()
|
|
// We substract 2, as the SDK is skipping the header automatically and Google Spreadsheets is base 1
|
|
const row = rows[rowIndex - 2]
|
|
return { sheet, row }
|
|
}
|
|
|
|
async update(query: {
|
|
sheet: string
|
|
rowIndex: number
|
|
row: any
|
|
table: Table
|
|
}) {
|
|
try {
|
|
await this.connect()
|
|
const { sheet, row } = await this.getRowByIndex(
|
|
query.sheet,
|
|
query.rowIndex
|
|
)
|
|
if (row) {
|
|
const updateValues =
|
|
typeof query.row === "string" ? JSON.parse(query.row) : query.row
|
|
for (let key in updateValues) {
|
|
row.set(key, updateValues[key])
|
|
|
|
if (row.get(key) === null) {
|
|
row.set(key, "")
|
|
}
|
|
|
|
const { type, subtype, constraints } = query.table.schema[key]
|
|
const isDeprecatedSingleUser =
|
|
type === FieldType.BB_REFERENCE &&
|
|
subtype === BBReferenceFieldSubType.USER &&
|
|
constraints?.type !== "array"
|
|
if (isDeprecatedSingleUser && Array.isArray(row.get(key))) {
|
|
row.set(key, row.get(key)[0])
|
|
}
|
|
}
|
|
await row.save()
|
|
return [
|
|
this.buildRowObject(
|
|
sheet.headerValues,
|
|
row.toObject(),
|
|
row.rowNumber
|
|
),
|
|
]
|
|
} else {
|
|
throw new Error("Row does not exist.")
|
|
}
|
|
} catch (err) {
|
|
console.error("Error reading from google sheets", err)
|
|
throw err
|
|
}
|
|
}
|
|
|
|
async delete(query: { sheet: string; rowIndex: number }) {
|
|
await this.connect()
|
|
const { row } = await this.getRowByIndex(query.sheet, query.rowIndex)
|
|
if (row) {
|
|
await row.delete()
|
|
return [
|
|
{
|
|
deleted: query.rowIndex,
|
|
[GOOGLE_SHEETS_PRIMARY_KEY]: query.rowIndex,
|
|
},
|
|
]
|
|
} else {
|
|
throw new Error("Row does not exist.")
|
|
}
|
|
}
|
|
}
|
|
|
|
export async function setupCreationAuth(datasouce: GoogleSheetsConfig) {
|
|
if (datasouce.continueSetupId) {
|
|
const appId = context.getAppId()
|
|
const tokens = await cache.get(
|
|
`datasource:creation:${appId}:google:${datasouce.continueSetupId}`
|
|
)
|
|
|
|
datasouce.auth = tokens.tokens
|
|
delete datasouce.continueSetupId
|
|
}
|
|
}
|
|
|
|
export default {
|
|
schema: SCHEMA,
|
|
integration: GoogleSheetsIntegration,
|
|
}
|