diff --git a/packages/server/src/integrations/googlesheets.ts b/packages/server/src/integrations/googlesheets.ts index ea62281d4b..0d766ac1ef 100644 --- a/packages/server/src/integrations/googlesheets.ts +++ b/packages/server/src/integrations/googlesheets.ts @@ -31,7 +31,7 @@ import { cache, configs, context, HTTPError } from "@budibase/backend-core" import { dataFilters, utils } from "@budibase/shared-core" import { GOOGLE_SHEETS_PRIMARY_KEY } from "../constants" -interface GoogleSheetsConfig { +export interface GoogleSheetsConfig { spreadsheetId: string auth: OAuthClientConfig continueSetupId?: string @@ -157,7 +157,7 @@ const SCHEMA: Integration = { }, } -class GoogleSheetsIntegration implements DatasourcePlus { +export class GoogleSheetsIntegration implements DatasourcePlus { private readonly config: GoogleSheetsConfig private readonly spreadsheetId: string private client: GoogleSpreadsheet = undefined! @@ -378,6 +378,10 @@ class GoogleSheetsIntegration implements DatasourcePlus { return this.create({ sheet, row: json.body as Row }) case Operation.BULK_CREATE: return this.createBulk({ sheet, rows: json.body as Row[] }) + case Operation.BULK_UPSERT: + // This is technically not correct because it won't update existing + // rows, but it's better than not having this functionality at all. + return this.createBulk({ sheet, rows: json.body as Row[] }) case Operation.READ: return this.read({ ...json, sheet }) case Operation.UPDATE: @@ -395,9 +399,19 @@ class GoogleSheetsIntegration implements DatasourcePlus { sheet, }) case Operation.CREATE_TABLE: - return this.createTable(json?.table?.name) + if (!json.table) { + throw new Error( + "attempted to create a table without specifying the table to create" + ) + } + return this.createTable(json.table) case Operation.UPDATE_TABLE: - return this.updateTable(json.table!) + if (!json.table) { + throw new Error( + "attempted to create a table without specifying the table to create" + ) + } + return this.updateTable(json.table) case Operation.DELETE_TABLE: return this.deleteTable(json?.table?.name) default: @@ -422,13 +436,13 @@ class GoogleSheetsIntegration implements DatasourcePlus { return rowObject } - private async createTable(name?: string) { - if (!name) { - throw new Error("Must provide name for new sheet.") - } + private async createTable(table: Table) { try { await this.connect() - await this.client.addSheet({ title: name, headerValues: [name] }) + await this.client.addSheet({ + title: table.name, + headerValues: Object.keys(table.schema), + }) } catch (err) { console.error("Error creating new table in google sheets", err) throw err @@ -552,32 +566,15 @@ class GoogleSheetsIntegration implements DatasourcePlus { } else { rows = await sheet.getRows() } - // this is a special case - need to handle the _id, it doesn't exist - // we cannot edit the returned structure from google, it does not have - // setter functions and is immutable, easier to update the filters - // to look for the _rowNumber property rather than rowNumber - if (query.filters?.equal) { - const idFilterKeys = Object.keys(query.filters.equal).filter(filter => - filter.includes(GOOGLE_SHEETS_PRIMARY_KEY) - ) - for (let idFilterKey of idFilterKeys) { - const id = query.filters.equal[idFilterKey] - delete query.filters.equal[idFilterKey] - query.filters.equal[`_${GOOGLE_SHEETS_PRIMARY_KEY}`] = id - } - } if (hasFilters && query.paginate) { rows = rows.slice(offset, offset + limit) } const headerValues = sheet.headerValues - let response = [] - for (let row of rows) { - response.push( - this.buildRowObject(headerValues, row.toObject(), row.rowNumber) - ) - } + let response = rows.map(row => + this.buildRowObject(headerValues, row.toObject(), row.rowNumber) + ) response = dataFilters.runQuery(response, query.filters || {}) if (query.sort) { diff --git a/packages/server/src/integrations/tests/googlesheets.spec.ts b/packages/server/src/integrations/tests/googlesheets.spec.ts index e9d5290b2c..d3044d17eb 100644 --- a/packages/server/src/integrations/tests/googlesheets.spec.ts +++ b/packages/server/src/integrations/tests/googlesheets.spec.ts @@ -1,50 +1,44 @@ import { setEnv as setCoreEnv } from "@budibase/backend-core" -import type { GoogleSpreadsheetWorksheet } from "google-spreadsheet" import nock from "nock" -jest.mock("google-auth-library") -const { OAuth2Client } = require("google-auth-library") - -const setCredentialsMock = jest.fn() -const getAccessTokenMock = jest.fn() - -OAuth2Client.mockImplementation(() => { - return { - setCredentials: setCredentialsMock, - getAccessToken: getAccessTokenMock, - } -}) - -jest.mock("google-spreadsheet") -const { GoogleSpreadsheet } = require("google-spreadsheet") - -const sheetsByTitle: { [title: string]: GoogleSpreadsheetWorksheet } = {} -const sheetsByIndex: GoogleSpreadsheetWorksheet[] = [] -const mockGoogleIntegration = { - useOAuth2Client: jest.fn(), - loadInfo: jest.fn(), - sheetsByTitle, - sheetsByIndex, -} - -GoogleSpreadsheet.mockImplementation(() => mockGoogleIntegration) - -import { structures } from "@budibase/backend-core/tests" import TestConfiguration from "../../tests/utilities/TestConfiguration" -import GoogleSheetsIntegration from "../googlesheets" -import { FieldType, Table, TableSchema, TableSourceType } from "@budibase/types" -import { generateDatasourceID } from "../../db/utils" +import { + Datasource, + FieldType, + SourceName, + Table, + TableSourceType, +} from "@budibase/types" +import { GoogleSheetsMock } from "./utils/googlesheets" describe("Google Sheets Integration", () => { - let integration: any, - config = new TestConfiguration() - let cleanupEnv: () => void + const config = new TestConfiguration() - beforeAll(() => { + let cleanupEnv: () => void + let datasource: Datasource + let mock: GoogleSheetsMock + + beforeAll(async () => { cleanupEnv = setCoreEnv({ GOOGLE_CLIENT_ID: "test", GOOGLE_CLIENT_SECRET: "test", }) + + await config.init() + + datasource = await config.api.datasource.create({ + name: "Test Datasource", + type: "datasource", + source: SourceName.GOOGLE_SHEETS, + config: { + spreadsheetId: "randomId", + auth: { + appId: "appId", + accessToken: "accessToken", + refreshToken: "refreshToken", + }, + }, + }) }) afterAll(async () => { @@ -53,125 +47,130 @@ describe("Google Sheets Integration", () => { }) beforeEach(async () => { - integration = new GoogleSheetsIntegration.integration({ - spreadsheetId: "randomId", - auth: { - appId: "appId", - accessToken: "accessToken", - refreshToken: "refreshToken", - }, - }) - await config.init() - - jest.clearAllMocks() - nock.cleanAll() - nock("https://www.googleapis.com/").post("/oauth2/v4/token").reply(200, { - grant_type: "client_credentials", - client_id: "your-client-id", - client_secret: "your-client-secret", - }) + mock = GoogleSheetsMock.forDatasource(datasource) }) - function createBasicTable(name: string, columns: string[]): Table { - return { - type: "table", - name, - sourceId: generateDatasourceID(), - sourceType: TableSourceType.EXTERNAL, - schema: { - ...columns.reduce((p, c) => { - p[c] = { - name: c, + describe("create", () => { + it("creates a new table", async () => { + await config.api.table.save({ + name: "Test Table", + type: "table", + sourceId: datasource._id!, + sourceType: TableSourceType.EXTERNAL, + schema: { + name: { + name: "name", type: FieldType.STRING, constraints: { type: "string", }, - } - return p - }, {} as TableSchema), - }, - } - } - - function createSheet({ - headerValues, - }: { - headerValues: string[] - }): GoogleSpreadsheetWorksheet { - return { - // to ignore the unmapped fields - ...({} as any), - loadHeaderRow: jest.fn(), - headerValues, - setHeaderRow: jest.fn(), - } - } - - describe("update table", () => { - it("adding a new field will be adding a new header row", async () => { - await config.doInContext(structures.uuid(), async () => { - const tableColumns = ["name", "description", "new field"] - const table = createBasicTable(structures.uuid(), tableColumns) - - const sheet = createSheet({ headerValues: ["name", "description"] }) - sheetsByTitle[table.name] = sheet - await integration.updateTable(table) - - expect(sheet.loadHeaderRow).toHaveBeenCalledTimes(1) - expect(sheet.setHeaderRow).toHaveBeenCalledTimes(1) - expect(sheet.setHeaderRow).toHaveBeenCalledWith(tableColumns) + }, + description: { + name: "description", + type: FieldType.STRING, + constraints: { + type: "string", + }, + }, + }, }) - }) - it("removing an existing field will remove the header from the google sheet", async () => { - const sheet = await config.doInContext(structures.uuid(), async () => { - const tableColumns = ["name"] - const table = createBasicTable(structures.uuid(), tableColumns) - - const sheet = createSheet({ - headerValues: ["name", "description", "location"], - }) - sheetsByTitle[table.name] = sheet - await integration.updateTable(table) - return sheet - }) - expect(sheet.loadHeaderRow).toHaveBeenCalledTimes(1) - expect(sheet.setHeaderRow).toHaveBeenCalledTimes(1) - expect(sheet.setHeaderRow).toHaveBeenCalledWith([ - "name", - "description", - "location", - ]) + expect(mock.cell("A1")).toEqual("name") + expect(mock.cell("B1")).toEqual("description") + expect(mock.cell("A2")).toEqual(null) + expect(mock.cell("B2")).toEqual(null) }) }) - describe("getTableNames", () => { - it("can fetch table names", async () => { - await config.doInContext(structures.uuid(), async () => { - const sheetNames: string[] = [] - for (let i = 0; i < 5; i++) { - const sheet = createSheet({ headerValues: [] }) - sheetsByIndex.push(sheet) - sheetNames.push(sheet.title) - } - - const res = await integration.getTableNames() - - expect(mockGoogleIntegration.loadInfo).toHaveBeenCalledTimes(1) - expect(res).toEqual(sheetNames) + describe("update", () => { + let table: Table + beforeEach(async () => { + table = await config.api.table.save({ + name: "Test Table", + type: "table", + sourceId: datasource._id!, + sourceType: TableSourceType.EXTERNAL, + schema: { + name: { + name: "name", + type: FieldType.STRING, + }, + description: { + name: "description", + type: FieldType.STRING, + }, + }, }) }) - }) - describe("testConnection", () => { - it("can test successful connections", async () => { - await config.doInContext(structures.uuid(), async () => { - const res = await integration.testConnection() - - expect(mockGoogleIntegration.loadInfo).toHaveBeenCalledTimes(1) - expect(res).toEqual({ connected: true }) + it("should be able to add a new row", async () => { + const row = await config.api.row.save(table._id!, { + name: "Test Contact", + description: "original description", }) + + expect(row.name).toEqual("Test Contact") + expect(row.description).toEqual("original description") + + expect(mock.cell("A2")).toEqual("Test Contact") + expect(mock.cell("B2")).toEqual("original description") + + const row2 = await config.api.row.save(table._id!, { + name: "Test Contact 2", + description: "original description 2", + }) + + expect(row2.name).toEqual("Test Contact 2") + expect(row2.description).toEqual("original description 2") + + // Notable that adding a new row adds it at the top, not the bottom. Not + // entirely sure if this is the intended behaviour or an incorrect + // implementation of the GoogleSheetsMock. + expect(mock.cell("A2")).toEqual("Test Contact 2") + expect(mock.cell("B2")).toEqual("original description 2") + + expect(mock.cell("A3")).toEqual("Test Contact") + expect(mock.cell("B3")).toEqual("original description") + }) + + it("should be able to add multiple rows", async () => { + await config.api.row.bulkImport(table._id!, { + rows: [ + { + name: "Test Contact 1", + description: "original description 1", + }, + { + name: "Test Contact 2", + description: "original description 2", + }, + ], + }) + + expect(mock.cell("A2")).toEqual("Test Contact 1") + expect(mock.cell("B2")).toEqual("original description 1") + expect(mock.cell("A3")).toEqual("Test Contact 2") + expect(mock.cell("B3")).toEqual("original description 2") + }) + + it("should be able to update a row", async () => { + const row = await config.api.row.save(table._id!, { + name: "Test Contact", + description: "original description", + }) + + expect(mock.cell("A2")).toEqual("Test Contact") + expect(mock.cell("B2")).toEqual("original description") + + await config.api.row.save(table._id!, { + ...row, + name: "Test Contact Updated", + description: "original description updated", + }) + + expect(mock.cell("A2")).toEqual("Test Contact Updated") + expect(mock.cell("B2")).toEqual("original description updated") }) }) }) diff --git a/packages/server/src/integrations/tests/utils/googlesheets.ts b/packages/server/src/integrations/tests/utils/googlesheets.ts new file mode 100644 index 0000000000..0f92fea6cd --- /dev/null +++ b/packages/server/src/integrations/tests/utils/googlesheets.ts @@ -0,0 +1,815 @@ +// In this file is a mock implementation of the Google Sheets API. It is used +// to test the Google Sheets integration, and it keeps track of a single +// spreadsheet with many sheets. It aims to be a faithful recreation of the +// Google Sheets API, but it is not a perfect recreation. Some fields are +// missing if they aren't relevant to our use of the API. It's possible that +// this will cause problems for future feature development, but the original +// development of these tests involved hitting Google's APIs directly and +// examining the responses. If we couldn't find a good example of something in +// use, it wasn't included. +import { Datasource } from "@budibase/types" +import nock from "nock" +import { GoogleSheetsConfig } from "../../googlesheets" +import type { + SpreadsheetProperties, + ExtendedValue, + WorksheetDimension, + WorksheetDimensionProperties, + WorksheetProperties, + CellData, + CellBorder, + CellFormat, + CellPadding, + Color, +} from "google-spreadsheet/src/lib/types/sheets-types" + +const BLACK: Color = { red: 0, green: 0, blue: 0 } +const WHITE: Color = { red: 1, green: 1, blue: 1 } +const NO_PADDING: CellPadding = { top: 0, right: 0, bottom: 0, left: 0 } +const DEFAULT_BORDER: CellBorder = { + style: "SOLID", + width: 1, + color: BLACK, + colorStyle: { rgbColor: BLACK }, +} +const DEFAULT_CELL_FORMAT: CellFormat = { + hyperlinkDisplayType: "PLAIN_TEXT", + horizontalAlignment: "LEFT", + verticalAlignment: "BOTTOM", + wrapStrategy: "OVERFLOW_CELL", + textDirection: "LEFT_TO_RIGHT", + textRotation: { angle: 0, vertical: false }, + padding: NO_PADDING, + backgroundColorStyle: { rgbColor: BLACK }, + borders: { + top: DEFAULT_BORDER, + bottom: DEFAULT_BORDER, + left: DEFAULT_BORDER, + right: DEFAULT_BORDER, + }, + numberFormat: { + type: "NUMBER", + pattern: "General", + }, + backgroundColor: WHITE, + textFormat: { + foregroundColor: BLACK, + fontFamily: "Arial", + fontSize: 10, + bold: false, + italic: false, + strikethrough: false, + underline: false, + }, +} + +// https://protobuf.dev/reference/protobuf/google.protobuf/#value +type Value = string | number | boolean | null + +interface Range { + row: number + column: number +} + +// https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets.values#ValueRange +interface ValueRange { + range: string + majorDimension: WorksheetDimension + values: Value[][] +} + +// https://developers.google.com/sheets/api/reference/rest/v4/UpdateValuesResponse +interface UpdateValuesResponse { + spreadsheetId: string + updatedRange: string + updatedRows: number + updatedColumns: number + updatedCells: number + updatedData: ValueRange +} + +// https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets/response#AddSheetResponse +interface AddSheetResponse { + properties: WorksheetProperties +} + +// https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets/response +interface BatchUpdateResponse { + spreadsheetId: string + replies: { + addSheet?: AddSheetResponse + }[] + updatedSpreadsheet: Spreadsheet +} + +// https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets/request#AddSheetRequest +interface AddSheetRequest { + properties: WorksheetProperties +} + +interface Request { + addSheet?: AddSheetRequest +} + +// https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets/request +interface BatchUpdateRequest { + requests: Request[] + includeSpreadsheetInResponse: boolean + responseRanges: string[] + responseIncludeGridData: boolean +} + +// https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets/sheets#RowData +interface RowData { + values: CellData[] +} + +// https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets/sheets#GridData +interface GridData { + startRow: number + startColumn: number + rowData: RowData[] + rowMetadata: WorksheetDimensionProperties[] + columnMetadata: WorksheetDimensionProperties[] +} + +// https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets/sheets#Sheet +interface Sheet { + properties: WorksheetProperties + data: GridData[] +} + +// https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets#Spreadsheet +interface Spreadsheet { + properties: SpreadsheetProperties + spreadsheetId: string + sheets: Sheet[] +} + +// https://developers.google.com/sheets/api/reference/rest/v4/ValueInputOption +type ValueInputOption = + | "USER_ENTERED" + | "RAW" + | "INPUT_VALUE_OPTION_UNSPECIFIED" + +// https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets.values/append#InsertDataOption +type InsertDataOption = "OVERWRITE" | "INSERT_ROWS" + +// https://developers.google.com/sheets/api/reference/rest/v4/ValueRenderOption +type ValueRenderOption = "FORMATTED_VALUE" | "UNFORMATTED_VALUE" | "FORMULA" + +// https://developers.google.com/sheets/api/reference/rest/v4/DateTimeRenderOption +type DateTimeRenderOption = "SERIAL_NUMBER" | "FORMATTED_STRING" + +// https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets.values/append#query-parameters +interface AppendParams { + valueInputOption?: ValueInputOption + insertDataOption?: InsertDataOption + includeValuesInResponse?: boolean + responseValueRenderOption?: ValueRenderOption + responseDateTimeRenderOption?: DateTimeRenderOption +} + +// https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets.values/batchGet#query-parameters +interface BatchGetParams { + ranges: string[] + majorDimension?: WorksheetDimension + valueRenderOption?: ValueRenderOption + dateTimeRenderOption?: DateTimeRenderOption +} + +// https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets.values/batchGet#response-body +interface BatchGetResponse { + spreadsheetId: string + valueRanges: ValueRange[] +} + +interface AppendRequest { + range: string + params: AppendParams + body: ValueRange +} + +// https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets.values/append#response-body +interface AppendResponse { + spreadsheetId: string + tableRange: string + updates: UpdateValuesResponse +} + +export class GoogleSheetsMock { + private config: GoogleSheetsConfig + private spreadsheet: Spreadsheet + + static forDatasource(datasource: Datasource): GoogleSheetsMock { + return new GoogleSheetsMock(datasource.config as GoogleSheetsConfig) + } + + private constructor(config: GoogleSheetsConfig) { + this.config = config + this.spreadsheet = { + properties: { + title: "Test Spreadsheet", + locale: "en_US", + autoRecalc: "ON_CHANGE", + timeZone: "America/New_York", + defaultFormat: {}, + iterativeCalculationSettings: {}, + spreadsheetTheme: {}, + }, + spreadsheetId: config.spreadsheetId, + sheets: [], + } + + this.mockAuth() + this.mockAPI() + } + + private route( + method: "get" | "put" | "post", + path: string | RegExp, + handler: (uri: string, request: nock.Body) => nock.Body + ): nock.Scope { + const headers = { reqheaders: { authorization: "Bearer test" } } + const scope = nock("https://sheets.googleapis.com/", headers) + return scope[method](path).reply(200, handler).persist() + } + + private get( + path: string | RegExp, + handler: (uri: string, request: nock.Body) => nock.Body + ): nock.Scope { + return this.route("get", path, handler) + } + + private put( + path: string | RegExp, + handler: (uri: string, request: nock.Body) => nock.Body + ): nock.Scope { + return this.route("put", path, handler) + } + + private post( + path: string | RegExp, + handler: (uri: string, request: nock.Body) => nock.Body + ): nock.Scope { + return this.route("post", path, handler) + } + + private mockAuth() { + nock("https://www.googleapis.com/") + .post("/oauth2/v4/token") + .reply(200, { + grant_type: "client_credentials", + client_id: "your-client-id", + client_secret: "your-client-secret", + }) + .persist() + + nock("https://oauth2.googleapis.com/") + .post("/token", { + client_id: "test", + client_secret: "test", + grant_type: "refresh_token", + refresh_token: "refreshToken", + }) + .reply(200, { + access_token: "test", + expires_in: 3600, + token_type: "Bearer", + scopes: "https://www.googleapis.com/auth/spreadsheets", + }) + .persist() + } + + private mockAPI() { + const spreadsheetId = this.config.spreadsheetId + + this.get(`/v4/spreadsheets/${spreadsheetId}/`, () => + this.handleGetSpreadsheet() + ) + + // https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets.values/batchUpdate + this.post( + `/v4/spreadsheets/${spreadsheetId}/:batchUpdate`, + (_uri, request) => this.handleBatchUpdate(request as BatchUpdateRequest) + ) + + // https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets.values/update + this.put( + new RegExp(`/v4/spreadsheets/${spreadsheetId}/values/.*`), + (_uri, request) => this.handleValueUpdate(request as ValueRange) + ) + + // https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets.values/batchGet + this.get( + new RegExp(`/v4/spreadsheets/${spreadsheetId}/values:batchGet.*`), + uri => { + const url = new URL(uri, "https://sheets.googleapis.com/") + const params: BatchGetParams = { + ranges: url.searchParams.getAll("ranges"), + majorDimension: + (url.searchParams.get("majorDimension") as WorksheetDimension) || + "ROWS", + valueRenderOption: + (url.searchParams.get("valueRenderOption") as ValueRenderOption) || + undefined, + dateTimeRenderOption: + (url.searchParams.get( + "dateTimeRenderOption" + ) as DateTimeRenderOption) || undefined, + } + return this.handleBatchGet(params as unknown as BatchGetParams) + } + ) + + // https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets.values/get + this.get(new RegExp(`/v4/spreadsheets/${spreadsheetId}/values/.*`), uri => { + const range = uri.split("/").pop() + if (!range) { + throw new Error("No range provided") + } + return this.getValueRange(decodeURIComponent(range)) + }) + + // https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets.values/append + this.post( + new RegExp(`/v4/spreadsheets/${spreadsheetId}/values/.*:append`), + (_uri, request) => { + const url = new URL(_uri, "https://sheets.googleapis.com/") + const params: Record = Object.fromEntries( + url.searchParams.entries() + ) + + if (params.includeValuesInResponse === "true") { + params.includeValuesInResponse = true + } else { + params.includeValuesInResponse = false + } + + let range = url.pathname.split("/").pop() + if (!range) { + throw new Error("No range provided") + } + + if (range.endsWith(":append")) { + range = range.slice(0, -7) + } + + range = decodeURIComponent(range) + + return this.handleValueAppend({ + range, + params, + body: request as ValueRange, + }) + } + ) + } + + private handleValueAppend(request: AppendRequest): AppendResponse { + const { range, params, body } = request + const { sheet, bottomRight } = this.parseA1Notation(range) + + const newRows = body.values.map(v => this.valuesToRowData(v)) + const toDelete = + params.insertDataOption === "INSERT_ROWS" ? newRows.length : 0 + sheet.data[0].rowData.splice(bottomRight.row + 1, toDelete, ...newRows) + sheet.data[0].rowMetadata.splice(bottomRight.row + 1, toDelete, { + hiddenByUser: false, + hiddenByFilter: false, + pixelSize: 100, + developerMetadata: [], + }) + + // It's important to give back a correct updated range because the API + // library we use makes use of it to assign the correct row IDs to rows. + const updatedRange = this.createA1FromRanges( + sheet, + { + row: bottomRight.row + 1, + column: 0, + }, + { + row: bottomRight.row + newRows.length, + column: 0, + } + ) + + return { + spreadsheetId: this.spreadsheet.spreadsheetId, + tableRange: range, + updates: { + spreadsheetId: this.spreadsheet.spreadsheetId, + updatedRange, + updatedRows: body.values.length, + updatedColumns: body.values[0].length, + updatedCells: body.values.length * body.values[0].length, + updatedData: body, + }, + } + } + + private handleBatchGet(params: BatchGetParams): BatchGetResponse { + const { ranges, majorDimension } = params + + if (majorDimension && majorDimension !== "ROWS") { + throw new Error("Only row-major updates are supported") + } + + return { + spreadsheetId: this.spreadsheet.spreadsheetId, + valueRanges: ranges.map(range => this.getValueRange(range)), + } + } + + private handleBatchUpdate( + batchUpdateRequest: BatchUpdateRequest + ): BatchUpdateResponse { + const response: BatchUpdateResponse = { + spreadsheetId: this.spreadsheet.spreadsheetId, + replies: [], + updatedSpreadsheet: this.spreadsheet, + } + + for (const request of batchUpdateRequest.requests) { + if (request.addSheet) { + response.replies.push({ + addSheet: this.handleAddSheet(request.addSheet), + }) + } + } + + return response + } + + private handleAddSheet(request: AddSheetRequest): AddSheetResponse { + const properties: Omit = { + index: this.spreadsheet.sheets.length, + hidden: false, + rightToLeft: false, + tabColor: BLACK, + tabColorStyle: { rgbColor: BLACK }, + sheetType: "GRID", + title: request.properties.title, + sheetId: this.spreadsheet.sheets.length, + gridProperties: { + rowCount: 100, + columnCount: 26, + frozenRowCount: 0, + frozenColumnCount: 0, + hideGridlines: false, + rowGroupControlAfter: false, + columnGroupControlAfter: false, + }, + } + + this.spreadsheet.sheets.push({ + properties: properties as WorksheetProperties, + data: [this.createEmptyGrid(100, 26)], + }) + + // dataSourceSheetProperties is only returned by the API if the sheet type is + // DATA_SOURCE, which we aren't using, so sadly we need to cast here. + return { properties: properties as WorksheetProperties } + } + + private handleGetSpreadsheet(): Spreadsheet { + return this.spreadsheet + } + + private handleValueUpdate(valueRange: ValueRange): UpdateValuesResponse { + this.iterateCells(valueRange, (cell, value) => { + cell.userEnteredValue = this.createValue(value) + }) + + const response: UpdateValuesResponse = { + spreadsheetId: this.spreadsheet.spreadsheetId, + updatedRange: valueRange.range, + updatedRows: valueRange.values.length, + updatedColumns: valueRange.values[0].length, + updatedCells: valueRange.values.length * valueRange.values[0].length, + updatedData: valueRange, + } + return response + } + + private iterateCells( + valueRange: ValueRange, + cb: (cell: CellData, value: Value) => void + ) { + if (valueRange.majorDimension !== "ROWS") { + throw new Error("Only row-major updates are supported") + } + + const { sheet, topLeft, bottomRight } = this.parseA1Notation( + valueRange.range + ) + for (let row = topLeft.row; row <= bottomRight.row; row++) { + for (let col = topLeft.column; col <= bottomRight.column; col++) { + const cell = this.getCellNumericIndexes(sheet, row, col) + if (!cell) { + throw new Error("Cell not found") + } + const value = valueRange.values[row - topLeft.row][col - topLeft.column] + cb(cell, value) + } + } + } + + private getValueRange(range: string): ValueRange { + const { sheet, topLeft, bottomRight } = this.parseA1Notation(range) + const valueRange: ValueRange = { + range, + majorDimension: "ROWS", + values: [], + } + + for (let row = topLeft.row; row <= bottomRight.row; row++) { + const values: Value[] = [] + for (let col = topLeft.column; col <= bottomRight.column; col++) { + const cell = this.getCellNumericIndexes(sheet, row, col) + if (!cell) { + throw new Error("Cell not found") + } + values.push(this.cellValue(cell)) + } + valueRange.values.push(values) + } + + return valueRange + } + + private valuesToRowData(values: Value[]): RowData { + return { + values: values.map(v => { + return this.createCellData(v) + }), + } + } + + private unwrapValue(from: ExtendedValue): Value { + if ("stringValue" in from) { + return from.stringValue + } else if ("numberValue" in from) { + return from.numberValue + } else if ("boolValue" in from) { + return from.boolValue + } else if ("formulaValue" in from) { + return from.formulaValue + } else { + return null + } + } + + private cellValue(from: CellData): Value { + return this.unwrapValue(from.userEnteredValue) + } + + private createValue(from: Value): ExtendedValue { + if (from == null) { + return {} as ExtendedValue + } else if (typeof from === "string") { + return { + stringValue: from, + } + } else if (typeof from === "number") { + return { + numberValue: from, + } + } else if (typeof from === "boolean") { + return { + boolValue: from, + } + } else { + throw new Error("Unsupported value type") + } + } + + /** + * Because the structure of a CellData is very nested and contains a lot of + * extraneous formatting information, this function abstracts it away and just + * lets you create a cell containing a given value. + * + * When you want to read the value back out, use {@link cellValue}. + * + * @param value value to store in the returned cell + * @returns a CellData containing the given value. Read it back out with + * {@link cellValue} + */ + private createCellData(value: Value): CellData { + return { + userEnteredValue: this.createValue(value), + effectiveValue: this.createValue(value), + formattedValue: value?.toString() || "", + userEnteredFormat: DEFAULT_CELL_FORMAT, + effectiveFormat: DEFAULT_CELL_FORMAT, + } + } + + private createEmptyGrid(numRows: number, numCols: number): GridData { + const rowData: RowData[] = [] + for (let row = 0; row < numRows; row++) { + const cells: CellData[] = [] + for (let col = 0; col < numCols; col++) { + cells.push(this.createCellData(null)) + } + rowData.push({ values: cells }) + } + const rowMetadata: WorksheetDimensionProperties[] = [] + for (let row = 0; row < numRows; row++) { + rowMetadata.push({ + hiddenByFilter: false, + hiddenByUser: false, + pixelSize: 100, + developerMetadata: [], + }) + } + const columnMetadata: WorksheetDimensionProperties[] = [] + for (let col = 0; col < numCols; col++) { + columnMetadata.push({ + hiddenByFilter: false, + hiddenByUser: false, + pixelSize: 100, + developerMetadata: [], + }) + } + + return { + startRow: 0, + startColumn: 0, + rowData, + rowMetadata, + columnMetadata, + } + } + + private cellData(cell: string): CellData | undefined { + const { + sheet, + topLeft: { row, column }, + } = this.parseA1Notation(cell) + return this.getCellNumericIndexes(sheet, row, column) + } + + cell(cell: string): Value | undefined { + const cellData = this.cellData(cell) + if (!cellData) { + return undefined + } + return this.cellValue(cellData) + } + + private getCellNumericIndexes( + sheet: Sheet, + row: number, + column: number + ): CellData | undefined { + const data = sheet.data[0] + const rowData = data.rowData[row] + if (!rowData) { + return undefined + } + const cell = rowData.values[column] + if (!cell) { + return undefined + } + return cell + } + + // https://developers.google.com/sheets/api/guides/concepts#cell + // + // Examples from + // https://code.luasoftware.com/tutorials/google-sheets-api/google-sheets-api-range-parameter-a1-notation + // + // "Sheet1!A1" -> First cell on Row 1 Col 1 + // "Sheet1!A1:C1" -> Col 1-3 (A, B, C) on Row 1 = A1, B1, C1 + // "A1" -> First visible sheet (if sheet name is ommitted) + // "'My Sheet'!A1" -> If sheet name which contain space or start with a bracket. + // "Sheet1" -> All cells in Sheet1. + // "Sheet1!A:A" -> All cells on Col 1. + // "Sheet1!A:B" -> All cells on Col 1 and 2. + // "Sheet1!1:1" -> All cells on Row 1. + // "Sheet1!1:2" -> All cells on Row 1 and 2. + // + // How that translates to our code below, omitting the `sheet` property: + // + // "Sheet1!A1" -> { topLeft: { row: 0, column: 0 }, bottomRight: { row: 0, column: 0 } } + // "Sheet1!A1:C1" -> { topLeft: { row: 0, column: 0 }, bottomRight: { row: 0, column: 2 } } + // "A1" -> { topLeft: { row: 0, column: 0 }, bottomRight: { row: 0, column: 0 } } + // "Sheet1" -> { topLeft: { row: 0, column: 0 }, bottomRight: { row: 100, column: 25 } } + // -> This is because we default to having a 100x26 grid. + // "Sheet1!A:A" -> { topLeft: { row: 0, column: 0 }, bottomRight: { row: 99, column: 0 } } + // "Sheet1!A:B" -> { topLeft: { row: 0, column: 0 }, bottomRight: { row: 99, column: 1 } } + // "Sheet1!1:1" -> { topLeft: { row: 0, column: 0 }, bottomRight: { row: 0, column: 25 } } + // "Sheet1!1:2" -> { topLeft: { row: 0, column: 0 }, bottomRight: { row: 1, column: 25 } } + private parseA1Notation(range: string): { + sheet: Sheet + topLeft: Range + bottomRight: Range + } { + let sheet: Sheet + let rest: string + if (!range.includes("!")) { + sheet = this.spreadsheet.sheets[0] + rest = range + } else { + let sheetName = range.split("!")[0] + if (sheetName.startsWith("'") && sheetName.endsWith("'")) { + sheetName = sheetName.slice(1, -1) + } + const foundSheet = this.getSheetByName(sheetName) + if (!foundSheet) { + throw new Error(`Sheet ${sheetName} not found`) + } + sheet = foundSheet + rest = range.split("!")[1] + } + + const [topLeft, bottomRight] = rest.split(":") + + const parsedTopLeft = topLeft ? this.parseCell(topLeft) : undefined + let parsedBottomRight = bottomRight + ? this.parseCell(bottomRight) + : undefined + + if (!parsedTopLeft && !parsedBottomRight) { + throw new Error("No range provided") + } + + if (!parsedTopLeft) { + throw new Error("No top left cell provided") + } + + if (!parsedBottomRight) { + parsedBottomRight = parsedTopLeft + } + + if (parsedTopLeft && parsedTopLeft.row === undefined) { + parsedTopLeft.row = 0 + } + if (parsedTopLeft && parsedTopLeft.column === undefined) { + parsedTopLeft.column = 0 + } + if (parsedBottomRight && parsedBottomRight.row === undefined) { + parsedBottomRight.row = sheet.properties.gridProperties.rowCount - 1 + } + if (parsedBottomRight && parsedBottomRight.column === undefined) { + parsedBottomRight.column = sheet.properties.gridProperties.columnCount - 1 + } + + return { + sheet, + topLeft: parsedTopLeft as Range, + bottomRight: parsedBottomRight as Range, + } + } + + private createA1FromRanges(sheet: Sheet, topLeft: Range, bottomRight: Range) { + let title = sheet.properties.title + if (title.includes(" ")) { + title = `'${title}'` + } + const topLeftLetter = this.numberToLetter(topLeft.column) + const bottomRightLetter = this.numberToLetter(bottomRight.column) + const topLeftRow = topLeft.row + 1 + const bottomRightRow = bottomRight.row + 1 + return `${title}!${topLeftLetter}${topLeftRow}:${bottomRightLetter}${bottomRightRow}` + } + + /** + * Parses a cell reference into a row and column. + * @param cell a string of the form A1, B2, etc. + * @returns + */ + private parseCell(cell: string): Partial { + const firstChar = cell.slice(0, 1) + if (this.isInteger(firstChar)) { + return { row: parseInt(cell) - 1 } + } + const column = this.letterToNumber(firstChar) + if (cell.length === 1) { + return { column } + } + const number = cell.slice(1) + return { row: parseInt(number) - 1, column } + } + + private isInteger(value: string): boolean { + return !isNaN(parseInt(value)) + } + + private letterToNumber(letter: string): number { + return letter.charCodeAt(0) - 65 + } + + private numberToLetter(number: number): string { + return String.fromCharCode(number + 65) + } + + private getSheetByName(name: string): Sheet | undefined { + return this.spreadsheet.sheets.find( + sheet => sheet.properties.title === name + ) + } +} diff --git a/packages/shared-core/src/filters.ts b/packages/shared-core/src/filters.ts index 31934371f6..4c263c48bd 100644 --- a/packages/shared-core/src/filters.ts +++ b/packages/shared-core/src/filters.ts @@ -448,10 +448,10 @@ export function fixupFilterArrays(filters: SearchFilters) { return filters } -export const search = ( - docs: Record[], +export function search( + docs: Record[], query: RowSearchParams -): SearchResponse> => { +): SearchResponse> { let result = runQuery(docs, query.query) if (query.sort) { result = sort(result, query.sort, query.sortOrder || SortOrder.ASCENDING) @@ -472,7 +472,10 @@ export const search = ( * @param docs the data * @param query the JSON query */ -export const runQuery = (docs: Record[], query: SearchFilters) => { +export function runQuery>( + docs: T[], + query: SearchFilters +): T[] { if (!docs || !Array.isArray(docs)) { return [] } @@ -495,7 +498,7 @@ export const runQuery = (docs: Record[], query: SearchFilters) => { type: SearchFilterOperator, test: (docValue: any, testValue: any) => boolean ) => - (doc: Record) => { + (doc: T) => { for (const [key, testValue] of Object.entries(query[type] || {})) { const valueToCheck = isLogicalSearchOperator(type) ? doc @@ -742,11 +745,8 @@ export const runQuery = (docs: Record[], query: SearchFilters) => { } ) - const docMatch = (doc: Record) => { - const filterFunctions: Record< - SearchFilterOperator, - (doc: Record) => boolean - > = { + const docMatch = (doc: T) => { + const filterFunctions: Record boolean> = { string: stringMatch, fuzzy: fuzzyMatch, range: rangeMatch, @@ -790,12 +790,12 @@ export const runQuery = (docs: Record[], query: SearchFilters) => { * @param sortOrder the sort order ("ascending" or "descending") * @param sortType the type of sort ("string" or "number") */ -export const sort = ( - docs: any[], - sort: string, +export function sort>( + docs: T[], + sort: keyof T, sortOrder: SortOrder, sortType = SortType.STRING -) => { +): T[] { if (!sort || !sortOrder || !sortType) { return docs } @@ -810,19 +810,17 @@ export const sort = ( return parseFloat(x) } - return docs - .slice() - .sort((a: { [x: string]: any }, b: { [x: string]: any }) => { - const colA = parse(a[sort]) - const colB = parse(b[sort]) + return docs.slice().sort((a, b) => { + const colA = parse(a[sort]) + const colB = parse(b[sort]) - const result = colB == null || colA > colB ? 1 : -1 - if (sortOrder.toLowerCase() === "descending") { - return result * -1 - } + const result = colB == null || colA > colB ? 1 : -1 + if (sortOrder.toLowerCase() === "descending") { + return result * -1 + } - return result - }) + return result + }) } /** @@ -831,7 +829,7 @@ export const sort = ( * @param docs the data * @param limit the number of docs to limit to */ -export const limit = (docs: any[], limit: string) => { +export function limit(docs: T[], limit: string): T[] { const numLimit = parseFloat(limit) if (isNaN(numLimit)) { return docs