diff --git a/lerna.json b/lerna.json index 070c0f5315..c710d888c7 100644 --- a/lerna.json +++ b/lerna.json @@ -1,6 +1,6 @@ { "$schema": "node_modules/lerna/schemas/lerna-schema.json", - "version": "2.32.3", + "version": "2.32.5", "npmClient": "yarn", "packages": [ "packages/*", diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index 2b5243f856..55f71d76b0 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -40,7 +40,6 @@ import { dataFilters, helpers } from "@budibase/shared-core" import { cloneDeep } from "lodash" type QueryFunction = (query: SqlQuery | SqlQuery[], operation: Operation) => any -const MAX_SQS_RELATIONSHIP_FIELDS = 63 function getBaseLimit() { const envLimit = environment.SQL_MAX_ROWS @@ -56,6 +55,20 @@ function getRelationshipLimit() { return envLimit || 500 } +function prioritisedArraySort(toSort: string[], priorities: string[]) { + return toSort.sort((a, b) => { + const aPriority = priorities.find(field => field && a.endsWith(field)) + const bPriority = priorities.find(field => field && b.endsWith(field)) + if (aPriority && !bPriority) { + return -1 + } + if (!aPriority && bPriority) { + return 1 + } + return a.localeCompare(b) + }) +} + function getTableName(table?: Table): string | undefined { // SQS uses the table ID rather than the table name if ( @@ -877,6 +890,22 @@ class InternalBuilder { return `'${unaliased}'${separator}${tableField}` } + maxFunctionParameters() { + // functions like say json_build_object() in SQL have a limit as to how many can be performed + // before a limit is met, this limit exists in Postgres/SQLite. This can be very important, such as + // for JSON column building as part of relationships. We also have a default limit to avoid very complex + // functions being built - it is likely this is not necessary or the best way to do it. + switch (this.client) { + case SqlClient.SQL_LITE: + return 127 + case SqlClient.POSTGRES: + return 100 + // other DBs don't have a limit, but set some sort of limit + default: + return 200 + } + } + addJsonRelationships( query: Knex.QueryBuilder, fromTable: string, @@ -884,7 +913,7 @@ class InternalBuilder { ): Knex.QueryBuilder { const sqlClient = this.client const knex = this.knex - const { resource, tableAliases: aliases, endpoint } = this.query + const { resource, tableAliases: aliases, endpoint, meta } = this.query const fields = resource?.fields || [] for (let relationship of relationships) { const { @@ -899,21 +928,27 @@ class InternalBuilder { if (!toTable || !fromTable) { continue } + const relatedTable = meta.tables?.[toTable] const toAlias = aliases?.[toTable] || toTable, fromAlias = aliases?.[fromTable] || fromTable let toTableWithSchema = this.tableNameWithSchema(toTable, { alias: toAlias, schema: endpoint.schema, }) - let relationshipFields = fields.filter( - field => field.split(".")[0] === toAlias + const requiredFields = [ + ...(relatedTable?.primary || []), + relatedTable?.primaryDisplay, + ].filter(field => field) as string[] + // sort the required fields to first in the list, so they don't get sliced out + let relationshipFields = prioritisedArraySort( + fields.filter(field => field.split(".")[0] === toAlias), + requiredFields + ) + + relationshipFields = relationshipFields.slice( + 0, + Math.floor(this.maxFunctionParameters() / 2) ) - if (this.client === SqlClient.SQL_LITE) { - relationshipFields = relationshipFields.slice( - 0, - MAX_SQS_RELATIONSHIP_FIELDS - ) - } const fieldList: string = relationshipFields .map(field => this.buildJsonField(field)) .join(",") diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts index ac2d1e8c39..9c9bd0b284 100644 --- a/packages/server/src/api/controllers/row/ExternalRequest.ts +++ b/packages/server/src/api/controllers/row/ExternalRequest.ts @@ -12,6 +12,7 @@ import { OneToManyRelationshipFieldMetadata, Operation, PaginationJson, + QueryJson, RelationshipFieldMetadata, Row, SearchFilters, @@ -161,7 +162,6 @@ export class ExternalRequest { private readonly tableId: string private datasource?: Datasource private tables: { [key: string]: Table } = {} - private tableList: Table[] constructor(operation: T, tableId: string, datasource?: Datasource) { this.operation = operation @@ -170,7 +170,6 @@ export class ExternalRequest { if (datasource && datasource.entities) { this.tables = datasource.entities } - this.tableList = Object.values(this.tables) } private prepareFilters( @@ -301,7 +300,6 @@ export class ExternalRequest { throw "No tables found, fetch tables before query." } this.tables = this.datasource.entities - this.tableList = Object.values(this.tables) } return { tables: this.tables, datasource: this.datasource } } @@ -463,7 +461,7 @@ export class ExternalRequest { breakExternalTableId(relatedTableId) // @ts-ignore const linkPrimaryKey = this.tables[relatedTableName].primary[0] - if (!lookupField || !row[lookupField]) { + if (!lookupField || !row?.[lookupField] == null) { continue } const endpoint = getEndpoint(relatedTableId, Operation.READ) @@ -631,7 +629,8 @@ export class ExternalRequest { const { datasource: ds } = await this.retrieveMetadata(datasourceId) datasource = ds } - const table = this.tables[tableName] + const tables = this.tables + const table = tables[tableName] let isSql = isSQL(datasource) if (!table) { throw new Error( @@ -686,7 +685,7 @@ export class ExternalRequest { ) { throw "Deletion must be filtered" } - let json = { + let json: QueryJson = { endpoint: { datasourceId: datasourceId!, entityId: tableName, @@ -715,7 +714,7 @@ export class ExternalRequest { }, meta: { table, - id: config.id, + tables: tables, }, } diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index c9e7f4c3c5..0b0802bab2 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -3080,4 +3080,46 @@ describe.each([ }).toHaveLength(4) }) }) + + isSql && + describe("max related columns", () => { + let relatedRows: Row[] + + beforeAll(async () => { + const relatedSchema: TableSchema = {} + const row: Row = {} + for (let i = 0; i < 100; i++) { + const name = `column${i}` + relatedSchema[name] = { name, type: FieldType.NUMBER } + row[name] = i + } + const relatedTable = await createTable(relatedSchema) + table = await createTable({ + name: { name: "name", type: FieldType.STRING }, + related1: { + type: FieldType.LINK, + name: "related1", + fieldName: "main1", + tableId: relatedTable._id!, + relationshipType: RelationshipType.MANY_TO_MANY, + }, + }) + relatedRows = await Promise.all([ + config.api.row.save(relatedTable._id!, row), + ]) + await config.api.row.save(table._id!, { + name: "foo", + related1: [relatedRows[0]._id], + }) + }) + + it("retrieve the row with relationships", async () => { + await expectQuery({}).toContainExactly([ + { + name: "foo", + related1: [{ _id: relatedRows[0]._id }], + }, + ]) + }) + }) }) diff --git a/packages/server/src/automations/triggers.ts b/packages/server/src/automations/triggers.ts index 18b033cdcf..110ccfa37a 100644 --- a/packages/server/src/automations/triggers.ts +++ b/packages/server/src/automations/triggers.ts @@ -18,6 +18,7 @@ import { SearchFilters, AutomationStoppedReason, AutomationStatus, + AutomationRowEvent, } from "@budibase/types" import { executeInThread } from "../threads/automation" import { dataFilters, sdk } from "@budibase/shared-core" @@ -28,6 +29,7 @@ const JOB_OPTS = { removeOnFail: true, } import * as automationUtils from "../automations/automationUtils" +import { doesTableExist } from "../sdk/app/tables/getters" async function getAllAutomations() { const db = context.getAppDB() @@ -38,25 +40,35 @@ async function getAllAutomations() { } async function queueRelevantRowAutomations( - event: { appId: string; row: Row; oldRow: Row }, - eventType: string + event: AutomationRowEvent, + eventType: AutomationEventType ) { + const tableId = event.row.tableId if (event.appId == null) { throw `No appId specified for ${eventType} - check event emitters.` } + // make sure table exists and is valid before proceeding + if (!tableId || !(await doesTableExist(tableId))) { + return + } + await context.doInAppContext(event.appId, async () => { let automations = await getAllAutomations() // filter down to the correct event type and enabled automations + // make sure it is the correct table ID as well automations = automations.filter(automation => { const trigger = automation.definition.trigger - return trigger && trigger.event === eventType && !automation.disabled + return ( + trigger && + trigger.event === eventType && + !automation.disabled && + trigger?.inputs?.tableId === event.row.tableId + ) }) for (const automation of automations) { - const automationDef = automation.definition - const automationTrigger = automationDef?.trigger // don't queue events which are for dev apps, only way to test automations is // running tests on them, in production the test flag will never // be checked due to lazy evaluation (first always false) @@ -72,11 +84,7 @@ async function queueRelevantRowAutomations( row: event.row, oldRow: event.oldRow, }) - if ( - automationTrigger?.inputs && - automationTrigger.inputs.tableId === event.row.tableId && - shouldTrigger - ) { + if (shouldTrigger) { try { await automationQueue.add({ automation, event }, JOB_OPTS) } catch (e) { @@ -87,6 +95,17 @@ async function queueRelevantRowAutomations( }) } +async function queueRowAutomations( + event: AutomationRowEvent, + type: AutomationEventType +) { + try { + await queueRelevantRowAutomations(event, type) + } catch (err: any) { + logging.logWarn("Unable to process row event", err) + } +} + emitter.on( AutomationEventType.ROW_SAVE, async function (event: UpdatedRowEventEmitter) { @@ -94,7 +113,7 @@ emitter.on( if (!event || !event.row || !event.row.tableId) { return } - await queueRelevantRowAutomations(event, AutomationEventType.ROW_SAVE) + await queueRowAutomations(event, AutomationEventType.ROW_SAVE) } ) @@ -103,7 +122,7 @@ emitter.on(AutomationEventType.ROW_UPDATE, async function (event) { if (!event || !event.row || !event.row.tableId) { return } - await queueRelevantRowAutomations(event, AutomationEventType.ROW_UPDATE) + await queueRowAutomations(event, AutomationEventType.ROW_UPDATE) }) emitter.on(AutomationEventType.ROW_DELETE, async function (event) { @@ -111,7 +130,7 @@ emitter.on(AutomationEventType.ROW_DELETE, async function (event) { if (!event || !event.row || !event.row.tableId) { return } - await queueRelevantRowAutomations(event, AutomationEventType.ROW_DELETE) + await queueRowAutomations(event, AutomationEventType.ROW_DELETE) }) function rowPassesFilters(row: Row, filters: SearchFilters) { diff --git a/packages/server/src/integrations/googlesheets.ts b/packages/server/src/integrations/googlesheets.ts index 0d766ac1ef..dd9bef84ab 100644 --- a/packages/server/src/integrations/googlesheets.ts +++ b/packages/server/src/integrations/googlesheets.ts @@ -551,11 +551,16 @@ export class GoogleSheetsIntegration implements DatasourcePlus { await this.connect() const hasFilters = dataFilters.hasFilters(query.filters) const limit = query.paginate?.limit || 100 - const page: number = - typeof query.paginate?.page === "number" - ? query.paginate.page - : parseInt(query.paginate?.page || "1") - const offset = (page - 1) * limit + let offset = query.paginate?.offset || 0 + + let page = query.paginate?.page + if (typeof page === "string") { + page = parseInt(page) + } + if (page !== undefined) { + offset = page * limit + } + const sheet = this.client.sheetsByTitle[query.sheet] let rows: GoogleSpreadsheetRow[] = [] if (query.paginate && !hasFilters) { diff --git a/packages/server/src/integrations/tests/googlesheets.spec.ts b/packages/server/src/integrations/tests/googlesheets.spec.ts index 9781b97972..62d56bb2c2 100644 --- a/packages/server/src/integrations/tests/googlesheets.spec.ts +++ b/packages/server/src/integrations/tests/googlesheets.spec.ts @@ -208,6 +208,42 @@ describe("Google Sheets Integration", () => { expect(row2.name).toEqual("Test Contact 2") expect(row2.description).toEqual("original description 2") }) + + it("can paginate correctly", async () => { + await config.api.row.bulkImport(table._id!, { + rows: Array.from({ length: 248 }, (_, i) => ({ + name: `${i}`, + description: "", + })), + }) + + let resp = await config.api.row.search(table._id!, { + tableId: table._id!, + query: {}, + paginate: true, + limit: 10, + }) + let rows = resp.rows + + while (resp.hasNextPage) { + resp = await config.api.row.search(table._id!, { + tableId: table._id!, + query: {}, + paginate: true, + limit: 10, + bookmark: resp.bookmark, + }) + rows = rows.concat(resp.rows) + if (rows.length > 250) { + throw new Error("Too many rows returned") + } + } + + expect(rows.length).toEqual(250) + expect(rows.map(row => row.name)).toEqual( + expect.arrayContaining(Array.from({ length: 248 }, (_, i) => `${i}`)) + ) + }) }) describe("update", () => { @@ -299,5 +335,160 @@ describe("Google Sheets Integration", () => { expect(mock.cell("A2")).toEqual("Test Contact Updated") expect(mock.cell("B2")).toEqual("original description updated") }) + + it("should be able to rename a column", async () => { + const row = await config.api.row.save(table._id!, { + name: "Test Contact", + description: "original description", + }) + + const { name, ...otherColumns } = table.schema + const renamedTable = await config.api.table.save({ + ...table, + schema: { + ...otherColumns, + renamed: { + ...table.schema.name, + }, + }, + _rename: { + old: "name", + updated: "renamed", + }, + }) + + expect(renamedTable.schema.name).not.toBeDefined() + expect(renamedTable.schema.renamed).toBeDefined() + + expect(mock.cell("A1")).toEqual("renamed") + expect(mock.cell("B1")).toEqual("description") + expect(mock.cell("A2")).toEqual("Test Contact") + expect(mock.cell("B2")).toEqual("original description") + expect(mock.cell("A3")).toEqual(null) + expect(mock.cell("B3")).toEqual(null) + + const renamedRow = await config.api.row.get(table._id!, row._id!) + expect(renamedRow.renamed).toEqual("Test Contact") + expect(renamedRow.description).toEqual("original description") + expect(renamedRow.name).not.toBeDefined() + }) + + // TODO: this gets the error "Sheet is not large enough to fit 27 columns. Resize the sheet first." + // eslint-disable-next-line jest/no-commented-out-tests + // it("should be able to add a new column", async () => { + // const updatedTable = await config.api.table.save({ + // ...table, + // schema: { + // ...table.schema, + // newColumn: { + // name: "newColumn", + // type: FieldType.STRING, + // }, + // }, + // }) + + // expect(updatedTable.schema.newColumn).toBeDefined() + + // expect(mock.cell("A1")).toEqual("name") + // expect(mock.cell("B1")).toEqual("description") + // expect(mock.cell("C1")).toEqual("newColumn") + // }) + + it("should be able to delete a column", async () => { + const row = await config.api.row.save(table._id!, { + name: "Test Contact", + description: "original description", + }) + + const updatedTable = await config.api.table.save({ + ...table, + schema: { + name: { + name: "name", + type: FieldType.STRING, + }, + }, + }) + + expect(updatedTable.schema.name).toBeDefined() + expect(updatedTable.schema.description).not.toBeDefined() + + // TODO: we don't delete data in deleted columns yet, should we? + // expect(mock.cell("A1")).toEqual("name") + // expect(mock.cell("B1")).toEqual(null) + + const updatedRow = await config.api.row.get(table._id!, row._id!) + expect(updatedRow.name).toEqual("Test Contact") + expect(updatedRow.description).not.toBeDefined() + }) + }) + + describe("delete", () => { + let table: Table + beforeEach(async () => { + table = await config.api.table.save({ + name: "Test Table", + type: "table", + sourceId: datasource._id!, + sourceType: TableSourceType.EXTERNAL, + schema: { + name: { + name: "name", + type: FieldType.STRING, + constraints: { + type: "string", + }, + }, + description: { + name: "description", + type: FieldType.STRING, + constraints: { + type: "string", + }, + }, + }, + }) + + await config.api.row.bulkImport(table._id!, { + rows: [ + { + name: "Test Contact 1", + description: "original description 1", + }, + { + name: "Test Contact 2", + description: "original description 2", + }, + ], + }) + }) + + it("can delete a table", async () => { + expect(mock.sheet(table.name)).toBeDefined() + await config.api.table.destroy(table._id!, table._rev!) + expect(mock.sheet(table.name)).toBeUndefined() + }) + + it("can delete a row", async () => { + const rows = await config.api.row.fetch(table._id!) + expect(rows.length).toEqual(2) + + // Because row IDs in Google Sheets are sequential and determined by the + // actual row in the sheet, deleting a row will shift the row IDs down by + // one. This is why we reverse the rows before deleting them. + for (const row of rows.reverse()) { + await config.api.row.delete(table._id!, { _id: row._id! }) + } + + expect(mock.cell("A1")).toEqual("name") + expect(mock.cell("B1")).toEqual("description") + expect(mock.cell("A2")).toEqual(null) + expect(mock.cell("B2")).toEqual(null) + expect(mock.cell("A3")).toEqual(null) + expect(mock.cell("B3")).toEqual(null) + + const emptyRows = await config.api.row.fetch(table._id!) + expect(emptyRows.length).toEqual(0) + }) }) }) diff --git a/packages/server/src/integrations/tests/sql.spec.ts b/packages/server/src/integrations/tests/sql.spec.ts index cf5b88b0fd..5a505fbb40 100644 --- a/packages/server/src/integrations/tests/sql.spec.ts +++ b/packages/server/src/integrations/tests/sql.spec.ts @@ -162,7 +162,7 @@ describe("SQL query builder", () => { const query = sql._query(generateRelationshipJson({ schema: "production" })) expect(query).toEqual({ bindings: [limit, relationshipLimit], - sql: `with "paginated" as (select "brands".* from "production"."brands" order by "test"."id" asc limit $1) select "brands".*, (select json_agg(json_build_object('product_id',"products"."product_id",'product_name',"products"."product_name",'brand_id',"products"."brand_id")) from (select "products".* from "production"."products" as "products" where "products"."brand_id" = "brands"."brand_id" order by "products"."brand_id" asc limit $2) as "products") as "products" from "paginated" as "brands" order by "test"."id" asc`, + sql: `with "paginated" as (select "brands".* from "production"."brands" order by "test"."id" asc limit $1) select "brands".*, (select json_agg(json_build_object('brand_id',"products"."brand_id",'product_id',"products"."product_id",'product_name',"products"."product_name")) from (select "products".* from "production"."products" as "products" where "products"."brand_id" = "brands"."brand_id" order by "products"."brand_id" asc limit $2) as "products") as "products" from "paginated" as "brands" order by "test"."id" asc`, }) }) @@ -170,7 +170,7 @@ describe("SQL query builder", () => { const query = sql._query(generateRelationshipJson()) expect(query).toEqual({ bindings: [limit, relationshipLimit], - sql: `with "paginated" as (select "brands".* from "brands" order by "test"."id" asc limit $1) select "brands".*, (select json_agg(json_build_object('product_id',"products"."product_id",'product_name',"products"."product_name",'brand_id',"products"."brand_id")) from (select "products".* from "products" as "products" where "products"."brand_id" = "brands"."brand_id" order by "products"."brand_id" asc limit $2) as "products") as "products" from "paginated" as "brands" order by "test"."id" asc`, + sql: `with "paginated" as (select "brands".* from "brands" order by "test"."id" asc limit $1) select "brands".*, (select json_agg(json_build_object('brand_id',"products"."brand_id",'product_id',"products"."product_id",'product_name',"products"."product_name")) from (select "products".* from "products" as "products" where "products"."brand_id" = "brands"."brand_id" order by "products"."brand_id" asc limit $2) as "products") as "products" from "paginated" as "brands" order by "test"."id" asc`, }) }) diff --git a/packages/server/src/integrations/tests/sqlAlias.spec.ts b/packages/server/src/integrations/tests/sqlAlias.spec.ts index 9548499c65..fc5af4238c 100644 --- a/packages/server/src/integrations/tests/sqlAlias.spec.ts +++ b/packages/server/src/integrations/tests/sqlAlias.spec.ts @@ -63,7 +63,7 @@ describe("Captures of real examples", () => { bindings: [primaryLimit, relationshipLimit, relationshipLimit], sql: expect.stringContaining( multiline( - `select json_agg(json_build_object('executorid',"b"."executorid",'taskname',"b"."taskname",'taskid',"b"."taskid",'completed',"b"."completed",'qaid',"b"."qaid",'executorid',"b"."executorid",'taskname',"b"."taskname",'taskid',"b"."taskid",'completed',"b"."completed",'qaid',"b"."qaid")` + `select json_agg(json_build_object('completed',"b"."completed",'completed',"b"."completed",'executorid',"b"."executorid",'executorid',"b"."executorid",'qaid',"b"."qaid",'qaid',"b"."qaid",'taskid',"b"."taskid",'taskid',"b"."taskid",'taskname',"b"."taskname",'taskname',"b"."taskname")` ) ), }) @@ -95,7 +95,7 @@ describe("Captures of real examples", () => { sql: expect.stringContaining( multiline( `with "paginated" as (select "a".* from "products" as "a" order by "a"."productname" asc nulls first, "a"."productid" asc limit $1) - select "a".*, (select json_agg(json_build_object('executorid',"b"."executorid",'taskname',"b"."taskname",'taskid',"b"."taskid",'completed',"b"."completed",'qaid',"b"."qaid")) + select "a".*, (select json_agg(json_build_object('completed',"b"."completed",'executorid',"b"."executorid",'qaid',"b"."qaid",'taskid',"b"."taskid",'taskname',"b"."taskname")) from (select "b".* from "tasks" as "b" inner join "products_tasks" as "c" on "b"."taskid" = "c"."taskid" where "c"."productid" = "a"."productid" order by "b"."taskid" asc limit $2) as "b") as "tasks" from "paginated" as "a" order by "a"."productname" asc nulls first, "a"."productid" asc` ) @@ -113,7 +113,7 @@ describe("Captures of real examples", () => { bindings: [...filters, relationshipLimit, relationshipLimit], sql: multiline( `with "paginated" as (select "a".* from "tasks" as "a" where "a"."taskid" in ($1, $2) order by "a"."taskid" asc limit $3) - select "a".*, (select json_agg(json_build_object('productname',"b"."productname",'productid',"b"."productid")) + select "a".*, (select json_agg(json_build_object('productid',"b"."productid",'productname',"b"."productname")) from (select "b".* from "products" as "b" inner join "products_tasks" as "c" on "b"."productid" = "c"."productid" where "c"."taskid" = "a"."taskid" order by "b"."productid" asc limit $4) as "b") as "products" from "paginated" as "a" order by "a"."taskid" asc` ), diff --git a/packages/server/src/integrations/tests/utils/googlesheets.ts b/packages/server/src/integrations/tests/utils/googlesheets.ts index c58066bee5..4b17c25b01 100644 --- a/packages/server/src/integrations/tests/utils/googlesheets.ts +++ b/packages/server/src/integrations/tests/utils/googlesheets.ts @@ -21,6 +21,7 @@ import type { CellFormat, CellPadding, Color, + GridRange, } from "google-spreadsheet/src/lib/types/sheets-types" const BLACK: Color = { red: 0, green: 0, blue: 0 } @@ -88,11 +89,38 @@ interface UpdateValuesResponse { updatedData: ValueRange } +// https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets/request#AddSheetRequest +interface AddSheetRequest { + properties: WorksheetProperties +} + // https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets/response#AddSheetResponse interface AddSheetResponse { properties: WorksheetProperties } +interface DeleteRangeRequest { + range: GridRange + shiftDimension: WorksheetDimension +} + +// https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets/request#DeleteSheetRequest +interface DeleteSheetRequest { + sheetId: number +} + +// https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets/request +interface BatchUpdateRequest { + requests: { + addSheet?: AddSheetRequest + deleteRange?: DeleteRangeRequest + deleteSheet?: DeleteSheetRequest + }[] + includeSpreadsheetInResponse: boolean + responseRanges: string[] + responseIncludeGridData: boolean +} + // https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets/response interface BatchUpdateResponse { spreadsheetId: string @@ -102,23 +130,6 @@ interface BatchUpdateResponse { updatedSpreadsheet: Spreadsheet } -// https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets/request#AddSheetRequest -interface AddSheetRequest { - properties: WorksheetProperties -} - -interface Request { - addSheet?: AddSheetRequest -} - -// https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets/request -interface BatchUpdateRequest { - requests: Request[] - includeSpreadsheetInResponse: boolean - responseRanges: string[] - responseIncludeGridData: boolean -} - // https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets/sheets#RowData interface RowData { values: CellData[] @@ -369,13 +380,17 @@ export class GoogleSheetsMock { private handleValueAppend(request: AppendRequest): AppendResponse { const { range, params, body } = request - const { sheet, bottomRight } = this.parseA1Notation(range) + const { sheetId, endRowIndex } = this.parseA1Notation(range) + const sheet = this.getSheetById(sheetId) + if (!sheet) { + throw new Error(`Sheet ${sheetId} not found`) + } const newRows = body.values.map(v => this.valuesToRowData(v)) const toDelete = params.insertDataOption === "INSERT_ROWS" ? newRows.length : 0 - sheet.data[0].rowData.splice(bottomRight.row + 1, toDelete, ...newRows) - sheet.data[0].rowMetadata.splice(bottomRight.row + 1, toDelete, { + sheet.data[0].rowData.splice(endRowIndex + 1, toDelete, ...newRows) + sheet.data[0].rowMetadata.splice(endRowIndex + 1, toDelete, { hiddenByUser: false, hiddenByFilter: false, pixelSize: 100, @@ -384,17 +399,13 @@ export class GoogleSheetsMock { // It's important to give back a correct updated range because the API // library we use makes use of it to assign the correct row IDs to rows. - const updatedRange = this.createA1FromRanges( - sheet, - { - row: bottomRight.row + 1, - column: 0, - }, - { - row: bottomRight.row + newRows.length, - column: 0, - } - ) + const updatedRange = this.createA1({ + sheetId, + startRowIndex: endRowIndex + 1, + startColumnIndex: 0, + endRowIndex: endRowIndex + newRows.length, + endColumnIndex: 0, + }) return { spreadsheetId: this.spreadsheet.spreadsheetId, @@ -438,6 +449,14 @@ export class GoogleSheetsMock { addSheet: this.handleAddSheet(request.addSheet), }) } + if (request.deleteRange) { + this.handleDeleteRange(request.deleteRange) + response.replies.push({}) + } + if (request.deleteSheet) { + this.handleDeleteSheet(request.deleteSheet) + response.replies.push({}) + } } return response @@ -474,12 +493,29 @@ export class GoogleSheetsMock { return { properties: properties as WorksheetProperties } } + private handleDeleteRange(request: DeleteRangeRequest) { + const { range, shiftDimension } = request + + if (shiftDimension !== "ROWS") { + throw new Error("Only row-based deletes are supported") + } + + this.iterateRange(range, cell => { + cell.userEnteredValue = this.createValue(null) + }) + } + + private handleDeleteSheet(request: DeleteSheetRequest) { + const { sheetId } = request + this.spreadsheet.sheets.splice(sheetId, 1) + } + private handleGetSpreadsheet(): Spreadsheet { return this.spreadsheet } private handleValueUpdate(valueRange: ValueRange): UpdateValuesResponse { - this.iterateCells(valueRange, (cell, value) => { + this.iterateValueRange(valueRange, (cell, value) => { cell.userEnteredValue = this.createValue(value) }) @@ -494,7 +530,27 @@ export class GoogleSheetsMock { return response } - private iterateCells( + private iterateRange(range: GridRange, cb: (cell: CellData) => void) { + const { + sheetId, + startRowIndex, + endRowIndex, + startColumnIndex, + endColumnIndex, + } = this.ensureGridRange(range) + + for (let row = startRowIndex; row <= endRowIndex; row++) { + for (let col = startColumnIndex; col <= endColumnIndex; col++) { + const cell = this.getCellNumericIndexes(sheetId, row, col) + if (!cell) { + throw new Error("Cell not found") + } + cb(cell) + } + } + } + + private iterateValueRange( valueRange: ValueRange, cb: (cell: CellData, value: Value) => void ) { @@ -502,33 +558,46 @@ export class GoogleSheetsMock { throw new Error("Only row-major updates are supported") } - const { sheet, topLeft, bottomRight } = this.parseA1Notation( - valueRange.range - ) - for (let row = topLeft.row; row <= bottomRight.row; row++) { - for (let col = topLeft.column; col <= bottomRight.column; col++) { - const cell = this.getCellNumericIndexes(sheet, row, col) + const { + sheetId, + startColumnIndex, + startRowIndex, + endColumnIndex, + endRowIndex, + } = this.parseA1Notation(valueRange.range) + + for (let row = startRowIndex; row <= endRowIndex; row++) { + for (let col = startColumnIndex; col <= endColumnIndex; col++) { + const cell = this.getCellNumericIndexes(sheetId, row, col) if (!cell) { throw new Error("Cell not found") } - const value = valueRange.values[row - topLeft.row][col - topLeft.column] + const value = + valueRange.values[row - startRowIndex][col - startColumnIndex] cb(cell, value) } } } private getValueRange(range: string): ValueRange { - const { sheet, topLeft, bottomRight } = this.parseA1Notation(range) + const { + sheetId, + startRowIndex, + endRowIndex, + startColumnIndex, + endColumnIndex, + } = this.parseA1Notation(range) + const valueRange: ValueRange = { range, majorDimension: "ROWS", values: [], } - for (let row = topLeft.row; row <= bottomRight.row; row++) { + for (let row = startRowIndex; row <= endRowIndex; row++) { const values: Value[] = [] - for (let col = topLeft.column; col <= bottomRight.column; col++) { - const cell = this.getCellNumericIndexes(sheet, row, col) + for (let col = startColumnIndex; col <= endColumnIndex; col++) { + const cell = this.getCellNumericIndexes(sheetId, row, col) if (!cell) { throw new Error("Cell not found") } @@ -693,14 +762,12 @@ export class GoogleSheetsMock { } private cellData(cell: string): CellData | undefined { - const { - sheet, - topLeft: { row, column }, - } = this.parseA1Notation(cell) - return this.getCellNumericIndexes(sheet, row, column) + const { sheetId, startColumnIndex, startRowIndex } = + this.parseA1Notation(cell) + return this.getCellNumericIndexes(sheetId, startRowIndex, startColumnIndex) } - cell(cell: string): Value | undefined { + public cell(cell: string): Value | undefined { const cellData = this.cellData(cell) if (!cellData) { return undefined @@ -708,11 +775,26 @@ export class GoogleSheetsMock { return this.cellValue(cellData) } + public sheet(name: string | number): Sheet | undefined { + if (typeof name === "number") { + return this.getSheetById(name) + } + return this.getSheetByName(name) + } + private getCellNumericIndexes( - sheet: Sheet, + sheet: Sheet | number, row: number, column: number ): CellData | undefined { + if (typeof sheet === "number") { + const foundSheet = this.getSheetById(sheet) + if (!foundSheet) { + return undefined + } + sheet = foundSheet + } + const data = sheet.data[0] const rowData = data.rowData[row] if (!rowData) { @@ -751,11 +833,7 @@ export class GoogleSheetsMock { // "Sheet1!A:B" -> { topLeft: { row: 0, column: 0 }, bottomRight: { row: 99, column: 1 } } // "Sheet1!1:1" -> { topLeft: { row: 0, column: 0 }, bottomRight: { row: 0, column: 25 } } // "Sheet1!1:2" -> { topLeft: { row: 0, column: 0 }, bottomRight: { row: 1, column: 25 } } - private parseA1Notation(range: string): { - sheet: Sheet - topLeft: Range - bottomRight: Range - } { + private parseA1Notation(range: string): Required { let sheet: Sheet let rest: string if (!range.includes("!")) { @@ -793,35 +871,54 @@ export class GoogleSheetsMock { parsedBottomRight = parsedTopLeft } - if (parsedTopLeft && parsedTopLeft.row === undefined) { - parsedTopLeft.row = 0 - } - if (parsedTopLeft && parsedTopLeft.column === undefined) { - parsedTopLeft.column = 0 - } - if (parsedBottomRight && parsedBottomRight.row === undefined) { - parsedBottomRight.row = sheet.properties.gridProperties.rowCount - 1 - } - if (parsedBottomRight && parsedBottomRight.column === undefined) { - parsedBottomRight.column = sheet.properties.gridProperties.columnCount - 1 + return this.ensureGridRange({ + sheetId: sheet.properties.sheetId, + startRowIndex: parsedTopLeft.row, + endRowIndex: parsedBottomRight.row, + startColumnIndex: parsedTopLeft.column, + endColumnIndex: parsedBottomRight.column, + }) + } + + private ensureGridRange(range: GridRange): Required { + const sheet = this.getSheetById(range.sheetId) + if (!sheet) { + throw new Error(`Sheet ${range.sheetId} not found`) } return { - sheet, - topLeft: parsedTopLeft as Range, - bottomRight: parsedBottomRight as Range, + sheetId: range.sheetId, + startRowIndex: range.startRowIndex ?? 0, + endRowIndex: + range.endRowIndex ?? sheet.properties.gridProperties.rowCount - 1, + startColumnIndex: range.startColumnIndex ?? 0, + endColumnIndex: + range.endColumnIndex ?? sheet.properties.gridProperties.columnCount - 1, } } - private createA1FromRanges(sheet: Sheet, topLeft: Range, bottomRight: Range) { + private createA1(range: Required) { + const { + sheetId, + startColumnIndex, + startRowIndex, + endColumnIndex, + endRowIndex, + } = range + + const sheet = this.getSheetById(sheetId) + if (!sheet) { + throw new Error(`Sheet ${range.sheetId} not found`) + } + let title = sheet.properties.title if (title.includes(" ")) { title = `'${title}'` } - const topLeftLetter = this.numberToLetter(topLeft.column) - const bottomRightLetter = this.numberToLetter(bottomRight.column) - const topLeftRow = topLeft.row + 1 - const bottomRightRow = bottomRight.row + 1 + const topLeftLetter = this.numberToLetter(startColumnIndex) + const bottomRightLetter = this.numberToLetter(endColumnIndex) + const topLeftRow = startRowIndex + 1 + const bottomRightRow = endRowIndex + 1 return `${title}!${topLeftLetter}${topLeftRow}:${bottomRightLetter}${bottomRightRow}` } @@ -860,4 +957,10 @@ export class GoogleSheetsMock { sheet => sheet.properties.title === name ) } + + private getSheetById(id: number): Sheet | undefined { + return this.spreadsheet.sheets.find( + sheet => sheet.properties.sheetId === id + ) + } } diff --git a/packages/server/src/sdk/app/tables/getters.ts b/packages/server/src/sdk/app/tables/getters.ts index 27e9962a1a..5ff000fe12 100644 --- a/packages/server/src/sdk/app/tables/getters.ts +++ b/packages/server/src/sdk/app/tables/getters.ts @@ -101,6 +101,15 @@ export async function getTable(tableId: string): Promise { return await processTable(output) } +export async function doesTableExist(tableId: string): Promise { + try { + const table = await getTable(tableId) + return !!table + } catch (err) { + return false + } +} + export async function getAllTables() { const [internal, external] = await Promise.all([ getAllInternalTables(), diff --git a/packages/types/src/sdk/automations/index.ts b/packages/types/src/sdk/automations/index.ts index 5ea22148a5..d04f126c32 100644 --- a/packages/types/src/sdk/automations/index.ts +++ b/packages/types/src/sdk/automations/index.ts @@ -15,4 +15,10 @@ export interface AutomationData { automation: Automation } +export interface AutomationRowEvent { + appId: string + row: Row + oldRow: Row +} + export type AutomationJob = Job