From 0202db3efe84c5ef244463bfb29b03a8171300f6 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 31 Jul 2024 12:20:45 +0200 Subject: [PATCH 01/15] Unify enums --- packages/builder/src/constants/backend/index.js | 11 ++++------- .../server/src/api/controllers/view/exporters.ts | 14 +++++--------- packages/types/src/sdk/row.ts | 6 ++++++ 3 files changed, 15 insertions(+), 16 deletions(-) diff --git a/packages/builder/src/constants/backend/index.js b/packages/builder/src/constants/backend/index.js index ea8d35704f..6fbc36afe2 100644 --- a/packages/builder/src/constants/backend/index.js +++ b/packages/builder/src/constants/backend/index.js @@ -9,7 +9,10 @@ import { Constants } from "@budibase/frontend-core" const { TypeIconMap } = Constants -export { RelationshipType } from "@budibase/types" +export { + RelationshipType, + RowExportFormat as ROW_EXPORT_FORMATS, +} from "@budibase/types" export const AUTO_COLUMN_SUB_TYPES = AutoFieldSubType @@ -307,9 +310,3 @@ export const DatasourceTypes = { GRAPH: "Graph", API: "API", } - -export const ROW_EXPORT_FORMATS = { - CSV: "csv", - JSON: "json", - JSON_WITH_SCHEMA: "jsonWithSchema", -} diff --git a/packages/server/src/api/controllers/view/exporters.ts b/packages/server/src/api/controllers/view/exporters.ts index 9cf114f4e5..946a1b346a 100644 --- a/packages/server/src/api/controllers/view/exporters.ts +++ b/packages/server/src/api/controllers/view/exporters.ts @@ -1,4 +1,6 @@ -import { Row, TableSchema } from "@budibase/types" +import { Row, RowExportFormat, TableSchema } from "@budibase/types" + +export { RowExportFormat as Format } from "@budibase/types" function getHeaders( headers: string[], @@ -46,14 +48,8 @@ export function jsonWithSchema(schema: TableSchema, rows: Row[]) { return JSON.stringify({ schema: newSchema, rows }, undefined, 2) } -export enum Format { - CSV = "csv", - JSON = "json", - JSON_WITH_SCHEMA = "jsonWithSchema", -} - -export function isFormat(format: any): format is Format { - return Object.values(Format).includes(format as Format) +export function isFormat(format: any): format is RowExportFormat { + return Object.values(RowExportFormat).includes(format as RowExportFormat) } export function parseCsvExport(value: string) { diff --git a/packages/types/src/sdk/row.ts b/packages/types/src/sdk/row.ts index b0b137034b..6850359cc3 100644 --- a/packages/types/src/sdk/row.ts +++ b/packages/types/src/sdk/row.ts @@ -30,3 +30,9 @@ export interface SearchResponse { bookmark?: string | number totalRows?: number } + +export enum RowExportFormat { + CSV = "csv", + JSON = "json", + JSON_WITH_SCHEMA = "jsonWithSchema", +} From 58a47b801a54f0ed1b281cb5693f7b14458ffa4f Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 31 Jul 2024 12:23:09 +0200 Subject: [PATCH 02/15] Remove magic strings --- packages/server/src/tests/utilities/api/legacyView.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/server/src/tests/utilities/api/legacyView.ts b/packages/server/src/tests/utilities/api/legacyView.ts index ae250a81e2..643969955c 100644 --- a/packages/server/src/tests/utilities/api/legacyView.ts +++ b/packages/server/src/tests/utilities/api/legacyView.ts @@ -1,5 +1,5 @@ import { Expectations, TestAPI } from "./base" -import { Row, View, ViewCalculation } from "@budibase/types" +import { Row, RowExportFormat, View, ViewCalculation } from "@budibase/types" export class LegacyViewAPI extends TestAPI { get = async ( @@ -24,7 +24,7 @@ export class LegacyViewAPI extends TestAPI { export = async ( viewName: string, - format: "json" | "csv" | "jsonWithSchema", + format: `${RowExportFormat}`, expectations?: Expectations ) => { const response = await this._requestRaw("get", `/api/views/export`, { From 62fa05a855205f43b0ef929fc051cd91b0fd8f58 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 31 Jul 2024 13:28:28 +0200 Subject: [PATCH 03/15] Type --- packages/server/src/sdk/app/rows/utils.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/sdk/app/rows/utils.ts b/packages/server/src/sdk/app/rows/utils.ts index cd1b663f6a..e463397ad9 100644 --- a/packages/server/src/sdk/app/rows/utils.ts +++ b/packages/server/src/sdk/app/rows/utils.ts @@ -76,7 +76,7 @@ export async function getDatasourceAndQuery( } export function cleanExportRows( - rows: any[], + rows: Row[], schema: TableSchema, format: string, columns?: string[], From fe2b2bb097fb4ee106754630d2419f0f6a8a3cd2 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 31 Jul 2024 13:33:20 +0200 Subject: [PATCH 04/15] Don't export couchdb fields --- .../server/src/sdk/app/rows/search/internal.ts | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/packages/server/src/sdk/app/rows/search/internal.ts b/packages/server/src/sdk/app/rows/search/internal.ts index 46d2cd8c61..f86b041597 100644 --- a/packages/server/src/sdk/app/rows/search/internal.ts +++ b/packages/server/src/sdk/app/rows/search/internal.ts @@ -11,6 +11,7 @@ import { SearchResponse, SortType, Table, + TableSchema, User, } from "@budibase/types" import { getGlobalUsersFromMetadata } from "../../../../utilities/global" @@ -137,6 +138,9 @@ export async function exportRows( let rows: Row[] = [] let schema = table.schema let headers + + result = trimFields(result, schema) + // Filter data to only specified columns if required if (columns && columns.length) { for (let i = 0; i < result.length; i++) { @@ -299,3 +303,13 @@ async function getView(db: Database, viewName: string) { } return viewInfo } + +function trimFields(rows: Row[], schema: TableSchema) { + const allowedFields = ["_id", ...Object.keys(schema)] + const result = rows.map(row => + Object.keys(row) + .filter(key => allowedFields.includes(key)) + .reduce((acc, key) => ({ ...acc, [key]: row[key] }), {} as Row) + ) + return result +} From 543d0e1ce619b7fbaeeb994c0d33c325b2d934eb Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 31 Jul 2024 14:01:38 +0200 Subject: [PATCH 05/15] Add tests --- .../server/src/api/routes/tests/row.spec.ts | 45 ++++++++++++------- 1 file changed, 30 insertions(+), 15 deletions(-) diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts index 8871841ee7..96a157893f 100644 --- a/packages/server/src/api/routes/tests/row.spec.ts +++ b/packages/server/src/api/routes/tests/row.spec.ts @@ -1640,23 +1640,38 @@ describe.each([ table = await config.api.table.save(defaultTable()) }) - it("should allow exporting all columns", async () => { - const existing = await config.api.row.save(table._id!, {}) - const res = await config.api.row.exportRows(table._id!, { - rows: [existing._id!], - }) - const results = JSON.parse(res) - expect(results.length).toEqual(1) - const row = results[0] + isInternal && + it("should not export internal couchdb fields", async () => { + const existing = await config.api.row.save(table._id!, { + name: generator.guid(), + description: generator.paragraph(), + }) + const res = await config.api.row.exportRows(table._id!, { + rows: [existing._id!], + }) + const results = JSON.parse(res) + expect(results.length).toEqual(1) + const row = results[0] - // Ensure all original columns were exported - expect(Object.keys(row).length).toBeGreaterThanOrEqual( - Object.keys(existing).length - ) - Object.keys(existing).forEach(key => { - expect(row[key]).toEqual(existing[key]) + expect(Object.keys(row)).toEqual(["_id", "name", "description"]) + }) + + !isInternal && + it("should allow exporting all columns", async () => { + const existing = await config.api.row.save(table._id!, {}) + const res = await config.api.row.exportRows(table._id!, { + rows: [existing._id!], + }) + const results = JSON.parse(res) + expect(results.length).toEqual(1) + const row = results[0] + + // Ensure all original columns were exported + expect(Object.keys(row).length).toBe(Object.keys(existing).length) + Object.keys(existing).forEach(key => { + expect(row[key]).toEqual(existing[key]) + }) }) - }) it("should allow exporting only certain columns", async () => { const existing = await config.api.row.save(table._id!, {}) From 4f65306c4fdf71deb90fe9283585c446a7b6c318 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 25 Jul 2024 17:20:10 +0200 Subject: [PATCH 06/15] Add basic validateNewTableImport test --- .../server/src/api/routes/tests/table.spec.ts | 33 +++++++++++++++++++ .../server/src/tests/utilities/api/table.ts | 22 ++++++++++++- 2 files changed, 54 insertions(+), 1 deletion(-) diff --git a/packages/server/src/api/routes/tests/table.spec.ts b/packages/server/src/api/routes/tests/table.spec.ts index a2cff7b395..b32983b8ad 100644 --- a/packages/server/src/api/routes/tests/table.spec.ts +++ b/packages/server/src/api/routes/tests/table.spec.ts @@ -10,6 +10,7 @@ import { Row, SaveTableRequest, Table, + TableSchema, TableSourceType, User, ViewCalculation, @@ -1022,4 +1023,36 @@ describe.each([ }) }) }) + + describe("import validation", () => { + const basicSchema: TableSchema = { + id: { + type: FieldType.NUMBER, + name: "id", + }, + name: { + type: FieldType.STRING, + name: "name", + }, + } + + describe("validateNewTableImport", () => { + it("can validate basic imports", async () => { + const result = await config.api.table.validateNewTableImport( + [{ id: generator.natural(), name: generator.first() }], + basicSchema + ) + + expect(result).toEqual({ + allValid: true, + errors: {}, + invalidColumns: [], + schemaValidation: { + id: true, + name: true, + }, + }) + }) + }) + }) }) diff --git a/packages/server/src/tests/utilities/api/table.ts b/packages/server/src/tests/utilities/api/table.ts index d918ba8b9a..c42247dc59 100644 --- a/packages/server/src/tests/utilities/api/table.ts +++ b/packages/server/src/tests/utilities/api/table.ts @@ -3,9 +3,12 @@ import { BulkImportResponse, MigrateRequest, MigrateResponse, + Row, SaveTableRequest, SaveTableResponse, Table, + TableSchema, + ValidateTableImportResponse, } from "@budibase/types" import { Expectations, TestAPI } from "./base" @@ -61,8 +64,25 @@ export class TableAPI extends TestAPI { revId: string, expectations?: Expectations ): Promise => { - return await this._delete(`/api/tables/${tableId}/${revId}`, { + return await this._delete(`/api/tables/${tableId}/${revId}`, { expectations, }) } + + validateNewTableImport = async ( + rows: Row[], + schema: TableSchema, + expectations?: Expectations + ): Promise => { + return await this._post( + `/api/tables/validateNewTableImport`, + { + body: { + rows, + schema, + }, + expectations, + } + ) + } } From 5896e94e56992964064e5cd43d19b7f7a667c608 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 25 Jul 2024 17:32:53 +0200 Subject: [PATCH 07/15] Add basic validateExistingTableImport test --- .../server/src/api/routes/tests/table.spec.ts | 25 +++++++++++++++++++ .../server/src/tests/utilities/api/table.ts | 14 +++++++++++ 2 files changed, 39 insertions(+) diff --git a/packages/server/src/api/routes/tests/table.spec.ts b/packages/server/src/api/routes/tests/table.spec.ts index b32983b8ad..67b1d64ae1 100644 --- a/packages/server/src/api/routes/tests/table.spec.ts +++ b/packages/server/src/api/routes/tests/table.spec.ts @@ -1054,5 +1054,30 @@ describe.each([ }) }) }) + + describe("validateExistingTableImport", () => { + it("can validate basic imports", async () => { + const table = await config.api.table.save( + tableForDatasource(datasource, { + primary: ["id"], + schema: basicSchema, + }) + ) + const result = await config.api.table.validateExistingTableImport({ + tableId: table._id, + rows: [{ id: generator.natural(), name: generator.first() }], + }) + + expect(result).toEqual({ + allValid: true, + errors: {}, + invalidColumns: [], + schemaValidation: { + id: true, + name: true, + }, + }) + }) + }) }) }) diff --git a/packages/server/src/tests/utilities/api/table.ts b/packages/server/src/tests/utilities/api/table.ts index c42247dc59..9d4a92250a 100644 --- a/packages/server/src/tests/utilities/api/table.ts +++ b/packages/server/src/tests/utilities/api/table.ts @@ -8,6 +8,7 @@ import { SaveTableResponse, Table, TableSchema, + ValidateTableImportRequest, ValidateTableImportResponse, } from "@budibase/types" import { Expectations, TestAPI } from "./base" @@ -85,4 +86,17 @@ export class TableAPI extends TestAPI { } ) } + + validateExistingTableImport = async ( + body: ValidateTableImportRequest, + expectations?: Expectations + ): Promise => { + return await this._post( + `/api/tables/validateExistingTableImport`, + { + body, + expectations, + } + ) + } } From 9d0fdeff68b19d0551498cbc72d2cf53568476bd Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 31 Jul 2024 14:17:51 +0200 Subject: [PATCH 08/15] Add validateExistingTableImport _id support test --- .../server/src/api/routes/tests/table.spec.ts | 33 ++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/packages/server/src/api/routes/tests/table.spec.ts b/packages/server/src/api/routes/tests/table.spec.ts index 67b1d64ae1..dd40a2420e 100644 --- a/packages/server/src/api/routes/tests/table.spec.ts +++ b/packages/server/src/api/routes/tests/table.spec.ts @@ -1,4 +1,4 @@ -import { context, events } from "@budibase/backend-core" +import { context, docIds, events } from "@budibase/backend-core" import { AutoFieldSubType, BBReferenceFieldSubType, @@ -1078,6 +1078,37 @@ describe.each([ }, }) }) + + isInternal && + it("can reimport _id fields for internal tables", async () => { + const table = await config.api.table.save( + tableForDatasource(datasource, { + primary: ["id"], + schema: basicSchema, + }) + ) + const result = await config.api.table.validateExistingTableImport({ + tableId: table._id, + rows: [ + { + _id: docIds.generateRowID(table._id!), + id: generator.natural(), + name: generator.first(), + }, + ], + }) + + expect(result).toEqual({ + allValid: true, + errors: {}, + invalidColumns: [], + schemaValidation: { + _id: true, + id: true, + name: true, + }, + }) + }) }) }) }) From b28aaa3a936dfc27c9263efc83f6c6e169ad5d3f Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 31 Jul 2024 14:22:54 +0200 Subject: [PATCH 09/15] Fix --- packages/server/src/api/controllers/table/index.ts | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/packages/server/src/api/controllers/table/index.ts b/packages/server/src/api/controllers/table/index.ts index ba861064bb..a02ecc665e 100644 --- a/packages/server/src/api/controllers/table/index.ts +++ b/packages/server/src/api/controllers/table/index.ts @@ -17,6 +17,7 @@ import { CsvToJsonRequest, CsvToJsonResponse, FetchTablesResponse, + FieldType, MigrateRequest, MigrateResponse, SaveTableRequest, @@ -178,9 +179,17 @@ export async function validateExistingTableImport( const { rows, tableId } = ctx.request.body let schema = null + if (tableId) { const table = await sdk.tables.getTable(tableId) schema = table.schema + + if (!isExternalTable(table)) { + schema._id = { + name: "_id", + type: FieldType.STRING, + } + } } else { ctx.status = 422 return From 24cdfb3443b28ab34f80ca9a7e6ed0db119a3fc0 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 31 Jul 2024 14:55:25 +0200 Subject: [PATCH 10/15] Fix re-importing --- .../src/api/controllers/table/internal.ts | 23 +++++++-- .../server/src/api/controllers/table/utils.ts | 9 ++-- .../server/src/api/routes/tests/row.spec.ts | 50 +++++++++++++++++++ 3 files changed, 73 insertions(+), 9 deletions(-) diff --git a/packages/server/src/api/controllers/table/internal.ts b/packages/server/src/api/controllers/table/internal.ts index 4286d51d3e..6d1c67e800 100644 --- a/packages/server/src/api/controllers/table/internal.ts +++ b/packages/server/src/api/controllers/table/internal.ts @@ -3,6 +3,7 @@ import { handleDataImport } from "./utils" import { BulkImportRequest, BulkImportResponse, + FieldType, RenameColumn, SaveTableRequest, SaveTableResponse, @@ -69,10 +70,22 @@ export async function bulkImport( ) { const table = await sdk.tables.getTable(ctx.params.tableId) const { rows, identifierFields } = ctx.request.body - await handleDataImport(table, { - importRows: rows, - identifierFields, - user: ctx.user, - }) + await handleDataImport( + { + ...table, + schema: { + _id: { + name: "_id", + type: FieldType.STRING, + }, + ...table.schema, + }, + }, + { + importRows: rows, + identifierFields, + user: ctx.user, + } + ) return table } diff --git a/packages/server/src/api/controllers/table/utils.ts b/packages/server/src/api/controllers/table/utils.ts index e2036c8115..51f7b0e589 100644 --- a/packages/server/src/api/controllers/table/utils.ts +++ b/packages/server/src/api/controllers/table/utils.ts @@ -124,11 +124,12 @@ export async function importToRows( table: Table, user?: ContextUser ) { - let originalTable = table - let finalData: any = [] + const originalTable = table + const finalData: Row[] = [] + const keepCouchId = "_id" in table.schema for (let i = 0; i < data.length; i++) { let row = data[i] - row._id = generateRowID(table._id!) + row._id = (keepCouchId && row._id) || generateRowID(table._id!) row.type = "row" row.tableId = table._id @@ -180,7 +181,7 @@ export async function handleDataImport( const db = context.getAppDB() const data = parse(importRows, table) - let finalData: any = await importToRows(data, table, user) + const finalData = await importToRows(data, table, user) //Set IDs of finalData to match existing row if an update is expected if (identifierFields.length > 0) { diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts index 96a157893f..b448d46e6a 100644 --- a/packages/server/src/api/routes/tests/row.spec.ts +++ b/packages/server/src/api/routes/tests/row.spec.ts @@ -1298,6 +1298,56 @@ describe.each([ await assertRowUsage(isInternal ? rowUsage + 2 : rowUsage) }) + isInternal && + it("should be able to update existing rows on bulkImport", async () => { + const table = await config.api.table.save( + saveTableRequest({ + schema: { + name: { + type: FieldType.STRING, + name: "name", + }, + description: { + type: FieldType.STRING, + name: "description", + }, + }, + }) + ) + + const existingRow = await config.api.row.save(table._id!, { + name: "Existing row", + description: "Existing description", + }) + + + await config.api.row.bulkImport(table._id!, { + rows: [ + { + name: "Row 1", + description: "Row 1 description", + }, + { ...existingRow, name: "Updated existing row" }, + { + name: "Row 2", + description: "Row 2 description", + }, + ], + identifierFields: ["_id"], + }) + + const rows = await config.api.row.fetch(table._id!) + expect(rows.length).toEqual(3) + + rows.sort((a, b) => a.name.localeCompare(b.name)) + expect(rows[0].name).toEqual("Row 1") + expect(rows[0].description).toEqual("Row 1 description") + expect(rows[1].name).toEqual("Row 2") + expect(rows[1].description).toEqual("Row 2 description") + expect(rows[2].name).toEqual("Updated existing row") + expect(rows[2].description).toEqual("Existing description") + }) + // Upserting isn't yet supported in MSSQL, see: // https://github.com/knex/knex/pull/6050 !isMSSQL && From f794f84e9035d92817259d63de39f178d3c94b04 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 31 Jul 2024 15:00:30 +0200 Subject: [PATCH 11/15] Fix quote count --- packages/server/src/api/controllers/table/utils.ts | 6 +++++- packages/server/src/api/routes/tests/row.spec.ts | 3 +++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/server/src/api/controllers/table/utils.ts b/packages/server/src/api/controllers/table/utils.ts index 51f7b0e589..417cb22fe3 100644 --- a/packages/server/src/api/controllers/table/utils.ts +++ b/packages/server/src/api/controllers/table/utils.ts @@ -183,6 +183,8 @@ export async function handleDataImport( const finalData = await importToRows(data, table, user) + let newRowCount = finalData.length + //Set IDs of finalData to match existing row if an update is expected if (identifierFields.length > 0) { const allDocs = await db.allDocs( @@ -204,12 +206,14 @@ export async function handleDataImport( if (match) { finalItem._id = doc._id finalItem._rev = doc._rev + + newRowCount-- } }) }) } - await quotas.addRows(finalData.length, () => db.bulkDocs(finalData), { + await quotas.addRows(newRowCount, () => db.bulkDocs(finalData), { tableId: table._id, }) diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts index b448d46e6a..c4586263f4 100644 --- a/packages/server/src/api/routes/tests/row.spec.ts +++ b/packages/server/src/api/routes/tests/row.spec.ts @@ -1320,6 +1320,7 @@ describe.each([ description: "Existing description", }) + const rowUsage = await getRowUsage() await config.api.row.bulkImport(table._id!, { rows: [ @@ -1346,6 +1347,8 @@ describe.each([ expect(rows[1].description).toEqual("Row 2 description") expect(rows[2].name).toEqual("Updated existing row") expect(rows[2].description).toEqual("Existing description") + + await assertRowUsage(rowUsage + 2) }) // Upserting isn't yet supported in MSSQL, see: From a6beb0fa82f125cf43896d02c04480f7ffca53eb Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 31 Jul 2024 15:14:29 +0200 Subject: [PATCH 12/15] Support no updating existing rows --- .../server/src/api/controllers/table/utils.ts | 9 ++-- .../server/src/api/routes/tests/row.spec.ts | 54 +++++++++++++++++++ 2 files changed, 60 insertions(+), 3 deletions(-) diff --git a/packages/server/src/api/controllers/table/utils.ts b/packages/server/src/api/controllers/table/utils.ts index 417cb22fe3..0e0a83e3b3 100644 --- a/packages/server/src/api/controllers/table/utils.ts +++ b/packages/server/src/api/controllers/table/utils.ts @@ -122,11 +122,12 @@ export function makeSureTableUpToDate(table: Table, tableToSave: Table) { export async function importToRows( data: Row[], table: Table, - user?: ContextUser + user?: ContextUser, + opts?: { keepCouchId: boolean } ) { const originalTable = table const finalData: Row[] = [] - const keepCouchId = "_id" in table.schema + const keepCouchId = !!opts?.keepCouchId for (let i = 0; i < data.length; i++) { let row = data[i] row._id = (keepCouchId && row._id) || generateRowID(table._id!) @@ -181,7 +182,9 @@ export async function handleDataImport( const db = context.getAppDB() const data = parse(importRows, table) - const finalData = await importToRows(data, table, user) + const finalData = await importToRows(data, table, user, { + keepCouchId: identifierFields.includes("_id"), + }) let newRowCount = finalData.length diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts index c4586263f4..edceb925d6 100644 --- a/packages/server/src/api/routes/tests/row.spec.ts +++ b/packages/server/src/api/routes/tests/row.spec.ts @@ -1351,6 +1351,60 @@ describe.each([ await assertRowUsage(rowUsage + 2) }) + isInternal && + it("should create new rows if not identifierFields are provided", async () => { + const table = await config.api.table.save( + saveTableRequest({ + schema: { + name: { + type: FieldType.STRING, + name: "name", + }, + description: { + type: FieldType.STRING, + name: "description", + }, + }, + }) + ) + + const existingRow = await config.api.row.save(table._id!, { + name: "Existing row", + description: "Existing description", + }) + + const rowUsage = await getRowUsage() + + await config.api.row.bulkImport(table._id!, { + rows: [ + { + name: "Row 1", + description: "Row 1 description", + }, + { ...existingRow, name: "Updated existing row" }, + { + name: "Row 2", + description: "Row 2 description", + }, + ], + }) + + const rows = await config.api.row.fetch(table._id!) + expect(rows.length).toEqual(4) + + rows.sort((a, b) => a.name.localeCompare(b.name)) + expect(rows[0].name).toEqual("Existing row") + expect(rows[0].description).toEqual("Existing description") + expect(rows[1].name).toEqual("Row 1") + expect(rows[1].description).toEqual("Row 1 description") + expect(rows[2].name).toEqual("Row 2") + expect(rows[2].description).toEqual("Row 2 description") + expect(rows[3].name).toEqual("Updated existing row") + expect(rows[3].description).toEqual("Existing description") + + await assertRowUsage(rowUsage + 3) + }) + // Upserting isn't yet supported in MSSQL, see: // https://github.com/knex/knex/pull/6050 !isMSSQL && From b74841d99da31ea616f34ab1d5741f4478dd354d Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 31 Jul 2024 15:20:06 +0200 Subject: [PATCH 13/15] Fix --- packages/server/src/db/defaultData/datasource_bb_default.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/server/src/db/defaultData/datasource_bb_default.ts b/packages/server/src/db/defaultData/datasource_bb_default.ts index a393888e51..6b553e2d36 100644 --- a/packages/server/src/db/defaultData/datasource_bb_default.ts +++ b/packages/server/src/db/defaultData/datasource_bb_default.ts @@ -651,10 +651,10 @@ export async function buildDefaultDocs() { return new LinkDocument( employeeData.table._id!, "Jobs", - employeeData.rows[index]._id, + employeeData.rows[index]._id!, jobData.table._id!, "Assigned", - jobData.rows[index]._id + jobData.rows[index]._id! ) } ) From 8539f6d8532673e167d7ffae21725ba0292dc0b9 Mon Sep 17 00:00:00 2001 From: Peter Clement Date: Wed, 31 Jul 2024 22:18:00 +0100 Subject: [PATCH 14/15] Add helper function / builder for creating Automations for tests and improved types (#14220) * basic class for generating and running an automation * change filename * add to existing tests * remove dupe tests * add types to automation steps * add types to triggers * update tests and typing * fix types * typo * move all step schema types do types folder * updated types * typing pr comments * remove unused param * some more typing and tests * more typing * improve type map * fix broken type * this will surely fix my issue --- packages/server/src/automations/actions.ts | 26 +- .../server/src/automations/automationUtils.ts | 9 +- packages/server/src/automations/steps/bash.ts | 11 +- .../server/src/automations/steps/collect.ts | 9 +- .../server/src/automations/steps/createRow.ts | 16 +- .../server/src/automations/steps/delay.ts | 9 +- .../server/src/automations/steps/deleteRow.ts | 14 +- .../server/src/automations/steps/discord.ts | 9 +- .../src/automations/steps/executeQuery.ts | 14 +- .../src/automations/steps/executeScript.ts | 11 +- .../server/src/automations/steps/filter.ts | 9 +- packages/server/src/automations/steps/make.ts | 9 +- packages/server/src/automations/steps/n8n.ts | 9 +- .../server/src/automations/steps/openai.ts | 9 +- .../src/automations/steps/outgoingWebhook.ts | 11 +- .../server/src/automations/steps/queryRows.ts | 13 +- .../src/automations/steps/sendSmtpEmail.ts | 26 +- .../server/src/automations/steps/serverLog.ts | 11 +- .../server/src/automations/steps/slack.ts | 9 +- .../automations/steps/triggerAutomationRun.ts | 13 +- .../server/src/automations/steps/updateRow.ts | 15 +- .../server/src/automations/steps/zapier.ts | 9 +- .../server/src/automations/tests/loop.spec.ts | 4 +- .../tests/scenarios/scenarios.spec.ts | 160 +++++++++ .../tests/utilities/AutomationBuilder.ts | 174 ++++++++++ .../src/automations/tests/utilities/index.ts | 5 +- .../server/src/automations/triggerInfo/app.ts | 8 + .../src/automations/triggerInfo/cron.ts | 8 + .../src/automations/triggerInfo/rowDeleted.ts | 9 + .../src/automations/triggerInfo/rowSaved.ts | 13 + .../src/automations/triggerInfo/rowUpdated.ts | 13 + .../unitTests/automationUtils.spec.ts | 2 +- .../server/src/definitions/automations.ts | 11 +- .../server/src/tests/utilities/structures.ts | 3 +- packages/server/src/threads/automation.ts | 7 +- .../server/src/utilities/workerRequests.ts | 4 +- .../app/{ => automation}/automation.ts | 24 +- .../src/documents/app/automation/index.ts | 2 + .../src/documents/app/automation/schema.ts | 320 ++++++++++++++++++ 39 files changed, 952 insertions(+), 86 deletions(-) create mode 100644 packages/server/src/automations/tests/scenarios/scenarios.spec.ts create mode 100644 packages/server/src/automations/tests/utilities/AutomationBuilder.ts rename packages/types/src/documents/app/{ => automation}/automation.ts (93%) create mode 100644 packages/types/src/documents/app/automation/index.ts create mode 100644 packages/types/src/documents/app/automation/schema.ts diff --git a/packages/server/src/automations/actions.ts b/packages/server/src/automations/actions.ts index eee8ab4a7b..cff269cd80 100644 --- a/packages/server/src/automations/actions.ts +++ b/packages/server/src/automations/actions.ts @@ -20,17 +20,21 @@ import * as triggerAutomationRun from "./steps/triggerAutomationRun" import env from "../environment" import { AutomationStepSchema, - AutomationStepInput, PluginType, AutomationStep, + AutomationActionStepId, + ActionImplementations, + Hosting, + ActionImplementation, } from "@budibase/types" import sdk from "../sdk" import { getAutomationPlugin } from "../utilities/fileSystem" -const ACTION_IMPLS: Record< - string, - (opts: AutomationStepInput) => Promise -> = { +type ActionImplType = ActionImplementations< + typeof env.SELF_HOSTED extends "true" ? Hosting.SELF : Hosting.CLOUD +> + +const ACTION_IMPLS: ActionImplType = { SEND_EMAIL_SMTP: sendSmtpEmail.run, CREATE_ROW: createRow.run, UPDATE_ROW: updateRow.run, @@ -51,6 +55,7 @@ const ACTION_IMPLS: Record< integromat: make.run, n8n: n8n.run, } + export const BUILTIN_ACTION_DEFINITIONS: Record = { SEND_EMAIL_SMTP: sendSmtpEmail.definition, @@ -86,7 +91,7 @@ if (env.SELF_HOSTED) { ACTION_IMPLS["EXECUTE_BASH"] = bash.run // @ts-ignore BUILTIN_ACTION_DEFINITIONS["EXECUTE_BASH"] = bash.definition - + // @ts-ignore ACTION_IMPLS.OPENAI = openai.run BUILTIN_ACTION_DEFINITIONS.OPENAI = openai.definition } @@ -107,10 +112,13 @@ export async function getActionDefinitions() { } /* istanbul ignore next */ -export async function getAction(stepId: string) { - if (ACTION_IMPLS[stepId] != null) { - return ACTION_IMPLS[stepId] +export async function getAction( + stepId: AutomationActionStepId +): Promise | undefined> { + if (ACTION_IMPLS[stepId as keyof ActionImplType] != null) { + return ACTION_IMPLS[stepId as keyof ActionImplType] } + // must be a plugin if (env.SELF_HOSTED) { const plugins = await sdk.plugins.fetch(PluginType.AUTOMATION) diff --git a/packages/server/src/automations/automationUtils.ts b/packages/server/src/automations/automationUtils.ts index bb63be8bce..6e42f8e4bc 100644 --- a/packages/server/src/automations/automationUtils.ts +++ b/packages/server/src/automations/automationUtils.ts @@ -4,8 +4,13 @@ import { encodeJSBinding, } from "@budibase/string-templates" import sdk from "../sdk" -import { AutomationAttachment, FieldType, Row } from "@budibase/types" -import { LoopInput, LoopStepType } from "../definitions/automations" +import { + AutomationAttachment, + FieldType, + Row, + LoopStepType, +} from "@budibase/types" +import { LoopInput } from "../definitions/automations" import { objectStore, context } from "@budibase/backend-core" import * as uuid from "uuid" import path from "path" diff --git a/packages/server/src/automations/steps/bash.ts b/packages/server/src/automations/steps/bash.ts index 1a13f651ec..d33bfb3d6c 100644 --- a/packages/server/src/automations/steps/bash.ts +++ b/packages/server/src/automations/steps/bash.ts @@ -7,9 +7,10 @@ import { AutomationCustomIOType, AutomationFeature, AutomationIOType, - AutomationStepInput, AutomationStepSchema, AutomationStepType, + BashStepInputs, + BashStepOutputs, } from "@budibase/types" export const definition: AutomationStepSchema = { @@ -51,7 +52,13 @@ export const definition: AutomationStepSchema = { }, } -export async function run({ inputs, context }: AutomationStepInput) { +export async function run({ + inputs, + context, +}: { + inputs: BashStepInputs + context: object +}): Promise { if (inputs.code == null) { return { stdout: "Budibase bash automation failed: Invalid inputs", diff --git a/packages/server/src/automations/steps/collect.ts b/packages/server/src/automations/steps/collect.ts index 035bd36a46..2451fd1cf6 100644 --- a/packages/server/src/automations/steps/collect.ts +++ b/packages/server/src/automations/steps/collect.ts @@ -1,9 +1,10 @@ import { AutomationActionStepId, AutomationStepSchema, - AutomationStepInput, AutomationStepType, AutomationIOType, + CollectStepInputs, + CollectStepOutputs, } from "@budibase/types" export const definition: AutomationStepSchema = { @@ -43,7 +44,11 @@ export const definition: AutomationStepSchema = { }, } -export async function run({ inputs }: AutomationStepInput) { +export async function run({ + inputs, +}: { + inputs: CollectStepInputs +}): Promise { if (!inputs.collection) { return { success: false, diff --git a/packages/server/src/automations/steps/createRow.ts b/packages/server/src/automations/steps/createRow.ts index c7f5fcff3b..9908f138b8 100644 --- a/packages/server/src/automations/steps/createRow.ts +++ b/packages/server/src/automations/steps/createRow.ts @@ -10,10 +10,12 @@ import { AutomationCustomIOType, AutomationFeature, AutomationIOType, - AutomationStepInput, AutomationStepSchema, AutomationStepType, + CreateRowStepInputs, + CreateRowStepOutputs, } from "@budibase/types" +import { EventEmitter } from "events" export const definition: AutomationStepSchema = { name: "Create Row", @@ -74,7 +76,15 @@ export const definition: AutomationStepSchema = { }, } -export async function run({ inputs, appId, emitter }: AutomationStepInput) { +export async function run({ + inputs, + appId, + emitter, +}: { + inputs: CreateRowStepInputs + appId: string + emitter: EventEmitter +}): Promise { if (inputs.row == null || inputs.row.tableId == null) { return { success: false, @@ -93,7 +103,7 @@ export async function run({ inputs, appId, emitter }: AutomationStepInput) { try { inputs.row = await cleanUpRow(inputs.row.tableId, inputs.row) inputs.row = await sendAutomationAttachmentsToStorage( - inputs.row.tableId, + inputs.row.tableId!, inputs.row ) await save(ctx) diff --git a/packages/server/src/automations/steps/delay.ts b/packages/server/src/automations/steps/delay.ts index 4e04539998..5392f42b4b 100644 --- a/packages/server/src/automations/steps/delay.ts +++ b/packages/server/src/automations/steps/delay.ts @@ -2,9 +2,10 @@ import { wait } from "../../utilities" import { AutomationActionStepId, AutomationIOType, - AutomationStepInput, AutomationStepSchema, AutomationStepType, + DelayStepInputs, + DelayStepOutputs, } from "@budibase/types" export const definition: AutomationStepSchema = { @@ -39,7 +40,11 @@ export const definition: AutomationStepSchema = { type: AutomationStepType.LOGIC, } -export async function run({ inputs }: AutomationStepInput) { +export async function run({ + inputs, +}: { + inputs: DelayStepInputs +}): Promise { await wait(inputs.time) return { success: true, diff --git a/packages/server/src/automations/steps/deleteRow.ts b/packages/server/src/automations/steps/deleteRow.ts index c8b6585cae..fa26dddb45 100644 --- a/packages/server/src/automations/steps/deleteRow.ts +++ b/packages/server/src/automations/steps/deleteRow.ts @@ -1,14 +1,16 @@ +import { EventEmitter } from "events" import { destroy } from "../../api/controllers/row" import { buildCtx } from "./utils" import { getError } from "../automationUtils" import { AutomationActionStepId, - AutomationStepInput, AutomationStepSchema, AutomationStepType, AutomationIOType, AutomationCustomIOType, AutomationFeature, + DeleteRowStepInputs, + DeleteRowStepOutputs, } from "@budibase/types" export const definition: AutomationStepSchema = { @@ -59,7 +61,15 @@ export const definition: AutomationStepSchema = { }, } -export async function run({ inputs, appId, emitter }: AutomationStepInput) { +export async function run({ + inputs, + appId, + emitter, +}: { + inputs: DeleteRowStepInputs + appId: string + emitter: EventEmitter +}): Promise { if (inputs.id == null) { return { success: false, diff --git a/packages/server/src/automations/steps/discord.ts b/packages/server/src/automations/steps/discord.ts index c80e4ba66f..355f84b987 100644 --- a/packages/server/src/automations/steps/discord.ts +++ b/packages/server/src/automations/steps/discord.ts @@ -3,10 +3,11 @@ import { getFetchResponse } from "./utils" import { AutomationActionStepId, AutomationStepSchema, - AutomationStepInput, AutomationStepType, AutomationIOType, AutomationFeature, + ExternalAppStepOutputs, + DiscordStepInputs, } from "@budibase/types" const DEFAULT_USERNAME = "Budibase Automate" @@ -65,7 +66,11 @@ export const definition: AutomationStepSchema = { }, } -export async function run({ inputs }: AutomationStepInput) { +export async function run({ + inputs, +}: { + inputs: DiscordStepInputs +}): Promise { let { url, username, avatar_url, content } = inputs if (!username) { username = DEFAULT_USERNAME diff --git a/packages/server/src/automations/steps/executeQuery.ts b/packages/server/src/automations/steps/executeQuery.ts index a9517b01a0..eb033b8883 100644 --- a/packages/server/src/automations/steps/executeQuery.ts +++ b/packages/server/src/automations/steps/executeQuery.ts @@ -1,3 +1,4 @@ +import { EventEmitter } from "events" import * as queryController from "../../api/controllers/query" import { buildCtx } from "./utils" import * as automationUtils from "../automationUtils" @@ -6,9 +7,10 @@ import { AutomationCustomIOType, AutomationFeature, AutomationIOType, - AutomationStepInput, AutomationStepSchema, AutomationStepType, + ExecuteQueryStepInputs, + ExecuteQueryStepOutputs, } from "@budibase/types" export const definition: AutomationStepSchema = { @@ -62,7 +64,15 @@ export const definition: AutomationStepSchema = { }, } -export async function run({ inputs, appId, emitter }: AutomationStepInput) { +export async function run({ + inputs, + appId, + emitter, +}: { + inputs: ExecuteQueryStepInputs + appId: string + emitter: EventEmitter +}): Promise { if (inputs.query == null) { return { success: false, diff --git a/packages/server/src/automations/steps/executeScript.ts b/packages/server/src/automations/steps/executeScript.ts index 7ce6c1d0f4..3962da53ca 100644 --- a/packages/server/src/automations/steps/executeScript.ts +++ b/packages/server/src/automations/steps/executeScript.ts @@ -6,10 +6,12 @@ import { AutomationCustomIOType, AutomationFeature, AutomationIOType, - AutomationStepInput, AutomationStepSchema, AutomationStepType, + ExecuteScriptStepInputs, + ExecuteScriptStepOutputs, } from "@budibase/types" +import { EventEmitter } from "events" export const definition: AutomationStepSchema = { name: "JS Scripting", @@ -55,7 +57,12 @@ export async function run({ appId, context, emitter, -}: AutomationStepInput) { +}: { + inputs: ExecuteScriptStepInputs + appId: string + context: object + emitter: EventEmitter +}): Promise { if (inputs.code == null) { return { success: false, diff --git a/packages/server/src/automations/steps/filter.ts b/packages/server/src/automations/steps/filter.ts index 624619bb95..6d35a72698 100644 --- a/packages/server/src/automations/steps/filter.ts +++ b/packages/server/src/automations/steps/filter.ts @@ -1,9 +1,10 @@ import { AutomationActionStepId, AutomationStepSchema, - AutomationStepInput, AutomationStepType, AutomationIOType, + FilterStepInputs, + FilterStepOutputs, } from "@budibase/types" export const FilterConditions = { @@ -69,7 +70,11 @@ export const definition: AutomationStepSchema = { }, } -export async function run({ inputs }: AutomationStepInput) { +export async function run({ + inputs, +}: { + inputs: FilterStepInputs +}): Promise { try { let { field, condition, value } = inputs // coerce types so that we can use them diff --git a/packages/server/src/automations/steps/make.ts b/packages/server/src/automations/steps/make.ts index 555df8308a..45e31fbaa2 100644 --- a/packages/server/src/automations/steps/make.ts +++ b/packages/server/src/automations/steps/make.ts @@ -3,10 +3,11 @@ import { getFetchResponse } from "./utils" import { AutomationActionStepId, AutomationStepSchema, - AutomationStepInput, AutomationStepType, AutomationIOType, AutomationFeature, + ExternalAppStepOutputs, + MakeIntegrationInputs, } from "@budibase/types" export const definition: AutomationStepSchema = { @@ -57,7 +58,11 @@ export const definition: AutomationStepSchema = { }, } -export async function run({ inputs }: AutomationStepInput) { +export async function run({ + inputs, +}: { + inputs: MakeIntegrationInputs +}): Promise { const { url, body } = inputs let payload = {} diff --git a/packages/server/src/automations/steps/n8n.ts b/packages/server/src/automations/steps/n8n.ts index c400c7037a..b2fbce6de7 100644 --- a/packages/server/src/automations/steps/n8n.ts +++ b/packages/server/src/automations/steps/n8n.ts @@ -3,11 +3,12 @@ import { getFetchResponse } from "./utils" import { AutomationActionStepId, AutomationStepSchema, - AutomationStepInput, AutomationStepType, AutomationIOType, AutomationFeature, HttpMethod, + ExternalAppStepOutputs, + n8nStepInputs, } from "@budibase/types" export const definition: AutomationStepSchema = { @@ -67,7 +68,11 @@ export const definition: AutomationStepSchema = { }, } -export async function run({ inputs }: AutomationStepInput) { +export async function run({ + inputs, +}: { + inputs: n8nStepInputs +}): Promise { const { url, body, method, authorization } = inputs let payload = {} diff --git a/packages/server/src/automations/steps/openai.ts b/packages/server/src/automations/steps/openai.ts index 380a6b9536..279a0a9df0 100644 --- a/packages/server/src/automations/steps/openai.ts +++ b/packages/server/src/automations/steps/openai.ts @@ -3,9 +3,10 @@ import { OpenAI } from "openai" import { AutomationActionStepId, AutomationStepSchema, - AutomationStepInput, AutomationStepType, AutomationIOType, + OpenAIStepInputs, + OpenAIStepOutputs, } from "@budibase/types" import { env } from "@budibase/backend-core" import * as automationUtils from "../automationUtils" @@ -59,7 +60,11 @@ export const definition: AutomationStepSchema = { }, } -export async function run({ inputs }: AutomationStepInput) { +export async function run({ + inputs, +}: { + inputs: OpenAIStepInputs +}): Promise { if (!env.OPENAI_API_KEY) { return { success: false, diff --git a/packages/server/src/automations/steps/outgoingWebhook.ts b/packages/server/src/automations/steps/outgoingWebhook.ts index 269c8c7157..04972fefae 100644 --- a/packages/server/src/automations/steps/outgoingWebhook.ts +++ b/packages/server/src/automations/steps/outgoingWebhook.ts @@ -6,9 +6,10 @@ import { AutomationCustomIOType, AutomationFeature, AutomationIOType, - AutomationStepInput, AutomationStepSchema, AutomationStepType, + ExternalAppStepOutputs, + OutgoingWebhookStepInputs, } from "@budibase/types" enum RequestType { @@ -88,7 +89,13 @@ export const definition: AutomationStepSchema = { }, } -export async function run({ inputs }: AutomationStepInput) { +export async function run({ + inputs, +}: { + inputs: OutgoingWebhookStepInputs +}): Promise< + Omit | ExternalAppStepOutputs +> { let { requestMethod, url, requestBody, headers } = inputs if (!url.startsWith("http")) { url = `http://${url}` diff --git a/packages/server/src/automations/steps/queryRows.ts b/packages/server/src/automations/steps/queryRows.ts index aa3efa5425..172bbf7f55 100644 --- a/packages/server/src/automations/steps/queryRows.ts +++ b/packages/server/src/automations/steps/queryRows.ts @@ -8,13 +8,14 @@ import { AutomationCustomIOType, AutomationFeature, AutomationIOType, - AutomationStepInput, AutomationStepSchema, AutomationStepType, EmptyFilterOption, SearchFilters, Table, SortOrder, + QueryRowsStepInputs, + QueryRowsStepOutputs, } from "@budibase/types" import { db as dbCore } from "@budibase/backend-core" @@ -133,7 +134,13 @@ function hasNullFilters(filters: any[]) { ) } -export async function run({ inputs, appId }: AutomationStepInput) { +export async function run({ + inputs, + appId, +}: { + inputs: QueryRowsStepInputs + appId: string +}): Promise { const { tableId, filters, sortColumn, sortOrder, limit } = inputs if (!tableId) { return { @@ -145,7 +152,7 @@ export async function run({ inputs, appId }: AutomationStepInput) { } const table = await getTable(appId, tableId) let sortType = FieldType.STRING - if (table && table.schema && table.schema[sortColumn] && sortColumn) { + if (sortColumn && table && table.schema && table.schema[sortColumn]) { const fieldType = table.schema[sortColumn].type sortType = fieldType === FieldType.NUMBER ? FieldType.NUMBER : FieldType.STRING diff --git a/packages/server/src/automations/steps/sendSmtpEmail.ts b/packages/server/src/automations/steps/sendSmtpEmail.ts index bcb1699c6b..4950bfb3f3 100644 --- a/packages/server/src/automations/steps/sendSmtpEmail.ts +++ b/packages/server/src/automations/steps/sendSmtpEmail.ts @@ -3,11 +3,12 @@ import * as automationUtils from "../automationUtils" import { AutomationActionStepId, AutomationStepSchema, - AutomationStepInput, AutomationStepType, AutomationIOType, AutomationFeature, AutomationCustomIOType, + SmtpEmailStepInputs, + BaseAutomationOutputs, } from "@budibase/types" export const definition: AutomationStepSchema = { @@ -97,7 +98,11 @@ export const definition: AutomationStepSchema = { }, } -export async function run({ inputs }: AutomationStepInput) { +export async function run({ + inputs, +}: { + inputs: SmtpEmailStepInputs +}): Promise { let { to, from, @@ -116,17 +121,16 @@ export async function run({ inputs }: AutomationStepInput) { if (!contents) { contents = "

No content

" } - to = to || undefined - - if (attachments) { - if (Array.isArray(attachments)) { - attachments.forEach(item => automationUtils.guardAttachment(item)) - } else { - automationUtils.guardAttachment(attachments) - } - } try { + if (attachments) { + if (Array.isArray(attachments)) { + attachments.forEach(item => automationUtils.guardAttachment(item)) + } else { + automationUtils.guardAttachment(attachments) + } + } + let response = await sendSmtpEmail({ to, from, diff --git a/packages/server/src/automations/steps/serverLog.ts b/packages/server/src/automations/steps/serverLog.ts index eb75ca1f3b..482325b744 100644 --- a/packages/server/src/automations/steps/serverLog.ts +++ b/packages/server/src/automations/steps/serverLog.ts @@ -1,10 +1,11 @@ import { AutomationActionStepId, AutomationStepSchema, - AutomationStepInput, AutomationStepType, AutomationIOType, AutomationFeature, + ServerLogStepInputs, + ServerLogStepOutputs, } from "@budibase/types" /** @@ -53,7 +54,13 @@ export const definition: AutomationStepSchema = { }, } -export async function run({ inputs, appId }: AutomationStepInput) { +export async function run({ + inputs, + appId, +}: { + inputs: ServerLogStepInputs + appId: string +}): Promise { const message = `App ${appId} - ${inputs.text}` console.log(message) return { diff --git a/packages/server/src/automations/steps/slack.ts b/packages/server/src/automations/steps/slack.ts index 79544bf001..3ed462796b 100644 --- a/packages/server/src/automations/steps/slack.ts +++ b/packages/server/src/automations/steps/slack.ts @@ -3,10 +3,11 @@ import { getFetchResponse } from "./utils" import { AutomationActionStepId, AutomationStepSchema, - AutomationStepInput, AutomationStepType, AutomationIOType, AutomationFeature, + ExternalAppStepOutputs, + SlackStepInputs, } from "@budibase/types" export const definition: AutomationStepSchema = { @@ -54,7 +55,11 @@ export const definition: AutomationStepSchema = { }, } -export async function run({ inputs }: AutomationStepInput) { +export async function run({ + inputs, +}: { + inputs: SlackStepInputs +}): Promise { let { url, text } = inputs if (!url?.trim()?.length) { return { diff --git a/packages/server/src/automations/steps/triggerAutomationRun.ts b/packages/server/src/automations/steps/triggerAutomationRun.ts index 83e1722877..cc73200ab3 100644 --- a/packages/server/src/automations/steps/triggerAutomationRun.ts +++ b/packages/server/src/automations/steps/triggerAutomationRun.ts @@ -1,12 +1,13 @@ import { AutomationActionStepId, AutomationStepSchema, - AutomationStepInput, AutomationStepType, AutomationIOType, AutomationResults, Automation, AutomationCustomIOType, + TriggerAutomationStepInputs, + TriggerAutomationStepOutputs, } from "@budibase/types" import * as triggers from "../triggers" import { context } from "@budibase/backend-core" @@ -61,7 +62,11 @@ export const definition: AutomationStepSchema = { }, } -export async function run({ inputs }: AutomationStepInput) { +export async function run({ + inputs, +}: { + inputs: TriggerAutomationStepInputs +}): Promise { const { automationId, ...fieldParams } = inputs.automation if (await features.isTriggerAutomationRunEnabled()) { @@ -88,5 +93,9 @@ export async function run({ inputs }: AutomationStepInput) { value: response.steps, } } + } else { + return { + success: false, + } } } diff --git a/packages/server/src/automations/steps/updateRow.ts b/packages/server/src/automations/steps/updateRow.ts index c1e7e286ce..a029fb7413 100644 --- a/packages/server/src/automations/steps/updateRow.ts +++ b/packages/server/src/automations/steps/updateRow.ts @@ -1,3 +1,4 @@ +import { EventEmitter } from "events" import * as rowController from "../../api/controllers/row" import * as automationUtils from "../automationUtils" import { buildCtx } from "./utils" @@ -6,9 +7,10 @@ import { AutomationCustomIOType, AutomationFeature, AutomationIOType, - AutomationStepInput, AutomationStepSchema, AutomationStepType, + UpdateRowStepInputs, + UpdateRowStepOutputs, } from "@budibase/types" export const definition: AutomationStepSchema = { @@ -70,8 +72,15 @@ export const definition: AutomationStepSchema = { }, }, } - -export async function run({ inputs, appId, emitter }: AutomationStepInput) { +export async function run({ + inputs, + appId, + emitter, +}: { + inputs: UpdateRowStepInputs + appId: string + emitter: EventEmitter +}): Promise { if (inputs.rowId == null || inputs.row == null) { return { success: false, diff --git a/packages/server/src/automations/steps/zapier.ts b/packages/server/src/automations/steps/zapier.ts index e48d677228..6de94b0486 100644 --- a/packages/server/src/automations/steps/zapier.ts +++ b/packages/server/src/automations/steps/zapier.ts @@ -3,10 +3,11 @@ import { getFetchResponse } from "./utils" import { AutomationActionStepId, AutomationStepSchema, - AutomationStepInput, AutomationStepType, AutomationIOType, AutomationFeature, + ZapierStepInputs, + ZapierStepOutputs, } from "@budibase/types" export const definition: AutomationStepSchema = { @@ -50,7 +51,11 @@ export const definition: AutomationStepSchema = { }, } -export async function run({ inputs }: AutomationStepInput) { +export async function run({ + inputs, +}: { + inputs: ZapierStepInputs +}): Promise { const { url, body } = inputs let payload = {} diff --git a/packages/server/src/automations/tests/loop.spec.ts b/packages/server/src/automations/tests/loop.spec.ts index ba237d3253..372c3855b3 100644 --- a/packages/server/src/automations/tests/loop.spec.ts +++ b/packages/server/src/automations/tests/loop.spec.ts @@ -3,9 +3,9 @@ import * as triggers from "../triggers" import { loopAutomation } from "../../tests/utilities/structures" import { context } from "@budibase/backend-core" import * as setup from "./utilities" -import { Table } from "@budibase/types" +import { Table, LoopStepType } from "@budibase/types" import * as loopUtils from "../loopUtils" -import { LoopInput, LoopStepType } from "../../definitions/automations" +import { LoopInput } from "../../definitions/automations" describe("Attempt to run a basic loop automation", () => { let config = setup.getConfig(), diff --git a/packages/server/src/automations/tests/scenarios/scenarios.spec.ts b/packages/server/src/automations/tests/scenarios/scenarios.spec.ts new file mode 100644 index 0000000000..4f4ad70814 --- /dev/null +++ b/packages/server/src/automations/tests/scenarios/scenarios.spec.ts @@ -0,0 +1,160 @@ +import * as automation from "../../index" +import * as setup from "../utilities" +import { Table, LoopStepType } from "@budibase/types" +import { createAutomationBuilder } from "../utilities/AutomationBuilder" + +describe("Automation Scenarios", () => { + let config = setup.getConfig(), + table: Table + + beforeEach(async () => { + await automation.init() + await config.init() + table = await config.createTable() + await config.createRow() + }) + + afterAll(setup.afterAll) + + describe("Loop automations", () => { + it("should run an automation with a trigger, loop, and create row step", async () => { + const builder = createAutomationBuilder({ + name: "Test Trigger with Loop and Create Row", + }) + + const results = await builder + .rowSaved( + { tableId: table._id! }, + { + row: { + name: "Trigger Row", + description: "This row triggers the automation", + }, + id: "1234", + revision: "1", + } + ) + .loop({ + option: LoopStepType.ARRAY, + binding: [1, 2, 3], + }) + .createRow({ + row: { + name: "Item {{ loop.currentItem }}", + description: "Created from loop", + tableId: table._id, + }, + }) + .run() + + expect(results.trigger).toBeDefined() + expect(results.steps).toHaveLength(1) + + expect(results.steps[0].outputs.iterations).toBe(3) + expect(results.steps[0].outputs.items).toHaveLength(3) + + results.steps[0].outputs.items.forEach((output: any, index: number) => { + expect(output).toMatchObject({ + success: true, + row: { + name: `Item ${index + 1}`, + description: "Created from loop", + }, + }) + }) + }) + }) + + describe("Row Automations", () => { + it("should trigger an automation which then creates a row", async () => { + const table = await config.createTable() + + const builder = createAutomationBuilder({ + name: "Test Row Save and Create", + }) + + const results = await builder + .rowUpdated( + { tableId: table._id! }, + { + row: { name: "Test", description: "TEST" }, + id: "1234", + } + ) + .createRow({ + row: { + name: "{{trigger.row.name}}", + description: "{{trigger.row.description}}", + tableId: table._id, + }, + }) + .run() + + expect(results.steps).toHaveLength(1) + + expect(results.steps[0].outputs).toMatchObject({ + success: true, + row: { + name: "Test", + description: "TEST", + }, + }) + }) + }) + + it("should trigger an automation which querys the database", async () => { + const table = await config.createTable() + const row = { + name: "Test Row", + description: "original description", + tableId: table._id, + } + await config.createRow(row) + await config.createRow(row) + const builder = createAutomationBuilder({ + name: "Test Row Save and Create", + }) + + const results = await builder + .appAction({ fields: {} }) + .queryRows({ + tableId: table._id!, + }) + .run() + + expect(results.steps).toHaveLength(1) + expect(results.steps[0].outputs.rows).toHaveLength(2) + }) + + it("should trigger an automation which querys the database then deletes a row", async () => { + const table = await config.createTable() + const row = { + name: "DFN", + description: "original description", + tableId: table._id, + } + await config.createRow(row) + await config.createRow(row) + const builder = createAutomationBuilder({ + name: "Test Row Save and Create", + }) + + const results = await builder + .appAction({ fields: {} }) + .queryRows({ + tableId: table._id!, + }) + .deleteRow({ + tableId: table._id!, + id: "{{ steps.1.rows.0._id }}", + }) + .queryRows({ + tableId: table._id!, + }) + .run() + + expect(results.steps).toHaveLength(3) + expect(results.steps[1].outputs.success).toBeTruthy() + expect(results.steps[2].outputs.rows).toHaveLength(1) + }) +}) diff --git a/packages/server/src/automations/tests/utilities/AutomationBuilder.ts b/packages/server/src/automations/tests/utilities/AutomationBuilder.ts new file mode 100644 index 0000000000..49f60795d1 --- /dev/null +++ b/packages/server/src/automations/tests/utilities/AutomationBuilder.ts @@ -0,0 +1,174 @@ +import { v4 as uuidv4 } from "uuid" +import { testAutomation } from "../../../api/routes/tests/utilities/TestFunctions" +import { + RowCreatedTriggerInputs, + RowCreatedTriggerOutputs, +} from "../../triggerInfo/rowSaved" +import { + RowUpdatedTriggerInputs, + RowUpdatedTriggerOutputs, +} from "../../triggerInfo/rowUpdated" +import {} from "../../steps/createRow" +import { BUILTIN_ACTION_DEFINITIONS } from "../../actions" +import { TRIGGER_DEFINITIONS } from "../../triggers" +import { + RowDeletedTriggerInputs, + RowDeletedTriggerOutputs, +} from "../../triggerInfo/rowDeleted" +import { + AutomationStepSchema, + AutomationTriggerSchema, + LoopStepInputs, + DeleteRowStepInputs, + UpdateRowStepInputs, + CreateRowStepInputs, + Automation, + AutomationTrigger, + AutomationResults, + SmtpEmailStepInputs, + ExecuteQueryStepInputs, + QueryRowsStepInputs, +} from "@budibase/types" +import {} from "../../steps/loop" +import TestConfiguration from "../../../tests/utilities/TestConfiguration" +import * as setup from "../utilities" +import { + AppActionTriggerInputs, + AppActionTriggerOutputs, +} from "../../triggerInfo/app" +import { CronTriggerOutputs } from "../../triggerInfo/cron" + +type TriggerOutputs = + | RowCreatedTriggerOutputs + | RowUpdatedTriggerOutputs + | RowDeletedTriggerOutputs + | AppActionTriggerOutputs + | CronTriggerOutputs + | undefined + +class AutomationBuilder { + private automationConfig: Automation = { + name: "", + definition: { + steps: [], + trigger: {} as AutomationTrigger, + }, + type: "automation", + appId: setup.getConfig().getAppId(), + } + private config: TestConfiguration = setup.getConfig() + private triggerOutputs: TriggerOutputs + private triggerSet: boolean = false + + constructor(options: { name?: string } = {}) { + this.automationConfig.name = options.name || `Test Automation ${uuidv4()}` + } + + // TRIGGERS + rowSaved(inputs: RowCreatedTriggerInputs, outputs: RowCreatedTriggerOutputs) { + this.triggerOutputs = outputs + return this.trigger(TRIGGER_DEFINITIONS.ROW_SAVED, inputs, outputs) + } + + rowUpdated( + inputs: RowUpdatedTriggerInputs, + outputs: RowUpdatedTriggerOutputs + ) { + this.triggerOutputs = outputs + return this.trigger(TRIGGER_DEFINITIONS.ROW_UPDATED, inputs, outputs) + } + + rowDeleted( + inputs: RowDeletedTriggerInputs, + outputs: RowDeletedTriggerOutputs + ) { + this.triggerOutputs = outputs + return this.trigger(TRIGGER_DEFINITIONS.ROW_DELETED, inputs, outputs) + } + + appAction(outputs: AppActionTriggerOutputs, inputs?: AppActionTriggerInputs) { + this.triggerOutputs = outputs + return this.trigger(TRIGGER_DEFINITIONS.APP, inputs, outputs) + } + + // STEPS + createRow(inputs: CreateRowStepInputs): this { + return this.step(BUILTIN_ACTION_DEFINITIONS.CREATE_ROW, inputs) + } + + updateRow(inputs: UpdateRowStepInputs): this { + return this.step(BUILTIN_ACTION_DEFINITIONS.UPDATE_ROW, inputs) + } + + deleteRow(inputs: DeleteRowStepInputs): this { + return this.step(BUILTIN_ACTION_DEFINITIONS.DELETE_ROW, inputs) + } + + sendSmtpEmail(inputs: SmtpEmailStepInputs): this { + return this.step(BUILTIN_ACTION_DEFINITIONS.SEND_EMAIL_SMTP, inputs) + } + + executeQuery(inputs: ExecuteQueryStepInputs): this { + return this.step(BUILTIN_ACTION_DEFINITIONS.EXECUTE_QUERY, inputs) + } + + queryRows(inputs: QueryRowsStepInputs): this { + return this.step(BUILTIN_ACTION_DEFINITIONS.QUERY_ROWS, inputs) + } + loop(inputs: LoopStepInputs): this { + return this.step(BUILTIN_ACTION_DEFINITIONS.LOOP, inputs) + } + + private trigger( + triggerSchema: AutomationTriggerSchema, + inputs?: T, + outputs?: TriggerOutputs + ): this { + if (this.triggerSet) { + throw new Error("Only one trigger can be set for an automation.") + } + this.automationConfig.definition.trigger = { + ...triggerSchema, + inputs: inputs || {}, + id: uuidv4(), + } + this.triggerOutputs = outputs + this.triggerSet = true + + return this + } + + private step( + stepSchema: AutomationStepSchema, + inputs: T + ): this { + this.automationConfig.definition.steps.push({ + ...stepSchema, + inputs, + id: uuidv4(), + }) + return this + } + + async run() { + const automation = await this.config.createAutomation(this.automationConfig) + const results = await testAutomation( + this.config, + automation, + this.triggerOutputs + ) + return this.processResults(results) + } + + private processResults(results: { body: AutomationResults }) { + results.body.steps.shift() + return { + trigger: results.body.trigger, + steps: results.body.steps, + } + } +} + +export function createAutomationBuilder(options?: { name?: string }) { + return new AutomationBuilder(options) +} diff --git a/packages/server/src/automations/tests/utilities/index.ts b/packages/server/src/automations/tests/utilities/index.ts index cd3ea289ca..7952f7a80b 100644 --- a/packages/server/src/automations/tests/utilities/index.ts +++ b/packages/server/src/automations/tests/utilities/index.ts @@ -3,6 +3,7 @@ import { context } from "@budibase/backend-core" import { BUILTIN_ACTION_DEFINITIONS, getAction } from "../../actions" import emitter from "../../../events/index" import env from "../../../environment" +import { AutomationActionStepId } from "@budibase/types" let config: TestConfig @@ -33,7 +34,7 @@ export async function runInProd(fn: any) { export async function runStep(stepId: string, inputs: any, stepContext?: any) { async function run() { - let step = await getAction(stepId) + let step = await getAction(stepId as AutomationActionStepId) expect(step).toBeDefined() if (!step) { throw new Error("No step found") @@ -41,7 +42,7 @@ export async function runStep(stepId: string, inputs: any, stepContext?: any) { return step({ context: stepContext || {}, inputs, - appId: config ? config.getAppId() : null, + appId: config ? config.getAppId() : "", // don't really need an API key, mocked out usage quota, not being tested here apiKey, emitter, diff --git a/packages/server/src/automations/triggerInfo/app.ts b/packages/server/src/automations/triggerInfo/app.ts index bfd284cc53..c1945eb23d 100644 --- a/packages/server/src/automations/triggerInfo/app.ts +++ b/packages/server/src/automations/triggerInfo/app.ts @@ -39,3 +39,11 @@ export const definition: AutomationTriggerSchema = { }, type: AutomationStepType.TRIGGER, } + +export type AppActionTriggerInputs = { + fields: object +} + +export type AppActionTriggerOutputs = { + fields: object +} diff --git a/packages/server/src/automations/triggerInfo/cron.ts b/packages/server/src/automations/triggerInfo/cron.ts index be4b60cb27..781c1a8708 100644 --- a/packages/server/src/automations/triggerInfo/cron.ts +++ b/packages/server/src/automations/triggerInfo/cron.ts @@ -38,3 +38,11 @@ export const definition: AutomationTriggerSchema = { }, type: AutomationStepType.TRIGGER, } + +export type CronTriggerInputs = { + cron: string +} + +export type CronTriggerOutputs = { + timestamp: number +} diff --git a/packages/server/src/automations/triggerInfo/rowDeleted.ts b/packages/server/src/automations/triggerInfo/rowDeleted.ts index 06e53ce63f..0ebf908ec1 100644 --- a/packages/server/src/automations/triggerInfo/rowDeleted.ts +++ b/packages/server/src/automations/triggerInfo/rowDeleted.ts @@ -5,6 +5,7 @@ import { AutomationTriggerSchema, AutomationTriggerStepId, AutomationEventType, + Row, } from "@budibase/types" export const definition: AutomationTriggerSchema = { @@ -39,3 +40,11 @@ export const definition: AutomationTriggerSchema = { }, type: AutomationStepType.TRIGGER, } + +export type RowDeletedTriggerInputs = { + tableId: string +} + +export type RowDeletedTriggerOutputs = { + row: Row +} diff --git a/packages/server/src/automations/triggerInfo/rowSaved.ts b/packages/server/src/automations/triggerInfo/rowSaved.ts index d128934dcc..93f036d13a 100644 --- a/packages/server/src/automations/triggerInfo/rowSaved.ts +++ b/packages/server/src/automations/triggerInfo/rowSaved.ts @@ -5,7 +5,9 @@ import { AutomationTriggerSchema, AutomationTriggerStepId, AutomationEventType, + Row, } from "@budibase/types" +import { SearchFilters } from "aws-sdk/clients/elasticbeanstalk" export const definition: AutomationTriggerSchema = { name: "Row Created", @@ -52,3 +54,14 @@ export const definition: AutomationTriggerSchema = { }, type: AutomationStepType.TRIGGER, } + +export type RowCreatedTriggerInputs = { + tableId: string + filters?: SearchFilters +} + +export type RowCreatedTriggerOutputs = { + row: Row + id: string + revision: string +} diff --git a/packages/server/src/automations/triggerInfo/rowUpdated.ts b/packages/server/src/automations/triggerInfo/rowUpdated.ts index f6aefd1464..148a0bd3f3 100644 --- a/packages/server/src/automations/triggerInfo/rowUpdated.ts +++ b/packages/server/src/automations/triggerInfo/rowUpdated.ts @@ -5,6 +5,8 @@ import { AutomationTriggerSchema, AutomationTriggerStepId, AutomationEventType, + Row, + SearchFilters, } from "@budibase/types" export const definition: AutomationTriggerSchema = { @@ -59,3 +61,14 @@ export const definition: AutomationTriggerSchema = { }, type: AutomationStepType.TRIGGER, } + +export type RowUpdatedTriggerInputs = { + tableId: string + filters?: SearchFilters +} + +export type RowUpdatedTriggerOutputs = { + row: Row + id: string + revision?: string +} diff --git a/packages/server/src/automations/unitTests/automationUtils.spec.ts b/packages/server/src/automations/unitTests/automationUtils.spec.ts index afb6219ef2..7706dab4cc 100644 --- a/packages/server/src/automations/unitTests/automationUtils.spec.ts +++ b/packages/server/src/automations/unitTests/automationUtils.spec.ts @@ -1,9 +1,9 @@ -import { LoopStepType } from "../../definitions/automations" import { typecastForLooping, cleanInputValues, substituteLoopStep, } from "../automationUtils" +import { LoopStepType } from "@budibase/types" describe("automationUtils", () => { describe("substituteLoopStep", () => { diff --git a/packages/server/src/definitions/automations.ts b/packages/server/src/definitions/automations.ts index c205149a5b..2a51c737f2 100644 --- a/packages/server/src/definitions/automations.ts +++ b/packages/server/src/definitions/automations.ts @@ -1,9 +1,8 @@ -import { AutomationResults, AutomationStep } from "@budibase/types" - -export enum LoopStepType { - ARRAY = "Array", - STRING = "String", -} +import { + AutomationResults, + AutomationStep, + LoopStepType, +} from "@budibase/types" export interface LoopStep extends AutomationStep { inputs: LoopInput diff --git a/packages/server/src/tests/utilities/structures.ts b/packages/server/src/tests/utilities/structures.ts index 16ab049eb4..8f67ad1af9 100644 --- a/packages/server/src/tests/utilities/structures.ts +++ b/packages/server/src/tests/utilities/structures.ts @@ -25,8 +25,9 @@ import { Webhook, WebhookActionType, AutomationEventType, + LoopStepType, } from "@budibase/types" -import { LoopInput, LoopStepType } from "../../definitions/automations" +import { LoopInput } from "../../definitions/automations" import { merge } from "lodash" import { generator } from "@budibase/backend-core/tests" diff --git a/packages/server/src/threads/automation.ts b/packages/server/src/threads/automation.ts index a7cf71de4b..2e95542229 100644 --- a/packages/server/src/threads/automation.ts +++ b/packages/server/src/threads/automation.ts @@ -16,6 +16,7 @@ import { AutomationErrors, MAX_AUTOMATION_RECURRING_ERRORS } from "../constants" import { storeLog } from "../automations/logging" import { Automation, + AutomationActionStepId, AutomationData, AutomationJob, AutomationMetadata, @@ -108,7 +109,7 @@ class Orchestrator { return triggerOutput } - async getStepFunctionality(stepId: string) { + async getStepFunctionality(stepId: AutomationActionStepId) { let step = await actions.getAction(stepId) if (step == null) { throw `Cannot find automation step by name ${stepId}` @@ -422,7 +423,9 @@ class Orchestrator { continue } - let stepFn = await this.getStepFunctionality(step.stepId) + let stepFn = await this.getStepFunctionality( + step.stepId as AutomationActionStepId + ) let inputs = await processObject(originalStepInput, this._context) inputs = automationUtils.cleanInputValues( inputs, diff --git a/packages/server/src/utilities/workerRequests.ts b/packages/server/src/utilities/workerRequests.ts index 474f857b0a..0f487d9f31 100644 --- a/packages/server/src/utilities/workerRequests.ts +++ b/packages/server/src/utilities/workerRequests.ts @@ -103,8 +103,8 @@ export async function sendSmtpEmail({ from: string subject: string contents: string - cc: string - bcc: string + cc?: string + bcc?: string automation: boolean attachments?: EmailAttachment[] invite?: EmailInvite diff --git a/packages/types/src/documents/app/automation.ts b/packages/types/src/documents/app/automation/automation.ts similarity index 93% rename from packages/types/src/documents/app/automation.ts rename to packages/types/src/documents/app/automation/automation.ts index d5d7fe667c..d8fad4c8e8 100644 --- a/packages/types/src/documents/app/automation.ts +++ b/packages/types/src/documents/app/automation/automation.ts @@ -1,9 +1,9 @@ -import { Document } from "../document" +import { Document } from "../../document" import { EventEmitter } from "events" -import { User } from "../global" +import { User } from "../../global" import { ReadStream } from "fs" -import { Row } from "./row" -import { Table } from "./table" +import { Row } from "../row" +import { Table } from "../table" export enum AutomationIOType { OBJECT = "object", @@ -93,6 +93,7 @@ export interface EmailAttachment { } export interface SendEmailOpts { + to?: string // workspaceId If finer grain controls being used then this will lookup config for workspace. workspaceId?: string // user If sending to an existing user the object can be provided, this is used in the context. @@ -102,7 +103,7 @@ export interface SendEmailOpts { // contents If sending a custom email then can supply contents which will be added to it. contents?: string // subject A custom subject can be specified if the config one is not desired. - subject?: string + subject: string // info Pass in a structure of information to be stored alongside the invitation. info?: any cc?: boolean @@ -242,14 +243,18 @@ export interface AutomationLogPage { nextPage?: string } -export type AutomationStepInput = { - inputs: Record +export interface AutomationStepInputBase { context: Record emitter: EventEmitter appId: string apiKey?: string } +export type ActionImplementation = ( + params: { + inputs: TInputs + } & AutomationStepInputBase +) => Promise export interface AutomationMetadata extends Document { errorCount?: number automationChainCount?: number @@ -286,3 +291,8 @@ export type UpdatedRowEventEmitter = { table: Table appId: string } + +export enum LoopStepType { + ARRAY = "Array", + STRING = "String", +} diff --git a/packages/types/src/documents/app/automation/index.ts b/packages/types/src/documents/app/automation/index.ts new file mode 100644 index 0000000000..017596670d --- /dev/null +++ b/packages/types/src/documents/app/automation/index.ts @@ -0,0 +1,2 @@ +export * from "./automation" +export * from "./schema" diff --git a/packages/types/src/documents/app/automation/schema.ts b/packages/types/src/documents/app/automation/schema.ts new file mode 100644 index 0000000000..0da82f2f6e --- /dev/null +++ b/packages/types/src/documents/app/automation/schema.ts @@ -0,0 +1,320 @@ +import { SortOrder } from "../../../api" +import { EmptyFilterOption, Hosting, SearchFilters } from "../../../sdk" +import { HttpMethod } from "../query" +import { Row } from "../row" +import { + AutomationActionStepId, + AutomationResults, + EmailAttachment, + LoopStepType, + ActionImplementation, +} from "./automation" + +export type ActionImplementations = { + [AutomationActionStepId.COLLECT]: ActionImplementation< + CollectStepInputs, + CollectStepOutputs + > + [AutomationActionStepId.CREATE_ROW]: ActionImplementation< + CreateRowStepInputs, + CreateRowStepOutputs + > + [AutomationActionStepId.DELAY]: ActionImplementation< + DelayStepInputs, + DelayStepOutputs + > + [AutomationActionStepId.DELETE_ROW]: ActionImplementation< + DeleteRowStepInputs, + DeleteRowStepOutputs + > + + [AutomationActionStepId.EXECUTE_QUERY]: ActionImplementation< + ExecuteQueryStepInputs, + ExecuteQueryStepOutputs + > + [AutomationActionStepId.EXECUTE_SCRIPT]: ActionImplementation< + ExecuteScriptStepInputs, + ExecuteScriptStepOutputs + > + [AutomationActionStepId.FILTER]: ActionImplementation< + FilterStepInputs, + FilterStepOutputs + > + [AutomationActionStepId.QUERY_ROWS]: ActionImplementation< + QueryRowsStepInputs, + QueryRowsStepOutputs + > + [AutomationActionStepId.SEND_EMAIL_SMTP]: ActionImplementation< + SmtpEmailStepInputs, + BaseAutomationOutputs + > + [AutomationActionStepId.SERVER_LOG]: ActionImplementation< + ServerLogStepInputs, + ServerLogStepOutputs + > + [AutomationActionStepId.TRIGGER_AUTOMATION_RUN]: ActionImplementation< + TriggerAutomationStepInputs, + TriggerAutomationStepOutputs + > + [AutomationActionStepId.UPDATE_ROW]: ActionImplementation< + UpdateRowStepInputs, + UpdateRowStepOutputs + > + [AutomationActionStepId.OUTGOING_WEBHOOK]: ActionImplementation< + OutgoingWebhookStepInputs, + ExternalAppStepOutputs + > + [AutomationActionStepId.discord]: ActionImplementation< + DiscordStepInputs, + ExternalAppStepOutputs + > + [AutomationActionStepId.slack]: ActionImplementation< + SlackStepInputs, + ExternalAppStepOutputs + > + + [AutomationActionStepId.zapier]: ActionImplementation< + ZapierStepInputs, + ZapierStepOutputs + > + [AutomationActionStepId.integromat]: ActionImplementation< + MakeIntegrationInputs, + ExternalAppStepOutputs + > + [AutomationActionStepId.n8n]: ActionImplementation< + n8nStepInputs, + ExternalAppStepOutputs + > +} & (T extends "self" + ? { + [AutomationActionStepId.EXECUTE_BASH]: ActionImplementation< + BashStepInputs, + BashStepOutputs + > + [AutomationActionStepId.OPENAI]: ActionImplementation< + OpenAIStepInputs, + OpenAIStepOutputs + > + } + : {}) + +export type BaseAutomationOutputs = { + success?: boolean + response?: { + [key: string]: any + message?: string + } +} + +export type ExternalAppStepOutputs = { + httpStatus?: number + response: string + success: boolean +} + +export type BashStepInputs = { + code: string +} + +export type BashStepOutputs = BaseAutomationOutputs & { + stdout?: string +} + +export type CollectStepInputs = { + collection: string +} + +export type CollectStepOutputs = BaseAutomationOutputs & { + value?: any +} + +export type CreateRowStepInputs = { + row: Row +} + +export type CreateRowStepOutputs = BaseAutomationOutputs & { + row?: Row + id?: string + revision?: string +} + +export type DelayStepInputs = { + time: number +} + +export type DelayStepOutputs = BaseAutomationOutputs + +export type DeleteRowStepInputs = { + tableId: string + id: string + revision?: string +} + +export type DeleteRowStepOutputs = BaseAutomationOutputs & { + row?: Row +} + +export type DiscordStepInputs = { + url: string + username?: string + avatar_url?: string + content: string +} + +export type ExecuteQueryStepInputs = { + query: { + queryId: string + } +} + +export type ExecuteQueryStepOutputs = BaseAutomationOutputs & { + info?: any +} + +export type ExecuteScriptStepInputs = { + code: string +} + +export type ExecuteScriptStepOutputs = BaseAutomationOutputs & { + value?: string +} + +export type FilterStepInputs = { + field: any + condition: string + value: any +} + +export type FilterStepOutputs = BaseAutomationOutputs & { + result: boolean + refValue?: any + comparisonValue?: any +} + +export type LoopStepInputs = { + option: LoopStepType + binding: any + iterations?: number + failure?: string +} + +export type LoopStepOutputs = { + items: string + success: boolean + iterations: number +} + +export type MakeIntegrationInputs = { + url: string + body: any +} + +export type n8nStepInputs = { + url: string + method: HttpMethod + authorization: string + body: any +} + +export type OpenAIStepInputs = { + prompt: string + model: Model +} + +enum Model { + GPT_35_TURBO = "gpt-3.5-turbo", + // will only work with api keys that have access to the GPT4 API + GPT_4 = "gpt-4", +} + +export type OpenAIStepOutputs = Omit & { + response?: string | null +} + +export type QueryRowsStepInputs = { + tableId: string + filters?: SearchFilters + "filters-def"?: any + sortColumn?: string + sortOrder?: SortOrder + limit?: number + onEmptyFilter?: EmptyFilterOption +} + +export type QueryRowsStepOutputs = BaseAutomationOutputs & { + rows?: Row[] +} + +export type SmtpEmailStepInputs = { + to: string + from: string + subject: string + contents: string + cc: string + bcc: string + addInvite?: boolean + startTime: Date + endTime: Date + summary: string + location?: string + url?: string + attachments?: EmailAttachment[] +} +export type ServerLogStepInputs = { + text: string +} + +export type ServerLogStepOutputs = BaseAutomationOutputs & { + message: string +} +export type SlackStepInputs = { + url: string + text: string +} + +export type TriggerAutomationStepInputs = { + automation: { + automationId: string + } + timeout: number +} + +export type TriggerAutomationStepOutputs = BaseAutomationOutputs & { + value?: AutomationResults["steps"] +} + +export type UpdateRowStepInputs = { + meta: Record + row: Row + rowId: string +} + +export type UpdateRowStepOutputs = BaseAutomationOutputs & { + row?: Row + id?: string + revision?: string +} + +export type ZapierStepInputs = { + url: string + body: any +} + +export type ZapierStepOutputs = Omit & { + response: string +} + +enum RequestType { + POST = "POST", + GET = "GET", + PUT = "PUT", + DELETE = "DELETE", + PATCH = "PATCH", +} + +export type OutgoingWebhookStepInputs = { + requestMethod: RequestType + url: string + requestBody: string + headers: string +} From 86f59fb71de43245720888126cc7de5b6ee54610 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 1 Aug 2024 11:20:23 +0100 Subject: [PATCH 15/15] Fixes an issue with MS-SQL timeouts, it doesn't respect query timeout value as the client has its own built in timeout by default of 15000ms. --- packages/server/scripts/integrations/mssql/data/Dockerfile | 2 +- packages/server/src/integrations/microsoftSqlServer.ts | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/server/scripts/integrations/mssql/data/Dockerfile b/packages/server/scripts/integrations/mssql/data/Dockerfile index c26af556a9..2bdca4490c 100644 --- a/packages/server/scripts/integrations/mssql/data/Dockerfile +++ b/packages/server/scripts/integrations/mssql/data/Dockerfile @@ -1,4 +1,4 @@ -FROM mcr.microsoft.com/mssql/server:2022-latest +FROM mcr.microsoft.com/mssql/server@sha256:c4369c38385eba011c10906dc8892425831275bb035d5ce69656da8e29de50d8 ENV ACCEPT_EULA=Y ENV SA_PASSWORD=Passw0rd diff --git a/packages/server/src/integrations/microsoftSqlServer.ts b/packages/server/src/integrations/microsoftSqlServer.ts index af535891cf..88c75891e6 100644 --- a/packages/server/src/integrations/microsoftSqlServer.ts +++ b/packages/server/src/integrations/microsoftSqlServer.ts @@ -29,6 +29,7 @@ import { getReadableErrorMessage } from "./base/errorMapping" import sqlServer from "mssql" import { sql } from "@budibase/backend-core" import { ConfidentialClientApplication } from "@azure/msal-node" +import env from "../environment" import { utils } from "@budibase/shared-core" @@ -246,6 +247,7 @@ class SqlServerIntegration extends Sql implements DatasourcePlus { options: { encrypt, enableArithAbort: true, + requestTimeout: env.QUERY_THREAD_TIMEOUT, }, } if (encrypt) {