Merge branch 'master' of github.com:Budibase/budibase into fix/sql-pagination-fixes

packages/builder/src/constants/backend/index.js

@@ -9,7 +9,10 @@ import { Constants } from "@budibase/frontend-core"
 
 const { TypeIconMap } = Constants
 
-export { RelationshipType } from "@budibase/types"
+export {
+  RelationshipType,
+  RowExportFormat as ROW_EXPORT_FORMATS,
+} from "@budibase/types"
 
 export const AUTO_COLUMN_SUB_TYPES = AutoFieldSubType
 
@@ -307,9 +310,3 @@ export const DatasourceTypes = {
   GRAPH: "Graph",
   API: "API",
 }
-
-export const ROW_EXPORT_FORMATS = {
-  CSV: "csv",
-  JSON: "json",
-  JSON_WITH_SCHEMA: "jsonWithSchema",
-}

packages/server/scripts/integrations/mssql/data/Dockerfile

@@ -1,4 +1,4 @@
-FROM mcr.microsoft.com/mssql/server:2022-latest
+FROM mcr.microsoft.com/mssql/server@sha256:c4369c38385eba011c10906dc8892425831275bb035d5ce69656da8e29de50d8
 
 ENV ACCEPT_EULA=Y
 ENV SA_PASSWORD=Passw0rd

packages/server/src/api/controllers/table/index.ts

@@ -17,6 +17,7 @@ import {
   CsvToJsonRequest,
   CsvToJsonResponse,
   FetchTablesResponse,
+  FieldType,
   MigrateRequest,
   MigrateResponse,
   SaveTableRequest,
@@ -178,9 +179,17 @@ export async function validateExistingTableImport(
   const { rows, tableId } = ctx.request.body
 
   let schema = null
+
   if (tableId) {
     const table = await sdk.tables.getTable(tableId)
     schema = table.schema
+
+    if (!isExternalTable(table)) {
+      schema._id = {
+        name: "_id",
+        type: FieldType.STRING,
+      }
+    }
   } else {
     ctx.status = 422
     return

packages/server/src/api/controllers/table/internal.ts

@@ -3,6 +3,7 @@ import { handleDataImport } from "./utils"
 import {
   BulkImportRequest,
   BulkImportResponse,
+  FieldType,
   RenameColumn,
   SaveTableRequest,
   SaveTableResponse,
@@ -69,10 +70,22 @@ export async function bulkImport(
 ) {
   const table = await sdk.tables.getTable(ctx.params.tableId)
   const { rows, identifierFields } = ctx.request.body
-  await handleDataImport(table, {
+  await handleDataImport(
+    {
+      ...table,
+      schema: {
+        _id: {
+          name: "_id",
+          type: FieldType.STRING,
+        },
+        ...table.schema,
+      },
+    },
+    {
       importRows: rows,
       identifierFields,
       user: ctx.user,
-  })
+    }
+  )
   return table
 }

packages/server/src/api/controllers/table/utils.ts

@@ -122,13 +122,15 @@ export function makeSureTableUpToDate(table: Table, tableToSave: Table) {
 export async function importToRows(
   data: Row[],
   table: Table,
-  user?: ContextUser
+  user?: ContextUser,
+  opts?: { keepCouchId: boolean }
 ) {
-  let originalTable = table
-  let finalData: any = []
+  const originalTable = table
+  const finalData: Row[] = []
+  const keepCouchId = !!opts?.keepCouchId
   for (let i = 0; i < data.length; i++) {
     let row = data[i]
-    row._id = generateRowID(table._id!)
+    row._id = (keepCouchId && row._id) || generateRowID(table._id!)
     row.type = "row"
     row.tableId = table._id
 
@@ -180,7 +182,11 @@ export async function handleDataImport(
   const db = context.getAppDB()
   const data = parse(importRows, table)
 
-  let finalData: any = await importToRows(data, table, user)
+  const finalData = await importToRows(data, table, user, {
+    keepCouchId: identifierFields.includes("_id"),
+  })
+
+  let newRowCount = finalData.length
 
   //Set IDs of finalData to match existing row if an update is expected
   if (identifierFields.length > 0) {
@@ -203,12 +209,14 @@ export async function handleDataImport(
           if (match) {
             finalItem._id = doc._id
             finalItem._rev = doc._rev
+
+            newRowCount--
           }
         })
       })
   }
 
-  await quotas.addRows(finalData.length, () => db.bulkDocs(finalData), {
+  await quotas.addRows(newRowCount, () => db.bulkDocs(finalData), {
     tableId: table._id,
   })
 

packages/server/src/api/controllers/view/exporters.ts

@@ -1,4 +1,6 @@
-import { Row, TableSchema } from "@budibase/types"
+import { Row, RowExportFormat, TableSchema } from "@budibase/types"
+
+export { RowExportFormat as Format } from "@budibase/types"
 
 function getHeaders(
   headers: string[],
@@ -46,14 +48,8 @@ export function jsonWithSchema(schema: TableSchema, rows: Row[]) {
   return JSON.stringify({ schema: newSchema, rows }, undefined, 2)
 }
 
-export enum Format {
-  CSV = "csv",
-  JSON = "json",
-  JSON_WITH_SCHEMA = "jsonWithSchema",
-}
-
-export function isFormat(format: any): format is Format {
-  return Object.values(Format).includes(format as Format)
+export function isFormat(format: any): format is RowExportFormat {
+  return Object.values(RowExportFormat).includes(format as RowExportFormat)
 }
 
 export function parseCsvExport<T>(value: string) {

packages/server/src/api/routes/tests/row.spec.ts

@@ -1298,6 +1298,113 @@ describe.each([
       await assertRowUsage(isInternal ? rowUsage + 2 : rowUsage)
     })
 
+    isInternal &&
+      it("should be able to update existing rows on bulkImport", async () => {
+        const table = await config.api.table.save(
+          saveTableRequest({
+            schema: {
+              name: {
+                type: FieldType.STRING,
+                name: "name",
+              },
+              description: {
+                type: FieldType.STRING,
+                name: "description",
+              },
+            },
+          })
+        )
+
+        const existingRow = await config.api.row.save(table._id!, {
+          name: "Existing row",
+          description: "Existing description",
+        })
+
+        const rowUsage = await getRowUsage()
+
+        await config.api.row.bulkImport(table._id!, {
+          rows: [
+            {
+              name: "Row 1",
+              description: "Row 1 description",
+            },
+            { ...existingRow, name: "Updated existing row" },
+            {
+              name: "Row 2",
+              description: "Row 2 description",
+            },
+          ],
+          identifierFields: ["_id"],
+        })
+
+        const rows = await config.api.row.fetch(table._id!)
+        expect(rows.length).toEqual(3)
+
+        rows.sort((a, b) => a.name.localeCompare(b.name))
+        expect(rows[0].name).toEqual("Row 1")
+        expect(rows[0].description).toEqual("Row 1 description")
+        expect(rows[1].name).toEqual("Row 2")
+        expect(rows[1].description).toEqual("Row 2 description")
+        expect(rows[2].name).toEqual("Updated existing row")
+        expect(rows[2].description).toEqual("Existing description")
+
+        await assertRowUsage(rowUsage + 2)
+      })
+
+    isInternal &&
+      it("should create new rows if not identifierFields are provided", async () => {
+        const table = await config.api.table.save(
+          saveTableRequest({
+            schema: {
+              name: {
+                type: FieldType.STRING,
+                name: "name",
+              },
+              description: {
+                type: FieldType.STRING,
+                name: "description",
+              },
+            },
+          })
+        )
+
+        const existingRow = await config.api.row.save(table._id!, {
+          name: "Existing row",
+          description: "Existing description",
+        })
+
+        const rowUsage = await getRowUsage()
+
+        await config.api.row.bulkImport(table._id!, {
+          rows: [
+            {
+              name: "Row 1",
+              description: "Row 1 description",
+            },
+            { ...existingRow, name: "Updated existing row" },
+            {
+              name: "Row 2",
+              description: "Row 2 description",
+            },
+          ],
+        })
+
+        const rows = await config.api.row.fetch(table._id!)
+        expect(rows.length).toEqual(4)
+
+        rows.sort((a, b) => a.name.localeCompare(b.name))
+        expect(rows[0].name).toEqual("Existing row")
+        expect(rows[0].description).toEqual("Existing description")
+        expect(rows[1].name).toEqual("Row 1")
+        expect(rows[1].description).toEqual("Row 1 description")
+        expect(rows[2].name).toEqual("Row 2")
+        expect(rows[2].description).toEqual("Row 2 description")
+        expect(rows[3].name).toEqual("Updated existing row")
+        expect(rows[3].description).toEqual("Existing description")
+
+        await assertRowUsage(rowUsage + 3)
+      })
+
     // Upserting isn't yet supported in MSSQL, see:
     //   https://github.com/knex/knex/pull/6050
     !isMSSQL &&
@@ -1640,6 +1747,23 @@ describe.each([
       table = await config.api.table.save(defaultTable())
     })
 
+    isInternal &&
+      it("should not export internal couchdb fields", async () => {
+        const existing = await config.api.row.save(table._id!, {
+          name: generator.guid(),
+          description: generator.paragraph(),
+        })
+        const res = await config.api.row.exportRows(table._id!, {
+          rows: [existing._id!],
+        })
+        const results = JSON.parse(res)
+        expect(results.length).toEqual(1)
+        const row = results[0]
+
+        expect(Object.keys(row)).toEqual(["_id", "name", "description"])
+      })
+
+    !isInternal &&
       it("should allow exporting all columns", async () => {
         const existing = await config.api.row.save(table._id!, {})
         const res = await config.api.row.exportRows(table._id!, {
@@ -1650,9 +1774,7 @@ describe.each([
         const row = results[0]
 
         // Ensure all original columns were exported
-      expect(Object.keys(row).length).toBeGreaterThanOrEqual(
-        Object.keys(existing).length
-      )
+        expect(Object.keys(row).length).toBe(Object.keys(existing).length)
         Object.keys(existing).forEach(key => {
           expect(row[key]).toEqual(existing[key])
         })

packages/server/src/api/routes/tests/table.spec.ts

@@ -1,4 +1,4 @@
-import { context, events } from "@budibase/backend-core"
+import { context, docIds, events } from "@budibase/backend-core"
 import {
   AutoFieldSubType,
   BBReferenceFieldSubType,
@@ -10,6 +10,7 @@ import {
   Row,
   SaveTableRequest,
   Table,
+  TableSchema,
   TableSourceType,
   User,
   ViewCalculation,
@@ -1022,4 +1023,92 @@ describe.each([
       })
     })
   })
+
+  describe("import validation", () => {
+    const basicSchema: TableSchema = {
+      id: {
+        type: FieldType.NUMBER,
+        name: "id",
+      },
+      name: {
+        type: FieldType.STRING,
+        name: "name",
+      },
+    }
+
+    describe("validateNewTableImport", () => {
+      it("can validate basic imports", async () => {
+        const result = await config.api.table.validateNewTableImport(
+          [{ id: generator.natural(), name: generator.first() }],
+          basicSchema
+        )
+
+        expect(result).toEqual({
+          allValid: true,
+          errors: {},
+          invalidColumns: [],
+          schemaValidation: {
+            id: true,
+            name: true,
+          },
+        })
+      })
+    })
+
+    describe("validateExistingTableImport", () => {
+      it("can validate basic imports", async () => {
+        const table = await config.api.table.save(
+          tableForDatasource(datasource, {
+            primary: ["id"],
+            schema: basicSchema,
+          })
+        )
+        const result = await config.api.table.validateExistingTableImport({
+          tableId: table._id,
+          rows: [{ id: generator.natural(), name: generator.first() }],
+        })
+
+        expect(result).toEqual({
+          allValid: true,
+          errors: {},
+          invalidColumns: [],
+          schemaValidation: {
+            id: true,
+            name: true,
+          },
+        })
+      })
+
+      isInternal &&
+        it("can reimport _id fields for internal tables", async () => {
+          const table = await config.api.table.save(
+            tableForDatasource(datasource, {
+              primary: ["id"],
+              schema: basicSchema,
+            })
+          )
+          const result = await config.api.table.validateExistingTableImport({
+            tableId: table._id,
+            rows: [
+              {
+                _id: docIds.generateRowID(table._id!),
+                id: generator.natural(),
+                name: generator.first(),
+              },
+            ],
+          })
+
+          expect(result).toEqual({
+            allValid: true,
+            errors: {},
+            invalidColumns: [],
+            schemaValidation: {
+              _id: true,
+              id: true,
+              name: true,
+            },
+          })
+        })
+    })
+  })
 })

packages/server/src/automations/actions.ts

@@ -20,17 +20,21 @@ import * as triggerAutomationRun from "./steps/triggerAutomationRun"
 import env from "../environment"
 import {
   AutomationStepSchema,
-  AutomationStepInput,
   PluginType,
   AutomationStep,
+  AutomationActionStepId,
+  ActionImplementations,
+  Hosting,
+  ActionImplementation,
 } from "@budibase/types"
 import sdk from "../sdk"
 import { getAutomationPlugin } from "../utilities/fileSystem"
 
-const ACTION_IMPLS: Record<
-  string,
-  (opts: AutomationStepInput) => Promise<any>
-> = {
+type ActionImplType = ActionImplementations<
+  typeof env.SELF_HOSTED extends "true" ? Hosting.SELF : Hosting.CLOUD
+>
+
+const ACTION_IMPLS: ActionImplType = {
   SEND_EMAIL_SMTP: sendSmtpEmail.run,
   CREATE_ROW: createRow.run,
   UPDATE_ROW: updateRow.run,
@@ -51,6 +55,7 @@ const ACTION_IMPLS: Record<
   integromat: make.run,
   n8n: n8n.run,
 }
+
 export const BUILTIN_ACTION_DEFINITIONS: Record<string, AutomationStepSchema> =
   {
     SEND_EMAIL_SMTP: sendSmtpEmail.definition,
@@ -86,7 +91,7 @@ if (env.SELF_HOSTED) {
   ACTION_IMPLS["EXECUTE_BASH"] = bash.run
   // @ts-ignore
   BUILTIN_ACTION_DEFINITIONS["EXECUTE_BASH"] = bash.definition
-
+  // @ts-ignore
   ACTION_IMPLS.OPENAI = openai.run
   BUILTIN_ACTION_DEFINITIONS.OPENAI = openai.definition
 }
@@ -107,10 +112,13 @@ export async function getActionDefinitions() {
 }
 
 /* istanbul ignore next */
-export async function getAction(stepId: string) {
-  if (ACTION_IMPLS[stepId] != null) {
-    return ACTION_IMPLS[stepId]
+export async function getAction(
+  stepId: AutomationActionStepId
+): Promise<ActionImplementation<any, any> | undefined> {
+  if (ACTION_IMPLS[stepId as keyof ActionImplType] != null) {
+    return ACTION_IMPLS[stepId as keyof ActionImplType]
   }
+
   // must be a plugin
   if (env.SELF_HOSTED) {
     const plugins = await sdk.plugins.fetch(PluginType.AUTOMATION)

packages/server/src/automations/automationUtils.ts

@@ -4,8 +4,13 @@ import {
   encodeJSBinding,
 } from "@budibase/string-templates"
 import sdk from "../sdk"
-import { AutomationAttachment, FieldType, Row } from "@budibase/types"
-import { LoopInput, LoopStepType } from "../definitions/automations"
+import {
+  AutomationAttachment,
+  FieldType,
+  Row,
+  LoopStepType,
+} from "@budibase/types"
+import { LoopInput } from "../definitions/automations"
 import { objectStore, context } from "@budibase/backend-core"
 import * as uuid from "uuid"
 import path from "path"

packages/server/src/automations/steps/bash.ts

@@ -7,9 +7,10 @@ import {
   AutomationCustomIOType,
   AutomationFeature,
   AutomationIOType,
-  AutomationStepInput,
   AutomationStepSchema,
   AutomationStepType,
+  BashStepInputs,
+  BashStepOutputs,
 } from "@budibase/types"
 
 export const definition: AutomationStepSchema = {
@@ -51,7 +52,13 @@ export const definition: AutomationStepSchema = {
   },
 }
 
-export async function run({ inputs, context }: AutomationStepInput) {
+export async function run({
+  inputs,
+  context,
+}: {
+  inputs: BashStepInputs
+  context: object
+}): Promise<BashStepOutputs> {
   if (inputs.code == null) {
     return {
       stdout: "Budibase bash automation failed: Invalid inputs",

packages/server/src/automations/steps/collect.ts

@@ -1,9 +1,10 @@
 import {
   AutomationActionStepId,
   AutomationStepSchema,
-  AutomationStepInput,
   AutomationStepType,
   AutomationIOType,
+  CollectStepInputs,
+  CollectStepOutputs,
 } from "@budibase/types"
 
 export const definition: AutomationStepSchema = {
@@ -43,7 +44,11 @@ export const definition: AutomationStepSchema = {
   },
 }
 
-export async function run({ inputs }: AutomationStepInput) {
+export async function run({
+  inputs,
+}: {
+  inputs: CollectStepInputs
+}): Promise<CollectStepOutputs> {
   if (!inputs.collection) {
     return {
       success: false,

packages/server/src/automations/steps/createRow.ts

@@ -10,10 +10,12 @@ import {
   AutomationCustomIOType,
   AutomationFeature,
   AutomationIOType,
-  AutomationStepInput,
   AutomationStepSchema,
   AutomationStepType,
+  CreateRowStepInputs,
+  CreateRowStepOutputs,
 } from "@budibase/types"
+import { EventEmitter } from "events"
 
 export const definition: AutomationStepSchema = {
   name: "Create Row",
@@ -74,7 +76,15 @@ export const definition: AutomationStepSchema = {
   },
 }
 
-export async function run({ inputs, appId, emitter }: AutomationStepInput) {
+export async function run({
+  inputs,
+  appId,
+  emitter,
+}: {
+  inputs: CreateRowStepInputs
+  appId: string
+  emitter: EventEmitter
+}): Promise<CreateRowStepOutputs> {
   if (inputs.row == null || inputs.row.tableId == null) {
     return {
       success: false,
@@ -93,7 +103,7 @@ export async function run({ inputs, appId, emitter }: AutomationStepInput) {
   try {
     inputs.row = await cleanUpRow(inputs.row.tableId, inputs.row)
     inputs.row = await sendAutomationAttachmentsToStorage(
-      inputs.row.tableId,
+      inputs.row.tableId!,
       inputs.row
     )
     await save(ctx)

packages/server/src/automations/steps/delay.ts

@@ -2,9 +2,10 @@ import { wait } from "../../utilities"
 import {
   AutomationActionStepId,
   AutomationIOType,
-  AutomationStepInput,
   AutomationStepSchema,
   AutomationStepType,
+  DelayStepInputs,
+  DelayStepOutputs,
 } from "@budibase/types"
 
 export const definition: AutomationStepSchema = {
@@ -39,7 +40,11 @@ export const definition: AutomationStepSchema = {
   type: AutomationStepType.LOGIC,
 }
 
-export async function run({ inputs }: AutomationStepInput) {
+export async function run({
+  inputs,
+}: {
+  inputs: DelayStepInputs
+}): Promise<DelayStepOutputs> {
   await wait(inputs.time)
   return {
     success: true,

packages/server/src/automations/steps/deleteRow.ts

@@ -1,14 +1,16 @@
+import { EventEmitter } from "events"
 import { destroy } from "../../api/controllers/row"
 import { buildCtx } from "./utils"
 import { getError } from "../automationUtils"
 import {
   AutomationActionStepId,
-  AutomationStepInput,
   AutomationStepSchema,
   AutomationStepType,
   AutomationIOType,
   AutomationCustomIOType,
   AutomationFeature,
+  DeleteRowStepInputs,
+  DeleteRowStepOutputs,
 } from "@budibase/types"
 
 export const definition: AutomationStepSchema = {
@@ -59,7 +61,15 @@ export const definition: AutomationStepSchema = {
   },
 }
 
-export async function run({ inputs, appId, emitter }: AutomationStepInput) {
+export async function run({
+  inputs,
+  appId,
+  emitter,
+}: {
+  inputs: DeleteRowStepInputs
+  appId: string
+  emitter: EventEmitter
+}): Promise<DeleteRowStepOutputs> {
   if (inputs.id == null) {
     return {
       success: false,

packages/server/src/automations/steps/discord.ts

@@ -3,10 +3,11 @@ import { getFetchResponse } from "./utils"
 import {
   AutomationActionStepId,
   AutomationStepSchema,
-  AutomationStepInput,
   AutomationStepType,
   AutomationIOType,
   AutomationFeature,
+  ExternalAppStepOutputs,
+  DiscordStepInputs,
 } from "@budibase/types"
 
 const DEFAULT_USERNAME = "Budibase Automate"
@@ -65,7 +66,11 @@ export const definition: AutomationStepSchema = {
   },
 }
 
-export async function run({ inputs }: AutomationStepInput) {
+export async function run({
+  inputs,
+}: {
+  inputs: DiscordStepInputs
+}): Promise<ExternalAppStepOutputs> {
   let { url, username, avatar_url, content } = inputs
   if (!username) {
     username = DEFAULT_USERNAME

packages/server/src/automations/steps/executeQuery.ts

@@ -1,3 +1,4 @@
+import { EventEmitter } from "events"
 import * as queryController from "../../api/controllers/query"
 import { buildCtx } from "./utils"
 import * as automationUtils from "../automationUtils"
@@ -6,9 +7,10 @@ import {
   AutomationCustomIOType,
   AutomationFeature,
   AutomationIOType,
-  AutomationStepInput,
   AutomationStepSchema,
   AutomationStepType,
+  ExecuteQueryStepInputs,
+  ExecuteQueryStepOutputs,
 } from "@budibase/types"
 
 export const definition: AutomationStepSchema = {
@@ -62,7 +64,15 @@ export const definition: AutomationStepSchema = {
   },
 }
 
-export async function run({ inputs, appId, emitter }: AutomationStepInput) {
+export async function run({
+  inputs,
+  appId,
+  emitter,
+}: {
+  inputs: ExecuteQueryStepInputs
+  appId: string
+  emitter: EventEmitter
+}): Promise<ExecuteQueryStepOutputs> {
   if (inputs.query == null) {
     return {
       success: false,

packages/server/src/automations/steps/executeScript.ts

@@ -6,10 +6,12 @@ import {
   AutomationCustomIOType,
   AutomationFeature,
   AutomationIOType,
-  AutomationStepInput,
   AutomationStepSchema,
   AutomationStepType,
+  ExecuteScriptStepInputs,
+  ExecuteScriptStepOutputs,
 } from "@budibase/types"
+import { EventEmitter } from "events"
 
 export const definition: AutomationStepSchema = {
   name: "JS Scripting",
@@ -55,7 +57,12 @@ export async function run({
   appId,
   context,
   emitter,
-}: AutomationStepInput) {
+}: {
+  inputs: ExecuteScriptStepInputs
+  appId: string
+  context: object
+  emitter: EventEmitter
+}): Promise<ExecuteScriptStepOutputs> {
   if (inputs.code == null) {
     return {
       success: false,

packages/server/src/automations/steps/filter.ts

@@ -1,9 +1,10 @@
 import {
   AutomationActionStepId,
   AutomationStepSchema,
-  AutomationStepInput,
   AutomationStepType,
   AutomationIOType,
+  FilterStepInputs,
+  FilterStepOutputs,
 } from "@budibase/types"
 
 export const FilterConditions = {
@@ -69,7 +70,11 @@ export const definition: AutomationStepSchema = {
   },
 }
 
-export async function run({ inputs }: AutomationStepInput) {
+export async function run({
+  inputs,
+}: {
+  inputs: FilterStepInputs
+}): Promise<FilterStepOutputs> {
   try {
     let { field, condition, value } = inputs
     // coerce types so that we can use them

packages/server/src/automations/steps/make.ts

@@ -3,10 +3,11 @@ import { getFetchResponse } from "./utils"
 import {
   AutomationActionStepId,
   AutomationStepSchema,
-  AutomationStepInput,
   AutomationStepType,
   AutomationIOType,
   AutomationFeature,
+  ExternalAppStepOutputs,
+  MakeIntegrationInputs,
 } from "@budibase/types"
 
 export const definition: AutomationStepSchema = {
@@ -57,7 +58,11 @@ export const definition: AutomationStepSchema = {
   },
 }
 
-export async function run({ inputs }: AutomationStepInput) {
+export async function run({
+  inputs,
+}: {
+  inputs: MakeIntegrationInputs
+}): Promise<ExternalAppStepOutputs> {
   const { url, body } = inputs
 
   let payload = {}

packages/server/src/automations/steps/n8n.ts

@@ -3,11 +3,12 @@ import { getFetchResponse } from "./utils"
 import {
   AutomationActionStepId,
   AutomationStepSchema,
-  AutomationStepInput,
   AutomationStepType,
   AutomationIOType,
   AutomationFeature,
   HttpMethod,
+  ExternalAppStepOutputs,
+  n8nStepInputs,
 } from "@budibase/types"
 
 export const definition: AutomationStepSchema = {
@@ -67,7 +68,11 @@ export const definition: AutomationStepSchema = {
   },
 }
 
-export async function run({ inputs }: AutomationStepInput) {
+export async function run({
+  inputs,
+}: {
+  inputs: n8nStepInputs
+}): Promise<ExternalAppStepOutputs> {
   const { url, body, method, authorization } = inputs
 
   let payload = {}

packages/server/src/automations/steps/openai.ts

@@ -3,9 +3,10 @@ import { OpenAI } from "openai"
 import {
   AutomationActionStepId,
   AutomationStepSchema,
-  AutomationStepInput,
   AutomationStepType,
   AutomationIOType,
+  OpenAIStepInputs,
+  OpenAIStepOutputs,
 } from "@budibase/types"
 import { env } from "@budibase/backend-core"
 import * as automationUtils from "../automationUtils"
@@ -59,7 +60,11 @@ export const definition: AutomationStepSchema = {
   },
 }
 
-export async function run({ inputs }: AutomationStepInput) {
+export async function run({
+  inputs,
+}: {
+  inputs: OpenAIStepInputs
+}): Promise<OpenAIStepOutputs> {
   if (!env.OPENAI_API_KEY) {
     return {
       success: false,

packages/server/src/automations/steps/outgoingWebhook.ts

@@ -6,9 +6,10 @@ import {
   AutomationCustomIOType,
   AutomationFeature,
   AutomationIOType,
-  AutomationStepInput,
   AutomationStepSchema,
   AutomationStepType,
+  ExternalAppStepOutputs,
+  OutgoingWebhookStepInputs,
 } from "@budibase/types"
 
 enum RequestType {
@@ -88,7 +89,13 @@ export const definition: AutomationStepSchema = {
   },
 }
 
-export async function run({ inputs }: AutomationStepInput) {
+export async function run({
+  inputs,
+}: {
+  inputs: OutgoingWebhookStepInputs
+}): Promise<
+  Omit<ExternalAppStepOutputs, "httpStatus"> | ExternalAppStepOutputs
+> {
   let { requestMethod, url, requestBody, headers } = inputs
   if (!url.startsWith("http")) {
     url = `http://${url}`

packages/server/src/automations/steps/queryRows.ts

@@ -8,13 +8,14 @@ import {
   AutomationCustomIOType,
   AutomationFeature,
   AutomationIOType,
-  AutomationStepInput,
   AutomationStepSchema,
   AutomationStepType,
   EmptyFilterOption,
   SearchFilters,
   Table,
   SortOrder,
+  QueryRowsStepInputs,
+  QueryRowsStepOutputs,
 } from "@budibase/types"
 import { db as dbCore } from "@budibase/backend-core"
 
@@ -133,7 +134,13 @@ function hasNullFilters(filters: any[]) {
   )
 }
 
-export async function run({ inputs, appId }: AutomationStepInput) {
+export async function run({
+  inputs,
+  appId,
+}: {
+  inputs: QueryRowsStepInputs
+  appId: string
+}): Promise<QueryRowsStepOutputs> {
   const { tableId, filters, sortColumn, sortOrder, limit } = inputs
   if (!tableId) {
     return {
@@ -145,7 +152,7 @@ export async function run({ inputs, appId }: AutomationStepInput) {
   }
   const table = await getTable(appId, tableId)
   let sortType = FieldType.STRING
-  if (table && table.schema && table.schema[sortColumn] && sortColumn) {
+  if (sortColumn && table && table.schema && table.schema[sortColumn]) {
     const fieldType = table.schema[sortColumn].type
     sortType =
       fieldType === FieldType.NUMBER ? FieldType.NUMBER : FieldType.STRING

packages/server/src/automations/steps/sendSmtpEmail.ts

@@ -3,11 +3,12 @@ import * as automationUtils from "../automationUtils"
 import {
   AutomationActionStepId,
   AutomationStepSchema,
-  AutomationStepInput,
   AutomationStepType,
   AutomationIOType,
   AutomationFeature,
   AutomationCustomIOType,
+  SmtpEmailStepInputs,
+  BaseAutomationOutputs,
 } from "@budibase/types"
 
 export const definition: AutomationStepSchema = {
@@ -97,7 +98,11 @@ export const definition: AutomationStepSchema = {
   },
 }
 
-export async function run({ inputs }: AutomationStepInput) {
+export async function run({
+  inputs,
+}: {
+  inputs: SmtpEmailStepInputs
+}): Promise<BaseAutomationOutputs> {
   let {
     to,
     from,
@@ -116,8 +121,8 @@ export async function run({ inputs }: AutomationStepInput) {
   if (!contents) {
     contents = "<h1>No content</h1>"
   }
-  to = to || undefined
 
+  try {
     if (attachments) {
       if (Array.isArray(attachments)) {
         attachments.forEach(item => automationUtils.guardAttachment(item))
@@ -126,7 +131,6 @@ export async function run({ inputs }: AutomationStepInput) {
       }
     }
 
-  try {
     let response = await sendSmtpEmail({
       to,
       from,

packages/server/src/automations/steps/serverLog.ts

@@ -1,10 +1,11 @@
 import {
   AutomationActionStepId,
   AutomationStepSchema,
-  AutomationStepInput,
   AutomationStepType,
   AutomationIOType,
   AutomationFeature,
+  ServerLogStepInputs,
+  ServerLogStepOutputs,
 } from "@budibase/types"
 
 /**
@@ -53,7 +54,13 @@ export const definition: AutomationStepSchema = {
   },
 }
 
-export async function run({ inputs, appId }: AutomationStepInput) {
+export async function run({
+  inputs,
+  appId,
+}: {
+  inputs: ServerLogStepInputs
+  appId: string
+}): Promise<ServerLogStepOutputs> {
   const message = `App ${appId} - ${inputs.text}`
   console.log(message)
   return {

packages/server/src/automations/steps/slack.ts

@@ -3,10 +3,11 @@ import { getFetchResponse } from "./utils"
 import {
   AutomationActionStepId,
   AutomationStepSchema,
-  AutomationStepInput,
   AutomationStepType,
   AutomationIOType,
   AutomationFeature,
+  ExternalAppStepOutputs,
+  SlackStepInputs,
 } from "@budibase/types"
 
 export const definition: AutomationStepSchema = {
@@ -54,7 +55,11 @@ export const definition: AutomationStepSchema = {
   },
 }
 
-export async function run({ inputs }: AutomationStepInput) {
+export async function run({
+  inputs,
+}: {
+  inputs: SlackStepInputs
+}): Promise<ExternalAppStepOutputs> {
   let { url, text } = inputs
   if (!url?.trim()?.length) {
     return {

packages/server/src/automations/steps/triggerAutomationRun.ts

@@ -1,12 +1,13 @@
 import {
   AutomationActionStepId,
   AutomationStepSchema,
-  AutomationStepInput,
   AutomationStepType,
   AutomationIOType,
   AutomationResults,
   Automation,
   AutomationCustomIOType,
+  TriggerAutomationStepInputs,
+  TriggerAutomationStepOutputs,
 } from "@budibase/types"
 import * as triggers from "../triggers"
 import { context } from "@budibase/backend-core"
@@ -61,7 +62,11 @@ export const definition: AutomationStepSchema = {
   },
 }
 
-export async function run({ inputs }: AutomationStepInput) {
+export async function run({
+  inputs,
+}: {
+  inputs: TriggerAutomationStepInputs
+}): Promise<TriggerAutomationStepOutputs> {
   const { automationId, ...fieldParams } = inputs.automation
 
   if (await features.isTriggerAutomationRunEnabled()) {
@@ -88,5 +93,9 @@ export async function run({ inputs }: AutomationStepInput) {
         value: response.steps,
       }
     }
+  } else {
+    return {
+      success: false,
+    }
   }
 }

packages/server/src/automations/steps/updateRow.ts

@@ -1,3 +1,4 @@
+import { EventEmitter } from "events"
 import * as rowController from "../../api/controllers/row"
 import * as automationUtils from "../automationUtils"
 import { buildCtx } from "./utils"
@@ -6,9 +7,10 @@ import {
   AutomationCustomIOType,
   AutomationFeature,
   AutomationIOType,
-  AutomationStepInput,
   AutomationStepSchema,
   AutomationStepType,
+  UpdateRowStepInputs,
+  UpdateRowStepOutputs,
 } from "@budibase/types"
 
 export const definition: AutomationStepSchema = {
@@ -70,8 +72,15 @@ export const definition: AutomationStepSchema = {
     },
   },
 }
-
-export async function run({ inputs, appId, emitter }: AutomationStepInput) {
+export async function run({
+  inputs,
+  appId,
+  emitter,
+}: {
+  inputs: UpdateRowStepInputs
+  appId: string
+  emitter: EventEmitter
+}): Promise<UpdateRowStepOutputs> {
   if (inputs.rowId == null || inputs.row == null) {
     return {
       success: false,

packages/server/src/automations/steps/zapier.ts

@@ -3,10 +3,11 @@ import { getFetchResponse } from "./utils"
 import {
   AutomationActionStepId,
   AutomationStepSchema,
-  AutomationStepInput,
   AutomationStepType,
   AutomationIOType,
   AutomationFeature,
+  ZapierStepInputs,
+  ZapierStepOutputs,
 } from "@budibase/types"
 
 export const definition: AutomationStepSchema = {
@@ -50,7 +51,11 @@ export const definition: AutomationStepSchema = {
   },
 }
 
-export async function run({ inputs }: AutomationStepInput) {
+export async function run({
+  inputs,
+}: {
+  inputs: ZapierStepInputs
+}): Promise<ZapierStepOutputs> {
   const { url, body } = inputs
 
   let payload = {}

packages/server/src/automations/tests/loop.spec.ts

@@ -3,9 +3,9 @@ import * as triggers from "../triggers"
 import { loopAutomation } from "../../tests/utilities/structures"
 import { context } from "@budibase/backend-core"
 import * as setup from "./utilities"
-import { Table } from "@budibase/types"
+import { Table, LoopStepType } from "@budibase/types"
 import * as loopUtils from "../loopUtils"
-import { LoopInput, LoopStepType } from "../../definitions/automations"
+import { LoopInput } from "../../definitions/automations"
 
 describe("Attempt to run a basic loop automation", () => {
   let config = setup.getConfig(),

packages/server/src/automations/tests/scenarios/scenarios.spec.ts

@@ -0,0 +1,160 @@
+import * as automation from "../../index"
+import * as setup from "../utilities"
+import { Table, LoopStepType } from "@budibase/types"
+import { createAutomationBuilder } from "../utilities/AutomationBuilder"
+
+describe("Automation Scenarios", () => {
+  let config = setup.getConfig(),
+    table: Table
+
+  beforeEach(async () => {
+    await automation.init()
+    await config.init()
+    table = await config.createTable()
+    await config.createRow()
+  })
+
+  afterAll(setup.afterAll)
+
+  describe("Loop automations", () => {
+    it("should run an automation with a trigger, loop, and create row step", async () => {
+      const builder = createAutomationBuilder({
+        name: "Test Trigger with Loop and Create Row",
+      })
+
+      const results = await builder
+        .rowSaved(
+          { tableId: table._id! },
+          {
+            row: {
+              name: "Trigger Row",
+              description: "This row triggers the automation",
+            },
+            id: "1234",
+            revision: "1",
+          }
+        )
+        .loop({
+          option: LoopStepType.ARRAY,
+          binding: [1, 2, 3],
+        })
+        .createRow({
+          row: {
+            name: "Item {{ loop.currentItem }}",
+            description: "Created from loop",
+            tableId: table._id,
+          },
+        })
+        .run()
+
+      expect(results.trigger).toBeDefined()
+      expect(results.steps).toHaveLength(1)
+
+      expect(results.steps[0].outputs.iterations).toBe(3)
+      expect(results.steps[0].outputs.items).toHaveLength(3)
+
+      results.steps[0].outputs.items.forEach((output: any, index: number) => {
+        expect(output).toMatchObject({
+          success: true,
+          row: {
+            name: `Item ${index + 1}`,
+            description: "Created from loop",
+          },
+        })
+      })
+    })
+  })
+
+  describe("Row Automations", () => {
+    it("should trigger an automation which then creates a row", async () => {
+      const table = await config.createTable()
+
+      const builder = createAutomationBuilder({
+        name: "Test Row Save and Create",
+      })
+
+      const results = await builder
+        .rowUpdated(
+          { tableId: table._id! },
+          {
+            row: { name: "Test", description: "TEST" },
+            id: "1234",
+          }
+        )
+        .createRow({
+          row: {
+            name: "{{trigger.row.name}}",
+            description: "{{trigger.row.description}}",
+            tableId: table._id,
+          },
+        })
+        .run()
+
+      expect(results.steps).toHaveLength(1)
+
+      expect(results.steps[0].outputs).toMatchObject({
+        success: true,
+        row: {
+          name: "Test",
+          description: "TEST",
+        },
+      })
+    })
+  })
+
+  it("should trigger an automation which querys the database", async () => {
+    const table = await config.createTable()
+    const row = {
+      name: "Test Row",
+      description: "original description",
+      tableId: table._id,
+    }
+    await config.createRow(row)
+    await config.createRow(row)
+    const builder = createAutomationBuilder({
+      name: "Test Row Save and Create",
+    })
+
+    const results = await builder
+      .appAction({ fields: {} })
+      .queryRows({
+        tableId: table._id!,
+      })
+      .run()
+
+    expect(results.steps).toHaveLength(1)
+    expect(results.steps[0].outputs.rows).toHaveLength(2)
+  })
+
+  it("should trigger an automation which querys the database then deletes a row", async () => {
+    const table = await config.createTable()
+    const row = {
+      name: "DFN",
+      description: "original description",
+      tableId: table._id,
+    }
+    await config.createRow(row)
+    await config.createRow(row)
+    const builder = createAutomationBuilder({
+      name: "Test Row Save and Create",
+    })
+
+    const results = await builder
+      .appAction({ fields: {} })
+      .queryRows({
+        tableId: table._id!,
+      })
+      .deleteRow({
+        tableId: table._id!,
+        id: "{{ steps.1.rows.0._id }}",
+      })
+      .queryRows({
+        tableId: table._id!,
+      })
+      .run()
+
+    expect(results.steps).toHaveLength(3)
+    expect(results.steps[1].outputs.success).toBeTruthy()
+    expect(results.steps[2].outputs.rows).toHaveLength(1)
+  })
+})

packages/server/src/automations/tests/utilities/AutomationBuilder.ts

@@ -0,0 +1,174 @@
+import { v4 as uuidv4 } from "uuid"
+import { testAutomation } from "../../../api/routes/tests/utilities/TestFunctions"
+import {
+  RowCreatedTriggerInputs,
+  RowCreatedTriggerOutputs,
+} from "../../triggerInfo/rowSaved"
+import {
+  RowUpdatedTriggerInputs,
+  RowUpdatedTriggerOutputs,
+} from "../../triggerInfo/rowUpdated"
+import {} from "../../steps/createRow"
+import { BUILTIN_ACTION_DEFINITIONS } from "../../actions"
+import { TRIGGER_DEFINITIONS } from "../../triggers"
+import {
+  RowDeletedTriggerInputs,
+  RowDeletedTriggerOutputs,
+} from "../../triggerInfo/rowDeleted"
+import {
+  AutomationStepSchema,
+  AutomationTriggerSchema,
+  LoopStepInputs,
+  DeleteRowStepInputs,
+  UpdateRowStepInputs,
+  CreateRowStepInputs,
+  Automation,
+  AutomationTrigger,
+  AutomationResults,
+  SmtpEmailStepInputs,
+  ExecuteQueryStepInputs,
+  QueryRowsStepInputs,
+} from "@budibase/types"
+import {} from "../../steps/loop"
+import TestConfiguration from "../../../tests/utilities/TestConfiguration"
+import * as setup from "../utilities"
+import {
+  AppActionTriggerInputs,
+  AppActionTriggerOutputs,
+} from "../../triggerInfo/app"
+import { CronTriggerOutputs } from "../../triggerInfo/cron"
+
+type TriggerOutputs =
+  | RowCreatedTriggerOutputs
+  | RowUpdatedTriggerOutputs
+  | RowDeletedTriggerOutputs
+  | AppActionTriggerOutputs
+  | CronTriggerOutputs
+  | undefined
+
+class AutomationBuilder {
+  private automationConfig: Automation = {
+    name: "",
+    definition: {
+      steps: [],
+      trigger: {} as AutomationTrigger,
+    },
+    type: "automation",
+    appId: setup.getConfig().getAppId(),
+  }
+  private config: TestConfiguration = setup.getConfig()
+  private triggerOutputs: TriggerOutputs
+  private triggerSet: boolean = false
+
+  constructor(options: { name?: string } = {}) {
+    this.automationConfig.name = options.name || `Test Automation ${uuidv4()}`
+  }
+
+  // TRIGGERS
+  rowSaved(inputs: RowCreatedTriggerInputs, outputs: RowCreatedTriggerOutputs) {
+    this.triggerOutputs = outputs
+    return this.trigger(TRIGGER_DEFINITIONS.ROW_SAVED, inputs, outputs)
+  }
+
+  rowUpdated(
+    inputs: RowUpdatedTriggerInputs,
+    outputs: RowUpdatedTriggerOutputs
+  ) {
+    this.triggerOutputs = outputs
+    return this.trigger(TRIGGER_DEFINITIONS.ROW_UPDATED, inputs, outputs)
+  }
+
+  rowDeleted(
+    inputs: RowDeletedTriggerInputs,
+    outputs: RowDeletedTriggerOutputs
+  ) {
+    this.triggerOutputs = outputs
+    return this.trigger(TRIGGER_DEFINITIONS.ROW_DELETED, inputs, outputs)
+  }
+
+  appAction(outputs: AppActionTriggerOutputs, inputs?: AppActionTriggerInputs) {
+    this.triggerOutputs = outputs
+    return this.trigger(TRIGGER_DEFINITIONS.APP, inputs, outputs)
+  }
+
+  // STEPS
+  createRow(inputs: CreateRowStepInputs): this {
+    return this.step(BUILTIN_ACTION_DEFINITIONS.CREATE_ROW, inputs)
+  }
+
+  updateRow(inputs: UpdateRowStepInputs): this {
+    return this.step(BUILTIN_ACTION_DEFINITIONS.UPDATE_ROW, inputs)
+  }
+
+  deleteRow(inputs: DeleteRowStepInputs): this {
+    return this.step(BUILTIN_ACTION_DEFINITIONS.DELETE_ROW, inputs)
+  }
+
+  sendSmtpEmail(inputs: SmtpEmailStepInputs): this {
+    return this.step(BUILTIN_ACTION_DEFINITIONS.SEND_EMAIL_SMTP, inputs)
+  }
+
+  executeQuery(inputs: ExecuteQueryStepInputs): this {
+    return this.step(BUILTIN_ACTION_DEFINITIONS.EXECUTE_QUERY, inputs)
+  }
+
+  queryRows(inputs: QueryRowsStepInputs): this {
+    return this.step(BUILTIN_ACTION_DEFINITIONS.QUERY_ROWS, inputs)
+  }
+  loop(inputs: LoopStepInputs): this {
+    return this.step(BUILTIN_ACTION_DEFINITIONS.LOOP, inputs)
+  }
+
+  private trigger<T extends { [key: string]: any }>(
+    triggerSchema: AutomationTriggerSchema,
+    inputs?: T,
+    outputs?: TriggerOutputs
+  ): this {
+    if (this.triggerSet) {
+      throw new Error("Only one trigger can be set for an automation.")
+    }
+    this.automationConfig.definition.trigger = {
+      ...triggerSchema,
+      inputs: inputs || {},
+      id: uuidv4(),
+    }
+    this.triggerOutputs = outputs
+    this.triggerSet = true
+
+    return this
+  }
+
+  private step<T extends { [key: string]: any }>(
+    stepSchema: AutomationStepSchema,
+    inputs: T
+  ): this {
+    this.automationConfig.definition.steps.push({
+      ...stepSchema,
+      inputs,
+      id: uuidv4(),
+    })
+    return this
+  }
+
+  async run() {
+    const automation = await this.config.createAutomation(this.automationConfig)
+    const results = await testAutomation(
+      this.config,
+      automation,
+      this.triggerOutputs
+    )
+    return this.processResults(results)
+  }
+
+  private processResults(results: { body: AutomationResults }) {
+    results.body.steps.shift()
+    return {
+      trigger: results.body.trigger,
+      steps: results.body.steps,
+    }
+  }
+}
+
+export function createAutomationBuilder(options?: { name?: string }) {
+  return new AutomationBuilder(options)
+}

packages/server/src/automations/tests/utilities/index.ts

@@ -3,6 +3,7 @@ import { context } from "@budibase/backend-core"
 import { BUILTIN_ACTION_DEFINITIONS, getAction } from "../../actions"
 import emitter from "../../../events/index"
 import env from "../../../environment"
+import { AutomationActionStepId } from "@budibase/types"
 
 let config: TestConfig
 
@@ -33,7 +34,7 @@ export async function runInProd(fn: any) {
 
 export async function runStep(stepId: string, inputs: any, stepContext?: any) {
   async function run() {
-    let step = await getAction(stepId)
+    let step = await getAction(stepId as AutomationActionStepId)
     expect(step).toBeDefined()
     if (!step) {
       throw new Error("No step found")
@@ -41,7 +42,7 @@ export async function runStep(stepId: string, inputs: any, stepContext?: any) {
     return step({
       context: stepContext || {},
       inputs,
-      appId: config ? config.getAppId() : null,
+      appId: config ? config.getAppId() : "",
       // don't really need an API key, mocked out usage quota, not being tested here
       apiKey,
       emitter,

packages/server/src/automations/triggerInfo/app.ts

@@ -39,3 +39,11 @@ export const definition: AutomationTriggerSchema = {
   },
   type: AutomationStepType.TRIGGER,
 }
+
+export type AppActionTriggerInputs = {
+  fields: object
+}
+
+export type AppActionTriggerOutputs = {
+  fields: object
+}

packages/server/src/automations/triggerInfo/cron.ts

@@ -38,3 +38,11 @@ export const definition: AutomationTriggerSchema = {
   },
   type: AutomationStepType.TRIGGER,
 }
+
+export type CronTriggerInputs = {
+  cron: string
+}
+
+export type CronTriggerOutputs = {
+  timestamp: number
+}

packages/server/src/automations/triggerInfo/rowDeleted.ts

@@ -5,6 +5,7 @@ import {
   AutomationTriggerSchema,
   AutomationTriggerStepId,
   AutomationEventType,
+  Row,
 } from "@budibase/types"
 
 export const definition: AutomationTriggerSchema = {
@@ -39,3 +40,11 @@ export const definition: AutomationTriggerSchema = {
   },
   type: AutomationStepType.TRIGGER,
 }
+
+export type RowDeletedTriggerInputs = {
+  tableId: string
+}
+
+export type RowDeletedTriggerOutputs = {
+  row: Row
+}

packages/server/src/automations/triggerInfo/rowSaved.ts

@@ -5,7 +5,9 @@ import {
   AutomationTriggerSchema,
   AutomationTriggerStepId,
   AutomationEventType,
+  Row,
 } from "@budibase/types"
+import { SearchFilters } from "aws-sdk/clients/elasticbeanstalk"
 
 export const definition: AutomationTriggerSchema = {
   name: "Row Created",
@@ -52,3 +54,14 @@ export const definition: AutomationTriggerSchema = {
   },
   type: AutomationStepType.TRIGGER,
 }
+
+export type RowCreatedTriggerInputs = {
+  tableId: string
+  filters?: SearchFilters
+}
+
+export type RowCreatedTriggerOutputs = {
+  row: Row
+  id: string
+  revision: string
+}

packages/server/src/automations/triggerInfo/rowUpdated.ts

@@ -5,6 +5,8 @@ import {
   AutomationTriggerSchema,
   AutomationTriggerStepId,
   AutomationEventType,
+  Row,
+  SearchFilters,
 } from "@budibase/types"
 
 export const definition: AutomationTriggerSchema = {
@@ -59,3 +61,14 @@ export const definition: AutomationTriggerSchema = {
   },
   type: AutomationStepType.TRIGGER,
 }
+
+export type RowUpdatedTriggerInputs = {
+  tableId: string
+  filters?: SearchFilters
+}
+
+export type RowUpdatedTriggerOutputs = {
+  row: Row
+  id: string
+  revision?: string
+}

packages/server/src/automations/unitTests/automationUtils.spec.ts

@@ -1,9 +1,9 @@
-import { LoopStepType } from "../../definitions/automations"
 import {
   typecastForLooping,
   cleanInputValues,
   substituteLoopStep,
 } from "../automationUtils"
+import { LoopStepType } from "@budibase/types"
 
 describe("automationUtils", () => {
   describe("substituteLoopStep", () => {

packages/server/src/db/defaultData/datasource_bb_default.ts

@@ -651,10 +651,10 @@ export async function buildDefaultDocs() {
       return new LinkDocument(
         employeeData.table._id!,
         "Jobs",
-        employeeData.rows[index]._id,
+        employeeData.rows[index]._id!,
         jobData.table._id!,
         "Assigned",
-        jobData.rows[index]._id
+        jobData.rows[index]._id!
       )
     }
   )

packages/server/src/definitions/automations.ts

@@ -1,9 +1,8 @@
-import { AutomationResults, AutomationStep } from "@budibase/types"
-
-export enum LoopStepType {
-  ARRAY = "Array",
-  STRING = "String",
-}
+import {
+  AutomationResults,
+  AutomationStep,
+  LoopStepType,
+} from "@budibase/types"
 
 export interface LoopStep extends AutomationStep {
   inputs: LoopInput

packages/server/src/integrations/microsoftSqlServer.ts

@@ -29,6 +29,7 @@ import { getReadableErrorMessage } from "./base/errorMapping"
 import sqlServer from "mssql"
 import { sql } from "@budibase/backend-core"
 import { ConfidentialClientApplication } from "@azure/msal-node"
+import env from "../environment"
 
 import { utils } from "@budibase/shared-core"
 
@@ -246,6 +247,7 @@ class SqlServerIntegration extends Sql implements DatasourcePlus {
         options: {
           encrypt,
           enableArithAbort: true,
+          requestTimeout: env.QUERY_THREAD_TIMEOUT,
         },
       }
       if (encrypt) {

packages/server/src/sdk/app/rows/search/internal.ts

@@ -11,6 +11,7 @@ import {
   SearchResponse,
   SortType,
   Table,
+  TableSchema,
   User,
 } from "@budibase/types"
 import { getGlobalUsersFromMetadata } from "../../../../utilities/global"
@@ -137,6 +138,9 @@ export async function exportRows(
   let rows: Row[] = []
   let schema = table.schema
   let headers
+
+  result = trimFields(result, schema)
+
   // Filter data to only specified columns if required
   if (columns && columns.length) {
     for (let i = 0; i < result.length; i++) {
@@ -299,3 +303,13 @@ async function getView(db: Database, viewName: string) {
   }
   return viewInfo
 }
+
+function trimFields(rows: Row[], schema: TableSchema) {
+  const allowedFields = ["_id", ...Object.keys(schema)]
+  const result = rows.map(row =>
+    Object.keys(row)
+      .filter(key => allowedFields.includes(key))
+      .reduce((acc, key) => ({ ...acc, [key]: row[key] }), {} as Row)
+  )
+  return result
+}

packages/server/src/sdk/app/rows/utils.ts

@@ -76,7 +76,7 @@ export async function getDatasourceAndQuery(
 }
 
 export function cleanExportRows(
-  rows: any[],
+  rows: Row[],
   schema: TableSchema,
   format: string,
   columns?: string[],

packages/server/src/tests/utilities/api/legacyView.ts

@@ -1,5 +1,5 @@
 import { Expectations, TestAPI } from "./base"
-import { Row, View, ViewCalculation } from "@budibase/types"
+import { Row, RowExportFormat, View, ViewCalculation } from "@budibase/types"
 
 export class LegacyViewAPI extends TestAPI {
   get = async (
@@ -24,7 +24,7 @@ export class LegacyViewAPI extends TestAPI {
 
   export = async (
     viewName: string,
-    format: "json" | "csv" | "jsonWithSchema",
+    format: `${RowExportFormat}`,
     expectations?: Expectations
   ) => {
     const response = await this._requestRaw("get", `/api/views/export`, {

packages/server/src/tests/utilities/api/table.ts

@@ -3,9 +3,13 @@ import {
   BulkImportResponse,
   MigrateRequest,
   MigrateResponse,
+  Row,
   SaveTableRequest,
   SaveTableResponse,
   Table,
+  TableSchema,
+  ValidateTableImportRequest,
+  ValidateTableImportResponse,
 } from "@budibase/types"
 import { Expectations, TestAPI } from "./base"
 
@@ -61,8 +65,38 @@ export class TableAPI extends TestAPI {
     revId: string,
     expectations?: Expectations
   ): Promise<void> => {
-    return await this._delete<void>(`/api/tables/${tableId}/${revId}`, {
+    return await this._delete(`/api/tables/${tableId}/${revId}`, {
       expectations,
     })
   }
+
+  validateNewTableImport = async (
+    rows: Row[],
+    schema: TableSchema,
+    expectations?: Expectations
+  ): Promise<ValidateTableImportResponse> => {
+    return await this._post<ValidateTableImportResponse>(
+      `/api/tables/validateNewTableImport`,
+      {
+        body: {
+          rows,
+          schema,
+        },
+        expectations,
+      }
+    )
+  }
+
+  validateExistingTableImport = async (
+    body: ValidateTableImportRequest,
+    expectations?: Expectations
+  ): Promise<ValidateTableImportResponse> => {
+    return await this._post<ValidateTableImportResponse>(
+      `/api/tables/validateExistingTableImport`,
+      {
+        body,
+        expectations,
+      }
+    )
+  }
 }

packages/server/src/tests/utilities/structures.ts

@@ -25,8 +25,9 @@ import {
   Webhook,
   WebhookActionType,
   AutomationEventType,
+  LoopStepType,
 } from "@budibase/types"
-import { LoopInput, LoopStepType } from "../../definitions/automations"
+import { LoopInput } from "../../definitions/automations"
 import { merge } from "lodash"
 import { generator } from "@budibase/backend-core/tests"
 

packages/server/src/threads/automation.ts

@@ -16,6 +16,7 @@ import { AutomationErrors, MAX_AUTOMATION_RECURRING_ERRORS } from "../constants"
 import { storeLog } from "../automations/logging"
 import {
   Automation,
+  AutomationActionStepId,
   AutomationData,
   AutomationJob,
   AutomationMetadata,
@@ -108,7 +109,7 @@ class Orchestrator {
     return triggerOutput
   }
 
-  async getStepFunctionality(stepId: string) {
+  async getStepFunctionality(stepId: AutomationActionStepId) {
     let step = await actions.getAction(stepId)
     if (step == null) {
       throw `Cannot find automation step by name ${stepId}`
@@ -422,7 +423,9 @@ class Orchestrator {
                 continue
               }
 
-              let stepFn = await this.getStepFunctionality(step.stepId)
+              let stepFn = await this.getStepFunctionality(
+                step.stepId as AutomationActionStepId
+              )
               let inputs = await processObject(originalStepInput, this._context)
               inputs = automationUtils.cleanInputValues(
                 inputs,

packages/server/src/utilities/workerRequests.ts

@@ -103,8 +103,8 @@ export async function sendSmtpEmail({
   from: string
   subject: string
   contents: string
-  cc: string
-  bcc: string
+  cc?: string
+  bcc?: string
   automation: boolean
   attachments?: EmailAttachment[]
   invite?: EmailInvite

packages/types/src/documents/app/automation/automation.ts

@@ -1,9 +1,9 @@
-import { Document } from "../document"
+import { Document } from "../../document"
 import { EventEmitter } from "events"
-import { User } from "../global"
+import { User } from "../../global"
 import { ReadStream } from "fs"
-import { Row } from "./row"
-import { Table } from "./table"
+import { Row } from "../row"
+import { Table } from "../table"
 
 export enum AutomationIOType {
   OBJECT = "object",
@@ -93,6 +93,7 @@ export interface EmailAttachment {
 }
 
 export interface SendEmailOpts {
+  to?: string
   // workspaceId If finer grain controls being used then this will lookup config for workspace.
   workspaceId?: string
   // user If sending to an existing user the object can be provided, this is used in the context.
@@ -102,7 +103,7 @@ export interface SendEmailOpts {
   // contents If sending a custom email then can supply contents which will be added to it.
   contents?: string
   // subject A custom subject can be specified if the config one is not desired.
-  subject?: string
+  subject: string
   // info Pass in a structure of information to be stored alongside the invitation.
   info?: any
   cc?: boolean
@@ -242,14 +243,18 @@ export interface AutomationLogPage {
   nextPage?: string
 }
 
-export type AutomationStepInput = {
-  inputs: Record<string, any>
+export interface AutomationStepInputBase {
   context: Record<string, any>
   emitter: EventEmitter
   appId: string
   apiKey?: string
 }
 
+export type ActionImplementation<TInputs, TOutputs> = (
+  params: {
+    inputs: TInputs
+  } & AutomationStepInputBase
+) => Promise<TOutputs>
 export interface AutomationMetadata extends Document {
   errorCount?: number
   automationChainCount?: number
@@ -286,3 +291,8 @@ export type UpdatedRowEventEmitter = {
   table: Table
   appId: string
 }
+
+export enum LoopStepType {
+  ARRAY = "Array",
+  STRING = "String",
+}

packages/types/src/documents/app/automation/index.ts

@@ -0,0 +1,2 @@
+export * from "./automation"
+export * from "./schema"

packages/types/src/documents/app/automation/schema.ts

@@ -0,0 +1,320 @@
+import { SortOrder } from "../../../api"
+import { EmptyFilterOption, Hosting, SearchFilters } from "../../../sdk"
+import { HttpMethod } from "../query"
+import { Row } from "../row"
+import {
+  AutomationActionStepId,
+  AutomationResults,
+  EmailAttachment,
+  LoopStepType,
+  ActionImplementation,
+} from "./automation"
+
+export type ActionImplementations<T extends Hosting> = {
+  [AutomationActionStepId.COLLECT]: ActionImplementation<
+    CollectStepInputs,
+    CollectStepOutputs
+  >
+  [AutomationActionStepId.CREATE_ROW]: ActionImplementation<
+    CreateRowStepInputs,
+    CreateRowStepOutputs
+  >
+  [AutomationActionStepId.DELAY]: ActionImplementation<
+    DelayStepInputs,
+    DelayStepOutputs
+  >
+  [AutomationActionStepId.DELETE_ROW]: ActionImplementation<
+    DeleteRowStepInputs,
+    DeleteRowStepOutputs
+  >
+
+  [AutomationActionStepId.EXECUTE_QUERY]: ActionImplementation<
+    ExecuteQueryStepInputs,
+    ExecuteQueryStepOutputs
+  >
+  [AutomationActionStepId.EXECUTE_SCRIPT]: ActionImplementation<
+    ExecuteScriptStepInputs,
+    ExecuteScriptStepOutputs
+  >
+  [AutomationActionStepId.FILTER]: ActionImplementation<
+    FilterStepInputs,
+    FilterStepOutputs
+  >
+  [AutomationActionStepId.QUERY_ROWS]: ActionImplementation<
+    QueryRowsStepInputs,
+    QueryRowsStepOutputs
+  >
+  [AutomationActionStepId.SEND_EMAIL_SMTP]: ActionImplementation<
+    SmtpEmailStepInputs,
+    BaseAutomationOutputs
+  >
+  [AutomationActionStepId.SERVER_LOG]: ActionImplementation<
+    ServerLogStepInputs,
+    ServerLogStepOutputs
+  >
+  [AutomationActionStepId.TRIGGER_AUTOMATION_RUN]: ActionImplementation<
+    TriggerAutomationStepInputs,
+    TriggerAutomationStepOutputs
+  >
+  [AutomationActionStepId.UPDATE_ROW]: ActionImplementation<
+    UpdateRowStepInputs,
+    UpdateRowStepOutputs
+  >
+  [AutomationActionStepId.OUTGOING_WEBHOOK]: ActionImplementation<
+    OutgoingWebhookStepInputs,
+    ExternalAppStepOutputs
+  >
+  [AutomationActionStepId.discord]: ActionImplementation<
+    DiscordStepInputs,
+    ExternalAppStepOutputs
+  >
+  [AutomationActionStepId.slack]: ActionImplementation<
+    SlackStepInputs,
+    ExternalAppStepOutputs
+  >
+
+  [AutomationActionStepId.zapier]: ActionImplementation<
+    ZapierStepInputs,
+    ZapierStepOutputs
+  >
+  [AutomationActionStepId.integromat]: ActionImplementation<
+    MakeIntegrationInputs,
+    ExternalAppStepOutputs
+  >
+  [AutomationActionStepId.n8n]: ActionImplementation<
+    n8nStepInputs,
+    ExternalAppStepOutputs
+  >
+} & (T extends "self"
+  ? {
+      [AutomationActionStepId.EXECUTE_BASH]: ActionImplementation<
+        BashStepInputs,
+        BashStepOutputs
+      >
+      [AutomationActionStepId.OPENAI]: ActionImplementation<
+        OpenAIStepInputs,
+        OpenAIStepOutputs
+      >
+    }
+  : {})
+
+export type BaseAutomationOutputs = {
+  success?: boolean
+  response?: {
+    [key: string]: any
+    message?: string
+  }
+}
+
+export type ExternalAppStepOutputs = {
+  httpStatus?: number
+  response: string
+  success: boolean
+}
+
+export type BashStepInputs = {
+  code: string
+}
+
+export type BashStepOutputs = BaseAutomationOutputs & {
+  stdout?: string
+}
+
+export type CollectStepInputs = {
+  collection: string
+}
+
+export type CollectStepOutputs = BaseAutomationOutputs & {
+  value?: any
+}
+
+export type CreateRowStepInputs = {
+  row: Row
+}
+
+export type CreateRowStepOutputs = BaseAutomationOutputs & {
+  row?: Row
+  id?: string
+  revision?: string
+}
+
+export type DelayStepInputs = {
+  time: number
+}
+
+export type DelayStepOutputs = BaseAutomationOutputs
+
+export type DeleteRowStepInputs = {
+  tableId: string
+  id: string
+  revision?: string
+}
+
+export type DeleteRowStepOutputs = BaseAutomationOutputs & {
+  row?: Row
+}
+
+export type DiscordStepInputs = {
+  url: string
+  username?: string
+  avatar_url?: string
+  content: string
+}
+
+export type ExecuteQueryStepInputs = {
+  query: {
+    queryId: string
+  }
+}
+
+export type ExecuteQueryStepOutputs = BaseAutomationOutputs & {
+  info?: any
+}
+
+export type ExecuteScriptStepInputs = {
+  code: string
+}
+
+export type ExecuteScriptStepOutputs = BaseAutomationOutputs & {
+  value?: string
+}
+
+export type FilterStepInputs = {
+  field: any
+  condition: string
+  value: any
+}
+
+export type FilterStepOutputs = BaseAutomationOutputs & {
+  result: boolean
+  refValue?: any
+  comparisonValue?: any
+}
+
+export type LoopStepInputs = {
+  option: LoopStepType
+  binding: any
+  iterations?: number
+  failure?: string
+}
+
+export type LoopStepOutputs = {
+  items: string
+  success: boolean
+  iterations: number
+}
+
+export type MakeIntegrationInputs = {
+  url: string
+  body: any
+}
+
+export type n8nStepInputs = {
+  url: string
+  method: HttpMethod
+  authorization: string
+  body: any
+}
+
+export type OpenAIStepInputs = {
+  prompt: string
+  model: Model
+}
+
+enum Model {
+  GPT_35_TURBO = "gpt-3.5-turbo",
+  // will only work with api keys that have access to the GPT4 API
+  GPT_4 = "gpt-4",
+}
+
+export type OpenAIStepOutputs = Omit<BaseAutomationOutputs, "response"> & {
+  response?: string | null
+}
+
+export type QueryRowsStepInputs = {
+  tableId: string
+  filters?: SearchFilters
+  "filters-def"?: any
+  sortColumn?: string
+  sortOrder?: SortOrder
+  limit?: number
+  onEmptyFilter?: EmptyFilterOption
+}
+
+export type QueryRowsStepOutputs = BaseAutomationOutputs & {
+  rows?: Row[]
+}
+
+export type SmtpEmailStepInputs = {
+  to: string
+  from: string
+  subject: string
+  contents: string
+  cc: string
+  bcc: string
+  addInvite?: boolean
+  startTime: Date
+  endTime: Date
+  summary: string
+  location?: string
+  url?: string
+  attachments?: EmailAttachment[]
+}
+export type ServerLogStepInputs = {
+  text: string
+}
+
+export type ServerLogStepOutputs = BaseAutomationOutputs & {
+  message: string
+}
+export type SlackStepInputs = {
+  url: string
+  text: string
+}
+
+export type TriggerAutomationStepInputs = {
+  automation: {
+    automationId: string
+  }
+  timeout: number
+}
+
+export type TriggerAutomationStepOutputs = BaseAutomationOutputs & {
+  value?: AutomationResults["steps"]
+}
+
+export type UpdateRowStepInputs = {
+  meta: Record<string, any>
+  row: Row
+  rowId: string
+}
+
+export type UpdateRowStepOutputs = BaseAutomationOutputs & {
+  row?: Row
+  id?: string
+  revision?: string
+}
+
+export type ZapierStepInputs = {
+  url: string
+  body: any
+}
+
+export type ZapierStepOutputs = Omit<ExternalAppStepOutputs, "response"> & {
+  response: string
+}
+
+enum RequestType {
+  POST = "POST",
+  GET = "GET",
+  PUT = "PUT",
+  DELETE = "DELETE",
+  PATCH = "PATCH",
+}
+
+export type OutgoingWebhookStepInputs = {
+  requestMethod: RequestType
+  url: string
+  requestBody: string
+  headers: string
+}

packages/types/src/sdk/row.ts

@@ -30,3 +30,9 @@ export interface SearchResponse<T> {
   bookmark?: string | number
   totalRows?: number
 }
+
+export enum RowExportFormat {
+  CSV = "csv",
+  JSON = "json",
+  JSON_WITH_SCHEMA = "jsonWithSchema",
+}