Merge remote-tracking branch 'origin/v3-ui' into feature/builder-filtering-update

lerna.json

@@ -1,6 +1,6 @@
 {
   "$schema": "node_modules/lerna/schemas/lerna-schema.json",
-  "version": "2.32.4",
+  "version": "2.32.5",
   "npmClient": "yarn",
   "packages": [
     "packages/*",

packages/backend-core/src/sql/sql.ts

@@ -40,7 +40,6 @@ import { dataFilters, helpers } from "@budibase/shared-core"
 import { cloneDeep } from "lodash"
 
 type QueryFunction = (query: SqlQuery | SqlQuery[], operation: Operation) => any
-const MAX_SQS_RELATIONSHIP_FIELDS = 63
 
 function getBaseLimit() {
   const envLimit = environment.SQL_MAX_ROWS
@@ -56,6 +55,20 @@ function getRelationshipLimit() {
   return envLimit || 500
 }
 
+function prioritisedArraySort(toSort: string[], priorities: string[]) {
+  return toSort.sort((a, b) => {
+    const aPriority = priorities.find(field => field && a.endsWith(field))
+    const bPriority = priorities.find(field => field && b.endsWith(field))
+    if (aPriority && !bPriority) {
+      return -1
+    }
+    if (!aPriority && bPriority) {
+      return 1
+    }
+    return a.localeCompare(b)
+  })
+}
+
 function getTableName(table?: Table): string | undefined {
   // SQS uses the table ID rather than the table name
   if (
@@ -877,6 +890,22 @@ class InternalBuilder {
     return `'${unaliased}'${separator}${tableField}`
   }
 
+  maxFunctionParameters() {
+    // functions like say json_build_object() in SQL have a limit as to how many can be performed
+    // before a limit is met, this limit exists in Postgres/SQLite. This can be very important, such as
+    // for JSON column building as part of relationships. We also have a default limit to avoid very complex
+    // functions being built - it is likely this is not necessary or the best way to do it.
+    switch (this.client) {
+      case SqlClient.SQL_LITE:
+        return 127
+      case SqlClient.POSTGRES:
+        return 100
+      // other DBs don't have a limit, but set some sort of limit
+      default:
+        return 200
+    }
+  }
+
   addJsonRelationships(
     query: Knex.QueryBuilder,
     fromTable: string,
@@ -884,7 +913,7 @@ class InternalBuilder {
   ): Knex.QueryBuilder {
     const sqlClient = this.client
     const knex = this.knex
-    const { resource, tableAliases: aliases, endpoint } = this.query
+    const { resource, tableAliases: aliases, endpoint, meta } = this.query
     const fields = resource?.fields || []
     for (let relationship of relationships) {
       const {
@@ -899,21 +928,27 @@ class InternalBuilder {
       if (!toTable || !fromTable) {
         continue
       }
+      const relatedTable = meta.tables?.[toTable]
       const toAlias = aliases?.[toTable] || toTable,
         fromAlias = aliases?.[fromTable] || fromTable
       let toTableWithSchema = this.tableNameWithSchema(toTable, {
         alias: toAlias,
         schema: endpoint.schema,
       })
-      let relationshipFields = fields.filter(
+      const requiredFields = [
-        field => field.split(".")[0] === toAlias
+        ...(relatedTable?.primary || []),
+        relatedTable?.primaryDisplay,
+      ].filter(field => field) as string[]
+      // sort the required fields to first in the list, so they don't get sliced out
+      let relationshipFields = prioritisedArraySort(
+        fields.filter(field => field.split(".")[0] === toAlias),
+        requiredFields
+      )
+
+      relationshipFields = relationshipFields.slice(
+        0,
+        Math.floor(this.maxFunctionParameters() / 2)
       )
-      if (this.client === SqlClient.SQL_LITE) {
-        relationshipFields = relationshipFields.slice(
-          0,
-          MAX_SQS_RELATIONSHIP_FIELDS
-        )
-      }
       const fieldList: string = relationshipFields
         .map(field => this.buildJsonField(field))
         .join(",")

packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte

@@ -602,9 +602,13 @@
     let loopBlockCount = 0
     const addBinding = (name, value, icon, idx, isLoopBlock, bindingName) => {
       if (!name) return
-      const runtimeBinding = determineRuntimeBinding(name, idx, isLoopBlock)
+      const runtimeBinding = determineRuntimeBinding(
+        name,
+        idx,
+        isLoopBlock,
+        bindingName
+      )
       const categoryName = determineCategoryName(idx, isLoopBlock, bindingName)
-
       bindings.push(
         createBindingObject(
           name,
@@ -620,7 +624,7 @@
       )
     }
 
-    const determineRuntimeBinding = (name, idx, isLoopBlock) => {
+    const determineRuntimeBinding = (name, idx, isLoopBlock, bindingName) => {
       let runtimeName
 
       /* Begin special cases for generating custom schemas based on triggers */
@@ -641,12 +645,17 @@
       }
       /* End special cases for generating custom schemas based on triggers */
 
+      let hasUserDefinedName = automation.stepNames?.[allSteps[idx]?.id]
       if (isLoopBlock) {
         runtimeName = `loop.${name}`
       } else if (block.name.startsWith("JS")) {
-        runtimeName = `steps[${idx - loopBlockCount}].${name}`
+        runtimeName = hasUserDefinedName
+          ? `stepsByName[${bindingName}].${name}`
+          : `steps[${idx - loopBlockCount}].${name}`
       } else {
-        runtimeName = `steps.${idx - loopBlockCount}.${name}`
+        runtimeName = hasUserDefinedName
+          ? `stepsByName.${bindingName}.${name}`
+          : `steps.${idx - loopBlockCount}.${name}`
       }
       return idx === 0 ? `trigger.${name}` : runtimeName
     }
@@ -673,11 +682,11 @@
       const field = Object.values(FIELDS).find(
         field => field.type === value.type && field.subtype === value.subtype
       )
-
       return {
-        readableBinding: bindingName
+        readableBinding:
-          ? `${bindingName}.${name}`
+          bindingName && !isLoopBlock
-          : runtimeBinding,
+            ? `steps.${bindingName}.${name}`
+            : runtimeBinding,
         runtimeBinding,
         type: value.type,
         description: value.description,
@@ -697,8 +706,12 @@
         allSteps[idx]?.stepId === ActionStepID.LOOP &&
         allSteps.some(x => x.blockToLoop === block.id)
       let schema = cloneDeep(allSteps[idx]?.schema?.outputs?.properties) ?? {}
+      if (allSteps[idx]?.name.includes("Looping")) {
+        isLoopBlock = true
+        loopBlockCount++
+      }
       let bindingName =
-        automation.stepNames?.[allSteps[idx - loopBlockCount].id]
+        automation.stepNames?.[allSteps[idx].id] || allSteps[idx].name
 
       if (isLoopBlock) {
         schema = {
@@ -747,13 +760,12 @@
 
       if (wasLoopBlock) {
         loopBlockCount++
-        continue
+        schema = cloneDeep(allSteps[idx - 1]?.schema?.outputs?.properties)
       }
-      Object.entries(schema).forEach(([name, value]) =>
+      Object.entries(schema).forEach(([name, value]) => {
         addBinding(name, value, icon, idx, isLoopBlock, bindingName)
-      )
+      })
     }
-
     return bindings
   }
 

packages/builder/src/helpers/automations/nameHelpers.js

@@ -0,0 +1,117 @@
+import { AutomationActionStepId } from "@budibase/types"
+
+export const updateBindingsInInputs = (inputs, oldName, newName, stepIndex) => {
+  if (typeof inputs === "string") {
+    return inputs
+      .replace(
+        new RegExp(`stepsByName\\.${oldName}\\.`, "g"),
+        `stepsByName.${newName}.`
+      )
+      .replace(
+        new RegExp(`steps\\.${stepIndex}\\.`, "g"),
+        `stepsByName.${newName}.`
+      )
+  }
+
+  if (Array.isArray(inputs)) {
+    return inputs.map(item =>
+      updateBindingsInInputs(item, oldName, newName, stepIndex)
+    )
+  }
+
+  if (typeof inputs === "object" && inputs !== null) {
+    const updatedInputs = {}
+    for (const [key, value] of Object.entries(inputs)) {
+      const updatedKey = updateBindingsInInputs(
+        key,
+        oldName,
+        newName,
+        stepIndex
+      )
+      updatedInputs[updatedKey] = updateBindingsInInputs(
+        value,
+        oldName,
+        newName,
+        stepIndex
+      )
+    }
+    return updatedInputs
+  }
+  return inputs
+}
+
+export const updateBindingsInSteps = (
+  steps,
+  oldName,
+  newName,
+  changedStepIndex
+) => {
+  return steps.map(step => {
+    const updatedStep = {
+      ...step,
+      inputs: updateBindingsInInputs(
+        step.inputs,
+        oldName,
+        newName,
+        changedStepIndex
+      ),
+    }
+
+    if ("branches" in updatedStep.inputs) {
+      updatedStep.inputs.branches = updatedStep.inputs.branches.map(branch => ({
+        ...branch,
+        condition: updateBindingsInInputs(
+          branch.condition,
+          oldName,
+          newName,
+          changedStepIndex
+        ),
+      }))
+
+      if (updatedStep.inputs.children) {
+        for (const [key, childSteps] of Object.entries(
+          updatedStep.inputs.children
+        )) {
+          updatedStep.inputs.children[key] = updateBindingsInSteps(
+            childSteps,
+            oldName,
+            newName,
+            changedStepIndex
+          )
+        }
+      }
+    }
+
+    return updatedStep
+  })
+}
+export const getNewStepName = (automation, step) => {
+  const baseName = step.name
+
+  const countExistingSteps = steps => {
+    return steps.reduce((count, currentStep) => {
+      if (currentStep.name && currentStep.name.startsWith(baseName)) {
+        count++
+      }
+      if (
+        currentStep.stepId === AutomationActionStepId.BRANCH &&
+        currentStep.inputs &&
+        currentStep.inputs.children
+      ) {
+        Object.values(currentStep.inputs.children).forEach(branchSteps => {
+          count += countExistingSteps(branchSteps)
+        })
+      }
+      return count
+    }, 0)
+  }
+  let existingCount = 0
+  if (automation?.definition) {
+    existingCount = countExistingSteps(automation.definition.steps)
+  }
+  if (existingCount === 0) {
+    return baseName
+  }
+
+  return `${baseName} ${existingCount + 1}`
+}

packages/builder/src/helpers/tests/nameHelpers.spec.js

@@ -0,0 +1,177 @@
+import { cloneDeep } from "lodash"
+import {
+  updateBindingsInInputs,
+  updateBindingsInSteps,
+} from "../automations/nameHelpers"
+describe("Automation Binding Update Functions", () => {
+  const sampleAutomation = {
+    definition: {
+      steps: [
+        {
+          name: "First Step",
+          inputs: {
+            text: "Starting automation",
+          },
+          id: "step1",
+        },
+        {
+          name: "Second Step",
+          inputs: {
+            text: "{{ steps.0.success }} and {{ stepsByName.First Step.message }}",
+          },
+          id: "step2",
+        },
+        {
+          name: "Branch",
+          inputs: {
+            branches: [
+              {
+                name: "branch1",
+                condition: {
+                  equal: {
+                    "steps.1.success": true,
+                  },
+                },
+              },
+            ],
+            children: {
+              branch1: [
+                {
+                  name: "Nested Step",
+                  inputs: {
+                    text: "{{ stepsByName.Second Step.message }} and {{ steps.1.success }}",
+                  },
+                  id: "nestedStep",
+                },
+              ],
+            },
+          },
+          id: "branchStep",
+        },
+      ],
+      stepNames: {
+        step1: "First Step",
+        step2: "Second Step",
+        branchStep: "Branch",
+      },
+    },
+  }
+
+  it("updateBindingsInInputs updates string bindings correctly", () => {
+    const input = "{{ stepsByName.oldName.success }} and {{ steps.1.message }}"
+    const result = updateBindingsInInputs(input, "oldName", "newName", 1)
+    expect(result).toBe(
+      "{{ stepsByName.newName.success }} and {{ stepsByName.newName.message }}"
+    )
+  })
+
+  it("updateBindingsInInputs handles nested objects", () => {
+    const input = {
+      text: "{{ stepsByName.oldName.success }}",
+      nested: {
+        value: "{{ steps.1.message }}",
+      },
+    }
+    const result = updateBindingsInInputs(input, "oldName", "newName", 1)
+    expect(result).toEqual({
+      text: "{{ stepsByName.newName.success }}",
+      nested: {
+        value: "{{ stepsByName.newName.message }}",
+      },
+    })
+  })
+
+  it("updateBindingsInSteps updates bindings in all steps", () => {
+    const steps = cloneDeep(sampleAutomation.definition.steps)
+    const result = updateBindingsInSteps(
+      steps,
+      "Second Step",
+      "Renamed Step",
+      1
+    )
+
+    expect(result[1].name).toBe("Second Step")
+
+    expect(result[2].inputs.branches[0].condition.equal).toEqual({
+      "stepsByName.Renamed Step.success": true,
+    })
+
+    const nestedStepText = result[2].inputs.children.branch1[0].inputs.text
+    expect(nestedStepText).toBe(
+      "{{ stepsByName.Renamed Step.message }} and {{ stepsByName.Renamed Step.success }}"
+    )
+  })
+
+  it("updateBindingsInSteps handles steps with no bindings", () => {
+    const steps = [
+      {
+        name: "No Binding Step",
+        inputs: {
+          text: "Plain text",
+        },
+        id: "noBindingStep",
+      },
+    ]
+    const result = updateBindingsInSteps(steps, "Old Name", "New Name", 0)
+    expect(result).toEqual(steps)
+  })
+
+  it("updateBindingsInSteps updates bindings in deeply nested branches", () => {
+    const deeplyNestedStep = {
+      name: "Deep Branch",
+      inputs: {
+        branches: [
+          {
+            name: "deepBranch",
+            condition: {
+              equal: {
+                "stepsByName.Second Step.success": true,
+              },
+            },
+          },
+        ],
+        children: {
+          deepBranch: [
+            {
+              name: "Deep Log",
+              inputs: {
+                text: "{{ steps.1.message }}",
+              },
+            },
+          ],
+        },
+      },
+    }
+
+    const steps = [...sampleAutomation.definition.steps, deeplyNestedStep]
+    const result = updateBindingsInSteps(
+      steps,
+      "Second Step",
+      "Renamed Step",
+      1
+    )
+
+    expect(
+      result[3].inputs.branches[0].condition.equal[
+        "stepsByName.Renamed Step.success"
+      ]
+    ).toBe(true)
+    expect(result[3].inputs.children.deepBranch[0].inputs.text).toBe(
+      "{{ stepsByName.Renamed Step.message }}"
+    )
+  })
+
+  it("updateBindingsInSteps does not affect unrelated bindings", () => {
+    const steps = cloneDeep(sampleAutomation.definition.steps)
+    const result = updateBindingsInSteps(
+      steps,
+      "Second Step",
+      "Renamed Step",
+      1
+    )
+
+    expect(result[1].inputs.text).toBe(
+      "{{ steps.0.success }} and {{ stepsByName.First Step.message }}"
+    )
+  })
+})

packages/builder/src/stores/builder/automations.js

@@ -8,6 +8,10 @@ import { updateReferencesInObject } from "dataBinding"
 import { AutomationTriggerStepId } from "@budibase/types"
 import { sdk } from "@budibase/shared-core"
 import { rowActions } from "./rowActions"
+import {
+  updateBindingsInSteps,
+  getNewStepName,
+} from "helpers/automations/nameHelpers"
 
 const initialAutomationState = {
   automations: [],
@@ -285,13 +289,17 @@ const automationActions = store => ({
     await store.actions.save(newAutomation)
   },
   constructBlock(type, stepId, blockDefinition) {
-    return {
+    let newName
+    const newStep = {
       ...blockDefinition,
       inputs: blockDefinition.inputs || {},
       stepId,
       type,
       id: generate(),
     }
+    newName = getNewStepName(get(selectedAutomation), newStep)
+    newStep.name = newName
+    return newStep
   },
   addBlockToAutomation: async (block, blockIdx) => {
     const automation = get(selectedAutomation)
@@ -311,15 +319,34 @@ const automationActions = store => ({
   saveAutomationName: async (blockId, name) => {
     const automation = get(selectedAutomation)
     let newAutomation = cloneDeep(automation)
-    if (!automation) {
+    if (!newAutomation) {
       return
     }
-    newAutomation.definition.stepNames = {
-      ...newAutomation.definition.stepNames,
-      [blockId]: name.trim(),
-    }
 
-    await store.actions.save(newAutomation)
+    const stepIndex = newAutomation.definition.steps.findIndex(
+      step => step.id === blockId
+    )
+
+    if (stepIndex !== -1) {
+      const oldName = newAutomation.definition.steps[stepIndex].name
+      const newName = name.trim()
+
+      newAutomation.definition.stepNames = {
+        ...newAutomation.definition.stepNames,
+        [blockId]: newName,
+      }
+
+      newAutomation.definition.steps[stepIndex].name = newName
+
+      newAutomation.definition.steps = updateBindingsInSteps(
+        newAutomation.definition.steps,
+        oldName,
+        newName,
+        stepIndex
+      )
+
+      await store.actions.save(newAutomation)
+    }
   },
   deleteAutomationName: async blockId => {
     const automation = get(selectedAutomation)

packages/server/src/api/controllers/row/ExternalRequest.ts

@@ -12,6 +12,7 @@ import {
   OneToManyRelationshipFieldMetadata,
   Operation,
   PaginationJson,
+  QueryJson,
   RelationshipFieldMetadata,
   Row,
   SearchFilters,
@@ -161,7 +162,6 @@ export class ExternalRequest<T extends Operation> {
   private readonly tableId: string
   private datasource?: Datasource
   private tables: { [key: string]: Table } = {}
-  private tableList: Table[]
 
   constructor(operation: T, tableId: string, datasource?: Datasource) {
     this.operation = operation
@@ -170,7 +170,6 @@ export class ExternalRequest<T extends Operation> {
     if (datasource && datasource.entities) {
       this.tables = datasource.entities
     }
-    this.tableList = Object.values(this.tables)
   }
 
   private prepareFilters(
@@ -301,7 +300,6 @@ export class ExternalRequest<T extends Operation> {
         throw "No tables found, fetch tables before query."
       }
       this.tables = this.datasource.entities
-      this.tableList = Object.values(this.tables)
     }
     return { tables: this.tables, datasource: this.datasource }
   }
@@ -463,7 +461,7 @@ export class ExternalRequest<T extends Operation> {
         breakExternalTableId(relatedTableId)
       // @ts-ignore
       const linkPrimaryKey = this.tables[relatedTableName].primary[0]
-      if (!lookupField || !row[lookupField]) {
+      if (!lookupField || !row?.[lookupField] == null) {
         continue
       }
       const endpoint = getEndpoint(relatedTableId, Operation.READ)
@@ -631,7 +629,8 @@ export class ExternalRequest<T extends Operation> {
       const { datasource: ds } = await this.retrieveMetadata(datasourceId)
       datasource = ds
     }
-    const table = this.tables[tableName]
+    const tables = this.tables
+    const table = tables[tableName]
     let isSql = isSQL(datasource)
     if (!table) {
       throw new Error(
@@ -686,7 +685,7 @@ export class ExternalRequest<T extends Operation> {
     ) {
       throw "Deletion must be filtered"
     }
-    let json = {
+    let json: QueryJson = {
       endpoint: {
         datasourceId: datasourceId!,
         entityId: tableName,
@@ -715,7 +714,7 @@ export class ExternalRequest<T extends Operation> {
       },
       meta: {
         table,
-        id: config.id,
+        tables: tables,
       },
     }
 

packages/server/src/api/routes/tests/search.spec.ts

@@ -3080,4 +3080,46 @@ describe.each([
         }).toHaveLength(4)
       })
     })
+
+  isSql &&
+    describe("max related columns", () => {
+      let relatedRows: Row[]
+
+      beforeAll(async () => {
+        const relatedSchema: TableSchema = {}
+        const row: Row = {}
+        for (let i = 0; i < 100; i++) {
+          const name = `column${i}`
+          relatedSchema[name] = { name, type: FieldType.NUMBER }
+          row[name] = i
+        }
+        const relatedTable = await createTable(relatedSchema)
+        table = await createTable({
+          name: { name: "name", type: FieldType.STRING },
+          related1: {
+            type: FieldType.LINK,
+            name: "related1",
+            fieldName: "main1",
+            tableId: relatedTable._id!,
+            relationshipType: RelationshipType.MANY_TO_MANY,
+          },
+        })
+        relatedRows = await Promise.all([
+          config.api.row.save(relatedTable._id!, row),
+        ])
+        await config.api.row.save(table._id!, {
+          name: "foo",
+          related1: [relatedRows[0]._id],
+        })
+      })
+
+      it("retrieve the row with relationships", async () => {
+        await expectQuery({}).toContainExactly([
+          {
+            name: "foo",
+            related1: [{ _id: relatedRows[0]._id }],
+          },
+        ])
+      })
+    })
 })

packages/server/src/automations/tests/scenarios/looping.spec.ts

@@ -242,4 +242,31 @@ describe("Loop automations", () => {
     expect(results.steps[1].outputs.message).toContain("- 3")
     expect(results.steps[3].outputs.message).toContain("- 3")
   })
+
+  it("should use automation names to loop with", async () => {
+    const builder = createAutomationBuilder({
+      name: "Test Trigger with Loop and Create Row",
+    })
+
+    const results = await builder
+      .appAction({ fields: {} })
+      .loop(
+        {
+          option: LoopStepType.ARRAY,
+          binding: [1, 2, 3],
+        },
+        { stepName: "FirstLoopStep" }
+      )
+      .serverLog(
+        { text: "Message {{loop.currentItem}}" },
+        { stepName: "FirstLoopLog" }
+      )
+      .serverLog(
+        { text: "{{steps.FirstLoopLog.iterations}}" },
+        { stepName: "FirstLoopIterationLog" }
+      )
+      .run()
+
+    expect(results.steps[1].outputs.message).toContain("- 3")
+  })
 })

packages/server/src/automations/tests/scenarios/scenarios.spec.ts

@@ -49,151 +49,188 @@ describe("Automation Scenarios", () => {
         },
       })
     })
-  })
 
-  it("should trigger an automation which querys the database", async () => {
+    it("should trigger an automation which querys the database", async () => {
-    const table = await config.createTable()
+      const table = await config.createTable()
-    const row = {
+      const row = {
-      name: "Test Row",
+        name: "Test Row",
-      description: "original description",
+        description: "original description",
-      tableId: table._id,
+        tableId: table._id,
-    }
+      }
-    await config.createRow(row)
+      await config.createRow(row)
-    await config.createRow(row)
+      await config.createRow(row)
-    const builder = createAutomationBuilder({
+      const builder = createAutomationBuilder({
-      name: "Test Row Save and Create",
+        name: "Test Row Save and Create",
+      })
+
+      const results = await builder
+        .appAction({ fields: {} })
+        .queryRows({
+          tableId: table._id!,
+        })
+        .run()
+
+      expect(results.steps).toHaveLength(1)
+      expect(results.steps[0].outputs.rows).toHaveLength(2)
     })
 
-    const results = await builder
+    it("should trigger an automation which querys the database then deletes a row", async () => {
-      .appAction({ fields: {} })
+      const table = await config.createTable()
-      .queryRows({
+      const row = {
-        tableId: table._id!,
+        name: "DFN",
+        description: "original description",
+        tableId: table._id,
+      }
+      await config.createRow(row)
+      await config.createRow(row)
+      const builder = createAutomationBuilder({
+        name: "Test Row Save and Create",
       })
-      .run()
 
-    expect(results.steps).toHaveLength(1)
+      const results = await builder
-    expect(results.steps[0].outputs.rows).toHaveLength(2)
+        .appAction({ fields: {} })
-  })
+        .queryRows({
+          tableId: table._id!,
+        })
+        .deleteRow({
+          tableId: table._id!,
+          id: "{{ steps.1.rows.0._id }}",
+        })
+        .queryRows({
+          tableId: table._id!,
+        })
+        .run()
 
-  it("should trigger an automation which querys the database then deletes a row", async () => {
+      expect(results.steps).toHaveLength(3)
-    const table = await config.createTable()
+      expect(results.steps[1].outputs.success).toBeTruthy()
-    const row = {
+      expect(results.steps[2].outputs.rows).toHaveLength(1)
-      name: "DFN",
-      description: "original description",
-      tableId: table._id,
-    }
-    await config.createRow(row)
-    await config.createRow(row)
-    const builder = createAutomationBuilder({
-      name: "Test Row Save and Create",
     })
 
-    const results = await builder
+    it("should query an external database for some data then insert than into an internal table", async () => {
-      .appAction({ fields: {} })
+      const { datasource, client } = await setup.setupTestDatasource(
-      .queryRows({
+        config,
-        tableId: table._id!,
+        DatabaseName.MYSQL
-      })
-      .deleteRow({
-        tableId: table._id!,
-        id: "{{ steps.1.rows.0._id }}",
-      })
-      .queryRows({
-        tableId: table._id!,
-      })
-      .run()
-
-    expect(results.steps).toHaveLength(3)
-    expect(results.steps[1].outputs.success).toBeTruthy()
-    expect(results.steps[2].outputs.rows).toHaveLength(1)
-  })
-
-  it("should query an external database for some data then insert than into an internal table", async () => {
-    const { datasource, client } = await setup.setupTestDatasource(
-      config,
-      DatabaseName.MYSQL
-    )
-
-    const newTable = await config.createTable({
-      name: "table",
-      type: "table",
-      schema: {
-        name: {
-          name: "name",
-          type: FieldType.STRING,
-          constraints: {
-            presence: true,
-          },
-        },
-        age: {
-          name: "age",
-          type: FieldType.NUMBER,
-          constraints: {
-            presence: true,
-          },
-        },
-      },
-    })
-
-    const tableName = await setup.createTestTable(client, {
-      name: { type: "string" },
-      age: { type: "number" },
-    })
-
-    const rows = [
-      { name: "Joe", age: 20 },
-      { name: "Bob", age: 25 },
-      { name: "Paul", age: 30 },
-    ]
-
-    await setup.insertTestData(client, tableName, rows)
-
-    const query = await setup.saveTestQuery(
-      config,
-      client,
-      tableName,
-      datasource
-    )
-
-    const builder = createAutomationBuilder({
-      name: "Test external query and save",
-    })
-
-    const results = await builder
-      .appAction({
-        fields: {},
-      })
-      .executeQuery({
-        query: {
-          queryId: query._id!,
-        },
-      })
-      .loop({
-        option: LoopStepType.ARRAY,
-        binding: "{{ steps.1.response }}",
-      })
-      .createRow({
-        row: {
-          name: "{{ loop.currentItem.name }}",
-          age: "{{ loop.currentItem.age }}",
-          tableId: newTable._id!,
-        },
-      })
-      .queryRows({
-        tableId: newTable._id!,
-      })
-      .run()
-
-    expect(results.steps).toHaveLength(3)
-
-    expect(results.steps[1].outputs.iterations).toBe(3)
-    expect(results.steps[1].outputs.items).toHaveLength(3)
-
-    expect(results.steps[2].outputs.rows).toHaveLength(3)
-
-    rows.forEach(expectedRow => {
-      expect(results.steps[2].outputs.rows).toEqual(
-        expect.arrayContaining([expect.objectContaining(expectedRow)])
       )
+
+      const newTable = await config.createTable({
+        name: "table",
+        type: "table",
+        schema: {
+          name: {
+            name: "name",
+            type: FieldType.STRING,
+            constraints: {
+              presence: true,
+            },
+          },
+          age: {
+            name: "age",
+            type: FieldType.NUMBER,
+            constraints: {
+              presence: true,
+            },
+          },
+        },
+      })
+
+      const tableName = await setup.createTestTable(client, {
+        name: { type: "string" },
+        age: { type: "number" },
+      })
+
+      const rows = [
+        { name: "Joe", age: 20 },
+        { name: "Bob", age: 25 },
+        { name: "Paul", age: 30 },
+      ]
+
+      await setup.insertTestData(client, tableName, rows)
+
+      const query = await setup.saveTestQuery(
+        config,
+        client,
+        tableName,
+        datasource
+      )
+
+      const builder = createAutomationBuilder({
+        name: "Test external query and save",
+      })
+
+      const results = await builder
+        .appAction({
+          fields: {},
+        })
+        .executeQuery({
+          query: {
+            queryId: query._id!,
+          },
+        })
+        .loop({
+          option: LoopStepType.ARRAY,
+          binding: "{{ steps.1.response }}",
+        })
+        .createRow({
+          row: {
+            name: "{{ loop.currentItem.name }}",
+            age: "{{ loop.currentItem.age }}",
+            tableId: newTable._id!,
+          },
+        })
+        .queryRows({
+          tableId: newTable._id!,
+        })
+        .run()
+
+      expect(results.steps).toHaveLength(3)
+
+      expect(results.steps[1].outputs.iterations).toBe(3)
+      expect(results.steps[1].outputs.items).toHaveLength(3)
+
+      expect(results.steps[2].outputs.rows).toHaveLength(3)
+
+      rows.forEach(expectedRow => {
+        expect(results.steps[2].outputs.rows).toEqual(
+          expect.arrayContaining([expect.objectContaining(expectedRow)])
+        )
+      })
+    })
+  })
+
+  describe("Name Based Automations", () => {
+    it("should fetch and delete a rpw using automation naming", async () => {
+      const table = await config.createTable()
+      const row = {
+        name: "DFN",
+        description: "original description",
+        tableId: table._id,
+      }
+      await config.createRow(row)
+      await config.createRow(row)
+      const builder = createAutomationBuilder({
+        name: "Test Query and Delete Row",
+      })
+
+      const results = await builder
+        .appAction({ fields: {} })
+        .queryRows(
+          {
+            tableId: table._id!,
+          },
+          { stepName: "InitialQueryStep" }
+        )
+        .deleteRow({
+          tableId: table._id!,
+          id: "{{ steps.InitialQueryStep.rows.0._id }}",
+        })
+        .queryRows({
+          tableId: table._id!,
+        })
+        .run()
+
+      expect(results.steps).toHaveLength(3)
+      expect(results.steps[1].outputs.success).toBeTruthy()
+      expect(results.steps[2].outputs.rows).toHaveLength(1)
     })
   })
 })

packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts

@@ -57,21 +57,27 @@ type BranchConfig = {
 
 class BaseStepBuilder {
   protected steps: AutomationStep[] = []
+  protected stepNames: { [key: string]: string } = {}
 
   protected step<TStep extends AutomationActionStepId>(
     stepId: TStep,
     stepSchema: Omit<AutomationStep, "id" | "stepId" | "inputs">,
-    inputs: AutomationStepInputs<TStep>
+    inputs: AutomationStepInputs<TStep>,
+    stepName?: string
   ): this {
+    const id = uuidv4()
     this.steps.push({
       ...stepSchema,
       inputs: inputs as any,
-      id: uuidv4(),
+      id,
       stepId,
+      name: stepName || stepSchema.name,
     })
+    if (stepName) {
+      this.stepNames[id] = stepName
+    }
     return this
   }
-
   protected addBranchStep(branchConfig: BranchConfig): void {
     const branchStepInputs: BranchStepInputs = {
       branches: [] as Branch[],
@@ -99,66 +105,80 @@ class BaseStepBuilder {
   }
 
   // STEPS
-  createRow(inputs: CreateRowStepInputs): this {
+  createRow(inputs: CreateRowStepInputs, opts?: { stepName?: string }): this {
     return this.step(
       AutomationActionStepId.CREATE_ROW,
       BUILTIN_ACTION_DEFINITIONS.CREATE_ROW,
-      inputs
+      inputs,
+      opts?.stepName
     )
   }
 
-  updateRow(inputs: UpdateRowStepInputs): this {
+  updateRow(inputs: UpdateRowStepInputs, opts?: { stepName?: string }): this {
     return this.step(
       AutomationActionStepId.UPDATE_ROW,
       BUILTIN_ACTION_DEFINITIONS.UPDATE_ROW,
-      inputs
+      inputs,
+      opts?.stepName
     )
   }
 
-  deleteRow(inputs: DeleteRowStepInputs): this {
+  deleteRow(inputs: DeleteRowStepInputs, opts?: { stepName?: string }): this {
     return this.step(
       AutomationActionStepId.DELETE_ROW,
       BUILTIN_ACTION_DEFINITIONS.DELETE_ROW,
-      inputs
+      inputs,
+      opts?.stepName
     )
   }
 
-  sendSmtpEmail(inputs: SmtpEmailStepInputs): this {
+  sendSmtpEmail(
+    inputs: SmtpEmailStepInputs,
+    opts?: { stepName?: string }
+  ): this {
     return this.step(
       AutomationActionStepId.SEND_EMAIL_SMTP,
       BUILTIN_ACTION_DEFINITIONS.SEND_EMAIL_SMTP,
-      inputs
+      inputs,
+      opts?.stepName
     )
   }
 
-  executeQuery(inputs: ExecuteQueryStepInputs): this {
+  executeQuery(
+    inputs: ExecuteQueryStepInputs,
+    opts?: { stepName?: string }
+  ): this {
     return this.step(
       AutomationActionStepId.EXECUTE_QUERY,
       BUILTIN_ACTION_DEFINITIONS.EXECUTE_QUERY,
-      inputs
+      inputs,
+      opts?.stepName
     )
   }
 
-  queryRows(inputs: QueryRowsStepInputs): this {
+  queryRows(inputs: QueryRowsStepInputs, opts?: { stepName?: string }): this {
     return this.step(
       AutomationActionStepId.QUERY_ROWS,
       BUILTIN_ACTION_DEFINITIONS.QUERY_ROWS,
-      inputs
+      inputs,
+      opts?.stepName
     )
   }
-  loop(inputs: LoopStepInputs): this {
+  loop(inputs: LoopStepInputs, opts?: { stepName?: string }): this {
     return this.step(
       AutomationActionStepId.LOOP,
       BUILTIN_ACTION_DEFINITIONS.LOOP,
-      inputs
+      inputs,
+      opts?.stepName
     )
   }
 
-  serverLog(input: ServerLogStepInputs): this {
+  serverLog(input: ServerLogStepInputs, opts?: { stepName?: string }): this {
     return this.step(
       AutomationActionStepId.SERVER_LOG,
       BUILTIN_ACTION_DEFINITIONS.SERVER_LOG,
-      input
+      input,
+      opts?.stepName
     )
   }
 }
@@ -186,6 +206,7 @@ class AutomationBuilder extends BaseStepBuilder {
       definition: {
         steps: [],
         trigger: {} as AutomationTrigger,
+        stepNames: {},
       },
       type: "automation",
       appId: options.appId ?? setup.getConfig().getAppId(),
@@ -268,6 +289,7 @@ class AutomationBuilder extends BaseStepBuilder {
 
   build(): Automation {
     this.automationConfig.definition.steps = this.steps
+    this.automationConfig.definition.stepNames = this.stepNames
     return this.automationConfig
   }
 

packages/server/src/definitions/automations.ts

@@ -15,6 +15,7 @@ export interface TriggerOutput {
 
 export interface AutomationContext extends AutomationResults {
   steps: any[]
+  stepsByName?: Record<string, any>
   env?: Record<string, string>
   trigger: any
 }

packages/server/src/integrations/googlesheets.ts

@@ -551,11 +551,16 @@ export class GoogleSheetsIntegration implements DatasourcePlus {
       await this.connect()
       const hasFilters = dataFilters.hasFilters(query.filters)
       const limit = query.paginate?.limit || 100
-      const page: number =
+      let offset = query.paginate?.offset || 0
-        typeof query.paginate?.page === "number"
+
-          ? query.paginate.page
+      let page = query.paginate?.page
-          : parseInt(query.paginate?.page || "1")
+      if (typeof page === "string") {
-      const offset = (page - 1) * limit
+        page = parseInt(page)
+      }
+      if (page !== undefined) {
+        offset = page * limit
+      }
+
       const sheet = this.client.sheetsByTitle[query.sheet]
       let rows: GoogleSpreadsheetRow[] = []
       if (query.paginate && !hasFilters) {

packages/server/src/integrations/tests/googlesheets.spec.ts

@@ -208,6 +208,42 @@ describe("Google Sheets Integration", () => {
       expect(row2.name).toEqual("Test Contact 2")
       expect(row2.description).toEqual("original description 2")
     })
+
+    it("can paginate correctly", async () => {
+      await config.api.row.bulkImport(table._id!, {
+        rows: Array.from({ length: 248 }, (_, i) => ({
+          name: `${i}`,
+          description: "",
+        })),
+      })
+
+      let resp = await config.api.row.search(table._id!, {
+        tableId: table._id!,
+        query: {},
+        paginate: true,
+        limit: 10,
+      })
+      let rows = resp.rows
+
+      while (resp.hasNextPage) {
+        resp = await config.api.row.search(table._id!, {
+          tableId: table._id!,
+          query: {},
+          paginate: true,
+          limit: 10,
+          bookmark: resp.bookmark,
+        })
+        rows = rows.concat(resp.rows)
+        if (rows.length > 250) {
+          throw new Error("Too many rows returned")
+        }
+      }
+
+      expect(rows.length).toEqual(250)
+      expect(rows.map(row => row.name)).toEqual(
+        expect.arrayContaining(Array.from({ length: 248 }, (_, i) => `${i}`))
+      )
+    })
   })
 
   describe("update", () => {

packages/server/src/integrations/tests/sql.spec.ts

@@ -162,7 +162,7 @@ describe("SQL query builder", () => {
     const query = sql._query(generateRelationshipJson({ schema: "production" }))
     expect(query).toEqual({
       bindings: [limit, relationshipLimit],
-      sql: `with "paginated" as (select "brands".* from "production"."brands" order by "test"."id" asc limit $1) select "brands".*, (select json_agg(json_build_object('product_id',"products"."product_id",'product_name',"products"."product_name",'brand_id',"products"."brand_id")) from (select "products".* from "production"."products" as "products" where "products"."brand_id" = "brands"."brand_id" order by "products"."brand_id" asc limit $2) as "products") as "products" from "paginated" as "brands" order by "test"."id" asc`,
+      sql: `with "paginated" as (select "brands".* from "production"."brands" order by "test"."id" asc limit $1) select "brands".*, (select json_agg(json_build_object('brand_id',"products"."brand_id",'product_id',"products"."product_id",'product_name',"products"."product_name")) from (select "products".* from "production"."products" as "products" where "products"."brand_id" = "brands"."brand_id" order by "products"."brand_id" asc limit $2) as "products") as "products" from "paginated" as "brands" order by "test"."id" asc`,
     })
   })
 
@@ -170,7 +170,7 @@ describe("SQL query builder", () => {
     const query = sql._query(generateRelationshipJson())
     expect(query).toEqual({
       bindings: [limit, relationshipLimit],
-      sql: `with "paginated" as (select "brands".* from "brands" order by "test"."id" asc limit $1) select "brands".*, (select json_agg(json_build_object('product_id',"products"."product_id",'product_name',"products"."product_name",'brand_id',"products"."brand_id")) from (select "products".* from "products" as "products" where "products"."brand_id" = "brands"."brand_id" order by "products"."brand_id" asc limit $2) as "products") as "products" from "paginated" as "brands" order by "test"."id" asc`,
+      sql: `with "paginated" as (select "brands".* from "brands" order by "test"."id" asc limit $1) select "brands".*, (select json_agg(json_build_object('brand_id',"products"."brand_id",'product_id',"products"."product_id",'product_name',"products"."product_name")) from (select "products".* from "products" as "products" where "products"."brand_id" = "brands"."brand_id" order by "products"."brand_id" asc limit $2) as "products") as "products" from "paginated" as "brands" order by "test"."id" asc`,
     })
   })
 

packages/server/src/integrations/tests/sqlAlias.spec.ts

@@ -63,7 +63,7 @@ describe("Captures of real examples", () => {
         bindings: [primaryLimit, relationshipLimit, relationshipLimit],
         sql: expect.stringContaining(
           multiline(
-            `select json_agg(json_build_object('executorid',"b"."executorid",'taskname',"b"."taskname",'taskid',"b"."taskid",'completed',"b"."completed",'qaid',"b"."qaid",'executorid',"b"."executorid",'taskname',"b"."taskname",'taskid',"b"."taskid",'completed',"b"."completed",'qaid',"b"."qaid")`
+            `select json_agg(json_build_object('completed',"b"."completed",'completed',"b"."completed",'executorid',"b"."executorid",'executorid',"b"."executorid",'qaid',"b"."qaid",'qaid',"b"."qaid",'taskid',"b"."taskid",'taskid',"b"."taskid",'taskname',"b"."taskname",'taskname',"b"."taskname")`
           )
         ),
       })
@@ -95,7 +95,7 @@ describe("Captures of real examples", () => {
         sql: expect.stringContaining(
           multiline(
             `with "paginated" as (select "a".* from "products" as "a" order by "a"."productname" asc nulls first, "a"."productid" asc limit $1) 
-                 select "a".*, (select json_agg(json_build_object('executorid',"b"."executorid",'taskname',"b"."taskname",'taskid',"b"."taskid",'completed',"b"."completed",'qaid',"b"."qaid")) 
+                 select "a".*, (select json_agg(json_build_object('completed',"b"."completed",'executorid',"b"."executorid",'qaid',"b"."qaid",'taskid',"b"."taskid",'taskname',"b"."taskname")) 
                  from (select "b".* from "tasks" as "b" inner join "products_tasks" as "c" on "b"."taskid" = "c"."taskid" where "c"."productid" = "a"."productid" order by "b"."taskid" asc limit $2) as "b") as "tasks" 
                  from "paginated" as "a" order by "a"."productname" asc nulls first, "a"."productid" asc`
           )
@@ -113,7 +113,7 @@ describe("Captures of real examples", () => {
         bindings: [...filters, relationshipLimit, relationshipLimit],
         sql: multiline(
           `with "paginated" as (select "a".* from "tasks" as "a" where "a"."taskid" in ($1, $2) order by "a"."taskid" asc limit $3) 
-               select "a".*, (select json_agg(json_build_object('productname',"b"."productname",'productid',"b"."productid")) 
+               select "a".*, (select json_agg(json_build_object('productid',"b"."productid",'productname',"b"."productname")) 
                from (select "b".* from "products" as "b" inner join "products_tasks" as "c" on "b"."productid" = "c"."productid" 
                where "c"."taskid" = "a"."taskid" order by "b"."productid" asc limit $4) as "b") as "products" from "paginated" as "a" order by "a"."taskid" asc`
         ),

packages/server/src/threads/automation.ts

@@ -89,7 +89,12 @@ class Orchestrator {
     delete triggerOutput.appId
     delete triggerOutput.metadata
     // step zero is never used as the template string is zero indexed for customer facing
-    this.context = { steps: [{}], trigger: triggerOutput }
+    this.context = {
+      steps: [{}],
+      stepsByName: {},
+      trigger: triggerOutput,
+    }
+
     this.automation = automation
     // create an emitter which has the chain count for this automation run in it, so it can block
     // excessive chaining if required
@@ -451,6 +456,9 @@ class Orchestrator {
         outputs: tempOutput,
         inputs: steps[stepToLoopIndex].inputs,
       })
+
+      const stepName = steps[stepToLoopIndex].name || steps[stepToLoopIndex].id
+      this.context.stepsByName![stepName] = tempOutput
       this.context.steps[this.context.steps.length] = tempOutput
       this.context.steps = this.context.steps.filter(
         item => !item.hasOwnProperty.call(item, "currentItem")
@@ -555,8 +563,13 @@ class Orchestrator {
             loopIteration
           )
         }
+
         const stepFn = await this.getStepFunctionality(step.stepId)
-        let inputs = await processObject(originalStepInput, this.context)
+        let inputs = await this.addContextAndProcess(
+          originalStepInput,
+          this.context
+        )
+
         inputs = automationUtils.cleanInputValues(inputs, step.schema.inputs)
 
         const outputs = await stepFn({
@@ -583,6 +596,18 @@ class Orchestrator {
     return null
   }
 
+  private async addContextAndProcess(inputs: any, context: any) {
+    const processContext = {
+      ...context,
+      steps: {
+        ...context.steps,
+        ...context.stepsByName,
+      },
+    }
+
+    return processObject(inputs, processContext)
+  }
+
   private handleStepOutput(
     step: AutomationStep,
     outputs: any,
@@ -600,6 +625,8 @@ class Orchestrator {
     } else {
       this.updateExecutionOutput(step.id, step.stepId, step.inputs, outputs)
       this.context.steps[this.context.steps.length] = outputs
+      const stepName = step.name || step.id
+      this.context.stepsByName![stepName] = outputs
     }
   }
 }

packages/types/src/documents/app/automation/automation.ts

@@ -124,6 +124,8 @@ export interface Automation extends Document {
   definition: {
     steps: AutomationStep[]
     trigger: AutomationTrigger
+    // stepNames is used to lookup step names from their correspnding step ID.
+    stepNames?: Record<string, string>
   }
   screenId?: string
   uiTree?: any