Merge branch 'master' into BUDI-9038/validate-js-helpers

lerna.json

@@ -1,6 +1,6 @@
 {
   "$schema": "node_modules/lerna/schemas/lerna-schema.json",
-  "version": "3.4.13",
+  "version": "3.4.15",
   "npmClient": "yarn",
   "concurrency": 20,
   "command": {

packages/backend-core/src/context/mainContext.ts

@@ -123,7 +123,7 @@ export async function doInAutomationContext<T>(params: {
   task: () => T
 }): Promise<T> {
   await ensureSnippetContext()
-  return newContext(
+  return await newContext(
     {
       tenantId: getTenantIDFromAppID(params.appId),
       appId: params.appId,

packages/backend-core/src/sql/utils.ts

@@ -5,10 +5,10 @@ import {
   SqlQuery,
   Table,
   TableSourceType,
+  SEPARATOR,
 } from "@budibase/types"
 import { DEFAULT_BB_DATASOURCE_ID } from "../constants"
 import { Knex } from "knex"
-import { SEPARATOR } from "../db"
 import environment from "../environment"
 
 const DOUBLE_SEPARATOR = `${SEPARATOR}${SEPARATOR}`

packages/builder/src/stores/builder/screenComponent.ts

@@ -8,6 +8,7 @@ import {
   UIComponentError,
   ComponentDefinition,
   DependsOnComponentSetting,
+  Screen,
 } from "@budibase/types"
 import { queries } from "./queries"
 import { views } from "./views"
@@ -66,6 +67,7 @@ export const screenComponentErrorList = derived(
     if (!$selectedScreen) {
       return []
     }
+    const screen = $selectedScreen
 
     const datasources = {
       ...reduceBy("_id", $tables.list),
@@ -79,7 +81,9 @@ export const screenComponentErrorList = derived(
     const errors: UIComponentError[] = []
 
     function checkComponentErrors(component: Component, ancestors: string[]) {
-      errors.push(...getInvalidDatasources(component, datasources, definitions))
+      errors.push(
+        ...getInvalidDatasources(screen, component, datasources, definitions)
+      )
       errors.push(...getMissingRequiredSettings(component, definitions))
       errors.push(...getMissingAncestors(component, definitions, ancestors))
 
@@ -95,6 +99,7 @@ export const screenComponentErrorList = derived(
 )
 
 function getInvalidDatasources(
+  screen: Screen,
   component: Component,
   datasources: Record<string, any>,
   definitions: Record<string, ComponentDefinition>

packages/server/src/automations/tests/steps/loop.spec.ts

@@ -7,6 +7,7 @@ import {
   CreateRowStepOutputs,
   FieldType,
   FilterCondition,
+  AutomationStepStatus,
 } from "@budibase/types"
 import { createAutomationBuilder } from "../utilities/AutomationTestBuilder"
 import TestConfiguration from "../../../tests/utilities/TestConfiguration"
@@ -560,5 +561,25 @@ describe("Attempt to run a basic loop automation", () => {
         status: "stopped",
       })
     })
+
+    it("should not fail if queryRows returns nothing", async () => {
+      const table = await config.api.table.save(basicTable())
+      const results = await createAutomationBuilder(config)
+        .onAppAction()
+        .queryRows({
+          tableId: table._id!,
+        })
+        .loop({
+          option: LoopStepType.ARRAY,
+          binding: "{{ steps.1.rows }}",
+        })
+        .serverLog({ text: "Message {{loop.currentItem}}" })
+        .test({ fields: {} })
+
+      expect(results.steps[1].outputs.success).toBe(true)
+      expect(results.steps[1].outputs.status).toBe(
+        AutomationStepStatus.NO_ITERATIONS
+      )
+    })
   })
 })

packages/server/src/automations/utils.ts

@@ -40,21 +40,17 @@ function loggingArgs(job: AutomationJob) {
 }
 
 export async function processEvent(job: AutomationJob) {
-  return tracer.trace(
+  return tracer.trace("processEvent", async span => {
-    "processEvent",
-    { resource: "automation" },
-    async span => {
     const appId = job.data.event.appId!
     const automationId = job.data.automation._id!
 
-      span?.addTags({
+    span.addTags({
       appId,
       automationId,
       job: {
         id: job.id,
         name: job.name,
         attemptsMade: job.attemptsMade,
-          opts: {
         attempts: job.opts.attempts,
         priority: job.opts.priority,
         delay: job.opts.delay,
@@ -68,11 +64,11 @@ export async function processEvent(job: AutomationJob) {
         stackTraceLimit: job.opts.stackTraceLimit,
         preventParsingData: job.opts.preventParsingData,
       },
-        },
     })
 
     const task = async () => {
       try {
+        return await tracer.trace("task", async () => {
           if (isCronTrigger(job.data.automation) && !job.data.event.timestamp) {
             // Requires the timestamp at run time
             job.data.event.timestamp = Date.now()
@@ -81,25 +77,19 @@ export async function processEvent(job: AutomationJob) {
           console.log("automation running", ...loggingArgs(job))
 
           const runFn = () => Runner.run(job)
-          const result = await quotas.addAutomation(runFn, {
+          const result = await quotas.addAutomation(runFn, { automationId })
-            automationId,
-          })
           console.log("automation completed", ...loggingArgs(job))
           return result
+        })
       } catch (err) {
-          span?.addTags({ error: true })
+        span.addTags({ error: true })
-          console.error(
+        console.error(`automation was unable to run`, err, ...loggingArgs(job))
-            `automation was unable to run`,
-            err,
-            ...loggingArgs(job)
-          )
         return { err }
       }
     }
 
     return await context.doInAutomationContext({ appId, automationId, task })
-    }
+  })
-  )
 }
 
 export async function updateTestHistory(

packages/server/src/integrations/couchdb.ts

@@ -62,12 +62,16 @@ const SCHEMA: Integration = {
           type: DatasourceFieldType.STRING,
           required: true,
         },
+        rev: {
+          type: DatasourceFieldType.STRING,
+          required: true,
+        },
       },
     },
   },
 }
 
-class CouchDBIntegration implements IntegrationBase {
+export class CouchDBIntegration implements IntegrationBase {
   private readonly client: Database
 
   constructor(config: CouchDBConfig) {
@@ -82,7 +86,8 @@ class CouchDBIntegration implements IntegrationBase {
       connected: false,
     }
     try {
-      response.connected = await this.client.exists()
+      await this.client.allDocs({ limit: 1 })
+      response.connected = true
     } catch (e: any) {
       response.error = e.message as string
     }
@@ -99,13 +104,9 @@ class CouchDBIntegration implements IntegrationBase {
   }
 
   async read(query: { json: string | object }) {
-    const parsed = this.parse(query)
+    const params = { include_docs: true, ...this.parse(query) }
-    const params = {
-      include_docs: true,
-      ...parsed,
-    }
     const result = await this.client.allDocs(params)
-    return result.rows.map(row => row.doc)
+    return result.rows.map(row => row.doc!)
   }
 
   async update(query: { json: string | object }) {
@@ -121,8 +122,8 @@ class CouchDBIntegration implements IntegrationBase {
     return await this.client.get(query.id)
   }
 
-  async delete(query: { id: string }) {
+  async delete(query: { id: string; rev: string }) {
-    return await this.client.remove(query.id)
+    return await this.client.remove(query.id, query.rev)
   }
 }
 

packages/server/src/integrations/tests/couchdb.spec.ts

@@ -1,84 +1,87 @@
-jest.mock("@budibase/backend-core", () => {
+import { env } from "@budibase/backend-core"
-  const core = jest.requireActual("@budibase/backend-core")
+import { CouchDBIntegration } from "../couchdb"
-  return {
+import { generator } from "@budibase/backend-core/tests"
-    ...core,
-    db: {
-      ...core.db,
-      DatabaseWithConnection: function () {
-        return {
-          allDocs: jest.fn().mockReturnValue({ rows: [] }),
-          put: jest.fn(),
-          get: jest.fn().mockReturnValue({ _rev: "a" }),
-          remove: jest.fn(),
-        }
-      },
-    },
-  }
-})
 
-import { default as CouchDBIntegration } from "../couchdb"
+function couchSafeID(): string {
-
+  // CouchDB IDs must start with a letter, so we prepend an 'a'.
-class TestConfiguration {
+  return `a${generator.guid()}`
-  integration: any
-
-  constructor(
-    config: any = { url: "http://somewhere", database: "something" }
-  ) {
-    this.integration = new CouchDBIntegration.integration(config)
 }
+
+function doc(data: Record<string, any>): string {
+  return JSON.stringify({ _id: couchSafeID(), ...data })
+}
+
+function query(data?: Record<string, any>): { json: string } {
+  return { json: doc(data || {}) }
 }
 
 describe("CouchDB Integration", () => {
-  let config: any
+  let couchdb: CouchDBIntegration
 
   beforeEach(() => {
-    config = new TestConfiguration()
+    couchdb = new CouchDBIntegration({
+      url: env.COUCH_DB_URL,
+      database: couchSafeID(),
+    })
   })
 
-  it("calls the create method with the correct params", async () => {
+  it("successfully connects", async () => {
-    const doc = {
+    const { connected } = await couchdb.testConnection()
+    expect(connected).toBe(true)
+  })
+
+  it("can create documents", async () => {
+    const { id, ok, rev } = await couchdb.create(query({ test: 1 }))
+    expect(id).toBeDefined()
+    expect(ok).toBe(true)
+    expect(rev).toBeDefined()
+  })
+
+  it("can read created documents", async () => {
+    const { id, ok, rev } = await couchdb.create(query({ test: 1 }))
+    expect(id).toBeDefined()
+    expect(ok).toBe(true)
+    expect(rev).toBeDefined()
+
+    const docs = await couchdb.read(query())
+    expect(docs).toEqual([
+      {
+        _id: id,
+        _rev: rev,
         test: 1,
-    }
+        createdAt: expect.any(String),
-    await config.integration.create({
+        updatedAt: expect.any(String),
-      json: JSON.stringify(doc),
+      },
-    })
+    ])
-    expect(config.integration.client.put).toHaveBeenCalledWith(doc)
   })
 
-  it("calls the read method with the correct params", async () => {
+  it("can update documents", async () => {
-    const doc = {
+    const { id, ok, rev } = await couchdb.create(query({ test: 1 }))
-      name: "search",
+    expect(ok).toBe(true)
-    }
 
-    await config.integration.read({
+    const { id: newId, rev: newRev } = await couchdb.update(
-      json: JSON.stringify(doc),
+      query({ _id: id, _rev: rev, test: 2 })
+    )
+    const docs = await couchdb.read(query())
+    expect(docs).toEqual([
+      {
+        _id: newId,
+        _rev: newRev,
+        test: 2,
+        createdAt: expect.any(String),
+        updatedAt: expect.any(String),
+      },
+    ])
   })
 
-    expect(config.integration.client.allDocs).toHaveBeenCalledWith({
+  it("can delete documents", async () => {
-      include_docs: true,
+    const { id, ok, rev } = await couchdb.create(query({ test: 1 }))
-      name: "search",
+    expect(ok).toBe(true)
-    })
-  })
 
-  it("calls the update method with the correct params", async () => {
+    const deleteResponse = await couchdb.delete({ id, rev })
-    const doc = {
+    expect(deleteResponse.ok).toBe(true)
-      _id: "1234",
-      name: "search",
-    }
 
-    await config.integration.update({
+    const docs = await couchdb.read(query())
-      json: JSON.stringify(doc),
+    expect(docs).toBeEmpty()
-    })
-
-    expect(config.integration.client.put).toHaveBeenCalledWith({
-      ...doc,
-      _rev: "a",
-    })
-  })
-
-  it("calls the delete method with the correct params", async () => {
-    const id = "1234"
-    await config.integration.delete({ id })
-    expect(config.integration.client.remove).toHaveBeenCalledWith(id)
   })
 })

packages/server/src/threads/automation.ts

@@ -68,8 +68,12 @@ function getLoopIterable(step: LoopStep): any[] {
   let input = step.inputs.binding
 
   if (option === LoopStepType.ARRAY && typeof input === "string") {
+    if (input === "") {
+      input = []
+    } else {
       input = JSON.parse(input)
     }
+  }
 
   if (option === LoopStepType.STRING && Array.isArray(input)) {
     input = input.join(",")
@@ -310,11 +314,8 @@ class Orchestrator {
   }
 
   async execute(): Promise<AutomationResults> {
-    return tracer.trace(
+    return await tracer.trace("execute", async span => {
-      "Orchestrator.execute",
+      span.addTags({ appId: this.appId, automationId: this.automation._id })
-      { resource: "automation" },
-      async span => {
-        span?.addTags({ appId: this.appId, automationId: this.automation._id })
 
       const job = cloneDeep(this.job)
       delete job.data.event.appId
@@ -382,15 +383,14 @@ class Orchestrator {
       }
 
       return result
-      }
+    })
-    )
   }
 
   private async executeSteps(
     ctx: AutomationContext,
     steps: AutomationStep[]
   ): Promise<AutomationStepResult[]> {
-    return tracer.trace("Orchestrator.executeSteps", async () => {
+    return await tracer.trace("executeSteps", async () => {
       let stepIndex = 0
       const results: AutomationStepResult[] = []
 
@@ -446,6 +446,7 @@ class Orchestrator {
     step: LoopStep,
     stepToLoop: AutomationStep
   ): Promise<AutomationStepResult> {
+    return await tracer.trace("executeLoopStep", async span => {
       await processObject(step.inputs, prepareContext(ctx))
 
       const maxIterations = getLoopMaxIterations(step)
@@ -455,6 +456,10 @@ class Orchestrator {
       try {
         iterable = getLoopIterable(step)
       } catch (err) {
+        span.addTags({
+          status: AutomationStepStatus.INCORRECT_TYPE,
+          iterations,
+        })
         return stepFailure(stepToLoop, {
           status: AutomationStepStatus.INCORRECT_TYPE,
         })
@@ -464,6 +469,10 @@ class Orchestrator {
         const currentItem = iterable[iterations]
 
         if (iterations === maxIterations) {
+          span.addTags({
+            status: AutomationStepStatus.MAX_ITERATIONS,
+            iterations,
+          })
           return stepFailure(stepToLoop, {
             status: AutomationStepStatus.MAX_ITERATIONS,
             iterations,
@@ -471,6 +480,10 @@ class Orchestrator {
         }
 
         if (matchesLoopFailureCondition(step, currentItem)) {
+          span.addTags({
+            status: AutomationStepStatus.FAILURE_CONDITION,
+            iterations,
+          })
           return stepFailure(stepToLoop, {
             status: AutomationStepStatus.FAILURE_CONDITION,
           })
@@ -483,18 +496,21 @@ class Orchestrator {
       }
 
       const status =
-      iterations === 0 ? AutomationStatus.NO_CONDITION_MET : undefined
+        iterations === 0 ? AutomationStepStatus.NO_ITERATIONS : undefined
       return stepSuccess(stepToLoop, { status, iterations, items })
+    })
   }
 
   private async executeBranchStep(
     ctx: AutomationContext,
     step: BranchStep
   ): Promise<AutomationStepResult[]> {
+    return await tracer.trace("executeBranchStep", async span => {
       const { branches, children } = step.inputs
 
       for (const branch of branches) {
         if (await branchMatches(ctx, branch)) {
+          span.addTags({ branchName: branch.name, branchId: branch.id })
           return [
             stepSuccess(step, {
               branchName: branch.name,
@@ -506,14 +522,16 @@ class Orchestrator {
         }
       }
 
+      span.addTags({ status: AutomationStatus.NO_CONDITION_MET })
       return [stepFailure(step, { status: AutomationStatus.NO_CONDITION_MET })]
+    })
   }
 
   private async executeStep(
     ctx: AutomationContext,
     step: Readonly<AutomationStep>
   ): Promise<AutomationStepResult> {
-    return tracer.trace("Orchestrator.executeStep", async span => {
+    return await tracer.trace(step.stepId, async span => {
       span.addTags({
         step: {
           stepId: step.stepId,
@@ -524,6 +542,7 @@ class Orchestrator {
           internal: step.internal,
           deprecated: step.deprecated,
         },
+        inputsKeys: Object.keys(step.inputs),
       })
 
       if (this.stopped) {
@@ -557,6 +576,7 @@ class Orchestrator {
         ;(outputs as any).status = AutomationStatus.STOPPED
       }
 
+      span.addTags({ outputsKeys: Object.keys(outputs) })
       return stepSuccess(step, outputs, inputs)
     })
   }