Merge branch 'master' into grid-new-component-dnd

packages/backend-core/src/objectStore/objectStore.ts

@@ -424,7 +424,7 @@ export async function retrieveDirectory(bucketName: string, path: string) {
     stream.pipe(writeStream)
     writePromises.push(
       new Promise((resolve, reject) => {
-        stream.on("finish", resolve)
+        writeStream.on("finish", resolve)
         stream.on("error", reject)
         writeStream.on("error", reject)
       })

packages/server/jest.config.ts

@@ -21,20 +21,7 @@ const baseConfig: Config.InitialProjectOptions = {
   transform: {
     "^.+\\.ts?$": "@swc/jest",
     "^.+\\.js?$": "@swc/jest",
-    "^.+\\.svelte$": [
-      "jest-chain-transform", // https://github.com/svelteness/svelte-jester/issues/166
-      {
-        transformers: [
-          [
-            "svelte-jester",
-            {
-              preprocess: true,
-            },
-          ],
-          "@swc/jest",
-        ],
-      },
-    ],
+    "^.+\\.svelte?$": "<rootDir>/scripts/svelteTransformer.js",
   },
   transformIgnorePatterns: ["/node_modules/(?!svelte/).*"],
   moduleNameMapper: {

packages/server/package.json

@@ -139,7 +139,6 @@
     "@babel/core": "^7.22.5",
     "@babel/preset-env": "7.16.11",
     "@jest/types": "^29.6.3",
-    "@sveltejs/vite-plugin-svelte": "1.4.0",
     "@swc/core": "1.3.71",
     "@swc/jest": "0.2.27",
     "@types/archiver": "6.0.2",
@@ -165,7 +164,6 @@
     "docker-compose": "0.23.17",
     "ioredis-mock": "8.9.0",
     "jest": "29.7.0",
-    "jest-chain-transform": "^0.0.8",
     "jest-extended": "^4.0.2",
     "jest-openapi": "0.14.2",
     "nock": "13.5.4",

packages/server/scripts/dev/manage.js

@@ -46,6 +46,7 @@ async function init() {
     HTTP_LOGGING: "0",
     VERSION: "0.0.0+local",
     PASSWORD_MIN_LENGTH: "1",
+    OPENAI_API_KEY: "sk-abcdefghijklmnopqrstuvwxyz1234567890abcd",
   }
 
   config = { ...config, ...existingConfig }

packages/server/scripts/svelteTransformer.js

@@ -0,0 +1,11 @@
+const { compile } = require("svelte/compiler")
+const { transformSync } = require("@babel/core")
+
+module.exports = {
+  process(sourceText) {
+    const { js } = compile(sourceText, { css: "injected", generate: "ssr" })
+    const { code } = transformSync(js.code, { babelrc: true })
+
+    return { code: code }
+  },
+}

packages/server/src/api/controllers/static/templates/BudibaseApp.svelte

@@ -1,8 +1,9 @@
-<script lang="ts">
+<script>
   import ClientAppSkeleton from "@budibase/frontend-core/src/components/ClientAppSkeleton.svelte"
-  import type { BudibaseAppProps } from "@budibase/types"
 
-  export let props: BudibaseAppProps
+  /** @type {BudibaseAppProps} this receives all the props in one structure, following
+   * the type from @budibase/types */
+  export let props
 </script>
 
 <svelte:head>

packages/server/src/automations/tests/automation.spec.ts

@@ -1,100 +0,0 @@
-jest.mock("../../threads/automation")
-jest.mock("../../utilities/redis", () => ({
-  init: jest.fn(),
-  checkTestFlag: () => {
-    return false
-  },
-  shutdown: jest.fn(),
-}))
-
-jest.spyOn(global.console, "error")
-
-import "../../environment"
-import * as automation from "../index"
-import * as thread from "../../threads/automation"
-import * as triggers from "../triggers"
-import { basicAutomation } from "../../tests/utilities/structures"
-import { wait } from "../../utilities"
-import { makePartial } from "../../tests/utilities"
-import { cleanInputValues } from "../automationUtils"
-import { Automation } from "@budibase/types"
-import TestConfiguration from "../../tests/utilities/TestConfiguration"
-
-describe("Run through some parts of the automations system", () => {
-  const config = new TestConfiguration()
-
-  beforeAll(async () => {
-    await automation.init()
-    await config.init()
-  })
-
-  afterAll(async () => {
-    await automation.shutdown()
-    config.end()
-  })
-
-  it("should be able to init in builder", async () => {
-    const automation: Automation = {
-      ...basicAutomation(),
-      appId: config.appId!,
-    }
-    const fields: any = { a: 1, appId: config.appId }
-    await triggers.externalTrigger(automation, fields)
-    await wait(100)
-    expect(thread.execute).toHaveBeenCalled()
-  })
-
-  it("should check coercion", async () => {
-    const table = await config.createTable()
-    const automation: any = basicAutomation()
-    automation.definition.trigger.inputs.tableId = table._id
-    automation.definition.trigger.stepId = "APP"
-    automation.definition.trigger.inputs.fields = { a: "number" }
-    const fields: any = {
-      appId: config.getAppId(),
-      fields: {
-        a: "1",
-      },
-    }
-    await triggers.externalTrigger(automation, fields)
-    await wait(100)
-    expect(thread.execute).toHaveBeenCalledWith(
-      makePartial({
-        data: {
-          event: {
-            fields: {
-              a: 1,
-            },
-          },
-        },
-      }),
-      expect.any(Function)
-    )
-  })
-
-  it("should be able to clean inputs with the utilities", () => {
-    // can't clean without a schema
-    let output = cleanInputValues({ a: "1" })
-    expect(output.a).toBe("1")
-    output = cleanInputValues(
-      { a: "1", b: "true", c: "false", d: 1, e: "help" },
-      {
-        properties: {
-          a: {
-            type: "number",
-          },
-          b: {
-            type: "boolean",
-          },
-          c: {
-            type: "boolean",
-          },
-        },
-      }
-    )
-    expect(output.a).toBe(1)
-    expect(output.b).toBe(true)
-    expect(output.c).toBe(false)
-    expect(output.d).toBe(1)
-  })
-})

packages/server/src/automations/tests/automationUtils.spec.ts

@@ -119,5 +119,31 @@ describe("automationUtils", () => {
         schema,
       })
     })
+
+    it("should be able to clean inputs with the utilities", () => {
+      // can't clean without a schema
+      let output = cleanInputValues({ a: "1" })
+      expect(output.a).toBe("1")
+      output = cleanInputValues(
+        { a: "1", b: "true", c: "false", d: 1, e: "help" },
+        {
+          properties: {
+            a: {
+              type: "number",
+            },
+            b: {
+              type: "boolean",
+            },
+            c: {
+              type: "boolean",
+            },
+          },
+        }
+      )
+      expect(output.a).toBe(1)
+      expect(output.b).toBe(true)
+      expect(output.c).toBe(false)
+      expect(output.d).toBe(1)
+    })
   })
 })

packages/server/src/automations/tests/steps/loop.spec.ts

@@ -1,17 +1,13 @@
 import * as automation from "../../index"
-import * as triggers from "../../triggers"
-import { basicTable, loopAutomation } from "../../../tests/utilities/structures"
-import { context } from "@budibase/backend-core"
+import { basicTable } from "../../../tests/utilities/structures"
 import {
   Table,
   LoopStepType,
-  AutomationResults,
   ServerLogStepOutputs,
   CreateRowStepOutputs,
   FieldType,
 } from "@budibase/types"
 import * as loopUtils from "../../loopUtils"
-import { LoopInput } from "../../../definitions/automations"
 import { createAutomationBuilder } from "../utilities/AutomationTestBuilder"
 import TestConfiguration from "../../../tests/utilities/TestConfiguration"
 
@@ -34,41 +30,46 @@ describe("Attempt to run a basic loop automation", () => {
     config.end()
   })
 
-  async function runLoop(loopOpts?: LoopInput): Promise<AutomationResults> {
-    const appId = config.getAppId()
-    return await context.doInAppContext(appId, async () => {
-      const params = { fields: { appId } }
-      const result = await triggers.externalTrigger(
-        loopAutomation(table._id!, loopOpts),
-        params,
-        { getResponses: true }
-      )
-      if ("outputs" in result && !result.outputs.success) {
-        throw new Error("Unable to proceed - failed to return anything.")
-      }
-      return result as AutomationResults
-    })
-  }
-
   it("attempt to run a basic loop", async () => {
-    const resp = await runLoop()
-    expect(resp.steps[2].outputs.iterations).toBe(1)
+    const result = await createAutomationBuilder(config)
+      .onAppAction()
+      .queryRows({
+        tableId: table._id!,
+      })
+      .loop({
+        option: LoopStepType.ARRAY,
+        binding: "{{ steps.1.rows }}",
+      })
+      .serverLog({ text: "log statement" })
+      .test({ fields: {} })
+
+    expect(result.steps[1].outputs.iterations).toBe(1)
   })
 
   it("test a loop with a string", async () => {
-    const resp = await runLoop({
-      option: LoopStepType.STRING,
-      binding: "a,b,c",
-    })
-    expect(resp.steps[2].outputs.iterations).toBe(3)
+    const result = await createAutomationBuilder(config)
+      .onAppAction()
+      .loop({
+        option: LoopStepType.STRING,
+        binding: "a,b,c",
+      })
+      .serverLog({ text: "log statement" })
+      .test({ fields: {} })
+
+    expect(result.steps[0].outputs.iterations).toBe(3)
   })
 
   it("test a loop with a binding that returns an integer", async () => {
-    const resp = await runLoop({
-      option: LoopStepType.ARRAY,
-      binding: "{{ 1 }}",
-    })
-    expect(resp.steps[2].outputs.iterations).toBe(1)
+    const result = await createAutomationBuilder(config)
+      .onAppAction()
+      .loop({
+        option: LoopStepType.ARRAY,
+        binding: "{{ 1 }}",
+      })
+      .serverLog({ text: "log statement" })
+      .test({ fields: {} })
+
+    expect(result.steps[0].outputs.iterations).toBe(1)
   })
 
   it("should run an automation with a trigger, loop, and create row step", async () => {

packages/server/src/automations/tests/triggers/appAction.spec.ts

@@ -0,0 +1,74 @@
+import { createAutomationBuilder } from "../utilities/AutomationTestBuilder"
+import TestConfiguration from "../../../tests/utilities/TestConfiguration"
+import { captureAutomationResults } from "../utilities"
+import { Automation } from "@budibase/types"
+
+describe("app action trigger", () => {
+  const config = new TestConfiguration()
+  let automation: Automation
+
+  beforeAll(async () => {
+    await config.init()
+    automation = await createAutomationBuilder(config)
+      .onAppAction()
+      .serverLog({
+        text: "App action triggered",
+      })
+      .save()
+      .then(({ automation }) => automation)
+
+    await config.api.application.publish()
+  })
+
+  afterAll(() => {
+    config.end()
+  })
+
+  it("should trigger when the app action is performed", async () => {
+    const jobs = await captureAutomationResults(automation, async () => {
+      await config.withProdApp(async () => {
+        await config.api.automation.trigger(automation._id!, {
+          fields: {},
+          timeout: 1000,
+        })
+      })
+    })
+
+    expect(jobs).toHaveLength(1)
+    expect(jobs[0].data.event).toEqual(
+      expect.objectContaining({
+        fields: {},
+        timeout: 1000,
+      })
+    )
+  })
+
+  it("should correct coerce values based on the schema", async () => {
+    const { automation } = await createAutomationBuilder(config)
+      .onAppAction({
+        fields: { text: "string", number: "number", boolean: "boolean" },
+      })
+      .serverLog({
+        text: "{{ fields.text }} {{ fields.number }} {{ fields.boolean }}",
+      })
+      .save()
+
+    await config.api.application.publish()
+
+    const jobs = await captureAutomationResults(automation, async () => {
+      await config.withProdApp(async () => {
+        await config.api.automation.trigger(automation._id!, {
+          fields: { text: "1", number: "2", boolean: "true" },
+          timeout: 1000,
+        })
+      })
+    })
+
+    expect(jobs).toHaveLength(1)
+    expect(jobs[0].data.event.fields).toEqual({
+      text: "1",
+      number: 2,
+      boolean: true,
+    })
+  })
+})

packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts

@@ -15,6 +15,8 @@ import {
   isDidNotTriggerResponse,
   SearchFilters,
   TestAutomationRequest,
+  TriggerAutomationRequest,
+  TriggerAutomationResponse,
 } from "@budibase/types"
 import TestConfiguration from "../../../tests/utilities/TestConfiguration"
 import { automations } from "@budibase/shared-core"
@@ -143,13 +145,13 @@ class StepBuilder<
   TStep extends AutomationTriggerStepId
 > extends BranchStepBuilder<TStep> {
   private readonly config: TestConfiguration
-  private readonly trigger: AutomationTrigger
+  private readonly _trigger: AutomationTrigger
   private _name: string | undefined = undefined
 
   constructor(config: TestConfiguration, trigger: AutomationTrigger) {
     super()
     this.config = config
-    this.trigger = trigger
+    this._trigger = trigger
   }
 
   name(n: string): this {
@@ -163,7 +165,7 @@ class StepBuilder<
       name,
       definition: {
         steps: this.steps,
-        trigger: this.trigger,
+        trigger: this._trigger,
         stepNames: this.stepNames,
       },
       type: "automation",
@@ -180,6 +182,13 @@ class StepBuilder<
     const runner = await this.save()
     return await runner.test(triggerOutput)
   }
+
+  async trigger(
+    request: TriggerAutomationRequest
+  ): Promise<TriggerAutomationResponse> {
+    const runner = await this.save()
+    return await runner.trigger(request)
+  }
 }
 
 class AutomationRunner<TStep extends AutomationTriggerStepId> {
@@ -207,6 +216,15 @@ class AutomationRunner<TStep extends AutomationTriggerStepId> {
 
     return response
   }
+
+  async trigger(
+    request: TriggerAutomationRequest
+  ): Promise<TriggerAutomationResponse> {
+    return await this.config.api.automation.trigger(
+      this.automation._id!,
+      request
+    )
+  }
 }
 
 export function createAutomationBuilder(config: TestConfiguration) {

packages/server/svelte.config.js

@@ -1,5 +0,0 @@
-const { vitePreprocess } = require("@sveltejs/vite-plugin-svelte")
-
-module.exports = {
-  preprocess: vitePreprocess(),
-}

packages/string-templates/package.json

@@ -33,6 +33,7 @@
     "@rollup/plugin-json": "^4.1.0",
     "@rollup/plugin-node-resolve": "^15.2.3",
     "@rollup/plugin-typescript": "8.3.0",
+    "@types/doctrine": "^0.0.9",
     "doctrine": "^3.0.0",
     "jest": "29.7.0",
     "marked": "^4.0.10",

packages/string-templates/scripts/gen-collection-info.ts

@@ -7,18 +7,30 @@ import { marked } from "marked"
 import { join, dirname } from "path"
 
 const helpers = require("@budibase/handlebars-helpers")
-const doctrine = require("doctrine")
+import doctrine, { Annotation } from "doctrine"
 
-type HelperInfo = {
+type BudibaseAnnotation = Annotation & {
+  example?: string
   acceptsInline?: boolean
   acceptsBlock?: boolean
+}
+
+type Helper = {
+  args: string[]
+  numArgs: number
   example?: string
   description: string
-  tags?: any[]
+  requiresBlock?: boolean
+}
+
+type Manifest = {
+  [category: string]: {
+    [helper: string]: Helper
+  }
 }
 
 const FILENAME = join(__dirname, "..", "src", "manifest.json")
-const outputJSON: any = {}
+const outputJSON: Manifest = {}
 const ADDED_HELPERS = {
   date: {
     date: {
@@ -38,11 +50,10 @@ const ADDED_HELPERS = {
   },
 }
 
-function fixSpecialCases(name: string, obj: any) {
-  const args = obj.args
+function fixSpecialCases(name: string, obj: Helper) {
   if (name === "ifNth") {
-    args[0] = "a"
-    args[1] = "b"
+    obj.args = ["a", "b", "options"]
+    obj.numArgs = 3
   }
   if (name === "eachIndex") {
     obj.description = "Iterates the array, listing an item and the index of it."
@@ -66,10 +77,10 @@ function lookForward(lines: string[], funcLines: string[], idx: number) {
   return true
 }
 
-function getCommentInfo(file: string, func: string): HelperInfo {
+function getCommentInfo(file: string, func: string): BudibaseAnnotation {
   const lines = file.split("\n")
   const funcLines = func.split("\n")
-  let comment = null
+  let comment: string | null = null
   for (let idx = 0; idx < lines.length; ++idx) {
     // from here work back until we have the comment
     if (lookForward(lines, funcLines, idx)) {
@@ -91,15 +102,9 @@ function getCommentInfo(file: string, func: string): HelperInfo {
     }
   }
   if (comment == null) {
-    return { description: "" }
+    return { description: "", tags: [] }
   }
-  const docs: {
-    acceptsInline?: boolean
-    acceptsBlock?: boolean
-    example: string
-    description: string
-    tags: any[]
-  } = doctrine.parse(comment, { unwrap: true })
+  const docs: BudibaseAnnotation = doctrine.parse(comment, { unwrap: true })
   // some hacky fixes
   docs.description = docs.description.replace(/\n/g, " ")
   docs.description = docs.description.replace(/[ ]{2,}/g, " ")
@@ -135,7 +140,7 @@ function run() {
       )}/lib/${collection}.js`,
       "utf8"
     )
-    const collectionInfo: any = {}
+    const collectionInfo: { [name: string]: Helper } = {}
     // collect information about helper
     let hbsHelperInfo = helpers[collection]()
     for (let entry of Object.entries(hbsHelperInfo)) {
@@ -154,11 +159,8 @@ function run() {
       const jsDocInfo = getCommentInfo(collectionFile, fnc)
       let args = jsDocInfo.tags
         .filter(tag => tag.title === "param")
-        .map(
-          tag =>
-            tag.description &&
-            tag.description.replace(/`/g, "").split(" ")[0].trim()
-        )
+        .filter(tag => tag.description)
+        .map(tag => tag.description!.replace(/`/g, "").split(" ")[0].trim())
       collectionInfo[name] = fixSpecialCases(name, {
         args,
         numArgs: args.length,

packages/types/src/sdk/automations/index.ts

@@ -15,6 +15,7 @@ export interface AutomationDataEvent {
   oldRow?: Row
   user?: UserBindings
   timestamp?: number
+  fields?: Record<string, any>
 }
 
 export interface AutomationData {

yarn.lock

@@ -6456,6 +6456,11 @@
     "@types/node" "*"
     "@types/ssh2" "*"
 
+"@types/doctrine@^0.0.9":
+  version "0.0.9"
+  resolved "https://registry.yarnpkg.com/@types/doctrine/-/doctrine-0.0.9.tgz#d86a5f452a15e3e3113b99e39616a9baa0f9863f"
+  integrity sha512-eOIHzCUSH7SMfonMG1LsC2f8vxBFtho6NGBznK41R84YzPuvSBzrhEps33IsQiOW9+VL6NQ9DbjQJznk/S4uRA==
+
 "@types/estree@*", "@types/estree@1.0.5", "@types/estree@^1.0.0", "@types/estree@^1.0.1":
   version "1.0.5"
   resolved "https://registry.yarnpkg.com/@types/estree/-/estree-1.0.5.tgz#a6ce3e556e00fd9895dd872dd172ad0d4bd687f4"
@@ -13739,11 +13744,6 @@ jake@^10.8.5:
     filelist "^1.0.1"
     minimatch "^3.0.4"
 
-jest-chain-transform@^0.0.8:
-  version "0.0.8"
-  resolved "https://registry.yarnpkg.com/jest-chain-transform/-/jest-chain-transform-0.0.8.tgz#cbb4d3aef8d02678b1852968a9b0c861f75eef5a"
-  integrity sha512-AELTTzYJ34WrmQKAbxUGT+xqnAHu0/XJZhahYNGvBVUhnAayjm1QmT45DQjwEbQPQp7gn6CXzu6rZA03riwBuw==
-
 jest-changed-files@^29.7.0:
   version "29.7.0"
   resolved "https://registry.yarnpkg.com/jest-changed-files/-/jest-changed-files-29.7.0.tgz#1c06d07e77c78e1585d020424dedc10d6e17ac3a"