Merge branch 'master' into jest-mock-excision

hosting/nginx.dev.conf

@@ -62,6 +62,12 @@ http {
       proxy_connect_timeout 120s;
       proxy_send_timeout 120s;
       proxy_http_version 1.1;
+
+      # Enable buffering for potentially large OIDC configs
+      proxy_buffering on;
+      proxy_buffer_size 16k;
+      proxy_buffers 4 32k;
+
       proxy_set_header Host $host;
       proxy_set_header Connection "";
 

lerna.json

@@ -1,6 +1,6 @@
 {
   "$schema": "node_modules/lerna/schemas/lerna-schema.json",
-  "version": "3.4.16",
+  "version": "3.4.17",
   "npmClient": "yarn",
   "concurrency": 20,
   "command": {

packages/builder/src/stores/builder/automations.ts

@@ -484,7 +484,7 @@ const automationActions = (store: AutomationStore) => ({
         branches.forEach((branch, bIdx) => {
           children[branch.id].forEach(
             (bBlock: AutomationStep, sIdx: number, array: AutomationStep[]) => {
-              const ended = array.length - 1 === sIdx && !branches.length
+              const ended = array.length - 1 === sIdx
               treeTraverse(bBlock, pathToCurrentNode, sIdx, bIdx, ended)
             }
           )
@@ -505,7 +505,6 @@ const automationActions = (store: AutomationStore) => ({
     blocks.forEach((block, idx, array) => {
       treeTraverse(block, null, idx, null, array.length - 1 === idx)
     })
-
     return blockRefs
   },
 

packages/server/src/api/routes/tests/search.spec.ts

@@ -3553,6 +3553,31 @@ if (descriptions.length) {
                     limit: 1,
                   }).toContainExactly([row])
                 })
+
+                isInternal &&
+                  describe("search by _id for relations", () => {
+                    it("can filter by the related _id", async () => {
+                      await expectSearch({
+                        query: {
+                          equal: { "rel._id": row.rel[0]._id },
+                        },
+                      }).toContainExactly([row])
+
+                      await expectSearch({
+                        query: {
+                          equal: { "rel._id": row.rel[1]._id },
+                        },
+                      }).toContainExactly([row])
+                    })
+
+                    it("can filter by the related _id and find nothing", async () => {
+                      await expectSearch({
+                        query: {
+                          equal: { "rel._id": "rel_none" },
+                        },
+                      }).toFindNothing()
+                    })
+                  })
               })
 
             !isInternal &&

packages/server/src/automations/tests/branching.spec.ts

@@ -1,5 +1,5 @@
 import * as automation from "../index"
-import { Table, AutomationStatus } from "@budibase/types"
+import { Table, AutomationStatus, EmptyFilterOption } from "@budibase/types"
 import { createAutomationBuilder } from "./utilities/AutomationTestBuilder"
 import TestConfiguration from "../../tests/utilities/TestConfiguration"
 
@@ -280,4 +280,23 @@ describe("Branching automations", () => {
 
     expect(results.steps[2].outputs.message).toContain("Special user")
   })
+
+  it("should not fail with empty conditions", async () => {
+    const results = await createAutomationBuilder(config)
+      .onAppAction()
+      .branch({
+        specialBranch: {
+          steps: stepBuilder => stepBuilder.serverLog({ text: "Hello!" }),
+          condition: {
+            onEmptyFilter: EmptyFilterOption.RETURN_NONE,
+          },
+        },
+      })
+      .test({ fields: { test_trigger: true } })
+
+    expect(results.steps[0].outputs.success).toEqual(false)
+    expect(results.steps[0].outputs.status).toEqual(
+      AutomationStatus.NO_CONDITION_MET
+    )
+  })
 })

packages/server/src/automations/tests/steps/sendSmtpEmail.spec.ts

@@ -1,3 +1,4 @@
+import { SendEmailResponse } from "@budibase/types"
 import TestConfiguration from "../../../tests/utilities/TestConfiguration"
 import * as workerRequests from "../../../utilities/workerRequests"
 
@@ -5,17 +6,18 @@ jest.mock("../../../utilities/workerRequests", () => ({
   sendSmtpEmail: jest.fn(),
 }))
 
-function generateResponse(to: string, from: string) {
+function generateResponse(to: string, from: string): SendEmailResponse {
   return {
-    success: true,
+    message: `Email sent to ${to}.`,
-    response: {
     accepted: [to],
     envelope: {
       from: from,
       to: [to],
     },
-      message: `Email sent to ${to}.`,
+    messageId: "messageId",
-    },
+    pending: [],
+    rejected: [],
+    response: "response",
   }
 }
 

packages/server/src/sdk/app/rows/queryUtils.ts

@@ -7,6 +7,7 @@ import {
 } from "@budibase/types"
 import { cloneDeep } from "lodash/fp"
 import sdk from "../../../sdk"
+import { isInternal } from "../tables/utils"
 
 export const removeInvalidFilters = (
   filters: SearchFilters,
@@ -70,6 +71,10 @@ export const getQueryableFields = async (
     opts?: { noRelationships?: boolean }
   ): Promise<string[]> => {
     const result = []
+    if (isInternal({ table })) {
+      result.push("_id")
+    }
+
     for (const field of Object.keys(table.schema).filter(
       f => allowedFields.includes(f) && table.schema[f].visible !== false
     )) {
@@ -113,14 +118,13 @@ export const getQueryableFields = async (
     return result
   }
 
-  const result = [
+  // Querying by _id is always allowed, even if it's never part of the schema
-    "_id", // Querying by _id is always allowed, even if it's never part of the schema
+  const result = ["_id"]
-  ]
 
   if (fields == null) {
     fields = Object.keys(table.schema)
   }
   result.push(...(await extractTableFields(table, fields, [table._id!])))
 
-  return result
+  return Array.from(new Set(result))
 }

packages/server/src/sdk/app/rows/tests/queryUtils.spec.ts

@@ -250,6 +250,8 @@ describe("query utils", () => {
       expect(result).toEqual([
         "_id",
         "name",
+        "aux._id",
+        "auxTable._id",
         "aux.title",
         "auxTable.title",
         "aux.name",
@@ -284,7 +286,14 @@ describe("query utils", () => {
       const result = await config.doInContext(config.appId, () => {
         return getQueryableFields(table)
       })
-      expect(result).toEqual(["_id", "name", "aux.name", "auxTable.name"])
+      expect(result).toEqual([
+        "_id",
+        "name",
+        "aux._id",
+        "auxTable._id",
+        "aux.name",
+        "auxTable.name",
+      ])
     })
 
     it("excludes all relationship fields if hidden", async () => {
@@ -387,10 +396,14 @@ describe("query utils", () => {
             "_id",
             "name",
             // aux1 primitive props
+            "aux1._id",
+            "aux1Table._id",
             "aux1.name",
             "aux1Table.name",
 
             // aux2 primitive props
+            "aux2._id",
+            "aux2Table._id",
             "aux2.title",
             "aux2Table.title",
           ])
@@ -405,14 +418,18 @@ describe("query utils", () => {
             "name",
 
             // aux2_1 primitive props
+            "aux2_1._id",
+            "aux2Table._id",
             "aux2_1.title",
             "aux2Table.title",
 
             // aux2_2 primitive props
+            "aux2_2._id",
             "aux2_2.title",
-            "aux2Table.title",
 
             // table primitive props
+            "table._id",
+            "TestTable._id",
             "table.name",
             "TestTable.name",
           ])
@@ -427,14 +444,18 @@ describe("query utils", () => {
             "title",
 
             // aux1_1 primitive props
+            "aux1_1._id",
+            "aux1Table._id",
             "aux1_1.name",
             "aux1Table.name",
 
             // aux1_2 primitive props
+            "aux1_2._id",
             "aux1_2.name",
-            "aux1Table.name",
 
             // table primitive props
+            "table._id",
+            "TestTable._id",
             "table.name",
             "TestTable.name",
           ])
@@ -481,6 +502,8 @@ describe("query utils", () => {
             "name",
 
             // deep 1 aux primitive props
+            "aux._id",
+            "auxTable._id",
             "aux.title",
             "auxTable.title",
           ])
@@ -495,6 +518,8 @@ describe("query utils", () => {
             "title",
 
             // deep 1 dependency primitive props
+            "table._id",
+            "TestTable._id",
             "table.name",
             "TestTable.name",
           ])

packages/server/src/threads/automation.ts

@@ -367,6 +367,8 @@ class Orchestrator {
         if (e.errno === "ETIME") {
           span?.addTags({ timedOut: true })
           console.warn(`Automation execution timed out after ${timeout}ms`)
+        } else {
+          throw e
         }
       }
 

packages/server/src/utilities/workerRequests.ts

@@ -8,7 +8,15 @@ import {
   logging,
   env as coreEnv,
 } from "@budibase/backend-core"
-import { Ctx, User, EmailInvite, EmailAttachment } from "@budibase/types"
+import {
+  Ctx,
+  User,
+  EmailInvite,
+  EmailAttachment,
+  SendEmailResponse,
+  SendEmailRequest,
+  EmailTemplatePurpose,
+} from "@budibase/types"
 
 interface Request {
   ctx?: Ctx
@@ -110,25 +118,23 @@ export async function sendSmtpEmail({
   invite?: EmailInvite
 }) {
   // tenant ID will be set in header
-  const response = await fetch(
+  const request: SendEmailRequest = {
-    checkSlashesInUrl(env.WORKER_URL + `/api/global/email/send`),
-    createRequest({
-      method: "POST",
-      body: {
     email: to,
     from,
     contents,
     subject,
     cc,
     bcc,
-        purpose: "custom",
+    purpose: EmailTemplatePurpose.CUSTOM,
     automation,
     invite,
     attachments,
-      },
+  }
-    })
+  const response = await fetch(
+    checkSlashesInUrl(env.WORKER_URL + `/api/global/email/send`),
+    createRequest({ method: "POST", body: request })
   )
-  return checkResponse(response, "send email")
+  return (await checkResponse(response, "send email")) as SendEmailResponse
 }
 
 export async function removeAppFromUserRoles(ctx: Ctx, appId: string) {

packages/types/package.json

@@ -17,6 +17,7 @@
     "@budibase/nano": "10.1.5",
     "@types/json-schema": "^7.0.15",
     "@types/koa": "2.13.4",
+    "@types/nodemailer": "^6.4.17",
     "@types/redlock": "4.0.7",
     "koa-useragent": "^4.1.0",
     "rimraf": "3.0.2",

packages/types/src/api/web/global/email.ts

@@ -1,4 +1,5 @@
 import { EmailAttachment, EmailInvite } from "../../../documents"
+import SMTPTransport from "nodemailer/lib/smtp-transport"
 
 export enum EmailTemplatePurpose {
   CORE = "core",
@@ -12,17 +13,17 @@ export enum EmailTemplatePurpose {
 export interface SendEmailRequest {
   workspaceId?: string
   email: string
-  userId: string
+  userId?: string
   purpose: EmailTemplatePurpose
   contents?: string
   from?: string
   subject: string
-  cc?: boolean
+  cc?: string
-  bcc?: boolean
+  bcc?: string
   automation?: boolean
   invite?: EmailInvite
   attachments?: EmailAttachment[]
 }
-export interface SendEmailResponse extends Record<string, any> {
+export interface SendEmailResponse extends SMTPTransport.SentMessageInfo {
   message: string
 }

packages/types/src/documents/app/automation/automation.ts

@@ -1,10 +1,10 @@
 import { Document } from "../../document"
 import { User } from "../../global"
-import { ReadStream } from "fs"
 import { Row } from "../row"
 import { Table } from "../table"
 import { AutomationStep, AutomationTrigger } from "./schema"
 import { ContextEmitter } from "../../../sdk"
+import { Readable } from "stream"
 
 export enum AutomationIOType {
   OBJECT = "object",
@@ -108,8 +108,8 @@ export interface SendEmailOpts {
   subject: string
   // info Pass in a structure of information to be stored alongside the invitation.
   info?: any
-  cc?: boolean
+  cc?: string
-  bcc?: boolean
+  bcc?: string
   automation?: boolean
   invite?: EmailInvite
   attachments?: EmailAttachment[]
@@ -269,7 +269,7 @@ export type AutomationAttachment = {
 
 export type AutomationAttachmentContent = {
   filename: string
-  content: ReadStream | NodeJS.ReadableStream
+  content: Readable
 }
 
 export type BucketedContent = AutomationAttachmentContent & {

packages/types/src/sdk/search.ts

@@ -3,6 +3,7 @@ import { Row, DocumentType, Table, Datasource } from "../documents"
 import { SortOrder, SortType } from "../api"
 import { Knex } from "knex"
 import { Aggregation } from "./row"
+import _ from "lodash"
 
 export enum BasicOperator {
   EQUAL = "equal",
@@ -83,7 +84,7 @@ type RangeFilter = Record<
 type LogicalFilter = { conditions: SearchFilters[] }
 
 export function isLogicalFilter(filter: any): filter is LogicalFilter {
-  return "conditions" in filter
+  return _.isPlainObject(filter) && "conditions" in filter
 }
 
 export type AnySearchFilter = BasicFilter | ArrayFilter | RangeFilter

packages/worker/package.json

@@ -62,6 +62,7 @@
     "koa-body": "4.2.0",
     "koa-compress": "4.0.1",
     "koa-passport": "4.1.4",
+    "koa-redis": "^4.0.1",
     "koa-send": "5.0.1",
     "koa-session": "5.13.1",
     "koa-static": "5.0.0",
@@ -85,6 +86,7 @@
     "@types/koa__router": "12.0.4",
     "@types/lodash": "4.14.200",
     "@types/node-fetch": "2.6.4",
+    "@types/nodemailer": "^6.4.17",
     "@types/server-destroy": "1.0.1",
     "@types/supertest": "2.0.14",
     "@types/uuid": "8.3.4",

packages/worker/src/api/controllers/global/email.ts

@@ -24,10 +24,13 @@ export async function sendEmail(
     invite,
     attachments,
   } = ctx.request.body
-  let user: any
+  let user: User | undefined = undefined
   if (userId) {
     const db = tenancy.getGlobalDB()
-    user = await db.get<User>(userId)
+    user = await db.tryGet<User>(userId)
+  }
+  if (!user) {
+    ctx.throw(404, "User not found.")
   }
   const response = await sendEmailFn(email, purpose, {
     workspaceId,

packages/worker/src/api/routes/global/tests/auth.spec.ts

@@ -311,7 +311,7 @@ describe("/api/global/auth", () => {
       })
     })
 
-    describe("GET /api/global/auth/:tenantId/oidc/callback", () => {
+    describe.skip("GET /api/global/auth/:tenantId/oidc/callback", () => {
       it("logs in", async () => {
         const email = `${generator.guid()}@example.com`
 

packages/worker/src/index.ts

@@ -4,7 +4,7 @@ if (process.env.DD_APM_ENABLED) {
 
 // need to load environment first
 import env from "./environment"
-import Application from "koa"
+import Application, { Middleware } from "koa"
 import { bootstrap } from "global-agent"
 import * as db from "./db"
 import { sdk as proSdk } from "@budibase/pro"
@@ -20,6 +20,7 @@ import {
   cache,
   features,
 } from "@budibase/backend-core"
+import RedisStore from "koa-redis"
 
 db.init()
 import koaBody from "koa-body"
@@ -52,7 +53,23 @@ app.proxy = true
 app.use(handleScimBody)
 app.use(koaBody({ multipart: true }))
 
-app.use(koaSession(app))
+const sessionMiddleware: Middleware = async (ctx: any, next: any) => {
+  const redisClient = await new redis.Client(
+    redis.utils.Databases.SESSIONS
+  ).init()
+  return koaSession(
+    {
+      // @ts-ignore
+      store: new RedisStore({ client: redisClient.getClient() }),
+      key: "koa:sess",
+      maxAge: 86400000, // one day
+    },
+    app
+  )(ctx, next)
+}
+
+app.use(sessionMiddleware)
+
 app.use(middleware.correlation)
 app.use(middleware.pino)
 app.use(middleware.ip)

packages/worker/src/koa-redis.d.ts

@@ -0,0 +1 @@
+declare module "koa-redis" {}

packages/worker/src/utilities/email.ts

@@ -13,7 +13,8 @@ import { configs, cache, objectStore } from "@budibase/backend-core"
 import ical from "ical-generator"
 import _ from "lodash"
 
-const nodemailer = require("nodemailer")
+import nodemailer from "nodemailer"
+import SMTPTransport from "nodemailer/lib/smtp-transport"
 
 const TEST_MODE = env.ENABLE_EMAIL_TEST_MODE && env.isDev()
 const TYPE = TemplateType.EMAIL
@@ -26,7 +27,7 @@ const FULL_EMAIL_PURPOSES = [
 ]
 
 function createSMTPTransport(config?: SMTPInnerConfig) {
-  let options: any
+  let options: SMTPTransport.Options
   let secure = config?.secure
   // default it if not specified
   if (secure == null) {
@@ -161,7 +162,7 @@ export async function sendEmail(
   const code = await getLinkCode(purpose, email, opts.user, opts?.info)
   let context = await getSettingsTemplateContext(purpose, code)
 
-  let message: any = {
+  let message: Parameters<typeof transport.sendMail>[0] = {
     from: opts?.from || config?.from,
     html: await buildEmail(purpose, email, context, {
       user: opts?.user,

yarn.lock

@@ -2695,6 +2695,13 @@
   dependencies:
     regenerator-runtime "^0.14.0"
 
+"@babel/runtime@^7.8.3":
+  version "7.26.9"
+  resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.26.9.tgz#aa4c6facc65b9cb3f87d75125ffd47781b475433"
+  integrity sha512-aA63XwOkcl4xxQa3HjPMqOP6LiK0ZDv3mUPYEFXkpHbaFjtGggE1A61FjFzJnB+p7/oy2gA8E+rcBNl/zC1tMg==
+  dependencies:
+    regenerator-runtime "^0.14.0"
+
 "@babel/template@^7.22.15", "@babel/template@^7.22.5", "@babel/template@^7.25.9", "@babel/template@^7.3.3":
   version "7.25.9"
   resolved "https://registry.yarnpkg.com/@babel/template/-/template-7.25.9.tgz#ecb62d81a8a6f5dc5fe8abfc3901fc52ddf15016"
@@ -2778,9 +2785,9 @@
     through2 "^2.0.0"
 
 "@budibase/pro@npm:@budibase/pro@latest":
-  version "3.4.12"
+  version "3.4.16"
-  resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-3.4.12.tgz#60e630944de4e2de970a04179d8f0f57d48ce75e"
+  resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-3.4.16.tgz#c482a400e27b7e89ca73092c4c81bdeac1d24581"
-  integrity sha512-msUBmcWxRDg+ugjZvd27XudERQqtQRdiARsO8MaDVTcp5ejIXgshEIVVshHOCj3hcbRblw9pXvBIMI53iTMUsA==
+  integrity sha512-8ECnqOh9jQ10KlQEwmKPFcoVGE+2gGgSybj+vbshwDp1zAW76doyMR2DMNjEatNpWVnpoMnTkDWtE9aqQ5v0vQ==
   dependencies:
     "@anthropic-ai/sdk" "^0.27.3"
     "@budibase/backend-core" "*"
@@ -6768,6 +6775,13 @@
   dependencies:
     undici-types "~6.19.2"
 
+"@types/nodemailer@^6.4.17":
+  version "6.4.17"
+  resolved "https://registry.yarnpkg.com/@types/nodemailer/-/nodemailer-6.4.17.tgz#5c82a42aee16a3dd6ea31446a1bd6a447f1ac1a4"
+  integrity sha512-I9CCaIp6DTldEg7vyUTZi8+9Vo0hi1/T8gv3C89yk1rSAAzoKQ8H8ki/jBYJSFoH/BisgLP8tkZMlQ91CIquww==
+  dependencies:
+    "@types/node" "*"
+
 "@types/normalize-package-data@^2.4.0":
   version "2.4.1"
   resolved "https://registry.yarnpkg.com/@types/normalize-package-data/-/normalize-package-data-2.4.1.tgz#d3357479a0fdfdd5907fe67e17e0a85c906e1301"
@@ -9041,7 +9055,14 @@ co-body@^5.1.1:
     raw-body "^2.2.0"
     type-is "^1.6.14"
 
-co@^4.6.0:
+co-wrap-all@^1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/co-wrap-all/-/co-wrap-all-1.0.0.tgz#370ae3e8333510a53f6b2f7fdfbe4568a11b7ecf"
+  integrity sha512-aru6gLi2vTUazr+MxVm3Rv6ST7/EKtFj9BrfkcOrbCO2Qv6LqJdE71m88HhHiBEviKw/ucVrwoGLrq2xHpOsJA==
+  dependencies:
+    co "^4.0.0"
+
+co@^4.0.0, co@^4.6.0:
   version "4.6.0"
   resolved "https://registry.yarnpkg.com/co/-/co-4.6.0.tgz#6ea6bdf3d853ae54ccb8e47bfa0bf3f9031fb184"
   integrity sha512-QVb0dM5HvG+uaxitm8wONl7jltx8dqhfU33DcqtOZcLSVIKSDDLDi7+0LbAKiyI8hD9u42m2YxXSkMGWThaecQ==
@@ -13177,7 +13198,7 @@ ioredis@5.3.2:
     redis-parser "^3.0.0"
     standard-as-callback "^2.1.0"
 
-ioredis@^4.28.5:
+ioredis@^4.14.1, ioredis@^4.28.5:
   version "4.28.5"
   resolved "https://registry.yarnpkg.com/ioredis/-/ioredis-4.28.5.tgz#5c149e6a8d76a7f8fa8a504ffc85b7d5b6797f9f"
   integrity sha512-3GYo0GJtLqgNXj4YhrisLaNNvWSNwSS2wS4OELGfGxH8I69+XfNdnmV1AyN+ZqMh0i7eX+SWjrwFKDBDgfBC1A==
@@ -14677,6 +14698,16 @@ koa-pino-logger@4.0.0:
   dependencies:
     pino-http "^6.5.0"
 
+koa-redis@^4.0.1:
+  version "4.0.1"
+  resolved "https://registry.yarnpkg.com/koa-redis/-/koa-redis-4.0.1.tgz#57ac1b46d9ab851221a9f4952c1e8d4bf289db40"
+  integrity sha512-o2eTVNo1NBnloeUGhHed5Q2ZvJSLpUEj/+E1/7oH5EmH8WuQ+QLdl/VawkshxdFQ47W1p6V09lM3hCTu7D0YnQ==
+  dependencies:
+    "@babel/runtime" "^7.8.3"
+    co-wrap-all "^1.0.0"
+    debug "^4.1.1"
+    ioredis "^4.14.1"
+
 koa-router@^10.0.0:
   version "10.1.1"
   resolved "https://registry.yarnpkg.com/koa-router/-/koa-router-10.1.1.tgz#20809f82648518b84726cd445037813cd99f17ff"