Merge branch 'master' into execute-script-v2

docs/CONTRIBUTING.md

@@ -108,7 +108,7 @@ You can install them following any of the steps described below:
   - Installation steps: https://asdf-vm.com/guide/getting-started.html
   - asdf plugin add nodejs
   - asdf plugin add python
-  - npm install -g yarn
+  - asdf plugin add yarn
 
 ### Using NVM and pyenv
 

lerna.json

@@ -1,6 +1,6 @@
 {
   "$schema": "node_modules/lerna/schemas/lerna-schema.json",
-  "version": "3.4.20",
+  "version": "3.4.21",
   "npmClient": "yarn",
   "concurrency": 20,
   "command": {

packages/builder/src/stores/builder/screenComponent.ts

@@ -81,11 +81,11 @@ export const screenComponentErrorList = derived(
     const errors: UIComponentError[] = []
 
     function checkComponentErrors(component: Component, ancestors: string[]) {
+      errors.push(...getMissingAncestors(component, definitions, ancestors))
       errors.push(
         ...getInvalidDatasources(screen, component, datasources, definitions)
       )
       errors.push(...getMissingRequiredSettings(component, definitions))
-      errors.push(...getMissingAncestors(component, definitions, ancestors))
 
       for (const child of component._children || []) {
         checkComponentErrors(child, [...ancestors, component._component])
@@ -239,7 +239,10 @@ function getMissingAncestors(
   ancestors: string[]
 ): UIComponentError[] {
   const definition = definitions[component._component]
-
+  if (ancestors.some(a => !a.startsWith(BudibasePrefix))) {
+    // We don't have a way to know what components are used within a plugin component
+    return []
+  }
   if (!definition?.requiredAncestors?.length) {
     return []
   }

packages/client/src/index.js

@@ -1,142 +0,0 @@
-import ClientApp from "./components/ClientApp.svelte"
-import UpdatingApp from "./components/UpdatingApp.svelte"
-import {
-  builderStore,
-  appStore,
-  blockStore,
-  componentStore,
-  environmentStore,
-  dndStore,
-  eventStore,
-  hoverStore,
-  stateStore,
-  routeStore,
-} from "./stores"
-import loadSpectrumIcons from "@budibase/bbui/spectrum-icons-vite.js"
-import { get } from "svelte/store"
-import { initWebsocket } from "./websocket.js"
-
-// Provide svelte and svelte/internal as globals for custom components
-import * as svelte from "svelte"
-import * as internal from "svelte/internal"
-
-window.svelte_internal = internal
-window.svelte = svelte
-
-// Initialise spectrum icons
-loadSpectrumIcons()
-
-let app
-
-const loadBudibase = async () => {
-  // Update builder store with any builder flags
-  builderStore.set({
-    ...get(builderStore),
-    inBuilder: !!window["##BUDIBASE_IN_BUILDER##"],
-    layout: window["##BUDIBASE_PREVIEW_LAYOUT##"],
-    screen: window["##BUDIBASE_PREVIEW_SCREEN##"],
-    selectedComponentId: window["##BUDIBASE_SELECTED_COMPONENT_ID##"],
-    previewId: window["##BUDIBASE_PREVIEW_ID##"],
-    theme: window["##BUDIBASE_PREVIEW_THEME##"],
-    customTheme: window["##BUDIBASE_PREVIEW_CUSTOM_THEME##"],
-    previewDevice: window["##BUDIBASE_PREVIEW_DEVICE##"],
-    navigation: window["##BUDIBASE_PREVIEW_NAVIGATION##"],
-    hiddenComponentIds: window["##BUDIBASE_HIDDEN_COMPONENT_IDS##"],
-    usedPlugins: window["##BUDIBASE_USED_PLUGINS##"],
-    location: window["##BUDIBASE_LOCATION##"],
-    snippets: window["##BUDIBASE_SNIPPETS##"],
-    componentErrors: window["##BUDIBASE_COMPONENT_ERRORS##"],
-  })
-
-  // Set app ID - this window flag is set by both the preview and the real
-  // server rendered app HTML
-  appStore.actions.setAppId(window["##BUDIBASE_APP_ID##"])
-
-  // Set the flag used to determine if the app is being loaded via an iframe
-  appStore.actions.setAppEmbedded(
-    window["##BUDIBASE_APP_EMBEDDED##"] === "true"
-  )
-
-  if (window.MIGRATING_APP) {
-    new UpdatingApp({
-      target: window.document.body,
-    })
-    return
-  }
-
-  // Fetch environment info
-  if (!get(environmentStore)?.loaded) {
-    await environmentStore.actions.fetchEnvironment()
-  }
-
-  // Register handler for runtime events from the builder
-  window.handleBuilderRuntimeEvent = (type, data) => {
-    if (!window["##BUDIBASE_IN_BUILDER##"]) {
-      return
-    }
-    if (type === "event-completed") {
-      eventStore.actions.resolveEvent(data)
-    } else if (type === "eject-block") {
-      const block = blockStore.actions.getBlock(data)
-      block?.eject()
-    } else if (type === "dragging-new-component") {
-      const { dragging, component } = data
-      if (dragging) {
-        const definition =
-          componentStore.actions.getComponentDefinition(component)
-        dndStore.actions.startDraggingNewComponent({ component, definition })
-      } else {
-        dndStore.actions.reset()
-      }
-    } else if (type === "request-context") {
-      const { selectedComponentInstance, screenslotInstance } =
-        get(componentStore)
-      const instance = selectedComponentInstance || screenslotInstance
-      const context = instance?.getDataContext()
-      let stringifiedContext = null
-      try {
-        stringifiedContext = JSON.stringify(context)
-      } catch (error) {
-        // Ignore - invalid context
-      }
-      eventStore.actions.dispatchEvent("provide-context", {
-        context: stringifiedContext,
-      })
-    } else if (type === "hover-component") {
-      hoverStore.actions.hoverComponent(data, false)
-    } else if (type === "builder-meta") {
-      builderStore.actions.setMetadata(data)
-    } else if (type === "builder-state") {
-      const [[key, value]] = Object.entries(data)
-      stateStore.actions.setValue(key, value)
-    } else if (type === "builder-url-test-data") {
-      const { route, testValue } = data
-      routeStore.actions.setTestUrlParams(route, testValue)
-    }
-  }
-
-  // Register any custom components
-  if (window["##BUDIBASE_CUSTOM_COMPONENTS##"]) {
-    window["##BUDIBASE_CUSTOM_COMPONENTS##"].forEach(component => {
-      componentStore.actions.registerCustomComponent(component)
-    })
-  }
-
-  // Make a callback available for custom component bundles to register
-  // themselves at runtime
-  window.registerCustomComponent =
-    componentStore.actions.registerCustomComponent
-
-  // Initialise websocket
-  initWebsocket()
-
-  // Create app if one hasn't been created yet
-  if (!app) {
-    app = new ClientApp({
-      target: window.document.body,
-    })
-  }
-}
-
-// Attach to window so the HTML template can call this when it loads
-window.loadBudibase = loadBudibase

packages/pro

@@ -1 +1 @@
-Subproject commit 45f5673d5e5ab3c22deb6663cea2e31a628aa133
+Subproject commit e3843dd4eaced68ae063355b77df200dbc789c98

packages/server/src/api/controllers/plugin/index.ts

@@ -11,6 +11,7 @@ import {
   UploadPluginResponse,
   FetchPluginResponse,
   DeletePluginResponse,
+  PluginMetadata,
 } from "@budibase/types"
 import env from "../../../environment"
 import { clientAppSocket } from "../../../websockets"
@@ -53,10 +54,11 @@ export async function create(
   const { source, url, headers, githubToken } = ctx.request.body
 
   try {
-    let metadata
+    let metadata: PluginMetadata
-    let directory
+    let directory: string
+
     // Generating random name as a backup and needed for url
-    let name = "PLUGIN_" + Math.floor(100000 + Math.random() * 900000)
+    const name = "PLUGIN_" + Math.floor(100000 + Math.random() * 900000)
 
     switch (source) {
       case PluginSource.NPM: {
@@ -81,12 +83,14 @@ export async function create(
         directory = directoryUrl
         break
       }
+      default:
+        ctx.throw(400, "Invalid source")
     }
 
-    pluginCore.validate(metadata?.schema)
+    pluginCore.validate(metadata.schema)
 
     // Only allow components in cloud
-    if (!env.SELF_HOSTED && metadata?.schema?.type !== PluginType.COMPONENT) {
+    if (!env.SELF_HOSTED && metadata.schema?.type !== PluginType.COMPONENT) {
       throw new Error(
         "Only component plugins are supported outside of self-host"
       )

packages/server/src/automations/tests/steps/sendSmtpEmail.spec.ts

@@ -1,3 +1,4 @@
+import { SendEmailResponse } from "@budibase/types"
 import TestConfiguration from "../../../tests/utilities/TestConfiguration"
 import * as workerRequests from "../../../utilities/workerRequests"
 
@@ -5,17 +6,18 @@ jest.mock("../../../utilities/workerRequests", () => ({
   sendSmtpEmail: jest.fn(),
 }))
 
-function generateResponse(to: string, from: string) {
+function generateResponse(to: string, from: string): SendEmailResponse {
   return {
-    success: true,
+    message: `Email sent to ${to}.`,
-    response: {
+    accepted: [to],
-      accepted: [to],
+    envelope: {
-      envelope: {
+      from: from,
-        from: from,
+      to: [to],
-        to: [to],
-      },
-      message: `Email sent to ${to}.`,
     },
+    messageId: "messageId",
+    pending: [],
+    rejected: [],
+    response: "response",
   }
 }
 

packages/server/src/sdk/index.ts

@@ -7,7 +7,7 @@ import { default as queries } from "./app/queries"
 import { default as rows } from "./app/rows"
 import { default as links } from "./app/links"
 import { default as users } from "./users"
-import { default as plugins } from "./plugins"
+import * as plugins from "./plugins"
 import * as views from "./app/views"
 import * as permissions from "./app/permissions"
 import * as rowActions from "./app/rowActions"

packages/server/src/sdk/plugins/index.ts

@@ -1,5 +1,41 @@
-import * as plugins from "./plugins"
+import { KoaFile, Plugin, PluginSource, PluginType } from "@budibase/types"
+import {
+  db as dbCore,
+  objectStore,
+  plugins as pluginCore,
+  tenancy,
+} from "@budibase/backend-core"
+import { fileUpload } from "../../api/controllers/plugin/file"
+import env from "../../environment"
+import { clientAppSocket } from "../../websockets"
+import { sdk as pro } from "@budibase/pro"
 
-export default {
+export async function fetch(type?: PluginType): Promise<Plugin[]> {
-  ...plugins,
+  const db = tenancy.getGlobalDB()
+  const response = await db.allDocs(
+    dbCore.getPluginParams(null, {
+      include_docs: true,
+    })
+  )
+  let plugins = response.rows.map((row: any) => row.doc) as Plugin[]
+  plugins = await objectStore.enrichPluginURLs(plugins)
+  if (type) {
+    return plugins.filter((plugin: Plugin) => plugin.schema?.type === type)
+  } else {
+    return plugins
+  }
+}
+
+export async function processUploaded(plugin: KoaFile, source: PluginSource) {
+  const { metadata, directory } = await fileUpload(plugin)
+  pluginCore.validate(metadata.schema)
+
+  // Only allow components in cloud
+  if (!env.SELF_HOSTED && metadata.schema?.type !== PluginType.COMPONENT) {
+    throw new Error("Only component plugins are supported outside of self-host")
+  }
+
+  const doc = await pro.plugins.storePlugin(metadata, directory, source)
+  clientAppSocket?.emit("plugin-update", { name: doc.name, hash: doc.hash })
+  return doc
 }

packages/server/src/sdk/plugins/plugins.ts

@@ -1,41 +0,0 @@
-import { KoaFile, Plugin, PluginSource, PluginType } from "@budibase/types"
-import {
-  db as dbCore,
-  objectStore,
-  plugins as pluginCore,
-  tenancy,
-} from "@budibase/backend-core"
-import { fileUpload } from "../../api/controllers/plugin/file"
-import env from "../../environment"
-import { clientAppSocket } from "../../websockets"
-import { sdk as pro } from "@budibase/pro"
-
-export async function fetch(type?: PluginType): Promise<Plugin[]> {
-  const db = tenancy.getGlobalDB()
-  const response = await db.allDocs(
-    dbCore.getPluginParams(null, {
-      include_docs: true,
-    })
-  )
-  let plugins = response.rows.map((row: any) => row.doc) as Plugin[]
-  plugins = await objectStore.enrichPluginURLs(plugins)
-  if (type) {
-    return plugins.filter((plugin: Plugin) => plugin.schema?.type === type)
-  } else {
-    return plugins
-  }
-}
-
-export async function processUploaded(plugin: KoaFile, source?: PluginSource) {
-  const { metadata, directory } = await fileUpload(plugin)
-  pluginCore.validate(metadata?.schema)
-
-  // Only allow components in cloud
-  if (!env.SELF_HOSTED && metadata?.schema?.type !== PluginType.COMPONENT) {
-    throw new Error("Only component plugins are supported outside of self-host")
-  }
-
-  const doc = await pro.plugins.storePlugin(metadata, directory, source)
-  clientAppSocket?.emit("plugin-update", { name: doc.name, hash: doc.hash })
-  return doc
-}

packages/server/src/utilities/fileSystem/plugin.ts

@@ -1,4 +1,4 @@
-import { Plugin } from "@budibase/types"
+import { Plugin, PluginUpload } from "@budibase/types"
 import { budibaseTempDir } from "../budibaseDir"
 import fs from "fs"
 import { join } from "path"
@@ -8,31 +8,31 @@ import stream from "stream"
 const DATASOURCE_PATH = join(budibaseTempDir(), "datasource")
 const AUTOMATION_PATH = join(budibaseTempDir(), "automation")
 
-export const getPluginMetadata = async (path: string) => {
+export const getPluginMetadata = async (
-  let metadata: any = {}
+  path: string
+): Promise<PluginUpload> => {
+  let pkg: any
+  let schema: any
   try {
-    const pkg = fs.readFileSync(join(path, "package.json"), "utf8")
+    pkg = JSON.parse(fs.readFileSync(join(path, "package.json"), "utf8"))
-    const schema = fs.readFileSync(join(path, "schema.json"), "utf8")
+    schema = JSON.parse(fs.readFileSync(join(path, "schema.json"), "utf8"))
-
+    if (!pkg.name) {
-    metadata.schema = JSON.parse(schema)
+      throw new Error("package.json is missing 'name'.")
-    metadata.package = JSON.parse(pkg)
+    }
-
+    if (!pkg.version) {
-    if (
+      throw new Error("package.json is missing 'version'.")
-      !metadata.package.name ||
+    }
-      !metadata.package.version ||
+    if (!pkg.description) {
-      !metadata.package.description
+      throw new Error("package.json is missing 'description'.")
-    ) {
-      throw new Error(
-        "package.json is missing one of 'name', 'version' or 'description'."
-      )
     }
   } catch (err: any) {
     throw new Error(
-      `Unable to process schema.json/package.json in plugin. ${err.message}`
+      `Unable to process schema.json/package.json in plugin. ${err.message}`,
+      { cause: err }
     )
   }
 
-  return { metadata, directory: path }
+  return { metadata: { package: pkg, schema }, directory: path }
 }
 
 async function getPluginImpl(path: string, plugin: Plugin) {

packages/server/src/utilities/workerRequests.ts

@@ -8,7 +8,15 @@ import {
   logging,
   env as coreEnv,
 } from "@budibase/backend-core"
-import { Ctx, User, EmailInvite, EmailAttachment } from "@budibase/types"
+import {
+  Ctx,
+  User,
+  EmailInvite,
+  EmailAttachment,
+  SendEmailResponse,
+  SendEmailRequest,
+  EmailTemplatePurpose,
+} from "@budibase/types"
 
 interface Request {
   ctx?: Ctx
@@ -110,25 +118,23 @@ export async function sendSmtpEmail({
   invite?: EmailInvite
 }) {
   // tenant ID will be set in header
+  const request: SendEmailRequest = {
+    email: to,
+    from,
+    contents,
+    subject,
+    cc,
+    bcc,
+    purpose: EmailTemplatePurpose.CUSTOM,
+    automation,
+    invite,
+    attachments,
+  }
   const response = await fetch(
     checkSlashesInUrl(env.WORKER_URL + `/api/global/email/send`),
-    createRequest({
+    createRequest({ method: "POST", body: request })
-      method: "POST",
-      body: {
-        email: to,
-        from,
-        contents,
-        subject,
-        cc,
-        bcc,
-        purpose: "custom",
-        automation,
-        invite,
-        attachments,
-      },
-    })
   )
-  return checkResponse(response, "send email")
+  return (await checkResponse(response, "send email")) as SendEmailResponse
 }
 
 export async function removeAppFromUserRoles(ctx: Ctx, appId: string) {

packages/server/src/watch.ts

@@ -3,7 +3,8 @@ import env from "./environment"
 import chokidar from "chokidar"
 import fs from "fs"
 import { constants, tenancy } from "@budibase/backend-core"
-import pluginsSdk from "./sdk/plugins"
+import { processUploaded } from "./sdk/plugins"
+import { PluginSource } from "@budibase/types"
 
 export function watch() {
   const watchPath = path.join(env.PLUGINS_DIR, "./**/*.tar.gz")
@@ -27,7 +28,7 @@ export function watch() {
           const split = path.split("/")
           const name = split[split.length - 1]
           console.log("Importing plugin:", path)
-          await pluginsSdk.processUploaded({ name, path })
+          await processUploaded({ name, path }, PluginSource.FILE)
         } catch (err: any) {
           const message = err?.message ? err?.message : err
           console.error("Failed to import plugin:", message)

packages/types/package.json

@@ -17,6 +17,7 @@
     "@budibase/nano": "10.1.5",
     "@types/json-schema": "^7.0.15",
     "@types/koa": "2.13.4",
+    "@types/nodemailer": "^6.4.17",
     "@types/redlock": "4.0.7",
     "koa-useragent": "^4.1.0",
     "rimraf": "3.0.2",

packages/types/src/api/web/global/email.ts

@@ -1,4 +1,5 @@
 import { EmailAttachment, EmailInvite } from "../../../documents"
+import SMTPTransport from "nodemailer/lib/smtp-transport"
 
 export enum EmailTemplatePurpose {
   CORE = "core",
@@ -10,19 +11,18 @@ export enum EmailTemplatePurpose {
 }
 
 export interface SendEmailRequest {
-  workspaceId?: string
   email: string
-  userId: string
+  userId?: string
   purpose: EmailTemplatePurpose
   contents?: string
   from?: string
   subject: string
-  cc?: boolean
+  cc?: string
-  bcc?: boolean
+  bcc?: string
   automation?: boolean
   invite?: EmailInvite
   attachments?: EmailAttachment[]
 }
-export interface SendEmailResponse extends Record<string, any> {
+export interface SendEmailResponse extends SMTPTransport.SentMessageInfo {
   message: string
 }

packages/types/src/documents/app/automation/automation.ts

@@ -1,10 +1,10 @@
 import { Document } from "../../document"
 import { User } from "../../global"
-import { ReadStream } from "fs"
 import { Row } from "../row"
 import { Table } from "../table"
 import { AutomationStep, AutomationTrigger } from "./schema"
 import { ContextEmitter } from "../../../sdk"
+import { Readable } from "stream"
 
 export enum AutomationIOType {
   OBJECT = "object",
@@ -100,7 +100,7 @@ export interface SendEmailOpts {
   // workspaceId If finer grain controls being used then this will lookup config for workspace.
   workspaceId?: string
   // user If sending to an existing user the object can be provided, this is used in the context.
-  user: User
+  user?: User
   // from If sending from an address that is not what is configured in the SMTP config.
   from?: string
   // contents If sending a custom email then can supply contents which will be added to it.
@@ -109,8 +109,8 @@ export interface SendEmailOpts {
   subject: string
   // info Pass in a structure of information to be stored alongside the invitation.
   info?: any
-  cc?: boolean
+  cc?: string
-  bcc?: boolean
+  bcc?: string
   automation?: boolean
   invite?: EmailInvite
   attachments?: EmailAttachment[]
@@ -270,7 +270,7 @@ export type AutomationAttachment = {
 
 export type AutomationAttachmentContent = {
   filename: string
-  content: ReadStream | NodeJS.ReadableStream
+  content: Readable
 }
 
 export type BucketedContent = AutomationAttachmentContent & {

packages/types/src/documents/global/plugin.ts

@@ -24,10 +24,7 @@ export interface Plugin extends Document {
   source: PluginSource
   package: { [key: string]: any }
   hash: string
-  schema: {
+  schema: PluginSchema
-    type: PluginType
-    [key: string]: any
-  }
   iconFileName?: string
   // Populated on read
   jsUrl?: string
@@ -36,3 +33,24 @@ export interface Plugin extends Document {
 }
 
 export const PLUGIN_TYPE_ARR = Object.values(PluginType)
+
+export interface PluginSchema {
+  type: PluginType
+  [key: string]: any
+}
+
+interface Package {
+  name: string
+  version: string
+  description: string
+}
+
+export interface PluginMetadata {
+  schema: PluginSchema
+  package: Package
+}
+
+export interface PluginUpload {
+  metadata: PluginMetadata
+  directory: string
+}

packages/worker/package.json

@@ -85,11 +85,14 @@
     "@types/jsonwebtoken": "9.0.3",
     "@types/koa__router": "12.0.4",
     "@types/lodash": "4.14.200",
+    "@types/maildev": "^0.0.7",
     "@types/node-fetch": "2.6.4",
+    "@types/nodemailer": "^6.4.17",
     "@types/server-destroy": "1.0.1",
     "@types/supertest": "2.0.14",
     "@types/uuid": "8.3.4",
     "jest": "29.7.0",
+    "maildev": "^2.2.1",
     "nock": "^13.5.4",
     "nodemon": "2.0.15",
     "rimraf": "3.0.2",

packages/worker/src/api/controllers/global/email.ts

@@ -11,7 +11,6 @@ export async function sendEmail(
   ctx: UserCtx<SendEmailRequest, SendEmailResponse>
 ) {
   let {
-    workspaceId,
     email,
     userId,
     purpose,
@@ -24,13 +23,15 @@ export async function sendEmail(
     invite,
     attachments,
   } = ctx.request.body
-  let user: any
+  let user: User | undefined = undefined
   if (userId) {
     const db = tenancy.getGlobalDB()
-    user = await db.get<User>(userId)
+    user = await db.tryGet<User>(userId)
+    if (!user) {
+      ctx.throw(404, "User not found.")
+    }
   }
   const response = await sendEmailFn(email, purpose, {
-    workspaceId,
     user,
     contents,
     from,

packages/worker/src/api/routes/global/tests/email.spec.ts

@@ -1,33 +1,269 @@
-jest.mock("nodemailer")
+import { EmailTemplatePurpose, SendEmailRequest } from "@budibase/types"
-import { EmailTemplatePurpose } from "@budibase/types"
+import { TestConfiguration } from "../../../../tests"
-import { TestConfiguration, mocks } from "../../../../tests"
+import {
-
+  captureEmail,
-const sendMailMock = mocks.email.mock()
+  deleteAllEmail,
+  getAttachments,
+  Mailserver,
+  startMailserver,
+  stopMailserver,
+} from "../../../../tests/mocks/email"
+import { objectStore } from "@budibase/backend-core"
 
 describe("/api/global/email", () => {
   const config = new TestConfiguration()
+  let mailserver: Mailserver
 
   beforeAll(async () => {
     await config.beforeAll()
+    mailserver = await startMailserver(config)
   })
 
   afterAll(async () => {
+    await stopMailserver(mailserver)
     await config.afterAll()
   })
 
-  it("should be able to send an email (with mocking)", async () => {
+  beforeEach(async () => {
-    // initially configure settings
+    await deleteAllEmail(mailserver)
-    await config.saveSmtpConfig()
+  })
-    await config.saveSettingsConfig()
 
-    const res = await config.api.emails.sendEmail(
+  interface TestCase {
-      EmailTemplatePurpose.INVITATION
+    req: Partial<SendEmailRequest>
+    expectedStatus?: number
+    expectedContents?: string
+  }
+
+  const testCases: TestCase[] = [
+    {
+      req: {
+        purpose: EmailTemplatePurpose.WELCOME,
+      },
+      expectedContents: `Thanks for getting started with Budibase's Budibase platform.`,
+    },
+    {
+      req: {
+        purpose: EmailTemplatePurpose.INVITATION,
+      },
+      expectedContents: `Use the button below to set up your account and get started:`,
+    },
+    {
+      req: {
+        purpose: EmailTemplatePurpose.PASSWORD_RECOVERY,
+      },
+      expectedContents: `You recently requested to reset your password for your Budibase account in your Budibase platform`,
+    },
+    {
+      req: {
+        purpose: EmailTemplatePurpose.CUSTOM,
+        contents: "Hello, world!",
+      },
+      expectedContents: "Hello, world!",
+    },
+  ]
+
+  it.each(testCases)(
+    "can send $req.purpose emails",
+    async ({ req, expectedContents, expectedStatus }) => {
+      const email = await captureEmail(mailserver, async () => {
+        const res = await config.api.emails.sendEmail(
+          {
+            email: "to@example.com",
+            subject: "Test",
+            userId: config.user!._id,
+            purpose: EmailTemplatePurpose.WELCOME,
+            ...req,
+          },
+          {
+            status: expectedStatus || 200,
+          }
+        )
+        expect(res.message).toBeDefined()
+      })
+
+      expect(email.html).toContain(expectedContents)
+      expect(email.html).not.toContain("Invalid binding")
+    }
+  )
+
+  it("should be able to send an email with an attachment", async () => {
+    let bucket = "testbucket"
+    let filename = "test.txt"
+    await objectStore.upload({
+      bucket,
+      filename,
+      body: Buffer.from("test data"),
+    })
+    let presignedUrl = await objectStore.getPresignedUrl(
+      bucket,
+      filename,
+      60000
     )
 
-    expect(res.body.message).toBeDefined()
+    let attachmentObject = {
-    expect(sendMailMock).toHaveBeenCalled()
+      url: presignedUrl,
-    const emailCall = sendMailMock.mock.calls[0][0]
+      filename,
-    expect(emailCall.subject).toBe("Hello!")
+    }
-    expect(emailCall.html).not.toContain("Invalid binding")
+
+    const email = await captureEmail(mailserver, async () => {
+      const res = await config.api.emails.sendEmail({
+        email: "to@example.com",
+        subject: "Test",
+        userId: config.user!._id,
+        purpose: EmailTemplatePurpose.WELCOME,
+        attachments: [attachmentObject],
+      })
+      expect(res.message).toBeDefined()
+    })
+
+    expect(email.html).toContain(
+      "Thanks for getting started with Budibase's Budibase platform."
+    )
+    expect(email.html).not.toContain("Invalid binding")
+
+    const attachments = await getAttachments(mailserver, email)
+    expect(attachments).toEqual(["test data"])
+  })
+
+  it("should be able to send email without a userId", async () => {
+    const res = await config.api.emails.sendEmail({
+      email: "to@example.com",
+      subject: "Test",
+      purpose: EmailTemplatePurpose.WELCOME,
+    })
+    expect(res.message).toBeDefined()
+  })
+
+  it("should fail to send a password reset email without a userId", async () => {
+    const res = await config.api.emails.sendEmail(
+      {
+        email: "to@example.com",
+        subject: "Test",
+        purpose: EmailTemplatePurpose.PASSWORD_RECOVERY,
+      },
+      {
+        status: 400,
+      }
+    )
+    expect(res.message).toBeDefined()
+  })
+
+  it("can cc people", async () => {
+    const email = await captureEmail(mailserver, async () => {
+      await config.api.emails.sendEmail({
+        email: "to@example.com",
+        cc: "cc@example.com",
+        subject: "Test",
+        purpose: EmailTemplatePurpose.CUSTOM,
+        contents: "Hello, world!",
+      })
+    })
+
+    expect(email.cc).toEqual([{ address: "cc@example.com", name: "" }])
+  })
+
+  it("can bcc people", async () => {
+    const email = await captureEmail(mailserver, async () => {
+      await config.api.emails.sendEmail({
+        email: "to@example.com",
+        bcc: "bcc@example.com",
+        subject: "Test",
+        purpose: EmailTemplatePurpose.CUSTOM,
+        contents: "Hello, world!",
+      })
+    })
+
+    expect(email.calculatedBcc).toEqual([
+      { address: "bcc@example.com", name: "" },
+    ])
+  })
+
+  it("can change the from address", async () => {
+    const email = await captureEmail(mailserver, async () => {
+      const res = await config.api.emails.sendEmail({
+        email: "to@example.com",
+        from: "from@example.com",
+        subject: "Test",
+        purpose: EmailTemplatePurpose.CUSTOM,
+        contents: "Hello, world!",
+      })
+      expect(res.message).toBeDefined()
+    })
+
+    expect(email.to).toEqual([{ address: "to@example.com", name: "" }])
+    expect(email.from).toEqual([{ address: "from@example.com", name: "" }])
+  })
+
+  it("can send a calendar invite", async () => {
+    const startTime = new Date()
+    const endTime = new Date()
+
+    const email = await captureEmail(mailserver, async () => {
+      await config.api.emails.sendEmail({
+        email: "to@example.com",
+        subject: "Test",
+        purpose: EmailTemplatePurpose.CUSTOM,
+        contents: "Hello, world!",
+        invite: {
+          startTime,
+          endTime,
+          summary: "Summary",
+          location: "Location",
+          url: "http://example.com",
+        },
+      })
+    })
+
+    expect(email.alternatives).toEqual([
+      {
+        charset: "utf-8",
+        contentType: "text/calendar",
+        method: "REQUEST",
+        transferEncoding: "7bit",
+        content: expect.any(String),
+      },
+    ])
+
+    // Reference iCal invite:
+    //   BEGIN:VCALENDAR
+    //   VERSION:2.0
+    //   PRODID:-//sebbo.net//ical-generator//EN
+    //   NAME:Invite
+    //   X-WR-CALNAME:Invite
+    //   BEGIN:VEVENT
+    //   UID:2b5947b7-ec5a-4341-8d70-8d8130183f2a
+    //   SEQUENCE:0
+    //   DTSTAMP:20200101T000000Z
+    //   DTSTART:20200101T000000Z
+    //   DTEND:20200101T000000Z
+    //   SUMMARY:Summary
+    //   LOCATION:Location
+    //   URL;VALUE=URI:http://example.com
+    //   END:VEVENT
+    //   END:VCALENDAR
+    expect(email.alternatives[0].content).toContain("BEGIN:VCALENDAR")
+    expect(email.alternatives[0].content).toContain("BEGIN:VEVENT")
+    expect(email.alternatives[0].content).toContain("UID:")
+    expect(email.alternatives[0].content).toContain("SEQUENCE:0")
+    expect(email.alternatives[0].content).toContain("SUMMARY:Summary")
+    expect(email.alternatives[0].content).toContain("LOCATION:Location")
+    expect(email.alternatives[0].content).toContain(
+      "URL;VALUE=URI:http://example.com"
+    )
+    expect(email.alternatives[0].content).toContain("END:VEVENT")
+    expect(email.alternatives[0].content).toContain("END:VCALENDAR")
+
+    const formatDate = (date: Date) =>
+      date.toISOString().replace(/[-:]/g, "").split(".")[0] + "Z"
+
+    expect(email.alternatives[0].content).toContain(
+      `DTSTAMP:${formatDate(startTime)}`
+    )
+    expect(email.alternatives[0].content).toContain(
+      `DTSTART:${formatDate(startTime)}`
+    )
+    expect(email.alternatives[0].content).toContain(
+      `DTEND:${formatDate(endTime)}`
+    )
   })
 })

packages/worker/src/api/routes/global/tests/realEmail.spec.ts

@@ -1,108 +0,0 @@
-jest.unmock("node-fetch")
-import { TestConfiguration } from "../../../../tests"
-import { objectStore } from "@budibase/backend-core"
-import { helpers } from "@budibase/shared-core"
-
-import tk from "timekeeper"
-import { EmailAttachment, EmailTemplatePurpose } from "@budibase/types"
-
-const fetch = require("node-fetch")
-
-const nodemailer = require("nodemailer")
-
-// for the real email tests give them a long time to try complete/fail
-jest.setTimeout(30000)
-
-describe("/api/global/email", () => {
-  const config = new TestConfiguration()
-
-  beforeAll(async () => {
-    tk.reset()
-    await config.beforeAll()
-  })
-
-  afterAll(async () => {
-    await config.afterAll()
-  })
-
-  async function sendRealEmail(
-    purpose: string,
-    attachments?: EmailAttachment[]
-  ) {
-    let response, text
-    try {
-      await helpers.withTimeout(20000, () => config.saveEtherealSmtpConfig())
-      await helpers.withTimeout(20000, () => config.saveSettingsConfig())
-      let res
-      if (attachments) {
-        res = await config.api.emails
-          .sendEmail(purpose, attachments)
-          .timeout(20000)
-      } else {
-        res = await config.api.emails.sendEmail(purpose).timeout(20000)
-      }
-      // ethereal hiccup, can't test right now
-      if (res.status >= 300) {
-        return
-      }
-      expect(res.body.message).toBeDefined()
-      const testUrl = nodemailer.getTestMessageUrl(res.body)
-      expect(testUrl).toBeDefined()
-      response = await fetch(testUrl)
-      text = await response.text()
-    } catch (err: any) {
-      // ethereal hiccup, can't test right now
-      if (parseInt(err.status) >= 300 || (err && err.errno === "ETIME")) {
-        return
-      } else {
-        throw err
-      }
-    }
-    let toCheckFor
-    switch (purpose) {
-      case EmailTemplatePurpose.WELCOME:
-        toCheckFor = `Thanks for getting started with Budibase's Budibase platform.`
-        break
-      case EmailTemplatePurpose.INVITATION:
-        toCheckFor = `Use the button below to set up your account and get started:`
-        break
-      case EmailTemplatePurpose.PASSWORD_RECOVERY:
-        toCheckFor = `You recently requested to reset your password for your Budibase account in your Budibase platform`
-        break
-    }
-    expect(text).toContain(toCheckFor)
-  }
-
-  it("should be able to send a welcome email", async () => {
-    await sendRealEmail(EmailTemplatePurpose.WELCOME)
-  })
-
-  it("should be able to send a invitation email", async () => {
-    await sendRealEmail(EmailTemplatePurpose.INVITATION)
-  })
-
-  it("should be able to send a password recovery email", async () => {
-    await sendRealEmail(EmailTemplatePurpose.PASSWORD_RECOVERY)
-  })
-
-  it("should be able to send an email with attachments", async () => {
-    let bucket = "testbucket"
-    let filename = "test.txt"
-    await objectStore.upload({
-      bucket,
-      filename,
-      body: Buffer.from("test data"),
-    })
-    let presignedUrl = await objectStore.getPresignedUrl(
-      bucket,
-      filename,
-      60000
-    )
-
-    let attachmentObject = {
-      url: presignedUrl,
-      filename,
-    }
-    await sendRealEmail(EmailTemplatePurpose.WELCOME, [attachmentObject])
-  })
-})

packages/worker/src/tests/TestConfiguration.ts

@@ -32,6 +32,8 @@ import {
   AuthToken,
   SCIMConfig,
   ConfigType,
+  SMTPConfig,
+  SMTPInnerConfig,
 } from "@budibase/types"
 import API from "./api"
 import jwt, { Secret } from "jsonwebtoken"
@@ -348,9 +350,15 @@ class TestConfiguration {
 
   // CONFIGS - SMTP
 
-  async saveSmtpConfig() {
+  async saveSmtpConfig(config?: SMTPInnerConfig) {
     await this.deleteConfig(Config.SMTP)
-    await this._req(structures.configs.smtp(), null, controllers.config.save)
+
+    let smtpConfig: SMTPConfig = structures.configs.smtp()
+    if (config) {
+      smtpConfig = { type: ConfigType.SMTP, config }
+    }
+
+    await this._req(smtpConfig, null, controllers.config.save)
   }
 
   async saveEtherealSmtpConfig() {

packages/worker/src/tests/api/email.ts

@@ -1,19 +1,18 @@
-import { EmailAttachment } from "@budibase/types"
+import { SendEmailRequest, SendEmailResponse } from "@budibase/types"
 import { TestAPI } from "./base"
 
 export class EmailAPI extends TestAPI {
-  sendEmail = (purpose: string, attachments?: EmailAttachment[]) => {
+  sendEmail = async (
-    return this.request
+    req: SendEmailRequest,
+    expectations?: { status?: number }
+  ): Promise<SendEmailResponse> => {
+    const res = await this.request
       .post(`/api/global/email/send`)
-      .send({
+      .send(req)
-        email: "test@example.com",
-        attachments,
-        purpose,
-        tenantId: this.config.getTenantId(),
-        userId: this.config.user!._id!,
-      })
       .set(this.config.defaultHeaders())
       .expect("Content-Type", /json/)
-      .expect(200)
+      .expect(expectations?.status || 200)
+
+    return res.body as SendEmailResponse
   }
 }

packages/worker/src/tests/mocks/email.ts

@@ -1,3 +1,10 @@
+import MailDev from "maildev"
+import { promisify } from "util"
+import TestConfiguration from "../TestConfiguration"
+
+/**
+ * @deprecated please use the `MailDev` email server instead of this mock.
+ */
 export function mock() {
   // mock the email system
   const sendMailMock = jest.fn()
@@ -8,3 +15,170 @@ export function mock() {
   })
   return sendMailMock
 }
+
+export type Mailserver = InstanceType<typeof MailDev>
+export type MailserverConfig = ConstructorParameters<typeof MailDev>[0]
+
+export interface Attachment {
+  checksum: string
+  contentId: string
+  contentType: string
+  fileName: string
+  generatedFileName: string
+  length: number
+  transferEncoding: string
+  transformed: boolean
+}
+
+export interface Address {
+  address: string
+  args?: boolean
+  name?: string
+}
+
+export interface Alternative {
+  contentType: string
+  content: string
+  charset: string
+  method: string
+  transferEncoding: string
+}
+
+export interface Envelope {
+  from: Address
+  to: Address[]
+  host: string
+  remoteAddress: string
+}
+
+export interface Email {
+  attachments: Attachment[]
+  alternatives: Alternative[]
+  calculatedBcc: Address[]
+  cc: Address[]
+  date: string
+  envelope: Envelope
+  from: Address[]
+  headers: Record<string, string>
+  html: string
+  id: string
+  messageId: string
+  priority: string
+  read: boolean
+  size: number
+  sizeHuman: string
+  source: string
+  time: Date
+  to: Address[]
+}
+
+export function getUnusedPort(): Promise<number> {
+  return new Promise((resolve, reject) => {
+    const server = require("net").createServer()
+    server.unref()
+    server.on("error", reject)
+    server.listen(0, () => {
+      const port = server.address().port
+      server.close(() => {
+        resolve(port)
+      })
+    })
+  })
+}
+
+export async function captureEmail(
+  mailserver: Mailserver,
+  f: () => Promise<void>
+): Promise<Email> {
+  const timeoutMs = 5000
+  let timeout: ReturnType<typeof setTimeout> | undefined = undefined
+  const cancel = () => {
+    if (timeout) {
+      clearTimeout(timeout)
+      timeout = undefined
+    }
+  }
+  const timeoutPromise = new Promise<never>((_, reject) => {
+    timeout = setTimeout(() => {
+      reject(new Error("Timed out waiting for email"))
+    }, timeoutMs)
+  })
+  const mailPromise = new Promise<Email>(resolve => {
+    // @ts-expect-error - types are wrong
+    mailserver.once("new", email => {
+      resolve(email as Email)
+      cancel()
+    })
+  })
+  const emailPromise = Promise.race([mailPromise, timeoutPromise])
+  try {
+    await f()
+  } finally {
+    cancel()
+  }
+  return await emailPromise
+}
+
+export async function startMailserver(
+  config: TestConfiguration,
+  opts?: MailserverConfig
+): Promise<Mailserver> {
+  if (!opts) {
+    opts = {}
+  }
+  if (!opts.smtp) {
+    opts.smtp = await getUnusedPort()
+  }
+  const mailserver = new MailDev(opts || {})
+  await new Promise((resolve, reject) => {
+    mailserver.listen(err => {
+      if (err) {
+        return reject(err)
+      }
+      resolve(mailserver)
+    })
+  })
+  await config.saveSmtpConfig({
+    host: "localhost",
+    port: opts.smtp,
+    secure: false,
+    from: "test@example.com",
+  })
+  return mailserver
+}
+
+export function deleteAllEmail(mailserver: Mailserver) {
+  return promisify(mailserver.deleteAllEmail).bind(mailserver)()
+}
+
+export function stopMailserver(mailserver: Mailserver) {
+  return promisify(mailserver.close).bind(mailserver)()
+}
+
+export function getAttachment(
+  mailserver: Mailserver,
+  email: Email,
+  attachment: Attachment
+) {
+  return new Promise<string>(resolve => {
+    // @ts-expect-error - types are wrong
+    mailserver.getEmailAttachment(
+      email.id,
+      attachment.generatedFileName,
+      (err: any, _contentType: string, stream: ReadableStream) => {
+        if (err) {
+          throw err
+        }
+        resolve(new Response(stream).text())
+      }
+    )
+  })
+}
+
+export function getAttachments(mailserver: Mailserver, email: Email) {
+  return Promise.all(
+    email.attachments.map(attachment =>
+      getAttachment(mailserver, email, attachment)
+    )
+  )
+}

packages/worker/src/utilities/email.ts

@@ -4,16 +4,17 @@ import { getTemplateByPurpose, EmailTemplates } from "../constants/templates"
 import { getSettingsTemplateContext } from "./templates"
 import { processString } from "@budibase/string-templates"
 import {
-  User,
   SendEmailOpts,
   SMTPInnerConfig,
   EmailTemplatePurpose,
+  User,
 } from "@budibase/types"
-import { configs, cache, objectStore } from "@budibase/backend-core"
+import { configs, cache, objectStore, HTTPError } from "@budibase/backend-core"
 import ical from "ical-generator"
 import _ from "lodash"
 
-const nodemailer = require("nodemailer")
+import nodemailer from "nodemailer"
+import SMTPTransport from "nodemailer/lib/smtp-transport"
 
 const TEST_MODE = env.ENABLE_EMAIL_TEST_MODE && env.isDev()
 const TYPE = TemplateType.EMAIL
@@ -26,7 +27,7 @@ const FULL_EMAIL_PURPOSES = [
 ]
 
 function createSMTPTransport(config?: SMTPInnerConfig) {
-  let options: any
+  let options: SMTPTransport.Options
   let secure = config?.secure
   // default it if not specified
   if (secure == null) {
@@ -59,22 +60,6 @@ function createSMTPTransport(config?: SMTPInnerConfig) {
   return nodemailer.createTransport(options)
 }
 
-async function getLinkCode(
-  purpose: EmailTemplatePurpose,
-  email: string,
-  user: User,
-  info: any = null
-) {
-  switch (purpose) {
-    case EmailTemplatePurpose.PASSWORD_RECOVERY:
-      return cache.passwordReset.createCode(user._id!, info)
-    case EmailTemplatePurpose.INVITATION:
-      return cache.invite.createCode(email, info)
-    default:
-      return null
-  }
-}
-
 /**
  * Builds an email using handlebars and the templates found in the system (default or otherwise).
  * @param purpose the purpose of the email being built, e.g. invitation, password reset.
@@ -87,8 +72,8 @@ async function getLinkCode(
 async function buildEmail(
   purpose: EmailTemplatePurpose,
   email: string,
-  context: any,
+  context: Record<string, any>,
-  { user, contents }: any = {}
+  { user, contents }: { user?: User; contents?: string } = {}
 ) {
   // this isn't a full email
   if (FULL_EMAIL_PURPOSES.indexOf(purpose) === -1) {
@@ -106,8 +91,8 @@ async function buildEmail(
     throw "Unable to build email, missing base components"
   }
 
-  let name = user ? user.name : undefined
+  let name: string | undefined
-  if (user && !name && user.firstName) {
+  if (user && user.firstName) {
     name = user.lastName ? `${user.firstName} ${user.lastName}` : user.firstName
   }
   context = {
@@ -158,10 +143,21 @@ export async function sendEmail(
   }
   const transport = createSMTPTransport(config)
   // if there is a link code needed this will retrieve it
-  const code = await getLinkCode(purpose, email, opts.user, opts?.info)
+  let code: string | null = null
+  switch (purpose) {
+    case EmailTemplatePurpose.PASSWORD_RECOVERY:
+      if (!opts.user || !opts.user._id) {
+        throw new HTTPError("User must be provided for password recovery.", 400)
+      }
+      code = await cache.passwordReset.createCode(opts.user._id, opts.info)
+      break
+    case EmailTemplatePurpose.INVITATION:
+      code = await cache.invite.createCode(email, opts.info)
+      break
+  }
   let context = await getSettingsTemplateContext(purpose, code)
 
-  let message: any = {
+  let message: Parameters<typeof transport.sendMail>[0] = {
     from: opts?.from || config?.from,
     html: await buildEmail(purpose, email, context, {
       user: opts?.user,