Merge remote-tracking branch 'origin/master' into dean-fixes

packages/backend-core/package.json

@@ -45,7 +45,7 @@
     "passport-oauth2-refresh": "^2.1.0",
     "pino": "8.11.0",
     "pino-http": "8.3.3",
-    "posthog-node": "1.3.0",
+    "posthog-node": "4.0.1",
     "pouchdb": "7.3.0",
     "pouchdb-find": "7.2.2",
     "redlock": "4.2.0",

packages/backend-core/src/environment.ts

@@ -1,5 +1,6 @@
 import { existsSync, readFileSync } from "fs"
 import { ServiceType } from "@budibase/types"
+import { cloneDeep } from "lodash"
 
 function isTest() {
   return isJest()
@@ -144,6 +145,8 @@ const environment = {
   COOKIE_DOMAIN: process.env.COOKIE_DOMAIN,
   PLATFORM_URL: process.env.PLATFORM_URL || "",
   POSTHOG_TOKEN: process.env.POSTHOG_TOKEN,
+  POSTHOG_PERSONAL_TOKEN: process.env.POSTHOG_PERSONAL_TOKEN,
+  POSTHOG_API_HOST: process.env.POSTHOG_API_HOST || "https://us.i.posthog.com",
   ENABLE_ANALYTICS: process.env.ENABLE_ANALYTICS,
   TENANT_FEATURE_FLAGS: process.env.TENANT_FEATURE_FLAGS,
   CLOUDFRONT_CDN: process.env.CLOUDFRONT_CDN,
@@ -208,6 +211,32 @@ const environment = {
   OPENAI_API_KEY: process.env.OPENAI_API_KEY,
 }
 
+export function setEnv(newEnvVars: Partial<typeof environment>): () => void {
+  const oldEnv = cloneDeep(environment)
+
+  let key: keyof typeof newEnvVars
+  for (key in newEnvVars) {
+    environment._set(key, newEnvVars[key])
+  }
+
+  return () => {
+    for (const [key, value] of Object.entries(oldEnv)) {
+      environment._set(key, value)
+    }
+  }
+}
+
+export function withEnv<T>(envVars: Partial<typeof environment>, f: () => T) {
+  const cleanup = setEnv(envVars)
+  const result = f()
+  if (result instanceof Promise) {
+    return result.finally(cleanup)
+  } else {
+    cleanup()
+    return result
+  }
+}
+
 type EnvironmentKey = keyof typeof environment
 export const SECRETS: EnvironmentKey[] = [
   "API_ENCRYPTION_KEY",

packages/backend-core/src/events/processors/posthog/PosthogProcessor.ts

@@ -1,4 +1,4 @@
-import PostHog from "posthog-node"
+import { PostHog } from "posthog-node"
 import { Event, Identity, Group, BaseEvent } from "@budibase/types"
 import { EventProcessor } from "../types"
 import env from "../../../environment"

packages/backend-core/src/events/processors/posthog/tests/PosthogProcessor.spec.ts

@@ -1,9 +1,7 @@
 import { testEnv } from "../../../../../tests/extra"
 import PosthogProcessor from "../PosthogProcessor"
 import { Event, IdentityType, Hosting } from "@budibase/types"
-
-const tk = require("timekeeper")
-
+import tk from "timekeeper"
 import * as cache from "../../../../cache/generic"
 import { CacheKey } from "../../../../cache/generic"
 import * as context from "../../../../context"
@@ -18,6 +16,9 @@ const newIdentity = () => {
 }
 
 describe("PosthogProcessor", () => {
+  let processor: PosthogProcessor
+  let spy: jest.SpyInstance
+
   beforeAll(() => {
     testEnv.singleTenant()
   })
@@ -27,33 +28,29 @@ describe("PosthogProcessor", () => {
     await cache.bustCache(
       `${CacheKey.EVENTS_RATE_LIMIT}:${Event.SERVED_BUILDER}`
     )
+
+    processor = new PosthogProcessor("test")
+    spy = jest.spyOn(processor.posthog, "capture")
   })
 
   describe("processEvent", () => {
     it("processes event", async () => {
-      const processor = new PosthogProcessor("test")
-
       const identity = newIdentity()
       const properties = {}
 
       await processor.processEvent(Event.APP_CREATED, identity, properties)
-
-      expect(processor.posthog.capture).toHaveBeenCalledTimes(1)
+      expect(spy).toHaveBeenCalledTimes(1)
     })
 
     it("honours exclusions", async () => {
-      const processor = new PosthogProcessor("test")
-
       const identity = newIdentity()
       const properties = {}
 
       await processor.processEvent(Event.AUTH_SSO_UPDATED, identity, properties)
-      expect(processor.posthog.capture).toHaveBeenCalledTimes(0)
+      expect(spy).toHaveBeenCalledTimes(0)
     })
 
     it("removes audited information", async () => {
-      const processor = new PosthogProcessor("test")
-
       const identity = newIdentity()
       const properties = {
         email: "test",
@@ -63,7 +60,8 @@ describe("PosthogProcessor", () => {
       }
 
       await processor.processEvent(Event.USER_CREATED, identity, properties)
-      expect(processor.posthog.capture).toHaveBeenCalled()
+      expect(spy).toHaveBeenCalled()
+
       // @ts-ignore
       const call = processor.posthog.capture.mock.calls[0][0]
       expect(call.properties.audited).toBeUndefined()
@@ -72,7 +70,6 @@ describe("PosthogProcessor", () => {
 
     describe("rate limiting", () => {
       it("sends daily event once in same day", async () => {
-        const processor = new PosthogProcessor("test")
         const identity = newIdentity()
         const properties = {}
 
@@ -82,11 +79,10 @@ describe("PosthogProcessor", () => {
         tk.freeze(new Date(2022, 0, 1, 15, 0))
         await processor.processEvent(Event.SERVED_BUILDER, identity, properties)
 
-        expect(processor.posthog.capture).toHaveBeenCalledTimes(1)
+        expect(spy).toHaveBeenCalledTimes(1)
       })
 
       it("sends daily event once per unique day", async () => {
-        const processor = new PosthogProcessor("test")
         const identity = newIdentity()
         const properties = {}
 
@@ -102,11 +98,10 @@ describe("PosthogProcessor", () => {
         tk.freeze(new Date(2022, 0, 3, 6, 0))
         await processor.processEvent(Event.SERVED_BUILDER, identity, properties)
 
-        expect(processor.posthog.capture).toHaveBeenCalledTimes(3)
+        expect(spy).toHaveBeenCalledTimes(3)
       })
 
       it("sends event again after cache expires", async () => {
-        const processor = new PosthogProcessor("test")
         const identity = newIdentity()
         const properties = {}
 
@@ -120,11 +115,10 @@ describe("PosthogProcessor", () => {
         tk.freeze(new Date(2022, 0, 1, 14, 0))
         await processor.processEvent(Event.SERVED_BUILDER, identity, properties)
 
-        expect(processor.posthog.capture).toHaveBeenCalledTimes(2)
+        expect(spy).toHaveBeenCalledTimes(2)
       })
 
       it("sends per app events once per day per app", async () => {
-        const processor = new PosthogProcessor("test")
         const identity = newIdentity()
         const properties = {}
 
@@ -160,10 +154,10 @@ describe("PosthogProcessor", () => {
         }
 
         await runAppEvents("app_1")
-        expect(processor.posthog.capture).toHaveBeenCalledTimes(4)
+        expect(spy).toHaveBeenCalledTimes(4)
 
         await runAppEvents("app_2")
-        expect(processor.posthog.capture).toHaveBeenCalledTimes(8)
+        expect(spy).toHaveBeenCalledTimes(8)
       })
     })
   })

packages/backend-core/src/features/index.ts

@@ -1,108 +1,258 @@
 import env from "../environment"
 import * as context from "../context"
-import { cloneDeep } from "lodash"
+import { PostHog, PostHogOptions } from "posthog-node"
+import { IdentityType, UserCtx } from "@budibase/types"
+import tracer from "dd-trace"
 
-class Flag<T> {
-  static withDefault<T>(value: T) {
-    return new Flag(value)
+let posthog: PostHog | undefined
+export function init(opts?: PostHogOptions) {
+  if (env.POSTHOG_TOKEN && env.POSTHOG_API_HOST) {
+    console.log("initializing posthog client...")
+    posthog = new PostHog(env.POSTHOG_TOKEN, {
+      host: env.POSTHOG_API_HOST,
+      personalApiKey: env.POSTHOG_PERSONAL_TOKEN,
+      ...opts,
+    })
+  } else {
+    console.log("posthog disabled")
+  }
+}
+
+export abstract class Flag<T> {
+  static boolean(defaultValue: boolean): Flag<boolean> {
+    return new BooleanFlag(defaultValue)
   }
 
-  private constructor(public defaultValue: T) {}
+  static string(defaultValue: string): Flag<string> {
+    return new StringFlag(defaultValue)
+  }
+
+  static number(defaultValue: number): Flag<number> {
+    return new NumberFlag(defaultValue)
+  }
+
+  protected constructor(public defaultValue: T) {}
+
+  abstract parse(value: any): T
+}
+
+type UnwrapFlag<F> = F extends Flag<infer U> ? U : never
+
+export type FlagValues<T> = {
+  [K in keyof T]: UnwrapFlag<T[K]>
+}
+
+type KeysOfType<T, U> = {
+  [K in keyof T]: T[K] extends Flag<U> ? K : never
+}[keyof T]
+
+class BooleanFlag extends Flag<boolean> {
+  parse(value: any) {
+    if (typeof value === "string") {
+      return ["true", "t", "1"].includes(value.toLowerCase())
+    }
+
+    if (typeof value === "boolean") {
+      return value
+    }
+
+    throw new Error(`could not parse value "${value}" as boolean`)
+  }
+}
+
+class StringFlag extends Flag<string> {
+  parse(value: any) {
+    if (typeof value === "string") {
+      return value
+    }
+    throw new Error(`could not parse value "${value}" as string`)
+  }
+}
+
+class NumberFlag extends Flag<number> {
+  parse(value: any) {
+    if (typeof value === "number") {
+      return value
+    }
+
+    if (typeof value === "string") {
+      const parsed = parseFloat(value)
+      if (!isNaN(parsed)) {
+        return parsed
+      }
+    }
+
+    throw new Error(`could not parse value "${value}" as number`)
+  }
+}
+
+export class FlagSet<V extends Flag<any>, T extends { [key: string]: V }> {
+  constructor(private readonly flagSchema: T) {}
+
+  defaults(): FlagValues<T> {
+    return Object.keys(this.flagSchema).reduce((acc, key) => {
+      const typedKey = key as keyof T
+      acc[typedKey] = this.flagSchema[key].defaultValue
+      return acc
+    }, {} as FlagValues<T>)
+  }
+
+  isFlagName(name: string | number | symbol): name is keyof T {
+    return this.flagSchema[name as keyof T] !== undefined
+  }
+
+  async get<K extends keyof T>(
+    key: K,
+    ctx?: UserCtx
+  ): Promise<FlagValues<T>[K]> {
+    const flags = await this.fetch(ctx)
+    return flags[key]
+  }
+
+  async isEnabled<K extends KeysOfType<T, boolean>>(
+    key: K,
+    ctx?: UserCtx
+  ): Promise<boolean> {
+    const flags = await this.fetch(ctx)
+    return flags[key]
+  }
+
+  async fetch(ctx?: UserCtx): Promise<FlagValues<T>> {
+    return await tracer.trace("features.fetch", async span => {
+      const tags: Record<string, any> = {}
+      const flagValues = this.defaults()
+      const currentTenantId = context.getTenantId()
+      const specificallySetFalse = new Set<string>()
+
+      const split = (env.TENANT_FEATURE_FLAGS || "")
+        .split(",")
+        .map(x => x.split(":"))
+      for (const [tenantId, ...features] of split) {
+        if (!tenantId || (tenantId !== "*" && tenantId !== currentTenantId)) {
+          continue
+        }
+
+        tags[`readFromEnvironmentVars`] = true
+
+        for (let feature of features) {
+          let value = true
+          if (feature.startsWith("!")) {
+            feature = feature.slice(1)
+            value = false
+            specificallySetFalse.add(feature)
+          }
+
+          if (!this.isFlagName(feature)) {
+            throw new Error(`Feature: ${feature} is not an allowed option`)
+          }
+
+          if (typeof flagValues[feature] !== "boolean") {
+            throw new Error(`Feature: ${feature} is not a boolean`)
+          }
+
+          // @ts-expect-error - TS does not like you writing into a generic type,
+          // but we know that it's okay in this case because it's just an object.
+          flagValues[feature] = value
+          tags[`flags.${feature}.source`] = "environment"
+        }
+      }
+
+      const license = ctx?.user?.license
+      if (license) {
+        tags[`readFromLicense`] = true
+
+        for (const feature of license.features) {
+          if (!this.isFlagName(feature)) {
+            continue
+          }
+
+          if (
+            flagValues[feature] === true ||
+            specificallySetFalse.has(feature)
+          ) {
+            // If the flag is already set to through environment variables, we
+            // don't want to override it back to false here.
+            continue
+          }
+
+          // @ts-expect-error - TS does not like you writing into a generic type,
+          // but we know that it's okay in this case because it's just an object.
+          flagValues[feature] = true
+          tags[`flags.${feature}.source`] = "license"
+        }
+      }
+
+      const identity = context.getIdentity()
+      tags[`identity.type`] = identity?.type
+      tags[`identity.tenantId`] = identity?.tenantId
+      tags[`identity._id`] = identity?._id
+
+      // Until we're confident this performs well, we're only enabling it in QA
+      // and test environments.
+      const usePosthog = env.isTest() || env.isQA()
+      if (usePosthog && posthog && identity?.type === IdentityType.USER) {
+        tags[`readFromPostHog`] = true
+
+        const personProperties: Record<string, string> = {}
+        if (identity.tenantId) {
+          personProperties.tenantId = identity.tenantId
+        }
+
+        const posthogFlags = await posthog.getAllFlagsAndPayloads(
+          identity._id,
+          {
+            personProperties,
+          }
+        )
+        console.log("posthog flags", JSON.stringify(posthogFlags))
+
+        for (const [name, value] of Object.entries(posthogFlags.featureFlags)) {
+          if (!this.isFlagName(name)) {
+            // We don't want an unexpected PostHog flag to break the app, so we
+            // just log it and continue.
+            console.warn(`Unexpected posthog flag "${name}": ${value}`)
+            continue
+          }
+
+          if (flagValues[name] === true || specificallySetFalse.has(name)) {
+            // If the flag is already set to through environment variables, we
+            // don't want to override it back to false here.
+            continue
+          }
+
+          const payload = posthogFlags.featureFlagPayloads?.[name]
+          const flag = this.flagSchema[name]
+          try {
+            // @ts-expect-error - TS does not like you writing into a generic
+            // type, but we know that it's okay in this case because it's just
+            // an object.
+            flagValues[name] = flag.parse(payload || value)
+            tags[`flags.${name}.source`] = "posthog"
+          } catch (err) {
+            // We don't want an invalid PostHog flag to break the app, so we just
+            // log it and continue.
+            console.warn(`Error parsing posthog flag "${name}": ${value}`, err)
+          }
+        }
+      }
+
+      for (const [key, value] of Object.entries(flagValues)) {
+        tags[`flags.${key}.value`] = value
+      }
+      span?.addTags(tags)
+
+      return flagValues
+    })
+  }
 }
 
 // This is the primary source of truth for feature flags. If you want to add a
 // new flag, add it here and use the `fetch` and `get` functions to access it.
 // All of the machinery in this file is to make sure that flags have their
 // default values set correctly and their types flow through the system.
-const FLAGS = {
-  LICENSING: Flag.withDefault(false),
-  GOOGLE_SHEETS: Flag.withDefault(false),
-  USER_GROUPS: Flag.withDefault(false),
-  ONBOARDING_TOUR: Flag.withDefault(false),
-}
-
-const DEFAULTS = Object.keys(FLAGS).reduce((acc, key) => {
-  const typedKey = key as keyof typeof FLAGS
-  // @ts-ignore
-  acc[typedKey] = FLAGS[typedKey].defaultValue
-  return acc
-}, {} as Flags)
-
-type UnwrapFlag<F> = F extends Flag<infer U> ? U : never
-export type Flags = {
-  [K in keyof typeof FLAGS]: UnwrapFlag<(typeof FLAGS)[K]>
-}
-
-// Exported for use in tests, should not be used outside of this file.
-export function defaultFlags(): Flags {
-  return cloneDeep(DEFAULTS)
-}
-
-function isFlagName(name: string): name is keyof Flags {
-  return FLAGS[name as keyof typeof FLAGS] !== undefined
-}
-
-/**
- * Reads the TENANT_FEATURE_FLAGS environment variable and returns a Flags object
- * populated with the flags for the current tenant, filling in the default values
- * if the flag is not set.
- *
- * Check the tests for examples of how TENANT_FEATURE_FLAGS should be formatted.
- *
- * In future we plan to add more ways of setting feature flags, e.g. PostHog, and
- * they will be accessed through this function as well.
- */
-export async function fetch(): Promise<Flags> {
-  const currentTenantId = context.getTenantId()
-  const flags = defaultFlags()
-
-  const split = (env.TENANT_FEATURE_FLAGS || "")
-    .split(",")
-    .map(x => x.split(":"))
-  for (const [tenantId, ...features] of split) {
-    if (!tenantId || (tenantId !== "*" && tenantId !== currentTenantId)) {
-      continue
-    }
-
-    for (let feature of features) {
-      let value = true
-      if (feature.startsWith("!")) {
-        feature = feature.slice(1)
-        value = false
-      }
-
-      if (!isFlagName(feature)) {
-        throw new Error(`Feature: ${feature} is not an allowed option`)
-      }
-
-      if (typeof flags[feature] !== "boolean") {
-        throw new Error(`Feature: ${feature} is not a boolean`)
-      }
-
-      // @ts-ignore
-      flags[feature] = value
-    }
-  }
-
-  return flags
-}
-
-// Gets a single feature flag value. This is a convenience function for
-// `fetch().then(flags => flags[name])`.
-export async function get<K extends keyof Flags>(name: K): Promise<Flags[K]> {
-  const flags = await fetch()
-  return flags[name]
-}
-
-type BooleanFlags = {
-  [K in keyof typeof FLAGS]: (typeof FLAGS)[K] extends Flag<boolean> ? K : never
-}[keyof typeof FLAGS]
-
-// Convenience function for boolean flag values. This makes callsites more
-// readable for boolean flags.
-export async function isEnabled<K extends BooleanFlags>(
-  name: K
-): Promise<boolean> {
-  const flags = await fetch()
-  return flags[name]
-}
+export const flags = new FlagSet({
+  LICENSING: Flag.boolean(false),
+  GOOGLE_SHEETS: Flag.boolean(false),
+  USER_GROUPS: Flag.boolean(false),
+  ONBOARDING_TOUR: Flag.boolean(false),
+})

packages/backend-core/src/features/tests/features.spec.ts

@@ -1,86 +1,230 @@
-import { defaultFlags, fetch, get, Flags } from "../"
+import { IdentityContext, IdentityType, UserCtx } from "@budibase/types"
+import { Flag, FlagSet, FlagValues, init } from "../"
 import { context } from "../.."
-import env from "../../environment"
+import environment, { withEnv } from "../../environment"
+import nodeFetch from "node-fetch"
+import nock from "nock"
 
-async function withFlags<T>(flags: string, f: () => T): Promise<T> {
-  const oldFlags = env.TENANT_FEATURE_FLAGS
-  env._set("TENANT_FEATURE_FLAGS", flags)
-  try {
-    return await f()
-  } finally {
-    env._set("TENANT_FEATURE_FLAGS", oldFlags)
-  }
+const schema = {
+  TEST_BOOLEAN: Flag.boolean(false),
+  TEST_STRING: Flag.string("default value"),
+  TEST_NUMBER: Flag.number(0),
+}
+const flags = new FlagSet(schema)
+
+interface TestCase {
+  it: string
+  identity?: Partial<IdentityContext>
+  environmentFlags?: string
+  posthogFlags?: PostHogFlags
+  licenseFlags?: Array<string>
+  expected?: Partial<FlagValues<typeof schema>>
+  errorMessage?: string | RegExp
+}
+
+interface PostHogFlags {
+  featureFlags?: Record<string, boolean>
+  featureFlagPayloads?: Record<string, string>
+}
+
+function mockPosthogFlags(flags: PostHogFlags) {
+  nock("https://us.i.posthog.com")
+    .post("/decide/?v=3", body => {
+      return body.token === "test" && body.distinct_id === "us_1234"
+    })
+    .reply(200, flags)
+    .persist()
 }
 
 describe("feature flags", () => {
-  interface TestCase {
-    tenant: string
-    flags: string
-    expected: Partial<Flags>
-  }
+  beforeEach(() => {
+    nock.cleanAll()
+  })
 
   it.each<TestCase>([
     {
-      tenant: "tenant1",
-      flags: "tenant1:ONBOARDING_TOUR",
-      expected: { ONBOARDING_TOUR: true },
+      it: "should should find a simple boolean flag in the environment",
+      environmentFlags: "default:TEST_BOOLEAN",
+      expected: { TEST_BOOLEAN: true },
     },
     {
-      tenant: "tenant1",
-      flags: "tenant1:!ONBOARDING_TOUR",
-      expected: { ONBOARDING_TOUR: false },
+      it: "should should find a simple netgative boolean flag in the environment",
+      environmentFlags: "default:!TEST_BOOLEAN",
+      expected: { TEST_BOOLEAN: false },
     },
     {
-      tenant: "tenant1",
-      flags: "*:ONBOARDING_TOUR",
-      expected: { ONBOARDING_TOUR: true },
+      it: "should should match stars in the environment",
+      environmentFlags: "*:TEST_BOOLEAN",
+      expected: { TEST_BOOLEAN: true },
     },
     {
-      tenant: "tenant1",
-      flags: "tenant2:ONBOARDING_TOUR",
-      expected: { ONBOARDING_TOUR: false },
+      it: "should not match a different tenant's flags",
+      environmentFlags: "otherTenant:TEST_BOOLEAN",
+      expected: { TEST_BOOLEAN: false },
     },
     {
-      tenant: "tenant1",
-      flags: "",
-      expected: defaultFlags(),
+      it: "should return the defaults when no flags are set",
+      expected: flags.defaults(),
+    },
+    {
+      it: "should fail when an environment flag is not recognised",
+      environmentFlags: "default:TEST_BOOLEAN,default:FOO",
+      errorMessage: "Feature: FOO is not an allowed option",
+    },
+    {
+      it: "should be able to read boolean flags from PostHog",
+      posthogFlags: {
+        featureFlags: { TEST_BOOLEAN: true },
+      },
+      expected: { TEST_BOOLEAN: true },
+    },
+    {
+      it: "should be able to read string flags from PostHog",
+      posthogFlags: {
+        featureFlags: { TEST_STRING: true },
+        featureFlagPayloads: { TEST_STRING: "test" },
+      },
+      expected: { TEST_STRING: "test" },
+    },
+    {
+      it: "should be able to read numeric flags from PostHog",
+      posthogFlags: {
+        featureFlags: { TEST_NUMBER: true },
+        featureFlagPayloads: { TEST_NUMBER: "123" },
+      },
+      expected: { TEST_NUMBER: 123 },
+    },
+    {
+      it: "should not be able to override a negative environment flag from PostHog",
+      environmentFlags: "default:!TEST_BOOLEAN",
+      posthogFlags: {
+        featureFlags: { TEST_BOOLEAN: true },
+      },
+      expected: { TEST_BOOLEAN: false },
+    },
+    {
+      it: "should not be able to override a positive environment flag from PostHog",
+      environmentFlags: "default:TEST_BOOLEAN",
+      posthogFlags: {
+        featureFlags: {
+          TEST_BOOLEAN: false,
+        },
+      },
+      expected: { TEST_BOOLEAN: true },
+    },
+    {
+      it: "should be able to set boolean flags through the license",
+      licenseFlags: ["TEST_BOOLEAN"],
+      expected: { TEST_BOOLEAN: true },
+    },
+    {
+      it: "should not be able to override a negative environment flag from license",
+      environmentFlags: "default:!TEST_BOOLEAN",
+      licenseFlags: ["TEST_BOOLEAN"],
+      expected: { TEST_BOOLEAN: false },
+    },
+    {
+      it: "should not error on unrecognised PostHog flag",
+      posthogFlags: {
+        featureFlags: { UNDEFINED: true },
+      },
+      expected: flags.defaults(),
+    },
+    {
+      it: "should not error on unrecognised license flag",
+      licenseFlags: ["UNDEFINED"],
+      expected: flags.defaults(),
     },
   ])(
-    'should find flags $expected for $tenant with string "$flags"',
-    ({ tenant, flags, expected }) =>
-      context.doInTenant(tenant, () =>
-        withFlags(flags, async () => {
-          const flags = await fetch()
-          expect(flags).toMatchObject(expected)
+    "$it",
+    async ({
+      identity,
+      environmentFlags,
+      posthogFlags,
+      licenseFlags,
+      expected,
+      errorMessage,
+    }) => {
+      const env: Partial<typeof environment> = {
+        TENANT_FEATURE_FLAGS: environmentFlags,
+      }
 
-          for (const [key, expectedValue] of Object.entries(expected)) {
-            const value = await get(key as keyof Flags)
-            expect(value).toBe(expectedValue)
+      if (posthogFlags) {
+        mockPosthogFlags(posthogFlags)
+        env.POSTHOG_TOKEN = "test"
+        env.POSTHOG_API_HOST = "https://us.i.posthog.com"
+        env.POSTHOG_PERSONAL_TOKEN = "test"
+      }
+
+      const ctx = { user: { license: { features: licenseFlags || [] } } }
+
+      await withEnv(env, async () => {
+        // We need to pass in node-fetch here otherwise nock won't get used
+        // because posthog-node uses axios under the hood.
+        init({
+          fetch: (url, opts) => {
+            return nodeFetch(url, opts)
+          },
+        })
+
+        const fullIdentity: IdentityContext = {
+          _id: "us_1234",
+          tenantId: "default",
+          type: IdentityType.USER,
+          email: "test@example.com",
+          firstName: "Test",
+          lastName: "User",
+          ...identity,
+        }
+
+        await context.doInIdentityContext(fullIdentity, async () => {
+          if (errorMessage) {
+            await expect(flags.fetch(ctx as UserCtx)).rejects.toThrow(
+              errorMessage
+            )
+          } else if (expected) {
+            const values = await flags.fetch(ctx as UserCtx)
+            expect(values).toMatchObject(expected)
+
+            for (const [key, expectedValue] of Object.entries(expected)) {
+              const value = await flags.get(
+                key as keyof typeof schema,
+                ctx as UserCtx
+              )
+              expect(value).toBe(expectedValue)
+            }
+          } else {
+            throw new Error("No expected value")
           }
         })
-      )
-  )
-
-  interface FailedTestCase {
-    tenant: string
-    flags: string
-    expected: string | RegExp
-  }
-
-  it.each<FailedTestCase>([
-    {
-      tenant: "tenant1",
-      flags: "tenant1:ONBOARDING_TOUR,tenant1:FOO",
-      expected: "Feature: FOO is not an allowed option",
-    },
-  ])(
-    "should fail with message \"$expected\" for $tenant with string '$flags'",
-    async ({ tenant, flags, expected }) => {
-      context.doInTenant(tenant, () =>
-        withFlags(flags, async () => {
-          await expect(fetch()).rejects.toThrow(expected)
-        })
-      )
+      })
     }
   )
+
+  it("should not error if PostHog is down", async () => {
+    const identity: IdentityContext = {
+      _id: "us_1234",
+      tenantId: "default",
+      type: IdentityType.USER,
+      email: "test@example.com",
+      firstName: "Test",
+      lastName: "User",
+    }
+
+    nock("https://us.i.posthog.com")
+      .post("/decide/?v=3", body => {
+        return body.token === "test" && body.distinct_id === "us_1234"
+      })
+      .reply(503)
+      .persist()
+
+    await withEnv(
+      { POSTHOG_TOKEN: "test", POSTHOG_API_HOST: "https://us.i.posthog.com" },
+      async () => {
+        await context.doInIdentityContext(identity, async () => {
+          await flags.fetch()
+        })
+      }
+    )
+  })
 })

packages/backend-core/src/index.ts

@@ -27,7 +27,7 @@ export * as locks from "./redis/redlockImpl"
 export * as utils from "./utils"
 export * as errors from "./errors"
 export * as timers from "./timers"
-export { default as env } from "./environment"
+export { default as env, withEnv, setEnv } from "./environment"
 export * as blacklist from "./blacklist"
 export * as docUpdates from "./docUpdates"
 export * from "./utils/Duration"

packages/backend-core/src/middleware/passport/sso/tests/oidc.spec.ts

@@ -1,4 +1,4 @@
-import { generator, mocks, structures } from "../../../../../tests"
+import { generator, structures } from "../../../../../tests"
 import {
   JwtClaims,
   OIDCInnerConfig,
@@ -7,6 +7,7 @@ import {
 } from "@budibase/types"
 import * as _sso from "../sso"
 import * as oidc from "../oidc"
+import nock from "nock"
 
 jest.mock("@techpass/passport-openidconnect")
 const mockStrategy = require("@techpass/passport-openidconnect").Strategy
@@ -22,16 +23,9 @@ describe("oidc", () => {
   const oidcConfig: OIDCInnerConfig = structures.sso.oidcConfig()
   const wellKnownConfig = structures.sso.oidcWellKnownConfig()
 
-  function mockRetrieveWellKnownConfig() {
-    // mock the request to retrieve the oidc configuration
-    mocks.fetch.mockReturnValue({
-      ok: true,
-      json: () => wellKnownConfig,
-    })
-  }
-
   beforeEach(() => {
-    mockRetrieveWellKnownConfig()
+    nock.cleanAll()
+    nock(oidcConfig.configUrl).get("/").reply(200, wellKnownConfig)
   })
 
   describe("strategyFactory", () => {
@@ -42,8 +36,6 @@ describe("oidc", () => {
       )
       await oidc.strategyFactory(strategyConfiguration, mockSaveUser)
 
-      expect(mocks.fetch).toHaveBeenCalledWith(oidcConfig.configUrl)
-
       const expectedOptions = {
         issuer: wellKnownConfig.issuer,
         authorizationURL: wellKnownConfig.authorization_endpoint,

packages/backend-core/src/middleware/passport/sso/tests/sso.spec.ts

@@ -1,10 +1,11 @@
-import { structures, mocks } from "../../../../../tests"
+import { structures } from "../../../../../tests"
 import { testEnv } from "../../../../../tests/extra"
 import { SSOAuthDetails, User } from "@budibase/types"
 
 import { HTTPError } from "../../../../errors"
 import * as sso from "../sso"
 import * as context from "../../../../context"
+import nock from "nock"
 
 const mockDone = jest.fn()
 const mockSaveUser = jest.fn()
@@ -23,6 +24,7 @@ describe("sso", () => {
     beforeEach(() => {
       jest.clearAllMocks()
       testEnv.singleTenant()
+      nock.cleanAll()
     })
 
     describe("validation", () => {
@@ -51,15 +53,6 @@ describe("sso", () => {
       })
     })
 
-    function mockGetProfilePicture() {
-      mocks.fetch.mockReturnValueOnce(
-        Promise.resolve({
-          status: 200,
-          headers: { get: () => "image/" },
-        })
-      )
-    }
-
     describe("when the user doesn't exist", () => {
       let user: User
       let details: SSOAuthDetails
@@ -68,7 +61,10 @@ describe("sso", () => {
         users.getById.mockImplementationOnce(() => {
           throw new HTTPError("", 404)
         })
-        mockGetProfilePicture()
+
+        nock("http://example.com").get("/").reply(200, undefined, {
+          "Content-Type": "image/png",
+        })
 
         user = structures.users.user()
         delete user._rev
@@ -131,7 +127,9 @@ describe("sso", () => {
         existingUser = structures.users.user()
         existingUser._id = structures.uuid()
         details = structures.sso.authDetails(existingUser)
-        mockGetProfilePicture()
+        nock("http://example.com").get("/").reply(200, undefined, {
+          "Content-Type": "image/png",
+        })
       })
 
       describe("exists by email", () => {

packages/backend-core/src/plugin/tests/validation.spec.ts

@@ -1,12 +1,129 @@
 import { validate } from "../utils"
 import fetch from "node-fetch"
 import { PluginType } from "@budibase/types"
+import nock from "nock"
 
-const repoUrl =
-  "https://raw.githubusercontent.com/Budibase/budibase-skeleton/master"
-const automationLink = `${repoUrl}/automation/schema.json.hbs`
-const componentLink = `${repoUrl}/component/schema.json.hbs`
-const datasourceLink = `${repoUrl}/datasource/schema.json.hbs`
+const automationLink = `http://example.com/automation/schema.json`
+const componentLink = `http://example.com/component/schema.json`
+const datasourceLink = `http://example.com/datasource/schema.json`
+
+function mockDatasourceSchema() {
+  nock("http://example.com")
+    .get("/datasource/schema.json")
+    .reply(200, {
+      type: "datasource",
+      metadata: {},
+      schema: {
+        docs: "https://docs.budibase.com",
+        friendlyName: "Basic HTTP",
+        type: "API",
+        description: "Performs a basic HTTP calls to a URL",
+        datasource: {
+          url: {
+            type: "string",
+            required: true,
+          },
+          cookie: {
+            type: "string",
+            required: false,
+          },
+        },
+        query: {
+          create: {
+            type: "json",
+          },
+          read: {
+            type: "fields",
+            fields: {
+              queryString: {
+                display: "Query string",
+                type: "string",
+                required: false,
+              },
+            },
+          },
+          update: {
+            type: "json",
+          },
+          delete: {
+            type: "fields",
+            fields: {
+              id: {
+                type: "string",
+                required: true,
+              },
+            },
+          },
+        },
+      },
+    })
+}
+
+function mockAutomationSchema() {
+  nock("http://example.com")
+    .get("/automation/schema.json")
+    .reply(200, {
+      type: "automation",
+      metadata: {},
+      schema: {
+        name: "{{ name }}",
+        tagline: "{{ description }}",
+        icon: "Actions",
+        description: "{{ description }}",
+        type: "action",
+        stepId: "{{ name }}",
+        inputs: {
+          text: "",
+        },
+        schema: {
+          inputs: {
+            properties: {
+              text: {
+                type: "string",
+                title: "Log",
+              },
+            },
+            required: ["text"],
+          },
+          outputs: {
+            properties: {
+              success: {
+                type: "boolean",
+                description: "Whether the action was successful",
+              },
+              message: {
+                type: "string",
+                description: "What was output",
+              },
+            },
+            required: ["success", "message"],
+          },
+        },
+      },
+    })
+}
+
+function mockComponentSchema() {
+  nock("http://example.com")
+    .get("/component/schema.json")
+    .reply(200, {
+      type: "component",
+      metadata: {},
+      schema: {
+        name: "{{ name }}",
+        friendlyName: "{{ name }}",
+        description: "{{ description }}",
+        icon: "Text",
+        settings: [
+          {
+            type: "text",
+            key: "text",
+            label: "Text",
+          },
+        ],
+      },
+    })
+}
 
 async function getSchema(link: string) {
   const response = await fetch(link)
@@ -31,53 +148,62 @@ async function runTest(opts: { link?: string; schema?: any }) {
   return error
 }
 
-describe("it should be able to validate an automation schema", () => {
-  it("should return automation skeleton schema is valid", async () => {
-    const error = await runTest({ link: automationLink })
-    expect(error).toBeUndefined()
+describe("plugin validation", () => {
+  beforeEach(() => {
+    nock.cleanAll()
+    mockAutomationSchema()
+    mockComponentSchema()
+    mockDatasourceSchema()
   })
 
-  it("should fail given invalid automation schema", async () => {
-    const error = await runTest({
-      schema: {
-        type: PluginType.AUTOMATION,
-        schema: {},
-      },
+  describe("it should be able to validate an automation schema", () => {
+    it("should return automation skeleton schema is valid", async () => {
+      const error = await runTest({ link: automationLink })
+      expect(error).toBeUndefined()
+    })
+
+    it("should fail given invalid automation schema", async () => {
+      const error = await runTest({
+        schema: {
+          type: PluginType.AUTOMATION,
+          schema: {},
+        },
+      })
+      expect(error).toBeDefined()
+    })
+  })
+
+  describe("it should be able to validate a component schema", () => {
+    it("should return component skeleton schema is valid", async () => {
+      const error = await runTest({ link: componentLink })
+      expect(error).toBeUndefined()
+    })
+
+    it("should fail given invalid component schema", async () => {
+      const error = await runTest({
+        schema: {
+          type: PluginType.COMPONENT,
+          schema: {},
+        },
+      })
+      expect(error).toBeDefined()
+    })
+  })
+
+  describe("it should be able to validate a datasource schema", () => {
+    it("should return datasource skeleton schema is valid", async () => {
+      const error = await runTest({ link: datasourceLink })
+      expect(error).toBeUndefined()
+    })
+
+    it("should fail given invalid datasource schema", async () => {
+      const error = await runTest({
+        schema: {
+          type: PluginType.DATASOURCE,
+          schema: {},
+        },
+      })
+      expect(error).toBeDefined()
     })
-    expect(error).toBeDefined()
-  })
-})
-
-describe("it should be able to validate a component schema", () => {
-  it("should return component skeleton schema is valid", async () => {
-    const error = await runTest({ link: componentLink })
-    expect(error).toBeUndefined()
-  })
-
-  it("should fail given invalid component schema", async () => {
-    const error = await runTest({
-      schema: {
-        type: PluginType.COMPONENT,
-        schema: {},
-      },
-    })
-    expect(error).toBeDefined()
-  })
-})
-
-describe("it should be able to validate a datasource schema", () => {
-  it("should return datasource skeleton schema is valid", async () => {
-    const error = await runTest({ link: datasourceLink })
-    expect(error).toBeUndefined()
-  })
-
-  it("should fail given invalid datasource schema", async () => {
-    const error = await runTest({
-      schema: {
-        type: PluginType.DATASOURCE,
-        schema: {},
-      },
-    })
-    expect(error).toBeDefined()
   })
 })

packages/backend-core/src/redis/redis.ts

@@ -111,6 +111,10 @@ function init(selectDb = DEFAULT_SELECT_DB) {
   CLIENTS[selectDb] = client
 }
 
+export function closeAll() {
+  Object.values(CLIENTS).forEach(client => client.disconnect())
+}
+
 function waitForConnection(selectDb: number = DEFAULT_SELECT_DB) {
   return new Promise(resolve => {
     if (pickClient(selectDb) == null) {

packages/backend-core/src/redis/tests/redis.spec.ts

@@ -1,6 +1,6 @@
 import { GenericContainer, StartedTestContainer } from "testcontainers"
 import { generator, structures } from "../../../tests"
-import RedisWrapper from "../redis"
+import RedisWrapper, { closeAll } from "../redis"
 import { env } from "../.."
 import { randomUUID } from "crypto"
 
@@ -23,7 +23,10 @@ describe("redis", () => {
     env._set("REDIS_PASSWORD", 0)
   })
 
-  afterAll(() => container?.stop())
+  afterAll(() => {
+    container?.stop()
+    closeAll()
+  })
 
   beforeEach(async () => {
     redis = new RedisWrapper(structures.db.id())

packages/backend-core/tests/core/utilities/mocks/fetch.ts

@@ -1,17 +0,0 @@
-const mockFetch = jest.fn((url: any, opts: any) => {
-  const fetch = jest.requireActual("node-fetch")
-  const env = jest.requireActual("../../../../src/environment").default
-  if (url.includes(env.COUCH_DB_URL) || url.includes("raw.github")) {
-    return fetch(url, opts)
-  }
-  return undefined
-})
-
-const enable = () => {
-  jest.mock("node-fetch", () => mockFetch)
-}
-
-export default {
-  ...mockFetch,
-  enable,
-}

packages/backend-core/tests/core/utilities/mocks/index.ts

@@ -5,7 +5,5 @@ export const accounts = jest.mocked(_accounts)
 
 export * as date from "./date"
 export * as licenses from "./licenses"
-export { default as fetch } from "./fetch"
 export * from "./alerts"
 import "./events"
-import "./posthog"

packages/backend-core/tests/core/utilities/mocks/posthog.ts

@@ -1,7 +0,0 @@
-jest.mock("posthog-node", () => {
-  return jest.fn().mockImplementation(() => {
-    return {
-      capture: jest.fn(),
-    }
-  })
-})

packages/backend-core/tests/jestSetup.ts

@@ -2,14 +2,21 @@ import "./core/logging"
 import env from "../src/environment"
 import { cleanup } from "../src/timers"
 import { mocks, testContainerUtils } from "./core/utilities"
-
-// must explicitly enable fetch mock
-mocks.fetch.enable()
+import nock from "nock"
 
 // mock all dates to 2020-01-01T00:00:00.000Z
 // use tk.reset() to use real dates in individual tests
 import tk from "timekeeper"
 
+nock.disableNetConnect()
+nock.enableNetConnect(host => {
+  return (
+    host.includes("localhost") ||
+    host.includes("127.0.0.1") ||
+    host.includes("::1")
+  )
+})
+
 tk.freeze(mocks.date.MOCK_DATE)
 
 if (!process.env.DEBUG) {

packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte

@@ -19,6 +19,8 @@
     helpers,
     PROTECTED_INTERNAL_COLUMNS,
     PROTECTED_EXTERNAL_COLUMNS,
+    canBeDisplayColumn,
+    canHaveDefaultColumn,
   } from "@budibase/shared-core"
   import { createEventDispatcher, getContext, onMount } from "svelte"
   import { cloneDeep } from "lodash/fp"
@@ -44,6 +46,7 @@
   import { RowUtils } from "@budibase/frontend-core"
   import ServerBindingPanel from "components/common/bindings/ServerBindingPanel.svelte"
   import OptionsEditor from "./OptionsEditor.svelte"
+  import { isEnabled, TENANT_FEATURE_FLAGS } from "helpers/featureFlags"
 
   const AUTO_TYPE = FieldType.AUTO
   const FORMULA_TYPE = FieldType.FORMULA
@@ -133,7 +136,9 @@
   }
   $: initialiseField(field, savingColumn)
   $: checkConstraints(editableColumn)
-  $: required = !!editableColumn?.constraints?.presence || primaryDisplay
+  $: required = hasDefault
+    ? false
+    : !!editableColumn?.constraints?.presence || primaryDisplay
   $: uneditable =
     $tables.selected?._id === TableNames.USERS &&
     UNEDITABLE_USER_FIELDS.includes(editableColumn.name)
@@ -161,15 +166,17 @@
     : availableAutoColumns
   // used to select what different options can be displayed for column type
   $: canBeDisplay =
-    editableColumn?.type !== LINK_TYPE &&
-    editableColumn?.type !== AUTO_TYPE &&
-    editableColumn?.type !== JSON_TYPE &&
-    !editableColumn.autocolumn
+    canBeDisplayColumn(editableColumn.type) && !editableColumn.autocolumn
+  $: canHaveDefault =
+    isEnabled(TENANT_FEATURE_FLAGS.DEFAULT_VALUES) &&
+    canHaveDefaultColumn(editableColumn.type)
   $: canBeRequired =
     editableColumn?.type !== LINK_TYPE &&
     !uneditable &&
     editableColumn?.type !== AUTO_TYPE &&
     !editableColumn.autocolumn
+  $: hasDefault =
+    editableColumn?.default != null && editableColumn?.default !== ""
   $: externalTable = table.sourceType === DB_TYPE_EXTERNAL
   // in the case of internal tables the sourceId will just be undefined
   $: tableOptions = $tables.list.filter(
@@ -349,12 +356,15 @@
     }
   }
 
-  function onChangeRequired(e) {
-    const req = e.detail
+  function setRequired(req) {
     editableColumn.constraints.presence = req ? { allowEmpty: false } : false
     required = req
   }
 
+  function onChangeRequired(e) {
+    setRequired(e.detail)
+  }
+
   function openJsonSchemaEditor() {
     jsonSchemaModal.show()
   }
@@ -748,13 +758,37 @@
         <Toggle
           value={required}
           on:change={onChangeRequired}
-          disabled={primaryDisplay}
+          disabled={primaryDisplay || hasDefault}
           thin
           text="Required"
         />
       {/if}
     </div>
   {/if}
+
+  {#if canHaveDefault}
+    <div>
+      <ModalBindableInput
+        panel={ServerBindingPanel}
+        title="Default"
+        label="Default"
+        value={editableColumn.default}
+        on:change={e => {
+          editableColumn = {
+            ...editableColumn,
+            default: e.detail,
+          }
+
+          if (e.detail) {
+            setRequired(false)
+          }
+        }}
+        bindings={getBindings({ table })}
+        allowJS
+        context={rowGoldenSample}
+      />
+    </div>
+  {/if}
 </Layout>
 
 <div class="action-buttons">

packages/builder/src/helpers/featureFlags.js

@@ -6,6 +6,7 @@ export const TENANT_FEATURE_FLAGS = {
   USER_GROUPS: "USER_GROUPS",
   ONBOARDING_TOUR: "ONBOARDING_TOUR",
   GOOGLE_SHEETS: "GOOGLE_SHEETS",
+  DEFAULT_VALUES: "DEFAULT_VALUES",
 }
 
 export const isEnabled = featureFlag => {

packages/builder/src/pages/builder/portal/users/users/_components/AddUserModal.svelte

@@ -16,7 +16,7 @@
 
   export let showOnboardingTypeModal
 
-  const password = Math.random().toString(36).substring(2, 22)
+  const password = generatePassword(12)
   let disabled
   let userGroups = []
 
@@ -44,7 +44,7 @@
       {
         email: "",
         role: "appUser",
-        password: Math.random().toString(36).substring(2, 22),
+        password: generatePassword(12),
         forceResetPassword: true,
         error: null,
       },
@@ -69,6 +69,14 @@
     return userData[index].error == null
   }
 
+  function generatePassword(length) {
+    const array = new Uint8Array(length)
+    window.crypto.getRandomValues(array)
+    return Array.from(array, byte => byte.toString(36).padStart(2, "0"))
+      .join("")
+      .slice(0, length)
+  }
+
   const onConfirm = () => {
     let valid = true
     userData.forEach((input, index) => {

packages/builder/src/pages/builder/portal/users/users/index.svelte

@@ -216,7 +216,7 @@
       const newUser = {
         email: email,
         role: usersRole,
-        password: Math.random().toString(36).substring(2, 22),
+        password: generatePassword(12),
         forceResetPassword: true,
       }
 
@@ -288,6 +288,14 @@
     }
   }
 
+  const generatePassword = length => {
+    const array = new Uint8Array(length)
+    window.crypto.getRandomValues(array)
+    return Array.from(array, byte => byte.toString(36).padStart(2, "0"))
+      .join("")
+      .slice(0, length)
+  }
+
   onMount(async () => {
     try {
       await groups.actions.init()

packages/pro

@@ -1 +1 @@
-Subproject commit 62ef0e2d6e83522b6732fb3c61338de303f06ff0
+Subproject commit 94747fd5bb67c218244bb60b9540f3a6f1c3f6f1

packages/server/src/api/routes/tests/apikeys.spec.ts

@@ -1,11 +1,12 @@
-const setup = require("./utilities")
-const { checkBuilderEndpoint } = require("./utilities/TestFunctions")
+import { withEnv } from "../../../environment"
+import { getRequest, getConfig, afterAll as _afterAll } from "./utilities"
+import { checkBuilderEndpoint } from "./utilities/TestFunctions"
 
 describe("/api/keys", () => {
-  let request = setup.getRequest()
-  let config = setup.getConfig()
+  let request = getRequest()
+  let config = getConfig()
 
-  afterAll(setup.afterAll)
+  afterAll(_afterAll)
 
   beforeAll(async () => {
     await config.init()
@@ -13,7 +14,7 @@ describe("/api/keys", () => {
 
   describe("fetch", () => {
     it("should allow fetching", async () => {
-      await config.withEnv({ SELF_HOSTED: "true" }, async () => {
+      await withEnv({ SELF_HOSTED: "true" }, async () => {
         const res = await request
           .get(`/api/keys`)
           .set(config.defaultHeaders())
@@ -34,7 +35,7 @@ describe("/api/keys", () => {
 
   describe("update", () => {
     it("should allow updating a value", async () => {
-      await config.withEnv({ SELF_HOSTED: "true" }, async () => {
+      await withEnv({ SELF_HOSTED: "true" }, async () => {
         const res = await request
           .put(`/api/keys/TEST`)
           .send({

packages/server/src/api/routes/tests/application.spec.ts

@@ -14,7 +14,12 @@ jest.mock("../../../utilities/redis", () => ({
 import { checkBuilderEndpoint } from "./utilities/TestFunctions"
 import * as setup from "./utilities"
 import { AppStatus } from "../../../db/utils"
-import { events, utils, context } from "@budibase/backend-core"
+import {
+  events,
+  utils,
+  context,
+  withEnv as withCoreEnv,
+} from "@budibase/backend-core"
 import env from "../../../environment"
 import { type App } from "@budibase/types"
 import tk from "timekeeper"
@@ -353,7 +358,7 @@ describe("/applications", () => {
         .delete(`/api/global/roles/${prodAppId}`)
         .reply(200, {})
 
-      await config.withCoreEnv({ SQS_SEARCH_ENABLE: "true" }, async () => {
+      await withCoreEnv({ SQS_SEARCH_ENABLE: "true" }, async () => {
         await config.api.application.delete(app.appId)
       })
     })

packages/server/src/api/routes/tests/attachment.spec.ts

@@ -1,3 +1,4 @@
+import { withEnv } from "../../../environment"
 import * as setup from "./utilities"
 import { APIError } from "@budibase/types"
 
@@ -28,7 +29,7 @@ describe("/api/applications/:appId/sync", () => {
     })
 
     it("should reject an upload with a malicious file extension", async () => {
-      await config.withEnv({ SELF_HOSTED: undefined }, async () => {
+      await withEnv({ SELF_HOSTED: undefined }, async () => {
         let resp = (await config.api.attachment.process(
           "ohno.exe",
           Buffer.from([0]),
@@ -39,7 +40,7 @@ describe("/api/applications/:appId/sync", () => {
     })
 
     it("should reject an upload with a malicious uppercase file extension", async () => {
-      await config.withEnv({ SELF_HOSTED: undefined }, async () => {
+      await withEnv({ SELF_HOSTED: undefined }, async () => {
         let resp = (await config.api.attachment.process(
           "OHNO.EXE",
           Buffer.from([0]),

packages/server/src/api/routes/tests/row.spec.ts

@@ -40,6 +40,7 @@ import _, { merge } from "lodash"
 import * as uuid from "uuid"
 import { Knex } from "knex"
 import { InternalTables } from "../../../db/utils"
+import { withEnv } from "../../../environment"
 
 const timestamp = new Date("2023-01-26T11:48:57.597Z").toISOString()
 tk.freeze(timestamp)
@@ -1688,7 +1689,7 @@ describe.each([
         }
         const row = await config.api.row.save(testTable._id!, draftRow)
 
-        await config.withEnv({ SELF_HOSTED: "true" }, async () => {
+        await withEnv({ SELF_HOSTED: "true" }, async () => {
           return context.doInAppContext(config.getAppId(), async () => {
             const enriched: Row[] = await outputProcessing(table, [row])
             const [targetRow] = enriched
@@ -2456,7 +2457,7 @@ describe.each([
 
   describe("Formula JS protection", () => {
     it("should time out JS execution if a single cell takes too long", async () => {
-      await config.withEnv({ JS_PER_INVOCATION_TIMEOUT_MS: 40 }, async () => {
+      await withEnv({ JS_PER_INVOCATION_TIMEOUT_MS: 40 }, async () => {
         const js = Buffer.from(
           `
               let i = 0;
@@ -2494,7 +2495,7 @@ describe.each([
     })
 
     it("should time out JS execution if a multiple cells take too long", async () => {
-      await config.withEnv(
+      await withEnv(
         {
           JS_PER_INVOCATION_TIMEOUT_MS: 40,
           JS_PER_REQUEST_TIMEOUT_MS: 80,

packages/server/src/api/routes/tests/search.spec.ts

@@ -11,6 +11,8 @@ import {
   MIN_VALID_DATE,
   SQLITE_DESIGN_DOC_ID,
   utils,
+  withEnv as withCoreEnv,
+  setEnv as setCoreEnv,
 } from "@budibase/backend-core"
 
 import * as setup from "./utilities"
@@ -65,9 +67,9 @@ describe.each([
   let rows: Row[]
 
   beforeAll(async () => {
-    await config.withCoreEnv({ SQS_SEARCH_ENABLE: "true" }, () => config.init())
+    await withCoreEnv({ SQS_SEARCH_ENABLE: "true" }, () => config.init())
     if (isSqs) {
-      envCleanup = config.setCoreEnv({
+      envCleanup = setCoreEnv({
         SQS_SEARCH_ENABLE: "true",
         SQS_SEARCH_ENABLE_TENANTS: [config.getTenantId()],
       })
@@ -2734,7 +2736,7 @@ describe.each([
       })
 
       it("can still page when the hard limit is hit", async () => {
-        await config.withCoreEnv(
+        await withCoreEnv(
           {
             SQL_MAX_ROWS: "6",
           },

packages/server/src/api/routes/tests/static.spec.ts

@@ -8,23 +8,24 @@ jest.mock("aws-sdk", () => ({
   })),
 }))
 
-const setup = require("./utilities")
-const { constants } = require("@budibase/backend-core")
+import { Datasource, SourceName } from "@budibase/types"
+import { setEnv } from "../../../environment"
+import { getRequest, getConfig, afterAll as _afterAll } from "./utilities"
+import { constants } from "@budibase/backend-core"
 
 describe("/static", () => {
-  let request = setup.getRequest()
-  let config = setup.getConfig()
-  let app
-  let cleanupEnv
+  let request = getRequest()
+  let config = getConfig()
+  let cleanupEnv: () => void
 
   afterAll(() => {
-    setup.afterAll()
+    _afterAll()
     cleanupEnv()
   })
 
   beforeAll(async () => {
-    cleanupEnv = config.setEnv({ SELF_HOSTED: "true" })
-    app = await config.init()
+    cleanupEnv = setEnv({ SELF_HOSTED: "true" })
+    await config.init()
   })
 
   describe("/app", () => {
@@ -49,7 +50,7 @@ describe("/static", () => {
       delete headers[constants.Header.APP_ID]
 
       const res = await request
-        .get(`/app${config.prodApp.url}`)
+        .get(`/app${config.getProdApp().url}`)
         .set(headers)
         .expect(200)
 
@@ -68,14 +69,14 @@ describe("/static", () => {
 
   describe("/attachments", () => {
     describe("generateSignedUrls", () => {
-      let datasource
+      let datasource: Datasource
 
       beforeEach(async () => {
         datasource = await config.createDatasource({
           datasource: {
             type: "datasource",
             name: "Test",
-            source: "S3",
+            source: SourceName.S3,
             config: {},
           },
         })

packages/server/src/api/routes/tests/templates.spec.ts

@@ -2,6 +2,7 @@ import * as setup from "./utilities"
 import path from "path"
 import nock from "nock"
 import { generator } from "@budibase/backend-core/tests"
+import { withEnv as withCoreEnv } from "@budibase/backend-core"
 
 interface App {
   background: string
@@ -89,7 +90,7 @@ describe("/templates", () => {
           SQS_SEARCH_ENABLE_TENANTS: [config.getTenantId()],
         }
 
-        await config.withCoreEnv(env, async () => {
+        await withCoreEnv(env, async () => {
           const name = generator.guid().replaceAll("-", "")
           const url = `/${name}`
 

packages/server/src/api/routes/tests/viewV2.spec.ts

@@ -24,7 +24,12 @@ import { generator, mocks } from "@budibase/backend-core/tests"
 import { DatabaseName, getDatasource } from "../../../integrations/tests/utils"
 import merge from "lodash/merge"
 import { quotas } from "@budibase/pro"
-import { db, roles } from "@budibase/backend-core"
+import {
+  db,
+  roles,
+  withEnv as withCoreEnv,
+  setEnv as setCoreEnv,
+} from "@budibase/backend-core"
 
 describe.each([
   ["lucene", undefined],
@@ -89,12 +94,11 @@ describe.each([
   }
 
   beforeAll(async () => {
-    await config.withCoreEnv(
-      { SQS_SEARCH_ENABLE: isSqs ? "true" : "false" },
-      () => config.init()
+    await withCoreEnv({ SQS_SEARCH_ENABLE: isSqs ? "true" : "false" }, () =>
+      config.init()
     )
     if (isSqs) {
-      envCleanup = config.setCoreEnv({
+      envCleanup = setCoreEnv({
         SQS_SEARCH_ENABLE: "true",
         SQS_SEARCH_ENABLE_TENANTS: [config.getTenantId()],
       })

packages/server/src/api/routes/tests/webhook.spec.ts

@@ -2,6 +2,7 @@ import { Webhook } from "@budibase/types"
 import * as setup from "./utilities"
 import { checkBuilderEndpoint } from "./utilities/TestFunctions"
 import { mocks } from "@budibase/backend-core/tests"
+import { setEnv } from "../../../environment"
 
 const { basicWebhook, basicAutomation, collectAutomation } = setup.structures
 
@@ -17,7 +18,7 @@ describe("/webhooks", () => {
   })
 
   const setupTest = async () => {
-    cleanupEnv = config.setEnv({ SELF_HOSTED: "true" })
+    cleanupEnv = setEnv({ SELF_HOSTED: "true" })
     await config.init()
     const autoConfig = basicAutomation()
     autoConfig.definition.trigger.schema = {

packages/server/src/appMigrations/migrations/tests/20240604153647_initial_sqs.spec.ts

@@ -1,6 +1,10 @@
 import * as setup from "../../../api/routes/tests/utilities"
 import { basicTable } from "../../../tests/utilities/structures"
-import { db as dbCore, SQLITE_DESIGN_DOC_ID } from "@budibase/backend-core"
+import {
+  db as dbCore,
+  SQLITE_DESIGN_DOC_ID,
+  withEnv as withCoreEnv,
+} from "@budibase/backend-core"
 import {
   LinkDocument,
   DocumentType,
@@ -69,11 +73,11 @@ function oldLinkDocument(): Omit<LinkDocument, "tableId"> {
 type SQSEnvVar = "SQS_MIGRATION_ENABLE" | "SQS_SEARCH_ENABLE"
 
 async function sqsDisabled(envVar: SQSEnvVar, cb: () => Promise<void>) {
-  await config.withCoreEnv({ [envVar]: "", SQS_SEARCH_ENABLE_TENANTS: [] }, cb)
+  await withCoreEnv({ [envVar]: "", SQS_SEARCH_ENABLE_TENANTS: [] }, cb)
 }
 
 async function sqsEnabled(envVar: SQSEnvVar, cb: () => Promise<void>) {
-  await config.withCoreEnv(
+  await withCoreEnv(
     { [envVar]: "1", SQS_SEARCH_ENABLE_TENANTS: [config.getTenantId()] },
     cb
   )

packages/server/src/automations/tests/openai.spec.ts

@@ -1,5 +1,9 @@
 import { getConfig, runStep, afterAll as _afterAll } from "./utilities"
 import { OpenAI } from "openai"
+import {
+  withEnv as withCoreEnv,
+  setEnv as setCoreEnv,
+} from "@budibase/backend-core"
 
 jest.mock("openai", () => ({
   OpenAI: jest.fn().mockImplementation(() => ({
@@ -32,7 +36,7 @@ describe("test the openai action", () => {
   })
 
   beforeEach(() => {
-    resetEnv = config.setCoreEnv({ OPENAI_API_KEY: "abc123" })
+    resetEnv = setCoreEnv({ OPENAI_API_KEY: "abc123" })
   })
 
   afterEach(() => {
@@ -42,7 +46,7 @@ describe("test the openai action", () => {
   afterAll(_afterAll)
 
   it("should present the correct error message when the OPENAI_API_KEY variable isn't set", async () => {
-    await config.withCoreEnv({ OPENAI_API_KEY: "" }, async () => {
+    await withCoreEnv({ OPENAI_API_KEY: "" }, async () => {
       let res = await runStep("OPENAI", { prompt: OPENAI_PROMPT })
       expect(res.response).toEqual(
         "OpenAI API Key not configured - please add the OPENAI_API_KEY environment variable."

packages/server/src/environment.ts

@@ -1,5 +1,6 @@
 import { env as coreEnv } from "@budibase/backend-core"
 import { ServiceType } from "@budibase/types"
+import cloneDeep from "lodash/cloneDeep"
 
 coreEnv._set("SERVICE_TYPE", ServiceType.APPS)
 import { join } from "path"
@@ -133,6 +134,32 @@ const environment = {
   },
 }
 
+export function setEnv(newEnvVars: Partial<typeof environment>): () => void {
+  const oldEnv = cloneDeep(environment)
+
+  let key: keyof typeof newEnvVars
+  for (key in newEnvVars) {
+    environment._set(key, newEnvVars[key])
+  }
+
+  return () => {
+    for (const [key, value] of Object.entries(oldEnv)) {
+      environment._set(key, value)
+    }
+  }
+}
+
+export function withEnv<T>(envVars: Partial<typeof environment>, f: () => T) {
+  const cleanup = setEnv(envVars)
+  const result = f()
+  if (result instanceof Promise) {
+    return result.finally(cleanup)
+  } else {
+    cleanup()
+    return result
+  }
+}
+
 function cleanVariables() {
   // clean up any environment variable edge cases
   for (let [key, value] of Object.entries(environment)) {

packages/server/src/integrations/tests/googlesheets.spec.ts

@@ -1,3 +1,4 @@
+import { setEnv as setCoreEnv } from "@budibase/backend-core"
 import type { GoogleSpreadsheetWorksheet } from "google-spreadsheet"
 import nock from "nock"
 
@@ -40,7 +41,7 @@ describe("Google Sheets Integration", () => {
   let cleanupEnv: () => void
 
   beforeAll(() => {
-    cleanupEnv = config.setCoreEnv({
+    cleanupEnv = setCoreEnv({
       GOOGLE_CLIENT_ID: "test",
       GOOGLE_CLIENT_SECRET: "test",
     })

packages/server/src/sdk/app/rows/search/tests/search.spec.ts

@@ -3,6 +3,10 @@ import { Datasource, FieldType, Row, Table } from "@budibase/types"
 import TestConfiguration from "../../../../../tests/utilities/TestConfiguration"
 import { search } from "../../../../../sdk/app/rows/search"
 import { generator } from "@budibase/backend-core/tests"
+import {
+  withEnv as withCoreEnv,
+  setEnv as setCoreEnv,
+} from "@budibase/backend-core"
 import {
   DatabaseName,
   getDatasource,
@@ -31,13 +35,12 @@ describe.each([
   let rows: Row[]
 
   beforeAll(async () => {
-    await config.withCoreEnv(
-      { SQS_SEARCH_ENABLE: isSqs ? "true" : "false" },
-      () => config.init()
+    await withCoreEnv({ SQS_SEARCH_ENABLE: isSqs ? "true" : "false" }, () =>
+      config.init()
     )
 
     if (isSqs) {
-      envCleanup = config.setCoreEnv({
+      envCleanup = setCoreEnv({
         SQS_SEARCH_ENABLE: "true",
         SQS_SEARCH_ENABLE_TENANTS: [config.getTenantId()],
       })

packages/server/src/startup/index.ts

@@ -9,6 +9,7 @@ import {
   users,
   cache,
   env as coreEnv,
+  features,
 } from "@budibase/backend-core"
 import { watch } from "../watch"
 import * as automations from "../automations"
@@ -96,6 +97,9 @@ export async function startup(
   console.log("Initialising events")
   eventInit()
 
+  console.log("Initialising feature flags")
+  features.init()
+
   if (app && server) {
     console.log("Initialising websockets")
     initialiseWebsockets(app, server)

packages/server/src/startup/tests/startup.spec.ts

@@ -1,6 +1,12 @@
+import { withEnv } from "../../environment"
 import TestConfiguration from "../../tests/utilities/TestConfiguration"
 import { startup } from "../index"
-import { users, utils, tenancy } from "@budibase/backend-core"
+import {
+  users,
+  utils,
+  tenancy,
+  withEnv as withCoreEnv,
+} from "@budibase/backend-core"
 import nock from "nock"
 
 describe("check BB_ADMIN environment variables", () => {
@@ -23,13 +29,13 @@ describe("check BB_ADMIN environment variables", () => {
     const EMAIL = "budibase@budibase.com",
       PASSWORD = "budibase"
     await tenancy.doInTenant(tenancy.DEFAULT_TENANT_ID, async () => {
-      await config.withEnv(
+      await withEnv(
         {
           MULTI_TENANCY: "0",
           SELF_HOSTED: "1",
         },
         () =>
-          config.withCoreEnv(
+          withCoreEnv(
             {
               BB_ADMIN_USER_EMAIL: EMAIL,
               BB_ADMIN_USER_PASSWORD: PASSWORD,

packages/server/src/tests/utilities/TestConfiguration.ts

@@ -70,7 +70,6 @@ import {
 } from "@budibase/types"
 
 import API from "./api"
-import { cloneDeep } from "lodash"
 import jwt, { Secret } from "jsonwebtoken"
 import { Server } from "http"
 
@@ -247,65 +246,6 @@ export default class TestConfiguration {
     }
   }
 
-  async withEnv<T>(newEnvVars: Partial<typeof env>, f: () => Promise<T>) {
-    let cleanup = this.setEnv(newEnvVars)
-    try {
-      return await f()
-    } finally {
-      cleanup()
-    }
-  }
-
-  /*
-   * Sets the environment variables to the given values and returns a function
-   * that can be called to reset the environment variables to their original values.
-   */
-  setEnv(newEnvVars: Partial<typeof env>): () => void {
-    const oldEnv = cloneDeep(env)
-
-    let key: keyof typeof newEnvVars
-    for (key in newEnvVars) {
-      env._set(key, newEnvVars[key])
-    }
-
-    return () => {
-      for (const [key, value] of Object.entries(oldEnv)) {
-        env._set(key, value)
-      }
-    }
-  }
-
-  async withCoreEnv<T>(
-    newEnvVars: Partial<typeof coreEnv>,
-    f: () => Promise<T>
-  ) {
-    let cleanup = this.setCoreEnv(newEnvVars)
-    try {
-      return await f()
-    } finally {
-      cleanup()
-    }
-  }
-
-  /*
-   * Sets the environment variables to the given values and returns a function
-   * that can be called to reset the environment variables to their original values.
-   */
-  setCoreEnv(newEnvVars: Partial<typeof coreEnv>): () => void {
-    const oldEnv = cloneDeep(coreEnv)
-
-    let key: keyof typeof newEnvVars
-    for (key in newEnvVars) {
-      coreEnv._set(key, newEnvVars[key])
-    }
-
-    return () => {
-      for (const [key, value] of Object.entries(oldEnv)) {
-        coreEnv._set(key, value)
-      }
-    }
-  }
-
   async withUser(user: User, f: () => Promise<void>) {
     const oldUser = this.user
     this.user = user

packages/shared-core/src/table.ts

@@ -11,6 +11,7 @@ const allowDisplayColumnByType: Record<FieldType, boolean> = {
   [FieldType.AUTO]: true,
   [FieldType.INTERNAL]: true,
   [FieldType.BARCODEQR]: true,
+
   [FieldType.BIGINT]: true,
   [FieldType.BOOLEAN]: false,
   [FieldType.ARRAY]: false,
@@ -35,6 +36,30 @@ const allowSortColumnByType: Record<FieldType, boolean> = {
   [FieldType.BIGINT]: true,
   [FieldType.BOOLEAN]: true,
   [FieldType.JSON]: true,
+
+  [FieldType.FORMULA]: false,
+  [FieldType.ATTACHMENTS]: false,
+  [FieldType.ATTACHMENT_SINGLE]: false,
+  [FieldType.SIGNATURE_SINGLE]: false,
+  [FieldType.ARRAY]: false,
+  [FieldType.LINK]: false,
+  [FieldType.BB_REFERENCE]: false,
+  [FieldType.BB_REFERENCE_SINGLE]: false,
+}
+
+const allowDefaultColumnByType: Record<FieldType, boolean> = {
+  [FieldType.NUMBER]: true,
+  [FieldType.JSON]: true,
+  [FieldType.DATETIME]: true,
+  [FieldType.LONGFORM]: true,
+  [FieldType.STRING]: true,
+
+  [FieldType.OPTIONS]: false,
+  [FieldType.AUTO]: false,
+  [FieldType.INTERNAL]: false,
+  [FieldType.BARCODEQR]: false,
+  [FieldType.BIGINT]: false,
+  [FieldType.BOOLEAN]: false,
   [FieldType.FORMULA]: false,
   [FieldType.ATTACHMENTS]: false,
   [FieldType.ATTACHMENT_SINGLE]: false,
@@ -53,6 +78,10 @@ export function canBeSortColumn(type: FieldType): boolean {
   return !!allowSortColumnByType[type]
 }
 
+export function canHaveDefaultColumn(type: FieldType): boolean {
+  return !!allowDefaultColumnByType[type]
+}
+
 export function findDuplicateInternalColumns(table: Table): string[] {
   // maintains the case of keys
   const casedKeys = Object.keys(table.schema)

packages/types/src/documents/app/table/schema.ts

@@ -114,7 +114,6 @@ export interface FormulaFieldMetadata extends BaseFieldSchema {
   type: FieldType.FORMULA
   formula: string
   formulaType?: FormulaType
-  default?: string
 }
 
 export interface BBReferenceFieldMetadata

packages/worker/src/api/controllers/global/self.ts

@@ -104,7 +104,7 @@ export async function getSelf(ctx: any) {
   ctx.body = await groups.enrichUserRolesFromGroups(user)
 
   // add the feature flags for this tenant
-  const flags = await features.fetch()
+  const flags = await features.flags.fetch()
   ctx.body.flags = flags
 
   addSessionAttributesToUser(ctx)

packages/worker/src/api/controllers/global/users.ts

@@ -41,6 +41,14 @@ import { BpmStatusKey, BpmStatusValue } from "@budibase/shared-core"
 
 const MAX_USERS_UPLOAD_LIMIT = 1000
 
+const generatePassword = (length: number) => {
+  const array = new Uint8Array(length)
+  crypto.getRandomValues(array)
+  return Array.from(array, byte => byte.toString(36).padStart(2, "0"))
+    .join("")
+    .slice(0, length)
+}
+
 export const save = async (ctx: UserCtx<User, SaveUserResponse>) => {
   try {
     const currentUserId = ctx.user?._id
@@ -296,7 +304,7 @@ export const onboardUsers = async (
 
   let createdPasswords: Record<string, string> = {}
   const users: User[] = ctx.request.body.map(invite => {
-    let password = Math.random().toString(36).substring(2, 22)
+    const password = generatePassword(12)
     createdPasswords[invite.email] = password
 
     return {

packages/worker/src/api/routes/global/tests/auth.spec.ts

@@ -292,9 +292,9 @@ describe("/api/global/auth", () => {
       it("redirects to auth provider", async () => {
         nock("http://someconfigurl").get("/").times(1).reply(200, {
           issuer: "test",
-          authorization_endpoint: "http://localhost/auth",
-          token_endpoint: "http://localhost/token",
-          userinfo_endpoint: "http://localhost/userinfo",
+          authorization_endpoint: "http://example.com/auth",
+          token_endpoint: "http://example.com/token",
+          userinfo_endpoint: "http://example.com/userinfo",
         })
 
         const configId = await generateOidcConfig()
@@ -305,7 +305,7 @@ describe("/api/global/auth", () => {
         const location: string = res.get("location")
         expect(
           location.startsWith(
-            `http://localhost/auth?response_type=code&client_id=clientId&redirect_uri=http%3A%2F%2Flocalhost%3A10000%2Fapi%2Fglobal%2Fauth%2F${config.tenantId}%2Foidc%2Fcallback&scope=openid%20profile%20email%20offline_access`
+            `http://example.com/auth?response_type=code&client_id=clientId&redirect_uri=http%3A%2F%2Flocalhost%3A10000%2Fapi%2Fglobal%2Fauth%2F${config.tenantId}%2Foidc%2Fcallback&scope=openid%20profile%20email%20offline_access`
           )
         ).toBe(true)
       })
@@ -313,11 +313,13 @@ describe("/api/global/auth", () => {
 
     describe("GET /api/global/auth/:tenantId/oidc/callback", () => {
       it("logs in", async () => {
+        const email = `${generator.guid()}@example.com`
+
         nock("http://someconfigurl").get("/").times(2).reply(200, {
           issuer: "test",
-          authorization_endpoint: "http://localhost/auth",
-          token_endpoint: "http://localhost/token",
-          userinfo_endpoint: "http://localhost/userinfo",
+          authorization_endpoint: "http://example.com/auth",
+          token_endpoint: "http://example.com/token",
+          userinfo_endpoint: "http://example.com/userinfo",
         })
 
         const token = jwt.sign(
@@ -326,20 +328,20 @@ describe("/api/global/auth", () => {
             sub: "sub",
             aud: "clientId",
             exp: Math.floor(Date.now() / 1000) + 60 * 60,
-            email: "oauth@example.com",
+            email,
           },
           "secret"
         )
 
-        nock("http://localhost").post("/token").reply(200, {
+        nock("http://example.com").post("/token").reply(200, {
           access_token: "access",
           refresh_token: "refresh",
           id_token: token,
         })
 
-        nock("http://localhost").get("/userinfo?schema=openid").reply(200, {
+        nock("http://example.com").get("/userinfo?schema=openid").reply(200, {
           sub: "sub",
-          email: "oauth@example.com",
+          email,
         })
 
         const configId = await generateOidcConfig()
@@ -351,10 +353,7 @@ describe("/api/global/auth", () => {
           )
         }
 
-        expect(events.auth.login).toHaveBeenCalledWith(
-          "oidc",
-          "oauth@example.com"
-        )
+        expect(events.auth.login).toHaveBeenCalledWith("oidc", email)
         expect(events.auth.login).toHaveBeenCalledTimes(1)
         expect(res.status).toBe(302)
         const location: string = res.get("location")

packages/worker/src/api/routes/global/tests/realEmail.spec.ts

@@ -12,6 +12,33 @@ const nodemailer = require("nodemailer")
 // for the real email tests give them a long time to try complete/fail
 jest.setTimeout(30000)
 
+function cancelableTimeout(timeout: number): [Promise<unknown>, () => void] {
+  let timeoutId: NodeJS.Timeout
+  return [
+    new Promise((resolve, reject) => {
+      timeoutId = setTimeout(() => {
+        reject({
+          status: 301,
+          errno: "ETIME",
+        })
+      }, timeout)
+    }),
+    () => {
+      clearTimeout(timeoutId)
+    },
+  ]
+}
+
+async function withTimeout<T>(
+  timeout: number,
+  promise: Promise<T>
+): Promise<T> {
+  const [timeoutPromise, cancel] = cancelableTimeout(timeout)
+  const result = (await Promise.race([promise, timeoutPromise])) as T
+  cancel()
+  return result
+}
+
 describe("/api/global/email", () => {
   const config = new TestConfiguration()
 
@@ -30,19 +57,8 @@ describe("/api/global/email", () => {
   ) {
     let response, text
     try {
-      const timeout = () =>
-        new Promise((resolve, reject) =>
-          setTimeout(
-            () =>
-              reject({
-                status: 301,
-                errno: "ETIME",
-              }),
-            20000
-          )
-        )
-      await Promise.race([config.saveEtherealSmtpConfig(), timeout()])
-      await Promise.race([config.saveSettingsConfig(), timeout()])
+      await withTimeout(20000, config.saveEtherealSmtpConfig())
+      await withTimeout(20000, config.saveSettingsConfig())
       let res
       if (attachments) {
         res = await config.api.emails

packages/worker/src/index.ts

@@ -18,6 +18,7 @@ import {
   timers,
   redis,
   cache,
+  features,
 } from "@budibase/backend-core"
 
 db.init()
@@ -95,6 +96,7 @@ export default server.listen(parseInt(env.PORT || "4002"), async () => {
   console.log(startupLog)
   await initPro()
   await redis.clients.init()
+  features.init()
   cache.docWritethrough.init()
   // configure events to use the pro audit log write
   // can't integrate directly into backend-core due to cyclic issues

packages/worker/src/tests/api/configs.ts

@@ -40,7 +40,7 @@ export class ConfigAPI extends TestAPI {
     const sessionContent = JSON.parse(
       Buffer.from(koaSession, "base64").toString("utf-8")
     )
-    const handle = sessionContent["openidconnect:localhost"].state.handle
+    const handle = sessionContent["openidconnect:example.com"].state.handle
     return this.request
       .get(`/api/global/auth/${this.config.getTenantId()}/oidc/callback`)
       .query({ code: "test", state: handle })

packages/worker/src/tests/jestSetup.ts

@@ -1,13 +1,21 @@
 import { mocks, testContainerUtils } from "@budibase/backend-core/tests"
 import env from "../environment"
 import { env as coreEnv, timers } from "@budibase/backend-core"
-
-// must explicitly enable fetch mock
-mocks.fetch.enable()
+import nock from "nock"
 
 // mock all dates to 2020-01-01T00:00:00.000Z
 // use tk.reset() to use real dates in individual tests
-const tk = require("timekeeper")
+import tk from "timekeeper"
+
+nock.disableNetConnect()
+nock.enableNetConnect(host => {
+  return (
+    host.includes("localhost") ||
+    host.includes("127.0.0.1") ||
+    host.includes("::1") ||
+    host.includes("ethereal.email") // used in realEmail.spec.ts
+  )
+})
 
 tk.freeze(mocks.date.MOCK_DATE)
 

yarn.lock

@@ -2045,6 +2045,44 @@
   resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
   integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==
 
+"@budibase/backend-core@2.29.24":
+  version "0.0.0"
+  dependencies:
+    "@budibase/nano" "10.1.5"
+    "@budibase/pouchdb-replication-stream" "1.2.11"
+    "@budibase/shared-core" "0.0.0"
+    "@budibase/types" "0.0.0"
+    aws-cloudfront-sign "3.0.2"
+    aws-sdk "2.1030.0"
+    bcrypt "5.1.0"
+    bcryptjs "2.4.3"
+    bull "4.10.1"
+    correlation-id "4.0.0"
+    dd-trace "5.2.0"
+    dotenv "16.0.1"
+    ioredis "5.3.2"
+    joi "17.6.0"
+    jsonwebtoken "9.0.2"
+    knex "2.4.2"
+    koa-passport "^6.0.0"
+    koa-pino-logger "4.0.0"
+    lodash "4.17.21"
+    node-fetch "2.6.7"
+    passport-google-oauth "2.0.0"
+    passport-local "1.0.0"
+    passport-oauth2-refresh "^2.1.0"
+    pino "8.11.0"
+    pino-http "8.3.3"
+    posthog-node "4.0.1"
+    pouchdb "7.3.0"
+    pouchdb-find "7.2.2"
+    redlock "4.2.0"
+    rotating-file-stream "3.1.0"
+    sanitize-s3-objectkey "0.0.1"
+    semver "^7.5.4"
+    tar-fs "2.1.1"
+    uuid "^8.3.2"
+
 "@budibase/handlebars-helpers@^0.13.2":
   version "0.13.2"
   resolved "https://registry.yarnpkg.com/@budibase/handlebars-helpers/-/handlebars-helpers-0.13.2.tgz#73ab51c464e91fd955b429017648e0257060db77"
@@ -2087,6 +2125,44 @@
     pouchdb-promise "^6.0.4"
     through2 "^2.0.0"
 
+"@budibase/pro@npm:@budibase/pro@latest":
+  version "2.29.24"
+  resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.29.24.tgz#2dbd4c6c0f757aab7e17c413c6d6e4520086f9ac"
+  integrity sha512-m1v24UD6O21Vbrfsuo5kC5oeg7FzjWO2w8TQMw1VvPKmdIqqclaKDPTPytxwllTMkapMDRNzM5cQzqnQ3yHf6A==
+  dependencies:
+    "@budibase/backend-core" "2.29.24"
+    "@budibase/shared-core" "2.29.24"
+    "@budibase/string-templates" "2.29.24"
+    "@budibase/types" "2.29.24"
+    "@koa/router" "8.0.8"
+    bull "4.10.1"
+    joi "17.6.0"
+    jsonwebtoken "9.0.2"
+    lru-cache "^7.14.1"
+    memorystream "^0.3.1"
+    node-fetch "2.6.7"
+    scim-patch "^0.8.1"
+    scim2-parse-filter "^0.2.8"
+
+"@budibase/shared-core@2.29.24":
+  version "0.0.0"
+  dependencies:
+    "@budibase/types" "0.0.0"
+    cron-validate "1.4.5"
+
+"@budibase/string-templates@2.29.24":
+  version "0.0.0"
+  dependencies:
+    "@budibase/handlebars-helpers" "^0.13.2"
+    dayjs "^1.10.8"
+    handlebars "^4.7.8"
+    lodash.clonedeep "^4.5.0"
+
+"@budibase/types@2.29.24":
+  version "0.0.0"
+  dependencies:
+    scim-patch "^0.8.1"
+
 "@bull-board/api@5.10.2":
   version "5.10.2"
   resolved "https://registry.yarnpkg.com/@bull-board/api/-/api-5.10.2.tgz#ae8ff6918b23897bf879a6ead3683f964374c4b3"
@@ -7267,30 +7343,7 @@ axios-retry@^3.1.9:
     "@babel/runtime" "^7.15.4"
     is-retry-allowed "^2.2.0"
 
-axios@0.24.0:
-  version "0.24.0"
-  resolved "https://registry.yarnpkg.com/axios/-/axios-0.24.0.tgz#804e6fa1e4b9c5288501dd9dff56a7a0940d20d6"
-  integrity sha512-Q6cWsys88HoPgAaFAVUb0WpPk0O8iTeisR9IMqy9G8AbO4NlpVknrnQS03zzF9PGAWgO3cgletO3VjV/P7VztA==
-  dependencies:
-    follow-redirects "^1.14.4"
-
-axios@1.1.3:
-  version "1.1.3"
-  resolved "https://registry.yarnpkg.com/axios/-/axios-1.1.3.tgz#8274250dada2edf53814ed7db644b9c2866c1e35"
-  integrity sha512-00tXVRwKx/FZr/IDVFt4C+f9FYairX517WoGCL6dpOntqLkZofjhu43F/Xl44UOpqa+9sLFDrG/XAnFsUYgkDA==
-  dependencies:
-    follow-redirects "^1.15.0"
-    form-data "^4.0.0"
-    proxy-from-env "^1.1.0"
-
-axios@^0.21.1:
-  version "0.21.4"
-  resolved "https://registry.yarnpkg.com/axios/-/axios-0.21.4.tgz#c67b90dc0568e5c1cf2b0b858c43ba28e2eda575"
-  integrity sha512-ut5vewkiu8jjGBdqpM44XxjuCjq9LAKeHVmoVfHVzy8eHgxxq8SbAVQNovDA8mVi05kP0Ea/n/UzcSHcTJQfNg==
-  dependencies:
-    follow-redirects "^1.14.0"
-
-axios@^1.0.0, axios@^1.1.3, axios@^1.4.0, axios@^1.5.0:
+axios@0.24.0, axios@1.1.3, axios@1.6.3, axios@^0.21.1, axios@^1.0.0, axios@^1.1.3, axios@^1.4.0, axios@^1.5.0, axios@^1.6.2:
   version "1.6.3"
   resolved "https://registry.yarnpkg.com/axios/-/axios-1.6.3.tgz#7f50f23b3aa246eff43c54834272346c396613f4"
   integrity sha512-fWyNdeawGam70jXSVlKl+SUNVcL6j6W79CuSIPfi6HnDUmSCH6gyUys/HrqHeA/wU0Az41rRgean494d0Jb+ww==
@@ -11291,7 +11344,7 @@ fn.name@1.x.x:
   resolved "https://registry.yarnpkg.com/fn.name/-/fn.name-1.1.0.tgz#26cad8017967aea8731bc42961d04a3d5988accc"
   integrity sha512-GRnmB5gPyJpAhTQdSZTSp9uaPSvl09KoYcMQtsB9rQoOmzs9dH6ffeccH+Z+cv6P68Hu5bC6JjRh4Ah/mHSNRw==
 
-follow-redirects@^1.14.0, follow-redirects@^1.14.4, follow-redirects@^1.15.0:
+follow-redirects@^1.15.0:
   version "1.15.6"
   resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.6.tgz#7f815c0cda4249c74ff09e95ef97c23b5fd0399b"
   integrity sha512-wWN62YITEaOpSK584EZXJafH1AGpO8RVgElfkuXbTOrPX4fIfOyEpW/CsiNd8JdYrAoOvafRTOEnvsO++qCqFA==
@@ -12379,12 +12432,7 @@ http-assert@^1.3.0:
     deep-equal "~1.0.1"
     http-errors "~1.8.0"
 
-http-cache-semantics@3.8.1:
-  version "3.8.1"
-  resolved "https://registry.yarnpkg.com/http-cache-semantics/-/http-cache-semantics-3.8.1.tgz#39b0e16add9b605bf0a9ef3d9daaf4843b4cacd2"
-  integrity sha512-5ai2iksyV8ZXmnZhHH4rWPoxxistEexSi5936zIQ1bnNTW5VnA85B6P/VpXiRM017IgRvb2kKo1a//y+0wSp3w==
-
-http-cache-semantics@^4.0.0, http-cache-semantics@^4.1.0, http-cache-semantics@^4.1.1:
+http-cache-semantics@3.8.1, http-cache-semantics@4.1.1, http-cache-semantics@^4.0.0, http-cache-semantics@^4.1.0, http-cache-semantics@^4.1.1:
   version "4.1.1"
   resolved "https://registry.yarnpkg.com/http-cache-semantics/-/http-cache-semantics-4.1.1.tgz#abe02fcb2985460bf0323be664436ec3476a6d5a"
   integrity sha512-er295DKPVsV82j5kw1Gjt+ADA/XYHsajl82cGNQG2eyoPkvgUhX+nDIyelzhIWbbsXP39EHcI6l5tYs2FYqYXQ==
@@ -13381,6 +13429,11 @@ isobject@^3.0.1:
   resolved "https://registry.yarnpkg.com/isobject/-/isobject-3.0.1.tgz#4e431e92b11a9731636aa1f9c8d1ccbcfdab78df"
   integrity sha512-WhB9zCku7EGTj/HQQRz5aUQEUeoQZH2bWcltRErOpymJ4boYE6wL9Tbr23krRPSZ+C5zqNSrSw+Cc7sZZ4b7vg==
 
+isobject@^4.0.0:
+  version "4.0.0"
+  resolved "https://registry.yarnpkg.com/isobject/-/isobject-4.0.0.tgz#3f1c9155e73b192022a80819bacd0343711697b0"
+  integrity sha512-S/2fF5wH8SJA/kmwr6HYhK/RI/OkhD84k8ntalo0iJjZikgq1XFvR5M8NPT1x5F7fBwCG3qHfnzeP/Vh/ZxCUA==
+
 isolated-vm@^4.7.2:
   version "4.7.2"
   resolved "https://registry.yarnpkg.com/isolated-vm/-/isolated-vm-4.7.2.tgz#5670d5cce1d92004f9b825bec5b0b11fc7501b65"
@@ -15939,7 +15992,7 @@ msgpackr-extract@^3.0.2:
     "@msgpackr-extract/msgpackr-extract-linux-x64" "3.0.2"
     "@msgpackr-extract/msgpackr-extract-win32-x64" "3.0.2"
 
-msgpackr@^1.5.2:
+msgpackr@1.10.1, msgpackr@^1.5.2:
   version "1.10.1"
   resolved "https://registry.yarnpkg.com/msgpackr/-/msgpackr-1.10.1.tgz#51953bb4ce4f3494f0c4af3f484f01cfbb306555"
   integrity sha512-r5VRLv9qouXuLiIBrLpl2d5ZvPt8svdQTl5/vMvE4nzDMyEX4sgW5yWhuBBj5UmgwOTWj8CIdSXn5sAfsHAWIQ==
@@ -16133,25 +16186,13 @@ node-domexception@1.0.0:
   resolved "https://registry.yarnpkg.com/node-domexception/-/node-domexception-1.0.0.tgz#6888db46a1f71c0b76b3f7555016b63fe64766e5"
   integrity sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==
 
-node-fetch@2.6.0:
-  version "2.6.0"
-  resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.0.tgz#e633456386d4aa55863f676a7ab0daa8fdecb0fd"
-  integrity sha512-8dG4H5ujfvFiqDmVu9fQ5bOHUC15JMjMY/Zumv26oOvvVJjM67KF8koCWIabKQ1GJIa9r2mMZscBq/TbdOcmNA==
-
-node-fetch@2.6.7, node-fetch@^2.6.0, node-fetch@^2.6.1, node-fetch@^2.6.7:
+node-fetch@2.6.0, node-fetch@2.6.7, node-fetch@^2.6.0, node-fetch@^2.6.1, node-fetch@^2.6.7, node-fetch@^2.6.9, node-fetch@^2.7.0:
   version "2.6.7"
   resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.7.tgz#24de9fba827e3b4ae44dc8b20256a379160052ad"
   integrity sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==
   dependencies:
     whatwg-url "^5.0.0"
 
-node-fetch@^2.6.9, node-fetch@^2.7.0:
-  version "2.7.0"
-  resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.7.0.tgz#d0f0fa6e3e2dc1d27efcd8ad99d550bda94d187d"
-  integrity sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==
-  dependencies:
-    whatwg-url "^5.0.0"
-
 node-forge@^1.2.1, node-forge@^1.3.1:
   version "1.3.1"
   resolved "https://registry.yarnpkg.com/node-forge/-/node-forge-1.3.1.tgz#be8da2af243b2417d5f646a770663a92b7e9ded3"
@@ -17304,15 +17345,7 @@ passport-strategy@1.x.x, passport-strategy@^1.0.0:
   resolved "https://registry.yarnpkg.com/passport-strategy/-/passport-strategy-1.0.0.tgz#b5539aa8fc225a3d1ad179476ddf236b440f52e4"
   integrity sha512-CB97UUvDKJde2V0KDWWB3lyf6PC3FaZP7YxZ2G8OAtn9p4HI9j9JLP9qjOGZFvyl8uwNT8qM+hGnz/n16NI7oA==
 
-passport@^0.4.0:
-  version "0.4.1"
-  resolved "https://registry.yarnpkg.com/passport/-/passport-0.4.1.tgz#941446a21cb92fc688d97a0861c38ce9f738f270"
-  integrity sha512-IxXgZZs8d7uFSt3eqNjM9NQ3g3uQCW5avD8mRNoXV99Yig50vjuaez6dQK2qC0kVWPRTujxY0dWgGfT09adjYg==
-  dependencies:
-    passport-strategy "1.x.x"
-    pause "0.0.1"
-
-passport@^0.6.0:
+passport@0.6.0, passport@^0.4.0, passport@^0.6.0:
   version "0.6.0"
   resolved "https://registry.yarnpkg.com/passport/-/passport-0.6.0.tgz#e869579fab465b5c0b291e841e6cc95c005fac9d"
   integrity sha512-0fe+p3ZnrWRW74fe8+SvCyf4a3Pb2/h7gFkQ8yTJpAO50gDzlfjZUZTO1k5Eg9kUct22OxHLqDZoKUWRHOh9ug==
@@ -18085,6 +18118,14 @@ posthog-node@1.3.0:
     remove-trailing-slash "^0.1.1"
     uuid "^8.3.2"
 
+posthog-node@4.0.1:
+  version "4.0.1"
+  resolved "https://registry.yarnpkg.com/posthog-node/-/posthog-node-4.0.1.tgz#eb8b6cdf68c3fdd0dc2b75e8aab2e0ec3727fb2a"
+  integrity sha512-rtqm2h22QxLGBrW2bLYzbRhliIrqgZ0k+gF0LkQ1SNdeD06YE5eilV0MxZppFSxC8TfH0+B0cWCuebEnreIDgQ==
+  dependencies:
+    axios "^1.6.2"
+    rusha "^0.8.14"
+
 pouch-stream@^0.4.0:
   version "0.4.1"
   resolved "https://registry.yarnpkg.com/pouch-stream/-/pouch-stream-0.4.1.tgz#0c6d8475c9307677627991a2f079b301c3b89bdd"
@@ -18581,7 +18622,7 @@ pseudomap@^1.0.2:
   resolved "https://registry.yarnpkg.com/pseudomap/-/pseudomap-1.0.2.tgz#f052a28da70e618917ef0a8ac34c1ae5a68286b3"
   integrity sha512-b/YwNhb8lk1Zz2+bXXpS/LK9OisiZZ1SNsSLxN1x2OXVEhW2Ckr/7mWE5vrC1ZTiJlD9g19jWszTmJsB+oEpFQ==
 
-psl@^1.1.28, psl@^1.1.33:
+psl@^1.1.33:
   version "1.9.0"
   resolved "https://registry.yarnpkg.com/psl/-/psl-1.9.0.tgz#d0df2a137f00794565fcaf3b2c00cd09f8d5a5a7"
   integrity sha512-E/ZsdU4HLs/68gYzgGTkMicWTLPdAftJLfJFlLUAAKZGkStNU72sZjT66SnMDVOfOWY/YAoiD7Jxa9iHvngcag==
@@ -19549,6 +19590,11 @@ run-parallel@^1.1.9:
   dependencies:
     queue-microtask "^1.2.2"
 
+rusha@^0.8.14:
+  version "0.8.14"
+  resolved "https://registry.yarnpkg.com/rusha/-/rusha-0.8.14.tgz#a977d0de9428406138b7bb90d3de5dcd024e2f68"
+  integrity sha512-cLgakCUf6PedEu15t8kbsjnwIFFR2D4RfL+W3iWFJ4iac7z4B0ZI8fxy4R3J956kAI68HclCFGL8MPoUVC3qVA==
+
 rxjs@^6.6.6:
   version "6.6.7"
   resolved "https://registry.yarnpkg.com/rxjs/-/rxjs-6.6.7.tgz#90ac018acabf491bf65044235d5863c4dab804c9"
@@ -19643,11 +19689,6 @@ sax@1.2.1:
   resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.1.tgz#7b8e656190b228e81a66aea748480d828cd2d37a"
   integrity sha512-8I2a3LovHTOpm7NV5yOyO8IHqgVsfK4+UuySrXU8YXkSRX7k6hCV9b3HrkKCr3nMpgj+0bmocaJJWpvp1oc7ZA==
 
-sax@>=0.1.1:
-  version "1.4.1"
-  resolved "https://registry.yarnpkg.com/sax/-/sax-1.4.1.tgz#44cc8988377f126304d3b3fc1010c733b929ef0f"
-  integrity sha512-+aWOz7yVScEGoKNd4PA10LZ8sk0A/z5+nXQG5giUO5rprX9jgYsTdov9qCchZiPIZezbZH+jRut8nPodFAX4Jg==
-
 sax@>=0.6.0:
   version "1.2.4"
   resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.4.tgz#2816234e2378bddc4e5354fab5caa895df7100d9"
@@ -19720,33 +19761,13 @@ semver-diff@^3.1.1:
   dependencies:
     semver "^6.3.0"
 
-"semver@2 || 3 || 4 || 5", semver@^5.5.0, semver@^5.6.0, semver@^5.7.1:
-  version "5.7.2"
-  resolved "https://registry.yarnpkg.com/semver/-/semver-5.7.2.tgz#48d55db737c3287cd4835e17fa13feace1c41ef8"
-  integrity sha512-cBznnQ9KjJqU67B52RMC65CMarK2600WFnbkcaiwWq3xy/5haFJlshgnpjovMVJ+Hff49d8GEn0b87C5pDQ10g==
-
-semver@7.5.3, semver@^7.0.0, semver@^7.1.1, semver@^7.3.2, semver@^7.3.4, semver@^7.3.5, semver@^7.3.7, semver@^7.3.8, semver@^7.5.3:
+"semver@2 || 3 || 4 || 5", semver@7.5.3, semver@^5.5.0, semver@^5.6.0, semver@^5.7.1, semver@^6.0.0, semver@^6.1.1, semver@^6.1.2, semver@^6.2.0, semver@^6.3.0, semver@^6.3.1, semver@^7.0.0, semver@^7.1.1, semver@^7.3.2, semver@^7.3.4, semver@^7.3.5, semver@^7.3.7, semver@^7.3.8, semver@^7.5.3, semver@^7.5.4, semver@^7.6.3, semver@~2.3.1:
   version "7.5.3"
   resolved "https://registry.yarnpkg.com/semver/-/semver-7.5.3.tgz#161ce8c2c6b4b3bdca6caadc9fa3317a4c4fe88e"
   integrity sha512-QBlUtyVk/5EeHbi7X0fw6liDZc7BBmEaSYn01fMU1OUYbf6GPsbTtd8WmnqbI20SeycoHSeiybkE/q1Q+qlThQ==
   dependencies:
     lru-cache "^6.0.0"
 
-semver@^6.0.0, semver@^6.1.1, semver@^6.1.2, semver@^6.2.0, semver@^6.3.0, semver@^6.3.1:
-  version "6.3.1"
-  resolved "https://registry.yarnpkg.com/semver/-/semver-6.3.1.tgz#556d2ef8689146e46dcea4bfdd095f3434dffcb4"
-  integrity sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==
-
-semver@^7.5.4, semver@^7.6.3:
-  version "7.6.3"
-  resolved "https://registry.yarnpkg.com/semver/-/semver-7.6.3.tgz#980f7b5550bc175fb4dc09403085627f9eb33143"
-  integrity sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A==
-
-semver@~2.3.1:
-  version "2.3.2"
-  resolved "https://registry.yarnpkg.com/semver/-/semver-2.3.2.tgz#b9848f25d6cf36333073ec9ef8856d42f1233e52"
-  integrity sha512-abLdIKCosKfpnmhS52NCTjO4RiLspDfsn37prjzGrp9im5DPJOgh82Os92vtwGh6XdQryKI/7SREZnV+aqiXrA==
-
 seq-queue@^0.0.5:
   version "0.0.5"
   resolved "https://registry.yarnpkg.com/seq-queue/-/seq-queue-0.0.5.tgz#d56812e1c017a6e4e7c3e3a37a1da6d78dd3c93e"
@@ -21311,7 +21332,7 @@ touch@^3.1.0:
   dependencies:
     nopt "~1.0.10"
 
-"tough-cookie@^2.3.3 || ^3.0.1 || ^4.0.0", tough-cookie@^4.0.0, tough-cookie@^4.1.2:
+tough-cookie@4.1.3, "tough-cookie@^2.3.3 || ^3.0.1 || ^4.0.0", tough-cookie@^4.0.0, tough-cookie@^4.1.2, tough-cookie@~2.5.0:
   version "4.1.3"
   resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-4.1.3.tgz#97b9adb0728b42280aa3d814b6b999b2ff0318bf"
   integrity sha512-aX/y5pVRkfRnfmuX+OdbSdXvPe6ieKX/G2s7e98f4poJHnqH3281gDPm/metm6E/WRamfx7WC4HUqkWHfQHprw==
@@ -21321,14 +21342,6 @@ touch@^3.1.0:
     universalify "^0.2.0"
     url-parse "^1.5.3"
 
-tough-cookie@~2.5.0:
-  version "2.5.0"
-  resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-2.5.0.tgz#cd9fb2a0aa1d5a12b473bd9fb96fa3dcff65ade2"
-  integrity sha512-nlLsUzgm1kfLXSXfRZMc1KLAugd4hqJHDTvc2hDIwS3mZAfMEuMbc03SujMF+GEcpaX/qboeycw6iO8JwVv2+g==
-  dependencies:
-    psl "^1.1.28"
-    punycode "^2.1.1"
-
 tr46@^2.1.0:
   version "2.1.0"
   resolved "https://registry.yarnpkg.com/tr46/-/tr46-2.1.0.tgz#fa87aa81ca5d5941da8cbf1f9b749dc969a4e240"
@@ -21812,6 +21825,14 @@ unpipe@1.0.0:
   resolved "https://registry.yarnpkg.com/unpipe/-/unpipe-1.0.0.tgz#b2bf4ee8514aae6165b4817829d21b2ef49904ec"
   integrity sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==
 
+unset-value@2.0.1:
+  version "2.0.1"
+  resolved "https://registry.yarnpkg.com/unset-value/-/unset-value-2.0.1.tgz#57bed0c22d26f28d69acde5df9a11b77c74d2df3"
+  integrity sha512-2hvrBfjUE00PkqN+q0XP6yRAOGrR06uSiUoIQGZkc7GxvQ9H7v8quUPNtZjMg4uux69i8HWpIjLPUKwCuRGyNg==
+  dependencies:
+    has-value "^2.0.2"
+    isobject "^4.0.0"
+
 untildify@^4.0.0:
   version "4.0.0"
   resolved "https://registry.yarnpkg.com/untildify/-/untildify-4.0.0.tgz#2bc947b953652487e4600949fb091e3ae8cd919b"
@@ -22578,33 +22599,10 @@ xml-parse-from-string@^1.0.0:
   resolved "https://registry.yarnpkg.com/xml-parse-from-string/-/xml-parse-from-string-1.0.1.tgz#a9029e929d3dbcded169f3c6e28238d95a5d5a28"
   integrity sha512-ErcKwJTF54uRzzNMXq2X5sMIy88zJvfN2DmdoQvy7PAFJ+tPRU6ydWuOKNMyfmOjdyBQTFREi60s0Y0SyI0G0g==
 
-xml2js@0.1.x:
-  version "0.1.14"
-  resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.1.14.tgz#5274e67f5a64c5f92974cd85139e0332adc6b90c"
-  integrity sha512-pbdws4PPPNc1HPluSUKamY4GWMk592K7qwcj6BExbVOhhubub8+pMda/ql68b6L3luZs/OGjGSB5goV7SnmgnA==
-  dependencies:
-    sax ">=0.1.1"
-
-xml2js@0.4.19:
-  version "0.4.19"
-  resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.4.19.tgz#686c20f213209e94abf0d1bcf1efaa291c7827a7"
-  integrity sha512-esZnJZJOiJR9wWKMyuvSE1y6Dq5LCuJanqhxslH2bxM6duahNZ+HMpCLhBQGZkbX6xRf8x1Y2eJlgt2q3qo49Q==
-  dependencies:
-    sax ">=0.6.0"
-    xmlbuilder "~9.0.1"
-
-xml2js@0.5.0:
-  version "0.5.0"
-  resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.5.0.tgz#d9440631fbb2ed800203fad106f2724f62c493b7"
-  integrity sha512-drPFnkQJik/O+uPKpqSgr22mpuFHqKdbS835iAQrUC73L2F5WkboIRd63ai/2Yg6I1jzifPFKH2NTK+cfglkIA==
-  dependencies:
-    sax ">=0.6.0"
-    xmlbuilder "~11.0.0"
-
-xml2js@^0.4.19, xml2js@^0.4.5:
-  version "0.4.23"
-  resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.4.23.tgz#a0c69516752421eb2ac758ee4d4ccf58843eac66"
-  integrity sha512-ySPiMjM0+pLDftHgXY4By0uswI3SPKLDw/i3UXbnO8M/p28zqexCUoPmQFrYD+/1BzhGJSs2i1ERWKJAtiLrug==
+xml2js@0.1.x, xml2js@0.4.19, xml2js@0.5.0, xml2js@0.6.2, xml2js@^0.4.19, xml2js@^0.4.5:
+  version "0.6.2"
+  resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.6.2.tgz#dd0b630083aa09c161e25a4d0901e2b2a929b499"
+  integrity sha512-T4rieHaC1EXcES0Kxxj4JWgaUQHDk+qwHcYOCFHfiwKz7tOVPLq7Hjq9dM1WCMhylqMEfP7hMcOIChvotiZegA==
   dependencies:
     sax ">=0.6.0"
     xmlbuilder "~11.0.0"
@@ -22614,11 +22612,6 @@ xmlbuilder@~11.0.0:
   resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-11.0.1.tgz#be9bae1c8a046e76b31127726347d0ad7002beb3"
   integrity sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA==
 
-xmlbuilder@~9.0.1:
-  version "9.0.7"
-  resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-9.0.7.tgz#132ee63d2ec5565c557e20f4c22df9aca686b10d"
-  integrity sha512-7YXTQc3P2l9+0rjaUbLwMKRhtmwg1M1eDf6nag7urC7pIPYLD9W/jmzQ4ptRSUbodw5S0jfoGTflLemQibSpeQ==
-
 xmlchars@^2.2.0:
   version "2.2.0"
   resolved "https://registry.yarnpkg.com/xmlchars/-/xmlchars-2.2.0.tgz#060fe1bcb7f9c76fe2a17db86a9bc3ab894210cb"