MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge master.

This commit is contained in:
Sam Rose 2024-10-08 15:06:57 +01:00
parent 9b3ab049a4 ad7fd46767
commit f1b04d1252
No known key found for this signature in database
24 changed files with 546 additions and 454 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

300
packages/backend-core/src/features/features.ts Normal file
View File

@ -0,0 +1,300 @@
import env from "../environment"
import * as context from "../context"
import { PostHog, PostHogOptions } from "posthog-node"
import { FeatureFlag, IdentityType, UserCtx } from "@budibase/types"
import tracer from "dd-trace"
import { Duration } from "../utils"
let posthog: PostHog | undefined
export function init(opts?: PostHogOptions) {
if (
env.POSTHOG_TOKEN &&
env.POSTHOG_API_HOST &&
!env.SELF_HOSTED &&
env.POSTHOG_FEATURE_FLAGS_ENABLED
) {
console.log("initializing posthog client...")
posthog = new PostHog(env.POSTHOG_TOKEN, {
host: env.POSTHOG_API_HOST,
personalApiKey: env.POSTHOG_PERSONAL_TOKEN,
featureFlagsPollingInterval: Duration.fromMinutes(3).toMs(),
...opts,
})
} else {
console.log("posthog disabled")
}
}
export function shutdown() {
posthog?.shutdown()
}
export abstract class Flag<T> {
static boolean(defaultValue: boolean): Flag<boolean> {
return new BooleanFlag(defaultValue)
}
static string(defaultValue: string): Flag<string> {
return new StringFlag(defaultValue)
}
static number(defaultValue: number): Flag<number> {
return new NumberFlag(defaultValue)
}
protected constructor(public defaultValue: T) {}
abstract parse(value: any): T
}
type UnwrapFlag<F> = F extends Flag<infer U> ? U : never
export type FlagValues<T> = {
[K in keyof T]: UnwrapFlag<T[K]>
}
type KeysOfType<T, U> = {
[K in keyof T]: T[K] extends Flag<U> ? K : never
}[keyof T]
class BooleanFlag extends Flag<boolean> {
parse(value: any) {
if (typeof value === "string") {
return ["true", "t", "1"].includes(value.toLowerCase())
}
if (typeof value === "boolean") {
return value
}
throw new Error(`could not parse value "${value}" as boolean`)
}
}
class StringFlag extends Flag<string> {
parse(value: any) {
if (typeof value === "string") {
return value
}
throw new Error(`could not parse value "${value}" as string`)
}
}
class NumberFlag extends Flag<number> {
parse(value: any) {
if (typeof value === "number") {
return value
}
if (typeof value === "string") {
const parsed = parseFloat(value)
if (!isNaN(parsed)) {
return parsed
}
}
throw new Error(`could not parse value "${value}" as number`)
}
}
export interface EnvFlagEntry {
tenantId: string
key: string
value: boolean
}
export function parseEnvFlags(flags: string): EnvFlagEntry[] {
const split = flags.split(",").map(x => x.split(":"))
const result: EnvFlagEntry[] = []
for (const [tenantId, ...features] of split) {
for (let feature of features) {
let value = true
if (feature.startsWith("!")) {
feature = feature.slice(1)
value = false
}
result.push({ tenantId, key: feature, value })
}
}
return result
}
export class FlagSet<V extends Flag<any>, T extends { [key: string]: V }> {
// This is used to safely cache flags sets in the current request context.
// Because multiple sets could theoretically exist, we don't want the cache of
// one to leak into another.
private readonly setId: string
constructor(private readonly flagSchema: T) {
this.setId = crypto.randomUUID()
}
defaults(): FlagValues<T> {
return Object.keys(this.flagSchema).reduce((acc, key) => {
const typedKey = key as keyof T
acc[typedKey] = this.flagSchema[key].defaultValue
return acc
}, {} as FlagValues<T>)
}
isFlagName(name: string | number | symbol): name is keyof T {
return this.flagSchema[name as keyof T] !== undefined
}
async get<K extends keyof T>(
key: K,
ctx?: UserCtx
): Promise<FlagValues<T>[K]> {
const flags = await this.fetch(ctx)
return flags[key]
}
async isEnabled<K extends KeysOfType<T, boolean>>(
key: K,
ctx?: UserCtx
): Promise<boolean> {
const flags = await this.fetch(ctx)
return flags[key]
}
async fetch(ctx?: UserCtx): Promise<FlagValues<T>> {
return await tracer.trace("features.fetch", async span => {
const cachedFlags = context.getFeatureFlags<FlagValues<T>>(this.setId)
if (cachedFlags) {
span?.addTags({ fromCache: true })
return cachedFlags
}
const tags: Record<string, any> = {}
const flagValues = this.defaults()
const currentTenantId = context.getTenantId()
const specificallySetFalse = new Set<string>()
for (const { tenantId, key, value } of parseEnvFlags(
env.TENANT_FEATURE_FLAGS || ""
)) {
if (!tenantId || (tenantId !== "*" && tenantId !== currentTenantId)) {
continue
}
tags[`readFromEnvironmentVars`] = true
if (value === false) {
specificallySetFalse.add(key)
}
// ignore unknown flags
if (!this.isFlagName(key)) {
continue
}
if (typeof flagValues[key] !== "boolean") {
throw new Error(`Feature: ${key} is not a boolean`)
}
// @ts-expect-error - TS does not like you writing into a generic type,
// but we know that it's okay in this case because it's just an object.
flagValues[key as keyof FlagValues] = value
tags[`flags.${key}.source`] = "environment"
}
const license = ctx?.user?.license
if (license) {
tags[`readFromLicense`] = true
for (const feature of license.features) {
if (!this.isFlagName(feature)) {
continue
}
if (
flagValues[feature] === true ||
specificallySetFalse.has(feature)
) {
// If the flag is already set to through environment variables, we
// don't want to override it back to false here.
continue
}
// @ts-expect-error - TS does not like you writing into a generic type,
// but we know that it's okay in this case because it's just an object.
flagValues[feature] = true
tags[`flags.${feature}.source`] = "license"
}
}
const identity = context.getIdentity()
tags[`identity.type`] = identity?.type
tags[`identity.tenantId`] = identity?.tenantId
tags[`identity._id`] = identity?._id
if (posthog && identity?.type === IdentityType.USER) {
tags[`readFromPostHog`] = true
const personProperties: Record<string, string> = {}
if (identity.tenantId) {
personProperties.tenantId = identity.tenantId
}
const posthogFlags = await posthog.getAllFlagsAndPayloads(
identity._id,
{
personProperties,
}
)
for (const [name, value] of Object.entries(posthogFlags.featureFlags)) {
if (!this.isFlagName(name)) {
// We don't want an unexpected PostHog flag to break the app, so we
// just log it and continue.
console.warn(`Unexpected posthog flag "${name}": ${value}`)
continue
}
if (flagValues[name] === true || specificallySetFalse.has(name)) {
// If the flag is already set to through environment variables, we
// don't want to override it back to false here.
continue
}
const payload = posthogFlags.featureFlagPayloads?.[name]
const flag = this.flagSchema[name]
try {
// @ts-expect-error - TS does not like you writing into a generic
// type, but we know that it's okay in this case because it's just
// an object.
flagValues[name] = flag.parse(payload || value)
tags[`flags.${name}.source`] = "posthog"
} catch (err) {
// We don't want an invalid PostHog flag to break the app, so we just
// log it and continue.
console.warn(`Error parsing posthog flag "${name}": ${value}`, err)
}
}
}
context.setFeatureFlags(this.setId, flagValues)
for (const [key, value] of Object.entries(flagValues)) {
tags[`flags.${key}.value`] = value
}
span?.addTags(tags)
return flagValues
})
}
}
// This is the primary source of truth for feature flags. If you want to add a
// new flag, add it here and use the `fetch` and `get` functions to access it.
// All of the machinery in this file is to make sure that flags have their
// default values set correctly and their types flow through the system.
export const flags = new FlagSet({
DEFAULT_VALUES: Flag.boolean(env.isDev()),
AUTOMATION_BRANCHING: Flag.boolean(env.isDev()),
SQS: Flag.boolean(env.isDev()),
[FeatureFlag.AI_CUSTOM_CONFIGS]: Flag.boolean(env.isDev()),
[FeatureFlag.ENRICHED_RELATIONSHIPS]: Flag.boolean(env.isDev()),
})
type UnwrapPromise<T> = T extends Promise<infer U> ? U : T
export type FeatureFlags = UnwrapPromise<ReturnType<typeof flags.fetch>>

283
packages/backend-core/src/features/index.ts
View File

@ -1,281 +1,2 @@
import env from "../environment"
import * as context from "../context"
import { PostHog, PostHogOptions } from "posthog-node"
import { FeatureFlag, IdentityType, UserCtx } from "@budibase/types"
import tracer from "dd-trace"
import { Duration } from "../utils"
let posthog: PostHog | undefined
export function init(opts?: PostHogOptions) {
if (
env.POSTHOG_TOKEN &&
env.POSTHOG_API_HOST &&
!env.SELF_HOSTED &&
env.POSTHOG_FEATURE_FLAGS_ENABLED
) {
console.log("initializing posthog client...")
posthog = new PostHog(env.POSTHOG_TOKEN, {
host: env.POSTHOG_API_HOST,
personalApiKey: env.POSTHOG_PERSONAL_TOKEN,
featureFlagsPollingInterval: Duration.fromMinutes(3).toMs(),
...opts,
})
} else {
console.log("posthog disabled")
}
}
export function shutdown() {
posthog?.shutdown()
}
export abstract class Flag<T> {
static boolean(defaultValue: boolean): Flag<boolean> {
return new BooleanFlag(defaultValue)
}
static string(defaultValue: string): Flag<string> {
return new StringFlag(defaultValue)
}
static number(defaultValue: number): Flag<number> {
return new NumberFlag(defaultValue)
}
protected constructor(public defaultValue: T) {}
abstract parse(value: any): T
}
type UnwrapFlag<F> = F extends Flag<infer U> ? U : never
export type FlagValues<T> = {
[K in keyof T]: UnwrapFlag<T[K]>
}
type KeysOfType<T, U> = {
[K in keyof T]: T[K] extends Flag<U> ? K : never
}[keyof T]
class BooleanFlag extends Flag<boolean> {
parse(value: any) {
if (typeof value === "string") {
return ["true", "t", "1"].includes(value.toLowerCase())
}
if (typeof value === "boolean") {
return value
}
throw new Error(`could not parse value "${value}" as boolean`)
}
}
class StringFlag extends Flag<string> {
parse(value: any) {
if (typeof value === "string") {
return value
}
throw new Error(`could not parse value "${value}" as string`)
}
}
class NumberFlag extends Flag<number> {
parse(value: any) {
if (typeof value === "number") {
return value
}
if (typeof value === "string") {
const parsed = parseFloat(value)
if (!isNaN(parsed)) {
return parsed
}
}
throw new Error(`could not parse value "${value}" as number`)
}
}
export class FlagSet<V extends Flag<any>, T extends { [key: string]: V }> {
// This is used to safely cache flags sets in the current request context.
// Because multiple sets could theoretically exist, we don't want the cache of
// one to leak into another.
private readonly setId: string
constructor(private readonly flagSchema: T) {
this.setId = crypto.randomUUID()
}
defaults(): FlagValues<T> {
return Object.keys(this.flagSchema).reduce((acc, key) => {
const typedKey = key as keyof T
acc[typedKey] = this.flagSchema[key].defaultValue
return acc
}, {} as FlagValues<T>)
}
isFlagName(name: string | number | symbol): name is keyof T {
return this.flagSchema[name as keyof T] !== undefined
}
async get<K extends keyof T>(
key: K,
ctx?: UserCtx
): Promise<FlagValues<T>[K]> {
const flags = await this.fetch(ctx)
return flags[key]
}
async isEnabled<K extends KeysOfType<T, boolean>>(
key: K,
ctx?: UserCtx
): Promise<boolean> {
const flags = await this.fetch(ctx)
return flags[key]
}
async fetch(ctx?: UserCtx): Promise<FlagValues<T>> {
return await tracer.trace("features.fetch", async span => {
const cachedFlags = context.getFeatureFlags<FlagValues<T>>(this.setId)
if (cachedFlags) {
span?.addTags({ fromCache: true })
return cachedFlags
}
const tags: Record<string, any> = {}
const flagValues = this.defaults()
const currentTenantId = context.getTenantId()
const specificallySetFalse = new Set<string>()
const split = (env.TENANT_FEATURE_FLAGS || "")
.split(",")
.map(x => x.split(":"))
for (const [tenantId, ...features] of split) {
if (!tenantId || (tenantId !== "*" && tenantId !== currentTenantId)) {
continue
}
tags[`readFromEnvironmentVars`] = true
for (let feature of features) {
let value = true
if (feature.startsWith("!")) {
feature = feature.slice(1)
value = false
specificallySetFalse.add(feature)
}
// ignore unknown flags
if (!this.isFlagName(feature)) {
continue
}
if (typeof flagValues[feature] !== "boolean") {
throw new Error(`Feature: ${feature} is not a boolean`)
}
// @ts-expect-error - TS does not like you writing into a generic type,
// but we know that it's okay in this case because it's just an object.
flagValues[feature as keyof FlagValues] = value
tags[`flags.${feature}.source`] = "environment"
}
}
const license = ctx?.user?.license
if (license) {
tags[`readFromLicense`] = true
for (const feature of license.features) {
if (!this.isFlagName(feature)) {
continue
}
if (
flagValues[feature] === true ||
specificallySetFalse.has(feature)
) {
// If the flag is already set to through environment variables, we
// don't want to override it back to false here.
continue
}
// @ts-expect-error - TS does not like you writing into a generic type,
// but we know that it's okay in this case because it's just an object.
flagValues[feature] = true
tags[`flags.${feature}.source`] = "license"
}
}
const identity = context.getIdentity()
tags[`identity.type`] = identity?.type
tags[`identity.tenantId`] = identity?.tenantId
tags[`identity._id`] = identity?._id
if (posthog && identity?.type === IdentityType.USER) {
tags[`readFromPostHog`] = true
const personProperties: Record<string, string> = {}
if (identity.tenantId) {
personProperties.tenantId = identity.tenantId
}
const posthogFlags = await posthog.getAllFlagsAndPayloads(
identity._id,
{
personProperties,
}
)
for (const [name, value] of Object.entries(posthogFlags.featureFlags)) {
if (!this.isFlagName(name)) {
// We don't want an unexpected PostHog flag to break the app, so we
// just log it and continue.
console.warn(`Unexpected posthog flag "${name}": ${value}`)
continue
}
if (flagValues[name] === true || specificallySetFalse.has(name)) {
// If the flag is already set to through environment variables, we
// don't want to override it back to false here.
continue
}
const payload = posthogFlags.featureFlagPayloads?.[name]
const flag = this.flagSchema[name]
try {
// @ts-expect-error - TS does not like you writing into a generic
// type, but we know that it's okay in this case because it's just
// an object.
flagValues[name] = flag.parse(payload || value)
tags[`flags.${name}.source`] = "posthog"
} catch (err) {
// We don't want an invalid PostHog flag to break the app, so we just
// log it and continue.
console.warn(`Error parsing posthog flag "${name}": ${value}`, err)
}
}
}
context.setFeatureFlags(this.setId, flagValues)
for (const [key, value] of Object.entries(flagValues)) {
tags[`flags.${key}.value`] = value
}
span?.addTags(tags)
return flagValues
})
}
}
// This is the primary source of truth for feature flags. If you want to add a
// new flag, add it here and use the `fetch` and `get` functions to access it.
// All of the machinery in this file is to make sure that flags have their
// default values set correctly and their types flow through the system.
export const flags = new FlagSet({
DEFAULT_VALUES: Flag.boolean(env.isDev()),
AUTOMATION_BRANCHING: Flag.boolean(env.isDev()),
SQS: Flag.boolean(env.isDev()),
[FeatureFlag.AI_CUSTOM_CONFIGS]: Flag.boolean(env.isDev()),
[FeatureFlag.ENRICHED_RELATIONSHIPS]: Flag.boolean(env.isDev()),
})
export * from "./features"
export * as testutils from "./tests/utils"

64
packages/backend-core/src/features/tests/utils.ts Normal file
View File

@ -0,0 +1,64 @@
import { FeatureFlags, parseEnvFlags } from ".."
import { setEnv } from "../../environment"
function getCurrentFlags(): Record<string, Record<string, boolean>> {
const result: Record<string, Record<string, boolean>> = {}
for (const { tenantId, key, value } of parseEnvFlags(
process.env.TENANT_FEATURE_FLAGS || ""
)) {
const tenantFlags = result[tenantId] || {}
// Don't allow overwriting specifically false flags, to match the beheaviour
// of FlagSet.
if (tenantFlags[key] === false) {
continue
}
tenantFlags[key] = value
result[tenantId] = tenantFlags
}
return result
}
function buildFlagString(
flags: Record<string, Record<string, boolean>>
): string {
const parts: string[] = []
for (const [tenantId, tenantFlags] of Object.entries(flags)) {
for (const [key, value] of Object.entries(tenantFlags)) {
if (value === false) {
parts.push(`${tenantId}:!${key}`)
} else {
parts.push(`${tenantId}:${key}`)
}
}
}
return parts.join(",")
}
export function setFeatureFlags(
tenantId: string,
flags: Partial<FeatureFlags>
): () => void {
const current = getCurrentFlags()
for (const [key, value] of Object.entries(flags)) {
const tenantFlags = current[tenantId] || {}
tenantFlags[key] = value
current[tenantId] = tenantFlags
}
const flagString = buildFlagString(current)
return setEnv({ TENANT_FEATURE_FLAGS: flagString })
}
export function withFeatureFlags<T>(
tenantId: string,
flags: Partial<FeatureFlags>,
f: () => T
) {
const cleanup = setFeatureFlags(tenantId, flags)
const result = f()
if (result instanceof Promise) {
return result.finally(cleanup)
} else {
cleanup()
return result
}
}

24
packages/backend-core/src/middleware/passport/sso/sso.ts
View File

@ -2,7 +2,6 @@ import { generateGlobalUserID } from "../../../db"
import { authError } from "../utils"
import * as users from "../../../users"
import * as context from "../../../context"
import fetch from "node-fetch"
import {
SaveSSOUserFunction,
SSOAuthDetails,
@ -97,28 +96,13 @@ export async function authenticate(
return done(null, ssoUser)
}
async function getProfilePictureUrl(user: User, details: SSOAuthDetails) {
const pictureUrl = details.profile?._json.picture
if (pictureUrl) {
const response = await fetch(pictureUrl)
if (response.status === 200) {
const type = response.headers.get("content-type") as string
if (type.startsWith("image/")) {
return pictureUrl
}
}
}
}
/**
* @returns a user that has been sync'd with third party information
*/
async function syncUser(user: User, details: SSOAuthDetails): Promise<SSOUser> {
let firstName
let lastName
let pictureUrl
let oauth2
let thirdPartyProfile
if (details.profile) {
const profile = details.profile
@ -134,12 +118,6 @@ async function syncUser(user: User, details: SSOAuthDetails): Promise<SSOUser> {
lastName = name.familyName
}
}
pictureUrl = await getProfilePictureUrl(user, details)
thirdPartyProfile = {
...profile._json,
}
}
// oauth tokens for future use
@ -155,8 +133,6 @@ async function syncUser(user: User, details: SSOAuthDetails): Promise<SSOUser> {
providerType: details.providerType,
firstName,
lastName,
thirdPartyProfile,
pictureUrl,
oauth2,
}
}

4
packages/backend-core/tests/core/utilities/structures/accounts.ts
View File

@ -59,10 +59,8 @@ export function ssoAccount(account: Account = cloudAccount()): SSOAccount {
accessToken: generator.string(),
refreshToken: generator.string(),
},
pictureUrl: generator.url(),
provider: provider(),
providerType: providerType(),
thirdPartyProfile: {},
}
}
@ -76,9 +74,7 @@ export function verifiableSsoAccount(
accessToken: generator.string(),
refreshToken: generator.string(),
},
pictureUrl: generator.url(),
provider: AccountSSOProvider.MICROSOFT,
providerType: AccountSSOProviderType.MICROSOFT,
thirdPartyProfile: { id: "abc123" },
}
}

5
packages/backend-core/tests/core/utilities/structures/users.ts
View File

@ -25,7 +25,6 @@ export const user = (userProps?: Partial<Omit<User, "userId">>): User => {
roles: { app_test: "admin" },
firstName: generator.first(),
lastName: generator.last(),
pictureUrl: "http://example.com",
tenantId: tenant.id(),
...userProps,
}
@ -86,9 +85,5 @@ export function ssoUser(
oauth2: opts.details?.oauth2,
provider: opts.details?.provider!,
providerType: opts.details?.providerType!,
thirdPartyProfile: {
email: base.email,
picture: base.pictureUrl,
},
}
}

4
packages/server/src/api/controllers/view/viewsV2.ts
View File

@ -127,13 +127,13 @@ export async function create(ctx: Ctx<CreateViewRequest, ViewResponse>) {
const parsedView: Omit<RequiredKeys<ViewV2>, "id" | "version"> = {
name: view.name,
type: view.type,
tableId: view.tableId,
query: view.query,
queryUI: view.queryUI,
sort: view.sort,
schema,
primaryDisplay: view.primaryDisplay,
uiMetadata: view.uiMetadata,
}
const result = await sdk.views.create(tableId, parsedView)
ctx.status = 201
@ -163,6 +163,7 @@ export async function update(ctx: Ctx<UpdateViewRequest, ViewResponse>) {
const parsedView: RequiredKeys<ViewV2> = {
id: view.id,
name: view.name,
type: view.type,
version: view.version,
tableId: view.tableId,
query: view.query,
@ -170,7 +171,6 @@ export async function update(ctx: Ctx<UpdateViewRequest, ViewResponse>) {
sort: view.sort,
schema,
primaryDisplay: view.primaryDisplay,
uiMetadata: view.uiMetadata,
}
const result = await sdk.views.update(tableId, parsedView)

17
packages/server/src/api/routes/tests/application.spec.ts
View File

@ -14,12 +14,7 @@ jest.mock("../../../utilities/redis", () => ({
import { checkBuilderEndpoint } from "./utilities/TestFunctions"
import * as setup from "./utilities"
import { AppStatus } from "../../../db/utils"
import {
events,
utils,
context,
withEnv as withCoreEnv,
} from "@budibase/backend-core"
import { events, utils, context, features } from "@budibase/backend-core"
import env from "../../../environment"
import { type App } from "@budibase/types"
import tk from "timekeeper"
@ -358,9 +353,13 @@ describe("/applications", () => {
.delete(`/api/global/roles/${prodAppId}`)
.reply(200, {})
await withCoreEnv({ TENANT_FEATURE_FLAGS: "*:SQS" }, async () => {
await config.api.application.delete(app.appId)
})
await features.testutils.withFeatureFlags(
"*",
{ SQS: true },
async () => {
await config.api.application.delete(app.appId)
}
)
})
})

33
packages/server/src/api/routes/tests/row.spec.ts
View File

@ -13,8 +13,7 @@ import {
context,
InternalTable,
tenancy,
withEnv as withCoreEnv,
setEnv as setCoreEnv,
features,
} from "@budibase/backend-core"
import { quotas } from "@budibase/pro"
import {
@ -40,7 +39,6 @@ import {
TableSchema,
JsonFieldSubType,
RowExportFormat,
FeatureFlag,
RelationSchemaField,
} from "@budibase/types"
import { generator, mocks } from "@budibase/backend-core/tests"
@ -98,12 +96,12 @@ describe.each([
let envCleanup: (() => void) | undefined
beforeAll(async () => {
await withCoreEnv({ TENANT_FEATURE_FLAGS: "*:SQS" }, () => config.init())
if (isLucene) {
envCleanup = setCoreEnv({ TENANT_FEATURE_FLAGS: "*:!SQS" })
} else if (isSqs) {
envCleanup = setCoreEnv({ TENANT_FEATURE_FLAGS: "*:SQS" })
}
await features.testutils.withFeatureFlags("*", { SQS: true }, () =>
config.init()
)
envCleanup = features.testutils.setFeatureFlags("*", {
SQS: isSqs,
})
if (dsProvider) {
const rawDatasource = await dsProvider
@ -2517,15 +2515,9 @@ describe.each([
let flagCleanup: (() => void) | undefined
beforeAll(async () => {
const env = {
TENANT_FEATURE_FLAGS: `*:${FeatureFlag.ENRICHED_RELATIONSHIPS}`,
}
if (isSqs) {
env.TENANT_FEATURE_FLAGS = `${env.TENANT_FEATURE_FLAGS},*:SQS`
} else {
env.TENANT_FEATURE_FLAGS = `${env.TENANT_FEATURE_FLAGS},*:!SQS`
}
flagCleanup = setCoreEnv(env)
flagCleanup = features.testutils.setFeatureFlags("*", {
ENRICHED_RELATIONSHIPS: true,
})
const aux2Table = await config.api.table.save(saveTableRequest())
const aux2Data = await config.api.row.save(aux2Table._id!, {})
@ -2752,9 +2744,10 @@ describe.each([
it.each(testScenarios)(
"does not enrich relationships when not enabled (via %s)",
async (__, retrieveDelegate) => {
await withCoreEnv(
await features.testutils.withFeatureFlags(
"*",
{
TENANT_FEATURE_FLAGS: `*:!${FeatureFlag.ENRICHED_RELATIONSHIPS}`,
ENRICHED_RELATIONSHIPS: false,
},
async () => {
const otherRows = _.sampleSize(auxData, 5)

18
packages/server/src/api/routes/tests/search.spec.ts
View File

@ -7,9 +7,9 @@ import {
import {
context,
db as dbCore,
features,
MAX_VALID_DATE,
MIN_VALID_DATE,
setEnv as setCoreEnv,
SQLITE_DESIGN_DOC_ID,
utils,
withEnv as withCoreEnv,
@ -94,16 +94,12 @@ describe.each([
}
beforeAll(async () => {
await withCoreEnv({ TENANT_FEATURE_FLAGS: "*:SQS" }, () => config.init())
if (isLucene) {
envCleanup = setCoreEnv({
TENANT_FEATURE_FLAGS: "*:!SQS",
})
} else if (isSqs) {
envCleanup = setCoreEnv({
TENANT_FEATURE_FLAGS: "*:SQS",
})
}
await features.testutils.withFeatureFlags("*", { SQS: true }, () =>
config.init()
)
envCleanup = features.testutils.setFeatureFlags("*", {
SQS: isSqs,
})
if (config.app?.appId) {
config.app = await config.api.application.update(config.app?.appId, {

69
packages/server/src/api/routes/tests/templates.spec.ts
View File

@ -2,7 +2,7 @@ import * as setup from "./utilities"
import path from "path"
import nock from "nock"
import { generator } from "@budibase/backend-core/tests"
import { withEnv as withCoreEnv, env as coreEnv } from "@budibase/backend-core"
import { features } from "@budibase/backend-core"
interface App {
background: string
@ -85,41 +85,44 @@ describe("/templates", () => {
it.each(["sqs", "lucene"])(
`should be able to create an app from a template (%s)`,
async source => {
const env: Partial<typeof coreEnv> = {
TENANT_FEATURE_FLAGS: source === "sqs" ? "*:SQS" : "",
}
await features.testutils.withFeatureFlags(
"*",
{ SQS: source === "sqs" },
async () => {
const name = generator.guid().replaceAll("-", "")
const url = `/${name}`
await withCoreEnv(env, async () => {
const name = generator.guid().replaceAll("-", "")
const url = `/${name}`
const app = await config.api.application.create({
name,
url,
useTemplate: "true",
templateName: "Agency Client Portal",
templateKey: "app/agency-client-portal",
})
expect(app.name).toBe(name)
expect(app.url).toBe(url)
await config.withApp(app, async () => {
const tables = await config.api.table.fetch()
expect(tables).toHaveLength(2)
tables.sort((a, b) => a.name.localeCompare(b.name))
const [agencyProjects, users] = tables
expect(agencyProjects.name).toBe("Agency Projects")
expect(users.name).toBe("Users")
const { rows } = await config.api.row.search(agencyProjects._id!, {
tableId: agencyProjects._id!,
query: {},
const app = await config.api.application.create({
name,
url,
useTemplate: "true",
templateName: "Agency Client Portal",
templateKey: "app/agency-client-portal",
})
expect(app.name).toBe(name)
expect(app.url).toBe(url)
expect(rows).toHaveLength(3)
})
})
await config.withApp(app, async () => {
const tables = await config.api.table.fetch()
expect(tables).toHaveLength(2)
tables.sort((a, b) => a.name.localeCompare(b.name))
const [agencyProjects, users] = tables
expect(agencyProjects.name).toBe("Agency Projects")
expect(users.name).toBe("Users")
const { rows } = await config.api.row.search(
agencyProjects._id!,
{
tableId: agencyProjects._id!,
query: {},
}
)
expect(rows).toHaveLength(3)
})
}
)
}
)
})

106
packages/server/src/api/routes/tests/viewV2.spec.ts
View File

@ -22,7 +22,6 @@ import {
RelationshipType,
TableSchema,
RenameColumn,
FeatureFlag,
BBReferenceFieldSubType,
NumericCalculationFieldMetadata,
ViewV2Schema,
@ -33,13 +32,7 @@ import { generator, mocks } from "@budibase/backend-core/tests"
import { DatabaseName, getDatasource } from "../../../integrations/tests/utils"
import merge from "lodash/merge"
import { quotas } from "@budibase/pro"
import {
db,
roles,
withEnv as withCoreEnv,
setEnv as setCoreEnv,
env,
} from "@budibase/backend-core"
import { db, roles, features } from "@budibase/backend-core"
describe.each([
["lucene", undefined],
@ -104,18 +97,13 @@ describe.each([
}
beforeAll(async () => {
await withCoreEnv({ TENANT_FEATURE_FLAGS: isSqs ? "*:SQS" : "" }, () =>
await features.testutils.withFeatureFlags("*", { SQS: isSqs }, () =>
config.init()
)
if (isLucene) {
envCleanup = setCoreEnv({
TENANT_FEATURE_FLAGS: "*:!SQS",
})
} else if (isSqs) {
envCleanup = setCoreEnv({
TENANT_FEATURE_FLAGS: "*:SQS",
})
}
envCleanup = features.testutils.setFeatureFlags("*", {
SQS: isSqs,
})
if (dsProvider) {
datasource = await config.createDatasource({
@ -157,7 +145,7 @@ describe.each([
})
it("can persist views with all fields", async () => {
const newView: Required<Omit<CreateViewRequest, "queryUI">> = {
const newView: Required<Omit<CreateViewRequest, "queryUI" | "type">> = {
name: generator.name(),
tableId: table._id!,
primaryDisplay: "id",
@ -179,9 +167,6 @@ describe.each([
visible: true,
},
},
uiMetadata: {
foo: "bar",
},
}
const res = await config.api.viewV2.create(newView)
@ -551,6 +536,7 @@ describe.each([
let view = await config.api.viewV2.create({
tableId: table._id!,
name: generator.guid(),
type: ViewV2Type.CALCULATION,
schema: {
sum: {
visible: true,
@ -574,11 +560,35 @@ describe.each([
expect(sum.field).toEqual("Price")
})
it("cannot create a view with calculation fields unless it has the right type", async () => {
await config.api.viewV2.create(
{
tableId: table._id!,
name: generator.guid(),
schema: {
sum: {
visible: true,
calculationType: CalculationType.SUM,
field: "Price",
},
},
},
{
status: 400,
body: {
message:
"Calculation fields are not allowed in non-calculation views",
},
}
)
})
it("cannot create a calculation view with more than 5 aggregations", async () => {
await config.api.viewV2.create(
{
tableId: table._id!,
name: generator.guid(),
type: ViewV2Type.CALCULATION,
schema: {
sum: {
visible: true,
@ -626,6 +636,7 @@ describe.each([
{
tableId: table._id!,
name: generator.guid(),
type: ViewV2Type.CALCULATION,
schema: {
sum: {
visible: true,
@ -654,6 +665,7 @@ describe.each([
{
tableId: table._id!,
name: generator.guid(),
type: ViewV2Type.CALCULATION,
schema: {
count: {
visible: true,
@ -680,6 +692,7 @@ describe.each([
{
tableId: table._id!,
name: generator.guid(),
type: ViewV2Type.CALCULATION,
schema: {
count: {
visible: true,
@ -709,6 +722,7 @@ describe.each([
await config.api.viewV2.create({
tableId: table._id!,
name: generator.guid(),
type: ViewV2Type.CALCULATION,
schema: {
count: {
visible: true,
@ -795,7 +809,9 @@ describe.each([
it("can update all fields", async () => {
const tableId = table._id!
const updatedData: Required<Omit<UpdateViewRequest, "queryUI">> = {
const updatedData: Required<
Omit<UpdateViewRequest, "queryUI" | "type">
> = {
version: view.version,
id: view.id,
tableId,
@ -823,9 +839,6 @@ describe.each([
readonly: true,
},
},
uiMetadata: {
foo: "bar",
},
}
await config.api.viewV2.update(updatedData)
@ -1010,6 +1023,32 @@ describe.each([
)
})
it("cannot update view type after creation", async () => {
const view = await config.api.viewV2.create({
tableId: table._id!,
name: generator.guid(),
schema: {
id: { visible: true },
Price: {
visible: true,
},
},
})
await config.api.viewV2.update(
{
...view,
type: ViewV2Type.CALCULATION,
},
{
status: 400,
body: {
message: "Cannot update view type after creation",
},
}
)
})
isInternal &&
it("updating schema will only validate modified field", async () => {
let view = await config.api.viewV2.create({
@ -1082,6 +1121,7 @@ describe.each([
view = await config.api.viewV2.create({
tableId: table._id!,
name: generator.guid(),
type: ViewV2Type.CALCULATION,
schema: {
country: {
visible: true,
@ -2643,12 +2683,8 @@ describe.each([
describe("foreign relationship columns", () => {
let envCleanup: () => void
beforeAll(() => {
const flags = [`*:${FeatureFlag.ENRICHED_RELATIONSHIPS}`]
if (env.TENANT_FEATURE_FLAGS) {
flags.push(...env.TENANT_FEATURE_FLAGS.split(","))
}
envCleanup = setCoreEnv({
TENANT_FEATURE_FLAGS: flags.join(","),
envCleanup = features.testutils.setFeatureFlags("*", {
ENRICHED_RELATIONSHIPS: true,
})
})
@ -2839,6 +2875,7 @@ describe.each([
it("should be able to search by calculations", async () => {
const view = await config.api.viewV2.create({
tableId: table._id!,
type: ViewV2Type.CALCULATION,
name: generator.guid(),
schema: {
"Quantity Sum": {
@ -2873,6 +2910,7 @@ describe.each([
const view = await config.api.viewV2.create({
tableId: table._id!,
name: generator.guid(),
type: ViewV2Type.CALCULATION,
schema: {
quantity: {
visible: true,
@ -2911,6 +2949,7 @@ describe.each([
const view = await config.api.viewV2.create({
tableId: table._id!,
name: generator.guid(),
type: ViewV2Type.CALCULATION,
schema: {
aggregate: {
visible: true,
@ -2971,6 +3010,7 @@ describe.each([
const view = await config.api.viewV2.create({
tableId: table._id!,
name: generator.guid(),
type: ViewV2Type.CALCULATION,
schema: {
count: {
visible: true,
@ -3005,6 +3045,7 @@ describe.each([
{
tableId: table._id!,
name: generator.guid(),
type: ViewV2Type.CALCULATION,
schema: {
count: {
visible: true,
@ -3053,6 +3094,7 @@ describe.each([
const view = await config.api.viewV2.create({
tableId: table._id!,
name: generator.guid(),
type: ViewV2Type.CALCULATION,
schema: {
sum: {
visible: true,

6
packages/server/src/appMigrations/migrations/tests/20240604153647_initial_sqs.spec.ts
View File

@ -2,8 +2,8 @@ import * as setup from "../../../api/routes/tests/utilities"
import { basicTable } from "../../../tests/utilities/structures"
import {
db as dbCore,
features,
SQLITE_DESIGN_DOC_ID,
withEnv as withCoreEnv,
} from "@budibase/backend-core"
import {
LinkDocument,
@ -71,11 +71,11 @@ function oldLinkDocument(): Omit<LinkDocument, "tableId"> {
}
async function sqsDisabled(cb: () => Promise<void>) {
await withCoreEnv({ TENANT_FEATURE_FLAGS: "*:!SQS" }, cb)
await features.testutils.withFeatureFlags("*", { SQS: false }, cb)
}
async function sqsEnabled(cb: () => Promise<void>) {
await withCoreEnv({ TENANT_FEATURE_FLAGS: "*:SQS" }, cb)
await features.testutils.withFeatureFlags("*", { SQS: true }, cb)
}
describe("SQS migration", () => {

19
packages/server/src/sdk/app/rows/search/tests/search.spec.ts
View File

@ -10,10 +10,7 @@ import {
import TestConfiguration from "../../../../../tests/utilities/TestConfiguration"
import { search } from "../../../../../sdk/app/rows/search"
import { generator } from "@budibase/backend-core/tests"
import {
withEnv as withCoreEnv,
setEnv as setCoreEnv,
} from "@budibase/backend-core"
import { features } from "@budibase/backend-core"
import {
DatabaseName,
getDatasource,
@ -41,19 +38,13 @@ describe.each([
let table: Table
beforeAll(async () => {
await withCoreEnv({ TENANT_FEATURE_FLAGS: isSqs ? "*:SQS" : "" }, () =>
await features.testutils.withFeatureFlags("*", { SQS: isSqs }, () =>
config.init()
)
if (isLucene) {
envCleanup = setCoreEnv({
TENANT_FEATURE_FLAGS: "*:!SQS",
})
} else if (isSqs) {
envCleanup = setCoreEnv({
TENANT_FEATURE_FLAGS: "*:SQS",
})
}
envCleanup = features.testutils.setFeatureFlags("*", {
SQS: isSqs,
})
if (dsProvider) {
datasource = await config.createDatasource({

3
packages/server/src/sdk/app/views/external.ts
View File

@ -70,6 +70,9 @@ export async function update(tableId: string, view: ViewV2): Promise<ViewV2> {
if (!existingView || !existingView.name) {
throw new HTTPError(`View ${view.id} not found in table ${tableId}`, 404)
}
if (isV2(existingView) && existingView.type !== view.type) {
throw new HTTPError(`Cannot update view type after creation`, 400)
}
delete views[existingView.name]
views[view.name] = view

7
packages/server/src/sdk/app/views/index.ts
View File

@ -132,6 +132,13 @@ async function guardViewSchema(
if (helpers.views.isCalculationView(view)) {
await guardCalculationViewSchema(table, view)
} else {
if (helpers.views.hasCalculationFields(view)) {
throw new HTTPError(
"Calculation fields are not allowed in non-calculation views",
400
)
}
}
await checkReadonlyFields(table, view)

4
packages/server/src/sdk/app/views/internal.ts
View File

@ -59,6 +59,10 @@ export async function update(tableId: string, view: ViewV2): Promise<ViewV2> {
throw new HTTPError(`View ${view.id} not found in table ${tableId}`, 404)
}
if (isV2(existingView) && existingView.type !== view.type) {
throw new HTTPError(`Cannot update view type after creation`, 400)
}
delete table.views[existingView.name]
table.views[view.name] = view
await db.put(table)

8
packages/server/src/utilities/rowProcessor/tests/outputProcessing.spec.ts
View File

@ -8,7 +8,7 @@ import {
} from "@budibase/types"
import { outputProcessing } from ".."
import { generator, structures } from "@budibase/backend-core/tests"
import { setEnv as setCoreEnv } from "@budibase/backend-core"
import { features } from "@budibase/backend-core"
import * as bbReferenceProcessor from "../bbReferenceProcessor"
import TestConfiguration from "../../../tests/utilities/TestConfiguration"
@ -21,7 +21,7 @@ jest.mock("../bbReferenceProcessor", (): typeof bbReferenceProcessor => ({
describe("rowProcessor - outputProcessing", () => {
const config = new TestConfiguration()
let cleanupEnv: () => void = () => {}
let cleanupFlags: () => void = () => {}
beforeAll(async () => {
await config.init()
@ -33,11 +33,11 @@ describe("rowProcessor - outputProcessing", () => {
beforeEach(() => {
jest.resetAllMocks()
cleanupEnv = setCoreEnv({ TENANT_FEATURE_FLAGS: "*SQS" })
cleanupFlags = features.testutils.setFeatureFlags("*", { SQS: true })
})
afterEach(() => {
cleanupEnv()
cleanupFlags()
})
const processOutputBBReferenceMock =

5
packages/shared-core/src/helpers/views.ts
View File

@ -3,6 +3,7 @@ import {
ViewCalculationFieldMetadata,
ViewFieldMetadata,
ViewV2,
ViewV2Type,
} from "@budibase/types"
import { pickBy } from "lodash"
@ -21,6 +22,10 @@ export function isBasicViewField(
type UnsavedViewV2 = Omit<ViewV2, "id" | "version">
export function isCalculationView(view: UnsavedViewV2) {
return view.type === ViewV2Type.CALCULATION
}
export function hasCalculationFields(view: UnsavedViewV2) {
return Object.values(view.schema || {}).some(isCalculationField)
}

1
packages/types/src/api/account/accounts.ts
View File

@ -12,7 +12,6 @@ export interface CreateAccountRequest {
name?: string
password: string
provider?: AccountSSOProvider
thirdPartyProfile: object
}
export interface SearchAccountsRequest {

2
packages/types/src/documents/account/account.ts
View File

@ -98,8 +98,6 @@ export interface AccountSSO {
provider: AccountSSOProvider
providerType: AccountSSOProviderType
oauth2?: OAuthTokens
pictureUrl?: string
thirdPartyProfile: any // TODO: define what the google profile looks like
}
export type SSOAccount = (Account | CloudAccount) & AccountSSO

6
packages/types/src/documents/app/view.ts
View File

@ -79,10 +79,15 @@ export enum CalculationType {
MAX = "max",
}
export enum ViewV2Type {
CALCULATION = "calculation",
}
export interface ViewV2 {
version: 2
id: string
name: string
type?: ViewV2Type
primaryDisplay?: string
tableId: string
query?: LegacyFilter[] | SearchFilters
@ -94,7 +99,6 @@ export interface ViewV2 {
type?: SortType
}
schema?: ViewV2Schema
uiMetadata?: Record<string, any>
}
export type ViewV2Schema = Record<string, ViewFieldMetadata>

2
packages/types/src/documents/global/user.ts
View File

@ -21,7 +21,6 @@ export interface UserSSO {
provider: string // the individual provider e.g. Okta, Auth0, Google
providerType: SSOProviderType
oauth2?: OAuth2
thirdPartyProfile?: SSOProfileJson
profile?: {
displayName?: string
name?: {
@ -45,7 +44,6 @@ export interface User extends Document {
userId?: string
firstName?: string
lastName?: string
pictureUrl?: string
forceResetPassword?: boolean
roles: UserRoles
builder?: {

10
packages/worker/src/api/routes/global/tests/auditLogs.spec.ts
View File

@ -1,5 +1,5 @@
import { mocks, structures } from "@budibase/backend-core/tests"
import { context, events, setEnv as setCoreEnv } from "@budibase/backend-core"
import { context, events, features } from "@budibase/backend-core"
import { Event, IdentityType } from "@budibase/types"
import { TestConfiguration } from "../../../../tests"
@ -17,11 +17,9 @@ describe.each(["lucene", "sql"])("/api/global/auditlogs (%s)", method => {
let envCleanup: (() => void) | undefined
beforeAll(async () => {
if (method === "lucene") {
envCleanup = setCoreEnv({ TENANT_FEATURE_FLAGS: "*:!SQS" })
} else if (method === "sql") {
envCleanup = setCoreEnv({ TENANT_FEATURE_FLAGS: "*:SQS" })
}
envCleanup = features.testutils.setFeatureFlags("*", {
SQS: method === "sql",
})
await config.beforeAll()
})