MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into fix/related-rows-dont-exist

This commit is contained in:
Michael Drury 2024-10-09 12:18:37 +01:00 committed by GitHub
parent 11dc2766cc abd7b3b84e
commit 64492dca2a
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
55 changed files with 1615 additions and 668 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.github/workflows/deploy-featurebranch.yml vendored
View File

@ -23,7 +23,6 @@ jobs:
PAYLOAD_BRANCH: ${{ github.head_ref }}
PAYLOAD_PR_NUMBER: ${{ github.event.pull_request.number }}
PAYLOAD_LICENSE_TYPE: "free"
PAYLOAD_DEPLOY: true
with:
repository: budibase/budibase-deploys
event: featurebranch-qa-deploy

4
charts/budibase/templates/app-service-deployment.yaml
View File

@ -184,6 +184,10 @@ spec:
- name: NODE_DEBUG
value: {{ .Values.services.apps.nodeDebug | quote }}
{{ end }}
{{ if .Values.services.apps.xssSafeMode }}
- name: XSS_SAFE_MODE
value: {{ .Values.services.apps.xssSafeMode | quote }}
{{ end }}
{{ if .Values.globals.datadogApmEnabled }}
- name: DD_LOGS_INJECTION
value: {{ .Values.globals.datadogApmEnabled | quote }}

2
lerna.json
View File

@ -1,6 +1,6 @@
{
"$schema": "node_modules/lerna/schemas/lerna-schema.json",
"version": "2.32.11",
"version": "2.32.13",
"npmClient": "yarn",
"packages": [
"packages/*",

2
packages/account-portal

@ -1 +1 @@
Subproject commit 3e24f6293ff5ee5f9b42822e001504e3bbf19cc0
Subproject commit 8cd052ce8288f343812a514d06c5a9459b3ba1a8

14
packages/backend-core/src/db/couch/DatabaseImpl.ts
View File

@ -375,11 +375,21 @@ export class DatabaseImpl implements Database {
return this.performCall(() => {
return async () => {
const response = await directCouchUrlCall(args)
const json = await response.json()
const text = await response.text()
if (response.status > 300) {
let json
try {
json = JSON.parse(text)
} catch (err) {
console.error(`SQS error: ${text}`)
throw new CouchDBError(
"error while running SQS query, please try again later",
{ name: "sqs_error", status: response.status }
)
}
throw json
}
return json as T
return JSON.parse(text) as T
}
})
}

300
packages/backend-core/src/features/features.ts Normal file
View File

@ -0,0 +1,300 @@
import env from "../environment"
import * as context from "../context"
import { PostHog, PostHogOptions } from "posthog-node"
import { FeatureFlag, IdentityType, UserCtx } from "@budibase/types"
import tracer from "dd-trace"
import { Duration } from "../utils"
let posthog: PostHog | undefined
export function init(opts?: PostHogOptions) {
if (
env.POSTHOG_TOKEN &&
env.POSTHOG_API_HOST &&
!env.SELF_HOSTED &&
env.POSTHOG_FEATURE_FLAGS_ENABLED
) {
console.log("initializing posthog client...")
posthog = new PostHog(env.POSTHOG_TOKEN, {
host: env.POSTHOG_API_HOST,
personalApiKey: env.POSTHOG_PERSONAL_TOKEN,
featureFlagsPollingInterval: Duration.fromMinutes(3).toMs(),
...opts,
})
} else {
console.log("posthog disabled")
}
}
export function shutdown() {
posthog?.shutdown()
}
export abstract class Flag<T> {
static boolean(defaultValue: boolean): Flag<boolean> {
return new BooleanFlag(defaultValue)
}
static string(defaultValue: string): Flag<string> {
return new StringFlag(defaultValue)
}
static number(defaultValue: number): Flag<number> {
return new NumberFlag(defaultValue)
}
protected constructor(public defaultValue: T) {}
abstract parse(value: any): T
}
type UnwrapFlag<F> = F extends Flag<infer U> ? U : never
export type FlagValues<T> = {
[K in keyof T]: UnwrapFlag<T[K]>
}
type KeysOfType<T, U> = {
[K in keyof T]: T[K] extends Flag<U> ? K : never
}[keyof T]
class BooleanFlag extends Flag<boolean> {
parse(value: any) {
if (typeof value === "string") {
return ["true", "t", "1"].includes(value.toLowerCase())
}
if (typeof value === "boolean") {
return value
}
throw new Error(`could not parse value "${value}" as boolean`)
}
}
class StringFlag extends Flag<string> {
parse(value: any) {
if (typeof value === "string") {
return value
}
throw new Error(`could not parse value "${value}" as string`)
}
}
class NumberFlag extends Flag<number> {
parse(value: any) {
if (typeof value === "number") {
return value
}
if (typeof value === "string") {
const parsed = parseFloat(value)
if (!isNaN(parsed)) {
return parsed
}
}
throw new Error(`could not parse value "${value}" as number`)
}
}
export interface EnvFlagEntry {
tenantId: string
key: string
value: boolean
}
export function parseEnvFlags(flags: string): EnvFlagEntry[] {
const split = flags.split(",").map(x => x.split(":"))
const result: EnvFlagEntry[] = []
for (const [tenantId, ...features] of split) {
for (let feature of features) {
let value = true
if (feature.startsWith("!")) {
feature = feature.slice(1)
value = false
}
result.push({ tenantId, key: feature, value })
}
}
return result
}
export class FlagSet<V extends Flag<any>, T extends { [key: string]: V }> {
// This is used to safely cache flags sets in the current request context.
// Because multiple sets could theoretically exist, we don't want the cache of
// one to leak into another.
private readonly setId: string
constructor(private readonly flagSchema: T) {
this.setId = crypto.randomUUID()
}
defaults(): FlagValues<T> {
return Object.keys(this.flagSchema).reduce((acc, key) => {
const typedKey = key as keyof T
acc[typedKey] = this.flagSchema[key].defaultValue
return acc
}, {} as FlagValues<T>)
}
isFlagName(name: string | number | symbol): name is keyof T {
return this.flagSchema[name as keyof T] !== undefined
}
async get<K extends keyof T>(
key: K,
ctx?: UserCtx
): Promise<FlagValues<T>[K]> {
const flags = await this.fetch(ctx)
return flags[key]
}
async isEnabled<K extends KeysOfType<T, boolean>>(
key: K,
ctx?: UserCtx
): Promise<boolean> {
const flags = await this.fetch(ctx)
return flags[key]
}
async fetch(ctx?: UserCtx): Promise<FlagValues<T>> {
return await tracer.trace("features.fetch", async span => {
const cachedFlags = context.getFeatureFlags<FlagValues<T>>(this.setId)
if (cachedFlags) {
span?.addTags({ fromCache: true })
return cachedFlags
}
const tags: Record<string, any> = {}
const flagValues = this.defaults()
const currentTenantId = context.getTenantId()
const specificallySetFalse = new Set<string>()
for (const { tenantId, key, value } of parseEnvFlags(
env.TENANT_FEATURE_FLAGS || ""
)) {
if (!tenantId || (tenantId !== "*" && tenantId !== currentTenantId)) {
continue
}
tags[`readFromEnvironmentVars`] = true
if (value === false) {
specificallySetFalse.add(key)
}
// ignore unknown flags
if (!this.isFlagName(key)) {
continue
}
if (typeof flagValues[key] !== "boolean") {
throw new Error(`Feature: ${key} is not a boolean`)
}
// @ts-expect-error - TS does not like you writing into a generic type,
// but we know that it's okay in this case because it's just an object.
flagValues[key as keyof FlagValues] = value
tags[`flags.${key}.source`] = "environment"
}
const license = ctx?.user?.license
if (license) {
tags[`readFromLicense`] = true
for (const feature of license.features) {
if (!this.isFlagName(feature)) {
continue
}
if (
flagValues[feature] === true ||
specificallySetFalse.has(feature)
) {
// If the flag is already set to through environment variables, we
// don't want to override it back to false here.
continue
}
// @ts-expect-error - TS does not like you writing into a generic type,
// but we know that it's okay in this case because it's just an object.
flagValues[feature] = true
tags[`flags.${feature}.source`] = "license"
}
}
const identity = context.getIdentity()
tags[`identity.type`] = identity?.type
tags[`identity.tenantId`] = identity?.tenantId
tags[`identity._id`] = identity?._id
if (posthog && identity?.type === IdentityType.USER) {
tags[`readFromPostHog`] = true
const personProperties: Record<string, string> = {}
if (identity.tenantId) {
personProperties.tenantId = identity.tenantId
}
const posthogFlags = await posthog.getAllFlagsAndPayloads(
identity._id,
{
personProperties,
}
)
for (const [name, value] of Object.entries(posthogFlags.featureFlags)) {
if (!this.isFlagName(name)) {
// We don't want an unexpected PostHog flag to break the app, so we
// just log it and continue.
console.warn(`Unexpected posthog flag "${name}": ${value}`)
continue
}
if (flagValues[name] === true || specificallySetFalse.has(name)) {
// If the flag is already set to through environment variables, we
// don't want to override it back to false here.
continue
}
const payload = posthogFlags.featureFlagPayloads?.[name]
const flag = this.flagSchema[name]
try {
// @ts-expect-error - TS does not like you writing into a generic
// type, but we know that it's okay in this case because it's just
// an object.
flagValues[name] = flag.parse(payload || value)
tags[`flags.${name}.source`] = "posthog"
} catch (err) {
// We don't want an invalid PostHog flag to break the app, so we just
// log it and continue.
console.warn(`Error parsing posthog flag "${name}": ${value}`, err)
}
}
}
context.setFeatureFlags(this.setId, flagValues)
for (const [key, value] of Object.entries(flagValues)) {
tags[`flags.${key}.value`] = value
}
span?.addTags(tags)
return flagValues
})
}
}
// This is the primary source of truth for feature flags. If you want to add a
// new flag, add it here and use the `fetch` and `get` functions to access it.
// All of the machinery in this file is to make sure that flags have their
// default values set correctly and their types flow through the system.
export const flags = new FlagSet({
DEFAULT_VALUES: Flag.boolean(env.isDev()),
AUTOMATION_BRANCHING: Flag.boolean(env.isDev()),
SQS: Flag.boolean(env.isDev()),
[FeatureFlag.AI_CUSTOM_CONFIGS]: Flag.boolean(env.isDev()),
[FeatureFlag.ENRICHED_RELATIONSHIPS]: Flag.boolean(env.isDev()),
})
type UnwrapPromise<T> = T extends Promise<infer U> ? U : T
export type FeatureFlags = UnwrapPromise<ReturnType<typeof flags.fetch>>

283
packages/backend-core/src/features/index.ts
View File

@ -1,281 +1,2 @@
import env from "../environment"
import * as context from "../context"
import { PostHog, PostHogOptions } from "posthog-node"
import { FeatureFlag, IdentityType, UserCtx } from "@budibase/types"
import tracer from "dd-trace"
import { Duration } from "../utils"
let posthog: PostHog | undefined
export function init(opts?: PostHogOptions) {
if (
env.POSTHOG_TOKEN &&
env.POSTHOG_API_HOST &&
!env.SELF_HOSTED &&
env.POSTHOG_FEATURE_FLAGS_ENABLED
) {
console.log("initializing posthog client...")
posthog = new PostHog(env.POSTHOG_TOKEN, {
host: env.POSTHOG_API_HOST,
personalApiKey: env.POSTHOG_PERSONAL_TOKEN,
featureFlagsPollingInterval: Duration.fromMinutes(3).toMs(),
...opts,
})
} else {
console.log("posthog disabled")
}
}
export function shutdown() {
posthog?.shutdown()
}
export abstract class Flag<T> {
static boolean(defaultValue: boolean): Flag<boolean> {
return new BooleanFlag(defaultValue)
}
static string(defaultValue: string): Flag<string> {
return new StringFlag(defaultValue)
}
static number(defaultValue: number): Flag<number> {
return new NumberFlag(defaultValue)
}
protected constructor(public defaultValue: T) {}
abstract parse(value: any): T
}
type UnwrapFlag<F> = F extends Flag<infer U> ? U : never
export type FlagValues<T> = {
[K in keyof T]: UnwrapFlag<T[K]>
}
type KeysOfType<T, U> = {
[K in keyof T]: T[K] extends Flag<U> ? K : never
}[keyof T]
class BooleanFlag extends Flag<boolean> {
parse(value: any) {
if (typeof value === "string") {
return ["true", "t", "1"].includes(value.toLowerCase())
}
if (typeof value === "boolean") {
return value
}
throw new Error(`could not parse value "${value}" as boolean`)
}
}
class StringFlag extends Flag<string> {
parse(value: any) {
if (typeof value === "string") {
return value
}
throw new Error(`could not parse value "${value}" as string`)
}
}
class NumberFlag extends Flag<number> {
parse(value: any) {
if (typeof value === "number") {
return value
}
if (typeof value === "string") {
const parsed = parseFloat(value)
if (!isNaN(parsed)) {
return parsed
}
}
throw new Error(`could not parse value "${value}" as number`)
}
}
export class FlagSet<V extends Flag<any>, T extends { [key: string]: V }> {
// This is used to safely cache flags sets in the current request context.
// Because multiple sets could theoretically exist, we don't want the cache of
// one to leak into another.
private readonly setId: string
constructor(private readonly flagSchema: T) {
this.setId = crypto.randomUUID()
}
defaults(): FlagValues<T> {
return Object.keys(this.flagSchema).reduce((acc, key) => {
const typedKey = key as keyof T
acc[typedKey] = this.flagSchema[key].defaultValue
return acc
}, {} as FlagValues<T>)
}
isFlagName(name: string | number | symbol): name is keyof T {
return this.flagSchema[name as keyof T] !== undefined
}
async get<K extends keyof T>(
key: K,
ctx?: UserCtx
): Promise<FlagValues<T>[K]> {
const flags = await this.fetch(ctx)
return flags[key]
}
async isEnabled<K extends KeysOfType<T, boolean>>(
key: K,
ctx?: UserCtx
): Promise<boolean> {
const flags = await this.fetch(ctx)
return flags[key]
}
async fetch(ctx?: UserCtx): Promise<FlagValues<T>> {
return await tracer.trace("features.fetch", async span => {
const cachedFlags = context.getFeatureFlags<FlagValues<T>>(this.setId)
if (cachedFlags) {
span?.addTags({ fromCache: true })
return cachedFlags
}
const tags: Record<string, any> = {}
const flagValues = this.defaults()
const currentTenantId = context.getTenantId()
const specificallySetFalse = new Set<string>()
const split = (env.TENANT_FEATURE_FLAGS || "")
.split(",")
.map(x => x.split(":"))
for (const [tenantId, ...features] of split) {
if (!tenantId || (tenantId !== "*" && tenantId !== currentTenantId)) {
continue
}
tags[`readFromEnvironmentVars`] = true
for (let feature of features) {
let value = true
if (feature.startsWith("!")) {
feature = feature.slice(1)
value = false
specificallySetFalse.add(feature)
}
// ignore unknown flags
if (!this.isFlagName(feature)) {
continue
}
if (typeof flagValues[feature] !== "boolean") {
throw new Error(`Feature: ${feature} is not a boolean`)
}
// @ts-expect-error - TS does not like you writing into a generic type,
// but we know that it's okay in this case because it's just an object.
flagValues[feature as keyof FlagValues] = value
tags[`flags.${feature}.source`] = "environment"
}
}
const license = ctx?.user?.license
if (license) {
tags[`readFromLicense`] = true
for (const feature of license.features) {
if (!this.isFlagName(feature)) {
continue
}
if (
flagValues[feature] === true ||
specificallySetFalse.has(feature)
) {
// If the flag is already set to through environment variables, we
// don't want to override it back to false here.
continue
}
// @ts-expect-error - TS does not like you writing into a generic type,
// but we know that it's okay in this case because it's just an object.
flagValues[feature] = true
tags[`flags.${feature}.source`] = "license"
}
}
const identity = context.getIdentity()
tags[`identity.type`] = identity?.type
tags[`identity.tenantId`] = identity?.tenantId
tags[`identity._id`] = identity?._id
if (posthog && identity?.type === IdentityType.USER) {
tags[`readFromPostHog`] = true
const personProperties: Record<string, string> = {}
if (identity.tenantId) {
personProperties.tenantId = identity.tenantId
}
const posthogFlags = await posthog.getAllFlagsAndPayloads(
identity._id,
{
personProperties,
}
)
for (const [name, value] of Object.entries(posthogFlags.featureFlags)) {
if (!this.isFlagName(name)) {
// We don't want an unexpected PostHog flag to break the app, so we
// just log it and continue.
console.warn(`Unexpected posthog flag "${name}": ${value}`)
continue
}
if (flagValues[name] === true || specificallySetFalse.has(name)) {
// If the flag is already set to through environment variables, we
// don't want to override it back to false here.
continue
}
const payload = posthogFlags.featureFlagPayloads?.[name]
const flag = this.flagSchema[name]
try {
// @ts-expect-error - TS does not like you writing into a generic
// type, but we know that it's okay in this case because it's just
// an object.
flagValues[name] = flag.parse(payload || value)
tags[`flags.${name}.source`] = "posthog"
} catch (err) {
// We don't want an invalid PostHog flag to break the app, so we just
// log it and continue.
console.warn(`Error parsing posthog flag "${name}": ${value}`, err)
}
}
}
context.setFeatureFlags(this.setId, flagValues)
for (const [key, value] of Object.entries(flagValues)) {
tags[`flags.${key}.value`] = value
}
span?.addTags(tags)
return flagValues
})
}
}
// This is the primary source of truth for feature flags. If you want to add a
// new flag, add it here and use the `fetch` and `get` functions to access it.
// All of the machinery in this file is to make sure that flags have their
// default values set correctly and their types flow through the system.
export const flags = new FlagSet({
DEFAULT_VALUES: Flag.boolean(env.isDev()),
AUTOMATION_BRANCHING: Flag.boolean(env.isDev()),
SQS: Flag.boolean(env.isDev()),
[FeatureFlag.AI_CUSTOM_CONFIGS]: Flag.boolean(env.isDev()),
[FeatureFlag.ENRICHED_RELATIONSHIPS]: Flag.boolean(env.isDev()),
})
export * from "./features"
export * as testutils from "./tests/utils"

64
packages/backend-core/src/features/tests/utils.ts Normal file
View File

@ -0,0 +1,64 @@
import { FeatureFlags, parseEnvFlags } from ".."
import { setEnv } from "../../environment"
function getCurrentFlags(): Record<string, Record<string, boolean>> {
const result: Record<string, Record<string, boolean>> = {}
for (const { tenantId, key, value } of parseEnvFlags(
process.env.TENANT_FEATURE_FLAGS || ""
)) {
const tenantFlags = result[tenantId] || {}
// Don't allow overwriting specifically false flags, to match the beheaviour
// of FlagSet.
if (tenantFlags[key] === false) {
continue
}
tenantFlags[key] = value
result[tenantId] = tenantFlags
}
return result
}
function buildFlagString(
flags: Record<string, Record<string, boolean>>
): string {
const parts: string[] = []
for (const [tenantId, tenantFlags] of Object.entries(flags)) {
for (const [key, value] of Object.entries(tenantFlags)) {
if (value === false) {
parts.push(`${tenantId}:!${key}`)
} else {
parts.push(`${tenantId}:${key}`)
}
}
}
return parts.join(",")
}
export function setFeatureFlags(
tenantId: string,
flags: Partial<FeatureFlags>
): () => void {
const current = getCurrentFlags()
for (const [key, value] of Object.entries(flags)) {
const tenantFlags = current[tenantId] || {}
tenantFlags[key] = value
current[tenantId] = tenantFlags
}
const flagString = buildFlagString(current)
return setEnv({ TENANT_FEATURE_FLAGS: flagString })
}
export function withFeatureFlags<T>(
tenantId: string,
flags: Partial<FeatureFlags>,
f: () => T
) {
const cleanup = setFeatureFlags(tenantId, flags)
const result = f()
if (result instanceof Promise) {
return result.finally(cleanup)
} else {
cleanup()
return result
}
}

24
packages/backend-core/src/middleware/passport/sso/sso.ts
View File

@ -2,7 +2,6 @@ import { generateGlobalUserID } from "../../../db"
import { authError } from "../utils"
import * as users from "../../../users"
import * as context from "../../../context"
import fetch from "node-fetch"
import {
SaveSSOUserFunction,
SSOAuthDetails,
@ -97,28 +96,13 @@ export async function authenticate(
return done(null, ssoUser)
}
async function getProfilePictureUrl(user: User, details: SSOAuthDetails) {
const pictureUrl = details.profile?._json.picture
if (pictureUrl) {
const response = await fetch(pictureUrl)
if (response.status === 200) {
const type = response.headers.get("content-type") as string
if (type.startsWith("image/")) {
return pictureUrl
}
}
}
}
/**
* @returns a user that has been sync'd with third party information
*/
async function syncUser(user: User, details: SSOAuthDetails): Promise<SSOUser> {
let firstName
let lastName
let pictureUrl
let oauth2
let thirdPartyProfile
if (details.profile) {
const profile = details.profile
@ -134,12 +118,6 @@ async function syncUser(user: User, details: SSOAuthDetails): Promise<SSOUser> {
lastName = name.familyName
}
}
pictureUrl = await getProfilePictureUrl(user, details)
thirdPartyProfile = {
...profile._json,
}
}
// oauth tokens for future use
@ -155,8 +133,6 @@ async function syncUser(user: User, details: SSOAuthDetails): Promise<SSOUser> {
providerType: details.providerType,
firstName,
lastName,
thirdPartyProfile,
pictureUrl,
oauth2,
}
}

12
packages/backend-core/src/users/users.ts
View File

@ -24,6 +24,7 @@ import * as context from "../context"
import { getGlobalDB } from "../context"
import { isCreator } from "./utils"
import { UserDB } from "./db"
import { dataFilters } from "@budibase/shared-core"
type GetOpts = { cleanup?: boolean }
@ -262,10 +263,17 @@ export async function paginatedUsers({
userList = await bulkGetGlobalUsersById(query?.oneOf?._id, {
cleanup: true,
})
} else if (query) {
// TODO: this should use SQS search, but the logic is built in the 'server' package. Using the in-memory filtering to get this working meanwhile
const response = await db.allDocs<User>(
getGlobalUserParams(null, { ...opts, limit: undefined })
)
userList = response.rows.map(row => row.doc!)
userList = dataFilters.search(userList, { query, limit: opts.limit }).rows
} else {
// no search, query allDocs
const response = await db.allDocs(getGlobalUserParams(null, opts))
userList = response.rows.map((row: any) => row.doc)
const response = await db.allDocs<User>(getGlobalUserParams(null, opts))
userList = response.rows.map(row => row.doc!)
}
return pagination(userList, pageSize, {
paginate: true,

67
packages/backend-core/tests/core/utilities/structures/accounts.ts
View File

@ -6,9 +6,6 @@ import {
AccountSSOProviderType,
AuthType,
CloudAccount,
CreateAccount,
CreatePassswordAccount,
CreateVerifiableSSOAccount,
Hosting,
SSOAccount,
} from "@budibase/types"
@ -19,6 +16,7 @@ export const account = (partial: Partial<Account> = {}): Account => {
accountId: uuid(),
tenantId: generator.word(),
email: generator.email({ domain: "example.com" }),
accountName: generator.word(),
tenantName: generator.word(),
hosting: Hosting.SELF,
createdAt: Date.now(),
@ -61,10 +59,8 @@ export function ssoAccount(account: Account = cloudAccount()): SSOAccount {
accessToken: generator.string(),
refreshToken: generator.string(),
},
pictureUrl: generator.url(),
provider: provider(),
providerType: providerType(),
thirdPartyProfile: {},
}
}
@ -78,68 +74,7 @@ export function verifiableSsoAccount(
accessToken: generator.string(),
refreshToken: generator.string(),
},
pictureUrl: generator.url(),
provider: AccountSSOProvider.MICROSOFT,
providerType: AccountSSOProviderType.MICROSOFT,
thirdPartyProfile: { id: "abc123" },
}
}
export const cloudCreateAccount: CreatePassswordAccount = {
email: "cloud@budibase.com",
tenantId: "cloud",
hosting: Hosting.CLOUD,
authType: AuthType.PASSWORD,
password: "Password123!",
tenantName: "cloud",
name: "Budi Armstrong",
size: "10+",
profession: "Software Engineer",
}
export const cloudSSOCreateAccount: CreateAccount = {
email: "cloud-sso@budibase.com",
tenantId: "cloud-sso",
hosting: Hosting.CLOUD,
authType: AuthType.SSO,
tenantName: "cloudsso",
name: "Budi Armstrong",
size: "10+",
profession: "Software Engineer",
}
export const cloudVerifiableSSOCreateAccount: CreateVerifiableSSOAccount = {
email: "cloud-sso@budibase.com",
tenantId: "cloud-sso",
hosting: Hosting.CLOUD,
authType: AuthType.SSO,
tenantName: "cloudsso",
name: "Budi Armstrong",
size: "10+",
profession: "Software Engineer",
provider: AccountSSOProvider.MICROSOFT,
thirdPartyProfile: { id: "abc123" },
}
export const selfCreateAccount: CreatePassswordAccount = {
email: "self@budibase.com",
tenantId: "self",
hosting: Hosting.SELF,
authType: AuthType.PASSWORD,
password: "Password123!",
tenantName: "self",
name: "Budi Armstrong",
size: "10+",
profession: "Software Engineer",
}
export const selfSSOCreateAccount: CreateAccount = {
email: "self-sso@budibase.com",
tenantId: "self-sso",
hosting: Hosting.SELF,
authType: AuthType.SSO,
tenantName: "selfsso",
name: "Budi Armstrong",
size: "10+",
profession: "Software Engineer",
}

5
packages/backend-core/tests/core/utilities/structures/users.ts
View File

@ -25,7 +25,6 @@ export const user = (userProps?: Partial<Omit<User, "userId">>): User => {
roles: { app_test: "admin" },
firstName: generator.first(),
lastName: generator.last(),
pictureUrl: "http://example.com",
tenantId: tenant.id(),
...userProps,
}
@ -86,9 +85,5 @@ export function ssoUser(
oauth2: opts.details?.oauth2,
provider: opts.details?.provider!,
providerType: opts.details?.providerType!,
thirdPartyProfile: {
email: base.email,
picture: base.pictureUrl,
},
}
}

3
packages/builder/src/pages/builder/portal/apps/index.svelte
View File

@ -26,6 +26,7 @@
licensing,
environment,
enrichedApps,
sortBy,
} from "stores/portal"
import { goto } from "@roxi/routify"
import AppRow from "components/start/AppRow.svelte"
@ -247,7 +248,7 @@
<div class="app-actions">
<Select
autoWidth
value={$appsStore.sortBy}
value={$sortBy}
on:change={e => {
appsStore.updateSort(e.detail)
}}

89
packages/builder/src/stores/portal/apps.js
View File

@ -9,7 +9,6 @@ const DEV_PROPS = ["updatedBy", "updatedAt"]
export const INITIAL_APPS_STATE = {
apps: [],
sortBy: "name",
}
export class AppsStore extends BudiStore {
@ -53,6 +52,15 @@ export class AppsStore extends BudiStore {
...state,
sortBy,
}))
this.updateUserSort(sortBy)
}
async updateUserSort(sortBy) {
try {
await auth.updateSelf({ appSort: sortBy })
} catch (err) {
console.error("couldn't save user sort: ", err)
}
}
async load() {
@ -140,43 +148,50 @@ export class AppsStore extends BudiStore {
export const appsStore = new AppsStore()
// Centralise any logic that enriches the apps list
export const enrichedApps = derived([appsStore, auth], ([$store, $auth]) => {
const enrichedApps = $store.apps
? $store.apps.map(app => ({
...app,
deployed: app.status === AppStatus.DEPLOYED,
lockedYou: app.lockedBy && app.lockedBy.email === $auth.user?.email,
lockedOther: app.lockedBy && app.lockedBy.email !== $auth.user?.email,
favourite: $auth.user?.appFavourites?.includes(app.appId),
}))
: []
export const sortBy = derived([appsStore, auth], ([$store, $auth]) => {
return $store.sortBy || $auth.user?.appSort || "name"
})
if ($store.sortBy === "status") {
return enrichedApps.sort((a, b) => {
if (a.favourite === b.favourite) {
if (a.status === b.status) {
// Centralise any logic that enriches the apps list
export const enrichedApps = derived(
[appsStore, auth, sortBy],
([$store, $auth, $sortBy]) => {
const enrichedApps = $store.apps
? $store.apps.map(app => ({
...app,
deployed: app.status === AppStatus.DEPLOYED,
lockedYou: app.lockedBy && app.lockedBy.email === $auth.user?.email,
lockedOther: app.lockedBy && app.lockedBy.email !== $auth.user?.email,
favourite: $auth.user?.appFavourites?.includes(app.appId),
}))
: []
if ($sortBy === "status") {
return enrichedApps.sort((a, b) => {
if (a.favourite === b.favourite) {
if (a.status === b.status) {
return a.name?.toLowerCase() < b.name?.toLowerCase() ? -1 : 1
}
return a.status === AppStatus.DEPLOYED ? -1 : 1
}
return a.favourite ? -1 : 1
})
} else if ($sortBy === "updated") {
return enrichedApps?.sort((a, b) => {
if (a.favourite === b.favourite) {
const aUpdated = a.updatedAt || "9999"
const bUpdated = b.updatedAt || "9999"
return aUpdated < bUpdated ? 1 : -1
}
return a.favourite ? -1 : 1
})
} else {
return enrichedApps?.sort((a, b) => {
if (a.favourite === b.favourite) {
return a.name?.toLowerCase() < b.name?.toLowerCase() ? -1 : 1
}
return a.status === AppStatus.DEPLOYED ? -1 : 1
}
return a.favourite ? -1 : 1
})
} else if ($store.sortBy === "updated") {
return enrichedApps?.sort((a, b) => {
if (a.favourite === b.favourite) {
const aUpdated = a.updatedAt || "9999"
const bUpdated = b.updatedAt || "9999"
return aUpdated < bUpdated ? 1 : -1
}
return a.favourite ? -1 : 1
})
} else {
return enrichedApps?.sort((a, b) => {
if (a.favourite === b.favourite) {
return a.name?.toLowerCase() < b.name?.toLowerCase() ? -1 : 1
}
return a.favourite ? -1 : 1
})
return a.favourite ? -1 : 1
})
}
}
})
)

2
packages/builder/src/stores/portal/index.js
View File

@ -3,7 +3,7 @@ import { writable } from "svelte/store"
export { organisation } from "./organisation"
export { users } from "./users"
export { admin } from "./admin"
export { appsStore, enrichedApps } from "./apps"
export { appsStore, enrichedApps, sortBy } from "./apps"
export { email } from "./email"
export { auth } from "./auth"
export { oidc } from "./oidc"

2
packages/frontend-core/src/api/tables.js
View File

@ -40,7 +40,7 @@ export const buildTableEndpoints = API => ({
sortType,
paginate,
}) => {
if (!tableId || !query) {
if (!tableId) {
return {
rows: [],
}

87
packages/server/src/api/controllers/rowAction/crud.ts
View File

@ -1,6 +1,7 @@
import {
CreateRowActionRequest,
Ctx,
RowActionPermissions,
RowActionResponse,
RowActionsResponse,
UpdateRowActionRequest,
@ -18,25 +19,26 @@ async function getTable(ctx: Ctx) {
export async function find(ctx: Ctx<void, RowActionsResponse>) {
const table = await getTable(ctx)
const tableId = table._id!
if (!(await sdk.rowActions.docExists(table._id!))) {
if (!(await sdk.rowActions.docExists(tableId))) {
ctx.body = {
actions: {},
}
return
}
const { actions } = await sdk.rowActions.getAll(table._id!)
const { actions } = await sdk.rowActions.getAll(tableId)
const result: RowActionsResponse = {
actions: Object.entries(actions).reduce<Record<string, RowActionResponse>>(
(acc, [key, action]) => ({
...acc,
[key]: {
id: key,
tableId: table._id!,
tableId,
name: action.name,
automationId: action.automationId,
allowedViews: flattenAllowedViews(action.permissions.views),
allowedSources: flattenAllowedSources(tableId, action.permissions),
},
}),
{}
@ -49,17 +51,18 @@ export async function create(
ctx: Ctx<CreateRowActionRequest, RowActionResponse>
) {
const table = await getTable(ctx)
const tableId = table._id!
const createdAction = await sdk.rowActions.create(table._id!, {
const createdAction = await sdk.rowActions.create(tableId, {
name: ctx.request.body.name,
})
ctx.body = {
tableId: table._id!,
tableId,
id: createdAction.id,
name: createdAction.name,
automationId: createdAction.automationId,
allowedViews: undefined,
allowedSources: flattenAllowedSources(tableId, createdAction.permissions),
}
ctx.status = 201
}
@ -68,18 +71,19 @@ export async function update(
ctx: Ctx<UpdateRowActionRequest, RowActionResponse>
) {
const table = await getTable(ctx)
const tableId = table._id!
const { actionId } = ctx.params
const action = await sdk.rowActions.update(table._id!, actionId, {
const action = await sdk.rowActions.update(tableId, actionId, {
name: ctx.request.body.name,
})
ctx.body = {
tableId: table._id!,
tableId,
id: action.id,
name: action.name,
automationId: action.automationId,
allowedViews: undefined,
allowedSources: flattenAllowedSources(tableId, action.permissions),
}
}
@ -91,52 +95,89 @@ export async function remove(ctx: Ctx<void, void>) {
ctx.status = 204
}
export async function setTablePermission(ctx: Ctx<void, RowActionResponse>) {
const table = await getTable(ctx)
const tableId = table._id!
const { actionId } = ctx.params
const action = await sdk.rowActions.setTablePermission(tableId, actionId)
ctx.body = {
tableId,
id: action.id,
name: action.name,
automationId: action.automationId,
allowedSources: flattenAllowedSources(tableId, action.permissions),
}
}
export async function unsetTablePermission(ctx: Ctx<void, RowActionResponse>) {
const table = await getTable(ctx)
const tableId = table._id!
const { actionId } = ctx.params
const action = await sdk.rowActions.unsetTablePermission(tableId, actionId)
ctx.body = {
tableId,
id: action.id,
name: action.name,
automationId: action.automationId,
allowedSources: flattenAllowedSources(tableId, action.permissions),
}
}
export async function setViewPermission(ctx: Ctx<void, RowActionResponse>) {
const table = await getTable(ctx)
const tableId = table._id!
const { actionId, viewId } = ctx.params
const action = await sdk.rowActions.setViewPermission(
table._id!,
tableId,
actionId,
viewId
)
ctx.body = {
tableId: table._id!,
tableId,
id: action.id,
name: action.name,
automationId: action.automationId,
allowedViews: flattenAllowedViews(action.permissions.views),
allowedSources: flattenAllowedSources(tableId, action.permissions),
}
}
export async function unsetViewPermission(ctx: Ctx<void, RowActionResponse>) {
const table = await getTable(ctx)
const tableId = table._id!
const { actionId, viewId } = ctx.params
const action = await sdk.rowActions.unsetViewPermission(
table._id!,
tableId,
actionId,
viewId
)
ctx.body = {
tableId: table._id!,
tableId,
id: action.id,
name: action.name,
automationId: action.automationId,
allowedViews: flattenAllowedViews(action.permissions.views),
allowedSources: flattenAllowedSources(tableId, action.permissions),
}
}
function flattenAllowedViews(
permissions: Record<string, { runAllowed: boolean }>
function flattenAllowedSources(
tableId: string,
permissions: RowActionPermissions
) {
const allowedPermissions = Object.entries(permissions || {})
.filter(([_, p]) => p.runAllowed)
.map(([viewId]) => viewId)
if (!allowedPermissions.length) {
return undefined
const allowedPermissions = []
if (permissions.table.runAllowed) {
allowedPermissions.push(tableId)
}
allowedPermissions.push(
...Object.keys(permissions.views || {}).filter(
viewId => permissions.views[viewId].runAllowed
)
)
return allowedPermissions
}

2
packages/server/src/api/controllers/view/viewsV2.ts
View File

@ -127,6 +127,7 @@ export async function create(ctx: Ctx<CreateViewRequest, ViewResponse>) {
const parsedView: Omit<RequiredKeys<ViewV2>, "id" | "version"> = {
name: view.name,
type: view.type,
tableId: view.tableId,
query: view.query,
queryUI: view.queryUI,
@ -162,6 +163,7 @@ export async function update(ctx: Ctx<UpdateViewRequest, ViewResponse>) {
const parsedView: RequiredKeys<ViewV2> = {
id: view.id,
name: view.name,
type: view.type,
version: view.version,
tableId: view.tableId,
query: view.query,

10
packages/server/src/api/routes/rowAction.ts
View File

@ -51,6 +51,16 @@ router
authorized(BUILDER),
rowActionController.remove
)
.post(
"/api/tables/:tableId/actions/:actionId/permissions",
authorized(BUILDER),
rowActionController.setTablePermission
)
.delete(
"/api/tables/:tableId/actions/:actionId/permissions",
authorized(BUILDER),
rowActionController.unsetTablePermission
)
.post(
"/api/tables/:tableId/actions/:actionId/permissions/:viewId",
authorized(BUILDER),

17
packages/server/src/api/routes/tests/application.spec.ts
View File

@ -14,12 +14,7 @@ jest.mock("../../../utilities/redis", () => ({
import { checkBuilderEndpoint } from "./utilities/TestFunctions"
import * as setup from "./utilities"
import { AppStatus } from "../../../db/utils"
import {
events,
utils,
context,
withEnv as withCoreEnv,
} from "@budibase/backend-core"
import { events, utils, context, features } from "@budibase/backend-core"
import env from "../../../environment"
import { type App } from "@budibase/types"
import tk from "timekeeper"
@ -358,9 +353,13 @@ describe("/applications", () => {
.delete(`/api/global/roles/${prodAppId}`)
.reply(200, {})
await withCoreEnv({ TENANT_FEATURE_FLAGS: "*:SQS" }, async () => {
await config.api.application.delete(app.appId)
})
await features.testutils.withFeatureFlags(
"*",
{ SQS: true },
async () => {
await config.api.application.delete(app.appId)
}
)
})
})

33
packages/server/src/api/routes/tests/row.spec.ts
View File

@ -13,8 +13,7 @@ import {
context,
InternalTable,
tenancy,
withEnv as withCoreEnv,
setEnv as setCoreEnv,
features,
} from "@budibase/backend-core"
import { quotas } from "@budibase/pro"
import {
@ -40,7 +39,6 @@ import {
TableSchema,
JsonFieldSubType,
RowExportFormat,
FeatureFlag,
RelationSchemaField,
} from "@budibase/types"
import { generator, mocks } from "@budibase/backend-core/tests"
@ -98,12 +96,12 @@ describe.each([
let envCleanup: (() => void) | undefined
beforeAll(async () => {
await withCoreEnv({ TENANT_FEATURE_FLAGS: "*:SQS" }, () => config.init())
if (isLucene) {
envCleanup = setCoreEnv({ TENANT_FEATURE_FLAGS: "*:!SQS" })
} else if (isSqs) {
envCleanup = setCoreEnv({ TENANT_FEATURE_FLAGS: "*:SQS" })
}
await features.testutils.withFeatureFlags("*", { SQS: true }, () =>
config.init()
)
envCleanup = features.testutils.setFeatureFlags("*", {
SQS: isSqs,
})
if (dsProvider) {
const rawDatasource = await dsProvider
@ -2517,15 +2515,9 @@ describe.each([
let flagCleanup: (() => void) | undefined
beforeAll(async () => {
const env = {
TENANT_FEATURE_FLAGS: `*:${FeatureFlag.ENRICHED_RELATIONSHIPS}`,
}
if (isSqs) {
env.TENANT_FEATURE_FLAGS = `${env.TENANT_FEATURE_FLAGS},*:SQS`
} else {
env.TENANT_FEATURE_FLAGS = `${env.TENANT_FEATURE_FLAGS},*:!SQS`
}
flagCleanup = setCoreEnv(env)
flagCleanup = features.testutils.setFeatureFlags("*", {
ENRICHED_RELATIONSHIPS: true,
})
const aux2Table = await config.api.table.save(saveTableRequest())
const aux2Data = await config.api.row.save(aux2Table._id!, {})
@ -2752,9 +2744,10 @@ describe.each([
it.each(testScenarios)(
"does not enrich relationships when not enabled (via %s)",
async (__, retrieveDelegate) => {
await withCoreEnv(
await features.testutils.withFeatureFlags(
"*",
{
TENANT_FEATURE_FLAGS: `*:!${FeatureFlag.ENRICHED_RELATIONSHIPS}`,
ENRICHED_RELATIONSHIPS: false,
},
async () => {
const otherRows = _.sampleSize(auxData, 5)

143
packages/server/src/api/routes/tests/rowAction.spec.ts
View File

@ -133,6 +133,7 @@ describe("/rowsActions", () => {
id: expect.stringMatching(/^row_action_\w+/),
tableId: tableId,
automationId: expectAutomationId(),
allowedSources: [tableId],
})
expect(await config.api.rowAction.find(tableId)).toEqual({
@ -142,6 +143,7 @@ describe("/rowsActions", () => {
id: res.id,
tableId: tableId,
automationId: expectAutomationId(),
allowedSources: [tableId],
},
},
})
@ -180,18 +182,21 @@ describe("/rowsActions", () => {
id: responses[0].id,
tableId,
automationId: expectAutomationId(),
allowedSources: [tableId],
},
[responses[1].id]: {
name: rowActions[1].name,
id: responses[1].id,
tableId,
automationId: expectAutomationId(),
allowedSources: [tableId],
},
[responses[2].id]: {
name: rowActions[2].name,
id: responses[2].id,
tableId,
automationId: expectAutomationId(),
allowedSources: [tableId],
},
},
})
@ -224,6 +229,7 @@ describe("/rowsActions", () => {
id: expect.any(String),
tableId,
automationId: expectAutomationId(),
allowedSources: [tableId],
})
expect(await config.api.rowAction.find(tableId)).toEqual({
@ -233,6 +239,7 @@ describe("/rowsActions", () => {
id: res.id,
tableId: tableId,
automationId: expectAutomationId(),
allowedSources: [tableId],
},
},
})
@ -354,6 +361,7 @@ describe("/rowsActions", () => {
tableId,
name: updatedName,
automationId: actionData.automationId,
allowedSources: [tableId],
})
expect(await config.api.rowAction.find(tableId)).toEqual(
@ -364,6 +372,7 @@ describe("/rowsActions", () => {
id: actionData.id,
tableId: actionData.tableId,
automationId: actionData.automationId,
allowedSources: [tableId],
},
}),
})
@ -515,6 +524,81 @@ describe("/rowsActions", () => {
})
})
describe("set/unsetTablePermission", () => {
describe.each([
["setTablePermission", config.api.rowAction.setTablePermission],
["unsetTablePermission", config.api.rowAction.unsetTablePermission],
])("unauthorisedTests for %s", (__, delegateTest) => {
unauthorisedTests((expectations, testConfig) =>
delegateTest(tableId, generator.guid(), expectations, testConfig)
)
})
let actionId1: string, actionId2: string
beforeEach(async () => {
for (const rowAction of createRowActionRequests(3)) {
await createRowAction(tableId, rowAction)
}
const persisted = await config.api.rowAction.find(tableId)
const actions = _.sampleSize(Object.keys(persisted.actions), 2)
actionId1 = actions[0]
actionId2 = actions[1]
})
it("can set table permission", async () => {
await config.api.rowAction.unsetTablePermission(tableId, actionId1)
await config.api.rowAction.unsetTablePermission(tableId, actionId2)
const actionResult = await config.api.rowAction.setTablePermission(
tableId,
actionId1
)
const expectedAction1 = expect.objectContaining({
allowedSources: [tableId],
})
const expectedActions = expect.objectContaining({
[actionId1]: expectedAction1,
[actionId2]: expect.objectContaining({
allowedSources: [],
}),
})
expect(actionResult).toEqual(expectedAction1)
expect((await config.api.rowAction.find(tableId)).actions).toEqual(
expectedActions
)
})
it("can unset table permission", async () => {
const actionResult = await config.api.rowAction.unsetTablePermission(
tableId,
actionId1
)
const expectedAction = expect.objectContaining({
allowedSources: [],
})
expect(actionResult).toEqual(expectedAction)
expect(
(await config.api.rowAction.find(tableId)).actions[actionId1]
).toEqual(expectedAction)
})
it.each([
["setTablePermission", config.api.rowAction.setTablePermission],
["unsetTablePermission", config.api.rowAction.unsetTablePermission],
])(
"cannot update permission for unexisting tables (%s)",
async (__, delegateTest) => {
const tableId = generator.guid()
await delegateTest(tableId, actionId1, {
status: 404,
})
}
)
})
describe("set/unsetViewPermission", () => {
describe.each([
["setViewPermission", config.api.rowAction.setViewPermission],
@ -531,11 +615,9 @@ describe("/rowsActions", () => {
)
})
let tableIdForDescribe: string
let actionId1: string, actionId2: string
let viewId1: string, viewId2: string
beforeAll(async () => {
tableIdForDescribe = tableId
beforeEach(async () => {
for (const rowAction of createRowActionRequests(3)) {
await createRowAction(tableId, rowAction)
}
@ -557,11 +639,6 @@ describe("/rowsActions", () => {
).id
})
beforeEach(() => {
// Hack to reuse tables for these given tests
tableId = tableIdForDescribe
})
it("can set permission views", async () => {
await config.api.rowAction.setViewPermission(tableId, viewId1, actionId1)
const action1Result = await config.api.rowAction.setViewPermission(
@ -576,10 +653,10 @@ describe("/rowsActions", () => {
)
const expectedAction1 = expect.objectContaining({
allowedViews: [viewId1, viewId2],
allowedSources: [tableId, viewId1, viewId2],
})
const expectedAction2 = expect.objectContaining({
allowedViews: [viewId1],
allowedSources: [tableId, viewId1],
})
const expectedActions = expect.objectContaining({
@ -594,6 +671,8 @@ describe("/rowsActions", () => {
})
it("can unset permission views", async () => {
await config.api.rowAction.setViewPermission(tableId, viewId2, actionId1)
await config.api.rowAction.setViewPermission(tableId, viewId1, actionId2)
const actionResult = await config.api.rowAction.unsetViewPermission(
tableId,
viewId1,
@ -601,7 +680,7 @@ describe("/rowsActions", () => {
)
const expectedAction = expect.objectContaining({
allowedViews: [viewId2],
allowedSources: [tableId, viewId2],
})
expect(actionResult).toEqual(expectedAction)
expect(
@ -672,6 +751,7 @@ describe("/rowsActions", () => {
)
await config.publish()
// Travel time in order to "trim" the selected `getAutomationLogs`
tk.travel(Date.now() + 100)
})
@ -712,6 +792,38 @@ describe("/rowsActions", () => {
])
})
it("triggers from an allowed table", async () => {
expect(await getAutomationLogs()).toBeEmpty()
await config.api.rowAction.trigger(tableId, rowAction.id, { rowId })
const automationLogs = await getAutomationLogs()
expect(automationLogs).toEqual([
expect.objectContaining({
automationId: rowAction.automationId,
}),
])
})
it("rejects triggering from a non-allowed table", async () => {
await config.api.rowAction.unsetTablePermission(tableId, rowAction.id)
await config.publish()
await config.api.rowAction.trigger(
tableId,
rowAction.id,
{ rowId },
{
status: 403,
body: {
message: `Row action '${rowAction.id}' is not enabled for table '${tableId}'`,
},
}
)
const automationLogs = await getAutomationLogs()
expect(automationLogs).toEqual([])
})
it("rejects triggering from a non-allowed view", async () => {
const viewId = (
await config.api.viewV2.create(
@ -901,7 +1013,7 @@ describe("/rowsActions", () => {
})
it.each(allowedRoleConfig)(
"does not allow running row actions for tables by default even",
"allow running row actions for tables by default",
async (userRole, resourcePermission) => {
await config.api.permission.add({
level: PermissionLevel.READ,
@ -918,15 +1030,12 @@ describe("/rowsActions", () => {
rowAction.id,
{ rowId },
{
status: 403,
body: {
message: `Row action '${rowAction.id}' is not enabled for table '${tableId}'`,
},
status: 200,
}
)
const automationLogs = await getAutomationLogs()
expect(automationLogs).toBeEmpty()
expect(automationLogs).toHaveLength(1)
})
}
)

19
packages/server/src/api/routes/tests/search.spec.ts
View File

@ -7,9 +7,9 @@ import {
import {
context,
db as dbCore,
features,
MAX_VALID_DATE,
MIN_VALID_DATE,
setEnv as setCoreEnv,
SQLITE_DESIGN_DOC_ID,
utils,
withEnv as withCoreEnv,
@ -94,16 +94,12 @@ describe.each([
}
beforeAll(async () => {
await withCoreEnv({ TENANT_FEATURE_FLAGS: "*:SQS" }, () => config.init())
if (isLucene) {
envCleanup = setCoreEnv({
TENANT_FEATURE_FLAGS: "*:!SQS",
})
} else if (isSqs) {
envCleanup = setCoreEnv({
TENANT_FEATURE_FLAGS: "*:SQS",
})
}
await features.testutils.withFeatureFlags("*", { SQS: true }, () =>
config.init()
)
envCleanup = features.testutils.setFeatureFlags("*", {
SQS: isSqs,
})
if (config.app?.appId) {
config.app = await config.api.application.update(config.app?.appId, {
@ -191,7 +187,6 @@ describe.each([
if (isInMemory) {
return dataFilters.search(_.cloneDeep(rows), {
...this.query,
tableId: tableOrViewId,
})
} else {
return config.api.row.search(tableOrViewId, this.query)

69
packages/server/src/api/routes/tests/templates.spec.ts
View File

@ -2,7 +2,7 @@ import * as setup from "./utilities"
import path from "path"
import nock from "nock"
import { generator } from "@budibase/backend-core/tests"
import { withEnv as withCoreEnv, env as coreEnv } from "@budibase/backend-core"
import { features } from "@budibase/backend-core"
interface App {
background: string
@ -85,41 +85,44 @@ describe("/templates", () => {
it.each(["sqs", "lucene"])(
`should be able to create an app from a template (%s)`,
async source => {
const env: Partial<typeof coreEnv> = {
TENANT_FEATURE_FLAGS: source === "sqs" ? "*:SQS" : "",
}
await features.testutils.withFeatureFlags(
"*",
{ SQS: source === "sqs" },
async () => {
const name = generator.guid().replaceAll("-", "")
const url = `/${name}`
await withCoreEnv(env, async () => {
const name = generator.guid().replaceAll("-", "")
const url = `/${name}`
const app = await config.api.application.create({
name,
url,
useTemplate: "true",
templateName: "Agency Client Portal",
templateKey: "app/agency-client-portal",
})
expect(app.name).toBe(name)
expect(app.url).toBe(url)
await config.withApp(app, async () => {
const tables = await config.api.table.fetch()
expect(tables).toHaveLength(2)
tables.sort((a, b) => a.name.localeCompare(b.name))
const [agencyProjects, users] = tables
expect(agencyProjects.name).toBe("Agency Projects")
expect(users.name).toBe("Users")
const { rows } = await config.api.row.search(agencyProjects._id!, {
tableId: agencyProjects._id!,
query: {},
const app = await config.api.application.create({
name,
url,
useTemplate: "true",
templateName: "Agency Client Portal",
templateKey: "app/agency-client-portal",
})
expect(app.name).toBe(name)
expect(app.url).toBe(url)
expect(rows).toHaveLength(3)
})
})
await config.withApp(app, async () => {
const tables = await config.api.table.fetch()
expect(tables).toHaveLength(2)
tables.sort((a, b) => a.name.localeCompare(b.name))
const [agencyProjects, users] = tables
expect(agencyProjects.name).toBe("Agency Projects")
expect(users.name).toBe("Users")
const { rows } = await config.api.row.search(
agencyProjects._id!,
{
tableId: agencyProjects._id!,
query: {},
}
)
expect(rows).toHaveLength(3)
})
}
)
}
)
})

271
packages/server/src/api/routes/tests/viewV2.spec.ts
View File

@ -22,22 +22,16 @@ import {
RelationshipType,
TableSchema,
RenameColumn,
FeatureFlag,
BBReferenceFieldSubType,
NumericCalculationFieldMetadata,
ViewV2Schema,
ViewV2Type,
} from "@budibase/types"
import { generator, mocks } from "@budibase/backend-core/tests"
import { DatabaseName, getDatasource } from "../../../integrations/tests/utils"
import merge from "lodash/merge"
import { quotas } from "@budibase/pro"
import {
db,
roles,
withEnv as withCoreEnv,
setEnv as setCoreEnv,
env,
} from "@budibase/backend-core"
import { db, roles, features } from "@budibase/backend-core"
describe.each([
["lucene", undefined],
@ -102,18 +96,13 @@ describe.each([
}
beforeAll(async () => {
await withCoreEnv({ TENANT_FEATURE_FLAGS: isSqs ? "*:SQS" : "" }, () =>
await features.testutils.withFeatureFlags("*", { SQS: isSqs }, () =>
config.init()
)
if (isLucene) {
envCleanup = setCoreEnv({
TENANT_FEATURE_FLAGS: "*:!SQS",
})
} else if (isSqs) {
envCleanup = setCoreEnv({
TENANT_FEATURE_FLAGS: "*:SQS",
})
}
envCleanup = features.testutils.setFeatureFlags("*", {
SQS: isSqs,
})
if (dsProvider) {
datasource = await config.createDatasource({
@ -155,7 +144,7 @@ describe.each([
})
it("can persist views with all fields", async () => {
const newView: Required<Omit<CreateViewRequest, "queryUI">> = {
const newView: Required<Omit<CreateViewRequest, "queryUI" | "type">> = {
name: generator.name(),
tableId: table._id!,
primaryDisplay: "id",
@ -546,6 +535,7 @@ describe.each([
let view = await config.api.viewV2.create({
tableId: table._id!,
name: generator.guid(),
type: ViewV2Type.CALCULATION,
schema: {
sum: {
visible: true,
@ -568,6 +558,184 @@ describe.each([
expect(sum.calculationType).toEqual(CalculationType.SUM)
expect(sum.field).toEqual("Price")
})
it("cannot create a view with calculation fields unless it has the right type", async () => {
await config.api.viewV2.create(
{
tableId: table._id!,
name: generator.guid(),
schema: {
sum: {
visible: true,
calculationType: CalculationType.SUM,
field: "Price",
},
},
},
{
status: 400,
body: {
message:
"Calculation fields are not allowed in non-calculation views",
},
}
)
})
it("cannot create a calculation view with more than 5 aggregations", async () => {
await config.api.viewV2.create(
{
tableId: table._id!,
name: generator.guid(),
type: ViewV2Type.CALCULATION,
schema: {
sum: {
visible: true,
calculationType: CalculationType.SUM,
field: "Price",
},
count: {
visible: true,
calculationType: CalculationType.COUNT,
field: "Price",
},
min: {
visible: true,
calculationType: CalculationType.MIN,
field: "Price",
},
max: {
visible: true,
calculationType: CalculationType.MAX,
field: "Price",
},
avg: {
visible: true,
calculationType: CalculationType.AVG,
field: "Price",
},
sum2: {
visible: true,
calculationType: CalculationType.SUM,
field: "Price",
},
},
},
{
status: 400,
body: {
message: "Calculation views can only have a maximum of 5 fields",
},
}
)
})
it("cannot create a calculation view with duplicate calculations", async () => {
await config.api.viewV2.create(
{
tableId: table._id!,
name: generator.guid(),
type: ViewV2Type.CALCULATION,
schema: {
sum: {
visible: true,
calculationType: CalculationType.SUM,
field: "Price",
},
sum2: {
visible: true,
calculationType: CalculationType.SUM,
field: "Price",
},
},
},
{
status: 400,
body: {
message:
'Duplicate calculation on field "Price", calculation type "sum"',
},
}
)
})
it("finds duplicate counts", async () => {
await config.api.viewV2.create(
{
tableId: table._id!,
name: generator.guid(),
type: ViewV2Type.CALCULATION,
schema: {
count: {
visible: true,
calculationType: CalculationType.COUNT,
},
count2: {
visible: true,
calculationType: CalculationType.COUNT,
},
},
},
{
status: 400,
body: {
message:
'Duplicate calculation on field "*", calculation type "count"',
},
}
)
})
it("finds duplicate count distincts", async () => {
await config.api.viewV2.create(
{
tableId: table._id!,
name: generator.guid(),
type: ViewV2Type.CALCULATION,
schema: {
count: {
visible: true,
calculationType: CalculationType.COUNT,
distinct: true,
field: "Price",
},
count2: {
visible: true,
calculationType: CalculationType.COUNT,
distinct: true,
field: "Price",
},
},
},
{
status: 400,
body: {
message:
'Duplicate calculation on field "Price", calculation type "count"',
},
}
)
})
it("does not confuse counts and count distincts in the duplicate check", async () => {
await config.api.viewV2.create({
tableId: table._id!,
name: generator.guid(),
type: ViewV2Type.CALCULATION,
schema: {
count: {
visible: true,
calculationType: CalculationType.COUNT,
},
count2: {
visible: true,
calculationType: CalculationType.COUNT,
distinct: true,
field: "Price",
},
},
})
})
})
describe("update", () => {
@ -612,7 +780,9 @@ describe.each([
it("can update all fields", async () => {
const tableId = table._id!
const updatedData: Required<Omit<UpdateViewRequest, "queryUI">> = {
const updatedData: Required<
Omit<UpdateViewRequest, "queryUI" | "type">
> = {
version: view.version,
id: view.id,
tableId,
@ -824,6 +994,32 @@ describe.each([
)
})
it("cannot update view type after creation", async () => {
const view = await config.api.viewV2.create({
tableId: table._id!,
name: generator.guid(),
schema: {
id: { visible: true },
Price: {
visible: true,
},
},
})
await config.api.viewV2.update(
{
...view,
type: ViewV2Type.CALCULATION,
},
{
status: 400,
body: {
message: "Cannot update view type after creation",
},
}
)
})
isInternal &&
it("updating schema will only validate modified field", async () => {
let view = await config.api.viewV2.create({
@ -896,6 +1092,7 @@ describe.each([
view = await config.api.viewV2.create({
tableId: table._id!,
name: generator.guid(),
type: ViewV2Type.CALCULATION,
schema: {
country: {
visible: true,
@ -1018,6 +1215,26 @@ describe.each([
)
})
it("can add a new group by field that is invisible, even if required on the table", async () => {
view.schema!.name = { visible: false }
await config.api.viewV2.update(view)
const { rows } = await config.api.row.search(view.id)
expect(rows).toHaveLength(2)
expect(rows).toEqual(
expect.arrayContaining([
{
country: "USA",
age: 65,
},
{
country: "UK",
age: 61,
},
])
)
})
it("can add a new calculation field", async () => {
view.schema!.count = {
visible: true,
@ -2437,12 +2654,8 @@ describe.each([
describe("foreign relationship columns", () => {
let envCleanup: () => void
beforeAll(() => {
const flags = [`*:${FeatureFlag.ENRICHED_RELATIONSHIPS}`]
if (env.TENANT_FEATURE_FLAGS) {
flags.push(...env.TENANT_FEATURE_FLAGS.split(","))
}
envCleanup = setCoreEnv({
TENANT_FEATURE_FLAGS: flags.join(","),
envCleanup = features.testutils.setFeatureFlags("*", {
ENRICHED_RELATIONSHIPS: true,
})
})
@ -2633,6 +2846,7 @@ describe.each([
it("should be able to search by calculations", async () => {
const view = await config.api.viewV2.create({
tableId: table._id!,
type: ViewV2Type.CALCULATION,
name: generator.guid(),
schema: {
"Quantity Sum": {
@ -2667,6 +2881,7 @@ describe.each([
const view = await config.api.viewV2.create({
tableId: table._id!,
name: generator.guid(),
type: ViewV2Type.CALCULATION,
schema: {
quantity: {
visible: true,
@ -2705,6 +2920,7 @@ describe.each([
const view = await config.api.viewV2.create({
tableId: table._id!,
name: generator.guid(),
type: ViewV2Type.CALCULATION,
schema: {
aggregate: {
visible: true,
@ -2765,6 +2981,7 @@ describe.each([
const view = await config.api.viewV2.create({
tableId: table._id!,
name: generator.guid(),
type: ViewV2Type.CALCULATION,
schema: {
count: {
visible: true,
@ -2799,6 +3016,7 @@ describe.each([
{
tableId: table._id!,
name: generator.guid(),
type: ViewV2Type.CALCULATION,
schema: {
count: {
visible: true,
@ -2847,6 +3065,7 @@ describe.each([
const view = await config.api.viewV2.create({
tableId: table._id!,
name: generator.guid(),
type: ViewV2Type.CALCULATION,
schema: {
sum: {
visible: true,

6
packages/server/src/appMigrations/migrations/tests/20240604153647_initial_sqs.spec.ts
View File

@ -2,8 +2,8 @@ import * as setup from "../../../api/routes/tests/utilities"
import { basicTable } from "../../../tests/utilities/structures"
import {
db as dbCore,
features,
SQLITE_DESIGN_DOC_ID,
withEnv as withCoreEnv,
} from "@budibase/backend-core"
import {
LinkDocument,
@ -71,11 +71,11 @@ function oldLinkDocument(): Omit<LinkDocument, "tableId"> {
}
async function sqsDisabled(cb: () => Promise<void>) {
await withCoreEnv({ TENANT_FEATURE_FLAGS: "*:!SQS" }, cb)
await features.testutils.withFeatureFlags("*", { SQS: false }, cb)
}
async function sqsEnabled(cb: () => Promise<void>) {
await withCoreEnv({ TENANT_FEATURE_FLAGS: "*:SQS" }, cb)
await features.testutils.withFeatureFlags("*", { SQS: true }, cb)
}
describe("SQS migration", () => {

1
packages/server/src/environment.ts
View File

@ -83,6 +83,7 @@ const environment = {
PLUGINS_DIR: process.env.PLUGINS_DIR || DEFAULTS.PLUGINS_DIR,
MAX_IMPORT_SIZE_MB: process.env.MAX_IMPORT_SIZE_MB,
SESSION_EXPIRY_SECONDS: process.env.SESSION_EXPIRY_SECONDS,
XSS_SAFE_MODE: process.env.XSS_SAFE_MODE,
// SQL
SQL_MAX_ROWS: process.env.SQL_MAX_ROWS,
SQL_LOGGING_ENABLE: process.env.SQL_LOGGING_ENABLE,

28
packages/server/src/jsRunner/tests/jsRunner.spec.ts
View File

@ -8,7 +8,6 @@ import { init } from ".."
import TestConfiguration from "../../tests/utilities/TestConfiguration"
const DATE = "2021-01-21T12:00:00"
tk.freeze(DATE)
describe("jsRunner (using isolated-vm)", () => {
@ -48,6 +47,33 @@ describe("jsRunner (using isolated-vm)", () => {
).toEqual("ReferenceError: process is not defined")
})
it("should not allow the context to be mutated", async () => {
const context = { array: [1] }
const result = await processJS(
`
const array = $("array");
array.push(2);
return array[1]
`,
context
)
expect(result).toEqual(2)
expect(context.array).toEqual([1])
})
it("should copy values whenever returning them from $", async () => {
const context = { array: [1] }
const result = await processJS(
`
$("array").push(2);
return $("array")[1];
`,
context
)
expect(result).toEqual(undefined)
expect(context.array).toEqual([1])
})
describe("helpers", () => {
runJsHelpersTests({
funcWrap: (func: any) => config.doInContext(config.getAppId(), func),

19
packages/server/src/sdk/app/rowActions.ts
View File

@ -75,7 +75,7 @@ export async function create(tableId: string, rowAction: { name: string }) {
name: action.name,
automationId: automation._id!,
permissions: {
table: { runAllowed: false },
table: { runAllowed: true },
views: {},
},
}
@ -163,6 +163,23 @@ async function guardView(tableId: string, viewId: string) {
}
}
export async function setTablePermission(tableId: string, rowActionId: string) {
return await updateDoc(tableId, rowActionId, async actionsDoc => {
actionsDoc.actions[rowActionId].permissions.table.runAllowed = true
return actionsDoc
})
}
export async function unsetTablePermission(
tableId: string,
rowActionId: string
) {
return await updateDoc(tableId, rowActionId, async actionsDoc => {
actionsDoc.actions[rowActionId].permissions.table.runAllowed = false
return actionsDoc
})
}
export async function setViewPermission(
tableId: string,
rowActionId: string,

19
packages/server/src/sdk/app/rows/search/tests/search.spec.ts
View File

@ -10,10 +10,7 @@ import {
import TestConfiguration from "../../../../../tests/utilities/TestConfiguration"
import { search } from "../../../../../sdk/app/rows/search"
import { generator } from "@budibase/backend-core/tests"
import {
withEnv as withCoreEnv,
setEnv as setCoreEnv,
} from "@budibase/backend-core"
import { features } from "@budibase/backend-core"
import {
DatabaseName,
getDatasource,
@ -41,19 +38,13 @@ describe.each([
let table: Table
beforeAll(async () => {
await withCoreEnv({ TENANT_FEATURE_FLAGS: isSqs ? "*:SQS" : "" }, () =>
await features.testutils.withFeatureFlags("*", { SQS: isSqs }, () =>
config.init()
)
if (isLucene) {
envCleanup = setCoreEnv({
TENANT_FEATURE_FLAGS: "*:!SQS",
})
} else if (isSqs) {
envCleanup = setCoreEnv({
TENANT_FEATURE_FLAGS: "*:SQS",
})
}
envCleanup = features.testutils.setFeatureFlags("*", {
SQS: isSqs,
})
if (dsProvider) {
datasource = await config.createDatasource({

43
packages/server/src/sdk/app/rows/tests/utils.spec.ts
View File

@ -8,6 +8,7 @@ import {
import { generateTableID } from "../../../../db/utils"
import { validate } from "../utils"
import { generator } from "@budibase/backend-core/tests"
import { withEnv } from "../../../../environment"
describe("validate", () => {
const hour = () => generator.hour().toString().padStart(2, "0")
@ -332,4 +333,46 @@ describe("validate", () => {
})
})
})
describe("XSS Safe mode", () => {
const getTable = (): Table => ({
type: "table",
_id: generateTableID(),
name: "table",
sourceId: INTERNAL_TABLE_SOURCE_ID,
sourceType: TableSourceType.INTERNAL,
schema: {
text: {
name: "sometext",
type: FieldType.STRING,
},
},
})
it.each([
"SELECT * FROM users WHERE username = 'admin' --",
"SELECT * FROM users WHERE id = 1; DROP TABLE users;",
"1' OR '1' = '1",
"' OR 'a' = 'a",
"<script>alert('XSS');</script>",
'"><img src=x onerror=alert(1)>',
"</script><script>alert('test')</script>",
"<div onmouseover=\"alert('XSS')\">Hover over me!</div>",
"'; EXEC sp_msforeachtable 'DROP TABLE ?'; --",
"{alert('Injected')}",
"UNION SELECT * FROM users",
"INSERT INTO users (username, password) VALUES ('admin', 'password')",
"/* This is a comment */ SELECT * FROM users",
'<iframe src="http://malicious-site.com"></iframe>',
])("test potentially unsafe input: %s", async input => {
await withEnv({ XSS_SAFE_MODE: "1" }, async () => {
const table = getTable()
const row = { text: input }
const output = await validate({ source: table, row })
expect(output.valid).toBe(false)
expect(output.errors).toStrictEqual({
text: ["Input not sanitised - potentially vulnerable to XSS"],
})
})
})
})
})

13
packages/server/src/sdk/app/rows/utils.ts
View File

@ -22,6 +22,7 @@ import { extractViewInfoFromID, isRelationshipColumn } from "../../../db/utils"
import { isSQL } from "../../../integrations/utils"
import { docIds, sql } from "@budibase/backend-core"
import { getTableFromSource } from "../../../api/controllers/row/utils"
import env from "../../../environment"
const SQL_CLIENT_SOURCE_MAP: Record<SourceName, SqlClient | undefined> = {
[SourceName.POSTGRES]: SqlClient.POSTGRES,
@ -43,6 +44,9 @@ const SQL_CLIENT_SOURCE_MAP: Record<SourceName, SqlClient | undefined> = {
[SourceName.BUDIBASE]: undefined,
}
const XSS_INPUT_REGEX =
/[<>;"'(){}]|--|\/\*|\*\/|union|select|insert|drop|delete|update|exec|script/i
export function getSQLClient(datasource: Datasource): SqlClient {
if (!isSQL(datasource)) {
throw new Error("Cannot get SQL Client for non-SQL datasource")
@ -222,6 +226,15 @@ export async function validate({
} else {
res = validateJs.single(row[fieldName], constraints)
}
if (env.XSS_SAFE_MODE && typeof row[fieldName] === "string") {
if (XSS_INPUT_REGEX.test(row[fieldName])) {
errors[fieldName] = [
"Input not sanitised - potentially vulnerable to XSS",
]
}
}
if (res) errors[fieldName] = res
}
return { valid: Object.keys(errors).length === 0, errors }

3
packages/server/src/sdk/app/views/external.ts
View File

@ -70,6 +70,9 @@ export async function update(tableId: string, view: ViewV2): Promise<ViewV2> {
if (!existingView || !existingView.name) {
throw new HTTPError(`View ${view.id} not found in table ${tableId}`, 404)
}
if (isV2(existingView) && existingView.type !== view.type) {
throw new HTTPError(`Cannot update view type after creation`, 400)
}
delete views[existingView.name]
views[view.name] = view

41
packages/server/src/sdk/app/views/index.ts
View File

@ -64,11 +64,31 @@ async function guardCalculationViewSchema(
view: Omit<ViewV2, "id" | "version">
) {
const calculationFields = helpers.views.calculationFields(view)
for (const calculationFieldName of Object.keys(calculationFields)) {
const schema = calculationFields[calculationFieldName]
if (Object.keys(calculationFields).length > 5) {
throw new HTTPError(
"Calculation views can only have a maximum of 5 fields",
400
)
}
const seen: Record<string, Record<CalculationType, boolean>> = {}
for (const name of Object.keys(calculationFields)) {
const schema = calculationFields[name]
const isCount = schema.calculationType === CalculationType.COUNT
const isDistinct = isCount && "distinct" in schema && schema.distinct
const field = isCount && !isDistinct ? "*" : schema.field
if (seen[field]?.[schema.calculationType]) {
throw new HTTPError(
`Duplicate calculation on field "${field}", calculation type "${schema.calculationType}"`,
400
)
}
seen[field] ??= {} as Record<CalculationType, boolean>
seen[field][schema.calculationType] = true
// Count fields that aren't distinct don't need to reference another field,
// so we don't validate it.
if (isCount && !isDistinct) {
@ -78,14 +98,14 @@ async function guardCalculationViewSchema(
const targetSchema = table.schema[schema.field]
if (!targetSchema) {
throw new HTTPError(
`Calculation field "${calculationFieldName}" references field "${schema.field}" which does not exist in the table schema`,
`Calculation field "${name}" references field "${schema.field}" which does not exist in the table schema`,
400
)
}
if (!isCount && !helpers.schema.isNumeric(targetSchema)) {
throw new HTTPError(
`Calculation field "${calculationFieldName}" references field "${schema.field}" which is not a numeric field`,
`Calculation field "${name}" references field "${schema.field}" which is not a numeric field`,
400
)
}
@ -111,10 +131,21 @@ async function guardViewSchema(
if (helpers.views.isCalculationView(view)) {
await guardCalculationViewSchema(table, view)
} else {
if (helpers.views.hasCalculationFields(view)) {
throw new HTTPError(
"Calculation fields are not allowed in non-calculation views",
400
)
}
}
await checkReadonlyFields(table, view)
checkRequiredFields(table, view)
if (!helpers.views.isCalculationView(view)) {
checkRequiredFields(table, view)
}
checkDisplayField(view)
}

4
packages/server/src/sdk/app/views/internal.ts
View File

@ -59,6 +59,10 @@ export async function update(tableId: string, view: ViewV2): Promise<ViewV2> {
throw new HTTPError(`View ${view.id} not found in table ${tableId}`, 404)
}
if (isV2(existingView) && existingView.type !== view.type) {
throw new HTTPError(`Cannot update view type after creation`, 400)
}
delete table.views[existingView.name]
table.views[view.name] = view
await db.put(table)

36
packages/server/src/tests/utilities/api/rowAction.ts
View File

@ -72,6 +72,42 @@ export class RowActionAPI extends TestAPI {
)
}
setTablePermission = async (
tableId: string,
rowActionId: string,
expectations?: Expectations,
config?: { publicUser?: boolean }
) => {
return await this._post<RowActionResponse>(
`/api/tables/${tableId}/actions/${rowActionId}/permissions`,
{
expectations: {
status: 200,
...expectations,
},
...config,
}
)
}
unsetTablePermission = async (
tableId: string,
rowActionId: string,
expectations?: Expectations,
config?: { publicUser?: boolean }
) => {
return await this._delete<RowActionResponse>(
`/api/tables/${tableId}/actions/${rowActionId}/permissions`,
{
expectations: {
status: 200,
...expectations,
},
...config,
}
)
}
setViewPermission = async (
tableId: string,
viewId: string,

8
packages/server/src/utilities/rowProcessor/tests/outputProcessing.spec.ts
View File

@ -8,7 +8,7 @@ import {
} from "@budibase/types"
import { outputProcessing } from ".."
import { generator, structures } from "@budibase/backend-core/tests"
import { setEnv as setCoreEnv } from "@budibase/backend-core"
import { features } from "@budibase/backend-core"
import * as bbReferenceProcessor from "../bbReferenceProcessor"
import TestConfiguration from "../../../tests/utilities/TestConfiguration"
@ -21,7 +21,7 @@ jest.mock("../bbReferenceProcessor", (): typeof bbReferenceProcessor => ({
describe("rowProcessor - outputProcessing", () => {
const config = new TestConfiguration()
let cleanupEnv: () => void = () => {}
let cleanupFlags: () => void = () => {}
beforeAll(async () => {
await config.init()
@ -33,11 +33,11 @@ describe("rowProcessor - outputProcessing", () => {
beforeEach(() => {
jest.resetAllMocks()
cleanupEnv = setCoreEnv({ TENANT_FEATURE_FLAGS: "*SQS" })
cleanupFlags = features.testutils.setFeatureFlags("*", { SQS: true })
})
afterEach(() => {
cleanupEnv()
cleanupFlags()
})
const processOutputBBReferenceMock =

12
packages/shared-core/src/filters.ts
View File

@ -639,19 +639,19 @@ export function fixupFilterArrays(filters: SearchFilters) {
return filters
}
export function search<T>(
docs: Record<string, T>[],
query: RowSearchParams
): SearchResponse<Record<string, T>> {
export function search<T extends Record<string, any>>(
docs: T[],
query: Omit<RowSearchParams, "tableId">
): SearchResponse<T> {
let result = runQuery(docs, query.query)
if (query.sort) {
result = sort(result, query.sort, query.sortOrder || SortOrder.ASCENDING)
}
let totalRows = result.length
const totalRows = result.length
if (query.limit) {
result = limit(result, query.limit.toString())
}
const response: SearchResponse<Record<string, any>> = { rows: result }
const response: SearchResponse<T> = { rows: result }
if (query.countRows) {
response.totalRows = totalRows
}

5
packages/shared-core/src/helpers/views.ts
View File

@ -3,6 +3,7 @@ import {
ViewCalculationFieldMetadata,
ViewFieldMetadata,
ViewV2,
ViewV2Type,
} from "@budibase/types"
import { pickBy } from "lodash"
@ -21,6 +22,10 @@ export function isBasicViewField(
type UnsavedViewV2 = Omit<ViewV2, "id" | "version">
export function isCalculationView(view: UnsavedViewV2) {
return view.type === ViewV2Type.CALCULATION
}
export function hasCalculationFields(view: UnsavedViewV2) {
return Object.values(view.schema || {}).some(isCalculationField)
}

13
packages/shared-core/src/utils.ts
View File

@ -5,6 +5,7 @@ import {
SearchFilters,
BasicOperator,
ArrayOperator,
isLogicalSearchOperator,
} from "@budibase/types"
import * as Constants from "./constants"
import { removeKeyNumbering } from "./filters"
@ -97,10 +98,20 @@ export function isSupportedUserSearch(query: SearchFilters) {
{ op: BasicOperator.EQUAL, key: "_id" },
{ op: ArrayOperator.ONE_OF, key: "_id" },
]
for (let [key, operation] of Object.entries(query)) {
for (const [key, operation] of Object.entries(query)) {
if (typeof operation !== "object") {
return false
}
if (isLogicalSearchOperator(key)) {
for (const condition of query[key]!.conditions) {
if (!isSupportedUserSearch(condition)) {
return false
}
}
return true
}
const fields = Object.keys(operation || {})
// this filter doesn't contain options - ignore
if (fields.length === 0) {

39
packages/string-templates/src/helpers/javascript.ts
View File

@ -1,13 +1,14 @@
import { atob, isJSAllowed } from "../utilities"
import cloneDeep from "lodash/fp/cloneDeep"
import { atob, isBackendService, isJSAllowed } from "../utilities"
import { LITERAL_MARKER } from "../helpers/constants"
import { getJsHelperList } from "./list"
import { iifeWrapper } from "../iife"
import { JsTimeoutError, UserScriptError } from "../errors"
import { cloneDeep } from "lodash/fp"
// The method of executing JS scripts depends on the bundle being built.
// This setter is used in the entrypoint (either index.js or index.mjs).
let runJS: ((js: string, context: any) => any) | undefined = undefined
let runJS: ((js: string, context: Record<string, any>) => any) | undefined =
undefined
export const setJSRunner = (runner: typeof runJS) => (runJS = runner)
export const removeJSRunner = () => {
@ -31,9 +32,19 @@ const removeSquareBrackets = (value: string) => {
return value
}
const isReservedKey = (key: string) =>
key === "snippets" ||
key === "helpers" ||
key.startsWith("snippets.") ||
key.startsWith("helpers.")
// Our context getter function provided to JS code as $.
// Extracts a value from context.
const getContextValue = (path: string, context: any) => {
// We populate `snippets` ourselves, don't allow access to it.
if (isReservedKey(path)) {
return undefined
}
const literalStringRegex = /^(["'`]).*\1$/
let data = context
// check if it's a literal string - just return path if its quoted
@ -46,6 +57,7 @@ const getContextValue = (path: string, context: any) => {
}
data = data[removeSquareBrackets(key)]
})
return data
}
@ -67,10 +79,23 @@ export function processJS(handlebars: string, context: any) {
snippetMap[snippet.name] = snippet.code
}
// Our $ context function gets a value from context.
// We clone the context to avoid mutation in the binding affecting real
// app context.
const clonedContext = cloneDeep({ ...context, snippets: null })
let clonedContext: Record<string, any>
if (isBackendService()) {
// On the backned, values are copied across the isolated-vm boundary and
// so we don't need to do any cloning here. This does create a fundamental
// difference in how JS executes on the frontend vs the backend, e.g.
// consider this snippet:
//
// $("array").push(2)
// return $("array")[1]
//
// With the context of `{ array: [1] }`, the backend will return
// `undefined` whereas the frontend will return `2`. We should fix this.
clonedContext = context
} else {
clonedContext = cloneDeep(context)
}
const sandboxContext = {
$: (path: string) => getContextValue(path, clonedContext),
helpers: getJsHelperList(),

40
packages/string-templates/src/index.ts
View File

@ -1,4 +1,4 @@
import { Context, createContext, runInNewContext } from "vm"
import { createContext, runInNewContext } from "vm"
import { create, TemplateDelegate } from "handlebars"
import { registerAll, registerMinimum } from "./helpers/index"
import { postprocess, preprocess } from "./processors"
@ -455,21 +455,14 @@ export function convertToJS(hbs: string) {
export { JsTimeoutError, UserScriptError } from "./errors"
export function defaultJSSetup() {
if (!isBackendService()) {
/**
* Use polyfilled vm to run JS scripts in a browser Env
*/
setJSRunner((js: string, context: Context) => {
context = {
...context,
alert: undefined,
setInterval: undefined,
setTimeout: undefined,
}
createContext(context)
export function browserJSSetup() {
/**
* Use polyfilled vm to run JS scripts in a browser Env
*/
setJSRunner((js: string, context: Record<string, any>) => {
createContext(context)
const wrappedJs = `
const wrappedJs = `
result = {
result: null,
error: null,
@ -484,12 +477,17 @@ export function defaultJSSetup() {
result;
`
const result = runInNewContext(wrappedJs, context, { timeout: 1000 })
if (result.error) {
throw new UserScriptError(result.error)
}
return result.result
})
const result = runInNewContext(wrappedJs, context, { timeout: 1000 })
if (result.error) {
throw new UserScriptError(result.error)
}
return result.result
})
}
export function defaultJSSetup() {
if (!isBackendService()) {
browserJSSetup()
} else {
removeJSRunner()
}

7
packages/string-templates/src/utilities.ts
View File

@ -4,7 +4,14 @@ export const FIND_HBS_REGEX = /{{([^{].*?)}}/g
export const FIND_ANY_HBS_REGEX = /{?{{([^{].*?)}}}?/g
export const FIND_TRIPLE_HBS_REGEX = /{{{([^{].*?)}}}/g
const isJest = () => typeof jest !== "undefined"
export const isBackendService = () => {
// We consider the tests for string-templates to be frontend, so that they
// test the frontend JS functionality.
if (isJest()) {
return false
}
return typeof window === "undefined"
}

350
packages/string-templates/test/javascript.spec.ts
View File

@ -1,7 +1,13 @@
import vm from "vm"
import { processStringSync, encodeJSBinding, setJSRunner } from "../src/index"
import {
processStringSync,
encodeJSBinding,
defaultJSSetup,
} from "../src/index"
import { UUID_REGEX } from "./constants"
import tk from "timekeeper"
const DATE = "2021-01-21T12:00:00"
tk.freeze(DATE)
const processJS = (js: string, context?: object): any => {
return processStringSync(encodeJSBinding(js), context)
@ -9,9 +15,7 @@ const processJS = (js: string, context?: object): any => {
describe("Javascript", () => {
beforeAll(() => {
setJSRunner((js, context) => {
return vm.runInNewContext(js, context, { timeout: 1000 })
})
defaultJSSetup()
})
describe("Test the JavaScript helper", () => {
@ -118,8 +122,7 @@ describe("Javascript", () => {
})
it("should handle errors", () => {
const output = processJS(`throw "Error"`)
expect(output).toBe("Error while executing JS")
expect(processJS(`throw "Error"`)).toEqual("Error")
})
it("should timeout after one second", () => {
@ -127,16 +130,18 @@ describe("Javascript", () => {
expect(output).toBe("Timed out while executing JS")
})
it("should prevent access to the process global", () => {
const output = processJS(`return process`)
expect(output).toBe("Error while executing JS")
it("should prevent access to the process global", async () => {
expect(processJS(`return process`)).toEqual(
"ReferenceError: process is not defined"
)
})
})
describe("check JS helpers", () => {
it("should error if using the format helper. not helpers.", () => {
const output = processJS(`return helper.toInt(4.3)`)
expect(output).toBe("Error while executing JS")
expect(processJS(`return helper.toInt(4.3)`)).toEqual(
"ReferenceError: helper is not defined"
)
})
it("should be able to use toInt", () => {
@ -156,4 +161,323 @@ describe("Javascript", () => {
expect(output).toBe("Custom")
})
})
describe("mutability", () => {
it("should not allow the context to be mutated", async () => {
const context = { array: [1] }
const result = await processJS(
`
const array = $("array");
array.push(2);
return array[1]
`,
context
)
expect(result).toEqual(2)
expect(context.array).toEqual([1])
})
})
describe("malice", () => {
it("should not be able to call JS functions", () => {
expect(processJS(`return alert("hello")`)).toEqual(
"ReferenceError: alert is not defined"
)
expect(processJS(`return prompt("hello")`)).toEqual(
"ReferenceError: prompt is not defined"
)
expect(processJS(`return confirm("hello")`)).toEqual(
"ReferenceError: confirm is not defined"
)
expect(processJS(`return setTimeout(() => {}, 1000)`)).toEqual(
"ReferenceError: setTimeout is not defined"
)
expect(processJS(`return setInterval(() => {}, 1000)`)).toEqual(
"ReferenceError: setInterval is not defined"
)
})
})
// the test cases here were extracted from templates/real world examples of JS in Budibase
describe("real test cases from Budicloud", () => {
const context = {
"Unit Value": 2,
Quantity: 1,
}
it("handle test case 1", async () => {
const result = await processJS(
`
var Gross = $("[Unit Value]") * $("[Quantity]")
return Gross.toFixed(2)`,
context
)
expect(result).toBeDefined()
expect(result).toBe("2.00")
})
it("handle test case 2", async () => {
const todayDate = new Date()
// add a year and a month
todayDate.setMonth(new Date().getMonth() + 1)
todayDate.setFullYear(todayDate.getFullYear() + 1)
const context = {
"Purchase Date": DATE,
today: todayDate.toISOString(),
}
const result = await processJS(
`
var purchase = new Date($("[Purchase Date]"));
let purchaseyear = purchase.getFullYear();
let purchasemonth = purchase.getMonth();
var today = new Date($("today"));
let todayyear = today.getFullYear();
let todaymonth = today.getMonth();
var age = todayyear - purchaseyear
if (((todaymonth - purchasemonth) < 6) == true){
return age
}
`,
context
)
expect(result).toBeDefined()
expect(result).toBe(1)
})
it("should handle test case 3", async () => {
const context = {
Escalate: true,
"Budget ($)": 1100,
}
const result = await processJS(
`
if ($("[Escalate]") == true) {
if ($("Budget ($)") <= 1000)
{return 2;}
if ($("Budget ($)") > 1000)
{return 3;}
}
else {
if ($("Budget ($)") <= 1000)
{return 1;}
if ($("Budget ($)") > 1000)
if ($("Budget ($)") < 10000)
{return 2;}
else
{return 3}
}
`,
context
)
expect(result).toBeDefined()
expect(result).toBe(3)
})
it("should handle test case 4", async () => {
const context = {
"Time Sheets": ["a", "b"],
}
const result = await processJS(
`
let hours = 0
if (($("[Time Sheets]") != null) == true){
for (i = 0; i < $("[Time Sheets]").length; i++){
let hoursLogged = "Time Sheets." + i + ".Hours"
hours += $(hoursLogged)
}
return hours
}
if (($("[Time Sheets]") != null) == false){
return hours
}
`,
context
)
expect(result).toBeDefined()
expect(result).toBe("0ab")
})
it("should handle test case 5", async () => {
const context = {
change: JSON.stringify({ a: 1, primaryDisplay: "a" }),
previous: JSON.stringify({ a: 2, primaryDisplay: "b" }),
}
const result = await processJS(
`
let change = $("[change]") ? JSON.parse($("[change]")) : {}
let previous = $("[previous]") ? JSON.parse($("[previous]")) : {}
function simplifyLink(originalKey, value, parent) {
if (Array.isArray(value)) {
if (value.filter(item => Object.keys(item || {}).includes("primaryDisplay")).length > 0) {
parent[originalKey] = value.map(link => link.primaryDisplay)
}
}
}
for (let entry of Object.entries(change)) {
simplifyLink(entry[0], entry[1], change)
}
for (let entry of Object.entries(previous)) {
simplifyLink(entry[0], entry[1], previous)
}
let diff = Object.fromEntries(Object.entries(change).filter(([k, v]) => previous[k]?.toString() !== v?.toString()))
delete diff.audit_change
delete diff.audit_previous
delete diff._id
delete diff._rev
delete diff.tableId
delete diff.audit
for (let entry of Object.entries(diff)) {
simplifyLink(entry[0], entry[1], diff)
}
return JSON.stringify(change)?.replaceAll(",\\"", ",\\n\\t\\"").replaceAll("{\\"", "{\\n\\t\\"").replaceAll("}", "\\n}")
`,
context
)
expect(result).toBe(`{\n\t"a":1,\n\t"primaryDisplay":"a"\n}`)
})
it("should handle test case 6", async () => {
const context = {
"Join Date": DATE,
}
const result = await processJS(
`
var rate = 5;
var today = new Date();
// comment
function monthDiff(dateFrom, dateTo) {
return dateTo.getMonth() - dateFrom.getMonth() +
(12 * (dateTo.getFullYear() - dateFrom.getFullYear()))
}
var serviceMonths = monthDiff( new Date($("[Join Date]")), today);
var serviceYears = serviceMonths / 12;
if (serviceYears >= 1 && serviceYears < 5){
rate = 10;
}
if (serviceYears >= 5 && serviceYears < 10){
rate = 15;
}
if (serviceYears >= 10){
rate = 15;
rate += 0.5 * (Number(serviceYears.toFixed(0)) - 10);
}
return rate;
`,
context
)
expect(result).toBe(10)
})
it("should handle test case 7", async () => {
const context = {
"P I": "Pass",
"PA I": "Pass",
"F I": "Fail",
"V I": "Pass",
}
const result = await processJS(
`if (($("[P I]") == "Pass") == true)
if (($("[ P I]") == "Pass") == true)
if (($("[F I]") == "Pass") == true)
if (($("[V I]") == "Pass") == true)
{return "Pass"}
if (($("[PA I]") == "Fail") == true)
{return "Fail"}
if (($("[ P I]") == "Fail") == true)
{return "Fail"}
if (($("[F I]") == "Fail") == true)
{return "Fail"}
if (($("[V I]") == "Fail") == true)
{return "Fail"}
else
{return ""}`,
context
)
expect(result).toBe("Fail")
})
it("should handle test case 8", async () => {
const context = {
"T L": [{ Hours: 10 }],
"B H": 50,
}
const result = await processJS(
`var totalHours = 0;
if (($("[T L]") != null) == true){
for (let i = 0; i < ($("[T L]").length); i++){
var individualHours = "T L." + i + ".Hours";
var hoursNum = Number($(individualHours));
totalHours += hoursNum;
}
return totalHours.toFixed(2);
}
if (($("[T L]") != null) == false) {
return totalHours.toFixed(2);
}
`,
context
)
expect(result).toBe("10.00")
})
it("should handle test case 9", async () => {
const context = {
"T L": [{ Hours: 10 }],
"B H": 50,
}
const result = await processJS(
`var totalHours = 0;
if (($("[T L]") != null) == true){
for (let i = 0; i < ($("[T L]").length); i++){
var individualHours = "T L." + i + ".Hours";
var hoursNum = Number($(individualHours));
totalHours += hoursNum;
}
return ($("[B H]") - totalHours).toFixed(2);
}
if (($("[T L]") != null) == false) {
return ($("[B H]") - totalHours).toFixed(2);
}`,
context
)
expect(result).toBe("40.00")
})
it("should handle test case 10", async () => {
const context = {
"F F": [{ "F S": 10 }],
}
const result = await processJS(
`var rating = 0;
if ($("[F F]") != null){
for (i = 0; i < $("[F F]").length; i++){
var individualRating = $("F F." + i + ".F S");
rating += individualRating;
}
rating = (rating / $("[F F]").length);
}
return rating;
`,
context
)
expect(result).toBe(10)
})
})
})

8
packages/string-templates/test/manifest.spec.ts
View File

@ -1,5 +1,3 @@
import vm from "vm"
jest.mock("@budibase/handlebars-helpers/lib/math", () => {
const actual = jest.requireActual("@budibase/handlebars-helpers/lib/math")
@ -17,7 +15,7 @@ jest.mock("@budibase/handlebars-helpers/lib/uuid", () => {
}
})
import { processString, setJSRunner } from "../src/index"
import { defaultJSSetup, processString } from "../src/index"
import tk from "timekeeper"
import { getParsedManifest, runJsHelpersTests } from "./utils"
@ -32,9 +30,7 @@ describe("manifest", () => {
const manifest = getParsedManifest()
beforeAll(() => {
setJSRunner((js, context) => {
return vm.runInNewContext(js, context, { timeout: 1000 })
})
defaultJSSetup()
})
describe("examples are valid", () => {

1
packages/types/src/api/account/accounts.ts
View File

@ -12,7 +12,6 @@ export interface CreateAccountRequest {
name?: string
password: string
provider?: AccountSSOProvider
thirdPartyProfile: object
}
export interface SearchAccountsRequest {

2
packages/types/src/api/web/app/rowAction.ts
View File

@ -8,7 +8,7 @@ export interface RowActionResponse extends RowActionData {
id: string
tableId: string
automationId: string
allowedViews: string[] | undefined
allowedSources: string[] | undefined
}
export interface RowActionsResponse {

1
packages/types/src/api/web/auth.ts
View File

@ -21,6 +21,7 @@ export interface UpdateSelfRequest {
freeTrialConfirmedAt?: string
appFavourites?: string[]
tours?: Record<string, Date>
appSort?: string
}
export interface UpdateSelfResponse {

11
packages/types/src/documents/account/account.ts
View File

@ -7,10 +7,9 @@ export interface CreateAccount {
tenantId: string
hosting: Hosting
authType: AuthType
accountName: string
// optional fields - for sso based sign ups
registrationStep?: string
// profile
tenantName?: string
name?: string
size?: string
profession?: string
@ -20,11 +19,6 @@ export interface CreatePassswordAccount extends CreateAccount {
password: string
}
export interface CreateVerifiableSSOAccount extends CreateAccount {
provider?: AccountSSOProvider
thirdPartyProfile?: any
}
export const isCreatePasswordAccount = (
account: CreateAccount
): account is CreatePassswordAccount => account.authType === AuthType.PASSWORD
@ -56,6 +50,7 @@ export interface Account extends CreateAccount {
providerType?: AccountSSOProviderType
quotaUsage?: QuotaUsage
offlineLicenseToken?: string
tenantName?: string
}
export interface PasswordAccount extends Account {
@ -103,8 +98,6 @@ export interface AccountSSO {
provider: AccountSSOProvider
providerType: AccountSSOProviderType
oauth2?: OAuthTokens
pictureUrl?: string
thirdPartyProfile: any // TODO: define what the google profile looks like
}
export type SSOAccount = (Account | CloudAccount) & AccountSSO

10
packages/types/src/documents/app/rowAction.ts
View File

@ -8,8 +8,10 @@ export interface TableRowActions extends Document {
export interface RowActionData {
name: string
automationId: string
permissions: {
table: { runAllowed: boolean }
views: Record<string, { runAllowed: boolean }>
}
permissions: RowActionPermissions
}
export interface RowActionPermissions {
table: { runAllowed: boolean }
views: Record<string, { runAllowed: boolean }>
}

5
packages/types/src/documents/app/view.ts
View File

@ -79,10 +79,15 @@ export enum CalculationType {
MAX = "max",
}
export enum ViewV2Type {
CALCULATION = "calculation",
}
export interface ViewV2 {
version: 2
id: string
name: string
type?: ViewV2Type
primaryDisplay?: string
tableId: string
query?: LegacyFilter[] | SearchFilters

3
packages/types/src/documents/global/user.ts
View File

@ -21,7 +21,6 @@ export interface UserSSO {
provider: string // the individual provider e.g. Okta, Auth0, Google
providerType: SSOProviderType
oauth2?: OAuth2
thirdPartyProfile?: SSOProfileJson
profile?: {
displayName?: string
name?: {
@ -45,7 +44,6 @@ export interface User extends Document {
userId?: string
firstName?: string
lastName?: string
pictureUrl?: string
forceResetPassword?: boolean
roles: UserRoles
builder?: {
@ -67,6 +65,7 @@ export interface User extends Document {
scimInfo?: { isSync: true } & Record<string, any>
appFavourites?: string[]
ssoId?: string
appSort?: string
}
export enum UserStatus {

10
packages/worker/src/api/routes/global/tests/auditLogs.spec.ts
View File

@ -1,5 +1,5 @@
import { mocks, structures } from "@budibase/backend-core/tests"
import { context, events, setEnv as setCoreEnv } from "@budibase/backend-core"
import { context, events, features } from "@budibase/backend-core"
import { Event, IdentityType } from "@budibase/types"
import { TestConfiguration } from "../../../../tests"
@ -17,11 +17,9 @@ describe.each(["lucene", "sql"])("/api/global/auditlogs (%s)", method => {
let envCleanup: (() => void) | undefined
beforeAll(async () => {
if (method === "lucene") {
envCleanup = setCoreEnv({ TENANT_FEATURE_FLAGS: "*:!SQS" })
} else if (method === "sql") {
envCleanup = setCoreEnv({ TENANT_FEATURE_FLAGS: "*:SQS" })
}
envCleanup = features.testutils.setFeatureFlags("*", {
SQS: method === "sql",
})
await config.beforeAll()
})

19
packages/worker/src/api/routes/global/tests/users.spec.ts
View File

@ -741,6 +741,25 @@ describe("/api/global/users", () => {
it("should throw an error if public query performed", async () => {
await config.api.users.searchUsers({}, { status: 403, noHeaders: true })
})
it("should be able to search using logical conditions", async () => {
const user = await config.createUser()
const response = await config.api.users.searchUsers({
query: {
$and: {
conditions: [
{
$and: {
conditions: [{ string: { email: user.email } }],
},
},
],
},
},
})
expect(response.body.data.length).toBe(1)
expect(response.body.data[0].email).toBe(user.email)
})
})
describe("DELETE /api/global/users/:userId", () => {

1
packages/worker/src/api/routes/validation/users.ts
View File

@ -29,6 +29,7 @@ export const buildSelfSaveValidation = () => {
freeTrialConfirmedAt: Joi.string().optional(),
appFavourites: Joi.array().optional(),
tours: Joi.object().optional(),
appSort: Joi.string().optional(),
}
return auth.joiValidator.body(Joi.object(schema).required().unknown(false))
}