MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into v3-ui

This commit is contained in:
Adria Navarro 2024-10-09 17:32:21 +02:00
parent c1333c66db fab54b6ff3
commit 5ce619e9bc
55 changed files with 1276 additions and 676 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
lerna.json
View File

@ -1,6 +1,6 @@
{
"$schema": "node_modules/lerna/schemas/lerna-schema.json",
"version": "2.32.12",
"version": "2.32.15",
"npmClient": "yarn",
"packages": [
"packages/*",

9
packages/backend-core/src/context/mainContext.ts
View File

@ -253,6 +253,11 @@ export function getAppId(): string | undefined {
}
}
export function getIP(): string | undefined {
const context = Context.get()
return context?.ip
}
export const getProdAppId = () => {
const appId = getAppId()
if (!appId) {
@ -281,6 +286,10 @@ export function doInScimContext(task: any) {
return newContext(updates, task)
}
export function doInIPContext(ip: string, task: any) {
return newContext({ ip }, task)
}
export async function ensureSnippetContext(enabled = !env.isTest()) {
const ctx = getCurrentContext()

1
packages/backend-core/src/context/types.ts
View File

@ -9,6 +9,7 @@ export type ContextMap = {
identity?: IdentityContext
environmentVariables?: Record<string, string>
isScim?: boolean
ip?: string
automationId?: string
isMigrating?: boolean
vm?: VM

12
packages/backend-core/src/db/couch/DatabaseImpl.ts
View File

@ -213,17 +213,21 @@ export class DatabaseImpl implements Database {
async getMultiple<T extends Document>(
ids: string[],
opts?: { allowMissing?: boolean }
opts?: { allowMissing?: boolean; excludeDocs?: boolean }
): Promise<T[]> {
// get unique
ids = [...new Set(ids)]
const includeDocs = !opts?.excludeDocs
const response = await this.allDocs<T>({
keys: ids,
include_docs: true,
include_docs: includeDocs,
})
const rowUnavailable = (row: RowResponse<T>) => {
// row is deleted - key lookup can return this
if (row.doc == null || ("deleted" in row.value && row.value.deleted)) {
if (
(includeDocs && row.doc == null) ||
(row.value && "deleted" in row.value && row.value.deleted)
) {
return true
}
return row.error === "not_found"
@ -237,7 +241,7 @@ export class DatabaseImpl implements Database {
const missingIds = missing.map(row => row.key).join(", ")
throw new Error(`Unable to get documents: ${missingIds}`)
}
return rows.map(row => row.doc!)
return rows.map(row => (includeDocs ? row.doc! : row.value))
}
async remove(idOrDoc: string | Document, rev?: string) {

278
packages/backend-core/src/features/features.ts Normal file
View File

@ -0,0 +1,278 @@
import env from "../environment"
import * as crypto from "crypto"
import * as context from "../context"
import { PostHog, PostHogOptions } from "posthog-node"
import { FeatureFlag } from "@budibase/types"
import tracer from "dd-trace"
import { Duration } from "../utils"
let posthog: PostHog | undefined
export function init(opts?: PostHogOptions) {
if (
env.POSTHOG_TOKEN &&
env.POSTHOG_API_HOST &&
!env.SELF_HOSTED &&
env.POSTHOG_FEATURE_FLAGS_ENABLED
) {
console.log("initializing posthog client...")
posthog = new PostHog(env.POSTHOG_TOKEN, {
host: env.POSTHOG_API_HOST,
personalApiKey: env.POSTHOG_PERSONAL_TOKEN,
featureFlagsPollingInterval: Duration.fromMinutes(3).toMs(),
...opts,
})
} else {
console.log("posthog disabled")
}
}
export function shutdown() {
posthog?.shutdown()
}
export abstract class Flag<T> {
static boolean(defaultValue: boolean): Flag<boolean> {
return new BooleanFlag(defaultValue)
}
static string(defaultValue: string): Flag<string> {
return new StringFlag(defaultValue)
}
static number(defaultValue: number): Flag<number> {
return new NumberFlag(defaultValue)
}
protected constructor(public defaultValue: T) {}
abstract parse(value: any): T
}
type UnwrapFlag<F> = F extends Flag<infer U> ? U : never
export type FlagValues<T> = {
[K in keyof T]: UnwrapFlag<T[K]>
}
type KeysOfType<T, U> = {
[K in keyof T]: T[K] extends Flag<U> ? K : never
}[keyof T]
class BooleanFlag extends Flag<boolean> {
parse(value: any) {
if (typeof value === "string") {
return ["true", "t", "1"].includes(value.toLowerCase())
}
if (typeof value === "boolean") {
return value
}
throw new Error(`could not parse value "${value}" as boolean`)
}
}
class StringFlag extends Flag<string> {
parse(value: any) {
if (typeof value === "string") {
return value
}
throw new Error(`could not parse value "${value}" as string`)
}
}
class NumberFlag extends Flag<number> {
parse(value: any) {
if (typeof value === "number") {
return value
}
if (typeof value === "string") {
const parsed = parseFloat(value)
if (!isNaN(parsed)) {
return parsed
}
}
throw new Error(`could not parse value "${value}" as number`)
}
}
export interface EnvFlagEntry {
tenantId: string
key: string
value: boolean
}
export function parseEnvFlags(flags: string): EnvFlagEntry[] {
const split = flags.split(",").map(x => x.split(":"))
const result: EnvFlagEntry[] = []
for (const [tenantId, ...features] of split) {
for (let feature of features) {
let value = true
if (feature.startsWith("!")) {
feature = feature.slice(1)
value = false
}
result.push({ tenantId, key: feature, value })
}
}
return result
}
export class FlagSet<V extends Flag<any>, T extends { [key: string]: V }> {
// This is used to safely cache flags sets in the current request context.
// Because multiple sets could theoretically exist, we don't want the cache of
// one to leak into another.
private readonly setId: string
constructor(private readonly flagSchema: T) {
this.setId = crypto.randomUUID()
}
defaults(): FlagValues<T> {
return Object.keys(this.flagSchema).reduce((acc, key) => {
const typedKey = key as keyof T
acc[typedKey] = this.flagSchema[key].defaultValue
return acc
}, {} as FlagValues<T>)
}
isFlagName(name: string | number | symbol): name is keyof T {
return this.flagSchema[name as keyof T] !== undefined
}
async get<K extends keyof T>(key: K): Promise<FlagValues<T>[K]> {
const flags = await this.fetch()
return flags[key]
}
async isEnabled<K extends KeysOfType<T, boolean>>(key: K): Promise<boolean> {
const flags = await this.fetch()
return flags[key]
}
async fetch(): Promise<FlagValues<T>> {
return await tracer.trace("features.fetch", async span => {
const cachedFlags = context.getFeatureFlags<FlagValues<T>>(this.setId)
if (cachedFlags) {
span?.addTags({ fromCache: true })
return cachedFlags
}
const tags: Record<string, any> = {}
const flagValues = this.defaults()
const currentTenantId = context.getTenantId()
const specificallySetFalse = new Set<string>()
for (const { tenantId, key, value } of parseEnvFlags(
env.TENANT_FEATURE_FLAGS || ""
)) {
if (!tenantId || (tenantId !== "*" && tenantId !== currentTenantId)) {
continue
}
tags[`readFromEnvironmentVars`] = true
if (value === false) {
specificallySetFalse.add(key)
}
// ignore unknown flags
if (!this.isFlagName(key)) {
continue
}
if (typeof flagValues[key] !== "boolean") {
throw new Error(`Feature: ${key} is not a boolean`)
}
// @ts-expect-error - TS does not like you writing into a generic type,
// but we know that it's okay in this case because it's just an object.
flagValues[key as keyof FlagValues] = value
tags[`flags.${key}.source`] = "environment"
}
const identity = context.getIdentity()
let userId = identity?._id
if (!userId) {
const ip = context.getIP()
if (ip) {
userId = crypto.createHash("sha512").update(ip).digest("hex")
}
}
let tenantId = identity?.tenantId
if (!tenantId) {
tenantId = currentTenantId
}
tags[`identity.type`] = identity?.type
tags[`identity._id`] = identity?._id
tags[`tenantId`] = tenantId
tags[`userId`] = userId
if (posthog && userId) {
tags[`readFromPostHog`] = true
const personProperties: Record<string, string> = { tenantId }
const posthogFlags = await posthog.getAllFlagsAndPayloads(userId, {
personProperties,
})
for (const [name, value] of Object.entries(posthogFlags.featureFlags)) {
if (!this.isFlagName(name)) {
// We don't want an unexpected PostHog flag to break the app, so we
// just log it and continue.
console.warn(`Unexpected posthog flag "${name}": ${value}`)
continue
}
if (flagValues[name] === true || specificallySetFalse.has(name)) {
// If the flag is already set to through environment variables, we
// don't want to override it back to false here.
continue
}
const payload = posthogFlags.featureFlagPayloads?.[name]
const flag = this.flagSchema[name]
try {
// @ts-expect-error - TS does not like you writing into a generic
// type, but we know that it's okay in this case because it's just
// an object.
flagValues[name] = flag.parse(payload || value)
tags[`flags.${name}.source`] = "posthog"
} catch (err) {
// We don't want an invalid PostHog flag to break the app, so we just
// log it and continue.
console.warn(`Error parsing posthog flag "${name}": ${value}`, err)
}
}
}
context.setFeatureFlags(this.setId, flagValues)
for (const [key, value] of Object.entries(flagValues)) {
tags[`flags.${key}.value`] = value
}
span?.addTags(tags)
return flagValues
})
}
}
// This is the primary source of truth for feature flags. If you want to add a
// new flag, add it here and use the `fetch` and `get` functions to access it.
// All of the machinery in this file is to make sure that flags have their
// default values set correctly and their types flow through the system.
export const flags = new FlagSet({
DEFAULT_VALUES: Flag.boolean(env.isDev()),
AUTOMATION_BRANCHING: Flag.boolean(env.isDev()),
SQS: Flag.boolean(env.isDev()),
[FeatureFlag.AI_CUSTOM_CONFIGS]: Flag.boolean(env.isDev()),
[FeatureFlag.ENRICHED_RELATIONSHIPS]: Flag.boolean(env.isDev()),
})
type UnwrapPromise<T> = T extends Promise<infer U> ? U : T
export type FeatureFlags = UnwrapPromise<ReturnType<typeof flags.fetch>>

283
packages/backend-core/src/features/index.ts
View File

@ -1,281 +1,2 @@
import env from "../environment"
import * as context from "../context"
import { PostHog, PostHogOptions } from "posthog-node"
import { FeatureFlag, IdentityType, UserCtx } from "@budibase/types"
import tracer from "dd-trace"
import { Duration } from "../utils"
let posthog: PostHog | undefined
export function init(opts?: PostHogOptions) {
if (
env.POSTHOG_TOKEN &&
env.POSTHOG_API_HOST &&
!env.SELF_HOSTED &&
env.POSTHOG_FEATURE_FLAGS_ENABLED
) {
console.log("initializing posthog client...")
posthog = new PostHog(env.POSTHOG_TOKEN, {
host: env.POSTHOG_API_HOST,
personalApiKey: env.POSTHOG_PERSONAL_TOKEN,
featureFlagsPollingInterval: Duration.fromMinutes(3).toMs(),
...opts,
})
} else {
console.log("posthog disabled")
}
}
export function shutdown() {
posthog?.shutdown()
}
export abstract class Flag<T> {
static boolean(defaultValue: boolean): Flag<boolean> {
return new BooleanFlag(defaultValue)
}
static string(defaultValue: string): Flag<string> {
return new StringFlag(defaultValue)
}
static number(defaultValue: number): Flag<number> {
return new NumberFlag(defaultValue)
}
protected constructor(public defaultValue: T) {}
abstract parse(value: any): T
}
type UnwrapFlag<F> = F extends Flag<infer U> ? U : never
export type FlagValues<T> = {
[K in keyof T]: UnwrapFlag<T[K]>
}
type KeysOfType<T, U> = {
[K in keyof T]: T[K] extends Flag<U> ? K : never
}[keyof T]
class BooleanFlag extends Flag<boolean> {
parse(value: any) {
if (typeof value === "string") {
return ["true", "t", "1"].includes(value.toLowerCase())
}
if (typeof value === "boolean") {
return value
}
throw new Error(`could not parse value "${value}" as boolean`)
}
}
class StringFlag extends Flag<string> {
parse(value: any) {
if (typeof value === "string") {
return value
}
throw new Error(`could not parse value "${value}" as string`)
}
}
class NumberFlag extends Flag<number> {
parse(value: any) {
if (typeof value === "number") {
return value
}
if (typeof value === "string") {
const parsed = parseFloat(value)
if (!isNaN(parsed)) {
return parsed
}
}
throw new Error(`could not parse value "${value}" as number`)
}
}
export class FlagSet<V extends Flag<any>, T extends { [key: string]: V }> {
// This is used to safely cache flags sets in the current request context.
// Because multiple sets could theoretically exist, we don't want the cache of
// one to leak into another.
private readonly setId: string
constructor(private readonly flagSchema: T) {
this.setId = crypto.randomUUID()
}
defaults(): FlagValues<T> {
return Object.keys(this.flagSchema).reduce((acc, key) => {
const typedKey = key as keyof T
acc[typedKey] = this.flagSchema[key].defaultValue
return acc
}, {} as FlagValues<T>)
}
isFlagName(name: string | number | symbol): name is keyof T {
return this.flagSchema[name as keyof T] !== undefined
}
async get<K extends keyof T>(
key: K,
ctx?: UserCtx
): Promise<FlagValues<T>[K]> {
const flags = await this.fetch(ctx)
return flags[key]
}
async isEnabled<K extends KeysOfType<T, boolean>>(
key: K,
ctx?: UserCtx
): Promise<boolean> {
const flags = await this.fetch(ctx)
return flags[key]
}
async fetch(ctx?: UserCtx): Promise<FlagValues<T>> {
return await tracer.trace("features.fetch", async span => {
const cachedFlags = context.getFeatureFlags<FlagValues<T>>(this.setId)
if (cachedFlags) {
span?.addTags({ fromCache: true })
return cachedFlags
}
const tags: Record<string, any> = {}
const flagValues = this.defaults()
const currentTenantId = context.getTenantId()
const specificallySetFalse = new Set<string>()
const split = (env.TENANT_FEATURE_FLAGS || "")
.split(",")
.map(x => x.split(":"))
for (const [tenantId, ...features] of split) {
if (!tenantId || (tenantId !== "*" && tenantId !== currentTenantId)) {
continue
}
tags[`readFromEnvironmentVars`] = true
for (let feature of features) {
let value = true
if (feature.startsWith("!")) {
feature = feature.slice(1)
value = false
specificallySetFalse.add(feature)
}
// ignore unknown flags
if (!this.isFlagName(feature)) {
continue
}
if (typeof flagValues[feature] !== "boolean") {
throw new Error(`Feature: ${feature} is not a boolean`)
}
// @ts-expect-error - TS does not like you writing into a generic type,
// but we know that it's okay in this case because it's just an object.
flagValues[feature as keyof FlagValues] = value
tags[`flags.${feature}.source`] = "environment"
}
}
const license = ctx?.user?.license
if (license) {
tags[`readFromLicense`] = true
for (const feature of license.features) {
if (!this.isFlagName(feature)) {
continue
}
if (
flagValues[feature] === true ||
specificallySetFalse.has(feature)
) {
// If the flag is already set to through environment variables, we
// don't want to override it back to false here.
continue
}
// @ts-expect-error - TS does not like you writing into a generic type,
// but we know that it's okay in this case because it's just an object.
flagValues[feature] = true
tags[`flags.${feature}.source`] = "license"
}
}
const identity = context.getIdentity()
tags[`identity.type`] = identity?.type
tags[`identity.tenantId`] = identity?.tenantId
tags[`identity._id`] = identity?._id
if (posthog && identity?.type === IdentityType.USER) {
tags[`readFromPostHog`] = true
const personProperties: Record<string, string> = {}
if (identity.tenantId) {
personProperties.tenantId = identity.tenantId
}
const posthogFlags = await posthog.getAllFlagsAndPayloads(
identity._id,
{
personProperties,
}
)
for (const [name, value] of Object.entries(posthogFlags.featureFlags)) {
if (!this.isFlagName(name)) {
// We don't want an unexpected PostHog flag to break the app, so we
// just log it and continue.
console.warn(`Unexpected posthog flag "${name}": ${value}`)
continue
}
if (flagValues[name] === true || specificallySetFalse.has(name)) {
// If the flag is already set to through environment variables, we
// don't want to override it back to false here.
continue
}
const payload = posthogFlags.featureFlagPayloads?.[name]
const flag = this.flagSchema[name]
try {
// @ts-expect-error - TS does not like you writing into a generic
// type, but we know that it's okay in this case because it's just
// an object.
flagValues[name] = flag.parse(payload || value)
tags[`flags.${name}.source`] = "posthog"
} catch (err) {
// We don't want an invalid PostHog flag to break the app, so we just
// log it and continue.
console.warn(`Error parsing posthog flag "${name}": ${value}`, err)
}
}
}
context.setFeatureFlags(this.setId, flagValues)
for (const [key, value] of Object.entries(flagValues)) {
tags[`flags.${key}.value`] = value
}
span?.addTags(tags)
return flagValues
})
}
}
// This is the primary source of truth for feature flags. If you want to add a
// new flag, add it here and use the `fetch` and `get` functions to access it.
// All of the machinery in this file is to make sure that flags have their
// default values set correctly and their types flow through the system.
export const flags = new FlagSet({
DEFAULT_VALUES: Flag.boolean(env.isDev()),
AUTOMATION_BRANCHING: Flag.boolean(env.isDev()),
SQS: Flag.boolean(env.isDev()),
[FeatureFlag.AI_CUSTOM_CONFIGS]: Flag.boolean(env.isDev()),
[FeatureFlag.ENRICHED_RELATIONSHIPS]: Flag.boolean(env.isDev()),
})
export * from "./features"
export * as testutils from "./tests/utils"

90
packages/backend-core/src/features/tests/features.spec.ts
View File

@ -1,9 +1,10 @@
import { IdentityContext, IdentityType, UserCtx } from "@budibase/types"
import { IdentityContext, IdentityType } from "@budibase/types"
import { Flag, FlagSet, FlagValues, init, shutdown } from "../"
import * as context from "../../context"
import environment, { withEnv } from "../../environment"
import nodeFetch from "node-fetch"
import nock from "nock"
import * as crypto from "crypto"
const schema = {
TEST_BOOLEAN: Flag.boolean(false),
@ -17,7 +18,6 @@ interface TestCase {
identity?: Partial<IdentityContext>
environmentFlags?: string
posthogFlags?: PostHogFlags
licenseFlags?: Array<string>
expected?: Partial<FlagValues<typeof schema>>
errorMessage?: string | RegExp
}
@ -27,10 +27,14 @@ interface PostHogFlags {
featureFlagPayloads?: Record<string, string>
}
function mockPosthogFlags(flags: PostHogFlags) {
function mockPosthogFlags(
flags: PostHogFlags,
opts?: { token?: string; distinct_id?: string }
) {
const { token = "test", distinct_id = "us_1234" } = opts || {}
nock("https://us.i.posthog.com")
.post("/decide/?v=3", body => {
return body.token === "test" && body.distinct_id === "us_1234"
return body.token === token && body.distinct_id === distinct_id
})
.reply(200, flags)
.persist()
@ -112,17 +116,6 @@ describe("feature flags", () => {
},
expected: { TEST_BOOLEAN: true },
},
{
it: "should be able to set boolean flags through the license",
licenseFlags: ["TEST_BOOLEAN"],
expected: { TEST_BOOLEAN: true },
},
{
it: "should not be able to override a negative environment flag from license",
environmentFlags: "default:!TEST_BOOLEAN",
licenseFlags: ["TEST_BOOLEAN"],
expected: { TEST_BOOLEAN: false },
},
{
it: "should not error on unrecognised PostHog flag",
posthogFlags: {
@ -130,18 +123,12 @@ describe("feature flags", () => {
},
expected: flags.defaults(),
},
{
it: "should not error on unrecognised license flag",
licenseFlags: ["UNDEFINED"],
expected: flags.defaults(),
},
])(
"$it",
async ({
identity,
environmentFlags,
posthogFlags,
licenseFlags,
expected,
errorMessage,
}) => {
@ -157,8 +144,6 @@ describe("feature flags", () => {
env.POSTHOG_API_HOST = "https://us.i.posthog.com"
}
const ctx = { user: { license: { features: licenseFlags || [] } } }
await withEnv(env, async () => {
// We need to pass in node-fetch here otherwise nock won't get used
// because posthog-node uses axios under the hood.
@ -180,18 +165,13 @@ describe("feature flags", () => {
await context.doInIdentityContext(fullIdentity, async () => {
if (errorMessage) {
await expect(flags.fetch(ctx as UserCtx)).rejects.toThrow(
errorMessage
)
await expect(flags.fetch()).rejects.toThrow(errorMessage)
} else if (expected) {
const values = await flags.fetch(ctx as UserCtx)
const values = await flags.fetch()
expect(values).toMatchObject(expected)
for (const [key, expectedValue] of Object.entries(expected)) {
const value = await flags.get(
key as keyof typeof schema,
ctx as UserCtx
)
const value = await flags.get(key as keyof typeof schema)
expect(value).toBe(expectedValue)
}
} else {
@ -214,6 +194,14 @@ describe("feature flags", () => {
lastName: "User",
}
// We need to pass in node-fetch here otherwise nock won't get used
// because posthog-node uses axios under the hood.
init({
fetch: (url, opts) => {
return nodeFetch(url, opts)
},
})
nock("https://us.i.posthog.com")
.post("/decide/?v=3", body => {
return body.token === "test" && body.distinct_id === "us_1234"
@ -230,4 +218,44 @@ describe("feature flags", () => {
}
)
})
it("should still get flags when user is logged out", async () => {
const env: Partial<typeof environment> = {
SELF_HOSTED: false,
POSTHOG_FEATURE_FLAGS_ENABLED: "true",
POSTHOG_API_HOST: "https://us.i.posthog.com",
POSTHOG_TOKEN: "test",
}
const ip = "127.0.0.1"
const hashedIp = crypto.createHash("sha512").update(ip).digest("hex")
await withEnv(env, async () => {
mockPosthogFlags(
{
featureFlags: { TEST_BOOLEAN: true },
},
{
distinct_id: hashedIp,
}
)
// We need to pass in node-fetch here otherwise nock won't get used
// because posthog-node uses axios under the hood.
init({
fetch: (url, opts) => {
return nodeFetch(url, opts)
},
})
await context.doInIPContext(ip, async () => {
await context.doInTenant("default", async () => {
const result = await flags.fetch()
expect(result.TEST_BOOLEAN).toBe(true)
})
})
shutdown()
})
})
})

64
packages/backend-core/src/features/tests/utils.ts Normal file
View File

@ -0,0 +1,64 @@
import { FeatureFlags, parseEnvFlags } from ".."
import { setEnv } from "../../environment"
function getCurrentFlags(): Record<string, Record<string, boolean>> {
const result: Record<string, Record<string, boolean>> = {}
for (const { tenantId, key, value } of parseEnvFlags(
process.env.TENANT_FEATURE_FLAGS || ""
)) {
const tenantFlags = result[tenantId] || {}
// Don't allow overwriting specifically false flags, to match the beheaviour
// of FlagSet.
if (tenantFlags[key] === false) {
continue
}
tenantFlags[key] = value
result[tenantId] = tenantFlags
}
return result
}
function buildFlagString(
flags: Record<string, Record<string, boolean>>
): string {
const parts: string[] = []
for (const [tenantId, tenantFlags] of Object.entries(flags)) {
for (const [key, value] of Object.entries(tenantFlags)) {
if (value === false) {
parts.push(`${tenantId}:!${key}`)
} else {
parts.push(`${tenantId}:${key}`)
}
}
}
return parts.join(",")
}
export function setFeatureFlags(
tenantId: string,
flags: Partial<FeatureFlags>
): () => void {
const current = getCurrentFlags()
for (const [key, value] of Object.entries(flags)) {
const tenantFlags = current[tenantId] || {}
tenantFlags[key] = value
current[tenantId] = tenantFlags
}
const flagString = buildFlagString(current)
return setEnv({ TENANT_FEATURE_FLAGS: flagString })
}
export function withFeatureFlags<T>(
tenantId: string,
flags: Partial<FeatureFlags>,
f: () => T
) {
const cleanup = setFeatureFlags(tenantId, flags)
const result = f()
if (result instanceof Promise) {
return result.finally(cleanup)
} else {
cleanup()
return result
}
}

1
packages/backend-core/src/middleware/index.ts
View File

@ -20,3 +20,4 @@ export { default as correlation } from "../logging/correlation/middleware"
export { default as errorHandling } from "./errorHandling"
export { default as querystringToBody } from "./querystringToBody"
export * as joiValidator from "./joi-validator"
export { default as ip } from "./ip"

12
packages/backend-core/src/middleware/ip.ts Normal file
View File

@ -0,0 +1,12 @@
import { Ctx } from "@budibase/types"
import { doInIPContext } from "../context"
export default async (ctx: Ctx, next: any) => {
if (ctx.ip) {
return await doInIPContext(ctx.ip, () => {
return next()
})
} else {
return next()
}
}

8
packages/backend-core/src/security/permissions.ts
View File

@ -65,7 +65,13 @@ export enum BuiltinPermissionID {
POWER = "power",
}
export const BUILTIN_PERMISSIONS = {
export const BUILTIN_PERMISSIONS: {
[key in keyof typeof BuiltinPermissionID]: {
_id: (typeof BuiltinPermissionID)[key]
name: string
permissions: Permission[]
}
} = {
PUBLIC: {
_id: BuiltinPermissionID.PUBLIC,
name: "Public",

12
packages/backend-core/src/users/users.ts
View File

@ -24,6 +24,7 @@ import * as context from "../context"
import { getGlobalDB } from "../context"
import { isCreator } from "./utils"
import { UserDB } from "./db"
import { dataFilters } from "@budibase/shared-core"
type GetOpts = { cleanup?: boolean }
@ -262,10 +263,17 @@ export async function paginatedUsers({
userList = await bulkGetGlobalUsersById(query?.oneOf?._id, {
cleanup: true,
})
} else if (query) {
// TODO: this should use SQS search, but the logic is built in the 'server' package. Using the in-memory filtering to get this working meanwhile
const response = await db.allDocs<User>(
getGlobalUserParams(null, { ...opts, limit: undefined })
)
userList = response.rows.map(row => row.doc!)
userList = dataFilters.search(userList, { query, limit: opts.limit }).rows
} else {
// no search, query allDocs
const response = await db.allDocs(getGlobalUserParams(null, opts))
userList = response.rows.map((row: any) => row.doc)
const response = await db.allDocs<User>(getGlobalUserParams(null, opts))
userList = response.rows.map(row => row.doc!)
}
return pagination(userList, pageSize, {
paginate: true,

23
packages/server/src/api/controllers/permission.ts
View File

@ -1,7 +1,6 @@
import { permissions, roles, context } from "@budibase/backend-core"
import {
UserCtx,
Role,
GetResourcePermsResponse,
ResourcePermissionInfo,
GetDependantResourcesResponse,
@ -9,6 +8,7 @@ import {
AddPermissionRequest,
RemovePermissionRequest,
RemovePermissionResponse,
FetchResourcePermissionInfoResponse,
} from "@budibase/types"
import {
CURRENTLY_SUPPORTED_LEVELS,
@ -28,10 +28,12 @@ export function fetchLevels(ctx: UserCtx) {
ctx.body = SUPPORTED_LEVELS
}
export async function fetch(ctx: UserCtx) {
export async function fetch(
ctx: UserCtx<void, FetchResourcePermissionInfoResponse>
) {
const db = context.getAppDB()
const dbRoles: Role[] = await sdk.permissions.getAllDBRoles(db)
let permissions: any = {}
const dbRoles = await sdk.permissions.getAllDBRoles(db)
let permissions: Record<string, Record<string, string>> = {}
// create an object with structure role ID -> resource ID -> level
for (let role of dbRoles) {
if (!role.permissions) {
@ -43,13 +45,13 @@ export async function fetch(ctx: UserCtx) {
}
for (let [resource, levelArr] of Object.entries(role.permissions)) {
const levels: string[] = Array.isArray(levelArr) ? levelArr : [levelArr]
const perms: Record<string, string> = {}
const perms: Record<string, string> = permissions[resource] || {}
levels.forEach(level => (perms[level] = roleId!))
permissions[resource] = perms
}
}
// apply the base permissions
const finalPermissions: Record<string, Record<string, string>> = {}
const finalPermissions: FetchResourcePermissionInfoResponse = {}
for (let [resource, permission] of Object.entries(permissions)) {
const basePerms = getBasePermissions(resource)
finalPermissions[resource] = Object.assign(basePerms, permission)
@ -92,18 +94,17 @@ export async function getDependantResources(
export async function addPermission(ctx: UserCtx<void, AddPermissionResponse>) {
const params: AddPermissionRequest = ctx.params
ctx.body = await sdk.permissions.updatePermissionOnRole(
params,
PermissionUpdateType.ADD
)
await sdk.permissions.updatePermissionOnRole(params, PermissionUpdateType.ADD)
ctx.status = 200
}
export async function removePermission(
ctx: UserCtx<void, RemovePermissionResponse>
) {
const params: RemovePermissionRequest = ctx.params
ctx.body = await sdk.permissions.updatePermissionOnRole(
await sdk.permissions.updatePermissionOnRole(
params,
PermissionUpdateType.REMOVE
)
ctx.status = 200
}

6
packages/server/src/api/controllers/row/external.ts
View File

@ -27,6 +27,7 @@ import {
} from "../../../utilities/rowProcessor"
import { cloneDeep } from "lodash"
import { generateIdForRow } from "./utils"
import { helpers } from "@budibase/shared-core"
export async function handleRequest<T extends Operation>(
operation: T,
@ -42,6 +43,11 @@ export async function handleRequest<T extends Operation>(
export async function patch(ctx: UserCtx<PatchRowRequest, PatchRowResponse>) {
const source = await utils.getSource(ctx)
if (sdk.views.isView(source) && helpers.views.isCalculationView(source)) {
ctx.throw(400, "Cannot update rows through a calculation view")
}
const table = await utils.getTableFromSource(source)
const { _id, ...rowData } = ctx.request.body

7
packages/server/src/api/controllers/row/internal.ts
View File

@ -22,13 +22,20 @@ import sdk from "../../../sdk"
import { getLinkedTableIDs } from "../../../db/linkedRows/linkUtils"
import { flatten } from "lodash"
import { findRow } from "../../../sdk/app/rows/internal"
import { helpers } from "@budibase/shared-core"
export async function patch(ctx: UserCtx<PatchRowRequest, PatchRowResponse>) {
const { tableId } = utils.getSourceId(ctx)
const source = await utils.getSource(ctx)
if (sdk.views.isView(source) && helpers.views.isCalculationView(source)) {
ctx.throw(400, "Cannot update rows through a calculation view")
}
const table = sdk.views.isView(source)
? await sdk.views.getTable(source.id)
: source
const inputs = ctx.request.body
const isUserTable = tableId === InternalTables.USER_METADATA
let oldRow

2
packages/server/src/api/controllers/table/external.ts
View File

@ -31,7 +31,7 @@ function getDatasourceId(table: Table) {
return breakExternalTableId(table._id).datasourceId
}
export async function save(
export async function updateTable(
ctx: UserCtx<SaveTableRequest, SaveTableResponse>,
renaming?: RenameColumn
) {

14
packages/server/src/api/controllers/table/index.ts
View File

@ -102,18 +102,22 @@ export async function find(ctx: UserCtx<void, TableResponse>) {
export async function save(ctx: UserCtx<SaveTableRequest, SaveTableResponse>) {
const appId = ctx.appId
const table = ctx.request.body
const isImport = table.rows
const { rows, ...table } = ctx.request.body
const isImport = rows
const renaming = ctx.request.body._rename
const isCreate = !table._id
checkDefaultFields(table)
const api = pickApi({ table })
let savedTable = await api.save(ctx, renaming)
if (!table._id) {
let savedTable: Table
if (isCreate) {
savedTable = await sdk.tables.create(table, rows, ctx.user._id)
savedTable = await sdk.tables.enrichViewSchemas(savedTable)
await events.table.created(savedTable)
} else {
const api = pickApi({ table })
savedTable = await api.updateTable(ctx, renaming)
await events.table.updated(savedTable)
}
if (renaming) {

9
packages/server/src/api/controllers/table/internal.ts
View File

@ -12,7 +12,7 @@ import {
} from "@budibase/types"
import sdk from "../../../sdk"
export async function save(
export async function updateTable(
ctx: UserCtx<SaveTableRequest, SaveTableResponse>,
renaming?: RenameColumn
) {
@ -25,19 +25,16 @@ export async function save(
sourceType: rest.sourceType || TableSourceType.INTERNAL,
}
const isImport = !!rows
if (!tableToSave.views) {
tableToSave.views = {}
}
try {
const { table } = await sdk.tables.internal.save(tableToSave, {
user: ctx.user,
userId: ctx.user._id,
rowsToImport: rows,
tableId: ctx.request.body._id,
renaming,
isImport,
})
return table
@ -72,7 +69,7 @@ export async function bulkImport(
await handleDataImport(table, {
importRows: rows,
identifierFields,
user: ctx.user,
userId: ctx.user._id,
})
return table
}

2
packages/server/src/api/controllers/table/tests/utils.spec.ts
View File

@ -41,7 +41,7 @@ describe("utils", () => {
const data = [{ name: "Alice" }, { name: "Bob" }, { name: "Claire" }]
const result = await importToRows(data, table, config.user)
const result = await importToRows(data, table, config.user?._id)
expect(result).toEqual([
expect.objectContaining({
autoId: 1,

20
packages/server/src/api/controllers/table/utils.ts
View File

@ -18,7 +18,6 @@ import { quotas } from "@budibase/pro"
import { events, context, features } from "@budibase/backend-core"
import {
AutoFieldSubType,
ContextUser,
Datasource,
Row,
SourceName,
@ -122,7 +121,7 @@ export function makeSureTableUpToDate(table: Table, tableToSave: Table) {
export async function importToRows(
data: Row[],
table: Table,
user?: ContextUser,
userId?: string,
opts?: { keepCouchId: boolean }
) {
const originalTable = table
@ -136,7 +135,7 @@ export async function importToRows(
// We use a reference to table here and update it after input processing,
// so that we can auto increment auto IDs in imported data properly
const processed = await inputProcessing(user?._id, table, row, {
const processed = await inputProcessing(userId, table, row, {
noAutoRelationships: true,
})
row = processed
@ -167,11 +166,10 @@ export async function importToRows(
export async function handleDataImport(
table: Table,
opts?: { identifierFields?: string[]; user?: ContextUser; importRows?: Row[] }
opts?: { identifierFields?: string[]; userId?: string; importRows?: Row[] }
) {
const schema = table.schema
const identifierFields = opts?.identifierFields || []
const user = opts?.user
const importRows = opts?.importRows
if (!importRows || !isRows(importRows) || !isSchema(schema)) {
@ -181,7 +179,7 @@ export async function handleDataImport(
const db = context.getAppDB()
const data = parse(importRows, table)
const finalData = await importToRows(data, table, user, {
const finalData = await importToRows(data, table, opts?.userId, {
keepCouchId: identifierFields.includes("_id"),
})
@ -282,22 +280,22 @@ export function checkStaticTables(table: Table) {
class TableSaveFunctions {
db: Database
user?: ContextUser
userId?: string
oldTable?: Table
importRows?: Row[]
rows: Row[]
constructor({
user,
userId,
oldTable,
importRows,
}: {
user?: ContextUser
userId?: string
oldTable?: Table
importRows?: Row[]
}) {
this.db = context.getAppDB()
this.user = user
this.userId = userId
this.oldTable = oldTable
this.importRows = importRows
// any rows that need updated
@ -329,7 +327,7 @@ class TableSaveFunctions {
table = await handleSearchIndexes(table)
table = await handleDataImport(table, {
importRows: this.importRows,
user: this.user,
userId: this.userId,
})
if (await features.flags.isEnabled("SQS")) {
await sdk.tables.sqs.addTable(table)

17
packages/server/src/api/routes/tests/application.spec.ts
View File

@ -14,12 +14,7 @@ jest.mock("../../../utilities/redis", () => ({
import { checkBuilderEndpoint } from "./utilities/TestFunctions"
import * as setup from "./utilities"
import { AppStatus } from "../../../db/utils"
import {
events,
utils,
context,
withEnv as withCoreEnv,
} from "@budibase/backend-core"
import { events, utils, context, features } from "@budibase/backend-core"
import env from "../../../environment"
import { type App } from "@budibase/types"
import tk from "timekeeper"
@ -358,9 +353,13 @@ describe("/applications", () => {
.delete(`/api/global/roles/${prodAppId}`)
.reply(200, {})
await withCoreEnv({ TENANT_FEATURE_FLAGS: "*:SQS" }, async () => {
await config.api.application.delete(app.appId)
})
await features.testutils.withFeatureFlags(
"*",
{ SQS: true },
async () => {
await config.api.application.delete(app.appId)
}
)
})
})

379
packages/server/src/api/routes/tests/permissions.spec.ts
View File

@ -1,5 +1,5 @@
import { roles } from "@budibase/backend-core"
import { Document, PermissionLevel, Row, Table, ViewV2 } from "@budibase/types"
import { Document, PermissionLevel, Row } from "@budibase/types"
import * as setup from "./utilities"
import { generator, mocks } from "@budibase/backend-core/tests"
@ -9,13 +9,11 @@ const { BUILTIN_ROLE_IDS } = roles
const HIGHER_ROLE_ID = BUILTIN_ROLE_IDS.BASIC
const STD_ROLE_ID = BUILTIN_ROLE_IDS.PUBLIC
const DEFAULT_TABLE_ROLE_ID = BUILTIN_ROLE_IDS.ADMIN
describe("/permission", () => {
let request = setup.getRequest()
let config = setup.getConfig()
let table: Table & { _id: string }
let perms: Document[]
let row: Row
let view: ViewV2
afterAll(setup.afterAll)
@ -25,18 +23,6 @@ describe("/permission", () => {
beforeEach(async () => {
mocks.licenses.useCloudFree()
table = (await config.createTable()) as typeof table
row = await config.createRow()
view = await config.api.viewV2.create({
tableId: table._id!,
name: generator.guid(),
})
perms = await config.api.permission.add({
roleId: STD_ROLE_ID,
resourceId: table._id,
level: PermissionLevel.READ,
})
})
describe("levels", () => {
@ -54,134 +40,251 @@ describe("/permission", () => {
})
})
describe("add", () => {
it("should be able to add permission to a role for the table", async () => {
expect(perms.length).toEqual(1)
expect(perms[0]._id).toEqual(`${STD_ROLE_ID}`)
})
describe("table permissions", () => {
let tableId: string
it("should get the resource permissions", async () => {
const res = await request
.get(`/api/permission/${table._id}`)
.set(config.defaultHeaders())
.expect("Content-Type", /json/)
.expect(200)
expect(res.body).toEqual({
permissions: {
read: { permissionType: "EXPLICIT", role: STD_ROLE_ID },
write: { permissionType: "BASE", role: HIGHER_ROLE_ID },
},
})
})
it("should get resource permissions with multiple roles", async () => {
perms = await config.api.permission.add({
roleId: HIGHER_ROLE_ID,
resourceId: table._id,
level: PermissionLevel.WRITE,
})
const res = await config.api.permission.get(table._id)
expect(res).toEqual({
permissions: {
read: { permissionType: "EXPLICIT", role: STD_ROLE_ID },
write: { permissionType: "EXPLICIT", role: HIGHER_ROLE_ID },
},
})
const allRes = await request
.get(`/api/permission`)
.set(config.defaultHeaders())
.expect("Content-Type", /json/)
.expect(200)
expect(allRes.body[table._id]["read"]).toEqual(STD_ROLE_ID)
expect(allRes.body[table._id]["write"]).toEqual(HIGHER_ROLE_ID)
})
})
describe("remove", () => {
it("should be able to remove the permission", async () => {
const res = await config.api.permission.revoke({
roleId: STD_ROLE_ID,
resourceId: table._id,
level: PermissionLevel.READ,
})
expect(res[0]._id).toEqual(STD_ROLE_ID)
const permsRes = await config.api.permission.get(table._id)
expect(permsRes.permissions[STD_ROLE_ID]).toBeUndefined()
})
})
describe("check public user allowed", () => {
it("should be able to read the row", async () => {
// replicate changes before checking permissions
await config.publish()
const res = await request
.get(`/api/${table._id}/rows`)
.set(config.publicHeaders())
.expect("Content-Type", /json/)
.expect(200)
expect(res.body[0]._id).toEqual(row._id)
})
it("should be able to access the view data when the table is set to public and with no view permissions overrides", async () => {
// Make view inherit table permissions. Needed for backwards compatibility with existing views.
await config.api.permission.revoke({
roleId: STD_ROLE_ID,
resourceId: view.id,
level: PermissionLevel.READ,
})
// replicate changes before checking permissions
await config.publish()
const res = await config.api.viewV2.publicSearch(view.id)
expect(res.rows[0]._id).toEqual(row._id)
})
it("should not be able to access the view data when the table is not public and there are no view permissions overrides", async () => {
await config.api.permission.revoke({
roleId: STD_ROLE_ID,
resourceId: table._id,
level: PermissionLevel.READ,
})
// Make view inherit table permissions. Needed for backwards compatibility with existing views.
await config.api.permission.revoke({
roleId: STD_ROLE_ID,
resourceId: view.id,
level: PermissionLevel.READ,
})
// replicate changes before checking permissions
await config.publish()
await config.api.viewV2.publicSearch(view.id, undefined, { status: 401 })
})
it("should use the view permissions", async () => {
beforeEach(async () => {
const table = await config.createTable()
tableId = table._id!
await config.api.permission.add({
roleId: STD_ROLE_ID,
resourceId: view.id,
resourceId: tableId,
level: PermissionLevel.READ,
})
await config.api.permission.revoke({
roleId: STD_ROLE_ID,
resourceId: table._id,
level: PermissionLevel.READ,
})
// replicate changes before checking permissions
await config.publish()
const res = await config.api.viewV2.publicSearch(view.id)
expect(res.rows[0]._id).toEqual(row._id)
})
it("shouldn't allow writing from a public user", async () => {
const res = await request
.post(`/api/${table._id}/rows`)
.send(basicRow(table._id))
.set(config.publicHeaders())
.expect("Content-Type", /json/)
.expect(401)
expect(res.status).toEqual(401)
it("tables should be defaulted to admin", async () => {
const table = await config.createTable()
const { permissions } = await config.api.permission.get(table._id!)
expect(permissions).toEqual({
read: {
permissionType: "EXPLICIT",
role: DEFAULT_TABLE_ROLE_ID,
},
write: {
permissionType: "EXPLICIT",
role: DEFAULT_TABLE_ROLE_ID,
},
})
})
describe("add", () => {
it("should be able to add permission to a role for the table", async () => {
const res = await request
.get(`/api/permission/${tableId}`)
.set(config.defaultHeaders())
.expect("Content-Type", /json/)
.expect(200)
expect(res.body).toEqual({
permissions: {
read: { permissionType: "EXPLICIT", role: STD_ROLE_ID },
write: { permissionType: "EXPLICIT", role: DEFAULT_TABLE_ROLE_ID },
},
})
})
it("should get resource permissions with multiple roles", async () => {
await config.api.permission.add({
roleId: HIGHER_ROLE_ID,
resourceId: tableId,
level: PermissionLevel.WRITE,
})
const res = await config.api.permission.get(tableId)
expect(res).toEqual({
permissions: {
read: { permissionType: "EXPLICIT", role: STD_ROLE_ID },
write: { permissionType: "EXPLICIT", role: HIGHER_ROLE_ID },
},
})
const allRes = await request
.get(`/api/permission`)
.set(config.defaultHeaders())
.expect("Content-Type", /json/)
.expect(200)
expect(allRes.body[tableId]["read"]).toEqual(STD_ROLE_ID)
expect(allRes.body[tableId]["write"]).toEqual(HIGHER_ROLE_ID)
})
})
describe("remove", () => {
it("should be able to remove the permission", async () => {
await config.api.permission.revoke({
roleId: STD_ROLE_ID,
resourceId: tableId,
level: PermissionLevel.READ,
})
const permsRes = await config.api.permission.get(tableId)
expect(permsRes.permissions[STD_ROLE_ID]).toBeUndefined()
})
})
describe("check public user allowed", () => {
let viewId: string
let row: Row
beforeEach(async () => {
const view = await config.api.viewV2.create({
tableId,
name: generator.guid(),
})
viewId = view.id
row = await config.createRow()
})
it("should be able to read the row", async () => {
// replicate changes before checking permissions
await config.publish()
const res = await request
.get(`/api/${tableId}/rows`)
.set(config.publicHeaders())
.expect("Content-Type", /json/)
.expect(200)
expect(res.body[0]._id).toEqual(row._id)
})
it("should be able to access the view data when the table is set to public and with no view permissions overrides", async () => {
// Make view inherit table permissions. Needed for backwards compatibility with existing views.
await config.api.permission.revoke({
roleId: STD_ROLE_ID,
resourceId: viewId,
level: PermissionLevel.READ,
})
// replicate changes before checking permissions
await config.publish()
const res = await config.api.viewV2.publicSearch(viewId)
expect(res.rows[0]._id).toEqual(row._id)
})
it("should not be able to access the view data when the table is not public and there are no view permissions overrides", async () => {
await config.api.permission.revoke({
roleId: STD_ROLE_ID,
resourceId: tableId,
level: PermissionLevel.READ,
})
// Make view inherit table permissions. Needed for backwards compatibility with existing views.
await config.api.permission.revoke({
roleId: STD_ROLE_ID,
resourceId: viewId,
level: PermissionLevel.READ,
})
// replicate changes before checking permissions
await config.publish()
await config.api.viewV2.publicSearch(viewId, undefined, {
status: 401,
})
})
it("should use the view permissions", async () => {
await config.api.permission.add({
roleId: STD_ROLE_ID,
resourceId: viewId,
level: PermissionLevel.READ,
})
await config.api.permission.revoke({
roleId: STD_ROLE_ID,
resourceId: tableId,
level: PermissionLevel.READ,
})
// replicate changes before checking permissions
await config.publish()
const res = await config.api.viewV2.publicSearch(viewId)
expect(res.rows[0]._id).toEqual(row._id)
})
it("shouldn't allow writing from a public user", async () => {
const res = await request
.post(`/api/${tableId}/rows`)
.send(basicRow(tableId))
.set(config.publicHeaders())
.expect("Content-Type", /json/)
.expect(401)
expect(res.status).toEqual(401)
})
})
})
describe("view permissions", () => {
let tableId: string
let viewId: string
beforeEach(async () => {
const table = await config.createTable()
tableId = table._id!
const view = await config.api.viewV2.create({
tableId,
name: generator.guid(),
})
viewId = view.id
})
it("default permissions inherits and persists the table default value", async () => {
const { permissions } = await config.api.permission.get(viewId)
expect(permissions).toEqual({
read: {
permissionType: "EXPLICIT",
role: DEFAULT_TABLE_ROLE_ID,
inheritablePermission: DEFAULT_TABLE_ROLE_ID,
},
write: {
permissionType: "EXPLICIT",
role: DEFAULT_TABLE_ROLE_ID,
inheritablePermission: DEFAULT_TABLE_ROLE_ID,
},
})
})
it("does not update view permissions once persisted, even if table permissions change", async () => {
await config.api.permission.add({
roleId: STD_ROLE_ID,
resourceId: tableId,
level: PermissionLevel.READ,
})
const { permissions } = await config.api.permission.get(viewId)
expect(permissions).toEqual({
read: {
permissionType: "EXPLICIT",
role: DEFAULT_TABLE_ROLE_ID,
inheritablePermission: STD_ROLE_ID,
},
write: {
permissionType: "EXPLICIT",
role: DEFAULT_TABLE_ROLE_ID,
inheritablePermission: DEFAULT_TABLE_ROLE_ID,
},
})
})
it("can sets permissions inherits explicit view permissions", async () => {
await config.api.permission.add({
roleId: HIGHER_ROLE_ID,
resourceId: viewId,
level: PermissionLevel.WRITE,
})
const { permissions } = await config.api.permission.get(viewId)
expect(permissions).toEqual({
read: {
permissionType: "EXPLICIT",
role: DEFAULT_TABLE_ROLE_ID,
inheritablePermission: DEFAULT_TABLE_ROLE_ID,
},
write: {
permissionType: "EXPLICIT",
role: HIGHER_ROLE_ID,
inheritablePermission: DEFAULT_TABLE_ROLE_ID,
},
})
})
})

29
packages/server/src/api/routes/tests/queries/generic-sql.spec.ts
View File

@ -28,6 +28,7 @@ describe.each(
const config = setup.getConfig()
const isOracle = dbName === DatabaseName.ORACLE
const isMsSQL = dbName === DatabaseName.SQL_SERVER
const isPostgres = dbName === DatabaseName.POSTGRES
let rawDatasource: Datasource
let datasource: Datasource
@ -47,6 +48,9 @@ describe.each(
transformer: "return data",
readable: true,
}
if (query.fields?.sql && typeof query.fields.sql !== "string") {
throw new Error("Unable to create with knex structure in 'sql' field")
}
return await config.api.query.save(
{ ...defaultQuery, ...query },
expectations
@ -207,6 +211,31 @@ describe.each(
expect(prodQuery.parameters).toBeUndefined()
expect(prodQuery.schema).toBeDefined()
})
isPostgres &&
it("should be able to handle a JSON aggregate with newlines", async () => {
const jsonStatement = `COALESCE(json_build_object('name', name),'{"name":{}}'::json)`
const query = await createQuery({
fields: {
sql: client("test_table")
.select([
"*",
client.raw(
`${jsonStatement} as json,\n${jsonStatement} as json2`
),
])
.toString(),
},
})
const res = await config.api.query.execute(
query._id!,
{},
{
status: 200,
}
)
expect(res).toBeDefined()
})
})
})

45
packages/server/src/api/routes/tests/row.spec.ts
View File

@ -13,8 +13,7 @@ import {
context,
InternalTable,
tenancy,
withEnv as withCoreEnv,
setEnv as setCoreEnv,
features,
} from "@budibase/backend-core"
import { quotas } from "@budibase/pro"
import {
@ -40,7 +39,6 @@ import {
TableSchema,
JsonFieldSubType,
RowExportFormat,
FeatureFlag,
RelationSchemaField,
} from "@budibase/types"
import { generator, mocks } from "@budibase/backend-core/tests"
@ -98,12 +96,12 @@ describe.each([
let envCleanup: (() => void) | undefined
beforeAll(async () => {
await withCoreEnv({ TENANT_FEATURE_FLAGS: "*:SQS" }, () => config.init())
if (isLucene) {
envCleanup = setCoreEnv({ TENANT_FEATURE_FLAGS: "*:!SQS" })
} else if (isSqs) {
envCleanup = setCoreEnv({ TENANT_FEATURE_FLAGS: "*:SQS" })
}
await features.testutils.withFeatureFlags("*", { SQS: true }, () =>
config.init()
)
envCleanup = features.testutils.setFeatureFlags("*", {
SQS: isSqs,
})
if (dsProvider) {
const rawDatasource = await dsProvider
@ -1848,7 +1846,7 @@ describe.each([
})
describe("exportRows", () => {
beforeAll(async () => {
beforeEach(async () => {
table = await config.api.table.save(defaultTable())
})
@ -1885,6 +1883,16 @@ describe.each([
})
})
it("should allow exporting without filtering", async () => {
const existing = await config.api.row.save(table._id!, {})
const res = await config.api.row.exportRows(table._id!)
const results = JSON.parse(res)
expect(results.length).toEqual(1)
const row = results[0]
expect(row._id).toEqual(existing._id)
})
it("should allow exporting only certain columns", async () => {
const existing = await config.api.row.save(table._id!, {})
const res = await config.api.row.exportRows(table._id!, {
@ -2517,15 +2525,9 @@ describe.each([
let flagCleanup: (() => void) | undefined
beforeAll(async () => {
const env = {
TENANT_FEATURE_FLAGS: `*:${FeatureFlag.ENRICHED_RELATIONSHIPS}`,
}
if (isSqs) {
env.TENANT_FEATURE_FLAGS = `${env.TENANT_FEATURE_FLAGS},*:SQS`
} else {
env.TENANT_FEATURE_FLAGS = `${env.TENANT_FEATURE_FLAGS},*:!SQS`
}
flagCleanup = setCoreEnv(env)
flagCleanup = features.testutils.setFeatureFlags("*", {
ENRICHED_RELATIONSHIPS: true,
})
const aux2Table = await config.api.table.save(saveTableRequest())
const aux2Data = await config.api.row.save(aux2Table._id!, {})
@ -2752,9 +2754,10 @@ describe.each([
it.each(testScenarios)(
"does not enrich relationships when not enabled (via %s)",
async (__, retrieveDelegate) => {
await withCoreEnv(
await features.testutils.withFeatureFlags(
"*",
{
TENANT_FEATURE_FLAGS: `*:!${FeatureFlag.ENRICHED_RELATIONSHIPS}`,
ENRICHED_RELATIONSHIPS: false,
},
async () => {
const otherRows = _.sampleSize(auxData, 5)

19
packages/server/src/api/routes/tests/search.spec.ts
View File

@ -7,9 +7,9 @@ import {
import {
context,
db as dbCore,
features,
MAX_VALID_DATE,
MIN_VALID_DATE,
setEnv as setCoreEnv,
SQLITE_DESIGN_DOC_ID,
utils,
withEnv as withCoreEnv,
@ -94,16 +94,12 @@ describe.each([
}
beforeAll(async () => {
await withCoreEnv({ TENANT_FEATURE_FLAGS: "*:SQS" }, () => config.init())
if (isLucene) {
envCleanup = setCoreEnv({
TENANT_FEATURE_FLAGS: "*:!SQS",
})
} else if (isSqs) {
envCleanup = setCoreEnv({
TENANT_FEATURE_FLAGS: "*:SQS",
})
}
await features.testutils.withFeatureFlags("*", { SQS: true }, () =>
config.init()
)
envCleanup = features.testutils.setFeatureFlags("*", {
SQS: isSqs,
})
if (config.app?.appId) {
config.app = await config.api.application.update(config.app?.appId, {
@ -191,7 +187,6 @@ describe.each([
if (isInMemory) {
return dataFilters.search(_.cloneDeep(rows), {
...this.query,
tableId: tableOrViewId,
})
} else {
return config.api.row.search(tableOrViewId, this.query)

69
packages/server/src/api/routes/tests/templates.spec.ts
View File

@ -2,7 +2,7 @@ import * as setup from "./utilities"
import path from "path"
import nock from "nock"
import { generator } from "@budibase/backend-core/tests"
import { withEnv as withCoreEnv, env as coreEnv } from "@budibase/backend-core"
import { features } from "@budibase/backend-core"
interface App {
background: string
@ -85,41 +85,44 @@ describe("/templates", () => {
it.each(["sqs", "lucene"])(
`should be able to create an app from a template (%s)`,
async source => {
const env: Partial<typeof coreEnv> = {
TENANT_FEATURE_FLAGS: source === "sqs" ? "*:SQS" : "",
}
await features.testutils.withFeatureFlags(
"*",
{ SQS: source === "sqs" },
async () => {
const name = generator.guid().replaceAll("-", "")
const url = `/${name}`
await withCoreEnv(env, async () => {
const name = generator.guid().replaceAll("-", "")
const url = `/${name}`
const app = await config.api.application.create({
name,
url,
useTemplate: "true",
templateName: "Agency Client Portal",
templateKey: "app/agency-client-portal",
})
expect(app.name).toBe(name)
expect(app.url).toBe(url)
await config.withApp(app, async () => {
const tables = await config.api.table.fetch()
expect(tables).toHaveLength(2)
tables.sort((a, b) => a.name.localeCompare(b.name))
const [agencyProjects, users] = tables
expect(agencyProjects.name).toBe("Agency Projects")
expect(users.name).toBe("Users")
const { rows } = await config.api.row.search(agencyProjects._id!, {
tableId: agencyProjects._id!,
query: {},
const app = await config.api.application.create({
name,
url,
useTemplate: "true",
templateName: "Agency Client Portal",
templateKey: "app/agency-client-portal",
})
expect(app.name).toBe(name)
expect(app.url).toBe(url)
expect(rows).toHaveLength(3)
})
})
await config.withApp(app, async () => {
const tables = await config.api.table.fetch()
expect(tables).toHaveLength(2)
tables.sort((a, b) => a.name.localeCompare(b.name))
const [agencyProjects, users] = tables
expect(agencyProjects.name).toBe("Agency Projects")
expect(users.name).toBe("Users")
const { rows } = await config.api.row.search(
agencyProjects._id!,
{
tableId: agencyProjects._id!,
query: {},
}
)
expect(rows).toHaveLength(3)
})
}
)
}
)
})

154
packages/server/src/api/routes/tests/viewV2.spec.ts
View File

@ -22,23 +22,17 @@ import {
RelationshipType,
TableSchema,
RenameColumn,
FeatureFlag,
BBReferenceFieldSubType,
NumericCalculationFieldMetadata,
ViewV2Schema,
ViewV2Type,
JsonTypes,
} from "@budibase/types"
import { generator, mocks } from "@budibase/backend-core/tests"
import { DatabaseName, getDatasource } from "../../../integrations/tests/utils"
import merge from "lodash/merge"
import { quotas } from "@budibase/pro"
import {
db,
roles,
withEnv as withCoreEnv,
setEnv as setCoreEnv,
env,
} from "@budibase/backend-core"
import { db, roles, features } from "@budibase/backend-core"
describe.each([
["lucene", undefined],
@ -103,18 +97,13 @@ describe.each([
}
beforeAll(async () => {
await withCoreEnv({ TENANT_FEATURE_FLAGS: isSqs ? "*:SQS" : "" }, () =>
await features.testutils.withFeatureFlags("*", { SQS: isSqs }, () =>
config.init()
)
if (isLucene) {
envCleanup = setCoreEnv({
TENANT_FEATURE_FLAGS: "*:!SQS",
})
} else if (isSqs) {
envCleanup = setCoreEnv({
TENANT_FEATURE_FLAGS: "*:SQS",
})
}
envCleanup = features.testutils.setFeatureFlags("*", {
SQS: isSqs,
})
if (dsProvider) {
datasource = await config.createDatasource({
@ -748,6 +737,69 @@ describe.each([
},
})
})
!isLucene &&
it("does not get confused when a calculation field shadows a basic one", async () => {
const table = await config.api.table.save(
saveTableRequest({
schema: {
age: {
name: "age",
type: FieldType.NUMBER,
},
},
})
)
await config.api.row.bulkImport(table._id!, {
rows: [{ age: 1 }, { age: 2 }, { age: 3 }],
})
const view = await config.api.viewV2.create({
tableId: table._id!,
name: generator.guid(),
type: ViewV2Type.CALCULATION,
schema: {
age: {
visible: true,
calculationType: CalculationType.SUM,
field: "age",
},
},
})
const { rows } = await config.api.row.search(view.id)
expect(rows).toHaveLength(1)
expect(rows[0].age).toEqual(6)
})
// We don't allow the creation of tables with most JsonTypes when using
// external datasources.
isInternal &&
it("cannot use complex types as group-by fields", async () => {
for (const type of JsonTypes) {
const field = { name: "field", type } as FieldSchema
const table = await config.api.table.save(
saveTableRequest({ schema: { field } })
)
await config.api.viewV2.create(
{
tableId: table._id!,
name: generator.guid(),
type: ViewV2Type.CALCULATION,
schema: {
field: { visible: true },
},
},
{
status: 400,
body: {
message: `Grouping by fields of type "${type}" is not supported`,
},
}
)
}
})
})
describe("update", () => {
@ -1926,6 +1978,30 @@ describe.each([
expect(newRow.one).toBeUndefined()
expect(newRow.two).toEqual("bar")
})
it("should not be possible to create a row in a calculation view", async () => {
const view = await config.api.viewV2.create({
tableId: table._id!,
name: generator.guid(),
type: ViewV2Type.CALCULATION,
schema: {
id: { visible: true },
one: { visible: true },
},
})
await config.api.row.save(
view.id,
{ one: "foo" },
{
status: 400,
body: {
message: "Cannot insert rows through a calculation view",
status: 400,
},
}
)
})
})
describe("patch", () => {
@ -1990,6 +2066,40 @@ describe.each([
expect(row.one).toEqual("foo")
expect(row.two).toEqual("newBar")
})
it("should not be possible to modify a row in a calculation view", async () => {
const view = await config.api.viewV2.create({
tableId: table._id!,
name: generator.guid(),
type: ViewV2Type.CALCULATION,
schema: {
id: { visible: true },
one: { visible: true },
},
})
const newRow = await config.api.row.save(table._id!, {
one: "foo",
two: "bar",
})
await config.api.row.patch(
view.id,
{
tableId: table._id!,
_id: newRow._id!,
_rev: newRow._rev!,
one: "newFoo",
two: "newBar",
},
{
status: 400,
body: {
message: "Cannot update rows through a calculation view",
},
}
)
})
})
describe("destroy", () => {
@ -2666,12 +2776,8 @@ describe.each([
describe("foreign relationship columns", () => {
let envCleanup: () => void
beforeAll(() => {
const flags = [`*:${FeatureFlag.ENRICHED_RELATIONSHIPS}`]
if (env.TENANT_FEATURE_FLAGS) {
flags.push(...env.TENANT_FEATURE_FLAGS.split(","))
}
envCleanup = setCoreEnv({
TENANT_FEATURE_FLAGS: flags.join(","),
envCleanup = features.testutils.setFeatureFlags("*", {
ENRICHED_RELATIONSHIPS: true,
})
})

6
packages/server/src/appMigrations/migrations/tests/20240604153647_initial_sqs.spec.ts
View File

@ -2,8 +2,8 @@ import * as setup from "../../../api/routes/tests/utilities"
import { basicTable } from "../../../tests/utilities/structures"
import {
db as dbCore,
features,
SQLITE_DESIGN_DOC_ID,
withEnv as withCoreEnv,
} from "@budibase/backend-core"
import {
LinkDocument,
@ -71,11 +71,11 @@ function oldLinkDocument(): Omit<LinkDocument, "tableId"> {
}
async function sqsDisabled(cb: () => Promise<void>) {
await withCoreEnv({ TENANT_FEATURE_FLAGS: "*:!SQS" }, cb)
await features.testutils.withFeatureFlags("*", { SQS: false }, cb)
}
async function sqsEnabled(cb: () => Promise<void>) {
await withCoreEnv({ TENANT_FEATURE_FLAGS: "*:SQS" }, cb)
await features.testutils.withFeatureFlags("*", { SQS: true }, cb)
}
describe("SQS migration", () => {

8
packages/server/src/db/linkedRows/LinkController.ts
View File

@ -221,9 +221,15 @@ class LinkController {
link.id !== row._id && link.fieldName === linkedSchema.name
)
// check all the related rows exist
const foundRecords = await this._db.getMultiple(
links.map(l => l.id),
{ allowMissing: true, excludeDocs: true }
)
// The 1 side of 1:N is already related to something else
// You must remove the existing relationship
if (links.length > 0) {
if (foundRecords.length > 0) {
throw new Error(
`1:N Relationship Error: Record already linked to another.`
)

2
packages/server/src/integrations/postgres.ts
View File

@ -41,7 +41,7 @@ if (types) {
types.setTypeParser(1184, (val: any) => val) // timestampz
}
const JSON_REGEX = /'{.*}'::json/s
const JSON_REGEX = /'{\s*.*?\s*}'::json/gs
const Sql = sql.Sql
interface PostgresConfig {

2
packages/server/src/koa.ts
View File

@ -12,6 +12,7 @@ import { userAgent } from "koa-useragent"
export default function createKoaApp() {
const app = new Koa()
app.proxy = true
let mbNumber = parseInt(env.HTTP_MB_LIMIT || "10")
if (!mbNumber || isNaN(mbNumber)) {
@ -35,6 +36,7 @@ export default function createKoaApp() {
app.use(middleware.correlation)
app.use(middleware.pino)
app.use(middleware.ip)
app.use(userAgent)
const server = http.createServer(app.callback())

4
packages/server/src/migrations/tests/index.spec.ts
View File

@ -71,7 +71,7 @@ describe("migrations", () => {
expect(events.datasource.created).toHaveBeenCalledTimes(2)
expect(events.layout.created).toHaveBeenCalledTimes(1)
expect(events.query.created).toHaveBeenCalledTimes(2)
expect(events.role.created).toHaveBeenCalledTimes(2)
expect(events.role.created).toHaveBeenCalledTimes(3) // created roles + admin (created on table creation)
expect(events.table.created).toHaveBeenCalledTimes(3)
expect(events.view.created).toHaveBeenCalledTimes(2)
expect(events.view.calculationCreated).toHaveBeenCalledTimes(1)
@ -82,7 +82,7 @@ describe("migrations", () => {
// to make sure caching is working as expected
expect(
events.processors.analyticsProcessor.processEvent
).toHaveBeenCalledTimes(23)
).toHaveBeenCalledTimes(24) // Addtion of of the events above
})
})
})

20
packages/server/src/sdk/app/permissions/index.ts
View File

@ -185,6 +185,26 @@ export async function updatePermissionOnRole(
})
}
export async function setPermissions(
resourceId: string,
{
writeRole,
readRole,
}: {
writeRole: string
readRole: string
}
) {
await updatePermissionOnRole(
{ roleId: writeRole, resourceId, level: PermissionLevel.WRITE },
PermissionUpdateType.ADD
)
await updatePermissionOnRole(
{ roleId: readRole, resourceId, level: PermissionLevel.READ },
PermissionUpdateType.ADD
)
}
// utility function to stop this repetition - permissions always stored under roles
export async function getAllDBRoles(db: Database) {
const body = await db.allDocs<Role>(

5
packages/server/src/sdk/app/rows/external.ts
View File

@ -15,6 +15,7 @@ import {
} from "../../../utilities/rowProcessor"
import cloneDeep from "lodash/fp/cloneDeep"
import { tryExtractingTableAndViewId } from "./utils"
import { helpers } from "@budibase/shared-core"
export async function getRow(
sourceId: string | Table | ViewV2,
@ -54,6 +55,10 @@ export async function save(
source = await sdk.tables.getTable(tableId)
}
if (sdk.views.isView(source) && helpers.views.isCalculationView(source)) {
throw new HTTPError("Cannot insert rows through a calculation view", 400)
}
const row = await inputProcessing(userId, cloneDeep(source), inputs)
const validateResult = await sdk.rows.utils.validate({

7
packages/server/src/sdk/app/rows/internal.ts
View File

@ -1,4 +1,4 @@
import { context, db } from "@budibase/backend-core"
import { context, db, HTTPError } from "@budibase/backend-core"
import { Row, Table, ViewV2 } from "@budibase/types"
import sdk from "../../../sdk"
import { finaliseRow } from "../../../api/controllers/row/staticFormula"
@ -10,6 +10,7 @@ import * as linkRows from "../../../db/linkedRows"
import { InternalTables } from "../../../db/utils"
import { getFullUser } from "../../../utilities/users"
import { getSource, tryExtractingTableAndViewId } from "./utils"
import { helpers } from "@budibase/shared-core"
export async function save(
tableOrViewId: string,
@ -29,6 +30,10 @@ export async function save(
table = source
}
if (sdk.views.isView(source) && helpers.views.isCalculationView(source)) {
throw new HTTPError("Cannot insert rows through a calculation view", 400)
}
if (!inputs._rev && !inputs._id) {
inputs._id = db.generateRowID(inputs.tableId)
}

4
packages/server/src/sdk/app/rows/search/internal/internal.ts
View File

@ -62,10 +62,10 @@ export async function exportRows(
).rows.map(row => row.doc!)
result = await outputProcessing(table, response)
} else if (query) {
} else {
let searchResponse = await sdk.rows.search({
tableId,
query,
query: query || {},
sort,
sortOrder,
})

19
packages/server/src/sdk/app/rows/search/tests/search.spec.ts
View File

@ -10,10 +10,7 @@ import {
import TestConfiguration from "../../../../../tests/utilities/TestConfiguration"
import { search } from "../../../../../sdk/app/rows/search"
import { generator } from "@budibase/backend-core/tests"
import {
withEnv as withCoreEnv,
setEnv as setCoreEnv,
} from "@budibase/backend-core"
import { features } from "@budibase/backend-core"
import {
DatabaseName,
getDatasource,
@ -41,19 +38,13 @@ describe.each([
let table: Table
beforeAll(async () => {
await withCoreEnv({ TENANT_FEATURE_FLAGS: isSqs ? "*:SQS" : "" }, () =>
await features.testutils.withFeatureFlags("*", { SQS: isSqs }, () =>
config.init()
)
if (isLucene) {
envCleanup = setCoreEnv({
TENANT_FEATURE_FLAGS: "*:!SQS",
})
} else if (isSqs) {
envCleanup = setCoreEnv({
TENANT_FEATURE_FLAGS: "*:SQS",
})
}
envCleanup = features.testutils.setFeatureFlags("*", {
SQS: isSqs,
})
if (dsProvider) {
datasource = await config.createDatasource({

27
packages/server/src/sdk/app/tables/create.ts Normal file
View File

@ -0,0 +1,27 @@
import { Row, Table } from "@budibase/types"
import * as external from "./external"
import * as internal from "./internal"
import { isExternal } from "./utils"
import { setPermissions } from "../permissions"
import { roles } from "@budibase/backend-core"
export async function create(
table: Omit<Table, "_id" | "_rev">,
rows?: Row[],
userId?: string
): Promise<Table> {
let createdTable: Table
if (isExternal({ table })) {
createdTable = await external.create(table)
} else {
createdTable = await internal.create(table, rows, userId)
}
await setPermissions(createdTable._id!, {
writeRole: roles.BUILTIN_ROLE_IDS.ADMIN,
readRole: roles.BUILTIN_ROLE_IDS.ADMIN,
})
return createdTable
}

36
packages/server/src/sdk/app/tables/external/index.ts vendored
View File

@ -8,8 +8,11 @@ import {
ViewV2,
AutoFieldSubType,
} from "@budibase/types"
import { context } from "@budibase/backend-core"
import { buildExternalTableId } from "../../../../integrations/utils"
import { context, HTTPError } from "@budibase/backend-core"
import {
breakExternalTableId,
buildExternalTableId,
} from "../../../../integrations/utils"
import {
foreignKeyStructure,
hasTypeChanged,
@ -86,6 +89,35 @@ function validate(table: Table, oldTable?: Table) {
}
}
function getDatasourceId(table: Table) {
if (!table) {
throw new Error("No table supplied")
}
if (table.sourceId) {
return table.sourceId
}
if (!table._id) {
throw new Error("No table ID supplied")
}
return breakExternalTableId(table._id).datasourceId
}
export async function create(table: Omit<Table, "_id" | "_rev">) {
const datasourceId = getDatasourceId(table)
const tableToCreate = { ...table, created: true }
try {
const result = await save(datasourceId!, tableToCreate)
return result.table
} catch (err: any) {
if (err instanceof Error) {
throw new HTTPError(err.message, 400)
} else {
throw new HTTPError(err?.message || err, err.status || 500)
}
}
}
export async function save(
datasourceId: string,
update: Table,

2
packages/server/src/sdk/app/tables/index.ts
View File

@ -1,5 +1,6 @@
import { populateExternalTableSchemas } from "./validation"
import * as getters from "./getters"
import * as create from "./create"
import * as updates from "./update"
import * as utils from "./utils"
import { migrate } from "./migration"
@ -7,6 +8,7 @@ import * as sqs from "./internal/sqs"
export default {
populateExternalTableSchemas,
...create,
...updates,
...getters,
...utils,

48
packages/server/src/sdk/app/tables/internal/index.ts
View File

@ -5,7 +5,7 @@ import {
ViewStatisticsSchema,
ViewV2,
Row,
ContextUser,
TableSourceType,
} from "@budibase/types"
import {
hasTypeChanged,
@ -16,18 +16,56 @@ import { EventType, updateLinks } from "../../../../db/linkedRows"
import { cloneDeep } from "lodash/fp"
import isEqual from "lodash/isEqual"
import { runStaticFormulaChecks } from "../../../../api/controllers/table/bulkFormula"
import { context } from "@budibase/backend-core"
import { context, HTTPError } from "@budibase/backend-core"
import { findDuplicateInternalColumns } from "@budibase/shared-core"
import { getTable } from "../getters"
import { checkAutoColumns } from "./utils"
import * as viewsSdk from "../../views"
import { getRowParams } from "../../../../db/utils"
import { generateTableID, getRowParams } from "../../../../db/utils"
import { quotas } from "@budibase/pro"
export async function create(
table: Omit<Table, "_id" | "_rev">,
rows?: Row[],
userId?: string
) {
const tableId = generateTableID()
let tableToSave: Table = {
_id: tableId,
...table,
// Ensure these fields are populated, even if not sent in the request
type: table.type || "table",
sourceType: TableSourceType.INTERNAL,
}
const isImport = !!rows
if (!tableToSave.views) {
tableToSave.views = {}
}
try {
const { table } = await save(tableToSave, {
userId,
rowsToImport: rows,
isImport,
})
return table
} catch (err: any) {
if (err instanceof Error) {
throw new HTTPError(err.message, 400)
} else {
throw new HTTPError(err.message || err, err.status || 500)
}
}
}
export async function save(
table: Table,
opts?: {
user?: ContextUser
userId?: string
tableId?: string
rowsToImport?: Row[]
renaming?: RenameColumn
@ -63,7 +101,7 @@ export async function save(
// saving a table is a complex operation, involving many different steps, this
// has been broken out into a utility to make it more obvious/easier to manipulate
const tableSaveFunctions = new TableSaveFunctions({
user: opts?.user,
userId: opts?.userId,
oldTable,
importRows: opts?.rowsToImport,
})

36
packages/server/src/sdk/app/views/index.ts
View File

@ -1,6 +1,8 @@
import {
CalculationType,
canGroupBy,
FieldType,
isNumeric,
PermissionLevel,
RelationSchemaField,
RenameColumn,
@ -11,7 +13,7 @@ import {
ViewV2ColumnEnriched,
ViewV2Enriched,
} from "@budibase/types"
import { context, docIds, HTTPError, roles } from "@budibase/backend-core"
import { context, docIds, HTTPError } from "@budibase/backend-core"
import {
helpers,
PROTECTED_EXTERNAL_COLUMNS,
@ -22,7 +24,6 @@ import { isExternalTableID } from "../../../integrations/utils"
import * as internal from "./internal"
import * as external from "./external"
import sdk from "../../../sdk"
import { PermissionUpdateType, updatePermissionOnRole } from "../permissions"
function pickApi(tableId: any) {
if (isExternalTableID(tableId)) {
@ -101,7 +102,7 @@ async function guardCalculationViewSchema(
)
}
if (!isCount && !helpers.schema.isNumeric(targetSchema)) {
if (!isCount && !isNumeric(targetSchema.type)) {
throw new HTTPError(
`Calculation field "${name}" references field "${schema.field}" which is not a numeric field`,
400
@ -118,6 +119,13 @@ async function guardCalculationViewSchema(
400
)
}
if (!canGroupBy(targetSchema.type)) {
throw new HTTPError(
`Grouping by fields of type "${targetSchema.type}" is not supported`,
400
)
}
}
}
@ -235,24 +243,10 @@ export async function create(
// Set permissions to be the same as the table
const tablePerms = await sdk.permissions.getResourcePerms(tableId)
const readRole = tablePerms[PermissionLevel.READ]?.role
const writeRole = tablePerms[PermissionLevel.WRITE]?.role
await updatePermissionOnRole(
{
roleId: readRole || roles.BUILTIN_ROLE_IDS.BASIC,
resourceId: view.id,
level: PermissionLevel.READ,
},
PermissionUpdateType.ADD
)
await updatePermissionOnRole(
{
roleId: writeRole || roles.BUILTIN_ROLE_IDS.BASIC,
resourceId: view.id,
level: PermissionLevel.WRITE,
},
PermissionUpdateType.ADD
)
await sdk.permissions.setPermissions(view.id, {
writeRole: tablePerms[PermissionLevel.WRITE].role,
readRole: tablePerms[PermissionLevel.READ].role,
})
return view
}

10
packages/server/src/tests/utilities/api/permission.ts
View File

@ -1,6 +1,7 @@
import {
AddPermissionRequest,
AddPermissionResponse,
FetchResourcePermissionInfoResponse,
GetResourcePermsResponse,
RemovePermissionRequest,
RemovePermissionResponse,
@ -26,6 +27,15 @@ export class PermissionAPI extends TestAPI {
)
}
fetch = async (
expectations?: Expectations
): Promise<FetchResourcePermissionInfoResponse> => {
return await this._get<FetchResourcePermissionInfoResponse>(
`/api/permission`,
{ expectations }
)
}
revoke = async (
request: RemovePermissionRequest,
expectations?: Expectations

2
packages/server/src/tests/utilities/api/row.ts
View File

@ -105,7 +105,7 @@ export class RowAPI extends TestAPI {
exportRows = async (
tableId: string,
body: ExportRowsRequest,
body?: ExportRowsRequest,
format: RowExportFormat = RowExportFormat.JSON,
expectations?: Expectations
) => {

8
packages/server/src/utilities/rowProcessor/tests/outputProcessing.spec.ts
View File

@ -8,7 +8,7 @@ import {
} from "@budibase/types"
import { outputProcessing } from ".."
import { generator, structures } from "@budibase/backend-core/tests"
import { setEnv as setCoreEnv } from "@budibase/backend-core"
import { features } from "@budibase/backend-core"
import * as bbReferenceProcessor from "../bbReferenceProcessor"
import TestConfiguration from "../../../tests/utilities/TestConfiguration"
@ -21,7 +21,7 @@ jest.mock("../bbReferenceProcessor", (): typeof bbReferenceProcessor => ({
describe("rowProcessor - outputProcessing", () => {
const config = new TestConfiguration()
let cleanupEnv: () => void = () => {}
let cleanupFlags: () => void = () => {}
beforeAll(async () => {
await config.init()
@ -33,11 +33,11 @@ describe("rowProcessor - outputProcessing", () => {
beforeEach(() => {
jest.resetAllMocks()
cleanupEnv = setCoreEnv({ TENANT_FEATURE_FLAGS: "*SQS" })
cleanupFlags = features.testutils.setFeatureFlags("*", { SQS: true })
})
afterEach(() => {
cleanupEnv()
cleanupFlags()
})
const processOutputBBReferenceMock =

29
packages/server/src/utilities/security.ts
View File

@ -1,5 +1,6 @@
import { permissions, roles } from "@budibase/backend-core"
import { DocumentType, VirtualDocumentType } from "../db/utils"
import { getDocumentType, getVirtualDocumentType } from "@budibase/types"
export const CURRENTLY_SUPPORTED_LEVELS: string[] = [
permissions.PermissionLevel.WRITE,
@ -8,13 +9,16 @@ export const CURRENTLY_SUPPORTED_LEVELS: string[] = [
]
export function getPermissionType(resourceId: string) {
const docType = Object.values(DocumentType).filter(docType =>
resourceId.startsWith(docType)
)[0]
switch (docType as DocumentType | VirtualDocumentType) {
const virtualDocType = getVirtualDocumentType(resourceId)
switch (virtualDocType) {
case VirtualDocumentType.VIEW:
return permissions.PermissionType.TABLE
}
const docType = getDocumentType(resourceId)
switch (docType) {
case DocumentType.TABLE:
case DocumentType.ROW:
case VirtualDocumentType.VIEW:
return permissions.PermissionType.TABLE
case DocumentType.AUTOMATION:
return permissions.PermissionType.AUTOMATION
@ -32,22 +36,25 @@ export function getPermissionType(resourceId: string) {
/**
* works out the basic permissions based on builtin roles for a resource, using its ID
*/
export function getBasePermissions(resourceId: string) {
export function getBasePermissions(resourceId: string): Record<string, string> {
const type = getPermissionType(resourceId)
const basePermissions: { [key: string]: string } = {}
const basePermissions: Record<string, string> = {}
for (let [roleId, role] of Object.entries(roles.getBuiltinRoles())) {
if (!role.permissionId) {
continue
}
const perms = permissions.getBuiltinPermissionByID(role.permissionId)
if (!perms) {
continue
}
const typedPermission = perms.permissions.find(perm => perm.type === type)
if (
typedPermission &&
CURRENTLY_SUPPORTED_LEVELS.indexOf(typedPermission.level) !== -1
) {
if (!typedPermission) {
continue
}
if (CURRENTLY_SUPPORTED_LEVELS.includes(typedPermission.level)) {
const level = typedPermission.level
basePermissions[level] = roles.lowerBuiltinRoleID(
basePermissions[level],

12
packages/shared-core/src/filters.ts
View File

@ -642,19 +642,19 @@ export function fixupFilterArrays(filters: SearchFilters) {
return filters
}
export function search<T>(
docs: Record<string, T>[],
query: RowSearchParams
): SearchResponse<Record<string, T>> {
export function search<T extends Record<string, any>>(
docs: T[],
query: Omit<RowSearchParams, "tableId">
): SearchResponse<T> {
let result = runQuery(docs, query.query)
if (query.sort) {
result = sort(result, query.sort, query.sortOrder || SortOrder.ASCENDING)
}
let totalRows = result.length
const totalRows = result.length
if (query.limit) {
result = limit(result, query.limit.toString())
}
const response: SearchResponse<Record<string, any>> = { rows: result }
const response: SearchResponse<T> = { rows: result }
if (query.countRows) {
response.totalRows = totalRows
}

13
packages/shared-core/src/utils.ts
View File

@ -5,6 +5,7 @@ import {
SearchFilters,
BasicOperator,
ArrayOperator,
isLogicalSearchOperator,
} from "@budibase/types"
import * as Constants from "./constants"
import { removeKeyNumbering } from "./filters"
@ -97,10 +98,20 @@ export function isSupportedUserSearch(query: SearchFilters) {
{ op: BasicOperator.EQUAL, key: "_id" },
{ op: ArrayOperator.ONE_OF, key: "_id" },
]
for (let [key, operation] of Object.entries(query)) {
for (const [key, operation] of Object.entries(query)) {
if (typeof operation !== "object") {
return false
}
if (isLogicalSearchOperator(key)) {
for (const condition of query[key]!.conditions) {
if (!isSupportedUserSearch(condition)) {
return false
}
}
return true
}
const fields = Object.keys(operation || {})
// this filter doesn't contain options - ignore
if (fields.length === 0) {

8
packages/types/src/api/web/app/permission.ts
View File

@ -1,5 +1,9 @@
import { PermissionLevel } from "../../../sdk"
export interface FetchResourcePermissionInfoResponse {
[key: string]: Record<string, string>
}
export interface ResourcePermissionInfo {
role: string
permissionType: string
@ -21,7 +25,7 @@ export interface AddedPermission {
reason?: string
}
export type AddPermissionResponse = AddedPermission[]
export interface AddPermissionResponse {}
export interface AddPermissionRequest {
roleId: string
@ -30,4 +34,4 @@ export interface AddPermissionRequest {
}
export interface RemovePermissionRequest extends AddPermissionRequest {}
export interface RemovePermissionResponse extends AddPermissionResponse {}
export interface RemovePermissionResponse {}

20
packages/types/src/documents/app/row.ts
View File

@ -127,6 +127,26 @@ export const JsonTypes = [
FieldType.ARRAY,
]
export const NumericTypes = [FieldType.NUMBER, FieldType.BIGINT]
export function isNumeric(type: FieldType) {
return NumericTypes.includes(type)
}
export const GroupByTypes = [
FieldType.STRING,
FieldType.LONGFORM,
FieldType.OPTIONS,
FieldType.NUMBER,
FieldType.BOOLEAN,
FieldType.DATETIME,
FieldType.BIGINT,
]
export function canGroupBy(type: FieldType) {
return GroupByTypes.includes(type)
}
export interface RowAttachment {
size: number
name: string

24
packages/types/src/documents/document.ts
View File

@ -42,6 +42,17 @@ export enum DocumentType {
ROW_ACTIONS = "ra",
}
// Because DocumentTypes can overlap, we need to make sure that we search
// longest first to ensure we get the correct type.
const sortedDocumentTypes = Object.values(DocumentType).sort(
(a, b) => b.length - a.length // descending
)
export function getDocumentType(id: string): DocumentType | undefined {
return sortedDocumentTypes.find(docType =>
id.startsWith(`${docType}${SEPARATOR}`)
)
}
// these are the core documents that make up the data, design
// and automation sections of an app. This excludes any internal
// rows as we shouldn't import data.
@ -72,6 +83,19 @@ export enum VirtualDocumentType {
ROW_ACTION = "row_action",
}
// Because VirtualDocumentTypes can overlap, we need to make sure that we search
// longest first to ensure we get the correct type.
const sortedVirtualDocumentTypes = Object.values(VirtualDocumentType).sort(
(a, b) => b.length - a.length // descending
)
export function getVirtualDocumentType(
id: string
): VirtualDocumentType | undefined {
return sortedVirtualDocumentTypes.find(docType =>
id.startsWith(`${docType}${SEPARATOR}`)
)
}
export interface Document {
_id?: string
_rev?: string

2
packages/types/src/sdk/db.ts
View File

@ -133,7 +133,7 @@ export interface Database {
exists(docId: string): Promise<boolean>
getMultiple<T extends Document>(
ids: string[],
opts?: { allowMissing?: boolean }
opts?: { allowMissing?: boolean; excludeDocs?: boolean }
): Promise<T[]>
remove(idOrDoc: Document): Promise<Nano.DocumentDestroyResponse>
remove(idOrDoc: string, rev?: string): Promise<Nano.DocumentDestroyResponse>

10
packages/worker/src/api/routes/global/tests/auditLogs.spec.ts
View File

@ -1,5 +1,5 @@
import { mocks, structures } from "@budibase/backend-core/tests"
import { context, events, setEnv as setCoreEnv } from "@budibase/backend-core"
import { context, events, features } from "@budibase/backend-core"
import { Event, IdentityType } from "@budibase/types"
import { TestConfiguration } from "../../../../tests"
@ -17,11 +17,9 @@ describe.each(["lucene", "sql"])("/api/global/auditlogs (%s)", method => {
let envCleanup: (() => void) | undefined
beforeAll(async () => {
if (method === "lucene") {
envCleanup = setCoreEnv({ TENANT_FEATURE_FLAGS: "*:!SQS" })
} else if (method === "sql") {
envCleanup = setCoreEnv({ TENANT_FEATURE_FLAGS: "*:SQS" })
}
envCleanup = features.testutils.setFeatureFlags("*", {
SQS: method === "sql",
})
await config.beforeAll()
})

19
packages/worker/src/api/routes/global/tests/users.spec.ts
View File

@ -741,6 +741,25 @@ describe("/api/global/users", () => {
it("should throw an error if public query performed", async () => {
await config.api.users.searchUsers({}, { status: 403, noHeaders: true })
})
it("should be able to search using logical conditions", async () => {
const user = await config.createUser()
const response = await config.api.users.searchUsers({
query: {
$and: {
conditions: [
{
$and: {
conditions: [{ string: { email: user.email } }],
},
},
],
},
},
})
expect(response.body.data.length).toBe(1)
expect(response.body.data[0].email).toBe(user.email)
})
})
describe("DELETE /api/global/users/:userId", () => {

2
packages/worker/src/index.ts
View File

@ -46,6 +46,7 @@ bootstrap()
const app: Application = new Application()
app.keys = ["secret", "key"]
app.proxy = true
// set up top level koa middleware
app.use(handleScimBody)
@ -54,6 +55,7 @@ app.use(koaBody({ multipart: true }))
app.use(koaSession(app))
app.use(middleware.correlation)
app.use(middleware.pino)
app.use(middleware.ip)
app.use(userAgent)
// authentication