MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'v3-ui' of github.com:Budibase/budibase into view-calculation-ui

This commit is contained in:
Andrew Kingston 2024-10-09 18:53:48 +01:00
parent 6d294be646 156a2b96ab
commit bd635454ed
No known key found for this signature in database
58 changed files with 1342 additions and 694 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
lerna.json
View File

@ -1,6 +1,6 @@
{
"$schema": "node_modules/lerna/schemas/lerna-schema.json",
"version": "2.32.12",
"version": "2.32.15",
"npmClient": "yarn",
"packages": [
"packages/*",

9
packages/backend-core/src/context/mainContext.ts
View File

@ -253,6 +253,11 @@ export function getAppId(): string | undefined {
}
}
export function getIP(): string | undefined {
const context = Context.get()
return context?.ip
}
export const getProdAppId = () => {
const appId = getAppId()
if (!appId) {
@ -281,6 +286,10 @@ export function doInScimContext(task: any) {
return newContext(updates, task)
}
export function doInIPContext(ip: string, task: any) {
return newContext({ ip }, task)
}
export async function ensureSnippetContext(enabled = !env.isTest()) {
const ctx = getCurrentContext()

1
packages/backend-core/src/context/types.ts
View File

@ -9,6 +9,7 @@ export type ContextMap = {
identity?: IdentityContext
environmentVariables?: Record<string, string>
isScim?: boolean
ip?: string
automationId?: string
isMigrating?: boolean
vm?: VM

12
packages/backend-core/src/db/couch/DatabaseImpl.ts
View File

@ -213,17 +213,21 @@ export class DatabaseImpl implements Database {
async getMultiple<T extends Document>(
ids: string[],
opts?: { allowMissing?: boolean }
opts?: { allowMissing?: boolean; excludeDocs?: boolean }
): Promise<T[]> {
// get unique
ids = [...new Set(ids)]
const includeDocs = !opts?.excludeDocs
const response = await this.allDocs<T>({
keys: ids,
include_docs: true,
include_docs: includeDocs,
})
const rowUnavailable = (row: RowResponse<T>) => {
// row is deleted - key lookup can return this
if (row.doc == null || ("deleted" in row.value && row.value.deleted)) {
if (
(includeDocs && row.doc == null) ||
(row.value && "deleted" in row.value && row.value.deleted)
) {
return true
}
return row.error === "not_found"
@ -237,7 +241,7 @@ export class DatabaseImpl implements Database {
const missingIds = missing.map(row => row.key).join(", ")
throw new Error(`Unable to get documents: ${missingIds}`)
}
return rows.map(row => row.doc!)
return rows.map(row => (includeDocs ? row.doc! : row.value))
}
async remove(idOrDoc: string | Document, rev?: string) {

278
packages/backend-core/src/features/features.ts Normal file
View File

@ -0,0 +1,278 @@
import env from "../environment"
import * as crypto from "crypto"
import * as context from "../context"
import { PostHog, PostHogOptions } from "posthog-node"
import { FeatureFlag } from "@budibase/types"
import tracer from "dd-trace"
import { Duration } from "../utils"
let posthog: PostHog | undefined
export function init(opts?: PostHogOptions) {
if (
env.POSTHOG_TOKEN &&
env.POSTHOG_API_HOST &&
!env.SELF_HOSTED &&
env.POSTHOG_FEATURE_FLAGS_ENABLED
) {
console.log("initializing posthog client...")
posthog = new PostHog(env.POSTHOG_TOKEN, {
host: env.POSTHOG_API_HOST,
personalApiKey: env.POSTHOG_PERSONAL_TOKEN,
featureFlagsPollingInterval: Duration.fromMinutes(3).toMs(),
...opts,
})
} else {
console.log("posthog disabled")
}
}
export function shutdown() {
posthog?.shutdown()
}
export abstract class Flag<T> {
static boolean(defaultValue: boolean): Flag<boolean> {
return new BooleanFlag(defaultValue)
}
static string(defaultValue: string): Flag<string> {
return new StringFlag(defaultValue)
}
static number(defaultValue: number): Flag<number> {
return new NumberFlag(defaultValue)
}
protected constructor(public defaultValue: T) {}
abstract parse(value: any): T
}
type UnwrapFlag<F> = F extends Flag<infer U> ? U : never
export type FlagValues<T> = {
[K in keyof T]: UnwrapFlag<T[K]>
}
type KeysOfType<T, U> = {
[K in keyof T]: T[K] extends Flag<U> ? K : never
}[keyof T]
class BooleanFlag extends Flag<boolean> {
parse(value: any) {
if (typeof value === "string") {
return ["true", "t", "1"].includes(value.toLowerCase())
}
if (typeof value === "boolean") {
return value
}
throw new Error(`could not parse value "${value}" as boolean`)
}
}
class StringFlag extends Flag<string> {
parse(value: any) {
if (typeof value === "string") {
return value
}
throw new Error(`could not parse value "${value}" as string`)
}
}
class NumberFlag extends Flag<number> {
parse(value: any) {
if (typeof value === "number") {
return value
}
if (typeof value === "string") {
const parsed = parseFloat(value)
if (!isNaN(parsed)) {
return parsed
}
}
throw new Error(`could not parse value "${value}" as number`)
}
}
export interface EnvFlagEntry {
tenantId: string
key: string
value: boolean
}
export function parseEnvFlags(flags: string): EnvFlagEntry[] {
const split = flags.split(",").map(x => x.split(":"))
const result: EnvFlagEntry[] = []
for (const [tenantId, ...features] of split) {
for (let feature of features) {
let value = true
if (feature.startsWith("!")) {
feature = feature.slice(1)
value = false
}
result.push({ tenantId, key: feature, value })
}
}
return result
}
export class FlagSet<V extends Flag<any>, T extends { [key: string]: V }> {
// This is used to safely cache flags sets in the current request context.
// Because multiple sets could theoretically exist, we don't want the cache of
// one to leak into another.
private readonly setId: string
constructor(private readonly flagSchema: T) {
this.setId = crypto.randomUUID()
}
defaults(): FlagValues<T> {
return Object.keys(this.flagSchema).reduce((acc, key) => {
const typedKey = key as keyof T
acc[typedKey] = this.flagSchema[key].defaultValue
return acc
}, {} as FlagValues<T>)
}
isFlagName(name: string | number | symbol): name is keyof T {
return this.flagSchema[name as keyof T] !== undefined
}
async get<K extends keyof T>(key: K): Promise<FlagValues<T>[K]> {
const flags = await this.fetch()
return flags[key]
}
async isEnabled<K extends KeysOfType<T, boolean>>(key: K): Promise<boolean> {
const flags = await this.fetch()
return flags[key]
}
async fetch(): Promise<FlagValues<T>> {
return await tracer.trace("features.fetch", async span => {
const cachedFlags = context.getFeatureFlags<FlagValues<T>>(this.setId)
if (cachedFlags) {
span?.addTags({ fromCache: true })
return cachedFlags
}
const tags: Record<string, any> = {}
const flagValues = this.defaults()
const currentTenantId = context.getTenantId()
const specificallySetFalse = new Set<string>()
for (const { tenantId, key, value } of parseEnvFlags(
env.TENANT_FEATURE_FLAGS || ""
)) {
if (!tenantId || (tenantId !== "*" && tenantId !== currentTenantId)) {
continue
}
tags[`readFromEnvironmentVars`] = true
if (value === false) {
specificallySetFalse.add(key)
}
// ignore unknown flags
if (!this.isFlagName(key)) {
continue
}
if (typeof flagValues[key] !== "boolean") {
throw new Error(`Feature: ${key} is not a boolean`)
}
// @ts-expect-error - TS does not like you writing into a generic type,
// but we know that it's okay in this case because it's just an object.
flagValues[key as keyof FlagValues] = value
tags[`flags.${key}.source`] = "environment"
}
const identity = context.getIdentity()
let userId = identity?._id
if (!userId) {
const ip = context.getIP()
if (ip) {
userId = crypto.createHash("sha512").update(ip).digest("hex")
}
}
let tenantId = identity?.tenantId
if (!tenantId) {
tenantId = currentTenantId
}
tags[`identity.type`] = identity?.type
tags[`identity._id`] = identity?._id
tags[`tenantId`] = tenantId
tags[`userId`] = userId
if (posthog && userId) {
tags[`readFromPostHog`] = true
const personProperties: Record<string, string> = { tenantId }
const posthogFlags = await posthog.getAllFlagsAndPayloads(userId, {
personProperties,
})
for (const [name, value] of Object.entries(posthogFlags.featureFlags)) {
if (!this.isFlagName(name)) {
// We don't want an unexpected PostHog flag to break the app, so we
// just log it and continue.
console.warn(`Unexpected posthog flag "${name}": ${value}`)
continue
}
if (flagValues[name] === true || specificallySetFalse.has(name)) {
// If the flag is already set to through environment variables, we
// don't want to override it back to false here.
continue
}
const payload = posthogFlags.featureFlagPayloads?.[name]
const flag = this.flagSchema[name]
try {
// @ts-expect-error - TS does not like you writing into a generic
// type, but we know that it's okay in this case because it's just
// an object.
flagValues[name] = flag.parse(payload || value)
tags[`flags.${name}.source`] = "posthog"
} catch (err) {
// We don't want an invalid PostHog flag to break the app, so we just
// log it and continue.
console.warn(`Error parsing posthog flag "${name}": ${value}`, err)
}
}
}
context.setFeatureFlags(this.setId, flagValues)
for (const [key, value] of Object.entries(flagValues)) {
tags[`flags.${key}.value`] = value
}
span?.addTags(tags)
return flagValues
})
}
}
// This is the primary source of truth for feature flags. If you want to add a
// new flag, add it here and use the `fetch` and `get` functions to access it.
// All of the machinery in this file is to make sure that flags have their
// default values set correctly and their types flow through the system.
export const flags = new FlagSet({
DEFAULT_VALUES: Flag.boolean(env.isDev()),
AUTOMATION_BRANCHING: Flag.boolean(env.isDev()),
SQS: Flag.boolean(env.isDev()),
[FeatureFlag.AI_CUSTOM_CONFIGS]: Flag.boolean(env.isDev()),
[FeatureFlag.ENRICHED_RELATIONSHIPS]: Flag.boolean(env.isDev()),
})
type UnwrapPromise<T> = T extends Promise<infer U> ? U : T
export type FeatureFlags = UnwrapPromise<ReturnType<typeof flags.fetch>>

283
packages/backend-core/src/features/index.ts
View File

@ -1,281 +1,2 @@
import env from "../environment"
import * as context from "../context"
import { PostHog, PostHogOptions } from "posthog-node"
import { FeatureFlag, IdentityType, UserCtx } from "@budibase/types"
import tracer from "dd-trace"
import { Duration } from "../utils"
let posthog: PostHog | undefined
export function init(opts?: PostHogOptions) {
if (
env.POSTHOG_TOKEN &&
env.POSTHOG_API_HOST &&
!env.SELF_HOSTED &&
env.POSTHOG_FEATURE_FLAGS_ENABLED
) {
console.log("initializing posthog client...")
posthog = new PostHog(env.POSTHOG_TOKEN, {
host: env.POSTHOG_API_HOST,
personalApiKey: env.POSTHOG_PERSONAL_TOKEN,
featureFlagsPollingInterval: Duration.fromMinutes(3).toMs(),
...opts,
})
} else {
console.log("posthog disabled")
}
}
export function shutdown() {
posthog?.shutdown()
}
export abstract class Flag<T> {
static boolean(defaultValue: boolean): Flag<boolean> {
return new BooleanFlag(defaultValue)
}
static string(defaultValue: string): Flag<string> {
return new StringFlag(defaultValue)
}
static number(defaultValue: number): Flag<number> {
return new NumberFlag(defaultValue)
}
protected constructor(public defaultValue: T) {}
abstract parse(value: any): T
}
type UnwrapFlag<F> = F extends Flag<infer U> ? U : never
export type FlagValues<T> = {
[K in keyof T]: UnwrapFlag<T[K]>
}
type KeysOfType<T, U> = {
[K in keyof T]: T[K] extends Flag<U> ? K : never
}[keyof T]
class BooleanFlag extends Flag<boolean> {
parse(value: any) {
if (typeof value === "string") {
return ["true", "t", "1"].includes(value.toLowerCase())
}
if (typeof value === "boolean") {
return value
}
throw new Error(`could not parse value "${value}" as boolean`)
}
}
class StringFlag extends Flag<string> {
parse(value: any) {
if (typeof value === "string") {
return value
}
throw new Error(`could not parse value "${value}" as string`)
}
}
class NumberFlag extends Flag<number> {
parse(value: any) {
if (typeof value === "number") {
return value
}
if (typeof value === "string") {
const parsed = parseFloat(value)
if (!isNaN(parsed)) {
return parsed
}
}
throw new Error(`could not parse value "${value}" as number`)
}
}
export class FlagSet<V extends Flag<any>, T extends { [key: string]: V }> {
// This is used to safely cache flags sets in the current request context.
// Because multiple sets could theoretically exist, we don't want the cache of
// one to leak into another.
private readonly setId: string
constructor(private readonly flagSchema: T) {
this.setId = crypto.randomUUID()
}
defaults(): FlagValues<T> {
return Object.keys(this.flagSchema).reduce((acc, key) => {
const typedKey = key as keyof T
acc[typedKey] = this.flagSchema[key].defaultValue
return acc
}, {} as FlagValues<T>)
}
isFlagName(name: string | number | symbol): name is keyof T {
return this.flagSchema[name as keyof T] !== undefined
}
async get<K extends keyof T>(
key: K,
ctx?: UserCtx
): Promise<FlagValues<T>[K]> {
const flags = await this.fetch(ctx)
return flags[key]
}
async isEnabled<K extends KeysOfType<T, boolean>>(
key: K,
ctx?: UserCtx
): Promise<boolean> {
const flags = await this.fetch(ctx)
return flags[key]
}
async fetch(ctx?: UserCtx): Promise<FlagValues<T>> {
return await tracer.trace("features.fetch", async span => {
const cachedFlags = context.getFeatureFlags<FlagValues<T>>(this.setId)
if (cachedFlags) {
span?.addTags({ fromCache: true })
return cachedFlags
}
const tags: Record<string, any> = {}
const flagValues = this.defaults()
const currentTenantId = context.getTenantId()
const specificallySetFalse = new Set<string>()
const split = (env.TENANT_FEATURE_FLAGS || "")
.split(",")
.map(x => x.split(":"))
for (const [tenantId, ...features] of split) {
if (!tenantId || (tenantId !== "*" && tenantId !== currentTenantId)) {
continue
}
tags[`readFromEnvironmentVars`] = true
for (let feature of features) {
let value = true
if (feature.startsWith("!")) {
feature = feature.slice(1)
value = false
specificallySetFalse.add(feature)
}
// ignore unknown flags
if (!this.isFlagName(feature)) {
continue
}
if (typeof flagValues[feature] !== "boolean") {
throw new Error(`Feature: ${feature} is not a boolean`)
}
// @ts-expect-error - TS does not like you writing into a generic type,
// but we know that it's okay in this case because it's just an object.
flagValues[feature as keyof FlagValues] = value
tags[`flags.${feature}.source`] = "environment"
}
}
const license = ctx?.user?.license
if (license) {
tags[`readFromLicense`] = true
for (const feature of license.features) {
if (!this.isFlagName(feature)) {
continue
}
if (
flagValues[feature] === true ||
specificallySetFalse.has(feature)
) {
// If the flag is already set to through environment variables, we
// don't want to override it back to false here.
continue
}
// @ts-expect-error - TS does not like you writing into a generic type,
// but we know that it's okay in this case because it's just an object.
flagValues[feature] = true
tags[`flags.${feature}.source`] = "license"
}
}
const identity = context.getIdentity()
tags[`identity.type`] = identity?.type
tags[`identity.tenantId`] = identity?.tenantId
tags[`identity._id`] = identity?._id
if (posthog && identity?.type === IdentityType.USER) {
tags[`readFromPostHog`] = true
const personProperties: Record<string, string> = {}
if (identity.tenantId) {
personProperties.tenantId = identity.tenantId
}
const posthogFlags = await posthog.getAllFlagsAndPayloads(
identity._id,
{
personProperties,
}
)
for (const [name, value] of Object.entries(posthogFlags.featureFlags)) {
if (!this.isFlagName(name)) {
// We don't want an unexpected PostHog flag to break the app, so we
// just log it and continue.
console.warn(`Unexpected posthog flag "${name}": ${value}`)
continue
}
if (flagValues[name] === true || specificallySetFalse.has(name)) {
// If the flag is already set to through environment variables, we
// don't want to override it back to false here.
continue
}
const payload = posthogFlags.featureFlagPayloads?.[name]
const flag = this.flagSchema[name]
try {
// @ts-expect-error - TS does not like you writing into a generic
// type, but we know that it's okay in this case because it's just
// an object.
flagValues[name] = flag.parse(payload || value)
tags[`flags.${name}.source`] = "posthog"
} catch (err) {
// We don't want an invalid PostHog flag to break the app, so we just
// log it and continue.
console.warn(`Error parsing posthog flag "${name}": ${value}`, err)
}
}
}
context.setFeatureFlags(this.setId, flagValues)
for (const [key, value] of Object.entries(flagValues)) {
tags[`flags.${key}.value`] = value
}
span?.addTags(tags)
return flagValues
})
}
}
// This is the primary source of truth for feature flags. If you want to add a
// new flag, add it here and use the `fetch` and `get` functions to access it.
// All of the machinery in this file is to make sure that flags have their
// default values set correctly and their types flow through the system.
export const flags = new FlagSet({
DEFAULT_VALUES: Flag.boolean(env.isDev()),
AUTOMATION_BRANCHING: Flag.boolean(env.isDev()),
SQS: Flag.boolean(env.isDev()),
[FeatureFlag.AI_CUSTOM_CONFIGS]: Flag.boolean(env.isDev()),
[FeatureFlag.ENRICHED_RELATIONSHIPS]: Flag.boolean(env.isDev()),
})
export * from "./features"
export * as testutils from "./tests/utils"

90
packages/backend-core/src/features/tests/features.spec.ts
View File

@ -1,9 +1,10 @@
import { IdentityContext, IdentityType, UserCtx } from "@budibase/types"
import { IdentityContext, IdentityType } from "@budibase/types"
import { Flag, FlagSet, FlagValues, init, shutdown } from "../"
import * as context from "../../context"
import environment, { withEnv } from "../../environment"
import nodeFetch from "node-fetch"
import nock from "nock"
import * as crypto from "crypto"
const schema = {
TEST_BOOLEAN: Flag.boolean(false),
@ -17,7 +18,6 @@ interface TestCase {
identity?: Partial<IdentityContext>
environmentFlags?: string
posthogFlags?: PostHogFlags
licenseFlags?: Array<string>
expected?: Partial<FlagValues<typeof schema>>
errorMessage?: string | RegExp
}
@ -27,10 +27,14 @@ interface PostHogFlags {
featureFlagPayloads?: Record<string, string>
}
function mockPosthogFlags(flags: PostHogFlags) {
function mockPosthogFlags(
flags: PostHogFlags,
opts?: { token?: string; distinct_id?: string }
) {
const { token = "test", distinct_id = "us_1234" } = opts || {}
nock("https://us.i.posthog.com")
.post("/decide/?v=3", body => {
return body.token === "test" && body.distinct_id === "us_1234"
return body.token === token && body.distinct_id === distinct_id
})
.reply(200, flags)
.persist()
@ -112,17 +116,6 @@ describe("feature flags", () => {
},
expected: { TEST_BOOLEAN: true },
},
{
it: "should be able to set boolean flags through the license",
licenseFlags: ["TEST_BOOLEAN"],
expected: { TEST_BOOLEAN: true },
},
{
it: "should not be able to override a negative environment flag from license",
environmentFlags: "default:!TEST_BOOLEAN",
licenseFlags: ["TEST_BOOLEAN"],
expected: { TEST_BOOLEAN: false },
},
{
it: "should not error on unrecognised PostHog flag",
posthogFlags: {
@ -130,18 +123,12 @@ describe("feature flags", () => {
},
expected: flags.defaults(),
},
{
it: "should not error on unrecognised license flag",
licenseFlags: ["UNDEFINED"],
expected: flags.defaults(),
},
])(
"$it",
async ({
identity,
environmentFlags,
posthogFlags,
licenseFlags,
expected,
errorMessage,
}) => {
@ -157,8 +144,6 @@ describe("feature flags", () => {
env.POSTHOG_API_HOST = "https://us.i.posthog.com"
}
const ctx = { user: { license: { features: licenseFlags || [] } } }
await withEnv(env, async () => {
// We need to pass in node-fetch here otherwise nock won't get used
// because posthog-node uses axios under the hood.
@ -180,18 +165,13 @@ describe("feature flags", () => {
await context.doInIdentityContext(fullIdentity, async () => {
if (errorMessage) {
await expect(flags.fetch(ctx as UserCtx)).rejects.toThrow(
errorMessage
)
await expect(flags.fetch()).rejects.toThrow(errorMessage)
} else if (expected) {
const values = await flags.fetch(ctx as UserCtx)
const values = await flags.fetch()
expect(values).toMatchObject(expected)
for (const [key, expectedValue] of Object.entries(expected)) {
const value = await flags.get(
key as keyof typeof schema,
ctx as UserCtx
)
const value = await flags.get(key as keyof typeof schema)
expect(value).toBe(expectedValue)
}
} else {
@ -214,6 +194,14 @@ describe("feature flags", () => {
lastName: "User",
}
// We need to pass in node-fetch here otherwise nock won't get used
// because posthog-node uses axios under the hood.
init({
fetch: (url, opts) => {
return nodeFetch(url, opts)
},
})
nock("https://us.i.posthog.com")
.post("/decide/?v=3", body => {
return body.token === "test" && body.distinct_id === "us_1234"
@ -230,4 +218,44 @@ describe("feature flags", () => {
}
)
})
it("should still get flags when user is logged out", async () => {
const env: Partial<typeof environment> = {
SELF_HOSTED: false,
POSTHOG_FEATURE_FLAGS_ENABLED: "true",
POSTHOG_API_HOST: "https://us.i.posthog.com",
POSTHOG_TOKEN: "test",
}
const ip = "127.0.0.1"
const hashedIp = crypto.createHash("sha512").update(ip).digest("hex")
await withEnv(env, async () => {
mockPosthogFlags(
{
featureFlags: { TEST_BOOLEAN: true },
},
{
distinct_id: hashedIp,
}
)
// We need to pass in node-fetch here otherwise nock won't get used
// because posthog-node uses axios under the hood.
init({
fetch: (url, opts) => {
return nodeFetch(url, opts)
},
})
await context.doInIPContext(ip, async () => {
await context.doInTenant("default", async () => {
const result = await flags.fetch()
expect(result.TEST_BOOLEAN).toBe(true)
})
})
shutdown()
})
})
})

64
packages/backend-core/src/features/tests/utils.ts Normal file
View File

@ -0,0 +1,64 @@
import { FeatureFlags, parseEnvFlags } from ".."
import { setEnv } from "../../environment"
function getCurrentFlags(): Record<string, Record<string, boolean>> {
const result: Record<string, Record<string, boolean>> = {}
for (const { tenantId, key, value } of parseEnvFlags(
process.env.TENANT_FEATURE_FLAGS || ""
)) {
const tenantFlags = result[tenantId] || {}
// Don't allow overwriting specifically false flags, to match the beheaviour
// of FlagSet.
if (tenantFlags[key] === false) {
continue
}
tenantFlags[key] = value
result[tenantId] = tenantFlags
}
return result
}
function buildFlagString(
flags: Record<string, Record<string, boolean>>
): string {
const parts: string[] = []
for (const [tenantId, tenantFlags] of Object.entries(flags)) {
for (const [key, value] of Object.entries(tenantFlags)) {
if (value === false) {
parts.push(`${tenantId}:!${key}`)
} else {
parts.push(`${tenantId}:${key}`)
}
}
}
return parts.join(",")
}
export function setFeatureFlags(
tenantId: string,
flags: Partial<FeatureFlags>
): () => void {
const current = getCurrentFlags()
for (const [key, value] of Object.entries(flags)) {
const tenantFlags = current[tenantId] || {}
tenantFlags[key] = value
current[tenantId] = tenantFlags
}
const flagString = buildFlagString(current)
return setEnv({ TENANT_FEATURE_FLAGS: flagString })
}
export function withFeatureFlags<T>(
tenantId: string,
flags: Partial<FeatureFlags>,
f: () => T
) {
const cleanup = setFeatureFlags(tenantId, flags)
const result = f()
if (result instanceof Promise) {
return result.finally(cleanup)
} else {
cleanup()
return result
}
}

1
packages/backend-core/src/middleware/index.ts
View File

@ -20,3 +20,4 @@ export { default as correlation } from "../logging/correlation/middleware"
export { default as errorHandling } from "./errorHandling"
export { default as querystringToBody } from "./querystringToBody"
export * as joiValidator from "./joi-validator"
export { default as ip } from "./ip"

12
packages/backend-core/src/middleware/ip.ts Normal file
View File

@ -0,0 +1,12 @@
import { Ctx } from "@budibase/types"
import { doInIPContext } from "../context"
export default async (ctx: Ctx, next: any) => {
if (ctx.ip) {
return await doInIPContext(ctx.ip, () => {
return next()
})
} else {
return next()
}
}

8
packages/backend-core/src/security/permissions.ts
View File

@ -65,7 +65,13 @@ export enum BuiltinPermissionID {
POWER = "power",
}
export const BUILTIN_PERMISSIONS = {
export const BUILTIN_PERMISSIONS: {
[key in keyof typeof BuiltinPermissionID]: {
_id: (typeof BuiltinPermissionID)[key]
name: string
permissions: Permission[]
}
} = {
PUBLIC: {
_id: BuiltinPermissionID.PUBLIC,
name: "Public",

9
packages/backend-core/src/sql/sql.ts
View File

@ -325,7 +325,7 @@ class InternalBuilder {
return input
}
private parseBody(body: any) {
private parseBody(body: Record<string, any>) {
for (let [key, value] of Object.entries(body)) {
const { column } = this.splitter.run(key)
const schema = this.table.schema[column]
@ -1259,6 +1259,10 @@ class InternalBuilder {
create(opts: QueryOptions): Knex.QueryBuilder {
const { body } = this.query
if (!body) {
throw new Error("Cannot create without row body")
}
let query = this.qualifiedKnex({ alias: false })
const parsedBody = this.parseBody(body)
@ -1417,6 +1421,9 @@ class InternalBuilder {
update(opts: QueryOptions): Knex.QueryBuilder {
const { body, filters } = this.query
if (!body) {
throw new Error("Cannot update without row body")
}
let query = this.qualifiedKnex()
const parsedBody = this.parseBody(body)
query = this.addFilters(query, filters)

12
packages/backend-core/src/users/users.ts
View File

@ -24,6 +24,7 @@ import * as context from "../context"
import { getGlobalDB } from "../context"
import { isCreator } from "./utils"
import { UserDB } from "./db"
import { dataFilters } from "@budibase/shared-core"
type GetOpts = { cleanup?: boolean }
@ -262,10 +263,17 @@ export async function paginatedUsers({
userList = await bulkGetGlobalUsersById(query?.oneOf?._id, {
cleanup: true,
})
} else if (query) {
// TODO: this should use SQS search, but the logic is built in the 'server' package. Using the in-memory filtering to get this working meanwhile
const response = await db.allDocs<User>(
getGlobalUserParams(null, { ...opts, limit: undefined })
)
userList = response.rows.map(row => row.doc!)
userList = dataFilters.search(userList, { query, limit: opts.limit }).rows
} else {
// no search, query allDocs
const response = await db.allDocs(getGlobalUserParams(null, opts))
userList = response.rows.map((row: any) => row.doc)
const response = await db.allDocs<User>(getGlobalUserParams(null, opts))
userList = response.rows.map(row => row.doc!)
}
return pagination(userList, pageSize, {
paginate: true,

2
packages/frontend-core/src/components/grid/stores/datasources/viewV2.js
View File

@ -191,7 +191,7 @@ export const initialise = context => {
if ($view?.id !== $datasource.id) {
return
}
if (JSON.stringify($filter) !== JSON.stringify($view.query)) {
if (JSON.stringify($filter) !== JSON.stringify($view.queryUI)) {
await datasource.actions.saveDefinition({
...$view,
query: $filter,

23
packages/server/src/api/controllers/permission.ts
View File

@ -1,7 +1,6 @@
import { permissions, roles, context } from "@budibase/backend-core"
import {
UserCtx,
Role,
GetResourcePermsResponse,
ResourcePermissionInfo,
GetDependantResourcesResponse,
@ -9,6 +8,7 @@ import {
AddPermissionRequest,
RemovePermissionRequest,
RemovePermissionResponse,
FetchResourcePermissionInfoResponse,
} from "@budibase/types"
import {
CURRENTLY_SUPPORTED_LEVELS,
@ -28,10 +28,12 @@ export function fetchLevels(ctx: UserCtx) {
ctx.body = SUPPORTED_LEVELS
}
export async function fetch(ctx: UserCtx) {
export async function fetch(
ctx: UserCtx<void, FetchResourcePermissionInfoResponse>
) {
const db = context.getAppDB()
const dbRoles: Role[] = await sdk.permissions.getAllDBRoles(db)
let permissions: any = {}
const dbRoles = await sdk.permissions.getAllDBRoles(db)
let permissions: Record<string, Record<string, string>> = {}
// create an object with structure role ID -> resource ID -> level
for (let role of dbRoles) {
if (!role.permissions) {
@ -43,13 +45,13 @@ export async function fetch(ctx: UserCtx) {
}
for (let [resource, levelArr] of Object.entries(role.permissions)) {
const levels: string[] = Array.isArray(levelArr) ? levelArr : [levelArr]
const perms: Record<string, string> = {}
const perms: Record<string, string> = permissions[resource] || {}
levels.forEach(level => (perms[level] = roleId!))
permissions[resource] = perms
}
}
// apply the base permissions
const finalPermissions: Record<string, Record<string, string>> = {}
const finalPermissions: FetchResourcePermissionInfoResponse = {}
for (let [resource, permission] of Object.entries(permissions)) {
const basePerms = getBasePermissions(resource)
finalPermissions[resource] = Object.assign(basePerms, permission)
@ -92,18 +94,17 @@ export async function getDependantResources(
export async function addPermission(ctx: UserCtx<void, AddPermissionResponse>) {
const params: AddPermissionRequest = ctx.params
ctx.body = await sdk.permissions.updatePermissionOnRole(
params,
PermissionUpdateType.ADD
)
await sdk.permissions.updatePermissionOnRole(params, PermissionUpdateType.ADD)
ctx.status = 200
}
export async function removePermission(
ctx: UserCtx<void, RemovePermissionResponse>
) {
const params: RemovePermissionRequest = ctx.params
ctx.body = await sdk.permissions.updatePermissionOnRole(
await sdk.permissions.updatePermissionOnRole(
params,
PermissionUpdateType.REMOVE
)
ctx.status = 200
}

46
packages/server/src/api/controllers/row/ExternalRequest.ts
View File

@ -4,6 +4,7 @@ import {
AutoFieldSubType,
AutoReason,
Datasource,
DatasourcePlusQueryResponse,
FieldSchema,
FieldType,
FilterType,
@ -269,18 +270,13 @@ export class ExternalRequest<T extends Operation> {
}
}
private async removeManyToManyRelationships(
rowId: string,
table: Table,
colName: string
) {
private async removeManyToManyRelationships(rowId: string, table: Table) {
const tableId = table._id!
const filters = this.prepareFilters(rowId, {}, table)
// safety check, if there are no filters on deletion bad things happen
if (Object.keys(filters).length !== 0) {
return getDatasourceAndQuery({
endpoint: getEndpoint(tableId, Operation.DELETE),
body: { [colName]: null },
filters,
meta: {
table,
@ -291,13 +287,18 @@ export class ExternalRequest<T extends Operation> {
}
}
private async removeOneToManyRelationships(rowId: string, table: Table) {
private async removeOneToManyRelationships(
rowId: string,
table: Table,
colName: string
) {
const tableId = table._id!
const filters = this.prepareFilters(rowId, {}, table)
// safety check, if there are no filters on deletion bad things happen
if (Object.keys(filters).length !== 0) {
return getDatasourceAndQuery({
endpoint: getEndpoint(tableId, Operation.UPDATE),
body: { [colName]: null },
filters,
meta: {
table,
@ -557,8 +558,9 @@ export class ExternalRequest<T extends Operation> {
return matchesPrimaryLink
}
const matchesSecondayLink = row[linkSecondary] === body?.[linkSecondary]
return matchesPrimaryLink && matchesSecondayLink
const matchesSecondaryLink =
row[linkSecondary] === body?.[linkSecondary]
return matchesPrimaryLink && matchesSecondaryLink
}
const existingRelationship = rows.find((row: { [key: string]: any }) =>
@ -595,8 +597,8 @@ export class ExternalRequest<T extends Operation> {
for (let row of rows) {
const rowId = generateIdForRow(row, table)
const promise: Promise<any> = isMany
? this.removeManyToManyRelationships(rowId, table, colName)
: this.removeOneToManyRelationships(rowId, table)
? this.removeManyToManyRelationships(rowId, table)
: this.removeOneToManyRelationships(rowId, table, colName)
if (promise) {
promises.push(promise)
}
@ -619,12 +621,12 @@ export class ExternalRequest<T extends Operation> {
rows.map(row => {
const rowId = generateIdForRow(row, table)
return isMany
? this.removeManyToManyRelationships(
? this.removeManyToManyRelationships(rowId, table)
: this.removeOneToManyRelationships(
rowId,
table,
relationshipColumn.fieldName
)
: this.removeOneToManyRelationships(rowId, table)
})
)
}
@ -669,6 +671,7 @@ export class ExternalRequest<T extends Operation> {
config.includeSqlRelationships === IncludeRelationship.INCLUDE
// clean up row on ingress using schema
const unprocessedRow = config.row
const processed = this.inputProcessing(row, table)
row = processed.row
let manyRelationships = processed.manyRelationships
@ -743,9 +746,20 @@ export class ExternalRequest<T extends Operation> {
// aliasing can be disabled fully if desired
const aliasing = new sdk.rows.AliasTables(Object.keys(this.tables))
let response = env.SQL_ALIASING_DISABLE
? await getDatasourceAndQuery(json)
: await aliasing.queryWithAliasing(json, makeExternalQuery)
let response: DatasourcePlusQueryResponse
// there's a chance after input processing nothing needs updated, so pass over the call
// we might still need to perform other operations like updating the foreign keys on other rows
if (
this.operation === Operation.UPDATE &&
Object.keys(row || {}).length === 0 &&
unprocessedRow
) {
response = [unprocessedRow]
} else {
response = env.SQL_ALIASING_DISABLE
? await getDatasourceAndQuery(json)
: await aliasing.queryWithAliasing(json, makeExternalQuery)
}
// if it's a counting operation there will be no more processing, just return the number
if (this.operation === Operation.COUNT) {

6
packages/server/src/api/controllers/row/external.ts
View File

@ -27,6 +27,7 @@ import {
} from "../../../utilities/rowProcessor"
import { cloneDeep } from "lodash"
import { generateIdForRow } from "./utils"
import { helpers } from "@budibase/shared-core"
export async function handleRequest<T extends Operation>(
operation: T,
@ -42,6 +43,11 @@ export async function handleRequest<T extends Operation>(
export async function patch(ctx: UserCtx<PatchRowRequest, PatchRowResponse>) {
const source = await utils.getSource(ctx)
if (sdk.views.isView(source) && helpers.views.isCalculationView(source)) {
ctx.throw(400, "Cannot update rows through a calculation view")
}
const table = await utils.getTableFromSource(source)
const { _id, ...rowData } = ctx.request.body

7
packages/server/src/api/controllers/row/internal.ts
View File

@ -22,13 +22,20 @@ import sdk from "../../../sdk"
import { getLinkedTableIDs } from "../../../db/linkedRows/linkUtils"
import { flatten } from "lodash"
import { findRow } from "../../../sdk/app/rows/internal"
import { helpers } from "@budibase/shared-core"
export async function patch(ctx: UserCtx<PatchRowRequest, PatchRowResponse>) {
const { tableId } = utils.getSourceId(ctx)
const source = await utils.getSource(ctx)
if (sdk.views.isView(source) && helpers.views.isCalculationView(source)) {
ctx.throw(400, "Cannot update rows through a calculation view")
}
const table = sdk.views.isView(source)
? await sdk.views.getTable(source.id)
: source
const inputs = ctx.request.body
const isUserTable = tableId === InternalTables.USER_METADATA
let oldRow

2
packages/server/src/api/controllers/table/external.ts
View File

@ -31,7 +31,7 @@ function getDatasourceId(table: Table) {
return breakExternalTableId(table._id).datasourceId
}
export async function save(
export async function updateTable(
ctx: UserCtx<SaveTableRequest, SaveTableResponse>,
renaming?: RenameColumn
) {

14
packages/server/src/api/controllers/table/index.ts
View File

@ -102,18 +102,22 @@ export async function find(ctx: UserCtx<void, TableResponse>) {
export async function save(ctx: UserCtx<SaveTableRequest, SaveTableResponse>) {
const appId = ctx.appId
const table = ctx.request.body
const isImport = table.rows
const { rows, ...table } = ctx.request.body
const isImport = rows
const renaming = ctx.request.body._rename
const isCreate = !table._id
checkDefaultFields(table)
const api = pickApi({ table })
let savedTable = await api.save(ctx, renaming)
if (!table._id) {
let savedTable: Table
if (isCreate) {
savedTable = await sdk.tables.create(table, rows, ctx.user._id)
savedTable = await sdk.tables.enrichViewSchemas(savedTable)
await events.table.created(savedTable)
} else {
const api = pickApi({ table })
savedTable = await api.updateTable(ctx, renaming)
await events.table.updated(savedTable)
}
if (renaming) {

9
packages/server/src/api/controllers/table/internal.ts
View File

@ -12,7 +12,7 @@ import {
} from "@budibase/types"
import sdk from "../../../sdk"
export async function save(
export async function updateTable(
ctx: UserCtx<SaveTableRequest, SaveTableResponse>,
renaming?: RenameColumn
) {
@ -25,19 +25,16 @@ export async function save(
sourceType: rest.sourceType || TableSourceType.INTERNAL,
}
const isImport = !!rows
if (!tableToSave.views) {
tableToSave.views = {}
}
try {
const { table } = await sdk.tables.internal.save(tableToSave, {
user: ctx.user,
userId: ctx.user._id,
rowsToImport: rows,
tableId: ctx.request.body._id,
renaming,
isImport,
})
return table
@ -72,7 +69,7 @@ export async function bulkImport(
await handleDataImport(table, {
importRows: rows,
identifierFields,
user: ctx.user,
userId: ctx.user._id,
})
return table
}

2
packages/server/src/api/controllers/table/tests/utils.spec.ts
View File

@ -41,7 +41,7 @@ describe("utils", () => {
const data = [{ name: "Alice" }, { name: "Bob" }, { name: "Claire" }]
const result = await importToRows(data, table, config.user)
const result = await importToRows(data, table, config.user?._id)
expect(result).toEqual([
expect.objectContaining({
autoId: 1,

20
packages/server/src/api/controllers/table/utils.ts
View File

@ -18,7 +18,6 @@ import { quotas } from "@budibase/pro"
import { events, context, features } from "@budibase/backend-core"
import {
AutoFieldSubType,
ContextUser,
Datasource,
Row,
SourceName,
@ -122,7 +121,7 @@ export function makeSureTableUpToDate(table: Table, tableToSave: Table) {
export async function importToRows(
data: Row[],
table: Table,
user?: ContextUser,
userId?: string,
opts?: { keepCouchId: boolean }
) {
const originalTable = table
@ -136,7 +135,7 @@ export async function importToRows(
// We use a reference to table here and update it after input processing,
// so that we can auto increment auto IDs in imported data properly
const processed = await inputProcessing(user?._id, table, row, {
const processed = await inputProcessing(userId, table, row, {
noAutoRelationships: true,
})
row = processed
@ -167,11 +166,10 @@ export async function importToRows(
export async function handleDataImport(
table: Table,
opts?: { identifierFields?: string[]; user?: ContextUser; importRows?: Row[] }
opts?: { identifierFields?: string[]; userId?: string; importRows?: Row[] }
) {
const schema = table.schema
const identifierFields = opts?.identifierFields || []
const user = opts?.user
const importRows = opts?.importRows
if (!importRows || !isRows(importRows) || !isSchema(schema)) {
@ -181,7 +179,7 @@ export async function handleDataImport(
const db = context.getAppDB()
const data = parse(importRows, table)
const finalData = await importToRows(data, table, user, {
const finalData = await importToRows(data, table, opts?.userId, {
keepCouchId: identifierFields.includes("_id"),
})
@ -282,22 +280,22 @@ export function checkStaticTables(table: Table) {
class TableSaveFunctions {
db: Database
user?: ContextUser
userId?: string
oldTable?: Table
importRows?: Row[]
rows: Row[]
constructor({
user,
userId,
oldTable,
importRows,
}: {
user?: ContextUser
userId?: string
oldTable?: Table
importRows?: Row[]
}) {
this.db = context.getAppDB()
this.user = user
this.userId = userId
this.oldTable = oldTable
this.importRows = importRows
// any rows that need updated
@ -329,7 +327,7 @@ class TableSaveFunctions {
table = await handleSearchIndexes(table)
table = await handleDataImport(table, {
importRows: this.importRows,
user: this.user,
userId: this.userId,
})
if (await features.flags.isEnabled("SQS")) {
await sdk.tables.sqs.addTable(table)

17
packages/server/src/api/routes/tests/application.spec.ts
View File

@ -14,12 +14,7 @@ jest.mock("../../../utilities/redis", () => ({
import { checkBuilderEndpoint } from "./utilities/TestFunctions"
import * as setup from "./utilities"
import { AppStatus } from "../../../db/utils"
import {
events,
utils,
context,
withEnv as withCoreEnv,
} from "@budibase/backend-core"
import { events, utils, context, features } from "@budibase/backend-core"
import env from "../../../environment"
import { type App } from "@budibase/types"
import tk from "timekeeper"
@ -358,9 +353,13 @@ describe("/applications", () => {
.delete(`/api/global/roles/${prodAppId}`)
.reply(200, {})
await withCoreEnv({ TENANT_FEATURE_FLAGS: "*:SQS" }, async () => {
await config.api.application.delete(app.appId)
})
await features.testutils.withFeatureFlags(
"*",
{ SQS: true },
async () => {
await config.api.application.delete(app.appId)
}
)
})
})

379
packages/server/src/api/routes/tests/permissions.spec.ts
View File

@ -1,5 +1,5 @@
import { roles } from "@budibase/backend-core"
import { Document, PermissionLevel, Row, Table, ViewV2 } from "@budibase/types"
import { Document, PermissionLevel, Row } from "@budibase/types"
import * as setup from "./utilities"
import { generator, mocks } from "@budibase/backend-core/tests"
@ -9,13 +9,11 @@ const { BUILTIN_ROLE_IDS } = roles
const HIGHER_ROLE_ID = BUILTIN_ROLE_IDS.BASIC
const STD_ROLE_ID = BUILTIN_ROLE_IDS.PUBLIC
const DEFAULT_TABLE_ROLE_ID = BUILTIN_ROLE_IDS.ADMIN
describe("/permission", () => {
let request = setup.getRequest()
let config = setup.getConfig()
let table: Table & { _id: string }
let perms: Document[]
let row: Row
let view: ViewV2
afterAll(setup.afterAll)
@ -25,18 +23,6 @@ describe("/permission", () => {
beforeEach(async () => {
mocks.licenses.useCloudFree()
table = (await config.createTable()) as typeof table
row = await config.createRow()
view = await config.api.viewV2.create({
tableId: table._id!,
name: generator.guid(),
})
perms = await config.api.permission.add({
roleId: STD_ROLE_ID,
resourceId: table._id,
level: PermissionLevel.READ,
})
})
describe("levels", () => {
@ -54,134 +40,251 @@ describe("/permission", () => {
})
})
describe("add", () => {
it("should be able to add permission to a role for the table", async () => {
expect(perms.length).toEqual(1)
expect(perms[0]._id).toEqual(`${STD_ROLE_ID}`)
})
describe("table permissions", () => {
let tableId: string
it("should get the resource permissions", async () => {
const res = await request
.get(`/api/permission/${table._id}`)
.set(config.defaultHeaders())
.expect("Content-Type", /json/)
.expect(200)
expect(res.body).toEqual({
permissions: {
read: { permissionType: "EXPLICIT", role: STD_ROLE_ID },
write: { permissionType: "BASE", role: HIGHER_ROLE_ID },
},
})
})
it("should get resource permissions with multiple roles", async () => {
perms = await config.api.permission.add({
roleId: HIGHER_ROLE_ID,
resourceId: table._id,
level: PermissionLevel.WRITE,
})
const res = await config.api.permission.get(table._id)
expect(res).toEqual({
permissions: {
read: { permissionType: "EXPLICIT", role: STD_ROLE_ID },
write: { permissionType: "EXPLICIT", role: HIGHER_ROLE_ID },
},
})
const allRes = await request
.get(`/api/permission`)
.set(config.defaultHeaders())
.expect("Content-Type", /json/)
.expect(200)
expect(allRes.body[table._id]["read"]).toEqual(STD_ROLE_ID)
expect(allRes.body[table._id]["write"]).toEqual(HIGHER_ROLE_ID)
})
})
describe("remove", () => {
it("should be able to remove the permission", async () => {
const res = await config.api.permission.revoke({
roleId: STD_ROLE_ID,
resourceId: table._id,
level: PermissionLevel.READ,
})
expect(res[0]._id).toEqual(STD_ROLE_ID)
const permsRes = await config.api.permission.get(table._id)
expect(permsRes.permissions[STD_ROLE_ID]).toBeUndefined()
})
})
describe("check public user allowed", () => {
it("should be able to read the row", async () => {
// replicate changes before checking permissions
await config.publish()
const res = await request
.get(`/api/${table._id}/rows`)
.set(config.publicHeaders())
.expect("Content-Type", /json/)
.expect(200)
expect(res.body[0]._id).toEqual(row._id)
})
it("should be able to access the view data when the table is set to public and with no view permissions overrides", async () => {
// Make view inherit table permissions. Needed for backwards compatibility with existing views.
await config.api.permission.revoke({
roleId: STD_ROLE_ID,
resourceId: view.id,
level: PermissionLevel.READ,
})
// replicate changes before checking permissions
await config.publish()
const res = await config.api.viewV2.publicSearch(view.id)
expect(res.rows[0]._id).toEqual(row._id)
})
it("should not be able to access the view data when the table is not public and there are no view permissions overrides", async () => {
await config.api.permission.revoke({
roleId: STD_ROLE_ID,
resourceId: table._id,
level: PermissionLevel.READ,
})
// Make view inherit table permissions. Needed for backwards compatibility with existing views.
await config.api.permission.revoke({
roleId: STD_ROLE_ID,
resourceId: view.id,
level: PermissionLevel.READ,
})
// replicate changes before checking permissions
await config.publish()
await config.api.viewV2.publicSearch(view.id, undefined, { status: 401 })
})
it("should use the view permissions", async () => {
beforeEach(async () => {
const table = await config.createTable()
tableId = table._id!
await config.api.permission.add({
roleId: STD_ROLE_ID,
resourceId: view.id,
resourceId: tableId,
level: PermissionLevel.READ,
})
await config.api.permission.revoke({
roleId: STD_ROLE_ID,
resourceId: table._id,
level: PermissionLevel.READ,
})
// replicate changes before checking permissions
await config.publish()
const res = await config.api.viewV2.publicSearch(view.id)
expect(res.rows[0]._id).toEqual(row._id)
})
it("shouldn't allow writing from a public user", async () => {
const res = await request
.post(`/api/${table._id}/rows`)
.send(basicRow(table._id))
.set(config.publicHeaders())
.expect("Content-Type", /json/)
.expect(401)
expect(res.status).toEqual(401)
it("tables should be defaulted to admin", async () => {
const table = await config.createTable()
const { permissions } = await config.api.permission.get(table._id!)
expect(permissions).toEqual({
read: {
permissionType: "EXPLICIT",
role: DEFAULT_TABLE_ROLE_ID,
},
write: {
permissionType: "EXPLICIT",
role: DEFAULT_TABLE_ROLE_ID,
},
})
})
describe("add", () => {
it("should be able to add permission to a role for the table", async () => {
const res = await request
.get(`/api/permission/${tableId}`)
.set(config.defaultHeaders())
.expect("Content-Type", /json/)
.expect(200)
expect(res.body).toEqual({
permissions: {
read: { permissionType: "EXPLICIT", role: STD_ROLE_ID },
write: { permissionType: "EXPLICIT", role: DEFAULT_TABLE_ROLE_ID },
},
})
})
it("should get resource permissions with multiple roles", async () => {
await config.api.permission.add({
roleId: HIGHER_ROLE_ID,
resourceId: tableId,
level: PermissionLevel.WRITE,
})
const res = await config.api.permission.get(tableId)
expect(res).toEqual({
permissions: {
read: { permissionType: "EXPLICIT", role: STD_ROLE_ID },
write: { permissionType: "EXPLICIT", role: HIGHER_ROLE_ID },
},
})
const allRes = await request
.get(`/api/permission`)
.set(config.defaultHeaders())
.expect("Content-Type", /json/)
.expect(200)
expect(allRes.body[tableId]["read"]).toEqual(STD_ROLE_ID)
expect(allRes.body[tableId]["write"]).toEqual(HIGHER_ROLE_ID)
})
})
describe("remove", () => {
it("should be able to remove the permission", async () => {
await config.api.permission.revoke({
roleId: STD_ROLE_ID,
resourceId: tableId,
level: PermissionLevel.READ,
})
const permsRes = await config.api.permission.get(tableId)
expect(permsRes.permissions[STD_ROLE_ID]).toBeUndefined()
})
})
describe("check public user allowed", () => {
let viewId: string
let row: Row
beforeEach(async () => {
const view = await config.api.viewV2.create({
tableId,
name: generator.guid(),
})
viewId = view.id
row = await config.createRow()
})
it("should be able to read the row", async () => {
// replicate changes before checking permissions
await config.publish()
const res = await request
.get(`/api/${tableId}/rows`)
.set(config.publicHeaders())
.expect("Content-Type", /json/)
.expect(200)
expect(res.body[0]._id).toEqual(row._id)
})
it("should be able to access the view data when the table is set to public and with no view permissions overrides", async () => {
// Make view inherit table permissions. Needed for backwards compatibility with existing views.
await config.api.permission.revoke({
roleId: STD_ROLE_ID,
resourceId: viewId,
level: PermissionLevel.READ,
})
// replicate changes before checking permissions
await config.publish()
const res = await config.api.viewV2.publicSearch(viewId)
expect(res.rows[0]._id).toEqual(row._id)
})
it("should not be able to access the view data when the table is not public and there are no view permissions overrides", async () => {
await config.api.permission.revoke({
roleId: STD_ROLE_ID,
resourceId: tableId,
level: PermissionLevel.READ,
})
// Make view inherit table permissions. Needed for backwards compatibility with existing views.
await config.api.permission.revoke({
roleId: STD_ROLE_ID,
resourceId: viewId,
level: PermissionLevel.READ,
})
// replicate changes before checking permissions
await config.publish()
await config.api.viewV2.publicSearch(viewId, undefined, {
status: 401,
})
})
it("should use the view permissions", async () => {
await config.api.permission.add({
roleId: STD_ROLE_ID,
resourceId: viewId,
level: PermissionLevel.READ,
})
await config.api.permission.revoke({
roleId: STD_ROLE_ID,
resourceId: tableId,
level: PermissionLevel.READ,
})
// replicate changes before checking permissions
await config.publish()
const res = await config.api.viewV2.publicSearch(viewId)
expect(res.rows[0]._id).toEqual(row._id)
})
it("shouldn't allow writing from a public user", async () => {
const res = await request
.post(`/api/${tableId}/rows`)
.send(basicRow(tableId))
.set(config.publicHeaders())
.expect("Content-Type", /json/)
.expect(401)
expect(res.status).toEqual(401)
})
})
})
describe("view permissions", () => {
let tableId: string
let viewId: string
beforeEach(async () => {
const table = await config.createTable()
tableId = table._id!
const view = await config.api.viewV2.create({
tableId,
name: generator.guid(),
})
viewId = view.id
})
it("default permissions inherits and persists the table default value", async () => {
const { permissions } = await config.api.permission.get(viewId)
expect(permissions).toEqual({
read: {
permissionType: "EXPLICIT",
role: DEFAULT_TABLE_ROLE_ID,
inheritablePermission: DEFAULT_TABLE_ROLE_ID,
},
write: {
permissionType: "EXPLICIT",
role: DEFAULT_TABLE_ROLE_ID,
inheritablePermission: DEFAULT_TABLE_ROLE_ID,
},
})
})
it("does not update view permissions once persisted, even if table permissions change", async () => {
await config.api.permission.add({
roleId: STD_ROLE_ID,
resourceId: tableId,
level: PermissionLevel.READ,
})
const { permissions } = await config.api.permission.get(viewId)
expect(permissions).toEqual({
read: {
permissionType: "EXPLICIT",
role: DEFAULT_TABLE_ROLE_ID,
inheritablePermission: STD_ROLE_ID,
},
write: {
permissionType: "EXPLICIT",
role: DEFAULT_TABLE_ROLE_ID,
inheritablePermission: DEFAULT_TABLE_ROLE_ID,
},
})
})
it("can sets permissions inherits explicit view permissions", async () => {
await config.api.permission.add({
roleId: HIGHER_ROLE_ID,
resourceId: viewId,
level: PermissionLevel.WRITE,
})
const { permissions } = await config.api.permission.get(viewId)
expect(permissions).toEqual({
read: {
permissionType: "EXPLICIT",
role: DEFAULT_TABLE_ROLE_ID,
inheritablePermission: DEFAULT_TABLE_ROLE_ID,
},
write: {
permissionType: "EXPLICIT",
role: HIGHER_ROLE_ID,
inheritablePermission: DEFAULT_TABLE_ROLE_ID,
},
})
})
})

29
packages/server/src/api/routes/tests/queries/generic-sql.spec.ts
View File

@ -28,6 +28,7 @@ describe.each(
const config = setup.getConfig()
const isOracle = dbName === DatabaseName.ORACLE
const isMsSQL = dbName === DatabaseName.SQL_SERVER
const isPostgres = dbName === DatabaseName.POSTGRES
let rawDatasource: Datasource
let datasource: Datasource
@ -47,6 +48,9 @@ describe.each(
transformer: "return data",
readable: true,
}
if (query.fields?.sql && typeof query.fields.sql !== "string") {
throw new Error("Unable to create with knex structure in 'sql' field")
}
return await config.api.query.save(
{ ...defaultQuery, ...query },
expectations
@ -207,6 +211,31 @@ describe.each(
expect(prodQuery.parameters).toBeUndefined()
expect(prodQuery.schema).toBeDefined()
})
isPostgres &&
it("should be able to handle a JSON aggregate with newlines", async () => {
const jsonStatement = `COALESCE(json_build_object('name', name),'{"name":{}}'::json)`
const query = await createQuery({
fields: {
sql: client("test_table")
.select([
"*",
client.raw(
`${jsonStatement} as json,\n${jsonStatement} as json2`
),
])
.toString(),
},
})
const res = await config.api.query.execute(
query._id!,
{},
{
status: 200,
}
)
expect(res).toBeDefined()
})
})
})

72
packages/server/src/api/routes/tests/row.spec.ts
View File

@ -13,8 +13,7 @@ import {
context,
InternalTable,
tenancy,
withEnv as withCoreEnv,
setEnv as setCoreEnv,
features,
} from "@budibase/backend-core"
import { quotas } from "@budibase/pro"
import {
@ -40,7 +39,6 @@ import {
TableSchema,
JsonFieldSubType,
RowExportFormat,
FeatureFlag,
RelationSchemaField,
} from "@budibase/types"
import { generator, mocks } from "@budibase/backend-core/tests"
@ -98,12 +96,12 @@ describe.each([
let envCleanup: (() => void) | undefined
beforeAll(async () => {
await withCoreEnv({ TENANT_FEATURE_FLAGS: "*:SQS" }, () => config.init())
if (isLucene) {
envCleanup = setCoreEnv({ TENANT_FEATURE_FLAGS: "*:!SQS" })
} else if (isSqs) {
envCleanup = setCoreEnv({ TENANT_FEATURE_FLAGS: "*:SQS" })
}
await features.testutils.withFeatureFlags("*", { SQS: true }, () =>
config.init()
)
envCleanup = features.testutils.setFeatureFlags("*", {
SQS: isSqs,
})
if (dsProvider) {
const rawDatasource = await dsProvider
@ -1116,6 +1114,33 @@ describe.each([
expect(getResp.user2[0]._id).toEqual(user2._id)
})
it("should be able to remove a relationship from many side", async () => {
const row = await config.api.row.save(otherTable._id!, {
name: "test",
description: "test",
})
const row2 = await config.api.row.save(otherTable._id!, {
name: "test",
description: "test",
})
const { _id } = await config.api.row.save(table._id!, {
relationship: [{ _id: row._id }, { _id: row2._id }],
})
const relatedRow = await config.api.row.get(table._id!, _id!, {
status: 200,
})
expect(relatedRow.relationship.length).toEqual(2)
await config.api.row.save(table._id!, {
...relatedRow,
relationship: [{ _id: row._id }],
})
const afterRelatedRow = await config.api.row.get(table._id!, _id!, {
status: 200,
})
expect(afterRelatedRow.relationship.length).toEqual(1)
expect(afterRelatedRow.relationship[0]._id).toEqual(row._id)
})
it("should be able to update relationships when both columns are same name", async () => {
let row = await config.api.row.save(table._id!, {
name: "test",
@ -1848,7 +1873,7 @@ describe.each([
})
describe("exportRows", () => {
beforeAll(async () => {
beforeEach(async () => {
table = await config.api.table.save(defaultTable())
})
@ -1885,6 +1910,16 @@ describe.each([
})
})
it("should allow exporting without filtering", async () => {
const existing = await config.api.row.save(table._id!, {})
const res = await config.api.row.exportRows(table._id!)
const results = JSON.parse(res)
expect(results.length).toEqual(1)
const row = results[0]
expect(row._id).toEqual(existing._id)
})
it("should allow exporting only certain columns", async () => {
const existing = await config.api.row.save(table._id!, {})
const res = await config.api.row.exportRows(table._id!, {
@ -2517,15 +2552,9 @@ describe.each([
let flagCleanup: (() => void) | undefined
beforeAll(async () => {
const env = {
TENANT_FEATURE_FLAGS: `*:${FeatureFlag.ENRICHED_RELATIONSHIPS}`,
}
if (isSqs) {
env.TENANT_FEATURE_FLAGS = `${env.TENANT_FEATURE_FLAGS},*:SQS`
} else {
env.TENANT_FEATURE_FLAGS = `${env.TENANT_FEATURE_FLAGS},*:!SQS`
}
flagCleanup = setCoreEnv(env)
flagCleanup = features.testutils.setFeatureFlags("*", {
ENRICHED_RELATIONSHIPS: true,
})
const aux2Table = await config.api.table.save(saveTableRequest())
const aux2Data = await config.api.row.save(aux2Table._id!, {})
@ -2752,9 +2781,10 @@ describe.each([
it.each(testScenarios)(
"does not enrich relationships when not enabled (via %s)",
async (__, retrieveDelegate) => {
await withCoreEnv(
await features.testutils.withFeatureFlags(
"*",
{
TENANT_FEATURE_FLAGS: `*:!${FeatureFlag.ENRICHED_RELATIONSHIPS}`,
ENRICHED_RELATIONSHIPS: false,
},
async () => {
const otherRows = _.sampleSize(auxData, 5)

19
packages/server/src/api/routes/tests/search.spec.ts
View File

@ -7,9 +7,9 @@ import {
import {
context,
db as dbCore,
features,
MAX_VALID_DATE,
MIN_VALID_DATE,
setEnv as setCoreEnv,
SQLITE_DESIGN_DOC_ID,
utils,
withEnv as withCoreEnv,
@ -94,16 +94,12 @@ describe.each([
}
beforeAll(async () => {
await withCoreEnv({ TENANT_FEATURE_FLAGS: "*:SQS" }, () => config.init())
if (isLucene) {
envCleanup = setCoreEnv({
TENANT_FEATURE_FLAGS: "*:!SQS",
})
} else if (isSqs) {
envCleanup = setCoreEnv({
TENANT_FEATURE_FLAGS: "*:SQS",
})
}
await features.testutils.withFeatureFlags("*", { SQS: true }, () =>
config.init()
)
envCleanup = features.testutils.setFeatureFlags("*", {
SQS: isSqs,
})
if (config.app?.appId) {
config.app = await config.api.application.update(config.app?.appId, {
@ -191,7 +187,6 @@ describe.each([
if (isInMemory) {
return dataFilters.search(_.cloneDeep(rows), {
...this.query,
tableId: tableOrViewId,
})
} else {
return config.api.row.search(tableOrViewId, this.query)

69
packages/server/src/api/routes/tests/templates.spec.ts
View File

@ -2,7 +2,7 @@ import * as setup from "./utilities"
import path from "path"
import nock from "nock"
import { generator } from "@budibase/backend-core/tests"
import { withEnv as withCoreEnv, env as coreEnv } from "@budibase/backend-core"
import { features } from "@budibase/backend-core"
interface App {
background: string
@ -85,41 +85,44 @@ describe("/templates", () => {
it.each(["sqs", "lucene"])(
`should be able to create an app from a template (%s)`,
async source => {
const env: Partial<typeof coreEnv> = {
TENANT_FEATURE_FLAGS: source === "sqs" ? "*:SQS" : "",
}
await features.testutils.withFeatureFlags(
"*",
{ SQS: source === "sqs" },
async () => {
const name = generator.guid().replaceAll("-", "")
const url = `/${name}`
await withCoreEnv(env, async () => {
const name = generator.guid().replaceAll("-", "")
const url = `/${name}`
const app = await config.api.application.create({
name,
url,
useTemplate: "true",
templateName: "Agency Client Portal",
templateKey: "app/agency-client-portal",
})
expect(app.name).toBe(name)
expect(app.url).toBe(url)
await config.withApp(app, async () => {
const tables = await config.api.table.fetch()
expect(tables).toHaveLength(2)
tables.sort((a, b) => a.name.localeCompare(b.name))
const [agencyProjects, users] = tables
expect(agencyProjects.name).toBe("Agency Projects")
expect(users.name).toBe("Users")
const { rows } = await config.api.row.search(agencyProjects._id!, {
tableId: agencyProjects._id!,
query: {},
const app = await config.api.application.create({
name,
url,
useTemplate: "true",
templateName: "Agency Client Portal",
templateKey: "app/agency-client-portal",
})
expect(app.name).toBe(name)
expect(app.url).toBe(url)
expect(rows).toHaveLength(3)
})
})
await config.withApp(app, async () => {
const tables = await config.api.table.fetch()
expect(tables).toHaveLength(2)
tables.sort((a, b) => a.name.localeCompare(b.name))
const [agencyProjects, users] = tables
expect(agencyProjects.name).toBe("Agency Projects")
expect(users.name).toBe("Users")
const { rows } = await config.api.row.search(
agencyProjects._id!,
{
tableId: agencyProjects._id!,
query: {},
}
)
expect(rows).toHaveLength(3)
})
}
)
}
)
})

154
packages/server/src/api/routes/tests/viewV2.spec.ts
View File

@ -22,23 +22,17 @@ import {
RelationshipType,
TableSchema,
RenameColumn,
FeatureFlag,
BBReferenceFieldSubType,
NumericCalculationFieldMetadata,
ViewV2Schema,
ViewV2Type,
JsonTypes,
} from "@budibase/types"
import { generator, mocks } from "@budibase/backend-core/tests"
import { DatabaseName, getDatasource } from "../../../integrations/tests/utils"
import merge from "lodash/merge"
import { quotas } from "@budibase/pro"
import {
db,
roles,
withEnv as withCoreEnv,
setEnv as setCoreEnv,
env,
} from "@budibase/backend-core"
import { db, roles, features } from "@budibase/backend-core"
describe.each([
["lucene", undefined],
@ -103,18 +97,13 @@ describe.each([
}
beforeAll(async () => {
await withCoreEnv({ TENANT_FEATURE_FLAGS: isSqs ? "*:SQS" : "" }, () =>
await features.testutils.withFeatureFlags("*", { SQS: isSqs }, () =>
config.init()
)
if (isLucene) {
envCleanup = setCoreEnv({
TENANT_FEATURE_FLAGS: "*:!SQS",
})
} else if (isSqs) {
envCleanup = setCoreEnv({
TENANT_FEATURE_FLAGS: "*:SQS",
})
}
envCleanup = features.testutils.setFeatureFlags("*", {
SQS: isSqs,
})
if (dsProvider) {
datasource = await config.createDatasource({
@ -748,6 +737,69 @@ describe.each([
},
})
})
!isLucene &&
it("does not get confused when a calculation field shadows a basic one", async () => {
const table = await config.api.table.save(
saveTableRequest({
schema: {
age: {
name: "age",
type: FieldType.NUMBER,
},
},
})
)
await config.api.row.bulkImport(table._id!, {
rows: [{ age: 1 }, { age: 2 }, { age: 3 }],
})
const view = await config.api.viewV2.create({
tableId: table._id!,
name: generator.guid(),
type: ViewV2Type.CALCULATION,
schema: {
age: {
visible: true,
calculationType: CalculationType.SUM,
field: "age",
},
},
})
const { rows } = await config.api.row.search(view.id)
expect(rows).toHaveLength(1)
expect(rows[0].age).toEqual(6)
})
// We don't allow the creation of tables with most JsonTypes when using
// external datasources.
isInternal &&
it("cannot use complex types as group-by fields", async () => {
for (const type of JsonTypes) {
const field = { name: "field", type } as FieldSchema
const table = await config.api.table.save(
saveTableRequest({ schema: { field } })
)
await config.api.viewV2.create(
{
tableId: table._id!,
name: generator.guid(),
type: ViewV2Type.CALCULATION,
schema: {
field: { visible: true },
},
},
{
status: 400,
body: {
message: `Grouping by fields of type "${type}" is not supported`,
},
}
)
}
})
})
describe("update", () => {
@ -1926,6 +1978,30 @@ describe.each([
expect(newRow.one).toBeUndefined()
expect(newRow.two).toEqual("bar")
})
it("should not be possible to create a row in a calculation view", async () => {
const view = await config.api.viewV2.create({
tableId: table._id!,
name: generator.guid(),
type: ViewV2Type.CALCULATION,
schema: {
id: { visible: true },
one: { visible: true },
},
})
await config.api.row.save(
view.id,
{ one: "foo" },
{
status: 400,
body: {
message: "Cannot insert rows through a calculation view",
status: 400,
},
}
)
})
})
describe("patch", () => {
@ -1990,6 +2066,40 @@ describe.each([
expect(row.one).toEqual("foo")
expect(row.two).toEqual("newBar")
})
it("should not be possible to modify a row in a calculation view", async () => {
const view = await config.api.viewV2.create({
tableId: table._id!,
name: generator.guid(),
type: ViewV2Type.CALCULATION,
schema: {
id: { visible: true },
one: { visible: true },
},
})
const newRow = await config.api.row.save(table._id!, {
one: "foo",
two: "bar",
})
await config.api.row.patch(
view.id,
{
tableId: table._id!,
_id: newRow._id!,
_rev: newRow._rev!,
one: "newFoo",
two: "newBar",
},
{
status: 400,
body: {
message: "Cannot update rows through a calculation view",
},
}
)
})
})
describe("destroy", () => {
@ -2666,12 +2776,8 @@ describe.each([
describe("foreign relationship columns", () => {
let envCleanup: () => void
beforeAll(() => {
const flags = [`*:${FeatureFlag.ENRICHED_RELATIONSHIPS}`]
if (env.TENANT_FEATURE_FLAGS) {
flags.push(...env.TENANT_FEATURE_FLAGS.split(","))
}
envCleanup = setCoreEnv({
TENANT_FEATURE_FLAGS: flags.join(","),
envCleanup = features.testutils.setFeatureFlags("*", {
ENRICHED_RELATIONSHIPS: true,
})
})

6
packages/server/src/appMigrations/migrations/tests/20240604153647_initial_sqs.spec.ts
View File

@ -2,8 +2,8 @@ import * as setup from "../../../api/routes/tests/utilities"
import { basicTable } from "../../../tests/utilities/structures"
import {
db as dbCore,
features,
SQLITE_DESIGN_DOC_ID,
withEnv as withCoreEnv,
} from "@budibase/backend-core"
import {
LinkDocument,
@ -71,11 +71,11 @@ function oldLinkDocument(): Omit<LinkDocument, "tableId"> {
}
async function sqsDisabled(cb: () => Promise<void>) {
await withCoreEnv({ TENANT_FEATURE_FLAGS: "*:!SQS" }, cb)
await features.testutils.withFeatureFlags("*", { SQS: false }, cb)
}
async function sqsEnabled(cb: () => Promise<void>) {
await withCoreEnv({ TENANT_FEATURE_FLAGS: "*:SQS" }, cb)
await features.testutils.withFeatureFlags("*", { SQS: true }, cb)
}
describe("SQS migration", () => {

8
packages/server/src/db/linkedRows/LinkController.ts
View File

@ -221,9 +221,15 @@ class LinkController {
link.id !== row._id && link.fieldName === linkedSchema.name
)
// check all the related rows exist
const foundRecords = await this._db.getMultiple(
links.map(l => l.id),
{ allowMissing: true, excludeDocs: true }
)
// The 1 side of 1:N is already related to something else
// You must remove the existing relationship
if (links.length > 0) {
if (foundRecords.length > 0) {
throw new Error(
`1:N Relationship Error: Record already linked to another.`
)

2
packages/server/src/integrations/postgres.ts
View File

@ -41,7 +41,7 @@ if (types) {
types.setTypeParser(1184, (val: any) => val) // timestampz
}
const JSON_REGEX = /'{.*}'::json/s
const JSON_REGEX = /'{\s*.*?\s*}'::json/gs
const Sql = sql.Sql
interface PostgresConfig {

2
packages/server/src/koa.ts
View File

@ -12,6 +12,7 @@ import { userAgent } from "koa-useragent"
export default function createKoaApp() {
const app = new Koa()
app.proxy = true
let mbNumber = parseInt(env.HTTP_MB_LIMIT || "10")
if (!mbNumber || isNaN(mbNumber)) {
@ -35,6 +36,7 @@ export default function createKoaApp() {
app.use(middleware.correlation)
app.use(middleware.pino)
app.use(middleware.ip)
app.use(userAgent)
const server = http.createServer(app.callback())

4
packages/server/src/migrations/tests/index.spec.ts
View File

@ -71,7 +71,7 @@ describe("migrations", () => {
expect(events.datasource.created).toHaveBeenCalledTimes(2)
expect(events.layout.created).toHaveBeenCalledTimes(1)
expect(events.query.created).toHaveBeenCalledTimes(2)
expect(events.role.created).toHaveBeenCalledTimes(2)
expect(events.role.created).toHaveBeenCalledTimes(3) // created roles + admin (created on table creation)
expect(events.table.created).toHaveBeenCalledTimes(3)
expect(events.view.created).toHaveBeenCalledTimes(2)
expect(events.view.calculationCreated).toHaveBeenCalledTimes(1)
@ -82,7 +82,7 @@ describe("migrations", () => {
// to make sure caching is working as expected
expect(
events.processors.analyticsProcessor.processEvent
).toHaveBeenCalledTimes(23)
).toHaveBeenCalledTimes(24) // Addtion of of the events above
})
})
})

20
packages/server/src/sdk/app/permissions/index.ts
View File

@ -185,6 +185,26 @@ export async function updatePermissionOnRole(
})
}
export async function setPermissions(
resourceId: string,
{
writeRole,
readRole,
}: {
writeRole: string
readRole: string
}
) {
await updatePermissionOnRole(
{ roleId: writeRole, resourceId, level: PermissionLevel.WRITE },
PermissionUpdateType.ADD
)
await updatePermissionOnRole(
{ roleId: readRole, resourceId, level: PermissionLevel.READ },
PermissionUpdateType.ADD
)
}
// utility function to stop this repetition - permissions always stored under roles
export async function getAllDBRoles(db: Database) {
const body = await db.allDocs<Role>(

5
packages/server/src/sdk/app/rows/external.ts
View File

@ -15,6 +15,7 @@ import {
} from "../../../utilities/rowProcessor"
import cloneDeep from "lodash/fp/cloneDeep"
import { tryExtractingTableAndViewId } from "./utils"
import { helpers } from "@budibase/shared-core"
export async function getRow(
sourceId: string | Table | ViewV2,
@ -54,6 +55,10 @@ export async function save(
source = await sdk.tables.getTable(tableId)
}
if (sdk.views.isView(source) && helpers.views.isCalculationView(source)) {
throw new HTTPError("Cannot insert rows through a calculation view", 400)
}
const row = await inputProcessing(userId, cloneDeep(source), inputs)
const validateResult = await sdk.rows.utils.validate({

7
packages/server/src/sdk/app/rows/internal.ts
View File

@ -1,4 +1,4 @@
import { context, db } from "@budibase/backend-core"
import { context, db, HTTPError } from "@budibase/backend-core"
import { Row, Table, ViewV2 } from "@budibase/types"
import sdk from "../../../sdk"
import { finaliseRow } from "../../../api/controllers/row/staticFormula"
@ -10,6 +10,7 @@ import * as linkRows from "../../../db/linkedRows"
import { InternalTables } from "../../../db/utils"
import { getFullUser } from "../../../utilities/users"
import { getSource, tryExtractingTableAndViewId } from "./utils"
import { helpers } from "@budibase/shared-core"
export async function save(
tableOrViewId: string,
@ -29,6 +30,10 @@ export async function save(
table = source
}
if (sdk.views.isView(source) && helpers.views.isCalculationView(source)) {
throw new HTTPError("Cannot insert rows through a calculation view", 400)
}
if (!inputs._rev && !inputs._id) {
inputs._id = db.generateRowID(inputs.tableId)
}

4
packages/server/src/sdk/app/rows/search/internal/internal.ts
View File

@ -62,10 +62,10 @@ export async function exportRows(
).rows.map(row => row.doc!)
result = await outputProcessing(table, response)
} else if (query) {
} else {
let searchResponse = await sdk.rows.search({
tableId,
query,
query: query || {},
sort,
sortOrder,
})

19
packages/server/src/sdk/app/rows/search/tests/search.spec.ts
View File

@ -10,10 +10,7 @@ import {
import TestConfiguration from "../../../../../tests/utilities/TestConfiguration"
import { search } from "../../../../../sdk/app/rows/search"
import { generator } from "@budibase/backend-core/tests"
import {
withEnv as withCoreEnv,
setEnv as setCoreEnv,
} from "@budibase/backend-core"
import { features } from "@budibase/backend-core"
import {
DatabaseName,
getDatasource,
@ -41,19 +38,13 @@ describe.each([
let table: Table
beforeAll(async () => {
await withCoreEnv({ TENANT_FEATURE_FLAGS: isSqs ? "*:SQS" : "" }, () =>
await features.testutils.withFeatureFlags("*", { SQS: isSqs }, () =>
config.init()
)
if (isLucene) {
envCleanup = setCoreEnv({
TENANT_FEATURE_FLAGS: "*:!SQS",
})
} else if (isSqs) {
envCleanup = setCoreEnv({
TENANT_FEATURE_FLAGS: "*:SQS",
})
}
envCleanup = features.testutils.setFeatureFlags("*", {
SQS: isSqs,
})
if (dsProvider) {
datasource = await config.createDatasource({

27
packages/server/src/sdk/app/tables/create.ts Normal file
View File

@ -0,0 +1,27 @@
import { Row, Table } from "@budibase/types"
import * as external from "./external"
import * as internal from "./internal"
import { isExternal } from "./utils"
import { setPermissions } from "../permissions"
import { roles } from "@budibase/backend-core"
export async function create(
table: Omit<Table, "_id" | "_rev">,
rows?: Row[],
userId?: string
): Promise<Table> {
let createdTable: Table
if (isExternal({ table })) {
createdTable = await external.create(table)
} else {
createdTable = await internal.create(table, rows, userId)
}
await setPermissions(createdTable._id!, {
writeRole: roles.BUILTIN_ROLE_IDS.ADMIN,
readRole: roles.BUILTIN_ROLE_IDS.ADMIN,
})
return createdTable
}

36
packages/server/src/sdk/app/tables/external/index.ts vendored
View File

@ -8,8 +8,11 @@ import {
ViewV2,
AutoFieldSubType,
} from "@budibase/types"
import { context } from "@budibase/backend-core"
import { buildExternalTableId } from "../../../../integrations/utils"
import { context, HTTPError } from "@budibase/backend-core"
import {
breakExternalTableId,
buildExternalTableId,
} from "../../../../integrations/utils"
import {
foreignKeyStructure,
hasTypeChanged,
@ -86,6 +89,35 @@ function validate(table: Table, oldTable?: Table) {
}
}
function getDatasourceId(table: Table) {
if (!table) {
throw new Error("No table supplied")
}
if (table.sourceId) {
return table.sourceId
}
if (!table._id) {
throw new Error("No table ID supplied")
}
return breakExternalTableId(table._id).datasourceId
}
export async function create(table: Omit<Table, "_id" | "_rev">) {
const datasourceId = getDatasourceId(table)
const tableToCreate = { ...table, created: true }
try {
const result = await save(datasourceId!, tableToCreate)
return result.table
} catch (err: any) {
if (err instanceof Error) {
throw new HTTPError(err.message, 400)
} else {
throw new HTTPError(err?.message || err, err.status || 500)
}
}
}
export async function save(
datasourceId: string,
update: Table,

2
packages/server/src/sdk/app/tables/index.ts
View File

@ -1,5 +1,6 @@
import { populateExternalTableSchemas } from "./validation"
import * as getters from "./getters"
import * as create from "./create"
import * as updates from "./update"
import * as utils from "./utils"
import { migrate } from "./migration"
@ -7,6 +8,7 @@ import * as sqs from "./internal/sqs"
export default {
populateExternalTableSchemas,
...create,
...updates,
...getters,
...utils,

48
packages/server/src/sdk/app/tables/internal/index.ts
View File

@ -5,7 +5,7 @@ import {
ViewStatisticsSchema,
ViewV2,
Row,
ContextUser,
TableSourceType,
} from "@budibase/types"
import {
hasTypeChanged,
@ -16,18 +16,56 @@ import { EventType, updateLinks } from "../../../../db/linkedRows"
import { cloneDeep } from "lodash/fp"
import isEqual from "lodash/isEqual"
import { runStaticFormulaChecks } from "../../../../api/controllers/table/bulkFormula"
import { context } from "@budibase/backend-core"
import { context, HTTPError } from "@budibase/backend-core"
import { findDuplicateInternalColumns } from "@budibase/shared-core"
import { getTable } from "../getters"
import { checkAutoColumns } from "./utils"
import * as viewsSdk from "../../views"
import { getRowParams } from "../../../../db/utils"
import { generateTableID, getRowParams } from "../../../../db/utils"
import { quotas } from "@budibase/pro"
export async function create(
table: Omit<Table, "_id" | "_rev">,
rows?: Row[],
userId?: string
) {
const tableId = generateTableID()
let tableToSave: Table = {
_id: tableId,
...table,
// Ensure these fields are populated, even if not sent in the request
type: table.type || "table",
sourceType: TableSourceType.INTERNAL,
}
const isImport = !!rows
if (!tableToSave.views) {
tableToSave.views = {}
}
try {
const { table } = await save(tableToSave, {
userId,
rowsToImport: rows,
isImport,
})
return table
} catch (err: any) {
if (err instanceof Error) {
throw new HTTPError(err.message, 400)
} else {
throw new HTTPError(err.message || err, err.status || 500)
}
}
}
export async function save(
table: Table,
opts?: {
user?: ContextUser
userId?: string
tableId?: string
rowsToImport?: Row[]
renaming?: RenameColumn
@ -63,7 +101,7 @@ export async function save(
// saving a table is a complex operation, involving many different steps, this
// has been broken out into a utility to make it more obvious/easier to manipulate
const tableSaveFunctions = new TableSaveFunctions({
user: opts?.user,
userId: opts?.userId,
oldTable,
importRows: opts?.rowsToImport,
})

36
packages/server/src/sdk/app/views/index.ts
View File

@ -1,6 +1,8 @@
import {
CalculationType,
canGroupBy,
FieldType,
isNumeric,
PermissionLevel,
RelationSchemaField,
RenameColumn,
@ -11,7 +13,7 @@ import {
ViewV2ColumnEnriched,
ViewV2Enriched,
} from "@budibase/types"
import { context, docIds, HTTPError, roles } from "@budibase/backend-core"
import { context, docIds, HTTPError } from "@budibase/backend-core"
import {
helpers,
PROTECTED_EXTERNAL_COLUMNS,
@ -22,7 +24,6 @@ import { isExternalTableID } from "../../../integrations/utils"
import * as internal from "./internal"
import * as external from "./external"
import sdk from "../../../sdk"
import { PermissionUpdateType, updatePermissionOnRole } from "../permissions"
function pickApi(tableId: any) {
if (isExternalTableID(tableId)) {
@ -101,7 +102,7 @@ async function guardCalculationViewSchema(
)
}
if (!isCount && !helpers.schema.isNumeric(targetSchema)) {
if (!isCount && !isNumeric(targetSchema.type)) {
throw new HTTPError(
`Calculation field "${name}" references field "${schema.field}" which is not a numeric field`,
400
@ -118,6 +119,13 @@ async function guardCalculationViewSchema(
400
)
}
if (!canGroupBy(targetSchema.type)) {
throw new HTTPError(
`Grouping by fields of type "${targetSchema.type}" is not supported`,
400
)
}
}
}
@ -235,24 +243,10 @@ export async function create(
// Set permissions to be the same as the table
const tablePerms = await sdk.permissions.getResourcePerms(tableId)
const readRole = tablePerms[PermissionLevel.READ]?.role
const writeRole = tablePerms[PermissionLevel.WRITE]?.role
await updatePermissionOnRole(
{
roleId: readRole || roles.BUILTIN_ROLE_IDS.BASIC,
resourceId: view.id,
level: PermissionLevel.READ,
},
PermissionUpdateType.ADD
)
await updatePermissionOnRole(
{
roleId: writeRole || roles.BUILTIN_ROLE_IDS.BASIC,
resourceId: view.id,
level: PermissionLevel.WRITE,
},
PermissionUpdateType.ADD
)
await sdk.permissions.setPermissions(view.id, {
writeRole: tablePerms[PermissionLevel.WRITE].role,
readRole: tablePerms[PermissionLevel.READ].role,
})
return view
}

10
packages/server/src/tests/utilities/api/permission.ts
View File

@ -1,6 +1,7 @@
import {
AddPermissionRequest,
AddPermissionResponse,
FetchResourcePermissionInfoResponse,
GetResourcePermsResponse,
RemovePermissionRequest,
RemovePermissionResponse,
@ -26,6 +27,15 @@ export class PermissionAPI extends TestAPI {
)
}
fetch = async (
expectations?: Expectations
): Promise<FetchResourcePermissionInfoResponse> => {
return await this._get<FetchResourcePermissionInfoResponse>(
`/api/permission`,
{ expectations }
)
}
revoke = async (
request: RemovePermissionRequest,
expectations?: Expectations

2
packages/server/src/tests/utilities/api/row.ts
View File

@ -105,7 +105,7 @@ export class RowAPI extends TestAPI {
exportRows = async (
tableId: string,
body: ExportRowsRequest,
body?: ExportRowsRequest,
format: RowExportFormat = RowExportFormat.JSON,
expectations?: Expectations
) => {

8
packages/server/src/utilities/rowProcessor/tests/outputProcessing.spec.ts
View File

@ -8,7 +8,7 @@ import {
} from "@budibase/types"
import { outputProcessing } from ".."
import { generator, structures } from "@budibase/backend-core/tests"
import { setEnv as setCoreEnv } from "@budibase/backend-core"
import { features } from "@budibase/backend-core"
import * as bbReferenceProcessor from "../bbReferenceProcessor"
import TestConfiguration from "../../../tests/utilities/TestConfiguration"
@ -21,7 +21,7 @@ jest.mock("../bbReferenceProcessor", (): typeof bbReferenceProcessor => ({
describe("rowProcessor - outputProcessing", () => {
const config = new TestConfiguration()
let cleanupEnv: () => void = () => {}
let cleanupFlags: () => void = () => {}
beforeAll(async () => {
await config.init()
@ -33,11 +33,11 @@ describe("rowProcessor - outputProcessing", () => {
beforeEach(() => {
jest.resetAllMocks()
cleanupEnv = setCoreEnv({ TENANT_FEATURE_FLAGS: "*SQS" })
cleanupFlags = features.testutils.setFeatureFlags("*", { SQS: true })
})
afterEach(() => {
cleanupEnv()
cleanupFlags()
})
const processOutputBBReferenceMock =

29
packages/server/src/utilities/security.ts
View File

@ -1,5 +1,6 @@
import { permissions, roles } from "@budibase/backend-core"
import { DocumentType, VirtualDocumentType } from "../db/utils"
import { getDocumentType, getVirtualDocumentType } from "@budibase/types"
export const CURRENTLY_SUPPORTED_LEVELS: string[] = [
permissions.PermissionLevel.WRITE,
@ -8,13 +9,16 @@ export const CURRENTLY_SUPPORTED_LEVELS: string[] = [
]
export function getPermissionType(resourceId: string) {
const docType = Object.values(DocumentType).filter(docType =>
resourceId.startsWith(docType)
)[0]
switch (docType as DocumentType | VirtualDocumentType) {
const virtualDocType = getVirtualDocumentType(resourceId)
switch (virtualDocType) {
case VirtualDocumentType.VIEW:
return permissions.PermissionType.TABLE
}
const docType = getDocumentType(resourceId)
switch (docType) {
case DocumentType.TABLE:
case DocumentType.ROW:
case VirtualDocumentType.VIEW:
return permissions.PermissionType.TABLE
case DocumentType.AUTOMATION:
return permissions.PermissionType.AUTOMATION
@ -32,22 +36,25 @@ export function getPermissionType(resourceId: string) {
/**
* works out the basic permissions based on builtin roles for a resource, using its ID
*/
export function getBasePermissions(resourceId: string) {
export function getBasePermissions(resourceId: string): Record<string, string> {
const type = getPermissionType(resourceId)
const basePermissions: { [key: string]: string } = {}
const basePermissions: Record<string, string> = {}
for (let [roleId, role] of Object.entries(roles.getBuiltinRoles())) {
if (!role.permissionId) {
continue
}
const perms = permissions.getBuiltinPermissionByID(role.permissionId)
if (!perms) {
continue
}
const typedPermission = perms.permissions.find(perm => perm.type === type)
if (
typedPermission &&
CURRENTLY_SUPPORTED_LEVELS.indexOf(typedPermission.level) !== -1
) {
if (!typedPermission) {
continue
}
if (CURRENTLY_SUPPORTED_LEVELS.includes(typedPermission.level)) {
const level = typedPermission.level
basePermissions[level] = roles.lowerBuiltinRoleID(
basePermissions[level],

12
packages/shared-core/src/filters.ts
View File

@ -642,19 +642,19 @@ export function fixupFilterArrays(filters: SearchFilters) {
return filters
}
export function search<T>(
docs: Record<string, T>[],
query: RowSearchParams
): SearchResponse<Record<string, T>> {
export function search<T extends Record<string, any>>(
docs: T[],
query: Omit<RowSearchParams, "tableId">
): SearchResponse<T> {
let result = runQuery(docs, query.query)
if (query.sort) {
result = sort(result, query.sort, query.sortOrder || SortOrder.ASCENDING)
}
let totalRows = result.length
const totalRows = result.length
if (query.limit) {
result = limit(result, query.limit.toString())
}
const response: SearchResponse<Record<string, any>> = { rows: result }
const response: SearchResponse<T> = { rows: result }
if (query.countRows) {
response.totalRows = totalRows
}

13
packages/shared-core/src/utils.ts
View File

@ -5,6 +5,7 @@ import {
SearchFilters,
BasicOperator,
ArrayOperator,
isLogicalSearchOperator,
} from "@budibase/types"
import * as Constants from "./constants"
import { removeKeyNumbering } from "./filters"
@ -97,10 +98,20 @@ export function isSupportedUserSearch(query: SearchFilters) {
{ op: BasicOperator.EQUAL, key: "_id" },
{ op: ArrayOperator.ONE_OF, key: "_id" },
]
for (let [key, operation] of Object.entries(query)) {
for (const [key, operation] of Object.entries(query)) {
if (typeof operation !== "object") {
return false
}
if (isLogicalSearchOperator(key)) {
for (const condition of query[key]!.conditions) {
if (!isSupportedUserSearch(condition)) {
return false
}
}
return true
}
const fields = Object.keys(operation || {})
// this filter doesn't contain options - ignore
if (fields.length === 0) {

8
packages/types/src/api/web/app/permission.ts
View File

@ -1,5 +1,9 @@
import { PermissionLevel } from "../../../sdk"
export interface FetchResourcePermissionInfoResponse {
[key: string]: Record<string, string>
}
export interface ResourcePermissionInfo {
role: string
permissionType: string
@ -21,7 +25,7 @@ export interface AddedPermission {
reason?: string
}
export type AddPermissionResponse = AddedPermission[]
export interface AddPermissionResponse {}
export interface AddPermissionRequest {
roleId: string
@ -30,4 +34,4 @@ export interface AddPermissionRequest {
}
export interface RemovePermissionRequest extends AddPermissionRequest {}
export interface RemovePermissionResponse extends AddPermissionResponse {}
export interface RemovePermissionResponse {}

20
packages/types/src/documents/app/row.ts
View File

@ -127,6 +127,26 @@ export const JsonTypes = [
FieldType.ARRAY,
]
export const NumericTypes = [FieldType.NUMBER, FieldType.BIGINT]
export function isNumeric(type: FieldType) {
return NumericTypes.includes(type)
}
export const GroupByTypes = [
FieldType.STRING,
FieldType.LONGFORM,
FieldType.OPTIONS,
FieldType.NUMBER,
FieldType.BOOLEAN,
FieldType.DATETIME,
FieldType.BIGINT,
]
export function canGroupBy(type: FieldType) {
return GroupByTypes.includes(type)
}
export interface RowAttachment {
size: number
name: string

24
packages/types/src/documents/document.ts
View File

@ -42,6 +42,17 @@ export enum DocumentType {
ROW_ACTIONS = "ra",
}
// Because DocumentTypes can overlap, we need to make sure that we search
// longest first to ensure we get the correct type.
const sortedDocumentTypes = Object.values(DocumentType).sort(
(a, b) => b.length - a.length // descending
)
export function getDocumentType(id: string): DocumentType | undefined {
return sortedDocumentTypes.find(docType =>
id.startsWith(`${docType}${SEPARATOR}`)
)
}
// these are the core documents that make up the data, design
// and automation sections of an app. This excludes any internal
// rows as we shouldn't import data.
@ -72,6 +83,19 @@ export enum VirtualDocumentType {
ROW_ACTION = "row_action",
}
// Because VirtualDocumentTypes can overlap, we need to make sure that we search
// longest first to ensure we get the correct type.
const sortedVirtualDocumentTypes = Object.values(VirtualDocumentType).sort(
(a, b) => b.length - a.length // descending
)
export function getVirtualDocumentType(
id: string
): VirtualDocumentType | undefined {
return sortedVirtualDocumentTypes.find(docType =>
id.startsWith(`${docType}${SEPARATOR}`)
)
}
export interface Document {
_id?: string
_rev?: string

2
packages/types/src/sdk/db.ts
View File

@ -133,7 +133,7 @@ export interface Database {
exists(docId: string): Promise<boolean>
getMultiple<T extends Document>(
ids: string[],
opts?: { allowMissing?: boolean }
opts?: { allowMissing?: boolean; excludeDocs?: boolean }
): Promise<T[]>
remove(idOrDoc: Document): Promise<Nano.DocumentDestroyResponse>
remove(idOrDoc: string, rev?: string): Promise<Nano.DocumentDestroyResponse>

10
packages/worker/src/api/routes/global/tests/auditLogs.spec.ts
View File

@ -1,5 +1,5 @@
import { mocks, structures } from "@budibase/backend-core/tests"
import { context, events, setEnv as setCoreEnv } from "@budibase/backend-core"
import { context, events, features } from "@budibase/backend-core"
import { Event, IdentityType } from "@budibase/types"
import { TestConfiguration } from "../../../../tests"
@ -17,11 +17,9 @@ describe.each(["lucene", "sql"])("/api/global/auditlogs (%s)", method => {
let envCleanup: (() => void) | undefined
beforeAll(async () => {
if (method === "lucene") {
envCleanup = setCoreEnv({ TENANT_FEATURE_FLAGS: "*:!SQS" })
} else if (method === "sql") {
envCleanup = setCoreEnv({ TENANT_FEATURE_FLAGS: "*:SQS" })
}
envCleanup = features.testutils.setFeatureFlags("*", {
SQS: method === "sql",
})
await config.beforeAll()
})

19
packages/worker/src/api/routes/global/tests/users.spec.ts
View File

@ -741,6 +741,25 @@ describe("/api/global/users", () => {
it("should throw an error if public query performed", async () => {
await config.api.users.searchUsers({}, { status: 403, noHeaders: true })
})
it("should be able to search using logical conditions", async () => {
const user = await config.createUser()
const response = await config.api.users.searchUsers({
query: {
$and: {
conditions: [
{
$and: {
conditions: [{ string: { email: user.email } }],
},
},
],
},
},
})
expect(response.body.data.length).toBe(1)
expect(response.body.data[0].email).toBe(user.email)
})
})
describe("DELETE /api/global/users/:userId", () => {

2
packages/worker/src/index.ts
View File

@ -46,6 +46,7 @@ bootstrap()
const app: Application = new Application()
app.keys = ["secret", "key"]
app.proxy = true
// set up top level koa middleware
app.use(handleScimBody)
@ -54,6 +55,7 @@ app.use(koaBody({ multipart: true }))
app.use(koaSession(app))
app.use(middleware.correlation)
app.use(middleware.pino)
app.use(middleware.ip)
app.use(userAgent)
// authentication