diff --git a/lerna.json b/lerna.json index 55721b6b2d..4d36affd04 100644 --- a/lerna.json +++ b/lerna.json @@ -1,6 +1,6 @@ { "$schema": "node_modules/lerna/schemas/lerna-schema.json", - "version": "2.32.12", + "version": "2.32.15", "npmClient": "yarn", "packages": [ "packages/*", diff --git a/packages/backend-core/src/context/mainContext.ts b/packages/backend-core/src/context/mainContext.ts index 25b273e51c..64ba240fa5 100644 --- a/packages/backend-core/src/context/mainContext.ts +++ b/packages/backend-core/src/context/mainContext.ts @@ -253,6 +253,11 @@ export function getAppId(): string | undefined { } } +export function getIP(): string | undefined { + const context = Context.get() + return context?.ip +} + export const getProdAppId = () => { const appId = getAppId() if (!appId) { @@ -281,6 +286,10 @@ export function doInScimContext(task: any) { return newContext(updates, task) } +export function doInIPContext(ip: string, task: any) { + return newContext({ ip }, task) +} + export async function ensureSnippetContext(enabled = !env.isTest()) { const ctx = getCurrentContext() diff --git a/packages/backend-core/src/context/types.ts b/packages/backend-core/src/context/types.ts index ee84b49459..5549a47ff7 100644 --- a/packages/backend-core/src/context/types.ts +++ b/packages/backend-core/src/context/types.ts @@ -9,6 +9,7 @@ export type ContextMap = { identity?: IdentityContext environmentVariables?: Record isScim?: boolean + ip?: string automationId?: string isMigrating?: boolean vm?: VM diff --git a/packages/backend-core/src/db/couch/DatabaseImpl.ts b/packages/backend-core/src/db/couch/DatabaseImpl.ts index 1213dde8f5..2b37526dde 100644 --- a/packages/backend-core/src/db/couch/DatabaseImpl.ts +++ b/packages/backend-core/src/db/couch/DatabaseImpl.ts @@ -213,17 +213,21 @@ export class DatabaseImpl implements Database { async getMultiple( ids: string[], - opts?: { allowMissing?: boolean } + opts?: { allowMissing?: boolean; excludeDocs?: boolean } ): Promise { // get unique ids = [...new Set(ids)] + const includeDocs = !opts?.excludeDocs const response = await this.allDocs({ keys: ids, - include_docs: true, + include_docs: includeDocs, }) const rowUnavailable = (row: RowResponse) => { // row is deleted - key lookup can return this - if (row.doc == null || ("deleted" in row.value && row.value.deleted)) { + if ( + (includeDocs && row.doc == null) || + (row.value && "deleted" in row.value && row.value.deleted) + ) { return true } return row.error === "not_found" @@ -237,7 +241,7 @@ export class DatabaseImpl implements Database { const missingIds = missing.map(row => row.key).join(", ") throw new Error(`Unable to get documents: ${missingIds}`) } - return rows.map(row => row.doc!) + return rows.map(row => (includeDocs ? row.doc! : row.value)) } async remove(idOrDoc: string | Document, rev?: string) { diff --git a/packages/backend-core/src/features/features.ts b/packages/backend-core/src/features/features.ts new file mode 100644 index 0000000000..efe6495cb5 --- /dev/null +++ b/packages/backend-core/src/features/features.ts @@ -0,0 +1,278 @@ +import env from "../environment" +import * as crypto from "crypto" +import * as context from "../context" +import { PostHog, PostHogOptions } from "posthog-node" +import { FeatureFlag } from "@budibase/types" +import tracer from "dd-trace" +import { Duration } from "../utils" + +let posthog: PostHog | undefined +export function init(opts?: PostHogOptions) { + if ( + env.POSTHOG_TOKEN && + env.POSTHOG_API_HOST && + !env.SELF_HOSTED && + env.POSTHOG_FEATURE_FLAGS_ENABLED + ) { + console.log("initializing posthog client...") + posthog = new PostHog(env.POSTHOG_TOKEN, { + host: env.POSTHOG_API_HOST, + personalApiKey: env.POSTHOG_PERSONAL_TOKEN, + featureFlagsPollingInterval: Duration.fromMinutes(3).toMs(), + ...opts, + }) + } else { + console.log("posthog disabled") + } +} + +export function shutdown() { + posthog?.shutdown() +} + +export abstract class Flag { + static boolean(defaultValue: boolean): Flag { + return new BooleanFlag(defaultValue) + } + + static string(defaultValue: string): Flag { + return new StringFlag(defaultValue) + } + + static number(defaultValue: number): Flag { + return new NumberFlag(defaultValue) + } + + protected constructor(public defaultValue: T) {} + + abstract parse(value: any): T +} + +type UnwrapFlag = F extends Flag ? U : never + +export type FlagValues = { + [K in keyof T]: UnwrapFlag +} + +type KeysOfType = { + [K in keyof T]: T[K] extends Flag ? K : never +}[keyof T] + +class BooleanFlag extends Flag { + parse(value: any) { + if (typeof value === "string") { + return ["true", "t", "1"].includes(value.toLowerCase()) + } + + if (typeof value === "boolean") { + return value + } + + throw new Error(`could not parse value "${value}" as boolean`) + } +} + +class StringFlag extends Flag { + parse(value: any) { + if (typeof value === "string") { + return value + } + throw new Error(`could not parse value "${value}" as string`) + } +} + +class NumberFlag extends Flag { + parse(value: any) { + if (typeof value === "number") { + return value + } + + if (typeof value === "string") { + const parsed = parseFloat(value) + if (!isNaN(parsed)) { + return parsed + } + } + + throw new Error(`could not parse value "${value}" as number`) + } +} + +export interface EnvFlagEntry { + tenantId: string + key: string + value: boolean +} + +export function parseEnvFlags(flags: string): EnvFlagEntry[] { + const split = flags.split(",").map(x => x.split(":")) + const result: EnvFlagEntry[] = [] + for (const [tenantId, ...features] of split) { + for (let feature of features) { + let value = true + if (feature.startsWith("!")) { + feature = feature.slice(1) + value = false + } + result.push({ tenantId, key: feature, value }) + } + } + return result +} + +export class FlagSet, T extends { [key: string]: V }> { + // This is used to safely cache flags sets in the current request context. + // Because multiple sets could theoretically exist, we don't want the cache of + // one to leak into another. + private readonly setId: string + + constructor(private readonly flagSchema: T) { + this.setId = crypto.randomUUID() + } + + defaults(): FlagValues { + return Object.keys(this.flagSchema).reduce((acc, key) => { + const typedKey = key as keyof T + acc[typedKey] = this.flagSchema[key].defaultValue + return acc + }, {} as FlagValues) + } + + isFlagName(name: string | number | symbol): name is keyof T { + return this.flagSchema[name as keyof T] !== undefined + } + + async get(key: K): Promise[K]> { + const flags = await this.fetch() + return flags[key] + } + + async isEnabled>(key: K): Promise { + const flags = await this.fetch() + return flags[key] + } + + async fetch(): Promise> { + return await tracer.trace("features.fetch", async span => { + const cachedFlags = context.getFeatureFlags>(this.setId) + if (cachedFlags) { + span?.addTags({ fromCache: true }) + return cachedFlags + } + + const tags: Record = {} + const flagValues = this.defaults() + const currentTenantId = context.getTenantId() + const specificallySetFalse = new Set() + + for (const { tenantId, key, value } of parseEnvFlags( + env.TENANT_FEATURE_FLAGS || "" + )) { + if (!tenantId || (tenantId !== "*" && tenantId !== currentTenantId)) { + continue + } + + tags[`readFromEnvironmentVars`] = true + + if (value === false) { + specificallySetFalse.add(key) + } + + // ignore unknown flags + if (!this.isFlagName(key)) { + continue + } + + if (typeof flagValues[key] !== "boolean") { + throw new Error(`Feature: ${key} is not a boolean`) + } + + // @ts-expect-error - TS does not like you writing into a generic type, + // but we know that it's okay in this case because it's just an object. + flagValues[key as keyof FlagValues] = value + tags[`flags.${key}.source`] = "environment" + } + + const identity = context.getIdentity() + + let userId = identity?._id + if (!userId) { + const ip = context.getIP() + if (ip) { + userId = crypto.createHash("sha512").update(ip).digest("hex") + } + } + + let tenantId = identity?.tenantId + if (!tenantId) { + tenantId = currentTenantId + } + + tags[`identity.type`] = identity?.type + tags[`identity._id`] = identity?._id + tags[`tenantId`] = tenantId + tags[`userId`] = userId + + if (posthog && userId) { + tags[`readFromPostHog`] = true + + const personProperties: Record = { tenantId } + const posthogFlags = await posthog.getAllFlagsAndPayloads(userId, { + personProperties, + }) + + for (const [name, value] of Object.entries(posthogFlags.featureFlags)) { + if (!this.isFlagName(name)) { + // We don't want an unexpected PostHog flag to break the app, so we + // just log it and continue. + console.warn(`Unexpected posthog flag "${name}": ${value}`) + continue + } + + if (flagValues[name] === true || specificallySetFalse.has(name)) { + // If the flag is already set to through environment variables, we + // don't want to override it back to false here. + continue + } + + const payload = posthogFlags.featureFlagPayloads?.[name] + const flag = this.flagSchema[name] + try { + // @ts-expect-error - TS does not like you writing into a generic + // type, but we know that it's okay in this case because it's just + // an object. + flagValues[name] = flag.parse(payload || value) + tags[`flags.${name}.source`] = "posthog" + } catch (err) { + // We don't want an invalid PostHog flag to break the app, so we just + // log it and continue. + console.warn(`Error parsing posthog flag "${name}": ${value}`, err) + } + } + } + + context.setFeatureFlags(this.setId, flagValues) + for (const [key, value] of Object.entries(flagValues)) { + tags[`flags.${key}.value`] = value + } + span?.addTags(tags) + + return flagValues + }) + } +} + +// This is the primary source of truth for feature flags. If you want to add a +// new flag, add it here and use the `fetch` and `get` functions to access it. +// All of the machinery in this file is to make sure that flags have their +// default values set correctly and their types flow through the system. +export const flags = new FlagSet({ + DEFAULT_VALUES: Flag.boolean(env.isDev()), + AUTOMATION_BRANCHING: Flag.boolean(env.isDev()), + SQS: Flag.boolean(env.isDev()), + [FeatureFlag.AI_CUSTOM_CONFIGS]: Flag.boolean(env.isDev()), + [FeatureFlag.ENRICHED_RELATIONSHIPS]: Flag.boolean(env.isDev()), +}) + +type UnwrapPromise = T extends Promise ? U : T +export type FeatureFlags = UnwrapPromise> diff --git a/packages/backend-core/src/features/index.ts b/packages/backend-core/src/features/index.ts index 2b915e5689..f77a62fd4d 100644 --- a/packages/backend-core/src/features/index.ts +++ b/packages/backend-core/src/features/index.ts @@ -1,281 +1,2 @@ -import env from "../environment" -import * as context from "../context" -import { PostHog, PostHogOptions } from "posthog-node" -import { FeatureFlag, IdentityType, UserCtx } from "@budibase/types" -import tracer from "dd-trace" -import { Duration } from "../utils" - -let posthog: PostHog | undefined -export function init(opts?: PostHogOptions) { - if ( - env.POSTHOG_TOKEN && - env.POSTHOG_API_HOST && - !env.SELF_HOSTED && - env.POSTHOG_FEATURE_FLAGS_ENABLED - ) { - console.log("initializing posthog client...") - posthog = new PostHog(env.POSTHOG_TOKEN, { - host: env.POSTHOG_API_HOST, - personalApiKey: env.POSTHOG_PERSONAL_TOKEN, - featureFlagsPollingInterval: Duration.fromMinutes(3).toMs(), - ...opts, - }) - } else { - console.log("posthog disabled") - } -} - -export function shutdown() { - posthog?.shutdown() -} - -export abstract class Flag { - static boolean(defaultValue: boolean): Flag { - return new BooleanFlag(defaultValue) - } - - static string(defaultValue: string): Flag { - return new StringFlag(defaultValue) - } - - static number(defaultValue: number): Flag { - return new NumberFlag(defaultValue) - } - - protected constructor(public defaultValue: T) {} - - abstract parse(value: any): T -} - -type UnwrapFlag = F extends Flag ? U : never - -export type FlagValues = { - [K in keyof T]: UnwrapFlag -} - -type KeysOfType = { - [K in keyof T]: T[K] extends Flag ? K : never -}[keyof T] - -class BooleanFlag extends Flag { - parse(value: any) { - if (typeof value === "string") { - return ["true", "t", "1"].includes(value.toLowerCase()) - } - - if (typeof value === "boolean") { - return value - } - - throw new Error(`could not parse value "${value}" as boolean`) - } -} - -class StringFlag extends Flag { - parse(value: any) { - if (typeof value === "string") { - return value - } - throw new Error(`could not parse value "${value}" as string`) - } -} - -class NumberFlag extends Flag { - parse(value: any) { - if (typeof value === "number") { - return value - } - - if (typeof value === "string") { - const parsed = parseFloat(value) - if (!isNaN(parsed)) { - return parsed - } - } - - throw new Error(`could not parse value "${value}" as number`) - } -} - -export class FlagSet, T extends { [key: string]: V }> { - // This is used to safely cache flags sets in the current request context. - // Because multiple sets could theoretically exist, we don't want the cache of - // one to leak into another. - private readonly setId: string - - constructor(private readonly flagSchema: T) { - this.setId = crypto.randomUUID() - } - - defaults(): FlagValues { - return Object.keys(this.flagSchema).reduce((acc, key) => { - const typedKey = key as keyof T - acc[typedKey] = this.flagSchema[key].defaultValue - return acc - }, {} as FlagValues) - } - - isFlagName(name: string | number | symbol): name is keyof T { - return this.flagSchema[name as keyof T] !== undefined - } - - async get( - key: K, - ctx?: UserCtx - ): Promise[K]> { - const flags = await this.fetch(ctx) - return flags[key] - } - - async isEnabled>( - key: K, - ctx?: UserCtx - ): Promise { - const flags = await this.fetch(ctx) - return flags[key] - } - - async fetch(ctx?: UserCtx): Promise> { - return await tracer.trace("features.fetch", async span => { - const cachedFlags = context.getFeatureFlags>(this.setId) - if (cachedFlags) { - span?.addTags({ fromCache: true }) - return cachedFlags - } - - const tags: Record = {} - const flagValues = this.defaults() - const currentTenantId = context.getTenantId() - const specificallySetFalse = new Set() - - const split = (env.TENANT_FEATURE_FLAGS || "") - .split(",") - .map(x => x.split(":")) - for (const [tenantId, ...features] of split) { - if (!tenantId || (tenantId !== "*" && tenantId !== currentTenantId)) { - continue - } - - tags[`readFromEnvironmentVars`] = true - - for (let feature of features) { - let value = true - if (feature.startsWith("!")) { - feature = feature.slice(1) - value = false - specificallySetFalse.add(feature) - } - - // ignore unknown flags - if (!this.isFlagName(feature)) { - continue - } - - if (typeof flagValues[feature] !== "boolean") { - throw new Error(`Feature: ${feature} is not a boolean`) - } - - // @ts-expect-error - TS does not like you writing into a generic type, - // but we know that it's okay in this case because it's just an object. - flagValues[feature as keyof FlagValues] = value - tags[`flags.${feature}.source`] = "environment" - } - } - - const license = ctx?.user?.license - if (license) { - tags[`readFromLicense`] = true - - for (const feature of license.features) { - if (!this.isFlagName(feature)) { - continue - } - - if ( - flagValues[feature] === true || - specificallySetFalse.has(feature) - ) { - // If the flag is already set to through environment variables, we - // don't want to override it back to false here. - continue - } - - // @ts-expect-error - TS does not like you writing into a generic type, - // but we know that it's okay in this case because it's just an object. - flagValues[feature] = true - tags[`flags.${feature}.source`] = "license" - } - } - - const identity = context.getIdentity() - tags[`identity.type`] = identity?.type - tags[`identity.tenantId`] = identity?.tenantId - tags[`identity._id`] = identity?._id - - if (posthog && identity?.type === IdentityType.USER) { - tags[`readFromPostHog`] = true - - const personProperties: Record = {} - if (identity.tenantId) { - personProperties.tenantId = identity.tenantId - } - - const posthogFlags = await posthog.getAllFlagsAndPayloads( - identity._id, - { - personProperties, - } - ) - - for (const [name, value] of Object.entries(posthogFlags.featureFlags)) { - if (!this.isFlagName(name)) { - // We don't want an unexpected PostHog flag to break the app, so we - // just log it and continue. - console.warn(`Unexpected posthog flag "${name}": ${value}`) - continue - } - - if (flagValues[name] === true || specificallySetFalse.has(name)) { - // If the flag is already set to through environment variables, we - // don't want to override it back to false here. - continue - } - - const payload = posthogFlags.featureFlagPayloads?.[name] - const flag = this.flagSchema[name] - try { - // @ts-expect-error - TS does not like you writing into a generic - // type, but we know that it's okay in this case because it's just - // an object. - flagValues[name] = flag.parse(payload || value) - tags[`flags.${name}.source`] = "posthog" - } catch (err) { - // We don't want an invalid PostHog flag to break the app, so we just - // log it and continue. - console.warn(`Error parsing posthog flag "${name}": ${value}`, err) - } - } - } - - context.setFeatureFlags(this.setId, flagValues) - for (const [key, value] of Object.entries(flagValues)) { - tags[`flags.${key}.value`] = value - } - span?.addTags(tags) - - return flagValues - }) - } -} - -// This is the primary source of truth for feature flags. If you want to add a -// new flag, add it here and use the `fetch` and `get` functions to access it. -// All of the machinery in this file is to make sure that flags have their -// default values set correctly and their types flow through the system. -export const flags = new FlagSet({ - DEFAULT_VALUES: Flag.boolean(env.isDev()), - AUTOMATION_BRANCHING: Flag.boolean(env.isDev()), - SQS: Flag.boolean(env.isDev()), - [FeatureFlag.AI_CUSTOM_CONFIGS]: Flag.boolean(env.isDev()), - [FeatureFlag.ENRICHED_RELATIONSHIPS]: Flag.boolean(env.isDev()), -}) +export * from "./features" +export * as testutils from "./tests/utils" diff --git a/packages/backend-core/src/features/tests/features.spec.ts b/packages/backend-core/src/features/tests/features.spec.ts index 01c9bfa3c6..9af8a8f4bb 100644 --- a/packages/backend-core/src/features/tests/features.spec.ts +++ b/packages/backend-core/src/features/tests/features.spec.ts @@ -1,9 +1,10 @@ -import { IdentityContext, IdentityType, UserCtx } from "@budibase/types" +import { IdentityContext, IdentityType } from "@budibase/types" import { Flag, FlagSet, FlagValues, init, shutdown } from "../" import * as context from "../../context" import environment, { withEnv } from "../../environment" import nodeFetch from "node-fetch" import nock from "nock" +import * as crypto from "crypto" const schema = { TEST_BOOLEAN: Flag.boolean(false), @@ -17,7 +18,6 @@ interface TestCase { identity?: Partial environmentFlags?: string posthogFlags?: PostHogFlags - licenseFlags?: Array expected?: Partial> errorMessage?: string | RegExp } @@ -27,10 +27,14 @@ interface PostHogFlags { featureFlagPayloads?: Record } -function mockPosthogFlags(flags: PostHogFlags) { +function mockPosthogFlags( + flags: PostHogFlags, + opts?: { token?: string; distinct_id?: string } +) { + const { token = "test", distinct_id = "us_1234" } = opts || {} nock("https://us.i.posthog.com") .post("/decide/?v=3", body => { - return body.token === "test" && body.distinct_id === "us_1234" + return body.token === token && body.distinct_id === distinct_id }) .reply(200, flags) .persist() @@ -112,17 +116,6 @@ describe("feature flags", () => { }, expected: { TEST_BOOLEAN: true }, }, - { - it: "should be able to set boolean flags through the license", - licenseFlags: ["TEST_BOOLEAN"], - expected: { TEST_BOOLEAN: true }, - }, - { - it: "should not be able to override a negative environment flag from license", - environmentFlags: "default:!TEST_BOOLEAN", - licenseFlags: ["TEST_BOOLEAN"], - expected: { TEST_BOOLEAN: false }, - }, { it: "should not error on unrecognised PostHog flag", posthogFlags: { @@ -130,18 +123,12 @@ describe("feature flags", () => { }, expected: flags.defaults(), }, - { - it: "should not error on unrecognised license flag", - licenseFlags: ["UNDEFINED"], - expected: flags.defaults(), - }, ])( "$it", async ({ identity, environmentFlags, posthogFlags, - licenseFlags, expected, errorMessage, }) => { @@ -157,8 +144,6 @@ describe("feature flags", () => { env.POSTHOG_API_HOST = "https://us.i.posthog.com" } - const ctx = { user: { license: { features: licenseFlags || [] } } } - await withEnv(env, async () => { // We need to pass in node-fetch here otherwise nock won't get used // because posthog-node uses axios under the hood. @@ -180,18 +165,13 @@ describe("feature flags", () => { await context.doInIdentityContext(fullIdentity, async () => { if (errorMessage) { - await expect(flags.fetch(ctx as UserCtx)).rejects.toThrow( - errorMessage - ) + await expect(flags.fetch()).rejects.toThrow(errorMessage) } else if (expected) { - const values = await flags.fetch(ctx as UserCtx) + const values = await flags.fetch() expect(values).toMatchObject(expected) for (const [key, expectedValue] of Object.entries(expected)) { - const value = await flags.get( - key as keyof typeof schema, - ctx as UserCtx - ) + const value = await flags.get(key as keyof typeof schema) expect(value).toBe(expectedValue) } } else { @@ -214,6 +194,14 @@ describe("feature flags", () => { lastName: "User", } + // We need to pass in node-fetch here otherwise nock won't get used + // because posthog-node uses axios under the hood. + init({ + fetch: (url, opts) => { + return nodeFetch(url, opts) + }, + }) + nock("https://us.i.posthog.com") .post("/decide/?v=3", body => { return body.token === "test" && body.distinct_id === "us_1234" @@ -230,4 +218,44 @@ describe("feature flags", () => { } ) }) + + it("should still get flags when user is logged out", async () => { + const env: Partial = { + SELF_HOSTED: false, + POSTHOG_FEATURE_FLAGS_ENABLED: "true", + POSTHOG_API_HOST: "https://us.i.posthog.com", + POSTHOG_TOKEN: "test", + } + + const ip = "127.0.0.1" + const hashedIp = crypto.createHash("sha512").update(ip).digest("hex") + + await withEnv(env, async () => { + mockPosthogFlags( + { + featureFlags: { TEST_BOOLEAN: true }, + }, + { + distinct_id: hashedIp, + } + ) + + // We need to pass in node-fetch here otherwise nock won't get used + // because posthog-node uses axios under the hood. + init({ + fetch: (url, opts) => { + return nodeFetch(url, opts) + }, + }) + + await context.doInIPContext(ip, async () => { + await context.doInTenant("default", async () => { + const result = await flags.fetch() + expect(result.TEST_BOOLEAN).toBe(true) + }) + }) + + shutdown() + }) + }) }) diff --git a/packages/backend-core/src/features/tests/utils.ts b/packages/backend-core/src/features/tests/utils.ts new file mode 100644 index 0000000000..cc633c083d --- /dev/null +++ b/packages/backend-core/src/features/tests/utils.ts @@ -0,0 +1,64 @@ +import { FeatureFlags, parseEnvFlags } from ".." +import { setEnv } from "../../environment" + +function getCurrentFlags(): Record> { + const result: Record> = {} + for (const { tenantId, key, value } of parseEnvFlags( + process.env.TENANT_FEATURE_FLAGS || "" + )) { + const tenantFlags = result[tenantId] || {} + // Don't allow overwriting specifically false flags, to match the beheaviour + // of FlagSet. + if (tenantFlags[key] === false) { + continue + } + tenantFlags[key] = value + result[tenantId] = tenantFlags + } + return result +} + +function buildFlagString( + flags: Record> +): string { + const parts: string[] = [] + for (const [tenantId, tenantFlags] of Object.entries(flags)) { + for (const [key, value] of Object.entries(tenantFlags)) { + if (value === false) { + parts.push(`${tenantId}:!${key}`) + } else { + parts.push(`${tenantId}:${key}`) + } + } + } + return parts.join(",") +} + +export function setFeatureFlags( + tenantId: string, + flags: Partial +): () => void { + const current = getCurrentFlags() + for (const [key, value] of Object.entries(flags)) { + const tenantFlags = current[tenantId] || {} + tenantFlags[key] = value + current[tenantId] = tenantFlags + } + const flagString = buildFlagString(current) + return setEnv({ TENANT_FEATURE_FLAGS: flagString }) +} + +export function withFeatureFlags( + tenantId: string, + flags: Partial, + f: () => T +) { + const cleanup = setFeatureFlags(tenantId, flags) + const result = f() + if (result instanceof Promise) { + return result.finally(cleanup) + } else { + cleanup() + return result + } +} diff --git a/packages/backend-core/src/middleware/index.ts b/packages/backend-core/src/middleware/index.ts index e1eb7f1d26..20c2125b13 100644 --- a/packages/backend-core/src/middleware/index.ts +++ b/packages/backend-core/src/middleware/index.ts @@ -20,3 +20,4 @@ export { default as correlation } from "../logging/correlation/middleware" export { default as errorHandling } from "./errorHandling" export { default as querystringToBody } from "./querystringToBody" export * as joiValidator from "./joi-validator" +export { default as ip } from "./ip" diff --git a/packages/backend-core/src/middleware/ip.ts b/packages/backend-core/src/middleware/ip.ts new file mode 100644 index 0000000000..940f644ad6 --- /dev/null +++ b/packages/backend-core/src/middleware/ip.ts @@ -0,0 +1,12 @@ +import { Ctx } from "@budibase/types" +import { doInIPContext } from "../context" + +export default async (ctx: Ctx, next: any) => { + if (ctx.ip) { + return await doInIPContext(ctx.ip, () => { + return next() + }) + } else { + return next() + } +} diff --git a/packages/backend-core/src/security/permissions.ts b/packages/backend-core/src/security/permissions.ts index 98704f16c6..4ed2cd3954 100644 --- a/packages/backend-core/src/security/permissions.ts +++ b/packages/backend-core/src/security/permissions.ts @@ -65,7 +65,13 @@ export enum BuiltinPermissionID { POWER = "power", } -export const BUILTIN_PERMISSIONS = { +export const BUILTIN_PERMISSIONS: { + [key in keyof typeof BuiltinPermissionID]: { + _id: (typeof BuiltinPermissionID)[key] + name: string + permissions: Permission[] + } +} = { PUBLIC: { _id: BuiltinPermissionID.PUBLIC, name: "Public", diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index ed8dc929d6..f122ad1c41 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -325,7 +325,7 @@ class InternalBuilder { return input } - private parseBody(body: any) { + private parseBody(body: Record) { for (let [key, value] of Object.entries(body)) { const { column } = this.splitter.run(key) const schema = this.table.schema[column] @@ -1259,6 +1259,10 @@ class InternalBuilder { create(opts: QueryOptions): Knex.QueryBuilder { const { body } = this.query + if (!body) { + throw new Error("Cannot create without row body") + } + let query = this.qualifiedKnex({ alias: false }) const parsedBody = this.parseBody(body) @@ -1417,6 +1421,9 @@ class InternalBuilder { update(opts: QueryOptions): Knex.QueryBuilder { const { body, filters } = this.query + if (!body) { + throw new Error("Cannot update without row body") + } let query = this.qualifiedKnex() const parsedBody = this.parseBody(body) query = this.addFilters(query, filters) diff --git a/packages/backend-core/src/users/users.ts b/packages/backend-core/src/users/users.ts index d8546afa8b..f4838597b6 100644 --- a/packages/backend-core/src/users/users.ts +++ b/packages/backend-core/src/users/users.ts @@ -24,6 +24,7 @@ import * as context from "../context" import { getGlobalDB } from "../context" import { isCreator } from "./utils" import { UserDB } from "./db" +import { dataFilters } from "@budibase/shared-core" type GetOpts = { cleanup?: boolean } @@ -262,10 +263,17 @@ export async function paginatedUsers({ userList = await bulkGetGlobalUsersById(query?.oneOf?._id, { cleanup: true, }) + } else if (query) { + // TODO: this should use SQS search, but the logic is built in the 'server' package. Using the in-memory filtering to get this working meanwhile + const response = await db.allDocs( + getGlobalUserParams(null, { ...opts, limit: undefined }) + ) + userList = response.rows.map(row => row.doc!) + userList = dataFilters.search(userList, { query, limit: opts.limit }).rows } else { // no search, query allDocs - const response = await db.allDocs(getGlobalUserParams(null, opts)) - userList = response.rows.map((row: any) => row.doc) + const response = await db.allDocs(getGlobalUserParams(null, opts)) + userList = response.rows.map(row => row.doc!) } return pagination(userList, pageSize, { paginate: true, diff --git a/packages/frontend-core/src/components/grid/stores/datasources/viewV2.js b/packages/frontend-core/src/components/grid/stores/datasources/viewV2.js index 8e4321be96..84d5eb153e 100644 --- a/packages/frontend-core/src/components/grid/stores/datasources/viewV2.js +++ b/packages/frontend-core/src/components/grid/stores/datasources/viewV2.js @@ -191,7 +191,7 @@ export const initialise = context => { if ($view?.id !== $datasource.id) { return } - if (JSON.stringify($filter) !== JSON.stringify($view.query)) { + if (JSON.stringify($filter) !== JSON.stringify($view.queryUI)) { await datasource.actions.saveDefinition({ ...$view, query: $filter, diff --git a/packages/server/src/api/controllers/permission.ts b/packages/server/src/api/controllers/permission.ts index c7afb6a351..ead48d3db8 100644 --- a/packages/server/src/api/controllers/permission.ts +++ b/packages/server/src/api/controllers/permission.ts @@ -1,7 +1,6 @@ import { permissions, roles, context } from "@budibase/backend-core" import { UserCtx, - Role, GetResourcePermsResponse, ResourcePermissionInfo, GetDependantResourcesResponse, @@ -9,6 +8,7 @@ import { AddPermissionRequest, RemovePermissionRequest, RemovePermissionResponse, + FetchResourcePermissionInfoResponse, } from "@budibase/types" import { CURRENTLY_SUPPORTED_LEVELS, @@ -28,10 +28,12 @@ export function fetchLevels(ctx: UserCtx) { ctx.body = SUPPORTED_LEVELS } -export async function fetch(ctx: UserCtx) { +export async function fetch( + ctx: UserCtx +) { const db = context.getAppDB() - const dbRoles: Role[] = await sdk.permissions.getAllDBRoles(db) - let permissions: any = {} + const dbRoles = await sdk.permissions.getAllDBRoles(db) + let permissions: Record> = {} // create an object with structure role ID -> resource ID -> level for (let role of dbRoles) { if (!role.permissions) { @@ -43,13 +45,13 @@ export async function fetch(ctx: UserCtx) { } for (let [resource, levelArr] of Object.entries(role.permissions)) { const levels: string[] = Array.isArray(levelArr) ? levelArr : [levelArr] - const perms: Record = {} + const perms: Record = permissions[resource] || {} levels.forEach(level => (perms[level] = roleId!)) permissions[resource] = perms } } // apply the base permissions - const finalPermissions: Record> = {} + const finalPermissions: FetchResourcePermissionInfoResponse = {} for (let [resource, permission] of Object.entries(permissions)) { const basePerms = getBasePermissions(resource) finalPermissions[resource] = Object.assign(basePerms, permission) @@ -92,18 +94,17 @@ export async function getDependantResources( export async function addPermission(ctx: UserCtx) { const params: AddPermissionRequest = ctx.params - ctx.body = await sdk.permissions.updatePermissionOnRole( - params, - PermissionUpdateType.ADD - ) + await sdk.permissions.updatePermissionOnRole(params, PermissionUpdateType.ADD) + ctx.status = 200 } export async function removePermission( ctx: UserCtx ) { const params: RemovePermissionRequest = ctx.params - ctx.body = await sdk.permissions.updatePermissionOnRole( + await sdk.permissions.updatePermissionOnRole( params, PermissionUpdateType.REMOVE ) + ctx.status = 200 } diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts index ed8836626c..56522acb33 100644 --- a/packages/server/src/api/controllers/row/ExternalRequest.ts +++ b/packages/server/src/api/controllers/row/ExternalRequest.ts @@ -4,6 +4,7 @@ import { AutoFieldSubType, AutoReason, Datasource, + DatasourcePlusQueryResponse, FieldSchema, FieldType, FilterType, @@ -269,18 +270,13 @@ export class ExternalRequest { } } - private async removeManyToManyRelationships( - rowId: string, - table: Table, - colName: string - ) { + private async removeManyToManyRelationships(rowId: string, table: Table) { const tableId = table._id! const filters = this.prepareFilters(rowId, {}, table) // safety check, if there are no filters on deletion bad things happen if (Object.keys(filters).length !== 0) { return getDatasourceAndQuery({ endpoint: getEndpoint(tableId, Operation.DELETE), - body: { [colName]: null }, filters, meta: { table, @@ -291,13 +287,18 @@ export class ExternalRequest { } } - private async removeOneToManyRelationships(rowId: string, table: Table) { + private async removeOneToManyRelationships( + rowId: string, + table: Table, + colName: string + ) { const tableId = table._id! const filters = this.prepareFilters(rowId, {}, table) // safety check, if there are no filters on deletion bad things happen if (Object.keys(filters).length !== 0) { return getDatasourceAndQuery({ endpoint: getEndpoint(tableId, Operation.UPDATE), + body: { [colName]: null }, filters, meta: { table, @@ -557,8 +558,9 @@ export class ExternalRequest { return matchesPrimaryLink } - const matchesSecondayLink = row[linkSecondary] === body?.[linkSecondary] - return matchesPrimaryLink && matchesSecondayLink + const matchesSecondaryLink = + row[linkSecondary] === body?.[linkSecondary] + return matchesPrimaryLink && matchesSecondaryLink } const existingRelationship = rows.find((row: { [key: string]: any }) => @@ -595,8 +597,8 @@ export class ExternalRequest { for (let row of rows) { const rowId = generateIdForRow(row, table) const promise: Promise = isMany - ? this.removeManyToManyRelationships(rowId, table, colName) - : this.removeOneToManyRelationships(rowId, table) + ? this.removeManyToManyRelationships(rowId, table) + : this.removeOneToManyRelationships(rowId, table, colName) if (promise) { promises.push(promise) } @@ -619,12 +621,12 @@ export class ExternalRequest { rows.map(row => { const rowId = generateIdForRow(row, table) return isMany - ? this.removeManyToManyRelationships( + ? this.removeManyToManyRelationships(rowId, table) + : this.removeOneToManyRelationships( rowId, table, relationshipColumn.fieldName ) - : this.removeOneToManyRelationships(rowId, table) }) ) } @@ -669,6 +671,7 @@ export class ExternalRequest { config.includeSqlRelationships === IncludeRelationship.INCLUDE // clean up row on ingress using schema + const unprocessedRow = config.row const processed = this.inputProcessing(row, table) row = processed.row let manyRelationships = processed.manyRelationships @@ -743,9 +746,20 @@ export class ExternalRequest { // aliasing can be disabled fully if desired const aliasing = new sdk.rows.AliasTables(Object.keys(this.tables)) - let response = env.SQL_ALIASING_DISABLE - ? await getDatasourceAndQuery(json) - : await aliasing.queryWithAliasing(json, makeExternalQuery) + let response: DatasourcePlusQueryResponse + // there's a chance after input processing nothing needs updated, so pass over the call + // we might still need to perform other operations like updating the foreign keys on other rows + if ( + this.operation === Operation.UPDATE && + Object.keys(row || {}).length === 0 && + unprocessedRow + ) { + response = [unprocessedRow] + } else { + response = env.SQL_ALIASING_DISABLE + ? await getDatasourceAndQuery(json) + : await aliasing.queryWithAliasing(json, makeExternalQuery) + } // if it's a counting operation there will be no more processing, just return the number if (this.operation === Operation.COUNT) { diff --git a/packages/server/src/api/controllers/row/external.ts b/packages/server/src/api/controllers/row/external.ts index 18a9be5087..4fcd612ec3 100644 --- a/packages/server/src/api/controllers/row/external.ts +++ b/packages/server/src/api/controllers/row/external.ts @@ -27,6 +27,7 @@ import { } from "../../../utilities/rowProcessor" import { cloneDeep } from "lodash" import { generateIdForRow } from "./utils" +import { helpers } from "@budibase/shared-core" export async function handleRequest( operation: T, @@ -42,6 +43,11 @@ export async function handleRequest( export async function patch(ctx: UserCtx) { const source = await utils.getSource(ctx) + + if (sdk.views.isView(source) && helpers.views.isCalculationView(source)) { + ctx.throw(400, "Cannot update rows through a calculation view") + } + const table = await utils.getTableFromSource(source) const { _id, ...rowData } = ctx.request.body diff --git a/packages/server/src/api/controllers/row/internal.ts b/packages/server/src/api/controllers/row/internal.ts index bb9de6ce52..53b24d517c 100644 --- a/packages/server/src/api/controllers/row/internal.ts +++ b/packages/server/src/api/controllers/row/internal.ts @@ -22,13 +22,20 @@ import sdk from "../../../sdk" import { getLinkedTableIDs } from "../../../db/linkedRows/linkUtils" import { flatten } from "lodash" import { findRow } from "../../../sdk/app/rows/internal" +import { helpers } from "@budibase/shared-core" export async function patch(ctx: UserCtx) { const { tableId } = utils.getSourceId(ctx) const source = await utils.getSource(ctx) + + if (sdk.views.isView(source) && helpers.views.isCalculationView(source)) { + ctx.throw(400, "Cannot update rows through a calculation view") + } + const table = sdk.views.isView(source) ? await sdk.views.getTable(source.id) : source + const inputs = ctx.request.body const isUserTable = tableId === InternalTables.USER_METADATA let oldRow diff --git a/packages/server/src/api/controllers/table/external.ts b/packages/server/src/api/controllers/table/external.ts index 5b15d3d9c7..6f09bf4a61 100644 --- a/packages/server/src/api/controllers/table/external.ts +++ b/packages/server/src/api/controllers/table/external.ts @@ -31,7 +31,7 @@ function getDatasourceId(table: Table) { return breakExternalTableId(table._id).datasourceId } -export async function save( +export async function updateTable( ctx: UserCtx, renaming?: RenameColumn ) { diff --git a/packages/server/src/api/controllers/table/index.ts b/packages/server/src/api/controllers/table/index.ts index 7bec1581b4..404f82e57c 100644 --- a/packages/server/src/api/controllers/table/index.ts +++ b/packages/server/src/api/controllers/table/index.ts @@ -102,18 +102,22 @@ export async function find(ctx: UserCtx) { export async function save(ctx: UserCtx) { const appId = ctx.appId - const table = ctx.request.body - const isImport = table.rows + const { rows, ...table } = ctx.request.body + const isImport = rows const renaming = ctx.request.body._rename + const isCreate = !table._id + checkDefaultFields(table) - const api = pickApi({ table }) - let savedTable = await api.save(ctx, renaming) - if (!table._id) { + let savedTable: Table + if (isCreate) { + savedTable = await sdk.tables.create(table, rows, ctx.user._id) savedTable = await sdk.tables.enrichViewSchemas(savedTable) await events.table.created(savedTable) } else { + const api = pickApi({ table }) + savedTable = await api.updateTable(ctx, renaming) await events.table.updated(savedTable) } if (renaming) { diff --git a/packages/server/src/api/controllers/table/internal.ts b/packages/server/src/api/controllers/table/internal.ts index 4286d51d3e..40ce5e279d 100644 --- a/packages/server/src/api/controllers/table/internal.ts +++ b/packages/server/src/api/controllers/table/internal.ts @@ -12,7 +12,7 @@ import { } from "@budibase/types" import sdk from "../../../sdk" -export async function save( +export async function updateTable( ctx: UserCtx, renaming?: RenameColumn ) { @@ -25,19 +25,16 @@ export async function save( sourceType: rest.sourceType || TableSourceType.INTERNAL, } - const isImport = !!rows - if (!tableToSave.views) { tableToSave.views = {} } try { const { table } = await sdk.tables.internal.save(tableToSave, { - user: ctx.user, + userId: ctx.user._id, rowsToImport: rows, tableId: ctx.request.body._id, renaming, - isImport, }) return table @@ -72,7 +69,7 @@ export async function bulkImport( await handleDataImport(table, { importRows: rows, identifierFields, - user: ctx.user, + userId: ctx.user._id, }) return table } diff --git a/packages/server/src/api/controllers/table/tests/utils.spec.ts b/packages/server/src/api/controllers/table/tests/utils.spec.ts index dad0146696..68b00d3268 100644 --- a/packages/server/src/api/controllers/table/tests/utils.spec.ts +++ b/packages/server/src/api/controllers/table/tests/utils.spec.ts @@ -41,7 +41,7 @@ describe("utils", () => { const data = [{ name: "Alice" }, { name: "Bob" }, { name: "Claire" }] - const result = await importToRows(data, table, config.user) + const result = await importToRows(data, table, config.user?._id) expect(result).toEqual([ expect.objectContaining({ autoId: 1, diff --git a/packages/server/src/api/controllers/table/utils.ts b/packages/server/src/api/controllers/table/utils.ts index d568e5f33e..106d0e23a6 100644 --- a/packages/server/src/api/controllers/table/utils.ts +++ b/packages/server/src/api/controllers/table/utils.ts @@ -18,7 +18,6 @@ import { quotas } from "@budibase/pro" import { events, context, features } from "@budibase/backend-core" import { AutoFieldSubType, - ContextUser, Datasource, Row, SourceName, @@ -122,7 +121,7 @@ export function makeSureTableUpToDate(table: Table, tableToSave: Table) { export async function importToRows( data: Row[], table: Table, - user?: ContextUser, + userId?: string, opts?: { keepCouchId: boolean } ) { const originalTable = table @@ -136,7 +135,7 @@ export async function importToRows( // We use a reference to table here and update it after input processing, // so that we can auto increment auto IDs in imported data properly - const processed = await inputProcessing(user?._id, table, row, { + const processed = await inputProcessing(userId, table, row, { noAutoRelationships: true, }) row = processed @@ -167,11 +166,10 @@ export async function importToRows( export async function handleDataImport( table: Table, - opts?: { identifierFields?: string[]; user?: ContextUser; importRows?: Row[] } + opts?: { identifierFields?: string[]; userId?: string; importRows?: Row[] } ) { const schema = table.schema const identifierFields = opts?.identifierFields || [] - const user = opts?.user const importRows = opts?.importRows if (!importRows || !isRows(importRows) || !isSchema(schema)) { @@ -181,7 +179,7 @@ export async function handleDataImport( const db = context.getAppDB() const data = parse(importRows, table) - const finalData = await importToRows(data, table, user, { + const finalData = await importToRows(data, table, opts?.userId, { keepCouchId: identifierFields.includes("_id"), }) @@ -282,22 +280,22 @@ export function checkStaticTables(table: Table) { class TableSaveFunctions { db: Database - user?: ContextUser + userId?: string oldTable?: Table importRows?: Row[] rows: Row[] constructor({ - user, + userId, oldTable, importRows, }: { - user?: ContextUser + userId?: string oldTable?: Table importRows?: Row[] }) { this.db = context.getAppDB() - this.user = user + this.userId = userId this.oldTable = oldTable this.importRows = importRows // any rows that need updated @@ -329,7 +327,7 @@ class TableSaveFunctions { table = await handleSearchIndexes(table) table = await handleDataImport(table, { importRows: this.importRows, - user: this.user, + userId: this.userId, }) if (await features.flags.isEnabled("SQS")) { await sdk.tables.sqs.addTable(table) diff --git a/packages/server/src/api/routes/tests/application.spec.ts b/packages/server/src/api/routes/tests/application.spec.ts index 40ed828dce..fe8250bde5 100644 --- a/packages/server/src/api/routes/tests/application.spec.ts +++ b/packages/server/src/api/routes/tests/application.spec.ts @@ -14,12 +14,7 @@ jest.mock("../../../utilities/redis", () => ({ import { checkBuilderEndpoint } from "./utilities/TestFunctions" import * as setup from "./utilities" import { AppStatus } from "../../../db/utils" -import { - events, - utils, - context, - withEnv as withCoreEnv, -} from "@budibase/backend-core" +import { events, utils, context, features } from "@budibase/backend-core" import env from "../../../environment" import { type App } from "@budibase/types" import tk from "timekeeper" @@ -358,9 +353,13 @@ describe("/applications", () => { .delete(`/api/global/roles/${prodAppId}`) .reply(200, {}) - await withCoreEnv({ TENANT_FEATURE_FLAGS: "*:SQS" }, async () => { - await config.api.application.delete(app.appId) - }) + await features.testutils.withFeatureFlags( + "*", + { SQS: true }, + async () => { + await config.api.application.delete(app.appId) + } + ) }) }) diff --git a/packages/server/src/api/routes/tests/permissions.spec.ts b/packages/server/src/api/routes/tests/permissions.spec.ts index 625143e7fb..a479adb4cf 100644 --- a/packages/server/src/api/routes/tests/permissions.spec.ts +++ b/packages/server/src/api/routes/tests/permissions.spec.ts @@ -1,5 +1,5 @@ import { roles } from "@budibase/backend-core" -import { Document, PermissionLevel, Row, Table, ViewV2 } from "@budibase/types" +import { Document, PermissionLevel, Row } from "@budibase/types" import * as setup from "./utilities" import { generator, mocks } from "@budibase/backend-core/tests" @@ -9,13 +9,11 @@ const { BUILTIN_ROLE_IDS } = roles const HIGHER_ROLE_ID = BUILTIN_ROLE_IDS.BASIC const STD_ROLE_ID = BUILTIN_ROLE_IDS.PUBLIC +const DEFAULT_TABLE_ROLE_ID = BUILTIN_ROLE_IDS.ADMIN + describe("/permission", () => { let request = setup.getRequest() let config = setup.getConfig() - let table: Table & { _id: string } - let perms: Document[] - let row: Row - let view: ViewV2 afterAll(setup.afterAll) @@ -25,18 +23,6 @@ describe("/permission", () => { beforeEach(async () => { mocks.licenses.useCloudFree() - - table = (await config.createTable()) as typeof table - row = await config.createRow() - view = await config.api.viewV2.create({ - tableId: table._id!, - name: generator.guid(), - }) - perms = await config.api.permission.add({ - roleId: STD_ROLE_ID, - resourceId: table._id, - level: PermissionLevel.READ, - }) }) describe("levels", () => { @@ -54,134 +40,251 @@ describe("/permission", () => { }) }) - describe("add", () => { - it("should be able to add permission to a role for the table", async () => { - expect(perms.length).toEqual(1) - expect(perms[0]._id).toEqual(`${STD_ROLE_ID}`) - }) + describe("table permissions", () => { + let tableId: string - it("should get the resource permissions", async () => { - const res = await request - .get(`/api/permission/${table._id}`) - .set(config.defaultHeaders()) - .expect("Content-Type", /json/) - .expect(200) - expect(res.body).toEqual({ - permissions: { - read: { permissionType: "EXPLICIT", role: STD_ROLE_ID }, - write: { permissionType: "BASE", role: HIGHER_ROLE_ID }, - }, - }) - }) - - it("should get resource permissions with multiple roles", async () => { - perms = await config.api.permission.add({ - roleId: HIGHER_ROLE_ID, - resourceId: table._id, - level: PermissionLevel.WRITE, - }) - const res = await config.api.permission.get(table._id) - expect(res).toEqual({ - permissions: { - read: { permissionType: "EXPLICIT", role: STD_ROLE_ID }, - write: { permissionType: "EXPLICIT", role: HIGHER_ROLE_ID }, - }, - }) - - const allRes = await request - .get(`/api/permission`) - .set(config.defaultHeaders()) - .expect("Content-Type", /json/) - .expect(200) - expect(allRes.body[table._id]["read"]).toEqual(STD_ROLE_ID) - expect(allRes.body[table._id]["write"]).toEqual(HIGHER_ROLE_ID) - }) - }) - - describe("remove", () => { - it("should be able to remove the permission", async () => { - const res = await config.api.permission.revoke({ - roleId: STD_ROLE_ID, - resourceId: table._id, - level: PermissionLevel.READ, - }) - expect(res[0]._id).toEqual(STD_ROLE_ID) - const permsRes = await config.api.permission.get(table._id) - expect(permsRes.permissions[STD_ROLE_ID]).toBeUndefined() - }) - }) - - describe("check public user allowed", () => { - it("should be able to read the row", async () => { - // replicate changes before checking permissions - await config.publish() - - const res = await request - .get(`/api/${table._id}/rows`) - .set(config.publicHeaders()) - .expect("Content-Type", /json/) - .expect(200) - expect(res.body[0]._id).toEqual(row._id) - }) - - it("should be able to access the view data when the table is set to public and with no view permissions overrides", async () => { - // Make view inherit table permissions. Needed for backwards compatibility with existing views. - await config.api.permission.revoke({ - roleId: STD_ROLE_ID, - resourceId: view.id, - level: PermissionLevel.READ, - }) - // replicate changes before checking permissions - await config.publish() - - const res = await config.api.viewV2.publicSearch(view.id) - expect(res.rows[0]._id).toEqual(row._id) - }) - - it("should not be able to access the view data when the table is not public and there are no view permissions overrides", async () => { - await config.api.permission.revoke({ - roleId: STD_ROLE_ID, - resourceId: table._id, - level: PermissionLevel.READ, - }) - // Make view inherit table permissions. Needed for backwards compatibility with existing views. - await config.api.permission.revoke({ - roleId: STD_ROLE_ID, - resourceId: view.id, - level: PermissionLevel.READ, - }) - // replicate changes before checking permissions - await config.publish() - - await config.api.viewV2.publicSearch(view.id, undefined, { status: 401 }) - }) - - it("should use the view permissions", async () => { + beforeEach(async () => { + const table = await config.createTable() + tableId = table._id! await config.api.permission.add({ roleId: STD_ROLE_ID, - resourceId: view.id, + resourceId: tableId, level: PermissionLevel.READ, }) - await config.api.permission.revoke({ - roleId: STD_ROLE_ID, - resourceId: table._id, - level: PermissionLevel.READ, - }) - // replicate changes before checking permissions - await config.publish() - - const res = await config.api.viewV2.publicSearch(view.id) - expect(res.rows[0]._id).toEqual(row._id) }) - it("shouldn't allow writing from a public user", async () => { - const res = await request - .post(`/api/${table._id}/rows`) - .send(basicRow(table._id)) - .set(config.publicHeaders()) - .expect("Content-Type", /json/) - .expect(401) - expect(res.status).toEqual(401) + it("tables should be defaulted to admin", async () => { + const table = await config.createTable() + const { permissions } = await config.api.permission.get(table._id!) + expect(permissions).toEqual({ + read: { + permissionType: "EXPLICIT", + role: DEFAULT_TABLE_ROLE_ID, + }, + write: { + permissionType: "EXPLICIT", + role: DEFAULT_TABLE_ROLE_ID, + }, + }) + }) + + describe("add", () => { + it("should be able to add permission to a role for the table", async () => { + const res = await request + .get(`/api/permission/${tableId}`) + .set(config.defaultHeaders()) + .expect("Content-Type", /json/) + .expect(200) + expect(res.body).toEqual({ + permissions: { + read: { permissionType: "EXPLICIT", role: STD_ROLE_ID }, + write: { permissionType: "EXPLICIT", role: DEFAULT_TABLE_ROLE_ID }, + }, + }) + }) + + it("should get resource permissions with multiple roles", async () => { + await config.api.permission.add({ + roleId: HIGHER_ROLE_ID, + resourceId: tableId, + level: PermissionLevel.WRITE, + }) + const res = await config.api.permission.get(tableId) + expect(res).toEqual({ + permissions: { + read: { permissionType: "EXPLICIT", role: STD_ROLE_ID }, + write: { permissionType: "EXPLICIT", role: HIGHER_ROLE_ID }, + }, + }) + + const allRes = await request + .get(`/api/permission`) + .set(config.defaultHeaders()) + .expect("Content-Type", /json/) + .expect(200) + expect(allRes.body[tableId]["read"]).toEqual(STD_ROLE_ID) + expect(allRes.body[tableId]["write"]).toEqual(HIGHER_ROLE_ID) + }) + }) + + describe("remove", () => { + it("should be able to remove the permission", async () => { + await config.api.permission.revoke({ + roleId: STD_ROLE_ID, + resourceId: tableId, + level: PermissionLevel.READ, + }) + + const permsRes = await config.api.permission.get(tableId) + expect(permsRes.permissions[STD_ROLE_ID]).toBeUndefined() + }) + }) + + describe("check public user allowed", () => { + let viewId: string + let row: Row + + beforeEach(async () => { + const view = await config.api.viewV2.create({ + tableId, + name: generator.guid(), + }) + viewId = view.id + row = await config.createRow() + }) + + it("should be able to read the row", async () => { + // replicate changes before checking permissions + await config.publish() + + const res = await request + .get(`/api/${tableId}/rows`) + .set(config.publicHeaders()) + .expect("Content-Type", /json/) + .expect(200) + expect(res.body[0]._id).toEqual(row._id) + }) + + it("should be able to access the view data when the table is set to public and with no view permissions overrides", async () => { + // Make view inherit table permissions. Needed for backwards compatibility with existing views. + await config.api.permission.revoke({ + roleId: STD_ROLE_ID, + resourceId: viewId, + level: PermissionLevel.READ, + }) + + // replicate changes before checking permissions + await config.publish() + + const res = await config.api.viewV2.publicSearch(viewId) + expect(res.rows[0]._id).toEqual(row._id) + }) + + it("should not be able to access the view data when the table is not public and there are no view permissions overrides", async () => { + await config.api.permission.revoke({ + roleId: STD_ROLE_ID, + resourceId: tableId, + level: PermissionLevel.READ, + }) + + // Make view inherit table permissions. Needed for backwards compatibility with existing views. + await config.api.permission.revoke({ + roleId: STD_ROLE_ID, + resourceId: viewId, + level: PermissionLevel.READ, + }) + + // replicate changes before checking permissions + await config.publish() + + await config.api.viewV2.publicSearch(viewId, undefined, { + status: 401, + }) + }) + + it("should use the view permissions", async () => { + await config.api.permission.add({ + roleId: STD_ROLE_ID, + resourceId: viewId, + level: PermissionLevel.READ, + }) + await config.api.permission.revoke({ + roleId: STD_ROLE_ID, + resourceId: tableId, + level: PermissionLevel.READ, + }) + // replicate changes before checking permissions + await config.publish() + + const res = await config.api.viewV2.publicSearch(viewId) + expect(res.rows[0]._id).toEqual(row._id) + }) + + it("shouldn't allow writing from a public user", async () => { + const res = await request + .post(`/api/${tableId}/rows`) + .send(basicRow(tableId)) + .set(config.publicHeaders()) + .expect("Content-Type", /json/) + .expect(401) + expect(res.status).toEqual(401) + }) + }) + }) + + describe("view permissions", () => { + let tableId: string + let viewId: string + + beforeEach(async () => { + const table = await config.createTable() + tableId = table._id! + + const view = await config.api.viewV2.create({ + tableId, + name: generator.guid(), + }) + viewId = view.id + }) + + it("default permissions inherits and persists the table default value", async () => { + const { permissions } = await config.api.permission.get(viewId) + expect(permissions).toEqual({ + read: { + permissionType: "EXPLICIT", + role: DEFAULT_TABLE_ROLE_ID, + inheritablePermission: DEFAULT_TABLE_ROLE_ID, + }, + write: { + permissionType: "EXPLICIT", + role: DEFAULT_TABLE_ROLE_ID, + inheritablePermission: DEFAULT_TABLE_ROLE_ID, + }, + }) + }) + + it("does not update view permissions once persisted, even if table permissions change", async () => { + await config.api.permission.add({ + roleId: STD_ROLE_ID, + resourceId: tableId, + level: PermissionLevel.READ, + }) + + const { permissions } = await config.api.permission.get(viewId) + expect(permissions).toEqual({ + read: { + permissionType: "EXPLICIT", + role: DEFAULT_TABLE_ROLE_ID, + inheritablePermission: STD_ROLE_ID, + }, + write: { + permissionType: "EXPLICIT", + role: DEFAULT_TABLE_ROLE_ID, + inheritablePermission: DEFAULT_TABLE_ROLE_ID, + }, + }) + }) + + it("can sets permissions inherits explicit view permissions", async () => { + await config.api.permission.add({ + roleId: HIGHER_ROLE_ID, + resourceId: viewId, + level: PermissionLevel.WRITE, + }) + + const { permissions } = await config.api.permission.get(viewId) + expect(permissions).toEqual({ + read: { + permissionType: "EXPLICIT", + role: DEFAULT_TABLE_ROLE_ID, + inheritablePermission: DEFAULT_TABLE_ROLE_ID, + }, + write: { + permissionType: "EXPLICIT", + role: HIGHER_ROLE_ID, + inheritablePermission: DEFAULT_TABLE_ROLE_ID, + }, + }) }) }) diff --git a/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts b/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts index e6da4693eb..0979f8bed3 100644 --- a/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts +++ b/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts @@ -28,6 +28,7 @@ describe.each( const config = setup.getConfig() const isOracle = dbName === DatabaseName.ORACLE const isMsSQL = dbName === DatabaseName.SQL_SERVER + const isPostgres = dbName === DatabaseName.POSTGRES let rawDatasource: Datasource let datasource: Datasource @@ -47,6 +48,9 @@ describe.each( transformer: "return data", readable: true, } + if (query.fields?.sql && typeof query.fields.sql !== "string") { + throw new Error("Unable to create with knex structure in 'sql' field") + } return await config.api.query.save( { ...defaultQuery, ...query }, expectations @@ -207,6 +211,31 @@ describe.each( expect(prodQuery.parameters).toBeUndefined() expect(prodQuery.schema).toBeDefined() }) + + isPostgres && + it("should be able to handle a JSON aggregate with newlines", async () => { + const jsonStatement = `COALESCE(json_build_object('name', name),'{"name":{}}'::json)` + const query = await createQuery({ + fields: { + sql: client("test_table") + .select([ + "*", + client.raw( + `${jsonStatement} as json,\n${jsonStatement} as json2` + ), + ]) + .toString(), + }, + }) + const res = await config.api.query.execute( + query._id!, + {}, + { + status: 200, + } + ) + expect(res).toBeDefined() + }) }) }) diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts index 5222069460..b86ec38d08 100644 --- a/packages/server/src/api/routes/tests/row.spec.ts +++ b/packages/server/src/api/routes/tests/row.spec.ts @@ -13,8 +13,7 @@ import { context, InternalTable, tenancy, - withEnv as withCoreEnv, - setEnv as setCoreEnv, + features, } from "@budibase/backend-core" import { quotas } from "@budibase/pro" import { @@ -40,7 +39,6 @@ import { TableSchema, JsonFieldSubType, RowExportFormat, - FeatureFlag, RelationSchemaField, } from "@budibase/types" import { generator, mocks } from "@budibase/backend-core/tests" @@ -98,12 +96,12 @@ describe.each([ let envCleanup: (() => void) | undefined beforeAll(async () => { - await withCoreEnv({ TENANT_FEATURE_FLAGS: "*:SQS" }, () => config.init()) - if (isLucene) { - envCleanup = setCoreEnv({ TENANT_FEATURE_FLAGS: "*:!SQS" }) - } else if (isSqs) { - envCleanup = setCoreEnv({ TENANT_FEATURE_FLAGS: "*:SQS" }) - } + await features.testutils.withFeatureFlags("*", { SQS: true }, () => + config.init() + ) + envCleanup = features.testutils.setFeatureFlags("*", { + SQS: isSqs, + }) if (dsProvider) { const rawDatasource = await dsProvider @@ -1116,6 +1114,33 @@ describe.each([ expect(getResp.user2[0]._id).toEqual(user2._id) }) + it("should be able to remove a relationship from many side", async () => { + const row = await config.api.row.save(otherTable._id!, { + name: "test", + description: "test", + }) + const row2 = await config.api.row.save(otherTable._id!, { + name: "test", + description: "test", + }) + const { _id } = await config.api.row.save(table._id!, { + relationship: [{ _id: row._id }, { _id: row2._id }], + }) + const relatedRow = await config.api.row.get(table._id!, _id!, { + status: 200, + }) + expect(relatedRow.relationship.length).toEqual(2) + await config.api.row.save(table._id!, { + ...relatedRow, + relationship: [{ _id: row._id }], + }) + const afterRelatedRow = await config.api.row.get(table._id!, _id!, { + status: 200, + }) + expect(afterRelatedRow.relationship.length).toEqual(1) + expect(afterRelatedRow.relationship[0]._id).toEqual(row._id) + }) + it("should be able to update relationships when both columns are same name", async () => { let row = await config.api.row.save(table._id!, { name: "test", @@ -1848,7 +1873,7 @@ describe.each([ }) describe("exportRows", () => { - beforeAll(async () => { + beforeEach(async () => { table = await config.api.table.save(defaultTable()) }) @@ -1885,6 +1910,16 @@ describe.each([ }) }) + it("should allow exporting without filtering", async () => { + const existing = await config.api.row.save(table._id!, {}) + const res = await config.api.row.exportRows(table._id!) + const results = JSON.parse(res) + expect(results.length).toEqual(1) + const row = results[0] + + expect(row._id).toEqual(existing._id) + }) + it("should allow exporting only certain columns", async () => { const existing = await config.api.row.save(table._id!, {}) const res = await config.api.row.exportRows(table._id!, { @@ -2517,15 +2552,9 @@ describe.each([ let flagCleanup: (() => void) | undefined beforeAll(async () => { - const env = { - TENANT_FEATURE_FLAGS: `*:${FeatureFlag.ENRICHED_RELATIONSHIPS}`, - } - if (isSqs) { - env.TENANT_FEATURE_FLAGS = `${env.TENANT_FEATURE_FLAGS},*:SQS` - } else { - env.TENANT_FEATURE_FLAGS = `${env.TENANT_FEATURE_FLAGS},*:!SQS` - } - flagCleanup = setCoreEnv(env) + flagCleanup = features.testutils.setFeatureFlags("*", { + ENRICHED_RELATIONSHIPS: true, + }) const aux2Table = await config.api.table.save(saveTableRequest()) const aux2Data = await config.api.row.save(aux2Table._id!, {}) @@ -2752,9 +2781,10 @@ describe.each([ it.each(testScenarios)( "does not enrich relationships when not enabled (via %s)", async (__, retrieveDelegate) => { - await withCoreEnv( + await features.testutils.withFeatureFlags( + "*", { - TENANT_FEATURE_FLAGS: `*:!${FeatureFlag.ENRICHED_RELATIONSHIPS}`, + ENRICHED_RELATIONSHIPS: false, }, async () => { const otherRows = _.sampleSize(auxData, 5) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index 110899e292..1ccc9bfdc9 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -7,9 +7,9 @@ import { import { context, db as dbCore, + features, MAX_VALID_DATE, MIN_VALID_DATE, - setEnv as setCoreEnv, SQLITE_DESIGN_DOC_ID, utils, withEnv as withCoreEnv, @@ -94,16 +94,12 @@ describe.each([ } beforeAll(async () => { - await withCoreEnv({ TENANT_FEATURE_FLAGS: "*:SQS" }, () => config.init()) - if (isLucene) { - envCleanup = setCoreEnv({ - TENANT_FEATURE_FLAGS: "*:!SQS", - }) - } else if (isSqs) { - envCleanup = setCoreEnv({ - TENANT_FEATURE_FLAGS: "*:SQS", - }) - } + await features.testutils.withFeatureFlags("*", { SQS: true }, () => + config.init() + ) + envCleanup = features.testutils.setFeatureFlags("*", { + SQS: isSqs, + }) if (config.app?.appId) { config.app = await config.api.application.update(config.app?.appId, { @@ -191,7 +187,6 @@ describe.each([ if (isInMemory) { return dataFilters.search(_.cloneDeep(rows), { ...this.query, - tableId: tableOrViewId, }) } else { return config.api.row.search(tableOrViewId, this.query) diff --git a/packages/server/src/api/routes/tests/templates.spec.ts b/packages/server/src/api/routes/tests/templates.spec.ts index 6f4d468a68..d5483c54b4 100644 --- a/packages/server/src/api/routes/tests/templates.spec.ts +++ b/packages/server/src/api/routes/tests/templates.spec.ts @@ -2,7 +2,7 @@ import * as setup from "./utilities" import path from "path" import nock from "nock" import { generator } from "@budibase/backend-core/tests" -import { withEnv as withCoreEnv, env as coreEnv } from "@budibase/backend-core" +import { features } from "@budibase/backend-core" interface App { background: string @@ -85,41 +85,44 @@ describe("/templates", () => { it.each(["sqs", "lucene"])( `should be able to create an app from a template (%s)`, async source => { - const env: Partial = { - TENANT_FEATURE_FLAGS: source === "sqs" ? "*:SQS" : "", - } + await features.testutils.withFeatureFlags( + "*", + { SQS: source === "sqs" }, + async () => { + const name = generator.guid().replaceAll("-", "") + const url = `/${name}` - await withCoreEnv(env, async () => { - const name = generator.guid().replaceAll("-", "") - const url = `/${name}` - - const app = await config.api.application.create({ - name, - url, - useTemplate: "true", - templateName: "Agency Client Portal", - templateKey: "app/agency-client-portal", - }) - expect(app.name).toBe(name) - expect(app.url).toBe(url) - - await config.withApp(app, async () => { - const tables = await config.api.table.fetch() - expect(tables).toHaveLength(2) - - tables.sort((a, b) => a.name.localeCompare(b.name)) - const [agencyProjects, users] = tables - expect(agencyProjects.name).toBe("Agency Projects") - expect(users.name).toBe("Users") - - const { rows } = await config.api.row.search(agencyProjects._id!, { - tableId: agencyProjects._id!, - query: {}, + const app = await config.api.application.create({ + name, + url, + useTemplate: "true", + templateName: "Agency Client Portal", + templateKey: "app/agency-client-portal", }) + expect(app.name).toBe(name) + expect(app.url).toBe(url) - expect(rows).toHaveLength(3) - }) - }) + await config.withApp(app, async () => { + const tables = await config.api.table.fetch() + expect(tables).toHaveLength(2) + + tables.sort((a, b) => a.name.localeCompare(b.name)) + const [agencyProjects, users] = tables + expect(agencyProjects.name).toBe("Agency Projects") + expect(users.name).toBe("Users") + + const { rows } = await config.api.row.search( + agencyProjects._id!, + { + tableId: agencyProjects._id!, + query: {}, + } + ) + + expect(rows).toHaveLength(3) + }) + } + ) } ) }) diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts index 5d565dbdbc..195481ae10 100644 --- a/packages/server/src/api/routes/tests/viewV2.spec.ts +++ b/packages/server/src/api/routes/tests/viewV2.spec.ts @@ -22,23 +22,17 @@ import { RelationshipType, TableSchema, RenameColumn, - FeatureFlag, BBReferenceFieldSubType, NumericCalculationFieldMetadata, ViewV2Schema, ViewV2Type, + JsonTypes, } from "@budibase/types" import { generator, mocks } from "@budibase/backend-core/tests" import { DatabaseName, getDatasource } from "../../../integrations/tests/utils" import merge from "lodash/merge" import { quotas } from "@budibase/pro" -import { - db, - roles, - withEnv as withCoreEnv, - setEnv as setCoreEnv, - env, -} from "@budibase/backend-core" +import { db, roles, features } from "@budibase/backend-core" describe.each([ ["lucene", undefined], @@ -103,18 +97,13 @@ describe.each([ } beforeAll(async () => { - await withCoreEnv({ TENANT_FEATURE_FLAGS: isSqs ? "*:SQS" : "" }, () => + await features.testutils.withFeatureFlags("*", { SQS: isSqs }, () => config.init() ) - if (isLucene) { - envCleanup = setCoreEnv({ - TENANT_FEATURE_FLAGS: "*:!SQS", - }) - } else if (isSqs) { - envCleanup = setCoreEnv({ - TENANT_FEATURE_FLAGS: "*:SQS", - }) - } + + envCleanup = features.testutils.setFeatureFlags("*", { + SQS: isSqs, + }) if (dsProvider) { datasource = await config.createDatasource({ @@ -748,6 +737,69 @@ describe.each([ }, }) }) + + !isLucene && + it("does not get confused when a calculation field shadows a basic one", async () => { + const table = await config.api.table.save( + saveTableRequest({ + schema: { + age: { + name: "age", + type: FieldType.NUMBER, + }, + }, + }) + ) + + await config.api.row.bulkImport(table._id!, { + rows: [{ age: 1 }, { age: 2 }, { age: 3 }], + }) + + const view = await config.api.viewV2.create({ + tableId: table._id!, + name: generator.guid(), + type: ViewV2Type.CALCULATION, + schema: { + age: { + visible: true, + calculationType: CalculationType.SUM, + field: "age", + }, + }, + }) + + const { rows } = await config.api.row.search(view.id) + expect(rows).toHaveLength(1) + expect(rows[0].age).toEqual(6) + }) + + // We don't allow the creation of tables with most JsonTypes when using + // external datasources. + isInternal && + it("cannot use complex types as group-by fields", async () => { + for (const type of JsonTypes) { + const field = { name: "field", type } as FieldSchema + const table = await config.api.table.save( + saveTableRequest({ schema: { field } }) + ) + await config.api.viewV2.create( + { + tableId: table._id!, + name: generator.guid(), + type: ViewV2Type.CALCULATION, + schema: { + field: { visible: true }, + }, + }, + { + status: 400, + body: { + message: `Grouping by fields of type "${type}" is not supported`, + }, + } + ) + } + }) }) describe("update", () => { @@ -1926,6 +1978,30 @@ describe.each([ expect(newRow.one).toBeUndefined() expect(newRow.two).toEqual("bar") }) + + it("should not be possible to create a row in a calculation view", async () => { + const view = await config.api.viewV2.create({ + tableId: table._id!, + name: generator.guid(), + type: ViewV2Type.CALCULATION, + schema: { + id: { visible: true }, + one: { visible: true }, + }, + }) + + await config.api.row.save( + view.id, + { one: "foo" }, + { + status: 400, + body: { + message: "Cannot insert rows through a calculation view", + status: 400, + }, + } + ) + }) }) describe("patch", () => { @@ -1990,6 +2066,40 @@ describe.each([ expect(row.one).toEqual("foo") expect(row.two).toEqual("newBar") }) + + it("should not be possible to modify a row in a calculation view", async () => { + const view = await config.api.viewV2.create({ + tableId: table._id!, + name: generator.guid(), + type: ViewV2Type.CALCULATION, + schema: { + id: { visible: true }, + one: { visible: true }, + }, + }) + + const newRow = await config.api.row.save(table._id!, { + one: "foo", + two: "bar", + }) + + await config.api.row.patch( + view.id, + { + tableId: table._id!, + _id: newRow._id!, + _rev: newRow._rev!, + one: "newFoo", + two: "newBar", + }, + { + status: 400, + body: { + message: "Cannot update rows through a calculation view", + }, + } + ) + }) }) describe("destroy", () => { @@ -2666,12 +2776,8 @@ describe.each([ describe("foreign relationship columns", () => { let envCleanup: () => void beforeAll(() => { - const flags = [`*:${FeatureFlag.ENRICHED_RELATIONSHIPS}`] - if (env.TENANT_FEATURE_FLAGS) { - flags.push(...env.TENANT_FEATURE_FLAGS.split(",")) - } - envCleanup = setCoreEnv({ - TENANT_FEATURE_FLAGS: flags.join(","), + envCleanup = features.testutils.setFeatureFlags("*", { + ENRICHED_RELATIONSHIPS: true, }) }) diff --git a/packages/server/src/appMigrations/migrations/tests/20240604153647_initial_sqs.spec.ts b/packages/server/src/appMigrations/migrations/tests/20240604153647_initial_sqs.spec.ts index b4f708edb5..fe44b7b901 100644 --- a/packages/server/src/appMigrations/migrations/tests/20240604153647_initial_sqs.spec.ts +++ b/packages/server/src/appMigrations/migrations/tests/20240604153647_initial_sqs.spec.ts @@ -2,8 +2,8 @@ import * as setup from "../../../api/routes/tests/utilities" import { basicTable } from "../../../tests/utilities/structures" import { db as dbCore, + features, SQLITE_DESIGN_DOC_ID, - withEnv as withCoreEnv, } from "@budibase/backend-core" import { LinkDocument, @@ -71,11 +71,11 @@ function oldLinkDocument(): Omit { } async function sqsDisabled(cb: () => Promise) { - await withCoreEnv({ TENANT_FEATURE_FLAGS: "*:!SQS" }, cb) + await features.testutils.withFeatureFlags("*", { SQS: false }, cb) } async function sqsEnabled(cb: () => Promise) { - await withCoreEnv({ TENANT_FEATURE_FLAGS: "*:SQS" }, cb) + await features.testutils.withFeatureFlags("*", { SQS: true }, cb) } describe("SQS migration", () => { diff --git a/packages/server/src/db/linkedRows/LinkController.ts b/packages/server/src/db/linkedRows/LinkController.ts index 85a160713b..1cd4240d4b 100644 --- a/packages/server/src/db/linkedRows/LinkController.ts +++ b/packages/server/src/db/linkedRows/LinkController.ts @@ -221,9 +221,15 @@ class LinkController { link.id !== row._id && link.fieldName === linkedSchema.name ) + // check all the related rows exist + const foundRecords = await this._db.getMultiple( + links.map(l => l.id), + { allowMissing: true, excludeDocs: true } + ) + // The 1 side of 1:N is already related to something else // You must remove the existing relationship - if (links.length > 0) { + if (foundRecords.length > 0) { throw new Error( `1:N Relationship Error: Record already linked to another.` ) diff --git a/packages/server/src/integrations/postgres.ts b/packages/server/src/integrations/postgres.ts index 3652864991..28400f616f 100644 --- a/packages/server/src/integrations/postgres.ts +++ b/packages/server/src/integrations/postgres.ts @@ -41,7 +41,7 @@ if (types) { types.setTypeParser(1184, (val: any) => val) // timestampz } -const JSON_REGEX = /'{.*}'::json/s +const JSON_REGEX = /'{\s*.*?\s*}'::json/gs const Sql = sql.Sql interface PostgresConfig { diff --git a/packages/server/src/koa.ts b/packages/server/src/koa.ts index 9f90c04b50..acae433cc3 100644 --- a/packages/server/src/koa.ts +++ b/packages/server/src/koa.ts @@ -12,6 +12,7 @@ import { userAgent } from "koa-useragent" export default function createKoaApp() { const app = new Koa() + app.proxy = true let mbNumber = parseInt(env.HTTP_MB_LIMIT || "10") if (!mbNumber || isNaN(mbNumber)) { @@ -35,6 +36,7 @@ export default function createKoaApp() { app.use(middleware.correlation) app.use(middleware.pino) + app.use(middleware.ip) app.use(userAgent) const server = http.createServer(app.callback()) diff --git a/packages/server/src/migrations/tests/index.spec.ts b/packages/server/src/migrations/tests/index.spec.ts index d06cd37b69..6b3f3314ba 100644 --- a/packages/server/src/migrations/tests/index.spec.ts +++ b/packages/server/src/migrations/tests/index.spec.ts @@ -71,7 +71,7 @@ describe("migrations", () => { expect(events.datasource.created).toHaveBeenCalledTimes(2) expect(events.layout.created).toHaveBeenCalledTimes(1) expect(events.query.created).toHaveBeenCalledTimes(2) - expect(events.role.created).toHaveBeenCalledTimes(2) + expect(events.role.created).toHaveBeenCalledTimes(3) // created roles + admin (created on table creation) expect(events.table.created).toHaveBeenCalledTimes(3) expect(events.view.created).toHaveBeenCalledTimes(2) expect(events.view.calculationCreated).toHaveBeenCalledTimes(1) @@ -82,7 +82,7 @@ describe("migrations", () => { // to make sure caching is working as expected expect( events.processors.analyticsProcessor.processEvent - ).toHaveBeenCalledTimes(23) + ).toHaveBeenCalledTimes(24) // Addtion of of the events above }) }) }) diff --git a/packages/server/src/sdk/app/permissions/index.ts b/packages/server/src/sdk/app/permissions/index.ts index 2c3c0af95b..97af9ccc83 100644 --- a/packages/server/src/sdk/app/permissions/index.ts +++ b/packages/server/src/sdk/app/permissions/index.ts @@ -185,6 +185,26 @@ export async function updatePermissionOnRole( }) } +export async function setPermissions( + resourceId: string, + { + writeRole, + readRole, + }: { + writeRole: string + readRole: string + } +) { + await updatePermissionOnRole( + { roleId: writeRole, resourceId, level: PermissionLevel.WRITE }, + PermissionUpdateType.ADD + ) + await updatePermissionOnRole( + { roleId: readRole, resourceId, level: PermissionLevel.READ }, + PermissionUpdateType.ADD + ) +} + // utility function to stop this repetition - permissions always stored under roles export async function getAllDBRoles(db: Database) { const body = await db.allDocs( diff --git a/packages/server/src/sdk/app/rows/external.ts b/packages/server/src/sdk/app/rows/external.ts index 060ef3738a..f7cc38bf0a 100644 --- a/packages/server/src/sdk/app/rows/external.ts +++ b/packages/server/src/sdk/app/rows/external.ts @@ -15,6 +15,7 @@ import { } from "../../../utilities/rowProcessor" import cloneDeep from "lodash/fp/cloneDeep" import { tryExtractingTableAndViewId } from "./utils" +import { helpers } from "@budibase/shared-core" export async function getRow( sourceId: string | Table | ViewV2, @@ -54,6 +55,10 @@ export async function save( source = await sdk.tables.getTable(tableId) } + if (sdk.views.isView(source) && helpers.views.isCalculationView(source)) { + throw new HTTPError("Cannot insert rows through a calculation view", 400) + } + const row = await inputProcessing(userId, cloneDeep(source), inputs) const validateResult = await sdk.rows.utils.validate({ diff --git a/packages/server/src/sdk/app/rows/internal.ts b/packages/server/src/sdk/app/rows/internal.ts index 9306609132..bd03b77f94 100644 --- a/packages/server/src/sdk/app/rows/internal.ts +++ b/packages/server/src/sdk/app/rows/internal.ts @@ -1,4 +1,4 @@ -import { context, db } from "@budibase/backend-core" +import { context, db, HTTPError } from "@budibase/backend-core" import { Row, Table, ViewV2 } from "@budibase/types" import sdk from "../../../sdk" import { finaliseRow } from "../../../api/controllers/row/staticFormula" @@ -10,6 +10,7 @@ import * as linkRows from "../../../db/linkedRows" import { InternalTables } from "../../../db/utils" import { getFullUser } from "../../../utilities/users" import { getSource, tryExtractingTableAndViewId } from "./utils" +import { helpers } from "@budibase/shared-core" export async function save( tableOrViewId: string, @@ -29,6 +30,10 @@ export async function save( table = source } + if (sdk.views.isView(source) && helpers.views.isCalculationView(source)) { + throw new HTTPError("Cannot insert rows through a calculation view", 400) + } + if (!inputs._rev && !inputs._id) { inputs._id = db.generateRowID(inputs.tableId) } diff --git a/packages/server/src/sdk/app/rows/search/internal/internal.ts b/packages/server/src/sdk/app/rows/search/internal/internal.ts index 6617fc376c..0ca10e82d4 100644 --- a/packages/server/src/sdk/app/rows/search/internal/internal.ts +++ b/packages/server/src/sdk/app/rows/search/internal/internal.ts @@ -62,10 +62,10 @@ export async function exportRows( ).rows.map(row => row.doc!) result = await outputProcessing(table, response) - } else if (query) { + } else { let searchResponse = await sdk.rows.search({ tableId, - query, + query: query || {}, sort, sortOrder, }) diff --git a/packages/server/src/sdk/app/rows/search/tests/search.spec.ts b/packages/server/src/sdk/app/rows/search/tests/search.spec.ts index e7fd095865..4d8a6b6d69 100644 --- a/packages/server/src/sdk/app/rows/search/tests/search.spec.ts +++ b/packages/server/src/sdk/app/rows/search/tests/search.spec.ts @@ -10,10 +10,7 @@ import { import TestConfiguration from "../../../../../tests/utilities/TestConfiguration" import { search } from "../../../../../sdk/app/rows/search" import { generator } from "@budibase/backend-core/tests" -import { - withEnv as withCoreEnv, - setEnv as setCoreEnv, -} from "@budibase/backend-core" +import { features } from "@budibase/backend-core" import { DatabaseName, getDatasource, @@ -41,19 +38,13 @@ describe.each([ let table: Table beforeAll(async () => { - await withCoreEnv({ TENANT_FEATURE_FLAGS: isSqs ? "*:SQS" : "" }, () => + await features.testutils.withFeatureFlags("*", { SQS: isSqs }, () => config.init() ) - if (isLucene) { - envCleanup = setCoreEnv({ - TENANT_FEATURE_FLAGS: "*:!SQS", - }) - } else if (isSqs) { - envCleanup = setCoreEnv({ - TENANT_FEATURE_FLAGS: "*:SQS", - }) - } + envCleanup = features.testutils.setFeatureFlags("*", { + SQS: isSqs, + }) if (dsProvider) { datasource = await config.createDatasource({ diff --git a/packages/server/src/sdk/app/tables/create.ts b/packages/server/src/sdk/app/tables/create.ts new file mode 100644 index 0000000000..0b15cdb15a --- /dev/null +++ b/packages/server/src/sdk/app/tables/create.ts @@ -0,0 +1,27 @@ +import { Row, Table } from "@budibase/types" + +import * as external from "./external" +import * as internal from "./internal" +import { isExternal } from "./utils" +import { setPermissions } from "../permissions" +import { roles } from "@budibase/backend-core" + +export async function create( + table: Omit, + rows?: Row[], + userId?: string +): Promise { + let createdTable: Table + if (isExternal({ table })) { + createdTable = await external.create(table) + } else { + createdTable = await internal.create(table, rows, userId) + } + + await setPermissions(createdTable._id!, { + writeRole: roles.BUILTIN_ROLE_IDS.ADMIN, + readRole: roles.BUILTIN_ROLE_IDS.ADMIN, + }) + + return createdTable +} diff --git a/packages/server/src/sdk/app/tables/external/index.ts b/packages/server/src/sdk/app/tables/external/index.ts index e374e70c87..941d193b94 100644 --- a/packages/server/src/sdk/app/tables/external/index.ts +++ b/packages/server/src/sdk/app/tables/external/index.ts @@ -8,8 +8,11 @@ import { ViewV2, AutoFieldSubType, } from "@budibase/types" -import { context } from "@budibase/backend-core" -import { buildExternalTableId } from "../../../../integrations/utils" +import { context, HTTPError } from "@budibase/backend-core" +import { + breakExternalTableId, + buildExternalTableId, +} from "../../../../integrations/utils" import { foreignKeyStructure, hasTypeChanged, @@ -86,6 +89,35 @@ function validate(table: Table, oldTable?: Table) { } } +function getDatasourceId(table: Table) { + if (!table) { + throw new Error("No table supplied") + } + if (table.sourceId) { + return table.sourceId + } + if (!table._id) { + throw new Error("No table ID supplied") + } + return breakExternalTableId(table._id).datasourceId +} + +export async function create(table: Omit) { + const datasourceId = getDatasourceId(table) + + const tableToCreate = { ...table, created: true } + try { + const result = await save(datasourceId!, tableToCreate) + return result.table + } catch (err: any) { + if (err instanceof Error) { + throw new HTTPError(err.message, 400) + } else { + throw new HTTPError(err?.message || err, err.status || 500) + } + } +} + export async function save( datasourceId: string, update: Table, diff --git a/packages/server/src/sdk/app/tables/index.ts b/packages/server/src/sdk/app/tables/index.ts index fcf7051e7c..326b2e1456 100644 --- a/packages/server/src/sdk/app/tables/index.ts +++ b/packages/server/src/sdk/app/tables/index.ts @@ -1,5 +1,6 @@ import { populateExternalTableSchemas } from "./validation" import * as getters from "./getters" +import * as create from "./create" import * as updates from "./update" import * as utils from "./utils" import { migrate } from "./migration" @@ -7,6 +8,7 @@ import * as sqs from "./internal/sqs" export default { populateExternalTableSchemas, + ...create, ...updates, ...getters, ...utils, diff --git a/packages/server/src/sdk/app/tables/internal/index.ts b/packages/server/src/sdk/app/tables/internal/index.ts index c0beed0db8..fbcbed03dc 100644 --- a/packages/server/src/sdk/app/tables/internal/index.ts +++ b/packages/server/src/sdk/app/tables/internal/index.ts @@ -5,7 +5,7 @@ import { ViewStatisticsSchema, ViewV2, Row, - ContextUser, + TableSourceType, } from "@budibase/types" import { hasTypeChanged, @@ -16,18 +16,56 @@ import { EventType, updateLinks } from "../../../../db/linkedRows" import { cloneDeep } from "lodash/fp" import isEqual from "lodash/isEqual" import { runStaticFormulaChecks } from "../../../../api/controllers/table/bulkFormula" -import { context } from "@budibase/backend-core" +import { context, HTTPError } from "@budibase/backend-core" import { findDuplicateInternalColumns } from "@budibase/shared-core" import { getTable } from "../getters" import { checkAutoColumns } from "./utils" import * as viewsSdk from "../../views" -import { getRowParams } from "../../../../db/utils" +import { generateTableID, getRowParams } from "../../../../db/utils" import { quotas } from "@budibase/pro" +export async function create( + table: Omit, + rows?: Row[], + userId?: string +) { + const tableId = generateTableID() + + let tableToSave: Table = { + _id: tableId, + ...table, + // Ensure these fields are populated, even if not sent in the request + type: table.type || "table", + sourceType: TableSourceType.INTERNAL, + } + + const isImport = !!rows + + if (!tableToSave.views) { + tableToSave.views = {} + } + + try { + const { table } = await save(tableToSave, { + userId, + rowsToImport: rows, + isImport, + }) + + return table + } catch (err: any) { + if (err instanceof Error) { + throw new HTTPError(err.message, 400) + } else { + throw new HTTPError(err.message || err, err.status || 500) + } + } +} + export async function save( table: Table, opts?: { - user?: ContextUser + userId?: string tableId?: string rowsToImport?: Row[] renaming?: RenameColumn @@ -63,7 +101,7 @@ export async function save( // saving a table is a complex operation, involving many different steps, this // has been broken out into a utility to make it more obvious/easier to manipulate const tableSaveFunctions = new TableSaveFunctions({ - user: opts?.user, + userId: opts?.userId, oldTable, importRows: opts?.rowsToImport, }) diff --git a/packages/server/src/sdk/app/views/index.ts b/packages/server/src/sdk/app/views/index.ts index 853b4eb37c..27b3af87f1 100644 --- a/packages/server/src/sdk/app/views/index.ts +++ b/packages/server/src/sdk/app/views/index.ts @@ -1,6 +1,8 @@ import { CalculationType, + canGroupBy, FieldType, + isNumeric, PermissionLevel, RelationSchemaField, RenameColumn, @@ -11,7 +13,7 @@ import { ViewV2ColumnEnriched, ViewV2Enriched, } from "@budibase/types" -import { context, docIds, HTTPError, roles } from "@budibase/backend-core" +import { context, docIds, HTTPError } from "@budibase/backend-core" import { helpers, PROTECTED_EXTERNAL_COLUMNS, @@ -22,7 +24,6 @@ import { isExternalTableID } from "../../../integrations/utils" import * as internal from "./internal" import * as external from "./external" import sdk from "../../../sdk" -import { PermissionUpdateType, updatePermissionOnRole } from "../permissions" function pickApi(tableId: any) { if (isExternalTableID(tableId)) { @@ -101,7 +102,7 @@ async function guardCalculationViewSchema( ) } - if (!isCount && !helpers.schema.isNumeric(targetSchema)) { + if (!isCount && !isNumeric(targetSchema.type)) { throw new HTTPError( `Calculation field "${name}" references field "${schema.field}" which is not a numeric field`, 400 @@ -118,6 +119,13 @@ async function guardCalculationViewSchema( 400 ) } + + if (!canGroupBy(targetSchema.type)) { + throw new HTTPError( + `Grouping by fields of type "${targetSchema.type}" is not supported`, + 400 + ) + } } } @@ -235,24 +243,10 @@ export async function create( // Set permissions to be the same as the table const tablePerms = await sdk.permissions.getResourcePerms(tableId) - const readRole = tablePerms[PermissionLevel.READ]?.role - const writeRole = tablePerms[PermissionLevel.WRITE]?.role - await updatePermissionOnRole( - { - roleId: readRole || roles.BUILTIN_ROLE_IDS.BASIC, - resourceId: view.id, - level: PermissionLevel.READ, - }, - PermissionUpdateType.ADD - ) - await updatePermissionOnRole( - { - roleId: writeRole || roles.BUILTIN_ROLE_IDS.BASIC, - resourceId: view.id, - level: PermissionLevel.WRITE, - }, - PermissionUpdateType.ADD - ) + await sdk.permissions.setPermissions(view.id, { + writeRole: tablePerms[PermissionLevel.WRITE].role, + readRole: tablePerms[PermissionLevel.READ].role, + }) return view } diff --git a/packages/server/src/tests/utilities/api/permission.ts b/packages/server/src/tests/utilities/api/permission.ts index 986796d9a1..b4e641a1be 100644 --- a/packages/server/src/tests/utilities/api/permission.ts +++ b/packages/server/src/tests/utilities/api/permission.ts @@ -1,6 +1,7 @@ import { AddPermissionRequest, AddPermissionResponse, + FetchResourcePermissionInfoResponse, GetResourcePermsResponse, RemovePermissionRequest, RemovePermissionResponse, @@ -26,6 +27,15 @@ export class PermissionAPI extends TestAPI { ) } + fetch = async ( + expectations?: Expectations + ): Promise => { + return await this._get( + `/api/permission`, + { expectations } + ) + } + revoke = async ( request: RemovePermissionRequest, expectations?: Expectations diff --git a/packages/server/src/tests/utilities/api/row.ts b/packages/server/src/tests/utilities/api/row.ts index 6bec59fdf7..52317e142a 100644 --- a/packages/server/src/tests/utilities/api/row.ts +++ b/packages/server/src/tests/utilities/api/row.ts @@ -105,7 +105,7 @@ export class RowAPI extends TestAPI { exportRows = async ( tableId: string, - body: ExportRowsRequest, + body?: ExportRowsRequest, format: RowExportFormat = RowExportFormat.JSON, expectations?: Expectations ) => { diff --git a/packages/server/src/utilities/rowProcessor/tests/outputProcessing.spec.ts b/packages/server/src/utilities/rowProcessor/tests/outputProcessing.spec.ts index b6c2cdb11c..8cbe585d90 100644 --- a/packages/server/src/utilities/rowProcessor/tests/outputProcessing.spec.ts +++ b/packages/server/src/utilities/rowProcessor/tests/outputProcessing.spec.ts @@ -8,7 +8,7 @@ import { } from "@budibase/types" import { outputProcessing } from ".." import { generator, structures } from "@budibase/backend-core/tests" -import { setEnv as setCoreEnv } from "@budibase/backend-core" +import { features } from "@budibase/backend-core" import * as bbReferenceProcessor from "../bbReferenceProcessor" import TestConfiguration from "../../../tests/utilities/TestConfiguration" @@ -21,7 +21,7 @@ jest.mock("../bbReferenceProcessor", (): typeof bbReferenceProcessor => ({ describe("rowProcessor - outputProcessing", () => { const config = new TestConfiguration() - let cleanupEnv: () => void = () => {} + let cleanupFlags: () => void = () => {} beforeAll(async () => { await config.init() @@ -33,11 +33,11 @@ describe("rowProcessor - outputProcessing", () => { beforeEach(() => { jest.resetAllMocks() - cleanupEnv = setCoreEnv({ TENANT_FEATURE_FLAGS: "*SQS" }) + cleanupFlags = features.testutils.setFeatureFlags("*", { SQS: true }) }) afterEach(() => { - cleanupEnv() + cleanupFlags() }) const processOutputBBReferenceMock = diff --git a/packages/server/src/utilities/security.ts b/packages/server/src/utilities/security.ts index 01a3468c9c..4f93c33ee4 100644 --- a/packages/server/src/utilities/security.ts +++ b/packages/server/src/utilities/security.ts @@ -1,5 +1,6 @@ import { permissions, roles } from "@budibase/backend-core" import { DocumentType, VirtualDocumentType } from "../db/utils" +import { getDocumentType, getVirtualDocumentType } from "@budibase/types" export const CURRENTLY_SUPPORTED_LEVELS: string[] = [ permissions.PermissionLevel.WRITE, @@ -8,13 +9,16 @@ export const CURRENTLY_SUPPORTED_LEVELS: string[] = [ ] export function getPermissionType(resourceId: string) { - const docType = Object.values(DocumentType).filter(docType => - resourceId.startsWith(docType) - )[0] - switch (docType as DocumentType | VirtualDocumentType) { + const virtualDocType = getVirtualDocumentType(resourceId) + switch (virtualDocType) { + case VirtualDocumentType.VIEW: + return permissions.PermissionType.TABLE + } + + const docType = getDocumentType(resourceId) + switch (docType) { case DocumentType.TABLE: case DocumentType.ROW: - case VirtualDocumentType.VIEW: return permissions.PermissionType.TABLE case DocumentType.AUTOMATION: return permissions.PermissionType.AUTOMATION @@ -32,22 +36,25 @@ export function getPermissionType(resourceId: string) { /** * works out the basic permissions based on builtin roles for a resource, using its ID */ -export function getBasePermissions(resourceId: string) { +export function getBasePermissions(resourceId: string): Record { const type = getPermissionType(resourceId) - const basePermissions: { [key: string]: string } = {} + const basePermissions: Record = {} for (let [roleId, role] of Object.entries(roles.getBuiltinRoles())) { if (!role.permissionId) { continue } + const perms = permissions.getBuiltinPermissionByID(role.permissionId) if (!perms) { continue } + const typedPermission = perms.permissions.find(perm => perm.type === type) - if ( - typedPermission && - CURRENTLY_SUPPORTED_LEVELS.indexOf(typedPermission.level) !== -1 - ) { + if (!typedPermission) { + continue + } + + if (CURRENTLY_SUPPORTED_LEVELS.includes(typedPermission.level)) { const level = typedPermission.level basePermissions[level] = roles.lowerBuiltinRoleID( basePermissions[level], diff --git a/packages/shared-core/src/filters.ts b/packages/shared-core/src/filters.ts index fce355f7b9..95c500c43e 100644 --- a/packages/shared-core/src/filters.ts +++ b/packages/shared-core/src/filters.ts @@ -642,19 +642,19 @@ export function fixupFilterArrays(filters: SearchFilters) { return filters } -export function search( - docs: Record[], - query: RowSearchParams -): SearchResponse> { +export function search>( + docs: T[], + query: Omit +): SearchResponse { let result = runQuery(docs, query.query) if (query.sort) { result = sort(result, query.sort, query.sortOrder || SortOrder.ASCENDING) } - let totalRows = result.length + const totalRows = result.length if (query.limit) { result = limit(result, query.limit.toString()) } - const response: SearchResponse> = { rows: result } + const response: SearchResponse = { rows: result } if (query.countRows) { response.totalRows = totalRows } diff --git a/packages/shared-core/src/utils.ts b/packages/shared-core/src/utils.ts index 14b3c84425..4847e911e9 100644 --- a/packages/shared-core/src/utils.ts +++ b/packages/shared-core/src/utils.ts @@ -5,6 +5,7 @@ import { SearchFilters, BasicOperator, ArrayOperator, + isLogicalSearchOperator, } from "@budibase/types" import * as Constants from "./constants" import { removeKeyNumbering } from "./filters" @@ -97,10 +98,20 @@ export function isSupportedUserSearch(query: SearchFilters) { { op: BasicOperator.EQUAL, key: "_id" }, { op: ArrayOperator.ONE_OF, key: "_id" }, ] - for (let [key, operation] of Object.entries(query)) { + for (const [key, operation] of Object.entries(query)) { if (typeof operation !== "object") { return false } + + if (isLogicalSearchOperator(key)) { + for (const condition of query[key]!.conditions) { + if (!isSupportedUserSearch(condition)) { + return false + } + } + return true + } + const fields = Object.keys(operation || {}) // this filter doesn't contain options - ignore if (fields.length === 0) { diff --git a/packages/types/src/api/web/app/permission.ts b/packages/types/src/api/web/app/permission.ts index bead2a4279..b40310f21c 100644 --- a/packages/types/src/api/web/app/permission.ts +++ b/packages/types/src/api/web/app/permission.ts @@ -1,5 +1,9 @@ import { PermissionLevel } from "../../../sdk" +export interface FetchResourcePermissionInfoResponse { + [key: string]: Record +} + export interface ResourcePermissionInfo { role: string permissionType: string @@ -21,7 +25,7 @@ export interface AddedPermission { reason?: string } -export type AddPermissionResponse = AddedPermission[] +export interface AddPermissionResponse {} export interface AddPermissionRequest { roleId: string @@ -30,4 +34,4 @@ export interface AddPermissionRequest { } export interface RemovePermissionRequest extends AddPermissionRequest {} -export interface RemovePermissionResponse extends AddPermissionResponse {} +export interface RemovePermissionResponse {} diff --git a/packages/types/src/documents/app/row.ts b/packages/types/src/documents/app/row.ts index 42440b2988..673d481e3c 100644 --- a/packages/types/src/documents/app/row.ts +++ b/packages/types/src/documents/app/row.ts @@ -127,6 +127,26 @@ export const JsonTypes = [ FieldType.ARRAY, ] +export const NumericTypes = [FieldType.NUMBER, FieldType.BIGINT] + +export function isNumeric(type: FieldType) { + return NumericTypes.includes(type) +} + +export const GroupByTypes = [ + FieldType.STRING, + FieldType.LONGFORM, + FieldType.OPTIONS, + FieldType.NUMBER, + FieldType.BOOLEAN, + FieldType.DATETIME, + FieldType.BIGINT, +] + +export function canGroupBy(type: FieldType) { + return GroupByTypes.includes(type) +} + export interface RowAttachment { size: number name: string diff --git a/packages/types/src/documents/document.ts b/packages/types/src/documents/document.ts index f5facfae9d..2f0da1081a 100644 --- a/packages/types/src/documents/document.ts +++ b/packages/types/src/documents/document.ts @@ -42,6 +42,17 @@ export enum DocumentType { ROW_ACTIONS = "ra", } +// Because DocumentTypes can overlap, we need to make sure that we search +// longest first to ensure we get the correct type. +const sortedDocumentTypes = Object.values(DocumentType).sort( + (a, b) => b.length - a.length // descending +) +export function getDocumentType(id: string): DocumentType | undefined { + return sortedDocumentTypes.find(docType => + id.startsWith(`${docType}${SEPARATOR}`) + ) +} + // these are the core documents that make up the data, design // and automation sections of an app. This excludes any internal // rows as we shouldn't import data. @@ -72,6 +83,19 @@ export enum VirtualDocumentType { ROW_ACTION = "row_action", } +// Because VirtualDocumentTypes can overlap, we need to make sure that we search +// longest first to ensure we get the correct type. +const sortedVirtualDocumentTypes = Object.values(VirtualDocumentType).sort( + (a, b) => b.length - a.length // descending +) +export function getVirtualDocumentType( + id: string +): VirtualDocumentType | undefined { + return sortedVirtualDocumentTypes.find(docType => + id.startsWith(`${docType}${SEPARATOR}`) + ) +} + export interface Document { _id?: string _rev?: string diff --git a/packages/types/src/sdk/db.ts b/packages/types/src/sdk/db.ts index a081f4f1a2..49baaa5bb1 100644 --- a/packages/types/src/sdk/db.ts +++ b/packages/types/src/sdk/db.ts @@ -133,7 +133,7 @@ export interface Database { exists(docId: string): Promise getMultiple( ids: string[], - opts?: { allowMissing?: boolean } + opts?: { allowMissing?: boolean; excludeDocs?: boolean } ): Promise remove(idOrDoc: Document): Promise remove(idOrDoc: string, rev?: string): Promise diff --git a/packages/worker/src/api/routes/global/tests/auditLogs.spec.ts b/packages/worker/src/api/routes/global/tests/auditLogs.spec.ts index b09c27762d..b540836583 100644 --- a/packages/worker/src/api/routes/global/tests/auditLogs.spec.ts +++ b/packages/worker/src/api/routes/global/tests/auditLogs.spec.ts @@ -1,5 +1,5 @@ import { mocks, structures } from "@budibase/backend-core/tests" -import { context, events, setEnv as setCoreEnv } from "@budibase/backend-core" +import { context, events, features } from "@budibase/backend-core" import { Event, IdentityType } from "@budibase/types" import { TestConfiguration } from "../../../../tests" @@ -17,11 +17,9 @@ describe.each(["lucene", "sql"])("/api/global/auditlogs (%s)", method => { let envCleanup: (() => void) | undefined beforeAll(async () => { - if (method === "lucene") { - envCleanup = setCoreEnv({ TENANT_FEATURE_FLAGS: "*:!SQS" }) - } else if (method === "sql") { - envCleanup = setCoreEnv({ TENANT_FEATURE_FLAGS: "*:SQS" }) - } + envCleanup = features.testutils.setFeatureFlags("*", { + SQS: method === "sql", + }) await config.beforeAll() }) diff --git a/packages/worker/src/api/routes/global/tests/users.spec.ts b/packages/worker/src/api/routes/global/tests/users.spec.ts index a654c42359..c8e71f7eb4 100644 --- a/packages/worker/src/api/routes/global/tests/users.spec.ts +++ b/packages/worker/src/api/routes/global/tests/users.spec.ts @@ -741,6 +741,25 @@ describe("/api/global/users", () => { it("should throw an error if public query performed", async () => { await config.api.users.searchUsers({}, { status: 403, noHeaders: true }) }) + + it("should be able to search using logical conditions", async () => { + const user = await config.createUser() + const response = await config.api.users.searchUsers({ + query: { + $and: { + conditions: [ + { + $and: { + conditions: [{ string: { email: user.email } }], + }, + }, + ], + }, + }, + }) + expect(response.body.data.length).toBe(1) + expect(response.body.data[0].email).toBe(user.email) + }) }) describe("DELETE /api/global/users/:userId", () => { diff --git a/packages/worker/src/index.ts b/packages/worker/src/index.ts index 79e6f4493d..fb6a97a844 100644 --- a/packages/worker/src/index.ts +++ b/packages/worker/src/index.ts @@ -46,6 +46,7 @@ bootstrap() const app: Application = new Application() app.keys = ["secret", "key"] +app.proxy = true // set up top level koa middleware app.use(handleScimBody) @@ -54,6 +55,7 @@ app.use(koaBody({ multipart: true })) app.use(koaSession(app)) app.use(middleware.correlation) app.use(middleware.pino) +app.use(middleware.ip) app.use(userAgent) // authentication