diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml index a17ca352cc..1b236caab5 100644 --- a/.github/workflows/budibase_ci.yml +++ b/.github/workflows/budibase_ci.yml @@ -147,7 +147,7 @@ jobs: fi test-server: - runs-on: budi-tubby-tornado-quad-core-150gb + runs-on: budi-tubby-tornado-quad-core-300gb steps: - name: Checkout repo uses: actions/checkout@v4 diff --git a/README.md b/README.md index 4979f0ee8e..64492b97e4 100644 --- a/README.md +++ b/README.md @@ -116,7 +116,7 @@ As with anything that we build in Budibase, our new public API is simple to use, You can learn more about the Budibase API at the following places: - [General documentation](https://docs.budibase.com/docs/public-api): Learn how to get your API key, how to use spec, and how to use Postman -- [Interactive API documentation](https://docs.budibase.com/reference/post_applications) : Learn how to interact with the API +- [Interactive API documentation](https://docs.budibase.com/reference/appcreate) : Learn how to interact with the API

diff --git a/i18n/README.es.md b/i18n/README.es.md index a7d1112914..ee92ca24d5 100644 --- a/i18n/README.es.md +++ b/i18n/README.es.md @@ -144,7 +144,7 @@ del sistema. Budibase API ofrece: Puedes aprender mas acerca de Budibase API en los siguientes documentos: - [Documentacion general](https://docs.budibase.com/docs/public-api) : Como optener tu clave para la API, usar Insomnia y Postman -- [API Interactiva](https://docs.budibase.com/reference/post_applications) : Aprende como trabajar con la API +- [API Interactiva](https://docs.budibase.com/reference/appcreate) : Aprende como trabajar con la API #### Guias diff --git a/lerna.json b/lerna.json index d7d2bdce39..1b6574c7df 100644 --- a/lerna.json +++ b/lerna.json @@ -1,6 +1,6 @@ { "$schema": "node_modules/lerna/schemas/lerna-schema.json", - "version": "2.29.28", + "version": "2.29.30", "npmClient": "yarn", "packages": [ "packages/*", diff --git a/packages/backend-core/src/features/index.ts b/packages/backend-core/src/features/index.ts index ad517082de..d7f7c76436 100644 --- a/packages/backend-core/src/features/index.ts +++ b/packages/backend-core/src/features/index.ts @@ -1,79 +1,108 @@ import env from "../environment" import * as context from "../context" +import { cloneDeep } from "lodash" -export * from "./installation" - -/** - * Read the TENANT_FEATURE_FLAGS env var and return an array of features flags for each tenant. - * The env var is formatted as: - * tenant1:feature1:feature2,tenant2:feature1 - */ -export function buildFeatureFlags() { - if (!env.TENANT_FEATURE_FLAGS) { - return +class Flag { + static withDefault(value: T) { + return new Flag(value) } - const tenantFeatureFlags: Record = {} + private constructor(public defaultValue: T) {} +} - env.TENANT_FEATURE_FLAGS.split(",").forEach(tenantToFeatures => { - const [tenantId, ...features] = tenantToFeatures.split(":") +// This is the primary source of truth for feature flags. If you want to add a +// new flag, add it here and use the `fetch` and `get` functions to access it. +// All of the machinery in this file is to make sure that flags have their +// default values set correctly and their types flow through the system. +const FLAGS = { + LICENSING: Flag.withDefault(false), + GOOGLE_SHEETS: Flag.withDefault(false), + USER_GROUPS: Flag.withDefault(false), + ONBOARDING_TOUR: Flag.withDefault(false), +} - features.forEach(feature => { - if (!tenantFeatureFlags[tenantId]) { - tenantFeatureFlags[tenantId] = [] +const DEFAULTS = Object.keys(FLAGS).reduce((acc, key) => { + const typedKey = key as keyof typeof FLAGS + // @ts-ignore + acc[typedKey] = FLAGS[typedKey].defaultValue + return acc +}, {} as Flags) + +type UnwrapFlag = F extends Flag ? U : never +export type Flags = { + [K in keyof typeof FLAGS]: UnwrapFlag<(typeof FLAGS)[K]> +} + +// Exported for use in tests, should not be used outside of this file. +export function defaultFlags(): Flags { + return cloneDeep(DEFAULTS) +} + +function isFlagName(name: string): name is keyof Flags { + return FLAGS[name as keyof typeof FLAGS] !== undefined +} + +/** + * Reads the TENANT_FEATURE_FLAGS environment variable and returns a Flags object + * populated with the flags for the current tenant, filling in the default values + * if the flag is not set. + * + * Check the tests for examples of how TENANT_FEATURE_FLAGS should be formatted. + * + * In future we plan to add more ways of setting feature flags, e.g. PostHog, and + * they will be accessed through this function as well. + */ +export async function fetch(): Promise { + const currentTenantId = context.getTenantId() + const flags = defaultFlags() + + const split = (env.TENANT_FEATURE_FLAGS || "") + .split(",") + .map(x => x.split(":")) + for (const [tenantId, ...features] of split) { + if (!tenantId || (tenantId !== "*" && tenantId !== currentTenantId)) { + continue + } + + for (let feature of features) { + let value = true + if (feature.startsWith("!")) { + feature = feature.slice(1) + value = false } - tenantFeatureFlags[tenantId].push(feature) - }) - }) - return tenantFeatureFlags -} + if (!isFlagName(feature)) { + throw new Error(`Feature: ${feature} is not an allowed option`) + } -export function isEnabled(featureFlag: string) { - const tenantId = context.getTenantId() - const flags = getTenantFeatureFlags(tenantId) - return flags.includes(featureFlag) -} + if (typeof flags[feature] !== "boolean") { + throw new Error(`Feature: ${feature} is not a boolean`) + } -export function getTenantFeatureFlags(tenantId: string) { - let flags: string[] = [] - const envFlags = buildFeatureFlags() - if (envFlags) { - const globalFlags = envFlags["*"] - const tenantFlags = envFlags[tenantId] || [] - - // Explicitly exclude tenants from global features if required. - // Prefix the tenant flag with '!' - const tenantOverrides = tenantFlags.reduce( - (acc: string[], flag: string) => { - if (flag.startsWith("!")) { - let stripped = flag.substring(1) - acc.push(stripped) - } - return acc - }, - [] - ) - - if (globalFlags) { - flags.push(...globalFlags) + // @ts-ignore + flags[feature] = value } - if (tenantFlags.length) { - flags.push(...tenantFlags) - } - - // Purge any tenant specific overrides - flags = flags.filter(flag => { - return tenantOverrides.indexOf(flag) == -1 && !flag.startsWith("!") - }) } return flags } -export enum TenantFeatureFlag { - LICENSING = "LICENSING", - GOOGLE_SHEETS = "GOOGLE_SHEETS", - USER_GROUPS = "USER_GROUPS", - ONBOARDING_TOUR = "ONBOARDING_TOUR", +// Gets a single feature flag value. This is a convenience function for +// `fetch().then(flags => flags[name])`. +export async function get(name: K): Promise { + const flags = await fetch() + return flags[name] +} + +type BooleanFlags = { + [K in keyof typeof FLAGS]: (typeof FLAGS)[K] extends Flag ? K : never +}[keyof typeof FLAGS] + +// Convenience function for boolean flag values. This makes callsites more +// readable for boolean flags. +export async function isEnabled( + name: K +): Promise { + const flags = await fetch() + return flags[name] } diff --git a/packages/backend-core/src/features/installation.ts b/packages/backend-core/src/features/installation.ts deleted file mode 100644 index defc8bf987..0000000000 --- a/packages/backend-core/src/features/installation.ts +++ /dev/null @@ -1,17 +0,0 @@ -export function processFeatureEnvVar( - fullList: string[], - featureList?: string -) { - let list - if (!featureList) { - list = fullList - } else { - list = featureList.split(",") - } - for (let feature of list) { - if (!fullList.includes(feature)) { - throw new Error(`Feature: ${feature} is not an allowed option`) - } - } - return list as unknown as T[] -} diff --git a/packages/backend-core/src/features/tests/featureFlags.spec.ts b/packages/backend-core/src/features/tests/featureFlags.spec.ts deleted file mode 100644 index 1b68959329..0000000000 --- a/packages/backend-core/src/features/tests/featureFlags.spec.ts +++ /dev/null @@ -1,85 +0,0 @@ -import { - TenantFeatureFlag, - buildFeatureFlags, - getTenantFeatureFlags, -} from "../" -import env from "../../environment" - -const { ONBOARDING_TOUR, LICENSING, USER_GROUPS } = TenantFeatureFlag - -describe("featureFlags", () => { - beforeEach(() => { - env._set("TENANT_FEATURE_FLAGS", "") - }) - - it("Should return no flags when the TENANT_FEATURE_FLAG is empty", async () => { - let features = buildFeatureFlags() - expect(features).toBeUndefined() - }) - - it("Should generate a map of global and named tenant feature flags from the env value", async () => { - env._set( - "TENANT_FEATURE_FLAGS", - `*:${ONBOARDING_TOUR},tenant1:!${ONBOARDING_TOUR},tenant2:${USER_GROUPS},tenant1:${LICENSING}` - ) - - const parsedFlags: Record = { - "*": [ONBOARDING_TOUR], - tenant1: [`!${ONBOARDING_TOUR}`, LICENSING], - tenant2: [USER_GROUPS], - } - - let features = buildFeatureFlags() - - expect(features).toBeDefined() - expect(features).toEqual(parsedFlags) - }) - - it("Should add feature flag flag only to explicitly configured tenant", async () => { - env._set( - "TENANT_FEATURE_FLAGS", - `*:${LICENSING},*:${USER_GROUPS},tenant1:${ONBOARDING_TOUR}` - ) - - let tenant1Flags = getTenantFeatureFlags("tenant1") - let tenant2Flags = getTenantFeatureFlags("tenant2") - - expect(tenant1Flags).toBeDefined() - expect(tenant1Flags).toEqual([LICENSING, USER_GROUPS, ONBOARDING_TOUR]) - - expect(tenant2Flags).toBeDefined() - expect(tenant2Flags).toEqual([LICENSING, USER_GROUPS]) - }) -}) - -it("Should exclude tenant1 from global feature flag", async () => { - env._set( - "TENANT_FEATURE_FLAGS", - `*:${LICENSING},*:${ONBOARDING_TOUR},tenant1:!${ONBOARDING_TOUR}` - ) - - let tenant1Flags = getTenantFeatureFlags("tenant1") - let tenant2Flags = getTenantFeatureFlags("tenant2") - - expect(tenant1Flags).toBeDefined() - expect(tenant1Flags).toEqual([LICENSING]) - - expect(tenant2Flags).toBeDefined() - expect(tenant2Flags).toEqual([LICENSING, ONBOARDING_TOUR]) -}) - -it("Should explicitly add flags to configured tenants only", async () => { - env._set( - "TENANT_FEATURE_FLAGS", - `tenant1:${ONBOARDING_TOUR},tenant1:${LICENSING},tenant2:${LICENSING}` - ) - - let tenant1Flags = getTenantFeatureFlags("tenant1") - let tenant2Flags = getTenantFeatureFlags("tenant2") - - expect(tenant1Flags).toBeDefined() - expect(tenant1Flags).toEqual([ONBOARDING_TOUR, LICENSING]) - - expect(tenant2Flags).toBeDefined() - expect(tenant2Flags).toEqual([LICENSING]) -}) diff --git a/packages/backend-core/src/features/tests/features.spec.ts b/packages/backend-core/src/features/tests/features.spec.ts new file mode 100644 index 0000000000..83a89940b8 --- /dev/null +++ b/packages/backend-core/src/features/tests/features.spec.ts @@ -0,0 +1,86 @@ +import { defaultFlags, fetch, get, Flags } from "../" +import { context } from "../.." +import env from "../../environment" + +async function withFlags(flags: string, f: () => T): Promise { + const oldFlags = env.TENANT_FEATURE_FLAGS + env._set("TENANT_FEATURE_FLAGS", flags) + try { + return await f() + } finally { + env._set("TENANT_FEATURE_FLAGS", oldFlags) + } +} + +describe("feature flags", () => { + interface TestCase { + tenant: string + flags: string + expected: Partial + } + + it.each([ + { + tenant: "tenant1", + flags: "tenant1:ONBOARDING_TOUR", + expected: { ONBOARDING_TOUR: true }, + }, + { + tenant: "tenant1", + flags: "tenant1:!ONBOARDING_TOUR", + expected: { ONBOARDING_TOUR: false }, + }, + { + tenant: "tenant1", + flags: "*:ONBOARDING_TOUR", + expected: { ONBOARDING_TOUR: true }, + }, + { + tenant: "tenant1", + flags: "tenant2:ONBOARDING_TOUR", + expected: { ONBOARDING_TOUR: false }, + }, + { + tenant: "tenant1", + flags: "", + expected: defaultFlags(), + }, + ])( + 'should find flags $expected for $tenant with string "$flags"', + ({ tenant, flags, expected }) => + context.doInTenant(tenant, () => + withFlags(flags, async () => { + const flags = await fetch() + expect(flags).toMatchObject(expected) + + for (const [key, expectedValue] of Object.entries(expected)) { + const value = await get(key as keyof Flags) + expect(value).toBe(expectedValue) + } + }) + ) + ) + + interface FailedTestCase { + tenant: string + flags: string + expected: string | RegExp + } + + it.each([ + { + tenant: "tenant1", + flags: "tenant1:ONBOARDING_TOUR,tenant1:FOO", + expected: "Feature: FOO is not an allowed option", + }, + ])( + "should fail with message \"$expected\" for $tenant with string '$flags'", + async ({ tenant, flags, expected }) => { + context.doInTenant(tenant, () => + withFlags(flags, async () => { + await expect(fetch()).rejects.toThrow(expected) + }) + ) + } + ) +}) diff --git a/packages/backend-core/src/index.ts b/packages/backend-core/src/index.ts index 30c5fbdd7a..a14a344655 100644 --- a/packages/backend-core/src/index.ts +++ b/packages/backend-core/src/index.ts @@ -7,8 +7,7 @@ export * as roles from "./security/roles" export * as permissions from "./security/permissions" export * as accounts from "./accounts" export * as installation from "./installation" -export * as featureFlags from "./features" -export * as features from "./features/installation" +export * as features from "./features" export * as sessions from "./security/sessions" export * as platform from "./platform" export * as auth from "./auth" diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index 8ab8fea20e..ebae09e156 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -463,6 +463,24 @@ class InternalBuilder { } } + if (filters.$and) { + const { $and } = filters + query = query.where(x => { + for (const condition of $and.conditions) { + x = this.addFilters(x, condition, opts) + } + }) + } + + if (filters.$or) { + const { $or } = filters + query = query.where(x => { + for (const condition of $or.conditions) { + x = this.addFilters(x, { ...condition, allOr: true }, opts) + } + }) + } + if (filters.oneOf) { const fnc = allOr ? "orWhereIn" : "whereIn" iterate( diff --git a/packages/builder/src/helpers/featureFlags.js b/packages/builder/src/helpers/featureFlags.js index 462dae8c54..fe30fb9980 100644 --- a/packages/builder/src/helpers/featureFlags.js +++ b/packages/builder/src/helpers/featureFlags.js @@ -5,9 +5,10 @@ export const TENANT_FEATURE_FLAGS = { LICENSING: "LICENSING", USER_GROUPS: "USER_GROUPS", ONBOARDING_TOUR: "ONBOARDING_TOUR", + GOOGLE_SHEETS: "GOOGLE_SHEETS", } export const isEnabled = featureFlag => { const user = get(auth).user - return !!user?.featureFlags?.includes(featureFlag) + return !!user?.flags?.[featureFlag] } diff --git a/packages/pro b/packages/pro index 7dbe323aec..62ef0e2d6e 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 7dbe323aec724ae6336b13c06aaefa4a89837edf +Subproject commit 62ef0e2d6e83522b6732fb3c61338de303f06ff0 diff --git a/packages/server/package.json b/packages/server/package.json index 48ab0685d9..b835477489 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -80,7 +80,7 @@ "dotenv": "8.2.0", "form-data": "4.0.0", "global-agent": "3.0.0", - "google-spreadsheet": "npm:@budibase/google-spreadsheet@4.1.2", + "google-spreadsheet": "npm:@budibase/google-spreadsheet@4.1.3", "ioredis": "5.3.2", "isolated-vm": "^4.7.2", "jimp": "0.22.12", diff --git a/packages/server/src/api/controllers/row/views.ts b/packages/server/src/api/controllers/row/views.ts index 12e76155bc..c38a415aa2 100644 --- a/packages/server/src/api/controllers/row/views.ts +++ b/packages/server/src/api/controllers/row/views.ts @@ -6,11 +6,13 @@ import { RequiredKeys, RowSearchParams, SearchFilterKey, + LogicalOperator, } from "@budibase/types" import { dataFilters } from "@budibase/shared-core" import sdk from "../../../sdk" import { db, context } from "@budibase/backend-core" import { enrichSearchContext } from "./utils" +import { isExternalTableID } from "../../../integrations/utils" export async function searchView( ctx: UserCtx @@ -35,25 +37,33 @@ export async function searchView( // that could let users find rows they should not be allowed to access. let query = dataFilters.buildQuery(view.query || []) if (body.query) { - // Extract existing fields - const existingFields = - view.query - ?.filter(filter => filter.field) - .map(filter => db.removeKeyNumbering(filter.field)) || [] - // Delete extraneous search params that cannot be overridden delete body.query.allOr delete body.query.onEmptyFilter - // Carry over filters for unused fields - Object.keys(body.query).forEach(key => { - const operator = key as SearchFilterKey - Object.keys(body.query[operator] || {}).forEach(field => { - if (!existingFields.includes(db.removeKeyNumbering(field))) { - query[operator]![field] = body.query[operator]![field] - } + if (!isExternalTableID(view.tableId) && !db.isSqsEnabledForTenant()) { + // Extract existing fields + const existingFields = + view.query + ?.filter(filter => filter.field) + .map(filter => db.removeKeyNumbering(filter.field)) || [] + + // Carry over filters for unused fields + Object.keys(body.query).forEach(key => { + const operator = key as Exclude + Object.keys(body.query[operator] || {}).forEach(field => { + if (!existingFields.includes(db.removeKeyNumbering(field))) { + query[operator]![field] = body.query[operator]![field] + } + }) }) - }) + } else { + query = { + $and: { + conditions: [query, body.query], + }, + } + } } await context.ensureSnippetContext(true) diff --git a/packages/server/src/api/routes/tests/datasource.spec.ts b/packages/server/src/api/routes/tests/datasource.spec.ts index 4ca766247b..237133e639 100644 --- a/packages/server/src/api/routes/tests/datasource.spec.ts +++ b/packages/server/src/api/routes/tests/datasource.spec.ts @@ -17,9 +17,14 @@ import { SupportedSqlTypes, JsonFieldSubType, } from "@budibase/types" -import { DatabaseName, getDatasource } from "../../../integrations/tests/utils" +import { + DatabaseName, + getDatasource, + knexClient, +} from "../../../integrations/tests/utils" import { tableForDatasource } from "../../../tests/utilities/structures" import nock from "nock" +import { Knex } from "knex" describe("/datasources", () => { const config = setup.getConfig() @@ -164,11 +169,15 @@ describe("/datasources", () => { [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], + [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)], ])("%s", (_, dsProvider) => { let rawDatasource: Datasource + let client: Knex + beforeEach(async () => { rawDatasource = await dsProvider datasource = await config.api.datasource.create(rawDatasource) + client = await knexClient(rawDatasource) }) describe("get", () => { @@ -285,9 +294,6 @@ describe("/datasources", () => { [FieldType.STRING]: { name: stringName, type: FieldType.STRING, - constraints: { - presence: true, - }, }, [FieldType.LONGFORM]: { name: "longform", @@ -381,10 +387,6 @@ describe("/datasources", () => { ), schema: Object.entries(table.schema).reduce( (acc, [fieldName, field]) => { - // the constraint will be unset - as the DB doesn't recognise it as not null - if (fieldName === stringName) { - field.constraints = {} - } acc[fieldName] = expect.objectContaining({ ...field, }) @@ -441,20 +443,49 @@ describe("/datasources", () => { }) describe("info", () => { - it("should fetch information about postgres datasource", async () => { - const table = await config.api.table.save( - tableForDatasource(datasource, { - schema: { - name: { - name: "name", - type: FieldType.STRING, - }, - }, - }) - ) + it("should fetch information about a datasource with a single table", async () => { + const existingTableNames = ( + await config.api.datasource.info(datasource) + ).tableNames + + const tableName = generator.guid() + await client.schema.createTable(tableName, table => { + table.increments("id").primary() + table.string("name") + }) const info = await config.api.datasource.info(datasource) - expect(info.tableNames).toContain(table.name) + expect(info.tableNames).toEqual( + expect.arrayContaining([tableName, ...existingTableNames]) + ) + expect(info.tableNames).toHaveLength(existingTableNames.length + 1) + }) + + it("should fetch information about a datasource with multiple tables", async () => { + const existingTableNames = ( + await config.api.datasource.info(datasource) + ).tableNames + + const tableNames = [ + generator.guid(), + generator.guid(), + generator.guid(), + generator.guid(), + ] + for (const tableName of tableNames) { + await client.schema.createTable(tableName, table => { + table.increments("id").primary() + table.string("name") + }) + } + + const info = await config.api.datasource.info(datasource) + expect(info.tableNames).toEqual( + expect.arrayContaining([...tableNames, ...existingTableNames]) + ) + expect(info.tableNames).toHaveLength( + existingTableNames.length + tableNames.length + ) }) }) }) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index 8a1669d34c..d8c2a4a257 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -2693,4 +2693,239 @@ describe.each([ ) }) }) + + !isLucene && + describe("$and", () => { + beforeAll(async () => { + table = await createTable({ + age: { name: "age", type: FieldType.NUMBER }, + name: { name: "name", type: FieldType.STRING }, + }) + await createRows([ + { age: 1, name: "Jane" }, + { age: 10, name: "Jack" }, + { age: 7, name: "Hanna" }, + { age: 8, name: "Jan" }, + ]) + }) + + it("successfully finds a row for one level condition", async () => { + await expectQuery({ + $and: { + conditions: [{ equal: { age: 10 } }, { equal: { name: "Jack" } }], + }, + }).toContainExactly([{ age: 10, name: "Jack" }]) + }) + + it("successfully finds a row for one level with multiple conditions", async () => { + await expectQuery({ + $and: { + conditions: [{ equal: { age: 10 } }, { equal: { name: "Jack" } }], + }, + }).toContainExactly([{ age: 10, name: "Jack" }]) + }) + + it("successfully finds multiple rows for one level with multiple conditions", async () => { + await expectQuery({ + $and: { + conditions: [ + { range: { age: { low: 1, high: 9 } } }, + { string: { name: "Ja" } }, + ], + }, + }).toContainExactly([ + { age: 1, name: "Jane" }, + { age: 8, name: "Jan" }, + ]) + }) + + it("successfully finds rows for nested filters", async () => { + await expectQuery({ + $and: { + conditions: [ + { + $and: { + conditions: [ + { + range: { age: { low: 1, high: 10 } }, + }, + { string: { name: "Ja" } }, + ], + }, + equal: { name: "Jane" }, + }, + ], + }, + }).toContainExactly([{ age: 1, name: "Jane" }]) + }) + + it("returns nothing when filtering out all data", async () => { + await expectQuery({ + $and: { + conditions: [{ equal: { age: 7 } }, { equal: { name: "Jack" } }], + }, + }).toFindNothing() + }) + + !isInMemory && + it("validates conditions that are not objects", async () => { + await expect( + expectQuery({ + $and: { + conditions: [{ equal: { age: 10 } }, "invalidCondition" as any], + }, + }).toFindNothing() + ).rejects.toThrow( + 'Invalid body - "query.$and.conditions[1]" must be of type object' + ) + }) + + !isInMemory && + it("validates $and without conditions", async () => { + await expect( + expectQuery({ + $and: { + conditions: [ + { equal: { age: 10 } }, + { + $and: { + conditions: undefined as any, + }, + }, + ], + }, + }).toFindNothing() + ).rejects.toThrow( + 'Invalid body - "query.$and.conditions[1].$and.conditions" is required' + ) + }) + }) + + !isLucene && + describe("$or", () => { + beforeAll(async () => { + table = await createTable({ + age: { name: "age", type: FieldType.NUMBER }, + name: { name: "name", type: FieldType.STRING }, + }) + await createRows([ + { age: 1, name: "Jane" }, + { age: 10, name: "Jack" }, + { age: 7, name: "Hanna" }, + { age: 8, name: "Jan" }, + ]) + }) + + it("successfully finds a row for one level condition", async () => { + await expectQuery({ + $or: { + conditions: [{ equal: { age: 7 } }, { equal: { name: "Jack" } }], + }, + }).toContainExactly([ + { age: 10, name: "Jack" }, + { age: 7, name: "Hanna" }, + ]) + }) + + it("successfully finds a row for one level with multiple conditions", async () => { + await expectQuery({ + $or: { + conditions: [{ equal: { age: 7 } }, { equal: { name: "Jack" } }], + }, + }).toContainExactly([ + { age: 10, name: "Jack" }, + { age: 7, name: "Hanna" }, + ]) + }) + + it("successfully finds multiple rows for one level with multiple conditions", async () => { + await expectQuery({ + $or: { + conditions: [ + { range: { age: { low: 1, high: 9 } } }, + { string: { name: "Jan" } }, + ], + }, + }).toContainExactly([ + { age: 1, name: "Jane" }, + { age: 7, name: "Hanna" }, + { age: 8, name: "Jan" }, + ]) + }) + + it("successfully finds rows for nested filters", async () => { + await expectQuery({ + $or: { + conditions: [ + { + $or: { + conditions: [ + { + range: { age: { low: 1, high: 7 } }, + }, + { string: { name: "Jan" } }, + ], + }, + equal: { name: "Jane" }, + }, + ], + }, + }).toContainExactly([ + { age: 1, name: "Jane" }, + { age: 7, name: "Hanna" }, + { age: 8, name: "Jan" }, + ]) + }) + + it("returns nothing when filtering out all data", async () => { + await expectQuery({ + $or: { + conditions: [{ equal: { age: 6 } }, { equal: { name: "John" } }], + }, + }).toFindNothing() + }) + + it("can nest $and under $or filters", async () => { + await expectQuery({ + $or: { + conditions: [ + { + $and: { + conditions: [ + { + range: { age: { low: 1, high: 8 } }, + }, + { equal: { name: "Jan" } }, + ], + }, + equal: { name: "Jane" }, + }, + ], + }, + }).toContainExactly([ + { age: 1, name: "Jane" }, + { age: 8, name: "Jan" }, + ]) + }) + + it("can nest $or under $and filters", async () => { + await expectQuery({ + $and: { + conditions: [ + { + $or: { + conditions: [ + { + range: { age: { low: 1, high: 8 } }, + }, + { equal: { name: "Jan" } }, + ], + }, + equal: { name: "Jane" }, + }, + ], + }, + }).toContainExactly([{ age: 1, name: "Jane" }]) + }) + }) }) diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts index 8c0bc39234..ea6aedbe3c 100644 --- a/packages/server/src/api/routes/tests/viewV2.spec.ts +++ b/packages/server/src/api/routes/tests/viewV2.spec.ts @@ -1485,6 +1485,119 @@ describe.each([ } ) }) + + isLucene && + it("in lucene, cannot override a view filter", async () => { + await config.api.row.save(table._id!, { + one: "foo", + two: "bar", + }) + const two = await config.api.row.save(table._id!, { + one: "foo2", + two: "bar2", + }) + + const view = await config.api.viewV2.create({ + tableId: table._id!, + name: generator.guid(), + query: [ + { + operator: BasicOperator.EQUAL, + field: "two", + value: "bar2", + }, + ], + schema: { + id: { visible: true }, + one: { visible: false }, + two: { visible: true }, + }, + }) + + const response = await config.api.viewV2.search(view.id, { + query: { + equal: { + two: "bar", + }, + }, + }) + expect(response.rows).toHaveLength(1) + expect(response.rows).toEqual([ + expect.objectContaining({ _id: two._id }), + ]) + }) + + !isLucene && + it("can filter a view without a view filter", async () => { + const one = await config.api.row.save(table._id!, { + one: "foo", + two: "bar", + }) + await config.api.row.save(table._id!, { + one: "foo2", + two: "bar2", + }) + + const view = await config.api.viewV2.create({ + tableId: table._id!, + name: generator.guid(), + schema: { + id: { visible: true }, + one: { visible: false }, + two: { visible: true }, + }, + }) + + const response = await config.api.viewV2.search(view.id, { + query: { + equal: { + two: "bar", + }, + }, + }) + expect(response.rows).toHaveLength(1) + expect(response.rows).toEqual([ + expect.objectContaining({ _id: one._id }), + ]) + }) + + !isLucene && + it("cannot bypass a view filter", async () => { + await config.api.row.save(table._id!, { + one: "foo", + two: "bar", + }) + await config.api.row.save(table._id!, { + one: "foo2", + two: "bar2", + }) + + const view = await config.api.viewV2.create({ + tableId: table._id!, + name: generator.guid(), + query: [ + { + operator: BasicOperator.EQUAL, + field: "two", + value: "bar2", + }, + ], + schema: { + id: { visible: true }, + one: { visible: false }, + two: { visible: true }, + }, + }) + + const response = await config.api.viewV2.search(view.id, { + query: { + equal: { + two: "bar", + }, + }, + }) + expect(response.rows).toHaveLength(0) + }) }) describe("permissions", () => { diff --git a/packages/server/src/api/routes/utils/validators.ts b/packages/server/src/api/routes/utils/validators.ts index 671ce95038..9aa112cf4d 100644 --- a/packages/server/src/api/routes/utils/validators.ts +++ b/packages/server/src/api/routes/utils/validators.ts @@ -1,6 +1,11 @@ import { auth, permissions } from "@budibase/backend-core" import { DataSourceOperation } from "../../../constants" -import { Table, WebhookActionType } from "@budibase/types" +import { + EmptyFilterOption, + SearchFilters, + Table, + WebhookActionType, +} from "@budibase/types" import Joi, { CustomValidator } from "joi" import { ValidSnippetNameRegex, helpers } from "@budibase/shared-core" import sdk from "../../../sdk" @@ -84,7 +89,12 @@ export function datasourceValidator() { } function filterObject() { - return Joi.object({ + const conditionalFilteringObject = () => + Joi.object({ + conditions: Joi.array().items(Joi.link("#schema")).required(), + }) + + const filtersValidators: Record = { string: Joi.object().optional(), fuzzy: Joi.object().optional(), range: Joi.object().optional(), @@ -95,8 +105,17 @@ function filterObject() { oneOf: Joi.object().optional(), contains: Joi.object().optional(), notContains: Joi.object().optional(), + containsAny: Joi.object().optional(), allOr: Joi.boolean().optional(), - }).unknown(true) + onEmptyFilter: Joi.string() + .optional() + .valid(...Object.values(EmptyFilterOption)), + $and: conditionalFilteringObject(), + $or: conditionalFilteringObject(), + fuzzyOr: Joi.forbidden(), + documentType: Joi.forbidden(), + } + return Joi.object(filtersValidators).unknown(true).id("schema") } export function internalSearchValidator() { diff --git a/packages/server/src/automations/steps/queryRows.ts b/packages/server/src/automations/steps/queryRows.ts index 172bbf7f55..526e994c1f 100644 --- a/packages/server/src/automations/steps/queryRows.ts +++ b/packages/server/src/automations/steps/queryRows.ts @@ -11,13 +11,10 @@ import { AutomationStepSchema, AutomationStepType, EmptyFilterOption, - SearchFilters, - Table, SortOrder, QueryRowsStepInputs, QueryRowsStepOutputs, } from "@budibase/types" -import { db as dbCore } from "@budibase/backend-core" const SortOrderPretty = { [SortOrder.ASCENDING]: "Ascending", @@ -95,38 +92,6 @@ async function getTable(appId: string, tableId: string) { return ctx.body } -function typeCoercion(filters: SearchFilters, table: Table) { - if (!filters || !table) { - return filters - } - for (let key of Object.keys(filters)) { - const searchParam = filters[key as keyof SearchFilters] - if (typeof searchParam === "object") { - for (let [property, value] of Object.entries(searchParam)) { - // We need to strip numerical prefixes here, so that we can look up - // the correct field name in the schema - const columnName = dbCore.removeKeyNumbering(property) - const column = table.schema[columnName] - - // convert string inputs - if (!column || typeof value !== "string") { - continue - } - if (column.type === FieldType.NUMBER) { - if (key === "oneOf") { - searchParam[property] = value - .split(",") - .map(item => parseFloat(item)) - } else { - searchParam[property] = parseFloat(value) - } - } - } - } - } - return filters -} - function hasNullFilters(filters: any[]) { return ( filters.length === 0 || @@ -157,7 +122,7 @@ export async function run({ sortType = fieldType === FieldType.NUMBER ? FieldType.NUMBER : FieldType.STRING } - const ctx: any = buildCtx(appId, null, { + const ctx = buildCtx(appId, null, { params: { tableId, }, @@ -165,7 +130,7 @@ export async function run({ sortType, limit, sort: sortColumn, - query: typeCoercion(filters || {}, table), + query: filters || {}, // default to ascending, like data tab sortOrder: sortOrder || SortOrder.ASCENDING, }, diff --git a/packages/server/src/features.ts b/packages/server/src/features.ts index f040cf82a2..bf92ede18e 100644 --- a/packages/server/src/features.ts +++ b/packages/server/src/features.ts @@ -1,4 +1,3 @@ -import { features } from "@budibase/backend-core" import env from "./environment" enum AppFeature { @@ -6,7 +5,25 @@ enum AppFeature { AUTOMATIONS = "automations", } -const featureList = features.processFeatureEnvVar( +export function processFeatureEnvVar( + fullList: string[], + featureList?: string +) { + let list + if (!featureList) { + list = fullList + } else { + list = featureList.split(",") + } + for (let feature of list) { + if (!fullList.includes(feature)) { + throw new Error(`Feature: ${feature} is not an allowed option`) + } + } + return list as unknown as T[] +} + +const featureList = processFeatureEnvVar( Object.values(AppFeature), env.APP_FEATURES ) diff --git a/packages/server/src/integrations/oracle.ts b/packages/server/src/integrations/oracle.ts index 95e4943c7a..fcbc0731ea 100644 --- a/packages/server/src/integrations/oracle.ts +++ b/packages/server/src/integrations/oracle.ts @@ -400,7 +400,9 @@ class OracleIntegration extends Sql implements DatasourcePlus { if (oracleConstraint.type === OracleContraintTypes.PRIMARY) { table.primary!.push(columnName) } else if ( - oracleConstraint.type === OracleContraintTypes.NOT_NULL_OR_CHECK + oracleConstraint.type === + OracleContraintTypes.NOT_NULL_OR_CHECK && + oracleConstraint.searchCondition?.endsWith("IS NOT NULL") ) { table.schema[columnName].constraints = { presence: true, @@ -421,7 +423,11 @@ class OracleIntegration extends Sql implements DatasourcePlus { const columnsResponse = await this.internalQuery({ sql: OracleIntegration.COLUMNS_SQL, }) - return (columnsResponse.rows || []).map(row => row.TABLE_NAME) + const tableNames = new Set() + for (const row of columnsResponse.rows || []) { + tableNames.add(row.TABLE_NAME) + } + return Array.from(tableNames) } async testConnection() { diff --git a/packages/server/src/sdk/app/rows/search.ts b/packages/server/src/sdk/app/rows/search.ts index c008d43548..1ccd89639b 100644 --- a/packages/server/src/sdk/app/rows/search.ts +++ b/packages/server/src/sdk/app/rows/search.ts @@ -13,6 +13,7 @@ import { dataFilters } from "@budibase/shared-core" import sdk from "../../index" import { searchInputMapping } from "./search/utils" import { db as dbCore } from "@budibase/backend-core" +import tracer from "dd-trace" export { isValidFilter } from "../../../integrations/utils" @@ -32,32 +33,65 @@ function pickApi(tableId: any) { export async function search( options: RowSearchParams ): Promise> { - const isExternalTable = isExternalTableID(options.tableId) - options.query = dataFilters.cleanupQuery(options.query || {}) - options.query = dataFilters.fixupFilterArrays(options.query) - if ( - !dataFilters.hasFilters(options.query) && - options.query.onEmptyFilter === EmptyFilterOption.RETURN_NONE - ) { - return { - rows: [], + return await tracer.trace("search", async span => { + span?.addTags({ + tableId: options.tableId, + query: options.query, + sort: options.sort, + sortOrder: options.sortOrder, + sortType: options.sortType, + limit: options.limit, + bookmark: options.bookmark, + paginate: options.paginate, + fields: options.fields, + countRows: options.countRows, + }) + + const isExternalTable = isExternalTableID(options.tableId) + options.query = dataFilters.cleanupQuery(options.query || {}) + options.query = dataFilters.fixupFilterArrays(options.query) + + span?.addTags({ + cleanedQuery: options.query, + isExternalTable, + }) + + if ( + !dataFilters.hasFilters(options.query) && + options.query.onEmptyFilter === EmptyFilterOption.RETURN_NONE + ) { + span?.addTags({ emptyQuery: true }) + return { + rows: [], + } } - } - if (options.sortOrder) { - options.sortOrder = options.sortOrder.toLowerCase() as SortOrder - } + if (options.sortOrder) { + options.sortOrder = options.sortOrder.toLowerCase() as SortOrder + } - const table = await sdk.tables.getTable(options.tableId) - options = searchInputMapping(table, options) + const table = await sdk.tables.getTable(options.tableId) + options = searchInputMapping(table, options) - if (isExternalTable) { - return external.search(options, table) - } else if (dbCore.isSqsEnabledForTenant()) { - return internal.sqs.search(options, table) - } else { - return internal.lucene.search(options, table) - } + let result: SearchResponse + if (isExternalTable) { + span?.addTags({ searchType: "external" }) + result = await external.search(options, table) + } else if (dbCore.isSqsEnabledForTenant()) { + span?.addTags({ searchType: "sqs" }) + result = await internal.sqs.search(options, table) + } else { + span?.addTags({ searchType: "lucene" }) + result = await internal.lucene.search(options, table) + } + + span?.addTags({ + foundRows: result.rows.length, + totalRows: result.totalRows, + }) + + return result + }) } export async function exportRows( diff --git a/packages/server/src/sdk/app/rows/search/internal/sqs.ts b/packages/server/src/sdk/app/rows/search/internal/sqs.ts index 321ffbd9af..66ec905c61 100644 --- a/packages/server/src/sdk/app/rows/search/internal/sqs.ts +++ b/packages/server/src/sdk/app/rows/search/internal/sqs.ts @@ -2,6 +2,7 @@ import { Datasource, DocumentType, FieldType, + isLogicalSearchOperator, Operation, QueryJson, RelationshipFieldMetadata, @@ -137,20 +138,33 @@ function cleanupFilters( allTables.some(table => table.schema[key]) const splitter = new dataFilters.ColumnSplitter(allTables) - for (const filter of Object.values(filters)) { - for (const key of Object.keys(filter)) { - const { numberPrefix, relationshipPrefix, column } = splitter.run(key) - if (keyInAnyTable(column)) { - filter[ - `${numberPrefix || ""}${relationshipPrefix || ""}${mapToUserColumn( - column - )}` - ] = filter[key] - delete filter[key] + + const prefixFilters = (filters: SearchFilters) => { + for (const filterKey of Object.keys(filters) as (keyof SearchFilters)[]) { + if (isLogicalSearchOperator(filterKey)) { + for (const condition of filters[filterKey]!.conditions) { + prefixFilters(condition) + } + } else { + const filter = filters[filterKey]! + if (typeof filter !== "object") { + continue + } + for (const key of Object.keys(filter)) { + const { numberPrefix, relationshipPrefix, column } = splitter.run(key) + if (keyInAnyTable(column)) { + filter[ + `${numberPrefix || ""}${ + relationshipPrefix || "" + }${mapToUserColumn(column)}` + ] = filter[key] + delete filter[key] + } + } } } } - + prefixFilters(filters) return filters } diff --git a/packages/shared-core/src/filters.ts b/packages/shared-core/src/filters.ts index d30f591abc..e1a783175d 100644 --- a/packages/shared-core/src/filters.ts +++ b/packages/shared-core/src/filters.ts @@ -17,6 +17,8 @@ import { Table, BasicOperator, RangeOperator, + LogicalOperator, + isLogicalSearchOperator, } from "@budibase/types" import dayjs from "dayjs" import { OperatorOptions, SqlNumberTypeRangeMap } from "./constants" @@ -358,6 +360,8 @@ export const buildQuery = (filter: SearchFilter[]) => { high: value, } } + } else if (isLogicalSearchOperator(queryOperator)) { + // TODO } else if (query[queryOperator] && operator !== "onEmptyFilter") { if (type === "boolean") { // Transform boolean filters to cope with null. @@ -458,14 +462,17 @@ export const runQuery = (docs: Record[], query: SearchFilters) => { ) => (doc: Record) => { for (const [key, testValue] of Object.entries(query[type] || {})) { - const result = test(deepGet(doc, removeKeyNumbering(key)), testValue) + const valueToCheck = isLogicalSearchOperator(type) + ? doc + : deepGet(doc, removeKeyNumbering(key)) + const result = test(valueToCheck, testValue) if (query.allOr && result) { return true } else if (!query.allOr && !result) { return false } } - return true + return !query.allOr } const stringMatch = match( @@ -666,8 +673,45 @@ export const runQuery = (docs: Record[], query: SearchFilters) => { ) const containsAny = match(ArrayOperator.CONTAINS_ANY, _contains("some")) + const and = match( + LogicalOperator.AND, + (docValue: Record, conditions: SearchFilters[]) => { + if (!conditions.length) { + return false + } + for (const condition of conditions) { + const matchesCondition = runQuery([docValue], condition) + if (!matchesCondition.length) { + return false + } + } + return true + } + ) + const or = match( + LogicalOperator.OR, + (docValue: Record, conditions: SearchFilters[]) => { + if (!conditions.length) { + return false + } + for (const condition of conditions) { + const matchesCondition = runQuery([docValue], { + ...condition, + allOr: true, + }) + if (matchesCondition.length) { + return true + } + } + return false + } + ) + const docMatch = (doc: Record) => { - const filterFunctions = { + const filterFunctions: Record< + SearchFilterOperator, + (doc: Record) => boolean + > = { string: stringMatch, fuzzy: fuzzyMatch, range: rangeMatch, @@ -679,6 +723,8 @@ export const runQuery = (docs: Record[], query: SearchFilters) => { contains: contains, containsAny: containsAny, notContains: notContains, + [LogicalOperator.AND]: and, + [LogicalOperator.OR]: or, } const results = Object.entries(query || {}) diff --git a/packages/shared-core/tsconfig.build.json b/packages/shared-core/tsconfig.build.json index 13e298d71c..c1d5bc96e8 100644 --- a/packages/shared-core/tsconfig.build.json +++ b/packages/shared-core/tsconfig.build.json @@ -18,6 +18,6 @@ }, "tsBuildInfoFile": "dist/tsconfig.tsbuildinfo" }, - "include": ["src/**/*"], + "include": ["src/**/*.ts"], "exclude": ["**/*.spec.ts", "**/*.spec.js", "__mocks__", "src/tests"] } diff --git a/packages/shared-core/tsconfig.json b/packages/shared-core/tsconfig.json index d0c5134f1c..41dcee87c9 100644 --- a/packages/shared-core/tsconfig.json +++ b/packages/shared-core/tsconfig.json @@ -1,9 +1,6 @@ { "extends": "./tsconfig.build.json", "compilerOptions": { - "baseUrl": "..", - "rootDir": "src", - "composite": true, "types": ["node", "jest"] }, "exclude": ["node_modules", "dist"] diff --git a/packages/types/src/sdk/search.ts b/packages/types/src/sdk/search.ts index 5607efece8..6feea40766 100644 --- a/packages/types/src/sdk/search.ts +++ b/packages/types/src/sdk/search.ts @@ -23,7 +23,22 @@ export enum RangeOperator { RANGE = "range", } -export type SearchFilterOperator = BasicOperator | ArrayOperator | RangeOperator +export enum LogicalOperator { + AND = "$and", + OR = "$or", +} + +export function isLogicalSearchOperator( + value: string +): value is LogicalOperator { + return value === LogicalOperator.AND || value === LogicalOperator.OR +} + +export type SearchFilterOperator = + | BasicOperator + | ArrayOperator + | RangeOperator + | LogicalOperator export enum InternalSearchFilterOperator { COMPLEX_ID_OPERATOR = "_complexIdOperator", @@ -75,6 +90,13 @@ export interface SearchFilters { // to make sure the documents returned are always filtered down to a // specific document type (such as just rows) documentType?: DocumentType + + [LogicalOperator.AND]?: { + conditions: SearchFilters[] + } + [LogicalOperator.OR]?: { + conditions: SearchFilters[] + } } export type SearchFilterKey = keyof Omit< diff --git a/packages/worker/src/api/controllers/global/self.ts b/packages/worker/src/api/controllers/global/self.ts index d762f5168a..ec154adf7f 100644 --- a/packages/worker/src/api/controllers/global/self.ts +++ b/packages/worker/src/api/controllers/global/self.ts @@ -1,6 +1,6 @@ import * as userSdk from "../../../sdk/users" import { - featureFlags, + features, tenancy, db as dbCore, utils, @@ -104,8 +104,8 @@ export async function getSelf(ctx: any) { ctx.body = await groups.enrichUserRolesFromGroups(user) // add the feature flags for this tenant - const tenantId = tenancy.getTenantId() - ctx.body.featureFlags = featureFlags.getTenantFeatureFlags(tenantId) + const flags = await features.fetch() + ctx.body.flags = flags addSessionAttributesToUser(ctx) } diff --git a/packages/worker/src/environment.ts b/packages/worker/src/environment.ts index 9f7baf9e9b..6e36b45a3b 100644 --- a/packages/worker/src/environment.ts +++ b/packages/worker/src/environment.ts @@ -19,8 +19,6 @@ function parseIntSafe(number: any) { } const environment = { - // features - WORKER_FEATURES: process.env.WORKER_FEATURES, // auth MINIO_ACCESS_KEY: process.env.MINIO_ACCESS_KEY, MINIO_SECRET_KEY: process.env.MINIO_SECRET_KEY, diff --git a/packages/worker/src/features.ts b/packages/worker/src/features.ts deleted file mode 100644 index 075b3b81ca..0000000000 --- a/packages/worker/src/features.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { features } from "@budibase/backend-core" -import env from "./environment" - -enum WorkerFeature {} - -const featureList: WorkerFeature[] = features.processFeatureEnvVar( - Object.values(WorkerFeature), - env.WORKER_FEATURES -) - -export function isFeatureEnabled(feature: WorkerFeature) { - return featureList.includes(feature) -} diff --git a/yarn.lock b/yarn.lock index 607db0b7bb..0195f19a2a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -12072,10 +12072,10 @@ google-p12-pem@^4.0.0: dependencies: node-forge "^1.3.1" -"google-spreadsheet@npm:@budibase/google-spreadsheet@4.1.2": - version "4.1.2" - resolved "https://registry.yarnpkg.com/@budibase/google-spreadsheet/-/google-spreadsheet-4.1.2.tgz#90548ccba2284b3042b08d2974ef3caeaf772ad9" - integrity sha512-dxoY3rQGGnuNeZiXhNc9oYPduzU8xnIjWujFwNvaRRv3zWeUV7mj6HE2o/OJOeekPGt7o44B+w6DfkiaoteZgg== +"google-spreadsheet@npm:@budibase/google-spreadsheet@4.1.3": + version "4.1.3" + resolved "https://registry.yarnpkg.com/@budibase/google-spreadsheet/-/google-spreadsheet-4.1.3.tgz#bcee7bd9d90f82c54b16a9aca963b87aceb050ad" + integrity sha512-03VX3/K5NXIh6+XAIDZgcHPmR76xwd8vIDL7RedMpvM2IcXK0Iq/KU7FmLY0t/mKqORAGC7+0rajd0jLFezC4w== dependencies: axios "^1.4.0" lodash "^4.17.21"