MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into test-oracle

This commit is contained in:
Michael Drury 2024-08-07 16:44:54 +01:00 committed by GitHub
parent 0f3988caaa 95426db854
commit 7b41e7cb39
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
31 changed files with 838 additions and 313 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.github/workflows/budibase_ci.yml vendored
View File

@ -147,7 +147,7 @@ jobs:
fi
test-server:
runs-on: budi-tubby-tornado-quad-core-150gb
runs-on: budi-tubby-tornado-quad-core-300gb
steps:
- name: Checkout repo
uses: actions/checkout@v4

2
README.md
View File

@ -116,7 +116,7 @@ As with anything that we build in Budibase, our new public API is simple to use,
You can learn more about the Budibase API at the following places:
- [General documentation](https://docs.budibase.com/docs/public-api): Learn how to get your API key, how to use spec, and how to use Postman
- [Interactive API documentation](https://docs.budibase.com/reference/post_applications) : Learn how to interact with the API
- [Interactive API documentation](https://docs.budibase.com/reference/appcreate) : Learn how to interact with the API
<br /><br />

2
i18n/README.es.md
View File

@ -144,7 +144,7 @@ del sistema. Budibase API ofrece:
Puedes aprender mas acerca de Budibase API en los siguientes documentos:
- [Documentacion general](https://docs.budibase.com/docs/public-api) : Como optener tu clave para la API, usar Insomnia y Postman
- [API Interactiva](https://docs.budibase.com/reference/post_applications) : Aprende como trabajar con la API
- [API Interactiva](https://docs.budibase.com/reference/appcreate) : Aprende como trabajar con la API
#### Guias

2
lerna.json
View File

@ -1,6 +1,6 @@
{
"$schema": "node_modules/lerna/schemas/lerna-schema.json",
"version": "2.29.28",
"version": "2.29.30",
"npmClient": "yarn",
"packages": [
"packages/*",

151
packages/backend-core/src/features/index.ts
View File

@ -1,79 +1,108 @@
import env from "../environment"
import * as context from "../context"
import { cloneDeep } from "lodash"
export * from "./installation"
/**
* Read the TENANT_FEATURE_FLAGS env var and return an array of features flags for each tenant.
* The env var is formatted as:
* tenant1:feature1:feature2,tenant2:feature1
*/
export function buildFeatureFlags() {
if (!env.TENANT_FEATURE_FLAGS) {
return
class Flag<T> {
static withDefault<T>(value: T) {
return new Flag(value)
}
const tenantFeatureFlags: Record<string, string[]> = {}
private constructor(public defaultValue: T) {}
}
env.TENANT_FEATURE_FLAGS.split(",").forEach(tenantToFeatures => {
const [tenantId, ...features] = tenantToFeatures.split(":")
// This is the primary source of truth for feature flags. If you want to add a
// new flag, add it here and use the `fetch` and `get` functions to access it.
// All of the machinery in this file is to make sure that flags have their
// default values set correctly and their types flow through the system.
const FLAGS = {
LICENSING: Flag.withDefault(false),
GOOGLE_SHEETS: Flag.withDefault(false),
USER_GROUPS: Flag.withDefault(false),
ONBOARDING_TOUR: Flag.withDefault(false),
}
features.forEach(feature => {
if (!tenantFeatureFlags[tenantId]) {
tenantFeatureFlags[tenantId] = []
const DEFAULTS = Object.keys(FLAGS).reduce((acc, key) => {
const typedKey = key as keyof typeof FLAGS
// @ts-ignore
acc[typedKey] = FLAGS[typedKey].defaultValue
return acc
}, {} as Flags)
type UnwrapFlag<F> = F extends Flag<infer U> ? U : never
export type Flags = {
[K in keyof typeof FLAGS]: UnwrapFlag<(typeof FLAGS)[K]>
}
// Exported for use in tests, should not be used outside of this file.
export function defaultFlags(): Flags {
return cloneDeep(DEFAULTS)
}
function isFlagName(name: string): name is keyof Flags {
return FLAGS[name as keyof typeof FLAGS] !== undefined
}
/**
* Reads the TENANT_FEATURE_FLAGS environment variable and returns a Flags object
* populated with the flags for the current tenant, filling in the default values
* if the flag is not set.
*
* Check the tests for examples of how TENANT_FEATURE_FLAGS should be formatted.
*
* In future we plan to add more ways of setting feature flags, e.g. PostHog, and
* they will be accessed through this function as well.
*/
export async function fetch(): Promise<Flags> {
const currentTenantId = context.getTenantId()
const flags = defaultFlags()
const split = (env.TENANT_FEATURE_FLAGS || "")
.split(",")
.map(x => x.split(":"))
for (const [tenantId, ...features] of split) {
if (!tenantId || (tenantId !== "*" && tenantId !== currentTenantId)) {
continue
}
for (let feature of features) {
let value = true
if (feature.startsWith("!")) {
feature = feature.slice(1)
value = false
}
tenantFeatureFlags[tenantId].push(feature)
})
})
return tenantFeatureFlags
}
if (!isFlagName(feature)) {
throw new Error(`Feature: ${feature} is not an allowed option`)
}
export function isEnabled(featureFlag: string) {
const tenantId = context.getTenantId()
const flags = getTenantFeatureFlags(tenantId)
return flags.includes(featureFlag)
}
if (typeof flags[feature] !== "boolean") {
throw new Error(`Feature: ${feature} is not a boolean`)
}
export function getTenantFeatureFlags(tenantId: string) {
let flags: string[] = []
const envFlags = buildFeatureFlags()
if (envFlags) {
const globalFlags = envFlags["*"]
const tenantFlags = envFlags[tenantId] || []
// Explicitly exclude tenants from global features if required.
// Prefix the tenant flag with '!'
const tenantOverrides = tenantFlags.reduce(
(acc: string[], flag: string) => {
if (flag.startsWith("!")) {
let stripped = flag.substring(1)
acc.push(stripped)
}
return acc
},
[]
)
if (globalFlags) {
flags.push(...globalFlags)
// @ts-ignore
flags[feature] = value
}
if (tenantFlags.length) {
flags.push(...tenantFlags)
}
// Purge any tenant specific overrides
flags = flags.filter(flag => {
return tenantOverrides.indexOf(flag) == -1 && !flag.startsWith("!")
})
}
return flags
}
export enum TenantFeatureFlag {
LICENSING = "LICENSING",
GOOGLE_SHEETS = "GOOGLE_SHEETS",
USER_GROUPS = "USER_GROUPS",
ONBOARDING_TOUR = "ONBOARDING_TOUR",
// Gets a single feature flag value. This is a convenience function for
// `fetch().then(flags => flags[name])`.
export async function get<K extends keyof Flags>(name: K): Promise<Flags[K]> {
const flags = await fetch()
return flags[name]
}
type BooleanFlags = {
[K in keyof typeof FLAGS]: (typeof FLAGS)[K] extends Flag<boolean> ? K : never
}[keyof typeof FLAGS]
// Convenience function for boolean flag values. This makes callsites more
// readable for boolean flags.
export async function isEnabled<K extends BooleanFlags>(
name: K
): Promise<boolean> {
const flags = await fetch()
return flags[name]
}

17
packages/backend-core/src/features/installation.ts
View File

@ -1,17 +0,0 @@
export function processFeatureEnvVar<T>(
fullList: string[],
featureList?: string
) {
let list
if (!featureList) {
list = fullList
} else {
list = featureList.split(",")
}
for (let feature of list) {
if (!fullList.includes(feature)) {
throw new Error(`Feature: ${feature} is not an allowed option`)
}
}
return list as unknown as T[]
}

85
packages/backend-core/src/features/tests/featureFlags.spec.ts
View File

@ -1,85 +0,0 @@
import {
TenantFeatureFlag,
buildFeatureFlags,
getTenantFeatureFlags,
} from "../"
import env from "../../environment"
const { ONBOARDING_TOUR, LICENSING, USER_GROUPS } = TenantFeatureFlag
describe("featureFlags", () => {
beforeEach(() => {
env._set("TENANT_FEATURE_FLAGS", "")
})
it("Should return no flags when the TENANT_FEATURE_FLAG is empty", async () => {
let features = buildFeatureFlags()
expect(features).toBeUndefined()
})
it("Should generate a map of global and named tenant feature flags from the env value", async () => {
env._set(
"TENANT_FEATURE_FLAGS",
`*:${ONBOARDING_TOUR},tenant1:!${ONBOARDING_TOUR},tenant2:${USER_GROUPS},tenant1:${LICENSING}`
)
const parsedFlags: Record<string, string[]> = {
"*": [ONBOARDING_TOUR],
tenant1: [`!${ONBOARDING_TOUR}`, LICENSING],
tenant2: [USER_GROUPS],
}
let features = buildFeatureFlags()
expect(features).toBeDefined()
expect(features).toEqual(parsedFlags)
})
it("Should add feature flag flag only to explicitly configured tenant", async () => {
env._set(
"TENANT_FEATURE_FLAGS",
`*:${LICENSING},*:${USER_GROUPS},tenant1:${ONBOARDING_TOUR}`
)
let tenant1Flags = getTenantFeatureFlags("tenant1")
let tenant2Flags = getTenantFeatureFlags("tenant2")
expect(tenant1Flags).toBeDefined()
expect(tenant1Flags).toEqual([LICENSING, USER_GROUPS, ONBOARDING_TOUR])
expect(tenant2Flags).toBeDefined()
expect(tenant2Flags).toEqual([LICENSING, USER_GROUPS])
})
})
it("Should exclude tenant1 from global feature flag", async () => {
env._set(
"TENANT_FEATURE_FLAGS",
`*:${LICENSING},*:${ONBOARDING_TOUR},tenant1:!${ONBOARDING_TOUR}`
)
let tenant1Flags = getTenantFeatureFlags("tenant1")
let tenant2Flags = getTenantFeatureFlags("tenant2")
expect(tenant1Flags).toBeDefined()
expect(tenant1Flags).toEqual([LICENSING])
expect(tenant2Flags).toBeDefined()
expect(tenant2Flags).toEqual([LICENSING, ONBOARDING_TOUR])
})
it("Should explicitly add flags to configured tenants only", async () => {
env._set(
"TENANT_FEATURE_FLAGS",
`tenant1:${ONBOARDING_TOUR},tenant1:${LICENSING},tenant2:${LICENSING}`
)
let tenant1Flags = getTenantFeatureFlags("tenant1")
let tenant2Flags = getTenantFeatureFlags("tenant2")
expect(tenant1Flags).toBeDefined()
expect(tenant1Flags).toEqual([ONBOARDING_TOUR, LICENSING])
expect(tenant2Flags).toBeDefined()
expect(tenant2Flags).toEqual([LICENSING])
})

86
packages/backend-core/src/features/tests/features.spec.ts Normal file
View File

@ -0,0 +1,86 @@
import { defaultFlags, fetch, get, Flags } from "../"
import { context } from "../.."
import env from "../../environment"
async function withFlags<T>(flags: string, f: () => T): Promise<T> {
const oldFlags = env.TENANT_FEATURE_FLAGS
env._set("TENANT_FEATURE_FLAGS", flags)
try {
return await f()
} finally {
env._set("TENANT_FEATURE_FLAGS", oldFlags)
}
}
describe("feature flags", () => {
interface TestCase {
tenant: string
flags: string
expected: Partial<Flags>
}
it.each<TestCase>([
{
tenant: "tenant1",
flags: "tenant1:ONBOARDING_TOUR",
expected: { ONBOARDING_TOUR: true },
},
{
tenant: "tenant1",
flags: "tenant1:!ONBOARDING_TOUR",
expected: { ONBOARDING_TOUR: false },
},
{
tenant: "tenant1",
flags: "*:ONBOARDING_TOUR",
expected: { ONBOARDING_TOUR: true },
},
{
tenant: "tenant1",
flags: "tenant2:ONBOARDING_TOUR",
expected: { ONBOARDING_TOUR: false },
},
{
tenant: "tenant1",
flags: "",
expected: defaultFlags(),
},
])(
'should find flags $expected for $tenant with string "$flags"',
({ tenant, flags, expected }) =>
context.doInTenant(tenant, () =>
withFlags(flags, async () => {
const flags = await fetch()
expect(flags).toMatchObject(expected)
for (const [key, expectedValue] of Object.entries(expected)) {
const value = await get(key as keyof Flags)
expect(value).toBe(expectedValue)
}
})
)
)
interface FailedTestCase {
tenant: string
flags: string
expected: string | RegExp
}
it.each<FailedTestCase>([
{
tenant: "tenant1",
flags: "tenant1:ONBOARDING_TOUR,tenant1:FOO",
expected: "Feature: FOO is not an allowed option",
},
])(
"should fail with message \"$expected\" for $tenant with string '$flags'",
async ({ tenant, flags, expected }) => {
context.doInTenant(tenant, () =>
withFlags(flags, async () => {
await expect(fetch()).rejects.toThrow(expected)
})
)
}
)
})

3
packages/backend-core/src/index.ts
View File

@ -7,8 +7,7 @@ export * as roles from "./security/roles"
export * as permissions from "./security/permissions"
export * as accounts from "./accounts"
export * as installation from "./installation"
export * as featureFlags from "./features"
export * as features from "./features/installation"
export * as features from "./features"
export * as sessions from "./security/sessions"
export * as platform from "./platform"
export * as auth from "./auth"

18
packages/backend-core/src/sql/sql.ts
View File

@ -463,6 +463,24 @@ class InternalBuilder {
}
}
if (filters.$and) {
const { $and } = filters
query = query.where(x => {
for (const condition of $and.conditions) {
x = this.addFilters(x, condition, opts)
}
})
}
if (filters.$or) {
const { $or } = filters
query = query.where(x => {
for (const condition of $or.conditions) {
x = this.addFilters(x, { ...condition, allOr: true }, opts)
}
})
}
if (filters.oneOf) {
const fnc = allOr ? "orWhereIn" : "whereIn"
iterate(

3
packages/builder/src/helpers/featureFlags.js
View File

@ -5,9 +5,10 @@ export const TENANT_FEATURE_FLAGS = {
LICENSING: "LICENSING",
USER_GROUPS: "USER_GROUPS",
ONBOARDING_TOUR: "ONBOARDING_TOUR",
GOOGLE_SHEETS: "GOOGLE_SHEETS",
}
export const isEnabled = featureFlag => {
const user = get(auth).user
return !!user?.featureFlags?.includes(featureFlag)
return !!user?.flags?.[featureFlag]
}

2
packages/pro

@ -1 +1 @@
Subproject commit 7dbe323aec724ae6336b13c06aaefa4a89837edf
Subproject commit 62ef0e2d6e83522b6732fb3c61338de303f06ff0

2
packages/server/package.json
View File

@ -80,7 +80,7 @@
"dotenv": "8.2.0",
"form-data": "4.0.0",
"global-agent": "3.0.0",
"google-spreadsheet": "npm:@budibase/google-spreadsheet@4.1.2",
"google-spreadsheet": "npm:@budibase/google-spreadsheet@4.1.3",
"ioredis": "5.3.2",
"isolated-vm": "^4.7.2",
"jimp": "0.22.12",

38
packages/server/src/api/controllers/row/views.ts
View File

@ -6,11 +6,13 @@ import {
RequiredKeys,
RowSearchParams,
SearchFilterKey,
LogicalOperator,
} from "@budibase/types"
import { dataFilters } from "@budibase/shared-core"
import sdk from "../../../sdk"
import { db, context } from "@budibase/backend-core"
import { enrichSearchContext } from "./utils"
import { isExternalTableID } from "../../../integrations/utils"
export async function searchView(
ctx: UserCtx<SearchViewRowRequest, SearchRowResponse>
@ -35,25 +37,33 @@ export async function searchView(
// that could let users find rows they should not be allowed to access.
let query = dataFilters.buildQuery(view.query || [])
if (body.query) {
// Extract existing fields
const existingFields =
view.query
?.filter(filter => filter.field)
.map(filter => db.removeKeyNumbering(filter.field)) || []
// Delete extraneous search params that cannot be overridden
delete body.query.allOr
delete body.query.onEmptyFilter
// Carry over filters for unused fields
Object.keys(body.query).forEach(key => {
const operator = key as SearchFilterKey
Object.keys(body.query[operator] || {}).forEach(field => {
if (!existingFields.includes(db.removeKeyNumbering(field))) {
query[operator]![field] = body.query[operator]![field]
}
if (!isExternalTableID(view.tableId) && !db.isSqsEnabledForTenant()) {
// Extract existing fields
const existingFields =
view.query
?.filter(filter => filter.field)
.map(filter => db.removeKeyNumbering(filter.field)) || []
// Carry over filters for unused fields
Object.keys(body.query).forEach(key => {
const operator = key as Exclude<SearchFilterKey, LogicalOperator>
Object.keys(body.query[operator] || {}).forEach(field => {
if (!existingFields.includes(db.removeKeyNumbering(field))) {
query[operator]![field] = body.query[operator]![field]
}
})
})
})
} else {
query = {
$and: {
conditions: [query, body.query],
},
}
}
}
await context.ensureSnippetContext(true)

71
packages/server/src/api/routes/tests/datasource.spec.ts
View File

@ -17,9 +17,14 @@ import {
SupportedSqlTypes,
JsonFieldSubType,
} from "@budibase/types"
import { DatabaseName, getDatasource } from "../../../integrations/tests/utils"
import {
DatabaseName,
getDatasource,
knexClient,
} from "../../../integrations/tests/utils"
import { tableForDatasource } from "../../../tests/utilities/structures"
import nock from "nock"
import { Knex } from "knex"
describe("/datasources", () => {
const config = setup.getConfig()
@ -164,11 +169,15 @@ describe("/datasources", () => {
[DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)],
[DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)],
[DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)],
[DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)],
])("%s", (_, dsProvider) => {
let rawDatasource: Datasource
let client: Knex
beforeEach(async () => {
rawDatasource = await dsProvider
datasource = await config.api.datasource.create(rawDatasource)
client = await knexClient(rawDatasource)
})
describe("get", () => {
@ -285,9 +294,6 @@ describe("/datasources", () => {
[FieldType.STRING]: {
name: stringName,
type: FieldType.STRING,
constraints: {
presence: true,
},
},
[FieldType.LONGFORM]: {
name: "longform",
@ -381,10 +387,6 @@ describe("/datasources", () => {
),
schema: Object.entries(table.schema).reduce<TableSchema>(
(acc, [fieldName, field]) => {
// the constraint will be unset - as the DB doesn't recognise it as not null
if (fieldName === stringName) {
field.constraints = {}
}
acc[fieldName] = expect.objectContaining({
...field,
})
@ -441,20 +443,49 @@ describe("/datasources", () => {
})
describe("info", () => {
it("should fetch information about postgres datasource", async () => {
const table = await config.api.table.save(
tableForDatasource(datasource, {
schema: {
name: {
name: "name",
type: FieldType.STRING,
},
},
})
)
it("should fetch information about a datasource with a single table", async () => {
const existingTableNames = (
await config.api.datasource.info(datasource)
).tableNames
const tableName = generator.guid()
await client.schema.createTable(tableName, table => {
table.increments("id").primary()
table.string("name")
})
const info = await config.api.datasource.info(datasource)
expect(info.tableNames).toContain(table.name)
expect(info.tableNames).toEqual(
expect.arrayContaining([tableName, ...existingTableNames])
)
expect(info.tableNames).toHaveLength(existingTableNames.length + 1)
})
it("should fetch information about a datasource with multiple tables", async () => {
const existingTableNames = (
await config.api.datasource.info(datasource)
).tableNames
const tableNames = [
generator.guid(),
generator.guid(),
generator.guid(),
generator.guid(),
]
for (const tableName of tableNames) {
await client.schema.createTable(tableName, table => {
table.increments("id").primary()
table.string("name")
})
}
const info = await config.api.datasource.info(datasource)
expect(info.tableNames).toEqual(
expect.arrayContaining([...tableNames, ...existingTableNames])
)
expect(info.tableNames).toHaveLength(
existingTableNames.length + tableNames.length
)
})
})
})

235
packages/server/src/api/routes/tests/search.spec.ts
View File

@ -2693,4 +2693,239 @@ describe.each([
)
})
})
!isLucene &&
describe("$and", () => {
beforeAll(async () => {
table = await createTable({
age: { name: "age", type: FieldType.NUMBER },
name: { name: "name", type: FieldType.STRING },
})
await createRows([
{ age: 1, name: "Jane" },
{ age: 10, name: "Jack" },
{ age: 7, name: "Hanna" },
{ age: 8, name: "Jan" },
])
})
it("successfully finds a row for one level condition", async () => {
await expectQuery({
$and: {
conditions: [{ equal: { age: 10 } }, { equal: { name: "Jack" } }],
},
}).toContainExactly([{ age: 10, name: "Jack" }])
})
it("successfully finds a row for one level with multiple conditions", async () => {
await expectQuery({
$and: {
conditions: [{ equal: { age: 10 } }, { equal: { name: "Jack" } }],
},
}).toContainExactly([{ age: 10, name: "Jack" }])
})
it("successfully finds multiple rows for one level with multiple conditions", async () => {
await expectQuery({
$and: {
conditions: [
{ range: { age: { low: 1, high: 9 } } },
{ string: { name: "Ja" } },
],
},
}).toContainExactly([
{ age: 1, name: "Jane" },
{ age: 8, name: "Jan" },
])
})
it("successfully finds rows for nested filters", async () => {
await expectQuery({
$and: {
conditions: [
{
$and: {
conditions: [
{
range: { age: { low: 1, high: 10 } },
},
{ string: { name: "Ja" } },
],
},
equal: { name: "Jane" },
},
],
},
}).toContainExactly([{ age: 1, name: "Jane" }])
})
it("returns nothing when filtering out all data", async () => {
await expectQuery({
$and: {
conditions: [{ equal: { age: 7 } }, { equal: { name: "Jack" } }],
},
}).toFindNothing()
})
!isInMemory &&
it("validates conditions that are not objects", async () => {
await expect(
expectQuery({
$and: {
conditions: [{ equal: { age: 10 } }, "invalidCondition" as any],
},
}).toFindNothing()
).rejects.toThrow(
'Invalid body - "query.$and.conditions[1]" must be of type object'
)
})
!isInMemory &&
it("validates $and without conditions", async () => {
await expect(
expectQuery({
$and: {
conditions: [
{ equal: { age: 10 } },
{
$and: {
conditions: undefined as any,
},
},
],
},
}).toFindNothing()
).rejects.toThrow(
'Invalid body - "query.$and.conditions[1].$and.conditions" is required'
)
})
})
!isLucene &&
describe("$or", () => {
beforeAll(async () => {
table = await createTable({
age: { name: "age", type: FieldType.NUMBER },
name: { name: "name", type: FieldType.STRING },
})
await createRows([
{ age: 1, name: "Jane" },
{ age: 10, name: "Jack" },
{ age: 7, name: "Hanna" },
{ age: 8, name: "Jan" },
])
})
it("successfully finds a row for one level condition", async () => {
await expectQuery({
$or: {
conditions: [{ equal: { age: 7 } }, { equal: { name: "Jack" } }],
},
}).toContainExactly([
{ age: 10, name: "Jack" },
{ age: 7, name: "Hanna" },
])
})
it("successfully finds a row for one level with multiple conditions", async () => {
await expectQuery({
$or: {
conditions: [{ equal: { age: 7 } }, { equal: { name: "Jack" } }],
},
}).toContainExactly([
{ age: 10, name: "Jack" },
{ age: 7, name: "Hanna" },
])
})
it("successfully finds multiple rows for one level with multiple conditions", async () => {
await expectQuery({
$or: {
conditions: [
{ range: { age: { low: 1, high: 9 } } },
{ string: { name: "Jan" } },
],
},
}).toContainExactly([
{ age: 1, name: "Jane" },
{ age: 7, name: "Hanna" },
{ age: 8, name: "Jan" },
])
})
it("successfully finds rows for nested filters", async () => {
await expectQuery({
$or: {
conditions: [
{
$or: {
conditions: [
{
range: { age: { low: 1, high: 7 } },
},
{ string: { name: "Jan" } },
],
},
equal: { name: "Jane" },
},
],
},
}).toContainExactly([
{ age: 1, name: "Jane" },
{ age: 7, name: "Hanna" },
{ age: 8, name: "Jan" },
])
})
it("returns nothing when filtering out all data", async () => {
await expectQuery({
$or: {
conditions: [{ equal: { age: 6 } }, { equal: { name: "John" } }],
},
}).toFindNothing()
})
it("can nest $and under $or filters", async () => {
await expectQuery({
$or: {
conditions: [
{
$and: {
conditions: [
{
range: { age: { low: 1, high: 8 } },
},
{ equal: { name: "Jan" } },
],
},
equal: { name: "Jane" },
},
],
},
}).toContainExactly([
{ age: 1, name: "Jane" },
{ age: 8, name: "Jan" },
])
})
it("can nest $or under $and filters", async () => {
await expectQuery({
$and: {
conditions: [
{
$or: {
conditions: [
{
range: { age: { low: 1, high: 8 } },
},
{ equal: { name: "Jan" } },
],
},
equal: { name: "Jane" },
},
],
},
}).toContainExactly([{ age: 1, name: "Jane" }])
})
})
})

113
packages/server/src/api/routes/tests/viewV2.spec.ts
View File

@ -1485,6 +1485,119 @@ describe.each([
}
)
})
isLucene &&
it("in lucene, cannot override a view filter", async () => {
await config.api.row.save(table._id!, {
one: "foo",
two: "bar",
})
const two = await config.api.row.save(table._id!, {
one: "foo2",
two: "bar2",
})
const view = await config.api.viewV2.create({
tableId: table._id!,
name: generator.guid(),
query: [
{
operator: BasicOperator.EQUAL,
field: "two",
value: "bar2",
},
],
schema: {
id: { visible: true },
one: { visible: false },
two: { visible: true },
},
})
const response = await config.api.viewV2.search(view.id, {
query: {
equal: {
two: "bar",
},
},
})
expect(response.rows).toHaveLength(1)
expect(response.rows).toEqual([
expect.objectContaining({ _id: two._id }),
])
})
!isLucene &&
it("can filter a view without a view filter", async () => {
const one = await config.api.row.save(table._id!, {
one: "foo",
two: "bar",
})
await config.api.row.save(table._id!, {
one: "foo2",
two: "bar2",
})
const view = await config.api.viewV2.create({
tableId: table._id!,
name: generator.guid(),
schema: {
id: { visible: true },
one: { visible: false },
two: { visible: true },
},
})
const response = await config.api.viewV2.search(view.id, {
query: {
equal: {
two: "bar",
},
},
})
expect(response.rows).toHaveLength(1)
expect(response.rows).toEqual([
expect.objectContaining({ _id: one._id }),
])
})
!isLucene &&
it("cannot bypass a view filter", async () => {
await config.api.row.save(table._id!, {
one: "foo",
two: "bar",
})
await config.api.row.save(table._id!, {
one: "foo2",
two: "bar2",
})
const view = await config.api.viewV2.create({
tableId: table._id!,
name: generator.guid(),
query: [
{
operator: BasicOperator.EQUAL,
field: "two",
value: "bar2",
},
],
schema: {
id: { visible: true },
one: { visible: false },
two: { visible: true },
},
})
const response = await config.api.viewV2.search(view.id, {
query: {
equal: {
two: "bar",
},
},
})
expect(response.rows).toHaveLength(0)
})
})
describe("permissions", () => {

25
packages/server/src/api/routes/utils/validators.ts
View File

@ -1,6 +1,11 @@
import { auth, permissions } from "@budibase/backend-core"
import { DataSourceOperation } from "../../../constants"
import { Table, WebhookActionType } from "@budibase/types"
import {
EmptyFilterOption,
SearchFilters,
Table,
WebhookActionType,
} from "@budibase/types"
import Joi, { CustomValidator } from "joi"
import { ValidSnippetNameRegex, helpers } from "@budibase/shared-core"
import sdk from "../../../sdk"
@ -84,7 +89,12 @@ export function datasourceValidator() {
}
function filterObject() {
return Joi.object({
const conditionalFilteringObject = () =>
Joi.object({
conditions: Joi.array().items(Joi.link("#schema")).required(),
})
const filtersValidators: Record<keyof SearchFilters, any> = {
string: Joi.object().optional(),
fuzzy: Joi.object().optional(),
range: Joi.object().optional(),
@ -95,8 +105,17 @@ function filterObject() {
oneOf: Joi.object().optional(),
contains: Joi.object().optional(),
notContains: Joi.object().optional(),
containsAny: Joi.object().optional(),
allOr: Joi.boolean().optional(),
}).unknown(true)
onEmptyFilter: Joi.string()
.optional()
.valid(...Object.values(EmptyFilterOption)),
$and: conditionalFilteringObject(),
$or: conditionalFilteringObject(),
fuzzyOr: Joi.forbidden(),
documentType: Joi.forbidden(),
}
return Joi.object(filtersValidators).unknown(true).id("schema")
}
export function internalSearchValidator() {

39
packages/server/src/automations/steps/queryRows.ts
View File

@ -11,13 +11,10 @@ import {
AutomationStepSchema,
AutomationStepType,
EmptyFilterOption,
SearchFilters,
Table,
SortOrder,
QueryRowsStepInputs,
QueryRowsStepOutputs,
} from "@budibase/types"
import { db as dbCore } from "@budibase/backend-core"
const SortOrderPretty = {
[SortOrder.ASCENDING]: "Ascending",
@ -95,38 +92,6 @@ async function getTable(appId: string, tableId: string) {
return ctx.body
}
function typeCoercion(filters: SearchFilters, table: Table) {
if (!filters || !table) {
return filters
}
for (let key of Object.keys(filters)) {
const searchParam = filters[key as keyof SearchFilters]
if (typeof searchParam === "object") {
for (let [property, value] of Object.entries(searchParam)) {
// We need to strip numerical prefixes here, so that we can look up
// the correct field name in the schema
const columnName = dbCore.removeKeyNumbering(property)
const column = table.schema[columnName]
// convert string inputs
if (!column || typeof value !== "string") {
continue
}
if (column.type === FieldType.NUMBER) {
if (key === "oneOf") {
searchParam[property] = value
.split(",")
.map(item => parseFloat(item))
} else {
searchParam[property] = parseFloat(value)
}
}
}
}
}
return filters
}
function hasNullFilters(filters: any[]) {
return (
filters.length === 0 ||
@ -157,7 +122,7 @@ export async function run({
sortType =
fieldType === FieldType.NUMBER ? FieldType.NUMBER : FieldType.STRING
}
const ctx: any = buildCtx(appId, null, {
const ctx = buildCtx(appId, null, {
params: {
tableId,
},
@ -165,7 +130,7 @@ export async function run({
sortType,
limit,
sort: sortColumn,
query: typeCoercion(filters || {}, table),
query: filters || {},
// default to ascending, like data tab
sortOrder: sortOrder || SortOrder.ASCENDING,
},

21
packages/server/src/features.ts
View File

@ -1,4 +1,3 @@
import { features } from "@budibase/backend-core"
import env from "./environment"
enum AppFeature {
@ -6,7 +5,25 @@ enum AppFeature {
AUTOMATIONS = "automations",
}
const featureList = features.processFeatureEnvVar<AppFeature>(
export function processFeatureEnvVar<T>(
fullList: string[],
featureList?: string
) {
let list
if (!featureList) {
list = fullList
} else {
list = featureList.split(",")
}
for (let feature of list) {
if (!fullList.includes(feature)) {
throw new Error(`Feature: ${feature} is not an allowed option`)
}
}
return list as unknown as T[]
}
const featureList = processFeatureEnvVar<AppFeature>(
Object.values(AppFeature),
env.APP_FEATURES
)

10
packages/server/src/integrations/oracle.ts
View File

@ -400,7 +400,9 @@ class OracleIntegration extends Sql implements DatasourcePlus {
if (oracleConstraint.type === OracleContraintTypes.PRIMARY) {
table.primary!.push(columnName)
} else if (
oracleConstraint.type === OracleContraintTypes.NOT_NULL_OR_CHECK
oracleConstraint.type ===
OracleContraintTypes.NOT_NULL_OR_CHECK &&
oracleConstraint.searchCondition?.endsWith("IS NOT NULL")
) {
table.schema[columnName].constraints = {
presence: true,
@ -421,7 +423,11 @@ class OracleIntegration extends Sql implements DatasourcePlus {
const columnsResponse = await this.internalQuery<OracleColumnsResponse>({
sql: OracleIntegration.COLUMNS_SQL,
})
return (columnsResponse.rows || []).map(row => row.TABLE_NAME)
const tableNames = new Set<string>()
for (const row of columnsResponse.rows || []) {
tableNames.add(row.TABLE_NAME)
}
return Array.from(tableNames)
}
async testConnection() {

78
packages/server/src/sdk/app/rows/search.ts
View File

@ -13,6 +13,7 @@ import { dataFilters } from "@budibase/shared-core"
import sdk from "../../index"
import { searchInputMapping } from "./search/utils"
import { db as dbCore } from "@budibase/backend-core"
import tracer from "dd-trace"
export { isValidFilter } from "../../../integrations/utils"
@ -32,32 +33,65 @@ function pickApi(tableId: any) {
export async function search(
options: RowSearchParams
): Promise<SearchResponse<Row>> {
const isExternalTable = isExternalTableID(options.tableId)
options.query = dataFilters.cleanupQuery(options.query || {})
options.query = dataFilters.fixupFilterArrays(options.query)
if (
!dataFilters.hasFilters(options.query) &&
options.query.onEmptyFilter === EmptyFilterOption.RETURN_NONE
) {
return {
rows: [],
return await tracer.trace("search", async span => {
span?.addTags({
tableId: options.tableId,
query: options.query,
sort: options.sort,
sortOrder: options.sortOrder,
sortType: options.sortType,
limit: options.limit,
bookmark: options.bookmark,
paginate: options.paginate,
fields: options.fields,
countRows: options.countRows,
})
const isExternalTable = isExternalTableID(options.tableId)
options.query = dataFilters.cleanupQuery(options.query || {})
options.query = dataFilters.fixupFilterArrays(options.query)
span?.addTags({
cleanedQuery: options.query,
isExternalTable,
})
if (
!dataFilters.hasFilters(options.query) &&
options.query.onEmptyFilter === EmptyFilterOption.RETURN_NONE
) {
span?.addTags({ emptyQuery: true })
return {
rows: [],
}
}
}
if (options.sortOrder) {
options.sortOrder = options.sortOrder.toLowerCase() as SortOrder
}
if (options.sortOrder) {
options.sortOrder = options.sortOrder.toLowerCase() as SortOrder
}
const table = await sdk.tables.getTable(options.tableId)
options = searchInputMapping(table, options)
const table = await sdk.tables.getTable(options.tableId)
options = searchInputMapping(table, options)
if (isExternalTable) {
return external.search(options, table)
} else if (dbCore.isSqsEnabledForTenant()) {
return internal.sqs.search(options, table)
} else {
return internal.lucene.search(options, table)
}
let result: SearchResponse<Row>
if (isExternalTable) {
span?.addTags({ searchType: "external" })
result = await external.search(options, table)
} else if (dbCore.isSqsEnabledForTenant()) {
span?.addTags({ searchType: "sqs" })
result = await internal.sqs.search(options, table)
} else {
span?.addTags({ searchType: "lucene" })
result = await internal.lucene.search(options, table)
}
span?.addTags({
foundRows: result.rows.length,
totalRows: result.totalRows,
})
return result
})
}
export async function exportRows(

36
packages/server/src/sdk/app/rows/search/internal/sqs.ts
View File

@ -2,6 +2,7 @@ import {
Datasource,
DocumentType,
FieldType,
isLogicalSearchOperator,
Operation,
QueryJson,
RelationshipFieldMetadata,
@ -137,20 +138,33 @@ function cleanupFilters(
allTables.some(table => table.schema[key])
const splitter = new dataFilters.ColumnSplitter(allTables)
for (const filter of Object.values(filters)) {
for (const key of Object.keys(filter)) {
const { numberPrefix, relationshipPrefix, column } = splitter.run(key)
if (keyInAnyTable(column)) {
filter[
`${numberPrefix || ""}${relationshipPrefix || ""}${mapToUserColumn(
column
)}`
] = filter[key]
delete filter[key]
const prefixFilters = (filters: SearchFilters) => {
for (const filterKey of Object.keys(filters) as (keyof SearchFilters)[]) {
if (isLogicalSearchOperator(filterKey)) {
for (const condition of filters[filterKey]!.conditions) {
prefixFilters(condition)
}
} else {
const filter = filters[filterKey]!
if (typeof filter !== "object") {
continue
}
for (const key of Object.keys(filter)) {
const { numberPrefix, relationshipPrefix, column } = splitter.run(key)
if (keyInAnyTable(column)) {
filter[
`${numberPrefix || ""}${
relationshipPrefix || ""
}${mapToUserColumn(column)}`
] = filter[key]
delete filter[key]
}
}
}
}
}
prefixFilters(filters)
return filters
}

52
packages/shared-core/src/filters.ts
View File

@ -17,6 +17,8 @@ import {
Table,
BasicOperator,
RangeOperator,
LogicalOperator,
isLogicalSearchOperator,
} from "@budibase/types"
import dayjs from "dayjs"
import { OperatorOptions, SqlNumberTypeRangeMap } from "./constants"
@ -358,6 +360,8 @@ export const buildQuery = (filter: SearchFilter[]) => {
high: value,
}
}
} else if (isLogicalSearchOperator(queryOperator)) {
// TODO
} else if (query[queryOperator] && operator !== "onEmptyFilter") {
if (type === "boolean") {
// Transform boolean filters to cope with null.
@ -458,14 +462,17 @@ export const runQuery = (docs: Record<string, any>[], query: SearchFilters) => {
) =>
(doc: Record<string, any>) => {
for (const [key, testValue] of Object.entries(query[type] || {})) {
const result = test(deepGet(doc, removeKeyNumbering(key)), testValue)
const valueToCheck = isLogicalSearchOperator(type)
? doc
: deepGet(doc, removeKeyNumbering(key))
const result = test(valueToCheck, testValue)
if (query.allOr && result) {
return true
} else if (!query.allOr && !result) {
return false
}
}
return true
return !query.allOr
}
const stringMatch = match(
@ -666,8 +673,45 @@ export const runQuery = (docs: Record<string, any>[], query: SearchFilters) => {
)
const containsAny = match(ArrayOperator.CONTAINS_ANY, _contains("some"))
const and = match(
LogicalOperator.AND,
(docValue: Record<string, any>, conditions: SearchFilters[]) => {
if (!conditions.length) {
return false
}
for (const condition of conditions) {
const matchesCondition = runQuery([docValue], condition)
if (!matchesCondition.length) {
return false
}
}
return true
}
)
const or = match(
LogicalOperator.OR,
(docValue: Record<string, any>, conditions: SearchFilters[]) => {
if (!conditions.length) {
return false
}
for (const condition of conditions) {
const matchesCondition = runQuery([docValue], {
...condition,
allOr: true,
})
if (matchesCondition.length) {
return true
}
}
return false
}
)
const docMatch = (doc: Record<string, any>) => {
const filterFunctions = {
const filterFunctions: Record<
SearchFilterOperator,
(doc: Record<string, any>) => boolean
> = {
string: stringMatch,
fuzzy: fuzzyMatch,
range: rangeMatch,
@ -679,6 +723,8 @@ export const runQuery = (docs: Record<string, any>[], query: SearchFilters) => {
contains: contains,
containsAny: containsAny,
notContains: notContains,
[LogicalOperator.AND]: and,
[LogicalOperator.OR]: or,
}
const results = Object.entries(query || {})

2
packages/shared-core/tsconfig.build.json
View File

@ -18,6 +18,6 @@
},
"tsBuildInfoFile": "dist/tsconfig.tsbuildinfo"
},
"include": ["src/**/*"],
"include": ["src/**/*.ts"],
"exclude": ["**/*.spec.ts", "**/*.spec.js", "__mocks__", "src/tests"]
}

3
packages/shared-core/tsconfig.json
View File

@ -1,9 +1,6 @@
{
"extends": "./tsconfig.build.json",
"compilerOptions": {
"baseUrl": "..",
"rootDir": "src",
"composite": true,
"types": ["node", "jest"]
},
"exclude": ["node_modules", "dist"]

24
packages/types/src/sdk/search.ts
View File

@ -23,7 +23,22 @@ export enum RangeOperator {
RANGE = "range",
}
export type SearchFilterOperator = BasicOperator | ArrayOperator | RangeOperator
export enum LogicalOperator {
AND = "$and",
OR = "$or",
}
export function isLogicalSearchOperator(
value: string
): value is LogicalOperator {
return value === LogicalOperator.AND || value === LogicalOperator.OR
}
export type SearchFilterOperator =
| BasicOperator
| ArrayOperator
| RangeOperator
| LogicalOperator
export enum InternalSearchFilterOperator {
COMPLEX_ID_OPERATOR = "_complexIdOperator",
@ -75,6 +90,13 @@ export interface SearchFilters {
// to make sure the documents returned are always filtered down to a
// specific document type (such as just rows)
documentType?: DocumentType
[LogicalOperator.AND]?: {
conditions: SearchFilters[]
}
[LogicalOperator.OR]?: {
conditions: SearchFilters[]
}
}
export type SearchFilterKey = keyof Omit<

6
packages/worker/src/api/controllers/global/self.ts
View File

@ -1,6 +1,6 @@
import * as userSdk from "../../../sdk/users"
import {
featureFlags,
features,
tenancy,
db as dbCore,
utils,
@ -104,8 +104,8 @@ export async function getSelf(ctx: any) {
ctx.body = await groups.enrichUserRolesFromGroups(user)
// add the feature flags for this tenant
const tenantId = tenancy.getTenantId()
ctx.body.featureFlags = featureFlags.getTenantFeatureFlags(tenantId)
const flags = await features.fetch()
ctx.body.flags = flags
addSessionAttributesToUser(ctx)
}

2
packages/worker/src/environment.ts
View File

@ -19,8 +19,6 @@ function parseIntSafe(number: any) {
}
const environment = {
// features
WORKER_FEATURES: process.env.WORKER_FEATURES,
// auth
MINIO_ACCESS_KEY: process.env.MINIO_ACCESS_KEY,
MINIO_SECRET_KEY: process.env.MINIO_SECRET_KEY,

13
packages/worker/src/features.ts
View File

@ -1,13 +0,0 @@
import { features } from "@budibase/backend-core"
import env from "./environment"
enum WorkerFeature {}
const featureList: WorkerFeature[] = features.processFeatureEnvVar(
Object.values(WorkerFeature),
env.WORKER_FEATURES
)
export function isFeatureEnabled(feature: WorkerFeature) {
return featureList.includes(feature)
}

8
yarn.lock
View File

@ -12072,10 +12072,10 @@ google-p12-pem@^4.0.0:
dependencies:
node-forge "^1.3.1"
"google-spreadsheet@npm:@budibase/google-spreadsheet@4.1.2":
version "4.1.2"
resolved "https://registry.yarnpkg.com/@budibase/google-spreadsheet/-/google-spreadsheet-4.1.2.tgz#90548ccba2284b3042b08d2974ef3caeaf772ad9"
integrity sha512-dxoY3rQGGnuNeZiXhNc9oYPduzU8xnIjWujFwNvaRRv3zWeUV7mj6HE2o/OJOeekPGt7o44B+w6DfkiaoteZgg==
"google-spreadsheet@npm:@budibase/google-spreadsheet@4.1.3":
version "4.1.3"
resolved "https://registry.yarnpkg.com/@budibase/google-spreadsheet/-/google-spreadsheet-4.1.3.tgz#bcee7bd9d90f82c54b16a9aca963b87aceb050ad"
integrity sha512-03VX3/K5NXIh6+XAIDZgcHPmR76xwd8vIDL7RedMpvM2IcXK0Iq/KU7FmLY0t/mKqORAGC7+0rajd0jLFezC4w==
dependencies:
axios "^1.4.0"
lodash "^4.17.21"