MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' of github.com:Budibase/budibase into grid-layout-improved

This commit is contained in:
Andrew Kingston 2024-08-08 16:07:23 +01:00
parent 8e04e85df2 f3f6f8d9b0
commit 91120c673c
No known key found for this signature in database
34 changed files with 854 additions and 318 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.github/workflows/budibase_ci.yml vendored
View File

@ -147,7 +147,7 @@ jobs:
fi
test-server:
runs-on: budi-tubby-tornado-quad-core-150gb
runs-on: budi-tubby-tornado-quad-core-300gb
steps:
- name: Checkout repo
uses: actions/checkout@v4

2
README.md
View File

@ -116,7 +116,7 @@ As with anything that we build in Budibase, our new public API is simple to use,
You can learn more about the Budibase API at the following places:
- [General documentation](https://docs.budibase.com/docs/public-api): Learn how to get your API key, how to use spec, and how to use Postman
- [Interactive API documentation](https://docs.budibase.com/reference/post_applications) : Learn how to interact with the API
- [Interactive API documentation](https://docs.budibase.com/reference/appcreate) : Learn how to interact with the API
<br /><br />

6
hosting/single/runner.sh
View File

@ -80,7 +80,11 @@ ln -s ${DATA_DIR}/.env /worker/.env
# make these directories in runner, incase of mount
mkdir -p ${DATA_DIR}/minio
chown -R couchdb:couchdb ${DATA_DIR}/couch
redis-server --requirepass $REDIS_PASSWORD > /dev/stdout 2>&1 &
if [[ -n "${REDIS_PASSWORD}" ]]; then
redis-server --requirepass $REDIS_PASSWORD > /dev/stdout 2>&1 &
else
redis-server > /dev/stdout 2>&1 &
fi
/bbcouch-runner.sh &
# only start minio if use s3 isn't passed

2
i18n/README.es.md
View File

@ -144,7 +144,7 @@ del sistema. Budibase API ofrece:
Puedes aprender mas acerca de Budibase API en los siguientes documentos:
- [Documentacion general](https://docs.budibase.com/docs/public-api) : Como optener tu clave para la API, usar Insomnia y Postman
- [API Interactiva](https://docs.budibase.com/reference/post_applications) : Aprende como trabajar con la API
- [API Interactiva](https://docs.budibase.com/reference/appcreate) : Aprende como trabajar con la API
#### Guias

2
lerna.json
View File

@ -1,6 +1,6 @@
{
"$schema": "node_modules/lerna/schemas/lerna-schema.json",
"version": "2.29.29",
"version": "2.30.2",
"npmClient": "yarn",
"packages": [
"packages/*",

151
packages/backend-core/src/features/index.ts
View File

@ -1,79 +1,108 @@
import env from "../environment"
import * as context from "../context"
import { cloneDeep } from "lodash"
export * from "./installation"
/**
* Read the TENANT_FEATURE_FLAGS env var and return an array of features flags for each tenant.
* The env var is formatted as:
* tenant1:feature1:feature2,tenant2:feature1
*/
export function buildFeatureFlags() {
if (!env.TENANT_FEATURE_FLAGS) {
return
class Flag<T> {
static withDefault<T>(value: T) {
return new Flag(value)
}
const tenantFeatureFlags: Record<string, string[]> = {}
private constructor(public defaultValue: T) {}
}
env.TENANT_FEATURE_FLAGS.split(",").forEach(tenantToFeatures => {
const [tenantId, ...features] = tenantToFeatures.split(":")
// This is the primary source of truth for feature flags. If you want to add a
// new flag, add it here and use the `fetch` and `get` functions to access it.
// All of the machinery in this file is to make sure that flags have their
// default values set correctly and their types flow through the system.
const FLAGS = {
LICENSING: Flag.withDefault(false),
GOOGLE_SHEETS: Flag.withDefault(false),
USER_GROUPS: Flag.withDefault(false),
ONBOARDING_TOUR: Flag.withDefault(false),
}
features.forEach(feature => {
if (!tenantFeatureFlags[tenantId]) {
tenantFeatureFlags[tenantId] = []
const DEFAULTS = Object.keys(FLAGS).reduce((acc, key) => {
const typedKey = key as keyof typeof FLAGS
// @ts-ignore
acc[typedKey] = FLAGS[typedKey].defaultValue
return acc
}, {} as Flags)
type UnwrapFlag<F> = F extends Flag<infer U> ? U : never
export type Flags = {
[K in keyof typeof FLAGS]: UnwrapFlag<(typeof FLAGS)[K]>
}
// Exported for use in tests, should not be used outside of this file.
export function defaultFlags(): Flags {
return cloneDeep(DEFAULTS)
}
function isFlagName(name: string): name is keyof Flags {
return FLAGS[name as keyof typeof FLAGS] !== undefined
}
/**
* Reads the TENANT_FEATURE_FLAGS environment variable and returns a Flags object
* populated with the flags for the current tenant, filling in the default values
* if the flag is not set.
*
* Check the tests for examples of how TENANT_FEATURE_FLAGS should be formatted.
*
* In future we plan to add more ways of setting feature flags, e.g. PostHog, and
* they will be accessed through this function as well.
*/
export async function fetch(): Promise<Flags> {
const currentTenantId = context.getTenantId()
const flags = defaultFlags()
const split = (env.TENANT_FEATURE_FLAGS || "")
.split(",")
.map(x => x.split(":"))
for (const [tenantId, ...features] of split) {
if (!tenantId || (tenantId !== "*" && tenantId !== currentTenantId)) {
continue
}
for (let feature of features) {
let value = true
if (feature.startsWith("!")) {
feature = feature.slice(1)
value = false
}
tenantFeatureFlags[tenantId].push(feature)
})
})
return tenantFeatureFlags
}
if (!isFlagName(feature)) {
throw new Error(`Feature: ${feature} is not an allowed option`)
}
export function isEnabled(featureFlag: string) {
const tenantId = context.getTenantId()
const flags = getTenantFeatureFlags(tenantId)
return flags.includes(featureFlag)
}
if (typeof flags[feature] !== "boolean") {
throw new Error(`Feature: ${feature} is not a boolean`)
}
export function getTenantFeatureFlags(tenantId: string) {
let flags: string[] = []
const envFlags = buildFeatureFlags()
if (envFlags) {
const globalFlags = envFlags["*"]
const tenantFlags = envFlags[tenantId] || []
// Explicitly exclude tenants from global features if required.
// Prefix the tenant flag with '!'
const tenantOverrides = tenantFlags.reduce(
(acc: string[], flag: string) => {
if (flag.startsWith("!")) {
let stripped = flag.substring(1)
acc.push(stripped)
}
return acc
},
[]
)
if (globalFlags) {
flags.push(...globalFlags)
// @ts-ignore
flags[feature] = value
}
if (tenantFlags.length) {
flags.push(...tenantFlags)
}
// Purge any tenant specific overrides
flags = flags.filter(flag => {
return tenantOverrides.indexOf(flag) == -1 && !flag.startsWith("!")
})
}
return flags
}
export enum TenantFeatureFlag {
LICENSING = "LICENSING",
GOOGLE_SHEETS = "GOOGLE_SHEETS",
USER_GROUPS = "USER_GROUPS",
ONBOARDING_TOUR = "ONBOARDING_TOUR",
// Gets a single feature flag value. This is a convenience function for
// `fetch().then(flags => flags[name])`.
export async function get<K extends keyof Flags>(name: K): Promise<Flags[K]> {
const flags = await fetch()
return flags[name]
}
type BooleanFlags = {
[K in keyof typeof FLAGS]: (typeof FLAGS)[K] extends Flag<boolean> ? K : never
}[keyof typeof FLAGS]
// Convenience function for boolean flag values. This makes callsites more
// readable for boolean flags.
export async function isEnabled<K extends BooleanFlags>(
name: K
): Promise<boolean> {
const flags = await fetch()
return flags[name]
}

17
packages/backend-core/src/features/installation.ts
View File

@ -1,17 +0,0 @@
export function processFeatureEnvVar<T>(
fullList: string[],
featureList?: string
) {
let list
if (!featureList) {
list = fullList
} else {
list = featureList.split(",")
}
for (let feature of list) {
if (!fullList.includes(feature)) {
throw new Error(`Feature: ${feature} is not an allowed option`)
}
}
return list as unknown as T[]
}

85
packages/backend-core/src/features/tests/featureFlags.spec.ts
View File

@ -1,85 +0,0 @@
import {
TenantFeatureFlag,
buildFeatureFlags,
getTenantFeatureFlags,
} from "../"
import env from "../../environment"
const { ONBOARDING_TOUR, LICENSING, USER_GROUPS } = TenantFeatureFlag
describe("featureFlags", () => {
beforeEach(() => {
env._set("TENANT_FEATURE_FLAGS", "")
})
it("Should return no flags when the TENANT_FEATURE_FLAG is empty", async () => {
let features = buildFeatureFlags()
expect(features).toBeUndefined()
})
it("Should generate a map of global and named tenant feature flags from the env value", async () => {
env._set(
"TENANT_FEATURE_FLAGS",
`*:${ONBOARDING_TOUR},tenant1:!${ONBOARDING_TOUR},tenant2:${USER_GROUPS},tenant1:${LICENSING}`
)
const parsedFlags: Record<string, string[]> = {
"*": [ONBOARDING_TOUR],
tenant1: [`!${ONBOARDING_TOUR}`, LICENSING],
tenant2: [USER_GROUPS],
}
let features = buildFeatureFlags()
expect(features).toBeDefined()
expect(features).toEqual(parsedFlags)
})
it("Should add feature flag flag only to explicitly configured tenant", async () => {
env._set(
"TENANT_FEATURE_FLAGS",
`*:${LICENSING},*:${USER_GROUPS},tenant1:${ONBOARDING_TOUR}`
)
let tenant1Flags = getTenantFeatureFlags("tenant1")
let tenant2Flags = getTenantFeatureFlags("tenant2")
expect(tenant1Flags).toBeDefined()
expect(tenant1Flags).toEqual([LICENSING, USER_GROUPS, ONBOARDING_TOUR])
expect(tenant2Flags).toBeDefined()
expect(tenant2Flags).toEqual([LICENSING, USER_GROUPS])
})
})
it("Should exclude tenant1 from global feature flag", async () => {
env._set(
"TENANT_FEATURE_FLAGS",
`*:${LICENSING},*:${ONBOARDING_TOUR},tenant1:!${ONBOARDING_TOUR}`
)
let tenant1Flags = getTenantFeatureFlags("tenant1")
let tenant2Flags = getTenantFeatureFlags("tenant2")
expect(tenant1Flags).toBeDefined()
expect(tenant1Flags).toEqual([LICENSING])
expect(tenant2Flags).toBeDefined()
expect(tenant2Flags).toEqual([LICENSING, ONBOARDING_TOUR])
})
it("Should explicitly add flags to configured tenants only", async () => {
env._set(
"TENANT_FEATURE_FLAGS",
`tenant1:${ONBOARDING_TOUR},tenant1:${LICENSING},tenant2:${LICENSING}`
)
let tenant1Flags = getTenantFeatureFlags("tenant1")
let tenant2Flags = getTenantFeatureFlags("tenant2")
expect(tenant1Flags).toBeDefined()
expect(tenant1Flags).toEqual([ONBOARDING_TOUR, LICENSING])
expect(tenant2Flags).toBeDefined()
expect(tenant2Flags).toEqual([LICENSING])
})

86
packages/backend-core/src/features/tests/features.spec.ts Normal file
View File

@ -0,0 +1,86 @@
import { defaultFlags, fetch, get, Flags } from "../"
import { context } from "../.."
import env from "../../environment"
async function withFlags<T>(flags: string, f: () => T): Promise<T> {
const oldFlags = env.TENANT_FEATURE_FLAGS
env._set("TENANT_FEATURE_FLAGS", flags)
try {
return await f()
} finally {
env._set("TENANT_FEATURE_FLAGS", oldFlags)
}
}
describe("feature flags", () => {
interface TestCase {
tenant: string
flags: string
expected: Partial<Flags>
}
it.each<TestCase>([
{
tenant: "tenant1",
flags: "tenant1:ONBOARDING_TOUR",
expected: { ONBOARDING_TOUR: true },
},
{
tenant: "tenant1",
flags: "tenant1:!ONBOARDING_TOUR",
expected: { ONBOARDING_TOUR: false },
},
{
tenant: "tenant1",
flags: "*:ONBOARDING_TOUR",
expected: { ONBOARDING_TOUR: true },
},
{
tenant: "tenant1",
flags: "tenant2:ONBOARDING_TOUR",
expected: { ONBOARDING_TOUR: false },
},
{
tenant: "tenant1",
flags: "",
expected: defaultFlags(),
},
])(
'should find flags $expected for $tenant with string "$flags"',
({ tenant, flags, expected }) =>
context.doInTenant(tenant, () =>
withFlags(flags, async () => {
const flags = await fetch()
expect(flags).toMatchObject(expected)
for (const [key, expectedValue] of Object.entries(expected)) {
const value = await get(key as keyof Flags)
expect(value).toBe(expectedValue)
}
})
)
)
interface FailedTestCase {
tenant: string
flags: string
expected: string | RegExp
}
it.each<FailedTestCase>([
{
tenant: "tenant1",
flags: "tenant1:ONBOARDING_TOUR,tenant1:FOO",
expected: "Feature: FOO is not an allowed option",
},
])(
"should fail with message \"$expected\" for $tenant with string '$flags'",
async ({ tenant, flags, expected }) => {
context.doInTenant(tenant, () =>
withFlags(flags, async () => {
await expect(fetch()).rejects.toThrow(expected)
})
)
}
)
})

3
packages/backend-core/src/index.ts
View File

@ -7,8 +7,7 @@ export * as roles from "./security/roles"
export * as permissions from "./security/permissions"
export * as accounts from "./accounts"
export * as installation from "./installation"
export * as featureFlags from "./features"
export * as features from "./features/installation"
export * as features from "./features"
export * as sessions from "./security/sessions"
export * as platform from "./platform"
export * as auth from "./auth"

20
packages/backend-core/src/sql/sql.ts
View File

@ -463,6 +463,24 @@ class InternalBuilder {
}
}
if (filters.$and) {
const { $and } = filters
query = query.where(x => {
for (const condition of $and.conditions) {
x = this.addFilters(x, condition, opts)
}
})
}
if (filters.$or) {
const { $or } = filters
query = query.where(x => {
for (const condition of $or.conditions) {
x = this.addFilters(x, { ...condition, allOr: true }, opts)
}
})
}
if (filters.oneOf) {
const fnc = allOr ? "orWhereIn" : "whereIn"
iterate(
@ -682,6 +700,8 @@ class InternalBuilder {
const direction =
value.direction === SortOrder.ASCENDING ? "asc" : "desc"
// TODO: figure out a way to remove this conditional, not relying on
// the defaults of each datastore.
let nulls: "first" | "last" | undefined = undefined
if (
this.client === SqlClient.POSTGRES ||

3
packages/builder/src/helpers/featureFlags.js
View File

@ -5,9 +5,10 @@ export const TENANT_FEATURE_FLAGS = {
LICENSING: "LICENSING",
USER_GROUPS: "USER_GROUPS",
ONBOARDING_TOUR: "ONBOARDING_TOUR",
GOOGLE_SHEETS: "GOOGLE_SHEETS",
}
export const isEnabled = featureFlag => {
const user = get(auth).user
return !!user?.featureFlags?.includes(featureFlag)
return !!user?.flags?.[featureFlag]
}

2
packages/pro

@ -1 +1 @@
Subproject commit 7dbe323aec724ae6336b13c06aaefa4a89837edf
Subproject commit 62ef0e2d6e83522b6732fb3c61338de303f06ff0

2
packages/server/package.json
View File

@ -80,7 +80,7 @@
"dotenv": "8.2.0",
"form-data": "4.0.0",
"global-agent": "3.0.0",
"google-spreadsheet": "npm:@budibase/google-spreadsheet@4.1.2",
"google-spreadsheet": "npm:@budibase/google-spreadsheet@4.1.3",
"ioredis": "5.3.2",
"isolated-vm": "^4.7.2",
"jimp": "0.22.12",

38
packages/server/src/api/controllers/row/views.ts
View File

@ -6,11 +6,13 @@ import {
RequiredKeys,
RowSearchParams,
SearchFilterKey,
LogicalOperator,
} from "@budibase/types"
import { dataFilters } from "@budibase/shared-core"
import sdk from "../../../sdk"
import { db, context } from "@budibase/backend-core"
import { enrichSearchContext } from "./utils"
import { isExternalTableID } from "../../../integrations/utils"
export async function searchView(
ctx: UserCtx<SearchViewRowRequest, SearchRowResponse>
@ -35,25 +37,33 @@ export async function searchView(
// that could let users find rows they should not be allowed to access.
let query = dataFilters.buildQuery(view.query || [])
if (body.query) {
// Extract existing fields
const existingFields =
view.query
?.filter(filter => filter.field)
.map(filter => db.removeKeyNumbering(filter.field)) || []
// Delete extraneous search params that cannot be overridden
delete body.query.allOr
delete body.query.onEmptyFilter
// Carry over filters for unused fields
Object.keys(body.query).forEach(key => {
const operator = key as SearchFilterKey
Object.keys(body.query[operator] || {}).forEach(field => {
if (!existingFields.includes(db.removeKeyNumbering(field))) {
query[operator]![field] = body.query[operator]![field]
}
if (!isExternalTableID(view.tableId) && !db.isSqsEnabledForTenant()) {
// Extract existing fields
const existingFields =
view.query
?.filter(filter => filter.field)
.map(filter => db.removeKeyNumbering(filter.field)) || []
// Carry over filters for unused fields
Object.keys(body.query).forEach(key => {
const operator = key as Exclude<SearchFilterKey, LogicalOperator>
Object.keys(body.query[operator] || {}).forEach(field => {
if (!existingFields.includes(db.removeKeyNumbering(field))) {
query[operator]![field] = body.query[operator]![field]
}
})
})
})
} else {
query = {
$and: {
conditions: [query, body.query],
},
}
}
}
await context.ensureSnippetContext(true)

2
packages/server/src/api/controllers/static/index.ts
View File

@ -304,7 +304,7 @@ export const getSignedUploadURL = async function (ctx: Ctx) {
try {
const s3 = new AWS.S3({
region: awsRegion,
endpoint: datasource?.config?.endpoint as string,
endpoint: datasource?.config?.endpoint || undefined,
accessKeyId: datasource?.config?.accessKeyId as string,
secretAccessKey: datasource?.config?.secretAccessKey as string,
apiVersion: "2006-03-01",

71
packages/server/src/api/routes/tests/datasource.spec.ts
View File

@ -17,9 +17,14 @@ import {
SupportedSqlTypes,
JsonFieldSubType,
} from "@budibase/types"
import { DatabaseName, getDatasource } from "../../../integrations/tests/utils"
import {
DatabaseName,
getDatasource,
knexClient,
} from "../../../integrations/tests/utils"
import { tableForDatasource } from "../../../tests/utilities/structures"
import nock from "nock"
import { Knex } from "knex"
describe("/datasources", () => {
const config = setup.getConfig()
@ -164,11 +169,15 @@ describe("/datasources", () => {
[DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)],
[DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)],
[DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)],
[DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)],
])("%s", (_, dsProvider) => {
let rawDatasource: Datasource
let client: Knex
beforeEach(async () => {
rawDatasource = await dsProvider
datasource = await config.api.datasource.create(rawDatasource)
client = await knexClient(rawDatasource)
})
describe("get", () => {
@ -285,9 +294,6 @@ describe("/datasources", () => {
[FieldType.STRING]: {
name: stringName,
type: FieldType.STRING,
constraints: {
presence: true,
},
},
[FieldType.LONGFORM]: {
name: "longform",
@ -381,10 +387,6 @@ describe("/datasources", () => {
),
schema: Object.entries(table.schema).reduce<TableSchema>(
(acc, [fieldName, field]) => {
// the constraint will be unset - as the DB doesn't recognise it as not null
if (fieldName === stringName) {
field.constraints = {}
}
acc[fieldName] = expect.objectContaining({
...field,
})
@ -441,20 +443,49 @@ describe("/datasources", () => {
})
describe("info", () => {
it("should fetch information about postgres datasource", async () => {
const table = await config.api.table.save(
tableForDatasource(datasource, {
schema: {
name: {
name: "name",
type: FieldType.STRING,
},
},
})
)
it("should fetch information about a datasource with a single table", async () => {
const existingTableNames = (
await config.api.datasource.info(datasource)
).tableNames
const tableName = generator.guid()
await client.schema.createTable(tableName, table => {
table.increments("id").primary()
table.string("name")
})
const info = await config.api.datasource.info(datasource)
expect(info.tableNames).toContain(table.name)
expect(info.tableNames).toEqual(
expect.arrayContaining([tableName, ...existingTableNames])
)
expect(info.tableNames).toHaveLength(existingTableNames.length + 1)
})
it("should fetch information about a datasource with multiple tables", async () => {
const existingTableNames = (
await config.api.datasource.info(datasource)
).tableNames
const tableNames = [
generator.guid(),
generator.guid(),
generator.guid(),
generator.guid(),
]
for (const tableName of tableNames) {
await client.schema.createTable(tableName, table => {
table.increments("id").primary()
table.string("name")
})
}
const info = await config.api.datasource.info(datasource)
expect(info.tableNames).toEqual(
expect.arrayContaining([...tableNames, ...existingTableNames])
)
expect(info.tableNames).toHaveLength(
existingTableNames.length + tableNames.length
)
})
})
})

238
packages/server/src/api/routes/tests/search.spec.ts
View File

@ -1596,9 +1596,6 @@ describe.each([
// Our bigints are int64s in most datasources.
let BIG = "9223372036854775807"
if (name === DatabaseName.ORACLE) {
// BIG = "9223372036854775808"
}
beforeAll(async () => {
table = await createTable({
@ -2696,4 +2693,239 @@ describe.each([
)
})
})
!isLucene &&
describe("$and", () => {
beforeAll(async () => {
table = await createTable({
age: { name: "age", type: FieldType.NUMBER },
name: { name: "name", type: FieldType.STRING },
})
await createRows([
{ age: 1, name: "Jane" },
{ age: 10, name: "Jack" },
{ age: 7, name: "Hanna" },
{ age: 8, name: "Jan" },
])
})
it("successfully finds a row for one level condition", async () => {
await expectQuery({
$and: {
conditions: [{ equal: { age: 10 } }, { equal: { name: "Jack" } }],
},
}).toContainExactly([{ age: 10, name: "Jack" }])
})
it("successfully finds a row for one level with multiple conditions", async () => {
await expectQuery({
$and: {
conditions: [{ equal: { age: 10 } }, { equal: { name: "Jack" } }],
},
}).toContainExactly([{ age: 10, name: "Jack" }])
})
it("successfully finds multiple rows for one level with multiple conditions", async () => {
await expectQuery({
$and: {
conditions: [
{ range: { age: { low: 1, high: 9 } } },
{ string: { name: "Ja" } },
],
},
}).toContainExactly([
{ age: 1, name: "Jane" },
{ age: 8, name: "Jan" },
])
})
it("successfully finds rows for nested filters", async () => {
await expectQuery({
$and: {
conditions: [
{
$and: {
conditions: [
{
range: { age: { low: 1, high: 10 } },
},
{ string: { name: "Ja" } },
],
},
equal: { name: "Jane" },
},
],
},
}).toContainExactly([{ age: 1, name: "Jane" }])
})
it("returns nothing when filtering out all data", async () => {
await expectQuery({
$and: {
conditions: [{ equal: { age: 7 } }, { equal: { name: "Jack" } }],
},
}).toFindNothing()
})
!isInMemory &&
it("validates conditions that are not objects", async () => {
await expect(
expectQuery({
$and: {
conditions: [{ equal: { age: 10 } }, "invalidCondition" as any],
},
}).toFindNothing()
).rejects.toThrow(
'Invalid body - "query.$and.conditions[1]" must be of type object'
)
})
!isInMemory &&
it("validates $and without conditions", async () => {
await expect(
expectQuery({
$and: {
conditions: [
{ equal: { age: 10 } },
{
$and: {
conditions: undefined as any,
},
},
],
},
}).toFindNothing()
).rejects.toThrow(
'Invalid body - "query.$and.conditions[1].$and.conditions" is required'
)
})
})
!isLucene &&
describe("$or", () => {
beforeAll(async () => {
table = await createTable({
age: { name: "age", type: FieldType.NUMBER },
name: { name: "name", type: FieldType.STRING },
})
await createRows([
{ age: 1, name: "Jane" },
{ age: 10, name: "Jack" },
{ age: 7, name: "Hanna" },
{ age: 8, name: "Jan" },
])
})
it("successfully finds a row for one level condition", async () => {
await expectQuery({
$or: {
conditions: [{ equal: { age: 7 } }, { equal: { name: "Jack" } }],
},
}).toContainExactly([
{ age: 10, name: "Jack" },
{ age: 7, name: "Hanna" },
])
})
it("successfully finds a row for one level with multiple conditions", async () => {
await expectQuery({
$or: {
conditions: [{ equal: { age: 7 } }, { equal: { name: "Jack" } }],
},
}).toContainExactly([
{ age: 10, name: "Jack" },
{ age: 7, name: "Hanna" },
])
})
it("successfully finds multiple rows for one level with multiple conditions", async () => {
await expectQuery({
$or: {
conditions: [
{ range: { age: { low: 1, high: 9 } } },
{ string: { name: "Jan" } },
],
},
}).toContainExactly([
{ age: 1, name: "Jane" },
{ age: 7, name: "Hanna" },
{ age: 8, name: "Jan" },
])
})
it("successfully finds rows for nested filters", async () => {
await expectQuery({
$or: {
conditions: [
{
$or: {
conditions: [
{
range: { age: { low: 1, high: 7 } },
},
{ string: { name: "Jan" } },
],
},
equal: { name: "Jane" },
},
],
},
}).toContainExactly([
{ age: 1, name: "Jane" },
{ age: 7, name: "Hanna" },
{ age: 8, name: "Jan" },
])
})
it("returns nothing when filtering out all data", async () => {
await expectQuery({
$or: {
conditions: [{ equal: { age: 6 } }, { equal: { name: "John" } }],
},
}).toFindNothing()
})
it("can nest $and under $or filters", async () => {
await expectQuery({
$or: {
conditions: [
{
$and: {
conditions: [
{
range: { age: { low: 1, high: 8 } },
},
{ equal: { name: "Jan" } },
],
},
equal: { name: "Jane" },
},
],
},
}).toContainExactly([
{ age: 1, name: "Jane" },
{ age: 8, name: "Jan" },
])
})
it("can nest $or under $and filters", async () => {
await expectQuery({
$and: {
conditions: [
{
$or: {
conditions: [
{
range: { age: { low: 1, high: 8 } },
},
{ equal: { name: "Jan" } },
],
},
equal: { name: "Jane" },
},
],
},
}).toContainExactly([{ age: 1, name: "Jane" }])
})
})
})

113
packages/server/src/api/routes/tests/viewV2.spec.ts
View File

@ -1485,6 +1485,119 @@ describe.each([
}
)
})
isLucene &&
it("in lucene, cannot override a view filter", async () => {
await config.api.row.save(table._id!, {
one: "foo",
two: "bar",
})
const two = await config.api.row.save(table._id!, {
one: "foo2",
two: "bar2",
})
const view = await config.api.viewV2.create({
tableId: table._id!,
name: generator.guid(),
query: [
{
operator: BasicOperator.EQUAL,
field: "two",
value: "bar2",
},
],
schema: {
id: { visible: true },
one: { visible: false },
two: { visible: true },
},
})
const response = await config.api.viewV2.search(view.id, {
query: {
equal: {
two: "bar",
},
},
})
expect(response.rows).toHaveLength(1)
expect(response.rows).toEqual([
expect.objectContaining({ _id: two._id }),
])
})
!isLucene &&
it("can filter a view without a view filter", async () => {
const one = await config.api.row.save(table._id!, {
one: "foo",
two: "bar",
})
await config.api.row.save(table._id!, {
one: "foo2",
two: "bar2",
})
const view = await config.api.viewV2.create({
tableId: table._id!,
name: generator.guid(),
schema: {
id: { visible: true },
one: { visible: false },
two: { visible: true },
},
})
const response = await config.api.viewV2.search(view.id, {
query: {
equal: {
two: "bar",
},
},
})
expect(response.rows).toHaveLength(1)
expect(response.rows).toEqual([
expect.objectContaining({ _id: one._id }),
])
})
!isLucene &&
it("cannot bypass a view filter", async () => {
await config.api.row.save(table._id!, {
one: "foo",
two: "bar",
})
await config.api.row.save(table._id!, {
one: "foo2",
two: "bar2",
})
const view = await config.api.viewV2.create({
tableId: table._id!,
name: generator.guid(),
query: [
{
operator: BasicOperator.EQUAL,
field: "two",
value: "bar2",
},
],
schema: {
id: { visible: true },
one: { visible: false },
two: { visible: true },
},
})
const response = await config.api.viewV2.search(view.id, {
query: {
equal: {
two: "bar",
},
},
})
expect(response.rows).toHaveLength(0)
})
})
describe("permissions", () => {

25
packages/server/src/api/routes/utils/validators.ts
View File

@ -1,6 +1,11 @@
import { auth, permissions } from "@budibase/backend-core"
import { DataSourceOperation } from "../../../constants"
import { Table, WebhookActionType } from "@budibase/types"
import {
EmptyFilterOption,
SearchFilters,
Table,
WebhookActionType,
} from "@budibase/types"
import Joi, { CustomValidator } from "joi"
import { ValidSnippetNameRegex, helpers } from "@budibase/shared-core"
import sdk from "../../../sdk"
@ -84,7 +89,12 @@ export function datasourceValidator() {
}
function filterObject() {
return Joi.object({
const conditionalFilteringObject = () =>
Joi.object({
conditions: Joi.array().items(Joi.link("#schema")).required(),
})
const filtersValidators: Record<keyof SearchFilters, any> = {
string: Joi.object().optional(),
fuzzy: Joi.object().optional(),
range: Joi.object().optional(),
@ -95,8 +105,17 @@ function filterObject() {
oneOf: Joi.object().optional(),
contains: Joi.object().optional(),
notContains: Joi.object().optional(),
containsAny: Joi.object().optional(),
allOr: Joi.boolean().optional(),
}).unknown(true)
onEmptyFilter: Joi.string()
.optional()
.valid(...Object.values(EmptyFilterOption)),
$and: conditionalFilteringObject(),
$or: conditionalFilteringObject(),
fuzzyOr: Joi.forbidden(),
documentType: Joi.forbidden(),
}
return Joi.object(filtersValidators).unknown(true).id("schema")
}
export function internalSearchValidator() {

39
packages/server/src/automations/steps/queryRows.ts
View File

@ -11,13 +11,10 @@ import {
AutomationStepSchema,
AutomationStepType,
EmptyFilterOption,
SearchFilters,
Table,
SortOrder,
QueryRowsStepInputs,
QueryRowsStepOutputs,
} from "@budibase/types"
import { db as dbCore } from "@budibase/backend-core"
const SortOrderPretty = {
[SortOrder.ASCENDING]: "Ascending",
@ -95,38 +92,6 @@ async function getTable(appId: string, tableId: string) {
return ctx.body
}
function typeCoercion(filters: SearchFilters, table: Table) {
if (!filters || !table) {
return filters
}
for (let key of Object.keys(filters)) {
const searchParam = filters[key as keyof SearchFilters]
if (typeof searchParam === "object") {
for (let [property, value] of Object.entries(searchParam)) {
// We need to strip numerical prefixes here, so that we can look up
// the correct field name in the schema
const columnName = dbCore.removeKeyNumbering(property)
const column = table.schema[columnName]
// convert string inputs
if (!column || typeof value !== "string") {
continue
}
if (column.type === FieldType.NUMBER) {
if (key === "oneOf") {
searchParam[property] = value
.split(",")
.map(item => parseFloat(item))
} else {
searchParam[property] = parseFloat(value)
}
}
}
}
}
return filters
}
function hasNullFilters(filters: any[]) {
return (
filters.length === 0 ||
@ -157,7 +122,7 @@ export async function run({
sortType =
fieldType === FieldType.NUMBER ? FieldType.NUMBER : FieldType.STRING
}
const ctx: any = buildCtx(appId, null, {
const ctx = buildCtx(appId, null, {
params: {
tableId,
},
@ -165,7 +130,7 @@ export async function run({
sortType,
limit,
sort: sortColumn,
query: typeCoercion(filters || {}, table),
query: filters || {},
// default to ascending, like data tab
sortOrder: sortOrder || SortOrder.ASCENDING,
},

21
packages/server/src/features.ts
View File

@ -1,4 +1,3 @@
import { features } from "@budibase/backend-core"
import env from "./environment"
enum AppFeature {
@ -6,7 +5,25 @@ enum AppFeature {
AUTOMATIONS = "automations",
}
const featureList = features.processFeatureEnvVar<AppFeature>(
export function processFeatureEnvVar<T>(
fullList: string[],
featureList?: string
) {
let list
if (!featureList) {
list = fullList
} else {
list = featureList.split(",")
}
for (let feature of list) {
if (!fullList.includes(feature)) {
throw new Error(`Feature: ${feature} is not an allowed option`)
}
}
return list as unknown as T[]
}
const featureList = processFeatureEnvVar<AppFeature>(
Object.values(AppFeature),
env.APP_FEATURES
)

17
packages/server/src/integrations/oracle.ts
View File

@ -400,7 +400,9 @@ class OracleIntegration extends Sql implements DatasourcePlus {
if (oracleConstraint.type === OracleContraintTypes.PRIMARY) {
table.primary!.push(columnName)
} else if (
oracleConstraint.type === OracleContraintTypes.NOT_NULL_OR_CHECK
oracleConstraint.type ===
OracleContraintTypes.NOT_NULL_OR_CHECK &&
oracleConstraint.searchCondition?.endsWith("IS NOT NULL")
) {
table.schema[columnName].constraints = {
presence: true,
@ -421,7 +423,11 @@ class OracleIntegration extends Sql implements DatasourcePlus {
const columnsResponse = await this.internalQuery<OracleColumnsResponse>({
sql: OracleIntegration.COLUMNS_SQL,
})
return (columnsResponse.rows || []).map(row => row.TABLE_NAME)
const tableNames = new Set<string>()
for (const row of columnsResponse.rows || []) {
tableNames.add(row.TABLE_NAME)
}
return Array.from(tableNames)
}
async testConnection() {
@ -500,6 +506,13 @@ class OracleIntegration extends Sql implements DatasourcePlus {
password: this.config.password,
connectString,
}
// We set the timezone of the connection to match the timezone of the
// Budibase server, this is because several column types (e.g. time-only
// timestamps) do not store timezone information, so to avoid storing one
// time and getting a different one back we need to make sure the timezone
// of the server matches the timezone of the database. There's an assumption
// here that the server is running in the same timezone as the database.
const tz = Intl.DateTimeFormat().resolvedOptions().timeZone
const connection = await oracledb.getConnection(attributes)
await connection.execute(`ALTER SESSION SET TIME_ZONE = '${tz}'`)

78
packages/server/src/sdk/app/rows/search.ts
View File

@ -13,6 +13,7 @@ import { dataFilters } from "@budibase/shared-core"
import sdk from "../../index"
import { searchInputMapping } from "./search/utils"
import { db as dbCore } from "@budibase/backend-core"
import tracer from "dd-trace"
export { isValidFilter } from "../../../integrations/utils"
@ -32,32 +33,65 @@ function pickApi(tableId: any) {
export async function search(
options: RowSearchParams
): Promise<SearchResponse<Row>> {
const isExternalTable = isExternalTableID(options.tableId)
options.query = dataFilters.cleanupQuery(options.query || {})
options.query = dataFilters.fixupFilterArrays(options.query)
if (
!dataFilters.hasFilters(options.query) &&
options.query.onEmptyFilter === EmptyFilterOption.RETURN_NONE
) {
return {
rows: [],
return await tracer.trace("search", async span => {
span?.addTags({
tableId: options.tableId,
query: options.query,
sort: options.sort,
sortOrder: options.sortOrder,
sortType: options.sortType,
limit: options.limit,
bookmark: options.bookmark,
paginate: options.paginate,
fields: options.fields,
countRows: options.countRows,
})
const isExternalTable = isExternalTableID(options.tableId)
options.query = dataFilters.cleanupQuery(options.query || {})
options.query = dataFilters.fixupFilterArrays(options.query)
span?.addTags({
cleanedQuery: options.query,
isExternalTable,
})
if (
!dataFilters.hasFilters(options.query) &&
options.query.onEmptyFilter === EmptyFilterOption.RETURN_NONE
) {
span?.addTags({ emptyQuery: true })
return {
rows: [],
}
}
}
if (options.sortOrder) {
options.sortOrder = options.sortOrder.toLowerCase() as SortOrder
}
if (options.sortOrder) {
options.sortOrder = options.sortOrder.toLowerCase() as SortOrder
}
const table = await sdk.tables.getTable(options.tableId)
options = searchInputMapping(table, options)
const table = await sdk.tables.getTable(options.tableId)
options = searchInputMapping(table, options)
if (isExternalTable) {
return external.search(options, table)
} else if (dbCore.isSqsEnabledForTenant()) {
return internal.sqs.search(options, table)
} else {
return internal.lucene.search(options, table)
}
let result: SearchResponse<Row>
if (isExternalTable) {
span?.addTags({ searchType: "external" })
result = await external.search(options, table)
} else if (dbCore.isSqsEnabledForTenant()) {
span?.addTags({ searchType: "sqs" })
result = await internal.sqs.search(options, table)
} else {
span?.addTags({ searchType: "lucene" })
result = await internal.lucene.search(options, table)
}
span?.addTags({
foundRows: result.rows.length,
totalRows: result.totalRows,
})
return result
})
}
export async function exportRows(

36
packages/server/src/sdk/app/rows/search/internal/sqs.ts
View File

@ -2,6 +2,7 @@ import {
Datasource,
DocumentType,
FieldType,
isLogicalSearchOperator,
Operation,
QueryJson,
RelationshipFieldMetadata,
@ -137,20 +138,33 @@ function cleanupFilters(
allTables.some(table => table.schema[key])
const splitter = new dataFilters.ColumnSplitter(allTables)
for (const filter of Object.values(filters)) {
for (const key of Object.keys(filter)) {
const { numberPrefix, relationshipPrefix, column } = splitter.run(key)
if (keyInAnyTable(column)) {
filter[
`${numberPrefix || ""}${relationshipPrefix || ""}${mapToUserColumn(
column
)}`
] = filter[key]
delete filter[key]
const prefixFilters = (filters: SearchFilters) => {
for (const filterKey of Object.keys(filters) as (keyof SearchFilters)[]) {
if (isLogicalSearchOperator(filterKey)) {
for (const condition of filters[filterKey]!.conditions) {
prefixFilters(condition)
}
} else {
const filter = filters[filterKey]!
if (typeof filter !== "object") {
continue
}
for (const key of Object.keys(filter)) {
const { numberPrefix, relationshipPrefix, column } = splitter.run(key)
if (keyInAnyTable(column)) {
filter[
`${numberPrefix || ""}${
relationshipPrefix || ""
}${mapToUserColumn(column)}`
] = filter[key]
delete filter[key]
}
}
}
}
}
prefixFilters(filters)
return filters
}

1
packages/shared-core/src/constants/index.ts
View File

@ -182,6 +182,7 @@ export enum BpmStatusValue {
COMPLETING_ACCOUNT_INFO = "completing_account_info",
VERIFYING_EMAIL = "verifying_email",
COMPLETED = "completed",
FAILED = "failed",
}
export const DEFAULT_BB_DATASOURCE_ID = "datasource_internal_bb_default"

52
packages/shared-core/src/filters.ts
View File

@ -17,6 +17,8 @@ import {
Table,
BasicOperator,
RangeOperator,
LogicalOperator,
isLogicalSearchOperator,
} from "@budibase/types"
import dayjs from "dayjs"
import { OperatorOptions, SqlNumberTypeRangeMap } from "./constants"
@ -358,6 +360,8 @@ export const buildQuery = (filter: SearchFilter[]) => {
high: value,
}
}
} else if (isLogicalSearchOperator(queryOperator)) {
// TODO
} else if (query[queryOperator] && operator !== "onEmptyFilter") {
if (type === "boolean") {
// Transform boolean filters to cope with null.
@ -458,14 +462,17 @@ export const runQuery = (docs: Record<string, any>[], query: SearchFilters) => {
) =>
(doc: Record<string, any>) => {
for (const [key, testValue] of Object.entries(query[type] || {})) {
const result = test(deepGet(doc, removeKeyNumbering(key)), testValue)
const valueToCheck = isLogicalSearchOperator(type)
? doc
: deepGet(doc, removeKeyNumbering(key))
const result = test(valueToCheck, testValue)
if (query.allOr && result) {
return true
} else if (!query.allOr && !result) {
return false
}
}
return true
return !query.allOr
}
const stringMatch = match(
@ -666,8 +673,45 @@ export const runQuery = (docs: Record<string, any>[], query: SearchFilters) => {
)
const containsAny = match(ArrayOperator.CONTAINS_ANY, _contains("some"))
const and = match(
LogicalOperator.AND,
(docValue: Record<string, any>, conditions: SearchFilters[]) => {
if (!conditions.length) {
return false
}
for (const condition of conditions) {
const matchesCondition = runQuery([docValue], condition)
if (!matchesCondition.length) {
return false
}
}
return true
}
)
const or = match(
LogicalOperator.OR,
(docValue: Record<string, any>, conditions: SearchFilters[]) => {
if (!conditions.length) {
return false
}
for (const condition of conditions) {
const matchesCondition = runQuery([docValue], {
...condition,
allOr: true,
})
if (matchesCondition.length) {
return true
}
}
return false
}
)
const docMatch = (doc: Record<string, any>) => {
const filterFunctions = {
const filterFunctions: Record<
SearchFilterOperator,
(doc: Record<string, any>) => boolean
> = {
string: stringMatch,
fuzzy: fuzzyMatch,
range: rangeMatch,
@ -679,6 +723,8 @@ export const runQuery = (docs: Record<string, any>[], query: SearchFilters) => {
contains: contains,
containsAny: containsAny,
notContains: notContains,
[LogicalOperator.AND]: and,
[LogicalOperator.OR]: or,
}
const results = Object.entries(query || {})

2
packages/shared-core/tsconfig.build.json
View File

@ -18,6 +18,6 @@
},
"tsBuildInfoFile": "dist/tsconfig.tsbuildinfo"
},
"include": ["src/**/*"],
"include": ["src/**/*.ts"],
"exclude": ["**/*.spec.ts", "**/*.spec.js", "__mocks__", "src/tests"]
}

3
packages/shared-core/tsconfig.json
View File

@ -1,9 +1,6 @@
{
"extends": "./tsconfig.build.json",
"compilerOptions": {
"baseUrl": "..",
"rootDir": "src",
"composite": true,
"types": ["node", "jest"]
},
"exclude": ["node_modules", "dist"]

24
packages/types/src/sdk/search.ts
View File

@ -23,7 +23,22 @@ export enum RangeOperator {
RANGE = "range",
}
export type SearchFilterOperator = BasicOperator | ArrayOperator | RangeOperator
export enum LogicalOperator {
AND = "$and",
OR = "$or",
}
export function isLogicalSearchOperator(
value: string
): value is LogicalOperator {
return value === LogicalOperator.AND || value === LogicalOperator.OR
}
export type SearchFilterOperator =
| BasicOperator
| ArrayOperator
| RangeOperator
| LogicalOperator
export enum InternalSearchFilterOperator {
COMPLEX_ID_OPERATOR = "_complexIdOperator",
@ -75,6 +90,13 @@ export interface SearchFilters {
// to make sure the documents returned are always filtered down to a
// specific document type (such as just rows)
documentType?: DocumentType
[LogicalOperator.AND]?: {
conditions: SearchFilters[]
}
[LogicalOperator.OR]?: {
conditions: SearchFilters[]
}
}
export type SearchFilterKey = keyof Omit<

6
packages/worker/src/api/controllers/global/self.ts
View File

@ -1,6 +1,6 @@
import * as userSdk from "../../../sdk/users"
import {
featureFlags,
features,
tenancy,
db as dbCore,
utils,
@ -104,8 +104,8 @@ export async function getSelf(ctx: any) {
ctx.body = await groups.enrichUserRolesFromGroups(user)
// add the feature flags for this tenant
const tenantId = tenancy.getTenantId()
ctx.body.featureFlags = featureFlags.getTenantFeatureFlags(tenantId)
const flags = await features.fetch()
ctx.body.flags = flags
addSessionAttributesToUser(ctx)
}

2
packages/worker/src/environment.ts
View File

@ -19,8 +19,6 @@ function parseIntSafe(number: any) {
}
const environment = {
// features
WORKER_FEATURES: process.env.WORKER_FEATURES,
// auth
MINIO_ACCESS_KEY: process.env.MINIO_ACCESS_KEY,
MINIO_SECRET_KEY: process.env.MINIO_SECRET_KEY,

13
packages/worker/src/features.ts
View File

@ -1,13 +0,0 @@
import { features } from "@budibase/backend-core"
import env from "./environment"
enum WorkerFeature {}
const featureList: WorkerFeature[] = features.processFeatureEnvVar(
Object.values(WorkerFeature),
env.WORKER_FEATURES
)
export function isFeatureEnabled(feature: WorkerFeature) {
return featureList.includes(feature)
}

8
yarn.lock
View File

@ -12077,10 +12077,10 @@ google-p12-pem@^4.0.0:
dependencies:
node-forge "^1.3.1"
"google-spreadsheet@npm:@budibase/google-spreadsheet@4.1.2":
version "4.1.2"
resolved "https://registry.yarnpkg.com/@budibase/google-spreadsheet/-/google-spreadsheet-4.1.2.tgz#90548ccba2284b3042b08d2974ef3caeaf772ad9"
integrity sha512-dxoY3rQGGnuNeZiXhNc9oYPduzU8xnIjWujFwNvaRRv3zWeUV7mj6HE2o/OJOeekPGt7o44B+w6DfkiaoteZgg==
"google-spreadsheet@npm:@budibase/google-spreadsheet@4.1.3":
version "4.1.3"
resolved "https://registry.yarnpkg.com/@budibase/google-spreadsheet/-/google-spreadsheet-4.1.3.tgz#bcee7bd9d90f82c54b16a9aca963b87aceb050ad"
integrity sha512-03VX3/K5NXIh6+XAIDZgcHPmR76xwd8vIDL7RedMpvM2IcXK0Iq/KU7FmLY0t/mKqORAGC7+0rajd0jLFezC4w==
dependencies:
axios "^1.4.0"
lodash "^4.17.21"