MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'develop' into fix/automation-improvements

This commit is contained in:
Michael Drury 2023-08-17 17:35:02 +01:00 committed by GitHub
parent 4c6c0f3002 e174211440
commit df31fb1b8d
16 changed files with 282 additions and 148 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
.vscode/launch.json vendored
View File

@ -8,30 +8,18 @@
"name": "Budibase Server",
"type": "node",
"request": "launch",
"runtimeArgs": [
"--nolazy",
"-r",
"ts-node/register/transpile-only"
],
"args": [
"${workspaceFolder}/packages/server/src/index.ts"
],
"runtimeArgs": ["--nolazy", "-r", "ts-node/register/transpile-only"],
"args": ["${workspaceFolder}/packages/server/src/index.ts"],
"cwd": "${workspaceFolder}/packages/server"
},
{
"name": "Budibase Worker",
"type": "node",
"request": "launch",
"runtimeArgs": [
"--nolazy",
"-r",
"ts-node/register/transpile-only"
],
"args": [
"${workspaceFolder}/packages/worker/src/index.ts"
],
"runtimeArgs": ["--nolazy", "-r", "ts-node/register/transpile-only"],
"args": ["${workspaceFolder}/packages/worker/src/index.ts"],
"cwd": "${workspaceFolder}/packages/worker"
},
}
],
"compounds": [
{

5
charts/budibase/values.yaml
View File

@ -137,7 +137,6 @@ services:
path: /health
port: 10000
scheme: HTTP
enabled: true
periodSeconds: 3
failureThreshold: 1
livenessProbe:
@ -170,7 +169,6 @@ services:
path: /health
port: 4002
scheme: HTTP
enabled: true
periodSeconds: 3
failureThreshold: 1
livenessProbe:
@ -204,7 +202,6 @@ services:
path: /health
port: 4003
scheme: HTTP
enabled: true
periodSeconds: 3
failureThreshold: 1
livenessProbe:
@ -411,14 +408,12 @@ couchdb:
## Ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/#configure-probes
# FOR COUCHDB
livenessProbe:
enabled: true
failureThreshold: 3
initialDelaySeconds: 0
periodSeconds: 10
successThreshold: 1
timeoutSeconds: 1
readinessProbe:
enabled: true
failureThreshold: 3
initialDelaySeconds: 0
periodSeconds: 10

2
hosting/docker-compose.yaml
View File

@ -27,6 +27,7 @@ services:
BB_ADMIN_USER_EMAIL: ${BB_ADMIN_USER_EMAIL}
BB_ADMIN_USER_PASSWORD: ${BB_ADMIN_USER_PASSWORD}
PLUGINS_DIR: ${PLUGINS_DIR}
OFFLINE_MODE: ${OFFLINE_MODE}
depends_on:
- worker-service
- redis-service
@ -54,6 +55,7 @@ services:
INTERNAL_API_KEY: ${INTERNAL_API_KEY}
REDIS_URL: redis-service:6379
REDIS_PASSWORD: ${REDIS_PASSWORD}
OFFLINE_MODE: ${OFFLINE_MODE}
depends_on:
- redis-service
- minio-service

2
lerna.json
View File

@ -1,5 +1,5 @@
{
"version": "2.9.25",
"version": "2.9.29",
"npmClient": "yarn",
"packages": [
"packages/*"

26
packages/backend-core/src/db/lucene.ts
View File

@ -1,7 +1,6 @@
import fetch from "node-fetch"
import { getCouchInfo } from "./couch"
import { SearchFilters, Row } from "@budibase/types"
import { createUserIndex } from "./searchIndexes/searchIndexes"
import { SearchFilters, Row, EmptyFilterOption } from "@budibase/types"
const QUERY_START_REGEX = /\d[0-9]*:/g
@ -65,6 +64,7 @@ export class QueryBuilder<T> {
this.#index = index
this.#query = {
allOr: false,
onEmptyFilter: EmptyFilterOption.RETURN_ALL,
string: {},
fuzzy: {},
range: {},
@ -218,6 +218,10 @@ export class QueryBuilder<T> {
this.#query.allOr = true
}
setOnEmptyFilter(value: EmptyFilterOption) {
this.#query.onEmptyFilter = value
}
handleSpaces(input: string) {
if (this.#noEscaping) {
return input
@ -289,8 +293,9 @@ export class QueryBuilder<T> {
const builder = this
let allOr = this.#query && this.#query.allOr
let query = allOr ? "" : "*:*"
let allFiltersEmpty = true
const allPreProcessingOpts = { escape: true, lowercase: true, wrap: true }
let tableId
let tableId: string = ""
if (this.#query.equal!.tableId) {
tableId = this.#query.equal!.tableId
delete this.#query.equal!.tableId
@ -305,7 +310,7 @@ export class QueryBuilder<T> {
}
const contains = (key: string, value: any, mode = "AND") => {
if (Array.isArray(value) && value.length === 0) {
if (!value || (Array.isArray(value) && value.length === 0)) {
return null
}
if (!Array.isArray(value)) {
@ -384,6 +389,12 @@ export class QueryBuilder<T> {
built += ` ${mode} `
}
built += expression
if (
(typeof value !== "string" && value != null) ||
(typeof value === "string" && value !== tableId && value !== "")
) {
allFiltersEmpty = false
}
}
if (opts?.returnBuilt) {
return built
@ -463,6 +474,13 @@ export class QueryBuilder<T> {
allOr = false
build({ tableId }, equal)
}
if (allFiltersEmpty) {
if (this.#query.onEmptyFilter === EmptyFilterOption.RETURN_NONE) {
return ""
} else if (this.#query?.allOr) {
return query.replace("()", "(*:*)")
}
}
return query
}

72
packages/backend-core/src/db/tests/lucene.spec.ts
View File

@ -1,6 +1,6 @@
import { newid } from "../../docIds/newid"
import { getDB } from "../db"
import { Database } from "@budibase/types"
import { Database, EmptyFilterOption } from "@budibase/types"
import { QueryBuilder, paginatedSearch, fullSearch } from "../lucene"
const INDEX_NAME = "main"
@ -156,6 +156,76 @@ describe("lucene", () => {
expect(resp.rows.length).toBe(2)
})
describe("empty filters behaviour", () => {
it("should return all rows by default", async () => {
const builder = new QueryBuilder(dbName, INDEX_NAME)
builder.addEqual("property", "")
builder.addEqual("number", null)
builder.addString("property", "")
builder.addFuzzy("property", "")
builder.addNotEqual("number", undefined)
builder.addOneOf("number", null)
builder.addContains("array", undefined)
builder.addNotContains("array", null)
builder.addContainsAny("array", null)
const resp = await builder.run()
expect(resp.rows.length).toBe(3)
})
it("should return all rows when onEmptyFilter is ALL", async () => {
const builder = new QueryBuilder(dbName, INDEX_NAME)
builder.setOnEmptyFilter(EmptyFilterOption.RETURN_ALL)
builder.setAllOr()
builder.addEqual("property", "")
builder.addEqual("number", null)
builder.addString("property", "")
builder.addFuzzy("property", "")
builder.addNotEqual("number", undefined)
builder.addOneOf("number", null)
builder.addContains("array", undefined)
builder.addNotContains("array", null)
builder.addContainsAny("array", null)
const resp = await builder.run()
expect(resp.rows.length).toBe(3)
})
it("should return no rows when onEmptyFilter is NONE", async () => {
const builder = new QueryBuilder(dbName, INDEX_NAME)
builder.setOnEmptyFilter(EmptyFilterOption.RETURN_NONE)
builder.addEqual("property", "")
builder.addEqual("number", null)
builder.addString("property", "")
builder.addFuzzy("property", "")
builder.addNotEqual("number", undefined)
builder.addOneOf("number", null)
builder.addContains("array", undefined)
builder.addNotContains("array", null)
builder.addContainsAny("array", null)
const resp = await builder.run()
expect(resp.rows.length).toBe(0)
})
it("should return all matching rows when onEmptyFilter is NONE, but a filter value is provided", async () => {
const builder = new QueryBuilder(dbName, INDEX_NAME)
builder.setOnEmptyFilter(EmptyFilterOption.RETURN_NONE)
builder.addEqual("property", "")
builder.addEqual("number", 1)
builder.addString("property", "")
builder.addFuzzy("property", "")
builder.addNotEqual("number", undefined)
builder.addOneOf("number", null)
builder.addContains("array", undefined)
builder.addNotContains("array", null)
builder.addContainsAny("array", null)
const resp = await builder.run()
expect(resp.rows.length).toBe(1)
})
})
describe("skip", () => {
const skipDbName = `db-${newid()}`
let docs: {

30
packages/builder/src/components/design/settings/controls/FilterEditor/FilterDrawer.svelte
View File

@ -35,22 +35,28 @@
{ value: "and", label: "Match all filters" },
{ value: "or", label: "Match any filter" },
]
const onEmptyOptions = [
{ value: "all", label: "Return all table rows" },
{ value: "none", label: "Return no rows" },
]
let rawFilters
let matchAny = false
let onEmptyFilter = "all"
$: parseFilters(filters)
$: dispatch("change", enrichFilters(rawFilters, matchAny))
$: dispatch("change", enrichFilters(rawFilters, matchAny, onEmptyFilter))
$: enrichedSchemaFields = getFields(schemaFields || [], { allowLinks: true })
$: fieldOptions = enrichedSchemaFields.map(field => field.name) || []
$: valueTypeOptions = allowBindings ? ["Value", "Binding"] : ["Value"]
// Remove field key prefixes and determine whether to use the "match all"
// or "match any" behaviour
// Remove field key prefixes and determine which behaviours to use
const parseFilters = filters => {
matchAny = filters?.find(filter => filter.operator === "allOr") != null
onEmptyFilter =
filters?.find(filter => filter.onEmptyFilter)?.onEmptyFilter ?? "all"
rawFilters = (filters || [])
.filter(filter => filter.operator !== "allOr")
.filter(filter => filter.operator !== "allOr" && !filter.onEmptyFilter)
.map(filter => {
const { field } = filter
let newFilter = { ...filter }
@ -65,8 +71,8 @@
}
// Add field key prefixes and a special metadata filter object to indicate
// whether to use the "match all" or "match any" behaviour
const enrichFilters = (rawFilters, matchAny) => {
// how to handle filter behaviour
const enrichFilters = (rawFilters, matchAny, onEmptyFilter) => {
let count = 1
return rawFilters
.filter(filter => filter.field)
@ -75,6 +81,7 @@
field: `${count++}:${filter.field}`,
}))
.concat(matchAny ? [{ operator: "allOr" }] : [])
.concat([{ onEmptyFilter }])
}
const addFilter = () => {
@ -186,6 +193,17 @@
on:change={e => (matchAny = e.detail === "or")}
placeholder={null}
/>
{#if datasource?.type === "table"}
<Select
label="When filter empty"
value={onEmptyFilter}
options={onEmptyOptions}
getOptionLabel={opt => opt.label}
getOptionValue={opt => opt.value}
on:change={e => (onEmptyFilter = e.detail)}
placeholder={null}
/>
{/if}
</div>
<div>
<div class="filter-label">

1
packages/server/jest.config.ts
View File

@ -12,6 +12,7 @@ const baseConfig: Config.InitialProjectOptions = {
},
moduleNameMapper: {
"@budibase/backend-core/(.*)": "<rootDir>/../backend-core/$1",
"@budibase/shared-core/(.*)": "<rootDir>/../shared-core/$1",
"@budibase/backend-core": "<rootDir>/../backend-core/src",
"@budibase/shared-core": "<rootDir>/../shared-core/src",
"@budibase/types": "<rootDir>/../types/src",

9
packages/server/src/api/controllers/row/external.ts
View File

@ -13,8 +13,10 @@ import {
Row,
Table,
UserCtx,
EmptyFilterOption,
} from "@budibase/types"
import sdk from "../../../sdk"
import { hasFilters } from "@budibase/shared-core/src/filters"
export async function handleRequest(
operation: Operation,
@ -37,6 +39,13 @@ export async function handleRequest(
}
}
if (
!hasFilters(opts?.filters) &&
opts?.filters?.onEmptyFilter === EmptyFilterOption.RETURN_NONE
) {
return []
}
return new ExternalRequest(operation, tableId, opts?.datasource).run(
opts || {}
)

17
packages/server/src/automations/steps/queryRows.ts
View File

@ -11,6 +11,7 @@ import {
AutomationStepInput,
AutomationStepSchema,
AutomationStepType,
EmptyFilterOption,
SearchFilters,
Table,
} from "@budibase/types"
@ -26,16 +27,6 @@ const SortOrderPretty = {
[SortOrder.DESCENDING]: "Descending",
}
enum EmptyFilterOption {
RETURN_ALL = "all",
RETURN_NONE = "none",
}
const EmptyFilterOptionPretty = {
[EmptyFilterOption.RETURN_ALL]: "Return all table rows",
[EmptyFilterOption.RETURN_NONE]: "Return no rows",
}
export const definition: AutomationStepSchema = {
description: "Query rows from the database",
icon: "Search",
@ -77,12 +68,6 @@ export const definition: AutomationStepSchema = {
title: "Limit",
customType: AutomationCustomIOType.QUERY_LIMIT,
},
onEmptyFilter: {
pretty: Object.values(EmptyFilterOptionPretty),
enum: Object.values(EmptyFilterOption),
type: AutomationIOType.STRING,
title: "When Filter Empty",
},
},
required: ["tableId"],
},

12
packages/server/src/integrations/microsoftSqlServer.ts
View File

@ -341,10 +341,10 @@ class SqlServerIntegration extends Sql implements DatasourcePlus {
}
}
getDefinitionSQL(tableName: string) {
getDefinitionSQL(tableName: string, schemaName: string) {
return `select *
from INFORMATION_SCHEMA.COLUMNS
where TABLE_NAME='${tableName}'`
where TABLE_NAME='${tableName}' AND TABLE_SCHEMA='${schemaName}'`
}
getConstraintsSQL(tableName: string) {
@ -388,16 +388,18 @@ class SqlServerIntegration extends Sql implements DatasourcePlus {
throw "Unable to get list of tables in database"
}
const schema = this.config.schema || DEFAULT_SCHEMA
const schemaName = this.config.schema || DEFAULT_SCHEMA
const tableNames = tableInfo
.filter((record: any) => record.TABLE_SCHEMA === schema)
.filter((record: any) => record.TABLE_SCHEMA === schemaName)
.map((record: any) => record.TABLE_NAME)
.filter((name: string) => this.MASTER_TABLES.indexOf(name) === -1)
const tables: Record<string, ExternalTable> = {}
for (let tableName of tableNames) {
// get the column definition (type)
const definition = await this.runSQL(this.getDefinitionSQL(tableName))
const definition = await this.runSQL(
this.getDefinitionSQL(tableName, schemaName)
)
// find primary key constraints
const constraints = await this.runSQL(this.getConstraintsSQL(tableName))
// find the computed and identity columns (auto columns)

26
packages/server/src/sdk/app/rows/search/tests/external.spec.ts
View File

@ -1,6 +1,12 @@
import { GenericContainer } from "testcontainers"
import { Datasource, FieldType, Row, SourceName, Table } from "@budibase/types"
import {
Datasource,
EmptyFilterOption,
FieldType,
Row,
SourceName,
Table,
} from "@budibase/types"
import TestConfiguration from "../../../../../tests/utilities/TestConfiguration"
import { SearchParams } from "../../search"
import { search } from "../external"
@ -116,6 +122,22 @@ describe.skip("external", () => {
})
})
it("empty filters search returns no data", async () => {
await config.doInContext(config.appId, async () => {
const tableId = config.table!._id!
const searchParams: SearchParams = {
tableId,
query: {
onEmptyFilter: EmptyFilterOption.RETURN_NONE,
},
}
const result = await search(searchParams)
expect(result.rows).toHaveLength(0)
})
})
it("querying by fields will always return data attribute columns", async () => {
await config.doInContext(config.appId, async () => {
const tableId = config.table!._id!

97
packages/shared-core/src/filters.ts
View File

@ -1,4 +1,12 @@
import { Datasource, FieldType, SortDirection, SortType } from "@budibase/types"
import {
Datasource,
FieldType,
SortDirection,
SortType,
SearchFilter,
SearchQuery,
SearchQueryFields,
} from "@budibase/types"
import { OperatorOptions, SqlNumberTypeRangeMap } from "./constants"
import { deepGet } from "./helpers"
@ -73,13 +81,13 @@ export const NoEmptyFilterStrings = [
OperatorOptions.NotEquals.value,
OperatorOptions.Contains.value,
OperatorOptions.NotContains.value,
] as (keyof QueryFields)[]
] as (keyof SearchQueryFields)[]
/**
* Removes any fields that contain empty strings that would cause inconsistent
* behaviour with how backend tables are filtered (no value means no filter).
*/
const cleanupQuery = (query: Query) => {
const cleanupQuery = (query: SearchQuery) => {
if (!query) {
return query
}
@ -110,66 +118,12 @@ const removeKeyNumbering = (key: string) => {
}
}
type Filter = {
operator: keyof Query
field: string
type: any
value: any
externalType: keyof typeof SqlNumberTypeRangeMap
}
type Query = QueryFields & QueryConfig
type QueryFields = {
string?: {
[key: string]: string
}
fuzzy?: {
[key: string]: string
}
range?: {
[key: string]: {
high: number | string
low: number | string
}
}
equal?: {
[key: string]: any
}
notEqual?: {
[key: string]: any
}
empty?: {
[key: string]: any
}
notEmpty?: {
[key: string]: any
}
oneOf?: {
[key: string]: any[]
}
contains?: {
[key: string]: any[]
}
notContains?: {
[key: string]: any[]
}
containsAny?: {
[key: string]: any[]
}
}
type QueryConfig = {
allOr?: boolean
}
type QueryFieldsType = keyof QueryFields
/**
* Builds a lucene JSON query from the filter structure generated in the builder
* @param filter the builder filter structure
*/
export const buildLuceneQuery = (filter: Filter[]) => {
let query: Query = {
export const buildLuceneQuery = (filter: SearchFilter[]) => {
let query: SearchQuery = {
string: {},
fuzzy: {},
range: {},
@ -184,7 +138,8 @@ export const buildLuceneQuery = (filter: Filter[]) => {
}
if (Array.isArray(filter)) {
filter.forEach(expression => {
let { operator, field, type, value, externalType } = expression
let { operator, field, type, value, externalType, onEmptyFilter } =
expression
const isHbs =
typeof value === "string" && (value.match(HBS_REGEX) || []).length > 0
// Parse all values into correct types
@ -192,6 +147,10 @@ export const buildLuceneQuery = (filter: Filter[]) => {
query.allOr = true
return
}
if (onEmptyFilter) {
query.onEmptyFilter = onEmptyFilter
return
}
if (
type === "datetime" &&
!isHbs &&
@ -227,9 +186,13 @@ export const buildLuceneQuery = (filter: Filter[]) => {
}
if (operator.startsWith("range") && query.range) {
const minint =
SqlNumberTypeRangeMap[externalType]?.min || Number.MIN_SAFE_INTEGER
SqlNumberTypeRangeMap[
externalType as keyof typeof SqlNumberTypeRangeMap
]?.min || Number.MIN_SAFE_INTEGER
const maxint =
SqlNumberTypeRangeMap[externalType]?.max || Number.MAX_SAFE_INTEGER
SqlNumberTypeRangeMap[
externalType as keyof typeof SqlNumberTypeRangeMap
]?.max || Number.MAX_SAFE_INTEGER
if (!query.range[field]) {
query.range[field] = {
low: type === "number" ? minint : "0000-00-00T00:00:00.000Z",
@ -245,7 +208,7 @@ export const buildLuceneQuery = (filter: Filter[]) => {
) {
query.range[field].high = value
}
} else if (query[operator]) {
} else if (query[operator] && operator !== "onEmptyFilter") {
if (type === "boolean") {
// Transform boolean filters to cope with null.
// "equals false" needs to be "not equals true"
@ -275,7 +238,7 @@ export const buildLuceneQuery = (filter: Filter[]) => {
* @param docs the data
* @param query the JSON lucene query
*/
export const runLuceneQuery = (docs: any[], query?: Query) => {
export const runLuceneQuery = (docs: any[], query?: SearchQuery) => {
if (!docs || !Array.isArray(docs)) {
return []
}
@ -289,7 +252,7 @@ export const runLuceneQuery = (docs: any[], query?: Query) => {
// Iterates over a set of filters and evaluates a fail function against a doc
const match =
(
type: QueryFieldsType,
type: keyof SearchQueryFields,
failFn: (docValue: any, testValue: any) => boolean
) =>
(doc: any) => {
@ -456,11 +419,11 @@ export const luceneLimit = (docs: any[], limit: string) => {
return docs.slice(0, numLimit)
}
export const hasFilters = (query?: Query) => {
export const hasFilters = (query?: SearchQuery) => {
if (!query) {
return false
}
const skipped = ["allOr"]
const skipped = ["allOr", "onEmptyFilter"]
for (let [key, value] of Object.entries(query)) {
if (skipped.includes(key) || typeof value !== "object") {
continue

1
packages/types/src/api/web/index.ts
View File

@ -8,3 +8,4 @@ export * from "./system"
export * from "./app"
export * from "./global"
export * from "./pagination"
export * from "./searchFilter"

54
packages/types/src/api/web/searchFilter.ts Normal file
View File

@ -0,0 +1,54 @@
import { FieldType } from "../../documents"
import { EmptyFilterOption } from "../../sdk"
export type SearchFilter = {
operator: keyof SearchQuery
onEmptyFilter?: EmptyFilterOption
field: string
type?: FieldType
value: any
externalType?: string
}
export type SearchQuery = {
allOr?: boolean
onEmptyFilter?: EmptyFilterOption
string?: {
[key: string]: string
}
fuzzy?: {
[key: string]: string
}
range?: {
[key: string]: {
high: number | string
low: number | string
}
}
equal?: {
[key: string]: any
}
notEqual?: {
[key: string]: any
}
empty?: {
[key: string]: any
}
notEmpty?: {
[key: string]: any
}
oneOf?: {
[key: string]: any[]
}
contains?: {
[key: string]: any[]
}
notContains?: {
[key: string]: any[]
}
containsAny?: {
[key: string]: any[]
}
}
export type SearchQueryFields = Omit<SearchQuery, "allOr" | "onEmptyFilter">

6
packages/types/src/sdk/search.ts
View File

@ -4,6 +4,7 @@ import { SortType } from "../api"
export interface SearchFilters {
allOr?: boolean
onEmptyFilter?: EmptyFilterOption
string?: {
[key: string]: string
}
@ -99,3 +100,8 @@ export interface SqlQuery {
sql: string
bindings?: string[]
}
export enum EmptyFilterOption {
RETURN_ALL = "all",
RETURN_NONE = "none",
}