MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'fix/automation-improvements' of github.com:Budibase/budibase into fix/automation-improvements

This commit is contained in:
mike12345567 2023-08-17 17:45:17 +01:00
parent ba5e390b3f df31fb1b8d
commit 9cfab6fbfd
16 changed files with 282 additions and 148 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
.vscode/launch.json vendored
View File

@ -8,30 +8,18 @@
"name": "Budibase Server",
"type": "node",
"request": "launch",
"runtimeArgs": [
"--nolazy",
"-r",
"ts-node/register/transpile-only"
],
"args": [
"${workspaceFolder}/packages/server/src/index.ts"
],
"runtimeArgs": ["--nolazy", "-r", "ts-node/register/transpile-only"],
"args": ["${workspaceFolder}/packages/server/src/index.ts"],
"cwd": "${workspaceFolder}/packages/server"
},
{
"name": "Budibase Worker",
"type": "node",
"request": "launch",
"runtimeArgs": [
"--nolazy",
"-r",
"ts-node/register/transpile-only"
],
"args": [
"${workspaceFolder}/packages/worker/src/index.ts"
],
"runtimeArgs": ["--nolazy", "-r", "ts-node/register/transpile-only"],
"args": ["${workspaceFolder}/packages/worker/src/index.ts"],
"cwd": "${workspaceFolder}/packages/worker"
},
}
],
"compounds": [
{

5
charts/budibase/values.yaml
View File

@ -137,7 +137,6 @@ services:
path: /health
port: 10000
scheme: HTTP
enabled: true
periodSeconds: 3
failureThreshold: 1
livenessProbe:
@ -170,7 +169,6 @@ services:
path: /health
port: 4002
scheme: HTTP
enabled: true
periodSeconds: 3
failureThreshold: 1
livenessProbe:
@ -204,7 +202,6 @@ services:
path: /health
port: 4003
scheme: HTTP
enabled: true
periodSeconds: 3
failureThreshold: 1
livenessProbe:
@ -411,14 +408,12 @@ couchdb:
## Ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/#configure-probes
# FOR COUCHDB
livenessProbe:
enabled: true
failureThreshold: 3
initialDelaySeconds: 0
periodSeconds: 10
successThreshold: 1
timeoutSeconds: 1
readinessProbe:
enabled: true
failureThreshold: 3
initialDelaySeconds: 0
periodSeconds: 10

2
hosting/docker-compose.yaml
View File

@ -27,6 +27,7 @@ services:
BB_ADMIN_USER_EMAIL: ${BB_ADMIN_USER_EMAIL}
BB_ADMIN_USER_PASSWORD: ${BB_ADMIN_USER_PASSWORD}
PLUGINS_DIR: ${PLUGINS_DIR}
OFFLINE_MODE: ${OFFLINE_MODE}
depends_on:
- worker-service
- redis-service
@ -54,6 +55,7 @@ services:
INTERNAL_API_KEY: ${INTERNAL_API_KEY}
REDIS_URL: redis-service:6379
REDIS_PASSWORD: ${REDIS_PASSWORD}
OFFLINE_MODE: ${OFFLINE_MODE}
depends_on:
- redis-service
- minio-service

2
lerna.json
View File

@ -1,5 +1,5 @@
{
"version": "2.9.25",
"version": "2.9.29",
"npmClient": "yarn",
"packages": [
"packages/*"

26
packages/backend-core/src/db/lucene.ts
View File

@ -1,7 +1,6 @@
import fetch from "node-fetch"
import { getCouchInfo } from "./couch"
import { SearchFilters, Row } from "@budibase/types"
import { createUserIndex } from "./searchIndexes/searchIndexes"
import { SearchFilters, Row, EmptyFilterOption } from "@budibase/types"
const QUERY_START_REGEX = /\d[0-9]*:/g
@ -65,6 +64,7 @@ export class QueryBuilder<T> {
this.#index = index
this.#query = {
allOr: false,
onEmptyFilter: EmptyFilterOption.RETURN_ALL,
string: {},
fuzzy: {},
range: {},
@ -218,6 +218,10 @@ export class QueryBuilder<T> {
this.#query.allOr = true
}
setOnEmptyFilter(value: EmptyFilterOption) {
this.#query.onEmptyFilter = value
}
handleSpaces(input: string) {
if (this.#noEscaping) {
return input
@ -289,8 +293,9 @@ export class QueryBuilder<T> {
const builder = this
let allOr = this.#query && this.#query.allOr
let query = allOr ? "" : "*:*"
let allFiltersEmpty = true
const allPreProcessingOpts = { escape: true, lowercase: true, wrap: true }
let tableId
let tableId: string = ""
if (this.#query.equal!.tableId) {
tableId = this.#query.equal!.tableId
delete this.#query.equal!.tableId
@ -305,7 +310,7 @@ export class QueryBuilder<T> {
}
const contains = (key: string, value: any, mode = "AND") => {
if (Array.isArray(value) && value.length === 0) {
if (!value || (Array.isArray(value) && value.length === 0)) {
return null
}
if (!Array.isArray(value)) {
@ -384,6 +389,12 @@ export class QueryBuilder<T> {
built += ` ${mode} `
}
built += expression
if (
(typeof value !== "string" && value != null) ||
(typeof value === "string" && value !== tableId && value !== "")
) {
allFiltersEmpty = false
}
}
if (opts?.returnBuilt) {
return built
@ -463,6 +474,13 @@ export class QueryBuilder<T> {
allOr = false
build({ tableId }, equal)
}
if (allFiltersEmpty) {
if (this.#query.onEmptyFilter === EmptyFilterOption.RETURN_NONE) {
return ""
} else if (this.#query?.allOr) {
return query.replace("()", "(*:*)")
}
}
return query
}

72
packages/backend-core/src/db/tests/lucene.spec.ts
View File

@ -1,6 +1,6 @@
import { newid } from "../../docIds/newid"
import { getDB } from "../db"
import { Database } from "@budibase/types"
import { Database, EmptyFilterOption } from "@budibase/types"
import { QueryBuilder, paginatedSearch, fullSearch } from "../lucene"
const INDEX_NAME = "main"
@ -156,6 +156,76 @@ describe("lucene", () => {
expect(resp.rows.length).toBe(2)
})
describe("empty filters behaviour", () => {
it("should return all rows by default", async () => {
const builder = new QueryBuilder(dbName, INDEX_NAME)
builder.addEqual("property", "")
builder.addEqual("number", null)
builder.addString("property", "")
builder.addFuzzy("property", "")
builder.addNotEqual("number", undefined)
builder.addOneOf("number", null)
builder.addContains("array", undefined)
builder.addNotContains("array", null)
builder.addContainsAny("array", null)
const resp = await builder.run()
expect(resp.rows.length).toBe(3)
})
it("should return all rows when onEmptyFilter is ALL", async () => {
const builder = new QueryBuilder(dbName, INDEX_NAME)
builder.setOnEmptyFilter(EmptyFilterOption.RETURN_ALL)
builder.setAllOr()
builder.addEqual("property", "")
builder.addEqual("number", null)
builder.addString("property", "")
builder.addFuzzy("property", "")
builder.addNotEqual("number", undefined)
builder.addOneOf("number", null)
builder.addContains("array", undefined)
builder.addNotContains("array", null)
builder.addContainsAny("array", null)
const resp = await builder.run()
expect(resp.rows.length).toBe(3)
})
it("should return no rows when onEmptyFilter is NONE", async () => {
const builder = new QueryBuilder(dbName, INDEX_NAME)
builder.setOnEmptyFilter(EmptyFilterOption.RETURN_NONE)
builder.addEqual("property", "")
builder.addEqual("number", null)
builder.addString("property", "")
builder.addFuzzy("property", "")
builder.addNotEqual("number", undefined)
builder.addOneOf("number", null)
builder.addContains("array", undefined)
builder.addNotContains("array", null)
builder.addContainsAny("array", null)
const resp = await builder.run()
expect(resp.rows.length).toBe(0)
})
it("should return all matching rows when onEmptyFilter is NONE, but a filter value is provided", async () => {
const builder = new QueryBuilder(dbName, INDEX_NAME)
builder.setOnEmptyFilter(EmptyFilterOption.RETURN_NONE)
builder.addEqual("property", "")
builder.addEqual("number", 1)
builder.addString("property", "")
builder.addFuzzy("property", "")
builder.addNotEqual("number", undefined)
builder.addOneOf("number", null)
builder.addContains("array", undefined)
builder.addNotContains("array", null)
builder.addContainsAny("array", null)
const resp = await builder.run()
expect(resp.rows.length).toBe(1)
})
})
describe("skip", () => {
const skipDbName = `db-${newid()}`
let docs: {

30
packages/builder/src/components/design/settings/controls/FilterEditor/FilterDrawer.svelte
View File

@ -35,22 +35,28 @@
{ value: "and", label: "Match all filters" },
{ value: "or", label: "Match any filter" },
]
const onEmptyOptions = [
{ value: "all", label: "Return all table rows" },
{ value: "none", label: "Return no rows" },
]
let rawFilters
let matchAny = false
let onEmptyFilter = "all"
$: parseFilters(filters)
$: dispatch("change", enrichFilters(rawFilters, matchAny))
$: dispatch("change", enrichFilters(rawFilters, matchAny, onEmptyFilter))
$: enrichedSchemaFields = getFields(schemaFields || [], { allowLinks: true })
$: fieldOptions = enrichedSchemaFields.map(field => field.name) || []
$: valueTypeOptions = allowBindings ? ["Value", "Binding"] : ["Value"]
// Remove field key prefixes and determine whether to use the "match all"
// or "match any" behaviour
// Remove field key prefixes and determine which behaviours to use
const parseFilters = filters => {
matchAny = filters?.find(filter => filter.operator === "allOr") != null
onEmptyFilter =
filters?.find(filter => filter.onEmptyFilter)?.onEmptyFilter ?? "all"
rawFilters = (filters || [])
.filter(filter => filter.operator !== "allOr")
.filter(filter => filter.operator !== "allOr" && !filter.onEmptyFilter)
.map(filter => {
const { field } = filter
let newFilter = { ...filter }
@ -65,8 +71,8 @@
}
// Add field key prefixes and a special metadata filter object to indicate
// whether to use the "match all" or "match any" behaviour
const enrichFilters = (rawFilters, matchAny) => {
// how to handle filter behaviour
const enrichFilters = (rawFilters, matchAny, onEmptyFilter) => {
let count = 1
return rawFilters
.filter(filter => filter.field)
@ -75,6 +81,7 @@
field: `${count++}:${filter.field}`,
}))
.concat(matchAny ? [{ operator: "allOr" }] : [])
.concat([{ onEmptyFilter }])
}
const addFilter = () => {
@ -186,6 +193,17 @@
on:change={e => (matchAny = e.detail === "or")}
placeholder={null}
/>
{#if datasource?.type === "table"}
<Select
label="When filter empty"
value={onEmptyFilter}
options={onEmptyOptions}
getOptionLabel={opt => opt.label}
getOptionValue={opt => opt.value}
on:change={e => (onEmptyFilter = e.detail)}
placeholder={null}
/>
{/if}
</div>
<div>
<div class="filter-label">

1
packages/server/jest.config.ts
View File

@ -12,6 +12,7 @@ const baseConfig: Config.InitialProjectOptions = {
},
moduleNameMapper: {
"@budibase/backend-core/(.*)": "<rootDir>/../backend-core/$1",
"@budibase/shared-core/(.*)": "<rootDir>/../shared-core/$1",
"@budibase/backend-core": "<rootDir>/../backend-core/src",
"@budibase/shared-core": "<rootDir>/../shared-core/src",
"@budibase/types": "<rootDir>/../types/src",

9
packages/server/src/api/controllers/row/external.ts
View File

@ -13,8 +13,10 @@ import {
Row,
Table,
UserCtx,
EmptyFilterOption,
} from "@budibase/types"
import sdk from "../../../sdk"
import { hasFilters } from "@budibase/shared-core/src/filters"
export async function handleRequest(
operation: Operation,
@ -37,6 +39,13 @@ export async function handleRequest(
}
}
if (
!hasFilters(opts?.filters) &&
opts?.filters?.onEmptyFilter === EmptyFilterOption.RETURN_NONE
) {
return []
}
return new ExternalRequest(operation, tableId, opts?.datasource).run(
opts || {}
)

17
packages/server/src/automations/steps/queryRows.ts
View File

@ -11,6 +11,7 @@ import {
AutomationStepInput,
AutomationStepSchema,
AutomationStepType,
EmptyFilterOption,
SearchFilters,
Table,
} from "@budibase/types"
@ -26,16 +27,6 @@ const SortOrderPretty = {
[SortOrder.DESCENDING]: "Descending",
}
enum EmptyFilterOption {
RETURN_ALL = "all",
RETURN_NONE = "none",
}
const EmptyFilterOptionPretty = {
[EmptyFilterOption.RETURN_ALL]: "Return all table rows",
[EmptyFilterOption.RETURN_NONE]: "Return no rows",
}
export const definition: AutomationStepSchema = {
description: "Query rows from the database",
icon: "Search",
@ -77,12 +68,6 @@ export const definition: AutomationStepSchema = {
title: "Limit",
customType: AutomationCustomIOType.QUERY_LIMIT,
},
onEmptyFilter: {
pretty: Object.values(EmptyFilterOptionPretty),
enum: Object.values(EmptyFilterOption),
type: AutomationIOType.STRING,
title: "When Filter Empty",
},
},
required: ["tableId"],
},

12
packages/server/src/integrations/microsoftSqlServer.ts
View File

@ -341,10 +341,10 @@ class SqlServerIntegration extends Sql implements DatasourcePlus {
}
}
getDefinitionSQL(tableName: string) {
getDefinitionSQL(tableName: string, schemaName: string) {
return `select *
from INFORMATION_SCHEMA.COLUMNS
where TABLE_NAME='${tableName}'`
where TABLE_NAME='${tableName}' AND TABLE_SCHEMA='${schemaName}'`
}
getConstraintsSQL(tableName: string) {
@ -388,16 +388,18 @@ class SqlServerIntegration extends Sql implements DatasourcePlus {
throw "Unable to get list of tables in database"
}
const schema = this.config.schema || DEFAULT_SCHEMA
const schemaName = this.config.schema || DEFAULT_SCHEMA
const tableNames = tableInfo
.filter((record: any) => record.TABLE_SCHEMA === schema)
.filter((record: any) => record.TABLE_SCHEMA === schemaName)
.map((record: any) => record.TABLE_NAME)
.filter((name: string) => this.MASTER_TABLES.indexOf(name) === -1)
const tables: Record<string, ExternalTable> = {}
for (let tableName of tableNames) {
// get the column definition (type)
const definition = await this.runSQL(this.getDefinitionSQL(tableName))
const definition = await this.runSQL(
this.getDefinitionSQL(tableName, schemaName)
)
// find primary key constraints
const constraints = await this.runSQL(this.getConstraintsSQL(tableName))
// find the computed and identity columns (auto columns)

26
packages/server/src/sdk/app/rows/search/tests/external.spec.ts
View File

@ -1,6 +1,12 @@
import { GenericContainer } from "testcontainers"
import { Datasource, FieldType, Row, SourceName, Table } from "@budibase/types"
import {
Datasource,
EmptyFilterOption,
FieldType,
Row,
SourceName,
Table,
} from "@budibase/types"
import TestConfiguration from "../../../../../tests/utilities/TestConfiguration"
import { SearchParams } from "../../search"
import { search } from "../external"
@ -116,6 +122,22 @@ describe.skip("external", () => {
})
})
it("empty filters search returns no data", async () => {
await config.doInContext(config.appId, async () => {
const tableId = config.table!._id!
const searchParams: SearchParams = {
tableId,
query: {
onEmptyFilter: EmptyFilterOption.RETURN_NONE,
},
}
const result = await search(searchParams)
expect(result.rows).toHaveLength(0)
})
})
it("querying by fields will always return data attribute columns", async () => {
await config.doInContext(config.appId, async () => {
const tableId = config.table!._id!

97
packages/shared-core/src/filters.ts
View File

@ -1,4 +1,12 @@
import { Datasource, FieldType, SortDirection, SortType } from "@budibase/types"
import {
Datasource,
FieldType,
SortDirection,
SortType,
SearchFilter,
SearchQuery,
SearchQueryFields,
} from "@budibase/types"
import { OperatorOptions, SqlNumberTypeRangeMap } from "./constants"
import { deepGet } from "./helpers"
@ -73,13 +81,13 @@ export const NoEmptyFilterStrings = [
OperatorOptions.NotEquals.value,
OperatorOptions.Contains.value,
OperatorOptions.NotContains.value,
] as (keyof QueryFields)[]
] as (keyof SearchQueryFields)[]
/**
* Removes any fields that contain empty strings that would cause inconsistent
* behaviour with how backend tables are filtered (no value means no filter).
*/
const cleanupQuery = (query: Query) => {
const cleanupQuery = (query: SearchQuery) => {
if (!query) {
return query
}
@ -110,66 +118,12 @@ const removeKeyNumbering = (key: string) => {
}
}
type Filter = {
operator: keyof Query
field: string
type: any
value: any
externalType: keyof typeof SqlNumberTypeRangeMap
}
type Query = QueryFields & QueryConfig
type QueryFields = {
string?: {
[key: string]: string
}
fuzzy?: {
[key: string]: string
}
range?: {
[key: string]: {
high: number | string
low: number | string
}
}
equal?: {
[key: string]: any
}
notEqual?: {
[key: string]: any
}
empty?: {
[key: string]: any
}
notEmpty?: {
[key: string]: any
}
oneOf?: {
[key: string]: any[]
}
contains?: {
[key: string]: any[]
}
notContains?: {
[key: string]: any[]
}
containsAny?: {
[key: string]: any[]
}
}
type QueryConfig = {
allOr?: boolean
}
type QueryFieldsType = keyof QueryFields
/**
* Builds a lucene JSON query from the filter structure generated in the builder
* @param filter the builder filter structure
*/
export const buildLuceneQuery = (filter: Filter[]) => {
let query: Query = {
export const buildLuceneQuery = (filter: SearchFilter[]) => {
let query: SearchQuery = {
string: {},
fuzzy: {},
range: {},
@ -184,7 +138,8 @@ export const buildLuceneQuery = (filter: Filter[]) => {
}
if (Array.isArray(filter)) {
filter.forEach(expression => {
let { operator, field, type, value, externalType } = expression
let { operator, field, type, value, externalType, onEmptyFilter } =
expression
const isHbs =
typeof value === "string" && (value.match(HBS_REGEX) || []).length > 0
// Parse all values into correct types
@ -192,6 +147,10 @@ export const buildLuceneQuery = (filter: Filter[]) => {
query.allOr = true
return
}
if (onEmptyFilter) {
query.onEmptyFilter = onEmptyFilter
return
}
if (
type === "datetime" &&
!isHbs &&
@ -227,9 +186,13 @@ export const buildLuceneQuery = (filter: Filter[]) => {
}
if (operator.startsWith("range") && query.range) {
const minint =
SqlNumberTypeRangeMap[externalType]?.min || Number.MIN_SAFE_INTEGER
SqlNumberTypeRangeMap[
externalType as keyof typeof SqlNumberTypeRangeMap
]?.min || Number.MIN_SAFE_INTEGER
const maxint =
SqlNumberTypeRangeMap[externalType]?.max || Number.MAX_SAFE_INTEGER
SqlNumberTypeRangeMap[
externalType as keyof typeof SqlNumberTypeRangeMap
]?.max || Number.MAX_SAFE_INTEGER
if (!query.range[field]) {
query.range[field] = {
low: type === "number" ? minint : "0000-00-00T00:00:00.000Z",
@ -245,7 +208,7 @@ export const buildLuceneQuery = (filter: Filter[]) => {
) {
query.range[field].high = value
}
} else if (query[operator]) {
} else if (query[operator] && operator !== "onEmptyFilter") {
if (type === "boolean") {
// Transform boolean filters to cope with null.
// "equals false" needs to be "not equals true"
@ -275,7 +238,7 @@ export const buildLuceneQuery = (filter: Filter[]) => {
* @param docs the data
* @param query the JSON lucene query
*/
export const runLuceneQuery = (docs: any[], query?: Query) => {
export const runLuceneQuery = (docs: any[], query?: SearchQuery) => {
if (!docs || !Array.isArray(docs)) {
return []
}
@ -289,7 +252,7 @@ export const runLuceneQuery = (docs: any[], query?: Query) => {
// Iterates over a set of filters and evaluates a fail function against a doc
const match =
(
type: QueryFieldsType,
type: keyof SearchQueryFields,
failFn: (docValue: any, testValue: any) => boolean
) =>
(doc: any) => {
@ -456,11 +419,11 @@ export const luceneLimit = (docs: any[], limit: string) => {
return docs.slice(0, numLimit)
}
export const hasFilters = (query?: Query) => {
export const hasFilters = (query?: SearchQuery) => {
if (!query) {
return false
}
const skipped = ["allOr"]
const skipped = ["allOr", "onEmptyFilter"]
for (let [key, value] of Object.entries(query)) {
if (skipped.includes(key) || typeof value !== "object") {
continue

1
packages/types/src/api/web/index.ts
View File

@ -8,3 +8,4 @@ export * from "./system"
export * from "./app"
export * from "./global"
export * from "./pagination"
export * from "./searchFilter"

54
packages/types/src/api/web/searchFilter.ts Normal file
View File

@ -0,0 +1,54 @@
import { FieldType } from "../../documents"
import { EmptyFilterOption } from "../../sdk"
export type SearchFilter = {
operator: keyof SearchQuery
onEmptyFilter?: EmptyFilterOption
field: string
type?: FieldType
value: any
externalType?: string
}
export type SearchQuery = {
allOr?: boolean
onEmptyFilter?: EmptyFilterOption
string?: {
[key: string]: string
}
fuzzy?: {
[key: string]: string
}
range?: {
[key: string]: {
high: number | string
low: number | string
}
}
equal?: {
[key: string]: any
}
notEqual?: {
[key: string]: any
}
empty?: {
[key: string]: any
}
notEmpty?: {
[key: string]: any
}
oneOf?: {
[key: string]: any[]
}
contains?: {
[key: string]: any[]
}
notContains?: {
[key: string]: any[]
}
containsAny?: {
[key: string]: any[]
}
}
export type SearchQueryFields = Omit<SearchQuery, "allOr" | "onEmptyFilter">

6
packages/types/src/sdk/search.ts
View File

@ -4,6 +4,7 @@ import { SortType } from "../api"
export interface SearchFilters {
allOr?: boolean
onEmptyFilter?: EmptyFilterOption
string?: {
[key: string]: string
}
@ -99,3 +100,8 @@ export interface SqlQuery {
sql: string
bindings?: string[]
}
export enum EmptyFilterOption {
RETURN_ALL = "all",
RETURN_NONE = "none",
}