Merge branch 'fix/automation-improvements' of github.com:Budibase/budibase into fix/automation-improvements
This commit is contained in:
commit
9cfab6fbfd
|
@ -1,42 +1,30 @@
|
||||||
{
|
{
|
||||||
// Use IntelliSense to learn about possible attributes.
|
// Use IntelliSense to learn about possible attributes.
|
||||||
// Hover to view descriptions of existing attributes.
|
// Hover to view descriptions of existing attributes.
|
||||||
// For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
|
// For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
|
||||||
"version": "0.2.0",
|
"version": "0.2.0",
|
||||||
"configurations": [
|
"configurations": [
|
||||||
{
|
{
|
||||||
"name": "Budibase Server",
|
"name": "Budibase Server",
|
||||||
"type": "node",
|
"type": "node",
|
||||||
"request": "launch",
|
"request": "launch",
|
||||||
"runtimeArgs": [
|
"runtimeArgs": ["--nolazy", "-r", "ts-node/register/transpile-only"],
|
||||||
"--nolazy",
|
"args": ["${workspaceFolder}/packages/server/src/index.ts"],
|
||||||
"-r",
|
"cwd": "${workspaceFolder}/packages/server"
|
||||||
"ts-node/register/transpile-only"
|
},
|
||||||
],
|
{
|
||||||
"args": [
|
"name": "Budibase Worker",
|
||||||
"${workspaceFolder}/packages/server/src/index.ts"
|
"type": "node",
|
||||||
],
|
"request": "launch",
|
||||||
"cwd": "${workspaceFolder}/packages/server"
|
"runtimeArgs": ["--nolazy", "-r", "ts-node/register/transpile-only"],
|
||||||
},
|
"args": ["${workspaceFolder}/packages/worker/src/index.ts"],
|
||||||
{
|
"cwd": "${workspaceFolder}/packages/worker"
|
||||||
"name": "Budibase Worker",
|
}
|
||||||
"type": "node",
|
],
|
||||||
"request": "launch",
|
"compounds": [
|
||||||
"runtimeArgs": [
|
{
|
||||||
"--nolazy",
|
"name": "Start Budibase",
|
||||||
"-r",
|
"configurations": ["Budibase Server", "Budibase Worker"]
|
||||||
"ts-node/register/transpile-only"
|
}
|
||||||
],
|
]
|
||||||
"args": [
|
|
||||||
"${workspaceFolder}/packages/worker/src/index.ts"
|
|
||||||
],
|
|
||||||
"cwd": "${workspaceFolder}/packages/worker"
|
|
||||||
},
|
|
||||||
],
|
|
||||||
"compounds": [
|
|
||||||
{
|
|
||||||
"name": "Start Budibase",
|
|
||||||
"configurations": ["Budibase Server", "Budibase Worker"]
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
}
|
|
@ -137,7 +137,6 @@ services:
|
||||||
path: /health
|
path: /health
|
||||||
port: 10000
|
port: 10000
|
||||||
scheme: HTTP
|
scheme: HTTP
|
||||||
enabled: true
|
|
||||||
periodSeconds: 3
|
periodSeconds: 3
|
||||||
failureThreshold: 1
|
failureThreshold: 1
|
||||||
livenessProbe:
|
livenessProbe:
|
||||||
|
@ -170,7 +169,6 @@ services:
|
||||||
path: /health
|
path: /health
|
||||||
port: 4002
|
port: 4002
|
||||||
scheme: HTTP
|
scheme: HTTP
|
||||||
enabled: true
|
|
||||||
periodSeconds: 3
|
periodSeconds: 3
|
||||||
failureThreshold: 1
|
failureThreshold: 1
|
||||||
livenessProbe:
|
livenessProbe:
|
||||||
|
@ -204,7 +202,6 @@ services:
|
||||||
path: /health
|
path: /health
|
||||||
port: 4003
|
port: 4003
|
||||||
scheme: HTTP
|
scheme: HTTP
|
||||||
enabled: true
|
|
||||||
periodSeconds: 3
|
periodSeconds: 3
|
||||||
failureThreshold: 1
|
failureThreshold: 1
|
||||||
livenessProbe:
|
livenessProbe:
|
||||||
|
@ -411,14 +408,12 @@ couchdb:
|
||||||
## Ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/#configure-probes
|
## Ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/#configure-probes
|
||||||
# FOR COUCHDB
|
# FOR COUCHDB
|
||||||
livenessProbe:
|
livenessProbe:
|
||||||
enabled: true
|
|
||||||
failureThreshold: 3
|
failureThreshold: 3
|
||||||
initialDelaySeconds: 0
|
initialDelaySeconds: 0
|
||||||
periodSeconds: 10
|
periodSeconds: 10
|
||||||
successThreshold: 1
|
successThreshold: 1
|
||||||
timeoutSeconds: 1
|
timeoutSeconds: 1
|
||||||
readinessProbe:
|
readinessProbe:
|
||||||
enabled: true
|
|
||||||
failureThreshold: 3
|
failureThreshold: 3
|
||||||
initialDelaySeconds: 0
|
initialDelaySeconds: 0
|
||||||
periodSeconds: 10
|
periodSeconds: 10
|
||||||
|
|
|
@ -27,6 +27,7 @@ services:
|
||||||
BB_ADMIN_USER_EMAIL: ${BB_ADMIN_USER_EMAIL}
|
BB_ADMIN_USER_EMAIL: ${BB_ADMIN_USER_EMAIL}
|
||||||
BB_ADMIN_USER_PASSWORD: ${BB_ADMIN_USER_PASSWORD}
|
BB_ADMIN_USER_PASSWORD: ${BB_ADMIN_USER_PASSWORD}
|
||||||
PLUGINS_DIR: ${PLUGINS_DIR}
|
PLUGINS_DIR: ${PLUGINS_DIR}
|
||||||
|
OFFLINE_MODE: ${OFFLINE_MODE}
|
||||||
depends_on:
|
depends_on:
|
||||||
- worker-service
|
- worker-service
|
||||||
- redis-service
|
- redis-service
|
||||||
|
@ -54,6 +55,7 @@ services:
|
||||||
INTERNAL_API_KEY: ${INTERNAL_API_KEY}
|
INTERNAL_API_KEY: ${INTERNAL_API_KEY}
|
||||||
REDIS_URL: redis-service:6379
|
REDIS_URL: redis-service:6379
|
||||||
REDIS_PASSWORD: ${REDIS_PASSWORD}
|
REDIS_PASSWORD: ${REDIS_PASSWORD}
|
||||||
|
OFFLINE_MODE: ${OFFLINE_MODE}
|
||||||
depends_on:
|
depends_on:
|
||||||
- redis-service
|
- redis-service
|
||||||
- minio-service
|
- minio-service
|
||||||
|
|
|
@ -1,5 +1,5 @@
|
||||||
{
|
{
|
||||||
"version": "2.9.25",
|
"version": "2.9.29",
|
||||||
"npmClient": "yarn",
|
"npmClient": "yarn",
|
||||||
"packages": [
|
"packages": [
|
||||||
"packages/*"
|
"packages/*"
|
||||||
|
@ -19,4 +19,4 @@
|
||||||
"loadEnvFiles": false
|
"loadEnvFiles": false
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,7 +1,6 @@
|
||||||
import fetch from "node-fetch"
|
import fetch from "node-fetch"
|
||||||
import { getCouchInfo } from "./couch"
|
import { getCouchInfo } from "./couch"
|
||||||
import { SearchFilters, Row } from "@budibase/types"
|
import { SearchFilters, Row, EmptyFilterOption } from "@budibase/types"
|
||||||
import { createUserIndex } from "./searchIndexes/searchIndexes"
|
|
||||||
|
|
||||||
const QUERY_START_REGEX = /\d[0-9]*:/g
|
const QUERY_START_REGEX = /\d[0-9]*:/g
|
||||||
|
|
||||||
|
@ -65,6 +64,7 @@ export class QueryBuilder<T> {
|
||||||
this.#index = index
|
this.#index = index
|
||||||
this.#query = {
|
this.#query = {
|
||||||
allOr: false,
|
allOr: false,
|
||||||
|
onEmptyFilter: EmptyFilterOption.RETURN_ALL,
|
||||||
string: {},
|
string: {},
|
||||||
fuzzy: {},
|
fuzzy: {},
|
||||||
range: {},
|
range: {},
|
||||||
|
@ -218,6 +218,10 @@ export class QueryBuilder<T> {
|
||||||
this.#query.allOr = true
|
this.#query.allOr = true
|
||||||
}
|
}
|
||||||
|
|
||||||
|
setOnEmptyFilter(value: EmptyFilterOption) {
|
||||||
|
this.#query.onEmptyFilter = value
|
||||||
|
}
|
||||||
|
|
||||||
handleSpaces(input: string) {
|
handleSpaces(input: string) {
|
||||||
if (this.#noEscaping) {
|
if (this.#noEscaping) {
|
||||||
return input
|
return input
|
||||||
|
@ -289,8 +293,9 @@ export class QueryBuilder<T> {
|
||||||
const builder = this
|
const builder = this
|
||||||
let allOr = this.#query && this.#query.allOr
|
let allOr = this.#query && this.#query.allOr
|
||||||
let query = allOr ? "" : "*:*"
|
let query = allOr ? "" : "*:*"
|
||||||
|
let allFiltersEmpty = true
|
||||||
const allPreProcessingOpts = { escape: true, lowercase: true, wrap: true }
|
const allPreProcessingOpts = { escape: true, lowercase: true, wrap: true }
|
||||||
let tableId
|
let tableId: string = ""
|
||||||
if (this.#query.equal!.tableId) {
|
if (this.#query.equal!.tableId) {
|
||||||
tableId = this.#query.equal!.tableId
|
tableId = this.#query.equal!.tableId
|
||||||
delete this.#query.equal!.tableId
|
delete this.#query.equal!.tableId
|
||||||
|
@ -305,7 +310,7 @@ export class QueryBuilder<T> {
|
||||||
}
|
}
|
||||||
|
|
||||||
const contains = (key: string, value: any, mode = "AND") => {
|
const contains = (key: string, value: any, mode = "AND") => {
|
||||||
if (Array.isArray(value) && value.length === 0) {
|
if (!value || (Array.isArray(value) && value.length === 0)) {
|
||||||
return null
|
return null
|
||||||
}
|
}
|
||||||
if (!Array.isArray(value)) {
|
if (!Array.isArray(value)) {
|
||||||
|
@ -384,6 +389,12 @@ export class QueryBuilder<T> {
|
||||||
built += ` ${mode} `
|
built += ` ${mode} `
|
||||||
}
|
}
|
||||||
built += expression
|
built += expression
|
||||||
|
if (
|
||||||
|
(typeof value !== "string" && value != null) ||
|
||||||
|
(typeof value === "string" && value !== tableId && value !== "")
|
||||||
|
) {
|
||||||
|
allFiltersEmpty = false
|
||||||
|
}
|
||||||
}
|
}
|
||||||
if (opts?.returnBuilt) {
|
if (opts?.returnBuilt) {
|
||||||
return built
|
return built
|
||||||
|
@ -463,6 +474,13 @@ export class QueryBuilder<T> {
|
||||||
allOr = false
|
allOr = false
|
||||||
build({ tableId }, equal)
|
build({ tableId }, equal)
|
||||||
}
|
}
|
||||||
|
if (allFiltersEmpty) {
|
||||||
|
if (this.#query.onEmptyFilter === EmptyFilterOption.RETURN_NONE) {
|
||||||
|
return ""
|
||||||
|
} else if (this.#query?.allOr) {
|
||||||
|
return query.replace("()", "(*:*)")
|
||||||
|
}
|
||||||
|
}
|
||||||
return query
|
return query
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -1,6 +1,6 @@
|
||||||
import { newid } from "../../docIds/newid"
|
import { newid } from "../../docIds/newid"
|
||||||
import { getDB } from "../db"
|
import { getDB } from "../db"
|
||||||
import { Database } from "@budibase/types"
|
import { Database, EmptyFilterOption } from "@budibase/types"
|
||||||
import { QueryBuilder, paginatedSearch, fullSearch } from "../lucene"
|
import { QueryBuilder, paginatedSearch, fullSearch } from "../lucene"
|
||||||
|
|
||||||
const INDEX_NAME = "main"
|
const INDEX_NAME = "main"
|
||||||
|
@ -156,6 +156,76 @@ describe("lucene", () => {
|
||||||
expect(resp.rows.length).toBe(2)
|
expect(resp.rows.length).toBe(2)
|
||||||
})
|
})
|
||||||
|
|
||||||
|
describe("empty filters behaviour", () => {
|
||||||
|
it("should return all rows by default", async () => {
|
||||||
|
const builder = new QueryBuilder(dbName, INDEX_NAME)
|
||||||
|
builder.addEqual("property", "")
|
||||||
|
builder.addEqual("number", null)
|
||||||
|
builder.addString("property", "")
|
||||||
|
builder.addFuzzy("property", "")
|
||||||
|
builder.addNotEqual("number", undefined)
|
||||||
|
builder.addOneOf("number", null)
|
||||||
|
builder.addContains("array", undefined)
|
||||||
|
builder.addNotContains("array", null)
|
||||||
|
builder.addContainsAny("array", null)
|
||||||
|
|
||||||
|
const resp = await builder.run()
|
||||||
|
expect(resp.rows.length).toBe(3)
|
||||||
|
})
|
||||||
|
|
||||||
|
it("should return all rows when onEmptyFilter is ALL", async () => {
|
||||||
|
const builder = new QueryBuilder(dbName, INDEX_NAME)
|
||||||
|
builder.setOnEmptyFilter(EmptyFilterOption.RETURN_ALL)
|
||||||
|
builder.setAllOr()
|
||||||
|
builder.addEqual("property", "")
|
||||||
|
builder.addEqual("number", null)
|
||||||
|
builder.addString("property", "")
|
||||||
|
builder.addFuzzy("property", "")
|
||||||
|
builder.addNotEqual("number", undefined)
|
||||||
|
builder.addOneOf("number", null)
|
||||||
|
builder.addContains("array", undefined)
|
||||||
|
builder.addNotContains("array", null)
|
||||||
|
builder.addContainsAny("array", null)
|
||||||
|
|
||||||
|
const resp = await builder.run()
|
||||||
|
expect(resp.rows.length).toBe(3)
|
||||||
|
})
|
||||||
|
|
||||||
|
it("should return no rows when onEmptyFilter is NONE", async () => {
|
||||||
|
const builder = new QueryBuilder(dbName, INDEX_NAME)
|
||||||
|
builder.setOnEmptyFilter(EmptyFilterOption.RETURN_NONE)
|
||||||
|
builder.addEqual("property", "")
|
||||||
|
builder.addEqual("number", null)
|
||||||
|
builder.addString("property", "")
|
||||||
|
builder.addFuzzy("property", "")
|
||||||
|
builder.addNotEqual("number", undefined)
|
||||||
|
builder.addOneOf("number", null)
|
||||||
|
builder.addContains("array", undefined)
|
||||||
|
builder.addNotContains("array", null)
|
||||||
|
builder.addContainsAny("array", null)
|
||||||
|
|
||||||
|
const resp = await builder.run()
|
||||||
|
expect(resp.rows.length).toBe(0)
|
||||||
|
})
|
||||||
|
|
||||||
|
it("should return all matching rows when onEmptyFilter is NONE, but a filter value is provided", async () => {
|
||||||
|
const builder = new QueryBuilder(dbName, INDEX_NAME)
|
||||||
|
builder.setOnEmptyFilter(EmptyFilterOption.RETURN_NONE)
|
||||||
|
builder.addEqual("property", "")
|
||||||
|
builder.addEqual("number", 1)
|
||||||
|
builder.addString("property", "")
|
||||||
|
builder.addFuzzy("property", "")
|
||||||
|
builder.addNotEqual("number", undefined)
|
||||||
|
builder.addOneOf("number", null)
|
||||||
|
builder.addContains("array", undefined)
|
||||||
|
builder.addNotContains("array", null)
|
||||||
|
builder.addContainsAny("array", null)
|
||||||
|
|
||||||
|
const resp = await builder.run()
|
||||||
|
expect(resp.rows.length).toBe(1)
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
describe("skip", () => {
|
describe("skip", () => {
|
||||||
const skipDbName = `db-${newid()}`
|
const skipDbName = `db-${newid()}`
|
||||||
let docs: {
|
let docs: {
|
||||||
|
|
|
@ -35,22 +35,28 @@
|
||||||
{ value: "and", label: "Match all filters" },
|
{ value: "and", label: "Match all filters" },
|
||||||
{ value: "or", label: "Match any filter" },
|
{ value: "or", label: "Match any filter" },
|
||||||
]
|
]
|
||||||
|
const onEmptyOptions = [
|
||||||
|
{ value: "all", label: "Return all table rows" },
|
||||||
|
{ value: "none", label: "Return no rows" },
|
||||||
|
]
|
||||||
|
|
||||||
let rawFilters
|
let rawFilters
|
||||||
let matchAny = false
|
let matchAny = false
|
||||||
|
let onEmptyFilter = "all"
|
||||||
|
|
||||||
$: parseFilters(filters)
|
$: parseFilters(filters)
|
||||||
$: dispatch("change", enrichFilters(rawFilters, matchAny))
|
$: dispatch("change", enrichFilters(rawFilters, matchAny, onEmptyFilter))
|
||||||
$: enrichedSchemaFields = getFields(schemaFields || [], { allowLinks: true })
|
$: enrichedSchemaFields = getFields(schemaFields || [], { allowLinks: true })
|
||||||
$: fieldOptions = enrichedSchemaFields.map(field => field.name) || []
|
$: fieldOptions = enrichedSchemaFields.map(field => field.name) || []
|
||||||
$: valueTypeOptions = allowBindings ? ["Value", "Binding"] : ["Value"]
|
$: valueTypeOptions = allowBindings ? ["Value", "Binding"] : ["Value"]
|
||||||
|
|
||||||
// Remove field key prefixes and determine whether to use the "match all"
|
// Remove field key prefixes and determine which behaviours to use
|
||||||
// or "match any" behaviour
|
|
||||||
const parseFilters = filters => {
|
const parseFilters = filters => {
|
||||||
matchAny = filters?.find(filter => filter.operator === "allOr") != null
|
matchAny = filters?.find(filter => filter.operator === "allOr") != null
|
||||||
|
onEmptyFilter =
|
||||||
|
filters?.find(filter => filter.onEmptyFilter)?.onEmptyFilter ?? "all"
|
||||||
rawFilters = (filters || [])
|
rawFilters = (filters || [])
|
||||||
.filter(filter => filter.operator !== "allOr")
|
.filter(filter => filter.operator !== "allOr" && !filter.onEmptyFilter)
|
||||||
.map(filter => {
|
.map(filter => {
|
||||||
const { field } = filter
|
const { field } = filter
|
||||||
let newFilter = { ...filter }
|
let newFilter = { ...filter }
|
||||||
|
@ -65,8 +71,8 @@
|
||||||
}
|
}
|
||||||
|
|
||||||
// Add field key prefixes and a special metadata filter object to indicate
|
// Add field key prefixes and a special metadata filter object to indicate
|
||||||
// whether to use the "match all" or "match any" behaviour
|
// how to handle filter behaviour
|
||||||
const enrichFilters = (rawFilters, matchAny) => {
|
const enrichFilters = (rawFilters, matchAny, onEmptyFilter) => {
|
||||||
let count = 1
|
let count = 1
|
||||||
return rawFilters
|
return rawFilters
|
||||||
.filter(filter => filter.field)
|
.filter(filter => filter.field)
|
||||||
|
@ -75,6 +81,7 @@
|
||||||
field: `${count++}:${filter.field}`,
|
field: `${count++}:${filter.field}`,
|
||||||
}))
|
}))
|
||||||
.concat(matchAny ? [{ operator: "allOr" }] : [])
|
.concat(matchAny ? [{ operator: "allOr" }] : [])
|
||||||
|
.concat([{ onEmptyFilter }])
|
||||||
}
|
}
|
||||||
|
|
||||||
const addFilter = () => {
|
const addFilter = () => {
|
||||||
|
@ -186,6 +193,17 @@
|
||||||
on:change={e => (matchAny = e.detail === "or")}
|
on:change={e => (matchAny = e.detail === "or")}
|
||||||
placeholder={null}
|
placeholder={null}
|
||||||
/>
|
/>
|
||||||
|
{#if datasource?.type === "table"}
|
||||||
|
<Select
|
||||||
|
label="When filter empty"
|
||||||
|
value={onEmptyFilter}
|
||||||
|
options={onEmptyOptions}
|
||||||
|
getOptionLabel={opt => opt.label}
|
||||||
|
getOptionValue={opt => opt.value}
|
||||||
|
on:change={e => (onEmptyFilter = e.detail)}
|
||||||
|
placeholder={null}
|
||||||
|
/>
|
||||||
|
{/if}
|
||||||
</div>
|
</div>
|
||||||
<div>
|
<div>
|
||||||
<div class="filter-label">
|
<div class="filter-label">
|
||||||
|
|
|
@ -12,6 +12,7 @@ const baseConfig: Config.InitialProjectOptions = {
|
||||||
},
|
},
|
||||||
moduleNameMapper: {
|
moduleNameMapper: {
|
||||||
"@budibase/backend-core/(.*)": "<rootDir>/../backend-core/$1",
|
"@budibase/backend-core/(.*)": "<rootDir>/../backend-core/$1",
|
||||||
|
"@budibase/shared-core/(.*)": "<rootDir>/../shared-core/$1",
|
||||||
"@budibase/backend-core": "<rootDir>/../backend-core/src",
|
"@budibase/backend-core": "<rootDir>/../backend-core/src",
|
||||||
"@budibase/shared-core": "<rootDir>/../shared-core/src",
|
"@budibase/shared-core": "<rootDir>/../shared-core/src",
|
||||||
"@budibase/types": "<rootDir>/../types/src",
|
"@budibase/types": "<rootDir>/../types/src",
|
||||||
|
|
|
@ -13,8 +13,10 @@ import {
|
||||||
Row,
|
Row,
|
||||||
Table,
|
Table,
|
||||||
UserCtx,
|
UserCtx,
|
||||||
|
EmptyFilterOption,
|
||||||
} from "@budibase/types"
|
} from "@budibase/types"
|
||||||
import sdk from "../../../sdk"
|
import sdk from "../../../sdk"
|
||||||
|
import { hasFilters } from "@budibase/shared-core/src/filters"
|
||||||
|
|
||||||
export async function handleRequest(
|
export async function handleRequest(
|
||||||
operation: Operation,
|
operation: Operation,
|
||||||
|
@ -37,6 +39,13 @@ export async function handleRequest(
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (
|
||||||
|
!hasFilters(opts?.filters) &&
|
||||||
|
opts?.filters?.onEmptyFilter === EmptyFilterOption.RETURN_NONE
|
||||||
|
) {
|
||||||
|
return []
|
||||||
|
}
|
||||||
|
|
||||||
return new ExternalRequest(operation, tableId, opts?.datasource).run(
|
return new ExternalRequest(operation, tableId, opts?.datasource).run(
|
||||||
opts || {}
|
opts || {}
|
||||||
)
|
)
|
||||||
|
|
|
@ -11,6 +11,7 @@ import {
|
||||||
AutomationStepInput,
|
AutomationStepInput,
|
||||||
AutomationStepSchema,
|
AutomationStepSchema,
|
||||||
AutomationStepType,
|
AutomationStepType,
|
||||||
|
EmptyFilterOption,
|
||||||
SearchFilters,
|
SearchFilters,
|
||||||
Table,
|
Table,
|
||||||
} from "@budibase/types"
|
} from "@budibase/types"
|
||||||
|
@ -26,16 +27,6 @@ const SortOrderPretty = {
|
||||||
[SortOrder.DESCENDING]: "Descending",
|
[SortOrder.DESCENDING]: "Descending",
|
||||||
}
|
}
|
||||||
|
|
||||||
enum EmptyFilterOption {
|
|
||||||
RETURN_ALL = "all",
|
|
||||||
RETURN_NONE = "none",
|
|
||||||
}
|
|
||||||
|
|
||||||
const EmptyFilterOptionPretty = {
|
|
||||||
[EmptyFilterOption.RETURN_ALL]: "Return all table rows",
|
|
||||||
[EmptyFilterOption.RETURN_NONE]: "Return no rows",
|
|
||||||
}
|
|
||||||
|
|
||||||
export const definition: AutomationStepSchema = {
|
export const definition: AutomationStepSchema = {
|
||||||
description: "Query rows from the database",
|
description: "Query rows from the database",
|
||||||
icon: "Search",
|
icon: "Search",
|
||||||
|
@ -77,12 +68,6 @@ export const definition: AutomationStepSchema = {
|
||||||
title: "Limit",
|
title: "Limit",
|
||||||
customType: AutomationCustomIOType.QUERY_LIMIT,
|
customType: AutomationCustomIOType.QUERY_LIMIT,
|
||||||
},
|
},
|
||||||
onEmptyFilter: {
|
|
||||||
pretty: Object.values(EmptyFilterOptionPretty),
|
|
||||||
enum: Object.values(EmptyFilterOption),
|
|
||||||
type: AutomationIOType.STRING,
|
|
||||||
title: "When Filter Empty",
|
|
||||||
},
|
|
||||||
},
|
},
|
||||||
required: ["tableId"],
|
required: ["tableId"],
|
||||||
},
|
},
|
||||||
|
|
|
@ -341,10 +341,10 @@ class SqlServerIntegration extends Sql implements DatasourcePlus {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
getDefinitionSQL(tableName: string) {
|
getDefinitionSQL(tableName: string, schemaName: string) {
|
||||||
return `select *
|
return `select *
|
||||||
from INFORMATION_SCHEMA.COLUMNS
|
from INFORMATION_SCHEMA.COLUMNS
|
||||||
where TABLE_NAME='${tableName}'`
|
where TABLE_NAME='${tableName}' AND TABLE_SCHEMA='${schemaName}'`
|
||||||
}
|
}
|
||||||
|
|
||||||
getConstraintsSQL(tableName: string) {
|
getConstraintsSQL(tableName: string) {
|
||||||
|
@ -388,16 +388,18 @@ class SqlServerIntegration extends Sql implements DatasourcePlus {
|
||||||
throw "Unable to get list of tables in database"
|
throw "Unable to get list of tables in database"
|
||||||
}
|
}
|
||||||
|
|
||||||
const schema = this.config.schema || DEFAULT_SCHEMA
|
const schemaName = this.config.schema || DEFAULT_SCHEMA
|
||||||
const tableNames = tableInfo
|
const tableNames = tableInfo
|
||||||
.filter((record: any) => record.TABLE_SCHEMA === schema)
|
.filter((record: any) => record.TABLE_SCHEMA === schemaName)
|
||||||
.map((record: any) => record.TABLE_NAME)
|
.map((record: any) => record.TABLE_NAME)
|
||||||
.filter((name: string) => this.MASTER_TABLES.indexOf(name) === -1)
|
.filter((name: string) => this.MASTER_TABLES.indexOf(name) === -1)
|
||||||
|
|
||||||
const tables: Record<string, ExternalTable> = {}
|
const tables: Record<string, ExternalTable> = {}
|
||||||
for (let tableName of tableNames) {
|
for (let tableName of tableNames) {
|
||||||
// get the column definition (type)
|
// get the column definition (type)
|
||||||
const definition = await this.runSQL(this.getDefinitionSQL(tableName))
|
const definition = await this.runSQL(
|
||||||
|
this.getDefinitionSQL(tableName, schemaName)
|
||||||
|
)
|
||||||
// find primary key constraints
|
// find primary key constraints
|
||||||
const constraints = await this.runSQL(this.getConstraintsSQL(tableName))
|
const constraints = await this.runSQL(this.getConstraintsSQL(tableName))
|
||||||
// find the computed and identity columns (auto columns)
|
// find the computed and identity columns (auto columns)
|
||||||
|
|
|
@ -1,6 +1,12 @@
|
||||||
import { GenericContainer } from "testcontainers"
|
import { GenericContainer } from "testcontainers"
|
||||||
|
import {
|
||||||
import { Datasource, FieldType, Row, SourceName, Table } from "@budibase/types"
|
Datasource,
|
||||||
|
EmptyFilterOption,
|
||||||
|
FieldType,
|
||||||
|
Row,
|
||||||
|
SourceName,
|
||||||
|
Table,
|
||||||
|
} from "@budibase/types"
|
||||||
import TestConfiguration from "../../../../../tests/utilities/TestConfiguration"
|
import TestConfiguration from "../../../../../tests/utilities/TestConfiguration"
|
||||||
import { SearchParams } from "../../search"
|
import { SearchParams } from "../../search"
|
||||||
import { search } from "../external"
|
import { search } from "../external"
|
||||||
|
@ -116,6 +122,22 @@ describe.skip("external", () => {
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
|
it("empty filters search returns no data", async () => {
|
||||||
|
await config.doInContext(config.appId, async () => {
|
||||||
|
const tableId = config.table!._id!
|
||||||
|
|
||||||
|
const searchParams: SearchParams = {
|
||||||
|
tableId,
|
||||||
|
query: {
|
||||||
|
onEmptyFilter: EmptyFilterOption.RETURN_NONE,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
const result = await search(searchParams)
|
||||||
|
|
||||||
|
expect(result.rows).toHaveLength(0)
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
it("querying by fields will always return data attribute columns", async () => {
|
it("querying by fields will always return data attribute columns", async () => {
|
||||||
await config.doInContext(config.appId, async () => {
|
await config.doInContext(config.appId, async () => {
|
||||||
const tableId = config.table!._id!
|
const tableId = config.table!._id!
|
||||||
|
|
|
@ -1,4 +1,12 @@
|
||||||
import { Datasource, FieldType, SortDirection, SortType } from "@budibase/types"
|
import {
|
||||||
|
Datasource,
|
||||||
|
FieldType,
|
||||||
|
SortDirection,
|
||||||
|
SortType,
|
||||||
|
SearchFilter,
|
||||||
|
SearchQuery,
|
||||||
|
SearchQueryFields,
|
||||||
|
} from "@budibase/types"
|
||||||
import { OperatorOptions, SqlNumberTypeRangeMap } from "./constants"
|
import { OperatorOptions, SqlNumberTypeRangeMap } from "./constants"
|
||||||
import { deepGet } from "./helpers"
|
import { deepGet } from "./helpers"
|
||||||
|
|
||||||
|
@ -73,13 +81,13 @@ export const NoEmptyFilterStrings = [
|
||||||
OperatorOptions.NotEquals.value,
|
OperatorOptions.NotEquals.value,
|
||||||
OperatorOptions.Contains.value,
|
OperatorOptions.Contains.value,
|
||||||
OperatorOptions.NotContains.value,
|
OperatorOptions.NotContains.value,
|
||||||
] as (keyof QueryFields)[]
|
] as (keyof SearchQueryFields)[]
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Removes any fields that contain empty strings that would cause inconsistent
|
* Removes any fields that contain empty strings that would cause inconsistent
|
||||||
* behaviour with how backend tables are filtered (no value means no filter).
|
* behaviour with how backend tables are filtered (no value means no filter).
|
||||||
*/
|
*/
|
||||||
const cleanupQuery = (query: Query) => {
|
const cleanupQuery = (query: SearchQuery) => {
|
||||||
if (!query) {
|
if (!query) {
|
||||||
return query
|
return query
|
||||||
}
|
}
|
||||||
|
@ -110,66 +118,12 @@ const removeKeyNumbering = (key: string) => {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
type Filter = {
|
|
||||||
operator: keyof Query
|
|
||||||
field: string
|
|
||||||
type: any
|
|
||||||
value: any
|
|
||||||
externalType: keyof typeof SqlNumberTypeRangeMap
|
|
||||||
}
|
|
||||||
|
|
||||||
type Query = QueryFields & QueryConfig
|
|
||||||
type QueryFields = {
|
|
||||||
string?: {
|
|
||||||
[key: string]: string
|
|
||||||
}
|
|
||||||
fuzzy?: {
|
|
||||||
[key: string]: string
|
|
||||||
}
|
|
||||||
range?: {
|
|
||||||
[key: string]: {
|
|
||||||
high: number | string
|
|
||||||
low: number | string
|
|
||||||
}
|
|
||||||
}
|
|
||||||
equal?: {
|
|
||||||
[key: string]: any
|
|
||||||
}
|
|
||||||
notEqual?: {
|
|
||||||
[key: string]: any
|
|
||||||
}
|
|
||||||
empty?: {
|
|
||||||
[key: string]: any
|
|
||||||
}
|
|
||||||
notEmpty?: {
|
|
||||||
[key: string]: any
|
|
||||||
}
|
|
||||||
oneOf?: {
|
|
||||||
[key: string]: any[]
|
|
||||||
}
|
|
||||||
contains?: {
|
|
||||||
[key: string]: any[]
|
|
||||||
}
|
|
||||||
notContains?: {
|
|
||||||
[key: string]: any[]
|
|
||||||
}
|
|
||||||
containsAny?: {
|
|
||||||
[key: string]: any[]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
type QueryConfig = {
|
|
||||||
allOr?: boolean
|
|
||||||
}
|
|
||||||
|
|
||||||
type QueryFieldsType = keyof QueryFields
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Builds a lucene JSON query from the filter structure generated in the builder
|
* Builds a lucene JSON query from the filter structure generated in the builder
|
||||||
* @param filter the builder filter structure
|
* @param filter the builder filter structure
|
||||||
*/
|
*/
|
||||||
export const buildLuceneQuery = (filter: Filter[]) => {
|
export const buildLuceneQuery = (filter: SearchFilter[]) => {
|
||||||
let query: Query = {
|
let query: SearchQuery = {
|
||||||
string: {},
|
string: {},
|
||||||
fuzzy: {},
|
fuzzy: {},
|
||||||
range: {},
|
range: {},
|
||||||
|
@ -184,7 +138,8 @@ export const buildLuceneQuery = (filter: Filter[]) => {
|
||||||
}
|
}
|
||||||
if (Array.isArray(filter)) {
|
if (Array.isArray(filter)) {
|
||||||
filter.forEach(expression => {
|
filter.forEach(expression => {
|
||||||
let { operator, field, type, value, externalType } = expression
|
let { operator, field, type, value, externalType, onEmptyFilter } =
|
||||||
|
expression
|
||||||
const isHbs =
|
const isHbs =
|
||||||
typeof value === "string" && (value.match(HBS_REGEX) || []).length > 0
|
typeof value === "string" && (value.match(HBS_REGEX) || []).length > 0
|
||||||
// Parse all values into correct types
|
// Parse all values into correct types
|
||||||
|
@ -192,6 +147,10 @@ export const buildLuceneQuery = (filter: Filter[]) => {
|
||||||
query.allOr = true
|
query.allOr = true
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
if (onEmptyFilter) {
|
||||||
|
query.onEmptyFilter = onEmptyFilter
|
||||||
|
return
|
||||||
|
}
|
||||||
if (
|
if (
|
||||||
type === "datetime" &&
|
type === "datetime" &&
|
||||||
!isHbs &&
|
!isHbs &&
|
||||||
|
@ -227,9 +186,13 @@ export const buildLuceneQuery = (filter: Filter[]) => {
|
||||||
}
|
}
|
||||||
if (operator.startsWith("range") && query.range) {
|
if (operator.startsWith("range") && query.range) {
|
||||||
const minint =
|
const minint =
|
||||||
SqlNumberTypeRangeMap[externalType]?.min || Number.MIN_SAFE_INTEGER
|
SqlNumberTypeRangeMap[
|
||||||
|
externalType as keyof typeof SqlNumberTypeRangeMap
|
||||||
|
]?.min || Number.MIN_SAFE_INTEGER
|
||||||
const maxint =
|
const maxint =
|
||||||
SqlNumberTypeRangeMap[externalType]?.max || Number.MAX_SAFE_INTEGER
|
SqlNumberTypeRangeMap[
|
||||||
|
externalType as keyof typeof SqlNumberTypeRangeMap
|
||||||
|
]?.max || Number.MAX_SAFE_INTEGER
|
||||||
if (!query.range[field]) {
|
if (!query.range[field]) {
|
||||||
query.range[field] = {
|
query.range[field] = {
|
||||||
low: type === "number" ? minint : "0000-00-00T00:00:00.000Z",
|
low: type === "number" ? minint : "0000-00-00T00:00:00.000Z",
|
||||||
|
@ -245,7 +208,7 @@ export const buildLuceneQuery = (filter: Filter[]) => {
|
||||||
) {
|
) {
|
||||||
query.range[field].high = value
|
query.range[field].high = value
|
||||||
}
|
}
|
||||||
} else if (query[operator]) {
|
} else if (query[operator] && operator !== "onEmptyFilter") {
|
||||||
if (type === "boolean") {
|
if (type === "boolean") {
|
||||||
// Transform boolean filters to cope with null.
|
// Transform boolean filters to cope with null.
|
||||||
// "equals false" needs to be "not equals true"
|
// "equals false" needs to be "not equals true"
|
||||||
|
@ -275,7 +238,7 @@ export const buildLuceneQuery = (filter: Filter[]) => {
|
||||||
* @param docs the data
|
* @param docs the data
|
||||||
* @param query the JSON lucene query
|
* @param query the JSON lucene query
|
||||||
*/
|
*/
|
||||||
export const runLuceneQuery = (docs: any[], query?: Query) => {
|
export const runLuceneQuery = (docs: any[], query?: SearchQuery) => {
|
||||||
if (!docs || !Array.isArray(docs)) {
|
if (!docs || !Array.isArray(docs)) {
|
||||||
return []
|
return []
|
||||||
}
|
}
|
||||||
|
@ -289,7 +252,7 @@ export const runLuceneQuery = (docs: any[], query?: Query) => {
|
||||||
// Iterates over a set of filters and evaluates a fail function against a doc
|
// Iterates over a set of filters and evaluates a fail function against a doc
|
||||||
const match =
|
const match =
|
||||||
(
|
(
|
||||||
type: QueryFieldsType,
|
type: keyof SearchQueryFields,
|
||||||
failFn: (docValue: any, testValue: any) => boolean
|
failFn: (docValue: any, testValue: any) => boolean
|
||||||
) =>
|
) =>
|
||||||
(doc: any) => {
|
(doc: any) => {
|
||||||
|
@ -456,11 +419,11 @@ export const luceneLimit = (docs: any[], limit: string) => {
|
||||||
return docs.slice(0, numLimit)
|
return docs.slice(0, numLimit)
|
||||||
}
|
}
|
||||||
|
|
||||||
export const hasFilters = (query?: Query) => {
|
export const hasFilters = (query?: SearchQuery) => {
|
||||||
if (!query) {
|
if (!query) {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
const skipped = ["allOr"]
|
const skipped = ["allOr", "onEmptyFilter"]
|
||||||
for (let [key, value] of Object.entries(query)) {
|
for (let [key, value] of Object.entries(query)) {
|
||||||
if (skipped.includes(key) || typeof value !== "object") {
|
if (skipped.includes(key) || typeof value !== "object") {
|
||||||
continue
|
continue
|
||||||
|
|
|
@ -8,3 +8,4 @@ export * from "./system"
|
||||||
export * from "./app"
|
export * from "./app"
|
||||||
export * from "./global"
|
export * from "./global"
|
||||||
export * from "./pagination"
|
export * from "./pagination"
|
||||||
|
export * from "./searchFilter"
|
||||||
|
|
|
@ -0,0 +1,54 @@
|
||||||
|
import { FieldType } from "../../documents"
|
||||||
|
import { EmptyFilterOption } from "../../sdk"
|
||||||
|
|
||||||
|
export type SearchFilter = {
|
||||||
|
operator: keyof SearchQuery
|
||||||
|
onEmptyFilter?: EmptyFilterOption
|
||||||
|
field: string
|
||||||
|
type?: FieldType
|
||||||
|
value: any
|
||||||
|
externalType?: string
|
||||||
|
}
|
||||||
|
|
||||||
|
export type SearchQuery = {
|
||||||
|
allOr?: boolean
|
||||||
|
onEmptyFilter?: EmptyFilterOption
|
||||||
|
string?: {
|
||||||
|
[key: string]: string
|
||||||
|
}
|
||||||
|
fuzzy?: {
|
||||||
|
[key: string]: string
|
||||||
|
}
|
||||||
|
range?: {
|
||||||
|
[key: string]: {
|
||||||
|
high: number | string
|
||||||
|
low: number | string
|
||||||
|
}
|
||||||
|
}
|
||||||
|
equal?: {
|
||||||
|
[key: string]: any
|
||||||
|
}
|
||||||
|
notEqual?: {
|
||||||
|
[key: string]: any
|
||||||
|
}
|
||||||
|
empty?: {
|
||||||
|
[key: string]: any
|
||||||
|
}
|
||||||
|
notEmpty?: {
|
||||||
|
[key: string]: any
|
||||||
|
}
|
||||||
|
oneOf?: {
|
||||||
|
[key: string]: any[]
|
||||||
|
}
|
||||||
|
contains?: {
|
||||||
|
[key: string]: any[]
|
||||||
|
}
|
||||||
|
notContains?: {
|
||||||
|
[key: string]: any[]
|
||||||
|
}
|
||||||
|
containsAny?: {
|
||||||
|
[key: string]: any[]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export type SearchQueryFields = Omit<SearchQuery, "allOr" | "onEmptyFilter">
|
|
@ -4,6 +4,7 @@ import { SortType } from "../api"
|
||||||
|
|
||||||
export interface SearchFilters {
|
export interface SearchFilters {
|
||||||
allOr?: boolean
|
allOr?: boolean
|
||||||
|
onEmptyFilter?: EmptyFilterOption
|
||||||
string?: {
|
string?: {
|
||||||
[key: string]: string
|
[key: string]: string
|
||||||
}
|
}
|
||||||
|
@ -99,3 +100,8 @@ export interface SqlQuery {
|
||||||
sql: string
|
sql: string
|
||||||
bindings?: string[]
|
bindings?: string[]
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export enum EmptyFilterOption {
|
||||||
|
RETURN_ALL = "all",
|
||||||
|
RETURN_NONE = "none",
|
||||||
|
}
|
||||||
|
|
Loading…
Reference in New Issue