MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into fix/app-title-alignment

This commit is contained in:
deanhannigan 2024-07-15 12:21:01 +01:00 committed by GitHub
parent 0a50ab284f ec8c6edf68
commit bc40d20206
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
44 changed files with 581 additions and 289 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.github/workflows/budibase_ci.yml vendored
View File

@ -108,7 +108,7 @@ jobs:
- name: Pull testcontainers images
run: |
docker pull testcontainers/ryuk:0.5.1 &
docker pull budibase/couchdb:v3.2.1-sql &
docker pull budibase/couchdb:v3.2.1-sqs &
docker pull redis &
wait $(jobs -p)

2
lerna.json
View File

@ -1,6 +1,6 @@
{
"$schema": "node_modules/lerna/schemas/lerna-schema.json",
"version": "2.29.15",
"version": "2.29.20",
"npmClient": "yarn",
"packages": [
"packages/*",

5
packages/backend-core/src/db/couch/connections.ts
View File

@ -25,7 +25,10 @@ export const getCouchInfo = (connection?: string) => {
}
const authCookie = Buffer.from(`${username}:${password}`).toString("base64")
let sqlUrl = env.COUCH_DB_SQL_URL
if (!sqlUrl && urlInfo.url) {
// default for dev
if (env.isDev() && !sqlUrl) {
sqlUrl = "http://localhost:4006"
} else if (!sqlUrl && urlInfo.url) {
const parsed = new URL(urlInfo.url)
// attempt to connect on default port
sqlUrl = urlInfo.url.replace(parsed.port, "4984")

33
packages/backend-core/src/db/utils.ts
View File

@ -1,6 +1,6 @@
import env from "../environment"
import { DEFAULT_TENANT_ID, SEPARATOR, DocumentType } from "../constants"
import { getTenantId, getGlobalDBName } from "../context"
import { getTenantId, getGlobalDBName, isMultiTenant } from "../context"
import { doWithDB, directCouchAllDbs } from "./db"
import { AppState, DeletedApp, getAppMetadata } from "../cache/appMetadata"
import { isDevApp, isDevAppID, getProdAppID } from "../docIds/conversions"
@ -206,3 +206,34 @@ export function pagination<T>(
nextPage,
}
}
export function isSqsEnabledForTenant(): boolean {
const tenantId = getTenantId()
if (!env.SQS_SEARCH_ENABLE) {
return false
}
// single tenant (self host and dev) always enabled if flag set
if (!isMultiTenant()) {
return true
}
// This is to guard against the situation in tests where tests pass because
// we're not actually using SQS, we're using Lucene and the tests pass due to
// parity.
if (env.isTest() && env.SQS_SEARCH_ENABLE_TENANTS.length === 0) {
throw new Error(
"to enable SQS you must specify a list of tenants in the SQS_SEARCH_ENABLE_TENANTS env var"
)
}
// Special case to enable all tenants, for testing in QA.
if (
env.SQS_SEARCH_ENABLE_TENANTS.length === 1 &&
env.SQS_SEARCH_ENABLE_TENANTS[0] === "*"
) {
return true
}
return env.SQS_SEARCH_ENABLE_TENANTS.includes(tenantId)
}

5
packages/backend-core/src/environment.ts
View File

@ -114,8 +114,11 @@ const environment = {
ENCRYPTION_KEY: process.env.ENCRYPTION_KEY,
API_ENCRYPTION_KEY: getAPIEncryptionKey(),
COUCH_DB_URL: process.env.COUCH_DB_URL || "http://localhost:4005",
COUCH_DB_SQL_URL: process.env.COUCH_DB_SQL_URL || "http://localhost:4006",
COUCH_DB_SQL_URL: process.env.COUCH_DB_SQL_URL,
SQS_SEARCH_ENABLE: process.env.SQS_SEARCH_ENABLE,
SQS_SEARCH_ENABLE_TENANTS:
process.env.SQS_SEARCH_ENABLE_TENANTS?.split(",") || [],
SQS_MIGRATION_ENABLE: process.env.SQS_MIGRATION_ENABLE,
COUCH_DB_USERNAME: process.env.COUCH_DB_USER,
COUCH_DB_PASSWORD: process.env.COUCH_DB_PASSWORD,
GOOGLE_CLIENT_ID: process.env.GOOGLE_CLIENT_ID,

5
packages/backend-core/src/redis/tests/redis.spec.ts
View File

@ -2,6 +2,7 @@ import { GenericContainer, StartedTestContainer } from "testcontainers"
import { generator, structures } from "../../../tests"
import RedisWrapper from "../redis"
import { env } from "../.."
import { randomUUID } from "crypto"
jest.setTimeout(30000)
@ -52,10 +53,10 @@ describe("redis", () => {
describe("bulkStore", () => {
function createRandomObject(
keyLength: number,
valueGenerator: () => any = () => generator.word()
valueGenerator: () => any = () => randomUUID()
) {
return generator
.unique(() => generator.word(), keyLength)
.unique(() => randomUUID(), keyLength)
.reduce((acc, key) => {
acc[key] = valueGenerator()
return acc

9
packages/backend-core/src/users/users.ts
View File

@ -18,9 +18,10 @@ import {
CouchFindOptions,
DatabaseQueryOpts,
SearchFilters,
SearchFilterOperator,
SearchUsersRequest,
User,
BasicOperator,
ArrayOperator,
} from "@budibase/types"
import * as context from "../context"
import { getGlobalDB } from "../context"
@ -46,9 +47,9 @@ function removeUserPassword(users: User | User[]) {
export function isSupportedUserSearch(query: SearchFilters) {
const allowed = [
{ op: SearchFilterOperator.STRING, key: "email" },
{ op: SearchFilterOperator.EQUAL, key: "_id" },
{ op: SearchFilterOperator.ONE_OF, key: "_id" },
{ op: BasicOperator.STRING, key: "email" },
{ op: BasicOperator.EQUAL, key: "_id" },
{ op: ArrayOperator.ONE_OF, key: "_id" },
]
for (let [key, operation] of Object.entries(query)) {
if (typeof operation !== "object") {

52
packages/builder/src/components/automation/AutomationPanel/AutomationPanel.svelte
View File

@ -30,6 +30,16 @@
return lowerA > lowerB ? 1 : -1
})
$: groupedAutomations = filteredAutomations.reduce((acc, auto) => {
acc[auto.definition.trigger.event] ??= {
icon: auto.definition.trigger.icon,
name: (auto.definition.trigger?.name || "").toUpperCase(),
entries: [],
}
acc[auto.definition.trigger.event].entries.push(auto)
return acc
}, {})
$: showNoResults = searchString && !filteredAutomations.length
onMount(async () => {
@ -55,16 +65,25 @@
/>
</div>
<div class="side-bar-nav">
{#each filteredAutomations as automation}
<NavItem
text={automation.name}
selected={automation._id === selectedAutomationId}
on:click={() => selectAutomation(automation._id)}
selectedBy={$userSelectedResourceMap[automation._id]}
disabled={automation.disabled}
>
<EditAutomationPopover {automation} />
</NavItem>
{#each Object.values(groupedAutomations || {}) as triggerGroup}
<div class="nav-group">
<div class="nav-group-header" title={triggerGroup?.name}>
{triggerGroup?.name}
</div>
{#each triggerGroup.entries as automation}
<NavItem
icon={triggerGroup.icon}
iconColor={"var(--spectrum-global-color-gray-900)"}
text={automation.name}
selected={automation._id === selectedAutomationId}
on:click={() => selectAutomation(automation._id)}
selectedBy={$userSelectedResourceMap[automation._id]}
disabled={automation.disabled}
>
<EditAutomationPopover {automation} />
</NavItem>
{/each}
</div>
{/each}
{#if showNoResults}
@ -82,6 +101,17 @@
</Modal>
<style>
.nav-group {
padding-top: var(--spacing-l);
}
.nav-group-header {
color: var(--spectrum-global-color-gray-600);
padding: 0px calc(var(--spacing-l) + 4px);
padding-bottom: var(--spacing-l);
white-space: nowrap;
overflow: hidden;
text-overflow: ellipsis;
}
.side-bar {
flex: 0 0 260px;
display: flex;
@ -104,7 +134,7 @@
justify-content: flex-start;
align-items: center;
gap: var(--spacing-l);
padding: 0 var(--spacing-l);
padding: 0 calc(var(--spacing-l) + 4px);
}
.side-bar-nav {
flex: 1 1 auto;

18
packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte
View File

@ -250,6 +250,7 @@
},
}))
},
placeholder: false,
getOptionLabel: type => type.name,
getOptionValue: type => type.id,
options: [
@ -280,14 +281,14 @@
tableId: inputData["row"].tableId,
},
meta: {
fields: inputData["meta"].oldFields || {},
fields: inputData["meta"]?.oldFields || {},
},
onChange: e => {
onChange({
oldRow: e.detail.row,
meta: {
fields: inputData["meta"].fields,
oldFields: e.detail.meta.fields,
fields: inputData["meta"]?.fields || {},
oldFields: e.detail?.meta?.fields || {},
},
})
},
@ -304,7 +305,15 @@
row: inputData["row"],
meta: inputData["meta"] || {},
onChange: e => {
onChange(e.detail)
onChange({
row: e.detail.row,
meta: {
fields: e.detail?.meta?.fields || {},
...(isTestModal
? { oldFields: inputData["meta"]?.oldFields || {} }
: {}),
},
})
},
...baseProps,
},
@ -452,7 +461,6 @@
*/
const onChange = Utils.sequential(async update => {
const request = cloneDeep(update)
// Process app trigger updates
if (isTrigger && !isTestModal) {
// Row trigger

34
packages/builder/src/components/automation/SetupPanel/RowSelector.svelte
View File

@ -288,29 +288,27 @@
{/each}
{#if table && schemaFields}
{#key editableFields}
<div
class="add-fields-btn"
class:empty={Object.is(editableFields, {})}
bind:this={popoverAnchor}
>
<ActionButton
icon="Add"
fullWidth
on:click={() => {
customPopover.show()
}}
disabled={!schemaFields}
>Add fields
</ActionButton>
</div>
{/key}
<div
class="add-fields-btn"
class:empty={Object.is(editableFields, {})}
bind:this={popoverAnchor}
>
<ActionButton
icon="Add"
fullWidth
on:click={() => {
customPopover.show()
}}
disabled={!schemaFields}
>Add fields
</ActionButton>
</div>
{/if}
<Popover
align="center"
bind:this={customPopover}
anchor={popoverAnchor}
anchor={editableFields ? popoverAnchor : null}
useAnchorWidth
maxHeight={300}
resizable={false}

4
packages/frontend-core/src/components/FilterBuilder.svelte
View File

@ -11,7 +11,7 @@
Label,
Multiselect,
} from "@budibase/bbui"
import { FieldType, SearchFilterOperator } from "@budibase/types"
import { ArrayOperator, FieldType } from "@budibase/types"
import { generate } from "shortid"
import { QueryUtils, Constants } from "@budibase/frontend-core"
import { getContext } from "svelte"
@ -268,7 +268,7 @@
<slot name="binding" {filter} />
{:else if [FieldType.STRING, FieldType.LONGFORM, FieldType.NUMBER, FieldType.BIGINT, FieldType.FORMULA].includes(filter.type)}
<Input disabled={filter.noValue} bind:value={filter.value} />
{:else if filter.type === FieldType.ARRAY || (filter.type === FieldType.OPTIONS && filter.operator === SearchFilterOperator.ONE_OF)}
{:else if filter.type === FieldType.ARRAY || (filter.type === FieldType.OPTIONS && filter.operator === ArrayOperator.ONE_OF)}
<Multiselect
disabled={filter.noValue}
options={getFieldOptions(filter.field)}

5
packages/server/src/api/controllers/deploy/Deployment.ts
View File

@ -1,5 +1,4 @@
import newid from "../../../db/newid"
import { context } from "@budibase/backend-core"
import { context, utils } from "@budibase/backend-core"
/**
* This is used to pass around information about the deployment that is occurring
@ -12,7 +11,7 @@ export default class Deployment {
appUrl?: string
constructor(id = null) {
this._id = id || newid()
this._id = id || utils.newid()
}
setVerification(verification: any) {

21
packages/server/src/api/controllers/row/utils/sqlUtils.ts
View File

@ -22,6 +22,20 @@ export function isManyToMany(
return !!(field as ManyToManyRelationshipFieldMetadata).through
}
function isCorrectRelationship(
relationship: RelationshipsJson,
table1: Table,
table2: Table,
row: Row
): boolean {
const junctionTableId = generateJunctionTableID(table1._id!, table2._id!)
const possibleColumns = [
`${junctionTableId}.doc1.fieldName`,
`${junctionTableId}.doc2.fieldName`,
]
return !!possibleColumns.find(col => row[col] === relationship.column)
}
/**
* This iterates through the returned rows and works out what elements of the rows
* actually match up to another row (based on primary keys) - this is pretty specific
@ -64,7 +78,12 @@ export async function updateRelationshipColumns(
if (!linked._id) {
continue
}
columns[relationship.column] = linked
if (
!opts?.sqs ||
isCorrectRelationship(relationship, table, linkedTable, row)
) {
columns[relationship.column] = linked
}
}
for (let [column, related] of Object.entries(columns)) {
if (!row._id) {

6
packages/server/src/api/controllers/table/utils.ts
View File

@ -15,7 +15,7 @@ import { getViews, saveView } from "../view/utils"
import viewTemplate from "../view/viewBuilder"
import { cloneDeep } from "lodash/fp"
import { quotas } from "@budibase/pro"
import { events, context } from "@budibase/backend-core"
import { events, context, db as dbCore } from "@budibase/backend-core"
import {
AutoFieldSubType,
ContextUser,
@ -324,7 +324,7 @@ class TableSaveFunctions {
importRows: this.importRows,
user: this.user,
})
if (env.SQS_SEARCH_ENABLE) {
if (dbCore.isSqsEnabledForTenant()) {
await sdk.tables.sqs.addTable(table)
}
return table
@ -518,7 +518,7 @@ export async function internalTableCleanup(table: Table, rows?: Row[]) {
if (rows) {
await AttachmentCleanup.tableDelete(table, rows)
}
if (env.SQS_SEARCH_ENABLE) {
if (dbCore.isSqsEnabledForTenant()) {
await sdk.tables.sqs.removeTable(table)
}
}

8
packages/server/src/api/routes/tests/permissions.spec.ts
View File

@ -203,7 +203,7 @@ describe("/permission", () => {
// replicate changes before checking permissions
await config.publish()
await config.api.viewV2.publicSearch(view.id, undefined, { status: 403 })
await config.api.viewV2.publicSearch(view.id, undefined, { status: 401 })
})
it("should ignore the view permissions if the flag is not on", async () => {
@ -221,7 +221,7 @@ describe("/permission", () => {
await config.publish()
await config.api.viewV2.publicSearch(view.id, undefined, {
status: 403,
status: 401,
})
})
@ -250,8 +250,8 @@ describe("/permission", () => {
.send(basicRow(table._id))
.set(config.publicHeaders())
.expect("Content-Type", /json/)
.expect(403)
expect(res.status).toEqual(403)
.expect(401)
expect(res.status).toEqual(401)
})
})

18
packages/server/src/api/routes/tests/row.spec.ts
View File

@ -819,7 +819,10 @@ describe.each([
const table = await config.api.table.save(tableRequest)
const stringValue = generator.word()
const naturalValue = generator.integer({ min: 0, max: 1000 })
// MySQL and MariaDB auto-increment fields have a minimum value of 1. If
// you try to save a row with a value of 0 it will use 1 instead.
const naturalValue = generator.integer({ min: 1, max: 1000 })
const existing = await config.api.row.save(table._id!, {
string: stringValue,
@ -1457,17 +1460,12 @@ describe.each([
delete tableRequest.schema.id
const table = await config.api.table.save(tableRequest)
const toCreate = generator
.unique(() => generator.integer({ min: 0, max: 10000 }), 10)
.map(number => ({ number, string: generator.word({ length: 30 }) }))
const rows = await Promise.all(
generator
.unique(
() => ({
string: generator.word({ length: 30 }),
number: generator.integer({ min: 0, max: 10000 }),
}),
10
)
.map(d => config.api.row.save(table._id!, d))
toCreate.map(d => config.api.row.save(table._id!, d))
)
const res = await config.api.row.exportRows(table._id!, {

173
packages/server/src/api/routes/tests/search.spec.ts
View File

@ -54,10 +54,13 @@ describe.each([
let rows: Row[]
beforeAll(async () => {
await config.withCoreEnv({ SQS_SEARCH_ENABLE: "true" }, () => config.init())
if (isSqs) {
envCleanup = config.setEnv({ SQS_SEARCH_ENABLE: "true" })
envCleanup = config.setCoreEnv({
SQS_SEARCH_ENABLE: "true",
SQS_SEARCH_ENABLE_TENANTS: [config.getTenantId()],
})
}
await config.init()
if (config.app?.appId) {
config.app = await config.api.application.update(config.app?.appId, {
@ -780,6 +783,46 @@ describe.each([
it("fails to find nonexistent row", async () => {
await expectQuery({ oneOf: { name: ["none"] } }).toFindNothing()
})
it("can have multiple values for same column", async () => {
await expectQuery({
oneOf: {
name: ["foo", "bar"],
},
}).toContainExactly([{ name: "foo" }, { name: "bar" }])
})
it("splits comma separated strings", async () => {
await expectQuery({
oneOf: {
// @ts-ignore
name: "foo,bar",
},
}).toContainExactly([{ name: "foo" }, { name: "bar" }])
})
it("trims whitespace", async () => {
await expectQuery({
oneOf: {
// @ts-ignore
name: "foo, bar",
},
}).toContainExactly([{ name: "foo" }, { name: "bar" }])
})
it("empty arrays returns all when onEmptyFilter is set to return 'all'", async () => {
await expectQuery({
onEmptyFilter: EmptyFilterOption.RETURN_ALL,
oneOf: { name: [] },
}).toContainExactly([{ name: "foo" }, { name: "bar" }])
})
it("empty arrays returns all when onEmptyFilter is set to return 'none'", async () => {
await expectQuery({
onEmptyFilter: EmptyFilterOption.RETURN_NONE,
oneOf: { name: [] },
}).toContainExactly([])
})
})
describe("fuzzy", () => {
@ -1002,6 +1045,32 @@ describe.each([
it("fails to find nonexistent row", async () => {
await expectQuery({ oneOf: { age: [2] } }).toFindNothing()
})
// I couldn't find a way to make this work in Lucene and given that
// we're getting rid of Lucene soon I wasn't inclined to spend time on
// it.
!isLucene &&
it("can convert from a string", async () => {
await expectQuery({
oneOf: {
// @ts-ignore
age: "1",
},
}).toContainExactly([{ age: 1 }])
})
// I couldn't find a way to make this work in Lucene and given that
// we're getting rid of Lucene soon I wasn't inclined to spend time on
// it.
!isLucene &&
it("can find multiple values for same column", async () => {
await expectQuery({
oneOf: {
// @ts-ignore
age: "1,10",
},
}).toContainExactly([{ age: 1 }, { age: 10 }])
})
})
describe("range", () => {
@ -2085,6 +2154,106 @@ describe.each([
})
})
isInternal &&
describe("relations to same table", () => {
let relatedTable: Table, relatedRows: Row[]
beforeAll(async () => {
relatedTable = await createTable(
{
name: { name: "name", type: FieldType.STRING },
},
"productCategory"
)
table = await createTable({
name: { name: "name", type: FieldType.STRING },
related1: {
type: FieldType.LINK,
name: "related1",
fieldName: "main1",
tableId: relatedTable._id!,
relationshipType: RelationshipType.MANY_TO_MANY,
},
related2: {
type: FieldType.LINK,
name: "related2",
fieldName: "main2",
tableId: relatedTable._id!,
relationshipType: RelationshipType.MANY_TO_MANY,
},
})
relatedRows = await Promise.all([
config.api.row.save(relatedTable._id!, { name: "foo" }),
config.api.row.save(relatedTable._id!, { name: "bar" }),
config.api.row.save(relatedTable._id!, { name: "baz" }),
config.api.row.save(relatedTable._id!, { name: "boo" }),
])
await Promise.all([
config.api.row.save(table._id!, {
name: "test",
related1: [relatedRows[0]._id!],
related2: [relatedRows[1]._id!],
}),
config.api.row.save(table._id!, {
name: "test2",
related1: [relatedRows[2]._id!],
related2: [relatedRows[3]._id!],
}),
])
})
it("should be able to relate to same table", async () => {
await expectSearch({
query: {},
}).toContainExactly([
{
name: "test",
related1: [{ _id: relatedRows[0]._id }],
related2: [{ _id: relatedRows[1]._id }],
},
{
name: "test2",
related1: [{ _id: relatedRows[2]._id }],
related2: [{ _id: relatedRows[3]._id }],
},
])
})
isSqs &&
it("should be able to filter down to second row with equal", async () => {
await expectSearch({
query: {
equal: {
["related1.name"]: "baz",
},
},
}).toContainExactly([
{
name: "test2",
related1: [{ _id: relatedRows[2]._id }],
},
])
})
isSqs &&
it("should be able to filter down to first row with not equal", async () => {
await expectSearch({
query: {
notEqual: {
["1:related2.name"]: "bar",
["2:related2.name"]: "baz",
["3:related2.name"]: "boo",
},
},
}).toContainExactly([
{
name: "test",
related1: [{ _id: relatedRows[0]._id }],
},
])
})
})
isInternal &&
describe("no column error backwards compat", () => {
beforeAll(async () => {

3
packages/server/src/api/routes/tests/templates.spec.ts
View File

@ -86,9 +86,10 @@ describe("/templates", () => {
async source => {
const env = {
SQS_SEARCH_ENABLE: source === "sqs" ? "true" : "false",
SQS_SEARCH_ENABLE_TENANTS: [config.getTenantId()],
}
await config.withEnv(env, async () => {
await config.withCoreEnv(env, async () => {
const name = generator.guid().replaceAll("-", "")
const url = `/${name}`

2
packages/server/src/api/routes/tests/utilities/TestFunctions.ts
View File

@ -151,7 +151,7 @@ export const checkPermissionsEndpoint = async ({
await exports
.createRequest(config.request, method, url, body)
.set(failHeader)
.expect(403)
.expect(401)
}
export const getDB = () => {

26
packages/server/src/api/routes/tests/viewV2.spec.ts
View File

@ -9,7 +9,6 @@ import {
QuotaUsageType,
Row,
SaveTableRequest,
SearchFilterOperator,
SortOrder,
SortType,
StaticQuotaName,
@ -19,6 +18,7 @@ import {
ViewUIFieldMetadata,
ViewV2,
SearchResponse,
BasicOperator,
} from "@budibase/types"
import { generator, mocks } from "@budibase/backend-core/tests"
import { DatabaseName, getDatasource } from "../../../integrations/tests/utils"
@ -88,10 +88,16 @@ describe.each([
}
beforeAll(async () => {
await config.withCoreEnv(
{ SQS_SEARCH_ENABLE: isSqs ? "true" : "false" },
() => config.init()
)
if (isSqs) {
envCleanup = config.setEnv({ SQS_SEARCH_ENABLE: "true" })
envCleanup = config.setCoreEnv({
SQS_SEARCH_ENABLE: "true",
SQS_SEARCH_ENABLE_TENANTS: [config.getTenantId()],
})
}
await config.init()
if (dsProvider) {
datasource = await config.createDatasource({
@ -149,7 +155,7 @@ describe.each([
primaryDisplay: "id",
query: [
{
operator: SearchFilterOperator.EQUAL,
operator: BasicOperator.EQUAL,
field: "field",
value: "value",
},
@ -561,7 +567,7 @@ describe.each([
...view,
query: [
{
operator: SearchFilterOperator.EQUAL,
operator: BasicOperator.EQUAL,
field: "newField",
value: "thatValue",
},
@ -589,7 +595,7 @@ describe.each([
primaryDisplay: "Price",
query: [
{
operator: SearchFilterOperator.EQUAL,
operator: BasicOperator.EQUAL,
field: generator.word(),
value: generator.word(),
},
@ -673,7 +679,7 @@ describe.each([
tableId: generator.guid(),
query: [
{
operator: SearchFilterOperator.EQUAL,
operator: BasicOperator.EQUAL,
field: "newField",
value: "thatValue",
},
@ -1194,7 +1200,7 @@ describe.each([
name: generator.guid(),
query: [
{
operator: SearchFilterOperator.EQUAL,
operator: BasicOperator.EQUAL,
field: "two",
value: "bar2",
},
@ -1484,7 +1490,7 @@ describe.each([
it("does not allow public users to fetch by default", async () => {
await config.publish()
await config.api.viewV2.publicSearch(view.id, undefined, {
status: 403,
status: 401,
})
})
@ -1528,7 +1534,7 @@ describe.each([
await config.publish()
await config.api.viewV2.publicSearch(view.id, undefined, {
status: 403,
status: 401,
})
})
})

2
packages/server/src/appMigrations/migrations.ts
View File

@ -1,6 +1,6 @@
// This file should never be manually modified, use `yarn add-app-migration` in order to add a new one
import env from "../environment"
import { env } from "@budibase/backend-core"
import { AppMigration } from "."
import m20240604153647_initial_sqs from "./migrations/20240604153647_initial_sqs"

3
packages/server/src/appMigrations/migrations/20240604153647_initial_sqs.ts
View File

@ -1,8 +1,7 @@
import { context } from "@budibase/backend-core"
import { context, env } from "@budibase/backend-core"
import { allLinkDocs } from "../../db/utils"
import LinkDocumentImpl from "../../db/linkedRows/LinkDocument"
import sdk from "../../sdk"
import env from "../../environment"
const migration = async () => {
const linkDocs = await allLinkDocs()

7
packages/server/src/appMigrations/migrations/tests/20240604153647_initial_sqs.spec.ts
View File

@ -69,11 +69,14 @@ function oldLinkDocument(): Omit<LinkDocument, "tableId"> {
type SQSEnvVar = "SQS_MIGRATION_ENABLE" | "SQS_SEARCH_ENABLE"
async function sqsDisabled(envVar: SQSEnvVar, cb: () => Promise<void>) {
await config.withEnv({ [envVar]: "" }, cb)
await config.withCoreEnv({ [envVar]: "", SQS_SEARCH_ENABLE_TENANTS: [] }, cb)
}
async function sqsEnabled(envVar: SQSEnvVar, cb: () => Promise<void>) {
await config.withEnv({ [envVar]: "1" }, cb)
await config.withCoreEnv(
{ [envVar]: "1", SQS_SEARCH_ENABLE_TENANTS: [config.getTenantId()] },
cb
)
}
describe.each(["SQS_MIGRATION_ENABLE", "SQS_SEARCH_ENABLE"] as SQSEnvVar[])(

5
packages/server/src/automations/utils.ts
View File

@ -1,10 +1,9 @@
import { Thread, ThreadType } from "../threads"
import { definitions } from "./triggerInfo"
import { automationQueue } from "./bullboard"
import newid from "../db/newid"
import { updateEntityMetadata } from "../utilities"
import { MetadataTypes } from "../constants"
import { db as dbCore, context } from "@budibase/backend-core"
import { db as dbCore, context, utils } from "@budibase/backend-core"
import { getAutomationMetadataParams } from "../db/utils"
import { cloneDeep } from "lodash/fp"
import { quotas } from "@budibase/pro"
@ -207,7 +206,7 @@ export async function enableCronTrigger(appId: any, automation: Automation) {
)
}
// make a job id rather than letting Bull decide, makes it easier to handle on way out
const jobId = `${appId}_cron_${newid()}`
const jobId = `${appId}_cron_${utils.newid()}`
const job: any = await automationQueue.add(
{
automation,

10
packages/server/src/constants/index.ts
View File

@ -24,16 +24,6 @@ export enum FilterTypes {
ONE_OF = "oneOf",
}
export const NoEmptyFilterStrings = [
FilterTypes.STRING,
FilterTypes.FUZZY,
FilterTypes.EQUAL,
FilterTypes.NOT_EQUAL,
FilterTypes.CONTAINS,
FilterTypes.NOT_CONTAINS,
FilterTypes.CONTAINS_ANY,
]
export const CanSwitchTypes = [
[FieldType.JSON, FieldType.ARRAY],
[

5
packages/server/src/db/inMemoryView.ts
View File

@ -1,9 +1,8 @@
import newid from "./newid"
import { Row, Document, DBView } from "@budibase/types"
// bypass the main application db config
// use in memory pouchdb directly
import { db as dbCore } from "@budibase/backend-core"
import { db as dbCore, utils } from "@budibase/backend-core"
const Pouch = dbCore.getPouch({ inMemory: true })
@ -16,7 +15,7 @@ export async function runView(
// use a different ID each time for the DB, make sure they
// are always unique for each query, don't want overlap
// which could cause 409s
const db = new Pouch(newid())
const db = new Pouch(utils.newid())
try {
// write all the docs to the in memory Pouch (remove revs)
await db.bulkDocs(

5
packages/server/src/db/newid.ts
View File

@ -1,5 +0,0 @@
import { v4 } from "uuid"
export default function (): string {
return v4().replace(/-/g, "")
}

5
packages/server/src/db/utils.ts
View File

@ -1,5 +1,4 @@
import newid from "./newid"
import { context, db as dbCore } from "@budibase/backend-core"
import { context, db as dbCore, utils } from "@budibase/backend-core"
import {
DatabaseQueryOpts,
Datasource,
@ -15,6 +14,8 @@ import {
export { DocumentType, VirtualDocumentType } from "@budibase/types"
const newid = utils.newid
type Optional = string | null
export const enum AppStatus {

5
packages/server/src/environment.ts
View File

@ -28,7 +28,6 @@ const DEFAULTS = {
PLUGINS_DIR: "/plugins",
FORKED_PROCESS_NAME: "main",
JS_RUNNER_MEMORY_LIMIT: 64,
COUCH_DB_SQL_URL: "http://localhost:4006",
}
const QUERY_THREAD_TIMEOUT =
@ -44,7 +43,7 @@ const environment = {
// important - prefer app port to generic port
PORT: process.env.APP_PORT || process.env.PORT,
COUCH_DB_URL: process.env.COUCH_DB_URL,
COUCH_DB_SQL_URL: process.env.COUCH_DB_SQL_URL || DEFAULTS.COUCH_DB_SQL_URL,
COUCH_DB_SQL_URL: process.env.COUCH_DB_SQL_URL,
MINIO_URL: process.env.MINIO_URL,
WORKER_URL: process.env.WORKER_URL,
AWS_REGION: process.env.AWS_REGION,
@ -87,8 +86,6 @@ const environment = {
SQL_MAX_ROWS: process.env.SQL_MAX_ROWS,
SQL_LOGGING_ENABLE: process.env.SQL_LOGGING_ENABLE,
SQL_ALIASING_DISABLE: process.env.SQL_ALIASING_DISABLE,
SQS_SEARCH_ENABLE: process.env.SQS_SEARCH_ENABLE,
SQS_MIGRATION_ENABLE: process.env.SQS_MIGRATION_ENABLE,
// flags
ALLOW_DEV_AUTOMATIONS: process.env.ALLOW_DEV_AUTOMATIONS,
DISABLE_THREADING: process.env.DISABLE_THREADING,

4
packages/server/src/middleware/authorized.ts
View File

@ -96,7 +96,7 @@ const authorized =
}
if (!ctx.user) {
return ctx.throw(403, "No user info found")
return ctx.throw(401, "No user info found")
}
// get the resource roles
@ -148,7 +148,7 @@ const authorized =
// check authenticated
if (!ctx.isAuthenticated) {
return ctx.throw(403, "Session not authenticated")
return ctx.throw(401, "Session not authenticated")
}
// check general builder stuff, this middleware is a good way

4
packages/server/src/middleware/tests/authorized.spec.ts
View File

@ -105,7 +105,7 @@ describe("Authorization middleware", () => {
it("throws when no user data is present in context", async () => {
await config.executeMiddleware()
expect(config.throw).toHaveBeenCalledWith(403, "No user info found")
expect(config.throw).toHaveBeenCalledWith(401, "No user info found")
})
it("passes on to next() middleware if user is an admin", async () => {
@ -157,7 +157,7 @@ describe("Authorization middleware", () => {
await config.executeMiddleware()
expect(config.throw).toHaveBeenCalledWith(
403,
401,
"Session not authenticated"
)
})

69
packages/server/src/sdk/app/rows/search.ts
View File

@ -2,21 +2,18 @@ import {
EmptyFilterOption,
Row,
RowSearchParams,
SearchFilterOperator,
SearchFilters,
SearchResponse,
SortOrder,
} from "@budibase/types"
import { isExternalTableID } from "../../../integrations/utils"
import * as internal from "./search/internal"
import * as external from "./search/external"
import { NoEmptyFilterStrings } from "../../../constants"
import * as sqs from "./search/sqs"
import env from "../../../environment"
import { ExportRowsParams, ExportRowsResult } from "./search/types"
import { dataFilters } from "@budibase/shared-core"
import sdk from "../../index"
import { searchInputMapping } from "./search/utils"
import { db as dbCore } from "@budibase/backend-core"
export { isValidFilter } from "../../../integrations/utils"
@ -33,70 +30,12 @@ function pickApi(tableId: any) {
return internal
}
function isEmptyArray(value: any) {
return Array.isArray(value) && value.length === 0
}
// don't do a pure falsy check, as 0 is included
// https://github.com/Budibase/budibase/issues/10118
export function removeEmptyFilters(filters: SearchFilters) {
for (let filterField of NoEmptyFilterStrings) {
if (!filters[filterField]) {
continue
}
for (let filterType of Object.keys(filters)) {
if (filterType !== filterField) {
continue
}
// don't know which one we're checking, type could be anything
const value = filters[filterType] as unknown
if (typeof value === "object") {
for (let [key, value] of Object.entries(
filters[filterType] as object
)) {
if (value == null || value === "" || isEmptyArray(value)) {
// @ts-ignore
delete filters[filterField][key]
}
}
}
}
}
return filters
}
// The frontend can send single values for array fields sometimes, so to handle
// this we convert them to arrays at the controller level so that nothing below
// this has to worry about the non-array values.
function fixupFilterArrays(filters: SearchFilters) {
const arrayFields = [
SearchFilterOperator.ONE_OF,
SearchFilterOperator.CONTAINS,
SearchFilterOperator.NOT_CONTAINS,
SearchFilterOperator.CONTAINS_ANY,
]
for (const searchField of arrayFields) {
const field = filters[searchField]
if (field == null) {
continue
}
for (const key of Object.keys(field)) {
if (!Array.isArray(field[key])) {
field[key] = [field[key]]
}
}
}
return filters
}
export async function search(
options: RowSearchParams
): Promise<SearchResponse<Row>> {
const isExternalTable = isExternalTableID(options.tableId)
options.query = removeEmptyFilters(options.query || {})
options.query = fixupFilterArrays(options.query)
options.query = dataFilters.cleanupQuery(options.query || {})
options.query = dataFilters.fixupFilterArrays(options.query)
if (
!dataFilters.hasFilters(options.query) &&
options.query.onEmptyFilter === EmptyFilterOption.RETURN_NONE
@ -115,7 +54,7 @@ export async function search(
if (isExternalTable) {
return external.search(options, table)
} else if (env.SQS_SEARCH_ENABLE) {
} else if (dbCore.isSqsEnabledForTenant()) {
return sqs.search(options, table)
} else {
return internal.search(options, table)

73
packages/server/src/sdk/app/rows/search/sqs.ts
View File

@ -5,6 +5,7 @@ import {
Operation,
QueryJson,
RelationshipFieldMetadata,
RelationshipsJson,
Row,
RowSearchParams,
SearchFilters,
@ -30,7 +31,10 @@ import {
SQLITE_DESIGN_DOC_ID,
SQS_DATASOURCE_INTERNAL,
} from "@budibase/backend-core"
import { CONSTANT_INTERNAL_ROW_COLS } from "../../../../db/utils"
import {
CONSTANT_INTERNAL_ROW_COLS,
generateJunctionTableID,
} from "../../../../db/utils"
import AliasTables from "../sqlAlias"
import { outputProcessing } from "../../../../utilities/rowProcessor"
import pick from "lodash/pick"
@ -41,39 +45,43 @@ import {
getTableIDList,
} from "./filters"
import { dataFilters } from "@budibase/shared-core"
import { DEFAULT_TABLE_IDS } from "../../../../constants"
const builder = new sql.Sql(SqlClient.SQL_LITE)
const MISSING_COLUMN_REGEX = new RegExp(`no such column: .+`)
const USER_COLUMN_PREFIX_REGEX = new RegExp(
`no such column: .+${USER_COLUMN_PREFIX}`
)
const MISSING_TABLE_REGX = new RegExp(`no such table: .+`)
function buildInternalFieldList(
table: Table,
tables: Table[],
opts: { relationships: boolean } = { relationships: true }
opts?: { relationships?: RelationshipsJson[] }
) {
let fieldList: string[] = []
const addJunctionFields = (relatedTable: Table, fields: string[]) => {
fields.forEach(field => {
fieldList.push(
`${generateJunctionTableID(table._id!, relatedTable._id!)}.${field}`
)
})
}
fieldList = fieldList.concat(
CONSTANT_INTERNAL_ROW_COLS.map(col => `${table._id}.${col}`)
)
for (let col of Object.values(table.schema)) {
const isRelationship = col.type === FieldType.LINK
if (!opts.relationships && isRelationship) {
if (!opts?.relationships && isRelationship) {
continue
}
if (isRelationship) {
const linkCol = col as RelationshipFieldMetadata
const relatedTable = tables.find(table => table._id === linkCol.tableId)!
fieldList = fieldList.concat(
buildInternalFieldList(relatedTable, tables, { relationships: false })
)
// no relationships provided, don't go more than a layer deep
fieldList = fieldList.concat(buildInternalFieldList(relatedTable, tables))
addJunctionFields(relatedTable, ["doc1.fieldName", "doc2.fieldName"])
} else {
fieldList.push(`${table._id}.${mapToUserColumn(col.name)}`)
}
}
return fieldList
return [...new Set(fieldList)]
}
function cleanupFilters(
@ -165,18 +173,27 @@ function reverseUserColumnMapping(rows: Row[]) {
})
}
function runSqlQuery(json: QueryJson, tables: Table[]): Promise<Row[]>
function runSqlQuery(
json: QueryJson,
tables: Table[],
relationships: RelationshipsJson[]
): Promise<Row[]>
function runSqlQuery(
json: QueryJson,
tables: Table[],
relationships: RelationshipsJson[],
opts: { countTotalRows: true }
): Promise<number>
async function runSqlQuery(
json: QueryJson,
tables: Table[],
relationships: RelationshipsJson[],
opts?: { countTotalRows?: boolean }
) {
const alias = new AliasTables(tables.map(table => table.name))
const relationshipJunctionTableIds = relationships.map(rel => rel.through!)
const alias = new AliasTables(
tables.map(table => table.name).concat(relationshipJunctionTableIds)
)
if (opts?.countTotalRows) {
json.endpoint.operation = Operation.COUNT
}
@ -193,8 +210,13 @@ async function runSqlQuery(
let bindings = query.bindings
// quick hack for docIds
sql = sql.replace(/`doc1`.`rowId`/g, "`doc1.rowId`")
sql = sql.replace(/`doc2`.`rowId`/g, "`doc2.rowId`")
const fixJunctionDocs = (field: string) =>
["doc1", "doc2"].forEach(doc => {
sql = sql.replaceAll(`\`${doc}\`.\`${field}\``, `\`${doc}.${field}\``)
})
fixJunctionDocs("rowId")
fixJunctionDocs("fieldName")
if (Array.isArray(query)) {
throw new Error("SQS cannot currently handle multiple queries")
@ -215,10 +237,10 @@ async function runSqlQuery(
function resyncDefinitionsRequired(status: number, message: string) {
// pre data_ prefix on column names, need to resync
return (
(status === 400 && message?.match(USER_COLUMN_PREFIX_REGEX)) ||
// default tables aren't included in definition
(status === 400 &&
DEFAULT_TABLE_IDS.find(tableId => message?.includes(tableId))) ||
// there are tables missing - try a resync
(status === 400 && message.match(MISSING_TABLE_REGX)) ||
// there are columns missing - try a resync
(status === 400 && message.match(MISSING_COLUMN_REGEX)) ||
// no design document found, needs a full sync
(status === 404 && message?.includes(SQLITE_DESIGN_DOC_ID))
)
@ -226,7 +248,8 @@ function resyncDefinitionsRequired(status: number, message: string) {
export async function search(
options: RowSearchParams,
table: Table
table: Table,
opts?: { retrying?: boolean }
): Promise<SearchResponse<Row>> {
let { paginate, query, ...params } = options
@ -260,7 +283,7 @@ export async function search(
columnPrefix: USER_COLUMN_PREFIX,
},
resource: {
fields: buildInternalFieldList(table, allTables),
fields: buildInternalFieldList(table, allTables, { relationships }),
},
relationships,
}
@ -292,11 +315,11 @@ export async function search(
try {
const queries: Promise<Row[] | number>[] = []
queries.push(runSqlQuery(request, allTables))
queries.push(runSqlQuery(request, allTables, relationships))
if (options.countRows) {
// get the total count of rows
queries.push(
runSqlQuery(request, allTables, {
runSqlQuery(request, allTables, relationships, {
countTotalRows: true,
})
)
@ -351,9 +374,9 @@ export async function search(
return response
} catch (err: any) {
const msg = typeof err === "string" ? err : err.message
if (resyncDefinitionsRequired(err.status, msg)) {
if (!opts?.retrying && resyncDefinitionsRequired(err.status, msg)) {
await sdk.tables.sqs.syncDefinition()
return search(options, table)
return search(options, table, { retrying: true })
}
// previously the internal table didn't error when a column didn't exist in search
if (err.status === 400 && msg?.match(MISSING_COLUMN_REGEX)) {

11
packages/server/src/sdk/app/rows/search/tests/search.spec.ts
View File

@ -31,10 +31,17 @@ describe.each([
let rows: Row[]
beforeAll(async () => {
await config.withCoreEnv(
{ SQS_SEARCH_ENABLE: isSqs ? "true" : "false" },
() => config.init()
)
if (isSqs) {
envCleanup = config.setEnv({ SQS_SEARCH_ENABLE: "true" })
envCleanup = config.setCoreEnv({
SQS_SEARCH_ENABLE: "true",
SQS_SEARCH_ENABLE_TENANTS: [config.getTenantId()],
})
}
await config.init()
if (dsProvider) {
datasource = await config.createDatasource({

3
packages/server/src/sdk/app/rows/sqlAlias.ts
View File

@ -111,7 +111,8 @@ export default class AliasTables {
aliasField(field: string) {
const tableNames = this.tableNames
if (field.includes(".")) {
const [tableName, column] = field.split(".")
const [tableName, ...rest] = field.split(".")
const column = rest.join(".")
const foundTableName = tableNames.find(name => {
const idx = tableName.indexOf(name)
if (idx === -1 || idx > 1) {

5
packages/server/src/sdk/app/tables/getters.ts
View File

@ -1,4 +1,4 @@
import { context } from "@budibase/backend-core"
import { context, db as dbCore, env } from "@budibase/backend-core"
import { getTableParams } from "../../../db/utils"
import {
breakExternalTableId,
@ -15,7 +15,6 @@ import {
} from "@budibase/types"
import datasources from "../datasources"
import sdk from "../../../sdk"
import env from "../../../environment"
export function processTable(table: Table): Table {
if (!table) {
@ -34,7 +33,7 @@ export function processTable(table: Table): Table {
sourceId: table.sourceId || INTERNAL_TABLE_SOURCE_ID,
sourceType: TableSourceType.INTERNAL,
}
if (env.SQS_SEARCH_ENABLE) {
if (dbCore.isSqsEnabledForTenant()) {
processed.sql = !!env.SQS_SEARCH_ENABLE
}
return processed

28
packages/server/src/sdk/app/tables/internal/sqs.ts
View File

@ -127,9 +127,14 @@ function mapTable(table: Table): SQLiteTables {
// nothing exists, need to iterate though existing tables
async function buildBaseDefinition(): Promise<PreSaveSQLiteDefinition> {
const tables = await tablesSdk.getAllInternalTables()
const defaultTables = DEFAULT_TABLES
for (const defaultTable of DEFAULT_TABLES) {
// the default table doesn't exist in Couch, use the in-memory representation
if (!tables.find(table => table._id === defaultTable._id)) {
tables.push(defaultTable)
}
}
const definition = sql.designDoc.base("tableId")
for (let table of tables.concat(defaultTables)) {
for (let table of tables) {
definition.sql.tables = {
...definition.sql.tables,
...mapTable(table),
@ -176,9 +181,22 @@ export async function addTable(table: Table) {
export async function removeTable(table: Table) {
const db = context.getAppDB()
try {
const definition = await db.get<SQLiteDefinition>(SQLITE_DESIGN_DOC_ID)
if (definition.sql?.tables?.[table._id!]) {
delete definition.sql.tables[table._id!]
const [tables, definition] = await Promise.all([
tablesSdk.getAllInternalTables(),
db.get<SQLiteDefinition>(SQLITE_DESIGN_DOC_ID),
])
const tableIds = tables
.map(tbl => tbl._id!)
.filter(id => !id.includes(table._id!))
let cleanup = false
for (let tableKey of Object.keys(definition.sql?.tables || {})) {
// there are no tables matching anymore
if (!tableIds.find(id => tableKey.includes(id))) {
delete definition.sql.tables[tableKey]
cleanup = true
}
}
if (cleanup) {
await db.put(definition)
// make sure SQS is cleaned up, tables removed
await db.sqlDiskCleanup()

4
packages/server/src/sdk/tests/attachments.spec.ts
View File

@ -39,7 +39,9 @@ describe("should be able to re-write attachment URLs", () => {
}
const db = dbCore.getDB(config.getAppId())
await sdk.backups.updateAttachmentColumns(db.name, db)
await config.doInContext(config.getAppId(), () =>
sdk.backups.updateAttachmentColumns(db.name, db)
)
return {
db,

14
packages/server/src/tests/utilities/TestConfiguration.ts
View File

@ -26,6 +26,7 @@ import {
roles,
sessions,
tenancy,
utils,
} from "@budibase/backend-core"
import {
app as appController,
@ -40,7 +41,6 @@ import {
} from "./controllers"
import { cleanup } from "../../utilities/fileSystem"
import newid from "../../db/newid"
import { generateUserMetadataID } from "../../db/utils"
import { startup } from "../../startup"
import supertest from "supertest"
@ -74,6 +74,8 @@ import { cloneDeep } from "lodash"
import jwt, { Secret } from "jsonwebtoken"
import { Server } from "http"
const newid = utils.newid
mocks.licenses.init(pro)
// use unlimited license by default
@ -245,10 +247,10 @@ export default class TestConfiguration {
}
}
async withEnv(newEnvVars: Partial<typeof env>, f: () => Promise<void>) {
async withEnv<T>(newEnvVars: Partial<typeof env>, f: () => Promise<T>) {
let cleanup = this.setEnv(newEnvVars)
try {
await f()
return await f()
} finally {
cleanup()
}
@ -273,13 +275,13 @@ export default class TestConfiguration {
}
}
async withCoreEnv(
async withCoreEnv<T>(
newEnvVars: Partial<typeof coreEnv>,
f: () => Promise<void>
f: () => Promise<T>
) {
let cleanup = this.setCoreEnv(newEnvVars)
try {
await f()
return await f()
} finally {
cleanup()
}

122
packages/shared-core/src/filters.ts
View File

@ -6,6 +6,7 @@ import {
SearchFilter,
SearchFilters,
SearchQueryFields,
ArrayOperator,
SearchFilterOperator,
SortType,
FieldConstraints,
@ -14,11 +15,13 @@ import {
EmptyFilterOption,
SearchResponse,
Table,
BasicOperator,
RangeOperator,
} from "@budibase/types"
import dayjs from "dayjs"
import { OperatorOptions, SqlNumberTypeRangeMap } from "./constants"
import { deepGet, schema } from "./helpers"
import _ from "lodash"
import { isPlainObject, isEmpty } from "lodash"
const HBS_REGEX = /{{([^{].*?)}}/g
@ -103,31 +106,49 @@ export const NoEmptyFilterStrings = [
OperatorOptions.NotEquals.value,
OperatorOptions.Contains.value,
OperatorOptions.NotContains.value,
OperatorOptions.ContainsAny.value,
OperatorOptions.In.value,
] as (keyof SearchQueryFields)[]
/**
* Removes any fields that contain empty strings that would cause inconsistent
* behaviour with how backend tables are filtered (no value means no filter).
*
* don't do a pure falsy check, as 0 is included
* https://github.com/Budibase/budibase/issues/10118
*/
const cleanupQuery = (query: SearchFilters) => {
export const cleanupQuery = (query: SearchFilters) => {
if (!query) {
return query
}
for (let filterField of NoEmptyFilterStrings) {
const operator = filterField as SearchFilterOperator
if (!query[operator]) {
if (!query[filterField]) {
continue
}
for (let [key, value] of Object.entries(query[operator]!)) {
if (value == null || value === "") {
delete query[operator]![key]
for (let filterType of Object.keys(query)) {
if (filterType !== filterField) {
continue
}
// don't know which one we're checking, type could be anything
const value = query[filterType] as unknown
if (typeof value === "object") {
for (let [key, value] of Object.entries(query[filterType] as object)) {
if (value == null || value === "" || isEmptyArray(value)) {
// @ts-ignore
delete query[filterField][key]
}
}
}
}
}
return query
}
function isEmptyArray(value: any) {
return Array.isArray(value) && value.length === 0
}
/**
* Removes a numeric prefix on field names designed to give fields uniqueness
*/
@ -323,6 +344,32 @@ export const buildQuery = (filter: SearchFilter[]) => {
return query
}
// The frontend can send single values for array fields sometimes, so to handle
// this we convert them to arrays at the controller level so that nothing below
// this has to worry about the non-array values.
export function fixupFilterArrays(filters: SearchFilters) {
for (const searchField of Object.values(ArrayOperator)) {
const field = filters[searchField]
if (field == null || !isPlainObject(field)) {
continue
}
for (const key of Object.keys(field)) {
if (Array.isArray(field[key])) {
continue
}
const value = field[key] as any
if (typeof value === "string") {
field[key] = value.split(",").map((x: string) => x.trim())
} else {
field[key] = [value]
}
}
}
return filters
}
export const search = (
docs: Record<string, any>[],
query: RowSearchParams
@ -356,6 +403,7 @@ export const runQuery = (docs: Record<string, any>[], query: SearchFilters) => {
}
query = cleanupQuery(query)
query = fixupFilterArrays(query)
if (
!hasFilters(query) &&
@ -382,7 +430,7 @@ export const runQuery = (docs: Record<string, any>[], query: SearchFilters) => {
}
const stringMatch = match(
SearchFilterOperator.STRING,
BasicOperator.STRING,
(docValue: any, testValue: any) => {
if (!(typeof docValue === "string")) {
return false
@ -395,7 +443,7 @@ export const runQuery = (docs: Record<string, any>[], query: SearchFilters) => {
)
const fuzzyMatch = match(
SearchFilterOperator.FUZZY,
BasicOperator.FUZZY,
(docValue: any, testValue: any) => {
if (!(typeof docValue === "string")) {
return false
@ -408,17 +456,17 @@ export const runQuery = (docs: Record<string, any>[], query: SearchFilters) => {
)
const rangeMatch = match(
SearchFilterOperator.RANGE,
RangeOperator.RANGE,
(docValue: any, testValue: any) => {
if (docValue == null || docValue === "") {
return false
}
if (_.isObject(testValue.low) && _.isEmpty(testValue.low)) {
if (isPlainObject(testValue.low) && isEmpty(testValue.low)) {
testValue.low = undefined
}
if (_.isObject(testValue.high) && _.isEmpty(testValue.high)) {
if (isPlainObject(testValue.high) && isEmpty(testValue.high)) {
testValue.high = undefined
}
@ -497,11 +545,8 @@ export const runQuery = (docs: Record<string, any>[], query: SearchFilters) => {
(...args: T): boolean =>
!f(...args)
const equalMatch = match(SearchFilterOperator.EQUAL, _valueMatches)
const notEqualMatch = match(
SearchFilterOperator.NOT_EQUAL,
not(_valueMatches)
)
const equalMatch = match(BasicOperator.EQUAL, _valueMatches)
const notEqualMatch = match(BasicOperator.NOT_EQUAL, not(_valueMatches))
const _empty = (docValue: any) => {
if (typeof docValue === "string") {
@ -516,26 +561,24 @@ export const runQuery = (docs: Record<string, any>[], query: SearchFilters) => {
return docValue == null
}
const emptyMatch = match(SearchFilterOperator.EMPTY, _empty)
const notEmptyMatch = match(SearchFilterOperator.NOT_EMPTY, not(_empty))
const emptyMatch = match(BasicOperator.EMPTY, _empty)
const notEmptyMatch = match(BasicOperator.NOT_EMPTY, not(_empty))
const oneOf = match(
SearchFilterOperator.ONE_OF,
(docValue: any, testValue: any) => {
if (typeof testValue === "string") {
testValue = testValue.split(",")
if (typeof docValue === "number") {
testValue = testValue.map((item: string) => parseFloat(item))
}
}
if (!Array.isArray(testValue)) {
return false
}
return testValue.some(item => _valueMatches(docValue, item))
const oneOf = match(ArrayOperator.ONE_OF, (docValue: any, testValue: any) => {
if (typeof testValue === "string") {
testValue = testValue.split(",")
}
)
if (typeof docValue === "number") {
testValue = testValue.map((item: string) => parseFloat(item))
}
if (!Array.isArray(testValue)) {
return false
}
return testValue.some(item => _valueMatches(docValue, item))
})
const _contains =
(f: "some" | "every") => (docValue: any, testValue: any) => {
@ -562,7 +605,7 @@ export const runQuery = (docs: Record<string, any>[], query: SearchFilters) => {
}
const contains = match(
SearchFilterOperator.CONTAINS,
ArrayOperator.CONTAINS,
(docValue: any, testValue: any) => {
if (Array.isArray(testValue) && testValue.length === 0) {
return true
@ -571,7 +614,7 @@ export const runQuery = (docs: Record<string, any>[], query: SearchFilters) => {
}
)
const notContains = match(
SearchFilterOperator.NOT_CONTAINS,
ArrayOperator.NOT_CONTAINS,
(docValue: any, testValue: any) => {
// Not sure if this is logically correct, but at the time this code was
// written the search endpoint behaved this way and we wanted to make this
@ -582,10 +625,7 @@ export const runQuery = (docs: Record<string, any>[], query: SearchFilters) => {
return not(_contains("every"))(docValue, testValue)
}
)
const containsAny = match(
SearchFilterOperator.CONTAINS_ANY,
_contains("some")
)
const containsAny = match(ArrayOperator.CONTAINS_ANY, _contains("some"))
const docMatch = (doc: Record<string, any>) => {
const filterFunctions = {

40
packages/types/src/sdk/search.ts
View File

@ -3,20 +3,28 @@ import { Row, Table, DocumentType } from "../documents"
import { SortOrder, SortType } from "../api"
import { Knex } from "knex"
export enum SearchFilterOperator {
STRING = "string",
FUZZY = "fuzzy",
RANGE = "range",
export enum BasicOperator {
EQUAL = "equal",
NOT_EQUAL = "notEqual",
EMPTY = "empty",
NOT_EMPTY = "notEmpty",
ONE_OF = "oneOf",
FUZZY = "fuzzy",
STRING = "string",
}
export enum ArrayOperator {
CONTAINS = "contains",
NOT_CONTAINS = "notContains",
CONTAINS_ANY = "containsAny",
ONE_OF = "oneOf",
}
export enum RangeOperator {
RANGE = "range",
}
export type SearchFilterOperator = BasicOperator | ArrayOperator | RangeOperator
export enum InternalSearchFilterOperator {
COMPLEX_ID_OPERATOR = "_complexIdOperator",
}
@ -52,17 +60,17 @@ export interface SearchFilters {
// allows just fuzzy to be or - all the fuzzy/like parameters
fuzzyOr?: boolean
onEmptyFilter?: EmptyFilterOption
[SearchFilterOperator.STRING]?: BasicFilter<string>
[SearchFilterOperator.FUZZY]?: BasicFilter<string>
[SearchFilterOperator.RANGE]?: RangeFilter
[SearchFilterOperator.EQUAL]?: BasicFilter
[SearchFilterOperator.NOT_EQUAL]?: BasicFilter
[SearchFilterOperator.EMPTY]?: BasicFilter
[SearchFilterOperator.NOT_EMPTY]?: BasicFilter
[SearchFilterOperator.ONE_OF]?: ArrayFilter
[SearchFilterOperator.CONTAINS]?: ArrayFilter
[SearchFilterOperator.NOT_CONTAINS]?: ArrayFilter
[SearchFilterOperator.CONTAINS_ANY]?: ArrayFilter
[BasicOperator.STRING]?: BasicFilter<string>
[BasicOperator.FUZZY]?: BasicFilter<string>
[RangeOperator.RANGE]?: RangeFilter
[BasicOperator.EQUAL]?: BasicFilter
[BasicOperator.NOT_EQUAL]?: BasicFilter
[BasicOperator.EMPTY]?: BasicFilter
[BasicOperator.NOT_EMPTY]?: BasicFilter
[ArrayOperator.ONE_OF]?: ArrayFilter
[ArrayOperator.CONTAINS]?: ArrayFilter
[ArrayOperator.NOT_CONTAINS]?: ArrayFilter
[ArrayOperator.CONTAINS_ANY]?: ArrayFilter
// specific to SQS/SQLite search on internal tables this can be used
// to make sure the documents returned are always filtered down to a
// specific document type (such as just rows)

11
packages/worker/src/api/controllers/system/environment.ts
View File

@ -1,6 +1,6 @@
import { Ctx, MaintenanceType } from "@budibase/types"
import env from "../../../environment"
import { env as coreEnv } from "@budibase/backend-core"
import { env as coreEnv, db as dbCore } from "@budibase/backend-core"
import nodeFetch from "node-fetch"
let sqsAvailable: boolean
@ -12,7 +12,12 @@ async function isSqsAvailable() {
}
try {
await nodeFetch(coreEnv.COUCH_DB_SQL_URL, {
const couchInfo = dbCore.getCouchInfo()
if (!couchInfo.sqlUrl) {
sqsAvailable = false
return false
}
await nodeFetch(couchInfo.sqlUrl, {
timeout: 1000,
})
sqsAvailable = true
@ -24,7 +29,7 @@ async function isSqsAvailable() {
}
async function isSqsMissing() {
return env.SQS_SEARCH_ENABLE && !(await isSqsAvailable())
return coreEnv.SQS_SEARCH_ENABLE && !(await isSqsAvailable())
}
export const fetch = async (ctx: Ctx) => {

3
packages/worker/src/api/index.ts
View File

@ -5,8 +5,7 @@ const compress = require("koa-compress")
import zlib from "zlib"
import { routes } from "./routes"
import { middleware as pro, sdk } from "@budibase/pro"
import { auth, middleware } from "@budibase/backend-core"
import env from "../environment"
import { auth, middleware, env } from "@budibase/backend-core"
if (env.SQS_SEARCH_ENABLE) {
sdk.auditLogs.useSQLSearch()

1
packages/worker/src/environment.ts
View File

@ -46,7 +46,6 @@ const environment = {
DISABLE_ACCOUNT_PORTAL: process.env.DISABLE_ACCOUNT_PORTAL,
SMTP_FALLBACK_ENABLED: process.env.SMTP_FALLBACK_ENABLED,
DISABLE_DEVELOPER_LICENSE: process.env.DISABLE_DEVELOPER_LICENSE,
SQS_SEARCH_ENABLE: process.env.SQS_SEARCH_ENABLE,
BUDIBASE_ENVIRONMENT: process.env.BUDIBASE_ENVIRONMENT,
// smtp
SMTP_USER: process.env.SMTP_USER,