MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into BUDI-8428/row-action-crud

This commit is contained in:
Adria Navarro 2024-07-12 11:53:12 +02:00 committed by GitHub
parent b44397d027 5ff9599e3a
commit 99b4aae7de
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
17 changed files with 158 additions and 118 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
lerna.json
View File

@ -1,6 +1,6 @@
{
"$schema": "node_modules/lerna/schemas/lerna-schema.json",
"version": "2.29.15",
"version": "2.29.19",
"npmClient": "yarn",
"packages": [
"packages/*",

5
packages/backend-core/src/db/couch/connections.ts
View File

@ -25,7 +25,10 @@ export const getCouchInfo = (connection?: string) => {
}
const authCookie = Buffer.from(`${username}:${password}`).toString("base64")
let sqlUrl = env.COUCH_DB_SQL_URL
if (!sqlUrl && urlInfo.url) {
// default for dev
if (env.isDev() && !sqlUrl) {
sqlUrl = "http://localhost:4006"
} else if (!sqlUrl && urlInfo.url) {
const parsed = new URL(urlInfo.url)
// attempt to connect on default port
sqlUrl = urlInfo.url.replace(parsed.port, "4984")

15
packages/backend-core/src/db/utils.ts
View File

@ -1,6 +1,6 @@
import env from "../environment"
import { DEFAULT_TENANT_ID, SEPARATOR, DocumentType } from "../constants"
import { getTenantId, getGlobalDBName } from "../context"
import { getTenantId, getGlobalDBName, isMultiTenant } from "../context"
import { doWithDB, directCouchAllDbs } from "./db"
import { AppState, DeletedApp, getAppMetadata } from "../cache/appMetadata"
import { isDevApp, isDevAppID, getProdAppID } from "../docIds/conversions"
@ -213,6 +213,11 @@ export function isSqsEnabledForTenant(): boolean {
return false
}
// single tenant (self host and dev) always enabled if flag set
if (!isMultiTenant()) {
return true
}
// This is to guard against the situation in tests where tests pass because
// we're not actually using SQS, we're using Lucene and the tests pass due to
// parity.
@ -222,5 +227,13 @@ export function isSqsEnabledForTenant(): boolean {
)
}
// Special case to enable all tenants, for testing in QA.
if (
env.SQS_SEARCH_ENABLE_TENANTS.length === 1 &&
env.SQS_SEARCH_ENABLE_TENANTS[0] === "*"
) {
return true
}
return env.SQS_SEARCH_ENABLE_TENANTS.includes(tenantId)
}

2
packages/backend-core/src/environment.ts
View File

@ -114,7 +114,7 @@ const environment = {
ENCRYPTION_KEY: process.env.ENCRYPTION_KEY,
API_ENCRYPTION_KEY: getAPIEncryptionKey(),
COUCH_DB_URL: process.env.COUCH_DB_URL || "http://localhost:4005",
COUCH_DB_SQL_URL: process.env.COUCH_DB_SQL_URL || "http://localhost:4006",
COUCH_DB_SQL_URL: process.env.COUCH_DB_SQL_URL,
SQS_SEARCH_ENABLE: process.env.SQS_SEARCH_ENABLE,
SQS_SEARCH_ENABLE_TENANTS:
process.env.SQS_SEARCH_ENABLE_TENANTS?.split(",") || [],

5
packages/backend-core/src/redis/tests/redis.spec.ts
View File

@ -2,6 +2,7 @@ import { GenericContainer, StartedTestContainer } from "testcontainers"
import { generator, structures } from "../../../tests"
import RedisWrapper from "../redis"
import { env } from "../.."
import { randomUUID } from "crypto"
jest.setTimeout(30000)
@ -52,10 +53,10 @@ describe("redis", () => {
describe("bulkStore", () => {
function createRandomObject(
keyLength: number,
valueGenerator: () => any = () => generator.word()
valueGenerator: () => any = () => randomUUID()
) {
return generator
.unique(() => generator.word(), keyLength)
.unique(() => randomUUID(), keyLength)
.reduce((acc, key) => {
acc[key] = valueGenerator()
return acc

52
packages/builder/src/components/automation/AutomationPanel/AutomationPanel.svelte
View File

@ -30,6 +30,16 @@
return lowerA > lowerB ? 1 : -1
})
$: groupedAutomations = filteredAutomations.reduce((acc, auto) => {
acc[auto.definition.trigger.event] ??= {
icon: auto.definition.trigger.icon,
name: (auto.definition.trigger?.name || "").toUpperCase(),
entries: [],
}
acc[auto.definition.trigger.event].entries.push(auto)
return acc
}, {})
$: showNoResults = searchString && !filteredAutomations.length
onMount(async () => {
@ -55,16 +65,25 @@
/>
</div>
<div class="side-bar-nav">
{#each filteredAutomations as automation}
<NavItem
text={automation.name}
selected={automation._id === selectedAutomationId}
on:click={() => selectAutomation(automation._id)}
selectedBy={$userSelectedResourceMap[automation._id]}
disabled={automation.disabled}
>
<EditAutomationPopover {automation} />
</NavItem>
{#each Object.values(groupedAutomations || {}) as triggerGroup}
<div class="nav-group">
<div class="nav-group-header" title={triggerGroup?.name}>
{triggerGroup?.name}
</div>
{#each triggerGroup.entries as automation}
<NavItem
icon={triggerGroup.icon}
iconColor={"var(--spectrum-global-color-gray-900)"}
text={automation.name}
selected={automation._id === selectedAutomationId}
on:click={() => selectAutomation(automation._id)}
selectedBy={$userSelectedResourceMap[automation._id]}
disabled={automation.disabled}
>
<EditAutomationPopover {automation} />
</NavItem>
{/each}
</div>
{/each}
{#if showNoResults}
@ -82,6 +101,17 @@
</Modal>
<style>
.nav-group {
padding-top: var(--spacing-l);
}
.nav-group-header {
color: var(--spectrum-global-color-gray-600);
padding: 0px calc(var(--spacing-l) + 4px);
padding-bottom: var(--spacing-l);
white-space: nowrap;
overflow: hidden;
text-overflow: ellipsis;
}
.side-bar {
flex: 0 0 260px;
display: flex;
@ -104,7 +134,7 @@
justify-content: flex-start;
align-items: center;
gap: var(--spacing-l);
padding: 0 var(--spacing-l);
padding: 0 calc(var(--spacing-l) + 4px);
}
.side-bar-nav {
flex: 1 1 auto;

18
packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte
View File

@ -250,6 +250,7 @@
},
}))
},
placeholder: false,
getOptionLabel: type => type.name,
getOptionValue: type => type.id,
options: [
@ -280,14 +281,14 @@
tableId: inputData["row"].tableId,
},
meta: {
fields: inputData["meta"].oldFields || {},
fields: inputData["meta"]?.oldFields || {},
},
onChange: e => {
onChange({
oldRow: e.detail.row,
meta: {
fields: inputData["meta"].fields,
oldFields: e.detail.meta.fields,
fields: inputData["meta"]?.fields || {},
oldFields: e.detail?.meta?.fields || {},
},
})
},
@ -304,7 +305,15 @@
row: inputData["row"],
meta: inputData["meta"] || {},
onChange: e => {
onChange(e.detail)
onChange({
row: e.detail.row,
meta: {
fields: e.detail?.meta?.fields || {},
...(isTestModal
? { oldFields: inputData["meta"]?.oldFields || {} }
: {}),
},
})
},
...baseProps,
},
@ -452,7 +461,6 @@
*/
const onChange = Utils.sequential(async update => {
const request = cloneDeep(update)
// Process app trigger updates
if (isTrigger && !isTestModal) {
// Row trigger

34
packages/builder/src/components/automation/SetupPanel/RowSelector.svelte
View File

@ -288,29 +288,27 @@
{/each}
{#if table && schemaFields}
{#key editableFields}
<div
class="add-fields-btn"
class:empty={Object.is(editableFields, {})}
bind:this={popoverAnchor}
>
<ActionButton
icon="Add"
fullWidth
on:click={() => {
customPopover.show()
}}
disabled={!schemaFields}
>Add fields
</ActionButton>
</div>
{/key}
<div
class="add-fields-btn"
class:empty={Object.is(editableFields, {})}
bind:this={popoverAnchor}
>
<ActionButton
icon="Add"
fullWidth
on:click={() => {
customPopover.show()
}}
disabled={!schemaFields}
>Add fields
</ActionButton>
</div>
{/if}
<Popover
align="center"
bind:this={customPopover}
anchor={popoverAnchor}
anchor={editableFields ? popoverAnchor : null}
useAnchorWidth
maxHeight={300}
resizable={false}

18
packages/server/src/api/routes/tests/row.spec.ts
View File

@ -819,7 +819,10 @@ describe.each([
const table = await config.api.table.save(tableRequest)
const stringValue = generator.word()
const naturalValue = generator.integer({ min: 0, max: 1000 })
// MySQL and MariaDB auto-increment fields have a minimum value of 1. If
// you try to save a row with a value of 0 it will use 1 instead.
const naturalValue = generator.integer({ min: 1, max: 1000 })
const existing = await config.api.row.save(table._id!, {
string: stringValue,
@ -1457,17 +1460,12 @@ describe.each([
delete tableRequest.schema.id
const table = await config.api.table.save(tableRequest)
const toCreate = generator
.unique(() => generator.integer({ min: 0, max: 10000 }), 10)
.map(number => ({ number, string: generator.word({ length: 30 }) }))
const rows = await Promise.all(
generator
.unique(
() => ({
string: generator.word({ length: 30 }),
number: generator.integer({ min: 0, max: 10000 }),
}),
10
)
.map(d => config.api.row.save(table._id!, d))
toCreate.map(d => config.api.row.save(table._id!, d))
)
const res = await config.api.row.exportRows(table._id!, {

14
packages/server/src/api/routes/tests/search.spec.ts
View File

@ -809,6 +809,20 @@ describe.each([
},
}).toContainExactly([{ name: "foo" }, { name: "bar" }])
})
it("empty arrays returns all when onEmptyFilter is set to return 'all'", async () => {
await expectQuery({
onEmptyFilter: EmptyFilterOption.RETURN_ALL,
oneOf: { name: [] },
}).toContainExactly([{ name: "foo" }, { name: "bar" }])
})
it("empty arrays returns all when onEmptyFilter is set to return 'none'", async () => {
await expectQuery({
onEmptyFilter: EmptyFilterOption.RETURN_NONE,
oneOf: { name: [] },
}).toContainExactly([])
})
})
describe("fuzzy", () => {

2
packages/server/src/api/routes/tests/utilities/TestFunctions.ts
View File

@ -151,7 +151,7 @@ export const checkPermissionsEndpoint = async ({
await exports
.createRequest(config.request, method, url, body)
.set(failHeader)
.expect(403)
.expect(401)
}
export const getDB = () => {

10
packages/server/src/constants/index.ts
View File

@ -24,16 +24,6 @@ export enum FilterTypes {
ONE_OF = "oneOf",
}
export const NoEmptyFilterStrings = [
FilterTypes.STRING,
FilterTypes.FUZZY,
FilterTypes.EQUAL,
FilterTypes.NOT_EQUAL,
FilterTypes.CONTAINS,
FilterTypes.NOT_CONTAINS,
FilterTypes.CONTAINS_ANY,
]
export const CanSwitchTypes = [
[FieldType.JSON, FieldType.ARRAY],
[

3
packages/server/src/environment.ts
View File

@ -28,7 +28,6 @@ const DEFAULTS = {
PLUGINS_DIR: "/plugins",
FORKED_PROCESS_NAME: "main",
JS_RUNNER_MEMORY_LIMIT: 64,
COUCH_DB_SQL_URL: "http://localhost:4006",
}
const QUERY_THREAD_TIMEOUT =
@ -44,7 +43,7 @@ const environment = {
// important - prefer app port to generic port
PORT: process.env.APP_PORT || process.env.PORT,
COUCH_DB_URL: process.env.COUCH_DB_URL,
COUCH_DB_SQL_URL: process.env.COUCH_DB_SQL_URL || DEFAULTS.COUCH_DB_SQL_URL,
COUCH_DB_SQL_URL: process.env.COUCH_DB_SQL_URL,
MINIO_URL: process.env.MINIO_URL,
WORKER_URL: process.env.WORKER_URL,
AWS_REGION: process.env.AWS_REGION,

37
packages/server/src/sdk/app/rows/search.ts
View File

@ -2,14 +2,12 @@ import {
EmptyFilterOption,
Row,
RowSearchParams,
SearchFilters,
SearchResponse,
SortOrder,
} from "@budibase/types"
import { isExternalTableID } from "../../../integrations/utils"
import * as internal from "./search/internal"
import * as external from "./search/external"
import { NoEmptyFilterStrings } from "../../../constants"
import * as sqs from "./search/sqs"
import { ExportRowsParams, ExportRowsResult } from "./search/types"
import { dataFilters } from "@budibase/shared-core"
@ -32,44 +30,11 @@ function pickApi(tableId: any) {
return internal
}
function isEmptyArray(value: any) {
return Array.isArray(value) && value.length === 0
}
// don't do a pure falsy check, as 0 is included
// https://github.com/Budibase/budibase/issues/10118
export function removeEmptyFilters(filters: SearchFilters) {
for (let filterField of NoEmptyFilterStrings) {
if (!filters[filterField]) {
continue
}
for (let filterType of Object.keys(filters)) {
if (filterType !== filterField) {
continue
}
// don't know which one we're checking, type could be anything
const value = filters[filterType] as unknown
if (typeof value === "object") {
for (let [key, value] of Object.entries(
filters[filterType] as object
)) {
if (value == null || value === "" || isEmptyArray(value)) {
// @ts-ignore
delete filters[filterField][key]
}
}
}
}
}
return filters
}
export async function search(
options: RowSearchParams
): Promise<SearchResponse<Row>> {
const isExternalTable = isExternalTableID(options.tableId)
options.query = removeEmptyFilters(options.query || {})
options.query = dataFilters.cleanupQuery(options.query || {})
options.query = dataFilters.fixupFilterArrays(options.query)
if (
!dataFilters.hasFilters(options.query) &&

20
packages/server/src/sdk/app/rows/search/sqs.ts
View File

@ -45,13 +45,10 @@ import {
getTableIDList,
} from "./filters"
import { dataFilters } from "@budibase/shared-core"
import { DEFAULT_TABLE_IDS } from "../../../../constants"
const builder = new sql.Sql(SqlClient.SQL_LITE)
const MISSING_COLUMN_REGEX = new RegExp(`no such column: .+`)
const USER_COLUMN_PREFIX_REGEX = new RegExp(
`no such column: .+${USER_COLUMN_PREFIX}`
)
const MISSING_TABLE_REGX = new RegExp(`no such table: .+`)
function buildInternalFieldList(
table: Table,
@ -240,10 +237,10 @@ async function runSqlQuery(
function resyncDefinitionsRequired(status: number, message: string) {
// pre data_ prefix on column names, need to resync
return (
(status === 400 && message?.match(USER_COLUMN_PREFIX_REGEX)) ||
// default tables aren't included in definition
(status === 400 &&
DEFAULT_TABLE_IDS.find(tableId => message?.includes(tableId))) ||
// there are tables missing - try a resync
(status === 400 && message.match(MISSING_TABLE_REGX)) ||
// there are columns missing - try a resync
(status === 400 && message.match(MISSING_COLUMN_REGEX)) ||
// no design document found, needs a full sync
(status === 404 && message?.includes(SQLITE_DESIGN_DOC_ID))
)
@ -251,7 +248,8 @@ function resyncDefinitionsRequired(status: number, message: string) {
export async function search(
options: RowSearchParams,
table: Table
table: Table,
opts?: { retrying?: boolean }
): Promise<SearchResponse<Row>> {
let { paginate, query, ...params } = options
@ -376,9 +374,9 @@ export async function search(
return response
} catch (err: any) {
const msg = typeof err === "string" ? err : err.message
if (resyncDefinitionsRequired(err.status, msg)) {
if (!opts?.retrying && resyncDefinitionsRequired(err.status, msg)) {
await sdk.tables.sqs.syncDefinition()
return search(options, table)
return search(options, table, { retrying: true })
}
// previously the internal table didn't error when a column didn't exist in search
if (err.status === 400 && msg?.match(MISSING_COLUMN_REGEX)) {

30
packages/shared-core/src/filters.ts
View File

@ -106,31 +106,49 @@ export const NoEmptyFilterStrings = [
OperatorOptions.NotEquals.value,
OperatorOptions.Contains.value,
OperatorOptions.NotContains.value,
OperatorOptions.ContainsAny.value,
OperatorOptions.In.value,
] as (keyof SearchQueryFields)[]
/**
* Removes any fields that contain empty strings that would cause inconsistent
* behaviour with how backend tables are filtered (no value means no filter).
*
* don't do a pure falsy check, as 0 is included
* https://github.com/Budibase/budibase/issues/10118
*/
const cleanupQuery = (query: SearchFilters) => {
export const cleanupQuery = (query: SearchFilters) => {
if (!query) {
return query
}
for (let filterField of NoEmptyFilterStrings) {
const operator = filterField as SearchFilterOperator
if (!query[operator]) {
if (!query[filterField]) {
continue
}
for (let [key, value] of Object.entries(query[operator]!)) {
if (value == null || value === "") {
delete query[operator]![key]
for (let filterType of Object.keys(query)) {
if (filterType !== filterField) {
continue
}
// don't know which one we're checking, type could be anything
const value = query[filterType] as unknown
if (typeof value === "object") {
for (let [key, value] of Object.entries(query[filterType] as object)) {
if (value == null || value === "" || isEmptyArray(value)) {
// @ts-ignore
delete query[filterField][key]
}
}
}
}
}
return query
}
function isEmptyArray(value: any) {
return Array.isArray(value) && value.length === 0
}
/**
* Removes a numeric prefix on field names designed to give fields uniqueness
*/

9
packages/worker/src/api/controllers/system/environment.ts
View File

@ -1,6 +1,6 @@
import { Ctx, MaintenanceType } from "@budibase/types"
import env from "../../../environment"
import { env as coreEnv } from "@budibase/backend-core"
import { env as coreEnv, db as dbCore } from "@budibase/backend-core"
import nodeFetch from "node-fetch"
let sqsAvailable: boolean
@ -12,7 +12,12 @@ async function isSqsAvailable() {
}
try {
await nodeFetch(coreEnv.COUCH_DB_SQL_URL, {
const couchInfo = dbCore.getCouchInfo()
if (!couchInfo.sqlUrl) {
sqsAvailable = false
return false
}
await nodeFetch(couchInfo.sqlUrl, {
timeout: 1000,
})
sqsAvailable = true