MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into demock-dynamodb

This commit is contained in:
Sam Rose 2025-03-05 14:35:46 +00:00 committed by GitHub
parent 9cbef46512 eaa6997c47
commit d930621bd1
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
24 changed files with 544 additions and 421 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
globalSetup.ts
View File

@ -88,6 +88,16 @@ export default async function setup() {
content: `
[log]
level = warn
[httpd]
socket_options = [{nodelay, true}]
[couchdb]
single_node = true
[cluster]
n = 1
q = 1
`,
target: "/opt/couchdb/etc/local.d/test-couchdb.ini",
},

2
lerna.json
View File

@ -1,6 +1,6 @@
{
"$schema": "node_modules/lerna/schemas/lerna-schema.json",
"version": "3.4.22",
"version": "3.4.24",
"npmClient": "yarn",
"concurrency": 20,
"command": {

8
packages/backend-core/src/queue/inMemoryQueue.ts
View File

@ -3,7 +3,6 @@ import { newid } from "../utils"
import { Queue, QueueOptions, JobOptions } from "./queue"
import { helpers } from "@budibase/shared-core"
import { Job, JobId, JobInformation } from "bull"
import { cloneDeep } from "lodash"
function jobToJobInformation(job: Job): JobInformation {
let cron = ""
@ -88,9 +87,7 @@ export class InMemoryQueue<T = any> implements Partial<Queue<T>> {
*/
async process(concurrencyOrFunc: number | any, func?: any) {
func = typeof concurrencyOrFunc === "number" ? func : concurrencyOrFunc
this._emitter.on("message", async msg => {
const message = cloneDeep(msg)
this._emitter.on("message", async message => {
// For the purpose of testing, don't trigger cron jobs immediately.
// Require the test to trigger them manually with timestamps.
if (!message.manualTrigger && message.opts?.repeat != null) {
@ -165,6 +162,9 @@ export class InMemoryQueue<T = any> implements Partial<Queue<T>> {
opts,
}
this._messages.push(message)
if (this._messages.length > 1000) {
this._messages.shift()
}
this._addCount++
this._emitter.emit("message", message)
}

74
packages/bbui/src/Form/Core/Multiselect.svelte
View File

@ -1,22 +1,26 @@
<script>
<script lang="ts" context="module">
type Option = any
</script>
<script lang="ts">
import Picker from "./Picker.svelte"
import { createEventDispatcher } from "svelte"
export let value = []
export let id = null
export let placeholder = null
export let disabled = false
export let options = []
export let getOptionLabel = option => option
export let getOptionValue = option => option
export let readonly = false
export let autocomplete = false
export let sort = false
export let autoWidth = false
export let searchTerm = null
export let customPopoverHeight = undefined
export let open = false
export let loading
export let value: string[] = []
export let id: string | undefined = undefined
export let placeholder: string | null = null
export let disabled: boolean = false
export let options: Option[] = []
export let getOptionLabel = (option: Option, _index?: number) => option
export let getOptionValue = (option: Option, _index?: number) => option
export let readonly: boolean = false
export let autocomplete: boolean = false
export let sort: boolean = false
export let autoWidth: boolean = false
export let searchTerm: string | null = null
export let customPopoverHeight: string | undefined = undefined
export let open: boolean = false
export let loading: boolean
export let onOptionMouseenter = () => {}
export let onOptionMouseleave = () => {}
@ -27,10 +31,15 @@
$: optionLookupMap = getOptionLookupMap(options)
$: fieldText = getFieldText(arrayValue, optionLookupMap, placeholder)
$: isOptionSelected = optionValue => selectedLookupMap[optionValue] === true
$: isOptionSelected = (optionValue: string) =>
selectedLookupMap[optionValue] === true
$: toggleOption = makeToggleOption(selectedLookupMap, arrayValue)
const getFieldText = (value, map, placeholder) => {
const getFieldText = (
value: string[],
map: Record<string, any> | null,
placeholder: string | null
) => {
if (Array.isArray(value) && value.length > 0) {
if (!map) {
return ""
@ -42,8 +51,8 @@
}
}
const getSelectedLookupMap = value => {
let map = {}
const getSelectedLookupMap = (value: string[]) => {
const map: Record<string, boolean> = {}
if (Array.isArray(value) && value.length > 0) {
value.forEach(option => {
if (option) {
@ -54,22 +63,23 @@
return map
}
const getOptionLookupMap = options => {
let map = null
if (options?.length) {
map = {}
options.forEach((option, idx) => {
const optionValue = getOptionValue(option, idx)
if (optionValue != null) {
map[optionValue] = getOptionLabel(option, idx) || ""
}
})
const getOptionLookupMap = (options: Option[]) => {
if (!options?.length) {
return null
}
const map: Record<string, any> = {}
options.forEach((option, idx) => {
const optionValue = getOptionValue(option, idx)
if (optionValue != null) {
map[optionValue] = getOptionLabel(option, idx) || ""
}
})
return map
}
const makeToggleOption = (map, value) => {
return optionValue => {
const makeToggleOption = (map: Record<string, boolean>, value: string[]) => {
return (optionValue: string) => {
if (map[optionValue]) {
const filtered = value.filter(option => option !== optionValue)
dispatch("change", filtered)

47
packages/builder/src/analytics/PosthogClient.js → packages/builder/src/analytics/PosthogClient.ts
View File

@ -1,9 +1,12 @@
import posthog from "posthog-js"
import { Events } from "./constants"
export default class PosthogClient {
constructor(token) {
token: string
initialised: boolean
constructor(token: string) {
this.token = token
this.initialised = false
}
init() {
@ -12,6 +15,8 @@ export default class PosthogClient {
posthog.init(this.token, {
autocapture: false,
capture_pageview: false,
// disable by default
disable_session_recording: true,
})
posthog.set_config({ persistence: "cookie" })
@ -22,7 +27,7 @@ export default class PosthogClient {
* Set the posthog context to the current user
* @param {String} id - unique user id
*/
identify(id) {
identify(id: string) {
if (!this.initialised) return
posthog.identify(id)
@ -32,7 +37,7 @@ export default class PosthogClient {
* Update user metadata associated with current user in posthog
* @param {Object} meta - user fields
*/
updateUser(meta) {
updateUser(meta: Record<string, any>) {
if (!this.initialised) return
posthog.people.set(meta)
@ -43,28 +48,22 @@ export default class PosthogClient {
* @param {String} event - event identifier
* @param {Object} props - properties for the event
*/
captureEvent(eventName, props) {
if (!this.initialised) return
props.sourceApp = "builder"
posthog.capture(eventName, props)
}
/**
* Submit NPS feedback to posthog.
* @param {Object} values - NPS Values
*/
npsFeedback(values) {
if (!this.initialised) return
localStorage.setItem(Events.NPS.SUBMITTED, Date.now())
const prefixedFeedback = {}
for (let key in values) {
prefixedFeedback[`feedback_${key}`] = values[key]
captureEvent(event: string, props: Record<string, any>) {
if (!this.initialised) {
return
}
posthog.capture(Events.NPS.SUBMITTED, prefixedFeedback)
props.sourceApp = "builder"
posthog.capture(event, props)
}
enableSessionRecording() {
if (!this.initialised) {
return
}
posthog.set_config({
disable_session_recording: false,
})
}
/**

4
packages/builder/src/analytics/index.js
View File

@ -31,6 +31,10 @@ class AnalyticsHub {
posthog.captureEvent(eventName, props)
}
enableSessionRecording() {
posthog.enableSessionRecording()
}
async logout() {
posthog.logout()
}

15
packages/builder/src/components/common/bindings/BindingSidePanel.svelte
View File

@ -145,9 +145,11 @@
return
}
popoverAnchor = target
const doc = new DOMParser().parseFromString(helper.description, "text/html")
hoverTarget = {
type: "helper",
description: helper.description,
description: doc.body.textContent || "",
code: getHelperExample(helper, mode === BindingMode.JavaScript),
}
popover.show()
@ -241,20 +243,19 @@
>
{#if hoverTarget.description}
<div>
<!-- eslint-disable-next-line svelte/no-at-html-tags-->
{@html hoverTarget.description}
{hoverTarget.description}
</div>
{/if}
{#if hoverTarget.code}
{#if mode === BindingMode.JavaScript}
{#if mode === BindingMode.Text || (mode === BindingMode.JavaScript && hoverTarget.type === "binding")}
<!-- eslint-disable-next-line svelte/no-at-html-tags-->
<pre>{@html hoverTarget.code}</pre>
{:else}
<CodeEditor
value={hoverTarget.code?.trim()}
mode={EditorModes.JS}
readonly
/>
{:else if mode === BindingMode.Text}
<!-- eslint-disable-next-line svelte/no-at-html-tags-->
<pre>{@html hoverTarget.code}</pre>
{/if}
{/if}
</div>

3
packages/builder/src/stores/portal/admin.ts
View File

@ -8,6 +8,7 @@ import {
SystemStatusResponse,
} from "@budibase/types"
import { BudiStore } from "../BudiStore"
import Analytics from "../../analytics"
interface AdminState extends GetEnvironmentResponse {
loaded: boolean
@ -33,6 +34,8 @@ export class AdminStore extends BudiStore<AdminState> {
await this.getEnvironment()
// enable system status checks in the cloud
if (get(this.store).cloud) {
// in cloud allow this
Analytics.enableSessionRecording()
await this.getSystemStatus()
this.checkStatus()
}

3
packages/client/src/components/app/forms/Field.svelte
View File

@ -5,11 +5,12 @@
import { memo } from "@budibase/frontend-core"
import Placeholder from "../Placeholder.svelte"
import InnerForm from "./InnerForm.svelte"
import type { FieldApi } from "."
export let label: string | undefined = undefined
export let field: string | undefined = undefined
export let fieldState: any
export let fieldApi: any
export let fieldApi: FieldApi
export let fieldSchema: any
export let defaultValue: string | undefined = undefined
export let type: any

27
packages/client/src/components/app/forms/RelationshipField.svelte
View File

@ -1,6 +1,6 @@
<script lang="ts">
import { CoreSelect, CoreMultiselect } from "@budibase/bbui"
import { FieldType, InternalTable } from "@budibase/types"
import { BasicOperator, FieldType, InternalTable } from "@budibase/types"
import { fetchData, Utils } from "@budibase/frontend-core"
import { getContext } from "svelte"
import Field from "./Field.svelte"
@ -9,10 +9,11 @@
RelationshipFieldMetadata,
Row,
} from "@budibase/types"
import type { FieldApi, FieldState } from "."
export let field: string | undefined = undefined
export let label: string | undefined = undefined
export let placeholder: any = undefined
export let placeholder: string | undefined = undefined
export let disabled: boolean = false
export let readonly: boolean = false
export let validation: any
@ -35,12 +36,13 @@
const { API } = getContext("sdk")
// Field state
let fieldState: any
let fieldApi: any
let fieldState: FieldState<string | string[]> | undefined
let fieldApi: FieldApi
let fieldSchema: RelationshipFieldMetadata | undefined
// Local UI state
let searchTerm: any
let searchTerm: string
let open: boolean = false
// Options state
@ -106,17 +108,14 @@
filter: SearchFilter[],
linkedTableId?: string
) => {
if (!linkedTableId) {
return undefined
}
const datasource =
datasourceType === "table"
dsType === "table"
? {
type: datasourceType,
tableId: fieldSchema?.tableId!,
type: dsType,
tableId: linkedTableId!,
}
: {
type: datasourceType,
type: dsType,
tableId: InternalTable.USER_METADATA,
}
return fetchData({
@ -306,14 +305,14 @@
}
// Ensure we match all filters, rather than any
let newFilter: any = filter
let newFilter = filter
if (searchTerm) {
// @ts-expect-error this doesn't fit types, but don't want to change it yet
newFilter = (newFilter || []).filter(x => x.operator !== "allOr")
newFilter.push({
// Use a big numeric prefix to avoid clashing with an existing filter
field: `999:${primaryDisplay}`,
operator: "string",
operator: BasicOperator.STRING,
value: searchTerm,
})
}

12
packages/client/src/components/app/forms/index.js → packages/client/src/components/app/forms/index.ts
View File

@ -19,3 +19,15 @@ export { default as codescanner } from "./CodeScannerField.svelte"
export { default as signaturesinglefield } from "./SignatureField.svelte"
export { default as bbreferencefield } from "./BBReferenceField.svelte"
export { default as bbreferencesinglefield } from "./BBReferenceSingleField.svelte"
export interface FieldApi {
setValue(value: any): boolean
deregister(): void
}
export interface FieldState<T> {
value: T
fieldId: string
disabled: boolean
readonly: boolean
}

3
packages/server/src/api/routes/tests/automation.spec.ts
View File

@ -290,8 +290,7 @@ describe("/automations", () => {
await setup.delay(500)
let elements = await getAllTableRows(config)
// don't test it unless there are values to test
if (elements.length > 1) {
expect(elements.length).toBeGreaterThanOrEqual(MAX_RETRIES)
if (elements.length >= 1) {
expect(elements[0].name).toEqual("Test")
expect(elements[0].description).toEqual("TEST")
return

538
packages/server/src/api/routes/tests/row.spec.ts
View File

@ -166,18 +166,6 @@ if (descriptions.length) {
)
}
const resetRowUsage = async () => {
await config.doInContext(
undefined,
async () =>
await quotas.setUsage(
0,
StaticQuotaName.ROWS,
QuotaUsageType.STATIC
)
)
}
const getRowUsage = async () => {
const { total } = await config.doInContext(undefined, () =>
quotas.getCurrentUsageValues(
@ -188,19 +176,27 @@ if (descriptions.length) {
return total
}
const assertRowUsage = async (expected: number) => {
const usage = await getRowUsage()
async function expectRowUsage(expected: number, f: () => Promise<void>) {
const before = await getRowUsage()
await f()
const after = await getRowUsage()
const usage = after - before
// Because our quota tracking is not perfect, we allow a 10% margin of
// error. This is to account for the fact that parallel writes can result
// in some quota updates getting lost. We don't have any need to solve this
// right now, so we just allow for some error.
// error. This is to account for the fact that parallel writes can
// result in some quota updates getting lost. We don't have any need
// to solve this right now, so we just allow for some error.
if (expected === 0) {
expect(usage).toEqual(0)
return
}
expect(usage).toBeGreaterThan(expected * 0.9)
expect(usage).toBeLessThan(expected * 1.1)
if (usage < 0) {
expect(usage).toBeGreaterThan(expected * 1.1)
expect(usage).toBeLessThan(expected * 0.9)
} else {
expect(usage).toBeGreaterThan(expected * 0.9)
expect(usage).toBeLessThan(expected * 1.1)
}
}
const defaultRowFields = isInternal
@ -215,91 +211,86 @@ if (descriptions.length) {
table = await config.api.table.save(defaultTable())
})
beforeEach(async () => {
await resetRowUsage()
})
describe("create", () => {
it("creates a new row successfully", async () => {
const rowUsage = await getRowUsage()
const row = await config.api.row.save(table._id!, {
name: "Test Contact",
await expectRowUsage(isInternal ? 1 : 0, async () => {
const row = await config.api.row.save(table._id!, {
name: "Test Contact",
})
expect(row.name).toEqual("Test Contact")
expect(row._rev).toBeDefined()
})
expect(row.name).toEqual("Test Contact")
expect(row._rev).toBeDefined()
await assertRowUsage(isInternal ? rowUsage + 1 : rowUsage)
})
it("fails to create a row for a table that does not exist", async () => {
const rowUsage = await getRowUsage()
await config.api.row.save("1234567", {}, { status: 404 })
await assertRowUsage(rowUsage)
await expectRowUsage(0, async () => {
await config.api.row.save("1234567", {}, { status: 404 })
})
})
it("fails to create a row if required fields are missing", async () => {
const rowUsage = await getRowUsage()
const table = await config.api.table.save(
saveTableRequest({
schema: {
required: {
type: FieldType.STRING,
name: "required",
constraints: {
type: "string",
presence: true,
},
},
},
})
)
await config.api.row.save(
table._id!,
{},
{
status: 500,
body: {
validationErrors: {
required: ["can't be blank"],
},
},
}
)
await assertRowUsage(rowUsage)
})
isInternal &&
it("increment row autoId per create row request", async () => {
const rowUsage = await getRowUsage()
const newTable = await config.api.table.save(
await expectRowUsage(0, async () => {
const table = await config.api.table.save(
saveTableRequest({
schema: {
"Row ID": {
name: "Row ID",
type: FieldType.NUMBER,
subtype: AutoFieldSubType.AUTO_ID,
icon: "ri-magic-line",
autocolumn: true,
required: {
type: FieldType.STRING,
name: "required",
constraints: {
type: "number",
type: "string",
presence: true,
numericality: {
greaterThanOrEqualTo: "",
lessThanOrEqualTo: "",
},
},
},
},
})
)
await config.api.row.save(
table._id!,
{},
{
status: 500,
body: {
validationErrors: {
required: ["can't be blank"],
},
},
}
)
})
})
let previousId = 0
for (let i = 0; i < 10; i++) {
const row = await config.api.row.save(newTable._id!, {})
expect(row["Row ID"]).toBeGreaterThan(previousId)
previousId = row["Row ID"]
}
await assertRowUsage(isInternal ? rowUsage + 10 : rowUsage)
isInternal &&
it("increment row autoId per create row request", async () => {
await expectRowUsage(isInternal ? 10 : 0, async () => {
const newTable = await config.api.table.save(
saveTableRequest({
schema: {
"Row ID": {
name: "Row ID",
type: FieldType.NUMBER,
subtype: AutoFieldSubType.AUTO_ID,
icon: "ri-magic-line",
autocolumn: true,
constraints: {
type: "number",
presence: true,
numericality: {
greaterThanOrEqualTo: "",
lessThanOrEqualTo: "",
},
},
},
},
})
)
let previousId = 0
for (let i = 0; i < 10; i++) {
const row = await config.api.row.save(newTable._id!, {})
expect(row["Row ID"]).toBeGreaterThan(previousId)
previousId = row["Row ID"]
}
})
})
isInternal &&
@ -985,16 +976,16 @@ if (descriptions.length) {
describe("update", () => {
it("updates an existing row successfully", async () => {
const existing = await config.api.row.save(table._id!, {})
const rowUsage = await getRowUsage()
const res = await config.api.row.save(table._id!, {
_id: existing._id,
_rev: existing._rev,
name: "Updated Name",
await expectRowUsage(0, async () => {
const res = await config.api.row.save(table._id!, {
_id: existing._id,
_rev: existing._rev,
name: "Updated Name",
})
expect(res.name).toEqual("Updated Name")
})
expect(res.name).toEqual("Updated Name")
await assertRowUsage(rowUsage)
})
!isInternal &&
@ -1177,23 +1168,22 @@ if (descriptions.length) {
it("should update only the fields that are supplied", async () => {
const existing = await config.api.row.save(table._id!, {})
const rowUsage = await getRowUsage()
await expectRowUsage(0, async () => {
const row = await config.api.row.patch(table._id!, {
_id: existing._id!,
_rev: existing._rev!,
tableId: table._id!,
name: "Updated Name",
})
const row = await config.api.row.patch(table._id!, {
_id: existing._id!,
_rev: existing._rev!,
tableId: table._id!,
name: "Updated Name",
expect(row.name).toEqual("Updated Name")
expect(row.description).toEqual(existing.description)
const savedRow = await config.api.row.get(table._id!, row._id!)
expect(savedRow.description).toEqual(existing.description)
expect(savedRow.name).toEqual("Updated Name")
})
expect(row.name).toEqual("Updated Name")
expect(row.description).toEqual(existing.description)
const savedRow = await config.api.row.get(table._id!, row._id!)
expect(savedRow.description).toEqual(existing.description)
expect(savedRow.name).toEqual("Updated Name")
await assertRowUsage(rowUsage)
})
it("should update only the fields that are supplied and emit the correct oldRow", async () => {
@ -1224,20 +1214,19 @@ if (descriptions.length) {
it("should throw an error when given improper types", async () => {
const existing = await config.api.row.save(table._id!, {})
const rowUsage = await getRowUsage()
await config.api.row.patch(
table._id!,
{
_id: existing._id!,
_rev: existing._rev!,
tableId: table._id!,
name: 1,
},
{ status: 400 }
)
await assertRowUsage(rowUsage)
await expectRowUsage(0, async () => {
await config.api.row.patch(
table._id!,
{
_id: existing._id!,
_rev: existing._rev!,
tableId: table._id!,
name: 1,
},
{ status: 400 }
)
})
})
it("should not overwrite links if those links are not set", async () => {
@ -1452,25 +1441,25 @@ if (descriptions.length) {
it("should be able to delete a row", async () => {
const createdRow = await config.api.row.save(table._id!, {})
const rowUsage = await getRowUsage()
const res = await config.api.row.bulkDelete(table._id!, {
rows: [createdRow],
await expectRowUsage(isInternal ? -1 : 0, async () => {
const res = await config.api.row.bulkDelete(table._id!, {
rows: [createdRow],
})
expect(res[0]._id).toEqual(createdRow._id)
})
expect(res[0]._id).toEqual(createdRow._id)
await assertRowUsage(isInternal ? rowUsage - 1 : rowUsage)
})
it("should be able to delete a row with ID only", async () => {
const createdRow = await config.api.row.save(table._id!, {})
const rowUsage = await getRowUsage()
const res = await config.api.row.bulkDelete(table._id!, {
rows: [createdRow._id!],
await expectRowUsage(isInternal ? -1 : 0, async () => {
const res = await config.api.row.bulkDelete(table._id!, {
rows: [createdRow._id!],
})
expect(res[0]._id).toEqual(createdRow._id)
expect(res[0].tableId).toEqual(table._id!)
})
expect(res[0]._id).toEqual(createdRow._id)
expect(res[0].tableId).toEqual(table._id!)
await assertRowUsage(isInternal ? rowUsage - 1 : rowUsage)
})
it("should be able to bulk delete rows, including a row that doesn't exist", async () => {
@ -1560,31 +1549,29 @@ if (descriptions.length) {
})
it("should return no errors on valid row", async () => {
const rowUsage = await getRowUsage()
await expectRowUsage(0, async () => {
const res = await config.api.row.validate(table._id!, {
name: "ivan",
})
const res = await config.api.row.validate(table._id!, {
name: "ivan",
expect(res.valid).toBe(true)
expect(Object.keys(res.errors)).toEqual([])
})
expect(res.valid).toBe(true)
expect(Object.keys(res.errors)).toEqual([])
await assertRowUsage(rowUsage)
})
it("should errors on invalid row", async () => {
const rowUsage = await getRowUsage()
await expectRowUsage(0, async () => {
const res = await config.api.row.validate(table._id!, { name: 1 })
const res = await config.api.row.validate(table._id!, { name: 1 })
if (isInternal) {
expect(res.valid).toBe(false)
expect(Object.keys(res.errors)).toEqual(["name"])
} else {
// Validation for external is not implemented, so it will always return valid
expect(res.valid).toBe(true)
expect(Object.keys(res.errors)).toEqual([])
}
await assertRowUsage(rowUsage)
if (isInternal) {
expect(res.valid).toBe(false)
expect(Object.keys(res.errors)).toEqual(["name"])
} else {
// Validation for external is not implemented, so it will always return valid
expect(res.valid).toBe(true)
expect(Object.keys(res.errors)).toEqual([])
}
})
})
})
@ -1596,15 +1583,15 @@ if (descriptions.length) {
it("should be able to delete a bulk set of rows", async () => {
const row1 = await config.api.row.save(table._id!, {})
const row2 = await config.api.row.save(table._id!, {})
const rowUsage = await getRowUsage()
const res = await config.api.row.bulkDelete(table._id!, {
rows: [row1, row2],
await expectRowUsage(isInternal ? -2 : 0, async () => {
const res = await config.api.row.bulkDelete(table._id!, {
rows: [row1, row2],
})
expect(res.length).toEqual(2)
await config.api.row.get(table._id!, row1._id!, { status: 404 })
})
expect(res.length).toEqual(2)
await config.api.row.get(table._id!, row1._id!, { status: 404 })
await assertRowUsage(isInternal ? rowUsage - 2 : rowUsage)
})
it("should be able to delete a variety of row set types", async () => {
@ -1613,41 +1600,42 @@ if (descriptions.length) {
config.api.row.save(table._id!, {}),
config.api.row.save(table._id!, {}),
])
const rowUsage = await getRowUsage()
const res = await config.api.row.bulkDelete(table._id!, {
rows: [row1, row2._id!, { _id: row3._id }],
await expectRowUsage(isInternal ? -3 : 0, async () => {
const res = await config.api.row.bulkDelete(table._id!, {
rows: [row1, row2._id!, { _id: row3._id }],
})
expect(res.length).toEqual(3)
await config.api.row.get(table._id!, row1._id!, { status: 404 })
})
expect(res.length).toEqual(3)
await config.api.row.get(table._id!, row1._id!, { status: 404 })
await assertRowUsage(isInternal ? rowUsage - 3 : rowUsage)
})
it("should accept a valid row object and delete the row", async () => {
const row1 = await config.api.row.save(table._id!, {})
const rowUsage = await getRowUsage()
const res = await config.api.row.delete(table._id!, row1 as DeleteRow)
await expectRowUsage(isInternal ? -1 : 0, async () => {
const res = await config.api.row.delete(
table._id!,
row1 as DeleteRow
)
expect(res.id).toEqual(row1._id)
await config.api.row.get(table._id!, row1._id!, { status: 404 })
await assertRowUsage(isInternal ? rowUsage - 1 : rowUsage)
expect(res.id).toEqual(row1._id)
await config.api.row.get(table._id!, row1._id!, { status: 404 })
})
})
it.each([{ not: "valid" }, { rows: 123 }, "invalid"])(
"should ignore malformed/invalid delete request: %s",
async (request: any) => {
const rowUsage = await getRowUsage()
await config.api.row.delete(table._id!, request, {
status: 400,
body: {
message: "Invalid delete rows request",
},
await expectRowUsage(0, async () => {
await config.api.row.delete(table._id!, request, {
status: 400,
body: {
message: "Invalid delete rows request",
},
})
})
await assertRowUsage(rowUsage)
}
)
})
@ -1733,31 +1721,29 @@ if (descriptions.length) {
})
)
const rowUsage = await getRowUsage()
await expectRowUsage(isInternal ? 2 : 0, async () => {
await config.api.row.bulkImport(table._id!, {
rows: [
{
name: "Row 1",
description: "Row 1 description",
},
{
name: "Row 2",
description: "Row 2 description",
},
],
})
await config.api.row.bulkImport(table._id!, {
rows: [
{
name: "Row 1",
description: "Row 1 description",
},
{
name: "Row 2",
description: "Row 2 description",
},
],
const rows = await config.api.row.fetch(table._id!)
expect(rows.length).toEqual(2)
rows.sort((a, b) => a.name.localeCompare(b.name))
expect(rows[0].name).toEqual("Row 1")
expect(rows[0].description).toEqual("Row 1 description")
expect(rows[1].name).toEqual("Row 2")
expect(rows[1].description).toEqual("Row 2 description")
})
const rows = await config.api.row.fetch(table._id!)
expect(rows.length).toEqual(2)
rows.sort((a, b) => a.name.localeCompare(b.name))
expect(rows[0].name).toEqual("Row 1")
expect(rows[0].description).toEqual("Row 1 description")
expect(rows[1].name).toEqual("Row 2")
expect(rows[1].description).toEqual("Row 2 description")
await assertRowUsage(isInternal ? rowUsage + 2 : rowUsage)
})
isInternal &&
@ -1782,35 +1768,33 @@ if (descriptions.length) {
description: "Existing description",
})
const rowUsage = await getRowUsage()
await expectRowUsage(2, async () => {
await config.api.row.bulkImport(table._id!, {
rows: [
{
name: "Row 1",
description: "Row 1 description",
},
{ ...existingRow, name: "Updated existing row" },
{
name: "Row 2",
description: "Row 2 description",
},
],
identifierFields: ["_id"],
})
await config.api.row.bulkImport(table._id!, {
rows: [
{
name: "Row 1",
description: "Row 1 description",
},
{ ...existingRow, name: "Updated existing row" },
{
name: "Row 2",
description: "Row 2 description",
},
],
identifierFields: ["_id"],
const rows = await config.api.row.fetch(table._id!)
expect(rows.length).toEqual(3)
rows.sort((a, b) => a.name.localeCompare(b.name))
expect(rows[0].name).toEqual("Row 1")
expect(rows[0].description).toEqual("Row 1 description")
expect(rows[1].name).toEqual("Row 2")
expect(rows[1].description).toEqual("Row 2 description")
expect(rows[2].name).toEqual("Updated existing row")
expect(rows[2].description).toEqual("Existing description")
})
const rows = await config.api.row.fetch(table._id!)
expect(rows.length).toEqual(3)
rows.sort((a, b) => a.name.localeCompare(b.name))
expect(rows[0].name).toEqual("Row 1")
expect(rows[0].description).toEqual("Row 1 description")
expect(rows[1].name).toEqual("Row 2")
expect(rows[1].description).toEqual("Row 2 description")
expect(rows[2].name).toEqual("Updated existing row")
expect(rows[2].description).toEqual("Existing description")
await assertRowUsage(rowUsage + 2)
})
isInternal &&
@ -1835,36 +1819,34 @@ if (descriptions.length) {
description: "Existing description",
})
const rowUsage = await getRowUsage()
await expectRowUsage(3, async () => {
await config.api.row.bulkImport(table._id!, {
rows: [
{
name: "Row 1",
description: "Row 1 description",
},
{ ...existingRow, name: "Updated existing row" },
{
name: "Row 2",
description: "Row 2 description",
},
],
})
await config.api.row.bulkImport(table._id!, {
rows: [
{
name: "Row 1",
description: "Row 1 description",
},
{ ...existingRow, name: "Updated existing row" },
{
name: "Row 2",
description: "Row 2 description",
},
],
const rows = await config.api.row.fetch(table._id!)
expect(rows.length).toEqual(4)
rows.sort((a, b) => a.name.localeCompare(b.name))
expect(rows[0].name).toEqual("Existing row")
expect(rows[0].description).toEqual("Existing description")
expect(rows[1].name).toEqual("Row 1")
expect(rows[1].description).toEqual("Row 1 description")
expect(rows[2].name).toEqual("Row 2")
expect(rows[2].description).toEqual("Row 2 description")
expect(rows[3].name).toEqual("Updated existing row")
expect(rows[3].description).toEqual("Existing description")
})
const rows = await config.api.row.fetch(table._id!)
expect(rows.length).toEqual(4)
rows.sort((a, b) => a.name.localeCompare(b.name))
expect(rows[0].name).toEqual("Existing row")
expect(rows[0].description).toEqual("Existing description")
expect(rows[1].name).toEqual("Row 1")
expect(rows[1].description).toEqual("Row 1 description")
expect(rows[2].name).toEqual("Row 2")
expect(rows[2].description).toEqual("Row 2 description")
expect(rows[3].name).toEqual("Updated existing row")
expect(rows[3].description).toEqual("Existing description")
await assertRowUsage(rowUsage + 3)
})
// Upserting isn't yet supported in MSSQL / Oracle, see:
@ -2187,29 +2169,29 @@ if (descriptions.length) {
return { linkedTable, firstRow, secondRow }
}
)
const rowUsage = await getRowUsage()
// test basic enrichment
const resBasic = await config.api.row.get(
linkedTable._id!,
secondRow._id!
)
expect(resBasic.link.length).toBe(1)
expect(resBasic.link[0]).toEqual({
_id: firstRow._id,
primaryDisplay: firstRow.name,
await expectRowUsage(0, async () => {
// test basic enrichment
const resBasic = await config.api.row.get(
linkedTable._id!,
secondRow._id!
)
expect(resBasic.link.length).toBe(1)
expect(resBasic.link[0]).toEqual({
_id: firstRow._id,
primaryDisplay: firstRow.name,
})
// test full enrichment
const resEnriched = await config.api.row.getEnriched(
linkedTable._id!,
secondRow._id!
)
expect(resEnriched.link.length).toBe(1)
expect(resEnriched.link[0]._id).toBe(firstRow._id)
expect(resEnriched.link[0].name).toBe("Test Contact")
expect(resEnriched.link[0].description).toBe("original description")
})
// test full enrichment
const resEnriched = await config.api.row.getEnriched(
linkedTable._id!,
secondRow._id!
)
expect(resEnriched.link.length).toBe(1)
expect(resEnriched.link[0]._id).toBe(firstRow._id)
expect(resEnriched.link[0].name).toBe("Test Contact")
expect(resEnriched.link[0].description).toBe("original description")
await assertRowUsage(rowUsage)
})
})

42
packages/server/src/api/routes/tests/viewV2.spec.ts
View File

@ -2826,34 +2826,44 @@ if (descriptions.length) {
return total
}
const assertRowUsage = async (expected: number) => {
const usage = await getRowUsage()
async function expectRowUsage<T>(
expected: number,
f: () => Promise<T>
): Promise<T> {
const before = await getRowUsage()
const result = await f()
const after = await getRowUsage()
const usage = after - before
expect(usage).toBe(expected)
return result
}
it("should be able to delete a row", async () => {
const createdRow = await config.api.row.save(table._id!, {})
const rowUsage = await getRowUsage()
await config.api.row.bulkDelete(view.id, { rows: [createdRow] })
await assertRowUsage(isInternal ? rowUsage - 1 : rowUsage)
const createdRow = await expectRowUsage(isInternal ? 1 : 0, () =>
config.api.row.save(table._id!, {})
)
await expectRowUsage(isInternal ? -1 : 0, () =>
config.api.row.bulkDelete(view.id, { rows: [createdRow] })
)
await config.api.row.get(table._id!, createdRow._id!, {
status: 404,
})
})
it("should be able to delete multiple rows", async () => {
const rows = await Promise.all([
config.api.row.save(table._id!, {}),
config.api.row.save(table._id!, {}),
config.api.row.save(table._id!, {}),
])
const rowUsage = await getRowUsage()
await config.api.row.bulkDelete(view.id, {
rows: [rows[0], rows[2]],
const rows = await expectRowUsage(isInternal ? 3 : 0, async () => {
return [
await config.api.row.save(table._id!, {}),
await config.api.row.save(table._id!, {}),
await config.api.row.save(table._id!, {}),
]
})
await assertRowUsage(isInternal ? rowUsage - 2 : rowUsage)
await expectRowUsage(isInternal ? -2 : 0, async () => {
await config.api.row.bulkDelete(view.id, {
rows: [rows[0], rows[2]],
})
})
await config.api.row.get(table._id!, rows[0]._id!, {
status: 404,

27
packages/server/src/automations/tests/steps/loop.spec.ts
View File

@ -195,7 +195,34 @@ describe("Attempt to run a basic loop automation", () => {
.serverLog({ text: "{{steps.1.iterations}}" })
.test({ fields: {} })
expect(results.steps[0].outputs.status).toBe(
AutomationStepStatus.MAX_ITERATIONS
)
expect(results.steps[0].outputs.iterations).toBe(2)
expect(results.steps[0].outputs.items).toHaveLength(2)
expect(results.steps[0].outputs.items[0].message).toEndWith("test")
expect(results.steps[0].outputs.items[1].message).toEndWith("test2")
})
it("should stop when a failure condition is hit", async () => {
const results = await createAutomationBuilder(config)
.onAppAction()
.loop({
option: LoopStepType.ARRAY,
binding: ["test", "test2", "test3"],
failure: "test3",
})
.serverLog({ text: "{{loop.currentItem}}" })
.serverLog({ text: "{{steps.1.iterations}}" })
.test({ fields: {} })
expect(results.steps[0].outputs.status).toBe(
AutomationStepStatus.FAILURE_CONDITION
)
expect(results.steps[0].outputs.iterations).toBe(2)
expect(results.steps[0].outputs.items).toHaveLength(2)
expect(results.steps[0].outputs.items[0].message).toEndWith("test")
expect(results.steps[0].outputs.items[1].message).toEndWith("test2")
})
it("should run an automation with loop and max iterations to ensure context correctness further down the tree", async () => {

9
packages/server/src/definitions/automations.ts
View File

@ -20,9 +20,12 @@ export interface TriggerOutput {
export interface AutomationContext {
trigger: AutomationTriggerResultOutputs
steps: [AutomationTriggerResultOutputs, ...AutomationStepResultOutputs[]]
stepsById: Record<string, AutomationStepResultOutputs>
steps: Record<
string,
AutomationStepResultOutputs | AutomationTriggerResultOutputs
>
stepsByName: Record<string, AutomationStepResultOutputs>
stepsById: Record<string, AutomationStepResultOutputs>
env?: Record<string, string>
user?: UserBindings
settings?: {
@ -31,4 +34,6 @@ export interface AutomationContext {
company?: string
}
loop?: { currentItem: any }
_stepIndex: number
_error: boolean
}

2
packages/server/src/events/AutomationEmitter.ts
View File

@ -32,6 +32,8 @@ class AutomationEmitter implements ContextEmitter {
if (chainAutomations === true) {
return MAX_AUTOMATIONS_ALLOWED
} else if (env.isTest()) {
return 0
} else if (chainAutomations === undefined && env.SELF_HOSTED) {
return MAX_AUTOMATIONS_ALLOWED
} else {

2
packages/server/src/tests/jestSetup.ts
View File

@ -23,6 +23,6 @@ nock.enableNetConnect(host => {
testContainerUtils.setupEnv(env, coreEnv)
afterAll(() => {
afterAll(async () => {
timers.cleanup()
})

4
packages/server/src/tests/utilities/api/base.ts
View File

@ -146,8 +146,9 @@ export abstract class TestAPI {
}
}
let resp: Response | undefined = undefined
try {
return await req
resp = await req
} catch (e: any) {
// We've found that occasionally the connection between supertest and the
// server supertest starts gets reset. Not sure why, but retrying it
@ -161,6 +162,7 @@ export abstract class TestAPI {
}
throw e
}
return resp
}
protected async getHeaders(

70
packages/server/src/threads/automation.ts
View File

@ -143,7 +143,6 @@ async function branchMatches(
branch: Readonly<Branch>
): Promise<boolean> {
const toFilter: Record<string, any> = {}
const preparedCtx = prepareContext(ctx)
// Because we allow bindings on both the left and right of each condition in
// automation branches, we can't pass the BranchSearchFilters directly to
@ -160,9 +159,9 @@ async function branchMatches(
filter.conditions = filter.conditions.map(evaluateBindings)
} else {
for (const [field, value] of Object.entries(filter)) {
toFilter[field] = processStringSync(field, preparedCtx)
toFilter[field] = processStringSync(field, ctx)
if (typeof value === "string" && findHBSBlocks(value).length > 0) {
filter[field] = processStringSync(value, preparedCtx)
filter[field] = processStringSync(value, ctx)
}
}
}
@ -178,17 +177,6 @@ async function branchMatches(
return result.length > 0
}
function prepareContext(context: AutomationContext) {
return {
...context,
steps: {
...context.steps,
...context.stepsById,
...context.stepsByName,
},
}
}
async function enrichBaseContext(context: AutomationContext) {
context.env = await sdkUtils.getEnvironmentVariables()
@ -304,41 +292,37 @@ class Orchestrator {
}
hasErrored(context: AutomationContext): boolean {
const [_trigger, ...steps] = context.steps
for (const step of steps) {
if (step.success === false) {
return true
}
}
return false
return context._error === true
}
async execute(): Promise<AutomationResults> {
return await tracer.trace("execute", async span => {
span.addTags({ appId: this.appId, automationId: this.automation._id })
const job = cloneDeep(this.job)
delete job.data.event.appId
delete job.data.event.metadata
const data = cloneDeep(this.job.data)
delete data.event.appId
delete data.event.metadata
if (this.isCron() && !job.data.event.timestamp) {
job.data.event.timestamp = Date.now()
if (this.isCron() && !data.event.timestamp) {
data.event.timestamp = Date.now()
}
const trigger: AutomationTriggerResult = {
id: job.data.automation.definition.trigger.id,
stepId: job.data.automation.definition.trigger.stepId,
id: data.automation.definition.trigger.id,
stepId: data.automation.definition.trigger.stepId,
inputs: null,
outputs: job.data.event,
outputs: data.event,
}
const result: AutomationResults = { trigger, steps: [trigger] }
const ctx: AutomationContext = {
trigger: trigger.outputs,
steps: [trigger.outputs],
stepsById: {},
steps: { "0": trigger.outputs },
stepsByName: {},
stepsById: {},
user: trigger.outputs.user,
_error: false,
_stepIndex: 1,
}
await enrichBaseContext(ctx)
@ -348,7 +332,7 @@ class Orchestrator {
try {
await helpers.withTimeout(timeout, async () => {
const [stepOutputs, executionTime] = await utils.time(() =>
this.executeSteps(ctx, job.data.automation.definition.steps)
this.executeSteps(ctx, data.automation.definition.steps)
)
result.steps.push(...stepOutputs)
@ -400,9 +384,20 @@ class Orchestrator {
step: AutomationStep,
result: AutomationStepResult
) {
ctx.steps.push(result.outputs)
ctx.steps[step.id] = result.outputs
ctx.steps[step.name || step.id] = result.outputs
ctx.stepsById[step.id] = result.outputs
ctx.stepsByName[step.name || step.id] = result.outputs
ctx._stepIndex ||= 0
ctx.steps[ctx._stepIndex] = result.outputs
ctx._stepIndex++
if (result.outputs.success === false) {
ctx._error = true
}
results.push(result)
}
@ -449,7 +444,7 @@ class Orchestrator {
stepToLoop: AutomationStep
): Promise<AutomationStepResult> {
return await tracer.trace("executeLoopStep", async span => {
await processObject(step.inputs, prepareContext(ctx))
await processObject(step.inputs, ctx)
const maxIterations = getLoopMaxIterations(step)
const items: Record<string, any>[] = []
@ -478,6 +473,7 @@ class Orchestrator {
return stepFailure(stepToLoop, {
status: AutomationStepStatus.MAX_ITERATIONS,
iterations,
items,
})
}
@ -488,6 +484,8 @@ class Orchestrator {
})
return stepFailure(stepToLoop, {
status: AutomationStepStatus.FAILURE_CONDITION,
iterations,
items,
})
}
@ -558,7 +556,7 @@ class Orchestrator {
}
const inputs = automationUtils.cleanInputValues(
await processObject(cloneDeep(step.inputs), prepareContext(ctx)),
await processObject(cloneDeep(step.inputs), ctx),
step.schema.inputs.properties
)
@ -566,7 +564,7 @@ class Orchestrator {
inputs,
appId: this.appId,
emitter: this.emitter,
context: prepareContext(ctx),
context: ctx,
})
if (

3
packages/shared-core/src/utils.ts
View File

@ -117,7 +117,8 @@ export function isSupportedUserSearch(
{ op: BasicOperator.EQUAL, key: "_id" },
{ op: ArrayOperator.ONE_OF, key: "_id" },
]
for (const [key, operation] of Object.entries(query)) {
const { allOr, onEmptyFilter, ...filters } = query
for (const [key, operation] of Object.entries(filters)) {
if (typeof operation !== "object") {
return false
}

1
packages/string-templates/src/helpers/constants.ts
View File

@ -36,6 +36,7 @@ export const HelperFunctionNames = {
ALL: "all",
LITERAL: "literal",
JS: "js",
DECODE_ID: "decodeId",
}
export const LITERAL_MARKER = "%LITERAL%"

18
packages/string-templates/src/helpers/index.ts
View File

@ -25,13 +25,29 @@ function isObject(value: string | any[]) {
)
}
const HELPERS = [
export const HELPERS = [
// external helpers
new Helper(HelperFunctionNames.OBJECT, (value: any) => {
return new Handlebars.SafeString(JSON.stringify(value))
}),
// javascript helper
new Helper(HelperFunctionNames.JS, processJS, false),
new Helper(HelperFunctionNames.DECODE_ID, (_id: string | { _id: string }) => {
if (!_id) {
return []
}
// have to replace on the way back as we swapped out the double quotes
// when encoding, but JSON can't handle the single quotes
const id = typeof _id === "string" ? _id : _id._id
const decoded: string = decodeURIComponent(id).replace(/'/g, '"')
try {
const parsed = JSON.parse(decoded)
return Array.isArray(parsed) ? parsed : [parsed]
} catch (err) {
// wasn't json - likely was handlebars for a many to many
return [_id]
}
}),
// this help is applied to all statements
new Helper(
HelperFunctionNames.ALL,

41
packages/string-templates/test/helpers.spec.ts
View File

@ -517,3 +517,44 @@ describe("helper overlap", () => {
expect(output).toEqual("a")
})
})
describe("Test the decodeId helper", () => {
it("should decode a valid encoded ID", async () => {
const encodedId = encodeURIComponent("[42]") // "%5B42%5D"
const output = await processString("{{ decodeId id }}", { id: encodedId })
expect(output).toBe("42")
})
it("Should return an unchanged string if the string isn't encoded", async () => {
const unencodedId = "forty-two"
const output = await processString("{{ decodeId id }}", { id: unencodedId })
expect(output).toBe("forty-two")
})
it("Should return a string of comma-separated IDs when passed multiple IDs in a URI encoded array", async () => {
const encodedIds = encodeURIComponent("[1,2,3]") // "%5B1%2C2%2C3%5D"
const output = await processString("{{ decodeId id }}", { id: encodedIds })
expect(output).toBe("1,2,3")
})
it("Handles empty array gracefully", async () => {
const output = await processString("{{ decodeId value }}", {
value: [],
})
expect(output).toBe("[[]]")
})
it("Handles undefined gracefully", async () => {
const output = await processString("{{ decodeId value }}", {
value: undefined,
})
expect(output).toBe("")
})
it("Handles null gracefully", async () => {
const output = await processString("{{ decodeId value }}", {
value: undefined,
})
expect(output).toBe("")
})
})