Merge branch 'master' into more-db-typing-improvements

This commit is contained in:
Michael Drury 2023-11-08 18:50:39 +00:00 committed by GitHub
commit c511f621d3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
22 changed files with 243 additions and 146 deletions

View File

@ -188,4 +188,17 @@ describe("utils", () => {
expectResult(false)
})
})
describe("hasCircularStructure", () => {
it("should detect a circular structure", () => {
const a: any = { b: "b" }
const b = { a }
a.b = b
expect(utils.hasCircularStructure(b)).toBe(true)
})
it("should allow none circular structures", () => {
expect(utils.hasCircularStructure({ a: "b" })).toBe(false)
})
})
})

View File

@ -237,3 +237,17 @@ export function timeout(timeMs: number) {
export function isAudited(event: Event) {
return !!AuditedEventFriendlyName[event]
}
export function hasCircularStructure(json: any) {
if (typeof json !== "object") {
return false
}
try {
JSON.stringify(json)
} catch (err) {
if (err instanceof Error && err?.message.includes("circular structure")) {
return true
}
}
return false
}

View File

@ -48,15 +48,14 @@
<UndoRedoControl store={automationHistoryStore} />
</div>
<div class="controls">
<div class="buttons">
<div
on:click={() => {
testDataModal.show()
}}
class="buttons"
>
<Icon hoverable size="M" name="Play" />
<div
on:click={() => {
testDataModal.show()
}}
>
Run test
</div>
<div>Run test</div>
</div>
<div class="buttons">
<Icon

View File

@ -27,7 +27,7 @@
$: if (value?.queryId == null) value = { queryId: "" }
</script>
<div class="schema-fields">
<div class="schema-field">
<Label>Query</Label>
<div class="field-width">
<Select
@ -41,8 +41,8 @@
</div>
{#if parameters.length}
<div class="schema-fields">
{#each parameters as field}
{#each parameters as field}
<div class="schema-field">
<Label>{field.name}</Label>
<div class="field-width">
<DrawerBindableInput
@ -56,8 +56,8 @@
updateOnChange={false}
/>
</div>
{/each}
</div>
</div>
{/each}
{/if}
<style>
@ -65,7 +65,7 @@
width: 320px;
}
.schema-fields {
.schema-field {
display: flex;
justify-content: space-between;
align-items: center;
@ -76,7 +76,7 @@
margin-bottom: 10px;
}
.schema-fields :global(label) {
.schema-field :global(label) {
text-transform: capitalize;
}
</style>

View File

@ -114,10 +114,10 @@
</div>
{#if schemaFields.length}
{#each schemaFields as [field, schema]}
<div class="schema-fields">
<Label>{field}</Label>
<div class="field-width">
{#if !schema.autocolumn && schema.type !== "attachment"}
{#if !schema.autocolumn && schema.type !== "attachment"}
<div class="schema-fields">
<Label>{field}</Label>
<div class="field-width">
{#if isTestModal}
<RowSelectorTypes
{isTestModal}
@ -151,20 +151,20 @@
/>
</DrawerBindableSlot>
{/if}
{/if}
{#if isUpdateRow && schema.type === "link"}
<div class="checkbox-field">
<Checkbox
value={meta.fields?.[field]?.clearRelationships}
text={"Clear relationships if empty?"}
size={"S"}
on:change={e => onChangeSetting(e, field)}
/>
</div>
{/if}
{#if isUpdateRow && schema.type === "link"}
<div class="checkbox-field">
<Checkbox
value={meta.fields?.[field]?.clearRelationships}
text={"Clear relationships if empty?"}
size={"S"}
on:change={e => onChangeSetting(e, field)}
/>
</div>
{/if}
</div>
</div>
</div>
{/if}
{/each}
{/if}

View File

@ -67,6 +67,7 @@
bind:linkedRows={value[field]}
{schema}
on:change={e => onChange(e, field)}
useLabel={false}
/>
{:else if schema.type === "string" || schema.type === "number"}
<svelte:component

View File

@ -7,7 +7,7 @@
export let schema
export let linkedRows = []
export let useLabel = true
const dispatch = createEventDispatcher()
let rows = []
@ -51,7 +51,7 @@
linkedIds = e.detail ? [e.detail] : []
dispatch("change", linkedIds)
}}
{label}
label={useLabel ? label : null}
sort
/>
{:else}

View File

@ -106,7 +106,6 @@ export async function fetchDeployments(ctx: any) {
}
ctx.body = Object.values(deployments.history).reverse()
} catch (err) {
console.error(err)
ctx.body = []
}
}

View File

@ -23,7 +23,6 @@ import {
breakRowIdField,
convertRowId,
generateRowIdField,
getPrimaryDisplay,
isRowId,
isSQL,
} from "../../../integrations/utils"
@ -237,7 +236,7 @@ function basicProcessing({
thisRow._id = generateIdForRow(row, table, isLinked)
thisRow.tableId = table._id
thisRow._rev = "rev"
return processFormulas(table, thisRow)
return thisRow
}
function fixArrayTypes(row: Row, table: Table) {
@ -392,7 +391,7 @@ export class ExternalRequest<T extends Operation> {
return { row: newRow, manyRelationships }
}
squashRelationshipColumns(
processRelationshipFields(
table: Table,
row: Row,
relationships: RelationshipsJson[]
@ -402,7 +401,6 @@ export class ExternalRequest<T extends Operation> {
if (!linkedTable || !row[relationship.column]) {
continue
}
const display = linkedTable.primaryDisplay
for (let key of Object.keys(row[relationship.column])) {
let relatedRow: Row = row[relationship.column][key]
// add this row as context for the relationship
@ -411,15 +409,10 @@ export class ExternalRequest<T extends Operation> {
relatedRow[col.name] = [row]
}
}
// process additional types
relatedRow = processDates(table, relatedRow)
relatedRow = processFormulas(linkedTable, relatedRow)
let relatedDisplay
if (display) {
relatedDisplay = getPrimaryDisplay(relatedRow[display])
}
row[relationship.column][key] = {
primaryDisplay: relatedDisplay || "Invalid display column",
_id: relatedRow._id,
}
row[relationship.column][key] = relatedRow
}
}
return row
@ -521,14 +514,14 @@ export class ExternalRequest<T extends Operation> {
)
}
// Process some additional data types
let finalRowArray = Object.values(finalRows)
finalRowArray = processDates(table, finalRowArray)
finalRowArray = processFormulas(table, finalRowArray) as Row[]
return finalRowArray.map((row: Row) =>
this.squashRelationshipColumns(table, row, relationships)
// make sure all related rows are correct
let finalRowArray = Object.values(finalRows).map(row =>
this.processRelationshipFields(table, row, relationships)
)
// process some additional types
finalRowArray = processDates(table, finalRowArray)
return finalRowArray
}
/**
@ -663,7 +656,7 @@ export class ExternalRequest<T extends Operation> {
linkPrimary,
linkSecondary,
}: {
row: { [key: string]: any }
row: Row
linkPrimary: string
linkSecondary?: string
}) {

View File

@ -76,6 +76,7 @@ export async function patch(ctx: UserCtx<PatchRowRequest, PatchRowResponse>) {
relationships: true,
})
const enrichedRow = await outputProcessing(table, row, {
squash: true,
preserveLinks: true,
})
return {
@ -119,7 +120,10 @@ export async function save(ctx: UserCtx) {
})
return {
...response,
row: await outputProcessing(table, row, { preserveLinks: true }),
row: await outputProcessing(table, row, {
preserveLinks: true,
squash: true,
}),
}
} else {
return response
@ -140,7 +144,7 @@ export async function find(ctx: UserCtx): Promise<Row> {
const table = await sdk.tables.getTable(tableId)
// Preserving links, as the outputProcessing does not support external rows yet and we don't need it in this use case
return await outputProcessing(table, row, {
squash: false,
squash: true,
preserveLinks: true,
})
}
@ -207,7 +211,7 @@ export async function fetchEnrichedRow(ctx: UserCtx) {
// don't support composite keys right now
const linkedIds = links.map((link: Row) => breakRowIdField(link._id!)[0])
const primaryLink = linkedTable.primary?.[0] as string
row[fieldName] = await handleRequest(Operation.READ, linkedTableId!, {
const relatedRows = await handleRequest(Operation.READ, linkedTableId!, {
tables,
filters: {
oneOf: {
@ -216,6 +220,10 @@ export async function fetchEnrichedRow(ctx: UserCtx) {
},
includeSqlRelationships: IncludeRelationship.INCLUDE,
})
row[fieldName] = await outputProcessing(linkedTable, relatedRows, {
squash: true,
preserveLinks: true,
})
}
return row
}

View File

@ -101,12 +101,12 @@ export async function updateAllFormulasInTable(table: Table) {
for (let row of rows) {
// find the enriched row, if found process the formulas
const enrichedRow = enrichedRows.find(
(enriched: any) => enriched._id === row._id
(enriched: Row) => enriched._id === row._id
)
if (enrichedRow) {
const processed = processFormulas(table, cloneDeep(row), {
dynamic: false,
contextRows: enrichedRow,
contextRows: [enrichedRow],
})
// values have changed, need to add to bulk docs to update
if (!isEqual(processed, row)) {
@ -139,7 +139,7 @@ export async function finaliseRow(
// use enriched row to generate formulas for saving, specifically only use as context
row = processFormulas(table, row, {
dynamic: false,
contextRows: enrichedRow,
contextRows: [enrichedRow],
})
// don't worry about rev, tables handle rev/lastID updates
// if another row has been written since processing this will
@ -163,7 +163,9 @@ export async function finaliseRow(
const response = await db.put(row)
// for response, calculate the formulas for the enriched row
enrichedRow._rev = response.rev
enrichedRow = await processFormulas(table, enrichedRow, { dynamic: false })
enrichedRow = processFormulas(table, enrichedRow, {
dynamic: false,
})
// this updates the related formulas in other rows based on the relations to this row
if (updateFormula) {
await updateRelatedFormula(table, enrichedRow)

View File

@ -27,51 +27,59 @@ interface KoaRateLimitOptions {
}
const PREFIX = "/api/public/v1"
// allow a lot more requests when in test
const DEFAULT_API_REQ_LIMIT_PER_SEC = env.isTest() ? 100 : 10
function getApiLimitPerSecond(): number {
if (!env.API_REQ_LIMIT_PER_SEC) {
return DEFAULT_API_REQ_LIMIT_PER_SEC
}
return parseInt(env.API_REQ_LIMIT_PER_SEC)
}
// type can't be known - untyped libraries
let limiter: any, rateLimitStore: any
if (!env.DISABLE_RATE_LIMITING) {
// allow a lot more requests when in test
const DEFAULT_API_REQ_LIMIT_PER_SEC = env.isTest() ? 100 : 10
let rateLimitStore: any = null
if (!env.isTest()) {
const { password, host, port } = redis.utils.getRedisConnectionDetails()
let options: KoaRateLimitOptions = {
socket: {
host: host,
port: port,
},
function getApiLimitPerSecond(): number {
if (!env.API_REQ_LIMIT_PER_SEC) {
return DEFAULT_API_REQ_LIMIT_PER_SEC
}
return parseInt(env.API_REQ_LIMIT_PER_SEC)
}
if (password) {
options.password = password
}
if (!env.isTest()) {
const { password, host, port } = redis.utils.getRedisConnectionDetails()
let options: KoaRateLimitOptions = {
socket: {
host: host,
port: port,
},
}
if (!env.REDIS_CLUSTERED) {
// Can't set direct redis db in clustered env
options.database = SelectableDatabase.RATE_LIMITING
if (password) {
options.password = password
}
if (!env.REDIS_CLUSTERED) {
// Can't set direct redis db in clustered env
options.database = SelectableDatabase.RATE_LIMITING
}
rateLimitStore = new Stores.Redis(options)
RateLimit.defaultOptions({
store: rateLimitStore,
})
}
rateLimitStore = new Stores.Redis(options)
RateLimit.defaultOptions({
store: rateLimitStore,
// rate limiting, allows for 2 requests per second
limiter = RateLimit.middleware({
interval: { sec: 1 },
// per ip, per interval
max: getApiLimitPerSecond(),
})
} else {
console.log("**** PUBLIC API RATE LIMITING DISABLED ****")
}
// rate limiting, allows for 2 requests per second
const limiter = RateLimit.middleware({
interval: { sec: 1 },
// per ip, per interval
max: getApiLimitPerSecond(),
})
const publicRouter = new Router({
prefix: PREFIX,
})
publicRouter.use(limiter)
if (limiter) {
publicRouter.use(limiter)
}
function addMiddleware(
endpoints: any,

View File

@ -10,6 +10,7 @@ import {
FieldSchema,
FieldType,
FieldTypeSubtypes,
FormulaTypes,
INTERNAL_TABLE_SOURCE_ID,
MonthlyQuotaName,
PermissionLevel,
@ -2000,4 +2001,52 @@ describe.each([
})
})
})
describe("Formula fields", () => {
let relationshipTable: Table, tableId: string, relatedRow: Row
beforeAll(async () => {
const otherTableId = config.table!._id!
const cfg = generateTableConfig()
relationshipTable = await config.createLinkedTable(
RelationshipType.ONE_TO_MANY,
["links"],
{
...cfg,
// needs to be a short name
name: "b",
schema: {
...cfg.schema,
formula: {
name: "formula",
type: FieldType.FORMULA,
formula: "{{ links.0.name }}",
formulaType: FormulaTypes.DYNAMIC,
},
},
}
)
tableId = relationshipTable._id!
relatedRow = await config.api.row.save(otherTableId, {
name: generator.word(),
description: generator.paragraph(),
})
await config.api.row.save(tableId, {
name: generator.word(),
description: generator.paragraph(),
tableId,
links: [relatedRow._id],
})
})
it("should be able to search for rows containing formulas", async () => {
const { rows } = await config.api.row.search(tableId)
expect(rows.length).toBe(1)
expect(rows[0].links.length).toBe(1)
const row = rows[0]
expect(row.formula).toBe(relatedRow.name)
})
})
})

View File

@ -2,7 +2,6 @@ import LinkController from "./LinkController"
import {
IncludeDocs,
getLinkDocuments,
createLinkView,
getUniqueByProp,
getRelatedTableForField,
getLinkedTableIDs,

View File

@ -8,6 +8,7 @@ import {
LinkDocumentValue,
Table,
} from "@budibase/types"
import sdk from "../../sdk"
export { createLinkView } from "../views/staticViews"
@ -110,12 +111,11 @@ export function getLinkedTableIDs(table: Table): string[] {
}
export async function getLinkedTable(id: string, tables: Table[]) {
const db = context.getAppDB()
let linkedTable = tables.find(table => table._id === id)
if (linkedTable) {
return linkedTable
}
linkedTable = await db.get(id)
linkedTable = await sdk.tables.getTable(id)
if (linkedTable) {
tables.push(linkedTable)
}

View File

@ -61,6 +61,7 @@ const environment = {
ALLOW_DEV_AUTOMATIONS: process.env.ALLOW_DEV_AUTOMATIONS,
DISABLE_THREADING: process.env.DISABLE_THREADING,
DISABLE_AUTOMATION_LOGS: process.env.DISABLE_AUTOMATION_LOGS,
DISABLE_RATE_LIMITING: process.env.DISABLE_RATE_LIMITING,
MULTI_TENANCY: process.env.MULTI_TENANCY,
ENABLE_ANALYTICS: process.env.ENABLE_ANALYTICS,
SELF_HOSTED: process.env.SELF_HOSTED,

View File

@ -923,7 +923,6 @@ describe("postgres integrations", () => {
[m2mFieldName]: [
{
_id: row._id,
primaryDisplay: "Invalid display column",
},
],
})
@ -932,7 +931,6 @@ describe("postgres integrations", () => {
[m2mFieldName]: [
{
_id: row._id,
primaryDisplay: "Invalid display column",
},
],
})

View File

@ -80,7 +80,10 @@ export async function search(options: SearchParams) {
rows = rows.map((r: any) => pick(r, fields))
}
rows = await outputProcessing(table, rows, { preserveLinks: true })
rows = await outputProcessing(table, rows, {
preserveLinks: true,
squash: true,
})
// need wrapper object for bookmarks etc when paginating
return { rows, hasNextPage, bookmark: bookmark && bookmark + 1 }
@ -185,6 +188,7 @@ export async function fetch(tableId: string): Promise<Row[]> {
const table = await sdk.tables.getTable(tableId)
return await outputProcessing<Row[]>(table, response, {
preserveLinks: true,
squash: true,
})
}

View File

@ -6,6 +6,7 @@ import {
ExportRowsRequest,
BulkImportRequest,
BulkImportResponse,
SearchRowResponse,
} from "@budibase/types"
import TestConfiguration from "../TestConfiguration"
import { TestAPI } from "./base"
@ -154,7 +155,7 @@ export class RowAPI extends TestAPI {
search = async (
sourceId: string,
{ expectStatus } = { expectStatus: 200 }
): Promise<Row[]> => {
): Promise<SearchRowResponse> => {
const request = this.request
.post(`/api/${sourceId}/search`)
.set(this.config.defaultHeaders())

View File

@ -2,16 +2,15 @@ import * as linkRows from "../../db/linkedRows"
import { FieldTypes, AutoFieldSubTypes } from "../../constants"
import { processFormulas, fixAutoColumnSubType } from "./utils"
import { ObjectStoreBuckets } from "../../constants"
import { context, db as dbCore, objectStore } from "@budibase/backend-core"
import {
context,
db as dbCore,
objectStore,
utils,
} from "@budibase/backend-core"
import { InternalTables } from "../../db/utils"
import { TYPE_TRANSFORM_MAP } from "./map"
import {
AutoColumnFieldMetadata,
FieldSubtype,
Row,
RowAttachment,
Table,
} from "@budibase/types"
import { FieldSubtype, Row, RowAttachment, Table } from "@budibase/types"
import { cloneDeep } from "lodash/fp"
import {
processInputBBReferences,
@ -233,6 +232,11 @@ export async function outputProcessing<T extends Row[] | Row>(
})
: safeRows
// make sure squash is enabled if needed
if (!opts.squash && utils.hasCircularStructure(rows)) {
opts.squash = true
}
// process complex types: attachements, bb references...
for (let [property, column] of Object.entries(table.schema)) {
if (column.type === FieldTypes.ATTACHMENT) {
@ -258,7 +262,7 @@ export async function outputProcessing<T extends Row[] | Row>(
}
// process formulas after the complex types had been processed
enriched = processFormulas(table, enriched, { dynamic: true }) as Row[]
enriched = processFormulas(table, enriched, { dynamic: true })
if (opts.squash) {
enriched = (await linkRows.squashLinksToPrimaryDisplay(

View File

@ -12,6 +12,11 @@ import {
Table,
} from "@budibase/types"
interface FormulaOpts {
dynamic?: boolean
contextRows?: Row[]
}
/**
* If the subtype has been lost for any reason this works out what
* subtype the auto column should be.
@ -40,52 +45,50 @@ export function fixAutoColumnSubType(
/**
* Looks through the rows provided and finds formulas - which it then processes.
*/
export function processFormulas(
export function processFormulas<T extends Row | Row[]>(
table: Table,
rows: Row[] | Row,
{ dynamic, contextRows }: any = { dynamic: true }
) {
const single = !Array.isArray(rows)
let rowArray: Row[]
if (single) {
rowArray = [rows]
contextRows = contextRows ? [contextRows] : contextRows
} else {
rowArray = rows
}
for (let [column, schema] of Object.entries(table.schema)) {
if (schema.type !== FieldTypes.FORMULA) {
continue
}
inputRows: T,
{ dynamic, contextRows }: FormulaOpts = { dynamic: true }
): T {
const rows = Array.isArray(inputRows) ? inputRows : [inputRows]
if (rows)
for (let [column, schema] of Object.entries(table.schema)) {
if (schema.type !== FieldTypes.FORMULA) {
continue
}
const isStatic = schema.formulaType === FormulaTypes.STATIC
const isStatic = schema.formulaType === FormulaTypes.STATIC
if (
schema.formula == null ||
(dynamic && isStatic) ||
(!dynamic && !isStatic)
) {
continue
}
// iterate through rows and process formula
for (let i = 0; i < rowArray.length; i++) {
let row = rowArray[i]
let context = contextRows ? contextRows[i] : row
rowArray[i] = {
...row,
[column]: processStringSync(schema.formula, context),
if (
schema.formula == null ||
(dynamic && isStatic) ||
(!dynamic && !isStatic)
) {
continue
}
// iterate through rows and process formula
for (let i = 0; i < rows.length; i++) {
let row = rows[i]
let context = contextRows ? contextRows[i] : row
rows[i] = {
...row,
[column]: processStringSync(schema.formula, context),
}
}
}
}
return single ? rowArray[0] : rowArray
return Array.isArray(inputRows) ? rows : rows[0]
}
/**
* Processes any date columns and ensures that those without the ignoreTimezones
* flag set are parsed as UTC rather than local time.
*/
export function processDates(table: Table, rows: Row[]) {
let datesWithTZ = []
export function processDates<T extends Row | Row[]>(
table: Table,
inputRows: T
): T {
let rows = Array.isArray(inputRows) ? inputRows : [inputRows]
let datesWithTZ: string[] = []
for (let [column, schema] of Object.entries(table.schema)) {
if (schema.type !== FieldTypes.DATETIME) {
continue
@ -102,5 +105,6 @@ export function processDates(table: Table, rows: Row[]) {
}
}
}
return rows
return Array.isArray(inputRows) ? rows : rows[0]
}

View File

@ -20,7 +20,7 @@
"test:self:ci": "yarn run test --testPathIgnorePatterns=\\.integration\\. \\.cloud\\. \\.licensing\\.",
"serve:test:self:ci": "start-server-and-test dev:built http://localhost:4001/health test:self:ci",
"serve": "start-server-and-test dev:built http://localhost:4001/health",
"dev:built": "cd ../ && yarn dev:built"
"dev:built": "cd ../ && DISABLE_RATE_LIMITING=1 yarn dev:built"
},
"devDependencies": {
"@budibase/types": "^2.3.17",