Merge branch 'master' into more-db-typing-improvements

packages/backend-core/src/utils/tests/utils.spec.ts

@@ -188,4 +188,17 @@ describe("utils", () => {
       expectResult(false)
     })
   })
+
+  describe("hasCircularStructure", () => {
+    it("should detect a circular structure", () => {
+      const a: any = { b: "b" }
+      const b = { a }
+      a.b = b
+      expect(utils.hasCircularStructure(b)).toBe(true)
+    })
+
+    it("should allow none circular structures", () => {
+      expect(utils.hasCircularStructure({ a: "b" })).toBe(false)
+    })
+  })
 })

packages/backend-core/src/utils/utils.ts

@@ -237,3 +237,17 @@ export function timeout(timeMs: number) {
 export function isAudited(event: Event) {
   return !!AuditedEventFriendlyName[event]
 }
+
+export function hasCircularStructure(json: any) {
+  if (typeof json !== "object") {
+    return false
+  }
+  try {
+    JSON.stringify(json)
+  } catch (err) {
+    if (err instanceof Error && err?.message.includes("circular structure")) {
+      return true
+    }
+  }
+  return false
+}

packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowChart.svelte

@@ -48,15 +48,14 @@
     <UndoRedoControl store={automationHistoryStore} />
   </div>
   <div class="controls">
-    <div class="buttons">
-      <Icon hoverable size="M" name="Play" />
     <div
       on:click={() => {
         testDataModal.show()
       }}
+      class="buttons"
     >
-        Run test
-      </div>
+      <Icon hoverable size="M" name="Play" />
+      <div>Run test</div>
     </div>
     <div class="buttons">
       <Icon

packages/builder/src/components/automation/SetupPanel/QueryParamSelector.svelte

@@ -27,7 +27,7 @@
   $: if (value?.queryId == null) value = { queryId: "" }
 </script>
 
-<div class="schema-fields">
+<div class="schema-field">
   <Label>Query</Label>
   <div class="field-width">
     <Select
@@ -41,8 +41,8 @@
 </div>
 
 {#if parameters.length}
-  <div class="schema-fields">
   {#each parameters as field}
+    <div class="schema-field">
       <Label>{field.name}</Label>
       <div class="field-width">
         <DrawerBindableInput
@@ -56,8 +56,8 @@
           updateOnChange={false}
         />
       </div>
-    {/each}
     </div>
+  {/each}
 {/if}
 
 <style>
@@ -65,7 +65,7 @@
     width: 320px;
   }
 
-  .schema-fields {
+  .schema-field {
     display: flex;
     justify-content: space-between;
     align-items: center;
@@ -76,7 +76,7 @@
     margin-bottom: 10px;
   }
 
-  .schema-fields :global(label) {
+  .schema-field :global(label) {
     text-transform: capitalize;
   }
 </style>

packages/builder/src/components/automation/SetupPanel/RowSelector.svelte

@@ -114,10 +114,10 @@
 </div>
 {#if schemaFields.length}
   {#each schemaFields as [field, schema]}
+    {#if !schema.autocolumn && schema.type !== "attachment"}
       <div class="schema-fields">
         <Label>{field}</Label>
         <div class="field-width">
-        {#if !schema.autocolumn && schema.type !== "attachment"}
           {#if isTestModal}
             <RowSelectorTypes
               {isTestModal}
@@ -151,7 +151,6 @@
               />
             </DrawerBindableSlot>
           {/if}
-        {/if}
 
           {#if isUpdateRow && schema.type === "link"}
             <div class="checkbox-field">
@@ -165,6 +164,7 @@
           {/if}
         </div>
       </div>
+    {/if}
   {/each}
 {/if}
 

packages/builder/src/components/automation/SetupPanel/RowSelectorTypes.svelte

@@ -67,6 +67,7 @@
     bind:linkedRows={value[field]}
     {schema}
     on:change={e => onChange(e, field)}
+    useLabel={false}
   />
 {:else if schema.type === "string" || schema.type === "number"}
   <svelte:component

packages/builder/src/components/common/LinkedRowSelector.svelte

@@ -7,7 +7,7 @@
 
   export let schema
   export let linkedRows = []
-
+  export let useLabel = true
   const dispatch = createEventDispatcher()
 
   let rows = []
@@ -51,7 +51,7 @@
       linkedIds = e.detail ? [e.detail] : []
       dispatch("change", linkedIds)
     }}
-    {label}
+    label={useLabel ? label : null}
     sort
   />
 {:else}

packages/server/src/api/controllers/deploy/index.ts

@@ -106,7 +106,6 @@ export async function fetchDeployments(ctx: any) {
     }
     ctx.body = Object.values(deployments.history).reverse()
   } catch (err) {
-    console.error(err)
     ctx.body = []
   }
 }

packages/server/src/api/controllers/row/ExternalRequest.ts

@@ -23,7 +23,6 @@ import {
   breakRowIdField,
   convertRowId,
   generateRowIdField,
-  getPrimaryDisplay,
   isRowId,
   isSQL,
 } from "../../../integrations/utils"
@@ -237,7 +236,7 @@ function basicProcessing({
   thisRow._id = generateIdForRow(row, table, isLinked)
   thisRow.tableId = table._id
   thisRow._rev = "rev"
-  return processFormulas(table, thisRow)
+  return thisRow
 }
 
 function fixArrayTypes(row: Row, table: Table) {
@@ -392,7 +391,7 @@ export class ExternalRequest<T extends Operation> {
     return { row: newRow, manyRelationships }
   }
 
-  squashRelationshipColumns(
+  processRelationshipFields(
     table: Table,
     row: Row,
     relationships: RelationshipsJson[]
@@ -402,7 +401,6 @@ export class ExternalRequest<T extends Operation> {
       if (!linkedTable || !row[relationship.column]) {
         continue
       }
-      const display = linkedTable.primaryDisplay
       for (let key of Object.keys(row[relationship.column])) {
         let relatedRow: Row = row[relationship.column][key]
         // add this row as context for the relationship
@@ -411,15 +409,10 @@ export class ExternalRequest<T extends Operation> {
             relatedRow[col.name] = [row]
           }
         }
+        // process additional types
+        relatedRow = processDates(table, relatedRow)
         relatedRow = processFormulas(linkedTable, relatedRow)
-        let relatedDisplay
-        if (display) {
-          relatedDisplay = getPrimaryDisplay(relatedRow[display])
-        }
-        row[relationship.column][key] = {
-          primaryDisplay: relatedDisplay || "Invalid display column",
-          _id: relatedRow._id,
-        }
+        row[relationship.column][key] = relatedRow
       }
     }
     return row
@@ -521,14 +514,14 @@ export class ExternalRequest<T extends Operation> {
       )
     }
 
-    // Process some additional data types
-    let finalRowArray = Object.values(finalRows)
-    finalRowArray = processDates(table, finalRowArray)
-    finalRowArray = processFormulas(table, finalRowArray) as Row[]
-
-    return finalRowArray.map((row: Row) =>
-      this.squashRelationshipColumns(table, row, relationships)
+    // make sure all related rows are correct
+    let finalRowArray = Object.values(finalRows).map(row =>
+      this.processRelationshipFields(table, row, relationships)
     )
+
+    // process some additional types
+    finalRowArray = processDates(table, finalRowArray)
+    return finalRowArray
   }
 
   /**
@@ -663,7 +656,7 @@ export class ExternalRequest<T extends Operation> {
         linkPrimary,
         linkSecondary,
       }: {
-        row: { [key: string]: any }
+        row: Row
         linkPrimary: string
         linkSecondary?: string
       }) {

packages/server/src/api/controllers/row/external.ts

@@ -76,6 +76,7 @@ export async function patch(ctx: UserCtx<PatchRowRequest, PatchRowResponse>) {
     relationships: true,
   })
   const enrichedRow = await outputProcessing(table, row, {
+    squash: true,
     preserveLinks: true,
   })
   return {
@@ -119,7 +120,10 @@ export async function save(ctx: UserCtx) {
     })
     return {
       ...response,
-      row: await outputProcessing(table, row, { preserveLinks: true }),
+      row: await outputProcessing(table, row, {
+        preserveLinks: true,
+        squash: true,
+      }),
     }
   } else {
     return response
@@ -140,7 +144,7 @@ export async function find(ctx: UserCtx): Promise<Row> {
   const table = await sdk.tables.getTable(tableId)
   // Preserving links, as the outputProcessing does not support external rows yet and we don't need it in this use case
   return await outputProcessing(table, row, {
-    squash: false,
+    squash: true,
     preserveLinks: true,
   })
 }
@@ -207,7 +211,7 @@ export async function fetchEnrichedRow(ctx: UserCtx) {
     // don't support composite keys right now
     const linkedIds = links.map((link: Row) => breakRowIdField(link._id!)[0])
     const primaryLink = linkedTable.primary?.[0] as string
-    row[fieldName] = await handleRequest(Operation.READ, linkedTableId!, {
+    const relatedRows = await handleRequest(Operation.READ, linkedTableId!, {
       tables,
       filters: {
         oneOf: {
@@ -216,6 +220,10 @@ export async function fetchEnrichedRow(ctx: UserCtx) {
       },
       includeSqlRelationships: IncludeRelationship.INCLUDE,
     })
+    row[fieldName] = await outputProcessing(linkedTable, relatedRows, {
+      squash: true,
+      preserveLinks: true,
+    })
   }
   return row
 }

packages/server/src/api/controllers/row/staticFormula.ts

@@ -101,12 +101,12 @@ export async function updateAllFormulasInTable(table: Table) {
   for (let row of rows) {
     // find the enriched row, if found process the formulas
     const enrichedRow = enrichedRows.find(
-      (enriched: any) => enriched._id === row._id
+      (enriched: Row) => enriched._id === row._id
     )
     if (enrichedRow) {
       const processed = processFormulas(table, cloneDeep(row), {
         dynamic: false,
-        contextRows: enrichedRow,
+        contextRows: [enrichedRow],
       })
       // values have changed, need to add to bulk docs to update
       if (!isEqual(processed, row)) {
@@ -139,7 +139,7 @@ export async function finaliseRow(
   // use enriched row to generate formulas for saving, specifically only use as context
   row = processFormulas(table, row, {
     dynamic: false,
-    contextRows: enrichedRow,
+    contextRows: [enrichedRow],
   })
   // don't worry about rev, tables handle rev/lastID updates
   // if another row has been written since processing this will
@@ -163,7 +163,9 @@ export async function finaliseRow(
   const response = await db.put(row)
   // for response, calculate the formulas for the enriched row
   enrichedRow._rev = response.rev
-  enrichedRow = await processFormulas(table, enrichedRow, { dynamic: false })
+  enrichedRow = processFormulas(table, enrichedRow, {
+    dynamic: false,
+  })
   // this updates the related formulas in other rows based on the relations to this row
   if (updateFormula) {
     await updateRelatedFormula(table, enrichedRow)

packages/server/src/api/routes/public/index.ts

@@ -27,6 +27,10 @@ interface KoaRateLimitOptions {
 }
 
 const PREFIX = "/api/public/v1"
+
+// type can't be known - untyped libraries
+let limiter: any, rateLimitStore: any
+if (!env.DISABLE_RATE_LIMITING) {
   // allow a lot more requests when in test
   const DEFAULT_API_REQ_LIMIT_PER_SEC = env.isTest() ? 100 : 10
 
@@ -37,7 +41,6 @@ function getApiLimitPerSecond(): number {
     return parseInt(env.API_REQ_LIMIT_PER_SEC)
   }
 
-let rateLimitStore: any = null
   if (!env.isTest()) {
     const { password, host, port } = redis.utils.getRedisConnectionDetails()
     let options: KoaRateLimitOptions = {
@@ -61,17 +64,22 @@ if (!env.isTest()) {
     })
   }
   // rate limiting, allows for 2 requests per second
-const limiter = RateLimit.middleware({
+  limiter = RateLimit.middleware({
     interval: { sec: 1 },
     // per ip, per interval
     max: getApiLimitPerSecond(),
   })
+} else {
+  console.log("**** PUBLIC API RATE LIMITING DISABLED ****")
+}
 
 const publicRouter = new Router({
   prefix: PREFIX,
 })
 
+if (limiter) {
   publicRouter.use(limiter)
+}
 
 function addMiddleware(
   endpoints: any,

packages/server/src/api/routes/tests/row.spec.ts

@@ -10,6 +10,7 @@ import {
   FieldSchema,
   FieldType,
   FieldTypeSubtypes,
+  FormulaTypes,
   INTERNAL_TABLE_SOURCE_ID,
   MonthlyQuotaName,
   PermissionLevel,
@@ -2000,4 +2001,52 @@ describe.each([
       })
     })
   })
+
+  describe("Formula fields", () => {
+    let relationshipTable: Table, tableId: string, relatedRow: Row
+
+    beforeAll(async () => {
+      const otherTableId = config.table!._id!
+      const cfg = generateTableConfig()
+      relationshipTable = await config.createLinkedTable(
+        RelationshipType.ONE_TO_MANY,
+        ["links"],
+        {
+          ...cfg,
+          // needs to be a short name
+          name: "b",
+          schema: {
+            ...cfg.schema,
+            formula: {
+              name: "formula",
+              type: FieldType.FORMULA,
+              formula: "{{ links.0.name }}",
+              formulaType: FormulaTypes.DYNAMIC,
+            },
+          },
+        }
+      )
+
+      tableId = relationshipTable._id!
+
+      relatedRow = await config.api.row.save(otherTableId, {
+        name: generator.word(),
+        description: generator.paragraph(),
+      })
+      await config.api.row.save(tableId, {
+        name: generator.word(),
+        description: generator.paragraph(),
+        tableId,
+        links: [relatedRow._id],
+      })
+    })
+
+    it("should be able to search for rows containing formulas", async () => {
+      const { rows } = await config.api.row.search(tableId)
+      expect(rows.length).toBe(1)
+      expect(rows[0].links.length).toBe(1)
+      const row = rows[0]
+      expect(row.formula).toBe(relatedRow.name)
+    })
+  })
 })

packages/server/src/db/linkedRows/index.ts

@@ -2,7 +2,6 @@ import LinkController from "./LinkController"
 import {
   IncludeDocs,
   getLinkDocuments,
-  createLinkView,
   getUniqueByProp,
   getRelatedTableForField,
   getLinkedTableIDs,

packages/server/src/db/linkedRows/linkUtils.ts

@@ -8,6 +8,7 @@ import {
   LinkDocumentValue,
   Table,
 } from "@budibase/types"
+import sdk from "../../sdk"
 
 export { createLinkView } from "../views/staticViews"
 
@@ -110,12 +111,11 @@ export function getLinkedTableIDs(table: Table): string[] {
 }
 
 export async function getLinkedTable(id: string, tables: Table[]) {
-  const db = context.getAppDB()
   let linkedTable = tables.find(table => table._id === id)
   if (linkedTable) {
     return linkedTable
   }
-  linkedTable = await db.get(id)
+  linkedTable = await sdk.tables.getTable(id)
   if (linkedTable) {
     tables.push(linkedTable)
   }

packages/server/src/environment.ts

@@ -61,6 +61,7 @@ const environment = {
   ALLOW_DEV_AUTOMATIONS: process.env.ALLOW_DEV_AUTOMATIONS,
   DISABLE_THREADING: process.env.DISABLE_THREADING,
   DISABLE_AUTOMATION_LOGS: process.env.DISABLE_AUTOMATION_LOGS,
+  DISABLE_RATE_LIMITING: process.env.DISABLE_RATE_LIMITING,
   MULTI_TENANCY: process.env.MULTI_TENANCY,
   ENABLE_ANALYTICS: process.env.ENABLE_ANALYTICS,
   SELF_HOSTED: process.env.SELF_HOSTED,

packages/server/src/integration-test/postgres.spec.ts

@@ -923,7 +923,6 @@ describe("postgres integrations", () => {
             [m2mFieldName]: [
               {
                 _id: row._id,
-                primaryDisplay: "Invalid display column",
               },
             ],
           })
@@ -932,7 +931,6 @@ describe("postgres integrations", () => {
             [m2mFieldName]: [
               {
                 _id: row._id,
-                primaryDisplay: "Invalid display column",
               },
             ],
           })

packages/server/src/sdk/app/rows/search/external.ts

@@ -80,7 +80,10 @@ export async function search(options: SearchParams) {
       rows = rows.map((r: any) => pick(r, fields))
     }
 
-    rows = await outputProcessing(table, rows, { preserveLinks: true })
+    rows = await outputProcessing(table, rows, {
+      preserveLinks: true,
+      squash: true,
+    })
 
     // need wrapper object for bookmarks etc when paginating
     return { rows, hasNextPage, bookmark: bookmark && bookmark + 1 }
@@ -185,6 +188,7 @@ export async function fetch(tableId: string): Promise<Row[]> {
   const table = await sdk.tables.getTable(tableId)
   return await outputProcessing<Row[]>(table, response, {
     preserveLinks: true,
+    squash: true,
   })
 }
 

packages/server/src/tests/utilities/api/row.ts

@@ -6,6 +6,7 @@ import {
   ExportRowsRequest,
   BulkImportRequest,
   BulkImportResponse,
+  SearchRowResponse,
 } from "@budibase/types"
 import TestConfiguration from "../TestConfiguration"
 import { TestAPI } from "./base"
@@ -154,7 +155,7 @@ export class RowAPI extends TestAPI {
   search = async (
     sourceId: string,
     { expectStatus } = { expectStatus: 200 }
-  ): Promise<Row[]> => {
+  ): Promise<SearchRowResponse> => {
     const request = this.request
       .post(`/api/${sourceId}/search`)
       .set(this.config.defaultHeaders())

packages/server/src/utilities/rowProcessor/index.ts

@@ -2,16 +2,15 @@ import * as linkRows from "../../db/linkedRows"
 import { FieldTypes, AutoFieldSubTypes } from "../../constants"
 import { processFormulas, fixAutoColumnSubType } from "./utils"
 import { ObjectStoreBuckets } from "../../constants"
-import { context, db as dbCore, objectStore } from "@budibase/backend-core"
+import {
+  context,
+  db as dbCore,
+  objectStore,
+  utils,
+} from "@budibase/backend-core"
 import { InternalTables } from "../../db/utils"
 import { TYPE_TRANSFORM_MAP } from "./map"
-import {
-  AutoColumnFieldMetadata,
-  FieldSubtype,
-  Row,
-  RowAttachment,
-  Table,
-} from "@budibase/types"
+import { FieldSubtype, Row, RowAttachment, Table } from "@budibase/types"
 import { cloneDeep } from "lodash/fp"
 import {
   processInputBBReferences,
@@ -233,6 +232,11 @@ export async function outputProcessing<T extends Row[] | Row>(
       })
     : safeRows
 
+  // make sure squash is enabled if needed
+  if (!opts.squash && utils.hasCircularStructure(rows)) {
+    opts.squash = true
+  }
+
   // process complex types: attachements, bb references...
   for (let [property, column] of Object.entries(table.schema)) {
     if (column.type === FieldTypes.ATTACHMENT) {
@@ -258,7 +262,7 @@ export async function outputProcessing<T extends Row[] | Row>(
   }
 
   // process formulas after the complex types had been processed
-  enriched = processFormulas(table, enriched, { dynamic: true }) as Row[]
+  enriched = processFormulas(table, enriched, { dynamic: true })
 
   if (opts.squash) {
     enriched = (await linkRows.squashLinksToPrimaryDisplay(

packages/server/src/utilities/rowProcessor/utils.ts

@@ -12,6 +12,11 @@ import {
   Table,
 } from "@budibase/types"
 
+interface FormulaOpts {
+  dynamic?: boolean
+  contextRows?: Row[]
+}
+
 /**
  * If the subtype has been lost for any reason this works out what
  * subtype the auto column should be.
@@ -40,19 +45,13 @@ export function fixAutoColumnSubType(
 /**
  * Looks through the rows provided and finds formulas - which it then processes.
  */
-export function processFormulas(
+export function processFormulas<T extends Row | Row[]>(
   table: Table,
-  rows: Row[] | Row,
-  { dynamic, contextRows }: any = { dynamic: true }
-) {
-  const single = !Array.isArray(rows)
-  let rowArray: Row[]
-  if (single) {
-    rowArray = [rows]
-    contextRows = contextRows ? [contextRows] : contextRows
-  } else {
-    rowArray = rows
-  }
+  inputRows: T,
+  { dynamic, contextRows }: FormulaOpts = { dynamic: true }
+): T {
+  const rows = Array.isArray(inputRows) ? inputRows : [inputRows]
+  if (rows)
     for (let [column, schema] of Object.entries(table.schema)) {
       if (schema.type !== FieldTypes.FORMULA) {
         continue
@@ -68,24 +67,28 @@ export function processFormulas(
         continue
       }
       // iterate through rows and process formula
-    for (let i = 0; i < rowArray.length; i++) {
-      let row = rowArray[i]
+      for (let i = 0; i < rows.length; i++) {
+        let row = rows[i]
         let context = contextRows ? contextRows[i] : row
-      rowArray[i] = {
+        rows[i] = {
           ...row,
           [column]: processStringSync(schema.formula, context),
         }
       }
     }
-  return single ? rowArray[0] : rowArray
+  return Array.isArray(inputRows) ? rows : rows[0]
 }
 
 /**
  * Processes any date columns and ensures that those without the ignoreTimezones
  * flag set are parsed as UTC rather than local time.
  */
-export function processDates(table: Table, rows: Row[]) {
-  let datesWithTZ = []
+export function processDates<T extends Row | Row[]>(
+  table: Table,
+  inputRows: T
+): T {
+  let rows = Array.isArray(inputRows) ? inputRows : [inputRows]
+  let datesWithTZ: string[] = []
   for (let [column, schema] of Object.entries(table.schema)) {
     if (schema.type !== FieldTypes.DATETIME) {
       continue
@@ -102,5 +105,6 @@ export function processDates(table: Table, rows: Row[]) {
       }
     }
   }
-  return rows
+
+  return Array.isArray(inputRows) ? rows : rows[0]
 }

qa-core/package.json

@@ -20,7 +20,7 @@
     "test:self:ci": "yarn run test --testPathIgnorePatterns=\\.integration\\. \\.cloud\\. \\.licensing\\.",
     "serve:test:self:ci": "start-server-and-test dev:built http://localhost:4001/health test:self:ci",
     "serve": "start-server-and-test dev:built http://localhost:4001/health",
-    "dev:built": "cd ../ && yarn dev:built"
+    "dev:built": "cd ../ && DISABLE_RATE_LIMITING=1 yarn dev:built"
   },
   "devDependencies": {
     "@budibase/types": "^2.3.17",