Merge branch 'master' into fix/disable-external-auto-columns

packages/server/src/api/routes/tests/row.spec.ts

@@ -32,8 +32,6 @@ import * as uuid from "uuid"
 const timestamp = new Date("2023-01-26T11:48:57.597Z").toISOString()
 tk.freeze(timestamp)
 
-jest.unmock("mssql")
-
 describe.each([
   ["internal", undefined],
   [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)],
@@ -131,7 +129,13 @@ describe.each([
 
   const assertRowUsage = async (expected: number) => {
     const usage = await getRowUsage()
-    expect(usage).toBe(expected)
+
+    // Because our quota tracking is not perfect, we allow a 10% margin of
+    // error.  This is to account for the fact that parallel writes can result
+    // in some quota updates getting lost. We don't have any need to solve this
+    // right now, so we just allow for some error.
+    expect(usage).toBeGreaterThan(expected * 0.9)
+    expect(usage).toBeLessThan(expected * 1.1)
   }
 
   const defaultRowFields = isInternal
@@ -194,39 +198,99 @@ describe.each([
       await assertRowUsage(rowUsage)
     })
 
-    it("increment row autoId per create row request", async () => {
+    isInternal &&
-      const rowUsage = await getRowUsage()
+      it("increment row autoId per create row request", async () => {
+        const rowUsage = await getRowUsage()
 
-      const newTable = await config.api.table.save(
+        const newTable = await config.api.table.save(
-        saveTableRequest({
+          saveTableRequest({
-          schema: {
+            schema: {
-            "Row ID": {
+              "Row ID": {
-              name: "Row ID",
+                name: "Row ID",
-              type: FieldType.NUMBER,
+                type: FieldType.NUMBER,
-              subtype: AutoFieldSubType.AUTO_ID,
+                subtype: AutoFieldSubType.AUTO_ID,
-              icon: "ri-magic-line",
+                icon: "ri-magic-line",
-              autocolumn: true,
+                autocolumn: true,
-              constraints: {
+                constraints: {
-                type: "number",
+                  type: "number",
-                presence: true,
+                  presence: true,
-                numericality: {
+                  numericality: {
-                  greaterThanOrEqualTo: "",
+                    greaterThanOrEqualTo: "",
-                  lessThanOrEqualTo: "",
+                    lessThanOrEqualTo: "",
+                  },
                 },
               },
             },
-          },
+          })
-        })
+        )
-      )
 
-      let previousId = 0
+        let previousId = 0
-      for (let i = 0; i < 10; i++) {
+        for (let i = 0; i < 10; i++) {
-        const row = await config.api.row.save(newTable._id!, {})
+          const row = await config.api.row.save(newTable._id!, {})
-        expect(row["Row ID"]).toBeGreaterThan(previousId)
+          expect(row["Row ID"]).toBeGreaterThan(previousId)
-        previousId = row["Row ID"]
+          previousId = row["Row ID"]
-      }
+        }
-      await assertRowUsage(rowUsage + 10)
+        await assertRowUsage(rowUsage + 10)
-    })
+      })
+
+    isInternal &&
+      it("should increment auto ID correctly when creating rows in parallel", async () => {
+        const table = await config.api.table.save(
+          saveTableRequest({
+            schema: {
+              "Row ID": {
+                name: "Row ID",
+                type: FieldType.NUMBER,
+                subtype: AutoFieldSubType.AUTO_ID,
+                icon: "ri-magic-line",
+                autocolumn: true,
+                constraints: {
+                  type: "number",
+                  presence: true,
+                  numericality: {
+                    greaterThanOrEqualTo: "",
+                    lessThanOrEqualTo: "",
+                  },
+                },
+              },
+            },
+          })
+        )
+
+        const sequence = Array(50)
+          .fill(0)
+          .map((_, i) => i + 1)
+
+        // This block of code is simulating users creating auto ID rows at the
+        // same time. It's expected that this operation will sometimes return
+        // a document conflict error (409), but the idea is to retry in those
+        // situations. The code below does this a large number of times with
+        // small, random delays between them to try and get through the list
+        // as quickly as possible.
+        await Promise.all(
+          sequence.map(async () => {
+            const attempts = 20
+            for (let attempt = 0; attempt < attempts; attempt++) {
+              try {
+                await config.api.row.save(table._id!, {})
+                return
+              } catch (e) {
+                await new Promise(r => setTimeout(r, Math.random() * 15))
+              }
+            }
+            throw new Error(`Failed to create row after ${attempts} attempts`)
+          })
+        )
+
+        const rows = await config.api.row.fetch(table._id!)
+        expect(rows).toHaveLength(50)
+
+        // The main purpose of this test is to ensure that even under pressure,
+        // we maintain data integrity. An auto ID column should hand out
+        // monotonically increasing unique integers no matter what.
+        const ids = rows.map(r => r["Row ID"])
+        expect(ids).toEqual(expect.arrayContaining(sequence))
+      })
 
     isInternal &&
       it("row values are coerced", async () => {

packages/server/src/api/routes/tests/search.spec.ts

@@ -16,8 +16,6 @@ import {
 } from "@budibase/types"
 import _ from "lodash"
 
-jest.unmock("mssql")
-
 describe.each([
   ["lucene", undefined],
   ["sqs", undefined],

packages/server/src/api/routes/tests/viewV2.spec.ts

@@ -24,8 +24,6 @@ import merge from "lodash/merge"
 import { quotas } from "@budibase/pro"
 import { roles } from "@budibase/backend-core"
 
-jest.unmock("mssql")
-
 describe.each([
   ["internal", undefined],
   [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)],

packages/server/src/sdk/app/rows/tests/internal.spec.ts

@@ -1,230 +0,0 @@
-import tk from "timekeeper"
-import * as internalSdk from "../internal"
-
-import { generator } from "@budibase/backend-core/tests"
-import {
-  INTERNAL_TABLE_SOURCE_ID,
-  TableSourceType,
-  FieldType,
-  Table,
-  AutoFieldSubType,
-  AutoColumnFieldMetadata,
-} from "@budibase/types"
-
-import TestConfiguration from "../../../../tests/utilities/TestConfiguration"
-
-tk.freeze(Date.now())
-
-describe("sdk >> rows >> internal", () => {
-  const config = new TestConfiguration()
-
-  beforeAll(async () => {
-    await config.init()
-  })
-
-  function makeRow() {
-    return {
-      name: generator.first(),
-      surname: generator.last(),
-      age: generator.age(),
-      address: generator.address(),
-    }
-  }
-
-  describe("save", () => {
-    const tableData: Table = {
-      name: generator.word(),
-      type: "table",
-      sourceId: INTERNAL_TABLE_SOURCE_ID,
-      sourceType: TableSourceType.INTERNAL,
-      schema: {
-        name: {
-          name: "name",
-          type: FieldType.STRING,
-          constraints: {
-            type: FieldType.STRING,
-          },
-        },
-        surname: {
-          name: "surname",
-          type: FieldType.STRING,
-          constraints: {
-            type: FieldType.STRING,
-          },
-        },
-        age: {
-          name: "age",
-          type: FieldType.NUMBER,
-          constraints: {
-            type: FieldType.NUMBER,
-          },
-        },
-        address: {
-          name: "address",
-          type: FieldType.STRING,
-          constraints: {
-            type: FieldType.STRING,
-          },
-        },
-      },
-    }
-
-    beforeEach(() => {
-      jest.clearAllMocks()
-    })
-
-    it("save will persist the row properly", async () => {
-      const table = await config.createTable(tableData)
-      const row = makeRow()
-
-      await config.doInContext(config.appId, async () => {
-        const response = await internalSdk.save(
-          table._id!,
-          row,
-          config.getUser()._id
-        )
-
-        expect(response).toEqual({
-          table,
-          row: {
-            ...row,
-            type: "row",
-            _rev: expect.stringMatching("1-.*"),
-          },
-          squashed: {
-            ...row,
-            type: "row",
-            _rev: expect.stringMatching("1-.*"),
-          },
-        })
-
-        const persistedRow = await config.api.row.get(
-          table._id!,
-          response.row._id!
-        )
-        expect(persistedRow).toEqual({
-          ...row,
-          type: "row",
-          _rev: expect.stringMatching("1-.*"),
-          createdAt: expect.any(String),
-          updatedAt: expect.any(String),
-        })
-      })
-    })
-
-    it("auto ids will update when creating new rows", async () => {
-      const table = await config.createTable({
-        ...tableData,
-        schema: {
-          ...tableData.schema,
-          id: {
-            name: "id",
-            type: FieldType.AUTO,
-            subtype: AutoFieldSubType.AUTO_ID,
-            autocolumn: true,
-            lastID: 0,
-          },
-        },
-      })
-      const row = makeRow()
-
-      await config.doInContext(config.appId, async () => {
-        const response = await internalSdk.save(
-          table._id!,
-          row,
-          config.getUser()._id
-        )
-
-        expect(response).toEqual({
-          table: {
-            ...table,
-            schema: {
-              ...table.schema,
-              id: {
-                ...table.schema.id,
-                lastID: 1,
-              },
-            },
-            _rev: expect.stringMatching("2-.*"),
-          },
-          row: {
-            ...row,
-            id: 1,
-            type: "row",
-            _rev: expect.stringMatching("1-.*"),
-          },
-          squashed: {
-            ...row,
-            id: 1,
-            type: "row",
-            _rev: expect.stringMatching("1-.*"),
-          },
-        })
-
-        const persistedRow = await config.api.row.get(
-          table._id!,
-          response.row._id!
-        )
-        expect(persistedRow).toEqual({
-          ...row,
-          type: "row",
-          id: 1,
-          _rev: expect.stringMatching("1-.*"),
-          createdAt: expect.any(String),
-          updatedAt: expect.any(String),
-        })
-      })
-    })
-
-    it("auto ids will update when creating new rows in parallel", async () => {
-      function makeRows(count: number) {
-        return Array.from({ length: count }, () => makeRow())
-      }
-
-      const table = await config.createTable({
-        ...tableData,
-        schema: {
-          ...tableData.schema,
-          id: {
-            name: "id",
-            type: FieldType.AUTO,
-            subtype: AutoFieldSubType.AUTO_ID,
-            autocolumn: true,
-          },
-        },
-      })
-
-      await config.doInContext(config.appId, async () => {
-        for (const row of makeRows(5)) {
-          await internalSdk.save(table._id!, row, config.getUser()._id)
-        }
-        await Promise.all(
-          makeRows(20).map(row =>
-            internalSdk.save(table._id!, row, config.getUser()._id)
-          )
-        )
-        for (const row of makeRows(5)) {
-          await internalSdk.save(table._id!, row, config.getUser()._id)
-        }
-      })
-
-      const persistedRows = await config.getRows(table._id!)
-      expect(persistedRows).toHaveLength(30)
-      expect(persistedRows).toEqual(
-        expect.arrayContaining(
-          Array.from({ length: 30 }).map((_, i) =>
-            expect.objectContaining({ id: i + 1 })
-          )
-        )
-      )
-
-      const persistedTable = await config.getTable(table._id)
-      expect(
-        (table.schema.id as AutoColumnFieldMetadata).lastID
-      ).toBeUndefined()
-      expect((persistedTable.schema.id as AutoColumnFieldMetadata).lastID).toBe(
-        30
-      )
-    })
-  })
-})

packages/server/src/utilities/rowProcessor/index.ts

@@ -1,6 +1,6 @@
 import * as linkRows from "../../db/linkedRows"
 import { processFormulas, fixAutoColumnSubType } from "./utils"
-import { context, objectStore, utils } from "@budibase/backend-core"
+import { objectStore, utils } from "@budibase/backend-core"
 import { InternalTables } from "../../db/utils"
 import { TYPE_TRANSFORM_MAP } from "./map"
 import {
@@ -25,45 +25,6 @@ type AutoColumnProcessingOpts = {
   noAutoRelationships?: boolean
 }
 
-// Returns the next auto ID for a column in a table. On success, the table will
-// be updated which is why it gets returned. The nextID returned is guaranteed
-// to be given only to you, and if you don't use it it's gone forever (a gap
-// will be left in the auto ID sequence).
-//
-// This function can throw if it fails to generate an auto ID after so many
-// attempts.
-async function getNextAutoId(
-  table: Table,
-  column: string
-): Promise<{ table: Table; nextID: number }> {
-  const db = context.getAppDB()
-  for (let attempt = 0; attempt < 5; attempt++) {
-    const schema = table.schema[column]
-    if (schema.type !== FieldType.NUMBER && schema.type !== FieldType.AUTO) {
-      throw new Error(`Column ${column} is not an auto column`)
-    }
-    schema.lastID = (schema.lastID || 0) + 1
-    try {
-      const resp = await db.put(table)
-      table._rev = resp.rev
-      return { table, nextID: schema.lastID }
-    } catch (e: any) {
-      if (e.status !== 409) {
-        throw e
-      }
-      // We wait for a random amount of time before retrying. The randomness
-      // makes it less likely for multiple requests modifying this table to
-      // collide.
-      await new Promise(resolve =>
-        setTimeout(resolve, Math.random() * 1.2 ** attempt * 1000)
-      )
-      table = await db.get(table._id)
-    }
-  }
-
-  throw new Error("Failed to generate an auto ID")
-}
-
 /**
  * This will update any auto columns that are found on the row/table with the correct information based on
  * time now and the current logged in user making the request.
@@ -116,9 +77,10 @@ export async function processAutoColumn(
         break
       case AutoFieldSubType.AUTO_ID:
         if (creating) {
-          const { table: newTable, nextID } = await getNextAutoId(table, key)
+          schema.lastID = schema.lastID || 0
-          table = newTable
+          row[key] = schema.lastID + 1
-          row[key] = nextID
+          schema.lastID++
+          table.schema[key] = schema
         }
         break
     }

packages/worker/src/api/routes/global/tests/realEmail.spec.ts

@@ -1,5 +1,4 @@
 jest.unmock("node-fetch")
-jest.unmock("aws-sdk")
 import { TestConfiguration } from "../../../../tests"
 import { EmailTemplatePurpose } from "../../../../constants"
 import { objectStore } from "@budibase/backend-core"