Merge branch 'master' into execute-script-v2

hosting/nginx.dev.conf

@@ -62,6 +62,12 @@ http {
       proxy_connect_timeout 120s;
       proxy_send_timeout 120s;
       proxy_http_version 1.1;
+
+      # Enable buffering for potentially large OIDC configs
+      proxy_buffering on;
+      proxy_buffer_size 16k;
+      proxy_buffers 4 32k;
+
       proxy_set_header Host $host;
       proxy_set_header Connection "";
 

packages/builder/src/stores/builder/automations.ts

@@ -489,7 +489,7 @@ const automationActions = (store: AutomationStore) => ({
         branches.forEach((branch, bIdx) => {
           children[branch.id].forEach(
             (bBlock: AutomationStep, sIdx: number, array: AutomationStep[]) => {
-              const ended = array.length - 1 === sIdx && !branches.length
+              const ended = array.length - 1 === sIdx
               treeTraverse(bBlock, pathToCurrentNode, sIdx, bIdx, ended)
             }
           )
@@ -510,7 +510,6 @@ const automationActions = (store: AutomationStore) => ({
     blocks.forEach((block, idx, array) => {
       treeTraverse(block, null, idx, null, array.length - 1 === idx)
     })
-
     return blockRefs
   },
 

packages/server/src/api/controllers/row/utils/tests/sqlUtils.spec.ts

@@ -1,45 +1,49 @@
 import {
   AIOperationEnum,
   CalculationType,
+  Datasource,
   FieldType,
   RelationshipType,
-  SourceName,
   Table,
   ViewV2,
   ViewV2Type,
 } from "@budibase/types"
 import { buildSqlFieldList } from "../sqlUtils"
 import { structures } from "../../../../routes/tests/utilities"
-import { sql } from "@budibase/backend-core"
 import { generator } from "@budibase/backend-core/tests"
 import { generateViewID } from "../../../../../db/utils"
 
-import sdk from "../../../../../sdk"
-import { cloneDeep } from "lodash"
 import { utils } from "@budibase/shared-core"
+import {
+  DatabaseName,
+  datasourceDescribe,
+} from "../../../../../integrations/tests/utils"
+import { context } from "@budibase/backend-core"
 
-jest.mock("../../../../../sdk/app/views", () => ({
+const descriptions = datasourceDescribe({
-  ...jest.requireActual("../../../../../sdk/app/views"),
+  only: [DatabaseName.POSTGRES],
-  getTable: jest.fn(),
+})
-}))
-const getTableMock = sdk.views.getTable as jest.MockedFunction<
-  typeof sdk.views.getTable
->
 
-describe("buildSqlFieldList", () => {
+if (descriptions.length) {
+  describe.each(descriptions)(
+    "buildSqlFieldList ($dbName)",
+    ({ config, dsProvider }) => {
       let allTables: Record<string, Table>
+      let datasource: Datasource
+
+      beforeEach(async () => {
+        const ds = await dsProvider()
+        datasource = ds.datasource!
+        allTables = {}
+      })
 
       class TableConfig {
-    private _table: Table & { _id: string }
+        private _table: Table
 
         constructor(name: string) {
           this._table = {
-        ...structures.tableForDatasource({
+            ...structures.tableForDatasource(datasource),
-          type: "datasource",
-          source: SourceName.POSTGRES,
-        }),
             name,
-        _id: sql.utils.buildExternalTableId("ds_id", name),
             schema: {
               name: {
                 name: "name",
@@ -55,8 +59,6 @@ describe("buildSqlFieldList", () => {
               },
             },
           }
-
-      allTables[name] = this._table
         }
 
         withHiddenField(field: string) {
@@ -110,6 +112,7 @@ describe("buildSqlFieldList", () => {
             type: FieldType.LINK,
             relationshipType: RelationshipType.ONE_TO_MANY,
             fieldName: "link",
+            foreignKey: "link",
             tableId: toTableId,
           }
           return this
@@ -125,17 +128,17 @@ describe("buildSqlFieldList", () => {
           return this
         }
 
-    create() {
+        async create() {
-      return cloneDeep(this._table)
+          const table = await config.api.table.save(this._table)
+          allTables[table.name] = table
+          return table
         }
       }
 
       class ViewConfig {
-    private _table: Table
         private _view: ViewV2
 
         constructor(table: Table) {
-      this._table = table
           this._view = {
             version: 2,
             id: generateViewID(table._id!),
@@ -183,197 +186,208 @@ describe("buildSqlFieldList", () => {
           return this
         }
 
-    create() {
+        async create() {
-      getTableMock.mockResolvedValueOnce(this._table)
+          return await config.api.viewV2.create(this._view)
-      return cloneDeep(this._view)
         }
       }
 
-  beforeEach(() => {
+      const buildSqlFieldListInApp: typeof buildSqlFieldList = async (
-    jest.clearAllMocks()
+        table,
-    allTables = {}
+        allTables,
-  })
+        opts
+      ) => {
+        return context.doInAppContext(config.getAppId(), () =>
+          buildSqlFieldList(table, allTables, opts)
+        )
+      }
 
       describe("table", () => {
         it("extracts fields from table schema", async () => {
-      const table = new TableConfig("table").create()
+          const table = await new TableConfig("table").create()
-      const result = await buildSqlFieldList(table, {})
+          const result = await buildSqlFieldListInApp(table, {})
           expect(result).toEqual([
             "table.name",
             "table.description",
             "table.amount",
+            "table.id",
           ])
         })
 
         it("excludes hidden fields", async () => {
-      const table = new TableConfig("table")
+          const table = await new TableConfig("table")
             .withHiddenField("description")
             .create()
-      const result = await buildSqlFieldList(table, {})
+          const result = await buildSqlFieldListInApp(table, {})
-      expect(result).toEqual(["table.name", "table.amount"])
+          expect(result).toEqual(["table.name", "table.amount", "table.id"])
         })
 
         it("excludes non-sql fields fields", async () => {
-      const table = new TableConfig("table")
+          const table = await new TableConfig("table")
             .withField("formula", FieldType.FORMULA)
             .withField("ai", FieldType.AI)
-        .withRelation("link", "otherTableId")
             .create()
 
-      const result = await buildSqlFieldList(table, {})
+          const result = await buildSqlFieldListInApp(table, {})
           expect(result).toEqual([
             "table.name",
             "table.description",
             "table.amount",
+            "table.id",
           ])
         })
 
         it("includes hidden fields if there is a formula column", async () => {
-      const table = new TableConfig("table")
+          const table = await new TableConfig("table")
             .withHiddenField("description")
             .withField("formula", FieldType.FORMULA)
             .create()
 
-      const result = await buildSqlFieldList(table, {})
+          const result = await buildSqlFieldListInApp(table, {})
           expect(result).toEqual([
             "table.name",
             "table.description",
             "table.amount",
+            "table.id",
           ])
         })
 
         it("includes relationships fields when flagged", async () => {
-      const otherTable = new TableConfig("linkedTable")
+          const otherTable = await new TableConfig("linkedTable")
             .withField("id", FieldType.NUMBER)
             .withPrimary("id")
             .withDisplay("name")
             .create()
 
-      const table = new TableConfig("table")
+          const table = await new TableConfig("table")
-        .withRelation("link", otherTable._id)
+            .withRelation("link", otherTable._id!)
             .create()
 
-      const result = await buildSqlFieldList(table, allTables, {
+          const result = await buildSqlFieldListInApp(table, allTables, {
             relationships: true,
           })
           expect(result).toEqual([
             "table.name",
             "table.description",
             "table.amount",
+            "table.id",
             "linkedTable.id",
             "linkedTable.name",
           ])
         })
 
         it("includes all relationship fields if there is a formula column", async () => {
-      const otherTable = new TableConfig("linkedTable")
+          const otherTable = await new TableConfig("linkedTable")
             .withField("hidden", FieldType.STRING, { visible: false })
             .create()
 
-      const table = new TableConfig("table")
+          const table = await new TableConfig("table")
-        .withRelation("link", otherTable._id)
+            .withRelation("link", otherTable._id!)
             .withField("formula", FieldType.FORMULA)
             .create()
 
-      const result = await buildSqlFieldList(table, allTables, {
+          const result = await buildSqlFieldListInApp(table, allTables, {
             relationships: true,
           })
           expect(result).toEqual([
             "table.name",
             "table.description",
             "table.amount",
+            "table.id",
             "linkedTable.name",
             "linkedTable.description",
             "linkedTable.amount",
             "linkedTable.hidden",
+            "linkedTable.id",
           ])
         })
 
         it("never includes non-sql columns from relationships", async () => {
-      const otherTable = new TableConfig("linkedTable")
+          const otherTable = await new TableConfig("linkedTable")
-        .withField("id", FieldType.NUMBER)
             .withField("hidden", FieldType.STRING, { visible: false })
             .withField("formula", FieldType.FORMULA)
             .withField("ai", FieldType.AI)
-        .withRelation("link", "otherTableId")
             .create()
 
-      const table = new TableConfig("table")
+          const table = await new TableConfig("table")
-        .withRelation("link", otherTable._id)
+            .withRelation("link", otherTable._id!)
             .withField("formula", FieldType.FORMULA)
             .create()
 
-      const result = await buildSqlFieldList(table, allTables, {
+          const result = await buildSqlFieldListInApp(table, allTables, {
             relationships: true,
           })
           expect(result).toEqual([
             "table.name",
             "table.description",
             "table.amount",
+            "table.id",
             "linkedTable.name",
             "linkedTable.description",
             "linkedTable.amount",
-        "linkedTable.id",
             "linkedTable.hidden",
+            "linkedTable.id",
           ])
         })
       })
 
       describe("view", () => {
         it("extracts fields from table schema", async () => {
-      const view = new ViewConfig(new TableConfig("table").create())
+          const view = await new ViewConfig(
+            await new TableConfig("table").create()
+          )
             .withVisible("amount")
             .withHidden("name")
             .create()
 
-      const result = await buildSqlFieldList(view, {})
+          const result = await buildSqlFieldListInApp(view, {})
-      expect(result).toEqual(["table.amount"])
+          expect(result).toEqual(["table.amount", "table.id"])
         })
 
         it("includes all fields if there is a formula column", async () => {
-      const table = new TableConfig("table")
+          const table = await new TableConfig("table")
             .withField("formula", FieldType.FORMULA)
             .create()
-      const view = new ViewConfig(table)
+          const view = await new ViewConfig(table)
             .withHidden("name")
             .withVisible("amount")
             .withVisible("formula")
             .create()
 
-      const result = await buildSqlFieldList(view, {})
+          const result = await buildSqlFieldListInApp(view, {})
           expect(result).toEqual([
             "table.name",
             "table.description",
             "table.amount",
+            "table.id",
           ])
         })
 
         it("does not includes all fields if the formula column is not included", async () => {
-      const table = new TableConfig("table")
+          const table = await new TableConfig("table")
             .withField("formula", FieldType.FORMULA)
             .create()
-      const view = new ViewConfig(table)
+          const view = await new ViewConfig(table)
             .withHidden("name")
             .withVisible("amount")
             .withHidden("formula")
             .create()
 
-      const result = await buildSqlFieldList(view, {})
+          const result = await buildSqlFieldListInApp(view, {})
-      expect(result).toEqual(["table.amount"])
+          expect(result).toEqual(["table.amount", "table.id"])
         })
 
         it("includes relationships columns", async () => {
-      const otherTable = new TableConfig("linkedTable")
+          const otherTable = await new TableConfig("linkedTable")
             .withField("id", FieldType.NUMBER)
             .withField("formula", FieldType.FORMULA)
             .withPrimary("id")
             .create()
 
-      const table = new TableConfig("table")
+          const table = await new TableConfig("table")
-        .withRelation("link", otherTable._id)
+            .withRelation("link", otherTable._id!)
             .create()
 
-      const view = new ViewConfig(table)
+          const view = await new ViewConfig(table)
             .withVisible("name")
             .withVisible("link")
             .withRelationshipColumns("link", {
@@ -383,51 +397,52 @@ describe("buildSqlFieldList", () => {
             })
             .create()
 
-      const result = await buildSqlFieldList(view, allTables, {
+          const result = await buildSqlFieldListInApp(view, allTables, {
             relationships: true,
           })
           expect(result).toEqual([
             "table.name",
+            "table.id",
             "linkedTable.id",
             "linkedTable.amount",
           ])
         })
 
         it("excludes relationships fields when view is not included in the view", async () => {
-      const otherTable = new TableConfig("linkedTable")
+          const otherTable = await new TableConfig("linkedTable")
             .withField("id", FieldType.NUMBER)
             .withPrimary("id")
             .withDisplay("name")
             .create()
 
-      const table = new TableConfig("table")
+          const table = await new TableConfig("table")
-        .withRelation("link", otherTable._id)
+            .withRelation("link", otherTable._id!)
             .withField("formula", FieldType.FORMULA)
             .create()
 
-      const view = new ViewConfig(table)
+          const view = await new ViewConfig(table)
             .withVisible("name")
             .withHidden("amount")
             .create()
 
-      const result = await buildSqlFieldList(view, allTables, {
+          const result = await buildSqlFieldListInApp(view, allTables, {
             relationships: true,
           })
-      expect(result).toEqual(["table.name"])
+          expect(result).toEqual(["table.name", "table.id"])
         })
 
         it("does not include relationships columns for hidden links", async () => {
-      const otherTable = new TableConfig("linkedTable")
+          const otherTable = await new TableConfig("linkedTable")
             .withField("id", FieldType.NUMBER)
             .withField("formula", FieldType.FORMULA)
             .withPrimary("id")
             .create()
 
-      const table = new TableConfig("table")
+          const table = await new TableConfig("table")
-        .withRelation("link", otherTable._id)
+            .withRelation("link", otherTable._id!)
             .create()
 
-      const view = new ViewConfig(table)
+          const view = await new ViewConfig(table)
             .withVisible("name")
             .withHidden("link")
             .withRelationshipColumns("link", {
@@ -437,28 +452,27 @@ describe("buildSqlFieldList", () => {
             })
             .create()
 
-      const result = await buildSqlFieldList(view, allTables, {
+          const result = await buildSqlFieldListInApp(view, allTables, {
             relationships: true,
           })
-      expect(result).toEqual(["table.name"])
+          expect(result).toEqual(["table.name", "table.id"])
         })
 
         it("includes all relationship fields if there is a formula column", async () => {
-      const otherTable = new TableConfig("linkedTable")
+          const otherTable = await new TableConfig("linkedTable")
             .withField("id", FieldType.NUMBER)
             .withField("hidden", FieldType.STRING, { visible: false })
             .withField("formula", FieldType.FORMULA)
             .withField("ai", FieldType.AI)
-        .withRelation("link", "otherTableId")
             .withPrimary("id")
             .create()
 
-      const table = new TableConfig("table")
+          const table = await new TableConfig("table")
-        .withRelation("link", otherTable._id)
+            .withRelation("link", otherTable._id!)
             .withField("formula", FieldType.FORMULA)
             .create()
 
-      const view = new ViewConfig(table)
+          const view = await new ViewConfig(table)
             .withVisible("name")
             .withVisible("formula")
             .withHidden("link")
@@ -469,13 +483,14 @@ describe("buildSqlFieldList", () => {
             })
             .create()
 
-      const result = await buildSqlFieldList(view, allTables, {
+          const result = await buildSqlFieldListInApp(view, allTables, {
             relationships: true,
           })
           expect(result).toEqual([
             "table.name",
             "table.description",
             "table.amount",
+            "table.id",
             "linkedTable.name",
             "linkedTable.description",
             "linkedTable.amount",
@@ -487,25 +502,31 @@ describe("buildSqlFieldList", () => {
 
       describe("calculation view", () => {
         it("does not include calculation fields", async () => {
-      const view = new ViewConfig(new TableConfig("table").create())
+          const view = await new ViewConfig(
+            await new TableConfig("table").create()
+          )
             .withCalculation("average", "amount", CalculationType.AVG)
 
             .create()
 
-      const result = await buildSqlFieldList(view, {})
+          const result = await buildSqlFieldListInApp(view, {})
           expect(result).toEqual([])
         })
 
         it("includes visible fields calculation fields", async () => {
-      const view = new ViewConfig(new TableConfig("table").create())
+          const view = await new ViewConfig(
+            await new TableConfig("table").create()
+          )
             .withCalculation("average", "amount", CalculationType.AVG)
             .withHidden("name")
             .withVisible("amount")
 
             .create()
 
-      const result = await buildSqlFieldList(view, {})
+          const result = await buildSqlFieldListInApp(view, {})
           expect(result).toEqual(["table.amount"])
         })
       })
-})
+    }
+  )
+}

packages/worker/package.json

@@ -62,6 +62,7 @@
     "koa-body": "4.2.0",
     "koa-compress": "4.0.1",
     "koa-passport": "4.1.4",
+    "koa-redis": "^4.0.1",
     "koa-send": "5.0.1",
     "koa-session": "5.13.1",
     "koa-static": "5.0.0",

packages/worker/src/api/routes/global/tests/auth.spec.ts

@@ -311,7 +311,7 @@ describe("/api/global/auth", () => {
       })
     })
 
-    describe("GET /api/global/auth/:tenantId/oidc/callback", () => {
+    describe.skip("GET /api/global/auth/:tenantId/oidc/callback", () => {
       it("logs in", async () => {
         const email = `${generator.guid()}@example.com`
 

packages/worker/src/index.ts

@@ -4,7 +4,7 @@ if (process.env.DD_APM_ENABLED) {
 
 // need to load environment first
 import env from "./environment"
-import Application from "koa"
+import Application, { Middleware } from "koa"
 import { bootstrap } from "global-agent"
 import * as db from "./db"
 import { sdk as proSdk } from "@budibase/pro"
@@ -20,6 +20,7 @@ import {
   cache,
   features,
 } from "@budibase/backend-core"
+import RedisStore from "koa-redis"
 
 db.init()
 import koaBody from "koa-body"
@@ -52,7 +53,23 @@ app.proxy = true
 app.use(handleScimBody)
 app.use(koaBody({ multipart: true }))
 
-app.use(koaSession(app))
+const sessionMiddleware: Middleware = async (ctx: any, next: any) => {
+  const redisClient = await new redis.Client(
+    redis.utils.Databases.SESSIONS
+  ).init()
+  return koaSession(
+    {
+      // @ts-ignore
+      store: new RedisStore({ client: redisClient.getClient() }),
+      key: "koa:sess",
+      maxAge: 86400000, // one day
+    },
+    app
+  )(ctx, next)
+}
+
+app.use(sessionMiddleware)
+
 app.use(middleware.correlation)
 app.use(middleware.pino)
 app.use(middleware.ip)

packages/worker/src/koa-redis.d.ts

@@ -0,0 +1 @@
+declare module "koa-redis" {}

yarn.lock

@@ -2695,6 +2695,13 @@
   dependencies:
     regenerator-runtime "^0.14.0"
 
+"@babel/runtime@^7.8.3":
+  version "7.26.9"
+  resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.26.9.tgz#aa4c6facc65b9cb3f87d75125ffd47781b475433"
+  integrity sha512-aA63XwOkcl4xxQa3HjPMqOP6LiK0ZDv3mUPYEFXkpHbaFjtGggE1A61FjFzJnB+p7/oy2gA8E+rcBNl/zC1tMg==
+  dependencies:
+    regenerator-runtime "^0.14.0"
+
 "@babel/template@^7.22.15", "@babel/template@^7.22.5", "@babel/template@^7.25.9", "@babel/template@^7.3.3":
   version "7.25.9"
   resolved "https://registry.yarnpkg.com/@babel/template/-/template-7.25.9.tgz#ecb62d81a8a6f5dc5fe8abfc3901fc52ddf15016"
@@ -9041,7 +9048,14 @@ co-body@^5.1.1:
     raw-body "^2.2.0"
     type-is "^1.6.14"
 
-co@^4.6.0:
+co-wrap-all@^1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/co-wrap-all/-/co-wrap-all-1.0.0.tgz#370ae3e8333510a53f6b2f7fdfbe4568a11b7ecf"
+  integrity sha512-aru6gLi2vTUazr+MxVm3Rv6ST7/EKtFj9BrfkcOrbCO2Qv6LqJdE71m88HhHiBEviKw/ucVrwoGLrq2xHpOsJA==
+  dependencies:
+    co "^4.0.0"
+
+co@^4.0.0, co@^4.6.0:
   version "4.6.0"
   resolved "https://registry.yarnpkg.com/co/-/co-4.6.0.tgz#6ea6bdf3d853ae54ccb8e47bfa0bf3f9031fb184"
   integrity sha512-QVb0dM5HvG+uaxitm8wONl7jltx8dqhfU33DcqtOZcLSVIKSDDLDi7+0LbAKiyI8hD9u42m2YxXSkMGWThaecQ==
@@ -13177,7 +13191,7 @@ ioredis@5.3.2:
     redis-parser "^3.0.0"
     standard-as-callback "^2.1.0"
 
-ioredis@^4.28.5:
+ioredis@^4.14.1, ioredis@^4.28.5:
   version "4.28.5"
   resolved "https://registry.yarnpkg.com/ioredis/-/ioredis-4.28.5.tgz#5c149e6a8d76a7f8fa8a504ffc85b7d5b6797f9f"
   integrity sha512-3GYo0GJtLqgNXj4YhrisLaNNvWSNwSS2wS4OELGfGxH8I69+XfNdnmV1AyN+ZqMh0i7eX+SWjrwFKDBDgfBC1A==
@@ -14677,6 +14691,16 @@ koa-pino-logger@4.0.0:
   dependencies:
     pino-http "^6.5.0"
 
+koa-redis@^4.0.1:
+  version "4.0.1"
+  resolved "https://registry.yarnpkg.com/koa-redis/-/koa-redis-4.0.1.tgz#57ac1b46d9ab851221a9f4952c1e8d4bf289db40"
+  integrity sha512-o2eTVNo1NBnloeUGhHed5Q2ZvJSLpUEj/+E1/7oH5EmH8WuQ+QLdl/VawkshxdFQ47W1p6V09lM3hCTu7D0YnQ==
+  dependencies:
+    "@babel/runtime" "^7.8.3"
+    co-wrap-all "^1.0.0"
+    debug "^4.1.1"
+    ioredis "^4.14.1"
+
 koa-router@^10.0.0:
   version "10.1.1"
   resolved "https://registry.yarnpkg.com/koa-router/-/koa-router-10.1.1.tgz#20809f82648518b84726cd445037813cd99f17ff"