From 0f170b6601f838eba7df5b877a54b85e377a2b59 Mon Sep 17 00:00:00 2001 From: jvcalderon Date: Wed, 27 Mar 2024 11:32:08 +0100 Subject: [PATCH 02/31] Update submodule --- packages/account-portal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/account-portal b/packages/account-portal index f5b467b6b1..eb5315adb5 160000 --- a/packages/account-portal +++ b/packages/account-portal @@ -1 +1 @@ -Subproject commit f5b467b6b1c55c48847545db41be7b1c035e167a +Subproject commit eb5315adb5c0023a340d8e1c1009ec667566ce75 From 5939a3f4f1bf65d6f11438db039f0ea33e8dda63 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 27 Mar 2024 12:15:30 +0100 Subject: [PATCH 03/31] Ignore merge conflicts on pro submodule checks --- .github/workflows/budibase_ci.yml | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml index 5a0d09033a..62db7e3826 100644 --- a/.github/workflows/budibase_ci.yml +++ b/.github/workflows/budibase_ci.yml @@ -231,27 +231,28 @@ jobs: echo "pro_commit=$pro_commit" echo "pro_commit=$pro_commit" >> "$GITHUB_OUTPUT" echo "base_commit=$base_commit" - echo "base_commit=$base_commit" >> "$GITHUB_OUTPUT" + + base_commit_excluding_merges=$(git log --no-merges -n 1 --format=format:%H $base_commit) + echo "base_commit_excluding_merges=$base_commit_excluding_merges" + echo "base_commit_excluding_merges=$base_commit_excluding_merges" >> "$GITHUB_OUTPUT" else echo "Nothing to do - branch to branch merge." fi - name: Check submodule merged to base branch - if: ${{ steps.get_pro_commits.outputs.base_commit != '' }} - uses: actions/github-script@v7 - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - script: | - const submoduleCommit = '${{ steps.get_pro_commits.outputs.pro_commit }}'; - const baseCommit = '${{ steps.get_pro_commits.outputs.base_commit }}'; + if: ${{ steps.get_pro_commits.outputs.base_commit_excluding_merges != '' }} + run: | + base_commit_excluding_merges=${{ steps.get_pro_commits.outputs.base_commit_excluding_merges }}' + any_commit=$(git log origin/master...$base_commit_excluding_merges) + if [ -n "$any_commit" ]; then + echo "An error occurred: " + echo 'Submodule commit does not match the latest commit on the "${{ steps.get_pro_commits.outputs.target_branch }}" branch.' + echo 'Refer to the pro repo to merge your changes: https://github.com/Budibase/budibase-pro/blob/master/docs/getting_started.md' - if (submoduleCommit !== baseCommit) { - console.error('Submodule commit does not match the latest commit on the "${{ steps.get_pro_commits.outputs.target_branch }}" branch.'); - console.error('Refer to the pro repo to merge your changes: https://github.com/Budibase/budibase-pro/blob/master/docs/getting_started.md') - process.exit(1); - } else { - console.log('All good, the submodule had been merged and setup correctly!') - } + exit 1 + else + echo 'All good, the submodule had been merged and setup correctly!' + fi check-accountportal-submodule: runs-on: ubuntu-latest From 02e2d26985c272b9f6fe30c8edb18a9d5a4ec751 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 27 Mar 2024 12:20:07 +0100 Subject: [PATCH 04/31] Fix --- .github/workflows/budibase_ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml index 62db7e3826..9164e9974f 100644 --- a/.github/workflows/budibase_ci.yml +++ b/.github/workflows/budibase_ci.yml @@ -242,7 +242,7 @@ jobs: - name: Check submodule merged to base branch if: ${{ steps.get_pro_commits.outputs.base_commit_excluding_merges != '' }} run: | - base_commit_excluding_merges=${{ steps.get_pro_commits.outputs.base_commit_excluding_merges }}' + base_commit_excluding_merges='${{ steps.get_pro_commits.outputs.base_commit_excluding_merges }}' any_commit=$(git log origin/master...$base_commit_excluding_merges) if [ -n "$any_commit" ]; then echo "An error occurred: " From 4fb62177305048b2652f9b398f63f82aa56d219f Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 27 Mar 2024 12:24:16 +0100 Subject: [PATCH 05/31] Fix checks --- .github/workflows/budibase_ci.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml index 9164e9974f..ade611a08c 100644 --- a/.github/workflows/budibase_ci.yml +++ b/.github/workflows/budibase_ci.yml @@ -243,8 +243,13 @@ jobs: if: ${{ steps.get_pro_commits.outputs.base_commit_excluding_merges != '' }} run: | base_commit_excluding_merges='${{ steps.get_pro_commits.outputs.base_commit_excluding_merges }}' - any_commit=$(git log origin/master...$base_commit_excluding_merges) + pro_commit='${{ steps.get_pro_commits.outputs.pro_commit }}' + + any_commit=$(git log $base_commit_excluding_merges...$pro_commit) + if [ -n "$any_commit" ]; then + echo $any_commit + echo "An error occurred: " echo 'Submodule commit does not match the latest commit on the "${{ steps.get_pro_commits.outputs.target_branch }}" branch.' echo 'Refer to the pro repo to merge your changes: https://github.com/Budibase/budibase-pro/blob/master/docs/getting_started.md' From c255c70954346d8e35b87c678ffaaac859582038 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 27 Mar 2024 12:25:37 +0100 Subject: [PATCH 06/31] cd --- .github/workflows/budibase_ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml index ade611a08c..4c1110fb08 100644 --- a/.github/workflows/budibase_ci.yml +++ b/.github/workflows/budibase_ci.yml @@ -242,6 +242,7 @@ jobs: - name: Check submodule merged to base branch if: ${{ steps.get_pro_commits.outputs.base_commit_excluding_merges != '' }} run: | + cd packages/pro base_commit_excluding_merges='${{ steps.get_pro_commits.outputs.base_commit_excluding_merges }}' pro_commit='${{ steps.get_pro_commits.outputs.pro_commit }}' From 89d9b787df98ba521d37758d33015902b3a47143 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 27 Mar 2024 12:26:10 +0100 Subject: [PATCH 07/31] Rename --- .github/workflows/budibase_ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml index 4c1110fb08..aef88a30d6 100644 --- a/.github/workflows/budibase_ci.yml +++ b/.github/workflows/budibase_ci.yml @@ -239,7 +239,7 @@ jobs: echo "Nothing to do - branch to branch merge." fi - - name: Check submodule merged to base branch + - name: Check submodule merged and latest on base branch if: ${{ steps.get_pro_commits.outputs.base_commit_excluding_merges != '' }} run: | cd packages/pro From 1ed687f6c483728c245afa8232da399be81d5b56 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 27 Mar 2024 12:37:07 +0100 Subject: [PATCH 08/31] Fix checks --- .github/workflows/budibase_ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml index aef88a30d6..b8bd6063ef 100644 --- a/.github/workflows/budibase_ci.yml +++ b/.github/workflows/budibase_ci.yml @@ -246,7 +246,7 @@ jobs: base_commit_excluding_merges='${{ steps.get_pro_commits.outputs.base_commit_excluding_merges }}' pro_commit='${{ steps.get_pro_commits.outputs.pro_commit }}' - any_commit=$(git log $base_commit_excluding_merges...$pro_commit) + any_commit=$(git log --no-merges $base_commit_excluding_merges...$pro_commit) if [ -n "$any_commit" ]; then echo $any_commit From 4493905e74302ac4116b2e6c2858b722482c4323 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 27 Mar 2024 12:39:20 +0100 Subject: [PATCH 09/31] Update submodule --- packages/pro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pro b/packages/pro index f8e8f87bd5..8a10ac52a4 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit f8e8f87bd52081e1303a5ae92c432ea5b38f3bb4 +Subproject commit 8a10ac52a4fdb6eb1f444aa0a308fd28ede526ec From ab46a3e5e8e9b20d956b56ac539b0a6c2301f3c0 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 27 Mar 2024 12:41:05 +0100 Subject: [PATCH 10/31] Update submodule --- packages/pro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pro b/packages/pro index 8a10ac52a4..6b62505be0 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 8a10ac52a4fdb6eb1f444aa0a308fd28ede526ec +Subproject commit 6b62505be0c0b50a57b4f4980d86541ebdc86428 From 1e7608a7908030343d6bf634ba3f98840b2327ea Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 27 Mar 2024 11:46:45 +0000 Subject: [PATCH 11/31] Trying yarn 1.22.19. --- .github/workflows/budibase_ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml index 5a0d09033a..bab8a689b1 100644 --- a/.github/workflows/budibase_ci.yml +++ b/.github/workflows/budibase_ci.yml @@ -43,6 +43,7 @@ jobs: with: node-version: 20.x cache: yarn + - run: yarn set version 1.22.19 - run: yarn --frozen-lockfile - run: yarn lint From 4fa98f17d2a6b9140b00e79f0277f53f0245b41f Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 27 Mar 2024 11:55:22 +0000 Subject: [PATCH 12/31] Different way to set yarn version. --- .github/workflows/budibase_ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml index bab8a689b1..2898d06682 100644 --- a/.github/workflows/budibase_ci.yml +++ b/.github/workflows/budibase_ci.yml @@ -43,7 +43,7 @@ jobs: with: node-version: 20.x cache: yarn - - run: yarn set version 1.22.19 + - run: yarn policies set-version 1.22.19 - run: yarn --frozen-lockfile - run: yarn lint From e36ec141637b12513b50e4e17cc526cbc3b32760 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 27 Mar 2024 12:05:44 +0000 Subject: [PATCH 13/31] Use yarn 1.22.19 across all CI steps. --- .github/workflows/budibase_ci.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml index 2898d06682..d1d0ea214e 100644 --- a/.github/workflows/budibase_ci.yml +++ b/.github/workflows/budibase_ci.yml @@ -43,7 +43,7 @@ jobs: with: node-version: 20.x cache: yarn - - run: yarn policies set-version 1.22.19 + - run: yarn set version 1.22.19 - run: yarn --frozen-lockfile - run: yarn lint @@ -62,6 +62,7 @@ jobs: with: node-version: 20.x cache: yarn + - run: yarn set version 1.22.19 - run: yarn --frozen-lockfile # Run build all the projects @@ -104,6 +105,7 @@ jobs: with: node-version: 20.x cache: yarn + - run: yarn set version 1.22.19 - run: yarn --frozen-lockfile - name: Test run: | @@ -128,6 +130,7 @@ jobs: with: node-version: 20.x cache: yarn + - run: yarn set version 1.22.19 - run: yarn --frozen-lockfile - name: Test worker run: | @@ -165,6 +168,7 @@ jobs: docker pull testcontainers/ryuk:0.5.1 docker pull budibase/couchdb + - run: yarn set version 1.22.19 - run: yarn --frozen-lockfile - name: Test server @@ -189,6 +193,7 @@ jobs: with: node-version: 20.x cache: yarn + - run: yarn set version 1.22.19 - run: yarn --frozen-lockfile - name: Build packages run: yarn build --scope @budibase/server --scope @budibase/worker From 36fa7b75deb0b5807f4e74b3d77df2c5ba185687 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 27 Mar 2024 12:24:03 +0000 Subject: [PATCH 14/31] Removing yarn version set - made no difference. --- .github/workflows/budibase_ci.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml index d1d0ea214e..5a0d09033a 100644 --- a/.github/workflows/budibase_ci.yml +++ b/.github/workflows/budibase_ci.yml @@ -43,7 +43,6 @@ jobs: with: node-version: 20.x cache: yarn - - run: yarn set version 1.22.19 - run: yarn --frozen-lockfile - run: yarn lint @@ -62,7 +61,6 @@ jobs: with: node-version: 20.x cache: yarn - - run: yarn set version 1.22.19 - run: yarn --frozen-lockfile # Run build all the projects @@ -105,7 +103,6 @@ jobs: with: node-version: 20.x cache: yarn - - run: yarn set version 1.22.19 - run: yarn --frozen-lockfile - name: Test run: | @@ -130,7 +127,6 @@ jobs: with: node-version: 20.x cache: yarn - - run: yarn set version 1.22.19 - run: yarn --frozen-lockfile - name: Test worker run: | @@ -168,7 +164,6 @@ jobs: docker pull testcontainers/ryuk:0.5.1 docker pull budibase/couchdb - - run: yarn set version 1.22.19 - run: yarn --frozen-lockfile - name: Test server @@ -193,7 +188,6 @@ jobs: with: node-version: 20.x cache: yarn - - run: yarn set version 1.22.19 - run: yarn --frozen-lockfile - name: Build packages run: yarn build --scope @budibase/server --scope @budibase/worker From 81ef8033ddfb231baf8930da28a848140f3b5b7b Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 27 Mar 2024 12:41:17 +0000 Subject: [PATCH 15/31] Trying an earlier account portal reference. --- packages/account-portal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/account-portal b/packages/account-portal index eb5315adb5..f46a310b81 160000 --- a/packages/account-portal +++ b/packages/account-portal @@ -1 +1 @@ -Subproject commit eb5315adb5c0023a340d8e1c1009ec667566ce75 +Subproject commit f46a310b813d32931ecf3e03235e75ab881edfc1 From 2427936b70100ee72766ebb1de74c6512fad43f8 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 27 Mar 2024 12:59:29 +0000 Subject: [PATCH 16/31] Build in CI in two steps, one for account portal and one for OSS/monorepo. --- .github/workflows/budibase_ci.yml | 3 ++- package.json | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml index 5a0d09033a..93be55f059 100644 --- a/.github/workflows/budibase_ci.yml +++ b/.github/workflows/budibase_ci.yml @@ -66,7 +66,8 @@ jobs: # Run build all the projects - name: Build run: | - yarn build + yarn build:oss + yarn build:account-portal # Check the types of the projects built via esbuild - name: Check types run: | diff --git a/package.json b/package.json index 79a7b06eff..6b7db65ed8 100644 --- a/package.json +++ b/package.json @@ -33,7 +33,9 @@ "preinstall": "node scripts/syncProPackage.js", "get-past-client-version": "node scripts/getPastClientVersion.js", "setup": "git config submodule.recurse true && git submodule update && node ./hosting/scripts/setup.js && yarn && yarn build && yarn dev", - "build": "NODE_OPTIONS=--max-old-space-size=1500 lerna run build --stream", + "build": "NODE_OPTIONS=--max-old-space-size=2000 lerna run build --stream", + "build:oss": "NODE_OPTIONS=--max-old-space-size=1500 lerna run build --stream --ignore @budibase/account-portal-ui --ignore @budibase/account-portal-server", + "build:account-portal": "NODE_OPTIONS=--max-old-space-size=1500 lerna run build --stream --scope @budibase/account-portal-ui --scope @budibase/account-portal-server", "build:dev": "lerna run --stream prebuild && yarn nx run-many --target=build --output-style=dynamic --watch --preserveWatchOutput", "check:types": "lerna run check:types", "build:sdk": "lerna run --stream build:sdk", From 84516e85a19d653f321b7cd93b8e7d9e82168b0a Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 27 Mar 2024 13:00:02 +0000 Subject: [PATCH 17/31] Updating to latest account portal master. --- packages/account-portal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/account-portal b/packages/account-portal index f46a310b81..eb5315adb5 160000 --- a/packages/account-portal +++ b/packages/account-portal @@ -1 +1 @@ -Subproject commit f46a310b813d32931ecf3e03235e75ab881edfc1 +Subproject commit eb5315adb5c0023a340d8e1c1009ec667566ce75 From 77621f87b6d0779a670a5f49264c9e4ecd2c031b Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 27 Mar 2024 13:15:02 +0000 Subject: [PATCH 18/31] Adding missing package from ignore. --- package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index 6b7db65ed8..8d7a372da4 100644 --- a/package.json +++ b/package.json @@ -34,8 +34,8 @@ "get-past-client-version": "node scripts/getPastClientVersion.js", "setup": "git config submodule.recurse true && git submodule update && node ./hosting/scripts/setup.js && yarn && yarn build && yarn dev", "build": "NODE_OPTIONS=--max-old-space-size=2000 lerna run build --stream", - "build:oss": "NODE_OPTIONS=--max-old-space-size=1500 lerna run build --stream --ignore @budibase/account-portal-ui --ignore @budibase/account-portal-server", - "build:account-portal": "NODE_OPTIONS=--max-old-space-size=1500 lerna run build --stream --scope @budibase/account-portal-ui --scope @budibase/account-portal-server", + "build:oss": "NODE_OPTIONS=--max-old-space-size=1500 lerna run build --stream --ignore @budibase/account-portal-ui --ignore @budibase/account-portal-server --ignore @budibase/account-portal", + "build:account-portal": "NODE_OPTIONS=--max-old-space-size=1500 lerna run build --stream --scope @budibase/account-portal-ui --scope @budibase/account-portal-server --scope @budibase/account-portal", "build:dev": "lerna run --stream prebuild && yarn nx run-many --target=build --output-style=dynamic --watch --preserveWatchOutput", "check:types": "lerna run check:types", "build:sdk": "lerna run --stream build:sdk", From cd694cb31237f9969af23079d2657d3c9ee774cd Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 27 Mar 2024 13:32:24 +0000 Subject: [PATCH 19/31] Disabling warnings for eval (required). --- packages/bbui/rollup.config.js | 7 +++++++ packages/client/rollup.config.js | 3 ++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/packages/bbui/rollup.config.js b/packages/bbui/rollup.config.js index e285d548d6..da274e0ba5 100644 --- a/packages/bbui/rollup.config.js +++ b/packages/bbui/rollup.config.js @@ -12,6 +12,13 @@ export default { format: "esm", file: "dist/bbui.es.js", }, + onwarn(warning, warn) { + // suppress eval warnings + if (warning.code === "EVAL") { + return + } + warn(warning) + }, plugins: [ resolve(), commonjs(), diff --git a/packages/client/rollup.config.js b/packages/client/rollup.config.js index ee839424ee..9839414f7e 100644 --- a/packages/client/rollup.config.js +++ b/packages/client/rollup.config.js @@ -45,7 +45,8 @@ export default { onwarn(warning, warn) { if ( warning.code === "THIS_IS_UNDEFINED" || - warning.code === "CIRCULAR_DEPENDENCY" + warning.code === "CIRCULAR_DEPENDENCY" || + warning.code === "EVAL" ) { return } From 936762c8ee992c1f57ad41fd3c77d6e0fb127e17 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 27 Mar 2024 13:35:47 +0000 Subject: [PATCH 20/31] Removing all eval warnings from rollup. --- packages/string-templates/rollup.config.js | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/packages/string-templates/rollup.config.js b/packages/string-templates/rollup.config.js index a843286a82..ee02c7a14a 100644 --- a/packages/string-templates/rollup.config.js +++ b/packages/string-templates/rollup.config.js @@ -17,6 +17,12 @@ const config = (format, outputFile) => ({ format, file: outputFile, }, + onwarn(warning, warn) { + if (warning.code === "EVAL") { + return + } + warn(warning) + }, plugins: [ typescript(), resolve({ From 4d6ceb142ac61545ae9b972ff6ed238cfd28af45 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 27 Mar 2024 14:00:43 +0000 Subject: [PATCH 21/31] Updating account portal reference. --- packages/account-portal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/account-portal b/packages/account-portal index eb5315adb5..60658a052d 160000 --- a/packages/account-portal +++ b/packages/account-portal @@ -1 +1 @@ -Subproject commit eb5315adb5c0023a340d8e1c1009ec667566ce75 +Subproject commit 60658a052d2642e5f4a8038e253f771a24f34907 From e4d2ac4fc2abaaf0a43c762a70ebec2a5fd3df36 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 27 Mar 2024 14:25:14 +0000 Subject: [PATCH 22/31] Changing build back. --- .github/workflows/budibase_ci.yml | 4 +--- package.json | 4 +--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml index 93be55f059..c3f2cf7953 100644 --- a/.github/workflows/budibase_ci.yml +++ b/.github/workflows/budibase_ci.yml @@ -65,9 +65,7 @@ jobs: # Run build all the projects - name: Build - run: | - yarn build:oss - yarn build:account-portal + run: yarn build # Check the types of the projects built via esbuild - name: Check types run: | diff --git a/package.json b/package.json index 8d7a372da4..79a7b06eff 100644 --- a/package.json +++ b/package.json @@ -33,9 +33,7 @@ "preinstall": "node scripts/syncProPackage.js", "get-past-client-version": "node scripts/getPastClientVersion.js", "setup": "git config submodule.recurse true && git submodule update && node ./hosting/scripts/setup.js && yarn && yarn build && yarn dev", - "build": "NODE_OPTIONS=--max-old-space-size=2000 lerna run build --stream", - "build:oss": "NODE_OPTIONS=--max-old-space-size=1500 lerna run build --stream --ignore @budibase/account-portal-ui --ignore @budibase/account-portal-server --ignore @budibase/account-portal", - "build:account-portal": "NODE_OPTIONS=--max-old-space-size=1500 lerna run build --stream --scope @budibase/account-portal-ui --scope @budibase/account-portal-server --scope @budibase/account-portal", + "build": "NODE_OPTIONS=--max-old-space-size=1500 lerna run build --stream", "build:dev": "lerna run --stream prebuild && yarn nx run-many --target=build --output-style=dynamic --watch --preserveWatchOutput", "check:types": "lerna run check:types", "build:sdk": "lerna run --stream build:sdk", From 9f80d7fe435b259b50934431fa29872781fdb5f2 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 27 Mar 2024 16:59:45 +0000 Subject: [PATCH 23/31] Adding a mechanism to split build of account portal and monorepo. --- package.json | 2 ++ 1 file changed, 2 insertions(+) diff --git a/package.json b/package.json index 79a7b06eff..32693a0b6f 100644 --- a/package.json +++ b/package.json @@ -34,6 +34,8 @@ "get-past-client-version": "node scripts/getPastClientVersion.js", "setup": "git config submodule.recurse true && git submodule update && node ./hosting/scripts/setup.js && yarn && yarn build && yarn dev", "build": "NODE_OPTIONS=--max-old-space-size=1500 lerna run build --stream", + "build:oss": "NODE_OPTIONS=--max-old-space-size=1500 lerna run build --stream --ignore @budibase/account-portal --ignore @budibase/account-portal-server --ignore @budibase/account-portal-ui", + "build:account-portal": "NODE_OPTIONS=--max-old-space-size=1500 lerna run build --stream --scope @budibase/account-portal --scope @budibase/account-portal-server --scope @budibase/account-portal-ui", "build:dev": "lerna run --stream prebuild && yarn nx run-many --target=build --output-style=dynamic --watch --preserveWatchOutput", "check:types": "lerna run check:types", "build:sdk": "lerna run --stream build:sdk", From 84f87c4a48e17be36fc1ee866c57a93e4a09f80a Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 27 Mar 2024 17:00:56 +0000 Subject: [PATCH 24/31] Updating CI. --- .github/workflows/budibase_ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml index a9cb89d4e6..030ad6578e 100644 --- a/.github/workflows/budibase_ci.yml +++ b/.github/workflows/budibase_ci.yml @@ -65,7 +65,9 @@ jobs: # Run build all the projects - name: Build - run: yarn build + run: | + yarn build:oss + yarn build:account-portal # Check the types of the projects built via esbuild - name: Check types run: | From b77106480e2dba9dff5cf3b818c494a9ee8d2522 Mon Sep 17 00:00:00 2001 From: jvcalderon Date: Thu, 28 Mar 2024 11:46:58 +0100 Subject: [PATCH 25/31] Adds POST /api/global/users/sso endpoint --- packages/backend-core/src/platform/users.ts | 17 ++++++-- .../tests/core/utilities/structures/users.ts | 1 + packages/types/src/api/web/user.ts | 5 +++ .../src/api/controllers/global/users.ts | 19 +++++++++ packages/worker/src/api/index.ts | 9 +++++ .../src/api/routes/global/tests/users.spec.ts | 40 +++++++++++++++++++ .../worker/src/api/routes/global/users.ts | 6 +++ .../worker/src/api/routes/validation/users.ts | 9 +++++ packages/worker/src/tests/api/users.ts | 14 +++++++ 9 files changed, 116 insertions(+), 4 deletions(-) diff --git a/packages/backend-core/src/platform/users.ts b/packages/backend-core/src/platform/users.ts index 6f030afb7c..ccaad76b19 100644 --- a/packages/backend-core/src/platform/users.ts +++ b/packages/backend-core/src/platform/users.ts @@ -20,7 +20,7 @@ export async function lookupTenantId(userId: string) { return user.tenantId } -async function getUserDoc(emailOrId: string): Promise { +export async function getUserDoc(emailOrId: string): Promise { const db = getPlatformDB() return db.get(emailOrId) } @@ -79,6 +79,17 @@ async function addUserDoc(emailOrId: string, newDocFn: () => PlatformUser) { } } +export async function addSsoUser( + ssoId: string, + email: string, + userId: string, + tenantId: string +) { + return addUserDoc(ssoId, () => + newUserSsoIdDoc(ssoId, email, userId, tenantId) + ) +} + export async function addUser( tenantId: string, userId: string, @@ -91,9 +102,7 @@ export async function addUser( ] if (ssoId) { - promises.push( - addUserDoc(ssoId, () => newUserSsoIdDoc(ssoId, email, userId, tenantId)) - ) + promises.push(addSsoUser(ssoId, email, userId, tenantId)) } await Promise.all(promises) diff --git a/packages/backend-core/tests/core/utilities/structures/users.ts b/packages/backend-core/tests/core/utilities/structures/users.ts index db90887af2..88fb02617f 100644 --- a/packages/backend-core/tests/core/utilities/structures/users.ts +++ b/packages/backend-core/tests/core/utilities/structures/users.ts @@ -86,6 +86,7 @@ export function ssoUser( oauth2: opts.details?.oauth2, provider: opts.details?.provider!, providerType: opts.details?.providerType!, + ssoId: opts.details?.userId || uuid(), thirdPartyProfile: { email: base.email, picture: base.pictureUrl, diff --git a/packages/types/src/api/web/user.ts b/packages/types/src/api/web/user.ts index d68d687dcb..0ef7493016 100644 --- a/packages/types/src/api/web/user.ts +++ b/packages/types/src/api/web/user.ts @@ -68,6 +68,11 @@ export interface CreateAdminUserRequest { ssoId?: string } +export interface AddSSoUserRequest { + ssoId: string + email: string +} + export interface CreateAdminUserResponse { _id: string _rev: string diff --git a/packages/worker/src/api/controllers/global/users.ts b/packages/worker/src/api/controllers/global/users.ts index 6b9e533f78..4e59873e33 100644 --- a/packages/worker/src/api/controllers/global/users.ts +++ b/packages/worker/src/api/controllers/global/users.ts @@ -3,6 +3,7 @@ import env from "../../../environment" import { AcceptUserInviteRequest, AcceptUserInviteResponse, + AddSSoUserRequest, BulkUserRequest, BulkUserResponse, CloudAccount, @@ -15,6 +16,7 @@ import { LockName, LockType, MigrationType, + PlatformUserByEmail, SaveUserResponse, SearchUsersRequest, User, @@ -53,6 +55,23 @@ export const save = async (ctx: UserCtx) => { } } +export const addSsoSupport = async (ctx: Ctx) => { + const { email, ssoId } = ctx.request.body + try { + // Status is changed to 404 from getUserDoc if user is not found + let userByEmail = (await platform.users.getUserDoc(email)) as PlatformUserByEmail + await platform.users.addSsoUser( + ssoId, + email, + userByEmail.userId, + userByEmail.tenantId + ) + ctx.status = 200 + } catch (err: any) { + ctx.throw(err.status || 400, err) + } +} + const bulkDelete = async (userIds: string[], currentUserId: string) => { if (userIds?.indexOf(currentUserId) !== -1) { throw new Error("Unable to delete self.") diff --git a/packages/worker/src/api/index.ts b/packages/worker/src/api/index.ts index d7aef0b274..4936c104e1 100644 --- a/packages/worker/src/api/index.ts +++ b/packages/worker/src/api/index.ts @@ -41,6 +41,10 @@ const PUBLIC_ENDPOINTS = [ route: "/api/global/users/init", method: "POST", }, + { + route: "/api/global/users/sso", + method: "POST", + }, { route: "/api/global/users/invite/accept", method: "POST", @@ -81,6 +85,11 @@ const NO_TENANCY_ENDPOINTS = [ route: "/api/global/users/init", method: "POST", }, + // tenant is retrieved from the user found by the requested email + { + route: "/api/global/users/sso", + method: "POST", + }, // deprecated single tenant sso callback { route: "/api/admin/auth/google/callback", diff --git a/packages/worker/src/api/routes/global/tests/users.spec.ts b/packages/worker/src/api/routes/global/tests/users.spec.ts index 37f5721881..6923a7839b 100644 --- a/packages/worker/src/api/routes/global/tests/users.spec.ts +++ b/packages/worker/src/api/routes/global/tests/users.spec.ts @@ -520,10 +520,50 @@ describe("/api/global/users", () => { }) } + function createPasswordUser() { + return config.doInTenant(() => { + const user = structures.users.user() + return userSdk.db.save(user) + }) + } + it("should be able to update an sso user that has no password", async () => { const user = await createSSOUser() await config.api.users.saveUser(user) }) + + it("sso support couldn't be used by admin. It is cloud restricted and needs internal key", async () => { + const user = await config.createUser() + const ssoId = "fake-ssoId" + await config.api.users + .addSsoSupportDefaultAuth(ssoId, user.email) + .expect("Content-Type", /json/) + .expect(403) + }) + + it("if user email doesn't exist, SSO support couldn't be added. Not found error returned", async () => { + const ssoId = "fake-ssoId" + const email = "fake-email@budibase.com" + await config.api.users + .addSsoSupportInternalAPIAuth(ssoId, email) + .expect("Content-Type", /json/) + .expect(404) + }) + + it("if user email exist, SSO support is added", async () => { + const user = await createPasswordUser() + const ssoId = "fakessoId" + await config.api.users + .addSsoSupportInternalAPIAuth(ssoId, user.email) + .expect(200) + }) + + it("if user ssoId is already assigned, no change will be applied", async () => { + const user = await createSSOUser() + await config.api.users + .addSsoSupportInternalAPIAuth(user.ssoId, user.email) + .expect(200) + }) }) }) diff --git a/packages/worker/src/api/routes/global/users.ts b/packages/worker/src/api/routes/global/users.ts index 6b9291b88b..e7c77678fc 100644 --- a/packages/worker/src/api/routes/global/users.ts +++ b/packages/worker/src/api/routes/global/users.ts @@ -65,6 +65,12 @@ router users.buildUserSaveValidation(), controller.save ) + .post( + "/api/global/users/sso", + cloudRestricted, + users.buildAddSsoSupport(), + controller.addSsoSupport + ) .post( "/api/global/users/bulk", auth.adminOnly, diff --git a/packages/worker/src/api/routes/validation/users.ts b/packages/worker/src/api/routes/validation/users.ts index fbc85af2d3..cbd7567457 100644 --- a/packages/worker/src/api/routes/validation/users.ts +++ b/packages/worker/src/api/routes/validation/users.ts @@ -41,6 +41,15 @@ export const buildUserSaveValidation = () => { return auth.joiValidator.body(Joi.object(schema).required().unknown(true)) } +export const buildAddSsoSupport = () => { + return auth.joiValidator.body( + Joi.object({ + ssoId: Joi.string().required(), + email: Joi.string().required(), + }).required() + ) +} + export const buildUserBulkUserValidation = (isSelf = false) => { if (!isSelf) { schema = { diff --git a/packages/worker/src/tests/api/users.ts b/packages/worker/src/tests/api/users.ts index 45105c99da..d08a4ef8c7 100644 --- a/packages/worker/src/tests/api/users.ts +++ b/packages/worker/src/tests/api/users.ts @@ -127,6 +127,20 @@ export class UserAPI extends TestAPI { .expect(status ? status : 200) } + addSsoSupportInternalAPIAuth = (ssoId: string, email: string) => { + return this.request + .post(`/api/global/users/sso`) + .send({ ssoId, email }) + .set(this.config.internalAPIHeaders()) + } + + addSsoSupportDefaultAuth = (ssoId: string, email: string) => { + return this.request + .post(`/api/global/users/sso`) + .send({ ssoId, email }) + .set(this.config.defaultHeaders()) + } + deleteUser = (userId: string, status?: number) => { return this.request .delete(`/api/global/users/${userId}`) From af2407e8cdeaee721e14c6a64a0b1ae0f5b107a8 Mon Sep 17 00:00:00 2001 From: jvcalderon Date: Thu, 28 Mar 2024 12:04:27 +0100 Subject: [PATCH 26/31] Lint --- packages/worker/src/api/controllers/global/users.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/worker/src/api/controllers/global/users.ts b/packages/worker/src/api/controllers/global/users.ts index 4e59873e33..93f35b4c37 100644 --- a/packages/worker/src/api/controllers/global/users.ts +++ b/packages/worker/src/api/controllers/global/users.ts @@ -59,7 +59,9 @@ export const addSsoSupport = async (ctx: Ctx) => { const { email, ssoId } = ctx.request.body try { // Status is changed to 404 from getUserDoc if user is not found - let userByEmail = (await platform.users.getUserDoc(email)) as PlatformUserByEmail + let userByEmail = (await platform.users.getUserDoc( + email + )) as PlatformUserByEmail await platform.users.addSsoUser( ssoId, email, From d8de0a31541820a855427f7e794769abb2ac7f47 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 28 Mar 2024 12:14:56 +0000 Subject: [PATCH 27/31] An issue which comes up relatively often is that localhost is not a valid option for a datasource in our cloud and our self host environments. Fixing this so it only shows this in development. --- packages/server/src/integrations/couchdb.ts | 3 ++- packages/server/src/integrations/elasticsearch.ts | 3 ++- packages/server/src/integrations/microsoftSqlServer.ts | 4 ++-- packages/server/src/integrations/mongodb.ts | 3 ++- packages/server/src/integrations/mysql.ts | 3 ++- packages/server/src/integrations/oracle.ts | 3 ++- packages/server/src/integrations/postgres.ts | 3 ++- packages/server/src/integrations/redis.ts | 3 ++- packages/server/src/integrations/utils.ts | 9 +++++++++ 9 files changed, 25 insertions(+), 9 deletions(-) diff --git a/packages/server/src/integrations/couchdb.ts b/packages/server/src/integrations/couchdb.ts index 079f646b60..c271fb12b2 100644 --- a/packages/server/src/integrations/couchdb.ts +++ b/packages/server/src/integrations/couchdb.ts @@ -9,6 +9,7 @@ import { QueryType, } from "@budibase/types" import { db as dbCore } from "@budibase/backend-core" +import { HOST_ADDRESS } from "./utils" interface CouchDBConfig { url: string @@ -28,7 +29,7 @@ const SCHEMA: Integration = { url: { type: DatasourceFieldType.STRING, required: true, - default: "http://localhost:5984", + default: `http://${HOST_ADDRESS}:5984`, }, database: { type: DatasourceFieldType.STRING, diff --git a/packages/server/src/integrations/elasticsearch.ts b/packages/server/src/integrations/elasticsearch.ts index 7ae0295298..af03baaef1 100644 --- a/packages/server/src/integrations/elasticsearch.ts +++ b/packages/server/src/integrations/elasticsearch.ts @@ -8,6 +8,7 @@ import { } from "@budibase/types" import { Client, ClientOptions } from "@elastic/elasticsearch" +import { HOST_ADDRESS } from "./utils" interface ElasticsearchConfig { url: string @@ -29,7 +30,7 @@ const SCHEMA: Integration = { url: { type: DatasourceFieldType.STRING, required: true, - default: "http://localhost:9200", + default: `http://${HOST_ADDRESS}:9200`, }, ssl: { type: DatasourceFieldType.BOOLEAN, diff --git a/packages/server/src/integrations/microsoftSqlServer.ts b/packages/server/src/integrations/microsoftSqlServer.ts index 831fcd3d20..8e8e84f533 100644 --- a/packages/server/src/integrations/microsoftSqlServer.ts +++ b/packages/server/src/integrations/microsoftSqlServer.ts @@ -22,6 +22,7 @@ import { finaliseExternalTables, SqlClient, checkExternalTables, + HOST_ADDRESS, } from "./utils" import Sql from "./base/sql" import { MSSQLTablesResponse, MSSQLColumn } from "./base/types" @@ -88,7 +89,6 @@ const SCHEMA: Integration = { user: { type: DatasourceFieldType.STRING, required: true, - default: "localhost", }, password: { type: DatasourceFieldType.PASSWORD, @@ -96,7 +96,7 @@ const SCHEMA: Integration = { }, server: { type: DatasourceFieldType.STRING, - default: "localhost", + default: HOST_ADDRESS, }, port: { type: DatasourceFieldType.NUMBER, diff --git a/packages/server/src/integrations/mongodb.ts b/packages/server/src/integrations/mongodb.ts index c9852e4c7a..dea752502d 100644 --- a/packages/server/src/integrations/mongodb.ts +++ b/packages/server/src/integrations/mongodb.ts @@ -22,6 +22,7 @@ import { InsertManyResult, } from "mongodb" import environment from "../environment" +import { HOST_ADDRESS } from "./utils" export interface MongoDBConfig { connectionString: string @@ -51,7 +52,7 @@ const getSchema = () => { connectionString: { type: DatasourceFieldType.STRING, required: true, - default: "mongodb://localhost:27017", + default: `mongodb://${HOST_ADDRESS}:27017`, display: "Connection string", }, db: { diff --git a/packages/server/src/integrations/mysql.ts b/packages/server/src/integrations/mysql.ts index db3819f468..fd9d57d255 100644 --- a/packages/server/src/integrations/mysql.ts +++ b/packages/server/src/integrations/mysql.ts @@ -21,6 +21,7 @@ import { generateColumnDefinition, finaliseExternalTables, checkExternalTables, + HOST_ADDRESS, } from "./utils" import dayjs from "dayjs" import { NUMBER_REGEX } from "../utilities" @@ -49,7 +50,7 @@ const SCHEMA: Integration = { datasource: { host: { type: DatasourceFieldType.STRING, - default: "localhost", + default: HOST_ADDRESS, required: true, }, port: { diff --git a/packages/server/src/integrations/oracle.ts b/packages/server/src/integrations/oracle.ts index 83803906be..9104aadbcc 100644 --- a/packages/server/src/integrations/oracle.ts +++ b/packages/server/src/integrations/oracle.ts @@ -22,6 +22,7 @@ import { finaliseExternalTables, getSqlQuery, SqlClient, + HOST_ADDRESS, } from "./utils" import Sql from "./base/sql" import { @@ -63,7 +64,7 @@ const SCHEMA: Integration = { datasource: { host: { type: DatasourceFieldType.STRING, - default: "localhost", + default: HOST_ADDRESS, required: true, }, port: { diff --git a/packages/server/src/integrations/postgres.ts b/packages/server/src/integrations/postgres.ts index 635d834761..05a519da64 100644 --- a/packages/server/src/integrations/postgres.ts +++ b/packages/server/src/integrations/postgres.ts @@ -21,6 +21,7 @@ import { finaliseExternalTables, SqlClient, checkExternalTables, + HOST_ADDRESS, } from "./utils" import Sql from "./base/sql" import { PostgresColumn } from "./base/types" @@ -72,7 +73,7 @@ const SCHEMA: Integration = { datasource: { host: { type: DatasourceFieldType.STRING, - default: "localhost", + default: HOST_ADDRESS, required: true, }, port: { diff --git a/packages/server/src/integrations/redis.ts b/packages/server/src/integrations/redis.ts index 6a6331ccd4..e127cddd56 100644 --- a/packages/server/src/integrations/redis.ts +++ b/packages/server/src/integrations/redis.ts @@ -6,6 +6,7 @@ import { QueryType, } from "@budibase/types" import Redis from "ioredis" +import { HOST_ADDRESS } from "./utils" interface RedisConfig { host: string @@ -28,7 +29,7 @@ const SCHEMA: Integration = { host: { type: DatasourceFieldType.STRING, required: true, - default: "localhost", + default: HOST_ADDRESS, }, port: { type: DatasourceFieldType.NUMBER, diff --git a/packages/server/src/integrations/utils.ts b/packages/server/src/integrations/utils.ts index f73f522357..1ba379da9e 100644 --- a/packages/server/src/integrations/utils.ts +++ b/packages/server/src/integrations/utils.ts @@ -13,6 +13,7 @@ import { DEFAULT_BB_DATASOURCE_ID, } from "../constants" import { helpers } from "@budibase/shared-core" +import env from "../environment" const DOUBLE_SEPARATOR = `${SEPARATOR}${SEPARATOR}` const ROW_ID_REGEX = /^\[.*]$/g @@ -92,6 +93,14 @@ export enum SqlClient { ORACLE = "oracledb", } +const isCloud = env.isProd() && !env.SELF_HOSTED +const isSelfHost = env.isProd() && env.SELF_HOSTED +export const HOST_ADDRESS = isSelfHost + ? "host.docker.internal" + : isCloud + ? "" + : "localhost" + export function isExternalTableID(tableId: string) { return tableId.includes(DocumentType.DATASOURCE) } From 32a5ce24ec7f20003e671ac2c8d99bde322d59fa Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Thu, 28 Mar 2024 15:46:43 +0000 Subject: [PATCH 28/31] remove automatic ssoId generation --- packages/backend-core/tests/core/utilities/structures/users.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/backend-core/tests/core/utilities/structures/users.ts b/packages/backend-core/tests/core/utilities/structures/users.ts index 88fb02617f..db90887af2 100644 --- a/packages/backend-core/tests/core/utilities/structures/users.ts +++ b/packages/backend-core/tests/core/utilities/structures/users.ts @@ -86,7 +86,6 @@ export function ssoUser( oauth2: opts.details?.oauth2, provider: opts.details?.provider!, providerType: opts.details?.providerType!, - ssoId: opts.details?.userId || uuid(), thirdPartyProfile: { email: base.email, picture: base.pictureUrl, From a7722bccc094646de07607ca1a9edc3930237816 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Thu, 28 Mar 2024 16:18:51 +0000 Subject: [PATCH 29/31] fix worker test --- packages/worker/src/api/routes/global/tests/users.spec.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/worker/src/api/routes/global/tests/users.spec.ts b/packages/worker/src/api/routes/global/tests/users.spec.ts index 6923a7839b..2198757be1 100644 --- a/packages/worker/src/api/routes/global/tests/users.spec.ts +++ b/packages/worker/src/api/routes/global/tests/users.spec.ts @@ -560,6 +560,7 @@ describe("/api/global/users", () => { it("if user ssoId is already assigned, no change will be applied", async () => { const user = await createSSOUser() + user.ssoId = "testssoId" await config.api.users .addSsoSupportInternalAPIAuth(user.ssoId, user.email) .expect(200) From 0fe6804290564756b95a5539a6ba20ac439a285a Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Thu, 28 Mar 2024 16:49:13 +0000 Subject: [PATCH 30/31] Bump version to 2.22.13 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index 9cffdba08a..bacdcb782f 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.22.12", + "version": "2.22.13", "npmClient": "yarn", "packages": [ "packages/*", From 29d0b91cc2da8ab725111cbe22c60a1130c7beda Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Thu, 28 Mar 2024 17:28:41 +0000 Subject: [PATCH 31/31] update account-portal ref --- packages/account-portal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/account-portal b/packages/account-portal index 60658a052d..63ce32bca8 160000 --- a/packages/account-portal +++ b/packages/account-portal @@ -1 +1 @@ -Subproject commit 60658a052d2642e5f4a8038e253f771a24f34907 +Subproject commit 63ce32bca871f0a752323f5f7ebb5ec16bbbacc3