Merge branch 'master' into fix/use-direct-describe

.eslintignore

@@ -9,8 +9,5 @@ packages/server/client
 packages/server/coverage
 packages/builder/.routify
 packages/sdk/sdk
-packages/account-portal/packages/server/build
-packages/account-portal/packages/ui/.routify
-packages/account-portal/packages/ui/build
 **/*.ivm.bundle.js
 packages/server/build/oldClientVersions/**/**

.github/workflows/budibase_ci.yml

@@ -64,18 +64,15 @@ jobs:
       - run: yarn --frozen-lockfile
 
       # Run build all the projects
-      - name: Build OSS
+      - name: Build
-        run: yarn build:oss
+        run: yarn build
-      - name: Build account portal
-        run: yarn build:account-portal
-        if: ${{ env.IS_OSS_CONTRIBUTOR == 'false' }}
       # Check the types of the projects built via esbuild
       - name: Check types
         run: |
           if ${{ env.ONLY_AFFECTED_TASKS }}; then
-            yarn check:types --since=${{ env.NX_BASE_BRANCH }} --ignore @budibase/account-portal-server
+            yarn check:types --since=${{ env.NX_BASE_BRANCH }}
           else
-            yarn check:types --ignore @budibase/account-portal-server
+            yarn check:types
           fi
 
   helm-lint:
@@ -150,7 +147,8 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        datasource: [mssql, mysql, postgres, mongodb, mariadb, oracle, none]
+        datasource:
+          [mssql, mysql, postgres, mongodb, mariadb, oracle, sqs, none]
     steps:
       - name: Checkout repo
         uses: actions/checkout@v4
@@ -274,64 +272,6 @@ jobs:
             echo 'All good, the submodule had been merged and setup correctly!'
           fi
 
-  check-accountportal-submodule:
-    runs-on: ubuntu-latest
-    if: inputs.run_as_oss != true  && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase')
-    steps:
-      - name: Checkout repo and submodules
-        uses: actions/checkout@v4
-        with:
-          submodules: true
-          token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }}
-          fetch-depth: 0
-
-      - uses: dorny/paths-filter@v3
-        id: changes
-        with:
-          filters: |
-            src:
-              - packages/account-portal/**
-
-      - if: steps.changes.outputs.src == 'true'
-        name: Check account portal commit
-        id: get_accountportal_commits
-        run: |
-          cd packages/account-portal
-          accountportal_commit=$(git rev-parse HEAD)
-
-          branch="${{ github.base_ref || github.ref_name  }}"
-          echo "Running on branch '$branch' (base_ref=${{ github.base_ref }}, ref_name=${{ github.head_ref }})"
-
-          base_commit=$(git rev-parse origin/master)
-
-          if [[ ! -z $base_commit ]]; then
-            echo "target_branch=$branch"
-            echo "target_branch=$branch" >> "$GITHUB_OUTPUT"
-            echo "accountportal_commit=$accountportal_commit"
-            echo "accountportal_commit=$accountportal_commit" >> "$GITHUB_OUTPUT"
-            echo "base_commit=$base_commit"
-            echo "base_commit=$base_commit" >> "$GITHUB_OUTPUT"
-          else
-            echo "Nothing to do - branch to branch merge."
-          fi
-
-      - name: Check submodule merged to base branch
-        if: ${{ steps.get_accountportal_commits.outputs.base_commit != '' }}
-        uses: actions/github-script@v7
-        with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
-          script: |
-            const submoduleCommit = '${{ steps.get_accountportal_commits.outputs.accountportal_commit }}';
-            const baseCommit = '${{ steps.get_accountportal_commits.outputs.base_commit }}';
-
-            if (submoduleCommit !== baseCommit) {
-              console.error('Submodule commit does not match the latest commit on the "${{ steps.get_accountportal_commits.outputs.target_branch }}" branch.');
-              console.error('Refer to the account portal repo to merge your changes: https://github.com/Budibase/account-portal/blob/master/docs/index.md')
-              process.exit(1);
-            } else {
-              console.log('All good, the submodule had been merged and setup correctly!')
-            }
-
   check-lockfile:
     runs-on: ubuntu-latest
     steps:

.gitignore

@@ -8,6 +8,7 @@ packages/server/build/oldClientVersions/**/*
 packages/builder/src/components/deploy/clientVersions.json
 packages/server/src/integrations/tests/utils/*.lock
 packages/builder/vite.config.mjs.timestamp*
+packages/account-portal
 
 # Logs
 logs
@@ -110,4 +111,4 @@ budibase-component
 budibase-datasource
 
 *.iml
-.nx
+.nx

.gitmodules

@@ -1,6 +1,3 @@
 [submodule "packages/pro"]
 	path = packages/pro
 	url = git@github.com:Budibase/budibase-pro.git
-[submodule "packages/account-portal"]
-	path = packages/account-portal
-	url = git@github.com:Budibase/account-portal.git

.prettierignore

@@ -9,8 +9,4 @@ packages/backend-core/coverage
 packages/builder/.routify
 packages/sdk/sdk
 packages/pro/coverage
-packages/account-portal/packages/ui/build
-packages/account-portal/packages/ui/.routify
-packages/account-portal/packages/server/build
-packages/account-portal/packages/server/coverage
 **/*.ivm.bundle.js

.vscode/launch.json

@@ -20,16 +20,6 @@
       "args": ["${workspaceFolder}/packages/worker/src/index.ts"],
       "cwd": "${workspaceFolder}/packages/worker"
     },
-    {
-      "name": "Camunda Worker",
-      "type": "node",
-      "request": "launch",
-      "runtimeArgs": ["--nolazy", "-r", "ts-node/register/transpile-only"],
-      "args": [
-        "${workspaceFolder}/packages/account-portal/packages/server/src/v2/run.ts"
-      ],
-      "cwd": "${workspaceFolder}/packages/account-portal/packages/server"
-    },
     {
       "type": "chrome",
       "request": "launch",

lerna.json

@@ -1,12 +1,7 @@
 {
   "$schema": "node_modules/lerna/schemas/lerna-schema.json",
-  "version": "3.2.7",
+  "version": "3.2.9",
   "npmClient": "yarn",
-  "packages": [
-    "packages/*",
-    "!packages/account-portal",
-    "packages/account-portal/packages/*"
-  ],
   "concurrency": 20,
   "command": {
     "publish": {

nx.json

@@ -2,7 +2,6 @@
   "$schema": "./node_modules/nx/schemas/nx-schema.json",
   "tasksRunnerOptions": {
     "default": {
-      "runner": "nx-cloud",
       "options": {
         "cacheableOperations": ["build", "test", "check:types"]
       }

package.json

@@ -9,6 +9,7 @@
     "@types/node": "20.10.0",
     "@types/proper-lockfile": "^4.1.4",
     "@typescript-eslint/parser": "6.9.0",
+    "depcheck": "^1.4.7",
     "esbuild": "^0.18.17",
     "esbuild-node-externals": "^1.14.0",
     "eslint": "^8.52.0",
@@ -35,11 +36,10 @@
     "setup": "git config submodule.recurse true && git submodule update && node ./hosting/scripts/setup.js && yarn && yarn build && yarn dev",
     "build": "DISABLE_V8_COMPILE_CACHE=1 NODE_OPTIONS=--max-old-space-size=1500 lerna run build --stream",
     "build:apps": "DISABLE_V8_COMPILE_CACHE=1 yarn build --scope @budibase/server --scope @budibase/worker",
-    "build:oss": "DISABLE_V8_COMPILE_CACHE=1 NODE_OPTIONS=--max-old-space-size=1500 lerna run build --stream --ignore @budibase/account-portal-server --ignore @budibase/account-portal-ui",
     "build:cli": "yarn build --scope @budibase/cli",
-    "build:account-portal": "NODE_OPTIONS=--max-old-space-size=1500 lerna run build --stream --scope @budibase/account-portal-server --scope @budibase/account-portal-ui",
     "build:dev": "lerna run --stream prebuild && yarn nx run-many --target=build --output-style=dynamic --watch --preserveWatchOutput",
-    "check:types": "lerna run --concurrency 2 check:types --ignore @budibase/account-portal-server",
+    "check:types": "yarn check:dependencies && lerna run --concurrency 2 check:types",
+    "check:dependencies": "lerna run --concurrency 2 check:dependencies",
     "build:sdk": "lerna run --stream build:sdk",
     "deps:circular": "madge packages/server/dist/index.js packages/worker/src/index.ts packages/backend-core/dist/src/index.js packages/cli/src/index.js --circular",
     "release": "lerna publish from-package --yes --force-publish --no-git-tag-version --no-push --no-git-reset",
@@ -52,12 +52,9 @@
     "kill-server": "kill-port 4001 4002",
     "kill-accountportal": "kill-port 3001 4003",
     "kill-all": "yarn run kill-builder && yarn run kill-server && yarn kill-accountportal",
-    "dev": "yarn run kill-all && lerna run --parallel prebuild && lerna run --stream dev --ignore=@budibase/account-portal-ui --ignore @budibase/account-portal-server",
+    "dev": "yarn run kill-all && lerna run --parallel prebuild && lerna run --stream dev",
-    "dev:noserver": "yarn run kill-builder && lerna run --stream dev:stack:up --ignore @budibase/account-portal-server && lerna run --stream dev --ignore @budibase/backend-core --ignore @budibase/server --ignore @budibase/worker --ignore=@budibase/account-portal-ui --ignore @budibase/account-portal-server",
+    "dev:noserver": "yarn run kill-builder && lerna run --stream dev:stack:up && lerna run --stream dev --ignore @budibase/backend-core --ignore @budibase/server --ignore @budibase/worker",
     "dev:server": "yarn run kill-server && lerna run --stream dev --scope @budibase/worker --scope @budibase/server",
-    "dev:accountportal": "yarn kill-accountportal && lerna run dev --stream --scope @budibase/account-portal-ui --scope @budibase/account-portal-server",
-    "dev:camunda": "./scripts/deploy-camunda.sh",
-    "dev:all": "yarn run kill-all && lerna run --stream dev",
     "dev:built": "yarn run kill-all && cd packages/server && yarn dev:stack:up && cd ../../ && lerna run --stream dev:built",
     "dev:docker": "./scripts/devDocker.sh",
     "test": "lerna run --concurrency 1 --stream test --stream",
@@ -98,9 +95,7 @@
   },
   "workspaces": {
     "packages": [
-      "packages/*",
+      "packages/*"
-      "!packages/account-portal",
-      "packages/account-portal/packages/*"
     ]
   },
   "resolutions": {

packages/account-portal

@@ -1 +0,0 @@
-Subproject commit 9bef5d1656b4f3c991447ded6d65b0eba393a140

packages/backend-core/package.json

@@ -24,6 +24,7 @@
     "build": "tsc -p tsconfig.build.json --paths null && node ./scripts/build.js",
     "build:dev": "yarn prebuild && tsc --build --watch --preserveWatchOutput",
     "check:types": "tsc -p tsconfig.json --noEmit --paths null --target es2020",
+    "check:dependencies": "node ../../scripts/depcheck.js",
     "test": "bash scripts/test.sh",
     "test:watch": "jest --watchAll"
   },
@@ -43,6 +44,7 @@
     "ioredis": "5.3.2",
     "joi": "17.6.0",
     "jsonwebtoken": "9.0.2",
+    "knex": "2.4.2",
     "koa-passport": "^6.0.0",
     "koa-pino-logger": "4.0.0",
     "lodash": "4.17.21",
@@ -61,9 +63,12 @@
     "semver": "^7.5.4",
     "tar-fs": "2.1.1",
     "uuid": "^8.3.2",
-    "knex": "2.4.2"
+    "@techpass/passport-openidconnect": "0.3.3",
+    "google-auth-library": "^8.0.1",
+    "google-spreadsheet": "npm:@budibase/google-spreadsheet@4.1.5"
   },
   "devDependencies": {
+    "@jest/types": "^29.6.3",
     "@shopify/jest-koa-mocks": "5.1.1",
     "@swc/core": "1.3.71",
     "@swc/jest": "0.2.27",
@@ -71,6 +76,7 @@
     "@types/cookies": "0.7.8",
     "@types/jest": "29.5.5",
     "@types/lodash": "4.14.200",
+    "@types/node": "^22.9.0",
     "@types/node-fetch": "2.6.4",
     "@types/pouchdb": "6.4.0",
     "@types/redlock": "4.0.7",
@@ -81,6 +87,7 @@
     "ioredis-mock": "8.9.0",
     "jest": "29.7.0",
     "jest-serial-runner": "1.2.1",
+    "nock": "^13.5.6",
     "pino-pretty": "10.0.0",
     "pouchdb-adapter-memory": "7.2.2",
     "testcontainers": "^10.7.2",

packages/server/package.json

@@ -13,6 +13,7 @@
     "build": "node ./scripts/build.js",
     "postbuild": "copyfiles -f ../client/dist/budibase-client.js ../client/manifest.json client && copyfiles -f ../../yarn.lock ./dist/",
     "check:types": "tsc -p tsconfig.json --noEmit --paths null --target es2020",
+    "check:dependencies": "node ../../scripts/depcheck.js",
     "build:isolated-vm-lib:snippets": "esbuild --minify --bundle src/jsRunner/bundles/snippets.ts --outfile=src/jsRunner/bundles/snippets.ivm.bundle.js --platform=node --format=iife --global-name=snippets",
     "build:isolated-vm-lib:string-templates": "esbuild --minify --bundle src/jsRunner/bundles/index-helpers.ts --outfile=src/jsRunner/bundles/index-helpers.ivm.bundle.js --platform=node --format=iife --external:handlebars --global-name=helpers",
     "build:isolated-vm-lib:bson": "esbuild --minify --bundle src/jsRunner/bundles/bsonPackage.ts --outfile=src/jsRunner/bundles/bson.ivm.bundle.js --platform=node --format=iife --global-name=bson",
@@ -49,9 +50,11 @@
   "license": "GPL-3.0",
   "dependencies": {
     "@apidevtools/swagger-parser": "10.0.3",
+    "@azure/msal-node": "^2.5.1",
     "@budibase/backend-core": "0.0.0",
     "@budibase/client": "0.0.0",
     "@budibase/frontend-core": "0.0.0",
+    "@budibase/nano": "10.1.5",
     "@budibase/pro": "0.0.0",
     "@budibase/shared-core": "0.0.0",
     "@budibase/string-templates": "0.0.0",
@@ -69,6 +72,7 @@
     "aws-sdk": "2.1030.0",
     "bcrypt": "5.1.0",
     "bcryptjs": "2.4.3",
+    "bson": "^6.9.0",
     "buffer": "6.0.3",
     "bull": "4.10.1",
     "chokidar": "3.5.3",
@@ -76,10 +80,12 @@
     "cookies": "0.8.0",
     "csvtojson": "2.0.10",
     "curlconverter": "3.21.0",
+    "dayjs": "^1.10.8",
     "dd-trace": "5.2.0",
     "dotenv": "8.2.0",
     "form-data": "4.0.0",
     "global-agent": "3.0.0",
+    "google-auth-library": "^8.0.1",
     "google-spreadsheet": "npm:@budibase/google-spreadsheet@4.1.5",
     "ioredis": "5.3.2",
     "isolated-vm": "^4.7.2",
@@ -87,6 +93,7 @@
     "joi": "17.6.0",
     "js-yaml": "4.1.0",
     "jsonschema": "1.4.0",
+    "jsonwebtoken": "9.0.2",
     "knex": "2.4.2",
     "koa": "2.13.4",
     "koa-body": "4.2.0",
@@ -109,10 +116,12 @@
     "pouchdb-all-dbs": "1.1.1",
     "pouchdb-find": "7.2.2",
     "redis": "4",
+    "semver": "^7.5.4",
     "serialize-error": "^7.0.1",
     "server-destroy": "1.0.1",
     "snowflake-promise": "^4.5.0",
     "socket.io": "4.7.5",
+    "svelte": "^4.2.10",
     "tar": "6.2.1",
     "tmp": "0.2.3",
     "to-json-schema": "0.2.5",
@@ -123,6 +132,7 @@
   },
   "devDependencies": {
     "@babel/preset-env": "7.16.11",
+    "@jest/types": "^29.6.3",
     "@swc/core": "1.3.71",
     "@swc/jest": "0.2.27",
     "@types/archiver": "6.0.2",
@@ -133,6 +143,7 @@
     "@types/koa__router": "8.0.8",
     "@types/lodash": "4.14.200",
     "@types/mssql": "9.1.4",
+    "@types/node": "^22.9.0",
     "@types/node-fetch": "2.6.4",
     "@types/oracledb": "6.5.1",
     "@types/pg": "8.6.6",
@@ -141,8 +152,10 @@
     "@types/tar": "6.1.5",
     "@types/tmp": "0.2.6",
     "@types/uuid": "8.3.4",
+    "chance": "^1.1.12",
     "copyfiles": "2.4.1",
     "docker-compose": "0.23.17",
+    "ioredis-mock": "8.9.0",
     "jest": "29.7.0",
     "jest-extended": "^4.0.2",
     "jest-openapi": "0.14.2",
@@ -158,7 +171,8 @@
     "tsconfig-paths": "4.0.0",
     "typescript": "5.5.2",
     "update-dotenv": "1.1.1",
-    "yargs": "13.2.4"
+    "yargs": "13.2.4",
+    "@babel/core": "^7.22.5"
   },
   "nx": {
     "targets": {

packages/worker/package.json

@@ -16,6 +16,7 @@
     "build": "node ../../scripts/build.js",
     "postbuild": "copyfiles -f ../../yarn.lock ./dist/",
     "check:types": "tsc -p tsconfig.json --noEmit --paths null --target es2020",
+    "check:dependencies": "node ../../scripts/depcheck.js",
     "build:dev": "yarn prebuild && tsc --build --watch --preserveWatchOutput",
     "run:docker": "node dist/index.js",
     "debug": "yarn build && node --expose-gc --inspect=9223 dist/index.js",
@@ -41,6 +42,7 @@
     "@budibase/pro": "0.0.0",
     "@budibase/string-templates": "0.0.0",
     "@budibase/types": "0.0.0",
+    "@budibase/shared-core": "0.0.0",
     "@koa/router": "8.0.8",
     "@techpass/passport-openidconnect": "0.3.3",
     "@types/global-agent": "2.1.1",
@@ -53,6 +55,8 @@
     "global-agent": "3.0.0",
     "ical-generator": "4.1.0",
     "joi": "17.6.0",
+    "jsonwebtoken": "9.0.2",
+    "knex": "2.4.2",
     "koa": "2.13.4",
     "koa-body": "4.2.0",
     "koa-compress": "4.0.1",
@@ -69,9 +73,10 @@
     "pouchdb": "7.3.0",
     "pouchdb-all-dbs": "1.1.1",
     "server-destroy": "1.0.1",
-    "knex": "2.4.2"
+    "uuid": "^8.3.2"
   },
   "devDependencies": {
+    "@jest/types": "^29.6.3",
     "@swc/core": "1.3.71",
     "@swc/jest": "0.2.27",
     "@types/jest": "29.5.5",
@@ -79,6 +84,7 @@
     "@types/koa": "2.13.4",
     "@types/koa__router": "8.0.8",
     "@types/lodash": "4.14.200",
+    "@types/node": "^22.9.0",
     "@types/node-fetch": "2.6.4",
     "@types/server-destroy": "1.0.1",
     "@types/supertest": "2.0.14",
@@ -87,6 +93,7 @@
     "nock": "^13.5.4",
     "nodemon": "2.0.15",
     "rimraf": "3.0.2",
+    "superagent": "^10.1.1",
     "supertest": "6.3.3",
     "timekeeper": "2.2.0",
     "typescript": "5.5.2",

scripts/depcheck.js

@@ -0,0 +1,29 @@
+#!/usr/bin/node
+
+const depcheck = require("depcheck")
+
+function filterResults(missing) {
+  if (missing.src) {
+    delete missing.src
+  }
+  return missing
+}
+
+function printMissing(missing) {
+  for (let [key, value] of Object.entries(filterResults(missing))) {
+    console.log(`Package ${key} missing in: ${value.join(", ")}`)
+  }
+}
+
+depcheck(process.cwd(), {
+  ignorePatterns: ["dist"],
+  skipMissing: false,
+}).then(results => {
+  if (Object.values(filterResults(results.missing)).length > 0) {
+    printMissing(results.missing)
+    console.error("Missing packages found - stopping.")
+    process.exit(-1)
+  } else {
+    console.log("No missing dependencies.")
+  }
+})

scripts/deploy-camunda.sh

@@ -1,32 +0,0 @@
-#!/bin/bash
-yarn global add zbctl
-export ZEEBE_ADDRESS='localhost:26500'
-
-cd ../budibase-bpm
-
-is_camunda_ready() {
-    if (zbctl --insecure status 2>/dev/null) | grep -q 'Healthy'; then
-        return 1
-    else
-        return 0
-    fi
-}
-
-docker-compose up -d
-echo "waiting for Camunda to be ready..."
-
-while is_camunda_ready -eq 0; do sleep 1; done
-
-echo "deploy processes..."
-for file in src/main/resources/models/*; do
-    zbctl deploy resource $file --insecure
-done
-
-cd ../budibase/packages/pro
-yarn && yarn build
-
-cd ../account-portal/packages/server 
-yarn worker:run & cd ../../../.. && yarn dev:accountportal
-
-
-

scripts/localdomain.sh

@@ -5,12 +5,12 @@ domain=$2
 
 if [ "$enable" = "enable" ]; then
   lerna run env:localdomain:enable -- "$domain"
-  cd packages/account-portal
+  cd ../account-portal
   yarn env:localdomain:enable "$domain"
   cd -
 else
   lerna run env:localdomain:disable
-  cd packages/account-portal
+  cd ../account-portal
   yarn env:localdomain:disable
   cd -
 fi