Merge pull request #10267 from Budibase/chore/pro_as_submodule

Chore - use pro as submodule

.github/workflows/budibase_ci.yml

@@ -1,36 +1,25 @@
 name: Budibase CI
 
 on:
- # Trigger the workflow on push or pull request,
+  # Trigger the workflow on push or pull request,
- # but only for the master branch
+  # but only for the master branch
- push:
+  push:
     branches:
       - master
       - develop
- pull_request:
+  pull_request:
     branches:
       - master
       - develop
- workflow_dispatch:
+  workflow_dispatch:
 
 env:
   BRANCH: ${{ github.event.pull_request.head.ref }}
   BASE_BRANCH: ${{ github.event.pull_request.base.ref}}
-  PERSONAL_ACCESS_TOKEN : ${{ secrets.PERSONAL_ACCESS_TOKEN }}
+  PERSONAL_ACCESS_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
 
 jobs:
   lint:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-    - name: Use Node.js 14.x
-      uses: actions/setup-node@v1
-      with:
-        node-version: 14.x
-    - run: yarn
-    - run: yarn lint
-
-  build:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
@@ -38,8 +27,20 @@ jobs:
         uses: actions/setup-node@v1
         with:
           node-version: 14.x
-      - name: Install Pro
+      - run: yarn
-        run: yarn install:pro $BRANCH $BASE_BRANCH
+      - run: yarn lint
+
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          submodules: true
+          token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
+      - name: Use Node.js 14.x
+        uses: actions/setup-node@v1
+        with:
+          node-version: 14.x
       - run: yarn
       - run: yarn bootstrap
       - run: yarn build
@@ -48,16 +49,17 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
+        with:
+          submodules: true
+          token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
       - name: Use Node.js 14.x
         uses: actions/setup-node@v1
         with:
           node-version: 14.x
-      - name: Install Pro
-        run: yarn install:pro $BRANCH $BASE_BRANCH
       - run: yarn
       - run: yarn bootstrap
       - run: yarn build
-      - run: yarn test
+      - run: yarn test --ignore=@budibase/pro
       - uses: codecov/codecov-action@v3
         with:
           token: ${{ secrets.CODECOV_TOKEN }} # not required for public repos
@@ -68,26 +70,28 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
+        with:
+          submodules: true
+          token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
       - name: Use Node.js 14.x
         uses: actions/setup-node@v1
         with:
           node-version: 14.x
-      - name: Install Pro
-        run: yarn install:pro $BRANCH $BASE_BRANCH
       - run: yarn
       - run: yarn bootstrap
-      - run: yarn test:pro
+      - run: yarn test --scope=@budibase/pro
 
   integration-test:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
+        with:
+          submodules: true
+          token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
       - name: Use Node.js 14.x
         uses: actions/setup-node@v1
         with:
           node-version: 14.x
-      - name: Install Pro
-        run: yarn install:pro $BRANCH $BASE_BRANCH
       - run: yarn && yarn bootstrap && yarn build
       - run: |
           cd qa-core
@@ -96,3 +100,24 @@ jobs:
         env:
           BB_ADMIN_USER_EMAIL: admin
           BB_ADMIN_USER_PASSWORD: admin
+
+  check-pro-submodule:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+        with:
+          submodules: true
+          token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
+          fetch-depth: 0
+      - name: Check submodule
+        run: |
+          cd packages/pro
+          git fetch
+          if ! git merge-base --is-ancestor $(git log -n 1 --pretty=format:%H) origin/develop; then
+            echo "Current commit has not been merged to develop"
+            echo "Refer to the pro repo to merge your changes: https://github.com/Budibase/budibase-pro/blob/develop/docs/getting_started.md"
+            exit 1
+          else
+            echo "All good, the submodule had been merged!"
+          fi

.github/workflows/release-develop.yml

@@ -2,20 +2,20 @@ name: Budibase Prerelease
 concurrency: release-prerelease
 
 on:
- push:
+  push:
     branches:
       - develop
     paths:
-      - '.aws/**'
+      - ".aws/**"
-      - '.github/**'
+      - ".github/**"
-      - 'charts/**'
+      - "charts/**"
-      - 'packages/**'
+      - "packages/**"
-      - 'scripts/**'
+      - "scripts/**"
-      - 'package.json'
+      - "package.json"
-      - 'yarn.lock'
+      - "yarn.lock"
-      - 'package.json'
+      - "package.json"
-      - 'yarn.lock'
+      - "yarn.lock"
- workflow_dispatch:
+  workflow_dispatch:
 
 env:
   # Posthog token used by ui at build time
@@ -36,18 +36,18 @@ jobs:
           echo "Ref is not develop, you must run this job from develop."
           exit 1
       - uses: actions/checkout@v2
+        with:
+          submodules: true
+          token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
       - uses: actions/setup-node@v1
         with:
           node-version: 14.x
 
-      - name: Install Pro
-        run: yarn install:pro develop
-
       - run: yarn
       - run: yarn bootstrap
       - run: yarn build
       - run: yarn build:sdk
-#      - run: yarn test
+      #      - run: yarn test
 
       - name: Publish budibase packages to NPM
         env:

.github/workflows/release-master.yml

@@ -2,37 +2,37 @@ name: Budibase Release
 concurrency: release
 
 on:
- push:
+  push:
     branches:
       - master
     paths:
-      - '.aws/**'
+      - ".aws/**"
-      - '.github/**'
+      - ".github/**"
-      - 'charts/**'
+      - "charts/**"
-      - 'packages/**'
+      - "packages/**"
-      - 'scripts/**'
+      - "scripts/**"
-      - 'package.json'
+      - "package.json"
-      - 'yarn.lock'
+      - "yarn.lock"
-      - 'package.json'
+      - "package.json"
-      - 'yarn.lock'
+      - "yarn.lock"
- workflow_dispatch:
+  workflow_dispatch:
-  inputs:
+    inputs:
-    versioning:
+      versioning:
-      type: choice
+        type: choice
-      description: "Versioning type: patch, minor, major"
+        description: "Versioning type: patch, minor, major"
-      default: patch
+        default: patch
-      options:
+        options:
-        - patch
+          - patch
-        - minor
+          - minor
-        - major
+          - major
-      required: true
+        required: true
 
 env:
   # Posthog token used by ui at build time
   POSTHOG_TOKEN: phc_bIjZL7oh2GEUd2vqvTBH8WvrX0fWTFQMs6H5KQxiUxU
   INTERCOM_TOKEN: ${{ secrets.INTERCOM_TOKEN }}
   SENTRY_DSN: ${{ secrets.SENTRY_DSN }}
-  PERSONAL_ACCESS_TOKEN : ${{ secrets.PERSONAL_ACCESS_TOKEN }}
+  PERSONAL_ACCESS_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
 
 jobs:
   release-images:
@@ -44,13 +44,13 @@ jobs:
           echo "Ref is not master, you must run this job from master."
           exit 1
       - uses: actions/checkout@v2
+        with:
+          submodules: true
+          token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
       - uses: actions/setup-node@v1
         with:
           node-version: 14.x
 
-      - name: Install Pro
-        run: yarn install:pro master
-
       - run: yarn
       - run: yarn bootstrap
       - run: yarn lint
@@ -68,7 +68,7 @@ jobs:
           echo //registry.npmjs.org/:_authToken=${NPM_TOKEN} >> .npmrc
           yarn release
 
-      - name: 'Get Previous tag'
+      - name: "Get Previous tag"
         id: previoustag
         uses: "WyriHaximus/github-action-get-previous-tag@v1"
 

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "packages/pro"]
+	path = packages/pro
+	url = git@github.com:Budibase/budibase-pro.git

.husky/post-checkout

@@ -0,0 +1,4 @@
+# .husky/post-checkout
+# ...
+
+git config submodule.recurse true

docs/DEV-SETUP-DEBIAN.md

@@ -1,13 +1,17 @@
 ## Dev Environment on Debian 11
 
 ### Install NVM & Node 14
+
 NVM documentation: https://github.com/nvm-sh/nvm#installing-and-updating
 
 Install NVM
+
 ```
 curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.0/install.sh | bash
 ```
+
 Install Node 14
+
 ```
 nvm install 14
 ```
@@ -17,13 +21,16 @@ nvm install 14
 ```
 npm install -g yarn jest lerna
 ```
+
 ### Install Docker and Docker Compose
 
 ```
 apt install docker.io
 pip3 install docker-compose
 ```
+
 ### Clone the repo
+
 ```
 git clone https://github.com/Budibase/budibase.git
 ```
@@ -44,10 +51,13 @@ This setup process was tested on Debian 11 (bullseye) with version numbers show
 cd budibase
 yarn setup
 ```
+
 The yarn setup command runs several build steps i.e.
+
 ```
 node ./hosting/scripts/setup.js && yarn && yarn bootstrap && yarn build && yarn dev
 ```
+
 So this command will actually run the application in dev mode. It creates .env files under `./packages/server` and `./packages/worker` and runs docker containers for each service via docker-compose.
 
 The dev version will be available on port 10000 i.e.
@@ -55,6 +65,7 @@ The dev version will be available on port 10000 i.e.
 http://127.0.0.1:10000/builder/admin
 
 ### File descriptor issues with Vite and Chrome in Linux
+
 If your dev environment stalls forever, with some network requests stuck in flight, it's likely that Chrome is trying to open more file descriptors than your system allows.
 To fix this, apply the following tweaks.
 

docs/DEV-SETUP-MACOSX.md

@@ -8,10 +8,10 @@ Install instructions [here](https://brew.sh/)
 `eval $(/opt/homebrew/bin/brew shellenv)` line to your `.zshrc`. This will make your zsh to find the apps you install
 through brew.
 
-
 ### Install Node
 
 Budibase requires a recent version of node 14:
+
 ```
 brew install node npm
 node -v
@@ -22,12 +22,15 @@ node -v
 ```
 npm install -g yarn jest lerna
 ```
+
 ### Install Docker and Docker Compose
 
 ```
 brew install docker docker-compose
 ```
+
 ### Clone the repo
+
 ```
 git clone https://github.com/Budibase/budibase.git
 ```
@@ -48,10 +51,13 @@ This setup process was tested on Mac OSX 12 (Monterey) with version numbers show
 cd budibase
 yarn setup
 ```
+
 The yarn setup command runs several build steps i.e.
+
 ```
 node ./hosting/scripts/setup.js && yarn && yarn bootstrap && yarn build && yarn dev
 ```
+
 So this command will actually run the application in dev mode. It creates .env files under `./packages/server` and `./packages/worker` and runs docker containers for each service via docker-compose.
 
 The dev version will be available on port 10000 i.e.
@@ -62,4 +68,5 @@ http://127.0.0.1:10000/builder/admin
 [hosting/docker-compose-dev.yaml](../hosting/docker-compose.dev.yaml)
 
 ### Troubleshooting
+
 If there are errors with the `yarn setup` command, you can try installing nvm and node 14. This is the same as the instructions for Debian 11.

docs/DEV-SETUP-WINDOWS.md

@@ -1,13 +1,15 @@
 ## Dev Environment on Windows 10/11 (WSL2)
 
-
 ### Install WSL with Ubuntu LTS
 
 Enable WSL 2 on Windows 10/11 for docker support.
+
 ```
 wsl --set-default-version 2
 ```
+
 Install Ubuntu LTS.
+
 ```
 wsl --install Ubuntu
 ```
@@ -16,6 +18,7 @@ Or follow the instruction here:
 https://learn.microsoft.com/en-us/windows/wsl/install
 
 ### Install Docker in windows
+
 Download the installer from docker and install it.
 
 Check this url for more detailed instructions:
@@ -24,18 +27,21 @@ https://docs.docker.com/desktop/install/windows-install/
 You should follow the next steps from within the Ubuntu terminal.
 
 ### Install NVM & Node 14
+
 NVM documentation: https://github.com/nvm-sh/nvm#installing-and-updating
 
 Install NVM
+
 ```
 curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.0/install.sh | bash
 ```
+
 Install Node 14
+
 ```
 nvm install 14
 ```
 
-
 ### Install npm requirements
 
 ```
@@ -43,6 +49,7 @@ npm install -g yarn jest lerna
 ```
 
 ### Clone the repo
+
 ```
 git clone https://github.com/Budibase/budibase.git
 ```
@@ -63,10 +70,13 @@ This setup process was tested on Windows 11 with version numbers show below. You
 cd budibase
 yarn setup
 ```
+
 The yarn setup command runs several build steps i.e.
+
 ```
 node ./hosting/scripts/setup.js && yarn && yarn bootstrap && yarn build && yarn dev
 ```
+
 So this command will actually run the application in dev mode. It creates .env files under `./packages/server` and `./packages/worker` and runs docker containers for each service via docker-compose.
 
 The dev version will be available on port 10000 i.e.
@@ -74,6 +84,7 @@ The dev version will be available on port 10000 i.e.
 http://127.0.0.1:10000/builder/admin
 
 ### Working with the code
+
 Here are the instructions to work on the application from within Visual Studio Code (in Windows) through the WSL. All the commands and files are within the Ubuntu system and it should run as if you were working on a Linux machine.
 
 https://code.visualstudio.com/docs/remote/wsl

lerna.json

@@ -1,8 +1,22 @@
 {
   "version": "2.5.6-alpha.6",
   "npmClient": "yarn",
+  "packages": [
+    "packages/backend-core",
+    "packages/bbui",
+    "packages/builder",
+    "packages/cli",
+    "packages/client",
+    "packages/frontend-core",
+    "packages/sdk",
+    "packages/server",
+    "packages/shared-core",
+    "packages/string-templates",
+    "packages/types",
+    "packages/worker",
+    "packages/pro/packages/pro"
+  ],
   "useWorkspaces": true,
-  "packages": ["packages/*"],
   "command": {
     "publish": {
       "ignoreChanges": [

package.json

@@ -8,7 +8,7 @@
     "eslint": "^7.28.0",
     "eslint-plugin-cypress": "^2.11.3",
     "eslint-plugin-svelte3": "^3.2.0",
-    "husky": "^7.0.1",
+    "husky": "^8.0.3",
     "js-yaml": "^4.1.0",
     "kill-port": "^1.6.1",
     "lerna": "^6.6.1",
@@ -21,18 +21,16 @@
     "typescript": "4.7.3"
   },
   "scripts": {
-    "setup": "node ./hosting/scripts/setup.js && yarn && yarn bootstrap && yarn build && yarn dev",
+    "setup": "git config submodule.recurse true && git submodule update && node ./hosting/scripts/setup.js && yarn && yarn bootstrap && yarn build && yarn dev",
-    "bootstrap": "lerna link && ./scripts/link-dependencies.sh",
+    "bootstrap": "./scripts/bootstrap.sh && lerna link && ./scripts/link-dependencies.sh",
     "build": "lerna run --stream build",
-    "build:dev": "lerna run --stream prebuild && tsc --build --watch --preserveWatchOutput",
+    "build:dev": "lerna run --stream prebuild && yarn nx run-many --target=build --output-style=dynamic --watch --preserveWatchOutput",
     "backend:bootstrap": "./scripts/scopeBackend.sh && yarn run bootstrap",
     "backend:build": "./scripts/scopeBackend.sh 'lerna run --stream build'",
     "build:sdk": "lerna run --stream build:sdk",
     "deps:circular": "madge packages/server/dist/index.js packages/worker/src/index.ts packages/backend-core/dist/src/index.js packages/cli/src/index.js --circular",
     "release": "lerna publish ${RELEASE_VERSION_TYPE:-patch} --yes --force-publish && yarn release:pro",
     "release:develop": "lerna publish prerelease --yes --force-publish --dist-tag develop --exact && yarn release:pro:develop",
-    "release:pro": "bash scripts/pro/release.sh",
-    "release:pro:develop": "bash scripts/pro/release.sh develop",
     "restore": "yarn run clean && yarn run bootstrap && yarn run build",
     "nuke": "yarn run nuke:packages && yarn run nuke:docker",
     "nuke:packages": "yarn run restore",
@@ -46,7 +44,6 @@
     "dev:server": "yarn run kill-server && lerna run --stream --parallel dev:builder --concurrency 1 --scope @budibase/backend-core --scope @budibase/worker --scope @budibase/server",
     "dev:built": "cd packages/server && yarn dev:stack:up && cd ../../ && lerna run --stream --parallel dev:built",
     "test": "lerna run --stream test --stream",
-    "test:pro": "bash scripts/pro/test.sh",
     "lint:eslint": "eslint packages && eslint qa-core",
     "lint:prettier": "prettier --check \"packages/**/*.{js,ts,svelte}\" && prettier --write \"examples/**/*.{js,ts,svelte}\" && prettier --check \"qa-core/**/*.{js,ts,svelte}\"",
     "lint": "yarn run lint:eslint && yarn run lint:prettier",
@@ -82,12 +79,25 @@
     "mode:account": "yarn mode:cloud && yarn env:account:enable",
     "security:audit": "node scripts/audit.js",
     "postinstall": "husky install",
-    "install:pro": "bash scripts/pro/install.sh",
+    "dep:clean": "yarn clean -y && yarn bootstrap",
-    "dep:clean": "yarn clean && yarn bootstrap"
+    "submodules:load": "git submodule init && git submodule update && yarn && yarn bootstrap",
+    "submodules:unload": "git submodule deinit --all && yarn && yarn bootstrap"
   },
   "workspaces": {
     "packages": [
-      "packages/*"
+      "packages/backend-core",
+      "packages/bbui",
+      "packages/builder",
+      "packages/cli",
+      "packages/client",
+      "packages/frontend-core",
+      "packages/sdk",
+      "packages/server",
+      "packages/shared-core",
+      "packages/string-templates",
+      "packages/types",
+      "packages/worker",
+      "packages/pro/packages/pro"
     ]
   }
 }

packages/backend-core/package.json

@@ -15,8 +15,6 @@
     "prebuild": "rimraf dist/",
     "prepack": "cp package.json dist",
     "build": "tsc -p tsconfig.build.json",
-    "build:pro": "../../scripts/pro/build.sh",
-    "postbuild": "yarn run build:pro",
     "build:dev": "yarn prebuild && tsc --build --watch --preserveWatchOutput",
     "test": "bash scripts/test.sh",
     "test:watch": "jest --watchAll"

packages/backend-core/tsconfig.json

@@ -7,11 +7,6 @@
       "@budibase/types": ["../types/src"]
     }
   },
-  "references": [
+
-    { "path": "../types" }
+  "exclude": ["node_modules", "dist"]
-  ],
-  "exclude": [
-    "node_modules",
-    "dist",
-  ]
 }

packages/builder/tsconfig.json

@@ -13,9 +13,5 @@
   },
   "ts-node": {
     "require": ["tsconfig-paths/register"]
-  },
+  }
-  "references": [
-    { "path": "../types" },
-    { "path": "../backend-core" },
-  ]
 }

packages/cli/tsconfig.json

@@ -8,23 +8,14 @@
     "paths": {
       "@budibase/types": ["../types/src"],
       "@budibase/backend-core": ["../backend-core/src"],
-      "@budibase/backend-core/*": ["../backend-core/*"],
+      "@budibase/backend-core/*": ["../backend-core/*"]
     }
   },
   "ts-node": {
     "require": ["tsconfig-paths/register"],
     "swc": true
   },
-  "references": [
+
-    { "path": "../types" },
+  "include": ["src/**/*", "package.json"],
-    { "path": "../backend-core" },
+  "exclude": ["node_modules", "dist"]
-  ],
-  "include": [
-    "src/**/*",
-    "package.json"
-  ],
-  "exclude": [
-    "node_modules",
-    "dist"
-  ]
 }

packages/pro

@@ -0,0 +1 @@
+Subproject commit 315e57dc9185dede169af073a01cc23e673b7091

packages/server/jest.config.ts

@@ -20,9 +20,9 @@ const baseConfig: Config.InitialProjectOptions = {
 }
 
 // add pro sources if they exist
-if (fs.existsSync("../../../budibase-pro")) {
+if (fs.existsSync("../pro/packages")) {
-  baseConfig.moduleNameMapper["@budibase/pro"] =
+  baseConfig.moduleNameMapper!["@budibase/pro"] =
-    "<rootDir>/../../../budibase-pro/packages/pro/src"
+    "<rootDir>/../pro/packages/pro/src"
 }
 
 const config: Config.InitialOptions = {

packages/server/nodemon.json

@@ -1,6 +1,10 @@
 {
-  "watch": ["src", "../backend-core", "../../../budibase-pro/packages/pro"],
+  "watch": ["src", "../backend-core", "../pro/packages/pro"],
   "ext": "js,ts,json",
-  "ignore": ["src/**/*.spec.ts", "src/**/*.spec.js", "../backend-core/dist/**/*"],
+  "ignore": [
+    "src/**/*.spec.ts",
+    "src/**/*.spec.js",
+    "../backend-core/dist/**/*"
+  ],
   "exec": "ts-node src/index.ts"
 }

packages/server/tsconfig.json

@@ -10,19 +10,13 @@
       "@budibase/backend-core": ["../backend-core/src"],
       "@budibase/backend-core/*": ["../backend-core/*"],
       "@budibase/shared-core": ["../shared-core/src"],
-      "@budibase/pro": ["../../../budibase-pro/packages/pro/src"]
+      "@budibase/pro": ["../pro/packages/pro/src"]
     }
   },
   "ts-node": {
     "require": ["tsconfig-paths/register"],
     "swc": true
   },
-  "references": [
-    { "path": "../types" },
-    { "path": "../backend-core" },
-    { "path": "../shared-core" },
-    { "path": "../../../budibase-pro/packages/pro" }
-  ],
   "include": ["src/**/*", "specs", "package.json"],
   "exclude": ["node_modules", "dist"]
 }

packages/shared-core/tsconfig.json

@@ -6,6 +6,5 @@
     "paths": {
       "@budibase/types": ["../types/src"]
     }
-  },
+  }
-  "references": [{ "path": "../types" }]
 }

packages/worker/jest.config.ts

@@ -20,11 +20,11 @@ const config: Config.InitialOptions = {
 }
 
 // add pro sources if they exist
-if (fs.existsSync("../../../budibase-pro")) {
+if (fs.existsSync("../pro/packages")) {
-  config.moduleNameMapper["@budibase/pro/(.*)"] =
+  config.moduleNameMapper!["@budibase/pro/(.*)"] =
-    "<rootDir>/../../../budibase-pro/packages/pro/$1"
+    "<rootDir>/../pro/packages/pro/$1"
-  config.moduleNameMapper["@budibase/pro"] =
+  config.moduleNameMapper!["@budibase/pro"] =
-    "<rootDir>/../../../budibase-pro/packages/pro/src"
+    "<rootDir>/../pro/packages/pro/src"
 }
 
 export default config

packages/worker/nodemon.json

@@ -1,6 +1,10 @@
 {
-  "watch": ["src", "../backend-core", "../../../budibase-pro/packages/pro"],
+  "watch": ["src", "../backend-core", "../pro/packages/pro"],
   "ext": "js,ts,json",
-  "ignore": ["src/**/*.spec.ts", "src/**/*.spec.js", "../backend-core/dist/**/*"],
+  "ignore": [
+    "src/**/*.spec.ts",
+    "src/**/*.spec.js",
+    "../backend-core/dist/**/*"
+  ],
   "exec": "ts-node src/index.ts"
 }

packages/worker/tsconfig.json

@@ -9,23 +9,13 @@
       "@budibase/types": ["../types/src"],
       "@budibase/backend-core": ["../backend-core/src"],
       "@budibase/backend-core/*": ["../backend-core/*"],
-      "@budibase/pro": ["../../../budibase-pro/packages/pro/src"]
+      "@budibase/pro": ["../pro/packages/pro/src"]
     }
   },
   "ts-node": {
     "require": ["tsconfig-paths/register"],
     "swc": true
   },
-  "references": [
+  "include": ["src/**/*", "package.json"],
-    { "path": "../types" },
+  "exclude": ["dist"]
-    { "path": "../backend-core" },
-    { "path": "../../../budibase-pro/packages/pro" }
-  ],
-  "include": [
-    "src/**/*",
-    "package.json"
-  ],
-  "exclude": [
-    "dist"
-  ]
 }

qa-core/tsconfig.json

@@ -16,22 +16,12 @@
       "@budibase/types": ["../packages/types/src"],
       "@budibase/backend-core": ["../packages/backend-core/src"],
       "@budibase/backend-core/*": ["../packages/backend-core/*"],
-      "@budibase/server/*": ["../packages/server/src/*"],
+      "@budibase/server/*": ["../packages/server/src/*"]
     }
   },
   "ts-node": {
     "require": ["tsconfig-paths/register"]
   },
-  "references": [
+  "include": ["src/**/*", "package.json"],
-    { "path": "../packages/types" },
+  "exclude": ["node_modules", "dist"]
-    { "path": "../packages/backend-core" },
-  ],
-  "include": [
-    "src/**/*",
-    "package.json"
-  ],
-  "exclude": [
-    "node_modules",
-    "dist"
-  ]
 }

scripts/bootstrap.sh

@@ -0,0 +1,7 @@
+if [ -d "packages/pro/packages" ]; then
+  cd packages/pro
+
+  yarn
+  lerna bootstrap
+  yarn setup
+fi

scripts/link-dependencies.sh

@@ -34,33 +34,17 @@ yarn unlink
 yarn link
 cd -
 
-if [ -d "../budibase-pro" ]; then
+if [ -d packages/pro/packages ]; then
-  cd ../budibase-pro
+  pro_loaded_locally=true
-  echo "Bootstrapping budibase-pro"
+else
-  yarn bootstrap
+  pro_loaded_locally=false
+fi
 
-  cd packages/pro
+if [ $pro_loaded_locally = true ]; then
   echo "Linking pro"
+  cd packages/pro/packages/pro
   yarn unlink
   yarn link
-
-  echo "Linking backend-core to pro"
-  yarn link '@budibase/backend-core'
-
-  echo "Linking types to pro"
-  yarn link '@budibase/types'
-
-  echo "Linking string-templates to pro"
-  yarn link '@budibase/string-templates'
-
-  cd ../../../budibase
-
-  echo "Linking pro to worker"
-  cd packages/worker && yarn link '@budibase/pro'
-  cd -
-
-  echo "Linking pro to server"
-  cd packages/server && yarn link '@budibase/pro'
   cd -
 fi
 
@@ -79,7 +63,7 @@ if [ -d "../account-portal" ]; then
   echo "Linking types to account-portal"
   yarn link "@budibase/types"
 
-  if [ -d "../../../budibase-pro" ]; then
+  if [ $pro_loaded_locally = true ]; then
     echo "Linking pro to account-portal"
     yarn link "@budibase/pro"
   fi

scripts/pro/build.sh

@@ -1,15 +0,0 @@
-#!/bin/bash
-
-# This script is designed for building the pro repo after the backend-core build has completed.
-# This ensures that any changes in core that are required by pro are done in the correct order.
-
-set -e
-
-# Go to parent of budibase
-cd ../../../
-
-if [[ -d "budibase-pro" ]]; then
-  cd budibase-pro
-  echo "Building pro"
-  yarn build
-fi

scripts/pro/install.sh

@@ -1,39 +0,0 @@
-#!/bin/bash
-
-if [[ -z "${CI}" ]]; then
-  echo 'Cannot run install.sh unless in CI'
-  exit 0
-fi
-
-BRANCH=$1
-BASE_BRANCH=$2
-
-cd ../
-echo "Cloning pro repo..."
-git clone https://$PERSONAL_ACCESS_TOKEN@github.com/Budibase/budibase-pro.git
-
-# Community forks won't be able to clone the pro repo as they can't access secrets
-# Skip the rest of the installation and rely on npm version instead
-# This should be ok as forks will not rely on pro changes
-if [[ -d "budibase-pro" ]]; then
-  cd budibase-pro
-
-  if [[ -z "${BRANCH}" ]]; then
-    echo Using GITHUB_REF_NAME: $GITHUB_REF_NAME
-    export BRANCH=$GITHUB_REF_NAME
-  fi
-
-  # Try to checkout the matching pro branch
-  git checkout $BRANCH
-
-  if [[ $? == "1" ]] && [[ $BASE_BRANCH ]]; then
-    # There is no matching branch, try to match the base branch
-    git checkout $BASE_BRANCH
-  fi
-
-  # If neither branch exists continue with default branch 'develop'
-  git pull
-
-  echo "Initializing pro repo..."
-  yarn
-fi

scripts/pro/release.sh

@@ -1,94 +0,0 @@
-#!/bin/bash 
-
-# Fail when any command fails
-set -e
-
-if [[ -z "${CI}" ]]; then
-  echo 'Cannot run release.sh unless in CI'
-  exit 0
-fi
-
-#############################################
-#                   SETUP                   #
-#############################################
-
-# Release pro with same version as budibase
-VERSION=$(jq -r .version lerna.json)
-echo "Version: $VERSION"
-COMMAND=$1
-echo "Command: $COMMAND"
-
-# Determine tag to use
-TAG=""
-if [[ $COMMAND == "develop" ]]; then
-  TAG="develop"
-else
-  TAG="latest"
-fi
-
-echo "Releasing version $VERSION"
-echo "Releasing tag $TAG"
-
-#############################################
-#                PRE-PUBLISH                #
-#############################################
-
-# Go to pro repo root
-cd ../budibase-pro
-
-# Install NPM credentials
-echo //registry.npmjs.org/:_authToken=${NPM_TOKEN} >> .npmrc 
-
-# Sync budibase dependency versions in packages/pro/package.json
-# Ensures pro does not use out of date dependency
-cd packages/pro
-jq '.dependencies."@budibase/backend-core"="'$VERSION'"' package.json > package.json.tmp && mv package.json.tmp package.json
-jq '.dependencies."@budibase/types"="'$VERSION'"' package.json > package.json.tmp && mv package.json.tmp package.json
-
-# Go back to pro repo root
-cd -
-
-# Update lockfile with new dependency versions
-yarn clean -y && yarn bootstrap
-
-# Commit and push
-git add packages/pro/yarn.lock
-git commit -m "Update dependency versions to $VERSION" -n
-git push
-
-#############################################
-#                  PUBLISH                  #
-#############################################
-
-lerna publish $VERSION --yes --force-publish --dist-tag $TAG
-
-#############################################
-#            POST-PUBLISH - BUDIBASE        #
-#############################################
-
-# Go to budibase repo root
-cd ../budibase
-
-# Update pro version in packages/server/package.json
-cd packages/server
-jq '.dependencies."@budibase/pro"="'$VERSION'"' package.json > package.json.tmp && mv package.json.tmp package.json
-
-# Go back to budibase repo root
-cd -
-
-# Update pro version in packages/worker/package.json
-cd packages/worker
-jq '.dependencies."@budibase/pro"="'$VERSION'"' package.json > package.json.tmp && mv package.json.tmp package.json
-
-# Go back to budibase repo root
-cd -
-
-# Update lockfile with new pro version
-yarn clean -y && yarn bootstrap
-
-# Commit and push changes
-git add packages/server/package.json
-git add packages/worker/package.json
-git add yarn.lock
-git commit -m "Update pro version to $VERSION" -n
-git push

scripts/pro/test.sh

@@ -1,11 +0,0 @@
-#!/bin/bash
-
-# Fail when any command fails
-set -e
-
-cd ../
-if [[ -d "budibase-pro" ]]; then
-  cd budibase-pro
-  yarn test
-  cd ../budibase
-fi

tsconfig.json

@@ -1,9 +0,0 @@
-{
-  "references": [
-    { "path": "./packages/types" },
-    { "path": "./packages/backend-core" },
-    { "path": "./packages/server" },
-    { "path": "./packages/worker" }
-  ],
-  "files" :[]
-}

yarn.lock

@@ -1386,6 +1386,45 @@
   resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
   integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==
 
+"@budibase/backend-core@2.5.6-alpha.3":
+  version "2.5.6-alpha.3"
+  resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-2.5.6-alpha.3.tgz#686d1533a3218507e4c82a58bf73caf875e0d895"
+  integrity sha512-TYaGj1G1h3K/QNwSanEFbwxlA04T3dc0DDHDgauyvqX8kHXYhY11BDn8xqrCQrpqXC7XlC4nL78NrhZiDJ+eFA==
+  dependencies:
+    "@budibase/nano" "10.1.2"
+    "@budibase/pouchdb-replication-stream" "1.2.10"
+    "@budibase/types" "2.5.6-alpha.3"
+    "@shopify/jest-koa-mocks" "5.0.1"
+    "@techpass/passport-openidconnect" "0.3.2"
+    aws-cloudfront-sign "2.2.0"
+    aws-sdk "2.1030.0"
+    bcrypt "5.0.1"
+    bcryptjs "2.4.3"
+    bull "4.10.1"
+    correlation-id "4.0.0"
+    dotenv "16.0.1"
+    emitter-listener "1.1.2"
+    ioredis "4.28.0"
+    joi "17.6.0"
+    jsonwebtoken "9.0.0"
+    koa-passport "4.1.4"
+    koa-pino-logger "4.0.0"
+    lodash "4.17.21"
+    lodash.isarguments "3.1.0"
+    node-fetch "2.6.7"
+    passport-google-oauth "2.0.0"
+    passport-jwt "4.0.0"
+    passport-local "1.0.0"
+    passport-oauth2-refresh "^2.1.0"
+    posthog-node "1.3.0"
+    pouchdb "7.3.0"
+    pouchdb-find "7.2.2"
+    redlock "4.2.0"
+    sanitize-s3-objectkey "0.0.1"
+    semver "7.3.7"
+    tar-fs "2.1.1"
+    uuid "8.3.2"
+
 "@budibase/bbui@^0.9.139":
   version "0.9.190"
   resolved "https://registry.yarnpkg.com/@budibase/bbui/-/bbui-0.9.190.tgz#e1ec400ac90f556bfbc80fc23a04506f1585ea81"
@@ -1547,6 +1586,13 @@
   resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.4.44-alpha.1.tgz#1679657aa180d9c59afa1dffa611bff0638bd933"
   integrity sha512-Sq+8HfM75EBMoOvKYFwELdlxmVN6wNZMofDjT/2G+9aF+Zfe5Tzw69C+unmdBgcGGjGCHEYWSz4mF0v8FPAGbg==
 
+"@budibase/types@2.5.6-alpha.3":
+  version "2.5.6-alpha.3"
+  resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.5.6-alpha.3.tgz#fd7656599d388d2d5a21806b9058a2656793ec87"
+  integrity sha512-SD9WTXb2A+3jqEEk8HoEDwphm5tiwfQWiBFOzTZlooWiS5M6UGxKCgXzPv0Ad9Pfxi2oGxfGh7qJcNgHQDoE3Q==
+  dependencies:
+    scim-patch "^0.7.0"
+
 "@bull-board/api@3.7.0":
   version "3.7.0"
   resolved "https://registry.yarnpkg.com/@bull-board/api/-/api-3.7.0.tgz#231f687187c0cb34e0b97f463917b6aaeb4ef6af"
@@ -3007,7 +3053,7 @@
   resolved "https://registry.yarnpkg.com/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.14.tgz#add4c98d341472a289190b424efbdb096991bb24"
   integrity sha512-XPSJHWmi394fuUuzDnGz1wiKqWfo1yXecHQMRf2l6hztTO+nPru658AyDngaBe7isIxEkRsPR3FZh+s7iVa4Uw==
 
-"@jridgewell/sourcemap-codec@^1.4.10":
+"@jridgewell/sourcemap-codec@^1.4.10", "@jridgewell/sourcemap-codec@^1.4.13":
   version "1.4.15"
   resolved "https://registry.yarnpkg.com/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.15.tgz#d7c6e6755c78567a951e04ab52ef0fd26de59f32"
   integrity sha512-eF2rxCRulEKXHTRiDrDy6erMYWqNw4LPdQ8UQA4huuxaQsVeRPFl2oM8oDGxMFhJUWZf9McpLtJasDDZb/Bpeg==
@@ -3717,7 +3763,7 @@
   dependencies:
     slash "^3.0.0"
 
-"@rollup/plugin-commonjs@^16.0.0":
+"@rollup/plugin-commonjs@16.0.0", "@rollup/plugin-commonjs@^16.0.0":
   version "16.0.0"
   resolved "https://registry.yarnpkg.com/@rollup/plugin-commonjs/-/plugin-commonjs-16.0.0.tgz#169004d56cd0f0a1d0f35915d31a036b0efe281f"
   integrity sha512-LuNyypCP3msCGVQJ7ki8PqYdpjfEkE/xtFa5DqlF+7IBD0JsfMZ87C58heSwIMint58sAUZbt3ITqOmdQv/dXw==
@@ -3800,6 +3846,22 @@
     "@rollup/pluginutils" "^3.1.0"
     magic-string "^0.25.7"
 
+"@rollup/plugin-replace@^5.0.2":
+  version "5.0.2"
+  resolved "https://registry.yarnpkg.com/@rollup/plugin-replace/-/plugin-replace-5.0.2.tgz#45f53501b16311feded2485e98419acb8448c61d"
+  integrity sha512-M9YXNekv/C/iHHK+cvORzfRYfPbq0RDD8r0G+bMiTXjNGKulPnCT9O3Ss46WfhI6ZOCgApOP7xAdmCQJ+U2LAA==
+  dependencies:
+    "@rollup/pluginutils" "^5.0.1"
+    magic-string "^0.27.0"
+
+"@rollup/plugin-typescript@8.3.0":
+  version "8.3.0"
+  resolved "https://registry.yarnpkg.com/@rollup/plugin-typescript/-/plugin-typescript-8.3.0.tgz#bc1077fa5897b980fc27e376c4e377882c63e68b"
+  integrity sha512-I5FpSvLbtAdwJ+naznv+B4sjXZUcIvLLceYpITAn7wAP8W0wqc5noLdGIp9HGVntNhRWXctwPYrSSFQxtl0FPA==
+  dependencies:
+    "@rollup/pluginutils" "^3.1.0"
+    resolve "^1.17.0"
+
 "@rollup/pluginutils@^3.0.8", "@rollup/pluginutils@^3.1.0":
   version "3.1.0"
   resolved "https://registry.yarnpkg.com/@rollup/pluginutils/-/pluginutils-3.1.0.tgz#706b4524ee6dc8b103b3c995533e5ad680c02b9b"
@@ -11698,7 +11760,7 @@ fs.realpath@^1.0.0:
   resolved "https://registry.yarnpkg.com/fs.realpath/-/fs.realpath-1.0.0.tgz#1504ad2523158caa40db4a2787cb01411994ea4f"
   integrity sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==
 
-fsevents@^2.1.2, fsevents@^2.3.2, fsevents@~2.3.2:
+fsevents@^2.1.2, fsevents@^2.3.2, fsevents@~2.3.1, fsevents@~2.3.2:
   version "2.3.2"
   resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-2.3.2.tgz#8a526f78b8fdf4623b709e0b975c52c24c02fd1a"
   integrity sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==
@@ -12880,10 +12942,10 @@ humanize-ms@^1.2.0, humanize-ms@^1.2.1:
   dependencies:
     ms "^2.0.0"
 
-husky@^7.0.1:
+husky@^8.0.3:
-  version "7.0.4"
+  version "8.0.3"
-  resolved "https://registry.yarnpkg.com/husky/-/husky-7.0.4.tgz#242048245dc49c8fb1bf0cc7cfb98dd722531535"
+  resolved "https://registry.yarnpkg.com/husky/-/husky-8.0.3.tgz#4936d7212e46d1dea28fef29bb3a108872cd9184"
-  integrity sha512-vbaCKN2QLtP/vD4yvs6iz6hBEo6wkSzs8HpRah1Z6aGmF2KW5PdYuAd7uX5a+OyBZHBhd+TFLqgjUgytQr4RvQ==
+  integrity sha512-+dQSyqPh4x1hlO1swXBiNb2HzTDN1I2IGLQx1GrBuiqFJfoMrnZWwVmatvSiO+Iz8fBUnf+lekwNo4c2LlXItg==
 
 iconv-lite@0.4.24, iconv-lite@^0.4.15, iconv-lite@^0.4.24, iconv-lite@^0.4.5:
   version "0.4.24"
@@ -16688,6 +16750,13 @@ magic-string@^0.26.2:
   dependencies:
     sourcemap-codec "^1.4.8"
 
+magic-string@^0.27.0:
+  version "0.27.0"
+  resolved "https://registry.yarnpkg.com/magic-string/-/magic-string-0.27.0.tgz#e4a3413b4bab6d98d2becffd48b4a257effdbbf3"
+  integrity sha512-8UnnX2PeRAPZuN12svgR9j7M1uWMovg/CEnIwIG0LFkXSJJe4PdfUGiTGl8V9bsBHFUtfVINcSyYxd7q+kx9fA==
+  dependencies:
+    "@jridgewell/sourcemap-codec" "^1.4.13"
+
 make-dir@3.1.0, make-dir@^3.0.0, make-dir@^3.1.0:
   version "3.1.0"
   resolved "https://registry.yarnpkg.com/make-dir/-/make-dir-3.1.0.tgz#415e967046b3a7f1d185277d84aa58203726a13f"
@@ -21417,6 +21486,13 @@ rollup-pluginutils@^2.3.1, rollup-pluginutils@^2.5.0, rollup-pluginutils@^2.6.0,
   dependencies:
     estree-walker "^0.6.1"
 
+rollup@2.45.2:
+  version "2.45.2"
+  resolved "https://registry.yarnpkg.com/rollup/-/rollup-2.45.2.tgz#8fb85917c9f35605720e92328f3ccbfba6f78b48"
+  integrity sha512-kRRU7wXzFHUzBIv0GfoFFIN3m9oteY4uAsKllIpQDId5cfnkWF2J130l+27dzDju0E6MScKiV0ZM5Bw8m4blYQ==
+  optionalDependencies:
+    fsevents "~2.3.1"
+
 rollup@^2.36.2, rollup@^2.44.0, rollup@^2.45.2, rollup@^2.79.1:
   version "2.79.1"
   resolved "https://registry.yarnpkg.com/rollup/-/rollup-2.79.1.tgz#bedee8faef7c9f93a2647ac0108748f497f081c7"
@@ -23379,7 +23455,7 @@ timed-out@^4.0.1:
   resolved "https://registry.yarnpkg.com/timed-out/-/timed-out-4.0.1.tgz#f32eacac5a175bea25d7fab565ab3ed8741ef56f"
   integrity sha512-G7r3AhovYtr5YKOWQkta8RKAPb+J9IsO4uVmzjl8AZwfhs8UcUwTiD6gcJYSgOtzyjvQKrKYn41syHbUWMkafA==
 
-timekeeper@2.2.0:
+timekeeper@2.2.0, timekeeper@^2.2.0:
   version "2.2.0"
   resolved "https://registry.yarnpkg.com/timekeeper/-/timekeeper-2.2.0.tgz#9645731fce9e3280a18614a57a9d1b72af3ca368"
   integrity sha512-W3AmPTJWZkRwu+iSNxPIsLZ2ByADsOLbbLxe46UJyWj3mlYLlwucKiq+/dPm0l9wTzqoF3/2PH0AGFCebjq23A==