Merge branch 'develop' into tests/qa-22-improve-qa-core-boilerplate

.github/workflows/budibase_ci.yml

@@ -1,36 +1,25 @@
 name: Budibase CI
 
-on: 
+on:
- # Trigger the workflow on push or pull request,
+  # Trigger the workflow on push or pull request,
- # but only for the master branch
+  # but only for the master branch
- push:
+  push:
-    branches:
-      - master
-      - develop 
- pull_request:
     branches:
       - master
       - develop
- workflow_dispatch:
+  pull_request:
+    branches:
+      - master
+      - develop
+  workflow_dispatch:
 
 env:
   BRANCH: ${{ github.event.pull_request.head.ref }}
   BASE_BRANCH: ${{ github.event.pull_request.base.ref}}
-  PERSONAL_ACCESS_TOKEN : ${{ secrets.PERSONAL_ACCESS_TOKEN }}
+  PERSONAL_ACCESS_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
 
 jobs:
   lint:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-    - name: Use Node.js 14.x
-      uses: actions/setup-node@v1
-      with:
-        node-version: 14.x
-    - run: yarn
-    - run: yarn lint
-
-  build:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
@@ -38,8 +27,20 @@ jobs:
         uses: actions/setup-node@v1
         with:
           node-version: 14.x
-      - name: Install Pro
+      - run: yarn
-        run: yarn install:pro $BRANCH $BASE_BRANCH
+      - run: yarn lint
+
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          submodules: true
+          token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
+      - name: Use Node.js 14.x
+        uses: actions/setup-node@v1
+        with:
+          node-version: 14.x
       - run: yarn
       - run: yarn bootstrap
       - run: yarn build
@@ -48,16 +49,17 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
+        with:
+          submodules: true
+          token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
       - name: Use Node.js 14.x
         uses: actions/setup-node@v1
         with:
           node-version: 14.x
-      - name: Install Pro
-        run: yarn install:pro $BRANCH $BASE_BRANCH
       - run: yarn
       - run: yarn bootstrap
       - run: yarn build
-      - run: yarn test
+      - run: yarn test --ignore=@budibase/pro
       - uses: codecov/codecov-action@v3
         with:
           token: ${{ secrets.CODECOV_TOKEN }} # not required for public repos
@@ -68,26 +70,29 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
+        with:
+          submodules: true
+          token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
       - name: Use Node.js 14.x
         uses: actions/setup-node@v1
         with:
           node-version: 14.x
-      - name: Install Pro
-        run: yarn install:pro $BRANCH $BASE_BRANCH
       - run: yarn
       - run: yarn bootstrap
-      - run: yarn test:pro
+      - run: yarn build --scope=@budibase/types --scope=@budibase/shared-core
+      - run: yarn test --scope=@budibase/pro
 
   integration-test:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
+        with:
+          submodules: true
+          token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
       - name: Use Node.js 14.x
         uses: actions/setup-node@v1
         with:
           node-version: 14.x
-      - name: Install Pro
-        run: yarn install:pro $BRANCH $BASE_BRANCH
       - run: yarn && yarn bootstrap && yarn build
       - run: |
           cd qa-core
@@ -96,3 +101,24 @@ jobs:
         env:
           BB_ADMIN_USER_EMAIL: admin
           BB_ADMIN_USER_PASSWORD: admin
+
+  check-pro-submodule:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+        with:
+          submodules: true
+          token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
+          fetch-depth: 0
+      - name: Check submodule
+        run: |
+          cd packages/pro
+          git fetch
+          if ! git merge-base --is-ancestor $(git log -n 1 --pretty=format:%H) origin/develop; then
+            echo "Current commit has not been merged to develop"
+            echo "Refer to the pro repo to merge your changes: https://github.com/Budibase/budibase-pro/blob/develop/docs/getting_started.md"
+            exit 1
+          else
+            echo "All good, the submodule had been merged!"
+          fi

.github/workflows/release-develop.yml

@@ -1,21 +1,11 @@
 name: Budibase Prerelease
 concurrency: release-prerelease
 
-on: 
+on:
- push:
+  push:
-    branches:
+    tags:
-      - develop
+      - v*-alpha.*
-    paths:
+  workflow_dispatch:
-      - '.aws/**'
-      - '.github/**'
-      - 'charts/**'
-      - 'packages/**'
-      - 'scripts/**'
-      - 'package.json'
-      - 'yarn.lock'
-      - 'package.json'
-      - 'yarn.lock'
- workflow_dispatch:
 
 env:
   # Posthog token used by ui at build time
@@ -24,43 +14,60 @@ env:
   INTERCOM_TOKEN: ${{ secrets.INTERCOM_TOKEN }}
   PERSONAL_ACCESS_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
   FEATURE_PREVIEW_URL: https://budirelease.live
-      
+
 jobs:
   release-images:
-    runs-on: ubuntu-latest 
+    runs-on: ubuntu-latest
 
     steps:
-      - name: Fail if branch is not develop
-        if: github.ref != 'refs/heads/develop' 
-        run: | 
-          echo "Ref is not develop, you must run this job from develop."
-          exit 1
       - uses: actions/checkout@v2
+        with:
+          submodules: true
+          token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
+          fetch-depth: 0
+
+      - name: Fail if tag is not develop
+        run: |
+          if ! git merge-base --is-ancestor ${{ github.sha }} origin/develop; then
+            echo "Tag is not in develop"
+            exit 1
+          fi
+
       - uses: actions/setup-node@v1
         with:
           node-version: 14.x
 
-      - name: Install Pro
+      - run: yarn
-        run: yarn install:pro develop
+      - name: Update versions
-
+        run: |
-      - run: yarn 
+          version=$(cat lerna.json \
-      - run: yarn bootstrap
+            | grep version \
+            | head -1 \
+            | awk -F: '{gsub(/"/,"",$2);gsub(/[[:space:]]*/,"",$2); print $2}' \
+            | sed 's/[",]//g')
+          echo "Setting version $version"
+          yarn lerna exec "yarn version --no-git-tag-version --new-version=$version"
+          echo "Updating dependencies"
+          node scripts/syncLocalDependencies.js $version
+          echo "Syncing yarn workspace"
+          yarn
       - run: yarn build
       - run: yarn build:sdk
-#      - run: yarn test
 
       - name: Publish budibase packages to NPM
         env:
           NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
-        run: | 
+        run: |
           # setup the username and email.
           git config --global user.name "Budibase Staging Release Bot"
           git config --global user.email "<>"
+          git submodule foreach git commit -a -m 'Release process'
+          git commit -a -m 'Release process'
           echo //registry.npmjs.org/:_authToken=${NPM_TOKEN} >> .npmrc 
           yarn release:develop
 
       - name: Build/release Docker images
-        run: | 
+        run: |
           docker login -u $DOCKER_USER -p $DOCKER_PASSWORD
           yarn build:docker:develop
         env:
@@ -84,7 +91,7 @@ jobs:
           git config user.name "Budibase Helm Bot"
           git config user.email "<>"
           git reset --hard
-          git pull
+          git fetch
           mkdir sync
           echo "Packaging chart to sync dir"
           helm package charts/budibase --version 0.0.0-develop --app-version develop --destination sync

.github/workflows/release-master.yml

@@ -2,57 +2,60 @@ name: Budibase Release
 concurrency: release
 
 on:
- push:
+  push:
-    branches:
+    tags:
-      - master
+      - "v[0-9]+.[0-9]+.[0-9]+"
-    paths:
+      # Exclude all pre-releases
-      - '.aws/**'
+      - "!v*[0-9]+.[0-9]+.[0-9]+-*"
-      - '.github/**'
+  workflow_dispatch:
-      - 'charts/**'
+    inputs:
-      - 'packages/**'
+      tags:
-      - 'scripts/**'
+        description: "Release tag"
-      - 'package.json'
+        required: true
-      - 'yarn.lock'
+        type: boolean
-      - 'package.json'
-      - 'yarn.lock'
- workflow_dispatch:
-  inputs:
-    versioning:
-      type: choice
-      description: "Versioning type: patch, minor, major"
-      default: patch
-      options:
-        - patch
-        - minor
-        - major
-      required: true
 
 env:
-  # Posthog token used by ui at build time 
+  # Posthog token used by ui at build time
   POSTHOG_TOKEN: phc_bIjZL7oh2GEUd2vqvTBH8WvrX0fWTFQMs6H5KQxiUxU
   INTERCOM_TOKEN: ${{ secrets.INTERCOM_TOKEN }}
   SENTRY_DSN: ${{ secrets.SENTRY_DSN }}
-  PERSONAL_ACCESS_TOKEN : ${{ secrets.PERSONAL_ACCESS_TOKEN }}
+  PERSONAL_ACCESS_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
 
 jobs:
   release-images:
     runs-on: ubuntu-latest
     steps:
-      - name: Fail if branch is not master
-        if: github.ref != 'refs/heads/master' 
-        run: | 
-          echo "Ref is not master, you must run this job from master."
-          exit 1
       - uses: actions/checkout@v2
+        with:
+          submodules: true
+          token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
+          fetch-depth: 0
+
+      - name: Fail if branch is not master
+        if: github.ref != 'refs/heads/master'
+        run: |
+          echo "Ref is not master, you must run this job from master."
+          // Change to "exit 1" when merged. Left to 0 to not fail all the pipelines and not to cause noise
+          exit 0
+
       - uses: actions/setup-node@v1
         with:
           node-version: 14.x
 
-      - name: Install Pro
-        run: yarn install:pro master
-
       - run: yarn
-      - run: yarn bootstrap
+      - name: Update versions
+        run: |
+          version=$(cat lerna.json \
+            | grep version \
+            | head -1 \
+            | awk -F: '{gsub(/"/,"",$2);gsub(/[[:space:]]*/,"",$2); print $2}' \
+            | sed 's/[",]//g')
+          echo "Setting version $version"
+          yarn lerna exec "yarn version --no-git-tag-version --new-version=$version"
+          echo "Updating dependencies"
+          node scripts/syncLocalDependencies.js $version
+          echo "Syncing yarn workspace"
+          yarn
       - run: yarn lint
       - run: yarn build
       - run: yarn build:sdk
@@ -65,15 +68,17 @@ jobs:
           # setup the username and email. I tend to use 'GitHub Actions Bot' with no email by default
           git config --global user.name "Budibase Release Bot"
           git config --global user.email "<>"
+          git submodule foreach git commit -a -m 'Release process'
+          git commit -a -m 'Release process'
           echo //registry.npmjs.org/:_authToken=${NPM_TOKEN} >> .npmrc
           yarn release
 
-      - name: 'Get Previous tag'
+      - name: "Get Previous tag"
         id: previoustag
         uses: "WyriHaximus/github-action-get-previous-tag@v1"
 
       - name: Build/release Docker images
-        run: | 
+        run: |
           docker login -u $DOCKER_USER -p $DOCKER_PASSWORD
           yarn build:docker
         env:
@@ -103,7 +108,7 @@ jobs:
           git config user.name "Budibase Helm Bot"
           git config user.email "<>"
           git reset --hard
-          git pull
+          git fetch
           mkdir sync
           echo "Packaging chart to sync dir"
           helm package charts/budibase --version 0.0.0-master --app-version v"$RELEASE_VERSION" --destination sync

.github/workflows/tag-prerelease.yml

@@ -0,0 +1,39 @@
+name: Tag prerelease
+concurrency: release-prerelease
+
+on:
+  push:
+    branches:
+      - develop
+    paths:
+      - ".aws/**"
+      - ".github/**"
+      - "charts/**"
+      - "packages/**"
+      - "scripts/**"
+      - "package.json"
+      - "yarn.lock"
+  workflow_dispatch:
+
+jobs:
+  tag-prerelease:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Fail if branch is not develop
+        if: github.ref != 'refs/heads/develop'
+        run: |
+          echo "Ref is not develop, you must run this job from develop."
+          exit 1
+      - uses: actions/checkout@v2
+        with:
+          submodules: true
+          token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
+
+      - run: yarn
+      - name: Tag prerelease
+        run: |
+          # setup the username and email.
+          git config --global user.name "Budibase Staging Release Bot"
+          git config --global user.email "<>"
+          ./scripts/versionCommit.sh prerelease

.github/workflows/tag-release.yml

@@ -0,0 +1,49 @@
+name: Tag release
+concurrency: release-prerelease
+
+on:
+  push:
+    branches:
+      - master
+    paths:
+      - ".aws/**"
+      - ".github/**"
+      - "charts/**"
+      - "packages/**"
+      - "scripts/**"
+      - "package.json"
+      - "yarn.lock"
+  workflow_dispatch:
+    inputs:
+      versioning:
+        type: choice
+        description: "Versioning type: patch, minor, major"
+        default: patch
+        options:
+          - patch
+          - minor
+          - major
+        required: true
+
+jobs:
+  tag-prerelease:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Fail if branch is not master
+        if: github.ref != 'refs/heads/master'
+        run: |
+          echo "Ref is not master, you must run this job from master."
+          exit 1
+      - uses: actions/checkout@v2
+        with:
+          submodules: true
+          token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
+
+      - run: yarn
+      - name: Tag prerelease
+        run: |
+          # setup the username and email.
+          git config --global user.name "Budibase Staging Release Bot"
+          git config --global user.email "<>"
+          ./scripts/versionCommit.sh ${{ github.event.inputs.versioning }}

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "packages/pro"]
+	path = packages/pro
+	url = git@github.com:Budibase/budibase-pro.git

.husky/post-checkout

@@ -0,0 +1,4 @@
+# .husky/post-checkout
+# ...
+
+git config submodule.recurse true

docs/DEV-SETUP-DEBIAN.md

@@ -1,13 +1,17 @@
 ## Dev Environment on Debian 11
 
 ### Install NVM & Node 14
+
 NVM documentation: https://github.com/nvm-sh/nvm#installing-and-updating
 
 Install NVM
+
 ```
 curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.0/install.sh | bash
 ```
+
 Install Node 14
+
 ```
 nvm install 14
 ```
@@ -17,13 +21,16 @@ nvm install 14
 ```
 npm install -g yarn jest lerna
 ```
+
 ### Install Docker and Docker Compose
 
 ```
 apt install docker.io
 pip3 install docker-compose
 ```
+
 ### Clone the repo
+
 ```
 git clone https://github.com/Budibase/budibase.git
 ```
@@ -44,10 +51,13 @@ This setup process was tested on Debian 11 (bullseye) with version numbers show
 cd budibase
 yarn setup
 ```
+
 The yarn setup command runs several build steps i.e.
+
 ```
 node ./hosting/scripts/setup.js && yarn && yarn bootstrap && yarn build && yarn dev
 ```
+
 So this command will actually run the application in dev mode. It creates .env files under `./packages/server` and `./packages/worker` and runs docker containers for each service via docker-compose.
 
 The dev version will be available on port 10000 i.e.
@@ -55,6 +65,7 @@ The dev version will be available on port 10000 i.e.
 http://127.0.0.1:10000/builder/admin
 
 ### File descriptor issues with Vite and Chrome in Linux
+
 If your dev environment stalls forever, with some network requests stuck in flight, it's likely that Chrome is trying to open more file descriptors than your system allows.
 To fix this, apply the following tweaks.
 
@@ -62,4 +73,4 @@ Debian based distros:
 Add `* - nofile 65536` to `/etc/security/limits.conf`.
 
 Arch:
-Add `DefaultLimitNOFILE=65536` to `/etc/systemd/system.conf`.
+Add `DefaultLimitNOFILE=65536` to `/etc/systemd/system.conf`.

docs/DEV-SETUP-MACOSX.md

@@ -4,14 +4,14 @@
 
 Install instructions [here](https://brew.sh/)
 
-| **NOTE**: If you are working on a M1 Apple Silicon which is running Z shell, you could need to add 
+| **NOTE**: If you are working on a M1 Apple Silicon which is running Z shell, you could need to add
-`eval $(/opt/homebrew/bin/brew shellenv)` line to your `.zshrc`. This will make your zsh to find the apps you install 
+`eval $(/opt/homebrew/bin/brew shellenv)` line to your `.zshrc`. This will make your zsh to find the apps you install
 through brew.
 
-
 ### Install Node
 
 Budibase requires a recent version of node 14:
+
 ```
 brew install node npm
 node -v
@@ -22,12 +22,15 @@ node -v
 ```
 npm install -g yarn jest lerna
 ```
+
 ### Install Docker and Docker Compose
 
 ```
 brew install docker docker-compose
 ```
+
 ### Clone the repo
+
 ```
 git clone https://github.com/Budibase/budibase.git
 ```
@@ -48,10 +51,13 @@ This setup process was tested on Mac OSX 12 (Monterey) with version numbers show
 cd budibase
 yarn setup
 ```
+
 The yarn setup command runs several build steps i.e.
+
 ```
 node ./hosting/scripts/setup.js && yarn && yarn bootstrap && yarn build && yarn dev
 ```
+
 So this command will actually run the application in dev mode. It creates .env files under `./packages/server` and `./packages/worker` and runs docker containers for each service via docker-compose.
 
 The dev version will be available on port 10000 i.e.

docs/DEV-SETUP-WINDOWS.md

@@ -1,13 +1,15 @@
 ## Dev Environment on Windows 10/11 (WSL2)
 
-
 ### Install WSL with Ubuntu LTS
 
 Enable WSL 2 on Windows 10/11 for docker support.
+
 ```
 wsl --set-default-version 2
 ```
+
 Install Ubuntu LTS.
+
 ```
 wsl --install Ubuntu
 ```
@@ -16,6 +18,7 @@ Or follow the instruction here:
 https://learn.microsoft.com/en-us/windows/wsl/install
 
 ### Install Docker in windows
+
 Download the installer from docker and install it.
 
 Check this url for more detailed instructions:
@@ -24,18 +27,21 @@ https://docs.docker.com/desktop/install/windows-install/
 You should follow the next steps from within the Ubuntu terminal.
 
 ### Install NVM & Node 14
+
 NVM documentation: https://github.com/nvm-sh/nvm#installing-and-updating
 
 Install NVM
+
 ```
 curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.0/install.sh | bash
 ```
+
 Install Node 14
+
 ```
 nvm install 14
 ```
 
-
 ### Install npm requirements
 
 ```
@@ -43,6 +49,7 @@ npm install -g yarn jest lerna
 ```
 
 ### Clone the repo
+
 ```
 git clone https://github.com/Budibase/budibase.git
 ```
@@ -63,10 +70,13 @@ This setup process was tested on Windows 11 with version numbers show below. You
 cd budibase
 yarn setup
 ```
+
 The yarn setup command runs several build steps i.e.
+
 ```
 node ./hosting/scripts/setup.js && yarn && yarn bootstrap && yarn build && yarn dev
 ```
+
 So this command will actually run the application in dev mode. It creates .env files under `./packages/server` and `./packages/worker` and runs docker containers for each service via docker-compose.
 
 The dev version will be available on port 10000 i.e.
@@ -74,8 +84,9 @@ The dev version will be available on port 10000 i.e.
 http://127.0.0.1:10000/builder/admin
 
 ### Working with the code
+
 Here are the instructions to work on the application from within Visual Studio Code (in Windows) through the WSL. All the commands and files are within the Ubuntu system and it should run as if you were working on a Linux machine.
 
 https://code.visualstudio.com/docs/remote/wsl
 
-Note you will be able to run the application from within the WSL terminal and you will be able to access the application from the a browser in Windows.
+Note you will be able to run the application from within the WSL terminal and you will be able to access the application from the a browser in Windows.

hosting/couchdb/Dockerfile

@@ -5,8 +5,11 @@ ENV COUCHDB_PASSWORD admin
 EXPOSE 5984
 
 RUN apt-get update && apt-get install -y --no-install-recommends software-properties-common wget unzip curl && \
-    apt-add-repository 'deb http://security.debian.org/debian-security stretch/updates main' && \
+    wget -qO - https://adoptopenjdk.jfrog.io/adoptopenjdk/api/gpg/key/public | apt-key add - && \
-    apt-get update && apt-get install -y --no-install-recommends openjdk-8-jre && \
+    apt-add-repository 'deb http://security.debian.org/debian-security bullseye-security/updates main' && \
+    apt-add-repository 'deb http://archive.debian.org/debian stretch-backports main' && \
+    apt-add-repository --yes https://adoptopenjdk.jfrog.io/adoptopenjdk/deb/ && \
+    apt-get update && apt-get install -y --no-install-recommends adoptopenjdk-8-hotspot && \
     rm -rf /var/lib/apt/lists/
 
 # setup clouseau

hosting/single/Dockerfile

@@ -22,7 +22,7 @@ FROM budibase/couchdb
 ARG TARGETARCH
 ENV TARGETARCH $TARGETARCH
 #TARGETBUILD can be set to single (for single docker image) or aas (for azure app service)
-# e.g. docker build --build-arg TARGETBUILD=aas ....
+# e.g. docker build --build-arg TARGETBUILD=aas ....
 ARG TARGETBUILD=single
 ENV TARGETBUILD $TARGETBUILD
 
@@ -32,7 +32,7 @@ COPY --from=build /worker /worker
 # install base dependencies
 RUN apt-get update && \
   apt-get install -y --no-install-recommends software-properties-common nginx uuid-runtime redis-server && \
-  apt-add-repository 'deb http://security.debian.org/debian-security stretch/updates main' && \
+  apt-add-repository 'deb http://security.debian.org/debian-security bullseye-security/updates main' && \
   apt-get update
 
 # install other dependencies, nodejs, oracle requirements, jdk8, redis, nginx

lerna.json

@@ -1,8 +1,22 @@
 {
-  "version": "2.5.10-alpha.0",
+  "version": "2.6.8-alpha.9",
   "npmClient": "yarn",
+  "packages": [
+    "packages/backend-core",
+    "packages/bbui",
+    "packages/builder",
+    "packages/cli",
+    "packages/client",
+    "packages/frontend-core",
+    "packages/sdk",
+    "packages/server",
+    "packages/shared-core",
+    "packages/string-templates",
+    "packages/types",
+    "packages/worker",
+    "packages/pro/packages/pro"
+  ],
   "useWorkspaces": true,
-  "packages": ["packages/*"],
   "command": {
     "publish": {
       "ignoreChanges": [
@@ -17,4 +31,4 @@
       "loadEnvFiles": false
     }
   }
-}
+}

package.json

@@ -8,7 +8,7 @@
     "eslint": "^7.28.0",
     "eslint-plugin-cypress": "^2.11.3",
     "eslint-plugin-svelte3": "^3.2.0",
-    "husky": "^7.0.1",
+    "husky": "^8.0.3",
     "js-yaml": "^4.1.0",
     "kill-port": "^1.6.1",
     "lerna": "^6.6.1",
@@ -17,22 +17,22 @@
     "prettier-plugin-svelte": "^2.3.0",
     "rimraf": "^3.0.2",
     "rollup-plugin-replace": "^2.2.0",
+    "semver": "^7.5.0",
     "svelte": "^3.38.2",
     "typescript": "4.7.3"
   },
   "scripts": {
-    "setup": "node ./hosting/scripts/setup.js && yarn && yarn bootstrap && yarn build && yarn dev",
+    "preinstall": "node scripts/syncProPackage.js",
-    "bootstrap": "lerna link && ./scripts/link-dependencies.sh",
+    "setup": "git config submodule.recurse true && git submodule update && node ./hosting/scripts/setup.js && yarn && yarn bootstrap && yarn build && yarn dev",
+    "bootstrap": "./scripts/bootstrap.sh && lerna link && ./scripts/link-dependencies.sh",
     "build": "lerna run --stream build",
-    "build:dev": "lerna run --stream prebuild && tsc --build --watch --preserveWatchOutput",
+    "build:dev": "lerna run --stream prebuild && yarn nx run-many --target=build --output-style=dynamic --watch --preserveWatchOutput",
     "backend:bootstrap": "./scripts/scopeBackend.sh && yarn run bootstrap",
     "backend:build": "./scripts/scopeBackend.sh 'lerna run --stream build'",
     "build:sdk": "lerna run --stream build:sdk",
     "deps:circular": "madge packages/server/dist/index.js packages/worker/src/index.ts packages/backend-core/dist/src/index.js packages/cli/src/index.js --circular",
-    "release": "lerna publish ${RELEASE_VERSION_TYPE:-patch} --yes --force-publish && yarn release:pro",
+    "release": "lerna publish ${RELEASE_VERSION_TYPE:-patch} --yes --force-publish --no-git-tag-version --no-push --no-git-reset",
-    "release:develop": "lerna publish prerelease --yes --force-publish --dist-tag develop --exact && yarn release:pro:develop",
+    "release:develop": "lerna publish from-package --yes --force-publish --dist-tag develop --exact --no-git-tag-version --no-push --no-git-reset",
-    "release:pro": "bash scripts/pro/release.sh",
-    "release:pro:develop": "bash scripts/pro/release.sh develop",
     "restore": "yarn run clean && yarn run bootstrap && yarn run build",
     "nuke": "yarn run nuke:packages && yarn run nuke:docker",
     "nuke:packages": "yarn run restore",
@@ -46,7 +46,6 @@
     "dev:server": "yarn run kill-server && lerna run --stream --parallel dev:builder --concurrency 1 --scope @budibase/backend-core --scope @budibase/worker --scope @budibase/server",
     "dev:built": "yarn run kill-all && cd packages/server && yarn dev:stack:up && cd ../../ && lerna run --stream --parallel dev:built",
     "test": "lerna run --stream test --stream",
-    "test:pro": "bash scripts/pro/test.sh",
     "lint:eslint": "eslint packages && eslint qa-core",
     "lint:prettier": "prettier --check \"packages/**/*.{js,ts,svelte}\" && prettier --write \"examples/**/*.{js,ts,svelte}\" && prettier --check \"qa-core/**/*.{js,ts,svelte}\"",
     "lint": "yarn run lint:eslint && yarn run lint:prettier",
@@ -82,12 +81,25 @@
     "mode:account": "yarn mode:cloud && yarn env:account:enable",
     "security:audit": "node scripts/audit.js",
     "postinstall": "husky install",
-    "install:pro": "bash scripts/pro/install.sh",
+    "dep:clean": "yarn clean -y && yarn bootstrap",
-    "dep:clean": "yarn clean && yarn bootstrap"
+    "submodules:load": "git submodule init && git submodule update && yarn && yarn bootstrap",
+    "submodules:unload": "git submodule deinit --all && yarn && yarn bootstrap"
   },
   "workspaces": {
     "packages": [
-      "packages/*"
+      "packages/backend-core",
+      "packages/bbui",
+      "packages/builder",
+      "packages/cli",
+      "packages/client",
+      "packages/frontend-core",
+      "packages/sdk",
+      "packages/server",
+      "packages/shared-core",
+      "packages/string-templates",
+      "packages/types",
+      "packages/worker",
+      "packages/pro/packages/pro"
     ]
   }
 }

packages/backend-core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/backend-core",
-  "version": "2.5.10-alpha.0",
+  "version": "0.0.1",
   "description": "Budibase backend core libraries used in server and worker",
   "main": "dist/src/index.js",
   "types": "dist/src/index.d.ts",
@@ -15,8 +15,6 @@
     "prebuild": "rimraf dist/",
     "prepack": "cp package.json dist",
     "build": "tsc -p tsconfig.build.json",
-    "build:pro": "../../scripts/pro/build.sh",
-    "postbuild": "yarn run build:pro",
     "build:dev": "yarn prebuild && tsc --build --watch --preserveWatchOutput",
     "test": "bash scripts/test.sh",
     "test:watch": "jest --watchAll"
@@ -24,7 +22,7 @@
   "dependencies": {
     "@budibase/nano": "10.1.2",
     "@budibase/pouchdb-replication-stream": "1.2.10",
-    "@budibase/types": "2.5.10-alpha.0",
+    "@budibase/types": "0.0.1",
     "@shopify/jest-koa-mocks": "5.0.1",
     "@techpass/passport-openidconnect": "0.3.2",
     "aws-cloudfront-sign": "2.2.0",

packages/backend-core/src/environment.ts

@@ -96,6 +96,7 @@ const environment = {
   SALT_ROUNDS: process.env.SALT_ROUNDS,
   REDIS_URL: process.env.REDIS_URL || "localhost:6379",
   REDIS_PASSWORD: process.env.REDIS_PASSWORD || "budibase",
+  REDIS_CLUSTERED: process.env.REDIS_CLUSTERED,
   MOCK_REDIS: process.env.MOCK_REDIS,
   MINIO_ACCESS_KEY: process.env.MINIO_ACCESS_KEY,
   MINIO_SECRET_KEY: process.env.MINIO_SECRET_KEY,

packages/backend-core/src/queue/listeners.ts

@@ -40,6 +40,12 @@ function logging(queue: Queue, jobQueue: JobQueue) {
     case JobQueue.APP_BACKUP:
       eventType = "app-backup-event"
       break
+    case JobQueue.AUDIT_LOG:
+      eventType = "audit-log-event"
+      break
+    case JobQueue.SYSTEM_EVENT_QUEUE:
+      eventType = "system-event"
+      break
   }
   if (process.env.NODE_DEBUG?.includes("bull")) {
     queue

packages/backend-core/src/redis/redis.ts

@@ -12,7 +12,7 @@ import * as timers from "../timers"
 
 const RETRY_PERIOD_MS = 2000
 const STARTUP_TIMEOUT_MS = 5000
-const CLUSTERED = false
+const CLUSTERED = env.REDIS_CLUSTERED
 const DEFAULT_SELECT_DB = SelectableDatabase.DEFAULT
 
 // for testing just generate the client once
@@ -81,7 +81,7 @@ function init(selectDb = DEFAULT_SELECT_DB) {
   if (client) {
     client.disconnect()
   }
-  const { redisProtocolUrl, opts, host, port } = getRedisOptions(CLUSTERED)
+  const { redisProtocolUrl, opts, host, port } = getRedisOptions()
 
   if (CLUSTERED) {
     client = new Redis.Cluster([{ host, port }], opts)

packages/backend-core/src/redis/redlockImpl.ts

@@ -85,7 +85,7 @@ export const doWithLock = async <T>(
   opts: LockOptions,
   task: () => Promise<T>
 ): Promise<RedlockExecution<T>> => {
-  const redlock = await getClient(opts.type)
+  const redlock = await getClient(opts.type, opts.customOptions)
   let lock
   try {
     // determine lock name

packages/backend-core/src/redis/utils.ts

@@ -57,7 +57,7 @@ export enum SelectableDatabase {
   UNUSED_14 = 15,
 }
 
-export function getRedisOptions(clustered = false) {
+export function getRedisOptions() {
   let password = env.REDIS_PASSWORD
   let url: string[] | string = env.REDIS_URL.split("//")
   // get rid of the protocol
@@ -83,7 +83,7 @@ export function getRedisOptions(clustered = false) {
   const opts: any = {
     connectTimeout: CONNECT_TIMEOUT_MS,
   }
-  if (clustered) {
+  if (env.REDIS_CLUSTERED) {
     opts.redisOptions = {}
     opts.redisOptions.tls = {}
     opts.redisOptions.password = password

packages/backend-core/tsconfig.json

@@ -7,11 +7,6 @@
       "@budibase/types": ["../types/src"]
     }
   },
-  "references": [
+
-    { "path": "../types" }
+  "exclude": ["node_modules", "dist"]
-  ],
+}
-  "exclude": [
-    "node_modules",
-    "dist",
-  ]
-}

packages/bbui/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/bbui",
   "description": "A UI solution used in the different Budibase projects.",
-  "version": "2.5.10-alpha.0",
+  "version": "0.0.1",
   "license": "MPL-2.0",
   "svelte": "src/index.js",
   "module": "dist/bbui.es.js",
@@ -38,8 +38,8 @@
   ],
   "dependencies": {
     "@adobe/spectrum-css-workflow-icons": "1.2.1",
-    "@budibase/shared-core": "2.5.10-alpha.0",
+    "@budibase/shared-core": "0.0.1",
-    "@budibase/string-templates": "2.5.10-alpha.0",
+    "@budibase/string-templates": "0.0.1",
     "@spectrum-css/accordion": "3.0.24",
     "@spectrum-css/actionbutton": "1.0.1",
     "@spectrum-css/actiongroup": "1.0.1",

packages/bbui/src/Actions/click_outside.js

@@ -1,4 +1,8 @@
-const ignoredClasses = [".flatpickr-calendar", ".spectrum-Popover"]
+const ignoredClasses = [
+  ".flatpickr-calendar",
+  ".spectrum-Popover",
+  ".download-js-link",
+]
 let clickHandlers = []
 
 /**
@@ -22,8 +26,8 @@ const handleClick = event => {
     }
 
     // Ignore clicks for modals, unless the handler is registered from a modal
-    const sourceInModal = handler.anchor.closest(".spectrum-Modal") != null
+    const sourceInModal = handler.anchor.closest(".spectrum-Underlay") != null
-    const clickInModal = event.target.closest(".spectrum-Modal") != null
+    const clickInModal = event.target.closest(".spectrum-Underlay") != null
     if (clickInModal && !sourceInModal) {
       return
     }

packages/bbui/src/Form/Core/DatePicker.svelte

@@ -18,10 +18,14 @@
   export let ignoreTimezones = false
   export let time24hr = false
   export let range = false
+  export let flatpickr
+  export let useKeyboardShortcuts = true
+
   const dispatch = createEventDispatcher()
   const flatpickrId = `${uuid()}-wrapper`
+
   let open = false
-  let flatpickr, flatpickrOptions
+  let flatpickrOptions
 
   // Another classic flatpickr issue. Errors were randomly being thrown due to
   // flatpickr internal code. Making sure that "destroy" is a valid function
@@ -59,6 +63,8 @@
         dispatch("change", timestamp.toISOString())
       }
     },
+    onOpen: () => dispatch("open"),
+    onClose: () => dispatch("close"),
   }
 
   $: redrawOptions = {
@@ -113,12 +119,16 @@
 
   const onOpen = () => {
     open = true
-    document.addEventListener("keyup", clearDateOnBackspace)
+    if (useKeyboardShortcuts) {
+      document.addEventListener("keyup", clearDateOnBackspace)
+    }
   }
 
   const onClose = () => {
     open = false
-    document.removeEventListener("keyup", clearDateOnBackspace)
+    if (useKeyboardShortcuts) {
+      document.removeEventListener("keyup", clearDateOnBackspace)
+    }
 
     // Manually blur all input fields since flatpickr creates a second
     // duplicate input field.

packages/builder/assets/make.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 512 512">   <g id="Make-App-Icon-Circle" transform="translate(3757 -1767)">     <circle id="Ellipse_10" data-name="Ellipse 10" cx="256" cy="256" r="256" transform="translate(-3757 1767)" fill="#6d00cc"/>     <path id="Path_141560" data-name="Path 141560" d="M244.78,14.544a7.187,7.187,0,0,0-7.186,7.192V213.927a7.19,7.19,0,0,0,7.186,7.192h52.063a7.187,7.187,0,0,0,7.186-7.192V21.736a7.183,7.183,0,0,0-7.186-7.192ZM92.066,17.083,5.77,188.795a7.191,7.191,0,0,0,3.192,9.654l46.514,23.379a7.184,7.184,0,0,0,9.654-3.2l86.3-171.711a7.184,7.184,0,0,0-3.2-9.654L101.719,13.886a7.2,7.2,0,0,0-9.654,3.2m72.592.614L127.731,204.876a7.189,7.189,0,0,0,5.632,8.442l51.028,10.306a7.2,7.2,0,0,0,8.481-5.665L229.8,30.786a7.19,7.19,0,0,0-5.637-8.442L173.133,12.038a7.391,7.391,0,0,0-1.427-.144,7.194,7.194,0,0,0-7.048,5.8" transform="translate(-3676.356 1905.425)" fill="#fff"/>   </g> </svg>

packages/builder/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/builder",
-  "version": "2.5.10-alpha.0",
+  "version": "0.0.1",
   "license": "GPL-3.0",
   "private": true,
   "scripts": {
@@ -58,10 +58,10 @@
     }
   },
   "dependencies": {
-    "@budibase/bbui": "2.5.10-alpha.0",
+    "@budibase/bbui": "0.0.1",
-    "@budibase/frontend-core": "2.5.10-alpha.0",
+    "@budibase/frontend-core": "0.0.1",
-    "@budibase/shared-core": "2.5.10-alpha.0",
+    "@budibase/shared-core": "0.0.1",
-    "@budibase/string-templates": "2.5.10-alpha.0",
+    "@budibase/string-templates": "0.0.1",
     "@fortawesome/fontawesome-svg-core": "^6.2.1",
     "@fortawesome/free-brands-svg-icons": "^6.2.1",
     "@fortawesome/free-solid-svg-icons": "^6.2.1",

packages/builder/src/builderStore/store/automation/index.js

@@ -147,6 +147,9 @@ const automationActions = store => ({
       testData,
     })
     if (!result?.trigger && !result?.steps?.length) {
+      if (result?.err?.code === "usage_limit_exceeded") {
+        throw "You have exceeded your automation quota"
+      }
       throw "Something went wrong testing your automation"
     }
     store.update(state => {

packages/builder/src/components/automation/AutomationBuilder/FlowChart/ActionModal.svelte

@@ -94,7 +94,7 @@
             />
             <span class="icon-spacing">
               <Body size="XS">
-                {idx.charAt(0).toUpperCase() + idx.slice(1)}
+                {action.stepTitle || idx.charAt(0).toUpperCase() + idx.slice(1)}
               </Body>
             </span>
           </div>

packages/builder/src/components/automation/AutomationBuilder/FlowChart/ExternalActions.js

@@ -1,11 +1,11 @@
 import DiscordLogo from "assets/discord.svg"
 import ZapierLogo from "assets/zapier.png"
-import IntegromatLogo from "assets/integromat.png"
+import MakeLogo from "assets/make.svg"
 import SlackLogo from "assets/slack.svg"
 
 export const externalActions = {
   zapier: { name: "zapier", icon: ZapierLogo },
   discord: { name: "discord", icon: DiscordLogo },
   slack: { name: "slack", icon: SlackLogo },
-  integromat: { name: "integromat", icon: IntegromatLogo },
+  integromat: { name: "integromat", icon: MakeLogo },
 }

packages/builder/src/components/automation/AutomationBuilder/FlowChart/TestDataModal.svelte

@@ -52,7 +52,7 @@
       await automationStore.actions.test($selectedAutomation, testData)
       $automationStore.showTestPanel = true
     } catch (error) {
-      notifications.error("Error testing automation")
+      notifications.error(error)
     }
   }
 </script>

packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte

@@ -61,11 +61,63 @@
   $: isTrigger = block?.type === "TRIGGER"
   $: isUpdateRow = stepId === ActionStepID.UPDATE_ROW
 
+  /**
+   * TODO - Remove after November 2023
+   * *******************************
+   * Code added to provide backwards compatibility between Values 1,2,3,4,5
+   * and the new JSON body.
+   */
+  let deprecatedSchemaProperties
+  $: {
+    if (block?.stepId === "integromat" || block?.stepId === "zapier") {
+      deprecatedSchemaProperties = schemaProperties.filter(
+        prop => !prop[0].startsWith("value")
+      )
+      if (!deprecatedSchemaProperties.map(entry => entry[0]).includes("body")) {
+        deprecatedSchemaProperties.push([
+          "body",
+          {
+            title: "Payload",
+            type: "json",
+          },
+        ])
+      }
+    } else {
+      deprecatedSchemaProperties = schemaProperties
+    }
+  }
+  /****************************************************/
+
   const getInputData = (testData, blockInputs) => {
     let newInputData = testData || blockInputs
     if (block.event === "app:trigger" && !newInputData?.fields) {
       newInputData = cloneDeep(blockInputs)
     }
+
+    /**
+     * TODO - Remove after November 2023
+     * *******************************
+     * Code added to provide backwards compatibility between Values 1,2,3,4,5
+     * and the new JSON body.
+     */
+    if (
+      (block?.stepId === "integromat" || block?.stepId === "zapier") &&
+      !newInputData?.body?.value
+    ) {
+      let deprecatedValues = {
+        ...newInputData,
+      }
+      delete deprecatedValues.url
+      delete deprecatedValues.body
+      newInputData = {
+        url: newInputData.url,
+        body: {
+          value: JSON.stringify(deprecatedValues),
+        },
+      }
+    }
+    /**********************************/
+
     inputData = newInputData
     setDefaultEnumValues()
   }
@@ -239,7 +291,7 @@
 </script>
 
 <div class="fields">
-  {#each schemaProperties as [key, value]}
+  {#each deprecatedSchemaProperties as [key, value]}
     <div class="block-field">
       {#if key !== "fields"}
         <Label
@@ -256,6 +308,28 @@
           options={value.enum}
           getOptionLabel={(x, idx) => (value.pretty ? value.pretty[idx] : x)}
         />
+      {:else if value.type === "json"}
+        <Editor
+          editorHeight="250"
+          editorWidth="448"
+          mode="json"
+          value={inputData[key]?.value}
+          on:change={e => {
+            /**
+             * TODO - Remove after November 2023
+             * *******************************
+             * Code added to provide backwards compatibility between Values 1,2,3,4,5
+             * and the new JSON body.
+             */
+            delete inputData.value1
+            delete inputData.value2
+            delete inputData.value3
+            delete inputData.value4
+            delete inputData.value5
+            /***********************/
+            onChange(e, key)
+          }}
+        />
       {:else if value.customType === "column"}
         <Select
           on:change={e => onChange(e, key)}

packages/builder/src/components/backend/DataTable/buttons/ExistingRelationshipButton.svelte

@@ -39,7 +39,7 @@
 {#if datasource}
   <div>
     <ActionButton icon="DataCorrelated" primary quiet on:click={modal.show}>
-      Define existing relationship
+      Define relationship
     </ActionButton>
   </div>
   <Modal bind:this={modal}>

packages/builder/src/components/backend/DataTable/buttons/grid/GridExportButton.svelte

@@ -9,13 +9,21 @@
   $: selectedRowArray = Object.keys($selectedRows).map(id => ({ _id: id }))
 </script>
 
-<ExportButton
+<span data-ignore-click-outside="true">
-  {disabled}
+  <ExportButton
-  view={$tableId}
+    {disabled}
-  filters={$filter}
+    view={$tableId}
-  sorting={{
+    filters={$filter}
-    sortColumn: $sort.column,
+    sorting={{
-    sortOrder: $sort.order,
+      sortColumn: $sort.column,
-  }}
+      sortOrder: $sort.order,
-  selectedRows={selectedRowArray}
+    }}
-/>
+    selectedRows={selectedRowArray}
+  />
+</span>
+
+<style>
+  span {
+    display: contents;
+  }
+</style>

packages/builder/src/components/backend/DataTable/buttons/grid/GridFilterButton.svelte

@@ -2,19 +2,19 @@
   import TableFilterButton from "../TableFilterButton.svelte"
   import { getContext } from "svelte"
 
-  const { columns, config, filter, table } = getContext("grid")
+  const { columns, tableId, filter, table } = getContext("grid")
 
   const onFilter = e => {
     filter.set(e.detail || [])
   }
 </script>
 
-{#key $config.tableId}
+{#key $tableId}
   <TableFilterButton
     schema={$table?.schema}
     filters={$filter}
     on:change={onFilter}
     disabled={!$columns.length}
-    tableId={$config.tableId}
+    tableId={$tableId}
   />
 {/key}

packages/builder/src/components/backend/DataTable/buttons/grid/GridManageAccessButton.svelte

@@ -2,7 +2,7 @@
   import ManageAccessButton from "../ManageAccessButton.svelte"
   import { getContext } from "svelte"
 
-  const { config } = getContext("grid")
+  const { tableId } = getContext("grid")
 </script>
 
-<ManageAccessButton resourceId={$config.tableId} />
+<ManageAccessButton resourceId={$tableId} />

packages/builder/src/components/integration/QueryEditor.svelte

@@ -18,6 +18,7 @@
   export let tab = true
   export let mode
   export let editorHeight = 500
+  export let editorWidth = 640
   // export let parameters = []
 
   let width
@@ -169,7 +170,9 @@
 {#if label}
   <Label small>{label}</Label>
 {/if}
-<div style={`--code-mirror-height: ${editorHeight}px`}>
+<div
+  style={`--code-mirror-height: ${editorHeight}px; --code-mirror-width: ${editorWidth}px;`}
+>
   <textarea tabindex="0" bind:this={refs.editor} readonly {value} />
 </div>
 
@@ -183,6 +186,7 @@
   }
 
   div :global(.CodeMirror) {
+    width: var(--code-mirror-width) !important;
     height: var(--code-mirror-height) !important;
     border-radius: var(--border-radius-s);
     font-family: var(--font-mono);

packages/builder/src/components/portal/licensing/licensingBanners.js

@@ -147,6 +147,9 @@ const buildUsersAboveLimitBanner = EXPIRY_KEY => {
   return {
     key: EXPIRY_KEY,
     type: BANNER_TYPES.WARNING,
+    onChange: () => {
+      defaultCacheFn(EXPIRY_KEY)
+    },
     criteria: () => {
       return userLicensing.warnUserLimit
     },

packages/builder/src/pages/builder/portal/users/groups/[groupId].svelte

@@ -1,47 +1,28 @@
 <script>
-  import { url, goto } from "@roxi/routify"
   import {
-    Button,
+    ActionMenu,
-    Layout,
     Heading,
     Icon,
-    Popover,
+    Layout,
-    notifications,
-    Table,
-    ActionMenu,
     MenuItem,
     Modal,
+    Table,
+    notifications,
   } from "@budibase/bbui"
-  import UserGroupPicker from "components/settings/UserGroupPicker.svelte"
+  import { goto, url } from "@roxi/routify"
-  import { createPaginationStore } from "helpers/pagination"
-  import { users, apps, groups, auth, features } from "stores/portal"
-  import { onMount, setContext } from "svelte"
-  import { roles } from "stores/backend"
   import ConfirmDialog from "components/common/ConfirmDialog.svelte"
+  import { Breadcrumb, Breadcrumbs } from "components/portal/page"
+  import { roles } from "stores/backend"
+  import { apps, auth, features, groups } from "stores/portal"
+  import { onMount, setContext } from "svelte"
+  import AppNameTableRenderer from "../users/_components/AppNameTableRenderer.svelte"
+  import AppRoleTableRenderer from "../users/_components/AppRoleTableRenderer.svelte"
   import CreateEditGroupModal from "./_components/CreateEditGroupModal.svelte"
   import GroupIcon from "./_components/GroupIcon.svelte"
-  import { Breadcrumbs, Breadcrumb } from "components/portal/page"
+  import GroupUsers from "./_components/GroupUsers.svelte"
-  import AppNameTableRenderer from "../users/_components/AppNameTableRenderer.svelte"
-  import RemoveUserTableRenderer from "./_components/RemoveUserTableRenderer.svelte"
-  import AppRoleTableRenderer from "../users/_components/AppRoleTableRenderer.svelte"
-  import ScimBanner from "../_components/SCIMBanner.svelte"
 
   export let groupId
 
-  $: userSchema = {
-    email: {
-      width: "1fr",
-    },
-    ...(readonly
-      ? {}
-      : {
-          _id: {
-            displayName: "",
-            width: "auto",
-            borderLeft: true,
-          },
-        }),
-  }
   const appSchema = {
     name: {
       width: "2fr",
@@ -50,12 +31,6 @@
       width: "1fr",
     },
   }
-  const customUserTableRenderers = [
-    {
-      column: "_id",
-      component: RemoveUserTableRenderer,
-    },
-  ]
   const customAppTableRenderers = [
     {
       column: "name",
@@ -67,20 +42,12 @@
     },
   ]
 
-  let popoverAnchor
-  let popover
-  let searchTerm = ""
-  let prevSearch = undefined
-  let pageInfo = createPaginationStore()
   let loaded = false
   let editModal, deleteModal
 
   $: scimEnabled = $features.isScimEnabled
   $: readonly = !$auth.isAdmin || scimEnabled
-  $: page = $pageInfo.page
-  $: fetchUsers(page, searchTerm)
   $: group = $groups.find(x => x._id === groupId)
-  $: filtered = $users.data
   $: groupApps = $apps
     .filter(app =>
       groups.actions
@@ -97,25 +64,6 @@
     }
   }
 
-  async function fetchUsers(page, search) {
-    if ($pageInfo.loading) {
-      return
-    }
-    // need to remove the page if they've started searching
-    if (search && !prevSearch) {
-      pageInfo.reset()
-      page = undefined
-    }
-    prevSearch = search
-    try {
-      pageInfo.loading()
-      await users.search({ page, email: search })
-      pageInfo.fetched($users.hasNextPage, $users.nextPage)
-    } catch (error) {
-      notifications.error("Error getting user list")
-    }
-  }
-
   async function deleteGroup() {
     try {
       await groups.actions.delete(group)
@@ -130,21 +78,17 @@
     try {
       await groups.actions.save(group)
     } catch (error) {
-      notifications.error(`Failed to save user group`)
+      if (error.message) {
+        notifications.error(error.message)
+      } else {
+        notifications.error(`Failed to save user group`)
+      }
     }
   }
 
-  const removeUser = async id => {
-    await groups.actions.removeUser(groupId, id)
-  }
-
   const removeApp = async app => {
     await groups.actions.removeApp(groupId, apps.getProdAppID(app.devId))
   }
-
-  setContext("users", {
-    removeUser,
-  })
   setContext("roles", {
     updateRole: () => {},
     removeRole: removeApp,
@@ -186,41 +130,7 @@
     </div>
 
     <Layout noPadding gap="S">
-      <div class="header">
+      <GroupUsers {groupId} />
-        <Heading size="S">Users</Heading>
-        {#if !scimEnabled}
-          <div bind:this={popoverAnchor}>
-            <Button disabled={readonly} on:click={popover.show()} cta
-              >Add user</Button
-            >
-          </div>
-        {:else}
-          <ScimBanner />
-        {/if}
-        <Popover align="right" bind:this={popover} anchor={popoverAnchor}>
-          <UserGroupPicker
-            bind:searchTerm
-            labelKey="email"
-            selected={group.users?.map(user => user._id)}
-            list={$users.data}
-            on:select={e => groups.actions.addUser(groupId, e.detail)}
-            on:deselect={e => groups.actions.removeUser(groupId, e.detail)}
-          />
-        </Popover>
-      </div>
-
-      <Table
-        schema={userSchema}
-        data={group?.users}
-        allowEditRows={false}
-        customPlaceholder
-        customRenderers={customUserTableRenderers}
-        on:click={e => $goto(`../users/${e.detail._id}`)}
-      >
-        <div class="placeholder" slot="placeholder">
-          <Heading size="S">This user group doesn't have any users</Heading>
-        </div>
-      </Table>
     </Layout>
 
     <Layout noPadding gap="S">

packages/builder/src/pages/builder/portal/users/groups/_components/CreateEditGroupModal.svelte

@@ -9,15 +9,23 @@
 
   export let group
   export let saveGroup
+
+  let nameError
 </script>
 
 <ModalContent
-  onConfirm={() => saveGroup(group)}
+  onConfirm={() => {
+    if (!group.name?.trim()) {
+      nameError = "Group name cannot be empty"
+      return false
+    }
+    saveGroup(group)
+  }}
   size="M"
   title={group?._rev ? "Edit group" : "Create group"}
   confirmText="Save"
 >
-  <Input bind:value={group.name} label="Name" />
+  <Input bind:value={group.name} label="Name" error={nameError} />
   <div class="modal-format">
     <div class="modal-inner">
       <Body size="XS">Icon</Body>

packages/builder/src/pages/builder/portal/users/groups/_components/EditUserPicker.svelte

@@ -0,0 +1,59 @@
+<script>
+  import { Button, Popover, notifications } from "@budibase/bbui"
+  import UserGroupPicker from "components/settings/UserGroupPicker.svelte"
+  import { createPaginationStore } from "helpers/pagination"
+  import { auth, groups, users } from "stores/portal"
+
+  export let groupId
+  export let onUsersUpdated
+
+  let popoverAnchor
+  let popover
+  let searchTerm = ""
+  let prevSearch = undefined
+  let pageInfo = createPaginationStore()
+
+  $: readonly = !$auth.isAdmin
+  $: page = $pageInfo.page
+  $: searchUsers(page, searchTerm)
+  $: group = $groups.find(x => x._id === groupId)
+
+  async function searchUsers(page, search) {
+    if ($pageInfo.loading) {
+      return
+    }
+    // need to remove the page if they've started searching
+    if (search && !prevSearch) {
+      pageInfo.reset()
+      page = undefined
+    }
+    prevSearch = search
+    try {
+      pageInfo.loading()
+      await users.search({ page, email: search })
+      pageInfo.fetched($users.hasNextPage, $users.nextPage)
+    } catch (error) {
+      notifications.error("Error getting user list")
+    }
+  }
+</script>
+
+<div bind:this={popoverAnchor}>
+  <Button disabled={readonly} on:click={popover.show()} cta>Add user</Button>
+</div>
+<Popover align="right" bind:this={popover} anchor={popoverAnchor}>
+  <UserGroupPicker
+    bind:searchTerm
+    labelKey="email"
+    selected={group.users?.map(user => user._id)}
+    list={$users.data}
+    on:select={async e => {
+      await groups.actions.addUser(groupId, e.detail)
+      onUsersUpdated()
+    }}
+    on:deselect={async e => {
+      await groups.actions.removeUser(groupId, e.detail)
+      onUsersUpdated()
+    }}
+  />
+</Popover>

packages/builder/src/pages/builder/portal/users/groups/_components/GroupUsers.svelte

@@ -0,0 +1,112 @@
+<script>
+  import EditUserPicker from "./EditUserPicker.svelte"
+
+  import { Heading, Pagination, Table } from "@budibase/bbui"
+  import { fetchData } from "@budibase/frontend-core"
+  import { goto } from "@roxi/routify"
+  import { API } from "api"
+  import { auth, features, groups } from "stores/portal"
+  import { setContext } from "svelte"
+  import ScimBanner from "../../_components/SCIMBanner.svelte"
+  import RemoveUserTableRenderer from "../_components/RemoveUserTableRenderer.svelte"
+
+  export let groupId
+
+  const fetchGroupUsers = fetchData({
+    API,
+    datasource: {
+      type: "groupUser",
+    },
+    options: {
+      query: {
+        groupId,
+      },
+    },
+  })
+
+  $: userSchema = {
+    email: {
+      width: "1fr",
+    },
+    ...(readonly
+      ? {}
+      : {
+          _id: {
+            displayName: "",
+            width: "auto",
+            borderLeft: true,
+          },
+        }),
+  }
+  const customUserTableRenderers = [
+    {
+      column: "_id",
+      component: RemoveUserTableRenderer,
+    },
+  ]
+
+  $: scimEnabled = $features.isScimEnabled
+  $: readonly = !$auth.isAdmin || scimEnabled
+
+  const removeUser = async id => {
+    await groups.actions.removeUser(groupId, id)
+    fetchGroupUsers.refresh()
+  }
+
+  setContext("users", {
+    removeUser,
+  })
+</script>
+
+<div class="header">
+  <Heading size="S">Users</Heading>
+  {#if !scimEnabled}
+    <EditUserPicker {groupId} onUsersUpdated={fetchGroupUsers.getInitialData} />
+  {:else}
+    <ScimBanner />
+  {/if}
+</div>
+
+<Table
+  schema={userSchema}
+  data={$fetchGroupUsers?.rows}
+  allowEditRows={false}
+  customPlaceholder
+  customRenderers={customUserTableRenderers}
+  on:click={e => $goto(`../users/${e.detail._id}`)}
+>
+  <div class="placeholder" slot="placeholder">
+    <Heading size="S">This user group doesn't have any users</Heading>
+  </div>
+</Table>
+
+<div class="pagination">
+  <Pagination
+    page={$fetchGroupUsers.pageNumber + 1}
+    hasPrevPage={$fetchGroupUsers.loading
+      ? false
+      : $fetchGroupUsers.hasPrevPage}
+    hasNextPage={$fetchGroupUsers.loading
+      ? false
+      : $fetchGroupUsers.hasNextPage}
+    goToPrevPage={$fetchGroupUsers.loading ? null : fetchGroupUsers.prevPage}
+    goToNextPage={$fetchGroupUsers.loading ? null : fetchGroupUsers.nextPage}
+  />
+</div>
+
+<style>
+  .header {
+    display: flex;
+    flex-direction: row;
+    justify-content: flex-start;
+    align-items: center;
+    gap: var(--spacing-l);
+  }
+  .header :global(.spectrum-Heading) {
+    flex: 1 1 auto;
+  }
+  .placeholder {
+    width: 100%;
+    text-align: center;
+  }
+</style>

packages/builder/src/pages/builder/portal/users/groups/index.svelte

@@ -66,6 +66,8 @@
     } catch (error) {
       if (error.status === 400) {
         notifications.error(error.message)
+      } else if (error.message) {
+        notifications.error(error.message)
       } else {
         notifications.error(`Failed to save group`)
       }

packages/builder/src/pages/builder/portal/users/users/index.svelte

@@ -261,7 +261,7 @@
       header={`Users will soon be limited to ${staticUserLimit}`}
       message={`Our free plan is going to be limited to ${staticUserLimit} users in ${$licensing.userLimitDays}.
     
-    This means any users exceeding the limit have been de-activated.
+    This means any users exceeding the limit will be de-activated.
 
     De-activated users will not able to access the builder or any published apps until you upgrade to one of our paid plans.
     `}

packages/builder/src/stores/portal/groups.js

@@ -28,7 +28,7 @@ export function createGroupsStore() {
       // on the backend anyway
       if (get(licensing).groupsEnabled) {
         const groups = await API.getGroups()
-        store.set(groups)
+        store.set(groups.data)
       }
     },
 

packages/builder/tsconfig.json

@@ -13,9 +13,5 @@
   },
   "ts-node": {
     "require": ["tsconfig-paths/register"]
-  },
+  }
-  "references": [
+}
-    { "path": "../types" },
-    { "path": "../backend-core" },
-  ]
-}

packages/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/cli",
-  "version": "2.5.10-alpha.0",
+  "version": "0.0.1",
   "description": "Budibase CLI, for developers, self hosting and migrations.",
   "main": "dist/index.js",
   "bin": {
@@ -29,9 +29,9 @@
     "outputPath": "build"
   },
   "dependencies": {
-    "@budibase/backend-core": "2.5.10-alpha.0",
+    "@budibase/backend-core": "0.0.1",
-    "@budibase/string-templates": "2.5.10-alpha.0",
+    "@budibase/string-templates": "0.0.1",
-    "@budibase/types": "2.5.10-alpha.0",
+    "@budibase/types": "0.0.1",
     "axios": "0.21.2",
     "chalk": "4.1.0",
     "cli-progress": "3.11.2",

packages/client/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/client",
-  "version": "2.5.10-alpha.0",
+  "version": "0.0.1",
   "license": "MPL-2.0",
   "module": "dist/budibase-client.js",
   "main": "dist/budibase-client.js",
@@ -19,11 +19,11 @@
     "dev:builder": "rollup -cw"
   },
   "dependencies": {
-    "@budibase/bbui": "2.5.10-alpha.0",
+    "@budibase/bbui": "0.0.1",
-    "@budibase/frontend-core": "2.5.10-alpha.0",
+    "@budibase/frontend-core": "0.0.1",
-    "@budibase/shared-core": "2.5.10-alpha.0",
+    "@budibase/shared-core": "0.0.1",
-    "@budibase/string-templates": "2.5.10-alpha.0",
+    "@budibase/string-templates": "0.0.1",
-    "@budibase/types": "2.5.10-alpha.0",
+    "@budibase/types": "0.0.1",
     "@spectrum-css/button": "^3.0.3",
     "@spectrum-css/card": "^3.0.3",
     "@spectrum-css/divider": "^1.0.3",

packages/client/src/components/app/Button.svelte

@@ -5,6 +5,8 @@
   const { styleable, builderStore } = getContext("sdk")
   const component = getContext("component")
 
+  let handlingOnClick = false
+
   export let disabled = false
   export let text = ""
   export let onClick
@@ -16,6 +18,16 @@
   export let icon = null
   export let active = false
 
+  const handleOnClick = async () => {
+    handlingOnClick = true
+
+    if (onClick) {
+      await onClick()
+    }
+
+    handlingOnClick = false
+  }
+
   let node
 
   $: $component.editing && node?.focus()
@@ -37,9 +49,9 @@
   <button
     class={`spectrum-Button spectrum-Button--size${size} spectrum-Button--${type}`}
     class:spectrum-Button--quiet={quiet}
-    {disabled}
+    disabled={disabled || handlingOnClick}
     use:styleable={$component.styles}
-    on:click={onClick}
+    on:click={handleOnClick}
     contenteditable={$component.editing && !icon}
     on:blur={$component.editing ? updateText : null}
     bind:this={node}

packages/client/src/utils/buttonActions.js

@@ -384,7 +384,7 @@ const confirmTextMap = {
   ["Save Row"]: "Are you sure you want to save this row?",
   ["Execute Query"]: "Are you sure you want to execute this query?",
   ["Trigger Automation"]: "Are you sure you want to trigger this automation?",
-  ["Prompt User"]: "Are you sure you want to contiune?",
+  ["Prompt User"]: "Are you sure you want to continue?",
 }
 
 /**

packages/frontend-core/package.json

@@ -1,13 +1,13 @@
 {
   "name": "@budibase/frontend-core",
-  "version": "2.5.10-alpha.0",
+  "version": "0.0.1",
   "description": "Budibase frontend core libraries used in builder and client",
   "author": "Budibase",
   "license": "MPL-2.0",
   "svelte": "src/index.js",
   "dependencies": {
-    "@budibase/bbui": "2.5.10-alpha.0",
+    "@budibase/bbui": "0.0.1",
-    "@budibase/shared-core": "2.5.10-alpha.0",
+    "@budibase/shared-core": "0.0.1",
     "dayjs": "^1.11.7",
     "lodash": "^4.17.21",
     "socket.io-client": "^4.6.1",

packages/frontend-core/src/api/groups.js

@@ -52,6 +52,20 @@ export const buildGroupsEndpoints = API => {
       })
     },
 
+    /**
+     * Gets a group users by the group id
+     */
+    getGroupUsers: async ({ id, bookmark }) => {
+      let url = `/api/global/groups/${id}/users?`
+      if (bookmark) {
+        url += `bookmark=${bookmark}`
+      }
+
+      return await API.get({
+        url,
+      })
+    },
+
     /**
      * Adds users to a group
      * @param groupId The group to update

packages/frontend-core/src/components/grid/cells/DataCell.svelte

@@ -32,6 +32,7 @@
   $: readonly =
     column.schema.autocolumn ||
     column.schema.disabled ||
+    column.schema.type === "formula" ||
     (!$config.allowEditRows && row._id)
 
   // Register this cell API if the row is focused

packages/frontend-core/src/components/grid/cells/DateCell.svelte

@@ -1,12 +1,17 @@
 <script>
   import dayjs from "dayjs"
   import { CoreDatePicker, Icon } from "@budibase/bbui"
+  import { onMount } from "svelte"
 
   export let value
   export let schema
   export let onChange
   export let focused = false
   export let readonly = false
+  export let api
+
+  let flatpickr
+  let isOpen
 
   // adding the 0- will turn a string like 00:00:00 into a valid ISO
   // date, but will make actual ISO dates invalid
@@ -19,6 +24,26 @@
     ? "MMM D YYYY"
     : "MMM D YYYY, HH:mm"
   $: editable = focused && !readonly
+
+  // Ensure we close flatpickr when unselected
+  $: {
+    if (!focused) {
+      flatpickr?.close()
+    }
+  }
+
+  const onKeyDown = () => {
+    return isOpen
+  }
+
+  onMount(() => {
+    api = {
+      onKeyDown,
+      focus: () => flatpickr?.open(),
+      blur: () => flatpickr?.close(),
+      isActive: () => isOpen,
+    }
+  })
 </script>
 
 <div class="container">
@@ -42,6 +67,10 @@
       {timeOnly}
       time24hr
       ignoreTimezones={schema.ignoreTimezones}
+      bind:flatpickr
+      on:open={() => (isOpen = true)}
+      on:close={() => (isOpen = false)}
+      useKeyboardShortcuts={false}
     />
   </div>
 {/if}

packages/frontend-core/src/components/grid/cells/GutterCell.svelte

@@ -70,7 +70,15 @@
         </div>
       {/if}
     {/if}
-    {#if $config.allowExpandRows}
+    {#if rowSelected && $config.allowDeleteRows}
+      <div class="delete" on:click={() => dispatch("request-bulk-delete")}>
+        <Icon
+          name="Delete"
+          size="S"
+          color="var(--spectrum-global-color-red-400)"
+        />
+      </div>
+    {:else if $config.allowExpandRows}
       <div
         class="expand"
         class:visible={!disableExpand && (rowFocused || rowHovered)}
@@ -111,9 +119,12 @@
   .number.visible {
     display: flex;
   }
+  .delete,
+  .expand {
+    margin-right: 4px;
+  }
   .expand {
     opacity: 0;
-    margin-right: 4px;
   }
   .expand :global(.spectrum-Icon) {
     pointer-events: none;
@@ -124,4 +135,11 @@
   .expand.visible :global(.spectrum-Icon) {
     pointer-events: all;
   }
+
+  .delete:hover {
+    cursor: pointer;
+  }
+  .delete:hover :global(.spectrum-Icon) {
+    color: var(--spectrum-global-color-red-600) !important;
+  }
 </style>

packages/frontend-core/src/components/grid/controls/BulkDeleteHandler.svelte

@@ -1,8 +1,8 @@
 <script>
-  import { Modal, ModalContent, Button, notifications } from "@budibase/bbui"
+  import { Modal, ModalContent, notifications } from "@budibase/bbui"
   import { getContext, onMount } from "svelte"
 
-  const { selectedRows, rows, config, subscribe } = getContext("grid")
+  const { selectedRows, rows, subscribe } = getContext("grid")
 
   let modal
 
@@ -27,20 +27,6 @@
   onMount(() => subscribe("request-bulk-delete", () => modal?.show()))
 </script>
 
-{#if selectedRowCount}
-  <div class="delete-button" data-ignore-click-outside="true">
-    <Button
-      icon="Delete"
-      size="M"
-      on:click={modal.show}
-      disabled={!$config.allowEditRows}
-      cta
-    >
-      Delete {selectedRowCount} row{selectedRowCount === 1 ? "" : "s"}
-    </Button>
-  </div>
-{/if}
-
 <Modal bind:this={modal}>
   <ModalContent
     title="Delete rows"
@@ -53,14 +39,3 @@
     row{selectedRowCount === 1 ? "" : "s"}?
   </ModalContent>
 </Modal>
-
-<style>
-  .delete-button :global(.spectrum-Button:not(:disabled)) {
-    background-color: var(--spectrum-global-color-red-400);
-    border-color: var(--spectrum-global-color-red-400);
-  }
-  .delete-button :global(.spectrum-Button:not(:disabled):hover) {
-    background-color: var(--spectrum-global-color-red-500);
-    border-color: var(--spectrum-global-color-red-500);
-  }
-</style>

packages/frontend-core/src/components/grid/controls/ColumnWidthButton.svelte

@@ -3,7 +3,7 @@
   import { ActionButton, Popover } from "@budibase/bbui"
   import { DefaultColumnWidth } from "../lib/constants"
 
-  const { stickyColumn, columns } = getContext("grid")
+  const { stickyColumn, columns, compact } = getContext("grid")
   const smallSize = 120
   const mediumSize = DefaultColumnWidth
   const largeSize = DefaultColumnWidth * 1.5
@@ -59,12 +59,13 @@
     on:click={() => (open = !open)}
     selected={open}
     disabled={!allCols.length}
+    tooltip={$compact ? "Width" : null}
   >
-    Width
+    {$compact ? "" : "Width"}
   </ActionButton>
 </div>
 
-<Popover bind:open {anchor} align="left">
+<Popover bind:open {anchor} align={$compact ? "right" : "left"}>
   <div class="content">
     {#each sizeOptions as option}
       <ActionButton

packages/frontend-core/src/components/grid/controls/HideColumnsButton.svelte

@@ -1,8 +1,9 @@
 <script>
   import { getContext } from "svelte"
-  import { ActionButton, Popover, Toggle } from "@budibase/bbui"
+  import { ActionButton, Popover, Toggle, Icon } from "@budibase/bbui"
+  import { getColumnIcon } from "../lib/utils"
 
-  const { columns, stickyColumn } = getContext("grid")
+  const { columns, stickyColumn, compact } = getContext("grid")
 
   let open = false
   let anchor
@@ -47,25 +48,32 @@
     on:click={() => (open = !open)}
     selected={open || anyHidden}
     disabled={!$columns.length}
+    tooltip={$compact ? "Columns" : ""}
   >
-    Columns
+    {$compact ? "" : "Columns"}
   </ActionButton>
 </div>
 
-<Popover bind:open {anchor} align="left">
+<Popover bind:open {anchor} align={$compact ? "right" : "left"}>
   <div class="content">
     <div class="columns">
       {#if $stickyColumn}
+        <div class="column">
+          <Icon size="S" name={getColumnIcon($stickyColumn)} />
+          {$stickyColumn.label}
+        </div>
         <Toggle disabled size="S" value={true} />
-        <span>{$stickyColumn.label}</span>
       {/if}
       {#each $columns as column}
+        <div class="column">
+          <Icon size="S" name={getColumnIcon(column)} />
+          {column.label}
+        </div>
         <Toggle
           size="S"
           value={column.visible}
           on:change={e => toggleVisibility(column, e.detail)}
         />
-        <span>{column.label}</span>
       {/each}
     </div>
     <div class="buttons">
@@ -90,6 +98,13 @@
   .columns {
     display: grid;
     align-items: center;
-    grid-template-columns: auto 1fr;
+    grid-template-columns: 1fr auto;
+  }
+  .columns :global(.spectrum-Switch) {
+    margin-right: 0;
+  }
+  .column {
+    display: flex;
+    gap: 8px;
   }
 </style>

packages/frontend-core/src/components/grid/controls/RowHeightButton.svelte

@@ -7,7 +7,7 @@
     SmallRowHeight,
   } from "../lib/constants"
 
-  const { rowHeight, columns, table } = getContext("grid")
+  const { rowHeight, columns, table, compact } = getContext("grid")
   const sizeOptions = [
     {
       label: "Small",
@@ -41,12 +41,13 @@
     size="M"
     on:click={() => (open = !open)}
     selected={open}
+    tooltip={$compact ? "Height" : null}
   >
-    Height
+    {$compact ? "" : "Height"}
   </ActionButton>
 </div>
 
-<Popover bind:open {anchor} align="left">
+<Popover bind:open {anchor} align={$compact ? "right" : "left"}>
   <div class="content">
     {#each sizeOptions as option}
       <ActionButton

packages/frontend-core/src/components/grid/controls/SortButton.svelte

@@ -2,7 +2,7 @@
   import { getContext } from "svelte"
   import { ActionButton, Popover, Select } from "@budibase/bbui"
 
-  const { sort, columns, stickyColumn } = getContext("grid")
+  const { sort, columns, stickyColumn, compact } = getContext("grid")
 
   let open = false
   let anchor
@@ -90,12 +90,13 @@
     on:click={() => (open = !open)}
     selected={open}
     disabled={!columnOptions.length}
+    tooltip={$compact ? "Sort" : ""}
   >
-    Sort
+    {$compact ? "" : "Sort"}
   </ActionButton>
 </div>
 
-<Popover bind:open {anchor} align="left">
+<Popover bind:open {anchor} align={$compact ? "right" : "left"}>
   <div class="content">
     <Select
       placeholder={null}

packages/frontend-core/src/components/grid/layout/Grid.svelte

@@ -6,7 +6,7 @@
   import { createEventManagers } from "../lib/events"
   import { createAPIClient } from "../../../api"
   import { attachStores } from "../stores"
-  import DeleteButton from "../controls/DeleteButton.svelte"
+  import BulkDeleteHandler from "../controls/BulkDeleteHandler.svelte"
   import BetaButton from "../controls/BetaButton.svelte"
   import GridBody from "./GridBody.svelte"
   import ResizeOverlay from "../overlays/ResizeOverlay.svelte"
@@ -112,13 +112,12 @@
       <AddRowButton />
       <AddColumnButton />
       <slot name="controls" />
+      <SortButton />
+      <HideColumnsButton />
       <ColumnWidthButton />
       <RowHeightButton />
-      <HideColumnsButton />
-      <SortButton />
     </div>
     <div class="controls-right">
-      <DeleteButton />
       <UserAvatars />
     </div>
   </div>
@@ -131,7 +130,9 @@
           <GridBody />
         </div>
         <BetaButton />
-        <NewRow />
+        {#if allowAddRows}
+          <NewRow />
+        {/if}
         <div class="overlays">
           <ResizeOverlay />
           <ReorderOverlay />
@@ -146,6 +147,9 @@
       <ProgressCircle />
     </div>
   {/if}
+  {#if allowDeleteRows}
+    <BulkDeleteHandler />
+  {/if}
   <KeyboardManager />
 </div>
 
@@ -214,6 +218,7 @@
     padding: var(--cell-padding);
     gap: var(--cell-spacing);
     background: var(--background);
+    z-index: 2;
   }
   .controls-left,
   .controls-right {
@@ -239,7 +244,7 @@
     height: 100%;
     display: grid;
     place-items: center;
-    z-index: 10;
+    z-index: 100;
   }
   .grid-loading:before {
     content: "";

packages/frontend-core/src/components/grid/layout/HeaderRow.svelte

@@ -4,8 +4,14 @@
   import HeaderCell from "../cells/HeaderCell.svelte"
   import { Icon } from "@budibase/bbui"
 
-  const { renderedColumns, dispatch, scroll, hiddenColumnsWidth, width } =
+  const {
-    getContext("grid")
+    renderedColumns,
+    dispatch,
+    scroll,
+    hiddenColumnsWidth,
+    width,
+    config,
+  } = getContext("grid")
 
   $: columnsWidth = $renderedColumns.reduce(
     (total, col) => (total += col.width),
@@ -23,13 +29,15 @@
       {/each}
     </div>
   </GridScrollWrapper>
-  <div
+  {#if $config.allowAddColumns}
-    class="add"
+    <div
-    style="left:{left}px"
+      class="add"
-    on:click={() => dispatch("add-column")}
+      style="left:{left}px"
-  >
+      on:click={() => dispatch("add-column")}
-    <Icon name="Add" />
+    >
-  </div>
+      <Icon name="Add" />
+    </div>
+  {/if}
 </div>
 
 <style>
@@ -38,7 +46,6 @@
     border-bottom: var(--cell-border);
     position: relative;
     height: var(--default-row-height);
-    z-index: 1;
   }
   .row {
     display: flex;
@@ -54,6 +61,7 @@
     border-right: var(--cell-border);
     border-bottom: var(--cell-border);
     background: var(--spectrum-global-color-gray-100);
+    z-index: 20;
   }
   .add:hover {
     background: var(--spectrum-global-color-gray-200);

packages/frontend-core/src/components/grid/layout/NewRow.svelte

@@ -270,7 +270,7 @@
     z-index: 3;
     position: absolute;
     top: calc(var(--row-height) + var(--offset) + 24px);
-    left: var(--gutter-width);
+    left: 18px;
   }
   .button-with-keys {
     display: flex;

packages/frontend-core/src/components/grid/lib/utils.js

@@ -21,6 +21,9 @@ const TypeIconMap = {
 }
 
 export const getColumnIcon = column => {
+  if (column.schema.autocolumn) {
+    return "MagicWand"
+  }
   const type = column.schema.type
   return TypeIconMap[type] || "Text"
 }

packages/frontend-core/src/components/grid/overlays/KeyboardManager.svelte

@@ -13,6 +13,7 @@
     clipboard,
     dispatch,
     selectedRows,
+    config,
   } = getContext("grid")
 
   const ignoredOriginSelectors = [
@@ -37,10 +38,12 @@
         e.preventDefault()
         focusFirstCell()
       } else if (e.key === "Enter" && (e.ctrlKey || e.metaKey)) {
-        e.preventDefault()
+        if ($config.allowAddRows) {
-        dispatch("add-row-inline")
+          e.preventDefault()
+          dispatch("add-row-inline")
+        }
       } else if (e.key === "Delete" || e.key === "Backspace") {
-        if (Object.keys($selectedRows).length) {
+        if (Object.keys($selectedRows).length && $config.allowDeleteRows) {
           dispatch("request-bulk-delete")
         }
       }
@@ -88,7 +91,9 @@
           }
           break
         case "Enter":
-          dispatch("add-row-inline")
+          if ($config.allowAddRows) {
+            dispatch("add-row-inline")
+          }
       }
     } else {
       switch (e.key) {
@@ -106,7 +111,7 @@
           break
         case "Delete":
         case "Backspace":
-          if (Object.keys($selectedRows).length) {
+          if (Object.keys($selectedRows).length && $config.allowDeleteRows) {
             dispatch("request-bulk-delete")
           } else {
             deleteSelectedCell()
@@ -117,7 +122,9 @@
           break
         case " ":
         case "Space":
-          toggleSelectRow()
+          if ($config.allowDeleteRows) {
+            toggleSelectRow()
+          }
           break
         default:
           startEnteringValue(e.key, e.which)

packages/frontend-core/src/components/grid/overlays/MenuOverlay.svelte

@@ -1,6 +1,13 @@
 <script>
-  import { clickOutside, Menu, MenuItem, notifications } from "@budibase/bbui"
+  import {
+    clickOutside,
+    Menu,
+    MenuItem,
+    Helpers,
+    notifications,
+  } from "@budibase/bbui"
   import { getContext } from "svelte"
+  import { NewRowID } from "../lib/constants"
 
   const {
     focusedRow,
@@ -14,9 +21,11 @@
     clipboard,
     dispatch,
     focusedCellAPI,
+    focusedRowId,
   } = getContext("grid")
 
   $: style = makeStyle($menu)
+  $: isNewRow = $focusedRowId === NewRowID
 
   const makeStyle = menu => {
     return `left:${menu.left}px; top:${menu.top}px;`
@@ -36,6 +45,11 @@
       $focusedCellId = `${newRow._id}-${column}`
     }
   }
+
+  const copyToClipboard = async value => {
+    await Helpers.copyToClipboard(value)
+    notifications.success("Copied to clipboard")
+  }
 </script>
 
 {#if $menu.visible}
@@ -58,22 +72,38 @@
       </MenuItem>
       <MenuItem
         icon="Maximize"
-        disabled={!$config.allowEditRows}
+        disabled={isNewRow || !$config.allowEditRows}
         on:click={() => dispatch("edit-row", $focusedRow)}
         on:click={menu.actions.close}
       >
         Edit row in modal
       </MenuItem>
+      <MenuItem
+        icon="Copy"
+        disabled={isNewRow || !$focusedRow?._id}
+        on:click={() => copyToClipboard($focusedRow?._id)}
+        on:click={menu.actions.close}
+      >
+        Copy row _id
+      </MenuItem>
+      <MenuItem
+        icon="Copy"
+        disabled={isNewRow || !$focusedRow?._rev}
+        on:click={() => copyToClipboard($focusedRow?._rev)}
+        on:click={menu.actions.close}
+      >
+        Copy row _rev
+      </MenuItem>
       <MenuItem
         icon="Duplicate"
-        disabled={!$config.allowAddRows}
+        disabled={isNewRow || !$config.allowAddRows}
         on:click={duplicate}
       >
         Duplicate row
       </MenuItem>
       <MenuItem
         icon="Delete"
-        disabled={!$config.allowDeleteRows}
+        disabled={isNewRow || !$config.allowDeleteRows}
         on:click={deleteRow}
       >
         Delete row

packages/frontend-core/src/components/grid/stores/rows.js

@@ -338,15 +338,11 @@ export const deriveStores = context => {
         ...state,
         [rowId]: true,
       }))
-      const newRow = { ...row, ...get(rowChangeCache)[rowId] }
+      const saved = await API.saveRow({ ...row, ...get(rowChangeCache)[rowId] })
-      const saved = await API.saveRow(newRow)
 
       // Update state after a successful change
       rows.update(state => {
-        state[index] = {
+        state[index] = saved
-          ...newRow,
-          _rev: saved._rev,
-        }
         return state.slice()
       })
       rowChangeCache.update(state => {

packages/frontend-core/src/components/grid/stores/ui.js

@@ -2,6 +2,7 @@ import { writable, get, derived } from "svelte/store"
 import { tick } from "svelte"
 import {
   DefaultRowHeight,
+  GutterWidth,
   LargeRowHeight,
   MediumRowHeight,
   NewRowID,
@@ -43,6 +44,8 @@ export const deriveStores = context => {
     enrichedRows,
     rowLookupMap,
     rowHeight,
+    stickyColumn,
+    width,
   } = context
 
   // Derive the row that contains the selected cell
@@ -70,6 +73,7 @@ export const deriveStores = context => {
     hoveredRowId.set(null)
   }
 
+  // Derive the amount of content lines to show in cells depending on row height
   const contentLines = derived(rowHeight, $rowHeight => {
     if ($rowHeight === LargeRowHeight) {
       return 3
@@ -79,9 +83,15 @@ export const deriveStores = context => {
     return 1
   })
 
+  // Derive whether we should use the compact UI, depending on width
+  const compact = derived([stickyColumn, width], ([$stickyColumn, $width]) => {
+    return ($stickyColumn?.width || 0) + $width + GutterWidth < 1100
+  })
+
   return {
     focusedRow,
     contentLines,
+    compact,
     ui: {
       actions: {
         blur,

packages/frontend-core/src/fetch/DataFetch.js

@@ -362,13 +362,35 @@ export default class DataFetch {
       return
     }
     this.store.update($store => ({ ...$store, loading: true }))
-    const { rows, info, error } = await this.getPage()
+    const { rows, info, error, cursor } = await this.getPage()
+
+    let { cursors } = get(this.store)
+    const { pageNumber } = get(this.store)
+
+    if (!rows.length && pageNumber > 0) {
+      // If the full page is gone but we have previous pages, navigate to the previous page
+      this.store.update($store => ({
+        ...$store,
+        loading: false,
+        cursors: cursors.slice(0, pageNumber),
+      }))
+      return await this.prevPage()
+    }
+
+    const currentNextCursor = cursors[pageNumber + 1]
+    if (currentNextCursor != cursor) {
+      // If the current cursor changed, all the next pages need to be updated, so we mark them as stale
+      cursors = cursors.slice(0, pageNumber + 1)
+      cursors[pageNumber + 1] = cursor
+    }
+
     this.store.update($store => ({
       ...$store,
       rows,
       info,
       loading: false,
       error,
+      cursors,
     }))
   }
 

packages/frontend-core/src/fetch/GroupUserFetch.js

@@ -0,0 +1,50 @@
+import { get } from "svelte/store"
+import DataFetch from "./DataFetch.js"
+import { TableNames } from "../constants"
+
+export default class GroupUserFetch extends DataFetch {
+  constructor(opts) {
+    super({
+      ...opts,
+      datasource: {
+        tableId: TableNames.USERS,
+      },
+    })
+  }
+
+  determineFeatureFlags() {
+    return {
+      supportsSearch: true,
+      supportsSort: false,
+      supportsPagination: true,
+    }
+  }
+
+  async getDefinition() {
+    return {
+      schema: {},
+    }
+  }
+
+  async getData() {
+    const { query, cursor } = get(this.store)
+    try {
+      const res = await this.API.getGroupUsers({
+        id: query.groupId,
+        bookmark: cursor,
+      })
+
+      return {
+        rows: res?.users || [],
+        hasNextPage: res?.hasNextPage || false,
+        cursor: res?.bookmark || null,
+      }
+    } catch (error) {
+      return {
+        rows: [],
+        hasNextPage: false,
+        error,
+      }
+    }
+  }
+}

packages/frontend-core/src/fetch/fetchData.js

@@ -6,6 +6,7 @@ import NestedProviderFetch from "./NestedProviderFetch.js"
 import FieldFetch from "./FieldFetch.js"
 import JSONArrayFetch from "./JSONArrayFetch.js"
 import UserFetch from "./UserFetch.js"
+import GroupUserFetch from "./GroupUserFetch.js"
 
 const DataFetchMap = {
   table: TableFetch,
@@ -13,6 +14,7 @@ const DataFetchMap = {
   query: QueryFetch,
   link: RelationshipFetch,
   user: UserFetch,
+  groupUser: GroupUserFetch,
 
   // Client specific datasource types
   provider: NestedProviderFetch,

packages/pro

@@ -0,0 +1 @@
+Subproject commit c57a98d246a50a43905d8572a88c901ec598390c

packages/sdk/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/sdk",
-  "version": "2.5.10-alpha.0",
+  "version": "0.0.1",
   "description": "Budibase Public API SDK",
   "author": "Budibase",
   "license": "MPL-2.0",

packages/server/jest.config.ts

@@ -20,9 +20,9 @@ const baseConfig: Config.InitialProjectOptions = {
 }
 
 // add pro sources if they exist
-if (fs.existsSync("../../../budibase-pro")) {
+if (fs.existsSync("../pro/packages")) {
-  baseConfig.moduleNameMapper["@budibase/pro"] =
+  baseConfig.moduleNameMapper!["@budibase/pro"] =
-    "<rootDir>/../../../budibase-pro/packages/pro/src"
+    "<rootDir>/../pro/packages/pro/src"
 }
 
 const config: Config.InitialOptions = {

packages/server/nodemon.json

@@ -1,6 +1,10 @@
 {
-  "watch": ["src", "../backend-core", "../../../budibase-pro/packages/pro"],
+  "watch": ["src", "../backend-core", "../pro/packages/pro"],
   "ext": "js,ts,json",
-  "ignore": ["src/**/*.spec.ts", "src/**/*.spec.js", "../backend-core/dist/**/*"],
+  "ignore": [
+    "src/**/*.spec.ts",
+    "src/**/*.spec.js",
+    "../backend-core/dist/**/*"
+  ],
   "exec": "ts-node src/index.ts"
-}
+}

packages/server/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "2.5.10-alpha.0",
+  "version": "0.0.1",
   "description": "Budibase Web Server",
   "main": "src/index.ts",
   "repository": {
@@ -45,12 +45,12 @@
   "license": "GPL-3.0",
   "dependencies": {
     "@apidevtools/swagger-parser": "10.0.3",
-    "@budibase/backend-core": "2.5.10-alpha.0",
+    "@budibase/backend-core": "0.0.1",
-    "@budibase/client": "2.5.10-alpha.0",
+    "@budibase/client": "0.0.1",
-    "@budibase/pro": "2.5.10-alpha.0",
+    "@budibase/pro": "0.0.1",
-    "@budibase/shared-core": "2.5.10-alpha.0",
+    "@budibase/shared-core": "0.0.1",
-    "@budibase/string-templates": "2.5.10-alpha.0",
+    "@budibase/string-templates": "0.0.1",
-    "@budibase/types": "2.5.10-alpha.0",
+    "@budibase/types": "0.0.1",
     "@bull-board/api": "3.7.0",
     "@bull-board/koa": "3.9.4",
     "@elastic/elasticsearch": "7.10.0",

packages/server/src/api/controllers/row/internal.ts

@@ -37,7 +37,7 @@ import {
   Table,
 } from "@budibase/types"
 
-const { cleanExportRows } = require("./utils")
+import { cleanExportRows } from "./utils"
 
 const CALCULATION_TYPES = {
   SUM: "sum",
@@ -391,6 +391,9 @@ export async function exportRows(ctx: UserCtx) {
   const table = await db.get(ctx.params.tableId)
   const rowIds = ctx.request.body.rows
   let format = ctx.query.format
+  if (typeof format !== "string") {
+    ctx.throw(400, "Format parameter is not valid")
+  }
   const { columns, query } = ctx.request.body
 
   let result

packages/server/src/api/controllers/row/utils.ts

@@ -69,9 +69,9 @@ export async function validate({
     if (type === FieldTypes.FORMULA || column.autocolumn) {
       continue
     }
-    // special case for options, need to always allow unselected (null)
+    // special case for options, need to always allow unselected (empty)
     if (type === FieldTypes.OPTIONS && constraints.inclusion) {
-      constraints.inclusion.push(null)
+      constraints.inclusion.push(null, "")
     }
     let res
 
@@ -137,8 +137,8 @@ export function cleanExportRows(
     delete schema[column]
   })
 
-  // Intended to avoid 'undefined' in export
   if (format === Format.CSV) {
+    // Intended to append empty values in export
     const schemaKeys = Object.keys(schema)
     for (let key of schemaKeys) {
       if (columns?.length && columns.indexOf(key) > 0) {
@@ -146,7 +146,7 @@ export function cleanExportRows(
       }
       for (let row of cleanRows) {
         if (row[key] == null) {
-          row[key] = ""
+          row[key] = undefined
         }
       }
     }

packages/server/src/api/controllers/table/index.ts

@@ -10,7 +10,7 @@ import { getDatasourceParams } from "../../../db/utils"
 import { context, events } from "@budibase/backend-core"
 import { Table, UserCtx } from "@budibase/types"
 import sdk from "../../../sdk"
-import csv from "csvtojson"
+import { jsonFromCsvString } from "../../../utilities/csv"
 
 function pickApi({ tableId, table }: { tableId?: string; table?: Table }) {
   if (table && !tableId) {
@@ -104,7 +104,7 @@ export async function bulkImport(ctx: UserCtx) {
 export async function csvToJson(ctx: UserCtx) {
   const { csvString } = ctx.request.body
 
-  const result = await csv().fromString(csvString)
+  const result = await jsonFromCsvString(csvString)
 
   ctx.status = 200
   ctx.body = result

packages/server/src/api/controllers/view/exporters.ts

@@ -10,7 +10,9 @@ export function csv(headers: string[], rows: Row[]) {
         val =
           typeof val === "object" && !(val instanceof Date)
             ? `"${JSON.stringify(val).replace(/"/g, "'")}"`
-            : `"${val}"`
+            : val !== undefined
+            ? `"${val}"`
+            : ""
         return val.trim()
       })
       .join(",")}`

packages/server/src/api/routes/public/index.ts

@@ -42,8 +42,14 @@ if (!env.isTest()) {
         host: REDIS_OPTS.host,
         port: REDIS_OPTS.port,
       },
-      password: REDIS_OPTS.opts.password,
+      password:
-      database: 1,
+        REDIS_OPTS.opts.password || REDIS_OPTS.opts.redisOptions.password,
+    }
+
+    if (!env.REDIS_CLUSTERED) {
+      // Can't set direct redis db in clustered env
+      // @ts-ignore
+      options.database = 1
     }
   }
   rateLimitStore = new Stores.Redis(options)

packages/server/src/api/routes/tests/row.spec.js

@@ -212,6 +212,7 @@ describe("/rows", () => {
           attachmentNull: attachment,
           attachmentUndefined: attachment,
           attachmentEmpty: attachment,
+          attachmentEmptyArrayStr: attachment
         },
       })
 
@@ -239,6 +240,7 @@ describe("/rows", () => {
         attachmentNull: null,
         attachmentUndefined: undefined,
         attachmentEmpty: "",
+        attachmentEmptyArrayStr: "[]",
       }
 
       const id = (await config.createRow(row))._id
@@ -268,6 +270,7 @@ describe("/rows", () => {
       expect(saved.attachmentNull).toEqual([])
       expect(saved.attachmentUndefined).toBe(undefined)
       expect(saved.attachmentEmpty).toEqual([])
+      expect(saved.attachmentEmptyArrayStr).toEqual([])
     })
   })
 

packages/server/src/automations/actions.ts

@@ -9,7 +9,7 @@ import * as serverLog from "./steps/serverLog"
 import * as discord from "./steps/discord"
 import * as slack from "./steps/slack"
 import * as zapier from "./steps/zapier"
-import * as integromat from "./steps/integromat"
+import * as make from "./steps/make"
 import * as filter from "./steps/filter"
 import * as delay from "./steps/delay"
 import * as queryRow from "./steps/queryRows"
@@ -43,7 +43,7 @@ const ACTION_IMPLS: Record<
   discord: discord.run,
   slack: slack.run,
   zapier: zapier.run,
-  integromat: integromat.run,
+  integromat: make.run,
 }
 export const BUILTIN_ACTION_DEFINITIONS: Record<string, AutomationStepSchema> =
   {
@@ -63,7 +63,7 @@ export const BUILTIN_ACTION_DEFINITIONS: Record<string, AutomationStepSchema> =
     discord: discord.definition,
     slack: slack.definition,
     zapier: zapier.definition,
-    integromat: integromat.definition,
+    integromat: make.definition,
   }
 
 // don't add the bash script/definitions unless in self host

packages/server/src/automations/steps/make.ts

@@ -9,10 +9,11 @@ import {
 } from "@budibase/types"
 
 export const definition: AutomationStepSchema = {
-  name: "Integromat Integration",
+  name: "Make Integration",
-  tagline: "Trigger an Integromat scenario",
+  stepTitle: "Make",
+  tagline: "Trigger a Make scenario",
   description:
-    "Performs a webhook call to Integromat and gets the response (if configured)",
+    "Performs a webhook call to Make and gets the response (if configured)",
   icon: "ri-shut-down-line",
   stepId: AutomationActionStepId.integromat,
   type: AutomationStepType.ACTION,
@@ -25,6 +26,10 @@ export const definition: AutomationStepSchema = {
           type: AutomationIOType.STRING,
           title: "Webhook URL",
         },
+        body: {
+          type: AutomationIOType.JSON,
+          title: "Payload",
+        },
         value1: {
           type: AutomationIOType.STRING,
           title: "Input Value 1",
@@ -69,7 +74,19 @@ export const definition: AutomationStepSchema = {
 }
 
 export async function run({ inputs }: AutomationStepInput) {
-  const { url, value1, value2, value3, value4, value5 } = inputs
+  //TODO - Remove deprecated values 1,2,3,4,5 after November 2023
+  const { url, value1, value2, value3, value4, value5, body } = inputs
+
+  let payload = {}
+  try {
+    payload = body?.value ? JSON.parse(body?.value) : {}
+  } catch (err) {
+    return {
+      httpStatus: 400,
+      response: "Invalid payload JSON",
+      success: false,
+    }
+  }
 
   if (!url?.trim()?.length) {
     return {
@@ -88,6 +105,7 @@ export async function run({ inputs }: AutomationStepInput) {
         value3,
         value4,
         value5,
+        ...payload,
       }),
       headers: {
         "Content-Type": "application/json",

packages/server/src/automations/steps/zapier.ts

@@ -24,6 +24,10 @@ export const definition: AutomationStepSchema = {
           type: AutomationIOType.STRING,
           title: "Webhook URL",
         },
+        body: {
+          type: AutomationIOType.JSON,
+          title: "Payload",
+        },
         value1: {
           type: AutomationIOType.STRING,
           title: "Payload Value 1",
@@ -63,7 +67,19 @@ export const definition: AutomationStepSchema = {
 }
 
 export async function run({ inputs }: AutomationStepInput) {
-  const { url, value1, value2, value3, value4, value5 } = inputs
+  //TODO - Remove deprecated values 1,2,3,4,5 after November 2023
+  const { url, value1, value2, value3, value4, value5, body } = inputs
+
+  let payload = {}
+  try {
+    payload = body?.value ? JSON.parse(body?.value) : {}
+  } catch (err) {
+    return {
+      httpStatus: 400,
+      response: "Invalid payload JSON",
+      success: false,
+    }
+  }
 
   if (!url?.trim()?.length) {
     return {
@@ -85,6 +101,7 @@ export async function run({ inputs }: AutomationStepInput) {
         value3,
         value4,
         value5,
+        ...payload,
       }),
       headers: {
         "Content-Type": "application/json",

packages/server/src/automations/tests/make.spec.ts

@@ -0,0 +1,54 @@
+import { getConfig, afterAll, runStep, actions } from "./utilities"
+
+describe("test the outgoing webhook action", () => {
+  let config = getConfig()
+
+  beforeAll(async () => {
+    await config.init()
+  })
+
+  afterAll()
+
+  it("should be able to run the action", async () => {
+    const res = await runStep(actions.integromat.stepId, {
+      value1: "test",
+      url: "http://www.test.com",
+    })
+    expect(res.response.url).toEqual("http://www.test.com")
+    expect(res.response.method).toEqual("post")
+    expect(res.success).toEqual(true)
+  })
+
+  it("should add the payload props when a JSON string is provided", async () => {
+    const payload = `{"value1":1,"value2":2,"value3":3,"value4":4,"value5":5,"name":"Adam","age":9}`
+    const res = await runStep(actions.integromat.stepId, {
+      value1: "ONE",
+      value2: "TWO",
+      value3: "THREE",
+      value4: "FOUR",
+      value5: "FIVE",
+      body: {
+        value: payload,
+      },
+      url: "http://www.test.com",
+    })
+    expect(res.response.url).toEqual("http://www.test.com")
+    expect(res.response.method).toEqual("post")
+    expect(res.response.body).toEqual(payload)
+    expect(res.success).toEqual(true)
+  })
+
+  it("should return a 400 if the JSON payload string is malformed", async () => {
+    const payload = `{ value1 1 }`
+    const res = await runStep(actions.integromat.stepId, {
+      value1: "ONE",
+      body: {
+        value: payload,
+      },
+      url: "http://www.test.com",
+    })
+    expect(res.httpStatus).toEqual(400)
+    expect(res.response).toEqual("Invalid payload JSON")
+    expect(res.success).toEqual(false)
+  })
+})

packages/server/src/automations/tests/zapier.spec.js

@@ -1,27 +0,0 @@
-const setup = require("./utilities")
-const fetch = require("node-fetch")
-
-jest.mock("node-fetch")
-
-describe("test the outgoing webhook action", () => {
-    let inputs
-    let config = setup.getConfig()
-
-    beforeAll(async () => {
-        await config.init()
-        inputs = {
-            value1: "test",
-            url: "http://www.test.com",
-        }
-    })
-
-    afterAll(setup.afterAll)
-
-    it("should be able to run the action", async () => {
-        const res = await setup.runStep(setup.actions.zapier.stepId, inputs)
-        expect(res.response.url).toEqual("http://www.test.com")
-        expect(res.response.method).toEqual("post")
-        expect(res.success).toEqual(true)
-    })
-
-})

packages/server/src/automations/tests/zapier.spec.ts

@@ -0,0 +1,56 @@
+import { getConfig, afterAll, runStep, actions } from "./utilities"
+
+describe("test the outgoing webhook action", () => {
+  let config = getConfig()
+
+  beforeAll(async () => {
+    await config.init()
+  })
+
+  afterAll()
+
+  it("should be able to run the action", async () => {
+    const res = await runStep(actions.zapier.stepId, {
+      value1: "test",
+      url: "http://www.test.com",
+    })
+    expect(res.response.url).toEqual("http://www.test.com")
+    expect(res.response.method).toEqual("post")
+    expect(res.success).toEqual(true)
+  })
+
+  it("should add the payload props when a JSON string is provided", async () => {
+    const payload = `{ "value1": 1, "value2": 2, "value3": 3, "value4": 4, "value5": 5, "name": "Adam", "age": 9 }`
+    const res = await runStep(actions.zapier.stepId, {
+      value1: "ONE",
+      value2: "TWO",
+      value3: "THREE",
+      value4: "FOUR",
+      value5: "FIVE",
+      body: {
+        value: payload,
+      },
+      url: "http://www.test.com",
+    })
+    expect(res.response.url).toEqual("http://www.test.com")
+    expect(res.response.method).toEqual("post")
+    expect(res.response.body).toEqual(
+      `{"platform":"budibase","value1":1,"value2":2,"value3":3,"value4":4,"value5":5,"name":"Adam","age":9}`
+    )
+    expect(res.success).toEqual(true)
+  })
+
+  it("should return a 400 if the JSON payload string is malformed", async () => {
+    const payload = `{ value1 1 }`
+    const res = await runStep(actions.zapier.stepId, {
+      value1: "ONE",
+      body: {
+        value: payload,
+      },
+      url: "http://www.test.com",
+    })
+    expect(res.httpStatus).toEqual(400)
+    expect(res.response).toEqual("Invalid payload JSON")
+    expect(res.success).toEqual(false)
+  })
+})

packages/server/src/environment.ts

@@ -34,8 +34,6 @@ function parseIntSafe(number?: string) {
   }
 }
 
-let inThread = false
-
 const environment = {
   // important - prefer app port to generic port
   PORT: process.env.APP_PORT || process.env.PORT,
@@ -47,6 +45,7 @@ const environment = {
   MINIO_SECRET_KEY: process.env.MINIO_SECRET_KEY,
   REDIS_URL: process.env.REDIS_URL,
   REDIS_PASSWORD: process.env.REDIS_PASSWORD,
+  REDIS_CLUSTERED: process.env.REDIS_CLUSTERED,
   HTTP_MIGRATIONS: process.env.HTTP_MIGRATIONS,
   API_REQ_LIMIT_PER_SEC: process.env.API_REQ_LIMIT_PER_SEC,
   GOOGLE_CLIENT_ID: process.env.GOOGLE_CLIENT_ID,
@@ -94,12 +93,8 @@ const environment = {
   isProd: () => {
     return !isDev()
   },
-  // used to check if already in a thread, don't thread further
-  setInThread: () => {
-    inThread = true
-  },
   isInThread: () => {
-    return inThread
+    return process.env.FORKED_PROCESS
   },
 }
 

packages/server/src/integrations/googlesheets.ts

@@ -15,7 +15,7 @@ import {
 } from "@budibase/types"
 import { OAuth2Client } from "google-auth-library"
 import { buildExternalTableId, finaliseExternalTables } from "./utils"
-import { GoogleSpreadsheet } from "google-spreadsheet"
+import { GoogleSpreadsheet, GoogleSpreadsheetRow } from "google-spreadsheet"
 import fetch from "node-fetch"
 import { configs, HTTPError } from "@budibase/backend-core"
 import { dataFilters } from "@budibase/shared-core"
@@ -434,7 +434,20 @@ class GoogleSheetsIntegration implements DatasourcePlus {
     try {
       await this.connect()
       const sheet = this.client.sheetsByTitle[query.sheet]
-      const rows = await sheet.getRows()
+      let rows: GoogleSpreadsheetRow[] = []
+      if (query.paginate) {
+        const limit = query.paginate.limit || 100
+        let page: number =
+          typeof query.paginate.page === "number"
+            ? query.paginate.page
+            : parseInt(query.paginate.page || "1")
+        rows = await sheet.getRows({
+          limit,
+          offset: (page - 1) * limit,
+        })
+      } else {
+        rows = await sheet.getRows()
+      }
       const filtered = dataFilters.runLuceneQuery(rows, query.filters)
       const headerValues = sheet.headerValues
       let response = []

packages/server/src/threads/index.ts

@@ -39,6 +39,12 @@ export class Thread {
       const workerOpts: any = {
         autoStart: true,
         maxConcurrentWorkers: this.count,
+        workerOptions: {
+          env: {
+            ...process.env,
+            FORKED_PROCESS: "1",
+          },
+        },
       }
       if (opts.timeoutMs) {
         this.timeoutMs = opts.timeoutMs

packages/server/src/threads/utils.ts

@@ -25,11 +25,9 @@ function makeVariableKey(queryId: string, variable: string) {
 
 export function threadSetup() {
   // don't run this if not threading
-  if (env.isTest() || env.DISABLE_THREADING) {
+  if (env.isTest() || env.DISABLE_THREADING || !env.isInThread()) {
     return
   }
-  // when thread starts, make sure it is recorded
-  env.setInThread()
   db.init()
 }
 

packages/server/src/utilities/csv.ts

@@ -0,0 +1,22 @@
+import csv from "csvtojson"
+
+export async function jsonFromCsvString(csvString: string) {
+  const castedWithEmptyValues = await csv({ ignoreEmpty: true }).fromString(
+    csvString
+  )
+
+  // By default the csvtojson library casts empty values as empty strings. This is causing issues on conversion.
+  // ignoreEmpty will remove the key completly if empty, so creating this empty object will ensure we return the values with the keys but empty values
+  const result = await csv({ ignoreEmpty: false }).fromString(csvString)
+  result.forEach((r, i) => {
+    for (const [key] of Object.entries(r).filter(
+      ([key, value]) => value === ""
+    )) {
+      if (castedWithEmptyValues[i][key] === undefined) {
+        r[key] = null
+      }
+    }
+  })
+
+  return result
+}

packages/server/src/utilities/rowProcessor/map.ts

@@ -1,5 +1,6 @@
 // @ts-nocheck
 import { FieldTypes } from "../../constants"
+import { logging } from "@budibase/backend-core"
 
 /**
  * A map of how we convert various properties in rows to each other based on the row type.
@@ -67,9 +68,23 @@ export const TYPE_TRANSFORM_MAP: any = {
     },
   },
   [FieldTypes.ATTACHMENT]: {
-    "": [],
     [null]: [],
     [undefined]: undefined,
+    parse: attachments => {
+      if (typeof attachments === "string") {
+        if (attachments === "") {
+          return []
+        }
+        let result
+        try {
+          result = JSON.parse(attachments)
+        } catch (e) {
+          logging.logAlert("Could not parse attachments", e)
+        }
+        return result
+      }
+      return attachments
+    },
   },
   [FieldTypes.BOOLEAN]: {
     "": null,

packages/server/src/utilities/schema.ts

@@ -4,6 +4,9 @@ interface SchemaColumn {
   readonly name: string
   readonly type: FieldTypes
   readonly autocolumn?: boolean
+  readonly constraints?: {
+    presence: boolean
+  }
 }
 
 interface Schema {
@@ -76,6 +79,11 @@ export function validate(rows: Rows, schema: Schema): ValidationResults {
       // If the columnType is not a string, then it's not present in the schema, and should be added to the invalid columns array
       if (typeof columnType !== "string") {
         results.invalidColumns.push(columnName)
+      } else if (
+        columnData == null &&
+        !schema[columnName].constraints?.presence
+      ) {
+        results.schemaValidation[columnName] = true
       } else if (
         // If there's no data for this field don't bother with further checks
         // If the field is already marked as invalid there's no need for further checks

packages/server/src/utilities/tests/csv.spec.ts

@@ -0,0 +1,33 @@
+import { jsonFromCsvString } from "../csv"
+
+describe("csv", () => {
+  describe("jsonFromCsvString", () => {
+    test("multiple lines csv can be casted", async () => {
+      const csvString = '"id","title"\n"1","aaa"\n"2","bbb"'
+
+      const result = await jsonFromCsvString(csvString)
+
+      expect(result).toEqual([
+        { id: "1", title: "aaa" },
+        { id: "2", title: "bbb" },
+      ])
+      result.forEach(r => expect(Object.keys(r)).toEqual(["id", "title"]))
+    })
+
+    test("empty values are casted as undefined", async () => {
+      const csvString =
+        '"id","optional","title"\n1,,"aaa"\n2,"value","bbb"\n3,,"ccc"'
+
+      const result = await jsonFromCsvString(csvString)
+
+      expect(result).toEqual([
+        { id: "1", optional: null, title: "aaa" },
+        { id: "2", optional: "value", title: "bbb" },
+        { id: "3", optional: null, title: "ccc" },
+      ])
+      result.forEach(r =>
+        expect(Object.keys(r)).toEqual(["id", "optional", "title"])
+      )
+    })
+  })
+})

packages/server/tsconfig.json

@@ -11,7 +11,7 @@
       "@budibase/backend-core": ["../backend-core/src"],
       "@budibase/backend-core/*": ["../backend-core/*"],
       "@budibase/shared-core": ["../shared-core/src"],
-      "@budibase/pro": ["../../../budibase-pro/packages/pro/src"]
+      "@budibase/pro": ["../pro/packages/pro/src"]
     }
   },
   "ts-node": {

packages/shared-core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/shared-core",
-  "version": "2.5.10-alpha.0",
+  "version": "0.0.1",
   "description": "Shared data utils",
   "main": "dist/cjs/src/index.js",
   "types": "dist/mjs/src/index.d.ts",
@@ -20,7 +20,7 @@
     "dev:builder": "yarn prebuild && concurrently \"tsc -p tsconfig.build.json --watch\" \"tsc -p tsconfig-cjs.build.json --watch\""
   },
   "dependencies": {
-    "@budibase/types": "2.5.10-alpha.0"
+    "@budibase/types": "0.0.1"
   },
   "devDependencies": {
     "concurrently": "^7.6.0",

packages/shared-core/tsconfig.json

@@ -6,6 +6,5 @@
     "paths": {
       "@budibase/types": ["../types/src"]
     }
-  },
+  }
-  "references": [{ "path": "../types" }]
 }

packages/string-templates/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/string-templates",
-  "version": "2.5.10-alpha.0",
+  "version": "0.0.1",
   "description": "Handlebars wrapper for Budibase templating.",
   "main": "src/index.cjs",
   "module": "dist/bundle.mjs",