Merge branch 'master' of github.com:Budibase/budibase into fix/internal-db-enrich-perf

.dockerignore

@@ -1,9 +1,14 @@
-packages/server/node_modules
-packages/builder
-packages/frontend-core
-packages/backend-core
-packages/worker/node_modules
-packages/cli
-packages/client
-packages/bbui
-packages/string-templates
+*
+!/packages/
+!/scripts/
+/packages/*/node_modules
+packages/server/scripts/
+!packages/server/scripts/integrations/oracle
+!nx.json
+!/hosting/single/
+!/hosting/letsencrypt /
+
+!package.json
+!yarn.lock
+!lerna.json
+!.yarnrc

.github/workflows/close-featurebranch.yml

@@ -20,7 +20,7 @@ jobs:
       - uses: passeidireto/trigger-external-workflow-action@main
         env:
           PAYLOAD_BRANCH: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.BRANCH || github.head_ref }}
-          PAYLOAD_PR_NUMBER: ${{ github.ref }}
+          PAYLOAD_PR_NUMBER: ${{ github.event.pull_request.number }}
         with:
           repository: budibase/budibase-deploys
           event: featurebranch-qa-close

.github/workflows/release-master.yml

@@ -14,7 +14,6 @@ env:
   # Posthog token used by ui at build time
   POSTHOG_TOKEN: phc_bIjZL7oh2GEUd2vqvTBH8WvrX0fWTFQMs6H5KQxiUxU
   INTERCOM_TOKEN: ${{ secrets.INTERCOM_TOKEN }}
-  SENTRY_DSN: ${{ secrets.SENTRY_DSN }}
   PERSONAL_ACCESS_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
 
 jobs:
@@ -110,7 +109,6 @@ jobs:
           git commit -m "Helm Release: ${{ env.RELEASE_VERSION }}"
           git push
 
-
   trigger-deploy-to-qa-env:
     needs: [release-helm-chart]
     runs-on: ubuntu-latest

.github/workflows/release-singleimage-test.yml

@@ -0,0 +1,69 @@
+name: Test
+
+on:
+  workflow_dispatch:
+
+env:
+  CI: true
+  PERSONAL_ACCESS_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
+  REGISTRY_URL: registry.hub.docker.com
+  NX_CLOUD_ACCESS_TOKEN: ${{ secrets.NX_CLOUD_ACCESS_TOKEN }}
+jobs:
+  build:
+    name: "build"
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        node-version: [18.x]
+    steps:
+      - name: "Checkout"
+        uses: actions/checkout@v4
+        with:
+          submodules: true
+          token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
+
+      - name: Use Node.js ${{ matrix.node-version }}
+        uses: actions/setup-node@v3
+        with:
+          node-version: ${{ matrix.node-version }}
+          cache: "yarn"
+      - name: Setup QEMU
+        uses: docker/setup-qemu-action@v3
+      - name: Setup Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Run Yarn
+        run: yarn
+      - name: Run Yarn Build
+        run: yarn build --scope @budibase/server --scope @budibase/worker
+      - name: Login to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_API_KEY }}
+      - name: Get the latest release version
+        id: version
+        run: |
+          release_version=$(cat lerna.json | jq -r '.version')
+          echo $release_version
+          echo "RELEASE_VERSION=$release_version" >> $GITHUB_ENV
+      - name: Tag and release Budibase service docker image
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          push: true
+          pull: true
+          platforms: linux/amd64,linux/arm64
+          tags: budibase/budibase-test:test
+          file: ./hosting/single/Dockerfile.v2
+          cache-from: type=registry,ref=budibase/budibase-test:test
+          cache-to: type=inline
+      - name: Tag and release Budibase Azure App Service docker image
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          push: true
+          platforms: linux/amd64
+          build-args: TARGETBUILD=aas
+          tags: budibase/budibase-test:aas
+          file: ./hosting/single/Dockerfile.v2

.github/workflows/release-singleimage.yml

@@ -20,8 +20,8 @@ jobs:
         with:
           root-reserve-mb: 30000
           swap-size-mb: 1024
-          remove-android: 'true'
-          remove-dotnet: 'true'
+          remove-android: "true"
+          remove-dotnet: "true"
       - name: Fail if not a tag
         run: |
           if [[ $GITHUB_REF != refs/tags/* ]]; then 
@@ -48,7 +48,7 @@ jobs:
       - name: Update versions
         run: ./scripts/updateVersions.sh
       - name: Run Yarn Build
-        run: yarn build:docker:pre
+        run: yarn build
       - name: Login to Docker Hub
         uses: docker/login-action@v2
         with:

.yarnrc

@@ -1 +1 @@
-network-timeout 100000
+network-timeout 1000000

charts/budibase/templates/app-service-deployment.yaml

@@ -134,8 +134,6 @@ spec:
           {{ end }}
         - name: SELF_HOSTED
           value: {{ .Values.globals.selfHosted | quote }}
-        - name: SENTRY_DSN
-          value: {{ .Values.globals.sentryDSN | quote }}
         - name: POSTHOG_TOKEN
           value: {{ .Values.globals.posthogToken | quote }}
         - name: WORKER_URL

charts/budibase/templates/worker-service-deployment.yaml

@@ -130,8 +130,6 @@ spec:
           {{ end }}
         - name: SELF_HOSTED
           value: {{ .Values.globals.selfHosted | quote }}
-        - name: SENTRY_DSN
-          value: {{ .Values.globals.sentryDSN }}
         - name: ENABLE_ANALYTICS
           value: {{ .Values.globals.enableAnalytics | quote }}
         - name: POSTHOG_TOKEN

charts/budibase/values.yaml

@@ -78,7 +78,6 @@ globals:
   budibaseEnv: PRODUCTION
   tenantFeatureFlags: "*:LICENSING,*:USER_GROUPS,*:ONBOARDING_TOUR"
   enableAnalytics: "1"
-  sentryDSN: ""
   posthogToken: "phc_bIjZL7oh2GEUd2vqvTBH8WvrX0fWTFQMs6H5KQxiUxU"
   selfHosted: "1" # set to 0 for budibase cloud environment, set to 1 for self-hosted setup
   multiTenancy: "0" # set to 0 to disable multiple orgs, set to 1 to enable multiple orgs

docs/CONTRIBUTING.md

@@ -138,6 +138,8 @@ To develop the Budibase platform you'll need [Docker](https://www.docker.com/) a
 
 `yarn setup` will check that all necessary components are installed and setup the repo for usage.
 
+If you have access to the `@budibase/pro` submodule then please follow the Pro section of this guide before running the above command.
+
 ##### Manual method
 
 The following commands can be executed to manually get Budibase up and running (assuming Docker/Docker Compose has been installed).
@@ -146,6 +148,8 @@ The following commands can be executed to manually get Budibase up and running (
 
 `yarn build` will build all budibase packages.
 
+If you have access to the `@budibase/pro` submodule then please follow the Pro section of this guide before running the above commands.
+
 #### 4. Running
 
 To run the budibase server and builder in dev mode (i.e. with live reloading):

hosting/couchdb/couch/local.ini

@@ -3,3 +3,6 @@
 [couchdb]
 database_dir = DATA_DIR/couch/dbs
 view_index_dir = DATA_DIR/couch/views
+
+[chttpd_auth]
+timeout = 7200 ; 2 hours in seconds

hosting/docker-compose.build.yaml

@@ -19,7 +19,6 @@ services:
       API_ENCRYPTION_KEY: ${API_ENCRYPTION_KEY}
       JWT_SECRET: ${JWT_SECRET}
       LOG_LEVEL: info
-      SENTRY_DSN: https://a34ae347621946bf8acded18e5b7d4b8@o420233.ingest.sentry.io/5338131
       ENABLE_ANALYTICS: "true"
       REDIS_URL: redis-service:6379
       REDIS_PASSWORD: ${REDIS_PASSWORD}
@@ -48,7 +47,6 @@ services:
       COUCH_DB_USERNAME: ${COUCH_DB_USER}
       COUCH_DB_PASSWORD: ${COUCH_DB_PASSWORD}
       COUCH_DB_URL: http://${COUCH_DB_USER}:${COUCH_DB_PASSWORD}@couchdb-service:5984
-      SENTRY_DSN: https://a34ae347621946bf8acded18e5b7d4b8@o420233.ingest.sentry.io/5338131
       INTERNAL_API_KEY: ${INTERNAL_API_KEY}
       REDIS_URL: redis-service:6379
       REDIS_PASSWORD: ${REDIS_PASSWORD}

hosting/docker-compose.yaml

@@ -20,7 +20,6 @@ services:
       API_ENCRYPTION_KEY: ${API_ENCRYPTION_KEY}
       JWT_SECRET: ${JWT_SECRET}
       LOG_LEVEL: info
-      SENTRY_DSN: https://a34ae347621946bf8acded18e5b7d4b8@o420233.ingest.sentry.io/5338131
       ENABLE_ANALYTICS: "true"
       REDIS_URL: redis-service:6379
       REDIS_PASSWORD: ${REDIS_PASSWORD}
@@ -31,8 +30,8 @@ services:
     depends_on:
       - worker-service
       - redis-service
-#    volumes:
-#      - /some/path/to/plugins:/plugins
+  #    volumes:
+  #      - /some/path/to/plugins:/plugins
 
   worker-service:
     restart: unless-stopped
@@ -51,7 +50,6 @@ services:
       COUCH_DB_USERNAME: ${COUCH_DB_USER}
       COUCH_DB_PASSWORD: ${COUCH_DB_PASSWORD}
       COUCH_DB_URL: http://${COUCH_DB_USER}:${COUCH_DB_PASSWORD}@couchdb-service:5984
-      SENTRY_DSN: https://a34ae347621946bf8acded18e5b7d4b8@o420233.ingest.sentry.io/5338131
       INTERNAL_API_KEY: ${INTERNAL_API_KEY}
       REDIS_URL: redis-service:6379
       REDIS_PASSWORD: ${REDIS_PASSWORD}
@@ -113,7 +111,12 @@ services:
       PUT_CALL: "curl -u ${COUCH_DB_USER}:${COUCH_DB_PASSWORD} -X PUT couchdb-service:5984"
     depends_on:
       - couchdb-service
-    command: ["sh","-c","sleep 10 && $${PUT_CALL}/_users && $${PUT_CALL}/_replicator; fg;"]
+    command:
+      [
+        "sh",
+        "-c",
+        "sleep 10 && $${PUT_CALL}/_users && $${PUT_CALL}/_replicator; fg;",
+      ]
 
   redis-service:
     restart: unless-stopped

hosting/single/Dockerfile

@@ -12,14 +12,14 @@ RUN chmod +x /cleanup.sh
 WORKDIR /app
 ADD packages/server .
 COPY yarn.lock .
-RUN yarn install --production=true --network-timeout 100000
+RUN yarn install --production=true --network-timeout 1000000
 RUN /cleanup.sh
 
 # build worker
 WORKDIR /worker
 ADD packages/worker .
 COPY yarn.lock .
-RUN yarn install --production=true --network-timeout 100000
+RUN yarn install --production=true --network-timeout 1000000
 RUN /cleanup.sh
 
 FROM budibase/couchdb

hosting/single/Dockerfile.v2

@@ -0,0 +1,126 @@
+FROM node:18-slim as build
+
+# install node-gyp dependencies
+RUN apt-get update && apt-get install -y --no-install-recommends g++ make python3 jq
+
+
+# copy and install dependencies
+WORKDIR /app
+COPY package.json .
+COPY yarn.lock .
+COPY lerna.json .
+COPY .yarnrc .
+
+COPY packages/server/package.json packages/server/package.json
+COPY packages/worker/package.json packages/worker/package.json
+# string-templates does not get bundled during the esbuild process, so we want to use the local version
+COPY packages/string-templates/package.json packages/string-templates/package.json
+
+
+COPY scripts/removeWorkspaceDependencies.sh scripts/removeWorkspaceDependencies.sh
+RUN chmod +x ./scripts/removeWorkspaceDependencies.sh
+RUN ./scripts/removeWorkspaceDependencies.sh
+
+
+# We will never want to sync pro, but the script is still required
+RUN echo '' > scripts/syncProPackage.js
+RUN jq 'del(.scripts.postinstall)' package.json > temp.json && mv temp.json package.json
+RUN --mount=type=cache,target=/root/.yarn YARN_CACHE_FOLDER=/root/.yarn  yarn install --production
+
+# copy the actual code
+COPY packages/server/dist packages/server/dist
+COPY packages/server/pm2.config.js packages/server/pm2.config.js
+COPY packages/server/client packages/server/client
+COPY packages/server/builder packages/server/builder
+COPY packages/worker/dist packages/worker/dist
+COPY packages/worker/pm2.config.js packages/worker/pm2.config.js
+COPY packages/string-templates packages/string-templates
+
+
+FROM budibase/couchdb as runner
+ARG TARGETARCH
+ENV TARGETARCH $TARGETARCH
+#TARGETBUILD can be set to single (for single docker image) or aas (for azure app service)
+# e.g. docker build --build-arg TARGETBUILD=aas ....
+ARG TARGETBUILD=single
+ENV TARGETBUILD $TARGETBUILD
+
+# install base dependencies
+RUN apt-get update && \
+  apt-get install -y --no-install-recommends software-properties-common nginx uuid-runtime redis-server
+
+# Install postgres client for pg_dump utils
+RUN apt install software-properties-common apt-transport-https gpg -y \
+    && curl -fsSl https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor | tee /usr/share/keyrings/postgresql.gpg > /dev/null \
+    && echo deb [arch=amd64,arm64,ppc64el signed-by=/usr/share/keyrings/postgresql.gpg] http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main | tee /etc/apt/sources.list.d/postgresql.list \
+    && apt update -y \
+    && apt install postgresql-client-15 -y \
+    && apt remove software-properties-common apt-transport-https gpg -y
+
+# install other dependencies, nodejs, oracle requirements, jdk8, redis, nginx
+WORKDIR /nodejs
+RUN curl -sL https://deb.nodesource.com/setup_18.x -o /tmp/nodesource_setup.sh && \
+  bash /tmp/nodesource_setup.sh && \
+  apt-get install -y --no-install-recommends libaio1 nodejs && \
+  npm install --global yarn pm2
+
+# setup nginx
+COPY hosting/single/nginx/nginx.conf /etc/nginx
+COPY hosting/single/nginx/nginx-default-site.conf /etc/nginx/sites-enabled/default
+RUN mkdir -p /var/log/nginx && \
+  touch /var/log/nginx/error.log && \
+  touch /var/run/nginx.pid && \
+  usermod -a -G tty www-data
+
+WORKDIR /
+RUN mkdir -p scripts/integrations/oracle
+COPY packages/server/scripts/integrations/oracle scripts/integrations/oracle
+RUN /bin/bash -e ./scripts/integrations/oracle/instantclient/linux/install.sh
+
+# setup minio
+WORKDIR /minio
+COPY scripts/install-minio.sh ./install.sh
+RUN chmod +x install.sh && ./install.sh
+
+# setup runner file
+WORKDIR /
+COPY hosting/single/runner.sh .
+RUN chmod +x ./runner.sh
+COPY hosting/single/healthcheck.sh .
+RUN chmod +x ./healthcheck.sh
+
+# Script below sets the path for storing data based on $DATA_DIR
+# For Azure App Service install SSH & point data locations to /home
+COPY hosting/single/ssh/sshd_config /etc/
+COPY hosting/single/ssh/ssh_setup.sh /tmp
+RUN /build-target-paths.sh
+
+
+#  setup letsencrypt certificate
+RUN apt-get install -y certbot python3-certbot-nginx
+COPY hosting/letsencrypt /app/letsencrypt
+RUN chmod +x /app/letsencrypt/certificate-request.sh /app/letsencrypt/certificate-renew.sh
+
+COPY --from=build /app/node_modules /node_modules
+COPY --from=build /app/package.json /package.json
+COPY --from=build /app/packages/server /app
+COPY --from=build /app/packages/worker /worker
+COPY --from=build /app/packages/string-templates /string-templates
+
+RUN cd /string-templates && yarn link && cd ../app && yarn link @budibase/string-templates && cd ../worker && yarn link @budibase/string-templates
+
+
+EXPOSE 80
+EXPOSE 443
+# Expose port 2222 for SSH on Azure App Service build
+EXPOSE 2222
+VOLUME /data
+
+
+HEALTHCHECK --interval=15s --timeout=15s --start-period=45s CMD "/healthcheck.sh"
+
+# must set this just before running
+ENV NODE_ENV=production
+WORKDIR /
+
+CMD ["./runner.sh"]

hosting/single/runner.sh

@@ -7,16 +7,16 @@ declare -a DOCKER_VARS=("APP_PORT" "APPS_URL" "ARCHITECTURE" "BUDIBASE_ENVIRONME
 [[ -z "${BUDIBASE_ENVIRONMENT}" ]] && export BUDIBASE_ENVIRONMENT=PRODUCTION
 [[ -z "${CLUSTER_PORT}" ]] && export CLUSTER_PORT=80
 [[ -z "${DEPLOYMENT_ENVIRONMENT}" ]] && export DEPLOYMENT_ENVIRONMENT=docker
-[[ -z "${MINIO_URL}" ]] && export MINIO_URL=http://localhost:9000
+[[ -z "${MINIO_URL}" ]] && export MINIO_URL=http://127.0.0.1:9000
 [[ -z "${NODE_ENV}" ]] && export NODE_ENV=production
 [[ -z "${POSTHOG_TOKEN}" ]] && export POSTHOG_TOKEN=phc_bIjZL7oh2GEUd2vqvTBH8WvrX0fWTFQMs6H5KQxiUxU
 [[ -z "${TENANT_FEATURE_FLAGS}" ]] && export TENANT_FEATURE_FLAGS="*:LICENSING,*:USER_GROUPS,*:ONBOARDING_TOUR"
 [[ -z "${ACCOUNT_PORTAL_URL}" ]] && export ACCOUNT_PORTAL_URL=https://account.budibase.app
-[[ -z "${REDIS_URL}" ]] && export REDIS_URL=localhost:6379
+[[ -z "${REDIS_URL}" ]] && export REDIS_URL=127.0.0.1:6379
 [[ -z "${SELF_HOSTED}" ]] && export SELF_HOSTED=1
 [[ -z "${WORKER_PORT}" ]] && export WORKER_PORT=4002
-[[ -z "${WORKER_URL}" ]] && export WORKER_URL=http://localhost:4002
-[[ -z "${APPS_URL}" ]] && export APPS_URL=http://localhost:4001
+[[ -z "${WORKER_URL}" ]] && export WORKER_URL=http://127.0.0.1:4002
+[[ -z "${APPS_URL}" ]] && export APPS_URL=http://127.0.0.1:4001
 [[ -z "${SERVER_TOP_LEVEL_PATH}" ]] && export SERVER_TOP_LEVEL_PATH=/app
 #  export CUSTOM_DOMAIN=budi001.custom.com
 
@@ -51,7 +51,7 @@ do
     fi
 done
 if [[ -z "${COUCH_DB_URL}" ]]; then
-    export COUCH_DB_URL=http://$COUCHDB_USER:$COUCHDB_PASSWORD@localhost:5984
+    export COUCH_DB_URL=http://$COUCHDB_USER:$COUCHDB_PASSWORD@127.0.0.1:5984
 fi
 if [ ! -f "${DATA_DIR}/.env" ]; then
     touch ${DATA_DIR}/.env

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "2.11.34",
+  "version": "2.11.38",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

nx.json

@@ -8,5 +8,9 @@
       }
     }
   },
-  "targetDefaults": {}
+  "targetDefaults": {
+    "build": {
+      "inputs": ["{workspaceRoot}/scripts/build.js"]
+    }
+  }
 }

package.json

@@ -3,14 +3,11 @@
   "private": true,
   "devDependencies": {
     "@esbuild-plugins/tsconfig-paths": "^0.1.2",
-    "@nx/js": "16.4.3",
-    "@rollup/plugin-json": "^4.0.2",
     "@typescript-eslint/parser": "6.7.2",
     "esbuild": "^0.18.17",
     "esbuild-node-externals": "^1.8.0",
     "eslint": "^8.44.0",
     "husky": "^8.0.3",
-    "js-yaml": "^4.1.0",
     "kill-port": "^1.6.1",
     "lerna": "7.1.1",
     "madge": "^6.0.0",
@@ -19,8 +16,6 @@
     "nx-cloud": "16.0.5",
     "prettier": "2.8.8",
     "prettier-plugin-svelte": "^2.3.0",
-    "rimraf": "^3.0.2",
-    "rollup-plugin-replace": "^2.2.0",
     "svelte": "3.49.0",
     "typescript": "5.2.2",
     "@babel/core": "^7.22.5",
@@ -51,7 +46,7 @@
     "dev:noserver": "yarn run kill-builder && lerna run --stream dev:stack:up && lerna run --stream dev:builder --ignore @budibase/backend-core --ignore @budibase/server --ignore @budibase/worker",
     "dev:server": "yarn run kill-server && lerna run --stream dev:builder --scope @budibase/worker --scope @budibase/server",
     "dev:built": "yarn run kill-all && cd packages/server && yarn dev:stack:up && cd ../../ && lerna run --stream dev:built",
-    "dev:docker": "yarn build:docker:pre && docker-compose -f hosting/docker-compose.build.yaml -f hosting/docker-compose.dev.yaml --env-file hosting/.env up --build --scale proxy-service=0",
+    "dev:docker": "yarn build && docker-compose -f hosting/docker-compose.build.yaml -f hosting/docker-compose.dev.yaml --env-file hosting/.env up --build --scale proxy-service=0",
     "test": "lerna run --stream test --stream",
     "lint:eslint": "eslint packages qa-core --max-warnings=0",
     "lint:prettier": "prettier --check \"packages/**/*.{js,ts,svelte}\" && prettier --write \"examples/**/*.{js,ts,svelte}\" && prettier --check \"qa-core/**/*.{js,ts,svelte}\"",
@@ -61,7 +56,6 @@
     "lint:fix": "yarn run lint:fix:prettier && yarn run lint:fix:eslint",
     "build:specs": "lerna run --stream specs",
     "build:docker": "lerna run --stream build:docker && yarn build:docker:proxy && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh $BUDIBASE_RELEASE_VERSION && cd -",
-    "build:docker:pre": "yarn build && lerna run --stream predocker",
     "build:docker:proxy": "docker build hosting/proxy -t proxy-service",
     "build:docker:selfhost": "lerna run --stream build:docker && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh latest && cd -",
     "build:docker:develop": "node scripts/pinVersions && lerna run --stream build:docker && yarn build:docker:proxy && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh develop && cd -",
@@ -69,8 +63,7 @@
     "build:docker:airgap:single": "SINGLE_IMAGE=1 node hosting/scripts/airgapped/airgappedDockerBuild",
     "build:digitalocean": "cd hosting/digitalocean && ./build.sh && cd -",
     "build:docker:single:multiarch": "docker buildx build --platform linux/arm64,linux/amd64 -f hosting/single/Dockerfile -t budibase:latest .",
-    "build:docker:single:image": "docker build -f hosting/single/Dockerfile -t budibase:latest .",
-    "build:docker:single": "yarn build && lerna run --concurrency 1 predocker && yarn build:docker:single:image",
+    "build:docker:single": "./scripts/build-single-image.sh",
     "build:docker:dependencies": "docker build -f hosting/dependencies/Dockerfile -t budibase/dependencies:latest ./hosting",
     "publish:docker:couch": "docker buildx build --platform linux/arm64,linux/amd64 -f hosting/couchdb/Dockerfile -t budibase/couchdb:latest -t budibase/couchdb:v3.2.1 --push ./hosting/couchdb",
     "publish:docker:dependencies": "docker buildx build --platform linux/arm64,linux/amd64 -f hosting/dependencies/Dockerfile -t budibase/dependencies:latest -t budibase/dependencies:v3.2.1 --push ./hosting",

packages/backend-core/package.json

@@ -26,7 +26,7 @@
     "@budibase/shared-core": "0.0.0",
     "@budibase/types": "0.0.0",
     "@techpass/passport-openidconnect": "0.3.2",
-    "aws-cloudfront-sign": "2.2.0",
+    "aws-cloudfront-sign": "3.0.2",
     "aws-sdk": "2.1030.0",
     "bcrypt": "5.1.0",
     "bcryptjs": "2.4.3",

packages/backend-core/src/cache/appMetadata.ts

@@ -33,8 +33,8 @@ function isInvalid(metadata?: { state: string }) {
  * Get the requested app metadata by id.
  * Use redis cache to first read the app metadata.
  * If not present fallback to loading the app metadata directly and re-caching.
- * @param {string} appId the id of the app to get metadata from.
- * @returns {object} the app metadata.
+ * @param appId the id of the app to get metadata from.
+ * @returns the app metadata.
  */
 export async function getAppMetadata(appId: string): Promise<App | DeletedApp> {
   const client = await getAppClient()
@@ -72,9 +72,9 @@ export async function getAppMetadata(appId: string): Promise<App | DeletedApp> {
 
 /**
  * Invalidate/reset the cached metadata when a change occurs in the db.
- * @param appId {string} the cache key to bust/update.
- * @param newMetadata {object|undefined} optional - can simply provide the new metadata to update with.
- * @return {Promise<void>} will respond with success when cache is updated.
+ * @param appId the cache key to bust/update.
+ * @param newMetadata optional - can simply provide the new metadata to update with.
+ * @return will respond with success when cache is updated.
  */
 export async function invalidateAppMetadata(appId: string, newMetadata?: any) {
   if (!appId) {

packages/backend-core/src/cache/user.ts

@@ -61,9 +61,9 @@ async function populateUsersFromDB(
  * Get the requested user by id.
  * Use redis cache to first read the user.
  * If not present fallback to loading the user directly and re-caching.
- * @param {*} userId the id of the user to get
- * @param {*} tenantId the tenant of the user to get
- * @param {*} populateUser function to provide the user for re-caching. default to couch db
+ * @param userId the id of the user to get
+ * @param tenantId the tenant of the user to get
+ * @param populateUser function to provide the user for re-caching. default to couch db
  * @returns
  */
 export async function getUser(
@@ -111,8 +111,8 @@ export async function getUser(
  * Get the requested users by id.
  * Use redis cache to first read the users.
  * If not present fallback to loading the users directly and re-caching.
- * @param {*} userIds the ids of the user to get
- * @param {*} tenantId the tenant of the users to get
+ * @param userIds the ids of the user to get
+ * @param tenantId the tenant of the users to get
  * @returns
  */
 export async function getUsers(

packages/backend-core/src/cache/writethrough.ts

@@ -119,8 +119,8 @@ export class Writethrough {
     this.writeRateMs = writeRateMs
   }
 
-  async put(doc: any) {
-    return put(this.db, doc, this.writeRateMs)
+  async put(doc: any, writeRateMs: number = this.writeRateMs) {
+    return put(this.db, doc, writeRateMs)
   }
 
   async get(id: string) {

packages/backend-core/src/configs/configs.ts

@@ -23,7 +23,7 @@ import environment from "../environment"
 
 /**
  * Generates a new configuration ID.
- * @returns {string} The new configuration ID which the config doc can be stored under.
+ * @returns The new configuration ID which the config doc can be stored under.
  */
 export function generateConfigID(type: ConfigType) {
   return `${DocumentType.CONFIG}${SEPARATOR}${type}`

packages/backend-core/src/context/mainContext.ts

@@ -62,7 +62,7 @@ export function isTenancyEnabled() {
 
 /**
  * Given an app ID this will attempt to retrieve the tenant ID from it.
- * @return {null|string} The tenant ID found within the app ID.
+ * @return The tenant ID found within the app ID.
  */
 export function getTenantIDFromAppID(appId: string) {
   if (!appId) {

packages/backend-core/src/db/Replication.ts

@@ -8,8 +8,8 @@ class Replication {
 
   /**
    *
-   * @param {String} source - the DB you want to replicate or rollback to
-   * @param {String} target - the DB you want to replicate to, or rollback from
+   * @param source - the DB you want to replicate or rollback to
+   * @param target - the DB you want to replicate to, or rollback from
    */
   constructor({ source, target }: any) {
     this.source = getPouchDB(source)
@@ -38,7 +38,7 @@ class Replication {
 
   /**
    * Two way replication operation, intended to be promise based.
-   * @param {Object} opts - PouchDB replication options
+   * @param opts - PouchDB replication options
    */
   sync(opts = {}) {
     this.replication = this.promisify(this.source.sync, opts)
@@ -47,7 +47,7 @@ class Replication {
 
   /**
    * One way replication operation, intended to be promise based.
-   * @param {Object} opts - PouchDB replication options
+   * @param opts - PouchDB replication options
    */
   replicate(opts = {}) {
     this.replication = this.promisify(this.source.replicate.to, opts)

packages/backend-core/src/db/lucene.ts

@@ -599,10 +599,10 @@ async function runQuery<T>(
  * Gets round the fixed limit of 200 results from a query by fetching as many
  * pages as required and concatenating the results. This recursively operates
  * until enough results have been found.
- * @param dbName {string} Which database to run a lucene query on
- * @param index {string} Which search index to utilise
- * @param query {object} The JSON query structure
- * @param params {object} The search params including:
+ * @param dbName Which database to run a lucene query on
+ * @param index Which search index to utilise
+ * @param query The JSON query structure
+ * @param params The search params including:
  *   tableId {string} The table ID to search
  *   sort {string} The sort column
  *   sortOrder {string} The sort order ("ascending" or "descending")
@@ -655,10 +655,10 @@ async function recursiveSearch<T>(
  * Performs a paginated search. A bookmark will be returned to allow the next
  * page to be fetched. There is a max limit off 200 results per page in a
  * paginated search.
- * @param dbName {string} Which database to run a lucene query on
- * @param index {string} Which search index to utilise
- * @param query {object} The JSON query structure
- * @param params {object} The search params including:
+ * @param dbName Which database to run a lucene query on
+ * @param index Which search index to utilise
+ * @param query The JSON query structure
+ * @param params The search params including:
  *   tableId {string} The table ID to search
  *   sort {string} The sort column
  *   sortOrder {string} The sort order ("ascending" or "descending")
@@ -722,10 +722,10 @@ export async function paginatedSearch<T>(
  * desired amount of results. There is a limit of 1000 results to avoid
  * heavy performance hits, and to avoid client components breaking from
  * handling too much data.
- * @param dbName {string} Which database to run a lucene query on
- * @param index {string} Which search index to utilise
- * @param query {object} The JSON query structure
- * @param params {object} The search params including:
+ * @param dbName Which database to run a lucene query on
+ * @param index Which search index to utilise
+ * @param query The JSON query structure
+ * @param params The search params including:
  *   tableId {string} The table ID to search
  *   sort {string} The sort column
  *   sortOrder {string} The sort order ("ascending" or "descending")

packages/backend-core/src/db/utils.ts

@@ -45,7 +45,7 @@ export async function getAllDbs(opts = { efficient: false }) {
  * Lots of different points in the system need to find the full list of apps, this will
  * enumerate the entire CouchDB cluster and get the list of databases (every app).
  *
- * @return {Promise<object[]>} returns the app information document stored in each app database.
+ * @return returns the app information document stored in each app database.
  */
 export async function getAllApps({
   dev,

packages/backend-core/src/docIds/conversions.ts

@@ -25,7 +25,7 @@ export function isDevApp(app: App) {
 
 /**
  * Generates a development app ID from a real app ID.
- * @returns {string} the dev app ID which can be used for dev database.
+ * @returns the dev app ID which can be used for dev database.
  */
 export function getDevelopmentAppID(appId: string) {
   if (!appId || appId.startsWith(APP_DEV_PREFIX)) {

packages/backend-core/src/docIds/ids.ts

@@ -8,7 +8,7 @@ import { newid } from "./newid"
 
 /**
  * Generates a new app ID.
- * @returns {string} The new app ID which the app doc can be stored under.
+ * @returns The new app ID which the app doc can be stored under.
  */
 export const generateAppID = (tenantId?: string | null) => {
   let id = APP_PREFIX
@@ -20,9 +20,9 @@ export const generateAppID = (tenantId?: string | null) => {
 
 /**
  * Gets a new row ID for the specified table.
- * @param {string} tableId The table which the row is being created for.
- * @param {string|null} id If an ID is to be used then the UUID can be substituted for this.
- * @returns {string} The new ID which a row doc can be stored under.
+ * @param tableId The table which the row is being created for.
+ * @param id If an ID is to be used then the UUID can be substituted for this.
+ * @returns The new ID which a row doc can be stored under.
  */
 export function generateRowID(tableId: string, id?: string) {
   id = id || newid()
@@ -31,7 +31,7 @@ export function generateRowID(tableId: string, id?: string) {
 
 /**
  * Generates a new workspace ID.
- * @returns {string} The new workspace ID which the workspace doc can be stored under.
+ * @returns The new workspace ID which the workspace doc can be stored under.
  */
 export function generateWorkspaceID() {
   return `${DocumentType.WORKSPACE}${SEPARATOR}${newid()}`
@@ -39,7 +39,7 @@ export function generateWorkspaceID() {
 
 /**
  * Generates a new global user ID.
- * @returns {string} The new user ID which the user doc can be stored under.
+ * @returns The new user ID which the user doc can be stored under.
  */
 export function generateGlobalUserID(id?: any) {
   return `${DocumentType.USER}${SEPARATOR}${id || newid()}`
@@ -52,8 +52,8 @@ export function isGlobalUserID(id: string) {
 
 /**
  * Generates a new user ID based on the passed in global ID.
- * @param {string} globalId The ID of the global user.
- * @returns {string} The new user ID which the user doc can be stored under.
+ * @param globalId The ID of the global user.
+ * @returns The new user ID which the user doc can be stored under.
  */
 export function generateUserMetadataID(globalId: string) {
   return generateRowID(InternalTable.USER_METADATA, globalId)
@@ -84,7 +84,7 @@ export function generateAppUserID(prodAppId: string, userId: string) {
 
 /**
  * Generates a new role ID.
- * @returns {string} The new role ID which the role doc can be stored under.
+ * @returns The new role ID which the role doc can be stored under.
  */
 export function generateRoleID(name: string) {
   const prefix = `${DocumentType.ROLE}${SEPARATOR}`
@@ -103,7 +103,7 @@ export function prefixRoleID(name: string) {
 
 /**
  * Generates a new dev info document ID - this is scoped to a user.
- * @returns {string} The new dev info ID which info for dev (like api key) can be stored under.
+ * @returns The new dev info ID which info for dev (like api key) can be stored under.
  */
 export const generateDevInfoID = (userId: any) => {
   return `${DocumentType.DEV_INFO}${SEPARATOR}${userId}`
@@ -111,7 +111,7 @@ export const generateDevInfoID = (userId: any) => {
 
 /**
  * Generates a new plugin ID - to be used in the global DB.
- * @returns {string} The new plugin ID which a plugin metadata document can be stored under.
+ * @returns The new plugin ID which a plugin metadata document can be stored under.
  */
 export const generatePluginID = (name: string) => {
   return `${DocumentType.PLUGIN}${SEPARATOR}${name}`

packages/backend-core/src/docIds/params.ts

@@ -12,12 +12,12 @@ import { getProdAppID } from "./conversions"
  * is usually the case as most of our docs are top level e.g. tables, automations, users and so on.
  * More complex cases such as link docs and rows which have multiple levels of IDs that their
  * ID consists of need their own functions to build the allDocs parameters.
- * @param {string} docType The type of document which input params are being built for, e.g. user,
+ * @param docType The type of document which input params are being built for, e.g. user,
  * link, app, table and so on.
- * @param {string|null} docId The ID of the document minus its type - this is only needed if looking
+ * @param docId The ID of the document minus its type - this is only needed if looking
  * for a singular document.
- * @param {object} otherProps Add any other properties onto the request, e.g. include_docs.
- * @returns {object} Parameters which can then be used with an allDocs request.
+ * @param otherProps Add any other properties onto the request, e.g. include_docs.
+ * @returns Parameters which can then be used with an allDocs request.
  */
 export function getDocParams(
   docType: string,
@@ -36,11 +36,11 @@ export function getDocParams(
 
 /**
  * Gets the DB allDocs/query params for retrieving a row.
- * @param {string|null} tableId The table in which the rows have been stored.
- * @param {string|null} rowId The ID of the row which is being specifically queried for. This can be
+ * @param tableId The table in which the rows have been stored.
+ * @param rowId The ID of the row which is being specifically queried for. This can be
  * left null to get all the rows in the table.
- * @param {object} otherProps Any other properties to add to the request.
- * @returns {object} Parameters which can then be used with an allDocs request.
+ * @param otherProps Any other properties to add to the request.
+ * @returns Parameters which can then be used with an allDocs request.
  */
 export function getRowParams(
   tableId?: string | null,

packages/backend-core/src/helpers.ts

@@ -1,8 +1,8 @@
 /**
  * Makes sure that a URL has the correct number of slashes, while maintaining the
  * http(s):// double slashes.
- * @param {string} url The URL to test and remove any extra double slashes.
- * @return {string} The updated url.
+ * @param url The URL to test and remove any extra double slashes.
+ * @return The updated url.
  */
 export function checkSlashesInUrl(url: string) {
   return url.replace(/(https?:\/\/)|(\/)+/g, "$1$2")

packages/backend-core/src/middleware/passport/local.ts

@@ -13,10 +13,10 @@ export const options = {
 
 /**
  * Passport Local Authentication Middleware.
- * @param {*} ctx the request structure
- * @param {*} email username to login with
- * @param {*} password plain text password to log in with
- * @param {*} done callback from passport to return user information and errors
+ * @param ctx the request structure
+ * @param email username to login with
+ * @param password plain text password to log in with
+ * @param done callback from passport to return user information and errors
  * @returns The authenticated user, or errors if they occur
  */
 export async function authenticate(

packages/backend-core/src/middleware/passport/sso/oidc.ts

@@ -17,15 +17,15 @@ const OIDCStrategy = require("@techpass/passport-openidconnect").Strategy
 
 export function buildVerifyFn(saveUserFn: SaveSSOUserFunction) {
   /**
-   * @param {*} issuer The identity provider base URL
-   * @param {*} sub The user ID
-   * @param {*} profile The user profile information. Created by passport from the /userinfo response
-   * @param {*} jwtClaims The parsed id_token claims
-   * @param {*} accessToken The access_token for contacting the identity provider - may or may not be a JWT
-   * @param {*} refreshToken The refresh_token for obtaining a new access_token - usually not a JWT
-   * @param {*} idToken The id_token - always a JWT
-   * @param {*} params The response body from requesting an access_token
-   * @param {*} done The passport callback: err, user, info
+   * @param issuer The identity provider base URL
+   * @param sub The user ID
+   * @param profile The user profile information. Created by passport from the /userinfo response
+   * @param jwtClaims The parsed id_token claims
+   * @param accessToken The access_token for contacting the identity provider - may or may not be a JWT
+   * @param refreshToken The refresh_token for obtaining a new access_token - usually not a JWT
+   * @param idToken The id_token - always a JWT
+   * @param params The response body from requesting an access_token
+   * @param done The passport callback: err, user, info
    */
   return async (
     issuer: string,
@@ -61,8 +61,8 @@ export function buildVerifyFn(saveUserFn: SaveSSOUserFunction) {
 }
 
 /**
- * @param {*} profile The structured profile created by passport using the user info endpoint
- * @param {*} jwtClaims The claims returned in the id token
+ * @param profile The structured profile created by passport using the user info endpoint
+ * @param jwtClaims The claims returned in the id token
  */
 function getEmail(profile: SSOProfile, jwtClaims: JwtClaims) {
   // profile not guaranteed to contain email e.g. github connected azure ad account

packages/backend-core/src/middleware/passport/utils.ts

@@ -5,9 +5,9 @@ import { ConfigType, GoogleInnerConfig } from "@budibase/types"
 /**
  * Utility to handle authentication errors.
  *
- * @param {*} done The passport callback.
- * @param {*} message Message that will be returned in the response body
- * @param {*} err (Optional) error that will be logged
+ * @param done The passport callback.
+ * @param message Message that will be returned in the response body
+ * @param err (Optional) error that will be logged
  */
 
 export function authError(done: Function, message: string, err?: any) {

packages/backend-core/src/objectStore/buckets/app.ts

@@ -6,10 +6,10 @@ import * as cloudfront from "../cloudfront"
  * In production the client library is stored in the object store, however in development
  * we use the symlinked version produced by lerna, located in node modules. We link to this
  * via a specific endpoint (under /api/assets/client).
- * @param {string} appId In production we need the appId to look up the correct bucket, as the
+ * @param appId In production we need the appId to look up the correct bucket, as the
  * version of the client lib may differ between apps.
- * @param {string} version The version to retrieve.
- * @return {string} The URL to be inserted into appPackage response or server rendered
+ * @param version The version to retrieve.
+ * @return The URL to be inserted into appPackage response or server rendered
  * app index file.
  */
 export const clientLibraryUrl = (appId: string, version: string) => {

packages/backend-core/src/objectStore/cloudfront.ts

@@ -1,5 +1,5 @@
 import env from "../environment"
-const cfsign = require("aws-cloudfront-sign")
+import * as cfsign from "aws-cloudfront-sign"
 
 let PRIVATE_KEY: string | undefined
 
@@ -21,7 +21,7 @@ function getPrivateKey() {
 
 const getCloudfrontSignParams = () => {
   return {
-    keypairId: env.CLOUDFRONT_PUBLIC_KEY_ID,
+    keypairId: env.CLOUDFRONT_PUBLIC_KEY_ID!,
     privateKeyString: getPrivateKey(),
     expireTime: new Date().getTime() + 1000 * 60 * 60, // 1 hour
   }

packages/backend-core/src/objectStore/objectStore.ts

@@ -61,9 +61,9 @@ export function sanitizeBucket(input: string) {
 
 /**
  * Gets a connection to the object store using the S3 SDK.
- * @param {string} bucket the name of the bucket which blobs will be uploaded/retrieved from.
- * @param {object} opts configuration for the object store.
- * @return {Object} an S3 object store object, check S3 Nodejs SDK for usage.
+ * @param bucket the name of the bucket which blobs will be uploaded/retrieved from.
+ * @param opts configuration for the object store.
+ * @return an S3 object store object, check S3 Nodejs SDK for usage.
  * @constructor
  */
 export const ObjectStore = (

packages/backend-core/src/plugin/utils.ts

@@ -6,6 +6,7 @@ import {
   AutomationStepIdArray,
   AutomationIOType,
   AutomationCustomIOType,
+  DatasourceFeature,
 } from "@budibase/types"
 import joi from "joi"
 
@@ -67,9 +68,27 @@ function validateDatasource(schema: any) {
     version: joi.string().optional(),
     schema: joi.object({
       docs: joi.string(),
+      plus: joi.boolean().optional(),
+      isSQL: joi.boolean().optional(),
+      auth: joi
+        .object({
+          type: joi.string().required(),
+        })
+        .optional(),
+      features: joi
+        .object(
+          Object.fromEntries(
+            Object.values(DatasourceFeature).map(key => [
+              key,
+              joi.boolean().optional(),
+            ])
+          )
+        )
+        .optional(),
+      relationships: joi.boolean().optional(),
+      description: joi.string().required(),
       friendlyName: joi.string().required(),
       type: joi.string().allow(...DATASOURCE_TYPES),
-      description: joi.string().required(),
       datasource: joi.object().pattern(joi.string(), fieldValidator).required(),
       query: joi
         .object()

packages/backend-core/src/queue/inMemoryQueue.ts

@@ -5,9 +5,9 @@ import { timeout } from "../utils"
  * Bull works with a Job wrapper around all messages that contains a lot more information about
  * the state of the message, this object constructor implements the same schema of Bull jobs
  * for the sake of maintaining API consistency.
- * @param {string} queue The name of the queue which the message will be carried on.
- * @param {object} message The JSON message which will be passed back to the consumer.
- * @returns {Object} A new job which can now be put onto the queue, this is mostly an
+ * @param queue The name of the queue which the message will be carried on.
+ * @param message The JSON message which will be passed back to the consumer.
+ * @returns A new job which can now be put onto the queue, this is mostly an
  * internal structure so that an in memory queue can be easily swapped for a Bull queue.
  */
 function newJob(queue: string, message: any) {
@@ -32,8 +32,8 @@ class InMemoryQueue {
   _addCount: number
   /**
    * The constructor the queue, exactly the same as that of Bulls.
-   * @param {string} name The name of the queue which is being configured.
-   * @param {object|null} opts This is not used by the in memory queue as there is no real use
+   * @param name The name of the queue which is being configured.
+   * @param opts This is not used by the in memory queue as there is no real use
    * case when in memory, but is the same API as Bull
    */
   constructor(name: string, opts = null) {
@@ -49,7 +49,7 @@ class InMemoryQueue {
    * Same callback API as Bull, each callback passed to this will consume messages as they are
    * available. Please note this is a queue service, not a notification service, so each
    * consumer will receive different messages.
-   * @param {function<object>} func The callback function which will return a "Job", the same
+   * @param func The callback function which will return a "Job", the same
    * as the Bull API, within this job the property "data" contains the JSON message. Please
    * note this is incredibly limited compared to Bull as in reality the Job would contain
    * a lot more information about the queue and current status of Bull cluster.
@@ -73,9 +73,9 @@ class InMemoryQueue {
    * Simple function to replicate the add message functionality of Bull, putting
    * a new message on the queue. This then emits an event which will be used to
    * return the message to a consumer (if one is attached).
-   * @param {object} msg A message to be transported over the queue, this should be
+   * @param msg A message to be transported over the queue, this should be
    * a JSON message as this is required by Bull.
-   * @param {boolean} repeat serves no purpose for the import queue.
+   * @param repeat serves no purpose for the import queue.
    */
   // eslint-disable-next-line no-unused-vars
   add(msg: any, repeat: boolean) {
@@ -96,7 +96,7 @@ class InMemoryQueue {
 
   /**
    * This removes a cron which has been implemented, this is part of Bull API.
-   * @param {string} cronJobId The cron which is to be removed.
+   * @param cronJobId The cron which is to be removed.
    */
   removeRepeatableByKey(cronJobId: string) {
     // TODO: implement for testing

packages/backend-core/src/redis/redis.ts

@@ -142,7 +142,7 @@ function waitForConnection(selectDb: number = DEFAULT_SELECT_DB) {
  * this can only be done with redis streams because they will have an end.
  * @param stream A redis stream, specifically as this type of stream will have an end.
  * @param client The client to use for further lookups.
- * @return {Promise<object>} The final output of the stream
+ * @return The final output of the stream
  */
 function promisifyStream(stream: any, client: RedisWrapper) {
   return new Promise((resolve, reject) => {

packages/backend-core/src/security/permissions.ts

@@ -36,8 +36,8 @@ export function levelToNumber(perm: PermissionLevel) {
 
 /**
  * Given the specified permission level for the user return the levels they are allowed to carry out.
- * @param {string} userPermLevel The permission level of the user.
- * @return {string[]} All the permission levels this user is allowed to carry out.
+ * @param userPermLevel The permission level of the user.
+ * @return All the permission levels this user is allowed to carry out.
  */
 export function getAllowedLevels(userPermLevel: PermissionLevel): string[] {
   switch (userPermLevel) {

packages/backend-core/src/security/roles.ts

@@ -149,9 +149,9 @@ export function lowerBuiltinRoleID(roleId1?: string, roleId2?: string): string {
 /**
  * Gets the role object, this is mainly useful for two purposes, to check if the level exists and
  * to check if the role inherits any others.
- * @param {string|null} roleId The level ID to lookup.
- * @param {object|null} opts options for the function, like whether to halt errors, instead return public.
- * @returns {Promise<Role|object|null>} The role object, which may contain an "inherits" property.
+ * @param roleId The level ID to lookup.
+ * @param opts options for the function, like whether to halt errors, instead return public.
+ * @returns The role object, which may contain an "inherits" property.
  */
 export async function getRole(
   roleId?: string,
@@ -225,8 +225,8 @@ export async function getUserRoleIdHierarchy(
 /**
  * Returns an ordered array of the user's inherited role IDs, this can be used
  * to determine if a user can access something that requires a specific role.
- * @param {string} userRoleId The user's role ID, this can be found in their access token.
- * @returns {Promise<object[]>} returns an ordered array of the roles, with the first being their
+ * @param userRoleId The user's role ID, this can be found in their access token.
+ * @returns returns an ordered array of the roles, with the first being their
  * highest level of access and the last being the lowest level.
  */
 export async function getUserRoleHierarchy(userRoleId?: string) {
@@ -258,7 +258,7 @@ export async function getAllRoleIds(appId?: string) {
 
 /**
  * Given an app ID this will retrieve all of the roles that are currently within that app.
- * @return {Promise<object[]>} An array of the role objects that were found.
+ * @return An array of the role objects that were found.
  */
 export async function getAllRoles(appId?: string): Promise<RoleDoc[]> {
   if (appId) {

packages/backend-core/src/users/db.ts

@@ -21,17 +21,21 @@ import {
   User,
   UserStatus,
   UserGroup,
-  ContextUser,
 } from "@budibase/types"
 import {
   getAccountHolderFromUserIds,
   isAdmin,
+  isCreator,
   validateUniqueUser,
 } from "./utils"
 import { searchExistingEmails } from "./lookup"
 import { hash } from "../utils"
 
-type QuotaUpdateFn = (change: number, cb?: () => Promise<any>) => Promise<any>
+type QuotaUpdateFn = (
+  change: number,
+  creatorsChange: number,
+  cb?: () => Promise<any>
+) => Promise<any>
 type GroupUpdateFn = (groupId: string, userIds: string[]) => Promise<any>
 type FeatureFn = () => Promise<Boolean>
 type GroupGetFn = (ids: string[]) => Promise<UserGroup[]>
@@ -135,7 +139,7 @@ export class UserDB {
     if (!fullUser.roles) {
       fullUser.roles = {}
     }
-    // add the active status to a user if its not provided
+    // add the active status to a user if it's not provided
     if (fullUser.status == null) {
       fullUser.status = UserStatus.ACTIVE
     }
@@ -246,7 +250,8 @@ export class UserDB {
     }
 
     const change = dbUser ? 0 : 1 // no change if there is existing user
-    return UserDB.quotas.addUsers(change, async () => {
+    const creatorsChange = isCreator(dbUser) !== isCreator(user) ? 1 : 0
+    return UserDB.quotas.addUsers(change, creatorsChange, async () => {
       await validateUniqueUser(email, tenantId)
 
       let builtUser = await UserDB.buildUser(user, opts, tenantId, dbUser)
@@ -308,6 +313,7 @@ export class UserDB {
 
     let usersToSave: any[] = []
     let newUsers: any[] = []
+    let newCreators: any[] = []
 
     const emails = newUsersRequested.map((user: User) => user.email)
     const existingEmails = await searchExistingEmails(emails)
@@ -328,59 +334,66 @@ export class UserDB {
       }
       newUser.userGroups = groups
       newUsers.push(newUser)
+      if (isCreator(newUser)) {
+        newCreators.push(newUser)
+      }
     }
 
     const account = await accountSdk.getAccountByTenantId(tenantId)
-    return UserDB.quotas.addUsers(newUsers.length, async () => {
-      // create the promises array that will be called by bulkDocs
-      newUsers.forEach((user: any) => {
-        usersToSave.push(
-          UserDB.buildUser(
-            user,
-            {
-              hashPassword: true,
-              requirePassword: user.requirePassword,
-            },
-            tenantId,
-            undefined, // no dbUser
-            account
+    return UserDB.quotas.addUsers(
+      newUsers.length,
+      newCreators.length,
+      async () => {
+        // create the promises array that will be called by bulkDocs
+        newUsers.forEach((user: any) => {
+          usersToSave.push(
+            UserDB.buildUser(
+              user,
+              {
+                hashPassword: true,
+                requirePassword: user.requirePassword,
+              },
+              tenantId,
+              undefined, // no dbUser
+              account
+            )
           )
-        )
-      })
+        })
 
-      const usersToBulkSave = await Promise.all(usersToSave)
-      await usersCore.bulkUpdateGlobalUsers(usersToBulkSave)
+        const usersToBulkSave = await Promise.all(usersToSave)
+        await usersCore.bulkUpdateGlobalUsers(usersToBulkSave)
 
-      // Post-processing of bulk added users, e.g. events and cache operations
-      for (const user of usersToBulkSave) {
-        // TODO: Refactor to bulk insert users into the info db
-        // instead of relying on looping tenant creation
-        await platform.users.addUser(tenantId, user._id, user.email)
-        await eventHelpers.handleSaveEvents(user, undefined)
-      }
+        // Post-processing of bulk added users, e.g. events and cache operations
+        for (const user of usersToBulkSave) {
+          // TODO: Refactor to bulk insert users into the info db
+          // instead of relying on looping tenant creation
+          await platform.users.addUser(tenantId, user._id, user.email)
+          await eventHelpers.handleSaveEvents(user, undefined)
+        }
+
+        const saved = usersToBulkSave.map(user => {
+          return {
+            _id: user._id,
+            email: user.email,
+          }
+        })
+
+        // now update the groups
+        if (Array.isArray(saved) && groups) {
+          const groupPromises = []
+          const createdUserIds = saved.map(user => user._id)
+          for (let groupId of groups) {
+            groupPromises.push(UserDB.groups.addUsers(groupId, createdUserIds))
+          }
+          await Promise.all(groupPromises)
+        }
 
-      const saved = usersToBulkSave.map(user => {
         return {
-          _id: user._id,
-          email: user.email,
+          successful: saved,
+          unsuccessful,
         }
-      })
-
-      // now update the groups
-      if (Array.isArray(saved) && groups) {
-        const groupPromises = []
-        const createdUserIds = saved.map(user => user._id)
-        for (let groupId of groups) {
-          groupPromises.push(UserDB.groups.addUsers(groupId, createdUserIds))
-        }
-        await Promise.all(groupPromises)
       }
-
-      return {
-        successful: saved,
-        unsuccessful,
-      }
-    })
+    )
   }
 
   static async bulkDelete(userIds: string[]): Promise<BulkUserDeleted> {
@@ -420,11 +433,12 @@ export class UserDB {
       _deleted: true,
     }))
     const dbResponse = await usersCore.bulkUpdateGlobalUsers(toDelete)
+    const creatorsToDelete = usersToDelete.filter(isCreator)
 
-    await UserDB.quotas.removeUsers(toDelete.length)
     for (let user of usersToDelete) {
       await bulkDeleteProcessing(user)
     }
+    await UserDB.quotas.removeUsers(toDelete.length, creatorsToDelete.length)
 
     // Build Response
     // index users by id
@@ -473,7 +487,8 @@ export class UserDB {
 
     await db.remove(userId, dbUser._rev)
 
-    await UserDB.quotas.removeUsers(1)
+    const creatorsToDelete = isCreator(dbUser) ? 1 : 0
+    await UserDB.quotas.removeUsers(1, creatorsToDelete)
     await eventHelpers.handleDeleteEvents(dbUser)
     await cache.user.invalidateUser(userId)
     await sessions.invalidateSessions(userId, { reason: "deletion" })

packages/backend-core/src/users/users.ts

@@ -14,14 +14,15 @@ import {
 } from "../db"
 import {
   BulkDocsResponse,
-  ContextUser,
   SearchQuery,
   SearchQueryOperators,
   SearchUsersRequest,
   User,
+  ContextUser,
 } from "@budibase/types"
-import * as context from "../context"
 import { getGlobalDB } from "../context"
+import * as context from "../context"
+import { isCreator } from "./utils"
 
 type GetOpts = { cleanup?: boolean }
 
@@ -283,6 +284,19 @@ export async function getUserCount() {
   return response.total_rows
 }
 
+export async function getCreatorCount() {
+  let creators = 0
+  async function iterate(startPage?: string) {
+    const page = await paginatedUsers({ bookmark: startPage })
+    creators += page.data.filter(isCreator).length
+    if (page.hasNextPage) {
+      await iterate(page.nextPage)
+    }
+  }
+  await iterate()
+  return creators
+}
+
 // used to remove the builder/admin permissions, for processing the
 // user as an app user (they may have some specific role/group
 export function removePortalUserPermissions(user: User | ContextUser) {

packages/backend-core/src/users/utils.ts

@@ -10,6 +10,7 @@ import { getAccountByTenantId } from "../accounts"
 // extract from shared-core to make easily accessible from backend-core
 export const isBuilder = sdk.users.isBuilder
 export const isAdmin = sdk.users.isAdmin
+export const isCreator = sdk.users.isCreator
 export const isGlobalBuilder = sdk.users.isGlobalBuilder
 export const isAdminOrBuilder = sdk.users.isAdminOrBuilder
 export const hasAdminPermissions = sdk.users.hasAdminPermissions

packages/backend-core/src/utils/utils.ts

@@ -79,8 +79,8 @@ export function isPublicApiRequest(ctx: Ctx): boolean {
 
 /**
  * Given a request tries to find the appId, which can be located in various places
- * @param {object} ctx The main request body to look through.
- * @returns {string|undefined} If an appId was found it will be returned.
+ * @param ctx The main request body to look through.
+ * @returns If an appId was found it will be returned.
  */
 export async function getAppIdFromCtx(ctx: Ctx) {
   // look in headers
@@ -135,7 +135,7 @@ function parseAppIdFromUrl(url?: string) {
 
 /**
  * opens the contents of the specified encrypted JWT.
- * @return {object} the contents of the token.
+ * @return the contents of the token.
  */
 export function openJwt(token: string) {
   if (!token) {
@@ -169,8 +169,8 @@ export function isValidInternalAPIKey(apiKey: string) {
 
 /**
  * Get a cookie from context, and decrypt if necessary.
- * @param {object} ctx The request which is to be manipulated.
- * @param {string} name The name of the cookie to get.
+ * @param ctx The request which is to be manipulated.
+ * @param name The name of the cookie to get.
  */
 export function getCookie(ctx: Ctx, name: string) {
   const cookie = ctx.cookies.get(name)
@@ -184,10 +184,10 @@ export function getCookie(ctx: Ctx, name: string) {
 
 /**
  * Store a cookie for the request - it will not expire.
- * @param {object} ctx The request which is to be manipulated.
- * @param {string} name The name of the cookie to set.
- * @param {string|object} value The value of cookie which will be set.
- * @param {object} opts options like whether to sign.
+ * @param ctx The request which is to be manipulated.
+ * @param name The name of the cookie to set.
+ * @param value The value of cookie which will be set.
+ * @param opts options like whether to sign.
  */
 export function setCookie(
   ctx: Ctx,
@@ -223,8 +223,8 @@ export function clearCookie(ctx: Ctx, name: string) {
 /**
  * Checks if the API call being made (based on the provided ctx object) is from the client. If
  * the call is not from a client app then it is from the builder.
- * @param {object} ctx The koa context object to be tested.
- * @return {boolean} returns true if the call is from the client lib (a built app rather than the builder).
+ * @param ctx The koa context object to be tested.
+ * @return returns true if the call is from the client lib (a built app rather than the builder).
  */
 export function isClient(ctx: Ctx) {
   return ctx.headers[Header.TYPE] === "client"

packages/backend-core/tests/core/users/users.spec.js

@@ -0,0 +1,54 @@
+const _ = require('lodash/fp')
+const {structures} = require("../../../tests")
+
+jest.mock("../../../src/context")
+jest.mock("../../../src/db")
+
+const context = require("../../../src/context")
+const db = require("../../../src/db")
+
+const {getCreatorCount} = require('../../../src/users/users')
+
+describe("Users", () => {
+
+  let getGlobalDBMock
+  let getGlobalUserParamsMock
+  let paginationMock
+
+  beforeEach(() => {
+    jest.resetAllMocks()
+
+    getGlobalDBMock = jest.spyOn(context, "getGlobalDB")
+    getGlobalUserParamsMock = jest.spyOn(db, "getGlobalUserParams")
+    paginationMock = jest.spyOn(db, "pagination")
+  })
+
+  it("Retrieves the number of creators", async () => {
+    const getUsers = (offset, limit, creators = false) => {
+      const range = _.range(offset, limit)
+      const opts = creators ? {builder: {global: true}} : undefined
+      return range.map(() => structures.users.user(opts))
+    }
+    const page1Data = getUsers(0, 8)
+    const page2Data = getUsers(8, 12, true)
+    getGlobalDBMock.mockImplementation(() => ({
+      name   : "fake-db",
+      allDocs: () => ({
+        rows: [...page1Data, ...page2Data]
+      })
+    }))
+    paginationMock.mockImplementationOnce(() => ({
+      data: page1Data,
+      hasNextPage: true,
+      nextPage: "1"
+    }))
+    paginationMock.mockImplementation(() => ({
+      data: page2Data,
+      hasNextPage: false,
+      nextPage: undefined
+    }))
+    const creatorsCount = await getCreatorCount()
+    expect(creatorsCount).toBe(4)
+    expect(paginationMock).toHaveBeenCalledTimes(2)
+  })
+})

packages/backend-core/tests/core/utilities/structures/licenses.ts

@@ -72,6 +72,11 @@ export function quotas(): Quotas {
           value: 1,
           triggers: [],
         },
+        creators: {
+          name: "Creators",
+          value: 1,
+          triggers: [],
+        },
         userGroups: {
           name: "User Groups",
           value: 1,
@@ -118,6 +123,10 @@ export function customer(): Customer {
 export function subscription(): Subscription {
   return {
     amount: 10000,
+    amounts: {
+      user: 10000,
+      creator: 0,
+    },
     cancelAt: undefined,
     currency: "usd",
     currentPeriodEnd: 0,
@@ -126,6 +135,10 @@ export function subscription(): Subscription {
     duration: PriceDuration.MONTHLY,
     pastDueAt: undefined,
     quantity: 0,
+    quantities: {
+      user: 0,
+      creator: 0,
+    },
     status: "active",
   }
 }

packages/backend-core/tests/core/utilities/structures/quotas.ts

@@ -1,6 +1,6 @@
 import { MonthlyQuotaName, QuotaUsage } from "@budibase/types"
 
-export const usage = (): QuotaUsage => {
+export const usage = (users: number = 0, creators: number = 0): QuotaUsage => {
   return {
     _id: "usage_quota",
     quotaReset: new Date().toISOString(),
@@ -58,7 +58,8 @@ export const usage = (): QuotaUsage => {
     usageQuota: {
       apps: 0,
       plugins: 0,
-      users: 0,
+      users,
+      creators,
       userGroups: 0,
       rows: 0,
       triggers: {},

packages/bbui/package.json

@@ -82,9 +82,9 @@
     "@spectrum-css/vars": "3.0.1",
     "dayjs": "^1.10.8",
     "easymde": "^2.16.1",
+    "svelte-dnd-action": "^0.9.8",
     "svelte-flatpickr": "3.2.3",
-    "svelte-portal": "^1.0.0",
-    "svelte-dnd-action": "^0.9.8"
+    "svelte-portal": "^1.0.0"
   },
   "resolutions": {
     "loader-utils": "1.4.1"

packages/builder/build/copy.js

@@ -1,8 +0,0 @@
-const ncp = require("ncp").ncp
-
-ncp("./dist", "../server/builder", function (err) {
-  if (err) {
-    return console.error(err)
-  }
-  console.log("Copied dist folder to ../server/builder")
-})

packages/builder/package.json

@@ -64,7 +64,6 @@
     "@fortawesome/fontawesome-svg-core": "^6.2.1",
     "@fortawesome/free-brands-svg-icons": "^6.2.1",
     "@fortawesome/free-solid-svg-icons": "^6.2.1",
-    "@sentry/browser": "5.19.1",
     "@spectrum-css/page": "^3.0.1",
     "@spectrum-css/vars": "^3.0.1",
     "codemirror": "^5.59.0",
@@ -85,8 +84,8 @@
     "@babel/core": "^7.12.14",
     "@babel/plugin-transform-runtime": "^7.13.10",
     "@babel/preset-env": "^7.13.12",
-    "@rollup/plugin-replace": "^2.4.2",
-    "@roxi/routify": "2.18.5",
+    "@rollup/plugin-replace": "^5.0.3",
+    "@roxi/routify": "2.18.12",
     "@sveltejs/vite-plugin-svelte": "1.0.1",
     "@testing-library/jest-dom": "5.17.0",
     "@testing-library/svelte": "^3.2.2",
@@ -95,16 +94,18 @@
     "jest": "29.6.2",
     "jsdom": "^21.1.1",
     "ncp": "^2.0.0",
-    "rollup": "^2.44.0",
     "svelte": "^3.48.0",
     "svelte-jester": "^1.3.2",
-    "vite": "^3.0.8",
-    "vite-plugin-static-copy": "^0.16.0",
+    "vite": "^4.4.11",
+    "vite-plugin-static-copy": "^0.17.0",
     "vitest": "^0.29.2"
   },
   "nx": {
     "targets": {
       "build": {
+        "outputs": [
+          "{workspaceRoot}/packages/server/builder"
+        ],
         "dependsOn": [
           {
             "projects": [

packages/builder/src/analytics/SentryClient.js

@@ -1,37 +0,0 @@
-import * as Sentry from "@sentry/browser"
-
-export default class SentryClient {
-  constructor(dsn) {
-    this.dsn = dsn
-  }
-
-  init() {
-    if (this.dsn) {
-      Sentry.init({ dsn: this.dsn })
-
-      this.initalised = true
-    }
-  }
-
-  /**
-   * Capture an exception and send it to sentry.
-   * @param {Error} err - JS error object
-   */
-  captureException(err) {
-    if (!this.initalised) return
-
-    Sentry.captureException(err)
-  }
-
-  /**
-   * Identify user in sentry.
-   * @param {String} id - Unique user id
-   */
-  identify(id) {
-    if (!this.initalised) return
-
-    Sentry.configureScope(scope => {
-      scope.setUser({ id })
-    })
-  }
-}

packages/builder/src/analytics/index.js

@@ -1,16 +1,14 @@
 import { API } from "api"
 import PosthogClient from "./PosthogClient"
 import IntercomClient from "./IntercomClient"
-import SentryClient from "./SentryClient"
 import { Events, EventSource } from "./constants"
 
 const posthog = new PosthogClient(process.env.POSTHOG_TOKEN)
-const sentry = new SentryClient(process.env.SENTRY_DSN)
 const intercom = new IntercomClient(process.env.INTERCOM_TOKEN)
 
 class AnalyticsHub {
   constructor() {
-    this.clients = [posthog, sentry, intercom]
+    this.clients = [posthog, intercom]
   }
 
   async activate() {
@@ -23,12 +21,9 @@ class AnalyticsHub {
 
   identify(id) {
     posthog.identify(id)
-    sentry.identify(id)
   }
 
-  captureException(err) {
-    sentry.captureException(err)
-  }
+  captureException(_err) {}
 
   captureEvent(eventName, props = {}) {
     posthog.captureEvent(eventName, props)

packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte

@@ -36,7 +36,7 @@
   import { FieldType, FieldSubtype, SourceName } from "@budibase/types"
   import RelationshipSelector from "components/common/RelationshipSelector.svelte"
 
-  const AUTO_TYPE = "auto"
+  const AUTO_TYPE = FIELDS.AUTO.type
   const FORMULA_TYPE = FIELDS.FORMULA.type
   const LINK_TYPE = FIELDS.LINK.type
   const STRING_TYPE = FIELDS.STRING.type
@@ -60,8 +60,13 @@
     {}
   )
 
-  function makeFieldId(type, subtype) {
-    return `${type}${subtype || ""}`.toUpperCase()
+  function makeFieldId(type, subtype, autocolumn) {
+    // don't make field IDs for auto types
+    if (type === AUTO_TYPE || autocolumn) {
+      return type.toUpperCase()
+    } else {
+      return `${type}${subtype || ""}`.toUpperCase()
+    }
   }
 
   let originalName
@@ -183,7 +188,8 @@
     if (!savingColumn) {
       editableColumn.fieldId = makeFieldId(
         editableColumn.type,
-        editableColumn.subtype
+        editableColumn.subtype,
+        editableColumn.autocolumn
       )
 
       allowedTypes = getAllowedTypes().map(t => ({
@@ -419,7 +425,7 @@
         FIELDS.FORMULA,
         FIELDS.JSON,
         isUsers ? FIELDS.USERS : FIELDS.USER,
-        { name: "Auto Column", type: AUTO_TYPE },
+        FIELDS.AUTO,
       ]
     } else {
       let fields = [
@@ -538,7 +544,7 @@
     getOptionValue={field => field.fieldId}
     getOptionIcon={field => field.icon}
     isOptionEnabled={option => {
-      if (option.type == AUTO_TYPE) {
+      if (option.type === AUTO_TYPE) {
         return availableAutoColumnKeys?.length > 0
       }
       return true

packages/builder/src/components/backend/Datasources/CreateEditRelationship.svelte

@@ -13,6 +13,8 @@
   import { Helpers } from "@budibase/bbui"
   import { RelationshipErrorChecker } from "./relationshipErrors"
   import { onMount } from "svelte"
+  import RelationshipSelector from "components/common/RelationshipSelector.svelte"
+  import { PrettyRelationshipDefinitions } from "constants/backend"
 
   export let save
   export let datasource
@@ -22,16 +24,21 @@
   export let selectedFromTable
   export let close
 
-  const relationshipTypes = [
-    {
-      label: "One to Many",
-      value: RelationshipType.MANY_TO_ONE,
+  let relationshipMap = {
+    [RelationshipType.MANY_TO_MANY]: {
+      part1: PrettyRelationshipDefinitions.MANY,
+      part2: PrettyRelationshipDefinitions.MANY,
     },
-    {
-      label: "Many to Many",
-      value: RelationshipType.MANY_TO_MANY,
+    [RelationshipType.MANY_TO_ONE]: {
+      part1: PrettyRelationshipDefinitions.ONE,
+      part2: PrettyRelationshipDefinitions.MANY,
     },
-  ]
+  }
+  let relationshipOpts1 = Object.values(PrettyRelationshipDefinitions)
+  let relationshipOpts2 = Object.values(PrettyRelationshipDefinitions)
+
+  let relationshipPart1 = PrettyRelationshipDefinitions.MANY
+  let relationshipPart2 = PrettyRelationshipDefinitions.ONE
 
   let originalFromColumnName = toRelationship.name,
     originalToColumnName = fromRelationship.name
@@ -49,14 +56,32 @@
   )
   let errors = {}
   let fromPrimary, fromForeign, fromColumn, toColumn
-  let fromId, toId, throughId, throughToKey, throughFromKey
+
+  let throughId, throughToKey, throughFromKey
   let isManyToMany, isManyToOne, relationshipType
   let hasValidated = false
 
+  $: fromId = null
+  $: toId = null
+
   $: tableOptions = plusTables.map(table => ({
     label: table.name,
     value: table._id,
+    name: table.name,
+    _id: table._id,
   }))
+
+  $: {
+    // Determine the relationship type based on the selected values of both parts
+    relationshipType = Object.entries(relationshipMap).find(
+      ([_, parts]) =>
+        parts.part1 === relationshipPart1 && parts.part2 === relationshipPart2
+    )?.[0]
+
+    changed(() => {
+      hasValidated = false
+    })
+  }
   $: valid =
     getErrorCount(errors) === 0 && allRequiredAttributesSet(relationshipType)
   $: isManyToMany = relationshipType === RelationshipType.MANY_TO_MANY
@@ -338,33 +363,34 @@
   onConfirm={saveRelationship}
   disabled={!valid}
 >
-  <Select
-    label="Relationship type"
-    options={relationshipTypes}
-    bind:value={relationshipType}
-    bind:error={errors.relationshipType}
-    on:change={() =>
-      changed(() => {
-        hasValidated = false
-      })}
-  />
   <div class="headings">
     <Detail>Tables</Detail>
   </div>
-  {#if !selectedFromTable}
-    <Select
-      label="Select from table"
-      options={tableOptions}
-      bind:value={fromId}
-      bind:error={errors.fromTable}
-      on:change={e =>
-        changed(() => {
-          const table = plusTables.find(tbl => tbl._id === e.detail)
-          fromColumn = table?.name || ""
-          fromPrimary = table?.primary?.[0]
-        })}
-    />
-  {/if}
+
+  <RelationshipSelector
+    bind:relationshipPart1
+    bind:relationshipPart2
+    bind:relationshipTableIdPrimary={fromId}
+    bind:relationshipTableIdSecondary={toId}
+    {relationshipOpts1}
+    {relationshipOpts2}
+    {tableOptions}
+    {errors}
+    primaryDisabled={selectedFromTable}
+    primaryTableChanged={e =>
+      changed(() => {
+        const table = plusTables.find(tbl => tbl._id === e.detail)
+        fromColumn = table?.name || ""
+        fromPrimary = table?.primary?.[0]
+      })}
+    secondaryTableChanged={e =>
+      changed(() => {
+        const table = plusTables.find(tbl => tbl._id === e.detail)
+        toColumn = table.name || ""
+        fromForeign = null
+      })}
+  />
+
   {#if isManyToOne && fromId}
     <Select
       label={`Primary Key (${getTable(fromId).name})`}
@@ -374,18 +400,6 @@
       on:change={changed}
     />
   {/if}
-  <Select
-    label={"Select to table"}
-    options={tableOptions}
-    bind:value={toId}
-    bind:error={errors.toTable}
-    on:change={e =>
-      changed(() => {
-        const table = plusTables.find(tbl => tbl._id === e.detail)
-        toColumn = table.name || ""
-        fromForeign = null
-      })}
-  />
   {#if isManyToMany}
     <Select
       label={"Through"}

packages/builder/src/components/backend/Datasources/TableImportSelection/index.svelte

@@ -57,7 +57,7 @@
       {#if $store.error}
         <InlineAlert
           type="error"
-          header={$store.error.title}
+          header="Error fetching {tableType}"
           message={$store.error.description}
         />
       {/if}

packages/builder/src/components/backend/Datasources/TableImportSelection/tableSelectionStore.js

@@ -1,6 +1,6 @@
 import { derived, writable, get } from "svelte/store"
 import { keepOpen, notifications } from "@budibase/bbui"
-import { datasources, ImportTableError, tables } from "stores/backend"
+import { datasources, tables } from "stores/backend"
 
 export const createTableSelectionStore = (integration, datasource) => {
   const tableNamesStore = writable([])
@@ -30,12 +30,7 @@ export const createTableSelectionStore = (integration, datasource) => {
       notifications.success(`Tables fetched successfully.`)
       await onComplete()
     } catch (err) {
-      if (err instanceof ImportTableError) {
-        errorStore.set(err)
-      } else {
-        notifications.error("Error fetching tables.")
-      }
-
+      errorStore.set(err)
       return keepOpen
     }
   }

packages/builder/src/components/common/RelationshipSelector.svelte

@@ -6,11 +6,14 @@
   export let relationshipTableIdPrimary
   export let relationshipTableIdSecondary
   export let editableColumn
-  export let linkEditDisabled
+  export let linkEditDisabled = false
   export let tableOptions
   export let errors
   export let relationshipOpts1
   export let relationshipOpts2
+  export let primaryTableChanged
+  export let secondaryTableChanged
+  export let primaryDisabled = true
 </script>
 
 <div class="relationship-container">
@@ -19,16 +22,19 @@
       disabled={linkEditDisabled}
       bind:value={relationshipPart1}
       options={relationshipOpts1}
+      bind:error={errors.relationshipType}
     />
   </div>
   <div class="relationship-label">in</div>
   <div class="relationship-part">
     <Select
-      disabled
+      disabled={primaryDisabled}
       options={tableOptions}
       getOptionLabel={table => table.name}
       getOptionValue={table => table._id}
       bind:value={relationshipTableIdPrimary}
+      on:change={primaryTableChanged}
+      bind:error={errors.fromTable}
     />
   </div>
 </div>
@@ -46,20 +52,24 @@
     <Select
       disabled={linkEditDisabled}
       bind:value={relationshipTableIdSecondary}
+      bind:error={errors.toTable}
       options={tableOptions.filter(
         table => table._id !== relationshipTableIdPrimary
       )}
       getOptionLabel={table => table.name}
       getOptionValue={table => table._id}
+      on:change={secondaryTableChanged}
     />
   </div>
 </div>
-<Input
-  disabled={linkEditDisabled}
-  label={`Column name in other table`}
-  bind:value={editableColumn.fieldName}
-  error={errors.relatedName}
-/>
+{#if editableColumn}
+  <Input
+    disabled={linkEditDisabled}
+    label={`Column name in other table`}
+    bind:value={editableColumn.fieldName}
+    error={errors.relatedName}
+  />
+{/if}
 
 <style>
   .relationship-container {

packages/builder/src/components/design/settings/controls/FieldConfiguration.svelte

@@ -1,91 +0,0 @@
-<script>
-  import { Button, ActionButton, Drawer } from "@budibase/bbui"
-  import { createEventDispatcher } from "svelte"
-  import ColumnDrawer from "./ColumnEditor/ColumnDrawer.svelte"
-  import { cloneDeep } from "lodash/fp"
-  import {
-    getDatasourceForProvider,
-    getSchemaForDatasource,
-  } from "builderStore/dataBinding"
-  import { currentAsset } from "builderStore"
-  import { getFields } from "helpers/searchFields"
-
-  export let componentInstance
-  export let value = []
-  export let allowCellEditing = true
-  export let subject = "Table"
-
-  const dispatch = createEventDispatcher()
-
-  let drawer
-  let boundValue
-
-  $: datasource = getDatasourceForProvider($currentAsset, componentInstance)
-  $: schema = getSchema($currentAsset, datasource)
-  $: options = allowCellEditing
-    ? Object.keys(schema || {})
-    : enrichedSchemaFields?.map(field => field.name)
-  $: sanitisedValue = getValidColumns(value, options)
-  $: updateBoundValue(sanitisedValue)
-  $: enrichedSchemaFields = getFields(Object.values(schema || {}), {
-    allowLinks: true,
-  })
-
-  const getSchema = (asset, datasource) => {
-    const schema = getSchemaForDatasource(asset, datasource).schema
-
-    // Don't show ID and rev in tables
-    if (schema) {
-      delete schema._id
-      delete schema._rev
-    }
-
-    return schema
-  }
-
-  const updateBoundValue = value => {
-    boundValue = cloneDeep(value)
-  }
-
-  const getValidColumns = (columns, options) => {
-    if (!Array.isArray(columns) || !columns.length) {
-      return []
-    }
-    // We need to account for legacy configs which would just be an array
-    // of strings
-    if (typeof columns[0] === "string") {
-      columns = columns.map(col => ({
-        name: col,
-        displayName: col,
-      }))
-    }
-    return columns.filter(column => {
-      return options.includes(column.name)
-    })
-  }
-
-  const open = () => {
-    updateBoundValue(sanitisedValue)
-    drawer.show()
-  }
-
-  const save = () => {
-    dispatch("change", getValidColumns(boundValue, options))
-    drawer.hide()
-  }
-</script>
-
-<ActionButton on:click={open}>Configure columns</ActionButton>
-<Drawer bind:this={drawer} title="{subject} Columns">
-  <svelte:fragment slot="description">
-    Configure the columns in your {subject.toLowerCase()}.
-  </svelte:fragment>
-  <Button cta slot="buttons" on:click={save}>Save</Button>
-  <ColumnDrawer
-    slot="body"
-    bind:columns={boundValue}
-    {options}
-    {schema}
-    {allowCellEditing}
-  />
-</Drawer>

packages/builder/src/components/start/ImportAppModal.svelte

@@ -54,6 +54,7 @@
       label="App export"
       on:change={e => {
         file = e.detail?.[0]
+        encrypted = file?.name?.endsWith(".enc.tar.gz")
       }}
     />
     <Toggle text="Encrypted" bind:value={encrypted} />

packages/builder/src/constants/backend/index.js

@@ -1,5 +1,21 @@
 import { FieldType, FieldSubtype } from "@budibase/types"
 
+export const AUTO_COLUMN_SUB_TYPES = {
+  AUTO_ID: "autoID",
+  CREATED_BY: "createdBy",
+  CREATED_AT: "createdAt",
+  UPDATED_BY: "updatedBy",
+  UPDATED_AT: "updatedAt",
+}
+
+export const AUTO_COLUMN_DISPLAY_NAMES = {
+  AUTO_ID: "Auto ID",
+  CREATED_BY: "Created By",
+  CREATED_AT: "Created At",
+  UPDATED_BY: "Updated By",
+  UPDATED_AT: "Updated At",
+}
+
 export const FIELDS = {
   STRING: {
     name: "Text",
@@ -107,6 +123,12 @@ export const FIELDS = {
       presence: false,
     },
   },
+  AUTO: {
+    name: "Auto Column",
+    type: FieldType.AUTO,
+    icon: "MagicWand",
+    constraints: {},
+  },
   FORMULA: {
     name: "Formula",
     type: FieldType.FORMULA,
@@ -139,22 +161,6 @@ export const FIELDS = {
   },
 }
 
-export const AUTO_COLUMN_SUB_TYPES = {
-  AUTO_ID: "autoID",
-  CREATED_BY: "createdBy",
-  CREATED_AT: "createdAt",
-  UPDATED_BY: "updatedBy",
-  UPDATED_AT: "updatedAt",
-}
-
-export const AUTO_COLUMN_DISPLAY_NAMES = {
-  AUTO_ID: "Auto ID",
-  CREATED_BY: "Created By",
-  CREATED_AT: "Created At",
-  UPDATED_BY: "Updated By",
-  UPDATED_AT: "Updated At",
-}
-
 export const FILE_TYPES = {
   IMAGE: ["png", "tiff", "gif", "raw", "jpg", "jpeg"],
   CODE: ["js", "rs", "py", "java", "rb", "hs", "yml"],

packages/builder/src/pages/builder/app/[application]/data/datasource/[datasourceId]/_components/EditDatasourceConfigButton.svelte

@@ -62,7 +62,14 @@
       </div>
     {/if}
     <div class="truncate">
-      <Body>{getSubtitle(datasource)}</Body>
+      <Body>
+        {@const subtitle = getSubtitle(datasource)}
+        {#if subtitle}
+          {subtitle}
+        {:else}
+          {Object.values(datasource.config).join(" / ")}
+        {/if}
+      </Body>
     </div>
   </div>
   <div class="right">

packages/builder/src/pages/builder/app/[application]/data/datasource/[datasourceId]/_layout.svelte

@@ -23,5 +23,7 @@
 </script>
 
 {#key $params.datasourceId}
-  <slot />
+  {#if $datasources.selected}
+    <slot />
+  {/if}
 {/key}

packages/builder/src/pages/builder/app/[application]/data/datasource/[datasourceId]/index.svelte

@@ -16,8 +16,7 @@
   let selectedPanel = null
   let panelOptions = []
 
-  // datasources.selected can return null temporarily on datasource deletion
-  $: datasource = $datasources.selected || {}
+  $: datasource = $datasources.selected
 
   $: getOptions(datasource)
 

packages/builder/src/pages/builder/app/[application]/settings/exportImport.svelte

@@ -13,7 +13,7 @@
   import ExportAppModal from "components/start/ExportAppModal.svelte"
   import ImportAppModal from "components/start/ImportAppModal.svelte"
 
-  $: filteredApps = $apps.filter(app => app.devId == $store.appId)
+  $: filteredApps = $apps.filter(app => app.devId === $store.appId)
   $: app = filteredApps.length ? filteredApps[0] : {}
   $: appDeployed = app?.status === AppStatus.DEPLOYED
 

packages/builder/src/pages/builder/auth/forgot.svelte

@@ -43,7 +43,7 @@
   })
 </script>
 
-<TestimonialPage>
+<TestimonialPage enabled={$organisation.testimonialsEnabled}>
   <Layout gap="S" noPadding>
     <img alt="logo" src={$organisation.logoUrl || Logo} />
     <span class="heading-wrap">

packages/builder/src/pages/builder/auth/reset.svelte

@@ -53,7 +53,7 @@
   })
 </script>
 
-<TestimonialPage>
+<TestimonialPage enabled={$organisation.testimonialsEnabled}>
   <Layout gap="S" noPadding>
     {#if loaded}
       <img alt="logo" src={$organisation.logoUrl || Logo} />

packages/builder/src/stores/backend/datasources.js

@@ -9,15 +9,19 @@ import { API } from "api"
 import { DatasourceFeature } from "@budibase/types"
 import { TableNames } from "constants"
 
-export class ImportTableError extends Error {
-  constructor(message) {
-    super(message)
-    const [title, description] = message.split(" - ")
+class TableImportError extends Error {
+  constructor(errors) {
+    super()
+    this.name = "TableImportError"
+    this.errors = errors
+  }
 
-    this.name = "TableSelectionError"
-    // Capitalize the first character of both the title and description
-    this.title = title[0].toUpperCase() + title.substr(1)
-    this.description = description[0].toUpperCase() + description.substr(1)
+  get description() {
+    let message = ""
+    for (const key in this.errors) {
+      message += `${key}: ${this.errors[key]}\n`
+    }
+    return message
   }
 }
 
@@ -25,7 +29,6 @@ export function createDatasourcesStore() {
   const store = writable({
     list: [],
     selectedDatasourceId: null,
-    schemaError: null,
   })
 
   const derivedStore = derived([store, tables], ([$store, $tables]) => {
@@ -75,18 +78,13 @@ export function createDatasourcesStore() {
     store.update(state => ({
       ...state,
       selectedDatasourceId: id,
-      // Remove any possible schema error
-      schemaError: null,
     }))
   }
 
   const updateDatasource = response => {
-    const { datasource, error } = response
-    if (error) {
-      store.update(state => ({
-        ...state,
-        schemaError: error,
-      }))
+    const { datasource, errors } = response
+    if (errors && Object.keys(errors).length > 0) {
+      throw new TableImportError(errors)
     }
     replaceDatasource(datasource._id, datasource)
     select(datasource._id)
@@ -94,20 +92,11 @@ export function createDatasourcesStore() {
   }
 
   const updateSchema = async (datasource, tablesFilter) => {
-    try {
-      const response = await API.buildDatasourceSchema({
-        datasourceId: datasource?._id,
-        tablesFilter,
-      })
-      updateDatasource(response)
-    } catch (e) {
-      // buildDatasourceSchema call returns user presentable errors with two parts divided with a " - ".
-      if (e.message.split(" - ").length === 2) {
-        throw new ImportTableError(e.message)
-      } else {
-        throw e
-      }
-    }
+    const response = await API.buildDatasourceSchema({
+      datasourceId: datasource?._id,
+      tablesFilter,
+    })
+    updateDatasource(response)
   }
 
   const sourceCount = source => {
@@ -136,6 +125,7 @@ export function createDatasourcesStore() {
       config,
       name: `${integration.friendlyName}${nameModifier}`,
       plus: integration.plus && integration.name !== IntegrationTypes.REST,
+      isSQL: integration.isSQL,
     }
 
     if (await checkDatasourceValidity(integration, datasource)) {
@@ -171,12 +161,6 @@ export function createDatasourcesStore() {
     replaceDatasource(datasource._id, null)
   }
 
-  const removeSchemaError = () => {
-    store.update(state => {
-      return { ...state, schemaError: null }
-    })
-  }
-
   const replaceDatasource = (datasourceId, datasource) => {
     if (!datasourceId) {
       return
@@ -229,7 +213,6 @@ export function createDatasourcesStore() {
     create,
     update,
     delete: deleteDatasource,
-    removeSchemaError,
     replaceDatasource,
     getTableNames,
   }

packages/builder/src/stores/backend/index.js

@@ -4,7 +4,7 @@ export { views } from "./views"
 export { viewsV2 } from "./viewsV2"
 export { permissions } from "./permissions"
 export { roles } from "./roles"
-export { datasources, ImportTableError } from "./datasources"
+export { datasources } from "./datasources"
 export { integrations } from "./integrations"
 export { sortedIntegrations } from "./sortedIntegrations"
 export { queries } from "./queries"

packages/builder/vite.config.js

@@ -80,7 +80,6 @@ export default defineConfig(({ mode }) => {
         "process.env.INTERCOM_TOKEN": JSON.stringify(
           process.env.INTERCOM_TOKEN
         ),
-        "process.env.SENTRY_DSN": JSON.stringify(process.env.SENTRY_DSN),
       }),
       copyFonts("fonts"),
       ...(isProduction ? [] : devOnlyPlugins),

packages/client/manifest.json

@@ -5609,6 +5609,21 @@
           }
         ]
       },
+      {
+        "type": "event",
+        "label": "On row click",
+        "key": "onRowClick",
+        "context": [
+          {
+            "label": "Clicked row",
+            "key": "row"
+          }
+        ],
+        "dependsOn": {
+          "setting": "allowEditRows",
+          "value": false
+        }
+      },
       {
         "type": "boolean",
         "label": "Add rows",

packages/client/src/api/api.js

@@ -32,7 +32,7 @@ export const API = createAPIClient({
   },
 
   // Show an error notification for all API failures.
-  // We could also log these to sentry.
+  // We could also log these to Posthog.
   // Or we could check error.status and redirect to login on a 403 etc.
   onError: error => {
     const { status, method, url, message, handled, suppressErrors } =

packages/client/src/components/app/GridBlock.svelte

@@ -14,12 +14,14 @@
   export let initialSortOrder = null
   export let fixedRowHeight = null
   export let columns = null
+  export let onRowClick = null
 
   const component = getContext("component")
   const { styleable, API, builderStore, notificationStore } = getContext("sdk")
 
   $: columnWhitelist = columns?.map(col => col.name)
   $: schemaOverrides = getSchemaOverrides(columns)
+  $: handleRowClick = allowEditRows ? undefined : onRowClick
 
   const getSchemaOverrides = columns => {
     let overrides = {}
@@ -56,6 +58,7 @@
     showControls={false}
     notifySuccess={notificationStore.actions.success}
     notifyError={notificationStore.actions.error}
+    on:rowclick={e => handleRowClick?.({ row: e.detail })}
   />
 </div>
 

packages/frontend-core/src/components/grid/cells/GutterCell.svelte

@@ -17,13 +17,24 @@
   const { config, dispatch, selectedRows } = getContext("grid")
   const svelteDispatch = createEventDispatcher()
 
-  const select = () => {
+  const select = e => {
+    e.stopPropagation()
     svelteDispatch("select")
     const id = row?._id
     if (id) {
       selectedRows.actions.toggleRow(id)
     }
   }
+
+  const bulkDelete = e => {
+    e.stopPropagation()
+    dispatch("request-bulk-delete")
+  }
+
+  const expand = e => {
+    e.stopPropagation()
+    svelteDispatch("expand")
+  }
 </script>
 
 <GridCell
@@ -56,7 +67,7 @@
       {/if}
     {/if}
     {#if rowSelected && $config.canDeleteRows}
-      <div class="delete" on:click={() => dispatch("request-bulk-delete")}>
+      <div class="delete" on:click={bulkDelete}>
         <Icon
           name="Delete"
           size="S"
@@ -65,12 +76,7 @@
       </div>
     {:else}
       <div class="expand" class:visible={$config.canExpandRows && expandable}>
-        <Icon
-          size="S"
-          name="Maximize"
-          hoverable
-          on:click={() => svelteDispatch("expand")}
-        />
+        <Icon size="S" name="Maximize" hoverable on:click={expand} />
       </div>
     {/if}
   </div>

packages/frontend-core/src/components/grid/cells/RelationshipCell.svelte

@@ -260,29 +260,31 @@
       class:wrap={editable || contentLines > 1}
       on:wheel={e => (focused ? e.stopPropagation() : null)}
     >
-      {#each value || [] as relationship}
-        {#if relationship[primaryDisplay] || relationship.primaryDisplay}
-          <div class="badge">
-            <span
-              on:click={editable
-                ? () => showRelationship(relationship._id)
-                : null}
-            >
-              {readable(
-                relationship[primaryDisplay] || relationship.primaryDisplay
-              )}
-            </span>
-            {#if editable}
-              <Icon
-                name="Close"
-                size="XS"
-                hoverable
-                on:click={() => toggleRow(relationship)}
-              />
-            {/if}
-          </div>
-        {/if}
-      {/each}
+      {#if Array.isArray(value) && value.length}
+        {#each value as relationship}
+          {#if relationship[primaryDisplay] || relationship.primaryDisplay}
+            <div class="badge">
+              <span
+                on:click={editable
+                  ? () => showRelationship(relationship._id)
+                  : null}
+              >
+                {readable(
+                  relationship[primaryDisplay] || relationship.primaryDisplay
+                )}
+              </span>
+              {#if editable}
+                <Icon
+                  name="Close"
+                  size="XS"
+                  hoverable
+                  on:click={() => toggleRow(relationship)}
+                />
+              {/if}
+            </div>
+          {/if}
+        {/each}
+      {/if}
       {#if editable}
         <div class="add" on:click={open}>
           <Icon name="Add" size="S" />
@@ -318,7 +320,7 @@
         <div class="searching">
           <ProgressCircle size="S" />
         </div>
-      {:else if searchResults?.length}
+      {:else if Array.isArray(searchResults) && searchResults.length}
         <div class="results">
           {#each searchResults as row, idx}
             <div

packages/frontend-core/src/components/grid/layout/GridBody.svelte

@@ -35,7 +35,7 @@
 </script>
 
 <div bind:this={body} class="grid-body">
-  <GridScrollWrapper scrollHorizontally scrollVertically wheelInteractive>
+  <GridScrollWrapper scrollHorizontally scrollVertically attachHandlers>
     {#each $renderedRows as row, idx}
       <GridRow
         {row}

packages/frontend-core/src/components/grid/layout/GridRow.svelte

@@ -17,6 +17,7 @@
     columnHorizontalInversionIndex,
     contentLines,
     isDragging,
+    dispatch,
   } = getContext("grid")
 
   $: rowSelected = !!$selectedRows[row._id]
@@ -30,6 +31,7 @@
   on:focus
   on:mouseenter={$isDragging ? null : () => ($hoveredRowId = row._id)}
   on:mouseleave={$isDragging ? null : () => ($hoveredRowId = null)}
+  on:click={() => dispatch("rowclick", row)}
 >
   {#each $renderedColumns as column, columnIdx (column.name)}
     {@const cellId = `${row._id}-${column.name}`}

packages/frontend-core/src/components/grid/layout/GridScrollWrapper.svelte

@@ -17,7 +17,11 @@
 
   export let scrollVertically = false
   export let scrollHorizontally = false
-  export let wheelInteractive = false
+  export let attachHandlers = false
+
+  // Used for tracking touch events
+  let initialTouchX
+  let initialTouchY
 
   $: style = generateStyle($scroll, $rowHeight, $hiddenColumnsWidth)
 
@@ -27,17 +31,47 @@
     return `transform: translate3d(${offsetX}px, ${offsetY}px, 0);`
   }
 
-  // Handles a wheel even and updates the scroll offsets
+  // Handles a mouse wheel event and updates scroll state
   const handleWheel = e => {
     e.preventDefault()
-    debouncedHandleWheel(e.deltaX, e.deltaY, e.clientY)
+    updateScroll(e.deltaX, e.deltaY, e.clientY)
 
     // If a context menu was visible, hide it
     if ($menu.visible) {
       menu.actions.close()
     }
   }
-  const debouncedHandleWheel = domDebounce((deltaX, deltaY, clientY) => {
+
+  // Handles touch start events
+  const handleTouchStart = e => {
+    if (!e.touches?.[0]) return
+    initialTouchX = e.touches[0].clientX
+    initialTouchY = e.touches[0].clientY
+  }
+
+  // Handles touch move events and updates scroll state
+  const handleTouchMove = e => {
+    if (!e.touches?.[0]) return
+    e.preventDefault()
+
+    // Compute delta from previous event, and update scroll
+    const deltaX = initialTouchX - e.touches[0].clientX
+    const deltaY = initialTouchY - e.touches[0].clientY
+    updateScroll(deltaX, deltaY)
+
+    // Store position to reference in next event
+    initialTouchX = e.touches[0].clientX
+    initialTouchY = e.touches[0].clientY
+
+    // If a context menu was visible, hide it
+    if ($menu.visible) {
+      menu.actions.close()
+    }
+  }
+
+  // Updates the scroll offset by a certain delta, and ensure scrolling
+  // stays within sensible bounds. Debounced for performance.
+  const updateScroll = domDebounce((deltaX, deltaY, clientY) => {
     const { top, left } = $scroll
 
     // Calculate new scroll top
@@ -55,15 +89,19 @@
     })
 
     // Hover row under cursor
-    const y = clientY - $bounds.top + (newScrollTop % $rowHeight)
-    const hoveredRow = $renderedRows[Math.floor(y / $rowHeight)]
-    hoveredRowId.set(hoveredRow?._id)
+    if (clientY != null) {
+      const y = clientY - $bounds.top + (newScrollTop % $rowHeight)
+      const hoveredRow = $renderedRows[Math.floor(y / $rowHeight)]
+      hoveredRowId.set(hoveredRow?._id)
+    }
   })
 </script>
 
 <div
   class="outer"
-  on:wheel={wheelInteractive ? handleWheel : null}
+  on:wheel={attachHandlers ? handleWheel : null}
+  on:touchstart={attachHandlers ? handleTouchStart : null}
+  on:touchmove={attachHandlers ? handleTouchMove : null}
   on:click|self={() => ($focusedCellId = null)}
 >
   <div {style} class="inner">

packages/frontend-core/src/components/grid/layout/NewRow.svelte

@@ -205,7 +205,7 @@
       {/if}
     </div>
     <div class="normal-columns" transition:fade|local={{ duration: 130 }}>
-      <GridScrollWrapper scrollHorizontally wheelInteractive>
+      <GridScrollWrapper scrollHorizontally attachHandlers>
         <div class="row">
           {#each $renderedColumns as column, columnIdx}
             {@const cellId = `new-${column.name}`}

packages/frontend-core/src/components/grid/layout/StickyColumn.svelte

@@ -64,7 +64,7 @@
   </div>
 
   <div class="content" on:mouseleave={() => ($hoveredRowId = null)}>
-    <GridScrollWrapper scrollVertically wheelInteractive>
+    <GridScrollWrapper scrollVertically attachHandlers>
       {#each $renderedRows as row, idx}
         {@const rowSelected = !!$selectedRows[row._id]}
         {@const rowHovered = $hoveredRowId === row._id}
@@ -74,6 +74,7 @@
           class="row"
           on:mouseenter={$isDragging ? null : () => ($hoveredRowId = row._id)}
           on:mouseleave={$isDragging ? null : () => ($hoveredRowId = null)}
+          on:click={() => dispatch("rowclick", row)}
         >
           <GutterCell {row} {rowFocused} {rowHovered} {rowSelected} />
           {#if $stickyColumn}

packages/frontend-core/src/components/grid/overlays/ScrollOverlay.svelte

@@ -53,18 +53,27 @@
     }
   }
 
+  const getLocation = e => {
+    return {
+      y: e.touches?.[0]?.clientY ?? e.clientY,
+      x: e.touches?.[0]?.clientX ?? e.clientX,
+    }
+  }
+
   // V scrollbar drag handlers
   const startVDragging = e => {
     e.preventDefault()
-    initialMouse = e.clientY
+    initialMouse = getLocation(e).y
     initialScroll = $scrollTop
     document.addEventListener("mousemove", moveVDragging)
+    document.addEventListener("touchmove", moveVDragging)
     document.addEventListener("mouseup", stopVDragging)
+    document.addEventListener("touchend", stopVDragging)
     isDraggingV = true
     closeMenu()
   }
   const moveVDragging = domDebounce(e => {
-    const delta = e.clientY - initialMouse
+    const delta = getLocation(e).y - initialMouse
     const weight = delta / availHeight
     const newScrollTop = initialScroll + weight * $maxScrollTop
     scroll.update(state => ({
@@ -74,22 +83,26 @@
   })
   const stopVDragging = () => {
     document.removeEventListener("mousemove", moveVDragging)
+    document.removeEventListener("touchmove", moveVDragging)
     document.removeEventListener("mouseup", stopVDragging)
+    document.removeEventListener("touchend", stopVDragging)
     isDraggingV = false
   }
 
   // H scrollbar drag handlers
   const startHDragging = e => {
     e.preventDefault()
-    initialMouse = e.clientX
+    initialMouse = getLocation(e).x
     initialScroll = $scrollLeft
     document.addEventListener("mousemove", moveHDragging)
+    document.addEventListener("touchmove", moveHDragging)
     document.addEventListener("mouseup", stopHDragging)
+    document.addEventListener("touchend", stopHDragging)
     isDraggingH = true
     closeMenu()
   }
   const moveHDragging = domDebounce(e => {
-    const delta = e.clientX - initialMouse
+    const delta = getLocation(e).x - initialMouse
     const weight = delta / availWidth
     const newScrollLeft = initialScroll + weight * $maxScrollLeft
     scroll.update(state => ({
@@ -99,7 +112,9 @@
   })
   const stopHDragging = () => {
     document.removeEventListener("mousemove", moveHDragging)
+    document.removeEventListener("touchmove", moveHDragging)
     document.removeEventListener("mouseup", stopHDragging)
+    document.removeEventListener("touchend", stopHDragging)
     isDraggingH = false
   }
 </script>
@@ -109,6 +124,7 @@
     class="v-scrollbar"
     style="--size:{ScrollBarSize}px; top:{barTop}px; height:{barHeight}px;"
     on:mousedown={startVDragging}
+    on:touchstart={startVDragging}
     class:dragging={isDraggingV}
   />
 {/if}
@@ -117,6 +133,7 @@
     class="h-scrollbar"
     style="--size:{ScrollBarSize}px; left:{barLeft}px; width:{barWidth}px;"
     on:mousedown={startHDragging}
+    on:touchstart={startHDragging}
     class:dragging={isDraggingH}
   />
 {/if}

packages/frontend-core/src/components/grid/stores/clipboard.js

@@ -1,4 +1,5 @@
 import { writable, get } from "svelte/store"
+import { Helpers } from "@budibase/bbui"
 
 export const createStores = () => {
   const copiedCell = writable(null)
@@ -12,7 +13,16 @@ export const createActions = context => {
   const { copiedCell, focusedCellAPI } = context
 
   const copy = () => {
-    copiedCell.set(get(focusedCellAPI)?.getValue())
+    const value = get(focusedCellAPI)?.getValue()
+    copiedCell.set(value)
+
+    // Also copy a stringified version to the clipboard
+    let stringified = ""
+    if (value != null && value !== "") {
+      // Only conditionally stringify to avoid redundant quotes around text
+      stringified = typeof value === "object" ? JSON.stringify(value) : value
+    }
+    Helpers.copyToClipboard(stringified)
   }
 
   const paste = () => {

packages/pro

@@ -1 +1 @@
-Subproject commit 044bec6447066b215932d6726c437e7ec5a9e42e
+Subproject commit 570d14aa44aa88f4d053856322210f0008ba5c76

packages/server/package.json

@@ -11,15 +11,14 @@
   "scripts": {
     "prebuild": "rimraf dist/",
     "build": "node ./scripts/build.js",
+    "postbuild": "copyfiles -f ../client/dist/budibase-client.js ../client/manifest.json client && copyfiles -f ../../yarn.lock ./dist/",
     "check:types": "tsc -p tsconfig.json --noEmit --paths null",
-    "postbuild": "copyfiles -f ../client/dist/budibase-client.js ../client/manifest.json client",
     "build:dev": "yarn prebuild && tsc --build --watch --preserveWatchOutput",
     "debug": "yarn build && node --expose-gc --inspect=9222 dist/index.js",
     "test": "bash scripts/test.sh",
     "test:memory": "jest --maxWorkers=2 --logHeapUsage --forceExit",
     "test:watch": "jest --watch",
-    "predocker": "copyfiles -f ../client/dist/budibase-client.js ../client/manifest.json client && yarn build && cp ../../yarn.lock ./dist/",
-    "build:docker": "yarn predocker && docker build . -t app-service --label version=$BUDIBASE_RELEASE_VERSION",
+    "build:docker": "yarn build && docker build . -t app-service --label version=$BUDIBASE_RELEASE_VERSION",
     "run:docker": "node dist/index.js",
     "run:docker:cluster": "pm2-runtime start pm2.config.js",
     "dev:stack:up": "node scripts/dev/manage.js up",
@@ -54,9 +53,8 @@
     "@bull-board/api": "3.7.0",
     "@bull-board/koa": "3.9.4",
     "@elastic/elasticsearch": "7.10.0",
-    "@google-cloud/firestore": "5.0.2",
+    "@google-cloud/firestore": "6.8.0",
     "@koa/router": "8.0.8",
-    "@sentry/node": "6.17.7",
     "@socket.io/redis-adapter": "^8.2.1",
     "airtable": "0.10.1",
     "arangojs": "7.2.0",
@@ -70,7 +68,6 @@
     "curlconverter": "3.21.0",
     "dd-trace": "3.13.2",
     "dotenv": "8.2.0",
-    "fix-path": "3.0.0",
     "form-data": "4.0.0",
     "global-agent": "3.0.0",
     "google-auth-library": "7.12.0",
@@ -96,12 +93,11 @@
     "object-sizeof": "2.6.1",
     "open": "8.4.0",
     "openai": "^3.2.1",
+    "openapi-types": "9.3.1",
     "pg": "8.10.0",
-    "posthog-node": "1.3.0",
     "pouchdb": "7.3.0",
-    "pouchdb-all-dbs": "1.0.2",
+    "pouchdb-all-dbs": "1.1.1",
     "pouchdb-find": "7.2.2",
-    "pouchdb-replication-stream": "1.2.9",
     "redis": "4",
     "server-destroy": "1.0.1",
     "snowflake-promise": "^4.5.0",
@@ -113,8 +109,7 @@
     "validate.js": "0.13.1",
     "vm2": "^3.9.19",
     "worker-farm": "1.7.0",
-    "xml2js": "0.5.0",
-    "yargs": "13.2.4"
+    "xml2js": "0.5.0"
   },
   "devDependencies": {
     "@babel/core": "7.17.4",
@@ -144,7 +139,6 @@
     "jest-runner": "29.6.2",
     "jest-serial-runner": "1.2.1",
     "nodemon": "2.0.15",
-    "openapi-types": "9.3.1",
     "openapi-typescript": "5.2.0",
     "path-to-regexp": "6.2.0",
     "rimraf": "3.0.2",
@@ -154,7 +148,8 @@
     "ts-node": "10.8.1",
     "tsconfig-paths": "4.0.0",
     "typescript": "5.2.2",
-    "update-dotenv": "1.1.1"
+    "update-dotenv": "1.1.1",
+    "yargs": "13.2.4"
   },
   "optionalDependencies": {
     "oracledb": "5.3.0"
@@ -171,6 +166,22 @@
             "target": "build"
           }
         ]
+      },
+      "build": {
+        "outputs": [
+          "{projectRoot}/builder",
+          "{projectRoot}/client",
+          "{projectRoot}/dist"
+        ],
+        "dependsOn": [
+          {
+            "projects": [
+              "@budibase/client",
+              "@budibase/builder"
+            ],
+            "target": "build"
+          }
+        ]
       }
     }
   }

packages/server/src/api/controllers/datasource.ts

@@ -5,7 +5,6 @@ import {
   getTableParams,
 } from "../../db/utils"
 import { destroy as tableDestroy } from "./table/internal"
-import { BuildSchemaErrors, InvalidColumns } from "../../constants"
 import { getIntegration } from "../../integrations"
 import { invalidateDynamicVariables } from "../../threads/utils"
 import { context, db as dbCore, events } from "@budibase/backend-core"
@@ -14,10 +13,13 @@ import {
   CreateDatasourceResponse,
   Datasource,
   DatasourcePlus,
+  ExternalTable,
   FetchDatasourceInfoRequest,
   FetchDatasourceInfoResponse,
   IntegrationBase,
+  Schema,
   SourceName,
+  Table,
   UpdateDatasourceResponse,
   UserCtx,
   VerifyDatasourceRequest,
@@ -27,23 +29,6 @@ import sdk from "../../sdk"
 import { builderSocket } from "../../websockets"
 import { setupCreationAuth as googleSetupCreationAuth } from "../../integrations/googlesheets"
 
-function getErrorTables(errors: any, errorType: string) {
-  return Object.entries(errors)
-    .filter(entry => entry[1] === errorType)
-    .map(([name]) => name)
-}
-
-function updateError(error: any, newError: any, tables: string[]) {
-  if (!error) {
-    error = ""
-  }
-  if (error.length > 0) {
-    error += "\n"
-  }
-  error += `${newError} ${tables.join(", ")}`
-  return error
-}
-
 async function getConnector(
   datasource: Datasource
 ): Promise<IntegrationBase | DatasourcePlus> {
@@ -71,48 +56,36 @@ async function getAndMergeDatasource(datasource: Datasource) {
   return await sdk.datasources.enrich(enrichedDatasource)
 }
 
-async function buildSchemaHelper(datasource: Datasource) {
+async function buildSchemaHelper(datasource: Datasource): Promise<Schema> {
   const connector = (await getConnector(datasource)) as DatasourcePlus
-  await connector.buildSchema(datasource._id!, datasource.entities!)
-
-  const errors = connector.schemaErrors
-  let error = null
-  if (errors && Object.keys(errors).length > 0) {
-    const noKey = getErrorTables(errors, BuildSchemaErrors.NO_KEY)
-    const invalidCol = getErrorTables(errors, BuildSchemaErrors.INVALID_COLUMN)
-    if (noKey.length) {
-      error = updateError(
-        error,
-        "No primary key constraint found for the following:",
-        noKey
-      )
-    }
-    if (invalidCol.length) {
-      const invalidCols = Object.values(InvalidColumns).join(", ")
-      error = updateError(
-        error,
-        `Cannot use columns ${invalidCols} found in following:`,
-        invalidCol
-      )
-    }
-  }
-  return { tables: connector.tables, error }
+  return await connector.buildSchema(
+    datasource._id!,
+    datasource.entities! as Record<string, ExternalTable>
+  )
 }
 
-async function buildFilteredSchema(datasource: Datasource, filter?: string[]) {
-  let { tables, error } = await buildSchemaHelper(datasource)
-  let finalTables = tables
-  if (filter) {
-    finalTables = {}
-    for (let key in tables) {
-      if (
-        filter.some((filter: any) => filter.toLowerCase() === key.toLowerCase())
-      ) {
-        finalTables[key] = tables[key]
-      }
+async function buildFilteredSchema(
+  datasource: Datasource,
+  filter?: string[]
+): Promise<Schema> {
+  let schema = await buildSchemaHelper(datasource)
+  if (!filter) {
+    return schema
+  }
+
+  let filteredSchema: Schema = { tables: {}, errors: {} }
+  for (let key in schema.tables) {
+    if (filter.some(filter => filter.toLowerCase() === key.toLowerCase())) {
+      filteredSchema.tables[key] = schema.tables[key]
     }
   }
-  return { tables: finalTables, error }
+
+  for (let key in schema.errors) {
+    if (filter.some(filter => filter.toLowerCase() === key.toLowerCase())) {
+      filteredSchema.errors[key] = schema.errors[key]
+    }
+  }
+  return filteredSchema
 }
 
 export async function fetch(ctx: UserCtx) {
@@ -156,7 +129,7 @@ export async function buildSchemaFromDb(ctx: UserCtx) {
   const tablesFilter = ctx.request.body.tablesFilter
   const datasource = await sdk.datasources.get(ctx.params.datasourceId)
 
-  const { tables, error } = await buildFilteredSchema(datasource, tablesFilter)
+  const { tables, errors } = await buildFilteredSchema(datasource, tablesFilter)
   datasource.entities = tables
 
   setDefaultDisplayColumns(datasource)
@@ -164,13 +137,11 @@ export async function buildSchemaFromDb(ctx: UserCtx) {
     sdk.tables.populateExternalTableSchemas(datasource)
   )
   datasource._rev = dbResp.rev
-  const cleanedDatasource = await sdk.datasources.removeSecretSingle(datasource)
 
-  const res: any = { datasource: cleanedDatasource }
-  if (error) {
-    res.error = error
+  ctx.body = {
+    datasource: await sdk.datasources.removeSecretSingle(datasource),
+    errors,
   }
-  ctx.body = res
 }
 
 /**
@@ -298,15 +269,12 @@ export async function save(
     type: plus ? DocumentType.DATASOURCE_PLUS : DocumentType.DATASOURCE,
   }
 
-  let schemaError = null
+  let errors: Record<string, string> = {}
   if (fetchSchema) {
-    const { tables, error } = await buildFilteredSchema(
-      datasource,
-      tablesFilter
-    )
-    schemaError = error
-    datasource.entities = tables
+    const schema = await buildFilteredSchema(datasource, tablesFilter)
+    datasource.entities = schema.tables
     setDefaultDisplayColumns(datasource)
+    errors = schema.errors
   }
 
   if (preSaveAction[datasource.source]) {
@@ -327,13 +295,10 @@ export async function save(
     }
   }
 
-  const response: CreateDatasourceResponse = {
+  ctx.body = {
     datasource: await sdk.datasources.removeSecretSingle(datasource),
+    errors,
   }
-  if (schemaError) {
-    response.error = schemaError
-  }
-  ctx.body = response
   builderSocket?.emitDatasourceUpdate(ctx, datasource)
 }
 

packages/server/src/api/controllers/routing.ts

@@ -40,7 +40,7 @@ class Routing {
 
 /**
  * Gets the full routing structure by querying the routing view and processing the result into the tree.
- * @returns {Promise<object>} The routing structure, this is the full structure designed for use in the builder,
+ * @returns The routing structure, this is the full structure designed for use in the builder,
  * if the client routing is required then the updateRoutingStructureForUserRole should be used.
  */
 async function getRoutingStructure() {

packages/server/src/api/controllers/row/ExternalRequest.ts

@@ -280,17 +280,8 @@ function isEditableColumn(column: FieldSchema) {
   return !(isExternalAutoColumn || isFormula)
 }
 
-export type ExternalRequestReturnType<T> = T extends Operation.READ
-  ?
-      | Row[]
-      | {
-          row: Row
-          table: Table
-        }
-  : {
-      row: Row
-      table: Table
-    }
+export type ExternalRequestReturnType<T extends Operation> =
+  T extends Operation.READ ? Row[] : { row: Row; table: Table }
 
 export class ExternalRequest<T extends Operation> {
   private readonly operation: T
@@ -857,11 +848,12 @@ export class ExternalRequest<T extends Operation> {
     }
     const output = this.outputProcessing(response, table, relationships)
     // if reading it'll just be an array of rows, return whole thing
-    const result = (
-      operation === Operation.READ && Array.isArray(response)
-        ? output
-        : { row: output[0], table }
-    ) as ExternalRequestReturnType<T>
-    return result
+    if (operation === Operation.READ) {
+      return (
+        Array.isArray(output) ? output : [output]
+      ) as ExternalRequestReturnType<T>
+    } else {
+      return { row: output[0], table } as ExternalRequestReturnType<T>
+    }
   }
 }

packages/server/src/api/controllers/row/external.ts

@@ -44,7 +44,7 @@ export async function handleRequest<T extends Operation>(
     return [] as any
   }
 
-  return new ExternalRequest(operation, tableId, opts?.datasource).run(
+  return new ExternalRequest<T>(operation, tableId, opts?.datasource).run(
     opts || {}
   )
 }
@@ -148,17 +148,17 @@ export async function find(ctx: UserCtx): Promise<Row> {
 export async function destroy(ctx: UserCtx) {
   const tableId = utils.getTableId(ctx)
   const _id = ctx.request.body._id
-  const { row } = (await handleRequest(Operation.DELETE, tableId, {
+  const { row } = await handleRequest(Operation.DELETE, tableId, {
     id: breakRowIdField(_id),
     includeSqlRelationships: IncludeRelationship.EXCLUDE,
-  })) as { row: Row }
+  })
   return { response: { ok: true, id: _id }, row }
 }
 
 export async function bulkDestroy(ctx: UserCtx) {
   const { rows } = ctx.request.body
   const tableId = utils.getTableId(ctx)
-  let promises: Promise<Row[] | { row: Row; table: Table }>[] = []
+  let promises: Promise<{ row: Row; table: Table }>[] = []
   for (let row of rows) {
     promises.push(
       handleRequest(Operation.DELETE, tableId, {
@@ -167,7 +167,7 @@ export async function bulkDestroy(ctx: UserCtx) {
       })
     )
   }
-  const responses = (await Promise.all(promises)) as { row: Row }[]
+  const responses = await Promise.all(promises)
   return { response: { ok: true }, rows: responses.map(resp => resp.row) }
 }
 
@@ -183,11 +183,11 @@ export async function fetchEnrichedRow(ctx: UserCtx) {
     ctx.throw(400, "Datasource has not been configured for plus API.")
   }
   const tables = datasource.entities
-  const response = (await handleRequest(Operation.READ, tableId, {
+  const response = await handleRequest(Operation.READ, tableId, {
     id,
     datasource,
     includeSqlRelationships: IncludeRelationship.INCLUDE,
-  })) as Row[]
+  })
   const table: Table = tables[tableName]
   const row = response[0]
   // this seems like a lot of work, but basically we need to dig deeper for the enrich

packages/server/src/api/controllers/view/viewBuilder.ts

@@ -88,8 +88,8 @@ const SCHEMA_MAP: Record<string, any> = {
 /**
  * Iterates through the array of filters to create a JS
  * expression that gets used in a CouchDB view.
- * @param {Array} filters - an array of filter objects
- * @returns {String} JS Expression
+ * @param filters - an array of filter objects
+ * @returns JS Expression
  */
 function parseFilterExpression(filters: ViewFilter[]) {
   const expression = []
@@ -125,8 +125,8 @@ function parseFilterExpression(filters: ViewFilter[]) {
 /**
  * Returns a CouchDB compliant emit() expression that is used to emit the
  * correct key/value pairs for custom views.
- * @param {String?} field - field to use for calculations, if any
- * @param {String?} groupBy - field to group calculation results on, if any
+ * @param field - field to use for calculations, if any
+ * @param groupBy - field to group calculation results on, if any
  */
 function parseEmitExpression(field: string, groupBy: string) {
   return `emit(doc["${groupBy || "_id"}"], doc["${field}"]);`
@@ -136,7 +136,7 @@ function parseEmitExpression(field: string, groupBy: string) {
  * Return a fully parsed CouchDB compliant view definition
  * that will be stored in the design document in the database.
  *
- * @param {Object} viewDefinition - the JSON definition for a custom view.
+ * @param viewDefinition - the JSON definition for a custom view.
  * field: field that calculations will be performed on
  * tableId: tableId of the table this view was created from
  * groupBy: field that calculations will be grouped by. Field must be present for this to be useful

packages/server/src/api/routes/tests/appImport.spec.ts

@@ -23,7 +23,10 @@ describe("/applications/:appId/import", () => {
       .set(config.defaultHeaders())
       .expect("Content-Type", /json/)
       .expect(200)
-    expect(res.body.message).toBe("app updated")
+    const appPackage = await config.api.application.get(appId!)
+    expect(appPackage.navigation?.links?.length).toBe(2)
+    expect(expect(appPackage.navigation?.links?.[0].url).toBe("/blank"))
+    expect(expect(appPackage.navigation?.links?.[1].url).toBe("/derp"))
     const screens = await config.api.screen.list()
     expect(screens.length).toBe(2)
     expect(screens[0].routing.route).toBe("/derp")

packages/server/src/api/routes/tests/datasource.spec.ts

@@ -37,7 +37,7 @@ describe("/datasources", () => {
         .expect(200)
 
       expect(res.body.datasource.name).toEqual("Test")
-      expect(res.body.errors).toBeUndefined()
+      expect(res.body.errors).toEqual({})
       expect(events.datasource.created).toBeCalledTimes(1)
     })
   })

packages/server/src/app.ts

@@ -1,11 +1,7 @@
-import Sentry from "@sentry/node"
-
 if (process.env.DD_APM_ENABLED) {
   require("./ddApm")
 }
 
-// need to load environment first
-import env from "./environment"
 import * as db from "./db"
 db.init()
 import { ServiceType } from "@budibase/types"
@@ -28,10 +24,6 @@ async function start() {
   }
   // startup includes automation runner - if enabled
   await startup(app, server)
-  if (env.isProd()) {
-    env._set("NODE_ENV", "production")
-    Sentry.init()
-  }
 }
 
 start().catch(err => {

packages/server/src/automations/automationUtils.ts

@@ -14,13 +14,13 @@ import { LoopStep, LoopStepType, LoopInput } from "../definitions/automations"
  * make sure that the post template statement can be cast into the correct type, this function does this for numbers
  * and booleans.
  *
- * @param {object} inputs An object of inputs, please note this will not recurse down into any objects within, it simply
+ * @param inputs An object of inputs, please note this will not recurse down into any objects within, it simply
  * cleanses the top level inputs, however it can be used by recursively calling it deeper into the object structures if
  * the schema is known.
- * @param {object} schema The defined schema of the inputs, in the form of JSON schema. The schema definition of an
+ * @param schema The defined schema of the inputs, in the form of JSON schema. The schema definition of an
  * automation is the likely use case of this, however validate.js syntax can be converted closely enough to use this by
  * wrapping the schema properties in a top level "properties" object.
- * @returns {object} The inputs object which has had all the various types supported by this function converted to their
+ * @returns The inputs object which has had all the various types supported by this function converted to their
  * primitive types.
  */
 export function cleanInputValues(inputs: Record<string, any>, schema?: any) {
@@ -74,9 +74,9 @@ export function cleanInputValues(inputs: Record<string, any>, schema?: any) {
  * the automation but is instead part of the Table/Table. This function will get the table schema and use it to instead
  * perform the cleanInputValues function on the input row.
  *
- * @param {string} tableId The ID of the Table/Table which the schema is to be retrieved for.
- * @param {object} row The input row structure which requires clean-up after having been through template statements.
- * @returns {Promise<Object>} The cleaned up rows object, will should now have all the required primitive types.
+ * @param tableId The ID of the Table/Table which the schema is to be retrieved for.
+ * @param row The input row structure which requires clean-up after having been through template statements.
+ * @returns The cleaned up rows object, will should now have all the required primitive types.
  */
 export async function cleanUpRow(tableId: string, row: Row) {
   let table = await sdk.tables.getTable(tableId)

packages/server/src/automations/utils.ts

@@ -148,8 +148,8 @@ export function isRebootTrigger(auto: Automation) {
 
 /**
  * This function handles checking of any cron jobs that need to be enabled/updated.
- * @param {string} appId The ID of the app in which we are checking for webhooks
- * @param {object|undefined} automation The automation object to be updated.
+ * @param appId The ID of the app in which we are checking for webhooks
+ * @param automation The automation object to be updated.
  */
 export async function enableCronTrigger(appId: any, automation: Automation) {
   const trigger = automation ? automation.definition.trigger : null
@@ -187,10 +187,10 @@ export async function enableCronTrigger(appId: any, automation: Automation) {
 
 /**
  * This function handles checking if any webhooks need to be created or deleted for automations.
- * @param {string} appId The ID of the app in which we are checking for webhooks
- * @param {object|undefined} oldAuto The old automation object if updating/deleting
- * @param {object|undefined} newAuto The new automation object if creating/updating
- * @returns {Promise<object|undefined>} After this is complete the new automation object may have been updated and should be
+ * @param appId The ID of the app in which we are checking for webhooks
+ * @param oldAuto The old automation object if updating/deleting
+ * @param newAuto The new automation object if creating/updating
+ * @returns After this is complete the new automation object may have been updated and should be
  * written to DB (this does not write to DB as it would be wasteful to repeat).
  */
 export async function checkForWebhooks({ oldAuto, newAuto }: any) {
@@ -257,8 +257,8 @@ export async function checkForWebhooks({ oldAuto, newAuto }: any) {
 
 /**
  * When removing an app/unpublishing it need to make sure automations are cleaned up (cron).
- * @param appId {string} the app that is being removed.
- * @return {Promise<void>} clean is complete if this succeeds.
+ * @param appId the app that is being removed.
+ * @return clean is complete if this succeeds.
  */
 export async function cleanupAutomations(appId: any) {
   await disableAllCrons(appId)
@@ -267,7 +267,7 @@ export async function cleanupAutomations(appId: any) {
 /**
  * Checks if the supplied automation is of a recurring type.
  * @param automation The automation to check.
- * @return {boolean} if it is recurring (cron).
+ * @return if it is recurring (cron).
  */
 export function isRecurring(automation: Automation) {
   return automation.definition.trigger.stepId === definitions.CRON.stepId

packages/server/src/constants/index.ts

@@ -159,11 +159,6 @@ export enum InvalidColumns {
   TABLE_ID = "tableId",
 }
 
-export enum BuildSchemaErrors {
-  NO_KEY = "no_key",
-  INVALID_COLUMN = "invalid_column",
-}
-
 export enum AutomationErrors {
   INCORRECT_TYPE = "INCORRECT_TYPE",
   MAX_ITERATIONS = "MAX_ITERATIONS_REACHED",

packages/server/src/db/linkedRows/LinkController.ts

@@ -1,8 +1,7 @@
 import { IncludeDocs, getLinkDocuments } from "./linkUtils"
 import { InternalTables, getUserMetadataParams } from "../utils"
-import Sentry from "@sentry/node"
 import { FieldTypes } from "../../constants"
-import { context } from "@budibase/backend-core"
+import { context, logging } from "@budibase/backend-core"
 import LinkDocument from "./LinkDocument"
 import {
   Database,
@@ -39,7 +38,7 @@ class LinkController {
 
   /**
    * Retrieves the table, if it was not already found in the eventData.
-   * @returns {Promise<object>} This will return a table based on the event data, either
+   * @returns This will return a table based on the event data, either
    * if it was in the event already, or it uses the specified tableId to get it.
    */
   async table() {
@@ -53,8 +52,8 @@ class LinkController {
   /**
    * Checks if the table this was constructed with has any linking columns currently.
    * If the table has not been retrieved this will retrieve it based on the eventData.
-   * @params {object|null} table If a table that is not known to the link controller is to be tested.
-   * @returns {Promise<boolean>} True if there are any linked fields, otherwise it will return
+   * @params table If a table that is not known to the link controller is to be tested.
+   * @returns True if there are any linked fields, otherwise it will return
    * false.
    */
   async doesTableHaveLinkedFields(table?: Table) {
@@ -160,7 +159,7 @@ class LinkController {
   /**
    * When a row is saved this will carry out the necessary operations to make sure
    * the link has been created/updated.
-   * @returns {Promise<object>} returns the row that has been cleaned and prepared to be written to the DB - links
+   * @returns returns the row that has been cleaned and prepared to be written to the DB - links
    * have also been created.
    */
   async rowSaved() {
@@ -272,7 +271,7 @@ class LinkController {
   /**
    * When a row is deleted this will carry out the necessary operations to make sure
    * any links that existed have been removed.
-   * @returns {Promise<object>} The operation has been completed and the link documents should now
+   * @returns The operation has been completed and the link documents should now
    * be accurate. This also returns the row that was deleted.
    */
   async rowDeleted() {
@@ -294,8 +293,8 @@ class LinkController {
 
   /**
    * Remove a field from a table as well as any linked rows that pertained to it.
-   * @param {string} fieldName The field to be removed from the table.
-   * @returns {Promise<void>} The table has now been updated.
+   * @param fieldName The field to be removed from the table.
+   * @returns The table has now been updated.
    */
   async removeFieldFromTable(fieldName: string) {
     let oldTable = this._oldTable
@@ -334,7 +333,7 @@ class LinkController {
   /**
    * When a table is saved this will carry out the necessary operations to make sure
    * any linked tables are notified and updated correctly.
-   * @returns {Promise<object>} The operation has been completed and the link documents should now
+   * @returns The operation has been completed and the link documents should now
    * be accurate. Also returns the table that was operated on.
    */
   async tableSaved() {
@@ -395,7 +394,7 @@ class LinkController {
   /**
    * Update a table, this means if a field is removed need to handle removing from other table and removing
    * any link docs that pertained to it.
-   * @returns {Promise<Object>} The table which has been saved, same response as with the tableSaved function.
+   * @returns The table which has been saved, same response as with the tableSaved function.
    */
   async tableUpdated() {
     const oldTable = this._oldTable
@@ -419,7 +418,7 @@ class LinkController {
    * When a table is deleted this will carry out the necessary operations to make sure
    * any linked tables have the joining column correctly removed as well as removing any
    * now stale linking documents.
-   * @returns {Promise<object>} The operation has been completed and the link documents should now
+   * @returns The operation has been completed and the link documents should now
    * be accurate. Also returns the table that was operated on.
    */
   async tableDeleted() {
@@ -433,9 +432,8 @@ class LinkController {
           delete linkedTable.schema[field.fieldName]
           await this._db.put(linkedTable)
         }
-      } catch (err) {
-        /* istanbul ignore next */
-        Sentry.captureException(err)
+      } catch (err: any) {
+        logging.logWarn(err?.message, err)
       }
     }
     // need to get the full link docs to delete them

packages/server/src/db/linkedRows/LinkDocument.ts

@@ -6,12 +6,12 @@ import { LinkDocument } from "@budibase/types"
  * Creates a new link document structure which can be put to the database. It is important to
  * note that while this talks about linker/linked the link is bi-directional and for all intent
  * and purposes it does not matter from which direction the link was initiated.
- * @param {string} tableId1 The ID of the first table (the linker).
- * @param {string} tableId2 The ID of the second table (the linked).
- * @param {string} fieldName1 The name of the field in the linker table.
- * @param {string} fieldName2 The name of the field in the linked table.
- * @param {string} rowId1 The ID of the row which is acting as the linker.
- * @param {string} rowId2 The ID of the row which is acting as the linked.
+ * @param tableId1 The ID of the first table (the linker).
+ * @param tableId2 The ID of the second table (the linked).
+ * @param fieldName1 The name of the field in the linker table.
+ * @param fieldName2 The name of the field in the linked table.
+ * @param rowId1 The ID of the row which is acting as the linker.
+ * @param rowId2 The ID of the row which is acting as the linked.
  * @constructor
  */
 class LinkDocumentImpl implements LinkDocument {

packages/server/src/db/linkedRows/index.ts

@@ -90,13 +90,13 @@ async function getFullLinkedDocs(links: LinkDocumentValue[]) {
 
 /**
  * Update link documents for a row or table - this is to be called by the API controller when a change is occurring.
- * @param {string} args.eventType states what type of change which is occurring, means this can be expanded upon in the
+ * @param args.eventType states what type of change which is occurring, means this can be expanded upon in the
  * future quite easily (all updates go through one function).
- * @param {string} args.tableId The ID of the of the table which is being changed.
- * @param {object|undefined} args.row The row which is changing, e.g. created, updated or deleted.
- * @param {object|undefined} args.table If the table has already been retrieved this can be used to reduce database gets.
- * @param {object|undefined} args.oldTable If the table is being updated then the old table can be provided for differencing.
- * @returns {Promise<object>} When the update is complete this will respond successfully. Returns the row for
+ * @param args.tableId The ID of the of the table which is being changed.
+ * @param args.row The row which is changing, e.g. created, updated or deleted.
+ * @param args.table If the table has already been retrieved this can be used to reduce database gets.
+ * @param args.oldTable If the table is being updated then the old table can be provided for differencing.
+ * @returns When the update is complete this will respond successfully. Returns the row for
  * row operations and the table for table operations.
  */
 export async function updateLinks(args: {
@@ -144,10 +144,10 @@ export async function updateLinks(args: {
 /**
  * Given a table and a list of rows this will retrieve all of the attached docs and enrich them into the row.
  * This is required for formula fields, this may only be utilised internally (for now).
- * @param {object} table The table from which the rows originated.
- * @param {array<object>} rows The rows which are to be enriched.
- * @param {object} opts optional - options like passing in a base row to use for enrichment.
- * @return {Promise<*>} returns the rows with all of the enriched relationships on it.
+ * @param table The table from which the rows originated.
+ * @param rows The rows which are to be enriched.
+ * @param opts optional - options like passing in a base row to use for enrichment.
+ * @return returns the rows with all of the enriched relationships on it.
  */
 export async function attachFullLinkedDocs(
   table: Table,
@@ -208,9 +208,9 @@ export async function attachFullLinkedDocs(
 
 /**
  * This function will take the given enriched rows and squash the links to only contain the primary display field.
- * @param {object} table The table from which the rows originated.
- * @param {array<object>} enriched The pre-enriched rows (full docs) which are to be squashed.
- * @returns {Promise<Array>} The rows after having their links squashed to only contain the ID and primary display.
+ * @param table The table from which the rows originated.
+ * @param enriched The pre-enriched rows (full docs) which are to be squashed.
+ * @returns The rows after having their links squashed to only contain the ID and primary display.
  */
 export async function squashLinksToPrimaryDisplay(
   table: Table,