Merge branch 'master' into BUDI-8312

.eslintrc.json

@@ -92,7 +92,8 @@
         // differs to external, but the API is broadly the same
         "jest/no-conditional-expect": "off",
         // have to turn this off to allow function overloading in typescript
-        "no-dupe-class-members": "off"
+        "no-dupe-class-members": "off",
+        "no-redeclare": "off"
       }
     },
     {

.github/workflows/budibase_ci.yml

@@ -73,9 +73,9 @@ jobs:
       - name: Check types
         run: |
           if ${{ env.USE_NX_AFFECTED }}; then
-            yarn check:types --since=${{ env.NX_BASE_BRANCH }}
+            yarn check:types --since=${{ env.NX_BASE_BRANCH }} --ignore @budibase/account-portal-server
           else
-            yarn check:types
+            yarn check:types --ignore @budibase/account-portal-server
           fi
 
   helm-lint:
@@ -226,10 +226,11 @@ jobs:
         if: ${{ steps.get_pro_commits.outputs.base_commit_excluding_merges != '' }}
         run: |
           cd packages/pro
+          base_commit='${{ steps.get_pro_commits.outputs.base_commit }}'
           base_commit_excluding_merges='${{ steps.get_pro_commits.outputs.base_commit_excluding_merges }}'
           pro_commit='${{ steps.get_pro_commits.outputs.pro_commit }}'
 
-          any_commit=$(git log --no-merges $base_commit_excluding_merges...$pro_commit)
+          any_commit=$(git log --no-merges $base_commit...$pro_commit)
 
           if [ -n "$any_commit" ]; then
             echo $any_commit

.github/workflows/force-release.yml

@@ -9,7 +9,7 @@ on:
 jobs:
   ensure-is-master-tag:
     name: Ensure is a master tag
-    runs-on: qa-arc-runner-set
+    runs-on: ubuntu-latest
     steps:
       - name: Checkout monorepo
         uses: actions/checkout@v4

.github/workflows/pr-labeler.yml

@@ -0,0 +1,38 @@
+name: PR labeler
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
+  cancel-in-progress: true
+
+on:
+  pull_request:
+    types: [opened, synchronize]
+
+jobs:
+  size-labeler:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: codelytv/pr-size-labeler@v1
+        with:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          xs_max_size: "10"
+          s_max_size: "100"
+          m_max_size: "500"
+          l_max_size: "1000"
+          fail_if_xl: "false"
+          files_to_ignore: "yarn.lock"
+
+  team-labeler:
+    runs-on: ubuntu-latest
+    if: ${{ github.event.action == 'opened' }}
+    steps:
+      - uses: rodrigoarias/auto-label-per-user@v1.0.0
+        with:
+          git-token: ${{ secrets.GITHUB_TOKEN }}
+          user-team-map: |
+            {
+                "adrinr": "firestorm",
+                "samwho": "firestorm",
+                "PClmnt": "firestorm",
+                "mike12345567": "firestorm"
+            }

.gitignore

@@ -8,6 +8,8 @@ bb-airgapped.tar.gz
 packages/server/build/oldClientVersions/**/*
 packages/builder/src/components/deploy/clientVersions.json
 
+packages/server/src/integrations/tests/utils/*.lock
+
 # Logs
 logs
 *.log

charts/budibase/Chart.lock

@@ -1,6 +1,6 @@
 dependencies:
 - name: couchdb
   repository: https://apache.github.io/couchdb-helm
-  version: 4.3.0
+  version: 4.5.6
-digest: sha256:94449a7f195b186f5af33ec5aa66d58b36bede240fae710f021ca87837b30606
+digest: sha256:405f098633e632d6f4e140175f156ed4f02918b0d89193f1b66c9cbea211d6c9
-generated: "2023-11-20T17:43:02.777596Z"
+generated: "2024-06-05T14:41:05.979052+01:00"

charts/budibase/Chart.yaml

@@ -17,6 +17,6 @@ version: 0.0.0
 appVersion: 0.0.0
 dependencies:
   - name: couchdb
-    version: 4.3.0
+    version: 4.5.6
     repository: https://apache.github.io/couchdb-helm
     condition: services.couchdb.enabled

charts/budibase/README.md

@@ -112,7 +112,9 @@ $ helm install --create-namespace --namespace budibase budibase . -f values.yaml
 | awsAlbIngress.enabled | bool | `false` | Whether to create an ALB Ingress resource pointing to the Budibase proxy. Requires the AWS ALB Ingress Controller. |
 | couchdb.clusterSize | int | `1` | The number of replicas to run in the CouchDB cluster. We set this to 1 by default to make things simpler, but you can set it to 3 if you need a high-availability CouchDB cluster. |
 | couchdb.couchdbConfig.couchdb.uuid | string | `"budibase-couchdb"` | Unique identifier for this CouchDB server instance. You shouldn't need to change this. |
+| couchdb.extraPorts[0] | object | `{"containerPort":4984,"name":"sqs"}` | Extra ports to expose on the CouchDB service. We expose the SQS port by default, but you can add more ports here if you need to. |
 | couchdb.image | object | `{}` | We use a custom CouchDB image for running Budibase and we don't support using any other CouchDB image. You shouldn't change this, and if you do we can't guarantee that Budibase will work. |
+| couchdb.service.extraPorts[0] | object | `{"name":"sqs","port":4984,"protocol":"TCP","targetPort":4984}` | Extra ports to expose on the CouchDB service. We expose the SQS port by default, but you can add more ports here if you need to. |
 | globals.apiEncryptionKey | string | `""` | Used for encrypting API keys and environment variables when stored in the database. You don't need to set this if `createSecrets` is true. |
 | globals.appVersion | string | `""` | The version of Budibase to deploy. Defaults to what's specified by {{ .Chart.AppVersion }}. Ends up being used as the image version tag for the apps, proxy, and worker images. |
 | globals.automationMaxIterations | string | `"200"` | The maximum number of iterations allows for an automation loop step. You can read more about looping here: <https://docs.budibase.com/docs/looping>. |
@@ -135,6 +137,8 @@ $ helm install --create-namespace --namespace budibase budibase . -f values.yaml
 | globals.smtp.password | string | `""` | The password to use when authenticating with your SMTP server. |
 | globals.smtp.port | string | `"587"` | The port of your SMTP server. |
 | globals.smtp.user | string | `""` | The username to use when authenticating with your SMTP server. |
+| globals.sqs.enabled | bool | `false` | Whether to use the CouchDB "structured query service" or not. This is disabled by default for now, but will become the default in a future release. |
+| globals.tempBucketName | string | `""` |  |
 | globals.tenantFeatureFlags | string | `"*:LICENSING,*:USER_GROUPS,*:ONBOARDING_TOUR"` | Sets what feature flags are enabled and for which tenants. Should not ordinarily need to be changed. |
 | imagePullSecrets | list | `[]` | Passed to all pods created by this chart. Should not ordinarily need to be changed. |
 | ingress.className | string | `""` | What ingress class to use. |
@@ -152,6 +156,7 @@ $ helm install --create-namespace --namespace budibase budibase . -f values.yaml
 | services.apps.autoscaling.targetCPUUtilizationPercentage | int | `80` | Target CPU utilization percentage for the apps service. Note that for autoscaling to work, you will need to have metrics-server configured, and resources set for the apps pods. |
 | services.apps.extraContainers | list | `[]` | Additional containers to be added to the apps pod. |
 | services.apps.extraEnv | list | `[]` | Extra environment variables to set for apps pods. Takes a list of name=value pairs. |
+| services.apps.extraEnvFromSecret | list | `[]` | Name of the K8s Secret in the same namespace which contains the extra environment variables. This can be used to avoid storing sensitive information in the values.yaml file. |
 | services.apps.extraVolumeMounts | list | `[]` | Additional volumeMounts to the main apps container. |
 | services.apps.extraVolumes | list | `[]` | Additional volumes to the apps pod. |
 | services.apps.httpLogging | int | `1` | Whether or not to log HTTP requests to the apps service. |
@@ -168,6 +173,7 @@ $ helm install --create-namespace --namespace budibase budibase . -f values.yaml
 | services.automationWorkers.enabled | bool | `true` | Whether or not to enable the automation worker service. If you disable this, automations will be processed by the apps service. |
 | services.automationWorkers.extraContainers | list | `[]` | Additional containers to be added to the automationWorkers pod. |
 | services.automationWorkers.extraEnv | list | `[]` | Extra environment variables to set for automation worker pods. Takes a list of name=value pairs. |
+| services.automationWorkers.extraEnvFromSecret | list | `[]` | Name of the K8s Secret in the same namespace which contains the extra environment variables. This can be used to avoid storing sensitive information in the values.yaml file. |
 | services.automationWorkers.extraVolumeMounts | list | `[]` | Additional volumeMounts to the main automationWorkers container. |
 | services.automationWorkers.extraVolumes | list | `[]` | Additional volumes to the automationWorkers pod. |
 | services.automationWorkers.livenessProbe | object | HTTP health checks. | Liveness probe configuration for automation worker pods. You shouldn't need to change this, but if you want to you can find more information here: <https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/> |
@@ -195,7 +201,7 @@ $ helm install --create-namespace --namespace budibase budibase . -f values.yaml
 | services.objectStore.region | string | `""` | AWS_REGION if using S3 |
 | services.objectStore.resources | object | `{}` | The resources to use for Minio pods. See <https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/> for more information on how to set these. |
 | services.objectStore.secretKey | string | `""` | AWS_SECRET_ACCESS_KEY if using S3 |
-| services.objectStore.storage | string | `"100Mi"` | How much storage to give Minio in its PersistentVolumeClaim. |
+| services.objectStore.storage | string | `"2Gi"` | How much storage to give Minio in its PersistentVolumeClaim. |
 | services.objectStore.storageClass | string | `""` | If defined, storageClassName: <storageClass> If set to "-", storageClassName: "", which disables dynamic provisioning If undefined (the default) or set to null, no storageClassName spec is set, choosing the default provisioner. |
 | services.objectStore.url | string | `"http://minio-service:9000"` | URL to use for object storage. Only change this if you're using an external object store, such as S3. Remember to set `minio: false` if you do this. |
 | services.proxy.autoscaling.enabled | bool | `false` | Whether to enable horizontal pod autoscaling for the proxy service. |
@@ -227,6 +233,7 @@ $ helm install --create-namespace --namespace budibase budibase . -f values.yaml
 | services.worker.autoscaling.targetCPUUtilizationPercentage | int | `80` | Target CPU utilization percentage for the worker service. Note that for autoscaling to work, you will need to have metrics-server configured, and resources set for the worker pods. |
 | services.worker.extraContainers | list | `[]` | Additional containers to be added to the worker pod. |
 | services.worker.extraEnv | list | `[]` | Extra environment variables to set for worker pods. Takes a list of name=value pairs. |
+| services.worker.extraEnvFromSecret | list | `[]` | Name of the K8s Secret in the same namespace which contains the extra environment variables. This can be used to avoid storing sensitive information in the values.yaml file. |
 | services.worker.extraVolumeMounts | list | `[]` | Additional volumeMounts to the main worker container. |
 | services.worker.extraVolumes | list | `[]` | Additional volumes to the worker pod. |
 | services.worker.httpLogging | int | `1` | Whether or not to log HTTP requests to the worker service. |

charts/budibase/templates/app-service-deployment.yaml

@@ -42,6 +42,14 @@ spec:
           {{ else }}
           value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.services.couchdb.port }}
           {{ end }}
+        {{ if .Values.globals.sqs.enabled }}
+        - name: COUCH_DB_SQL_URL
+        {{ if .Values.globals.sqs.url }}
+          value: {{ .Values.globals.sqs.url }}
+        {{ else }}
+          value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.globals.sqs.port }}
+        {{ end }}
+        {{ end }}
         {{ if .Values.services.couchdb.enabled }}
         - name: COUCH_DB_USER
           valueFrom:
@@ -198,10 +206,21 @@ spec:
         - name: APP_FEATURES
           value: "api"
         {{- end }}
+        {{- if .Values.globals.sqs.enabled }}
+        - name: SQS_SEARCH_ENABLE
+          value: "true"
+        {{- end }}
         {{- range .Values.services.apps.extraEnv }}
         - name: {{ .name }}
           value: {{ .value | quote }}
         {{- end }}
+        {{- range .Values.services.apps.extraEnvFromSecret}}
+        - name: {{ .name }}
+          valueFrom:
+            secretKeyRef:
+              name: {{ .secretName }}
+              key: {{ .secretKey | quote }}
+        {{- end}}
         image: budibase/apps:{{ .Values.globals.appVersion | default .Chart.AppVersion }}
         imagePullPolicy: Always
         {{- if .Values.services.apps.startupProbe }}

charts/budibase/templates/automation-worker-service-deployment.yaml

@@ -201,6 +201,13 @@ spec:
         - name: {{ .name }}
           value: {{ .value | quote }}
         {{- end }}
+        {{- range .Values.services.automationWorkers.extraEnvFromSecret}}
+        - name: {{ .name }}
+          valueFrom:
+            secretKeyRef:
+              name: {{ .secretName }}
+              key: {{ .secretKey | quote }}
+        {{- end}}
 
         image: budibase/apps:{{ .Values.globals.appVersion | default .Chart.AppVersion }}
         imagePullPolicy: Always
@@ -272,4 +279,4 @@ spec:
       {{- toYaml .Values.services.automationWorkers.extraVolumes | nindent 8 }}
       {{ end }}
 status: {}
-{{- end }}
+{{- end }}

charts/budibase/templates/worker-service-deployment.yaml

@@ -56,6 +56,14 @@ spec:
           {{ else }}
           value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.services.couchdb.port }}
           {{ end }}
+        {{ if .Values.globals.sqs.enabled }}
+        - name: COUCH_DB_SQL_URL
+        {{ if .Values.globals.sqs.url }}
+          value: {{ .Values.globals.sqs.url }}
+        {{ else }}
+          value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.globals.sqs.port }}
+        {{ end }}
+        {{ end }}
         - name: API_ENCRYPTION_KEY
           value: {{ .Values.globals.apiEncryptionKey | quote }}
         - name: HTTP_LOGGING
@@ -184,10 +192,21 @@ spec:
         - name: NODE_TLS_REJECT_UNAUTHORIZED
           value: {{ .Values.services.tlsRejectUnauthorized }}
         {{ end }}
+        {{- if .Values.globals.sqs.enabled }}
+        - name: SQS_SEARCH_ENABLE
+          value: "true"
+        {{- end }}
         {{- range .Values.services.worker.extraEnv }}
         - name: {{ .name }}
           value: {{ .value | quote }}
         {{- end }}
+        {{- range .Values.services.worker.extraEnvFromSecret}}
+        - name: {{ .name }}
+          valueFrom:
+            secretKeyRef:
+              name: {{ .secretName }}
+              key: {{ .secretKey | quote }}
+        {{- end}}
         image: budibase/worker:{{ .Values.globals.appVersion | default .Chart.AppVersion }}
         imagePullPolicy: Always
         {{- if .Values.services.worker.startupProbe }}

charts/budibase/values.yaml

@@ -138,6 +138,15 @@ globals:
     # -- The password to use when authenticating with your SMTP server.
     password: ""
 
+  sqs:
+    # -- Whether to use the CouchDB "structured query service" or not. This is disabled by
+    # default for now, but will become the default in a future release.
+    enabled: false
+    # @ignore
+    url: ""
+    # @ignore
+    port: "4984"
+
 services:
   # -- The DNS suffix to use for service discovery. You only need to change this
   # if you've configured your cluster to use a different DNS suffix.
@@ -240,6 +249,13 @@ services:
     # -- Extra environment variables to set for apps pods. Takes a list of
     # name=value pairs.
     extraEnv: []
+    # -- Name of the K8s Secret in the same namespace which contains the extra environment variables.
+    # This can be used to avoid storing sensitive information in the values.yaml file.
+    extraEnvFromSecret: []
+    #  - name: MY_SECRET_KEY
+    #    secretName : my-secret
+    #    secretKey: my-secret-key
+
     # -- Startup probe configuration for apps pods. You shouldn't need to
     # change this, but if you want to you can find more information here:
     # <https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/>
@@ -323,6 +339,13 @@ services:
     # -- Extra environment variables to set for automation worker pods. Takes a list of
     # name=value pairs.
     extraEnv: []
+    # -- Name of the K8s Secret in the same namespace which contains the extra environment variables.
+    # This can be used to avoid storing sensitive information in the values.yaml file.
+    extraEnvFromSecret: []
+    #  - name: MY_SECRET_KEY
+    #    secretName : my-secret
+    #    secretKey: my-secret-key
+
     # -- Startup probe configuration for automation worker pods. You shouldn't
     # need to change this, but if you want to you can find more information
     # here:
@@ -408,6 +431,13 @@ services:
     # -- Extra environment variables to set for worker pods. Takes a list of
     # name=value pairs.
     extraEnv: []
+    # -- Name of the K8s Secret in the same namespace which contains the extra environment variables.
+    # This can be used to avoid storing sensitive information in the values.yaml file.
+    extraEnvFromSecret: []
+    #  - name: MY_SECRET_KEY
+    #    secretName : my-secret
+    #    secretKey: my-secret-key
+
     # -- Startup probe configuration for worker pods. You shouldn't need to
     # change this, but if you want to you can find more information here:
     # <https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/>
@@ -611,10 +641,25 @@ couchdb:
     # @ignore
     repository: budibase/couchdb
     # @ignore
-    tag: v3.2.1
+    tag: v3.3.3
     # @ignore
     pullPolicy: Always
 
+  extraPorts:
+    # -- Extra ports to expose on the CouchDB service. We expose the SQS port
+    # by default, but you can add more ports here if you need to.
+    - name: sqs
+      containerPort: 4984
+
+  service:
+    extraPorts:
+      # -- Extra ports to expose on the CouchDB service. We expose the SQS port
+      # by default, but you can add more ports here if you need to.
+      - name: sqs
+        port: 4984
+        targetPort: 4984
+        protocol: TCP
+
   # @ignore
   # This should remain false. We ship Clouseau ourselves as part of the
   # budibase/couchdb image, and it's not possible to disable it because it's a

examples/nextjs-api-sales/package.json

@@ -22,6 +22,6 @@
     "@types/react": "17.0.39",
     "eslint": "8.10.0",
     "eslint-config-next": "12.1.0",
-    "typescript": "5.2.2"
+    "typescript": "5.5.2"
   }
 }

examples/nextjs-api-sales/yarn.lock

@@ -333,11 +333,11 @@ brace-expansion@^1.1.7:
     concat-map "0.0.1"
 
 braces@^3.0.1, braces@~3.0.2:
-  version "3.0.2"
+  version "3.0.3"
-  resolved "https://registry.yarnpkg.com/braces/-/braces-3.0.2.tgz#3454e1a462ee8d599e236df336cd9ea4f8afe107"
+  resolved "https://registry.yarnpkg.com/braces/-/braces-3.0.3.tgz#490332f40919452272d55a8480adc0c441358789"
-  integrity sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==
+  integrity sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==
   dependencies:
-    fill-range "^7.0.1"
+    fill-range "^7.1.1"
 
 bulma@^0.9.3:
   version "0.9.3"
@@ -781,10 +781,10 @@ file-entry-cache@^6.0.1:
   dependencies:
     flat-cache "^3.0.4"
 
-fill-range@^7.0.1:
+fill-range@^7.1.1:
-  version "7.0.1"
+  version "7.1.1"
-  resolved "https://registry.yarnpkg.com/fill-range/-/fill-range-7.0.1.tgz#1919a6a7c75fe38b2c7c77e5198535da9acdda40"
+  resolved "https://registry.yarnpkg.com/fill-range/-/fill-range-7.1.1.tgz#44265d3cac07e3ea7dc247516380643754a05292"
-  integrity sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==
+  integrity sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==
   dependencies:
     to-regex-range "^5.0.1"
 
@@ -1709,10 +1709,10 @@ type-fest@^0.20.2:
   resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-0.20.2.tgz#1bf207f4b28f91583666cb5fbd327887301cd5f4"
   integrity sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ==
 
-typescript@4.6.2:
+typescript@5.2.2:
-  version "4.6.2"
+  version "5.2.2"
-  resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.6.2.tgz#fe12d2727b708f4eef40f51598b3398baa9611d4"
+  resolved "https://registry.yarnpkg.com/typescript/-/typescript-5.2.2.tgz#5ebb5e5a5b75f085f22bc3f8460fba308310fa78"
-  integrity sha512-HM/hFigTBHZhLXshn9sN37H085+hQGeJHJ/X7LpBWLID/fbc2acUMfU+lGD98X81sKP+pFa9f0DZmCwB9GnbAg==
+  integrity sha512-mI4WrpHsbCIcwT9cF4FZvr80QUeKvsUsUvKDoR+X/7XHQH98xYD8YHZg7ANtz2GtZt/CBq2QJ0thkGJMHfqc1w==
 
 unbox-primitive@^1.0.1:
   version "1.0.1"

hosting/proxy/nginx.prod.conf

@@ -74,6 +74,7 @@ http {
     add_header X-Content-Type-Options nosniff always;
     add_header X-XSS-Protection "1; mode=block" always;
     add_header Content-Security-Policy "${csp_default}; ${csp_script}; ${csp_style}; ${csp_object}; ${csp_base_uri}; ${csp_connect}; ${csp_font}; ${csp_frame}; ${csp_img}; ${csp_manifest}; ${csp_media}; ${csp_worker};" always;
+    add_header Strict-Transport-Security "max-age=31536000; includeSubDomains; preload" always;
 
     # upstreams
     set $apps ${APPS_UPSTREAM_URL};

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "2.27.5",
+  "version": "2.29.5",
   "npmClient": "yarn",
   "packages": [
     "packages/*",

package.json

@@ -27,7 +27,7 @@
     "proper-lockfile": "^4.1.2",
     "svelte": "^4.2.10",
     "svelte-eslint-parser": "^0.33.1",
-    "typescript": "5.2.2",
+    "typescript": "5.5.2",
     "typescript-eslint": "^7.3.1",
     "yargs": "^17.7.2"
   },
@@ -37,10 +37,10 @@
     "build": "NODE_OPTIONS=--max-old-space-size=1500 lerna run build --stream",
     "build:apps": "yarn build --scope @budibase/server --scope @budibase/worker",
     "build:cli": "yarn build --scope @budibase/cli",
-    "build:oss": "NODE_OPTIONS=--max-old-space-size=1500 lerna run build --stream --ignore @budibase/account-portal --ignore @budibase/account-portal-server --ignore @budibase/account-portal-ui",
+    "build:oss": "NODE_OPTIONS=--max-old-space-size=1500 lerna run build --stream --ignore @budibase/account-portal-server --ignore @budibase/account-portal-ui",
-    "build:account-portal": "NODE_OPTIONS=--max-old-space-size=1500 lerna run build --stream --scope @budibase/account-portal --scope @budibase/account-portal-server --scope @budibase/account-portal-ui",
+    "build:account-portal": "NODE_OPTIONS=--max-old-space-size=1500 lerna run build --stream --scope @budibase/account-portal-server --scope @budibase/account-portal-ui",
     "build:dev": "lerna run --stream prebuild && yarn nx run-many --target=build --output-style=dynamic --watch --preserveWatchOutput",
-    "check:types": "lerna run --concurrency 2 check:types",
+    "check:types": "lerna run --concurrency 2 check:types --ignore @budibase/account-portal-server",
     "build:sdk": "lerna run --stream build:sdk",
     "deps:circular": "madge packages/server/dist/index.js packages/worker/src/index.ts packages/backend-core/dist/src/index.js packages/cli/src/index.js --circular",
     "release": "lerna publish from-package --yes --force-publish --no-git-tag-version --no-push --no-git-reset",

packages/account-portal

@@ -1 +1 @@
-Subproject commit 39acfff42a063e5a8a7d58d36721ec3103e16348
+Subproject commit ff16525b73c5751d344f5c161a682609c0a993f2

packages/backend-core/package.json

@@ -16,7 +16,7 @@
     "prepack": "cp package.json dist",
     "build": "tsc -p tsconfig.build.json --paths null && node ./scripts/build.js",
     "build:dev": "yarn prebuild && tsc --build --watch --preserveWatchOutput",
-    "check:types": "tsc -p tsconfig.json --noEmit --paths null",
+    "check:types": "tsc -p tsconfig.json --noEmit --paths null --target es2020",
     "test": "bash scripts/test.sh",
     "test:watch": "jest --watchAll"
   },
@@ -79,7 +79,7 @@
     "pouchdb-adapter-memory": "7.2.2",
     "testcontainers": "^10.7.2",
     "timekeeper": "2.2.0",
-    "typescript": "5.2.2"
+    "typescript": "5.5.2"
   },
   "nx": {
     "targets": {

packages/backend-core/src/constants/db.ts

@@ -72,4 +72,4 @@ export const DEFAULT_JOBS_TABLE_ID = "ta_bb_jobs"
 export const DEFAULT_INVENTORY_TABLE_ID = "ta_bb_inventory"
 export const DEFAULT_EXPENSES_TABLE_ID = "ta_bb_expenses"
 export const DEFAULT_EMPLOYEE_TABLE_ID = "ta_bb_employee"
-export const DEFAULT_BB_DATASOURCE_ID = "datasource_internal_bb_default"
+export { DEFAULT_BB_DATASOURCE_ID } from "@budibase/shared-core"

packages/backend-core/src/db/constants.ts

@@ -1,14 +1,5 @@
-export const CONSTANT_INTERNAL_ROW_COLS = [
+export {
-  "_id",
+  CONSTANT_INTERNAL_ROW_COLS,
-  "_rev",
+  CONSTANT_EXTERNAL_ROW_COLS,
-  "type",
+  isInternalColumnName,
-  "createdAt",
+} from "@budibase/shared-core"
-  "updatedAt",
-  "tableId",
-] as const
-
-export const CONSTANT_EXTERNAL_ROW_COLS = ["_id", "_rev", "tableId"] as const
-
-export function isInternalColumnName(name: string): boolean {
-  return (CONSTANT_INTERNAL_ROW_COLS as readonly string[]).includes(name)
-}

packages/backend-core/src/db/couch/DatabaseImpl.ts

@@ -8,6 +8,7 @@ import {
   DatabaseOpts,
   DatabasePutOpts,
   DatabaseQueryOpts,
+  DBError,
   Document,
   isDocument,
   RowResponse,
@@ -41,7 +42,7 @@ function buildNano(couchInfo: { url: string; cookie: string }) {
 
 type DBCall<T> = () => Promise<T>
 
-class CouchDBError extends Error {
+class CouchDBError extends Error implements DBError {
   status: number
   statusCode: number
   reason: string
@@ -328,7 +329,14 @@ export class DatabaseImpl implements Database {
   async sqlDiskCleanup(): Promise<void> {
     const dbName = this.name
     const url = `/${dbName}/_cleanup`
-    return await this._sqlQuery<void>(url, "POST")
+    try {
+      await this._sqlQuery<void>(url, "POST")
+    } catch (err: any) {
+      // hack for now - SQS throws a 500 when there is nothing to clean-up
+      if (err.status !== 500) {
+        throw err
+      }
+    }
   }
 
   // removes a document from sqlite
@@ -352,18 +360,15 @@ export class DatabaseImpl implements Database {
   }
 
   async destroy() {
+    if (env.SQS_SEARCH_ENABLE && (await this.exists(SQLITE_DESIGN_DOC_ID))) {
+      // delete the design document, then run the cleanup operation
+      const definition = await this.get<SQLiteDefinition>(SQLITE_DESIGN_DOC_ID)
+      // remove all tables - save the definition then trigger a cleanup
+      definition.sql.tables = {}
+      await this.put(definition)
+      await this.sqlDiskCleanup()
+    }
     try {
-      if (env.SQS_SEARCH_ENABLE) {
-        // delete the design document, then run the cleanup operation
-        try {
-          const definition = await this.get<SQLiteDefinition>(
-            SQLITE_DESIGN_DOC_ID
-          )
-          await this.remove(SQLITE_DESIGN_DOC_ID, definition._rev)
-        } finally {
-          await this.sqlDiskCleanup()
-        }
-      }
       return await this.nano().db.destroy(this.name)
     } catch (err: any) {
       // didn't exist, don't worry

packages/backend-core/src/environment.ts

@@ -93,15 +93,21 @@ function isApps() {
   return environment.SERVICE_TYPE === ServiceType.APPS
 }
 
+function isQA() {
+  return environment.BUDIBASE_ENVIRONMENT === "QA"
+}
+
 const environment = {
   isTest,
   isJest,
   isDev,
   isWorker,
   isApps,
+  isQA,
   isProd: () => {
     return !isDev()
   },
+  BUDIBASE_ENVIRONMENT: process.env.BUDIBASE_ENVIRONMENT,
   JS_BCRYPT: process.env.JS_BCRYPT,
   JWT_SECRET: process.env.JWT_SECRET,
   JWT_SECRET_FALLBACK: process.env.JWT_SECRET_FALLBACK,
@@ -120,6 +126,7 @@ const environment = {
   REDIS_CLUSTERED: process.env.REDIS_CLUSTERED,
   MINIO_ACCESS_KEY: process.env.MINIO_ACCESS_KEY,
   MINIO_SECRET_KEY: process.env.MINIO_SECRET_KEY,
+  AWS_SESSION_TOKEN: process.env.AWS_SESSION_TOKEN,
   AWS_REGION: process.env.AWS_REGION,
   MINIO_URL: process.env.MINIO_URL,
   MINIO_ENABLED: process.env.MINIO_ENABLED || 1,

packages/backend-core/src/middleware/joi-validator.ts

@@ -3,7 +3,8 @@ import { Ctx } from "@budibase/types"
 
 function validate(
   schema: Joi.ObjectSchema | Joi.ArraySchema,
-  property: string
+  property: string,
+  opts: { errorPrefix: string } = { errorPrefix: `Invalid ${property}` }
 ) {
   // Return a Koa middleware function
   return (ctx: Ctx, next: any) => {
@@ -29,16 +30,26 @@ function validate(
 
     const { error } = schema.validate(params)
     if (error) {
-      ctx.throw(400, `Invalid ${property} - ${error.message}`)
+      let message = error.message
+      if (opts.errorPrefix) {
+        message = `Invalid ${property} - ${message}`
+      }
+      ctx.throw(400, message)
     }
     return next()
   }
 }
 
-export function body(schema: Joi.ObjectSchema | Joi.ArraySchema) {
+export function body(
-  return validate(schema, "body")
+  schema: Joi.ObjectSchema | Joi.ArraySchema,
+  opts?: { errorPrefix: string }
+) {
+  return validate(schema, "body", opts)
 }
 
-export function params(schema: Joi.ObjectSchema | Joi.ArraySchema) {
+export function params(
-  return validate(schema, "params")
+  schema: Joi.ObjectSchema | Joi.ArraySchema,
+  opts?: { errorPrefix: string }
+) {
+  return validate(schema, "params", opts)
 }

packages/backend-core/src/objectStore/objectStore.ts

@@ -14,6 +14,7 @@ import { v4 } from "uuid"
 import { APP_PREFIX, APP_DEV_PREFIX } from "../db"
 import fsp from "fs/promises"
 import { HeadObjectOutput } from "aws-sdk/clients/s3"
+import { ReadableStream } from "stream/web"
 
 const streamPipeline = promisify(stream.pipeline)
 // use this as a temporary store of buckets that are being created
@@ -41,10 +42,7 @@ type UploadParams = BaseUploadParams & {
   path?: string | PathLike
 }
 
-export type StreamTypes =
+export type StreamTypes = ReadStream | NodeJS.ReadableStream
-  | ReadStream
-  | NodeJS.ReadableStream
-  | ReadableStream<Uint8Array>
 
 export type StreamUploadParams = BaseUploadParams & {
   stream?: StreamTypes
@@ -103,6 +101,11 @@ export function ObjectStore(
     }
   }
 
+  // for AWS Credentials using temporary session token
+  if (!env.MINIO_ENABLED && env.AWS_SESSION_TOKEN) {
+    config.sessionToken = env.AWS_SESSION_TOKEN
+  }
+
   // custom S3 is in use i.e. minio
   if (env.MINIO_URL) {
     if (opts.presigning && env.MINIO_ENABLED) {
@@ -222,6 +225,9 @@ export async function streamUpload({
   extra,
   ttl,
 }: StreamUploadParams) {
+  if (!stream) {
+    throw new Error("Stream to upload is invalid/undefined")
+  }
   const extension = filename.split(".").pop()
   const objectStore = ObjectStore(bucketName)
   const bucketCreated = await createBucketIfNotExists(objectStore, bucketName)
@@ -251,14 +257,27 @@ export async function streamUpload({
       : CONTENT_TYPE_MAP.txt
   }
 
+  const bucket = sanitizeBucket(bucketName),
+    objKey = sanitizeKey(filename)
   const params = {
-    Bucket: sanitizeBucket(bucketName),
+    Bucket: bucket,
-    Key: sanitizeKey(filename),
+    Key: objKey,
     Body: stream,
     ContentType: contentType,
     ...extra,
   }
-  return objectStore.upload(params).promise()
+
+  const details = await objectStore.upload(params).promise()
+  const headDetails = await objectStore
+    .headObject({
+      Bucket: bucket,
+      Key: objKey,
+    })
+    .promise()
+  return {
+    ...details,
+    ContentLength: headDetails.ContentLength,
+  }
 }
 
 /**

packages/backend-core/src/queue/inMemoryQueue.ts

@@ -63,12 +63,12 @@ class InMemoryQueue implements Partial<Queue> {
    * Same callback API as Bull, each callback passed to this will consume messages as they are
    * available. Please note this is a queue service, not a notification service, so each
    * consumer will receive different messages.
-   * @param func The callback function which will return a "Job", the same
    * as the Bull API, within this job the property "data" contains the JSON message. Please
    * note this is incredibly limited compared to Bull as in reality the Job would contain
    * a lot more information about the queue and current status of Bull cluster.
    */
-  async process(func: any) {
+  async process(concurrencyOrFunc: number | any, func?: any) {
+    func = typeof concurrencyOrFunc === "number" ? func : concurrencyOrFunc
     this._emitter.on("message", async () => {
       if (this._messages.length <= 0) {
         return

packages/backend-core/src/queue/queue.ts

@@ -21,6 +21,7 @@ let cleanupInterval: NodeJS.Timeout
 async function cleanup() {
   for (let queue of QUEUES) {
     await queue.clean(CLEANUP_PERIOD_MS, "completed")
+    await queue.clean(CLEANUP_PERIOD_MS, "failed")
   }
 }
 

packages/backend-core/src/sql/sql.ts

@@ -1,10 +1,10 @@
 import { Knex, knex } from "knex"
 import * as dbCore from "../db"
 import {
-  isIsoDateString,
-  isValidFilter,
   getNativeSql,
   isExternalTable,
+  isIsoDateString,
+  isValidFilter,
 } from "./utils"
 import { SqlStatements } from "./sqlStatements"
 import SqlTableQueryBuilder from "./sqlTable"
@@ -12,21 +12,21 @@ import {
   BBReferenceFieldMetadata,
   FieldSchema,
   FieldType,
+  INTERNAL_TABLE_SOURCE_ID,
   JsonFieldMetadata,
+  JsonTypes,
   Operation,
+  prefixed,
   QueryJson,
-  SqlQuery,
+  QueryOptions,
   RelationshipsJson,
   SearchFilters,
-  SortDirection,
+  SortOrder,
+  SqlClient,
+  SqlQuery,
   SqlQueryBinding,
   Table,
   TableSourceType,
-  INTERNAL_TABLE_SOURCE_ID,
-  SqlClient,
-  QueryOptions,
-  JsonTypes,
-  prefixed,
 } from "@budibase/types"
 import environment from "../environment"
 import { helpers } from "@budibase/shared-core"
@@ -114,7 +114,7 @@ function generateSelectStatement(
 ): (string | Knex.Raw)[] | "*" {
   const { resource, meta } = json
 
-  if (!resource) {
+  if (!resource || !resource.fields || resource.fields.length === 0) {
     return "*"
   }
 
@@ -184,7 +184,11 @@ class InternalBuilder {
     query: Knex.QueryBuilder,
     filters: SearchFilters | undefined,
     table: Table,
-    opts: { aliases?: Record<string, string>; relationship?: boolean }
+    opts: {
+      aliases?: Record<string, string>
+      relationship?: boolean
+      columnPrefix?: string
+    }
   ): Knex.QueryBuilder {
     if (!filters) {
       return query
@@ -192,7 +196,10 @@ class InternalBuilder {
     filters = parseFilters(filters)
     // if all or specified in filters, then everything is an or
     const allOr = filters.allOr
-    const sqlStatements = new SqlStatements(this.client, table, { allOr })
+    const sqlStatements = new SqlStatements(this.client, table, {
+      allOr,
+      columnPrefix: opts.columnPrefix,
+    })
     const tableName =
       this.client === SqlClient.SQL_LITE ? table._id! : table.name
 
@@ -397,9 +404,9 @@ class InternalBuilder {
       contains(filters.containsAny, true)
     }
 
+    const tableRef = opts?.aliases?.[table._id!] || table._id
     // when searching internal tables make sure long looking for rows
-    if (filters.documentType && !isExternalTable(table)) {
+    if (filters.documentType && !isExternalTable(table) && tableRef) {
-      const tableRef = opts?.aliases?.[table._id!] || table._id
       // has to be its own option, must always be AND onto the search
       query.andWhereLike(
         `${tableRef}._id`,
@@ -410,28 +417,50 @@ class InternalBuilder {
     return query
   }
 
-  addSorting(query: Knex.QueryBuilder, json: QueryJson): Knex.QueryBuilder {
+  addDistinctCount(
-    let { sort, paginate } = json
+    query: Knex.QueryBuilder,
+    json: QueryJson
+  ): Knex.QueryBuilder {
     const table = json.meta.table
+    const primary = table.primary
+    const aliases = json.tableAliases
+    const aliased =
+      table.name && aliases?.[table.name] ? aliases[table.name] : table.name
+    if (!primary) {
+      throw new Error("SQL counting requires primary key to be supplied")
+    }
+    return query.countDistinct(`${aliased}.${primary[0]} as total`)
+  }
+
+  addSorting(query: Knex.QueryBuilder, json: QueryJson): Knex.QueryBuilder {
+    let { sort } = json
+    const table = json.meta.table
+    const primaryKey = table.primary
     const tableName = getTableName(table)
     const aliases = json.tableAliases
     const aliased =
       tableName && aliases?.[tableName] ? aliases[tableName] : table?.name
+    if (!Array.isArray(primaryKey)) {
+      throw new Error("Sorting requires primary key to be specified for table")
+    }
     if (sort && Object.keys(sort || {}).length > 0) {
       for (let [key, value] of Object.entries(sort)) {
         const direction =
-          value.direction === SortDirection.ASCENDING ? "asc" : "desc"
+          value.direction === SortOrder.ASCENDING ? "asc" : "desc"
         let nulls
         if (this.client === SqlClient.POSTGRES) {
           // All other clients already sort this as expected by default, and adding this to the rest of the clients is causing issues
-          nulls = value.direction === SortDirection.ASCENDING ? "first" : "last"
+          nulls = value.direction === SortOrder.ASCENDING ? "first" : "last"
         }
 
         query = query.orderBy(`${aliased}.${key}`, direction, nulls)
       }
-    } else if (this.client === SqlClient.MS_SQL && paginate?.limit) {
+    }
-      // @ts-ignore
+
-      query = query.orderBy(`${aliased}.${table?.primary[0]}`)
+    // add sorting by the primary key if the result isn't already sorted by it,
+    // to make sure result is deterministic
+    if (!sort || sort[primaryKey[0]] === undefined) {
+      query = query.orderBy(`${aliased}.${primaryKey[0]}`)
     }
     return query
   }
@@ -522,7 +551,7 @@ class InternalBuilder {
           })
       }
     }
-    return query.limit(BASE_LIMIT)
+    return query
   }
 
   knexWithAlias(
@@ -533,13 +562,12 @@ class InternalBuilder {
     const tableName = endpoint.entityId
     const tableAlias = aliases?.[tableName]
 
-    const query = knex(
+    return knex(
       this.tableNameWithSchema(tableName, {
         alias: tableAlias,
         schema: endpoint.schema,
       })
     )
-    return query
   }
 
   create(knex: Knex, json: QueryJson, opts: QueryOptions): Knex.QueryBuilder {
@@ -571,52 +599,95 @@ class InternalBuilder {
     return query.insert(parsedBody)
   }
 
-  read(knex: Knex, json: QueryJson, limit: number): Knex.QueryBuilder {
+  bulkUpsert(knex: Knex, json: QueryJson): Knex.QueryBuilder {
-    let { endpoint, resource, filters, paginate, relationships, tableAliases } =
+    const { endpoint, body } = json
-      json
+    let query = this.knexWithAlias(knex, endpoint)
+    if (!Array.isArray(body)) {
+      return query
+    }
+    const parsedBody = body.map(row => parseBody(row))
+    if (
+      this.client === SqlClient.POSTGRES ||
+      this.client === SqlClient.SQL_LITE ||
+      this.client === SqlClient.MY_SQL
+    ) {
+      const primary = json.meta.table.primary
+      if (!primary) {
+        throw new Error("Primary key is required for upsert")
+      }
+      const ret = query.insert(parsedBody).onConflict(primary).merge()
+      return ret
+    } else if (this.client === SqlClient.MS_SQL) {
+      // No upsert or onConflict support in MSSQL yet, see:
+      //   https://github.com/knex/knex/pull/6050
+      return query.insert(parsedBody)
+    }
+    return query.upsert(parsedBody)
+  }
+
+  read(
+    knex: Knex,
+    json: QueryJson,
+    opts: {
+      limits?: { base: number; query: number }
+    } = {}
+  ): Knex.QueryBuilder {
+    let { endpoint, filters, paginate, relationships, tableAliases } = json
+    const { limits } = opts
+    const counting = endpoint.operation === Operation.COUNT
 
     const tableName = endpoint.entityId
-    // select all if not specified
+    // start building the query
-    if (!resource) {
+    let query = this.knexWithAlias(knex, endpoint, tableAliases)
-      resource = { fields: [] }
-    }
-    let selectStatement: string | (string | Knex.Raw)[] = "*"
-    // handle select
-    if (resource.fields && resource.fields.length > 0) {
-      // select the resources as the format "table.columnName" - this is what is provided
-      // by the resource builder further up
-      selectStatement = generateSelectStatement(json, knex)
-    }
-    let foundLimit = limit || BASE_LIMIT
     // handle pagination
     let foundOffset: number | null = null
+    let foundLimit = limits?.query || limits?.base
     if (paginate && paginate.page && paginate.limit) {
       // @ts-ignore
       const page = paginate.page <= 1 ? 0 : paginate.page - 1
       const offset = page * paginate.limit
       foundLimit = paginate.limit
       foundOffset = offset
+    } else if (paginate && paginate.offset && paginate.limit) {
+      foundLimit = paginate.limit
+      foundOffset = paginate.offset
     } else if (paginate && paginate.limit) {
       foundLimit = paginate.limit
     }
-    // start building the query
+    // counting should not sort, limit or offset
-    let query = this.knexWithAlias(knex, endpoint, tableAliases)
+    if (!counting) {
-    query = query.limit(foundLimit)
+      // add the found limit if supplied
-    if (foundOffset) {
+      if (foundLimit != null) {
-      query = query.offset(foundOffset)
+        query = query.limit(foundLimit)
+      }
+      // add overall pagination
+      if (foundOffset != null) {
+        query = query.offset(foundOffset)
+      }
+      // add sorting to pre-query
+      // no point in sorting when counting
+      query = this.addSorting(query, json)
     }
+    // add filters to the query (where)
     query = this.addFilters(query, filters, json.meta.table, {
+      columnPrefix: json.meta.columnPrefix,
       aliases: tableAliases,
     })
 
-    // add sorting to pre-query
-    query = this.addSorting(query, json)
     const alias = tableAliases?.[tableName] || tableName
-    let preQuery = knex({
+    let preQuery: Knex.QueryBuilder = knex({
-      [alias]: query,
+      // the typescript definition for the knex constructor doesn't support this
-    } as any).select(selectStatement) as any
+      // syntax, but it is the only way to alias a pre-query result as part of
+      // a query - there is an alias dictionary type, but it assumes it can only
+      // be a table name, not a pre-query
+      [alias]: query as any,
+    })
+    // if counting, use distinct count, else select
+    preQuery = !counting
+      ? preQuery.select(generateSelectStatement(json, knex))
+      : this.addDistinctCount(preQuery, json)
     // have to add after as well (this breaks MS-SQL)
-    if (this.client !== SqlClient.MS_SQL) {
+    if (this.client !== SqlClient.MS_SQL && !counting) {
       preQuery = this.addSorting(preQuery, json)
     }
     // handle joins
@@ -627,7 +698,15 @@ class InternalBuilder {
       endpoint.schema,
       tableAliases
     )
+
+    // add a base limit over the whole query
+    // if counting we can't set this limit
+    if (limits?.base) {
+      query = query.limit(limits.base)
+    }
+
     return this.addFilters(query, filters, json.meta.table, {
+      columnPrefix: json.meta.columnPrefix,
       relationship: true,
       aliases: tableAliases,
     })
@@ -638,6 +717,7 @@ class InternalBuilder {
     let query = this.knexWithAlias(knex, endpoint, tableAliases)
     const parsedBody = parseBody(body)
     query = this.addFilters(query, filters, json.meta.table, {
+      columnPrefix: json.meta.columnPrefix,
       aliases: tableAliases,
     })
     // mysql can't use returning
@@ -652,6 +732,7 @@ class InternalBuilder {
     const { endpoint, filters, tableAliases } = json
     let query = this.knexWithAlias(knex, endpoint, tableAliases)
     query = this.addFilters(query, filters, json.meta.table, {
+      columnPrefix: json.meta.columnPrefix,
       aliases: tableAliases,
     })
     // mysql can't use returning
@@ -671,6 +752,19 @@ class SqlQueryBuilder extends SqlTableQueryBuilder {
     this.limit = limit
   }
 
+  private convertToNative(query: Knex.QueryBuilder, opts: QueryOptions = {}) {
+    const sqlClient = this.getSqlClient()
+    if (opts?.disableBindings) {
+      return { sql: query.toString() }
+    } else {
+      let native = getNativeSql(query)
+      if (sqlClient === SqlClient.SQL_LITE) {
+        native = convertBooleans(native)
+      }
+      return native
+    }
+  }
+
   /**
    * @param json The JSON query DSL which is to be converted to SQL.
    * @param opts extra options which are to be passed into the query builder, e.g. disableReturning
@@ -694,7 +788,16 @@ class SqlQueryBuilder extends SqlTableQueryBuilder {
         query = builder.create(client, json, opts)
         break
       case Operation.READ:
-        query = builder.read(client, json, this.limit)
+        query = builder.read(client, json, {
+          limits: {
+            query: this.limit,
+            base: BASE_LIMIT,
+          },
+        })
+        break
+      case Operation.COUNT:
+        // read without any limits to count
+        query = builder.read(client, json)
         break
       case Operation.UPDATE:
         query = builder.update(client, json, opts)
@@ -705,6 +808,9 @@ class SqlQueryBuilder extends SqlTableQueryBuilder {
       case Operation.BULK_CREATE:
         query = builder.bulkCreate(client, json)
         break
+      case Operation.BULK_UPSERT:
+        query = builder.bulkUpsert(client, json)
+        break
       case Operation.CREATE_TABLE:
       case Operation.UPDATE_TABLE:
       case Operation.DELETE_TABLE:
@@ -713,15 +819,7 @@ class SqlQueryBuilder extends SqlTableQueryBuilder {
         throw `Operation type is not supported by SQL query builder`
     }
 
-    if (opts?.disableBindings) {
+    return this.convertToNative(query, opts)
-      return { sql: query.toString() }
-    } else {
-      let native = getNativeSql(query)
-      if (sqlClient === SqlClient.SQL_LITE) {
-        native = convertBooleans(native)
-      }
-      return native
-    }
   }
 
   async getReturningRow(queryFn: QueryFunction, json: QueryJson) {
@@ -797,6 +895,9 @@ class SqlQueryBuilder extends SqlTableQueryBuilder {
         await this.getReturningRow(queryFn, this.checkLookupKeys(id, json))
       )
     }
+    if (operation === Operation.COUNT) {
+      return results
+    }
     if (operation !== Operation.READ) {
       return row
     }

packages/backend-core/src/sql/sqlStatements.ts

@@ -5,19 +5,27 @@ export class SqlStatements {
   client: string
   table: Table
   allOr: boolean | undefined
+  columnPrefix: string | undefined
+
   constructor(
     client: string,
     table: Table,
-    { allOr }: { allOr?: boolean } = {}
+    { allOr, columnPrefix }: { allOr?: boolean; columnPrefix?: string } = {}
   ) {
     this.client = client
     this.table = table
     this.allOr = allOr
+    this.columnPrefix = columnPrefix
   }
 
   getField(key: string): FieldSchema | undefined {
     const fieldName = key.split(".")[1]
-    return this.table.schema[fieldName]
+    let found = this.table.schema[fieldName]
+    if (!found && this.columnPrefix) {
+      const prefixRemovedFieldName = fieldName.replace(this.columnPrefix, "")
+      found = this.table.schema[prefixRemovedFieldName]
+    }
+    return found
   }
 
   between(

packages/backend-core/src/sql/sqlTable.ts

@@ -109,8 +109,10 @@ function generateSchema(
           const { tableName } = breakExternalTableId(column.tableId)
           // @ts-ignore
           const relatedTable = tables[tableName]
-          if (!relatedTable) {
+          if (!relatedTable || !relatedTable.primary) {
-            throw new Error("Referenced table doesn't exist")
+            throw new Error(
+              "Referenced table doesn't exist or has no primary keys"
+            )
           }
           const relatedPrimary = relatedTable.primary[0]
           const externalType = relatedTable.schema[relatedPrimary].externalType

packages/backend-core/src/sql/utils.ts

@@ -55,10 +55,7 @@ export function buildExternalTableId(datasourceId: string, tableName: string) {
   return `${datasourceId}${DOUBLE_SEPARATOR}${tableName}`
 }
 
-export function breakExternalTableId(tableId: string | undefined) {
+export function breakExternalTableId(tableId: string) {
-  if (!tableId) {
-    return {}
-  }
   const parts = tableId.split(DOUBLE_SEPARATOR)
   let datasourceId = parts.shift()
   // if they need joined
@@ -67,6 +64,9 @@ export function breakExternalTableId(tableId: string | undefined) {
   if (tableName.includes(ENCODED_SPACE)) {
     tableName = decodeURIComponent(tableName)
   }
+  if (!datasourceId || !tableName) {
+    throw new Error("Unable to get datasource/table name from table ID")
+  }
   return { datasourceId, tableName }
 }
 

packages/backend-core/src/tenancy/db.ts

@@ -1,6 +1,21 @@
 import { getDB } from "../db/db"
 import { getGlobalDBName } from "../context"
+import { TenantInfo } from "@budibase/types"
 
 export function getTenantDB(tenantId: string) {
   return getDB(getGlobalDBName(tenantId))
 }
+
+export async function saveTenantInfo(tenantInfo: TenantInfo) {
+  const db = getTenantDB(tenantInfo.tenantId)
+  // save the tenant info to db
+  return db.put({
+    _id: "tenant_info",
+    ...tenantInfo,
+  })
+}
+
+export async function getTenantInfo(tenantId: string): Promise<TenantInfo> {
+  const db = getTenantDB(tenantId)
+  return db.get("tenant_info")
+}

packages/backend-core/tests/core/utilities/structures/accounts.ts

@@ -24,7 +24,6 @@ export const account = (partial: Partial<Account> = {}): Account => {
     createdAt: Date.now(),
     verified: true,
     verificationSent: true,
-    tier: "FREE", // DEPRECATED
     authType: AuthType.PASSWORD,
     name: generator.name(),
     size: "10+",

packages/bbui/src/Modal/Modal.svelte

@@ -162,6 +162,7 @@
     max-height: 100%;
   }
   .modal-inner-wrapper {
+    padding: 40px;
     flex: 1 1 auto;
     display: flex;
     flex-direction: row;
@@ -176,7 +177,6 @@
     border: 2px solid var(--spectrum-global-color-gray-200);
     overflow: visible;
     max-height: none;
-    margin: 40px 0;
     transform: none;
     --spectrum-dialog-confirm-border-radius: var(
       --spectrum-global-dimension-size-100

packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte

@@ -15,6 +15,9 @@
     Checkbox,
     DatePicker,
     DrawerContent,
+    Toggle,
+    Icon,
+    Divider,
   } from "@budibase/bbui"
   import CreateWebhookModal from "components/automation/Shared/CreateWebhookModal.svelte"
   import { automationStore, selectedAutomation, tables } from "stores/builder"
@@ -40,7 +43,7 @@
     EditorModes,
   } from "components/common/CodeEditor"
   import FilterBuilder from "components/design/settings/controls/FilterEditor/FilterBuilder.svelte"
-  import { LuceneUtils, Utils } from "@budibase/frontend-core"
+  import { QueryUtils, Utils, search } from "@budibase/frontend-core"
   import {
     getSchemaForDatasourcePlus,
     getEnvironmentBindings,
@@ -72,7 +75,11 @@
   $: schema = getSchemaForDatasourcePlus(tableId, {
     searchableSchema: true,
   }).schema
-  $: schemaFields = Object.values(schema || {})
+  $: schemaFields = search.getFields(
+    $tables.list,
+    Object.values(schema || {}),
+    { allowLinks: true }
+  )
   $: queryLimit = tableId?.includes("datasource") ? "∞" : "1000"
   $: isTrigger = block?.type === "TRIGGER"
   $: isUpdateRow = stepId === ActionStepID.UPDATE_ROW
@@ -88,6 +95,8 @@
       ? [hbAutocomplete([...bindingsToCompletions(bindings, codeMode)])]
       : []
 
+  let testDataRowVisibility = {}
+
   const getInputData = (testData, blockInputs) => {
     // Test data is not cloned for reactivity
     let newInputData = testData || cloneDeep(blockInputs)
@@ -118,7 +127,6 @@
         searchableSchema: true,
       }).schema
     }
-
     try {
       if (isTestModal) {
         let newTestData = { schema }
@@ -196,7 +204,8 @@
         (automation.trigger?.event === "row:update" ||
           automation.trigger?.event === "row:save")
       ) {
-        if (name !== "id" && name !== "revision") return `trigger.row.${name}`
+        let noRowKeywordBindings = ["id", "revision", "oldRow"]
+        if (!noRowKeywordBindings.includes(name)) return `trigger.row.${name}`
       }
       /* End special cases for generating custom schemas based on triggers */
 
@@ -343,7 +352,7 @@
   }
 
   function saveFilters(key) {
-    const filters = LuceneUtils.buildLuceneQuery(tempFilters)
+    const filters = QueryUtils.buildQuery(tempFilters)
     const defKey = `${key}-def`
     onChange({ detail: filters }, key)
     // need to store the builder definition in the automation
@@ -372,7 +381,11 @@
 
   function getFieldLabel(key, value) {
     const requiredSuffix = requiredProperties.includes(key) ? "*" : ""
-    return `${value.title || (key === "row" ? "Table" : key)} ${requiredSuffix}`
+    return `${value.title || (key === "row" ? "Row" : key)} ${requiredSuffix}`
+  }
+
+  function toggleTestDataRowVisibility(key) {
+    testDataRowVisibility[key] = !testDataRowVisibility[key]
   }
 
   function handleAttachmentParams(keyValueObj) {
@@ -385,6 +398,16 @@
     return params
   }
 
+  function toggleAttachmentBinding(e, key) {
+    onChange(
+      {
+        detail: "",
+      },
+      key
+    )
+    onChange({ detail: { useAttachmentBinding: e.detail } }, "meta")
+  }
+
   onMount(async () => {
     try {
       await environment.loadVariables()
@@ -462,27 +485,64 @@
               <div class="label-wrapper">
                 <Label>{label}</Label>
               </div>
-              <div class="attachment-field-width">
+              <div class="toggle-container">
-                <KeyValueBuilder
+                <Toggle
-                  on:change={e =>
+                  value={inputData?.meta?.useAttachmentBinding}
-                    onChange(
+                  text={"Use bindings"}
-                      {
+                  size={"XS"}
-                        detail: e.detail.map(({ name, value }) => ({
+                  on:change={e => toggleAttachmentBinding(e, key)}
-                          url: name,
-                          filename: value,
-                        })),
-                      },
-                      key
-                    )}
-                  object={handleAttachmentParams(inputData[key])}
-                  allowJS
-                  {bindings}
-                  keyBindings
-                  customButtonText={"Add attachment"}
-                  keyPlaceholder={"URL"}
-                  valuePlaceholder={"Filename"}
                 />
               </div>
+
+              <div class="attachment-field-width">
+                {#if !inputData?.meta?.useAttachmentBinding}
+                  <KeyValueBuilder
+                    on:change={e =>
+                      onChange(
+                        {
+                          detail: e.detail.map(({ name, value }) => ({
+                            url: name,
+                            filename: value,
+                          })),
+                        },
+                        key
+                      )}
+                    object={handleAttachmentParams(inputData[key])}
+                    allowJS
+                    {bindings}
+                    keyBindings
+                    customButtonText={"Add attachment"}
+                    keyPlaceholder={"URL"}
+                    valuePlaceholder={"Filename"}
+                  />
+                {:else if isTestModal}
+                  <ModalBindableInput
+                    title={value.title || label}
+                    value={inputData[key]}
+                    panel={AutomationBindingPanel}
+                    type={value.customType}
+                    on:change={e => onChange(e, key)}
+                    {bindings}
+                    updateOnChange={false}
+                  />
+                {:else}
+                  <div class="test">
+                    <DrawerBindableInput
+                      title={value.title ?? label}
+                      panel={AutomationBindingPanel}
+                      type={value.customType}
+                      value={inputData[key]}
+                      on:change={e => onChange(e, key)}
+                      {bindings}
+                      updateOnChange={false}
+                      placeholder={value.customType === "queryLimit"
+                        ? queryLimit
+                        : ""}
+                      drawerLeft="260px"
+                    />
+                  </div>
+                {/if}
+              </div>
             </div>
           {:else if value.customType === "filters"}
             <ActionButton on:click={drawer.show}>Define filters</ActionButton>
@@ -560,20 +620,48 @@
               on:change={e => onChange(e, key)}
             />
           {:else if value.customType === "row"}
-            <RowSelector
+            {#if isTestModal}
-              value={inputData[key]}
+              <div class="align-horizontally">
-              meta={inputData["meta"] || {}}
+                <Icon
-              on:change={e => {
+                  name={testDataRowVisibility[key] ? "Remove" : "Add"}
-                if (e.detail?.key) {
+                  hoverable
-                  onChange(e, e.detail.key)
+                  on:click={() => toggleTestDataRowVisibility(key)}
-                } else {
+                />
-                  onChange(e, key)
+                <Label size="XL">{label}</Label>
-                }
+              </div>
-              }}
+              {#if testDataRowVisibility[key]}
-              {bindings}
+                <RowSelector
-              {isTestModal}
+                  value={inputData[key]}
-              {isUpdateRow}
+                  meta={inputData["meta"] || {}}
-            />
+                  on:change={e => {
+                    if (e.detail?.key) {
+                      onChange(e, e.detail.key)
+                    } else {
+                      onChange(e, key)
+                    }
+                  }}
+                  {bindings}
+                  {isTestModal}
+                  {isUpdateRow}
+                />
+              {/if}
+              <Divider />
+            {:else}
+              <RowSelector
+                value={inputData[key]}
+                meta={inputData["meta"] || {}}
+                on:change={e => {
+                  if (e.detail?.key) {
+                    onChange(e, e.detail.key)
+                  } else {
+                    onChange(e, key)
+                  }
+                }}
+                {bindings}
+                {isTestModal}
+                {isUpdateRow}
+              />
+            {/if}
           {:else if value.customType === "webhookUrl"}
             <WebhookDisplay
               on:change={e => onChange(e, key)}
@@ -689,6 +777,12 @@
     width: 320px;
   }
 
+  .align-horizontally {
+    display: flex;
+    gap: var(--spacing-s);
+    align-items: center;
+  }
+
   .fields {
     display: flex;
     flex-direction: column;

packages/builder/src/components/automation/SetupPanel/RowSelector.svelte

@@ -10,12 +10,12 @@
   import { TableNames } from "constants"
 
   const dispatch = createEventDispatcher()
-
   export let value
   export let meta
   export let bindings
   export let isTestModal
   export let isUpdateRow
+
   $: parsedBindings = bindings.map(binding => {
     let clone = Object.assign({}, binding)
     clone.icon = "ShareAndroid"
@@ -94,17 +94,22 @@
     dispatch("change", newValue)
   }
 
-  const onChangeSetting = (e, field) => {
+  const onChangeSetting = (field, key, value) => {
-    let fields = {}
+    let newField = {}
-    fields[field] = {
+    newField[field] = {
-      clearRelationships: e.detail,
+      [key]: value,
     }
+
+    let updatedFields = {
+      ...meta?.fields,
+      ...newField,
+    }
+
     dispatch("change", {
       key: "meta",
-      fields,
+      fields: updatedFields,
     })
   }
-
   // Ensure any nullish tableId values get set to empty string so
   // that the select works
   $: if (value?.tableId == null) value = { tableId: "" }
@@ -157,6 +162,9 @@
                 bindings={parsedBindings}
                 {value}
                 {onChange}
+                useAttachmentBinding={meta?.fields?.[field]
+                  ?.useAttachmentBinding}
+                {onChangeSetting}
               />
             </DrawerBindableSlot>
           {/if}
@@ -167,7 +175,8 @@
                 value={meta.fields?.[field]?.clearRelationships}
                 text={"Clear relationships if empty?"}
                 size={"S"}
-                on:change={e => onChangeSetting(e, field)}
+                on:change={e =>
+                  onChangeSetting(field, "clearRelationships", e.detail)}
               />
             </div>
           {/if}

packages/builder/src/components/automation/SetupPanel/RowSelectorTypes.svelte

@@ -1,5 +1,11 @@
 <script>
-  import { Select, DatePicker, Multiselect, TextArea } from "@budibase/bbui"
+  import {
+    Select,
+    DatePicker,
+    Multiselect,
+    TextArea,
+    Toggle,
+  } from "@budibase/bbui"
   import { FieldType } from "@budibase/types"
   import LinkedRowSelector from "components/common/LinkedRowSelector.svelte"
   import DrawerBindableInput from "../../common/bindings/DrawerBindableInput.svelte"
@@ -14,6 +20,8 @@
   export let value
   export let bindings
   export let isTestModal
+  export let useAttachmentBinding
+  export let onChangeSetting
 
   $: parsedBindings = bindings.map(binding => {
     let clone = Object.assign({}, binding)
@@ -27,6 +35,8 @@
     FieldType.SIGNATURE_SINGLE,
   ]
 
+  let previousBindingState = useAttachmentBinding
+
   function schemaHasOptions(schema) {
     return !!schema.constraints?.inclusion?.length
   }
@@ -34,13 +44,6 @@
   function handleAttachmentParams(keyValueObj) {
     let params = {}
 
-    if (
-      (schema.type === FieldType.ATTACHMENT_SINGLE ||
-        schema.type === FieldType.SIGNATURE_SINGLE) &&
-      Object.keys(keyValueObj).length === 0
-    ) {
-      return []
-    }
     if (!Array.isArray(keyValueObj) && keyValueObj) {
       keyValueObj = [keyValueObj]
     }
@@ -52,6 +55,26 @@
     }
     return params
   }
+
+  async function handleToggleChange(toggleField, event) {
+    if (event.detail === true) {
+      value[toggleField] = []
+    } else {
+      value[toggleField] = ""
+    }
+    previousBindingState = event.detail
+    onChangeSetting(toggleField, "useAttachmentBinding", event.detail)
+    onChange({ detail: value[toggleField] }, toggleField)
+  }
+
+  $: if (useAttachmentBinding !== previousBindingState) {
+    if (useAttachmentBinding) {
+      value[field] = []
+    } else {
+      value[field] = ""
+    }
+    previousBindingState = useAttachmentBinding
+  }
 </script>
 
 {#if schemaHasOptions(schema) && schema.type !== "array"}
@@ -108,38 +131,65 @@
     useLabel={false}
   />
 {:else if attachmentTypes.includes(schema.type)}
-  <div class="attachment-field-spacinng">
+  <div class="attachment-field-container">
-    <KeyValueBuilder
+    <div class="toggle-container">
-      on:change={e =>
+      <Toggle
-        onChange(
+        value={useAttachmentBinding}
-          {
+        text={"Use bindings"}
-            detail:
+        size={"XS"}
-              schema.type === FieldType.ATTACHMENT_SINGLE ||
+        on:change={e => handleToggleChange(field, e)}
-              schema.type === FieldType.SIGNATURE_SINGLE
+      />
-                ? e.detail.length > 0
+    </div>
-                  ? {
+    {#if !useAttachmentBinding}
-                      url: e.detail[0].name,
+      <div class="attachment-field-spacing">
-                      filename: e.detail[0].value,
+        <KeyValueBuilder
-                    }
+          on:change={async e => {
-                  : {}
+            onChange(
-                : e.detail.map(({ name, value }) => ({
+              {
-                    url: name,
+                detail:
-                    filename: value,
+                  schema.type === FieldType.ATTACHMENT_SINGLE ||
-                  })),
+                  schema.type === FieldType.SIGNATURE_SINGLE
-          },
+                    ? e.detail.length > 0
-          field
+                      ? {
-        )}
+                          url: e.detail[0].name,
-      object={handleAttachmentParams(value[field])}
+                          filename: e.detail[0].value,
-      allowJS
+                        }
-      {bindings}
+                      : {}
-      keyBindings
+                    : e.detail.map(({ name, value }) => ({
-      customButtonText={"Add attachment"}
+                        url: name,
-      keyPlaceholder={"URL"}
+                        filename: value,
-      valuePlaceholder={"Filename"}
+                      })),
-      actionButtonDisabled={(schema.type === FieldType.ATTACHMENT_SINGLE ||
+              },
-        schema.type === FieldType.SIGNATURE) &&
+              field
-        Object.keys(value[field]).length >= 1}
+            )
-    />
+          }}
+          object={handleAttachmentParams(value[field])}
+          allowJS
+          {bindings}
+          keyBindings
+          customButtonText={"Add attachment"}
+          keyPlaceholder={"URL"}
+          valuePlaceholder={"Filename"}
+          actionButtonDisabled={(schema.type === FieldType.ATTACHMENT_SINGLE ||
+            schema.type === FieldType.SIGNATURE) &&
+            Object.keys(value[field]).length >= 1}
+        />
+      </div>
+    {:else}
+      <div class="json-input-spacing">
+        <svelte:component
+          this={isTestModal ? ModalBindableInput : DrawerBindableInput}
+          panel={AutomationBindingPanel}
+          value={value[field]}
+          on:change={e => onChange(e, field)}
+          type="string"
+          bindings={parsedBindings}
+          allowJS={true}
+          updateOnChange={false}
+          title={schema.name}
+        />
+      </div>
+    {/if}
   </div>
 {:else if ["string", "number", "bigint", "barcodeqr", "array"].includes(schema.type)}
   <svelte:component
@@ -156,7 +206,8 @@
 {/if}
 
 <style>
-  .attachment-field-spacinng {
+  .attachment-field-spacing,
+  .json-input-spacing {
     margin-top: var(--spacing-s);
     margin-bottom: var(--spacing-l);
   }

packages/builder/src/components/backend/DataTable/ViewV2DataTable.svelte

@@ -1,6 +1,6 @@
 <script>
   import { viewsV2 } from "stores/builder"
-  import { admin } from "stores/portal"
+  import { admin, licensing } from "stores/portal"
   import { Grid } from "@budibase/frontend-core"
   import { API } from "api"
   import GridCreateEditRowModal from "components/backend/DataTable/modals/grid/GridCreateEditRowModal.svelte"
@@ -28,6 +28,7 @@
     showAvatars={false}
     on:updatedatasource={handleGridViewUpdate}
     isCloud={$admin.cloud}
+    allowViewReadonlyColumns={$licensing.isViewReadonlyColumnsEnabled}
   >
     <svelte:fragment slot="filter">
       <GridFilterButton />

packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte

@@ -17,6 +17,8 @@
     SWITCHABLE_TYPES,
     ValidColumnNameRegex,
     helpers,
+    CONSTANT_INTERNAL_ROW_COLS,
+    CONSTANT_EXTERNAL_ROW_COLS,
   } from "@budibase/shared-core"
   import { createEventDispatcher, getContext, onMount } from "svelte"
   import { cloneDeep } from "lodash/fp"
@@ -52,7 +54,6 @@
   const DATE_TYPE = FieldType.DATETIME
 
   const dispatch = createEventDispatcher()
-  const PROHIBITED_COLUMN_NAMES = ["type", "_id", "_rev", "tableId"]
   const { dispatch: gridDispatch, rows } = getContext("grid")
 
   export let field
@@ -334,7 +335,7 @@
     // Add in defaults and initial definition
     const definition = fieldDefinitions[type?.toUpperCase()]
     if (definition?.constraints) {
-      editableColumn.constraints = definition.constraints
+      editableColumn.constraints = cloneDeep(definition.constraints)
     }
 
     editableColumn.type = definition.type
@@ -487,20 +488,23 @@
       })
     }
     const newError = {}
+    const prohibited = externalTable
+      ? CONSTANT_EXTERNAL_ROW_COLS
+      : CONSTANT_INTERNAL_ROW_COLS
     if (!externalTable && fieldInfo.name?.startsWith("_")) {
       newError.name = `Column name cannot start with an underscore.`
     } else if (fieldInfo.name && !fieldInfo.name.match(ValidColumnNameRegex)) {
       newError.name = `Illegal character; must be alpha-numeric.`
-    } else if (PROHIBITED_COLUMN_NAMES.some(name => fieldInfo.name === name)) {
+    } else if (prohibited.some(name => fieldInfo?.name === name)) {
-      newError.name = `${PROHIBITED_COLUMN_NAMES.join(
+      newError.name = `${prohibited.join(
         ", "
-      )} are not allowed as column names`
+      )} are not allowed as column names - case insensitive.`
     } else if (inUse($tables.selected, fieldInfo.name, originalName)) {
       newError.name = `Column name already in use.`
     }
 
     if (fieldInfo.type === FieldType.AUTO && !fieldInfo.subtype) {
-      newError.subtype = `Auto Column requires a type`
+      newError.subtype = `Auto Column requires a type.`
     }
 
     if (fieldInfo.fieldName && fieldInfo.tableId) {

packages/builder/src/components/backend/DataTable/modals/ExportModal.svelte

@@ -8,7 +8,7 @@
   } from "@budibase/bbui"
   import download from "downloadjs"
   import { API } from "api"
-  import { LuceneUtils } from "@budibase/frontend-core"
+  import { QueryUtils } from "@budibase/frontend-core"
   import { utils } from "@budibase/shared-core"
   import { ROW_EXPORT_FORMATS } from "constants/backend"
 
@@ -49,7 +49,7 @@
     exportFormat = Array.isArray(options) ? options[0]?.key : []
   }
 
-  $: luceneFilter = LuceneUtils.buildLuceneQuery(appliedFilters)
+  $: query = QueryUtils.buildQuery(appliedFilters)
   $: exportOpDisplay = buildExportOpDisplay(
     sorting,
     filterDisplay,
@@ -139,7 +139,7 @@
           tableId: view,
           format: exportFormat,
           search: {
-            query: luceneFilter,
+            query,
             sort: sorting?.sortColumn,
             sortOrder: sorting?.sortOrder,
             paginate: false,

packages/builder/src/components/backend/TableNavigator/ExistingTableDataImport.svelte

@@ -1,9 +1,14 @@
 <script>
-  import { FieldType, BBReferenceFieldSubType } from "@budibase/types"
+  import {
+    FieldType,
+    BBReferenceFieldSubType,
+    SourceName,
+  } from "@budibase/types"
   import { Select, Toggle, Multiselect } from "@budibase/bbui"
   import { DB_TYPE_INTERNAL } from "constants/backend"
   import { API } from "api"
   import { parseFile } from "./utils"
+  import { tables, datasources } from "stores/builder"
 
   let error = null
   let fileName = null
@@ -80,6 +85,9 @@
     schema = fetchSchema(tableId)
   }
 
+  $: table = $tables.list.find(table => table._id === tableId)
+  $: datasource = $datasources.list.find(ds => ds._id === table?.sourceId)
+
   async function fetchSchema(tableId) {
     try {
       const definition = await API.fetchTableDefinition(tableId)
@@ -185,20 +193,25 @@
       </div>
     {/each}
   </div>
-  {#if tableType === DB_TYPE_INTERNAL}
+  <br />
-    <br />
+  <!-- SQL Server doesn't yet support overwriting rows by existing keys -->
+  {#if datasource?.source !== SourceName.SQL_SERVER}
     <Toggle
       bind:value={updateExistingRows}
       on:change={() => (identifierFields = [])}
       thin
       text="Update existing rows"
     />
-    {#if updateExistingRows}
+  {/if}
+  {#if updateExistingRows}
+    {#if tableType === DB_TYPE_INTERNAL}
       <Multiselect
         label="Identifier field(s)"
         options={Object.keys(validation)}
         bind:value={identifierFields}
       />
+    {:else}
+      <p>Rows will be updated based on the table's primary key.</p>
     {/if}
   {/if}
   {#if invalidColumns.length > 0}

packages/builder/src/components/common/Dropzone.svelte

@@ -38,4 +38,5 @@
   {processFiles}
   handleFileTooLarge={$admin.cloud ? handleFileTooLarge : null}
   {fileSizeLimit}
+  on:change
 />

packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/CloseModal.svelte

@@ -0,0 +1,8 @@
+<div class="root">This action doesn't require any settings.</div>
+
+<style>
+  .root {
+    max-width: 400px;
+    margin: 0 auto;
+  }
+</style>

packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/DeleteRow.svelte

@@ -53,6 +53,12 @@
         placeholder="Are you sure you want to delete?"
         bind:value={parameters.confirmText}
       />
+
+      <Label small>Confirm Text</Label>
+      <Input placeholder="Confirm" bind:value={parameters.confirmButtonText} />
+
+      <Label small>Cancel Text</Label>
+      <Input placeholder="Cancel" bind:value={parameters.cancelButtonText} />
     {/if}
   </div>
 </div>

packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/DuplicateRow.svelte

@@ -83,6 +83,12 @@
         placeholder="Are you sure you want to duplicate this row?"
         bind:value={parameters.confirmText}
       />
+
+      <Label small>Confirm Text</Label>
+      <Input placeholder="Confirm" bind:value={parameters.confirmButtonText} />
+
+      <Label small>Cancel Text</Label>
+      <Input placeholder="Cancel" bind:value={parameters.cancelButtonText} />
     {/if}
   </div>
 

packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/ExecuteQuery.svelte

@@ -74,6 +74,18 @@
           placeholder="Are you sure you want to execute this query?"
           bind:value={parameters.confirmText}
         />
+
+        <Input
+          label="Confirm Text"
+          placeholder="Confirm"
+          bind:value={parameters.confirmButtonText}
+        />
+
+        <Input
+          label="Cancel Text"
+          placeholder="Cancel"
+          bind:value={parameters.cancelButtonText}
+        />
       {/if}
 
       {#if query?.parameters?.length > 0}

packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/OpenModal.svelte

@@ -0,0 +1,36 @@
+<script>
+  import { Select, Label } from "@budibase/bbui"
+  import { selectedScreen } from "stores/builder"
+  import { findAllMatchingComponents } from "helpers/components"
+
+  export let parameters
+
+  $: modalOptions = getModalOptions($selectedScreen)
+
+  const getModalOptions = screen => {
+    const modalComponents = findAllMatchingComponents(screen.props, component =>
+      component._component.endsWith("/modal")
+    )
+    return modalComponents.map(modal => ({
+      label: modal._instanceName,
+      value: modal._id,
+    }))
+  }
+</script>
+
+<div class="root">
+  <Label small>Modal</Label>
+  <Select bind:value={parameters.id} options={modalOptions} />
+</div>
+
+<style>
+  .root {
+    display: grid;
+    column-gap: var(--spacing-l);
+    row-gap: var(--spacing-s);
+    grid-template-columns: 60px 1fr;
+    align-items: center;
+    max-width: 400px;
+    margin: 0 auto;
+  }
+</style>

packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/PromptUser.svelte

@@ -18,7 +18,7 @@
   <div class="params">
     <Label small>Title</Label>
     <DrawerBindableInput
-      placeholder="Title"
+      placeholder="Prompt User"
       value={parameters.customTitleText}
       on:change={e => (parameters.customTitleText = e.detail)}
       {bindings}
@@ -30,6 +30,22 @@
       on:change={e => (parameters.confirmText = e.detail)}
       {bindings}
     />
+
+    <Label small>Confirm Text</Label>
+    <DrawerBindableInput
+      placeholder="Confirm"
+      value={parameters.confirmButtonText}
+      on:change={e => (parameters.confirmButtonText = e.detail)}
+      {bindings}
+    />
+
+    <Label small>Cancel Text</Label>
+    <DrawerBindableInput
+      placeholder="Cancel"
+      value={parameters.cancelButtonText}
+      on:change={e => (parameters.cancelButtonText = e.detail)}
+      {bindings}
+    />
   </div>
 </div>
 

packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/SaveRow.svelte

@@ -80,6 +80,12 @@
         placeholder="Are you sure you want to save this row?"
         bind:value={parameters.confirmText}
       />
+
+      <Label small>Confirm Text</Label>
+      <Input placeholder="Confirm" bind:value={parameters.confirmButtonText} />
+
+      <Label small>Cancel Text</Label>
+      <Input placeholder="Cancel" bind:value={parameters.cancelButtonText} />
     {/if}
   </div>
 

packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/ShowNotification.svelte

@@ -25,6 +25,8 @@
     },
   ]
 
+  const MAX_DURATION = 120000 // Maximum duration in milliseconds (2 minutes)
+
   onMount(() => {
     if (!parameters.type) {
       parameters.type = "success"
@@ -33,6 +35,14 @@
       parameters.autoDismiss = true
     }
   })
+
+  function handleDurationChange(event) {
+    let newDuration = event.detail
+    if (newDuration > MAX_DURATION) {
+      newDuration = MAX_DURATION
+    }
+    parameters.duration = newDuration
+  }
 </script>
 
 <div class="root">
@@ -47,6 +57,16 @@
   />
   <Label />
   <Checkbox text="Auto dismiss" bind:value={parameters.autoDismiss} />
+  {#if parameters.autoDismiss}
+    <Label>Duration (ms)</Label>
+    <DrawerBindableInput
+      title="Duration"
+      {bindings}
+      value={parameters.duration}
+      placeholder="3000"
+      on:change={handleDurationChange}
+    />
+  {/if}
 </div>
 
 <style>

packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/index.js

@@ -21,5 +21,7 @@ export { default as ShowNotification } from "./ShowNotification.svelte"
 export { default as PromptUser } from "./PromptUser.svelte"
 export { default as OpenSidePanel } from "./OpenSidePanel.svelte"
 export { default as CloseSidePanel } from "./CloseSidePanel.svelte"
+export { default as OpenModal } from "./OpenModal.svelte"
+export { default as CloseModal } from "./CloseModal.svelte"
 export { default as ClearRowSelection } from "./ClearRowSelection.svelte"
 export { default as DownloadFile } from "./DownloadFile.svelte"

packages/builder/src/components/design/settings/controls/ButtonActionEditor/manifest.json

@@ -157,6 +157,18 @@
       "component": "CloseSidePanel",
       "dependsOnFeature": "sidePanel"
     },
+    {
+      "name": "Open Modal",
+      "type": "application",
+      "component": "OpenModal",
+      "dependsOnFeature": "modal"
+    },
+    {
+      "name": "Close Modal",
+      "type": "application",
+      "component": "CloseModal",
+      "dependsOnFeature": "modal"
+    },
     {
       "name": "Clear Row Selection",
       "type": "data",

packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceCategory.svelte

@@ -29,7 +29,7 @@
       on:click={() => onSelect(data)}
     >
       <span class="spectrum-Menu-itemLabel">
-        {data.label}
+        {data.datasource?.name ? `${data.datasource.name} - ` : ""}{data.label}
       </span>
       <svg
         class="spectrum-Icon spectrum-UIIcon-Checkmark100 spectrum-Menu-checkmark spectrum-Menu-itemIcon"

packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceSelect.svelte

@@ -55,6 +55,9 @@
     label: m.name,
     tableId: m._id,
     type: "table",
+    datasource: $datasources.list.find(
+      ds => ds._id === m.sourceId || m.datasourceId
+    ),
   }))
   $: viewsV1 = $viewsStore.list.map(view => ({
     ...view,

packages/builder/src/components/design/settings/controls/Explanation/Explanation.svelte

@@ -18,14 +18,11 @@
   import subjects from "./subjects"
   import { appStore } from "stores/builder"
 
-  export let explanation
-  export let columnIcon
-  export let columnType
-  export let columnName
-
   export let tableHref = () => {}
-
   export let schema
+  export let name
+  export let explanation
+  export let componentName
 
   $: explanationWithPresets = getExplanationWithPresets(
     explanation,
@@ -54,14 +51,8 @@
 </script>
 
 <div bind:this={root} class="tooltipContents">
-  <Column
+  <Column {name} {schema} {tableHref} {setExplanationSubject} />
-    {columnName}
+  <Support {componentName} {support} {setExplanationSubject} />
-    {columnIcon}
-    {columnType}
-    {tableHref}
-    {setExplanationSubject}
-  />
-  <Support {support} {setExplanationSubject} />
   {#if messages.includes(messageConstants.stringAsNumber)}
     <StringAsNumber {setExplanationSubject} />
   {/if}
@@ -84,7 +75,7 @@
 
 {#if detailsModalSubject !== subjects.none}
   <DetailsModal
-    {columnName}
+    columnName={name}
     anchor={root}
     {schema}
     subject={detailsModalSubject}

packages/builder/src/components/design/settings/controls/Explanation/lines/Column.svelte

@@ -1,69 +1,124 @@
 <script>
-  import {
+  import { Line, InfoWord, DocumentationLink, Text } from "../typography"
-    Line,
+  import { FieldType } from "@budibase/types"
-    InfoWord,
+  import { FIELDS } from "constants/backend"
-    DocumentationLink,
-    Text,
-    Period,
-  } from "../typography"
   import subjects from "../subjects"
 
-  export let columnName
+  export let schema
-  export let columnIcon
+  export let name
-  export let columnType
   export let tableHref
   export let setExplanationSubject
 
+  const getTypeName = schema => {
+    const fieldDefinition = Object.values(FIELDS).find(
+      f => f.type === schema?.type
+    )
+
+    if (schema?.type === "jsonarray") {
+      return "JSON Array"
+    }
+    if (schema?.type === "options") {
+      return "Options"
+    }
+
+    return fieldDefinition?.name || schema?.type || "Unknown"
+  }
+
+  const getTypeIcon = schema => {
+    const fieldDefinition = Object.values(FIELDS).find(
+      f => f.type === schema?.type
+    )
+
+    if (schema?.type === "jsonarray") {
+      return "BracketsSquare"
+    }
+
+    return fieldDefinition?.icon || "Circle"
+  }
+
   const getDocLink = columnType => {
-    if (columnType === "Number") {
+    if (columnType === FieldType.NUMBER) {
       return "https://docs.budibase.com/docs/number"
     }
-    if (columnType === "Text") {
+    if (columnType === FieldType.STRING) {
       return "https://docs.budibase.com/docs/text"
     }
-    if (columnType === "Attachment") {
+    if (columnType === FieldType.LONGFORM) {
+      return "https://docs.budibase.com/docs/text"
+    }
+    if (columnType === FieldType.ATTACHMENT_SINGLE) {
       return "https://docs.budibase.com/docs/attachments"
     }
-    if (columnType === "Multi-select") {
+    if (columnType === FieldType.ATTACHMENTS) {
+      // No distinct multi attachment docs, link to attachment instead
+      return "https://docs.budibase.com/docs/attachments"
+    }
+    if (columnType === FieldType.ARRAY) {
       return "https://docs.budibase.com/docs/multi-select"
     }
-    if (columnType === "JSON") {
+    if (columnType === FieldType.JSON) {
       return "https://docs.budibase.com/docs/json"
     }
-    if (columnType === "Date/Time") {
+    if (columnType === "jsonarray") {
+      return "https://docs.budibase.com/docs/json"
+    }
+    if (columnType === FieldType.DATETIME) {
       return "https://docs.budibase.com/docs/datetime"
     }
-    if (columnType === "User") {
+    if (columnType === FieldType.BB_REFERENCE_SINGLE) {
-      return "https://docs.budibase.com/docs/user"
+      return "https://docs.budibase.com/docs/users-1"
     }
-    if (columnType === "QR") {
+    if (columnType === FieldType.BB_REFERENCE) {
+      return "https://docs.budibase.com/docs/users-1"
+    }
+    if (columnType === FieldType.BARCODEQR) {
       return "https://docs.budibase.com/docs/barcodeqr"
     }
-    if (columnType === "Relationship") {
+    if (columnType === FieldType.LINK) {
       return "https://docs.budibase.com/docs/relationships"
     }
-    if (columnType === "Formula") {
+    if (columnType === FieldType.FORMULA) {
       return "https://docs.budibase.com/docs/formula"
     }
-    if (columnType === "Options") {
+    if (columnType === FieldType.OPTIONS) {
       return "https://docs.budibase.com/docs/options"
     }
-    if (columnType === "BigInt") {
+    if (columnType === FieldType.BOOLEAN) {
-      // No BigInt docs
-      return null
-    }
-    if (columnType === "Boolean") {
       return "https://docs.budibase.com/docs/boolean-truefalse"
     }
-    if (columnType === "Signature") {
+    if (columnType === FieldType.SIGNATURE_SINGLE) {
       // No Signature docs
       return null
     }
+    if (columnType === FieldType.BIGINT) {
+      // No BigInt docs
+      return null
+    }
 
     return null
   }
 
-  $: docLink = getDocLink(columnType)
+  // NOTE The correct indefinite article is based on the pronounciation of the word it precedes, not the spelling. So simply checking if the word begins with a vowel is not sufficient.
+
+  // e.g., `an honor`, `a user`
+  const getIndefiniteArticle = schema => {
+    const anTypes = [
+      FieldType.OPTIONS,
+      null, // `null` gets parsed as "unknown"
+      undefined, // `undefined` gets parsed as "unknown"
+    ]
+
+    if (anTypes.includes(schema?.type)) {
+      return "an"
+    }
+
+    return "a"
+  }
+
+  $: columnTypeName = getTypeName(schema)
+  $: columnIcon = getTypeIcon(schema)
+  $: docLink = getDocLink(schema?.type)
+  $: indefiniteArticle = getIndefiniteArticle(schema)
 </script>
 
 <Line noWrap>
@@ -71,14 +126,14 @@
     on:mouseenter={() => setExplanationSubject(subjects.column)}
     on:mouseleave={() => setExplanationSubject(subjects.none)}
     href={tableHref}
-    text={columnName}
+    text={name}
   />
-  <Text value=" is a " />
+  <Text value={` is ${indefiniteArticle} `} />
   <DocumentationLink
     disabled={docLink === null}
     href={docLink}
     icon={columnIcon}
-    text={`${columnType} column`}
+    text={columnTypeName}
   />
-  <Period />
+  <Text value=" column." />
 </Line>

packages/builder/src/components/design/settings/controls/Explanation/lines/Support.svelte

@@ -2,9 +2,16 @@
   import { Line, InfoWord, DocumentationLink, Text } from "../typography"
   import subjects from "../subjects"
   import * as explanation from "../explanation"
+  import { componentStore } from "stores/builder"
 
   export let setExplanationSubject
   export let support
+  export let componentName
+
+  const getComponentDefinition = componentName => {
+    const components = $componentStore.components || {}
+    return components[componentName] || null
+  }
 
   const getIcon = support => {
     if (support === explanation.support.unsupported) {
@@ -39,21 +46,24 @@
   $: icon = getIcon(support)
   $: color = getColor(support)
   $: text = getText(support)
+  $: componentDefinition = getComponentDefinition(componentName)
 </script>
 
-<Line>
+{#if componentDefinition}
-  <InfoWord
+  <Line>
-    on:mouseenter={() => setExplanationSubject(subjects.support)}
+    <InfoWord
-    on:mouseleave={() => setExplanationSubject(subjects.none)}
+      on:mouseenter={() => setExplanationSubject(subjects.support)}
-    {icon}
+      on:mouseleave={() => setExplanationSubject(subjects.none)}
-    {color}
+      {icon}
-    {text}
+      {color}
-  />
+      {text}
-  <Text value=" with this " />
+    />
-  <DocumentationLink
+    <Text value=" with this " />
-    href="https://docs.budibase.com/docs/chart"
+    <DocumentationLink
-    icon="GraphPie"
+      href={componentDefinition.documentationLink}
-    text="Chart component"
+      icon={componentDefinition.icon}
-  />
+      text={componentDefinition.name}
-  <Text value=" input." />
+    />
-</Line>
+    <Text value=" input." />
+  </Line>
+{/if}

packages/builder/src/components/design/settings/controls/FieldSelect.svelte

@@ -6,8 +6,6 @@
   import { Explanation } from "./Explanation"
   import { debounce } from "lodash"
   import { params } from "@roxi/routify"
-  import { Constants } from "@budibase/frontend-core"
-  import { FIELDS } from "constants/backend"
 
   export let componentInstance = {}
   export let value = ""
@@ -60,35 +58,6 @@
   const onOptionMouseleave = e => {
     updateTooltip(e, null)
   }
-  const getOptionIcon = optionKey => {
-    const option = schema[optionKey]
-    if (!option) return ""
-
-    if (option.autocolumn) {
-      return "MagicWand"
-    }
-    const { type, subtype } = option
-
-    const result =
-      typeof Constants.TypeIconMap[type] === "object" && subtype
-        ? Constants.TypeIconMap[type][subtype]
-        : Constants.TypeIconMap[type]
-
-    return result || "Text"
-  }
-
-  const getOptionIconTooltip = optionKey => {
-    const option = schema[optionKey]
-
-    const type = option?.type
-    const field = Object.values(FIELDS).find(f => f.type === type)
-
-    if (field) {
-      return field.name
-    }
-
-    return ""
-  }
 </script>
 
 <Select
@@ -109,10 +78,9 @@
     <Explanation
       tableHref={`/builder/app/${$params.application}/data/table/${datasource?.tableId}`}
       schema={schema[currentOption]}
-      columnIcon={getOptionIcon(currentOption)}
+      name={currentOption}
-      columnName={currentOption}
-      columnType={getOptionIconTooltip(currentOption)}
       {explanation}
+      componentName={componentInstance._component}
     />
   </ContextTooltip>
 {/if}

packages/builder/src/components/design/settings/controls/FilterEditor/FilterEditor.svelte

@@ -9,7 +9,8 @@
   import { createEventDispatcher } from "svelte"
   import { getDatasourceForProvider, getSchemaForDatasource } from "dataBinding"
   import FilterBuilder from "./FilterBuilder.svelte"
-  import { selectedScreen } from "stores/builder"
+  import { tables, selectedScreen } from "stores/builder"
+  import { search } from "@budibase/frontend-core"
 
   const dispatch = createEventDispatcher()
 
@@ -23,7 +24,11 @@
   $: tempValue = value
   $: datasource = getDatasourceForProvider($selectedScreen, componentInstance)
   $: dsSchema = getSchemaForDatasource($selectedScreen, datasource)?.schema
-  $: schemaFields = Object.values(schema || dsSchema || {})
+  $: schemaFields = search.getFields(
+    $tables.list,
+    Object.values(schema || dsSchema || {}),
+    { allowLinks: true }
+  )
   $: text = getText(value?.filter(filter => filter.field))
 
   async function saveFilter() {

packages/builder/src/components/design/settings/controls/MultiFieldSelect.svelte

@@ -4,10 +4,8 @@
   import { selectedScreen } from "stores/builder"
   import { createEventDispatcher } from "svelte"
   import { Explanation } from "./Explanation"
-  import { FIELDS } from "constants/backend"
   import { params } from "@roxi/routify"
   import { debounce } from "lodash"
-  import { Constants } from "@budibase/frontend-core"
 
   export let componentInstance = {}
   export let value = ""
@@ -37,40 +35,6 @@
     dispatch("change", boundValue)
   }
 
-  const getOptionIcon = optionKey => {
-    const option = schema[optionKey]
-    if (!option) return ""
-
-    if (option.autocolumn) {
-      return "MagicWand"
-    }
-    const { type, subtype } = option
-
-    const result =
-      typeof Constants.TypeIconMap[type] === "object" && subtype
-        ? Constants.TypeIconMap[type][subtype]
-        : Constants.TypeIconMap[type]
-
-    return result || "Text"
-  }
-
-  const getOptionIconTooltip = optionKey => {
-    const option = schema[optionKey]
-
-    const type = option?.type
-    const field = Object.values(FIELDS).find(f => f.type === type)
-
-    if (field) {
-      return field.name
-    } else if (type === "jsonarray") {
-      // `jsonarray` isn't present in the above FIELDS constant
-
-      return "JSON Array"
-    }
-
-    return ""
-  }
-
   const updateTooltip = debounce((e, option) => {
     if (option == null) {
       contextTooltipVisible = false
@@ -110,10 +74,9 @@
     <Explanation
       tableHref={`/builder/app/${$params.application}/data/table/${datasource?.tableId}`}
       schema={schema[currentOption]}
-      columnIcon={getOptionIcon(currentOption)}
+      name={currentOption}
-      columnName={currentOption}
-      columnType={getOptionIconTooltip(currentOption)}
       {explanation}
+      componentName={componentInstance._component}
     />
   </ContextTooltip>
 {/if}

packages/builder/src/components/integration/RestQueryViewer.svelte

@@ -233,9 +233,9 @@
         response.info = response.info || { code: 200 }
         // if existing schema, copy over what it is
         if (schema) {
-          for (let [name, field] of Object.entries(schema)) {
+          for (let [name, field] of Object.entries(response.schema)) {
-            if (response.schema[name]) {
+            if (!schema[name]) {
-              response.schema[name] = field
+              schema[name] = field
             }
           }
         }

packages/builder/src/dataBinding.js

@@ -30,6 +30,7 @@ import ActionDefinitions from "components/design/settings/controls/ButtonActionE
 import { environment, licensing } from "stores/portal"
 import { convertOldFieldFormat } from "components/design/settings/controls/FieldConfiguration/utils"
 import { FIELDS } from "constants/backend"
+import { FieldType } from "@budibase/types"
 
 const { ContextScopes } = Constants
 
@@ -555,6 +556,9 @@ const getComponentBindingCategory = (component, context, def) => {
 export const getUserBindings = () => {
   let bindings = []
   const { schema } = getSchemaForDatasourcePlus(TableNames.USERS)
+  // add props that are not in the user metadata table schema
+  // but will be there for logged-in user
+  schema["globalId"] = { type: FieldType.STRING }
   const keys = Object.keys(schema).sort()
   const safeUser = makePropSafe("user")
 
@@ -728,7 +732,7 @@ const getRoleBindings = () => {
   return (get(rolesStore) || []).map(role => {
     return {
       type: "context",
-      runtimeBinding: `trim "${role._id}"`,
+      runtimeBinding: `'${role._id}'`,
       readableBinding: `Role.${role.name}`,
       category: "Role",
       icon: "UserGroup",

packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ComponentSettingsPanel.svelte

@@ -70,7 +70,7 @@
         <input
           class="input"
           value={title}
-          {title}
+          title={componentName}
           placeholder={componentName}
           on:keypress={e => {
             if (e.key.toLowerCase() === "enter") {
@@ -158,7 +158,32 @@
     overflow: hidden;
     text-overflow: ellipsis;
     white-space: nowrap;
+    position: relative;
+    padding: 5px;
+    right: 6px;
+    border: 1px solid transparent;
+    border-radius: 3px;
+    transition: 150ms background-color, 150ms border-color, 150ms color;
   }
+
+  .input:hover,
+  .input:focus {
+    cursor: text;
+    background-color: var(
+      --spectrum-textfield-m-background-color,
+      var(--spectrum-global-color-gray-50)
+    );
+    border: 1px solid white;
+    border-color: var(
+      --spectrum-textfield-m-border-color,
+      var(--spectrum-alias-border-color)
+    );
+    color: var(
+      --spectrum-textfield-m-text-color,
+      var(--spectrum-alias-text-color)
+    );
+  }
+
   .panel-title-content {
     display: contents;
   }

packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ConditionalUIDrawer.svelte

@@ -12,7 +12,7 @@
   import { dndzone } from "svelte-dnd-action"
   import { generate } from "shortid"
   import DrawerBindableInput from "components/common/bindings/DrawerBindableInput.svelte"
-  import { LuceneUtils, Constants } from "@budibase/frontend-core"
+  import { QueryUtils, Constants } from "@budibase/frontend-core"
   import { selectedComponent, componentStore } from "stores/builder"
   import { getComponentForSetting } from "components/design/settings/componentSettings"
   import PropertyControl from "components/design/settings/controls/PropertyControl.svelte"
@@ -119,7 +119,7 @@
   }
 
   const getOperatorOptions = condition => {
-    return LuceneUtils.getValidOperatorsForType({ type: condition.valueType })
+    return QueryUtils.getValidOperatorsForType({ type: condition.valueType })
   }
 
   const onOperatorChange = (condition, newOperator) => {
@@ -138,7 +138,7 @@
     condition.referenceValue = null
 
     // Ensure a valid operator is set
-    const validOperators = LuceneUtils.getValidOperatorsForType({
+    const validOperators = QueryUtils.getValidOperatorsForType({
       type: newType,
     }).map(x => x.value)
     if (!validOperators.includes(condition.operator)) {

packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/new/_components/NewComponentPanel.svelte

@@ -59,7 +59,14 @@
     // Build up list of illegal children from ancestors
     let illegalChildren = definition.illegalChildren || []
     path.forEach(ancestor => {
-      if (ancestor._component === `@budibase/standard-components/sidepanel`) {
+      // Sidepanels and modals can be nested anywhere in the component tree, but really they are always rendered at the top level.
+      // Because of this, it doesn't make sense to carry over any parent illegal children to them, so the array is reset here.
+      if (
+        [
+          "@budibase/standard-components/sidepanel",
+          "@budibase/standard-components/modal",
+        ].includes(ancestor._component)
+      ) {
         illegalChildren = []
       }
       const def = componentStore.getDefinition(ancestor._component)

packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/new/_components/componentStructure.json

@@ -14,7 +14,7 @@
   {
     "name": "Layout",
     "icon": "ClassicGridView",
-    "children": ["container", "section", "sidepanel"]
+    "children": ["container", "section", "sidepanel", "modal"]
   },
   {
     "name": "Data",

packages/builder/src/pages/builder/maintenance/index.svelte

@@ -33,7 +33,8 @@
             </Body>
           </Layout>
           <Button
-            on:click={() => (window.location = "https://docs.budibase.com")}
+            on:click={() =>
+              (window.location = "https://docs.budibase.com/docs/migrations")}
             >Migration guide</Button
           >
         {/if}

packages/builder/src/pages/builder/portal/users/users/index.svelte

@@ -60,6 +60,7 @@
     userLimitReachedModal
   let searchEmail = undefined
   let selectedRows = []
+  let selectedInvites = []
   let bulkSaveResponse
   let customRenderers = [
     { column: "email", component: EmailTableRenderer },
@@ -123,7 +124,7 @@
       return {}
     }
     let pendingSchema = JSON.parse(JSON.stringify(tblSchema))
-    pendingSchema.email.displayName = "Pending Invites"
+    pendingSchema.email.displayName = "Pending Users"
     return pendingSchema
   }
 
@@ -132,6 +133,7 @@
       const { admin, builder, userGroups, apps } = invite.info
 
       return {
+        _id: invite.code,
         email: invite.email,
         builder,
         admin,
@@ -260,9 +262,26 @@
         return
       }
 
-      await users.bulkDelete(ids)
+      if (ids.length > 0) {
-      notifications.success(`Successfully deleted ${selectedRows.length} rows`)
+        await users.bulkDelete(ids)
+      }
+
+      if (selectedInvites.length > 0) {
+        await users.removeInvites(
+          selectedInvites.map(invite => ({
+            code: invite._id,
+          }))
+        )
+        pendingInvites = await users.getInvites()
+      }
+
+      notifications.success(
+        `Successfully deleted ${
+          selectedRows.length + selectedInvites.length
+        } users`
+      )
       selectedRows = []
+      selectedInvites = []
       await fetch.refresh()
     } catch (error) {
       notifications.error("Error deleting users")
@@ -328,15 +347,15 @@
       </div>
     {/if}
     <div class="controls-right">
-      <Search bind:value={searchEmail} placeholder="Search" />
+      {#if selectedRows.length > 0 || selectedInvites.length > 0}
-      {#if selectedRows.length > 0}
         <DeleteRowsButton
           item="user"
           on:updaterows
-          {selectedRows}
+          selectedRows={[...selectedRows, ...selectedInvites]}
           deleteRows={deleteUsers}
         />
       {/if}
+      <Search bind:value={searchEmail} placeholder="Search" />
     </div>
   </div>
   <Table
@@ -362,10 +381,12 @@
   </div>
 
   <Table
+    bind:selectedRows={selectedInvites}
     schema={pendingSchema}
     data={parsedInvites}
     allowEditColumns={false}
     allowEditRows={false}
+    allowSelectRows={!readonly}
     {customRenderers}
     loading={!invitesLoaded}
     allowClickRows={false}

packages/builder/src/stores/builder/screens.js

@@ -125,7 +125,14 @@ export class ScreenStore extends BudiStore {
         return
       }
 
-      if (type === "@budibase/standard-components/sidepanel") {
+      // Sidepanels and modals can be nested anywhere in the component tree, but really they are always rendered at the top level.
+      // Because of this, it doesn't make sense to carry over any parent illegal children to them, so the array is reset here.
+      if (
+        [
+          "@budibase/standard-components/sidepanel",
+          "@budibase/standard-components/modal",
+        ].includes(type)
+      ) {
         illegalChildren = []
       }
 

packages/builder/src/stores/portal/licensing.js

@@ -138,6 +138,11 @@ export const createLicensingStore = () => {
       const isViewPermissionsEnabled = license.features.includes(
         Constants.Features.VIEW_PERMISSIONS
       )
+
+      const isViewReadonlyColumnsEnabled = license.features.includes(
+        Constants.Features.VIEW_READONLY_COLUMNS
+      )
+
       store.update(state => {
         return {
           ...state,
@@ -157,6 +162,7 @@ export const createLicensingStore = () => {
           triggerAutomationRunEnabled,
           isViewPermissionsEnabled,
           perAppBuildersEnabled,
+          isViewReadonlyColumnsEnabled,
         }
       })
     },

packages/builder/src/stores/portal/users.js

@@ -38,6 +38,10 @@ export function createUsersStore() {
     return API.inviteUsers(payload)
   }
 
+  async function removeInvites(payload) {
+    return API.removeUserInvites(payload)
+  }
+
   async function acceptInvite(inviteCode, password, firstName, lastName) {
     return API.acceptInvite({
       inviteCode,
@@ -154,6 +158,7 @@ export function createUsersStore() {
     onboard,
     fetchInvite,
     getInvites,
+    removeInvites,
     updateInvite,
     getUserCountByApp,
     addAppBuilder,

packages/cli/package.json

@@ -11,7 +11,7 @@
   "scripts": {
     "tsc": "node ../../scripts/build.js",
     "build": "yarn tsc",
-    "check:types": "tsc -p tsconfig.json --noEmit --paths null",
+    "check:types": "tsc -p tsconfig.json --noEmit --paths null --target es2020",
     "start": "ts-node ./src/index.ts"
   },
   "dependencies": {
@@ -32,7 +32,7 @@
     "pouchdb": "7.3.0",
     "pouchdb-replication-stream": "1.2.9",
     "randomstring": "1.1.5",
-    "tar": "6.1.15",
+    "tar": "6.2.1",
     "yaml": "^2.1.1"
   },
   "devDependencies": {
@@ -40,6 +40,6 @@
     "@types/node-fetch": "2.6.4",
     "@types/pouchdb": "^6.4.0",
     "ts-node": "10.8.1",
-    "typescript": "5.2.2"
+    "typescript": "5.5.2"
   }
 }

packages/client/manifest.json

@@ -11,6 +11,7 @@
     "continueIfAction": true,
     "showNotificationAction": true,
     "sidePanel": true,
+    "modal": true,
     "skeletonLoader": true
   },
   "typeSupportPresets": {
@@ -22,17 +23,21 @@
         { "type": "bigint", "message": "stringAsNumber" },
         { "type": "options", "message": "stringAsNumber" },
         { "type": "formula", "message": "stringAsNumber" },
-        { "type": "datetime", "message": "dateAsNumber"}
+        { "type": "datetime", "message": "dateAsNumber" }
       ],
-      "unsupported": [
+      "unsupported": [{ "type": "json", "message": "jsonPrimitivesOnly" }]
-        { "type": "json", "message": "jsonPrimitivesOnly" }
-      ]
     },
     "stringLike": {
-      "supported": ["string", "number", "bigint", "options", "longform", "boolean", "datetime"],
+      "supported": [
-      "unsupported": [
+        "string",
-        { "type": "json", "message": "jsonPrimitivesOnly" }
+        "number",
-      ]
+        "bigint",
+        "options",
+        "longform",
+        "boolean",
+        "datetime"
+      ],
+      "unsupported": [{ "type": "json", "message": "jsonPrimitivesOnly" }]
     },
     "datetimeLike": {
       "supported": ["datetime"],
@@ -42,11 +47,9 @@
         { "type": "options", "message": "stringAsDate" },
         { "type": "formula", "message": "stringAsDate" },
         { "type": "bigint", "message": "stringAsDate" },
-        { "type": "number", "message": "numberAsDate"}
+        { "type": "number", "message": "numberAsDate" }
       ],
-      "unsupported": [
+      "unsupported": [{ "type": "json", "message": "jsonPrimitivesOnly" }]
-        { "type": "json", "message": "jsonPrimitivesOnly" }
-      ]
     }
   },
   "layout": {
@@ -5223,6 +5226,7 @@
     ]
   },
   "chartblock": {
+    "documentationLink": "https://docs.budibase.com/docs/chart",
     "block": true,
     "name": "Chart Block",
     "icon": "GraphPie",
@@ -6974,7 +6978,7 @@
     "name": "Side Panel",
     "icon": "RailRight",
     "hasChildren": true,
-    "illegalChildren": ["section", "sidepanel"],
+    "illegalChildren": ["section", "sidepanel", "modal"],
     "showEmptyState": false,
     "draggable": false,
     "info": "Side panels are hidden by default. They will only be revealed when triggered by the 'Open Side Panel' action.",
@@ -6992,6 +6996,52 @@
       }
     ]
   },
+  "modal": {
+    "name": "Modal",
+    "icon": "MBox",
+    "hasChildren": true,
+    "illegalChildren": ["section", "modal", "sidepanel"],
+    "showEmptyState": false,
+    "draggable": false,
+    "info": "Modals are hidden by default. They will only be revealed when triggered by the 'Open Modal' action.",
+    "settings": [
+      {
+        "type": "boolean",
+        "key": "ignoreClicksOutside",
+        "label": "Ignore clicks outside",
+        "defaultValue": false
+      },
+      {
+        "type": "event",
+        "key": "onClose",
+        "label": "On close"
+      },
+      {
+        "type": "select",
+        "label": "Size",
+        "key": "size",
+        "defaultValue": "small",
+        "options": [
+          {
+            "label": "Small",
+            "value": "small"
+          },
+          {
+            "label": "Medium",
+            "value": "medium"
+          },
+          {
+            "label": "Large",
+            "value": "large"
+          },
+          {
+            "label": "Fullscreen",
+            "value": "fullscreen"
+          }
+        ]
+      }
+    ]
+  },
   "rowexplorer": {
     "block": true,
     "name": "Row Explorer Block",

packages/client/src/components/ClientApp.svelte

@@ -19,6 +19,8 @@
     devToolsStore,
     devToolsEnabled,
     environmentStore,
+    sidePanelStore,
+    modalStore,
   } from "stores"
   import NotificationDisplay from "components/overlay/NotificationDisplay.svelte"
   import ConfirmationDisplay from "components/overlay/ConfirmationDisplay.svelte"
@@ -102,6 +104,21 @@
         embedded: !!$appStore.embedded,
       })
     }
+    const handleHashChange = () => {
+      const { open: sidePanelOpen } = $sidePanelStore
+      if (sidePanelOpen) {
+        sidePanelStore.actions.close()
+      }
+
+      const { open: modalOpen } = $modalStore
+      if (modalOpen) {
+        modalStore.actions.close()
+      }
+    }
+    window.addEventListener("hashchange", handleHashChange)
+    return () => {
+      window.removeEventListener("hashchange", handleHashChange)
+    }
   })
 
   $: {

packages/client/src/components/app/Button.svelte

@@ -5,8 +5,6 @@
   const { styleable, builderStore } = getContext("sdk")
   const component = getContext("component")
 
-  let handlingOnClick = false
-
   export let disabled = false
   export let text = ""
   export let onClick
@@ -19,17 +17,9 @@
   // For internal use only for now - not defined in the manifest
   export let active = false
 
-  const handleOnClick = async () => {
-    handlingOnClick = true
-
-    if (onClick) {
-      await onClick()
-    }
-
-    handlingOnClick = false
-  }
-
   let node
+  let touched = false
+  let handlingOnClick = false
 
   $: $component.editing && node?.focus()
   $: componentText = getComponentText(text, $builderStore, $component)
@@ -42,7 +32,18 @@
   }
 
   const updateText = e => {
-    builderStore.actions.updateProp("text", e.target.textContent)
+    if (touched) {
+      builderStore.actions.updateProp("text", e.target.textContent)
+    }
+    touched = false
+  }
+
+  const handleOnClick = async () => {
+    handlingOnClick = true
+    if (onClick) {
+      await onClick()
+    }
+    handlingOnClick = false
   }
 </script>
 
@@ -57,6 +58,7 @@
     on:blur={$component.editing ? updateText : null}
     bind:this={node}
     class:active
+    on:input={() => (touched = true)}
   >
     {#if icon}
       <i class="{icon} {size}" />

packages/client/src/components/app/DataProvider.svelte

@@ -1,7 +1,7 @@
 <script>
   import { getContext } from "svelte"
   import { Pagination, ProgressCircle } from "@budibase/bbui"
-  import { fetchData, LuceneUtils } from "@budibase/frontend-core"
+  import { fetchData, QueryUtils } from "@budibase/frontend-core"
 
   export let dataSource
   export let filter
@@ -19,7 +19,7 @@
 
   // We need to manage our lucene query manually as we want to allow components
   // to extend it
-  $: defaultQuery = LuceneUtils.buildLuceneQuery(filter)
+  $: defaultQuery = QueryUtils.buildQuery(filter)
   $: query = extendQuery(defaultQuery, queryExtensions)
   $: fetch = createFetch(dataSource)
   $: fetch.update({

packages/client/src/components/app/GridBlock.svelte

@@ -90,9 +90,11 @@
     columns.forEach((column, idx) => {
       overrides[column.field] = {
         displayName: column.label,
-        width: column.width,
         order: idx,
       }
+      if (column.width) {
+        overrides[column.field].width = column.width
+      }
     })
     return overrides
   }

packages/client/src/components/app/Heading.svelte

@@ -14,6 +14,7 @@
   export let size
 
   let node
+  let touched = false
 
   $: $component.editing && node?.focus()
   $: placeholder = $builderStore.inBuilder && !text && !$component.editing
@@ -47,7 +48,10 @@
 
   // Convert contenteditable HTML to text and save
   const updateText = e => {
-    builderStore.actions.updateProp("text", e.target.textContent)
+    if (touched) {
+      builderStore.actions.updateProp("text", e.target.textContent)
+    }
+    touched = false
   }
 </script>
 
@@ -62,6 +66,7 @@
     class:underline
     class="spectrum-Heading {sizeClass} {alignClass}"
     on:blur={$component.editing ? updateText : null}
+    on:input={() => (touched = true)}
   >
     {componentText}
   </h1>

packages/client/src/components/app/Layout.svelte

@@ -12,6 +12,7 @@
     linkable,
     builderStore,
     sidePanelStore,
+    modalStore,
     appStore,
   } = sdk
   const context = getContext("context")
@@ -77,6 +78,7 @@
     !$builderStore.inBuilder &&
     $sidePanelStore.open &&
     !$sidePanelStore.ignoreClicksOutside
+
   $: screenId = $builderStore.inBuilder
     ? `${$builderStore.screen?._id}-screen`
     : "screen"
@@ -198,6 +200,7 @@
   const handleClickLink = () => {
     mobileOpen = false
     sidePanelStore.actions.close()
+    modalStore.actions.close()
   }
 </script>
 

packages/client/src/components/app/Link.svelte

@@ -1,7 +1,7 @@
 <script>
   import { getContext } from "svelte"
 
-  const { linkable, styleable, builderStore, sidePanelStore } =
+  const { linkable, styleable, builderStore, sidePanelStore, modalStore } =
     getContext("sdk")
   const component = getContext("component")
 
@@ -16,6 +16,7 @@
   export let size
 
   let node
+  let touched = false
 
   $: $component.editing && node?.focus()
   $: externalLink = url && typeof url === "string" && !url.startsWith("/")
@@ -28,6 +29,11 @@
   // overrides the color when it's passed as inline style.
   $: styles = enrichStyles($component.styles, color)
 
+  const handleUrlChange = () => {
+    sidePanelStore.actions.close()
+    modalStore.actions.close()
+  }
+
   const getSanitizedUrl = (url, externalLink, newTab) => {
     if (!url) {
       return externalLink || newTab ? "#/" : "/"
@@ -62,7 +68,10 @@
   }
 
   const updateText = e => {
-    builderStore.actions.updateProp("text", e.target.textContent)
+    if (touched) {
+      builderStore.actions.updateProp("text", e.target.textContent)
+    }
+    touched = false
   }
 </script>
 
@@ -76,6 +85,7 @@
     class:underline
     class="align--{align || 'left'} size--{size || 'M'}"
     on:blur={$component.editing ? updateText : null}
+    on:input={() => (touched = true)}
   >
     {componentText}
   </div>
@@ -104,7 +114,7 @@
         class:italic
         class:underline
         class="align--{align || 'left'} size--{size || 'M'}"
-        on:click={sidePanelStore.actions.close}
+        on:click={handleUrlChange}
       >
         {componentText}
       </a>

packages/client/src/components/app/Modal.svelte

@@ -0,0 +1,141 @@
+<script>
+  import { getContext } from "svelte"
+  import { Modal, Icon } from "@budibase/bbui"
+
+  const component = getContext("component")
+  const { styleable, modalStore, builderStore, dndIsDragging } =
+    getContext("sdk")
+
+  export let onClose
+  export let ignoreClicksOutside
+  export let size
+  let modal
+
+  // Open modal automatically in builder
+  $: {
+    if ($builderStore.inBuilder) {
+      if (
+        $component.inSelectedPath &&
+        $modalStore.contentId !== $component.id
+      ) {
+        modalStore.actions.open($component.id)
+      } else if (
+        !$component.inSelectedPath &&
+        $modalStore.contentId === $component.id &&
+        !$dndIsDragging
+      ) {
+        modalStore.actions.close()
+      }
+    }
+  }
+
+  $: open = $modalStore.contentId === $component.id
+
+  const handleModalClose = async () => {
+    if (onClose) {
+      await onClose()
+    }
+    modalStore.actions.close()
+  }
+
+  const handleOpen = (open, modal) => {
+    if (!modal) return
+
+    if (open) {
+      modal.show()
+    } else {
+      modal.hide()
+    }
+  }
+
+  $: handleOpen(open, modal)
+</script>
+
+<!-- Conditional displaying in the builder is necessary otherwise previews don't update properly upon component deletion -->
+{#if !$builderStore.inBuilder || open}
+  <Modal
+    on:cancel={handleModalClose}
+    bind:this={modal}
+    disableCancel={$builderStore.inBuilder}
+    zIndex={2}
+  >
+    <div use:styleable={$component.styles} class={`modal-content ${size}`}>
+      <div class="modal-header">
+        <Icon
+          color="var(--spectrum-global-color-gray-800)"
+          name="Close"
+          hoverable
+          on:click={handleModalClose}
+        />
+      </div>
+      <div class="modal-main">
+        <div class="modal-main-inner">
+          <slot />
+        </div>
+      </div>
+    </div>
+  </Modal>
+{/if}
+
+<style>
+  .modal-content {
+    display: flex;
+    flex-direction: column;
+    max-width: 100%;
+    box-sizing: border-box;
+    padding: 12px 0px 40px;
+  }
+
+  .small {
+    width: 400px;
+    min-height: 200px;
+  }
+
+  .medium {
+    width: 600px;
+    min-height: 400px;
+  }
+
+  .large {
+    width: 800px;
+    min-height: 600px;
+  }
+
+  .fullscreen {
+    width: calc(100vw - 80px);
+    min-height: calc(100vh - 80px);
+  }
+
+  .modal-header {
+    display: flex;
+    flex-direction: row;
+    justify-content: flex-end;
+    flex-shrink: 0;
+    flex-grow: 0;
+    padding: 0 12px 12px;
+    box-sizing: border-box;
+  }
+
+  .modal-main {
+    padding: 0 40px;
+    flex-grow: 1;
+    display: flex;
+    flex-direction: column;
+  }
+
+  .modal-main :global(.component > *) {
+    max-width: 100%;
+  }
+
+  .modal-main-inner {
+    flex-grow: 1;
+    display: flex;
+    flex-direction: column;
+    word-break: break-word;
+  }
+
+  .modal-main-inner:empty {
+    border-radius: 3px;
+    border: 2px dashed var(--spectrum-global-color-gray-400);
+  }
+</style>

packages/client/src/components/app/SidePanel.svelte

@@ -29,10 +29,6 @@
     }
   }
 
-  // $: {
-
-  // }
-
   // Derive visibility
   $: open = $sidePanelStore.contentId === $component.id
 

packages/client/src/components/app/Text.svelte

@@ -13,6 +13,7 @@
   export let size
 
   let node
+  let touched = false
 
   $: $component.editing && node?.focus()
   $: placeholder = $builderStore.inBuilder && !text && !$component.editing
@@ -46,7 +47,10 @@
 
   // Convert contenteditable HTML to text and save
   const updateText = e => {
-    builderStore.actions.updateProp("text", e.target.textContent)
+    if (touched) {
+      builderStore.actions.updateProp("text", e.target.textContent)
+    }
+    touched = false
   }
 </script>
 
@@ -61,6 +65,7 @@
     class:underline
     class="spectrum-Body {sizeClass} {alignClass}"
     on:blur={$component.editing ? updateText : null}
+    on:input={() => (touched = true)}
   >
     {componentText}
   </p>

packages/client/src/components/app/blocks/ChartBlock.svelte

@@ -35,6 +35,7 @@
   export let valueUnits
   export let yAxisLabel
   export let xAxisLabel
+  export let yAxisUnits
   export let curve
 
   // Area
@@ -85,6 +86,7 @@
           valueUnits,
           yAxisLabel,
           xAxisLabel,
+          yAxisUnits,
           stacked,
           horizontal,
           curve,

packages/client/src/components/app/blocks/FormBlockComponent.svelte

@@ -68,6 +68,15 @@
           maximum: schema?.constraints?.length?.maximum,
         }
       },
+      [FieldType.DATETIME]: (_field, schema) => {
+        const props = {
+          valueAsTimestamp: !schema?.timeOnly,
+        }
+        if (schema?.dateOnly) {
+          props.enableTime = false
+        }
+        return props
+      },
     }
 
     const fieldSchema = getFieldSchema(field)

packages/client/src/components/app/blocks/form/FormBlock.svelte

@@ -31,41 +31,23 @@
 
   let schema
 
-  $: formattedFields = convertOldFieldFormat(fields)
-  $: fieldsOrDefault = getDefaultFields(formattedFields, schema)
   $: fetchSchema(dataSource)
   $: id = $component.id
-  // We could simply spread $$props into the inner form and append our
+  $: formattedFields = convertOldFieldFormat(fields)
-  // additions, but that would create svelte warnings about unused props and
+  $: fieldsOrDefault = getDefaultFields(formattedFields, schema)
-  // make maintenance in future more confusing as we typically always have a
+  $: buttonsOrDefault =
-  // proper mapping of schema settings to component exports, without having to
+    buttons ||
-  // search multiple files
+    Utils.buildFormBlockButtonConfig({
-  $: innerProps = {
+      _id: id,
-    dataSource,
+      showDeleteButton,
-    actionUrl,
+      showSaveButton,
-    actionType,
+      saveButtonLabel,
-    size,
+      deleteButtonLabel,
-    disabled,
+      notificationOverride,
-    fields: fieldsOrDefault,
+      actionType,
-    title,
+      actionUrl,
-    description,
+      dataSource,
-    schema,
+    })
-    notificationOverride,
-    buttons:
-      buttons ||
-      Utils.buildFormBlockButtonConfig({
-        _id: id,
-        showDeleteButton,
-        showSaveButton,
-        saveButtonLabel,
-        deleteButtonLabel,
-        notificationOverride,
-        actionType,
-        actionUrl,
-        dataSource,
-      }),
-    buttonPosition: buttons ? buttonPosition : "top",
-  }
 
   // Provide additional data context for live binding eval
   export const getAdditionalDataContext = () => {
@@ -123,5 +105,18 @@
 </script>
 
 <FormBlockWrapper {actionType} {dataSource} {rowId} {noRowsMessage}>
-  <InnerFormBlock {...innerProps} />
+  <InnerFormBlock
+    {dataSource}
+    {actionUrl}
+    {actionType}
+    {size}
+    {disabled}
+    fields={fieldsOrDefault}
+    {title}
+    {description}
+    {schema}
+    {notificationOverride}
+    buttons={buttonsOrDefault}
+    buttonPosition={buttons ? buttonPosition : "top"}
+  />
 </FormBlockWrapper>

packages/client/src/components/app/blocks/form/InnerFormBlock.svelte

@@ -91,15 +91,13 @@
       {#if description}
         <BlockComponent type="text" props={{ text: description }} order={1} />
       {/if}
-      {#key fields}
+      <BlockComponent type="container">
-        <BlockComponent type="container">
+        <div class="form-block fields" class:mobile={$context.device.mobile}>
-          <div class="form-block fields" class:mobile={$context.device.mobile}>
+          {#each fields as field, idx}
-            {#each fields as field, idx}
+            <FormBlockComponent {field} {schema} order={idx} />
-              <FormBlockComponent {field} {schema} order={idx} />
+          {/each}
-            {/each}
+        </div>
-          </div>
+      </BlockComponent>
-        </BlockComponent>
-      {/key}
     </BlockComponent>
     {#if buttonPosition === "bottom"}
       <BlockComponent

packages/client/src/components/app/charts/AreaChart.svelte

@@ -74,7 +74,6 @@
       },
     },
     xaxis: {
-      type: labelType,
       categories,
       labels: {
         formatter: xAxisFormatter,

packages/client/src/components/app/charts/BarChart.svelte

@@ -72,7 +72,6 @@
     },
     // We can just always provide the categories to the xaxis and horizontal mode automatically handles "tranposing" the categories to the yaxis, but certain things like labels need to be manually put on a certain axis based on the selected mode. Titles do not need to be handled this way, they are exposed to the user as "X axis" and Y Axis" so flipping them would be confusing.
     xaxis: {
-      type: labelType,
       categories,
       labels: {
         formatter: xAxisFormatter,

packages/client/src/components/app/charts/LineChart.svelte

@@ -66,7 +66,6 @@
       },
     },
     xaxis: {
-      type: labelType,
       categories,
       labels: {
         formatter: xAxisFormatter,

packages/client/src/components/app/dynamic-filter/DynamicFilter.svelte

@@ -3,7 +3,7 @@
   import { getContext, onDestroy } from "svelte"
   import { ModalContent, Modal } from "@budibase/bbui"
   import FilterModal from "./FilterModal.svelte"
-  import { LuceneUtils } from "@budibase/frontend-core"
+  import { QueryUtils } from "@budibase/frontend-core"
   import Button from "../Button.svelte"
 
   export let dataProvider
@@ -36,7 +36,7 @@
   // Add query extension to data provider
   $: {
     if (filters?.length) {
-      const queryExtension = LuceneUtils.buildLuceneQuery(filters)
+      const queryExtension = QueryUtils.buildQuery(filters)
       addExtension?.($component.id, queryExtension)
     } else {
       removeExtension?.($component.id)

packages/client/src/components/app/forms/DateTimeField.svelte

@@ -16,15 +16,37 @@
   export let onChange
   export let span
   export let helpText = null
+  export let valueAsTimestamp = false
 
   let fieldState
   let fieldApi
 
   const handleChange = e => {
-    const changed = fieldApi.setValue(e.detail)
+    let value = e.detail
-    if (onChange && changed) {
+    if (timeOnly && valueAsTimestamp) {
-      onChange({ value: e.detail })
+      if (!isValidDate(value)) {
+        // Handle time only fields that are timestamps under the hood
+        value = timeToDateISOString(value)
+      }
     }
+
+    const changed = fieldApi.setValue(value)
+    if (onChange && changed) {
+      onChange({ value })
+    }
+  }
+
+  const isValidDate = value => !isNaN(new Date(value))
+
+  const timeToDateISOString = value => {
+    let [hours, minutes] = value.split(":").map(Number)
+
+    const date = new Date()
+    date.setHours(hours)
+    date.setMinutes(minutes)
+    date.setSeconds(0)
+    date.setMilliseconds(0)
+    return date.toISOString()
   }
 </script>
 

packages/client/src/components/app/forms/Field.svelte

@@ -26,6 +26,10 @@
   // Register field with form
   const formApi = formContext?.formApi
   const labelPos = fieldGroupContext?.labelPosition || "above"
+
+  let touched = false
+  let labelNode
+
   $: formStep = formStepContext ? $formStepContext || 1 : 1
   $: formField = formApi?.registerField(
     field,
@@ -36,14 +40,12 @@
     validation,
     formStep
   )
-
   $: schemaType =
     fieldSchema?.type !== "formula" && fieldSchema?.type !== "bigint"
       ? fieldSchema?.type
       : "string"
 
   // Focus label when editing
-  let labelNode
   $: $component.editing && labelNode?.focus()
 
   // Update form properties in parent component on every store change
@@ -57,7 +59,10 @@
   $: labelClass = labelPos === "above" ? "" : `spectrum-FieldLabel--${labelPos}`
 
   const updateLabel = e => {
-    builderStore.actions.updateProp("label", e.target.textContent)
+    if (touched) {
+      builderStore.actions.updateProp("label", e.target.textContent)
+    }
+    touched = false
   }
 
   onDestroy(() => {
@@ -79,6 +84,7 @@
       bind:this={labelNode}
       contenteditable={$component.editing}
       on:blur={$component.editing ? updateLabel : null}
+      on:input={() => (touched = true)}
       class:hidden={!label}
       class:readonly
       for={fieldState?.fieldId}

packages/client/src/components/app/forms/InnerForm.svelte

@@ -206,7 +206,7 @@
           error: initialError,
           disabled:
             disabled || fieldDisabled || (isAutoColumn && !editAutoColumns),
-          readonly: readonly || fieldReadOnly,
+          readonly: readonly || fieldReadOnly || schema?.[field]?.readonly,
           defaultValue,
           validator,
           lastUpdate: Date.now(),

packages/client/src/components/app/index.js

@@ -37,6 +37,7 @@ export { default as markdownviewer } from "./MarkdownViewer.svelte"
 export { default as embeddedmap } from "./embedded-map/EmbeddedMap.svelte"
 export { default as grid } from "./Grid.svelte"
 export { default as sidepanel } from "./SidePanel.svelte"
+export { default as modal } from "./Modal.svelte"
 export { default as gridblock } from "./GridBlock.svelte"
 export * from "./charts"
 export * from "./forms"

packages/client/src/components/overlay/ConfirmationDisplay.svelte

@@ -8,6 +8,8 @@
     <ModalContent
       title={$confirmationStore.title}
       onConfirm={confirmationStore.actions.confirm}
+      confirmText={$confirmationStore.confirmButtonText}
+      cancelText={$confirmationStore.cancelButtonText}
     >
       {$confirmationStore.text}
     </ModalContent>

packages/client/src/components/preview/IndicatorSet.svelte

@@ -57,7 +57,9 @@
       return
     }
     nextState.indicators[idx].visible =
-      nextState.indicators[idx].insideSidePanel || entries[0].isIntersecting
+      nextState.indicators[idx].insideModal ||
+      nextState.indicators[idx].insideSidePanel ||
+      entries[0].isIntersecting
     if (++callbackCount === observers.length) {
       state = nextState
       updating = false
@@ -139,6 +141,7 @@
         height: elBounds.height + 4,
         visible: false,
         insideSidePanel: !!child.closest(".side-panel"),
+        insideModal: !!child.closest(".modal-content"),
       })
     })
   }

packages/client/src/components/preview/SettingsBar.svelte

@@ -41,7 +41,7 @@
         allSettings.push(setting)
       }
     })
-    return allSettings.filter(setting => setting.showInBar)
+    return allSettings.filter(setting => setting.showInBar && !setting.hidden)
   }
 
   const updatePosition = () => {