Merge branch 'master' into qa-core-changes

.eslintignore

@@ -10,4 +10,5 @@ packages/builder/.routify
 packages/sdk/sdk
 packages/account-portal/packages/server/build
 packages/account-portal/packages/ui/.routify
-packages/account-portal/packages/ui/build
+packages/account-portal/packages/ui/build
+**/*.ivm.bundle.js

.eslintrc.json

@@ -43,7 +43,8 @@
         "no-useless-escape": "off",
         "no-undef": "off",
         "no-prototype-builtins": "off",
-        "local-rules/no-budibase-imports": "error"
+        "local-rules/no-budibase-imports": "error",
+        "local-rules/no-test-com": "error"
       }
     },
     {
@@ -53,7 +54,7 @@
         "packages/frontend-core/**/*"
       ],
       "rules": {
-        "no-console": ["error", { "allow": ["warn", "error", "debug"] } ]
+        "no-console": ["error", { "allow": ["warn", "error", "debug"] }]
       }
     }
   ],

.prettierignore

@@ -11,4 +11,5 @@ packages/sdk/sdk
 packages/pro/coverage
 packages/account-portal/packages/ui/build
 packages/account-portal/packages/ui/.routify
-packages/account-portal/packages/server/build
+packages/account-portal/packages/server/build
+**/*.ivm.bundle.js

eslint-local-rules/index.js

@@ -18,4 +18,37 @@ module.exports = {
       }
     },
   },
+  "no-test-com": {
+    meta: {
+      type: "problem",
+      docs: {
+        description:
+          "disallow the use of 'test.com' in strings and replace it with 'example.com'",
+        category: "Possible Errors",
+        recommended: false,
+      },
+      schema: [], // no options
+      fixable: "code", // Indicates that this rule supports automatic fixing
+    },
+    create: function (context) {
+      return {
+        Literal(node) {
+          if (
+            typeof node.value === "string" &&
+            node.value.includes("test.com")
+          ) {
+            context.report({
+              node,
+              message:
+                "test.com is a privately owned domain and could point anywhere, use example.com instead.",
+              fix: function (fixer) {
+                const newText = node.raw.replace(/test\.com/g, "example.com")
+                return fixer.replaceText(node, newText)
+              },
+            })
+          }
+        },
+      }
+    },
+  },
 }

hosting/couchdb/Dockerfile

@@ -1,4 +1,101 @@
-FROM couchdb:3.2.1
+# Modified from https://github.com/apache/couchdb-docker/blob/main/3.3.3/Dockerfile
+#
+# Everything in this `base` image is adapted from the official `couchdb` image's
+# Dockerfile. Only modifications related to upgrading from Debian bullseye to
+# bookworm have been included. The `runner` image contains Budibase's
+# customisations to the image, e.g. adding Clouseau.
+FROM node:20-slim AS base
+
+# Add CouchDB user account to make sure the IDs are assigned consistently
+RUN groupadd -g 5984 -r couchdb && useradd -u 5984 -d /opt/couchdb -g couchdb couchdb
+
+# be sure GPG and apt-transport-https are available and functional
+RUN set -ex; \
+    apt-get update; \
+    apt-get install -y --no-install-recommends \
+        apt-transport-https \
+        ca-certificates \
+        dirmngr \
+        gnupg \
+     ; \
+    rm -rf /var/lib/apt/lists/*
+
+# grab tini for signal handling and zombie reaping
+# see https://github.com/apache/couchdb-docker/pull/28#discussion_r141112407
+RUN set -eux; \
+    apt-get update; \
+    apt-get install -y --no-install-recommends tini; \
+    rm -rf /var/lib/apt/lists/*; \
+    tini --version
+
+# http://docs.couchdb.org/en/latest/install/unix.html#installing-the-apache-couchdb-packages
+ENV GPG_COUCH_KEY \
+# gpg: rsa8192 205-01-19 The Apache Software Foundation (Package repository signing key) <root@apache.org>
+    390EF70BB1EA12B2773962950EE62FB37A00258D
+RUN set -eux; \
+    apt-get update; \
+    apt-get install -y curl; \
+    export GNUPGHOME="$(mktemp -d)"; \
+    curl -fL -o keys.asc https://couchdb.apache.org/repo/keys.asc; \
+    gpg --batch --import keys.asc; \
+    gpg --batch --export "${GPG_COUCH_KEY}" > /usr/share/keyrings/couchdb-archive-keyring.gpg; \
+    command -v gpgconf && gpgconf --kill all || :; \
+    rm -rf "$GNUPGHOME"; \
+    apt-key list; \
+    apt purge -y --autoremove curl; \
+    rm -rf /var/lib/apt/lists/*
+
+ENV COUCHDB_VERSION 3.3.3
+
+RUN . /etc/os-release; \
+    echo "deb [signed-by=/usr/share/keyrings/couchdb-archive-keyring.gpg] https://apache.jfrog.io/artifactory/couchdb-deb/ ${VERSION_CODENAME} main" | \
+        tee /etc/apt/sources.list.d/couchdb.list >/dev/null
+
+# https://github.com/apache/couchdb-pkg/blob/master/debian/README.Debian
+RUN set -eux; \
+    apt-get update; \
+    \
+    echo "couchdb couchdb/mode select none" | debconf-set-selections; \
+# we DO want recommends this time
+    DEBIAN_FRONTEND=noninteractive apt-get install -y --allow-downgrades --allow-remove-essential --allow-change-held-packages \
+            couchdb="$COUCHDB_VERSION"~bookworm \
+    ; \
+# Undo symlinks to /var/log and /var/lib
+    rmdir /var/lib/couchdb /var/log/couchdb; \
+    rm /opt/couchdb/data /opt/couchdb/var/log; \
+    mkdir -p /opt/couchdb/data /opt/couchdb/var/log; \
+    chown couchdb:couchdb /opt/couchdb/data /opt/couchdb/var/log; \
+    chmod 777 /opt/couchdb/data /opt/couchdb/var/log; \
+# Remove file that sets logging to a file
+    rm /opt/couchdb/etc/default.d/10-filelog.ini; \
+# Check we own everything in /opt/couchdb. Matches the command in dockerfile_entrypoint.sh
+    find /opt/couchdb \! \( -user couchdb -group couchdb \) -exec chown -f couchdb:couchdb '{}' +; \
+# Setup directories and permissions for config. Technically these could be 555 and 444 respectively
+# but we keep them as 755 and 644 for consistency with CouchDB defaults and the dockerfile_entrypoint.sh.
+    find /opt/couchdb/etc -type d ! -perm 0755 -exec chmod -f 0755 '{}' +; \
+    find /opt/couchdb/etc -type f ! -perm 0644 -exec chmod -f 0644 '{}' +; \
+# only local.d needs to be writable for the docker_entrypoint.sh
+    chmod -f 0777 /opt/couchdb/etc/local.d; \
+# apt clean-up
+    rm -rf /var/lib/apt/lists/*;
+
+# Add configuration
+COPY --chown=couchdb:couchdb couch/10-docker-default.ini /opt/couchdb/etc/default.d/
+# COPY --chown=couchdb:couchdb vm.args /opt/couchdb/etc/
+
+COPY docker-entrypoint.sh /usr/local/bin
+RUN ln -s usr/local/bin/docker-entrypoint.sh /docker-entrypoint.sh # backwards compat
+ENTRYPOINT ["tini", "--", "/docker-entrypoint.sh"]
+
+VOLUME /opt/couchdb/data
+
+# 5984: Main CouchDB endpoint
+# 4369: Erlang portmap daemon (epmd)
+# 9100: CouchDB cluster communication port
+EXPOSE 5984 4369 9100
+CMD ["/opt/couchdb/bin/couchdb"]
+
+FROM base as runner
 
 ENV COUCHDB_USER admin
 ENV COUCHDB_PASSWORD admin
@@ -6,9 +103,9 @@ EXPOSE 5984
 
 RUN apt-get update && apt-get install -y --no-install-recommends software-properties-common wget unzip curl && \
     wget -O - https://packages.adoptium.net/artifactory/api/gpg/key/public | apt-key add - && \
-    apt-add-repository 'deb http://security.debian.org/debian-security bullseye-security/updates main' && \
+    apt-add-repository 'deb http://security.debian.org/debian-security bookworm-security/updates main' && \
     apt-add-repository 'deb http://archive.debian.org/debian stretch-backports main' && \
-    apt-add-repository 'deb https://packages.adoptium.net/artifactory/deb bullseye main' && \
+    apt-add-repository 'deb https://packages.adoptium.net/artifactory/deb bookworm main' && \
     apt-get update && apt-get install -y --no-install-recommends temurin-8-jdk && \
     rm -rf /var/lib/apt/lists/
 

hosting/couchdb/clouseau/clouseau.ini

@@ -4,7 +4,7 @@
 name=clouseau@127.0.0.1
 
 ; set this to the same distributed Erlang cookie used by the CouchDB nodes
-cookie=monster
+cookie=COUCHDB_ERLANG_COOKIE
 
 ; the path where you would like to store the search index files
 dir=DATA_DIR/search

hosting/couchdb/couch/10-docker-default.ini

@@ -0,0 +1,8 @@
+; CouchDB Configuration Settings
+
+; Custom settings should be made in this file. They will override settings
+; in default.ini, but unlike changes made to default.ini, this file won't be
+; overwritten on server upgrade.
+
+[chttpd]
+bind_address = any

hosting/couchdb/couch/vm.args

@@ -12,7 +12,7 @@
 
 # erlang cookie for clouseau security
 -name couchdb@127.0.0.1
--setcookie monster
+-setcookie COUCHDB_ERLANG_COOKIE
 
 # Ensure that the Erlang VM listens on a known port
 -kernel inet_dist_listen_min 9100

hosting/couchdb/docker-entrypoint.sh

@@ -0,0 +1,122 @@
+#!/bin/bash
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+set -e
+
+# first arg is `-something` or `+something`
+if [ "${1#-}" != "$1" ] || [ "${1#+}" != "$1" ]; then
+	set -- /opt/couchdb/bin/couchdb "$@"
+fi
+
+# first arg is the bare word `couchdb`
+if [ "$1" = 'couchdb' ]; then
+	shift
+	set -- /opt/couchdb/bin/couchdb "$@"
+fi
+
+if [ "$1" = '/opt/couchdb/bin/couchdb' ]; then
+	# this is where runtime configuration changes will be written.
+	# we need to explicitly touch it here in case /opt/couchdb/etc has
+	# been mounted as an external volume, in which case it won't exist.
+	# If running as the couchdb user (i.e. container starts as root),
+	# write permissions will be granted below.
+	touch /opt/couchdb/etc/local.d/docker.ini
+
+	# if user is root, assume running under the couchdb user (default)
+	# and ensure it is able to access files and directories that may be mounted externally
+	if [ "$(id -u)" = '0' ]; then
+		# Check that we own everything in /opt/couchdb and fix if necessary. We also
+		# add the `-f` flag in all the following invocations because there may be
+		# cases where some of these ownership and permissions issues are non-fatal
+		# (e.g. a config file owned by root with o+r is actually fine), and we don't
+		# to be too aggressive about crashing here ...
+		find /opt/couchdb \! \( -user couchdb -group couchdb \) -exec chown -f couchdb:couchdb '{}' +
+
+		# Ensure that data files have the correct permissions. We were previously
+		# preventing any access to these files outside of couchdb:couchdb, but it
+		# turns out that CouchDB itself does not set such restrictive permissions
+		# when it creates the files. The approach taken here ensures that the
+		# contents of the datadir have the same permissions as they had when they
+		# were initially created. This should minimize any startup delay.
+		find /opt/couchdb/data -type d ! -perm 0755 -exec chmod -f 0755 '{}' +
+		find /opt/couchdb/data -type f ! -perm 0644 -exec chmod -f 0644 '{}' +
+
+		# Do the same thing for configuration files and directories. Technically
+		# CouchDB only needs read access to the configuration files as all online
+		# changes will be applied to the "docker.ini" file below, but we set 644
+		# for the sake of consistency.
+		find /opt/couchdb/etc -type d ! -perm 0755 -exec chmod -f 0755 '{}' +
+		find /opt/couchdb/etc -type f ! -perm 0644 -exec chmod -f 0644 '{}' +
+	fi
+
+	if [ ! -z "$NODENAME" ] && ! grep "couchdb@" /opt/couchdb/etc/vm.args; then
+		echo "-name couchdb@$NODENAME" >> /opt/couchdb/etc/vm.args
+	fi
+
+	if [ "$COUCHDB_USER" ] && [ "$COUCHDB_PASSWORD" ]; then
+		# Create admin only if not already present
+		if ! grep -Pzoqr "\[admins\]\n$COUCHDB_USER =" /opt/couchdb/etc/local.d/*.ini /opt/couchdb/etc/local.ini; then
+			printf "\n[admins]\n%s = %s\n" "$COUCHDB_USER" "$COUCHDB_PASSWORD" >> /opt/couchdb/etc/local.d/docker.ini
+		fi
+	fi
+
+	if [ "$COUCHDB_SECRET" ]; then
+		# Set secret only if not already present
+		if ! grep -Pzoqr "\[chttpd_auth\]\nsecret =" /opt/couchdb/etc/local.d/*.ini /opt/couchdb/etc/local.ini; then
+			printf "\n[chttpd_auth]\nsecret = %s\n" "$COUCHDB_SECRET" >> /opt/couchdb/etc/local.d/docker.ini
+		fi
+	fi
+
+	if [ "$COUCHDB_ERLANG_COOKIE" ]; then
+		cookieFile='/opt/couchdb/.erlang.cookie'
+		if [ -e "$cookieFile" ]; then
+			if [ "$(cat "$cookieFile" 2>/dev/null)" != "$COUCHDB_ERLANG_COOKIE" ]; then
+				echo >&2
+				echo >&2 "warning: $cookieFile contents do not match COUCHDB_ERLANG_COOKIE"
+				echo >&2
+			fi
+		else
+			echo "$COUCHDB_ERLANG_COOKIE" > "$cookieFile"
+		fi
+		chown couchdb:couchdb "$cookieFile"
+		chmod 600 "$cookieFile"
+	fi
+
+	if [ "$(id -u)" = '0' ]; then
+		chown -f couchdb:couchdb /opt/couchdb/etc/local.d/docker.ini || true
+	fi
+
+	# if we don't find an [admins] section followed by a non-comment, display a warning
+        if ! grep -Pzoqr '\[admins\]\n[^;]\w+' /opt/couchdb/etc/default.d/*.ini /opt/couchdb/etc/local.d/*.ini /opt/couchdb/etc/local.ini; then
+		# The - option suppresses leading tabs but *not* spaces. :)
+		cat >&2 <<-'EOWARN'
+*************************************************************
+ERROR: CouchDB 3.0+ will no longer run in "Admin Party"
+       mode. You *MUST* specify an admin user and
+       password, either via your own .ini file mapped
+       into the container at /opt/couchdb/etc/local.ini
+       or inside /opt/couchdb/etc/local.d, or with
+       "-e COUCHDB_USER=admin -e COUCHDB_PASSWORD=password"
+       to set it via "docker run".
+*************************************************************
+EOWARN
+		exit 1
+	fi
+
+	if [ "$(id -u)" = '0' ]; then
+		export HOME=$(echo ~couchdb)
+		exec setpriv --reuid=couchdb --regid=couchdb --clear-groups "$@"
+	fi
+fi
+
+exec "$@"

hosting/couchdb/runner.sh

@@ -1,6 +1,7 @@
 #!/bin/bash
 
 DATA_DIR=${DATA_DIR:-/data}
+COUCHDB_ERLANG_COOKIE=${COUCHDB_ERLANG_COOKIE:-B9CFC32C-3458-4A86-8448-B3C753991CA7}
 
 mkdir -p ${DATA_DIR}
 mkdir -p ${DATA_DIR}/couch/{dbs,views}
@@ -60,6 +61,9 @@ else
     sed -i "s#DATA_DIR#/data#g" /opt/couchdb/etc/local.ini
 fi
 
+sed -i "s#COUCHDB_ERLANG_COOKIE#${COUCHDB_ERLANG_COOKIE}#g" /opt/couchdb/etc/vm.args
+sed -i "s#COUCHDB_ERLANG_COOKIE#${COUCHDB_ERLANG_COOKIE}#g" /opt/clouseau/clouseau.ini
+
 # Start Clouseau. Budibase won't function correctly without Clouseau running, it
 # powers the search API endpoints which are used to do all sorts, including
 # populating app grids.

hosting/docker-compose.yaml

@@ -98,7 +98,6 @@ services:
   couchdb-service:
     restart: unless-stopped
     image: budibase/couchdb
-    pull_policy: always
     environment:
       - COUCHDB_PASSWORD=${COUCH_DB_PASSWORD}
       - COUCHDB_USER=${COUCH_DB_USER}

hosting/single/Dockerfile

@@ -3,7 +3,6 @@ FROM node:20-slim as build
 # install node-gyp dependencies
 RUN apt-get update && apt-get install -y --no-install-recommends g++ make python3 jq
 
-
 # copy and install dependencies
 WORKDIR /app
 COPY package.json .
@@ -39,10 +38,9 @@ COPY packages/worker/pm2.config.js packages/worker/pm2.config.js
 COPY packages/string-templates packages/string-templates
 
 
-FROM budibase/couchdb as runner
+FROM budibase/couchdb:v3.3.3 as runner
 ARG TARGETARCH
 ENV TARGETARCH $TARGETARCH
-ENV NODE_MAJOR 20
 #TARGETBUILD can be set to single (for single docker image) or aas (for azure app service)
 # e.g. docker build --build-arg TARGETBUILD=aas ....
 ARG TARGETBUILD=single
@@ -60,10 +58,8 @@ RUN apt install -y software-properties-common apt-transport-https ca-certificate
     && apt install postgresql-client-15 -y \
     && apt remove software-properties-common apt-transport-https gpg -y
 
-# install other dependencies, nodejs, oracle requirements, jdk8, redis, nginx
+# We use pm2 in order to run multiple node processes in a single container
-WORKDIR /nodejs
+RUN npm install --global pm2
-COPY scripts/install-node.sh ./install.sh
-RUN chmod +x install.sh && ./install.sh
 
 # setup nginx
 COPY hosting/single/nginx/nginx.conf /etc/nginx
@@ -124,6 +120,8 @@ HEALTHCHECK --interval=15s --timeout=15s --start-period=45s CMD "/healthcheck.sh
 
 # must set this just before running
 ENV NODE_ENV=production
+# this is required for isolated-vm to work on Node 20+
+ENV NODE_OPTIONS="--no-node-snapshot"
 WORKDIR /
 
 CMD ["./runner.sh"]

hosting/single/runner.sh

@@ -97,10 +97,12 @@ fi
 sleep 10
 
 pushd app
-pm2 start -l /dev/stdout --name app "yarn run:docker"
+pm2 start --name app "yarn run:docker"
 popd
 pushd worker
-pm2 start -l /dev/stdout --name worker "yarn run:docker"
+pm2 start --name worker "yarn run:docker"
 popd
 echo "end of runner.sh, sleeping ..."
+
+tail -f $HOME/.pm2/logs/*.log
 sleep infinity

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "2.17.6",
+  "version": "2.20.5",
   "npmClient": "yarn",
   "packages": [
     "packages/*",

package.json

@@ -58,7 +58,7 @@
     "lint": "yarn run lint:eslint && yarn run lint:prettier",
     "lint:fix:eslint": "eslint --fix --max-warnings=0 packages qa-core",
     "lint:fix:prettier": "prettier --write \"packages/**/*.{js,ts,svelte}\" && prettier --write \"examples/**/*.{js,ts,svelte}\" && prettier --write \"qa-core/**/*.{js,ts,svelte}\"",
-    "lint:fix": "yarn run lint:fix:prettier && yarn run lint:fix:eslint",
+    "lint:fix": "yarn run lint:fix:eslint && yarn run lint:fix:prettier",
     "build:specs": "lerna run --stream specs",
     "build:docker:airgap": "node hosting/scripts/airgapped/airgappedDockerBuild",
     "build:docker:airgap:single": "SINGLE_IMAGE=1 node hosting/scripts/airgapped/airgappedDockerBuild",
@@ -97,7 +97,16 @@
     "@budibase/backend-core": "0.0.0",
     "@budibase/shared-core": "0.0.0",
     "@budibase/string-templates": "0.0.0",
-    "@budibase/types": "0.0.0"
+    "@budibase/types": "0.0.0",
+    "tough-cookie": "4.1.3",
+    "node-fetch": "2.6.7",
+    "semver": "7.5.3",
+    "http-cache-semantics": "4.1.1",
+    "msgpackr": "1.10.1",
+    "axios": "1.6.3",
+    "xml2js": "0.6.2",
+    "unset-value": "2.0.1",
+    "passport": "0.6.0"
   },
   "engines": {
     "node": ">=20.0.0 <21.0.0"

packages/account-portal

@@ -1 +1 @@
-Subproject commit cc12291732ee902dc832bc7d93cf2086ffdf0cff
+Subproject commit 4384bc742ca22fb1e9bf91843e65ae929daf17e2

packages/backend-core/package.json

@@ -25,19 +25,19 @@
     "@budibase/pouchdb-replication-stream": "1.2.10",
     "@budibase/shared-core": "0.0.0",
     "@budibase/types": "0.0.0",
-    "@techpass/passport-openidconnect": "0.3.2",
+    "@govtechsg/passport-openidconnect": "^1.0.2",
     "aws-cloudfront-sign": "3.0.2",
     "aws-sdk": "2.1030.0",
     "bcrypt": "5.1.0",
     "bcryptjs": "2.4.3",
     "bull": "4.10.1",
     "correlation-id": "4.0.0",
-    "dd-trace": "5.0.0",
+    "dd-trace": "5.2.0",
     "dotenv": "16.0.1",
     "ioredis": "5.3.2",
     "joi": "17.6.0",
     "jsonwebtoken": "9.0.2",
-    "koa-passport": "4.1.4",
+    "koa-passport": "^6.0.0",
     "koa-pino-logger": "4.0.0",
     "lodash": "4.17.21",
     "node-fetch": "2.6.7",
@@ -52,9 +52,9 @@
     "redlock": "4.2.0",
     "rotating-file-stream": "3.1.0",
     "sanitize-s3-objectkey": "0.0.1",
-    "semver": "7.3.7",
+    "semver": "^7.5.4",
     "tar-fs": "2.1.1",
-    "uuid": "8.3.2"
+    "uuid": "^8.3.2"
   },
   "devDependencies": {
     "@shopify/jest-koa-mocks": "5.1.1",

packages/backend-core/src/context/types.ts

@@ -1,4 +1,4 @@
-import { IdentityContext } from "@budibase/types"
+import { IdentityContext, VM } from "@budibase/types"
 import { ExecutionTimeTracker } from "../timers"
 
 // keep this out of Budibase types, don't want to expose context info
@@ -11,4 +11,5 @@ export type ContextMap = {
   automationId?: string
   isMigrating?: boolean
   jsExecutionTracker?: ExecutionTimeTracker
+  vm?: VM
 }

packages/backend-core/src/events/publishers/query.ts

@@ -3,6 +3,7 @@ import {
   Event,
   Datasource,
   Query,
+  QueryPreview,
   QueryCreatedEvent,
   QueryUpdatedEvent,
   QueryDeletedEvent,
@@ -68,9 +69,9 @@ const run = async (count: number, timestamp?: string | number) => {
   await publishEvent(Event.QUERIES_RUN, properties, timestamp)
 }
 
-const previewed = async (datasource: Datasource, query: Query) => {
+const previewed = async (datasource: Datasource, query: QueryPreview) => {
   const properties: QueryPreviewedEvent = {
-    queryId: query._id,
+    queryId: query.queryId,
     datasourceId: datasource._id as string,
     source: datasource.source,
     queryVerb: query.queryVerb,

packages/backend-core/src/installation.ts

@@ -6,6 +6,7 @@ import * as context from "./context"
 import semver from "semver"
 import { bustCache, withCache, TTL, CacheKey } from "./cache/generic"
 import environment from "./environment"
+import { logAlert } from "./logging"
 
 export const getInstall = async (): Promise<Installation> => {
   return withCache(CacheKey.INSTALLATION, TTL.ONE_DAY, getInstallFromDB, {
@@ -80,27 +81,35 @@ export const checkInstallVersion = async (): Promise<void> => {
   const currentVersion = install.version
   const newVersion = environment.VERSION
 
-  if (currentVersion !== newVersion) {
+  try {
-    const isUpgrade = semver.gt(newVersion, currentVersion)
+    if (currentVersion !== newVersion) {
-    const isDowngrade = semver.lt(newVersion, currentVersion)
+      const isUpgrade = semver.gt(newVersion, currentVersion)
+      const isDowngrade = semver.lt(newVersion, currentVersion)
 
-    const success = await updateVersion(newVersion)
+      const success = await updateVersion(newVersion)
 
-    if (success) {
+      if (success) {
-      await context.doInIdentityContext(
+        await context.doInIdentityContext(
-        {
+          {
-          _id: install.installId,
+            _id: install.installId,
-          type: IdentityType.INSTALLATION,
+            type: IdentityType.INSTALLATION,
-        },
+          },
-        async () => {
+          async () => {
-          if (isUpgrade) {
+            if (isUpgrade) {
-            await events.installation.upgraded(currentVersion, newVersion)
+              await events.installation.upgraded(currentVersion, newVersion)
-          } else if (isDowngrade) {
+            } else if (isDowngrade) {
-            await events.installation.downgraded(currentVersion, newVersion)
+              await events.installation.downgraded(currentVersion, newVersion)
+            }
           }
-        }
+        )
-      )
+        await events.identification.identifyInstallationGroup(install.installId)
-      await events.identification.identifyInstallationGroup(install.installId)
+      }
+    }
+  } catch (err: any) {
+    if (err?.message?.includes("Invalid Version")) {
+      logAlert(`Invalid version "${newVersion}" - is it semver?`)
+    } else {
+      logAlert("Failed to retrieve version", err)
     }
   }
 }

packages/backend-core/src/logging/correlation/correlation.ts

@@ -2,11 +2,12 @@ import { Header } from "../../constants"
 
 const correlator = require("correlation-id")
 
-export const setHeader = (headers: any) => {
+export const setHeader = (headers: Record<string, string>) => {
   const correlationId = correlator.getId()
-  if (correlationId) {
+  if (!correlationId) {
-    headers[Header.CORRELATION_ID] = correlationId
+    return
   }
+  headers[Header.CORRELATION_ID] = correlationId
 }
 
 export function getId() {

packages/backend-core/src/objectStore/objectStore.ts

@@ -255,7 +255,8 @@ export async function listAllObjects(bucketName: string, path: string) {
       objects = objects.concat(response.Contents)
     }
     isTruncated = !!response.IsTruncated
-  } while (isTruncated)
+    token = response.NextContinuationToken
+  } while (isTruncated && token)
   return objects
 }
 

packages/backend-core/src/queue/queue.ts

@@ -2,7 +2,7 @@ import env from "../environment"
 import { getRedisOptions } from "../redis/utils"
 import { JobQueue } from "./constants"
 import InMemoryQueue from "./inMemoryQueue"
-import BullQueue, { QueueOptions } from "bull"
+import BullQueue, { QueueOptions, JobOptions } from "bull"
 import { addListeners, StalledFn } from "./listeners"
 import { Duration } from "../utils"
 import * as timers from "../timers"
@@ -24,17 +24,24 @@ async function cleanup() {
 
 export function createQueue<T>(
   jobQueue: JobQueue,
-  opts: { removeStalledCb?: StalledFn } = {}
+  opts: {
+    removeStalledCb?: StalledFn
+    maxStalledCount?: number
+    jobOptions?: JobOptions
+  } = {}
 ): BullQueue.Queue<T> {
   const redisOpts = getRedisOptions()
   const queueConfig: QueueOptions = {
     redis: redisOpts,
     settings: {
-      maxStalledCount: 0,
+      maxStalledCount: opts.maxStalledCount ? opts.maxStalledCount : 0,
       lockDuration: QUEUE_LOCK_MS,
       lockRenewTime: QUEUE_LOCK_RENEW_INTERNAL_MS,
     },
   }
+  if (opts.jobOptions) {
+    queueConfig.defaultJobOptions = opts.jobOptions
+  }
   let queue: any
   if (!env.isTest()) {
     queue = new BullQueue(jobQueue, queueConfig)

packages/backend-core/src/utils/tests/utils.spec.ts

@@ -44,11 +44,11 @@ describe("utils", () => {
 
     it("gets appId from url", async () => {
       await config.doInTenant(async () => {
-        const url = "http://test.com"
+        const url = "http://example.com"
         env._set("PLATFORM_URL", url)
 
         const ctx = structures.koa.newContext()
-        ctx.host = `${config.tenantId}.test.com`
+        ctx.host = `${config.tenantId}.example.com`
 
         const expected = db.generateAppID(config.tenantId)
         const app = structures.apps.app(expected)
@@ -89,7 +89,7 @@ describe("utils", () => {
       const ctx = structures.koa.newContext()
       const expected = db.generateAppID()
       ctx.request.headers = {
-        referer: `http://test.com/builder/app/${expected}/design/screen_123/screens`,
+        referer: `http://example.com/builder/app/${expected}/design/screen_123/screens`,
       }
 
       const actual = await utils.getAppIdFromCtx(ctx)
@@ -100,7 +100,7 @@ describe("utils", () => {
       const ctx = structures.koa.newContext()
       const appId = db.generateAppID()
       ctx.request.headers = {
-        referer: `http://test.com/foo/app/${appId}/bar`,
+        referer: `http://example.com/foo/app/${appId}/bar`,
       }
 
       const actual = await utils.getAppIdFromCtx(ctx)

packages/backend-core/tests/core/utilities/structures/common.ts

@@ -3,5 +3,5 @@ import { v4 as uuid } from "uuid"
 export { v4 as uuid } from "uuid"
 
 export const email = () => {
-  return `${uuid()}@test.com`
+  return `${uuid()}@example.com`
 }

packages/backend-core/tests/core/utilities/structures/sso.ts

@@ -61,7 +61,7 @@ export function ssoProfile(user?: User): SSOProfile {
     },
     _json: {
       email: user.email,
-      picture: "http://test.com",
+      picture: "http://example.com",
     },
     provider: generator.string(),
   }

packages/backend-core/tests/core/utilities/structures/users.ts

@@ -25,7 +25,7 @@ export const user = (userProps?: Partial<Omit<User, "userId">>): User => {
     roles: { app_test: "admin" },
     firstName: generator.first(),
     lastName: generator.last(),
-    pictureUrl: "http://test.com",
+    pictureUrl: "http://example.com",
     tenantId: tenant.id(),
     ...userProps,
   }

packages/bbui/src/Form/Core/EnvDropdown.svelte

@@ -184,7 +184,7 @@
       {#if environmentVariablesEnabled}
         <div on:click={() => showModal()} class="add-variable">
           <svg
-            class="spectrum-Icon spectrum-Icon--sizeS "
+            class="spectrum-Icon spectrum-Icon--sizeS"
             focusable="false"
             aria-hidden="true"
           >
@@ -195,7 +195,7 @@
       {:else}
         <div on:click={() => handleUpgradePanel()} class="add-variable">
           <svg
-            class="spectrum-Icon spectrum-Icon--sizeS "
+            class="spectrum-Icon spectrum-Icon--sizeS"
             focusable="false"
             aria-hidden="true"
           >

packages/builder/package.json

@@ -23,7 +23,6 @@
       "\\.(jpg|jpeg|png|gif|eot|otf|webp|svg|ttf|woff|woff2|mp4|webm|wav|mp3|m4a|aac|oga)$": "<rootDir>/internals/mocks/fileMock.js",
       "\\.(css|less|sass|scss)$": "identity-obj-proxy",
       "components(.*)$": "<rootDir>/src/components$1",
-      "builderStore(.*)$": "<rootDir>/src/builderStore$1",
       "stores(.*)$": "<rootDir>/src/stores$1",
       "analytics(.*)$": "<rootDir>/src/analytics$1",
       "constants/backend": "<rootDir>/src/constants/backend/index.js"

packages/builder/src/api.js

@@ -3,14 +3,14 @@ import {
   CookieUtils,
   Constants,
 } from "@budibase/frontend-core"
-import { store } from "./builderStore"
+import { appStore } from "stores/builder"
 import { get } from "svelte/store"
 import { auth, navigation } from "./stores/portal"
 
 export const API = createAPIClient({
   attachHeaders: headers => {
     // Attach app ID header from store
-    let appId = get(store).appId
+    let appId = get(appStore).appId
     if (appId) {
       headers["x-budibase-app-id"] = appId
     }

packages/builder/src/builderStore/index.js

@@ -1,158 +0,0 @@
-import { getFrontendStore } from "./store/frontend"
-import { getAutomationStore } from "./store/automation"
-import { getTemporalStore } from "./store/temporal"
-import { getThemeStore } from "./store/theme"
-import { getUserStore } from "./store/users"
-import { getDeploymentStore } from "./store/deployments"
-import { derived, get } from "svelte/store"
-import { findComponent, findComponentPath } from "./componentUtils"
-import { RoleUtils } from "@budibase/frontend-core"
-import { createHistoryStore } from "builderStore/store/history"
-import { cloneDeep } from "lodash/fp"
-import { getHoverStore } from "./store/hover"
-
-export const store = getFrontendStore()
-export const automationStore = getAutomationStore()
-export const themeStore = getThemeStore()
-export const temporalStore = getTemporalStore()
-export const userStore = getUserStore()
-export const deploymentStore = getDeploymentStore()
-export const hoverStore = getHoverStore()
-
-// Setup history for screens
-export const screenHistoryStore = createHistoryStore({
-  getDoc: id => get(store).screens?.find(screen => screen._id === id),
-  selectDoc: store.actions.screens.select,
-  afterAction: () => {
-    // Ensure a valid component is selected
-    if (!get(selectedComponent)) {
-      store.update(state => ({
-        ...state,
-        selectedComponentId: get(selectedScreen)?.props._id,
-      }))
-    }
-  },
-})
-store.actions.screens.save = screenHistoryStore.wrapSaveDoc(
-  store.actions.screens.save
-)
-store.actions.screens.delete = screenHistoryStore.wrapDeleteDoc(
-  store.actions.screens.delete
-)
-
-// Setup history for automations
-export const automationHistoryStore = createHistoryStore({
-  getDoc: automationStore.actions.getDefinition,
-  selectDoc: automationStore.actions.select,
-})
-automationStore.actions.save = automationHistoryStore.wrapSaveDoc(
-  automationStore.actions.save
-)
-automationStore.actions.delete = automationHistoryStore.wrapDeleteDoc(
-  automationStore.actions.delete
-)
-
-export const selectedScreen = derived(store, $store => {
-  return $store.screens.find(screen => screen._id === $store.selectedScreenId)
-})
-
-export const selectedLayout = derived(store, $store => {
-  return $store.layouts?.find(layout => layout._id === $store.selectedLayoutId)
-})
-
-export const selectedComponent = derived(
-  [store, selectedScreen],
-  ([$store, $selectedScreen]) => {
-    if (
-      $selectedScreen &&
-      $store.selectedComponentId?.startsWith(`${$selectedScreen._id}-`)
-    ) {
-      return $selectedScreen?.props
-    }
-    if (!$selectedScreen || !$store.selectedComponentId) {
-      return null
-    }
-    const selected = findComponent(
-      $selectedScreen?.props,
-      $store.selectedComponentId
-    )
-
-    const clone = selected ? cloneDeep(selected) : selected
-    store.actions.components.migrateSettings(clone)
-    return clone
-  }
-)
-
-// For legacy compatibility only, but with the new design UI this is just
-// the selected screen
-export const currentAsset = selectedScreen
-
-export const sortedScreens = derived(store, $store => {
-  return $store.screens.slice().sort((a, b) => {
-    // Sort by role first
-    const roleA = RoleUtils.getRolePriority(a.routing.roleId)
-    const roleB = RoleUtils.getRolePriority(b.routing.roleId)
-    if (roleA !== roleB) {
-      return roleA > roleB ? -1 : 1
-    }
-    // Then put home screens first
-    const homeA = !!a.routing.homeScreen
-    const homeB = !!b.routing.homeScreen
-    if (homeA !== homeB) {
-      return homeA ? -1 : 1
-    }
-    // Then sort alphabetically by each URL param
-    const aParams = a.routing.route.split("/")
-    const bParams = b.routing.route.split("/")
-    let minParams = Math.min(aParams.length, bParams.length)
-    for (let i = 0; i < minParams; i++) {
-      if (aParams[i] === bParams[i]) {
-        continue
-      }
-      return aParams[i] < bParams[i] ? -1 : 1
-    }
-    // Then sort by the fewest amount of URL params
-    return aParams.length < bParams.length ? -1 : 1
-  })
-})
-
-export const selectedComponentPath = derived(
-  [store, selectedScreen],
-  ([$store, $selectedScreen]) => {
-    return findComponentPath(
-      $selectedScreen?.props,
-      $store.selectedComponentId
-    ).map(component => component._id)
-  }
-)
-
-// Derived automation state
-export const selectedAutomation = derived(automationStore, $automationStore => {
-  if (!$automationStore.selectedAutomationId) {
-    return null
-  }
-  return $automationStore.automations?.find(
-    x => x._id === $automationStore.selectedAutomationId
-  )
-})
-
-// Derive map of resource IDs to other users.
-// We only ever care about a single user in each resource, so if multiple users
-// share the same datasource we can just overwrite them.
-export const userSelectedResourceMap = derived(userStore, $userStore => {
-  let map = {}
-  $userStore.forEach(user => {
-    const resource = user.builderMetadata?.selectedResourceId
-    if (resource) {
-      if (!map[resource]) {
-        map[resource] = []
-      }
-      map[resource].push(user)
-    }
-  })
-  return map
-})
-
-export const isOnlyUser = derived(userStore, $userStore => {
-  return $userStore.length < 2
-})

packages/builder/src/builderStore/store/hover.js

@@ -1,27 +0,0 @@
-import { get, writable } from "svelte/store"
-import { store as builder } from "builderStore"
-
-export const getHoverStore = () => {
-  const initialValue = {
-    componentId: null,
-  }
-
-  const store = writable(initialValue)
-
-  const update = (componentId, notifyClient = true) => {
-    if (componentId === get(store).componentId) {
-      return
-    }
-    store.update(state => {
-      state.componentId = componentId
-      return state
-    })
-    if (notifyClient) {
-      builder.actions.preview.sendEvent("hover-component", componentId)
-    }
-  }
-  return {
-    subscribe: store.subscribe,
-    actions: { update },
-  }
-}

packages/builder/src/builderStore/tests/dataBinding.test.js

@@ -1,545 +0,0 @@
-import { expect, describe, it, vi } from "vitest"
-import {
-  runtimeToReadableBinding,
-  readableToRuntimeBinding,
-  updateReferencesInObject,
-} from "../dataBinding"
-
-vi.mock("@budibase/frontend-core")
-vi.mock("builderStore/componentUtils")
-vi.mock("builderStore/store")
-vi.mock("builderStore/store/theme")
-vi.mock("builderStore/store/temporal")
-
-describe("runtimeToReadableBinding", () => {
-  const bindableProperties = [
-    {
-      category: "Current User",
-      icon: "User",
-      providerId: "user",
-      readableBinding: "Current User.firstName",
-      runtimeBinding: "[user].[firstName]",
-      type: "context",
-    },
-    {
-      category: "Bindings",
-      icon: "Brackets",
-      readableBinding: "Binding.count",
-      runtimeBinding: "count",
-      type: "context",
-    },
-  ]
-  it("should convert a runtime binding to a readable one", () => {
-    const textWithBindings = `Hello {{ [user].[firstName] }}! The count is {{ count }}.`
-    expect(
-      runtimeToReadableBinding(
-        bindableProperties,
-        textWithBindings,
-        "readableBinding"
-      )
-    ).toEqual(
-      `Hello {{ Current User.firstName }}! The count is {{ Binding.count }}.`
-    )
-  })
-
-  it("should not convert to readable binding if it is already readable", () => {
-    const textWithBindings = `Hello {{ [user].[firstName] }}! The count is {{ Binding.count }}.`
-    expect(
-      runtimeToReadableBinding(
-        bindableProperties,
-        textWithBindings,
-        "readableBinding"
-      )
-    ).toEqual(
-      `Hello {{ Current User.firstName }}! The count is {{ Binding.count }}.`
-    )
-  })
-})
-
-describe("readableToRuntimeBinding", () => {
-  const bindableProperties = [
-    {
-      category: "Current User",
-      icon: "User",
-      providerId: "user",
-      readableBinding: "Current User.firstName",
-      runtimeBinding: "[user].[firstName]",
-      type: "context",
-    },
-    {
-      category: "Bindings",
-      icon: "Brackets",
-      readableBinding: "Binding.count",
-      runtimeBinding: "count",
-      type: "context",
-    },
-  ]
-  it("should convert a readable binding to a runtime one", () => {
-    const textWithBindings = `Hello {{ Current User.firstName }}! The count is {{ Binding.count }}.`
-    expect(
-      readableToRuntimeBinding(
-        bindableProperties,
-        textWithBindings,
-        "runtimeBinding"
-      )
-    ).toEqual(`Hello {{ [user].[firstName] }}! The count is {{ count }}.`)
-  })
-})
-
-describe("updateReferencesInObject", () => {
-  it("should increment steps in sequence on 'add'", () => {
-    let obj = [
-      {
-        id: "a0",
-        parameters: {
-          text: "Alpha",
-        },
-      },
-      {
-        id: "a1",
-        parameters: {
-          text: "Apple",
-        },
-      },
-      {
-        id: "b2",
-        parameters: {
-          text: "Banana {{ actions.1.row }}",
-        },
-      },
-      {
-        id: "c3",
-        parameters: {
-          text: "Carrot {{ actions.1.row }}",
-        },
-      },
-      {
-        id: "d4",
-        parameters: {
-          text: "Dog {{ actions.3.row }}",
-        },
-      },
-      {
-        id: "e5",
-        parameters: {
-          text: "Eagle {{ actions.4.row }}",
-        },
-      },
-    ]
-    updateReferencesInObject({
-      obj,
-      modifiedIndex: 0,
-      action: "add",
-      label: "actions",
-    })
-
-    expect(obj).toEqual([
-      {
-        id: "a0",
-        parameters: {
-          text: "Alpha",
-        },
-      },
-      {
-        id: "a1",
-        parameters: {
-          text: "Apple",
-        },
-      },
-      {
-        id: "b2",
-        parameters: {
-          text: "Banana {{ actions.2.row }}",
-        },
-      },
-      {
-        id: "c3",
-        parameters: {
-          text: "Carrot {{ actions.2.row }}",
-        },
-      },
-      {
-        id: "d4",
-        parameters: {
-          text: "Dog {{ actions.4.row }}",
-        },
-      },
-      {
-        id: "e5",
-        parameters: {
-          text: "Eagle {{ actions.5.row }}",
-        },
-      },
-    ])
-  })
-
-  it("should decrement steps in sequence on 'delete'", () => {
-    let obj = [
-      {
-        id: "a1",
-        parameters: {
-          text: "Apple",
-        },
-      },
-      {
-        id: "b2",
-        parameters: {
-          text: "Banana {{ actions.1.row }}",
-        },
-      },
-      {
-        id: "d4",
-        parameters: {
-          text: "Dog {{ actions.3.row }}",
-        },
-      },
-      {
-        id: "e5",
-        parameters: {
-          text: "Eagle {{ actions.4.row }}",
-        },
-      },
-    ]
-    updateReferencesInObject({
-      obj,
-      modifiedIndex: 2,
-      action: "delete",
-      label: "actions",
-    })
-
-    expect(obj).toEqual([
-      {
-        id: "a1",
-        parameters: {
-          text: "Apple",
-        },
-      },
-      {
-        id: "b2",
-        parameters: {
-          text: "Banana {{ actions.1.row }}",
-        },
-      },
-      {
-        id: "d4",
-        parameters: {
-          text: "Dog {{ actions.2.row }}",
-        },
-      },
-      {
-        id: "e5",
-        parameters: {
-          text: "Eagle {{ actions.3.row }}",
-        },
-      },
-    ])
-  })
-
-  it("should handle on 'move' to a lower index", () => {
-    let obj = [
-      {
-        id: "a1",
-        parameters: {
-          text: "Apple",
-        },
-      },
-      {
-        id: "b2",
-        parameters: {
-          text: "Banana {{ actions.0.row }}",
-        },
-      },
-      {
-        id: "e5",
-        parameters: {
-          text: "Eagle {{ actions.3.row }}",
-        },
-      },
-      {
-        id: "c3",
-        parameters: {
-          text: "Carrot {{ actions.0.row }}",
-        },
-      },
-      {
-        id: "d4",
-        parameters: {
-          text: "Dog {{ actions.2.row }}",
-        },
-      },
-    ]
-    updateReferencesInObject({
-      obj,
-      modifiedIndex: 2,
-      action: "move",
-      label: "actions",
-      originalIndex: 4,
-    })
-
-    expect(obj).toEqual([
-      {
-        id: "a1",
-        parameters: {
-          text: "Apple",
-        },
-      },
-      {
-        id: "b2",
-        parameters: {
-          text: "Banana {{ actions.0.row }}",
-        },
-      },
-      {
-        id: "e5",
-        parameters: {
-          text: "Eagle {{ actions.4.row }}",
-        },
-      },
-      {
-        id: "c3",
-        parameters: {
-          text: "Carrot {{ actions.0.row }}",
-        },
-      },
-      {
-        id: "d4",
-        parameters: {
-          text: "Dog {{ actions.3.row }}",
-        },
-      },
-    ])
-  })
-
-  it("should handle on 'move' to a higher index", () => {
-    let obj = [
-      {
-        id: "b2",
-        parameters: {
-          text: "Banana {{ actions.0.row }}",
-        },
-      },
-      {
-        id: "c3",
-        parameters: {
-          text: "Carrot {{ actions.0.row }}",
-        },
-      },
-      {
-        id: "a1",
-        parameters: {
-          text: "Apple",
-        },
-      },
-      {
-        id: "d4",
-        parameters: {
-          text: "Dog {{ actions.2.row }}",
-        },
-      },
-      {
-        id: "e5",
-        parameters: {
-          text: "Eagle {{ actions.3.row }}",
-        },
-      },
-    ]
-    updateReferencesInObject({
-      obj,
-      modifiedIndex: 2,
-      action: "move",
-      label: "actions",
-      originalIndex: 0,
-    })
-
-    expect(obj).toEqual([
-      {
-        id: "b2",
-        parameters: {
-          text: "Banana {{ actions.2.row }}",
-        },
-      },
-      {
-        id: "c3",
-        parameters: {
-          text: "Carrot {{ actions.2.row }}",
-        },
-      },
-      {
-        id: "a1",
-        parameters: {
-          text: "Apple",
-        },
-      },
-      {
-        id: "d4",
-        parameters: {
-          text: "Dog {{ actions.1.row }}",
-        },
-      },
-      {
-        id: "e5",
-        parameters: {
-          text: "Eagle {{ actions.3.row }}",
-        },
-      },
-    ])
-  })
-
-  it("should handle on 'move' of action being referenced, dragged to a higher index", () => {
-    let obj = [
-      {
-        "##eventHandlerType": "Validate Form",
-        id: "cCD0Dwcnq",
-      },
-      {
-        "##eventHandlerType": "Close Screen Modal",
-        id: "3fbbIOfN0H",
-      },
-      {
-        "##eventHandlerType": "Save Row",
-        parameters: {
-          tableId: "ta_bb_employee",
-        },
-        id: "aehg5cTmhR",
-      },
-      {
-        "##eventHandlerType": "Close Side Panel",
-        id: "mzkpf86cxo",
-      },
-      {
-        "##eventHandlerType": "Navigate To",
-        id: "h0uDFeJa8A",
-      },
-      {
-        parameters: {
-          autoDismiss: true,
-          type: "success",
-          message: "{{ actions.1.row }}",
-        },
-        "##eventHandlerType": "Show Notification",
-        id: "JEI5lAyJZ",
-      },
-    ]
-    updateReferencesInObject({
-      obj,
-      modifiedIndex: 2,
-      action: "move",
-      label: "actions",
-      originalIndex: 1,
-    })
-
-    expect(obj).toEqual([
-      {
-        "##eventHandlerType": "Validate Form",
-        id: "cCD0Dwcnq",
-      },
-      {
-        "##eventHandlerType": "Close Screen Modal",
-        id: "3fbbIOfN0H",
-      },
-      {
-        "##eventHandlerType": "Save Row",
-        parameters: {
-          tableId: "ta_bb_employee",
-        },
-        id: "aehg5cTmhR",
-      },
-      {
-        "##eventHandlerType": "Close Side Panel",
-        id: "mzkpf86cxo",
-      },
-      {
-        "##eventHandlerType": "Navigate To",
-        id: "h0uDFeJa8A",
-      },
-      {
-        parameters: {
-          autoDismiss: true,
-          type: "success",
-          message: "{{ actions.2.row }}",
-        },
-        "##eventHandlerType": "Show Notification",
-        id: "JEI5lAyJZ",
-      },
-    ])
-  })
-
-  it("should handle on 'move' of action being referenced, dragged to a lower index", () => {
-    let obj = [
-      {
-        "##eventHandlerType": "Save Row",
-        parameters: {
-          tableId: "ta_bb_employee",
-        },
-        id: "aehg5cTmhR",
-      },
-      {
-        "##eventHandlerType": "Validate Form",
-        id: "cCD0Dwcnq",
-      },
-      {
-        "##eventHandlerType": "Close Screen Modal",
-        id: "3fbbIOfN0H",
-      },
-      {
-        "##eventHandlerType": "Close Side Panel",
-        id: "mzkpf86cxo",
-      },
-      {
-        "##eventHandlerType": "Navigate To",
-        id: "h0uDFeJa8A",
-      },
-      {
-        parameters: {
-          autoDismiss: true,
-          type: "success",
-          message: "{{ actions.4.row }}",
-        },
-        "##eventHandlerType": "Show Notification",
-        id: "JEI5lAyJZ",
-      },
-    ]
-    updateReferencesInObject({
-      obj,
-      modifiedIndex: 0,
-      action: "move",
-      label: "actions",
-      originalIndex: 4,
-    })
-
-    expect(obj).toEqual([
-      {
-        "##eventHandlerType": "Save Row",
-        parameters: {
-          tableId: "ta_bb_employee",
-        },
-        id: "aehg5cTmhR",
-      },
-      {
-        "##eventHandlerType": "Validate Form",
-        id: "cCD0Dwcnq",
-      },
-      {
-        "##eventHandlerType": "Close Screen Modal",
-        id: "3fbbIOfN0H",
-      },
-      {
-        "##eventHandlerType": "Close Side Panel",
-        id: "mzkpf86cxo",
-      },
-      {
-        "##eventHandlerType": "Navigate To",
-        id: "h0uDFeJa8A",
-      },
-      {
-        parameters: {
-          autoDismiss: true,
-          type: "success",
-          message: "{{ actions.0.row }}",
-        },
-        "##eventHandlerType": "Show Notification",
-        id: "JEI5lAyJZ",
-      },
-    ])
-  })
-})

packages/builder/src/components/automation/AutomationBuilder/AutomationBuilder.svelte

@@ -1,5 +1,5 @@
 <script>
-  import { selectedAutomation } from "builderStore"
+  import { selectedAutomation } from "stores/builder"
   import Flowchart from "./FlowChart/FlowChart.svelte"
 </script>
 

packages/builder/src/components/automation/AutomationBuilder/FlowChart/ActionModal.svelte

@@ -9,11 +9,11 @@
     Tags,
     Tag,
   } from "@budibase/bbui"
-  import { automationStore, selectedAutomation } from "builderStore"
+  import { automationStore, selectedAutomation } from "stores/builder"
   import { admin, licensing } from "stores/portal"
   import { externalActions } from "./ExternalActions"
   import { TriggerStepID, ActionStepID } from "constants/backend/automations"
-  import { checkForCollectStep } from "builderStore/utils"
+  import { checkForCollectStep } from "helpers/utils"
 
   export let blockIdx
   export let lastStep
@@ -128,10 +128,10 @@
         >
           <div class="item-body">
             <img
-              width="20"
+              width={20}
-              height="20"
+              height={20}
               src={externalActions[action.stepId].icon}
-              alt="zapier"
+              alt={externalActions[action.stepId].name}
             />
             <span class="icon-spacing">
               <Body size="XS">

packages/builder/src/components/automation/AutomationBuilder/FlowChart/AutomationBlockTagline.svelte

@@ -1,7 +1,7 @@
 <script>
   import { processStringSync } from "@budibase/string-templates"
   import { get } from "lodash/fp"
-  import { tables } from "stores/backend"
+  import { tables } from "stores/builder"
 
   export let block
 

packages/builder/src/components/automation/AutomationBuilder/FlowChart/ExternalActions.js

@@ -1,5 +1,6 @@
 import DiscordLogo from "assets/discord.svg"
 import ZapierLogo from "assets/zapier.png"
+import n8nLogo from "assets/n8n_square.png"
 import MakeLogo from "assets/make.svg"
 import SlackLogo from "assets/slack.svg"
 
@@ -8,4 +9,5 @@ export const externalActions = {
   discord: { name: "discord", icon: DiscordLogo },
   slack: { name: "slack", icon: SlackLogo },
   integromat: { name: "integromat", icon: MakeLogo },
+  n8n: { name: "n8n", icon: n8nLogo },
 }

packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowChart.svelte

@@ -3,7 +3,7 @@
     automationStore,
     selectedAutomation,
     automationHistoryStore,
-  } from "builderStore"
+  } from "stores/builder"
   import ConfirmDialog from "components/common/ConfirmDialog.svelte"
   import FlowItem from "./FlowItem.svelte"
   import TestDataModal from "./TestDataModal.svelte"

packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowItem.svelte

@@ -1,5 +1,9 @@
 <script>
-  import { automationStore, selectedAutomation } from "builderStore"
+  import {
+    automationStore,
+    selectedAutomation,
+    permissions,
+  } from "stores/builder"
   import {
     Icon,
     Divider,
@@ -17,7 +21,6 @@
   import FlowItemHeader from "./FlowItemHeader.svelte"
   import RoleSelect from "components/design/settings/controls/RoleSelect.svelte"
   import { ActionStepID, TriggerStepID } from "constants/backend/automations"
-  import { permissions } from "stores/backend"
 
   export let block
   export let testDataModal

packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowItemHeader.svelte

@@ -1,6 +1,6 @@
 <script>
-  import { automationStore, selectedAutomation } from "builderStore"
+  import { automationStore, selectedAutomation } from "stores/builder"
-  import { Icon, Body, StatusLight, AbsTooltip } from "@budibase/bbui"
+  import { Icon, Body, AbsTooltip, StatusLight } from "@budibase/bbui"
   import { externalActions } from "./ExternalActions"
   import { createEventDispatcher } from "svelte"
   import { Features } from "constants/backend/automations"

packages/builder/src/components/automation/AutomationBuilder/FlowChart/TestDataModal.svelte

@@ -5,7 +5,7 @@
     notifications,
     ActionButton,
   } from "@budibase/bbui"
-  import { automationStore, selectedAutomation } from "builderStore"
+  import { automationStore, selectedAutomation } from "stores/builder"
   import AutomationBlockSetup from "../../SetupPanel/AutomationBlockSetup.svelte"
   import { cloneDeep } from "lodash/fp"
 

packages/builder/src/components/automation/AutomationBuilder/TestPanel.svelte

@@ -1,7 +1,7 @@
 <script>
   import { Icon, Divider } from "@budibase/bbui"
   import TestDisplay from "./TestDisplay.svelte"
-  import { automationStore } from "builderStore"
+  import { automationStore } from "stores/builder"
 
   export let automation
 </script>

packages/builder/src/components/automation/AutomationPanel/AutomationList.svelte

@@ -4,7 +4,7 @@
     automationStore,
     selectedAutomation,
     userSelectedResourceMap,
-  } from "builderStore"
+  } from "stores/builder"
   import NavItem from "components/common/NavItem.svelte"
   import EditAutomationPopover from "./EditAutomationPopover.svelte"
   import { notifications } from "@budibase/bbui"

packages/builder/src/components/automation/AutomationPanel/AutomationPanel.svelte

@@ -7,7 +7,7 @@
     automationStore,
     selectedAutomation,
     userSelectedResourceMap,
-  } from "builderStore"
+  } from "stores/builder"
   import NavItem from "components/common/NavItem.svelte"
   import EditAutomationPopover from "./EditAutomationPopover.svelte"
 

packages/builder/src/components/automation/AutomationPanel/CreateAutomationModal.svelte

@@ -1,5 +1,5 @@
 <script>
-  import { automationStore } from "builderStore"
+  import { automationStore } from "stores/builder"
   import {
     notifications,
     Input,

packages/builder/src/components/automation/AutomationPanel/EditAutomationPopover.svelte

@@ -1,5 +1,5 @@
 <script>
-  import { automationStore } from "builderStore"
+  import { automationStore } from "stores/builder"
   import { ActionMenu, MenuItem, notifications, Icon } from "@budibase/bbui"
   import ConfirmDialog from "components/common/ConfirmDialog.svelte"
   import UpdateAutomationModal from "components/automation/AutomationPanel/UpdateAutomationModal.svelte"

packages/builder/src/components/automation/AutomationPanel/UpdateAutomationModal.svelte

@@ -1,5 +1,5 @@
 <script>
-  import { automationStore } from "builderStore"
+  import { automationStore } from "stores/builder"
   import {
     notifications,
     Icon,

packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte

@@ -17,8 +17,7 @@
     DatePicker,
   } from "@budibase/bbui"
   import CreateWebhookModal from "components/automation/Shared/CreateWebhookModal.svelte"
-  import { automationStore, selectedAutomation } from "builderStore"
+  import { automationStore, selectedAutomation, tables } from "stores/builder"
-  import { tables } from "stores/backend"
   import { environment, licensing } from "stores/portal"
   import WebhookDisplay from "../Shared/WebhookDisplay.svelte"
   import DrawerBindableInput from "../../common/bindings/DrawerBindableInput.svelte"
@@ -44,7 +43,7 @@
   import {
     getSchemaForDatasourcePlus,
     getEnvironmentBindings,
-  } from "builderStore/dataBinding"
+  } from "dataBinding"
   import { TriggerStepID, ActionStepID } from "constants/backend/automations"
   import { onMount } from "svelte"
   import { cloneDeep } from "lodash/fp"
@@ -80,6 +79,7 @@
     disableWrapping: true,
   })
   $: editingJs = codeMode === EditorModes.JS
+  $: requiredProperties = block.schema.inputs.required || []
 
   $: stepCompletions =
     codeMode === EditorModes.Handlebars
@@ -360,6 +360,11 @@
     )
   }
 
+  function getFieldLabel(key, value) {
+    const requiredSuffix = requiredProperties.includes(key) ? "*" : ""
+    return `${value.title || (key === "row" ? "Table" : key)} ${requiredSuffix}`
+  }
+
   onMount(async () => {
     try {
       await environment.loadVariables()
@@ -377,7 +382,7 @@
           <Label
             tooltip={value.title === "Binding / Value"
               ? "If using the String input type, please use a comma or newline separated string"
-              : null}>{value.title || (key === "row" ? "Table" : key)}</Label
+              : null}>{getFieldLabel(key, value)}</Label
           >
         {/if}
         <div class:field-width={shouldRenderField(value)}>

packages/builder/src/components/automation/SetupPanel/AutomationSelector.svelte

@@ -1,7 +1,7 @@
 <script>
   import { Select, Label } from "@budibase/bbui"
   import { createEventDispatcher } from "svelte"
-  import { automationStore, selectedAutomation } from "builderStore"
+  import { automationStore, selectedAutomation } from "stores/builder"
   import { TriggerStepID } from "constants/backend/automations"
   import DrawerBindableInput from "../../common/bindings/DrawerBindableInput.svelte"
   import AutomationBindingPanel from "../../common/bindings/ServerBindingPanel.svelte"

packages/builder/src/components/automation/SetupPanel/CronBuilder.svelte

@@ -1,16 +1,35 @@
 <script>
   import { Button, Select, Input, Label } from "@budibase/bbui"
   import { onMount, createEventDispatcher } from "svelte"
-  import { flags } from "stores/backend"
+  import { flags } from "stores/builder"
+  import { helpers, REBOOT_CRON } from "@budibase/shared-core"
 
   const dispatch = createEventDispatcher()
 
   export let value
+  let error
+
+  $: {
+    const exists = CRON_EXPRESSIONS.some(cron => cron.value === value)
+    const customIndex = CRON_EXPRESSIONS.findIndex(
+      cron => cron.label === "Custom"
+    )
+
+    if (!exists && customIndex === -1) {
+      CRON_EXPRESSIONS[0] = { label: "Custom", value: value }
+    } else if (exists && customIndex !== -1) {
+      CRON_EXPRESSIONS.splice(customIndex, 1)
+    }
+  }
 
   const onChange = e => {
-    if (e.detail === value) {
+    if (value !== REBOOT_CRON) {
+      error = helpers.cron.validate(e.detail).err
+    }
+    if (e.detail === value || error) {
       return
     }
+
     value = e.detail
     dispatch("change", e.detail)
   }
@@ -41,7 +60,7 @@
     if (!$flags.cloud) {
       CRON_EXPRESSIONS.push({
         label: "Every Budibase Reboot",
-        value: "@reboot",
+        value: REBOOT_CRON,
       })
     }
   })
@@ -49,6 +68,7 @@
 
 <div class="block-field">
   <Input
+    {error}
     on:change={onChange}
     {value}
     on:blur={() => (touched = true)}
@@ -64,7 +84,7 @@
     {#if presets}
       <Select
         on:change={onChange}
-        {value}
+        value={value || "Custom"}
         secondary
         extraThin
         label="Presets"

packages/builder/src/components/automation/SetupPanel/QueryParamSelector.svelte

@@ -1,6 +1,6 @@
 <script>
   import { createEventDispatcher } from "svelte"
-  import { queries } from "stores/backend"
+  import { queries } from "stores/builder"
   import { Select, Label } from "@budibase/bbui"
   import DrawerBindableInput from "../../common/bindings/DrawerBindableInput.svelte"
   import AutomationBindingPanel from "../../common/bindings/ServerBindingPanel.svelte"

packages/builder/src/components/automation/SetupPanel/QuerySelector.svelte

@@ -1,5 +1,5 @@
 <script>
-  import { queries } from "stores/backend"
+  import { queries } from "stores/builder"
   import { Select } from "@budibase/bbui"
 
   export let value

packages/builder/src/components/automation/SetupPanel/RowSelector.svelte

@@ -1,5 +1,5 @@
 <script>
-  import { tables } from "stores/backend"
+  import { tables } from "stores/builder"
   import { Select, Checkbox, Label } from "@budibase/bbui"
   import { createEventDispatcher } from "svelte"
   import RowSelectorTypes from "./RowSelectorTypes.svelte"

packages/builder/src/components/automation/SetupPanel/TableSelector.svelte

@@ -1,5 +1,5 @@
 <script>
-  import { tables } from "stores/backend"
+  import { tables } from "stores/builder"
   import { Select } from "@budibase/bbui"
   import { createEventDispatcher } from "svelte"
   import { TableNames } from "constants"

packages/builder/src/components/automation/Shared/CreateWebhookModal.svelte

@@ -1,6 +1,6 @@
 <script>
   import { Icon, notifications, ModalContent } from "@budibase/bbui"
-  import { automationStore, selectedAutomation } from "builderStore"
+  import { automationStore, selectedAutomation } from "stores/builder"
   import WebhookDisplay from "./WebhookDisplay.svelte"
   import { onMount, onDestroy } from "svelte"
 

packages/builder/src/components/backend/DataTable/RelationshipDataTable.svelte

@@ -1,7 +1,7 @@
 <script>
   import { API } from "api"
   import Table from "./Table.svelte"
-  import { tables } from "stores/backend"
+  import { tables } from "stores/builder"
   import { notifications } from "@budibase/bbui"
 
   export let tableId

packages/builder/src/components/backend/DataTable/TableDataTable.svelte

@@ -1,10 +1,9 @@
 <script>
-  import { datasources, tables, integrations } from "stores/backend"
+  import { datasources, tables, integrations, appStore } from "stores/builder"
   import EditRolesButton from "./buttons/EditRolesButton.svelte"
   import { TableNames } from "constants"
   import { Grid } from "@budibase/frontend-core"
   import { API } from "api"
-  import { store } from "builderStore"
   import GridAddColumnModal from "components/backend/DataTable/modals/grid/GridCreateColumnModal.svelte"
   import GridCreateEditRowModal from "components/backend/DataTable/modals/grid/GridCreateEditRowModal.svelte"
   import GridEditUserModal from "components/backend/DataTable/modals/grid/GridEditUserModal.svelte"
@@ -59,14 +58,14 @@
     datasource={gridDatasource}
     canAddRows={!isUsersTable}
     canDeleteRows={!isUsersTable}
-    canEditRows={!isUsersTable || !$store.features.disableUserMetadata}
+    canEditRows={!isUsersTable || !$appStore.features.disableUserMetadata}
-    canEditColumns={!isUsersTable || !$store.features.disableUserMetadata}
+    canEditColumns={!isUsersTable || !$appStore.features.disableUserMetadata}
     schemaOverrides={isUsersTable ? userSchemaOverrides : null}
     showAvatars={false}
     on:updatedatasource={handleGridTableUpdate}
   >
     <svelte:fragment slot="filter">
-      {#if isUsersTable && $store.features.disableUserMetadata}
+      {#if isUsersTable && $appStore.features.disableUserMetadata}
         <GridUsersTableButton />
       {/if}
       <GridFilterButton />

packages/builder/src/components/backend/DataTable/ViewDataTable.svelte

@@ -1,6 +1,6 @@
 <script>
   import { API } from "api"
-  import { tables } from "stores/backend"
+  import { tables } from "stores/builder"
 
   import Table from "./Table.svelte"
   import CalculateButton from "./buttons/CalculateButton.svelte"

packages/builder/src/components/backend/DataTable/ViewV2DataTable.svelte

@@ -1,5 +1,5 @@
 <script>
-  import { viewsV2 } from "stores/backend"
+  import { viewsV2 } from "stores/builder"
   import { Grid } from "@budibase/frontend-core"
   import { API } from "api"
   import GridCreateEditRowModal from "components/backend/DataTable/modals/grid/GridCreateEditRowModal.svelte"

packages/builder/src/components/backend/DataTable/buttons/ExistingRelationshipButton.svelte

@@ -1,10 +1,8 @@
 <script>
   import { ActionButton, notifications } from "@budibase/bbui"
   import CreateEditRelationshipModal from "../../Datasources/CreateEditRelationshipModal.svelte"
-  import {
+
-    datasources,
+  import { datasources, tables as tablesStore } from "stores/builder"
-    tables as tablesStore,
-  } from "../../../../stores/backend"
   import { createEventDispatcher } from "svelte"
 
   export let table

packages/builder/src/components/backend/DataTable/buttons/ManageAccessButton.svelte

@@ -1,6 +1,6 @@
 <script>
   import { ActionButton, Modal } from "@budibase/bbui"
-  import { permissions } from "stores/backend"
+  import { permissions } from "stores/builder"
   import ManageAccessModal from "../modals/ManageAccessModal.svelte"
 
   export let resourceId

packages/builder/src/components/backend/DataTable/cells/RoleCell.svelte

@@ -1,5 +1,5 @@
 <script>
-  import { roles } from "stores/backend"
+  import { roles } from "stores/builder"
 
   export let value
 

packages/builder/src/components/backend/DataTable/formula.js

@@ -1,5 +1,5 @@
 import { FIELDS } from "constants/backend"
-import { tables } from "stores/backend"
+import { tables } from "stores/builder"
 import { get as svelteGet } from "svelte/store"
 
 // currently supported level of relationship depth (server side)

packages/builder/src/components/backend/DataTable/modals/CalculateModal.svelte

@@ -1,6 +1,6 @@
 <script>
   import { Select, Label, notifications, ModalContent } from "@budibase/bbui"
-  import { tables, views } from "stores/backend"
+  import { tables, views } from "stores/builder"
   import { FIELDS } from "constants/backend"
 
   const CALCULATIONS = [

packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte

@@ -15,7 +15,7 @@
   } from "@budibase/bbui"
   import { createEventDispatcher, getContext, onMount } from "svelte"
   import { cloneDeep } from "lodash/fp"
-  import { tables, datasources } from "stores/backend"
+  import { tables, datasources } from "stores/builder"
   import { TableNames, UNEDITABLE_USER_FIELDS } from "constants"
   import {
     FIELDS,
@@ -28,7 +28,7 @@
     PrettyRelationshipDefinitions,
     DB_TYPE_EXTERNAL,
   } from "constants/backend"
-  import { getAutoColumnInformation, buildAutoColumn } from "builderStore/utils"
+  import { getAutoColumnInformation, buildAutoColumn } from "helpers/utils"
   import ConfirmDialog from "components/common/ConfirmDialog.svelte"
   import ModalBindableInput from "components/common/bindings/ModalBindableInput.svelte"
   import { getBindings } from "components/backend/DataTable/formula"

packages/builder/src/components/backend/DataTable/modals/CreateEditRow.svelte

@@ -1,6 +1,6 @@
 <script>
   import { createEventDispatcher } from "svelte"
-  import { tables } from "stores/backend"
+  import { tables } from "stores/builder"
   import { ModalContent, keepOpen, notifications } from "@budibase/bbui"
   import RowFieldControl from "../RowFieldControl.svelte"
   import { API } from "api"

packages/builder/src/components/backend/DataTable/modals/CreateEditUser.svelte

@@ -1,6 +1,6 @@
 <script>
   import { createEventDispatcher } from "svelte"
-  import { tables, roles } from "stores/backend"
+  import { tables, roles } from "stores/builder"
   import {
     notifications,
     keepOpen,

packages/builder/src/components/backend/DataTable/modals/EditRoles.svelte

@@ -10,7 +10,7 @@
   import { onMount } from "svelte"
   import { API } from "api"
   import ErrorsBox from "components/common/ErrorsBox.svelte"
-  import { roles } from "stores/backend"
+  import { roles } from "stores/builder"
 
   const BASE_ROLE = { _id: "", inherits: "BASIC", permissionId: "write" }
 

packages/builder/src/components/backend/DataTable/modals/FilterModal.svelte

@@ -10,7 +10,7 @@
     notifications,
     Icon,
   } from "@budibase/bbui"
-  import { tables, views } from "stores/backend"
+  import { tables, views } from "stores/builder"
 
   const CONDITIONS = [
     {

packages/builder/src/components/backend/DataTable/modals/GroupByModal.svelte

@@ -1,6 +1,6 @@
 <script>
   import { Select, ModalContent, notifications } from "@budibase/bbui"
-  import { tables, views } from "stores/backend"
+  import { tables, views } from "stores/builder"
   import { FIELDS } from "constants/backend"
 
   export let view = {}

packages/builder/src/components/backend/DataTable/modals/JSONSchemaModal.svelte

@@ -13,7 +13,7 @@
   } from "@budibase/bbui"
   import { onMount, createEventDispatcher } from "svelte"
   import { FIELDS } from "constants/backend"
-  import { generate } from "builderStore/schemaGenerator"
+  import { generate } from "helpers/schemaGenerator"
 
   export let schema = {}
   export let json

packages/builder/src/components/backend/DataTable/modals/ManageAccessModal.svelte

@@ -1,6 +1,6 @@
 <script>
   import { PermissionSource } from "@budibase/types"
-  import { roles, permissions as permissionsStore } from "stores/backend"
+  import { roles, permissions as permissionsStore } from "stores/builder"
   import {
     Label,
     Input,

packages/builder/src/components/backend/DataTable/modals/grid/GridCreateViewModal.svelte

@@ -2,7 +2,7 @@
   import { getContext } from "svelte"
   import { Input, notifications, ModalContent } from "@budibase/bbui"
   import { goto } from "@roxi/routify"
-  import { viewsV2 } from "stores/backend"
+  import { viewsV2 } from "stores/builder"
 
   const { filter, sort, definition } = getContext("grid")
 

packages/builder/src/components/backend/DataTable/modals/grid/GridUsersTableButton.svelte

@@ -1,15 +1,12 @@
 <script>
   import { ActionButton, Popover, Heading, Body, Button } from "@budibase/bbui"
-  import { store } from "builderStore"
+  import { builderStore } from "stores/builder"
 
   let anchor
   let open = false
 
   const openSidePanel = () => {
-    store.update(state => ({
+    builderStore.showBuilderSidePanel()
-      ...state,
-      builderSidePanel: true,
-    }))
     open = false
   }
 </script>

packages/builder/src/components/backend/DatasourceNavigator/DatasourceNavigator.svelte

@@ -3,13 +3,13 @@
   import { Layout } from "@budibase/bbui"
   import { BUDIBASE_INTERNAL_DB_ID } from "constants/backend"
   import {
-    database,
     datasources,
     queries,
     tables,
     views,
     viewsV2,
-  } from "stores/backend"
+    userSelectedResourceMap,
+  } from "stores/builder"
   import EditDatasourcePopover from "./popovers/EditDatasourcePopover.svelte"
   import EditQueryPopover from "./popovers/EditQueryPopover.svelte"
   import NavItem from "components/common/NavItem.svelte"
@@ -21,7 +21,6 @@
   } from "helpers/data/utils"
   import IntegrationIcon from "./IntegrationIcon.svelte"
   import { TableNames } from "constants"
-  import { userSelectedResourceMap } from "builderStore"
   import { enrichDatasources } from "./datasourceUtils"
   import { onMount } from "svelte"
 
@@ -75,69 +74,67 @@
     searchTerm && !showAppUsersTable && !enrichedDataSources.find(ds => ds.show)
 </script>
 
-{#if $database?._id}
+<div class="hierarchy-items-container">
-  <div class="hierarchy-items-container">
+  {#if showAppUsersTable}
-    {#if showAppUsersTable}
+    <NavItem
-      <NavItem
+      icon="UserGroup"
-        icon="UserGroup"
+      text={appUsersTableName}
-        text={appUsersTableName}
+      selected={$isActive("./table/:tableId") &&
-        selected={$isActive("./table/:tableId") &&
+        $tables.selected?._id === TableNames.USERS}
-          $tables.selected?._id === TableNames.USERS}
+      on:click={() => selectTable(TableNames.USERS)}
-        on:click={() => selectTable(TableNames.USERS)}
+      selectedBy={$userSelectedResourceMap[TableNames.USERS]}
-        selectedBy={$userSelectedResourceMap[TableNames.USERS]}
+    />
-      />
+  {/if}
-    {/if}
+  {#each enrichedDataSources.filter(ds => ds.show) as datasource}
-    {#each enrichedDataSources.filter(ds => ds.show) as datasource}
+    <NavItem
-      <NavItem
+      border
-        border
+      text={datasource.name}
-        text={datasource.name}
+      opened={datasource.open}
-        opened={datasource.open}
+      selected={$isActive("./datasource") && datasource.selected}
-        selected={$isActive("./datasource") && datasource.selected}
+      withArrow={true}
-        withArrow={true}
+      on:click={() => selectDatasource(datasource)}
-        on:click={() => selectDatasource(datasource)}
+      on:iconClick={() => toggleNode(datasource)}
-        on:iconClick={() => toggleNode(datasource)}
+      selectedBy={$userSelectedResourceMap[datasource._id]}
-        selectedBy={$userSelectedResourceMap[datasource._id]}
+    >
-      >
+      <div class="datasource-icon" slot="icon">
-        <div class="datasource-icon" slot="icon">
+        <IntegrationIcon
-          <IntegrationIcon
+          integrationType={datasource.source}
-            integrationType={datasource.source}
+          schema={datasource.schema}
-            schema={datasource.schema}
+          size="18"
-            size="18"
+        />
-          />
+      </div>
-        </div>
+      {#if datasource._id !== BUDIBASE_INTERNAL_DB_ID}
-        {#if datasource._id !== BUDIBASE_INTERNAL_DB_ID}
+        <EditDatasourcePopover {datasource} />
-          <EditDatasourcePopover {datasource} />
-        {/if}
-      </NavItem>
-
-      {#if datasource.open}
-        <TableNavigator tables={datasource.tables} {selectTable} />
-        {#each datasource.queries as query}
-          <NavItem
-            indentLevel={1}
-            icon="SQLQuery"
-            iconText={customQueryIconText(datasource, query)}
-            iconColor={customQueryIconColor(datasource, query)}
-            text={customQueryText(datasource, query)}
-            selected={$isActive("./query/:queryId") &&
-              $queries.selectedQueryId === query._id}
-            on:click={() => $goto(`./query/${query._id}`)}
-            selectedBy={$userSelectedResourceMap[query._id]}
-          >
-            <EditQueryPopover {query} />
-          </NavItem>
-        {/each}
       {/if}
-    {/each}
+    </NavItem>
-    {#if showNoResults}
+
-      <Layout paddingY="none" paddingX="L">
+    {#if datasource.open}
-        <div class="no-results">
+      <TableNavigator tables={datasource.tables} {selectTable} />
-          There aren't any datasources matching that name
+      {#each datasource.queries as query}
-        </div>
+        <NavItem
-      </Layout>
+          indentLevel={1}
+          icon="SQLQuery"
+          iconText={customQueryIconText(datasource, query)}
+          iconColor={customQueryIconColor(datasource, query)}
+          text={customQueryText(datasource, query)}
+          selected={$isActive("./query/:queryId") &&
+            $queries.selectedQueryId === query._id}
+          on:click={() => $goto(`./query/${query._id}`)}
+          selectedBy={$userSelectedResourceMap[query._id]}
+        >
+          <EditQueryPopover {query} />
+        </NavItem>
+      {/each}
     {/if}
-  </div>
+  {/each}
-{/if}
+  {#if showNoResults}
+    <Layout paddingY="none" paddingX="L">
+      <div class="no-results">
+        There aren't any datasources matching that name
+      </div>
+    </Layout>
+  {/if}
+</div>
 
 <style>
   .hierarchy-items-container {

packages/builder/src/components/backend/DatasourceNavigator/icons/index.js

@@ -16,7 +16,7 @@ import Firebase from "./Firebase.svelte"
 import Redis from "./Redis.svelte"
 import Snowflake from "./Snowflake.svelte"
 import Custom from "./Custom.svelte"
-import { integrations } from "stores/backend"
+import { integrations } from "stores/builder"
 import { get } from "svelte/store"
 
 const ICONS = {

packages/builder/src/components/backend/DatasourceNavigator/modals/ImportRestQueriesModal.svelte

@@ -12,7 +12,7 @@
     TextArea,
     Dropzone,
   } from "@budibase/bbui"
-  import { datasources, queries } from "stores/backend"
+  import { datasources, queries } from "stores/builder"
   import { writable } from "svelte/store"
 
   export let navigateDatasource = false

packages/builder/src/components/backend/DatasourceNavigator/modals/UpdateDatasourceModal.svelte

@@ -1,6 +1,6 @@
 <script>
   import { get } from "svelte/store"
-  import { datasources, integrations } from "stores/backend"
+  import { datasources, integrations } from "stores/builder"
   import { notifications, Input, ModalContent, Modal } from "@budibase/bbui"
   import { integrationForDatasource } from "stores/selectors"
 

packages/builder/src/components/backend/DatasourceNavigator/popovers/EditDatasourcePopover.svelte

@@ -1,6 +1,6 @@
 <script>
   import { goto } from "@roxi/routify"
-  import { datasources } from "stores/backend"
+  import { datasources } from "stores/builder"
   import { notifications, ActionMenu, MenuItem, Icon } from "@budibase/bbui"
   import ConfirmDialog from "components/common/ConfirmDialog.svelte"
   import UpdateDatasourceModal from "components/backend/DatasourceNavigator/modals/UpdateDatasourceModal.svelte"

packages/builder/src/components/backend/DatasourceNavigator/popovers/EditQueryPopover.svelte

@@ -2,7 +2,7 @@
   import { goto } from "@roxi/routify"
   import { ActionMenu, MenuItem, Icon, notifications } from "@budibase/bbui"
   import ConfirmDialog from "components/common/ConfirmDialog.svelte"
-  import { datasources, queries } from "stores/backend"
+  import { datasources, queries } from "stores/builder"
 
   export let query
 

packages/builder/src/components/backend/Datasources/CreateEditRelationship.svelte

@@ -13,7 +13,7 @@
     Body,
     Helpers,
   } from "@budibase/bbui"
-  import { tables } from "stores/backend"
+  import { tables } from "stores/builder"
   import { RelationshipErrorChecker } from "./relationshipErrors"
   import { onMount } from "svelte"
   import RelationshipSelector from "components/common/RelationshipSelector.svelte"

packages/builder/src/components/backend/Datasources/CreateEditRelationshipModal.svelte

@@ -2,7 +2,7 @@
   import { Modal } from "@budibase/bbui"
   import { get } from "svelte/store"
   import CreateEditRelationship from "./CreateEditRelationship.svelte"
-  import { integrations, datasources } from "stores/backend"
+  import { integrations, datasources } from "stores/builder"
   import { integrationForDatasource } from "stores/selectors"
 
   export let datasource

packages/builder/src/components/backend/Datasources/TableImportSelection/tableSelectionStore.js

@@ -1,6 +1,6 @@
 import { derived, writable, get } from "svelte/store"
 import { keepOpen, notifications } from "@budibase/bbui"
-import { datasources, tables } from "stores/backend"
+import { datasources, tables } from "stores/builder"
 
 export const createTableSelectionStore = (integration, datasource) => {
   const tableNamesStore = writable([])

packages/builder/src/components/backend/TableNavigator/TableNavigator.svelte

@@ -3,14 +3,13 @@
     tables as tablesStore,
     views,
     viewsV2,
-    database,
+    userSelectedResourceMap,
-  } from "stores/backend"
+  } from "stores/builder"
   import { TableNames } from "constants"
   import EditTablePopover from "./popovers/EditTablePopover.svelte"
   import EditViewPopover from "./popovers/EditViewPopover.svelte"
   import NavItem from "components/common/NavItem.svelte"
   import { goto, isActive } from "@roxi/routify"
-  import { userSelectedResourceMap } from "builderStore"
 
   export let tables
   export let selectTable
@@ -29,42 +28,40 @@
   }
 </script>
 
-{#if $database?._id}
+<div class="hierarchy-items-container">
-  <div class="hierarchy-items-container">
+  {#each sortedTables as table, idx}
-    {#each sortedTables as table, idx}
+    <NavItem
+      indentLevel={1}
+      border={idx > 0}
+      icon={table._id === TableNames.USERS ? "UserGroup" : "Table"}
+      text={table.name}
+      selected={$isActive("./table/:tableId") &&
+        $tablesStore.selected?._id === table._id}
+      on:click={() => selectTable(table._id)}
+      selectedBy={$userSelectedResourceMap[table._id]}
+    >
+      {#if table._id !== TableNames.USERS}
+        <EditTablePopover {table} />
+      {/if}
+    </NavItem>
+    {#each [...Object.entries(table.views || {})].sort() as [name, view], idx (idx)}
       <NavItem
-        indentLevel={1}
+        indentLevel={2}
-        border={idx > 0}
+        icon="Remove"
-        icon={table._id === TableNames.USERS ? "UserGroup" : "Table"}
+        text={name}
-        text={table.name}
+        selected={isViewActive(view, $isActive, $views, $viewsV2)}
-        selected={$isActive("./table/:tableId") &&
+        on:click={() => {
-          $tablesStore.selected?._id === table._id}
+          if (view.version === 2) {
-        on:click={() => selectTable(table._id)}
+            $goto(`./view/v2/${encodeURIComponent(view.id)}`)
-        selectedBy={$userSelectedResourceMap[table._id]}
+          } else {
+            $goto(`./view/v1/${encodeURIComponent(name)}`)
+          }
+        }}
+        selectedBy={$userSelectedResourceMap[name] ||
+          $userSelectedResourceMap[view.id]}
       >
-        {#if table._id !== TableNames.USERS}
+        <EditViewPopover {view} />
-          <EditTablePopover {table} />
-        {/if}
       </NavItem>
-      {#each [...Object.entries(table.views || {})].sort() as [name, view], idx (idx)}
-        <NavItem
-          indentLevel={2}
-          icon="Remove"
-          text={name}
-          selected={isViewActive(view, $isActive, $views, $viewsV2)}
-          on:click={() => {
-            if (view.version === 2) {
-              $goto(`./view/v2/${encodeURIComponent(view.id)}`)
-            } else {
-              $goto(`./view/v1/${encodeURIComponent(name)}`)
-            }
-          }}
-          selectedBy={$userSelectedResourceMap[name] ||
-            $userSelectedResourceMap[view.id]}
-        >
-          <EditViewPopover {view} />
-        </NavItem>
-      {/each}
     {/each}
-  </div>
+  {/each}
-{/if}
+</div>

packages/builder/src/components/backend/TableNavigator/modals/CreateTableModal.svelte

@@ -1,6 +1,6 @@
 <script>
   import { goto, url } from "@roxi/routify"
-  import { tables, datasources } from "stores/backend"
+  import { tables, datasources } from "stores/builder"
   import {
     notifications,
     Input,

packages/builder/src/components/backend/TableNavigator/popovers/EditTablePopover.svelte

@@ -1,8 +1,7 @@
 <script>
   import { goto, params } from "@roxi/routify"
-  import { store } from "builderStore"
   import { cloneDeep } from "lodash/fp"
-  import { tables, datasources } from "stores/backend"
+  import { tables, datasources, screenStore } from "stores/builder"
   import {
     ActionMenu,
     Icon,
@@ -32,7 +31,7 @@
   $: allowDeletion = !externalTable || table?.created
 
   function showDeleteModal() {
-    templateScreens = $store.screens.filter(
+    templateScreens = $screenStore.screens.filter(
       screen => screen.autoTableId === table._id
     )
     willBeDeleted = ["All table data"].concat(
@@ -47,7 +46,7 @@
       await tables.delete(table)
       // Screens need deleted one at a time because of undo/redo
       for (let screen of templateScreens) {
-        await store.actions.screens.delete(screen)
+        await screenStore.delete(screen)
       }
       if (table.sourceType === DB_TYPE_EXTERNAL) {
         await datasources.fetch()

packages/builder/src/components/backend/TableNavigator/popovers/EditViewPopover.svelte

@@ -1,5 +1,5 @@
 <script>
-  import { views, viewsV2 } from "stores/backend"
+  import { views, viewsV2 } from "stores/builder"
   import { cloneDeep } from "lodash/fp"
   import ConfirmDialog from "components/common/ConfirmDialog.svelte"
   import {

packages/builder/src/components/commandPalette/CommandPalette.svelte

@@ -10,12 +10,17 @@
   import { API } from "api"
   import { goto } from "@roxi/routify"
   import {
-    store,
-    sortedScreens,
     automationStore,
-    themeStore,
+    previewStore,
-  } from "builderStore"
+    builderStore,
-  import { datasources, queries, tables, views } from "stores/backend"
+    sortedScreens,
+    appStore,
+    datasources,
+    queries,
+    tables,
+    views,
+  } from "stores/builder"
+  import { themeStore } from "stores/portal"
   import { getContext } from "svelte"
   import { Constants } from "@budibase/frontend-core"
 
@@ -26,8 +31,7 @@
       name: "Invite users and manage app access",
       description: "",
       icon: "User",
-      action: () =>
+      action: () => builderStore.showBuilderSidePanel(),
-        store.update(state => ({ ...state, builderSidePanel: true })),
     },
     {
       type: "Navigate",
@@ -69,13 +73,13 @@
       name: "App",
       description: "",
       icon: "Play",
-      action: () => store.update(state => ({ ...state, showPreview: true })),
+      action: () => previewStore.showPreview(true),
     },
     {
       type: "Preview",
       name: "Published App",
       icon: "Play",
-      action: () => window.open(`/app${$store.url}`),
+      action: () => window.open(`/app${$appStore.url}`),
     },
     {
       type: "Support",
@@ -216,7 +220,7 @@
 
   async function deployApp() {
     try {
-      await API.publishAppChanges($store.appId)
+      await API.publishAppChanges($appStore.appId)
       notifications.success("App published successfully")
     } catch (error) {
       notifications.error("Error publishing app")

packages/builder/src/components/common/CodeEditor/CodeEditor.svelte

@@ -43,7 +43,7 @@
   import { Compartment } from "@codemirror/state"
   import { javascript } from "@codemirror/lang-javascript"
   import { EditorModes, getDefaultTheme } from "./"
-  import { themeStore } from "builderStore"
+  import { themeStore } from "stores/portal"
 
   export let label
   export let completions = []

packages/builder/src/components/common/CodeEditor/index.js

@@ -286,7 +286,13 @@ export const hbInsert = (value, from, to, text) => {
   return parsedInsert
 }
 
-export function jsInsert(value, from, to, text, { helper, disableWrapping }) {
+export function jsInsert(
+  value,
+  from,
+  to,
+  text,
+  { helper, disableWrapping } = {}
+) {
   let parsedInsert = ""
 
   const left = from ? value.substring(0, from) : ""

packages/builder/src/components/common/CodeMirrorEditor.svelte

@@ -27,7 +27,7 @@
 <script>
   import { Label } from "@budibase/bbui"
   import CodeMirror from "components/integration/codemirror"
-  import { themeStore } from "builderStore"
+  import { themeStore } from "stores/portal"
   import { createEventDispatcher, onMount } from "svelte"
 
   export let mode = EditorModes.JS

packages/builder/src/components/common/LinkedRowSelector.svelte

@@ -1,8 +1,8 @@
 <script>
-  import { tables } from "stores/backend"
+  import { tables } from "stores/builder"
   import { API } from "api"
   import { Select, Label, Multiselect } from "@budibase/bbui"
-  import { capitalise } from "../../helpers"
+  import { capitalise } from "helpers"
   import { createEventDispatcher } from "svelte"
 
   export let schema

packages/builder/src/components/common/RoleSelect.svelte

@@ -1,6 +1,6 @@
 <script>
   import { Select, FancySelect } from "@budibase/bbui"
-  import { roles } from "stores/backend"
+  import { roles } from "stores/builder"
   import { licensing } from "stores/portal"
 
   import { Constants, RoleUtils } from "@budibase/frontend-core"

packages/builder/src/components/common/bindings/BindableCombobox.svelte

@@ -3,7 +3,7 @@
   import {
     readableToRuntimeBinding,
     runtimeToReadableBinding,
-  } from "builderStore/dataBinding"
+  } from "dataBinding"
   import { createEventDispatcher } from "svelte"
   import { isJSBinding } from "@budibase/string-templates"
 

packages/builder/src/components/common/bindings/BindingPanel.svelte

@@ -19,7 +19,7 @@
   import {
     readableToRuntimeBinding,
     runtimeToReadableBinding,
-  } from "builderStore/dataBinding"
+  } from "dataBinding"
 
   import { admin } from "stores/portal"
   import CodeEditor from "../CodeEditor/CodeEditor.svelte"

packages/builder/src/components/common/bindings/DrawerBindableCombobox.svelte

@@ -3,7 +3,7 @@
   import {
     readableToRuntimeBinding,
     runtimeToReadableBinding,
-  } from "builderStore/dataBinding"
+  } from "dataBinding"
   import ClientBindingPanel from "components/common/bindings/ClientBindingPanel.svelte"
   import { createEventDispatcher, setContext } from "svelte"
   import { isJSBinding } from "@budibase/string-templates"

packages/builder/src/components/common/bindings/DrawerBindableInput.svelte

@@ -3,11 +3,12 @@
   import {
     readableToRuntimeBinding,
     runtimeToReadableBinding,
-  } from "builderStore/dataBinding"
+  } from "dataBinding"
 
   import ClientBindingPanel from "components/common/bindings/ClientBindingPanel.svelte"
   import { createEventDispatcher, setContext } from "svelte"
   import { isJSBinding } from "@budibase/string-templates"
+  import { builderStore } from "stores/builder"
 
   export let panel = ClientBindingPanel
   export let value = ""
@@ -21,6 +22,7 @@
   export let allowHelpers = true
   export let updateOnChange = true
   export let drawerLeft
+  export let key
   export let disableBindings = false
 
   const dispatch = createEventDispatcher()
@@ -35,6 +37,7 @@
   const saveBinding = () => {
     onChange(tempValue)
     onBlur()
+    builderStore.propertyFocus()
     bindingDrawer.hide()
   }
 
@@ -50,6 +53,11 @@
   const onBlur = () => {
     dispatch("blur", currentVal)
   }
+
+  const onDrawerHide = e => {
+    builderStore.propertyFocus()
+    dispatch("drawerHide", e.detail)
+  }
 </script>
 
 <div class="control" class:disabled>
@@ -67,6 +75,7 @@
     <div
       class="icon"
       on:click={() => {
+        builderStore.propertyFocus(key)
         bindingDrawer.show()
       }}
     >
@@ -75,7 +84,7 @@
   {/if}
 </div>
 <Drawer
-  on:drawerHide
+  on:drawerHide={onDrawerHide}
   on:drawerShow
   {fillWidth}
   bind:this={bindingDrawer}

packages/builder/src/components/common/bindings/DrawerBindableSlot.svelte

@@ -3,7 +3,7 @@
   import {
     readableToRuntimeBinding,
     runtimeToReadableBinding,
-  } from "builderStore/dataBinding"
+  } from "dataBinding"
 
   import ClientBindingPanel from "components/common/bindings/ClientBindingPanel.svelte"
   import { createEventDispatcher, setContext } from "svelte"