From ea9811b9864bacd7b520639723b2eaac68e2c0bb Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 13 Feb 2024 11:51:45 +0000 Subject: [PATCH 01/43] Update CouchDB image to bookworm and couchdb 3.3.3. --- hosting/couchdb/Dockerfile | 97 +++++++++++++++- hosting/couchdb/clouseau/clouseau.ini | 2 +- hosting/couchdb/couch/10-docker-default.ini | 8 ++ hosting/couchdb/couch/vm.args | 2 +- hosting/couchdb/docker-entrypoint.sh | 121 ++++++++++++++++++++ hosting/couchdb/runner.sh | 4 + hosting/single/runner.sh | 6 +- packages/server/src/jsRunner/index.ts | 4 + 8 files changed, 237 insertions(+), 7 deletions(-) create mode 100644 hosting/couchdb/couch/10-docker-default.ini create mode 100755 hosting/couchdb/docker-entrypoint.sh diff --git a/hosting/couchdb/Dockerfile b/hosting/couchdb/Dockerfile index f83df7038b..6f940d6c48 100644 --- a/hosting/couchdb/Dockerfile +++ b/hosting/couchdb/Dockerfile @@ -1,4 +1,95 @@ -FROM couchdb:3.2.1 +# Modified from https://github.com/apache/couchdb-docker/blob/main/3.2.1/Dockerfile +FROM debian:bookworm-slim + +# Add CouchDB user account to make sure the IDs are assigned consistently +RUN groupadd -g 5984 -r couchdb && useradd -u 5984 -d /opt/couchdb -g couchdb couchdb + +# be sure GPG and apt-transport-https are available and functional +RUN set -ex; \ + apt-get update; \ + apt-get install -y --no-install-recommends \ + apt-transport-https \ + ca-certificates \ + dirmngr \ + gnupg \ + ; \ + rm -rf /var/lib/apt/lists/* + +# grab gosu for easy step-down from root and tini for signal handling and zombie reaping +# see https://github.com/apache/couchdb-docker/pull/28#discussion_r141112407 +RUN set -eux; \ + apt-get update; \ + apt-get install -y --no-install-recommends gosu tini; \ + rm -rf /var/lib/apt/lists/*; \ + gosu nobody true; \ + tini --version + +# http://docs.couchdb.org/en/latest/install/unix.html#installing-the-apache-couchdb-packages +ENV GPG_COUCH_KEY \ +# gpg: rsa8192 205-01-19 The Apache Software Foundation (Package repository signing key) + 390EF70BB1EA12B2773962950EE62FB37A00258D +RUN set -eux; \ + apt-get update; \ + apt-get install -y curl; \ + export GNUPGHOME="$(mktemp -d)"; \ + curl -fL -o keys.asc https://couchdb.apache.org/repo/keys.asc; \ + gpg --batch --import keys.asc; \ + gpg --batch --export "${GPG_COUCH_KEY}" > /usr/share/keyrings/couchdb-archive-keyring.gpg; \ + command -v gpgconf && gpgconf --kill all || :; \ + rm -rf "$GNUPGHOME"; \ + apt-key list; \ + apt purge -y --autoremove curl; \ + rm -rf /var/lib/apt/lists/* + +ENV COUCHDB_VERSION 3.3.3 + +RUN . /etc/os-release; \ + echo "deb [signed-by=/usr/share/keyrings/couchdb-archive-keyring.gpg] https://apache.jfrog.io/artifactory/couchdb-deb/ ${VERSION_CODENAME} main" | \ + tee /etc/apt/sources.list.d/couchdb.list >/dev/null + +# https://github.com/apache/couchdb-pkg/blob/master/debian/README.Debian +RUN set -eux; \ + apt-get update; \ + \ + echo "couchdb couchdb/mode select none" | debconf-set-selections; \ +# we DO want recommends this time + DEBIAN_FRONTEND=noninteractive apt-get install -y --allow-downgrades --allow-remove-essential --allow-change-held-packages \ + couchdb="$COUCHDB_VERSION"~bookworm \ + ; \ +# Undo symlinks to /var/log and /var/lib + rmdir /var/lib/couchdb /var/log/couchdb; \ + rm /opt/couchdb/data /opt/couchdb/var/log; \ + mkdir -p /opt/couchdb/data /opt/couchdb/var/log; \ + chown couchdb:couchdb /opt/couchdb/data /opt/couchdb/var/log; \ + chmod 777 /opt/couchdb/data /opt/couchdb/var/log; \ +# Remove file that sets logging to a file + rm /opt/couchdb/etc/default.d/10-filelog.ini; \ +# Check we own everything in /opt/couchdb. Matches the command in dockerfile_entrypoint.sh + find /opt/couchdb \! \( -user couchdb -group couchdb \) -exec chown -f couchdb:couchdb '{}' +; \ +# Setup directories and permissions for config. Technically these could be 555 and 444 respectively +# but we keep them as 755 and 644 for consistency with CouchDB defaults and the dockerfile_entrypoint.sh. + find /opt/couchdb/etc -type d ! -perm 0755 -exec chmod -f 0755 '{}' +; \ + find /opt/couchdb/etc -type f ! -perm 0644 -exec chmod -f 0644 '{}' +; \ +# only local.d needs to be writable for the docker_entrypoint.sh + chmod -f 0777 /opt/couchdb/etc/local.d; \ +# apt clean-up + rm -rf /var/lib/apt/lists/*; + +# Add configuration +COPY --chown=couchdb:couchdb couch/10-docker-default.ini /opt/couchdb/etc/default.d/ +# COPY --chown=couchdb:couchdb vm.args /opt/couchdb/etc/ + +COPY docker-entrypoint.sh /usr/local/bin +RUN ln -s usr/local/bin/docker-entrypoint.sh /docker-entrypoint.sh # backwards compat +ENTRYPOINT ["tini", "--", "/docker-entrypoint.sh"] + +VOLUME /opt/couchdb/data + +# 5984: Main CouchDB endpoint +# 4369: Erlang portmap daemon (epmd) +# 9100: CouchDB cluster communication port +EXPOSE 5984 4369 9100 +# CMD ["/opt/couchdb/bin/couchdb"] ENV COUCHDB_USER admin ENV COUCHDB_PASSWORD admin @@ -6,9 +97,9 @@ EXPOSE 5984 RUN apt-get update && apt-get install -y --no-install-recommends software-properties-common wget unzip curl && \ wget -O - https://packages.adoptium.net/artifactory/api/gpg/key/public | apt-key add - && \ - apt-add-repository 'deb http://security.debian.org/debian-security bullseye-security/updates main' && \ + apt-add-repository 'deb http://security.debian.org/debian-security bookworm-security/updates main' && \ apt-add-repository 'deb http://archive.debian.org/debian stretch-backports main' && \ - apt-add-repository 'deb https://packages.adoptium.net/artifactory/deb bullseye main' && \ + apt-add-repository 'deb https://packages.adoptium.net/artifactory/deb bookworm main' && \ apt-get update && apt-get install -y --no-install-recommends temurin-8-jdk && \ rm -rf /var/lib/apt/lists/ diff --git a/hosting/couchdb/clouseau/clouseau.ini b/hosting/couchdb/clouseau/clouseau.ini index 578a5acafa..014fb854f3 100644 --- a/hosting/couchdb/clouseau/clouseau.ini +++ b/hosting/couchdb/clouseau/clouseau.ini @@ -4,7 +4,7 @@ name=clouseau@127.0.0.1 ; set this to the same distributed Erlang cookie used by the CouchDB nodes -cookie=monster +cookie=COUCHDB_ERLANG_COOKIE ; the path where you would like to store the search index files dir=DATA_DIR/search diff --git a/hosting/couchdb/couch/10-docker-default.ini b/hosting/couchdb/couch/10-docker-default.ini new file mode 100644 index 0000000000..1aa633cbfc --- /dev/null +++ b/hosting/couchdb/couch/10-docker-default.ini @@ -0,0 +1,8 @@ +; CouchDB Configuration Settings + +; Custom settings should be made in this file. They will override settings +; in default.ini, but unlike changes made to default.ini, this file won't be +; overwritten on server upgrade. + +[chttpd] +bind_address = any diff --git a/hosting/couchdb/couch/vm.args b/hosting/couchdb/couch/vm.args index e9e4416863..11845dd6d4 100644 --- a/hosting/couchdb/couch/vm.args +++ b/hosting/couchdb/couch/vm.args @@ -12,7 +12,7 @@ # erlang cookie for clouseau security -name couchdb@127.0.0.1 --setcookie monster +-setcookie COUCHDB_ERLANG_COOKIE # Ensure that the Erlang VM listens on a known port -kernel inet_dist_listen_min 9100 diff --git a/hosting/couchdb/docker-entrypoint.sh b/hosting/couchdb/docker-entrypoint.sh new file mode 100755 index 0000000000..8d6456d577 --- /dev/null +++ b/hosting/couchdb/docker-entrypoint.sh @@ -0,0 +1,121 @@ +#!/bin/bash +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. + +set -e + +# first arg is `-something` or `+something` +if [ "${1#-}" != "$1" ] || [ "${1#+}" != "$1" ]; then + set -- /opt/couchdb/bin/couchdb "$@" +fi + +# first arg is the bare word `couchdb` +if [ "$1" = 'couchdb' ]; then + shift + set -- /opt/couchdb/bin/couchdb "$@" +fi + +if [ "$1" = '/opt/couchdb/bin/couchdb' ]; then + # this is where runtime configuration changes will be written. + # we need to explicitly touch it here in case /opt/couchdb/etc has + # been mounted as an external volume, in which case it won't exist. + # If running as the couchdb user (i.e. container starts as root), + # write permissions will be granted below. + touch /opt/couchdb/etc/local.d/docker.ini + + # if user is root, assume running under the couchdb user (default) + # and ensure it is able to access files and directories that may be mounted externally + if [ "$(id -u)" = '0' ]; then + # Check that we own everything in /opt/couchdb and fix if necessary. We also + # add the `-f` flag in all the following invocations because there may be + # cases where some of these ownership and permissions issues are non-fatal + # (e.g. a config file owned by root with o+r is actually fine), and we don't + # to be too aggressive about crashing here ... + find /opt/couchdb \! \( -user couchdb -group couchdb \) -exec chown -f couchdb:couchdb '{}' + + + # Ensure that data files have the correct permissions. We were previously + # preventing any access to these files outside of couchdb:couchdb, but it + # turns out that CouchDB itself does not set such restrictive permissions + # when it creates the files. The approach taken here ensures that the + # contents of the datadir have the same permissions as they had when they + # were initially created. This should minimize any startup delay. + find /opt/couchdb/data -type d ! -perm 0755 -exec chmod -f 0755 '{}' + + find /opt/couchdb/data -type f ! -perm 0644 -exec chmod -f 0644 '{}' + + + # Do the same thing for configuration files and directories. Technically + # CouchDB only needs read access to the configuration files as all online + # changes will be applied to the "docker.ini" file below, but we set 644 + # for the sake of consistency. + find /opt/couchdb/etc -type d ! -perm 0755 -exec chmod -f 0755 '{}' + + find /opt/couchdb/etc -type f ! -perm 0644 -exec chmod -f 0644 '{}' + + fi + + if [ ! -z "$NODENAME" ] && ! grep "couchdb@" /opt/couchdb/etc/vm.args; then + echo "-name couchdb@$NODENAME" >> /opt/couchdb/etc/vm.args + fi + + if [ "$COUCHDB_USER" ] && [ "$COUCHDB_PASSWORD" ]; then + # Create admin only if not already present + if ! grep -Pzoqr "\[admins\]\n$COUCHDB_USER =" /opt/couchdb/etc/local.d/*.ini /opt/couchdb/etc/local.ini; then + printf "\n[admins]\n%s = %s\n" "$COUCHDB_USER" "$COUCHDB_PASSWORD" >> /opt/couchdb/etc/local.d/docker.ini + fi + fi + + if [ "$COUCHDB_SECRET" ]; then + # Set secret only if not already present + if ! grep -Pzoqr "\[chttpd_auth\]\nsecret =" /opt/couchdb/etc/local.d/*.ini /opt/couchdb/etc/local.ini; then + printf "\n[chttpd_auth]\nsecret = %s\n" "$COUCHDB_SECRET" >> /opt/couchdb/etc/local.d/docker.ini + fi + fi + + if [ "$COUCHDB_ERLANG_COOKIE" ]; then + cookieFile='/opt/couchdb/.erlang.cookie' + if [ -e "$cookieFile" ]; then + if [ "$(cat "$cookieFile" 2>/dev/null)" != "$COUCHDB_ERLANG_COOKIE" ]; then + echo >&2 + echo >&2 "warning: $cookieFile contents do not match COUCHDB_ERLANG_COOKIE" + echo >&2 + fi + else + echo "$COUCHDB_ERLANG_COOKIE" > "$cookieFile" + fi + chown couchdb:couchdb "$cookieFile" + chmod 600 "$cookieFile" + fi + + if [ "$(id -u)" = '0' ]; then + chown -f couchdb:couchdb /opt/couchdb/etc/local.d/docker.ini || true + fi + + # if we don't find an [admins] section followed by a non-comment, display a warning + if ! grep -Pzoqr '\[admins\]\n[^;]\w+' /opt/couchdb/etc/default.d/*.ini /opt/couchdb/etc/local.d/*.ini /opt/couchdb/etc/local.ini; then + # The - option suppresses leading tabs but *not* spaces. :) + cat >&2 <<-'EOWARN' +************************************************************* +ERROR: CouchDB 3.0+ will no longer run in "Admin Party" + mode. You *MUST* specify an admin user and + password, either via your own .ini file mapped + into the container at /opt/couchdb/etc/local.ini + or inside /opt/couchdb/etc/local.d, or with + "-e COUCHDB_USER=admin -e COUCHDB_PASSWORD=password" + to set it via "docker run". +************************************************************* +EOWARN + exit 1 + fi + + if [ "$(id -u)" = '0' ]; then + exec gosu couchdb "$@" + fi +fi + +exec "$@" diff --git a/hosting/couchdb/runner.sh b/hosting/couchdb/runner.sh index 9f6a853ca7..aaadee6b43 100644 --- a/hosting/couchdb/runner.sh +++ b/hosting/couchdb/runner.sh @@ -1,6 +1,7 @@ #!/bin/bash DATA_DIR=${DATA_DIR:-/data} +COUCHDB_ERLANG_COOKIE=${COUCHDB_ERLANG_COOKIE:-B9CFC32C-3458-4A86-8448-B3C753991CA7} mkdir -p ${DATA_DIR} mkdir -p ${DATA_DIR}/couch/{dbs,views} @@ -60,6 +61,9 @@ else sed -i "s#DATA_DIR#/data#g" /opt/couchdb/etc/local.ini fi +sed -i "s#COUCHDB_ERLANG_COOKIE#${COUCHDB_ERLANG_COOKIE}#g" /opt/couchdb/etc/vm.args +sed -i "s#COUCHDB_ERLANG_COOKIE#${COUCHDB_ERLANG_COOKIE}#g" /opt/clouseau/clouseau.ini + # Start Clouseau. Budibase won't function correctly without Clouseau running, it # powers the search API endpoints which are used to do all sorts, including # populating app grids. diff --git a/hosting/single/runner.sh b/hosting/single/runner.sh index 886da4c916..3126eedfb1 100644 --- a/hosting/single/runner.sh +++ b/hosting/single/runner.sh @@ -97,10 +97,12 @@ fi sleep 10 pushd app -pm2 start -l /dev/stdout --name app "yarn run:docker" +pm2 start --name app "yarn run:docker" popd pushd worker -pm2 start -l /dev/stdout --name worker "yarn run:docker" +pm2 start --name worker "yarn run:docker" popd echo "end of runner.sh, sleeping ..." + +tail -f $HOME/.pm2/logs/*.log sleep infinity diff --git a/packages/server/src/jsRunner/index.ts b/packages/server/src/jsRunner/index.ts index e39dab1313..84fe6e65c8 100644 --- a/packages/server/src/jsRunner/index.ts +++ b/packages/server/src/jsRunner/index.ts @@ -4,11 +4,15 @@ import { setJSRunner, setOnErrorLog } from "@budibase/string-templates" import { context, logging, timers } from "@budibase/backend-core" import tracer from "dd-trace" import { serializeError } from "serialize-error" +import { IsolatedVM } from "./vm" type TrackerFn = (f: () => T) => T export function init() { setJSRunner((js: string, ctx: vm.Context) => { + const ivm = new IsolatedVM({ memoryLimit: 10, invocationTimeout: 1000 }) + ivm.withContext(ctx) + return tracer.trace("runJS", {}, span => { const perRequestLimit = env.JS_PER_REQUEST_TIMEOUT_MS let track: TrackerFn = f => f() From 140ff2d985876084a510a16031e19fb68178f500 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 13 Feb 2024 12:08:25 +0000 Subject: [PATCH 02/43] Use node:20-slim as the base for consistency with app images. --- hosting/couchdb/Dockerfile | 2 +- hosting/single/Dockerfile | 10 +++------- scripts/install-node.sh | 8 -------- 3 files changed, 4 insertions(+), 16 deletions(-) delete mode 100644 scripts/install-node.sh diff --git a/hosting/couchdb/Dockerfile b/hosting/couchdb/Dockerfile index 6f940d6c48..b4dbdd36dc 100644 --- a/hosting/couchdb/Dockerfile +++ b/hosting/couchdb/Dockerfile @@ -1,5 +1,5 @@ # Modified from https://github.com/apache/couchdb-docker/blob/main/3.2.1/Dockerfile -FROM debian:bookworm-slim +FROM node:20-slim # Add CouchDB user account to make sure the IDs are assigned consistently RUN groupadd -g 5984 -r couchdb && useradd -u 5984 -d /opt/couchdb -g couchdb couchdb diff --git a/hosting/single/Dockerfile b/hosting/single/Dockerfile index f9044cd124..a928766541 100644 --- a/hosting/single/Dockerfile +++ b/hosting/single/Dockerfile @@ -3,7 +3,6 @@ FROM node:20-slim as build # install node-gyp dependencies RUN apt-get update && apt-get install -y --no-install-recommends g++ make python3 jq - # copy and install dependencies WORKDIR /app COPY package.json . @@ -39,10 +38,9 @@ COPY packages/worker/pm2.config.js packages/worker/pm2.config.js COPY packages/string-templates packages/string-templates -FROM budibase/couchdb as runner +FROM budicouch as runner ARG TARGETARCH ENV TARGETARCH $TARGETARCH -ENV NODE_MAJOR 20 #TARGETBUILD can be set to single (for single docker image) or aas (for azure app service) # e.g. docker build --build-arg TARGETBUILD=aas .... ARG TARGETBUILD=single @@ -60,10 +58,8 @@ RUN apt install -y software-properties-common apt-transport-https ca-certificate && apt install postgresql-client-15 -y \ && apt remove software-properties-common apt-transport-https gpg -y -# install other dependencies, nodejs, oracle requirements, jdk8, redis, nginx -WORKDIR /nodejs -COPY scripts/install-node.sh ./install.sh -RUN chmod +x install.sh && ./install.sh +# We use pm2 in order to run multiple node processes in a single container +RUN npm install --global pm2 # setup nginx COPY hosting/single/nginx/nginx.conf /etc/nginx diff --git a/scripts/install-node.sh b/scripts/install-node.sh deleted file mode 100644 index 562bdf2cd3..0000000000 --- a/scripts/install-node.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash -apt-get install -y gnupg -curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor | tee /usr/share/keyrings/nodesource.gpg > /dev/null -echo "deb [signed-by=/usr/share/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$NODE_MAJOR.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list -apt-get update -echo "INSTALLING NODE $NODE_MAJOR" -apt-get install -y --no-install-recommends nodejs -npm install --global yarn pm2 \ No newline at end of file From f6137e097457c9fcafae88085f3cb8ecbeebb568 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 13 Feb 2024 12:09:00 +0000 Subject: [PATCH 03/43] Put the couchdb reference back in the single image build. --- hosting/single/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hosting/single/Dockerfile b/hosting/single/Dockerfile index a928766541..f89967cb0b 100644 --- a/hosting/single/Dockerfile +++ b/hosting/single/Dockerfile @@ -38,7 +38,7 @@ COPY packages/worker/pm2.config.js packages/worker/pm2.config.js COPY packages/string-templates packages/string-templates -FROM budicouch as runner +FROM budibase/couchdb as runner ARG TARGETARCH ENV TARGETARCH $TARGETARCH #TARGETBUILD can be set to single (for single docker image) or aas (for azure app service) From 4bcf1332593f298c0bb6781d4f0902bdc4e449df Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 13 Feb 2024 12:18:16 +0000 Subject: [PATCH 04/43] Remove unneeded jsRunner/index.ts code. --- packages/server/src/jsRunner/index.ts | 4 ---- 1 file changed, 4 deletions(-) diff --git a/packages/server/src/jsRunner/index.ts b/packages/server/src/jsRunner/index.ts index 84fe6e65c8..e39dab1313 100644 --- a/packages/server/src/jsRunner/index.ts +++ b/packages/server/src/jsRunner/index.ts @@ -4,15 +4,11 @@ import { setJSRunner, setOnErrorLog } from "@budibase/string-templates" import { context, logging, timers } from "@budibase/backend-core" import tracer from "dd-trace" import { serializeError } from "serialize-error" -import { IsolatedVM } from "./vm" type TrackerFn = (f: () => T) => T export function init() { setJSRunner((js: string, ctx: vm.Context) => { - const ivm = new IsolatedVM({ memoryLimit: 10, invocationTimeout: 1000 }) - ivm.withContext(ctx) - return tracer.trace("runJS", {}, span => { const perRequestLimit = env.JS_PER_REQUEST_TIMEOUT_MS let track: TrackerFn = f => f() From 634b3b43dd3ba22b7e3a320436311bf5a9402d96 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 13 Feb 2024 15:35:44 +0000 Subject: [PATCH 05/43] Better delineate what is our CouchDB image and what is from the official one. --- hosting/couchdb/Dockerfile | 16 +++++++++++----- hosting/couchdb/docker-entrypoint.sh | 5 +++-- hosting/single/Dockerfile | 2 +- 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/hosting/couchdb/Dockerfile b/hosting/couchdb/Dockerfile index b4dbdd36dc..6dbf2be08b 100644 --- a/hosting/couchdb/Dockerfile +++ b/hosting/couchdb/Dockerfile @@ -1,5 +1,10 @@ # Modified from https://github.com/apache/couchdb-docker/blob/main/3.2.1/Dockerfile -FROM node:20-slim +# +# Everything in this `base` image is adapted from the official `couchdb` image's +# Dockerfile. Only modifications related to upgrading from Debian bullseye to +# bookworm have been included. The `runner` image contains Budibase's +# customisations to the image, e.g. adding Clouseau. +FROM node:20-slim AS base # Add CouchDB user account to make sure the IDs are assigned consistently RUN groupadd -g 5984 -r couchdb && useradd -u 5984 -d /opt/couchdb -g couchdb couchdb @@ -15,13 +20,12 @@ RUN set -ex; \ ; \ rm -rf /var/lib/apt/lists/* -# grab gosu for easy step-down from root and tini for signal handling and zombie reaping +# grab tini for signal handling and zombie reaping # see https://github.com/apache/couchdb-docker/pull/28#discussion_r141112407 RUN set -eux; \ apt-get update; \ - apt-get install -y --no-install-recommends gosu tini; \ + apt-get install -y --no-install-recommends tini; \ rm -rf /var/lib/apt/lists/*; \ - gosu nobody true; \ tini --version # http://docs.couchdb.org/en/latest/install/unix.html#installing-the-apache-couchdb-packages @@ -89,7 +93,9 @@ VOLUME /opt/couchdb/data # 4369: Erlang portmap daemon (epmd) # 9100: CouchDB cluster communication port EXPOSE 5984 4369 9100 -# CMD ["/opt/couchdb/bin/couchdb"] +CMD ["/opt/couchdb/bin/couchdb"] + +FROM base as runner ENV COUCHDB_USER admin ENV COUCHDB_PASSWORD admin diff --git a/hosting/couchdb/docker-entrypoint.sh b/hosting/couchdb/docker-entrypoint.sh index 8d6456d577..bd709b7b73 100755 --- a/hosting/couchdb/docker-entrypoint.sh +++ b/hosting/couchdb/docker-entrypoint.sh @@ -114,8 +114,9 @@ EOWARN fi if [ "$(id -u)" = '0' ]; then - exec gosu couchdb "$@" + export HOME=$(echo ~couchdb) + exec setpriv --reuid=couchdb --regid=couchdb --clear-groups "$@" fi fi -exec "$@" +exec "$@" \ No newline at end of file diff --git a/hosting/single/Dockerfile b/hosting/single/Dockerfile index f89967cb0b..a928766541 100644 --- a/hosting/single/Dockerfile +++ b/hosting/single/Dockerfile @@ -38,7 +38,7 @@ COPY packages/worker/pm2.config.js packages/worker/pm2.config.js COPY packages/string-templates packages/string-templates -FROM budibase/couchdb as runner +FROM budicouch as runner ARG TARGETARCH ENV TARGETARCH $TARGETARCH #TARGETBUILD can be set to single (for single docker image) or aas (for azure app service) From 22240659eb2a0107e9421bba3e030f44ed318800 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 13 Feb 2024 15:36:09 +0000 Subject: [PATCH 06/43] Link to the correct Dockerfile we've modified. --- hosting/couchdb/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hosting/couchdb/Dockerfile b/hosting/couchdb/Dockerfile index 6dbf2be08b..36abc2dd19 100644 --- a/hosting/couchdb/Dockerfile +++ b/hosting/couchdb/Dockerfile @@ -1,4 +1,4 @@ -# Modified from https://github.com/apache/couchdb-docker/blob/main/3.2.1/Dockerfile +# Modified from https://github.com/apache/couchdb-docker/blob/main/3.3.3/Dockerfile # # Everything in this `base` image is adapted from the official `couchdb` image's # Dockerfile. Only modifications related to upgrading from Debian bullseye to From 30abf188a3aa0e03442ffe8d5b4b29340e0ce4bc Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 16 Feb 2024 15:13:26 +0000 Subject: [PATCH 07/43] Bail out if server startup fails. --- hosting/single/Dockerfile | 2 +- packages/server/src/app.ts | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/hosting/single/Dockerfile b/hosting/single/Dockerfile index a928766541..f89967cb0b 100644 --- a/hosting/single/Dockerfile +++ b/hosting/single/Dockerfile @@ -38,7 +38,7 @@ COPY packages/worker/pm2.config.js packages/worker/pm2.config.js COPY packages/string-templates packages/string-templates -FROM budicouch as runner +FROM budibase/couchdb as runner ARG TARGETARCH ENV TARGETARCH $TARGETARCH #TARGETBUILD can be set to single (for single docker image) or aas (for azure app service) diff --git a/packages/server/src/app.ts b/packages/server/src/app.ts index f6f1780030..4e84422dec 100644 --- a/packages/server/src/app.ts +++ b/packages/server/src/app.ts @@ -26,6 +26,7 @@ async function start() { start().catch(err => { console.error(`Failed server startup - ${err.message}`) + throw err }) export function getServer() { From 43a284283367d78726d3fd43f252eaa5ea085270 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 19 Feb 2024 21:04:33 +0100 Subject: [PATCH 08/43] Fix defaulting values --- packages/server/src/jsRunner/vm/isolated-vm.ts | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/packages/server/src/jsRunner/vm/isolated-vm.ts b/packages/server/src/jsRunner/vm/isolated-vm.ts index 12559df0b7..83a7646cfa 100644 --- a/packages/server/src/jsRunner/vm/isolated-vm.ts +++ b/packages/server/src/jsRunner/vm/isolated-vm.ts @@ -38,10 +38,9 @@ export class IsolatedVM implements VM { invocationTimeout?: number isolateAccumulatedTimeout?: number } = {}) { - memoryLimit = memoryLimit || environment.JS_RUNNER_MEMORY_LIMIT - invocationTimeout = memoryLimit || 1000 - - this.isolate = new ivm.Isolate({ memoryLimit }) + this.isolate = new ivm.Isolate({ + memoryLimit: memoryLimit || environment.JS_RUNNER_MEMORY_LIMIT, + }) this.vm = this.isolate.createContextSync() this.jail = this.vm.global this.jail.setSync("global", this.jail.derefInto()) @@ -51,7 +50,7 @@ export class IsolatedVM implements VM { [this.resultKey]: { [this.runResultKey]: "" }, }) - this.invocationTimeout = invocationTimeout + this.invocationTimeout = invocationTimeout || 1000 this.isolateAccumulatedTimeout = isolateAccumulatedTimeout } From d1e0b37dc21313a65187387afc7d549ef238b539 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 19 Feb 2024 21:07:45 +0100 Subject: [PATCH 09/43] Remove magic number --- packages/server/src/jsRunner/vm/isolated-vm.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/server/src/jsRunner/vm/isolated-vm.ts b/packages/server/src/jsRunner/vm/isolated-vm.ts index 83a7646cfa..f431ff644b 100644 --- a/packages/server/src/jsRunner/vm/isolated-vm.ts +++ b/packages/server/src/jsRunner/vm/isolated-vm.ts @@ -50,7 +50,8 @@ export class IsolatedVM implements VM { [this.resultKey]: { [this.runResultKey]: "" }, }) - this.invocationTimeout = invocationTimeout || 1000 + this.invocationTimeout = + invocationTimeout || environment.JS_PER_INVOCATION_TIMEOUT_MS this.isolateAccumulatedTimeout = isolateAccumulatedTimeout } From 634bf5f8ba7851d58dd8781dc0b124b45bfcb48e Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Mon, 19 Feb 2024 20:26:23 +0000 Subject: [PATCH 10/43] Bump version to 2.20.0 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index 62068e25a8..3386214dcf 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.19.6", + "version": "2.20.0", "npmClient": "yarn", "packages": [ "packages/*", From f99dbeb2ffe5457a71421ec9c9c3f719750a1f05 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Mon, 19 Feb 2024 18:28:43 -0300 Subject: [PATCH 11/43] update pro submodule --- packages/pro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pro b/packages/pro index aaf7101cd1..32e4d4f946 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit aaf7101cd1493215155cc8f83124c70d53eb1be4 +Subproject commit 32e4d4f946c881bd87777d99b17f13ca4a38ea99 From 9443a113ecb4b098b38898105d6f0eaeed0d24e3 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Mon, 19 Feb 2024 18:44:58 -0300 Subject: [PATCH 12/43] update reference --- packages/pro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pro b/packages/pro index 32e4d4f946..7bc9f00fac 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 32e4d4f946c881bd87777d99b17f13ca4a38ea99 +Subproject commit 7bc9f00fac37571b45ca41a034004860bf058cb3 From 2d019ccb6573a89353777de5dd02d26a7b2a551c Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Mon, 19 Feb 2024 18:46:13 -0300 Subject: [PATCH 13/43] account portal submodule --- packages/account-portal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/account-portal b/packages/account-portal index cc12291732..8c446c4ba3 160000 --- a/packages/account-portal +++ b/packages/account-portal @@ -1 +1 @@ -Subproject commit cc12291732ee902dc832bc7d93cf2086ffdf0cff +Subproject commit 8c446c4ba385592127fa31755d3b64467b291882 From 93b18b81e003da221cb065d9587f0a6ed337fbfb Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 20 Feb 2024 10:49:45 +0000 Subject: [PATCH 14/43] Fix re-used context in JS runner. --- packages/server/src/api/controllers/script.ts | 7 ++++--- packages/server/src/jsRunner/index.ts | 15 +++++-------- .../server/src/jsRunner/vm/isolated-vm.ts | 14 +++++++++++-- packages/server/src/jsRunner/vm/vm2.ts | 14 +++++++++++-- packages/server/src/threads/query.ts | 21 ++++++++----------- packages/types/src/sdk/vm.ts | 1 + 6 files changed, 43 insertions(+), 29 deletions(-) diff --git a/packages/server/src/api/controllers/script.ts b/packages/server/src/api/controllers/script.ts index bdca2d6e18..93e1ad7df9 100644 --- a/packages/server/src/api/controllers/script.ts +++ b/packages/server/src/api/controllers/script.ts @@ -3,9 +3,10 @@ import { IsolatedVM } from "../../jsRunner/vm" export async function execute(ctx: Ctx) { const { script, context } = ctx.request.body - const runner = new IsolatedVM().withContext(context) - - const result = runner.execute(`(function(){\n${script}\n})();`) + const vm = new IsolatedVM() + const result = vm.withContext(context, () => + vm.execute(`(function(){\n${script}\n})();`) + ) ctx.body = result } diff --git a/packages/server/src/jsRunner/index.ts b/packages/server/src/jsRunner/index.ts index 3c13aef1d4..1e8bee04c3 100644 --- a/packages/server/src/jsRunner/index.ts +++ b/packages/server/src/jsRunner/index.ts @@ -23,22 +23,17 @@ export function init() { try { const bbCtx = context.getCurrentContext()! - let { vm } = bbCtx - if (!vm) { - // Can't copy the native helpers into the isolate. We just ignore them as they are handled properly from the helpersSource - const { helpers, ...ctxToPass } = ctx - - vm = new IsolatedVM({ + if (!bbCtx.vm) { + const vm = new IsolatedVM({ memoryLimit: env.JS_RUNNER_MEMORY_LIMIT, invocationTimeout: env.JS_PER_INVOCATION_TIMEOUT_MS, isolateAccumulatedTimeout: env.JS_PER_REQUEST_TIMEOUT_MS, - }) - .withContext(ctxToPass) - .withHelpers() + }).withHelpers() bbCtx.vm = vm } - return vm.execute(js) + const { helpers, ...rest } = ctx + return bbCtx.vm.withContext(rest, () => bbCtx.vm!.execute(js)) } catch (error: any) { if (error.message === "Script execution timed out.") { throw new JsErrorTimeout() diff --git a/packages/server/src/jsRunner/vm/isolated-vm.ts b/packages/server/src/jsRunner/vm/isolated-vm.ts index f431ff644b..78250acdd2 100644 --- a/packages/server/src/jsRunner/vm/isolated-vm.ts +++ b/packages/server/src/jsRunner/vm/isolated-vm.ts @@ -97,10 +97,14 @@ export class IsolatedVM implements VM { return this } - withContext(context: Record) { + withContext(context: Record, f: () => T) { this.addToContext(context) - return this + try { + return f() + } finally { + this.removeFromContext(Object.keys(context)) + } } withParsingBson(data: any) { @@ -224,6 +228,12 @@ export class IsolatedVM implements VM { } } + private removeFromContext(keys: string[]) { + for (let key of keys) { + this.jail.deleteSync(key) + } + } + private getFromContext(key: string) { const ref = this.vm.global.getSync(key, { reference: true }) const result = ref.copySync() diff --git a/packages/server/src/jsRunner/vm/vm2.ts b/packages/server/src/jsRunner/vm/vm2.ts index 6d05943d25..75e3899064 100644 --- a/packages/server/src/jsRunner/vm/vm2.ts +++ b/packages/server/src/jsRunner/vm/vm2.ts @@ -7,16 +7,26 @@ export class VM2 implements VM { vm: vm2.VM results: { out: string } - constructor(context: any) { + constructor() { this.vm = new vm2.VM({ timeout: JS_TIMEOUT_MS, }) this.results = { out: "" } - this.vm.setGlobals(context) this.vm.setGlobal("fetch", fetch) this.vm.setGlobal("results", this.results) } + withContext(context: Record, fn: () => T): T { + this.vm.setGlobals(context) + try { + return fn() + } finally { + for (const key in context) { + this.vm.setGlobal(key, undefined) + } + } + } + execute(script: string) { const code = `let fn = () => {\n${script}\n}; results.out = fn();` const vmScript = new vm2.VMScript(code) diff --git a/packages/server/src/threads/query.ts b/packages/server/src/threads/query.ts index c159f24268..6cc07c1256 100644 --- a/packages/server/src/threads/query.ts +++ b/packages/server/src/threads/query.ts @@ -131,24 +131,21 @@ class QueryRunner { if (transformer) { let runner: VM if (!USE_ISOLATED_VM) { - runner = new VM2({ - data: rows, - params: enrichedParameters, - }) + runner = new VM2() } else { transformer = `(function(){\n${transformer}\n})();` - let isolatedVm = new IsolatedVM().withContext({ - data: rows, - params: enrichedParameters, - }) + let vm = new IsolatedVM() if (datasource.source === SourceName.MONGODB) { - isolatedVm = isolatedVm.withParsingBson(rows) + vm = vm.withParsingBson(rows) } - - runner = isolatedVm + runner = vm } - rows = runner.execute(transformer) + const ctx = { + data: rows, + params: enrichedParameters, + } + rows = runner.withContext(ctx, () => runner.execute(transformer)) } // if the request fails we retry once, invalidating the cached value diff --git a/packages/types/src/sdk/vm.ts b/packages/types/src/sdk/vm.ts index 43b7775d3b..314883e0c5 100644 --- a/packages/types/src/sdk/vm.ts +++ b/packages/types/src/sdk/vm.ts @@ -1,3 +1,4 @@ export interface VM { execute(code: string): any + withContext(context: Record, fn: () => T): T } From cdad301e7e39fd3d0258a5c5b79802d0cd768143 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 20 Feb 2024 11:52:56 +0100 Subject: [PATCH 15/43] Undefined checks for context --- packages/server/src/jsRunner/index.ts | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/packages/server/src/jsRunner/index.ts b/packages/server/src/jsRunner/index.ts index 3c13aef1d4..5db461e023 100644 --- a/packages/server/src/jsRunner/index.ts +++ b/packages/server/src/jsRunner/index.ts @@ -21,9 +21,9 @@ export function init() { } try { - const bbCtx = context.getCurrentContext()! + const bbCtx = context.getCurrentContext() - let { vm } = bbCtx + let vm = bbCtx?.vm if (!vm) { // Can't copy the native helpers into the isolate. We just ignore them as they are handled properly from the helpersSource const { helpers, ...ctxToPass } = ctx @@ -36,7 +36,10 @@ export function init() { .withContext(ctxToPass) .withHelpers() - bbCtx.vm = vm + if (bbCtx) { + // If we have a context, we want to persist it to reuse the isolate + bbCtx.vm = vm + } } return vm.execute(js) } catch (error: any) { From a866677080bde9cbac804b306dc32d6798bd9e3e Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 20 Feb 2024 10:59:04 +0000 Subject: [PATCH 16/43] Add tests. --- .../server/src/api/routes/tests/row.spec.ts | 43 +++++++++++++++++++ packages/server/src/jsRunner/index.ts | 4 +- 2 files changed, 44 insertions(+), 3 deletions(-) diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts index 10dac3c0ea..239da36351 100644 --- a/packages/server/src/api/routes/tests/row.spec.ts +++ b/packages/server/src/api/routes/tests/row.spec.ts @@ -2135,5 +2135,48 @@ describe.each([ } ) }) + + it("should not carry over context between formulas", async () => { + const js = Buffer.from(`return $("[text]");`).toString("base64") + const table = await config.createTable({ + name: "table", + type: "table", + schema: { + text: { + name: "text", + type: FieldType.STRING, + }, + formula: { + name: "formula", + type: FieldType.FORMULA, + formula: `{{ js "${js}"}}`, + formulaType: FormulaType.DYNAMIC, + }, + }, + }) + + for (let i = 0; i < 10; i++) { + await config.api.row.save(table._id!, { text: `foo${i}` }) + } + + const { rows } = await config.api.row.search(table._id!) + expect(rows).toHaveLength(10) + + const formulaValues = rows.map(r => r.formula) + expect(formulaValues).toEqual( + expect.arrayContaining([ + "foo0", + "foo1", + "foo2", + "foo3", + "foo4", + "foo5", + "foo6", + "foo7", + "foo8", + "foo9", + ]) + ) + }) }) }) diff --git a/packages/server/src/jsRunner/index.ts b/packages/server/src/jsRunner/index.ts index 1e8bee04c3..d0385404fe 100644 --- a/packages/server/src/jsRunner/index.ts +++ b/packages/server/src/jsRunner/index.ts @@ -24,13 +24,11 @@ export function init() { const bbCtx = context.getCurrentContext()! if (!bbCtx.vm) { - const vm = new IsolatedVM({ + bbCtx.vm = new IsolatedVM({ memoryLimit: env.JS_RUNNER_MEMORY_LIMIT, invocationTimeout: env.JS_PER_INVOCATION_TIMEOUT_MS, isolateAccumulatedTimeout: env.JS_PER_REQUEST_TIMEOUT_MS, }).withHelpers() - - bbCtx.vm = vm } const { helpers, ...rest } = ctx return bbCtx.vm.withContext(rest, () => bbCtx.vm!.execute(js)) From 904476de5ca1b34a59edb3f85237daad2b0757fd Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Tue, 20 Feb 2024 11:04:03 +0000 Subject: [PATCH 17/43] Bump version to 2.20.1 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index 3386214dcf..8210fbae99 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.20.0", + "version": "2.20.1", "npmClient": "yarn", "packages": [ "packages/*", From 285916d0bfbfe2f56e2d5c7365348ad0de2e40ff Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 20 Feb 2024 11:11:27 +0000 Subject: [PATCH 18/43] Some PR comments/build issue. --- packages/server/src/jsRunner/vm/builtin-vm.ts | 11 +++++++++++ packages/server/src/jsRunner/vm/isolated-vm.ts | 4 ++-- packages/server/src/jsRunner/vm/vm2.ts | 4 ++-- packages/types/src/sdk/vm.ts | 2 +- 4 files changed, 16 insertions(+), 5 deletions(-) diff --git a/packages/server/src/jsRunner/vm/builtin-vm.ts b/packages/server/src/jsRunner/vm/builtin-vm.ts index b4c9f775f9..849abdd6f2 100644 --- a/packages/server/src/jsRunner/vm/builtin-vm.ts +++ b/packages/server/src/jsRunner/vm/builtin-vm.ts @@ -15,6 +15,17 @@ export class BuiltInVM implements VM { this.span = span } + withContext(context: Record, executeWithContext: () => T): T { + this.ctx = vm.createContext(context) + try { + return executeWithContext() + } finally { + for (const key in context) { + delete this.ctx[key] + } + } + } + execute(code: string) { const perRequestLimit = env.JS_PER_REQUEST_TIMEOUT_MS let track: TrackerFn = f => f() diff --git a/packages/server/src/jsRunner/vm/isolated-vm.ts b/packages/server/src/jsRunner/vm/isolated-vm.ts index 78250acdd2..e5c431666d 100644 --- a/packages/server/src/jsRunner/vm/isolated-vm.ts +++ b/packages/server/src/jsRunner/vm/isolated-vm.ts @@ -97,11 +97,11 @@ export class IsolatedVM implements VM { return this } - withContext(context: Record, f: () => T) { + withContext(context: Record, executeWithContext: () => T) { this.addToContext(context) try { - return f() + return executeWithContext() } finally { this.removeFromContext(Object.keys(context)) } diff --git a/packages/server/src/jsRunner/vm/vm2.ts b/packages/server/src/jsRunner/vm/vm2.ts index 75e3899064..5825025d26 100644 --- a/packages/server/src/jsRunner/vm/vm2.ts +++ b/packages/server/src/jsRunner/vm/vm2.ts @@ -16,10 +16,10 @@ export class VM2 implements VM { this.vm.setGlobal("results", this.results) } - withContext(context: Record, fn: () => T): T { + withContext(context: Record, executeWithContext: () => T): T { this.vm.setGlobals(context) try { - return fn() + return executeWithContext() } finally { for (const key in context) { this.vm.setGlobal(key, undefined) diff --git a/packages/types/src/sdk/vm.ts b/packages/types/src/sdk/vm.ts index 314883e0c5..f1099524bc 100644 --- a/packages/types/src/sdk/vm.ts +++ b/packages/types/src/sdk/vm.ts @@ -1,4 +1,4 @@ export interface VM { execute(code: string): any - withContext(context: Record, fn: () => T): T + withContext(context: Record, executeWithContext: () => T): T } From b7190f458d15ad6d519e2752324cd63c4cb0ac05 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 20 Feb 2024 12:26:10 +0100 Subject: [PATCH 19/43] Update pro ref --- packages/pro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pro b/packages/pro index 7bc9f00fac..6f3a0b7f72 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 7bc9f00fac37571b45ca41a034004860bf058cb3 +Subproject commit 6f3a0b7f72d16f9604179af98ff7602ca0df2737 From b9afe1b926dd420ca4d32d17e203d5b406346575 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 20 Feb 2024 12:09:54 +0100 Subject: [PATCH 20/43] Remove vm2 wrapper --- packages/server/src/jsRunner/vm/index.ts | 1 - packages/server/src/jsRunner/vm/vm2.ts | 36 ------------------------ packages/server/src/threads/query.ts | 22 +++++---------- 3 files changed, 7 insertions(+), 52 deletions(-) delete mode 100644 packages/server/src/jsRunner/vm/vm2.ts diff --git a/packages/server/src/jsRunner/vm/index.ts b/packages/server/src/jsRunner/vm/index.ts index 01e0daa354..cc50a5eeaa 100644 --- a/packages/server/src/jsRunner/vm/index.ts +++ b/packages/server/src/jsRunner/vm/index.ts @@ -1,3 +1,2 @@ export * from "./isolated-vm" export * from "./builtin-vm" -export * from "./vm2" diff --git a/packages/server/src/jsRunner/vm/vm2.ts b/packages/server/src/jsRunner/vm/vm2.ts deleted file mode 100644 index 5825025d26..0000000000 --- a/packages/server/src/jsRunner/vm/vm2.ts +++ /dev/null @@ -1,36 +0,0 @@ -import vm2 from "vm2" -import { VM } from "@budibase/types" - -const JS_TIMEOUT_MS = 1000 - -export class VM2 implements VM { - vm: vm2.VM - results: { out: string } - - constructor() { - this.vm = new vm2.VM({ - timeout: JS_TIMEOUT_MS, - }) - this.results = { out: "" } - this.vm.setGlobal("fetch", fetch) - this.vm.setGlobal("results", this.results) - } - - withContext(context: Record, executeWithContext: () => T): T { - this.vm.setGlobals(context) - try { - return executeWithContext() - } finally { - for (const key in context) { - this.vm.setGlobal(key, undefined) - } - } - } - - execute(script: string) { - const code = `let fn = () => {\n${script}\n}; results.out = fn();` - const vmScript = new vm2.VMScript(code) - this.vm.run(vmScript) - return this.results.out - } -} diff --git a/packages/server/src/threads/query.ts b/packages/server/src/threads/query.ts index 6cc07c1256..9366f2b12c 100644 --- a/packages/server/src/threads/query.ts +++ b/packages/server/src/threads/query.ts @@ -7,20 +7,18 @@ import { QueryVariable, QueryResponse, } from "./definitions" -import { IsolatedVM, VM2 } from "../jsRunner/vm" +import { IsolatedVM } from "../jsRunner/vm" import { getIntegration } from "../integrations" import { processStringSync } from "@budibase/string-templates" import { context, cache, auth } from "@budibase/backend-core" import { getGlobalIDFromUserMetadataID } from "../db/utils" import sdk from "../sdk" import { cloneDeep } from "lodash/fp" -import { Datasource, Query, SourceName, VM } from "@budibase/types" +import { Datasource, Query, SourceName } from "@budibase/types" import { isSQL } from "../integrations/utils" import { interpolateSQL } from "../integrations/queries/sql" -const USE_ISOLATED_VM = true - class QueryRunner { datasource: Datasource queryVerb: string @@ -129,23 +127,17 @@ class QueryRunner { // transform as required if (transformer) { - let runner: VM - if (!USE_ISOLATED_VM) { - runner = new VM2() - } else { - transformer = `(function(){\n${transformer}\n})();` - let vm = new IsolatedVM() - if (datasource.source === SourceName.MONGODB) { - vm = vm.withParsingBson(rows) - } - runner = vm + transformer = `(function(){\n${transformer}\n})();` + let vm = new IsolatedVM() + if (datasource.source === SourceName.MONGODB) { + vm = vm.withParsingBson(rows) } const ctx = { data: rows, params: enrichedParameters, } - rows = runner.withContext(ctx, () => runner.execute(transformer)) + rows = vm.withContext(ctx, () => vm.execute(transformer)) } // if the request fails we retry once, invalidating the cached value From 73d39836ab480deb555ca162268625665224d69a Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 20 Feb 2024 12:10:49 +0100 Subject: [PATCH 21/43] Remove vm wrapper --- packages/server/src/jsRunner/index.ts | 9 +-- packages/server/src/jsRunner/vm/builtin-vm.ts | 76 ------------------- packages/server/src/jsRunner/vm/index.ts | 1 - 3 files changed, 1 insertion(+), 85 deletions(-) delete mode 100644 packages/server/src/jsRunner/vm/builtin-vm.ts diff --git a/packages/server/src/jsRunner/index.ts b/packages/server/src/jsRunner/index.ts index 3b6b464dae..0fbfed3b66 100644 --- a/packages/server/src/jsRunner/index.ts +++ b/packages/server/src/jsRunner/index.ts @@ -8,18 +8,11 @@ import { import { context, logging } from "@budibase/backend-core" import tracer from "dd-trace" -import { BuiltInVM, IsolatedVM } from "./vm" - -const USE_ISOLATED_VM = true +import { IsolatedVM } from "./vm" export function init() { setJSRunner((js: string, ctx: Record) => { return tracer.trace("runJS", {}, span => { - if (!USE_ISOLATED_VM) { - const vm = new BuiltInVM(ctx, span) - return vm.execute(js) - } - try { const bbCtx = context.getCurrentContext() diff --git a/packages/server/src/jsRunner/vm/builtin-vm.ts b/packages/server/src/jsRunner/vm/builtin-vm.ts deleted file mode 100644 index 849abdd6f2..0000000000 --- a/packages/server/src/jsRunner/vm/builtin-vm.ts +++ /dev/null @@ -1,76 +0,0 @@ -import vm from "vm" -import env from "../../environment" -import { context, timers } from "@budibase/backend-core" -import tracer, { Span } from "dd-trace" -import { VM } from "@budibase/types" - -type TrackerFn = (f: () => T) => T - -export class BuiltInVM implements VM { - private ctx: vm.Context - private span?: Span - - constructor(ctx: vm.Context, span?: Span) { - this.ctx = ctx - this.span = span - } - - withContext(context: Record, executeWithContext: () => T): T { - this.ctx = vm.createContext(context) - try { - return executeWithContext() - } finally { - for (const key in context) { - delete this.ctx[key] - } - } - } - - execute(code: string) { - const perRequestLimit = env.JS_PER_REQUEST_TIMEOUT_MS - let track: TrackerFn = f => f() - if (perRequestLimit) { - const bbCtx = tracer.trace("runJS.getCurrentContext", {}, span => - context.getCurrentContext() - ) - if (bbCtx) { - if (!bbCtx.jsExecutionTracker) { - this.span?.addTags({ - createdExecutionTracker: true, - }) - bbCtx.jsExecutionTracker = tracer.trace( - "runJS.createExecutionTimeTracker", - {}, - span => timers.ExecutionTimeTracker.withLimit(perRequestLimit) - ) - } - this.span?.addTags({ - js: { - limitMS: bbCtx.jsExecutionTracker.limitMs, - elapsedMS: bbCtx.jsExecutionTracker.elapsedMS, - }, - }) - // We call checkLimit() here to prevent paying the cost of creating - // a new VM context below when we don't need to. - tracer.trace("runJS.checkLimitAndBind", {}, span => { - bbCtx.jsExecutionTracker!.checkLimit() - track = bbCtx.jsExecutionTracker!.track.bind(bbCtx.jsExecutionTracker) - }) - } - } - - this.ctx = { - ...this.ctx, - alert: undefined, - setInterval: undefined, - setTimeout: undefined, - } - - vm.createContext(this.ctx) - return track(() => - vm.runInNewContext(code, this.ctx, { - timeout: env.JS_PER_INVOCATION_TIMEOUT_MS, - }) - ) - } -} diff --git a/packages/server/src/jsRunner/vm/index.ts b/packages/server/src/jsRunner/vm/index.ts index cc50a5eeaa..286a277cfb 100644 --- a/packages/server/src/jsRunner/vm/index.ts +++ b/packages/server/src/jsRunner/vm/index.ts @@ -1,2 +1 @@ export * from "./isolated-vm" -export * from "./builtin-vm" From 8480fb0227506d13f0306cc26cda2012270fd555 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 20 Feb 2024 12:11:26 +0100 Subject: [PATCH 22/43] Remove vm2 package --- packages/server/package.json | 1 - yarn.lock | 10 +--------- 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/packages/server/package.json b/packages/server/package.json index 57e1a828b7..79daf1ba4b 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -114,7 +114,6 @@ "undici-types": "^6.0.1", "uuid": "^8.3.2", "validate.js": "0.13.1", - "vm2": "^3.9.19", "worker-farm": "1.7.0", "xml2js": "0.5.0" }, diff --git a/yarn.lock b/yarn.lock index 10acc829b3..47de6a67ec 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6463,7 +6463,7 @@ acorn@^7.1.1: resolved "https://registry.yarnpkg.com/acorn/-/acorn-7.4.1.tgz#feaed255973d2e77555b83dbc08851a6c63520fa" integrity sha512-nQyp0o1/mNdbTO1PO6kHkwSrmgZ0MT/jCCpNiwbUjGoRN4dlBhqJtoQuCnEOKzgTVwg0ZWiCoQy6SxMebQVh8A== -acorn@^8.1.0, acorn@^8.10.0, acorn@^8.2.4, acorn@^8.4.1, acorn@^8.5.0, acorn@^8.7.0, acorn@^8.7.1, acorn@^8.8.1, acorn@^8.8.2, acorn@^8.9.0: +acorn@^8.1.0, acorn@^8.10.0, acorn@^8.2.4, acorn@^8.4.1, acorn@^8.5.0, acorn@^8.7.1, acorn@^8.8.1, acorn@^8.8.2, acorn@^8.9.0: version "8.11.3" resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.11.3.tgz#71e0b14e13a4ec160724b38fb7b0f233b1b81d7a" integrity sha512-Y9rRfJG5jcKOE0CLisYbojUjIrIEE7AGMzA/Sm4BslANhbS+cDMpgBdcPT91oJ7OuJ9hYJBx59RjbhxVnrF8Xg== @@ -21633,14 +21633,6 @@ vlq@^0.2.2: resolved "https://registry.yarnpkg.com/vlq/-/vlq-0.2.3.tgz#8f3e4328cf63b1540c0d67e1b2778386f8975b26" integrity sha512-DRibZL6DsNhIgYQ+wNdWDL2SL3bKPlVrRiBqV5yuMm++op8W4kGFtaQfCs4KEJn0wBZcHVHJ3eoywX8983k1ow== -vm2@^3.9.19: - version "3.9.19" - resolved "https://registry.yarnpkg.com/vm2/-/vm2-3.9.19.tgz#be1e1d7a106122c6c492b4d51c2e8b93d3ed6a4a" - integrity sha512-J637XF0DHDMV57R6JyVsTak7nIL8gy5KH4r1HiwWLf/4GBbb5MKL5y7LpmF4A8E2nR6XmzpmMFQ7V7ppPTmUQg== - dependencies: - acorn "^8.7.0" - acorn-walk "^8.2.0" - vuvuzela@1.0.3: version "1.0.3" resolved "https://registry.yarnpkg.com/vuvuzela/-/vuvuzela-1.0.3.tgz#3be145e58271c73ca55279dd851f12a682114b0b" From f67173b757eb15204c6aff19d338c52e402d7a99 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 20 Feb 2024 11:35:29 +0000 Subject: [PATCH 23/43] Fix flaky table test. --- packages/server/src/api/routes/tests/table.spec.ts | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/packages/server/src/api/routes/tests/table.spec.ts b/packages/server/src/api/routes/tests/table.spec.ts index c8cb3ef21b..ce119e56f0 100644 --- a/packages/server/src/api/routes/tests/table.spec.ts +++ b/packages/server/src/api/routes/tests/table.spec.ts @@ -368,10 +368,12 @@ describe("/tables", () => { .set(config.defaultHeaders()) .expect("Content-Type", /json/) .expect(200) - const fetchedTable = res.body[0] - expect(fetchedTable.name).toEqual(testTable.name) - expect(fetchedTable.type).toEqual("table") - expect(fetchedTable.sourceType).toEqual("internal") + + const table = res.body.find((t: Table) => t._id === testTable._id) + expect(table).toBeDefined() + expect(table.name).toEqual(testTable.name) + expect(table.type).toEqual("table") + expect(table.sourceType).toEqual("internal") }) it("should apply authorization to endpoint", async () => { From 568f1547bc1a5844be78740a8d9bf2fb3658fa0c Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Tue, 20 Feb 2024 11:47:56 +0000 Subject: [PATCH 24/43] Bump version to 2.20.2 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index 8210fbae99..04f5c6e880 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.20.1", + "version": "2.20.2", "npmClient": "yarn", "packages": [ "packages/*", From 9127337f8b41eacdd0650978ed7bc079e46e608e Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 20 Feb 2024 13:14:49 +0100 Subject: [PATCH 25/43] Use latest image --- hosting/single/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hosting/single/Dockerfile b/hosting/single/Dockerfile index f89967cb0b..ee98b0729d 100644 --- a/hosting/single/Dockerfile +++ b/hosting/single/Dockerfile @@ -38,7 +38,7 @@ COPY packages/worker/pm2.config.js packages/worker/pm2.config.js COPY packages/string-templates packages/string-templates -FROM budibase/couchdb as runner +FROM budibase/couchdb:v3.3.3 as runner ARG TARGETARCH ENV TARGETARCH $TARGETARCH #TARGETBUILD can be set to single (for single docker image) or aas (for azure app service) From c7ec698d30d01051f8f16dcdbf9d73c06299f397 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 20 Feb 2024 12:20:35 +0000 Subject: [PATCH 26/43] Ensure a backup is complete before attempting to import it. --- packages/server/src/api/routes/tests/backup.spec.ts | 7 ++----- packages/server/src/tests/utilities/api/backup.ts | 13 +++++++++++++ 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/packages/server/src/api/routes/tests/backup.spec.ts b/packages/server/src/api/routes/tests/backup.spec.ts index d12b5e1507..acfac783db 100644 --- a/packages/server/src/api/routes/tests/backup.spec.ts +++ b/packages/server/src/api/routes/tests/backup.spec.ts @@ -8,7 +8,6 @@ import { mocks } from "@budibase/backend-core/tests" mocks.licenses.useBackups() describe("/backups", () => { - let request = setup.getRequest() let config = setup.getConfig() afterAll(setup.afterAll) @@ -59,10 +58,8 @@ describe("/backups", () => { await config.createScreen() const exportRes = await config.api.backup.createBackup(appId) expect(exportRes.backupId).toBeDefined() - const importRes = await config.api.backup.importBackup( - appId, - exportRes.backupId - ) + await config.api.backup.waitForBackupToComplete(appId, exportRes.backupId) + await config.api.backup.importBackup(appId, exportRes.backupId) }) }) diff --git a/packages/server/src/tests/utilities/api/backup.ts b/packages/server/src/tests/utilities/api/backup.ts index f9cbc7086e..8cd1e58a29 100644 --- a/packages/server/src/tests/utilities/api/backup.ts +++ b/packages/server/src/tests/utilities/api/backup.ts @@ -31,6 +31,19 @@ export class BackupAPI extends TestAPI { return result.body as CreateAppBackupResponse } + waitForBackupToComplete = async (appId: string, backupId: string) => { + for (let i = 0; i < 10; i++) { + await new Promise(resolve => setTimeout(resolve, 1000)) + const result = await this.request + .get(`/api/apps/${appId}/backups/${backupId}/file`) + .set(this.config.defaultHeaders()) + if (result.status === 200) { + return + } + } + throw new Error("Backup did not complete") + } + importBackup = async ( appId: string, backupId: string From e8d38b17d9b68f7a7263708d6000f6f674ba8230 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 20 Feb 2024 13:35:29 +0100 Subject: [PATCH 27/43] Update pro ref --- packages/pro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pro b/packages/pro index 6f3a0b7f72..60e47a8249 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 6f3a0b7f72d16f9604179af98ff7602ca0df2737 +Subproject commit 60e47a8249fd6291a6bc20fe3fe6776b11938fa1 From 91d45024e2b675668f2692185d8aba6ec376243c Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Tue, 20 Feb 2024 12:50:11 +0000 Subject: [PATCH 28/43] Bump version to 2.20.3 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index 04f5c6e880..1cba488aa3 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.20.2", + "version": "2.20.3", "npmClient": "yarn", "packages": [ "packages/*", From 5dfa46037408b1fe7e8026a15e81d90d7234c8af Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 20 Feb 2024 16:23:35 +0000 Subject: [PATCH 29/43] Disabling VM by default in string-templates, backend services *MUST* set their JS runner specifically rather than assuming the VM library by default. --- .../src/helpers/javascript.js | 7 +++- packages/string-templates/src/index.js | 37 +++++++++++-------- packages/string-templates/src/utilities.js | 8 ++++ packages/string-templates/test/vm.spec.js | 27 ++++++++++++++ 4 files changed, 62 insertions(+), 17 deletions(-) create mode 100644 packages/string-templates/test/vm.spec.js diff --git a/packages/string-templates/src/helpers/javascript.js b/packages/string-templates/src/helpers/javascript.js index 4a7f602690..7827736812 100644 --- a/packages/string-templates/src/helpers/javascript.js +++ b/packages/string-templates/src/helpers/javascript.js @@ -1,4 +1,4 @@ -const { atob } = require("../utilities") +const { atob, isBackendService, isJSAllowed } = require("../utilities") const cloneDeep = require("lodash.clonedeep") const { LITERAL_MARKER } = require("../helpers/constants") const { getJsHelperList } = require("./list") @@ -7,6 +7,9 @@ const { getJsHelperList } = require("./list") // This setter is used in the entrypoint (either index.js or index.mjs). let runJS module.exports.setJSRunner = runner => (runJS = runner) +module.exports.removeJSRunner = () => { + runJS = undefined +} let onErrorLog module.exports.setOnErrorLog = delegate => (onErrorLog = delegate) @@ -39,7 +42,7 @@ const getContextValue = (path, context) => { // Evaluates JS code against a certain context module.exports.processJS = (handlebars, context) => { - if (process && process.env.NO_JS) { + if (!isJSAllowed() || (isBackendService() && !runJS)) { throw new Error("JS disabled in environment.") } try { diff --git a/packages/string-templates/src/index.js b/packages/string-templates/src/index.js index bcd63d2e6f..2dc360bd96 100644 --- a/packages/string-templates/src/index.js +++ b/packages/string-templates/src/index.js @@ -2,7 +2,7 @@ const vm = require("vm") const handlebars = require("handlebars") const { registerAll, registerMinimum } = require("./helpers/index") const processors = require("./processors") -const { atob, btoa } = require("./utilities") +const { atob, btoa, isBackendService, isJSAllowed } = require("./utilities") const manifest = require("../manifest.json") const { FIND_HBS_REGEX, @@ -404,18 +404,25 @@ module.exports.JsErrorTimeout = errors.JsErrorTimeout module.exports.helpersToRemoveForJs = helpersToRemoveForJs -if (process && !process.env.NO_JS) { - /** - * Use polyfilled vm to run JS scripts in a browser Env - */ - javascript.setJSRunner((js, context) => { - context = { - ...context, - alert: undefined, - setInterval: undefined, - setTimeout: undefined, - } - vm.createContext(context) - return vm.runInNewContext(js, context, { timeout: 1000 }) - }) +function defaultJSSetup() { + if (!isBackendService()) { + /** + * Use polyfilled vm to run JS scripts in a browser Env + */ + javascript.setJSRunner((js, context) => { + context = { + ...context, + alert: undefined, + setInterval: undefined, + setTimeout: undefined, + } + vm.createContext(context) + return vm.runInNewContext(js, context, { timeout: 1000 }) + }) + } else { + javascript.removeJSRunner() + } } +defaultJSSetup() + +module.exports.defaultJSSetup = defaultJSSetup diff --git a/packages/string-templates/src/utilities.js b/packages/string-templates/src/utilities.js index 775c150e1b..00b2d7d855 100644 --- a/packages/string-templates/src/utilities.js +++ b/packages/string-templates/src/utilities.js @@ -4,6 +4,14 @@ module.exports.FIND_HBS_REGEX = /{{([^{].*?)}}/g module.exports.FIND_ANY_HBS_REGEX = /{?{{([^{].*?)}}}?/g module.exports.FIND_TRIPLE_HBS_REGEX = /{{{([^{].*?)}}}/g +module.exports.isBackendService = () => { + return typeof window === "undefined" +} + +module.exports.isJSAllowed = () => { + return process && !process.env.NO_JS +} + // originally this could be done with a single regex using look behinds // but safari does not support this feature // original regex: /(? { + const utilities = jest.requireActual("../src/utilities") + return { + ...utilities, + isBackendService: jest.fn().mockReturnValue(true), + } +}) +const { defaultJSSetup, processStringSync, encodeJSBinding } = require("../src") +const { isBackendService } = require("../src/utilities") +const mockedBackendService = jest.mocked(isBackendService) + +const binding = encodeJSBinding("return 1") +describe("confirm VM is available when expected and when not", () => { + it("shouldn't have JS available in a backend service by default", () => { + defaultJSSetup() + const result = processStringSync(binding, {}) + // shouldn't process at all + expect(result).toBe(binding) + }) + + it("should have JS available in frontend environments", () => { + mockedBackendService.mockReturnValue(false) + defaultJSSetup() + const result = processStringSync(binding, {}) + expect(result).toBe(1) + }) +}) From f294497b4a9e82cb1da47d99183e576cf6e03e16 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 20 Feb 2024 17:38:33 +0100 Subject: [PATCH 30/43] Update submodule ref --- packages/account-portal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/account-portal b/packages/account-portal index 8c446c4ba3..d643d74c57 160000 --- a/packages/account-portal +++ b/packages/account-portal @@ -1 +1 @@ -Subproject commit 8c446c4ba385592127fa31755d3b64467b291882 +Subproject commit d643d74c577a2ddb782489a6907639461cbcc438 From 4e61230c9a30be5f1c024c050fa8c12c4e01bf94 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 20 Feb 2024 17:17:13 +0000 Subject: [PATCH 31/43] Removing unused function. --- packages/string-templates/src/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/string-templates/src/index.js b/packages/string-templates/src/index.js index 2dc360bd96..0125b9e0ab 100644 --- a/packages/string-templates/src/index.js +++ b/packages/string-templates/src/index.js @@ -2,7 +2,7 @@ const vm = require("vm") const handlebars = require("handlebars") const { registerAll, registerMinimum } = require("./helpers/index") const processors = require("./processors") -const { atob, btoa, isBackendService, isJSAllowed } = require("./utilities") +const { atob, btoa, isBackendService } = require("./utilities") const manifest = require("../manifest.json") const { FIND_HBS_REGEX, From 41599540391cf8e5ef788ead457eb47088a4f23b Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Tue, 20 Feb 2024 14:23:42 -0300 Subject: [PATCH 32/43] acct portal submodule --- packages/account-portal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/account-portal b/packages/account-portal index d643d74c57..4384bc742c 160000 --- a/packages/account-portal +++ b/packages/account-portal @@ -1 +1 @@ -Subproject commit d643d74c577a2ddb782489a6907639461cbcc438 +Subproject commit 4384bc742ca22fb1e9bf91843e65ae929daf17e2 From 2145480572549d2fb1590e98e3da4521cf5649e4 Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Tue, 20 Feb 2024 18:09:43 +0000 Subject: [PATCH 33/43] Bump version to 2.20.4 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index 1cba488aa3..41b473161a 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.20.3", + "version": "2.20.4", "npmClient": "yarn", "packages": [ "packages/*", From dfb1774d2ce1044cd1c65e25e0f5ecd312d7bd0f Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Tue, 20 Feb 2024 16:32:55 -0300 Subject: [PATCH 34/43] bump default memory unit --- packages/server/src/environment.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/environment.ts b/packages/server/src/environment.ts index 873c392942..7842a9b3dc 100644 --- a/packages/server/src/environment.ts +++ b/packages/server/src/environment.ts @@ -27,7 +27,7 @@ const DEFAULTS = { TEMPLATE_REPOSITORY: "app", PLUGINS_DIR: "/plugins", FORKED_PROCESS_NAME: "main", - JS_RUNNER_MEMORY_LIMIT: 64, + JS_RUNNER_MEMORY_LIMIT: 96, } const QUERY_THREAD_TIMEOUT = From 73fe2e0d1d9ff4e8a5fc5a8f2cf3fded3304c16c Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Tue, 20 Feb 2024 16:35:34 -0300 Subject: [PATCH 35/43] update JS per execution time --- packages/server/src/environment.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/server/src/environment.ts b/packages/server/src/environment.ts index 7842a9b3dc..f304ce4eb2 100644 --- a/packages/server/src/environment.ts +++ b/packages/server/src/environment.ts @@ -23,11 +23,11 @@ const DEFAULTS = { AUTOMATION_THREAD_TIMEOUT: 12000, AUTOMATION_SYNC_TIMEOUT: 120000, AUTOMATION_MAX_ITERATIONS: 200, - JS_PER_EXECUTION_TIME_LIMIT_MS: 1000, + JS_PER_EXECUTION_TIME_LIMIT_MS: 1500, TEMPLATE_REPOSITORY: "app", PLUGINS_DIR: "/plugins", FORKED_PROCESS_NAME: "main", - JS_RUNNER_MEMORY_LIMIT: 96, + JS_RUNNER_MEMORY_LIMIT: 64, } const QUERY_THREAD_TIMEOUT = From 033ab7110906144117990606063823506247c92d Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Tue, 20 Feb 2024 19:51:22 +0000 Subject: [PATCH 36/43] Bump version to 2.20.5 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index 41b473161a..a62c15997d 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.20.4", + "version": "2.20.5", "npmClient": "yarn", "packages": [ "packages/*", From e988890a7ee6ee9458218d913788101e2cf2b523 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 21 Feb 2024 10:12:06 +0100 Subject: [PATCH 37/43] Remove defaultUserValues from test config --- .../src/tests/utilities/TestConfiguration.ts | 81 ++++++++----------- 1 file changed, 33 insertions(+), 48 deletions(-) diff --git a/packages/server/src/tests/utilities/TestConfiguration.ts b/packages/server/src/tests/utilities/TestConfiguration.ts index ea3204536a..00550c2c24 100644 --- a/packages/server/src/tests/utilities/TestConfiguration.ts +++ b/packages/server/src/tests/utilities/TestConfiguration.ts @@ -76,14 +76,6 @@ mocks.licenses.useUnlimited() dbInit() -type DefaultUserValues = { - globalUserId: string - email: string - firstName: string - lastName: string - csrfToken: string -} - export interface TableToBuild extends Omit { sourceId?: string sourceType?: TableSourceType @@ -105,8 +97,8 @@ export default class TestConfiguration { automation: any datasource?: Datasource tenantId?: string - defaultUserValues: DefaultUserValues api: API + csrfToken?: string constructor(openServer = true) { if (openServer) { @@ -121,21 +113,10 @@ export default class TestConfiguration { } this.appId = null this.allApps = [] - this.defaultUserValues = this.populateDefaultUserValues() this.api = new API(this) } - populateDefaultUserValues(): DefaultUserValues { - return { - globalUserId: `us_${newid()}`, - email: generator.email(), - firstName: generator.first(), - lastName: generator.last(), - csrfToken: generator.hash(), - } - } - getRequest() { return this.request } @@ -160,15 +141,6 @@ export default class TestConfiguration { return this.prodAppId } - getUserDetails() { - return { - globalId: this.defaultUserValues.globalUserId, - email: this.defaultUserValues.email, - firstName: this.defaultUserValues.firstName, - lastName: this.defaultUserValues.lastName, - } - } - async doInContext( appId: string | null, task: () => Promise @@ -300,15 +272,27 @@ export default class TestConfiguration { } // USER / AUTH - async globalUser({ - id = this.defaultUserValues.globalUserId, - firstName = this.defaultUserValues.firstName, - lastName = this.defaultUserValues.lastName, - builder = true, - admin = false, - email = this.defaultUserValues.email, - roles, - }: any = {}): Promise { + async globalUser( + config: { + id?: string + firstName?: string + lastName?: string + builder?: boolean + admin?: boolean + email?: string + roles?: any + } = {} + ): Promise { + const { + id = `us_${newid()}`, + firstName = generator.first(), + lastName = generator.last(), + builder = true, + admin = false, + email, + roles, + } = config + const db = tenancy.getTenantDB(this.getTenantId()) let existing try { @@ -327,7 +311,7 @@ export default class TestConfiguration { await sessions.createASession(id, { sessionId: "sessionid", tenantId: this.getTenantId(), - csrfToken: this.defaultUserValues.csrfToken, + csrfToken: this.csrfToken, }) if (builder) { user.builder = { global: true } @@ -358,9 +342,9 @@ export default class TestConfiguration { } = {} ): Promise { let { id, firstName, lastName, email, builder, admin, roles } = user - firstName = firstName || this.defaultUserValues.firstName - lastName = lastName || this.defaultUserValues.lastName - email = email || this.defaultUserValues.email + ;(firstName = firstName || generator.first()), + (lastName = lastName || generator.last()), + (email = email || generator.email()) roles = roles || {} if (builder == null) { builder = true @@ -448,7 +432,7 @@ export default class TestConfiguration { defaultHeaders(extras = {}, prodApp = false) { const tenantId = this.getTenantId() const authObj: AuthToken = { - userId: this.defaultUserValues.globalUserId, + userId: this.user.globalUserId, sessionId: "sessionid", tenantId, } @@ -457,7 +441,7 @@ export default class TestConfiguration { const headers: any = { Accept: "application/json", Cookie: [`${constants.Cookie.Auth}=${authToken}`], - [constants.Header.CSRF_TOKEN]: this.defaultUserValues.csrfToken, + [constants.Header.CSRF_TOKEN]: this.csrfToken, Host: this.tenantHost(), ...extras, } @@ -487,7 +471,7 @@ export default class TestConfiguration { async basicRoleHeaders() { return await this.roleHeaders({ - email: this.defaultUserValues.email, + email: generator.email(), builder: false, prodApp: true, roleId: roles.BUILTIN_ROLE_IDS.BASIC, @@ -495,7 +479,7 @@ export default class TestConfiguration { } async roleHeaders({ - email = this.defaultUserValues.email, + email = generator.email(), roleId = roles.BUILTIN_ROLE_IDS.ADMIN, builder = false, prodApp = true, @@ -519,11 +503,12 @@ export default class TestConfiguration { } async newTenant(appName = newid()): Promise { - this.defaultUserValues = this.populateDefaultUserValues() this.tenantId = structures.tenant.id() this.user = await this.globalUser() this.globalUserId = this.user._id this.userMetadataId = generateUserMetadataID(this.globalUserId) + + this.csrfToken = generator.hash() return this.createApp(appName) } @@ -533,7 +518,7 @@ export default class TestConfiguration { // API - async generateApiKey(userId = this.defaultUserValues.globalUserId) { + async generateApiKey(userId = this.user.globalUserId) { const db = tenancy.getTenantDB(this.getTenantId()) const id = dbCore.generateDevInfoID(userId) let devInfo: any From f6e968efe848b568b8edeb59a17bf764e8a506a1 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 21 Feb 2024 10:18:46 +0100 Subject: [PATCH 38/43] Fix test --- packages/server/src/sdk/users/tests/utils.spec.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/server/src/sdk/users/tests/utils.spec.ts b/packages/server/src/sdk/users/tests/utils.spec.ts index 86dc411caf..efe790d49b 100644 --- a/packages/server/src/sdk/users/tests/utils.spec.ts +++ b/packages/server/src/sdk/users/tests/utils.spec.ts @@ -84,7 +84,8 @@ describe("syncGlobalUsers", () => { await syncGlobalUsers() const metadata = await rawUserMetadata() - expect(metadata).toHaveLength(2) + + expect(metadata).toHaveLength(2 + 1) // ADMIN user created in test bootstrap still in the application expect(metadata).toContainEqual( expect.objectContaining({ _id: db.generateUserMetadataID(user1._id!), @@ -121,7 +122,7 @@ describe("syncGlobalUsers", () => { await syncGlobalUsers() const metadata = await rawUserMetadata() - expect(metadata).toHaveLength(1) //ADMIN user created in test bootstrap still in the application + expect(metadata).toHaveLength(1) // ADMIN user created in test bootstrap still in the application }) }) }) From 0b5226413b7d76353685c1e70eb9eb18c1a7f922 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 21 Feb 2024 10:36:17 +0100 Subject: [PATCH 39/43] Fix ids --- .../src/tests/utilities/TestConfiguration.ts | 21 ++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/packages/server/src/tests/utilities/TestConfiguration.ts b/packages/server/src/tests/utilities/TestConfiguration.ts index 00550c2c24..0204eeb178 100644 --- a/packages/server/src/tests/utilities/TestConfiguration.ts +++ b/packages/server/src/tests/utilities/TestConfiguration.ts @@ -91,7 +91,6 @@ export default class TestConfiguration { prodApp: any prodAppId: any user: any - globalUserId: any userMetadataId: any table?: Table automation: any @@ -100,6 +99,10 @@ export default class TestConfiguration { api: API csrfToken?: string + private get globalUserId() { + return this.user._id + } + constructor(openServer = true) { if (openServer) { // use a random port because it doesn't matter @@ -141,6 +144,15 @@ export default class TestConfiguration { return this.prodAppId } + getUserDetails() { + return { + globalId: this.globalUserId, + email: this.user.email, + firstName: this.user.firstName, + lastName: this.user.lastName, + } + } + async doInContext( appId: string | null, task: () => Promise @@ -432,7 +444,7 @@ export default class TestConfiguration { defaultHeaders(extras = {}, prodApp = false) { const tenantId = this.getTenantId() const authObj: AuthToken = { - userId: this.user.globalUserId, + userId: this.globalUserId, sessionId: "sessionid", tenantId, } @@ -505,8 +517,7 @@ export default class TestConfiguration { async newTenant(appName = newid()): Promise { this.tenantId = structures.tenant.id() this.user = await this.globalUser() - this.globalUserId = this.user._id - this.userMetadataId = generateUserMetadataID(this.globalUserId) + this.userMetadataId = generateUserMetadataID(this.user._id) this.csrfToken = generator.hash() return this.createApp(appName) @@ -518,7 +529,7 @@ export default class TestConfiguration { // API - async generateApiKey(userId = this.user.globalUserId) { + async generateApiKey(userId = this.user._id) { const db = tenancy.getTenantDB(this.getTenantId()) const id = dbCore.generateDevInfoID(userId) let devInfo: any From f7d5ccee3c95a68f406d3e3ced363dfd53cd50fb Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 21 Feb 2024 10:42:36 +0100 Subject: [PATCH 40/43] Fix setting up email --- packages/server/src/tests/utilities/TestConfiguration.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/tests/utilities/TestConfiguration.ts b/packages/server/src/tests/utilities/TestConfiguration.ts index 0204eeb178..5b20f83d77 100644 --- a/packages/server/src/tests/utilities/TestConfiguration.ts +++ b/packages/server/src/tests/utilities/TestConfiguration.ts @@ -301,7 +301,7 @@ export default class TestConfiguration { lastName = generator.last(), builder = true, admin = false, - email, + email = generator.email(), roles, } = config From f68e7359c6379096a55db163bb07bb4b6492853f Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 21 Feb 2024 10:57:49 +0100 Subject: [PATCH 41/43] Fix csrf usage --- packages/server/src/tests/utilities/TestConfiguration.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/server/src/tests/utilities/TestConfiguration.ts b/packages/server/src/tests/utilities/TestConfiguration.ts index 5b20f83d77..bbe3260d26 100644 --- a/packages/server/src/tests/utilities/TestConfiguration.ts +++ b/packages/server/src/tests/utilities/TestConfiguration.ts @@ -515,11 +515,12 @@ export default class TestConfiguration { } async newTenant(appName = newid()): Promise { + this.csrfToken = generator.hash() + this.tenantId = structures.tenant.id() this.user = await this.globalUser() this.userMetadataId = generateUserMetadataID(this.user._id) - this.csrfToken = generator.hash() return this.createApp(appName) } From 73bf29ab3c0edab5615f56d65c6410fcaf1cdc4d Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 21 Feb 2024 11:22:43 +0100 Subject: [PATCH 42/43] Clean code --- .../src/tests/utilities/TestConfiguration.ts | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/packages/server/src/tests/utilities/TestConfiguration.ts b/packages/server/src/tests/utilities/TestConfiguration.ts index bbe3260d26..8e6ecdfeb1 100644 --- a/packages/server/src/tests/utilities/TestConfiguration.ts +++ b/packages/server/src/tests/utilities/TestConfiguration.ts @@ -353,14 +353,16 @@ export default class TestConfiguration { roles?: UserRoles } = {} ): Promise { - let { id, firstName, lastName, email, builder, admin, roles } = user - ;(firstName = firstName || generator.first()), - (lastName = lastName || generator.last()), - (email = email || generator.email()) - roles = roles || {} - if (builder == null) { - builder = true - } + const { + id, + firstName = generator.first(), + lastName = generator.last(), + email = generator.email(), + builder = true, + admin, + roles, + } = user + const globalId = !id ? `us_${Math.random()}` : `us_${id}` const resp = await this.globalUser({ id: globalId, @@ -369,7 +371,7 @@ export default class TestConfiguration { email, builder, admin, - roles, + roles: roles || {}, }) await cache.user.invalidateUser(globalId) return resp From f417c2d8a4d96cc958ecb3a35281030cfe6c1f2c Mon Sep 17 00:00:00 2001 From: Joe <49767913+joebudi@users.noreply.github.com> Date: Wed, 21 Feb 2024 13:28:35 +0000 Subject: [PATCH 43/43] Joe's lab day minor updates (#12944) * Change default button type to CTA - change default button type to CTA - change ordering of types/variants * Fix layout shift within portal Within the portal, when navigating from screen to screen, there's a slight layout shift caused by the scrollbar. This is a small fix. * row/column icons change The current row/column icons for positioning components are confusing. I believe these icons are easier to understand. * Fix for horizontal scrollbar showing When adding/removing actions within automations, the horizontal scrollbar flashes. Fix. * Title change for Upload data Upload data is not wrong, but it's best to be explicit. * Increase size of upgrade button * small fix for the styling inconsistency * Dianostics padding fix * lint fix * update account-portal * update icons --------- Co-authored-by: melohagan <101575380+melohagan@users.noreply.github.com> Co-authored-by: Mel O'Hagan --- packages/bbui/src/Layout/Page.svelte | 1 + packages/bbui/src/Typography/Body.svelte | 6 +++++ packages/bbui/src/Typography/Heading.svelte | 4 +++ .../FlowChart/FlowChart.svelte | 1 + .../builder/app/[application]/data/new.svelte | 2 +- .../portal/_components/UpgradeButton.svelte | 4 +-- .../portal/settings/diagnostics.svelte | 9 ++++--- packages/client/manifest.json | 26 +++++++++---------- .../client/src/components/app/Button.svelte | 2 +- 9 files changed, 35 insertions(+), 20 deletions(-) diff --git a/packages/bbui/src/Layout/Page.svelte b/packages/bbui/src/Layout/Page.svelte index 57c264231b..2169a12459 100644 --- a/packages/bbui/src/Layout/Page.svelte +++ b/packages/bbui/src/Layout/Page.svelte @@ -43,6 +43,7 @@ flex-direction: row; justify-content: flex-start; align-items: stretch; + overflow-y: scroll !important; flex: 1 1 auto; overflow-x: hidden; } diff --git a/packages/bbui/src/Typography/Body.svelte b/packages/bbui/src/Typography/Body.svelte index 71b4dca248..2123eeee95 100644 --- a/packages/bbui/src/Typography/Body.svelte +++ b/packages/bbui/src/Typography/Body.svelte @@ -20,3 +20,9 @@ >

+ + diff --git a/packages/bbui/src/Typography/Heading.svelte b/packages/bbui/src/Typography/Heading.svelte index c0d0571143..50522fffc3 100644 --- a/packages/bbui/src/Typography/Heading.svelte +++ b/packages/bbui/src/Typography/Heading.svelte @@ -21,4 +21,8 @@ h1 { font-family: var(--font-accent); } + + h1 { + text-wrap: balance; + } diff --git a/packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowChart.svelte b/packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowChart.svelte index 4c458a5627..1ace6c0f00 100644 --- a/packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowChart.svelte +++ b/packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowChart.svelte @@ -130,6 +130,7 @@ flex-grow: 1; padding: 23px 23px 80px; box-sizing: border-box; + overflow-x: hidden; } .header.scrolling { diff --git a/packages/builder/src/pages/builder/app/[application]/data/new.svelte b/packages/builder/src/pages/builder/app/[application]/data/new.svelte index c07a9f563d..20efd3667b 100644 --- a/packages/builder/src/pages/builder/app/[application]/data/new.svelte +++ b/packages/builder/src/pages/builder/app/[application]/data/new.svelte @@ -77,7 +77,7 @@ internalTableModal.show({ promptUpload: true })} - title="Upload data" + title="Upload CSV / JSON" description="Non-relational" {disabled} > diff --git a/packages/builder/src/pages/builder/portal/_components/UpgradeButton.svelte b/packages/builder/src/pages/builder/portal/_components/UpgradeButton.svelte index b12efd6f03..59a791538a 100644 --- a/packages/builder/src/pages/builder/portal/_components/UpgradeButton.svelte +++ b/packages/builder/src/pages/builder/portal/_components/UpgradeButton.svelte @@ -10,7 +10,7 @@ {#if $admin.cloud && $auth?.user?.accountPortalAccess}