MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' of github.com:Budibase/budibase into fix/isolated-vm-tests

This commit is contained in:
mike12345567 2024-02-21 15:19:53 +00:00
parent fcea092667 f417c2d8a4
commit 050508b5b6
43 changed files with 510 additions and 283 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

103
hosting/couchdb/Dockerfile
View File

@ -1,4 +1,101 @@
FROM couchdb:3.2.1
# Modified from https://github.com/apache/couchdb-docker/blob/main/3.3.3/Dockerfile
#
# Everything in this `base` image is adapted from the official `couchdb` image's
# Dockerfile. Only modifications related to upgrading from Debian bullseye to
# bookworm have been included. The `runner` image contains Budibase's
# customisations to the image, e.g. adding Clouseau.
FROM node:20-slim AS base
# Add CouchDB user account to make sure the IDs are assigned consistently
RUN groupadd -g 5984 -r couchdb && useradd -u 5984 -d /opt/couchdb -g couchdb couchdb
# be sure GPG and apt-transport-https are available and functional
RUN set -ex; \
apt-get update; \
apt-get install -y --no-install-recommends \
apt-transport-https \
ca-certificates \
dirmngr \
gnupg \
; \
rm -rf /var/lib/apt/lists/*
# grab tini for signal handling and zombie reaping
# see https://github.com/apache/couchdb-docker/pull/28#discussion_r141112407
RUN set -eux; \
apt-get update; \
apt-get install -y --no-install-recommends tini; \
rm -rf /var/lib/apt/lists/*; \
tini --version
# http://docs.couchdb.org/en/latest/install/unix.html#installing-the-apache-couchdb-packages
ENV GPG_COUCH_KEY \
# gpg: rsa8192 205-01-19 The Apache Software Foundation (Package repository signing key) <root@apache.org>
390EF70BB1EA12B2773962950EE62FB37A00258D
RUN set -eux; \
apt-get update; \
apt-get install -y curl; \
export GNUPGHOME="$(mktemp -d)"; \
curl -fL -o keys.asc https://couchdb.apache.org/repo/keys.asc; \
gpg --batch --import keys.asc; \
gpg --batch --export "${GPG_COUCH_KEY}" > /usr/share/keyrings/couchdb-archive-keyring.gpg; \
command -v gpgconf && gpgconf --kill all || :; \
rm -rf "$GNUPGHOME"; \
apt-key list; \
apt purge -y --autoremove curl; \
rm -rf /var/lib/apt/lists/*
ENV COUCHDB_VERSION 3.3.3
RUN . /etc/os-release; \
echo "deb [signed-by=/usr/share/keyrings/couchdb-archive-keyring.gpg] https://apache.jfrog.io/artifactory/couchdb-deb/ ${VERSION_CODENAME} main" | \
tee /etc/apt/sources.list.d/couchdb.list >/dev/null
# https://github.com/apache/couchdb-pkg/blob/master/debian/README.Debian
RUN set -eux; \
apt-get update; \
\
echo "couchdb couchdb/mode select none" | debconf-set-selections; \
# we DO want recommends this time
DEBIAN_FRONTEND=noninteractive apt-get install -y --allow-downgrades --allow-remove-essential --allow-change-held-packages \
couchdb="$COUCHDB_VERSION"~bookworm \
; \
# Undo symlinks to /var/log and /var/lib
rmdir /var/lib/couchdb /var/log/couchdb; \
rm /opt/couchdb/data /opt/couchdb/var/log; \
mkdir -p /opt/couchdb/data /opt/couchdb/var/log; \
chown couchdb:couchdb /opt/couchdb/data /opt/couchdb/var/log; \
chmod 777 /opt/couchdb/data /opt/couchdb/var/log; \
# Remove file that sets logging to a file
rm /opt/couchdb/etc/default.d/10-filelog.ini; \
# Check we own everything in /opt/couchdb. Matches the command in dockerfile_entrypoint.sh
find /opt/couchdb \! \( -user couchdb -group couchdb \) -exec chown -f couchdb:couchdb '{}' +; \
# Setup directories and permissions for config. Technically these could be 555 and 444 respectively
# but we keep them as 755 and 644 for consistency with CouchDB defaults and the dockerfile_entrypoint.sh.
find /opt/couchdb/etc -type d ! -perm 0755 -exec chmod -f 0755 '{}' +; \
find /opt/couchdb/etc -type f ! -perm 0644 -exec chmod -f 0644 '{}' +; \
# only local.d needs to be writable for the docker_entrypoint.sh
chmod -f 0777 /opt/couchdb/etc/local.d; \
# apt clean-up
rm -rf /var/lib/apt/lists/*;
# Add configuration
COPY --chown=couchdb:couchdb couch/10-docker-default.ini /opt/couchdb/etc/default.d/
# COPY --chown=couchdb:couchdb vm.args /opt/couchdb/etc/
COPY docker-entrypoint.sh /usr/local/bin
RUN ln -s usr/local/bin/docker-entrypoint.sh /docker-entrypoint.sh # backwards compat
ENTRYPOINT ["tini", "--", "/docker-entrypoint.sh"]
VOLUME /opt/couchdb/data
# 5984: Main CouchDB endpoint
# 4369: Erlang portmap daemon (epmd)
# 9100: CouchDB cluster communication port
EXPOSE 5984 4369 9100
CMD ["/opt/couchdb/bin/couchdb"]
FROM base as runner
ENV COUCHDB_USER admin
ENV COUCHDB_PASSWORD admin
@ -6,9 +103,9 @@ EXPOSE 5984
RUN apt-get update && apt-get install -y --no-install-recommends software-properties-common wget unzip curl && \
wget -O - https://packages.adoptium.net/artifactory/api/gpg/key/public | apt-key add - && \
apt-add-repository 'deb http://security.debian.org/debian-security bullseye-security/updates main' && \
apt-add-repository 'deb http://security.debian.org/debian-security bookworm-security/updates main' && \
apt-add-repository 'deb http://archive.debian.org/debian stretch-backports main' && \
apt-add-repository 'deb https://packages.adoptium.net/artifactory/deb bullseye main' && \
apt-add-repository 'deb https://packages.adoptium.net/artifactory/deb bookworm main' && \
apt-get update && apt-get install -y --no-install-recommends temurin-8-jdk && \
rm -rf /var/lib/apt/lists/

2
hosting/couchdb/clouseau/clouseau.ini
View File

@ -4,7 +4,7 @@
name=clouseau@127.0.0.1
; set this to the same distributed Erlang cookie used by the CouchDB nodes
cookie=monster
cookie=COUCHDB_ERLANG_COOKIE
; the path where you would like to store the search index files
dir=DATA_DIR/search

8
hosting/couchdb/couch/10-docker-default.ini Normal file
View File

@ -0,0 +1,8 @@
; CouchDB Configuration Settings
; Custom settings should be made in this file. They will override settings
; in default.ini, but unlike changes made to default.ini, this file won't be
; overwritten on server upgrade.
[chttpd]
bind_address = any

2
hosting/couchdb/couch/vm.args
View File

@ -12,7 +12,7 @@
# erlang cookie for clouseau security
-name couchdb@127.0.0.1
-setcookie monster
-setcookie COUCHDB_ERLANG_COOKIE
# Ensure that the Erlang VM listens on a known port
-kernel inet_dist_listen_min 9100

122
hosting/couchdb/docker-entrypoint.sh Executable file
View File

@ -0,0 +1,122 @@
#!/bin/bash
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License. You may obtain a copy of
# the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations under
# the License.
set -e
# first arg is `-something` or `+something`
if [ "${1#-}" != "$1" ] || [ "${1#+}" != "$1" ]; then
set -- /opt/couchdb/bin/couchdb "$@"
fi
# first arg is the bare word `couchdb`
if [ "$1" = 'couchdb' ]; then
shift
set -- /opt/couchdb/bin/couchdb "$@"
fi
if [ "$1" = '/opt/couchdb/bin/couchdb' ]; then
# this is where runtime configuration changes will be written.
# we need to explicitly touch it here in case /opt/couchdb/etc has
# been mounted as an external volume, in which case it won't exist.
# If running as the couchdb user (i.e. container starts as root),
# write permissions will be granted below.
touch /opt/couchdb/etc/local.d/docker.ini
# if user is root, assume running under the couchdb user (default)
# and ensure it is able to access files and directories that may be mounted externally
if [ "$(id -u)" = '0' ]; then
# Check that we own everything in /opt/couchdb and fix if necessary. We also
# add the `-f` flag in all the following invocations because there may be
# cases where some of these ownership and permissions issues are non-fatal
# (e.g. a config file owned by root with o+r is actually fine), and we don't
# to be too aggressive about crashing here ...
find /opt/couchdb \! \( -user couchdb -group couchdb \) -exec chown -f couchdb:couchdb '{}' +
# Ensure that data files have the correct permissions. We were previously
# preventing any access to these files outside of couchdb:couchdb, but it
# turns out that CouchDB itself does not set such restrictive permissions
# when it creates the files. The approach taken here ensures that the
# contents of the datadir have the same permissions as they had when they
# were initially created. This should minimize any startup delay.
find /opt/couchdb/data -type d ! -perm 0755 -exec chmod -f 0755 '{}' +
find /opt/couchdb/data -type f ! -perm 0644 -exec chmod -f 0644 '{}' +
# Do the same thing for configuration files and directories. Technically
# CouchDB only needs read access to the configuration files as all online
# changes will be applied to the "docker.ini" file below, but we set 644
# for the sake of consistency.
find /opt/couchdb/etc -type d ! -perm 0755 -exec chmod -f 0755 '{}' +
find /opt/couchdb/etc -type f ! -perm 0644 -exec chmod -f 0644 '{}' +
fi
if [ ! -z "$NODENAME" ] && ! grep "couchdb@" /opt/couchdb/etc/vm.args; then
echo "-name couchdb@$NODENAME" >> /opt/couchdb/etc/vm.args
fi
if [ "$COUCHDB_USER" ] && [ "$COUCHDB_PASSWORD" ]; then
# Create admin only if not already present
if ! grep -Pzoqr "\[admins\]\n$COUCHDB_USER =" /opt/couchdb/etc/local.d/*.ini /opt/couchdb/etc/local.ini; then
printf "\n[admins]\n%s = %s\n" "$COUCHDB_USER" "$COUCHDB_PASSWORD" >> /opt/couchdb/etc/local.d/docker.ini
fi
fi
if [ "$COUCHDB_SECRET" ]; then
# Set secret only if not already present
if ! grep -Pzoqr "\[chttpd_auth\]\nsecret =" /opt/couchdb/etc/local.d/*.ini /opt/couchdb/etc/local.ini; then
printf "\n[chttpd_auth]\nsecret = %s\n" "$COUCHDB_SECRET" >> /opt/couchdb/etc/local.d/docker.ini
fi
fi
if [ "$COUCHDB_ERLANG_COOKIE" ]; then
cookieFile='/opt/couchdb/.erlang.cookie'
if [ -e "$cookieFile" ]; then
if [ "$(cat "$cookieFile" 2>/dev/null)" != "$COUCHDB_ERLANG_COOKIE" ]; then
echo >&2
echo >&2 "warning: $cookieFile contents do not match COUCHDB_ERLANG_COOKIE"
echo >&2
fi
else
echo "$COUCHDB_ERLANG_COOKIE" > "$cookieFile"
fi
chown couchdb:couchdb "$cookieFile"
chmod 600 "$cookieFile"
fi
if [ "$(id -u)" = '0' ]; then
chown -f couchdb:couchdb /opt/couchdb/etc/local.d/docker.ini || true
fi
# if we don't find an [admins] section followed by a non-comment, display a warning
if ! grep -Pzoqr '\[admins\]\n[^;]\w+' /opt/couchdb/etc/default.d/*.ini /opt/couchdb/etc/local.d/*.ini /opt/couchdb/etc/local.ini; then
# The - option suppresses leading tabs but *not* spaces. :)
cat >&2 <<-'EOWARN'
*************************************************************
ERROR: CouchDB 3.0+ will no longer run in "Admin Party"
mode. You *MUST* specify an admin user and
password, either via your own .ini file mapped
into the container at /opt/couchdb/etc/local.ini
or inside /opt/couchdb/etc/local.d, or with
"-e COUCHDB_USER=admin -e COUCHDB_PASSWORD=password"
to set it via "docker run".
*************************************************************
EOWARN
exit 1
fi
if [ "$(id -u)" = '0' ]; then
export HOME=$(echo ~couchdb)
exec setpriv --reuid=couchdb --regid=couchdb --clear-groups "$@"
fi
fi
exec "$@"

4
hosting/couchdb/runner.sh
View File

@ -1,6 +1,7 @@
#!/bin/bash
DATA_DIR=${DATA_DIR:-/data}
COUCHDB_ERLANG_COOKIE=${COUCHDB_ERLANG_COOKIE:-B9CFC32C-3458-4A86-8448-B3C753991CA7}
mkdir -p ${DATA_DIR}
mkdir -p ${DATA_DIR}/couch/{dbs,views}
@ -60,6 +61,9 @@ else
sed -i "s#DATA_DIR#/data#g" /opt/couchdb/etc/local.ini
fi
sed -i "s#COUCHDB_ERLANG_COOKIE#${COUCHDB_ERLANG_COOKIE}#g" /opt/couchdb/etc/vm.args
sed -i "s#COUCHDB_ERLANG_COOKIE#${COUCHDB_ERLANG_COOKIE}#g" /opt/clouseau/clouseau.ini
# Start Clouseau. Budibase won't function correctly without Clouseau running, it
# powers the search API endpoints which are used to do all sorts, including
# populating app grids.

10
hosting/single/Dockerfile
View File

@ -3,7 +3,6 @@ FROM node:20-slim as build
# install node-gyp dependencies
RUN apt-get update && apt-get install -y --no-install-recommends g++ make python3 jq
# copy and install dependencies
WORKDIR /app
COPY package.json .
@ -39,10 +38,9 @@ COPY packages/worker/pm2.config.js packages/worker/pm2.config.js
COPY packages/string-templates packages/string-templates
FROM budibase/couchdb as runner
FROM budibase/couchdb:v3.3.3 as runner
ARG TARGETARCH
ENV TARGETARCH $TARGETARCH
ENV NODE_MAJOR 20
#TARGETBUILD can be set to single (for single docker image) or aas (for azure app service)
# e.g. docker build --build-arg TARGETBUILD=aas ....
ARG TARGETBUILD=single
@ -60,10 +58,8 @@ RUN apt install -y software-properties-common apt-transport-https ca-certificate
&& apt install postgresql-client-15 -y \
&& apt remove software-properties-common apt-transport-https gpg -y
# install other dependencies, nodejs, oracle requirements, jdk8, redis, nginx
WORKDIR /nodejs
COPY scripts/install-node.sh ./install.sh
RUN chmod +x install.sh && ./install.sh
# We use pm2 in order to run multiple node processes in a single container
RUN npm install --global pm2
# setup nginx
COPY hosting/single/nginx/nginx.conf /etc/nginx

6
hosting/single/runner.sh
View File

@ -97,10 +97,12 @@ fi
sleep 10
pushd app
pm2 start -l /dev/stdout --name app "yarn run:docker"
pm2 start --name app "yarn run:docker"
popd
pushd worker
pm2 start -l /dev/stdout --name worker "yarn run:docker"
pm2 start --name worker "yarn run:docker"
popd
echo "end of runner.sh, sleeping ..."
tail -f $HOME/.pm2/logs/*.log
sleep infinity

2
lerna.json
View File

@ -1,5 +1,5 @@
{
"version": "2.19.6",
"version": "2.20.5",
"npmClient": "yarn",
"packages": [
"packages/*",

2
packages/account-portal

@ -1 +1 @@
Subproject commit 8c446c4ba385592127fa31755d3b64467b291882
Subproject commit 4384bc742ca22fb1e9bf91843e65ae929daf17e2

1
packages/bbui/src/Layout/Page.svelte
View File

@ -43,6 +43,7 @@
flex-direction: row;
justify-content: flex-start;
align-items: stretch;
overflow-y: scroll !important;
flex: 1 1 auto;
overflow-x: hidden;
}

6
packages/bbui/src/Typography/Body.svelte
View File

@ -20,3 +20,9 @@
>
<slot />
</p>
<style>
p {
text-wrap: pretty;
}
</style>

4
packages/bbui/src/Typography/Heading.svelte
View File

@ -21,4 +21,8 @@
h1 {
font-family: var(--font-accent);
}
h1 {
text-wrap: balance;
}
</style>

1
packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowChart.svelte
View File

@ -130,6 +130,7 @@
flex-grow: 1;
padding: 23px 23px 80px;
box-sizing: border-box;
overflow-x: hidden;
}
.header.scrolling {

2
packages/builder/src/pages/builder/app/[application]/data/new.svelte
View File

@ -77,7 +77,7 @@
</DatasourceOption>
<DatasourceOption
on:click={() => internalTableModal.show({ promptUpload: true })}
title="Upload data"
title="Upload CSV / JSON"
description="Non-relational"
{disabled}
>

4
packages/builder/src/pages/builder/portal/_components/UpgradeButton.svelte
View File

@ -10,7 +10,7 @@
{#if $admin.cloud && $auth?.user?.accountPortalAccess}
<Button
cta
size="S"
size="M"
on:click
on:click={() => {
window.open($admin.accountPortalUrl + "/portal/upgrade", "_blank")
@ -21,7 +21,7 @@
{:else if !$admin.cloud && sdk.users.isAdmin($auth.user)}
<Button
cta
size="S"
size="M"
on:click={() => $goto("/builder/portal/account/upgrade")}
on:click
>

9
packages/builder/src/pages/builder/portal/settings/diagnostics.svelte
View File

@ -49,10 +49,13 @@
{#if sdk.users.isAdmin($auth.user) && diagnosticInfo}
<Layout noPadding>
<Layout gap="XS">
<Layout gap="XS" noPadding>
<Heading size="M">Diagnostics</Heading>
Please include this diagnostic information in support requests and github issues
by clicking the button on the top right to copy to clipboard.
<Body>
Please include this diagnostic information in support requests and
github issues by clicking the button on the top right to copy to
clipboard.
</Body>
<Divider />
<Body size="M">
<section>

26
packages/client/manifest.json
View File

@ -89,13 +89,13 @@
{
"label": "Column",
"value": "column",
"barIcon": "ViewColumn",
"barIcon": "TableSelectColumn",
"barTitle": "Column layout"
},
{
"label": "Row",
"value": "row",
"barIcon": "ViewRow",
"barIcon": "TableSelectRow",
"barTitle": "Row layout"
}
],
@ -298,13 +298,13 @@
{
"label": "Column",
"value": "column",
"barIcon": "ViewColumn",
"barIcon": "TableSelectColumn",
"barTitle": "Column layout"
},
{
"label": "Row",
"value": "row",
"barIcon": "ViewRow",
"barIcon": "TableSelectRow",
"barTitle": "Row layout"
}
],
@ -460,6 +460,10 @@
"label": "Variant",
"key": "type",
"options": [
{
"label": "Action",
"value": "cta"
},
{
"label": "Primary",
"value": "primary"
@ -468,10 +472,6 @@
"label": "Secondary",
"value": "secondary"
},
{
"label": "Action",
"value": "cta"
},
{
"label": "Warning",
"value": "warning"
@ -481,7 +481,7 @@
"value": "overBackground"
}
],
"defaultValue": "primary"
"defaultValue": "cta"
},
{
"type": "select",
@ -602,13 +602,13 @@
{
"label": "Column",
"value": "column",
"barIcon": "ViewColumn",
"barIcon": "TableSelectColumn",
"barTitle": "Column layout"
},
{
"label": "Row",
"value": "row",
"barIcon": "ViewRow",
"barIcon": "TableSelectRow",
"barTitle": "Row layout"
}
],
@ -5917,13 +5917,13 @@
{
"label": "Column",
"value": "column",
"barIcon": "ViewRow",
"barIcon": "TableSelectColumn",
"barTitle": "Column layout"
},
{
"label": "Row",
"value": "row",
"barIcon": "ViewColumn",
"barIcon": "TableSelectRow",
"barTitle": "Row layout"
}
],

2
packages/client/src/components/app/Button.svelte
View File

@ -11,7 +11,7 @@
export let text = ""
export let onClick
export let size = "M"
export let type = "primary"
export let type = "cta"
export let quiet = false
// For internal use only for now - not defined in the manifest

2
packages/pro

@ -1 +1 @@
Subproject commit 336bf2184cf632fdc2bffbad5628e8b15dd381bd
Subproject commit 60e47a8249fd6291a6bc20fe3fe6776b11938fa1

1
packages/server/package.json
View File

@ -114,7 +114,6 @@
"undici-types": "^6.0.1",
"uuid": "^8.3.2",
"validate.js": "0.13.1",
"vm2": "^3.9.19",
"worker-farm": "1.7.0",
"xml2js": "0.5.0"
},

7
packages/server/src/api/controllers/script.ts
View File

@ -3,9 +3,10 @@ import { IsolatedVM } from "../../jsRunner/vm"
export async function execute(ctx: Ctx) {
const { script, context } = ctx.request.body
const runner = new IsolatedVM().withContext(context)
const result = runner.execute(`(function(){\n${script}\n})();`)
const vm = new IsolatedVM()
const result = vm.withContext(context, () =>
vm.execute(`(function(){\n${script}\n})();`)
)
ctx.body = result
}

7
packages/server/src/api/routes/tests/backup.spec.ts
View File

@ -8,7 +8,6 @@ import { mocks } from "@budibase/backend-core/tests"
mocks.licenses.useBackups()
describe("/backups", () => {
let request = setup.getRequest()
let config = setup.getConfig()
afterAll(setup.afterAll)
@ -59,10 +58,8 @@ describe("/backups", () => {
await config.createScreen()
const exportRes = await config.api.backup.createBackup(appId)
expect(exportRes.backupId).toBeDefined()
const importRes = await config.api.backup.importBackup(
appId,
exportRes.backupId
)
await config.api.backup.waitForBackupToComplete(appId, exportRes.backupId)
await config.api.backup.importBackup(appId, exportRes.backupId)
})
})

43
packages/server/src/api/routes/tests/row.spec.ts
View File

@ -2135,5 +2135,48 @@ describe.each([
}
)
})
it("should not carry over context between formulas", async () => {
const js = Buffer.from(`return $("[text]");`).toString("base64")
const table = await config.createTable({
name: "table",
type: "table",
schema: {
text: {
name: "text",
type: FieldType.STRING,
},
formula: {
name: "formula",
type: FieldType.FORMULA,
formula: `{{ js "${js}"}}`,
formulaType: FormulaType.DYNAMIC,
},
},
})
for (let i = 0; i < 10; i++) {
await config.api.row.save(table._id!, { text: `foo${i}` })
}
const { rows } = await config.api.row.search(table._id!)
expect(rows).toHaveLength(10)
const formulaValues = rows.map(r => r.formula)
expect(formulaValues).toEqual(
expect.arrayContaining([
"foo0",
"foo1",
"foo2",
"foo3",
"foo4",
"foo5",
"foo6",
"foo7",
"foo8",
"foo9",
])
)
})
})
})

10
packages/server/src/api/routes/tests/table.spec.ts
View File

@ -368,10 +368,12 @@ describe("/tables", () => {
.set(config.defaultHeaders())
.expect("Content-Type", /json/)
.expect(200)
const fetchedTable = res.body[0]
expect(fetchedTable.name).toEqual(testTable.name)
expect(fetchedTable.type).toEqual("table")
expect(fetchedTable.sourceType).toEqual("internal")
const table = res.body.find((t: Table) => t._id === testTable._id)
expect(table).toBeDefined()
expect(table.name).toEqual(testTable.name)
expect(table.type).toEqual("table")
expect(table.sourceType).toEqual("internal")
})
it("should apply authorization to endpoint", async () => {

1
packages/server/src/app.ts
View File

@ -26,6 +26,7 @@ async function start() {
start().catch(err => {
console.error(`Failed server startup - ${err.message}`)
throw err
})
export function getServer() {

2
packages/server/src/environment.ts
View File

@ -23,7 +23,7 @@ const DEFAULTS = {
AUTOMATION_THREAD_TIMEOUT: 12000,
AUTOMATION_SYNC_TIMEOUT: 120000,
AUTOMATION_MAX_ITERATIONS: 200,
JS_PER_EXECUTION_TIME_LIMIT_MS: 1000,
JS_PER_EXECUTION_TIME_LIMIT_MS: 1500,
TEMPLATE_REPOSITORY: "app",
PLUGINS_DIR: "/plugins",
FORKED_PROCESS_NAME: "main",

35
packages/server/src/jsRunner/index.ts
View File

@ -8,37 +8,28 @@ import {
import { context, logging } from "@budibase/backend-core"
import tracer from "dd-trace"
import { BuiltInVM, IsolatedVM } from "./vm"
const USE_ISOLATED_VM = true
import { IsolatedVM } from "./vm"
export function init() {
setJSRunner((js: string, ctx: Record<string, any>) => {
return tracer.trace("runJS", {}, span => {
if (!USE_ISOLATED_VM) {
const vm = new BuiltInVM(ctx, span)
return vm.execute(js)
}
try {
const bbCtx = context.getCurrentContext()!
const bbCtx = context.getCurrentContext()
let { vm } = bbCtx
if (!vm) {
// Can't copy the native helpers into the isolate. We just ignore them as they are handled properly from the helpersSource
const { helpers, ...ctxToPass } = ctx
vm = new IsolatedVM({
memoryLimit: env.JS_RUNNER_MEMORY_LIMIT,
invocationTimeout: env.JS_PER_INVOCATION_TIMEOUT_MS,
isolateAccumulatedTimeout: env.JS_PER_REQUEST_TIMEOUT_MS,
})
.withContext(ctxToPass)
.withHelpers()
const vm = bbCtx?.vm
? bbCtx.vm
: new IsolatedVM({
memoryLimit: env.JS_RUNNER_MEMORY_LIMIT,
invocationTimeout: env.JS_PER_INVOCATION_TIMEOUT_MS,
isolateAccumulatedTimeout: env.JS_PER_REQUEST_TIMEOUT_MS,
}).withHelpers()
if (bbCtx) {
// If we have a context, we want to persist it to reuse the isolate
bbCtx.vm = vm
}
return vm.execute(js)
const { helpers, ...rest } = ctx
return vm.withContext(rest, () => vm.execute(js))
} catch (error: any) {
if (error.message === "Script execution timed out.") {
throw new JsErrorTimeout()

65
packages/server/src/jsRunner/vm/builtin-vm.ts
View File

@ -1,65 +0,0 @@
import vm from "vm"
import env from "../../environment"
import { context, timers } from "@budibase/backend-core"
import tracer, { Span } from "dd-trace"
import { VM } from "@budibase/types"
type TrackerFn = <T>(f: () => T) => T
export class BuiltInVM implements VM {
private ctx: vm.Context
private span?: Span
constructor(ctx: vm.Context, span?: Span) {
this.ctx = ctx
this.span = span
}
execute(code: string) {
const perRequestLimit = env.JS_PER_REQUEST_TIMEOUT_MS
let track: TrackerFn = f => f()
if (perRequestLimit) {
const bbCtx = tracer.trace("runJS.getCurrentContext", {}, span =>
context.getCurrentContext()
)
if (bbCtx) {
if (!bbCtx.jsExecutionTracker) {
this.span?.addTags({
createdExecutionTracker: true,
})
bbCtx.jsExecutionTracker = tracer.trace(
"runJS.createExecutionTimeTracker",
{},
span => timers.ExecutionTimeTracker.withLimit(perRequestLimit)
)
}
this.span?.addTags({
js: {
limitMS: bbCtx.jsExecutionTracker.limitMs,
elapsedMS: bbCtx.jsExecutionTracker.elapsedMS,
},
})
// We call checkLimit() here to prevent paying the cost of creating
// a new VM context below when we don't need to.
tracer.trace("runJS.checkLimitAndBind", {}, span => {
bbCtx.jsExecutionTracker!.checkLimit()
track = bbCtx.jsExecutionTracker!.track.bind(bbCtx.jsExecutionTracker)
})
}
}
this.ctx = {
...this.ctx,
alert: undefined,
setInterval: undefined,
setTimeout: undefined,
}
vm.createContext(this.ctx)
return track(() =>
vm.runInNewContext(code, this.ctx, {
timeout: env.JS_PER_INVOCATION_TIMEOUT_MS,
})
)
}
}

2
packages/server/src/jsRunner/vm/index.ts
View File

@ -1,3 +1 @@
export * from "./isolated-vm"
export * from "./builtin-vm"
export * from "./vm2"

24
packages/server/src/jsRunner/vm/isolated-vm.ts
View File

@ -38,10 +38,9 @@ export class IsolatedVM implements VM {
invocationTimeout?: number
isolateAccumulatedTimeout?: number
} = {}) {
memoryLimit = memoryLimit || environment.JS_RUNNER_MEMORY_LIMIT
invocationTimeout = memoryLimit || 1000
this.isolate = new ivm.Isolate({ memoryLimit })
this.isolate = new ivm.Isolate({
memoryLimit: memoryLimit || environment.JS_RUNNER_MEMORY_LIMIT,
})
this.vm = this.isolate.createContextSync()
this.jail = this.vm.global
this.jail.setSync("global", this.jail.derefInto())
@ -51,7 +50,8 @@ export class IsolatedVM implements VM {
[this.resultKey]: { [this.runResultKey]: "" },
})
this.invocationTimeout = invocationTimeout
this.invocationTimeout =
invocationTimeout || environment.JS_PER_INVOCATION_TIMEOUT_MS
this.isolateAccumulatedTimeout = isolateAccumulatedTimeout
}
@ -97,10 +97,14 @@ export class IsolatedVM implements VM {
return this
}
withContext(context: Record<string, any>) {
withContext<T>(context: Record<string, any>, executeWithContext: () => T) {
this.addToContext(context)
return this
try {
return executeWithContext()
} finally {
this.removeFromContext(Object.keys(context))
}
}
withParsingBson(data: any) {
@ -224,6 +228,12 @@ export class IsolatedVM implements VM {
}
}
private removeFromContext(keys: string[]) {
for (let key of keys) {
this.jail.deleteSync(key)
}
}
private getFromContext(key: string) {
const ref = this.vm.global.getSync(key, { reference: true })
const result = ref.copySync()

26
packages/server/src/jsRunner/vm/vm2.ts
View File

@ -1,26 +0,0 @@
import vm2 from "vm2"
import { VM } from "@budibase/types"
const JS_TIMEOUT_MS = 1000
export class VM2 implements VM {
vm: vm2.VM
results: { out: string }
constructor(context: any) {
this.vm = new vm2.VM({
timeout: JS_TIMEOUT_MS,
})
this.results = { out: "" }
this.vm.setGlobals(context)
this.vm.setGlobal("fetch", fetch)
this.vm.setGlobal("results", this.results)
}
execute(script: string) {
const code = `let fn = () => {\n${script}\n}; results.out = fn();`
const vmScript = new vm2.VMScript(code)
this.vm.run(vmScript)
return this.results.out
}
}

5
packages/server/src/sdk/users/tests/utils.spec.ts
View File

@ -84,7 +84,8 @@ describe("syncGlobalUsers", () => {
await syncGlobalUsers()
const metadata = await rawUserMetadata()
expect(metadata).toHaveLength(2)
expect(metadata).toHaveLength(2 + 1) // ADMIN user created in test bootstrap still in the application
expect(metadata).toContainEqual(
expect.objectContaining({
_id: db.generateUserMetadataID(user1._id!),
@ -121,7 +122,7 @@ describe("syncGlobalUsers", () => {
await syncGlobalUsers()
const metadata = await rawUserMetadata()
expect(metadata).toHaveLength(1) //ADMIN user created in test bootstrap still in the application
expect(metadata).toHaveLength(1) // ADMIN user created in test bootstrap still in the application
})
})
})

103
packages/server/src/tests/utilities/TestConfiguration.ts
View File

@ -76,14 +76,6 @@ mocks.licenses.useUnlimited()
dbInit()
type DefaultUserValues = {
globalUserId: string
email: string
firstName: string
lastName: string
csrfToken: string
}
export interface TableToBuild extends Omit<Table, "sourceId" | "sourceType"> {
sourceId?: string
sourceType?: TableSourceType
@ -99,14 +91,17 @@ export default class TestConfiguration {
prodApp: any
prodAppId: any
user: any
globalUserId: any
userMetadataId: any
table?: Table
automation: any
datasource?: Datasource
tenantId?: string
defaultUserValues: DefaultUserValues
api: API
csrfToken?: string
private get globalUserId() {
return this.user._id
}
constructor(openServer = true) {
if (openServer) {
@ -121,21 +116,10 @@ export default class TestConfiguration {
}
this.appId = null
this.allApps = []
this.defaultUserValues = this.populateDefaultUserValues()
this.api = new API(this)
}
populateDefaultUserValues(): DefaultUserValues {
return {
globalUserId: `us_${newid()}`,
email: generator.email(),
firstName: generator.first(),
lastName: generator.last(),
csrfToken: generator.hash(),
}
}
getRequest() {
return this.request
}
@ -162,10 +146,10 @@ export default class TestConfiguration {
getUserDetails() {
return {
globalId: this.defaultUserValues.globalUserId,
email: this.defaultUserValues.email,
firstName: this.defaultUserValues.firstName,
lastName: this.defaultUserValues.lastName,
globalId: this.globalUserId,
email: this.user.email,
firstName: this.user.firstName,
lastName: this.user.lastName,
}
}
@ -300,15 +284,27 @@ export default class TestConfiguration {
}
// USER / AUTH
async globalUser({
id = this.defaultUserValues.globalUserId,
firstName = this.defaultUserValues.firstName,
lastName = this.defaultUserValues.lastName,
builder = true,
admin = false,
email = this.defaultUserValues.email,
roles,
}: any = {}): Promise<User> {
async globalUser(
config: {
id?: string
firstName?: string
lastName?: string
builder?: boolean
admin?: boolean
email?: string
roles?: any
} = {}
): Promise<User> {
const {
id = `us_${newid()}`,
firstName = generator.first(),
lastName = generator.last(),
builder = true,
admin = false,
email = generator.email(),
roles,
} = config
const db = tenancy.getTenantDB(this.getTenantId())
let existing
try {
@ -327,7 +323,7 @@ export default class TestConfiguration {
await sessions.createASession(id, {
sessionId: "sessionid",
tenantId: this.getTenantId(),
csrfToken: this.defaultUserValues.csrfToken,
csrfToken: this.csrfToken,
})
if (builder) {
user.builder = { global: true }
@ -357,14 +353,16 @@ export default class TestConfiguration {
roles?: UserRoles
} = {}
): Promise<User> {
let { id, firstName, lastName, email, builder, admin, roles } = user
firstName = firstName || this.defaultUserValues.firstName
lastName = lastName || this.defaultUserValues.lastName
email = email || this.defaultUserValues.email
roles = roles || {}
if (builder == null) {
builder = true
}
const {
id,
firstName = generator.first(),
lastName = generator.last(),
email = generator.email(),
builder = true,
admin,
roles,
} = user
const globalId = !id ? `us_${Math.random()}` : `us_${id}`
const resp = await this.globalUser({
id: globalId,
@ -373,7 +371,7 @@ export default class TestConfiguration {
email,
builder,
admin,
roles,
roles: roles || {},
})
await cache.user.invalidateUser(globalId)
return resp
@ -448,7 +446,7 @@ export default class TestConfiguration {
defaultHeaders(extras = {}, prodApp = false) {
const tenantId = this.getTenantId()
const authObj: AuthToken = {
userId: this.defaultUserValues.globalUserId,
userId: this.globalUserId,
sessionId: "sessionid",
tenantId,
}
@ -457,7 +455,7 @@ export default class TestConfiguration {
const headers: any = {
Accept: "application/json",
Cookie: [`${constants.Cookie.Auth}=${authToken}`],
[constants.Header.CSRF_TOKEN]: this.defaultUserValues.csrfToken,
[constants.Header.CSRF_TOKEN]: this.csrfToken,
Host: this.tenantHost(),
...extras,
}
@ -487,7 +485,7 @@ export default class TestConfiguration {
async basicRoleHeaders() {
return await this.roleHeaders({
email: this.defaultUserValues.email,
email: generator.email(),
builder: false,
prodApp: true,
roleId: roles.BUILTIN_ROLE_IDS.BASIC,
@ -495,7 +493,7 @@ export default class TestConfiguration {
}
async roleHeaders({
email = this.defaultUserValues.email,
email = generator.email(),
roleId = roles.BUILTIN_ROLE_IDS.ADMIN,
builder = false,
prodApp = true,
@ -519,11 +517,12 @@ export default class TestConfiguration {
}
async newTenant(appName = newid()): Promise<App> {
this.defaultUserValues = this.populateDefaultUserValues()
this.csrfToken = generator.hash()
this.tenantId = structures.tenant.id()
this.user = await this.globalUser()
this.globalUserId = this.user._id
this.userMetadataId = generateUserMetadataID(this.globalUserId)
this.userMetadataId = generateUserMetadataID(this.user._id)
return this.createApp(appName)
}
@ -533,7 +532,7 @@ export default class TestConfiguration {
// API
async generateApiKey(userId = this.defaultUserValues.globalUserId) {
async generateApiKey(userId = this.user._id) {
const db = tenancy.getTenantDB(this.getTenantId())
const id = dbCore.generateDevInfoID(userId)
let devInfo: any

13
packages/server/src/tests/utilities/api/backup.ts
View File

@ -31,6 +31,19 @@ export class BackupAPI extends TestAPI {
return result.body as CreateAppBackupResponse
}
waitForBackupToComplete = async (appId: string, backupId: string) => {
for (let i = 0; i < 10; i++) {
await new Promise(resolve => setTimeout(resolve, 1000))
const result = await this.request
.get(`/api/apps/${appId}/backups/${backupId}/file`)
.set(this.config.defaultHeaders())
if (result.status === 200) {
return
}
}
throw new Error("Backup did not complete")
}
importBackup = async (
appId: string,
backupId: string

33
packages/server/src/threads/query.ts
View File

@ -7,20 +7,18 @@ import {
QueryVariable,
QueryResponse,
} from "./definitions"
import { IsolatedVM, VM2 } from "../jsRunner/vm"
import { IsolatedVM } from "../jsRunner/vm"
import { getIntegration } from "../integrations"
import { processStringSync } from "@budibase/string-templates"
import { context, cache, auth } from "@budibase/backend-core"
import { getGlobalIDFromUserMetadataID } from "../db/utils"
import sdk from "../sdk"
import { cloneDeep } from "lodash/fp"
import { Datasource, Query, SourceName, VM } from "@budibase/types"
import { Datasource, Query, SourceName } from "@budibase/types"
import { isSQL } from "../integrations/utils"
import { interpolateSQL } from "../integrations/queries/sql"
const USE_ISOLATED_VM = true
class QueryRunner {
datasource: Datasource
queryVerb: string
@ -129,26 +127,17 @@ class QueryRunner {
// transform as required
if (transformer) {
let runner: VM
if (!USE_ISOLATED_VM) {
runner = new VM2({
data: rows,
params: enrichedParameters,
})
} else {
transformer = `(function(){\n${transformer}\n})();`
let isolatedVm = new IsolatedVM().withContext({
data: rows,
params: enrichedParameters,
})
if (datasource.source === SourceName.MONGODB) {
isolatedVm = isolatedVm.withParsingBson(rows)
}
runner = isolatedVm
transformer = `(function(){\n${transformer}\n})();`
let vm = new IsolatedVM()
if (datasource.source === SourceName.MONGODB) {
vm = vm.withParsingBson(rows)
}
rows = runner.execute(transformer)
const ctx = {
data: rows,
params: enrichedParameters,
}
rows = vm.withContext(ctx, () => vm.execute(transformer))
}
// if the request fails we retry once, invalidating the cached value

7
packages/string-templates/src/helpers/javascript.js
View File

@ -1,4 +1,4 @@
const { atob } = require("../utilities")
const { atob, isBackendService, isJSAllowed } = require("../utilities")
const cloneDeep = require("lodash.clonedeep")
const { LITERAL_MARKER } = require("../helpers/constants")
const { getJsHelperList } = require("./list")
@ -7,6 +7,9 @@ const { getJsHelperList } = require("./list")
// This setter is used in the entrypoint (either index.js or index.mjs).
let runJS
module.exports.setJSRunner = runner => (runJS = runner)
module.exports.removeJSRunner = () => {
runJS = undefined
}
let onErrorLog
module.exports.setOnErrorLog = delegate => (onErrorLog = delegate)
@ -39,7 +42,7 @@ const getContextValue = (path, context) => {
// Evaluates JS code against a certain context
module.exports.processJS = (handlebars, context) => {
if (process && process.env.NO_JS) {
if (!isJSAllowed() || (isBackendService() && !runJS)) {
throw new Error("JS disabled in environment.")
}
try {

37
packages/string-templates/src/index.js
View File

@ -2,7 +2,7 @@ const vm = require("vm")
const handlebars = require("handlebars")
const { registerAll, registerMinimum } = require("./helpers/index")
const processors = require("./processors")
const { atob, btoa } = require("./utilities")
const { atob, btoa, isBackendService } = require("./utilities")
const manifest = require("../manifest.json")
const {
FIND_HBS_REGEX,
@ -404,18 +404,25 @@ module.exports.JsErrorTimeout = errors.JsErrorTimeout
module.exports.helpersToRemoveForJs = helpersToRemoveForJs
if (process && !process.env.NO_JS) {
/**
* Use polyfilled vm to run JS scripts in a browser Env
*/
javascript.setJSRunner((js, context) => {
context = {
...context,
alert: undefined,
setInterval: undefined,
setTimeout: undefined,
}
vm.createContext(context)
return vm.runInNewContext(js, context, { timeout: 1000 })
})
function defaultJSSetup() {
if (!isBackendService()) {
/**
* Use polyfilled vm to run JS scripts in a browser Env
*/
javascript.setJSRunner((js, context) => {
context = {
...context,
alert: undefined,
setInterval: undefined,
setTimeout: undefined,
}
vm.createContext(context)
return vm.runInNewContext(js, context, { timeout: 1000 })
})
} else {
javascript.removeJSRunner()
}
}
defaultJSSetup()
module.exports.defaultJSSetup = defaultJSSetup

8
packages/string-templates/src/utilities.js
View File

@ -4,6 +4,14 @@ module.exports.FIND_HBS_REGEX = /{{([^{].*?)}}/g
module.exports.FIND_ANY_HBS_REGEX = /{?{{([^{].*?)}}}?/g
module.exports.FIND_TRIPLE_HBS_REGEX = /{{{([^{].*?)}}}/g
module.exports.isBackendService = () => {
return typeof window === "undefined"
}
module.exports.isJSAllowed = () => {
return process && !process.env.NO_JS
}
// originally this could be done with a single regex using look behinds
// but safari does not support this feature
// original regex: /(?<!{){{[^{}]+}}(?!})/g

27
packages/string-templates/test/vm.spec.js Normal file
View File

@ -0,0 +1,27 @@
jest.mock("../src/utilities", () => {
const utilities = jest.requireActual("../src/utilities")
return {
...utilities,
isBackendService: jest.fn().mockReturnValue(true),
}
})
const { defaultJSSetup, processStringSync, encodeJSBinding } = require("../src")
const { isBackendService } = require("../src/utilities")
const mockedBackendService = jest.mocked(isBackendService)
const binding = encodeJSBinding("return 1")
describe("confirm VM is available when expected and when not", () => {
it("shouldn't have JS available in a backend service by default", () => {
defaultJSSetup()
const result = processStringSync(binding, {})
// shouldn't process at all
expect(result).toBe(binding)
})
it("should have JS available in frontend environments", () => {
mockedBackendService.mockReturnValue(false)
defaultJSSetup()
const result = processStringSync(binding, {})
expect(result).toBe(1)
})
})

1
packages/types/src/sdk/vm.ts
View File

@ -1,3 +1,4 @@
export interface VM {
execute(code: string): any
withContext<T>(context: Record<string, any>, executeWithContext: () => T): T
}

8
scripts/install-node.sh
View File

@ -1,8 +0,0 @@
#!/bin/bash
apt-get install -y gnupg
curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor | tee /usr/share/keyrings/nodesource.gpg > /dev/null
echo "deb [signed-by=/usr/share/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$NODE_MAJOR.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list
apt-get update
echo "INSTALLING NODE $NODE_MAJOR"
apt-get install -y --no-install-recommends nodejs
npm install --global yarn pm2

10
yarn.lock
View File

@ -6463,7 +6463,7 @@ acorn@^7.1.1:
resolved "https://registry.yarnpkg.com/acorn/-/acorn-7.4.1.tgz#feaed255973d2e77555b83dbc08851a6c63520fa"
integrity sha512-nQyp0o1/mNdbTO1PO6kHkwSrmgZ0MT/jCCpNiwbUjGoRN4dlBhqJtoQuCnEOKzgTVwg0ZWiCoQy6SxMebQVh8A==
acorn@^8.1.0, acorn@^8.10.0, acorn@^8.2.4, acorn@^8.4.1, acorn@^8.5.0, acorn@^8.7.0, acorn@^8.7.1, acorn@^8.8.1, acorn@^8.8.2, acorn@^8.9.0:
acorn@^8.1.0, acorn@^8.10.0, acorn@^8.2.4, acorn@^8.4.1, acorn@^8.5.0, acorn@^8.7.1, acorn@^8.8.1, acorn@^8.8.2, acorn@^8.9.0:
version "8.11.3"
resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.11.3.tgz#71e0b14e13a4ec160724b38fb7b0f233b1b81d7a"
integrity sha512-Y9rRfJG5jcKOE0CLisYbojUjIrIEE7AGMzA/Sm4BslANhbS+cDMpgBdcPT91oJ7OuJ9hYJBx59RjbhxVnrF8Xg==
@ -21633,14 +21633,6 @@ vlq@^0.2.2:
resolved "https://registry.yarnpkg.com/vlq/-/vlq-0.2.3.tgz#8f3e4328cf63b1540c0d67e1b2778386f8975b26"
integrity sha512-DRibZL6DsNhIgYQ+wNdWDL2SL3bKPlVrRiBqV5yuMm++op8W4kGFtaQfCs4KEJn0wBZcHVHJ3eoywX8983k1ow==
vm2@^3.9.19:
version "3.9.19"
resolved "https://registry.yarnpkg.com/vm2/-/vm2-3.9.19.tgz#be1e1d7a106122c6c492b4d51c2e8b93d3ed6a4a"
integrity sha512-J637XF0DHDMV57R6JyVsTak7nIL8gy5KH4r1HiwWLf/4GBbb5MKL5y7LpmF4A8E2nR6XmzpmMFQ7V7ppPTmUQg==
dependencies:
acorn "^8.7.0"
acorn-walk "^8.2.0"
vuvuzela@1.0.3:
version "1.0.3"
resolved "https://registry.yarnpkg.com/vuvuzela/-/vuvuzela-1.0.3.tgz#3be145e58271c73ca55279dd851f12a682114b0b"