From df98d6bb5d6d0edf31090550426bd435739709fd Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Tue, 2 Jul 2024 12:14:01 +0100
Subject: [PATCH 1/9] Updating CouchDB directory to be SQS by default, this
image has both Lucene and SQS capabilities so it is backwards compatiable.
---
hosting/couchdb/Dockerfile | 12 ++-
hosting/couchdb/Dockerfile.v2 | 139 ----------------------------------
hosting/couchdb/runner.sh | 9 ++-
hosting/couchdb/runner.v2.sh | 88 ---------------------
package.json | 1 -
5 files changed, 16 insertions(+), 233 deletions(-)
delete mode 100644 hosting/couchdb/Dockerfile.v2
delete mode 100644 hosting/couchdb/runner.v2.sh
diff --git a/hosting/couchdb/Dockerfile b/hosting/couchdb/Dockerfile
index ca72153e78..b95fa348f8 100644
--- a/hosting/couchdb/Dockerfile
+++ b/hosting/couchdb/Dockerfile
@@ -96,10 +96,13 @@ EXPOSE 5984 4369 9100
CMD ["/opt/couchdb/bin/couchdb"]
FROM base as runner
+ARG TARGETARCH
+ENV TARGETARCH $TARGETARCH
ENV COUCHDB_USER admin
ENV COUCHDB_PASSWORD admin
EXPOSE 5984
+EXPOSE 4984
RUN apt-get update && apt-get install -y --no-install-recommends software-properties-common wget unzip curl && \
wget -O - https://packages.adoptium.net/artifactory/api/gpg/key/public | apt-key add - && \
@@ -125,7 +128,12 @@ ADD clouseau/log4j.properties clouseau/clouseau.ini ./
WORKDIR /opt/couchdb
ADD couch/vm.args couch/local.ini ./etc/
+# setup SQS
+WORKDIR /opt/sqs
+ADD sqs ./
+RUN chmod +x ./install.sh && ./install.sh
+
WORKDIR /
ADD runner.sh ./bbcouch-runner.sh
-RUN chmod +x ./bbcouch-runner.sh /opt/clouseau/bin/clouseau
-CMD ["./bbcouch-runner.sh"]
\ No newline at end of file
+RUN chmod +x ./bbcouch-runner.sh /opt/clouseau/bin/clouseau /opt/sqs/sqs
+CMD ["./bbcouch-runner.sh"]
diff --git a/hosting/couchdb/Dockerfile.v2 b/hosting/couchdb/Dockerfile.v2
deleted file mode 100644
index 126742cadb..0000000000
--- a/hosting/couchdb/Dockerfile.v2
+++ /dev/null
@@ -1,139 +0,0 @@
-# Modified from https://github.com/apache/couchdb-docker/blob/main/3.3.3/Dockerfile
-#
-# Everything in this `base` image is adapted from the official `couchdb` image's
-# Dockerfile. Only modifications related to upgrading from Debian bullseye to
-# bookworm have been included. The `runner` image contains Budibase's
-# customisations to the image, e.g. adding Clouseau.
-FROM node:20-slim AS base
-
-# Add CouchDB user account to make sure the IDs are assigned consistently
-RUN groupadd -g 5984 -r couchdb && useradd -u 5984 -d /opt/couchdb -g couchdb couchdb
-
-# be sure GPG and apt-transport-https are available and functional
-RUN set -ex; \
- apt-get update; \
- apt-get install -y --no-install-recommends \
- apt-transport-https \
- ca-certificates \
- dirmngr \
- gnupg \
- ; \
- rm -rf /var/lib/apt/lists/*
-
-# grab tini for signal handling and zombie reaping
-# see https://github.com/apache/couchdb-docker/pull/28#discussion_r141112407
-RUN set -eux; \
- apt-get update; \
- apt-get install -y --no-install-recommends tini; \
- rm -rf /var/lib/apt/lists/*; \
- tini --version
-
-# http://docs.couchdb.org/en/latest/install/unix.html#installing-the-apache-couchdb-packages
-ENV GPG_COUCH_KEY \
-# gpg: rsa8192 205-01-19 The Apache Software Foundation (Package repository signing key) <root@apache.org>
- 390EF70BB1EA12B2773962950EE62FB37A00258D
-RUN set -eux; \
- apt-get update; \
- apt-get install -y curl; \
- export GNUPGHOME="$(mktemp -d)"; \
- curl -fL -o keys.asc https://couchdb.apache.org/repo/keys.asc; \
- gpg --batch --import keys.asc; \
- gpg --batch --export "${GPG_COUCH_KEY}" > /usr/share/keyrings/couchdb-archive-keyring.gpg; \
- command -v gpgconf && gpgconf --kill all || :; \
- rm -rf "$GNUPGHOME"; \
- apt-key list; \
- apt purge -y --autoremove curl; \
- rm -rf /var/lib/apt/lists/*
-
-ENV COUCHDB_VERSION 3.3.3
-
-RUN . /etc/os-release; \
- echo "deb [signed-by=/usr/share/keyrings/couchdb-archive-keyring.gpg] https://apache.jfrog.io/artifactory/couchdb-deb/ ${VERSION_CODENAME} main" | \
- tee /etc/apt/sources.list.d/couchdb.list >/dev/null
-
-# https://github.com/apache/couchdb-pkg/blob/master/debian/README.Debian
-RUN set -eux; \
- apt-get update; \
- \
- echo "couchdb couchdb/mode select none" | debconf-set-selections; \
-# we DO want recommends this time
- DEBIAN_FRONTEND=noninteractive apt-get install -y --allow-downgrades --allow-remove-essential --allow-change-held-packages \
- couchdb="$COUCHDB_VERSION"~bookworm \
- ; \
-# Undo symlinks to /var/log and /var/lib
- rmdir /var/lib/couchdb /var/log/couchdb; \
- rm /opt/couchdb/data /opt/couchdb/var/log; \
- mkdir -p /opt/couchdb/data /opt/couchdb/var/log; \
- chown couchdb:couchdb /opt/couchdb/data /opt/couchdb/var/log; \
- chmod 777 /opt/couchdb/data /opt/couchdb/var/log; \
-# Remove file that sets logging to a file
- rm /opt/couchdb/etc/default.d/10-filelog.ini; \
-# Check we own everything in /opt/couchdb. Matches the command in dockerfile_entrypoint.sh
- find /opt/couchdb \! \( -user couchdb -group couchdb \) -exec chown -f couchdb:couchdb '{}' +; \
-# Setup directories and permissions for config. Technically these could be 555 and 444 respectively
-# but we keep them as 755 and 644 for consistency with CouchDB defaults and the dockerfile_entrypoint.sh.
- find /opt/couchdb/etc -type d ! -perm 0755 -exec chmod -f 0755 '{}' +; \
- find /opt/couchdb/etc -type f ! -perm 0644 -exec chmod -f 0644 '{}' +; \
-# only local.d needs to be writable for the docker_entrypoint.sh
- chmod -f 0777 /opt/couchdb/etc/local.d; \
-# apt clean-up
- rm -rf /var/lib/apt/lists/*;
-
-# Add configuration
-COPY --chown=couchdb:couchdb couch/10-docker-default.ini /opt/couchdb/etc/default.d/
-# COPY --chown=couchdb:couchdb vm.args /opt/couchdb/etc/
-
-COPY docker-entrypoint.sh /usr/local/bin
-RUN ln -s usr/local/bin/docker-entrypoint.sh /docker-entrypoint.sh # backwards compat
-ENTRYPOINT ["tini", "--", "/docker-entrypoint.sh"]
-
-VOLUME /opt/couchdb/data
-
-# 5984: Main CouchDB endpoint
-# 4369: Erlang portmap daemon (epmd)
-# 9100: CouchDB cluster communication port
-EXPOSE 5984 4369 9100
-CMD ["/opt/couchdb/bin/couchdb"]
-
-FROM base as runner
-ARG TARGETARCH
-ENV TARGETARCH $TARGETARCH
-
-ENV COUCHDB_USER admin
-ENV COUCHDB_PASSWORD admin
-EXPOSE 5984
-EXPOSE 4984
-
-RUN apt-get update && apt-get install -y --no-install-recommends software-properties-common wget unzip curl && \
- wget -O - https://packages.adoptium.net/artifactory/api/gpg/key/public | apt-key add - && \
- apt-add-repository 'deb http://security.debian.org/debian-security bookworm-security/updates main' && \
- apt-add-repository 'deb http://archive.debian.org/debian stretch-backports main' && \
- apt-add-repository 'deb https://packages.adoptium.net/artifactory/deb bookworm main' && \
- apt-get update && apt-get install -y --no-install-recommends temurin-8-jdk && \
- rm -rf /var/lib/apt/lists/
-
-# setup clouseau
-WORKDIR /
-RUN wget https://github.com/cloudant-labs/clouseau/releases/download/2.21.0/clouseau-2.21.0-dist.zip && \
- unzip clouseau-2.21.0-dist.zip && \
- mv clouseau-2.21.0 /opt/clouseau && \
- rm clouseau-2.21.0-dist.zip
-
-WORKDIR /opt/clouseau
-RUN mkdir ./bin
-ADD clouseau/clouseau ./bin/
-ADD clouseau/log4j.properties clouseau/clouseau.ini ./
-
-# setup CouchDB
-WORKDIR /opt/couchdb
-ADD couch/vm.args couch/local.ini ./etc/
-
-# setup SQS
-WORKDIR /opt/sqs
-ADD sqs ./
-RUN chmod +x ./install.sh && ./install.sh
-
-WORKDIR /
-ADD runner.v2.sh ./bbcouch-runner.sh
-RUN chmod +x ./bbcouch-runner.sh /opt/clouseau/bin/clouseau /opt/sqs/sqs
-CMD ["./bbcouch-runner.sh"]
diff --git a/hosting/couchdb/runner.sh b/hosting/couchdb/runner.sh
index aaadee6b43..f8cbe49b8f 100644
--- a/hosting/couchdb/runner.sh
+++ b/hosting/couchdb/runner.sh
@@ -70,9 +70,12 @@ sed -i "s#COUCHDB_ERLANG_COOKIE#${COUCHDB_ERLANG_COOKIE}#g" /opt/clouseau/clouse
/opt/clouseau/bin/clouseau > /dev/stdout 2>&1 &
# Start CouchDB.
-/docker-entrypoint.sh /opt/couchdb/bin/couchdb &
+/docker-entrypoint.sh /opt/couchdb/bin/couchdb > /dev/stdout 2>&1 &
-# Wati for CouchDB to start up.
+# Start SQS. Use 127.0.0.1 instead of localhost to avoid IPv6 issues.
+/opt/sqs/sqs --server "http://127.0.0.1:5984" --data-dir ${DATA_DIR}/sqs --bind-address=0.0.0.0 > /dev/stdout 2>&1 &
+
+# Wait for CouchDB to start up.
while [[ $(curl -s -w "%{http_code}\n" http://localhost:5984/_up -o /dev/null) -ne 200 ]]; do
echo 'Waiting for CouchDB to start...';
sleep 5;
@@ -82,4 +85,4 @@ done
# function correctly, so we create them here.
curl -X PUT -u "${COUCHDB_USER}:${COUCHDB_PASSWORD}" http://localhost:5984/_users
curl -X PUT -u "${COUCHDB_USER}:${COUCHDB_PASSWORD}" http://localhost:5984/_replicator
-sleep infinity
\ No newline at end of file
+sleep infinity
diff --git a/hosting/couchdb/runner.v2.sh b/hosting/couchdb/runner.v2.sh
deleted file mode 100644
index f8cbe49b8f..0000000000
--- a/hosting/couchdb/runner.v2.sh
+++ /dev/null
@@ -1,88 +0,0 @@
-#!/bin/bash
-
-DATA_DIR=${DATA_DIR:-/data}
-COUCHDB_ERLANG_COOKIE=${COUCHDB_ERLANG_COOKIE:-B9CFC32C-3458-4A86-8448-B3C753991CA7}
-
-mkdir -p ${DATA_DIR}
-mkdir -p ${DATA_DIR}/couch/{dbs,views}
-mkdir -p ${DATA_DIR}/search
-chown -R couchdb:couchdb ${DATA_DIR}/couch
-
-echo ${TARGETBUILD} > /buildtarget.txt
-if [[ "${TARGETBUILD}" = "aas" ]]; then
- # Azure AppService uses /home for persistent data & SSH on port 2222
- DATA_DIR="${DATA_DIR:-/home}"
- WEBSITES_ENABLE_APP_SERVICE_STORAGE=true
- mkdir -p $DATA_DIR/{search,minio,couch}
- mkdir -p $DATA_DIR/couch/{dbs,views}
- chown -R couchdb:couchdb $DATA_DIR/couch/
- apt update
- apt-get install -y openssh-server
- echo "root:Docker!" | chpasswd
- mkdir -p /tmp
- chmod +x /tmp/ssh_setup.sh \
- && (sleep 1;/tmp/ssh_setup.sh 2>&1 > /dev/null)
- cp /etc/sshd_config /etc/ssh/sshd_config
- /etc/init.d/ssh restart
- sed -i "s#DATA_DIR#/home#g" /opt/clouseau/clouseau.ini
- sed -i "s#DATA_DIR#/home#g" /opt/couchdb/etc/local.ini
-elif [[ "${TARGETBUILD}" = "single" ]]; then
- # In the single image build, the Dockerfile specifies /data as a volume
- # mount, so we use that for all persistent data.
- sed -i "s#DATA_DIR#/data#g" /opt/clouseau/clouseau.ini
- sed -i "s#DATA_DIR#/data#g" /opt/couchdb/etc/local.ini
-elif [[ "${TARGETBUILD}" = "docker-compose" ]]; then
- # We remove the database_dir and view_index_dir settings from the local.ini
- # in docker-compose because it will default to /opt/couchdb/data which is what
- # our docker-compose was using prior to us switching to using our own CouchDB
- # image.
- sed -i "s#^database_dir.*\$##g" /opt/couchdb/etc/local.ini
- sed -i "s#^view_index_dir.*\$##g" /opt/couchdb/etc/local.ini
- sed -i "s#^dir=.*\$#dir=/opt/couchdb/data#g" /opt/clouseau/clouseau.ini
-elif [[ -n $KUBERNETES_SERVICE_HOST ]]; then
- # In Kubernetes the directory /opt/couchdb/data has a persistent volume
- # mount for storing database data.
- sed -i "s#^dir=.*\$#dir=/opt/couchdb/data#g" /opt/clouseau/clouseau.ini
-
- # We remove the database_dir and view_index_dir settings from the local.ini
- # in Kubernetes because it will default to /opt/couchdb/data which is what
- # our Helm chart was using prior to us switching to using our own CouchDB
- # image.
- sed -i "s#^database_dir.*\$##g" /opt/couchdb/etc/local.ini
- sed -i "s#^view_index_dir.*\$##g" /opt/couchdb/etc/local.ini
-
- # We remove the -name setting from the vm.args file in Kubernetes because
- # it will default to the pod FQDN, which is what's required for clustering
- # to work.
- sed -i "s/^-name .*$//g" /opt/couchdb/etc/vm.args
-else
- # For all other builds, we use /data for persistent data.
- sed -i "s#DATA_DIR#/data#g" /opt/clouseau/clouseau.ini
- sed -i "s#DATA_DIR#/data#g" /opt/couchdb/etc/local.ini
-fi
-
-sed -i "s#COUCHDB_ERLANG_COOKIE#${COUCHDB_ERLANG_COOKIE}#g" /opt/couchdb/etc/vm.args
-sed -i "s#COUCHDB_ERLANG_COOKIE#${COUCHDB_ERLANG_COOKIE}#g" /opt/clouseau/clouseau.ini
-
-# Start Clouseau. Budibase won't function correctly without Clouseau running, it
-# powers the search API endpoints which are used to do all sorts, including
-# populating app grids.
-/opt/clouseau/bin/clouseau > /dev/stdout 2>&1 &
-
-# Start CouchDB.
-/docker-entrypoint.sh /opt/couchdb/bin/couchdb > /dev/stdout 2>&1 &
-
-# Start SQS. Use 127.0.0.1 instead of localhost to avoid IPv6 issues.
-/opt/sqs/sqs --server "http://127.0.0.1:5984" --data-dir ${DATA_DIR}/sqs --bind-address=0.0.0.0 > /dev/stdout 2>&1 &
-
-# Wait for CouchDB to start up.
-while [[ $(curl -s -w "%{http_code}\n" http://localhost:5984/_up -o /dev/null) -ne 200 ]]; do
- echo 'Waiting for CouchDB to start...';
- sleep 5;
-done
-
-# CouchDB needs the `_users` and `_replicator` databases to exist before it will
-# function correctly, so we create them here.
-curl -X PUT -u "${COUCHDB_USER}:${COUCHDB_PASSWORD}" http://localhost:5984/_users
-curl -X PUT -u "${COUCHDB_USER}:${COUCHDB_PASSWORD}" http://localhost:5984/_replicator
-sleep infinity
diff --git a/package.json b/package.json
index 7e5ce25923..6416397c94 100644
--- a/package.json
+++ b/package.json
@@ -77,7 +77,6 @@
"build:docker:single:sqs": "./scripts/build-single-image-sqs.sh",
"build:docker:dependencies": "docker build -f hosting/dependencies/Dockerfile -t budibase/dependencies:latest ./hosting",
"publish:docker:couch": "docker buildx build --platform linux/arm64,linux/amd64 -f hosting/couchdb/Dockerfile -t budibase/couchdb:latest -t budibase/couchdb:v3.3.3 --push ./hosting/couchdb",
- "publish:docker:couch-sqs": "docker buildx build --platform linux/arm64,linux/amd64 -f hosting/couchdb/Dockerfile.v2 -t budibase/couchdb:v3.3.3-sqs --push ./hosting/couchdb",
"publish:docker:dependencies": "docker buildx build --platform linux/arm64,linux/amd64 -f hosting/dependencies/Dockerfile -t budibase/dependencies:latest -t budibase/dependencies:v3.2.1 --push ./hosting",
"release:helm": "node scripts/releaseHelmChart",
"env:multi:enable": "lerna run --stream env:multi:enable",
From 882a056cd9a8a942330c5468f2f065430e2b960b Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 2 Jul 2024 13:58:16 +0200
Subject: [PATCH 2/9] Don't trim any fields on view crud endpoints
---
.../src/api/controllers/view/viewsV2.ts | 5 ---
.../src/api/routes/tests/viewV2.spec.ts | 36 +++++++++----------
2 files changed, 16 insertions(+), 25 deletions(-)
diff --git a/packages/server/src/api/controllers/view/viewsV2.ts b/packages/server/src/api/controllers/view/viewsV2.ts
index 76807b796a..4208772fa6 100644
--- a/packages/server/src/api/controllers/view/viewsV2.ts
+++ b/packages/server/src/api/controllers/view/viewsV2.ts
@@ -33,11 +33,6 @@ async function parseSchema(view: CreateViewRequest) {
p[fieldName] = fieldSchema
return p
}, {} as Record<string, RequiredKeys<ViewUIFieldMetadata>>)
- for (let [key, column] of Object.entries(finalViewSchema)) {
- if (!column.visible && !column.readonly) {
- delete finalViewSchema[key]
- }
- }
return finalViewSchema
}
diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts
index 99ff4f8db7..5d83cd8616 100644
--- a/packages/server/src/api/routes/tests/viewV2.spec.ts
+++ b/packages/server/src/api/routes/tests/viewV2.spec.ts
@@ -218,6 +218,10 @@ describe.each([
order: 1,
width: 100,
},
+ Category: {
+ visible: false,
+ icon: "ic",
+ },
},
id: createdView.id,
version: 2,
@@ -269,9 +273,8 @@ describe.each([
...newView,
schema: {
id: { visible: true },
- Price: {
- visible: true,
- },
+ Price: { visible: true },
+ Category: { visible: false },
},
id: expect.any(String),
version: 2,
@@ -759,6 +762,7 @@ describe.each([
order: 1,
width: 100,
},
+ Category: { visible: false, icon: "ic" },
},
id: view.id,
version: 2,
@@ -873,7 +877,8 @@ describe.each([
await db.getDB(config.appId!).put(tableToUpdate)
view = await config.api.viewV2.get(view.id)
- await config.api.viewV2.update({
+ await config.api.viewV2.update(
+ {
...view,
schema: {
...view.schema,
@@ -881,22 +886,14 @@ describe.each([
visible: false,
},
},
- })
-
- expect(await config.api.viewV2.get(view.id)).toEqual(
- expect.objectContaining({
- schema: {
- id: expect.objectContaining({
- visible: false,
- }),
- Price: expect.objectContaining({
- visible: false,
- }),
- Category: expect.objectContaining({
- visible: true,
- }),
+ },
+ {
+ status: 400,
+ body: {
+ message: 'You can\'t hide "id" because it is a required field.',
+ status: 400,
},
- })
+ }
)
})
})
@@ -938,7 +935,6 @@ describe.each([
Category: { visible: true },
},
})
- expect(res.schema?.Price).toBeUndefined()
const view = await config.api.viewV2.get(res.id)
const updatedTable = await config.api.table.get(table._id!)
From 53af94f971ba03d6ba2568dc7328b24e21329c6f Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 2 Jul 2024 14:02:16 +0200
Subject: [PATCH 3/9] Fix search returning non visible columns
---
packages/server/src/api/controllers/row/views.ts | 4 +++-
packages/server/src/api/routes/tests/viewV2.spec.ts | 13 +++++++------
2 files changed, 10 insertions(+), 7 deletions(-)
diff --git a/packages/server/src/api/controllers/row/views.ts b/packages/server/src/api/controllers/row/views.ts
index 63ce12f0ab..12e76155bc 100644
--- a/packages/server/src/api/controllers/row/views.ts
+++ b/packages/server/src/api/controllers/row/views.ts
@@ -25,7 +25,9 @@ export async function searchView(
ctx.throw(400, `This method only supports viewsV2`)
}
- const viewFields = Object.keys(view.schema || {})
+ const viewFields = Object.entries(view.schema || {})
+ .filter(([_, value]) => value.visible)
+ .map(([key]) => key)
const { body } = ctx.request
// Enrich saved query with ephemeral query params.
diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts
index 5d83cd8616..43a6d39172 100644
--- a/packages/server/src/api/routes/tests/viewV2.spec.ts
+++ b/packages/server/src/api/routes/tests/viewV2.spec.ts
@@ -879,14 +879,14 @@ describe.each([
view = await config.api.viewV2.get(view.id)
await config.api.viewV2.update(
{
- ...view,
- schema: {
- ...view.schema,
- Price: {
- visible: false,
+ ...view,
+ schema: {
+ ...view.schema,
+ Price: {
+ visible: false,
+ },
},
},
- },
{
status: 400,
body: {
@@ -1201,6 +1201,7 @@ describe.each([
],
schema: {
id: { visible: true },
+ one: { visible: false },
two: { visible: true },
},
})
From 07d538f6fc573d32588c041e4e3fc77c8d1b75ef Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 2 Jul 2024 14:02:49 +0200
Subject: [PATCH 4/9] Don't trim fields on view enrich schema
---
packages/server/src/sdk/app/views/index.ts | 12 ++++--------
1 file changed, 4 insertions(+), 8 deletions(-)
diff --git a/packages/server/src/sdk/app/views/index.ts b/packages/server/src/sdk/app/views/index.ts
index b6ac7b6f6b..fce57a390d 100644
--- a/packages/server/src/sdk/app/views/index.ts
+++ b/packages/server/src/sdk/app/views/index.ts
@@ -160,14 +160,10 @@ export function enrichSchema(
for (const key of Object.keys(schema)) {
// if nothing specified in view, then it is not visible
const ui = view.schema?.[key] || { visible: false }
- if (ui.visible === false) {
- schema[key].visible = false
- } else {
- schema[key] = {
- ...schema[key],
- ...ui,
- order: anyViewOrder ? ui?.order ?? undefined : schema[key].order,
- }
+ schema[key] = {
+ ...schema[key],
+ ...ui,
+ order: anyViewOrder ? ui?.order ?? undefined : schema[key].order,
}
}
From 9dacc5ea38ddadeb385a8c68b92138ff09672ab9 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Tue, 2 Jul 2024 14:45:19 +0100
Subject: [PATCH 5/9] Make the SQS environment check work off the
SQS_SEARCH_ENABLE env var.
---
packages/worker/src/api/controllers/system/environment.ts | 8 +-------
1 file changed, 1 insertion(+), 7 deletions(-)
diff --git a/packages/worker/src/api/controllers/system/environment.ts b/packages/worker/src/api/controllers/system/environment.ts
index 203d3d41ff..4deca5df6e 100644
--- a/packages/worker/src/api/controllers/system/environment.ts
+++ b/packages/worker/src/api/controllers/system/environment.ts
@@ -3,12 +3,6 @@ import env from "../../../environment"
import { env as coreEnv } from "@budibase/backend-core"
import nodeFetch from "node-fetch"
-// When we come to move to SQS fully and move away from Clouseau, we will need
-// to flip this to true (or remove it entirely). This will then be used to
-// determine if we should show the maintenance page that links to the SQS
-// migration docs.
-const sqsRequired = false
-
let sqsAvailable: boolean
async function isSqsAvailable() {
// We cache this value for the duration of the Node process because we don't
@@ -30,7 +24,7 @@ async function isSqsAvailable() {
}
async function isSqsMissing() {
- return sqsRequired && !(await isSqsAvailable())
+ return env.SQS_SEARCH_ENABLE && !(await isSqsAvailable())
}
export const fetch = async (ctx: Ctx) => {
From 2e95bd74c993576f4971f7cb9daa45bd714efbf6 Mon Sep 17 00:00:00 2001
From: Budibase Staging Release Bot <>
Date: Tue, 2 Jul 2024 14:23:13 +0000
Subject: [PATCH 6/9] Bump version to 2.29.6
---
lerna.json | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/lerna.json b/lerna.json
index 10bb67fd27..e92bdb224b 100644
--- a/lerna.json
+++ b/lerna.json
@@ -1,6 +1,6 @@
{
"$schema": "node_modules/lerna/schemas/lerna-schema.json",
- "version": "2.29.5",
+ "version": "2.29.6",
"npmClient": "yarn",
"packages": [
"packages/*",
@@ -22,4 +22,4 @@
"loadEnvFiles": false
}
}
-}
+}
\ No newline at end of file
From 39392a793db6015a91dc38a59ac4def910756fd6 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Tue, 2 Jul 2024 16:57:32 +0100
Subject: [PATCH 7/9] Making sure the app migration queue is created correctly
consistently.
---
packages/server/src/appMigrations/queue.ts | 36 +++++++++++-----------
1 file changed, 18 insertions(+), 18 deletions(-)
diff --git a/packages/server/src/appMigrations/queue.ts b/packages/server/src/appMigrations/queue.ts
index e2bc4406f1..12f301da0e 100644
--- a/packages/server/src/appMigrations/queue.ts
+++ b/packages/server/src/appMigrations/queue.ts
@@ -11,26 +11,26 @@ export type AppMigrationJob = {
appId: string
}
-let appMigrationQueue: queue.Queue<AppMigrationJob> | undefined
+// always create app migration queue - so that events can be pushed and read from it
+// across the different api and automation services
+let appMigrationQueue = queue.createQueue<AppMigrationJob>(
+ queue.JobQueue.APP_MIGRATION,
+ {
+ jobOptions: {
+ attempts: MAX_ATTEMPTS,
+ removeOnComplete: true,
+ removeOnFail: true,
+ },
+ maxStalledCount: MAX_ATTEMPTS,
+ removeStalledCb: async (job: Job) => {
+ logging.logAlert(
+ `App migration failed, queue job ID: ${job.id} - reason: ${job.failedReason}`
+ )
+ },
+ }
+)
export function init() {
- appMigrationQueue = queue.createQueue<AppMigrationJob>(
- queue.JobQueue.APP_MIGRATION,
- {
- jobOptions: {
- attempts: MAX_ATTEMPTS,
- removeOnComplete: true,
- removeOnFail: true,
- },
- maxStalledCount: MAX_ATTEMPTS,
- removeStalledCb: async (job: Job) => {
- logging.logAlert(
- `App migration failed, queue job ID: ${job.id} - reason: ${job.failedReason}`
- )
- },
- }
- )
-
return appMigrationQueue.process(MIGRATION_CONCURRENCY, processMessage)
}
From ac2b87df9edcb6224f7b9dd9c019159ba885238c Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Tue, 2 Jul 2024 16:59:42 +0100
Subject: [PATCH 8/9] Quick change.
---
packages/server/src/appMigrations/index.ts | 1 -
packages/server/src/appMigrations/queue.ts | 2 +-
2 files changed, 1 insertion(+), 2 deletions(-)
diff --git a/packages/server/src/appMigrations/index.ts b/packages/server/src/appMigrations/index.ts
index de15666215..5ee92543f0 100644
--- a/packages/server/src/appMigrations/index.ts
+++ b/packages/server/src/appMigrations/index.ts
@@ -49,7 +49,6 @@ export async function checkMissingMigrations(
const queue = getAppMigrationQueue()
if (
- queue &&
latestMigration &&
getTimestamp(currentVersion) < getTimestamp(latestMigration)
) {
diff --git a/packages/server/src/appMigrations/queue.ts b/packages/server/src/appMigrations/queue.ts
index 12f301da0e..d06d039fd0 100644
--- a/packages/server/src/appMigrations/queue.ts
+++ b/packages/server/src/appMigrations/queue.ts
@@ -13,7 +13,7 @@ export type AppMigrationJob = {
// always create app migration queue - so that events can be pushed and read from it
// across the different api and automation services
-let appMigrationQueue = queue.createQueue<AppMigrationJob>(
+const appMigrationQueue = queue.createQueue<AppMigrationJob>(
queue.JobQueue.APP_MIGRATION,
{
jobOptions: {
From 0b5205ad259b859f19b3c94a3b613d50568dc7b3 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Tue, 2 Jul 2024 18:05:59 +0100
Subject: [PATCH 9/9] Setting DISABLE_V8_COMPILE_CACHE=1 when running builds as
it may fix some issues with node 20 and esbuild causing v8 stack traces.
---
package.json | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/package.json b/package.json
index 6416397c94..29b87898ac 100644
--- a/package.json
+++ b/package.json
@@ -33,10 +33,10 @@
"scripts": {
"get-past-client-version": "node scripts/getPastClientVersion.js",
"setup": "git config submodule.recurse true && git submodule update && node ./hosting/scripts/setup.js && yarn && yarn build && yarn dev",
- "build": "NODE_OPTIONS=--max-old-space-size=1500 lerna run build --stream",
- "build:apps": "yarn build --scope @budibase/server --scope @budibase/worker",
+ "build": "DISABLE_V8_COMPILE_CACHE=1 NODE_OPTIONS=--max-old-space-size=1500 lerna run build --stream",
+ "build:apps": "DISABLE_V8_COMPILE_CACHE=1 yarn build --scope @budibase/server --scope @budibase/worker",
+ "build:oss": "DISABLE_V8_COMPILE_CACHE=1 NODE_OPTIONS=--max-old-space-size=1500 lerna run build --stream --ignore @budibase/account-portal-server --ignore @budibase/account-portal-ui",
"build:cli": "yarn build --scope @budibase/cli",
- "build:oss": "NODE_OPTIONS=--max-old-space-size=1500 lerna run build --stream --ignore @budibase/account-portal-server --ignore @budibase/account-portal-ui",
"build:account-portal": "NODE_OPTIONS=--max-old-space-size=1500 lerna run build --stream --scope @budibase/account-portal-server --scope @budibase/account-portal-ui",
"build:dev": "lerna run --stream prebuild && yarn nx run-many --target=build --output-style=dynamic --watch --preserveWatchOutput",
"check:types": "lerna run --concurrency 2 check:types --ignore @budibase/account-portal-server",