Merge remote-tracking branch 'origin/master' into feature/automation-row-ux-update

This commit is contained in:
Dean 2024-07-03 09:05:05 +01:00
commit f44aba763e
13 changed files with 69 additions and 303 deletions

View File

@ -96,10 +96,13 @@ EXPOSE 5984 4369 9100
CMD ["/opt/couchdb/bin/couchdb"]
FROM base as runner
ARG TARGETARCH
ENV TARGETARCH $TARGETARCH
ENV COUCHDB_USER admin
ENV COUCHDB_PASSWORD admin
EXPOSE 5984
EXPOSE 4984
RUN apt-get update && apt-get install -y --no-install-recommends software-properties-common wget unzip curl && \
wget -O - https://packages.adoptium.net/artifactory/api/gpg/key/public | apt-key add - && \
@ -125,7 +128,12 @@ ADD clouseau/log4j.properties clouseau/clouseau.ini ./
WORKDIR /opt/couchdb
ADD couch/vm.args couch/local.ini ./etc/
# setup SQS
WORKDIR /opt/sqs
ADD sqs ./
RUN chmod +x ./install.sh && ./install.sh
WORKDIR /
ADD runner.sh ./bbcouch-runner.sh
RUN chmod +x ./bbcouch-runner.sh /opt/clouseau/bin/clouseau
CMD ["./bbcouch-runner.sh"]
RUN chmod +x ./bbcouch-runner.sh /opt/clouseau/bin/clouseau /opt/sqs/sqs
CMD ["./bbcouch-runner.sh"]

View File

@ -1,139 +0,0 @@
# Modified from https://github.com/apache/couchdb-docker/blob/main/3.3.3/Dockerfile
#
# Everything in this `base` image is adapted from the official `couchdb` image's
# Dockerfile. Only modifications related to upgrading from Debian bullseye to
# bookworm have been included. The `runner` image contains Budibase's
# customisations to the image, e.g. adding Clouseau.
FROM node:20-slim AS base
# Add CouchDB user account to make sure the IDs are assigned consistently
RUN groupadd -g 5984 -r couchdb && useradd -u 5984 -d /opt/couchdb -g couchdb couchdb
# be sure GPG and apt-transport-https are available and functional
RUN set -ex; \
apt-get update; \
apt-get install -y --no-install-recommends \
apt-transport-https \
ca-certificates \
dirmngr \
gnupg \
; \
rm -rf /var/lib/apt/lists/*
# grab tini for signal handling and zombie reaping
# see https://github.com/apache/couchdb-docker/pull/28#discussion_r141112407
RUN set -eux; \
apt-get update; \
apt-get install -y --no-install-recommends tini; \
rm -rf /var/lib/apt/lists/*; \
tini --version
# http://docs.couchdb.org/en/latest/install/unix.html#installing-the-apache-couchdb-packages
ENV GPG_COUCH_KEY \
# gpg: rsa8192 205-01-19 The Apache Software Foundation (Package repository signing key) <root@apache.org>
390EF70BB1EA12B2773962950EE62FB37A00258D
RUN set -eux; \
apt-get update; \
apt-get install -y curl; \
export GNUPGHOME="$(mktemp -d)"; \
curl -fL -o keys.asc https://couchdb.apache.org/repo/keys.asc; \
gpg --batch --import keys.asc; \
gpg --batch --export "${GPG_COUCH_KEY}" > /usr/share/keyrings/couchdb-archive-keyring.gpg; \
command -v gpgconf && gpgconf --kill all || :; \
rm -rf "$GNUPGHOME"; \
apt-key list; \
apt purge -y --autoremove curl; \
rm -rf /var/lib/apt/lists/*
ENV COUCHDB_VERSION 3.3.3
RUN . /etc/os-release; \
echo "deb [signed-by=/usr/share/keyrings/couchdb-archive-keyring.gpg] https://apache.jfrog.io/artifactory/couchdb-deb/ ${VERSION_CODENAME} main" | \
tee /etc/apt/sources.list.d/couchdb.list >/dev/null
# https://github.com/apache/couchdb-pkg/blob/master/debian/README.Debian
RUN set -eux; \
apt-get update; \
\
echo "couchdb couchdb/mode select none" | debconf-set-selections; \
# we DO want recommends this time
DEBIAN_FRONTEND=noninteractive apt-get install -y --allow-downgrades --allow-remove-essential --allow-change-held-packages \
couchdb="$COUCHDB_VERSION"~bookworm \
; \
# Undo symlinks to /var/log and /var/lib
rmdir /var/lib/couchdb /var/log/couchdb; \
rm /opt/couchdb/data /opt/couchdb/var/log; \
mkdir -p /opt/couchdb/data /opt/couchdb/var/log; \
chown couchdb:couchdb /opt/couchdb/data /opt/couchdb/var/log; \
chmod 777 /opt/couchdb/data /opt/couchdb/var/log; \
# Remove file that sets logging to a file
rm /opt/couchdb/etc/default.d/10-filelog.ini; \
# Check we own everything in /opt/couchdb. Matches the command in dockerfile_entrypoint.sh
find /opt/couchdb \! \( -user couchdb -group couchdb \) -exec chown -f couchdb:couchdb '{}' +; \
# Setup directories and permissions for config. Technically these could be 555 and 444 respectively
# but we keep them as 755 and 644 for consistency with CouchDB defaults and the dockerfile_entrypoint.sh.
find /opt/couchdb/etc -type d ! -perm 0755 -exec chmod -f 0755 '{}' +; \
find /opt/couchdb/etc -type f ! -perm 0644 -exec chmod -f 0644 '{}' +; \
# only local.d needs to be writable for the docker_entrypoint.sh
chmod -f 0777 /opt/couchdb/etc/local.d; \
# apt clean-up
rm -rf /var/lib/apt/lists/*;
# Add configuration
COPY --chown=couchdb:couchdb couch/10-docker-default.ini /opt/couchdb/etc/default.d/
# COPY --chown=couchdb:couchdb vm.args /opt/couchdb/etc/
COPY docker-entrypoint.sh /usr/local/bin
RUN ln -s usr/local/bin/docker-entrypoint.sh /docker-entrypoint.sh # backwards compat
ENTRYPOINT ["tini", "--", "/docker-entrypoint.sh"]
VOLUME /opt/couchdb/data
# 5984: Main CouchDB endpoint
# 4369: Erlang portmap daemon (epmd)
# 9100: CouchDB cluster communication port
EXPOSE 5984 4369 9100
CMD ["/opt/couchdb/bin/couchdb"]
FROM base as runner
ARG TARGETARCH
ENV TARGETARCH $TARGETARCH
ENV COUCHDB_USER admin
ENV COUCHDB_PASSWORD admin
EXPOSE 5984
EXPOSE 4984
RUN apt-get update && apt-get install -y --no-install-recommends software-properties-common wget unzip curl && \
wget -O - https://packages.adoptium.net/artifactory/api/gpg/key/public | apt-key add - && \
apt-add-repository 'deb http://security.debian.org/debian-security bookworm-security/updates main' && \
apt-add-repository 'deb http://archive.debian.org/debian stretch-backports main' && \
apt-add-repository 'deb https://packages.adoptium.net/artifactory/deb bookworm main' && \
apt-get update && apt-get install -y --no-install-recommends temurin-8-jdk && \
rm -rf /var/lib/apt/lists/
# setup clouseau
WORKDIR /
RUN wget https://github.com/cloudant-labs/clouseau/releases/download/2.21.0/clouseau-2.21.0-dist.zip && \
unzip clouseau-2.21.0-dist.zip && \
mv clouseau-2.21.0 /opt/clouseau && \
rm clouseau-2.21.0-dist.zip
WORKDIR /opt/clouseau
RUN mkdir ./bin
ADD clouseau/clouseau ./bin/
ADD clouseau/log4j.properties clouseau/clouseau.ini ./
# setup CouchDB
WORKDIR /opt/couchdb
ADD couch/vm.args couch/local.ini ./etc/
# setup SQS
WORKDIR /opt/sqs
ADD sqs ./
RUN chmod +x ./install.sh && ./install.sh
WORKDIR /
ADD runner.v2.sh ./bbcouch-runner.sh
RUN chmod +x ./bbcouch-runner.sh /opt/clouseau/bin/clouseau /opt/sqs/sqs
CMD ["./bbcouch-runner.sh"]

View File

@ -70,9 +70,12 @@ sed -i "s#COUCHDB_ERLANG_COOKIE#${COUCHDB_ERLANG_COOKIE}#g" /opt/clouseau/clouse
/opt/clouseau/bin/clouseau > /dev/stdout 2>&1 &
# Start CouchDB.
/docker-entrypoint.sh /opt/couchdb/bin/couchdb &
/docker-entrypoint.sh /opt/couchdb/bin/couchdb > /dev/stdout 2>&1 &
# Wati for CouchDB to start up.
# Start SQS. Use 127.0.0.1 instead of localhost to avoid IPv6 issues.
/opt/sqs/sqs --server "http://127.0.0.1:5984" --data-dir ${DATA_DIR}/sqs --bind-address=0.0.0.0 > /dev/stdout 2>&1 &
# Wait for CouchDB to start up.
while [[ $(curl -s -w "%{http_code}\n" http://localhost:5984/_up -o /dev/null) -ne 200 ]]; do
echo 'Waiting for CouchDB to start...';
sleep 5;
@ -82,4 +85,4 @@ done
# function correctly, so we create them here.
curl -X PUT -u "${COUCHDB_USER}:${COUCHDB_PASSWORD}" http://localhost:5984/_users
curl -X PUT -u "${COUCHDB_USER}:${COUCHDB_PASSWORD}" http://localhost:5984/_replicator
sleep infinity
sleep infinity

View File

@ -1,88 +0,0 @@
#!/bin/bash
DATA_DIR=${DATA_DIR:-/data}
COUCHDB_ERLANG_COOKIE=${COUCHDB_ERLANG_COOKIE:-B9CFC32C-3458-4A86-8448-B3C753991CA7}
mkdir -p ${DATA_DIR}
mkdir -p ${DATA_DIR}/couch/{dbs,views}
mkdir -p ${DATA_DIR}/search
chown -R couchdb:couchdb ${DATA_DIR}/couch
echo ${TARGETBUILD} > /buildtarget.txt
if [[ "${TARGETBUILD}" = "aas" ]]; then
# Azure AppService uses /home for persistent data & SSH on port 2222
DATA_DIR="${DATA_DIR:-/home}"
WEBSITES_ENABLE_APP_SERVICE_STORAGE=true
mkdir -p $DATA_DIR/{search,minio,couch}
mkdir -p $DATA_DIR/couch/{dbs,views}
chown -R couchdb:couchdb $DATA_DIR/couch/
apt update
apt-get install -y openssh-server
echo "root:Docker!" | chpasswd
mkdir -p /tmp
chmod +x /tmp/ssh_setup.sh \
&& (sleep 1;/tmp/ssh_setup.sh 2>&1 > /dev/null)
cp /etc/sshd_config /etc/ssh/sshd_config
/etc/init.d/ssh restart
sed -i "s#DATA_DIR#/home#g" /opt/clouseau/clouseau.ini
sed -i "s#DATA_DIR#/home#g" /opt/couchdb/etc/local.ini
elif [[ "${TARGETBUILD}" = "single" ]]; then
# In the single image build, the Dockerfile specifies /data as a volume
# mount, so we use that for all persistent data.
sed -i "s#DATA_DIR#/data#g" /opt/clouseau/clouseau.ini
sed -i "s#DATA_DIR#/data#g" /opt/couchdb/etc/local.ini
elif [[ "${TARGETBUILD}" = "docker-compose" ]]; then
# We remove the database_dir and view_index_dir settings from the local.ini
# in docker-compose because it will default to /opt/couchdb/data which is what
# our docker-compose was using prior to us switching to using our own CouchDB
# image.
sed -i "s#^database_dir.*\$##g" /opt/couchdb/etc/local.ini
sed -i "s#^view_index_dir.*\$##g" /opt/couchdb/etc/local.ini
sed -i "s#^dir=.*\$#dir=/opt/couchdb/data#g" /opt/clouseau/clouseau.ini
elif [[ -n $KUBERNETES_SERVICE_HOST ]]; then
# In Kubernetes the directory /opt/couchdb/data has a persistent volume
# mount for storing database data.
sed -i "s#^dir=.*\$#dir=/opt/couchdb/data#g" /opt/clouseau/clouseau.ini
# We remove the database_dir and view_index_dir settings from the local.ini
# in Kubernetes because it will default to /opt/couchdb/data which is what
# our Helm chart was using prior to us switching to using our own CouchDB
# image.
sed -i "s#^database_dir.*\$##g" /opt/couchdb/etc/local.ini
sed -i "s#^view_index_dir.*\$##g" /opt/couchdb/etc/local.ini
# We remove the -name setting from the vm.args file in Kubernetes because
# it will default to the pod FQDN, which is what's required for clustering
# to work.
sed -i "s/^-name .*$//g" /opt/couchdb/etc/vm.args
else
# For all other builds, we use /data for persistent data.
sed -i "s#DATA_DIR#/data#g" /opt/clouseau/clouseau.ini
sed -i "s#DATA_DIR#/data#g" /opt/couchdb/etc/local.ini
fi
sed -i "s#COUCHDB_ERLANG_COOKIE#${COUCHDB_ERLANG_COOKIE}#g" /opt/couchdb/etc/vm.args
sed -i "s#COUCHDB_ERLANG_COOKIE#${COUCHDB_ERLANG_COOKIE}#g" /opt/clouseau/clouseau.ini
# Start Clouseau. Budibase won't function correctly without Clouseau running, it
# powers the search API endpoints which are used to do all sorts, including
# populating app grids.
/opt/clouseau/bin/clouseau > /dev/stdout 2>&1 &
# Start CouchDB.
/docker-entrypoint.sh /opt/couchdb/bin/couchdb > /dev/stdout 2>&1 &
# Start SQS. Use 127.0.0.1 instead of localhost to avoid IPv6 issues.
/opt/sqs/sqs --server "http://127.0.0.1:5984" --data-dir ${DATA_DIR}/sqs --bind-address=0.0.0.0 > /dev/stdout 2>&1 &
# Wait for CouchDB to start up.
while [[ $(curl -s -w "%{http_code}\n" http://localhost:5984/_up -o /dev/null) -ne 200 ]]; do
echo 'Waiting for CouchDB to start...';
sleep 5;
done
# CouchDB needs the `_users` and `_replicator` databases to exist before it will
# function correctly, so we create them here.
curl -X PUT -u "${COUCHDB_USER}:${COUCHDB_PASSWORD}" http://localhost:5984/_users
curl -X PUT -u "${COUCHDB_USER}:${COUCHDB_PASSWORD}" http://localhost:5984/_replicator
sleep infinity

View File

@ -1,6 +1,6 @@
{
"$schema": "node_modules/lerna/schemas/lerna-schema.json",
"version": "2.29.5",
"version": "2.29.6",
"npmClient": "yarn",
"packages": [
"packages/*",
@ -22,4 +22,4 @@
"loadEnvFiles": false
}
}
}
}

View File

@ -33,10 +33,10 @@
"scripts": {
"get-past-client-version": "node scripts/getPastClientVersion.js",
"setup": "git config submodule.recurse true && git submodule update && node ./hosting/scripts/setup.js && yarn && yarn build && yarn dev",
"build": "NODE_OPTIONS=--max-old-space-size=1500 lerna run build --stream",
"build:apps": "yarn build --scope @budibase/server --scope @budibase/worker",
"build": "DISABLE_V8_COMPILE_CACHE=1 NODE_OPTIONS=--max-old-space-size=1500 lerna run build --stream",
"build:apps": "DISABLE_V8_COMPILE_CACHE=1 yarn build --scope @budibase/server --scope @budibase/worker",
"build:oss": "DISABLE_V8_COMPILE_CACHE=1 NODE_OPTIONS=--max-old-space-size=1500 lerna run build --stream --ignore @budibase/account-portal-server --ignore @budibase/account-portal-ui",
"build:cli": "yarn build --scope @budibase/cli",
"build:oss": "NODE_OPTIONS=--max-old-space-size=1500 lerna run build --stream --ignore @budibase/account-portal-server --ignore @budibase/account-portal-ui",
"build:account-portal": "NODE_OPTIONS=--max-old-space-size=1500 lerna run build --stream --scope @budibase/account-portal-server --scope @budibase/account-portal-ui",
"build:dev": "lerna run --stream prebuild && yarn nx run-many --target=build --output-style=dynamic --watch --preserveWatchOutput",
"check:types": "lerna run --concurrency 2 check:types --ignore @budibase/account-portal-server",
@ -77,7 +77,6 @@
"build:docker:single:sqs": "./scripts/build-single-image-sqs.sh",
"build:docker:dependencies": "docker build -f hosting/dependencies/Dockerfile -t budibase/dependencies:latest ./hosting",
"publish:docker:couch": "docker buildx build --platform linux/arm64,linux/amd64 -f hosting/couchdb/Dockerfile -t budibase/couchdb:latest -t budibase/couchdb:v3.3.3 --push ./hosting/couchdb",
"publish:docker:couch-sqs": "docker buildx build --platform linux/arm64,linux/amd64 -f hosting/couchdb/Dockerfile.v2 -t budibase/couchdb:v3.3.3-sqs --push ./hosting/couchdb",
"publish:docker:dependencies": "docker buildx build --platform linux/arm64,linux/amd64 -f hosting/dependencies/Dockerfile -t budibase/dependencies:latest -t budibase/dependencies:v3.2.1 --push ./hosting",
"release:helm": "node scripts/releaseHelmChart",
"env:multi:enable": "lerna run --stream env:multi:enable",

View File

@ -25,7 +25,9 @@ export async function searchView(
ctx.throw(400, `This method only supports viewsV2`)
}
const viewFields = Object.keys(view.schema || {})
const viewFields = Object.entries(view.schema || {})
.filter(([_, value]) => value.visible)
.map(([key]) => key)
const { body } = ctx.request
// Enrich saved query with ephemeral query params.

View File

@ -33,11 +33,6 @@ async function parseSchema(view: CreateViewRequest) {
p[fieldName] = fieldSchema
return p
}, {} as Record<string, RequiredKeys<ViewUIFieldMetadata>>)
for (let [key, column] of Object.entries(finalViewSchema)) {
if (!column.visible && !column.readonly) {
delete finalViewSchema[key]
}
}
return finalViewSchema
}

View File

@ -218,6 +218,10 @@ describe.each([
order: 1,
width: 100,
},
Category: {
visible: false,
icon: "ic",
},
},
id: createdView.id,
version: 2,
@ -269,9 +273,8 @@ describe.each([
...newView,
schema: {
id: { visible: true },
Price: {
visible: true,
},
Price: { visible: true },
Category: { visible: false },
},
id: expect.any(String),
version: 2,
@ -759,6 +762,7 @@ describe.each([
order: 1,
width: 100,
},
Category: { visible: false, icon: "ic" },
},
id: view.id,
version: 2,
@ -873,30 +877,23 @@ describe.each([
await db.getDB(config.appId!).put(tableToUpdate)
view = await config.api.viewV2.get(view.id)
await config.api.viewV2.update({
...view,
schema: {
...view.schema,
Price: {
visible: false,
await config.api.viewV2.update(
{
...view,
schema: {
...view.schema,
Price: {
visible: false,
},
},
},
})
expect(await config.api.viewV2.get(view.id)).toEqual(
expect.objectContaining({
schema: {
id: expect.objectContaining({
visible: false,
}),
Price: expect.objectContaining({
visible: false,
}),
Category: expect.objectContaining({
visible: true,
}),
{
status: 400,
body: {
message: 'You can\'t hide "id" because it is a required field.',
status: 400,
},
})
}
)
})
})
@ -938,7 +935,6 @@ describe.each([
Category: { visible: true },
},
})
expect(res.schema?.Price).toBeUndefined()
const view = await config.api.viewV2.get(res.id)
const updatedTable = await config.api.table.get(table._id!)
@ -1205,6 +1201,7 @@ describe.each([
],
schema: {
id: { visible: true },
one: { visible: false },
two: { visible: true },
},
})

View File

@ -49,7 +49,6 @@ export async function checkMissingMigrations(
const queue = getAppMigrationQueue()
if (
queue &&
latestMigration &&
getTimestamp(currentVersion) < getTimestamp(latestMigration)
) {

View File

@ -11,26 +11,26 @@ export type AppMigrationJob = {
appId: string
}
let appMigrationQueue: queue.Queue<AppMigrationJob> | undefined
// always create app migration queue - so that events can be pushed and read from it
// across the different api and automation services
const appMigrationQueue = queue.createQueue<AppMigrationJob>(
queue.JobQueue.APP_MIGRATION,
{
jobOptions: {
attempts: MAX_ATTEMPTS,
removeOnComplete: true,
removeOnFail: true,
},
maxStalledCount: MAX_ATTEMPTS,
removeStalledCb: async (job: Job) => {
logging.logAlert(
`App migration failed, queue job ID: ${job.id} - reason: ${job.failedReason}`
)
},
}
)
export function init() {
appMigrationQueue = queue.createQueue<AppMigrationJob>(
queue.JobQueue.APP_MIGRATION,
{
jobOptions: {
attempts: MAX_ATTEMPTS,
removeOnComplete: true,
removeOnFail: true,
},
maxStalledCount: MAX_ATTEMPTS,
removeStalledCb: async (job: Job) => {
logging.logAlert(
`App migration failed, queue job ID: ${job.id} - reason: ${job.failedReason}`
)
},
}
)
return appMigrationQueue.process(MIGRATION_CONCURRENCY, processMessage)
}

View File

@ -160,14 +160,10 @@ export function enrichSchema(
for (const key of Object.keys(schema)) {
// if nothing specified in view, then it is not visible
const ui = view.schema?.[key] || { visible: false }
if (ui.visible === false) {
schema[key].visible = false
} else {
schema[key] = {
...schema[key],
...ui,
order: anyViewOrder ? ui?.order ?? undefined : schema[key].order,
}
schema[key] = {
...schema[key],
...ui,
order: anyViewOrder ? ui?.order ?? undefined : schema[key].order,
}
}

View File

@ -3,12 +3,6 @@ import env from "../../../environment"
import { env as coreEnv } from "@budibase/backend-core"
import nodeFetch from "node-fetch"
// When we come to move to SQS fully and move away from Clouseau, we will need
// to flip this to true (or remove it entirely). This will then be used to
// determine if we should show the maintenance page that links to the SQS
// migration docs.
const sqsRequired = false
let sqsAvailable: boolean
async function isSqsAvailable() {
// We cache this value for the duration of the Node process because we don't
@ -30,7 +24,7 @@ async function isSqsAvailable() {
}
async function isSqsMissing() {
return sqsRequired && !(await isSqsAvailable())
return env.SQS_SEARCH_ENABLE && !(await isSqsAvailable())
}
export const fetch = async (ctx: Ctx) => {