From b171e8e00fc2516d7a76191d6a744cf7da9795ee Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 18 Oct 2023 14:28:05 +0200 Subject: [PATCH 1/8] Update the script to be sh and workspace agnostic --- hosting/single/Dockerfile.v2 | 5 +-- scripts/removeWorkspaceDependencies.sh | 47 +++++++------------------- 2 files changed, 15 insertions(+), 37 deletions(-) diff --git a/hosting/single/Dockerfile.v2 b/hosting/single/Dockerfile.v2 index b1abe6d53e..ad11545a22 100644 --- a/hosting/single/Dockerfile.v2 +++ b/hosting/single/Dockerfile.v2 @@ -19,13 +19,14 @@ COPY packages/string-templates/package.json packages/string-templates/package.js COPY scripts/removeWorkspaceDependencies.sh scripts/removeWorkspaceDependencies.sh RUN chmod +x ./scripts/removeWorkspaceDependencies.sh -RUN ./scripts/removeWorkspaceDependencies.sh +RUN ./scripts/removeWorkspaceDependencies.sh packages/server/package.json +RUN ./scripts/removeWorkspaceDependencies.sh packages/worker/package.json # We will never want to sync pro, but the script is still required RUN echo '' > scripts/syncProPackage.js RUN jq 'del(.scripts.postinstall)' package.json > temp.json && mv temp.json package.json -RUN --mount=type=cache,target=/root/.yarn YARN_CACHE_FOLDER=/root/.yarn yarn install --production +RUN --mount=type=cache,target=/root/.yarn YARN_CACHE_FOLDER=/root/.yarn yarn install --production # copy the actual code COPY packages/server/dist packages/server/dist diff --git a/scripts/removeWorkspaceDependencies.sh b/scripts/removeWorkspaceDependencies.sh index 627058c92a..850dea924c 100755 --- a/scripts/removeWorkspaceDependencies.sh +++ b/scripts/removeWorkspaceDependencies.sh @@ -1,31 +1,17 @@ -#!/bin/bash +#!/bin/sh -packages_to_remove=( - @budibase/backend-core - @budibase/bbui - @budibase/builder - @budibase/cli - @budibase/client - @budibase/frontend-core - @budibase/pro - @budibase/sdk - @budibase/server - @budibase/shared-core # We cannot remove string-templates yet because it cannot be bundled by esbuild as a dependency - @budibase/string-templates - @budibase/types - @budibase/worker -) +packages_to_remove="@budibase/backend-core @budibase/bbui @budibase/builder @budibase/cli @budibase/client @budibase/frontend-core @budibase/pro @budibase/sdk @budibase/server @budibase/shared-core @budibase/types @budibase/worker" - -root_package_json=$(cat "package.json") +package_json_path="$1" +package_json=$(cat "$package_json_path") process_package() { - local pkg="$1" - local package_json=$(cat "$pkg/package.json") - local has_changes=false + pkg_path="$1" + package_json=$(cat "$pkg_path") + has_changes=false - for package_name in "${packages_to_remove[@]}"; do + for package_name in $packages_to_remove; do if echo "$package_json" | jq -e --arg package_name "$package_name" '.dependencies | has($package_name)' > /dev/null; then package_json=$(echo "$package_json" | jq "del(.dependencies[\"$package_name\"])") has_changes=true @@ -33,20 +19,11 @@ process_package() { done if [ "$has_changes" = true ]; then - echo "$package_json" > "$1/package.json" + echo "$package_json" > "$pkg_path" fi } +process_package "$package_json_path" -for pkg in $(echo "$root_package_json" | jq -r '.workspaces.packages[]' ); do - if [[ "$pkg" == *"*"* ]]; then - # Use find to iterate through immediate subdirectories - find "$pkg" -maxdepth 1 -type d -print | while read -r workspace_package; do - process_package "$workspace_package" - done - else - process_package "$pkg" - fi -done - -echo "$root_package_json" | jq "del(.resolutions)" > "package.json" \ No newline at end of file +package_json=$(cat "$package_json_path") +echo "$package_json" | jq "del(.resolutions)" > "$1" From e80a20d61fbbacbcaafb84138bbbe9e0142ecc89 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 18 Oct 2023 14:28:30 +0200 Subject: [PATCH 2/8] Use root as docker context for the app/worker images --- hosting/docker-compose.build.yaml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/hosting/docker-compose.build.yaml b/hosting/docker-compose.build.yaml index bc363fb0bf..f21b9519e4 100644 --- a/hosting/docker-compose.build.yaml +++ b/hosting/docker-compose.build.yaml @@ -4,7 +4,9 @@ version: "3" services: app-service: - build: ../packages/server + build: + context: .. + dockerfile: packages/server/Dockerfile container_name: build-bbapps environment: SELF_HOSTED: 1 @@ -32,7 +34,9 @@ services: # - /some/path/to/plugins:/plugins worker-service: - build: ../packages/worker + build: + context: .. + dockerfile: packages/worker/Dockerfile container_name: build-bbworker environment: SELF_HOSTED: 1 From ba053490dc2fd836df779531ef08496a97878e2d Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 18 Oct 2023 16:06:32 +0200 Subject: [PATCH 3/8] Npm-less images --- packages/server/Dockerfile | 39 +++++++++++++++++++------- packages/worker/Dockerfile | 30 +++++++++++++++----- scripts/removeWorkspaceDependencies.sh | 3 +- 3 files changed, 53 insertions(+), 19 deletions(-) diff --git a/packages/server/Dockerfile b/packages/server/Dockerfile index e1b3b208c7..d5a86b037d 100644 --- a/packages/server/Dockerfile +++ b/packages/server/Dockerfile @@ -18,12 +18,12 @@ ENV TOP_LEVEL_PATH=/ # handle node-gyp RUN apt-get update \ - && apt-get install -y --no-install-recommends g++ make python3 + && apt-get install -y --no-install-recommends g++ make python3 jq RUN yarn global add pm2 # Install client for oracle datasource RUN apt-get install unzip libaio1 -COPY scripts/integrations/oracle/ scripts/integrations/oracle/ +COPY packages/server/scripts/integrations/oracle/ scripts/integrations/oracle/ RUN /bin/bash -e scripts/integrations/oracle/instantclient/linux/x86-64/install.sh # Install postgres client for pg_dump utils @@ -35,18 +35,37 @@ RUN apt update && apt upgrade -y \ && apt install postgresql-client-15 -y \ && apt remove software-properties-common apt-transport-https curl gpg -y +WORKDIR / -COPY package.json . -COPY dist/yarn.lock . -RUN yarn install --production=true \ +COPY scripts/removeWorkspaceDependencies.sh scripts/removeWorkspaceDependencies.sh +RUN chmod +x ./scripts/removeWorkspaceDependencies.sh + + +WORKDIR /string-templates +COPY packages/string-templates/package.json package.json +RUN ../scripts/removeWorkspaceDependencies.sh package.json +RUN --mount=type=cache,target=/root/.yarn YARN_CACHE_FOLDER=/root/.yarn yarn install --production=true +COPY packages/string-templates . + + +WORKDIR /app +COPY packages/server/package.json . +COPY packages/server/dist/yarn.lock . +RUN cd ../string-templates && yarn link && cd - && yarn link @budibase/string-templates + +COPY scripts/removeWorkspaceDependencies.sh scripts/removeWorkspaceDependencies.sh +RUN chmod +x ./scripts/removeWorkspaceDependencies.sh +RUN ./scripts/removeWorkspaceDependencies.sh package.json + +RUN --mount=type=cache,target=/root/.yarn YARN_CACHE_FOLDER=/root/.yarn yarn install --production=true \ # Remove unneeded data from file system to reduce image size - && yarn cache clean && apt-get remove -y --purge --auto-remove g++ make python \ + && yarn cache clean && apt-get remove -y --purge --auto-remove g++ make python jq \ && rm -rf /tmp/* /root/.node-gyp /usr/local/lib/node_modules/npm/node_modules/node-gyp -COPY dist/ dist/ -COPY docker_run.sh . -COPY builder/ builder/ -COPY client/ client/ +COPY packages/server/dist/ dist/ +COPY packages/server/docker_run.sh . +COPY packages/server/builder/ builder/ +COPY packages/server/client/ client/ EXPOSE 4001 diff --git a/packages/worker/Dockerfile b/packages/worker/Dockerfile index 4230ee86f8..0d60db6fc5 100644 --- a/packages/worker/Dockerfile +++ b/packages/worker/Dockerfile @@ -5,22 +5,38 @@ LABEL com.centurylinklabs.watchtower.lifecycle.pre-update="scripts/watchtower-ho LABEL com.centurylinklabs.watchtower.lifecycle.post-update="scripts/watchtower-hooks/post-update.sh" LABEL com.centurylinklabs.watchtower.lifecycle.post-check="scripts/watchtower-hooks/post-check.sh" -WORKDIR /app # handle node-gyp -RUN apk add --no-cache --virtual .gyp python3 make g++ +RUN apk add --no-cache --virtual .gyp python3 make g++ jq RUN yarn global add pm2 +WORKDIR / -COPY package.json . -COPY dist/yarn.lock . -RUN yarn install --production=true +COPY scripts/removeWorkspaceDependencies.sh scripts/removeWorkspaceDependencies.sh +RUN chmod +x ./scripts/removeWorkspaceDependencies.sh + + +WORKDIR /string-templates +COPY packages/string-templates/package.json package.json +RUN ../scripts/removeWorkspaceDependencies.sh package.json +RUN --mount=type=cache,target=/root/.yarn YARN_CACHE_FOLDER=/root/.yarn yarn install --production=true +COPY packages/string-templates . + + +WORKDIR /app +COPY packages/worker/package.json . +COPY packages/worker/dist/yarn.lock . +RUN cd ../string-templates && yarn link && cd - && yarn link @budibase/string-templates + +RUN ../scripts/removeWorkspaceDependencies.sh package.json + +RUN --mount=type=cache,target=/root/.yarn YARN_CACHE_FOLDER=/root/.yarn yarn install --production=true # Remove unneeded data from file system to reduce image size RUN apk del .gyp \ && yarn cache clean -COPY dist/ dist/ -COPY docker_run.sh . +COPY packages/worker/dist/ dist/ +COPY packages/worker/docker_run.sh . EXPOSE 4001 diff --git a/scripts/removeWorkspaceDependencies.sh b/scripts/removeWorkspaceDependencies.sh index 850dea924c..ba27ae7edc 100755 --- a/scripts/removeWorkspaceDependencies.sh +++ b/scripts/removeWorkspaceDependencies.sh @@ -1,7 +1,6 @@ #!/bin/sh - # We cannot remove string-templates yet because it cannot be bundled by esbuild as a dependency -packages_to_remove="@budibase/backend-core @budibase/bbui @budibase/builder @budibase/cli @budibase/client @budibase/frontend-core @budibase/pro @budibase/sdk @budibase/server @budibase/shared-core @budibase/types @budibase/worker" +packages_to_remove="@budibase/backend-core @budibase/bbui @budibase/builder @budibase/cli @budibase/client @budibase/frontend-core @budibase/pro @budibase/sdk @budibase/server @budibase/shared-core @budibase/string-templates @budibase/types @budibase/worker" package_json_path="$1" package_json=$(cat "$package_json_path") From 3851a679157f9e789d22e0c5968038f62c5ce6be Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 19 Oct 2023 11:00:53 +0200 Subject: [PATCH 4/8] Unify jsonwebtoken and lodash packages --- packages/backend-core/package.json | 4 ++-- packages/pro | 2 +- packages/server/package.json | 2 +- packages/string-templates/package.json | 2 +- packages/worker/package.json | 2 +- yarn.lock | 27 +++++++++++--------------- 6 files changed, 17 insertions(+), 22 deletions(-) diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json index 22ca5b21cc..b23cd8e5b1 100644 --- a/packages/backend-core/package.json +++ b/packages/backend-core/package.json @@ -35,7 +35,7 @@ "dotenv": "16.0.1", "ioredis": "5.3.2", "joi": "17.6.0", - "jsonwebtoken": "9.0.0", + "jsonwebtoken": "9.0.2", "koa-passport": "4.1.4", "koa-pino-logger": "4.0.0", "lodash": "4.17.21", @@ -63,7 +63,7 @@ "@types/chance": "1.1.3", "@types/cookies": "0.7.8", "@types/jest": "29.5.5", - "@types/lodash": "4.14.180", + "@types/lodash": "4.14.200", "@types/node": "18.17.0", "@types/node-fetch": "2.6.4", "@types/pouchdb": "6.4.0", diff --git a/packages/pro b/packages/pro index 570d14aa44..7301ec1ecd 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 570d14aa44aa88f4d053856322210f0008ba5c76 +Subproject commit 7301ec1ecd32dc765b94dff60b17ad4db2937aa8 diff --git a/packages/server/package.json b/packages/server/package.json index 2e23e71b64..0bc3618a08 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -122,7 +122,7 @@ "@types/jest": "29.5.5", "@types/koa": "2.13.4", "@types/koa__router": "8.0.8", - "@types/lodash": "4.14.180", + "@types/lodash": "4.14.200", "@types/mssql": "8.1.2", "@types/node": "18.17.0", "@types/node-fetch": "2.6.4", diff --git a/packages/string-templates/package.json b/packages/string-templates/package.json index d9f56442a0..1e32e023f3 100644 --- a/packages/string-templates/package.json +++ b/packages/string-templates/package.json @@ -28,7 +28,7 @@ "@budibase/handlebars-helpers": "^0.11.9", "dayjs": "^1.10.8", "handlebars": "^4.7.6", - "lodash": "^4.17.20", + "lodash": "4.17.21", "vm2": "^3.9.19" }, "devDependencies": { diff --git a/packages/worker/package.json b/packages/worker/package.json index 882752215d..6895386504 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -77,7 +77,7 @@ "@types/jsonwebtoken": "8.5.1", "@types/koa": "2.13.4", "@types/koa__router": "8.0.8", - "@types/lodash": "^4.14.191", + "@types/lodash": "4.14.200", "@types/node": "18.17.0", "@types/node-fetch": "2.6.4", "@types/server-destroy": "1.0.1", diff --git a/yarn.lock b/yarn.lock index a258f08a1d..e3629a8dbc 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5595,15 +5595,10 @@ resolved "https://registry.yarnpkg.com/@types/linkify-it/-/linkify-it-3.0.3.tgz#15a0712296c5041733c79efe233ba17ae5a7587b" integrity sha512-pTjcqY9E4nOI55Wgpz7eiI8+LzdYnw3qxXCfHyBDdPbYvbyLgWLJGh8EdPvqawwMK1Uo1794AUkkR38Fr0g+2g== -"@types/lodash@4.14.180": - version "4.14.180" - resolved "https://registry.yarnpkg.com/@types/lodash/-/lodash-4.14.180.tgz#4ab7c9ddfc92ec4a887886483bc14c79fb380670" - integrity sha512-XOKXa1KIxtNXgASAnwj7cnttJxS4fksBRywK/9LzRV5YxrF80BXZIGeQSuoESQ/VkUj30Ae0+YcuHc15wJCB2g== - -"@types/lodash@^4.14.191": - version "4.14.192" - resolved "https://registry.yarnpkg.com/@types/lodash/-/lodash-4.14.192.tgz#5790406361a2852d332d41635d927f1600811285" - integrity sha512-km+Vyn3BYm5ytMO13k9KTp27O75rbQ0NFw+U//g+PX7VZyjCioXaRFisqSIJRECljcTv73G3i6BpglNGHgUQ5A== +"@types/lodash@4.14.200": + version "4.14.200" + resolved "https://registry.yarnpkg.com/@types/lodash/-/lodash-4.14.200.tgz#435b6035c7eba9cdf1e039af8212c9e9281e7149" + integrity sha512-YI/M/4HRImtNf3pJgbF+W6FrXovqj+T+/HpENLTooK9PnkacBsDpeP3IpHab40CClUfhNmdM2WTNP2sa2dni5Q== "@types/long@^4.0.0", "@types/long@^4.0.1": version "4.0.2" @@ -13835,10 +13830,10 @@ jsonschema@1.4.0: resolved "https://registry.yarnpkg.com/jsonschema/-/jsonschema-1.4.0.tgz#1afa34c4bc22190d8e42271ec17ac8b3404f87b2" integrity sha512-/YgW6pRMr6M7C+4o8kS+B/2myEpHCrxO4PEWnqJNBFMjn7EWXqlQ4tGwL6xTHeRplwuZmcAncdvfOad1nT2yMw== -jsonwebtoken@8.5.1: - version "8.5.1" - resolved "https://registry.yarnpkg.com/jsonwebtoken/-/jsonwebtoken-8.5.1.tgz#00e71e0b8df54c2121a1f26137df2280673bcc0d" - integrity sha512-XjwVfRS6jTMsqYs0EsuJ4LGxXV14zQybNd4L2r0UvbVnSF9Af8x7p5MzbJ90Ioz/9TI41/hTCvznF/loiSzn8w== +jsonwebtoken@9.0.2: + version "9.0.2" + resolved "https://registry.yarnpkg.com/jsonwebtoken/-/jsonwebtoken-9.0.2.tgz#65ff91f4abef1784697d40952bb1998c504caaf3" + integrity sha512-PRp66vJ865SSqOlgqS8hujT5U4AOgMfhrwYIuIhfKaoSCZcirrmASQr8CX7cUg+RMih+hgznrjp99o+W4pJLHQ== dependencies: jws "^3.2.2" lodash.includes "^4.3.0" @@ -13849,9 +13844,9 @@ jsonwebtoken@8.5.1: lodash.isstring "^4.0.1" lodash.once "^4.0.0" ms "^2.1.1" - semver "^5.6.0" + semver "^7.5.4" -jsonwebtoken@9.0.0, jsonwebtoken@^9.0.0: +jsonwebtoken@^9.0.0: version "9.0.0" resolved "https://registry.yarnpkg.com/jsonwebtoken/-/jsonwebtoken-9.0.0.tgz#d0faf9ba1cc3a56255fe49c0961a67e520c1926d" integrity sha512-tuGfYXxkQGDPnLJ7SibiQgVgeDgfbPq2k2ICcbgqW8WxWLBAxKQM/ZCu/IT8SOSwmaYl4dpTFCW5xZv7YbbWUw== @@ -14799,7 +14794,7 @@ lodash.xor@^4.5.0: resolved "https://registry.yarnpkg.com/lodash.xor/-/lodash.xor-4.5.0.tgz#4d48ed7e98095b0632582ba714d3ff8ae8fb1db6" integrity sha512-sVN2zimthq7aZ5sPGXnSz32rZPuqcparVW50chJQe+mzTYV+IsxSsl/2gnkWWE2Of7K3myBQBqtLKOUEHJKRsQ== -lodash@4.17.21, lodash@^4.17.11, lodash@^4.17.15, lodash@^4.17.19, lodash@^4.17.20, lodash@^4.17.21, lodash@^4.17.3, lodash@^4.7.0: +lodash@4.17.21, lodash@^4.17.11, lodash@^4.17.15, lodash@^4.17.19, lodash@^4.17.21, lodash@^4.17.3, lodash@^4.7.0: version "4.17.21" resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.21.tgz#679591c564c3bffaae8454cf0b3df370c3d6911c" integrity sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg== From 38144aa15ae619ef6b258b8083445dac903f6593 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 19 Oct 2023 11:08:56 +0200 Subject: [PATCH 5/8] Clean scripts --- .github/workflows/budibase_ci.yml | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml index 840d580892..1580be338e 100644 --- a/.github/workflows/budibase_ci.yml +++ b/.github/workflows/budibase_ci.yml @@ -231,7 +231,7 @@ jobs: cache: "yarn" - run: yarn --frozen-lockfile - name: Build packages - run: yarn build --scope @budibase/server --scope @budibase/worker --scope @budibase/client --scope @budibase/backend-core + run: yarn build --scope @budibase/server --scope @budibase/worker - name: Run tests run: | cd qa-core diff --git a/package.json b/package.json index 7f5c971009..100a306a35 100644 --- a/package.json +++ b/package.json @@ -46,7 +46,7 @@ "dev:noserver": "yarn run kill-builder && lerna run --stream dev:stack:up && lerna run --stream dev:builder --ignore @budibase/backend-core --ignore @budibase/server --ignore @budibase/worker", "dev:server": "yarn run kill-server && lerna run --stream dev:builder --scope @budibase/worker --scope @budibase/server", "dev:built": "yarn run kill-all && cd packages/server && yarn dev:stack:up && cd ../../ && lerna run --stream dev:built", - "dev:docker": "yarn build && docker-compose -f hosting/docker-compose.build.yaml -f hosting/docker-compose.dev.yaml --env-file hosting/.env up --build --scale proxy-service=0", + "dev:docker": "yarn build --scope @budibase/server --scope @budibase/worker && docker-compose -f hosting/docker-compose.build.yaml -f hosting/docker-compose.dev.yaml --env-file hosting/.env up --build --scale proxy-service=0", "test": "lerna run --stream test --stream", "lint:eslint": "eslint packages qa-core --max-warnings=0", "lint:prettier": "prettier --check \"packages/**/*.{js,ts,svelte}\" && prettier --write \"examples/**/*.{js,ts,svelte}\" && prettier --check \"qa-core/**/*.{js,ts,svelte}\"", From 63a26ac8239fb3101eaa7a530045de44b14e9219 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 19 Oct 2023 11:16:52 +0200 Subject: [PATCH 6/8] Fix dependencies --- packages/worker/package.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/worker/package.json b/packages/worker/package.json index 6895386504..1eee3f020f 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -48,6 +48,7 @@ "aws-sdk": "2.1030.0", "bcrypt": "5.1.0", "bcryptjs": "2.4.3", + "bull": "4.10.1", "dd-trace": "3.13.2", "dotenv": "8.6.0", "global-agent": "3.0.0", @@ -61,6 +62,7 @@ "koa-session": "5.13.1", "koa-static": "5.0.0", "koa-useragent": "^4.1.0", + "lodash": "4.17.21", "node-fetch": "2.6.7", "nodemailer": "6.7.2", "passport-google-oauth": "2.0.0", @@ -84,7 +86,6 @@ "@types/supertest": "2.0.12", "@types/uuid": "8.3.4", "jest": "29.6.2", - "lodash": "4.17.21", "nodemon": "2.0.15", "rimraf": "3.0.2", "supertest": "6.2.2", From a30982177f4ed1bfefbacd701fe03022f1d74b5a Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 19 Oct 2023 17:04:44 +0200 Subject: [PATCH 7/8] Update pro --- packages/pro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pro b/packages/pro index 7301ec1ecd..570d14aa44 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 7301ec1ecd32dc765b94dff60b17ad4db2937aa8 +Subproject commit 570d14aa44aa88f4d053856322210f0008ba5c76 From 6b68e5283681fa72abb78784b47ed20e1fb3f794 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 20 Oct 2023 10:02:18 +0200 Subject: [PATCH 8/8] Use v2 dockerfiles --- hosting/docker-compose.build.yaml | 4 +- packages/server/Dockerfile | 39 ++++----------- packages/server/Dockerfile.v2 | 79 +++++++++++++++++++++++++++++++ packages/worker/Dockerfile | 30 +++--------- packages/worker/Dockerfile.v2 | 53 +++++++++++++++++++++ 5 files changed, 151 insertions(+), 54 deletions(-) create mode 100644 packages/server/Dockerfile.v2 create mode 100644 packages/worker/Dockerfile.v2 diff --git a/hosting/docker-compose.build.yaml b/hosting/docker-compose.build.yaml index f21b9519e4..e192620b59 100644 --- a/hosting/docker-compose.build.yaml +++ b/hosting/docker-compose.build.yaml @@ -6,7 +6,7 @@ services: app-service: build: context: .. - dockerfile: packages/server/Dockerfile + dockerfile: packages/server/Dockerfile.v2 container_name: build-bbapps environment: SELF_HOSTED: 1 @@ -36,7 +36,7 @@ services: worker-service: build: context: .. - dockerfile: packages/worker/Dockerfile + dockerfile: packages/worker/Dockerfile.v2 container_name: build-bbworker environment: SELF_HOSTED: 1 diff --git a/packages/server/Dockerfile b/packages/server/Dockerfile index d5a86b037d..e1b3b208c7 100644 --- a/packages/server/Dockerfile +++ b/packages/server/Dockerfile @@ -18,12 +18,12 @@ ENV TOP_LEVEL_PATH=/ # handle node-gyp RUN apt-get update \ - && apt-get install -y --no-install-recommends g++ make python3 jq + && apt-get install -y --no-install-recommends g++ make python3 RUN yarn global add pm2 # Install client for oracle datasource RUN apt-get install unzip libaio1 -COPY packages/server/scripts/integrations/oracle/ scripts/integrations/oracle/ +COPY scripts/integrations/oracle/ scripts/integrations/oracle/ RUN /bin/bash -e scripts/integrations/oracle/instantclient/linux/x86-64/install.sh # Install postgres client for pg_dump utils @@ -35,37 +35,18 @@ RUN apt update && apt upgrade -y \ && apt install postgresql-client-15 -y \ && apt remove software-properties-common apt-transport-https curl gpg -y -WORKDIR / -COPY scripts/removeWorkspaceDependencies.sh scripts/removeWorkspaceDependencies.sh -RUN chmod +x ./scripts/removeWorkspaceDependencies.sh - - -WORKDIR /string-templates -COPY packages/string-templates/package.json package.json -RUN ../scripts/removeWorkspaceDependencies.sh package.json -RUN --mount=type=cache,target=/root/.yarn YARN_CACHE_FOLDER=/root/.yarn yarn install --production=true -COPY packages/string-templates . - - -WORKDIR /app -COPY packages/server/package.json . -COPY packages/server/dist/yarn.lock . -RUN cd ../string-templates && yarn link && cd - && yarn link @budibase/string-templates - -COPY scripts/removeWorkspaceDependencies.sh scripts/removeWorkspaceDependencies.sh -RUN chmod +x ./scripts/removeWorkspaceDependencies.sh -RUN ./scripts/removeWorkspaceDependencies.sh package.json - -RUN --mount=type=cache,target=/root/.yarn YARN_CACHE_FOLDER=/root/.yarn yarn install --production=true \ +COPY package.json . +COPY dist/yarn.lock . +RUN yarn install --production=true \ # Remove unneeded data from file system to reduce image size - && yarn cache clean && apt-get remove -y --purge --auto-remove g++ make python jq \ + && yarn cache clean && apt-get remove -y --purge --auto-remove g++ make python \ && rm -rf /tmp/* /root/.node-gyp /usr/local/lib/node_modules/npm/node_modules/node-gyp -COPY packages/server/dist/ dist/ -COPY packages/server/docker_run.sh . -COPY packages/server/builder/ builder/ -COPY packages/server/client/ client/ +COPY dist/ dist/ +COPY docker_run.sh . +COPY builder/ builder/ +COPY client/ client/ EXPOSE 4001 diff --git a/packages/server/Dockerfile.v2 b/packages/server/Dockerfile.v2 new file mode 100644 index 0000000000..d5a86b037d --- /dev/null +++ b/packages/server/Dockerfile.v2 @@ -0,0 +1,79 @@ +FROM node:18-slim + +LABEL com.centurylinklabs.watchtower.lifecycle.pre-check="scripts/watchtower-hooks/pre-check.sh" +LABEL com.centurylinklabs.watchtower.lifecycle.pre-update="scripts/watchtower-hooks/pre-update.sh" +LABEL com.centurylinklabs.watchtower.lifecycle.post-update="scripts/watchtower-hooks/post-update.sh" +LABEL com.centurylinklabs.watchtower.lifecycle.post-check="scripts/watchtower-hooks/post-check.sh" + +WORKDIR /app + +ENV PORT=4001 +ENV COUCH_DB_URL=https://couchdb.budi.live:5984 +ENV BUDIBASE_ENVIRONMENT=PRODUCTION +ENV SERVICE=app-service +ENV POSTHOG_TOKEN=phc_bIjZL7oh2GEUd2vqvTBH8WvrX0fWTFQMs6H5KQxiUxU +ENV TENANT_FEATURE_FLAGS=*:LICENSING,*:USER_GROUPS,*:ONBOARDING_TOUR +ENV ACCOUNT_PORTAL_URL=https://account.budibase.app +ENV TOP_LEVEL_PATH=/ + +# handle node-gyp +RUN apt-get update \ + && apt-get install -y --no-install-recommends g++ make python3 jq +RUN yarn global add pm2 + +# Install client for oracle datasource +RUN apt-get install unzip libaio1 +COPY packages/server/scripts/integrations/oracle/ scripts/integrations/oracle/ +RUN /bin/bash -e scripts/integrations/oracle/instantclient/linux/x86-64/install.sh + +# Install postgres client for pg_dump utils +RUN apt update && apt upgrade -y \ + && apt install software-properties-common apt-transport-https curl gpg -y \ + && curl -fsSl https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor | tee /usr/share/keyrings/postgresql.gpg > /dev/null \ + && echo deb [arch=amd64,arm64,ppc64el signed-by=/usr/share/keyrings/postgresql.gpg] http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main | tee /etc/apt/sources.list.d/postgresql.list \ + && apt update -y \ + && apt install postgresql-client-15 -y \ + && apt remove software-properties-common apt-transport-https curl gpg -y + +WORKDIR / + +COPY scripts/removeWorkspaceDependencies.sh scripts/removeWorkspaceDependencies.sh +RUN chmod +x ./scripts/removeWorkspaceDependencies.sh + + +WORKDIR /string-templates +COPY packages/string-templates/package.json package.json +RUN ../scripts/removeWorkspaceDependencies.sh package.json +RUN --mount=type=cache,target=/root/.yarn YARN_CACHE_FOLDER=/root/.yarn yarn install --production=true +COPY packages/string-templates . + + +WORKDIR /app +COPY packages/server/package.json . +COPY packages/server/dist/yarn.lock . +RUN cd ../string-templates && yarn link && cd - && yarn link @budibase/string-templates + +COPY scripts/removeWorkspaceDependencies.sh scripts/removeWorkspaceDependencies.sh +RUN chmod +x ./scripts/removeWorkspaceDependencies.sh +RUN ./scripts/removeWorkspaceDependencies.sh package.json + +RUN --mount=type=cache,target=/root/.yarn YARN_CACHE_FOLDER=/root/.yarn yarn install --production=true \ + # Remove unneeded data from file system to reduce image size + && yarn cache clean && apt-get remove -y --purge --auto-remove g++ make python jq \ + && rm -rf /tmp/* /root/.node-gyp /usr/local/lib/node_modules/npm/node_modules/node-gyp + +COPY packages/server/dist/ dist/ +COPY packages/server/docker_run.sh . +COPY packages/server/builder/ builder/ +COPY packages/server/client/ client/ + +EXPOSE 4001 + +# have to add node environment production after install +# due to this causing yarn to stop installing dev dependencies +# which are actually needed to get this environment up and running +ENV NODE_ENV=production +ENV CLUSTER_MODE=${CLUSTER_MODE} +ENV TOP_LEVEL_PATH=/app + +CMD ["./docker_run.sh"] diff --git a/packages/worker/Dockerfile b/packages/worker/Dockerfile index 0d60db6fc5..4230ee86f8 100644 --- a/packages/worker/Dockerfile +++ b/packages/worker/Dockerfile @@ -5,38 +5,22 @@ LABEL com.centurylinklabs.watchtower.lifecycle.pre-update="scripts/watchtower-ho LABEL com.centurylinklabs.watchtower.lifecycle.post-update="scripts/watchtower-hooks/post-update.sh" LABEL com.centurylinklabs.watchtower.lifecycle.post-check="scripts/watchtower-hooks/post-check.sh" +WORKDIR /app # handle node-gyp -RUN apk add --no-cache --virtual .gyp python3 make g++ jq +RUN apk add --no-cache --virtual .gyp python3 make g++ RUN yarn global add pm2 -WORKDIR / -COPY scripts/removeWorkspaceDependencies.sh scripts/removeWorkspaceDependencies.sh -RUN chmod +x ./scripts/removeWorkspaceDependencies.sh - - -WORKDIR /string-templates -COPY packages/string-templates/package.json package.json -RUN ../scripts/removeWorkspaceDependencies.sh package.json -RUN --mount=type=cache,target=/root/.yarn YARN_CACHE_FOLDER=/root/.yarn yarn install --production=true -COPY packages/string-templates . - - -WORKDIR /app -COPY packages/worker/package.json . -COPY packages/worker/dist/yarn.lock . -RUN cd ../string-templates && yarn link && cd - && yarn link @budibase/string-templates - -RUN ../scripts/removeWorkspaceDependencies.sh package.json - -RUN --mount=type=cache,target=/root/.yarn YARN_CACHE_FOLDER=/root/.yarn yarn install --production=true +COPY package.json . +COPY dist/yarn.lock . +RUN yarn install --production=true # Remove unneeded data from file system to reduce image size RUN apk del .gyp \ && yarn cache clean -COPY packages/worker/dist/ dist/ -COPY packages/worker/docker_run.sh . +COPY dist/ dist/ +COPY docker_run.sh . EXPOSE 4001 diff --git a/packages/worker/Dockerfile.v2 b/packages/worker/Dockerfile.v2 new file mode 100644 index 0000000000..0d60db6fc5 --- /dev/null +++ b/packages/worker/Dockerfile.v2 @@ -0,0 +1,53 @@ +FROM node:18-alpine + +LABEL com.centurylinklabs.watchtower.lifecycle.pre-check="scripts/watchtower-hooks/pre-check.sh" +LABEL com.centurylinklabs.watchtower.lifecycle.pre-update="scripts/watchtower-hooks/pre-update.sh" +LABEL com.centurylinklabs.watchtower.lifecycle.post-update="scripts/watchtower-hooks/post-update.sh" +LABEL com.centurylinklabs.watchtower.lifecycle.post-check="scripts/watchtower-hooks/post-check.sh" + + +# handle node-gyp +RUN apk add --no-cache --virtual .gyp python3 make g++ jq +RUN yarn global add pm2 + +WORKDIR / + +COPY scripts/removeWorkspaceDependencies.sh scripts/removeWorkspaceDependencies.sh +RUN chmod +x ./scripts/removeWorkspaceDependencies.sh + + +WORKDIR /string-templates +COPY packages/string-templates/package.json package.json +RUN ../scripts/removeWorkspaceDependencies.sh package.json +RUN --mount=type=cache,target=/root/.yarn YARN_CACHE_FOLDER=/root/.yarn yarn install --production=true +COPY packages/string-templates . + + +WORKDIR /app +COPY packages/worker/package.json . +COPY packages/worker/dist/yarn.lock . +RUN cd ../string-templates && yarn link && cd - && yarn link @budibase/string-templates + +RUN ../scripts/removeWorkspaceDependencies.sh package.json + +RUN --mount=type=cache,target=/root/.yarn YARN_CACHE_FOLDER=/root/.yarn yarn install --production=true +# Remove unneeded data from file system to reduce image size +RUN apk del .gyp \ + && yarn cache clean + +COPY packages/worker/dist/ dist/ +COPY packages/worker/docker_run.sh . + +EXPOSE 4001 + +# have to add node environment production after install +# due to this causing yarn to stop installing dev dependencies +# which are actually needed to get this environment up and running +ENV NODE_ENV=production +ENV CLUSTER_MODE=${CLUSTER_MODE} +ENV SERVICE=worker-service +ENV POSTHOG_TOKEN=phc_bIjZL7oh2GEUd2vqvTBH8WvrX0fWTFQMs6H5KQxiUxU +ENV TENANT_FEATURE_FLAGS=*:LICENSING,*:USER_GROUPS,*:ONBOARDING_TOUR +ENV ACCOUNT_PORTAL_URL=https://account.budibase.app + +CMD ["./docker_run.sh"]