Merge pull request #12106 from Budibase/BUDI-7641/remove-npm-from-apps-worker-docker

Remove npm dependency from apps and worker docker images
2023-10-20 12:30:39 +02:00 · 2023-10-20 12:30:39 +02:00 · 0176b8f31e
parent 525898a93c 806685014b
commit 0176b8f31e
12 changed files with 173 additions and 64 deletions
--- a/.github/workflows/budibase_ci.yml
+++ b/.github/workflows/budibase_ci.yml
@ -231,7 +231,7 @@ jobs:
          cache: "yarn"
      - run: yarn --frozen-lockfile
      - name: Build packages
-        run: yarn build --scope @budibase/server --scope @budibase/worker --scope @budibase/client --scope @budibase/backend-core
+        run: yarn build --scope @budibase/server --scope @budibase/worker
      - name: Run tests
        run: |
          cd qa-core
--- a/hosting/docker-compose.build.yaml
+++ b/hosting/docker-compose.build.yaml
@ -4,7 +4,9 @@ version: "3"

 services:
  app-service:
-    build: ../packages/server
+    build:
+      context: ..
+      dockerfile: packages/server/Dockerfile.v2
    container_name: build-bbapps
    environment:
      SELF_HOSTED: 1
@ -32,7 +34,9 @@ services:
  #      - /some/path/to/plugins:/plugins

  worker-service:
-    build: ../packages/worker
+    build:
+      context: ..
+      dockerfile: packages/worker/Dockerfile.v2
    container_name: build-bbworker
    environment:
      SELF_HOSTED: 1
--- a/hosting/single/Dockerfile.v2
+++ b/hosting/single/Dockerfile.v2
@ -19,7 +19,8 @@ COPY packages/string-templates/package.json packages/string-templates/package.js

 COPY scripts/removeWorkspaceDependencies.sh scripts/removeWorkspaceDependencies.sh
 RUN chmod +x ./scripts/removeWorkspaceDependencies.sh
-RUN ./scripts/removeWorkspaceDependencies.sh
+RUN ./scripts/removeWorkspaceDependencies.sh packages/server/package.json
+RUN ./scripts/removeWorkspaceDependencies.sh packages/worker/package.json


 # We will never want to sync pro, but the script is still required
--- a/package.json
+++ b/package.json
@ -46,7 +46,7 @@
    "dev:noserver": "yarn run kill-builder && lerna run --stream dev:stack:up && lerna run --stream dev:builder --ignore @budibase/backend-core --ignore @budibase/server --ignore @budibase/worker",
    "dev:server": "yarn run kill-server && lerna run --stream dev:builder --scope @budibase/worker --scope @budibase/server",
    "dev:built": "yarn run kill-all && cd packages/server && yarn dev:stack:up && cd ../../ && lerna run --stream dev:built",
-    "dev:docker": "yarn build && docker-compose -f hosting/docker-compose.build.yaml -f hosting/docker-compose.dev.yaml --env-file hosting/.env up --build --scale proxy-service=0",
+    "dev:docker": "yarn build --scope @budibase/server --scope @budibase/worker && docker-compose -f hosting/docker-compose.build.yaml -f hosting/docker-compose.dev.yaml --env-file hosting/.env up --build --scale proxy-service=0",
    "test": "lerna run --stream test --stream",
    "lint:eslint": "eslint packages qa-core --max-warnings=0",
    "lint:prettier": "prettier --check \"packages/**/*.{js,ts,svelte}\" && prettier --write \"examples/**/*.{js,ts,svelte}\" && prettier --check \"qa-core/**/*.{js,ts,svelte}\"",
--- a/packages/backend-core/package.json
+++ b/packages/backend-core/package.json
@ -35,7 +35,7 @@
    "dotenv": "16.0.1",
    "ioredis": "5.3.2",
    "joi": "17.6.0",
-    "jsonwebtoken": "9.0.0",
+    "jsonwebtoken": "9.0.2",
    "koa-passport": "4.1.4",
    "koa-pino-logger": "4.0.0",
    "lodash": "4.17.21",
@ -63,7 +63,7 @@
    "@types/chance": "1.1.3",
    "@types/cookies": "0.7.8",
    "@types/jest": "29.5.5",
-    "@types/lodash": "4.14.180",
+    "@types/lodash": "4.14.200",
    "@types/node": "18.17.0",
    "@types/node-fetch": "2.6.4",
    "@types/pouchdb": "6.4.0",
--- a/packages/server/Dockerfile.v2
+++ b/packages/server/Dockerfile.v2
@ -0,0 +1,79 @@
+FROM node:18-slim
+
+LABEL com.centurylinklabs.watchtower.lifecycle.pre-check="scripts/watchtower-hooks/pre-check.sh"
+LABEL com.centurylinklabs.watchtower.lifecycle.pre-update="scripts/watchtower-hooks/pre-update.sh"
+LABEL com.centurylinklabs.watchtower.lifecycle.post-update="scripts/watchtower-hooks/post-update.sh"
+LABEL com.centurylinklabs.watchtower.lifecycle.post-check="scripts/watchtower-hooks/post-check.sh"
+
+WORKDIR /app
+
+ENV PORT=4001
+ENV COUCH_DB_URL=https://couchdb.budi.live:5984
+ENV BUDIBASE_ENVIRONMENT=PRODUCTION
+ENV SERVICE=app-service
+ENV POSTHOG_TOKEN=phc_bIjZL7oh2GEUd2vqvTBH8WvrX0fWTFQMs6H5KQxiUxU
+ENV TENANT_FEATURE_FLAGS=*:LICENSING,*:USER_GROUPS,*:ONBOARDING_TOUR
+ENV ACCOUNT_PORTAL_URL=https://account.budibase.app
+ENV TOP_LEVEL_PATH=/
+
+# handle node-gyp
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends g++ make python3 jq
+RUN yarn global add pm2
+
+# Install client for oracle datasource
+RUN apt-get install unzip libaio1
+COPY packages/server/scripts/integrations/oracle/ scripts/integrations/oracle/
+RUN /bin/bash -e scripts/integrations/oracle/instantclient/linux/x86-64/install.sh 
+
+# Install postgres client for pg_dump utils
+RUN apt update && apt upgrade -y \
+    && apt install software-properties-common apt-transport-https curl gpg -y \
+    && curl -fsSl https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor | tee /usr/share/keyrings/postgresql.gpg > /dev/null \
+    && echo deb [arch=amd64,arm64,ppc64el signed-by=/usr/share/keyrings/postgresql.gpg] http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main | tee /etc/apt/sources.list.d/postgresql.list \
+    && apt update -y \
+    && apt install postgresql-client-15 -y \
+    && apt remove software-properties-common apt-transport-https curl gpg -y
+
+WORKDIR /
+
+COPY scripts/removeWorkspaceDependencies.sh scripts/removeWorkspaceDependencies.sh
+RUN chmod +x ./scripts/removeWorkspaceDependencies.sh
+
+
+WORKDIR /string-templates
+COPY packages/string-templates/package.json package.json
+RUN ../scripts/removeWorkspaceDependencies.sh package.json
+RUN --mount=type=cache,target=/root/.yarn YARN_CACHE_FOLDER=/root/.yarn yarn install --production=true
+COPY packages/string-templates .
+
+
+WORKDIR /app
+COPY packages/server/package.json .
+COPY packages/server/dist/yarn.lock .
+RUN cd ../string-templates && yarn link && cd - && yarn link @budibase/string-templates
+
+COPY scripts/removeWorkspaceDependencies.sh scripts/removeWorkspaceDependencies.sh
+RUN chmod +x ./scripts/removeWorkspaceDependencies.sh
+RUN ./scripts/removeWorkspaceDependencies.sh  package.json
+
+RUN --mount=type=cache,target=/root/.yarn YARN_CACHE_FOLDER=/root/.yarn yarn install --production=true \
+    # Remove unneeded data from file system to reduce image size
+    && yarn cache clean && apt-get remove -y --purge --auto-remove g++ make python jq \
+    && rm -rf /tmp/* /root/.node-gyp /usr/local/lib/node_modules/npm/node_modules/node-gyp
+
+COPY packages/server/dist/ dist/
+COPY packages/server/docker_run.sh .
+COPY packages/server/builder/ builder/
+COPY packages/server/client/ client/
+
+EXPOSE 4001
+
+# have to add node environment production after install
+# due to this causing yarn to stop installing dev dependencies
+# which are actually needed to get this environment up and running
+ENV NODE_ENV=production
+ENV CLUSTER_MODE=${CLUSTER_MODE}
+ENV TOP_LEVEL_PATH=/app
+
+CMD ["./docker_run.sh"]
--- a/packages/server/package.json
+++ b/packages/server/package.json
@ -122,7 +122,7 @@
    "@types/jest": "29.5.5",
    "@types/koa": "2.13.4",
    "@types/koa__router": "8.0.8",
-    "@types/lodash": "4.14.180",
+    "@types/lodash": "4.14.200",
    "@types/mssql": "8.1.2",
    "@types/node": "18.17.0",
    "@types/node-fetch": "2.6.4",
--- a/packages/string-templates/package.json
+++ b/packages/string-templates/package.json
@ -28,7 +28,7 @@
    "@budibase/handlebars-helpers": "^0.11.9",
    "dayjs": "^1.10.8",
    "handlebars": "^4.7.6",
-    "lodash": "^4.17.20",
+    "lodash": "4.17.21",
    "vm2": "^3.9.19"
  },
  "devDependencies": {
--- a/packages/worker/Dockerfile.v2
+++ b/packages/worker/Dockerfile.v2
@ -0,0 +1,53 @@
+FROM node:18-alpine
+
+LABEL com.centurylinklabs.watchtower.lifecycle.pre-check="scripts/watchtower-hooks/pre-check.sh"
+LABEL com.centurylinklabs.watchtower.lifecycle.pre-update="scripts/watchtower-hooks/pre-update.sh"
+LABEL com.centurylinklabs.watchtower.lifecycle.post-update="scripts/watchtower-hooks/post-update.sh"
+LABEL com.centurylinklabs.watchtower.lifecycle.post-check="scripts/watchtower-hooks/post-check.sh"
+
+
+# handle node-gyp
+RUN apk add --no-cache --virtual .gyp python3 make g++ jq
+RUN yarn global add pm2
+
+WORKDIR /
+
+COPY scripts/removeWorkspaceDependencies.sh scripts/removeWorkspaceDependencies.sh
+RUN chmod +x ./scripts/removeWorkspaceDependencies.sh
+
+
+WORKDIR /string-templates
+COPY packages/string-templates/package.json package.json
+RUN ../scripts/removeWorkspaceDependencies.sh package.json
+RUN --mount=type=cache,target=/root/.yarn YARN_CACHE_FOLDER=/root/.yarn yarn install --production=true
+COPY packages/string-templates .
+
+
+WORKDIR /app
+COPY packages/worker/package.json .
+COPY packages/worker/dist/yarn.lock .
+RUN cd ../string-templates && yarn link && cd - && yarn link @budibase/string-templates
+
+RUN ../scripts/removeWorkspaceDependencies.sh package.json
+
+RUN --mount=type=cache,target=/root/.yarn YARN_CACHE_FOLDER=/root/.yarn yarn install --production=true
+# Remove unneeded data from file system to reduce image size
+RUN apk del .gyp \
+    && yarn cache clean
+
+COPY packages/worker/dist/ dist/
+COPY packages/worker/docker_run.sh .
+
+EXPOSE 4001
+
+# have to add node environment production after install
+# due to this causing yarn to stop installing dev dependencies
+# which are actually needed to get this environment up and running
+ENV NODE_ENV=production
+ENV CLUSTER_MODE=${CLUSTER_MODE}
+ENV SERVICE=worker-service
+ENV POSTHOG_TOKEN=phc_bIjZL7oh2GEUd2vqvTBH8WvrX0fWTFQMs6H5KQxiUxU
+ENV TENANT_FEATURE_FLAGS=*:LICENSING,*:USER_GROUPS,*:ONBOARDING_TOUR
+ENV ACCOUNT_PORTAL_URL=https://account.budibase.app
+
+CMD ["./docker_run.sh"]
--- a/packages/worker/package.json
+++ b/packages/worker/package.json
@ -48,6 +48,7 @@
    "aws-sdk": "2.1030.0",
    "bcrypt": "5.1.0",
    "bcryptjs": "2.4.3",
+    "bull": "4.10.1",
    "dd-trace": "3.13.2",
    "dotenv": "8.6.0",
    "global-agent": "3.0.0",
@ -61,6 +62,7 @@
    "koa-session": "5.13.1",
    "koa-static": "5.0.0",
    "koa-useragent": "^4.1.0",
+    "lodash": "4.17.21",
    "node-fetch": "2.6.7",
    "nodemailer": "6.7.2",
    "passport-google-oauth": "2.0.0",
@ -77,14 +79,13 @@
    "@types/jsonwebtoken": "8.5.1",
    "@types/koa": "2.13.4",
    "@types/koa__router": "8.0.8",
-    "@types/lodash": "^4.14.191",
+    "@types/lodash": "4.14.200",
    "@types/node": "18.17.0",
    "@types/node-fetch": "2.6.4",
    "@types/server-destroy": "1.0.1",
    "@types/supertest": "2.0.12",
    "@types/uuid": "8.3.4",
    "jest": "29.6.2",
-    "lodash": "4.17.21",
    "nodemon": "2.0.15",
    "rimraf": "3.0.2",
    "supertest": "6.2.2",
--- a/scripts/removeWorkspaceDependencies.sh
+++ b/scripts/removeWorkspaceDependencies.sh
@ -1,31 +1,16 @@
-#!/bin/bash
+#!/bin/sh

-packages_to_remove=(
-  @budibase/backend-core
-  @budibase/bbui
-  @budibase/builder
-  @budibase/cli
-  @budibase/client
-  @budibase/frontend-core
-  @budibase/pro
-  @budibase/sdk
-  @budibase/server
-  @budibase/shared-core
-  # We cannot remove string-templates yet because it cannot be bundled by esbuild as a dependency
-  @budibase/string-templates
-  @budibase/types
-  @budibase/worker
-)
+packages_to_remove="@budibase/backend-core @budibase/bbui @budibase/builder @budibase/cli @budibase/client @budibase/frontend-core @budibase/pro @budibase/sdk @budibase/server @budibase/shared-core @budibase/string-templates @budibase/types @budibase/worker"

-
-root_package_json=$(cat "package.json")
+package_json_path="$1"
+package_json=$(cat "$package_json_path")

 process_package() {
-  local pkg="$1"
-  local package_json=$(cat "$pkg/package.json")
-  local has_changes=false
+  pkg_path="$1"
+  package_json=$(cat "$pkg_path")
+  has_changes=false

-  for package_name in "${packages_to_remove[@]}"; do
+  for package_name in $packages_to_remove; do
    if echo "$package_json" | jq -e --arg package_name "$package_name" '.dependencies | has($package_name)' > /dev/null; then
      package_json=$(echo "$package_json" | jq "del(.dependencies[\"$package_name\"])")
      has_changes=true
@ -33,20 +18,11 @@ process_package() {
  done

  if [ "$has_changes" = true ]; then
-    echo "$package_json" > "$1/package.json"
+    echo "$package_json" > "$pkg_path"
  fi
 }

+process_package "$package_json_path"

-for pkg in $(echo "$root_package_json" | jq -r '.workspaces.packages[]' ); do
-  if [[ "$pkg" == *"*"* ]]; then
-    # Use find to iterate through immediate subdirectories
-    find "$pkg" -maxdepth 1 -type d -print | while read -r workspace_package; do
-      process_package "$workspace_package"
-    done
-  else
-    process_package "$pkg"
-  fi
-done
-
-echo "$root_package_json" | jq "del(.resolutions)" > "package.json"
+package_json=$(cat "$package_json_path")
+echo "$package_json" | jq "del(.resolutions)" > "$1"
--- a/yarn.lock
+++ b/yarn.lock
@ -5595,15 +5595,10 @@
  resolved "https://registry.yarnpkg.com/@types/linkify-it/-/linkify-it-3.0.3.tgz#15a0712296c5041733c79efe233ba17ae5a7587b"
  integrity sha512-pTjcqY9E4nOI55Wgpz7eiI8+LzdYnw3qxXCfHyBDdPbYvbyLgWLJGh8EdPvqawwMK1Uo1794AUkkR38Fr0g+2g==

-"@types/lodash@4.14.180":
-  version "4.14.180"
-  resolved "https://registry.yarnpkg.com/@types/lodash/-/lodash-4.14.180.tgz#4ab7c9ddfc92ec4a887886483bc14c79fb380670"
-  integrity sha512-XOKXa1KIxtNXgASAnwj7cnttJxS4fksBRywK/9LzRV5YxrF80BXZIGeQSuoESQ/VkUj30Ae0+YcuHc15wJCB2g==
-
-"@types/lodash@^4.14.191":
-  version "4.14.192"
-  resolved "https://registry.yarnpkg.com/@types/lodash/-/lodash-4.14.192.tgz#5790406361a2852d332d41635d927f1600811285"
-  integrity sha512-km+Vyn3BYm5ytMO13k9KTp27O75rbQ0NFw+U//g+PX7VZyjCioXaRFisqSIJRECljcTv73G3i6BpglNGHgUQ5A==
+"@types/lodash@4.14.200":
+  version "4.14.200"
+  resolved "https://registry.yarnpkg.com/@types/lodash/-/lodash-4.14.200.tgz#435b6035c7eba9cdf1e039af8212c9e9281e7149"
+  integrity sha512-YI/M/4HRImtNf3pJgbF+W6FrXovqj+T+/HpENLTooK9PnkacBsDpeP3IpHab40CClUfhNmdM2WTNP2sa2dni5Q==

 "@types/long@^4.0.0", "@types/long@^4.0.1":
  version "4.0.2"
@ -13835,10 +13830,10 @@ jsonschema@1.4.0:
  resolved "https://registry.yarnpkg.com/jsonschema/-/jsonschema-1.4.0.tgz#1afa34c4bc22190d8e42271ec17ac8b3404f87b2"
  integrity sha512-/YgW6pRMr6M7C+4o8kS+B/2myEpHCrxO4PEWnqJNBFMjn7EWXqlQ4tGwL6xTHeRplwuZmcAncdvfOad1nT2yMw==

-jsonwebtoken@8.5.1:
-  version "8.5.1"
-  resolved "https://registry.yarnpkg.com/jsonwebtoken/-/jsonwebtoken-8.5.1.tgz#00e71e0b8df54c2121a1f26137df2280673bcc0d"
-  integrity sha512-XjwVfRS6jTMsqYs0EsuJ4LGxXV14zQybNd4L2r0UvbVnSF9Af8x7p5MzbJ90Ioz/9TI41/hTCvznF/loiSzn8w==
+jsonwebtoken@9.0.2:
+  version "9.0.2"
+  resolved "https://registry.yarnpkg.com/jsonwebtoken/-/jsonwebtoken-9.0.2.tgz#65ff91f4abef1784697d40952bb1998c504caaf3"
+  integrity sha512-PRp66vJ865SSqOlgqS8hujT5U4AOgMfhrwYIuIhfKaoSCZcirrmASQr8CX7cUg+RMih+hgznrjp99o+W4pJLHQ==
  dependencies:
    jws "^3.2.2"
    lodash.includes "^4.3.0"
@ -13849,9 +13844,9 @@ jsonwebtoken@8.5.1:
    lodash.isstring "^4.0.1"
    lodash.once "^4.0.0"
    ms "^2.1.1"
-    semver "^5.6.0"
+    semver "^7.5.4"

-jsonwebtoken@9.0.0, jsonwebtoken@^9.0.0:
+jsonwebtoken@^9.0.0:
  version "9.0.0"
  resolved "https://registry.yarnpkg.com/jsonwebtoken/-/jsonwebtoken-9.0.0.tgz#d0faf9ba1cc3a56255fe49c0961a67e520c1926d"
  integrity sha512-tuGfYXxkQGDPnLJ7SibiQgVgeDgfbPq2k2ICcbgqW8WxWLBAxKQM/ZCu/IT8SOSwmaYl4dpTFCW5xZv7YbbWUw==
@ -14799,7 +14794,7 @@ lodash.xor@^4.5.0:
  resolved "https://registry.yarnpkg.com/lodash.xor/-/lodash.xor-4.5.0.tgz#4d48ed7e98095b0632582ba714d3ff8ae8fb1db6"
  integrity sha512-sVN2zimthq7aZ5sPGXnSz32rZPuqcparVW50chJQe+mzTYV+IsxSsl/2gnkWWE2Of7K3myBQBqtLKOUEHJKRsQ==

-lodash@4.17.21, lodash@^4.17.11, lodash@^4.17.15, lodash@^4.17.19, lodash@^4.17.20, lodash@^4.17.21, lodash@^4.17.3, lodash@^4.7.0:
+lodash@4.17.21, lodash@^4.17.11, lodash@^4.17.15, lodash@^4.17.19, lodash@^4.17.21, lodash@^4.17.3, lodash@^4.7.0:
  version "4.17.21"
  resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.21.tgz#679591c564c3bffaae8454cf0b3df370c3d6911c"
  integrity sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==