diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml index 840d580892..1580be338e 100644 --- a/.github/workflows/budibase_ci.yml +++ b/.github/workflows/budibase_ci.yml @@ -231,7 +231,7 @@ jobs: cache: "yarn" - run: yarn --frozen-lockfile - name: Build packages - run: yarn build --scope @budibase/server --scope @budibase/worker --scope @budibase/client --scope @budibase/backend-core + run: yarn build --scope @budibase/server --scope @budibase/worker - name: Run tests run: | cd qa-core diff --git a/hosting/docker-compose.build.yaml b/hosting/docker-compose.build.yaml index bc363fb0bf..e192620b59 100644 --- a/hosting/docker-compose.build.yaml +++ b/hosting/docker-compose.build.yaml @@ -4,7 +4,9 @@ version: "3" services: app-service: - build: ../packages/server + build: + context: .. + dockerfile: packages/server/Dockerfile.v2 container_name: build-bbapps environment: SELF_HOSTED: 1 @@ -32,7 +34,9 @@ services: # - /some/path/to/plugins:/plugins worker-service: - build: ../packages/worker + build: + context: .. + dockerfile: packages/worker/Dockerfile.v2 container_name: build-bbworker environment: SELF_HOSTED: 1 diff --git a/hosting/single/Dockerfile.v2 b/hosting/single/Dockerfile.v2 index b1abe6d53e..ad11545a22 100644 --- a/hosting/single/Dockerfile.v2 +++ b/hosting/single/Dockerfile.v2 @@ -19,13 +19,14 @@ COPY packages/string-templates/package.json packages/string-templates/package.js COPY scripts/removeWorkspaceDependencies.sh scripts/removeWorkspaceDependencies.sh RUN chmod +x ./scripts/removeWorkspaceDependencies.sh -RUN ./scripts/removeWorkspaceDependencies.sh +RUN ./scripts/removeWorkspaceDependencies.sh packages/server/package.json +RUN ./scripts/removeWorkspaceDependencies.sh packages/worker/package.json # We will never want to sync pro, but the script is still required RUN echo '' > scripts/syncProPackage.js RUN jq 'del(.scripts.postinstall)' package.json > temp.json && mv temp.json package.json -RUN --mount=type=cache,target=/root/.yarn YARN_CACHE_FOLDER=/root/.yarn yarn install --production +RUN --mount=type=cache,target=/root/.yarn YARN_CACHE_FOLDER=/root/.yarn yarn install --production # copy the actual code COPY packages/server/dist packages/server/dist diff --git a/package.json b/package.json index 7f5c971009..100a306a35 100644 --- a/package.json +++ b/package.json @@ -46,7 +46,7 @@ "dev:noserver": "yarn run kill-builder && lerna run --stream dev:stack:up && lerna run --stream dev:builder --ignore @budibase/backend-core --ignore @budibase/server --ignore @budibase/worker", "dev:server": "yarn run kill-server && lerna run --stream dev:builder --scope @budibase/worker --scope @budibase/server", "dev:built": "yarn run kill-all && cd packages/server && yarn dev:stack:up && cd ../../ && lerna run --stream dev:built", - "dev:docker": "yarn build && docker-compose -f hosting/docker-compose.build.yaml -f hosting/docker-compose.dev.yaml --env-file hosting/.env up --build --scale proxy-service=0", + "dev:docker": "yarn build --scope @budibase/server --scope @budibase/worker && docker-compose -f hosting/docker-compose.build.yaml -f hosting/docker-compose.dev.yaml --env-file hosting/.env up --build --scale proxy-service=0", "test": "lerna run --stream test --stream", "lint:eslint": "eslint packages qa-core --max-warnings=0", "lint:prettier": "prettier --check \"packages/**/*.{js,ts,svelte}\" && prettier --write \"examples/**/*.{js,ts,svelte}\" && prettier --check \"qa-core/**/*.{js,ts,svelte}\"", diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json index 22ca5b21cc..b23cd8e5b1 100644 --- a/packages/backend-core/package.json +++ b/packages/backend-core/package.json @@ -35,7 +35,7 @@ "dotenv": "16.0.1", "ioredis": "5.3.2", "joi": "17.6.0", - "jsonwebtoken": "9.0.0", + "jsonwebtoken": "9.0.2", "koa-passport": "4.1.4", "koa-pino-logger": "4.0.0", "lodash": "4.17.21", @@ -63,7 +63,7 @@ "@types/chance": "1.1.3", "@types/cookies": "0.7.8", "@types/jest": "29.5.5", - "@types/lodash": "4.14.180", + "@types/lodash": "4.14.200", "@types/node": "18.17.0", "@types/node-fetch": "2.6.4", "@types/pouchdb": "6.4.0", diff --git a/packages/server/Dockerfile.v2 b/packages/server/Dockerfile.v2 new file mode 100644 index 0000000000..d5a86b037d --- /dev/null +++ b/packages/server/Dockerfile.v2 @@ -0,0 +1,79 @@ +FROM node:18-slim + +LABEL com.centurylinklabs.watchtower.lifecycle.pre-check="scripts/watchtower-hooks/pre-check.sh" +LABEL com.centurylinklabs.watchtower.lifecycle.pre-update="scripts/watchtower-hooks/pre-update.sh" +LABEL com.centurylinklabs.watchtower.lifecycle.post-update="scripts/watchtower-hooks/post-update.sh" +LABEL com.centurylinklabs.watchtower.lifecycle.post-check="scripts/watchtower-hooks/post-check.sh" + +WORKDIR /app + +ENV PORT=4001 +ENV COUCH_DB_URL=https://couchdb.budi.live:5984 +ENV BUDIBASE_ENVIRONMENT=PRODUCTION +ENV SERVICE=app-service +ENV POSTHOG_TOKEN=phc_bIjZL7oh2GEUd2vqvTBH8WvrX0fWTFQMs6H5KQxiUxU +ENV TENANT_FEATURE_FLAGS=*:LICENSING,*:USER_GROUPS,*:ONBOARDING_TOUR +ENV ACCOUNT_PORTAL_URL=https://account.budibase.app +ENV TOP_LEVEL_PATH=/ + +# handle node-gyp +RUN apt-get update \ + && apt-get install -y --no-install-recommends g++ make python3 jq +RUN yarn global add pm2 + +# Install client for oracle datasource +RUN apt-get install unzip libaio1 +COPY packages/server/scripts/integrations/oracle/ scripts/integrations/oracle/ +RUN /bin/bash -e scripts/integrations/oracle/instantclient/linux/x86-64/install.sh + +# Install postgres client for pg_dump utils +RUN apt update && apt upgrade -y \ + && apt install software-properties-common apt-transport-https curl gpg -y \ + && curl -fsSl https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor | tee /usr/share/keyrings/postgresql.gpg > /dev/null \ + && echo deb [arch=amd64,arm64,ppc64el signed-by=/usr/share/keyrings/postgresql.gpg] http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main | tee /etc/apt/sources.list.d/postgresql.list \ + && apt update -y \ + && apt install postgresql-client-15 -y \ + && apt remove software-properties-common apt-transport-https curl gpg -y + +WORKDIR / + +COPY scripts/removeWorkspaceDependencies.sh scripts/removeWorkspaceDependencies.sh +RUN chmod +x ./scripts/removeWorkspaceDependencies.sh + + +WORKDIR /string-templates +COPY packages/string-templates/package.json package.json +RUN ../scripts/removeWorkspaceDependencies.sh package.json +RUN --mount=type=cache,target=/root/.yarn YARN_CACHE_FOLDER=/root/.yarn yarn install --production=true +COPY packages/string-templates . + + +WORKDIR /app +COPY packages/server/package.json . +COPY packages/server/dist/yarn.lock . +RUN cd ../string-templates && yarn link && cd - && yarn link @budibase/string-templates + +COPY scripts/removeWorkspaceDependencies.sh scripts/removeWorkspaceDependencies.sh +RUN chmod +x ./scripts/removeWorkspaceDependencies.sh +RUN ./scripts/removeWorkspaceDependencies.sh package.json + +RUN --mount=type=cache,target=/root/.yarn YARN_CACHE_FOLDER=/root/.yarn yarn install --production=true \ + # Remove unneeded data from file system to reduce image size + && yarn cache clean && apt-get remove -y --purge --auto-remove g++ make python jq \ + && rm -rf /tmp/* /root/.node-gyp /usr/local/lib/node_modules/npm/node_modules/node-gyp + +COPY packages/server/dist/ dist/ +COPY packages/server/docker_run.sh . +COPY packages/server/builder/ builder/ +COPY packages/server/client/ client/ + +EXPOSE 4001 + +# have to add node environment production after install +# due to this causing yarn to stop installing dev dependencies +# which are actually needed to get this environment up and running +ENV NODE_ENV=production +ENV CLUSTER_MODE=${CLUSTER_MODE} +ENV TOP_LEVEL_PATH=/app + +CMD ["./docker_run.sh"] diff --git a/packages/server/package.json b/packages/server/package.json index 2e23e71b64..0bc3618a08 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -122,7 +122,7 @@ "@types/jest": "29.5.5", "@types/koa": "2.13.4", "@types/koa__router": "8.0.8", - "@types/lodash": "4.14.180", + "@types/lodash": "4.14.200", "@types/mssql": "8.1.2", "@types/node": "18.17.0", "@types/node-fetch": "2.6.4", diff --git a/packages/string-templates/package.json b/packages/string-templates/package.json index d9f56442a0..1e32e023f3 100644 --- a/packages/string-templates/package.json +++ b/packages/string-templates/package.json @@ -28,7 +28,7 @@ "@budibase/handlebars-helpers": "^0.11.9", "dayjs": "^1.10.8", "handlebars": "^4.7.6", - "lodash": "^4.17.20", + "lodash": "4.17.21", "vm2": "^3.9.19" }, "devDependencies": { diff --git a/packages/worker/Dockerfile.v2 b/packages/worker/Dockerfile.v2 new file mode 100644 index 0000000000..0d60db6fc5 --- /dev/null +++ b/packages/worker/Dockerfile.v2 @@ -0,0 +1,53 @@ +FROM node:18-alpine + +LABEL com.centurylinklabs.watchtower.lifecycle.pre-check="scripts/watchtower-hooks/pre-check.sh" +LABEL com.centurylinklabs.watchtower.lifecycle.pre-update="scripts/watchtower-hooks/pre-update.sh" +LABEL com.centurylinklabs.watchtower.lifecycle.post-update="scripts/watchtower-hooks/post-update.sh" +LABEL com.centurylinklabs.watchtower.lifecycle.post-check="scripts/watchtower-hooks/post-check.sh" + + +# handle node-gyp +RUN apk add --no-cache --virtual .gyp python3 make g++ jq +RUN yarn global add pm2 + +WORKDIR / + +COPY scripts/removeWorkspaceDependencies.sh scripts/removeWorkspaceDependencies.sh +RUN chmod +x ./scripts/removeWorkspaceDependencies.sh + + +WORKDIR /string-templates +COPY packages/string-templates/package.json package.json +RUN ../scripts/removeWorkspaceDependencies.sh package.json +RUN --mount=type=cache,target=/root/.yarn YARN_CACHE_FOLDER=/root/.yarn yarn install --production=true +COPY packages/string-templates . + + +WORKDIR /app +COPY packages/worker/package.json . +COPY packages/worker/dist/yarn.lock . +RUN cd ../string-templates && yarn link && cd - && yarn link @budibase/string-templates + +RUN ../scripts/removeWorkspaceDependencies.sh package.json + +RUN --mount=type=cache,target=/root/.yarn YARN_CACHE_FOLDER=/root/.yarn yarn install --production=true +# Remove unneeded data from file system to reduce image size +RUN apk del .gyp \ + && yarn cache clean + +COPY packages/worker/dist/ dist/ +COPY packages/worker/docker_run.sh . + +EXPOSE 4001 + +# have to add node environment production after install +# due to this causing yarn to stop installing dev dependencies +# which are actually needed to get this environment up and running +ENV NODE_ENV=production +ENV CLUSTER_MODE=${CLUSTER_MODE} +ENV SERVICE=worker-service +ENV POSTHOG_TOKEN=phc_bIjZL7oh2GEUd2vqvTBH8WvrX0fWTFQMs6H5KQxiUxU +ENV TENANT_FEATURE_FLAGS=*:LICENSING,*:USER_GROUPS,*:ONBOARDING_TOUR +ENV ACCOUNT_PORTAL_URL=https://account.budibase.app + +CMD ["./docker_run.sh"] diff --git a/packages/worker/package.json b/packages/worker/package.json index 882752215d..1eee3f020f 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -48,6 +48,7 @@ "aws-sdk": "2.1030.0", "bcrypt": "5.1.0", "bcryptjs": "2.4.3", + "bull": "4.10.1", "dd-trace": "3.13.2", "dotenv": "8.6.0", "global-agent": "3.0.0", @@ -61,6 +62,7 @@ "koa-session": "5.13.1", "koa-static": "5.0.0", "koa-useragent": "^4.1.0", + "lodash": "4.17.21", "node-fetch": "2.6.7", "nodemailer": "6.7.2", "passport-google-oauth": "2.0.0", @@ -77,14 +79,13 @@ "@types/jsonwebtoken": "8.5.1", "@types/koa": "2.13.4", "@types/koa__router": "8.0.8", - "@types/lodash": "^4.14.191", + "@types/lodash": "4.14.200", "@types/node": "18.17.0", "@types/node-fetch": "2.6.4", "@types/server-destroy": "1.0.1", "@types/supertest": "2.0.12", "@types/uuid": "8.3.4", "jest": "29.6.2", - "lodash": "4.17.21", "nodemon": "2.0.15", "rimraf": "3.0.2", "supertest": "6.2.2", diff --git a/scripts/removeWorkspaceDependencies.sh b/scripts/removeWorkspaceDependencies.sh index 627058c92a..ba27ae7edc 100755 --- a/scripts/removeWorkspaceDependencies.sh +++ b/scripts/removeWorkspaceDependencies.sh @@ -1,31 +1,16 @@ -#!/bin/bash +#!/bin/sh -packages_to_remove=( - @budibase/backend-core - @budibase/bbui - @budibase/builder - @budibase/cli - @budibase/client - @budibase/frontend-core - @budibase/pro - @budibase/sdk - @budibase/server - @budibase/shared-core - # We cannot remove string-templates yet because it cannot be bundled by esbuild as a dependency - @budibase/string-templates - @budibase/types - @budibase/worker -) +packages_to_remove="@budibase/backend-core @budibase/bbui @budibase/builder @budibase/cli @budibase/client @budibase/frontend-core @budibase/pro @budibase/sdk @budibase/server @budibase/shared-core @budibase/string-templates @budibase/types @budibase/worker" - -root_package_json=$(cat "package.json") +package_json_path="$1" +package_json=$(cat "$package_json_path") process_package() { - local pkg="$1" - local package_json=$(cat "$pkg/package.json") - local has_changes=false + pkg_path="$1" + package_json=$(cat "$pkg_path") + has_changes=false - for package_name in "${packages_to_remove[@]}"; do + for package_name in $packages_to_remove; do if echo "$package_json" | jq -e --arg package_name "$package_name" '.dependencies | has($package_name)' > /dev/null; then package_json=$(echo "$package_json" | jq "del(.dependencies[\"$package_name\"])") has_changes=true @@ -33,20 +18,11 @@ process_package() { done if [ "$has_changes" = true ]; then - echo "$package_json" > "$1/package.json" + echo "$package_json" > "$pkg_path" fi } +process_package "$package_json_path" -for pkg in $(echo "$root_package_json" | jq -r '.workspaces.packages[]' ); do - if [[ "$pkg" == *"*"* ]]; then - # Use find to iterate through immediate subdirectories - find "$pkg" -maxdepth 1 -type d -print | while read -r workspace_package; do - process_package "$workspace_package" - done - else - process_package "$pkg" - fi -done - -echo "$root_package_json" | jq "del(.resolutions)" > "package.json" \ No newline at end of file +package_json=$(cat "$package_json_path") +echo "$package_json" | jq "del(.resolutions)" > "$1" diff --git a/yarn.lock b/yarn.lock index a258f08a1d..e3629a8dbc 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5595,15 +5595,10 @@ resolved "https://registry.yarnpkg.com/@types/linkify-it/-/linkify-it-3.0.3.tgz#15a0712296c5041733c79efe233ba17ae5a7587b" integrity sha512-pTjcqY9E4nOI55Wgpz7eiI8+LzdYnw3qxXCfHyBDdPbYvbyLgWLJGh8EdPvqawwMK1Uo1794AUkkR38Fr0g+2g== -"@types/lodash@4.14.180": - version "4.14.180" - resolved "https://registry.yarnpkg.com/@types/lodash/-/lodash-4.14.180.tgz#4ab7c9ddfc92ec4a887886483bc14c79fb380670" - integrity sha512-XOKXa1KIxtNXgASAnwj7cnttJxS4fksBRywK/9LzRV5YxrF80BXZIGeQSuoESQ/VkUj30Ae0+YcuHc15wJCB2g== - -"@types/lodash@^4.14.191": - version "4.14.192" - resolved "https://registry.yarnpkg.com/@types/lodash/-/lodash-4.14.192.tgz#5790406361a2852d332d41635d927f1600811285" - integrity sha512-km+Vyn3BYm5ytMO13k9KTp27O75rbQ0NFw+U//g+PX7VZyjCioXaRFisqSIJRECljcTv73G3i6BpglNGHgUQ5A== +"@types/lodash@4.14.200": + version "4.14.200" + resolved "https://registry.yarnpkg.com/@types/lodash/-/lodash-4.14.200.tgz#435b6035c7eba9cdf1e039af8212c9e9281e7149" + integrity sha512-YI/M/4HRImtNf3pJgbF+W6FrXovqj+T+/HpENLTooK9PnkacBsDpeP3IpHab40CClUfhNmdM2WTNP2sa2dni5Q== "@types/long@^4.0.0", "@types/long@^4.0.1": version "4.0.2" @@ -13835,10 +13830,10 @@ jsonschema@1.4.0: resolved "https://registry.yarnpkg.com/jsonschema/-/jsonschema-1.4.0.tgz#1afa34c4bc22190d8e42271ec17ac8b3404f87b2" integrity sha512-/YgW6pRMr6M7C+4o8kS+B/2myEpHCrxO4PEWnqJNBFMjn7EWXqlQ4tGwL6xTHeRplwuZmcAncdvfOad1nT2yMw== -jsonwebtoken@8.5.1: - version "8.5.1" - resolved "https://registry.yarnpkg.com/jsonwebtoken/-/jsonwebtoken-8.5.1.tgz#00e71e0b8df54c2121a1f26137df2280673bcc0d" - integrity sha512-XjwVfRS6jTMsqYs0EsuJ4LGxXV14zQybNd4L2r0UvbVnSF9Af8x7p5MzbJ90Ioz/9TI41/hTCvznF/loiSzn8w== +jsonwebtoken@9.0.2: + version "9.0.2" + resolved "https://registry.yarnpkg.com/jsonwebtoken/-/jsonwebtoken-9.0.2.tgz#65ff91f4abef1784697d40952bb1998c504caaf3" + integrity sha512-PRp66vJ865SSqOlgqS8hujT5U4AOgMfhrwYIuIhfKaoSCZcirrmASQr8CX7cUg+RMih+hgznrjp99o+W4pJLHQ== dependencies: jws "^3.2.2" lodash.includes "^4.3.0" @@ -13849,9 +13844,9 @@ jsonwebtoken@8.5.1: lodash.isstring "^4.0.1" lodash.once "^4.0.0" ms "^2.1.1" - semver "^5.6.0" + semver "^7.5.4" -jsonwebtoken@9.0.0, jsonwebtoken@^9.0.0: +jsonwebtoken@^9.0.0: version "9.0.0" resolved "https://registry.yarnpkg.com/jsonwebtoken/-/jsonwebtoken-9.0.0.tgz#d0faf9ba1cc3a56255fe49c0961a67e520c1926d" integrity sha512-tuGfYXxkQGDPnLJ7SibiQgVgeDgfbPq2k2ICcbgqW8WxWLBAxKQM/ZCu/IT8SOSwmaYl4dpTFCW5xZv7YbbWUw== @@ -14799,7 +14794,7 @@ lodash.xor@^4.5.0: resolved "https://registry.yarnpkg.com/lodash.xor/-/lodash.xor-4.5.0.tgz#4d48ed7e98095b0632582ba714d3ff8ae8fb1db6" integrity sha512-sVN2zimthq7aZ5sPGXnSz32rZPuqcparVW50chJQe+mzTYV+IsxSsl/2gnkWWE2Of7K3myBQBqtLKOUEHJKRsQ== -lodash@4.17.21, lodash@^4.17.11, lodash@^4.17.15, lodash@^4.17.19, lodash@^4.17.20, lodash@^4.17.21, lodash@^4.17.3, lodash@^4.7.0: +lodash@4.17.21, lodash@^4.17.11, lodash@^4.17.15, lodash@^4.17.19, lodash@^4.17.21, lodash@^4.17.3, lodash@^4.7.0: version "4.17.21" resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.21.tgz#679591c564c3bffaae8454cf0b3df370c3d6911c" integrity sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==