From 148c7010e8a09ee357596aa10ed1d1992ccd48c1 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 16 Jun 2022 19:30:34 +0100 Subject: [PATCH 1/4] Updating the single image docker build to pin versions before build to allow it to work with develop etc. Also optimising the image a bit so that it can be built significantly faster. --- hosting/single/Dockerfile | 107 +++++++++++++++++--------------------- hosting/single/test.sh | 4 ++ scripts/pinVersions.js | 55 ++++++++++++-------- 3 files changed, 85 insertions(+), 81 deletions(-) create mode 100755 hosting/single/test.sh diff --git a/hosting/single/Dockerfile b/hosting/single/Dockerfile index 79d84f70c6..4490586906 100644 --- a/hosting/single/Dockerfile +++ b/hosting/single/Dockerfile @@ -1,79 +1,71 @@ -FROM couchdb +FROM couchdb:3.2.1 -ENV DEPLOYMENT_ENVIRONMENT=docker -ENV POSTHOG_TOKEN=phc_fg5I3nDOf6oJVMHSaycEhpPdlgS8rzXG2r6F2IpxCHS -ENV COUCHDB_PASSWORD=budibase -ENV COUCHDB_USER=budibase -ENV COUCH_DB_URL=http://budibase:budibase@localhost:5984 -ENV BUDIBASE_ENVIRONMENT=PRODUCTION -ENV MINIO_URL=http://localhost:9000 -ENV REDIS_URL=localhost:6379 -ENV WORKER_URL=http://localhost:4002 -ENV INTERNAL_API_KEY=budibase -ENV JWT_SECRET=testsecret -ENV MINIO_ACCESS_KEY=budibase -ENV MINIO_SECRET_KEY=budibase -ENV SELF_HOSTED=1 -ENV CLUSTER_PORT=10000 -ENV REDIS_PASSWORD=budibase -ENV ARCHITECTURE=amd -ENV APP_PORT=4001 -ENV WORKER_PORT=4002 +ENV DEPLOYMENT_ENVIRONMENT=docker \ + POSTHOG_TOKEN=phc_fg5I3nDOf6oJVMHSaycEhpPdlgS8rzXG2r6F2IpxCHS \ + COUCHDB_PASSWORD=budibase \ + COUCHDB_USER=budibase \ + COUCH_DB_URL=http://budibase:budibase@localhost:5984 \ + BUDIBASE_ENVIRONMENT=PRODUCTION \ + MINIO_URL=http://localhost:9000 \ + REDIS_URL=localhost:6379 \ + WORKER_URL=http://localhost:4002 \ + INTERNAL_API_KEY=budibase \ + JWT_SECRET=testsecret \ + MINIO_ACCESS_KEY=budibase \ + MINIO_SECRET_KEY=budibase \ + SELF_HOSTED=1 \ + CLUSTER_PORT=10000 \ + REDIS_PASSWORD=budibase \ + ARCHITECTURE=amd \ + APP_PORT=4001 \ + WORKER_PORT=4002 -RUN apt-get update -RUN apt-get install software-properties-common wget nginx -y -RUN apt-add-repository 'deb http://security.debian.org/debian-security stretch/updates main' -RUN apt-get update +# install base dependencies +RUN apt-get update && \ + apt-get install software-properties-common wget -y && \ + apt-add-repository 'deb http://security.debian.org/debian-security stretch/updates main' && \ + apt-get update + +# install other dependencies, nodejs, oracle requirements, jdk8, redis, nginx +WORKDIR /nodejs +RUN curl -sL https://deb.nodesource.com/setup_16.x -o /tmp/nodesource_setup.sh && \ + bash /tmp/nodesource_setup.sh && \ + apt-get install libaio1 nodejs nginx openjdk-8-jdk redis-server unzip -y && \ + npm install --global yarn pm2 # setup nginx ADD hosting/single/nginx.conf /etc/nginx -RUN mkdir /etc/nginx/logs -RUN useradd www -RUN touch /etc/nginx/logs/error.log -RUN touch /etc/nginx/logs/nginx.pid +RUN mkdir /etc/nginx/logs && \ + useradd www && \ + touch /etc/nginx/logs/error.log && \ + touch /etc/nginx/logs/nginx.pid -# install java -RUN apt-get install openjdk-8-jdk -y +# add pin script +WORKDIR / +ADD scripts/pinVersions.js . -# setup nodejs -WORKDIR /nodejs -RUN curl -sL https://deb.nodesource.com/setup_16.x -o /tmp/nodesource_setup.sh -RUN bash /tmp/nodesource_setup.sh -RUN apt-get install nodejs -RUN npm install --global yarn -RUN npm install --global pm2 - -# setup redis -RUN apt install redis-server -y - -# setup server +# setup server and install client for oracle datasource WORKDIR /app ADD packages/server . -RUN ls -al -RUN yarn -RUN yarn build -# Install client for oracle datasource -RUN apt-get install unzip libaio1 -RUN /bin/bash -e scripts/integrations/oracle/instantclient/linux/x86-64/install.sh +RUN node /pinVersions.js && yarn && yarn build && \ + /bin/bash -e scripts/integrations/oracle/instantclient/linux/x86-64/install.sh # setup worker WORKDIR /worker ADD packages/worker . -RUN yarn -RUN yarn build +RUN node /pinVersions.js && yarn && yarn build # setup clouseau WORKDIR / -RUN wget https://github.com/cloudant-labs/clouseau/releases/download/2.21.0/clouseau-2.21.0-dist.zip -RUN unzip clouseau-2.21.0-dist.zip -RUN mv clouseau-2.21.0 /opt/clouseau -RUN rm clouseau-2.21.0-dist.zip +RUN wget https://github.com/cloudant-labs/clouseau/releases/download/2.21.0/clouseau-2.21.0-dist.zip && \ + unzip clouseau-2.21.0-dist.zip && \ + mv clouseau-2.21.0 /opt/clouseau && \ + rm clouseau-2.21.0-dist.zip WORKDIR /opt/clouseau RUN mkdir ./bin ADD hosting/single/clouseau ./bin/ -ADD hosting/single/log4j.properties . -ADD hosting/single/clouseau.ini . +ADD hosting/single/log4j.properties hosting/single/clouseau.ini ./ RUN chmod +x ./bin/clouseau # setup CouchDB @@ -82,8 +74,7 @@ ADD hosting/single/vm.args ./etc/ # setup minio WORKDIR /minio -RUN wget https://dl.min.io/server/minio/release/linux-${ARCHITECTURE}64/minio -RUN chmod +x minio +RUN wget https://dl.min.io/server/minio/release/linux-${ARCHITECTURE}64/minio && chmod +x minio # setup runner file WORKDIR / diff --git a/hosting/single/test.sh b/hosting/single/test.sh new file mode 100755 index 0000000000..b9f9a4032c --- /dev/null +++ b/hosting/single/test.sh @@ -0,0 +1,4 @@ +#!/bin/bash +id=$(docker run -t -d -p 10000:10000 budibase:latest) +docker exec -it $id bash +docker kill $id diff --git a/scripts/pinVersions.js b/scripts/pinVersions.js index ecc605ad28..44cbba1090 100644 --- a/scripts/pinVersions.js +++ b/scripts/pinVersions.js @@ -3,35 +3,44 @@ const path = require("path") const MONOREPO_ROOT = "packages" -const packages = fs.readdirSync(MONOREPO_ROOT) +const packages = getPackages() + +function getPackages() { + if (fs.existsSync(MONOREPO_ROOT)) { + return fs.readdirSync(MONOREPO_ROOT).map(pkg => path.join(MONOREPO_ROOT, pkg)) + } else { + return ["./"] + } +} function pinDeps(dependencies) { - for (let dependency in dependencies) { - if (dependency.startsWith("@budibase")) { - dependencies[dependency] = dependencies[dependency].replace("^", "") - } - } + for (let dependency in dependencies) { + if (dependency.startsWith("@budibase")) { + dependencies[dependency] = dependencies[dependency].replace("^", "") + } + } + return dependencies } // iterate over the monorepo packages -for (let pkg of packages) { - const pkgPath = path.join(MONOREPO_ROOT, pkg) +for (let pkgPath of packages) { + // only directories + if (fs.statSync(pkgPath).isDirectory()) { + // get the package JSON file + const pkgJsonPath = path.join(pkgPath, "package.json") + if (!fs.existsSync(pkgJsonPath)) { + continue + } + const pkgJson = JSON.parse(fs.readFileSync(pkgJsonPath)) + - // only directories - if (fs.statSync(pkgPath).isDirectory()) { + // find any budibase dependencies, and pin them + pkgJson.dependencies = pinDeps(pkgJson.dependencies) + pkgJson.devDependencies = pinDeps(pkgJson.devDependencies) - - // get the package JSON file - const pkgJsonPath = path.join(pkgPath, "package.json") - const pkgJson = require(`../${pkgJsonPath}`) - - // find any budibase dependencies, and pin them - pinDeps(pkgJson.dependencies) - pinDeps(pkgJson.devDependencies) - - // update the package JSON files - fs.writeFileSync(pkgJsonPath, JSON.stringify(pkgJson, null, 2)) - } + // update the package JSON files + fs.writeFileSync(pkgJsonPath, JSON.stringify(pkgJson, null, 2)) + } } -console.log("Pinned dev versions for budibase packages successfully.") \ No newline at end of file +console.log("Pinned dev versions for budibase packages successfully.") From af60582dadd6574f1eef658fac3e4f3d724840b1 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 16 Jun 2022 20:38:24 +0100 Subject: [PATCH 2/4] Slimming build down from 4GB to 1.8GB for single image. --- hosting/single/Dockerfile | 44 +++++++++++++++++++++++++++------------ scripts/cleanup.sh | 11 ++++++++++ 2 files changed, 42 insertions(+), 13 deletions(-) create mode 100644 scripts/cleanup.sh diff --git a/hosting/single/Dockerfile b/hosting/single/Dockerfile index 4490586906..f0df9373c5 100644 --- a/hosting/single/Dockerfile +++ b/hosting/single/Dockerfile @@ -1,5 +1,28 @@ +FROM node:14-slim as build + +# install node-gyp dependencies +RUN apt-get update && apt-get install -y --no-install-recommends g++ make python + +# add pin script +WORKDIR / +ADD scripts/pinVersions.js scripts/cleanup.sh ./ +RUN chmod +x /cleanup.sh + +# build server +WORKDIR /app +ADD packages/server . +RUN node /pinVersions.js && yarn && yarn build && /cleanup.sh + +# build worker +WORKDIR /worker +ADD packages/worker . +RUN node /pinVersions.js && yarn && yarn build && /cleanup.sh + FROM couchdb:3.2.1 +COPY --from=build /app /app +COPY --from=build /worker /worker + ENV DEPLOYMENT_ENVIRONMENT=docker \ POSTHOG_TOKEN=phc_fg5I3nDOf6oJVMHSaycEhpPdlgS8rzXG2r6F2IpxCHS \ COUCHDB_PASSWORD=budibase \ @@ -40,20 +63,10 @@ RUN mkdir /etc/nginx/logs && \ touch /etc/nginx/logs/error.log && \ touch /etc/nginx/logs/nginx.pid -# add pin script WORKDIR / -ADD scripts/pinVersions.js . - -# setup server and install client for oracle datasource -WORKDIR /app -ADD packages/server . -RUN node /pinVersions.js && yarn && yarn build && \ - /bin/bash -e scripts/integrations/oracle/instantclient/linux/x86-64/install.sh - -# setup worker -WORKDIR /worker -ADD packages/worker . -RUN node /pinVersions.js && yarn && yarn build +RUN mkdir -p scripts/integrations/oracle +ADD packages/server/scripts/integrations/oracle scripts/integrations/oracle +RUN /bin/bash -e ./scripts/integrations/oracle/instantclient/linux/x86-64/install.sh # setup clouseau WORKDIR / @@ -81,10 +94,15 @@ WORKDIR / ADD hosting/single/runner.sh . RUN chmod +x ./runner.sh +# cleanup cache +RUN yarn cache clean -f + EXPOSE 10000 VOLUME /opt/couchdb/data VOLUME /minio # must set this just before running ENV NODE_ENV=production +WORKDIR / + CMD ["./runner.sh"] diff --git a/scripts/cleanup.sh b/scripts/cleanup.sh new file mode 100644 index 0000000000..d0f557a876 --- /dev/null +++ b/scripts/cleanup.sh @@ -0,0 +1,11 @@ +#!/bin/bash +dir=$(pwd) +mv dist / +mv package.json / +cd / +rm -r $dir +mkdir $dir +mv /dist $dir +mv /package.json $dir +cd $dir +NODE_ENV=production yarn From 81da3ab3fd83e0a4ee48e9be37639c202ae49b93 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 16 Jun 2022 20:49:50 +0100 Subject: [PATCH 3/4] Fixing a missing dependency. --- packages/backend-core/package.json | 2 +- packages/server/package.json | 1 + packages/worker/package.json | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json index daaa7580d3..4d14c37f6e 100644 --- a/packages/backend-core/package.json +++ b/packages/backend-core/package.json @@ -19,6 +19,7 @@ "test:watch": "jest --watchAll" }, "dependencies": { + "@budibase/types": "^1.0.200-alpha.3", "@techpass/passport-openidconnect": "0.3.2", "aws-sdk": "2.1030.0", "bcrypt": "5.0.1", @@ -56,7 +57,6 @@ ] }, "devDependencies": { - "@budibase/types": "^1.0.200-alpha.3", "@shopify/jest-koa-mocks": "3.1.5", "@types/jest": "27.5.1", "@types/koa": "2.0.52", diff --git a/packages/server/package.json b/packages/server/package.json index 73b9281df2..89d61b0837 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -81,6 +81,7 @@ "@budibase/client": "^1.0.200-alpha.3", "@budibase/pro": "1.0.200-alpha.3", "@budibase/string-templates": "^1.0.200-alpha.3", + "@budibase/types": "^1.0.200-alpha.3", "@bull-board/api": "3.7.0", "@bull-board/koa": "3.9.4", "@elastic/elasticsearch": "7.10.0", diff --git a/packages/worker/package.json b/packages/worker/package.json index cc4e39e441..dd230fa452 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -36,6 +36,7 @@ "dependencies": { "@budibase/backend-core": "^1.0.200-alpha.3", "@budibase/pro": "1.0.200-alpha.3", + "@budibase/types": "^1.0.200-alpha.3", "@budibase/string-templates": "^1.0.200-alpha.3", "@koa/router": "8.0.8", "@sentry/node": "6.17.7", From 82c2f1e05e3118459ee3f880a7e2eed39e90ff08 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 17 Jun 2022 14:12:04 +0100 Subject: [PATCH 4/4] Adding back dependencies. --- packages/backend-core/package.json | 2 +- packages/server/package.json | 1 + packages/server/yarn.lock | 18 +++++++++--------- packages/worker/package.json | 1 + packages/worker/yarn.lock | 18 +++++++++--------- 5 files changed, 21 insertions(+), 19 deletions(-) diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json index 04d50eff1a..03d03729ce 100644 --- a/packages/backend-core/package.json +++ b/packages/backend-core/package.json @@ -20,6 +20,7 @@ "test:watch": "jest --watchAll" }, "dependencies": { + "@budibase/types": "^1.0.206", "@techpass/passport-openidconnect": "0.3.2", "aws-sdk": "2.1030.0", "bcrypt": "5.0.1", @@ -57,7 +58,6 @@ ] }, "devDependencies": { - "@budibase/types": "^1.0.206", "@shopify/jest-koa-mocks": "3.1.5", "@types/jest": "27.5.1", "@types/koa": "2.0.52", diff --git a/packages/server/package.json b/packages/server/package.json index 20e4529352..eac2dc3d0c 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -81,6 +81,7 @@ "@budibase/client": "^1.0.206", "@budibase/pro": "1.0.206", "@budibase/string-templates": "^1.0.206", + "@budibase/types": "^1.0.206", "@bull-board/api": "3.7.0", "@bull-board/koa": "3.9.4", "@elastic/elasticsearch": "7.10.0", diff --git a/packages/server/yarn.lock b/packages/server/yarn.lock index b9f7f81a5b..c767ecec51 100644 --- a/packages/server/yarn.lock +++ b/packages/server/yarn.lock @@ -1094,10 +1094,10 @@ resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw== -"@budibase/backend-core@1.0.201-alpha.4": - version "1.0.201-alpha.4" - resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.201-alpha.4.tgz#51fda5838e9c51c88c85204ad867811e49b7d34c" - integrity sha512-5oQMfKPDMpB4x5MzGgOtWFIPzc7RG/uu+rSe65PVwWfp77bDa3kxtCtC1p6vtJuPlb4GYrHqGLtr7WePbJf4fA== +"@budibase/backend-core@1.0.206": + version "1.0.206" + resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.206.tgz#bf9f4153358e3b07c9df1dbc1cc8a100da880699" + integrity sha512-WjeWP58GumjW2sHqlsgbYYGi5nEm2oUjqo4UOX5QLKancVPfcKPB7DQal1K9il8vxiiT3ZVNG8XakUGjq2G7Uw== dependencies: "@techpass/passport-openidconnect" "0.3.2" aws-sdk "2.1030.0" @@ -1175,12 +1175,12 @@ svelte-flatpickr "^3.2.3" svelte-portal "^1.0.0" -"@budibase/pro@1.0.201-alpha.4": - version "1.0.201-alpha.4" - resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.201-alpha.4.tgz#5b92dc7a4bb2004bcd43339c863430ea640e0b74" - integrity sha512-fsDlE8O4Y4JpZI3NwBoN8udX4cEx7/ebZ41Q7qD5VwA6Q2niapRcX0MvDmGdjDSVECG+NJU1uqCdg5cDq1Odxw== +"@budibase/pro@1.0.206": + version "1.0.206" + resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.206.tgz#14343df74106d7bdc7f6e2a396305a06f6e446e5" + integrity sha512-SpXUX/xNaNRnDPemvQGe8bmH5yBK6u3TMG60KEBKHTou9OCsEw39x2qy2aHQkMhDFg8YTlfs42fVvhwKJlHbBw== dependencies: - "@budibase/backend-core" "1.0.201-alpha.4" + "@budibase/backend-core" "1.0.206" node-fetch "^2.6.1" "@budibase/standard-components@^0.9.139": diff --git a/packages/worker/package.json b/packages/worker/package.json index 3b8e0f7d6c..73a8b01af0 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -37,6 +37,7 @@ "@budibase/backend-core": "^1.0.206", "@budibase/pro": "1.0.206", "@budibase/string-templates": "^1.0.206", + "@budibase/types": "^1.0.206", "@koa/router": "8.0.8", "@sentry/node": "6.17.7", "@techpass/passport-openidconnect": "0.3.2", diff --git a/packages/worker/yarn.lock b/packages/worker/yarn.lock index 194ca3f061..5e23c4645b 100644 --- a/packages/worker/yarn.lock +++ b/packages/worker/yarn.lock @@ -291,10 +291,10 @@ resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw== -"@budibase/backend-core@1.0.201-alpha.4": - version "1.0.201-alpha.4" - resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.201-alpha.4.tgz#51fda5838e9c51c88c85204ad867811e49b7d34c" - integrity sha512-5oQMfKPDMpB4x5MzGgOtWFIPzc7RG/uu+rSe65PVwWfp77bDa3kxtCtC1p6vtJuPlb4GYrHqGLtr7WePbJf4fA== +"@budibase/backend-core@1.0.206": + version "1.0.206" + resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.206.tgz#bf9f4153358e3b07c9df1dbc1cc8a100da880699" + integrity sha512-WjeWP58GumjW2sHqlsgbYYGi5nEm2oUjqo4UOX5QLKancVPfcKPB7DQal1K9il8vxiiT3ZVNG8XakUGjq2G7Uw== dependencies: "@techpass/passport-openidconnect" "0.3.2" aws-sdk "2.1030.0" @@ -322,12 +322,12 @@ uuid "8.3.2" zlib "1.0.5" -"@budibase/pro@1.0.201-alpha.4": - version "1.0.201-alpha.4" - resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.201-alpha.4.tgz#5b92dc7a4bb2004bcd43339c863430ea640e0b74" - integrity sha512-fsDlE8O4Y4JpZI3NwBoN8udX4cEx7/ebZ41Q7qD5VwA6Q2niapRcX0MvDmGdjDSVECG+NJU1uqCdg5cDq1Odxw== +"@budibase/pro@1.0.206": + version "1.0.206" + resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.206.tgz#14343df74106d7bdc7f6e2a396305a06f6e446e5" + integrity sha512-SpXUX/xNaNRnDPemvQGe8bmH5yBK6u3TMG60KEBKHTou9OCsEw39x2qy2aHQkMhDFg8YTlfs42fVvhwKJlHbBw== dependencies: - "@budibase/backend-core" "1.0.201-alpha.4" + "@budibase/backend-core" "1.0.206" node-fetch "^2.6.1" "@cspotcode/source-map-consumer@0.8.0":