Merge branch 'develop' of github.com:Budibase/budibase into feature/automation-logs

hosting/single/Dockerfile

@@ -1,79 +1,84 @@
-FROM couchdb
+FROM node:14-slim as build
 
-ENV DEPLOYMENT_ENVIRONMENT=docker
+# install node-gyp dependencies
-ENV POSTHOG_TOKEN=phc_fg5I3nDOf6oJVMHSaycEhpPdlgS8rzXG2r6F2IpxCHS
+RUN apt-get update && apt-get install -y --no-install-recommends g++ make python
-ENV COUCHDB_PASSWORD=budibase
-ENV COUCHDB_USER=budibase
-ENV COUCH_DB_URL=http://budibase:budibase@localhost:5984
-ENV BUDIBASE_ENVIRONMENT=PRODUCTION
-ENV MINIO_URL=http://localhost:9000
-ENV REDIS_URL=localhost:6379
-ENV WORKER_URL=http://localhost:4002
-ENV INTERNAL_API_KEY=budibase
-ENV JWT_SECRET=testsecret
-ENV MINIO_ACCESS_KEY=budibase
-ENV MINIO_SECRET_KEY=budibase
-ENV SELF_HOSTED=1
-ENV CLUSTER_PORT=10000
-ENV REDIS_PASSWORD=budibase
-ENV ARCHITECTURE=amd
-ENV APP_PORT=4001
-ENV WORKER_PORT=4002
 
-RUN apt-get update
+# add pin script
-RUN apt-get install software-properties-common wget nginx -y
+WORKDIR /
-RUN apt-add-repository 'deb http://security.debian.org/debian-security stretch/updates main'
+ADD scripts/pinVersions.js scripts/cleanup.sh ./
-RUN apt-get update
+RUN chmod +x /cleanup.sh
+
+# build server
+WORKDIR /app
+ADD packages/server .
+RUN node /pinVersions.js && yarn && yarn build && /cleanup.sh
+
+# build worker
+WORKDIR /worker
+ADD packages/worker .
+RUN node /pinVersions.js && yarn && yarn build && /cleanup.sh
+
+FROM couchdb:3.2.1
+
+COPY --from=build /app /app
+COPY --from=build /worker /worker
+
+ENV DEPLOYMENT_ENVIRONMENT=docker \
+  POSTHOG_TOKEN=phc_fg5I3nDOf6oJVMHSaycEhpPdlgS8rzXG2r6F2IpxCHS \
+  COUCHDB_PASSWORD=budibase \
+  COUCHDB_USER=budibase \
+  COUCH_DB_URL=http://budibase:budibase@localhost:5984 \
+  BUDIBASE_ENVIRONMENT=PRODUCTION \
+  MINIO_URL=http://localhost:9000 \
+  REDIS_URL=localhost:6379 \
+  WORKER_URL=http://localhost:4002 \
+  INTERNAL_API_KEY=budibase \
+  JWT_SECRET=testsecret \
+  MINIO_ACCESS_KEY=budibase \
+  MINIO_SECRET_KEY=budibase \
+  SELF_HOSTED=1 \
+  CLUSTER_PORT=10000 \
+  REDIS_PASSWORD=budibase \
+  ARCHITECTURE=amd \
+  APP_PORT=4001 \
+  WORKER_PORT=4002
+
+# install base dependencies
+RUN apt-get update && \
+  apt-get install software-properties-common wget -y && \
+  apt-add-repository 'deb http://security.debian.org/debian-security stretch/updates main' && \
+  apt-get update
+
+# install other dependencies, nodejs, oracle requirements, jdk8, redis, nginx
+WORKDIR /nodejs
+RUN curl -sL https://deb.nodesource.com/setup_16.x -o /tmp/nodesource_setup.sh && \
+  bash /tmp/nodesource_setup.sh && \
+  apt-get install libaio1 nodejs nginx openjdk-8-jdk redis-server unzip -y && \
+  npm install --global yarn pm2
 
 # setup nginx
 ADD hosting/single/nginx.conf /etc/nginx
-RUN mkdir /etc/nginx/logs
+RUN mkdir /etc/nginx/logs && \
-RUN useradd www
+  useradd www && \
-RUN touch /etc/nginx/logs/error.log
+  touch /etc/nginx/logs/error.log && \
-RUN touch /etc/nginx/logs/nginx.pid
+  touch /etc/nginx/logs/nginx.pid
 
-# install java
+WORKDIR /
-RUN apt-get install openjdk-8-jdk -y
+RUN mkdir -p scripts/integrations/oracle
-
+ADD packages/server/scripts/integrations/oracle scripts/integrations/oracle
-# setup nodejs
+RUN /bin/bash -e ./scripts/integrations/oracle/instantclient/linux/x86-64/install.sh
-WORKDIR /nodejs
-RUN curl -sL https://deb.nodesource.com/setup_16.x -o /tmp/nodesource_setup.sh
-RUN bash /tmp/nodesource_setup.sh
-RUN apt-get install nodejs
-RUN npm install --global yarn
-RUN npm install --global pm2
-
-# setup redis
-RUN apt install redis-server -y
-
-# setup server
-WORKDIR /app
-ADD packages/server .
-RUN ls -al
-RUN yarn
-RUN yarn build
-# Install client for oracle datasource
-RUN apt-get install unzip libaio1
-RUN /bin/bash -e scripts/integrations/oracle/instantclient/linux/x86-64/install.sh
-
-# setup worker
-WORKDIR /worker
-ADD packages/worker .
-RUN yarn
-RUN yarn build
 
 # setup clouseau
 WORKDIR /
-RUN wget https://github.com/cloudant-labs/clouseau/releases/download/2.21.0/clouseau-2.21.0-dist.zip
+RUN wget https://github.com/cloudant-labs/clouseau/releases/download/2.21.0/clouseau-2.21.0-dist.zip && \
-RUN unzip clouseau-2.21.0-dist.zip
+  unzip clouseau-2.21.0-dist.zip && \
-RUN mv clouseau-2.21.0 /opt/clouseau
+  mv clouseau-2.21.0 /opt/clouseau && \
-RUN rm clouseau-2.21.0-dist.zip
+  rm clouseau-2.21.0-dist.zip
 
 WORKDIR /opt/clouseau
 RUN mkdir ./bin
 ADD hosting/single/clouseau ./bin/
-ADD hosting/single/log4j.properties .
+ADD hosting/single/log4j.properties hosting/single/clouseau.ini ./
-ADD hosting/single/clouseau.ini .
 RUN chmod +x ./bin/clouseau
 
 # setup CouchDB
@@ -82,18 +87,22 @@ ADD hosting/single/vm.args ./etc/
 
 # setup minio
 WORKDIR /minio
-RUN wget https://dl.min.io/server/minio/release/linux-${ARCHITECTURE}64/minio
+RUN wget https://dl.min.io/server/minio/release/linux-${ARCHITECTURE}64/minio && chmod +x minio
-RUN chmod +x minio
 
 # setup runner file
 WORKDIR /
 ADD hosting/single/runner.sh .
 RUN chmod +x ./runner.sh
 
+# cleanup cache
+RUN yarn cache clean -f
+
 EXPOSE 10000
 VOLUME /opt/couchdb/data
 VOLUME /minio
 
 # must set this just before running
 ENV NODE_ENV=production
+WORKDIR /
+
 CMD ["./runner.sh"]

hosting/single/test.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+id=$(docker run -t -d -p 10000:10000 budibase:latest)
+docker exec -it $id bash
+docker kill $id

packages/backend-core/package.json

@@ -20,6 +20,7 @@
     "test:watch": "jest --watchAll"
   },
   "dependencies": {
+    "@budibase/types": "^1.0.207-alpha.3",
     "@techpass/passport-openidconnect": "0.3.2",
     "aws-sdk": "2.1030.0",
     "bcrypt": "5.0.1",
@@ -57,7 +58,6 @@
     ]
   },
   "devDependencies": {
-    "@budibase/types": "^1.0.207-alpha.3",
     "@shopify/jest-koa-mocks": "3.1.5",
     "@types/jest": "27.5.1",
     "@types/koa": "2.0.52",

packages/server/package.json

@@ -77,6 +77,7 @@
   "license": "GPL-3.0",
   "dependencies": {
     "@apidevtools/swagger-parser": "10.0.3",
+    "@budibase/types": "^1.0.207-alpha.3",
     "@budibase/backend-core": "^1.0.207-alpha.3",
     "@budibase/client": "^1.0.207-alpha.3",
     "@budibase/pro": "1.0.207-alpha.3",

packages/worker/package.json

@@ -34,6 +34,7 @@
   "author": "Budibase",
   "license": "GPL-3.0",
   "dependencies": {
+    "@budibase/types": "^1.0.207-alpha.3",
     "@budibase/backend-core": "^1.0.207-alpha.3",
     "@budibase/pro": "1.0.207-alpha.3",
     "@budibase/string-templates": "^1.0.207-alpha.3",

scripts/cleanup.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+dir=$(pwd)
+mv dist /
+mv package.json /
+cd /
+rm -r $dir
+mkdir $dir
+mv /dist $dir
+mv /package.json $dir
+cd $dir
+NODE_ENV=production yarn

scripts/pinVersions.js

@@ -3,35 +3,44 @@ const path = require("path")
 
 const MONOREPO_ROOT = "packages"
 
-const packages = fs.readdirSync(MONOREPO_ROOT)
+const packages = getPackages()
+
+function getPackages() {
+  if (fs.existsSync(MONOREPO_ROOT)) {
+    return fs.readdirSync(MONOREPO_ROOT).map(pkg => path.join(MONOREPO_ROOT, pkg))
+  } else {
+    return ["./"]
+  }
+}
 
 function pinDeps(dependencies) {
-	for (let dependency in dependencies) {
+  for (let dependency in dependencies) {
-		if (dependency.startsWith("@budibase")) {
+    if (dependency.startsWith("@budibase")) {
-			dependencies[dependency] = dependencies[dependency].replace("^", "")
+      dependencies[dependency] = dependencies[dependency].replace("^", "")
-		}
+    }
-	}
+  }
+  return dependencies
 }
 
 // iterate over the monorepo packages
-for (let pkg of packages) {
+for (let pkgPath of packages) {
-	const pkgPath = path.join(MONOREPO_ROOT, pkg)
+  // only directories
-
+  if (fs.statSync(pkgPath).isDirectory()) {
-	// only directories
+    // get the package JSON file
-	if (fs.statSync(pkgPath).isDirectory()) {
+    const pkgJsonPath = path.join(pkgPath, "package.json")
+    if (!fs.existsSync(pkgJsonPath)) {
+      continue
+    }
+    const pkgJson = JSON.parse(fs.readFileSync(pkgJsonPath))
     
 
-		// get the package JSON file
+    // find any budibase dependencies, and pin them
-		const pkgJsonPath = path.join(pkgPath, "package.json")
+    pkgJson.dependencies = pinDeps(pkgJson.dependencies)
-		const pkgJson = require(`../${pkgJsonPath}`)
+    pkgJson.devDependencies = pinDeps(pkgJson.devDependencies)
 
-		// find any budibase dependencies, and pin them
+    // update the package JSON files
-		pinDeps(pkgJson.dependencies)
+    fs.writeFileSync(pkgJsonPath, JSON.stringify(pkgJson, null, 2))
-		pinDeps(pkgJson.devDependencies)
+  }
-
-		// update the package JSON files
-		fs.writeFileSync(pkgJsonPath, JSON.stringify(pkgJson, null, 2))
-	}
 }
 
 console.log("Pinned dev versions for budibase packages successfully.")