PR comments.

hosting/docker-compose.build.yaml

@@ -1,118 +0,0 @@
-version: "3"
-
-# optional ports are specified throughout for more advanced use cases.
-
-services:
-  app-service:
-    build: ../packages/server
-    container_name: build-bbapps
-    environment:
-      SELF_HOSTED: 1
-      COUCH_DB_URL: http://${COUCH_DB_USER}:${COUCH_DB_PASSWORD}@couchdb-service:5984
-      WORKER_URL: http://worker-service:4003
-      MINIO_URL: http://minio-service:9000
-      MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY}
-      MINIO_SECRET_KEY: ${MINIO_SECRET_KEY}
-      INTERNAL_API_KEY: ${INTERNAL_API_KEY}
-      BUDIBASE_ENVIRONMENT: ${BUDIBASE_ENVIRONMENT}
-      PORT: 4002
-      API_ENCRYPTION_KEY: ${API_ENCRYPTION_KEY}
-      JWT_SECRET: ${JWT_SECRET}
-      LOG_LEVEL: info
-      SENTRY_DSN: https://a34ae347621946bf8acded18e5b7d4b8@o420233.ingest.sentry.io/5338131
-      ENABLE_ANALYTICS: "true"
-      REDIS_URL: redis-service:6379
-      REDIS_PASSWORD: ${REDIS_PASSWORD}
-      BB_ADMIN_USER_EMAIL: ${BB_ADMIN_USER_EMAIL}
-      BB_ADMIN_USER_PASSWORD: ${BB_ADMIN_USER_PASSWORD}
-      PLUGINS_DIR: ${PLUGINS_DIR}
-    depends_on:
-      - worker-service
-      - redis-service
-  #    volumes:
-  #      - /some/path/to/plugins:/plugins
-
-  worker-service:
-    build: ../packages/worker
-    container_name: build-bbworker
-    environment:
-      SELF_HOSTED: 1
-      PORT: 4003
-      CLUSTER_PORT: ${MAIN_PORT}
-      API_ENCRYPTION_KEY: ${API_ENCRYPTION_KEY}
-      JWT_SECRET: ${JWT_SECRET}
-      MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY}
-      MINIO_SECRET_KEY: ${MINIO_SECRET_KEY}
-      MINIO_URL: http://minio-service:9000
-      APPS_URL: http://app-service:4002
-      COUCH_DB_USERNAME: ${COUCH_DB_USER}
-      COUCH_DB_PASSWORD: ${COUCH_DB_PASSWORD}
-      COUCH_DB_URL: http://${COUCH_DB_USER}:${COUCH_DB_PASSWORD}@couchdb-service:5984
-      SENTRY_DSN: https://a34ae347621946bf8acded18e5b7d4b8@o420233.ingest.sentry.io/5338131
-      INTERNAL_API_KEY: ${INTERNAL_API_KEY}
-      REDIS_URL: redis-service:6379
-      REDIS_PASSWORD: ${REDIS_PASSWORD}
-    depends_on:
-      - redis-service
-      - minio-service
-
-  minio-service:
-    image: minio/minio
-    container_name: build-minio
-    volumes:
-      - minio_data:/data
-    environment:
-      MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY}
-      MINIO_SECRET_KEY: ${MINIO_SECRET_KEY}
-      MINIO_BROWSER: "off"
-    command: server /data --console-address ":9001"
-    healthcheck:
-      test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
-      interval: 30s
-      timeout: 20s
-      retries: 3
-
-  proxy-service:
-    ports:
-      - "${MAIN_PORT}:10000"
-    container_name: build-bbproxy
-    image: budibase/proxy
-    environment:
-      - PROXY_RATE_LIMIT_WEBHOOKS_PER_SECOND=10
-      - PROXY_RATE_LIMIT_API_PER_SECOND=20
-      - APPS_UPSTREAM_URL=http://app-service:4002
-      - WORKER_UPSTREAM_URL=http://worker-service:4003
-      - MINIO_UPSTREAM_URL=http://minio-service:9000
-      - COUCHDB_UPSTREAM_URL=http://couchdb-service:5984
-      - WATCHTOWER_UPSTREAM_URL=http://watchtower-service:8080
-      - RESOLVER=127.0.0.11
-    depends_on:
-      - minio-service
-      - worker-service
-      - app-service
-      - couchdb-service
-
-  couchdb-service:
-    restart: unless-stopped
-    container_name: build-couch
-    image: budibase/couchdb
-    environment:
-      - COUCHDB_PASSWORD=${COUCH_DB_PASSWORD}
-      - COUCHDB_USER=${COUCH_DB_USER}
-    volumes:
-      - couchdb3_data:/opt/couchdb/data
-
-  redis-service:
-    container_name: build-redis
-    image: redis
-    command: redis-server --requirepass ${REDIS_PASSWORD}
-    volumes:
-      - redis_data:/data
-
-volumes:
-  couchdb3_data:
-    driver: local
-  minio_data:
-    driver: local
-  redis_data:
-    driver: local

hosting/docker-compose.dev.yaml

@@ -3,6 +3,59 @@ version: "3"
 # optional ports are specified throughout for more advanced use cases.
 
 services:
+  app-service:
+    build: ../packages/server
+    container_name: build-bbapps
+    profiles: ["services"]
+    environment:
+      SELF_HOSTED: 1
+      COUCH_DB_URL: http://${COUCH_DB_USER}:${COUCH_DB_PASSWORD}@couchdb-service:5984
+      WORKER_URL: http://worker-service:4003
+      MINIO_URL: http://minio-service:9000
+      MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY}
+      MINIO_SECRET_KEY: ${MINIO_SECRET_KEY}
+      INTERNAL_API_KEY: ${INTERNAL_API_KEY}
+      BUDIBASE_ENVIRONMENT: ${BUDIBASE_ENVIRONMENT}
+      PORT: 4002
+      API_ENCRYPTION_KEY: ${API_ENCRYPTION_KEY}
+      JWT_SECRET: ${JWT_SECRET}
+      LOG_LEVEL: info
+      SENTRY_DSN: https://a34ae347621946bf8acded18e5b7d4b8@o420233.ingest.sentry.io/5338131
+      ENABLE_ANALYTICS: "true"
+      REDIS_URL: redis-service:6379
+      REDIS_PASSWORD: ${REDIS_PASSWORD}
+      BB_ADMIN_USER_EMAIL: ${BB_ADMIN_USER_EMAIL}
+      BB_ADMIN_USER_PASSWORD: ${BB_ADMIN_USER_PASSWORD}
+      PLUGINS_DIR: ${PLUGINS_DIR}
+    depends_on:
+      - worker-service
+      - redis-service
+
+  worker-service:
+    build: ../packages/worker
+    container_name: build-bbworker
+    profiles: ["services"]
+    environment:
+      SELF_HOSTED: 1
+      PORT: 4003
+      CLUSTER_PORT: ${MAIN_PORT}
+      API_ENCRYPTION_KEY: ${API_ENCRYPTION_KEY}
+      JWT_SECRET: ${JWT_SECRET}
+      MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY}
+      MINIO_SECRET_KEY: ${MINIO_SECRET_KEY}
+      MINIO_URL: http://minio-service:9000
+      APPS_URL: http://app-service:4002
+      COUCH_DB_USERNAME: ${COUCH_DB_USER}
+      COUCH_DB_PASSWORD: ${COUCH_DB_PASSWORD}
+      COUCH_DB_URL: http://${COUCH_DB_USER}:${COUCH_DB_PASSWORD}@couchdb-service:5984
+      SENTRY_DSN: https://a34ae347621946bf8acded18e5b7d4b8@o420233.ingest.sentry.io/5338131
+      INTERNAL_API_KEY: ${INTERNAL_API_KEY}
+      REDIS_URL: redis-service:6379
+      REDIS_PASSWORD: ${REDIS_PASSWORD}
+    depends_on:
+      - redis-service
+      - minio-service
+
   minio-service:
     container_name: budi-minio-dev
     restart: on-failure

package.json

@@ -50,7 +50,7 @@
     "dev:noserver": "yarn run kill-builder && lerna link && lerna run --stream dev:stack:up && lerna run --stream --parallel dev:builder --concurrency 1 --ignore @budibase/backend-core --ignore @budibase/server --ignore @budibase/worker",
     "dev:server": "yarn run kill-server && lerna run --stream --parallel dev:builder --concurrency 1 --scope @budibase/worker --scope @budibase/server",
     "dev:built": "yarn run kill-all && cd packages/server && yarn dev:stack:up && cd ../../ && lerna run --stream --parallel dev:built",
-    "dev:docker": "yarn build && docker-compose -f hosting/docker-compose.build.yaml up --build",
+    "dev:docker": "yarn build && docker-compose -f hosting/docker-compose.dev.yaml up --build --profile services",
     "test": "lerna run --stream test --stream",
     "lint:eslint": "eslint packages && eslint qa-core",
     "lint:prettier": "prettier --check \"packages/**/*.{js,ts,svelte}\" && prettier --write \"examples/**/*.{js,ts,svelte}\" && prettier --check \"qa-core/**/*.{js,ts,svelte}\"",

packages/server/scripts/build.js

@@ -7,6 +7,24 @@ const dir = join(__dirname, "..")
 const entryPath = join(dir, "src")
 const outfilePath = join(dir, "dist")
 
+/**
+ * The reasoning for this is that now our built version is simple
+ * dist/index.js - any kind of threaded approach in Node.js requires
+ * a runner file to work from - I played around with a lot of
+ * different methods, but we really want to be able to use forks.
+ *
+ * Rather than trying to rewrite so that forks run the whole system,
+ * I instead went down a path of building the individual threads so
+ * that we have runner files for each of them e.g. dist/automations.js
+ * and dist/query.js - these can be ran totally independently and then
+ * the parent process can pass down data for processing to them.
+ *
+ * The ignoring is simply to remove the files which really don't need
+ * to be built - they could be built and it wouldn't cause any issues,
+ * but this just means if any further threads are added in future
+ * they will naturally work (rather than including, which would mean
+ * adjustments to the build files).
+ */
 const ignoredFiles = ["definitions", "index", "utils"]
 const threadNames = fs
   .readdirSync(join(dir, "src", "threads"))