Merge branch 'master' into grid-all-datasources

.github/workflows/budibase_ci.yml

@@ -231,7 +231,7 @@ jobs:
           cache: "yarn"
       - run: yarn --frozen-lockfile
       - name: Build packages
-        run: yarn build --scope @budibase/server --scope @budibase/worker --scope @budibase/client --scope @budibase/backend-core
+        run: yarn build --scope @budibase/server --scope @budibase/worker
       - name: Run tests
         run: |
           cd qa-core

hosting/docker-compose.build.yaml

@@ -4,7 +4,9 @@ version: "3"
 
 services:
   app-service:
-    build: ../packages/server
+    build:
+      context: ..
+      dockerfile: packages/server/Dockerfile.v2
     container_name: build-bbapps
     environment:
       SELF_HOSTED: 1
@@ -32,7 +34,9 @@ services:
   #      - /some/path/to/plugins:/plugins
 
   worker-service:
-    build: ../packages/worker
+    build:
+      context: ..
+      dockerfile: packages/worker/Dockerfile.v2
     container_name: build-bbworker
     environment:
       SELF_HOSTED: 1

hosting/single/Dockerfile.v2

@@ -19,13 +19,14 @@ COPY packages/string-templates/package.json packages/string-templates/package.js
 
 COPY scripts/removeWorkspaceDependencies.sh scripts/removeWorkspaceDependencies.sh
 RUN chmod +x ./scripts/removeWorkspaceDependencies.sh
-RUN ./scripts/removeWorkspaceDependencies.sh
+RUN ./scripts/removeWorkspaceDependencies.sh packages/server/package.json
+RUN ./scripts/removeWorkspaceDependencies.sh packages/worker/package.json
 
 
 # We will never want to sync pro, but the script is still required
 RUN echo '' > scripts/syncProPackage.js
 RUN jq 'del(.scripts.postinstall)' package.json > temp.json && mv temp.json package.json
-RUN --mount=type=cache,target=/root/.yarn YARN_CACHE_FOLDER=/root/.yarn  yarn install --production
+RUN --mount=type=cache,target=/root/.yarn YARN_CACHE_FOLDER=/root/.yarn yarn install --production
 
 # copy the actual code
 COPY packages/server/dist packages/server/dist

package.json

@@ -46,7 +46,7 @@
     "dev:noserver": "yarn run kill-builder && lerna run --stream dev:stack:up && lerna run --stream dev:builder --ignore @budibase/backend-core --ignore @budibase/server --ignore @budibase/worker",
     "dev:server": "yarn run kill-server && lerna run --stream dev:builder --scope @budibase/worker --scope @budibase/server",
     "dev:built": "yarn run kill-all && cd packages/server && yarn dev:stack:up && cd ../../ && lerna run --stream dev:built",
-    "dev:docker": "yarn build && docker-compose -f hosting/docker-compose.build.yaml -f hosting/docker-compose.dev.yaml --env-file hosting/.env up --build --scale proxy-service=0",
+    "dev:docker": "yarn build --scope @budibase/server --scope @budibase/worker && docker-compose -f hosting/docker-compose.build.yaml -f hosting/docker-compose.dev.yaml --env-file hosting/.env up --build --scale proxy-service=0",
     "test": "lerna run --stream test --stream",
     "lint:eslint": "eslint packages qa-core --max-warnings=0",
     "lint:prettier": "prettier --check \"packages/**/*.{js,ts,svelte}\" && prettier --write \"examples/**/*.{js,ts,svelte}\" && prettier --check \"qa-core/**/*.{js,ts,svelte}\"",

packages/backend-core/package.json

@@ -35,7 +35,7 @@
     "dotenv": "16.0.1",
     "ioredis": "5.3.2",
     "joi": "17.6.0",
-    "jsonwebtoken": "9.0.0",
+    "jsonwebtoken": "9.0.2",
     "koa-passport": "4.1.4",
     "koa-pino-logger": "4.0.0",
     "lodash": "4.17.21",
@@ -63,7 +63,7 @@
     "@types/chance": "1.1.3",
     "@types/cookies": "0.7.8",
     "@types/jest": "29.5.5",
-    "@types/lodash": "4.14.180",
+    "@types/lodash": "4.14.200",
     "@types/node": "18.17.0",
     "@types/node-fetch": "2.6.4",
     "@types/pouchdb": "6.4.0",

packages/server/Dockerfile.v2

@@ -0,0 +1,79 @@
+FROM node:18-slim
+
+LABEL com.centurylinklabs.watchtower.lifecycle.pre-check="scripts/watchtower-hooks/pre-check.sh"
+LABEL com.centurylinklabs.watchtower.lifecycle.pre-update="scripts/watchtower-hooks/pre-update.sh"
+LABEL com.centurylinklabs.watchtower.lifecycle.post-update="scripts/watchtower-hooks/post-update.sh"
+LABEL com.centurylinklabs.watchtower.lifecycle.post-check="scripts/watchtower-hooks/post-check.sh"
+
+WORKDIR /app
+
+ENV PORT=4001
+ENV COUCH_DB_URL=https://couchdb.budi.live:5984
+ENV BUDIBASE_ENVIRONMENT=PRODUCTION
+ENV SERVICE=app-service
+ENV POSTHOG_TOKEN=phc_bIjZL7oh2GEUd2vqvTBH8WvrX0fWTFQMs6H5KQxiUxU
+ENV TENANT_FEATURE_FLAGS=*:LICENSING,*:USER_GROUPS,*:ONBOARDING_TOUR
+ENV ACCOUNT_PORTAL_URL=https://account.budibase.app
+ENV TOP_LEVEL_PATH=/
+
+# handle node-gyp
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends g++ make python3 jq
+RUN yarn global add pm2
+
+# Install client for oracle datasource
+RUN apt-get install unzip libaio1
+COPY packages/server/scripts/integrations/oracle/ scripts/integrations/oracle/
+RUN /bin/bash -e scripts/integrations/oracle/instantclient/linux/x86-64/install.sh 
+
+# Install postgres client for pg_dump utils
+RUN apt update && apt upgrade -y \
+    && apt install software-properties-common apt-transport-https curl gpg -y \
+    && curl -fsSl https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor | tee /usr/share/keyrings/postgresql.gpg > /dev/null \
+    && echo deb [arch=amd64,arm64,ppc64el signed-by=/usr/share/keyrings/postgresql.gpg] http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main | tee /etc/apt/sources.list.d/postgresql.list \
+    && apt update -y \
+    && apt install postgresql-client-15 -y \
+    && apt remove software-properties-common apt-transport-https curl gpg -y
+
+WORKDIR /
+
+COPY scripts/removeWorkspaceDependencies.sh scripts/removeWorkspaceDependencies.sh
+RUN chmod +x ./scripts/removeWorkspaceDependencies.sh
+
+
+WORKDIR /string-templates
+COPY packages/string-templates/package.json package.json
+RUN ../scripts/removeWorkspaceDependencies.sh package.json
+RUN --mount=type=cache,target=/root/.yarn YARN_CACHE_FOLDER=/root/.yarn yarn install --production=true
+COPY packages/string-templates .
+
+
+WORKDIR /app
+COPY packages/server/package.json .
+COPY packages/server/dist/yarn.lock .
+RUN cd ../string-templates && yarn link && cd - && yarn link @budibase/string-templates
+
+COPY scripts/removeWorkspaceDependencies.sh scripts/removeWorkspaceDependencies.sh
+RUN chmod +x ./scripts/removeWorkspaceDependencies.sh
+RUN ./scripts/removeWorkspaceDependencies.sh  package.json
+
+RUN --mount=type=cache,target=/root/.yarn YARN_CACHE_FOLDER=/root/.yarn yarn install --production=true \
+    # Remove unneeded data from file system to reduce image size
+    && yarn cache clean && apt-get remove -y --purge --auto-remove g++ make python jq \
+    && rm -rf /tmp/* /root/.node-gyp /usr/local/lib/node_modules/npm/node_modules/node-gyp
+
+COPY packages/server/dist/ dist/
+COPY packages/server/docker_run.sh .
+COPY packages/server/builder/ builder/
+COPY packages/server/client/ client/
+
+EXPOSE 4001
+
+# have to add node environment production after install
+# due to this causing yarn to stop installing dev dependencies
+# which are actually needed to get this environment up and running
+ENV NODE_ENV=production
+ENV CLUSTER_MODE=${CLUSTER_MODE}
+ENV TOP_LEVEL_PATH=/app
+
+CMD ["./docker_run.sh"]

packages/server/package.json

@@ -122,7 +122,7 @@
     "@types/jest": "29.5.5",
     "@types/koa": "2.13.4",
     "@types/koa__router": "8.0.8",
-    "@types/lodash": "4.14.180",
+    "@types/lodash": "4.14.200",
     "@types/mssql": "8.1.2",
     "@types/node": "18.17.0",
     "@types/node-fetch": "2.6.4",

packages/string-templates/package.json

@@ -28,7 +28,7 @@
     "@budibase/handlebars-helpers": "^0.11.9",
     "dayjs": "^1.10.8",
     "handlebars": "^4.7.6",
-    "lodash": "^4.17.20",
+    "lodash": "4.17.21",
     "vm2": "^3.9.19"
   },
   "devDependencies": {

packages/worker/Dockerfile.v2

@@ -0,0 +1,53 @@
+FROM node:18-alpine
+
+LABEL com.centurylinklabs.watchtower.lifecycle.pre-check="scripts/watchtower-hooks/pre-check.sh"
+LABEL com.centurylinklabs.watchtower.lifecycle.pre-update="scripts/watchtower-hooks/pre-update.sh"
+LABEL com.centurylinklabs.watchtower.lifecycle.post-update="scripts/watchtower-hooks/post-update.sh"
+LABEL com.centurylinklabs.watchtower.lifecycle.post-check="scripts/watchtower-hooks/post-check.sh"
+
+
+# handle node-gyp
+RUN apk add --no-cache --virtual .gyp python3 make g++ jq
+RUN yarn global add pm2
+
+WORKDIR /
+
+COPY scripts/removeWorkspaceDependencies.sh scripts/removeWorkspaceDependencies.sh
+RUN chmod +x ./scripts/removeWorkspaceDependencies.sh
+
+
+WORKDIR /string-templates
+COPY packages/string-templates/package.json package.json
+RUN ../scripts/removeWorkspaceDependencies.sh package.json
+RUN --mount=type=cache,target=/root/.yarn YARN_CACHE_FOLDER=/root/.yarn yarn install --production=true
+COPY packages/string-templates .
+
+
+WORKDIR /app
+COPY packages/worker/package.json .
+COPY packages/worker/dist/yarn.lock .
+RUN cd ../string-templates && yarn link && cd - && yarn link @budibase/string-templates
+
+RUN ../scripts/removeWorkspaceDependencies.sh package.json
+
+RUN --mount=type=cache,target=/root/.yarn YARN_CACHE_FOLDER=/root/.yarn yarn install --production=true
+# Remove unneeded data from file system to reduce image size
+RUN apk del .gyp \
+    && yarn cache clean
+
+COPY packages/worker/dist/ dist/
+COPY packages/worker/docker_run.sh .
+
+EXPOSE 4001
+
+# have to add node environment production after install
+# due to this causing yarn to stop installing dev dependencies
+# which are actually needed to get this environment up and running
+ENV NODE_ENV=production
+ENV CLUSTER_MODE=${CLUSTER_MODE}
+ENV SERVICE=worker-service
+ENV POSTHOG_TOKEN=phc_bIjZL7oh2GEUd2vqvTBH8WvrX0fWTFQMs6H5KQxiUxU
+ENV TENANT_FEATURE_FLAGS=*:LICENSING,*:USER_GROUPS,*:ONBOARDING_TOUR
+ENV ACCOUNT_PORTAL_URL=https://account.budibase.app
+
+CMD ["./docker_run.sh"]

packages/worker/package.json

@@ -48,6 +48,7 @@
     "aws-sdk": "2.1030.0",
     "bcrypt": "5.1.0",
     "bcryptjs": "2.4.3",
+    "bull": "4.10.1",
     "dd-trace": "3.13.2",
     "dotenv": "8.6.0",
     "global-agent": "3.0.0",
@@ -61,6 +62,7 @@
     "koa-session": "5.13.1",
     "koa-static": "5.0.0",
     "koa-useragent": "^4.1.0",
+    "lodash": "4.17.21",
     "node-fetch": "2.6.7",
     "nodemailer": "6.7.2",
     "passport-google-oauth": "2.0.0",
@@ -77,14 +79,13 @@
     "@types/jsonwebtoken": "8.5.1",
     "@types/koa": "2.13.4",
     "@types/koa__router": "8.0.8",
-    "@types/lodash": "^4.14.191",
+    "@types/lodash": "4.14.200",
     "@types/node": "18.17.0",
     "@types/node-fetch": "2.6.4",
     "@types/server-destroy": "1.0.1",
     "@types/supertest": "2.0.12",
     "@types/uuid": "8.3.4",
     "jest": "29.6.2",
-    "lodash": "4.17.21",
     "nodemon": "2.0.15",
     "rimraf": "3.0.2",
     "supertest": "6.2.2",

scripts/removeWorkspaceDependencies.sh

@@ -1,31 +1,16 @@
-#!/bin/bash
+#!/bin/sh
 
-packages_to_remove=(
+packages_to_remove="@budibase/backend-core @budibase/bbui @budibase/builder @budibase/cli @budibase/client @budibase/frontend-core @budibase/pro @budibase/sdk @budibase/server @budibase/shared-core @budibase/string-templates @budibase/types @budibase/worker"
-  @budibase/backend-core
-  @budibase/bbui
-  @budibase/builder
-  @budibase/cli
-  @budibase/client
-  @budibase/frontend-core
-  @budibase/pro
-  @budibase/sdk
-  @budibase/server
-  @budibase/shared-core
-  # We cannot remove string-templates yet because it cannot be bundled by esbuild as a dependency
-  @budibase/string-templates
-  @budibase/types
-  @budibase/worker
-)
 
-
+package_json_path="$1"
-root_package_json=$(cat "package.json")
+package_json=$(cat "$package_json_path")
 
 process_package() {
-  local pkg="$1"
+  pkg_path="$1"
-  local package_json=$(cat "$pkg/package.json")
+  package_json=$(cat "$pkg_path")
-  local has_changes=false
+  has_changes=false
 
-  for package_name in "${packages_to_remove[@]}"; do
+  for package_name in $packages_to_remove; do
     if echo "$package_json" | jq -e --arg package_name "$package_name" '.dependencies | has($package_name)' > /dev/null; then
       package_json=$(echo "$package_json" | jq "del(.dependencies[\"$package_name\"])")
       has_changes=true
@@ -33,20 +18,11 @@ process_package() {
   done
 
   if [ "$has_changes" = true ]; then
-    echo "$package_json" > "$1/package.json"
+    echo "$package_json" > "$pkg_path"
   fi
 }
 
+process_package "$package_json_path"
 
-for pkg in $(echo "$root_package_json" | jq -r '.workspaces.packages[]' ); do
+package_json=$(cat "$package_json_path")
-  if [[ "$pkg" == *"*"* ]]; then
+echo "$package_json" | jq "del(.resolutions)" > "$1"
-    # Use find to iterate through immediate subdirectories
-    find "$pkg" -maxdepth 1 -type d -print | while read -r workspace_package; do
-      process_package "$workspace_package"
-    done
-  else
-    process_package "$pkg"
-  fi
-done
-
-echo "$root_package_json" | jq "del(.resolutions)" > "package.json"

yarn.lock

@@ -5595,15 +5595,10 @@
   resolved "https://registry.yarnpkg.com/@types/linkify-it/-/linkify-it-3.0.3.tgz#15a0712296c5041733c79efe233ba17ae5a7587b"
   integrity sha512-pTjcqY9E4nOI55Wgpz7eiI8+LzdYnw3qxXCfHyBDdPbYvbyLgWLJGh8EdPvqawwMK1Uo1794AUkkR38Fr0g+2g==
 
-"@types/lodash@4.14.180":
+"@types/lodash@4.14.200":
-  version "4.14.180"
+  version "4.14.200"
-  resolved "https://registry.yarnpkg.com/@types/lodash/-/lodash-4.14.180.tgz#4ab7c9ddfc92ec4a887886483bc14c79fb380670"
+  resolved "https://registry.yarnpkg.com/@types/lodash/-/lodash-4.14.200.tgz#435b6035c7eba9cdf1e039af8212c9e9281e7149"
-  integrity sha512-XOKXa1KIxtNXgASAnwj7cnttJxS4fksBRywK/9LzRV5YxrF80BXZIGeQSuoESQ/VkUj30Ae0+YcuHc15wJCB2g==
+  integrity sha512-YI/M/4HRImtNf3pJgbF+W6FrXovqj+T+/HpENLTooK9PnkacBsDpeP3IpHab40CClUfhNmdM2WTNP2sa2dni5Q==
-
-"@types/lodash@^4.14.191":
-  version "4.14.192"
-  resolved "https://registry.yarnpkg.com/@types/lodash/-/lodash-4.14.192.tgz#5790406361a2852d332d41635d927f1600811285"
-  integrity sha512-km+Vyn3BYm5ytMO13k9KTp27O75rbQ0NFw+U//g+PX7VZyjCioXaRFisqSIJRECljcTv73G3i6BpglNGHgUQ5A==
 
 "@types/long@^4.0.0", "@types/long@^4.0.1":
   version "4.0.2"
@@ -13835,10 +13830,10 @@ jsonschema@1.4.0:
   resolved "https://registry.yarnpkg.com/jsonschema/-/jsonschema-1.4.0.tgz#1afa34c4bc22190d8e42271ec17ac8b3404f87b2"
   integrity sha512-/YgW6pRMr6M7C+4o8kS+B/2myEpHCrxO4PEWnqJNBFMjn7EWXqlQ4tGwL6xTHeRplwuZmcAncdvfOad1nT2yMw==
 
-jsonwebtoken@8.5.1:
+jsonwebtoken@9.0.2:
-  version "8.5.1"
+  version "9.0.2"
-  resolved "https://registry.yarnpkg.com/jsonwebtoken/-/jsonwebtoken-8.5.1.tgz#00e71e0b8df54c2121a1f26137df2280673bcc0d"
+  resolved "https://registry.yarnpkg.com/jsonwebtoken/-/jsonwebtoken-9.0.2.tgz#65ff91f4abef1784697d40952bb1998c504caaf3"
-  integrity sha512-XjwVfRS6jTMsqYs0EsuJ4LGxXV14zQybNd4L2r0UvbVnSF9Af8x7p5MzbJ90Ioz/9TI41/hTCvznF/loiSzn8w==
+  integrity sha512-PRp66vJ865SSqOlgqS8hujT5U4AOgMfhrwYIuIhfKaoSCZcirrmASQr8CX7cUg+RMih+hgznrjp99o+W4pJLHQ==
   dependencies:
     jws "^3.2.2"
     lodash.includes "^4.3.0"
@@ -13849,9 +13844,9 @@ jsonwebtoken@8.5.1:
     lodash.isstring "^4.0.1"
     lodash.once "^4.0.0"
     ms "^2.1.1"
-    semver "^5.6.0"
+    semver "^7.5.4"
 
-jsonwebtoken@9.0.0, jsonwebtoken@^9.0.0:
+jsonwebtoken@^9.0.0:
   version "9.0.0"
   resolved "https://registry.yarnpkg.com/jsonwebtoken/-/jsonwebtoken-9.0.0.tgz#d0faf9ba1cc3a56255fe49c0961a67e520c1926d"
   integrity sha512-tuGfYXxkQGDPnLJ7SibiQgVgeDgfbPq2k2ICcbgqW8WxWLBAxKQM/ZCu/IT8SOSwmaYl4dpTFCW5xZv7YbbWUw==
@@ -14799,7 +14794,7 @@ lodash.xor@^4.5.0:
   resolved "https://registry.yarnpkg.com/lodash.xor/-/lodash.xor-4.5.0.tgz#4d48ed7e98095b0632582ba714d3ff8ae8fb1db6"
   integrity sha512-sVN2zimthq7aZ5sPGXnSz32rZPuqcparVW50chJQe+mzTYV+IsxSsl/2gnkWWE2Of7K3myBQBqtLKOUEHJKRsQ==
 
-lodash@4.17.21, lodash@^4.17.11, lodash@^4.17.15, lodash@^4.17.19, lodash@^4.17.20, lodash@^4.17.21, lodash@^4.17.3, lodash@^4.7.0:
+lodash@4.17.21, lodash@^4.17.11, lodash@^4.17.15, lodash@^4.17.19, lodash@^4.17.21, lodash@^4.17.3, lodash@^4.7.0:
   version "4.17.21"
   resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.21.tgz#679591c564c3bffaae8454cf0b3df370c3d6911c"
   integrity sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==