Merge pull request #9870 from Budibase/chore/esbuild

Chore - esbuild for server and builder

.github/workflows/budibase_ci.yml

@@ -44,9 +44,9 @@ jobs:
           node-version: 14.x
           cache: "yarn"
       - run: yarn
-      - run: yarn build
+      - run: yarn nx run-many -t=build --configuration=production
 
-  test:
+  test-libraries:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
@@ -59,8 +59,27 @@ jobs:
           node-version: 14.x
           cache: "yarn"
       - run: yarn
-      - run: yarn build --scope=@budibase/types --scope=@budibase/shared-core --scope=@budibase/string-templates
+      - run: yarn test --ignore=@budibase/worker --ignore=@budibase/server --ignore=@budibase/pro
-      - run: yarn test --ignore=@budibase/pro
+      - uses: codecov/codecov-action@v3
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }} # not required for public repos
+          name: codecov-umbrella
+          verbose: true
+
+  test-services:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          submodules: true
+          token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
+      - name: Use Node.js 14.x
+        uses: actions/setup-node@v3
+        with:
+          node-version: 14.x
+          cache: "yarn"
+      - run: yarn
+      - run: yarn test --scope=@budibase/worker --scope=@budibase/server
       - uses: codecov/codecov-action@v3
         with:
           token: ${{ secrets.CODECOV_TOKEN }} # not required for public repos

.github/workflows/release-develop.yml

@@ -51,7 +51,7 @@ jobs:
           node scripts/syncLocalDependencies.js $version
           echo "Syncing yarn workspace"
           yarn
-      - run: yarn build
+      - run: yarn build --configuration=production
       - run: yarn build:sdk
 
       - name: Publish budibase packages to NPM

.github/workflows/release-master.yml

@@ -57,7 +57,7 @@ jobs:
           echo "Syncing yarn workspace"
           yarn
       - run: yarn lint
-      - run: yarn build
+      - run: yarn build --configuration=production
       - run: yarn build:sdk
 
       - name: Publish budibase packages to NPM

hosting/docker-compose.build.yaml

@@ -0,0 +1,77 @@
+version: "3"
+
+# optional ports are specified throughout for more advanced use cases.
+
+services:
+  app-service:
+    build: ../packages/server
+    container_name: build-bbapps
+    environment:
+      SELF_HOSTED: 1
+      COUCH_DB_URL: http://${COUCH_DB_USER}:${COUCH_DB_PASSWORD}@couchdb-service:5984
+      WORKER_URL: http://worker-service:4003
+      MINIO_URL: http://minio-service:9000
+      MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY}
+      MINIO_SECRET_KEY: ${MINIO_SECRET_KEY}
+      INTERNAL_API_KEY: ${INTERNAL_API_KEY}
+      BUDIBASE_ENVIRONMENT: ${BUDIBASE_ENVIRONMENT}
+      PORT: 4002
+      API_ENCRYPTION_KEY: ${API_ENCRYPTION_KEY}
+      JWT_SECRET: ${JWT_SECRET}
+      LOG_LEVEL: info
+      SENTRY_DSN: https://a34ae347621946bf8acded18e5b7d4b8@o420233.ingest.sentry.io/5338131
+      ENABLE_ANALYTICS: "true"
+      REDIS_URL: redis-service:6379
+      REDIS_PASSWORD: ${REDIS_PASSWORD}
+      BB_ADMIN_USER_EMAIL: ${BB_ADMIN_USER_EMAIL}
+      BB_ADMIN_USER_PASSWORD: ${BB_ADMIN_USER_PASSWORD}
+      PLUGINS_DIR: ${PLUGINS_DIR}
+    depends_on:
+      - worker-service
+      - redis-service
+  #    volumes:
+  #      - /some/path/to/plugins:/plugins
+
+  worker-service:
+    build: ../packages/worker
+    container_name: build-bbworker
+    environment:
+      SELF_HOSTED: 1
+      PORT: 4003
+      CLUSTER_PORT: ${MAIN_PORT}
+      API_ENCRYPTION_KEY: ${API_ENCRYPTION_KEY}
+      JWT_SECRET: ${JWT_SECRET}
+      MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY}
+      MINIO_SECRET_KEY: ${MINIO_SECRET_KEY}
+      MINIO_URL: http://minio-service:9000
+      APPS_URL: http://app-service:4002
+      COUCH_DB_USERNAME: ${COUCH_DB_USER}
+      COUCH_DB_PASSWORD: ${COUCH_DB_PASSWORD}
+      COUCH_DB_URL: http://${COUCH_DB_USER}:${COUCH_DB_PASSWORD}@couchdb-service:5984
+      SENTRY_DSN: https://a34ae347621946bf8acded18e5b7d4b8@o420233.ingest.sentry.io/5338131
+      INTERNAL_API_KEY: ${INTERNAL_API_KEY}
+      REDIS_URL: redis-service:6379
+      REDIS_PASSWORD: ${REDIS_PASSWORD}
+    depends_on:
+      - redis-service
+      - minio-service
+
+  proxy-service-docker:
+    ports:
+      - "${MAIN_PORT}:10000"
+    container_name: build-bbproxy
+    image: budibase/proxy
+    environment:
+      - PROXY_RATE_LIMIT_WEBHOOKS_PER_SECOND=10
+      - PROXY_RATE_LIMIT_API_PER_SECOND=20
+      - APPS_UPSTREAM_URL=http://app-service:4002
+      - WORKER_UPSTREAM_URL=http://worker-service:4003
+      - MINIO_UPSTREAM_URL=http://minio-service:9000
+      - COUCHDB_UPSTREAM_URL=http://couchdb-service:5984
+      - WATCHTOWER_UPSTREAM_URL=http://watchtower-service:8080
+      - RESOLVER=127.0.0.11
+    depends_on:
+      - minio-service
+      - worker-service
+      - app-service
+      - couchdb-service

nx.json

@@ -6,5 +6,15 @@
         "cacheableOperations": ["build", "test"]
       }
     }
+  },
+  "targetDefaults": {
+    "dev:builder": {
+      "dependsOn": [
+        {
+          "projects": ["@budibase/string-templates"],
+          "target": "build"
+        }
+      ]
+    }
   }
 }

package.json

@@ -2,17 +2,23 @@
   "name": "root",
   "private": true,
   "devDependencies": {
+    "@esbuild-plugins/node-resolve": "^0.2.2",
+    "@nx/esbuild": "16.2.1",
+    "@nx/js": "16.2.1",
     "@rollup/plugin-json": "^4.0.2",
     "@typescript-eslint/parser": "5.45.0",
     "babel-eslint": "^10.0.3",
+    "esbuild": "^0.17.18",
     "eslint": "^7.28.0",
     "eslint-plugin-cypress": "^2.11.3",
     "eslint-plugin-svelte3": "^3.2.0",
     "husky": "^8.0.3",
     "js-yaml": "^4.1.0",
     "kill-port": "^1.6.1",
-    "lerna": "^6.6.1",
+    "lerna": "7.0.0-alpha.0",
     "madge": "^6.0.0",
+    "minimist": "^1.2.8",
+    "nx": "^16.2.1",
     "prettier": "^2.3.1",
     "prettier-plugin-svelte": "^2.3.0",
     "rimraf": "^3.0.2",
@@ -24,8 +30,8 @@
   "scripts": {
     "preinstall": "node scripts/syncProPackage.js",
     "setup": "git config submodule.recurse true && git submodule update && node ./hosting/scripts/setup.js && yarn && yarn bootstrap && yarn build && yarn dev",
-    "bootstrap": "./scripts/bootstrap.sh && lerna link && ./scripts/link-dependencies.sh",
+    "bootstrap": "./scripts/bootstrap.sh  && ./scripts/link-dependencies.sh",
-    "build": "lerna run --stream build",
+    "build": "yarn nx run-many -t=build",
     "build:dev": "lerna run --stream prebuild && yarn nx run-many --target=build --output-style=dynamic --watch --preserveWatchOutput",
     "backend:bootstrap": "./scripts/scopeBackend.sh && yarn run bootstrap",
     "backend:build": "./scripts/scopeBackend.sh 'lerna run --stream build'",
@@ -41,10 +47,11 @@
     "kill-builder": "kill-port 3000",
     "kill-server": "kill-port 4001 4002",
     "kill-all": "yarn run kill-builder && yarn run kill-server",
-    "dev": "yarn run kill-all && lerna link && lerna run --stream --parallel dev:builder --concurrency 1 --stream",
+    "dev": "yarn run kill-all  && lerna run --stream --parallel dev:builder --stream",
-    "dev:noserver": "yarn run kill-builder && lerna link && lerna run --stream dev:stack:up && lerna run --stream --parallel dev:builder --concurrency 1 --ignore @budibase/backend-core --ignore @budibase/server --ignore @budibase/worker",
+    "dev:noserver": "yarn run kill-builder  && lerna run --stream dev:stack:up && lerna run --stream --parallel dev:builder --ignore @budibase/backend-core --ignore @budibase/server --ignore @budibase/worker",
-    "dev:server": "yarn run kill-server && lerna run --stream --parallel dev:builder --concurrency 1 --scope @budibase/backend-core --scope @budibase/worker --scope @budibase/server",
+    "dev:server": "yarn run kill-server && lerna run --stream --parallel dev:builder --scope @budibase/worker --scope @budibase/server",
     "dev:built": "yarn run kill-all && cd packages/server && yarn dev:stack:up && cd ../../ && lerna run --stream --parallel dev:built",
+    "dev:docker": "yarn build && docker-compose -f hosting/docker-compose.dev.yaml -f hosting/docker-compose.build.yaml up --build --scale proxy-service=0 ",
     "test": "lerna run --stream test --stream",
     "lint:eslint": "eslint packages && eslint qa-core",
     "lint:prettier": "prettier --check \"packages/**/*.{js,ts,svelte}\" && prettier --write \"examples/**/*.{js,ts,svelte}\" && prettier --check \"qa-core/**/*.{js,ts,svelte}\"",
@@ -101,5 +108,6 @@
       "packages/worker",
       "packages/pro/packages/pro"
     ]
-  }
+  },
+  "dependencies": {}
 }

packages/backend-core/package.json

@@ -88,5 +88,19 @@
     "tsconfig-paths": "4.0.0",
     "typescript": "4.7.3"
   },
+  "nx": {
+    "targets": {
+      "build": {
+        "dependsOn": [
+          {
+            "projects": [
+              "@budibase/types"
+            ],
+            "target": "build"
+          }
+        ]
+      }
+    }
+  },
   "gitHead": "d1836a898cab3f8ab80ee6d8f42be1a9eed7dcdc"
 }

packages/bbui/package.json

@@ -90,5 +90,19 @@
   "resolutions": {
     "loader-utils": "1.4.1"
   },
+  "nx": {
+    "targets": {
+      "build": {
+        "dependsOn": [
+          {
+            "projects": [
+              "@budibase/string-templates"
+            ],
+            "target": "build"
+          }
+        ]
+      }
+    }
+  },
   "gitHead": "d1836a898cab3f8ab80ee6d8f42be1a9eed7dcdc"
 }

packages/bbui/src/ActionButton/ActionButton.svelte

@@ -102,7 +102,9 @@
     margin-left: 0;
     transition: color ease-out 130ms;
   }
-  .is-selected:not(.spectrum-ActionButton--emphasized):not(.spectrum-ActionButton--quiet) {
+  .is-selected:not(.spectrum-ActionButton--emphasized):not(
+      .spectrum-ActionButton--quiet
+    ) {
     background: var(--spectrum-global-color-gray-300);
     border-color: var(--spectrum-global-color-gray-500);
   }

packages/builder/package.json

@@ -9,7 +9,7 @@
     "dev:builder": "routify -c dev:vite",
     "dev:vite": "vite --host 0.0.0.0",
     "rollup": "rollup -c -w",
-    "test": "vitest"
+    "test": "vitest run"
   },
   "jest": {
     "globals": {
@@ -117,5 +117,31 @@
     "vite": "^3.0.8",
     "vitest": "^0.29.2"
   },
+  "nx": {
+    "targets": {
+      "build": {
+        "dependsOn": [
+          {
+            "projects": [
+              "@budibase/string-templates",
+              "@budibase/shared-core"
+            ],
+            "target": "build"
+          }
+        ]
+      },
+      "test": {
+        "dependsOn": [
+          {
+            "projects": [
+              "@budibase/shared-core",
+              "@budibase/string-templates"
+            ],
+            "target": "build"
+          }
+        ]
+      }
+    }
+  },
   "gitHead": "115189f72a850bfb52b65ec61d932531bf327072"
 }

packages/builder/src/components/common/CodeMirrorEditor.svelte

@@ -146,15 +146,18 @@
 
   /* Override default active line highlight colour in dark theme */
   div
-    :global(.CodeMirror-focused.cm-s-tomorrow-night-eighties
+    :global(
-      .CodeMirror-activeline-background) {
+      .CodeMirror-focused.cm-s-tomorrow-night-eighties
+        .CodeMirror-activeline-background
+    ) {
     background: rgba(255, 255, 255, 0.075);
   }
 
   /* Remove active line styling when not focused */
   div
-    :global(.CodeMirror:not(.CodeMirror-focused)
+    :global(
-      .CodeMirror-activeline-background) {
+      .CodeMirror:not(.CodeMirror-focused) .CodeMirror-activeline-background
+    ) {
     background: unset;
   }
 

packages/builder/src/pages/builder/portal/settings/auth/scim.svelte

@@ -115,27 +115,6 @@
     align-items: center;
   }
 
-  input[type="file"] {
-    display: none;
-  }
-  .sso-link-icon {
-    padding-top: 4px;
-    margin-left: 3px;
-  }
-  .sso-link {
-    margin-top: 12px;
-    display: flex;
-    flex-direction: row;
-    align-items: center;
-  }
-  .enforce-sso-title {
-    margin-right: 10px;
-  }
-  .enforce-sso-heading-container {
-    display: flex;
-    flex-direction: row;
-    align-items: start;
-  }
   .provider-title {
     display: flex;
     flex-direction: row;
@@ -143,9 +122,6 @@
     align-items: center;
     gap: var(--spacing-m);
   }
-  .provider-title span {
-    flex: 1 1 auto;
-  }
   .inputContainer {
     display: flex;
     flex-direction: row;

packages/cli/package.json

@@ -63,5 +63,19 @@
     "renamer": "^4.0.0",
     "ts-node": "^10.9.1",
     "typescript": "4.7.3"
+  },
+  "nx": {
+    "targets": {
+      "build": {
+        "dependsOn": [
+          {
+            "projects": [
+              "@budibase/backend-core"
+            ],
+            "target": "build"
+          }
+        ]
+      }
+    }
   }
 }

packages/client/package.json

@@ -65,5 +65,20 @@
   "resolutions": {
     "loader-utils": "1.4.1"
   },
+  "nx": {
+    "targets": {
+      "build": {
+        "dependsOn": [
+          {
+            "projects": [
+              "@budibase/string-templates",
+              "@budibase/shared-core"
+            ],
+            "target": "build"
+          }
+        ]
+      }
+    }
+  },
   "gitHead": "d1836a898cab3f8ab80ee6d8f42be1a9eed7dcdc"
 }

packages/client/src/components/app/charts/ApexChart.svelte

@@ -54,8 +54,9 @@
     color: white;
   }
   div
-    :global(.apexcharts-theme-dark
+    :global(
-      .apexcharts-tooltip-series-group.apexcharts-active) {
+      .apexcharts-theme-dark .apexcharts-tooltip-series-group.apexcharts-active
+    ) {
     padding-bottom: 0;
   }
 </style>

packages/client/src/components/app/forms/JSONField.svelte

@@ -72,9 +72,11 @@
   :global(.spectrum-Form-itemField .spectrum-Textfield--multiline) {
     min-height: calc(var(--height) - 24px);
   }
-  :global(.spectrum-Form--labelsAbove
+  :global(
+      .spectrum-Form--labelsAbove
         .spectrum-Form-itemField
-      .spectrum-Textfield--multiline) {
+        .spectrum-Textfield--multiline
+    ) {
     min-height: calc(var(--height) - 24px);
   }
 </style>

packages/pro

@@ -1 +1 @@
-Subproject commit a590dc237a16983b8f39dc8e65005b7736d23467
+Subproject commit aea8a4acb0bae6a1036520bf4c6d8cae428cc7d9

packages/server/.dockerignore

@@ -1,6 +1,7 @@
-node_modules
+*
-npm-debug.log
+!/dist/
-Dockerfile
+!/scripts/integrations/oracle/
-.dockerignore
+!/package.json
-.git
+!/docker_run.sh
-.gitignore
+!/builder/
+!/client/

packages/server/Dockerfile

@@ -15,22 +15,28 @@ ENV POSTHOG_TOKEN=phc_bIjZL7oh2GEUd2vqvTBH8WvrX0fWTFQMs6H5KQxiUxU
 ENV TENANT_FEATURE_FLAGS=*:LICENSING,*:USER_GROUPS,*:ONBOARDING_TOUR
 ENV ACCOUNT_PORTAL_URL=https://account.budibase.app
 
-# copy files and install dependencies
+
-COPY . ./
 # handle node-gyp
 RUN apt-get update \
-    && apt-get install -y --no-install-recommends g++ make python \
+    && apt-get install -y --no-install-recommends g++ make python
-    && yarn \
-    && yarn cache clean \
-    && apt-get remove -y --purge --auto-remove g++ make python \
-    && rm -rf /tmp/* /root/.node-gyp /usr/local/lib/node_modules/npm/node_modules/node-gyp
 RUN yarn global add pm2
-RUN yarn build
 
 # Install client for oracle datasource
 RUN apt-get install unzip libaio1
+COPY scripts/integrations/oracle/ scripts/integrations/oracle/
 RUN /bin/bash -e scripts/integrations/oracle/instantclient/linux/x86-64/install.sh 
 
+COPY package.json .
+RUN yarn 
+# Remove unneeded data from file system to reduce image size
+RUN yarn cache clean && apt-get remove -y --purge --auto-remove g++ make python \
+    && rm -rf /tmp/* /root/.node-gyp /usr/local/lib/node_modules/npm/node_modules/node-gyp
+
+COPY dist/ dist/
+COPY docker_run.sh .
+COPY builder/ builder/
+COPY client/ client/
+
 EXPOSE 4001
 
 # have to add node environment production after install
@@ -38,4 +44,6 @@ EXPOSE 4001
 # which are actually needed to get this environment up and running
 ENV NODE_ENV=production
 ENV CLUSTER_MODE=${CLUSTER_MODE}
+ENV TOP_LEVEL_PATH=/app
+
 CMD ["./docker_run.sh"]

packages/server/jest.config.ts

@@ -1,6 +1,7 @@
 import { Config } from "@jest/types"
 
 import * as fs from "fs"
+import { join } from "path"
 const preset = require("ts-jest/jest-preset")
 
 const baseConfig: Config.InitialProjectOptions = {
@@ -49,4 +50,6 @@ const config: Config.InitialOptions = {
   coverageReporters: ["lcov", "json", "clover"],
 }
 
+process.env.TOP_LEVEL_PATH = join(__dirname, "..", "..")
+
 export default config

packages/server/nodemon.json

@@ -6,5 +6,5 @@
     "src/**/*.spec.js",
     "../backend-core/dist/**/*"
   ],
-  "exec": "ts-node src/index.ts"
+  "exec": "node ../../scripts/build && node ./dist/index.js"
 }

packages/server/package.json

@@ -10,22 +10,22 @@
   },
   "scripts": {
     "prebuild": "rimraf dist/",
-    "build": "tsc -p tsconfig.build.json && mv dist/src/* dist/ && rimraf dist/src/",
+    "build": "node ./scripts/build.js",
+    "postbuild": "copyfiles -f ../client/dist/budibase-client.js ../client/manifest.json client",
     "build:dev": "yarn prebuild && tsc --build --watch --preserveWatchOutput",
     "debug": "yarn build && node --expose-gc --inspect=9222 dist/index.js",
-    "postbuild": "copyfiles -u 1 src/**/*.svelte dist/ && copyfiles -u 1 src/**/*.hbs dist/ && copyfiles -u 1 src/**/*.json dist/",
     "test": "bash scripts/test.sh",
     "test:memory": "jest --maxWorkers=2 --logHeapUsage --forceExit",
     "test:watch": "jest --watch",
-    "predocker": "copyfiles -f ../client/dist/budibase-client.js ../client/manifest.json client",
+    "predocker": "copyfiles -f ../client/dist/budibase-client.js ../client/manifest.json client && yarn build --configuration=production",
-    "build:docker": "yarn run predocker && docker build . -t app-service --label version=$BUDIBASE_RELEASE_VERSION",
+    "build:docker": "yarn predocker && docker build . -t app-service --label version=$BUDIBASE_RELEASE_VERSION",
     "build:docs": "node ./scripts/docs/generate.js open",
     "run:docker": "node dist/index.js",
     "run:docker:cluster": "pm2-runtime start pm2.config.js",
     "dev:stack:up": "node scripts/dev/manage.js up",
     "dev:stack:down": "node scripts/dev/manage.js down",
     "dev:stack:nuke": "node scripts/dev/manage.js nuke",
-    "dev:builder": "yarn run dev:stack:up && nodemon",
+    "dev:builder": "yarn run dev:stack:up && rimraf dist/ && nodemon",
     "dev:built": "yarn run dev:stack:up && yarn run run:docker",
     "specs": "ts-node specs/generate.ts && openapi-typescript specs/openapi.yaml --output src/definitions/openapi.ts",
     "initialise": "node scripts/initialise.js",
@@ -118,8 +118,8 @@
     "validate.js": "0.13.1",
     "vm2": "3.9.17",
     "worker-farm": "1.7.0",
-    "xml2js": "0.5.0",
+    "yargs": "13.2.4",
-    "yargs": "13.2.4"
+    "xml2js": "0.5.0"
   },
   "devDependencies": {
     "@babel/core": "7.17.4",
@@ -177,5 +177,20 @@
   "optionalDependencies": {
     "oracledb": "5.3.0"
   },
+  "nx": {
+    "targets": {
+      "test": {
+        "dependsOn": [
+          {
+            "projects": [
+              "@budibase/string-templates",
+              "@budibase/shared-core"
+            ],
+            "target": "build"
+          }
+        ]
+      }
+    }
+  },
   "gitHead": "d1836a898cab3f8ab80ee6d8f42be1a9eed7dcdc"
 }

packages/server/pm2.config.js

@@ -1,7 +1,7 @@
 module.exports = {
   apps: [
     {
-      script: "dist/index.js",
+      script: "./dist/index.js",
       instances: "max",
       exec_mode: "cluster",
     },

packages/server/scripts/build.js

@@ -0,0 +1,48 @@
+#!/usr/bin/node
+const { join } = require("path")
+const fs = require("fs")
+const coreBuild = require("../../../scripts/build")
+
+const dir = join(__dirname, "..")
+const entryPath = join(dir, "src")
+const outfilePath = join(dir, "dist")
+
+/**
+ * The reasoning for this is that now our built version is simple
+ * dist/index.js - any kind of threaded approach in Node.js requires
+ * a runner file to work from - I played around with a lot of
+ * different methods, but we really want to be able to use forks.
+ *
+ * Rather than trying to rewrite so that forks run the whole system,
+ * I instead went down a path of building the individual threads so
+ * that we have runner files for each of them e.g. dist/automations.js
+ * and dist/query.js - these can be ran totally independently and then
+ * the parent process can pass down data for processing to them.
+ *
+ * The ignoring is simply to remove the files which really don't need
+ * to be built - they could be built and it wouldn't cause any issues,
+ * but this just means if any further threads are added in future
+ * they will naturally work (rather than including, which would mean
+ * adjustments to the build files).
+ */
+const ignoredFiles = ["definitions", "index", "utils"]
+const threadNames = fs
+  .readdirSync(join(dir, "src", "threads"))
+  .filter(path => !ignoredFiles.find(file => path.includes(file)))
+  .map(path => path.replace(".ts", ""))
+const files = [
+  {
+    entry: join(entryPath, "index.ts"),
+    out: join(outfilePath, "index.js"),
+  },
+]
+for (let name of threadNames) {
+  files.push({
+    entry: join(entryPath, "threads", `${name}.ts`),
+    out: join(outfilePath, `${name}.js`),
+  })
+}
+
+for (let file of files) {
+  coreBuild(file.entry, file.out)
+}

packages/server/src/api/controllers/static/index.ts

@@ -134,7 +134,7 @@ export const serveApp = async function (ctx: any) {
             ? objectStore.getGlobalFileUrl("settings", "logoUrl")
             : "",
       })
-      const appHbs = loadHandlebarsFile(`${__dirname}/templates/app.hbs`)
+      const appHbs = loadHandlebarsFile(`${__dirname}/app.hbs`)
       ctx.body = await processString(appHbs, {
         head,
         body: html,
@@ -161,7 +161,7 @@ export const serveApp = async function (ctx: any) {
             : "",
       })
 
-      const appHbs = loadHandlebarsFile(`${__dirname}/templates/app.hbs`)
+      const appHbs = loadHandlebarsFile(`${__dirname}/app.hbs`)
       ctx.body = await processString(appHbs, {
         head,
         body: html,
@@ -177,7 +177,7 @@ export const serveBuilderPreview = async function (ctx: any) {
 
   if (!env.isJest()) {
     let appId = context.getAppId()
-    const previewHbs = loadHandlebarsFile(`${__dirname}/templates/preview.hbs`)
+    const previewHbs = loadHandlebarsFile(`${__dirname}/preview.hbs`)
     ctx.body = await processString(previewHbs, {
       clientLibPath: objectStore.clientLibraryUrl(appId!, appInfo.version),
     })

packages/server/src/api/routes/static.ts

@@ -5,6 +5,7 @@ import authorized from "../../middleware/authorized"
 import { permissions } from "@budibase/backend-core"
 import env from "../../environment"
 import { paramResource } from "../../middleware/resourceId"
+import { devClientLibPath } from "../../utilities/fileSystem"
 const { BUILDER, PermissionType, PermissionLevel } = permissions
 
 const router: Router = new Router()
@@ -17,7 +18,8 @@ router.param("file", async (file: any, ctx: any, next: any) => {
   }
   // test serves from require
   if (env.isTest()) {
-    ctx.devPath = require.resolve("@budibase/client").split(ctx.file)[0]
+    const path = devClientLibPath()
+    ctx.devPath = path.split(ctx.file)[0]
   } else if (env.isDev()) {
     // Serving the client library from your local dir in dev
     ctx.devPath = budibaseTempDir()

packages/server/src/constants/index.ts

@@ -1,4 +1,5 @@
 import { objectStore, roles, constants } from "@budibase/backend-core"
+import { FieldType as FieldTypes } from "@budibase/types"
 export { FieldType as FieldTypes, RelationshipTypes } from "@budibase/types"
 
 export enum FilterTypes {
@@ -24,14 +25,14 @@ export const NoEmptyFilterStrings = [
 ]
 
 export const CanSwitchTypes = [
-  [exports.FieldTypes.JSON, exports.FieldTypes.ARRAY],
+  [FieldTypes.JSON, FieldTypes.ARRAY],
   [
-    exports.FieldTypes.STRING,
+    FieldTypes.STRING,
-    exports.FieldTypes.OPTIONS,
+    FieldTypes.OPTIONS,
-    exports.FieldTypes.LONGFORM,
+    FieldTypes.LONGFORM,
-    exports.FieldTypes.BARCODEQR,
+    FieldTypes.BARCODEQR,
   ],
-  [exports.FieldTypes.BOOLEAN, exports.FieldTypes.NUMBER],
+  [FieldTypes.BOOLEAN, FieldTypes.NUMBER],
 ]
 
 export const SwitchableTypes = CanSwitchTypes.reduce((prev, current) =>
@@ -77,9 +78,9 @@ export const USERS_TABLE_SCHEMA = {
   // TODO: ADMIN PANEL - when implemented this doesn't need to be carried out
   schema: {
     email: {
-      type: exports.FieldTypes.STRING,
+      type: FieldTypes.STRING,
       constraints: {
-        type: exports.FieldTypes.STRING,
+        type: FieldTypes.STRING,
         email: true,
         length: {
           maximum: "",
@@ -92,27 +93,27 @@ export const USERS_TABLE_SCHEMA = {
     firstName: {
       name: "firstName",
       fieldName: "firstName",
-      type: exports.FieldTypes.STRING,
+      type: FieldTypes.STRING,
       constraints: {
-        type: exports.FieldTypes.STRING,
+        type: FieldTypes.STRING,
         presence: false,
       },
     },
     lastName: {
       name: "lastName",
       fieldName: "lastName",
-      type: exports.FieldTypes.STRING,
+      type: FieldTypes.STRING,
       constraints: {
-        type: exports.FieldTypes.STRING,
+        type: FieldTypes.STRING,
         presence: false,
       },
     },
     roleId: {
       fieldName: "roleId",
       name: "roleId",
-      type: exports.FieldTypes.OPTIONS,
+      type: FieldTypes.OPTIONS,
       constraints: {
-        type: exports.FieldTypes.STRING,
+        type: FieldTypes.STRING,
         presence: false,
         inclusion: Object.values(roles.BUILTIN_ROLE_IDS),
       },
@@ -120,9 +121,9 @@ export const USERS_TABLE_SCHEMA = {
     status: {
       fieldName: "status",
       name: "status",
-      type: exports.FieldTypes.OPTIONS,
+      type: FieldTypes.OPTIONS,
       constraints: {
-        type: exports.FieldTypes.STRING,
+        type: FieldTypes.STRING,
         presence: false,
         inclusion: Object.values(constants.UserStatus),
       },

packages/server/src/db/inMemoryView.ts

@@ -25,7 +25,9 @@ export async function runView(
       }))
     )
     let fn = (doc: Document, emit: any) => emit(doc._id)
-    eval("fn = " + view?.map?.replace("function (doc)", "function (doc, emit)"))
+    ;(0, eval)(
+      "fn = " + view?.map?.replace("function (doc)", "function (doc, emit)")
+    )
     const queryFns: any = {
       meta: view.meta,
       map: fn,

packages/server/src/environment.ts

@@ -96,6 +96,7 @@ const environment = {
   isInThread: () => {
     return process.env.FORKED_PROCESS
   },
+  TOP_LEVEL_PATH: process.env.TOP_LEVEL_PATH,
 }
 
 // threading can cause memory issues with node-ts in development

packages/server/src/integrations/base/sqlTable.ts

@@ -206,4 +206,3 @@ class SqlTableQueryBuilder {
 }
 
 export default SqlTableQueryBuilder
-module.exports = SqlTableQueryBuilder

packages/server/src/utilities/fileSystem/app.ts

@@ -35,7 +35,7 @@ export const getComponentLibraryManifest = async (library: string) => {
   const filename = "manifest.json"
 
   if (env.isDev() || env.isTest()) {
-    const path = join(TOP_LEVEL_PATH, "../client", filename)
+    const path = join(TOP_LEVEL_PATH, "packages/client", filename)
     // always load from new so that updates are refreshed
     delete require.cache[require.resolve(path)]
     return require(path)

packages/server/src/utilities/fileSystem/clientLibrary.ts

@@ -1,4 +1,4 @@
-import { join } from "path"
+import path, { join } from "path"
 import { ObjectStoreBuckets } from "../../constants"
 import fs from "fs"
 import { objectStore } from "@budibase/backend-core"
@@ -6,6 +6,10 @@ import { resolve } from "../centralPath"
 import env from "../../environment"
 import { TOP_LEVEL_PATH } from "./filesystem"
 
+export function devClientLibPath() {
+  return require.resolve("@budibase/client")
+}
+
 /**
  * Client library paths in the object store:
  * Previously, the entire client library package was downloaded from NPM
@@ -89,9 +93,10 @@ export async function updateClientLibrary(appId: string) {
   let manifest, client
 
   if (env.isDev()) {
+    const clientPath = devClientLibPath()
     // Load the symlinked version in dev which is always the newest
-    manifest = require.resolve("@budibase/client/manifest.json")
+    manifest = join(path.dirname(path.dirname(clientPath)), "manifest.json")
-    client = require.resolve("@budibase/client")
+    client = clientPath
   } else {
     // Load the bundled version in prod
     manifest = resolve(TOP_LEVEL_PATH, "client", "manifest.json")

packages/server/src/utilities/fileSystem/filesystem.ts

@@ -4,9 +4,11 @@ import { budibaseTempDir } from "../budibaseDir"
 import { join } from "path"
 import env from "../../environment"
 import tar from "tar"
+import environment from "../../environment"
 const uuid = require("uuid/v4")
 
-export const TOP_LEVEL_PATH = join(__dirname, "..", "..", "..")
+export const TOP_LEVEL_PATH =
+  environment.TOP_LEVEL_PATH || join(__dirname, "..", "..", "..")
 
 /**
  * Upon first startup of instance there may not be everything we need in tmp directory, set it up.

packages/server/tsconfig.build.json

@@ -10,7 +10,15 @@
     "incremental": true,
     "types": ["node", "jest"],
     "outDir": "dist/src",
-    "skipLibCheck": true
+    "skipLibCheck": true,
+    "baseUrl": ".",
+    "paths": {
+      "@budibase/types": ["../types/src"],
+      "@budibase/backend-core": ["../backend-core/src"],
+      "@budibase/backend-core/*": ["../backend-core/*"],
+      "@budibase/shared-core": ["../shared-core/src"],
+      "@budibase/pro": ["../pro/packages/pro/src"]
+    }
   },
   "include": ["src/**/*"],
   "exclude": [

packages/server/tsconfig.json

@@ -18,12 +18,6 @@
     "require": ["tsconfig-paths/register"],
     "swc": true
   },
-  "references": [
-    { "path": "../types" },
-    { "path": "../backend-core" },
-    { "path": "../shared-core" },
-    { "path": "../../../budibase-pro/packages/pro" }
-  ],
   "include": ["src/**/*", "specs"],
   "exclude": ["node_modules", "dist"]
 }

packages/shared-core/package.json

@@ -26,5 +26,19 @@
     "concurrently": "^7.6.0",
     "rimraf": "3.0.2",
     "typescript": "4.7.3"
+  },
+  "nx": {
+    "targets": {
+      "build": {
+        "dependsOn": [
+          {
+            "projects": [
+              "@budibase/types"
+            ],
+            "target": "build"
+          }
+        ]
+      }
+    }
   }
 }

packages/shared-core/src/filters.ts

@@ -328,8 +328,8 @@ export const runLuceneQuery = (docs: any[], query?: Query) => {
       return (
         docValue == null ||
         docValue === "" ||
-        docValue < testValue.low ||
+        +docValue < testValue.low ||
-        docValue > testValue.high
+        +docValue > testValue.high
       )
     }
   )

packages/string-templates/tsconfig.json

@@ -1,12 +1,11 @@
 {
   "include": ["src/**/*"],
-
   "compilerOptions": {
     "allowJs": true,
     "declaration": true,
     "emitDeclarationOnly": true,
     "outDir": "dist",
     "esModuleInterop": true,
-    "types": ["node"]
+    "types": ["node", "jest"]
   }
 }

packages/worker/.dockerignore

@@ -1,7 +1,3 @@
-node_modules
+*
-npm-debug.log
+!/dist/
-Dockerfile
+!/docker_run.sh
-.dockerignore
-.git
-.gitignore
-

packages/worker/Dockerfile

@@ -7,13 +7,20 @@ LABEL com.centurylinklabs.watchtower.lifecycle.post-check="scripts/watchtower-ho
 
 WORKDIR /app
 
-# copy files and install dependencies
-COPY . ./
 # handle node-gyp
-RUN apk add --no-cache --virtual .gyp python3 make g++ \
+RUN apk add --no-cache --virtual .gyp python3 make g++
-    && yarn && apk del .gyp
 RUN yarn global add pm2
 
+
+COPY dist/package.json .
+RUN yarn
+# Remove unneeded data from file system to reduce image size
+RUN apk del .gyp \
+    && yarn cache clean
+
+COPY dist/ dist/
+COPY docker_run.sh .
+
 EXPOSE 4001
 
 # have to add node environment production after install

packages/worker/nodemon.json

@@ -6,5 +6,5 @@
     "src/**/*.spec.js",
     "../backend-core/dist/**/*"
   ],
-  "exec": "ts-node src/index.ts"
+  "exec": "yarn build && node dist/index.js"
 }

packages/worker/package.json

@@ -13,15 +13,15 @@
   ],
   "scripts": {
     "prebuild": "rimraf dist/",
-    "build": "tsc -p tsconfig.build.json",
+    "build": "cd ../.. && nx build @budibase/worker",
-    "postbuild": "copyfiles -u 1 src/**/*.hbs dist/",
     "build:dev": "yarn prebuild && tsc --build --watch --preserveWatchOutput",
     "run:docker": "node dist/index.js",
     "debug": "yarn build && node --expose-gc --inspect=9223 dist/index.js",
     "run:docker:cluster": "pm2-runtime start pm2.config.js",
-    "build:docker": "docker build . -t worker-service --label version=$BUDIBASE_RELEASE_VERSION",
+    "predocker": "yarn build --configuration=production",
+    "build:docker": "yarn predocker && docker build . -t worker-service --label version=$BUDIBASE_RELEASE_VERSION",
     "dev:stack:init": "node ./scripts/dev/manage.js init",
-    "dev:builder": "npm run dev:stack:init && nodemon",
+    "dev:builder": "npm run dev:stack:init && rimraf dist/ && nodemon",
     "dev:built": "yarn run dev:stack:init && yarn run run:docker",
     "test": "bash scripts/test.sh",
     "test:watch": "jest --watch",

packages/worker/project.json

@@ -0,0 +1,47 @@
+{
+  "name": "@budibase/worker",
+  "$schema": "../../node_modules/nx/schemas/project-schema.json",
+  "targets": {
+    "build": {
+      "executor": "@nx/esbuild:esbuild",
+      "outputs": ["{options.outputPath}"],
+      "options": {
+        "main": "packages/worker/src/index.ts",
+        "outputPath": "packages/worker/dist",
+        "outputFileName": "index.js",
+        "tsConfig": "packages/worker/tsconfig.build.json",
+        "assets": [
+          {
+            "glob": "**/*.hbs",
+            "input": "packages/worker/src/constants/templates",
+            "output": "."
+          }
+        ],
+        "format": ["cjs"],
+        "esbuildOptions": {
+          "outExtension": {
+            ".js": ".js"
+          },
+          "sourcemap": true
+        },
+        "minify": true,
+        "generatePackageJson": true,
+        "skipTypeCheck": true
+      },
+      "configurations": {
+        "production": {
+          "skipTypeCheck": false,
+          "esbuildOptions": {
+            "sourcemap": false
+          }
+        }
+      },
+      "dependsOn": [
+        {
+          "projects": ["@budibase/types", "@budibase/string-templates"],
+          "target": "build"
+        }
+      ]
+    }
+  }
+}

packages/worker/tsconfig.build.json

@@ -8,13 +8,18 @@
     "esModuleInterop": true,
     "resolveJsonModule": true,
     "incremental": true,
-    "types": [ "node", "jest" ],
+    "types": ["node", "jest"],
     "outDir": "dist",
-    "skipLibCheck": true
+    "skipLibCheck": true,
+    "paths": {
+      "@budibase/types": ["../types/src"],
+      "@budibase/backend-core": ["../backend-core/src"],
+      "@budibase/backend-core/*": ["../backend-core/*"],
+      "@budibase/shared-core": ["../shared-core/src"],
+      "@budibase/pro": ["../pro/packages/pro/src"]
+    }
   },
-  "include": [
+  "include": ["src/**/*"],
-    "src/**/*"
-  ],
   "exclude": [
     "node_modules",
     "dist",

packages/worker/tsconfig.json

@@ -16,7 +16,6 @@
     "require": ["tsconfig-paths/register"],
     "swc": true
   },
-  "references": [{ "path": "../types" }, { "path": "../backend-core" }],
   "include": ["src/**/*"],
   "exclude": ["dist"]
 }

qa-core/tsconfig.json

@@ -22,10 +22,6 @@
   "ts-node": {
     "require": ["tsconfig-paths/register"]
   },
-  "references": [
-    { "path": "../packages/types" },
-    { "path": "../packages/backend-core" }
-  ],
   "include": ["src/**/*"],
   "exclude": ["node_modules", "dist"]
 }

scripts/build.js

@@ -0,0 +1,69 @@
+#!/usr/bin/node
+
+const start = Date.now()
+
+const glob = require("glob")
+const fs = require("fs")
+const path = require("path")
+
+const { build } = require("esbuild")
+
+const { default: NodeResolve } = require("@esbuild-plugins/node-resolve")
+
+var argv = require("minimist")(process.argv.slice(2))
+
+function runBuild(entry, outfile) {
+  const isDev = process.env.NODE_ENV !== "production"
+  const tsconfig = argv["p"] || `tsconfig.build.json`
+
+  const sharedConfig = {
+    entryPoints: [entry],
+    bundle: true,
+    minify: !isDev,
+    sourcemap: isDev,
+    tsconfig,
+    plugins: [
+      NodeResolve({
+        extensions: [".ts", ".js"],
+        onResolved: resolved => {
+          if (resolved.includes("node_modules")) {
+            return {
+              external: true,
+            }
+          }
+          return resolved
+        },
+      }),
+    ],
+    target: "node14",
+    preserveSymlinks: true,
+    loader: {
+      ".svelte": "copy",
+    },
+  }
+
+  build({
+    ...sharedConfig,
+    platform: "node",
+    outfile,
+  }).then(() => {
+    glob(`${process.cwd()}/src/**/*.hbs`, {}, (err, files) => {
+      for (const file of files) {
+        fs.copyFileSync(file, `${process.cwd()}/dist/${path.basename(file)}`)
+      }
+
+      console.log(
+        "\x1b[32m%s\x1b[0m",
+        `Build successfully in ${(Date.now() - start) / 1000} seconds`
+      )
+    })
+  })
+}
+
+if (require.main === module) {
+  const entry = argv["e"] || "./src/index.ts"
+  const outfile = `dist/${entry.split("/").pop().replace(".ts", ".js")}`
+  runBuild(entry, outfile)
+} else {
+  module.exports = runBuild
+}