Merge branch 'develop' into chore/refactor_pro_as_package

lerna.json

@@ -1,8 +1,9 @@
 {
-  "version": "2.7.25-alpha.3",
+  "version": "2.7.25-alpha.8",
   "npmClient": "yarn",
   "packages": ["packages/*"],
   "useWorkspaces": true,
+  "useNx": true,
   "command": {
     "publish": {
       "ignoreChanges": [

package.json

@@ -2,23 +2,22 @@
   "name": "root",
   "private": true,
   "devDependencies": {
-    "@esbuild-plugins/node-resolve": "^0.2.2",
     "@esbuild-plugins/tsconfig-paths": "^0.1.2",
     "@nx/js": "16.2.1",
     "@rollup/plugin-json": "^4.0.2",
     "@typescript-eslint/parser": "5.45.0",
     "babel-eslint": "^10.0.3",
     "esbuild": "^0.17.18",
+    "esbuild-node-externals": "^1.7.0",
     "eslint": "^7.28.0",
     "eslint-plugin-cypress": "^2.11.3",
     "eslint-plugin-svelte3": "^3.2.0",
     "husky": "^8.0.3",
     "js-yaml": "^4.1.0",
     "kill-port": "^1.6.1",
-    "lerna": "7.0.0-alpha.0",
+    "lerna": "7.0.2",
     "madge": "^6.0.0",
     "minimist": "^1.2.8",
-    "nx": "^16.2.1",
     "prettier": "^2.3.1",
     "prettier-plugin-svelte": "^2.3.0",
     "rimraf": "^3.0.2",
@@ -48,9 +47,9 @@
     "kill-builder": "kill-port 3000",
     "kill-server": "kill-port 4001 4002",
     "kill-all": "yarn run kill-builder && yarn run kill-server",
-    "dev": "yarn run kill-all  && lerna run --stream --parallel dev:builder --stream",
-    "dev:noserver": "yarn run kill-builder  && lerna run --stream dev:stack:up && lerna run --stream --parallel dev:builder --ignore @budibase/backend-core --ignore @budibase/server --ignore @budibase/worker",
-    "dev:server": "yarn run kill-server && lerna run --stream --parallel dev:builder --scope @budibase/worker --scope @budibase/server",
+    "dev": "yarn run kill-all && lerna run --stream --parallel dev:builder --stream",
+    "dev:noserver": "yarn run kill-builder && lerna run --stream dev:stack:up && lerna run --stream --parallel dev:builder --ignore @budibase/backend-core --ignore @budibase/server --ignore @budibase/worker",
+    "dev:server": "yarn run kill-server && yarn build --projects=@budibase/client && lerna run --stream --parallel dev:builder --scope @budibase/worker --scope @budibase/server",
     "dev:built": "yarn run kill-all && cd packages/server && yarn dev:stack:up && cd ../../ && lerna run --stream --parallel dev:built",
     "dev:docker": "yarn build:docker:pre && docker-compose -f hosting/docker-compose.build.yaml -f hosting/docker-compose.dev.yaml --env-file hosting/.env up --build --scale proxy-service=0",
     "test": "lerna run --stream test --stream",

packages/backend-core/jest.config.ts

@@ -31,4 +31,6 @@ const config: Config.InitialOptions = {
   coverageReporters: ["lcov", "json", "clover"],
 }
 
+process.env.DISABLE_PINO_LOGGER = "1"
+
 export default config

packages/backend-core/package.json

@@ -27,7 +27,7 @@
     "@techpass/passport-openidconnect": "0.3.2",
     "aws-cloudfront-sign": "2.2.0",
     "aws-sdk": "2.1030.0",
-    "bcrypt": "5.0.1",
+    "bcrypt": "5.1.0",
     "bcryptjs": "2.4.3",
     "bull": "4.10.1",
     "correlation-id": "4.0.0",

packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte

@@ -309,7 +309,7 @@
   }
 
   function canShowField(key, value) {
-    const dependsOn = value.dependsOn
+    const dependsOn = value?.dependsOn
     return !dependsOn || !!inputData[dependsOn]
   }
 

packages/server/nodemon.json

@@ -6,5 +6,5 @@
     "src/**/*.spec.js",
     "../backend-core/dist/**/*"
   ],
-  "exec": "node ./scripts/build.js && node ./dist/index.js"
+  "exec": "yarn build && node ./dist/index.js"
 }

packages/server/package.json

@@ -63,6 +63,7 @@
     "airtable": "0.10.1",
     "arangojs": "7.2.0",
     "aws-sdk": "2.1030.0",
+    "bcrypt": "5.1.0",
     "bcryptjs": "2.4.3",
     "bull": "4.10.1",
     "chmodr": "1.2.0",

packages/server/src/app.ts

@@ -15,7 +15,6 @@ import * as api from "./api"
 import * as automations from "./automations"
 import { Thread } from "./threads"
 import * as redis from "./utilities/redis"
-import { initialise as initialiseWebsockets } from "./websockets"
 import { events, logging, middleware, timers } from "@budibase/backend-core"
 import { startup } from "./startup"
 const Sentry = require("@sentry/node")

packages/server/src/integrations/mongodb.ts

@@ -489,7 +489,11 @@ class MongoIntegration implements IntegrationBase {
 
       switch (query.extra.actionType) {
         case "find": {
-          return await collection.find(json).toArray()
+          if (json) {
+            return await collection.find(json).toArray()
+          } else {
+            return await collection.find().toArray()
+          }
         }
         case "findOne": {
           return await collection.findOne(json)

packages/worker/package.json

@@ -47,6 +47,7 @@
     "@techpass/passport-openidconnect": "0.3.2",
     "@types/global-agent": "2.1.1",
     "aws-sdk": "2.1030.0",
+    "bcrypt": "5.1.0",
     "bcryptjs": "2.4.3",
     "dd-trace": "3.13.2",
     "dotenv": "8.6.0",

packages/worker/src/api/controllers/global/users.ts

@@ -38,7 +38,7 @@ const MAX_USERS_UPLOAD_LIMIT = 1000
 
 export const save = async (ctx: UserCtx<User, SaveUserResponse>) => {
   try {
-    const currentUserId = ctx.user._id
+    const currentUserId = ctx.user?._id
     const requestUser = ctx.request.body
 
     const user = await userSdk.save(requestUser, { currentUserId })

qa-core/package.json

@@ -10,6 +10,7 @@
   },
   "scripts": {
     "setup": "yarn && node scripts/createEnv.js",
+    "user": "yarn && node scripts/createEnv.js && node scripts/createUser.js",
     "test": "jest --runInBand --json --outputFile=testResults.json --forceExit",
     "test:watch": "yarn run test --watch",
     "test:debug": "DEBUG=1 yarn run test",

qa-core/scripts/createUser.js

@@ -0,0 +1,49 @@
+const dotenv = require("dotenv")
+const { join } = require("path")
+const fs = require("fs")
+const fetch = require("node-fetch")
+
+function getVarFromDotEnv(path, varName) {
+  const parsed = dotenv.parse(fs.readFileSync(path))
+  return parsed[varName]
+}
+
+async function createUser() {
+  const serverPath = join(__dirname, "..", "..", "packages", "server", ".env")
+  const qaCorePath = join(__dirname, "..", ".env")
+  const apiKey = getVarFromDotEnv(serverPath, "INTERNAL_API_KEY")
+  const username = getVarFromDotEnv(qaCorePath, "BB_ADMIN_USER_EMAIL")
+  const password = getVarFromDotEnv(qaCorePath, "BB_ADMIN_USER_PASSWORD")
+  const url = getVarFromDotEnv(qaCorePath, "BUDIBASE_URL")
+  const resp = await fetch(`${url}/api/public/v1/users`, {
+    method: "POST",
+    headers: {
+      "Content-Type": "application/json",
+      "x-budibase-api-key": apiKey,
+    },
+    body: JSON.stringify({
+      email: username,
+      password,
+      builder: {
+        global: true,
+      },
+      admin: {
+        global: true,
+      },
+      roles: {},
+    }),
+  })
+  if (resp.status !== 200) {
+    throw new Error(await resp.text())
+  } else {
+    return await resp.json()
+  }
+}
+
+createUser()
+  .then(() => {
+    console.log("User created - ready to use")
+  })
+  .catch(err => {
+    console.error("Failed to create user - ", err)
+  })

qa-core/src/account-api/api/AccountInternalAPIClient.ts

@@ -67,11 +67,12 @@ export default class AccountInternalAPIClient {
       }
       const message = `${method} ${url} - ${response.status}`
 
+      const isDebug = process.env.LOG_LEVEL === "debug"
       if (response.status > 499) {
         console.error(message, data)
       } else if (response.status >= 400) {
         console.warn(message, data)
-      } else {
+      } else if (isDebug) {
         console.debug(message, data)
       }
 

qa-core/src/internal-api/api/BudibaseInternalAPIClient.ts

@@ -58,11 +58,12 @@ class BudibaseInternalAPIClient {
       }
       const message = `${method} ${url} - ${response.status}`
 
+      const isDebug = process.env.LOG_LEVEL === "debug"
       if (response.status > 499) {
         console.error(message, data)
       } else if (response.status >= 400) {
         console.warn(message, data)
-      } else {
+      } else if (isDebug) {
         console.debug(message, data)
       }
 

qa-core/src/internal-api/tests/dataSources/mongoDB.integration.spec.ts

@@ -2,7 +2,7 @@ import TestConfiguration from "../../config/TestConfiguration"
 import * as fixtures from "../../fixtures"
 import { Query } from "@budibase/types"
 
-describe("Internal API - Data Sources: MongoDB", () => {
+xdescribe("Internal API - Data Sources: MongoDB", () => {
   const config = new TestConfiguration()
 
   beforeAll(async () => {

qa-core/src/jest/globalSetup.ts

@@ -1,3 +1,4 @@
+process.env.DISABLE_PINO_LOGGER = "1"
 import { DEFAULT_TENANT_ID, logging } from "@budibase/backend-core"
 import { AccountInternalAPI } from "../account-api"
 import * as fixtures from "../internal-api/fixtures"

qa-core/src/public-api/api/BudibasePublicAPIClient.ts

@@ -57,11 +57,12 @@ class BudibasePublicAPIClient {
       }
       const message = `${method} ${url} - ${response.status}`
 
+      const isDebug = process.env.LOG_LEVEL === "debug"
       if (response.status > 499) {
         console.error(message, data)
       } else if (response.status >= 400) {
         console.warn(message, data)
-      } else {
+      } else if (isDebug) {
         console.debug(message, data)
       }
 

scripts/build.js

@@ -8,10 +8,10 @@ const path = require("path")
 
 const { build } = require("esbuild")
 
-const { default: NodeResolve } = require("@esbuild-plugins/node-resolve")
 const {
   default: TsconfigPathsPlugin,
 } = require("@esbuild-plugins/tsconfig-paths")
+const { nodeExternalsPlugin } = require("esbuild-node-externals")
 
 var argv = require("minimist")(process.argv.slice(2))
 
@@ -25,32 +25,28 @@ function runBuild(entry, outfile) {
     minify: !isDev,
     sourcemap: isDev,
     tsconfig,
-    plugins: [
-      TsconfigPathsPlugin({ tsconfig }),
-      NodeResolve({
-        extensions: [".ts", ".js"],
-        onResolved: resolved => {
-          if (resolved.includes("node_modules") && !resolved.includes("/@budibase/pro/")) {
-            return {
-              external: true,
-            }
-          }
-          return resolved
-        },
-      }),
-    ],
+    plugins: [TsconfigPathsPlugin({ tsconfig }), nodeExternalsPlugin()],
     target: "node14",
     preserveSymlinks: true,
     loader: {
       ".svelte": "copy",
     },
+    metafile: true,
+    external: [
+      "deasync",
+      "mock-aws-s3",
+      "nock",
+      "pino",
+      "koa-pino-logger",
+      "bull",
+    ],
   }
 
   build({
     ...sharedConfig,
     platform: "node",
     outfile,
-  }).then(() => {
+  }).then(result => {
     glob(`${process.cwd()}/src/**/*.hbs`, {}, (err, files) => {
       for (const file of files) {
         fs.copyFileSync(file, `${process.cwd()}/dist/${path.basename(file)}`)
@@ -61,6 +57,11 @@ function runBuild(entry, outfile) {
         `Build successfully in ${(Date.now() - start) / 1000} seconds`
       )
     })
+
+    fs.writeFileSync(
+      `dist/${path.basename(outfile)}.meta.json`,
+      JSON.stringify(result.metafile)
+    )
   })
 }