Merge remote-tracking branch 'origin/develop' into feature/app-settings-section

lerna.json

@@ -1,6 +1,7 @@
 {
-  "version": "2.7.25-alpha.2",
+  "version": "2.7.25-alpha.7",
   "npmClient": "yarn",
+  "useNx": true,
   "packages": [
     "packages/backend-core",
     "packages/bbui",
@@ -16,7 +17,6 @@
     "packages/worker",
     "packages/pro/packages/pro"
   ],
-  "useWorkspaces": true,
   "command": {
     "publish": {
       "ignoreChanges": [

package.json

@@ -2,23 +2,22 @@
   "name": "root",
   "private": true,
   "devDependencies": {
-    "@esbuild-plugins/node-resolve": "^0.2.2",
     "@esbuild-plugins/tsconfig-paths": "^0.1.2",
     "@nx/js": "16.2.1",
     "@rollup/plugin-json": "^4.0.2",
     "@typescript-eslint/parser": "5.45.0",
     "babel-eslint": "^10.0.3",
     "esbuild": "^0.17.18",
+    "esbuild-node-externals": "^1.7.0",
     "eslint": "^7.28.0",
     "eslint-plugin-cypress": "^2.11.3",
     "eslint-plugin-svelte3": "^3.2.0",
     "husky": "^8.0.3",
     "js-yaml": "^4.1.0",
     "kill-port": "^1.6.1",
-    "lerna": "7.0.0-alpha.0",
+    "lerna": "7.0.2",
     "madge": "^6.0.0",
     "minimist": "^1.2.8",
-    "nx": "^16.2.1",
     "prettier": "^2.3.1",
     "prettier-plugin-svelte": "^2.3.0",
     "rimraf": "^3.0.2",
@@ -48,9 +47,9 @@
     "kill-builder": "kill-port 3000",
     "kill-server": "kill-port 4001 4002",
     "kill-all": "yarn run kill-builder && yarn run kill-server",
-    "dev": "yarn run kill-all  && lerna run --stream --parallel dev:builder --stream",
-    "dev:noserver": "yarn run kill-builder  && lerna run --stream dev:stack:up && lerna run --stream --parallel dev:builder --ignore @budibase/backend-core --ignore @budibase/server --ignore @budibase/worker",
-    "dev:server": "yarn run kill-server && lerna run --stream --parallel dev:builder --scope @budibase/worker --scope @budibase/server",
+    "dev": "yarn run kill-all && lerna run --stream --parallel dev:builder --stream",
+    "dev:noserver": "yarn run kill-builder && lerna run --stream dev:stack:up && lerna run --stream --parallel dev:builder --ignore @budibase/backend-core --ignore @budibase/server --ignore @budibase/worker",
+    "dev:server": "yarn run kill-server && yarn build --projects=@budibase/client && lerna run --stream --parallel dev:builder --scope @budibase/worker --scope @budibase/server",
     "dev:built": "yarn run kill-all && cd packages/server && yarn dev:stack:up && cd ../../ && lerna run --stream --parallel dev:built",
     "dev:docker": "yarn build:docker:pre && docker-compose -f hosting/docker-compose.build.yaml -f hosting/docker-compose.dev.yaml --env-file hosting/.env up --build --scale proxy-service=0",
     "test": "lerna run --stream test --stream",

packages/backend-core/jest.config.ts

@@ -31,4 +31,6 @@ const config: Config.InitialOptions = {
   coverageReporters: ["lcov", "json", "clover"],
 }
 
+process.env.DISABLE_PINO_LOGGER = "1"
+
 export default config

packages/backend-core/package.json

@@ -27,7 +27,7 @@
     "@techpass/passport-openidconnect": "0.3.2",
     "aws-cloudfront-sign": "2.2.0",
     "aws-sdk": "2.1030.0",
-    "bcrypt": "5.0.1",
+    "bcrypt": "5.1.0",
     "bcryptjs": "2.4.3",
     "bull": "4.10.1",
     "correlation-id": "4.0.0",

packages/builder/src/components/start/CreateAppModal.svelte

@@ -11,6 +11,7 @@
   import TemplateCard from "components/common/TemplateCard.svelte"
   import createFromScratchScreen from "builderStore/store/screenTemplates/createFromScratchScreen"
   import { Roles } from "constants/backend"
+  import { lowercase } from "helpers"
 
   export let template
 
@@ -19,6 +20,7 @@
 
   const values = writable({ name: "", url: null })
   const validation = createValidationStore()
+  const encryptionValidation = createValidationStore()
 
   $: {
     const { url } = $values
@@ -27,8 +29,11 @@
       ...$values,
       url: url?.[0] === "/" ? url.substring(1, url.length) : url,
     })
+    encryptionValidation.check({ ...$values })
   }
 
+  $: encryptedFile = $values.file?.name?.endsWith(".enc.tar.gz")
+
   onMount(async () => {
     const lastChar = $auth.user?.firstName
       ? $auth.user?.firstName[$auth.user?.firstName.length - 1]
@@ -87,6 +92,9 @@
     appValidation.name(validation, { apps: applications })
     appValidation.url(validation, { apps: applications })
     appValidation.file(validation, { template })
+
+    encryptionValidation.addValidatorType("encryptionPassword", "text", true)
+
     // init validation
     const { url } = $values
     validation.check({
@@ -110,6 +118,9 @@
         data.append("templateName", template.name)
         data.append("templateKey", template.key)
         data.append("templateFile", $values.file)
+        if ($values.encryptionPassword?.trim()) {
+          data.append("encryptionPassword", $values.encryptionPassword.trim())
+        }
       }
 
       // Create App
@@ -143,67 +154,119 @@
       $goto(`/builder/app/${createdApp.instance._id}`)
     } catch (error) {
       creating = false
-      console.error(error)
-      notifications.error("Error creating app")
+      throw error
     }
   }
+
+  const Step = { CONFIG: "config", SET_PASSWORD: "set_password" }
+  let currentStep = Step.CONFIG
+  $: stepConfig = {
+    [Step.CONFIG]: {
+      title: "Create your app",
+      confirmText: template?.fromFile ? "Import app" : "Create app",
+      onConfirm: async () => {
+        if (encryptedFile) {
+          currentStep = Step.SET_PASSWORD
+          return false
+        } else {
+          try {
+            await createNewApp()
+          } catch (error) {
+            notifications.error("Error creating app")
+          }
+        }
+      },
+      isValid: $validation.valid,
+    },
+    [Step.SET_PASSWORD]: {
+      title: "Provide the export password",
+      confirmText: "Import app",
+      onConfirm: async () => {
+        try {
+          await createNewApp()
+        } catch (e) {
+          let message = "Error creating app"
+          if (e.message) {
+            message += `: ${lowercase(e.message)}`
+          }
+          notifications.error(message)
+          return false
+        }
+      },
+      isValid: $encryptionValidation.valid,
+    },
+  }
 </script>
 
 <ModalContent
-  title={"Create your app"}
-  confirmText={template?.fromFile ? "Import app" : "Create app"}
-  onConfirm={createNewApp}
-  disabled={!$validation.valid}
+  title={stepConfig[currentStep].title}
+  confirmText={stepConfig[currentStep].confirmText}
+  onConfirm={stepConfig[currentStep].onConfirm}
+  disabled={!stepConfig[currentStep].isValid}
 >
-  {#if template && !template?.fromFile}
-    <TemplateCard
-      name={template.name}
-      imageSrc={template.image}
-      backgroundColour={template.background}
-      overlayEnabled={false}
-      icon={template.icon}
-    />
-  {/if}
-  {#if template?.fromFile}
-    <Dropzone
-      error={$validation.touched.file && $validation.errors.file}
-      gallery={false}
-      label="File to import"
-      value={[$values.file]}
-      on:change={e => {
-        $values.file = e.detail?.[0]
-        $validation.touched.file = true
-      }}
-    />
-  {/if}
-  <Input
-    autofocus={true}
-    bind:value={$values.name}
-    disabled={creating}
-    error={$validation.touched.name && $validation.errors.name}
-    on:blur={() => ($validation.touched.name = true)}
-    on:change={nameToUrl($values.name)}
-    label="Name"
-    placeholder={defaultAppName}
-  />
-  <span>
-    <Input
-      bind:value={$values.url}
-      disabled={creating}
-      error={$validation.touched.url && $validation.errors.url}
-      on:blur={() => ($validation.touched.url = true)}
-      on:change={tidyUrl($values.url)}
-      label="URL"
-      placeholder={$values.url
-        ? $values.url
-        : `/${resolveAppUrl(template, $values.name)}`}
-    />
-    {#if $values.url && $values.url !== "" && !$validation.errors.url}
-      <div class="app-server" title={appUrl}>
-        {appUrl}
-      </div>
+  {#if currentStep === Step.CONFIG}
+    {#if template && !template?.fromFile}
+      <TemplateCard
+        name={template.name}
+        imageSrc={template.image}
+        backgroundColour={template.background}
+        overlayEnabled={false}
+        icon={template.icon}
+      />
     {/if}
-  </span>
+    {#if template?.fromFile}
+      <Dropzone
+        error={$validation.touched.file && $validation.errors.file}
+        gallery={false}
+        label="File to import"
+        value={[$values.file]}
+        on:change={e => {
+          $values.file = e.detail?.[0]
+          $validation.touched.file = true
+        }}
+      />
+    {/if}
+    <Input
+      autofocus={true}
+      bind:value={$values.name}
+      disabled={creating}
+      error={$validation.touched.name && $validation.errors.name}
+      on:blur={() => ($validation.touched.name = true)}
+      on:change={nameToUrl($values.name)}
+      label="Name"
+      placeholder={defaultAppName}
+    />
+    <span>
+      <Input
+        bind:value={$values.url}
+        disabled={creating}
+        error={$validation.touched.url && $validation.errors.url}
+        on:blur={() => ($validation.touched.url = true)}
+        on:change={tidyUrl($values.url)}
+        label="URL"
+        placeholder={$values.url
+          ? $values.url
+          : `/${resolveAppUrl(template, $values.name)}`}
+      />
+      {#if $values.url && $values.url !== "" && !$validation.errors.url}
+        <div class="app-server" title={appUrl}>
+          {appUrl}
+        </div>
+      {/if}
+    </span>
+  {/if}
+  {#if currentStep === Step.SET_PASSWORD}
+    <Input
+      autofocus={true}
+      label="Imported file password"
+      type="password"
+      bind:value={$values.encryptionPassword}
+      disabled={creating}
+      on:blur={() => ($encryptionValidation.touched.encryptionPassword = true)}
+      error={$encryptionValidation.touched.encryptionPassword &&
+        $encryptionValidation.errors.encryptionPassword}
+    />
+  {/if}
 </ModalContent>
 
 <style>

packages/server/nodemon.json

@@ -6,5 +6,5 @@
     "src/**/*.spec.js",
     "../backend-core/dist/**/*"
   ],
-  "exec": "node ./scripts/build.js && node ./dist/index.js"
+  "exec": "yarn build && node ./dist/index.js"
 }

packages/server/package.json

@@ -63,6 +63,7 @@
     "airtable": "0.10.1",
     "arangojs": "7.2.0",
     "aws-sdk": "2.1030.0",
+    "bcrypt": "5.1.0",
     "bcryptjs": "2.4.3",
     "bull": "4.10.1",
     "chmodr": "1.2.0",

packages/server/src/api/controllers/application.ts

@@ -115,7 +115,18 @@ function checkAppName(
   }
 }
 
-async function createInstance(appId: string, template: any) {
+interface AppTemplate {
+  templateString: string
+  useTemplate: string
+  file?: {
+    type: string
+    path: string
+    password?: string
+  }
+  key?: string
+}
+
+async function createInstance(appId: string, template: AppTemplate) {
   const db = context.getAppDB()
   await db.put({
     _id: "_design/database",
@@ -240,19 +251,24 @@ export async function fetchAppPackage(ctx: UserCtx) {
 async function performAppCreate(ctx: UserCtx) {
   const apps = (await dbCore.getAllApps({ dev: true })) as App[]
   const name = ctx.request.body.name,
-    possibleUrl = ctx.request.body.url
+    possibleUrl = ctx.request.body.url,
+    encryptionPassword = ctx.request.body.encryptionPassword
+
   checkAppName(ctx, apps, name)
   const url = sdk.applications.getAppUrl({ name, url: possibleUrl })
   checkAppUrl(ctx, apps, url)
 
   const { useTemplate, templateKey, templateString } = ctx.request.body
-  const instanceConfig: any = {
+  const instanceConfig: AppTemplate = {
     useTemplate,
     key: templateKey,
     templateString,
   }
   if (ctx.request.files && ctx.request.files.templateFile) {
-    instanceConfig.file = ctx.request.files.templateFile
+    instanceConfig.file = {
+      ...(ctx.request.files.templateFile as any),
+      password: encryptionPassword,
+    }
   }
   const tenantId = tenancy.isMultiTenant() ? tenancy.getTenantId() : null
   const appId = generateDevAppID(generateAppID(tenantId))

packages/server/src/app.ts

@@ -15,7 +15,6 @@ import * as api from "./api"
 import * as automations from "./automations"
 import { Thread } from "./threads"
 import * as redis from "./utilities/redis"
-import { initialise as initialiseWebsockets } from "./websockets"
 import { events, logging, middleware, timers } from "@budibase/backend-core"
 import { startup } from "./startup"
 const Sentry = require("@sentry/node")

packages/server/src/integrations/mongodb.ts

@@ -489,7 +489,11 @@ class MongoIntegration implements IntegrationBase {
 
       switch (query.extra.actionType) {
         case "find": {
-          return await collection.find(json).toArray()
+          if (json) {
+            return await collection.find(json).toArray()
+          } else {
+            return await collection.find().toArray()
+          }
         }
         case "findOne": {
           return await collection.findOne(json)

packages/worker/package.json

@@ -47,6 +47,7 @@
     "@techpass/passport-openidconnect": "0.3.2",
     "@types/global-agent": "2.1.1",
     "aws-sdk": "2.1030.0",
+    "bcrypt": "5.1.0",
     "bcryptjs": "2.4.3",
     "dd-trace": "3.13.2",
     "dotenv": "8.6.0",

packages/worker/src/api/controllers/global/users.ts

@@ -38,7 +38,7 @@ const MAX_USERS_UPLOAD_LIMIT = 1000
 
 export const save = async (ctx: UserCtx<User, SaveUserResponse>) => {
   try {
-    const currentUserId = ctx.user._id
+    const currentUserId = ctx.user?._id
     const requestUser = ctx.request.body
 
     const user = await userSdk.save(requestUser, { currentUserId })

qa-core/package.json

@@ -10,6 +10,7 @@
   },
   "scripts": {
     "setup": "yarn && node scripts/createEnv.js",
+    "user": "yarn && node scripts/createEnv.js && node scripts/createUser.js",
     "test": "jest --runInBand --json --outputFile=testResults.json --forceExit",
     "test:watch": "yarn run test --watch",
     "test:debug": "DEBUG=1 yarn run test",

qa-core/scripts/createUser.js

@@ -0,0 +1,49 @@
+const dotenv = require("dotenv")
+const { join } = require("path")
+const fs = require("fs")
+const fetch = require("node-fetch")
+
+function getVarFromDotEnv(path, varName) {
+  const parsed = dotenv.parse(fs.readFileSync(path))
+  return parsed[varName]
+}
+
+async function createUser() {
+  const serverPath = join(__dirname, "..", "..", "packages", "server", ".env")
+  const qaCorePath = join(__dirname, "..", ".env")
+  const apiKey = getVarFromDotEnv(serverPath, "INTERNAL_API_KEY")
+  const username = getVarFromDotEnv(qaCorePath, "BB_ADMIN_USER_EMAIL")
+  const password = getVarFromDotEnv(qaCorePath, "BB_ADMIN_USER_PASSWORD")
+  const url = getVarFromDotEnv(qaCorePath, "BUDIBASE_URL")
+  const resp = await fetch(`${url}/api/public/v1/users`, {
+    method: "POST",
+    headers: {
+      "Content-Type": "application/json",
+      "x-budibase-api-key": apiKey,
+    },
+    body: JSON.stringify({
+      email: username,
+      password,
+      builder: {
+        global: true,
+      },
+      admin: {
+        global: true,
+      },
+      roles: {},
+    }),
+  })
+  if (resp.status !== 200) {
+    throw new Error(await resp.text())
+  } else {
+    return await resp.json()
+  }
+}
+
+createUser()
+  .then(() => {
+    console.log("User created - ready to use")
+  })
+  .catch(err => {
+    console.error("Failed to create user - ", err)
+  })

qa-core/src/account-api/api/AccountInternalAPIClient.ts

@@ -67,11 +67,12 @@ export default class AccountInternalAPIClient {
       }
       const message = `${method} ${url} - ${response.status}`
 
+      const isDebug = process.env.LOG_LEVEL === "debug"
       if (response.status > 499) {
         console.error(message, data)
       } else if (response.status >= 400) {
         console.warn(message, data)
-      } else {
+      } else if (isDebug) {
         console.debug(message, data)
       }
 

qa-core/src/internal-api/api/BudibaseInternalAPIClient.ts

@@ -58,11 +58,12 @@ class BudibaseInternalAPIClient {
       }
       const message = `${method} ${url} - ${response.status}`
 
+      const isDebug = process.env.LOG_LEVEL === "debug"
       if (response.status > 499) {
         console.error(message, data)
       } else if (response.status >= 400) {
         console.warn(message, data)
-      } else {
+      } else if (isDebug) {
         console.debug(message, data)
       }
 

qa-core/src/internal-api/tests/dataSources/mongoDB.integration.spec.ts

@@ -2,7 +2,7 @@ import TestConfiguration from "../../config/TestConfiguration"
 import * as fixtures from "../../fixtures"
 import { Query } from "@budibase/types"
 
-describe("Internal API - Data Sources: MongoDB", () => {
+xdescribe("Internal API - Data Sources: MongoDB", () => {
   const config = new TestConfiguration()
 
   beforeAll(async () => {

qa-core/src/jest/globalSetup.ts

@@ -1,3 +1,4 @@
+process.env.DISABLE_PINO_LOGGER = "1"
 import { DEFAULT_TENANT_ID, logging } from "@budibase/backend-core"
 import { AccountInternalAPI } from "../account-api"
 import * as fixtures from "../internal-api/fixtures"

qa-core/src/public-api/api/BudibasePublicAPIClient.ts

@@ -57,11 +57,12 @@ class BudibasePublicAPIClient {
       }
       const message = `${method} ${url} - ${response.status}`
 
+      const isDebug = process.env.LOG_LEVEL === "debug"
       if (response.status > 499) {
         console.error(message, data)
       } else if (response.status >= 400) {
         console.warn(message, data)
-      } else {
+      } else if (isDebug) {
         console.debug(message, data)
       }
 

scripts/build.js

@@ -8,10 +8,10 @@ const path = require("path")
 
 const { build } = require("esbuild")
 
-const { default: NodeResolve } = require("@esbuild-plugins/node-resolve")
 const {
   default: TsconfigPathsPlugin,
 } = require("@esbuild-plugins/tsconfig-paths")
+const { nodeExternalsPlugin } = require("esbuild-node-externals")
 
 var argv = require("minimist")(process.argv.slice(2))
 
@@ -25,32 +25,28 @@ function runBuild(entry, outfile) {
     minify: !isDev,
     sourcemap: isDev,
     tsconfig,
-    plugins: [
-      TsconfigPathsPlugin({ tsconfig }),
-      NodeResolve({
-        extensions: [".ts", ".js"],
-        onResolved: resolved => {
-          if (resolved.includes("node_modules") && !resolved.includes("/@budibase/pro/")) {
-            return {
-              external: true,
-            }
-          }
-          return resolved
-        },
-      }),
-    ],
+    plugins: [TsconfigPathsPlugin({ tsconfig }), nodeExternalsPlugin()],
     target: "node14",
     preserveSymlinks: true,
     loader: {
       ".svelte": "copy",
     },
+    metafile: true,
+    external: [
+      "deasync",
+      "mock-aws-s3",
+      "nock",
+      "pino",
+      "koa-pino-logger",
+      "bull",
+    ],
   }
 
   build({
     ...sharedConfig,
     platform: "node",
     outfile,
-  }).then(() => {
+  }).then(result => {
     glob(`${process.cwd()}/src/**/*.hbs`, {}, (err, files) => {
       for (const file of files) {
         fs.copyFileSync(file, `${process.cwd()}/dist/${path.basename(file)}`)
@@ -61,6 +57,11 @@ function runBuild(entry, outfile) {
         `Build successfully in ${(Date.now() - start) / 1000} seconds`
       )
     })
+
+    fs.writeFileSync(
+      `dist/${path.basename(outfile)}.meta.json`,
+      JSON.stringify(result.metafile)
+    )
   })
 }