From bec7d86739741c20634d0446399f3b64b159c57e Mon Sep 17 00:00:00 2001
From: Martin McKeaveney <martin@budibase.com>
Date: Tue, 28 Jun 2022 13:14:49 +0100
Subject: [PATCH] cherry pick new ui from develop

---
 .github/workflows/budibase_ci.yml             | 17 ++++++++
 .../backend-core/src/objectStore/index.ts     | 10 +++++
 .../builder/app/[application]/_layout.svelte  | 27 +++++++++++-
 packages/builder/src/stores/backend/flags.js  |  3 ++
 packages/builder/vite.config.js               |  3 ++
 packages/frontend-core/src/api/flags.js       |  9 ++++
 packages/frontend-core/src/api/other.js       |  9 ++++
 packages/server/.gitignore                    |  1 +
 .../server/src/api/controllers/application.ts |  6 ++-
 packages/server/src/api/controllers/dev.js    | 37 +++++++++++++++-
 .../src/api/controllers/static/index.js       | 43 ++++++++++++++++++-
 packages/server/src/api/routes/dev.js         |  1 +
 packages/server/src/api/routes/static.ts      |  1 +
 .../src/utilities/fileSystem/utilities.js     |  2 +
 packages/server/src/utilities/index.js        |  8 +++-
 packages/server/yarn.lock                     | 18 ++++----
 packages/worker/.dockerignore                 |  7 +++
 packages/worker/yarn.lock                     | 18 ++++----
 18 files changed, 195 insertions(+), 25 deletions(-)
 create mode 100644 packages/worker/.dockerignore

diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml
index e940e6fa10..1303d5921a 100644
--- a/.github/workflows/budibase_ci.yml
+++ b/.github/workflows/budibase_ci.yml
@@ -7,6 +7,7 @@ on:
     branches:
       - master
       - develop 
+      - new-design-ui
  pull_request:
     branches:
       - master
@@ -59,3 +60,19 @@ jobs:
       with:
         install: false
         command: yarn test:e2e:ci
+    
+    - name: Configure AWS Credentials
+      uses: aws-actions/configure-aws-credentials@v1
+      with:
+        aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+        aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+        aws-region: eu-west-1
+
+    - name: Upload to S3
+      if: github.ref == 'refs/heads/new-design-ui' 
+      run: | 
+        tar -czvf new_ui.tar.gz packages/server/assets packages/server/index.html 
+        aws s3 cp new_ui.tar.gz s3://prod-budi-app-assets/beta:design_ui/
+        aws s3 cp packages/client/dist/budibase-client.js s3://prod-budi-app-assets/beta:design_ui/budibase-client.js
+        aws cloudfront create-invalidation --distribution-id E3ELKP4RCEHVLW --paths "/beta:design_ui/*"
+        
diff --git a/packages/backend-core/src/objectStore/index.ts b/packages/backend-core/src/objectStore/index.ts
index 9bb0760f5b..a7e0b0c134 100644
--- a/packages/backend-core/src/objectStore/index.ts
+++ b/packages/backend-core/src/objectStore/index.ts
@@ -294,6 +294,16 @@ export const uploadDirectory = async (
   await Promise.all(uploads)
 }
 
+exports.downloadTarballDirect = async (url: string, path: string) => {
+  path = sanitizeKey(path)
+  const response = await fetch(url)
+  if (!response.ok) {
+    throw new Error(`unexpected response ${response.statusText}`)
+  }
+
+  await streamPipeline(response.body, zlib.Unzip(), tar.extract(path))
+}
+
 export const downloadTarball = async (url: any, bucketName: any, path: any) => {
   bucketName = sanitizeBucket(bucketName)
   path = sanitizeKey(path)
diff --git a/packages/builder/src/pages/builder/app/[application]/_layout.svelte b/packages/builder/src/pages/builder/app/[application]/_layout.svelte
index ebe5e32653..8c3de02202 100644
--- a/packages/builder/src/pages/builder/app/[application]/_layout.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/_layout.svelte
@@ -1,7 +1,14 @@
 <script>
   import { store, automationStore } from "builderStore"
   import { roles, flags } from "stores/backend"
-  import { Icon, ActionGroup, Tabs, Tab, notifications } from "@budibase/bbui"
+  import {
+    Icon,
+    ActionGroup,
+    Tabs,
+    Tab,
+    notifications,
+    Banner,
+  } from "@budibase/bbui"
   import RevertModal from "components/deploy/RevertModal.svelte"
   import VersionModal from "components/deploy/VersionModal.svelte"
   import DeployNavigation from "components/deploy/DeployNavigation.svelte"
@@ -17,6 +24,7 @@
 
   // Get Package and set store
   let promise = getPackage()
+  let betaAccess = false
 
   // Sync once when you load the app
   let hasSynced = false
@@ -58,10 +66,18 @@
     })
   }
 
+  async function newDesignUi() {
+    await flags.toggleUiFeature("design_ui")
+    window.location.reload()
+  }
+
   onMount(async () => {
     if (!hasSynced && application) {
       try {
         await API.syncApp(application)
+        // check if user has beta access
+        const betaResponse = await API.checkBetaAccess($auth?.user?.email)
+        betaAccess = betaResponse.access
       } catch (error) {
         notifications.error("Failed to sync with production database")
       }
@@ -79,6 +95,15 @@
   <div class="loading" />
 {:then _}
   <div class="root">
+    {#if betaAccess}
+      <Banner
+        extraButtonText="Try New UI (Beta)"
+        extraButtonAction={newDesignUi}
+      >
+        Try the <b>all new</b> budibase design interface. (Not recommended for existing
+        budibase apps)
+      </Banner>
+    {/if}
     <div class="top-nav">
       <div class="topleftnav">
         <button class="home-logo">
diff --git a/packages/builder/src/stores/backend/flags.js b/packages/builder/src/stores/backend/flags.js
index 449d010640..9f1652676c 100644
--- a/packages/builder/src/stores/backend/flags.js
+++ b/packages/builder/src/stores/backend/flags.js
@@ -16,6 +16,9 @@ export function createFlagsStore() {
       })
       await actions.fetch()
     },
+    toggleUiFeature: async feature => {
+      await API.toggleUiFeature({ value: feature })
+    },
   }
 
   return {
diff --git a/packages/builder/vite.config.js b/packages/builder/vite.config.js
index 56ffbf20f5..ff60efd960 100644
--- a/packages/builder/vite.config.js
+++ b/packages/builder/vite.config.js
@@ -32,6 +32,9 @@ export default ({ mode }) => {
           process.env.INTERCOM_TOKEN
         ),
         "process.env.SENTRY_DSN": JSON.stringify(process.env.SENTRY_DSN),
+        "process.env.FEATURE_PREVIEW_URL": JSON.stringify(
+          process.env.FEATURE_PREVIEW_URL
+        ),
       }),
     ],
     optimizeDeps: {
diff --git a/packages/frontend-core/src/api/flags.js b/packages/frontend-core/src/api/flags.js
index bb545e83b9..16adeb7b5d 100644
--- a/packages/frontend-core/src/api/flags.js
+++ b/packages/frontend-core/src/api/flags.js
@@ -22,4 +22,13 @@ export const buildFlagEndpoints = API => ({
       },
     })
   },
+  /**
+   * Allows us to experimentally toggle a beta UI feature through a cookie.
+   * @param value the feature to toggle
+   */
+  toggleUiFeature: async ({ value }) => {
+    return await API.post({
+      url: `/api/beta/${value}`,
+    })
+  },
 })
diff --git a/packages/frontend-core/src/api/other.js b/packages/frontend-core/src/api/other.js
index 6903a120db..e2cd7aac83 100644
--- a/packages/frontend-core/src/api/other.js
+++ b/packages/frontend-core/src/api/other.js
@@ -54,4 +54,13 @@ export const buildOtherEndpoints = API => ({
       url: "/api/permission/builtin",
     })
   },
+
+  /**
+   * Check if they are part of the budibase beta program.
+   */
+  checkBetaAccess: async email => {
+    return await API.get({
+      url: `${process.env.FEATURE_PREVIEW_URL}/api/beta/access?email=${email}`,
+    })
+  },
 })
diff --git a/packages/server/.gitignore b/packages/server/.gitignore
index f129368792..ced0d982e0 100644
--- a/packages/server/.gitignore
+++ b/packages/server/.gitignore
@@ -2,6 +2,7 @@ node_modules/
 myapps/
 .env
 builder/*
+new_design_ui/*
 client/*
 db/dev.db/
 dist
diff --git a/packages/server/src/api/controllers/application.ts b/packages/server/src/api/controllers/application.ts
index 7d78e5dded..9736ebaba5 100644
--- a/packages/server/src/api/controllers/application.ts
+++ b/packages/server/src/api/controllers/application.ts
@@ -227,7 +227,11 @@ export const fetchAppPackage = async (ctx: any) => {
     application,
     screens,
     layouts,
-    clientLibPath: clientLibraryPath(ctx.params.appId, application.version),
+    clientLibPath: clientLibraryPath(
+      ctx.params.appId,
+      application.version,
+      ctx
+    ),
   }
 }
 
diff --git a/packages/server/src/api/controllers/dev.js b/packages/server/src/api/controllers/dev.js
index 3c5835ac21..0ed3934c6c 100644
--- a/packages/server/src/api/controllers/dev.js
+++ b/packages/server/src/api/controllers/dev.js
@@ -3,8 +3,12 @@ const env = require("../../environment")
 const { checkSlashesInUrl } = require("../../utilities")
 const { request } = require("../../utilities/workerRequests")
 const { clearLock } = require("../../utilities/redis")
-const { Replication, getProdAppID } = require("@budibase/backend-core/db")
-const { DocumentTypes } = require("../../db/utils")
+const {
+  Replication,
+  getProdAppID,
+  dangerousGetDB,
+} = require("@budibase/backend-core/db")
+const { DocumentTypes, getRowParams } = require("../../db/utils")
 const { app: appCache } = require("@budibase/backend-core/cache")
 const { getProdAppDB, getAppDB } = require("@budibase/backend-core/context")
 const { events } = require("@budibase/backend-core")
@@ -133,3 +137,32 @@ exports.getBudibaseVersion = async ctx => {
   }
   await events.installation.versionChecked(version)
 }
+
+// TODO: remove as part of beta program
+exports.checkBetaAccess = async ctx => {
+  const userToCheck = ctx.query.email
+  const BETA_USERS_DB = "app_bb_f9b77d06b9db4e3ca185476ab87a2364"
+  const BETA_USERS_TABLE = "ta_8c2c6df1c03f49cfb6340e85e066dd15"
+
+  try {
+    const db = dangerousGetDB(BETA_USERS_DB)
+    const betaUsers = (
+      await db.allDocs(
+        getRowParams(BETA_USERS_TABLE, null, {
+          include_docs: true,
+        })
+      )
+    ).rows.map(row => row.doc)
+
+    let access = false
+    for (let betaUser of betaUsers) {
+      if (betaUser["Email address"].trim() === userToCheck) {
+        access = true
+        break
+      }
+    }
+    ctx.body = { access }
+  } catch (err) {
+    ctx.body = { access: false }
+  }
+}
diff --git a/packages/server/src/api/controllers/static/index.js b/packages/server/src/api/controllers/static/index.js
index 28295499b2..d99c476206 100644
--- a/packages/server/src/api/controllers/static/index.js
+++ b/packages/server/src/api/controllers/static/index.js
@@ -16,9 +16,15 @@ const { upload } = require("../../../utilities/fileSystem")
 const { attachmentsRelativeURL } = require("../../../utilities")
 const { DocumentTypes, isDevAppID } = require("../../../db/utils")
 const { getAppDB, getAppId } = require("@budibase/backend-core/context")
+const { setCookie, clearCookie } = require("@budibase/backend-core/utils")
 const AWS = require("aws-sdk")
 const { events } = require("@budibase/backend-core")
 
+const fs = require("fs")
+const {
+  downloadTarballDirect,
+} = require("../../../utilities/fileSystem/utilities")
+
 async function prepareUpload({ s3Key, bucket, metadata, file }) {
   const response = await upload({
     bucket,
@@ -38,8 +44,41 @@ async function prepareUpload({ s3Key, bucket, metadata, file }) {
   }
 }
 
+exports.toggleBetaUiFeature = async function (ctx) {
+  const cookieName = `beta:${ctx.params.feature}`
+
+  if (ctx.cookies.get(cookieName)) {
+    clearCookie(ctx, cookieName)
+    ctx.body = {
+      message: `${ctx.params.feature} disabled`,
+    }
+    return
+  }
+
+  let builderPath = resolve(TOP_LEVEL_PATH, "new_design_ui")
+
+  // // download it from S3
+  if (!fs.existsSync(builderPath)) {
+    fs.mkdirSync(builderPath)
+  }
+  await downloadTarballDirect(
+    "https://cdn.budi.live/beta:design_ui/new_ui.tar.gz",
+    builderPath
+  )
+  setCookie(ctx, {}, cookieName)
+
+  ctx.body = {
+    message: `${ctx.params.feature} enabled`,
+  }
+}
+
 exports.serveBuilder = async function (ctx) {
-  let builderPath = resolve(TOP_LEVEL_PATH, "builder")
+  // Temporary: New Design UI
+  const designUiCookie = ctx.cookies.get("beta:design_ui")
+  // TODO: get this from the tmp Dir that we downloaded from MinIO
+  const uiPath = designUiCookie ? "new_design_ui" : "builder"
+
+  let builderPath = resolve(TOP_LEVEL_PATH, uiPath)
   await send(ctx, ctx.file, { root: builderPath })
   if (!ctx.file.includes("assets/")) {
     await events.serve.servedBuilder()
@@ -78,7 +117,7 @@ exports.serveApp = async function (ctx) {
       title: appInfo.name,
       production: env.isProd(),
       appId,
-      clientLibPath: clientLibraryPath(appId, appInfo.version),
+      clientLibPath: clientLibraryPath(appId, appInfo.version, ctx),
     })
 
     const appHbs = loadHandlebarsFile(`${__dirname}/templates/app.hbs`)
diff --git a/packages/server/src/api/routes/dev.js b/packages/server/src/api/routes/dev.js
index 165149ca8b..854e4f7035 100644
--- a/packages/server/src/api/routes/dev.js
+++ b/packages/server/src/api/routes/dev.js
@@ -22,5 +22,6 @@ router
   .get("/api/dev/version", authorized(BUILDER), controller.getBudibaseVersion)
   .delete("/api/dev/:appId/lock", authorized(BUILDER), controller.clearLock)
   .post("/api/dev/:appId/revert", authorized(BUILDER), controller.revert)
+  .get("/api/beta/access", controller.checkBetaAccess)
 
 module.exports = router
diff --git a/packages/server/src/api/routes/static.ts b/packages/server/src/api/routes/static.ts
index d216d993b4..61cf2b1245 100644
--- a/packages/server/src/api/routes/static.ts
+++ b/packages/server/src/api/routes/static.ts
@@ -38,6 +38,7 @@ router
   // TODO: for now this builder endpoint is not authorized/secured, will need to be
   .get("/builder/:file*", controller.serveBuilder)
   .post("/api/attachments/process", authorized(BUILDER), controller.uploadFile)
+  .post("/api/beta/:feature", controller.toggleBetaUiFeature)
   .post(
     "/api/attachments/:tableId/upload",
     paramResource("tableId"),
diff --git a/packages/server/src/utilities/fileSystem/utilities.js b/packages/server/src/utilities/fileSystem/utilities.js
index e85f4f9c79..1c804c0142 100644
--- a/packages/server/src/utilities/fileSystem/utilities.js
+++ b/packages/server/src/utilities/fileSystem/utilities.js
@@ -9,6 +9,7 @@ const {
   deleteFolder,
   uploadDirectory,
   downloadTarball,
+  downloadTarballDirect,
 } = require("@budibase/backend-core/objectStore")
 
 /***********************************
@@ -29,4 +30,5 @@ exports.retrieveToTmp = retrieveToTmp
 exports.deleteFolder = deleteFolder
 exports.uploadDirectory = uploadDirectory
 exports.downloadTarball = downloadTarball
+exports.downloadTarballDirect = downloadTarballDirect
 exports.deleteFiles = deleteFiles
diff --git a/packages/server/src/utilities/index.js b/packages/server/src/utilities/index.js
index 66005fd9cd..0fc18a1ae6 100644
--- a/packages/server/src/utilities/index.js
+++ b/packages/server/src/utilities/index.js
@@ -54,11 +54,17 @@ exports.objectStoreUrl = () => {
  * @return {string} The URL to be inserted into appPackage response or server rendered
  * app index file.
  */
-exports.clientLibraryPath = (appId, version) => {
+exports.clientLibraryPath = (appId, version, ctx) => {
   if (env.isProd()) {
+    // TODO: remove - for beta testing UI
+    if (ctx && ctx.cookies.get("beta:design_ui")) {
+      return "https://cdn.budi.live/beta:design_ui/budibase-client.js"
+    }
+
     let url = `${exports.objectStoreUrl()}/${sanitizeKey(
       appId
     )}/budibase-client.js`
+
     // append app version to bust the cache
     if (version) {
       url += `?v=${version}`
diff --git a/packages/server/yarn.lock b/packages/server/yarn.lock
index 82d7943354..7dd2021116 100644
--- a/packages/server/yarn.lock
+++ b/packages/server/yarn.lock
@@ -1028,10 +1028,10 @@
   resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
   integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==
 
-"@budibase/backend-core@1.0.212":
-  version "1.0.212"
-  resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.212.tgz#7ce8bfa8c968f4aa7558bd2dd13ad3e1e3e40c8f"
-  integrity sha512-pCrAHr54d2onSbaUoCWP83LMJnm28PNIpBAmAhi2kNdSfaGTFY/Iw1sbGOE3G/9vNaB+RRXeibKEEPFjToOgAg==
+"@budibase/backend-core@1.0.213":
+  version "1.0.213"
+  resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.213.tgz#da10d014b5e39457413a9b7f6ead54322d482855"
+  integrity sha512-ARqPhrev/da9WNXVIYSXN5M+cYLKSBYL7pvVVcwMXewp6KCR0gdUBHxuksnrmTbxqT43h7Uc/Zg1H/jYc1xQQQ==
   dependencies:
     "@techpass/passport-openidconnect" "0.3.2"
     aws-sdk "2.1030.0"
@@ -1109,12 +1109,12 @@
     svelte-flatpickr "^3.2.3"
     svelte-portal "^1.0.0"
 
-"@budibase/pro@1.0.212":
-  version "1.0.212"
-  resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.212.tgz#d6d2528b9ec2ec47e79d3360d04feeb5d2900d1e"
-  integrity sha512-RcbDmz3pkReUHXAJDPzvgTYy0CBksw55XLbW0wNtDu2HVWP0ZXoiMMJjOxNDwMfL3q4eZitWgISa7QUDisDtMA==
+"@budibase/pro@1.0.213":
+  version "1.0.213"
+  resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.213.tgz#80e6005bec51927d373d278dd8d8672c2f25a4d5"
+  integrity sha512-zhTMPZBv0IkQsdKz1ywnWaxmt/PMrw/EkW1dS8bIOAqHgFTUgawiMGrqrzH43Iw3JemMK7AvtI1EOhs+zrMWVg==
   dependencies:
-    "@budibase/backend-core" "1.0.212"
+    "@budibase/backend-core" "1.0.213"
     node-fetch "^2.6.1"
 
 "@budibase/standard-components@^0.9.139":
diff --git a/packages/worker/.dockerignore b/packages/worker/.dockerignore
new file mode 100644
index 0000000000..93fd77103c
--- /dev/null
+++ b/packages/worker/.dockerignore
@@ -0,0 +1,7 @@
+node_modules
+npm-debug.log
+Dockerfile
+.dockerignore
+.git
+.gitignore
+
diff --git a/packages/worker/yarn.lock b/packages/worker/yarn.lock
index 96cc5c5750..4c394647a7 100644
--- a/packages/worker/yarn.lock
+++ b/packages/worker/yarn.lock
@@ -291,10 +291,10 @@
   resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
   integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==
 
-"@budibase/backend-core@1.0.212":
-  version "1.0.212"
-  resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.212.tgz#7ce8bfa8c968f4aa7558bd2dd13ad3e1e3e40c8f"
-  integrity sha512-pCrAHr54d2onSbaUoCWP83LMJnm28PNIpBAmAhi2kNdSfaGTFY/Iw1sbGOE3G/9vNaB+RRXeibKEEPFjToOgAg==
+"@budibase/backend-core@1.0.213":
+  version "1.0.213"
+  resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.213.tgz#da10d014b5e39457413a9b7f6ead54322d482855"
+  integrity sha512-ARqPhrev/da9WNXVIYSXN5M+cYLKSBYL7pvVVcwMXewp6KCR0gdUBHxuksnrmTbxqT43h7Uc/Zg1H/jYc1xQQQ==
   dependencies:
     "@techpass/passport-openidconnect" "0.3.2"
     aws-sdk "2.1030.0"
@@ -322,12 +322,12 @@
     uuid "8.3.2"
     zlib "1.0.5"
 
-"@budibase/pro@1.0.212":
-  version "1.0.212"
-  resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.212.tgz#d6d2528b9ec2ec47e79d3360d04feeb5d2900d1e"
-  integrity sha512-RcbDmz3pkReUHXAJDPzvgTYy0CBksw55XLbW0wNtDu2HVWP0ZXoiMMJjOxNDwMfL3q4eZitWgISa7QUDisDtMA==
+"@budibase/pro@1.0.213":
+  version "1.0.213"
+  resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.213.tgz#80e6005bec51927d373d278dd8d8672c2f25a4d5"
+  integrity sha512-zhTMPZBv0IkQsdKz1ywnWaxmt/PMrw/EkW1dS8bIOAqHgFTUgawiMGrqrzH43Iw3JemMK7AvtI1EOhs+zrMWVg==
   dependencies:
-    "@budibase/backend-core" "1.0.212"
+    "@budibase/backend-core" "1.0.213"
     node-fetch "^2.6.1"
 
 "@cspotcode/source-map-consumer@0.8.0":