Merge pull request #6471 from Budibase/infra/deliver-new-ui

Infra/deliver new UI

.github/workflows/budibase_ci.yml

@@ -7,6 +7,7 @@ on:
     branches:
       - master
       - develop 
+      - new-design-ui
  pull_request:
     branches:
       - master
@@ -59,3 +60,19 @@ jobs:
       with:
         install: false
         command: yarn test:e2e:ci
+    
+    - name: Configure AWS Credentials
+      uses: aws-actions/configure-aws-credentials@v1
+      with:
+        aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+        aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+        aws-region: eu-west-1
+
+    - name: Upload to S3
+      if: github.ref == 'refs/heads/new-design-ui' 
+      run: | 
+        tar -czvf new_ui.tar.gz packages/server/assets packages/server/index.html 
+        aws s3 cp new_ui.tar.gz s3://prod-budi-app-assets/beta:design_ui/
+        aws s3 cp packages/client/dist/budibase-client.js s3://prod-budi-app-assets/beta:design_ui/budibase-client.js
+        aws cloudfront create-invalidation --distribution-id E3ELKP4RCEHVLW --paths "/beta:design_ui/*"
+        

packages/backend-core/src/objectStore/index.ts

@@ -294,6 +294,16 @@ export const uploadDirectory = async (
   await Promise.all(uploads)
 }
 
+exports.downloadTarballDirect = async (url: string, path: string) => {
+  path = sanitizeKey(path)
+  const response = await fetch(url)
+  if (!response.ok) {
+    throw new Error(`unexpected response ${response.statusText}`)
+  }
+
+  await streamPipeline(response.body, zlib.Unzip(), tar.extract(path))
+}
+
 export const downloadTarball = async (url: any, bucketName: any, path: any) => {
   bucketName = sanitizeBucket(bucketName)
   path = sanitizeKey(path)

packages/builder/src/pages/builder/app/[application]/_layout.svelte

@@ -1,7 +1,14 @@
 <script>
   import { store, automationStore } from "builderStore"
   import { roles, flags } from "stores/backend"
-  import { Icon, ActionGroup, Tabs, Tab, notifications } from "@budibase/bbui"
+  import {
+    Icon,
+    ActionGroup,
+    Tabs,
+    Tab,
+    notifications,
+    Banner,
+  } from "@budibase/bbui"
   import RevertModal from "components/deploy/RevertModal.svelte"
   import VersionModal from "components/deploy/VersionModal.svelte"
   import DeployNavigation from "components/deploy/DeployNavigation.svelte"
@@ -17,6 +24,7 @@
 
   // Get Package and set store
   let promise = getPackage()
+  let betaAccess = false
 
   // Sync once when you load the app
   let hasSynced = false
@@ -58,10 +66,18 @@
     })
   }
 
+  async function newDesignUi() {
+    await flags.toggleUiFeature("design_ui")
+    window.location.reload()
+  }
+
   onMount(async () => {
     if (!hasSynced && application) {
       try {
         await API.syncApp(application)
+        // check if user has beta access
+        const betaResponse = await API.checkBetaAccess($auth?.user?.email)
+        betaAccess = betaResponse.access
       } catch (error) {
         notifications.error("Failed to sync with production database")
       }
@@ -79,6 +95,15 @@
   <div class="loading" />
 {:then _}
   <div class="root">
+    {#if betaAccess}
+      <Banner
+        extraButtonText="Try New UI (Beta)"
+        extraButtonAction={newDesignUi}
+      >
+        Try the <b>all new</b> budibase design interface. (Not recommended for existing
+        budibase apps)
+      </Banner>
+    {/if}
     <div class="top-nav">
       <div class="topleftnav">
         <button class="home-logo">

packages/builder/src/stores/backend/flags.js

@@ -16,6 +16,9 @@ export function createFlagsStore() {
       })
       await actions.fetch()
     },
+    toggleUiFeature: async feature => {
+      await API.toggleUiFeature({ value: feature })
+    },
   }
 
   return {

packages/builder/vite.config.js

@@ -32,6 +32,9 @@ export default ({ mode }) => {
           process.env.INTERCOM_TOKEN
         ),
         "process.env.SENTRY_DSN": JSON.stringify(process.env.SENTRY_DSN),
+        "process.env.FEATURE_PREVIEW_URL": JSON.stringify(
+          process.env.FEATURE_PREVIEW_URL
+        ),
       }),
     ],
     optimizeDeps: {

packages/frontend-core/src/api/flags.js

@@ -22,4 +22,13 @@ export const buildFlagEndpoints = API => ({
       },
     })
   },
+  /**
+   * Allows us to experimentally toggle a beta UI feature through a cookie.
+   * @param value the feature to toggle
+   */
+  toggleUiFeature: async ({ value }) => {
+    return await API.post({
+      url: `/api/beta/${value}`,
+    })
+  },
 })

packages/frontend-core/src/api/other.js

@@ -54,4 +54,13 @@ export const buildOtherEndpoints = API => ({
       url: "/api/permission/builtin",
     })
   },
+
+  /**
+   * Check if they are part of the budibase beta program.
+   */
+  checkBetaAccess: async email => {
+    return await API.get({
+      url: `${process.env.FEATURE_PREVIEW_URL}/api/beta/access?email=${email}`,
+    })
+  },
 })

packages/server/.gitignore

@@ -2,6 +2,7 @@ node_modules/
 myapps/
 .env
 builder/*
+new_design_ui/*
 client/*
 db/dev.db/
 dist

packages/server/src/api/controllers/application.ts

@@ -227,7 +227,11 @@ export const fetchAppPackage = async (ctx: any) => {
     application,
     screens,
     layouts,
-    clientLibPath: clientLibraryPath(ctx.params.appId, application.version),
+    clientLibPath: clientLibraryPath(
+      ctx.params.appId,
+      application.version,
+      ctx
+    ),
   }
 }
 

packages/server/src/api/controllers/dev.js

@@ -3,8 +3,12 @@ const env = require("../../environment")
 const { checkSlashesInUrl } = require("../../utilities")
 const { request } = require("../../utilities/workerRequests")
 const { clearLock } = require("../../utilities/redis")
-const { Replication, getProdAppID } = require("@budibase/backend-core/db")
+const {
-const { DocumentTypes } = require("../../db/utils")
+  Replication,
+  getProdAppID,
+  dangerousGetDB,
+} = require("@budibase/backend-core/db")
+const { DocumentTypes, getRowParams } = require("../../db/utils")
 const { app: appCache } = require("@budibase/backend-core/cache")
 const { getProdAppDB, getAppDB } = require("@budibase/backend-core/context")
 const { events } = require("@budibase/backend-core")
@@ -133,3 +137,32 @@ exports.getBudibaseVersion = async ctx => {
   }
   await events.installation.versionChecked(version)
 }
+
+// TODO: remove as part of beta program
+exports.checkBetaAccess = async ctx => {
+  const userToCheck = ctx.query.email
+  const BETA_USERS_DB = "app_bb_f9b77d06b9db4e3ca185476ab87a2364"
+  const BETA_USERS_TABLE = "ta_8c2c6df1c03f49cfb6340e85e066dd15"
+
+  try {
+    const db = dangerousGetDB(BETA_USERS_DB)
+    const betaUsers = (
+      await db.allDocs(
+        getRowParams(BETA_USERS_TABLE, null, {
+          include_docs: true,
+        })
+      )
+    ).rows.map(row => row.doc)
+
+    let access = false
+    for (let betaUser of betaUsers) {
+      if (betaUser["Email address"].trim() === userToCheck) {
+        access = true
+        break
+      }
+    }
+    ctx.body = { access }
+  } catch (err) {
+    ctx.body = { access: false }
+  }
+}

packages/server/src/api/controllers/static/index.js

@@ -16,9 +16,15 @@ const { upload } = require("../../../utilities/fileSystem")
 const { attachmentsRelativeURL } = require("../../../utilities")
 const { DocumentTypes, isDevAppID } = require("../../../db/utils")
 const { getAppDB, getAppId } = require("@budibase/backend-core/context")
+const { setCookie, clearCookie } = require("@budibase/backend-core/utils")
 const AWS = require("aws-sdk")
 const { events } = require("@budibase/backend-core")
 
+const fs = require("fs")
+const {
+  downloadTarballDirect,
+} = require("../../../utilities/fileSystem/utilities")
+
 async function prepareUpload({ s3Key, bucket, metadata, file }) {
   const response = await upload({
     bucket,
@@ -38,8 +44,41 @@ async function prepareUpload({ s3Key, bucket, metadata, file }) {
   }
 }
 
+exports.toggleBetaUiFeature = async function (ctx) {
+  const cookieName = `beta:${ctx.params.feature}`
+
+  if (ctx.cookies.get(cookieName)) {
+    clearCookie(ctx, cookieName)
+    ctx.body = {
+      message: `${ctx.params.feature} disabled`,
+    }
+    return
+  }
+
+  let builderPath = resolve(TOP_LEVEL_PATH, "new_design_ui")
+
+  // // download it from S3
+  if (!fs.existsSync(builderPath)) {
+    fs.mkdirSync(builderPath)
+  }
+  await downloadTarballDirect(
+    "https://cdn.budi.live/beta:design_ui/new_ui.tar.gz",
+    builderPath
+  )
+  setCookie(ctx, {}, cookieName)
+
+  ctx.body = {
+    message: `${ctx.params.feature} enabled`,
+  }
+}
+
 exports.serveBuilder = async function (ctx) {
-  let builderPath = resolve(TOP_LEVEL_PATH, "builder")
+  // Temporary: New Design UI
+  const designUiCookie = ctx.cookies.get("beta:design_ui")
+  // TODO: get this from the tmp Dir that we downloaded from MinIO
+  const uiPath = designUiCookie ? "new_design_ui" : "builder"
+
+  let builderPath = resolve(TOP_LEVEL_PATH, uiPath)
   await send(ctx, ctx.file, { root: builderPath })
   if (!ctx.file.includes("assets/")) {
     await events.serve.servedBuilder()
@@ -78,7 +117,7 @@ exports.serveApp = async function (ctx) {
       title: appInfo.name,
       production: env.isProd(),
       appId,
-      clientLibPath: clientLibraryPath(appId, appInfo.version),
+      clientLibPath: clientLibraryPath(appId, appInfo.version, ctx),
     })
 
     const appHbs = loadHandlebarsFile(`${__dirname}/templates/app.hbs`)

packages/server/src/api/routes/dev.js

@@ -22,5 +22,6 @@ router
   .get("/api/dev/version", authorized(BUILDER), controller.getBudibaseVersion)
   .delete("/api/dev/:appId/lock", authorized(BUILDER), controller.clearLock)
   .post("/api/dev/:appId/revert", authorized(BUILDER), controller.revert)
+  .get("/api/beta/access", controller.checkBetaAccess)
 
 module.exports = router

packages/server/src/api/routes/static.ts

@@ -38,6 +38,7 @@ router
   // TODO: for now this builder endpoint is not authorized/secured, will need to be
   .get("/builder/:file*", controller.serveBuilder)
   .post("/api/attachments/process", authorized(BUILDER), controller.uploadFile)
+  .post("/api/beta/:feature", controller.toggleBetaUiFeature)
   .post(
     "/api/attachments/:tableId/upload",
     paramResource("tableId"),

packages/server/src/utilities/fileSystem/utilities.js

@@ -9,6 +9,7 @@ const {
   deleteFolder,
   uploadDirectory,
   downloadTarball,
+  downloadTarballDirect,
 } = require("@budibase/backend-core/objectStore")
 
 /***********************************
@@ -29,4 +30,5 @@ exports.retrieveToTmp = retrieveToTmp
 exports.deleteFolder = deleteFolder
 exports.uploadDirectory = uploadDirectory
 exports.downloadTarball = downloadTarball
+exports.downloadTarballDirect = downloadTarballDirect
 exports.deleteFiles = deleteFiles

packages/server/src/utilities/index.js

@@ -54,11 +54,17 @@ exports.objectStoreUrl = () => {
  * @return {string} The URL to be inserted into appPackage response or server rendered
  * app index file.
  */
-exports.clientLibraryPath = (appId, version) => {
+exports.clientLibraryPath = (appId, version, ctx) => {
   if (env.isProd()) {
+    // TODO: remove - for beta testing UI
+    if (ctx && ctx.cookies.get("beta:design_ui")) {
+      return "https://cdn.budi.live/beta:design_ui/budibase-client.js"
+    }
+
     let url = `${exports.objectStoreUrl()}/${sanitizeKey(
       appId
     )}/budibase-client.js`
+
     // append app version to bust the cache
     if (version) {
       url += `?v=${version}`

packages/server/yarn.lock

@@ -1094,11 +1094,12 @@
   resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
   integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==
 
-"@budibase/backend-core@1.0.211":
+"@budibase/backend-core@1.0.212-alpha.0":
-  version "1.0.211"
+  version "1.0.212-alpha.0"
-  resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.211.tgz#579bcc97acf1df2510e302bb70b43245e5ab0e37"
+  resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.212-alpha.0.tgz#6ac3448c8272e918f1af1fff0cf8c5773ae61219"
-  integrity sha512-ham+Qk6WXQi37Lgnz1Gh/+ItcvCo+dnPZDcVvPESrVxqQkVVIejlxQinZSueIrsgOYHE1j3TUH5zER5M2t6RWw==
+  integrity sha512-hFvbQQEbF3w2u9fe/S+RhNw5HUETS6rhu9q5KDTDQ57k05D4YMPcpMBGSh7SPMqmVyEwUDgcL36mkFOc3AgjYQ==
   dependencies:
+    "@budibase/types" "^1.0.212-alpha.0"
     "@techpass/passport-openidconnect" "0.3.2"
     aws-sdk "2.1030.0"
     bcrypt "5.0.1"
@@ -1175,12 +1176,12 @@
     svelte-flatpickr "^3.2.3"
     svelte-portal "^1.0.0"
 
-"@budibase/pro@1.0.211":
+"@budibase/pro@1.0.212-alpha.0":
-  version "1.0.211"
+  version "1.0.212-alpha.0"
-  resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.211.tgz#d362e9af8c15f6ed386f27b7cca95cc096a91344"
+  resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.212-alpha.0.tgz#843f8291fcc0a2fbcb4d857a750bc116cdaee293"
-  integrity sha512-dfFByJhlTIURT3sXei5mVXq5rczFMM/ij2Scze0uqPZNpmIlWVqiYdGX7/HEcmIFSS0+UfcdBxjCJJlTGiK4/w==
+  integrity sha512-4nhWxjMcxSQBPXRy/U+37IaVLYOr4/RVe79/fUvnXrr5qAeecbEk/QbkJJd3dU1WaNxB2eGhNtH3uBUPQvcT9A==
   dependencies:
-    "@budibase/backend-core" "1.0.211"
+    "@budibase/backend-core" "1.0.212-alpha.0"
     node-fetch "^2.6.1"
 
 "@budibase/standard-components@^0.9.139":
@@ -1201,6 +1202,11 @@
     svelte-apexcharts "^1.0.2"
     svelte-flatpickr "^3.1.0"
 
+"@budibase/types@^1.0.212-alpha.0":
+  version "1.0.212"
+  resolved "https://registry.yarnpkg.com/@budibase/types/-/types-1.0.212.tgz#e66a15b711544b4fab7767261fd5f2f1dd7f40d7"
+  integrity sha512-DhGyw6snwJQZQlx7havVYnqPZfZERueKZfmVCBySzwInZZt0+sXZaBl1BVjGjYuwpaUQBMDBf7geBgHXp6DIKg==
+
 "@bull-board/api@3.7.0":
   version "3.7.0"
   resolved "https://registry.yarnpkg.com/@bull-board/api/-/api-3.7.0.tgz#231f687187c0cb34e0b97f463917b6aaeb4ef6af"

packages/worker/.dockerignore

@@ -0,0 +1,7 @@
+node_modules
+npm-debug.log
+Dockerfile
+.dockerignore
+.git
+.gitignore
+

packages/worker/yarn.lock

@@ -291,11 +291,12 @@
   resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
   integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==
 
-"@budibase/backend-core@1.0.207-alpha.3":
+"@budibase/backend-core@1.0.212-alpha.0":
-  version "1.0.207-alpha.3"
+  version "1.0.212-alpha.0"
-  resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.207-alpha.3.tgz#98bced0575ec4e2b158239a8c73b39ca2d816719"
+  resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.212-alpha.0.tgz#6ac3448c8272e918f1af1fff0cf8c5773ae61219"
-  integrity sha512-DU4X6jJ+DfhzOv4TTa1w4Dk5ZEdlK/z1joCTruT+SGM5qI75bXrGeol5OX2OaEbNKtXFKJ1zeVTmBCYcu7OFUg==
+  integrity sha512-hFvbQQEbF3w2u9fe/S+RhNw5HUETS6rhu9q5KDTDQ57k05D4YMPcpMBGSh7SPMqmVyEwUDgcL36mkFOc3AgjYQ==
   dependencies:
+    "@budibase/types" "^1.0.212-alpha.0"
     "@techpass/passport-openidconnect" "0.3.2"
     aws-sdk "2.1030.0"
     bcrypt "5.0.1"
@@ -322,14 +323,19 @@
     uuid "8.3.2"
     zlib "1.0.5"
 
-"@budibase/pro@1.0.207-alpha.3":
+"@budibase/pro@1.0.212-alpha.0":
-  version "1.0.207-alpha.3"
+  version "1.0.212-alpha.0"
-  resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.207-alpha.3.tgz#9bde845ceb685f1b43286a124620c21fdf891a01"
+  resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.212-alpha.0.tgz#843f8291fcc0a2fbcb4d857a750bc116cdaee293"
-  integrity sha512-WFEMujpKTVAMvAgLBnMdw8ou9PxsbM4Oa9Dq+DAUsWpPACsMWOProyHLsdRxJyvHlgGfwVjo5MEusvStjI4j6g==
+  integrity sha512-4nhWxjMcxSQBPXRy/U+37IaVLYOr4/RVe79/fUvnXrr5qAeecbEk/QbkJJd3dU1WaNxB2eGhNtH3uBUPQvcT9A==
   dependencies:
-    "@budibase/backend-core" "1.0.207-alpha.3"
+    "@budibase/backend-core" "1.0.212-alpha.0"
     node-fetch "^2.6.1"
 
+"@budibase/types@^1.0.212-alpha.0":
+  version "1.0.212"
+  resolved "https://registry.yarnpkg.com/@budibase/types/-/types-1.0.212.tgz#e66a15b711544b4fab7767261fd5f2f1dd7f40d7"
+  integrity sha512-DhGyw6snwJQZQlx7havVYnqPZfZERueKZfmVCBySzwInZZt0+sXZaBl1BVjGjYuwpaUQBMDBf7geBgHXp6DIKg==
+
 "@cspotcode/source-map-consumer@0.8.0":
   version "0.8.0"
   resolved "https://registry.yarnpkg.com/@cspotcode/source-map-consumer/-/source-map-consumer-0.8.0.tgz#33bf4b7b39c178821606f669bbc447a6a629786b"