Merge pull request #6513 from Budibase/infra/beta-ui

Infra/beta UI

.github/workflows/budibase_ci.yml

@@ -7,6 +7,7 @@ on:
     branches:
       - master
       - develop 
+      - new-design-ui
  pull_request:
     branches:
       - master
@@ -59,3 +60,19 @@ jobs:
       with:
         install: false
         command: yarn test:e2e:ci
+    
+    - name: Configure AWS Credentials
+      uses: aws-actions/configure-aws-credentials@v1
+      with:
+        aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+        aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+        aws-region: eu-west-1
+
+    - name: Upload to S3
+      if: github.ref == 'refs/heads/new-design-ui' 
+      run: | 
+        tar -czvf new_ui.tar.gz packages/server/assets packages/server/index.html 
+        aws s3 cp new_ui.tar.gz s3://prod-budi-app-assets/beta:design_ui/
+        aws s3 cp packages/client/dist/budibase-client.js s3://prod-budi-app-assets/beta:design_ui/budibase-client.js
+        aws cloudfront create-invalidation --distribution-id E3ELKP4RCEHVLW --paths "/beta:design_ui/*"
+        

.github/workflows/release-develop.yml

@@ -21,7 +21,7 @@ env:
   # Posthog token used by ui at build time 
   POSTHOG_TOKEN: phc_uDYOfnFt6wAbBAXkC6STjcrTpAFiWIhqgFcsC1UVO5F
   INTERCOM_TOKEN: ${{ secrets.INTERCOM_TOKEN }}
-  PERSONAL_ACCESS_TOKEN : ${{ secrets.PERSONAL_ACCESS_TOKEN }}
+  PERSONAL_ACCESS_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
       
 jobs:
   release:

packages/backend-core/src/objectStore/index.ts

@@ -294,6 +294,16 @@ export const uploadDirectory = async (
   await Promise.all(uploads)
 }
 
+exports.downloadTarballDirect = async (url: string, path: string) => {
+  path = sanitizeKey(path)
+  const response = await fetch(url)
+  if (!response.ok) {
+    throw new Error(`unexpected response ${response.statusText}`)
+  }
+
+  await streamPipeline(response.body, zlib.Unzip(), tar.extract(path))
+}
+
 export const downloadTarball = async (url: any, bucketName: any, path: any) => {
   bucketName = sanitizeBucket(bucketName)
   path = sanitizeKey(path)

packages/builder/src/pages/builder/app/[application]/_layout.svelte

@@ -1,7 +1,14 @@
 <script>
   import { store, automationStore } from "builderStore"
   import { roles, flags } from "stores/backend"
-  import { Icon, ActionGroup, Tabs, Tab, notifications } from "@budibase/bbui"
+  import {
+    Icon,
+    ActionGroup,
+    Tabs,
+    Tab,
+    notifications,
+    Banner,
+  } from "@budibase/bbui"
   import RevertModal from "components/deploy/RevertModal.svelte"
   import VersionModal from "components/deploy/VersionModal.svelte"
   import DeployNavigation from "components/deploy/DeployNavigation.svelte"
@@ -17,6 +24,7 @@
 
   // Get Package and set store
   let promise = getPackage()
+  let betaAccess = false
 
   // Sync once when you load the app
   let hasSynced = false
@@ -58,10 +66,18 @@
     })
   }
 
+  async function newDesignUi() {
+    await flags.toggleUiFeature("design_ui")
+    window.location.reload()
+  }
+
   onMount(async () => {
     if (!hasSynced && application) {
       try {
         await API.syncApp(application)
+        // check if user has beta access
+        const betaResponse = await API.checkBetaAccess($auth?.user?.email)
+        betaAccess = betaResponse.access
       } catch (error) {
         notifications.error("Failed to sync with production database")
       }
@@ -79,6 +95,15 @@
   <div class="loading" />
 {:then _}
   <div class="root">
+    {#if betaAccess}
+      <Banner
+        extraButtonText="Try New UI (Beta)"
+        extraButtonAction={newDesignUi}
+      >
+        Try the <b>all new</b> budibase design interface. (Not recommended for existing
+        budibase apps)
+      </Banner>
+    {/if}
     <div class="top-nav">
       <div class="topleftnav">
         <button class="home-logo">

packages/builder/src/stores/backend/flags.js

@@ -16,6 +16,9 @@ export function createFlagsStore() {
       })
       await actions.fetch()
     },
+    toggleUiFeature: async feature => {
+      await API.toggleUiFeature({ value: feature })
+    },
   }
 
   return {

packages/frontend-core/src/api/flags.js

@@ -22,4 +22,13 @@ export const buildFlagEndpoints = API => ({
       },
     })
   },
+  /**
+   * Allows us to experimentally toggle a beta UI feature through a cookie.
+   * @param value the feature to toggle
+   */
+  toggleUiFeature: async ({ value }) => {
+    return await API.post({
+      url: `/api/beta/${value}`,
+    })
+  },
 })

packages/frontend-core/src/api/other.js

@@ -54,4 +54,13 @@ export const buildOtherEndpoints = API => ({
       url: "/api/permission/builtin",
     })
   },
+
+  /**
+   * Check if they are part of the budibase beta program.
+   */
+  checkBetaAccess: async email => {
+    return await API.get({
+      url: `/api/beta/access?email=${email}`,
+    })
+  },
 })

packages/server/.gitignore

@@ -2,6 +2,7 @@ node_modules/
 myapps/
 .env
 builder/*
+new_design_ui/*
 client/*
 db/dev.db/
 dist

packages/server/src/api/controllers/application.ts

@@ -227,7 +227,11 @@ export const fetchAppPackage = async (ctx: any) => {
     application,
     screens,
     layouts,
-    clientLibPath: clientLibraryPath(ctx.params.appId, application.version),
+    clientLibPath: clientLibraryPath(
+      ctx.params.appId,
+      application.version,
+      ctx
+    ),
   }
 }
 

packages/server/src/api/controllers/dev.js

@@ -3,8 +3,12 @@ const env = require("../../environment")
 const { checkSlashesInUrl } = require("../../utilities")
 const { request } = require("../../utilities/workerRequests")
 const { clearLock } = require("../../utilities/redis")
-const { Replication, getProdAppID } = require("@budibase/backend-core/db")
+const {
-const { DocumentTypes } = require("../../db/utils")
+  Replication,
+  getProdAppID,
+  dangerousGetDB,
+} = require("@budibase/backend-core/db")
+const { DocumentTypes, getRowParams } = require("../../db/utils")
 const { app: appCache } = require("@budibase/backend-core/cache")
 const { getProdAppDB, getAppDB } = require("@budibase/backend-core/context")
 const { events } = require("@budibase/backend-core")
@@ -133,3 +137,50 @@ exports.getBudibaseVersion = async ctx => {
   }
   await events.installation.versionChecked(version)
 }
+
+// TODO: remove as part of beta program
+exports.checkBetaAccess = async ctx => {
+  // go to the cloud platform if running self hosted
+  if (env.SELF_HOSTED || !env.MULTI_TENANCY) {
+    let baseUrl = ""
+    if (env.ACCOUNT_PORTAL_URL) {
+      baseUrl = env.ACCOUNT_PORTAL_URL.replace("account.", "")
+    } else {
+      baseUrl = "https://budibase.app"
+    }
+
+    const response = await fetch(
+      `${baseUrl}/api/beta/access?email=${ctx.query.email}`
+    )
+    const json = await response.json()
+    ctx.body = json
+    return
+  }
+
+  const userToCheck = ctx.query.email
+  const BETA_USERS_DB = "app_bb_f9b77d06b9db4e3ca185476ab87a2364"
+  const BETA_USERS_TABLE = "ta_8c2c6df1c03f49cfb6340e85e066dd15"
+
+  try {
+    const db = dangerousGetDB(BETA_USERS_DB)
+    const betaUsers = (
+      await db.allDocs(
+        getRowParams(BETA_USERS_TABLE, null, {
+          include_docs: true,
+        })
+      )
+    ).rows.map(row => row.doc)
+
+    let access = false
+    for (let betaUser of betaUsers) {
+      if (betaUser["Email address"].trim() === userToCheck) {
+        access = true
+        break
+      }
+    }
+    ctx.body = { access }
+  } catch (err) {
+    console.error(err)
+    ctx.body = { access: false }
+  }
+}

packages/server/src/api/controllers/static/index.js

@@ -16,9 +16,15 @@ const { upload } = require("../../../utilities/fileSystem")
 const { attachmentsRelativeURL } = require("../../../utilities")
 const { DocumentTypes, isDevAppID } = require("../../../db/utils")
 const { getAppDB, getAppId } = require("@budibase/backend-core/context")
+const { setCookie, clearCookie } = require("@budibase/backend-core/utils")
 const AWS = require("aws-sdk")
 const { events } = require("@budibase/backend-core")
 
+const fs = require("fs")
+const {
+  downloadTarballDirect,
+} = require("../../../utilities/fileSystem/utilities")
+
 async function prepareUpload({ s3Key, bucket, metadata, file }) {
   const response = await upload({
     bucket,
@@ -38,8 +44,41 @@ async function prepareUpload({ s3Key, bucket, metadata, file }) {
   }
 }
 
+exports.toggleBetaUiFeature = async function (ctx) {
+  const cookieName = `beta:${ctx.params.feature}`
+
+  if (ctx.cookies.get(cookieName)) {
+    clearCookie(ctx, cookieName)
+    ctx.body = {
+      message: `${ctx.params.feature} disabled`,
+    }
+    return
+  }
+
+  let builderPath = resolve(TOP_LEVEL_PATH, "new_design_ui")
+
+  // // download it from S3
+  if (!fs.existsSync(builderPath)) {
+    fs.mkdirSync(builderPath)
+  }
+  await downloadTarballDirect(
+    "https://cdn.budi.live/beta:design_ui/new_ui.tar.gz",
+    builderPath
+  )
+  setCookie(ctx, {}, cookieName)
+
+  ctx.body = {
+    message: `${ctx.params.feature} enabled`,
+  }
+}
+
 exports.serveBuilder = async function (ctx) {
-  let builderPath = resolve(TOP_LEVEL_PATH, "builder")
+  // Temporary: New Design UI
+  const designUiCookie = ctx.cookies.get("beta:design_ui")
+  // TODO: get this from the tmp Dir that we downloaded from MinIO
+  const uiPath = designUiCookie ? "new_design_ui" : "builder"
+
+  let builderPath = resolve(TOP_LEVEL_PATH, uiPath)
   await send(ctx, ctx.file, { root: builderPath })
   if (!ctx.file.includes("assets/")) {
     await events.serve.servedBuilder()
@@ -78,7 +117,7 @@ exports.serveApp = async function (ctx) {
       title: appInfo.name,
       production: env.isProd(),
       appId,
-      clientLibPath: clientLibraryPath(appId, appInfo.version),
+      clientLibPath: clientLibraryPath(appId, appInfo.version, ctx),
     })
 
     const appHbs = loadHandlebarsFile(`${__dirname}/templates/app.hbs`)

packages/server/src/api/routes/dev.js

@@ -22,5 +22,6 @@ router
   .get("/api/dev/version", authorized(BUILDER), controller.getBudibaseVersion)
   .delete("/api/dev/:appId/lock", authorized(BUILDER), controller.clearLock)
   .post("/api/dev/:appId/revert", authorized(BUILDER), controller.revert)
+  .get("/api/beta/access", controller.checkBetaAccess)
 
 module.exports = router

packages/server/src/api/routes/static.ts

@@ -38,6 +38,7 @@ router
   // TODO: for now this builder endpoint is not authorized/secured, will need to be
   .get("/builder/:file*", controller.serveBuilder)
   .post("/api/attachments/process", authorized(BUILDER), controller.uploadFile)
+  .post("/api/beta/:feature", controller.toggleBetaUiFeature)
   .post(
     "/api/attachments/:tableId/upload",
     paramResource("tableId"),

packages/server/src/environment.js

@@ -67,7 +67,9 @@ module.exports = {
   SALT_ROUNDS: process.env.SALT_ROUNDS,
   LOGGER: process.env.LOGGER,
   LOG_LEVEL: process.env.LOG_LEVEL,
-  AUTOMATION_MAX_ITERATIONS: process.env.AUTOMATION_MAX_ITERATIONS,
+  ACCOUNT_PORTAL_URL: process.env.ACCOUNT_PORTAL_URL,
+  AUTOMATION_MAX_ITERATIONS:
+    parseIntSafe(process.env.AUTOMATION_MAX_ITERATIONS) || 200,
   SENDGRID_API_KEY: process.env.SENDGRID_API_KEY,
   DYNAMO_ENDPOINT: process.env.DYNAMO_ENDPOINT,
   QUERY_THREAD_TIMEOUT: parseIntSafe(process.env.QUERY_THREAD_TIMEOUT),

packages/server/src/utilities/fileSystem/utilities.js

@@ -9,6 +9,7 @@ const {
   deleteFolder,
   uploadDirectory,
   downloadTarball,
+  downloadTarballDirect,
 } = require("@budibase/backend-core/objectStore")
 
 /***********************************
@@ -29,4 +30,5 @@ exports.retrieveToTmp = retrieveToTmp
 exports.deleteFolder = deleteFolder
 exports.uploadDirectory = uploadDirectory
 exports.downloadTarball = downloadTarball
+exports.downloadTarballDirect = downloadTarballDirect
 exports.deleteFiles = deleteFiles

packages/server/src/utilities/index.js

@@ -54,11 +54,17 @@ exports.objectStoreUrl = () => {
  * @return {string} The URL to be inserted into appPackage response or server rendered
  * app index file.
  */
-exports.clientLibraryPath = (appId, version) => {
+exports.clientLibraryPath = (appId, version, ctx) => {
   if (env.isProd()) {
+    // TODO: remove - for beta testing UI
+    if (ctx && ctx.cookies.get("beta:design_ui")) {
+      return "https://cdn.budi.live/beta:design_ui/budibase-client.js"
+    }
+
     let url = `${exports.objectStoreUrl()}/${sanitizeKey(
       appId
     )}/budibase-client.js`
+
     // append app version to bust the cache
     if (version) {
       url += `?v=${version}`

packages/server/yarn.lock

@@ -1028,10 +1028,10 @@
   resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
   integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==
 
-"@budibase/backend-core@1.0.212":
+"@budibase/backend-core@1.0.213":
-  version "1.0.212"
+  version "1.0.213"
-  resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.212.tgz#7ce8bfa8c968f4aa7558bd2dd13ad3e1e3e40c8f"
+  resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.213.tgz#da10d014b5e39457413a9b7f6ead54322d482855"
-  integrity sha512-pCrAHr54d2onSbaUoCWP83LMJnm28PNIpBAmAhi2kNdSfaGTFY/Iw1sbGOE3G/9vNaB+RRXeibKEEPFjToOgAg==
+  integrity sha512-ARqPhrev/da9WNXVIYSXN5M+cYLKSBYL7pvVVcwMXewp6KCR0gdUBHxuksnrmTbxqT43h7Uc/Zg1H/jYc1xQQQ==
   dependencies:
     "@techpass/passport-openidconnect" "0.3.2"
     aws-sdk "2.1030.0"
@@ -1109,12 +1109,12 @@
     svelte-flatpickr "^3.2.3"
     svelte-portal "^1.0.0"
 
-"@budibase/pro@1.0.212":
+"@budibase/pro@1.0.213":
-  version "1.0.212"
+  version "1.0.213"
-  resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.212.tgz#d6d2528b9ec2ec47e79d3360d04feeb5d2900d1e"
+  resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.213.tgz#80e6005bec51927d373d278dd8d8672c2f25a4d5"
-  integrity sha512-RcbDmz3pkReUHXAJDPzvgTYy0CBksw55XLbW0wNtDu2HVWP0ZXoiMMJjOxNDwMfL3q4eZitWgISa7QUDisDtMA==
+  integrity sha512-zhTMPZBv0IkQsdKz1ywnWaxmt/PMrw/EkW1dS8bIOAqHgFTUgawiMGrqrzH43Iw3JemMK7AvtI1EOhs+zrMWVg==
   dependencies:
-    "@budibase/backend-core" "1.0.212"
+    "@budibase/backend-core" "1.0.213"
     node-fetch "^2.6.1"
 
 "@budibase/standard-components@^0.9.139":

packages/worker/.dockerignore

@@ -0,0 +1,7 @@
+node_modules
+npm-debug.log
+Dockerfile
+.dockerignore
+.git
+.gitignore
+

packages/worker/yarn.lock

@@ -291,10 +291,10 @@
   resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
   integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==
 
-"@budibase/backend-core@1.0.212":
+"@budibase/backend-core@1.0.213":
-  version "1.0.212"
+  version "1.0.213"
-  resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.212.tgz#7ce8bfa8c968f4aa7558bd2dd13ad3e1e3e40c8f"
+  resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.213.tgz#da10d014b5e39457413a9b7f6ead54322d482855"
-  integrity sha512-pCrAHr54d2onSbaUoCWP83LMJnm28PNIpBAmAhi2kNdSfaGTFY/Iw1sbGOE3G/9vNaB+RRXeibKEEPFjToOgAg==
+  integrity sha512-ARqPhrev/da9WNXVIYSXN5M+cYLKSBYL7pvVVcwMXewp6KCR0gdUBHxuksnrmTbxqT43h7Uc/Zg1H/jYc1xQQQ==
   dependencies:
     "@techpass/passport-openidconnect" "0.3.2"
     aws-sdk "2.1030.0"
@@ -322,12 +322,12 @@
     uuid "8.3.2"
     zlib "1.0.5"
 
-"@budibase/pro@1.0.212":
+"@budibase/pro@1.0.213":
-  version "1.0.212"
+  version "1.0.213"
-  resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.212.tgz#d6d2528b9ec2ec47e79d3360d04feeb5d2900d1e"
+  resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.213.tgz#80e6005bec51927d373d278dd8d8672c2f25a4d5"
-  integrity sha512-RcbDmz3pkReUHXAJDPzvgTYy0CBksw55XLbW0wNtDu2HVWP0ZXoiMMJjOxNDwMfL3q4eZitWgISa7QUDisDtMA==
+  integrity sha512-zhTMPZBv0IkQsdKz1ywnWaxmt/PMrw/EkW1dS8bIOAqHgFTUgawiMGrqrzH43Iw3JemMK7AvtI1EOhs+zrMWVg==
   dependencies:
-    "@budibase/backend-core" "1.0.212"
+    "@budibase/backend-core" "1.0.213"
     node-fetch "^2.6.1"
 
 "@cspotcode/source-map-consumer@0.8.0":