pulling from master

.github/workflows/release-develop.yml

@@ -26,7 +26,7 @@ env:
   FEATURE_PREVIEW_URL: https://budirelease.live
       
 jobs:
-  release:
+  release-images:
     runs-on: ubuntu-latest 
 
     steps:
@@ -44,19 +44,12 @@ jobs:
         run: yarn install:pro develop
 
       - run: yarn 
-      - run: yarn bootstrap 
+      - run: yarn bootstrap
       - run: yarn lint 
       - run: yarn build
       - run: yarn build:sdk
       - run: yarn test
 
-      - name: Configure AWS Credentials
-        uses: aws-actions/configure-aws-credentials@v1
-        with:
-          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-          aws-region: eu-west-1
-
       - name: Publish budibase packages to NPM
         env:
           NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
@@ -76,12 +69,6 @@ jobs:
           DOCKER_USER: ${{ secrets.DOCKER_USERNAME }}
           DOCKER_PASSWORD: ${{ secrets.DOCKER_API_KEY }}
 
-      - name: Get the latest budibase release version
-        id: version
-        run: | 
-          release_version=$(cat lerna.json | jq -r '.version')
-          echo "RELEASE_VERSION=$release_version" >> $GITHUB_ENV
-
       - name: Tag and release Proxy service docker image
         run: | 
           docker login -u $DOCKER_USER -p $DOCKER_PASSWORD
@@ -93,6 +80,26 @@ jobs:
           DOCKER_PASSWORD: ${{ secrets.DOCKER_API_KEY }}
           RELEASE_TAG: k8s-release
 
+  deploy-to-release-env:
+    needs: [release-images]
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Get the current budibase release version
+        id: version
+        run: |
+          release_version=$(cat lerna.json | jq -r '.version')
+          echo "RELEASE_VERSION=$release_version" >> $GITHUB_ENV
+
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: eu-west-1
+
       - name: Pull values.yaml from budibase-infra
         run: | 
           curl -H "Authorization: token ${{ secrets.GH_PERSONAL_TOKEN }}" \
@@ -149,3 +156,53 @@ jobs:
           webhook-url: ${{ secrets.PROD_DEPLOY_WEBHOOK_URL }}
           content: "Release Env Deployment Complete: ${{ env.RELEASE_VERSION }} deployed to Budibase Release Env."
           embed-title: ${{ env.RELEASE_VERSION }}
+
+  release-helm-chart:
+    needs: [release-images]
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Setup Helm
+        uses: azure/setup-helm@v1
+        id: helm-install
+
+      # due to helm repo index issue: https://github.com/helm/helm/issues/7363
+      # we need to create new package in a different dir, merge the index and move the package back
+      - name: Build and release helm chart
+        run: |
+          git config user.name "Budibase Helm Bot"
+          git config user.email "<>"
+          git reset --hard
+          git pull
+          mkdir sync
+          echo "Packaging chart to sync dir"
+          helm package charts/budibase --version 0.0.0-develop --app-version develop --destination sync
+          echo "Packaging successful"
+          git checkout gh-pages
+          echo "Indexing helm repo"         
+          helm repo index --merge docs/index.yaml sync
+          mv -f sync/* docs
+          rm -rf sync
+          echo "Pushing new helm release"
+          git add -A
+          git commit -m "Helm Release: develop"
+          git push
+
+  trigger-deploy-to-qa-env:
+    needs: [release-helm-chart]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Get the current budibase release version
+        id: version
+        run: |
+          release_version=$(cat lerna.json | jq -r '.version')
+          echo "RELEASE_VERSION=$release_version" >> $GITHUB_ENV
+
+      - uses: passeidireto/trigger-external-workflow-action@main
+        env:
+          PAYLOAD_VERSION: ${{ env.RELEASE_VERSION }}
+        with:
+          repository: budibase/budibase-deploys
+          event: deploy-develop-to-qa
+          github_pat: ${{ secrets.GH_ACCESS_TOKEN }}

.github/workflows/release-selfhost.yml

@@ -73,7 +73,7 @@ jobs:
           git config user.email "<>"
           git reset --hard
           git pull
-          helm package charts/budibase
+          helm package charts/budibase --version "$RELEASE_VERSION" --app-version "$RELEASE_VERSION"
           git checkout gh-pages
           mv *.tgz docs
           helm repo index docs

.gitignore

@@ -107,3 +107,5 @@ stats.html
 *.tsbuildinfo
 budibase-component
 budibase-datasource
+
+*.iml

.vscode/settings.json

@@ -8,7 +8,7 @@
         "editor.defaultFormatter": "vscode.json-language-features"
     },
     "[javascript]": {
-        "editor.defaultFormatter": "esbenp.prettier-vscode"
+        "editor.defaultFormatter": "vscode.typescript-language-features"
     },
     "debug.javascript.terminalOptions": {
         "skipFiles": [
@@ -16,4 +16,7 @@
             "<node_internals>/**"
         ]
     },
+    "[typescript]": {
+        "editor.defaultFormatter": "vscode.typescript-language-features"
+    },
 }

charts/budibase/Chart.yaml

@@ -11,8 +11,10 @@ sources:
   - https://github.com/Budibase/budibase
   - https://budibase.com
 type: application
-version: 0.2.11
+# populates on packaging
-appVersion: 1.0.214
+version: 0.0.0
+# populates on packaging
+appVersion: 0.0.0
 dependencies:
   - name: couchdb
     version: 3.6.1

charts/budibase/templates/app-service-deployment.yaml

@@ -67,6 +67,8 @@ spec:
         - name: AWS_REGION
           value: {{ .Values.services.objectStore.region }}
         {{ end }}
+        - name: MINIO_ENABLED
+          value: {{ .Values.services.objectStore.minio | quote }}
         - name: MINIO_ACCESS_KEY
           valueFrom:
             secretKeyRef:
@@ -77,13 +79,19 @@ spec:
             secretKeyRef:
               name: {{ template "budibase.fullname" . }}
               key: objectStoreSecret
+        - name: CLOUDFRONT_CDN
+          value: {{ .Values.services.objectStore.cloudfront.cdn | quote }}
+        - name: CLOUDFRONT_PUBLIC_KEY_ID
+          value: {{ .Values.services.objectStore.cloudfront.publicKeyId | quote }}
+        - name: CLOUDFRONT_PRIVATE_KEY_64
+          value: {{ .Values.services.objectStore.cloudfront.privateKey64 | quote }}
         - name: MINIO_URL
           value: {{ .Values.services.objectStore.url }}
         - name: PLUGIN_BUCKET_NAME
           value: {{ .Values.services.objectStore.pluginBucketName | quote }}
         - name: APPS_BUCKET_NAME
           value: {{ .Values.services.objectStore.appsBucketName | quote }}
-        - name: GLOBAL_CLOUD_BUCKET_NAME
+        - name: GLOBAL_BUCKET_NAME
           value: {{ .Values.services.objectStore.globalBucketName | quote }}
         - name: BACKUPS_BUCKET_NAME
           value: {{ .Values.services.objectStore.backupsBucketName | quote }}

charts/budibase/templates/worker-service-deployment.yaml

@@ -68,6 +68,8 @@ spec:
         - name: AWS_REGION
           value: {{ .Values.services.objectStore.region }}
         {{ end }}
+        - name: MINIO_ENABLED
+          value: {{ .Values.services.objectStore.minio | quote }}
         - name: MINIO_ACCESS_KEY
           valueFrom:
             secretKeyRef:
@@ -80,11 +82,17 @@ spec:
               key: objectStoreSecret
         - name: MINIO_URL
           value: {{ .Values.services.objectStore.url }}
+        - name: CLOUDFRONT_CDN
+          value: {{ .Values.services.objectStore.cloudfront.cdn | quote }}
+        - name: CLOUDFRONT_PUBLIC_KEY_ID
+          value: {{ .Values.services.objectStore.cloudfront.publicKeyId | quote }}
+        - name: CLOUDFRONT_PRIVATE_KEY_64
+          value: {{ .Values.services.objectStore.cloudfront.privateKey64 | quote }}
         - name: PLUGIN_BUCKET_NAME
           value: {{ .Values.services.objectStore.pluginBucketName | quote }}
         - name: APPS_BUCKET_NAME
           value: {{ .Values.services.objectStore.appsBucketName | quote }}
-        - name: GLOBAL_CLOUD_BUCKET_NAME
+        - name: GLOBAL_BUCKET_NAME
           value: {{ .Values.services.objectStore.globalBucketName | quote }}
         - name: BACKUPS_BUCKET_NAME
           value: {{ .Values.services.objectStore.backupsBucketName | quote }}

charts/budibase/values.yaml

@@ -167,6 +167,7 @@ services:
     resources: {}
 
   objectStore:
+    # Set to false if using another object store such as S3
     minio: true
     browser: true
     port: 9000
@@ -182,6 +183,13 @@ services:
     ##   set, choosing the default provisioner.
     storageClass: ""
     resources: {}
+    cloudfront:
+      # Set the url of a distribution to enable cloudfront
+      cdn: ""
+      # ID of public key stored in cloudfront
+      publicKeyId: ""
+      # Base64 encoded private key for the above public key
+      privateKey64: ""
 
 # Override values in couchDB subchart
 couchdb:

hosting/nginx.dev.conf.hbs

@@ -186,6 +186,26 @@ http {
       proxy_pass      http://minio-service:9000;
     }
 
+    location /files/signed/ {
+      proxy_set_header X-Real-IP $remote_addr;
+      proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+      proxy_set_header X-Forwarded-Proto $scheme;
+
+      # IMPORTANT: Signed urls will inspect the host header of the request.
+      # Normally a signed url will need to be generated with a specified client host in mind.
+      # To support dynamic hosts, e.g. some unknown self-hosted installation url,
+      # use a predefined host header. The host 'minio-service' is also used at the time of url signing.
+      proxy_set_header Host minio-service;
+
+      proxy_connect_timeout 300;
+      proxy_http_version 1.1;
+      proxy_set_header Connection "";
+      chunked_transfer_encoding off;
+
+      proxy_pass      http://minio-service:9000;
+      rewrite ^/files/signed/(.*)$ /$1 break;
+    }
+
     client_header_timeout 60;
     client_body_timeout   60;
     keepalive_timeout     60;

hosting/nginx.prod.conf.hbs

@@ -208,6 +208,26 @@ http {
       proxy_pass      http://$minio:9000;
     }
 
+    location /files/signed/ {
+      proxy_set_header X-Real-IP $remote_addr;
+      proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+      proxy_set_header X-Forwarded-Proto $scheme;
+
+      # IMPORTANT: Signed urls will inspect the host header of the request.
+      # Normally a signed url will need to be generated with a specified client host in mind.
+      # To support dynamic hosts, e.g. some unknown self-hosted installation url,
+      # use a predefined host header. The host 'minio-service' is also used at the time of url signing.
+      proxy_set_header Host minio-service;
+
+      proxy_connect_timeout 300;
+      proxy_http_version 1.1;
+      proxy_set_header Connection "";
+      chunked_transfer_encoding off;
+
+      proxy_pass      http://$minio:9000;
+      rewrite ^/files/signed/(.*)$ /$1 break;
+    }
+
     client_header_timeout 60;
     client_body_timeout   60;
     keepalive_timeout     60;

hosting/single/nginx/nginx-default-site.conf

@@ -95,15 +95,37 @@ server {
     }
 
     location / {
-        proxy_set_header X-Real-IP $remote_addr;
+      proxy_set_header X-Real-IP $remote_addr;
-        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+      proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-        proxy_set_header X-Forwarded-Proto $scheme;
+      proxy_set_header X-Forwarded-Proto $scheme;
+      proxy_set_header Host $http_host;
 
-        proxy_connect_timeout 300;
+      proxy_connect_timeout 300;
-        proxy_http_version 1.1;
+      proxy_http_version 1.1;
-        proxy_set_header Connection "";
+      proxy_set_header Connection "";
-        chunked_transfer_encoding off;
+      chunked_transfer_encoding off;
-        proxy_pass      http://127.0.0.1:9000;
+
+      proxy_pass      http://127.0.0.1:9000;
+    }
+
+    location /files/signed/ {
+      proxy_set_header X-Real-IP $remote_addr;
+      proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+      proxy_set_header X-Forwarded-Proto $scheme;
+
+      # IMPORTANT: Signed urls will inspect the host header of the request.
+      # Normally a signed url will need to be generated with a specified client host in mind.
+      # To support dynamic hosts, e.g. some unknown self-hosted installation url,
+      # use a predefined host header. The host 'minio-service' is also used at the time of url signing.
+      proxy_set_header Host minio-service;
+
+      proxy_connect_timeout 300;
+      proxy_http_version 1.1;
+      proxy_set_header Connection "";
+      chunked_transfer_encoding off;
+
+      proxy_pass      http://127.0.0.1:9000;
+      rewrite ^/files/signed/(.*)$ /$1 break;
     }
 
     client_header_timeout 60;

lerna.json

@@ -15,4 +15,4 @@
       ]
     }
   }
-}
+}

package.json

@@ -25,6 +25,7 @@
     "bootstrap": "lerna bootstrap && lerna link && ./scripts/link-dependencies.sh",
     "build": "lerna run build",
     "build:dev": "lerna run prebuild && tsc --build --watch --preserveWatchOutput",
+    "build:backend": "lerna run build --ignore @budibase/client --ignore @budibase/bbui --ignore @budibase/builder --ignore @budibase/cli",
     "build:sdk": "lerna run build:sdk",
     "deps:circular": "madge packages/server/dist/index.js packages/worker/src/index.ts packages/backend-core/dist/src/index.js packages/cli/src/index.js --circular",
     "release": "lerna publish ${RELEASE_VERSION_TYPE:-patch} --yes --force-publish && yarn release:pro",

packages/backend-core/__mocks__/aws-sdk.ts

@@ -3,7 +3,10 @@ const mockS3 = {
   deleteObject: jest.fn().mockReturnThis(),
   deleteObjects: jest.fn().mockReturnThis(),
   createBucket: jest.fn().mockReturnThis(),
-  listObjects: jest.fn().mockReturnThis(),
+  listObject: jest.fn().mockReturnThis(),
+  getSignedUrl: jest.fn((operation: string, params: any) => {
+    return `http://s3.example.com/${params.Bucket}/${params.Key}`
+  }),
   promise: jest.fn().mockReturnThis(),
   catch: jest.fn(),
 }

packages/backend-core/package.json

@@ -24,6 +24,7 @@
     "@budibase/types": "^2.2.9",
     "@shopify/jest-koa-mocks": "5.0.1",
     "@techpass/passport-openidconnect": "0.3.2",
+    "aws-cloudfront-sign": "2.2.0",
     "aws-sdk": "2.1030.0",
     "bcrypt": "5.0.1",
     "bcryptjs": "2.4.3",
@@ -78,4 +79,4 @@
     "typescript": "4.7.3"
   },
   "gitHead": "d1836a898cab3f8ab80ee6d8f42be1a9eed7dcdc"
-}
+}

packages/backend-core/src/context/mainContext.ts

@@ -2,7 +2,7 @@
 // store an app ID to pretend there is a context
 import env from "../environment"
 import Context from "./Context"
-import { getDevelopmentAppID, getProdAppID } from "../db/conversions"
+import * as conversions from "../db/conversions"
 import { getDB } from "../db/db"
 import {
   DocumentType,
@@ -181,6 +181,14 @@ export function getAppId(): string | undefined {
   }
 }
 
+export const getProdAppId = () => {
+  const appId = getAppId()
+  if (!appId) {
+    throw new Error("Could not get appId")
+  }
+  return conversions.getProdAppID(appId)
+}
+
 export function updateTenantId(tenantId?: string) {
   let context: ContextMap = updateContext({
     tenantId,
@@ -229,7 +237,7 @@ export function getProdAppDB(opts?: any): Database {
   if (!appId) {
     throw new Error("Unable to retrieve prod DB - no app ID.")
   }
-  return getDB(getProdAppID(appId), opts)
+  return getDB(conversions.getProdAppID(appId), opts)
 }
 
 /**
@@ -241,5 +249,5 @@ export function getDevAppDB(opts?: any): Database {
   if (!appId) {
     throw new Error("Unable to retrieve dev DB - no app ID.")
   }
-  return getDB(getDevelopmentAppID(appId), opts)
+  return getDB(conversions.getDevelopmentAppID(appId), opts)
 }

packages/backend-core/src/db/utils.ts

@@ -14,7 +14,7 @@ import { doWithDB, allDbs, directCouchAllDbs } from "./db"
 import { getAppMetadata } from "../cache/appMetadata"
 import { isDevApp, isDevAppID, getProdAppID } from "./conversions"
 import * as events from "../events"
-import { App, Database, ConfigType } from "@budibase/types"
+import { App, Database, ConfigType, isSettingsConfig } from "@budibase/types"
 
 /**
  * Generates a new app ID.
@@ -489,18 +489,12 @@ export const getScopedFullConfig = async function (
 
   // custom logic for settings doc
   if (type === ConfigType.SETTINGS) {
-    if (scopedConfig && scopedConfig.doc) {
+    if (!scopedConfig || !scopedConfig.doc) {
-      // overrides affected by environment variables
-      scopedConfig.doc.config.platformUrl = await getPlatformUrl({
-        tenantAware: true,
-      })
-      scopedConfig.doc.config.analyticsEnabled =
-        await events.analytics.enabled()
-    } else {
       // defaults
       scopedConfig = {
         doc: {
           _id: generateConfigID({ type, user, workspace }),
+          type: ConfigType.SETTINGS,
           config: {
             platformUrl: await getPlatformUrl({ tenantAware: true }),
             analyticsEnabled: await events.analytics.enabled(),
@@ -508,6 +502,16 @@ export const getScopedFullConfig = async function (
         },
       }
     }
+
+    // will always be true - use assertion function to get type access
+    if (isSettingsConfig(scopedConfig.doc)) {
+      // overrides affected by environment
+      scopedConfig.doc.config.platformUrl = await getPlatformUrl({
+        tenantAware: true,
+      })
+      scopedConfig.doc.config.analyticsEnabled =
+        await events.analytics.enabled()
+    }
   }
 
   return scopedConfig && scopedConfig.doc

packages/backend-core/src/environment.ts

@@ -1,9 +1,13 @@
 function isTest() {
-  return (
+  return isCypress() || isJest()
-    process.env.NODE_ENV === "jest" ||
+}
-    process.env.NODE_ENV === "cypress" ||
+
-    process.env.JEST_WORKER_ID != null
+function isJest() {
-  )
+  return !!(process.env.NODE_ENV === "jest" || process.env.JEST_WORKER_ID)
+}
+
+function isCypress() {
+  return process.env.NODE_ENV === "cypress"
 }
 
 function isDev() {
@@ -21,13 +25,16 @@ const DefaultBucketName = {
   APPS: "prod-budi-app-assets",
   TEMPLATES: "templates",
   GLOBAL: "global",
-  CLOUD: "prod-budi-tenant-uploads",
   PLUGINS: "plugins",
 }
 
 const environment = {
   isTest,
+  isJest,
   isDev,
+  isProd: () => {
+    return !isDev()
+  },
   JS_BCRYPT: process.env.JS_BCRYPT,
   JWT_SECRET: process.env.JWT_SECRET,
   COUCH_DB_URL: process.env.COUCH_DB_URL || "http://localhost:4005",
@@ -42,6 +49,7 @@ const environment = {
   MINIO_SECRET_KEY: process.env.MINIO_SECRET_KEY,
   AWS_REGION: process.env.AWS_REGION,
   MINIO_URL: process.env.MINIO_URL,
+  MINIO_ENABLED: process.env.MINIO_ENABLED || 1,
   INTERNAL_API_KEY: process.env.INTERNAL_API_KEY,
   MULTI_TENANCY: process.env.MULTI_TENANCY,
   ACCOUNT_PORTAL_URL:
@@ -54,6 +62,9 @@ const environment = {
   POSTHOG_TOKEN: process.env.POSTHOG_TOKEN,
   ENABLE_ANALYTICS: process.env.ENABLE_ANALYTICS,
   TENANT_FEATURE_FLAGS: process.env.TENANT_FEATURE_FLAGS,
+  CLOUDFRONT_CDN: process.env.CLOUDFRONT_CDN,
+  CLOUDFRONT_PRIVATE_KEY_64: process.env.CLOUDFRONT_PRIVATE_KEY_64,
+  CLOUDFRONT_PUBLIC_KEY_ID: process.env.CLOUDFRONT_PUBLIC_KEY_ID,
   BACKUPS_BUCKET_NAME:
     process.env.BACKUPS_BUCKET_NAME || DefaultBucketName.BACKUPS,
   APPS_BUCKET_NAME: process.env.APPS_BUCKET_NAME || DefaultBucketName.APPS,
@@ -61,12 +72,9 @@ const environment = {
     process.env.TEMPLATES_BUCKET_NAME || DefaultBucketName.TEMPLATES,
   GLOBAL_BUCKET_NAME:
     process.env.GLOBAL_BUCKET_NAME || DefaultBucketName.GLOBAL,
-  GLOBAL_CLOUD_BUCKET_NAME:
-    process.env.GLOBAL_CLOUD_BUCKET_NAME || DefaultBucketName.CLOUD,
   PLUGIN_BUCKET_NAME:
     process.env.PLUGIN_BUCKET_NAME || DefaultBucketName.PLUGINS,
   USE_COUCH: process.env.USE_COUCH || true,
-  DISABLE_DEVELOPER_LICENSE: process.env.DISABLE_DEVELOPER_LICENSE,
   DEFAULT_LICENSE: process.env.DEFAULT_LICENSE,
   SERVICE: process.env.SERVICE || "budibase",
   LOG_LEVEL: process.env.LOG_LEVEL,
@@ -87,6 +95,11 @@ for (let [key, value] of Object.entries(environment)) {
     // @ts-ignore
     environment[key] = 0
   }
+  // handle the edge case of "false" to disable an environment variable
+  if (value === "false") {
+    // @ts-ignore
+    environment[key] = 0
+  }
 }
 
 export = environment

packages/backend-core/src/objectStore/buckets/app.ts

@@ -0,0 +1,40 @@
+import env from "../../environment"
+import * as objectStore from "../objectStore"
+import * as cloudfront from "../cloudfront"
+
+/**
+ * In production the client library is stored in the object store, however in development
+ * we use the symlinked version produced by lerna, located in node modules. We link to this
+ * via a specific endpoint (under /api/assets/client).
+ * @param {string} appId In production we need the appId to look up the correct bucket, as the
+ * version of the client lib may differ between apps.
+ * @param {string} version The version to retrieve.
+ * @return {string} The URL to be inserted into appPackage response or server rendered
+ * app index file.
+ */
+export const clientLibraryUrl = (appId: string, version: string) => {
+  if (env.isProd()) {
+    let file = `${objectStore.sanitizeKey(appId)}/budibase-client.js`
+    if (env.CLOUDFRONT_CDN) {
+      // append app version to bust the cache
+      if (version) {
+        file += `?v=${version}`
+      }
+      // don't need to use presigned for client with cloudfront
+      // file is public
+      return cloudfront.getUrl(file)
+    } else {
+      return objectStore.getPresignedUrl(env.APPS_BUCKET_NAME, file)
+    }
+  } else {
+    return `/api/assets/client`
+  }
+}
+
+export const getAppFileUrl = (s3Key: string) => {
+  if (env.CLOUDFRONT_CDN) {
+    return cloudfront.getPresignedUrl(s3Key)
+  } else {
+    return objectStore.getPresignedUrl(env.APPS_BUCKET_NAME, s3Key)
+  }
+}

packages/backend-core/src/objectStore/buckets/global.ts

@@ -0,0 +1,29 @@
+import env from "../../environment"
+import * as tenancy from "../../tenancy"
+import * as objectStore from "../objectStore"
+import * as cloudfront from "../cloudfront"
+
+// URLs
+
+export const getGlobalFileUrl = (type: string, name: string, etag?: string) => {
+  let file = getGlobalFileS3Key(type, name)
+  if (env.CLOUDFRONT_CDN) {
+    if (etag) {
+      file = `${file}?etag=${etag}`
+    }
+    return cloudfront.getPresignedUrl(file)
+  } else {
+    return objectStore.getPresignedUrl(env.GLOBAL_BUCKET_NAME, file)
+  }
+}
+
+// KEYS
+
+export const getGlobalFileS3Key = (type: string, name: string) => {
+  let file = `${type}/${name}`
+  if (env.MULTI_TENANCY) {
+    const tenantId = tenancy.getTenantId()
+    file = `${tenantId}/${file}`
+  }
+  return file
+}

packages/backend-core/src/objectStore/buckets/index.ts

@@ -0,0 +1,3 @@
+export * from "./app"
+export * from "./global"
+export * from "./plugins"

packages/backend-core/src/objectStore/buckets/plugins.ts

@@ -0,0 +1,71 @@
+import env from "../../environment"
+import * as objectStore from "../objectStore"
+import * as tenancy from "../../tenancy"
+import * as cloudfront from "../cloudfront"
+import { Plugin } from "@budibase/types"
+
+// URLS
+
+export const enrichPluginURLs = (plugins: Plugin[]) => {
+  if (!plugins || !plugins.length) {
+    return []
+  }
+  return plugins.map(plugin => {
+    const jsUrl = getPluginJSUrl(plugin)
+    const iconUrl = getPluginIconUrl(plugin)
+    return { ...plugin, jsUrl, iconUrl }
+  })
+}
+
+const getPluginJSUrl = (plugin: Plugin) => {
+  const s3Key = getPluginJSKey(plugin)
+  return getPluginUrl(s3Key)
+}
+
+const getPluginIconUrl = (plugin: Plugin): string | undefined => {
+  const s3Key = getPluginIconKey(plugin)
+  if (!s3Key) {
+    return
+  }
+  return getPluginUrl(s3Key)
+}
+
+const getPluginUrl = (s3Key: string) => {
+  if (env.CLOUDFRONT_CDN) {
+    return cloudfront.getPresignedUrl(s3Key)
+  } else {
+    return objectStore.getPresignedUrl(env.PLUGIN_BUCKET_NAME, s3Key)
+  }
+}
+
+// S3 KEYS
+
+export const getPluginJSKey = (plugin: Plugin) => {
+  return getPluginS3Key(plugin, "plugin.min.js")
+}
+
+export const getPluginIconKey = (plugin: Plugin) => {
+  // stored iconUrl is deprecated - hardcode to icon.svg in this case
+  const iconFileName = plugin.iconUrl ? "icon.svg" : plugin.iconFileName
+  if (!iconFileName) {
+    return
+  }
+  return getPluginS3Key(plugin, iconFileName)
+}
+
+const getPluginS3Key = (plugin: Plugin, fileName: string) => {
+  const s3Key = getPluginS3Dir(plugin.name)
+  return `${s3Key}/${fileName}`
+}
+
+export const getPluginS3Dir = (pluginName: string) => {
+  let s3Key = `${pluginName}`
+  if (env.MULTI_TENANCY) {
+    const tenantId = tenancy.getTenantId()
+    s3Key = `${tenantId}/${s3Key}`
+  }
+  if (env.CLOUDFRONT_CDN) {
+    s3Key = `plugins/${s3Key}`
+  }
+  return s3Key
+}

packages/backend-core/src/objectStore/buckets/tests/app.spec.ts

@@ -0,0 +1,171 @@
+import * as app from "../app"
+import { getAppFileUrl } from "../app"
+import { testEnv } from "../../../../tests"
+
+describe("app", () => {
+  beforeEach(() => {
+    testEnv.nodeJest()
+  })
+
+  describe("clientLibraryUrl", () => {
+    function getClientUrl() {
+      return app.clientLibraryUrl("app_123/budibase-client.js", "2.0.0")
+    }
+
+    describe("single tenant", () => {
+      beforeAll(() => {
+        testEnv.singleTenant()
+      })
+
+      it("gets url in dev", () => {
+        testEnv.nodeDev()
+        const url = getClientUrl()
+        expect(url).toBe("/api/assets/client")
+      })
+
+      it("gets url with embedded minio", () => {
+        testEnv.withMinio()
+        const url = getClientUrl()
+        expect(url).toBe(
+          "/files/signed/prod-budi-app-assets/app_123/budibase-client.js/budibase-client.js"
+        )
+      })
+
+      it("gets url with custom S3", () => {
+        testEnv.withS3()
+        const url = getClientUrl()
+        expect(url).toBe(
+          "http://s3.example.com/prod-budi-app-assets/app_123/budibase-client.js/budibase-client.js"
+        )
+      })
+
+      it("gets url with cloudfront + s3", () => {
+        testEnv.withCloudfront()
+        const url = getClientUrl()
+        expect(url).toBe(
+          "http://cf.example.com/app_123/budibase-client.js/budibase-client.js?v=2.0.0"
+        )
+      })
+    })
+
+    describe("multi tenant", () => {
+      beforeAll(() => {
+        testEnv.multiTenant()
+      })
+
+      it("gets url in dev", async () => {
+        testEnv.nodeDev()
+        await testEnv.withTenant(tenantId => {
+          const url = getClientUrl()
+          expect(url).toBe("/api/assets/client")
+        })
+      })
+
+      it("gets url with embedded minio", async () => {
+        await testEnv.withTenant(tenantId => {
+          testEnv.withMinio()
+          const url = getClientUrl()
+          expect(url).toBe(
+            "/files/signed/prod-budi-app-assets/app_123/budibase-client.js/budibase-client.js"
+          )
+        })
+      })
+
+      it("gets url with custom S3", async () => {
+        await testEnv.withTenant(tenantId => {
+          testEnv.withS3()
+          const url = getClientUrl()
+          expect(url).toBe(
+            "http://s3.example.com/prod-budi-app-assets/app_123/budibase-client.js/budibase-client.js"
+          )
+        })
+      })
+
+      it("gets url with cloudfront + s3", async () => {
+        await testEnv.withTenant(tenantId => {
+          testEnv.withCloudfront()
+          const url = getClientUrl()
+          expect(url).toBe(
+            "http://cf.example.com/app_123/budibase-client.js/budibase-client.js?v=2.0.0"
+          )
+        })
+      })
+    })
+  })
+
+  describe("getAppFileUrl", () => {
+    function getAppFileUrl() {
+      return app.getAppFileUrl("app_123/attachments/image.jpeg")
+    }
+
+    describe("single tenant", () => {
+      beforeAll(() => {
+        testEnv.multiTenant()
+      })
+
+      it("gets url with embedded minio", () => {
+        testEnv.withMinio()
+        const url = getAppFileUrl()
+        expect(url).toBe(
+          "/files/signed/prod-budi-app-assets/app_123/attachments/image.jpeg"
+        )
+      })
+
+      it("gets url with custom S3", () => {
+        testEnv.withS3()
+        const url = getAppFileUrl()
+        expect(url).toBe(
+          "http://s3.example.com/prod-budi-app-assets/app_123/attachments/image.jpeg"
+        )
+      })
+
+      it("gets url with cloudfront + s3", () => {
+        testEnv.withCloudfront()
+        const url = getAppFileUrl()
+        // omit rest of signed params
+        expect(
+          url.includes("http://cf.example.com/app_123/attachments/image.jpeg?")
+        ).toBe(true)
+      })
+    })
+
+    describe("multi tenant", () => {
+      beforeAll(() => {
+        testEnv.multiTenant()
+      })
+
+      it("gets url with embedded minio", async () => {
+        testEnv.withMinio()
+        await testEnv.withTenant(tenantId => {
+          const url = getAppFileUrl()
+          expect(url).toBe(
+            "/files/signed/prod-budi-app-assets/app_123/attachments/image.jpeg"
+          )
+        })
+      })
+
+      it("gets url with custom S3", async () => {
+        testEnv.withS3()
+        await testEnv.withTenant(tenantId => {
+          const url = getAppFileUrl()
+          expect(url).toBe(
+            "http://s3.example.com/prod-budi-app-assets/app_123/attachments/image.jpeg"
+          )
+        })
+      })
+
+      it("gets url with cloudfront + s3", async () => {
+        testEnv.withCloudfront()
+        await testEnv.withTenant(tenantId => {
+          const url = getAppFileUrl()
+          // omit rest of signed params
+          expect(
+            url.includes(
+              "http://cf.example.com/app_123/attachments/image.jpeg?"
+            )
+          ).toBe(true)
+        })
+      })
+    })
+  })
+})

packages/backend-core/src/objectStore/buckets/tests/global.spec.ts

@@ -0,0 +1,74 @@
+import * as global from "../global"
+import { testEnv } from "../../../../tests"
+
+describe("global", () => {
+  describe("getGlobalFileUrl", () => {
+    function getGlobalFileUrl() {
+      return global.getGlobalFileUrl("settings", "logoUrl", "etag")
+    }
+
+    describe("single tenant", () => {
+      beforeAll(() => {
+        testEnv.singleTenant()
+      })
+
+      it("gets url with embedded minio", () => {
+        testEnv.withMinio()
+        const url = getGlobalFileUrl()
+        expect(url).toBe("/files/signed/global/settings/logoUrl")
+      })
+
+      it("gets url with custom S3", () => {
+        testEnv.withS3()
+        const url = getGlobalFileUrl()
+        expect(url).toBe("http://s3.example.com/global/settings/logoUrl")
+      })
+
+      it("gets url with cloudfront + s3", () => {
+        testEnv.withCloudfront()
+        const url = getGlobalFileUrl()
+        // omit rest of signed params
+        expect(
+          url.includes("http://cf.example.com/settings/logoUrl?etag=etag&")
+        ).toBe(true)
+      })
+    })
+
+    describe("multi tenant", () => {
+      beforeAll(() => {
+        testEnv.multiTenant()
+      })
+
+      it("gets url with embedded minio", async () => {
+        testEnv.withMinio()
+        await testEnv.withTenant(tenantId => {
+          const url = getGlobalFileUrl()
+          expect(url).toBe(`/files/signed/global/${tenantId}/settings/logoUrl`)
+        })
+      })
+
+      it("gets url with custom S3", async () => {
+        testEnv.withS3()
+        await testEnv.withTenant(tenantId => {
+          const url = getGlobalFileUrl()
+          expect(url).toBe(
+            `http://s3.example.com/global/${tenantId}/settings/logoUrl`
+          )
+        })
+      })
+
+      it("gets url with cloudfront + s3", async () => {
+        testEnv.withCloudfront()
+        await testEnv.withTenant(tenantId => {
+          const url = getGlobalFileUrl()
+          // omit rest of signed params
+          expect(
+            url.includes(
+              `http://cf.example.com/${tenantId}/settings/logoUrl?etag=etag&`
+            )
+          ).toBe(true)
+        })
+      })
+    })
+  })
+})

packages/backend-core/src/objectStore/buckets/tests/plugins.spec.ts

@@ -0,0 +1,110 @@
+import * as plugins from "../plugins"
+import { structures, testEnv } from "../../../../tests"
+
+describe("plugins", () => {
+  describe("enrichPluginURLs", () => {
+    const plugin = structures.plugins.plugin()
+
+    function getEnrichedPluginUrls() {
+      const enriched = plugins.enrichPluginURLs([plugin])[0]
+      return {
+        jsUrl: enriched.jsUrl!,
+        iconUrl: enriched.iconUrl!,
+      }
+    }
+
+    describe("single tenant", () => {
+      beforeAll(() => {
+        testEnv.singleTenant()
+      })
+
+      it("gets url with embedded minio", () => {
+        testEnv.withMinio()
+        const urls = getEnrichedPluginUrls()
+        expect(urls.jsUrl).toBe(
+          `/files/signed/plugins/${plugin.name}/plugin.min.js`
+        )
+        expect(urls.iconUrl).toBe(
+          `/files/signed/plugins/${plugin.name}/icon.svg`
+        )
+      })
+
+      it("gets url with custom S3", () => {
+        testEnv.withS3()
+        const urls = getEnrichedPluginUrls()
+        expect(urls.jsUrl).toBe(
+          `http://s3.example.com/plugins/${plugin.name}/plugin.min.js`
+        )
+        expect(urls.iconUrl).toBe(
+          `http://s3.example.com/plugins/${plugin.name}/icon.svg`
+        )
+      })
+
+      it("gets url with cloudfront + s3", () => {
+        testEnv.withCloudfront()
+        const urls = getEnrichedPluginUrls()
+        // omit rest of signed params
+        expect(
+          urls.jsUrl.includes(
+            `http://cf.example.com/plugins/${plugin.name}/plugin.min.js?`
+          )
+        ).toBe(true)
+        expect(
+          urls.iconUrl.includes(
+            `http://cf.example.com/plugins/${plugin.name}/icon.svg?`
+          )
+        ).toBe(true)
+      })
+    })
+
+    describe("multi tenant", () => {
+      beforeAll(() => {
+        testEnv.multiTenant()
+      })
+
+      it("gets url with embedded minio", async () => {
+        testEnv.withMinio()
+        await testEnv.withTenant(tenantId => {
+          const urls = getEnrichedPluginUrls()
+          expect(urls.jsUrl).toBe(
+            `/files/signed/plugins/${tenantId}/${plugin.name}/plugin.min.js`
+          )
+          expect(urls.iconUrl).toBe(
+            `/files/signed/plugins/${tenantId}/${plugin.name}/icon.svg`
+          )
+        })
+      })
+
+      it("gets url with custom S3", async () => {
+        testEnv.withS3()
+        await testEnv.withTenant(tenantId => {
+          const urls = getEnrichedPluginUrls()
+          expect(urls.jsUrl).toBe(
+            `http://s3.example.com/plugins/${tenantId}/${plugin.name}/plugin.min.js`
+          )
+          expect(urls.iconUrl).toBe(
+            `http://s3.example.com/plugins/${tenantId}/${plugin.name}/icon.svg`
+          )
+        })
+      })
+
+      it("gets url with cloudfront + s3", async () => {
+        testEnv.withCloudfront()
+        await testEnv.withTenant(tenantId => {
+          const urls = getEnrichedPluginUrls()
+          // omit rest of signed params
+          expect(
+            urls.jsUrl.includes(
+              `http://cf.example.com/plugins/${tenantId}/${plugin.name}/plugin.min.js?`
+            )
+          ).toBe(true)
+          expect(
+            urls.iconUrl.includes(
+              `http://cf.example.com/plugins/${tenantId}/${plugin.name}/icon.svg?`
+            )
+          ).toBe(true)
+        })
+      })
+    })
+  })
+})

packages/backend-core/src/objectStore/cloudfront.ts

@@ -0,0 +1,41 @@
+import env from "../environment"
+const cfsign = require("aws-cloudfront-sign")
+
+let PRIVATE_KEY: string | undefined
+
+function getPrivateKey() {
+  if (!env.CLOUDFRONT_PRIVATE_KEY_64) {
+    throw new Error("CLOUDFRONT_PRIVATE_KEY_64 is not set")
+  }
+
+  if (PRIVATE_KEY) {
+    return PRIVATE_KEY
+  }
+
+  PRIVATE_KEY = Buffer.from(env.CLOUDFRONT_PRIVATE_KEY_64, "base64").toString(
+    "utf-8"
+  )
+
+  return PRIVATE_KEY
+}
+
+const getCloudfrontSignParams = () => {
+  return {
+    keypairId: env.CLOUDFRONT_PUBLIC_KEY_ID,
+    privateKeyString: getPrivateKey(),
+    expireTime: new Date().getTime() + 1000 * 60 * 60, // 1 hour
+  }
+}
+
+export const getPresignedUrl = (s3Key: string) => {
+  const url = getUrl(s3Key)
+  return cfsign.getSignedUrl(url, getCloudfrontSignParams())
+}
+
+export const getUrl = (s3Key: string) => {
+  let prefix = "/"
+  if (s3Key.startsWith("/")) {
+    prefix = ""
+  }
+  return `${env.CLOUDFRONT_CDN}${prefix}${s3Key}`
+}

packages/backend-core/src/objectStore/index.ts

@@ -1,2 +1,3 @@
 export * from "./objectStore"
 export * from "./utils"
+export * from "./buckets"

packages/backend-core/src/objectStore/objectStore.ts

@@ -8,7 +8,7 @@ import { promisify } from "util"
 import { join } from "path"
 import fs from "fs"
 import env from "../environment"
-import { budibaseTempDir, ObjectStoreBuckets } from "./utils"
+import { budibaseTempDir } from "./utils"
 import { v4 } from "uuid"
 import { APP_PREFIX, APP_DEV_PREFIX } from "../db"
 
@@ -26,7 +26,7 @@ type UploadParams = {
   bucket: string
   filename: string
   path: string
-  type?: string
+  type?: string | null
   // can be undefined, we will remove it
   metadata?: {
     [key: string]: string | undefined
@@ -41,6 +41,7 @@ const CONTENT_TYPE_MAP: any = {
   json: "application/json",
   gz: "application/gzip",
 }
+
 const STRING_CONTENT_TYPES = [
   CONTENT_TYPE_MAP.html,
   CONTENT_TYPE_MAP.css,
@@ -58,35 +59,17 @@ export function sanitizeBucket(input: string) {
   return input.replace(new RegExp(APP_DEV_PREFIX, "g"), APP_PREFIX)
 }
 
-function publicPolicy(bucketName: string) {
-  return {
-    Version: "2012-10-17",
-    Statement: [
-      {
-        Effect: "Allow",
-        Principal: {
-          AWS: ["*"],
-        },
-        Action: "s3:GetObject",
-        Resource: [`arn:aws:s3:::${bucketName}/*`],
-      },
-    ],
-  }
-}
-
-const PUBLIC_BUCKETS = [
-  ObjectStoreBuckets.APPS,
-  ObjectStoreBuckets.GLOBAL,
-  ObjectStoreBuckets.PLUGINS,
-]
-
 /**
  * Gets a connection to the object store using the S3 SDK.
  * @param {string} bucket the name of the bucket which blobs will be uploaded/retrieved from.
+ * @param {object} opts configuration for the object store.
  * @return {Object} an S3 object store object, check S3 Nodejs SDK for usage.
  * @constructor
  */
-export const ObjectStore = (bucket: string) => {
+export const ObjectStore = (
+  bucket: string,
+  opts: { presigning: boolean } = { presigning: false }
+) => {
   const config: any = {
     s3ForcePathStyle: true,
     signatureVersion: "v4",
@@ -100,9 +83,20 @@ export const ObjectStore = (bucket: string) => {
       Bucket: sanitizeBucket(bucket),
     }
   }
+
+  // custom S3 is in use i.e. minio
   if (env.MINIO_URL) {
-    config.endpoint = env.MINIO_URL
+    if (opts.presigning && env.MINIO_ENABLED) {
+      // IMPORTANT: Signed urls will inspect the host header of the request.
+      // Normally a signed url will need to be generated with a specified host in mind.
+      // To support dynamic hosts, e.g. some unknown self-hosted installation url,
+      // use a predefined host. The host 'minio-service' is also forwarded to minio requests via nginx
+      config.endpoint = "minio-service"
+    } else {
+      config.endpoint = env.MINIO_URL
+    }
   }
+
   return new AWS.S3(config)
 }
 
@@ -135,16 +129,6 @@ export const makeSureBucketExists = async (client: any, bucketName: string) => {
         await promises[bucketName]
         delete promises[bucketName]
       }
-      // public buckets are quite hidden in the system, make sure
-      // no bucket is set accidentally
-      if (PUBLIC_BUCKETS.includes(bucketName)) {
-        await client
-          .putBucketPolicy({
-            Bucket: bucketName,
-            Policy: JSON.stringify(publicPolicy(bucketName)),
-          })
-          .promise()
-      }
     } else {
       throw new Error("Unable to write to object store bucket.")
     }
@@ -274,6 +258,36 @@ export const listAllObjects = async (bucketName: string, path: string) => {
   return objects
 }
 
+/**
+ * Generate a presigned url with a default TTL of 1 hour
+ */
+export const getPresignedUrl = (
+  bucketName: string,
+  key: string,
+  durationSeconds: number = 3600
+) => {
+  const objectStore = ObjectStore(bucketName, { presigning: true })
+  const params = {
+    Bucket: sanitizeBucket(bucketName),
+    Key: sanitizeKey(key),
+    Expires: durationSeconds,
+  }
+  const url = objectStore.getSignedUrl("getObject", params)
+
+  if (!env.MINIO_ENABLED) {
+    // return the full URL to the client
+    return url
+  } else {
+    // return the path only to the client
+    // use the presigned url route to ensure the static
+    // hostname will be used in the request
+    const signedUrl = new URL(url)
+    const path = signedUrl.pathname
+    const query = signedUrl.search
+    return `/files/signed${path}${query}`
+  }
+}
+
 /**
  * Same as retrieval function but puts to a temporary file.
  */

packages/backend-core/src/objectStore/utils.ts

@@ -14,7 +14,6 @@ export const ObjectStoreBuckets = {
   APPS: env.APPS_BUCKET_NAME,
   TEMPLATES: env.TEMPLATES_BUCKET_NAME,
   GLOBAL: env.GLOBAL_BUCKET_NAME,
-  GLOBAL_CLOUD: env.GLOBAL_CLOUD_BUCKET_NAME,
   PLUGINS: env.PLUGIN_BUCKET_NAME,
 }
 

packages/backend-core/src/tenancy/tenancy.ts

@@ -1,4 +1,4 @@
-import { doWithDB, queryPlatformView, getGlobalDBName } from "../db"
+import { doWithDB, getGlobalDBName } from "../db"
 import {
   DEFAULT_TENANT_ID,
   getTenantId,
@@ -8,11 +8,10 @@ import {
 import env from "../environment"
 import {
   BBContext,
-  PlatformUser,
   TenantResolutionStrategy,
   GetTenantIdOptions,
 } from "@budibase/types"
-import { Header, StaticDatabases, ViewName } from "../constants"
+import { Header, StaticDatabases } from "../constants"
 
 const TENANT_DOC = StaticDatabases.PLATFORM_INFO.docs.tenants
 const PLATFORM_INFO_DB = StaticDatabases.PLATFORM_INFO.name
@@ -111,27 +110,7 @@ export async function lookupTenantId(userId: string) {
   })
 }
 
-// lookup, could be email or userId, either will return a doc
+export const isUserInAppTenant = (appId: string, user?: any) => {
-export async function getTenantUser(
-  identifier: string
-): Promise<PlatformUser | undefined> {
-  // use the view here and allow to find anyone regardless of casing
-  // Use lowercase to ensure email login is case-insensitive
-  const users = await queryPlatformView<PlatformUser>(
-    ViewName.PLATFORM_USERS_LOWERCASE,
-    {
-      keys: [identifier.toLowerCase()],
-      include_docs: true,
-    }
-  )
-  if (Array.isArray(users)) {
-    return users[0]
-  } else {
-    return users
-  }
-}
-
-export function isUserInAppTenant(appId: string, user?: any) {
   let userTenantId
   if (user) {
     userTenantId = user.tenantId || DEFAULT_TENANT_ID

packages/backend-core/src/utils/tests/utils.spec.ts

@@ -1,8 +1,8 @@
-const { structures } = require("../../tests")
+import { structures } from "../../../tests"
-const utils = require("../utils")
+import * as utils from "../../utils"
-const events = require("../events")
+import * as events from "../../events"
-const { DEFAULT_TENANT_ID } = require("../constants")
+import { DEFAULT_TENANT_ID } from "../../constants"
-const { doInTenant } = require("../context")
+import { doInTenant } from "../../context"
 
 describe("utils", () => {
   describe("platformLogout", () => {
@@ -14,4 +14,4 @@ describe("utils", () => {
       })
     })
   })
-})
+})

packages/backend-core/src/utils/utils.ts

@@ -1,6 +1,13 @@
 import { getAllApps, queryGlobalView } from "../db"
 import { options } from "../middleware/passport/jwt"
-import { Header, Cookie, MAX_VALID_DATE } from "../constants"
+import {
+  Header,
+  Cookie,
+  MAX_VALID_DATE,
+  DocumentType,
+  SEPARATOR,
+  ViewName,
+} from "../constants"
 import env from "../environment"
 import * as userCache from "../cache/user"
 import { getSessionsForUser, invalidateSessions } from "../security/sessions"
@@ -8,12 +15,11 @@ import * as events from "../events"
 import * as tenancy from "../tenancy"
 import {
   App,
-  BBContext,
+  Ctx,
   PlatformLogoutOpts,
   TenantResolutionStrategy,
 } from "@budibase/types"
 import { SetOption } from "cookies"
-import { DocumentType, SEPARATOR, ViewName } from "../constants"
 const jwt = require("jsonwebtoken")
 
 const APP_PREFIX = DocumentType.APP + SEPARATOR
@@ -25,7 +31,7 @@ function confirmAppId(possibleAppId: string | undefined) {
     : undefined
 }
 
-async function resolveAppUrl(ctx: BBContext) {
+async function resolveAppUrl(ctx: Ctx) {
   const appUrl = ctx.path.split("/")[2]
   let possibleAppUrl = `/${appUrl.toLowerCase()}`
 
@@ -50,7 +56,7 @@ async function resolveAppUrl(ctx: BBContext) {
   return app && app.appId ? app.appId : undefined
 }
 
-export function isServingApp(ctx: BBContext) {
+export function isServingApp(ctx: Ctx) {
   // dev app
   if (ctx.path.startsWith(`/${APP_PREFIX}`)) {
     return true
@@ -67,7 +73,7 @@ export function isServingApp(ctx: BBContext) {
  * @param {object} ctx The main request body to look through.
  * @returns {string|undefined} If an appId was found it will be returned.
  */
-export async function getAppIdFromCtx(ctx: BBContext) {
+export async function getAppIdFromCtx(ctx: Ctx) {
   // look in headers
   const options = [ctx.headers[Header.APP_ID]]
   let appId
@@ -83,12 +89,16 @@ export async function getAppIdFromCtx(ctx: BBContext) {
     appId = confirmAppId(ctx.request.body.appId)
   }
 
-  // look in the url - dev app
+  // look in the path
-  let appPath =
+  const pathId = parseAppIdFromUrl(ctx.path)
-    ctx.request.headers.referrer ||
+  if (!appId && pathId) {
-    ctx.path.split("/").filter(subPath => subPath.startsWith(APP_PREFIX))
+    appId = confirmAppId(pathId)
-  if (!appId && appPath.length) {
+  }
-    appId = confirmAppId(appPath[0])
+
+  // look in the referer
+  const refererId = parseAppIdFromUrl(ctx.request.headers.referer)
+  if (!appId && refererId) {
+    appId = confirmAppId(refererId)
   }
 
   // look in the url - prod app
@@ -99,6 +109,13 @@ export async function getAppIdFromCtx(ctx: BBContext) {
   return appId
 }
 
+function parseAppIdFromUrl(url?: string) {
+  if (!url) {
+    return
+  }
+  return url.split("/").find(subPath => subPath.startsWith(APP_PREFIX))
+}
+
 /**
  * opens the contents of the specified encrypted JWT.
  * @return {object} the contents of the token.
@@ -115,7 +132,7 @@ export function openJwt(token: string) {
  * @param {object} ctx The request which is to be manipulated.
  * @param {string} name The name of the cookie to get.
  */
-export function getCookie(ctx: BBContext, name: string) {
+export function getCookie(ctx: Ctx, name: string) {
   const cookie = ctx.cookies.get(name)
 
   if (!cookie) {
@@ -133,7 +150,7 @@ export function getCookie(ctx: BBContext, name: string) {
  * @param {object} opts options like whether to sign.
  */
 export function setCookie(
-  ctx: BBContext,
+  ctx: Ctx,
   value: any,
   name = "builder",
   opts = { sign: true }
@@ -159,7 +176,7 @@ export function setCookie(
 /**
  * Utility function, simply calls setCookie with an empty string for value
  */
-export function clearCookie(ctx: BBContext, name: string) {
+export function clearCookie(ctx: Ctx, name: string) {
   setCookie(ctx, null, name)
 }
 
@@ -169,7 +186,7 @@ export function clearCookie(ctx: BBContext, name: string) {
  * @param {object} ctx The koa context object to be tested.
  * @return {boolean} returns true if the call is from the client lib (a built app rather than the builder).
  */
-export function isClient(ctx: BBContext) {
+export function isClient(ctx: Ctx) {
   return ctx.headers[Header.TYPE] === "client"
 }
 

packages/backend-core/tests/jestSetup.ts

@@ -17,7 +17,9 @@ env._set("MINIO_URL", "http://localhost")
 env._set("MINIO_ACCESS_KEY", "test")
 env._set("MINIO_SECRET_KEY", "test")
 
-global.console.log = jest.fn() // console.log are ignored in tests
+if (!process.env.DEBUG) {
+  global.console.log = jest.fn() // console.log are ignored in tests
+}
 
 if (!process.env.CI) {
   // set a longer timeout in dev for debugging

packages/backend-core/tests/utilities/index.ts

@@ -1,6 +1,7 @@
 export * as mocks from "./mocks"
 export * as structures from "./structures"
 export { generator } from "./structures"
+export * as testEnv from "./testEnv"
 
 import * as dbConfig from "./db"
 dbConfig.init()

packages/backend-core/tests/utilities/mocks/events.ts

@@ -117,3 +117,7 @@ jest.spyOn(events.view, "filterDeleted")
 jest.spyOn(events.view, "calculationCreated")
 jest.spyOn(events.view, "calculationUpdated")
 jest.spyOn(events.view, "calculationDeleted")
+
+jest.spyOn(events.plugin, "init")
+jest.spyOn(events.plugin, "imported")
+jest.spyOn(events.plugin, "deleted")

packages/backend-core/tests/utilities/mocks/index.ts

@@ -1,5 +1,6 @@
-import "./posthog"
-import "./events"
 export * as accounts from "./accounts"
 export * as date from "./date"
+export * as licenses from "./licenses"
 export { default as fetch } from "./fetch"
+import "./posthog"
+import "./events"

packages/backend-core/tests/utilities/mocks/licenses.ts

@@ -0,0 +1,83 @@
+import { Feature, License, Quotas } from "@budibase/types"
+import _ from "lodash"
+
+let CLOUD_FREE_LICENSE: License
+let UNLIMITED_LICENSE: License
+let getCachedLicense: any
+
+// init for the packages other than pro
+export function init(proPkg: any) {
+  initInternal({
+    CLOUD_FREE_LICENSE: proPkg.constants.licenses.CLOUD_FREE_LICENSE,
+    UNLIMITED_LICENSE: proPkg.constants.licenses.UNLIMITED_LICENSE,
+    getCachedLicense: proPkg.licensing.cache.getCachedLicense,
+  })
+}
+
+// init for the pro package
+export function initInternal(opts: {
+  CLOUD_FREE_LICENSE: License
+  UNLIMITED_LICENSE: License
+  getCachedLicense: any
+}) {
+  CLOUD_FREE_LICENSE = opts.CLOUD_FREE_LICENSE
+  UNLIMITED_LICENSE = opts.UNLIMITED_LICENSE
+  getCachedLicense = opts.getCachedLicense
+}
+
+export interface UseLicenseOpts {
+  features?: Feature[]
+  quotas?: Quotas
+}
+
+// LICENSES
+
+export const useLicense = (license: License, opts?: UseLicenseOpts) => {
+  if (opts) {
+    if (opts.features) {
+      license.features.push(...opts.features)
+    }
+    if (opts.quotas) {
+      license.quotas = opts.quotas
+    }
+  }
+
+  getCachedLicense.mockReturnValue(license)
+
+  return license
+}
+
+export const useUnlimited = (opts?: UseLicenseOpts) => {
+  return useLicense(UNLIMITED_LICENSE, opts)
+}
+
+export const useCloudFree = () => {
+  return useLicense(CLOUD_FREE_LICENSE)
+}
+
+// FEATURES
+
+const useFeature = (feature: Feature) => {
+  const license = _.cloneDeep(UNLIMITED_LICENSE)
+  const opts: UseLicenseOpts = {
+    features: [feature],
+  }
+
+  return useLicense(license, opts)
+}
+
+export const useBackups = () => {
+  return useFeature(Feature.APP_BACKUPS)
+}
+
+export const useGroups = () => {
+  return useFeature(Feature.USER_GROUPS)
+}
+
+// QUOTAS
+
+export const setAutomationLogsQuota = (value: number) => {
+  const license = _.cloneDeep(UNLIMITED_LICENSE)
+  license.quotas.constant.automationLogRetentionDays.value = value
+  return useLicense(license)
+}

packages/backend-core/tests/utilities/structures/index.ts

@@ -6,3 +6,4 @@ export const generator = new Chance()
 export * as koa from "./koa"
 export * as accounts from "./accounts"
 export * as licenses from "./licenses"
+export * as plugins from "./plugins"

packages/backend-core/tests/utilities/structures/plugins.ts

@@ -0,0 +1,19 @@
+import { generator } from "."
+import { Plugin, PluginSource, PluginType } from "@budibase/types"
+
+export function plugin(): Plugin {
+  return {
+    description: generator.word(),
+    name: generator.word(),
+    version: "1.0.0",
+    source: PluginSource.FILE,
+    package: {
+      name: generator.word,
+    },
+    hash: generator.hash(),
+    schema: {
+      type: PluginType.DATASOURCE,
+    },
+    iconFileName: "icon.svg",
+  }
+}

packages/backend-core/tests/utilities/testEnv.ts

@@ -0,0 +1,87 @@
+import env from "../../src/environment"
+import * as tenancy from "../../src/tenancy"
+import { newid } from "../../src/utils"
+
+// TENANCY
+
+export async function withTenant(task: (tenantId: string) => any) {
+  const tenantId = newid()
+  return tenancy.doInTenant(tenantId, async () => {
+    await task(tenantId)
+  })
+}
+
+export function singleTenant() {
+  env._set("MULTI_TENANCY", 0)
+}
+
+export function multiTenant() {
+  env._set("MULTI_TENANCY", 1)
+}
+
+// NODE
+
+export function nodeDev() {
+  env._set("NODE_ENV", "dev")
+}
+
+export function nodeJest() {
+  env._set("NODE_ENV", "jest")
+}
+
+// FILES
+
+export function withS3() {
+  env._set("NODE_ENV", "production")
+  env._set("MINIO_ENABLED", 0)
+  env._set("MINIO_URL", "http://s3.example.com")
+  env._set("CLOUDFRONT_CDN", undefined)
+}
+
+const CLOUDFRONT_TEST_KEY =
+  "-----BEGIN RSA PRIVATE KEY-----\n" +
+  "MIIEpAIBAAKCAQEAqXRsir/0Qba1xEnybUs7d7QEAE02GRc+4H7HD5l5VnAxkV1m\n" +
+  "tNTXTmoYkaIhLdebV1EwQs3T9knxoyd4cVcrDkDfDLZErfYWJsuE3/QYNknnZs4/\n" +
+  "Ai0cg+v9ZX3gcizvpYg9GQI3INM0uRG8lJwGP7FQ/kknhA2yVFVCSxX6kkNtOUh5\n" +
+  "dKSG7m6IwswcSwD++Z/94vsFkoZIGY0e1CD/drFJ6+1TFY2YgbDKT5wDFLJ9vHFx\n" +
+  "/5o4POwn3gz/ru2Db9jbRdfEAqRdy46nRKQgBGUmupAgSK1+BJEzafexp8RmCGb0\n" +
+  "WUffxOtj8/jNCeCF0JBgVHAe3crOQ8ySrtoaHQIDAQABAoIBAA+ipW07/u6dTDI7\n" +
+  "XHoHKgqGeqQIe8he47dVG0ruL0rxeTFfe92NkfwzP+cYHZWcQkIRRLG1Six8cCZM\n" +
+  "uwlCML/U7n++xaGDhlG4D5+WZzGDKi3LM/cgcHQfrzbRIYeHa+lLI9AN60ZFFqVI\n" +
+  "5KyVpOH1m3KLD3FYzi6H22EQOxmJpqWlt2uArny5LxlPJKmmGSFjvneb4N2ZAKGQ\n" +
+  "QfClJGz9tRjceWUUdJrpqmTmBQIosKmLPq8PEviUNAVG+6m4r8jiRbf8OKkAm+3L\n" +
+  "LVIsN8HfYB9jEuERYPnbuXdX0kDEkg0xEyTH5YbNZvfm5ptCU9Xn+Jz1trF+wCHD\n" +
+  "2RlxdQUCgYEA3U0nCf6NTmmeMCsAX6gvaPuM0iUfUfS3b3G57I6u46lLGNLsfJw6\n" +
+  "MTpVc164lKYQK9czw/ijKzb8e3mcyzbPorVkajMjUCNWGrMK+vFbOGmqQkhUi30U\n" +
+  "IJuuTktMd+21D/SpLlev4MLria23vUIKEqNenYpV6wkGLt/mKtISaPMCgYEAxAYx\n" +
+  "j+xJLTK9eN+rpekwjYE78hD9VoBkBnr/NBiGV302AsJRuq2+L4zcBnAsH+SidFim\n" +
+  "cwqoj3jeVT8ZQFXlK3fGVaEJsCXd6GWk8ZIWUTn9JZwi2KcCvCU/YiHfx8c7y7Gl\n" +
+  "SiPXUPsvvkcw6RRh2u4J5tHLIqJe3W58ENoBNK8CgYEApxTBDMKrXTBQxn0w4wfQ\n" +
+  "A6soPuDYLMBeXj226eswD6KZmDxnYA1zwgcQzPIO2ewm+XKZGrR2PQJezbqbrrHL\n" +
+  "QkVBcwz49GA5eh8Dg0MGZCki6rhBXK8qqxPfHi2rpkBKG6nUsbBykXeY7XHC75kU\n" +
+  "kc3WeYsgIzvE908EMAA69hECgYEAinbpiYVZh1DBH+G26MIYZswz4OB5YyHcBevZ\n" +
+  "2x27v48VmMtUWe4iWopAXVfdA0ZILrD0Gm0b9gRl4IdqudQyxgqcEZ5oLoIBBwjN\n" +
+  "g0oy83tnwqpQvwLx3p7c79+HqCGmrlK0s/MvQ+e6qMi21t1r5e6hFed5euSA6B8E\n" +
+  "Cg9ELMcCgYB9bGwlNAE+iuzMIhKev1s7h3TzqKtGw37TtHXvxcTQs3uawJQksQ2s\n" +
+  "K0Zy1Ta7vybbwAA5m+LxoMT04WUdJO7Cr8/3rBMrbKKO3H7IgC3G+nXnOBdshzn5\n" +
+  "ifMbhZslFThC/osD5ZV7snXZgTWyPexaINJhHmdrAWpmW1h+UFoiMw==\n" +
+  "-----END RSA PRIVATE KEY-----\n"
+
+const CLOUDFRONT_TEST_KEY_64 = Buffer.from(
+  CLOUDFRONT_TEST_KEY,
+  "utf-8"
+).toString("base64")
+
+export function withCloudfront() {
+  withS3()
+  env._set("CLOUDFRONT_CDN", "http://cf.example.com")
+  env._set("CLOUDFRONT_PUBLIC_KEY_ID", "keypair_123")
+  env._set("CLOUDFRONT_PRIVATE_KEY_64", CLOUDFRONT_TEST_KEY_64)
+}
+
+export function withMinio() {
+  env._set("NODE_ENV", "production")
+  env._set("MINIO_ENABLED", 1)
+  env._set("MINIO_URL", "http://minio.example.com")
+  env._set("CLOUDFRONT_CDN", undefined)
+}

packages/backend-core/tsconfig.json

@@ -8,6 +8,10 @@
     }
   },
   "references": [
-    { "path": "../types" },
+    { "path": "../types" }
+  ],
+  "exclude": [
+    "node_modules",
+    "dist"
   ]
 }

packages/backend-core/yarn.lock

@@ -1538,6 +1538,13 @@ asynckit@^0.4.0:
   resolved "https://registry.yarnpkg.com/asynckit/-/asynckit-0.4.0.tgz#c79ed97f7f34cb8f2ba1bc9790bcc366474b4b79"
   integrity sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==
 
+aws-cloudfront-sign@2.2.0:
+  version "2.2.0"
+  resolved "https://registry.yarnpkg.com/aws-cloudfront-sign/-/aws-cloudfront-sign-2.2.0.tgz#3910f5a6d0d90fec07f2b4ef8ab07f3eefb5625d"
+  integrity sha512-qG+rwZMP3KRTPPbVmWY8DlrT56AkA4iVOeo23vkdK2EXeW/brJFN2haSNKzVz+oYhFMEIzVVloeAcrEzuRkuVQ==
+  dependencies:
+    lodash "^3.6.0"
+
 aws-sdk@2.1030.0:
   version "2.1030.0"
   resolved "https://registry.yarnpkg.com/aws-sdk/-/aws-sdk-2.1030.0.tgz#24a856af3d2b8b37c14a8f59974993661c66fd82"
@@ -3839,6 +3846,11 @@ lodash@4.17.21, lodash@^4.17.21:
   resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.21.tgz#679591c564c3bffaae8454cf0b3df370c3d6911c"
   integrity sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==
 
+lodash@^3.6.0:
+  version "3.10.1"
+  resolved "https://registry.yarnpkg.com/lodash/-/lodash-3.10.1.tgz#5bf45e8e49ba4189e17d482789dfd15bd140b7b6"
+  integrity sha512-9mDDwqVIma6OZX79ZlDACZl8sBm0TEnkf99zV3iMA4GzkIT/9hiqP5mY0HoT1iNLCrKc/R1HByV+yJfRWVJryQ==
+
 lowercase-keys@^1.0.0, lowercase-keys@^1.0.1:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/lowercase-keys/-/lowercase-keys-1.0.1.tgz#6f9e30b47084d971a7c820ff15a6c5167b74c26f"

packages/bbui/package.json

@@ -89,4 +89,4 @@
     "loader-utils": "1.4.1"
   },
   "gitHead": "d1836a898cab3f8ab80ee6d8f42be1a9eed7dcdc"
-}
+}

packages/bbui/src/Badge/Badge.svelte

@@ -10,10 +10,13 @@
   export let green = false
   export let active = false
   export let inactive = false
+  export let hoverable = false
 </script>
 
 <span
+  on:click
   class="spectrum-Label"
+  class:hoverable
   class:spectrum-Label--small={size === "S"}
   class:spectrum-Label--large={size === "L"}
   class:spectrum-Label--grey={grey}
@@ -27,3 +30,13 @@
 >
   <slot />
 </span>
+
+<style>
+  .spectrum-Label--grey {
+    background-color: var(--spectrum-global-color-gray-500);
+    font-weight: 600;
+  }
+  .hoverable:hover {
+    cursor: pointer;
+  }
+</style>

packages/bbui/src/Form/Core/Dropzone.svelte

@@ -15,6 +15,7 @@
   export let value = []
   export let id = null
   export let disabled = false
+  export let compact = false
   export let fileSizeLimit = BYTES_IN_MB * 20
   export let processFiles = null
   export let deleteAttachments = null
@@ -239,70 +240,72 @@
           bind:this={fileInput}
           on:change={handleFile}
         />
-        <svg
+        {#if !compact}
-          class="spectrum-IllustratedMessage-illustration"
+          <svg
-          width="125"
+            class="spectrum-IllustratedMessage-illustration"
-          height="60"
+            width="125"
-          viewBox="0 0 199 97.7"
+            height="60"
-        >
+            viewBox="0 0 199 97.7"
-          <defs>
+          >
-            <style>
+            <defs>
-              .cls-1,
+              <style>
-              .cls-2 {
+                .cls-1,
-                fill: none;
+                .cls-2 {
-                stroke-linecap: round;
+                  fill: none;
-                stroke-linejoin: round;
+                  stroke-linecap: round;
-              }
+                  stroke-linejoin: round;
-              .cls-1 {
+                }
-                stroke-width: 3px;
+                .cls-1 {
-              }
+                  stroke-width: 3px;
-              .cls-2 {
+                }
-                stroke-width: 2px;
+                .cls-2 {
-              }
+                  stroke-width: 2px;
-            </style>
+                }
-          </defs>
+              </style>
-          <path
+            </defs>
-            class="cls-1"
+            <path
-            d="M110.53,85.66,100.26,95.89a1.09,1.09,0,0,1-1.52,0L88.47,85.66"
+              class="cls-1"
-          />
+              d="M110.53,85.66,100.26,95.89a1.09,1.09,0,0,1-1.52,0L88.47,85.66"
-          <line class="cls-1" x1="99.5" y1="95.5" x2="99.5" y2="58.5" />
+            />
-          <path class="cls-1" d="M105.5,73.5h19a2,2,0,0,0,2-2v-43" />
+            <line class="cls-1" x1="99.5" y1="95.5" x2="99.5" y2="58.5" />
-          <path
+            <path class="cls-1" d="M105.5,73.5h19a2,2,0,0,0,2-2v-43" />
-            class="cls-1"
+            <path
-            d="M126.5,22.5h-19a2,2,0,0,1-2-2V1.5h-31a2,2,0,0,0-2,2v68a2,2,0,0,0,2,2h19"
+              class="cls-1"
-          />
+              d="M126.5,22.5h-19a2,2,0,0,1-2-2V1.5h-31a2,2,0,0,0-2,2v68a2,2,0,0,0,2,2h19"
-          <line class="cls-1" x1="105.5" y1="1.5" x2="126.5" y2="22.5" />
+            />
-          <path
+            <line class="cls-1" x1="105.5" y1="1.5" x2="126.5" y2="22.5" />
-            class="cls-2"
+            <path
-            d="M47.93,50.49a5,5,0,1,0-4.83-5A4.93,4.93,0,0,0,47.93,50.49Z"
+              class="cls-2"
-          />
+              d="M47.93,50.49a5,5,0,1,0-4.83-5A4.93,4.93,0,0,0,47.93,50.49Z"
-          <path
+            />
-            class="cls-2"
+            <path
-            d="M36.6,65.93,42.05,60A2.06,2.06,0,0,1,45,60l12.68,13.2"
+              class="cls-2"
-          />
+              d="M36.6,65.93,42.05,60A2.06,2.06,0,0,1,45,60l12.68,13.2"
-          <path
+            />
-            class="cls-2"
+            <path
-            d="M3.14,73.23,22.42,53.76a1.65,1.65,0,0,1,2.38,0l19.05,19.7"
+              class="cls-2"
-          />
+              d="M3.14,73.23,22.42,53.76a1.65,1.65,0,0,1,2.38,0l19.05,19.7"
-          <path
+            />
-            class="cls-1"
+            <path
-            d="M139.5,36.5H196A1.49,1.49,0,0,1,197.5,38V72A1.49,1.49,0,0,1,196,73.5H141A1.49,1.49,0,0,1,139.5,72V32A1.49,1.49,0,0,1,141,30.5H154a2.43,2.43,0,0,1,1.67.66l6,5.66"
+              class="cls-1"
-          />
+              d="M139.5,36.5H196A1.49,1.49,0,0,1,197.5,38V72A1.49,1.49,0,0,1,196,73.5H141A1.49,1.49,0,0,1,139.5,72V32A1.49,1.49,0,0,1,141,30.5H154a2.43,2.43,0,0,1,1.67.66l6,5.66"
-          <rect
+            />
-            class="cls-1"
+            <rect
-            x="1.5"
+              class="cls-1"
-            y="34.5"
+              x="1.5"
-            width="58"
+              y="34.5"
-            height="39"
+              width="58"
-            rx="2"
+              height="39"
-            ry="2"
+              rx="2"
-          />
+              ry="2"
-        </svg>
+            />
-        <h2
+          </svg>
-          class="spectrum-Heading spectrum-Heading--sizeL spectrum-Heading--light spectrum-IllustratedMessage-heading"
+          <h2
-        >
+            class="spectrum-Heading spectrum-Heading--sizeL spectrum-Heading--light spectrum-IllustratedMessage-heading"
-          Drag and drop your file
+          >
-        </h2>
+            Drag and drop your file
+          </h2>
+        {/if}
         {#if !disabled}
           <p
             class="spectrum-Body spectrum-Body--sizeS spectrum-IllustratedMessage-description"
@@ -310,8 +313,10 @@
             <label for={fieldId} class="spectrum-Link">
               Select a file to upload
             </label>
-            <br />
+            {#if !compact}
-            from your computer
+              <br />
+              from your computer
+            {/if}
           </p>
           {#if fileTags.length}
             <Tags>

packages/bbui/src/Table/RelationshipRenderer.svelte

@@ -1,6 +1,7 @@
 <script>
   import "@spectrum-css/label/dist/index-vars.css"
   import { createEventDispatcher } from "svelte"
+  import Badge from "../Badge/Badge.svelte"
 
   export let row
   export let value
@@ -24,17 +25,11 @@
 
 {#each relationships as relationship}
   {#if relationship?.primaryDisplay}
-    <span class="spectrum-Label spectrum-Label--grey" on:click={onClick}>
+    <Badge hoverable grey on:click={onClick}>
       {relationship.primaryDisplay}
-    </span>
+    </Badge>
   {/if}
 {/each}
 {#if leftover}
   <div>+{leftover} more</div>
 {/if}
-
-<style>
-  span:hover {
-    cursor: pointer;
-  }
-</style>

packages/bbui/yarn.lock

@@ -2,7 +2,7 @@
 # yarn lockfile v1
 
 
-"@adobe/spectrum-css-workflow-icons@^1.2.1":
+"@adobe/spectrum-css-workflow-icons@1.2.1":
   version "1.2.1"
   resolved "https://registry.yarnpkg.com/@adobe/spectrum-css-workflow-icons/-/spectrum-css-workflow-icons-1.2.1.tgz#7e2cb3fcfb5c8b12d7275afafbb6ec44913551b4"
   integrity sha512-uVgekyBXnOVkxp+CUssjN/gefARtudZC8duEn1vm0lBQFwGRZFlDEzU1QC+aIRWCrD1Z8OgRpmBYlSZ7QS003w==

packages/builder/cypress/integration/adminAndManagement/accountPortals.spec.js

@@ -3,17 +3,17 @@ const interact = require('../../support/interact')
 
 filterTests(["smoke", "all"], () => {
   context("Account Portals", () => {
-    
+
     const bbUserEmail = "bbuser@test.com"
-    
+
     before(() => {
       cy.login()
       cy.deleteApp("Cypress Tests")
       cy.createApp("Cypress Tests", false)
-      
+
       // Create new user
       cy.wait(500)
-      cy.visit(`${Cypress.config().baseUrl}/builder`, { timeout: 5000})
+      cy.visit(`${Cypress.config().baseUrl}/builder`, { timeout: 5000 })
       cy.createUser(bbUserEmail)
       cy.contains("bbuser").click()
       cy.wait(500)
@@ -25,18 +25,18 @@ filterTests(["smoke", "all"], () => {
       cy.get(interact.SPECTRUM_MENU).within(() => {
         cy.get(interact.SPECTRUM_MENU_ITEM).contains("Force password reset").click({ force: true })
       })
-      
+
       cy.get(interact.SPECTRUM_DIALOG_GRID)
-      .find(interact.SPECTRUM_TEXTFIELD_INPUT).invoke('val').as('pwd')
+        .find(interact.SPECTRUM_TEXTFIELD_INPUT).invoke('val').as('pwd')
 
       cy.get(interact.SPECTRUM_BUTTON).contains("Reset password").click({ force: true })
-      
+
       // Login as new user and set password
       cy.logOut()
       cy.get('@pwd').then((pwd) => {
         cy.login(bbUserEmail, pwd)
       })
-      
+
       for (let i = 0; i < 2; i++) {
         cy.get(interact.SPECTRUM_TEXTFIELD_INPUT).eq(i).type("test")
       }
@@ -58,15 +58,15 @@ filterTests(["smoke", "all"], () => {
 
       cy.logoutNoAppGrid()
     })
-    
-    it("should verify Admin Portal", () => {
-        cy.login()
-        // Configure user role
-        cy.setUserRole("bbuser", "Admin")
-        bbUserLogin()
 
-        // Verify available options for Admin portal
+    xit("should verify Admin Portal", () => {
-        cy.get(interact.SPECTRUM_SIDENAV)
+      cy.login()
+      // Configure user role
+      cy.setUserRole("bbuser", "Admin")
+      bbUserLogin()
+
+      // Verify available options for Admin portal
+      cy.get(interact.SPECTRUM_SIDENAV)
         .should('contain', 'Apps')
         //.and('contain', 'Usage')
         .and('contain', 'Users')
@@ -75,12 +75,12 @@ filterTests(["smoke", "all"], () => {
         .and('contain', 'Organisation')
         .and('contain', 'Theming')
         .and('contain', 'Update')
-        //.and('contain', 'Upgrade')
+      //.and('contain', 'Upgrade')
 
-        cy.logOut()
+      cy.logOut()
     })
 
-    it("should verify Development Portal", () => {
+    xit("should verify Development Portal", () => {
       // Only Development access should be enabled
       cy.login()
       cy.setUserRole("bbuser", "Developer")
@@ -98,7 +98,7 @@ filterTests(["smoke", "all"], () => {
         .and('not.contain', 'Update')
         .and('not.contain', 'Upgrade')
 
-        cy.logOut()
+      cy.logOut()
     })
 
     const bbUserLogin = () => {

packages/builder/cypress/integration/appOverview.spec.js

@@ -9,7 +9,7 @@ filterTests(["all"], () => {
       cy.createApp("Cypress Tests")
     })
 
-    it("Should be accessible from the applications list", () => {
+    xit("Should be accessible from the applications list", () => {
       cy.visit(`${Cypress.config().baseUrl}/builder`)
       cy.get(".appTable .title")
         .eq(0)
@@ -27,7 +27,7 @@ filterTests(["all"], () => {
     })
 
     // Find a more suitable place for this.
-    it("Should allow unlocking in the app list", () => {
+    xit("Should allow unlocking in the app list", () => {
       cy.visit(`${Cypress.config().baseUrl}/builder`)
 
       cy.get(".appTable .lock-status").eq(0).contains("Locked by you").click()
@@ -38,7 +38,7 @@ filterTests(["all"], () => {
       cy.get(".lock-status").should("not.be.visible")
     })
 
-    it("Should allow unlocking in the app overview screen", () => {
+    xit("Should allow unlocking in the app overview screen", () => {
       cy.visit(`${Cypress.config().baseUrl}/builder`)
 
       cy.get(".appTable .app-row-actions button")
@@ -58,7 +58,7 @@ filterTests(["all"], () => {
       cy.get(".lock-status").should("not.be.visible")
     })
 
-    it("Should reflect the deploy state of an app that hasn't been published.", () => {
+    xit("Should reflect the deploy state of an app that hasn't been published.", () => {
       cy.visit(`${Cypress.config().baseUrl}/builder`)
 
       cy.get(".appTable .app-row-actions button")
@@ -81,7 +81,7 @@ filterTests(["all"], () => {
       })
     })
 
-    it("Should reflect the app deployment state", () => {
+    xit("Should reflect the app deployment state", () => {
       cy.visit(`${Cypress.config().baseUrl}/builder`, { timeout: 5000 })
       cy.get(".appTable .app-row-actions button")
         .contains("Edit")
@@ -117,7 +117,7 @@ filterTests(["all"], () => {
       })
     })
 
-    it("Should reflect an application that has been unpublished", () => {
+    xit("Should reflect an application that has been unpublished", () => {
       cy.visit(`${Cypress.config().baseUrl}/builder`)
       cy.get(".appTable .app-row-actions button")
         .contains("Edit")
@@ -154,7 +154,7 @@ filterTests(["all"], () => {
       })
     })
 
-    it("Should allow the editing of the application icon and colour", () => {
+    xit("Should allow the editing of the application icon and colour", () => {
       cy.visit(`${Cypress.config().baseUrl}/builder`)
       cy.get(".appTable .app-row-actions button")
         .contains("Manage")
@@ -196,7 +196,7 @@ filterTests(["all"], () => {
       })
     })
 
-    it("Should reflect the last time the application was edited", () => {
+    xit("Should reflect the last time the application was edited", () => {
       cy.visit(`${Cypress.config().baseUrl}/builder`)
       cy.get(".appTable .app-row-actions button")
         .contains("Manage")
@@ -221,7 +221,7 @@ filterTests(["all"], () => {
       })
     })
 
-    it("Should reflect application version is up-to-date", () => {
+    xit("Should reflect application version is up-to-date", () => {
       cy.visit(`${Cypress.config().baseUrl}/builder`)
       cy.get(".appTable .app-row-actions button")
         .contains("Manage")
@@ -302,7 +302,7 @@ filterTests(["all"], () => {
       })
     })
 
-    it("Should allow editing of the app details.", () => {
+    xit("Should allow editing of the app details.", () => {
       cy.visit(`${Cypress.config().baseUrl}/builder`, { timeout: 5000 })
       cy.get(".appTable .app-row-actions button")
         .contains("Manage")
@@ -373,13 +373,13 @@ filterTests(["all"], () => {
             .contains("Copy App ID")
             .click({ force: true })
         })
-      
+
       cy.get(".spectrum-Toast-content")
-      .contains("App ID copied to clipboard.")
+        .contains("App ID copied to clipboard.")
-      .should("be.visible")
+        .should("be.visible")
     })
 
-    it("Should allow unpublishing of the application via the Unpublish link", () => {
+    xit("Should allow unpublishing of the application via the Unpublish link", () => {
       cy.visit(`${Cypress.config().baseUrl}/builder`)
       cy.get(".appTable .app-row-actions button")
         .contains("Manage")
@@ -388,7 +388,7 @@ filterTests(["all"], () => {
 
       cy.get(`[data-cy="app-status"]`).within(() => {
         cy.contains("Unpublish").click({ force: true })
-        })
+      })
 
       cy.get("[data-cy='unpublish-modal']")
         .should("be.visible")
@@ -399,11 +399,11 @@ filterTests(["all"], () => {
       cy.get(".overview-tab [data-cy='app-status']").within(() => {
         cy.get(".status-display").contains("Unpublished")
         cy.get(".status-display .icon svg[aria-label='GlobeStrike']")
-        .should("exist")
+          .should("exist")
       })
     })
 
-    it("Should allow deleting of the application", () => {
+    xit("Should allow deleting of the application", () => {
       cy.visit(`${Cypress.config().baseUrl}/builder`)
       cy.get(".appTable .app-row-actions button")
         .contains("Manage")

packages/builder/cypress/integration/createAutomation.spec.js

@@ -2,13 +2,13 @@ import filterTests from "../support/filterTests"
 const interact = require('../support/interact')
 
 filterTests(['smoke', 'all'], () => {
-  context("Create a automation", () => {
+  xcontext("Create a automation", () => {
     before(() => {
       cy.login()
       cy.createTestApp()
     })
 
-    it("should create a automation", () => {
+    xit("should create a automation", () => {
       cy.createTestTableWithData()
       cy.wait(2000)
       cy.contains("Automate").click()

packages/builder/cypress/support/commands.js

@@ -461,10 +461,7 @@ Cypress.Commands.add("createTable", (tableName, initialTable) => {
   cy.get(".nav-item", { timeout: 2000 })
     .contains("Budibase DB")
     .click({ force: true })
-  cy.get(".spectrum-Tabs-content", { timeout: 2000 }).should(
+  cy.get(".nav-item-content", { timeout: 2000 }).should("contain", tableName)
-    "contain",
-    tableName
-  )
 })
 
 Cypress.Commands.add("createTestTableWithData", () => {

packages/builder/package.json

@@ -123,4 +123,4 @@
     "vite": "^3.0.8"
   },
   "gitHead": "115189f72a850bfb52b65ec61d932531bf327072"
-}
+}

packages/builder/src/components/backend/DataTable/Table.svelte

@@ -216,7 +216,6 @@
     flex-direction: row;
     justify-content: flex-start;
     align-items: center;
-    margin-top: var(--spacing-m);
   }
   .table-title > div {
     margin-left: var(--spacing-xs);

packages/builder/src/components/backend/DataTable/buttons/ExistingRelationshipButton.svelte

@@ -1,7 +1,7 @@
 <script>
   import { ActionButton, Modal, notifications } from "@budibase/bbui"
   import CreateEditRelationship from "../../Datasources/CreateEditRelationship.svelte"
-  import { datasources, tables } from "../../../../stores/backend"
+  import { datasources } from "../../../../stores/backend"
   import { createEventDispatcher } from "svelte"
 
   export let table
@@ -21,8 +21,6 @@
       // Create datasource
       await datasources.save(datasource)
       notifications.success(`Relationship information saved.`)
-      const tableList = await tables.fetch()
-      await tables.select(tableList.find(tbl => tbl._id === table._id))
       dispatch("updatecolumns")
     } catch (err) {
       notifications.error(`Error saving relationship info: ${err}`)

packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte

@@ -102,7 +102,7 @@
   // in the case of internal tables the sourceId will just be undefined
   $: tableOptions = $tables.list.filter(
     opt =>
-      opt._id !== $tables.draft._id &&
+      opt._id !== $tables.selected._id &&
       opt.type === table.type &&
       table.sourceId === opt.sourceId
   )
@@ -112,7 +112,7 @@
 
   async function saveColumn() {
     if (field.type === AUTO_TYPE) {
-      field = buildAutoColumn($tables.draft.name, field.name, field.subtype)
+      field = buildAutoColumn($tables.selected.name, field.name, field.subtype)
     }
     if (field.type !== LINK_TYPE) {
       delete field.fieldName
@@ -310,7 +310,7 @@
       newError.name = `${PROHIBITED_COLUMN_NAMES.join(
         ", "
       )} are not allowed as column names`
-    } else if (inUse($tables.draft, fieldInfo.name, originalName)) {
+    } else if (inUse($tables.selected, fieldInfo.name, originalName)) {
       newError.name = `Column name already in use.`
     }
     if (fieldInfo.fieldName && fieldInfo.tableId) {

packages/builder/src/components/backend/DataTable/modals/CreateEditRow.svelte

@@ -1,6 +1,6 @@
 <script>
   import { createEventDispatcher } from "svelte"
-  import { tables, rows } from "stores/backend"
+  import { tables } from "stores/backend"
   import { notifications } from "@budibase/bbui"
   import RowFieldControl from "../RowFieldControl.svelte"
   import { API } from "api"
@@ -25,7 +25,6 @@
     try {
       await API.saveRow({ ...row, tableId: table._id })
       notifications.success("Row saved successfully")
-      rows.save()
       dispatch("updaterows")
     } catch (error) {
       if (error.handled) {

packages/builder/src/components/backend/DataTable/modals/CreateEditUser.svelte

@@ -1,6 +1,6 @@
 <script>
   import { createEventDispatcher } from "svelte"
-  import { tables, rows } from "stores/backend"
+  import { tables } from "stores/backend"
   import { roles } from "stores/backend"
   import { notifications } from "@budibase/bbui"
   import RowFieldControl from "../RowFieldControl.svelte"
@@ -57,7 +57,6 @@
     try {
       await API.saveRow({ ...row, tableId: table._id })
       notifications.success("User saved successfully")
-      rows.save()
       dispatch("updaterows")
     } catch (error) {
       if (error.handled) {

packages/builder/src/components/backend/DataTable/modals/CreateViewModal.svelte

@@ -9,13 +9,13 @@
 
   $: views = $tables.list.flatMap(table => Object.keys(table.views || {}))
 
-  function saveView() {
+  const saveView = async () => {
     if (views.includes(name)) {
       notifications.error(`View exists with name ${name}`)
       return
     }
     try {
-      viewsStore.save({
+      await viewsStore.save({
         name,
         tableId: $tables.selected._id,
         field,

packages/builder/src/components/backend/DatasourceNavigator/DatasourceNavigator.svelte

@@ -1,7 +1,5 @@
 <script>
-  import { onMount } from "svelte"
+  import { goto, isActive, params } from "@roxi/routify"
-  import { get } from "svelte/store"
-  import { goto, params } from "@roxi/routify"
   import { BUDIBASE_INTERNAL_DB_ID } from "constants/backend"
   import { database, datasources, queries, tables, views } from "stores/backend"
   import EditDatasourcePopover from "./popovers/EditDatasourcePopover.svelte"
@@ -14,40 +12,61 @@
     customQueryText,
   } from "helpers/data/utils"
   import IntegrationIcon from "./IntegrationIcon.svelte"
-  import { notifications } from "@budibase/bbui"
 
   let openDataSources = []
-  $: enrichedDataSources = Array.isArray($datasources.list)
+  $: enrichedDataSources = enrichDatasources(
-    ? $datasources.list.map(datasource => {
+    $datasources,
-        const selected = $datasources.selected === datasource._id
+    $params,
-        const open = openDataSources.includes(datasource._id)
+    $isActive,
-        const containsSelected = containsActiveEntity(datasource)
+    $tables,
-        const onlySource = $datasources.list.length === 1
+    $queries,
-        return {
+    $views
-          ...datasource,
+  )
-          selected,
-          open: selected || open || containsSelected || onlySource,
-        }
-      })
-    : []
   $: openDataSource = enrichedDataSources.find(x => x.open)
   $: {
-    // Ensure the open datasource is always included in the list of open
+    // Ensure the open datasource is always actually open
-    // datasources
     if (openDataSource) {
       openNode(openDataSource)
     }
   }
 
-  function selectDatasource(datasource) {
+  const enrichDatasources = (
-    openNode(datasource)
+    datasources,
-    datasources.select(datasource._id)
+    params,
-    $goto(`./datasource/${datasource._id}`)
+    isActive,
+    tables,
+    queries,
+    views
+  ) => {
+    if (!datasources?.list?.length) {
+      return []
+    }
+    return datasources.list.map(datasource => {
+      const selected =
+        isActive("./datasource") &&
+        datasources.selectedDatasourceId === datasource._id
+      const open = openDataSources.includes(datasource._id)
+      const containsSelected = containsActiveEntity(
+        datasource,
+        params,
+        isActive,
+        tables,
+        queries,
+        views
+      )
+      const onlySource = datasources.list.length === 1
+      return {
+        ...datasource,
+        selected,
+        containsSelected,
+        open: selected || open || containsSelected || onlySource,
+      }
+    })
   }
 
-  function onClickQuery(query) {
+  function selectDatasource(datasource) {
-    queries.select(query)
+    openNode(datasource)
-    $goto(`./datasource/${query.datasourceId}/${query._id}`)
+    $goto(`./datasource/${datasource._id}`)
   }
 
   function closeNode(datasource) {
@@ -69,21 +88,39 @@
     }
   }
 
-  onMount(async () => {
+  const containsActiveEntity = (
-    try {
+    datasource,
-      await datasources.fetch()
+    params,
-      await queries.fetch()
+    isActive,
-    } catch (error) {
+    tables,
-      notifications.error("Error fetching datasources and queries")
+    queries,
-    }
+    views
-  })
+  ) => {
-
+    // Check for being on a datasource page
-  const containsActiveEntity = datasource => {
+    if (params.datasourceId === datasource._id) {
-    // If we're view a query then the datasource ID is in the URL
-    if ($params.selectedDatasource === datasource._id) {
       return true
     }
 
+    // Check for hardcoded datasource edge cases
+    if (
+      isActive("./datasource/bb_internal") &&
+      datasource._id === "bb_internal"
+    ) {
+      return true
+    }
+    if (
+      isActive("./datasource/datasource_internal_bb_default") &&
+      datasource._id === "datasource_internal_bb_default"
+    ) {
+      return true
+    }
+
+    // Check for a matching query
+    if (params.queryId) {
+      const query = queries.list?.find(q => q._id === params.queryId)
+      return datasource._id === query?.datasourceId
+    }
+
     // If there are no entities it can't contain anything
     if (!datasource.entities) {
       return false
@@ -96,13 +133,13 @@
     }
 
     // Check for a matching table
-    if ($params.selectedTable) {
+    if (params.tableId) {
-      const selectedTable = get(tables).selected?._id
+      const selectedTable = tables.selected?._id
       return options.find(x => x._id === selectedTable) != null
     }
 
     // Check for a matching view
-    const selectedView = get(views).selected?.name
+    const selectedView = views.selected?.name
     const table = options.find(table => {
       return table.views?.[selectedView] != null
     })
@@ -117,7 +154,7 @@
         border={idx > 0}
         text={datasource.name}
         opened={datasource.open}
-        selected={datasource.selected}
+        selected={$isActive("./datasource") && datasource.selected}
         withArrow={true}
         on:click={() => selectDatasource(datasource)}
         on:iconClick={() => toggleNode(datasource)}
@@ -143,11 +180,11 @@
             iconText={customQueryIconText(datasource, query)}
             iconColor={customQueryIconColor(datasource, query)}
             text={customQueryText(datasource, query)}
-            opened={$queries.selected === query._id}
+            selected={$isActive("./query/:queryId") &&
-            selected={$queries.selected === query._id}
+              $queries.selectedQueryId === query._id}
-            on:click={() => onClickQuery(query)}
+            on:click={() => $goto(`./query/${query._id}`)}
           >
-            <EditQueryPopover {query} {onClickQuery} />
+            <EditQueryPopover {query} />
           </NavItem>
         {/each}
       {/if}
@@ -156,6 +193,9 @@
 {/if}
 
 <style>
+  .hierarchy-items-container {
+    margin: 0 calc(-1 * var(--spacing-xl));
+  }
   .datasource-icon {
     display: grid;
     place-items: center;

packages/builder/src/components/backend/DatasourceNavigator/IntegrationIcon.svelte

@@ -1,18 +1,15 @@
 <script>
   import { getIcon } from "./icons"
   import CustomSVG from "components/common/CustomSVG.svelte"
-  import { admin } from "stores/portal"
 
   export let integrationType
   export let schema
   export let size = "18"
 
-  $: objectStoreUrl = $admin.cloud ? "https://cdn.budi.live" : ""
-  $: pluginsUrl = `${objectStoreUrl}/plugins`
   $: iconInfo = getIcon(integrationType, schema)
 
   async function getSvgFromUrl(info) {
-    const url = `${pluginsUrl}/${info.url}`
+    const url = `${info.url}`
     const resp = await fetch(url, {
       headers: {
         ["pragma"]: "no-cache",

packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/PlusConfigForm.svelte

@@ -104,7 +104,6 @@
   }
 
   function onClickTable(table) {
-    tables.select(table)
     $goto(`../../table/${table._id}`)
   }
 

packages/builder/src/components/backend/DatasourceNavigator/modals/CreateDatasourceModal.svelte

@@ -11,7 +11,7 @@
   import { onMount } from "svelte"
   import ICONS from "../icons"
   import { API } from "api"
-  import { IntegrationTypes } from "constants/backend"
+  import { IntegrationTypes, DatasourceTypes } from "constants/backend"
   import CreateTableModal from "components/backend/TableNavigator/modals/CreateTableModal.svelte"
   import DatasourceConfigModal from "components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte"
   import GoogleDatasourceConfigModal from "components/backend/DatasourceNavigator/modals/GoogleDatasourceConfigModal.svelte"
@@ -31,6 +31,7 @@
   $: customIntegrations = Object.entries(integrations).filter(
     entry => entry[1].custom
   )
+  $: sortedIntegrations = sortIntegrations(integrations)
 
   checkShowImport()
 
@@ -99,6 +100,29 @@
     }
     integrations = newIntegrations
   }
+
+  function sortIntegrations(integrations) {
+    let integrationsArray = Object.entries(integrations)
+    function getTypeOrder(schema) {
+      if (schema.type === DatasourceTypes.API) {
+        return 1
+      }
+      if (schema.type === DatasourceTypes.RELATIONAL) {
+        return 2
+      }
+      return schema.type?.charCodeAt(0)
+    }
+
+    integrationsArray.sort((a, b) => {
+      let typeOrderA = getTypeOrder(a[1])
+      let typeOrderB = getTypeOrder(b[1])
+      if (typeOrderA === typeOrderB) {
+        return a[1].friendlyName?.localeCompare(b[1].friendlyName)
+      }
+      return typeOrderA < typeOrderB ? -1 : 1
+    })
+    return integrationsArray
+  }
 </script>
 
 <Modal bind:this={internalTableModal}>
@@ -157,7 +181,7 @@
     <Layout noPadding gap="XS">
       <Body size="S">Connect to an external datasource</Body>
       <div class="item-list">
-        {#each Object.entries(integrations).filter(([key, val]) => key !== IntegrationTypes.INTERNAL && !val.custom) as [integrationType, schema]}
+        {#each sortedIntegrations.filter(([key, val]) => key !== IntegrationTypes.INTERNAL && !val.custom) as [integrationType, schema]}
           <DatasourceCard
             on:selected={evt => selectIntegration(evt.detail)}
             {schema}

packages/builder/src/components/backend/DatasourceNavigator/modals/ImportRestQueriesModal.svelte

@@ -64,7 +64,6 @@
       // reload
       await datasources.fetch()
       await queries.fetch()
-      await datasources.select(datasourceId)
 
       if (navigateDatasource) {
         $goto(`./datasource/${datasourceId}`)

packages/builder/src/components/backend/DatasourceNavigator/popovers/EditDatasourcePopover.svelte

@@ -1,6 +1,6 @@
 <script>
   import { goto } from "@roxi/routify"
-  import { datasources, queries, tables } from "stores/backend"
+  import { datasources } from "stores/backend"
   import { notifications } from "@budibase/bbui"
   import { ActionMenu, MenuItem, Icon } from "@budibase/bbui"
   import ConfirmDialog from "components/common/ConfirmDialog.svelte"
@@ -14,23 +14,10 @@
 
   async function deleteDatasource() {
     try {
-      let wasSelectedSource = $datasources.selected
+      const isSelected = datasource.selected || datasource.containsSelected
-      if (!wasSelectedSource && $queries.selected) {
-        const queryId = $queries.selected
-        wasSelectedSource = $datasources.list.find(ds =>
-          queryId.includes(ds._id)
-        )?._id
-      }
-      const wasSelectedTable = $tables.selected
       await datasources.delete(datasource)
       notifications.success("Datasource deleted")
-      // Navigate to first index page if the source you are deleting is selected
+      if (isSelected) {
-      const entities = Object.values(datasource?.entities || {})
-      if (
-        wasSelectedSource === datasource._id ||
-        (entities &&
-          entities.find(entity => entity._id === wasSelectedTable?._id))
-      ) {
         $goto("./datasource")
       }
     } catch (error) {

packages/builder/src/components/backend/DatasourceNavigator/popovers/EditQueryPopover.svelte

@@ -5,23 +5,17 @@
   import { datasources, queries } from "stores/backend"
 
   export let query
-  export let onClickQuery
 
   let confirmDeleteDialog
 
   async function deleteQuery() {
     try {
-      const wasSelectedQuery = $queries.selected
+      // Go back to the datasource if we are deleting the active query
-      // need to calculate this before the query is deleted
+      if ($queries.selectedQueryId === query._id) {
-      const navigateToDatasource = wasSelectedQuery === query._id
-
-      await queries.delete(query)
-      await datasources.fetch()
-
-      if (navigateToDatasource) {
-        await datasources.select(query.datasourceId)
         $goto(`./datasource/${query.datasourceId}`)
       }
+      await queries.delete(query)
+      await datasources.fetch()
       notifications.success("Query deleted")
     } catch (error) {
       notifications.error("Error deleting query")
@@ -31,7 +25,7 @@
   async function duplicateQuery() {
     try {
       const newQuery = await queries.duplicate(query)
-      onClickQuery(newQuery)
+      $goto(`./query/${newQuery._id}`)
     } catch (error) {
       notifications.error("Error duplicating query")
     }

packages/builder/src/components/backend/TableNavigator/TableNavigator.svelte

@@ -1,39 +1,18 @@
 <script>
-  import { goto } from "@roxi/routify"
   import { tables, views, database } from "stores/backend"
   import { TableNames } from "constants"
   import EditTablePopover from "./popovers/EditTablePopover.svelte"
   import EditViewPopover from "./popovers/EditViewPopover.svelte"
   import NavItem from "components/common/NavItem.svelte"
+  import { goto, isActive } from "@roxi/routify"
 
   const alphabetical = (a, b) => a.name?.toLowerCase() > b.name?.toLowerCase()
 
   export let sourceId
 
-  $: selectedView = $views.selected && $views.selected.name
   $: sortedTables = $tables.list
     .filter(table => table.sourceId === sourceId)
     .sort(alphabetical)
-
-  function selectTable(table) {
-    tables.select(table)
-    $goto(`./table/${table._id}`)
-  }
-
-  function selectView(view) {
-    views.select(view)
-    $goto(`./view/${view.name}`)
-  }
-
-  function onClickView(table, viewName) {
-    if (selectedView === viewName) {
-      return
-    }
-    selectView({
-      name: viewName,
-      ...table.views[viewName],
-    })
-  }
 </script>
 
 {#if $database?._id}
@@ -44,8 +23,9 @@
         border={idx > 0}
         icon={table._id === TableNames.USERS ? "UserGroup" : "Table"}
         text={table.name}
-        selected={$tables.selected?._id === table._id}
+        selected={$isActive("./table/:tableId") &&
-        on:click={() => selectTable(table)}
+          $tables.selected?._id === table._id}
+        on:click={() => $goto(`./table/${table._id}`)}
       >
         {#if table._id !== TableNames.USERS}
           <EditTablePopover {table} />
@@ -56,8 +36,9 @@
           indentLevel={2}
           icon="Remove"
           text={viewName}
-          selected={selectedView === viewName}
+          selected={$isActive("./view/:viewName") &&
-          on:click={() => onClickView(table, viewName)}
+            $views.selected?.name === viewName}
+          on:click={() => $goto(`./view/${viewName}`)}
         >
           <EditViewPopover
             view={{ name: viewName, ...table.views[viewName] }}

packages/builder/src/components/backend/TableNavigator/popovers/EditTablePopover.svelte

@@ -1,5 +1,5 @@
 <script>
-  import { goto } from "@roxi/routify"
+  import { goto, params } from "@roxi/routify"
   import { store } from "builderStore"
   import { cloneDeep } from "lodash/fp"
   import { tables, datasources } from "stores/backend"
@@ -41,17 +41,16 @@
   }
 
   async function deleteTable() {
-    const wasSelectedTable = $tables.selected
+    const isSelected = $params.tableId === table._id
     try {
       await tables.delete(table)
       await store.actions.screens.delete(templateScreens)
-      await tables.fetch()
       if (table.type === "external") {
         await datasources.fetch()
       }
       notifications.success("Table deleted")
-      if (wasSelectedTable && wasSelectedTable._id === table._id) {
+      if (isSelected) {
-        $goto("./table")
+        $goto(`./datasource/${table.datasourceId}`)
       }
     } catch (error) {
       notifications.error("Error deleting table")

packages/builder/src/components/backend/TableNavigator/popovers/EditViewPopover.svelte

@@ -1,5 +1,5 @@
 <script>
-  import { goto } from "@roxi/routify"
+  import { goto, params } from "@roxi/routify"
   import { views } from "stores/backend"
   import { cloneDeep } from "lodash/fp"
   import ConfirmDialog from "components/common/ConfirmDialog.svelte"
@@ -33,11 +33,14 @@
 
   async function deleteView() {
     try {
+      const isSelected = $params.viewName === $views.selectedViewName
       const name = view.name
       const id = view.tableId
       await views.delete(name)
       notifications.success("View deleted")
-      $goto(`./table/${id}`)
+      if (isSelected) {
+        $goto(`./table/${id}`)
+      }
     } catch (error) {
       notifications.error("Error deleting view")
     }

packages/builder/src/components/deploy/DeployModal.svelte

@@ -12,7 +12,6 @@
   import { ProgressCircle } from "@budibase/bbui"
   import CopyInput from "components/common/inputs/CopyInput.svelte"
 
-  let feedbackModal
   let publishModal
   let asyncModal
   let publishCompleteModal
@@ -23,13 +22,13 @@
 
   export let onOk
 
-  async function deployApp() {
+  async function publishApp() {
     try {
       //In Progress
       asyncModal.show()
       publishModal.hide()
 
-      published = await API.deployAppChanges()
+      published = await API.publishAppChanges($store.appId)
 
       if (typeof onOk === "function") {
         await onOk()
@@ -56,20 +55,11 @@
 </script>
 
 <Button cta on:click={publishModal.show}>Publish</Button>
-<Modal bind:this={feedbackModal}>
-  <ModalContent
-    title="Enjoying Budibase?"
-    size="L"
-    showConfirmButton={false}
-    showCancelButton={false}
-  />
-</Modal>
-
 <Modal bind:this={publishModal}>
   <ModalContent
     title="Publish to Production"
     confirmText="Publish"
-    onConfirm={deployApp}
+    onConfirm={publishApp}
     dataCy={"deploy-app-modal"}
   >
     <span

packages/builder/src/components/design/Panel.svelte

@@ -51,6 +51,7 @@
 <style>
   .panel {
     width: 260px;
+    flex: 0 0 260px;
     background: var(--background);
     display: flex;
     flex-direction: column;
@@ -66,6 +67,7 @@
   }
   .panel.wide {
     width: 420px;
+    flex: 0 0 420px;
   }
   .header {
     flex: 0 0 48px;

packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/PromptUser.svelte

@@ -0,0 +1,50 @@
+<script>
+  import { Body, Label, Input } from "@budibase/bbui"
+  import { onMount } from "svelte"
+
+  export let parameters
+
+  onMount(() => {
+    if (!parameters.confirm) {
+      parameters.confirm = true
+    }
+  })
+</script>
+
+<div class="root">
+  <Body size="S">Enter the message you wish to display to the user.</Body>
+  <div class="params">
+    <Label small>Title</Label>
+    <Input placeholder="Prompt User" bind:value={parameters.customTitleText} />
+    <Label small>Message</Label>
+    <Input
+      placeholder="Are you sure you want to continue?"
+      bind:value={parameters.confirmText}
+    />
+  </div>
+</div>
+
+<style>
+  .root {
+    width: 100%;
+    max-width: 800px;
+    margin: 0 auto;
+    display: flex;
+    flex-direction: column;
+    justify-content: flex-start;
+    align-items: stretch;
+    gap: var(--spacing-xl);
+  }
+
+  .root :global(p) {
+    line-height: 1.5;
+  }
+
+  .params {
+    display: grid;
+    column-gap: var(--spacing-l);
+    row-gap: var(--spacing-s);
+    grid-template-columns: 60px 1fr;
+    align-items: center;
+  }
+</style>

packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/index.js

@@ -16,5 +16,6 @@ export { default as ExportData } from "./ExportData.svelte"
 export { default as ContinueIf } from "./ContinueIf.svelte"
 export { default as UpdateFieldValue } from "./UpdateFieldValue.svelte"
 export { default as ShowNotification } from "./ShowNotification.svelte"
+export { default as PromptUser } from "./PromptUser.svelte"
 export { default as OpenSidePanel } from "./OpenSidePanel.svelte"
 export { default as CloseSidePanel } from "./CloseSidePanel.svelte"

packages/builder/src/components/design/settings/controls/ButtonActionEditor/manifest.json

@@ -117,6 +117,11 @@
       "component": "ShowNotification",
       "dependsOnFeature": "showNotificationAction"
     },
+    {
+      "name": "Prompt User",
+      "type": "application",
+      "component": "PromptUser"
+    },
     {
       "name": "Open Side Panel",
       "type": "application",

packages/builder/src/components/integration/QueryViewer.svelte

@@ -29,11 +29,12 @@
 
   export let query
 
+  const transformerDocs = "https://docs.budibase.com/docs/transformers"
+
   let fields = query?.schema ? schemaToFields(query.schema) : []
   let parameters
   let data = []
   let saveId
-  const transformerDocs = "https://docs.budibase.com/docs/transformers"
 
   $: datasource = $datasources.list.find(ds => ds._id === query.datasourceId)
   $: query.schema = fieldsToSchema(fields)
@@ -94,132 +95,144 @@
     try {
       const { _id } = await queries.save(query.datasourceId, query)
       saveId = _id
-      notifications.success(`Query saved successfully.`)
+      notifications.success(`Query saved successfully`)
-      $goto(`../${_id}`)
+
+      // Go to the correct URL if we just created a new query
+      if (!query._rev) {
+        $goto(`../../${_id}`)
+      }
     } catch (error) {
-      notifications.error("Error creating query")
+      notifications.error("Error saving query")
     }
   }
 </script>
 
-<Layout gap="S" noPadding>
+<div class="wrapper">
-  <Heading size="M">Query {integrationInfo?.friendlyName}</Heading>
+  <Layout gap="S" noPadding>
-  <Divider />
+    <Heading size="M">Query {integrationInfo?.friendlyName}</Heading>
-  <Heading size="S">Config</Heading>
-  <div class="config">
-    <div class="config-field">
-      <Label>Query Name</Label>
-      <Input bind:value={query.name} />
-    </div>
-    {#if queryConfig}
-      <div class="config-field">
-        <Label>Function</Label>
-        <Select
-          bind:value={query.queryVerb}
-          on:change={resetDependentFields}
-          options={Object.keys(queryConfig)}
-          getOptionLabel={verb =>
-            queryConfig[verb]?.displayName || capitalise(verb)}
-        />
-      </div>
-      <div class="config-field">
-        <AccessLevelSelect {saveId} {query} label="Access Level" />
-      </div>
-      {#if integrationInfo?.extra && query.queryVerb}
-        <ExtraQueryConfig
-          {query}
-          {populateExtraQuery}
-          config={integrationInfo.extra}
-        />
-      {/if}
-      {#key query.parameters}
-        <BindingBuilder
-          queryBindings={query.parameters}
-          bindable={false}
-          on:change={e => {
-            query.parameters = e.detail.map(binding => {
-              return {
-                name: binding.name,
-                default: binding.value,
-              }
-            })
-          }}
-        />
-      {/key}
-    {/if}
-  </div>
-  {#if shouldShowQueryConfig}
     <Divider />
+    <Heading size="S">Config</Heading>
     <div class="config">
-      <Heading size="S">Fields</Heading>
+      <div class="config-field">
-      <Body size="S">Fill in the fields specific to this query.</Body>
+        <Label>Query Name</Label>
-      <IntegrationQueryEditor
+        <Input bind:value={query.name} />
-        {datasource}
-        {query}
-        height={200}
-        schema={queryConfig[query.queryVerb]}
-        bind:parameters
-      />
-      <Divider />
-    </div>
-    <div class="config">
-      <div class="help-heading">
-        <Heading size="S">Transformer</Heading>
-        <Icon
-          on:click={() => window.open(transformerDocs)}
-          hoverable
-          name="Help"
-          size="L"
-        />
       </div>
-      <Body size="S"
+      {#if queryConfig}
-        >Add a JavaScript function to transform the query result.</Body
+        <div class="config-field">
-      >
+          <Label>Function</Label>
-      <CodeMirrorEditor
+          <Select
-        height={200}
+            bind:value={query.queryVerb}
-        label="Transformer"
+            on:change={resetDependentFields}
-        value={query.transformer}
+            options={Object.keys(queryConfig)}
-        resize="vertical"
+            getOptionLabel={verb =>
-        on:change={e => (query.transformer = e.detail)}
+              queryConfig[verb]?.displayName || capitalise(verb)}
-      />
+          />
-      <Divider />
+        </div>
-    </div>
+        <div class="config-field">
-    <div class="viewer-controls">
+          <AccessLevelSelect {saveId} {query} label="Access Level" />
-      <Heading size="S">Results</Heading>
+        </div>
-      <ButtonGroup gap="M">
+        {#if integrationInfo?.extra && query.queryVerb}
-        <Button cta disabled={queryInvalid} on:click={saveQuery}>
+          <ExtraQueryConfig
-          Save Query
+            {query}
-        </Button>
+            {populateExtraQuery}
-        <Button secondary on:click={previewQuery}>Run Query</Button>
+            config={integrationInfo.extra}
-      </ButtonGroup>
+          />
-    </div>
+        {/if}
-    <Body size="S">
+        {#key query.parameters}
-      Below, you can preview the results from your query and change the schema.
+          <BindingBuilder
-    </Body>
+            queryBindings={query.parameters}
-    <section class="viewer">
+            bindable={false}
-      {#if data}
+            on:change={e => {
-        <Tabs selected="JSON">
+              query.parameters = e.detail.map(binding => {
-          <Tab title="JSON">
+                return {
-            <JSONPreview data={data[0]} minHeight="120" />
+                  name: binding.name,
-          </Tab>
+                  default: binding.value,
-          <Tab title="Schema">
+                }
-            <KeyValueBuilder
+              })
-              bind:object={fields}
+            }}
-              name="field"
+          />
-              headings
+        {/key}
-              options={SchemaTypeOptions}
-            />
-          </Tab>
-          <Tab title="Preview">
-            <ExternalDataSourceTable {query} {data} />
-          </Tab>
-        </Tabs>
       {/if}
-    </section>
+    </div>
-  {/if}
+    {#if shouldShowQueryConfig}
-</Layout>
+      <Divider />
+      <div class="config">
+        <Heading size="S">Fields</Heading>
+        <Body size="S">Fill in the fields specific to this query.</Body>
+        <IntegrationQueryEditor
+          {datasource}
+          {query}
+          height={200}
+          schema={queryConfig[query.queryVerb]}
+          bind:parameters
+        />
+        <Divider />
+      </div>
+      <div class="config">
+        <div class="help-heading">
+          <Heading size="S">Transformer</Heading>
+          <Icon
+            on:click={() => window.open(transformerDocs)}
+            hoverable
+            name="Help"
+            size="L"
+          />
+        </div>
+        <Body size="S"
+          >Add a JavaScript function to transform the query result.</Body
+        >
+        <CodeMirrorEditor
+          height={200}
+          label="Transformer"
+          value={query.transformer}
+          resize="vertical"
+          on:change={e => (query.transformer = e.detail)}
+        />
+        <Divider />
+      </div>
+      <div class="viewer-controls">
+        <Heading size="S">Results</Heading>
+        <ButtonGroup gap="XS">
+          <Button cta disabled={queryInvalid} on:click={saveQuery}>
+            Save Query
+          </Button>
+          <Button secondary on:click={previewQuery}>Run Query</Button>
+        </ButtonGroup>
+      </div>
+      <Body size="S">
+        Below, you can preview the results from your query and change the
+        schema.
+      </Body>
+      <section class="viewer">
+        {#if data}
+          <Tabs selected="JSON">
+            <Tab title="JSON">
+              <JSONPreview data={data[0]} minHeight="120" />
+            </Tab>
+            <Tab title="Schema">
+              <KeyValueBuilder
+                bind:object={fields}
+                name="field"
+                headings
+                options={SchemaTypeOptions}
+              />
+            </Tab>
+            <Tab title="Preview">
+              <ExternalDataSourceTable {query} {data} />
+            </Tab>
+          </Tabs>
+        {/if}
+      </section>
+    {/if}
+  </Layout>
+</div>
 
 <style>
+  .wrapper {
+    width: 640px;
+    margin: auto;
+  }
+
   .config {
     display: grid;
     grid-gap: var(--spacing-s);

packages/builder/src/components/integration/RestQueryViewer.svelte

@@ -1,5 +1,5 @@
 <script>
-  import { params } from "@roxi/routify"
+  import { goto, params } from "@roxi/routify"
   import { datasources, flags, integrations, queries } from "stores/backend"
   import {
     Banner,
@@ -23,7 +23,7 @@
   import CodeMirrorEditor, {
     EditorModes,
   } from "components/common/CodeMirrorEditor.svelte"
-  import RestBodyInput from "../../_components/RestBodyInput.svelte"
+  import RestBodyInput from "./RestBodyInput.svelte"
   import { capitalise } from "helpers"
   import { onMount } from "svelte"
   import restUtils from "helpers/data/utils"
@@ -36,7 +36,7 @@
   } from "constants/backend"
   import JSONPreview from "components/integration/JSONPreview.svelte"
   import AccessLevelSelect from "components/integration/AccessLevelSelect.svelte"
-  import DynamicVariableModal from "../../_components/DynamicVariableModal.svelte"
+  import DynamicVariableModal from "./DynamicVariableModal.svelte"
   import Placeholder from "assets/bb-spaceship.svg"
   import { cloneDeep } from "lodash/fp"
 
@@ -49,6 +49,8 @@
     toBindingsArray,
   } from "builderStore/dataBinding"
 
+  export let queryId
+
   let query, datasource
   let breakQs = {},
     requestBindings = {}
@@ -102,8 +104,8 @@
 
   function getSelectedQuery() {
     return cloneDeep(
-      $queries.list.find(q => q._id === $queries.selected) || {
+      $queries.list.find(q => q._id === queryId) || {
-        datasourceId: $params.selectedDatasource,
+        datasourceId: $params.datasourceId,
         parameters: [],
         fields: {
           // only init the objects, everything else is optional strings
@@ -159,6 +161,7 @@
   async function saveQuery() {
     const toSave = buildQuery()
     try {
+      const isNew = !query._rev
       const { _id } = await queries.save(toSave.datasourceId, toSave)
       saveId = _id
       query = getSelectedQuery()
@@ -174,6 +177,9 @@
         staticVariables,
         restBindings
       )
+      if (isNew) {
+        $goto(`../../${_id}`)
+      }
     } catch (err) {
       notifications.error(`Error saving query`)
     }
@@ -464,8 +470,9 @@
             on:click={saveQuery}
             tooltip={!hasSchema
               ? "Saving a query before sending will mean no schema is generated"
-              : null}>Save</Button
+              : null}
-          >
+            >Save
+          </Button>
         </div>
         <Tabs selected="Bindings" quiet noPadding noHorizPadding onTop>
           <Tab title="Bindings">
@@ -708,26 +715,33 @@
     margin: 0 auto;
     height: 100%;
   }
+
   .table {
     width: 960px;
   }
+
   .url-block {
     display: flex;
     gap: var(--spacing-s);
     z-index: 200;
   }
+
   .verb {
     flex: 1;
   }
+
   .url {
     flex: 4;
   }
+
   .top {
     min-height: 50%;
   }
+
   .bottom {
     padding-bottom: 50px;
   }
+
   .stats {
     display: flex;
     gap: var(--spacing-xl);
@@ -735,40 +749,49 @@
     margin-right: 0;
     align-items: center;
   }
+
   .green {
     color: #53a761;
   }
+
   .red {
     color: #ea7d82;
   }
+
   .top-bar {
     display: flex;
     justify-content: space-between;
   }
+
   .access {
     display: flex;
     gap: var(--spacing-m);
     align-items: center;
   }
+
   .placeholder-internal {
     display: flex;
     flex-direction: column;
     width: 200px;
     gap: var(--spacing-l);
   }
+
   .placeholder {
     display: flex;
     margin-top: var(--spacing-xl);
     justify-content: center;
   }
+
   .auth-container {
     width: 100%;
     display: flex;
     justify-content: space-between;
   }
+
   .auth-select {
     width: 200px;
   }
+
   .pagination {
     display: grid;
     grid-template-columns: 1fr 1fr;

packages/builder/src/constants/backend/index.js

@@ -261,3 +261,12 @@ export const BannedSearchTypes = [
   "json",
   "jsonarray",
 ]
+
+export const DatasourceTypes = {
+  RELATIONAL: "Relational",
+  NON_RELATIONAL: "Non-relational",
+  SPREADSHEET: "Spreadsheet",
+  OBJECT_STORE: "Object store",
+  GRAPH: "Graph",
+  API: "API",
+}

packages/builder/src/helpers/urlStateSync.js

@@ -6,6 +6,7 @@ export const syncURLToState = options => {
     urlParam,
     stateKey,
     validate,
+    update,
     baseUrl = "..",
     fallbackUrl,
     store,
@@ -38,7 +39,7 @@ export const syncURLToState = options => {
   let cachedPage = get(routify.page)
   let previousParamsHash = null
   let debug = false
-  const log = (...params) => debug && console.log(...params)
+  const log = (...params) => debug && console.log(`[${urlParam}]`, ...params)
 
   // Navigate to a certain URL
   const gotoUrl = (url, params) => {
@@ -85,10 +86,16 @@ export const syncURLToState = options => {
     // Only update state if we have a new value
     if (urlValue !== stateValue) {
       log(`state.${stateKey} (${stateValue}) <= url.${urlParam} (${urlValue})`)
-      store.update(state => {
+      if (update) {
-        state[stateKey] = urlValue
+        // Use custom update function if provided
-        return state
+        update(urlValue)
-      })
+      } else {
+        // Otherwise manually update the store
+        store.update(state => ({
+          ...state,
+          [stateKey]: urlValue,
+        }))
+      }
     }
   }
 

packages/builder/src/pages/builder/app/[application]/data/_layout.svelte

@@ -1,75 +1,46 @@
 <script>
-  import { redirect } from "@roxi/routify"
+  import { Button, Layout } from "@budibase/bbui"
-  import { Button, Tabs, Tab, Layout } from "@budibase/bbui"
   import DatasourceNavigator from "components/backend/DatasourceNavigator/DatasourceNavigator.svelte"
   import CreateDatasourceModal from "components/backend/DatasourceNavigator/modals/CreateDatasourceModal.svelte"
-
+  import Panel from "components/design/Panel.svelte"
-  let selected = "Sources"
 
   let modal
-
-  function selectFirstDatasource() {
-    $redirect("./table")
-  }
 </script>
 
 <!-- routify:options index=1 -->
-<div class="root">
+<div class="data">
-  <div class="nav">
+  <Panel title="Sources" borderRight>
-    <Tabs {selected} on:select={selectFirstDatasource}>
+    <Layout paddingX="L" paddingY="XL" gap="S">
-      <Tab title="Sources">
+      <Button dataCy={`new-datasource`} cta on:click={modal.show}>
-        <Layout paddingX="L" paddingY="L" gap="S">
+        Add source
-          <Button dataCy={`new-datasource`} cta wide on:click={modal.show}
+      </Button>
-            >Add source</Button
+      <CreateDatasourceModal bind:modal />
-          >
+      <DatasourceNavigator />
-        </Layout>
+    </Layout>
-        <CreateDatasourceModal bind:modal />
+  </Panel>
-        <DatasourceNavigator />
+
-      </Tab>
-    </Tabs>
-  </div>
   <div class="content">
     <slot />
   </div>
 </div>
 
 <style>
-  .root {
+  .data {
     flex: 1 1 auto;
+    display: flex;
+    flex-direction: row;
+    justify-content: flex-start;
+    align-items: stretch;
     height: 0;
-    display: grid;
-    grid-template-columns: 260px minmax(0, 1fr);
   }
-
   .content {
-    flex: 1 1 auto;
+    padding: 28px 40px 40px 40px;
-    padding: var(--spacing-l) 40px 40px 40px;
     overflow-y: auto;
-    display: flex;
-    flex-direction: column;
-    justify-content: flex-start;
-    align-items: stretch;
     gap: var(--spacing-l);
-  }
-  .content :global(> span) {
-    display: contents;
-  }
-
-  .nav {
-    overflow-y: auto;
-    background: var(--background);
     display: flex;
     flex-direction: column;
     justify-content: flex-start;
     align-items: stretch;
-    position: relative;
+    flex: 1 1 auto;
-    border-right: var(--border-light);
-    padding-bottom: 60px;
-  }
-
-  .add-button {
-    position: absolute;
-    top: var(--spacing-l);
-    right: var(--spacing-xl);
   }
 </style>

packages/builder/src/pages/builder/app/[application]/data/datasource/[datasourceId]/_layout.svelte

@@ -0,0 +1,23 @@
+<script>
+  import { params } from "@roxi/routify"
+  import { datasources } from "stores/backend"
+  import { syncURLToState } from "helpers/urlStateSync"
+  import * as routify from "@roxi/routify"
+  import { onDestroy } from "svelte"
+
+  const stopSyncing = syncURLToState({
+    urlParam: "datasourceId",
+    stateKey: "selectedDatasourceId",
+    validate: id => $datasources.list?.some(ds => ds._id === id),
+    update: datasources.select,
+    fallbackUrl: "../",
+    store: datasources,
+    routify,
+  })
+
+  onDestroy(stopSyncing)
+</script>
+
+{#key $params.datasourceId}
+  <slot />
+{/key}

packages/builder/src/pages/builder/app/[application]/data/datasource/[datasourceId]/index.svelte

@@ -21,34 +21,31 @@
   import { cloneDeep } from "lodash/fp"
   import ImportRestQueriesModal from "components/backend/DatasourceNavigator/modals/ImportRestQueriesModal.svelte"
 
-  let importQueriesModal
-
-  let changed,
-    isValid = true
-  let integration, baseDatasource, datasource
-  let queryList
   const querySchema = {
     name: {},
     queryVerb: { displayName: "Method" },
   }
 
-  $: baseDatasource = $datasources.list.find(
+  let importQueriesModal
-    ds => ds._id === $datasources.selected
+  let changed = false
-  )
+  let isValid = true
+  let integration, baseDatasource, datasource
+  let queryList
 
+  $: baseDatasource = $datasources.selected
   $: queryList = $queries.list.filter(
     query => query.datasourceId === datasource?._id
   )
   $: hasChanged(baseDatasource, datasource)
   $: updateDatasource(baseDatasource)
 
-  function hasChanged(base, ds) {
+  const hasChanged = (base, ds) => {
     if (base && ds) {
       changed = !isEqual(base, ds)
     }
   }
 
-  async function saveDatasource() {
+  const saveDatasource = async () => {
     try {
       // Create datasource
       await datasources.save(datasource)
@@ -63,12 +60,7 @@
     }
   }
 
-  function onClickQuery(query) {
+  const updateDatasource = base => {
-    queries.select(query)
-    $goto(`./${query._id}`)
-  }
-
-  function updateDatasource(base) {
     if (base) {
       datasource = cloneDeep(base)
       integration = $integrations[datasource.source]
@@ -87,7 +79,7 @@
 
 {#if datasource && integration}
   <section>
-    <Layout>
+    <Layout noPadding>
       <Layout gap="XS" noPadding>
         <header>
           <svelte:component
@@ -95,16 +87,16 @@
             height="26"
             width="26"
           />
-          <Heading size="M">{datasource.name}</Heading>
+          <Heading size="M">{$datasources.selected?.name}</Heading>
         </header>
         <Body size="M">{integration.description}</Body>
       </Layout>
       <Divider />
       <div class="config-header">
         <Heading size="S">Configuration</Heading>
-        <Button disabled={!changed || !isValid} cta on:click={saveDatasource}
+        <Button disabled={!changed || !isValid} cta on:click={saveDatasource}>
-          >Save</Button
+          Save
-        >
+        </Button>
       </div>
       <IntegrationConfigForm
         on:change={hasChanged}
@@ -120,12 +112,16 @@
         <Heading size="S">Queries</Heading>
         <div class="query-buttons">
           {#if datasource?.source === IntegrationTypes.REST}
-            <Button secondary on:click={() => importQueriesModal.show()}
+            <Button secondary on:click={() => importQueriesModal.show()}>
-              >Import</Button
+              Import
-            >
+            </Button>
           {/if}
-          <Button cta icon="Add" on:click={() => $goto("./new")}
+          <Button
-            >Add query
+            cta
+            icon="Add"
+            on:click={() => $goto(`../../query/new/${datasource._id}`)}
+          >
+            Add query
           </Button>
         </div>
       </div>
@@ -137,7 +133,7 @@
       {#if queryList && queryList.length > 0}
         <div class="query-list">
           <Table
-            on:click={({ detail }) => onClickQuery(detail)}
+            on:click={({ detail }) => $goto(`../../query/${detail._id}`)}
             schema={querySchema}
             data={queryList}
             allowEditColumns={false}

packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/[query]/_layout.svelte

@@ -1,23 +0,0 @@
-<script>
-  import { params } from "@roxi/routify"
-  import { queries, datasources } from "stores/backend"
-  import { IntegrationTypes } from "constants/backend"
-  import { redirect } from "@roxi/routify"
-
-  let datasourceId
-  if ($params.query) {
-    const query = $queries.list.find(q => q._id === $params.query)
-    if (query) {
-      queries.select(query)
-      datasourceId = query.datasourceId
-    }
-  }
-  const datasource = $datasources.list.find(
-    ds => ds._id === $datasources.selected || ds._id === datasourceId
-  )
-  if (datasource?.source === IntegrationTypes.REST) {
-    $redirect(`../rest/${$params.query}`)
-  }
-</script>
-
-<slot />

packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/[query]/index.svelte

@@ -1,39 +0,0 @@
-<script>
-  import { params, redirect } from "@roxi/routify"
-  import { database, datasources, queries } from "stores/backend"
-  import QueryInterface from "components/integration/QueryViewer.svelte"
-  import { IntegrationTypes } from "constants/backend"
-
-  let selectedQuery, datasource
-  $: selectedQuery = $queries.list.find(
-    query => query._id === $queries.selected
-  ) || {
-    datasourceId: $params.selectedDatasource,
-    parameters: [],
-    fields: {},
-    queryVerb: "read",
-  }
-  $: datasource = $datasources.list.find(
-    ds => ds._id === $params.selectedDatasource
-  )
-  $: {
-    if (datasource?.source === IntegrationTypes.REST) {
-      $redirect(`../rest/${$params.query}`)
-    }
-  }
-</script>
-
-<section>
-  <div class="inner">
-    {#if $database._id && selectedQuery}
-      <QueryInterface query={selectedQuery} />
-    {/if}
-  </div>
-</section>
-
-<style>
-  .inner {
-    width: 640px;
-    margin: 0 auto;
-  }
-</style>

packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/_layout.svelte

@@ -1,17 +0,0 @@
-<script>
-  import { params } from "@roxi/routify"
-  import { datasources } from "stores/backend"
-
-  if ($params.selectedDatasource && !$params.query) {
-    const datasource = $datasources.list.find(
-      m => m._id === $params.selectedDatasource
-    )
-    if (datasource) {
-      datasources.select(datasource._id)
-    }
-  }
-</script>
-
-{#key $params.selectedDatasource}
-  <slot />
-{/key}

packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/rest/[query]/_layout.svelte

@@ -1,13 +0,0 @@
-<script>
-  import { params } from "@roxi/routify"
-  import { queries } from "stores/backend"
-
-  if ($params.query) {
-    const query = $queries.list.find(q => q._id === $params.query)
-    if (query) {
-      queries.select(query)
-    }
-  }
-</script>
-
-<slot />

packages/builder/src/pages/builder/app/[application]/data/datasource/bb_internal/_layout.svelte

@@ -1,7 +0,0 @@
-<script>
-  import { datasources } from "stores/backend"
-
-  datasources.select("bb_internal")
-</script>
-
-<slot />

packages/builder/src/pages/builder/app/[application]/data/datasource/bb_internal/index.svelte

@@ -4,13 +4,19 @@
   import ICONS from "components/backend/DatasourceNavigator/icons"
   import { tables, datasources } from "stores/backend"
   import { goto } from "@roxi/routify"
+  import { onMount } from "svelte"
+  import { BUDIBASE_INTERNAL_DB_ID } from "constants/backend"
 
   let modal
 
   $: internalTablesBySourceId = $tables.list.filter(
     table =>
-      table.type !== "external" && $datasources.selected === table.sourceId
+      table.type !== "external" && table.sourceId === BUDIBASE_INTERNAL_DB_ID
   )
+
+  onMount(() => {
+    datasources.select(BUDIBASE_INTERNAL_DB_ID)
+  })
 </script>
 
 <Modal bind:this={modal}>
@@ -73,7 +79,7 @@
     background: var(--background);
     border: var(--border-dark);
     display: grid;
-    grid-template-columns: 2fr 0.75fr 20px;
+    grid-template-columns: 1fr auto;
     align-items: center;
     padding: var(--spacing-m);
     gap: var(--layout-xs);

packages/builder/src/pages/builder/app/[application]/data/datasource/datasource_internal_bb_default/_layout.svelte

@@ -1,8 +0,0 @@
-<script>
-  import { datasources } from "stores/backend"
-  import { DEFAULT_BB_DATASOURCE_ID } from "constants/backend"
-
-  datasources.select(DEFAULT_BB_DATASOURCE_ID)
-</script>
-
-<slot />

packages/builder/src/pages/builder/app/[application]/data/datasource/datasource_internal_bb_default/index.svelte

@@ -4,12 +4,18 @@
   import ICONS from "components/backend/DatasourceNavigator/icons"
   import { tables, datasources } from "stores/backend"
   import { goto } from "@roxi/routify"
+  import { DEFAULT_BB_DATASOURCE_ID } from "constants/backend"
+  import { onMount } from "svelte"
 
   let modal
   $: internalTablesBySourceId = $tables.list.filter(
     table =>
-      table.type !== "external" && $datasources.selected === table.sourceId
+      table.type !== "external" && table.sourceId === DEFAULT_BB_DATASOURCE_ID
   )
+
+  onMount(() => {
+    datasources.select(DEFAULT_BB_DATASOURCE_ID)
+  })
 </script>
 
 <Modal bind:this={modal}>
@@ -23,10 +29,11 @@
         <svelte:component this={ICONS.BUDIBASE} height="26" width="26" />
         <Heading size="M">Sample Data</Heading>
       </header>
-      <Body size="M">A little something to get you up and running!</Body>
+      <Body size="M">
-      <Body size="M"
+        A little something to get you up and running!
-        >If you have no need for this datasource, feel free to delete it.</Body
+        <br />
-      >
+        If you have no need for this datasource, feel free to delete it.
+      </Body>
     </Layout>
     <Divider />
     <Heading size="S">Tables</Heading>
@@ -73,7 +80,7 @@
     background: var(--background);
     border: var(--border-dark);
     display: grid;
-    grid-template-columns: 2fr 0.75fr 20px;
+    grid-template-columns: 1fr auto;
     align-items: center;
     padding: var(--spacing-m);
     gap: var(--layout-xs);

packages/builder/src/pages/builder/app/[application]/data/datasource/index.svelte

@@ -4,12 +4,16 @@
   import { onMount } from "svelte"
 
   onMount(async () => {
-    // navigate to first table in list, if not already selected
+    const { list, selected } = $datasources
-    $datasources.list.length > 0 && $redirect(`./${$datasources.list[0]._id}`)
+    if (selected) {
+      $redirect(`./${selected?._id}`)
+    } else {
+      $redirect(`./${list[0]._id}`)
+    }
   })
 </script>
 
-{#if $datasources.list.length === 0}
+{#if !$datasources.list?.length}
   <i>Connect your first datasource to start building.</i>
 {:else}<i>Select a datasource to edit</i>{/if}
 

packages/builder/src/pages/builder/app/[application]/data/query/[queryId]/_layout.svelte

@@ -0,0 +1,22 @@
+<script>
+  import { queries } from "stores/backend"
+  import { syncURLToState } from "helpers/urlStateSync"
+  import * as routify from "@roxi/routify"
+  import { onDestroy } from "svelte"
+
+  const stopSyncing = syncURLToState({
+    urlParam: "queryId",
+    stateKey: "selectedQueryId",
+    validate: id => id === "new" || $queries.list?.some(q => q._id === id),
+    update: queries.select,
+    fallbackUrl: "../",
+    store: queries,
+    routify,
+  })
+
+  onDestroy(stopSyncing)
+</script>
+
+{#key $queries.selectedQueryId}
+  <slot />
+{/key}

packages/builder/src/pages/builder/app/[application]/data/query/[queryId]/index.svelte

@@ -0,0 +1,18 @@
+<script>
+  import { database, queries, datasources } from "stores/backend"
+  import QueryViewer from "components/integration/QueryViewer.svelte"
+  import RestQueryViewer from "components/integration/RestQueryViewer.svelte"
+  import { IntegrationTypes } from "constants/backend"
+
+  $: query = $queries.selected
+  $: datasource = $datasources.list.find(ds => ds._id === query?.datasourceId)
+  $: isRestQuery = datasource?.source === IntegrationTypes.REST
+</script>
+
+{#if $database._id && query}
+  {#if isRestQuery}
+    <RestQueryViewer queryId={$queries.selectedQueryId} />
+  {:else}
+    <QueryViewer {query} />
+  {/if}
+{/if}

packages/builder/src/pages/builder/app/[application]/data/query/index.svelte

@@ -0,0 +1,16 @@
+<script>
+  import { onMount } from "svelte"
+  import { queries } from "stores/backend"
+  import { redirect } from "@roxi/routify"
+
+  onMount(async () => {
+    const { list, selected } = $queries
+    if (selected) {
+      $redirect(`./${selected?._id}`)
+    } else if (list?.length) {
+      $redirect(`./${list[0]._id}`)
+    } else {
+      $redirect("../")
+    }
+  })
+</script>

packages/builder/src/pages/builder/app/[application]/data/query/new/[datasourceId]/index.svelte

@@ -0,0 +1,38 @@
+<script>
+  import { params, redirect } from "@roxi/routify"
+  import { database, datasources } from "stores/backend"
+  import QueryViewer from "components/integration/QueryViewer.svelte"
+  import RestQueryViewer from "components/integration/RestQueryViewer.svelte"
+  import { IntegrationTypes } from "constants/backend"
+
+  $: datasource = $datasources.list.find(ds => ds._id === $params.datasourceId)
+  $: {
+    if (!datasource) {
+      $redirect("../../../")
+    }
+  }
+  $: isRestQuery = datasource?.source === IntegrationTypes.REST
+  $: query = buildNewQuery(isRestQuery)
+
+  const buildNewQuery = isRestQuery => {
+    let query = {
+      datasourceId: $params.datasourceId,
+      parameters: [],
+      fields: {},
+      queryVerb: "read",
+    }
+    if (isRestQuery) {
+      query.flags = {}
+      query.fields = { disabledHeaders: {}, headers: {} }
+    }
+    return query
+  }
+</script>
+
+{#if $database._id && datasource && query}
+  {#if isRestQuery}
+    <RestQueryViewer />
+  {:else}
+    <QueryViewer {query} />
+  {/if}
+{/if}

packages/builder/src/pages/builder/app/[application]/data/query/new/index.svelte

@@ -0,0 +1,5 @@
+<script>
+  import { redirect } from "@roxi/routify"
+
+  $redirect("../")
+</script>

packages/builder/src/pages/builder/app/[application]/data/table/[selectedTable]/_layout.svelte

@@ -1,13 +0,0 @@
-<script>
-  import { params } from "@roxi/routify"
-  import { tables } from "stores/backend"
-
-  if ($params.selectedTable) {
-    const table = $tables.list.find(m => m._id === $params.selectedTable)
-    if (table) {
-      tables.select(table)
-    }
-  }
-</script>
-
-<slot />

packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_layout.svelte

@@ -0,0 +1,20 @@
+<script>
+  import { syncURLToState } from "helpers/urlStateSync"
+  import { tables } from "stores/backend"
+  import * as routify from "@roxi/routify"
+  import { onDestroy } from "svelte"
+
+  const stopSyncing = syncURLToState({
+    urlParam: "tableId",
+    stateKey: "selectedTableId",
+    validate: id => $tables.list?.some(table => table._id === id),
+    update: tables.select,
+    fallbackUrl: "../",
+    store: tables,
+    routify,
+  })
+
+  onDestroy(stopSyncing)
+</script>
+
+<slot />

packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/index.svelte

@@ -3,9 +3,11 @@
   import { tables, database } from "stores/backend"
 </script>
 
-{#if $database?._id && $tables?.selected?.name}
+{#if $database?._id && $tables?.selected}
   <TableDataTable />
-{:else}<i>Create your first table to start building</i>{/if}
+{:else}
+  <i>Create your first table to start building</i>
+{/if}
 
 <style>
   i {

packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/relationship/[rowId]/[field]/index.svelte

@@ -4,7 +4,7 @@
 </script>
 
 <RelationshipDataTable
-  tableId={$params.selectedTable}
+  tableId={$params.tableId}
-  rowId={$params.selectedRow}
+  rowId={$params.rowId}
-  fieldName={decodeURI($params.selectedField)}
+  fieldName={decodeURI($params.field)}
 />