Merge branch 'develop' into cypress-testing

hosting/kubernetes/budibase/templates/app-service-deployment.yaml

@@ -96,6 +96,10 @@ spec:
           value: worker-service:{{ .Values.services.worker.port }}
         - name: COOKIE_DOMAIN
           value: {{ .Values.globals.cookieDomain | quote }}
+        - name: ACCOUNT_PORTAL_URL
+          value: {{ .Values.globals.accountPortalUrl | quote }}
+        - name: ACCOUNT_PORTAL_API_KEY
+          value: {{ .Values.globals.accountPortalApiKey | quote }}
         image: budibase/apps
         imagePullPolicy: Always
         name: bbapps

hosting/kubernetes/budibase/templates/worker-service-deployment.yaml

@@ -89,6 +89,8 @@ spec:
           value: {{ .Values.globals.selfHosted | quote }}
         - name: ACCOUNT_PORTAL_URL
           value: {{ .Values.globals.accountPortalUrl | quote }}
+        - name: ACCOUNT_PORTAL_API_KEY
+          value: {{ .Values.globals.accountPortalApiKey | quote }}
         - name: COOKIE_DOMAIN
           value: {{ .Values.globals.cookieDomain | quote }}
         image: budibase/worker

hosting/kubernetes/budibase/values.yaml

@@ -90,6 +90,7 @@ globals:
   logLevel: info
   selfHosted: 1
   accountPortalUrL: ""
+  accountPortalApiKey: ""
   cookieDomain: ""
   createSecrets: true # creates an internal API key, JWT secrets and redis password for you
 

hosting/kubernetes/envoy/envoy.yaml

@@ -50,6 +50,11 @@ static_resources:
                 route:
                   cluster: app-service
 
+              - match: { path: "/api/deploy" }
+                route:
+                  timeout: 60s
+                  cluster: app-service
+
               # special case for when API requests are made, can just forward, not to minio
               - match: { prefix: "/api/" }
                 route:

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "0.9.146-alpha.4",
+  "version": "0.9.154-alpha.1",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

package.json

@@ -50,6 +50,8 @@
     "multi:disable": "lerna run multi:disable",
     "selfhost:enable": "lerna run selfhost:enable",
     "selfhost:disable": "lerna run selfhost:disable",
+    "localdomain:enable": "lerna run localdomain:enable",
+    "localdomain:disable": "lerna run localdomain:disable",
     "postinstall": "husky install"
   }
 }

packages/auth/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/auth",
-  "version": "0.9.146-alpha.4",
+  "version": "0.9.154-alpha.1",
   "description": "Authentication middlewares for budibase builder and apps",
   "main": "src/index.js",
   "author": "Budibase",

packages/auth/src/cache/user.js

@@ -12,7 +12,7 @@ const populateFromDB = async (userId, tenantId) => {
   const user = await getGlobalDB(tenantId).get(userId)
   user.budibaseAccess = true
 
-  if (!env.SELF_HOSTED) {
+  if (!env.SELF_HOSTED && !env.DISABLE_ACCOUNT_PORTAL) {
     const account = await accounts.getAccount(user.email)
     if (account) {
       user.account = account

packages/auth/src/cloud/accounts.js

@@ -1,16 +1,18 @@
 const API = require("./api")
 const env = require("../environment")
+const { Headers } = require("../constants")
 
 const api = new API(env.ACCOUNT_PORTAL_URL)
 
-// TODO: Authorization
-
 exports.getAccount = async email => {
   const payload = {
     email,
   }
   const response = await api.post(`/api/accounts/search`, {
     body: payload,
+    headers: {
+      [Headers.API_KEY]: env.ACCOUNT_PORTAL_API_KEY,
+    },
   })
   const json = await response.json()
 

packages/auth/src/environment.js

@@ -21,6 +21,8 @@ module.exports = {
   INTERNAL_API_KEY: process.env.INTERNAL_API_KEY,
   MULTI_TENANCY: process.env.MULTI_TENANCY,
   ACCOUNT_PORTAL_URL: process.env.ACCOUNT_PORTAL_URL,
+  ACCOUNT_PORTAL_API_KEY: process.env.ACCOUNT_PORTAL_API_KEY,
+  DISABLE_ACCOUNT_PORTAL: process.env.DISABLE_ACCOUNT_PORTAL,
   SELF_HOSTED: !!parseInt(process.env.SELF_HOSTED),
   COOKIE_DOMAIN: process.env.COOKIE_DOMAIN,
   isTest,

packages/auth/src/middleware/matchers.js

@@ -7,6 +7,7 @@ exports.buildMatcherRegex = patterns => {
   return patterns.map(pattern => {
     const isObj = typeof pattern === "object" && pattern.route
     const method = isObj ? pattern.method : "GET"
+    const strict = pattern.strict ? pattern.strict : false
     let route = isObj ? pattern.route : pattern
 
     const matches = route.match(PARAM_REGEX)
@@ -16,13 +17,19 @@ exports.buildMatcherRegex = patterns => {
         route = route.replace(match, pattern)
       }
     }
-    return { regex: new RegExp(route), method }
+    return { regex: new RegExp(route), method, strict, route }
   })
 }
 
 exports.matches = (ctx, options) => {
-  return options.find(({ regex, method }) => {
+  return options.find(({ regex, method, strict, route }) => {
-    const urlMatch = regex.test(ctx.request.url)
+    let urlMatch
+    if (strict) {
+      urlMatch = ctx.request.url === route
+    } else {
+      urlMatch = regex.test(ctx.request.url)
+    }
+
     const methodMatch =
       method === "ALL"
         ? true

packages/auth/src/middleware/passport/tests/third-party-common.spec.js

@@ -20,6 +20,10 @@ const getErrorMessage = () => {
   return done.mock.calls[0][2].message
 }
 
+const saveUser = async (user) => {
+  return await db.put(user)
+}
+
 describe("third party common", () => {
   describe("authenticateThirdParty", () => { 
     let thirdPartyUser
@@ -36,7 +40,7 @@ describe("third party common", () => {
   
     describe("validation", () => {
       const testValidation = async (message) => {
-        await authenticateThirdParty(thirdPartyUser, false, done)
+        await authenticateThirdParty(thirdPartyUser, false, done, saveUser)
         expect(done.mock.calls.length).toBe(1)
         expect(getErrorMessage()).toContain(message)
       }
@@ -78,7 +82,7 @@ describe("third party common", () => {
     describe("when the user doesn't exist", () => {
       describe("when a local account is required", () => {
         it("returns an error message", async () => {
-          await authenticateThirdParty(thirdPartyUser, true, done)
+          await authenticateThirdParty(thirdPartyUser, true, done, saveUser)
           expect(done.mock.calls.length).toBe(1)
           expect(getErrorMessage()).toContain("Email does not yet exist. You must set up your local budibase account first.")
         })
@@ -86,7 +90,7 @@ describe("third party common", () => {
       
       describe("when a local account isn't required", () => {
         it("creates and authenticates the user", async () => {
-          await authenticateThirdParty(thirdPartyUser, false, done)
+          await authenticateThirdParty(thirdPartyUser, false, done, saveUser)
           const user = expectUserIsAuthenticated()
           expectUserIsSynced(user, thirdPartyUser)
           expect(user.roles).toStrictEqual({})
@@ -123,7 +127,7 @@ describe("third party common", () => {
         })
 
         it("syncs and authenticates the user", async () => {
-          await authenticateThirdParty(thirdPartyUser, true, done)
+          await authenticateThirdParty(thirdPartyUser, true, done, saveUser)
         
           const user = expectUserIsAuthenticated()
           expectUserIsSynced(user, thirdPartyUser)
@@ -139,7 +143,7 @@ describe("third party common", () => {
         })
 
         it("syncs and authenticates the user", async () => {
-          await authenticateThirdParty(thirdPartyUser, true, done)
+          await authenticateThirdParty(thirdPartyUser, true, done, saveUser)
         
           const user = expectUserIsAuthenticated()
           expectUserIsSynced(user, thirdPartyUser)

packages/auth/src/middleware/passport/third-party-common.js

@@ -1,6 +1,7 @@
 const env = require("../../environment")
 const jwt = require("jsonwebtoken")
 const { generateGlobalUserID } = require("../../db/utils")
+const { saveUser } = require("../../utils")
 const { authError } = require("./utils")
 const { newid } = require("../../hashing")
 const { createASession } = require("../../security/sessions")
@@ -14,7 +15,8 @@ const fetch = require("node-fetch")
 exports.authenticateThirdParty = async function (
   thirdPartyUser,
   requireLocalAccount = true,
-  done
+  done,
+  saveUserFn = saveUser
 ) {
   if (!thirdPartyUser.provider) {
     return authError(done, "third party user provider required")
@@ -71,7 +73,13 @@ exports.authenticateThirdParty = async function (
   dbUser = await syncUser(dbUser, thirdPartyUser)
 
   // create or sync the user
-  const response = await db.put(dbUser)
+  let response
+  try {
+    response = await saveUserFn(dbUser, getTenantId(), false, false)
+  } catch (err) {
+    return authError(done, err)
+  }
+
   dbUser._rev = response.rev
 
   // authenticate

packages/auth/src/objectStore/index.js

@@ -265,7 +265,7 @@ exports.downloadTarball = async (url, bucketName, path) => {
 
   const tmpPath = join(budibaseTempDir(), path)
   await streamPipeline(response.body, zlib.Unzip(), tar.extract(tmpPath))
-  if (!env.isTest()) {
+  if (!env.isTest() && env.SELF_HOSTED) {
     await exports.uploadDirectory(bucketName, tmpPath, path)
   }
   // return the temporary path incase there is a use for it

packages/auth/src/redis/index.js

@@ -191,6 +191,12 @@ class RedisWrapper {
     }
   }
 
+  async getTTL(key) {
+    const db = this._db
+    const prefixedKey = addDbPrefix(db, key)
+    return CLIENT.ttl(prefixedKey)
+  }
+
   async setExpiry(key, expirySeconds) {
     const db = this._db
     const prefixedKey = addDbPrefix(db, key)

packages/auth/src/tenancy/deprovision.js

@@ -19,6 +19,22 @@ const removeTenantFromInfoDB = async tenantId => {
   }
 }
 
+exports.removeUserFromInfoDB = async dbUser => {
+  const infoDb = getDB(PLATFORM_INFO_DB)
+  const keys = [dbUser._id, dbUser.email]
+  const userDocs = await infoDb.allDocs({
+    keys,
+    include_docs: true,
+  })
+  const toDelete = userDocs.rows.map(row => {
+    return {
+      ...row.doc,
+      _deleted: true,
+    }
+  })
+  await infoDb.bulkDocs(toDelete)
+}
+
 const removeUsersFromInfoDB = async tenantId => {
   try {
     const globalDb = getGlobalDB(tenantId)

packages/auth/src/tenancy/tenancy.js

@@ -73,7 +73,7 @@ exports.tryAddTenant = async (tenantId, userId, email) => {
   await Promise.all(promises)
 }
 
-exports.getGlobalDB = (tenantId = null) => {
+exports.getGlobalDBName = (tenantId = null) => {
   // tenant ID can be set externally, for example user API where
   // new tenants are being created, this may be the case
   if (!tenantId) {
@@ -81,13 +81,16 @@ exports.getGlobalDB = (tenantId = null) => {
   }
 
   let dbName
-
   if (tenantId === DEFAULT_TENANT_ID) {
     dbName = StaticDatabases.GLOBAL.name
   } else {
     dbName = `${tenantId}${SEPARATOR}${StaticDatabases.GLOBAL.name}`
   }
+  return dbName
+}
 
+exports.getGlobalDB = (tenantId = null) => {
+  const dbName = exports.getGlobalDBName(tenantId)
   return getDB(dbName)
 }
 
@@ -104,3 +107,13 @@ exports.lookupTenantId = async userId => {
   }
   return tenantId
 }
+
+// lookup, could be email or userId, either will return a doc
+exports.getTenantUser = async identifier => {
+  const db = getDB(PLATFORM_INFO_DB)
+  try {
+    return await db.get(identifier)
+  } catch (err) {
+    return null
+  }
+}

packages/auth/src/utils.js

@@ -1,10 +1,24 @@
-const { DocumentTypes, SEPARATOR, ViewNames } = require("./db/utils")
+const {
+  DocumentTypes,
+  SEPARATOR,
+  ViewNames,
+  generateGlobalUserID,
+} = require("./db/utils")
 const jwt = require("jsonwebtoken")
 const { options } = require("./middleware/passport/jwt")
 const { createUserEmailView } = require("./db/views")
-const { Headers } = require("./constants")
+const { Headers, UserStatus } = require("./constants")
-const { getGlobalDB } = require("./tenancy")
+const {
+  getGlobalDB,
+  updateTenantId,
+  getTenantUser,
+  tryAddTenant,
+} = require("./tenancy")
 const environment = require("./environment")
+const accounts = require("./cloud/accounts")
+const { hash } = require("./hashing")
+const userCache = require("./cache/user")
+const env = require("./environment")
 
 const APP_PREFIX = DocumentTypes.APP + SEPARATOR
 
@@ -131,3 +145,93 @@ exports.getGlobalUserByEmail = async email => {
     }
   }
 }
+
+exports.saveUser = async (
+  user,
+  tenantId,
+  hashPassword = true,
+  requirePassword = true
+) => {
+  if (!tenantId) {
+    throw "No tenancy specified."
+  }
+  // need to set the context for this request, as specified
+  updateTenantId(tenantId)
+  // specify the tenancy incase we're making a new admin user (public)
+  const db = getGlobalDB(tenantId)
+  let { email, password, _id } = user
+  // make sure another user isn't using the same email
+  let dbUser
+  if (email) {
+    // check budibase users inside the tenant
+    dbUser = await exports.getGlobalUserByEmail(email)
+    if (dbUser != null && (dbUser._id !== _id || Array.isArray(dbUser))) {
+      throw `Email address ${email} already in use.`
+    }
+
+    // check budibase users in other tenants
+    if (env.MULTI_TENANCY) {
+      dbUser = await getTenantUser(email)
+      if (dbUser != null && dbUser.tenantId !== tenantId) {
+        throw `Email address ${email} already in use.`
+      }
+    }
+
+    // check root account users in account portal
+    if (!env.SELF_HOSTED && !env.DISABLE_ACCOUNT_PORTAL) {
+      const account = await accounts.getAccount(email)
+      if (account && account.verified && account.tenantId !== tenantId) {
+        throw `Email address ${email} already in use.`
+      }
+    }
+  } else {
+    dbUser = await db.get(_id)
+  }
+
+  // get the password, make sure one is defined
+  let hashedPassword
+  if (password) {
+    hashedPassword = hashPassword ? await hash(password) : password
+  } else if (dbUser) {
+    hashedPassword = dbUser.password
+  } else if (requirePassword) {
+    throw "Password must be specified."
+  }
+
+  _id = _id || generateGlobalUserID()
+  user = {
+    createdAt: Date.now(),
+    ...dbUser,
+    ...user,
+    _id,
+    password: hashedPassword,
+    tenantId,
+  }
+  // make sure the roles object is always present
+  if (!user.roles) {
+    user.roles = {}
+  }
+  // add the active status to a user if its not provided
+  if (user.status == null) {
+    user.status = UserStatus.ACTIVE
+  }
+  try {
+    const response = await db.put({
+      password: hashedPassword,
+      ...user,
+    })
+    await tryAddTenant(tenantId, _id, email)
+    await userCache.invalidateUser(response.id)
+    return {
+      _id: response.id,
+      _rev: response.rev,
+      email,
+    }
+  } catch (err) {
+    if (err.status === 409) {
+      throw "User exists already"
+    } else {
+      throw err
+    }
+  }
+}

packages/bbui/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/bbui",
   "description": "A UI solution used in the different Budibase projects.",
-  "version": "0.9.146-alpha.4",
+  "version": "0.9.154-alpha.1",
   "license": "AGPL-3.0",
   "svelte": "src/index.js",
   "module": "dist/bbui.es.js",
@@ -78,7 +78,7 @@
     "@spectrum-css/underlay": "^2.0.9",
     "@spectrum-css/vars": "^3.0.1",
     "dayjs": "^1.10.4",
-    "svelte-flatpickr": "^3.1.0",
+    "svelte-flatpickr": "^3.2.3",
     "svelte-portal": "^1.0.0"
   },
   "gitHead": "d1836a898cab3f8ab80ee6d8f42be1a9eed7dcdc"

packages/bbui/src/Form/Core/DatePicker.svelte

@@ -26,6 +26,7 @@
     altFormat: enableTime ? "F j Y, H:i" : "F j, Y",
     wrap: true,
     appendTo,
+    disableMobile: "true",
   }
 
   const handleChange = event => {

packages/bbui/src/Form/Core/TextField.svelte

@@ -90,6 +90,7 @@
     on:input={onInput}
     on:keyup={updateValueOnEnter}
     {type}
+    inputmode={type === "number" ? "decimal" : "text"}
     class="spectrum-Textfield-input"
   />
 </div>

packages/bbui/yarn.lock

@@ -2415,10 +2415,10 @@ supports-color@^7.0.0, supports-color@^7.1.0:
   dependencies:
     has-flag "^4.0.0"
 
-svelte-flatpickr@^3.1.0:
+svelte-flatpickr@^3.2.3:
-  version "3.1.0"
+  version "3.2.3"
-  resolved "https://registry.yarnpkg.com/svelte-flatpickr/-/svelte-flatpickr-3.1.0.tgz#ad83588430dbd55196a1a258b8ba27e7f9c1ee37"
+  resolved "https://registry.yarnpkg.com/svelte-flatpickr/-/svelte-flatpickr-3.2.3.tgz#db5dd7ad832ef83262b45e09737955ad3d591fc8"
-  integrity sha512-zKyV+ukeVuJ8CW0Ing3T19VSekc4bPkou/5Riutt1yATrLvSsanNqcgqi7Q5IePvIoOF9GJ5OtHvn1qK9Wx9BQ==
+  integrity sha512-PNkqK4Napx8nTvCwkaUXdnKo8dISThaxEOK+szTUXcY6H0dQM0TSyuoMaVWY2yX7pM+PN5cpCQCcVe8YvTRFSw==
   dependencies:
     flatpickr "^4.5.2"
 

packages/builder/cypress/integration/createTable.spec.js

@@ -24,9 +24,7 @@ context("Create a Table", () => {
   it("updates a column on the table", () => {
     cy.get(".title").click()
     cy.get(".spectrum-Table-editIcon > use").click()
-    cy.get("input")
+    cy.get("input").eq(1).type("updated", { force: true })
-      .eq(1)
-      .type("updated", { force: true })
     // Unset table display column
     cy.get(".spectrum-Switch-input").eq(1).click()
     cy.contains("Save Column").click()
@@ -45,9 +43,7 @@ context("Create a Table", () => {
   it("deletes a row", () => {
     cy.get(".spectrum-Checkbox-input").check({ force: true })
     cy.contains("Delete 1 row(s)").click()
-    cy.get(".spectrum-Modal")
+    cy.get(".spectrum-Modal").contains("Delete").click()
-      .contains("Delete")
-      .click()
     cy.contains("RoverUpdated").should("not.exist")
   })
 
@@ -56,15 +52,18 @@ context("Create a Table", () => {
     cy.get(".spectrum-Table-editIcon > use").click()
     cy.contains("Delete").click()
     cy.wait(50)
-    cy.contains("Delete Column")
+    cy.contains("Delete Column").click()
-      .click()
     cy.contains("nameupdated").should("not.exist")
   })
 
   it("deletes a table", () => {
-    cy.get(".actions > :nth-child(1) > .icon > .spectrum-Icon > use")
+    cy.get(".nav-item")
-      .eq(1)
+      .contains("dog")
-      .click({ force: true })
+      .parents(".nav-item")
+      .first()
+      .within(() => {
+        cy.get(".actions .spectrum-Icon").click({ force: true })
+      })
     cy.get(".spectrum-Menu > :nth-child(2)").click()
     cy.contains("Delete Table").click()
     cy.contains("dog").should("not.exist")

packages/builder/cypress/integration/createView.spec.js

@@ -28,11 +28,7 @@ context("Create a View", () => {
       const headers = Array.from($headers).map(header =>
         header.textContent.trim()
       )
-      expect(removeSpacing(headers)).to.deep.eq([
+      expect(removeSpacing(headers)).to.deep.eq(["group", "age", "rating"])
-        "group",
-        "age",
-        "rating",
-      ])
     })
   })
 
@@ -62,7 +58,7 @@ context("Create a View", () => {
     cy.get(".modal-inner-wrapper").within(() => {
       cy.get(".spectrum-Picker-label").eq(0).click()
       cy.contains("Statistics").click()
-      
+
       cy.get(".spectrum-Picker-label").eq(1).click()
       cy.contains("age").click({ force: true })
 
@@ -105,20 +101,20 @@ context("Create a View", () => {
     cy.get(".spectrum-Table-cell").then($values => {
       let values = Array.from($values).map(header => header.textContent.trim())
       expect(values).to.deep.eq([
-            "Students",
+        "Students",
-            "70",
+        "70",
-            "20",
+        "20",
-            "25",
+        "25",
-            "3",
+        "3",
-            "1650",
+        "1650",
-            "23.333333333333332",
+        "23.333333333333332",
-            "Teachers",
+        "Teachers",
-            "85",
+        "85",
-            "36",
+        "36",
-            "49",
+        "49",
-            "2",
+        "2",
-            "3697",
+        "3697",
-            "42.5",
+        "42.5",
       ])
     })
   })

packages/builder/cypress/support/commands.js

@@ -31,8 +31,7 @@ Cypress.Commands.add("login", () => {
 Cypress.Commands.add("createApp", name => {
   cy.visit(`localhost:${Cypress.env("PORT")}/builder`)
   cy.wait(500)
-  cy.contains(/Create (new )?app/).click()
+  cy.contains(/Start from scratch/).click()
-  cy.wait(500)
   cy.get(".spectrum-Modal")
     .within(() => {
       cy.get("input").eq(0).type(name).should("have.value", name).blur()
@@ -187,7 +186,7 @@ Cypress.Commands.add("getComponent", componentId => {
     .its("body")
     .should("not.be.null")
     .then(cy.wrap)
-    .find(`[data-component-id=${componentId}]`)
+    .find(`[data-id=${componentId}]`)
 })
 
 Cypress.Commands.add("navigateToFrontend", () => {

packages/builder/index.html

@@ -4,7 +4,7 @@
 	<meta charset='utf8'>
 	<meta name='viewport' content='width=device-width'>
 	<title>Budibase</title>
-	<link rel='icon' href='/src/favicon.ico'>
+	<link rel='icon' href='/src/favicon.png'>
 	<link rel="preconnect" href="https://fonts.gstatic.com" />
 	<link
 		href="https://fonts.googleapis.com/css2?family=Source+Sans+Pro:wght@400;600;700&display=swap"

packages/builder/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/builder",
-  "version": "0.9.146-alpha.4",
+  "version": "0.9.154-alpha.1",
   "license": "AGPL-3.0",
   "private": true,
   "scripts": {
@@ -65,10 +65,10 @@
     }
   },
   "dependencies": {
-    "@budibase/bbui": "^0.9.146-alpha.4",
+    "@budibase/bbui": "^0.9.154-alpha.1",
-    "@budibase/client": "^0.9.146-alpha.4",
+    "@budibase/client": "^0.9.154-alpha.1",
     "@budibase/colorpicker": "1.1.2",
-    "@budibase/string-templates": "^0.9.146-alpha.4",
+    "@budibase/string-templates": "^0.9.154-alpha.1",
     "@sentry/browser": "5.19.1",
     "@spectrum-css/page": "^3.0.1",
     "@spectrum-css/vars": "^3.0.1",

packages/builder/src/builderStore/dataBinding.js

@@ -1,12 +1,12 @@
 import { cloneDeep } from "lodash/fp"
 import { get } from "svelte/store"
 import {
+  findAllMatchingComponents,
   findComponent,
   findComponentPath,
-  findAllMatchingComponents,
 } from "./storeUtils"
 import { store } from "builderStore"
-import { tables as tablesStore, queries as queriesStores } from "stores/backend"
+import { queries as queriesStores, tables as tablesStore } from "stores/backend"
 import { makePropSafe } from "@budibase/string-templates"
 import { TableNames } from "../constants"
 
@@ -422,6 +422,10 @@ function shouldReplaceBinding(currentValue, from, convertTo) {
   return !invalids.find(invalid => noSpaces?.includes(invalid))
 }
 
+function replaceBetween(string, start, end, replacement) {
+  return string.substring(0, start) + replacement + string.substring(end)
+}
+
 /**
  * utility function for the readableToRuntimeBinding and runtimeToReadableBinding.
  */
@@ -431,6 +435,7 @@ function bindingReplacement(bindableProperties, textWithBindings, convertTo) {
   if (typeof textWithBindings !== "string") {
     return textWithBindings
   }
+  // work from longest to shortest
   const convertFromProps = bindableProperties
     .map(el => el[convertFrom])
     .sort((a, b) => {
@@ -440,12 +445,29 @@ function bindingReplacement(bindableProperties, textWithBindings, convertTo) {
   let result = textWithBindings
   for (let boundValue of boundValues) {
     let newBoundValue = boundValue
+    // we use a search string, where any time we replace something we blank it out
+    // in the search, working from longest to shortest so always use best match first
+    let searchString = newBoundValue
     for (let from of convertFromProps) {
       if (shouldReplaceBinding(newBoundValue, from, convertTo)) {
         const binding = bindableProperties.find(el => el[convertFrom] === from)
-        while (newBoundValue.includes(from)) {
+        let idx
-          newBoundValue = newBoundValue.replace(from, binding[convertTo])
+        do {
-        }
+          // see if any instances of this binding exist in the search string
+          idx = searchString.indexOf(from)
+          if (idx !== -1) {
+            let end = idx + from.length,
+              searchReplace = Array(binding[convertTo].length).join("*")
+            // blank out parts of the search string
+            searchString = replaceBetween(searchString, idx, end, searchReplace)
+            newBoundValue = replaceBetween(
+              newBoundValue,
+              idx,
+              end,
+              binding[convertTo]
+            )
+          }
+        } while (idx !== -1)
       }
     }
     result = result.replace(boundValue, newBoundValue)

packages/builder/src/builderStore/store/automation/Automation.js

@@ -17,7 +17,7 @@ export default class Automation {
     this.automation.testData = data
   }
 
-  addBlock(block) {
+  addBlock(block, idx) {
     // Make sure to add trigger if doesn't exist
     if (!this.hasTrigger() && block.type === "TRIGGER") {
       const trigger = { id: generate(), ...block }
@@ -26,10 +26,7 @@ export default class Automation {
     }
 
     const newBlock = { id: generate(), ...block }
-    this.automation.definition.steps = [
+    this.automation.definition.steps.splice(idx, 0, newBlock)
-      ...this.automation.definition.steps,
-      newBlock,
-    ]
     return newBlock
   }
 

packages/builder/src/builderStore/store/automation/index.js

@@ -104,9 +104,12 @@ const automationActions = store => ({
       return state
     })
   },
-  addBlockToAutomation: block => {
+  addBlockToAutomation: (block, blockIdx) => {
     store.update(state => {
-      const newBlock = state.selectedAutomation.addBlock(cloneDeep(block))
+      const newBlock = state.selectedAutomation.addBlock(
+        cloneDeep(block),
+        blockIdx
+      )
       state.selectedBlock = newBlock
       return state
     })

packages/builder/src/builderStore/store/frontend.js

@@ -15,7 +15,6 @@ import {
   database,
   tables,
 } from "stores/backend"
-
 import { fetchComponentLibDefinitions } from "../loadComponentLibraries"
 import api from "../api"
 import { FrontendTypes } from "constants"
@@ -25,6 +24,7 @@ import {
   findComponentParent,
   findClosestMatchingComponent,
   findAllMatchingComponents,
+  findComponent,
 } from "../storeUtils"
 import { uuid } from "../uuid"
 import { removeBindings } from "../dataBinding"
@@ -67,6 +67,14 @@ export const getFrontendStore = () => {
     initialise: async pkg => {
       const { layouts, screens, application, clientLibPath } = pkg
       const components = await fetchComponentLibDefinitions(application.appId)
+      // make sure app isn't locked
+      if (
+        components &&
+        components.status === 400 &&
+        components.message?.includes("lock")
+      ) {
+        throw { ok: false, reason: "locked" }
+      }
       store.update(state => ({
         ...state,
         libraries: application.componentLibraries,
@@ -464,6 +472,24 @@ export const getFrontendStore = () => {
         if (!asset) {
           return
         }
+
+        // Fetch full definition
+        component = findComponent(asset.props, component._id)
+
+        // Ensure we aren't deleting the screen slot
+        if (component._component?.endsWith("/screenslot")) {
+          throw "You can't delete the screen slot"
+        }
+
+        // Ensure we aren't deleting something that contains the screen slot
+        const screenslot = findComponentType(
+          component,
+          "@budibase/standard-components/screenslot"
+        )
+        if (screenslot != null) {
+          throw "You can't delete a component that contains the screen slot"
+        }
+
         const parent = findComponentParent(asset.props, component._id)
         if (parent) {
           parent._children = parent._children.filter(

packages/builder/src/components/automation/AutomationBuilder/FlowChart/ActionModal.svelte

@@ -1,10 +1,9 @@
 <script>
   import { ModalContent, Layout, Detail, Body, Icon } from "@budibase/bbui"
   import { automationStore } from "builderStore"
-  import { database } from "stores/backend"
   import { externalActions } from "./ExternalActions"
-  $: instanceId = $database._id
 
+  export let blockIdx
   let selectedAction
   let actionVal
   let actions = Object.entries($automationStore.blockDefinitions.ACTION)
@@ -39,7 +38,8 @@
     )
     automationStore.actions.addBlockToAutomation(newBlock)
     await automationStore.actions.save(
-      $automationStore.selectedAutomation?.automation
+      $automationStore.selectedAutomation?.automation,
+      blockIdx
     )
   }
 </script>

packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowChart.svelte

@@ -4,9 +4,9 @@
   import FlowItem from "./FlowItem.svelte"
   import TestDataModal from "./TestDataModal.svelte"
   import { flip } from "svelte/animate"
-  import { fade, fly } from "svelte/transition"
+  import { fly } from "svelte/transition"
   import {
-    Detail,
+    Heading,
     Icon,
     ActionButton,
     notifications,
@@ -57,26 +57,24 @@
   <div class="content">
     <div class="title">
       <div class="subtitle">
-        <Detail size="L">{automation.name}</Detail>
+        <Heading size="S">{automation.name}</Heading>
-        <div
+        <div style="display:flex;">
-          style="display:flex;
+          <div class="iconPadding">
-          color: var(--spectrum-global-color-gray-400);"
-        >
-          <span class="iconPadding">
             <div class="icon">
               <Icon
                 on:click={confirmDeleteDialog.show}
                 hoverable
+                size="M"
                 name="DeleteOutline"
               />
             </div>
-          </span>
+          </div>
           <ActionButton
             on:click={() => {
               testDataModal.show()
             }}
             icon="MultipleCheck"
-            size="S">Run test</ActionButton
+            size="M">Run test</ActionButton
           >
         </div>
       </div>
@@ -84,16 +82,11 @@
     {#each blocks as block, idx (block.id)}
       <div
         class="block"
-        animate:flip={{ duration: 800 }}
+        animate:flip={{ duration: 500 }}
-        in:fade|local
+        in:fly|local={{ x: 500, duration: 1500 }}
-        out:fly|local={{ x: 500 }}
+        out:fly|local={{ x: 500, duration: 800 }}
       >
         <FlowItem {testDataModal} {testAutomation} {onSelect} {block} />
-        {#if idx !== blocks.length - 1}
-          <div class="separator" />
-          <Icon name="AddCircle" size="S" />
-          <div class="separator" />
-        {/if}
       </div>
     {/each}
   </div>
@@ -114,14 +107,6 @@
 </div>
 
 <style>
-  .separator {
-    width: 1px;
-    height: 25px;
-    border-left: 1px dashed var(--grey-4);
-    color: var(--grey-4);
-    /* center horizontally */
-    align-self: center;
-  }
   .canvas {
     margin: 0 -40px calc(-1 * var(--spacing-l)) -40px;
     overflow-y: auto;
@@ -153,11 +138,14 @@
     padding-bottom: var(--spacing-xl);
     display: flex;
     justify-content: space-between;
+    align-items: center;
+  }
+  .iconPadding {
+    padding-top: var(--spacing-s);
   }
 
   .icon {
     cursor: pointer;
-    display: flex;
     padding-right: var(--spacing-m);
   }
 </style>

packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowItem.svelte

@@ -14,7 +14,6 @@
   import CreateWebhookModal from "components/automation/Shared/CreateWebhookModal.svelte"
   import ResultsModal from "./ResultsModal.svelte"
   import ActionModal from "./ActionModal.svelte"
-  import { database } from "stores/backend"
   import { externalActions } from "./ExternalActions"
 
   export let onSelect
@@ -29,7 +28,6 @@
   $: testResult = $automationStore.selectedAutomation.testResults?.steps.filter(
     step => step.stepId === block.stepId
   )
-  $: instanceId = $database._id
 
   $: isTrigger = block.type === "TRIGGER"
 
@@ -40,6 +38,10 @@
   $: blockIdx = steps.findIndex(step => step.id === block.id)
   $: lastStep = !isTrigger && blockIdx + 1 === steps.length
 
+  $: totalBlocks =
+    $automationStore.selectedAutomation?.automation?.definition?.steps.length +
+    1
+
   // Logic for hiding / showing the add button.first we check if it has a child
   // then we check to see whether its inputs have been commpleted
   $: disableAddButton = isTrigger
@@ -167,13 +169,24 @@
   </Modal>
 
   <Modal bind:this={actionModal} width="30%">
-    <ActionModal bind:blockComplete />
+    <ActionModal {blockIdx} bind:blockComplete />
   </Modal>
 
   <Modal bind:this={webhookModal} width="30%">
     <CreateWebhookModal />
   </Modal>
 </div>
+<div class="separator" />
+<Icon
+  on:click={() => actionModal.show()}
+  disabled={!hasCompletedInputs}
+  hoverable
+  name="AddCircle"
+  size="S"
+/>
+{#if isTrigger ? totalBlocks > 1 : blockIdx !== totalBlocks - 2}
+  <div class="separator" />
+{/if}
 
 <style>
   .center-items {
@@ -191,8 +204,7 @@
   .block {
     width: 360px;
     font-size: 16px;
-    background-color: var(--spectrum-alias-background-color-secondary);
+    background-color: var(--background);
-    color: var(--grey-9);
     border: 1px solid var(--spectrum-global-color-gray-300);
     border-radius: 4px 4px 4px 4px;
   }
@@ -200,4 +212,13 @@
   .blockSection {
     padding: var(--spacing-xl);
   }
+
+  .separator {
+    width: 1px;
+    height: 25px;
+    border-left: 1px dashed var(--grey-4);
+    color: var(--grey-4);
+    /* center horizontally */
+    align-self: center;
+  }
 </style>

packages/builder/src/components/automation/Shared/WebhookDisplay.svelte

@@ -1,22 +1,14 @@
 <script>
   import { Input, Icon, notifications } from "@budibase/bbui"
-  import { store, hostingStore } from "builderStore"
 
   export let value
-  export let production = false
-
-  $: appId = $store.appId
-  $: appUrl = $hostingStore.appUrl
 
   function fullWebhookURL(uri) {
     if (!uri) {
       return ""
     }
-    if (production) {
+
-      return `${appUrl}/${uri}`
+    return `${window.location.origin}/${uri}`
-    } else {
-      return `${window.location.origin}/${uri}`
-    }
   }
 
   function copyToClipboard() {

packages/builder/src/components/backend/DatasourceNavigator/DatasourceNavigator.svelte

@@ -11,16 +11,18 @@
   import ICONS from "./icons"
 
   let openDataSources = []
-  $: enrichedDataSources = $datasources.list.map(datasource => {
+  $: enrichedDataSources = Array.isArray($datasources.list)
-    const selected = $datasources.selected === datasource._id
+    ? $datasources.list.map(datasource => {
-    const open = openDataSources.includes(datasource._id)
+        const selected = $datasources.selected === datasource._id
-    const containsSelected = containsActiveEntity(datasource)
+        const open = openDataSources.includes(datasource._id)
-    return {
+        const containsSelected = containsActiveEntity(datasource)
-      ...datasource,
+        return {
-      selected,
+          ...datasource,
-      open: selected || open || containsSelected,
+          selected,
-    }
+          open: selected || open || containsSelected,
-  })
+        }
+      })
+    : []
   $: openDataSource = enrichedDataSources.find(x => x.open)
   $: {
     // Ensure the open data source is always included in the list of open

packages/builder/src/components/backend/DatasourceNavigator/modals/CreateDatasourceModal.svelte

@@ -75,7 +75,7 @@
     }}
   >
     <Layout noPadding>
-      <Body size="XS"
+      <Body size="S"
         >All apps need data. You can connect to a data source below, or add data
         to your app using Budibase's built-in database.
       </Body>

packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte

@@ -3,26 +3,33 @@
   import { ModalContent, notifications, Body, Layout } from "@budibase/bbui"
   import analytics, { Events } from "analytics"
   import IntegrationConfigForm from "components/backend/DatasourceNavigator/TableIntegrationMenu/IntegrationConfigForm.svelte"
-  import { datasources } from "stores/backend"
+  import { datasources, tables } from "stores/backend"
   import { IntegrationNames } from "constants"
+  import cloneDeep from "lodash/cloneDeepWith"
 
   export let integration
+  export let modal
+
+  // kill the reference so the input isn't saved
+  let config = cloneDeep(integration)
 
   function prepareData() {
     let datasource = {}
     let existingTypeCount = $datasources.list.filter(
-      ds => ds.source == integration.type
+      ds => ds.source == config.type
     ).length
 
-    let baseName = IntegrationNames[integration.type]
+    let baseName = IntegrationNames[config.type]
     let name =
-      existingTypeCount == 0 ? baseName : `${baseName}-${existingTypeCount + 1}`
+      existingTypeCount === 0
+        ? baseName
+        : `${baseName}-${existingTypeCount + 1}`
 
     datasource.type = "datasource"
-    datasource.source = integration.type
+    datasource.source = config.type
-    datasource.config = integration.config
+    datasource.config = config.config
     datasource.name = name
-    datasource.plus = integration.plus
+    datasource.plus = config.plus
 
     return datasource
   }
@@ -32,6 +39,8 @@
       // Create datasource
       const resp = await datasources.save(datasource, datasource.plus)
 
+      // update the tables incase data source plus
+      await tables.fetch()
       await datasources.select(resp._id)
       $goto(`./datasource/${resp._id}`)
       notifications.success(`Datasource updated successfully.`)
@@ -48,9 +57,10 @@
 </script>
 
 <ModalContent
-  title={`Connect to ${IntegrationNames[integration.type]}`}
+  title={`Connect to ${IntegrationNames[config.type]}`}
   onConfirm={() => saveDatasource()}
-  confirmText={integration.plus
+  onCancel={() => modal.show()}
+  confirmText={config.plus
     ? "Fetch tables from database"
     : "Save and continue to query"}
   cancelText="Back"
@@ -62,10 +72,7 @@
     </Body>
   </Layout>
 
-  <IntegrationConfigForm
+  <IntegrationConfigForm schema={config.schema} integration={config.config} />
-    schema={integration.schema}
-    bind:integration={integration.config}
-  />
 </ModalContent>
 
 <style>

packages/builder/src/components/deploy/DeployModal.svelte

@@ -1,21 +1,10 @@
 <script>
-  import { onMount, onDestroy } from "svelte"
   import { Button, Modal, notifications, ModalContent } from "@budibase/bbui"
   import api from "builderStore/api"
   import analytics, { Events } from "analytics"
   import { store } from "builderStore"
 
-  const DeploymentStatus = {
-    SUCCESS: "SUCCESS",
-    PENDING: "PENDING",
-    FAILURE: "FAILURE",
-  }
-
-  const POLL_INTERVAL = 10000
-
   let feedbackModal
-  let deployments = []
-  let poll
   let publishModal
 
   async function deployApp() {
@@ -34,62 +23,6 @@
       notifications.error(`Error publishing app: ${err}`)
     }
   }
-
-  async function fetchDeployments() {
-    try {
-      const response = await api.get(`/api/deployments`)
-      const json = await response.json()
-
-      if (deployments.length > 0) {
-        checkIncomingDeploymentStatus(deployments, json)
-      }
-
-      deployments = json
-    } catch (err) {
-      console.error(err)
-      clearInterval(poll)
-      notifications.error(
-        "Error fetching deployment history. Please try again."
-      )
-    }
-  }
-
-  // Required to check any updated deployment statuses between polls
-  function checkIncomingDeploymentStatus(current, incoming) {
-    for (let incomingDeployment of incoming) {
-      if (
-        incomingDeployment.status === DeploymentStatus.FAILURE ||
-        incomingDeployment.status === DeploymentStatus.SUCCESS
-      ) {
-        const currentDeployment = current.find(
-          deployment => deployment._id === incomingDeployment._id
-        )
-
-        // We have just been notified of an ongoing deployments status change
-        if (
-          !currentDeployment ||
-          currentDeployment.status === DeploymentStatus.PENDING
-        ) {
-          if (incomingDeployment.status === DeploymentStatus.FAILURE) {
-            notifications.error(incomingDeployment.err)
-          } else {
-            notifications.send(
-              "Published to Production.",
-              "success",
-              "CheckmarkCircle"
-            )
-          }
-        }
-      }
-    }
-  }
-
-  onMount(() => {
-    fetchDeployments()
-    poll = setInterval(fetchDeployments, POLL_INTERVAL)
-  })
-
-  onDestroy(() => clearInterval(poll))
 </script>
 
 <Button secondary on:click={publishModal.show}>Publish</Button>

packages/builder/src/components/design/AppPreview/CurrentItemPreview.svelte

@@ -1,12 +1,20 @@
 <script>
+  import { get } from "svelte/store"
   import { onMount, onDestroy } from "svelte"
   import { store, currentAsset } from "builderStore"
   import iframeTemplate from "./iframeTemplate"
   import { Screen } from "builderStore/store/screenTemplates/utils/Screen"
   import { FrontendTypes } from "constants"
   import ConfirmDialog from "components/common/ConfirmDialog.svelte"
-  import { ProgressCircle, Layout, Heading, Body } from "@budibase/bbui"
+  import {
+    ProgressCircle,
+    Layout,
+    Heading,
+    Body,
+    notifications,
+  } from "@budibase/bbui"
   import ErrorSVG from "assets/error.svg?raw"
+  import { findComponent, findComponentPath } from "builderStore/storeUtils"
 
   let iframe
   let layout
@@ -102,7 +110,7 @@
     iframe.contentWindow.addEventListener("keydown", handleKeydownEvent)
   })
 
-  // remove all iframe event listeners on component destroy
+  // Remove all iframe event listeners on component destroy
   onDestroy(() => {
     if (iframe.contentWindow) {
       iframe.contentWindow.removeEventListener("bb-event", handleBudibaseEvent)
@@ -122,6 +130,26 @@
       // Wait for this event to show the client library if intelligent
       // loading is supported
       loading = false
+    } else if (type === "move-component") {
+      const { componentId, destinationComponentId } = data
+      const rootComponent = get(currentAsset).props
+
+      // Get source and destination components
+      const source = findComponent(rootComponent, componentId)
+      const destination = findComponent(rootComponent, destinationComponentId)
+
+      // Stop if the target is a child of source
+      const path = findComponentPath(source, destinationComponentId)
+      const ids = path.map(component => component._id)
+      if (ids.includes(data.destinationComponentId)) {
+        return
+      }
+
+      // Cut and paste the component to the new destination
+      if (source && destination) {
+        store.actions.components.copy(source, true)
+        store.actions.components.paste(destination, data.mode)
+      }
     } else {
       console.warning(`Client sent unknown event type: ${type}`)
     }
@@ -144,10 +172,15 @@
     confirmDeleteDialog.show()
   }
 
-  const deleteComponent = () => {
+  const deleteComponent = async () => {
-    store.actions.components.delete({ _id: idToDelete })
+    try {
+      await store.actions.components.delete({ _id: idToDelete })
+    } catch (error) {
+      notifications.error(error)
+    }
     idToDelete = null
   }
+
   const cancelDeleteComponent = () => {
     idToDelete = null
   }

packages/builder/src/components/design/AppPreview/ThemeEditor.svelte

@@ -62,7 +62,7 @@
 <Modal bind:this={modal}>
   <ModalContent
     showConfirmButton={false}
-    cancelText="Close"
+    cancelText="View changes"
     showCloseIcon={false}
     title="Theme settings"
   >
@@ -84,7 +84,7 @@
         </div>
       </div>
       <div class="setting">
-        <Label size="L">Primary color</Label>
+        <Label size="L">Accent color</Label>
         <ColorPicker
           spectrumTheme={$store.theme}
           value={$store.customTheme?.primaryColor || defaultTheme.primaryColor}
@@ -92,7 +92,7 @@
         />
       </div>
       <div class="setting">
-        <Label size="L">Primary color (hover)</Label>
+        <Label size="L">Accent color (hover)</Label>
         <ColorPicker
           spectrumTheme={$store.theme}
           value={$store.customTheme?.primaryColorHover ||

packages/builder/src/components/design/NavigationPanel/ComponentDropdownMenu.svelte

@@ -3,7 +3,7 @@
   import { store, currentAsset } from "builderStore"
   import ConfirmDialog from "components/common/ConfirmDialog.svelte"
   import { findComponentParent } from "builderStore/storeUtils"
-  import { ActionMenu, MenuItem, Icon } from "@budibase/bbui"
+  import { ActionMenu, MenuItem, Icon, notifications } from "@budibase/bbui"
 
   export let component
 
@@ -51,7 +51,11 @@
   }
 
   const deleteComponent = async () => {
-    await store.actions.components.delete(component)
+    try {
+      await store.actions.components.delete(component)
+    } catch (error) {
+      notifications.error(error)
+    }
   }
 
   const storeComponentForCopy = (cut = false) => {

packages/builder/src/components/design/NavigationPanel/NewScreenModal.svelte

@@ -5,6 +5,7 @@
   import { Input, Select, ModalContent, Toggle } from "@budibase/bbui"
   import getTemplates from "builderStore/store/screenTemplates"
   import analytics, { Events } from "analytics"
+  import sanitizeUrl from "builderStore/store/screenTemplates/utils/sanitizeUrl"
 
   const CONTAINER = "@budibase/standard-components/container"
 
@@ -84,7 +85,7 @@
     if (!event.detail.startsWith("/")) {
       route = "/" + event.detail
     }
-    route = route.replace(/ +/g, "-")
+    route = sanitizeUrl(route)
   }
 </script>
 

packages/builder/src/components/design/PropertiesPanel/ScreenSettingsSection.svelte

@@ -7,6 +7,7 @@
   import RoleSelect from "./PropertyControls/RoleSelect.svelte"
   import { currentAsset, store } from "builderStore"
   import { FrontendTypes } from "constants"
+  import sanitizeUrl from "builderStore/store/screenTemplates/utils/sanitizeUrl"
 
   export let componentInstance
   export let bindings
@@ -37,7 +38,12 @@
       key: "routing.route",
       label: "Route",
       control: Input,
-      parser: val => val.replace(/ +/g, "-"),
+      parser: val => {
+        if (!val.startsWith("/")) {
+          val = "/" + val
+        }
+        return sanitizeUrl(val)
+      },
     },
     { key: "routing.roleId", label: "Access", control: RoleSelect },
     { key: "layoutId", label: "Layout", control: LayoutSelect },

packages/builder/src/components/start/CreateAppModal.svelte

@@ -17,6 +17,7 @@
   import { capitalise } from "helpers"
   import { goto } from "@roxi/routify"
   import { APP_NAME_REGEX } from "constants"
+  import TemplateList from "./TemplateList.svelte"
 
   export let template
 
@@ -31,12 +32,16 @@
         APP_NAME_REGEX,
         "App name must be letters, numbers and spaces only"
       ),
-    file: template ? mixed().required("Please choose a file to import") : null,
+    file: template?.fromFile
+      ? mixed().required("Please choose a file to import")
+      : null,
   }
 
   let submitting = false
   let valid = false
+
   $: checkValidity($values, validator)
+  $: showTemplateSelection = !template?.fromFile && !template?.key
 
   onMount(async () => {
     await hostingStore.actions.fetchDeployedApps()
@@ -73,7 +78,7 @@
     submitting = true
 
     // Check a template exists if we are important
-    if (template && !$values.file) {
+    if (template?.fromFile && !$values.file) {
       $errors.file = "Please choose a file to import"
       valid = false
       submitting = false
@@ -133,33 +138,59 @@
   }
 </script>
 
-<ModalContent
+{#if showTemplateSelection}
-  title={template ? "Import app" : "Create app"}
+  <ModalContent
-  confirmText={template ? "Import app" : "Create app"}
+    title={"Get started quickly"}
-  onConfirm={createNewApp}
+    showConfirmButton={false}
-  disabled={!valid}
+    size="L"
->
+    onConfirm={() => {
-  {#if template}
+      showTemplateSelection = false
-    <Dropzone
+      return false
-      error={$touched.file && $errors.file}
+    }}
-      gallery={false}
+    showCancelButton={false}
-      label="File to import"
+    showCloseIcon={false}
-      value={[$values.file]}
+  >
-      on:change={e => {
+    <Body size="M">Select a template below, or start from scratch.</Body>
-        $values.file = e.detail?.[0]
+    <TemplateList
-        $touched.file = true
+      onSelect={selected => {
+        if (!selected) {
+          showTemplateSelection = false
+          return
+        }
+
+        template = selected
       }}
     />
-  {/if}
+  </ModalContent>
-  <Body size="S">
+{:else}
-    Give your new app a name, and choose which groups have access (paid plans
+  <ModalContent
-    only).
+    title={template?.fromFile ? "Import app" : "Create app"}
-  </Body>
+    confirmText={template?.fromFile ? "Import app" : "Create app"}
-  <Input
+    onConfirm={createNewApp}
-    bind:value={$values.name}
+    disabled={!valid}
-    error={$touched.name && $errors.name}
+  >
-    on:blur={() => ($touched.name = true)}
+    {#if template?.fromFile}
-    label="Name"
+      <Dropzone
-  />
+        error={$touched.file && $errors.file}
-  <Checkbox label="Group access" disabled value={true} text="All users" />
+        gallery={false}
-</ModalContent>
+        label="File to import"
+        value={[$values.file]}
+        on:change={e => {
+          $values.file = e.detail?.[0]
+          $touched.file = true
+        }}
+      />
+    {/if}
+    <Body size="S">
+      Give your new app a name, and choose which groups have access (paid plans
+      only).
+    </Body>
+    <Input
+      bind:value={$values.name}
+      error={$touched.name && $errors.name}
+      on:blur={() => ($touched.name = true)}
+      label="Name"
+    />
+    <Checkbox label="Group access" disabled value={true} text="All users" />
+  </ModalContent>
+{/if}

packages/builder/src/components/start/TemplateList.svelte

@@ -1,5 +1,5 @@
 <script>
-  import { Button, Heading, Body } from "@budibase/bbui"
+  import { Heading, Layout, Icon } from "@budibase/bbui"
   import Spinner from "components/common/Spinner.svelte"
   import api from "builderStore/api"
 
@@ -13,8 +13,7 @@
   let templatesPromise = fetchTemplates()
 </script>
 
-<div class="root">
+<Layout gap="XS" noPadding>
-  <Heading size="M">Start With a Template</Heading>
   {#await templatesPromise}
     <div class="spinner-container">
       <Spinner size="30" />
@@ -22,41 +21,69 @@
   {:then templates}
     <div class="templates">
       {#each templates as template}
-        <div class="templates-card">
+        <div class="template" on:click={() => onSelect(template)}>
-          <Heading size="S">{template.name}</Heading>
+          <div
-          <Body size="M" grey>{template.category}</Body>
+            class="background-icon"
-          <Body size="S" black>{template.description}</Body>
+            style={`background: ${template.background};`}
-          <div><img alt="template" src={template.image} width="100%" /></div>
+          >
-          <div class="card-footer">
+            <Icon name={template.icon} />
-            <Button secondary on:click={() => onSelect(template)}>
-              Create
-              {template.name}
-            </Button>
           </div>
+          <Heading size="XS">{template.name}</Heading>
+          <p class="detail">{template?.category?.toUpperCase()}</p>
         </div>
       {/each}
+      <div class="template start-from-scratch" on:click={() => onSelect(null)}>
+        <div class="background-icon" style={`background: var(--background);`}>
+          <Icon name="Add" />
+        </div>
+        <Heading size="XS">Start from scratch</Heading>
+        <p class="detail">BLANK</p>
+      </div>
     </div>
   {:catch err}
     <h1 style="color:red">{err}</h1>
   {/await}
-</div>
+</Layout>
 
 <style>
   .templates {
     display: grid;
-    grid-template-columns: repeat(auto-fill, minmax(280px, 1fr));
+    width: 100%;
-    grid-gap: var(--layout-m);
+    grid-gap: var(--spacing-m);
+    grid-template-columns: 1fr;
     justify-content: start;
+    margin-top: 15px;
   }
 
-  .templates-card {
+  .background-icon {
-    background-color: var(--background);
+    padding: 10px;
-    padding: var(--spacing-xl);
+    border-radius: 4px;
-    border-radius: var(--border-radius-m);
+    display: flex;
-    border: var(--border-dark);
+    align-items: center;
+    justify-content: center;
+    width: 18px;
+    color: white;
   }
 
-  .card-footer {
+  .template {
-    margin-top: var(--spacing-m);
+    height: 60px;
+    display: grid;
+    grid-gap: var(--layout-m);
+    grid-template-columns: 5% 1fr 15%;
+    border: 1px solid #494949;
+    align-items: center;
+    cursor: pointer;
+    border-radius: 4px;
+    background: #1a1a1a;
+    padding: 8px 16px;
+  }
+
+  .detail {
+    text-align: right;
+  }
+
+  .start-from-scratch {
+    background: var(--spectrum-global-color-gray-50);
+    margin-top: 20px;
   }
 </style>

packages/builder/src/components/upgrade/UpgradeModal.svelte

@@ -1,10 +1,11 @@
 <script>
   import { Modal, ModalContent, Button } from "@budibase/bbui"
+  import { admin } from "stores/portal"
 
   let upgradeModal
 
   const onConfirm = () => {
-    window.open("https://account.budibase.app/portal/install", "_blank")
+    window.open(`${$admin.accountPortalUrl}/portal/install`, "_blank")
   }
 </script>
 
@@ -21,12 +22,12 @@
   <ModalContent
     size="M"
     {onConfirm}
-    title="Upgrade to self-hosted"
+    title="Self-host Budibase"
-    confirmText="Upgrade"
+    confirmText="Self-host Budibase"
   >
-    <span
+    <span>
-      >Upgrade to Budibase self-hosting for free, and get SSO, unlimited apps,
+      Self-host budibase for free to get unlimited apps and more - and it only
-      and more - and it only takes a few minutes!</span
+      takes a few minutes!
-    >
+    </span>
   </ModalContent>
 </Modal>

packages/builder/src/constants/lucene.js

@@ -44,6 +44,15 @@ export const OperatorOptions = {
   },
 }
 
+export const NoEmptyFilterStrings = [
+  OperatorOptions.StartsWith.value,
+  OperatorOptions.Like.value,
+  OperatorOptions.Equals.value,
+  OperatorOptions.NotEquals.value,
+  OperatorOptions.Contains.value,
+  OperatorOptions.NotContains.value,
+]
+
 /**
  * Returns the valid operator options for a certain data type
  * @param type the data type

packages/builder/src/helpers/lucene.js

@@ -1,3 +1,26 @@
+import { NoEmptyFilterStrings } from "../constants/lucene"
+
+/**
+ * Removes any fields that contain empty strings that would cause inconsistent
+ * behaviour with how backend tables are filtered (no value means no filter).
+ */
+function cleanupQuery(query) {
+  if (!query) {
+    return query
+  }
+  for (let filterField of NoEmptyFilterStrings) {
+    if (!query[filterField]) {
+      continue
+    }
+    for (let [key, value] of Object.entries(query[filterField])) {
+      if (!value || value === "") {
+        delete query[filterField][key]
+      }
+    }
+  }
+  return query
+}
+
 /**
  * Builds a lucene JSON query from the filter structure generated in the builder
  * @param filter the builder filter structure
@@ -76,6 +99,8 @@ export const luceneQuery = (docs, query) => {
   if (!query) {
     return docs
   }
+  // make query consistent first
+  query = cleanupQuery(query)
 
   // Iterates over a set of filters and evaluates a fail function against a doc
   const match = (type, failFn) => doc => {

packages/builder/src/pages/builder/_layout.svelte

@@ -11,18 +11,38 @@
   $: cloud = $admin.cloud
   $: user = $auth.user
 
-  const validateTenantId = async () => {
+  $: useAccountPortal = cloud && !$admin.disableAccountPortal
-    // set the tenant from the url in the cloud
-    const tenantId = window.location.host.split(".")[0]
 
-    if (!tenantId.includes("localhost:")) {
+  const validateTenantId = async () => {
-      // user doesn't have permission to access this tenant - kick them out
+    const host = window.location.host
-      if (user?.tenantId !== tenantId) {
+    if (host.includes("localhost:")) {
+      // ignore local dev
+      return
+    }
+
+    // e.g. ['tenant', 'budibase', 'app'] vs ['budibase', 'app']
+    let urlTenantId
+    const hostParts = host.split(".")
+    if (hostParts.length > 2) {
+      urlTenantId = hostParts[0]
+    }
+
+    if (user && user.tenantId) {
+      // no tenant in the url - send to account portal to fix this
+      if (!urlTenantId) {
+        window.location.href = $admin.accountPortalUrl
+        return
+      }
+
+      if (user.tenantId !== urlTenantId) {
+        // user should not be here - play it safe and log them out
         await auth.logout()
         await auth.setOrganisation(null)
-      } else {
+        return
-        await auth.setOrganisation(tenantId)
       }
+    } else {
+      // no user - set the org according to the url
+      await auth.setOrganisation(urlTenantId)
     }
   }
 
@@ -30,7 +50,7 @@
     await auth.checkAuth()
     await admin.init()
 
-    if (cloud && multiTenancyEnabled) {
+    if (useAccountPortal && multiTenancyEnabled) {
       await validateTenantId()
     }
 
@@ -38,31 +58,35 @@
   })
 
   $: {
-    // We should never see the org or admin user creation screens in the cloud
+    const apiReady = $admin.loaded && $auth.loaded
-    if (!cloud) {
+    // if tenant is not set go to it
-      const apiReady = $admin.loaded && $auth.loaded
-      // if tenant is not set go to it
-      if (loaded && apiReady && multiTenancyEnabled && !tenantSet) {
-        $redirect("./auth/org")
-      }
-      // Force creation of an admin user if one doesn't exist
-      else if (loaded && apiReady && !hasAdminUser) {
-        $redirect("./admin")
-      }
-    }
-  }
-  // Redirect to log in at any time if the user isn't authenticated
-  $: {
     if (
+      loaded &&
+      !useAccountPortal &&
+      apiReady &&
+      multiTenancyEnabled &&
+      !tenantSet
+    ) {
+      $redirect("./auth/org")
+    }
+    // Force creation of an admin user if one doesn't exist
+    else if (loaded && !useAccountPortal && apiReady && !hasAdminUser) {
+      $redirect("./admin")
+    }
+    // Redirect to log in at any time if the user isn't authenticated
+    else if (
       loaded &&
       (hasAdminUser || cloud) &&
       !$auth.user &&
       !$isActive("./auth") &&
-      !$isActive("./invite")
+      !$isActive("./invite") &&
+      !$isActive("./admin")
     ) {
       const returnUrl = encodeURIComponent(window.location.pathname)
       $redirect("./auth?", { returnUrl })
-    } else if ($auth?.user?.forceResetPassword) {
+    }
+    // check if password reset required for user
+    else if ($auth.user?.forceResetPassword) {
       $redirect("./auth/reset")
     }
   }

packages/builder/src/pages/builder/admin/_components/ImportAppsModal.svelte

@@ -0,0 +1,50 @@
+<script>
+  import { notifications, ModalContent, Dropzone, Body } from "@budibase/bbui"
+  import { post } from "builderStore/api"
+
+  let submitting = false
+
+  $: value = { file: null }
+
+  async function importApps() {
+    submitting = true
+
+    try {
+      // Create form data to create app
+      let data = new FormData()
+      data.append("importFile", value.file)
+
+      // Create App
+      const importResp = await post("/api/cloud/import", data, {})
+      const importJson = await importResp.json()
+      if (!importResp.ok) {
+        throw new Error(importJson.message)
+      }
+      // now reload to get to login
+      window.location.reload()
+    } catch (error) {
+      notifications.error(error)
+      submitting = false
+    }
+  }
+</script>
+
+<ModalContent
+  title="Import apps"
+  confirmText="Import apps"
+  onConfirm={importApps}
+  disabled={!value.file}
+>
+  <Body
+    >Please upload the file that was exported from your Cloud environment to get
+    started</Body
+  >
+  <Dropzone
+    gallery={false}
+    label="File to import"
+    value={[value.file]}
+    on:change={e => {
+      value.file = e.detail?.[0]
+    }}
+  />
+</ModalContent>

packages/builder/src/pages/builder/admin/_layout.svelte

@@ -5,8 +5,11 @@
 
   let loaded = false
 
+  $: cloud = $admin.cloud
+  $: useAccountPortal = cloud && !$admin.disableAccountPortal
+
   onMount(() => {
-    if ($admin?.checklist?.adminUser.checked) {
+    if ($admin?.checklist?.adminUser.checked || useAccountPortal) {
       $redirect("../")
     } else {
       loaded = true

packages/builder/src/pages/builder/admin/index.svelte

@@ -7,18 +7,22 @@
     Input,
     Body,
     ActionButton,
+    Modal,
   } from "@budibase/bbui"
   import { goto } from "@roxi/routify"
   import api from "builderStore/api"
   import { admin, auth } from "stores/portal"
   import PasswordRepeatInput from "components/common/users/PasswordRepeatInput.svelte"
+  import ImportAppsModal from "./_components/ImportAppsModal.svelte"
   import Logo from "assets/bb-emblem.svg"
 
   let adminUser = {}
   let error
+  let modal
 
   $: tenantId = $auth.tenantId
   $: multiTenancyEnabled = $admin.multiTenancy
+  $: cloud = $admin.cloud
 
   async function save() {
     try {
@@ -38,6 +42,9 @@
   }
 </script>
 
+<Modal bind:this={modal} padding={false} width="600px">
+  <ImportAppsModal />
+</Modal>
 <section>
   <div class="container">
     <Layout>
@@ -66,6 +73,15 @@
           >
             Change organisation
           </ActionButton>
+        {:else if !cloud}
+          <ActionButton
+            quiet
+            on:click={() => {
+              modal.show()
+            }}
+          >
+            Import from cloud
+          </ActionButton>
         {/if}
       </Layout>
     </Layout>

packages/builder/src/pages/builder/app/[application]/_layout.svelte

@@ -8,7 +8,7 @@
   import NPSFeedbackForm from "components/feedback/NPSFeedbackForm.svelte"
   import { get } from "builderStore/api"
   import { auth, admin } from "stores/portal"
-  import { isActive, goto, layout } from "@roxi/routify"
+  import { isActive, goto, layout, redirect } from "@roxi/routify"
   import Logo from "assets/bb-emblem.svg"
   import { capitalise } from "helpers"
   import UpgradeModal from "../../../../components/upgrade/UpgradeModal.svelte"
@@ -34,7 +34,16 @@
     const pkg = await res.json()
 
     if (res.ok) {
-      await store.actions.initialise(pkg)
+      try {
+        await store.actions.initialise(pkg)
+        // edge case, lock wasn't known to client when it re-directed, or user went directly
+      } catch (err) {
+        if (!err.ok && err.reason === "locked") {
+          $redirect("../../")
+        } else {
+          throw err
+        }
+      }
       await automationStore.actions.fetch()
       await roles.fetch()
       return pkg
@@ -92,7 +101,7 @@
         <ActionGroup />
       </div>
       <div class="toprightnav">
-        {#if $admin.cloud}
+        {#if $admin.cloud && $auth.user.account}
           <UpgradeModal />
         {/if}
         <VersionModal />

packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/CreateEditRelationship/CreateEditRelationship.svelte

@@ -156,6 +156,8 @@
         ...relateTo,
         through: through._id,
         fieldName: fromTable.primary[0],
+        throughFrom: relateFrom.throughTo,
+        throughTo: relateFrom.throughFrom,
       }
     } else {
       // the relateFrom.fieldName should remain the same, as it is the foreignKey in the other
@@ -251,6 +253,22 @@
       bind:error={errors.through}
       bind:value={fromRelationship.through}
     />
+    {#if fromTable && toTable && through}
+      <Select
+        label={`Foreign Key (${fromTable?.name})`}
+        options={Object.keys(through?.schema)}
+        on:change={() => ($touched.fromForeign = true)}
+        bind:error={errors.fromForeign}
+        bind:value={fromRelationship.throughTo}
+      />
+      <Select
+        label={`Foreign Key (${toTable?.name})`}
+        options={Object.keys(through?.schema)}
+        on:change={() => ($touched.toForeign = true)}
+        bind:error={errors.toForeign}
+        bind:value={fromRelationship.throughFrom}
+      />
+    {/if}
   {:else if fromRelationship?.relationshipType && toTable}
     <Select
       label={`Foreign Key (${toTable?.name})`}

packages/builder/src/pages/builder/apps/index.svelte

@@ -159,8 +159,6 @@
     cursor: pointer;
     filter: brightness(110%);
   }
-  .group {
-  }
   .app {
     display: grid;
     grid-template-columns: auto 1fr auto;

packages/builder/src/pages/builder/auth/_layout.svelte

@@ -9,10 +9,10 @@
     $redirect("../")
   }
 
-  // redirect to account portal for authentication in the cloud
   if (
     !$auth.user &&
     $admin.cloud &&
+    !$admin.disableAccountPortal &&
     $admin.accountPortalUrl &&
     !$admin?.checklist?.sso?.checked
   ) {

packages/builder/src/pages/builder/auth/login.svelte

@@ -8,6 +8,7 @@
     Input,
     Layout,
     notifications,
+    Link,
   } from "@budibase/bbui"
   import { goto, params } from "@roxi/routify"
   import { auth, organisation, oidc, admin } from "stores/portal"
@@ -22,6 +23,7 @@
 
   $: company = $organisation.company || "Budibase"
   $: multiTenancyEnabled = $admin.multiTenancy
+  $: cloud = $admin.cloud
 
   async function login() {
     try {
@@ -84,7 +86,7 @@
         <ActionButton quiet on:click={() => $goto("./forgot")}>
           Forgot password?
         </ActionButton>
-        {#if multiTenancyEnabled}
+        {#if multiTenancyEnabled && !cloud}
           <ActionButton
             quiet
             on:click={() => {
@@ -96,6 +98,16 @@
           </ActionButton>
         {/if}
       </Layout>
+      {#if cloud}
+        <Body size="xs" textAlign="center">
+          By using Budibase Cloud
+          <br />
+          you are agreeing to our
+          <Link href="https://budibase.com/eula" target="_blank"
+            >License Agreement</Link
+          >
+        </Body>
+      {/if}
     </Layout>
   </div>
 </div>

packages/builder/src/pages/builder/auth/org.svelte

@@ -10,6 +10,8 @@
   $: multiTenancyEnabled = $admin.multiTenancy
   $: cloud = $admin.cloud
 
+  $: useAccountPortal = cloud && !$admin.disableAccountPortal
+
   async function setOrg() {
     if (tenantId == null || tenantId === "") {
       tenantId = "default"
@@ -26,7 +28,7 @@
 
   onMount(async () => {
     await auth.checkQueryString()
-    if (!multiTenancyEnabled || cloud) {
+    if (!multiTenancyEnabled || useAccountPortal) {
       $goto("../")
     } else {
       admin.unload()

packages/builder/src/pages/builder/index.svelte

@@ -5,11 +5,9 @@
   auth.checkQueryString()
 
   $: {
-    if (!$auth.user) {
+    if ($auth.user?.builder?.global) {
-      $redirect(`./auth`)
-    } else if ($auth.user.builder?.global) {
       $redirect(`./portal`)
-    } else {
+    } else if ($auth.user) {
       $redirect(`./apps`)
     }
   }

packages/builder/src/pages/builder/portal/apps/index.svelte

@@ -8,10 +8,9 @@
     ButtonGroup,
     Select,
     Modal,
-    ModalContent,
     Page,
     notifications,
-    Body,
+    Search,
   } from "@budibase/bbui"
   import CreateAppModal from "components/start/CreateAppModal.svelte"
   import UpdateAppModal from "components/start/UpdateAppModal.svelte"
@@ -35,8 +34,13 @@
   let unpublishModal
   let creatingApp = false
   let loaded = false
+  let searchTerm = ""
+  let cloud = $admin.cloud
 
   $: enrichedApps = enrichApps($apps, $auth.user, sortBy)
+  $: filteredApps = enrichedApps.filter(app =>
+    app?.name?.toLowerCase().includes(searchTerm.toLowerCase())
+  )
 
   const enrichApps = (apps, user, sortBy) => {
     const enrichedApps = apps.map(app => ({
@@ -45,6 +49,7 @@
       lockedYou: app.lockedBy && app.lockedBy.email === user?.email,
       lockedOther: app.lockedBy && app.lockedBy.email !== user?.email,
     }))
+
     if (sortBy === "status") {
       return enrichedApps.sort((a, b) => {
         if (a.status === b.status) {
@@ -70,6 +75,15 @@
     creatingApp = true
   }
 
+  const initiateAppsExport = () => {
+    try {
+      download(`/api/cloud/export`)
+      notifications.success("Apps exported successfully")
+    } catch (err) {
+      notifications.error(`Error exporting apps: ${err}`)
+    }
+  }
+
   const initiateAppImport = () => {
     template = { fromFile: true }
     creationModal.show()
@@ -190,6 +204,9 @@
       <div class="title">
         <Heading>Apps</Heading>
         <ButtonGroup>
+          {#if cloud}
+            <Button secondary on:click={initiateAppsExport}>Export apps</Button>
+          {/if}
           <Button secondary on:click={initiateAppImport}>Import app</Button>
           <Button cta on:click={initiateAppCreation}>Create app</Button>
         </ButtonGroup>
@@ -197,6 +214,7 @@
       <div class="filter">
         <div class="select">
           <Select
+            autoWidth
             bind:value={sortBy}
             placeholder={null}
             options={[
@@ -205,6 +223,9 @@
               { label: "Sort by status", value: "status" },
             ]}
           />
+          <div class="desktop-search">
+            <Search placeholder="Search" bind:value={searchTerm} />
+          </div>
         </div>
         <ActionGroup>
           <ActionButton
@@ -221,11 +242,14 @@
           />
         </ActionGroup>
       </div>
+      <div class="mobile-search">
+        <Search placeholder="Search" bind:value={searchTerm} />
+      </div>
       <div
         class:appGrid={layout === "grid"}
         class:appTable={layout === "table"}
       >
-        {#each enrichedApps as app (app.appId)}
+        {#each filteredApps as app (app.appId)}
           <svelte:component
             this={layout === "grid" ? AppCard : AppRow}
             {releaseLock}
@@ -244,22 +268,7 @@
   {#if !enrichedApps.length && !creatingApp && loaded}
     <div class="empty-wrapper">
       <Modal inline>
-        <ModalContent
+        <CreateAppModal {template} />
-          title="Create your first app"
-          confirmText="Create app"
-          showCancelButton={false}
-          showCloseIcon={false}
-          onConfirm={initiateAppCreation}
-          size="M"
-        >
-          <div slot="footer">
-            <Button on:click={initiateAppImport} secondary>Import app</Button>
-          </div>
-          <Body size="S">
-            The purpose of the Budibase builder is to help you build beautiful,
-            powerful applications quickly and easily.
-          </Body>
-        </ModalContent>
       </Modal>
     </div>
   {/if}
@@ -298,10 +307,26 @@
     flex-direction: row;
     justify-content: space-between;
     align-items: center;
+    gap: 10px;
+  }
+
+  @media only screen and (max-width: 560px) {
+    .title {
+      flex-direction: column;
+      align-items: flex-start;
+    }
   }
 
   .select {
-    width: 190px;
+    display: grid;
+    grid-template-columns: 1fr 1fr;
+    grid-gap: 10px;
+  }
+  .filter :global(.spectrum-ActionGroup) {
+    flex-wrap: nowrap;
+  }
+  .mobile-search {
+    display: none;
   }
 
   .appGrid {
@@ -342,5 +367,11 @@
     .appTable {
       grid-template-columns: 1fr auto;
     }
+    .desktop-search {
+      display: none;
+    }
+    .mobile-search {
+      display: block;
+    }
   }
 </style>

packages/builder/src/pages/builder/portal/manage/users/[userId].svelte

@@ -52,11 +52,11 @@
 
   async function deleteUser() {
     const res = await users.delete(userId)
-    if (res.message) {
+    if (res.status === 200) {
       notifications.success(`User ${$userFetch?.data?.email} deleted.`)
       $goto("./")
     } else {
-      notifications.error("Failed to delete user.")
+      notifications.error(res?.message ? res.message : "Failed to delete user.")
     }
   }
 

packages/builder/src/stores/portal/admin.js

@@ -7,6 +7,7 @@ export function createAdminStore() {
     loaded: false,
     multiTenancy: false,
     cloud: false,
+    disableAccountPortal: false,
     accountPortalUrl: "",
     onboardingProgress: 0,
     checklist: {
@@ -47,12 +48,14 @@ export function createAdminStore() {
   async function getEnvironment() {
     let multiTenancyEnabled = false
     let cloud = false
+    let disableAccountPortal = false
     let accountPortalUrl = ""
     try {
       const response = await api.get(`/api/system/environment`)
       const json = await response.json()
       multiTenancyEnabled = json.multiTenancy
       cloud = json.cloud
+      disableAccountPortal = json.disableAccountPortal
       accountPortalUrl = json.accountPortalUrl
     } catch (err) {
       // just let it stay disabled
@@ -60,6 +63,7 @@ export function createAdminStore() {
     admin.update(store => {
       store.multiTenancy = multiTenancyEnabled
       store.cloud = cloud
+      store.disableAccountPortal = disableAccountPortal
       store.accountPortalUrl = accountPortalUrl
       return store
     })

packages/builder/src/stores/portal/users.js

@@ -55,7 +55,11 @@ export function createUsersStore() {
   async function del(id) {
     const response = await api.delete(`/api/global/users/${id}`)
     update(users => users.filter(user => user._id !== id))
-    return await response.json()
+    const json = await response.json()
+    return {
+      ...json,
+      status: response.status,
+    }
   }
 
   async function save(data) {

packages/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/cli",
-  "version": "0.9.146-alpha.4",
+  "version": "0.9.154-alpha.1",
   "description": "Budibase CLI, for developers, self hosting and migrations.",
   "main": "src/index.js",
   "bin": {

packages/client/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/client",
-  "version": "0.9.146-alpha.4",
+  "version": "0.9.154-alpha.1",
   "license": "MPL-2.0",
   "module": "dist/budibase-client.js",
   "main": "dist/budibase-client.js",
@@ -19,9 +19,9 @@
     "dev:builder": "rollup -cw"
   },
   "dependencies": {
-    "@budibase/bbui": "^0.9.146-alpha.4",
+    "@budibase/bbui": "^0.9.154-alpha.1",
     "@budibase/standard-components": "^0.9.139",
-    "@budibase/string-templates": "^0.9.146-alpha.4",
+    "@budibase/string-templates": "^0.9.154-alpha.1",
     "regexparam": "^1.3.0",
     "shortid": "^2.2.15",
     "svelte-spa-router": "^3.0.5"

packages/client/src/api/analytics.js

@@ -0,0 +1,10 @@
+import API from "./api"
+
+/**
+ * Notifies that an end user client app has been loaded.
+ */
+export const pingEndUser = async () => {
+  return await API.post({
+    url: `/api/analytics/ping`,
+  })
+}

packages/client/src/api/index.js

@@ -9,3 +9,4 @@ export * from "./routes"
 export * from "./queries"
 export * from "./app"
 export * from "./automations"
+export * from "./analytics"

packages/client/src/components/ClientApp.svelte

@@ -23,6 +23,7 @@
   import SelectionIndicator from "components/preview/SelectionIndicator.svelte"
   import HoverIndicator from "components/preview/HoverIndicator.svelte"
   import CustomThemeWrapper from "./CustomThemeWrapper.svelte"
+  import DNDHandler from "components/preview/DNDHandler.svelte"
   import ErrorSVG from "builder/assets/error.svg"
 
   // Provide contexts
@@ -40,6 +41,8 @@
     dataLoaded = true
     if ($builderStore.inBuilder) {
       builderStore.actions.notifyLoaded()
+    } else {
+      builderStore.actions.pingEndUser()
     }
   })
 
@@ -104,7 +107,10 @@
               <div id="app-root">
                 <CustomThemeWrapper>
                   {#key $screenStore.activeLayout._id}
-                    <Component instance={$screenStore.activeLayout.props} />
+                    <Component
+                      isLayout
+                      instance={$screenStore.activeLayout.props}
+                    />
                   {/key}
 
                   <!-- Layers on top of app -->
@@ -122,6 +128,7 @@
               {#if $builderStore.inBuilder}
                 <SelectionIndicator />
                 <HoverIndicator />
+                <DNDHandler />
               {/if}
             </div>
           </StateBindingsProvider>

packages/client/src/components/Component.svelte

@@ -11,6 +11,8 @@
   import Placeholder from "components/app/Placeholder.svelte"
 
   export let instance = {}
+  export let isLayout = false
+  export let isScreen = false
 
   // The enriched component settings
   let enrichedSettings
@@ -49,11 +51,11 @@
   $: children = instance._children || []
   $: id = instance._id
   $: name = instance._instanceName
-  $: empty =
+  $: interactive =
-    !children.length &&
+    $builderStore.inBuilder &&
-    definition?.hasChildren &&
+    ($builderStore.previewType === "layout" || insideScreenslot)
-    definition?.showEmptyState !== false &&
+  $: empty = interactive && !children.length && definition?.hasChildren
-    $builderStore.inBuilder
+  $: emptyState = empty && definition?.showEmptyState !== false
   $: rawProps = getRawProps(instance)
   $: instanceKey = JSON.stringify(rawProps)
   $: updateComponentProps(rawProps, instanceKey, $context)
@@ -61,16 +63,16 @@
     $builderStore.inBuilder &&
     $builderStore.selectedComponentId === instance._id
   $: inSelectedPath = $builderStore.selectedComponentPath?.includes(id)
-  $: interactive = $builderStore.previewType === "layout" || insideScreenslot
   $: evaluateConditions(enrichedSettings?._conditions)
   $: componentSettings = { ...enrichedSettings, ...conditionalSettings }
+  $: renderKey = `${propsHash}-${emptyState}`
 
   // Update component context
   $: componentStore.set({
     id,
     children: children.length,
-    styles: { ...instance._styles, id, empty, interactive },
+    styles: { ...instance._styles, id, empty: emptyState, interactive },
-    empty,
+    empty: emptyState,
     selected,
     name,
   })
@@ -169,13 +171,22 @@
     conditionalSettings = result.settingUpdates
     visible = nextVisible
   }
+
+  // Drag and drop helper tags
+  $: draggable = interactive && !isLayout && !isScreen
+  $: droppable = interactive && !isLayout && !isScreen
 </script>
 
-{#key propsHash}
+{#key renderKey}
   {#if constructor && componentSettings && (visible || inSelectedPath)}
+    <!-- The ID is used as a class because getElementsByClassName is O(1) -->
+    <!-- and the performance matters for the selection indicators -->
     <div
       class={`component ${id}`}
-      data-type={interactive ? "component" : ""}
+      class:draggable
+      class:droppable
+      class:empty
+      class:interactive
       data-id={id}
       data-name={name}
     >
@@ -184,7 +195,7 @@
           {#each children as child (child._id)}
             <svelte:self instance={child} />
           {/each}
-        {:else if empty}
+        {:else if emptyState}
           <Placeholder />
         {/if}
       </svelte:component>
@@ -196,4 +207,10 @@
   .component {
     display: contents;
   }
+  .interactive :global(*:hover) {
+    cursor: pointer;
+  }
+  .draggable :global(*:hover) {
+    cursor: grab;
+  }
 </style>

packages/client/src/components/Screen.svelte

@@ -22,6 +22,6 @@
 <!-- Ensure to fully remount when screen changes -->
 {#key screenDefinition?._id}
   <Provider key="url" data={params}>
-    <Component instance={screenDefinition} />
+    <Component isScreen instance={screenDefinition} />
   </Provider>
 {/key}

packages/client/src/components/app/Button.svelte

@@ -31,4 +31,7 @@
   .spectrum-Button--overBackground:hover {
     color: #555;
   }
+  .spectrum-Button::after {
+    display: none;
+  }
 </style>

packages/client/src/components/app/Container.svelte

@@ -34,7 +34,7 @@
     display: flex;
     max-width: 100%;
   }
-  .valid-container :global([data-type="component"] > *) {
+  .valid-container :global(.component > *) {
     max-width: 100%;
   }
   .direction-row {
@@ -46,7 +46,7 @@
 
   /* Grow containers inside a row need 0 width 0 so that they ignore content */
   /* The nested selector for data-type is the wrapper around all components */
-  .direction-row :global(> [data-type="component"] > .size-grow) {
+  .direction-row :global(> .component > .size-grow) {
     width: 0;
   }
 

packages/client/src/components/app/Layout.svelte

@@ -353,7 +353,7 @@
   }
 
   /* Reduce padding */
-  .mobile .main {
+  .mobile:not(.layout--none) .main {
     padding: 16px;
   }
 

packages/client/src/components/preview/DNDHandler.svelte

@@ -0,0 +1,240 @@
+<script context="module">
+  export const Sides = {
+    Top: "Top",
+    Right: "Right",
+    Bottom: "Bottom",
+    Left: "Left",
+  }
+</script>
+
+<script>
+  import { onMount } from "svelte"
+  import { get } from "svelte/store"
+  import IndicatorSet from "./IndicatorSet.svelte"
+  import DNDPositionIndicator from "./DNDPositionIndicator.svelte"
+  import { builderStore } from "stores"
+
+  let dragInfo
+  let dropInfo
+
+  const getEdges = (bounds, mousePoint) => {
+    const { width, height, top, left } = bounds
+    return {
+      [Sides.Top]: [mousePoint[0], top],
+      [Sides.Right]: [left + width, mousePoint[1]],
+      [Sides.Bottom]: [mousePoint[0], top + height],
+      [Sides.Left]: [left, mousePoint[1]],
+    }
+  }
+
+  const calculatePointDelta = (point1, point2) => {
+    const deltaX = Math.abs(point1[0] - point2[0])
+    const deltaY = Math.abs(point1[1] - point2[1])
+    return Math.sqrt(deltaX * deltaX + deltaY * deltaY)
+  }
+
+  const getDOMNodeForComponent = component => {
+    const parent = component.closest(".component")
+    const children = Array.from(parent.childNodes)
+    return children?.find(node => node?.nodeType === 1)
+  }
+
+  // Callback when initially starting a drag on a draggable component
+  const onDragStart = e => {
+    const parent = e.target.closest(".component")
+    if (!parent?.classList.contains("draggable")) {
+      return
+    }
+
+    // Update state
+    dragInfo = {
+      target: parent.dataset.id,
+      parent: parent.dataset.parent,
+    }
+    builderStore.actions.selectComponent(dragInfo.target)
+    builderStore.actions.setDragging(true)
+
+    // Highlight being dragged by setting opacity
+    const child = getDOMNodeForComponent(e.target)
+    if (child) {
+      child.style.opacity = "0.5"
+    }
+  }
+
+  // Callback when drag stops (whether dropped or not)
+  const onDragEnd = e => {
+    // Reset opacity style
+    if (dragInfo) {
+      const child = getDOMNodeForComponent(e.target)
+      if (child) {
+        child.style.opacity = ""
+      }
+    }
+
+    // Reset state and styles
+    dragInfo = null
+    dropInfo = null
+    builderStore.actions.setDragging(false)
+  }
+
+  // Callback when on top of a component
+  const onDragOver = e => {
+    // Skip if we aren't validly dragging currently
+    if (!dragInfo || !dropInfo) {
+      return
+    }
+
+    e.preventDefault()
+    const { droppableInside, bounds } = dropInfo
+    const { top, left, height, width } = bounds
+    const mouseY = e.clientY
+    const mouseX = e.clientX
+    const snapFactor = droppableInside ? 0.33 : 0.5
+    const snapLimitV = Math.min(40, height * snapFactor)
+    const snapLimitH = Math.min(40, width * snapFactor)
+
+    // Determine all sies we are within snap range of
+    let sides = []
+    if (mouseY <= top + snapLimitV) {
+      sides.push(Sides.Top)
+    } else if (mouseY >= top + height - snapLimitV) {
+      sides.push(Sides.Bottom)
+    }
+    if (mouseX < left + snapLimitH) {
+      sides.push(Sides.Left)
+    } else if (mouseX > left + width - snapLimitH) {
+      sides.push(Sides.Right)
+    }
+
+    // When no edges match, drop inside if possible
+    if (!sides.length) {
+      dropInfo.mode = droppableInside ? "inside" : null
+      dropInfo.side = null
+      return
+    }
+
+    // When one edge matches, use that edge
+    if (sides.length === 1) {
+      dropInfo.side = sides[0]
+      if ([Sides.Top, Sides.Left].includes(sides[0])) {
+        dropInfo.mode = "above"
+      } else {
+        dropInfo.mode = "below"
+      }
+      return
+    }
+
+    // When 2 edges match, work out which is closer
+    const mousePoint = [mouseX, mouseY]
+    const edges = getEdges(bounds, mousePoint)
+    const edge1 = edges[sides[0]]
+    const delta1 = calculatePointDelta(mousePoint, edge1)
+    const edge2 = edges[sides[1]]
+    const delta2 = calculatePointDelta(mousePoint, edge2)
+    const edge = delta1 < delta2 ? sides[0] : sides[1]
+    dropInfo.side = edge
+    if ([Sides.Top, Sides.Left].includes(edge)) {
+      dropInfo.mode = "above"
+    } else {
+      dropInfo.mode = "below"
+    }
+  }
+
+  // Callback when entering a potential drop target
+  const onDragEnter = e => {
+    // Skip if we aren't validly dragging currently
+    if (!dragInfo) {
+      return
+    }
+
+    const element = e.target.closest(".component")
+    if (
+      element &&
+      element.classList.contains("droppable") &&
+      element.dataset.id !== dragInfo.target
+    ) {
+      // Do nothing if this is the same target
+      if (element.dataset.id === dropInfo?.target) {
+        return
+      }
+
+      // Ensure the dragging flag is always set.
+      // There's a bit of a race condition between the app reinitialisation
+      // after selecting the DND component and setting this the first time
+      if (!get(builderStore).isDragging) {
+        builderStore.actions.setDragging(true)
+      }
+
+      // Store target ID
+      const target = element.dataset.id
+
+      // Precompute and store some info to avoid recalculating everything in
+      // dragOver
+      const child = getDOMNodeForComponent(e.target)
+      const bounds = child.getBoundingClientRect()
+      dropInfo = {
+        target,
+        name: element.dataset.name,
+        droppableInside: element.classList.contains("empty"),
+        bounds,
+      }
+    } else {
+      dropInfo = null
+    }
+  }
+
+  // Callback when leaving a potential drop target.
+  // Since we don't style our targets, we don't need to unset anything.
+  const onDragLeave = () => {}
+
+  // Callback when dropping a drag on top of some component
+  const onDrop = e => {
+    e.preventDefault()
+    if (dropInfo?.mode) {
+      builderStore.actions.moveComponent(
+        dragInfo.target,
+        dropInfo.target,
+        dropInfo.mode
+      )
+    }
+  }
+
+  onMount(() => {
+    // Events fired on the draggable target
+    document.addEventListener("dragstart", onDragStart, false)
+    document.addEventListener("dragend", onDragEnd, false)
+
+    // Events fired on the drop targets
+    document.addEventListener("dragover", onDragOver, false)
+    document.addEventListener("dragenter", onDragEnter, false)
+    document.addEventListener("dragleave", onDragLeave, false)
+    document.addEventListener("drop", onDrop, false)
+
+    return () => {
+      // Events fired on the draggable target
+      document.removeEventListener("dragstart", onDragStart, false)
+      document.removeEventListener("dragend", onDragEnd, false)
+
+      // Events fired on the drop targets
+      document.removeEventListener("dragover", onDragOver, false)
+      document.removeEventListener("dragenter", onDragEnter, false)
+      document.removeEventListener("dragleave", onDragLeave, false)
+      document.removeEventListener("drop", onDrop, false)
+    }
+  })
+</script>
+
+<IndicatorSet
+  componentId={dropInfo?.mode === "inside" ? dropInfo.target : null}
+  color="var(--spectrum-global-color-static-green-500)"
+  zIndex="930"
+  transition
+  prefix="Inside"
+/>
+
+<DNDPositionIndicator
+  {dropInfo}
+  color="var(--spectrum-global-color-static-green-500)"
+  zIndex="940"
+  transition
+/>

packages/client/src/components/preview/DNDPositionIndicator.svelte

@@ -0,0 +1,64 @@
+<script>
+  import Indicator from "./Indicator.svelte"
+  import { Sides } from "./DNDHandler.svelte"
+
+  export let dropInfo
+  export let zIndex
+  export let color
+  export let transition
+
+  $: dimensions = getDimensions(dropInfo)
+  $: prefix = dropInfo?.mode === "above" ? "Before" : "After"
+  $: text = `${prefix} ${dropInfo?.name}`
+  $: renderKey = `${dropInfo?.target}-${dropInfo?.side}`
+
+  const getDimensions = info => {
+    const { bounds, side } = info ?? {}
+    if (!bounds || !side) {
+      return null
+    }
+
+    // Get preview offset
+    const root = document.getElementById("clip-root")
+    const rootBounds = root.getBoundingClientRect()
+
+    // Subtract preview offset from bounds
+    let { left, top, width, height } = bounds
+    left -= rootBounds.left
+    top -= rootBounds.top
+
+    // Determine position
+    if (side === Sides.Top || side === Sides.Bottom) {
+      return {
+        top: side === Sides.Top ? top - 4 : top + height,
+        left: left - 2,
+        width: width + 4,
+        height: 0,
+      }
+    } else {
+      return {
+        top: top - 2,
+        left: side === Sides.Left ? left - 4 : left + width,
+        width: 0,
+        height: height + 4,
+      }
+    }
+  }
+</script>
+
+{#key renderKey}
+  {#if dimensions && dropInfo?.mode !== "inside"}
+    <Indicator
+      left={Math.round(dimensions.left)}
+      top={Math.round(dimensions.top)}
+      width={dimensions.width}
+      height={dimensions.height}
+      {text}
+      {zIndex}
+      {color}
+      {transition}
+      alignRight={dropInfo?.side === Sides.Right}
+      line
+    />
+  {/if}
+{/key}

packages/client/src/components/preview/HoverIndicator.svelte

@@ -7,7 +7,7 @@
   $: zIndex = componentId === $builderStore.selectedComponentId ? 900 : 920
 
   const onMouseOver = e => {
-    const element = e.target.closest("[data-type='component']")
+    const element = e.target.closest(".interactive.component")
     const newId = element?.dataset?.id
     if (newId !== componentId) {
       componentId = newId
@@ -30,7 +30,7 @@
 </script>
 
 <IndicatorSet
-  {componentId}
+  componentId={$builderStore.isDragging ? null : componentId}
   color="var(--spectrum-global-color-static-blue-200)"
   transition
   {zIndex}

packages/client/src/components/preview/Indicator.svelte

@@ -9,6 +9,10 @@
   export let color
   export let zIndex
   export let transition = false
+  export let line = false
+  export let alignRight = false
+
+  $: flipped = top < 20
 </script>
 
 <div
@@ -18,11 +22,12 @@
   }}
   out:fade={{ duration: transition ? 130 : 0 }}
   class="indicator"
-  class:flipped={top < 20}
+  class:flipped
+  class:line
   style="top: {top}px; left: {left}px; width: {width}px; height: {height}px; --color: {color}; --zIndex: {zIndex};"
 >
   {#if text}
-    <div class="text" class:flipped={top < 20}>
+    <div class="text" class:flipped class:line class:right={alignRight}>
       {text}
     </div>
   {/if}
@@ -30,6 +35,7 @@
 
 <style>
   .indicator {
+    right: 0;
     position: absolute;
     z-index: var(--zIndex);
     border: 2px solid var(--color);
@@ -42,6 +48,9 @@
   .indicator.flipped {
     border-top-left-radius: 4px;
   }
+  .indicator.line {
+    border-radius: 4px !important;
+  }
   .text {
     background-color: var(--color);
     color: white;
@@ -61,9 +70,18 @@
     justify-content: flex-start;
     align-items: center;
   }
+  .text.line {
+    transform: translateY(-50%);
+    border-radius: 4px;
+  }
   .text.flipped {
     border-top-left-radius: 4px;
+    border-bottom-left-radius: 4px;
     transform: translateY(0%);
     top: -2px;
   }
+  .text.right {
+    right: -2px;
+    left: auto;
+  }
 </style>

packages/client/src/components/preview/IndicatorSet.svelte

@@ -7,6 +7,7 @@
   export let color
   export let transition
   export let zIndex
+  export let prefix = null
 
   let indicators = []
   let interval
@@ -51,6 +52,9 @@
     const parents = document.getElementsByClassName(componentId)
     if (parents.length) {
       text = parents[0].dataset.name
+      if (prefix) {
+        text = `${prefix} ${text}`
+      }
     }
 
     // Batch reads to minimize reflow

packages/client/src/components/preview/SettingsBar.svelte

@@ -16,7 +16,7 @@
   let measured = false
 
   $: definition = $builderStore.selectedComponentDefinition
-  $: showBar = definition?.showSettingsBar
+  $: showBar = definition?.showSettingsBar && !$builderStore.isDragging
   $: settings = definition?.settings?.filter(setting => setting.showInBar) ?? []
 
   const updatePosition = () => {

packages/client/src/stores/builder.js

@@ -1,6 +1,7 @@
 import { writable, derived } from "svelte/store"
 import Manifest from "manifest.json"
 import { findComponentById, findComponentPathById } from "../utils/components"
+import { pingEndUser } from "../api"
 
 const dispatchEvent = (type, data = {}) => {
   window.dispatchEvent(
@@ -23,6 +24,7 @@ const createBuilderStore = () => {
     theme: null,
     customTheme: null,
     previewDevice: "desktop",
+    isDragging: false,
   }
   const writableStore = writable(initialState)
   const derivedStore = derived(writableStore, $state => {
@@ -63,14 +65,28 @@ const createBuilderStore = () => {
     notifyLoaded: () => {
       dispatchEvent("preview-loaded")
     },
+    pingEndUser: () => {
+      pingEndUser()
+    },
     setSelectedPath: path => {
-      console.log("set to ")
-      console.log(path)
       writableStore.update(state => {
         state.selectedPath = path
         return state
       })
     },
+    moveComponent: (componentId, destinationComponentId, mode) => {
+      dispatchEvent("move-component", {
+        componentId,
+        destinationComponentId,
+        mode,
+      })
+    },
+    setDragging: dragging => {
+      writableStore.update(state => {
+        state.isDragging = dragging
+        return state
+      })
+    },
   }
   return {
     ...writableStore,

packages/client/src/utils/styleable.js

@@ -23,10 +23,14 @@ export const styleable = (node, styles = {}) => {
   let applyHoverStyles
   let selectComponent
 
+  // Allow dragging if required
+  const parent = node.closest(".component")
+  if (parent && parent.classList.contains("draggable")) {
+    node.setAttribute("draggable", true)
+  }
+
   // Creates event listeners and applies initial styles
   const setupStyles = (newStyles = {}) => {
-    // Use empty state styles as base styles if required, but let them, get
-    // overridden by any user specified styles
     let baseStyles = {}
     if (newStyles.empty) {
       baseStyles.border = "2px dashed var(--spectrum-global-color-gray-600)"
@@ -45,7 +49,6 @@ export const styleable = (node, styles = {}) => {
     // Applies a style string to a DOM node
     const applyStyles = styleString => {
       node.style = styleString
-      node.dataset.componentId = componentId
     }
 
     // Applies the "normal" style definition

packages/client/yarn.lock

@@ -29,9 +29,9 @@
     js-tokens "^4.0.0"
 
 "@budibase/bbui@^0.9.139":
-  version "0.9.145"
+  version "0.9.142"
-  resolved "https://registry.yarnpkg.com/@budibase/bbui/-/bbui-0.9.145.tgz#e65425e927e9488847aaf8209ff3eb0cf00c219c"
+  resolved "https://registry.yarnpkg.com/@budibase/bbui/-/bbui-0.9.142.tgz#7edbda7967c9e5dfc96e5be5231656e5aab8d0e3"
-  integrity sha512-vHSi+J52U24YSJPd1cfH9ePN92kCGLxKw4naYDjavYGd568GbRPJWzerzyqhm4VQtWn8FFi47jbzAsfAhiFfLA==
+  integrity sha512-m2YlqqH87T4RwqD/oGhH6twHIgvFv4oUMEhKpkgLsbxjXVLVD0OOF7WqjpDnSa4khVQaixjdkI/Jiw2qhBUSaA==
   dependencies:
     "@adobe/spectrum-css-workflow-icons" "^1.2.1"
     "@spectrum-css/actionbutton" "^1.0.1"

packages/server/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "0.9.146-alpha.4",
+  "version": "0.9.154-alpha.1",
   "description": "Budibase Web Server",
   "main": "src/index.js",
   "repository": {
@@ -27,7 +27,9 @@
     "multi:enable": "node scripts/multiTenancy.js enable",
     "multi:disable": "node scripts/multiTenancy.js disable",
     "selfhost:enable": "node scripts/selfhost.js enable",
-    "selfhost:disable": "node scripts/selfhost.js disable"
+    "selfhost:disable": "node scripts/selfhost.js disable",
+    "localdomain:enable": "node scripts/localdomain.js enable",
+    "localdomain:disable": "node scripts/localdomain.js disable"
   },
   "jest": {
     "preset": "ts-jest",
@@ -64,9 +66,9 @@
   "author": "Budibase",
   "license": "AGPL-3.0-or-later",
   "dependencies": {
-    "@budibase/auth": "^0.9.146-alpha.4",
+    "@budibase/auth": "^0.9.154-alpha.1",
-    "@budibase/client": "^0.9.146-alpha.4",
+    "@budibase/client": "^0.9.154-alpha.1",
-    "@budibase/string-templates": "^0.9.146-alpha.4",
+    "@budibase/string-templates": "^0.9.154-alpha.1",
     "@elastic/elasticsearch": "7.10.0",
     "@koa/router": "8.0.0",
     "@sendgrid/mail": "7.1.1",
@@ -96,6 +98,7 @@
     "koa-session": "5.12.0",
     "koa-static": "5.0.0",
     "lodash": "4.17.21",
+    "memorystream": "^0.3.1",
     "mongodb": "3.6.3",
     "mssql": "6.2.3",
     "mysql": "2.18.1",
@@ -103,6 +106,7 @@
     "open": "7.3.0",
     "pg": "8.5.1",
     "pino-pretty": "4.0.0",
+    "posthog-node": "^1.1.4",
     "pouchdb": "7.2.1",
     "pouchdb-adapter-memory": "^7.2.1",
     "pouchdb-all-dbs": "1.0.2",

packages/server/scripts/dev/manage.js

@@ -37,7 +37,7 @@ async function init() {
     const envFileJson = {
       PORT: 4001,
       MINIO_URL: "http://localhost:10000/",
-      COUCH_DB_URL: "http://@localhost:10000/db/",
+      COUCH_DB_URL: "http://budibase:budibase@localhost:10000/db/",
       REDIS_URL: "localhost:6379",
       WORKER_URL: "http://localhost:4002",
       INTERNAL_API_KEY: "budibase",
@@ -48,6 +48,7 @@ async function init() {
       COUCH_DB_PASSWORD: "budibase",
       COUCH_DB_USER: "budibase",
       SELF_HOSTED: 1,
+      DISABLE_ACCOUNT_PORTAL: "",
       MULTI_TENANCY: "",
     }
     let envFile = ""

packages/server/scripts/integrations/customer-categories/docker-compose.yml

@@ -0,0 +1,28 @@
+version: "3.8"
+services:
+  db:
+    container_name: postgres
+    image: postgres
+    restart: always
+    environment:
+      POSTGRES_USER: root
+      POSTGRES_PASSWORD: root
+      POSTGRES_DB: main
+    ports:
+      - "5432:5432"
+    volumes:
+      #- pg_data:/var/lib/postgresql/data/
+      - ./init.sql:/docker-entrypoint-initdb.d/init.sql
+
+  pgadmin:
+    container_name: pgadmin-pg
+    image: dpage/pgadmin4
+    restart: always
+    environment:
+      PGADMIN_DEFAULT_EMAIL: root@root.com
+      PGADMIN_DEFAULT_PASSWORD: root
+    ports:
+      - "5050:80"
+
+#volumes:
+#  pg_data:

packages/server/scripts/integrations/customer-categories/init.sql

@@ -0,0 +1,41 @@
+SELECT 'CREATE DATABASE main'
+WHERE NOT EXISTS (SELECT FROM pg_database WHERE datname = 'main')\gexec
+
+CREATE TABLE categories
+(
+    name text COLLATE pg_catalog."default",
+    id integer NOT NULL GENERATED ALWAYS AS IDENTITY ( INCREMENT 1 START 1 MINVALUE 1 MAXVALUE 2147483647 CACHE 1 ),
+    CONSTRAINT categories_pkey PRIMARY KEY (id)
+);
+
+CREATE TABLE customers
+(
+    id integer NOT NULL GENERATED ALWAYS AS IDENTITY ( INCREMENT 1 START 1 MINVALUE 1 MAXVALUE 2147483647 CACHE 1 ),
+    name text COLLATE pg_catalog."default",
+    email text COLLATE pg_catalog."default",
+    age integer,
+    "dateOfBirth" date,
+    CONSTRAINT customers_pkey PRIMARY KEY (id)
+);
+
+CREATE TABLE customer_category
+(
+    customer_id integer,
+    category_id integer,
+    notes text COLLATE pg_catalog."default",
+    id integer NOT NULL GENERATED ALWAYS AS IDENTITY ( INCREMENT 1 START 1 MINVALUE 1 MAXVALUE 2147483647 CACHE 1 ),
+    CONSTRAINT "Category" FOREIGN KEY (category_id)
+        REFERENCES public.categories (id) MATCH SIMPLE
+        ON UPDATE NO ACTION
+        ON DELETE NO ACTION
+        NOT VALID,
+    CONSTRAINT "Customer" FOREIGN KEY (customer_id)
+        REFERENCES public.customers (id) MATCH SIMPLE
+        ON UPDATE NO ACTION
+        ON DELETE NO ACTION
+        NOT VALID
+);
+
+
+INSERT INTO customers (name, email, age) VALUES ('Mike', 'mike@mike.com', 30);
+INSERT INTO categories (name) VALUES ('Books');

packages/server/scripts/integrations/customer-categories/reset.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+docker-compose down
+docker volume prune -f

packages/server/scripts/localdomain.js

@@ -0,0 +1,22 @@
+#!/usr/bin/env node
+const updateDotEnv = require("update-dotenv")
+
+const arg = process.argv.slice(2)[0]
+
+/**
+ * For testing multi tenancy sub domains locally.
+ *
+ * Relies on an entry in /etc/hosts e.g:
+ *
+ * 127.0.0.1   local.com
+ *
+ * and an entry for each tenant you wish to test locally e.g:
+ *
+ * 127.0.0.1   t1.local.com
+ * 127.0.0.1   t2.local.com
+ */
+updateDotEnv({
+  ACCOUNT_PORTAL_URL:
+    arg === "enable" ? "http://local.com:10001" : "http://localhost:10001",
+  COOKIE_DOMAIN: arg === "enable" ? ".local.com" : "",
+}).then(() => console.log("Updated worker!"))

packages/server/src/api/controllers/analytics.js

@@ -1,7 +1,32 @@
 const env = require("../../environment")
+const PostHog = require("posthog-node")
 
-exports.isEnabled = async function (ctx) {
+let posthogClient
+
+if (env.POSTHOG_TOKEN && env.ENABLE_ANALYTICS && !env.SELF_HOSTED) {
+  posthogClient = new PostHog(env.POSTHOG_TOKEN)
+}
+
+exports.isEnabled = async ctx => {
   ctx.body = {
     enabled: !env.SELF_HOSTED && env.ENABLE_ANALYTICS === "true",
   }
 }
+
+exports.endUserPing = async ctx => {
+  if (!posthogClient) {
+    ctx.body = {
+      ping: false,
+    }
+    return
+  }
+
+  posthogClient.capture("budibase:end_user_ping", {
+    userId: ctx.user && ctx.user._id,
+    appId: ctx.appId,
+  })
+
+  ctx.body = {
+    ping: true,
+  }
+}

packages/server/src/api/controllers/application.js

@@ -31,7 +31,7 @@ const {
   getDeployedApps,
   removeAppFromUserRoles,
 } = require("../../utilities/workerRequests")
-const { clientLibraryPath } = require("../../utilities")
+const { clientLibraryPath, stringToReadStream } = require("../../utilities")
 const { getAllLocks } = require("../../utilities/redis")
 const {
   updateClientLibrary,
@@ -82,7 +82,7 @@ async function getAppUrlIfNotInUse(ctx) {
   if (!env.SELF_HOSTED) {
     return url
   }
-  const deployedApps = await getDeployedApps(ctx)
+  const deployedApps = await getDeployedApps()
   if (
     url &&
     deployedApps[url] != null &&
@@ -114,8 +114,13 @@ async function createInstance(template) {
 
   // replicate the template data to the instance DB
   // this is currently very hard to test, downloading and importing template files
-  /* istanbul ignore next */
+  if (template && template.templateString) {
-  if (template && template.useTemplate === "true") {
+    const { ok } = await db.load(stringToReadStream(template.templateString))
+    if (!ok) {
+      throw "Error loading database dump from memory."
+    }
+  } else if (template && template.useTemplate === "true") {
+    /* istanbul ignore next */
     const { ok } = await db.load(await getTemplateStream(template))
     if (!ok) {
       throw "Error loading database dump from template."
@@ -191,10 +196,11 @@ exports.fetchAppPackage = async function (ctx) {
 }
 
 exports.create = async function (ctx) {
-  const { useTemplate, templateKey } = ctx.request.body
+  const { useTemplate, templateKey, templateString } = ctx.request.body
   const instanceConfig = {
     useTemplate,
     key: templateKey,
+    templateString,
   }
   if (ctx.request.files && ctx.request.files.templateFile) {
     instanceConfig.file = ctx.request.files.templateFile

packages/server/src/api/controllers/cloud.js

@@ -0,0 +1,92 @@
+const env = require("../../environment")
+const { getAllApps } = require("@budibase/auth/db")
+const CouchDB = require("../../db")
+const {
+  exportDB,
+  sendTempFile,
+  readFileSync,
+} = require("../../utilities/fileSystem")
+const { stringToReadStream } = require("../../utilities")
+const { getGlobalDBName, getGlobalDB } = require("@budibase/auth/tenancy")
+const { create } = require("./application")
+const { getDocParams, DocumentTypes, isDevAppID } = require("../../db/utils")
+
+async function createApp(appName, appImport) {
+  const ctx = {
+    request: {
+      body: {
+        templateString: appImport,
+        name: appName,
+      },
+    },
+  }
+  return create(ctx)
+}
+
+exports.exportApps = async ctx => {
+  if (env.SELF_HOSTED || !env.MULTI_TENANCY) {
+    ctx.throw(400, "Exporting only allowed in multi-tenant cloud environments.")
+  }
+  const apps = await getAllApps(CouchDB, { all: true })
+  const globalDBString = await exportDB(getGlobalDBName())
+  let allDBs = {
+    global: globalDBString,
+  }
+  for (let app of apps) {
+    // only export the dev apps as they will be the latest, the user can republish the apps
+    // in their self hosted environment
+    if (isDevAppID(app._id)) {
+      allDBs[app.name] = await exportDB(app._id)
+    }
+  }
+  const filename = `cloud-export-${new Date().getTime()}.txt`
+  ctx.attachment(filename)
+  ctx.body = sendTempFile(JSON.stringify(allDBs))
+}
+
+async function getAllDocType(db, docType) {
+  const response = await db.allDocs(
+    getDocParams(docType, null, {
+      include_docs: true,
+    })
+  )
+  return response.rows.map(row => row.doc)
+}
+
+exports.importApps = async ctx => {
+  if (!env.SELF_HOSTED || env.MULTI_TENANCY) {
+    ctx.throw(400, "Importing only allowed in self hosted environments.")
+  }
+  const apps = await getAllApps(CouchDB, { all: true })
+  if (
+    apps.length !== 0 ||
+    !ctx.request.files ||
+    !ctx.request.files.importFile
+  ) {
+    ctx.throw(
+      400,
+      "Import file is required and environment must be fresh to import apps."
+    )
+  }
+  const importFile = ctx.request.files.importFile
+  const importString = readFileSync(importFile.path)
+  const dbs = JSON.parse(importString)
+  const globalDbImport = dbs.global
+  // remove from the list of apps
+  delete dbs.global
+  const globalDb = getGlobalDB()
+  // load the global db first
+  await globalDb.load(stringToReadStream(globalDbImport))
+  for (let [appName, appImport] of Object.entries(dbs)) {
+    await createApp(appName, appImport)
+  }
+  // once apps are created clean up the global db
+  let users = await getAllDocType(globalDb, DocumentTypes.USER)
+  for (let user of users) {
+    delete user.tenantId
+  }
+  await globalDb.bulkDocs(users)
+  ctx.body = {
+    message: "Apps successfully imported.",
+  }
+}

packages/server/src/api/controllers/deploy/index.js

@@ -64,6 +64,7 @@ async function storeDeploymentHistory(deployment) {
 
 async function initDeployedApp(prodAppId) {
   const db = new CouchDB(prodAppId)
+  console.log("Reading automation docs")
   const automations = (
     await db.allDocs(
       getAutomationParams(null, {
@@ -71,12 +72,17 @@ async function initDeployedApp(prodAppId) {
       })
     )
   ).rows.map(row => row.doc)
+  console.log("You have " + automations.length + " automations")
   const promises = []
+  console.log("Disabling prod crons..")
   await disableAllCrons(prodAppId)
+  console.log("Prod Cron triggers disabled..")
+  console.log("Enabling cron triggers for deployed app..")
   for (let automation of automations) {
     promises.push(enableCronTrigger(prodAppId, automation))
   }
   await Promise.all(promises)
+  console.log("Enabled cron triggers for deployed app..")
 }
 
 async function deployApp(deployment) {
@@ -88,13 +94,18 @@ async function deployApp(deployment) {
       target: productionAppId,
     })
 
+    console.log("Replication object created")
+
     await replication.replicate()
+    console.log("replication complete.. replacing app meta doc")
     const db = new CouchDB(productionAppId)
     const appDoc = await db.get(DocumentTypes.APP_METADATA)
     appDoc.appId = productionAppId
     appDoc.instance._id = productionAppId
     await db.put(appDoc)
+    console.log("New app doc written successfully.")
 
+    console.log("Setting up live repl between dev and prod")
     // Set up live sync between the live and dev instances
     const liveReplication = new Replication({
       source: productionAppId,
@@ -105,8 +116,11 @@ async function deployApp(deployment) {
         return doc._id !== DocumentTypes.APP_METADATA
       },
     })
+    console.log("Set up live repl between dev and prod")
 
+    console.log("Initialising deployed app")
     await initDeployedApp(productionAppId)
+    console.log("Init complete, setting deployment to successful")
     deployment.setStatus(DeploymentStatus.SUCCESS)
     await storeDeploymentHistory(deployment)
   } catch (err) {
@@ -153,9 +167,13 @@ exports.deploymentProgress = async function (ctx) {
 
 exports.deployApp = async function (ctx) {
   let deployment = new Deployment(ctx.appId)
+  console.log("Deployment object created")
   deployment.setStatus(DeploymentStatus.PENDING)
+  console.log("Deployment object set to pending")
   deployment = await storeDeploymentHistory(deployment)
+  console.log("Stored deployment history")
 
+  console.log("Deploying app...")
   await deployApp(deployment)
 
   ctx.body = deployment

packages/server/src/api/controllers/hosting.js

@@ -18,5 +18,5 @@ exports.fetchUrls = async ctx => {
 }
 
 exports.getDeployedApps = async ctx => {
-  ctx.body = await getDeployedApps(ctx)
+  ctx.body = await getDeployedApps()
 }

packages/server/src/api/controllers/row/ExternalRequest.ts

@@ -178,7 +178,12 @@ module External {
         manyRelationships: ManyRelationship[] = []
       for (let [key, field] of Object.entries(table.schema)) {
         // if set already, or not set just skip it
-        if (!row[key] || newRow[key] || field.autocolumn) {
+        if ((!row[key] && row[key] !== "") || newRow[key] || field.autocolumn) {
+          continue
+        }
+        // if its an empty string then it means return the column to null (if possible)
+        if (row[key] === "") {
+          newRow[key] = null
           continue
         }
         // parse floats/numbers
@@ -205,9 +210,13 @@ module External {
         } else {
           // we're not inserting a doc, will be a bunch of update calls
           const isUpdate = !field.through
-          const thisKey: string = isUpdate ? "id" : linkTablePrimary
+          const thisKey: string = isUpdate
+            ? "id"
+            : field.throughTo || linkTablePrimary
           // @ts-ignore
-          const otherKey: string = isUpdate ? field.fieldName : tablePrimary
+          const otherKey: string = isUpdate
+            ? field.fieldName
+            : field.throughFrom || tablePrimary
           row[key].map((relationship: any) => {
             // we don't really support composite keys for relationships, this is why [0] is used
             manyRelationships.push({
@@ -328,12 +337,11 @@ module External {
         if (!table.primary || !linkTable.primary) {
           continue
         }
-        const definition = {
+        const definition: any = {
           // if no foreign key specified then use the name of the field in other table
           from: field.foreignKey || table.primary[0],
           to: field.fieldName,
           tableName: linkTableName,
-          through: undefined,
           // need to specify where to put this back into
           column: fieldName,
         }
@@ -343,8 +351,10 @@ module External {
           )
           definition.through = throughTableName
           // don't support composite keys for relationships
-          definition.from = table.primary[0]
+          definition.from = field.throughFrom || table.primary[0]
-          definition.to = linkTable.primary[0]
+          definition.to = field.throughTo || linkTable.primary[0]
+          definition.fromPrimary = table.primary[0]
+          definition.toPrimary = linkTable.primary[0]
         }
         relationships.push(definition)
       }
@@ -369,7 +379,8 @@ module External {
         }
         const isMany = field.relationshipType === RelationshipTypes.MANY_TO_MANY
         const tableId = isMany ? field.through : field.tableId
-        const fieldName = isMany ? primaryKey : field.fieldName
+        const manyKey = field.throughFrom || primaryKey
+        const fieldName = isMany ? manyKey : field.fieldName
         const response = await makeExternalQuery(this.appId, {
           endpoint: getEndpoint(tableId, DataSourceOperation.READ),
           filters: {

packages/server/src/api/controllers/row/external.js

@@ -2,6 +2,7 @@ const {
   DataSourceOperation,
   SortDirection,
   FieldTypes,
+  NoEmptyFilterStrings,
 } = require("../../../constants")
 const {
   breakExternalTableId,
@@ -11,6 +12,19 @@ const ExternalRequest = require("./ExternalRequest")
 const CouchDB = require("../../../db")
 
 async function handleRequest(appId, operation, tableId, opts = {}) {
+  // make sure the filters are cleaned up, no empty strings for equals, fuzzy or string
+  if (opts && opts.filters) {
+    for (let filterField of NoEmptyFilterStrings) {
+      if (!opts.filters[filterField]) {
+        continue
+      }
+      for (let [key, value] of Object.entries(opts.filters[filterField])) {
+        if (!value || value === "") {
+          delete opts.filters[filterField][key]
+        }
+      }
+    }
+  }
   return new ExternalRequest(appId, operation, tableId, opts.datasource).run(
     opts
   )

packages/server/src/api/controllers/row/internal.js

@@ -5,7 +5,6 @@ const {
   generateRowID,
   DocumentTypes,
   InternalTables,
-  generateMemoryViewID,
 } = require("../../../db/utils")
 const userController = require("../user")
 const {
@@ -20,7 +19,12 @@ const { fullSearch, paginatedSearch } = require("./internalSearch")
 const { getGlobalUsersFromMetadata } = require("../../../utilities/global")
 const inMemoryViews = require("../../../db/inMemoryView")
 const env = require("../../../environment")
-const { migrateToInMemoryView } = require("../view/utils")
+const {
+  migrateToInMemoryView,
+  migrateToDesignView,
+  getFromDesignDoc,
+  getFromMemoryDoc,
+} = require("../view/utils")
 
 const CALCULATION_TYPES = {
   SUM: "sum",
@@ -74,33 +78,24 @@ async function getRawTableData(ctx, db, tableId) {
 }
 
 async function getView(db, viewName) {
-  let viewInfo
+  let mainGetter = env.SELF_HOSTED ? getFromDesignDoc : getFromMemoryDoc
-  async function getFromDesignDoc() {
+  let secondaryGetter = env.SELF_HOSTED ? getFromMemoryDoc : getFromDesignDoc
-    const designDoc = await db.get("_design/database")
+  let migration = env.SELF_HOSTED ? migrateToDesignView : migrateToInMemoryView
-    viewInfo = designDoc.views[viewName]
+  let viewInfo,
-    return viewInfo
+    migrate = false
-  }
+  try {
-  let migrate = false
+    viewInfo = await mainGetter(db, viewName)
-  if (env.SELF_HOSTED) {
+  } catch (err) {
-    viewInfo = await getFromDesignDoc()
+    // check if it can be retrieved from design doc (needs migrated)
-  } else {
+    if (err.status !== 404) {
-    try {
+      viewInfo = null
-      viewInfo = await db.get(generateMemoryViewID(viewName))
+    } else {
-      if (viewInfo) {
+      viewInfo = await secondaryGetter(db, viewName)
-        viewInfo = viewInfo.view
+      migrate = !!viewInfo
-      }
-    } catch (err) {
-      // check if it can be retrieved from design doc (needs migrated)
-      if (err.status !== 404) {
-        viewInfo = null
-      } else {
-        viewInfo = await getFromDesignDoc()
-        migrate = !!viewInfo
-      }
     }
   }
   if (migrate) {
-    await migrateToInMemoryView(db, viewName)
+    await migration(db, viewName)
   }
   if (!viewInfo) {
     throw "View does not exist."
@@ -408,16 +403,32 @@ exports.fetchEnrichedRow = async ctx => {
     rowId,
   })
   // look up the actual rows based on the ids
-  const response = await db.allDocs({
+  let response = (
-    include_docs: true,
+    await db.allDocs({
-    keys: linkVals.map(linkVal => linkVal.id),
+      include_docs: true,
-  })
+      keys: linkVals.map(linkVal => linkVal.id),
-  // need to include the IDs in these rows for any links they may have
+    })
-  let linkedRows = await outputProcessing(
+  ).rows.map(row => row.doc)
-    ctx,
+  // group responses by table
-    table,
+  let groups = {},
-    response.rows.map(row => row.doc)
+    tables = {}
-  )
+  for (let row of response) {
+    const linkedTableId = row.tableId
+    if (groups[linkedTableId] == null) {
+      groups[linkedTableId] = [row]
+      tables[linkedTableId] = await db.get(linkedTableId)
+    } else {
+      groups[linkedTableId].push(row)
+    }
+  }
+  let linkedRows = []
+  for (let [tableId, rows] of Object.entries(groups)) {
+    // need to include the IDs in these rows for any links they may have
+    linkedRows = linkedRows.concat(
+      await outputProcessing(ctx, tables[tableId], rows)
+    )
+  }
+
   // insert the link rows in the correct place throughout the main row
   for (let fieldName of Object.keys(table.schema)) {
     let field = table.schema[fieldName]

packages/server/src/api/controllers/static/index.js

@@ -40,7 +40,7 @@ async function prepareUpload({ s3Key, bucket, metadata, file }) {
 async function checkForSelfHostedURL(ctx) {
   // the "appId" component of the URL may actually be a specific self hosted URL
   let possibleAppUrl = `/${encodeURI(ctx.params.appId).toLowerCase()}`
-  const apps = await getDeployedApps(ctx)
+  const apps = await getDeployedApps()
   if (apps[possibleAppUrl] && apps[possibleAppUrl].appId) {
     return apps[possibleAppUrl].appId
   } else {

packages/server/src/api/controllers/view/utils.js

@@ -107,3 +107,30 @@ exports.migrateToInMemoryView = async (db, viewName) => {
   await db.put(designDoc)
   await exports.saveView(db, null, viewName, view)
 }
+
+exports.migrateToDesignView = async (db, viewName) => {
+  let view = await db.get(generateMemoryViewID(viewName))
+  const designDoc = await db.get("_design/database")
+  designDoc.views[viewName] = view.view
+  await db.put(designDoc)
+  await db.remove(view._id, view._rev)
+}
+
+exports.getFromDesignDoc = async (db, viewName) => {
+  const designDoc = await db.get("_design/database")
+  let view = designDoc.views[viewName]
+  if (view == null) {
+    throw { status: 404, message: "Unable to get view" }
+  }
+  return view
+}
+
+exports.getFromMemoryDoc = async (db, viewName) => {
+  let view = await db.get(generateMemoryViewID(viewName))
+  if (view) {
+    view = view.view
+  } else {
+    throw { status: 404, message: "Unable to get view" }
+  }
+  return view
+}