merge with master

.all-contributorsrc

@@ -75,6 +75,28 @@
         "design"
       ]
     },
+    {
+      "login": "Rory-Powell",
+      "name": "Rory Powell",
+      "avatar_url": "https://avatars.githubusercontent.com/u/8755148?v=4",
+      "profile": "https://github.com/Rory-Powell",
+      "contributions": [
+        "code",
+        "doc",
+        "test"
+      ]
+    },
+    {
+      "login": "PClmnt",
+      "name": "Peter Clement",
+      "avatar_url": "https://avatars.githubusercontent.com/u/5665926?v=4",
+      "profile": "https://github.com/PClmnt",
+      "contributions": [
+        "code",
+        "doc",
+        "test"
+      ]
+    },
     {
       "login": "Conor-Mack",
       "name": "Conor_Mack",

.github/AUTHORS.md

@@ -7,3 +7,5 @@ Contributors
 * Martin McKeaveney - [@shogunpurple](https://github.com/shogunpurple)
 * Andrew Kingston - [@aptkingston](https://github.com/aptkingston)
 * Michael Drury - [@mike12345567](https://github.com/mike12345567)
+* Peter Clement - [@PClmnt](https://github.com/PClmnt)
+* Rory Powell - [@Rory-Powell](https://github.com/Rory-Powell)

.github/CONTRIBUTING.md

@@ -126,7 +126,16 @@ To run the budibase server and builder in dev mode (i.e. with live reloading):
 
 This will enable watch mode for both the builder app, server, client library and any component libraries.
 
-### 5. Cleanup
+### 5. Debugging using VS Code
+
+To debug the budibase server and worker a VS Code launch configuration has been provided. 
+
+Visit the debug window and select `Budibase Server` or `Budibase Worker` to debug the respective component. 
+Alternatively to start both components simultaneously select `Start Budibase`.
+
+In addition to the above, the remaining budibase components may be ran in dev mode using: `yarn dev:noserver`.
+
+### 6. Cleanup
 
 If you wish to delete all the apps created in development and reset the environment then run the following:
 

.vscode/launch.json

@@ -21,6 +21,27 @@
             "args": [],
             "cwd":"C:/code/my-apps",
             "console": "externalTerminal" 
+        },
+        {
+            "name": "Budibase Server",
+            "type": "node",
+            "request": "launch",
+            "runtimeArgs": ["--nolazy", "-r", "ts-node/register/transpile-only"],
+            "args": ["${workspaceFolder}/packages/server/src/index.ts"],
+            "cwd": "${workspaceFolder}/packages/server"
+          },
+          {
+            "name": "Budibase Worker",
+            "type": "node",
+            "request": "launch",
+            "program": "${workspaceFolder}/packages/worker/src/index.js",
+            "cwd": "${workspaceFolder}/packages/worker"
+          }
+    ],
+    "compounds": [
+        {
+          "name": "Start Budibase",
+          "configurations": ["Budibase Server", "Budibase Worker"]
         }
     ]
 }

README.md

@@ -211,9 +211,11 @@ Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/d
     <td align="center"><a href="https://budibase.com/"><img src="https://avatars3.githubusercontent.com/u/3524181?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Michael Shanks</b></sub></a><br /><a href="https://github.com/Budibase/budibase/commits?author=mjashanks" title="Documentation">📖</a> <a href="https://github.com/Budibase/budibase/commits?author=mjashanks" title="Code">💻</a> <a href="https://github.com/Budibase/budibase/commits?author=mjashanks" title="Tests">⚠️</a></td>
     <td align="center"><a href="https://github.com/kevmodrome"><img src="https://avatars3.githubusercontent.com/u/534488?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Kevin Åberg Kultalahti</b></sub></a><br /><a href="https://github.com/Budibase/budibase/commits?author=kevmodrome" title="Documentation">📖</a> <a href="https://github.com/Budibase/budibase/commits?author=kevmodrome" title="Code">💻</a> <a href="https://github.com/Budibase/budibase/commits?author=kevmodrome" title="Tests">⚠️</a></td>
     <td align="center"><a href="https://www.budibase.com/"><img src="https://avatars2.githubusercontent.com/u/49767913?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Joe</b></sub></a><br /><a href="https://github.com/Budibase/budibase/commits?author=joebudi" title="Documentation">📖</a> <a href="https://github.com/Budibase/budibase/commits?author=joebudi" title="Code">💻</a> <a href="#content-joebudi" title="Content">🖋</a> <a href="#design-joebudi" title="Design">🎨</a></td>
-    <td align="center"><a href="https://github.com/Conor-Mack"><img src="https://avatars1.githubusercontent.com/u/36074859?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Conor_Mack</b></sub></a><br /><a href="https://github.com/Budibase/budibase/commits?author=Conor-Mack" title="Code">💻</a> <a href="https://github.com/Budibase/budibase/commits?author=Conor-Mack" title="Tests">⚠️</a></td>
+    <td align="center"><a href="https://github.com/Rory-Powell"><img src="https://avatars.githubusercontent.com/u/8755148?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Rory Powell</b></sub></a><br /><a href="https://github.com/Budibase/budibase/commits?author=Rory-Powell" title="Code">💻</a> <a href="https://github.com/Budibase/budibase/commits?author=Rory-Powell" title="Documentation">📖</a> <a href="https://github.com/Budibase/budibase/commits?author=Rory-Powell" title="Tests">⚠️</a></td>
   </tr>
   <tr>
+    <td align="center"><a href="https://github.com/PClmnt"><img src="https://avatars.githubusercontent.com/u/5665926?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Peter Clement</b></sub></a><br /><a href="https://github.com/Budibase/budibase/commits?author=PClmnt" title="Code">💻</a> <a href="https://github.com/Budibase/budibase/commits?author=PClmnt" title="Documentation">📖</a> <a href="https://github.com/Budibase/budibase/commits?author=PClmnt" title="Tests">⚠️</a></td>
+    <td align="center"><a href="https://github.com/Conor-Mack"><img src="https://avatars1.githubusercontent.com/u/36074859?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Conor_Mack</b></sub></a><br /><a href="https://github.com/Budibase/budibase/commits?author=Conor-Mack" title="Code">💻</a> <a href="https://github.com/Budibase/budibase/commits?author=Conor-Mack" title="Tests">⚠️</a></td>
     <td align="center"><a href="https://github.com/pngwn"><img src="https://avatars1.githubusercontent.com/u/12937446?v=4?s=100" width="100px;" alt=""/><br /><sub><b>pngwn</b></sub></a><br /><a href="https://github.com/Budibase/budibase/commits?author=pngwn" title="Code">💻</a> <a href="https://github.com/Budibase/budibase/commits?author=pngwn" title="Tests">⚠️</a></td>
     <td align="center"><a href="https://github.com/HugoLd"><img src="https://avatars0.githubusercontent.com/u/26521848?v=4?s=100" width="100px;" alt=""/><br /><sub><b>HugoLd</b></sub></a><br /><a href="https://github.com/Budibase/budibase/commits?author=HugoLd" title="Code">💻</a></td>
     <td align="center"><a href="https://github.com/victoriasloan"><img src="https://avatars.githubusercontent.com/u/9913651?v=4?s=100" width="100px;" alt=""/><br /><sub><b>victoriasloan</b></sub></a><br /><a href="https://github.com/Budibase/budibase/commits?author=victoriasloan" title="Code">💻</a></td>

packages/auth/cache.js

@@ -0,0 +1,3 @@
+module.exports = {
+  user: require("./src/cache/user"),
+}

packages/auth/sessions.js

@@ -0,0 +1 @@
+module.exports = require("./src/security/sessions")

packages/auth/src/cache/user.js

@@ -0,0 +1,21 @@
+const { getDB } = require("../db")
+const { StaticDatabases } = require("../db/utils")
+const redis = require("../redis/authRedis")
+
+const EXPIRY_SECONDS = 3600
+
+exports.getUser = async userId => {
+  const client = await redis.getUserClient()
+  // try cache
+  let user = await client.get(userId)
+  if (!user) {
+    user = await getDB(StaticDatabases.GLOBAL.name).get(userId)
+    client.store(userId, user, EXPIRY_SECONDS)
+  }
+  return user
+}
+
+exports.invalidateUser = async userId => {
+  const client = await redis.getUserClient()
+  await client.delete(userId)
+}

packages/auth/src/index.js

@@ -4,6 +4,7 @@ const JwtStrategy = require("passport-jwt").Strategy
 const { StaticDatabases } = require("./db/utils")
 const { jwt, local, authenticated, google, auditLog } = require("./middleware")
 const { setDB, getDB } = require("./db")
+const userCache = require("./cache/user")
 
 // Strategies
 passport.use(new LocalStrategy(local.options, local.authenticate))
@@ -47,6 +48,9 @@ module.exports = {
     jwt: require("jsonwebtoken"),
     auditLog,
   },
+  cache: {
+    user: userCache,
+  },
   StaticDatabases,
   constants: require("./constants"),
 }

packages/auth/src/middleware/authenticated.js

@@ -1,7 +1,7 @@
 const { Cookies } = require("../constants")
-const database = require("../db")
 const { getCookie, clearCookie } = require("../utils")
-const { StaticDatabases } = require("../db/utils")
+const { getUser } = require("../cache/user")
+const { getSession, updateSessionTTL } = require("../security/sessions")
 const env = require("../environment")
 
 const PARAM_REGEX = /\/:(.*?)\//g
@@ -48,14 +48,27 @@ module.exports = (noAuthPatterns = [], opts) => {
         user = null,
         internal = false
       if (authCookie) {
+        let error = null
+        const sessionId = authCookie.sessionId,
+          userId = authCookie.userId
+        const session = await getSession(userId, sessionId)
+        if (!session) {
+          error = "No session found"
+        } else {
           try {
-          const db = database.getDB(StaticDatabases.GLOBAL.name)
+            user = await getUser(userId)
-          user = await db.get(authCookie.userId)
             delete user.password
             authenticated = true
           } catch (err) {
-          // remove the cookie as the use does not exist anymore
+            error = err
+          }
+        }
+        if (error) {
+          // remove the cookie as the user does not exist anymore
           clearCookie(ctx, Cookies.Auth)
+        } else {
+          // make sure we denote that the session is still in use
+          await updateSessionTTL(session)
         }
       }
       const apiKey = ctx.request.headers["x-budibase-api-key"]

packages/auth/src/middleware/passport/google.js

@@ -7,6 +7,8 @@ const {
   generateGlobalUserID,
   ViewNames,
 } = require("../../db/utils")
+const { newid } = require("../../hashing")
+const { createASession } = require("../../security/sessions")
 
 async function authenticate(token, tokenSecret, profile, done) {
   // Check the user exists in the instance DB by email
@@ -59,15 +61,16 @@ async function authenticate(token, tokenSecret, profile, done) {
   }
 
   // authenticate
-  const payload = {
+  const sessionId = newid()
-    userId: dbUser._id,
+  await createASession(dbUser._id, sessionId)
-    builder: dbUser.builder,
-    email: dbUser.email,
-  }
 
-  dbUser.token = jwt.sign(payload, env.JWT_SECRET, {
+  dbUser.token = jwt.sign(
-    expiresIn: "1 day",
+    {
-  })
+      userId: dbUser._id,
+      sessionId,
+    },
+    env.JWT_SECRET
+  )
 
   return done(null, dbUser)
 }

packages/auth/src/middleware/passport/local.js

@@ -3,6 +3,8 @@ const { UserStatus } = require("../../constants")
 const { compare } = require("../../hashing")
 const env = require("../../environment")
 const { getGlobalUserByEmail } = require("../../utils")
+const { newid } = require("../../hashing")
+const { createASession } = require("../../security/sessions")
 
 const INVALID_ERR = "Invalid Credentials"
 
@@ -31,13 +33,16 @@ exports.authenticate = async function (email, password, done) {
 
   // authenticate
   if (await compare(password, dbUser.password)) {
-    const payload = {
+    const sessionId = newid()
-      userId: dbUser._id,
+    await createASession(dbUser._id, sessionId)
-    }
 
-    dbUser.token = jwt.sign(payload, env.JWT_SECRET, {
+    dbUser.token = jwt.sign(
-      expiresIn: "1 day",
+      {
-    })
+        userId: dbUser._id,
+        sessionId,
+      },
+      env.JWT_SECRET
+    )
     // Remove users password in payload
     delete dbUser.password
 

packages/auth/src/objectStore/index.js

@@ -22,11 +22,13 @@ const CONTENT_TYPE_MAP = {
   html: "text/html",
   css: "text/css",
   js: "application/javascript",
+  json: "application/json",
 }
 const STRING_CONTENT_TYPES = [
   CONTENT_TYPE_MAP.html,
   CONTENT_TYPE_MAP.css,
   CONTENT_TYPE_MAP.js,
+  CONTENT_TYPE_MAP.json,
 ]
 
 // does normal sanitization and then swaps dev apps to apps

packages/auth/src/redis/authRedis.js

@@ -0,0 +1,29 @@
+const Client = require("./index")
+const utils = require("./utils")
+
+let userClient, sessionClient
+
+async function init() {
+  userClient = await new Client(utils.Databases.USER_CACHE).init()
+  sessionClient = await new Client(utils.Databases.SESSIONS).init()
+}
+
+process.on("exit", async () => {
+  if (userClient) await userClient.finish()
+  if (sessionClient) await sessionClient.finish()
+})
+
+module.exports = {
+  getUserClient: async () => {
+    if (!userClient) {
+      await init()
+    }
+    return userClient
+  },
+  getSessionClient: async () => {
+    if (!sessionClient) {
+      await init()
+    }
+    return sessionClient
+  },
+}

packages/auth/src/redis/index.js

@@ -1,7 +1,12 @@
 const env = require("../environment")
 // ioredis mock is all in memory
 const Redis = env.isTest() ? require("ioredis-mock") : require("ioredis")
-const { addDbPrefix, removeDbPrefix, getRedisOptions } = require("./utils")
+const {
+  addDbPrefix,
+  removeDbPrefix,
+  getRedisOptions,
+  SEPARATOR,
+} = require("./utils")
 
 const RETRY_PERIOD_MS = 2000
 const STARTUP_TIMEOUT_MS = 5000
@@ -143,14 +148,15 @@ class RedisWrapper {
     CLIENT.disconnect()
   }
 
-  async scan() {
+  async scan(key = "") {
     const db = this._db
+    key = `${db}${SEPARATOR}${key}`
     let stream
     if (CLUSTERED) {
       let node = CLIENT.nodes("master")
-      stream = node[0].scanStream({ match: db + "-*", count: 100 })
+      stream = node[0].scanStream({ match: key + "*", count: 100 })
     } else {
-      stream = CLIENT.scanStream({ match: db + "-*", count: 100 })
+      stream = CLIENT.scanStream({ match: key + "*", count: 100 })
     }
     return promisifyStream(stream)
   }
@@ -182,6 +188,12 @@ class RedisWrapper {
     }
   }
 
+  async setExpiry(key, expirySeconds) {
+    const db = this._db
+    const prefixedKey = addDbPrefix(db, key)
+    await CLIENT.expire(prefixedKey, expirySeconds)
+  }
+
   async delete(key) {
     const db = this._db
     await CLIENT.del(addDbPrefix(db, key))

packages/auth/src/redis/utils.js

@@ -11,8 +11,12 @@ exports.Databases = {
   INVITATIONS: "invitation",
   DEV_LOCKS: "devLocks",
   DEBOUNCE: "debounce",
+  SESSIONS: "session",
+  USER_CACHE: "users",
 }
 
+exports.SEPARATOR = SEPARATOR
+
 exports.getRedisOptions = (clustered = false) => {
   const [host, port] = REDIS_URL.split(":")
   const opts = {

packages/auth/src/security/sessions.js

@@ -0,0 +1,69 @@
+const redis = require("../redis/authRedis")
+
+const EXPIRY_SECONDS = 86400
+
+async function getSessionsForUser(userId) {
+  const client = await redis.getSessionClient()
+  const sessions = await client.scan(userId)
+  return sessions.map(session => session.value)
+}
+
+function makeSessionID(userId, sessionId) {
+  return `${userId}/${sessionId}`
+}
+
+exports.createASession = async (userId, sessionId) => {
+  const client = await redis.getSessionClient()
+  const session = {
+    createdAt: new Date().toISOString(),
+    lastAccessedAt: new Date().toISOString(),
+    sessionId,
+    userId,
+  }
+  await client.store(makeSessionID(userId, sessionId), session, EXPIRY_SECONDS)
+}
+
+exports.invalidateSessions = async (userId, sessionId = null) => {
+  let sessions = []
+  if (sessionId) {
+    sessions.push({ key: makeSessionID(userId, sessionId) })
+  } else {
+    sessions = await getSessionsForUser(userId)
+  }
+  const client = await redis.getSessionClient()
+  const promises = []
+  for (let session of sessions) {
+    promises.push(client.delete(session.key))
+  }
+  await Promise.all(promises)
+}
+
+exports.updateSessionTTL = async session => {
+  const client = await redis.getSessionClient()
+  const key = makeSessionID(session.userId, session.sessionId)
+  session.lastAccessedAt = new Date().toISOString()
+  await client.store(key, session, EXPIRY_SECONDS)
+}
+
+exports.endSession = async (userId, sessionId) => {
+  const client = await redis.getSessionClient()
+  await client.delete(makeSessionID(userId, sessionId))
+}
+
+exports.getUserSessions = getSessionsForUser
+
+exports.getSession = async (userId, sessionId) => {
+  try {
+    const client = await redis.getSessionClient()
+    return client.get(makeSessionID(userId, sessionId))
+  } catch (err) {
+    // if can't get session don't error, just don't return anything
+    return null
+  }
+}
+
+exports.getAllSessions = async () => {
+  const client = await redis.getSessionClient()
+  const sessions = await client.scan()
+  return sessions.map(session => session.value)
+}

packages/auth/src/utils.js

@@ -64,23 +64,18 @@ exports.getCookie = (ctx, name) => {
 }
 
 /**
- * Store a cookie for the request, has a hardcoded expiry.
+ * Store a cookie for the request - it will not expire.
  * @param {object} ctx The request which is to be manipulated.
  * @param {string} name The name of the cookie to set.
  * @param {string|object} value The value of cookie which will be set.
  */
 exports.setCookie = (ctx, value, name = "builder") => {
-  const expires = new Date()
-  expires.setDate(expires.getDate() + 1)
-
   if (!value) {
     ctx.cookies.set(name)
   } else {
-    value = jwt.sign(value, options.secretOrKey, {
+    value = jwt.sign(value, options.secretOrKey)
-      expiresIn: "1 day",
-    })
     ctx.cookies.set(name, value, {
-      expires,
+      maxAge: Number.MAX_SAFE_INTEGER,
       path: "/",
       httpOnly: false,
       overwrite: true,

packages/bbui/src/ColorPicker/ColorPicker.svelte

@@ -9,10 +9,10 @@
 
   export let value
   export let size = "M"
+  export let spectrumTheme
 
   let open = false
 
-  $: color = value || "transparent"
   $: customValue = getCustomValue(value)
   $: checkColor = getCheckColor(value)
 
@@ -21,7 +21,8 @@
     {
       label: "Grays",
       colors: [
-        "white",
+        "gray-50",
+        "gray-75",
         "gray-100",
         "gray-200",
         "gray-300",
@@ -31,7 +32,6 @@
         "gray-700",
         "gray-800",
         "gray-900",
-        "black",
       ],
     },
     {
@@ -86,7 +86,7 @@
       return value
     }
     let found = false
-    const comparisonValue = value.substring(35, value.length - 1)
+    const comparisonValue = value.substring(28, value.length - 1)
     for (let category of categories) {
       found = category.colors.includes(comparisonValue)
       if (found) {
@@ -102,17 +102,19 @@
 
   const getCheckColor = value => {
     return /^.*(white|(gray-(50|75|100|200|300|400|500)))\)$/.test(value)
-      ? "black"
+      ? "var(--spectrum-global-color-gray-900)"
-      : "white"
+      : "var(--spectrum-global-color-gray-50)"
   }
 </script>
 
 <div class="container">
+  <div class="preview size--{size || 'M'}" on:click={() => (open = true)}>
     <div
-    class="preview size--{size || 'M'}"
+      class="fill {spectrumTheme || ''}"
-    style="background: {color};"
+      style={value ? `background: ${value};` : ""}
-    on:click={() => (open = true)}
+      class:placeholder={!value}
     />
+  </div>
   {#if open}
     <div
       use:clickOutside={() => (open = false)}
@@ -126,16 +128,20 @@
             {#each category.colors as color}
               <div
                 on:click={() => {
-                  onChange(`var(--spectrum-global-color-static-${color})`)
+                  onChange(`var(--spectrum-global-color-${color})`)
                 }}
                 class="color"
-                style="background: var(--spectrum-global-color-static-{color}); color: {checkColor};"
                 title={prettyPrint(color)}
               >
-                {#if value === `var(--spectrum-global-color-static-${color})`}
+                <div
+                  class="fill {spectrumTheme || ''}"
+                  style="background: var(--spectrum-global-color-{color}); color: {checkColor};"
+                >
+                  {#if value === `var(--spectrum-global-color-${color})`}
                     <Icon name="Checkmark" size="S" />
                   {/if}
                 </div>
+              </div>
             {/each}
           </div>
         </div>
@@ -170,12 +176,43 @@
     width: 32px;
     height: 32px;
     border-radius: 100%;
+    position: relative;
     transition: border-color 130ms ease-in-out;
-    box-shadow: 0 0 0 1px var(--spectrum-global-color-gray-300);
+    box-shadow: 0 0 0 1px var(--spectrum-global-color-gray-400);
   }
   .preview:hover {
     cursor: pointer;
-    box-shadow: 0 0 2px 2px var(--spectrum-global-color-gray-300);
+    box-shadow: 0 0 2px 2px var(--spectrum-global-color-gray-400);
+  }
+  .fill {
+    width: 100%;
+    height: 100%;
+    border-radius: 100%;
+    position: absolute;
+    top: 0;
+    left: 0;
+    display: grid;
+    place-items: center;
+  }
+  .fill.placeholder {
+    background-position: 0 0, 10px 10px;
+    background-size: 20px 20px;
+    background-image: linear-gradient(
+        45deg,
+        #eee 25%,
+        transparent 25%,
+        transparent 75%,
+        #eee 75%,
+        #eee 100%
+      ),
+      linear-gradient(
+        45deg,
+        #eee 25%,
+        white 25%,
+        white 75%,
+        #eee 75%,
+        #eee 100%
+      );
   }
   .size--S {
     width: 20px;
@@ -219,8 +256,7 @@
     width: 16px;
     border-radius: 100%;
     box-shadow: 0 0 0 1px var(--spectrum-global-color-gray-300);
-    display: grid;
+    position: relative;
-    place-items: center;
   }
   .color:hover {
     cursor: pointer;
@@ -236,4 +272,8 @@
   .category--custom .heading {
     margin-bottom: var(--spacing-xs);
   }
+
+  .spectrum-wrapper {
+    background-color: transparent;
+  }
 </style>

packages/bbui/src/Form/Core/DatePicker.svelte

@@ -145,4 +145,7 @@
     height: 100vh;
     z-index: 999;
   }
+  :global(.flatpickr-calendar) {
+    font-family: "Source Sans Pro", sans-serif;
+  }
 </style>

packages/builder/cypress/integration/createBinding.spec.js

@@ -29,7 +29,7 @@ context("Create Bindings", () => {
       // The builder preview pages don't have a real URL, so all we can do
       // is check that we were able to bind to the property, and that the
       // component exists on the page
-      cy.getComponent(componentId).should("have.text", "Placeholder text")
+      cy.getComponent(componentId).should("have.text", "New Paragraph")
     })
   })
 

packages/builder/src/builderStore/store/frontend.js

@@ -8,7 +8,6 @@ import {
   selectedComponent,
   selectedAccessRole,
 } from "builderStore"
-// Backendstores
 import {
   datasources,
   integrations,
@@ -33,6 +32,10 @@ const INITIAL_FRONTEND_STATE = {
   layouts: [],
   screens: [],
   components: [],
+  clientFeatures: {
+    spectrumThemes: false,
+    intelligentLoading: false,
+  },
   currentFrontEndType: "none",
   selectedScreenId: "",
   selectedLayoutId: "",
@@ -43,6 +46,7 @@ const INITIAL_FRONTEND_STATE = {
   appId: "",
   routes: {},
   clientLibPath: "",
+  theme: "",
 }
 
 export const getFrontendStore = () => {
@@ -56,16 +60,23 @@ export const getFrontendStore = () => {
         ...state,
         libraries: application.componentLibraries,
         components,
+        clientFeatures: {
+          ...state.clientFeatures,
+          ...components.features,
+        },
         name: application.name,
         description: application.description,
         appId: application.appId,
         url: application.url,
         layouts,
         screens,
+        theme: application.theme,
         hasAppPackage: true,
         appInstance: application.instance,
         clientLibPath,
         previousTopNavPath: {},
+        version: application.version,
+        revertableVersion: application.revertableVersion,
       }))
       await hostingStore.actions.fetch()
 
@@ -79,6 +90,20 @@ export const getFrontendStore = () => {
       database.set(application.instance)
       tables.init()
     },
+    theme: {
+      save: async theme => {
+        const appId = get(store).appId
+        const response = await api.put(`/api/applications/${appId}`, { theme })
+        if (response.status === 200) {
+          store.update(state => {
+            state.theme = theme
+            return state
+          })
+        } else {
+          throw new Error("Error updating theme")
+        }
+      },
+    },
     routing: {
       fetch: async () => {
         const response = await api.get("/api/routing")
@@ -199,6 +224,11 @@ export const getFrontendStore = () => {
         const response = await api.post(`/api/layouts`, layoutToSave)
         const savedLayout = await response.json()
 
+        // Abort if saving failed
+        if (response.status !== 200) {
+          return
+        }
+
         store.update(state => {
           const layoutIdx = state.layouts.findIndex(
             stateLayout => stateLayout._id === savedLayout._id
@@ -316,16 +346,6 @@ export const getFrontendStore = () => {
       create: async (componentName, presetProps) => {
         const selected = get(selectedComponent)
         const asset = get(currentAsset)
-        const state = get(store)
-
-        // Only allow one screen slot, and in the layout
-        if (componentName.endsWith("screenslot")) {
-          const isLayout = state.currentFrontEndType === FrontendTypes.LAYOUT
-          const slot = findComponentType(asset.props, componentName)
-          if (!isLayout || slot != null) {
-            return
-          }
-        }
 
         // Create new component
         const componentInstance = store.actions.components.createInstance(

packages/builder/src/builderStore/store/screenTemplates/newRowScreen.js

@@ -38,8 +38,6 @@ const createScreen = table => {
     .instanceName("Form")
     .customProps({
       actionType: "Create",
-      theme: "spectrum--lightest",
-      size: "spectrum--medium",
       dataSource: {
         label: table.name,
         tableId: table._id,

packages/builder/src/builderStore/store/screenTemplates/rowDetailScreen.js

@@ -8,7 +8,6 @@ import {
   makeTitleContainer,
   makeSaveButton,
   makeMainForm,
-  spectrumColor,
   makeDatasourceFormComponents,
 } from "./utils/commonComponents"
 
@@ -26,36 +25,13 @@ export const ROW_DETAIL_TEMPLATE = "ROW_DETAIL_TEMPLATE"
 export const rowDetailUrl = table => sanitizeUrl(`/${table.name}/:id`)
 
 function generateTitleContainer(table, title, formId, repeaterId) {
-  // have to override style for this, its missing margin
+  const saveButton = makeSaveButton(table, formId)
-  const saveButton = makeSaveButton(table, formId).normalStyle({
-    background: "#000000",
-    "border-width": "0",
-    "border-style": "None",
-    color: "#fff",
-    "font-weight": "600",
-    "font-size": "14px",
-  })
-
   const deleteButton = new Component("@budibase/standard-components/button")
-    .normalStyle({
-      background: "transparent",
-      "border-width": "0",
-      "border-style": "None",
-      color: "#9e9e9e",
-      "font-weight": "600",
-      "font-size": "14px",
-      "margin-right": "8px",
-      "margin-left": "16px",
-    })
-    .hoverStyle({
-      background: "transparent",
-      color: "#4285f4",
-    })
-    .customStyle(spectrumColor(700))
     .text("Delete")
     .customProps({
-      className: "",
+      type: "secondary",
-      disabled: false,
+      quiet: true,
+      size: "M",
       onClick: [
         {
           parameters: {
@@ -76,7 +52,19 @@ function generateTitleContainer(table, title, formId, repeaterId) {
     })
     .instanceName("Delete Button")
 
-  return makeTitleContainer(title).addChild(deleteButton).addChild(saveButton)
+  const buttons = new Component("@budibase/standard-components/container")
+    .instanceName("Button Container")
+    .customProps({
+      direction: "row",
+      hAlign: "right",
+      vAlign: "middle",
+      size: "shrink",
+      gap: "M",
+    })
+    .addChild(deleteButton)
+    .addChild(saveButton)
+
+  return makeTitleContainer(title).addChild(buttons)
 }
 
 const createScreen = table => {
@@ -98,7 +86,7 @@ const createScreen = table => {
           valueType: "Binding",
         },
       ],
-      limit: 1,
+      limit: table.type === "external" ? undefined : 1,
       paginate: false,
     })
 

packages/builder/src/builderStore/store/screenTemplates/rowListScreen.js

@@ -19,21 +19,10 @@ export const rowListUrl = table => sanitizeUrl(`/${table.name}`)
 
 function generateTitleContainer(table) {
   const newButton = new Component("@budibase/standard-components/button")
-    .normalStyle({
-      background: "#000000",
-      "border-width": "0",
-      "border-style": "None",
-      color: "#fff",
-      "font-weight": "600",
-      "font-size": "14px",
-    })
-    .hoverStyle({
-      background: "#4285f4",
-    })
     .text("Create New")
     .customProps({
-      className: "",
+      size: "M",
-      disabled: false,
+      type: "primary",
       onClick: [
         {
           parameters: {
@@ -46,12 +35,6 @@ function generateTitleContainer(table) {
     .instanceName("New Button")
 
   const heading = new Component("@budibase/standard-components/heading")
-    .normalStyle({
-      margin: "0px",
-      flex: "1 1 auto",
-      "text-transform": "capitalize",
-    })
-    .type("h2")
     .instanceName("Title")
     .text(table.name)
     .customProps({
@@ -60,14 +43,12 @@ function generateTitleContainer(table) {
     })
 
   return new Component("@budibase/standard-components/container")
-    .normalStyle({
-      "margin-bottom": "32px",
-    })
     .customProps({
       direction: "row",
       hAlign: "stretch",
       vAlign: "middle",
       size: "shrink",
+      gap: "M",
     })
     .instanceName("Title Container")
     .addChild(heading)
@@ -91,68 +72,35 @@ const createScreen = table => {
   const spectrumTable = new Component("@budibase/standard-components/table")
     .customProps({
       dataProvider: `{{ literal ${makePropSafe(provider._json._id)} }}`,
-      theme: "spectrum--lightest",
       showAutoColumns: false,
-      quiet: true,
+      quiet: false,
-      size: "spectrum--medium",
       rowCount: 8,
     })
     .instanceName(`${table.name} Table`)
 
   const safeTableId = makePropSafe(spectrumTable._json._id)
   const safeRowId = makePropSafe("_id")
-  const viewButton = new Component("@budibase/standard-components/button")
+  const viewLink = new Component("@budibase/standard-components/link")
     .customProps({
       text: "View",
-      onClick: [
-        {
-          "##eventHandlerType": "Navigate To",
-          parameters: {
       url: `${rowListUrl(table)}/{{ ${safeTableId}.${safeRowId} }}`,
-          },
+      size: "S",
-        },
+      color: "var(--spectrum-global-color-gray-600)",
-      ],
+      align: "left",
     })
-    .instanceName("View Button")
     .normalStyle({
-      background: "transparent",
+      ["margin-left"]: "16px",
-      "font-weight": "600",
+      ["margin-right"]: "16px",
-      color: "#888",
-      "border-width": "0",
-    })
-    .hoverStyle({
-      color: "#4285f4",
     })
+    .instanceName("View Link")
 
-  spectrumTable.addChild(viewButton)
+  spectrumTable.addChild(viewLink)
   provider.addChild(spectrumTable)
 
-  const mainContainer = new Component("@budibase/standard-components/container")
-    .normalStyle({
-      background: "white",
-      "border-radius": "0.5rem",
-      "box-shadow": "0 1px 2px 0 rgba(0, 0, 0, 0.05)",
-      "border-width": "2px",
-      "border-color": "rgba(0, 0, 0, 0.1)",
-      "border-style": "None",
-      "padding-top": "48px",
-      "padding-bottom": "48px",
-      "padding-right": "48px",
-      "padding-left": "48px",
-    })
-    .customProps({
-      direction: "column",
-      hAlign: "stretch",
-      vAlign: "top",
-      size: "shrink",
-    })
-    .instanceName("Container")
-    .addChild(generateTitleContainer(table))
-    .addChild(provider)
-
   return new Screen()
     .route(rowListUrl(table))
     .instanceName(`${table.name} - List`)
-    .addChild(mainContainer)
+    .addChild(generateTitleContainer(table))
+    .addChild(provider)
     .json()
 }

packages/builder/src/builderStore/store/screenTemplates/utils/commonComponents.js

@@ -8,23 +8,16 @@ export function spectrumColor(number) {
   // God knows why. It seems to think optional chaining further down the
   // file is invalid if the word g-l-o-b-a-l is found - hence the reason this
   // statement is split into parts.
-  return "color: var(--spectrum-glo" + `bal-color-gray-${number});`
+  return "var(--spectrum-glo" + `bal-color-gray-${number})`
 }
 
 export function makeLinkComponent(tableName) {
   return new Component("@budibase/standard-components/link")
-    .normalStyle({
-      color: "#757575",
-      "text-transform": "capitalize",
-    })
-    .hoverStyle({
-      color: "#4285f4",
-    })
-    .customStyle(spectrumColor(700))
     .text(tableName)
     .customProps({
       url: `/${tableName.toLowerCase()}`,
       openInNewTab: false,
+      color: spectrumColor(700),
       size: "S",
       align: "left",
     })
@@ -33,19 +26,12 @@ export function makeLinkComponent(tableName) {
 export function makeMainForm() {
   return new Component("@budibase/standard-components/form")
     .normalStyle({
-      width: "700px",
+      width: "600px",
-      padding: "0px",
-      "border-radius": "0.5rem",
-      "box-shadow": "0 1px 2px 0 rgba(0, 0, 0, 0.05)",
-      "padding-top": "48px",
-      "padding-bottom": "48px",
-      "padding-right": "48px",
-      "padding-left": "48px",
     })
     .instanceName("Form")
 }
 
-export function makeBreadcrumbContainer(tableName, text, capitalise = false) {
+export function makeBreadcrumbContainer(tableName, text) {
   const link = makeLinkComponent(tableName).instanceName("Back Link")
 
   const arrowText = new Component("@budibase/standard-components/text")
@@ -53,42 +39,27 @@ export function makeBreadcrumbContainer(tableName, text, capitalise = false) {
     .normalStyle({
       "margin-right": "4px",
       "margin-left": "4px",
-      "margin-top": "0px",
-      "margin-bottom": "0px",
     })
-    .customStyle(spectrumColor(700))
     .text(">")
     .instanceName("Arrow")
     .customProps({
+      color: spectrumColor(700),
       size: "S",
       align: "left",
     })
 
-  const textStyling = {
-    color: "#000000",
-    "margin-top": "0px",
-    "margin-bottom": "0px",
-  }
-  if (capitalise) {
-    textStyling["text-transform"] = "capitalize"
-  }
   const identifierText = new Component("@budibase/standard-components/text")
-    .type("none")
-    .normalStyle(textStyling)
-    .customStyle(spectrumColor(700))
     .text(text)
     .instanceName("Identifier")
     .customProps({
+      color: spectrumColor(700),
       size: "S",
       align: "left",
     })
 
   return new Component("@budibase/standard-components/container")
-    .normalStyle({
-      "font-size": "14px",
-      color: "#757575",
-    })
     .customProps({
+      gap: "N",
       direction: "row",
       hAlign: "left",
       vAlign: "middle",
@@ -102,22 +73,10 @@ export function makeBreadcrumbContainer(tableName, text, capitalise = false) {
 
 export function makeSaveButton(table, formId) {
   return new Component("@budibase/standard-components/button")
-    .normalStyle({
-      background: "#000000",
-      "border-width": "0",
-      "border-style": "None",
-      color: "#fff",
-      "font-weight": "600",
-      "font-size": "14px",
-      "margin-left": "16px",
-    })
-    .hoverStyle({
-      background: "#4285f4",
-    })
     .text("Save")
     .customProps({
-      className: "",
+      type: "primary",
-      disabled: false,
+      size: "M",
       onClick: [
         {
           "##eventHandlerType": "Validate Form",
@@ -145,12 +104,6 @@ export function makeSaveButton(table, formId) {
 
 export function makeTitleContainer(title) {
   const heading = new Component("@budibase/standard-components/heading")
-    .normalStyle({
-      margin: "0px",
-      flex: "1 1 auto",
-    })
-    .customStyle(spectrumColor(900))
-    .type("h2")
     .instanceName("Title")
     .text(title)
     .customProps({
@@ -168,6 +121,7 @@ export function makeTitleContainer(title) {
       hAlign: "stretch",
       vAlign: "middle",
       size: "shrink",
+      gap: "M",
     })
     .instanceName("Title Container")
     .addChild(heading)

packages/builder/src/components/backend/DataTable/DataTable.svelte

@@ -24,7 +24,7 @@
     name: $views.selected?.name,
   }
   $: type = $tables.selected?.type
-  $: isInternal = type === "internal"
+  $: isInternal = type !== "external"
 
   // Fetch rows for specified table
   $: {
@@ -72,9 +72,7 @@
     {#if isUsersTable}
       <EditRolesButton />
     {/if}
-    {#if isInternal}
     <HideAutocolumnButton bind:hideAutocolumns />
-    {/if}
     <!-- always have the export last -->
     <ExportButton view={$tables.selected?._id} />
   {/if}

packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte

@@ -53,7 +53,7 @@
   let deletion
 
   $: tableOptions = $tables.list.filter(
-    table => table._id !== $tables.draft._id
+    table => table._id !== $tables.draft._id && table.type !== "external"
   )
   $: required = !!field?.constraints?.presence || primaryDisplay
   $: uneditable =
@@ -172,11 +172,6 @@
         alt: `Many ${table.name} rows → many ${linkTable.name} rows`,
         value: RelationshipTypes.MANY_TO_MANY,
       },
-      {
-        name: `One ${linkName} row → many ${thisName} rows`,
-        alt: `One ${linkTable.name} rows → many ${table.name} rows`,
-        value: RelationshipTypes.ONE_TO_MANY,
-      },
       {
         name: `One ${thisName} row → many ${linkName} rows`,
         alt: `One ${table.name} rows → many ${linkTable.name} rows`,

packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/index.svelte

@@ -5,14 +5,17 @@
   import ICONS from "../icons"
 
   export let integration = {}
-
   let integrations = []
+  const INTERNAL = "BUDIBASE"
 
   async function fetchIntegrations() {
     const response = await api.get("/api/integrations")
     const json = await response.json()
 
-    integrations = json
+    integrations = {
+      [INTERNAL]: { datasource: {}, name: "INTERNAL/CSV" },
+      ...json,
+    }
     return json
   }
 
@@ -21,7 +24,7 @@
 
     // build the schema
     const schema = {}
-    for (let key in selected.datasource) {
+    for (let key of Object.keys(selected.datasource)) {
       schema[key] = selected.datasource[key].default
     }
 
@@ -39,7 +42,7 @@
 
 <section>
   <div class="integration-list">
-    {#each Object.keys(integrations) as integrationType}
+    {#each Object.entries(integrations) as [integrationType, schema]}
       <div
         class="integration hoverable"
         class:selected={integration.type === integrationType}
@@ -50,7 +53,7 @@
           height="50"
           width="50"
         />
-        <Body size="XS">{integrationType}</Body>
+        <Body size="XS">{schema.name || integrationType}</Body>
       </div>
     {/each}
   </div>

packages/builder/src/components/backend/DatasourceNavigator/modals/CreateDatasourceModal.svelte

@@ -2,15 +2,21 @@
   import { goto } from "@roxi/routify"
   import { datasources } from "stores/backend"
   import { notifications } from "@budibase/bbui"
-  import { Input, Label, ModalContent } from "@budibase/bbui"
+  import { Input, Label, ModalContent, Modal, Context } from "@budibase/bbui"
   import TableIntegrationMenu from "../TableIntegrationMenu/index.svelte"
+  import CreateTableModal from "components/backend/TableNavigator/modals/CreateTableModal.svelte"
   import analytics from "analytics"
+  import { getContext } from "svelte"
 
-  let error = ""
+  const modalContext = getContext(Context.Modal)
 
+  let tableModal
   let name
+  let error = ""
   let integration
 
+  $: checkOpenModal(integration && integration.type === "BUDIBASE")
+
   function checkValid(evt) {
     const datasourceName = evt.target.value
     if (
@@ -22,6 +28,12 @@
     error = ""
   }
 
+  function checkOpenModal(isInternal) {
+    if (isInternal) {
+      tableModal.show()
+    }
+  }
+
   async function saveDatasource() {
     const { type, plus, ...config } = integration
 
@@ -40,6 +52,9 @@
   }
 </script>
 
+<Modal bind:this={tableModal} on:hide={modalContext.hide}>
+  <CreateTableModal bind:name />
+</Modal>
 <ModalContent
   title="Create Datasource"
   size="L"

packages/builder/src/components/backend/TableNavigator/TableNavigator.svelte

@@ -6,9 +6,14 @@
   import EditViewPopover from "./popovers/EditViewPopover.svelte"
   import NavItem from "components/common/NavItem.svelte"
 
+  const alphabetical = (a, b) => a.name?.toLowerCase() > b.name?.toLowerCase()
+
   export let sourceId
 
   $: selectedView = $views.selected && $views.selected.name
+  $: sortedTables = $tables.list
+    .filter(table => table.sourceId === sourceId)
+    .sort(alphabetical)
 
   function selectTable(table) {
     tables.select(table)
@@ -33,7 +38,7 @@
 
 {#if $database?._id}
   <div class="hierarchy-items-container">
-    {#each $tables.list.filter(table => table.sourceId === sourceId) as table, idx}
+    {#each sortedTables as table, idx}
       <NavItem
         indentLevel={1}
         border={idx > 0}
@@ -46,7 +51,7 @@
           <EditTablePopover {table} />
         {/if}
       </NavItem>
-      {#each Object.keys(table.views || {}) as viewName, idx (idx)}
+      {#each [...Object.keys(table.views || {})].sort() as viewName, idx (idx)}
         <NavItem
           indentLevel={2}
           icon="Remove"

packages/builder/src/components/backend/TableNavigator/modals/CreateTableModal.svelte

@@ -1,5 +1,5 @@
 <script>
-  import { goto } from "@roxi/routify"
+  import { goto, url } from "@roxi/routify"
   import { store } from "builderStore"
   import { tables } from "stores/backend"
   import { notifications } from "@budibase/bbui"
@@ -27,7 +27,7 @@
 
   $: tableNames = $tables.list.map(table => table.name)
 
-  let name
+  export let name
   let dataImport
   let error = ""
   let createAutoscreens = true
@@ -91,7 +91,11 @@
     }
 
     // Navigate to new table
-    $goto(`../../table/${table._id}`)
+    const currentUrl = $url()
+    const path = currentUrl.endsWith("data")
+      ? `./table/${table._id}`
+      : `../../table/${table._id}`
+    $goto(path)
   }
 </script>
 

packages/builder/src/components/common/bindings/DrawerBindableCombobox.svelte

@@ -0,0 +1,97 @@
+<script>
+  import { Icon, Combobox, Drawer, Button } from "@budibase/bbui"
+  import {
+    readableToRuntimeBinding,
+    runtimeToReadableBinding,
+  } from "builderStore/dataBinding"
+  import BindingPanel from "components/common/bindings/BindingPanel.svelte"
+  import { createEventDispatcher } from "svelte"
+
+  export let panel = BindingPanel
+  export let value = ""
+  export let bindings = []
+  export let title = "Bindings"
+  export let placeholder
+  export let label
+  export let disabled = false
+  export let options
+
+  const dispatch = createEventDispatcher()
+  let bindingDrawer
+  $: tempValue = Array.isArray(value) ? value : []
+  $: readableValue = runtimeToReadableBinding(bindings, value)
+
+  const handleClose = () => {
+    onChange(tempValue)
+    bindingDrawer.hide()
+  }
+
+  const onChange = value => {
+    dispatch("change", readableToRuntimeBinding(bindings, value))
+  }
+</script>
+
+<div class="control">
+  <Combobox
+    {label}
+    {disabled}
+    value={readableValue}
+    on:change={event => onChange(event.detail)}
+    {placeholder}
+    {options}
+  />
+  {#if !disabled}
+    <div class="icon" on:click={bindingDrawer.show}>
+      <Icon size="S" name="FlashOn" />
+    </div>
+  {/if}
+</div>
+<Drawer bind:this={bindingDrawer} {title}>
+  <svelte:fragment slot="description">
+    Add the objects on the left to enrich your text.
+  </svelte:fragment>
+  <Button cta slot="buttons" on:click={handleClose}>Save</Button>
+  <svelte:component
+    this={panel}
+    slot="body"
+    value={readableValue}
+    close={handleClose}
+    on:update={event => (tempValue = event.detail)}
+    bindableProperties={bindings}
+  />
+</Drawer>
+
+<style>
+  .control {
+    flex: 1;
+    position: relative;
+  }
+
+  .icon {
+    right: 31px;
+    bottom: 1px;
+    position: absolute;
+    justify-content: center;
+    align-items: center;
+    display: flex;
+    flex-direction: row;
+    box-sizing: border-box;
+    border-left: 1px solid var(--spectrum-alias-border-color);
+    border-right: 1px solid var(--spectrum-alias-border-color);
+    width: 31px;
+    color: var(--spectrum-alias-text-color);
+    background-color: var(--spectrum-global-color-gray-75);
+    transition: background-color
+        var(--spectrum-global-animation-duration-100, 130ms),
+      box-shadow var(--spectrum-global-animation-duration-100, 130ms),
+      border-color var(--spectrum-global-animation-duration-100, 130ms);
+    height: calc(var(--spectrum-alias-item-height-m) - 2px);
+  }
+
+  .icon:hover {
+    cursor: pointer;
+    color: var(--spectrum-alias-text-color-hover);
+    background-color: var(--spectrum-global-color-gray-50);
+    border-color: var(--spectrum-alias-border-color-hover);
+  }
+</style>

packages/builder/src/components/deploy/VersionModal.svelte

@@ -0,0 +1,120 @@
+<script>
+  import {
+    Icon,
+    Modal,
+    notifications,
+    ModalContent,
+    Body,
+    Button,
+  } from "@budibase/bbui"
+  import { store } from "builderStore"
+  import api from "builderStore/api"
+  import clientPackage from "@budibase/client/package.json"
+
+  let updateModal
+
+  $: appId = $store.appId
+  $: updateAvailable = clientPackage.version !== $store.version
+  $: revertAvailable = $store.revertableVersion != null
+
+  const refreshAppPackage = async () => {
+    const applicationPkg = await api.get(
+      `/api/applications/${appId}/appPackage`
+    )
+    const pkg = await applicationPkg.json()
+    if (applicationPkg.ok) {
+      await store.actions.initialise(pkg)
+    } else {
+      throw new Error(pkg)
+    }
+  }
+
+  const update = async () => {
+    try {
+      const response = await api.post(
+        `/api/applications/${appId}/client/update`
+      )
+      const json = await response.json()
+      if (response.status !== 200) {
+        throw json.message
+      }
+
+      // Don't wait for the async refresh, since this causes modal flashing
+      refreshAppPackage()
+      notifications.success(
+        `App updated successfully to version ${clientPackage.version}`
+      )
+    } catch (err) {
+      notifications.error(`Error updating app: ${err}`)
+    }
+  }
+
+  const revert = async () => {
+    try {
+      const revertableVersion = $store.revertableVersion
+      const response = await api.post(
+        `/api/applications/${appId}/client/revert`
+      )
+      const json = await response.json()
+      if (response.status !== 200) {
+        throw json.message
+      }
+
+      // Don't wait for the async refresh, since this causes modal flashing
+      refreshAppPackage()
+      notifications.success(
+        `App reverted successfully to version ${revertableVersion}`
+      )
+    } catch (err) {
+      notifications.error(`Error reverting app: ${err}`)
+    }
+    updateModal.hide()
+  }
+</script>
+
+<div class="icon-wrapper" class:highlight={updateAvailable}>
+  <Icon name="Refresh" hoverable on:click={updateModal.show} />
+</div>
+<Modal bind:this={updateModal}>
+  <ModalContent
+    title="App version"
+    confirmText="Update"
+    cancelText={updateAvailable ? "Cancel" : "Close"}
+    onConfirm={update}
+    showConfirmButton={updateAvailable}
+  >
+    <div slot="footer">
+      {#if revertAvailable}
+        <Button quiet secondary on:click={revert}>Revert</Button>
+      {/if}
+    </div>
+    {#if updateAvailable}
+      <Body size="S">
+        This app is currently using version <b>{$store.version}</b>, but version
+        <b>{clientPackage.version}</b> is available. Updates can contain new features,
+        performance improvements and bug fixes.
+      </Body>
+    {:else}
+      <Body size="S">
+        This app is currently using version <b>{$store.version}</b> which is the
+        latest version available.
+      </Body>
+    {/if}
+    {#if revertAvailable}
+      <Body size="S">
+        You can revert this app to version
+        <b>{$store.revertableVersion}</b>
+        if you're experiencing issues with the current version.
+      </Body>
+    {/if}
+  </ModalContent>
+</Modal>
+
+<style>
+  .icon-wrapper {
+    display: contents;
+  }
+  .icon-wrapper.highlight :global(svg) {
+    color: var(--spectrum-global-color-blue-600);
+  }
+</style>

packages/builder/src/components/design/AppPreview/AppThemeSelect.svelte

@@ -0,0 +1,38 @@
+<script>
+  import { Select } from "@budibase/bbui"
+  import { store } from "builderStore"
+
+  const themeOptions = [
+    {
+      label: "Lightest",
+      value: "spectrum--lightest",
+    },
+    {
+      label: "Light",
+      value: "spectrum--light",
+    },
+    {
+      label: "Dark",
+      value: "spectrum--dark",
+    },
+    {
+      label: "Darkest",
+      value: "spectrum--darkest",
+    },
+  ]
+</script>
+
+<div>
+  <Select
+    value={$store.theme || "spectrum--light"}
+    options={themeOptions}
+    placeholder={null}
+    on:change={e => store.actions.theme.save(e.detail)}
+  />
+</div>
+
+<style>
+  div {
+    padding-right: 8px;
+  }
+</style>

packages/builder/src/components/design/AppPreview/CurrentItemPreview.svelte

@@ -48,6 +48,7 @@
     screen,
     selectedComponentId,
     previewType: $store.currentFrontEndType,
+    theme: $store.theme,
   }
 
   // Saving pages and screens to the DB causes them to have _revs.
@@ -74,19 +75,16 @@
     iframe.contentWindow.addEventListener(
       "ready",
       () => {
+        // Display preview immediately if the intelligent loading feature
+        // is not supported
+        if (!$store.clientFeatures.intelligentLoading) {
           loading = false
+        }
         refreshContent(strippedJson)
       },
       { once: true }
     )
 
-    // Use iframe loading event to support old client versions
-    iframe.contentWindow.addEventListener(
-      "iframe-loaded",
-      () => (loading = false),
-      { once: true }
-    )
-
     // Catch any app errors
     iframe.contentWindow.addEventListener(
       "error",
@@ -108,7 +106,9 @@
         idToDelete = data.id
         confirmDeleteDialog.show()
       } else if (type === "preview-loaded") {
-        // loading = false
+        // Wait for this event to show the client library if intelligent
+        // loading is supported
+        loading = false
       } else {
         console.warning(`Client sent unknown event type: ${type}`)
       }

packages/builder/src/components/design/AppPreview/componentStructure.json

@@ -27,9 +27,7 @@
     "name": "Card",
     "icon": "Card",
     "children": [
-      "stackedlist",
+      "spectrumcard",
-      "card",
-      "cardhorizontal",
       "cardstat"
     ]
   },
@@ -57,13 +55,6 @@
       "icon",
       "embed"
     ]
-  },
-  {
-    "name": "Other",
-    "icon": "More",
-    "children": [
-      "screenslot"
-    ]
   }
 ]
 

packages/builder/src/components/design/AppPreview/iframeTemplate.js

@@ -48,7 +48,14 @@ export default `
         }
 
         // Extract data from message
-        const { selectedComponentId, layout, screen, previewType, appId } = JSON.parse(event.data)
+        const {
+          selectedComponentId,
+          layout,
+          screen,
+          previewType,
+          appId,
+          theme
+        } = JSON.parse(event.data)
 
         // Set some flags so the app knows we're in the builder
         window["##BUDIBASE_IN_BUILDER##"] = true
@@ -58,6 +65,7 @@ export default `
         window["##BUDIBASE_SELECTED_COMPONENT_ID##"] = selectedComponentId
         window["##BUDIBASE_PREVIEW_ID##"] = Math.random()
         window["##BUDIBASE_PREVIEW_TYPE##"] = previewType
+        window["##BUDIBASE_PREVIEW_THEME##"] = theme
 
         // Initialise app
         try {

packages/builder/src/components/design/NavigationPanel/ComponentDropdownMenu.svelte

@@ -65,11 +65,14 @@
   }
 </script>
 
-<ActionMenu>
+{#if definition?.editable !== false}
+  <ActionMenu>
     <div slot="control" class="icon">
       <Icon size="S" hoverable name="MoreSmallList" />
     </div>
-  <MenuItem icon="Delete" on:click={confirmDeleteDialog.show}>Delete</MenuItem>
+    <MenuItem icon="Delete" on:click={confirmDeleteDialog.show}>
+      Delete
+    </MenuItem>
     <MenuItem noClose icon="ChevronUp" on:click={moveUpComponent}>
       Move up
     </MenuItem>
@@ -106,11 +109,12 @@
     >
       Paste inside
     </MenuItem>
-</ActionMenu>
+  </ActionMenu>
-<ConfirmDialog
+  <ConfirmDialog
     bind:this={confirmDeleteDialog}
     title="Confirm Deletion"
     body={`Are you sure you wish to delete this '${definition?.name}' component?`}
     okText="Delete Component"
     onOk={deleteComponent}
-/>
+  />
+{/if}

packages/builder/src/components/design/NavigationPanel/ComponentNavigationTree/ComponentTree.svelte

@@ -3,6 +3,7 @@
   import { DropEffect, DropPosition } from "./dragDropStore"
   import ComponentDropdownMenu from "../ComponentDropdownMenu.svelte"
   import NavItem from "components/common/NavItem.svelte"
+  import { capitalise } from "helpers"
 
   export let components = []
   export let currentComponent
@@ -10,8 +11,6 @@
   export let level = 0
   export let dragDropStore
 
-  const isScreenslot = name => name?.endsWith("screenslot")
-
   const selectComponent = component => {
     store.actions.components.select(component)
   }
@@ -42,6 +41,16 @@
 
     return false
   }
+
+  const getComponentText = component => {
+    if (component._instanceName) {
+      return component._instanceName
+    }
+    const type =
+      component._component.replace("@budibase/standard-components/", "") ||
+      "component"
+    return capitalise(type)
+  }
 </script>
 
 <ul>
@@ -63,9 +72,7 @@
         on:dragstart={dragstart(component)}
         on:dragover={dragover(component, index)}
         on:drop={dragDropStore.actions.drop}
-        text={isScreenslot(component._component)
+        text={getComponentText(component)}
-          ? "Screenslot"
-          : component._instanceName}
         withArrow
         indentLevel={level + 1}
         selected={$store.selectedComponentId === component._id}

packages/builder/src/components/design/PropertiesPanel/ComponentSettingsSection.svelte

@@ -1,12 +1,6 @@
 <script>
   import { isEmpty } from "lodash/fp"
-  import {
+  import { Checkbox, Input, Select, DetailSummary } from "@budibase/bbui"
-    Checkbox,
-    Input,
-    Select,
-    DetailSummary,
-    ColorPicker,
-  } from "@budibase/bbui"
   import { store } from "builderStore"
   import PropertyControl from "./PropertyControls/PropertyControl.svelte"
   import LayoutSelect from "./PropertyControls/LayoutSelect.svelte"
@@ -31,6 +25,8 @@
   import AttachmentFieldSelect from "./PropertyControls/AttachmentFieldSelect.svelte"
   import RelationshipFieldSelect from "./PropertyControls/RelationshipFieldSelect.svelte"
   import ResetFieldsButton from "./PropertyControls/ResetFieldsButton.svelte"
+  import ColorPicker from "./PropertyControls/ColorPicker.svelte"
+  import URLSelect from "./PropertyControls/URLSelect.svelte"
 
   export let componentDefinition
   export let componentInstance
@@ -66,6 +62,7 @@
     section: SectionSelect,
     navigation: NavigationEditor,
     filter: FilterEditor,
+    url: URLSelect,
     "field/string": StringFieldSelect,
     "field/number": NumberFieldSelect,
     "field/options": OptionsFieldSelect,

packages/builder/src/components/design/PropertiesPanel/CustomStylesSection.svelte

@@ -35,7 +35,8 @@
     <ActionButton on:click={openDrawer}>Edit custom CSS</ActionButton>
   </div>
 </DetailSummary>
-<Drawer bind:this={drawer} title="Custom CSS">
+{#key componentInstance?._id}
+  <Drawer bind:this={drawer} title="Custom CSS">
     <Button cta slot="buttons" on:click={save}>Save</Button>
     <DrawerContent slot="body">
       <div class="content">
@@ -45,7 +46,8 @@
         </Layout>
       </div>
     </DrawerContent>
-</Drawer>
+  </Drawer>
+{/key}
 
 <style>
   .content {

packages/builder/src/components/design/PropertiesPanel/PropertyControls/ColorPicker.svelte

@@ -1,42 +1,8 @@
 <script>
-  import { createEventDispatcher } from "svelte"
+  import { ColorPicker } from "@budibase/bbui"
-  import Colorpicker from "@budibase/colorpicker"
+  import { store } from "builderStore"
-
-  const dispatch = createEventDispatcher()
 
   export let value
-
-  const WAIT = 150
-
-  function throttle(callback, wait, immediate = false) {
-    let timeout = null
-    let initialCall = true
-
-    return function () {
-      const callNow = immediate && initialCall
-      const next = () => {
-        callback.apply(this, arguments)
-        timeout = null
-      }
-
-      if (callNow) {
-        initialCall = false
-        next()
-      }
-
-      if (!timeout) {
-        timeout = setTimeout(next, wait)
-      }
-    }
-  }
-
-  const onChange = throttle(
-    e => {
-      dispatch("change", e.detail)
-    },
-    WAIT,
-    true
-  )
 </script>
 
-<Colorpicker value={value || "#C4C4C4"} on:change={onChange} />
+<ColorPicker {value} on:change spectrumTheme={$store.theme} />

packages/builder/src/components/design/PropertiesPanel/PropertyControls/EventsEditor/EventPropertyControl.svelte

@@ -51,7 +51,7 @@
   }
 </script>
 
-<ActionButton on:click={drawer.show}>Define Actions</ActionButton>
+<ActionButton on:click={drawer.show}>Define actions</ActionButton>
 <Drawer bind:this={drawer} title={"Actions"}>
   <svelte:fragment slot="description">
     Define what actions to run.

packages/builder/src/components/design/PropertiesPanel/PropertyControls/FilterEditor/FilterEditor.svelte

@@ -38,7 +38,7 @@
   }
 </script>
 
-<ActionButton on:click={drawer.show}>Define Filters</ActionButton>
+<ActionButton on:click={drawer.show}>Define filters</ActionButton>
 <Drawer bind:this={drawer} title="Filtering">
   <Button cta slot="buttons" on:click={saveFilter}>Save</Button>
   <DrawerContent slot="body">

packages/builder/src/components/design/PropertiesPanel/PropertyControls/IconSelect/IconSelect.svelte

@@ -107,7 +107,7 @@
     loading = false
   }
 
-  $: displayValue = value ? value.substring(3) : "Pick Icon"
+  $: displayValue = value ? value.substring(3) : "Pick icon"
 
   $: totalPages = Math.ceil(filteredIcons.length / maxIconsPerPage)
   $: pageEndIdx = maxIconsPerPage * currentPage

packages/builder/src/components/design/PropertiesPanel/PropertyControls/URLSelect.svelte

@@ -0,0 +1,12 @@
+<script>
+  import { store } from "builderStore"
+  import DrawerBindableCombobox from "components/common/bindings/DrawerBindableCombobox.svelte"
+
+  export let value
+
+  $: urlOptions = $store.screens
+    .map(screen => screen.routing?.route)
+    .filter(x => x != null)
+</script>
+
+<DrawerBindableCombobox {value} on:change options={urlOptions} />

packages/builder/src/components/design/PropertiesPanel/componentStyles.js

@@ -1,4 +1,5 @@
-import { Input, Select, ColorPicker } from "@budibase/bbui"
+import { Input, Select } from "@budibase/bbui"
+import ColorPicker from "./PropertyControls/ColorPicker.svelte"
 
 export const margin = {
   label: "Margin",

packages/builder/src/pages/builder/app/[application]/_layout.svelte

@@ -4,6 +4,7 @@
   import { Icon, ActionGroup, Tabs, Tab } from "@budibase/bbui"
   import DeployModal from "components/deploy/DeployModal.svelte"
   import RevertModal from "components/deploy/RevertModal.svelte"
+  import VersionModal from "components/deploy/VersionModal.svelte"
   import { get } from "builderStore/api"
   import { isActive, goto, layout } from "@roxi/routify"
   import Logo from "assets/bb-emblem.svg"
@@ -80,6 +81,7 @@
         <ActionGroup />
       </div>
       <div class="toprightnav">
+        <VersionModal />
         <RevertModal />
         <Icon
           name="Play"

packages/builder/src/pages/builder/app/[application]/data/_layout.svelte

@@ -7,7 +7,6 @@
 
   let selected = "Sources"
   let modal
-
   $: isExternal =
     $params.selectedDatasource &&
     $params.selectedDatasource !== BUDIBASE_INTERNAL_DB

packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/CreateEditRelationship/CreateEditRelationship.svelte

@@ -0,0 +1,286 @@
+<script>
+  import { RelationshipTypes } from "constants/backend"
+  import {
+    Button,
+    Input,
+    ModalContent,
+    Select,
+    Detail,
+    Body,
+  } from "@budibase/bbui"
+  import { tables } from "stores/backend"
+  import { uuid } from "builderStore/uuid"
+  import { writable } from "svelte/store"
+
+  export let save
+  export let datasource
+  export let plusTables = []
+  export let fromRelationship = {}
+  export let toRelationship = {}
+  export let close
+
+  let originalFromName = fromRelationship.name,
+    originalToName = toRelationship.name
+
+  function inSchema(table, prop, ogName) {
+    if (!table || !prop || prop === ogName) {
+      return false
+    }
+    const keys = Object.keys(table.schema).map(key => key.toLowerCase())
+    return keys.indexOf(prop.toLowerCase()) !== -1
+  }
+
+  const touched = writable({})
+
+  function checkForErrors(
+    fromTable,
+    toTable,
+    throughTable,
+    fromRelate,
+    toRelate
+  ) {
+    const isMany =
+      fromRelate.relationshipType === RelationshipTypes.MANY_TO_MANY
+    const tableNotSet = "Please specify a table"
+    const errors = {}
+    if ($touched.from && !fromTable) {
+      errors.from = tableNotSet
+    }
+    if ($touched.to && !toTable) {
+      errors.to = tableNotSet
+    }
+    if ($touched.through && isMany && !fromRelate.through) {
+      errors.through = tableNotSet
+    }
+    if ($touched.foreign && !isMany && !fromRelate.fieldName) {
+      errors.foreign = "Please pick the foreign key"
+    }
+    const colNotSet = "Please specify a column name"
+    if ($touched.fromCol && !fromRelate.name) {
+      errors.fromCol = colNotSet
+    }
+    if ($touched.toCol && !toRelate.name) {
+      errors.toCol = colNotSet
+    }
+    // currently don't support relationships back onto the table itself, needs to relate out
+    const tableError = "From/to/through tables must be different"
+    if (fromTable && (fromTable === toTable || fromTable === throughTable)) {
+      errors.from = tableError
+    }
+    if (toTable && (toTable === fromTable || toTable === throughTable)) {
+      errors.to = tableError
+    }
+    if (
+      throughTable &&
+      (throughTable === fromTable || throughTable === toTable)
+    ) {
+      errors.through = tableError
+    }
+    const colError = "Column name cannot be an existing column"
+    if (inSchema(fromTable, fromRelate.name, originalFromName)) {
+      errors.fromCol = colError
+    }
+    if (inSchema(toTable, toRelate.name, originalToName)) {
+      errors.toCol = colError
+    }
+    return errors
+  }
+
+  $: tableOptions = plusTables.map(table => ({
+    label: table.name,
+    value: table._id,
+  }))
+  $: fromTable = plusTables.find(table => table._id === toRelationship?.tableId)
+  $: toTable = plusTables.find(table => table._id === fromRelationship?.tableId)
+  $: through = plusTables.find(table => table._id === fromRelationship?.through)
+  $: errors = checkForErrors(
+    fromTable,
+    toTable,
+    through,
+    fromRelationship,
+    toRelationship
+  )
+  $: valid =
+    Object.keys(errors).length === 0 && Object.keys($touched).length !== 0
+  $: linkTable = through || toTable
+  $: relationshipTypes = [
+    {
+      label: "Many",
+      value: RelationshipTypes.MANY_TO_MANY,
+    },
+    {
+      label: "One",
+      value: RelationshipTypes.MANY_TO_ONE,
+    },
+  ]
+  $: updateRelationshipType(fromRelationship?.relationshipType)
+
+  function updateRelationshipType(fromType) {
+    if (fromType === RelationshipTypes.MANY_TO_MANY) {
+      toRelationship.relationshipType = RelationshipTypes.MANY_TO_MANY
+    } else {
+      toRelationship.relationshipType = RelationshipTypes.MANY_TO_ONE
+    }
+  }
+
+  function buildRelationships() {
+    // if any to many only need to check from
+    const manyToMany =
+      fromRelationship.relationshipType === RelationshipTypes.MANY_TO_MANY
+    // main is simply used to know this is the side the user configured it from
+    const id = uuid()
+    if (!manyToMany) {
+      delete fromRelationship.through
+      delete toRelationship.through
+    }
+    let relateFrom = {
+      ...fromRelationship,
+      type: "link",
+      main: true,
+      _id: id,
+    }
+    let relateTo = {
+      ...toRelationship,
+      type: "link",
+      _id: id,
+    }
+
+    // [0] is because we don't support composite keys for relationships right now
+    if (manyToMany) {
+      relateFrom = {
+        ...relateFrom,
+        through: through._id,
+        fieldName: toTable.primary[0],
+      }
+      relateTo = {
+        ...relateTo,
+        through: through._id,
+        fieldName: fromTable.primary[0],
+      }
+    } else {
+      relateFrom = {
+        ...relateFrom,
+        foreignKey: relateFrom.fieldName,
+        fieldName: fromTable.primary[0],
+      }
+      relateTo = {
+        ...relateTo,
+        relationshipType: RelationshipTypes.ONE_TO_MANY,
+        foreignKey: relateFrom.fieldName,
+        fieldName: fromTable.primary[0],
+      }
+    }
+
+    fromRelationship = relateFrom
+    toRelationship = relateTo
+  }
+
+  // save the relationship on to the datasource
+  async function saveRelationship() {
+    buildRelationships()
+    // source of relationship
+    datasource.entities[fromTable.name].schema[fromRelationship.name] =
+      fromRelationship
+    // save other side of relationship in the other schema
+    datasource.entities[toTable.name].schema[toRelationship.name] =
+      toRelationship
+
+    // If relationship has been renamed
+    if (originalFromName !== fromRelationship.name) {
+      delete datasource.entities[fromTable.name].schema[originalFromName]
+    }
+    if (originalToName !== toRelationship.name) {
+      delete datasource.entities[toTable.name].schema[originalToName]
+    }
+
+    await save()
+    await tables.fetch()
+  }
+
+  async function deleteRelationship() {
+    delete datasource.entities[fromTable.name].schema[fromRelationship.name]
+    delete datasource.entities[toTable.name].schema[toRelationship.name]
+    await save()
+    await tables.fetch()
+    close()
+  }
+</script>
+
+<ModalContent
+  title="Create Relationship"
+  confirmText="Save"
+  onConfirm={saveRelationship}
+  disabled={!valid}
+>
+  <Select
+    label="Relationship type"
+    options={relationshipTypes}
+    bind:value={fromRelationship.relationshipType}
+  />
+  <div class="headings">
+    <Detail>Tables</Detail>
+  </div>
+  <Select
+    label="Select from table"
+    options={tableOptions}
+    on:change={() => ($touched.from = true)}
+    bind:error={errors.from}
+    bind:value={toRelationship.tableId}
+  />
+  <Select
+    label={"Select to table"}
+    options={tableOptions}
+    on:change={() => ($touched.to = true)}
+    bind:error={errors.to}
+    bind:value={fromRelationship.tableId}
+  />
+  {#if fromRelationship?.relationshipType === RelationshipTypes.MANY_TO_MANY}
+    <Select
+      label={"Through"}
+      options={tableOptions}
+      on:change={() => ($touched.through = true)}
+      bind:error={errors.through}
+      bind:value={fromRelationship.through}
+    />
+  {:else if fromRelationship?.relationshipType && toTable}
+    <Select
+      label={`Foreign Key (${toTable?.name})`}
+      options={Object.keys(toTable?.schema).filter(
+        field => toTable?.primary.indexOf(field) === -1
+      )}
+      on:change={() => ($touched.foreign = true)}
+      bind:error={errors.foreign}
+      bind:value={fromRelationship.fieldName}
+    />
+  {/if}
+  <div class="headings">
+    <Detail>Column names</Detail>
+  </div>
+  <Body>
+    Budibase manages SQL relationships as a new column in the table, please
+    provide a name for these columns.
+  </Body>
+  <Input
+    on:blur={() => ($touched.fromCol = true)}
+    bind:error={errors.fromCol}
+    label="From table column"
+    bind:value={fromRelationship.name}
+  />
+  <Input
+    on:blur={() => ($touched.toCol = true)}
+    bind:error={errors.toCol}
+    label="To table column"
+    bind:value={toRelationship.name}
+  />
+  <div slot="footer">
+    {#if originalFromName != null}
+      <Button warning text on:click={deleteRelationship}>Delete</Button>
+    {/if}
+  </div>
+</ModalContent>
+
+<style>
+  .headings {
+    margin-top: var(--spacing-s);
+  }
+</style>

packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/CreateEditRelationship/TableSelect.svelte

@@ -0,0 +1,21 @@
+<script>
+  import { Menu, Icon, MenuSection, MenuItem } from "@budibase/bbui"
+
+  export let heading
+  export let tables
+  export let selected = false
+  export let select
+</script>
+
+<Menu>
+  <MenuSection {heading}>
+    {#each tables as table}
+      <MenuItem noClose icon="Table" on:click={() => select(table)}>
+        {table.name}
+        {#if selected}
+          <Icon size="S" name="Checkmark" />
+        {/if}
+      </MenuItem>
+    {/each}
+  </MenuSection>
+</Menu>

packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/index.svelte

@@ -1,16 +1,70 @@
 <script>
   import { goto, beforeUrlChange } from "@roxi/routify"
-  import { Button, Heading, Body, Divider, Layout } from "@budibase/bbui"
+  import { Button, Heading, Body, Divider, Layout, Modal } from "@budibase/bbui"
   import { datasources, integrations, queries, tables } from "stores/backend"
   import { notifications } from "@budibase/bbui"
   import IntegrationConfigForm from "components/backend/DatasourceNavigator/TableIntegrationMenu/IntegrationConfigForm.svelte"
+  import CreateEditRelationship from "./CreateEditRelationship/CreateEditRelationship.svelte"
+  import DisplayColumnModal from "./modals/EditDisplayColumnsModal.svelte"
   import ICONS from "components/backend/DatasourceNavigator/icons"
   import { capitalise } from "helpers"
 
   let unsaved = false
+  let relationshipModal
+  let displayColumnModal
+  let selectedFromRelationship, selectedToRelationship
 
   $: datasource = $datasources.list.find(ds => ds._id === $datasources.selected)
   $: integration = datasource && $integrations[datasource.source]
+  $: plusTables = datasource?.plus
+    ? Object.values(datasource.entities || {})
+    : []
+  $: relationships = getRelationships(plusTables)
+
+  function getRelationships(tables) {
+    if (!tables || !Array.isArray(tables)) {
+      return {}
+    }
+    let pairs = {}
+    for (let table of tables) {
+      for (let column of Object.values(table.schema)) {
+        if (column.type !== "link") {
+          continue
+        }
+        // these relationships have an id to pair them to each other
+        // one has a main for the from side
+        const key = column.main ? "from" : "to"
+        pairs[column._id] = {
+          ...pairs[column._id],
+          [key]: column,
+        }
+      }
+    }
+    return pairs
+  }
+
+  function buildRelationshipDisplayString(fromCol, toCol) {
+    function getTableName(tableId) {
+      if (!tableId || typeof tableId !== "string") {
+        return null
+      }
+      return plusTables.find(table => table._id === tableId)?.name || "Unknown"
+    }
+    if (!toCol || !fromCol) {
+      return "Cannot build name"
+    }
+    const fromTableName = getTableName(toCol.tableId)
+    const toTableName = getTableName(fromCol.tableId)
+    const throughTableName = getTableName(fromCol.through)
+
+    let displayString
+    if (throughTableName) {
+      displayString = `${fromTableName} through ${throughTableName} → ${toTableName}`
+    } else {
+      displayString = `${fromTableName} → ${toTableName}`
+    }
+    return displayString
+  }
 
   async function saveDatasource() {
     try {
@@ -48,6 +102,16 @@
     unsaved = true
   }
 
+  function openRelationshipModal(fromRelationship, toRelationship) {
+    selectedFromRelationship = fromRelationship || {}
+    selectedToRelationship = toRelationship || {}
+    relationshipModal.show()
+  }
+
+  function openDisplayColumnModal() {
+    displayColumnModal.show()
+  }
+
   $beforeUrlChange(() => {
     if (unsaved) {
       notifications.error(
@@ -59,6 +123,21 @@
   })
 </script>
 
+<Modal bind:this={relationshipModal}>
+  <CreateEditRelationship
+    {datasource}
+    save={saveDatasource}
+    close={relationshipModal.hide}
+    {plusTables}
+    fromRelationship={selectedFromRelationship}
+    toRelationship={selectedToRelationship}
+  />
+</Modal>
+
+<Modal bind:this={displayColumnModal}>
+  <DisplayColumnModal {datasource} {plusTables} save={saveDatasource} />
+</Modal>
+
 {#if datasource && integration}
   <section>
     <Layout>
@@ -92,9 +171,18 @@
         <Divider />
         <div class="query-header">
           <Heading size="S">Tables</Heading>
-          <Button primary on:click={updateDatasourceSchema}
+          <div class="table-buttons">
-            >Fetch Tables From Database</Button
+            {#if plusTables && plusTables.length !== 0}
-          >
+              <Button primary on:click={openDisplayColumnModal}>
+                Update display columns
+              </Button>
+            {/if}
+            <div>
+              <Button primary on:click={updateDatasourceSchema}>
+                Fetch tables from database
+              </Button>
+            </div>
+          </div>
         </div>
         <Body>
           This datasource can determine tables automatically. Budibase can fetch
@@ -102,18 +190,44 @@
           having to write any queries at all.
         </Body>
         <div class="query-list">
-          {#if datasource.entities}
+          {#each plusTables as table}
-            {#each Object.keys(datasource.entities) as entity}
+            <div class="query-list-item" on:click={() => onClickTable(table)}>
-              <div
+              <p class="query-name">{table.name}</p>
-                class="query-list-item"
+              <p>Primary Key: {table.primary}</p>
-                on:click={() => onClickTable(datasource.entities[entity])}
-              >
-                <p class="query-name">{entity}</p>
-                <p>Primary Key: {datasource.entities[entity].primary}</p>
               <p>→</p>
             </div>
           {/each}
+        </div>
+        {#if plusTables?.length !== 0}
+          <Divider />
+          <div class="query-header">
+            <Heading size="S">Relationships</Heading>
+            <Button primary on:click={() => openRelationshipModal()}
+              >Create relationship</Button
+            >
+          </div>
+          <Body>
+            Tell budibase how your tables are related to get even more smart
+            features.
+          </Body>
         {/if}
+        <div class="query-list">
+          {#each Object.values(relationships) as relationship}
+            <div
+              class="query-list-item"
+              on:click={() =>
+                openRelationshipModal(relationship.from, relationship.to)}
+            >
+              <p class="query-name">
+                {buildRelationshipDisplayString(
+                  relationship.from,
+                  relationship.to
+                )}
+              </p>
+              <p>{relationship.from?.name} to {relationship.to?.name}</p>
+              <p>→</p>
+            </div>
+          {/each}
         </div>
       {/if}
       <Divider />
@@ -202,4 +316,14 @@
     text-overflow: ellipsis;
     font-size: var(--font-size-s);
   }
+
+  .table-buttons {
+    display: grid;
+    grid-gap: var(--spacing-l);
+    grid-template-columns: 1fr 1fr;
+  }
+
+  .table-buttons div {
+    grid-column-end: -1;
+  }
 </style>

packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/modals/EditDisplayColumnsModal.svelte

@@ -0,0 +1,43 @@
+<script>
+  import { ModalContent, Select, Body } from "@budibase/bbui"
+  import { tables } from "stores/backend"
+
+  export let datasource
+  export let plusTables
+  export let save
+
+  async function saveDisplayColumns() {
+    // be explicit about copying over
+    for (let table of plusTables) {
+      datasource.entities[table.name].primaryDisplay = table.primaryDisplay
+    }
+    save()
+    await tables.fetch()
+  }
+
+  function getColumnOptions(table) {
+    if (!table || !table.schema) {
+      return []
+    }
+    return Object.entries(table.schema)
+      .filter(field => field[1].type !== "link")
+      .map(([fieldName]) => fieldName)
+  }
+</script>
+
+<ModalContent
+  title="Edit display columns"
+  confirmText="Save"
+  onConfirm={saveDisplayColumns}
+>
+  <Body
+    >Select the columns that will be shown when displaying relationships.</Body
+  >
+  {#each plusTables as table}
+    <Select
+      label={table.name}
+      options={getColumnOptions(table)}
+      bind:value={table.primaryDisplay}
+    />
+  {/each}
+</ModalContent>

packages/builder/src/pages/builder/app/[application]/design/[assetType]/_layout.svelte

@@ -13,6 +13,7 @@
   import { FrontendTypes } from "constants"
   import { findComponent, findComponentPath } from "builderStore/storeUtils"
   import { get } from "svelte/store"
+  import AppThemeSelect from "components/design/AppPreview/AppThemeSelect.svelte"
 
   // Cache previous values so we don't update the URL more than necessary
   let previousType
@@ -147,9 +148,16 @@
 
   <div class="preview-pane">
     {#if $currentAsset}
+      <div class="preview-header">
         <ComponentSelectionList />
+        {#if $store.clientFeatures.spectrumThemes}
+          <AppThemeSelect />
+        {/if}
+      </div>
       <div class="preview-content">
+        {#key $store.version}
           <CurrentItemPreview />
+        {/key}
       </div>
     {/if}
   </div>
@@ -193,6 +201,10 @@
     gap: var(--spacing-m);
     padding: var(--spacing-xl) 40px;
   }
+  .preview-header {
+    display: grid;
+    grid-template-columns: 1fr 100px;
+  }
   .preview-content {
     flex: 1 1 auto;
   }

packages/builder/src/pages/builder/portal/manage/users/[userId].svelte

@@ -60,6 +60,16 @@
 
   let toggleDisabled = false
 
+  async function updateUserFirstName(evt) {
+    await users.save({ ...$userFetch?.data, firstName: evt.target.value })
+    await userFetch.refresh()
+  }
+
+  async function updateUserLastName(evt) {
+    await users.save({ ...$userFetch?.data, lastName: evt.target.value })
+    await userFetch.refresh()
+  }
+
   async function toggleFlag(flagName, detail) {
     toggleDisabled = true
     await users.save({ ...$userFetch?.data, [flagName]: { global: detail } })
@@ -113,11 +123,19 @@
       </div>
       <div class="field">
         <Label size="L">First name</Label>
-        <Input disabled thin value={$userFetch?.data?.firstName} />
+        <Input
+          thin
+          value={$userFetch?.data?.firstName}
+          on:blur={updateUserFirstName}
+        />
       </div>
       <div class="field">
         <Label size="L">Last name</Label>
-        <Input disabled thin value={$userFetch?.data?.lastName} />
+        <Input
+          thin
+          value={$userFetch?.data?.lastName}
+          on:blur={updateUserLastName}
+        />
       </div>
       <!-- don't let a user remove the privileges that let them be here -->
       {#if userId !== $auth.user._id}

packages/builder/src/pages/builder/portal/manage/users/_components/UpdateRolesModal.svelte

@@ -9,7 +9,9 @@
   const dispatch = createEventDispatcher()
 
   const roles = app.roles
-  let options = roles.map(role => role._id).filter(id => id !== "PUBLIC")
+  let options = roles
+    .map(role => ({ value: role._id, label: role.name }))
+    .filter(role => role.value !== "PUBLIC")
   let selectedRole = user?.roles?.[app?._id]
 
   async function updateUserRoles() {

packages/cli/package.json

@@ -24,7 +24,7 @@
     "docker-compose": "^0.23.6",
     "inquirer": "^8.0.0",
     "lookpath": "^1.1.0",
-    "pkg": "^4.4.9",
+    "pkg": "^5.3.0",
     "posthog-node": "1.0.7",
     "randomstring": "^1.1.5"
   },

packages/client/src/api/app.js

@@ -3,8 +3,8 @@ import API from "./api"
 /**
  * Fetches screen definition for an app.
  */
-export const fetchAppDefinition = async appId => {
+export const fetchAppPackage = async appId => {
   return await API.get({
-    url: `/api/applications/${appId}/definition`,
+    url: `/api/applications/${appId}/appPackage`,
   })
 }

packages/client/src/components/ClientApp.svelte

@@ -13,6 +13,7 @@
     authStore,
     routeStore,
     builderStore,
+    appStore,
   } from "../store"
   import { TableNames, ActionTypes } from "../constants"
   import SettingsBar from "./preview/SettingsBar.svelte"
@@ -70,6 +71,9 @@
       }
     }
   }
+
+  $: themeClass =
+    $builderStore.theme || $appStore.application?.theme || "spectrum--light"
 </script>
 
 {#if dataLoaded}
@@ -77,7 +81,7 @@
     id="spectrum-root"
     lang="en"
     dir="ltr"
-    class="spectrum spectrum--medium spectrum--light"
+    class="spectrum spectrum--medium {themeClass}"
   >
     {#if permissionError}
       <div class="error">
@@ -126,6 +130,28 @@
   }
   #app-root {
     position: relative;
+    border: 1px solid var(--spectrum-global-color-gray-300);
+  }
+
+  /* Custom scrollbars */
+  :global(::-webkit-scrollbar) {
+    width: 8px;
+    height: 8px;
+  }
+  :global(::-webkit-scrollbar-track) {
+    background: var(--spectrum-alias-background-color-default);
+  }
+  :global(::-webkit-scrollbar-thumb) {
+    background-color: var(--spectrum-global-color-gray-400);
+    border-radius: 4px;
+  }
+  :global(::-webkit-scrollbar-corner) {
+    background: var(--spectrum-alias-background-color-default);
+  }
+  :global(*) {
+    scrollbar-width: thin;
+    scrollbar-color: var(--spectrum-global-color-gray-400)
+      var(--spectrum-alias-background-color-default);
   }
 
   .error {

packages/client/src/components/preview/HoverIndicator.svelte

@@ -29,4 +29,9 @@
   })
 </script>
 
-<IndicatorSet {componentId} color="rgb(120, 170, 244)" transition {zIndex} />
+<IndicatorSet
+  {componentId}
+  color="var(--spectrum-global-color-static-blue-200)"
+  transition
+  {zIndex}
+/>

packages/client/src/components/preview/SelectionIndicator.svelte

@@ -5,7 +5,7 @@
 
 <IndicatorSet
   componentId={$builderStore.selectedComponentId}
-  color="rgb(66, 133, 244)"
+  color="var(--spectrum-global-color-static-blue-600)"
   zIndex="910"
   transition
 />

packages/client/src/components/preview/SettingsBar.svelte

@@ -138,11 +138,11 @@
     padding: 6px 8px;
     opacity: 0;
     flex-direction: row;
-    background: var(--background);
+    background: var(--spectrum-alias-background-color-primary);
     justify-content: center;
     align-items: center;
     border-radius: 4px;
-    box-shadow: 2px 2px 8px rgba(0, 0, 0, 0.2);
+    box-shadow: 0 2px 8px rgba(0, 0, 0, 0.2);
     gap: 2px;
     transition: opacity 0.13s ease-in-out;
   }
@@ -155,4 +155,14 @@
     margin: 0 4px;
     background-color: var(--spectrum-global-color-gray-300);
   }
+
+  /* Theme overrides */
+  :global(.spectrum--dark) .bar,
+  :global(.spectrum--darkest) .bar {
+    background: var(--spectrum-global-color-gray-200);
+  }
+  :global(.spectrum--dark) .divider,
+  :global(.spectrum--darkest) .divider {
+    background: var(--spectrum-global-color-gray-400);
+  }
 </style>

packages/client/src/index.js

@@ -1,5 +1,5 @@
 import ClientApp from "./components/ClientApp.svelte"
-import { builderStore } from "./store"
+import { builderStore, appStore } from "./store"
 
 let app
 
@@ -7,14 +7,18 @@ const loadBudibase = () => {
   // Update builder store with any builder flags
   builderStore.set({
     inBuilder: !!window["##BUDIBASE_IN_BUILDER##"],
-    appId: window["##BUDIBASE_APP_ID##"],
     layout: window["##BUDIBASE_PREVIEW_LAYOUT##"],
     screen: window["##BUDIBASE_PREVIEW_SCREEN##"],
     selectedComponentId: window["##BUDIBASE_SELECTED_COMPONENT_ID##"],
     previewId: window["##BUDIBASE_PREVIEW_ID##"],
     previewType: window["##BUDIBASE_PREVIEW_TYPE##"],
+    theme: window["##BUDIBASE_PREVIEW_THEME##"],
   })
 
+  // Set app ID - this window flag is set by both the preview and the real
+  // server rendered app HTML
+  appStore.actions.setAppID(window["##BUDIBASE_APP_ID##"])
+
   // Create app if one hasn't been created yet
   if (!app) {
     app = new ClientApp({

packages/client/src/store/app.js

@@ -0,0 +1,27 @@
+import * as API from "../api"
+import { get, writable } from "svelte/store"
+
+const createAppStore = () => {
+  const store = writable({})
+
+  // Fetches the app definition including screens, layouts and theme
+  const fetchAppDefinition = async () => {
+    const appDefinition = await API.fetchAppPackage(get(store).appId)
+    store.set(appDefinition)
+  }
+
+  // Sets the initial app ID
+  const setAppID = id => {
+    store.update(state => {
+      state.appId = id
+      return state
+    })
+  }
+
+  return {
+    subscribe: store.subscribe,
+    actions: { setAppID, fetchAppDefinition },
+  }
+}
+
+export const appStore = createAppStore()

packages/client/src/store/auth.js

@@ -1,27 +1,14 @@
 import * as API from "../api"
-import { writable, get } from "svelte/store"
+import { writable } from "svelte/store"
-import { builderStore } from "./builder"
-import { TableNames } from "../constants"
 
 const createAuthStore = () => {
   const store = writable(null)
 
   // Fetches the user object if someone is logged in and has reloaded the page
   const fetchUser = async () => {
-    // Fetch the first user if inside the builder
-    if (get(builderStore).inBuilder) {
-      const users = await API.fetchTableData(TableNames.USERS)
-      if (!users.error && users[0] != null) {
-        store.set(users[0])
-      }
-    }
-
-    // Or fetch the current user from localstorage in a real app
-    else {
     const user = await API.fetchSelf()
     store.set(user)
   }
-  }
 
   return {
     subscribe: store.subscribe,

packages/client/src/store/index.js

@@ -1,4 +1,5 @@
 export { authStore } from "./auth"
+export { appStore } from "./app"
 export { notificationStore } from "./notification"
 export { routeStore } from "./routes"
 export { screenStore } from "./screens"

packages/client/src/store/initialise.js

@@ -1,7 +1,7 @@
 import { routeStore } from "./routes"
-import { screenStore } from "./screens"
+import { appStore } from "./app"
 
 export async function initialise() {
   await routeStore.actions.fetchRoutes()
-  await screenStore.actions.fetchScreens()
+  await appStore.actions.fetchAppDefinition()
 }

packages/client/src/store/screens.js

@@ -1,16 +1,12 @@
-import { writable, derived, get } from "svelte/store"
+import { derived } from "svelte/store"
 import { routeStore } from "./routes"
 import { builderStore } from "./builder"
-import * as API from "../api"
+import { appStore } from "./app"
 
 const createScreenStore = () => {
-  const config = writable({
-    screens: [],
-    layouts: [],
-  })
   const store = derived(
-    [config, routeStore, builderStore],
+    [appStore, routeStore, builderStore],
-    ([$config, $routeStore, $builderStore]) => {
+    ([$appStore, $routeStore, $builderStore]) => {
       let activeLayout, activeScreen
       let layouts, screens
       if ($builderStore.inBuilder) {
@@ -23,8 +19,8 @@ const createScreenStore = () => {
         activeLayout = { props: { _component: "screenslot" } }
 
         // Find the correct screen by matching the current route
-        screens = $config.screens
+        screens = $appStore.screens
-        layouts = $config.layouts
+        layouts = $appStore.layouts
         if ($routeStore.activeRoute) {
           activeScreen = screens.find(
             screen => screen._id === $routeStore.activeRoute.screenId
@@ -40,17 +36,8 @@ const createScreenStore = () => {
     }
   )
 
-  const fetchScreens = async () => {
-    const appDefinition = await API.fetchAppDefinition(get(builderStore).appId)
-    config.set({
-      screens: appDefinition.screens,
-      layouts: appDefinition.layouts,
-    })
-  }
-
   return {
     subscribe: store.subscribe,
-    actions: { fetchScreens },
   }
 }
 

packages/client/src/utils/styleable.js

@@ -29,7 +29,7 @@ export const styleable = (node, styles = {}) => {
     // overridden by any user specified styles
     let baseStyles = {}
     if (newStyles.empty) {
-      baseStyles.border = "2px dashed var(--grey-5)"
+      baseStyles.border = "2px dashed var(--spectrum-global-color-gray-600)"
       baseStyles.padding = "var(--spacing-l)"
       baseStyles.overflow = "hidden"
     }

packages/server/.gitignore

@@ -2,6 +2,7 @@ node_modules/
 myapps/
 .env
 builder/*
+client/*
 public/
 db/dev.db/
 dist

packages/server/.vscode/launch.json

@@ -5,10 +5,13 @@
     "version": "0.2.0",
     "configurations": [
       {
+        "name": "Start Server",
         "type": "node",
         "request": "launch",
-        "name": "Start Server",
+        "runtimeExecutable": "node",
-        "program": "${workspaceFolder}/src/index.js"
+        "runtimeArgs": ["--nolazy", "-r", "ts-node/register/transpile-only"],
+        "args": ["src/index.ts"],
+        "cwd": "${workspaceRoot}",
       },
       {
         "type": "node",

packages/server/nodemon.json

@@ -0,0 +1,6 @@
+{
+  "watch": ["src"],
+  "ext": "js,ts,json",
+  "ignore": ["src/**/*.spec.ts", "src/**/*.spec.js"],
+  "exec": "ts-node src/index.ts"
+}

packages/server/package.json

@@ -13,12 +13,13 @@
     "postbuild": "copyfiles -u 1 src/**/*.svelte dist/ && copyfiles -u 1 src/**/*.hbs dist/ && copyfiles -u 1 src/**/*.json dist/",
     "test": "jest --coverage --maxWorkers=2",
     "test:watch": "jest --watch",
-    "build:docker": "docker build . -t app-service",
+    "predocker": "copyfiles -f ../client/dist/budibase-client.js ../standard-components/manifest.json client",
+    "build:docker": "yarn run predocker && docker build . -t app-service",
     "run:docker": "node dist/index.js",
     "dev:stack:up": "node scripts/dev/manage.js up",
     "dev:stack:down": "node scripts/dev/manage.js down",
     "dev:stack:nuke": "node scripts/dev/manage.js nuke",
-    "dev:builder": "yarn run dev:stack:up && ts-node src/index.ts",
+    "dev:builder": "yarn run dev:stack:up && nodemon",
     "format": "prettier --config ../../.prettierrc.json 'src/**/*.ts' --write",
     "lint": "eslint --fix src/",
     "lint:fix": "yarn run format && yarn run lint",

packages/server/scripts/integrations/mysql/init.sql

@@ -1,9 +1,22 @@
 CREATE DATABASE IF NOT EXISTS main;
 USE main;
 CREATE TABLE Persons (
-    PersonID int NOT NULL PRIMARY KEY,
+    PersonID int NOT NULL AUTO_INCREMENT,
     LastName varchar(255),
     FirstName varchar(255),
     Address varchar(255),
-    City varchar(255)
+    City varchar(255),
+    PRIMARY KEY (PersonID)
 );
+CREATE TABLE Tasks (
+    TaskID int NOT NULL AUTO_INCREMENT,
+    PersonID INT,
+    TaskName varchar(255),
+    PRIMARY KEY (TaskID),
+    CONSTRAINT fkPersons
+        FOREIGN KEY(PersonID)
+	    REFERENCES Persons(PersonID)
+);
+INSERT INTO Persons (FirstName, LastName, Address, City) VALUES ('Mike', 'Hughes', '123 Fake Street', 'Belfast');
+INSERT INTO Tasks (PersonID, TaskName) VALUES (1, 'assembling');
+INSERT INTO Tasks (PersonID, TaskName) VALUES (1, 'processing');

packages/server/scripts/integrations/mysql/reset.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+docker-compose down
+docker volume prune -f

packages/server/scripts/integrations/postgres/init.sql

@@ -1,17 +1,42 @@
 SELECT 'CREATE DATABASE main'
 WHERE NOT EXISTS (SELECT FROM pg_database WHERE datname = 'main')\gexec
 CREATE TABLE Persons (
-    PersonID INT NOT NULL PRIMARY KEY,
+    PersonID SERIAL PRIMARY KEY,
     LastName varchar(255),
     FirstName varchar(255),
     Address varchar(255),
-    City varchar(255)
+    City varchar(255) DEFAULT 'Belfast'
 );
 CREATE TABLE Tasks (
-    TaskID INT NOT NULL PRIMARY KEY,
+    TaskID SERIAL PRIMARY KEY,
     PersonID INT,
     TaskName varchar(255),
     CONSTRAINT fkPersons
         FOREIGN KEY(PersonID)
 	    REFERENCES Persons(PersonID)
 );
+CREATE TABLE Products (
+    ProductID SERIAL PRIMARY KEY,
+    ProductName varchar(255)
+);
+CREATE TABLE Products_Tasks (
+    ProductID INT NOT NULL,
+    TaskID INT NOT NULL,
+    CONSTRAINT fkProducts
+        FOREIGN KEY(ProductID)
+	    REFERENCES Products(ProductID),
+    CONSTRAINT fkTasks
+        FOREIGN KEY(TaskID)
+	    REFERENCES Tasks(TaskID),
+    PRIMARY KEY (ProductID, TaskID)
+);
+INSERT INTO Persons (FirstName, LastName, Address, City) VALUES ('Mike', 'Hughes', '123 Fake Street', 'Belfast');
+INSERT INTO Tasks (PersonID, TaskName) VALUES (1, 'assembling');
+INSERT INTO Tasks (PersonID, TaskName) VALUES (1, 'processing');
+INSERT INTO Products (ProductName) VALUES ('Computers');
+INSERT INTO Products (ProductName) VALUES ('Laptops');
+INSERT INTO Products (ProductName) VALUES ('Chairs');
+INSERT INTO Products_Tasks (ProductID, TaskID) VALUES (1, 1);
+INSERT INTO Products_Tasks (ProductID, TaskID) VALUES (2, 1);
+INSERT INTO Products_Tasks (ProductID, TaskID) VALUES (3, 1);
+INSERT INTO Products_Tasks (ProductID, TaskID) VALUES (1, 2);

packages/server/scripts/integrations/postgres/reset.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+docker-compose down
+docker volume prune -f

packages/server/src/api/controllers/application.js

@@ -33,6 +33,11 @@ const {
 } = require("../../utilities/workerRequests")
 const { clientLibraryPath } = require("../../utilities")
 const { getAllLocks } = require("../../utilities/redis")
+const {
+  updateClientLibrary,
+  backupClientLibrary,
+  revertClientLibrary,
+} = require("../../utilities/fileSystem/clientLibrary")
 
 const URL_REGEX_SLASH = /\/|\\/g
 
@@ -67,15 +72,18 @@ async function getAppUrlIfNotInUse(ctx) {
   let url
   if (ctx.request.body.url) {
     url = encodeURI(ctx.request.body.url)
-  } else {
+  } else if (ctx.request.body.name) {
     url = encodeURI(`${ctx.request.body.name}`)
   }
+  if (url) {
     url = `/${url.replace(URL_REGEX_SLASH, "")}`.toLowerCase()
+  }
   if (!env.SELF_HOSTED) {
     return url
   }
   const deployedApps = await getDeployedApps(ctx)
   if (
+    url &&
     deployedApps[url] != null &&
     deployedApps[url].appId !== ctx.params.appId
   ) {
@@ -161,7 +169,15 @@ exports.fetchAppDefinition = async function (ctx) {
 exports.fetchAppPackage = async function (ctx) {
   const db = new CouchDB(ctx.params.appId)
   const application = await db.get(DocumentTypes.APP_METADATA)
-  const [layouts, screens] = await Promise.all([getLayouts(db), getScreens(db)])
+  const layouts = await getLayouts(db)
+  let screens = await getScreens(db)
+
+  // Only filter screens if the user is not a builder
+  if (!(ctx.user.builder && ctx.user.builder.global)) {
+    const userRoleId = getUserRoleId(ctx)
+    const accessController = new AccessController(ctx.params.appId)
+    screens = await accessController.checkScreensAccess(screens, userRoleId)
+  }
 
   ctx.body = {
     application,
@@ -220,27 +236,54 @@ exports.create = async function (ctx) {
 }
 
 exports.update = async function (ctx) {
-  const url = await getAppUrlIfNotInUse(ctx)
+  const data = await updateAppPackage(ctx, ctx.request.body, ctx.params.appId)
+  ctx.status = 200
+  ctx.body = data
+}
+
+exports.updateClient = async function (ctx) {
+  // Get current app version
   const db = new CouchDB(ctx.params.appId)
   const application = await db.get(DocumentTypes.APP_METADATA)
+  const currentVersion = application.version
 
-  const data = ctx.request.body
+  // Update client library and manifest
-  const newData = { ...application, ...data, url }
+  if (!env.isTest()) {
-  if (ctx.request.body._rev !== application._rev) {
+    await backupClientLibrary(ctx.params.appId)
-    newData._rev = application._rev
+    await updateClientLibrary(ctx.params.appId)
   }
 
-  // the locked by property is attached by server but generated from
+  // Update versions in app package
-  // Redis, shouldn't ever store it
+  const appPackageUpdates = {
-  if (newData.lockedBy) {
+    version: packageJson.version,
-    delete newData.lockedBy
+    revertableVersion: currentVersion,
   }
-
+  const data = await updateAppPackage(ctx, appPackageUpdates, ctx.params.appId)
-  const response = await db.put(newData)
-  data._rev = response.rev
-
   ctx.status = 200
-  ctx.body = response
+  ctx.body = data
+}
+
+exports.revertClient = async function (ctx) {
+  // Check app can be reverted
+  const db = new CouchDB(ctx.params.appId)
+  const application = await db.get(DocumentTypes.APP_METADATA)
+  if (!application.revertableVersion) {
+    ctx.throw(400, "There is no version to revert to")
+  }
+
+  // Update client library and manifest
+  if (!env.isTest()) {
+    await revertClientLibrary(ctx.params.appId)
+  }
+
+  // Update versions in app package
+  const appPackageUpdates = {
+    version: application.revertableVersion,
+    revertableVersion: null,
+  }
+  const data = await updateAppPackage(ctx, appPackageUpdates, ctx.params.appId)
+  ctx.status = 200
+  ctx.body = data
 }
 
 exports.delete = async function (ctx) {
@@ -258,6 +301,23 @@ exports.delete = async function (ctx) {
   ctx.body = result
 }
 
+const updateAppPackage = async (ctx, appPackage, appId) => {
+  const url = await getAppUrlIfNotInUse(ctx)
+  const db = new CouchDB(appId)
+  const application = await db.get(DocumentTypes.APP_METADATA)
+
+  const newAppPackage = { ...application, ...appPackage, url }
+  if (appPackage._rev !== application._rev) {
+    newAppPackage._rev = application._rev
+  }
+
+  // the locked by property is attached by server but generated from
+  // Redis, shouldn't ever store it
+  delete newAppPackage.lockedBy
+
+  return await db.put(newAppPackage)
+}
+
 const createEmptyAppPackage = async (ctx, app) => {
   const db = new CouchDB(app.appId)
 

packages/server/src/api/controllers/component.js

@@ -20,6 +20,9 @@ exports.fetchAppComponentDefinitions = async function (ctx) {
   const definitions = {}
   for (let { manifest, library } of componentManifests) {
     for (let key of Object.keys(manifest)) {
+      if (key === "features") {
+        definitions[key] = manifest[key]
+      } else {
         const fullComponentName = `${library}/${key}`.toLowerCase()
         definitions[fullComponentName] = {
           component: fullComponentName,
@@ -27,5 +30,6 @@ exports.fetchAppComponentDefinitions = async function (ctx) {
         }
       }
     }
+  }
   ctx.body = definitions
 }

packages/server/src/api/controllers/datasource.js

@@ -48,7 +48,7 @@ exports.buildSchemaFromDb = async function (ctx) {
 
   // Connect to the DB and build the schema
   const connector = new Connector(datasource.config)
-  await connector.buildSchema(datasource._id)
+  await connector.buildSchema(datasource._id, datasource.entities)
   datasource.entities = connector.tables
 
   const response = await db.post(datasource)

packages/server/src/api/controllers/row/ExternalRequest.ts

@@ -0,0 +1,525 @@
+import {
+  Operation,
+  SearchFilters,
+  SortJson,
+  PaginationJson,
+  RelationshipsJson,
+} from "../../../definitions/datasource"
+import {
+  Row,
+  Table,
+  FieldSchema,
+  Datasource,
+} from "../../../definitions/common"
+import {
+  breakRowIdField,
+  generateRowIdField,
+} from "../../../integrations/utils"
+
+interface ManyRelationship {
+  tableId?: string
+  id?: string
+  isUpdate?: boolean
+  [key: string]: any
+}
+
+interface RunConfig {
+  id: string
+  row: Row
+  filters: SearchFilters
+  sort: SortJson
+  paginate: PaginationJson
+}
+
+module External {
+  const { makeExternalQuery } = require("./utils")
+  const { DataSourceOperation, FieldTypes } = require("../../../constants")
+  const { breakExternalTableId, isSQL } = require("../../../integrations/utils")
+  const { processObjectSync } = require("@budibase/string-templates")
+  const { cloneDeep } = require("lodash/fp")
+  const { isEqual } = require("lodash")
+  const CouchDB = require("../../../db")
+
+  function buildFilters(
+    id: string | undefined,
+    filters: SearchFilters,
+    table: Table
+  ) {
+    const primary = table.primary
+    // if passed in array need to copy for shifting etc
+    let idCopy = cloneDeep(id)
+    if (filters) {
+      // need to map over the filters and make sure the _id field isn't present
+      for (let filter of Object.values(filters)) {
+        if (filter._id && primary) {
+          const parts = breakRowIdField(filter._id)
+          for (let field of primary) {
+            filter[field] = parts.shift()
+          }
+        }
+        // make sure this field doesn't exist on any filter
+        delete filter._id
+      }
+    }
+    // there is no id, just use the user provided filters
+    if (!idCopy || !table) {
+      return filters
+    }
+    // if used as URL parameter it will have been joined
+    if (!Array.isArray(idCopy)) {
+      idCopy = breakRowIdField(idCopy)
+    }
+    const equal: any = {}
+    if (primary && idCopy) {
+      for (let field of primary) {
+        // work through the ID and get the parts
+        equal[field] = idCopy.shift()
+      }
+    }
+    return {
+      equal,
+    }
+  }
+
+  function generateIdForRow(row: Row, table: Table): string {
+    const primary = table.primary
+    if (!row || !primary) {
+      return ""
+    }
+    // build id array
+    let idParts = []
+    for (let field of primary) {
+      if (row[field]) {
+        idParts.push(row[field])
+      }
+    }
+    if (idParts.length === 0) {
+      return ""
+    }
+    return generateRowIdField(idParts)
+  }
+
+  function getEndpoint(tableId: string | undefined, operation: string) {
+    if (!tableId) {
+      return {}
+    }
+    const { datasourceId, tableName } = breakExternalTableId(tableId)
+    return {
+      datasourceId,
+      entityId: tableName,
+      operation,
+    }
+  }
+
+  function basicProcessing(row: Row, table: Table) {
+    const thisRow: { [key: string]: any } = {}
+    // filter the row down to what is actually the row (not joined)
+    for (let fieldName of Object.keys(table.schema)) {
+      thisRow[fieldName] = row[fieldName]
+    }
+    thisRow._id = generateIdForRow(row, table)
+    thisRow.tableId = table._id
+    thisRow._rev = "rev"
+    return thisRow
+  }
+
+  function isMany(field: FieldSchema) {
+    return (
+      field.relationshipType && field.relationshipType.split("-")[0] === "many"
+    )
+  }
+
+  class ExternalRequest {
+    private readonly appId: string
+    private operation: Operation
+    private tableId: string
+    private datasource: Datasource
+    private tables: { [key: string]: Table } = {}
+
+    constructor(
+      appId: string,
+      operation: Operation,
+      tableId: string,
+      datasource: Datasource
+    ) {
+      this.appId = appId
+      this.operation = operation
+      this.tableId = tableId
+      this.datasource = datasource
+      if (datasource && datasource.entities) {
+        this.tables = datasource.entities
+      }
+    }
+
+    inputProcessing(row: Row, table: Table) {
+      if (!row) {
+        return { row, manyRelationships: [] }
+      }
+      // we don't really support composite keys for relationships, this is why [0] is used
+      // @ts-ignore
+      const tablePrimary: string = table.primary[0]
+      let newRow: Row = {},
+        manyRelationships: ManyRelationship[] = []
+      for (let [key, field] of Object.entries(table.schema)) {
+        // if set already, or not set just skip it
+        if (!row[key] || newRow[key] || field.autocolumn) {
+          continue
+        }
+        // if its not a link then just copy it over
+        if (field.type !== FieldTypes.LINK) {
+          newRow[key] = row[key]
+          continue
+        }
+        const { tableName: linkTableName } = breakExternalTableId(field.tableId)
+        // table has to exist for many to many
+        if (!this.tables[linkTableName]) {
+          continue
+        }
+        const linkTable = this.tables[linkTableName]
+        // @ts-ignore
+        const linkTablePrimary = linkTable.primary[0]
+        if (!isMany(field)) {
+          newRow[field.foreignKey || linkTablePrimary] = breakRowIdField(
+            row[key][0]
+          )[0]
+        } else {
+          // we're not inserting a doc, will be a bunch of update calls
+          const isUpdate = !field.through
+          const thisKey: string = isUpdate ? "id" : linkTablePrimary
+          // @ts-ignore
+          const otherKey: string = isUpdate ? field.foreignKey : tablePrimary
+          row[key].map((relationship: any) => {
+            // we don't really support composite keys for relationships, this is why [0] is used
+            manyRelationships.push({
+              tableId: field.through || field.tableId,
+              isUpdate,
+              [thisKey]: breakRowIdField(relationship)[0],
+              // leave the ID for enrichment later
+              [otherKey]: `{{ literal ${tablePrimary} }}`,
+            })
+          })
+        }
+      }
+      // we return the relationships that may need to be created in the through table
+      // we do this so that if the ID is generated by the DB it can be inserted
+      // after the fact
+      return { row: newRow, manyRelationships }
+    }
+
+    /**
+     * This iterates through the returned rows and works out what elements of the rows
+     * actually match up to another row (based on primary keys) - this is pretty specific
+     * to SQL and the way that SQL relationships are returned based on joins.
+     */
+    updateRelationshipColumns(
+      row: Row,
+      rows: { [key: string]: Row },
+      relationships: RelationshipsJson[]
+    ) {
+      const columns: { [key: string]: any } = {}
+      for (let relationship of relationships) {
+        const linkedTable = this.tables[relationship.tableName]
+        if (!linkedTable) {
+          continue
+        }
+        let linked = basicProcessing(row, linkedTable)
+        if (!linked._id) {
+          continue
+        }
+        // if not returning full docs then get the minimal links out
+        const display = linkedTable.primaryDisplay
+        linked = {
+          primaryDisplay: display ? linked[display] : undefined,
+          _id: linked._id,
+        }
+        columns[relationship.column] = linked
+      }
+      for (let [column, related] of Object.entries(columns)) {
+        if (!row._id) {
+          continue
+        }
+        const rowId: string = row._id
+        if (!Array.isArray(rows[rowId][column])) {
+          rows[rowId][column] = []
+        }
+        // make sure relationship hasn't been found already
+        if (
+          !rows[rowId][column].find(
+            (relation: Row) => relation._id === related._id
+          )
+        ) {
+          rows[rowId][column].push(related)
+        }
+      }
+      return rows
+    }
+
+    outputProcessing(
+      rows: Row[],
+      table: Table,
+      relationships: RelationshipsJson[]
+    ) {
+      if (rows[0].read === true) {
+        return []
+      }
+      let finalRows: { [key: string]: Row } = {}
+      for (let row of rows) {
+        const rowId = generateIdForRow(row, table)
+        row._id = rowId
+        // this is a relationship of some sort
+        if (finalRows[rowId]) {
+          finalRows = this.updateRelationshipColumns(
+            row,
+            finalRows,
+            relationships
+          )
+          continue
+        }
+        const thisRow = basicProcessing(row, table)
+        finalRows[thisRow._id] = thisRow
+        // do this at end once its been added to the final rows
+        finalRows = this.updateRelationshipColumns(
+          row,
+          finalRows,
+          relationships
+        )
+      }
+      return Object.values(finalRows)
+    }
+
+    /**
+     * Gets the list of relationship JSON structures based on the columns in the table,
+     * this will be used by the underlying library to build whatever relationship mechanism
+     * it has (e.g. SQL joins).
+     */
+    buildRelationships(table: Table): RelationshipsJson[] {
+      const relationships = []
+      for (let [fieldName, field] of Object.entries(table.schema)) {
+        if (field.type !== FieldTypes.LINK) {
+          continue
+        }
+        const { tableName: linkTableName } = breakExternalTableId(field.tableId)
+        // no table to link to, this is not a valid relationships
+        if (!this.tables[linkTableName]) {
+          continue
+        }
+        const linkTable = this.tables[linkTableName]
+        if (!table.primary || !linkTable.primary) {
+          continue
+        }
+        const definition = {
+          // if no foreign key specified then use the name of the field in other table
+          from: field.foreignKey || table.primary[0],
+          to: field.fieldName,
+          tableName: linkTableName,
+          through: undefined,
+          // need to specify where to put this back into
+          column: fieldName,
+        }
+        if (field.through) {
+          const { tableName: throughTableName } = breakExternalTableId(
+            field.through
+          )
+          definition.through = throughTableName
+          // don't support composite keys for relationships
+          definition.from = table.primary[0]
+          definition.to = linkTable.primary[0]
+        }
+        relationships.push(definition)
+      }
+      return relationships
+    }
+
+    /**
+     * This is a cached lookup, of relationship records, this is mainly for creating/deleting junction
+     * information.
+     */
+    async lookup(
+      row: Row,
+      relationship: ManyRelationship,
+      cache: { [key: string]: Row[] } = {}
+    ) {
+      const { tableId, isUpdate, id, ...rest } = relationship
+      const { tableName } = breakExternalTableId(tableId)
+      const table = this.tables[tableName]
+      if (isUpdate) {
+        return { rows: [], table }
+      }
+      // if not updating need to make sure we have a list of all possible options
+      let fullKey: string = tableId + "/",
+        rowKey: string = ""
+      for (let key of Object.keys(rest)) {
+        if (row[key]) {
+          fullKey += key
+          rowKey = key
+        }
+      }
+      if (cache[fullKey] == null) {
+        cache[fullKey] = await makeExternalQuery(this.appId, {
+          endpoint: getEndpoint(tableId, DataSourceOperation.READ),
+          filters: {
+            equal: {
+              [rowKey]: row[rowKey],
+            },
+          },
+        })
+      }
+      return { rows: cache[fullKey], table }
+    }
+
+    /**
+     * Once a row has been written we may need to update a many field, e.g. updating foreign keys
+     * in a bunch of rows in another table, or inserting/deleting rows from a junction table (many to many).
+     * This is quite a complex process and is handled by this function, there are a few things going on here:
+     * 1. If updating foreign keys its relatively simple, just create a filter for the row that needs updated
+     * and write the various components.
+     * 2. If junction table, then we lookup what exists already, write what doesn't exist, work out what
+     * isn't supposed to exist anymore and delete those. This is better than the usual method of delete them
+     * all and then re-create, as theres no chance of losing data (e.g. delete succeed, but write fail).
+     */
+    async handleManyRelationships(row: Row, relationships: ManyRelationship[]) {
+      const { appId } = this
+      if (relationships.length === 0) {
+        return
+      }
+      // if we're creating (in a through table) need to wipe the existing ones first
+      const promises = []
+      const cache: { [key: string]: Row[] } = {}
+      for (let relationship of relationships) {
+        const { tableId, isUpdate, id, ...rest } = relationship
+        const body = processObjectSync(rest, row)
+        const { table, rows } = await this.lookup(row, relationship, cache)
+        const found = rows.find(row => isEqual(body, row))
+        const operation = isUpdate
+          ? DataSourceOperation.UPDATE
+          : DataSourceOperation.CREATE
+        if (!found) {
+          promises.push(
+            makeExternalQuery(appId, {
+              endpoint: getEndpoint(tableId, operation),
+              // if we're doing many relationships then we're writing, only one response
+              body,
+              filters: buildFilters(id, {}, table),
+            })
+          )
+        } else {
+          // remove the relationship from the rows
+          rows.splice(rows.indexOf(found), 1)
+        }
+      }
+      // finally if creating, cleanup any rows that aren't supposed to be here
+      for (let [key, rows] of Object.entries(cache)) {
+        // @ts-ignore
+        const tableId: string = key.split("/").shift()
+        const { tableName } = breakExternalTableId(tableId)
+        const table = this.tables[tableName]
+        for (let row of rows) {
+          promises.push(
+            makeExternalQuery(this.appId, {
+              endpoint: getEndpoint(tableId, DataSourceOperation.DELETE),
+              filters: buildFilters(generateIdForRow(row, table), {}, table),
+            })
+          )
+        }
+      }
+      await Promise.all(promises)
+    }
+
+    /**
+     * This function is a bit crazy, but the exact purpose of it is to protect against the scenario in which
+     * you have column overlap in relationships, e.g. we join a few different tables and they all have the
+     * concept of an ID, but for some of them it will be null (if they say don't have a relationship).
+     * Creating the specific list of fields that we desire, and excluding the ones that are no use to us
+     * is more performant and has the added benefit of protecting against this scenario.
+     */
+    buildFields(table: Table) {
+      function extractNonLinkFieldNames(table: Table, existing: string[] = []) {
+        return Object.entries(table.schema)
+          .filter(
+            column =>
+              column[1].type !== FieldTypes.LINK &&
+              !existing.find((field: string) => field.includes(column[0]))
+          )
+          .map(column => `${table.name}.${column[0]}`)
+      }
+      let fields = extractNonLinkFieldNames(table)
+      for (let field of Object.values(table.schema)) {
+        if (field.type !== FieldTypes.LINK) {
+          continue
+        }
+        const { tableName: linkTableName } = breakExternalTableId(field.tableId)
+        const linkTable = this.tables[linkTableName]
+        if (linkTable) {
+          const linkedFields = extractNonLinkFieldNames(linkTable, fields)
+          fields = fields.concat(linkedFields)
+        }
+      }
+      return fields
+    }
+
+    async run({ id, row, filters, sort, paginate }: RunConfig) {
+      const { appId, operation, tableId } = this
+      let { datasourceId, tableName } = breakExternalTableId(tableId)
+      if (!this.datasource) {
+        const db = new CouchDB(appId)
+        this.datasource = await db.get(datasourceId)
+        if (!this.datasource || !this.datasource.entities) {
+          throw "No tables found, fetch tables before query."
+        }
+        this.tables = this.datasource.entities
+      }
+      const table = this.tables[tableName]
+      let isSql = isSQL(this.datasource)
+      if (!table) {
+        throw `Unable to process query, table "${tableName}" not defined.`
+      }
+      // clean up row on ingress using schema
+      filters = buildFilters(id, filters, table)
+      const relationships = this.buildRelationships(table)
+      const processed = this.inputProcessing(row, table)
+      row = processed.row
+      if (
+        operation === DataSourceOperation.DELETE &&
+        (filters == null || Object.keys(filters).length === 0)
+      ) {
+        throw "Deletion must be filtered"
+      }
+      let json = {
+        endpoint: {
+          datasourceId,
+          entityId: tableName,
+          operation,
+        },
+        resource: {
+          // have to specify the fields to avoid column overlap (for SQL)
+          fields: isSql ? this.buildFields(table) : [],
+        },
+        filters,
+        sort,
+        paginate,
+        relationships,
+        body: row,
+        // pass an id filter into extra, purely for mysql/returning
+        extra: {
+          idFilter: buildFilters(id || generateIdForRow(row, table), {}, table),
+        },
+      }
+      // can't really use response right now
+      const response = await makeExternalQuery(appId, json)
+      // handle many to many relationships now if we know the ID (could be auto increment)
+      if (processed.manyRelationships) {
+        await this.handleManyRelationships(
+          response[0],
+          processed.manyRelationships
+        )
+      }
+      const output = this.outputProcessing(response, table, relationships)
+      // if reading it'll just be an array of rows, return whole thing
+      return operation === DataSourceOperation.READ && Array.isArray(response)
+        ? output
+        : { row: output[0], table }
+    }
+  }
+
+  module.exports = ExternalRequest
+}

packages/server/src/api/controllers/row/external.js

@@ -1,136 +1,19 @@
-const { makeExternalQuery } = require("./utils")
+const {
-const { DataSourceOperation, SortDirection } = require("../../../constants")
+  DataSourceOperation,
-const { getExternalTable } = require("../table/utils")
+  SortDirection,
+  FieldTypes,
+} = require("../../../constants")
 const {
   breakExternalTableId,
-  generateRowIdField,
   breakRowIdField,
 } = require("../../../integrations/utils")
-const { cloneDeep } = require("lodash/fp")
+const ExternalRequest = require("./ExternalRequest")
+const CouchDB = require("../../../db")
 
-function inputProcessing(row, table) {
+async function handleRequest(appId, operation, tableId, opts = {}) {
-  if (!row) {
+  return new ExternalRequest(appId, operation, tableId, opts.datasource).run(
-    return row
+    opts
-  }
+  )
-  let newRow = {}
-  for (let key of Object.keys(table.schema)) {
-    // currently excludes empty strings
-    if (row[key]) {
-      newRow[key] = row[key]
-    }
-  }
-  return newRow
-}
-
-function generateIdForRow(row, table) {
-  if (!row) {
-    return
-  }
-  const primary = table.primary
-  // build id array
-  let idParts = []
-  for (let field of primary) {
-    idParts.push(row[field])
-  }
-  return generateRowIdField(idParts)
-}
-
-function outputProcessing(rows, table) {
-  // if no rows this is what is returned? Might be PG only
-  if (rows[0].read === true) {
-    return []
-  }
-  for (let row of rows) {
-    row._id = generateIdForRow(row, table)
-    row.tableId = table._id
-    row._rev = "rev"
-  }
-  return rows
-}
-
-function buildFilters(id, filters, table) {
-  const primary = table.primary
-  // if passed in array need to copy for shifting etc
-  let idCopy = cloneDeep(id)
-  if (filters) {
-    // need to map over the filters and make sure the _id field isn't present
-    for (let filter of Object.values(filters)) {
-      if (filter._id) {
-        const parts = breakRowIdField(filter._id)
-        for (let field of primary) {
-          filter[field] = parts.shift()
-        }
-      }
-      // make sure this field doesn't exist on any filter
-      delete filter._id
-    }
-  }
-  // there is no id, just use the user provided filters
-  if (!idCopy || !table) {
-    return filters
-  }
-  // if used as URL parameter it will have been joined
-  if (typeof idCopy === "string") {
-    idCopy = breakRowIdField(idCopy)
-  }
-  const equal = {}
-  for (let field of primary) {
-    // work through the ID and get the parts
-    equal[field] = idCopy.shift()
-  }
-  return {
-    equal,
-  }
-}
-
-async function handleRequest(
-  appId,
-  operation,
-  tableId,
-  { id, row, filters, sort, paginate } = {}
-) {
-  let { datasourceId, tableName } = breakExternalTableId(tableId)
-  const table = await getExternalTable(appId, datasourceId, tableName)
-  if (!table) {
-    throw `Unable to process query, table "${tableName}" not defined.`
-  }
-  // clean up row on ingress using schema
-  filters = buildFilters(id, filters, table)
-  row = inputProcessing(row, table)
-  if (
-    operation === DataSourceOperation.DELETE &&
-    (filters == null || Object.keys(filters).length === 0)
-  ) {
-    throw "Deletion must be filtered"
-  }
-  let json = {
-    endpoint: {
-      datasourceId,
-      entityId: tableName,
-      operation,
-    },
-    resource: {
-      // not specifying any fields means "*"
-      fields: [],
-    },
-    filters,
-    sort,
-    paginate,
-    body: row,
-    // pass an id filter into extra, purely for mysql/returning
-    extra: {
-      idFilter: buildFilters(id || generateIdForRow(row, table), {}, table),
-    },
-  }
-  // can't really use response right now
-  const response = await makeExternalQuery(appId, json)
-  // we searched for rows in someway
-  if (operation === DataSourceOperation.READ && Array.isArray(response)) {
-    return outputProcessing(response, table)
-  } else {
-    row = outputProcessing(response, table)[0]
-    return { row, table }
-  }
 }
 
 exports.patch = async ctx => {
@@ -172,9 +55,15 @@ exports.find = async ctx => {
   const appId = ctx.appId
   const id = ctx.params.rowId
   const tableId = ctx.params.tableId
-  return handleRequest(appId, DataSourceOperation.READ, tableId, {
+  const response = await handleRequest(
+    appId,
+    DataSourceOperation.READ,
+    tableId,
+    {
       id,
-  })
+    }
+  )
+  return response ? response[0] : response
 }
 
 exports.destroy = async ctx => {
@@ -270,7 +159,56 @@ exports.validate = async () => {
   return { valid: true }
 }
 
-exports.fetchEnrichedRow = async () => {
+exports.fetchEnrichedRow = async ctx => {
-  // TODO: How does this work
+  const appId = ctx.appId
-  throw "Not Implemented"
+  const id = ctx.params.rowId
+  const tableId = ctx.params.tableId
+  const { datasourceId, tableName } = breakExternalTableId(tableId)
+  const db = new CouchDB(appId)
+  const datasource = await db.get(datasourceId)
+  if (!datasource || !datasource.entities) {
+    ctx.throw(400, "Datasource has not been configured for plus API.")
+  }
+  const tables = datasource.entities
+  const response = await handleRequest(
+    appId,
+    DataSourceOperation.READ,
+    tableId,
+    {
+      id,
+      datasource,
+    }
+  )
+  const table = tables[tableName]
+  const row = response[0]
+  // this seems like a lot of work, but basically we need to dig deeper for the enrich
+  // for a single row, there is probably a better way to do this with some smart multi-layer joins
+  for (let [fieldName, field] of Object.entries(table.schema)) {
+    if (
+      field.type !== FieldTypes.LINK ||
+      !row[fieldName] ||
+      row[fieldName].length === 0
+    ) {
+      continue
+    }
+    const links = row[fieldName]
+    const linkedTableId = field.tableId
+    const linkedTable = tables[breakExternalTableId(linkedTableId).tableName]
+    // don't support composite keys right now
+    const linkedIds = links.map(link => breakRowIdField(link._id)[0])
+    row[fieldName] = await handleRequest(
+      appId,
+      DataSourceOperation.READ,
+      linkedTableId,
+      {
+        tables,
+        filters: {
+          oneOf: {
+            [linkedTable.primary]: linkedIds,
+          },
+        },
+      }
+    )
+  }
+  return row
 }

packages/server/src/api/controllers/table/utils.js

@@ -204,15 +204,18 @@ class TableSaveFunctions {
   }
 }
 
-exports.getExternalTable = async (appId, datasourceId, tableName) => {
+exports.getAllExternalTables = async (appId, datasourceId) => {
   const db = new CouchDB(appId)
   const datasource = await db.get(datasourceId)
   if (!datasource || !datasource.entities) {
     throw "Datasource is not configured fully."
   }
-  return Object.values(datasource.entities).find(
+  return datasource.entities
-    entity => entity.name === tableName
+}
-  )
+
+exports.getExternalTable = async (appId, datasourceId, tableName) => {
+  const entities = await exports.getAllExternalTables(appId, datasourceId)
+  return entities[tableName]
 }
 
 exports.TableSaveFunctions = TableSaveFunctions

packages/server/src/api/routes/application.js

@@ -11,6 +11,16 @@ router
   .get("/api/applications/:appId/appPackage", controller.fetchAppPackage)
   .put("/api/applications/:appId", authorized(BUILDER), controller.update)
   .post("/api/applications", authorized(BUILDER), controller.create)
+  .post(
+    "/api/applications/:appId/client/update",
+    authorized(BUILDER),
+    controller.updateClient
+  )
+  .post(
+    "/api/applications/:appId/client/revert",
+    authorized(BUILDER),
+    controller.revertClient
+  )
   .delete("/api/applications/:appId", authorized(BUILDER), controller.delete)
 
 module.exports = router

packages/server/src/api/routes/tests/application.spec.js

@@ -94,7 +94,7 @@ describe("/applications", () => {
   })
 
   describe("update", () => {
-    it("should be able to fetch the app package", async () => {
+    it("should be able to update the app package", async () => {
       const res = await request
         .put(`/api/applications/${config.getAppId()}`)
         .send({
@@ -107,6 +107,30 @@ describe("/applications", () => {
     })
   })
 
+  describe("manage client library version", () => {
+    it("should be able to update the app client library version", async () => {
+      console.log(config.getAppId())
+      await request
+        .post(`/api/applications/${config.getAppId()}/client/update`)
+        .set(config.defaultHeaders())
+        .expect("Content-Type", /json/)
+        .expect(200)
+    })
+    it("should be able to revert the app client library version", async () => {
+      // We need to first update the version so that we can then revert
+      await request
+        .post(`/api/applications/${config.getAppId()}/client/update`)
+        .set(config.defaultHeaders())
+        .expect("Content-Type", /json/)
+        .expect(200)
+      await request
+        .post(`/api/applications/${config.getAppId()}/client/revert`)
+        .set(config.defaultHeaders())
+        .expect("Content-Type", /json/)
+        .expect(200)
+    })
+  })
+
   describe("edited at", () => {
     it("middleware should set edited at", async () => {
       const headers = config.defaultHeaders()

packages/server/src/api/routes/tests/datasource.spec.js

@@ -94,7 +94,7 @@ describe("/datasources", () => {
         .expect(200)
       // this is mock data, can't test it
       expect(res.body).toBeDefined()
-      expect(pg.queryMock).toHaveBeenCalledWith(`select "name", "age" from "users" where "name" like $1 limit $2`, ["John%", 5000])
+      expect(pg.queryMock).toHaveBeenCalledWith(`select "name", "age" from "users" where "users"."name" like $1 limit $2`, ["John%", 5000])
     })
   })
 

packages/server/src/api/routes/tests/routing.spec.js

@@ -2,7 +2,6 @@ const setup = require("./utilities")
 const { basicScreen } = setup.structures
 const { checkBuilderEndpoint } = require("./utilities/TestFunctions")
 const { BUILTIN_ROLE_IDS } = require("@budibase/auth/roles")
-const workerRequests = require("../../../utilities/workerRequests")
 
 const route = "/test"
 

packages/server/src/constants/definitions.ts

@@ -26,3 +26,17 @@ export interface Table {
   primaryDisplay?: string
   sourceId?: string
 }
+
+export interface BudibaseAppMetadata {
+  _id: string
+  _rev?: string
+  appId: string
+  type: string
+  version: string
+  componentlibraries: string[]
+  name: string
+  url: string
+  instance: { _id: string }
+  updatedAt: Date
+  createdAt: Date
+}

packages/server/src/constants/layouts.js

@@ -15,6 +15,7 @@ const EMPTY_LAYOUT = {
       {
         _id: "7fcf11e4-6f5b-4085-8e0d-9f3d44c98967",
         _component: "@budibase/standard-components/screenslot",
+        _instanceName: "Screen slot",
         _styles: {
           normal: {
             flex: "1 1 auto",
@@ -63,6 +64,7 @@ const BASE_LAYOUTS = [
         {
           _id: "7fcf11e4-6f5b-4085-8e0d-9f3d44c98967",
           _component: "@budibase/standard-components/screenslot",
+          _instanceName: "Screen slot",
           _styles: {
             normal: {
               flex: "1 1 auto",
@@ -84,6 +86,7 @@ const BASE_LAYOUTS = [
         normal: {},
         selected: {},
       },
+      title: "{{ name }}",
       navigation: "Top",
       width: "Large",
       links: [
@@ -109,6 +112,7 @@ const BASE_LAYOUTS = [
         {
           _id: "7fcf11e4-6f5b-4085-8e0d-9f3d44c98967",
           _component: "@budibase/standard-components/screenslot",
+          _instanceName: "Screen slot",
           _styles: {
             normal: {
               flex: "1 1 auto",

packages/server/src/definitions/common.ts

@@ -0,0 +1,101 @@
+import { SourceNames } from "./datasource"
+
+interface Base {
+  _id?: string
+  _rev?: string
+}
+
+export interface FieldSchema {
+  // TODO: replace with field types enum when done
+  type: string
+  fieldName?: string
+  name: string
+  tableId?: string
+  relationshipType?: string
+  through?: string
+  foreignKey?: string
+  autocolumn?: boolean
+  constraints?: {
+    type?: string
+    email?: boolean
+    inclusion?: string[]
+    length?: {
+      minimum?: string | number
+      maximum?: string | number
+    }
+    presence?: boolean
+  }
+}
+
+export interface TableSchema {
+  [key: string]: FieldSchema
+}
+
+export interface Table extends Base {
+  type?: string
+  views?: {}
+  name?: string
+  primary?: string[]
+  schema: TableSchema
+  primaryDisplay?: string
+  sourceId?: string
+}
+
+export interface Row extends Base {
+  type?: string
+  tableId?: string
+  [key: string]: any
+}
+
+interface JsonSchemaField {
+  properties: {
+    [key: string]: {
+      type: string
+      title: string
+      customType?: string
+    }
+  }
+  required?: string[]
+}
+
+export interface AutomationStep {
+  description: string
+  event?: string
+  icon: string
+  id: string
+  inputs: {
+    [key: string]: any
+  }
+  name: string
+  schema: {
+    inputs: JsonSchemaField
+    outputs: JsonSchemaField
+  }
+  stepId: string
+  tagline: string
+  type: string
+}
+
+export interface Automation extends Base {
+  name: string
+  type: string
+  appId?: string
+  definition: {
+    steps: AutomationStep[]
+    trigger?: AutomationStep
+  }
+}
+
+export interface Datasource extends Base {
+  type: string
+  name: string
+  source: SourceNames
+  // the config is defined by the schema
+  config: {
+    [key: string]: string | number | boolean
+  }
+  plus: boolean
+  entities?: {
+    [key: string]: Table
+  }
+}

packages/server/src/definitions/datasource.ts

@@ -26,6 +26,20 @@ export enum DatasourceFieldTypes {
   JSON = "json",
 }
 
+export enum SourceNames {
+  POSTGRES = "POSTGRES",
+  DYNAMODB = "DYNAMODB",
+  MONGODB = "MONGODB",
+  ELASTICSEARCH = "ELASTICSEARCH",
+  COUCHDB = "COUCHDB",
+  SQL_SERVER = "SQL_SERVER",
+  S3 = "S3",
+  AIRTABLE = "AIRTABLE",
+  MYSQL = "MYSQL",
+  ARANGODB = "ARANGODB",
+  REST = "REST",
+}
+
 export interface QueryDefinition {
   type: QueryTypes
   displayName?: string
@@ -47,7 +61,7 @@ export interface Integration {
 }
 
 export interface SearchFilters {
-  allOr: boolean
+  allOr?: boolean
   string?: {
     [key: string]: string
   }
@@ -72,6 +86,26 @@ export interface SearchFilters {
   notEmpty?: {
     [key: string]: any
   }
+  oneOf?: {
+    [key: string]: any[]
+  }
+}
+
+export interface SortJson {
+  [key: string]: SortDirection
+}
+
+export interface PaginationJson {
+  limit: number
+  page: string | number
+}
+
+export interface RelationshipsJson {
+  through?: string
+  from?: string
+  to?: string
+  tableName: string
+  column: string
 }
 
 export interface QueryJson {
@@ -84,17 +118,13 @@ export interface QueryJson {
     fields: string[]
   }
   filters?: SearchFilters
-  sort?: {
+  sort?: SortJson
-    [key: string]: SortDirection
+  paginate?: PaginationJson
-  }
-  paginate?: {
-    limit: number
-    page: string | number
-  }
   body?: object
-  extra: {
+  extra?: {
     idFilter?: SearchFilters
   }
+  relationships?: RelationshipsJson[]
 }
 
 export interface SqlQuery {

packages/server/src/integrations/airtable.ts

@@ -2,7 +2,7 @@ import {
   Integration,
   DatasourceFieldTypes,
   QueryTypes,
-} from "./base/definitions"
+} from "../definitions/datasource"
 
 module AirtableModule {
   const Airtable = require("airtable")

packages/server/src/integrations/arangodb.ts

@@ -2,7 +2,7 @@ import {
   Integration,
   DatasourceFieldTypes,
   QueryTypes,
-} from "./base/definitions"
+} from "../definitions/datasource"
 
 module ArangoModule {
   const { Database, aql } = require("arangojs")

packages/server/src/integrations/base/sql.ts

@@ -6,18 +6,23 @@ import {
   QueryOptions,
   SortDirection,
   Operation,
-} from "./definitions"
+  RelationshipsJson,
+} from "../../definitions/datasource"
 
+type KnexQuery = Knex.QueryBuilder | Knex
+
+// right now we only do filters on the specific table being queried
 function addFilters(
-  query: any,
+  tableName: string,
+  query: KnexQuery,
   filters: SearchFilters | undefined
-): Knex.QueryBuilder {
+): KnexQuery {
   function iterate(
     structure: { [key: string]: any },
     fn: (key: string, value: any) => void
   ) {
     for (let [key, value] of Object.entries(structure)) {
-      fn(key, value)
+      fn(`${tableName}.${key}`, value)
     }
   }
   if (!filters) {
@@ -25,6 +30,12 @@ function addFilters(
   }
   // if all or specified in filters, then everything is an or
   const allOr = filters.allOr
+  if (filters.oneOf) {
+    iterate(filters.oneOf, (key, array) => {
+      const fnc = allOr ? "orWhereIn" : "whereIn"
+      query = query[fnc](key, array)
+    })
+  }
   if (filters.string) {
     iterate(filters.string, (key, value) => {
       const fnc = allOr ? "orWhere" : "where"
@@ -67,9 +78,47 @@ function addFilters(
   return query
 }
 
-function buildCreate(knex: Knex, json: QueryJson, opts: QueryOptions) {
+function addRelationships(
+  query: KnexQuery,
+  fromTable: string,
+  relationships: RelationshipsJson[] | undefined
+): KnexQuery {
+  if (!relationships) {
+    return query
+  }
+  for (let relationship of relationships) {
+    const from = relationship.from,
+      to = relationship.to,
+      toTable = relationship.tableName
+    if (!relationship.through) {
+      // @ts-ignore
+      query = query.leftJoin(
+        toTable,
+        `${fromTable}.${from}`,
+        `${relationship.tableName}.${to}`
+      )
+    } else {
+      const throughTable = relationship.through
+      query = query
+        // @ts-ignore
+        .leftJoin(
+          throughTable,
+          `${fromTable}.${from}`,
+          `${throughTable}.${from}`
+        )
+        .leftJoin(toTable, `${toTable}.${to}`, `${throughTable}.${to}`)
+    }
+  }
+  return query
+}
+
+function buildCreate(
+  knex: Knex,
+  json: QueryJson,
+  opts: QueryOptions
+): KnexQuery {
   const { endpoint, body } = json
-  let query = knex(endpoint.entityId)
+  let query: KnexQuery = knex(endpoint.entityId)
   // mysql can't use returning
   if (opts.disableReturning) {
     return query.insert(body)
@@ -78,9 +127,10 @@ function buildCreate(knex: Knex, json: QueryJson, opts: QueryOptions) {
   }
 }
 
-function buildRead(knex: Knex, json: QueryJson, limit: number) {
+function buildRead(knex: Knex, json: QueryJson, limit: number): KnexQuery {
-  let { endpoint, resource, filters, sort, paginate } = json
+  let { endpoint, resource, filters, sort, paginate, relationships } = json
-  let query: Knex.QueryBuilder = knex(endpoint.entityId)
+  const tableName = endpoint.entityId
+  let query: KnexQuery = knex(tableName)
   // select all if not specified
   if (!resource) {
     resource = { fields: [] }
@@ -92,7 +142,9 @@ function buildRead(knex: Knex, json: QueryJson, limit: number) {
     query = query.select("*")
   }
   // handle where
-  query = addFilters(query, filters)
+  query = addFilters(tableName, query, filters)
+  // handle join
+  query = addRelationships(query, tableName, relationships)
   // handle sorting
   if (sort) {
     for (let [key, value] of Object.entries(sort)) {
@@ -114,10 +166,14 @@ function buildRead(knex: Knex, json: QueryJson, limit: number) {
   return query
 }
 
-function buildUpdate(knex: Knex, json: QueryJson, opts: QueryOptions) {
+function buildUpdate(
+  knex: Knex,
+  json: QueryJson,
+  opts: QueryOptions
+): KnexQuery {
   const { endpoint, body, filters } = json
-  let query = knex(endpoint.entityId)
+  let query: KnexQuery = knex(endpoint.entityId)
-  query = addFilters(query, filters)
+  query = addFilters(endpoint.entityId, query, filters)
   // mysql can't use returning
   if (opts.disableReturning) {
     return query.update(body)
@@ -126,10 +182,14 @@ function buildUpdate(knex: Knex, json: QueryJson, opts: QueryOptions) {
   }
 }
 
-function buildDelete(knex: Knex, json: QueryJson, opts: QueryOptions) {
+function buildDelete(
+  knex: Knex,
+  json: QueryJson,
+  opts: QueryOptions
+): KnexQuery {
   const { endpoint, filters } = json
-  let query = knex(endpoint.entityId)
+  let query: KnexQuery = knex(endpoint.entityId)
-  query = addFilters(query, filters)
+  query = addFilters(endpoint.entityId, query, filters)
   // mysql can't use returning
   if (opts.disableReturning) {
     return query.delete()
@@ -180,6 +240,8 @@ class SqlQueryBuilder {
       default:
         throw `Operation type is not supported by SQL query builder`
     }
+
+    // @ts-ignore
     return query.toSQL().toNative()
   }
 }

packages/server/src/integrations/couchdb.ts

@@ -2,7 +2,7 @@ import {
   Integration,
   DatasourceFieldTypes,
   QueryTypes,
-} from "./base/definitions"
+} from "../definitions/datasource"
 
 module CouchDBModule {
   const PouchDB = require("pouchdb")

packages/server/src/integrations/dynamodb.ts

@@ -2,7 +2,7 @@ import {
   Integration,
   DatasourceFieldTypes,
   QueryTypes,
-} from "./base/definitions"
+} from "../definitions/datasource"
 
 module DynamoModule {
   const AWS = require("aws-sdk")

packages/server/src/integrations/elasticsearch.ts

@@ -2,7 +2,7 @@ import {
   Integration,
   DatasourceFieldTypes,
   QueryTypes,
-} from "./base/definitions"
+} from "../definitions/datasource"
 
 module ElasticsearchModule {
   const { Client } = require("@elastic/elasticsearch")

packages/server/src/integrations/index.ts

@@ -9,33 +9,34 @@ const airtable = require("./airtable")
 const mysql = require("./mysql")
 const arangodb = require("./arangodb")
 const rest = require("./rest")
+const { SourceNames } = require("../definitions/datasource")
 
 const DEFINITIONS = {
-  POSTGRES: postgres.schema,
+  [SourceNames.POSTGRES]: postgres.schema,
-  DYNAMODB: dynamodb.schema,
+  [SourceNames.DYNAMODB]: dynamodb.schema,
-  MONGODB: mongodb.schema,
+  [SourceNames.MONGODB]: mongodb.schema,
-  ELASTICSEARCH: elasticsearch.schema,
+  [SourceNames.ELASTICSEARCH]: elasticsearch.schema,
-  COUCHDB: couchdb.schema,
+  [SourceNames.COUCHDB]: couchdb.schema,
-  SQL_SERVER: sqlServer.schema,
+  [SourceNames.SQL_SERVER]: sqlServer.schema,
-  S3: s3.schema,
+  [SourceNames.S3]: s3.schema,
-  AIRTABLE: airtable.schema,
+  [SourceNames.AIRTABLE]: airtable.schema,
-  MYSQL: mysql.schema,
+  [SourceNames.MYSQL]: mysql.schema,
-  ARANGODB: arangodb.schema,
+  [SourceNames.ARANGODB]: arangodb.schema,
-  REST: rest.schema,
+  [SourceNames.REST]: rest.schema,
 }
 
 const INTEGRATIONS = {
-  POSTGRES: postgres.integration,
+  [SourceNames.POSTGRES]: postgres.integration,
-  DYNAMODB: dynamodb.integration,
+  [SourceNames.DYNAMODB]: dynamodb.integration,
-  MONGODB: mongodb.integration,
+  [SourceNames.MONGODB]: mongodb.integration,
-  ELASTICSEARCH: elasticsearch.integration,
+  [SourceNames.ELASTICSEARCH]: elasticsearch.integration,
-  COUCHDB: couchdb.integration,
+  [SourceNames.COUCHDB]: couchdb.integration,
-  S3: s3.integration,
+  [SourceNames.SQL_SERVER]: s3.integration,
-  SQL_SERVER: sqlServer.integration,
+  [SourceNames.S3]: sqlServer.integration,
-  AIRTABLE: airtable.integration,
+  [SourceNames.AIRTABLE]: airtable.integration,
-  MYSQL: mysql.integration,
+  [SourceNames.MYSQL]: mysql.integration,
-  ARANGODB: arangodb.integration,
+  [SourceNames.ARANGODB]: arangodb.integration,
-  REST: rest.integration,
+  [SourceNames.REST]: rest.integration,
 }
 
 module.exports = {