Merge branch 'feature/oidc-support' of https://github.com/Budibase/budibase into oidc-config-management

.all-contributorsrc

@@ -75,6 +75,28 @@
         "design"
       ]
     },
+    {
+      "login": "Rory-Powell",
+      "name": "Rory Powell",
+      "avatar_url": "https://avatars.githubusercontent.com/u/8755148?v=4",
+      "profile": "https://github.com/Rory-Powell",
+      "contributions": [
+        "code",
+        "doc",
+        "test"
+      ]
+    },
+    {
+      "login": "PClmnt",
+      "name": "Peter Clement",
+      "avatar_url": "https://avatars.githubusercontent.com/u/5665926?v=4",
+      "profile": "https://github.com/PClmnt",
+      "contributions": [
+        "code",
+        "doc",
+        "test"
+      ]
+    },
     {
       "login": "Conor-Mack",
       "name": "Conor_Mack",

.github/AUTHORS.md

@@ -7,3 +7,5 @@ Contributors
 * Martin McKeaveney - [@shogunpurple](https://github.com/shogunpurple)
 * Andrew Kingston - [@aptkingston](https://github.com/aptkingston)
 * Michael Drury - [@mike12345567](https://github.com/mike12345567)
+* Peter Clement - [@PClmnt](https://github.com/PClmnt)
+* Rory Powell - [@Rory-Powell](https://github.com/Rory-Powell)

.github/CONTRIBUTING.md

@@ -126,7 +126,16 @@ To run the budibase server and builder in dev mode (i.e. with live reloading):
 
 This will enable watch mode for both the builder app, server, client library and any component libraries.
 
-### 5. Cleanup
+### 5. Debugging using VS Code
+
+To debug the budibase server and worker a VS Code launch configuration has been provided. 
+
+Visit the debug window and select `Budibase Server` or `Budibase Worker` to debug the respective component. 
+Alternatively to start both components simultaneously select `Start Budibase`.
+
+In addition to the above, the remaining budibase components may be ran in dev mode using: `yarn dev:noserver`.
+
+### 6. Cleanup
 
 If you wish to delete all the apps created in development and reset the environment then run the following:
 

.vscode/launch.json

@@ -21,6 +21,27 @@
             "args": [],
             "cwd":"C:/code/my-apps",
             "console": "externalTerminal" 
+        },
+        {
+            "name": "Budibase Server",
+            "type": "node",
+            "request": "launch",
+            "runtimeArgs": ["--nolazy", "-r", "ts-node/register/transpile-only"],
+            "args": ["${workspaceFolder}/packages/server/src/index.ts"],
+            "cwd": "${workspaceFolder}/packages/server"
+          },
+          {
+            "name": "Budibase Worker",
+            "type": "node",
+            "request": "launch",
+            "program": "${workspaceFolder}/packages/worker/src/index.js",
+            "cwd": "${workspaceFolder}/packages/worker"
+          }
+    ],
+    "compounds": [
+        {
+          "name": "Start Budibase",
+          "configurations": ["Budibase Server", "Budibase Worker"]
         }
     ]
 }

README.md

@@ -211,9 +211,11 @@ Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/d
     <td align="center"><a href="https://budibase.com/"><img src="https://avatars3.githubusercontent.com/u/3524181?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Michael Shanks</b></sub></a><br /><a href="https://github.com/Budibase/budibase/commits?author=mjashanks" title="Documentation">📖</a> <a href="https://github.com/Budibase/budibase/commits?author=mjashanks" title="Code">💻</a> <a href="https://github.com/Budibase/budibase/commits?author=mjashanks" title="Tests">⚠️</a></td>
     <td align="center"><a href="https://github.com/kevmodrome"><img src="https://avatars3.githubusercontent.com/u/534488?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Kevin Åberg Kultalahti</b></sub></a><br /><a href="https://github.com/Budibase/budibase/commits?author=kevmodrome" title="Documentation">📖</a> <a href="https://github.com/Budibase/budibase/commits?author=kevmodrome" title="Code">💻</a> <a href="https://github.com/Budibase/budibase/commits?author=kevmodrome" title="Tests">⚠️</a></td>
     <td align="center"><a href="https://www.budibase.com/"><img src="https://avatars2.githubusercontent.com/u/49767913?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Joe</b></sub></a><br /><a href="https://github.com/Budibase/budibase/commits?author=joebudi" title="Documentation">📖</a> <a href="https://github.com/Budibase/budibase/commits?author=joebudi" title="Code">💻</a> <a href="#content-joebudi" title="Content">🖋</a> <a href="#design-joebudi" title="Design">🎨</a></td>
-    <td align="center"><a href="https://github.com/Conor-Mack"><img src="https://avatars1.githubusercontent.com/u/36074859?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Conor_Mack</b></sub></a><br /><a href="https://github.com/Budibase/budibase/commits?author=Conor-Mack" title="Code">💻</a> <a href="https://github.com/Budibase/budibase/commits?author=Conor-Mack" title="Tests">⚠️</a></td>
+    <td align="center"><a href="https://github.com/Rory-Powell"><img src="https://avatars.githubusercontent.com/u/8755148?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Rory Powell</b></sub></a><br /><a href="https://github.com/Budibase/budibase/commits?author=Rory-Powell" title="Code">💻</a> <a href="https://github.com/Budibase/budibase/commits?author=Rory-Powell" title="Documentation">📖</a> <a href="https://github.com/Budibase/budibase/commits?author=Rory-Powell" title="Tests">⚠️</a></td>
   </tr>
   <tr>
+    <td align="center"><a href="https://github.com/PClmnt"><img src="https://avatars.githubusercontent.com/u/5665926?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Peter Clement</b></sub></a><br /><a href="https://github.com/Budibase/budibase/commits?author=PClmnt" title="Code">💻</a> <a href="https://github.com/Budibase/budibase/commits?author=PClmnt" title="Documentation">📖</a> <a href="https://github.com/Budibase/budibase/commits?author=PClmnt" title="Tests">⚠️</a></td>
+    <td align="center"><a href="https://github.com/Conor-Mack"><img src="https://avatars1.githubusercontent.com/u/36074859?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Conor_Mack</b></sub></a><br /><a href="https://github.com/Budibase/budibase/commits?author=Conor-Mack" title="Code">💻</a> <a href="https://github.com/Budibase/budibase/commits?author=Conor-Mack" title="Tests">⚠️</a></td>
     <td align="center"><a href="https://github.com/pngwn"><img src="https://avatars1.githubusercontent.com/u/12937446?v=4?s=100" width="100px;" alt=""/><br /><sub><b>pngwn</b></sub></a><br /><a href="https://github.com/Budibase/budibase/commits?author=pngwn" title="Code">💻</a> <a href="https://github.com/Budibase/budibase/commits?author=pngwn" title="Tests">⚠️</a></td>
     <td align="center"><a href="https://github.com/HugoLd"><img src="https://avatars0.githubusercontent.com/u/26521848?v=4?s=100" width="100px;" alt=""/><br /><sub><b>HugoLd</b></sub></a><br /><a href="https://github.com/Budibase/budibase/commits?author=HugoLd" title="Code">💻</a></td>
     <td align="center"><a href="https://github.com/victoriasloan"><img src="https://avatars.githubusercontent.com/u/9913651?v=4?s=100" width="100px;" alt=""/><br /><sub><b>victoriasloan</b></sub></a><br /><a href="https://github.com/Budibase/budibase/commits?author=victoriasloan" title="Code">💻</a></td>

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "0.9.67",
+  "version": "0.9.70",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

packages/auth/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/auth",
-  "version": "0.9.67",
+  "version": "0.9.70",
   "description": "Authentication middlewares for budibase builder and apps",
   "main": "src/index.js",
   "author": "Budibase",
@@ -13,6 +13,7 @@
     "koa-passport": "^4.1.4",
     "lodash": "^4.17.21",
     "node-fetch": "^2.6.1",
+    "@techpass/passport-openidconnect": "^0.3.0",
     "passport-google-auth": "^1.0.2",
     "passport-google-oauth": "^2.0.0",
     "passport-jwt": "^4.0.0",

packages/auth/src/index.js

@@ -2,7 +2,14 @@ const passport = require("koa-passport")
 const LocalStrategy = require("passport-local").Strategy
 const JwtStrategy = require("passport-jwt").Strategy
 const { StaticDatabases } = require("./db/utils")
-const { jwt, local, authenticated, google, auditLog } = require("./middleware")
+const {
+  jwt,
+  local,
+  authenticated,
+  google,
+  oidc,
+  auditLog,
+} = require("./middleware")
 const { setDB, getDB } = require("./db")
 
 // Strategies
@@ -44,6 +51,7 @@ module.exports = {
     buildAuthMiddleware: authenticated,
     passport,
     google,
+    oidc,
     jwt: require("jsonwebtoken"),
     auditLog,
   },

packages/auth/src/middleware/index.js

@@ -1,11 +1,13 @@
 const jwt = require("./passport/jwt")
 const local = require("./passport/local")
 const google = require("./passport/google")
+const oidc = require("./passport/oidc")
 const authenticated = require("./authenticated")
 const auditLog = require("./auditLog")
 
 module.exports = {
   google,
+  oidc,
   jwt,
   local,
   authenticated,

packages/auth/src/middleware/passport/oidc.js

@@ -0,0 +1,164 @@
+const env = require("../../environment")
+const jwt = require("jsonwebtoken")
+const database = require("../../db")
+const fetch = require("node-fetch")
+const OIDCStrategy = require("@techpass/passport-openidconnect").Strategy
+const {
+  StaticDatabases,
+  generateGlobalUserID,
+  ViewNames,
+} = require("../../db/utils")
+
+/**
+ * Attempt to parse the users email address. 
+ * 
+ * It is not guaranteed that the email will be returned by the user info endpoint (e.g. github connected account used in azure ad).
+ * Fallback to the id token where possible.
+ * 
+ * @param {*} profile The structured profile created by passport using the user info endpoint
+ * @param {*} jwtClaims The raw claims returned in the id token
+ */
+function getEmail(profile, jwtClaims) {
+  if (profile._json.email) {
+    return profile._json.email
+  }
+
+  if (jwtClaims.email) {
+    return jwtClaims.email
+  }
+
+  return null;
+}
+
+/**
+ * 
+ * @param {*} issuer The identity provider base URL
+ * @param {*} sub The user ID
+ * @param {*} profile The user profile information. Created by passport from the /userinfo response
+ * @param {*} jwtClaims The parsed id_token claims
+ * @param {*} accessToken The access_token for contacting the identity provider - may or may not be a JWT
+ * @param {*} refreshToken The refresh_token for obtaining a new access_token - usually not a JWT
+ * @param {*} idToken The id_token - always a JWT
+ * @param {*} params The response body from requesting an access_token
+ * @param {*} done The passport callback: err, user, info
+ * @returns 
+ */
+async function authenticate(
+  issuer,
+  sub,
+  profile,
+  jwtClaims,
+  accessToken,
+  refreshToken,
+  idToken,
+  params,
+  done
+) {
+  // Check the user exists in the instance DB by email
+  const db = database.getDB(StaticDatabases.GLOBAL.name)
+
+  let dbUser
+
+  const userId = generateGlobalUserID(profile.id)
+
+  try {
+    // use the OIDC profile id
+    dbUser = await db.get(userId)
+  } catch (err) {
+    const user = {
+      _id: userId,
+      provider: profile.provider,
+      roles: {},
+      ...profile._json,
+    }
+
+    // check if an account with the OIDC email address exists locally
+    const email = getEmail(profile, jwtClaims)
+    if (!email) {
+      return done(null, false, { message: "No email address found" })
+    }
+
+    const users = await db.query(`database/${ViewNames.USER_BY_EMAIL}`, {
+      key: email,
+      include_docs: true,
+    })
+
+    // OIDC user already exists by email
+    if (users.rows.length > 0) {
+      const existing = users.rows[0].doc
+
+      // remove the local account to avoid conflicts
+      await db.remove(existing._id, existing._rev)
+
+      // merge with existing account
+      user.roles = existing.roles
+      user.builder = existing.builder
+      user.admin = existing.admin
+
+      const response = await db.post(user)
+      dbUser = user
+      dbUser._rev = response.rev
+    } else {
+      return done(null, false, { message: "Email does not yet exist. You must set up your local budibase account first." })
+    }
+  }
+
+  // authenticate
+  const payload = {
+    userId: dbUser._id,
+    builder: dbUser.builder,
+    email: dbUser.email,
+  }
+
+  dbUser.token = jwt.sign(payload, env.JWT_SECRET, {
+    expiresIn: "1 day",
+  })
+
+  return done(null, dbUser)
+}
+
+/**
+ * Create an instance of the oidc passport strategy. This wrapper fetches the configuration
+ * from couchDB rather than environment variables, using this factory is necessary for dynamically configuring passport.
+ * @returns Dynamically configured Passport OIDC Strategy
+ */
+exports.strategyFactory = async function (callbackUrl) {
+  try {
+    const configurationUrl =
+      "https://login.microsoftonline.com/2668c0dd-7ed2-4db3-b387-05b6f9204a70/v2.0/.well-known/openid-configuration"
+    const clientSecret = "g-ty~2iW.bo.88xj_QI6~hdc-H8mP2Xbnd"
+    const clientId = "bed2017b-2f53-42a9-8ef9-e58918935e07"
+
+    if (!clientId || !clientSecret || !callbackUrl || !configurationUrl) {
+      throw new Error(
+        "Configuration invalid. Must contain clientID, clientSecret, callbackUrl and configurationUrl"
+      )
+    }
+
+    const response = await fetch(configurationUrl)
+
+    if (!response.ok) {
+      throw new Error(`Unexpected response when fetching openid-configuration: ${response.statusText}`)
+    }
+
+    const body = await response.json()
+
+    return new OIDCStrategy(
+      {
+        issuer: body.issuer,
+        authorizationURL: body.authorization_endpoint,
+        tokenURL: body.token_endpoint,
+        userInfoURL: body.userinfo_endpoint,
+        clientID: clientId,
+        clientSecret: clientSecret,
+        callbackURL: callbackUrl,
+        scope: "profile email",
+      },
+      authenticate
+    )
+    
+  } catch (err) {
+    console.error(err)
+    throw new Error("Error constructing OIDC authentication strategy", err)
+  }
+}

packages/auth/yarn.lock

@@ -2,6 +2,17 @@
 # yarn lockfile v1
 
 
+"@techpass/passport-openidconnect@^0.3.0":
+  version "0.3.0"
+  resolved "https://registry.yarnpkg.com/@techpass/passport-openidconnect/-/passport-openidconnect-0.3.0.tgz#a60b2bbf3f262649a5a02d5d186219944acc3010"
+  integrity sha512-bVsPwl66s7J7GHxTPlW/RJYhZol9SshNznQsx83OOh9G+JWFGoeWxh+xbX+FTdJNoUvGIGbJnpWPY2wC6NOHPw==
+  dependencies:
+    base64url "^3.0.1"
+    oauth "^0.9.15"
+    passport-strategy "^1.0.0"
+    request "^2.88.0"
+    webfinger "^0.4.2"
+
 ajv@^6.12.3:
   version "6.12.6"
   resolved "https://registry.yarnpkg.com/ajv/-/ajv-6.12.6.tgz#baf5a62e802b07d977034586f8c3baf5adf26df4"
@@ -71,7 +82,7 @@ base64-js@^1.0.2, base64-js@^1.3.1:
   resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.5.1.tgz#1b1b440160a5bf7ad40b650f095963481903930a"
   integrity sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==
 
-base64url@3.x.x:
+base64url@3.x.x, base64url@^3.0.1:
   version "3.0.1"
   resolved "https://registry.yarnpkg.com/base64url/-/base64url-3.0.1.tgz#6399d572e2bc3f90a9a8b22d5dbb0a32d33f788d"
   integrity sha512-ir1UPr3dkwexU7FdV8qBBbNDRUhMmIekYMFZfi+C/sLNnRESKPl23nB9b2pltqfOQNnGzsDdId90AEtG5tCx4A==
@@ -574,7 +585,7 @@ oauth-sign@~0.9.0:
   resolved "https://registry.yarnpkg.com/oauth-sign/-/oauth-sign-0.9.0.tgz#47a7b016baa68b5fa0ecf3dee08a85c679ac6455"
   integrity sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ==
 
-oauth@0.9.x:
+oauth@0.9.x, oauth@^0.9.15:
   version "0.9.15"
   resolved "https://registry.yarnpkg.com/oauth/-/oauth-0.9.15.tgz#bd1fefaf686c96b75475aed5196412ff60cfb9c1"
   integrity sha1-vR/vr2hslrdUda7VGWQS/2DPucE=
@@ -748,7 +759,7 @@ redis-parser@^3.0.0:
   dependencies:
     redis-errors "^1.0.0"
 
-request@^2.72.0, request@^2.74.0:
+request@^2.72.0, request@^2.74.0, request@^2.88.0:
   version "2.88.2"
   resolved "https://registry.yarnpkg.com/request/-/request-2.88.2.tgz#d73c918731cb5a87da047e207234146f664d12b3"
   integrity sha512-MsvtOrfG9ZcrOwAW+Qi+F6HbD0CWXEh9ou77uOb7FM2WPhwT7smM833PzanhJLsgXjN89Ir6V2PczXNnMpwKhw==
@@ -794,7 +805,7 @@ sax@1.2.1:
   resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.1.tgz#7b8e656190b228e81a66aea748480d828cd2d37a"
   integrity sha1-e45lYZCyKOgaZq6nSEgNgozS03o=
 
-sax@>=0.6.0:
+sax@>=0.1.1, sax@>=0.6.0:
   version "1.2.4"
   resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.4.tgz#2816234e2378bddc4e5354fab5caa895df7100d9"
   integrity sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw==
@@ -829,6 +840,11 @@ standard-as-callback@^2.1.0:
   resolved "https://registry.yarnpkg.com/standard-as-callback/-/standard-as-callback-2.1.0.tgz#8953fc05359868a77b5b9739a665c5977bb7df45"
   integrity sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==
 
+step@0.0.x:
+  version "0.0.6"
+  resolved "https://registry.yarnpkg.com/step/-/step-0.0.6.tgz#143e7849a5d7d3f4a088fe29af94915216eeede2"
+  integrity sha1-FD54SaXX0/SgiP4pr5SRUhbu7eI=
+
 string-template@~1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/string-template/-/string-template-1.0.0.tgz#9e9f2233dc00f218718ec379a28a5673ecca8b96"
@@ -943,11 +959,26 @@ verror@1.10.0:
     core-util-is "1.0.2"
     extsprintf "^1.2.0"
 
+webfinger@^0.4.2:
+  version "0.4.2"
+  resolved "https://registry.yarnpkg.com/webfinger/-/webfinger-0.4.2.tgz#3477a6d97799461896039fcffc650b73468ee76d"
+  integrity sha1-NHem2XeZRhiWA5/P/GULc0aO520=
+  dependencies:
+    step "0.0.x"
+    xml2js "0.1.x"
+
 wrappy@1:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/wrappy/-/wrappy-1.0.2.tgz#b5243d8f3ec1aa35f1364605bc0d1036e30ab69f"
   integrity sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=
 
+xml2js@0.1.x:
+  version "0.1.14"
+  resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.1.14.tgz#5274e67f5a64c5f92974cd85139e0332adc6b90c"
+  integrity sha1-UnTmf1pkxfkpdM2FE54DMq3GuQw=
+  dependencies:
+    sax ">=0.1.1"
+
 xml2js@0.4.19:
   version "0.4.19"
   resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.4.19.tgz#686c20f213209e94abf0d1bcf1efaa291c7827a7"

packages/bbui/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/bbui",
   "description": "A UI solution used in the different Budibase projects.",
-  "version": "0.9.67",
+  "version": "0.9.70",
   "license": "AGPL-3.0",
   "svelte": "src/index.js",
   "module": "dist/bbui.es.js",

packages/builder/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/builder",
-  "version": "0.9.67",
+  "version": "0.9.70",
   "license": "AGPL-3.0",
   "private": true,
   "scripts": {
@@ -65,10 +65,10 @@
     }
   },
   "dependencies": {
-    "@budibase/bbui": "^0.9.67",
+    "@budibase/bbui": "^0.9.70",
-    "@budibase/client": "^0.9.67",
+    "@budibase/client": "^0.9.70",
     "@budibase/colorpicker": "1.1.2",
-    "@budibase/string-templates": "^0.9.67",
+    "@budibase/string-templates": "^0.9.70",
     "@sentry/browser": "5.19.1",
     "@spectrum-css/page": "^3.0.1",
     "@spectrum-css/vars": "^3.0.1",

packages/builder/src/builderStore/store/screenTemplates/rowDetailScreen.js

@@ -98,7 +98,6 @@ const createScreen = table => {
           valueType: "Binding",
         },
       ],
-      limit: 1,
       paginate: false,
     })
 

packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte

@@ -53,7 +53,7 @@
   let deletion
 
   $: tableOptions = $tables.list.filter(
-    table => table._id !== $tables.draft._id
+    table => table._id !== $tables.draft._id && table.type !== "external"
   )
   $: required = !!field?.constraints?.presence || primaryDisplay
   $: uneditable =
@@ -172,11 +172,6 @@
         alt: `Many ${table.name} rows → many ${linkTable.name} rows`,
         value: RelationshipTypes.MANY_TO_MANY,
       },
-      {
-        name: `One ${linkName} row → many ${thisName} rows`,
-        alt: `One ${linkTable.name} rows → many ${table.name} rows`,
-        value: RelationshipTypes.ONE_TO_MANY,
-      },
       {
         name: `One ${thisName} row → many ${linkName} rows`,
         alt: `One ${table.name} rows → many ${linkTable.name} rows`,

packages/builder/src/components/backend/TableNavigator/TableNavigator.svelte

@@ -6,9 +6,14 @@
   import EditViewPopover from "./popovers/EditViewPopover.svelte"
   import NavItem from "components/common/NavItem.svelte"
 
+  const alphabetical = (a, b) => a.name?.toLowerCase() > b.name?.toLowerCase()
+
   export let sourceId
 
   $: selectedView = $views.selected && $views.selected.name
+  $: sortedTables = $tables.list
+    .filter(table => table.sourceId === sourceId)
+    .sort(alphabetical)
 
   function selectTable(table) {
     tables.select(table)
@@ -33,7 +38,7 @@
 
 {#if $database?._id}
   <div class="hierarchy-items-container">
-    {#each $tables.list.filter(table => table.sourceId === sourceId) as table, idx}
+    {#each sortedTables as table, idx}
       <NavItem
         indentLevel={1}
         border={idx > 0}
@@ -46,7 +51,7 @@
           <EditTablePopover {table} />
         {/if}
       </NavItem>
-      {#each Object.keys(table.views || {}) as viewName, idx (idx)}
+      {#each [...Object.keys(table.views || {})].sort() as viewName, idx (idx)}
         <NavItem
           indentLevel={2}
           icon="Remove"

packages/builder/src/components/design/PropertiesPanel/componentStyles.js

@@ -284,7 +284,7 @@ export const background = {
 
 export const border = {
   label: "Border",
-  columns: "auto 1fr",
+  columns: "1fr 1fr",
   settings: [
     {
       label: "Color",
@@ -295,17 +295,31 @@ export const border = {
       label: "Width",
       key: "border-width",
       control: Select,
+      column: "1 / 2",
+      placeholder: "None",
       options: [
         { label: "Small", value: "1px" },
         { label: "Medium", value: "2px" },
         { label: "Large", value: "4px" },
       ],
     },
+    {
+      label: "Style",
+      key: "border-style",
+      control: Select,
+      column: "2 / 3",
+      placeholder: "None",
+      options: [
+        { label: "Solid", value: "solid" },
+        { label: "Dotted", value: "dotted" },
+        { label: "Dashed", value: "dashed" },
+      ],
+    },
     {
       label: "Radius",
       key: "border-radius",
       control: Select,
-      column: "1 / 3",
+      placeholder: "None",
       options: [
         { label: "Small", value: "0.25rem" },
         { label: "Medium", value: "0.5rem" },
@@ -317,7 +331,7 @@ export const border = {
       label: "Shadow",
       key: "box-shadow",
       control: Select,
-      column: "1 / 3",
+      placeholder: "None",
       options: [
         {
           label: "Small",

packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/CreateEditRelationship/CreateEditRelationship.svelte

@@ -0,0 +1,194 @@
+<script>
+  import { RelationshipTypes } from "constants/backend"
+  import { Button, Input, ModalContent, Select, Detail } from "@budibase/bbui"
+  import { tables } from "stores/backend"
+  import { uuid } from "builderStore/uuid"
+
+  export let save
+  export let datasource
+  export let plusTables = []
+  export let fromRelationship = {}
+  export let toRelationship = {}
+  export let close
+
+  let originalFromName = fromRelationship.name,
+    originalToName = toRelationship.name
+
+  function isValid(relationship) {
+    if (
+      relationship.relationshipType === RelationshipTypes.MANY_TO_MANY &&
+      !relationship.through
+    ) {
+      return false
+    }
+    return (
+      relationship.name && relationship.tableId && relationship.relationshipType
+    )
+  }
+
+  $: tableOptions = plusTables.map(table => ({
+    label: table.name,
+    value: table._id,
+  }))
+  $: fromTable = plusTables.find(table => table._id === toRelationship?.tableId)
+  $: toTable = plusTables.find(table => table._id === fromRelationship?.tableId)
+  $: through = plusTables.find(table => table._id === fromRelationship?.through)
+  $: valid = toTable && fromTable && isValid(fromRelationship)
+  $: linkTable = through || toTable
+  $: relationshipTypes = [
+    {
+      label: "Many",
+      value: RelationshipTypes.MANY_TO_MANY,
+    },
+    {
+      label: "One",
+      value: RelationshipTypes.MANY_TO_ONE,
+    },
+  ]
+  $: updateRelationshipType(fromRelationship?.relationshipType)
+
+  function updateRelationshipType(fromType) {
+    if (fromType === RelationshipTypes.MANY_TO_MANY) {
+      toRelationship.relationshipType = RelationshipTypes.MANY_TO_MANY
+    } else {
+      toRelationship.relationshipType = RelationshipTypes.MANY_TO_ONE
+    }
+  }
+
+  function buildRelationships() {
+    // if any to many only need to check from
+    const manyToMany =
+      fromRelationship.relationshipType === RelationshipTypes.MANY_TO_MANY
+    // main is simply used to know this is the side the user configured it from
+    const id = uuid()
+    if (!manyToMany) {
+      delete fromRelationship.through
+      delete toRelationship.through
+    }
+    let relateFrom = {
+      ...fromRelationship,
+      type: "link",
+      main: true,
+      _id: id,
+    }
+    let relateTo = {
+      ...toRelationship,
+      type: "link",
+      _id: id,
+    }
+
+    // [0] is because we don't support composite keys for relationships right now
+    if (manyToMany) {
+      relateFrom = {
+        ...relateFrom,
+        through: through._id,
+        fieldName: toTable.primary[0],
+      }
+      relateTo = {
+        ...relateTo,
+        through: through._id,
+        fieldName: fromTable.primary[0],
+      }
+    } else {
+      relateFrom = {
+        ...relateFrom,
+        foreignKey: relateFrom.fieldName,
+        fieldName: fromTable.primary[0],
+      }
+      relateTo = {
+        ...relateTo,
+        relationshipType: RelationshipTypes.ONE_TO_MANY,
+        foreignKey: relateFrom.fieldName,
+        fieldName: fromTable.primary[0],
+      }
+    }
+
+    fromRelationship = relateFrom
+    toRelationship = relateTo
+  }
+
+  // save the relationship on to the datasource
+  async function saveRelationship() {
+    buildRelationships()
+    // source of relationship
+    datasource.entities[fromTable.name].schema[fromRelationship.name] =
+      fromRelationship
+    // save other side of relationship in the other schema
+    datasource.entities[toTable.name].schema[toRelationship.name] =
+      toRelationship
+
+    // If relationship has been renamed
+    if (originalFromName !== fromRelationship.name) {
+      delete datasource.entities[fromTable.name].schema[originalFromName]
+    }
+    if (originalToName !== toRelationship.name) {
+      delete datasource.entities[toTable.name].schema[originalToName]
+    }
+
+    await save()
+    await tables.fetch()
+  }
+
+  async function deleteRelationship() {
+    delete datasource.entities[fromTable.name].schema[fromRelationship.name]
+    delete datasource.entities[toTable.name].schema[toRelationship.name]
+    await save()
+    await tables.fetch()
+    close()
+  }
+</script>
+
+<ModalContent
+  title="Create Relationship"
+  confirmText="Save"
+  onConfirm={saveRelationship}
+  disabled={!valid}
+>
+  <Select
+    label="Relationship type"
+    options={relationshipTypes}
+    bind:value={fromRelationship.relationshipType}
+  />
+  <div class="headings">
+    <Detail>Tables</Detail>
+  </div>
+  <Select
+    label="Select from table"
+    options={tableOptions}
+    bind:value={toRelationship.tableId}
+  />
+  <Select
+    label={"Select to table"}
+    options={tableOptions}
+    bind:value={fromRelationship.tableId}
+  />
+  {#if fromRelationship?.relationshipType === RelationshipTypes.MANY_TO_MANY}
+    <Select
+      label={"Through"}
+      options={tableOptions}
+      bind:value={fromRelationship.through}
+    />
+  {:else if toTable}
+    <Select
+      label={`Foreign Key (${toTable?.name})`}
+      options={Object.keys(toTable?.schema)}
+      bind:value={fromRelationship.fieldName}
+    />
+  {/if}
+  <div class="headings">
+    <Detail>Column names</Detail>
+  </div>
+  <Input label="From table column" bind:value={fromRelationship.name} />
+  <Input label="To table column" bind:value={toRelationship.name} />
+  <div slot="footer">
+    {#if originalFromName != null}
+      <Button warning text on:click={deleteRelationship}>Delete</Button>
+    {/if}
+  </div>
+</ModalContent>
+
+<style>
+  .headings {
+    margin-top: var(--spacing-s);
+  }
+</style>

packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/CreateEditRelationship/TableSelect.svelte

@@ -0,0 +1,21 @@
+<script>
+  import { Menu, Icon, MenuSection, MenuItem } from "@budibase/bbui"
+
+  export let heading
+  export let tables
+  export let selected = false
+  export let select
+</script>
+
+<Menu>
+  <MenuSection {heading}>
+    {#each tables as table}
+      <MenuItem noClose icon="Table" on:click={() => select(table)}>
+        {table.name}
+        {#if selected}
+          <Icon size="S" name="Checkmark" />
+        {/if}
+      </MenuItem>
+    {/each}
+  </MenuSection>
+</Menu>

packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/index.svelte

@@ -1,16 +1,70 @@
 <script>
   import { goto, beforeUrlChange } from "@roxi/routify"
-  import { Button, Heading, Body, Divider, Layout } from "@budibase/bbui"
+  import { Button, Heading, Body, Divider, Layout, Modal } from "@budibase/bbui"
   import { datasources, integrations, queries, tables } from "stores/backend"
   import { notifications } from "@budibase/bbui"
   import IntegrationConfigForm from "components/backend/DatasourceNavigator/TableIntegrationMenu/IntegrationConfigForm.svelte"
+  import CreateEditRelationship from "./CreateEditRelationship/CreateEditRelationship.svelte"
+  import DisplayColumnModal from "./modals/EditDisplayColumnsModal.svelte"
   import ICONS from "components/backend/DatasourceNavigator/icons"
   import { capitalise } from "helpers"
 
   let unsaved = false
+  let relationshipModal
+  let displayColumnModal
+  let selectedFromRelationship, selectedToRelationship
 
   $: datasource = $datasources.list.find(ds => ds._id === $datasources.selected)
   $: integration = datasource && $integrations[datasource.source]
+  $: plusTables = datasource?.plus
+    ? Object.values(datasource.entities || {})
+    : []
+  $: relationships = getRelationships(plusTables)
+
+  function getRelationships(tables) {
+    if (!tables || !Array.isArray(tables)) {
+      return {}
+    }
+    let pairs = {}
+    for (let table of tables) {
+      for (let column of Object.values(table.schema)) {
+        if (column.type !== "link") {
+          continue
+        }
+        // these relationships have an id to pair them to each other
+        // one has a main for the from side
+        const key = column.main ? "from" : "to"
+        pairs[column._id] = {
+          ...pairs[column._id],
+          [key]: column,
+        }
+      }
+    }
+    return pairs
+  }
+
+  function buildRelationshipDisplayString(fromCol, toCol) {
+    function getTableName(tableId) {
+      if (!tableId || typeof tableId !== "string") {
+        return null
+      }
+      return plusTables.find(table => table._id === tableId)?.name || "Unknown"
+    }
+    if (!toCol || !fromCol) {
+      return "Cannot build name"
+    }
+    const fromTableName = getTableName(toCol.tableId)
+    const toTableName = getTableName(fromCol.tableId)
+    const throughTableName = getTableName(fromCol.through)
+
+    let displayString
+    if (throughTableName) {
+      displayString = `${fromTableName} through ${throughTableName} → ${toTableName}`
+    } else {
+      displayString = `${fromTableName} → ${toTableName}`
+    }
+    return displayString
+  }
 
   async function saveDatasource() {
     try {
@@ -48,6 +102,16 @@
     unsaved = true
   }
 
+  function openRelationshipModal(fromRelationship, toRelationship) {
+    selectedFromRelationship = fromRelationship || {}
+    selectedToRelationship = toRelationship || {}
+    relationshipModal.show()
+  }
+
+  function openDisplayColumnModal() {
+    displayColumnModal.show()
+  }
+
   $beforeUrlChange(() => {
     if (unsaved) {
       notifications.error(
@@ -59,6 +123,21 @@
   })
 </script>
 
+<Modal bind:this={relationshipModal}>
+  <CreateEditRelationship
+    {datasource}
+    save={saveDatasource}
+    close={relationshipModal.hide}
+    {plusTables}
+    fromRelationship={selectedFromRelationship}
+    toRelationship={selectedToRelationship}
+  />
+</Modal>
+
+<Modal bind:this={displayColumnModal}>
+  <DisplayColumnModal {datasource} {plusTables} save={saveDatasource} />
+</Modal>
+
 {#if datasource && integration}
   <section>
     <Layout>
@@ -92,9 +171,18 @@
         <Divider />
         <div class="query-header">
           <Heading size="S">Tables</Heading>
-          <Button primary on:click={updateDatasourceSchema}
+          <div class="table-buttons">
-            >Fetch Tables From Database</Button
+            {#if plusTables && plusTables.length !== 0}
-          >
+              <Button primary on:click={openDisplayColumnModal}>
+                Update display columns
+              </Button>
+            {/if}
+            <div>
+              <Button primary on:click={updateDatasourceSchema}>
+                Fetch tables from database
+              </Button>
+            </div>
+          </div>
         </div>
         <Body>
           This datasource can determine tables automatically. Budibase can fetch
@@ -102,18 +190,44 @@
           having to write any queries at all.
         </Body>
         <div class="query-list">
-          {#if datasource.entities}
+          {#each plusTables as table}
-            {#each Object.keys(datasource.entities) as entity}
+            <div class="query-list-item" on:click={() => onClickTable(table)}>
-              <div
+              <p class="query-name">{table.name}</p>
-                class="query-list-item"
+              <p>Primary Key: {table.primary}</p>
-                on:click={() => onClickTable(datasource.entities[entity])}
-              >
-                <p class="query-name">{entity}</p>
-                <p>Primary Key: {datasource.entities[entity].primary}</p>
               <p>→</p>
             </div>
           {/each}
+        </div>
+        {#if plusTables?.length !== 0}
+          <Divider />
+          <div class="query-header">
+            <Heading size="S">Relationships</Heading>
+            <Button primary on:click={() => openRelationshipModal()}
+              >Create relationship</Button
+            >
+          </div>
+          <Body>
+            Tell budibase how your tables are related to get even more smart
+            features.
+          </Body>
         {/if}
+        <div class="query-list">
+          {#each Object.values(relationships) as relationship}
+            <div
+              class="query-list-item"
+              on:click={() =>
+                openRelationshipModal(relationship.from, relationship.to)}
+            >
+              <p class="query-name">
+                {buildRelationshipDisplayString(
+                  relationship.from,
+                  relationship.to
+                )}
+              </p>
+              <p>{relationship.from?.name} to {relationship.to?.name}</p>
+              <p>→</p>
+            </div>
+          {/each}
         </div>
       {/if}
       <Divider />
@@ -202,4 +316,14 @@
     text-overflow: ellipsis;
     font-size: var(--font-size-s);
   }
+
+  .table-buttons {
+    display: grid;
+    grid-gap: var(--spacing-l);
+    grid-template-columns: 1fr 1fr;
+  }
+
+  .table-buttons div {
+    grid-column-end: -1;
+  }
 </style>

packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/modals/EditDisplayColumnsModal.svelte

@@ -0,0 +1,43 @@
+<script>
+  import { ModalContent, Select, Body } from "@budibase/bbui"
+  import { tables } from "stores/backend"
+
+  export let datasource
+  export let plusTables
+  export let save
+
+  async function saveDisplayColumns() {
+    // be explicit about copying over
+    for (let table of plusTables) {
+      datasource.entities[table.name].primaryDisplay = table.primaryDisplay
+    }
+    save()
+    await tables.fetch()
+  }
+
+  function getColumnOptions(table) {
+    if (!table || !table.schema) {
+      return []
+    }
+    return Object.entries(table.schema)
+      .filter(field => field[1].type !== "link")
+      .map(([fieldName]) => fieldName)
+  }
+</script>
+
+<ModalContent
+  title="Edit display columns"
+  confirmText="Save"
+  onConfirm={saveDisplayColumns}
+>
+  <Body
+    >Select the columns that will be shown when displaying relationships.</Body
+  >
+  {#each plusTables as table}
+    <Select
+      label={table.name}
+      options={getColumnOptions(table)}
+      bind:value={table.primaryDisplay}
+    />
+  {/each}
+</ModalContent>

packages/builder/src/pages/builder/auth/_components/OIDCButton.svelte

@@ -0,0 +1,32 @@
+<script>
+  import { ActionButton } from "@budibase/bbui"
+  // import { admin } from "stores/portal"
+
+  let show = true
+</script>
+
+{#if show}
+  <ActionButton on:click={() => window.open("/api/admin/auth/oidc", "_blank")}>
+    <div class="inner">
+      <p>Sign in with OIDC</p>
+    </div>
+  </ActionButton>
+{/if}
+
+<style>
+  .inner {
+    display: flex;
+    flex-direction: row;
+    align-items: center;
+    justify-content: center;
+    padding-top: var(--spacing-xs);
+    padding-bottom: var(--spacing-xs);
+  }
+  /* .inner img {
+    width: 18px;
+    margin: 3px 10px 3px 3px;
+  } */
+  .inner p {
+    margin: 0;
+  }
+</style>

packages/builder/src/pages/builder/auth/login.svelte

@@ -12,7 +12,7 @@
   import { goto, params } from "@roxi/routify"
   import { auth, organisation } from "stores/portal"
   import GoogleButton from "./_components/GoogleButton.svelte"
-  import OidcButton from "./_components/OidcButton.svelte"
+  import OIDCButton from "./_components/OIDCButton.svelte"
   import Logo from "assets/bb-emblem.svg"
   import { onMount } from "svelte"
 
@@ -62,7 +62,7 @@
         <Heading>Sign in to {company}</Heading>
       </Layout>
       <GoogleButton />
-      <OidcButton />
+      <OIDCButton />
       <Divider noGrid />
       <Layout gap="XS" noPadding>
         <Body size="S" textAlign="center">Sign in with email</Body>

packages/builder/src/pages/builder/portal/manage/users/[userId].svelte

@@ -60,6 +60,16 @@
 
   let toggleDisabled = false
 
+  async function updateUserFirstName(evt) {
+    await users.save({ ...$userFetch?.data, firstName: evt.target.value })
+    await userFetch.refresh()
+  }
+
+  async function updateUserLastName(evt) {
+    await users.save({ ...$userFetch?.data, lastName: evt.target.value })
+    await userFetch.refresh()
+  }
+
   async function toggleFlag(flagName, detail) {
     toggleDisabled = true
     await users.save({ ...$userFetch?.data, [flagName]: { global: detail } })
@@ -113,11 +123,19 @@
       </div>
       <div class="field">
         <Label size="L">First name</Label>
-        <Input disabled thin value={$userFetch?.data?.firstName} />
+        <Input
+          thin
+          value={$userFetch?.data?.firstName}
+          on:blur={updateUserFirstName}
+        />
       </div>
       <div class="field">
         <Label size="L">Last name</Label>
-        <Input disabled thin value={$userFetch?.data?.lastName} />
+        <Input
+          thin
+          value={$userFetch?.data?.lastName}
+          on:blur={updateUserLastName}
+        />
       </div>
       <!-- don't let a user remove the privileges that let them be here -->
       {#if userId !== $auth.user._id}

packages/builder/src/pages/builder/portal/manage/users/_components/UpdateRolesModal.svelte

@@ -9,7 +9,7 @@
   const dispatch = createEventDispatcher()
 
   const roles = app.roles
-  let options = roles.map(role => role._id)
+  let options = roles.map(role => ({ value: role._id, label: role.name }))
   let selectedRole = user?.roles?.[app?._id]
 
   async function updateUserRoles() {

packages/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/cli",
-  "version": "0.9.67",
+  "version": "0.9.70",
   "description": "Budibase CLI, for developers, self hosting and migrations.",
   "main": "src/index.js",
   "bin": {
@@ -24,7 +24,7 @@
     "docker-compose": "^0.23.6",
     "inquirer": "^8.0.0",
     "lookpath": "^1.1.0",
-    "pkg": "^4.4.9",
+    "pkg": "^5.3.0",
     "posthog-node": "1.0.7",
     "randomstring": "^1.1.5"
   },

packages/client/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/client",
-  "version": "0.9.67",
+  "version": "0.9.70",
   "license": "MPL-2.0",
   "module": "dist/budibase-client.js",
   "main": "dist/budibase-client.js",
@@ -18,9 +18,9 @@
     "dev:builder": "rollup -cw"
   },
   "dependencies": {
-    "@budibase/bbui": "^0.9.67",
+    "@budibase/bbui": "^0.9.70",
-    "@budibase/standard-components": "^0.9.67",
+    "@budibase/standard-components": "^0.9.70",
-    "@budibase/string-templates": "^0.9.67",
+    "@budibase/string-templates": "^0.9.70",
     "regexparam": "^1.3.0",
     "shortid": "^2.2.15",
     "svelte-spa-router": "^3.0.5"

packages/client/src/utils/styleable.js

@@ -34,11 +34,6 @@ export const styleable = (node, styles = {}) => {
       baseStyles.overflow = "hidden"
     }
 
-    // Append border-style css if border-width is specified
-    if (newStyles.normal?.["border-width"]) {
-      baseStyles["border-style"] = "solid"
-    }
-
     const componentId = newStyles.id
     const customStyles = newStyles.custom || ""
     const normalStyles = { ...baseStyles, ...newStyles.normal }

packages/server/.vscode/launch.json

@@ -5,10 +5,13 @@
     "version": "0.2.0",
     "configurations": [
       {
+        "name": "Start Server",
         "type": "node",
         "request": "launch",
-        "name": "Start Server",
+        "runtimeExecutable": "node",
-        "program": "${workspaceFolder}/src/index.js"
+        "runtimeArgs": ["--nolazy", "-r", "ts-node/register/transpile-only"],
+        "args": ["src/index.ts"],
+        "cwd": "${workspaceRoot}",
       },
       {
         "type": "node",

packages/server/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "0.9.67",
+  "version": "0.9.70",
   "description": "Budibase Web Server",
   "main": "src/index.js",
   "repository": {
@@ -59,9 +59,9 @@
   "author": "Budibase",
   "license": "AGPL-3.0-or-later",
   "dependencies": {
-    "@budibase/auth": "^0.9.67",
+    "@budibase/auth": "^0.9.70",
-    "@budibase/client": "^0.9.67",
+    "@budibase/client": "^0.9.70",
-    "@budibase/string-templates": "^0.9.67",
+    "@budibase/string-templates": "^0.9.70",
     "@elastic/elasticsearch": "7.10.0",
     "@koa/router": "8.0.0",
     "@sendgrid/mail": "7.1.1",
@@ -114,7 +114,7 @@
   "devDependencies": {
     "@babel/core": "^7.14.3",
     "@babel/preset-env": "^7.14.4",
-    "@budibase/standard-components": "^0.9.67",
+    "@budibase/standard-components": "^0.9.70",
     "@jest/test-sequencer": "^24.8.0",
     "@types/bull": "^3.15.1",
     "@types/jest": "^26.0.23",

packages/server/scripts/integrations/postgres/init.sql

@@ -15,3 +15,28 @@ CREATE TABLE Tasks (
         FOREIGN KEY(PersonID)
 	    REFERENCES Persons(PersonID)
 );
+CREATE TABLE Products (
+    ProductID INT NOT NULL PRIMARY KEY,
+    ProductName varchar(255)
+);
+CREATE TABLE Products_Tasks (
+    ProductID INT NOT NULL,
+    TaskID INT NOT NULL,
+    CONSTRAINT fkProducts
+        FOREIGN KEY(ProductID)
+	    REFERENCES Products(ProductID),
+    CONSTRAINT fkTasks
+        FOREIGN KEY(TaskID)
+	    REFERENCES Tasks(TaskID),
+    PRIMARY KEY (ProductID, TaskID)
+);
+INSERT INTO Persons (PersonID, FirstName, LastName, Address, City) VALUES (1, 'Mike', 'Hughes', '123 Fake Street', 'Belfast');
+INSERT INTO Tasks (TaskID, PersonID, TaskName) VALUES (1, 1, 'assembling');
+INSERT INTO Tasks (TaskID, PersonID, TaskName) VALUES (2, 1, 'processing');
+INSERT INTO Products (ProductID, ProductName) VALUES (1, 'Computers');
+INSERT INTO Products (ProductID, ProductName) VALUES (2, 'Laptops');
+INSERT INTO Products (ProductID, ProductName) VALUES (3, 'Chairs');
+INSERT INTO Products_Tasks (ProductID, TaskID) VALUES (1, 1);
+INSERT INTO Products_Tasks (ProductID, TaskID) VALUES (2, 1);
+INSERT INTO Products_Tasks (ProductID, TaskID) VALUES (3, 1);
+INSERT INTO Products_Tasks (ProductID, TaskID) VALUES (1, 2);

packages/server/scripts/integrations/postgres/reset.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+docker-compose down
+docker volume prune -f

packages/server/src/api/controllers/datasource.js

@@ -48,7 +48,7 @@ exports.buildSchemaFromDb = async function (ctx) {
 
   // Connect to the DB and build the schema
   const connector = new Connector(datasource.config)
-  await connector.buildSchema(datasource._id)
+  await connector.buildSchema(datasource._id, datasource.entities)
   datasource.entities = connector.tables
 
   const response = await db.post(datasource)

packages/server/src/api/controllers/row/ExternalRequest.ts

@@ -0,0 +1,525 @@
+import {
+  Operation,
+  SearchFilters,
+  SortJson,
+  PaginationJson,
+  RelationshipsJson,
+} from "../../../definitions/datasource"
+import {
+  Row,
+  Table,
+  FieldSchema,
+  Datasource,
+} from "../../../definitions/common"
+import {
+  breakRowIdField,
+  generateRowIdField,
+} from "../../../integrations/utils"
+
+interface ManyRelationship {
+  tableId?: string
+  id?: string
+  isUpdate?: boolean
+  [key: string]: any
+}
+
+interface RunConfig {
+  id: string
+  row: Row
+  filters: SearchFilters
+  sort: SortJson
+  paginate: PaginationJson
+}
+
+module External {
+  const { makeExternalQuery } = require("./utils")
+  const { DataSourceOperation, FieldTypes } = require("../../../constants")
+  const { breakExternalTableId, isSQL } = require("../../../integrations/utils")
+  const { processObjectSync } = require("@budibase/string-templates")
+  const { cloneDeep } = require("lodash/fp")
+  const { isEqual } = require("lodash")
+  const CouchDB = require("../../../db")
+
+  function buildFilters(
+    id: string | undefined,
+    filters: SearchFilters,
+    table: Table
+  ) {
+    const primary = table.primary
+    // if passed in array need to copy for shifting etc
+    let idCopy = cloneDeep(id)
+    if (filters) {
+      // need to map over the filters and make sure the _id field isn't present
+      for (let filter of Object.values(filters)) {
+        if (filter._id && primary) {
+          const parts = breakRowIdField(filter._id)
+          for (let field of primary) {
+            filter[field] = parts.shift()
+          }
+        }
+        // make sure this field doesn't exist on any filter
+        delete filter._id
+      }
+    }
+    // there is no id, just use the user provided filters
+    if (!idCopy || !table) {
+      return filters
+    }
+    // if used as URL parameter it will have been joined
+    if (!Array.isArray(idCopy)) {
+      idCopy = breakRowIdField(idCopy)
+    }
+    const equal: any = {}
+    if (primary && idCopy) {
+      for (let field of primary) {
+        // work through the ID and get the parts
+        equal[field] = idCopy.shift()
+      }
+    }
+    return {
+      equal,
+    }
+  }
+
+  function generateIdForRow(row: Row, table: Table): string {
+    const primary = table.primary
+    if (!row || !primary) {
+      return ""
+    }
+    // build id array
+    let idParts = []
+    for (let field of primary) {
+      if (row[field]) {
+        idParts.push(row[field])
+      }
+    }
+    if (idParts.length === 0) {
+      return ""
+    }
+    return generateRowIdField(idParts)
+  }
+
+  function getEndpoint(tableId: string | undefined, operation: string) {
+    if (!tableId) {
+      return {}
+    }
+    const { datasourceId, tableName } = breakExternalTableId(tableId)
+    return {
+      datasourceId,
+      entityId: tableName,
+      operation,
+    }
+  }
+
+  function basicProcessing(row: Row, table: Table) {
+    const thisRow: { [key: string]: any } = {}
+    // filter the row down to what is actually the row (not joined)
+    for (let fieldName of Object.keys(table.schema)) {
+      thisRow[fieldName] = row[fieldName]
+    }
+    thisRow._id = generateIdForRow(row, table)
+    thisRow.tableId = table._id
+    thisRow._rev = "rev"
+    return thisRow
+  }
+
+  function isMany(field: FieldSchema) {
+    return (
+      field.relationshipType && field.relationshipType.split("-")[0] === "many"
+    )
+  }
+
+  class ExternalRequest {
+    private readonly appId: string
+    private operation: Operation
+    private tableId: string
+    private datasource: Datasource
+    private tables: { [key: string]: Table } = {}
+
+    constructor(
+      appId: string,
+      operation: Operation,
+      tableId: string,
+      datasource: Datasource
+    ) {
+      this.appId = appId
+      this.operation = operation
+      this.tableId = tableId
+      this.datasource = datasource
+      if (datasource && datasource.entities) {
+        this.tables = datasource.entities
+      }
+    }
+
+    inputProcessing(row: Row, table: Table) {
+      if (!row) {
+        return { row, manyRelationships: [] }
+      }
+      // we don't really support composite keys for relationships, this is why [0] is used
+      // @ts-ignore
+      const tablePrimary: string = table.primary[0]
+      let newRow: Row = {},
+        manyRelationships: ManyRelationship[] = []
+      for (let [key, field] of Object.entries(table.schema)) {
+        // if set already, or not set just skip it
+        if (!row[key] || newRow[key]) {
+          continue
+        }
+        // if its not a link then just copy it over
+        if (field.type !== FieldTypes.LINK) {
+          newRow[key] = row[key]
+          continue
+        }
+        const { tableName: linkTableName } = breakExternalTableId(field.tableId)
+        // table has to exist for many to many
+        if (!this.tables[linkTableName]) {
+          continue
+        }
+        const linkTable = this.tables[linkTableName]
+        // @ts-ignore
+        const linkTablePrimary = linkTable.primary[0]
+        if (!isMany(field)) {
+          newRow[field.foreignKey || linkTablePrimary] = breakRowIdField(
+            row[key][0]
+          )[0]
+        } else {
+          // we're not inserting a doc, will be a bunch of update calls
+          const isUpdate = !field.through
+          const thisKey: string = isUpdate ? "id" : linkTablePrimary
+          // @ts-ignore
+          const otherKey: string = isUpdate ? field.foreignKey : tablePrimary
+          row[key].map((relationship: any) => {
+            // we don't really support composite keys for relationships, this is why [0] is used
+            manyRelationships.push({
+              tableId: field.through || field.tableId,
+              isUpdate,
+              [thisKey]: breakRowIdField(relationship)[0],
+              // leave the ID for enrichment later
+              [otherKey]: `{{ literal ${tablePrimary} }}`,
+            })
+          })
+        }
+      }
+      // we return the relationships that may need to be created in the through table
+      // we do this so that if the ID is generated by the DB it can be inserted
+      // after the fact
+      return { row: newRow, manyRelationships }
+    }
+
+    /**
+     * This iterates through the returned rows and works out what elements of the rows
+     * actually match up to another row (based on primary keys) - this is pretty specific
+     * to SQL and the way that SQL relationships are returned based on joins.
+     */
+    updateRelationshipColumns(
+      row: Row,
+      rows: { [key: string]: Row },
+      relationships: RelationshipsJson[]
+    ) {
+      const columns: { [key: string]: any } = {}
+      for (let relationship of relationships) {
+        const linkedTable = this.tables[relationship.tableName]
+        if (!linkedTable) {
+          continue
+        }
+        let linked = basicProcessing(row, linkedTable)
+        if (!linked._id) {
+          continue
+        }
+        // if not returning full docs then get the minimal links out
+        const display = linkedTable.primaryDisplay
+        linked = {
+          primaryDisplay: display ? linked[display] : undefined,
+          _id: linked._id,
+        }
+        columns[relationship.column] = linked
+      }
+      for (let [column, related] of Object.entries(columns)) {
+        if (!row._id) {
+          continue
+        }
+        const rowId: string = row._id
+        if (!Array.isArray(rows[rowId][column])) {
+          rows[rowId][column] = []
+        }
+        // make sure relationship hasn't been found already
+        if (
+          !rows[rowId][column].find(
+            (relation: Row) => relation._id === related._id
+          )
+        ) {
+          rows[rowId][column].push(related)
+        }
+      }
+      return rows
+    }
+
+    outputProcessing(
+      rows: Row[],
+      table: Table,
+      relationships: RelationshipsJson[]
+    ) {
+      if (rows[0].read === true) {
+        return []
+      }
+      let finalRows: { [key: string]: Row } = {}
+      for (let row of rows) {
+        const rowId = generateIdForRow(row, table)
+        row._id = rowId
+        // this is a relationship of some sort
+        if (finalRows[rowId]) {
+          finalRows = this.updateRelationshipColumns(
+            row,
+            finalRows,
+            relationships
+          )
+          continue
+        }
+        const thisRow = basicProcessing(row, table)
+        finalRows[thisRow._id] = thisRow
+        // do this at end once its been added to the final rows
+        finalRows = this.updateRelationshipColumns(
+          row,
+          finalRows,
+          relationships
+        )
+      }
+      return Object.values(finalRows)
+    }
+
+    /**
+     * Gets the list of relationship JSON structures based on the columns in the table,
+     * this will be used by the underlying library to build whatever relationship mechanism
+     * it has (e.g. SQL joins).
+     */
+    buildRelationships(table: Table): RelationshipsJson[] {
+      const relationships = []
+      for (let [fieldName, field] of Object.entries(table.schema)) {
+        if (field.type !== FieldTypes.LINK) {
+          continue
+        }
+        const { tableName: linkTableName } = breakExternalTableId(field.tableId)
+        // no table to link to, this is not a valid relationships
+        if (!this.tables[linkTableName]) {
+          continue
+        }
+        const linkTable = this.tables[linkTableName]
+        if (!table.primary || !linkTable.primary) {
+          continue
+        }
+        const definition = {
+          // if no foreign key specified then use the name of the field in other table
+          from: field.foreignKey || table.primary[0],
+          to: field.fieldName,
+          tableName: linkTableName,
+          through: undefined,
+          // need to specify where to put this back into
+          column: fieldName,
+        }
+        if (field.through) {
+          const { tableName: throughTableName } = breakExternalTableId(
+            field.through
+          )
+          definition.through = throughTableName
+          // don't support composite keys for relationships
+          definition.from = table.primary[0]
+          definition.to = linkTable.primary[0]
+        }
+        relationships.push(definition)
+      }
+      return relationships
+    }
+
+    /**
+     * This is a cached lookup, of relationship records, this is mainly for creating/deleting junction
+     * information.
+     */
+    async lookup(
+      row: Row,
+      relationship: ManyRelationship,
+      cache: { [key: string]: Row[] } = {}
+    ) {
+      const { tableId, isUpdate, id, ...rest } = relationship
+      const { tableName } = breakExternalTableId(tableId)
+      const table = this.tables[tableName]
+      if (isUpdate) {
+        return { rows: [], table }
+      }
+      // if not updating need to make sure we have a list of all possible options
+      let fullKey: string = tableId + "/",
+        rowKey: string = ""
+      for (let key of Object.keys(rest)) {
+        if (row[key]) {
+          fullKey += key
+          rowKey = key
+        }
+      }
+      if (cache[fullKey] == null) {
+        cache[fullKey] = await makeExternalQuery(this.appId, {
+          endpoint: getEndpoint(tableId, DataSourceOperation.READ),
+          filters: {
+            equal: {
+              [rowKey]: row[rowKey],
+            },
+          },
+        })
+      }
+      return { rows: cache[fullKey], table }
+    }
+
+    /**
+     * Once a row has been written we may need to update a many field, e.g. updating foreign keys
+     * in a bunch of rows in another table, or inserting/deleting rows from a junction table (many to many).
+     * This is quite a complex process and is handled by this function, there are a few things going on here:
+     * 1. If updating foreign keys its relatively simple, just create a filter for the row that needs updated
+     * and write the various components.
+     * 2. If junction table, then we lookup what exists already, write what doesn't exist, work out what
+     * isn't supposed to exist anymore and delete those. This is better than the usual method of delete them
+     * all and then re-create, as theres no chance of losing data (e.g. delete succeed, but write fail).
+     */
+    async handleManyRelationships(row: Row, relationships: ManyRelationship[]) {
+      const { appId } = this
+      if (relationships.length === 0) {
+        return
+      }
+      // if we're creating (in a through table) need to wipe the existing ones first
+      const promises = []
+      const cache: { [key: string]: Row[] } = {}
+      for (let relationship of relationships) {
+        const { tableId, isUpdate, id, ...rest } = relationship
+        const body = processObjectSync(rest, row)
+        const { table, rows } = await this.lookup(row, relationship, cache)
+        const found = rows.find(row => isEqual(body, row))
+        const operation = isUpdate
+          ? DataSourceOperation.UPDATE
+          : DataSourceOperation.CREATE
+        if (!found) {
+          promises.push(
+            makeExternalQuery(appId, {
+              endpoint: getEndpoint(tableId, operation),
+              // if we're doing many relationships then we're writing, only one response
+              body,
+              filters: buildFilters(id, {}, table),
+            })
+          )
+        } else {
+          // remove the relationship from the rows
+          rows.splice(rows.indexOf(found), 1)
+        }
+      }
+      // finally if creating, cleanup any rows that aren't supposed to be here
+      for (let [key, rows] of Object.entries(cache)) {
+        // @ts-ignore
+        const tableId: string = key.split("/").shift()
+        const { tableName } = breakExternalTableId(tableId)
+        const table = this.tables[tableName]
+        for (let row of rows) {
+          promises.push(
+            makeExternalQuery(this.appId, {
+              endpoint: getEndpoint(tableId, DataSourceOperation.DELETE),
+              filters: buildFilters(generateIdForRow(row, table), {}, table),
+            })
+          )
+        }
+      }
+      await Promise.all(promises)
+    }
+
+    /**
+     * This function is a bit crazy, but the exact purpose of it is to protect against the scenario in which
+     * you have column overlap in relationships, e.g. we join a few different tables and they all have the
+     * concept of an ID, but for some of them it will be null (if they say don't have a relationship).
+     * Creating the specific list of fields that we desire, and excluding the ones that are no use to us
+     * is more performant and has the added benefit of protecting against this scenario.
+     */
+    buildFields(table: Table) {
+      function extractNonLinkFieldNames(table: Table, existing: string[] = []) {
+        return Object.entries(table.schema)
+          .filter(
+            column =>
+              column[1].type !== FieldTypes.LINK &&
+              !existing.find((field: string) => field.includes(column[0]))
+          )
+          .map(column => `${table.name}.${column[0]}`)
+      }
+      let fields = extractNonLinkFieldNames(table)
+      for (let field of Object.values(table.schema)) {
+        if (field.type !== FieldTypes.LINK) {
+          continue
+        }
+        const { tableName: linkTableName } = breakExternalTableId(field.tableId)
+        const linkTable = this.tables[linkTableName]
+        if (linkTable) {
+          const linkedFields = extractNonLinkFieldNames(linkTable, fields)
+          fields = fields.concat(linkedFields)
+        }
+      }
+      return fields
+    }
+
+    async run({ id, row, filters, sort, paginate }: RunConfig) {
+      const { appId, operation, tableId } = this
+      let { datasourceId, tableName } = breakExternalTableId(tableId)
+      if (!this.datasource) {
+        const db = new CouchDB(appId)
+        this.datasource = await db.get(datasourceId)
+        if (!this.datasource || !this.datasource.entities) {
+          throw "No tables found, fetch tables before query."
+        }
+        this.tables = this.datasource.entities
+      }
+      const table = this.tables[tableName]
+      let isSql = isSQL(this.datasource)
+      if (!table) {
+        throw `Unable to process query, table "${tableName}" not defined.`
+      }
+      // clean up row on ingress using schema
+      filters = buildFilters(id, filters, table)
+      const relationships = this.buildRelationships(table)
+      const processed = this.inputProcessing(row, table)
+      row = processed.row
+      if (
+        operation === DataSourceOperation.DELETE &&
+        (filters == null || Object.keys(filters).length === 0)
+      ) {
+        throw "Deletion must be filtered"
+      }
+      let json = {
+        endpoint: {
+          datasourceId,
+          entityId: tableName,
+          operation,
+        },
+        resource: {
+          // have to specify the fields to avoid column overlap (for SQL)
+          fields: isSql ? this.buildFields(table) : [],
+        },
+        filters,
+        sort,
+        paginate,
+        relationships,
+        body: row,
+        // pass an id filter into extra, purely for mysql/returning
+        extra: {
+          idFilter: buildFilters(id || generateIdForRow(row, table), {}, table),
+        },
+      }
+      // can't really use response right now
+      const response = await makeExternalQuery(appId, json)
+      // handle many to many relationships now if we know the ID (could be auto increment)
+      if (processed.manyRelationships) {
+        await this.handleManyRelationships(
+          response[0],
+          processed.manyRelationships
+        )
+      }
+      const output = this.outputProcessing(response, table, relationships)
+      // if reading it'll just be an array of rows, return whole thing
+      return operation === DataSourceOperation.READ && Array.isArray(response)
+        ? output
+        : { row: output[0], table }
+    }
+  }
+
+  module.exports = ExternalRequest
+}

packages/server/src/api/controllers/row/external.js

@@ -1,136 +1,19 @@
-const { makeExternalQuery } = require("./utils")
+const {
-const { DataSourceOperation, SortDirection } = require("../../../constants")
+  DataSourceOperation,
-const { getExternalTable } = require("../table/utils")
+  SortDirection,
+  FieldTypes,
+} = require("../../../constants")
 const {
   breakExternalTableId,
-  generateRowIdField,
   breakRowIdField,
 } = require("../../../integrations/utils")
-const { cloneDeep } = require("lodash/fp")
+const ExternalRequest = require("./ExternalRequest")
+const CouchDB = require("../../../db")
 
-function inputProcessing(row, table) {
+async function handleRequest(appId, operation, tableId, opts = {}) {
-  if (!row) {
+  return new ExternalRequest(appId, operation, tableId, opts.datasource).run(
-    return row
+    opts
-  }
+  )
-  let newRow = {}
-  for (let key of Object.keys(table.schema)) {
-    // currently excludes empty strings
-    if (row[key]) {
-      newRow[key] = row[key]
-    }
-  }
-  return newRow
-}
-
-function generateIdForRow(row, table) {
-  if (!row) {
-    return
-  }
-  const primary = table.primary
-  // build id array
-  let idParts = []
-  for (let field of primary) {
-    idParts.push(row[field])
-  }
-  return generateRowIdField(idParts)
-}
-
-function outputProcessing(rows, table) {
-  // if no rows this is what is returned? Might be PG only
-  if (rows[0].read === true) {
-    return []
-  }
-  for (let row of rows) {
-    row._id = generateIdForRow(row, table)
-    row.tableId = table._id
-    row._rev = "rev"
-  }
-  return rows
-}
-
-function buildFilters(id, filters, table) {
-  const primary = table.primary
-  // if passed in array need to copy for shifting etc
-  let idCopy = cloneDeep(id)
-  if (filters) {
-    // need to map over the filters and make sure the _id field isn't present
-    for (let filter of Object.values(filters)) {
-      if (filter._id) {
-        const parts = breakRowIdField(filter._id)
-        for (let field of primary) {
-          filter[field] = parts.shift()
-        }
-      }
-      // make sure this field doesn't exist on any filter
-      delete filter._id
-    }
-  }
-  // there is no id, just use the user provided filters
-  if (!idCopy || !table) {
-    return filters
-  }
-  // if used as URL parameter it will have been joined
-  if (typeof idCopy === "string") {
-    idCopy = breakRowIdField(idCopy)
-  }
-  const equal = {}
-  for (let field of primary) {
-    // work through the ID and get the parts
-    equal[field] = idCopy.shift()
-  }
-  return {
-    equal,
-  }
-}
-
-async function handleRequest(
-  appId,
-  operation,
-  tableId,
-  { id, row, filters, sort, paginate } = {}
-) {
-  let { datasourceId, tableName } = breakExternalTableId(tableId)
-  const table = await getExternalTable(appId, datasourceId, tableName)
-  if (!table) {
-    throw `Unable to process query, table "${tableName}" not defined.`
-  }
-  // clean up row on ingress using schema
-  filters = buildFilters(id, filters, table)
-  row = inputProcessing(row, table)
-  if (
-    operation === DataSourceOperation.DELETE &&
-    (filters == null || Object.keys(filters).length === 0)
-  ) {
-    throw "Deletion must be filtered"
-  }
-  let json = {
-    endpoint: {
-      datasourceId,
-      entityId: tableName,
-      operation,
-    },
-    resource: {
-      // not specifying any fields means "*"
-      fields: [],
-    },
-    filters,
-    sort,
-    paginate,
-    body: row,
-    // pass an id filter into extra, purely for mysql/returning
-    extra: {
-      idFilter: buildFilters(id || generateIdForRow(row, table), {}, table),
-    },
-  }
-  // can't really use response right now
-  const response = await makeExternalQuery(appId, json)
-  // we searched for rows in someway
-  if (operation === DataSourceOperation.READ && Array.isArray(response)) {
-    return outputProcessing(response, table)
-  } else {
-    row = outputProcessing(response, table)[0]
-    return { row, table }
-  }
 }
 
 exports.patch = async ctx => {
@@ -172,9 +55,15 @@ exports.find = async ctx => {
   const appId = ctx.appId
   const id = ctx.params.rowId
   const tableId = ctx.params.tableId
-  return handleRequest(appId, DataSourceOperation.READ, tableId, {
+  const response = await handleRequest(
+    appId,
+    DataSourceOperation.READ,
+    tableId,
+    {
       id,
-  })
+    }
+  )
+  return response ? response[0] : response
 }
 
 exports.destroy = async ctx => {
@@ -270,7 +159,56 @@ exports.validate = async () => {
   return { valid: true }
 }
 
-exports.fetchEnrichedRow = async () => {
+exports.fetchEnrichedRow = async ctx => {
-  // TODO: How does this work
+  const appId = ctx.appId
-  throw "Not Implemented"
+  const id = ctx.params.rowId
+  const tableId = ctx.params.tableId
+  const { datasourceId, tableName } = breakExternalTableId(tableId)
+  const db = new CouchDB(appId)
+  const datasource = await db.get(datasourceId)
+  if (!datasource || !datasource.entities) {
+    ctx.throw(400, "Datasource has not been configured for plus API.")
+  }
+  const tables = datasource.entities
+  const response = await handleRequest(
+    appId,
+    DataSourceOperation.READ,
+    tableId,
+    {
+      id,
+      datasource,
+    }
+  )
+  const table = tables[tableName]
+  const row = response[0]
+  // this seems like a lot of work, but basically we need to dig deeper for the enrich
+  // for a single row, there is probably a better way to do this with some smart multi-layer joins
+  for (let [fieldName, field] of Object.entries(table.schema)) {
+    if (
+      field.type !== FieldTypes.LINK ||
+      !row[fieldName] ||
+      row[fieldName].length === 0
+    ) {
+      continue
+    }
+    const links = row[fieldName]
+    const linkedTableId = field.tableId
+    const linkedTable = tables[breakExternalTableId(linkedTableId).tableName]
+    // don't support composite keys right now
+    const linkedIds = links.map(link => breakRowIdField(link._id)[0])
+    row[fieldName] = await handleRequest(
+      appId,
+      DataSourceOperation.READ,
+      linkedTableId,
+      {
+        tables,
+        filters: {
+          oneOf: {
+            [linkedTable.primary]: linkedIds,
+          },
+        },
+      }
+    )
+  }
+  return row
 }

packages/server/src/api/controllers/table/utils.js

@@ -204,15 +204,18 @@ class TableSaveFunctions {
   }
 }
 
-exports.getExternalTable = async (appId, datasourceId, tableName) => {
+exports.getAllExternalTables = async (appId, datasourceId) => {
   const db = new CouchDB(appId)
   const datasource = await db.get(datasourceId)
   if (!datasource || !datasource.entities) {
     throw "Datasource is not configured fully."
   }
-  return Object.values(datasource.entities).find(
+  return datasource.entities
-    entity => entity.name === tableName
+}
-  )
+
+exports.getExternalTable = async (appId, datasourceId, tableName) => {
+  const entities = await exports.getAllExternalTables(appId, datasourceId)
+  return entities[tableName]
 }
 
 exports.TableSaveFunctions = TableSaveFunctions

packages/server/src/api/routes/tests/datasource.spec.js

@@ -94,7 +94,7 @@ describe("/datasources", () => {
         .expect(200)
       // this is mock data, can't test it
       expect(res.body).toBeDefined()
-      expect(pg.queryMock).toHaveBeenCalledWith(`select "name", "age" from "users" where "name" like $1 limit $2`, ["John%", 5000])
+      expect(pg.queryMock).toHaveBeenCalledWith(`select "name", "age" from "users" where "users"."name" like $1 limit $2`, ["John%", 5000])
     })
   })
 

packages/server/src/constants/definitions.ts

@@ -26,3 +26,17 @@ export interface Table {
   primaryDisplay?: string
   sourceId?: string
 }
+
+export interface BudibaseAppMetadata {
+  _id: string
+  _rev?: string
+  appId: string
+  type: string
+  version: string
+  componentlibraries: string[]
+  name: string
+  url: string
+  instance: { _id: string }
+  updatedAt: Date
+  createdAt: Date
+}

packages/server/src/definitions/common.ts

@@ -0,0 +1,100 @@
+import { SourceNames } from "./datasource"
+
+interface Base {
+  _id?: string
+  _rev?: string
+}
+
+export interface FieldSchema {
+  // TODO: replace with field types enum when done
+  type: string
+  fieldName?: string
+  name: string
+  tableId?: string
+  relationshipType?: string
+  through?: string
+  foreignKey?: string
+  constraints?: {
+    type?: string
+    email?: boolean
+    inclusion?: string[]
+    length?: {
+      minimum?: string | number
+      maximum?: string | number
+    }
+    presence?: boolean
+  }
+}
+
+export interface TableSchema {
+  [key: string]: FieldSchema
+}
+
+export interface Table extends Base {
+  type?: string
+  views?: {}
+  name?: string
+  primary?: string[]
+  schema: TableSchema
+  primaryDisplay?: string
+  sourceId?: string
+}
+
+export interface Row extends Base {
+  type?: string
+  tableId?: string
+  [key: string]: any
+}
+
+interface JsonSchemaField {
+  properties: {
+    [key: string]: {
+      type: string
+      title: string
+      customType?: string
+    }
+  }
+  required?: string[]
+}
+
+export interface AutomationStep {
+  description: string
+  event?: string
+  icon: string
+  id: string
+  inputs: {
+    [key: string]: any
+  }
+  name: string
+  schema: {
+    inputs: JsonSchemaField
+    outputs: JsonSchemaField
+  }
+  stepId: string
+  tagline: string
+  type: string
+}
+
+export interface Automation extends Base {
+  name: string
+  type: string
+  appId?: string
+  definition: {
+    steps: AutomationStep[]
+    trigger?: AutomationStep
+  }
+}
+
+export interface Datasource extends Base {
+  type: string
+  name: string
+  source: SourceNames
+  // the config is defined by the schema
+  config: {
+    [key: string]: string | number | boolean
+  }
+  plus: boolean
+  entities?: {
+    [key: string]: Table
+  }
+}

packages/server/src/definitions/datasource.ts

@@ -26,6 +26,20 @@ export enum DatasourceFieldTypes {
   JSON = "json",
 }
 
+export enum SourceNames {
+  POSTGRES = "POSTGRES",
+  DYNAMODB = "DYNAMODB",
+  MONGODB = "MONGODB",
+  ELASTICSEARCH = "ELASTICSEARCH",
+  COUCHDB = "COUCHDB",
+  SQL_SERVER = "SQL_SERVER",
+  S3 = "S3",
+  AIRTABLE = "AIRTABLE",
+  MYSQL = "MYSQL",
+  ARANGODB = "ARANGODB",
+  REST = "REST",
+}
+
 export interface QueryDefinition {
   type: QueryTypes
   displayName?: string
@@ -47,7 +61,7 @@ export interface Integration {
 }
 
 export interface SearchFilters {
-  allOr: boolean
+  allOr?: boolean
   string?: {
     [key: string]: string
   }
@@ -72,6 +86,26 @@ export interface SearchFilters {
   notEmpty?: {
     [key: string]: any
   }
+  oneOf?: {
+    [key: string]: any[]
+  }
+}
+
+export interface SortJson {
+  [key: string]: SortDirection
+}
+
+export interface PaginationJson {
+  limit: number
+  page: string | number
+}
+
+export interface RelationshipsJson {
+  through?: string
+  from?: string
+  to?: string
+  tableName: string
+  column: string
 }
 
 export interface QueryJson {
@@ -84,17 +118,13 @@ export interface QueryJson {
     fields: string[]
   }
   filters?: SearchFilters
-  sort?: {
+  sort?: SortJson
-    [key: string]: SortDirection
+  paginate?: PaginationJson
-  }
-  paginate?: {
-    limit: number
-    page: string | number
-  }
   body?: object
-  extra: {
+  extra?: {
     idFilter?: SearchFilters
   }
+  relationships?: RelationshipsJson[]
 }
 
 export interface SqlQuery {

packages/server/src/integrations/airtable.ts

@@ -2,7 +2,7 @@ import {
   Integration,
   DatasourceFieldTypes,
   QueryTypes,
-} from "./base/definitions"
+} from "../definitions/datasource"
 
 module AirtableModule {
   const Airtable = require("airtable")

packages/server/src/integrations/arangodb.ts

@@ -2,7 +2,7 @@ import {
   Integration,
   DatasourceFieldTypes,
   QueryTypes,
-} from "./base/definitions"
+} from "../definitions/datasource"
 
 module ArangoModule {
   const { Database, aql } = require("arangojs")

packages/server/src/integrations/base/sql.ts

@@ -6,18 +6,23 @@ import {
   QueryOptions,
   SortDirection,
   Operation,
-} from "./definitions"
+  RelationshipsJson,
+} from "../../definitions/datasource"
 
+type KnexQuery = Knex.QueryBuilder | Knex
+
+// right now we only do filters on the specific table being queried
 function addFilters(
-  query: any,
+  tableName: string,
+  query: KnexQuery,
   filters: SearchFilters | undefined
-): Knex.QueryBuilder {
+): KnexQuery {
   function iterate(
     structure: { [key: string]: any },
     fn: (key: string, value: any) => void
   ) {
     for (let [key, value] of Object.entries(structure)) {
-      fn(key, value)
+      fn(`${tableName}.${key}`, value)
     }
   }
   if (!filters) {
@@ -25,6 +30,12 @@ function addFilters(
   }
   // if all or specified in filters, then everything is an or
   const allOr = filters.allOr
+  if (filters.oneOf) {
+    iterate(filters.oneOf, (key, array) => {
+      const fnc = allOr ? "orWhereIn" : "whereIn"
+      query = query[fnc](key, array)
+    })
+  }
   if (filters.string) {
     iterate(filters.string, (key, value) => {
       const fnc = allOr ? "orWhere" : "where"
@@ -67,9 +78,47 @@ function addFilters(
   return query
 }
 
-function buildCreate(knex: Knex, json: QueryJson, opts: QueryOptions) {
+function addRelationships(
+  query: KnexQuery,
+  fromTable: string,
+  relationships: RelationshipsJson[] | undefined
+): KnexQuery {
+  if (!relationships) {
+    return query
+  }
+  for (let relationship of relationships) {
+    const from = relationship.from,
+      to = relationship.to,
+      toTable = relationship.tableName
+    if (!relationship.through) {
+      // @ts-ignore
+      query = query.leftJoin(
+        toTable,
+        `${fromTable}.${from}`,
+        `${relationship.tableName}.${to}`
+      )
+    } else {
+      const throughTable = relationship.through
+      query = query
+        // @ts-ignore
+        .leftJoin(
+          throughTable,
+          `${fromTable}.${from}`,
+          `${throughTable}.${from}`
+        )
+        .leftJoin(toTable, `${toTable}.${to}`, `${throughTable}.${to}`)
+    }
+  }
+  return query
+}
+
+function buildCreate(
+  knex: Knex,
+  json: QueryJson,
+  opts: QueryOptions
+): KnexQuery {
   const { endpoint, body } = json
-  let query = knex(endpoint.entityId)
+  let query: KnexQuery = knex(endpoint.entityId)
   // mysql can't use returning
   if (opts.disableReturning) {
     return query.insert(body)
@@ -78,9 +127,10 @@ function buildCreate(knex: Knex, json: QueryJson, opts: QueryOptions) {
   }
 }
 
-function buildRead(knex: Knex, json: QueryJson, limit: number) {
+function buildRead(knex: Knex, json: QueryJson, limit: number): KnexQuery {
-  let { endpoint, resource, filters, sort, paginate } = json
+  let { endpoint, resource, filters, sort, paginate, relationships } = json
-  let query: Knex.QueryBuilder = knex(endpoint.entityId)
+  const tableName = endpoint.entityId
+  let query: KnexQuery = knex(tableName)
   // select all if not specified
   if (!resource) {
     resource = { fields: [] }
@@ -92,7 +142,9 @@ function buildRead(knex: Knex, json: QueryJson, limit: number) {
     query = query.select("*")
   }
   // handle where
-  query = addFilters(query, filters)
+  query = addFilters(tableName, query, filters)
+  // handle join
+  query = addRelationships(query, tableName, relationships)
   // handle sorting
   if (sort) {
     for (let [key, value] of Object.entries(sort)) {
@@ -114,10 +166,14 @@ function buildRead(knex: Knex, json: QueryJson, limit: number) {
   return query
 }
 
-function buildUpdate(knex: Knex, json: QueryJson, opts: QueryOptions) {
+function buildUpdate(
+  knex: Knex,
+  json: QueryJson,
+  opts: QueryOptions
+): KnexQuery {
   const { endpoint, body, filters } = json
-  let query = knex(endpoint.entityId)
+  let query: KnexQuery = knex(endpoint.entityId)
-  query = addFilters(query, filters)
+  query = addFilters(endpoint.entityId, query, filters)
   // mysql can't use returning
   if (opts.disableReturning) {
     return query.update(body)
@@ -126,10 +182,14 @@ function buildUpdate(knex: Knex, json: QueryJson, opts: QueryOptions) {
   }
 }
 
-function buildDelete(knex: Knex, json: QueryJson, opts: QueryOptions) {
+function buildDelete(
+  knex: Knex,
+  json: QueryJson,
+  opts: QueryOptions
+): KnexQuery {
   const { endpoint, filters } = json
-  let query = knex(endpoint.entityId)
+  let query: KnexQuery = knex(endpoint.entityId)
-  query = addFilters(query, filters)
+  query = addFilters(endpoint.entityId, query, filters)
   // mysql can't use returning
   if (opts.disableReturning) {
     return query.delete()
@@ -180,6 +240,8 @@ class SqlQueryBuilder {
       default:
         throw `Operation type is not supported by SQL query builder`
     }
+
+    // @ts-ignore
     return query.toSQL().toNative()
   }
 }

packages/server/src/integrations/couchdb.ts

@@ -2,7 +2,7 @@ import {
   Integration,
   DatasourceFieldTypes,
   QueryTypes,
-} from "./base/definitions"
+} from "../definitions/datasource"
 
 module CouchDBModule {
   const PouchDB = require("pouchdb")

packages/server/src/integrations/dynamodb.ts

@@ -2,7 +2,7 @@ import {
   Integration,
   DatasourceFieldTypes,
   QueryTypes,
-} from "./base/definitions"
+} from "../definitions/datasource"
 
 module DynamoModule {
   const AWS = require("aws-sdk")

packages/server/src/integrations/elasticsearch.ts

@@ -2,7 +2,7 @@ import {
   Integration,
   DatasourceFieldTypes,
   QueryTypes,
-} from "./base/definitions"
+} from "../definitions/datasource"
 
 module ElasticsearchModule {
   const { Client } = require("@elastic/elasticsearch")

packages/server/src/integrations/index.ts

@@ -9,33 +9,34 @@ const airtable = require("./airtable")
 const mysql = require("./mysql")
 const arangodb = require("./arangodb")
 const rest = require("./rest")
+const { SourceNames } = require("../definitions/datasource")
 
 const DEFINITIONS = {
-  POSTGRES: postgres.schema,
+  [SourceNames.POSTGRES]: postgres.schema,
-  DYNAMODB: dynamodb.schema,
+  [SourceNames.DYNAMODB]: dynamodb.schema,
-  MONGODB: mongodb.schema,
+  [SourceNames.MONGODB]: mongodb.schema,
-  ELASTICSEARCH: elasticsearch.schema,
+  [SourceNames.ELASTICSEARCH]: elasticsearch.schema,
-  COUCHDB: couchdb.schema,
+  [SourceNames.COUCHDB]: couchdb.schema,
-  SQL_SERVER: sqlServer.schema,
+  [SourceNames.SQL_SERVER]: sqlServer.schema,
-  S3: s3.schema,
+  [SourceNames.S3]: s3.schema,
-  AIRTABLE: airtable.schema,
+  [SourceNames.AIRTABLE]: airtable.schema,
-  MYSQL: mysql.schema,
+  [SourceNames.MYSQL]: mysql.schema,
-  ARANGODB: arangodb.schema,
+  [SourceNames.ARANGODB]: arangodb.schema,
-  REST: rest.schema,
+  [SourceNames.REST]: rest.schema,
 }
 
 const INTEGRATIONS = {
-  POSTGRES: postgres.integration,
+  [SourceNames.POSTGRES]: postgres.integration,
-  DYNAMODB: dynamodb.integration,
+  [SourceNames.DYNAMODB]: dynamodb.integration,
-  MONGODB: mongodb.integration,
+  [SourceNames.MONGODB]: mongodb.integration,
-  ELASTICSEARCH: elasticsearch.integration,
+  [SourceNames.ELASTICSEARCH]: elasticsearch.integration,
-  COUCHDB: couchdb.integration,
+  [SourceNames.COUCHDB]: couchdb.integration,
-  S3: s3.integration,
+  [SourceNames.SQL_SERVER]: s3.integration,
-  SQL_SERVER: sqlServer.integration,
+  [SourceNames.S3]: sqlServer.integration,
-  AIRTABLE: airtable.integration,
+  [SourceNames.AIRTABLE]: airtable.integration,
-  MYSQL: mysql.integration,
+  [SourceNames.MYSQL]: mysql.integration,
-  ARANGODB: arangodb.integration,
+  [SourceNames.ARANGODB]: arangodb.integration,
-  REST: rest.integration,
+  [SourceNames.REST]: rest.integration,
 }
 
 module.exports = {

packages/server/src/integrations/microsoftSqlServer.ts

@@ -4,7 +4,7 @@ import {
   QueryTypes,
   QueryJson,
   SqlQuery,
-} from "./base/definitions"
+} from "../definitions/datasource"
 import { getSqlQuery } from "./utils"
 
 module MSSQLModule {

packages/server/src/integrations/mongodb.ts

@@ -2,7 +2,7 @@ import {
   Integration,
   DatasourceFieldTypes,
   QueryTypes,
-} from "./base/definitions"
+} from "../definitions/datasource"
 
 module MongoDBModule {
   const { MongoClient } = require("mongodb")

packages/server/src/integrations/mysql.ts

@@ -5,7 +5,8 @@ import {
   Operation,
   QueryJson,
   SqlQuery,
-} from "./base/definitions"
+} from "../definitions/datasource"
+import { Table, TableSchema } from "../definitions/common"
 import { getSqlQuery } from "./utils"
 
 module MySQLModule {
@@ -139,7 +140,7 @@ module MySQLModule {
     }
 
     async buildSchema(datasourceId: string) {
-      const tables: any = {}
+      const tables: { [key: string]: Table } = {}
       const database = this.config.database
       this.client.connect()
 
@@ -154,7 +155,7 @@ module MySQLModule {
       )
       for (let tableName of tableNames) {
         const primaryKeys = []
-        const schema: any = {}
+        const schema: TableSchema = {}
         const descResp = await internalQuery(
           this.client,
           { sql: `DESCRIBE ${tableName};` },
@@ -166,7 +167,7 @@ module MySQLModule {
             primaryKeys.push(columnName)
           }
           const constraints = {
-            required: column.Null !== "YES",
+            presence: column.Null !== "YES",
           }
           schema[columnName] = {
             name: columnName,
@@ -212,7 +213,7 @@ module MySQLModule {
     }
 
     async getReturningRow(json: QueryJson) {
-      if (!json.extra.idFilter) {
+      if (!json.extra || !json.extra.idFilter) {
         return {}
       }
       const input = this._query({

packages/server/src/integrations/postgres.ts

@@ -4,8 +4,8 @@ import {
   QueryTypes,
   QueryJson,
   SqlQuery,
-} from "./base/definitions"
+} from "../definitions/datasource"
-import { Table } from "../constants/definitions"
+import { Table } from "../definitions/common"
 import { getSqlQuery } from "./utils"
 
 module PostgresModule {
@@ -134,8 +134,9 @@ module PostgresModule {
     /**
      * Fetches the tables from the postgres table and assigns them to the datasource.
      * @param {*} datasourceId - datasourceId to fetch
+     * @param entities - the tables that are to be built
      */
-    async buildSchema(datasourceId: string) {
+    async buildSchema(datasourceId: string, entities: Record<string, Table>) {
       let tableKeys: { [key: string]: string[] } = {}
       try {
         const primaryKeysResponse = await this.client.query(
@@ -167,6 +168,19 @@ module PostgresModule {
             name: tableName,
             schema: {},
           }
+
+          // add the existing relationships from the entities if they exist, to prevent them from being overridden
+          if (entities && entities[tableName]) {
+            const existingTableSchema = entities[tableName].schema
+            for (let key in existingTableSchema) {
+              if (!existingTableSchema.hasOwnProperty(key)) {
+                continue
+              }
+              if (existingTableSchema[key].type === "link") {
+                tables[tableName].schema[key] = existingTableSchema[key]
+              }
+            }
+          }
         }
 
         const type: string = convertType(column.data_type, TYPE_MAP)

packages/server/src/integrations/rest.ts

@@ -2,7 +2,7 @@ import {
   Integration,
   DatasourceFieldTypes,
   QueryTypes,
-} from "./base/definitions"
+} from "../definitions/datasource"
 
 module RestModule {
   const fetch = require("node-fetch")

packages/server/src/integrations/s3.ts

@@ -1,4 +1,4 @@
-import { Integration, QueryTypes } from "./base/definitions"
+import { Integration, QueryTypes } from "../definitions/datasource"
 
 module S3Module {
   const AWS = require("aws-sdk")

packages/server/src/integrations/tests/sql.spec.js

@@ -81,7 +81,7 @@ describe("SQL query builder", () => {
     }))
     expect(query).toEqual({
       bindings: ["John%", limit],
-      sql: `select * from "${TABLE_NAME}" where "name" like $1 limit $2`
+      sql: `select * from "${TABLE_NAME}" where "${TABLE_NAME}"."name" like $1 limit $2`
     })
   })
 
@@ -98,7 +98,7 @@ describe("SQL query builder", () => {
     }))
     expect(query).toEqual({
       bindings: [2, 10, limit],
-      sql: `select * from "${TABLE_NAME}" where "age" between $1 and $2 limit $3`
+      sql: `select * from "${TABLE_NAME}" where "${TABLE_NAME}"."age" between $1 and $2 limit $3`
     })
   })
 
@@ -114,7 +114,7 @@ describe("SQL query builder", () => {
     }))
     expect(query).toEqual({
       bindings: [10, "John", limit],
-      sql: `select * from "${TABLE_NAME}" where ("age" = $1) or ("name" = $2) limit $3`
+      sql: `select * from "${TABLE_NAME}" where ("${TABLE_NAME}"."age" = $1) or ("${TABLE_NAME}"."name" = $2) limit $3`
     })
   })
 
@@ -139,7 +139,7 @@ describe("SQL query builder", () => {
     }))
     expect(query).toEqual({
       bindings: ["John", 1001],
-      sql: `update "${TABLE_NAME}" set "name" = $1 where "id" = $2 returning *`
+      sql: `update "${TABLE_NAME}" set "name" = $1 where "${TABLE_NAME}"."id" = $2 returning *`
     })
   })
 
@@ -151,7 +151,7 @@ describe("SQL query builder", () => {
     }))
     expect(query).toEqual({
       bindings: [1001],
-      sql: `delete from "${TABLE_NAME}" where "id" = $1 returning *`
+      sql: `delete from "${TABLE_NAME}" where "${TABLE_NAME}"."id" = $1 returning *`
     })
   })
 

packages/server/src/integrations/utils.ts

@@ -1,4 +1,6 @@
-import { SqlQuery } from "./base/definitions"
+import { SqlQuery } from "../definitions/datasource"
+import { Datasource } from "../definitions/common"
+import { SourceNames } from "../definitions/datasource"
 const { DocumentTypes, SEPARATOR } = require("../db/utils")
 const { FieldTypes } = require("../constants")
 
@@ -25,15 +27,21 @@ export function generateRowIdField(keyProps: any[] = []) {
     keyProps = [keyProps]
   }
   // this conserves order and types
+  // we have to swap the double quotes to single quotes for use in HBS statements
+  // when using the literal helper the double quotes can break things
   return encodeURIComponent(JSON.stringify(keyProps).replace(/"/g, "'"))
 }
 
 // should always return an array
-export function breakRowIdField(_id: string) {
+export function breakRowIdField(_id: string): any[] {
   if (!_id) {
-    return null
+    return []
   }
-  return JSON.parse(decodeURIComponent(_id))
+  // have to replace on the way back as we swapped out the double quotes
+  // when encoding, but JSON can't handle the single quotes
+  const decoded: string = decodeURIComponent(_id).replace(/'/g, '"')
+  const parsed = JSON.parse(decoded)
+  return Array.isArray(parsed) ? parsed : [parsed]
 }
 
 export function convertType(type: string, map: { [key: string]: any }) {
@@ -52,3 +60,11 @@ export function getSqlQuery(query: SqlQuery | string): SqlQuery {
     return query
   }
 }
+
+export function isSQL(datasource: Datasource): boolean {
+  if (!datasource || !datasource.source) {
+    return false
+  }
+  const SQL = [SourceNames.POSTGRES, SourceNames.SQL_SERVER, SourceNames.MYSQL]
+  return SQL.indexOf(datasource.source) !== -1
+}

packages/server/tsconfig.json

@@ -8,7 +8,8 @@
     "strict": true,
     "noImplicitAny": true,
     "esModuleInterop": true,
-    "resolveJsonModule": true
+    "resolveJsonModule": true,
+    "incremental": true
   },
   "include": [
     "./src/**/*"

packages/standard-components/package.json

@@ -29,11 +29,11 @@
   "keywords": [
     "svelte"
   ],
-  "version": "0.9.67",
+  "version": "0.9.70",
   "license": "MIT",
   "gitHead": "d1836a898cab3f8ab80ee6d8f42be1a9eed7dcdc",
   "dependencies": {
-    "@budibase/bbui": "^0.9.67",
+    "@budibase/bbui": "^0.9.70",
     "@spectrum-css/link": "^3.1.3",
     "@spectrum-css/page": "^3.0.1",
     "@spectrum-css/vars": "^3.0.1",

packages/standard-components/src/Heading.svelte

@@ -19,13 +19,20 @@
 
   // Add color styles to main styles object, otherwise the styleable helper
   // overrides the color when it's passed as inline style.
-  $: styles = {
+  $: styles = enrichStyles($component.styles, color)
-    ...$component.styles,
+
+  const enrichStyles = (styles, color) => {
+    if (!color) {
+      return styles
+    }
+    return {
+      ...styles,
       normal: {
-      ...$component.styles?.normal,
+        ...styles?.normal,
         color,
       },
     }
+  }
 </script>
 
 <h1

packages/standard-components/src/Link.svelte

@@ -23,13 +23,22 @@
 
   // Add color styles to main styles object, otherwise the styleable helper
   // overrides the color when it's passed as inline style.
-  $: styles = {
+  // Add color styles to main styles object, otherwise the styleable helper
-    ...$component.styles,
+  // overrides the color when it's passed as inline style.
+  $: styles = enrichStyles($component.styles, color)
+
+  const enrichStyles = (styles, color) => {
+    if (!color) {
+      return styles
+    }
+    return {
+      ...styles,
       normal: {
-      ...$component.styles?.normal,
+        ...styles?.normal,
         color,
       },
     }
+  }
 </script>
 
 {#if $builderStore.inBuilder || componentText}

packages/standard-components/src/Text.svelte

@@ -19,13 +19,20 @@
 
   // Add color styles to main styles object, otherwise the styleable helper
   // overrides the color when it's passed as inline style.
-  $: styles = {
+  $: styles = enrichStyles($component.styles, color)
-    ...$component.styles,
+
+  const enrichStyles = (styles, color) => {
+    if (!color) {
+      return styles
+    }
+    return {
+      ...styles,
       normal: {
-      ...$component.styles?.normal,
+        ...styles?.normal,
         color,
       },
     }
+  }
 </script>
 
 <p

packages/string-templates/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/string-templates",
-  "version": "0.9.67",
+  "version": "0.9.70",
   "description": "Handlebars wrapper for Budibase templating.",
   "main": "src/index.cjs",
   "module": "dist/bundle.mjs",

packages/string-templates/src/index.cjs

@@ -1,12 +1,7 @@
 const handlebars = require("handlebars")
 const { registerAll } = require("./helpers/index")
 const processors = require("./processors")
-const { cloneDeep } = require("lodash/fp")
+const { removeHandlebarsStatements } = require("./utilities")
-const {
-  removeNull,
-  updateContext,
-  removeHandlebarsStatements,
-} = require("./utilities")
 const manifest = require("../manifest.json")
 
 const hbsInstance = handlebars.create()

packages/worker/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/worker",
   "email": "hi@budibase.com",
-  "version": "0.9.67",
+  "version": "0.9.70",
   "description": "Budibase background service",
   "main": "src/index.js",
   "repository": {
@@ -21,8 +21,8 @@
   "author": "Budibase",
   "license": "AGPL-3.0-or-later",
   "dependencies": {
-    "@budibase/auth": "^0.9.67",
+    "@budibase/auth": "^0.9.70",
-    "@budibase/string-templates": "^0.9.67",
+    "@budibase/string-templates": "^0.9.70",
     "@koa/router": "^8.0.0",
     "aws-sdk": "^2.811.0",
     "bcryptjs": "^2.4.3",
@@ -39,6 +39,7 @@
     "koa-static": "^5.0.0",
     "node-fetch": "^2.6.1",
     "nodemailer": "^6.5.0",
+    "@techpass/passport-openidconnect": "^0.3.0",
     "passport-google-oauth": "^2.0.0",
     "passport-jwt": "^4.0.0",
     "passport-local": "^1.0.0",

packages/worker/src/api/controllers/admin/auth.js

@@ -1,5 +1,6 @@
 const authPkg = require("@budibase/auth")
 const { google } = require("@budibase/auth/src/middleware")
+const { oidc } = require("@budibase/auth/src/middleware")
 const { Configs, EmailTemplatePurpose } = require("../../../constants")
 const CouchDB = require("../../../db")
 const { sendEmail, isEmailConfigured } = require("../../../utilities/email")
@@ -10,16 +11,16 @@ const { checkResetPasswordCode } = require("../../../utilities/redis")
 
 const GLOBAL_DB = authPkg.StaticDatabases.GLOBAL.name
 
-function authInternal(ctx, user, err = null) {
+function authInternal(ctx, user, err = null, info = null) {
   if (err) {
-    return ctx.throw(403, "Unauthorized")
+    return ctx.throw(403, info? info : "Unauthorized")
   }
 
   const expires = new Date()
   expires.setDate(expires.getDate() + 1)
 
   if (!user) {
-    return ctx.throw(403, "Unauthorized")
+    return ctx.throw(403, info? info : "Unauthorized")
   }
 
   ctx.cookies.set(Cookies.Auth, user.token, {
@@ -129,3 +130,34 @@ exports.googleAuth = async (ctx, next) => {
     }
   )(ctx, next)
 }
+
+async function oidcStrategyFactory(ctx) {
+  const callbackUrl = `${ctx.protocol}://${ctx.host}/api/admin/auth/oidc/callback`
+  return oidc.strategyFactory(callbackUrl)
+}
+
+/**
+ * The initial call that OIDC authentication makes to take you to the configured OIDC login screen.
+ * On a successful login, you will be redirected to the oidcAuth callback route.
+ */
+exports.oidcPreAuth = async (ctx, next) => {
+  const strategy = await oidcStrategyFactory(ctx)
+
+  return passport.authenticate(strategy, {
+    scope: ["profile", "email"],
+  })(ctx, next)
+}
+
+exports.oidcAuth = async (ctx, next) => {
+  const strategy = await oidcStrategyFactory(ctx)
+
+  return passport.authenticate(
+    strategy,
+    { successRedirect: "/", failureRedirect: "/error" },
+    async (err, user, info) => {
+      authInternal(ctx, user, err, info)
+
+      ctx.redirect("/")
+    }
+  )(ctx, next)
+}

packages/worker/src/api/controllers/admin/roles.js

@@ -8,7 +8,7 @@ const CouchDB = require("../../../db")
 
 exports.fetch = async ctx => {
   // always use the dev apps as they'll be most up to date (true)
-  const apps = await getAllApps({ CouchDB, dev: true })
+  const apps = await getAllApps({ CouchDB, all: true })
   const promises = []
   for (let app of apps) {
     // use dev app IDs

packages/worker/src/api/index.js

@@ -25,6 +25,14 @@ const PUBLIC_ENDPOINTS = [
     route: "/api/admin/auth/google/callback",
     method: "GET",
   },
+  {
+    route: "/api/admin/auth/oidc",
+    method: "GET",
+  },
+  {
+    route: "/api/admin/auth/oidc/callback",
+    method: "GET",
+  },
   {
     route: "/api/admin/auth/reset",
     method: "POST",

packages/worker/src/api/routes/admin/auth.js

@@ -39,5 +39,7 @@ router
   .post("/api/admin/auth/logout", authController.logout)
   .get("/api/admin/auth/google", authController.googlePreAuth)
   .get("/api/admin/auth/google/callback", authController.googleAuth)
+  .get("/api/admin/auth/oidc", authController.oidcPreAuth)
+  .get("/api/admin/auth/oidc/callback", authController.oidcAuth)
 
 module.exports = router

packages/worker/src/index.js

@@ -5,6 +5,7 @@ require("@budibase/auth").init(CouchDB)
 const Koa = require("koa")
 const destroyable = require("server-destroy")
 const koaBody = require("koa-body")
+const koaSession = require("koa-session")
 const { passport } = require("@budibase/auth").auth
 const logger = require("koa-pino-logger")
 const http = require("http")
@@ -13,8 +14,11 @@ const redis = require("./utilities/redis")
 
 const app = new Koa()
 
+app.keys = ["secret", "key"]
+
 // set up top level koa middleware
 app.use(koaBody({ multipart: true }))
+app.use(koaSession(app))
 
 app.use(
   logger({

packages/worker/yarn.lock

@@ -566,6 +566,17 @@
   dependencies:
     defer-to-connect "^2.0.0"
 
+"@techpass/passport-openidconnect@^0.3.0":
+  version "0.3.0"
+  resolved "https://registry.yarnpkg.com/@techpass/passport-openidconnect/-/passport-openidconnect-0.3.0.tgz#a60b2bbf3f262649a5a02d5d186219944acc3010"
+  integrity sha512-bVsPwl66s7J7GHxTPlW/RJYhZol9SshNznQsx83OOh9G+JWFGoeWxh+xbX+FTdJNoUvGIGbJnpWPY2wC6NOHPw==
+  dependencies:
+    base64url "^3.0.1"
+    oauth "^0.9.15"
+    passport-strategy "^1.0.0"
+    request "^2.88.0"
+    webfinger "^0.4.2"
+
 "@types/babel__core@^7.0.0", "@types/babel__core@^7.1.7":
   version "7.1.14"
   resolved "https://registry.yarnpkg.com/@types/babel__core/-/babel__core-7.1.14.tgz#faaeefc4185ec71c389f4501ee5ec84b170cc402"
@@ -1058,7 +1069,7 @@ base64-js@^1.0.2, base64-js@^1.3.1:
   resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.5.1.tgz#1b1b440160a5bf7ad40b650f095963481903930a"
   integrity sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==
 
-base64url@3.x.x:
+base64url@3.x.x, base64url@^3.0.1:
   version "3.0.1"
   resolved "https://registry.yarnpkg.com/base64url/-/base64url-3.0.1.tgz#6399d572e2bc3f90a9a8b22d5dbb0a32d33f788d"
   integrity sha512-ir1UPr3dkwexU7FdV8qBBbNDRUhMmIekYMFZfi+C/sLNnRESKPl23nB9b2pltqfOQNnGzsDdId90AEtG5tCx4A==
@@ -4183,7 +4194,7 @@ oauth-sign@~0.9.0:
   resolved "https://registry.yarnpkg.com/oauth-sign/-/oauth-sign-0.9.0.tgz#47a7b016baa68b5fa0ecf3dee08a85c679ac6455"
   integrity sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ==
 
-oauth@0.9.x:
+oauth@0.9.x, oauth@^0.9.15:
   version "0.9.15"
   resolved "https://registry.yarnpkg.com/oauth/-/oauth-0.9.15.tgz#bd1fefaf686c96b75475aed5196412ff60cfb9c1"
   integrity sha1-vR/vr2hslrdUda7VGWQS/2DPucE=
@@ -4933,7 +4944,7 @@ request-promise-native@^1.0.9:
     stealthy-require "^1.1.1"
     tough-cookie "^2.3.3"
 
-request@^2.88.2:
+request@^2.88.0, request@^2.88.2:
   version "2.88.2"
   resolved "https://registry.yarnpkg.com/request/-/request-2.88.2.tgz#d73c918731cb5a87da047e207234146f664d12b3"
   integrity sha512-MsvtOrfG9ZcrOwAW+Qi+F6HbD0CWXEh9ou77uOb7FM2WPhwT7smM833PzanhJLsgXjN89Ir6V2PczXNnMpwKhw==
@@ -5080,7 +5091,7 @@ sax@1.2.1:
   resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.1.tgz#7b8e656190b228e81a66aea748480d828cd2d37a"
   integrity sha1-e45lYZCyKOgaZq6nSEgNgozS03o=
 
-sax@>=0.6.0:
+sax@>=0.1.1, sax@>=0.6.0:
   version "1.2.4"
   resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.4.tgz#2816234e2378bddc4e5354fab5caa895df7100d9"
   integrity sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw==
@@ -5390,6 +5401,11 @@ stealthy-require@^1.1.1:
   resolved "https://registry.yarnpkg.com/stealthy-require/-/stealthy-require-1.1.1.tgz#35b09875b4ff49f26a777e509b3090a3226bf24b"
   integrity sha1-NbCYdbT/SfJqd35QmzCQoyJr8ks=
 
+step@0.0.x:
+  version "0.0.6"
+  resolved "https://registry.yarnpkg.com/step/-/step-0.0.6.tgz#143e7849a5d7d3f4a088fe29af94915216eeede2"
+  integrity sha1-FD54SaXX0/SgiP4pr5SRUhbu7eI=
+
 string-length@^4.0.1:
   version "4.0.2"
   resolved "https://registry.yarnpkg.com/string-length/-/string-length-4.0.2.tgz#a8a8dc7bd5c1a82b9b3c8b87e125f66871b6e57a"
@@ -5923,6 +5939,14 @@ walker@^1.0.7, walker@~1.0.5:
   dependencies:
     makeerror "1.0.x"
 
+webfinger@^0.4.2:
+  version "0.4.2"
+  resolved "https://registry.yarnpkg.com/webfinger/-/webfinger-0.4.2.tgz#3477a6d97799461896039fcffc650b73468ee76d"
+  integrity sha1-NHem2XeZRhiWA5/P/GULc0aO520=
+  dependencies:
+    step "0.0.x"
+    xml2js "0.1.x"
+
 webidl-conversions@^5.0.0:
   version "5.0.0"
   resolved "https://registry.yarnpkg.com/webidl-conversions/-/webidl-conversions-5.0.0.tgz#ae59c8a00b121543a2acc65c0434f57b0fc11aff"
@@ -6031,6 +6055,13 @@ xml-name-validator@^3.0.0:
   resolved "https://registry.yarnpkg.com/xml-name-validator/-/xml-name-validator-3.0.0.tgz#6ae73e06de4d8c6e47f9fb181f78d648ad457c6a"
   integrity sha512-A5CUptxDsvxKJEU3yO6DuWBSJz/qizqzJKOMIfUJHETbBw/sFaDxgd6fxm1ewUaM0jZ444Fc5vC5ROYurg/4Pw==
 
+xml2js@0.1.x:
+  version "0.1.14"
+  resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.1.14.tgz#5274e67f5a64c5f92974cd85139e0332adc6b90c"
+  integrity sha1-UnTmf1pkxfkpdM2FE54DMq3GuQw=
+  dependencies:
+    sax ">=0.1.1"
+
 xml2js@0.4.19:
   version "0.4.19"
   resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.4.19.tgz#686c20f213209e94abf0d1bcf1efaa291c7827a7"