Merge pull request #4732 from Budibase/feature/budibase-api

Budibase public API

.github/workflows/release-selfhost.yml

@@ -47,6 +47,13 @@ jobs:
           yarn
           yarn build
           popd
+
+      - name: Build OpenAPI sepc
+        run: |
+          pushd packages/server
+          yarn
+          yarn specs
+          popd
       
       - name: Setup Helm 
         uses: azure/setup-helm@v1
@@ -77,3 +84,5 @@ jobs:
             packages/cli/build/cli-win.exe
             packages/cli/build/cli-linux
             packages/cli/build/cli-macos 
+            packages/server/specs/openapi.yaml
+            packages/server/specs/openapi.json

.gitignore

@@ -96,4 +96,5 @@ hosting/proxy/.generated-nginx.prod.conf
 *.sublime-workspace
 
 bin/
+hosting/.generated*
 packages/builder/cypress.env.json

.prettierignore

@@ -1,5 +1,4 @@
 node_modules
-public
 dist
 *.spec.js
 packages/builder/src/components/design/AppPreview/CurrentItemPreview.svelte
@@ -8,4 +7,4 @@ packages/server/coverage
 packages/server/client
 packages/builder/.routify
 packages/builder/cypress/support/queryLevelTransformerFunction.js
-packages/builder/cypress/support/queryLevelTransformerFunctionWithData.js
+packages/builder/cypress/support/queryLevelTransformerFunctionWithData.js

hosting/nginx.dev.conf.hbs

@@ -76,6 +76,7 @@ http {
       proxy_set_header X-Real-IP $remote_addr;
       proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
       proxy_set_header X-Forwarded-Proto $scheme;
+      proxy_set_header Host $http_host;
 
       proxy_connect_timeout 300;
       proxy_http_version 1.1;
@@ -91,4 +92,4 @@ http {
     gzip                  off;
     gzip_comp_level       4;
   }
-}
+}

package.json

@@ -44,6 +44,7 @@
     "lint:fix": "yarn run lint:fix:ts && yarn run lint:fix:prettier && yarn run lint:fix:eslint",
     "test:e2e": "lerna run cy:test --stream",
     "test:e2e:ci": "lerna run cy:ci --stream",
+    "build:specs": "lerna run specs",
     "build:docker": "lerna run build:docker && npm run build:docker:proxy:compose && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh $BUDIBASE_RELEASE_VERSION && cd -",
     "build:docker:proxy": "docker build hosting/proxy -t proxy-service",
     "build:docker:proxy:compose": "lerna run generate:proxy:compose && npm run build:docker:proxy",

packages/backend-core/encryption.js

@@ -0,0 +1 @@
+module.exports = require("./src/security/encryption")

packages/backend-core/src/cache/user.js

@@ -32,11 +32,10 @@ const populateFromDB = async (userId, tenantId) => {
  * @param {*} populateUser function to provide the user for re-caching. default to couch db
  * @returns
  */
-exports.getUser = async (
+exports.getUser = async (userId, tenantId = null, populateUser = null) => {
-  userId,
+  if (!populateUser) {
-  tenantId = null,
+    populateUser = populateFromDB
-  populateUser = populateFromDB
+  }
-) => {
   if (!tenantId) {
     try {
       tenantId = getTenantId()

packages/backend-core/src/db/constants.js

@@ -14,6 +14,7 @@ exports.DocumentTypes = {
   APP_METADATA: `${PRE_APP}${exports.SEPARATOR}metadata`,
   ROLE: "role",
   MIGRATIONS: "migrations",
+  DEV_INFO: "devinfo",
 }
 
 exports.StaticDatabases = {

packages/backend-core/src/db/utils.js

@@ -30,6 +30,7 @@ const UNICODE_MAX = "\ufff0"
 
 exports.ViewNames = {
   USER_BY_EMAIL: "by_email",
+  BY_API_KEY: "by_api_key",
 }
 
 exports.StaticDatabases = StaticDatabases
@@ -67,6 +68,7 @@ function getDocParams(docType, docId = null, otherProps = {}) {
     endkey: `${docType}${SEPARATOR}${docId}${UNICODE_MAX}`,
   }
 }
+exports.getDocParams = getDocParams
 
 /**
  * Generates a new workspace ID.
@@ -339,6 +341,14 @@ const getConfigParams = ({ type, workspace, user }, otherProps = {}) => {
   }
 }
 
+/**
+ * Generates a new dev info document ID - this is scoped to a user.
+ * @returns {string} The new dev info ID which info for dev (like api key) can be stored under.
+ */
+const generateDevInfoID = userId => {
+  return `${DocumentTypes.DEV_INFO}${SEPARATOR}${userId}`
+}
+
 /**
  * Returns the most granular configuration document from the DB based on the type, workspace and userID passed.
  * @param {Object} db - db instance to query
@@ -454,3 +464,4 @@ exports.generateConfigID = generateConfigID
 exports.getConfigParams = getConfigParams
 exports.getScopedFullConfig = getScopedFullConfig
 exports.generateNewUsageQuotaDoc = generateNewUsageQuotaDoc
+exports.generateDevInfoID = generateDevInfoID

packages/backend-core/src/db/views.js

@@ -1,4 +1,5 @@
 const { DocumentTypes, ViewNames } = require("./utils")
+const { getGlobalDB } = require("../tenancy")
 
 function DesignDoc() {
   return {
@@ -9,7 +10,8 @@ function DesignDoc() {
   }
 }
 
-exports.createUserEmailView = async db => {
+exports.createUserEmailView = async () => {
+  const db = getGlobalDB()
   let designDoc
   try {
     designDoc = await db.get("_design/database")
@@ -31,3 +33,51 @@ exports.createUserEmailView = async db => {
   }
   await db.put(designDoc)
 }
+
+exports.createApiKeyView = async () => {
+  const db = getGlobalDB()
+  let designDoc
+  try {
+    designDoc = await db.get("_design/database")
+  } catch (err) {
+    designDoc = DesignDoc()
+  }
+  const view = {
+    map: `function(doc) {
+      if (doc._id.startsWith("${DocumentTypes.DEV_INFO}") && doc.apiKey) {
+        emit(doc.apiKey, doc.userId)
+      }
+    }`,
+  }
+  designDoc.views = {
+    ...designDoc.views,
+    [ViewNames.BY_API_KEY]: view,
+  }
+  await db.put(designDoc)
+}
+
+exports.queryGlobalView = async (viewName, params, db = null) => {
+  const CreateFuncByName = {
+    [ViewNames.USER_BY_EMAIL]: exports.createUserEmailView,
+    [ViewNames.BY_API_KEY]: exports.createApiKeyView,
+  }
+  // can pass DB in if working with something specific
+  if (!db) {
+    db = getGlobalDB()
+  }
+  try {
+    let response = (await db.query(`database/${viewName}`, params)).rows
+    response = response.map(resp =>
+      params.include_docs ? resp.doc : resp.value
+    )
+    return response.length <= 1 ? response[0] : response
+  } catch (err) {
+    if (err != null && err.name === "not_found") {
+      const createFunc = CreateFuncByName[viewName]
+      await createFunc()
+      return exports.queryGlobalView(viewName, params)
+    } else {
+      throw err
+    }
+  }
+}

packages/backend-core/src/middleware/authenticated.js

@@ -4,6 +4,9 @@ const { getUser } = require("../cache/user")
 const { getSession, updateSessionTTL } = require("../security/sessions")
 const { buildMatcherRegex, matches } = require("./matchers")
 const env = require("../environment")
+const { SEPARATOR, ViewNames, queryGlobalView } = require("../../db")
+const { getGlobalDB } = require("../tenancy")
+const { decrypt } = require("../security/encryption")
 
 function finalise(
   ctx,
@@ -16,6 +19,28 @@ function finalise(
   ctx.version = version
 }
 
+async function checkApiKey(apiKey, populateUser) {
+  if (apiKey === env.INTERNAL_API_KEY) {
+    return { valid: true }
+  }
+  const decrypted = decrypt(apiKey)
+  const tenantId = decrypted.split(SEPARATOR)[0]
+  const db = getGlobalDB(tenantId)
+  // api key is encrypted in the database
+  const userId = await queryGlobalView(
+    ViewNames.BY_API_KEY,
+    {
+      key: apiKey,
+    },
+    db
+  )
+  if (userId) {
+    return { valid: true, user: await getUser(userId, tenantId, populateUser) }
+  } else {
+    throw "Invalid API key"
+  }
+}
+
 /**
  * This middleware is tenancy aware, so that it does not depend on other middlewares being used.
  * The tenancy modules should not be used here and it should be assumed that the tenancy context
@@ -79,9 +104,19 @@ module.exports = (
       const apiKey = ctx.request.headers[Headers.API_KEY]
       const tenantId = ctx.request.headers[Headers.TENANT_ID]
       // this is an internal request, no user made it
-      if (!authenticated && apiKey && apiKey === env.INTERNAL_API_KEY) {
+      if (!authenticated && apiKey) {
-        authenticated = true
+        const populateUser = opts.populateUser ? opts.populateUser(ctx) : null
-        internal = true
+        const { valid, user: foundUser } = await checkApiKey(
+          apiKey,
+          populateUser
+        )
+        if (valid && foundUser) {
+          authenticated = true
+          user = foundUser
+        } else if (valid) {
+          authenticated = true
+          internal = true
+        }
       }
       if (!user && tenantId) {
         user = { tenantId }
@@ -101,6 +136,7 @@ module.exports = (
       // allow configuring for public access
       if ((opts && opts.publicAllowed) || publicEndpoint) {
         finalise(ctx, { authenticated: false, version, publicEndpoint })
+        return next()
       } else {
         ctx.throw(err.status || 403, err)
       }

packages/backend-core/src/objectStore/index.js

@@ -78,6 +78,7 @@ exports.ObjectStore = bucket => {
   const config = {
     s3ForcePathStyle: true,
     signatureVersion: "v4",
+    apiVersion: "2006-03-01",
     params: {
       Bucket: sanitizeBucket(bucket),
     },
@@ -102,17 +103,21 @@ exports.makeSureBucketExists = async (client, bucketName) => {
       .promise()
   } catch (err) {
     const promises = STATE.bucketCreationPromises
+    const doesntExist = err.statusCode === 404,
+      noAccess = err.statusCode === 403
     if (promises[bucketName]) {
       await promises[bucketName]
-    } else if (err.statusCode === 404) {
+    } else if (doesntExist || noAccess) {
-      // bucket doesn't exist create it
+      if (doesntExist) {
-      promises[bucketName] = client
+        // bucket doesn't exist create it
-        .createBucket({
+        promises[bucketName] = client
-          Bucket: bucketName,
+          .createBucket({
-        })
+            Bucket: bucketName,
-        .promise()
+          })
-      await promises[bucketName]
+          .promise()
-      delete promises[bucketName]
+        await promises[bucketName]
+        delete promises[bucketName]
+      }
       // public buckets are quite hidden in the system, make sure
       // no bucket is set accidentally
       if (PUBLIC_BUCKETS.includes(bucketName)) {
@@ -124,7 +129,7 @@ exports.makeSureBucketExists = async (client, bucketName) => {
           .promise()
       }
     } else {
-      throw err
+      throw new Error("Unable to write to object store bucket.")
     }
   }
 }

packages/backend-core/src/security/apiKeys.js

@@ -0,0 +1 @@
+exports.lookupApiKey = async () => {}

packages/backend-core/src/security/encryption.js

@@ -0,0 +1,33 @@
+const crypto = require("crypto")
+const env = require("../environment")
+
+const ALGO = "aes-256-ctr"
+const SECRET = env.JWT_SECRET
+const SEPARATOR = "-"
+const ITERATIONS = 10000
+const RANDOM_BYTES = 16
+const STRETCH_LENGTH = 32
+
+function stretchString(string, salt) {
+  return crypto.pbkdf2Sync(string, salt, ITERATIONS, STRETCH_LENGTH, "sha512")
+}
+
+exports.encrypt = input => {
+  const salt = crypto.randomBytes(RANDOM_BYTES)
+  const stretched = stretchString(SECRET, salt)
+  const cipher = crypto.createCipheriv(ALGO, stretched, salt)
+  const base = cipher.update(input)
+  const final = cipher.final()
+  const encrypted = Buffer.concat([base, final]).toString("hex")
+  return `${salt.toString("hex")}${SEPARATOR}${encrypted}`
+}
+
+exports.decrypt = input => {
+  const [salt, encrypted] = input.split(SEPARATOR)
+  const saltBuffer = Buffer.from(salt, "hex")
+  const stretched = stretchString(SECRET, saltBuffer)
+  const decipher = crypto.createDecipheriv(ALGO, stretched, saltBuffer)
+  const base = decipher.update(Buffer.from(encrypted, "hex"))
+  const final = decipher.final()
+  return Buffer.concat([base, final]).toString()
+}

packages/backend-core/src/security/permissions.js

@@ -10,6 +10,7 @@ const PermissionLevels = {
 
 // these are the global types, that govern the underlying default behaviour
 const PermissionTypes = {
+  APP: "app",
   TABLE: "table",
   USER: "user",
   AUTOMATION: "automation",

packages/backend-core/src/utils.js

@@ -6,7 +6,7 @@ const {
 } = require("./db/utils")
 const jwt = require("jsonwebtoken")
 const { options } = require("./middleware/passport/jwt")
-const { createUserEmailView } = require("./db/views")
+const { queryGlobalView } = require("./db/views")
 const { Headers, UserStatus, Cookies, MAX_VALID_DATE } = require("./constants")
 const {
   getGlobalDB,
@@ -139,25 +139,11 @@ exports.getGlobalUserByEmail = async email => {
   if (email == null) {
     throw "Must supply an email address to view"
   }
-  const db = getGlobalDB()
 
-  try {
+  return queryGlobalView(ViewNames.USER_BY_EMAIL, {
-    let users = (
+    key: email.toLowerCase(),
-      await db.query(`database/${ViewNames.USER_BY_EMAIL}`, {
+    include_docs: true,
-        key: email.toLowerCase(),
+  })
-        include_docs: true,
-      })
-    ).rows
-    users = users.map(user => user.doc)
-    return users.length <= 1 ? users[0] : users
-  } catch (err) {
-    if (err != null && err.name === "not_found") {
-      await createUserEmailView(db)
-      return exports.getGlobalUserByEmail(email)
-    } else {
-      throw err
-    }
-  }
 }
 
 exports.saveUser = async (

packages/bbui/src/Modal/ModalContent.svelte

@@ -165,4 +165,8 @@
   .secondary-action {
     margin-right: auto;
   }
+
+  .spectrum-Dialog-buttonGroup {
+    padding-left: 0;
+  }
 </style>

packages/builder/src/components/automation/Shared/WebhookDisplay.svelte

@@ -1,5 +1,5 @@
 <script>
-  import { Input, Icon, notifications } from "@budibase/bbui"
+  import CopyInput from "components/common/inputs/CopyInput.svelte"
 
   export let value
 
@@ -10,55 +10,6 @@
 
     return `${window.location.origin}/${uri}`
   }
-
-  function copyToClipboard() {
-    const dummy = document.createElement("textarea")
-    document.body.appendChild(dummy)
-    dummy.value = fullWebhookURL(value)
-    dummy.select()
-    document.execCommand("copy")
-    document.body.removeChild(dummy)
-    notifications.success(`URL copied to clipboard`)
-  }
 </script>
 
-<div>
+<CopyInput {value} copyValue={fullWebhookURL(value)} />
-  <Input readonly value={fullWebhookURL(value)} />
-  <div class="icon" on:click={() => copyToClipboard()}>
-    <Icon size="S" name="Copy" />
-  </div>
-</div>
-
-<style>
-  div {
-    position: relative;
-  }
-
-  .icon {
-    right: 1px;
-    bottom: 1px;
-    position: absolute;
-    justify-content: center;
-    align-items: center;
-    display: flex;
-    flex-direction: row;
-    box-sizing: border-box;
-    border-left: 1px solid var(--spectrum-alias-border-color);
-    border-top-right-radius: var(--spectrum-alias-border-radius-regular);
-    border-bottom-right-radius: var(--spectrum-alias-border-radius-regular);
-    width: 31px;
-    color: var(--spectrum-alias-text-color);
-    background-color: var(--spectrum-global-color-gray-75);
-    transition: background-color
-        var(--spectrum-global-animation-duration-100, 130ms),
-      box-shadow var(--spectrum-global-animation-duration-100, 130ms),
-      border-color var(--spectrum-global-animation-duration-100, 130ms);
-    height: calc(var(--spectrum-alias-item-height-m) - 2px);
-  }
-  .icon:hover {
-    cursor: pointer;
-    color: var(--spectrum-alias-text-color-hover);
-    background-color: var(--spectrum-global-color-gray-50);
-    border-color: var(--spectrum-alias-border-color-hover);
-  }
-</style>

packages/builder/src/components/common/inputs/CopyInput.svelte

@@ -0,0 +1,58 @@
+<script>
+  import { Input, Icon, notifications } from "@budibase/bbui"
+
+  export let label = null
+  export let value
+  export let copyValue
+
+  const copyToClipboard = val => {
+    const dummy = document.createElement("textarea")
+    document.body.appendChild(dummy)
+    dummy.value = val
+    dummy.select()
+    document.execCommand("copy")
+    document.body.removeChild(dummy)
+    notifications.success(`URL copied to clipboard`)
+  }
+</script>
+
+<div>
+  <Input readonly {value} {label} />
+  <div class="icon" on:click={() => copyToClipboard(value || copyValue)}>
+    <Icon size="S" name="Copy" />
+  </div>
+</div>
+
+<style>
+  div {
+    position: relative;
+  }
+
+  .icon {
+    right: 1px;
+    bottom: 1px;
+    position: absolute;
+    justify-content: center;
+    align-items: center;
+    display: flex;
+    flex-direction: row;
+    box-sizing: border-box;
+    border-left: 1px solid var(--spectrum-alias-border-color);
+    border-top-right-radius: var(--spectrum-alias-border-radius-regular);
+    border-bottom-right-radius: var(--spectrum-alias-border-radius-regular);
+    width: 31px;
+    color: var(--spectrum-alias-text-color);
+    background-color: var(--spectrum-global-color-gray-75);
+    transition: background-color
+        var(--spectrum-global-animation-duration-100, 130ms),
+      box-shadow var(--spectrum-global-animation-duration-100, 130ms),
+      border-color var(--spectrum-global-animation-duration-100, 130ms);
+    height: calc(var(--spectrum-alias-item-height-m) - 2px);
+  }
+  .icon:hover {
+    cursor: pointer;
+    color: var(--spectrum-alias-text-color-hover);
+    background-color: var(--spectrum-global-color-gray-50);
+    border-color: var(--spectrum-alias-border-color-hover);
+  }
+</style>

packages/builder/src/components/settings/UpdateAPIKeyModal.svelte

@@ -0,0 +1,41 @@
+<script>
+  import { ModalContent, Body, notifications } from "@budibase/bbui"
+  import { auth } from "stores/portal"
+  import { onMount } from "svelte"
+  import CopyInput from "components/common/inputs/CopyInput.svelte"
+
+  let apiKey = null
+
+  async function generateAPIKey() {
+    try {
+      apiKey = await auth.generateAPIKey()
+      notifications.success("New API key generated")
+    } catch (err) {
+      notifications.error("Unable to generate new API key")
+    }
+    // need to return false to keep modal open
+    return false
+  }
+
+  onMount(async () => {
+    try {
+      apiKey = await auth.fetchAPIKey()
+    } catch (err) {
+      notifications.error("Unable to fetch API key")
+    }
+  })
+</script>
+
+<ModalContent
+  title="Developer information"
+  showConfirmButton={false}
+  showSecondaryButton={true}
+  secondaryButtonText="Re-generate key"
+  secondaryAction={generateAPIKey}
+>
+  <Body size="S">
+    You can find information about your developer account here, such as the API
+    key used to access the Budibase API.
+  </Body>
+  <CopyInput bind:value={apiKey} label="API key" />
+</ModalContent>

packages/builder/src/pages/builder/portal/_layout.svelte

@@ -18,11 +18,13 @@
   import { onMount } from "svelte"
   import UpdateUserInfoModal from "components/settings/UpdateUserInfoModal.svelte"
   import ChangePasswordModal from "components/settings/ChangePasswordModal.svelte"
+  import UpdateAPIKeyModal from "components/settings/UpdateAPIKeyModal.svelte"
   import Logo from "assets/bb-emblem.svg"
 
   let loaded = false
   let userInfoModal
   let changePasswordModal
+  let apiKeyModal
   let mobileMenuVisible = false
 
   $: menu = buildMenu($auth.isAdmin)
@@ -162,6 +164,11 @@
             <MenuItem icon="UserEdit" on:click={() => userInfoModal.show()}>
               Update user information
             </MenuItem>
+            {#if $auth.isBuilder}
+              <MenuItem icon="Key" on:click={() => apiKeyModal.show()}>
+                View API key
+              </MenuItem>
+            {/if}
             <MenuItem
               icon="LockClosed"
               on:click={() => changePasswordModal.show()}
@@ -186,6 +193,9 @@
   <Modal bind:this={changePasswordModal}>
     <ChangePasswordModal />
   </Modal>
+  <Modal bind:this={apiKeyModal}>
+    <UpdateAPIKeyModal />
+  </Modal>
 {/if}
 
 <style>

packages/builder/src/stores/portal/auth.js

@@ -172,6 +172,13 @@ export function createAuthStore() {
         resetCode,
       })
     },
+    generateAPIKey: async () => {
+      return API.generateAPIKey()
+    },
+    fetchAPIKey: async () => {
+      const info = await API.fetchDeveloperInfo()
+      return info?.apiKey
+    },
   }
 
   return {

packages/cli/.gitignore

@@ -3,3 +3,4 @@ docker-compose.yaml
 nginx.conf
 build/
 docker-error.log
+envoy.yaml

packages/client/yarn.lock

@@ -84,28 +84,28 @@
   integrity sha512-+oKLUI2a0QmQP9EzySeq/G4FpUkkdaDNbuEbqCj2IkPMc/2v/nwzsPhh1fj2UIghGAiiUwXfPpzax1e8fyhQUg==
 
 "@spectrum-css/divider@^1.0.3":
-  version "1.0.9"
+  version "1.0.17"
-  resolved "https://registry.yarnpkg.com/@spectrum-css/divider/-/divider-1.0.9.tgz#00246bd453981c4696149d26f5bcfeefd29b4b53"
+  resolved "https://registry.yarnpkg.com/@spectrum-css/divider/-/divider-1.0.17.tgz#cae86fdcb5eb6dae95798ae19ec962e5735fc27f"
-  integrity sha512-kmSMSXbm56FR0/OAGwT6tlsHuy1OpOve2DBggjND+AVWk6i3TpoTjvbVppy/f8fuLfbMDS5D3MPD27wTEj8wDA==
+  integrity sha512-wuijKLQ+hwZC/aN8x7fVY18KPoFrxnhohheoDB99OTH44nt5+jEggzcPtwMHiUyb5iWMZx045OrG2DdJHiGl1A==
   dependencies:
-    "@spectrum-css/vars" "^4.3.0"
+    "@spectrum-css/vars" "^7.0.0"
 
 "@spectrum-css/link@^3.1.3":
-  version "3.1.9"
+  version "3.1.15"
-  resolved "https://registry.yarnpkg.com/@spectrum-css/link/-/link-3.1.9.tgz#fe40db561c98bf2987489541ef39dcc71416908f"
+  resolved "https://registry.yarnpkg.com/@spectrum-css/link/-/link-3.1.15.tgz#08bcb78e3fe3e816968ba96399597f54e2a7d62d"
-  integrity sha512-/DpmLIbQGDBNZl+Fnf5VDQ34uC6E6Bz393CAYkzYFyadtvzVEy+PGCgUkT3Tgrwu833IW9fZOh7rkKjw1o/Zng==
+  integrity sha512-LKyI/zr8HXY/PGHCyQxT1Uv1zUzvg7Kuy8E1Itzp+yeCs82hg91aUYkXdQzeWm2eXB/w9cBfjr4NoCl9RWb5bQ==
 
 "@spectrum-css/page@^3.0.1":
-  version "3.0.8"
+  version "3.0.9"
-  resolved "https://registry.yarnpkg.com/@spectrum-css/page/-/page-3.0.8.tgz#001efa9e4c10095df9b2b37cf7d7d6eb60140190"
+  resolved "https://registry.yarnpkg.com/@spectrum-css/page/-/page-3.0.9.tgz#f8a705dee90af958e2ee20307218e4f82a018c36"
-  integrity sha512-naEGOyDv9zeK05oa8mZKdwenPILmHG9OTLyKcE8RwuYQDvb0EHcMGC54DOKtGJ5SMNMGCMdC4RwmYUYYKAhkNA==
+  integrity sha512-zxbzJHDHgbc6fq6DpgWPtQa73kCl/3bukMOV2l784jyEWfXx62nuhFYxFVUq8olvyHw2MNZEXF//P7y+W5axVw==
   dependencies:
-    "@spectrum-css/vars" "^4.3.0"
+    "@spectrum-css/vars" "^4.3.1"
 
 "@spectrum-css/tag@^3.1.4":
-  version "3.1.4"
+  version "3.3.3"
-  resolved "https://registry.yarnpkg.com/@spectrum-css/tag/-/tag-3.1.4.tgz#334384dd789ddf0562679cae62ef763883480ac5"
+  resolved "https://registry.yarnpkg.com/@spectrum-css/tag/-/tag-3.3.3.tgz#826bf03525d10f1ae034681095337973bd43f4af"
-  integrity sha512-9dYBMhCEkjy+p75XJIfCA2/zU4JAqsJrL7fkYIDXakS6/BzeVtIvAW/6JaIHtLIA9lrj0Sn4m+ZjceKnZNIv1w==
+  integrity sha512-sWcopo4Pgl5VMOF0TuP6on3KmnrcGcaYfBt1/LDAin8+pUoqv2NgLv5BkO7maaPsd9pCLU4K9Y8NPXbujDOefQ==
 
 "@spectrum-css/typography@^3.0.2":
   version "3.0.2"
@@ -117,10 +117,15 @@
   resolved "https://registry.yarnpkg.com/@spectrum-css/vars/-/vars-3.0.2.tgz#ea9062c3c98dfc6ba59e5df14a03025ad8969999"
   integrity sha512-vzS9KqYXot4J3AEER/u618MXWAS+IoMvYMNrOoscKiLLKYQWenaueakUWulFonToPd/9vIpqtdbwxznqrK5qDw==
 
-"@spectrum-css/vars@^4.3.0":
+"@spectrum-css/vars@^4.3.1":
-  version "4.3.0"
+  version "4.3.1"
-  resolved "https://registry.yarnpkg.com/@spectrum-css/vars/-/vars-4.3.0.tgz#03ddf67d3aa8a9a4cb0edbbd259465c9ced7e70d"
+  resolved "https://registry.yarnpkg.com/@spectrum-css/vars/-/vars-4.3.1.tgz#d333fa41909f691c8750b5c15ad9ba029df2248e"
-  integrity sha512-ZQ2XAhgu4G9yBeXQNDAz07Z8oZNnMt5o9vzf/mpBA7Teb/JI+8qXp2wt8D245SzmtNlFkG/bzRYvQc0scgZeCQ==
+  integrity sha512-rX6Iasu9BsFMVgEN0vGRPm9dmSxva+IK/uqQAa9HM0lliwqUiFrJxrFXHHpiAgNuux/U4srEJwbSpGzfF+CegQ==
+
+"@spectrum-css/vars@^7.0.0":
+  version "7.0.0"
+  resolved "https://registry.yarnpkg.com/@spectrum-css/vars/-/vars-7.0.0.tgz#61a10028f3eeb69d8e488cfeb6251ec75107076d"
+  integrity sha512-0c7i7B/OPrLjrSIQVAxhnDDmDs9tA46gMuuiiaqpRNqEjfS1pRv5eY06qxNMXO5TWL75bHxaiiTQy8NdGA/v6A==
 
 "@trysound/sax@0.2.0":
   version "0.2.0"
@@ -169,9 +174,9 @@ ansi-styles@^4.0.0, ansi-styles@^4.1.0:
     color-convert "^2.0.1"
 
 apexcharts@^3.19.2, apexcharts@^3.22.1:
-  version "3.30.0"
+  version "3.33.1"
-  resolved "https://registry.yarnpkg.com/apexcharts/-/apexcharts-3.30.0.tgz#09b008d0a58bb303904bed33b09b260e8fa5e283"
+  resolved "https://registry.yarnpkg.com/apexcharts/-/apexcharts-3.33.1.tgz#7159f45e7d726a548e5135a327c03e7894d0bf13"
-  integrity sha512-NHhFjkd4sqoQqHi+ECN/duVCRvqVZMdXX/UBzCs1xriq8NbNLvs+nIM8OXH1Siv+W50FrK1uTDZrW2cLsKWhBQ==
+  integrity sha512-5aVzrgJefd8EH4w7oRmuOhA3+cxJxQg27cYg3ANVGvPCOB4AY3mVVNtFHRFaIq7bv8ws4GRaA9MWfzoWQw3MPQ==
   dependencies:
     svg.draggable.js "^2.2.2"
     svg.easing.js "^2.0.0"
@@ -1355,9 +1360,9 @@ svelte-apexcharts@^1.0.2:
     apexcharts "^3.19.2"
 
 svelte-flatpickr@^3.1.0:
-  version "3.2.4"
+  version "3.2.6"
-  resolved "https://registry.yarnpkg.com/svelte-flatpickr/-/svelte-flatpickr-3.2.4.tgz#1824e26a5dc151d14906cfc7dfd100aefd1b072d"
+  resolved "https://registry.yarnpkg.com/svelte-flatpickr/-/svelte-flatpickr-3.2.6.tgz#595a97b2f25a669e61fe743f90a10dce783bbd49"
-  integrity sha512-EE2wbFfpZ3iCBOXRRW52w436Jv5lqFoJkd/1vB8XmkfASJgF9HrrZ6Er11NWSmmpaV1nPywwDYFXdWHCB+Wi9Q==
+  integrity sha512-0ePUyE9OjInYFqQwRKOxnFSu4dQX9+/rzFMynq2fKYXx406ZUThzSx72gebtjr0DoAQbsH2///BBZa5qk4qZXg==
   dependencies:
     flatpickr "^4.5.2"
 
@@ -1369,9 +1374,9 @@ svelte-spa-router@^3.0.5:
     regexparam "2.0.0"
 
 svelte@^3.38.2:
-  version "3.44.1"
+  version "3.46.4"
-  resolved "https://registry.yarnpkg.com/svelte/-/svelte-3.44.1.tgz#5cc772a8340f4519a4ecd1ac1a842325466b1a63"
+  resolved "https://registry.yarnpkg.com/svelte/-/svelte-3.46.4.tgz#0c46bc4a3e20a2617a1b7dc43a722f9d6c084a38"
-  integrity sha512-4DrCEJoBvdR689efHNSxIQn2pnFwB7E7j2yLEJtHE/P8hxwZWIphCtJ8are7bjl/iVMlcEf5uh5pJ68IwR09vQ==
+  integrity sha512-qKJzw6DpA33CIa+C/rGp4AUdSfii0DOTCzj/2YpSKKayw5WGSS624Et9L1nU1k2OVRS9vaENQXp2CVZNU+xvIg==
 
 svg.draggable.js@^2.2.2:
   version "2.2.2"

packages/frontend-core/src/api/index.js

@@ -20,6 +20,7 @@ import { buildScreenEndpoints } from "./screens"
 import { buildTableEndpoints } from "./tables"
 import { buildTemplateEndpoints } from "./templates"
 import { buildUserEndpoints } from "./user"
+import { buildSelfEndpoints } from "./self"
 import { buildViewEndpoints } from "./views"
 
 const defaultAPIClientConfig = {
@@ -231,5 +232,6 @@ export const createAPIClient = config => {
     ...buildTemplateEndpoints(API),
     ...buildUserEndpoints(API),
     ...buildViewEndpoints(API),
+    ...buildSelfEndpoints(API),
   }
 }

packages/frontend-core/src/api/self.js

@@ -0,0 +1,54 @@
+export const buildSelfEndpoints = API => ({
+  /**
+   * Using the logged in user, this will generate a new API key,
+   * assuming the user is a builder.
+   * @return {Promise<object>} returns the API response, including an API key.
+   */
+  generateAPIKey: async () => {
+    const response = await API.post({
+      url: "/api/global/self/api_key",
+    })
+    return response?.apiKey
+  },
+
+  /**
+   * retrieves the API key for the logged in user.
+   * @return {Promise<object>} An object containing the user developer information.
+   */
+  fetchDeveloperInfo: async () => {
+    return API.get({
+      url: "/api/global/self/api_key",
+    })
+  },
+
+  /**
+   * Fetches the currently logged-in user object.
+   * Used in client apps.
+   */
+  fetchSelf: async () => {
+    return await API.get({
+      url: "/api/self",
+    })
+  },
+
+  /**
+   * Fetches the currently logged-in user object.
+   * Used in the builder.
+   */
+  fetchBuilderSelf: async () => {
+    return await API.get({
+      url: "/api/global/self",
+    })
+  },
+
+  /**
+   * Updates the current logged-in user.
+   * @param user the new user object to save
+   */
+  updateSelf: async user => {
+    return await API.post({
+      url: "/api/global/self",
+      body: user,
+    })
+  },
+})

packages/frontend-core/src/api/user.js

@@ -1,24 +1,4 @@
 export const buildUserEndpoints = API => ({
-  /**
-   * Fetches the currently logged-in user object.
-   * Used in client apps.
-   */
-  fetchSelf: async () => {
-    return await API.get({
-      url: "/api/self",
-    })
-  },
-
-  /**
-   * Fetches the currently logged-in user object.
-   * Used in the builder.
-   */
-  fetchBuilderSelf: async () => {
-    return await API.get({
-      url: "/api/global/users/self",
-    })
-  },
-
   /**
    * Gets a list of users in the current tenant.
    */
@@ -61,17 +41,6 @@ export const buildUserEndpoints = API => ({
     })
   },
 
-  /**
-   * Updates the current logged-in user.
-   * @param user the new user object to save
-   */
-  updateSelf: async user => {
-    return await API.post({
-      url: "/api/global/users/self",
-      body: user,
-    })
-  },
-
   /**
    * Creates or updates a user in the current tenant.
    * @param user the new user to create

packages/server/.gitignore

@@ -3,8 +3,7 @@ myapps/
 .env
 builder/*
 client/*
-public/
 db/dev.db/
 dist
 coverage/
-watchtower-hook.json
+watchtower-hook.json

packages/server/__mocks__/node-fetch.ts

@@ -53,6 +53,7 @@ module FetchMock {
           {
             doc: {
               _id: "test",
+              tableId: opts.body.split("tableId:")[1].split('"')[0],
             },
           },
         ],

packages/server/package.json

@@ -25,6 +25,7 @@
     "generate:proxy:preprod": "node scripts/proxy/generateProxyConfig preprod",
     "generate:proxy:prod": "node scripts/proxy/generateProxyConfig prod",
     "format": "prettier --config ../../.prettierrc.json 'src/**/*.ts' --write",
+    "specs": "node specs/generate.js && openapi-typescript specs/openapi.yaml --output src/definitions/openapi.ts",
     "lint": "eslint --fix src/",
     "lint:fix": "yarn run format && yarn run lint",
     "initialise": "node scripts/initialise.js",
@@ -157,12 +158,15 @@
     "docker-compose": "^0.23.6",
     "eslint": "^6.8.0",
     "jest": "^27.0.5",
+    "jest-openapi": "^0.14.2",
     "nodemon": "^2.0.4",
     "openapi-types": "^9.3.1",
+    "openapi-typescript": "^5.2.0",
     "path-to-regexp": "^6.2.0",
     "prettier": "^2.3.1",
     "rimraf": "^3.0.2",
     "supertest": "^4.0.2",
+    "swagger-jsdoc": "^6.1.0",
     "ts-jest": "^27.0.3",
     "ts-node": "^10.0.0",
     "typescript": "^4.3.5",

packages/server/scripts/integrations/postgres/init.sql

@@ -1,11 +1,13 @@
 SELECT 'CREATE DATABASE main'
 WHERE NOT EXISTS (SELECT FROM pg_database WHERE datname = 'main')\gexec
+CREATE TYPE person_job AS ENUM ('qa', 'programmer', 'designer');
 CREATE TABLE Persons (
     PersonID SERIAL PRIMARY KEY,
     LastName varchar(255),
     FirstName varchar(255),
     Address varchar(255),
-    City varchar(255) DEFAULT 'Belfast'
+    City varchar(255) DEFAULT 'Belfast',
+    Type person_job
 );
 CREATE TABLE Tasks (
     TaskID SERIAL PRIMARY KEY,
@@ -35,8 +37,8 @@ CREATE TABLE Products_Tasks (
             REFERENCES Tasks(TaskID),
     PRIMARY KEY (ProductID, TaskID)
 );
-INSERT INTO Persons (FirstName, LastName, Address, City) VALUES ('Mike', 'Hughes', '123 Fake Street', 'Belfast');
+INSERT INTO Persons (FirstName, LastName, Address, City, Type) VALUES ('Mike', 'Hughes', '123 Fake Street', 'Belfast', 'qa');
-INSERT INTO Persons (FirstName, LastName, Address, City) Values ('John', 'Smith', '64 Updown Road', 'Dublin');
+INSERT INTO Persons (FirstName, LastName, Address, City, Type) VALUES ('John', 'Smith', '64 Updown Road', 'Dublin', 'programmer');
 INSERT INTO Tasks (ExecutorID, QaID, TaskName, Completed) VALUES (1, 2, 'assembling', TRUE);
 INSERT INTO Tasks (ExecutorID, QaID, TaskName, Completed) VALUES (2, 1, 'processing', FALSE);
 INSERT INTO Products (ProductName) VALUES ('Computers');

packages/server/specs/generate.js

@@ -0,0 +1,83 @@
+const swaggerJsdoc = require("swagger-jsdoc")
+const { join } = require("path")
+const { writeFileSync } = require("fs")
+const { examples, schemas } = require("./resources")
+const parameters = require("./parameters")
+const security = require("./security")
+
+const VARIABLES = {}
+
+const options = {
+  definition: {
+    openapi: "3.0.0",
+    info: {
+      title: "Budibase API",
+      description: "The public API for Budibase apps and its services.",
+      version: "1.0.0",
+    },
+    servers: [
+      {
+        url: "http://budibase.app/api/public/v1",
+        description: "Budibase Cloud API",
+      },
+      {
+        url: "{protocol}://{hostname}:10000/api/public/v1",
+        description: "Budibase self hosted API",
+      },
+    ],
+    components: {
+      parameters: {
+        ...parameters,
+      },
+      examples: {
+        ...examples,
+      },
+      securitySchemes: {
+        ...security,
+      },
+      schemas: {
+        ...schemas,
+      },
+    },
+    security: [
+      {
+        ApiKeyAuth: [],
+      },
+    ],
+  },
+  format: ".json",
+  apis: [join(__dirname, "..", "src", "api", "routes", "public", "*.ts")],
+}
+
+function writeFile(output, filename) {
+  try {
+    const path = join(__dirname, filename)
+    let spec = output
+    if (filename.endsWith("json")) {
+      spec = JSON.stringify(output, null, 2)
+    }
+    // input the static variables
+    for (let [key, replacement] of Object.entries(VARIABLES)) {
+      spec = spec.replace(new RegExp(`{${key}}`, "g"), replacement)
+    }
+    writeFileSync(path, spec)
+    console.log(`Wrote spec to ${path}`)
+    return path
+  } catch (err) {
+    console.error(err)
+  }
+}
+
+function run() {
+  const outputJSON = swaggerJsdoc(options)
+  options.format = ".yaml"
+  const outputYAML = swaggerJsdoc(options)
+  writeFile(outputJSON, "openapi.json")
+  return writeFile(outputYAML, "openapi.yaml")
+}
+
+if (require.main === module) {
+  run()
+}
+
+module.exports = run

packages/server/specs/parameters.js

@@ -0,0 +1,59 @@
+exports.tableId = {
+  in: "path",
+  name: "tableId",
+  required: true,
+  description: "The ID of the table which this request is targeting.",
+  schema: {
+    type: "string",
+  },
+}
+
+exports.rowId = {
+  in: "path",
+  name: "rowId",
+  required: true,
+  description: "The ID of the row which this request is targeting.",
+  schema: {
+    type: "string",
+  },
+}
+
+exports.appId = {
+  in: "header",
+  name: "x-budibase-app-id",
+  required: true,
+  description: "The ID of the app which this request is targeting.",
+  schema: {
+    type: "string",
+  },
+}
+
+exports.appIdUrl = {
+  in: "path",
+  name: "appId",
+  required: true,
+  description: "The ID of the app which this request is targeting.",
+  schema: {
+    type: "string",
+  },
+}
+
+exports.queryId = {
+  in: "path",
+  name: "queryId",
+  required: true,
+  description: "The ID of the query which this request is targeting.",
+  schema: {
+    type: "string",
+  },
+}
+
+exports.userId = {
+  in: "path",
+  name: "userId",
+  required: true,
+  description: "The ID of the user which this request is targeting.",
+  schema: {
+    type: "string",
+  },
+}

packages/server/specs/resources/application.js

@@ -0,0 +1,101 @@
+const userResource = require("./user")
+const { object } = require("./utils")
+const Resource = require("./utils/Resource")
+
+const application = {
+  _id: "app_metadata",
+  appId: "app_dev_957b12f943d348faa61db7e18e088d0f",
+  version: "1.0.58-alpha.0",
+  name: "App name",
+  url: "/app-url",
+  tenantId: "default",
+  updatedAt: "2022-02-22T13:00:54.035Z",
+  createdAt: "2022-02-11T18:02:26.961Z",
+  status: "development",
+  lockedBy: userResource.getExamples().user.value.user,
+}
+
+const base = {
+  name: {
+    description: "The name of the app.",
+    type: "string",
+  },
+  url: {
+    description:
+      "The URL by which the app is accessed, this must be URL encoded.",
+    type: "string",
+  },
+}
+
+const applicationSchema = object(base, { required: ["name", "url"] })
+
+const applicationOutputSchema = object(
+  {
+    ...base,
+    _id: {
+      description: "The ID of the app.",
+      type: "string",
+    },
+    status: {
+      description:
+        "The status of the app, stating it if is the development or published version.",
+      type: "string",
+      enum: ["development", "published"],
+    },
+    createdAt: {
+      description:
+        "States when the app was created, will be constant. Stored in ISO format.",
+      type: "string",
+    },
+    updatedAt: {
+      description:
+        "States the last time the app was updated - stored in ISO format.",
+      type: "string",
+    },
+    version: {
+      description:
+        "States the version of the Budibase client this app is currently based on.",
+      type: "string",
+    },
+    tenantId: {
+      description:
+        "In a multi-tenant environment this will state the tenant this app is within.",
+      type: "string",
+    },
+    lockedBy: {
+      description: "The user this app is currently being built by.",
+      type: "object",
+    },
+  },
+  {
+    required: [
+      "_id",
+      "name",
+      "url",
+      "status",
+      "createdAt",
+      "updatedAt",
+      "version",
+    ],
+  }
+)
+
+module.exports = new Resource()
+  .setExamples({
+    application: {
+      value: {
+        data: application,
+      },
+    },
+    applications: {
+      value: {
+        data: [application],
+      },
+    },
+  })
+  .setSchemas({
+    application: applicationSchema,
+    applicationOutput: object({
+      data: applicationOutputSchema,
+    }),
+  })

packages/server/specs/resources/index.js

@@ -0,0 +1,24 @@
+const application = require("./application")
+const row = require("./row")
+const table = require("./table")
+const query = require("./query")
+const user = require("./user")
+const misc = require("./misc")
+
+exports.examples = {
+  ...application.getExamples(),
+  ...row.getExamples(),
+  ...table.getExamples(),
+  ...query.getExamples(),
+  ...user.getExamples(),
+  ...misc.getExamples(),
+}
+
+exports.schemas = {
+  ...application.getSchemas(),
+  ...row.getSchemas(),
+  ...table.getSchemas(),
+  ...query.getSchemas(),
+  ...user.getSchemas(),
+  ...misc.getSchemas(),
+}

packages/server/specs/resources/misc.js

@@ -0,0 +1,12 @@
+const { object } = require("./utils")
+const Resource = require("./utils/Resource")
+
+module.exports = new Resource().setSchemas({
+  nameSearch: object({
+    name: {
+      type: "string",
+      description:
+        "The name to be used when searching - this will be used in a case insensitive starts with match.",
+    },
+  }),
+})

packages/server/specs/resources/query.js

@@ -0,0 +1,189 @@
+const Resource = require("./utils/Resource")
+const { object } = require("./utils")
+const { BaseQueryVerbs } = require("../../src/constants")
+
+const query = {
+  _id: "query_datasource_plus_4d8be0c506b9465daf4bf84d890fdab6_454854487c574d45bc4029b1e153219e",
+  datasourceId: "datasource_plus_4d8be0c506b9465daf4bf84d890fdab6",
+  parameters: [],
+  fields: {
+    sql: "select * from persons",
+  },
+  queryVerb: "read",
+  name: "Help",
+  schema: {
+    personid: {
+      name: "personid",
+      type: "string",
+    },
+    lastname: {
+      name: "lastname",
+      type: "string",
+    },
+    firstname: {
+      name: "firstname",
+      type: "string",
+    },
+    address: {
+      name: "address",
+      type: "string",
+    },
+    city: {
+      name: "city",
+      type: "string",
+    },
+  },
+  transformer: "return data",
+  readable: true,
+}
+
+const restResponse = {
+  value: {
+    data: [
+      {
+        value: "<html lang='en-GB'></html>",
+      },
+    ],
+    pagination: {
+      cursor: "2",
+    },
+    raw: "<html lang='en-GB'></html>",
+    headers: {
+      "content-type": "text/html; charset=ISO-8859-1",
+    },
+  },
+}
+
+const sqlResponse = {
+  value: {
+    data: [
+      {
+        personid: 1,
+        lastname: "Hughes",
+        firstname: "Mike",
+        address: "123 Fake Street",
+        city: "Belfast",
+      },
+      {
+        personid: 2,
+        lastname: "Smith",
+        firstname: "John",
+        address: "64 Updown Road",
+        city: "Dublin",
+      },
+    ],
+  },
+}
+
+const querySchema = object(
+  {
+    _id: {
+      description: "The ID of the query.",
+      type: "string",
+    },
+    datasourceId: {
+      description: "The ID of the data source the query belongs to.",
+      type: "string",
+    },
+    parameters: {
+      description: "The bindings which are required to perform this query.",
+      type: "array",
+      items: {
+        type: "string",
+      },
+    },
+    fields: {
+      description:
+        "The fields that are used to perform this query, e.g. the sql statement",
+      type: "object",
+    },
+    queryVerb: {
+      description: "The verb that describes this query.",
+      enum: Object.values(BaseQueryVerbs),
+    },
+    name: {
+      description: "The name of the query.",
+      type: "string",
+    },
+    schema: {
+      description:
+        "The schema of the data returned when the query is executed.",
+      type: "object",
+    },
+    transformer: {
+      description:
+        "The JavaScript transformer function, applied after the query responds with data.",
+      type: "string",
+    },
+    readable: {
+      description: "Whether the query has readable data.",
+      type: "boolean",
+    },
+  },
+  { required: ["name", "schema", "_id"] }
+)
+
+const executeQuerySchema = {
+  description:
+    "The query body must contain the required parameters for the query, this depends on query type, setup and bindings.",
+  type: "object",
+  additionalProperties: {
+    description:
+      "Key value properties of any type, depending on the query output schema.",
+  },
+}
+
+const executeQueryOutputSchema = object(
+  {
+    data: {
+      description: "The data response from the query.",
+      type: "array",
+      items: {
+        type: "object",
+      },
+    },
+    extra: {
+      description:
+        "Extra information that is not part of the main data, e.g. headers.",
+      type: "object",
+      properties: {
+        headers: {
+          description:
+            "If carrying out a REST request, this will contain the response headers.",
+          type: "object",
+        },
+        raw: {
+          description: "The raw query response, as a string.",
+          type: "string",
+        },
+      },
+    },
+    pagination: {
+      description:
+        "If pagination is supported, this will contain the bookmark/anchor information for it.",
+      type: "object",
+    },
+  },
+  { required: ["data"] }
+)
+
+module.exports = new Resource()
+  .setExamples({
+    query: {
+      value: {
+        data: query,
+      },
+    },
+    queries: {
+      value: {
+        data: [query],
+      },
+    },
+    restResponse,
+    sqlResponse,
+  })
+  .setSchemas({
+    executeQuery: executeQuerySchema,
+    executeQueryOutput: executeQueryOutputSchema,
+    query: querySchema,
+  })

packages/server/specs/resources/row.js

@@ -0,0 +1,125 @@
+const { object } = require("./utils")
+const Resource = require("./utils/Resource")
+
+const baseRow = {
+  _id: "ro_ta_5b1649e42a5b41dea4ef7742a36a7a70_e6dc7e38cf1343b2b56760265201cda4",
+  type: "row",
+  tableId: "ta_5b1649e42a5b41dea4ef7742a36a7a70",
+  name: "Mike",
+  age: 30,
+}
+
+const inputRow = {
+  ...baseRow,
+  relationship: ["ro_ta_..."],
+}
+
+const row = {
+  ...baseRow,
+  relationship: [
+    {
+      primaryDisplay: "Joe",
+      _id: "ro_ta_...",
+    },
+  ],
+}
+
+const enrichedRow = {
+  _id: "ro_ta_5b1649e42a5b41dea4ef7742a36a7a70_e6dc7e38cf1343b2b56760265201cda4",
+  name: "eg",
+  tableId: "ta_5b1649e42a5b41dea4ef7742a36a7a70",
+  type: "row",
+  relationship: [
+    {
+      _id: "ro_ta_users_us_8f3d717147d74d759d8cef5b6712062f",
+      name: "Joe",
+      tableId: "ta_users",
+      internal: [
+        {
+          _id: "ro_ta_5b1649e42a5b41dea4ef7742a36a7a70_e6dc7e38cf1343b2b56760265201cda4",
+          primaryDisplay: "eg",
+        },
+      ],
+    },
+  ],
+}
+
+const rowSchema = {
+  description: "The row to be created/updated, based on the table schema.",
+  type: "object",
+  additionalProperties: {
+    description:
+      "Key value properties of any type, depending on the table schema.",
+  },
+}
+
+const rowOutputSchema = {
+  ...rowSchema,
+  properties: {
+    ...rowSchema.properties,
+    _id: {
+      description: "The ID of the row.",
+      type: "string",
+    },
+    tableId: {
+      description: "The ID of the table this row comes from.",
+      type: "string",
+    },
+  },
+  required: ["tableId", "_id"],
+}
+
+const searchOutputSchema = {
+  type: "object",
+  required: ["data"],
+  properties: {
+    data: {
+      description:
+        "An array of rows, these will each contain an _id field which can be used to update or delete them.",
+      type: "array",
+      items: {
+        type: "object",
+      },
+    },
+    bookmark: {
+      description: "If pagination in use, this should be provided.",
+      oneOf: [{ type: "string" }, { type: "integer" }],
+    },
+    hasNextPage: {
+      description:
+        "If pagination in use, this will determine if there is another page to fetch.",
+      type: "boolean",
+    },
+  },
+}
+
+module.exports = new Resource()
+  .setExamples({
+    inputRow: {
+      value: inputRow,
+    },
+    row: {
+      value: {
+        data: row,
+      },
+    },
+    enrichedRow: {
+      value: {
+        data: enrichedRow,
+      },
+    },
+    rows: {
+      value: {
+        data: [row],
+        hasNextPage: true,
+        bookmark: 10,
+      },
+    },
+  })
+  .setSchemas({
+    row: rowSchema,
+    searchOutput: searchOutputSchema,
+    rowOutput: object({
+      data: rowOutputSchema,
+    }),
+  })

packages/server/specs/resources/table.js

@@ -0,0 +1,191 @@
+const {
+  FieldTypes,
+  RelationshipTypes,
+  FormulaTypes,
+} = require("../../src/constants")
+const { object } = require("./utils")
+const Resource = require("./utils/Resource")
+
+const table = {
+  _id: "ta_5b1649e42a5b41dea4ef7742a36a7a70",
+  name: "People",
+  schema: {
+    name: {
+      type: "string",
+      name: "name",
+    },
+    age: {
+      type: "number",
+      name: "age",
+    },
+    relationship: {
+      type: "link",
+      name: "relationship",
+      tableId: "ta_...",
+      fieldName: "relatedColumn",
+      relationshipType: "many-to-many",
+    },
+  },
+}
+
+const baseColumnDef = {
+  type: {
+    type: "string",
+    enum: Object.values(FieldTypes),
+    description:
+      "Defines the type of the column, most explain themselves, a link column is a relationship.",
+  },
+  constraints: {
+    type: "object",
+    description:
+      "A constraint can be applied to the column which will be validated against when a row is saved.",
+    properties: {
+      type: {
+        type: "string",
+        enum: ["string", "number", "object", "boolean"],
+      },
+      presence: {
+        type: "boolean",
+        description: "Defines whether the column is required or not.",
+      },
+    },
+  },
+  name: {
+    type: "string",
+    description: "The name of the column.",
+  },
+  autocolumn: {
+    type: "boolean",
+    description: "Defines whether the column is automatically generated.",
+  },
+}
+
+const tableSchema = {
+  description: "The table to be created/updated.",
+  type: "object",
+  required: ["name", "schema"],
+  properties: {
+    name: {
+      description: "The name of the table.",
+      type: "string",
+    },
+    primaryDisplay: {
+      type: "string",
+      description:
+        "The name of the column which should be used in relationship tags when relating to this table.",
+    },
+    schema: {
+      type: "object",
+      additionalProperties: {
+        oneOf: [
+          // relationship
+          {
+            type: "object",
+            properties: {
+              ...baseColumnDef,
+              type: {
+                type: "string",
+                enum: [FieldTypes.LINK],
+                description: "A relationship column.",
+              },
+              fieldName: {
+                type: "string",
+                description:
+                  "The name of the column which a relationship column is related to in another table.",
+              },
+              tableId: {
+                type: "string",
+                description:
+                  "The ID of the table which a relationship column is related to.",
+              },
+              relationshipType: {
+                type: "string",
+                enum: Object.values(RelationshipTypes),
+                description:
+                  "Defines the type of relationship that this column will be used for.",
+              },
+              through: {
+                type: "string",
+                description:
+                  "When using a SQL table that contains many to many relationships this defines the table the relationships are linked through.",
+              },
+              foreignKey: {
+                type: "string",
+                description:
+                  "When using a SQL table that contains a one to many relationship this defines the foreign key.",
+              },
+              throughFrom: {
+                type: "string",
+                description:
+                  "When using a SQL table that utilises a through table, this defines the primary key in the through table for this table.",
+              },
+              throughTo: {
+                type: "string",
+                description:
+                  "When using a SQL table that utilises a through table, this defines the primary key in the through table for the related table.",
+              },
+            },
+          },
+          {
+            type: "object",
+            properties: {
+              ...baseColumnDef,
+              type: {
+                type: "string",
+                enum: [FieldTypes.FORMULA],
+                description: "A formula column.",
+              },
+              formula: {
+                type: "string",
+                description:
+                  "Defines a Handlebars or JavaScript formula to use, note that Javascript formulas are expected to be provided in the base64 format.",
+              },
+              formulaType: {
+                type: "string",
+                enum: Object.values(FormulaTypes),
+                description:
+                  "Defines whether this is a static or dynamic formula.",
+              },
+            },
+          },
+          {
+            type: "object",
+            properties: baseColumnDef,
+          },
+        ],
+      },
+    },
+  },
+}
+
+const tableOutputSchema = {
+  ...tableSchema,
+  properties: {
+    ...tableSchema.properties,
+    _id: {
+      description: "The ID of the table.",
+      type: "string",
+    },
+  },
+  required: [...tableSchema.required, "_id"],
+}
+
+module.exports = new Resource()
+  .setExamples({
+    table: {
+      value: {
+        data: table,
+      },
+    },
+    tables: {
+      value: {
+        data: [table],
+      },
+    },
+  })
+  .setSchemas({
+    table: tableSchema,
+    tableOutput: object({
+      data: tableOutputSchema,
+    }),
+  })

packages/server/specs/resources/user.js

@@ -0,0 +1,126 @@
+const { object } = require("./utils")
+const Resource = require("./utils/Resource")
+
+const user = {
+  _id: "us_693a73206518477283a8d5ae31103252",
+  email: "test@test.com",
+  roles: {
+    app_957b12f943d348faa61db7e18e088d0f: "BASIC",
+  },
+  builder: {
+    global: false,
+  },
+  admin: {
+    global: true,
+  },
+  tenantId: "default",
+  status: "active",
+  budibaseAccess: true,
+  csrfToken: "9c70291d-7137-48f9-9166-99ab5473a3d4",
+  userId: "us_693a73206518477283a8d5ae31103252",
+  roleId: "ADMIN",
+  role: {
+    _id: "ADMIN",
+    name: "Admin",
+    permissionId: "admin",
+    inherits: "POWER",
+  },
+}
+
+const userSchema = object(
+  {
+    email: {
+      description: "The email address of the user, this must be unique.",
+      type: "string",
+    },
+    password: {
+      description:
+        "The password of the user if using password based login - this will never be returned. This can be" +
+        " left out of subsequent requests (updates) and will be enriched back into the user structure.",
+      type: "string",
+    },
+    status: {
+      description: "The status of the user, if they are active.",
+      type: "string",
+      enum: ["active"],
+    },
+    firstName: {
+      description: "The first name of the user",
+      type: "string",
+    },
+    lastName: {
+      description: "The last name of the user",
+      type: "string",
+    },
+    forceResetPassword: {
+      description:
+        "If set to true forces the user to reset their password on first login.",
+      type: "boolean",
+    },
+    builder: {
+      description: "Describes if the user is a builder user or not.",
+      type: "object",
+      properties: {
+        global: {
+          description:
+            "If set to true the user will be able to build any app in the system.",
+          type: "boolean",
+        },
+      },
+    },
+    admin: {
+      description: "Describes if the user is an admin user or not.",
+      type: "object",
+      properties: {
+        global: {
+          description:
+            "If set to true the user will be able to administrate the system.",
+          type: "boolean",
+        },
+      },
+    },
+    roles: {
+      description:
+        "Contains the roles of the user per app (assuming they are not a builder user).",
+      type: "object",
+      additionalProperties: {
+        type: "string",
+        description:
+          "A map of app ID (production app ID, minus the _dev component) to a role ID, e.g. ADMIN.",
+      },
+    },
+  },
+  { required: ["email", "roles"] }
+)
+
+const userOutputSchema = {
+  ...userSchema,
+  properties: {
+    ...userSchema.properties,
+    _id: {
+      description: "The ID of the user.",
+      type: "string",
+    },
+  },
+  required: [...userSchema.required, "_id"],
+}
+
+module.exports = new Resource()
+  .setExamples({
+    user: {
+      value: {
+        data: user,
+      },
+    },
+    users: {
+      value: {
+        data: [user],
+      },
+    },
+  })
+  .setSchemas({
+    user: userSchema,
+    userOutput: object({
+      data: userOutputSchema,
+    }),
+  })

packages/server/specs/resources/utils/Resource.js

@@ -0,0 +1,26 @@
+class Resource {
+  constructor() {
+    this.examples = {}
+    this.schemas = {}
+  }
+
+  setExamples(examples) {
+    this.examples = examples
+    return this
+  }
+
+  setSchemas(schemas) {
+    this.schemas = schemas
+    return this
+  }
+
+  getExamples() {
+    return this.examples
+  }
+
+  getSchemas() {
+    return this.schemas
+  }
+}
+
+module.exports = Resource

packages/server/specs/resources/utils/index.js

@@ -0,0 +1,11 @@
+exports.object = (props, opts) => {
+  const base = {
+    type: "object",
+    properties: props,
+    ...opts,
+  }
+  if (Object.keys(props).length > 0 && (!opts || !opts.required)) {
+    base.required = Object.keys(props)
+  }
+  return base
+}

packages/server/specs/security.js

@@ -0,0 +1,7 @@
+exports.ApiKeyAuth = {
+  type: "apiKey",
+  in: "header",
+  name: "x-budibase-api-key",
+  description:
+    "Your individual API key, this will provide access based on the configured RBAC settings of your user.",
+}

packages/server/src/api/controllers/application.js

@@ -263,6 +263,7 @@ exports.create = async ctx => {
     tenantId: getTenantId(),
     updatedAt: new Date().toISOString(),
     createdAt: new Date().toISOString(),
+    status: AppStatus.DEV,
   }
   const response = await db.put(newApplication, { force: true })
   newApplication._rev = response.rev

packages/server/src/api/controllers/public/applications.ts

@@ -0,0 +1,74 @@
+const { getAllApps } = require("@budibase/backend-core/db")
+const { updateAppId } = require("@budibase/backend-core/context")
+import { search as stringSearch } from "./utils"
+import { default as controller } from "../application"
+import { Application } from "../../../definitions/common"
+
+function fixAppID(app: Application, params: any) {
+  if (!params) {
+    return app
+  }
+  if (!app._id && params.appId) {
+    app._id = params.appId
+  }
+  return app
+}
+
+async function setResponseApp(ctx: any) {
+  if (ctx.body && ctx.body.appId && (!ctx.params || !ctx.params.appId)) {
+    ctx.params = { appId: ctx.body.appId }
+  }
+  await controller.fetchAppPackage(ctx)
+}
+
+export async function search(ctx: any, next: any) {
+  const { name } = ctx.request.body
+  const apps = await getAllApps({ all: true })
+  ctx.body = stringSearch(apps, name)
+  await next()
+}
+
+export async function create(ctx: any, next: any) {
+  if (!ctx.request.body || !ctx.request.body.useTemplate) {
+    ctx.request.body = {
+      useTemplate: false,
+      ...ctx.request.body,
+    }
+  }
+  await controller.create(ctx)
+  await setResponseApp(ctx)
+  await next()
+}
+
+export async function read(ctx: any, next: any) {
+  updateAppId(ctx.params.appId)
+  await setResponseApp(ctx)
+  await next()
+}
+
+export async function update(ctx: any, next: any) {
+  ctx.request.body = fixAppID(ctx.request.body, ctx.params)
+  updateAppId(ctx.params.appId)
+  await controller.update(ctx)
+  await setResponseApp(ctx)
+  await next()
+}
+
+export async function destroy(ctx: any, next: any) {
+  updateAppId(ctx.params.appId)
+  // get the app before deleting it
+  await setResponseApp(ctx)
+  const body = ctx.body
+  await controller.delete(ctx)
+  // overwrite the body again
+  ctx.body = body
+  await next()
+}
+
+export default {
+  create,
+  update,
+  read,
+  destroy,
+  search,
+}

packages/server/src/api/controllers/public/mapping/applications.ts

@@ -0,0 +1,32 @@
+import { Application } from "./types"
+
+function application(body: any): Application {
+  let app = body?.application ? body.application : body
+  return {
+    _id: app.appId,
+    name: app.name,
+    url: app.url,
+    status: app.status,
+    createdAt: app.createdAt,
+    updatedAt: app.updatedAt,
+    version: app.version,
+    tenantId: app.tenantId,
+    lockedBy: app.lockedBy,
+  }
+}
+
+function mapApplication(ctx: any): { data: Application } {
+  return {
+    data: application(ctx.body),
+  }
+}
+
+function mapApplications(ctx: any): { data: Application[] } {
+  const apps = ctx.body.map((body: any) => application(body))
+  return { data: apps }
+}
+
+export default {
+  mapApplication,
+  mapApplications,
+}

packages/server/src/api/controllers/public/mapping/index.ts

@@ -0,0 +1,13 @@
+import tables from "./tables"
+import applications from "./applications"
+import users from "./users"
+import rows from "./rows"
+import queries from "./queries"
+
+export default {
+  ...tables,
+  ...applications,
+  ...users,
+  ...rows,
+  ...queries,
+}

packages/server/src/api/controllers/public/mapping/queries.ts

@@ -0,0 +1,39 @@
+import { Query, ExecuteQuery } from "./types"
+
+function query(body: any): Query {
+  return {
+    _id: body._id,
+    datasourceId: body.datasourceId,
+    parameters: body.parameters,
+    fields: body.fields,
+    queryVerb: body.queryVerb,
+    name: body.name,
+    schema: body.schema,
+    transformer: body.transformer,
+    readable: body.readable,
+  }
+}
+
+function mapQueries(ctx: any): { data: Query[] } {
+  const queries = ctx.body.map((body: any) => query(body))
+  return {
+    data: queries,
+  }
+}
+
+function mapQueryExecution(ctx: any): ExecuteQuery {
+  // very little we can map here, structure mostly unknown
+  return {
+    data: ctx.body.data,
+    pagination: ctx.body.pagination,
+    extra: {
+      raw: ctx.body.raw,
+      headers: ctx.body.headers,
+    },
+  }
+}
+
+export default {
+  mapQueries,
+  mapQueryExecution,
+}

packages/server/src/api/controllers/public/mapping/rows.ts

@@ -0,0 +1,31 @@
+import { Row, RowSearch } from "./types"
+
+function row(body: any): Row {
+  delete body._rev
+  // have to input everything, since structure unknown
+  return {
+    ...body,
+    _id: body._id,
+    tableId: body.tableId,
+  }
+}
+
+function mapRowSearch(ctx: any): RowSearch {
+  const rows = ctx.body.rows.map((body: any) => row(body))
+  return {
+    data: rows,
+    hasNextPage: ctx.body.hasNextPage,
+    bookmark: ctx.body.bookmark,
+  }
+}
+
+function mapRow(ctx: any): { data: Row } {
+  return {
+    data: row(ctx.body),
+  }
+}
+
+export default {
+  mapRowSearch,
+  mapRow,
+}

packages/server/src/api/controllers/public/mapping/tables.ts

@@ -0,0 +1,26 @@
+import { Table } from "./types"
+
+function table(body: any): Table {
+  return {
+    _id: body._id,
+    name: body.name,
+    schema: body.schema,
+    primaryDisplay: body.primaryDisplay,
+  }
+}
+
+function mapTable(ctx: any): { data: Table } {
+  return {
+    data: table(ctx.body),
+  }
+}
+
+function mapTables(ctx: any): { data: Table[] } {
+  const tables = ctx.body.map((body: any) => table(body))
+  return { data: tables }
+}
+
+export default {
+  mapTable,
+  mapTables,
+}

packages/server/src/api/controllers/public/mapping/types.ts

@@ -0,0 +1,13 @@
+import { components } from "../../../../definitions/openapi"
+
+export type Query = components["schemas"]["query"]
+export type ExecuteQuery = components["schemas"]["executeQueryOutput"]
+
+export type Application = components["schemas"]["applicationOutput"]["data"]
+
+export type Table = components["schemas"]["tableOutput"]["data"]
+
+export type Row = components["schemas"]["rowOutput"]["data"]
+export type RowSearch = components["schemas"]["searchOutput"]
+
+export type User = components["schemas"]["userOutput"]["data"]

packages/server/src/api/controllers/public/mapping/users.ts

@@ -0,0 +1,32 @@
+import { User } from "./types"
+
+function user(body: any): User {
+  return {
+    _id: body._id,
+    email: body.email,
+    password: body.password,
+    status: body.status,
+    firstName: body.firstName,
+    lastName: body.lastName,
+    forceResetPassword: body.forceResetPassword,
+    builder: body.builder,
+    admin: body.admin,
+    roles: body.roles,
+  }
+}
+
+function mapUser(ctx: any): { data: User } {
+  return {
+    data: user(ctx.body),
+  }
+}
+
+function mapUsers(ctx: any): { data: User[] } {
+  const users = ctx.body.map((body: any) => user(body))
+  return { data: users }
+}
+
+export default {
+  mapUser,
+  mapUsers,
+}

packages/server/src/api/controllers/public/queries.ts

@@ -0,0 +1,20 @@
+import { search as stringSearch } from "./utils"
+import { default as queryController } from "../query"
+
+export async function search(ctx: any, next: any) {
+  await queryController.fetch(ctx)
+  const { name } = ctx.request.body
+  ctx.body = stringSearch(ctx.body, name)
+  await next()
+}
+
+export async function execute(ctx: any, next: any) {
+  // don't wrap this, already returns "data"
+  await queryController.executeV2(ctx)
+  await next()
+}
+
+export default {
+  search,
+  execute,
+}

packages/server/src/api/controllers/public/rows.ts

@@ -0,0 +1,77 @@
+import { default as rowController } from "../row"
+import { addRev } from "./utils"
+import { Row } from "../../../definitions/common"
+
+// makes sure that the user doesn't need to pass in the type, tableId or _id params for
+// the call to be correct
+function fixRow(row: Row, params: any) {
+  if (!params || !row) {
+    return row
+  }
+  if (params.rowId) {
+    row._id = params.rowId
+  }
+  if (params.tableId) {
+    row.tableId = params.tableId
+  }
+  if (!row.type) {
+    row.type = "row"
+  }
+  return row
+}
+
+export async function search(ctx: any, next: any) {
+  let { sort, paginate, bookmark, limit, query } = ctx.request.body
+  // update the body to the correct format of the internal search
+  if (!sort) {
+    sort = {}
+  }
+  ctx.request.body = {
+    sort: sort.column,
+    sortType: sort.type,
+    sortOrder: sort.order,
+    bookmark,
+    paginate,
+    limit,
+    query,
+  }
+  await rowController.search(ctx)
+  await next()
+}
+
+export async function create(ctx: any, next: any) {
+  ctx.request.body = fixRow(ctx.request.body, ctx.params)
+  await rowController.save(ctx)
+  await next()
+}
+
+export async function read(ctx: any, next: any) {
+  await rowController.fetchEnrichedRow(ctx)
+  await next()
+}
+
+export async function update(ctx: any, next: any) {
+  ctx.request.body = await addRev(fixRow(ctx.request.body, ctx.params.tableId))
+  await rowController.save(ctx)
+  await next()
+}
+
+export async function destroy(ctx: any, next: any) {
+  // set the body as expected, with the _id and _rev fields
+  ctx.request.body = await addRev(
+    fixRow({ _id: ctx.params.rowId }, ctx.params.tableId)
+  )
+  await rowController.destroy(ctx)
+  // destroy controller doesn't currently return the row as the body, need to adjust this
+  // in the public API to be correct
+  ctx.body = ctx.row
+  await next()
+}
+
+export default {
+  create,
+  read,
+  update,
+  destroy,
+  search,
+}

packages/server/src/api/controllers/public/tables.ts

@@ -0,0 +1,56 @@
+import { search as stringSearch, addRev } from "./utils"
+import { default as controller } from "../table"
+import { Table } from "../../../definitions/common"
+
+function fixTable(table: Table, params: any) {
+  if (!params || !table) {
+    return table
+  }
+  if (params.tableId) {
+    table._id = params.tableId
+  }
+  if (!table.type) {
+    table.type = "table"
+  }
+  return table
+}
+
+export async function search(ctx: any, next: any) {
+  const { name } = ctx.request.body
+  await controller.fetch(ctx)
+  ctx.body = stringSearch(ctx.body, name)
+  await next()
+}
+
+export async function create(ctx: any, next: any) {
+  await controller.save(ctx)
+  await next()
+}
+
+export async function read(ctx: any, next: any) {
+  await controller.find(ctx)
+  await next()
+}
+
+export async function update(ctx: any, next: any) {
+  ctx.request.body = await addRev(
+    fixTable(ctx.request.body, ctx.params),
+    ctx.params.tableId
+  )
+  await controller.save(ctx)
+  await next()
+}
+
+export async function destroy(ctx: any, next: any) {
+  await controller.destroy(ctx)
+  ctx.body = ctx.table
+  await next()
+}
+
+export default {
+  create,
+  read,
+  update,
+  destroy,
+  search,
+}

packages/server/src/api/controllers/public/users.ts

@@ -0,0 +1,82 @@
+import {
+  allGlobalUsers,
+  deleteGlobalUser,
+  readGlobalUser,
+  saveGlobalUser,
+} from "../../../utilities/workerRequests"
+import { search as stringSearch } from "./utils"
+
+const { getProdAppID } = require("@budibase/backend-core/db")
+
+function fixUser(ctx: any) {
+  if (!ctx.request.body) {
+    return ctx
+  }
+  if (!ctx.request.body._id && ctx.params.userId) {
+    ctx.request.body._id = ctx.params.userId
+  }
+  if (!ctx.request.body.roles) {
+    ctx.request.body.roles = {}
+  } else {
+    const newRoles: { [key: string]: string } = {}
+    for (let [appId, role] of Object.entries(ctx.request.body.roles)) {
+      // @ts-ignore
+      newRoles[getProdAppID(appId)] = role
+    }
+    ctx.request.body.roles = newRoles
+  }
+  return ctx
+}
+
+function getUser(ctx: any, userId?: string) {
+  if (userId) {
+    ctx.params = { userId }
+  } else if (!ctx.params?.userId) {
+    throw "No user ID provided for getting"
+  }
+  return readGlobalUser(ctx)
+}
+
+export async function search(ctx: any, next: any) {
+  const { name } = ctx.request.body
+  const users = await allGlobalUsers(ctx)
+  ctx.body = stringSearch(users, name, "email")
+  await next()
+}
+
+export async function create(ctx: any, next: any) {
+  const response = await saveGlobalUser(fixUser(ctx))
+  ctx.body = await getUser(ctx, response._id)
+  await next()
+}
+
+export async function read(ctx: any, next: any) {
+  ctx.body = await readGlobalUser(ctx)
+  await next()
+}
+
+export async function update(ctx: any, next: any) {
+  const user = await readGlobalUser(ctx)
+  ctx.request.body = {
+    ...ctx.request.body,
+    _rev: user._rev,
+  }
+  const response = await saveGlobalUser(fixUser(ctx))
+  ctx.body = await getUser(ctx, response._id)
+  await next()
+}
+
+export async function destroy(ctx: any, next: any) {
+  const user = await getUser(ctx)
+  await deleteGlobalUser(ctx)
+  ctx.body = user
+  await next()
+}
+
+export default {
+  create,
+  read,
+  update,
+  destroy,
+  search,
+}

packages/server/src/api/controllers/public/utils.ts

@@ -0,0 +1,38 @@
+const { getAppDB } = require("@budibase/backend-core/context")
+import { isExternalTable } from "../../../integrations/utils"
+
+export async function addRev(
+  body: { _id?: string; _rev?: string },
+  tableId?: string
+) {
+  if (!body._id || (tableId && isExternalTable(tableId))) {
+    return body
+  }
+  const db = getAppDB()
+  const dbDoc = await db.get(body._id)
+  body._rev = dbDoc._rev
+  return body
+}
+
+/**
+ * Performs a case insensitive search on the provided documents, using the
+ * provided key and value. This will be a string based search, using the
+ * startsWith function.
+ */
+export function search(docs: any[], value: any, key = "name") {
+  if (!value || typeof value !== "string") {
+    return docs
+  }
+  value = value.toLowerCase()
+  const filtered = []
+  for (let doc of docs) {
+    if (typeof doc[key] !== "string") {
+      continue
+    }
+    const toTest = doc[key].toLowerCase()
+    if (toTest.startsWith(value)) {
+      filtered.push(doc)
+    }
+  }
+  return filtered
+}

packages/server/src/api/controllers/row/internal.js

@@ -2,6 +2,7 @@ const linkRows = require("../../../db/linkedRows")
 const {
   generateRowID,
   getRowParams,
+  getTableIDFromRowID,
   DocumentTypes,
   InternalTables,
 } = require("../../../db/utils")
@@ -386,6 +387,9 @@ exports.fetchEnrichedRow = async ctx => {
   let groups = {},
     tables = {}
   for (let row of response) {
+    if (!row.tableId) {
+      row.tableId = getTableIDFromRowID(row._id)
+    }
     const linkedTableId = row.tableId
     if (groups[linkedTableId] == null) {
       groups[linkedTableId] = [row]

packages/server/src/api/controllers/table/index.js

@@ -48,7 +48,7 @@ exports.fetch = async function (ctx) {
 }
 
 exports.find = async function (ctx) {
-  const tableId = ctx.params.id
+  const tableId = ctx.params.tableId
   ctx.body = await getTable(tableId)
 }
 
@@ -70,6 +70,7 @@ exports.destroy = async function (ctx) {
   ctx.eventEmitter &&
     ctx.eventEmitter.emitTable(`table:delete`, appId, deletedTable)
   ctx.status = 200
+  ctx.table = deletedTable
   ctx.body = { message: `Table ${tableId} deleted.` }
 }
 

packages/server/src/api/controllers/webhook.js

@@ -1,6 +1,7 @@
 const { generateWebhookID, getWebhookParams } = require("../../db/utils")
 const toJsonSchema = require("to-json-schema")
 const validate = require("jsonschema").validate
+const { WebhookType } = require("../../constants")
 const triggers = require("../../automations/triggers")
 const { getProdAppID } = require("@budibase/backend-core/db")
 const { getAppDB, updateAppId } = require("@budibase/backend-core/context")
@@ -18,10 +19,6 @@ function Webhook(name, type, target) {
 
 exports.Webhook = Webhook
 
-exports.WebhookType = {
-  AUTOMATION: "automation",
-}
-
 exports.fetch = async ctx => {
   const db = getAppDB()
   const response = await db.allDocs(
@@ -62,7 +59,7 @@ exports.buildSchema = async ctx => {
   const webhook = await db.get(ctx.params.id)
   webhook.bodySchema = toJsonSchema(ctx.request.body)
   // update the automation outputs
-  if (webhook.action.type === exports.WebhookType.AUTOMATION) {
+  if (webhook.action.type === WebhookType.AUTOMATION) {
     let automation = await db.get(webhook.action.target)
     const autoOutputs = automation.definition.trigger.schema.outputs
     let properties = webhook.bodySchema.properties
@@ -92,7 +89,7 @@ exports.trigger = async ctx => {
       validate(ctx.request.body, webhook.bodySchema)
     }
     const target = await db.get(webhook.action.target)
-    if (webhook.action.type === exports.WebhookType.AUTOMATION) {
+    if (webhook.action.type === WebhookType.AUTOMATION) {
       // trigger with both the pure request and then expand it
       // incase the user has produced a schema to bind to
       await triggers.externalTrigger(target, {

packages/server/src/api/index.js

@@ -8,7 +8,7 @@ const {
 const currentApp = require("../middleware/currentapp")
 const compress = require("koa-compress")
 const zlib = require("zlib")
-const { mainRoutes, staticRoutes } = require("./routes")
+const { mainRoutes, staticRoutes, publicRoutes } = require("./routes")
 const pkg = require("../../package.json")
 const env = require("../environment")
 
@@ -82,6 +82,9 @@ for (let route of mainRoutes) {
   router.use(route.allowedMethods())
 }
 
+router.use(publicRoutes.routes())
+router.use(publicRoutes.allowedMethods())
+
 // WARNING - static routes will catch everything else after them this must be last
 router.use(staticRoutes.routes())
 router.use(staticRoutes.allowedMethods())

packages/server/src/api/routes/automation.js

@@ -1,50 +1,20 @@
 const Router = require("@koa/router")
 const controller = require("../controllers/automation")
 const authorized = require("../../middleware/authorized")
-const joiValidator = require("../../middleware/joi-validator")
 const {
   BUILDER,
   PermissionLevels,
   PermissionTypes,
 } = require("@budibase/backend-core/permissions")
-const Joi = require("joi")
 const { bodyResource, paramResource } = require("../../middleware/resourceId")
 const {
   middleware: appInfoMiddleware,
   AppType,
 } = require("../../middleware/appInfo")
+const { automationValidator } = require("./utils/validators")
 
 const router = Router()
 
-// prettier-ignore
-function generateStepSchema(allowStepTypes) {
-  return Joi.object({
-    stepId: Joi.string().required(),
-    id: Joi.string().required(),
-    description: Joi.string().required(),
-    name: Joi.string().required(),
-    tagline: Joi.string().required(),
-    icon: Joi.string().required(),
-    params: Joi.object(),
-    args: Joi.object(),
-    type: Joi.string().required().valid(...allowStepTypes),
-  }).unknown(true)
-}
-
-function generateValidator(existing = false) {
-  // prettier-ignore
-  return joiValidator.body(Joi.object({
-    _id: existing ? Joi.string().required() : Joi.string(),
-    _rev: existing ? Joi.string().required() : Joi.string(),
-    name: Joi.string().required(),
-    type: Joi.string().valid("automation").required(),
-    definition: Joi.object({
-      steps: Joi.array().required().items(generateStepSchema(["ACTION", "LOGIC"])),
-      trigger: generateStepSchema(["TRIGGER"]).allow(null),
-    }).required().unknown(true),
-  }).unknown(true))
-}
-
 router
   .get(
     "/api/automations/trigger/list",
@@ -72,13 +42,13 @@ router
     "/api/automations",
     bodyResource("_id"),
     authorized(BUILDER),
-    generateValidator(true),
+    automationValidator(true),
     controller.update
   )
   .post(
     "/api/automations",
     authorized(BUILDER),
-    generateValidator(false),
+    automationValidator(false),
     controller.create
   )
   .delete(

packages/server/src/api/routes/datasource.js

@@ -1,64 +1,18 @@
 const Router = require("@koa/router")
 const datasourceController = require("../controllers/datasource")
 const authorized = require("../../middleware/authorized")
-const joiValidator = require("../../middleware/joi-validator")
 const {
   BUILDER,
   PermissionLevels,
   PermissionTypes,
 } = require("@budibase/backend-core/permissions")
-const Joi = require("joi")
+const {
-const { DataSourceOperation } = require("../../constants")
+  datasourceValidator,
+  datasourceQueryValidator,
+} = require("./utils/validators")
 
 const router = Router()
 
-function generateDatasourceSchema() {
-  // prettier-ignore
-  return joiValidator.body(Joi.object({
-    _id: Joi.string(),
-    _rev: Joi.string(),
-    // source: Joi.string().valid("POSTGRES_PLUS"),
-    type: Joi.string().allow("datasource_plus"),
-    relationships: Joi.array().items(Joi.object({
-      from: Joi.string().required(),
-      to: Joi.string().required(),
-      cardinality: Joi.valid("1:N", "1:1", "N:N").required()
-    })),
-    // entities: Joi.array().items(Joi.object({
-    //   type: Joi.string().valid(...Object.values(FieldTypes)).required(),
-    //   name: Joi.string().required(),
-    // })),
-  }).unknown(true))
-}
-
-function generateQueryDatasourceSchema() {
-  // prettier-ignore
-  return joiValidator.body(Joi.object({
-    endpoint: Joi.object({
-      datasourceId: Joi.string().required(),
-      operation: Joi.string().required().valid(...Object.values(DataSourceOperation)),
-      entityId: Joi.string().required(),
-    }).required(),
-    resource: Joi.object({
-      fields: Joi.array().items(Joi.string()).optional(),
-    }).optional(),
-    body: Joi.object().optional(),
-    sort: Joi.object().optional(),
-    filters: Joi.object({
-      string: Joi.object().optional(),
-      range: Joi.object().optional(),
-      equal: Joi.object().optional(),
-      notEqual: Joi.object().optional(),
-      empty: Joi.object().optional(),
-      notEmpty: Joi.object().optional(),
-    }).optional(),
-    paginate: Joi.object({
-      page: Joi.string().alphanum().optional(),
-      limit: Joi.number().optional(),
-    }).optional(),
-  }))
-}
-
 router
   .get("/api/datasources", authorized(BUILDER), datasourceController.fetch)
   .get(
@@ -74,7 +28,7 @@ router
   .post(
     "/api/datasources/query",
     authorized(PermissionTypes.TABLE, PermissionLevels.READ),
-    generateQueryDatasourceSchema(),
+    datasourceQueryValidator(),
     datasourceController.query
   )
   .post(
@@ -85,7 +39,7 @@ router
   .post(
     "/api/datasources",
     authorized(BUILDER),
-    generateDatasourceSchema(),
+    datasourceValidator(),
     datasourceController.save
   )
   .delete(

packages/server/src/api/routes/index.js

@@ -25,6 +25,7 @@ const metadataRoutes = require("./metadata")
 const devRoutes = require("./dev")
 const cloudRoutes = require("./cloud")
 const migrationRoutes = require("./migrations")
+const publicRoutes = require("./public")
 
 exports.mainRoutes = [
   authRoutes,
@@ -57,4 +58,5 @@ exports.mainRoutes = [
   migrationRoutes,
 ]
 
+exports.publicRoutes = publicRoutes
 exports.staticRoutes = staticRoutes

packages/server/src/api/routes/permission.js

@@ -1,25 +1,11 @@
 const Router = require("@koa/router")
 const controller = require("../controllers/permission")
 const authorized = require("../../middleware/authorized")
-const {
+const { BUILDER } = require("@budibase/backend-core/permissions")
-  BUILDER,
+const { permissionValidator } = require("./utils/validators")
-  PermissionLevels,
-} = require("@budibase/backend-core/permissions")
-const Joi = require("joi")
-const joiValidator = require("../../middleware/joi-validator")
 
 const router = Router()
 
-function generateValidator() {
-  const permLevelArray = Object.values(PermissionLevels)
-  // prettier-ignore
-  return joiValidator.params(Joi.object({
-    level: Joi.string().valid(...permLevelArray).required(),
-    resourceId: Joi.string(),
-    roleId: Joi.string(),
-  }).unknown(true))
-}
-
 router
   .get("/api/permission/builtin", authorized(BUILDER), controller.fetchBuiltin)
   .get("/api/permission/levels", authorized(BUILDER), controller.fetchLevels)
@@ -33,14 +19,14 @@ router
   .post(
     "/api/permission/:roleId/:resourceId/:level",
     authorized(BUILDER),
-    generateValidator(),
+    permissionValidator(),
     controller.addPermission
   )
   // deleting the level defaults it back the underlying access control for the resource
   .delete(
     "/api/permission/:roleId/:resourceId/:level",
     authorized(BUILDER),
-    generateValidator(),
+    permissionValidator(),
     controller.removePermission
   )
 

packages/server/src/api/routes/public/applications.ts

@@ -0,0 +1,156 @@
+import controller from "../../controllers/public/applications"
+import Endpoint from "./utils/Endpoint"
+const { nameValidator, applicationValidator } = require("../utils/validators")
+
+const read = [],
+  write = []
+
+/**
+ * @openapi
+ * /applications:
+ *   post:
+ *     summary: Create an application
+ *     tags:
+ *       - applications
+ *     parameters:
+ *       - $ref: '#/components/parameters/appId'
+ *     requestBody:
+ *       required: true
+ *       content:
+ *         application/json:
+ *           schema:
+ *             $ref: '#/components/schemas/application'
+ *     responses:
+ *       200:
+ *         description: Returns the created application.
+ *         content:
+ *           application/json:
+ *             schema:
+ *               $ref: '#/components/schemas/applicationOutput'
+ *             examples:
+ *               application:
+ *                 $ref: '#/components/examples/application'
+ */
+write.push(
+  new Endpoint("post", "/applications", controller.create).addMiddleware(
+    applicationValidator()
+  )
+)
+
+/**
+ * @openapi
+ * /applications/{appId}:
+ *   put:
+ *     summary: Update an application
+ *     tags:
+ *       - applications
+ *     parameters:
+ *       - $ref: '#/components/parameters/appIdUrl'
+ *     requestBody:
+ *       required: true
+ *       content:
+ *         application/json:
+ *           schema:
+ *             $ref: '#/components/schemas/application'
+ *     responses:
+ *       200:
+ *         description: Returns the updated application.
+ *         content:
+ *           application/json:
+ *             schema:
+ *               $ref: '#/components/schemas/applicationOutput'
+ *             examples:
+ *               application:
+ *                 $ref: '#/components/examples/application'
+ */
+write.push(
+  new Endpoint("put", "/applications/:appId", controller.update).addMiddleware(
+    applicationValidator()
+  )
+)
+
+/**
+ * @openapi
+ * /applications/{appId}:
+ *   delete:
+ *     summary: Delete an application
+ *     tags:
+ *       - applications
+ *     parameters:
+ *       - $ref: '#/components/parameters/appIdUrl'
+ *     responses:
+ *       200:
+ *         description: Returns the deleted application.
+ *         content:
+ *           application/json:
+ *             schema:
+ *               $ref: '#/components/schemas/applicationOutput'
+ *             examples:
+ *               application:
+ *                 $ref: '#/components/examples/application'
+ */
+write.push(new Endpoint("delete", "/applications/:appId", controller.destroy))
+
+/**
+ * @openapi
+ * /applications/{appId}:
+ *   get:
+ *     summary: Retrieve an application
+ *     tags:
+ *       - applications
+ *     parameters:
+ *       - $ref: '#/components/parameters/appIdUrl'
+ *     responses:
+ *       200:
+ *         description: Returns the retrieved application.
+ *         content:
+ *           application/json:
+ *             schema:
+ *               $ref: '#/components/schemas/applicationOutput'
+ *             examples:
+ *               application:
+ *                 $ref: '#/components/examples/application'
+ */
+read.push(new Endpoint("get", "/applications/:appId", controller.read))
+
+/**
+ * @openapi
+ * /applications/search:
+ *   post:
+ *     summary: Search for applications
+ *     description: Based on application properties (currently only name) search for applications.
+ *     tags:
+ *       - applications
+ *     parameters:
+ *       - $ref: '#/components/parameters/appId'
+ *     requestBody:
+ *       required: true
+ *       content:
+ *         application/json:
+ *           schema:
+ *             $ref: '#/components/schemas/nameSearch'
+ *     responses:
+ *       200:
+ *         description: Returns the applications that were found based on the search parameters.
+ *         content:
+ *           application/json:
+ *             schema:
+ *               type: object
+ *               required:
+ *                 - data
+ *               properties:
+ *                 data:
+ *                   type: array
+ *                   items:
+ *                     $ref: '#/components/schemas/application'
+ *             examples:
+ *               applications:
+ *                 $ref: '#/components/examples/applications'
+ */
+read.push(
+  new Endpoint("post", "/applications/search", controller.search).addMiddleware(
+    nameValidator()
+  )
+)
+
+export default { read, write }

packages/server/src/api/routes/public/index.ts

@@ -0,0 +1,82 @@
+import appEndpoints from "./applications"
+import queryEndpoints from "./queries"
+import tableEndpoints from "./tables"
+import rowEndpoints from "./rows"
+import userEndpoints from "./users"
+import usage from "../../../middleware/usageQuota"
+import authorized from "../../../middleware/authorized"
+import { paramResource, paramSubResource } from "../../../middleware/resourceId"
+import { CtxFn } from "./utils/Endpoint"
+import mapperMiddleware from "./middleware/mapper"
+const Router = require("@koa/router")
+const {
+  PermissionLevels,
+  PermissionTypes,
+} = require("@budibase/backend-core/permissions")
+
+const PREFIX = "/api/public/v1"
+
+const publicRouter = new Router({
+  prefix: PREFIX,
+})
+
+function addMiddleware(
+  endpoints: any,
+  middleware: CtxFn,
+  opts: { output: boolean } = { output: false }
+) {
+  if (!endpoints) {
+    return
+  }
+  if (!Array.isArray(endpoints)) {
+    endpoints = [endpoints]
+  }
+  for (let endpoint of endpoints) {
+    if (opts?.output) {
+      endpoint.addOutputMiddleware(middleware)
+    } else {
+      endpoint.addMiddleware(middleware)
+    }
+  }
+}
+
+function addToRouter(endpoints: any) {
+  if (endpoints) {
+    for (let endpoint of endpoints) {
+      endpoint.apply(publicRouter)
+    }
+  }
+}
+
+function applyRoutes(
+  endpoints: any,
+  permType: string,
+  resource: string,
+  subResource?: string
+) {
+  const paramMiddleware = subResource
+    ? paramSubResource(resource, subResource)
+    : paramResource(resource)
+  // add the parameter capture middleware
+  addMiddleware(endpoints.read, paramMiddleware)
+  addMiddleware(endpoints.write, paramMiddleware)
+  // add the authorization middleware, using the correct perm type
+  addMiddleware(endpoints.read, authorized(permType, PermissionLevels.READ))
+  addMiddleware(endpoints.write, authorized(permType, PermissionLevels.WRITE))
+  // add the usage quota middleware
+  addMiddleware(endpoints.write, usage)
+  // add the output mapper middleware
+  addMiddleware(endpoints.read, mapperMiddleware, { output: true })
+  addMiddleware(endpoints.write, mapperMiddleware, { output: true })
+  addToRouter(endpoints.read)
+  addToRouter(endpoints.write)
+}
+
+applyRoutes(appEndpoints, PermissionTypes.APP, "appId")
+applyRoutes(tableEndpoints, PermissionTypes.TABLE, "tableId")
+applyRoutes(userEndpoints, PermissionTypes.USER, "userId")
+applyRoutes(queryEndpoints, PermissionTypes.QUERY, "queryId")
+// needs to be applied last for routing purposes, don't override other endpoints
+applyRoutes(rowEndpoints, PermissionTypes.TABLE, "tableId", "rowId")
+
+module.exports = publicRouter

packages/server/src/api/routes/public/middleware/mapper.ts

@@ -0,0 +1,81 @@
+import mapping from "../../../controllers/public/mapping"
+
+enum Resources {
+  APPLICATION = "applications",
+  TABLES = "tables",
+  ROWS = "rows",
+  USERS = "users",
+  QUERIES = "queries",
+  SEARCH = "search",
+}
+
+function isArrayResponse(ctx: any) {
+  return ctx.url.endsWith(Resources.SEARCH) || Array.isArray(ctx.body)
+}
+
+function processApplications(ctx: any) {
+  if (isArrayResponse(ctx)) {
+    return mapping.mapApplications(ctx)
+  } else {
+    return mapping.mapApplication(ctx)
+  }
+}
+
+function processTables(ctx: any) {
+  if (isArrayResponse(ctx)) {
+    return mapping.mapTables(ctx)
+  } else {
+    return mapping.mapTable(ctx)
+  }
+}
+
+function processRows(ctx: any) {
+  if (isArrayResponse(ctx)) {
+    return mapping.mapRowSearch(ctx)
+  } else {
+    return mapping.mapRow(ctx)
+  }
+}
+
+function processUsers(ctx: any) {
+  if (isArrayResponse(ctx)) {
+    return mapping.mapUsers(ctx)
+  } else {
+    return mapping.mapUser(ctx)
+  }
+}
+
+function processQueries(ctx: any) {
+  if (isArrayResponse(ctx)) {
+    return mapping.mapQueries(ctx)
+  } else {
+    return mapping.mapQueryExecution(ctx)
+  }
+}
+
+export default async (ctx: any, next: any) => {
+  let urlParts = ctx.url.split("/")
+  urlParts = urlParts.slice(4, urlParts.length)
+  let body = {}
+  switch (urlParts[0]) {
+    case Resources.APPLICATION:
+      body = processApplications(ctx)
+      break
+    case Resources.TABLES:
+      if (urlParts[2] === Resources.ROWS) {
+        body = processRows(ctx)
+      } else {
+        body = processTables(ctx)
+      }
+      break
+    case Resources.USERS:
+      body = processUsers(ctx)
+      break
+    case Resources.QUERIES:
+      body = processQueries(ctx)
+      break
+  }
+  // update the body based on what has occurred in the mapper
+  ctx.body = body
+  await next()
+}

packages/server/src/api/routes/public/queries.ts

@@ -0,0 +1,81 @@
+import controller from "../../controllers/public/queries"
+import Endpoint from "./utils/Endpoint"
+import { nameValidator } from "../utils/validators"
+
+const read = [],
+  write = []
+
+/**
+ * @openapi
+ * /queries/{queryId}:
+ *   post:
+ *     summary: Execute a query
+ *     description: Queries which have been created within a Budibase app can be executed using this,
+ *     tags:
+ *       - queries
+ *     parameters:
+ *       - $ref: '#/components/parameters/queryId'
+ *       - $ref: '#/components/parameters/appId'
+ *     requestBody:
+ *       required: true
+ *       content:
+ *         application/json:
+ *           schema:
+ *             $ref: '#/components/schemas/executeQuery'
+ *     responses:
+ *       200:
+ *         description: Returns the result of the query execution.
+ *         content:
+ *           application/json:
+ *             schema:
+ *               $ref: '#/components/schemas/executeQueryOutput'
+ *             examples:
+ *               REST:
+ *                 $ref: '#/components/examples/restResponse'
+ *               SQL:
+ *                 $ref: '#/components/examples/sqlResponse'
+ *
+ */
+write.push(new Endpoint("post", "/queries/:queryId", controller.execute))
+
+/**
+ * @openapi
+ * /queries/search:
+ *   post:
+ *     summary: Search for queries
+ *     description: Based on query properties (currently only name) search for queries.
+ *     tags:
+ *       - queries
+ *     parameters:
+ *       - $ref: '#/components/parameters/appId'
+ *     requestBody:
+ *       required: true
+ *       content:
+ *         application/json:
+ *           schema:
+ *             $ref: '#/components/schemas/nameSearch'
+ *     responses:
+ *       200:
+ *         description: Returns the queries found based on the search parameters.
+ *         content:
+ *           application/json:
+ *             schema:
+ *               type: object
+ *               required:
+ *                 - data
+ *               properties:
+ *                 data:
+ *                   type: array
+ *                   items:
+ *                     $ref: '#/components/schemas/query'
+ *             examples:
+ *               queries:
+ *                 $ref: '#/components/examples/queries'
+ */
+read.push(
+  new Endpoint("post", "/queries/search", controller.search).addMiddleware(
+    nameValidator()
+  )
+)
+
+export default { read, write }

packages/server/src/api/routes/public/rows.ts

@@ -0,0 +1,242 @@
+import controller from "../../controllers/public/rows"
+import Endpoint from "./utils/Endpoint"
+import { externalSearchValidator } from "../utils/validators"
+
+const read = [],
+  write = []
+
+/**
+ * @openapi
+ * /tables/{tableId}/rows:
+ *   post:
+ *     summary: Create a row
+ *     description: Creates a row within the specified table.
+ *     tags:
+ *       - rows
+ *     parameters:
+ *       - $ref: '#/components/parameters/tableId'
+ *       - $ref: '#/components/parameters/appId'
+ *     requestBody:
+ *       required: true
+ *       content:
+ *         application/json:
+ *           schema:
+ *             $ref: '#/components/schemas/row'
+ *           examples:
+ *             row:
+ *               $ref: '#/components/examples/inputRow'
+ *     responses:
+ *       200:
+ *         description: Returns the created row, including the ID which has been generated for it.
+ *           This can be found in the Budibase portal, viewed under the developer information.
+ *         content:
+ *           application/json:
+ *             schema:
+ *               $ref: '#/components/schemas/rowOutput'
+ *             examples:
+ *               row:
+ *                 $ref: '#/components/examples/row'
+ */
+write.push(new Endpoint("post", "/tables/:tableId/rows", controller.create))
+
+/**
+ * @openapi
+ * /tables/{tableId}/rows/{rowId}:
+ *   put:
+ *     summary: Update a row
+ *     description: Updates a row within the specified table.
+ *     tags:
+ *       - rows
+ *     parameters:
+ *       - $ref: '#/components/parameters/tableId'
+ *       - $ref: '#/components/parameters/rowId'
+ *       - $ref: '#/components/parameters/appId'
+ *     requestBody:
+ *       required: true
+ *       content:
+ *         application/json:
+ *           schema:
+ *             $ref: '#/components/schemas/row'
+ *           examples:
+ *             row:
+ *               $ref: '#/components/examples/inputRow'
+ *     responses:
+ *       200:
+ *         description: Returns the created row, including the ID which has been generated for it.
+ *         content:
+ *           application/json:
+ *             schema:
+ *               $ref: '#/components/schemas/rowOutput'
+ *             examples:
+ *               row:
+ *                 $ref: '#/components/examples/row'
+ */
+write.push(
+  new Endpoint("put", "/tables/:tableId/rows/:rowId", controller.update)
+)
+
+/**
+ * @openapi
+ * /tables/{tableId}/rows/{rowId}:
+ *   delete:
+ *     summary: Delete a row
+ *     description: Deletes a row within the specified table.
+ *     tags:
+ *       - rows
+ *     parameters:
+ *       - $ref: '#/components/parameters/tableId'
+ *       - $ref: '#/components/parameters/rowId'
+ *       - $ref: '#/components/parameters/appId'
+ *     responses:
+ *       200:
+ *         description: Returns the deleted row, including the ID which has been generated for it.
+ *         content:
+ *           application/json:
+ *             schema:
+ *               $ref: '#/components/schemas/rowOutput'
+ *             examples:
+ *               row:
+ *                 $ref: '#/components/examples/row'
+ */
+write.push(
+  new Endpoint("delete", "/tables/:tableId/rows/:rowId", controller.destroy)
+)
+
+/**
+ * @openapi
+ * /tables/{tableId}/rows/{rowId}:
+ *   get:
+ *     summary: Retrieve a row
+ *     description: This gets a single row, it will be enriched with the full related rows, rather than
+ *       the squashed "primaryDisplay" format returned by the search endpoint.
+ *     tags:
+ *       - rows
+ *     parameters:
+ *       - $ref: '#/components/parameters/tableId'
+ *       - $ref: '#/components/parameters/rowId'
+ *       - $ref: '#/components/parameters/appId'
+ *     responses:
+ *       200:
+ *         description: Returns the retrieved row.
+ *         content:
+ *           application/json:
+ *             schema:
+ *               $ref: '#/components/schemas/rowOutput'
+ *             examples:
+ *               enrichedRow:
+ *                 $ref: '#/components/examples/enrichedRow'
+ */
+read.push(new Endpoint("get", "/tables/:tableId/rows/:rowId", controller.read))
+
+/**
+ * @openapi
+ * /tables/{tableId}/rows/search:
+ *   post:
+ *     summary: Search for rows
+ *     tags:
+ *       - rows
+ *     parameters:
+ *       - $ref: '#/components/parameters/tableId'
+ *       - $ref: '#/components/parameters/appId'
+ *     requestBody:
+ *       required: true
+ *       content:
+ *         application/json:
+ *           schema:
+ *             type: object
+ *             required:
+ *               - query
+ *             properties:
+ *               query:
+ *                 type: object
+ *                 properties:
+ *                   string:
+ *                     type: object
+ *                     example:
+ *                       columnName1: value
+ *                       columnName2: value
+ *                     description: A map of field name to the string to search for,
+ *                       this will look for rows that have a value starting with the
+ *                       string value.
+ *                     additionalProperties:
+ *                       type: string
+ *                       description: The value to search for in the column.
+ *                   fuzzy:
+ *                     type: object
+ *                     description: A fuzzy search, only supported by internal tables.
+ *                   range:
+ *                     type: object
+ *                     description: Searches within a range, the format of this must be
+ *                       columnName -> [low, high].
+ *                     example:
+ *                       columnName1: [10, 20]
+ *                   equal:
+ *                     type: object
+ *                     description: Searches for rows that have a column value that is
+ *                       exactly the value set.
+ *                   notEqual:
+ *                     type: object
+ *                     description: Searches for any row which does not contain the specified
+ *                       column value.
+ *                   empty:
+ *                     type: object
+ *                     description: Searches for rows which do not contain the specified column.
+ *                       The object should simply contain keys of the column names, these
+ *                       can map to any value.
+ *                     example:
+ *                       columnName1: ""
+ *                   notEmpty:
+ *                     type: object
+ *                     description: Searches for rows which have the specified column.
+ *                   oneOf:
+ *                     type: object
+ *                     description: Searches for rows which have a column value that is any
+ *                       of the specified values. The format of this must be columnName -> [value1, value2].
+ *               paginate:
+ *                 type: boolean
+ *                 description: Enables pagination, by default this is disabled.
+ *               bookmark:
+ *                 oneOf:
+ *                   - type: string
+ *                   - type: integer
+ *                 description: If retrieving another page, the bookmark from the previous request must be supplied.
+ *               limit:
+ *                 type: integer
+ *                 description: The maximum number of rows to return, useful when paginating, for internal tables this
+ *                   will be limited to 1000, for SQL tables it will be 5000.
+ *               sort:
+ *                 type: object
+ *                 description: A set of parameters describing the sort behaviour of the search.
+ *                 properties:
+ *                   order:
+ *                     type: string
+ *                     enum: [ascending, descending]
+ *                     description: The order of the sort, by default this is ascending.
+ *                   column:
+ *                     type: string
+ *                     description: The name of the column by which the rows will be sorted.
+ *                   type:
+ *                     type: string
+ *                     enum: [string, number]
+ *                     description: Defines whether the column should be treated as a string
+ *                       or as numbers when sorting.
+ *     responses:
+ *       200:
+ *         description: The response will contain an array of rows that match the search parameters.
+ *         content:
+ *           application/json:
+ *             schema:
+ *               $ref: '#/components/schemas/searchOutput'
+ *             examples:
+ *               search:
+ *                 $ref: '#/components/examples/rows'
+ */
+read.push(
+  new Endpoint(
+    "post",
+    "/tables/:tableId/rows/search",
+    controller.search
+  ).addMiddleware(externalSearchValidator())
+)
+
+export default { read, write }

packages/server/src/api/routes/public/tables.ts

@@ -0,0 +1,169 @@
+import controller from "../../controllers/public/tables"
+import Endpoint from "./utils/Endpoint"
+import { tableValidator, nameValidator } from "../utils/validators"
+
+const read = [],
+  write = []
+
+/**
+ * @openapi
+ * /tables:
+ *   post:
+ *     summary: Create a table
+ *     description: Create a table, this could be internal or external.
+ *     tags:
+ *       - tables
+ *     parameters:
+ *       - $ref: '#/components/parameters/appId'
+ *     requestBody:
+ *       content:
+ *         application/json:
+ *           schema:
+ *             $ref: '#/components/schemas/table'
+ *           examples:
+ *             table:
+ *               $ref: '#/components/examples/table'
+ *     responses:
+ *       200:
+ *         description: Returns the created table, including the ID which has been generated for it. This can be
+ *           internal or external data sources.
+ *         content:
+ *           application/json:
+ *             schema:
+ *               $ref: '#/components/schemas/tableOutput'
+ *             examples:
+ *               table:
+ *                 $ref: '#/components/examples/table'
+ */
+write.push(
+  new Endpoint("post", "/tables", controller.create).addMiddleware(
+    tableValidator()
+  )
+)
+
+/**
+ * @openapi
+ * /tables/{tableId}:
+ *   put:
+ *     summary: Update a table
+ *     description: Update a table, this could be internal or external.
+ *     tags:
+ *       - tables
+ *     parameters:
+ *       - $ref: '#/components/parameters/tableId'
+ *       - $ref: '#/components/parameters/appId'
+ *     requestBody:
+ *       content:
+ *         application/json:
+ *           schema:
+ *             $ref: '#/components/schemas/table'
+ *           examples:
+ *             table:
+ *               $ref: '#/components/examples/table'
+ *     responses:
+ *       200:
+ *         description: Returns the updated table.
+ *         content:
+ *           application/json:
+ *             schema:
+ *               $ref: '#/components/schemas/tableOutput'
+ *             examples:
+ *               table:
+ *                 $ref: '#/components/examples/table'
+ */
+write.push(
+  new Endpoint("put", "/tables/:tableId", controller.update).addMiddleware(
+    tableValidator()
+  )
+)
+
+/**
+ * @openapi
+ * /tables/{tableId}:
+ *   delete:
+ *     summary: Delete a table
+ *     description: Delete a table, this could be internal or external.
+ *     tags:
+ *       - tables
+ *     parameters:
+ *       - $ref: '#/components/parameters/tableId'
+ *       - $ref: '#/components/parameters/appId'
+ *     responses:
+ *       200:
+ *         description: Returns the deleted table.
+ *         content:
+ *           application/json:
+ *             schema:
+ *               $ref: '#/components/schemas/tableOutput'
+ *             examples:
+ *               table:
+ *                 $ref: '#/components/examples/table'
+ */
+write.push(new Endpoint("delete", "/tables/:tableId", controller.destroy))
+
+/**
+ * @openapi
+ * /tables/{tableId}:
+ *   get:
+ *     summary: Retrieve a table
+ *     description: Lookup a table, this could be internal or external.
+ *     tags:
+ *       - tables
+ *     parameters:
+ *       - $ref: '#/components/parameters/tableId'
+ *       - $ref: '#/components/parameters/appId'
+ *     responses:
+ *       200:
+ *         description: Returns the retrieved table.
+ *         content:
+ *           application/json:
+ *             schema:
+ *               $ref: '#/components/schemas/tableOutput'
+ *             examples:
+ *               table:
+ *                 $ref: '#/components/examples/table'
+ */
+read.push(new Endpoint("get", "/tables/:tableId", controller.read))
+
+/**
+ * @openapi
+ * /tables/search:
+ *   post:
+ *     summary: Search for tables
+ *     description: Based on table properties (currently only name) search for tables. This could be
+ *       an internal or an external table.
+ *     tags:
+ *       - tables
+ *     parameters:
+ *       - $ref: '#/components/parameters/appId'
+ *     requestBody:
+ *       required: true
+ *       content:
+ *         application/json:
+ *           schema:
+ *             $ref: '#/components/schemas/nameSearch'
+ *     responses:
+ *       200:
+ *         description: Returns the found tables, based on the search parameters.
+ *         content:
+ *           application/json:
+ *             schema:
+ *               type: object
+ *               required:
+ *                 - data
+ *               properties:
+ *                 data:
+ *                   type: array
+ *                   items:
+ *                     $ref: '#/components/schemas/table'
+ *             examples:
+ *               tables:
+ *                 $ref: '#/components/examples/tables'
+ */
+read.push(
+  new Endpoint("post", "/tables/search", controller.search).addMiddleware(
+    nameValidator()
+  )
+)
+
+export default { read, write }

packages/server/src/api/routes/public/tests/compare.spec.js

@@ -0,0 +1,184 @@
+const jestOpenAPI = require("jest-openapi").default
+const generateSchema = require("../../../../../specs/generate")
+const setup = require("../../tests/utilities")
+const { checkSlashesInUrl } = require("../../../../utilities")
+
+const yamlPath = generateSchema()
+jestOpenAPI(yamlPath)
+
+let request = setup.getRequest()
+let config = setup.getConfig()
+let apiKey, table, app
+
+beforeAll(async () => {
+  app = await config.init()
+  table = await config.updateTable()
+  apiKey = await config.generateApiKey()
+})
+
+afterAll(setup.afterAll)
+
+async function makeRequest(method, endpoint, body, appId = config.getAppId()) {
+  const extraHeaders = {
+    "x-budibase-api-key": apiKey,
+  }
+  if (appId) {
+    extraHeaders["x-budibase-app-id"] = appId
+  }
+  const req = request
+    [method](checkSlashesInUrl(`/api/public/v1/${endpoint}`))
+    .set(config.defaultHeaders(extraHeaders))
+  if (body) {
+    req.send(body)
+  }
+  const res = await req.expect("Content-Type", /json/).expect(200)
+  expect(res.body).toBeDefined()
+  return res
+}
+
+describe("check the applications endpoints", () => {
+  it("should allow retrieving applications through search", async () => {
+    const res = await makeRequest("post", "/applications/search")
+    expect(res).toSatisfyApiSpec()
+  })
+
+  it("should allow creating an application", async () => {
+    const res = await makeRequest("post", "/applications", {
+      name: "new App"
+    }, null)
+    expect(res).toSatisfyApiSpec()
+  })
+
+  it("should allow updating an application", async () => {
+    const app = config.getApp()
+    const appId = config.getAppId()
+    const res = await makeRequest("put", `/applications/${appId}`, {
+      ...app,
+      name: "updated app name",
+    }, appId)
+    expect(res).toSatisfyApiSpec()
+  })
+
+  it("should allow retrieving an application", async () => {
+    const res = await makeRequest("get", `/applications/${config.getAppId()}`)
+    expect(res).toSatisfyApiSpec()
+  })
+
+  it("should allow deleting an application", async () => {
+    const res = await makeRequest("delete", `/applications/${config.getAppId()}`)
+    expect(res).toSatisfyApiSpec()
+  })
+})
+
+describe("check the tables endpoints", () => {
+  it("should allow retrieving tables through search", async () => {
+    await config.createApp("new app 1")
+    table = await config.updateTable()
+    const res = await makeRequest("post", "/tables/search")
+    expect(res).toSatisfyApiSpec()
+  })
+
+  it("should allow creating a table", async () => {
+    const res = await makeRequest("post", "/tables", {
+      name: "table name",
+      primaryDisplay: "column1",
+      schema: {
+        column1: {
+          type: "string",
+          constraints: {},
+        }
+      }
+    })
+    expect(res).toSatisfyApiSpec()
+  })
+
+  it("should allow updating a table", async () => {
+    const updated = { ...table, _rev: undefined, name: "new name" }
+    const res = await makeRequest("put", `/tables/${table._id}`, updated)
+    expect(res).toSatisfyApiSpec()
+  })
+
+  it("should allow retrieving a table", async () => {
+    const res = await makeRequest("get", `/tables/${table._id}`)
+    expect(res).toSatisfyApiSpec()
+  })
+
+  it("should allow deleting a table", async () => {
+    const res = await makeRequest("delete", `/tables/${table._id}`)
+    expect(res).toSatisfyApiSpec()
+  })
+})
+
+describe("check the rows endpoints", () => {
+  let row
+  it("should allow retrieving rows through search", async () => {
+    table = await config.updateTable()
+    const res = await makeRequest("post", `/tables/${table._id}/rows/search`, {
+      query: {
+      },
+    })
+    expect(res).toSatisfyApiSpec()
+  })
+
+  it("should allow creating a row", async () => {
+    const res = await makeRequest("post", `/tables/${table._id}/rows`, {
+      name: "test row",
+    })
+    expect(res).toSatisfyApiSpec()
+    row = res.body.data
+  })
+
+  it("should allow updating a row", async () => {
+    const res = await makeRequest("put", `/tables/${table._id}/rows/${row._id}`, {
+      name: "test row updated",
+    })
+    expect(res).toSatisfyApiSpec()
+  })
+
+  it("should allow retrieving a row", async () => {
+    const res = await makeRequest("get", `/tables/${table._id}/rows/${row._id}`)
+    expect(res).toSatisfyApiSpec()
+  })
+
+  it("should allow deleting a row", async () => {
+    const res = await makeRequest("delete", `/tables/${table._id}/rows/${row._id}`)
+    expect(res).toSatisfyApiSpec()
+  })
+})
+
+describe("check the users endpoints", () => {
+  let user
+  it("should allow retrieving users through search", async () => {
+    user = await config.createUser()
+    const res = await makeRequest("post", "/users/search")
+    expect(res).toSatisfyApiSpec()
+  })
+
+  it("should allow creating a user", async () => {
+    const res = await makeRequest("post", "/users")
+    expect(res).toSatisfyApiSpec()
+  })
+
+  it("should allow updating a user", async () => {
+    const res = await makeRequest("put", `/users/${user._id}`)
+    expect(res).toSatisfyApiSpec()
+  })
+
+  it("should allow retrieving a user", async () => {
+    const res = await makeRequest("get", `/users/${user._id}`)
+    expect(res).toSatisfyApiSpec()
+  })
+
+  it("should allow deleting a user", async () => {
+    const res = await makeRequest("delete", `/users/${user._id}`)
+    expect(res).toSatisfyApiSpec()
+  })
+})
+
+describe("check the queries endpoints", () => {
+  it("should allow retrieving queries through search", async () => {
+    const res = await makeRequest("post", "/queries/search")
+    expect(res).toSatisfyApiSpec()
+  })
+})
+

packages/server/src/api/routes/public/users.ts

@@ -0,0 +1,144 @@
+import controller from "../../controllers/public/users"
+import Endpoint from "./utils/Endpoint"
+import { nameValidator } from "../utils/validators"
+
+const read = [],
+  write = []
+
+/**
+ * @openapi
+ * /users:
+ *   post:
+ *     summary: Create a user
+ *     tags:
+ *       - users
+ *     requestBody:
+ *       required: true
+ *       content:
+ *         application/json:
+ *           schema:
+ *             $ref: '#/components/schemas/user'
+ *     responses:
+ *       200:
+ *         description: Returns the created user.
+ *         content:
+ *           application/json:
+ *             schema:
+ *               $ref: '#/components/schemas/userOutput'
+ *             examples:
+ *               user:
+ *                 $ref: '#/components/examples/user'
+ */
+write.push(new Endpoint("post", "/users", controller.create))
+
+/**
+ * @openapi
+ * /users/{userId}:
+ *   put:
+ *     summary: Update a user
+ *     tags:
+ *       - users
+ *     parameters:
+ *       - $ref: '#/components/parameters/userId'
+ *     requestBody:
+ *       required: true
+ *       content:
+ *         application/json:
+ *           schema:
+ *             $ref: '#/components/schemas/user'
+ *     responses:
+ *       200:
+ *         description: Returns the updated user.
+ *         content:
+ *           application/json:
+ *             schema:
+ *               $ref: '#/components/schemas/userOutput'
+ *             examples:
+ *               user:
+ *                 $ref: '#/components/examples/user'
+ */
+write.push(new Endpoint("put", "/users/:userId", controller.update))
+
+/**
+ * @openapi
+ * /users/{userId}:
+ *   delete:
+ *     summary: Delete a user
+ *     tags:
+ *       - users
+ *     parameters:
+ *       - $ref: '#/components/parameters/userId'
+ *     responses:
+ *       200:
+ *         description: Returns the deleted user.
+ *         content:
+ *           application/json:
+ *             schema:
+ *               $ref: '#/components/schemas/userOutput'
+ *             examples:
+ *               user:
+ *                 $ref: '#/components/examples/user'
+ */
+write.push(new Endpoint("delete", "/users/:userId", controller.destroy))
+
+/**
+ * @openapi
+ * /users/{userId}:
+ *   get:
+ *     summary: Retrieve a user
+ *     tags:
+ *       - users
+ *     parameters:
+ *       - $ref: '#/components/parameters/userId'
+ *     responses:
+ *       200:
+ *         description: Returns the retrieved user.
+ *         content:
+ *           application/json:
+ *             schema:
+ *               $ref: '#/components/schemas/userOutput'
+ *             examples:
+ *               user:
+ *                 $ref: '#/components/examples/user'
+ */
+read.push(new Endpoint("get", "/users/:userId", controller.read))
+
+/**
+ * @openapi
+ * /users/search:
+ *   post:
+ *     summary: Search for users
+ *     description: Based on user properties (currently only name) search for users.
+ *     tags:
+ *       - users
+ *     requestBody:
+ *       required: true
+ *       content:
+ *         application/json:
+ *           schema:
+ *             $ref: '#/components/schemas/nameSearch'
+ *     responses:
+ *       200:
+ *         description: Returns the found users based on search parameters.
+ *         content:
+ *           application/json:
+ *             schema:
+ *               type: object
+ *               required:
+ *                 - data
+ *               properties:
+ *                 data:
+ *                   type: array
+ *                   items:
+ *                     $ref: '#/components/schemas/user'
+ *             examples:
+ *               users:
+ *                 $ref: '#/components/examples/users'
+ */
+read.push(
+  new Endpoint("post", "/users/search", controller.search).addMiddleware(
+    nameValidator()
+  )
+)
+
+export default { read, write }

packages/server/src/api/routes/public/utils/Endpoint.ts

@@ -0,0 +1,51 @@
+import Router from "koa-router"
+
+export type CtxFn = (ctx: any, next?: any) => void | Promise<any>
+
+class Endpoint {
+  method: string
+  url: string
+  controller: CtxFn
+  middlewares: CtxFn[]
+  outputMiddlewares: CtxFn[]
+
+  constructor(method: string, url: string, controller: CtxFn) {
+    this.method = method
+    this.url = url
+    this.controller = controller
+    this.middlewares = []
+    this.outputMiddlewares = []
+  }
+
+  addMiddleware(middleware: CtxFn) {
+    this.middlewares.push(middleware)
+    return this
+  }
+
+  addOutputMiddleware(middleware: CtxFn) {
+    this.outputMiddlewares.push(middleware)
+    return this
+  }
+
+  apply(router: Router) {
+    const method = this.method,
+      url = this.url
+    const middlewares = this.middlewares,
+      controller = this.controller,
+      outputMiddlewares = this.outputMiddlewares
+    // need a function to do nothing to stop the execution at the end
+    // middlewares are circular so if they always keep calling next, it'll just keep looping
+    const complete = () => {}
+    const params = [
+      url,
+      ...middlewares,
+      controller,
+      ...outputMiddlewares,
+      complete,
+    ]
+    // @ts-ignore
+    router[method](...params)
+  }
+}
+
+export default Endpoint

packages/server/src/api/routes/role.js

@@ -1,34 +1,13 @@
 const Router = require("@koa/router")
 const controller = require("../controllers/role")
 const authorized = require("../../middleware/authorized")
-const Joi = require("joi")
+const { BUILDER } = require("@budibase/backend-core/permissions")
-const joiValidator = require("../../middleware/joi-validator")
+const { roleValidator } = require("./utils/validators")
-const {
-  BUILTIN_PERMISSION_IDS,
-  BUILDER,
-  PermissionLevels,
-} = require("@budibase/backend-core/permissions")
 
 const router = Router()
 
-function generateValidator() {
-  const permLevelArray = Object.values(PermissionLevels)
-  // prettier-ignore
-  return joiValidator.body(Joi.object({
-    _id: Joi.string().optional(),
-    _rev: Joi.string().optional(),
-    name: Joi.string().required(),
-    // this is the base permission ID (for now a built in)
-    permissionId: Joi.string().valid(...Object.values(BUILTIN_PERMISSION_IDS)).required(),
-    permissions: Joi.object()
-      .pattern(/.*/, [Joi.string().valid(...permLevelArray)])
-      .optional(),
-    inherits: Joi.string().optional(),
-  }).unknown(true))
-}
-
 router
-  .post("/api/roles", authorized(BUILDER), generateValidator(), controller.save)
+  .post("/api/roles", authorized(BUILDER), roleValidator(), controller.save)
   .get("/api/roles", authorized(BUILDER), controller.fetch)
   .get("/api/roles/:roleId", authorized(BUILDER), controller.find)
   .delete("/api/roles/:roleId/:rev", authorized(BUILDER), controller.destroy)

packages/server/src/api/routes/row.js

@@ -10,6 +10,7 @@ const {
   PermissionLevels,
   PermissionTypes,
 } = require("@budibase/backend-core/permissions")
+const { internalSearchValidator } = require("./utils/validators")
 
 const router = Router()
 
@@ -119,8 +120,6 @@ router
    *    "notEqual": {},
    *    "empty": {},
    *    "notEmpty": {},
-   *    "contains": {},
-   *    "notContains": {}
    *    "oneOf": {
    *      "columnName": ["value"]
    *    }
@@ -140,6 +139,7 @@ router
    */
   .post(
     "/api/:tableId/search",
+    internalSearchValidator(),
     paramResource("tableId"),
     authorized(PermissionTypes.TABLE, PermissionLevels.READ),
     rowController.search
@@ -195,6 +195,7 @@ router
     "/api/:tableId/rows",
     paramResource("tableId"),
     authorized(PermissionTypes.TABLE, PermissionLevels.WRITE),
+    usage,
     rowController.patch
   )
   /**

packages/server/src/api/routes/screen.js

@@ -2,40 +2,13 @@ const Router = require("@koa/router")
 const controller = require("../controllers/screen")
 const authorized = require("../../middleware/authorized")
 const { BUILDER } = require("@budibase/backend-core/permissions")
-const joiValidator = require("../../middleware/joi-validator")
+const { screenValidator } = require("./utils/validators")
-const Joi = require("joi")
 
 const router = Router()
 
-function generateSaveValidation() {
-  // prettier-ignore
-  return joiValidator.body(Joi.object({
-    name: Joi.string().required(),
-    routing: Joi.object({
-      route: Joi.string().required(),
-      roleId: Joi.string().required().allow(""),
-    }).required().unknown(true),
-    props: Joi.object({
-      _id: Joi.string().required(),
-      _component: Joi.string().required(),
-      _children: Joi.array().required(),
-      _instanceName: Joi.string().required(),
-      _styles: Joi.object().required(),
-      type: Joi.string().optional(),
-      table: Joi.string().optional(),
-      layoutId: Joi.string().optional(),
-    }).required().unknown(true),
-  }).unknown(true))
-}
-
 router
   .get("/api/screens", authorized(BUILDER), controller.fetch)
-  .post(
+  .post("/api/screens", authorized(BUILDER), screenValidator(), controller.save)
-    "/api/screens",
-    authorized(BUILDER),
-    generateSaveValidation(),
-    controller.save
-  )
   .delete(
     "/api/screens/:screenId/:screenRev",
     authorized(BUILDER),

packages/server/src/api/routes/table.js

@@ -7,25 +7,10 @@ const {
   PermissionLevels,
   PermissionTypes,
 } = require("@budibase/backend-core/permissions")
-const joiValidator = require("../../middleware/joi-validator")
+const { tableValidator } = require("./utils/validators")
-const Joi = require("joi")
 
 const router = Router()
 
-function generateSaveValidator() {
-  // prettier-ignore
-  return joiValidator.body(Joi.object({
-    _id: Joi.string(),
-    _rev: Joi.string(),
-    type: Joi.string().valid("table", "internal", "external"),
-    primaryDisplay: Joi.string(),
-    schema: Joi.object().required(),
-    name: Joi.string().required(),
-    views: Joi.object(),
-    dataImport: Joi.object(),
-  }).unknown(true))
-}
-
 router
   /**
    * @api {get} /api/tables Fetch all tables
@@ -53,8 +38,8 @@ router
    * @apiSuccess {object[]} body The response body will be the table that was found.
    */
   .get(
-    "/api/tables/:id",
+    "/api/tables/:tableId",
-    paramResource("id"),
+    paramResource("tableId"),
     authorized(PermissionTypes.TABLE, PermissionLevels.READ),
     tableController.find
   )
@@ -136,7 +121,7 @@ router
     // allows control over updating a table
     bodyResource("_id"),
     authorized(BUILDER),
-    generateSaveValidator(),
+    tableValidator(),
     tableController.save
   )
   /**

packages/server/src/api/routes/tests/utilities/index.js

@@ -2,6 +2,18 @@ const TestConfig = require("../../../../tests/utilities/TestConfiguration")
 const structures = require("../../../../tests/utilities/structures")
 const env = require("../../../../environment")
 
+function user() {
+  return {
+    _id: "user",
+    _rev: "rev",
+    createdAt: Date.now(),
+    email: "test@test.com",
+    roles: {},
+    tenantId: "default",
+    status: "active",
+  }
+}
+
 jest.mock("../../../../utilities/workerRequests", () => ({
   getGlobalUsers: jest.fn(() => {
     return {
@@ -13,6 +25,18 @@ jest.mock("../../../../utilities/workerRequests", () => ({
       _id: "us_uuid1",
     }
   }),
+  allGlobalUsers: jest.fn(() => {
+    return [user()]
+  }),
+  readGlobalUser: jest.fn(() => {
+    return user()
+  }),
+  saveGlobalUser: jest.fn(() => {
+    return { _id: "user", _rev: "rev" }
+  }),
+  deleteGlobalUser: jest.fn(() => {
+    return { message: "deleted user" }
+  }),
   removeAppFromUserRoles: jest.fn(),
 }))
 

packages/server/src/api/routes/utils/validators.js

@@ -0,0 +1,219 @@
+const joiValidator = require("../../../middleware/joi-validator")
+const { DataSourceOperation } = require("../../../constants")
+const { WebhookType } = require("../../../constants")
+const {
+  BUILTIN_PERMISSION_IDS,
+  PermissionLevels,
+} = require("@budibase/backend-core/permissions")
+const Joi = require("joi")
+
+const OPTIONAL_STRING = Joi.string().optional().allow(null).allow("")
+const OPTIONAL_NUMBER = Joi.number().optional().allow(null)
+
+exports.tableValidator = () => {
+  // prettier-ignore
+  return joiValidator.body(Joi.object({
+    _id: OPTIONAL_STRING,
+    _rev: OPTIONAL_STRING,
+    type: OPTIONAL_STRING.valid("table", "internal", "external"),
+    primaryDisplay: OPTIONAL_STRING,
+    schema: Joi.object().required(),
+    name: Joi.string().required(),
+    views: Joi.object(),
+    dataImport: Joi.object(),
+  }).unknown(true))
+}
+
+exports.nameValidator = () => {
+  // prettier-ignore
+  return joiValidator.body(Joi.object({
+    name: OPTIONAL_STRING,
+  }))
+}
+
+exports.datasourceValidator = () => {
+  // prettier-ignore
+  return joiValidator.body(Joi.object({
+    _id: Joi.string(),
+    _rev: Joi.string(),
+    type: OPTIONAL_STRING.allow("datasource_plus"),
+    relationships: Joi.array().items(Joi.object({
+      from: Joi.string().required(),
+      to: Joi.string().required(),
+      cardinality: Joi.valid("1:N", "1:1", "N:N").required()
+    })),
+  }).unknown(true))
+}
+
+function filterObject() {
+  // prettier-ignore
+  return Joi.object({
+    string: Joi.object().optional(),
+    fuzzy: Joi.object().optional(),
+    range: Joi.object().optional(),
+    equal: Joi.object().optional(),
+    notEqual: Joi.object().optional(),
+    empty: Joi.object().optional(),
+    notEmpty: Joi.object().optional(),
+    oneOf: Joi.object().optional(),
+    contains: Joi.object().optional(),
+    notContains: Joi.object().optional(),
+  }).unknown(true)
+}
+
+exports.internalSearchValidator = () => {
+  // prettier-ignore
+  return joiValidator.body(Joi.object({
+    tableId: OPTIONAL_STRING,
+    query: filterObject(),
+    limit: OPTIONAL_NUMBER,
+    sort: OPTIONAL_STRING,
+    sortOrder: OPTIONAL_STRING,
+    sortType: OPTIONAL_STRING,
+    paginate: Joi.boolean(),
+    bookmark: Joi.alternatives().try(OPTIONAL_STRING, OPTIONAL_NUMBER).optional(),
+  }))
+}
+
+exports.externalSearchValidator = () => {
+  return joiValidator.body(
+    Joi.object({
+      query: filterObject(),
+      paginate: Joi.boolean().optional(),
+      bookmark: Joi.alternatives()
+        .try(OPTIONAL_STRING, OPTIONAL_NUMBER)
+        .optional(),
+      limit: OPTIONAL_NUMBER,
+      sort: Joi.object({
+        column: Joi.string(),
+        order: OPTIONAL_STRING.valid("ascending", "descending"),
+        type: OPTIONAL_STRING.valid("string", "number"),
+      }).optional(),
+    })
+  )
+}
+
+exports.datasourceQueryValidator = () => {
+  // prettier-ignore
+  return joiValidator.body(Joi.object({
+    endpoint: Joi.object({
+      datasourceId: Joi.string().required(),
+      operation: Joi.string().required().valid(...Object.values(DataSourceOperation)),
+      entityId: Joi.string().required(),
+    }).required(),
+    resource: Joi.object({
+      fields: Joi.array().items(Joi.string()).optional(),
+    }).optional(),
+    body: Joi.object().optional(),
+    sort: Joi.object().optional(),
+    filters: filterObject().optional(),
+    paginate: Joi.object({
+      page: Joi.string().alphanum().optional(),
+      limit: Joi.number().optional(),
+    }).optional(),
+  }))
+}
+
+exports.webhookValidator = () => {
+  // prettier-ignore
+  return joiValidator.body(Joi.object({
+    live: Joi.bool(),
+    _id: OPTIONAL_STRING,
+    _rev: OPTIONAL_STRING,
+    name: Joi.string().required(),
+    bodySchema: Joi.object().optional(),
+    action: Joi.object({
+      type: Joi.string().required().valid(WebhookType.AUTOMATION),
+      target: Joi.string().required(),
+    }).required(),
+  }).unknown(true))
+}
+
+exports.roleValidator = () => {
+  const permLevelArray = Object.values(PermissionLevels)
+  // prettier-ignore
+  return joiValidator.body(Joi.object({
+    _id: OPTIONAL_STRING,
+    _rev: OPTIONAL_STRING,
+    name: Joi.string().required(),
+    // this is the base permission ID (for now a built in)
+    permissionId: Joi.string().valid(...Object.values(BUILTIN_PERMISSION_IDS)).required(),
+    permissions: Joi.object()
+      .pattern(/.*/, [Joi.string().valid(...permLevelArray)])
+      .optional(),
+    inherits: OPTIONAL_STRING,
+  }).unknown(true))
+}
+
+exports.permissionValidator = () => {
+  const permLevelArray = Object.values(PermissionLevels)
+  // prettier-ignore
+  return joiValidator.params(Joi.object({
+    level: Joi.string().valid(...permLevelArray).required(),
+    resourceId: Joi.string(),
+    roleId: Joi.string(),
+  }).unknown(true))
+}
+
+exports.screenValidator = () => {
+  // prettier-ignore
+  return joiValidator.body(Joi.object({
+    name: Joi.string().required(),
+    routing: Joi.object({
+      route: Joi.string().required(),
+      roleId: Joi.string().required().allow(""),
+    }).required().unknown(true),
+    props: Joi.object({
+      _id: Joi.string().required(),
+      _component: Joi.string().required(),
+      _children: Joi.array().required(),
+      _instanceName: Joi.string().required(),
+      _styles: Joi.object().required(),
+      type: OPTIONAL_STRING,
+      table: OPTIONAL_STRING,
+      layoutId: OPTIONAL_STRING,
+    }).required().unknown(true),
+  }).unknown(true))
+}
+
+function generateStepSchema(allowStepTypes) {
+  // prettier-ignore
+  return Joi.object({
+    stepId: Joi.string().required(),
+    id: Joi.string().required(),
+    description: Joi.string().required(),
+    name: Joi.string().required(),
+    tagline: Joi.string().required(),
+    icon: Joi.string().required(),
+    params: Joi.object(),
+    args: Joi.object(),
+    type: Joi.string().required().valid(...allowStepTypes),
+  }).unknown(true)
+}
+
+exports.automationValidator = (existing = false) => {
+  // prettier-ignore
+  return joiValidator.body(Joi.object({
+    _id: existing ? Joi.string().required() : OPTIONAL_STRING,
+    _rev: existing ? Joi.string().required() : OPTIONAL_STRING,
+    name: Joi.string().required(),
+    type: Joi.string().valid("automation").required(),
+    definition: Joi.object({
+      steps: Joi.array().required().items(generateStepSchema(["ACTION", "LOGIC"])),
+      trigger: generateStepSchema(["TRIGGER"]).allow(null),
+    }).required().unknown(true),
+  }).unknown(true))
+}
+
+exports.applicationValidator = () => {
+  // prettier-ignore
+  return joiValidator.body(Joi.object({
+    _id: OPTIONAL_STRING,
+    _rev: OPTIONAL_STRING,
+    name: Joi.string().required(),
+    url: OPTIONAL_STRING,
+    template: Joi.object({
+      templateString: OPTIONAL_STRING,
+    }).unknown(true),
+  }).unknown(true))
+}

packages/server/src/api/routes/webhook.js

@@ -1,33 +1,17 @@
 const Router = require("@koa/router")
 const controller = require("../controllers/webhook")
 const authorized = require("../../middleware/authorized")
-const joiValidator = require("../../middleware/joi-validator")
 const { BUILDER } = require("@budibase/backend-core/permissions")
-const Joi = require("joi")
+const { webhookValidator } = require("./utils/validators")
 
 const router = Router()
 
-function generateSaveValidator() {
-  // prettier-ignore
-  return joiValidator.body(Joi.object({
-    live: Joi.bool(),
-    _id: Joi.string().optional(),
-    _rev: Joi.string().optional(),
-    name: Joi.string().required(),
-    bodySchema: Joi.object().optional(),
-    action: Joi.object({
-      type: Joi.string().required().valid(controller.WebhookType.AUTOMATION),
-      target: Joi.string().required(),
-    }).required(),
-  }).unknown(true))
-}
-
 router
   .get("/api/webhooks", authorized(BUILDER), controller.fetch)
   .put(
     "/api/webhooks",
     authorized(BUILDER),
-    generateSaveValidator(),
+    webhookValidator(),
     controller.save
   )
   .delete("/api/webhooks/:id/:rev", authorized(BUILDER), controller.destroy)

packages/server/src/automations/steps/queryRows.js

@@ -74,7 +74,7 @@ exports.definition = {
 async function getTable(appId, tableId) {
   const ctx = buildCtx(appId, null, {
     params: {
-      id: tableId,
+      tableId,
     },
   })
   await tableController.find(ctx)

packages/server/src/automations/utils.js

@@ -5,7 +5,7 @@ const CouchDB = require("../db")
 const { queue } = require("./bullboard")
 const newid = require("../db/newid")
 const { updateEntityMetadata } = require("../utilities")
-const { MetadataTypes } = require("../constants")
+const { MetadataTypes, WebhookType } = require("../constants")
 const { getProdAppID } = require("@budibase/backend-core/db")
 const { cloneDeep } = require("lodash/fp")
 const { getAppDB, getAppId } = require("@budibase/backend-core/context")
@@ -159,7 +159,7 @@ exports.checkForWebhooks = async ({ oldAuto, newAuto }) => {
       request: {
         body: new webhooks.Webhook(
           "Automation webhook",
-          webhooks.WebhookType.AUTOMATION,
+          WebhookType.AUTOMATION,
           newAuto._id
         ),
       },

packages/server/src/constants/index.js

@@ -186,5 +186,9 @@ exports.BuildSchemaErrors = {
   INVALID_COLUMN: "invalid_column",
 }
 
+exports.WebhookType = {
+  AUTOMATION: "automation",
+}
+
 // pass through the list from the auth/core lib
 exports.ObjectStoreBuckets = ObjectStoreBuckets

packages/server/src/db/utils.js

@@ -146,6 +146,18 @@ exports.getRowParams = (tableId = null, rowId = null, otherProps = {}) => {
   return getDocParams(DocumentTypes.ROW, endOfKey, otherProps)
 }
 
+/**
+ * Given a row ID this will find the table ID within it (only works for internal tables).
+ * @param {string} rowId The ID of the row.
+ * @returns {string} The table ID.
+ */
+exports.getTableIDFromRowID = rowId => {
+  const components = rowId
+    .split(DocumentTypes.TABLE + SEPARATOR)[1]
+    .split(SEPARATOR)
+  return `${DocumentTypes.TABLE}${SEPARATOR}${components[0]}`
+}
+
 /**
  * Gets a new row ID for the specified table.
  * @param {string} tableId The table which the row is being created for.

packages/server/src/definitions/common.ts

@@ -5,6 +5,10 @@ export interface Base {
   _rev?: string
 }
 
+export interface Application extends Base {
+  appId?: string
+}
+
 export interface FieldSchema {
   // TODO: replace with field types enum when done
   type: string

packages/server/src/definitions/openapi.ts

@@ -0,0 +1,943 @@
+/**
+ * This file was auto-generated by openapi-typescript.
+ * Do not make direct changes to the file.
+ */
+
+export interface paths {
+  "/applications": {
+    post: {
+      parameters: {
+        header: {
+          /** The ID of the app which this request is targeting. */
+          "x-budibase-app-id": components["parameters"]["appId"]
+        }
+      }
+      responses: {
+        /** Returns the created application. */
+        200: {
+          content: {
+            "application/json": components["schemas"]["applicationOutput"]
+          }
+        }
+      }
+      requestBody: {
+        content: {
+          "application/json": components["schemas"]["application"]
+        }
+      }
+    }
+  }
+  "/applications/{appId}": {
+    get: {
+      parameters: {
+        path: {
+          /** The ID of the app which this request is targeting. */
+          appId: components["parameters"]["appIdUrl"]
+        }
+      }
+      responses: {
+        /** Returns the retrieved application. */
+        200: {
+          content: {
+            "application/json": components["schemas"]["applicationOutput"]
+          }
+        }
+      }
+    }
+    put: {
+      parameters: {
+        path: {
+          /** The ID of the app which this request is targeting. */
+          appId: components["parameters"]["appIdUrl"]
+        }
+      }
+      responses: {
+        /** Returns the updated application. */
+        200: {
+          content: {
+            "application/json": components["schemas"]["applicationOutput"]
+          }
+        }
+      }
+      requestBody: {
+        content: {
+          "application/json": components["schemas"]["application"]
+        }
+      }
+    }
+    delete: {
+      parameters: {
+        path: {
+          /** The ID of the app which this request is targeting. */
+          appId: components["parameters"]["appIdUrl"]
+        }
+      }
+      responses: {
+        /** Returns the deleted application. */
+        200: {
+          content: {
+            "application/json": components["schemas"]["applicationOutput"]
+          }
+        }
+      }
+    }
+  }
+  "/applications/search": {
+    /** Based on application properties (currently only name) search for applications. */
+    post: {
+      parameters: {
+        header: {
+          /** The ID of the app which this request is targeting. */
+          "x-budibase-app-id": components["parameters"]["appId"]
+        }
+      }
+      responses: {
+        /** Returns the applications that were found based on the search parameters. */
+        200: {
+          content: {
+            "application/json": {
+              data: components["schemas"]["application"][]
+            }
+          }
+        }
+      }
+      requestBody: {
+        content: {
+          "application/json": components["schemas"]["nameSearch"]
+        }
+      }
+    }
+  }
+  "/queries/{queryId}": {
+    /** Queries which have been created within a Budibase app can be executed using this, */
+    post: {
+      parameters: {
+        path: {
+          /** The ID of the query which this request is targeting. */
+          queryId: components["parameters"]["queryId"]
+        }
+        header: {
+          /** The ID of the app which this request is targeting. */
+          "x-budibase-app-id": components["parameters"]["appId"]
+        }
+      }
+      responses: {
+        /** Returns the result of the query execution. */
+        200: {
+          content: {
+            "application/json": components["schemas"]["executeQueryOutput"]
+          }
+        }
+      }
+      requestBody: {
+        content: {
+          "application/json": components["schemas"]["executeQuery"]
+        }
+      }
+    }
+  }
+  "/queries/search": {
+    /** Based on query properties (currently only name) search for queries. */
+    post: {
+      parameters: {
+        header: {
+          /** The ID of the app which this request is targeting. */
+          "x-budibase-app-id": components["parameters"]["appId"]
+        }
+      }
+      responses: {
+        /** Returns the queries found based on the search parameters. */
+        200: {
+          content: {
+            "application/json": {
+              data: components["schemas"]["query"][]
+            }
+          }
+        }
+      }
+      requestBody: {
+        content: {
+          "application/json": components["schemas"]["nameSearch"]
+        }
+      }
+    }
+  }
+  "/tables/{tableId}/rows": {
+    /** Creates a row within the specified table. */
+    post: {
+      parameters: {
+        path: {
+          /** The ID of the table which this request is targeting. */
+          tableId: components["parameters"]["tableId"]
+        }
+        header: {
+          /** The ID of the app which this request is targeting. */
+          "x-budibase-app-id": components["parameters"]["appId"]
+        }
+      }
+      responses: {
+        /** Returns the created row, including the ID which has been generated for it. This can be found in the Budibase portal, viewed under the developer information. */
+        200: {
+          content: {
+            "application/json": components["schemas"]["rowOutput"]
+          }
+        }
+      }
+      requestBody: {
+        content: {
+          "application/json": components["schemas"]["row"]
+        }
+      }
+    }
+  }
+  "/tables/{tableId}/rows/{rowId}": {
+    /** This gets a single row, it will be enriched with the full related rows, rather than the squashed "primaryDisplay" format returned by the search endpoint. */
+    get: {
+      parameters: {
+        path: {
+          /** The ID of the table which this request is targeting. */
+          tableId: components["parameters"]["tableId"]
+          /** The ID of the row which this request is targeting. */
+          rowId: components["parameters"]["rowId"]
+        }
+        header: {
+          /** The ID of the app which this request is targeting. */
+          "x-budibase-app-id": components["parameters"]["appId"]
+        }
+      }
+      responses: {
+        /** Returns the retrieved row. */
+        200: {
+          content: {
+            "application/json": components["schemas"]["rowOutput"]
+          }
+        }
+      }
+    }
+    /** Updates a row within the specified table. */
+    put: {
+      parameters: {
+        path: {
+          /** The ID of the table which this request is targeting. */
+          tableId: components["parameters"]["tableId"]
+          /** The ID of the row which this request is targeting. */
+          rowId: components["parameters"]["rowId"]
+        }
+        header: {
+          /** The ID of the app which this request is targeting. */
+          "x-budibase-app-id": components["parameters"]["appId"]
+        }
+      }
+      responses: {
+        /** Returns the created row, including the ID which has been generated for it. */
+        200: {
+          content: {
+            "application/json": components["schemas"]["rowOutput"]
+          }
+        }
+      }
+      requestBody: {
+        content: {
+          "application/json": components["schemas"]["row"]
+        }
+      }
+    }
+    /** Deletes a row within the specified table. */
+    delete: {
+      parameters: {
+        path: {
+          /** The ID of the table which this request is targeting. */
+          tableId: components["parameters"]["tableId"]
+          /** The ID of the row which this request is targeting. */
+          rowId: components["parameters"]["rowId"]
+        }
+        header: {
+          /** The ID of the app which this request is targeting. */
+          "x-budibase-app-id": components["parameters"]["appId"]
+        }
+      }
+      responses: {
+        /** Returns the deleted row, including the ID which has been generated for it. */
+        200: {
+          content: {
+            "application/json": components["schemas"]["rowOutput"]
+          }
+        }
+      }
+    }
+  }
+  "/tables/{tableId}/rows/search": {
+    post: {
+      parameters: {
+        path: {
+          /** The ID of the table which this request is targeting. */
+          tableId: components["parameters"]["tableId"]
+        }
+        header: {
+          /** The ID of the app which this request is targeting. */
+          "x-budibase-app-id": components["parameters"]["appId"]
+        }
+      }
+      responses: {
+        /** The response will contain an array of rows that match the search parameters. */
+        200: {
+          content: {
+            "application/json": components["schemas"]["searchOutput"]
+          }
+        }
+      }
+      requestBody: {
+        content: {
+          "application/json": {
+            query: {
+              /**
+               * @description A map of field name to the string to search for, this will look for rows that have a value starting with the string value.
+               * @example [object Object]
+               */
+              string?: { [key: string]: string }
+              /** @description A fuzzy search, only supported by internal tables. */
+              fuzzy?: { [key: string]: unknown }
+              /**
+               * @description Searches within a range, the format of this must be columnName -> [low, high].
+               * @example [object Object]
+               */
+              range?: { [key: string]: unknown }
+              /** @description Searches for rows that have a column value that is exactly the value set. */
+              equal?: { [key: string]: unknown }
+              /** @description Searches for any row which does not contain the specified column value. */
+              notEqual?: { [key: string]: unknown }
+              /**
+               * @description Searches for rows which do not contain the specified column. The object should simply contain keys of the column names, these can map to any value.
+               * @example [object Object]
+               */
+              empty?: { [key: string]: unknown }
+              /** @description Searches for rows which have the specified column. */
+              notEmpty?: { [key: string]: unknown }
+              /** @description Searches for rows which have a column value that is any of the specified values. The format of this must be columnName -> [value1, value2]. */
+              oneOf?: { [key: string]: unknown }
+            }
+            /** @description Enables pagination, by default this is disabled. */
+            paginate?: boolean
+            /** @description If retrieving another page, the bookmark from the previous request must be supplied. */
+            bookmark?: string | number
+            /** @description The maximum number of rows to return, useful when paginating, for internal tables this will be limited to 1000, for SQL tables it will be 5000. */
+            limit?: number
+            /** @description A set of parameters describing the sort behaviour of the search. */
+            sort?: {
+              /**
+               * @description The order of the sort, by default this is ascending.
+               * @enum {string}
+               */
+              order?: "ascending" | "descending"
+              /** @description The name of the column by which the rows will be sorted. */
+              column?: string
+              /**
+               * @description Defines whether the column should be treated as a string or as numbers when sorting.
+               * @enum {string}
+               */
+              type?: "string" | "number"
+            }
+          }
+        }
+      }
+    }
+  }
+  "/tables": {
+    /** Create a table, this could be internal or external. */
+    post: {
+      parameters: {
+        header: {
+          /** The ID of the app which this request is targeting. */
+          "x-budibase-app-id": components["parameters"]["appId"]
+        }
+      }
+      responses: {
+        /** Returns the created table, including the ID which has been generated for it. This can be internal or external data sources. */
+        200: {
+          content: {
+            "application/json": components["schemas"]["tableOutput"]
+          }
+        }
+      }
+      requestBody: {
+        content: {
+          "application/json": components["schemas"]["table"]
+        }
+      }
+    }
+  }
+  "/tables/{tableId}": {
+    /** Lookup a table, this could be internal or external. */
+    get: {
+      parameters: {
+        path: {
+          /** The ID of the table which this request is targeting. */
+          tableId: components["parameters"]["tableId"]
+        }
+        header: {
+          /** The ID of the app which this request is targeting. */
+          "x-budibase-app-id": components["parameters"]["appId"]
+        }
+      }
+      responses: {
+        /** Returns the retrieved table. */
+        200: {
+          content: {
+            "application/json": components["schemas"]["tableOutput"]
+          }
+        }
+      }
+    }
+    /** Update a table, this could be internal or external. */
+    put: {
+      parameters: {
+        path: {
+          /** The ID of the table which this request is targeting. */
+          tableId: components["parameters"]["tableId"]
+        }
+        header: {
+          /** The ID of the app which this request is targeting. */
+          "x-budibase-app-id": components["parameters"]["appId"]
+        }
+      }
+      responses: {
+        /** Returns the updated table. */
+        200: {
+          content: {
+            "application/json": components["schemas"]["tableOutput"]
+          }
+        }
+      }
+      requestBody: {
+        content: {
+          "application/json": components["schemas"]["table"]
+        }
+      }
+    }
+    /** Delete a table, this could be internal or external. */
+    delete: {
+      parameters: {
+        path: {
+          /** The ID of the table which this request is targeting. */
+          tableId: components["parameters"]["tableId"]
+        }
+        header: {
+          /** The ID of the app which this request is targeting. */
+          "x-budibase-app-id": components["parameters"]["appId"]
+        }
+      }
+      responses: {
+        /** Returns the deleted table. */
+        200: {
+          content: {
+            "application/json": components["schemas"]["tableOutput"]
+          }
+        }
+      }
+    }
+  }
+  "/tables/search": {
+    /** Based on table properties (currently only name) search for tables. This could be an internal or an external table. */
+    post: {
+      parameters: {
+        header: {
+          /** The ID of the app which this request is targeting. */
+          "x-budibase-app-id": components["parameters"]["appId"]
+        }
+      }
+      responses: {
+        /** Returns the found tables, based on the search parameters. */
+        200: {
+          content: {
+            "application/json": {
+              data: components["schemas"]["table"][]
+            }
+          }
+        }
+      }
+      requestBody: {
+        content: {
+          "application/json": components["schemas"]["nameSearch"]
+        }
+      }
+    }
+  }
+  "/users": {
+    post: {
+      responses: {
+        /** Returns the created user. */
+        200: {
+          content: {
+            "application/json": components["schemas"]["userOutput"]
+          }
+        }
+      }
+      requestBody: {
+        content: {
+          "application/json": components["schemas"]["user"]
+        }
+      }
+    }
+  }
+  "/users/{userId}": {
+    get: {
+      parameters: {
+        path: {
+          /** The ID of the user which this request is targeting. */
+          userId: components["parameters"]["userId"]
+        }
+      }
+      responses: {
+        /** Returns the retrieved user. */
+        200: {
+          content: {
+            "application/json": components["schemas"]["userOutput"]
+          }
+        }
+      }
+    }
+    put: {
+      parameters: {
+        path: {
+          /** The ID of the user which this request is targeting. */
+          userId: components["parameters"]["userId"]
+        }
+      }
+      responses: {
+        /** Returns the updated user. */
+        200: {
+          content: {
+            "application/json": components["schemas"]["userOutput"]
+          }
+        }
+      }
+      requestBody: {
+        content: {
+          "application/json": components["schemas"]["user"]
+        }
+      }
+    }
+    delete: {
+      parameters: {
+        path: {
+          /** The ID of the user which this request is targeting. */
+          userId: components["parameters"]["userId"]
+        }
+      }
+      responses: {
+        /** Returns the deleted user. */
+        200: {
+          content: {
+            "application/json": components["schemas"]["userOutput"]
+          }
+        }
+      }
+    }
+  }
+  "/users/search": {
+    /** Based on user properties (currently only name) search for users. */
+    post: {
+      responses: {
+        /** Returns the found users based on search parameters. */
+        200: {
+          content: {
+            "application/json": {
+              data: components["schemas"]["user"][]
+            }
+          }
+        }
+      }
+      requestBody: {
+        content: {
+          "application/json": components["schemas"]["nameSearch"]
+        }
+      }
+    }
+  }
+}
+
+export interface components {
+  schemas: {
+    application: {
+      /** @description The name of the app. */
+      name: string
+      /** @description The URL by which the app is accessed, this must be URL encoded. */
+      url: string
+    }
+    applicationOutput: {
+      data: {
+        /** @description The name of the app. */
+        name: string
+        /** @description The URL by which the app is accessed, this must be URL encoded. */
+        url: string
+        /** @description The ID of the app. */
+        _id: string
+        /**
+         * @description The status of the app, stating it if is the development or published version.
+         * @enum {string}
+         */
+        status: "development" | "published"
+        /** @description States when the app was created, will be constant. Stored in ISO format. */
+        createdAt: string
+        /** @description States the last time the app was updated - stored in ISO format. */
+        updatedAt: string
+        /** @description States the version of the Budibase client this app is currently based on. */
+        version: string
+        /** @description In a multi-tenant environment this will state the tenant this app is within. */
+        tenantId?: string
+        /** @description The user this app is currently being built by. */
+        lockedBy?: { [key: string]: unknown }
+      }
+    }
+    /** @description The row to be created/updated, based on the table schema. */
+    row: { [key: string]: unknown }
+    searchOutput: {
+      /** @description An array of rows, these will each contain an _id field which can be used to update or delete them. */
+      data: { [key: string]: unknown }[]
+      /** @description If pagination in use, this should be provided. */
+      bookmark?: string | number
+      /** @description If pagination in use, this will determine if there is another page to fetch. */
+      hasNextPage?: boolean
+    }
+    rowOutput: {
+      /** @description The row to be created/updated, based on the table schema. */
+      data: {
+        /** @description The ID of the row. */
+        _id: string
+        /** @description The ID of the table this row comes from. */
+        tableId: string
+      } & { [key: string]: unknown }
+    }
+    /** @description The table to be created/updated. */
+    table: {
+      /** @description The name of the table. */
+      name: string
+      /** @description The name of the column which should be used in relationship tags when relating to this table. */
+      primaryDisplay?: string
+      schema: {
+        [key: string]:
+          | {
+              /**
+               * @description A relationship column.
+               * @enum {string}
+               */
+              type?: "link"
+              /** @description A constraint can be applied to the column which will be validated against when a row is saved. */
+              constraints?: {
+                /** @enum {string} */
+                type?: "string" | "number" | "object" | "boolean"
+                /** @description Defines whether the column is required or not. */
+                presence?: boolean
+              }
+              /** @description The name of the column. */
+              name?: string
+              /** @description Defines whether the column is automatically generated. */
+              autocolumn?: boolean
+              /** @description The name of the column which a relationship column is related to in another table. */
+              fieldName?: string
+              /** @description The ID of the table which a relationship column is related to. */
+              tableId?: string
+              /**
+               * @description Defines the type of relationship that this column will be used for.
+               * @enum {string}
+               */
+              relationshipType?: "one-to-many" | "many-to-one" | "many-to-many"
+              /** @description When using a SQL table that contains many to many relationships this defines the table the relationships are linked through. */
+              through?: string
+              /** @description When using a SQL table that contains a one to many relationship this defines the foreign key. */
+              foreignKey?: string
+              /** @description When using a SQL table that utilises a through table, this defines the primary key in the through table for this table. */
+              throughFrom?: string
+              /** @description When using a SQL table that utilises a through table, this defines the primary key in the through table for the related table. */
+              throughTo?: string
+            }
+          | {
+              /**
+               * @description A formula column.
+               * @enum {string}
+               */
+              type?: "formula"
+              /** @description A constraint can be applied to the column which will be validated against when a row is saved. */
+              constraints?: {
+                /** @enum {string} */
+                type?: "string" | "number" | "object" | "boolean"
+                /** @description Defines whether the column is required or not. */
+                presence?: boolean
+              }
+              /** @description The name of the column. */
+              name?: string
+              /** @description Defines whether the column is automatically generated. */
+              autocolumn?: boolean
+              /** @description Defines a Handlebars or JavaScript formula to use, note that Javascript formulas are expected to be provided in the base64 format. */
+              formula?: string
+              /**
+               * @description Defines whether this is a static or dynamic formula.
+               * @enum {string}
+               */
+              formulaType?: "static" | "dynamic"
+            }
+          | {
+              /**
+               * @description Defines the type of the column, most explain themselves, a link column is a relationship.
+               * @enum {string}
+               */
+              type?:
+                | "string"
+                | "longform"
+                | "options"
+                | "number"
+                | "boolean"
+                | "array"
+                | "datetime"
+                | "attachment"
+                | "link"
+                | "formula"
+                | "auto"
+                | "json"
+                | "internal"
+              /** @description A constraint can be applied to the column which will be validated against when a row is saved. */
+              constraints?: {
+                /** @enum {string} */
+                type?: "string" | "number" | "object" | "boolean"
+                /** @description Defines whether the column is required or not. */
+                presence?: boolean
+              }
+              /** @description The name of the column. */
+              name?: string
+              /** @description Defines whether the column is automatically generated. */
+              autocolumn?: boolean
+            }
+      }
+    }
+    tableOutput: {
+      /** @description The table to be created/updated. */
+      data: {
+        /** @description The name of the table. */
+        name: string
+        /** @description The name of the column which should be used in relationship tags when relating to this table. */
+        primaryDisplay?: string
+        schema: {
+          [key: string]:
+            | {
+                /**
+                 * @description A relationship column.
+                 * @enum {string}
+                 */
+                type?: "link"
+                /** @description A constraint can be applied to the column which will be validated against when a row is saved. */
+                constraints?: {
+                  /** @enum {string} */
+                  type?: "string" | "number" | "object" | "boolean"
+                  /** @description Defines whether the column is required or not. */
+                  presence?: boolean
+                }
+                /** @description The name of the column. */
+                name?: string
+                /** @description Defines whether the column is automatically generated. */
+                autocolumn?: boolean
+                /** @description The name of the column which a relationship column is related to in another table. */
+                fieldName?: string
+                /** @description The ID of the table which a relationship column is related to. */
+                tableId?: string
+                /**
+                 * @description Defines the type of relationship that this column will be used for.
+                 * @enum {string}
+                 */
+                relationshipType?:
+                  | "one-to-many"
+                  | "many-to-one"
+                  | "many-to-many"
+                /** @description When using a SQL table that contains many to many relationships this defines the table the relationships are linked through. */
+                through?: string
+                /** @description When using a SQL table that contains a one to many relationship this defines the foreign key. */
+                foreignKey?: string
+                /** @description When using a SQL table that utilises a through table, this defines the primary key in the through table for this table. */
+                throughFrom?: string
+                /** @description When using a SQL table that utilises a through table, this defines the primary key in the through table for the related table. */
+                throughTo?: string
+              }
+            | {
+                /**
+                 * @description A formula column.
+                 * @enum {string}
+                 */
+                type?: "formula"
+                /** @description A constraint can be applied to the column which will be validated against when a row is saved. */
+                constraints?: {
+                  /** @enum {string} */
+                  type?: "string" | "number" | "object" | "boolean"
+                  /** @description Defines whether the column is required or not. */
+                  presence?: boolean
+                }
+                /** @description The name of the column. */
+                name?: string
+                /** @description Defines whether the column is automatically generated. */
+                autocolumn?: boolean
+                /** @description Defines a Handlebars or JavaScript formula to use, note that Javascript formulas are expected to be provided in the base64 format. */
+                formula?: string
+                /**
+                 * @description Defines whether this is a static or dynamic formula.
+                 * @enum {string}
+                 */
+                formulaType?: "static" | "dynamic"
+              }
+            | {
+                /**
+                 * @description Defines the type of the column, most explain themselves, a link column is a relationship.
+                 * @enum {string}
+                 */
+                type?:
+                  | "string"
+                  | "longform"
+                  | "options"
+                  | "number"
+                  | "boolean"
+                  | "array"
+                  | "datetime"
+                  | "attachment"
+                  | "link"
+                  | "formula"
+                  | "auto"
+                  | "json"
+                  | "internal"
+                /** @description A constraint can be applied to the column which will be validated against when a row is saved. */
+                constraints?: {
+                  /** @enum {string} */
+                  type?: "string" | "number" | "object" | "boolean"
+                  /** @description Defines whether the column is required or not. */
+                  presence?: boolean
+                }
+                /** @description The name of the column. */
+                name?: string
+                /** @description Defines whether the column is automatically generated. */
+                autocolumn?: boolean
+              }
+        }
+        /** @description The ID of the table. */
+        _id: string
+      }
+    }
+    /** @description The query body must contain the required parameters for the query, this depends on query type, setup and bindings. */
+    executeQuery: { [key: string]: unknown }
+    executeQueryOutput: {
+      /** @description The data response from the query. */
+      data: { [key: string]: unknown }[]
+      /** @description Extra information that is not part of the main data, e.g. headers. */
+      extra?: {
+        /** @description If carrying out a REST request, this will contain the response headers. */
+        headers?: { [key: string]: unknown }
+        /** @description The raw query response, as a string. */
+        raw?: string
+      }
+      /** @description If pagination is supported, this will contain the bookmark/anchor information for it. */
+      pagination?: { [key: string]: unknown }
+    }
+    query: {
+      /** @description The ID of the query. */
+      _id: string
+      /** @description The ID of the data source the query belongs to. */
+      datasourceId?: string
+      /** @description The bindings which are required to perform this query. */
+      parameters?: string[]
+      /** @description The fields that are used to perform this query, e.g. the sql statement */
+      fields?: { [key: string]: unknown }
+      /**
+       * @description The verb that describes this query.
+       * @enum {undefined}
+       */
+      queryVerb?: "create" | "read" | "update" | "delete"
+      /** @description The name of the query. */
+      name: string
+      /** @description The schema of the data returned when the query is executed. */
+      schema: { [key: string]: unknown }
+      /** @description The JavaScript transformer function, applied after the query responds with data. */
+      transformer?: string
+      /** @description Whether the query has readable data. */
+      readable?: boolean
+    }
+    user: {
+      /** @description The email address of the user, this must be unique. */
+      email: string
+      /** @description The password of the user if using password based login - this will never be returned. This can be left out of subsequent requests (updates) and will be enriched back into the user structure. */
+      password?: string
+      /**
+       * @description The status of the user, if they are active.
+       * @enum {string}
+       */
+      status?: "active"
+      /** @description The first name of the user */
+      firstName?: string
+      /** @description The last name of the user */
+      lastName?: string
+      /** @description If set to true forces the user to reset their password on first login. */
+      forceResetPassword?: boolean
+      /** @description Describes if the user is a builder user or not. */
+      builder?: {
+        /** @description If set to true the user will be able to build any app in the system. */
+        global?: boolean
+      }
+      /** @description Describes if the user is an admin user or not. */
+      admin?: {
+        /** @description If set to true the user will be able to administrate the system. */
+        global?: boolean
+      }
+      /** @description Contains the roles of the user per app (assuming they are not a builder user). */
+      roles: { [key: string]: string }
+    }
+    userOutput: {
+      data: {
+        /** @description The email address of the user, this must be unique. */
+        email: string
+        /** @description The password of the user if using password based login - this will never be returned. This can be left out of subsequent requests (updates) and will be enriched back into the user structure. */
+        password?: string
+        /**
+         * @description The status of the user, if they are active.
+         * @enum {string}
+         */
+        status?: "active"
+        /** @description The first name of the user */
+        firstName?: string
+        /** @description The last name of the user */
+        lastName?: string
+        /** @description If set to true forces the user to reset their password on first login. */
+        forceResetPassword?: boolean
+        /** @description Describes if the user is a builder user or not. */
+        builder?: {
+          /** @description If set to true the user will be able to build any app in the system. */
+          global?: boolean
+        }
+        /** @description Describes if the user is an admin user or not. */
+        admin?: {
+          /** @description If set to true the user will be able to administrate the system. */
+          global?: boolean
+        }
+        /** @description Contains the roles of the user per app (assuming they are not a builder user). */
+        roles: { [key: string]: string }
+        /** @description The ID of the user. */
+        _id: string
+      }
+    }
+    nameSearch: {
+      /** @description The name to be used when searching - this will be used in a case insensitive starts with match. */
+      name: string
+    }
+  }
+  parameters: {
+    /** @description The ID of the table which this request is targeting. */
+    tableId: string
+    /** @description The ID of the row which this request is targeting. */
+    rowId: string
+    /** @description The ID of the app which this request is targeting. */
+    appId: string
+    /** @description The ID of the app which this request is targeting. */
+    appIdUrl: string
+    /** @description The ID of the query which this request is targeting. */
+    queryId: string
+    /** @description The ID of the user which this request is targeting. */
+    userId: string
+  }
+}
+
+export interface operations {}
+
+export interface external {}

packages/server/src/tests/utilities/TestConfiguration.js

@@ -25,6 +25,8 @@ const { createASession } = require("@budibase/backend-core/sessions")
 const { user: userCache } = require("@budibase/backend-core/cache")
 const newid = require("../../db/newid")
 const context = require("@budibase/backend-core/context")
+const { generateDevInfoID, SEPARATOR } = require("@budibase/backend-core/db")
+const { encrypt } = require("@budibase/backend-core/encryption")
 
 const GLOBAL_USER_ID = "us_uuid1"
 const EMAIL = "babs@babs.com"
@@ -47,6 +49,10 @@ class TestConfiguration {
     return this.request
   }
 
+  getApp() {
+    return this.app
+  }
+
   getAppId() {
     return this.appId
   }
@@ -83,6 +89,20 @@ class TestConfiguration {
     }
   }
 
+  async generateApiKey(userId = GLOBAL_USER_ID) {
+    const db = getGlobalDB(TENANT_ID)
+    const id = generateDevInfoID(userId)
+    let devInfo
+    try {
+      devInfo = await db.get(id)
+    } catch (err) {
+      devInfo = { _id: id, userId }
+    }
+    devInfo.apiKey = encrypt(`${TENANT_ID}${SEPARATOR}${newid()}`)
+    await db.put(devInfo)
+    return devInfo.apiKey
+  }
+
   async globalUser({
     id = GLOBAL_USER_ID,
     builder = true,
@@ -135,7 +155,7 @@ class TestConfiguration {
     cleanup(this.allApps.map(app => app.appId))
   }
 
-  defaultHeaders() {
+  defaultHeaders(extras = {}) {
     const auth = {
       userId: GLOBAL_USER_ID,
       sessionId: "sessionid",
@@ -154,6 +174,7 @@ class TestConfiguration {
         `${Cookies.CurrentApp}=${appToken}`,
       ],
       [Headers.CSRF_TOKEN]: CSRF_TOKEN,
+      ...extras,
     }
     if (this.appId) {
       headers[Headers.APP_ID] = this.appId
@@ -226,7 +247,7 @@ class TestConfiguration {
 
   async getTable(tableId = null) {
     tableId = tableId || this.table._id
-    return this._req(null, { id: tableId }, controllers.table.find)
+    return this._req(null, { tableId }, controllers.table.find)
   }
 
   async createLinkedTable(relationshipType = null, links = ["link"]) {

packages/server/src/utilities/workerRequests.js

@@ -26,11 +26,31 @@ function request(ctx, request) {
     delete request.body
   }
   if (ctx && ctx.headers) {
-    request.headers.cookie = ctx.headers.cookie
+    request.headers = ctx.headers
   }
   return request
 }
 
+async function checkResponse(response, errorMsg, { ctx } = {}) {
+  if (response.status !== 200) {
+    let error
+    try {
+      error = await response.json()
+    } catch (err) {
+      error = await response.text()
+    }
+    const msg = `Unable to ${errorMsg} - ${
+      error.message ? error.message : error
+    }`
+    if (ctx) {
+      ctx.throw(400, msg)
+    } else {
+      throw msg
+    }
+  }
+  return response.json()
+}
+
 exports.request = request
 
 // have to pass in the tenant ID as this could be coming from an automation
@@ -50,25 +70,17 @@ exports.sendSmtpEmail = async (to, from, subject, contents, automation) => {
       },
     })
   )
-
+  return checkResponse(response, "send email")
-  if (response.status !== 200) {
-    const error = await response.text()
-    throw `Unable to send email - ${error}`
-  }
-  return response.json()
 }
 
 exports.getGlobalSelf = async (ctx, appId = null) => {
-  const endpoint = `/api/global/users/self`
+  const endpoint = `/api/global/self`
   const response = await fetch(
     checkSlashesInUrl(env.WORKER_URL + endpoint),
     // we don't want to use API key when getting self
     request(ctx, { method: "GET" })
   )
-  if (response.status !== 200) {
+  let json = await checkResponse(response, "get self globally", { ctx })
-    ctx.throw(400, "Unable to get self globally.")
-  }
-  let json = await response.json()
   if (appId) {
     json = updateAppRole(json)
   }
@@ -83,8 +95,45 @@ exports.removeAppFromUserRoles = async (ctx, appId) => {
       method: "DELETE",
     })
   )
-  if (response.status !== 200) {
+  return checkResponse(response, "remove app role")
-    throw "Unable to remove app role"
+}
-  }
+
-  return response.json()
+exports.allGlobalUsers = async ctx => {
+  const response = await fetch(
+    checkSlashesInUrl(env.WORKER_URL + "/api/global/users"),
+    // we don't want to use API key when getting self
+    request(ctx, { method: "GET" })
+  )
+  return checkResponse(response, "get users", { ctx })
+}
+
+exports.saveGlobalUser = async ctx => {
+  const response = await fetch(
+    checkSlashesInUrl(env.WORKER_URL + "/api/global/users"),
+    // we don't want to use API key when getting self
+    request(ctx, { method: "POST", body: ctx.request.body })
+  )
+  return checkResponse(response, "save user", { ctx })
+}
+
+exports.deleteGlobalUser = async ctx => {
+  const response = await fetch(
+    checkSlashesInUrl(
+      env.WORKER_URL + `/api/global/users/${ctx.params.userId}`
+    ),
+    // we don't want to use API key when getting self
+    request(ctx, { method: "DELETE" })
+  )
+  return checkResponse(response, "delete user", { ctx, body: ctx.request.body })
+}
+
+exports.readGlobalUser = async ctx => {
+  const response = await fetch(
+    checkSlashesInUrl(
+      env.WORKER_URL + `/api/global/users/${ctx.params.userId}`
+    ),
+    // we don't want to use API key when getting self
+    request(ctx, { method: "GET" })
+  )
+  return checkResponse(response, "get user", { ctx })
 }

packages/worker/src/api/controllers/global/roles.js

@@ -5,6 +5,9 @@ const {
   DocumentTypes,
 } = require("@budibase/backend-core/db")
 const { doInAppContext, getAppDB } = require("@budibase/backend-core/context")
+const { user: userCache } = require("@budibase/backend-core/cache")
+const { getGlobalDB } = require("@budibase/backend-core/tenancy")
+const { allUsers } = require("../../utilities")
 
 exports.fetch = async ctx => {
   const tenantId = ctx.user.tenantId
@@ -42,3 +45,23 @@ exports.find = async ctx => {
     }
   })
 }
+
+exports.removeAppRole = async ctx => {
+  const { appId } = ctx.params
+  const db = getGlobalDB()
+  const users = await allUsers(ctx)
+  const bulk = []
+  const cacheInvalidations = []
+  for (let user of users) {
+    if (user.roles[appId]) {
+      cacheInvalidations.push(userCache.invalidateUser(user._id))
+      delete user.roles[appId]
+      bulk.push(user)
+    }
+  }
+  await db.bulkDocs(bulk)
+  await Promise.all(cacheInvalidations)
+  ctx.body = {
+    message: "App role removed from all users",
+  }
+}

packages/worker/src/api/controllers/global/self.js

@@ -0,0 +1,95 @@
+const { getGlobalDB, getTenantId } = require("@budibase/backend-core/tenancy")
+const { generateDevInfoID, SEPARATOR } = require("@budibase/backend-core/db")
+const { user: userCache } = require("@budibase/backend-core/cache")
+const { hash, platformLogout } = require("@budibase/backend-core/utils")
+const { encrypt } = require("@budibase/backend-core/encryption")
+const { newid } = require("@budibase/backend-core/utils")
+const { getUser } = require("../../utilities")
+
+function newApiKey() {
+  return encrypt(`${getTenantId()}${SEPARATOR}${newid()}`)
+}
+
+function cleanupDevInfo(info) {
+  // user doesn't need to aware of dev doc info
+  delete info._id
+  delete info._rev
+  return info
+}
+
+exports.generateAPIKey = async ctx => {
+  const db = getGlobalDB()
+  const id = generateDevInfoID(ctx.user._id)
+  let devInfo
+  try {
+    devInfo = await db.get(id)
+  } catch (err) {
+    devInfo = { _id: id, userId: ctx.user._id }
+  }
+  devInfo.apiKey = await newApiKey()
+  await db.put(devInfo)
+  ctx.body = cleanupDevInfo(devInfo)
+}
+
+exports.fetchAPIKey = async ctx => {
+  const db = getGlobalDB()
+  const id = generateDevInfoID(ctx.user._id)
+  let devInfo
+  try {
+    devInfo = await db.get(id)
+  } catch (err) {
+    devInfo = {
+      _id: id,
+      userId: ctx.user._id,
+      apiKey: await newApiKey(),
+    }
+    await db.put(devInfo)
+  }
+  ctx.body = cleanupDevInfo(devInfo)
+}
+
+exports.getSelf = async ctx => {
+  if (!ctx.user) {
+    ctx.throw(403, "User not logged in")
+  }
+  const userId = ctx.user._id
+  ctx.params = {
+    id: userId,
+  }
+  // get the main body of the user
+  ctx.body = await getUser(userId)
+  // forward session information not found in db
+  ctx.body.account = ctx.user.account
+  ctx.body.budibaseAccess = ctx.user.budibaseAccess
+  ctx.body.accountPortalAccess = ctx.user.accountPortalAccess
+  ctx.body.csrfToken = ctx.user.csrfToken
+}
+
+exports.updateSelf = async ctx => {
+  const db = getGlobalDB()
+  const user = await db.get(ctx.user._id)
+  if (ctx.request.body.password) {
+    // changing password
+    ctx.request.body.password = await hash(ctx.request.body.password)
+    // Log all other sessions out apart from the current one
+    await platformLogout({
+      ctx,
+      userId: ctx.user._id,
+      keepActiveSession: true,
+    })
+  }
+  // don't allow sending up an ID/Rev, always use the existing one
+  delete ctx.request.body._id
+  delete ctx.request.body._rev
+  // don't allow setting the csrf token
+  delete ctx.request.body.csrfToken
+  const response = await db.put({
+    ...user,
+    ...ctx.request.body,
+  })
+  await userCache.invalidateUser(user._id)
+  ctx.body = {
+    _id: response.id,
+    _rev: response.rev,
+  }
+}

packages/worker/src/api/controllers/global/users.js

@@ -4,10 +4,8 @@ const {
   generateNewUsageQuotaDoc,
 } = require("@budibase/backend-core/db")
 const {
-  hash,
   getGlobalUserByEmail,
   saveUser,
-  platformLogout,
 } = require("@budibase/backend-core/utils")
 const { EmailTemplatePurpose } = require("../../../constants")
 const { checkInviteCode } = require("../../../utilities/redis")
@@ -24,16 +22,7 @@ const {
 const { removeUserFromInfoDB } = require("@budibase/backend-core/deprovision")
 const env = require("../../../environment")
 const { syncUserInApps } = require("../../../utilities/appService")
-
+const { allUsers, getUser } = require("../../utilities")
-async function allUsers() {
-  const db = getGlobalDB()
-  const response = await db.allDocs(
-    getGlobalUserParams(null, {
-      include_docs: true,
-    })
-  )
-  return response.rows.map(row => row.doc)
-}
 
 exports.save = async ctx => {
   try {
@@ -138,72 +127,6 @@ exports.destroy = async ctx => {
   }
 }
 
-exports.removeAppRole = async ctx => {
-  const { appId } = ctx.params
-  const db = getGlobalDB()
-  const users = await allUsers(ctx)
-  const bulk = []
-  const cacheInvalidations = []
-  for (let user of users) {
-    if (user.roles[appId]) {
-      cacheInvalidations.push(userCache.invalidateUser(user._id))
-      delete user.roles[appId]
-      bulk.push(user)
-    }
-  }
-  await db.bulkDocs(bulk)
-  await Promise.all(cacheInvalidations)
-  ctx.body = {
-    message: "App role removed from all users",
-  }
-}
-
-exports.getSelf = async ctx => {
-  if (!ctx.user) {
-    ctx.throw(403, "User not logged in")
-  }
-  ctx.params = {
-    id: ctx.user._id,
-  }
-  // this will set the body
-  await exports.find(ctx)
-
-  // forward session information not found in db
-  ctx.body.account = ctx.user.account
-  ctx.body.budibaseAccess = ctx.user.budibaseAccess
-  ctx.body.accountPortalAccess = ctx.user.accountPortalAccess
-  ctx.body.csrfToken = ctx.user.csrfToken
-}
-
-exports.updateSelf = async ctx => {
-  const db = getGlobalDB()
-  const user = await db.get(ctx.user._id)
-  if (ctx.request.body.password) {
-    // changing password
-    ctx.request.body.password = await hash(ctx.request.body.password)
-    // Log all other sessions out apart from the current one
-    await platformLogout({
-      ctx,
-      userId: ctx.user._id,
-      keepActiveSession: true,
-    })
-  }
-  // don't allow sending up an ID/Rev, always use the existing one
-  delete ctx.request.body._id
-  delete ctx.request.body._rev
-  // don't allow setting the csrf token
-  delete ctx.request.body.csrfToken
-  const response = await db.put({
-    ...user,
-    ...ctx.request.body,
-  })
-  await userCache.invalidateUser(user._id)
-  ctx.body = {
-    _id: response.id,
-    _rev: response.rev,
-  }
-}
-
 // called internally by app server user fetch
 exports.fetch = async ctx => {
   const users = await allUsers(ctx)
@@ -218,18 +141,7 @@ exports.fetch = async ctx => {
 
 // called internally by app server user find
 exports.find = async ctx => {
-  const db = getGlobalDB()
+  ctx.body = await getUser(ctx.params.id)
-  let user
-  try {
-    user = await db.get(ctx.params.id)
-  } catch (err) {
-    // no user found, just return nothing
-    user = {}
-  }
-  if (user) {
-    delete user.password
-  }
-  ctx.body = user
 }
 
 exports.tenantUserLookup = async ctx => {

packages/worker/src/api/index.js

@@ -67,6 +67,10 @@ const NO_TENANCY_ENDPOINTS = [
     route: "/api/global/users/self",
     method: "GET",
   },
+  {
+    route: "/api/global/self",
+    method: "GET",
+  },
 ]
 
 // most public endpoints are gets, but some are posts

packages/worker/src/api/routes/global/roles.js

@@ -7,5 +7,6 @@ const router = Router()
 router
   .get("/api/global/roles", adminOnly, controller.fetch)
   .get("/api/global/roles/:appId", adminOnly, controller.find)
+  .delete("/api/global/roles/:appId", adminOnly, controller.removeAppRole)
 
 module.exports = router

packages/worker/src/api/routes/global/self.js

@@ -0,0 +1,18 @@
+const Router = require("@koa/router")
+const controller = require("../../controllers/global/self")
+const builderOnly = require("../../../middleware/builderOnly")
+const { buildUserSaveValidation } = require("../../utilities/validation")
+
+const router = Router()
+
+router
+  .post("/api/global/self/api_key", builderOnly, controller.generateAPIKey)
+  .get("/api/global/self/api_key", builderOnly, controller.fetchAPIKey)
+  .get("/api/global/self", controller.getSelf)
+  .post(
+    "/api/global/self",
+    buildUserSaveValidation(true),
+    controller.updateSelf
+  )
+
+module.exports = router

packages/worker/src/api/routes/global/users.js

@@ -4,6 +4,8 @@ const joiValidator = require("../../../middleware/joi-validator")
 const adminOnly = require("../../../middleware/adminOnly")
 const Joi = require("joi")
 const cloudRestricted = require("../../../middleware/cloudRestricted")
+const { buildUserSaveValidation } = require("../../utilities/validation")
+const selfController = require("../../controllers/global/self")
 
 const router = Router()
 
@@ -19,32 +21,6 @@ function buildAdminInitValidation() {
   )
 }
 
-function buildUserSaveValidation(isSelf = false) {
-  let schema = {
-    email: Joi.string().allow(null, ""),
-    password: Joi.string().allow(null, ""),
-    forceResetPassword: Joi.boolean().optional(),
-    firstName: Joi.string().allow(null, ""),
-    lastName: Joi.string().allow(null, ""),
-    builder: Joi.object({
-      global: Joi.boolean().optional(),
-      apps: Joi.array().optional(),
-    })
-      .unknown(true)
-      .optional(),
-    // maps appId -> roleId for the user
-    roles: Joi.object().pattern(/.*/, Joi.string()).required().unknown(true),
-  }
-  if (!isSelf) {
-    schema = {
-      ...schema,
-      _id: Joi.string(),
-      _rev: Joi.string(),
-    }
-  }
-  return joiValidator.body(Joi.object(schema).required().unknown(true))
-}
-
 function buildInviteValidation() {
   // prettier-ignore
   return joiValidator.body(Joi.object({
@@ -69,7 +45,6 @@ router
     controller.save
   )
   .get("/api/global/users", adminOnly, controller.fetch)
-  .delete("/api/global/roles/:appId", adminOnly, controller.removeAppRole)
   .delete("/api/global/users/:id", adminOnly, controller.destroy)
   .get("/api/global/roles/:appId")
   .post(
@@ -79,11 +54,6 @@ router
     controller.invite
   )
   // non-global endpoints
-  .post(
-    "/api/global/users/self",
-    buildUserSaveValidation(true),
-    controller.updateSelf
-  )
   .post(
     "/api/global/users/invite/accept",
     buildInviteAcceptValidation(),
@@ -95,9 +65,15 @@ router
     buildAdminInitValidation(),
     controller.adminUser
   )
-  .get("/api/global/users/self", controller.getSelf)
   .get("/api/global/users/tenant/:id", controller.tenantUserLookup)
   // global endpoint but needs to come at end (blocks other endpoints otherwise)
   .get("/api/global/users/:id", adminOnly, controller.find)
+  // DEPRECATED - use new versions with self API
+  .get("/api/global/users/self", selfController.getSelf)
+  .post(
+    "/api/global/users/self",
+    buildUserSaveValidation(true),
+    selfController.updateSelf
+  )
 
 module.exports = router

packages/worker/src/api/routes/index.js

@@ -8,6 +8,7 @@ const roleRoutes = require("./global/roles")
 const sessionRoutes = require("./global/sessions")
 const environmentRoutes = require("./system/environment")
 const tenantsRoutes = require("./system/tenants")
+const selfRoutes = require("./global/self")
 
 exports.routes = [
   configRoutes,
@@ -20,4 +21,5 @@ exports.routes = [
   sessionRoutes,
   roleRoutes,
   environmentRoutes,
+  selfRoutes,
 ]