Merge branch 'develop' of github.com:Budibase/budibase into design-panel-v2

.github/workflows/budibase_ci.yml

@@ -43,13 +43,13 @@ jobs:
         verbose: true 
     - run: yarn test:e2e:ci
 
-    - name: Build and Push Staging Docker Image
+    - name: Build and Push Development Docker Image
       # Only run on push
       if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/develop' }}
       run: |
         docker login -u $DOCKER_USER -p $DOCKER_PASSWORD
         yarn build
-        yarn build:docker:staging
+        yarn build:docker:develop
       env:
         DOCKER_USER: ${{ secrets.DOCKER_USERNAME }}
         DOCKER_PASSWORD: ${{ secrets.DOCKER_API_KEY }}

hosting/envoy.dev.yaml.hbs

@@ -38,6 +38,11 @@ static_resources:
                 route:
                   cluster: server-dev
 
+              - match: { prefix: "/app/" }
+                route:
+                  cluster: server-dev
+                  prefix_rewrite: "/"
+
               # the below three cases are needed to make sure
               # all traffic prefixed for the builder is passed through
               # correctly.

hosting/envoy.yaml

@@ -139,5 +139,5 @@ static_resources:
             address:
               socket_address: 
                 address: watchtower-service
-                port_value: 6161
+                port_value: 8080
 

hosting/scripts/linux/release-to-docker-hub.sh

@@ -8,9 +8,5 @@ echo "Tagging images with SHA: $GITHUB_SHA and tag: $tag"
 docker tag app-service budibase/apps:$tag
 docker tag worker-service budibase/worker:$tag
 
-# Tag with git sha
+docker push budibase/apps:$tag 
-docker tag app-service budibase/apps:$GITHUB_SHA
+docker push budibase/worker:$tag
-docker tag worker-service budibase/worker:$GITHUB_SHA
-
-docker push budibase/apps
-docker push budibase/worker

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "0.9.51",
+  "version": "0.9.53",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

package.json

@@ -41,6 +41,6 @@
     "test:e2e": "lerna run cy:test",
     "test:e2e:ci": "lerna run cy:ci",
     "build:docker": "lerna run build:docker && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh && cd -",
-    "build:docker:staging": "lerna run build:docker && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh staging && cd -"
+    "build:docker:develop": "lerna run build:docker && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh develop && cd -"
   }
 }

packages/auth/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/auth",
-  "version": "0.9.51",
+  "version": "0.9.53",
   "description": "Authentication middlewares for budibase builder and apps",
   "main": "src/index.js",
   "author": "Budibase",

packages/auth/src/middleware/authenticated.js

@@ -22,6 +22,12 @@ function buildNoAuthRegex(patterns) {
   })
 }
 
+function finalise(ctx, { authenticated, user, internal } = {}) {
+  ctx.isAuthenticated = authenticated || false
+  ctx.user = user
+  ctx.internal = internal || false
+}
+
 module.exports = (noAuthPatterns = [], opts) => {
   const noAuthOptions = noAuthPatterns ? buildNoAuthRegex(noAuthPatterns) : []
   return async (ctx, next) => {
@@ -36,35 +42,39 @@ module.exports = (noAuthPatterns = [], opts) => {
       return next()
     }
     try {
-      const apiKey = ctx.request.headers["x-budibase-api-key"]
       // check the actual user is authenticated first
       const authCookie = getCookie(ctx, Cookies.Auth)
-
+      let authenticated = false,
-      // this is an internal request, no user made it
+        user = null,
-      if (apiKey && apiKey === env.INTERNAL_API_KEY) {
+        internal = false
-        ctx.isAuthenticated = true
+      if (authCookie) {
-        ctx.internal = true
-      } else if (authCookie) {
         try {
           const db = database.getDB(StaticDatabases.GLOBAL.name)
-          const user = await db.get(authCookie.userId)
+          user = await db.get(authCookie.userId)
           delete user.password
-          ctx.isAuthenticated = true
+          authenticated = true
-          ctx.user = user
         } catch (err) {
           // remove the cookie as the use does not exist anymore
           clearCookie(ctx, Cookies.Auth)
         }
       }
-      // be explicit
+      const apiKey = ctx.request.headers["x-budibase-api-key"]
-      if (ctx.isAuthenticated !== true) {
+      // this is an internal request, no user made it
-        ctx.isAuthenticated = false
+      if (!authenticated && apiKey && apiKey === env.INTERNAL_API_KEY) {
+        authenticated = true
+        internal = true
       }
+      // be explicit
+      if (authenticated !== true) {
+        authenticated = false
+      }
+      // isAuthenticated is a function, so use a variable to be able to check authed state
+      finalise(ctx, { authenticated, user, internal })
       return next()
     } catch (err) {
       // allow configuring for public access
       if (opts && opts.publicAllowed) {
-        ctx.isAuthenticated = false
+        finalise(ctx, { authenticated: false })
       } else {
         ctx.throw(err.status || 403, err)
       }

packages/bbui/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/bbui",
   "description": "A UI solution used in the different Budibase projects.",
-  "version": "0.9.51",
+  "version": "0.9.53",
   "license": "AGPL-3.0",
   "svelte": "src/index.js",
   "module": "dist/bbui.es.js",

packages/bbui/src/Table/StringRenderer.svelte

@@ -2,7 +2,7 @@
   export let value
 </script>
 
-<div>{value}</div>
+<div>{typeof value === "object" ? JSON.stringify(value) : value}</div>
 
 <style>
   div {

packages/builder/cypress/setup.js

@@ -20,6 +20,7 @@ process.env.MINIO_ACCESS_KEY = "budibase"
 process.env.MINIO_SECRET_KEY = "budibase"
 process.env.COUCH_DB_USER = "budibase"
 process.env.COUCH_DB_PASSWORD = "budibase"
+process.env.INTERNAL_API_KEY = "budibase"
 
 // Stop info logs polluting test outputs
 process.env.LOG_LEVEL = "error"

packages/builder/cypress/support/commands.js

@@ -37,7 +37,7 @@ Cypress.Commands.add("createApp", name => {
       cy.contains("Create app").click()
     })
     .then(() => {
-      cy.get("[data-cy=new-table]", {
+      cy.get(".selected > .content", {
         timeout: 20000,
       }).should("be.visible")
     })
@@ -51,7 +51,7 @@ Cypress.Commands.add("deleteApp", () => {
     .then(val => {
       console.log(val)
       if (val.length > 0) {
-        cy.get(".hoverable > use").click()
+        cy.get(".title > :nth-child(3) > .spectrum-Icon").click()
         cy.contains("Delete").click()
         cy.get(".spectrum-Button--warning").click()
       }
@@ -72,7 +72,8 @@ Cypress.Commands.add("createTestTableWithData", () => {
 
 Cypress.Commands.add("createTable", tableName => {
   // Enter table name
-  cy.get("[data-cy=new-table]").click()
+  cy.contains("Budibase DB").click()
+  cy.contains("Create new table").click()
   cy.get(".spectrum-Modal").within(() => {
     cy.get("input").first().type(tableName).blur()
     cy.get(".spectrum-ButtonGroup").contains("Create").click()

packages/builder/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/builder",
-  "version": "0.9.51",
+  "version": "0.9.53",
   "license": "AGPL-3.0",
   "private": true,
   "scripts": {
@@ -65,10 +65,10 @@
     }
   },
   "dependencies": {
-    "@budibase/bbui": "^0.9.51",
+    "@budibase/bbui": "^0.9.53",
-    "@budibase/client": "^0.9.51",
+    "@budibase/client": "^0.9.53",
     "@budibase/colorpicker": "1.1.2",
-    "@budibase/string-templates": "^0.9.51",
+    "@budibase/string-templates": "^0.9.53",
     "@sentry/browser": "5.19.1",
     "@spectrum-css/page": "^3.0.1",
     "@spectrum-css/vars": "^3.0.1",

packages/builder/src/App.svelte

@@ -5,6 +5,7 @@
   import { initialise } from "builderStore"
   import { NotificationDisplay } from "@budibase/bbui"
   import { parse, stringify } from "qs"
+  import HelpIcon from "components/common/HelpIcon.svelte"
 
   onMount(async () => {
     await initialise()
@@ -16,6 +17,7 @@
 <NotificationDisplay />
 <Router {routes} config={{ queryHandler }} />
 <div class="modal-container" />
+<HelpIcon />
 
 <style>
   .modal-container {

packages/builder/src/analytics.js

@@ -22,6 +22,7 @@ async function activate() {
   if (sentryConfigured) Sentry.init({ dsn: process.env.SENTRY_DSN })
   if (posthogConfigured) {
     posthog.init(process.env.POSTHOG_TOKEN, {
+      autocapture: false,
       api_host: process.env.POSTHOG_URL,
     })
     posthog.set_config({ persistence: "cookie" })

packages/builder/src/builderStore/store/screenTemplates/rowDetailScreen.js

@@ -85,7 +85,7 @@ const createScreen = table => {
     .customProps({
       dataSource: {
         label: table.name,
-        name: `all_${table._id}`,
+        name: table._id,
         tableId: table._id,
         type: "table",
       },

packages/builder/src/builderStore/store/screenTemplates/rowListScreen.js

@@ -76,7 +76,7 @@ const createScreen = table => {
     .customProps({
       dataSource: {
         label: table.name,
-        name: `all_${table._id}`,
+        name: table._id,
         tableId: table._id,
         type: "table",
       },

packages/builder/src/components/backend/DataTable/DataTable.svelte

@@ -17,31 +17,31 @@
   let data = []
   let loading = false
   $: isUsersTable = $tables.selected?._id === TableNames.USERS
-  $: title = $tables.selected.name
+  $: title = $tables.selected?.name
-  $: schema = $tables.selected.schema
+  $: schema = $tables.selected?.schema
   $: tableView = {
     schema,
     name: $views.selected?.name,
   }
+  $: type = $tables.selected?.type
+  $: isInternal = type === "internal"
 
   // Fetch rows for specified table
   $: {
-    if ($views.selected?.name?.startsWith("all_")) {
+    loading = true
-      loading = true
+    const loadingTableId = $tables.selected?._id
-      const loadingTableId = $tables.selected?._id
+    api.fetchDataForTable($tables.selected?._id).then(rows => {
-      api.fetchDataForView($views.selected).then(rows => {
+      loading = false
-        loading = false
 
-        // If we started a slow request then quickly change table, sometimes
+      // If we started a slow request then quickly change table, sometimes
-        // the old data overwrites the new data.
+      // the old data overwrites the new data.
-        // This check ensures that we don't do that.
+      // This check ensures that we don't do that.
-        if (loadingTableId !== $tables.selected?._id) {
+      if (loadingTableId !== $tables.selected?._id) {
-          return
+        return
-        }
+      }
 
-        data = rows || []
+      data = rows || []
-      })
+    })
-    }
   }
 </script>
 
@@ -50,11 +50,14 @@
   {schema}
   tableId={$tables.selected?._id}
   {data}
+  {type}
   allowEditing={true}
   bind:hideAutocolumns
   {loading}
 >
-  <CreateColumnButton />
+  {#if isInternal}
+    <CreateColumnButton />
+  {/if}
   {#if schema && Object.keys(schema).length > 0}
     {#if !isUsersTable}
       <CreateRowButton
@@ -62,13 +65,17 @@
         modalContentComponent={CreateEditRow}
       />
     {/if}
-    <CreateViewButton />
+    {#if isInternal}
+      <CreateViewButton />
+    {/if}
     <ManageAccessButton resourceId={$tables.selected?._id} />
     {#if isUsersTable}
       <EditRolesButton />
     {/if}
-    <HideAutocolumnButton bind:hideAutocolumns />
+    {#if isInternal}
+      <HideAutocolumnButton bind:hideAutocolumns />
+    {/if}
     <!-- always have the export last -->
-    <ExportButton view={tableView} />
+    <ExportButton view={$tables.selected?._id} />
   {/if}
 </Table>

packages/builder/src/components/backend/DataTable/ExternalDataSourceTable.svelte

@@ -6,12 +6,13 @@
 
   let loading = false
   let error = false
+  let type = "external"
 </script>
 
 {#if error}
   <div class="errors">{error}</div>
 {/if}
-<Table schema={query.schema} {data} {loading} rowCount={5} />
+<Table schema={query.schema} {data} {loading} {type} rowCount={5} />
 
 <style>
   .errors {

packages/builder/src/components/backend/DataTable/RelationshipDataTable.svelte

@@ -15,6 +15,7 @@
   $: linkedTable = $tables.list.find(table => table._id === linkedTableId)
   $: schema = linkedTable?.schema
   $: table = $tables.list.find(table => table._id === tableId)
+  $: type = table?.type
   $: fetchData(tableId, rowId)
   $: {
     let rowLabel = row?.[table?.primaryDisplay]
@@ -33,5 +34,5 @@
 </script>
 
 {#if row && row._id === rowId}
-  <Table {title} {schema} {data} />
+  <Table {title} {schema} {data} {type} />
 {/if}

packages/builder/src/components/backend/DataTable/Table.svelte

@@ -20,6 +20,7 @@
   export let loading = false
   export let hideAutocolumns
   export let rowCount
+  export let type
 
   let selectedRows = []
   let editableColumn
@@ -28,6 +29,7 @@
   let editColumnModal
   let customRenderers = []
 
+  $: isInternal = type !== "external"
   $: isUsersTable = tableId === TableNames.USERS
   $: data && resetSelectedRows()
   $: editRowComponent = isUsersTable ? CreateEditUser : CreateEditRow
@@ -73,9 +75,8 @@
   }
 
   const deleteRows = async () => {
-    await api.post(`/api/${tableId}/rows`, {
+    await api.delete(`/api/${tableId}/rows`, {
       rows: selectedRows,
-      type: "delete",
     })
     data = data.filter(row => !selectedRows.includes(row))
     notifications.success(`Successfully deleted ${selectedRows.length} rows`)
@@ -125,7 +126,7 @@
     bind:selectedRows
     allowSelectRows={allowEditing && !isUsersTable}
     allowEditRows={allowEditing}
-    allowEditColumns={allowEditing}
+    allowEditColumns={allowEditing && isInternal}
     showAutoColumns={!hideAutocolumns}
     on:editcolumn={e => editColumn(e.detail)}
     on:editrow={e => editRow(e.detail)}

packages/builder/src/components/backend/DataTable/ViewDataTable.svelte

@@ -11,19 +11,18 @@
   import HideAutocolumnButton from "./buttons/HideAutocolumnButton.svelte"
 
   export let view = {}
-  let hideAutocolumns = true
 
+  let hideAutocolumns = true
   let data = []
   let loading = false
+  let type = "internal"
 
   $: name = view.name
 
   // Fetch rows for specified view
   $: {
-    if (!name.startsWith("all_")) {
+    loading = true
-      loading = true
+    fetchViewData(name, view.field, view.groupBy, view.calculation)
-      fetchViewData(name, view.field, view.groupBy, view.calculation)
-    }
   }
 
   async function fetchViewData(name, field, groupBy, calculation) {
@@ -32,6 +31,7 @@
     const thisView = allTableViews.filter(
       views => views != null && views[name] != null
     )[0]
+
     // don't fetch view data if the view no longer exists
     if (!thisView) {
       return
@@ -57,6 +57,7 @@
   tableId={view.tableId}
   {data}
   {loading}
+  {type}
   allowEditing={!view?.calculation}
   bind:hideAutocolumns
 >

packages/builder/src/components/backend/DataTable/api.js

@@ -14,12 +14,15 @@ export async function saveRow(row, tableId) {
 }
 
 export async function deleteRow(row) {
-  const DELETE_ROWS_URL = `/api/${row.tableId}/rows/${row._id}/${row._rev}`
+  const DELETE_ROWS_URL = `/api/${row.tableId}/rows`
-  return api.delete(DELETE_ROWS_URL)
+  return api.delete(DELETE_ROWS_URL, {
+    _id: row._id,
+    _rev: row._rev,
+  })
 }
 
-export async function fetchDataForView(view) {
+export async function fetchDataForTable(tableId) {
-  const FETCH_ROWS_URL = `/api/views/${view.name}`
+  const FETCH_ROWS_URL = `/api/${tableId}/rows`
 
   const response = await api.get(FETCH_ROWS_URL)
   const json = await response.json()

packages/builder/src/components/backend/DataTable/buttons/DeleteRowsButton.svelte

@@ -9,7 +9,7 @@
 
   async function confirmDeletion() {
     await deleteRows()
-    modal.hide()
+    modal?.hide()
   }
 </script>
 

packages/builder/src/components/backend/DataTable/modals/CalculateModal.svelte

@@ -20,7 +20,9 @@
 
   export let view = {}
 
-  $: viewTable = $tables.list.find(({ _id }) => _id === $views.selected.tableId)
+  $: viewTable = $tables.list.find(
+    ({ _id }) => _id === $views.selected?.tableId
+  )
   $: fields =
     viewTable &&
     Object.keys(viewTable.schema).filter(

packages/builder/src/components/backend/DataTable/modals/ExportModal.svelte

@@ -20,7 +20,7 @@
   async function exportView() {
     download(
       `/api/views/export?view=${encodeURIComponent(
-        view.name
+        view
       )}&format=${exportFormat}`
     )
   }

packages/builder/src/components/backend/DataTable/modals/FilterModal.svelte

@@ -57,7 +57,9 @@
 
   export let view = {}
 
-  $: viewTable = $tables.list.find(({ _id }) => _id === $views.selected.tableId)
+  $: viewTable = $tables.list.find(
+    ({ _id }) => _id === $views.selected?.tableId
+  )
   $: fields = viewTable && Object.keys(viewTable.schema)
 
   function saveView() {

packages/builder/src/components/backend/DataTable/modals/GroupByModal.svelte

@@ -5,7 +5,9 @@
 
   export let view = {}
 
-  $: viewTable = $tables.list.find(({ _id }) => _id === $views.selected.tableId)
+  $: viewTable = $tables.list.find(
+    ({ _id }) => _id === $views.selected?.tableId
+  )
   $: fields =
     viewTable &&
     Object.entries(viewTable.schema)

packages/builder/src/components/backend/DatasourceNavigator/DatasourceNavigator.svelte

@@ -1,10 +1,12 @@
 <script>
   import { onMount } from "svelte"
   import { goto } from "@roxi/routify"
+  import { BUDIBASE_INTERNAL_DB } from "constants"
   import { database, datasources, queries } from "stores/backend"
   import EditDatasourcePopover from "./popovers/EditDatasourcePopover.svelte"
   import EditQueryPopover from "./popovers/EditQueryPopover.svelte"
   import NavItem from "components/common/NavItem.svelte"
+  import TableNavigator from "components/backend/TableNavigator/TableNavigator.svelte"
   import ICONS from "./icons"
 
   function selectDatasource(datasource) {
@@ -13,9 +15,6 @@
   }
 
   function onClickQuery(query) {
-    if ($queries.selected === query._id) {
-      return
-    }
     queries.select(query)
     $goto(`./datasource/${query.datasourceId}/${query._id}`)
   }
@@ -42,8 +41,13 @@
             width="18"
           />
         </div>
-        <EditDatasourcePopover {datasource} />
+        {#if datasource._id !== BUDIBASE_INTERNAL_DB}
+          <EditDatasourcePopover {datasource} />
+        {/if}
       </NavItem>
+
+      <TableNavigator sourceId={datasource._id} />
+
       {#each $queries.list.filter(query => query.datasourceId === datasource._id) as query}
         <NavItem
           indentLevel={1}

packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/index.svelte

@@ -6,7 +6,6 @@
 
   export let integration = {}
 
-  let schema
   let integrations = []
 
   async function fetchIntegrations() {
@@ -18,13 +17,18 @@
   }
 
   function selectIntegration(integrationType) {
-    schema = integrations[integrationType].datasource
+    const selected = integrations[integrationType]
+
+    // build the schema
+    const schema = {}
+    for (let key in selected.datasource) {
+      schema[key] = selected.datasource[key].default
+    }
+
     integration = {
       type: integrationType,
-      ...Object.keys(schema).reduce(
+      plus: selected.plus,
-        (acc, next) => ({ ...acc, [next]: schema[next].default }),
+      ...schema,
-        {}
-      ),
     }
   }
 

packages/builder/src/components/backend/DatasourceNavigator/icons/Budibase.svelte

@@ -0,0 +1,126 @@
+<script>
+  export let width = "100"
+  export let height = "100"
+</script>
+
+<svg
+  version="1.1"
+  id="Layer_1"
+  xmlns="http://www.w3.org/2000/svg"
+  xmlns:xlink="http://www.w3.org/1999/xlink"
+  x="0px"
+  y="0px"
+  viewBox="0 0 48 48"
+  style="enable-background:new 0 0 48 48;"
+  xml:space="preserve"
+  {height}
+  {width}
+>
+  <style type="text/css">
+    .st0 {
+      fill: #393c44;
+    }
+    .st1 {
+      fill: #ffffff;
+    }
+    .st2 {
+      fill: #4285f4;
+    }
+  </style>
+  <rect x="-152.17" y="-24.17" class="st0" width="96.17" height="96.17" />
+  <path
+    class="st1"
+    d="M-83.19,48h-41.79c-1.76,0-3.19-1.43-3.19-3.19V3.02c0-1.76,1.43-3.19,3.19-3.19h41.79
+	c1.76,0,3.19,1.43,3.19,3.19v41.79C-80,46.57-81.43,48-83.19,48z"
+  />
+  <g>
+    <g>
+      <path
+        class="st0"
+        d="M-99.62,12.57v9.94c1.15-1.21,2.59-1.81,4.32-1.81c1.03,0,1.97,0.19,2.82,0.58c0.86,0.39,1.59,0.91,2.19,1.57
+			c0.6,0.66,1.08,1.43,1.42,2.32c0.34,0.89,0.51,1.84,0.51,2.85c0,1.03-0.18,1.99-0.53,2.89c-0.35,0.9-0.84,1.68-1.47,2.35
+			c-0.63,0.67-1.37,1.19-2.23,1.58c-0.86,0.39-1.78,0.58-2.77,0.58c-1.8,0-3.22-0.66-4.27-1.97V35h-4.89V12.57H-99.62z
+			 M-93.46,28.11c0-0.43-0.08-0.84-0.24-1.23c-0.16-0.39-0.39-0.72-0.68-1.01c-0.29-0.29-0.62-0.52-1-0.69
+			c-0.38-0.17-0.79-0.26-1.24-0.26c-0.43,0-0.84,0.08-1.22,0.24c-0.38,0.16-0.71,0.39-0.99,0.68c-0.28,0.29-0.5,0.63-0.68,1.01
+			c-0.17,0.39-0.26,0.8-0.26,1.23c0,0.43,0.08,0.84,0.24,1.22c0.16,0.38,0.39,0.71,0.68,0.99c0.29,0.28,0.63,0.5,1.01,0.68
+			c0.39,0.17,0.8,0.26,1.23,0.26c0.43,0,0.84-0.08,1.22-0.24c0.38-0.16,0.71-0.39,0.99-0.68c0.28-0.29,0.5-0.62,0.68-1
+			C-93.55,28.92-93.46,28.52-93.46,28.11z"
+      />
+    </g>
+    <g>
+      <path
+        class="st0"
+        d="M-114.76,12.57v9.94c1.15-1.21,2.59-1.81,4.32-1.81c1.03,0,1.97,0.19,2.82,0.58
+			c0.86,0.39,1.59,0.91,2.19,1.57c0.6,0.66,1.08,1.43,1.42,2.32c0.34,0.89,0.51,1.84,0.51,2.85c0,1.03-0.18,1.99-0.53,2.89
+			c-0.35,0.9-0.84,1.68-1.47,2.35c-0.63,0.67-1.37,1.19-2.23,1.58c-0.86,0.39-1.78,0.58-2.77,0.58c-1.8,0-3.22-0.66-4.27-1.97V35
+			h-4.89V12.57H-114.76z M-108.6,28.11c0-0.43-0.08-0.84-0.24-1.23c-0.16-0.39-0.39-0.72-0.68-1.01c-0.29-0.29-0.62-0.52-1-0.69
+			c-0.38-0.17-0.79-0.26-1.24-0.26c-0.43,0-0.84,0.08-1.22,0.24c-0.38,0.16-0.71,0.39-0.99,0.68c-0.28,0.29-0.5,0.63-0.68,1.01
+			c-0.17,0.39-0.26,0.8-0.26,1.23c0,0.43,0.08,0.84,0.24,1.22c0.16,0.38,0.39,0.71,0.68,0.99c0.29,0.28,0.63,0.5,1.01,0.68
+			c0.39,0.17,0.8,0.26,1.23,0.26c0.43,0,0.84-0.08,1.22-0.24c0.38-0.16,0.71-0.39,0.99-0.68c0.28-0.29,0.5-0.62,0.68-1
+			C-108.68,28.92-108.6,28.52-108.6,28.11z"
+      />
+    </g>
+  </g>
+  <path
+    class="st2"
+    d="M44.81,159H3.02c-1.76,0-3.19-1.43-3.19-3.19v-41.79c0-1.76,1.43-3.19,3.19-3.19h41.79
+	c1.76,0,3.19,1.43,3.19,3.19v41.79C48,157.57,46.57,159,44.81,159z"
+  />
+  <g>
+    <g>
+      <path
+        class="st1"
+        d="M28.38,123.57v9.94c1.15-1.21,2.59-1.81,4.32-1.81c1.03,0,1.97,0.19,2.82,0.58c0.86,0.39,1.59,0.91,2.19,1.57
+			c0.6,0.66,1.08,1.43,1.42,2.32c0.34,0.89,0.51,1.84,0.51,2.85c0,1.03-0.18,1.99-0.53,2.89c-0.35,0.9-0.84,1.68-1.47,2.35
+			c-0.63,0.67-1.37,1.19-2.23,1.58c-0.86,0.39-1.78,0.58-2.77,0.58c-1.8,0-3.22-0.66-4.27-1.97V146h-4.89v-22.43H28.38z
+			 M34.54,139.11c0-0.43-0.08-0.84-0.24-1.23c-0.16-0.39-0.39-0.72-0.68-1.01c-0.29-0.29-0.62-0.52-1-0.69
+			c-0.38-0.17-0.79-0.26-1.24-0.26c-0.43,0-0.84,0.08-1.22,0.24c-0.38,0.16-0.71,0.39-0.99,0.68c-0.28,0.29-0.5,0.63-0.68,1.01
+			c-0.17,0.39-0.26,0.8-0.26,1.23c0,0.43,0.08,0.84,0.24,1.22c0.16,0.38,0.39,0.71,0.68,0.99c0.29,0.28,0.63,0.5,1.01,0.68
+			c0.39,0.17,0.8,0.26,1.23,0.26c0.43,0,0.84-0.08,1.22-0.24c0.38-0.16,0.71-0.39,0.99-0.68c0.28-0.29,0.5-0.62,0.68-1
+			C34.45,139.92,34.54,139.52,34.54,139.11z"
+      />
+    </g>
+    <g>
+      <path
+        class="st1"
+        d="M13.24,123.57v9.94c1.15-1.21,2.59-1.81,4.32-1.81c1.03,0,1.97,0.19,2.82,0.58c0.86,0.39,1.59,0.91,2.19,1.57
+			c0.6,0.66,1.08,1.43,1.42,2.32c0.34,0.89,0.51,1.84,0.51,2.85c0,1.03-0.18,1.99-0.53,2.89c-0.35,0.9-0.84,1.68-1.47,2.35
+			c-0.63,0.67-1.37,1.19-2.23,1.58c-0.86,0.39-1.78,0.58-2.77,0.58c-1.8,0-3.22-0.66-4.27-1.97V146H8.35v-22.43H13.24z M19.4,139.11
+			c0-0.43-0.08-0.84-0.24-1.23c-0.16-0.39-0.39-0.72-0.68-1.01c-0.29-0.29-0.62-0.52-1-0.69c-0.38-0.17-0.79-0.26-1.24-0.26
+			c-0.43,0-0.84,0.08-1.22,0.24c-0.38,0.16-0.71,0.39-0.99,0.68c-0.28,0.29-0.5,0.63-0.68,1.01c-0.17,0.39-0.26,0.8-0.26,1.23
+			c0,0.43,0.08,0.84,0.24,1.22c0.16,0.38,0.39,0.71,0.68,0.99c0.29,0.28,0.63,0.5,1.01,0.68c0.39,0.17,0.8,0.26,1.23,0.26
+			c0.43,0,0.84-0.08,1.22-0.24c0.38-0.16,0.71-0.39,0.99-0.68c0.28-0.29,0.5-0.62,0.68-1C19.32,139.92,19.4,139.52,19.4,139.11z"
+      />
+    </g>
+  </g>
+  <g>
+    <path
+      class="st0"
+      d="M44,48H4c-2.21,0-4-1.79-4-4V4c0-2.21,1.79-4,4-4h40c2.21,0,4,1.79,4,4v40C48,46.21,46.21,48,44,48z"
+    />
+    <g>
+      <path
+        class="st1"
+        d="M28.48,12v10.44c1.18-1.27,2.65-1.9,4.42-1.9c1.05,0,2.01,0.2,2.89,0.61c0.87,0.41,1.62,0.96,2.24,1.65
+			c0.62,0.69,1.1,1.5,1.45,2.44c0.35,0.94,0.52,1.93,0.52,2.99c0,1.08-0.18,2.09-0.54,3.04c-0.36,0.95-0.86,1.77-1.51,2.47
+			c-0.64,0.7-1.4,1.25-2.28,1.66C34.8,35.8,33.86,36,32.84,36c-1.84,0-3.3-0.69-4.37-2.07v1.62h-5V12H28.48z M34.78,28.31
+			c0-0.45-0.08-0.88-0.25-1.29c-0.17-0.41-0.4-0.76-0.69-1.06c-0.3-0.3-0.64-0.54-1.02-0.72c-0.39-0.18-0.81-0.27-1.27-0.27
+			c-0.44,0-0.86,0.09-1.24,0.26c-0.39,0.17-0.72,0.41-1.01,0.71c-0.29,0.3-0.52,0.66-0.69,1.06c-0.18,0.41-0.26,0.84-0.26,1.29
+			s0.08,0.88,0.25,1.28c0.17,0.4,0.4,0.74,0.69,1.04c0.29,0.29,0.64,0.53,1.04,0.71c0.4,0.18,0.82,0.27,1.26,0.27
+			c0.44,0,0.86-0.09,1.24-0.26c0.39-0.17,0.72-0.41,1.01-0.71c0.29-0.3,0.52-0.65,0.69-1.05C34.69,29.16,34.78,28.75,34.78,28.31z"
+      />
+    </g>
+    <g>
+      <path
+        class="st1"
+        d="M13,12v10.44c1.18-1.27,2.65-1.9,4.42-1.9c1.05,0,2.01,0.2,2.89,0.61c0.87,0.41,1.62,0.96,2.24,1.65
+			c0.62,0.69,1.1,1.5,1.45,2.44c0.35,0.94,0.52,1.93,0.52,2.99c0,1.08-0.18,2.09-0.54,3.04c-0.36,0.95-0.86,1.77-1.51,2.47
+			c-0.64,0.7-1.4,1.25-2.28,1.66C19.32,35.8,18.38,36,17.37,36c-1.84,0-3.3-0.69-4.37-2.07v1.62H8V12H13z M19.3,28.31
+			c0-0.45-0.08-0.88-0.25-1.29c-0.17-0.41-0.4-0.76-0.69-1.06c-0.3-0.3-0.64-0.54-1.02-0.72c-0.39-0.18-0.81-0.27-1.27-0.27
+			c-0.44,0-0.86,0.09-1.24,0.26c-0.39,0.17-0.72,0.41-1.01,0.71c-0.29,0.3-0.52,0.66-0.69,1.06c-0.18,0.41-0.26,0.84-0.26,1.29
+			s0.08,0.88,0.25,1.28c0.17,0.4,0.4,0.74,0.69,1.04c0.29,0.29,0.64,0.53,1.04,0.71c0.4,0.18,0.82,0.27,1.26,0.27
+			c0.44,0,0.86-0.09,1.24-0.26c0.39-0.17,0.72-0.41,1.01-0.71c0.29-0.3,0.52-0.65,0.69-1.05C19.21,29.16,19.3,28.75,19.3,28.31z"
+      />
+    </g>
+  </g>
+</svg>

packages/builder/src/components/backend/DatasourceNavigator/icons/index.js

@@ -9,8 +9,10 @@ import SqlServer from "./SQLServer.svelte"
 import MySQL from "./MySQL.svelte"
 import ArangoDB from "./ArangoDB.svelte"
 import Rest from "./Rest.svelte"
+import Budibase from "./Budibase.svelte"
 
 export default {
+  BUDIBASE: Budibase,
   POSTGRES: Postgres,
   DYNAMODB: DynamoDB,
   MONGODB: MongoDB,

packages/builder/src/components/backend/DatasourceNavigator/modals/CreateDatasourceModal.svelte

@@ -23,16 +23,17 @@
   }
 
   async function saveDatasource() {
-    const { type, ...config } = integration
+    const { type, plus, ...config } = integration
 
     // Create datasource
     const response = await datasources.save({
       name,
       source: type,
       config,
+      plus,
     })
     notifications.success(`Datasource ${name} created successfully.`)
-    analytics.captureEvent("Datasource Created", { name })
+    analytics.captureEvent("Datasource Created", { name, type })
 
     // Navigate to new datasource
     $goto(`./datasource/${response._id}`)

packages/builder/src/components/backend/TableNavigator/TableNavigator.svelte

@@ -6,6 +6,8 @@
   import EditViewPopover from "./popovers/EditViewPopover.svelte"
   import NavItem from "components/common/NavItem.svelte"
 
+  export let sourceId
+
   $: selectedView = $views.selected && $views.selected.name
 
   function selectTable(table) {
@@ -31,12 +33,13 @@
 
 {#if $database?._id}
   <div class="hierarchy-items-container">
-    {#each $tables.list as table, idx}
+    {#each $tables.list.filter(table => table.sourceId === sourceId) as table, idx}
       <NavItem
+        indentLevel={1}
         border={idx > 0}
         icon={table._id === TableNames.USERS ? "UserGroup" : "Table"}
         text={table.name}
-        selected={selectedView === `all_${table._id}`}
+        selected={$tables.selected?._id === table._id}
         on:click={() => selectTable(table)}
       >
         {#if table._id !== TableNames.USERS}
@@ -45,7 +48,7 @@
       </NavItem>
       {#each Object.keys(table.views || {}) as viewName, idx (idx)}
         <NavItem
-          indentLevel={1}
+          indentLevel={2}
           icon="Remove"
           text={viewName}
           selected={selectedView === viewName}

packages/builder/src/components/backend/TableNavigator/modals/CreateTableModal.svelte

@@ -91,7 +91,7 @@
     }
 
     // Navigate to new table
-    $goto(`./table/${table._id}`)
+    $goto(`../../table/${table._id}`)
   }
 </script>
 

packages/builder/src/components/backend/TableNavigator/popovers/EditTablePopover.svelte

@@ -1,15 +1,15 @@
 <script>
   import { goto } from "@roxi/routify"
-  import { store, allScreens } from "builderStore"
+  import { allScreens, store } from "builderStore"
   import { tables } from "stores/backend"
-  import { notifications } from "@budibase/bbui"
   import {
     ActionMenu,
-    MenuItem,
     Icon,
+    Input,
+    MenuItem,
     Modal,
     ModalContent,
-    Input,
+    notifications,
   } from "@budibase/bbui"
   import ConfirmDialog from "components/common/ConfirmDialog.svelte"
 
@@ -22,9 +22,12 @@
   let templateScreens
   let willBeDeleted
 
+  $: external = table?.type === "external"
+
   function showDeleteModal() {
-    const screens = $allScreens
+    templateScreens = $allScreens.filter(
-    templateScreens = screens.filter(screen => screen.autoTableId === table._id)
+      screen => screen.autoTableId === table._id
+    )
     willBeDeleted = ["All table data"].concat(
       templateScreens.map(screen => `Screen ${screen.props._instanceName}`)
     )
@@ -61,7 +64,9 @@
     <Icon s hoverable name="MoreSmallList" />
   </div>
   <MenuItem icon="Edit" on:click={editorModal.show}>Edit</MenuItem>
-  <MenuItem icon="Delete" on:click={showDeleteModal}>Delete</MenuItem>
+  {#if !external}
+    <MenuItem icon="Delete" on:click={showDeleteModal}>Delete</MenuItem>
+  {/if}
 </ActionMenu>
 
 <Modal bind:this={editorModal}>

packages/builder/src/components/common/HelpIcon.svelte

@@ -0,0 +1,17 @@
+<script>
+  import { Icon } from "@budibase/bbui"
+</script>
+
+<a target="_blank" href="https://github.com/Budibase/budibase/discussions">
+  <Icon hoverable name="Help" size="XXL" />
+</a>
+
+<style>
+  a {
+    color: inherit;
+    position: absolute;
+    bottom: var(--spacing-m);
+    right: var(--spacing-m);
+    border-radius: 55%;
+  }
+</style>

packages/builder/src/components/deploy/DeployModal.svelte

@@ -1,10 +1,10 @@
 <script>
   import { onMount, onDestroy } from "svelte"
   import { Button, Modal, notifications, ModalContent } from "@budibase/bbui"
+  import FeedbackIframe from "../feedback/FeedbackIframe.svelte"
   import { store } from "builderStore"
   import api from "builderStore/api"
   import analytics from "analytics"
-  import FeedbackIframe from "components/feedback/FeedbackIframe.svelte"
 
   const DeploymentStatus = {
     SUCCESS: "SUCCESS",
@@ -29,10 +29,6 @@
       } else {
         notifications.success(`Application published successfully`)
       }
-
-      if (analytics.requestFeedbackOnDeploy()) {
-        feedbackModal.show()
-      }
     } catch (err) {
       analytics.captureException(err)
       notifications.error(`Error publishing app: ${err}`)

packages/builder/src/components/design/PropertiesPanel/PropertyControls/DataSourceSelect.svelte

@@ -39,7 +39,7 @@
     type: "table",
   }))
   $: views = $tablesStore.list.reduce((acc, cur) => {
-    let viewsArr = Object.entries(cur.views).map(([key, value]) => ({
+    let viewsArr = Object.entries(cur.views || {}).map(([key, value]) => ({
       label: key,
       name: key,
       ...value,

packages/builder/src/components/design/PropertiesPanel/PropertyControls/EventsEditor/actions/LogOut.svelte

@@ -0,0 +1,14 @@
+<script>
+  import { Body } from "@budibase/bbui"
+</script>
+
+<div class="root">
+  <Body size="S">This action doesn't require any additional settings.</Body>
+</div>
+
+<style>
+  .root {
+    max-width: 800px;
+    margin: 0 auto;
+  }
+</style>

packages/builder/src/components/design/PropertiesPanel/PropertyControls/EventsEditor/actions/index.js

@@ -4,6 +4,7 @@ import DeleteRow from "./DeleteRow.svelte"
 import ExecuteQuery from "./ExecuteQuery.svelte"
 import TriggerAutomation from "./TriggerAutomation.svelte"
 import ValidateForm from "./ValidateForm.svelte"
+import LogOut from "./LogOut.svelte"
 
 // Defines which actions are available to configure in the front end.
 // Unfortunately the "name" property is used as the identifier so please don't
@@ -37,4 +38,8 @@ export default [
     name: "Validate Form",
     component: ValidateForm,
   },
+  {
+    name: "Log Out",
+    component: LogOut,
+  },
 ]

packages/builder/src/constants/index.js

@@ -31,3 +31,5 @@ export const LAYOUT_NAMES = {
     PUBLIC: "layout_private_master",
   },
 }
+
+export const BUDIBASE_INTERNAL_DB = "bb_internal"

packages/builder/src/pages/builder/app/[application]/data/_layout.svelte

@@ -1,49 +1,27 @@
 <script>
-  import { isActive, goto } from "@roxi/routify"
+  import { goto, params } from "@roxi/routify"
   import { Icon, Modal, Tabs, Tab } from "@budibase/bbui"
-  import TableNavigator from "components/backend/TableNavigator/TableNavigator.svelte"
+  import { BUDIBASE_INTERNAL_DB } from "constants"
   import DatasourceNavigator from "components/backend/DatasourceNavigator/DatasourceNavigator.svelte"
   import CreateDatasourceModal from "components/backend/DatasourceNavigator/modals/CreateDatasourceModal.svelte"
-  import CreateTableModal from "components/backend/TableNavigator/modals/CreateTableModal.svelte"
-
-  const tabs = [
-    {
-      title: "Internal",
-      key: "table",
-    },
-    {
-      title: "External",
-      key: "datasource",
-    },
-  ]
-
-  let selected = $isActive("./datasource") ? "External" : "Internal"
-
-  function selectFirstTableOrSource({ detail }) {
-    const { key } = tabs.find(t => t.title === detail)
-    if (key === "datasource") {
-      $goto("./datasource")
-    } else {
-      $goto("./table")
-    }
-  }
 
+  let selected = "Sources"
   let modal
+
+  $: isExternal =
+    $params.selectedDatasource &&
+    $params.selectedDatasource !== BUDIBASE_INTERNAL_DB
+
+  function selectFirstDatasource() {
+    $goto("./table")
+  }
 </script>
 
 <!-- routify:options index=0 -->
 <div class="root">
   <div class="nav">
-    <Tabs {selected} on:select={selectFirstTableOrSource}>
+    <Tabs {selected} on:select={selectFirstDatasource}>
-      <Tab title="Internal">
+      <Tab title="Sources">
-        <div class="tab-content-padding">
-          <TableNavigator />
-          <Modal bind:this={modal}>
-            <CreateTableModal />
-          </Modal>
-        </div>
-      </Tab>
-      <Tab title="External">
         <div class="tab-content-padding">
           <DatasourceNavigator />
           <Modal bind:this={modal}>
@@ -54,7 +32,7 @@
     </Tabs>
     <div
       class="add-button"
-      data-cy={`new-${selected === "External" ? "datasource" : "table"}`}
+      data-cy={`new-${isExternal ? "datasource" : "table"}`}
     >
       <Icon hoverable name="AddCircle" on:click={modal.show} />
     </div>

packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/[query]/_layout.svelte

@@ -1,13 +1 @@
-<script>
-  import { params } from "@roxi/routify"
-  import { queries } from "stores/backend"
-
-  if ($params.query) {
-    const query = $queries.list.find(m => m._id === $params.query)
-    if (query) {
-      queries.select(query)
-    }
-  }
-</script>
-
 <slot />

packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/index.svelte

@@ -1,7 +1,7 @@
 <script>
   import { goto, beforeUrlChange } from "@roxi/routify"
   import { Button, Heading, Body, Divider, Layout } from "@budibase/bbui"
-  import { datasources, integrations, queries } from "stores/backend"
+  import { datasources, integrations, queries, tables } from "stores/backend"
   import { notifications } from "@budibase/bbui"
   import IntegrationConfigForm from "components/backend/DatasourceNavigator/TableIntegrationMenu/IntegrationConfigForm.svelte"
   import ICONS from "components/backend/DatasourceNavigator/icons"
@@ -13,10 +13,25 @@
   $: integration = datasource && $integrations[datasource.source]
 
   async function saveDatasource() {
-    // Create datasource
+    try {
-    await datasources.save(datasource)
+      // Create datasource
-    notifications.success(`Datasource ${name} saved successfully.`)
+      await datasources.save(datasource)
-    unsaved = false
+      notifications.success(`Datasource ${name} updated successfully.`)
+      unsaved = false
+    } catch (err) {
+      notifications.error(`Error saving datasource: ${err}`)
+    }
+  }
+
+  async function updateDatasourceSchema() {
+    try {
+      await datasources.updateSchema(datasource)
+      notifications.success(`Datasource ${name} tables updated successfully.`)
+      unsaved = false
+      await tables.fetch()
+    } catch (err) {
+      notifications.error(`Error updating datasource schema: ${err}`)
+    }
   }
 
   function onClickQuery(query) {
@@ -24,6 +39,11 @@
     $goto(`./${query._id}`)
   }
 
+  function onClickTable(table) {
+    tables.select(table)
+    $goto(`../../table/${table._id}`)
+  }
+
   function setUnsaved() {
     unsaved = true
   }
@@ -39,7 +59,7 @@
   })
 </script>
 
-{#if datasource}
+{#if datasource && integration}
   <section>
     <Layout>
       <header>
@@ -66,6 +86,34 @@
           on:change={setUnsaved}
         />
       </div>
+      {#if datasource.plus}
+        <Divider />
+        <div class="query-header">
+          <Heading size="S">Tables</Heading>
+          <Button primary on:click={updateDatasourceSchema}
+            >Fetch Tables From Database</Button
+          >
+        </div>
+        <Body>
+          This datasource can determine tables automatically. Budibase can fetch
+          your tables directly from the database and you can use them without
+          having to write any queries at all.
+        </Body>
+        <div class="query-list">
+          {#if datasource.entities}
+            {#each Object.keys(datasource.entities) as entity}
+              <div
+                class="query-list-item"
+                on:click={() => onClickTable(datasource.entities[entity])}
+              >
+                <p class="query-name">{entity}</p>
+                <p>Primary Key: {datasource.entities[entity].primary}</p>
+                <p>→</p>
+              </div>
+            {/each}
+          {/if}
+        </div>
+      {/if}
       <Divider />
       <div class="query-header">
         <Heading size="S">Queries</Heading>

packages/builder/src/pages/builder/app/[application]/data/datasource/bb_internal/index.svelte

@@ -0,0 +1,81 @@
+<script>
+  import { Button, Heading, Body, Layout, Modal, Divider } from "@budibase/bbui"
+  import CreateTableModal from "components/backend/TableNavigator/modals/CreateTableModal.svelte"
+  import ICONS from "components/backend/DatasourceNavigator/icons"
+  import { tables } from "stores/backend"
+  import { goto } from "@roxi/routify"
+
+  let modal
+</script>
+
+<Modal bind:this={modal}>
+  <CreateTableModal />
+</Modal>
+
+<section>
+  <Layout>
+    <header>
+      <svelte:component this={ICONS.BUDIBASE} height="26" width="26" />
+      <Heading size="M">Budibase Internal</Heading>
+    </header>
+    <Body size="S" grey lh
+      >Budibase internal tables are part of your app, the data will be stored in
+      your apps context.</Body
+    >
+    <Divider />
+    <Heading size="S">Tables</Heading>
+    <div class="table-list">
+      {#each $tables.list.filter(table => table.type !== "external") as table}
+        <div
+          class="table-list-item"
+          on:click={$goto(`../../table/${table._id}`)}
+        >
+          <Body size="S">{table.name}</Body>
+          {#if table.primaryDisplay}
+            <Body size="S">display column: {table.primaryDisplay}</Body>
+          {/if}
+        </div>
+      {/each}
+    </div>
+    <div>
+      <Button cta on:click={modal.show}>Create new table</Button>
+    </div>
+  </Layout>
+</section>
+
+<style>
+  section {
+    margin: 0 auto;
+    width: 640px;
+  }
+
+  header {
+    margin: 0 0 var(--spacing-xs) 0;
+    display: flex;
+    gap: var(--spacing-l);
+    align-items: center;
+  }
+
+  .table-list {
+    display: flex;
+    flex-direction: column;
+    gap: var(--spacing-m);
+  }
+
+  .table-list-item {
+    border-radius: var(--border-radius-m);
+    background: var(--background);
+    border: var(--border-dark);
+    display: grid;
+    grid-template-columns: 2fr 0.75fr 20px;
+    align-items: center;
+    padding: var(--spacing-m);
+    gap: var(--layout-xs);
+    transition: 200ms background ease;
+  }
+
+  .table-list-item:hover {
+    background: var(--grey-1);
+    cursor: pointer;
+  }
+</style>

packages/builder/src/pages/builder/app/[application]/data/datasource/bb_internal/table/[selectedTable]/_layout.svelte

@@ -0,0 +1,13 @@
+<script>
+  import { params } from "@roxi/routify"
+  import { tables } from "stores/backend"
+
+  if ($params.selectedTable) {
+    const table = $tables.list.find(m => m._id === $params.selectedTable)
+    if (table) {
+      tables.select(table)
+    }
+  }
+</script>
+
+<slot />

packages/builder/src/pages/builder/app/[application]/data/datasource/bb_internal/table/[selectedTable]/index.svelte

@@ -0,0 +1,16 @@
+<script>
+  import TableDataTable from "components/backend/DataTable/DataTable.svelte"
+  import { tables, database } from "stores/backend"
+</script>
+
+{#if $database?._id && $tables?.selected?.name}
+  <TableDataTable />
+{:else}<i>Create your first table to start building</i>{/if}
+
+<style>
+  i {
+    font-size: var(--font-size-m);
+    color: var(--grey-5);
+    margin-top: 2px;
+  }
+</style>

packages/builder/src/pages/builder/app/[application]/data/datasource/bb_internal/table/[selectedTable]/relationship/[selectedRow]/[selectedField]/index.svelte

@@ -0,0 +1,10 @@
+<script>
+  import { params } from "@roxi/routify"
+  import RelationshipDataTable from "components/backend/DataTable/RelationshipDataTable.svelte"
+</script>
+
+<RelationshipDataTable
+  tableId={$params.selectedTable}
+  rowId={$params.selectedRow}
+  fieldName={decodeURI($params.selectedField)}
+/>

packages/builder/src/pages/builder/app/[application]/data/datasource/bb_internal/table/[selectedTable]/relationship/[selectedRow]/index.svelte

@@ -0,0 +1,6 @@
+<script>
+  import { goto } from "@roxi/routify"
+  $goto("../../")
+</script>
+
+<!-- routify:options index=false -->

packages/builder/src/pages/builder/app/[application]/data/datasource/bb_internal/table/[selectedTable]/relationship/index.svelte

@@ -0,0 +1,6 @@
+<script>
+  import { goto } from "@roxi/routify"
+  $goto("../")
+</script>
+
+<!-- routify:options index=false -->

packages/builder/src/pages/builder/app/[application]/data/datasource/bb_internal/table/_layout.svelte

@@ -0,0 +1,19 @@
+<script>
+  import { tables } from "stores/backend"
+  import { goto, leftover } from "@roxi/routify"
+  import { onMount } from "svelte"
+
+  onMount(async () => {
+    // navigate to first table in list, if not already selected
+    // and this is the final url (i.e. no selectedTable)
+    if (
+      !$leftover &&
+      $tables.list.length > 0
+      // (!$tables.selected || !$tables.selected._id)
+    ) {
+      $goto(`./${$tables.list[0]._id}`)
+    }
+  })
+</script>
+
+<slot />

packages/builder/src/pages/builder/app/[application]/data/datasource/bb_internal/table/index.svelte

@@ -0,0 +1,21 @@
+<script>
+  import { goto } from "@roxi/routify"
+  import { onMount } from "svelte"
+  import { tables } from "stores/backend"
+
+  onMount(async () => {
+    $tables.list.length > 0 && $goto(`./${$tables.list[0]._id}`)
+  })
+</script>
+
+{#if $tables.list.length === 0}
+  <i>Create your first table to start building</i>
+{:else}<i>Select a table to edit</i>{/if}
+
+<style>
+  i {
+    font-size: var(--font-size-m);
+    color: var(--grey-5);
+    margin-top: 2px;
+  }
+</style>

packages/builder/src/pages/builder/auth/forgot.svelte

@@ -9,6 +9,7 @@
   } from "@budibase/bbui"
   import { organisation, auth } from "stores/portal"
   import Logo from "assets/bb-emblem.svg"
+  import { onMount } from "svelte"
 
   let email = ""
 
@@ -20,6 +21,10 @@
       notifications.error("Unable to send reset password link")
     }
   }
+
+  onMount(async () => {
+    await organisation.init()
+  })
 </script>
 
 <div class="login">

packages/builder/src/pages/builder/auth/login.svelte

@@ -10,13 +10,16 @@
     notifications,
   } from "@budibase/bbui"
   import { goto, params } from "@roxi/routify"
-  import { auth } from "stores/portal"
+  import { auth, organisation } from "stores/portal"
   import GoogleButton from "./_components/GoogleButton.svelte"
   import Logo from "assets/bb-emblem.svg"
+  import { onMount } from "svelte"
 
   let username = ""
   let password = ""
 
+  $: company = $organisation.company || "Budibase"
+
   async function login() {
     try {
       await auth.login({
@@ -43,6 +46,10 @@
   function handleKeydown(evt) {
     if (evt.key === "Enter") login()
   }
+
+  onMount(async () => {
+    await organisation.init()
+  })
 </script>
 
 <svelte:window on:keydown={handleKeydown} />
@@ -50,8 +57,8 @@
   <div class="main">
     <Layout>
       <Layout noPadding justifyItems="center">
-        <img alt="logo" src={Logo} />
+        <img alt="logo" src={$organisation.logoUrl || Logo} />
-        <Heading>Sign in to Budibase</Heading>
+        <Heading>Sign in to {company}</Heading>
       </Layout>
       <GoogleButton />
       <Divider noGrid />
@@ -66,7 +73,7 @@
         />
       </Layout>
       <Layout gap="XS" noPadding>
-        <Button cta on:click={login}>Sign in to Budibase</Button>
+        <Button cta on:click={login}>Sign in to {company}</Button>
         <ActionButton quiet on:click={() => $goto("./forgot")}>
           Forgot password?
         </ActionButton>

packages/builder/src/pages/builder/auth/reset.svelte

@@ -2,8 +2,9 @@
   import { Body, Button, Heading, Layout, notifications } from "@budibase/bbui"
   import { goto, params } from "@roxi/routify"
   import PasswordRepeatInput from "components/common/users/PasswordRepeatInput.svelte"
-  import { auth } from "stores/portal"
+  import { auth, organisation } from "stores/portal"
   import Logo from "assets/bb-emblem.svg"
+  import { onMount } from "svelte"
 
   const resetCode = $params["?code"]
   let password, error
@@ -28,13 +29,17 @@
       notifications.error("Unable to reset password")
     }
   }
+
+  onMount(async () => {
+    await organisation.init()
+  })
 </script>
 
 <div class="login">
   <div class="main">
     <Layout>
       <Layout noPadding justifyItems="center">
-        <img src={Logo} alt="Organisation logo" />
+        <img src={$organisation.logoUrl || Logo} alt="Organisation logo" />
       </Layout>
       <Layout gap="XS" noPadding>
         <Heading textAlign="center">Reset your password</Heading>

packages/builder/src/pages/builder/portal/settings/organisation.svelte

@@ -57,11 +57,17 @@
       await organisation.init()
     }
 
-    // Update settings
+    const config = {
-    const res = await organisation.save({
       company: $values.company ?? "",
       platformUrl: $values.platformUrl ?? "",
-    })
+    }
+    // remove logo if required
+    if (!$values.logo) {
+      config.logoUrl = ""
+    }
+
+    // Update settings
+    const res = await organisation.save(config)
     if (res.status === 200) {
       notifications.success("Settings saved successfully")
     } else {
@@ -98,7 +104,11 @@
           <Dropzone
             value={[$values.logo]}
             on:change={e => {
-              $values.logo = e.detail?.[0]
+              if (!e.detail || e.detail.length === 0) {
+                $values.logo = null
+              } else {
+                $values.logo = e.detail[0]
+              }
             }}
           />
         </div>

packages/builder/src/pages/builder/portal/settings/update.svelte

@@ -1,4 +1,5 @@
 <script>
+  import { onMount } from "svelte"
   import {
     Layout,
     Heading,
@@ -7,9 +8,12 @@
     Divider,
     notifications,
   } from "@budibase/bbui"
+  import api from "builderStore/api"
   import { auth } from "stores/portal"
   import { redirect } from "@roxi/routify"
 
+  let version
+
   // Only admins allowed here
   $: {
     if (!$auth.isAdmin) {
@@ -26,10 +30,20 @@
         },
       })
       notifications.success("Your budibase installation is up to date.")
+      getVersion()
     } catch (err) {
       notifications.error(`Error installing budibase update ${err}`)
     }
   }
+
+  async function getVersion() {
+    const response = await api.get("/api/dev/version")
+    version = await response.text()
+  }
+
+  onMount(() => {
+    getVersion()
+  })
 </script>
 
 {#if $auth.isAdmin}
@@ -43,6 +57,11 @@
     </Layout>
     <Divider size="S" />
     <div class="fields">
+      <div class="field">
+        {#if version}
+          Current Version: {version}
+        {/if}
+      </div>
       <div class="field">
         <Button cta on:click={updateBudibase}>Check For Updates</Button>
       </div>

packages/builder/src/stores/backend/datasources.js

@@ -1,5 +1,5 @@
 import { writable } from "svelte/store"
-import { queries } from "./"
+import { queries, tables, views } from "./"
 import api from "../../builderStore/api"
 
 export const INITIAL_DATASOURCE_VALUES = {
@@ -21,17 +21,53 @@ export function createDatasourcesStore() {
     fetch: async () => {
       const response = await api.get(`/api/datasources`)
       const json = await response.json()
-      update(state => ({ ...state, list: json }))
+      update(state => ({ ...state, list: json, selected: null }))
       return json
     },
     select: async datasourceId => {
       update(state => ({ ...state, selected: datasourceId }))
-      queries.update(state => ({ ...state, selected: null }))
+      queries.unselect()
+      tables.unselect()
+      views.unselect()
+    },
+    unselect: () => {
+      update(state => ({ ...state, selected: null }))
+    },
+    updateSchema: async datasource => {
+      let url = `/api/datasources/${datasource._id}/schema`
+
+      const response = await api.post(url)
+      const json = await response.json()
+
+      if (response.status !== 200) {
+        throw new Error(json.message)
+      }
+
+      update(state => {
+        const currentIdx = state.list.findIndex(ds => ds._id === json._id)
+
+        const sources = state.list
+
+        if (currentIdx >= 0) {
+          sources.splice(currentIdx, 1, json)
+        } else {
+          sources.push(json)
+        }
+
+        return { list: sources, selected: json._id }
+      })
+      return json
     },
     save: async datasource => {
-      const response = await api.post("/api/datasources", datasource)
+      let url = "/api/datasources"
+
+      const response = await api.post(url, datasource)
       const json = await response.json()
 
+      if (response.status !== 200) {
+        throw new Error(json.message)
+      }
+
       update(state => {
         const currentIdx = state.list.findIndex(ds => ds._id === json._id)
 

packages/builder/src/stores/backend/queries.js

@@ -1,5 +1,5 @@
 import { writable, get } from "svelte/store"
-import { datasources, integrations } from "./"
+import { datasources, integrations, tables } from "./"
 import api from "builderStore/api"
 
 export function createQueriesStore() {
@@ -55,11 +55,14 @@ export function createQueriesStore() {
     },
     select: query => {
       update(state => ({ ...state, selected: query._id }))
-      datasources.update(state => ({
+      tables.update(state => ({
         ...state,
-        selected: query.datasourceId,
+        selected: null,
       }))
     },
+    unselect: () => {
+      update(state => ({ ...state, selected: null }))
+    },
     delete: async query => {
       const response = await api.delete(
         `/api/queries/${query._id}/${query._rev}`

packages/builder/src/stores/backend/rows.js

@@ -1,13 +1,13 @@
 import { writable, get } from "svelte/store"
-import { views } from "./"
+import { tables } from "./"
 
 export function createRowsStore() {
   const { subscribe } = writable([])
 
   return {
     subscribe,
-    save: () => views.select(get(views).selected),
+    save: () => tables.select(get(tables).selected),
-    delete: () => views.select(get(views).selected),
+    delete: () => tables.select(get(tables).selected),
   }
 }
 

packages/builder/src/stores/backend/tables.js

@@ -1,5 +1,5 @@
 import { writable, get } from "svelte/store"
-import { views } from "./"
+import { views, queries, datasources } from "./"
 import { cloneDeep } from "lodash/fp"
 import api from "builderStore/api"
 
@@ -25,7 +25,9 @@ export function createTablesStore() {
         selected: table,
         draft: cloneDeep(table),
       }))
-      views.select({ name: `all_${table._id}` })
+      views.unselect()
+      queries.unselect()
+      datasources.unselect()
     }
   }
 
@@ -66,8 +68,15 @@ export function createTablesStore() {
 
   return {
     subscribe,
+    update,
     fetch,
     select,
+    unselect: () => {
+      update(state => ({
+        ...state,
+        selected: null,
+      }))
+    },
     save,
     init: async () => {
       const response = await api.get("/api/tables")

packages/builder/src/stores/backend/tests/datasources.spec.js

@@ -24,10 +24,10 @@ describe("Datasources Store", () => {
   })
 
   it("fetches all the datasources and updates the store", async () => {
-    api.get.mockReturnValue({ json: () => [SOME_DATASOURCE]})
+    api.get.mockReturnValue({ json: () => [SOME_DATASOURCE] })
 
     await store.fetch()
-    expect(get(store)).toEqual({ list: [SOME_DATASOURCE], selected: null})  
+    expect(get(store)).toEqual({ list: [SOME_DATASOURCE], selected: null })  
   })
 
   it("selects a datasource", async () => {
@@ -44,7 +44,7 @@ describe("Datasources Store", () => {
   })
 
   it("saves the datasource, updates the store and returns status message", async () => {
-    api.post.mockReturnValue({ json: () => SAVE_DATASOURCE})    
+    api.post.mockReturnValue({ status: 200, json: () => SAVE_DATASOURCE})    
 
     await store.save({
       name: 'CoolDB',

packages/builder/src/stores/backend/tests/queries.spec.js

@@ -30,13 +30,6 @@ describe("Queries Store", () => {
     expect(get(store)).toEqual({ list: [SOME_QUERY], selected: null})  
   })
 
-  it("selects a query and updates selected datasource", async () => {
-    await store.select(SOME_QUERY)
-  
-    expect(get(store).selected).toEqual(SOME_QUERY._id) 
-    expect(get(datasources).selected).toEqual(SOME_QUERY.datasourceId)
-  })
-
   it("saves the query, updates the store and returns status message", async () => {
     api.post.mockReturnValue({ json: () => SAVE_QUERY_RESPONSE})    
 

packages/builder/src/stores/backend/tests/tables.spec.js

@@ -41,14 +41,6 @@ describe("Tables Store", () => {
     expect(get(store).draft).toEqual({}) 
   })
 
-  it("selecting a table updates the view store", async () => {
-    const tableToSelect = SOME_TABLES[0]
-    await store.select(tableToSelect)
-  
-    expect(get(store).selected).toEqual(tableToSelect) 
-    expect(get(views).selected).toEqual({ name: `all_${tableToSelect._id}` })
-  })
-
   it("saving a table also selects it", async () => {
     api.post.mockReturnValue({ json: () => SAVE_TABLES_RESPONSE})    
 

packages/builder/src/stores/backend/views.js

@@ -1,5 +1,5 @@
 import { writable, get } from "svelte/store"
-import { tables } from "./"
+import { tables, datasources, queries } from "./"
 import api from "builderStore/api"
 
 export function createViewsStore() {
@@ -10,11 +10,20 @@ export function createViewsStore() {
 
   return {
     subscribe,
+    update,
     select: async view => {
       update(state => ({
         ...state,
         selected: view,
       }))
+      queries.unselect()
+      datasources.unselect()
+    },
+    unselect: () => {
+      update(state => ({
+        ...state,
+        selected: null,
+      }))
     },
     delete: async view => {
       await api.delete(`/api/views/${view}`)

packages/builder/src/stores/portal/organisation.js

@@ -13,7 +13,7 @@ export function createOrganisationStore() {
   const { subscribe, set } = store
 
   async function init() {
-    const res = await api.get(`/api/admin/configs/settings`)
+    const res = await api.get(`/api/admin/configs/public`)
     const json = await res.json()
 
     if (json.status === 400) {

packages/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/cli",
-  "version": "0.9.51",
+  "version": "0.9.53",
   "description": "Budibase CLI, for developers, self hosting and migrations.",
   "main": "src/index.js",
   "bin": {

packages/client/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/client",
-  "version": "0.9.51",
+  "version": "0.9.53",
   "license": "MPL-2.0",
   "module": "dist/budibase-client.js",
   "main": "dist/budibase-client.js",
@@ -18,9 +18,9 @@
     "dev:builder": "rollup -cw"
   },
   "dependencies": {
-    "@budibase/bbui": "^0.9.51",
+    "@budibase/bbui": "^0.9.53",
-    "@budibase/standard-components": "^0.9.51",
+    "@budibase/standard-components": "^0.9.53",
-    "@budibase/string-templates": "^0.9.51",
+    "@budibase/string-templates": "^0.9.53",
     "regexparam": "^1.3.0",
     "shortid": "^2.2.15",
     "svelte-spa-router": "^3.0.5"

packages/client/src/api/rows.js

@@ -44,7 +44,7 @@ export const updateRow = async row => {
     return
   }
   const res = await API.patch({
-    url: `/api/${row.tableId}/rows/${row._id}`,
+    url: `/api/${row.tableId}/rows`,
     body: row,
   })
   res.error
@@ -65,7 +65,11 @@ export const deleteRow = async ({ tableId, rowId, revId }) => {
     return
   }
   const res = await API.del({
-    url: `/api/${tableId}/rows/${rowId}/${revId}`,
+    url: `/api/${tableId}/rows`,
+    body: {
+      _id: rowId,
+      _rev: revId,
+    },
   })
   res.error
     ? notificationStore.danger("An error has occurred")
@@ -84,11 +88,10 @@ export const deleteRows = async ({ tableId, rows }) => {
   if (!tableId || !rows) {
     return
   }
-  const res = await API.post({
+  const res = await API.del({
     url: `/api/${tableId}/rows`,
     body: {
       rows,
-      type: "delete",
     },
   })
   res.error

packages/client/src/api/tables.js

@@ -45,7 +45,7 @@ export const searchTable = async ({
     }
   }
   const res = await API.post({
-    url: `/api/search/${tableId}/rows`,
+    url: `/api/${tableId}/search`,
     body: {
       query,
       bookmark,

packages/client/yarn.lock

@@ -28,10 +28,10 @@
     chalk "^2.0.0"
     js-tokens "^4.0.0"
 
-"@budibase/bbui@^0.9.47":
+"@budibase/bbui@^0.9.53":
-  version "0.9.47"
+  version "0.9.53"
-  resolved "https://registry.yarnpkg.com/@budibase/bbui/-/bbui-0.9.47.tgz#d8664a05203432d522cd91a0bad1cdd8518baf93"
+  resolved "https://registry.yarnpkg.com/@budibase/bbui/-/bbui-0.9.53.tgz#b6841a31ff2c28feb929c57f7f10a1dae1b3aea3"
-  integrity sha512-LXvJCgUSoc4EJKafBaKfUzU4GUOQGmts/8F4V6LTFtTyMZavgq2/KFAgPbR3QeYvidLsshtwop/pQfoszXTQnQ==
+  integrity sha512-jO11Ky1KhPGRv922jMRO49FeTY1TeF3u2JaBJYBkVY95il3uPOI20M1AdA6w2emppDlyP6FSEHk+prdra4Lndw==
   dependencies:
     "@adobe/spectrum-css-workflow-icons" "^1.2.1"
     "@spectrum-css/actionbutton" "^1.0.1"
@@ -108,12 +108,12 @@
     to-gfm-code-block "^0.1.1"
     year "^0.2.1"
 
-"@budibase/standard-components@^0.9.47":
+"@budibase/standard-components@^0.9.53":
-  version "0.9.47"
+  version "0.9.53"
-  resolved "https://registry.yarnpkg.com/@budibase/standard-components/-/standard-components-0.9.47.tgz#8e4f27c43b5a6f65d3d296c61f842195e297f061"
+  resolved "https://registry.yarnpkg.com/@budibase/standard-components/-/standard-components-0.9.53.tgz#5e8d84bf4c3b1ceadfc40b5b5b6c0513b6283fc5"
-  integrity sha512-0+Ndg67Jgk7cqOYluGKpixNFvEqvy2oguKLEr1l83Sf0oWTQ3RCmUGs2mU66ljwnE+o4/JN/EdkA2uSqKInQtg==
+  integrity sha512-8QJmjwF51vh+rCiLbk+JLqCNZZBq9M8/LuyQOGvjnhB8l6DNrfjnCypP/xYoBf0uUvlki8TeNuZKQmDpBBnR7A==
   dependencies:
-    "@budibase/bbui" "^0.9.47"
+    "@budibase/bbui" "^0.9.53"
     "@spectrum-css/page" "^3.0.1"
     "@spectrum-css/vars" "^3.0.1"
     apexcharts "^3.22.1"
@@ -121,10 +121,10 @@
     svelte-apexcharts "^1.0.2"
     svelte-flatpickr "^3.1.0"
 
-"@budibase/string-templates@^0.9.47":
+"@budibase/string-templates@^0.9.53":
-  version "0.9.47"
+  version "0.9.53"
-  resolved "https://registry.yarnpkg.com/@budibase/string-templates/-/string-templates-0.9.47.tgz#484ce5ce29a6ddaef3480368b1a24ce8c3852324"
+  resolved "https://registry.yarnpkg.com/@budibase/string-templates/-/string-templates-0.9.53.tgz#9228965afcef4cc19f7d016b291d5f298bb3b725"
-  integrity sha512-I16Ps4AW7VW8MrSdsoZdwLutiX7GhRkiH6m1AdFcmzh2mZI6YyFM000PuKGEt+sREXK2NI6cBzmi9ZpKIAPJJw==
+  integrity sha512-TL3Zx6VN+YpbIT5vtuTXEv2SOAwmx+YN42pbWxH3ExHj54bvhmRxSS+xJySs75kWdvQU4OMnYywQcMeMsqkOqg==
   dependencies:
     "@budibase/handlebars-helpers" "^0.11.4"
     dayjs "^1.10.4"
@@ -4030,9 +4030,9 @@ serialize-javascript@^4.0.0:
     randombytes "^2.1.0"
 
 set-getter@^0.1.0:
-  version "0.1.0"
+  version "0.1.1"
-  resolved "https://registry.yarnpkg.com/set-getter/-/set-getter-0.1.0.tgz#d769c182c9d5a51f409145f2fba82e5e86e80376"
+  resolved "https://registry.yarnpkg.com/set-getter/-/set-getter-0.1.1.tgz#a3110e1b461d31a9cfc8c5c9ee2e9737ad447102"
-  integrity sha1-12nBgsnVpR9AkUXy+6guXoboA3Y=
+  integrity sha512-9sVWOy+gthr+0G9DzqqLaYNA7+5OKkSmcqjL9cBpDEaZrr3ShQlyX2cZ/O/ozE41oxn/Tt0LGEM/w4Rub3A3gw==
   dependencies:
     to-object-path "^0.3.0"
 
@@ -4233,9 +4233,9 @@ string_decoder@~1.1.1:
     safe-buffer "~5.1.0"
 
 striptags@^3.1.0:
-  version "3.1.1"
+  version "3.2.0"
-  resolved "https://registry.yarnpkg.com/striptags/-/striptags-3.1.1.tgz#c8c3e7fdd6fb4bb3a32a3b752e5b5e3e38093ebd"
+  resolved "https://registry.yarnpkg.com/striptags/-/striptags-3.2.0.tgz#cc74a137db2de8b0b9a370006334161f7dd67052"
-  integrity sha1-yMPn/db7S7OjKjt1LltePjgJPr0=
+  integrity sha512-g45ZOGzHDMe2bdYMdIvdAfCQkCTDMGBazSw1ypMowwGIee7ZQ5dU0rBJ8Jqgl+jAKIv4dbeE1jscZq9wid1Tkw==
 
 style-inject@^0.3.0:
   version "0.3.0"

packages/server/__mocks__/mysql.js

@@ -2,7 +2,9 @@ const mysql = {}
 
 const client = {
   connect: jest.fn(),
-  query: jest.fn(),
+  query: jest.fn((query, bindings, fn) => {
+    fn(null, [])
+  }),
 }
 
 mysql.createConnection = jest.fn(() => client)

packages/server/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "0.9.51",
+  "version": "0.9.53",
   "description": "Budibase Web Server",
   "main": "src/electron.js",
   "repository": {
@@ -55,9 +55,9 @@
   "author": "Budibase",
   "license": "AGPL-3.0-or-later",
   "dependencies": {
-    "@budibase/auth": "^0.9.51",
+    "@budibase/auth": "^0.9.53",
-    "@budibase/client": "^0.9.51",
+    "@budibase/client": "^0.9.53",
-    "@budibase/string-templates": "^0.9.51",
+    "@budibase/string-templates": "^0.9.53",
     "@elastic/elasticsearch": "7.10.0",
     "@koa/router": "8.0.0",
     "@sendgrid/mail": "7.1.1",
@@ -77,6 +77,7 @@
     "jimp": "0.16.1",
     "joi": "17.2.1",
     "jsonschema": "1.4.0",
+    "knex": "^0.95.6",
     "koa": "2.7.0",
     "koa-body": "4.2.0",
     "koa-compress": "4.0.1",
@@ -109,7 +110,7 @@
   "devDependencies": {
     "@babel/core": "^7.14.3",
     "@babel/preset-env": "^7.14.4",
-    "@budibase/standard-components": "^0.9.51",
+    "@budibase/standard-components": "^0.9.53",
     "@jest/test-sequencer": "^24.8.0",
     "babel-jest": "^27.0.2",
     "docker-compose": "^0.23.6",

packages/server/scripts/integrations/mysql/docker-compose.yaml

@@ -0,0 +1,21 @@
+# Use root/example as user/password credentials
+version: '3.1'
+
+services:
+
+  db:
+    image: mysql
+    restart: always
+    command: --init-file /data/application/init.sql --default-authentication-plugin=mysql_native_password
+    volumes:
+      - ./init.sql:/data/application/init.sql
+    environment:
+      MYSQL_ROOT_PASSWORD: root
+    ports:
+      - 3306:3306
+
+  adminer:
+    image: adminer
+    restart: always
+    ports:
+      - 8080:8080

packages/server/scripts/integrations/mysql/init.sql

@@ -0,0 +1,9 @@
+CREATE DATABASE IF NOT EXISTS main;
+USE main;
+CREATE TABLE Persons (
+    PersonID int NOT NULL PRIMARY KEY,
+    LastName varchar(255),
+    FirstName varchar(255),
+    Address varchar(255),
+    City varchar(255)
+);

packages/server/scripts/integrations/postgres/docker-compose.yml

@@ -0,0 +1,28 @@
+version: "3.8"
+services:
+  db:
+    container_name: postgres
+    image: postgres
+    restart: always
+    environment:
+      POSTGRES_USER: root
+      POSTGRES_PASSWORD: root
+      POSTGRES_DB: main
+    ports:
+      - "5432:5432"
+    volumes:
+      #- pg_data:/var/lib/postgresql/data/
+      - ./init.sql:/docker-entrypoint-initdb.d/init.sql
+
+  pgadmin:
+    container_name: pgadmin
+    image: dpage/pgadmin4
+    restart: always
+    environment:
+      PGADMIN_DEFAULT_EMAIL: root@root.com
+      PGADMIN_DEFAULT_PASSWORD: root
+    ports:
+      - "5050:80"
+
+#volumes:
+#  pg_data:

packages/server/scripts/integrations/postgres/init.sql

@@ -0,0 +1,9 @@
+SELECT 'CREATE DATABASE main'
+WHERE NOT EXISTS (SELECT FROM pg_database WHERE datname = 'main')\gexec
+CREATE TABLE Persons (
+    PersonID int NOT NULL PRIMARY KEY,
+    LastName varchar(255),
+    FirstName varchar(255),
+    Address varchar(255),
+    City varchar(255)
+);

packages/server/src/api/controllers/automation.js

@@ -59,7 +59,7 @@ async function checkForCronTriggers({ appId, oldAuto, newAuto }) {
 
   const cronTriggerActivated = isLive(newAuto) && !isLive(oldAuto)
 
-  if (cronTriggerRemoved || cronTriggerDeactivated) {
+  if (cronTriggerRemoved || (cronTriggerDeactivated && oldTrigger.cronJobId)) {
     await triggers.automationQueue.removeRepeatableByKey(oldTrigger.cronJobId)
   }
   // need to create cron job

packages/server/src/api/controllers/datasource.js

@@ -3,26 +3,67 @@ const {
   generateDatasourceID,
   getDatasourceParams,
   getQueryParams,
+  DocumentTypes,
+  BudibaseInternalDB,
+  getTableParams,
 } = require("../../db/utils")
 const { integrations } = require("../../integrations")
+const { makeExternalQuery } = require("./row/utils")
 
 exports.fetch = async function (ctx) {
   const database = new CouchDB(ctx.appId)
-  ctx.body = (
+
+  // Get internal tables
+  const db = new CouchDB(ctx.appId)
+  const internalTables = await db.allDocs(
+    getTableParams(null, {
+      include_docs: true,
+    })
+  )
+  const internal = internalTables.rows.map(row => row.doc)
+
+  const bbInternalDb = {
+    ...BudibaseInternalDB,
+    entities: internal,
+  }
+
+  // Get external datasources
+  const datasources = (
     await database.allDocs(
       getDatasourceParams(null, {
         include_docs: true,
       })
     )
   ).rows.map(row => row.doc)
+
+  ctx.body = [bbInternalDb, ...datasources]
+}
+
+exports.buildSchemaFromDb = async function (ctx) {
+  const db = new CouchDB(ctx.appId)
+  const datasourceId = ctx.params.datasourceId
+  const datasource = await db.get(datasourceId)
+
+  const Connector = integrations[datasource.source]
+
+  // Connect to the DB and build the schema
+  const connector = new Connector(datasource.config)
+  await connector.buildSchema(datasource._id)
+  datasource.entities = connector.tables
+
+  const response = await db.post(datasource)
+  datasource._rev = response.rev
+
+  ctx.body = datasource
 }
 
 exports.save = async function (ctx) {
   const db = new CouchDB(ctx.appId)
+  const plus = ctx.request.body.plus
 
   const datasource = {
-    _id: generateDatasourceID(),
+    _id: generateDatasourceID({ plus }),
-    type: "datasource",
+    type: plus ? DocumentTypes.DATASOURCE_PLUS : DocumentTypes.DATASOURCE,
     ...ctx.request.body,
   }
 
@@ -30,9 +71,11 @@ exports.save = async function (ctx) {
   datasource._rev = response.rev
 
   // Drain connection pools when configuration is changed
-  const source = integrations[datasource.source]
+  if (datasource.source) {
-  if (source && source.pool) {
+    const source = integrations[datasource.source]
-    await source.pool.end()
+    if (source && source.pool) {
+      await source.pool.end()
+    }
   }
 
   ctx.status = 200
@@ -58,3 +101,13 @@ exports.find = async function (ctx) {
   const database = new CouchDB(ctx.appId)
   ctx.body = await database.get(ctx.params.datasourceId)
 }
+
+// dynamic query functionality
+exports.query = async function (ctx) {
+  const queryJson = ctx.request.body
+  try {
+    ctx.body = await makeExternalQuery(ctx.appId, queryJson)
+  } catch (err) {
+    ctx.throw(400, err)
+  }
+}

packages/server/src/api/controllers/dev.js

@@ -11,10 +11,14 @@ async function redirect(ctx, method) {
   const { devPath } = ctx.params
   const response = await fetch(
     checkSlashesInUrl(`${env.WORKER_URL}/api/admin/${devPath}`),
-    request(ctx, {
+    request(
-      method,
+      ctx,
-      body: ctx.request.body,
+      {
-    })
+        method,
+        body: ctx.request.body,
+      },
+      true
+    )
   )
   if (response.status !== 200) {
     ctx.throw(response.status, response.statusText)
@@ -93,3 +97,7 @@ exports.revert = async ctx => {
     ctx.throw(400, `Unable to revert. ${err}`)
   }
 }
+
+exports.getBudibaseVersion = async ctx => {
+  ctx.body = require("../../../package.json").version
+}

packages/server/src/api/controllers/query.js

@@ -160,6 +160,8 @@ exports.execute = async function (ctx) {
   )
 
   const integration = new Integration(datasource.config)
+  console.log(query)
+  // ctx.body = {}
   // call the relevant CRUD method on the integration class
   ctx.body = formatResponse(await integration[query.queryVerb](enrichedQuery))
   // cleanup

packages/server/src/api/controllers/row/external.js

@@ -0,0 +1,276 @@
+const { makeExternalQuery } = require("./utils")
+const { DataSourceOperation, SortDirection } = require("../../../constants")
+const { getExternalTable } = require("../table/utils")
+const {
+  breakExternalTableId,
+  generateRowIdField,
+  breakRowIdField,
+} = require("../../../integrations/utils")
+const { cloneDeep } = require("lodash/fp")
+
+function inputProcessing(row, table) {
+  if (!row) {
+    return row
+  }
+  let newRow = {}
+  for (let key of Object.keys(table.schema)) {
+    // currently excludes empty strings
+    if (row[key]) {
+      newRow[key] = row[key]
+    }
+  }
+  return newRow
+}
+
+function generateIdForRow(row, table) {
+  if (!row) {
+    return
+  }
+  const primary = table.primary
+  // build id array
+  let idParts = []
+  for (let field of primary) {
+    idParts.push(row[field])
+  }
+  return generateRowIdField(idParts)
+}
+
+function outputProcessing(rows, table) {
+  // if no rows this is what is returned? Might be PG only
+  if (rows[0].read === true) {
+    return []
+  }
+  for (let row of rows) {
+    row._id = generateIdForRow(row, table)
+    row.tableId = table._id
+    row._rev = "rev"
+  }
+  return rows
+}
+
+function buildFilters(id, filters, table) {
+  const primary = table.primary
+  // if passed in array need to copy for shifting etc
+  let idCopy = cloneDeep(id)
+  if (filters) {
+    // need to map over the filters and make sure the _id field isn't present
+    for (let filter of Object.values(filters)) {
+      if (filter._id) {
+        const parts = breakRowIdField(filter._id)
+        for (let field of primary) {
+          filter[field] = parts.shift()
+        }
+      }
+      // make sure this field doesn't exist on any filter
+      delete filter._id
+    }
+  }
+  // there is no id, just use the user provided filters
+  if (!idCopy || !table) {
+    return filters
+  }
+  // if used as URL parameter it will have been joined
+  if (typeof idCopy === "string") {
+    idCopy = breakRowIdField(idCopy)
+  }
+  const equal = {}
+  for (let field of primary) {
+    // work through the ID and get the parts
+    equal[field] = idCopy.shift()
+  }
+  return {
+    equal,
+  }
+}
+
+async function handleRequest(
+  appId,
+  operation,
+  tableId,
+  { id, row, filters, sort, paginate } = {}
+) {
+  let { datasourceId, tableName } = breakExternalTableId(tableId)
+  const table = await getExternalTable(appId, datasourceId, tableName)
+  if (!table) {
+    throw `Unable to process query, table "${tableName}" not defined.`
+  }
+  // clean up row on ingress using schema
+  filters = buildFilters(id, filters, table)
+  row = inputProcessing(row, table)
+  if (
+    operation === DataSourceOperation.DELETE &&
+    (filters == null || Object.keys(filters).length === 0)
+  ) {
+    throw "Deletion must be filtered"
+  }
+  let json = {
+    endpoint: {
+      datasourceId,
+      entityId: tableName,
+      operation,
+    },
+    resource: {
+      // not specifying any fields means "*"
+      fields: [],
+    },
+    filters,
+    sort,
+    paginate,
+    body: row,
+    // pass an id filter into extra, purely for mysql/returning
+    extra: {
+      idFilter: buildFilters(id || generateIdForRow(row, table), {}, table),
+    },
+  }
+  // can't really use response right now
+  const response = await makeExternalQuery(appId, json)
+  // we searched for rows in someway
+  if (operation === DataSourceOperation.READ && Array.isArray(response)) {
+    return outputProcessing(response, table)
+  } else {
+    row = outputProcessing(response, table)[0]
+    return { row, table }
+  }
+}
+
+exports.patch = async ctx => {
+  const appId = ctx.appId
+  const inputs = ctx.request.body
+  const tableId = ctx.params.tableId
+  const id = breakRowIdField(inputs._id)
+  // don't save the ID to db
+  delete inputs._id
+  return handleRequest(appId, DataSourceOperation.UPDATE, tableId, {
+    id,
+    row: inputs,
+  })
+}
+
+exports.save = async ctx => {
+  const appId = ctx.appId
+  const inputs = ctx.request.body
+  const tableId = ctx.params.tableId
+  return handleRequest(appId, DataSourceOperation.CREATE, tableId, {
+    row: inputs,
+  })
+}
+
+exports.fetchView = async ctx => {
+  // there are no views in external data sources, shouldn't ever be called
+  // for now just fetch
+  ctx.params.tableId = ctx.params.viewName.split("all_")[1]
+  return exports.fetch(ctx)
+}
+
+exports.fetch = async ctx => {
+  const appId = ctx.appId
+  const tableId = ctx.params.tableId
+  return handleRequest(appId, DataSourceOperation.READ, tableId)
+}
+
+exports.find = async ctx => {
+  const appId = ctx.appId
+  const id = ctx.params.rowId
+  const tableId = ctx.params.tableId
+  return handleRequest(appId, DataSourceOperation.READ, tableId, {
+    id,
+  })
+}
+
+exports.destroy = async ctx => {
+  const appId = ctx.appId
+  const tableId = ctx.params.tableId
+  const id = ctx.request.body._id
+  const { row } = await handleRequest(
+    appId,
+    DataSourceOperation.DELETE,
+    tableId,
+    {
+      id,
+    }
+  )
+  return { response: { ok: true }, row }
+}
+
+exports.bulkDestroy = async ctx => {
+  const appId = ctx.appId
+  const { rows } = ctx.request.body
+  const tableId = ctx.params.tableId
+  let promises = []
+  for (let row of rows) {
+    promises.push(
+      handleRequest(appId, DataSourceOperation.DELETE, tableId, {
+        id: breakRowIdField(row._id),
+      })
+    )
+  }
+  const responses = await Promise.all(promises)
+  return { response: { ok: true }, rows: responses.map(resp => resp.row) }
+}
+
+exports.search = async ctx => {
+  const appId = ctx.appId
+  const tableId = ctx.params.tableId
+  const { paginate, query, ...params } = ctx.request.body
+  let { bookmark, limit } = params
+  if (!bookmark && paginate) {
+    bookmark = 1
+  }
+  let paginateObj = {}
+
+  if (paginate) {
+    paginateObj = {
+      // add one so we can track if there is another page
+      limit: limit,
+      page: bookmark,
+    }
+  } else if (params && limit) {
+    paginateObj = {
+      limit: limit,
+    }
+  }
+  let sort
+  if (params.sort) {
+    const direction =
+      params.sortOrder === "descending"
+        ? SortDirection.DESCENDING
+        : SortDirection.ASCENDING
+    sort = {
+      [params.sort]: direction,
+    }
+  }
+  const rows = await handleRequest(appId, DataSourceOperation.READ, tableId, {
+    filters: query,
+    sort,
+    paginate: paginateObj,
+  })
+  let hasNextPage = false
+  if (paginate && rows.length === limit) {
+    const nextRows = await handleRequest(
+      appId,
+      DataSourceOperation.READ,
+      tableId,
+      {
+        filters: query,
+        sort,
+        paginate: {
+          limit: 1,
+          page: bookmark * limit + 1,
+        },
+      }
+    )
+    hasNextPage = nextRows.length > 0
+  }
+  // need wrapper object for bookmarks etc when paginating
+  return { rows, hasNextPage, bookmark: bookmark + 1 }
+}
+
+exports.validate = async () => {
+  // can't validate external right now - maybe in future
+  return { valid: true }
+}
+
+exports.fetchEnrichedRow = async () => {
+  // TODO: How does this work
+  throw "Not Implemented"
+}

packages/server/src/api/controllers/row/index.js

@@ -0,0 +1,138 @@
+const internal = require("./internal")
+const external = require("./external")
+const { isExternalTable } = require("../../../integrations/utils")
+
+function pickApi(tableId) {
+  if (isExternalTable(tableId)) {
+    return external
+  }
+  return internal
+}
+
+function getTableId(ctx) {
+  if (ctx.request.body && ctx.request.body.tableId) {
+    return ctx.request.body.tableId
+  }
+  if (ctx.params && ctx.params.tableId) {
+    return ctx.params.tableId
+  }
+  if (ctx.params && ctx.params.viewName) {
+    return ctx.params.viewName
+  }
+}
+
+exports.patch = async ctx => {
+  const appId = ctx.appId
+  const tableId = getTableId(ctx)
+  const body = ctx.request.body
+  // if it doesn't have an _id then its save
+  if (body && !body._id) {
+    return exports.save(ctx)
+  }
+  try {
+    const { row, table } = await pickApi(tableId).patch(ctx)
+    ctx.status = 200
+    ctx.eventEmitter &&
+      ctx.eventEmitter.emitRow(`row:update`, appId, row, table)
+    ctx.message = `${table.name} updated successfully.`
+    ctx.body = row
+  } catch (err) {
+    ctx.throw(400, err)
+  }
+}
+
+exports.save = async function (ctx) {
+  const appId = ctx.appId
+  const tableId = getTableId(ctx)
+  const body = ctx.request.body
+  // if it has an ID already then its a patch
+  if (body && body._id) {
+    return exports.patch(ctx)
+  }
+  try {
+    const { row, table } = await pickApi(tableId).save(ctx)
+    ctx.status = 200
+    ctx.eventEmitter && ctx.eventEmitter.emitRow(`row:save`, appId, row, table)
+    ctx.message = `${table.name} saved successfully`
+    ctx.body = row
+  } catch (err) {
+    ctx.throw(400, err)
+  }
+}
+
+exports.fetchView = async function (ctx) {
+  const tableId = getTableId(ctx)
+  try {
+    ctx.body = await pickApi(tableId).fetchView(ctx)
+  } catch (err) {
+    ctx.throw(400, err)
+  }
+}
+
+exports.fetch = async function (ctx) {
+  const tableId = getTableId(ctx)
+  try {
+    ctx.body = await pickApi(tableId).fetch(ctx)
+  } catch (err) {
+    ctx.throw(400, err)
+  }
+}
+
+exports.find = async function (ctx) {
+  const tableId = getTableId(ctx)
+  try {
+    ctx.body = await pickApi(tableId).find(ctx)
+  } catch (err) {
+    ctx.throw(400, err)
+  }
+}
+
+exports.destroy = async function (ctx) {
+  const appId = ctx.appId
+  const inputs = ctx.request.body
+  const tableId = getTableId(ctx)
+  let response, row
+  if (inputs.rows) {
+    let { rows } = await pickApi(tableId).bulkDestroy(ctx)
+    response = rows
+    for (let row of rows) {
+      ctx.eventEmitter && ctx.eventEmitter.emitRow(`row:delete`, appId, row)
+    }
+  } else {
+    let resp = await pickApi(tableId).destroy(ctx)
+    response = resp.response
+    row = resp.row
+    ctx.eventEmitter && ctx.eventEmitter.emitRow(`row:delete`, appId, row)
+  }
+  ctx.status = 200
+  // for automations include the row that was deleted
+  ctx.row = row || {}
+  ctx.body = response
+}
+
+exports.search = async ctx => {
+  const tableId = getTableId(ctx)
+  try {
+    ctx.body = await pickApi(tableId).search(ctx)
+  } catch (err) {
+    ctx.throw(400, err)
+  }
+}
+
+exports.validate = async function (ctx) {
+  const tableId = getTableId(ctx)
+  try {
+    ctx.body = await pickApi(tableId).validate(ctx)
+  } catch (err) {
+    ctx.throw(400, err)
+  }
+}
+
+exports.fetchEnrichedRow = async function (ctx) {
+  const tableId = getTableId(ctx)
+  try {
+    ctx.body = await pickApi(tableId).fetchEnrichedRow(ctx)
+  } catch (err) {
+    ctx.throw(400, err)
+  }
+}

packages/server/src/api/controllers/row/internal.js

@@ -1,23 +1,20 @@
-const CouchDB = require("../../db")
+const CouchDB = require("../../../db")
-const validateJs = require("validate.js")
+const linkRows = require("../../../db/linkedRows")
-const linkRows = require("../../db/linkedRows")
 const {
   getRowParams,
   generateRowID,
   DocumentTypes,
-  SEPARATOR,
   InternalTables,
-} = require("../../db/utils")
+} = require("../../../db/utils")
-const userController = require("./user")
+const userController = require("../user")
 const {
   inputProcessing,
   outputProcessing,
-} = require("../../utilities/rowProcessor")
+} = require("../../../utilities/rowProcessor")
-const { FieldTypes } = require("../../constants")
+const { FieldTypes } = require("../../../constants")
 const { isEqual } = require("lodash")
-const { cloneDeep } = require("lodash/fp")
+const { validate, findRow } = require("./utils")
-
+const { fullSearch, paginatedSearch } = require("./internalSearch")
-const TABLE_VIEW_BEGINS_WITH = `all${SEPARATOR}${DocumentTypes.TABLE}${SEPARATOR}`
 
 const CALCULATION_TYPES = {
   SUM: "sum",
@@ -25,35 +22,7 @@ const CALCULATION_TYPES = {
   STATS: "stats",
 }
 
-validateJs.extend(validateJs.validators.datetime, {
+exports.patch = async ctx => {
-  parse: function (value) {
-    return new Date(value).getTime()
-  },
-  // Input is a unix timestamp
-  format: function (value) {
-    return new Date(value).toISOString()
-  },
-})
-
-async function findRow(ctx, db, tableId, rowId) {
-  let row
-  // TODO remove special user case in future
-  if (tableId === InternalTables.USER_METADATA) {
-    ctx.params = {
-      id: rowId,
-    }
-    await userController.findMetadata(ctx)
-    row = ctx.body
-  } else {
-    row = await db.get(rowId)
-  }
-  if (row.tableId !== tableId) {
-    throw "Supplied tableId does not match the rows tableId"
-  }
-  return row
-}
-
-exports.patch = async function (ctx) {
   const appId = ctx.appId
   const db = new CouchDB(appId)
   const inputs = ctx.request.body
@@ -61,7 +30,7 @@ exports.patch = async function (ctx) {
   const isUserTable = tableId === InternalTables.USER_METADATA
   let dbRow
   try {
-    dbRow = await db.get(ctx.params.rowId)
+    dbRow = await db.get(inputs._id)
   } catch (err) {
     if (isUserTable) {
       // don't include the rev, it'll be the global rev
@@ -70,7 +39,7 @@ exports.patch = async function (ctx) {
         _id: inputs._id,
       }
     } else {
-      ctx.throw(400, "Row does not exist")
+      throw "Row does not exist"
     }
   }
   let dbTable = await db.get(tableId)
@@ -88,12 +57,7 @@ exports.patch = async function (ctx) {
   })
 
   if (!validateResult.valid) {
-    ctx.status = 400
+    throw validateResult.errors
-    ctx.body = {
-      status: 400,
-      errors: validateResult.errors,
-    }
-    return
   }
 
   // returned row is cleaned and prepared for writing to DB
@@ -109,7 +73,7 @@ exports.patch = async function (ctx) {
     // the row has been updated, need to put it into the ctx
     ctx.request.body = row
     await userController.updateMetadata(ctx)
-    return
+    return { row: ctx.body, table }
   }
 
   const response = await db.put(row)
@@ -119,10 +83,7 @@ exports.patch = async function (ctx) {
   }
   row._rev = response.rev
   row.type = "row"
-  ctx.eventEmitter && ctx.eventEmitter.emitRow(`row:update`, appId, row, table)
+  return { row, table }
-  ctx.body = row
-  ctx.status = 200
-  ctx.message = `${table.name} updated successfully.`
 }
 
 exports.save = async function (ctx) {
@@ -131,20 +92,6 @@ exports.save = async function (ctx) {
   let inputs = ctx.request.body
   inputs.tableId = ctx.params.tableId
 
-  // TODO: find usage of this and break out into own endpoint
-  if (inputs.type === "delete") {
-    await bulkDelete(ctx)
-    ctx.body = inputs.rows
-    return
-  }
-
-  // if the row obj had an _id then it will have been retrieved
-  if (inputs._id && inputs._rev) {
-    ctx.params.rowId = inputs._id
-    await exports.patch(ctx)
-    return
-  }
-
   if (!inputs._rev && !inputs._id) {
     inputs._id = generateRowID(inputs.tableId)
   }
@@ -158,12 +105,7 @@ exports.save = async function (ctx) {
   })
 
   if (!validateResult.valid) {
-    ctx.status = 400
+    throw validateResult.errors
-    ctx.body = {
-      status: 400,
-      errors: validateResult.errors,
-    }
-    return
   }
 
   // make sure link rows are up to date
@@ -182,21 +124,17 @@ exports.save = async function (ctx) {
     await db.put(table)
   }
   row._rev = response.rev
-  ctx.eventEmitter && ctx.eventEmitter.emitRow(`row:save`, appId, row, table)
+  return { row, table }
-  ctx.body = row
-  ctx.status = 200
-  ctx.message = `${table.name} saved successfully`
 }
 
-exports.fetchView = async function (ctx) {
+exports.fetchView = async ctx => {
   const appId = ctx.appId
   const viewName = ctx.params.viewName
 
   // if this is a table view being looked for just transfer to that
-  if (viewName.startsWith(TABLE_VIEW_BEGINS_WITH)) {
+  if (viewName.includes(DocumentTypes.TABLE)) {
-    ctx.params.tableId = viewName.substring(4)
+    ctx.params.tableId = viewName
-    await exports.fetchTableRows(ctx)
+    return exports.fetch(ctx)
-    return
   }
 
   const db = new CouchDB(appId)
@@ -204,13 +142,14 @@ exports.fetchView = async function (ctx) {
   const designDoc = await db.get("_design/database")
   const viewInfo = designDoc.views[viewName]
   if (!viewInfo) {
-    ctx.throw(400, "View does not exist.")
+    throw "View does not exist."
   }
   const response = await db.query(`database/${viewName}`, {
     include_docs: !calculation,
     group: !!group,
   })
 
+  let rows
   if (!calculation) {
     response.rows = response.rows.map(row => row.doc)
     let table
@@ -222,7 +161,7 @@ exports.fetchView = async function (ctx) {
         schema: {},
       }
     }
-    ctx.body = await outputProcessing(appId, table, response.rows)
+    rows = await outputProcessing(appId, table, response.rows)
   }
 
   if (calculation === CALCULATION_TYPES.STATS) {
@@ -232,26 +171,26 @@ exports.fetchView = async function (ctx) {
       ...row.value,
       avg: row.value.sum / row.value.count,
     }))
-    ctx.body = response.rows
+    rows = response.rows
   }
 
   if (
     calculation === CALCULATION_TYPES.COUNT ||
     calculation === CALCULATION_TYPES.SUM
   ) {
-    ctx.body = response.rows.map(row => ({
+    rows = response.rows.map(row => ({
       group: row.key,
       field,
       value: row.value,
     }))
   }
+  return rows
 }
 
-exports.fetchTableRows = async function (ctx) {
+exports.fetch = async ctx => {
   const appId = ctx.appId
   const db = new CouchDB(appId)
 
-  // TODO remove special user case in future
   let rows,
     table = await db.get(ctx.params.tableId)
   if (ctx.params.tableId === InternalTables.USER_METADATA) {
@@ -265,27 +204,26 @@ exports.fetchTableRows = async function (ctx) {
     )
     rows = response.rows.map(row => row.doc)
   }
-  ctx.body = await outputProcessing(appId, table, rows)
+  return outputProcessing(appId, table, rows)
 }
 
-exports.find = async function (ctx) {
+exports.find = async ctx => {
   const appId = ctx.appId
   const db = new CouchDB(appId)
-  try {
+  const table = await db.get(ctx.params.tableId)
-    const table = await db.get(ctx.params.tableId)
+  let row = await findRow(ctx, db, ctx.params.tableId, ctx.params.rowId)
-    const row = await findRow(ctx, db, ctx.params.tableId, ctx.params.rowId)
+  row = await outputProcessing(appId, table, row)
-    ctx.body = await outputProcessing(appId, table, row)
+  return row
-  } catch (err) {
-    ctx.throw(400, err)
-  }
 }
 
 exports.destroy = async function (ctx) {
   const appId = ctx.appId
   const db = new CouchDB(appId)
-  const row = await db.get(ctx.params.rowId)
+  const { _id, _rev } = ctx.request.body
+  const row = await db.get(_id)
+
   if (row.tableId !== ctx.params.tableId) {
-    ctx.throw(400, "Supplied tableId doesn't match the row's tableId")
+    throw "Supplied tableId doesn't match the row's tableId"
   }
   await linkRows.updateLinks({
     appId,
@@ -293,54 +231,80 @@ exports.destroy = async function (ctx) {
     row,
     tableId: row.tableId,
   })
-  // TODO remove special user case in future
   if (ctx.params.tableId === InternalTables.USER_METADATA) {
     ctx.params = {
-      id: ctx.params.rowId,
+      id: _id,
     }
     await userController.destroyMetadata(ctx)
+    return { response: ctx.body, row }
   } else {
-    ctx.body = await db.remove(ctx.params.rowId, ctx.params.revId)
+    const response = await db.remove(_id, _rev)
+    return { response, row }
   }
-
-  // for automations include the row that was deleted
-  ctx.row = row
-  ctx.status = 200
-  ctx.eventEmitter && ctx.eventEmitter.emitRow(`row:delete`, appId, row)
 }
 
-exports.validate = async function (ctx) {
+exports.bulkDestroy = async ctx => {
-  const errors = await validate({
+  const appId = ctx.appId
+  const { rows } = ctx.request.body
+  const db = new CouchDB(appId)
+
+  let updates = rows.map(row =>
+    linkRows.updateLinks({
+      appId,
+      eventType: linkRows.EventType.ROW_DELETE,
+      row,
+      tableId: row.tableId,
+    })
+  )
+  // TODO remove special user case in future
+  if (ctx.params.tableId === InternalTables.USER_METADATA) {
+    updates = updates.concat(
+      rows.map(row => {
+        ctx.params = {
+          id: row._id,
+        }
+        return userController.destroyMetadata(ctx)
+      })
+    )
+  } else {
+    await db.bulkDocs(rows.map(row => ({ ...row, _deleted: true })))
+  }
+  await Promise.all(updates)
+  return { response: { ok: true }, rows }
+}
+
+exports.search = async ctx => {
+  const appId = ctx.appId
+  const { tableId } = ctx.params
+  const db = new CouchDB(appId)
+  const { paginate, query, ...params } = ctx.request.body
+  params.tableId = tableId
+
+  let response
+  if (paginate) {
+    response = await paginatedSearch(appId, query, params)
+  } else {
+    response = await fullSearch(appId, query, params)
+  }
+
+  // Enrich search results with relationships
+  if (response.rows && response.rows.length) {
+    const table = await db.get(tableId)
+    response.rows = await outputProcessing(appId, table, response.rows)
+  }
+
+  return response
+}
+
+exports.validate = async ctx => {
+  return validate({
     appId: ctx.appId,
     tableId: ctx.params.tableId,
     row: ctx.request.body,
   })
-  ctx.status = 200
-  ctx.body = errors
 }
 
-async function validate({ appId, tableId, row, table }) {
+exports.fetchEnrichedRow = async ctx => {
-  if (!table) {
-    const db = new CouchDB(appId)
-    table = await db.get(tableId)
-  }
-  const errors = {}
-  for (let fieldName of Object.keys(table.schema)) {
-    const constraints = cloneDeep(table.schema[fieldName].constraints)
-    // special case for options, need to always allow unselected (null)
-    if (
-      table.schema[fieldName].type === FieldTypes.OPTIONS &&
-      constraints.inclusion
-    ) {
-      constraints.inclusion.push(null)
-    }
-    const res = validateJs.single(row[fieldName], constraints)
-    if (res) errors[fieldName] = res
-  }
-  return { valid: Object.keys(errors).length === 0, errors }
-}
-
-exports.fetchEnrichedRow = async function (ctx) {
   const appId = ctx.appId
   const db = new CouchDB(appId)
   const tableId = ctx.params.tableId
@@ -381,39 +345,5 @@ exports.fetchEnrichedRow = async function (ctx) {
       )
     }
   }
-  ctx.body = row
+  return row
-  ctx.status = 200
-}
-
-async function bulkDelete(ctx) {
-  const appId = ctx.appId
-  const { rows } = ctx.request.body
-  const db = new CouchDB(appId)
-
-  let updates = rows.map(row =>
-    linkRows.updateLinks({
-      appId,
-      eventType: linkRows.EventType.ROW_DELETE,
-      row,
-      tableId: row.tableId,
-    })
-  )
-  // TODO remove special user case in future
-  if (ctx.params.tableId === InternalTables.USER_METADATA) {
-    updates = updates.concat(
-      rows.map(row => {
-        ctx.params = {
-          id: row._id,
-        }
-        return userController.destroyMetadata(ctx)
-      })
-    )
-  } else {
-    await db.bulkDocs(rows.map(row => ({ ...row, _deleted: true })))
-  }
-  await Promise.all(updates)
-
-  rows.forEach(row => {
-    ctx.eventEmitter && ctx.eventEmitter.emitRow(`row:delete`, appId, row)
-  })
 }

packages/server/src/api/controllers/row/utils.js

@@ -0,0 +1,70 @@
+const validateJs = require("validate.js")
+const { cloneDeep } = require("lodash/fp")
+const CouchDB = require("../../../db")
+const { InternalTables } = require("../../../db/utils")
+const userController = require("../user")
+const { FieldTypes } = require("../../../constants")
+const { integrations } = require("../../../integrations")
+
+validateJs.extend(validateJs.validators.datetime, {
+  parse: function (value) {
+    return new Date(value).getTime()
+  },
+  // Input is a unix timestamp
+  format: function (value) {
+    return new Date(value).toISOString()
+  },
+})
+
+exports.makeExternalQuery = async (appId, json) => {
+  const datasourceId = json.endpoint.datasourceId
+  const db = new CouchDB(appId)
+  const datasource = await db.get(datasourceId)
+  const Integration = integrations[datasource.source]
+  // query is the opinionated function
+  if (Integration.prototype.query) {
+    const integration = new Integration(datasource.config)
+    return integration.query(json)
+  } else {
+    throw "Datasource does not support query."
+  }
+}
+
+exports.findRow = async (ctx, db, tableId, rowId) => {
+  let row
+  // TODO remove special user case in future
+  if (tableId === InternalTables.USER_METADATA) {
+    ctx.params = {
+      id: rowId,
+    }
+    await userController.findMetadata(ctx)
+    row = ctx.body
+  } else {
+    row = await db.get(rowId)
+  }
+  if (row.tableId !== tableId) {
+    throw "Supplied tableId does not match the rows tableId"
+  }
+  return row
+}
+
+exports.validate = async ({ appId, tableId, row, table }) => {
+  if (!table) {
+    const db = new CouchDB(appId)
+    table = await db.get(tableId)
+  }
+  const errors = {}
+  for (let fieldName of Object.keys(table.schema)) {
+    const constraints = cloneDeep(table.schema[fieldName].constraints)
+    // special case for options, need to always allow unselected (null)
+    if (
+      table.schema[fieldName].type === FieldTypes.OPTIONS &&
+      constraints.inclusion
+    ) {
+      constraints.inclusion.push(null)
+    }
+    const res = validateJs.single(row[fieldName], constraints)
+    if (res) errors[fieldName] = res
+  }
+  return { valid: Object.keys(errors).length === 0, errors }
+}

packages/server/src/api/controllers/script.js

@@ -3,13 +3,15 @@ const vm = require("vm")
 
 class ScriptExecutor {
   constructor(body) {
-    this.script = new vm.Script(body.script)
+    const code = `let fn = () => {\n${body.script}\n}; out = fn();`
+    this.script = new vm.Script(code)
     this.context = vm.createContext(body.context)
     this.context.fetch = fetch
   }
 
   execute() {
-    return this.script.runInContext(this.context)
+    this.script.runInContext(this.context)
+    return this.context.out
   }
 }
 

packages/server/src/api/controllers/search/index.js

@@ -1,26 +0,0 @@
-const { fullSearch, paginatedSearch } = require("./utils")
-const CouchDB = require("../../../db")
-const { outputProcessing } = require("../../../utilities/rowProcessor")
-
-exports.rowSearch = async ctx => {
-  const appId = ctx.appId
-  const { tableId } = ctx.params
-  const db = new CouchDB(appId)
-  const { paginate, query, ...params } = ctx.request.body
-  params.tableId = tableId
-
-  let response
-  if (paginate) {
-    response = await paginatedSearch(appId, query, params)
-  } else {
-    response = await fullSearch(appId, query, params)
-  }
-
-  // Enrich search results with relationships
-  if (response.rows && response.rows.length) {
-    const table = await db.get(tableId)
-    response.rows = await outputProcessing(appId, table, response.rows)
-  }
-
-  ctx.body = response
-}

packages/server/src/api/controllers/table/index.js

@@ -5,23 +5,57 @@ const {
   getRowParams,
   getTableParams,
   generateTableID,
+  getDatasourceParams,
+  BudibaseInternalDB,
 } = require("../../../db/utils")
 const { FieldTypes } = require("../../../constants")
-const { TableSaveFunctions } = require("./utils")
+const { TableSaveFunctions, getExternalTable } = require("./utils")
+const {
+  isExternalTable,
+  breakExternalTableId,
+} = require("../../../integrations/utils")
 
 exports.fetch = async function (ctx) {
   const db = new CouchDB(ctx.appId)
-  const body = await db.allDocs(
+
+  const internalTables = await db.allDocs(
     getTableParams(null, {
       include_docs: true,
     })
   )
-  ctx.body = body.rows.map(row => row.doc)
+
+  const internal = internalTables.rows.map(row => ({
+    ...row.doc,
+    type: "internal",
+    sourceId: BudibaseInternalDB._id,
+  }))
+
+  const externalTables = await db.allDocs(
+    getDatasourceParams("plus", {
+      include_docs: true,
+    })
+  )
+
+  const external = externalTables.rows.flatMap(row => {
+    return Object.values(row.doc.entities || {}).map(entity => ({
+      ...entity,
+      type: "external",
+      sourceId: row.doc._id,
+    }))
+  })
+
+  ctx.body = [...internal, ...external]
 }
 
 exports.find = async function (ctx) {
   const db = new CouchDB(ctx.appId)
-  ctx.body = await db.get(ctx.params.id)
+  const tableId = ctx.params.id
+  if (isExternalTable(tableId)) {
+    let { datasourceId, tableName } = breakExternalTableId(tableId)
+    ctx.body = await getExternalTable(ctx.appId, datasourceId, tableName)
+  } else {
+    ctx.body = await db.get(ctx.params.id)
+  }
 }
 
 exports.save = async function (ctx) {

packages/server/src/api/controllers/table/utils.js

@@ -204,4 +204,15 @@ class TableSaveFunctions {
   }
 }
 
+exports.getExternalTable = async (appId, datasourceId, tableName) => {
+  const db = new CouchDB(appId)
+  const datasource = await db.get(datasourceId)
+  if (!datasource || !datasource.entities) {
+    throw "Datasource is not configured fully."
+  }
+  return Object.values(datasource.entities).find(
+    entity => entity.name === tableName
+  )
+}
+
 exports.TableSaveFunctions = TableSaveFunctions

packages/server/src/api/routes/datasource.js

@@ -1,14 +1,64 @@
 const Router = require("@koa/router")
 const datasourceController = require("../controllers/datasource")
 const authorized = require("../../middleware/authorized")
+const joiValidator = require("../../middleware/joi-validator")
 const {
   BUILDER,
   PermissionLevels,
   PermissionTypes,
 } = require("@budibase/auth/permissions")
+const Joi = require("joi")
+const { DataSourceOperation } = require("../../constants")
 
 const router = Router()
 
+function generateDatasourceSchema() {
+  // prettier-ignore
+  return joiValidator.body(Joi.object({
+    _id: Joi.string(),
+    _rev: Joi.string(),
+    // source: Joi.string().valid("POSTGRES_PLUS"),
+    type: Joi.string().allow("datasource_plus"),
+    relationships: Joi.array().items(Joi.object({
+      from: Joi.string().required(),
+      to: Joi.string().required(),
+      cardinality: Joi.valid("1:N", "1:1", "N:N").required()
+    })),
+    // entities: Joi.array().items(Joi.object({
+    //   type: Joi.string().valid(...Object.values(FieldTypes)).required(),
+    //   name: Joi.string().required(),
+    // })),
+  }).unknown(true))
+}
+
+function generateQueryDatasourceSchema() {
+  // prettier-ignore
+  return joiValidator.body(Joi.object({
+    endpoint: Joi.object({
+      datasourceId: Joi.string().required(),
+      operation: Joi.string().required().valid(...Object.values(DataSourceOperation)),
+      entityId: Joi.string().required(),
+    }).required(),
+    resource: Joi.object({
+      fields: Joi.array().items(Joi.string()).optional(),
+    }).optional(),
+    body: Joi.object().optional(),
+    sort: Joi.object().optional(),
+    filters: Joi.object({
+      string: Joi.object().optional(),
+      range: Joi.object().optional(),
+      equal: Joi.object().optional(),
+      notEqual: Joi.object().optional(),
+      empty: Joi.object().optional(),
+      notEmpty: Joi.object().optional(),
+    }).optional(),
+    paginate: Joi.object({
+      page: Joi.string().alphanum().optional(),
+      limit: Joi.number().optional(),
+    }).optional(),
+  }))
+}
+
 router
   .get("/api/datasources", authorized(BUILDER), datasourceController.fetch)
   .get(
@@ -16,7 +66,23 @@ router
     authorized(PermissionTypes.TABLE, PermissionLevels.READ),
     datasourceController.find
   )
-  .post("/api/datasources", authorized(BUILDER), datasourceController.save)
+  .post(
+    "/api/datasources/query",
+    authorized(PermissionTypes.TABLE, PermissionLevels.READ),
+    generateQueryDatasourceSchema(),
+    datasourceController.query
+  )
+  .post(
+    "/api/datasources/:datasourceId/schema",
+    authorized(BUILDER),
+    datasourceController.buildSchemaFromDb
+  )
+  .post(
+    "/api/datasources",
+    authorized(BUILDER),
+    generateDatasourceSchema(),
+    datasourceController.save
+  )
   .delete(
     "/api/datasources/:datasourceId/:revId",
     authorized(BUILDER),

packages/server/src/api/routes/dev.js

@@ -14,6 +14,7 @@ if (env.isDev() || env.isTest()) {
 }
 
 router
+  .get("/api/dev/version", authorized(BUILDER), controller.getBudibaseVersion)
   .delete("/api/dev/:appId/lock", authorized(BUILDER), controller.clearLock)
   .post("/api/dev/:appId/revert", authorized(BUILDER), controller.revert)
 

packages/server/src/api/routes/index.js

@@ -23,7 +23,6 @@ const queryRoutes = require("./query")
 const hostingRoutes = require("./hosting")
 const backupRoutes = require("./backup")
 const devRoutes = require("./dev")
-const searchRoutes = require("./search")
 
 exports.mainRoutes = [
   authRoutes,
@@ -52,7 +51,6 @@ exports.mainRoutes = [
   // this could be breaking as koa may recognise other routes as this
   tableRoutes,
   rowRoutes,
-  searchRoutes,
 ]
 
 exports.staticRoutes = staticRoutes

packages/server/src/api/routes/row.js

@@ -24,7 +24,7 @@ router
     "/api/:tableId/rows",
     paramResource("tableId"),
     authorized(PermissionTypes.TABLE, PermissionLevels.READ),
-    rowController.fetchTableRows
+    rowController.fetch
   )
   .get(
     "/api/:tableId/rows/:rowId",
@@ -32,6 +32,12 @@ router
     authorized(PermissionTypes.TABLE, PermissionLevels.READ),
     rowController.find
   )
+  .post(
+    "/api/:tableId/search",
+    paramResource("tableId"),
+    authorized(PermissionTypes.TABLE, PermissionLevels.READ),
+    rowController.search
+  )
   .post(
     "/api/:tableId/rows",
     paramResource("tableId"),
@@ -40,8 +46,8 @@ router
     rowController.save
   )
   .patch(
-    "/api/:tableId/rows/:rowId",
+    "/api/:tableId/rows",
-    paramSubResource("tableId", "rowId"),
+    paramResource("tableId"),
     authorized(PermissionTypes.TABLE, PermissionLevels.WRITE),
     rowController.patch
   )
@@ -52,8 +58,8 @@ router
     rowController.validate
   )
   .delete(
-    "/api/:tableId/rows/:rowId/:revId",
+    "/api/:tableId/rows",
-    paramSubResource("tableId", "rowId"),
+    paramResource("tableId"),
     authorized(PermissionTypes.TABLE, PermissionLevels.WRITE),
     usage,
     rowController.destroy

packages/server/src/api/routes/search.js

@@ -1,19 +0,0 @@
-const Router = require("@koa/router")
-const controller = require("../controllers/search")
-const {
-  PermissionTypes,
-  PermissionLevels,
-} = require("@budibase/auth/permissions")
-const authorized = require("../../middleware/authorized")
-const { paramResource } = require("../../middleware/resourceId")
-
-const router = Router()
-
-router.post(
-  "/api/search/:tableId/rows",
-  paramResource("tableId"),
-  authorized(PermissionTypes.TABLE, PermissionLevels.READ),
-  controller.rowSearch
-)
-
-module.exports = router

packages/server/src/api/routes/table.js

@@ -17,7 +17,7 @@ function generateSaveValidator() {
   return joiValidator.body(Joi.object({
     _id: Joi.string(),
     _rev: Joi.string(),
-    type: Joi.string().valid("table"),
+    type: Joi.string().valid("table", "internal", "external"),
     primaryDisplay: Joi.string(),
     schema: Joi.object().required(),
     name: Joi.string().required(),

packages/server/src/api/routes/tests/__snapshots__/datasource.spec.js.snap

@@ -0,0 +1,89 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`/datasources fetch returns all the datasources from the server 1`] = `
+Array [
+  Object {
+    "config": Object {},
+    "entities": Array [
+      Object {
+        "_id": "ta_users",
+        "_rev": "1-039883a06c1f9cb3945731d79838181e",
+        "name": "Users",
+        "primaryDisplay": "email",
+        "schema": Object {
+          "email": Object {
+            "constraints": Object {
+              "email": true,
+              "length": Object {
+                "maximum": "",
+              },
+              "presence": true,
+              "type": "string",
+            },
+            "fieldName": "email",
+            "name": "email",
+            "type": "string",
+          },
+          "firstName": Object {
+            "constraints": Object {
+              "presence": false,
+              "type": "string",
+            },
+            "fieldName": "firstName",
+            "name": "firstName",
+            "type": "string",
+          },
+          "lastName": Object {
+            "constraints": Object {
+              "presence": false,
+              "type": "string",
+            },
+            "fieldName": "lastName",
+            "name": "lastName",
+            "type": "string",
+          },
+          "roleId": Object {
+            "constraints": Object {
+              "inclusion": Array [
+                "ADMIN",
+                "POWER",
+                "BASIC",
+                "PUBLIC",
+              ],
+              "presence": false,
+              "type": "string",
+            },
+            "fieldName": "roleId",
+            "name": "roleId",
+            "type": "options",
+          },
+          "status": Object {
+            "constraints": Object {
+              "inclusion": Array [
+                "active",
+                "inactive",
+              ],
+              "presence": false,
+              "type": "string",
+            },
+            "fieldName": "status",
+            "name": "status",
+            "type": "options",
+          },
+        },
+        "type": "table",
+        "views": Object {},
+      },
+    ],
+    "name": "Budibase DB",
+    "source": "BUDIBASE",
+    "type": "budibase",
+  },
+  Object {
+    "config": Object {},
+    "name": "Test",
+    "source": "POSTGRES",
+    "type": "datasource",
+  },
+]
+`;

packages/server/src/api/routes/tests/datasource.spec.js

@@ -2,6 +2,9 @@ let setup = require("./utilities")
 let { basicDatasource } = setup.structures
 let { checkBuilderEndpoint } = require("./utilities/TestFunctions")
 
+jest.mock("pg")
+const pg = require("pg")
+
 describe("/datasources", () => {
   let request = setup.getRequest()
   let config = setup.getConfig()
@@ -37,13 +40,14 @@ describe("/datasources", () => {
         .expect(200)
 
       const datasources = res.body
-      expect(datasources).toEqual([
+
-        {
+      // remove non-deterministic fields
-          "_id": datasources[0]._id,
+      for (let source of datasources) {
-          "_rev": datasources[0]._rev,
+        delete source._id
-          ...basicDatasource()
+        delete source._rev
-        }
+      }
-      ])
+
+      expect(datasources).toMatchSnapshot()
     })
 
     it("should apply authorization to endpoint", async () => {
@@ -66,6 +70,34 @@ describe("/datasources", () => {
     })
   })
 
+  describe("query", () => {
+    it("should be able to query a pg datasource", async () => {
+      const res = await request
+        .post(`/api/datasources/query`)
+        .send({
+          endpoint: {
+            datasourceId: datasource._id,
+            operation: "READ",
+            // table name below
+            entityId: "users",
+          },
+          resource: {
+            fields: ["name", "age"],
+          },
+          filters: {
+            string: {
+              name: "John",
+            },
+          },
+        })
+        .set(config.defaultHeaders())
+        .expect(200)
+      // this is mock data, can't test it
+      expect(res.body).toBeDefined()
+      expect(pg.queryMock).toHaveBeenCalledWith(`select "name", "age" from "users" where "name" like $1 limit $2`, ["John%", 5000])
+    })
+  })
+
   describe("destroy", () => {
     it("deletes queries for the datasource after deletion and returns a success message", async () => {
       await config.createQuery()
@@ -81,7 +113,7 @@ describe("/datasources", () => {
         .expect('Content-Type', /json/)
         .expect(200)
 
-      expect(res.body).toEqual([])
+      expect(res.body.length).toEqual(1)
     })
 
     it("should apply authorization to endpoint", async () => {