Merge branch 'develop' of github.com:Budibase/budibase into feature/cloud-export

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "0.9.144-alpha.0",
+  "version": "0.9.146-alpha.3",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

packages/auth/deprovision.js

@@ -0,0 +1 @@
+module.exports = require("./src/tenancy/deprovision")

packages/auth/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/auth",
-  "version": "0.9.144-alpha.0",
+  "version": "0.9.146-alpha.3",
   "description": "Authentication middlewares for budibase builder and apps",
   "main": "src/index.js",
   "author": "Budibase",

packages/auth/src/tenancy/deprovision.js

@@ -0,0 +1,81 @@
+const { getGlobalUserParams, getAllApps } = require("../db/utils")
+const { getDB, getCouch } = require("../db")
+const { getGlobalDB } = require("./tenancy")
+const { StaticDatabases } = require("../db/constants")
+
+const TENANT_DOC = StaticDatabases.PLATFORM_INFO.docs.tenants
+const PLATFORM_INFO_DB = StaticDatabases.PLATFORM_INFO.name
+
+const removeTenantFromInfoDB = async tenantId => {
+  try {
+    const infoDb = getDB(PLATFORM_INFO_DB)
+    let tenants = await infoDb.get(TENANT_DOC)
+    tenants.tenantIds = tenants.tenantIds.filter(id => id !== tenantId)
+
+    await infoDb.put(tenants)
+  } catch (err) {
+    console.error(`Error removing tenant ${tenantId} from info db`, err)
+    throw err
+  }
+}
+
+const removeUsersFromInfoDB = async tenantId => {
+  try {
+    const globalDb = getGlobalDB(tenantId)
+    const infoDb = getDB(PLATFORM_INFO_DB)
+    const allUsers = await globalDb.allDocs(
+      getGlobalUserParams(null, {
+        include_docs: true,
+      })
+    )
+    const allEmails = allUsers.rows.map(row => row.doc.email)
+    // get the id docs
+    let keys = allUsers.rows.map(row => row.id)
+    // and the email docs
+    keys = keys.concat(allEmails)
+    // retrieve the docs and delete them
+    const userDocs = await infoDb.allDocs({
+      keys,
+      include_docs: true,
+    })
+    const toDelete = userDocs.rows.map(row => {
+      return {
+        ...row.doc,
+        _deleted: true,
+      }
+    })
+    await infoDb.bulkDocs(toDelete)
+  } catch (err) {
+    console.error(`Error removing tenant ${tenantId} users from info db`, err)
+    throw err
+  }
+}
+
+const removeGlobalDB = async tenantId => {
+  try {
+    const globalDb = getGlobalDB(tenantId)
+    await globalDb.destroy()
+  } catch (err) {
+    console.error(`Error removing tenant ${tenantId} users from info db`, err)
+    throw err
+  }
+}
+
+const removeTenantApps = async tenantId => {
+  try {
+    const apps = await getAllApps(getCouch(), { all: true })
+    const destroyPromises = apps.map(app => getDB(app.appId).destroy())
+    await Promise.allSettled(destroyPromises)
+  } catch (err) {
+    console.error(`Error removing tenant ${tenantId} apps`, err)
+    throw err
+  }
+}
+
+// can't live in tenancy package due to circular dependency on db/utils
+exports.deleteTenant = async tenantId => {
+  await removeTenantFromInfoDB(tenantId)
+  await removeUsersFromInfoDB(tenantId)
+  await removeGlobalDB(tenantId)
+  await removeTenantApps(tenantId)
+}

packages/auth/src/utils.js

@@ -67,10 +67,9 @@ exports.getCookie = (ctx, name) => {
  * @param {string|object} value The value of cookie which will be set.
  */
 exports.setCookie = (ctx, value, name = "builder") => {
-  if (!value) {
+  if (value) {
-    ctx.cookies.set(name)
-  } else {
     value = jwt.sign(value, options.secretOrKey)
+  }
 
   const config = {
     maxAge: Number.MAX_SAFE_INTEGER,
@@ -85,7 +84,6 @@ exports.setCookie = (ctx, value, name = "builder") => {
 
   ctx.cookies.set(name, value, config)
 }
-}
 
 /**
  * Utility function, simply calls setCookie with an empty string for value

packages/bbui/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/bbui",
   "description": "A UI solution used in the different Budibase projects.",
-  "version": "0.9.144-alpha.0",
+  "version": "0.9.146-alpha.3",
   "license": "AGPL-3.0",
   "svelte": "src/index.js",
   "module": "dist/bbui.es.js",

packages/bbui/src/Modal/ModalContent.svelte

@@ -14,6 +14,7 @@
   export let showConfirmButton = true
   export let showCloseIcon = true
   export let onConfirm = undefined
+  export let onCancel = undefined
   export let disabled = false
   export let showDivider = true
 
@@ -28,6 +29,14 @@
     }
     loading = false
   }
+
+  async function close() {
+    loading = true
+    if (!onCancel || (await onCancel()) !== false) {
+      cancel()
+    }
+    loading = false
+  }
 </script>
 
 <div
@@ -65,7 +74,7 @@
       >
         <slot name="footer" />
         {#if showCancelButton}
-          <Button group secondary on:click={cancel}>{cancelText}</Button>
+          <Button group secondary on:click={close}>{cancelText}</Button>
         {/if}
         {#if showConfirmButton}
           <Button

packages/bbui/src/Table/Table.svelte

@@ -27,6 +27,7 @@
   export let selectedRows = []
   export let editColumnTitle = "Edit"
   export let customRenderers = []
+  export let disableSorting = false
 
   const dispatch = createEventDispatcher()
 
@@ -63,7 +64,7 @@
   )
 
   // Reset state when data changes
-  $: data.length, reset()
+  $: rows.length, reset()
   const reset = () => {
     nextScrollTop = 0
     scrollTop = 0
@@ -107,7 +108,7 @@
   }
 
   const sortRows = (rows, sortColumn, sortOrder) => {
-    if (!sortColumn || !sortOrder) {
+    if (!sortColumn || !sortOrder || disableSorting) {
       return rows
     }
     return rows.slice().sort((a, b) => {
@@ -131,6 +132,7 @@
       sortColumn = fieldSchema.name
       sortOrder = "Descending"
     }
+    dispatch("sort", { column: sortColumn, order: sortOrder })
   }
 
   const getDisplayName = schema => {

packages/builder/cypress/integration/createTable.spec.js

@@ -6,7 +6,7 @@ context("Create a Table", () => {
 
   it("should create a new Table", () => {
     cy.createTable("dog")
-
+    cy.wait(1000)
     // Check if Table exists
     cy.get(".table-title h1").should("have.text", "dog")
   })
@@ -36,7 +36,8 @@ context("Create a Table", () => {
   it("edits a row", () => {
     cy.contains("button", "Edit").click({ force: true })
     cy.wait(1000)
-    cy.get(".spectrum-Modal input").type("Updated")
+    cy.get(".spectrum-Modal input").clear()
+    cy.get(".spectrum-Modal input").type("RoverUpdated")
     cy.contains("Save").click()
     cy.contains("RoverUpdated").should("have.text", "RoverUpdated")
   })
@@ -62,7 +63,7 @@ context("Create a Table", () => {
 
   it("deletes a table", () => {
     cy.get(".actions > :nth-child(1) > .icon > .spectrum-Icon > use")
-      .first()
+      .eq(1)
       .click({ force: true })
     cy.get(".spectrum-Menu > :nth-child(2)").click()
     cy.contains("Delete Table").click()

packages/builder/cypress/support/commands.js

@@ -35,8 +35,11 @@ Cypress.Commands.add("createApp", name => {
     .within(() => {
       cy.get("input").eq(0).type(name).should("have.value", name).blur()
       cy.get(".spectrum-ButtonGroup").contains("Create app").click()
+      cy.wait(7000)
     })
     .then(() => {
+      // Because we show the datasource modal on entry, we need to create a table to get rid of the modal in the future
+      cy.createInitialDatasource("initialTable")
       cy.expandBudibaseConnection()
       cy.get(".nav-item.selected > .content").should("be.visible")
     })
@@ -69,11 +72,28 @@ Cypress.Commands.add("createTestTableWithData", () => {
   cy.addColumn("dog", "age", "Number")
 })
 
-Cypress.Commands.add("createTable", tableName => {
+Cypress.Commands.add("createInitialDatasource", tableName => {
   // Enter table name
+  cy.get(".spectrum-Modal").within(() => {
+    cy.contains("Budibase DB").trigger("mouseover").click().click()
+    cy.wait(1000)
+    cy.contains("Continue").click()
+  })
+
+  cy.get(".spectrum-Modal").within(() => {
+    cy.wait(1000)
+    cy.get("input").first().type(tableName).blur()
+    cy.get(".spectrum-ButtonGroup").contains("Create").click()
+  })
+  cy.contains(tableName).should("be.visible")
+})
+
+Cypress.Commands.add("createTable", tableName => {
   cy.contains("Budibase DB").click()
   cy.contains("Create new table").click()
+
   cy.get(".spectrum-Modal").within(() => {
+    cy.wait(1000)
     cy.get("input").first().type(tableName).blur()
     cy.get(".spectrum-ButtonGroup").contains("Create").click()
   })

packages/builder/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/builder",
-  "version": "0.9.144-alpha.0",
+  "version": "0.9.146-alpha.3",
   "license": "AGPL-3.0",
   "private": true,
   "scripts": {
@@ -65,10 +65,10 @@
     }
   },
   "dependencies": {
-    "@budibase/bbui": "^0.9.144-alpha.0",
+    "@budibase/bbui": "^0.9.146-alpha.3",
-    "@budibase/client": "^0.9.144-alpha.0",
+    "@budibase/client": "^0.9.146-alpha.3",
     "@budibase/colorpicker": "1.1.2",
-    "@budibase/string-templates": "^0.9.144-alpha.0",
+    "@budibase/string-templates": "^0.9.146-alpha.3",
     "@sentry/browser": "5.19.1",
     "@spectrum-css/page": "^3.0.1",
     "@spectrum-css/vars": "^3.0.1",

packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte

@@ -24,7 +24,7 @@
   import ModalBindableInput from "components/common/bindings/ModalBindableInput.svelte"
   import FilterDrawer from "components/design/PropertiesPanel/PropertyControls/FilterEditor/FilterDrawer.svelte"
   // need the client lucene builder to convert to the structure API expects
-  import { buildLuceneQuery } from "../../../../../client/src/utils/lucene"
+  import { buildLuceneQuery } from "helpers/lucene"
 
   export let block
   export let webhookModal

packages/builder/src/components/backend/DataTable/DataTable.svelte

@@ -1,6 +1,6 @@
 <script>
-  import { tables, views } from "stores/backend"
+  import { fade } from "svelte/transition"
-
+  import { tables } from "stores/backend"
   import CreateRowButton from "./buttons/CreateRowButton.svelte"
   import CreateColumnButton from "./buttons/CreateColumnButton.svelte"
   import CreateViewButton from "./buttons/CreateViewButton.svelte"
@@ -8,59 +8,78 @@
   import EditRolesButton from "./buttons/EditRolesButton.svelte"
   import ManageAccessButton from "./buttons/ManageAccessButton.svelte"
   import HideAutocolumnButton from "./buttons/HideAutocolumnButton.svelte"
-  import * as api from "./api"
+  import TableFilterButton from "./buttons/TableFilterButton.svelte"
   import Table from "./Table.svelte"
   import { TableNames } from "constants"
   import CreateEditRow from "./modals/CreateEditRow.svelte"
+  import { fetchTableData } from "helpers/fetchTableData"
+  import { Pagination } from "@budibase/bbui"
 
   let hideAutocolumns = true
-  let data = []
+
-  let loading = false
   $: isUsersTable = $tables.selected?._id === TableNames.USERS
-  $: title = $tables.selected?.name
   $: schema = $tables.selected?.schema
-  $: tableView = {
-    schema,
-    name: $views.selected?.name,
-  }
   $: type = $tables.selected?.type
   $: isInternal = type !== "external"
+  $: id = $tables.selected?._id
+  $: search = searchTable(id)
+  $: columnOptions = Object.keys($search.schema || {})
 
-  // Fetch rows for specified table
+  // Fetches new data whenever the table changes
-  $: {
+  const searchTable = tableId => {
-    loading = true
+    return fetchTableData({
-    const loadingTableId = $tables.selected?._id
+      tableId,
-    api.fetchDataForTable($tables.selected?._id).then(rows => {
+      schema,
-      loading = false
+      limit: 10,
-
+      paginate: true,
-      // If we started a slow request then quickly change table, sometimes
+    })
-      // the old data overwrites the new data.
-      // This check ensures that we don't do that.
-      if (loadingTableId !== $tables.selected?._id) {
-        return
   }
 
-      data = rows || []
+  // Fetch data whenever sorting option changes
+  const onSort = e => {
+    search.update({
+      sortColumn: e.detail.column,
+      sortOrder: e.detail.order,
+    })
+  }
+
+  // Fetch data whenever filters change
+  const onFilter = e => {
+    search.update({
+      filters: e.detail,
+    })
+  }
+
+  // Fetch data whenever schema changes
+  const onUpdateColumns = () => {
+    search.update({
+      schema,
     })
   }
 </script>
 
+<div>
   <Table
-  {title}
+    title={$tables.selected?.name}
     {schema}
-  tableId={$tables.selected?._id}
-  {data}
     {type}
-  allowEditing={true}
+    tableId={id}
+    data={$search.rows}
     bind:hideAutocolumns
-  {loading}
+    loading={$search.loading}
+    on:sort={onSort}
+    allowEditing
+    disableSorting
+    on:updatecolumns={onUpdateColumns}
+    on:updaterows={search.refresh}
   >
     {#if isInternal}
-    <CreateColumnButton />
+      <CreateColumnButton on:updatecolumns={onUpdateColumns} />
     {/if}
     {#if schema && Object.keys(schema).length > 0}
       {#if !isUsersTable}
         <CreateRowButton
+          on:updaterows={search.refresh}
           title={"Create row"}
           modalContentComponent={CreateEditRow}
         />
@@ -75,5 +94,32 @@
       <HideAutocolumnButton bind:hideAutocolumns />
       <!-- always have the export last -->
       <ExportButton view={$tables.selected?._id} />
+      {#key id}
+        <TableFilterButton {schema} on:change={onFilter} />
+      {/key}
     {/if}
   </Table>
+  {#key id}
+    <div in:fade={{ delay: 200, duration: 100 }}>
+      <div class="pagination">
+        <Pagination
+          page={$search.pageNumber + 1}
+          hasPrevPage={$search.hasPrevPage}
+          hasNextPage={$search.hasNextPage}
+          goToPrevPage={$search.loading ? null : search.prevPage}
+          goToNextPage={$search.loading ? null : search.nextPage}
+        />
+      </div>
+    </div>
+  {/key}
+</div>
+
+<style>
+  .pagination {
+    display: flex;
+    flex-direction: row;
+    justify-content: flex-end;
+    align-items: center;
+    margin-top: var(--spacing-xl);
+  }
+</style>

packages/builder/src/components/backend/DataTable/Table.svelte

@@ -1,8 +1,7 @@
 <script>
   import { fade } from "svelte/transition"
   import { goto, params } from "@roxi/routify"
-  import { Table, Modal, Heading, notifications } from "@budibase/bbui"
+  import { Table, Modal, Heading, notifications, Layout } from "@budibase/bbui"
-
   import api from "builderStore/api"
   import Spinner from "components/common/Spinner.svelte"
   import DeleteRowsButton from "./buttons/DeleteRowsButton.svelte"
@@ -21,6 +20,7 @@
   export let hideAutocolumns
   export let rowCount
   export let type
+  export let disableSorting = false
 
   let selectedRows = []
   let editableColumn
@@ -98,13 +98,14 @@
   }
 </script>
 
+<Layout noPadding gap="S">
   <div>
     <div class="table-title">
       {#if title}
         <Heading size="S">{title}</Heading>
       {/if}
       {#if loading}
-      <div transition:fade>
+        <div transition:fade|local>
           <Spinner size="10" />
         </div>
       {/if}
@@ -112,17 +113,19 @@
     <div class="popovers">
       <slot />
       {#if !isUsersTable && selectedRows.length > 0}
-      <DeleteRowsButton {selectedRows} {deleteRows} />
+        <DeleteRowsButton on:updaterows {selectedRows} {deleteRows} />
       {/if}
     </div>
   </div>
   {#key tableId}
+    <div class="table-wrapper" in:fade={{ delay: 200, duration: 100 }}>
       <Table
         {data}
         {schema}
         {loading}
         {customRenderers}
         {rowCount}
+        {disableSorting}
         bind:selectedRows
         allowSelectRows={allowEditing && !isUsersTable}
         allowEditRows={allowEditing}
@@ -131,14 +134,21 @@
         on:editcolumn={e => editColumn(e.detail)}
         on:editrow={e => editRow(e.detail)}
         on:clickrelationship={e => selectRelationship(e.detail)}
+        on:sort
       />
+    </div>
   {/key}
+</Layout>
 
 <Modal bind:this={editRowModal}>
-  <svelte:component this={editRowComponent} row={editableRow} />
+  <svelte:component this={editRowComponent} on:updaterows row={editableRow} />
 </Modal>
 <Modal bind:this={editColumnModal}>
-  <CreateEditColumn field={editableColumn} onClosed={editColumnModal.hide} />
+  <CreateEditColumn
+    field={editableColumn}
+    on:updatecolumns
+    onClosed={editColumnModal.hide}
+  />
 </Modal>
 
 <style>
@@ -152,6 +162,9 @@
   .table-title > div {
     margin-left: var(--spacing-xs);
   }
+  .table-wrapper {
+    overflow: hidden;
+  }
 
   .popovers {
     display: flex;

packages/builder/src/components/backend/DataTable/ViewDataTable.svelte

@@ -5,7 +5,7 @@
   import Table from "./Table.svelte"
   import CalculateButton from "./buttons/CalculateButton.svelte"
   import GroupByButton from "./buttons/GroupByButton.svelte"
-  import FilterButton from "./buttons/FilterButton.svelte"
+  import ViewFilterButton from "./buttons/ViewFilterButton.svelte"
   import ExportButton from "./buttons/ExportButton.svelte"
   import ManageAccessButton from "./buttons/ManageAccessButton.svelte"
   import HideAutocolumnButton from "./buttons/HideAutocolumnButton.svelte"
@@ -61,7 +61,7 @@
   allowEditing={!view?.calculation}
   bind:hideAutocolumns
 >
-  <FilterButton {view} />
+  <ViewFilterButton {view} />
   <CalculateButton {view} />
   {#if view.calculation}
     <GroupByButton {view} />

packages/builder/src/components/backend/DataTable/buttons/CreateColumnButton.svelte

@@ -9,5 +9,5 @@
   Create column
 </ActionButton>
 <Modal bind:this={modal}>
-  <CreateEditColumn />
+  <CreateEditColumn on:updatecolumns />
 </Modal>

packages/builder/src/components/backend/DataTable/buttons/CreateRowButton.svelte

@@ -12,5 +12,5 @@
   {title}
 </ActionButton>
 <Modal bind:this={modal}>
-  <svelte:component this={modalContentComponent} />
+  <svelte:component this={modalContentComponent} on:updaterows />
 </Modal>

packages/builder/src/components/backend/DataTable/buttons/DeleteRowsButton.svelte

@@ -1,15 +1,18 @@
 <script>
+  import { createEventDispatcher } from "svelte"
   import { Button } from "@budibase/bbui"
   import ConfirmDialog from "components/common/ConfirmDialog.svelte"
 
   export let selectedRows
   export let deleteRows
 
+  const dispatch = createEventDispatcher()
   let modal
 
   async function confirmDeletion() {
     await deleteRows()
     modal?.hide()
+    dispatch("updaterows")
   }
 </script>
 

packages/builder/src/components/backend/DataTable/buttons/TableFilterButton.svelte

@@ -0,0 +1,46 @@
+<script>
+  import { createEventDispatcher } from "svelte"
+  import { ActionButton, Modal, ModalContent } from "@budibase/bbui"
+  import FilterDrawer from "components/design/PropertiesPanel/PropertyControls/FilterEditor/FilterDrawer.svelte"
+
+  export let schema
+  export let filters
+
+  const dispatch = createEventDispatcher()
+  let modal
+  let tempValue = filters || []
+
+  $: schemaFields = Object.values(schema || {})
+</script>
+
+<ActionButton
+  icon="Filter"
+  size="S"
+  quiet
+  on:click={modal.show}
+  active={tempValue?.length > 0}
+>
+  Filter
+</ActionButton>
+<Modal bind:this={modal}>
+  <ModalContent
+    title="Filter"
+    confirmText="Save"
+    size="XL"
+    onConfirm={() => dispatch("change", tempValue)}
+  >
+    <div class="wrapper">
+      <FilterDrawer
+        allowBindings={false}
+        bind:filters={tempValue}
+        {schemaFields}
+      />
+    </div>
+  </ModalContent>
+</Modal>
+
+<style>
+  .wrapper :global(.main) {
+    padding: 0;
+  }
+</style>

packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte

@@ -10,6 +10,7 @@
     ModalContent,
     Context,
   } from "@budibase/bbui"
+  import { createEventDispatcher } from "svelte"
   import { cloneDeep } from "lodash/fp"
   import { tables } from "stores/backend"
   import { TableNames, UNEDITABLE_USER_FIELDS } from "constants"
@@ -30,8 +31,9 @@
   const AUTO_TYPE = "auto"
   const FORMULA_TYPE = FIELDS.FORMULA.type
   const LINK_TYPE = FIELDS.LINK.type
-  let fieldDefinitions = cloneDeep(FIELDS)
+  const dispatch = createEventDispatcher()
   const { hide } = getContext(Context.Modal)
+  let fieldDefinitions = cloneDeep(FIELDS)
 
   export let field = {
     type: "string",
@@ -81,12 +83,13 @@
     if (field.type === AUTO_TYPE) {
       field = buildAutoColumn($tables.draft.name, field.name, field.subtype)
     }
-    tables.saveField({
+    await tables.saveField({
       originalName,
       field,
       primaryDisplay,
       indexes,
     })
+    dispatch("updatecolumns")
   }
 
   function deleteColumn() {
@@ -99,6 +102,7 @@
       hide()
       deletion = false
     }
+    dispatch("updatecolumns")
   }
 
   function handleTypeChange(event) {

packages/builder/src/components/backend/DataTable/modals/CreateEditRow.svelte

@@ -1,4 +1,5 @@
 <script>
+  import { createEventDispatcher } from "svelte"
   import { tables, rows } from "stores/backend"
   import { notifications } from "@budibase/bbui"
   import RowFieldControl from "../RowFieldControl.svelte"
@@ -12,6 +13,7 @@
   export let row = {}
 
   let errors = []
+  const dispatch = createEventDispatcher()
 
   $: creating = row?._id == null
   $: table = row.tableId
@@ -43,6 +45,7 @@
 
     notifications.success("Row saved successfully.")
     rows.save(rowResponse)
+    dispatch("updaterows")
   }
 </script>
 

packages/builder/src/components/backend/DataTable/modals/CreateEditUser.svelte

@@ -1,4 +1,5 @@
 <script>
+  import { createEventDispatcher } from "svelte"
   import { tables, rows } from "stores/backend"
   import { roles } from "stores/backend"
   import { notifications } from "@budibase/bbui"
@@ -9,6 +10,7 @@
 
   export let row = {}
 
+  const dispatch = createEventDispatcher()
   let errors = []
 
   $: creating = row?._id == null
@@ -71,6 +73,7 @@
 
     notifications.success("User saved successfully")
     rows.save(rowResponse)
+    dispatch("updaterows")
   }
 </script>
 

packages/builder/src/components/backend/DatasourceNavigator/DatasourceNavigator.svelte

@@ -1,9 +1,9 @@
 <script>
   import { onMount } from "svelte"
   import { get } from "svelte/store"
-  import { goto } from "@roxi/routify"
+  import { goto, params } from "@roxi/routify"
   import { BUDIBASE_INTERNAL_DB } from "constants"
-  import { database, datasources, queries, tables } from "stores/backend"
+  import { database, datasources, queries, tables, views } from "stores/backend"
   import EditDatasourcePopover from "./popovers/EditDatasourcePopover.svelte"
   import EditQueryPopover from "./popovers/EditQueryPopover.svelte"
   import NavItem from "components/common/NavItem.svelte"
@@ -11,16 +11,27 @@
   import ICONS from "./icons"
 
   let openDataSources = []
-  $: enrichedDataSources = $datasources.list.map(datasource => ({
+  $: enrichedDataSources = $datasources.list.map(datasource => {
+    const selected = $datasources.selected === datasource._id
+    const open = openDataSources.includes(datasource._id)
+    const containsSelected = containsActiveEntity(datasource)
+    return {
       ...datasource,
-    open:
+      selected,
-      openDataSources.includes(datasource._id) ||
+      open: selected || open || containsSelected,
-      containsActiveTable(datasource),
+    }
-    selected: $datasources.selected === datasource._id,
+  })
-  }))
+  $: openDataSource = enrichedDataSources.find(x => x.open)
+  $: {
+    // Ensure the open data source is always included in the list of open
+    // data sources
+    if (openDataSource) {
+      openNode(openDataSource)
+    }
+  }
 
   function selectDatasource(datasource) {
-    toggleNode(datasource)
+    openNode(datasource)
     datasources.select(datasource._id)
     $goto(`./datasource/${datasource._id}`)
   }
@@ -30,12 +41,22 @@
     $goto(`./datasource/${query.datasourceId}/${query._id}`)
   }
 
+  function closeNode(datasource) {
+    openDataSources = openDataSources.filter(id => datasource._id !== id)
+  }
+
+  function openNode(datasource) {
+    if (!openDataSources.includes(datasource._id)) {
+      openDataSources = [...openDataSources, datasource._id]
+    }
+  }
+
   function toggleNode(datasource) {
     const isOpen = openDataSources.includes(datasource._id)
     if (isOpen) {
-      openDataSources = openDataSources.filter(id => datasource._id !== id)
+      closeNode(datasource)
     } else {
-      openDataSources = [...openDataSources, datasource._id]
+      openNode(datasource)
     }
   }
 
@@ -44,16 +65,35 @@
     queries.fetch()
   })
 
-  const containsActiveTable = datasource => {
+  const containsActiveEntity = datasource => {
-    const activeTableId = get(tables).selected?._id
+    // If we're view a query then the data source ID is in the URL
+    if ($params.selectedDatasource === datasource._id) {
+      return true
+    }
+
+    // If there are no entities it can't contain anything
     if (!datasource.entities) {
       return false
     }
-    let tableOptions = datasource.entities
+
-    if (!Array.isArray(tableOptions)) {
+    // Get a list of table options
-      tableOptions = Object.values(tableOptions)
+    let options = datasource.entities
+    if (!Array.isArray(options)) {
+      options = Object.values(options)
     }
-    return tableOptions.find(x => x._id === activeTableId) != null
+
+    // Check for a matching table
+    if ($params.selectedTable) {
+      const selectedTable = get(tables).selected?._id
+      return options.find(x => x._id === selectedTable) != null
+    }
+
+    // Check for a matching view
+    const selectedView = get(views).selected?.name
+    const table = options.find(table => {
+      return table.views?.[selectedView] != null
+    })
+    return table != null
   }
 </script>
 

packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/IntegrationConfigForm.svelte

@@ -1,20 +1,28 @@
 <script>
-  import { Label, Input, Layout, Toggle } from "@budibase/bbui"
+  import { Label, Input, Layout, Toggle, Button } from "@budibase/bbui"
   import KeyValueBuilder from "components/integration/KeyValueBuilder.svelte"
   import { capitalise } from "helpers"
 
   export let integration
   export let schema
+  let addButton
 </script>
 
 <form>
   <Layout gap="S">
     {#each Object.keys(schema) as configKey}
       {#if schema[configKey].type === "object"}
+        <div class="form-row ssl">
           <Label>{capitalise(configKey)}</Label>
+          <Button secondary thin outline on:click={addButton.addEntry()}
+            >Add</Button
+          >
+        </div>
         <KeyValueBuilder
+          bind:this={addButton}
           defaults={schema[configKey].default}
           bind:object={integration[configKey]}
+          noAddButton={true}
         />
       {:else if schema[configKey].type === "boolean"}
         <div class="form-row">
@@ -42,4 +50,11 @@
     grid-gap: var(--spacing-l);
     align-items: center;
   }
+
+  .form-row.ssl {
+    display: grid;
+    grid-template-columns: 20% 20%;
+    grid-gap: var(--spacing-l);
+    align-items: center;
+  }
 </style>

packages/builder/src/components/backend/DatasourceNavigator/modals/CreateDatasourceModal.svelte

@@ -1,74 +1,160 @@
 <script>
-  import { goto } from "@roxi/routify"
+  import { ModalContent, Modal, Body, Layout, Detail } from "@budibase/bbui"
-  import { datasources } from "stores/backend"
+  import { onMount } from "svelte"
-  import { notifications } from "@budibase/bbui"
+  import ICONS from "../icons"
-  import { Input, Label, ModalContent, Modal, Context } from "@budibase/bbui"
+  import api from "builderStore/api"
-  import TableIntegrationMenu from "../TableIntegrationMenu/index.svelte"
+  import { IntegrationNames } from "constants"
   import CreateTableModal from "components/backend/TableNavigator/modals/CreateTableModal.svelte"
-  import analytics, { Events } from "analytics"
+  import DatasourceConfigModal from "components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte"
-  import { getContext } from "svelte"
 
-  const modalContext = getContext(Context.Modal)
+  export let modal
+  let integrations = []
+  let integration = {}
+  let internalTableModal
+  let externalDatasourceModal
 
-  let tableModal
+  const INTERNAL = "BUDIBASE"
-  let name
-  let error = ""
-  let integration
 
-  $: checkOpenModal(integration && integration.type === "BUDIBASE")
+  onMount(() => {
-
+    fetchIntegrations()
-  function checkValid(evt) {
-    const datasourceName = evt.target.value
-    if (
-      $datasources?.list.some(datasource => datasource.name === datasourceName)
-    ) {
-      error = `Datasource with name ${datasourceName} already exists. Please choose another name.`
-      return
-    }
-    error = ""
-  }
-
-  function checkOpenModal(isInternal) {
-    if (isInternal) {
-      tableModal.show()
-    }
-  }
-
-  async function saveDatasource() {
-    const { type, plus, ...config } = integration
-
-    // Create datasource
-    const response = await datasources.save({
-      name,
-      source: type,
-      config,
-      plus,
   })
-    notifications.success(`Datasource ${name} created successfully.`)
-    analytics.captureEvent(Events.DATASOURCE.CREATED, { name, type })
 
-    // Navigate to new datasource
+  function selectIntegration(integrationType) {
-    $goto(`./datasource/${response._id}`)
+    const selected = integrations[integrationType]
+
+    // build the schema
+    const config = {}
+    for (let key of Object.keys(selected.datasource)) {
+      config[key] = selected.datasource[key].default
+    }
+    integration = {
+      type: integrationType,
+      plus: selected.plus,
+      config,
+      schema: selected.datasource,
+    }
+  }
+
+  function chooseNextModal() {
+    if (integration.type === INTERNAL) {
+      externalDatasourceModal.hide()
+      internalTableModal.show()
+    } else {
+      externalDatasourceModal.show()
+    }
+  }
+
+  async function fetchIntegrations() {
+    const response = await api.get("/api/integrations")
+    const json = await response.json()
+    integrations = {
+      [INTERNAL]: { datasource: {}, name: "INTERNAL/CSV" },
+      ...json,
+    }
+    return json
   }
 </script>
 
-<Modal bind:this={tableModal} on:hide={modalContext.hide}>
+<Modal bind:this={internalTableModal}>
-  <CreateTableModal bind:name />
+  <CreateTableModal />
 </Modal>
+
+<Modal bind:this={externalDatasourceModal}>
+  <DatasourceConfigModal {integration} />
+</Modal>
+
+<Modal bind:this={modal}>
   <ModalContent
-  title="Create Datasource"
+    disabled={!Object.keys(integration).length}
-  size="L"
+    title="Data"
-  confirmText="Create"
+    confirmText="Continue"
-  onConfirm={saveDatasource}
+    showCancelButton={false}
-  disabled={error || !name || !integration?.type}
+    size="M"
+    onConfirm={() => {
+      chooseNextModal()
+    }}
   >
-  <Input
+    <Layout noPadding>
-    data-cy="datasource-name-input"
+      <Body size="XS"
-    label="Datasource Name"
+        >All apps need data. You can connect to a data source below, or add data
-    on:input={checkValid}
+        to your app using Budibase's built-in database.
-    bind:value={name}
+      </Body>
-    {error}
+      <div
+        class:selected={integration.type === INTERNAL}
+        on:click={() => selectIntegration(INTERNAL)}
+        class="item hoverable"
+      >
+        <div class="item-body">
+          <svelte:component this={ICONS.BUDIBASE} height="18" width="18" />
+          <span class="icon-spacing"> <Body size="S">Budibase DB</Body></span>
+        </div>
+      </div>
+    </Layout>
+
+    <Layout gap="XS" noPadding>
+      <div class="title-spacing">
+        <Detail size="S">Connect to data source</Detail>
+      </div>
+      <div class="item-list">
+        {#each Object.entries(integrations).filter(([key]) => key !== INTERNAL) as [integrationType, schema]}
+          <div
+            class:selected={integration.type === integrationType}
+            on:click={() => selectIntegration(integrationType)}
+            class="item hoverable"
+          >
+            <div class="item-body">
+              <svelte:component
+                this={ICONS[integrationType]}
+                height="18"
+                width="18"
               />
-  <Label>Datasource Type</Label>
+
-  <TableIntegrationMenu bind:integration />
+              <span class="icon-spacing">
+                <Body size="S"
+                  >{schema.name || IntegrationNames[integrationType]}</Body
+                ></span
+              >
+            </div>
+          </div>
+        {/each}
+      </div>
+    </Layout>
   </ModalContent>
+</Modal>
+
+<style>
+  .icon-spacing {
+    margin-left: var(--spacing-m);
+  }
+  .item-body {
+    display: flex;
+    margin-left: var(--spacing-m);
+  }
+  .item-list {
+    display: grid;
+    grid-template-columns: repeat(auto-fit, minmax(150px, 1fr));
+    grid-gap: var(--spectrum-alias-grid-baseline);
+  }
+
+  .item {
+    cursor: pointer;
+    display: grid;
+    grid-gap: var(--spectrum-alias-grid-margin-xsmall);
+    padding: var(--spectrum-alias-item-padding-s);
+    background: var(--spectrum-alias-background-color-secondary);
+    transition: 0.3s all;
+    border: solid var(--spectrum-alias-border-color);
+    border-radius: 5px;
+    box-sizing: border-box;
+    border-width: 2px;
+  }
+
+  .selected {
+    background: var(--spectrum-alias-background-color-tertiary);
+  }
+
+  .item:hover,
+  .selected {
+    background: var(--spectrum-alias-background-color-tertiary);
+  }
+</style>

packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte

@@ -0,0 +1,72 @@
+<script>
+  import { goto } from "@roxi/routify"
+  import { ModalContent, notifications, Body, Layout } from "@budibase/bbui"
+  import analytics, { Events } from "analytics"
+  import IntegrationConfigForm from "components/backend/DatasourceNavigator/TableIntegrationMenu/IntegrationConfigForm.svelte"
+  import { datasources } from "stores/backend"
+  import { IntegrationNames } from "constants"
+
+  export let integration
+
+  function prepareData() {
+    let datasource = {}
+    let existingTypeCount = $datasources.list.filter(
+      ds => ds.source == integration.type
+    ).length
+
+    let baseName = IntegrationNames[integration.type]
+    let name =
+      existingTypeCount == 0 ? baseName : `${baseName}-${existingTypeCount + 1}`
+
+    datasource.type = "datasource"
+    datasource.source = integration.type
+    datasource.config = integration.config
+    datasource.name = name
+    datasource.plus = integration.plus
+
+    return datasource
+  }
+  async function saveDatasource() {
+    const datasource = prepareData()
+    try {
+      // Create datasource
+      const resp = await datasources.save(datasource, datasource.plus)
+
+      await datasources.select(resp._id)
+      $goto(`./datasource/${resp._id}`)
+      notifications.success(`Datasource updated successfully.`)
+      analytics.captureEvent(Events.DATASOURCE.CREATED, {
+        name: resp.name,
+        source: resp.source,
+      })
+      return true
+    } catch (err) {
+      notifications.error(`Error saving datasource: ${err}`)
+      return false
+    }
+  }
+</script>
+
+<ModalContent
+  title={`Connect to ${IntegrationNames[integration.type]}`}
+  onConfirm={() => saveDatasource()}
+  confirmText={integration.plus
+    ? "Fetch tables from database"
+    : "Save and continue to query"}
+  cancelText="Back"
+  size="M"
+>
+  <Layout noPadding>
+    <Body size="XS"
+      >Connect your database to Budibase using the config below.
+    </Body>
+  </Layout>
+
+  <IntegrationConfigForm
+    schema={integration.schema}
+    bind:integration={integration.config}
+  />
+</ModalContent>
+
+<style>
+</style>

packages/builder/src/components/design/PropertiesPanel/PropertyControls/ConditionalUIDrawer.svelte

@@ -12,7 +12,7 @@
   import { dndzone } from "svelte-dnd-action"
   import { generate } from "shortid"
   import DrawerBindableInput from "components/common/bindings/DrawerBindableInput.svelte"
-  import { OperatorOptions, getValidOperatorsForType } from "helpers/lucene"
+  import { OperatorOptions, getValidOperatorsForType } from "constants/lucene"
   import { selectedComponent, store } from "builderStore"
   import { getComponentForSettingType } from "./componentSettings"
   import PropertyControl from "./PropertyControl.svelte"

packages/builder/src/components/design/PropertiesPanel/PropertyControls/FilterEditor/FilterDrawer.svelte

@@ -13,18 +13,20 @@
   import DrawerBindableInput from "components/common/bindings/DrawerBindableInput.svelte"
   import BindingPanel from "components/common/bindings/BindingPanel.svelte"
   import { generate } from "shortid"
-  import { getValidOperatorsForType, OperatorOptions } from "helpers/lucene"
+  import { getValidOperatorsForType, OperatorOptions } from "constants/lucene"
 
   export let schemaFields
   export let filters = []
   export let bindings = []
   export let panel = BindingPanel
+  export let allowBindings = true
 
   const BannedTypes = ["link", "attachment", "formula"]
 
   $: fieldOptions = (schemaFields ?? [])
     .filter(field => !BannedTypes.includes(field.type))
     .map(field => field.name)
+  $: valueTypeOptions = allowBindings ? ["Value", "Binding"] : ["Value"]
 
   const addFilter = () => {
     filters = [
@@ -93,7 +95,7 @@
     <Layout noPadding>
       <Body size="S">
         {#if !filters?.length}
-          Add your first filter column.
+          Add your first filter expression.
         {:else}
           Results are filtered to only those which match all of the following
           constraints.
@@ -117,7 +119,7 @@
             />
             <Select
               disabled={filter.noValue || !filter.field}
-              options={["Value", "Binding"]}
+              options={valueTypeOptions}
               bind:value={filter.valueType}
               placeholder={null}
             />

packages/builder/src/components/design/PropertiesPanel/PropertyControls/FilterEditor/FilterEditor.svelte

@@ -20,7 +20,6 @@
   $: dataSource = getDatasourceForProvider($currentAsset, componentInstance)
   $: schema = getSchemaForDatasource($currentAsset, dataSource)?.schema
   $: schemaFields = Object.values(schema || {})
-  $: internalTable = dataSource?.type === "table"
 
   const saveFilter = async () => {
     dispatch("change", tempValue)

packages/builder/src/components/integration/KeyValueBuilder.svelte

@@ -4,6 +4,7 @@
   export let defaults
   export let object = defaults || {}
   export let readOnly
+  export let noAddButton
 
   let fields = Object.entries(object).map(([name, value]) => ({ name, value }))
 
@@ -12,7 +13,7 @@
     {}
   )
 
-  function addEntry() {
+  export function addEntry() {
     fields = [...fields, {}]
   }
 
@@ -32,7 +33,7 @@
     {/if}
   {/each}
 </div>
-{#if !readOnly}
+{#if !readOnly && !noAddButton}
   <div>
     <Button secondary thin outline on:click={addEntry}>Add</Button>
   </div>

packages/builder/src/components/upgrade/UpgradeModal.svelte

@@ -4,7 +4,7 @@
   let upgradeModal
 
   const onConfirm = () => {
-    window.open("https://accounts.budibase.com/install", "_blank")
+    window.open("https://account.budibase.app/portal/install", "_blank")
   }
 </script>
 

packages/builder/src/constants/index.js

@@ -15,6 +15,20 @@ export const AppStatus = {
   DEPLOYED: "published",
 }
 
+export const IntegrationNames = {
+  POSTGRES: "PostgreSQL",
+  MONGODB: "MongoDB",
+  COUCHDB: "CouchDB",
+  S3: "S3",
+  MYSQL: "MySQL",
+  REST: "REST",
+  DYNAMODB: "DynamoDB",
+  ELASTICSEARCH: "ElasticSearch",
+  SQL_SERVER: "SQL Server",
+  AIRTABLE: "Airtable",
+  ARANGODB: "ArangoDB",
+}
+
 // fields on the user table that cannot be edited
 export const UNEDITABLE_USER_FIELDS = [
   "email",

packages/builder/src/constants/lucene.js

@@ -0,0 +1,97 @@
+/**
+ * Operator options for lucene queries
+ */
+export const OperatorOptions = {
+  Equals: {
+    value: "equal",
+    label: "Equals",
+  },
+  NotEquals: {
+    value: "notEqual",
+    label: "Not equals",
+  },
+  Empty: {
+    value: "empty",
+    label: "Is empty",
+  },
+  NotEmpty: {
+    value: "notEmpty",
+    label: "Is not empty",
+  },
+  StartsWith: {
+    value: "string",
+    label: "Starts with",
+  },
+  Like: {
+    value: "fuzzy",
+    label: "Like",
+  },
+  MoreThan: {
+    value: "rangeLow",
+    label: "More than",
+  },
+  LessThan: {
+    value: "rangeHigh",
+    label: "Less than",
+  },
+  Contains: {
+    value: "equal",
+    label: "Contains",
+  },
+  NotContains: {
+    value: "notEqual",
+    label: "Does Not Contain",
+  },
+}
+
+/**
+ * Returns the valid operator options for a certain data type
+ * @param type the data type
+ */
+export const getValidOperatorsForType = type => {
+  const Op = OperatorOptions
+  if (type === "string") {
+    return [
+      Op.Equals,
+      Op.NotEquals,
+      Op.StartsWith,
+      Op.Like,
+      Op.Empty,
+      Op.NotEmpty,
+    ]
+  } else if (type === "number") {
+    return [
+      Op.Equals,
+      Op.NotEquals,
+      Op.MoreThan,
+      Op.LessThan,
+      Op.Empty,
+      Op.NotEmpty,
+    ]
+  } else if (type === "options") {
+    return [Op.Equals, Op.NotEquals, Op.Empty, Op.NotEmpty]
+  } else if (type === "array") {
+    return [Op.Contains, Op.NotContains, Op.Empty, Op.NotEmpty]
+  } else if (type === "boolean") {
+    return [Op.Equals, Op.NotEquals, Op.Empty, Op.NotEmpty]
+  } else if (type === "longform") {
+    return [
+      Op.Equals,
+      Op.NotEquals,
+      Op.StartsWith,
+      Op.Like,
+      Op.Empty,
+      Op.NotEmpty,
+    ]
+  } else if (type === "datetime") {
+    return [
+      Op.Equals,
+      Op.NotEquals,
+      Op.MoreThan,
+      Op.LessThan,
+      Op.Empty,
+      Op.NotEmpty,
+    ]
+  }
+  return []
+}

packages/builder/src/helpers/fetchTableData.js

@@ -0,0 +1,206 @@
+// Do not use any aliased imports in common files, as these will be bundled
+// by multiple bundlers which may not be able to resolve them
+import { writable, derived, get } from "svelte/store"
+import * as API from "../builderStore/api"
+import { buildLuceneQuery } from "./lucene"
+
+const defaultOptions = {
+  tableId: null,
+  filters: null,
+  limit: 10,
+  sortColumn: null,
+  sortOrder: "ascending",
+  paginate: true,
+  schema: null,
+}
+
+export const fetchTableData = opts => {
+  // Save option set so we can override it later rather than relying on params
+  let options = {
+    ...defaultOptions,
+    ...opts,
+  }
+
+  // Local non-observable state
+  let query
+  let sortType
+  let lastBookmark
+
+  // Local observable state
+  const store = writable({
+    rows: [],
+    schema: null,
+    loading: false,
+    loaded: false,
+    bookmarks: [],
+    pageNumber: 0,
+  })
+
+  // Derive certain properties to return
+  const derivedStore = derived(store, $store => {
+    return {
+      ...$store,
+      hasNextPage: $store.bookmarks[$store.pageNumber + 1] != null,
+      hasPrevPage: $store.pageNumber > 0,
+    }
+  })
+
+  const fetchPage = async bookmark => {
+    lastBookmark = bookmark
+    const { tableId, limit, sortColumn, sortOrder, paginate } = options
+    store.update($store => ({ ...$store, loading: true }))
+    const res = await API.post(`/api/${options.tableId}/search`, {
+      tableId,
+      query,
+      limit,
+      sort: sortColumn,
+      sortOrder: sortOrder?.toLowerCase() ?? "ascending",
+      sortType,
+      paginate,
+      bookmark,
+    })
+    store.update($store => ({ ...$store, loading: false, loaded: true }))
+    return await res.json()
+  }
+
+  // Fetches a fresh set of results from the server
+  const fetchData = async () => {
+    const { tableId, schema, sortColumn, filters } = options
+
+    // Ensure table ID exists
+    if (!tableId) {
+      return
+    }
+
+    // Get and enrich schema.
+    // Ensure there are "name" properties for all fields and that field schema
+    // are objects
+    let enrichedSchema = schema
+    if (!enrichedSchema) {
+      const definition = await API.get(`/api/tables/${tableId}`)
+      enrichedSchema = definition?.schema ?? null
+    }
+    if (enrichedSchema) {
+      Object.entries(schema).forEach(([fieldName, fieldSchema]) => {
+        if (typeof fieldSchema === "string") {
+          enrichedSchema[fieldName] = {
+            type: fieldSchema,
+            name: fieldName,
+          }
+        } else {
+          enrichedSchema[fieldName] = {
+            ...fieldSchema,
+            name: fieldName,
+          }
+        }
+      })
+
+      // Save fixed schema so we can provide it later
+      options.schema = enrichedSchema
+    }
+
+    // Ensure schema exists
+    if (!schema) {
+      return
+    }
+    store.update($store => ({ ...$store, schema }))
+
+    // Work out what sort type to use
+    if (!sortColumn || !schema[sortColumn]) {
+      sortType = "string"
+    }
+    const type = schema?.[sortColumn]?.type
+    sortType = type === "number" ? "number" : "string"
+
+    // Build the lucene query
+    query = buildLuceneQuery(filters)
+
+    // Actually fetch data
+    const page = await fetchPage()
+    store.update($store => ({
+      ...$store,
+      loading: false,
+      loaded: true,
+      pageNumber: 0,
+      rows: page.rows,
+      bookmarks: page.hasNextPage ? [null, page.bookmark] : [null],
+    }))
+  }
+
+  // Fetches the next page of data
+  const nextPage = async () => {
+    const state = get(derivedStore)
+    if (state.loading || !options.paginate || !state.hasNextPage) {
+      return
+    }
+
+    // Fetch next page
+    const page = await fetchPage(state.bookmarks[state.pageNumber + 1])
+
+    // Update state
+    store.update($store => {
+      let { bookmarks, pageNumber } = $store
+      if (page.hasNextPage) {
+        bookmarks[pageNumber + 2] = page.bookmark
+      }
+      return {
+        ...$store,
+        pageNumber: pageNumber + 1,
+        rows: page.rows,
+        bookmarks,
+      }
+    })
+  }
+
+  // Fetches the previous page of data
+  const prevPage = async () => {
+    const state = get(derivedStore)
+    if (state.loading || !options.paginate || !state.hasPrevPage) {
+      return
+    }
+
+    // Fetch previous page
+    const page = await fetchPage(state.bookmarks[state.pageNumber - 1])
+
+    // Update state
+    store.update($store => {
+      return {
+        ...$store,
+        pageNumber: $store.pageNumber - 1,
+        rows: page.rows,
+      }
+    })
+  }
+
+  // Resets the data set and updates options
+  const update = async newOptions => {
+    if (newOptions) {
+      options = {
+        ...options,
+        ...newOptions,
+      }
+    }
+    await fetchData()
+  }
+
+  // Loads the same page again
+  const refresh = async () => {
+    if (get(store).loading) {
+      return
+    }
+    const page = await fetchPage(lastBookmark)
+    store.update($store => ({ ...$store, rows: page.rows }))
+  }
+
+  // Initially fetch data but don't bother waiting for the result
+  fetchData()
+
+  // Return our derived store which will be updated over time
+  return {
+    subscribe: derivedStore.subscribe,
+    nextPage,
+    prevPage,
+    update,
+    refresh,
+  }
+}

packages/builder/src/helpers/lucene.js

@@ -1,90 +1,179 @@
-export const OperatorOptions = {
+/**
-  Equals: {
+ * Builds a lucene JSON query from the filter structure generated in the builder
-    value: "equal",
+ * @param filter the builder filter structure
-    label: "Equals",
+ */
-  },
+export const buildLuceneQuery = filter => {
-  NotEquals: {
+  let query = {
-    value: "notEqual",
+    string: {},
-    label: "Not equals",
+    fuzzy: {},
-  },
+    range: {},
-  Empty: {
+    equal: {},
-    value: "empty",
+    notEqual: {},
-    label: "Is empty",
+    empty: {},
-  },
+    notEmpty: {},
-  NotEmpty: {
+    contains: {},
-    value: "notEmpty",
+    notContains: {},
-    label: "Is not empty",
+  }
-  },
+  if (Array.isArray(filter)) {
-  StartsWith: {
+    filter.forEach(expression => {
-    value: "string",
+      let { operator, field, type, value } = expression
-    label: "Starts with",
+      // Parse all values into correct types
-  },
+      if (type === "datetime" && value) {
-  Like: {
+        value = new Date(value).toISOString()
-    value: "fuzzy",
+      }
-    label: "Like",
+      if (type === "number") {
-  },
+        value = parseFloat(value)
-  MoreThan: {
+      }
-    value: "rangeLow",
+      if (type === "boolean") {
-    label: "More than",
+        value = `${value}`?.toLowerCase() === "true"
-  },
+      }
-  LessThan: {
+      if (operator.startsWith("range")) {
-    value: "rangeHigh",
+        if (!query.range[field]) {
-    label: "Less than",
+          query.range[field] = {
-  },
+            low:
-  Contains: {
+              type === "number"
-    value: "equal",
+                ? Number.MIN_SAFE_INTEGER
-    label: "Contains",
+                : "0000-00-00T00:00:00.000Z",
-  },
+            high:
-  NotContains: {
+              type === "number"
-    value: "notEqual",
+                ? Number.MAX_SAFE_INTEGER
-    label: "Does Not Contain",
+                : "9999-00-00T00:00:00.000Z",
-  },
+          }
+        }
+        if (operator === "rangeLow" && value != null && value !== "") {
+          query.range[field].low = value
+        } else if (operator === "rangeHigh" && value != null && value !== "") {
+          query.range[field].high = value
+        }
+      } else if (query[operator]) {
+        if (type === "boolean") {
+          // Transform boolean filters to cope with null.
+          // "equals false" needs to be "not equals true"
+          // "not equals false" needs to be "equals true"
+          if (operator === "equal" && value === false) {
+            query.notEqual[field] = true
+          } else if (operator === "notEqual" && value === false) {
+            query.equal[field] = true
+          } else {
+            query[operator][field] = value
+          }
+        } else {
+          query[operator][field] = value
+        }
+      }
+    })
   }
 
-export const getValidOperatorsForType = type => {
+  return query
-  const Op = OperatorOptions
-  if (type === "string") {
-    return [
-      Op.Equals,
-      Op.NotEquals,
-      Op.StartsWith,
-      Op.Like,
-      Op.Empty,
-      Op.NotEmpty,
-    ]
-  } else if (type === "number") {
-    return [
-      Op.Equals,
-      Op.NotEquals,
-      Op.MoreThan,
-      Op.LessThan,
-      Op.Empty,
-      Op.NotEmpty,
-    ]
-  } else if (type === "options") {
-    return [Op.Equals, Op.NotEquals, Op.Empty, Op.NotEmpty]
-  } else if (type === "array") {
-    return [Op.Contains, Op.NotContains, Op.Empty, Op.NotEmpty]
-  } else if (type === "boolean") {
-    return [Op.Equals, Op.NotEquals, Op.Empty, Op.NotEmpty]
-  } else if (type === "longform") {
-    return [
-      Op.Equals,
-      Op.NotEquals,
-      Op.StartsWith,
-      Op.Like,
-      Op.Empty,
-      Op.NotEmpty,
-    ]
-  } else if (type === "datetime") {
-    return [
-      Op.Equals,
-      Op.NotEquals,
-      Op.MoreThan,
-      Op.LessThan,
-      Op.Empty,
-      Op.NotEmpty,
-    ]
 }
-  return []
+
+/**
+ * Performs a client-side lucene search on an array of data
+ * @param docs the data
+ * @param query the JSON lucene query
+ */
+export const luceneQuery = (docs, query) => {
+  if (!query) {
+    return docs
+  }
+
+  // Iterates over a set of filters and evaluates a fail function against a doc
+  const match = (type, failFn) => doc => {
+    const filters = Object.entries(query[type] || {})
+    for (let i = 0; i < filters.length; i++) {
+      if (failFn(filters[i][0], filters[i][1], doc)) {
+        return false
+      }
+    }
+    return true
+  }
+
+  // Process a string match (fails if the value does not start with the string)
+  const stringMatch = match("string", (key, value, doc) => {
+    return !doc[key] || !doc[key].startsWith(value)
+  })
+
+  // Process a fuzzy match (treat the same as starts with when running locally)
+  const fuzzyMatch = match("fuzzy", (key, value, doc) => {
+    return !doc[key] || !doc[key].startsWith(value)
+  })
+
+  // Process a range match
+  const rangeMatch = match("range", (key, value, doc) => {
+    return !doc[key] || doc[key] < value.low || doc[key] > value.high
+  })
+
+  // Process an equal match (fails if the value is different)
+  const equalMatch = match("equal", (key, value, doc) => {
+    return value != null && value !== "" && doc[key] !== value
+  })
+
+  // Process a not-equal match (fails if the value is the same)
+  const notEqualMatch = match("notEqual", (key, value, doc) => {
+    return value != null && value !== "" && doc[key] === value
+  })
+
+  // Process an empty match (fails if the value is not empty)
+  const emptyMatch = match("empty", (key, value, doc) => {
+    return doc[key] != null && doc[key] !== ""
+  })
+
+  // Process a not-empty match (fails is the value is empty)
+  const notEmptyMatch = match("notEmpty", (key, value, doc) => {
+    return doc[key] == null || doc[key] === ""
+  })
+
+  // Match a document against all criteria
+  const docMatch = doc => {
+    return (
+      stringMatch(doc) &&
+      fuzzyMatch(doc) &&
+      rangeMatch(doc) &&
+      equalMatch(doc) &&
+      notEqualMatch(doc) &&
+      emptyMatch(doc) &&
+      notEmptyMatch(doc)
+    )
+  }
+
+  // Process all docs
+  return docs.filter(docMatch)
+}
+
+/**
+ * Performs a client-side sort from the equivalent server-side lucene sort
+ * parameters.
+ * @param docs the data
+ * @param sort the sort column
+ * @param sortOrder the sort order ("ascending" or "descending")
+ * @param sortType the type of sort ("string" or "number")
+ */
+export const luceneSort = (docs, sort, sortOrder, sortType = "string") => {
+  if (!sort || !sortOrder || !sortType) {
+    return docs
+  }
+  const parse = sortType === "string" ? x => `${x}` : x => parseFloat(x)
+  return docs.slice().sort((a, b) => {
+    const colA = parse(a[sort])
+    const colB = parse(b[sort])
+    if (sortOrder === "Descending") {
+      return colA > colB ? -1 : 1
+    } else {
+      return colA > colB ? 1 : -1
+    }
+  })
+}
+
+/**
+ * Limits the specified docs to the specified number of rows from the equivalent
+ * server-side lucene limit parameters.
+ * @param docs the data
+ * @param limit the number of docs to limit to
+ */
+export const luceneLimit = (docs, limit) => {
+  const numLimit = parseFloat(limit)
+  if (isNaN(numLimit)) {
+    return docs
+  }
+  return docs.slice(0, numLimit)
 }

packages/builder/src/helpers/warnings.js

@@ -0,0 +1,16 @@
+export const suppressWarnings = warnings => {
+  if (!warnings?.length) {
+    return
+  }
+  const regex = new RegExp(warnings.map(x => `(${x})`).join("|"), "gi")
+  const warn = console.warn
+  console.warn = (...params) => {
+    const msg = params[0]
+    if (msg && typeof msg === "string") {
+      if (msg.match(regex)) {
+        return
+      }
+    }
+    warn(...params)
+  }
+}

packages/builder/src/main.js

@@ -7,11 +7,19 @@ import "@spectrum-css/vars/dist/spectrum-light.css"
 import "@spectrum-css/vars/dist/spectrum-lightest.css"
 import "@spectrum-css/page/dist/index-vars.css"
 import "./global.css"
-
+import { suppressWarnings } from "./helpers/warnings"
 import loadSpectrumIcons from "@budibase/bbui/spectrum-icons-vite.js"
+import App from "./App.svelte"
+
+// Init spectrum icons
 loadSpectrumIcons()
 
-import App from "./App.svelte"
+// Suppress svelte runtime warnings
+suppressWarnings([
+  "was created with unknown prop",
+  "was created without expected prop",
+  "received an unexpected slot",
+])
 
 export default new App({
   target: document.getElementById("app"),

packages/builder/src/pages/builder/_layout.svelte

@@ -11,10 +11,32 @@
   $: multiTenancyEnabled = $admin.multiTenancy
   $: hasAdminUser = $admin?.checklist?.adminUser?.checked
   $: tenantSet = $auth.tenantSet
+  $: cloud = $admin.cloud
+  $: user = $auth.user
+
+  const validateTenantId = async () => {
+    // set the tenant from the url in the cloud
+    const tenantId = window.location.host.split(".")[0]
+
+    if (!tenantId.includes("localhost:")) {
+      // user doesn't have permission to access this tenant - kick them out
+      if (user?.tenantId !== tenantId) {
+        await auth.logout()
+        await auth.setOrganisation(null)
+      } else {
+        await auth.setOrganisation(tenantId)
+      }
+    }
+  }
 
   onMount(async () => {
     await auth.checkAuth()
     await admin.init()
+
+    if (cloud && multiTenancyEnabled) {
+      await validateTenantId()
+    }
+
     loaded = true
   })
 

packages/builder/src/pages/builder/app/[application]/data/_layout.svelte

@@ -1,12 +1,14 @@
 <script>
   import { goto, params } from "@roxi/routify"
-  import { Icon, Modal, Tabs, Tab } from "@budibase/bbui"
+  import { Icon, Tabs, Tab } from "@budibase/bbui"
   import { BUDIBASE_INTERNAL_DB } from "constants"
   import DatasourceNavigator from "components/backend/DatasourceNavigator/DatasourceNavigator.svelte"
   import CreateDatasourceModal from "components/backend/DatasourceNavigator/modals/CreateDatasourceModal.svelte"
 
   let selected = "Sources"
+
   let modal
+
   $: isExternal =
     $params.selectedDatasource &&
     $params.selectedDatasource !== BUDIBASE_INTERNAL_DB
@@ -23,9 +25,7 @@
       <Tab title="Sources">
         <div class="tab-content-padding">
           <DatasourceNavigator />
-          <Modal bind:this={modal}>
+          <CreateDatasourceModal bind:modal />
-            <CreateDatasourceModal />
-          </Modal>
         </div>
       </Tab>
     </Tabs>

packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/[query]/_layout.svelte

@@ -1 +1,13 @@
+<script>
+  import { params } from "@roxi/routify"
+  import { queries } from "stores/backend"
+
+  if ($params.query) {
+    const query = $queries.list.find(q => q._id === $params.query)
+    if (query) {
+      queries.select(query)
+    }
+  }
+</script>
+
 <slot />

packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/_layout.svelte

@@ -2,7 +2,7 @@
   import { params } from "@roxi/routify"
   import { datasources } from "stores/backend"
 
-  if ($params.selectedDatasource) {
+  if ($params.selectedDatasource && !$params.query) {
     const datasource = $datasources.list.find(
       m => m._id === $params.selectedDatasource
     )

packages/builder/src/pages/builder/app/[application]/data/datasource/_layout.svelte

@@ -1,14 +1 @@
-<script>
-  import { datasources } from "stores/backend"
-  import { goto, leftover } from "@roxi/routify"
-  import { onMount } from "svelte"
-
-  onMount(async () => {
-    // navigate to first datasource in list, if not already selected
-    if (!$leftover && $datasources.list.length > 0 && !$datasources.selected) {
-      $goto(`./${$datasources.list[0]._id}`)
-    }
-  })
-</script>
-
 <slot />

packages/builder/src/pages/builder/app/[application]/data/datasource/bb_internal/table/[selectedTable]/_layout.svelte

@@ -1,13 +0,0 @@
-<script>
-  import { params } from "@roxi/routify"
-  import { tables } from "stores/backend"
-
-  if ($params.selectedTable) {
-    const table = $tables.list.find(m => m._id === $params.selectedTable)
-    if (table) {
-      tables.select(table)
-    }
-  }
-</script>
-
-<slot />

packages/builder/src/pages/builder/app/[application]/data/datasource/bb_internal/table/[selectedTable]/index.svelte

@@ -1,16 +0,0 @@
-<script>
-  import TableDataTable from "components/backend/DataTable/DataTable.svelte"
-  import { tables, database } from "stores/backend"
-</script>
-
-{#if $database?._id && $tables?.selected?.name}
-  <TableDataTable />
-{:else}<i>Create your first table to start building</i>{/if}
-
-<style>
-  i {
-    font-size: var(--font-size-m);
-    color: var(--grey-5);
-    margin-top: 2px;
-  }
-</style>

packages/builder/src/pages/builder/app/[application]/data/datasource/bb_internal/table/[selectedTable]/relationship/[selectedRow]/[selectedField]/index.svelte

@@ -1,10 +0,0 @@
-<script>
-  import { params } from "@roxi/routify"
-  import RelationshipDataTable from "components/backend/DataTable/RelationshipDataTable.svelte"
-</script>
-
-<RelationshipDataTable
-  tableId={$params.selectedTable}
-  rowId={$params.selectedRow}
-  fieldName={decodeURI($params.selectedField)}
-/>

packages/builder/src/pages/builder/app/[application]/data/datasource/bb_internal/table/[selectedTable]/relationship/[selectedRow]/index.svelte

@@ -1,6 +0,0 @@
-<script>
-  import { goto } from "@roxi/routify"
-  $goto("../../")
-</script>
-
-<!-- routify:options index=false -->

packages/builder/src/pages/builder/app/[application]/data/datasource/bb_internal/table/[selectedTable]/relationship/index.svelte

@@ -1,6 +0,0 @@
-<script>
-  import { goto } from "@roxi/routify"
-  $goto("../")
-</script>
-
-<!-- routify:options index=false -->

packages/builder/src/pages/builder/app/[application]/data/datasource/bb_internal/table/_layout.svelte

@@ -1,19 +0,0 @@
-<script>
-  import { tables } from "stores/backend"
-  import { goto, leftover } from "@roxi/routify"
-  import { onMount } from "svelte"
-
-  onMount(async () => {
-    // navigate to first table in list, if not already selected
-    // and this is the final url (i.e. no selectedTable)
-    if (
-      !$leftover &&
-      $tables.list.length > 0
-      // (!$tables.selected || !$tables.selected._id)
-    ) {
-      $goto(`./${$tables.list[0]._id}`)
-    }
-  })
-</script>
-
-<slot />

packages/builder/src/pages/builder/app/[application]/data/datasource/bb_internal/table/index.svelte

@@ -1,21 +0,0 @@
-<script>
-  import { goto } from "@roxi/routify"
-  import { onMount } from "svelte"
-  import { tables } from "stores/backend"
-
-  onMount(async () => {
-    $tables.list.length > 0 && $goto(`./${$tables.list[0]._id}`)
-  })
-</script>
-
-{#if $tables.list.length === 0}
-  <i>Create your first table to start building</i>
-{:else}<i>Select a table to edit</i>{/if}
-
-<style>
-  i {
-    font-size: var(--font-size-m);
-    color: var(--grey-5);
-    margin-top: 2px;
-  }
-</style>

packages/builder/src/pages/builder/app/[application]/data/index.svelte

@@ -1,6 +1,22 @@
 <script>
   import { goto } from "@roxi/routify"
+  import { onMount } from "svelte"
+  import CreateDatasourceModal from "components/backend/DatasourceNavigator/modals/CreateDatasourceModal.svelte"
+  import { datasources } from "stores/backend"
+
+  let modal
+  $: setupComplete =
+    $datasources.list.find(x => (x._id = "bb_internal")).entities.length > 1 ||
+    $datasources.list.length > 1
+
+  onMount(() => {
+    if (!setupComplete) {
+      modal.show()
+    } else {
       $goto("./table")
+    }
+  })
 </script>
 
+<CreateDatasourceModal bind:modal />
 <!-- routify:options index=false -->

packages/builder/src/pages/builder/auth/_layout.svelte

@@ -1,17 +1,14 @@
 <script>
   import { auth, admin } from "stores/portal"
-  import { onMount } from "svelte"
   import { redirect } from "@roxi/routify"
 
   // If already authenticated, redirect away from the auth section.
   // Check this onMount rather than a reactive statement to avoid trumping
   // the login return URL functionality.
-  onMount(() => {
   if ($auth.user && !$auth.user.forceResetPassword) {
     $redirect("../")
   }
 
-    // redirect to account portal for authentication in the cloud
   if (
     !$auth.user &&
     $admin.cloud &&
@@ -21,7 +18,6 @@
   ) {
     window.location.href = $admin.accountPortalUrl
   }
-  })
 </script>
 
 {#if !$auth.user || $auth.user.forceResetPassword}

packages/builder/src/stores/backend/datasources.js

@@ -67,7 +67,7 @@ export function createDatasourcesStore() {
       })
       return json
     },
-    save: async datasource => {
+    save: async (datasource, fetchSchema = false) => {
       let response
       if (datasource._id) {
         response = await api.put(
@@ -75,7 +75,10 @@ export function createDatasourcesStore() {
           datasource
         )
       } else {
-        response = await api.post("/api/datasources", datasource)
+        response = await api.post("/api/datasources", {
+          datasource: datasource,
+          fetchSchema,
+        })
       }
 
       const json = await response.json()

packages/builder/src/stores/backend/queries.js

@@ -1,5 +1,5 @@
 import { writable, get } from "svelte/store"
-import { datasources, integrations, tables } from "./"
+import { datasources, integrations, tables, views } from "./"
 import api from "builderStore/api"
 
 export function createQueriesStore() {
@@ -55,10 +55,9 @@ export function createQueriesStore() {
     },
     select: query => {
       update(state => ({ ...state, selected: query._id }))
-      tables.update(state => ({
+      views.unselect()
-        ...state,
+      tables.unselect()
-        selected: null,
+      datasources.unselect()
-      }))
     },
     unselect: () => {
       update(state => ({ ...state, selected: null }))

packages/builder/src/stores/backend/tables.js

@@ -95,7 +95,13 @@ export function createTablesStore() {
         selected: {},
       }))
     },
-    saveField: ({ originalName, field, primaryDisplay = false, indexes }) => {
+    saveField: async ({
+      originalName,
+      field,
+      primaryDisplay = false,
+      indexes,
+    }) => {
+      let promise
       update(state => {
         // delete the original if renaming
         // need to handle if the column had no name, empty string
@@ -126,9 +132,12 @@ export function createTablesStore() {
           ...state.draft.schema,
           [field.name]: cloneDeep(field),
         }
-        save(state.draft)
+        promise = save(state.draft)
         return state
       })
+      if (promise) {
+        await promise
+      }
     },
     deleteField: field => {
       update(state => {

packages/builder/src/stores/backend/views.js

@@ -16,6 +16,7 @@ export function createViewsStore() {
         ...state,
         selected: view,
       }))
+      tables.unselect()
       queries.unselect()
       datasources.unselect()
     },

packages/builder/src/stores/portal/auth.js

@@ -80,6 +80,7 @@ export function createAuthStore() {
 
   return {
     subscribe: store.subscribe,
+    setOrganisation: setOrganisation,
     checkQueryString: async () => {
       const urlParams = new URLSearchParams(window.location.search)
       if (urlParams.has("tenantId")) {

packages/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/cli",
-  "version": "0.9.144-alpha.0",
+  "version": "0.9.146-alpha.3",
   "description": "Budibase CLI, for developers, self hosting and migrations.",
   "main": "src/index.js",
   "bin": {

packages/client/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/client",
-  "version": "0.9.144-alpha.0",
+  "version": "0.9.146-alpha.3",
   "license": "MPL-2.0",
   "module": "dist/budibase-client.js",
   "main": "dist/budibase-client.js",
@@ -19,9 +19,9 @@
     "dev:builder": "rollup -cw"
   },
   "dependencies": {
-    "@budibase/bbui": "^0.9.144-alpha.0",
+    "@budibase/bbui": "^0.9.146-alpha.3",
     "@budibase/standard-components": "^0.9.139",
-    "@budibase/string-templates": "^0.9.144-alpha.0",
+    "@budibase/string-templates": "^0.9.146-alpha.3",
     "regexparam": "^1.3.0",
     "shortid": "^2.2.15",
     "svelte-spa-router": "^3.0.5"

packages/client/rollup.config.js

@@ -58,6 +58,10 @@ export default {
           find: "sdk",
           replacement: path.resolve("./src/sdk"),
         },
+        {
+          find: "builder",
+          replacement: path.resolve("../builder"),
+        },
       ],
     }),
     svelte({

packages/client/src/components/ClientApp.svelte

@@ -23,7 +23,7 @@
   import SelectionIndicator from "components/preview/SelectionIndicator.svelte"
   import HoverIndicator from "components/preview/HoverIndicator.svelte"
   import CustomThemeWrapper from "./CustomThemeWrapper.svelte"
-  import ErrorSVG from "../../../builder/assets/error.svg"
+  import ErrorSVG from "builder/assets/error.svg"
 
   // Provide contexts
   setContext("sdk", SDK)

packages/client/src/components/app/DataProvider.svelte

@@ -6,7 +6,7 @@
     luceneQuery,
     luceneSort,
     luceneLimit,
-  } from "utils/lucene"
+  } from "builder/src/helpers/lucene"
   import Placeholder from "./Placeholder.svelte"
 
   export let dataSource

packages/client/src/stores/state.js

@@ -1,5 +1,5 @@
 import { writable, get, derived } from "svelte/store"
-import { localStorageStore } from "../../../builder/src/builderStore/store/localStorage"
+import { localStorageStore } from "builder/src/builderStore/store/localStorage"
 import { appStore } from "./app"
 
 const createStateStore = () => {

packages/client/src/utils/conditions.js

@@ -1,4 +1,4 @@
-import { buildLuceneQuery, luceneQuery } from "./lucene"
+import { buildLuceneQuery, luceneQuery } from "builder/src/helpers/lucene"
 
 export const getActiveConditions = conditions => {
   if (!conditions?.length) {

packages/client/src/utils/lucene.js

@@ -1,179 +0,0 @@
-/**
- * Builds a lucene JSON query from the filter structure generated in the builder
- * @param filter the builder filter structure
- */
-export const buildLuceneQuery = filter => {
-  let query = {
-    string: {},
-    fuzzy: {},
-    range: {},
-    equal: {},
-    notEqual: {},
-    empty: {},
-    notEmpty: {},
-    contains: {},
-    notContains: {},
-  }
-  if (Array.isArray(filter)) {
-    filter.forEach(expression => {
-      let { operator, field, type, value } = expression
-      // Parse all values into correct types
-      if (type === "datetime" && value) {
-        value = new Date(value).toISOString()
-      }
-      if (type === "number") {
-        value = parseFloat(value)
-      }
-      if (type === "boolean") {
-        value = `${value}`?.toLowerCase() === "true"
-      }
-      if (operator.startsWith("range")) {
-        if (!query.range[field]) {
-          query.range[field] = {
-            low:
-              type === "number"
-                ? Number.MIN_SAFE_INTEGER
-                : "0000-00-00T00:00:00.000Z",
-            high:
-              type === "number"
-                ? Number.MAX_SAFE_INTEGER
-                : "9999-00-00T00:00:00.000Z",
-          }
-        }
-        if (operator === "rangeLow" && value != null && value !== "") {
-          query.range[field].low = value
-        } else if (operator === "rangeHigh" && value != null && value !== "") {
-          query.range[field].high = value
-        }
-      } else if (query[operator]) {
-        if (type === "boolean") {
-          // Transform boolean filters to cope with null.
-          // "equals false" needs to be "not equals true"
-          // "not equals false" needs to be "equals true"
-          if (operator === "equal" && value === false) {
-            query.notEqual[field] = true
-          } else if (operator === "notEqual" && value === false) {
-            query.equal[field] = true
-          } else {
-            query[operator][field] = value
-          }
-        } else {
-          query[operator][field] = value
-        }
-      }
-    })
-  }
-
-  return query
-}
-
-/**
- * Performs a client-side lucene search on an array of data
- * @param docs the data
- * @param query the JSON lucene query
- */
-export const luceneQuery = (docs, query) => {
-  if (!query) {
-    return docs
-  }
-
-  // Iterates over a set of filters and evaluates a fail function against a doc
-  const match = (type, failFn) => doc => {
-    const filters = Object.entries(query[type] || {})
-    for (let i = 0; i < filters.length; i++) {
-      if (failFn(filters[i][0], filters[i][1], doc)) {
-        return false
-      }
-    }
-    return true
-  }
-
-  // Process a string match (fails if the value does not start with the string)
-  const stringMatch = match("string", (key, value, doc) => {
-    return !doc[key] || !doc[key].startsWith(value)
-  })
-
-  // Process a fuzzy match (treat the same as starts with when running locally)
-  const fuzzyMatch = match("fuzzy", (key, value, doc) => {
-    return !doc[key] || !doc[key].startsWith(value)
-  })
-
-  // Process a range match
-  const rangeMatch = match("range", (key, value, doc) => {
-    return !doc[key] || doc[key] < value.low || doc[key] > value.high
-  })
-
-  // Process an equal match (fails if the value is different)
-  const equalMatch = match("equal", (key, value, doc) => {
-    return value != null && value !== "" && doc[key] !== value
-  })
-
-  // Process a not-equal match (fails if the value is the same)
-  const notEqualMatch = match("notEqual", (key, value, doc) => {
-    return value != null && value !== "" && doc[key] === value
-  })
-
-  // Process an empty match (fails if the value is not empty)
-  const emptyMatch = match("empty", (key, value, doc) => {
-    return doc[key] != null && doc[key] !== ""
-  })
-
-  // Process a not-empty match (fails is the value is empty)
-  const notEmptyMatch = match("notEmpty", (key, value, doc) => {
-    return doc[key] == null || doc[key] === ""
-  })
-
-  // Match a document against all criteria
-  const docMatch = doc => {
-    return (
-      stringMatch(doc) &&
-      fuzzyMatch(doc) &&
-      rangeMatch(doc) &&
-      equalMatch(doc) &&
-      notEqualMatch(doc) &&
-      emptyMatch(doc) &&
-      notEmptyMatch(doc)
-    )
-  }
-
-  // Process all docs
-  return docs.filter(docMatch)
-}
-
-/**
- * Performs a client-side sort from the equivalent server-side lucene sort
- * parameters.
- * @param docs the data
- * @param sort the sort column
- * @param sortOrder the sort order ("ascending" or "descending")
- * @param sortType the type of sort ("string" or "number")
- */
-export const luceneSort = (docs, sort, sortOrder, sortType = "string") => {
-  if (!sort || !sortOrder || !sortType) {
-    return docs
-  }
-  const parse = sortType === "string" ? x => `${x}` : x => parseFloat(x)
-  return docs.slice().sort((a, b) => {
-    const colA = parse(a[sort])
-    const colB = parse(b[sort])
-    if (sortOrder === "Descending") {
-      return colA > colB ? -1 : 1
-    } else {
-      return colA > colB ? 1 : -1
-    }
-  })
-}
-
-/**
- * Limits the specified docs to the specified number of rows from the equivalent
- * server-side lucene limit parameters.
- * @param docs the data
- * @param limit the number of docs to limit to
- */
-export const luceneLimit = (docs, limit) => {
-  const numLimit = parseFloat(limit)
-  if (isNaN(numLimit)) {
-    return docs
-  }
-  return docs.slice(0, numLimit)
-}

packages/server/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "0.9.144-alpha.0",
+  "version": "0.9.146-alpha.3",
   "description": "Budibase Web Server",
   "main": "src/index.js",
   "repository": {
@@ -64,9 +64,9 @@
   "author": "Budibase",
   "license": "AGPL-3.0-or-later",
   "dependencies": {
-    "@budibase/auth": "^0.9.144-alpha.0",
+    "@budibase/auth": "^0.9.146-alpha.3",
-    "@budibase/client": "^0.9.144-alpha.0",
+    "@budibase/client": "^0.9.146-alpha.3",
-    "@budibase/string-templates": "^0.9.144-alpha.0",
+    "@budibase/string-templates": "^0.9.146-alpha.3",
     "@elastic/elasticsearch": "7.10.0",
     "@koa/router": "8.0.0",
     "@sendgrid/mail": "7.1.1",

packages/server/src/api/controllers/datasource.js

@@ -41,15 +41,10 @@ exports.fetch = async function (ctx) {
 
 exports.buildSchemaFromDb = async function (ctx) {
   const db = new CouchDB(ctx.appId)
-  const datasourceId = ctx.params.datasourceId
+  const datasource = await db.get(ctx.params.datasourceId)
-  const datasource = await db.get(datasourceId)
 
-  const Connector = integrations[datasource.source]
+  const tables = await buildSchemaHelper(datasource)
-
+  datasource.entities = tables
-  // Connect to the DB and build the schema
-  const connector = new Connector(datasource.config)
-  await connector.buildSchema(datasource._id, datasource.entities)
-  datasource.entities = connector.tables
 
   const response = await db.put(datasource)
   datasource._rev = response.rev
@@ -81,12 +76,18 @@ exports.update = async function (ctx) {
 
 exports.save = async function (ctx) {
   const db = new CouchDB(ctx.appId)
-  const plus = ctx.request.body.plus
+  const plus = ctx.request.body.datasource.plus
+  const fetchSchema = ctx.request.body.fetchSchema
 
   const datasource = {
     _id: generateDatasourceID({ plus }),
     type: plus ? DocumentTypes.DATASOURCE_PLUS : DocumentTypes.DATASOURCE,
-    ...ctx.request.body,
+    ...ctx.request.body.datasource,
+  }
+
+  if (fetchSchema) {
+    let tables = await buildSchemaHelper(datasource)
+    datasource.entities = tables
   }
 
   const response = await db.put(datasource)
@@ -133,3 +134,14 @@ exports.query = async function (ctx) {
     ctx.throw(400, err)
   }
 }
+
+const buildSchemaHelper = async datasource => {
+  const Connector = integrations[datasource.source]
+
+  // Connect to the DB and build the schema
+  const connector = new Connector(datasource.config)
+  await connector.buildSchema(datasource._id, datasource.entities)
+  datasource.entities = connector.tables
+
+  return connector.tables
+}

packages/server/src/api/controllers/row/internal.js

@@ -346,6 +346,11 @@ exports.bulkDestroy = async ctx => {
 }
 
 exports.search = async ctx => {
+  // Fetch the whole table when running in cypress, as search doesn't work
+  if (env.isCypress()) {
+    return { rows: await exports.fetch(ctx) }
+  }
+
   const appId = ctx.appId
   const { tableId } = ctx.params
   const db = new CouchDB(appId)

packages/server/src/api/routes/tests/query.spec.js

@@ -1,6 +1,7 @@
 // mock out postgres for this
 jest.mock("pg")
 
+const { findLastKey } = require("lodash/fp")
 const setup = require("./utilities")
 const { checkBuilderEndpoint } = require("./utilities/TestFunctions")
 const { basicQuery, basicDatasource } = setup.structures
@@ -19,10 +20,10 @@ describe("/queries", () => {
   })
 
   async function createInvalidIntegration() {
-    const datasource = await config.createDatasource({
+    const datasource = await config.createDatasource({datasource: {
-      ...basicDatasource(),
+      ...basicDatasource().datasource,
       source: "INVALID_INTEGRATION",
-    })
+    }})
     const query = await config.createQuery()
     return { datasource, query }
   }
@@ -183,11 +184,14 @@ describe("/queries", () => {
     })
 
     it("should fail with invalid integration type", async () => {
-      const { query } = await createInvalidIntegration()
+      const { query, datasource } = await createInvalidIntegration()
       await request
         .post(`/api/queries/${query._id}`)
         .send({
+          datasourceId: datasource._id,
           parameters: {},
+          fields: {},
+          queryVerb: "read",
         })
         .set(config.defaultHeaders())
         .expect(400)

packages/server/src/environment.js

@@ -13,6 +13,10 @@ function isDev() {
   )
 }
 
+function isCypress() {
+  return process.env.NODE_ENV === "cypress"
+}
+
 let LOADED = false
 if (!LOADED && isDev() && !isTest()) {
   require("dotenv").config()
@@ -62,6 +66,7 @@ module.exports = {
     module.exports[key] = value
   },
   isTest,
+  isCypress,
   isDev,
   isProd: () => {
     return !isDev()

packages/server/src/middleware/tests/usageQuota.spec.js

@@ -6,6 +6,9 @@ jest.mock("../../environment", () => ({
   isDev: () => true,
   _set: () => {},
 }))
+jest.mock("@budibase/auth/tenancy", () => ({
+  getTenantId: () => "testing123"
+}))
 
 const usageQuotaMiddleware = require("../usageQuota")
 const usageQuota = require("../../utilities/usageQuota")

packages/server/src/middleware/usageQuota.js

@@ -1,6 +1,10 @@
 const CouchDB = require("../db")
 const usageQuota = require("../utilities/usageQuota")
 const env = require("../environment")
+const { getTenantId } = require("@budibase/auth/tenancy")
+
+// tenants without limits
+const EXCLUDED_TENANTS = ["bb", "default", "bbtest", "bbstaging"]
 
 // currently only counting new writes and deletes
 const METHOD_MAP = {
@@ -28,8 +32,10 @@ function getProperty(url) {
 }
 
 module.exports = async (ctx, next) => {
+  const tenantId = getTenantId()
+
   // if in development or a self hosted cloud usage quotas should not be executed
-  if (env.isDev() || env.SELF_HOSTED) {
+  if (env.isDev() || env.SELF_HOSTED || EXCLUDED_TENANTS.includes(tenantId)) {
     return next()
   }
 

packages/server/src/tests/utilities/structures.js

@@ -70,10 +70,12 @@ exports.basicRole = () => {
 
 exports.basicDatasource = () => {
   return {
+    datasource: {
       type: "datasource",
       name: "Test",
       source: "POSTGRES",
       config: {},
+    },
   }
 }
 

packages/server/yarn.lock

@@ -943,6 +943,29 @@
   resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
   integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==
 
+"@budibase/auth@^0.9.146-alpha.3":
+  version "0.9.146"
+  resolved "https://registry.yarnpkg.com/@budibase/auth/-/auth-0.9.146.tgz#920fe02a78ca17903b72ccde307ca3e82b4176ad"
+  integrity sha512-T7DhI3WIolD0CjO2pRCEZfJBpJce4cmZWTFRIZ8lBnKe/6dxkK9fNrkZDYRhRkMwQbDQXoARADZM1hAfgUsSMg==
+  dependencies:
+    "@techpass/passport-openidconnect" "^0.3.0"
+    aws-sdk "^2.901.0"
+    bcryptjs "^2.4.3"
+    cls-hooked "^4.2.2"
+    ioredis "^4.27.1"
+    jsonwebtoken "^8.5.1"
+    koa-passport "^4.1.4"
+    lodash "^4.17.21"
+    node-fetch "^2.6.1"
+    passport-google-auth "^1.0.2"
+    passport-google-oauth "^2.0.0"
+    passport-jwt "^4.0.0"
+    passport-local "^1.0.0"
+    sanitize-s3-objectkey "^0.0.1"
+    tar-fs "^2.1.1"
+    uuid "^8.3.2"
+    zlib "^1.0.5"
+
 "@budibase/bbui@^0.9.139":
   version "0.9.139"
   resolved "https://registry.yarnpkg.com/@budibase/bbui/-/bbui-0.9.139.tgz#e6cfc90e8f6c2aa3526fc6a7bef251bccdaf51bb"
@@ -992,10 +1015,10 @@
     svelte-flatpickr "^3.1.0"
     svelte-portal "^1.0.0"
 
-"@budibase/bbui@^0.9.142":
+"@budibase/bbui@^0.9.146":
-  version "0.9.142"
+  version "0.9.146"
-  resolved "https://registry.yarnpkg.com/@budibase/bbui/-/bbui-0.9.142.tgz#7edbda7967c9e5dfc96e5be5231656e5aab8d0e3"
+  resolved "https://registry.yarnpkg.com/@budibase/bbui/-/bbui-0.9.146.tgz#7689b2c0f148321e62969181e3f6549f03dd3e78"
-  integrity sha512-m2YlqqH87T4RwqD/oGhH6twHIgvFv4oUMEhKpkgLsbxjXVLVD0OOF7WqjpDnSa4khVQaixjdkI/Jiw2qhBUSaA==
+  integrity sha512-Mq0oMyaN18Dg5e0IPtPXSGmu/TS4B74gW+l2ypJDNTzSRm934DOAPghDgkb53rFNZhsovCYjixJZmesUcv2o3g==
   dependencies:
     "@adobe/spectrum-css-workflow-icons" "^1.2.1"
     "@spectrum-css/actionbutton" "^1.0.1"
@@ -1041,14 +1064,14 @@
     svelte-flatpickr "^3.1.0"
     svelte-portal "^1.0.0"
 
-"@budibase/client@^0.9.140-alpha.11":
+"@budibase/client@^0.9.146-alpha.3":
-  version "0.9.142"
+  version "0.9.146"
-  resolved "https://registry.yarnpkg.com/@budibase/client/-/client-0.9.142.tgz#bfc3374b1414651cab799305def3e03354f070d0"
+  resolved "https://registry.yarnpkg.com/@budibase/client/-/client-0.9.146.tgz#d3b1bbd67245ab5a3870ccb580b9fc76f0344fd6"
-  integrity sha512-/iT0pAanwljWo9R5fD6YkkUC8OePGUiMwNzaxLlXkda+qrop3SZFGcbPlNGYA81jUQq1O4L39RON7wqLSiJ2oQ==
+  integrity sha512-vd/bMmiQVghFH3Pa9jrGXjYAAKo+lGrwWyfUSdXAb4XP6gCSnMK5BXf8NliNrQzQVmruYT+2rGMsnc+9q4lW1g==
   dependencies:
-    "@budibase/bbui" "^0.9.142"
+    "@budibase/bbui" "^0.9.146"
     "@budibase/standard-components" "^0.9.139"
-    "@budibase/string-templates" "^0.9.142"
+    "@budibase/string-templates" "^0.9.146"
     regexparam "^1.3.0"
     shortid "^2.2.15"
     svelte-spa-router "^3.0.5"
@@ -1099,10 +1122,10 @@
     svelte-apexcharts "^1.0.2"
     svelte-flatpickr "^3.1.0"
 
-"@budibase/string-templates@^0.9.140-alpha.11", "@budibase/string-templates@^0.9.142":
+"@budibase/string-templates@^0.9.146", "@budibase/string-templates@^0.9.146-alpha.3":
-  version "0.9.142"
+  version "0.9.146"
-  resolved "https://registry.yarnpkg.com/@budibase/string-templates/-/string-templates-0.9.142.tgz#2c026580edcc7e32c017b537921d06a1ad1148f5"
+  resolved "https://registry.yarnpkg.com/@budibase/string-templates/-/string-templates-0.9.146.tgz#85249c7a8777a5f0c280af6f6d0e3d3ff0bf20b5"
-  integrity sha512-7nGzVaOtOugwC/8NMoQ90B/L6Q/IzFub5Xws4r7tAeqZ5WUb1ITVDrNJ+w/PhzeVT6IdkXEUyMdkKBnJTmCD8g==
+  integrity sha512-4f91SVUaTKseB+j7ycWbP54XiqiFZ6bZvcKgzsg1mLF+VVJ1/ALUsLvCRaj6SlcSHrhhALiGVR1z18KOyBWoKw==
   dependencies:
     "@budibase/handlebars-helpers" "^0.11.4"
     dayjs "^1.10.4"
@@ -2123,6 +2146,17 @@
   dependencies:
     defer-to-connect "^1.0.1"
 
+"@techpass/passport-openidconnect@^0.3.0":
+  version "0.3.2"
+  resolved "https://registry.yarnpkg.com/@techpass/passport-openidconnect/-/passport-openidconnect-0.3.2.tgz#f8fd5d97256286665dbf26dac92431f977ab1e63"
+  integrity sha512-fnCtEiexXSHA029B//hJcCJlLJrT3lhpNCyA0rnz58Qttz0BLGCVv6yMT8HmOnGThH6vcDOVwdgKM3kbCQtEhw==
+  dependencies:
+    base64url "^3.0.1"
+    oauth "^0.9.15"
+    passport-strategy "^1.0.0"
+    request "^2.88.0"
+    webfinger "^0.4.2"
+
 "@tootallnate/once@1":
   version "1.1.2"
   resolved "https://registry.yarnpkg.com/@tootallnate/once/-/once-1.1.2.tgz#ccb91445360179a04e7fe6aff78c00ffc1eeaf82"
@@ -2867,6 +2901,13 @@ astral-regex@^1.0.0:
   resolved "https://registry.yarnpkg.com/astral-regex/-/astral-regex-1.0.0.tgz#6c8c3fb827dd43ee3918f27b82782ab7658a6fd9"
   integrity sha512-+Ryf6g3BKoRc7jfp7ad8tM4TtMiaWvbF/1/sQcZPkkS7ag3D5nMBCe2UfOTONtAkaG0tO0ij3C5Lwmf1EiyjHg==
 
+async-hook-jl@^1.7.6:
+  version "1.7.6"
+  resolved "https://registry.yarnpkg.com/async-hook-jl/-/async-hook-jl-1.7.6.tgz#4fd25c2f864dbaf279c610d73bf97b1b28595e68"
+  integrity sha512-gFaHkFfSxTjvoxDMYqDuGHlcRyUuamF8s+ZTtJdDzqjws4mCt7v0vuV79/E2Wr2/riMQgtG4/yUtXWs1gZ7JMg==
+  dependencies:
+    stack-chain "^1.3.7"
+
 async-limiter@~1.0.0:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/async-limiter/-/async-limiter-1.0.1.tgz#dd379e94f0db8310b08291f9d64c3209766617fd"
@@ -2884,6 +2925,13 @@ async@^2.6.3:
   dependencies:
     lodash "^4.17.14"
 
+async@~2.1.4:
+  version "2.1.5"
+  resolved "https://registry.yarnpkg.com/async/-/async-2.1.5.tgz#e587c68580994ac67fc56ff86d3ac56bdbe810bc"
+  integrity sha1-5YfGhYCZSsZ/xW/4bTrFa9voELw=
+  dependencies:
+    lodash "^4.14.0"
+
 asynckit@^0.4.0:
   version "0.4.0"
   resolved "https://registry.yarnpkg.com/asynckit/-/asynckit-0.4.0.tgz#c79ed97f7f34cb8f2ba1bc9790bcc366474b4b79"
@@ -2921,6 +2969,21 @@ aws-sdk@^2.767.0:
     uuid "3.3.2"
     xml2js "0.4.19"
 
+aws-sdk@^2.901.0:
+  version "2.997.0"
+  resolved "https://registry.yarnpkg.com/aws-sdk/-/aws-sdk-2.997.0.tgz#8598a5dd7bc6b6833a2fc3d737fba89020a79418"
+  integrity sha512-PiuDmC5hN+FsyLvl7GsZAnS6hQpo1pP+Ax2u8gyL19QlbBLwlhsFQF29vPcYatyv6WUxr51o6uymJdPxQg6uEA==
+  dependencies:
+    buffer "4.9.2"
+    events "1.1.1"
+    ieee754 "1.1.13"
+    jmespath "0.15.0"
+    querystring "0.2.0"
+    sax "1.2.1"
+    url "0.10.3"
+    uuid "3.3.2"
+    xml2js "0.4.19"
+
 aws-sign2@~0.7.0:
   version "0.7.0"
   resolved "https://registry.yarnpkg.com/aws-sign2/-/aws-sign2-0.7.0.tgz#b46e890934a9591f2d2f6f86d7e6a9f1b3fe76a8"
@@ -3088,6 +3151,11 @@ base64-js@^1.0.2, base64-js@^1.3.1:
   resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.5.1.tgz#1b1b440160a5bf7ad40b650f095963481903930a"
   integrity sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==
 
+base64url@3.x.x, base64url@^3.0.1:
+  version "3.0.1"
+  resolved "https://registry.yarnpkg.com/base64url/-/base64url-3.0.1.tgz#6399d572e2bc3f90a9a8b22d5dbb0a32d33f788d"
+  integrity sha512-ir1UPr3dkwexU7FdV8qBBbNDRUhMmIekYMFZfi+C/sLNnRESKPl23nB9b2pltqfOQNnGzsDdId90AEtG5tCx4A==
+
 base@^0.11.1:
   version "0.11.2"
   resolved "https://registry.yarnpkg.com/base/-/base-0.11.2.tgz#7bde5ced145b6d551a90db87f83c558b4eb48a8f"
@@ -3108,7 +3176,7 @@ bcrypt-pbkdf@^1.0.0:
   dependencies:
     tweetnacl "^0.14.3"
 
-bcryptjs@2.4.3:
+bcryptjs@2.4.3, bcryptjs@^2.4.3:
   version "2.4.3"
   resolved "https://registry.yarnpkg.com/bcryptjs/-/bcryptjs-2.4.3.tgz#9ab5627b93e60621ff7cdac5da9733027df1d0cb"
   integrity sha1-mrVie5PmBiH/fNrF2pczAn3x0Ms=
@@ -3153,6 +3221,15 @@ bl@^3.0.0:
   dependencies:
     readable-stream "^3.0.1"
 
+bl@^4.0.3:
+  version "4.1.0"
+  resolved "https://registry.yarnpkg.com/bl/-/bl-4.1.0.tgz#451535264182bec2fbbc83a62ab98cf11d9f7b3a"
+  integrity sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==
+  dependencies:
+    buffer "^5.5.0"
+    inherits "^2.0.4"
+    readable-stream "^3.4.0"
+
 bluebird@^3.5.1:
   version "3.7.2"
   resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.7.2.tgz#9f229c15be272454ffa973ace0dbee79a1b0c36f"
@@ -3517,6 +3594,11 @@ chokidar@^3.2.2:
   optionalDependencies:
     fsevents "~2.3.2"
 
+chownr@^1.1.1:
+  version "1.1.4"
+  resolved "https://registry.yarnpkg.com/chownr/-/chownr-1.1.4.tgz#6fc9d7b42d32a583596337666e7d08084da2cc6b"
+  integrity sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==
+
 ci-info@^2.0.0:
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/ci-info/-/ci-info-2.0.0.tgz#67a9e964be31a51e15e5010d58e6f12834002f46"
@@ -3589,6 +3671,15 @@ clone-response@1.0.2, clone-response@^1.0.2:
   dependencies:
     mimic-response "^1.0.0"
 
+cls-hooked@^4.2.2:
+  version "4.2.2"
+  resolved "https://registry.yarnpkg.com/cls-hooked/-/cls-hooked-4.2.2.tgz#ad2e9a4092680cdaffeb2d3551da0e225eae1908"
+  integrity sha512-J4Xj5f5wq/4jAvcdgoGsL3G103BtWpZrMo8NEinRltN+xpTZdI+M38pyQqhuFU/P792xkMFvnKSf+Lm81U1bxw==
+  dependencies:
+    async-hook-jl "^1.7.6"
+    emitter-listener "^1.0.1"
+    semver "^5.4.1"
+
 cluster-key-slot@^1.1.0:
   version "1.1.0"
   resolved "https://registry.yarnpkg.com/cluster-key-slot/-/cluster-key-slot-1.1.0.tgz#30474b2a981fb12172695833052bc0d01336d10d"
@@ -4319,6 +4410,13 @@ electron-to-chromium@^1.3.811:
   resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.3.827.tgz#c725e8db8c5be18b472a919e5f57904512df0fc1"
   integrity sha512-ye+4uQOY/jbjRutMcE/EmOcNwUeo1qo9aKL2tPyb09cU3lmxNeyDF4RWiemmkknW+p29h7dyDqy02higTxc9/A==
 
+emitter-listener@^1.0.1:
+  version "1.1.2"
+  resolved "https://registry.yarnpkg.com/emitter-listener/-/emitter-listener-1.1.2.tgz#56b140e8f6992375b3d7cb2cab1cc7432d9632e8"
+  integrity sha512-Bt1sBAGFHY9DKY+4/2cV6izcKJUf5T7/gkdmkxzX/qv9CcGH8xSwVRW5mtX03SWJtRTWSOpzCuWN9rBFYZepZQ==
+  dependencies:
+    shimmer "^1.2.0"
+
 emittery@^0.8.1:
   version "0.8.1"
   resolved "https://registry.yarnpkg.com/emittery/-/emittery-0.8.1.tgz#bb23cc86d03b30aa75a7f734819dee2e1ba70860"
@@ -4349,7 +4447,7 @@ encoding-down@^6.3.0:
     level-codec "^9.0.0"
     level-errors "^2.0.0"
 
-end-of-stream@^1.0.0, end-of-stream@^1.1.0:
+end-of-stream@^1.0.0, end-of-stream@^1.1.0, end-of-stream@^1.4.1:
   version "1.4.4"
   resolved "https://registry.yarnpkg.com/end-of-stream/-/end-of-stream-1.4.4.tgz#5ae64a5f45057baf3626ec14da0ca5e4b2431eb0"
   integrity sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==
@@ -5405,6 +5503,32 @@ globby@^11.0.3:
     merge2 "^1.3.0"
     slash "^3.0.0"
 
+google-auth-library@~0.10.0:
+  version "0.10.0"
+  resolved "https://registry.yarnpkg.com/google-auth-library/-/google-auth-library-0.10.0.tgz#6e15babee85fd1dd14d8d128a295b6838d52136e"
+  integrity sha1-bhW6vuhf0d0U2NEoopW2g41SE24=
+  dependencies:
+    gtoken "^1.2.1"
+    jws "^3.1.4"
+    lodash.noop "^3.0.1"
+    request "^2.74.0"
+
+google-p12-pem@^0.1.0:
+  version "0.1.2"
+  resolved "https://registry.yarnpkg.com/google-p12-pem/-/google-p12-pem-0.1.2.tgz#33c46ab021aa734fa0332b3960a9a3ffcb2f3177"
+  integrity sha1-M8RqsCGqc0+gMys5YKmj/8svMXc=
+  dependencies:
+    node-forge "^0.7.1"
+
+googleapis@^16.0.0:
+  version "16.1.0"
+  resolved "https://registry.yarnpkg.com/googleapis/-/googleapis-16.1.0.tgz#0f19f2d70572d918881a0f626e3b1a2fa8629576"
+  integrity sha1-Dxny1wVy2RiIGg9ibjsaL6hilXY=
+  dependencies:
+    async "~2.1.4"
+    google-auth-library "~0.10.0"
+    string-template "~1.0.0"
+
 got@^8.3.1:
   version "8.3.2"
   resolved "https://registry.yarnpkg.com/got/-/got-8.3.2.tgz#1d23f64390e97f776cac52e5b936e5f514d2e937"
@@ -5450,6 +5574,16 @@ graceful-fs@^4.1.10, graceful-fs@^4.1.11, graceful-fs@^4.1.15, graceful-fs@^4.1.
   resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.8.tgz#e412b8d33f5e006593cbd3cee6df9f2cebbe802a"
   integrity sha512-qkIilPUYcNhJpd33n0GBXTB1MMPp14TxEsEs0pTrsSVucApsYzW5V+Q8Qxhik6KU3evy+qkAAowTByymK0avdg==
 
+gtoken@^1.2.1:
+  version "1.2.3"
+  resolved "https://registry.yarnpkg.com/gtoken/-/gtoken-1.2.3.tgz#5509571b8afd4322e124cf66cf68115284c476d8"
+  integrity sha512-wQAJflfoqSgMWrSBk9Fg86q+sd6s7y6uJhIvvIPz++RElGlMtEqsdAR2oWwZ/WTEtp7P9xFbJRrT976oRgzJ/w==
+  dependencies:
+    google-p12-pem "^0.1.0"
+    jws "^3.0.0"
+    mime "^1.4.1"
+    request "^2.72.0"
+
 gulp-header@^1.7.1:
   version "1.8.12"
   resolved "https://registry.yarnpkg.com/gulp-header/-/gulp-header-1.8.12.tgz#ad306be0066599127281c4f8786660e705080a84"
@@ -5925,7 +6059,7 @@ invert-kv@^2.0.0:
   resolved "https://registry.yarnpkg.com/invert-kv/-/invert-kv-2.0.0.tgz#7393f5afa59ec9ff5f67a27620d11c226e3eec02"
   integrity sha512-wPVv/y/QQ/Uiirj/vh3oP+1Ww+AWehmi1g5fFWGPF6IpCBCDVrhgHRMvrLfdYcwDh3QJbGXDW4JAuzxElLSqKA==
 
-ioredis@^4.27.0:
+ioredis@^4.27.0, ioredis@^4.27.1:
   version "4.27.9"
   resolved "https://registry.yarnpkg.com/ioredis/-/ioredis-4.27.9.tgz#c27bbade9724f0b8f84c279fb1d567be785ba33d"
   integrity sha512-hAwrx9F+OQ0uIvaJefuS3UTqW+ByOLyLIV+j0EH8ClNVxvFyH9Vmb08hCL4yje6mDYT5zMquShhypkd50RRzkg==
@@ -7322,6 +7456,22 @@ jsonschema@1.4.0:
   resolved "https://registry.yarnpkg.com/jsonschema/-/jsonschema-1.4.0.tgz#1afa34c4bc22190d8e42271ec17ac8b3404f87b2"
   integrity sha512-/YgW6pRMr6M7C+4o8kS+B/2myEpHCrxO4PEWnqJNBFMjn7EWXqlQ4tGwL6xTHeRplwuZmcAncdvfOad1nT2yMw==
 
+jsonwebtoken@^8.2.0, jsonwebtoken@^8.5.1:
+  version "8.5.1"
+  resolved "https://registry.yarnpkg.com/jsonwebtoken/-/jsonwebtoken-8.5.1.tgz#00e71e0b8df54c2121a1f26137df2280673bcc0d"
+  integrity sha512-XjwVfRS6jTMsqYs0EsuJ4LGxXV14zQybNd4L2r0UvbVnSF9Af8x7p5MzbJ90Ioz/9TI41/hTCvznF/loiSzn8w==
+  dependencies:
+    jws "^3.2.2"
+    lodash.includes "^4.3.0"
+    lodash.isboolean "^3.0.3"
+    lodash.isinteger "^4.0.4"
+    lodash.isnumber "^3.0.3"
+    lodash.isplainobject "^4.0.6"
+    lodash.isstring "^4.0.1"
+    lodash.once "^4.0.0"
+    ms "^2.1.1"
+    semver "^5.6.0"
+
 jsprim@^1.2.2:
   version "1.4.1"
   resolved "https://registry.yarnpkg.com/jsprim/-/jsprim-1.4.1.tgz#313e66bc1e5cc06e438bc1b7499c2e5c56acb6a2"
@@ -7361,7 +7511,7 @@ jwa@^1.4.1:
     ecdsa-sig-formatter "1.0.11"
     safe-buffer "^5.0.1"
 
-jws@3.x.x:
+jws@3.x.x, jws@^3.0.0, jws@^3.1.4, jws@^3.2.2:
   version "3.2.2"
   resolved "https://registry.yarnpkg.com/jws/-/jws-3.2.2.tgz#001099f3639468c9414000e99995fa52fb478304"
   integrity sha512-YHlZCB6lMTllWDtSPHz/ZXTsi8S00usEV6v1tjq8tOUZzw7DpSDWVXjXDre6ed1w/pd495ODpHZYSdkRTsa0HA==
@@ -7486,6 +7636,13 @@ koa-is-json@^1.0.0:
   resolved "https://registry.yarnpkg.com/koa-is-json/-/koa-is-json-1.0.0.tgz#273c07edcdcb8df6a2c1ab7d59ee76491451ec14"
   integrity sha1-JzwH7c3Ljfaiwat9We52SRRR7BQ=
 
+koa-passport@^4.1.4:
+  version "4.1.4"
+  resolved "https://registry.yarnpkg.com/koa-passport/-/koa-passport-4.1.4.tgz#5f1665c1c2a37ace79af9f970b770885ca30ccfa"
+  integrity sha512-dJBCkl4X+zdYxbI2V2OtoGy0PUenpvp2ZLLWObc8UJhsId0iQpTFT8RVcuA0709AL2txGwRHnSPoT1bYNGa6Kg==
+  dependencies:
+    passport "^0.4.0"
+
 koa-pino-logger@3.0.0:
   version "3.0.0"
   resolved "https://registry.yarnpkg.com/koa-pino-logger/-/koa-pino-logger-3.0.0.tgz#27600b4f3639e8767dfc6b66493109c5457f53ba"
@@ -7787,16 +7944,46 @@ lodash.flatten@^4.4.0:
   resolved "https://registry.yarnpkg.com/lodash.flatten/-/lodash.flatten-4.4.0.tgz#f31c22225a9632d2bbf8e4addbef240aa765a61f"
   integrity sha1-8xwiIlqWMtK7+OSt2+8kCqdlph8=
 
+lodash.includes@^4.3.0:
+  version "4.3.0"
+  resolved "https://registry.yarnpkg.com/lodash.includes/-/lodash.includes-4.3.0.tgz#60bb98a87cb923c68ca1e51325483314849f553f"
+  integrity sha1-YLuYqHy5I8aMoeUTJUgzFISfVT8=
+
 lodash.isarguments@^3.1.0:
   version "3.1.0"
   resolved "https://registry.yarnpkg.com/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz#2f573d85c6a24289ff00663b491c1d338ff3458a"
   integrity sha1-L1c9hcaiQon/AGY7SRwdM4/zRYo=
 
+lodash.isboolean@^3.0.3:
+  version "3.0.3"
+  resolved "https://registry.yarnpkg.com/lodash.isboolean/-/lodash.isboolean-3.0.3.tgz#6c2e171db2a257cd96802fd43b01b20d5f5870f6"
+  integrity sha1-bC4XHbKiV82WgC/UOwGyDV9YcPY=
+
 lodash.isequal@^4.5.0:
   version "4.5.0"
   resolved "https://registry.yarnpkg.com/lodash.isequal/-/lodash.isequal-4.5.0.tgz#415c4478f2bcc30120c22ce10ed3226f7d3e18e0"
   integrity sha1-QVxEePK8wwEgwizhDtMib30+GOA=
 
+lodash.isinteger@^4.0.4:
+  version "4.0.4"
+  resolved "https://registry.yarnpkg.com/lodash.isinteger/-/lodash.isinteger-4.0.4.tgz#619c0af3d03f8b04c31f5882840b77b11cd68343"
+  integrity sha1-YZwK89A/iwTDH1iChAt3sRzWg0M=
+
+lodash.isnumber@^3.0.3:
+  version "3.0.3"
+  resolved "https://registry.yarnpkg.com/lodash.isnumber/-/lodash.isnumber-3.0.3.tgz#3ce76810c5928d03352301ac287317f11c0b1ffc"
+  integrity sha1-POdoEMWSjQM1IwGsKHMX8RwLH/w=
+
+lodash.isplainobject@^4.0.6:
+  version "4.0.6"
+  resolved "https://registry.yarnpkg.com/lodash.isplainobject/-/lodash.isplainobject-4.0.6.tgz#7c526a52d89b45c45cc690b88163be0497f550cb"
+  integrity sha1-fFJqUtibRcRcxpC4gWO+BJf1UMs=
+
+lodash.isstring@^4.0.1:
+  version "4.0.1"
+  resolved "https://registry.yarnpkg.com/lodash.isstring/-/lodash.isstring-4.0.1.tgz#d527dfb5456eca7cc9bb95d5daeaf88ba54a5451"
+  integrity sha1-1SfftUVuynzJu5XV2ur4i6VKVFE=
+
 lodash.keys@^4.2.0:
   version "4.2.0"
   resolved "https://registry.yarnpkg.com/lodash.keys/-/lodash.keys-4.2.0.tgz#a08602ac12e4fb83f91fc1fb7a360a4d9ba35205"
@@ -7807,11 +7994,21 @@ lodash.merge@^4.6.2:
   resolved "https://registry.yarnpkg.com/lodash.merge/-/lodash.merge-4.6.2.tgz#558aa53b43b661e1925a0afdfa36a9a1085fe57a"
   integrity sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==
 
+lodash.noop@^3.0.1:
+  version "3.0.1"
+  resolved "https://registry.yarnpkg.com/lodash.noop/-/lodash.noop-3.0.1.tgz#38188f4d650a3a474258439b96ec45b32617133c"
+  integrity sha1-OBiPTWUKOkdCWEObluxFsyYXEzw=
+
 lodash.omit@^4.5.0:
   version "4.5.0"
   resolved "https://registry.yarnpkg.com/lodash.omit/-/lodash.omit-4.5.0.tgz#6eb19ae5a1ee1dd9df0b969e66ce0b7fa30b5e60"
   integrity sha1-brGa5aHuHdnfC5aeZs4Lf6MLXmA=
 
+lodash.once@^4.0.0:
+  version "4.1.1"
+  resolved "https://registry.yarnpkg.com/lodash.once/-/lodash.once-4.1.1.tgz#0dd3971213c7c56df880977d504c88fb471a97ac"
+  integrity sha1-DdOXEhPHxW34gJd9UEyI+0cal6w=
+
 lodash.pick@^4.0.0:
   version "4.4.0"
   resolved "https://registry.yarnpkg.com/lodash.pick/-/lodash.pick-4.4.0.tgz#52f05610fff9ded422611441ed1fc123a03001b3"
@@ -7847,7 +8044,7 @@ lodash.xor@^4.5.0:
   resolved "https://registry.yarnpkg.com/lodash.xor/-/lodash.xor-4.5.0.tgz#4d48ed7e98095b0632582ba714d3ff8ae8fb1db6"
   integrity sha1-TUjtfpgJWwYyWCunFNP/iuj7HbY=
 
-lodash@4.17.21, lodash@4.x, lodash@^4.17.11, lodash@^4.17.14, lodash@^4.17.19, lodash@^4.17.20, lodash@^4.17.21, lodash@^4.17.3, lodash@^4.7.0:
+lodash@4.17.21, lodash@4.x, lodash@^4.14.0, lodash@^4.17.11, lodash@^4.17.14, lodash@^4.17.19, lodash@^4.17.20, lodash@^4.17.21, lodash@^4.17.3, lodash@^4.7.0:
   version "4.17.21"
   resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.21.tgz#679591c564c3bffaae8454cf0b3df370c3d6911c"
   integrity sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==
@@ -8097,6 +8294,11 @@ mixin-deep@^1.2.0:
     for-in "^1.0.2"
     is-extendable "^1.0.1"
 
+mkdirp-classic@^0.5.2:
+  version "0.5.3"
+  resolved "https://registry.yarnpkg.com/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz#fa10c9115cc6d8865be221ba47ee9bed78601113"
+  integrity sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==
+
 mkdirp@^0.5.0, mkdirp@^0.5.1:
   version "0.5.5"
   resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-0.5.5.tgz#d91cefd62d1436ca0f41620e251288d420099def"
@@ -8287,6 +8489,11 @@ node-fetch@^2.6.0, node-fetch@^2.6.1:
   resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.1.tgz#045bd323631f76ed2e2b55573394416b639a0052"
   integrity sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw==
 
+node-forge@^0.7.1:
+  version "0.7.6"
+  resolved "https://registry.yarnpkg.com/node-forge/-/node-forge-0.7.6.tgz#fdf3b418aee1f94f0ef642cd63486c77ca9724ac"
+  integrity sha512-sol30LUpz1jQFBjOKwbjxijiE3b6pjd74YwfD0fJOKPjF+fONKb2Yg8rYgS6+bK6VDl+/wfr4IYpC7jDzLUIfw==
+
 node-gyp-build@~4.1.0:
   version "4.1.1"
   resolved "https://registry.yarnpkg.com/node-gyp-build/-/node-gyp-build-4.1.1.tgz#d7270b5d86717068d114cc57fff352f96d745feb"
@@ -8398,6 +8605,11 @@ oauth-sign@~0.9.0:
   resolved "https://registry.yarnpkg.com/oauth-sign/-/oauth-sign-0.9.0.tgz#47a7b016baa68b5fa0ecf3dee08a85c679ac6455"
   integrity sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ==
 
+oauth@0.9.x, oauth@^0.9.15:
+  version "0.9.15"
+  resolved "https://registry.yarnpkg.com/oauth/-/oauth-0.9.15.tgz#bd1fefaf686c96b75475aed5196412ff60cfb9c1"
+  integrity sha1-vR/vr2hslrdUda7VGWQS/2DPucE=
+
 object-assign@^2.0.0:
   version "2.1.1"
   resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-2.1.1.tgz#43c36e5d569ff8e4816c4efa8be02d26967c18aa"
@@ -8695,6 +8907,84 @@ pascalcase@^0.1.1:
   resolved "https://registry.yarnpkg.com/pascalcase/-/pascalcase-0.1.1.tgz#b363e55e8006ca6fe21784d2db22bd15d7917f14"
   integrity sha1-s2PlXoAGym/iF4TS2yK9FdeRfxQ=
 
+passport-google-auth@^1.0.2:
+  version "1.0.2"
+  resolved "https://registry.yarnpkg.com/passport-google-auth/-/passport-google-auth-1.0.2.tgz#8b300b5aa442ef433de1d832ed3112877d0b2938"
+  integrity sha1-izALWqRC70M94dgy7TESh30LKTg=
+  dependencies:
+    googleapis "^16.0.0"
+    passport-strategy "1.x"
+
+passport-google-oauth1@1.x.x:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/passport-google-oauth1/-/passport-google-oauth1-1.0.0.tgz#af74a803df51ec646f66a44d82282be6f108e0cc"
+  integrity sha1-r3SoA99R7GRvZqRNgigr5vEI4Mw=
+  dependencies:
+    passport-oauth1 "1.x.x"
+
+passport-google-oauth20@2.x.x:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/passport-google-oauth20/-/passport-google-oauth20-2.0.0.tgz#0d241b2d21ebd3dc7f2b60669ec4d587e3a674ef"
+  integrity sha512-KSk6IJ15RoxuGq7D1UKK/8qKhNfzbLeLrG3gkLZ7p4A6DBCcv7xpyQwuXtWdpyR0+E0mwkpjY1VfPOhxQrKzdQ==
+  dependencies:
+    passport-oauth2 "1.x.x"
+
+passport-google-oauth@^2.0.0:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/passport-google-oauth/-/passport-google-oauth-2.0.0.tgz#f6eb4bc96dd6c16ec0ecfdf4e05ec48ca54d4dae"
+  integrity sha512-JKxZpBx6wBQXX1/a1s7VmdBgwOugohH+IxCy84aPTZNq/iIPX6u7Mqov1zY7MKRz3niFPol0KJz8zPLBoHKtYA==
+  dependencies:
+    passport-google-oauth1 "1.x.x"
+    passport-google-oauth20 "2.x.x"
+
+passport-jwt@^4.0.0:
+  version "4.0.0"
+  resolved "https://registry.yarnpkg.com/passport-jwt/-/passport-jwt-4.0.0.tgz#7f0be7ba942e28b9f5d22c2ebbb8ce96ef7cf065"
+  integrity sha512-BwC0n2GP/1hMVjR4QpnvqA61TxenUMlmfNjYNgK0ZAs0HK4SOQkHcSv4L328blNTLtHq7DbmvyNJiH+bn6C5Mg==
+  dependencies:
+    jsonwebtoken "^8.2.0"
+    passport-strategy "^1.0.0"
+
+passport-local@^1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/passport-local/-/passport-local-1.0.0.tgz#1fe63268c92e75606626437e3b906662c15ba6ee"
+  integrity sha1-H+YyaMkudWBmJkN+O5BmYsFbpu4=
+  dependencies:
+    passport-strategy "1.x.x"
+
+passport-oauth1@1.x.x:
+  version "1.2.0"
+  resolved "https://registry.yarnpkg.com/passport-oauth1/-/passport-oauth1-1.2.0.tgz#5229d431781bf5b265bec86ce9a9cce58a756cf9"
+  integrity sha512-Sv2YWodC6jN12M/OXwmR4BIXeeIHjjbwYTQw4kS6tHK4zYzSEpxBgSJJnknBjICA5cj0ju3FSnG1XmHgIhYnLg==
+  dependencies:
+    oauth "0.9.x"
+    passport-strategy "1.x.x"
+    utils-merge "1.x.x"
+
+passport-oauth2@1.x.x:
+  version "1.6.1"
+  resolved "https://registry.yarnpkg.com/passport-oauth2/-/passport-oauth2-1.6.1.tgz#c5aee8f849ce8bd436c7f81d904a3cd1666f181b"
+  integrity sha512-ZbV43Hq9d/SBSYQ22GOiglFsjsD1YY/qdiptA+8ej+9C1dL1TVB+mBE5kDH/D4AJo50+2i8f4bx0vg4/yDDZCQ==
+  dependencies:
+    base64url "3.x.x"
+    oauth "0.9.x"
+    passport-strategy "1.x.x"
+    uid2 "0.0.x"
+    utils-merge "1.x.x"
+
+passport-strategy@1.x, passport-strategy@1.x.x, passport-strategy@^1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/passport-strategy/-/passport-strategy-1.0.0.tgz#b5539aa8fc225a3d1ad179476ddf236b440f52e4"
+  integrity sha1-tVOaqPwiWj0a0XlHbd8ja0QPUuQ=
+
+passport@^0.4.0:
+  version "0.4.1"
+  resolved "https://registry.yarnpkg.com/passport/-/passport-0.4.1.tgz#941446a21cb92fc688d97a0861c38ce9f738f270"
+  integrity sha512-IxXgZZs8d7uFSt3eqNjM9NQ3g3uQCW5avD8mRNoXV99Yig50vjuaez6dQK2qC0kVWPRTujxY0dWgGfT09adjYg==
+  dependencies:
+    passport-strategy "1.x.x"
+    pause "0.0.1"
+
 path-exists@^3.0.0:
   version "3.0.0"
   resolved "https://registry.yarnpkg.com/path-exists/-/path-exists-3.0.0.tgz#ce0ebeaa5f78cb18925ea7d810d7b59b010fd515"
@@ -8749,6 +9039,11 @@ path-type@^4.0.0:
   resolved "https://registry.yarnpkg.com/path-type/-/path-type-4.0.0.tgz#84ed01c0a7ba380afe09d90a8c180dcd9d03043b"
   integrity sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==
 
+pause@0.0.1:
+  version "0.0.1"
+  resolved "https://registry.yarnpkg.com/pause/-/pause-0.0.1.tgz#1d408b3fdb76923b9543d96fb4c9dfd535d9cb5d"
+  integrity sha1-HUCLP9t2kjuVQ9lvtMnf1TXZy10=
+
 pend@~1.2.0:
   version "1.2.0"
   resolved "https://registry.yarnpkg.com/pend/-/pend-1.2.0.tgz#7a57eb550a6783f9115331fcf4663d5c8e007a50"
@@ -9447,7 +9742,7 @@ readable-stream@1.1.14, readable-stream@^1.0.27-1:
     isarray "0.0.1"
     string_decoder "~0.10.x"
 
-"readable-stream@2 || 3", readable-stream@^3.0.0, readable-stream@^3.0.1, readable-stream@^3.4.0, readable-stream@^3.6.0:
+"readable-stream@2 || 3", readable-stream@^3.0.0, readable-stream@^3.0.1, readable-stream@^3.1.1, readable-stream@^3.4.0, readable-stream@^3.6.0:
   version "3.6.0"
   resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-3.6.0.tgz#337bbda3adc0706bd3e024426a286d4b4b2c9198"
   integrity sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==
@@ -9680,7 +9975,7 @@ request-promise-native@^1.0.5:
     stealthy-require "^1.1.1"
     tough-cookie "^2.3.3"
 
-request@^2.87.0:
+request@^2.72.0, request@^2.74.0, request@^2.87.0, request@^2.88.0:
   version "2.88.2"
   resolved "https://registry.yarnpkg.com/request/-/request-2.88.2.tgz#d73c918731cb5a87da047e207234146f664d12b3"
   integrity sha512-MsvtOrfG9ZcrOwAW+Qi+F6HbD0CWXEh9ou77uOb7FM2WPhwT7smM833PzanhJLsgXjN89Ir6V2PczXNnMpwKhw==
@@ -9872,6 +10167,11 @@ sane@^4.0.3:
     minimist "^1.1.1"
     walker "~1.0.5"
 
+sanitize-s3-objectkey@^0.0.1:
+  version "0.0.1"
+  resolved "https://registry.yarnpkg.com/sanitize-s3-objectkey/-/sanitize-s3-objectkey-0.0.1.tgz#efa9887cd45275b40234fb4bb12fc5754fe64e7e"
+  integrity sha512-ZTk7aqLxy4sD40GWcYWoLfbe05XLmkKvh6vGKe13ADlei24xlezcvjgKy1qRArlaIbIMYaqK7PCalvZtulZlaQ==
+
 saslprep@^1.0.0:
   version "1.0.3"
   resolved "https://registry.yarnpkg.com/saslprep/-/saslprep-1.0.3.tgz#4c02f946b56cf54297e347ba1093e7acac4cf226"
@@ -9884,7 +10184,7 @@ sax@1.2.1:
   resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.1.tgz#7b8e656190b228e81a66aea748480d828cd2d37a"
   integrity sha1-e45lYZCyKOgaZq6nSEgNgozS03o=
 
-sax@>=0.6.0, sax@^1.2.4:
+sax@>=0.1.1, sax@>=0.6.0, sax@^1.2.4:
   version "1.2.4"
   resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.4.tgz#2816234e2378bddc4e5354fab5caa895df7100d9"
   integrity sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw==
@@ -9920,7 +10220,7 @@ semver-diff@^3.1.1:
   dependencies:
     semver "^6.3.0"
 
-"semver@2 || 3 || 4 || 5", semver@^5.1.0, semver@^5.5.0, semver@^5.6.0, semver@^5.7.1:
+"semver@2 || 3 || 4 || 5", semver@^5.1.0, semver@^5.4.1, semver@^5.5.0, semver@^5.6.0, semver@^5.7.1:
   version "5.7.1"
   resolved "https://registry.yarnpkg.com/semver/-/semver-5.7.1.tgz#a954f931aeba508d307bbf069eff0c01c96116f7"
   integrity sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==
@@ -10046,6 +10346,11 @@ shell-path@^2.1.0:
   dependencies:
     shell-env "^0.3.0"
 
+shimmer@^1.2.0:
+  version "1.2.1"
+  resolved "https://registry.yarnpkg.com/shimmer/-/shimmer-1.2.1.tgz#610859f7de327b587efebf501fb43117f9aff337"
+  integrity sha512-sQTKC1Re/rM6XyFM6fIAGHRPVGvyXfgzIDvzoq608vM+jeyVD0Tu1E6Np0Kc2zAIFWIj963V2800iF/9LPieQw==
+
 shortid@^2.2.15:
   version "2.2.16"
   resolved "https://registry.yarnpkg.com/shortid/-/shortid-2.2.16.tgz#b742b8f0cb96406fd391c76bfc18a67a57fe5608"
@@ -10297,6 +10602,11 @@ sshpk@^1.7.0:
     safer-buffer "^2.0.2"
     tweetnacl "~0.14.0"
 
+stack-chain@^1.3.7:
+  version "1.3.7"
+  resolved "https://registry.yarnpkg.com/stack-chain/-/stack-chain-1.3.7.tgz#d192c9ff4ea6a22c94c4dd459171e3f00cea1285"
+  integrity sha1-0ZLJ/06moiyUxN1FkXHj8AzqEoU=
+
 stack-utils@^1.0.1:
   version "1.0.5"
   resolved "https://registry.yarnpkg.com/stack-utils/-/stack-utils-1.0.5.tgz#a19b0b01947e0029c8e451d5d61a498f5bb1471b"
@@ -10339,6 +10649,11 @@ stealthy-require@^1.1.1:
   resolved "https://registry.yarnpkg.com/stealthy-require/-/stealthy-require-1.1.1.tgz#35b09875b4ff49f26a777e509b3090a3226bf24b"
   integrity sha1-NbCYdbT/SfJqd35QmzCQoyJr8ks=
 
+step@0.0.x:
+  version "0.0.6"
+  resolved "https://registry.yarnpkg.com/step/-/step-0.0.6.tgz#143e7849a5d7d3f4a088fe29af94915216eeede2"
+  integrity sha1-FD54SaXX0/SgiP4pr5SRUhbu7eI=
+
 strict-uri-encode@^1.0.0:
   version "1.1.0"
   resolved "https://registry.yarnpkg.com/strict-uri-encode/-/strict-uri-encode-1.1.0.tgz#279b225df1d582b1f54e65addd4352e18faa0713"
@@ -10352,6 +10667,11 @@ string-length@^4.0.1:
     char-regex "^1.0.2"
     strip-ansi "^6.0.0"
 
+string-template@~1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/string-template/-/string-template-1.0.0.tgz#9e9f2233dc00f218718ec379a28a5673ecca8b96"
+  integrity sha1-np8iM9wA8hhxjsN5oopWc+zKi5Y=
+
 string-width@^3.0.0, string-width@^3.1.0:
   version "3.1.0"
   resolved "https://registry.yarnpkg.com/string-width/-/string-width-3.1.0.tgz#22767be21b62af1081574306f69ac51b62203961"
@@ -10653,6 +10973,16 @@ table@^5.2.3:
     slice-ansi "^2.1.0"
     string-width "^3.0.0"
 
+tar-fs@^2.1.1:
+  version "2.1.1"
+  resolved "https://registry.yarnpkg.com/tar-fs/-/tar-fs-2.1.1.tgz#489a15ab85f1f0befabb370b7de4f9eb5cbe8784"
+  integrity sha512-V0r2Y9scmbDRLCNex/+hYzvp/zyYjvFbHPNgVTKfQvVrb6guiE/fxP+XblDNR011utopbkex2nM4dHNV6GDsng==
+  dependencies:
+    chownr "^1.1.1"
+    mkdirp-classic "^0.5.2"
+    pump "^3.0.0"
+    tar-stream "^2.1.4"
+
 tar-stream@^1.5.2:
   version "1.6.2"
   resolved "https://registry.yarnpkg.com/tar-stream/-/tar-stream-1.6.2.tgz#8ea55dab37972253d9a9af90fdcd559ae435c555"
@@ -10666,6 +10996,17 @@ tar-stream@^1.5.2:
     to-buffer "^1.1.1"
     xtend "^4.0.0"
 
+tar-stream@^2.1.4:
+  version "2.2.0"
+  resolved "https://registry.yarnpkg.com/tar-stream/-/tar-stream-2.2.0.tgz#acad84c284136b060dc3faa64474aa9aebd77287"
+  integrity sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==
+  dependencies:
+    bl "^4.0.3"
+    end-of-stream "^1.4.1"
+    fs-constants "^1.0.0"
+    inherits "^2.0.3"
+    readable-stream "^3.1.1"
+
 tarn@^1.1.5:
   version "1.1.5"
   resolved "https://registry.yarnpkg.com/tarn/-/tarn-1.1.5.tgz#7be88622e951738b9fa3fb77477309242cdddc2d"
@@ -11070,6 +11411,11 @@ uglify-js@^3.1.4:
   resolved "https://registry.yarnpkg.com/uglify-js/-/uglify-js-3.14.2.tgz#d7dd6a46ca57214f54a2d0a43cad0f35db82ac99"
   integrity sha512-rtPMlmcO4agTUfz10CbgJ1k6UAoXM2gWb3GoMPPZB/+/Ackf8lNWk11K4rYi2D0apgoFRLtQOZhb+/iGNJq26A==
 
+uid2@0.0.x:
+  version "0.0.4"
+  resolved "https://registry.yarnpkg.com/uid2/-/uid2-0.0.4.tgz#033f3b1d5d32505f5ce5f888b9f3b667123c0a44"
+  integrity sha512-IevTus0SbGwQzYh3+fRsAMTVVPOoIVufzacXcHPmdlle1jUpq7BRL+mw3dgeLanvGZdwwbWhRV6XrcFNdBmjWA==
+
 unbox-primitive@^1.0.1:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/unbox-primitive/-/unbox-primitive-1.0.1.tgz#085e215625ec3162574dc8859abee78a59b14471"
@@ -11261,7 +11607,7 @@ util.promisify@^1.0.0, util.promisify@^1.0.1:
     has-symbols "^1.0.1"
     object.getownpropertydescriptors "^2.1.1"
 
-utils-merge@1.0.1:
+utils-merge@1.0.1, utils-merge@1.x.x:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/utils-merge/-/utils-merge-1.0.1.tgz#9f95710f50a267947b2ccc124741c1028427e713"
   integrity sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM=
@@ -11358,6 +11704,14 @@ walker@^1.0.7, walker@~1.0.5:
   dependencies:
     makeerror "1.0.x"
 
+webfinger@^0.4.2:
+  version "0.4.2"
+  resolved "https://registry.yarnpkg.com/webfinger/-/webfinger-0.4.2.tgz#3477a6d97799461896039fcffc650b73468ee76d"
+  integrity sha1-NHem2XeZRhiWA5/P/GULc0aO520=
+  dependencies:
+    step "0.0.x"
+    xml2js "0.1.x"
+
 webidl-conversions@^4.0.2:
   version "4.0.2"
   resolved "https://registry.yarnpkg.com/webidl-conversions/-/webidl-conversions-4.0.2.tgz#a855980b1f0b6b359ba1d5d9fb39ae941faa63ad"
@@ -11557,6 +11911,13 @@ xml-parse-from-string@^1.0.0:
   resolved "https://registry.yarnpkg.com/xml-parse-from-string/-/xml-parse-from-string-1.0.1.tgz#a9029e929d3dbcded169f3c6e28238d95a5d5a28"
   integrity sha1-qQKekp09vN7RafPG4oI42VpdWig=
 
+xml2js@0.1.x:
+  version "0.1.14"
+  resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.1.14.tgz#5274e67f5a64c5f92974cd85139e0332adc6b90c"
+  integrity sha1-UnTmf1pkxfkpdM2FE54DMq3GuQw=
+  dependencies:
+    sax ">=0.1.1"
+
 xml2js@0.4.19:
   version "0.4.19"
   resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.4.19.tgz#686c20f213209e94abf0d1bcf1efaa291c7827a7"
@@ -11710,7 +12071,7 @@ yn@3.1.1:
   resolved "https://registry.yarnpkg.com/yn/-/yn-3.1.1.tgz#1e87401a09d767c1d5eab26a6e4c185182d2eb50"
   integrity sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q==
 
-zlib@1.0.5:
+zlib@1.0.5, zlib@^1.0.5:
   version "1.0.5"
   resolved "https://registry.yarnpkg.com/zlib/-/zlib-1.0.5.tgz#6e7c972fc371c645a6afb03ab14769def114fcc0"
   integrity sha1-bnyXL8NxxkWmr7A6sUdp3vEU/MA=

packages/string-templates/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/string-templates",
-  "version": "0.9.144-alpha.0",
+  "version": "0.9.146-alpha.3",
   "description": "Handlebars wrapper for Budibase templating.",
   "main": "src/index.cjs",
   "module": "dist/bundle.mjs",

packages/worker/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/worker",
   "email": "hi@budibase.com",
-  "version": "0.9.144-alpha.0",
+  "version": "0.9.146-alpha.3",
   "description": "Budibase background service",
   "main": "src/index.js",
   "repository": {
@@ -25,8 +25,8 @@
   "author": "Budibase",
   "license": "AGPL-3.0-or-later",
   "dependencies": {
-    "@budibase/auth": "^0.9.144-alpha.0",
+    "@budibase/auth": "^0.9.146-alpha.3",
-    "@budibase/string-templates": "^0.9.144-alpha.0",
+    "@budibase/string-templates": "^0.9.146-alpha.3",
     "@koa/router": "^8.0.0",
     "@techpass/passport-openidconnect": "^0.3.0",
     "aws-sdk": "^2.811.0",

packages/worker/src/api/controllers/global/users.js

@@ -53,22 +53,22 @@ async function saveUser(
     // check budibase users inside the tenant
     dbUser = await getGlobalUserByEmail(email)
     if (dbUser != null && (dbUser._id !== _id || Array.isArray(dbUser))) {
-      throw "Email address already in use."
+      throw `Email address ${email} already in use.`
     }
 
     // check budibase users in other tenants
     if (env.MULTI_TENANCY) {
       dbUser = await getTenantUser(email)
-      if (dbUser != null) {
+      if (dbUser != null && dbUser.tenantId !== tenantId) {
-        throw "Email address already in use."
+        throw `Email address ${email} already in use.`
       }
     }
 
     // check root account users in account portal
     if (!env.SELF_HOSTED && !env.DISABLE_ACCOUNT_PORTAL) {
       const account = await accounts.getAccount(email)
-      if (account) {
+      if (account && account.verified && account.tenantId !== tenantId) {
-        throw "Email address already in use."
+        throw `Email address ${email} already in use.`
       }
     }
   } else {

packages/worker/src/api/controllers/system/tenants.js

@@ -1,5 +1,7 @@
 const CouchDB = require("../../../db")
 const { StaticDatabases } = require("@budibase/auth/db")
+const { getTenantId } = require("@budibase/auth/tenancy")
+const { deleteTenant } = require("@budibase/auth/deprovision")
 
 exports.exists = async ctx => {
   const tenantId = ctx.request.params
@@ -31,3 +33,19 @@ exports.fetch = async ctx => {
   }
   ctx.body = tenants
 }
+
+exports.delete = async ctx => {
+  const tenantId = getTenantId()
+
+  if (ctx.params.tenantId !== tenantId) {
+    ctx.throw(403, "Unauthorized")
+  }
+
+  try {
+    await deleteTenant(tenantId)
+    ctx.status = 204
+  } catch (err) {
+    ctx.log.error(err)
+    throw err
+  }
+}

packages/worker/src/api/routes/system/tenants.js

@@ -7,5 +7,6 @@ const router = Router()
 router
   .get("/api/system/tenants/:tenantId/exists", controller.exists)
   .get("/api/system/tenants", adminOnly, controller.fetch)
+  .delete("/api/system/tenants/:tenantId", adminOnly, controller.delete)
 
 module.exports = router