code review and merge with develop

.github/workflows/smoke_test.yaml

@@ -0,0 +1,46 @@
+name: Budibase Smoke Test
+
+on:
+ workflow_dispatch:
+
+jobs:
+  release:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Use Node.js 14.x
+        uses: actions/setup-node@v1
+        with:
+          node-version: 14.x
+      - run: yarn
+      - run: yarn bootstrap
+      - run: yarn build
+      - name: Pull cypress.env.yaml from budibase-infra
+        run: | 
+          curl -H "Authorization: token ${{ secrets.GH_PERSONAL_TOKEN }}" \
+          -H 'Accept: application/vnd.github.v3.raw' \
+          -o packages/builder/cypress.env.json \
+          -L https://api.github.com/repos/budibase/budibase-infra/contents/test/cypress.env.json
+          wc -l packages/builder/cypress.env.json
+      - run: yarn test:e2e:ci
+        env:
+          CI: true
+          name: Budibase CI
+      
+      # TODO: upload recordings to s3
+      # - name: Configure AWS Credentials
+      #   uses: aws-actions/configure-aws-credentials@v1
+      #   with:
+      #     aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+      #     aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      #     aws-region: eu-west-1
+        
+      # TODO look at cypress reporters
+      # - name: Discord Webhook Action
+      #   uses: tsickert/discord-webhook@v4.0.0
+      #   with:
+      #     webhook-url: ${{ secrets.PROD_DEPLOY_WEBHOOK_URL }}
+      #     content: "Production Deployment Complete: ${{ env.RELEASE_VERSION }} deployed to Budibase Cloud."
+      #     embed-title: ${{ env.RELEASE_VERSION }}
+

charts/budibase/templates/app-service-deployment.yaml

@@ -99,7 +99,9 @@ spec:
         - name: PLATFORM_URL
           value: {{ .Values.globals.platformUrl | quote }}
         - name: USE_QUOTAS
-          value: "1" 
+          value: {{ .Values.globals.useQuotas | quote }}
+        - name: EXCLUDE_QUOTAS_TENANTS
+          value: {{ .Values.globals.excludeQuotasTenants | quote }}
         - name: ACCOUNT_PORTAL_URL
           value: {{ .Values.globals.accountPortalUrl | quote }}
         - name: ACCOUNT_PORTAL_API_KEY

charts/budibase/values.yaml

@@ -93,6 +93,8 @@ globals:
   logLevel: info
   selfHosted: "1" # set to 0 for budibase cloud environment, set to 1 for self-hosted setup
   multiTenancy: "0" # set to 0 to disable multiple orgs, set to 1 to enable multiple orgs
+  useQuotas: "0"
+  excludeQuotasTenants: "" # comma seperated list of tenants to exclude from quotas
   accountPortalUrl: ""
   accountPortalApiKey: ""
   cookieDomain: ""
@@ -239,7 +241,8 @@ couchdb:
     hosts:
       - chart-example.local
     path: /
-    annotations: []
+    annotations:
+      []
       # kubernetes.io/ingress.class: nginx
       # kubernetes.io/tls-acme: "true"
     tls:

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "1.0.47",
+  "version": "1.0.46-alpha.5",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

packages/backend-core/migrations.js

@@ -0,0 +1 @@
+module.exports = require("./src/migrations")

packages/backend-core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/backend-core",
-  "version": "1.0.47",
+  "version": "1.0.46-alpha.5",
   "description": "Budibase backend core libraries used in server and worker",
   "main": "src/index.js",
   "author": "Budibase",

packages/backend-core/src/db/constants.js

@@ -21,6 +21,7 @@ exports.StaticDatabases = {
     name: "global-db",
     docs: {
       apiKeys: "apikeys",
+      usageQuota: "usage_quota",
     },
   },
   // contains information about tenancy and so on
@@ -28,7 +29,6 @@ exports.StaticDatabases = {
     name: "global-info",
     docs: {
       tenants: "tenants",
-      usageQuota: "usage_quota",
     },
   },
 }

packages/backend-core/src/db/utils.js

@@ -450,7 +450,7 @@ async function getScopedConfig(db, params) {
 
 function generateNewUsageQuotaDoc() {
   return {
-    _id: StaticDatabases.PLATFORM_INFO.docs.usageQuota,
+    _id: StaticDatabases.GLOBAL.docs.usageQuota,
     quotaReset: Date.now() + 2592000000,
     usageQuota: {
       automationRuns: 0,

packages/backend-core/src/index.js

@@ -14,4 +14,5 @@ module.exports = {
   cache: require("../cache"),
   auth: require("../auth"),
   constants: require("../constants"),
+  migrations: require("../migrations"),
 }

packages/backend-core/src/migrations/index.js

@@ -1,5 +1,5 @@
 const { DocumentTypes } = require("../db/constants")
-const { getGlobalDB } = require("../tenancy")
+const { getGlobalDB, getTenantId } = require("../tenancy")
 
 exports.MIGRATION_DBS = {
   GLOBAL_DB: "GLOBAL_DB",
@@ -7,11 +7,13 @@ exports.MIGRATION_DBS = {
 
 exports.MIGRATIONS = {
   USER_EMAIL_VIEW_CASING: "user_email_view_casing",
+  QUOTAS_1: "quotas_1",
 }
 
 const DB_LOOKUP = {
   [exports.MIGRATION_DBS.GLOBAL_DB]: [
     exports.MIGRATIONS.USER_EMAIL_VIEW_CASING,
+    exports.MIGRATIONS.QUOTAS_1,
   ],
 }
 
@@ -27,6 +29,7 @@ exports.getMigrationsDoc = async db => {
 }
 
 exports.migrateIfRequired = async (migrationDb, migrationName, migrateFn) => {
+  const tenantId = getTenantId()
   try {
     let db
     if (migrationDb === exports.MIGRATION_DBS.GLOBAL_DB) {
@@ -47,15 +50,18 @@ exports.migrateIfRequired = async (migrationDb, migrationName, migrateFn) => {
       return
     }
 
-    console.log(`Performing migration: ${migrationName}`)
+    console.log(`[Tenant: ${tenantId}] Performing migration: ${migrationName}`)
     await migrateFn()
-    console.log(`Migration complete: ${migrationName}`)
+    console.log(`[Tenant: ${tenantId}] Migration complete: ${migrationName}`)
 
     // mark as complete
     doc[migrationName] = Date.now()
     await db.put(doc)
   } catch (err) {
-    console.error(`Error performing migration: ${migrationName}: `, err)
+    console.error(
+      `[Tenant: ${tenantId}] Error performing migration: ${migrationName}: `,
+      err
+    )
     throw err
   }
 }

packages/backend-core/yarn.lock

@@ -3410,9 +3410,9 @@ node-fetch@2.6.0:
   integrity sha512-8dG4H5ujfvFiqDmVu9fQ5bOHUC15JMjMY/Zumv26oOvvVJjM67KF8koCWIabKQ1GJIa9r2mMZscBq/TbdOcmNA==
 
 node-fetch@^2.6.1:
-  version "2.6.6"
+  version "2.6.7"
-  resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.6.tgz#1751a7c01834e8e1697758732e9efb6eeadfaf89"
+  resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.7.tgz#24de9fba827e3b4ae44dc8b20256a379160052ad"
-  integrity sha512-Z8/6vRlTUChSdIgMa51jxQ4lrw/Jy5SOW10ObaA47/RElsAN2c5Pn8bTgFGWn/ibwzXTE8qwr1Yzx28vsecXEA==
+  integrity sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==
   dependencies:
     whatwg-url "^5.0.0"
 

packages/bbui/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/bbui",
   "description": "A UI solution used in the different Budibase projects.",
-  "version": "1.0.47",
+  "version": "1.0.46-alpha.5",
   "license": "MPL-2.0",
   "svelte": "src/index.js",
   "module": "dist/bbui.es.js",

packages/bbui/src/Form/Core/Dropzone.svelte

@@ -22,6 +22,7 @@
   export let error = null
   export let fileTags = []
   export let maximum = null
+  export let extensions = "*"
 
   const dispatch = createEventDispatcher()
   const imageExtensions = [
@@ -146,7 +147,9 @@
           <img alt="preview" src={selectedUrl} />
         {:else}
           <div class="placeholder">
-            <div class="extension">{selectedImage.extension}</div>
+            <div class="extension">
+              {selectedImage.name || "Unknown file"}
+            </div>
             <div>Preview not supported</div>
           </div>
         {/if}
@@ -207,6 +210,7 @@
           {disabled}
           type="file"
           multiple
+          accept={extensions}
           on:change={handleFile}
         />
         <svg
@@ -357,18 +361,21 @@
     white-space: nowrap;
     width: 0;
     margin-right: 10px;
+    user-select: all;
   }
   .placeholder {
     display: flex;
     flex-direction: column;
     justify-content: center;
     align-items: center;
+    text-align: center;
   }
   .extension {
     color: var(--spectrum-global-color-gray-600);
     text-transform: uppercase;
     font-weight: 600;
     margin-bottom: 5px;
+    user-select: all;
   }
 
   .nav {

packages/builder/cypress/support/commands.js

@@ -35,7 +35,13 @@ Cypress.Commands.add("login", () => {
 Cypress.Commands.add("createApp", name => {
   cy.visit(`localhost:${Cypress.env("PORT")}/builder`)
   cy.wait(500)
-  cy.contains(/Start from scratch/).dblclick()
+  cy.request(`${Cypress.config().baseUrl}api/applications?status=all`)
+    .its("body")
+    .then(body => {
+      if (body.length > 0) {
+        cy.get(".spectrum-Button").contains("Create app").click({ force: true })
+      }
+    })
   cy.get(".spectrum-Modal").within(() => {
     cy.get("input").eq(0).type(name).should("have.value", name).blur()
     cy.get(".spectrum-ButtonGroup").contains("Create app").click()

packages/builder/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/builder",
-  "version": "1.0.47",
+  "version": "1.0.46-alpha.5",
   "license": "GPL-3.0",
   "private": true,
   "scripts": {
@@ -65,10 +65,10 @@
     }
   },
   "dependencies": {
-    "@budibase/bbui": "^1.0.47",
+    "@budibase/bbui": "^1.0.46-alpha.5",
-    "@budibase/client": "^1.0.47",
+    "@budibase/client": "^1.0.46-alpha.5",
     "@budibase/colorpicker": "1.1.2",
-    "@budibase/string-templates": "^1.0.47",
+    "@budibase/string-templates": "^1.0.46-alpha.5",
     "@sentry/browser": "5.19.1",
     "@spectrum-css/page": "^3.0.1",
     "@spectrum-css/vars": "^3.0.1",

packages/builder/src/builderStore/api.js

@@ -6,6 +6,7 @@ const apiCall =
   method =>
   async (url, body, headers = { "Content-Type": "application/json" }) => {
     headers["x-budibase-app-id"] = svelteGet(store).appId
+    headers["x-budibase-api-version"] = "1"
     const json = headers["Content-Type"] === "application/json"
     const resp = await fetch(url, {
       method: method,

packages/builder/src/builderStore/cookies.js

@@ -1,16 +1,26 @@
 export const Cookies = {
   Auth: "budibase:auth",
   CurrentApp: "budibase:currentapp",
+  ReturnUrl: "budibase:returnurl",
+}
+
+export function setCookie(name, value) {
+  if (getCookie(name)) {
+    removeCookie(name)
+  }
+  window.document.cookie = `${name}=${value}; Path=/;`
 }
 
 export function getCookie(cookieName) {
-  return document.cookie.split(";").some(cookie => {
+  const value = `; ${document.cookie}`
-    return cookie.trim().startsWith(`${cookieName}=`)
+  const parts = value.split(`; ${cookieName}=`)
-  })
+  if (parts.length === 2) {
+    return parts[1].split(";").shift()
+  }
 }
 
 export function removeCookie(cookieName) {
   if (getCookie(cookieName)) {
-    document.cookie = `${cookieName}=; Max-Age=-99999999;`
+    document.cookie = `${cookieName}=; Max-Age=-99999999; Path=/;`
   }
 }

packages/builder/src/builderStore/index.js

@@ -1,6 +1,5 @@
 import { getFrontendStore } from "./store/frontend"
 import { getAutomationStore } from "./store/automation"
-import { getHostingStore } from "./store/hosting"
 import { getThemeStore } from "./store/theme"
 import { derived, writable } from "svelte/store"
 import { FrontendTypes, LAYOUT_NAMES } from "../constants"
@@ -9,7 +8,6 @@ import { findComponent } from "./componentUtils"
 export const store = getFrontendStore()
 export const automationStore = getAutomationStore()
 export const themeStore = getThemeStore()
-export const hostingStore = getHostingStore()
 
 export const currentAsset = derived(store, $store => {
   const type = $store.currentFrontEndType

packages/builder/src/builderStore/store/frontend.js

@@ -2,7 +2,6 @@ import { get, writable } from "svelte/store"
 import { cloneDeep } from "lodash/fp"
 import {
   allScreens,
-  hostingStore,
   currentAsset,
   mainLayout,
   selectedComponent,
@@ -100,7 +99,6 @@ export const getFrontendStore = () => {
         version: application.version,
         revertableVersion: application.revertableVersion,
       }))
-      await hostingStore.actions.fetch()
 
       // Initialise backend stores
       const [_integrations] = await Promise.all([

packages/builder/src/builderStore/store/hosting.js

@@ -1,34 +0,0 @@
-import { writable } from "svelte/store"
-import api, { get } from "../api"
-
-const INITIAL_HOSTING_UI_STATE = {
-  appUrl: "",
-  deployedApps: {},
-  deployedAppNames: [],
-  deployedAppUrls: [],
-}
-
-export const getHostingStore = () => {
-  const store = writable({ ...INITIAL_HOSTING_UI_STATE })
-  store.actions = {
-    fetch: async () => {
-      const response = await api.get("/api/hosting/urls")
-      const urls = await response.json()
-      store.update(state => {
-        state.appUrl = urls.app
-        return state
-      })
-    },
-    fetchDeployedApps: async () => {
-      let deployments = await (await get("/api/hosting/apps")).json()
-      store.update(state => {
-        state.deployedApps = deployments
-        state.deployedAppNames = Object.values(deployments).map(app => app.name)
-        state.deployedAppUrls = Object.values(deployments).map(app => app.url)
-        return state
-      })
-      return deployments
-    },
-  }
-  return store
-}

packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte

@@ -126,7 +126,12 @@
     }
   }
 
+  function cancelEdit() {
+    field.name = originalName
+  }
+
   function deleteColumn() {
+    field.name = deleteColName
     if (field.name === $tables.selected.primaryDisplay) {
       notifications.error("You cannot delete the display column")
     } else {
@@ -307,6 +312,7 @@
   title={originalName ? "Edit Column" : "Create Column"}
   confirmText="Save Column"
   onConfirm={saveColumn}
+  onCancel={cancelEdit}
   disabled={invalid}
 >
   <Input
@@ -470,16 +476,16 @@
   onOk={deleteColumn}
   onCancel={hideDeleteDialog}
   title="Confirm Deletion"
-  disabled={deleteColName !== field.name}
+  disabled={deleteColName !== originalName}
 >
   <p>
-    Are you sure you wish to delete the column <b>{field.name}?</b>
+    Are you sure you wish to delete the column <b>{originalName}?</b>
     Your data will be deleted and this action cannot be undone - enter the column
     name to confirm.
   </p>
   <Input
     dataCy="delete-column-confirm"
     bind:value={deleteColName}
-    placeholder={field.name}
+    placeholder={originalName}
   />
 </ConfirmDialog>

packages/builder/src/components/backend/DataTable/modals/JSONSchemaModal.svelte

@@ -9,6 +9,7 @@
     Select,
     Body,
     Layout,
+    ActionButton,
   } from "@budibase/bbui"
   import { onMount, createEventDispatcher } from "svelte"
   import { FIELDS } from "constants/backend"
@@ -20,8 +21,8 @@
   let dispatcher = createEventDispatcher()
   let mode = "Form"
   let fieldCount = 0
-  let fieldKeys = {},
+  let fieldKeys = [],
-    fieldTypes = {}
+    fieldTypes = []
   let keyValueOptions = [
     { label: "String", value: FIELDS.STRING.type },
     { label: "Number", value: FIELDS.NUMBER.type },
@@ -50,27 +51,48 @@
     if (!schema) {
       schema = {}
     }
-    let i = 0
+    // find the entries which aren't in the list
-    for (let [key, value] of Object.entries(schema)) {
+    const schemaEntries = Object.entries(schema).filter(
-      fieldKeys[i] = key
+      ([key]) => !fieldKeys.includes(key)
-      fieldTypes[i] = value.type
+    )
-      i++
+    for (let [key, value] of schemaEntries) {
+      fieldKeys.push(key)
+      fieldTypes.push(value.type)
     }
-    fieldCount = i
+    fieldCount = fieldKeys.length
   }
 
   function saveSchema() {
-    for (let i of Object.keys(fieldKeys)) {
+    const newSchema = {}
-      const key = fieldKeys[i]
+    for (let [index, key] of fieldKeys.entries()) {
       // they were added to schema, rather than generated
-      if (!schema[key]) {
+      newSchema[key] = {
-        schema[key] = {
+        ...schema[key],
-          type: fieldTypes[i],
+        type: fieldTypes[index],
-        }
       }
     }
+    dispatcher("save", { schema: newSchema, json })
+    schema = newSchema
+  }
 
-    dispatcher("save", { schema, json })
+  function removeKey(index) {
+    const keyToRemove = fieldKeys[index]
+    if (fieldKeys[index + 1] != null) {
+      fieldKeys[index] = fieldKeys[index + 1]
+      fieldTypes[index] = fieldTypes[index + 1]
+    }
+    fieldKeys.splice(index, 1)
+    fieldTypes.splice(index, 1)
+    fieldCount--
+    if (json) {
+      try {
+        const parsed = JSON.parse(json)
+        delete parsed[keyToRemove]
+        json = JSON.stringify(parsed, null, 2)
+      } catch (err) {
+        // json not valid, ignore
+      }
+    }
   }
 
   onMount(() => {
@@ -97,6 +119,7 @@
             getOptionValue={field => field.value}
             getOptionLabel={field => field.label}
           />
+          <ActionButton icon="Close" quiet on:click={() => removeKey(i)} />
         </div>
       {/each}
       <div class:add-field-btn={fieldCount !== 0}>
@@ -118,9 +141,9 @@
 <style>
   .horizontal {
     display: grid;
-    grid-template-columns: 30% 1fr;
+    grid-template-columns: 30% 1fr 40px;
     grid-gap: var(--spacing-s);
-    align-items: center;
+    align-items: end;
   }
 
   .add-field-btn {

packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/CreateExternalTableModal.svelte

@@ -6,9 +6,12 @@
   export let datasource
 
   let name = ""
+  let submitted = false
   $: valid = name && name.length > 0 && !datasource?.entities[name]
   $: error =
-    name && datasource?.entities[name] ? "Table name already in use." : null
+    !submitted && name && datasource?.entities[name]
+      ? "Table name already in use."
+      : null
 
   function buildDefaultTable(tableName, datasourceId) {
     return {
@@ -26,6 +29,7 @@
   }
 
   async function saveTable() {
+    submitted = true
     const table = await tables.save(buildDefaultTable(name, datasource._id))
     await datasources.fetch()
     $goto(`../../table/${table._id}`)

packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/PlusConfigForm.svelte

@@ -188,7 +188,7 @@
 {:else}
   <Body size="S"><i>No tables found.</i></Body>
 {/if}
-{#if plusTables?.length !== 0}
+{#if plusTables?.length !== 0 && integration.relationships}
   <Divider size="S" />
   <div class="query-header">
     <Heading size="S">Relationships</Heading>

packages/builder/src/components/backend/DatasourceNavigator/_components/GoogleButton.svelte

@@ -5,13 +5,17 @@
   import { auth } from "stores/portal"
 
   export let preAuthStep
+  export let datasource
 
   $: tenantId = $auth.tenantId
 </script>
 
 <ActionButton
   on:click={async () => {
-    const datasource = await preAuthStep()
+    let ds = datasource
+    if (!ds) {
+      ds = await preAuthStep()
+    }
     window.open(
       `/api/global/auth/${tenantId}/datasource/google?datasourceId=${datasource._id}&appId=${$store.appId}`,
       "_blank"

packages/builder/src/components/backend/TableNavigator/modals/CreateTableModal.svelte

@@ -53,16 +53,23 @@
     }
 
     // Create table
-    const table = await tables.save(newTable)
+    let table
-    notifications.success(`Table ${name} created successfully.`)
+    try {
-    analytics.captureEvent(Events.TABLE.CREATED, { name })
+      table = await tables.save(newTable)
+      notifications.success(`Table ${name} created successfully.`)
+      analytics.captureEvent(Events.TABLE.CREATED, { name })
 
-    // Navigate to new table
+      // Navigate to new table
-    const currentUrl = $url()
+      const currentUrl = $url()
-    const path = currentUrl.endsWith("data")
+      const path = currentUrl.endsWith("data")
-      ? `./table/${table._id}`
+        ? `./table/${table._id}`
-      : `../../table/${table._id}`
+        : `../../table/${table._id}`
-    $goto(path)
+      $goto(path)
+    } catch (e) {
+      notifications.error(e)
+      // reload in case the table was created
+      await tables.fetch()
+    }
   }
 </script>
 

packages/builder/src/components/deploy/DeploymentHistory.svelte

@@ -6,7 +6,7 @@
   import api from "builderStore/api"
   import { notifications } from "@budibase/bbui"
   import CreateWebhookDeploymentModal from "./CreateWebhookDeploymentModal.svelte"
-  import { store, hostingStore } from "builderStore"
+  import { store } from "builderStore"
 
   const DeploymentStatus = {
     SUCCESS: "SUCCESS",
@@ -37,7 +37,7 @@
   let poll
   let deployments = []
   let urlComponent = $store.url || `/${appId}`
-  let deploymentUrl = `${$hostingStore.appUrl}${urlComponent}`
+  let deploymentUrl = `${urlComponent}`
 
   const formatDate = (date, format) =>
     Intl.DateTimeFormat("en-GB", DATE_OPTIONS[format]).format(date)

packages/builder/src/components/design/AppPreview/AppThemeSelect.svelte

@@ -1,6 +1,7 @@
 <script>
   import { Select } from "@budibase/bbui"
   import { store } from "builderStore"
+  import { get } from "svelte/store"
 
   const themeOptions = [
     {
@@ -20,6 +21,17 @@
       value: "spectrum--darkest",
     },
   ]
+
+  const onChangeTheme = async theme => {
+    await store.actions.theme.save(theme)
+    await store.actions.customTheme.save({
+      ...get(store).customTheme,
+      navBackground:
+        theme === "spectrum--light"
+          ? "var(--spectrum-global-color-gray-50)"
+          : "var(--spectrum-global-color-gray-100)",
+    })
+  }
 </script>
 
 <div>
@@ -27,7 +39,7 @@
     value={$store.theme}
     options={themeOptions}
     placeholder={null}
-    on:change={e => store.actions.theme.save(e.detail)}
+    on:change={e => onChangeTheme(e.detail)}
   />
 </div>
 

packages/builder/src/components/design/AppPreview/ThemeEditor.svelte

@@ -19,7 +19,7 @@
     primaryColor: "var(--spectrum-global-color-blue-600)",
     primaryColorHover: "var(--spectrum-global-color-blue-500)",
     buttonBorderRadius: "16px",
-    navBackground: "var(--spectrum-global-color-gray-100)",
+    navBackground: "var(--spectrum-global-color-gray-50)",
     navTextColor: "var(--spectrum-global-color-gray-800)",
   }
 
@@ -52,7 +52,14 @@
   }
 
   const resetTheme = () => {
-    store.actions.customTheme.save(null)
+    const theme = get(store).theme
+    store.actions.customTheme.save({
+      ...defaultTheme,
+      navBackground:
+        theme === "spectrum--light"
+          ? "var(--spectrum-global-color-gray-50)"
+          : "var(--spectrum-global-color-gray-100)",
+    })
   }
 </script>
 

packages/builder/src/components/design/AppPreview/componentStructure.json

@@ -44,7 +44,8 @@
       "relationshipfield",
       "daterangepicker",
       "multifieldselect",
-      "jsonfield"
+      "jsonfield",
+      "s3upload"
     ]
   },
   {

packages/builder/src/components/design/PropertiesPanel/PropertyControls/ButtonActionEditor/actions/LogOut.svelte

@@ -1,13 +1,38 @@
 <script>
-  import { Body } from "@budibase/bbui"
+  import { Label, Body, Layout } from "@budibase/bbui"
+  import DrawerBindableInput from "components/common/bindings/DrawerBindableInput.svelte"
+
+  export let parameters
+  export let bindings = []
 </script>
 
 <div class="root">
-  <Body size="S">This action doesn't require any additional settings.</Body>
+  <Layout noPadding gap="M">
+    <Body size="S">
+      Please enter the URL you would like to be redirected to after logging out.
+      If you don't enter a value, you'll be redirected to the login screen.
+    </Body>
+    <div class="content">
+      <Label small>Redirect URL</Label>
+      <DrawerBindableInput
+        title="Return URL"
+        value={parameters.redirectUrl}
+        on:change={value => (parameters.redirectUrl = value.detail)}
+        {bindings}
+      />
+    </div>
+  </Layout>
 </div>
 
 <style>
   .root {
+    max-width: 400px;
     margin: 0 auto;
   }
+  .content {
+    display: grid;
+    align-items: center;
+    gap: var(--spacing-m);
+    grid-template-columns: auto 1fr;
+  }
 </style>

packages/builder/src/components/design/PropertiesPanel/PropertyControls/ButtonActionEditor/actions/S3Upload.svelte

@@ -0,0 +1,33 @@
+<script>
+  import { Select, Label } from "@budibase/bbui"
+  import { currentAsset } from "builderStore"
+  import { findAllMatchingComponents } from "builderStore/componentUtils"
+
+  export let parameters
+
+  $: components = findAllMatchingComponents($currentAsset.props, component =>
+    component._component.endsWith("s3upload")
+  )
+</script>
+
+<div class="root">
+  <Label small>S3 Upload Component</Label>
+  <Select
+    bind:value={parameters.componentId}
+    options={components}
+    getOptionLabel={x => x._instanceName}
+    getOptionValue={x => x._id}
+  />
+</div>
+
+<style>
+  .root {
+    display: grid;
+    column-gap: var(--spacing-l);
+    row-gap: var(--spacing-s);
+    grid-template-columns: 120px 1fr;
+    align-items: center;
+    max-width: 400px;
+    margin: 0 auto;
+  }
+</style>

packages/builder/src/components/design/PropertiesPanel/PropertyControls/ButtonActionEditor/actions/index.js

@@ -11,3 +11,4 @@ export { default as ChangeFormStep } from "./ChangeFormStep.svelte"
 export { default as UpdateState } from "./UpdateState.svelte"
 export { default as RefreshDataProvider } from "./RefreshDataProvider.svelte"
 export { default as DuplicateRow } from "./DuplicateRow.svelte"
+export { default as S3Upload } from "./S3Upload.svelte"

packages/builder/src/components/design/PropertiesPanel/PropertyControls/ButtonActionEditor/manifest.json

@@ -70,6 +70,16 @@
       "name": "Update State",
       "component": "UpdateState",
       "dependsOnFeature": "state"
+    },
+    {
+      "name": "Upload File to S3",
+      "component": "S3Upload",
+      "context": [
+        {
+          "label": "File URL",
+          "value": "publicUrl"
+        }
+      ]
     }
   ]
 }

packages/builder/src/components/design/PropertiesPanel/PropertyControls/FilterEditor/FilterDrawer.svelte

@@ -14,6 +14,7 @@
   import ClientBindingPanel from "components/common/bindings/ClientBindingPanel.svelte"
   import { generate } from "shortid"
   import { getValidOperatorsForType, OperatorOptions } from "constants/lucene"
+  import { getFields } from "helpers/searchFields"
 
   export let schemaFields
   export let filters = []
@@ -21,11 +22,8 @@
   export let panel = ClientBindingPanel
   export let allowBindings = true
 
-  const BannedTypes = ["link", "attachment", "formula", "json", "jsonarray"]
+  $: enrichedSchemaFields = getFields(schemaFields || [])
-
+  $: fieldOptions = enrichedSchemaFields.map(field => field.name) || []
-  $: fieldOptions = (schemaFields ?? [])
-    .filter(field => !BannedTypes.includes(field.type))
-    .map(field => field.name)
   $: valueTypeOptions = allowBindings ? ["Value", "Binding"] : ["Value"]
 
   const addFilter = () => {
@@ -53,7 +51,7 @@
 
   const onFieldChange = (expression, field) => {
     // Update the field type
-    expression.type = schemaFields.find(x => x.name === field)?.type
+    expression.type = enrichedSchemaFields.find(x => x.name === field)?.type
 
     // Ensure a valid operator is set
     const validOperators = getValidOperatorsForType(expression.type).map(
@@ -85,7 +83,7 @@
   }
 
   const getFieldOptions = field => {
-    const schema = schemaFields.find(x => x.name === field)
+    const schema = enrichedSchemaFields.find(x => x.name === field)
     return schema?.constraints?.inclusion || []
   }
 </script>

packages/builder/src/components/design/PropertiesPanel/PropertyControls/S3DataSourceSelect.svelte

@@ -0,0 +1,15 @@
+<script>
+  import { Select } from "@budibase/bbui"
+  import { datasources } from "stores/backend"
+
+  export let value = null
+
+  $: dataSources = $datasources.list
+    .filter(ds => ds.source === "S3" && !ds.config?.endpoint)
+    .map(ds => ({
+      label: ds.name,
+      value: ds._id,
+    }))
+</script>
+
+<Select options={dataSources} {value} on:change />

packages/builder/src/components/design/PropertiesPanel/PropertyControls/SearchFieldSelect.svelte

@@ -0,0 +1,47 @@
+<script>
+  import { Multiselect } from "@budibase/bbui"
+  import {
+    getDatasourceForProvider,
+    getSchemaForDatasource,
+  } from "builderStore/dataBinding"
+  import { currentAsset } from "builderStore"
+  import { tables } from "stores/backend"
+  import { createEventDispatcher } from "svelte"
+  import { getFields } from "helpers/searchFields"
+
+  export let componentInstance = {}
+  export let value = ""
+  export let placeholder
+
+  const dispatch = createEventDispatcher()
+  $: datasource = getDatasourceForProvider($currentAsset, componentInstance)
+  $: schema = getSchemaForDatasource($currentAsset, datasource).schema
+  $: options = getOptions(datasource, schema || {})
+  $: boundValue = getSelectedOption(value, options)
+
+  function getOptions(ds, dsSchema) {
+    let base = Object.values(dsSchema)
+    if (!ds?.tableId) {
+      return base
+    }
+    const currentTable = $tables.list.find(table => table._id === ds.tableId)
+    return getFields(base, { allowLinks: currentTable.sql }).map(
+      field => field.name
+    )
+  }
+
+  function getSelectedOption(selectedOptions, allOptions) {
+    // Fix the hardcoded default string value
+    if (!Array.isArray(selectedOptions)) {
+      selectedOptions = []
+    }
+    return selectedOptions.filter(val => allOptions.indexOf(val) !== -1)
+  }
+
+  const setValue = value => {
+    boundValue = getSelectedOption(value.detail, options)
+    dispatch("change", boundValue)
+  }
+</script>
+
+<Multiselect {placeholder} value={boundValue} on:change={setValue} {options} />

packages/builder/src/components/design/PropertiesPanel/PropertyControls/componentSettings.js

@@ -1,5 +1,6 @@
 import { Checkbox, Select, Stepper } from "@budibase/bbui"
 import DataSourceSelect from "./DataSourceSelect.svelte"
+import S3DataSourceSelect from "./S3DataSourceSelect.svelte"
 import DataProviderSelect from "./DataProviderSelect.svelte"
 import ButtonActionEditor from "./ButtonActionEditor/ButtonActionEditor.svelte"
 import TableSelect from "./TableSelect.svelte"
@@ -7,6 +8,7 @@ import ColorPicker from "./ColorPicker.svelte"
 import { IconSelect } from "./IconSelect"
 import FieldSelect from "./FieldSelect.svelte"
 import MultiFieldSelect from "./MultiFieldSelect.svelte"
+import SearchFieldSelect from "./SearchFieldSelect.svelte"
 import SchemaSelect from "./SchemaSelect.svelte"
 import SectionSelect from "./SectionSelect.svelte"
 import NavigationEditor from "./NavigationEditor/NavigationEditor.svelte"
@@ -21,6 +23,7 @@ const componentMap = {
   text: DrawerBindableCombobox,
   select: Select,
   dataSource: DataSourceSelect,
+  "dataSource/s3": S3DataSourceSelect,
   dataProvider: DataProviderSelect,
   boolean: Checkbox,
   number: Stepper,
@@ -30,6 +33,7 @@ const componentMap = {
   icon: IconSelect,
   field: FieldSelect,
   multifield: MultiFieldSelect,
+  searchfield: SearchFieldSelect,
   options: OptionsEditor,
   schema: SchemaSelect,
   section: SectionSelect,

packages/builder/src/components/start/CreateAppModal.svelte

@@ -1,101 +1,46 @@
 <script>
   import { writable, get as svelteGet } from "svelte/store"
-
   import { notifications, Input, ModalContent, Dropzone } from "@budibase/bbui"
-  import { store, automationStore, hostingStore } from "builderStore"
+  import { store, automationStore } from "builderStore"
-  import { admin, auth } from "stores/portal"
+  import { apps, admin, auth } from "stores/portal"
-  import { string, mixed, object } from "yup"
   import api, { get, post } from "builderStore/api"
   import analytics, { Events } from "analytics"
   import { onMount } from "svelte"
-  import { capitalise } from "helpers"
   import { goto } from "@roxi/routify"
-  import { APP_NAME_REGEX } from "constants"
+  import { createValidationStore } from "helpers/validation/yup"
-  import TemplateList from "./TemplateList.svelte"
+  import * as appValidation from "helpers/validation/yup/app"
 
   export let template
-  export let inline
 
-  const values = writable({ name: null })
+  const values = writable({ name: "", url: null })
-  const errors = writable({})
+  const validation = createValidationStore()
-  const touched = writable({})
+  $: validation.check($values)
-  const validator = {
-    name: string()
-      .trim()
-      .required("Your application must have a name")
-      .matches(
-        APP_NAME_REGEX,
-        "App name must be letters, numbers and spaces only"
-      ),
-    file: template?.fromFile
-      ? mixed().required("Please choose a file to import")
-      : null,
-  }
-
-  let submitting = false
-  let valid = false
-  let initialTemplateInfo = template?.fromFile || template?.key
-
-  $: checkValidity($values, validator)
-  $: showTemplateSelection = !template && !initialTemplateInfo
 
   onMount(async () => {
-    await hostingStore.actions.fetchDeployedApps()
+    await setupValidation()
-    const existingAppNames = svelteGet(hostingStore).deployedAppNames
-    validator.name = string()
-      .trim()
-      .required("Your application must have a name")
-      .matches(APP_NAME_REGEX, "App name must be letters and numbers only")
-      .test(
-        "non-existing-app-name",
-        "Another app with the same name already exists",
-        value => {
-          return !existingAppNames.some(
-            appName => appName.toLowerCase() === value.toLowerCase()
-          )
-        }
-      )
   })
 
-  const checkValidity = async (values, validator) => {
+  const setupValidation = async () => {
-    const obj = object().shape(validator)
+    const applications = svelteGet(apps)
-    Object.keys(validator).forEach(key => ($errors[key] = null))
+    appValidation.name(validation, { apps: applications })
-    if (template?.fromFile && values.file == null) {
+    appValidation.url(validation, { apps: applications })
-      valid = false
+    appValidation.file(validation, { template })
-      return
+    // init validation
-    }
+    validation.check($values)
-
-    try {
-      await obj.validate(values, { abortEarly: false })
-    } catch (validationErrors) {
-      validationErrors.inner.forEach(error => {
-        $errors[error.path] = capitalise(error.message)
-      })
-    }
-
-    valid = await obj.isValid(values)
   }
 
   async function createNewApp() {
-    const templateToUse = Object.keys(template).length === 0 ? null : template
-    submitting = true
-
-    // Check a template exists if we are important
-    if (templateToUse?.fromFile && !$values.file) {
-      $errors.file = "Please choose a file to import"
-      valid = false
-      submitting = false
-      return false
-    }
-
     try {
       // Create form data to create app
       let data = new FormData()
       data.append("name", $values.name.trim())
-      data.append("useTemplate", templateToUse != null)
+      if ($values.url) {
-      if (templateToUse) {
+        data.append("url", $values.url.trim())
-        data.append("templateName", templateToUse.name)
+      }
-        data.append("templateKey", templateToUse.key)
+      data.append("useTemplate", template != null)
+      if (template) {
+        data.append("templateName", template.name)
+        data.append("templateKey", template.key)
         data.append("templateFile", $values.file)
       }
 
@@ -109,7 +54,7 @@
       analytics.captureEvent(Events.APP.CREATED, {
         name: $values.name,
         appId: appJson.instance._id,
-        templateToUse,
+        templateToUse: template,
       })
 
       // Select Correct Application/DB in prep for creating user
@@ -137,68 +82,51 @@
     } catch (error) {
       console.error(error)
       notifications.error(error)
-      submitting = false
     }
   }
 
-  async function onCancel() {
+  // auto add slash to url
-    template = null
+  $: {
-    await auth.setInitInfo({})
+    if ($values.url && !$values.url.startsWith("/")) {
+      $values.url = `/${$values.url}`
+    }
   }
 </script>
 
-{#if showTemplateSelection}
+<ModalContent
-  <ModalContent
+  title={"Create your app"}
-    title={"Get started quickly"}
+  confirmText={template?.fromFile ? "Import app" : "Create app"}
-    showConfirmButton={false}
+  onConfirm={createNewApp}
-    size="L"
+  disabled={!$validation.valid}
-    onConfirm={() => {
+>
-      template = {}
+  {#if template?.fromFile}
-      return false
+    <Dropzone
-    }}
+      error={$validation.touched.file && $validation.errors.file}
-    showCancelButton={!inline}
+      gallery={false}
-    showCloseIcon={!inline}
+      label="File to import"
-  >
+      value={[$values.file]}
-    <TemplateList
+      on:change={e => {
-      onSelect={(selected, { useImport } = {}) => {
+        $values.file = e.detail?.[0]
-        if (!selected) {
+        $validation.touched.file = true
-          template = useImport ? { fromFile: true } : {}
-          return
-        }
-        template = selected
       }}
     />
-  </ModalContent>
+  {/if}
-{:else}
+  <Input
-  <ModalContent
+    bind:value={$values.name}
-    title={"Name your app"}
+    error={$validation.touched.name && $validation.errors.name}
-    confirmText={template?.fromFile ? "Import app" : "Create app"}
+    on:blur={() => ($validation.touched.name = true)}
-    onConfirm={createNewApp}
+    label="Name"
-    onCancel={inline ? onCancel : null}
+    placeholder={$auth.user.firstName
-    cancelText={inline ? "Back" : undefined}
+      ? `${$auth.user.firstName}s app`
-    showCloseIcon={!inline}
+      : "My app"}
-    disabled={!valid}
+  />
-  >
+  <Input
-    {#if template?.fromFile}
+    bind:value={$values.url}
-      <Dropzone
+    error={$validation.touched.url && $validation.errors.url}
-        error={$touched.file && $errors.file}
+    on:blur={() => ($validation.touched.url = true)}
-        gallery={false}
+    label="URL"
-        label="File to import"
+    placeholder={$values.name
-        value={[$values.file]}
+      ? "/" + encodeURIComponent($values.name).toLowerCase()
-        on:change={e => {
+      : "/"}
-          $values.file = e.detail?.[0]
+  />
-          $touched.file = true
+</ModalContent>
-        }}
-      />
-    {/if}
-    <Input
-      bind:value={$values.name}
-      error={$touched.name && $errors.name}
-      on:blur={() => ($touched.name = true)}
-      label="Name"
-      placeholder={$auth.user.firstName
-        ? `${$auth.user.firstName}'s app`
-        : "My app"}
-    />
-  </ModalContent>
-{/if}

packages/builder/src/components/start/TemplateList.svelte

@@ -1,69 +0,0 @@
-<script>
-  import { Heading, Layout, Icon } from "@budibase/bbui"
-
-  export let onSelect
-</script>
-
-<Layout gap="XS" noPadding>
-  <div class="template start-from-scratch" on:click={() => onSelect(null)}>
-    <div
-      class="background-icon"
-      style={`background: rgb(50, 50, 50); color: white;`}
-    >
-      <Icon name="Add" />
-    </div>
-    <Heading size="XS">Start from scratch</Heading>
-    <p class="detail">BLANK</p>
-  </div>
-  <div
-    class="template import"
-    on:click={() => onSelect(null, { useImport: true })}
-  >
-    <div
-      class="background-icon"
-      style={`background: rgb(50, 50, 50); color: white;`}
-    >
-      <Icon name="Add" />
-    </div>
-    <Heading size="XS">Import an app</Heading>
-    <p class="detail">BLANK</p>
-  </div>
-</Layout>
-
-<style>
-  .background-icon {
-    padding: 10px;
-    border-radius: 4px;
-    display: flex;
-    align-items: center;
-    justify-content: center;
-    width: 18px;
-    color: white;
-  }
-
-  .template {
-    min-height: 60px;
-    display: grid;
-    grid-gap: var(--layout-s);
-    grid-template-columns: auto 1fr auto;
-    border: 1px solid #494949;
-    align-items: center;
-    cursor: pointer;
-    border-radius: 4px;
-    background: var(--background-alt);
-    padding: 8px 16px;
-  }
-
-  .detail {
-    text-align: right;
-  }
-
-  .start-from-scratch {
-    background: var(--spectrum-global-color-gray-50);
-    margin-top: 20px;
-  }
-
-  .import {
-    background: var(--spectrum-global-color-gray-50);
-  }
-</style>

packages/builder/src/components/start/UpdateAppModal.svelte

@@ -1,120 +1,75 @@
 <script>
   import { writable, get as svelteGet } from "svelte/store"
-  import {
+  import { notifications, Input, ModalContent, Body } from "@budibase/bbui"
-    notifications,
-    Input,
-    Modal,
-    ModalContent,
-    Body,
-  } from "@budibase/bbui"
-  import { hostingStore } from "builderStore"
   import { apps } from "stores/portal"
-  import { string, object } from "yup"
   import { onMount } from "svelte"
-  import { capitalise } from "helpers"
+  import { createValidationStore } from "helpers/validation/yup"
-  import { APP_NAME_REGEX } from "constants"
+  import * as appValidation from "helpers/validation/yup/app"
-
-  const values = writable({ name: null })
-  const errors = writable({})
-  const touched = writable({})
-  const validator = {
-    name: string()
-      .trim()
-      .required("Your application must have a name")
-      .matches(
-        APP_NAME_REGEX,
-        "App name must be letters, numbers and spaces only"
-      ),
-  }
 
   export let app
 
-  let modal
+  const values = writable({ name: "", url: null })
-  let valid = false
+  const validation = createValidationStore()
-  let dirty = false
+  $: validation.check($values)
-  $: checkValidity($values, validator)
-  $: {
-    // prevent validation by setting name to undefined without an app
-    if (app) {
-      $values.name = app?.name
-    }
-  }
 
   onMount(async () => {
-    await hostingStore.actions.fetchDeployedApps()
+    $values.name = app.name
-    const existingAppNames = svelteGet(hostingStore).deployedAppNames
+    $values.url = app.url
-    validator.name = string()
+    setupValidation()
-      .trim()
-      .required("Your application must have a name")
-      .matches(
-        APP_NAME_REGEX,
-        "App name must be letters, numbers and spaces only"
-      )
-      .test(
-        "non-existing-app-name",
-        "Another app with the same name already exists",
-        value => {
-          return !existingAppNames.some(
-            appName => dirty && appName.toLowerCase() === value.toLowerCase()
-          )
-        }
-      )
   })
 
-  const checkValidity = async (values, validator) => {
+  const setupValidation = async () => {
-    const obj = object().shape(validator)
+    const applications = svelteGet(apps)
-    Object.keys(validator).forEach(key => ($errors[key] = null))
+    appValidation.name(validation, { apps: applications, currentApp: app })
-    try {
+    appValidation.url(validation, { apps: applications, currentApp: app })
-      await obj.validate(values, { abortEarly: false })
+    // init validation
-    } catch (validationErrors) {
+    validation.check($values)
-      validationErrors.inner.forEach(error => {
-        $errors[error.path] = capitalise(error.message)
-      })
-    }
-    valid = await obj.isValid(values)
   }
 
   async function updateApp() {
     try {
       // Update App
-      await apps.update(app.instance._id, { name: $values.name.trim() })
+      const body = {
-      hide()
+        name: $values.name.trim(),
+      }
+      if ($values.url) {
+        body.url = $values.url.trim()
+      }
+      await apps.update(app.instance._id, body)
     } catch (error) {
       console.error(error)
       notifications.error(error)
     }
   }
 
-  export const show = () => {
+  // auto add slash to url
-    modal.show()
+  $: {
-  }
+    if ($values.url && !$values.url.startsWith("/")) {
-  export const hide = () => {
+      $values.url = `/${$values.url}`
-    modal.hide()
+    }
-  }
-
-  const onCancel = () => {
-    hide()
-  }
-
-  const onShow = () => {
-    dirty = false
   }
 </script>
 
-<Modal bind:this={modal} on:hide={onCancel} on:show={onShow}>
+<ModalContent
-  <ModalContent
+  title={"Edit app"}
-    title={"Edit app"}
+  confirmText={"Save"}
-    confirmText={"Save"}
+  onConfirm={updateApp}
-    onConfirm={updateApp}
+  disabled={!$validation.valid}
-    disabled={!(valid && dirty)}
+>
-  >
+  <Body size="S">Update the name of your app.</Body>
-    <Body size="S">Update the name of your app.</Body>
+  <Input
-    <Input
+    bind:value={$values.name}
-      bind:value={$values.name}
+    error={$validation.touched.name && $validation.errors.name}
-      error={$touched.name && $errors.name}
+    on:blur={() => ($validation.touched.name = true)}
-      on:blur={() => ($touched.name = true)}
+    label="Name"
-      on:change={() => (dirty = true)}
+  />
-      label="Name"
+  <Input
-    />
+    bind:value={$values.url}
-  </ModalContent>
+    error={$validation.touched.url && $validation.errors.url}
-</Modal>
+    on:blur={() => ($validation.touched.url = true)}
+    label="URL"
+    placeholder={$values.name
+      ? "/" + encodeURIComponent($values.name).toLowerCase()
+      : "/"}
+  />
+</ModalContent>

packages/builder/src/constants/backend/index.js

@@ -231,3 +231,11 @@ export const PaginationLocations = [
   { label: "Query parameters", value: "query" },
   { label: "Request body", value: "body" },
 ]
+
+export const BannedSearchTypes = [
+  "link",
+  "attachment",
+  "formula",
+  "json",
+  "jsonarray",
+]

packages/builder/src/constants/index.js

@@ -52,4 +52,7 @@ export const LAYOUT_NAMES = {
 
 export const BUDIBASE_INTERNAL_DB = "bb_internal"
 
+// one or more word characters and whitespace
 export const APP_NAME_REGEX = /^[\w\s]+$/
+// zero or more non-whitespace characters
+export const APP_URL_REGEX = /^\S*$/

packages/builder/src/helpers/searchFields.js

@@ -0,0 +1,31 @@
+import { tables } from "../stores/backend"
+import { BannedSearchTypes } from "../constants/backend"
+import { get } from "svelte/store"
+
+export function getTableFields(linkField) {
+  const table = get(tables).list.find(table => table._id === linkField.tableId)
+  if (!table || !table.sql) {
+    return []
+  }
+  const linkFields = getFields(Object.values(table.schema), {
+    allowLinks: false,
+  })
+  return linkFields.map(field => ({
+    ...field,
+    name: `${table.name}.${field.name}`,
+  }))
+}
+
+export function getFields(fields, { allowLinks } = { allowLinks: true }) {
+  let filteredFields = fields.filter(
+    field => !BannedSearchTypes.includes(field.type)
+  )
+  if (allowLinks) {
+    const linkFields = fields.filter(field => field.type === "link")
+    for (let linkField of linkFields) {
+      // only allow one depth of SQL relationship filtering
+      filteredFields = filteredFields.concat(getTableFields(linkField))
+    }
+  }
+  return filteredFields
+}

packages/builder/src/helpers/validation/validation.js

@@ -1,5 +1,7 @@
 import { writable, derived } from "svelte/store"
 
+// DEPRECATED - Use the yup based validators for future validation
+
 export function createValidationStore(initialValue, ...validators) {
   let touched = false
 

packages/builder/src/helpers/validation/validators.js

@@ -1,3 +1,5 @@
+// TODO: Convert to yup based validators
+
 export function emailValidator(value) {
   return (
     (value &&

packages/builder/src/helpers/validation/yup/app.js

@@ -0,0 +1,83 @@
+import { string, mixed } from "yup"
+import { APP_NAME_REGEX, APP_URL_REGEX } from "constants"
+
+export const name = (validation, { apps, currentApp } = { apps: [] }) => {
+  validation.addValidator(
+    "name",
+    string()
+      .trim()
+      .required("Your application must have a name")
+      .matches(
+        APP_NAME_REGEX,
+        "App name must be letters, numbers and spaces only"
+      )
+      .test(
+        "non-existing-app-name",
+        "Another app with the same name already exists",
+        value => {
+          if (!value) {
+            // exit early, above validator will fail
+            return true
+          }
+          if (currentApp) {
+            // filter out the current app if present
+            apps = apps.filter(app => app.appId !== currentApp.appId)
+          }
+          return !apps
+            .map(app => app.name)
+            .some(appName => appName.toLowerCase() === value.toLowerCase())
+        }
+      )
+  )
+}
+
+export const url = (validation, { apps, currentApp } = { apps: [] }) => {
+  validation.addValidator(
+    "url",
+    string()
+      .nullable()
+      .matches(APP_URL_REGEX, "App URL must not contain spaces")
+      .test(
+        "non-existing-app-url",
+        "Another app with the same URL already exists",
+        value => {
+          // url is nullable
+          if (!value) {
+            return true
+          }
+          if (currentApp) {
+            // filter out the current app if present
+            apps = apps.filter(app => app.appId !== currentApp.appId)
+          }
+          return !apps
+            .map(app => app.url)
+            .some(appUrl => appUrl?.toLowerCase() === value.toLowerCase())
+        }
+      )
+      .test("valid-url", "Not a valid URL", value => {
+        // url is nullable
+        if (!value) {
+          return true
+        }
+        // make it clear that this is a url path and cannot be a full url
+        return (
+          value.startsWith("/") &&
+          !value.includes("http") &&
+          !value.includes("www") &&
+          !value.includes(".") &&
+          value.length > 1 // just '/' is not valid
+        )
+      })
+  )
+}
+
+export const file = (validation, { template } = {}) => {
+  const templateToUse =
+    template && Object.keys(template).length === 0 ? null : template
+  validation.addValidator(
+    "file",
+    templateToUse?.fromFile
+      ? mixed().required("Please choose a file to import")
+      : null
+  )
+}

packages/builder/src/helpers/validation/yup/index.js

@@ -0,0 +1,66 @@
+import { capitalise } from "helpers"
+import { object } from "yup"
+import { writable, get } from "svelte/store"
+import { notifications } from "@budibase/bbui"
+
+export const createValidationStore = () => {
+  const DEFAULT = {
+    errors: {},
+    touched: {},
+    valid: false,
+  }
+
+  const validator = {}
+  const validation = writable(DEFAULT)
+
+  const addValidator = (propertyName, propertyValidator) => {
+    if (!propertyValidator || !propertyName) {
+      return
+    }
+    validator[propertyName] = propertyValidator
+  }
+
+  const check = async values => {
+    const obj = object().shape(validator)
+    // clear the previous errors
+    const properties = Object.keys(validator)
+    properties.forEach(property => (get(validation).errors[property] = null))
+
+    let validationError = false
+    try {
+      await obj.validate(values, { abortEarly: false })
+    } catch (error) {
+      if (!error.inner) {
+        notifications.error("Unexpected validation error", error)
+        validationError = true
+      } else {
+        error.inner.forEach(err => {
+          validation.update(store => {
+            store.errors[err.path] = capitalise(err.message)
+            return store
+          })
+        })
+      }
+    }
+
+    let valid
+    if (properties.length && !validationError) {
+      valid = await obj.isValid(values)
+    } else {
+      // don't say valid until validators have been loaded
+      valid = false
+    }
+
+    validation.update(store => {
+      store.valid = valid
+      return store
+    })
+  }
+
+  return {
+    subscribe: validation.subscribe,
+    set: validation.set,
+    check,
+    addValidator,
+  }
+}

packages/builder/src/pages/builder/_layout.svelte

@@ -2,6 +2,12 @@
   import { isActive, redirect, params } from "@roxi/routify"
   import { admin, auth } from "stores/portal"
   import { onMount } from "svelte"
+  import {
+    Cookies,
+    getCookie,
+    removeCookie,
+    setCookie,
+  } from "builderStore/cookies"
 
   let loaded = false
 
@@ -67,6 +73,24 @@
 
   $: {
     const apiReady = $admin.loaded && $auth.loaded
+
+    // firstly, set the return url
+    if (
+      loaded &&
+      apiReady &&
+      !$auth.user &&
+      !getCookie(Cookies.ReturnUrl) &&
+      // logout triggers a page refresh, so we don't want to set the return url
+      !$auth.postLogout &&
+      // don't set the return url on pre-login pages
+      !$isActive("./auth") &&
+      !$isActive("./invite") &&
+      !$isActive("./admin")
+    ) {
+      const url = window.location.pathname
+      setCookie(Cookies.ReturnUrl, url)
+    }
+
     // if tenant is not set go to it
     if (
       loaded &&
@@ -90,13 +114,20 @@
       !$isActive("./invite") &&
       !$isActive("./admin")
     ) {
-      const returnUrl = encodeURIComponent(window.location.pathname)
+      $redirect("./auth")
-      $redirect("./auth?", { returnUrl })
     }
     // check if password reset required for user
     else if ($auth.user?.forceResetPassword) {
       $redirect("./auth/reset")
     }
+    // lastly, redirect to the return url if it has been set
+    else if (loaded && apiReady && $auth.user) {
+      const returnUrl = getCookie(Cookies.ReturnUrl)
+      if (returnUrl) {
+        removeCookie(Cookies.ReturnUrl)
+        window.location.href = returnUrl
+      }
+    }
   }
 </script>
 

packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/index.svelte

@@ -19,8 +19,8 @@
   import { IntegrationTypes } from "constants/backend"
   import { isEqual } from "lodash"
   import { cloneDeep } from "lodash/fp"
-
   import ImportRestQueriesModal from "components/backend/DatasourceNavigator/modals/ImportRestQueriesModal.svelte"
+
   let importQueriesModal
 
   let changed

packages/builder/src/pages/builder/apps/index.svelte

@@ -12,7 +12,7 @@
     Modal,
   } from "@budibase/bbui"
   import { onMount } from "svelte"
-  import { apps, organisation, auth, admin } from "stores/portal"
+  import { apps, organisation, auth } from "stores/portal"
   import { goto } from "@roxi/routify"
   import { AppStatus } from "constants"
   import { gradient } from "actions"
@@ -34,7 +34,6 @@
   const publishedAppsOnly = app => app.status === AppStatus.DEPLOYED
 
   $: publishedApps = $apps.filter(publishedAppsOnly)
-  $: isCloud = $admin.cloud
   $: userApps = $auth.user?.builder?.global
     ? publishedApps
     : publishedApps.filter(app =>
@@ -42,7 +41,11 @@
       )
 
   function getUrl(app) {
-    return !isCloud ? `/app/${encodeURIComponent(app.name)}` : `/${app.prodId}`
+    if (app.url) {
+      return `/app${app.url}`
+    } else {
+      return `/${app.prodId}`
+    }
   }
 </script>
 

packages/builder/src/pages/builder/auth/login.svelte

@@ -10,7 +10,7 @@
     notifications,
     Link,
   } from "@budibase/bbui"
-  import { goto, params } from "@roxi/routify"
+  import { goto } from "@roxi/routify"
   import { auth, organisation, oidc, admin } from "stores/portal"
   import GoogleButton from "./_components/GoogleButton.svelte"
   import OIDCButton from "./_components/OIDCButton.svelte"
@@ -35,12 +35,8 @@
       if ($auth?.user?.forceResetPassword) {
         $goto("./reset")
       } else {
-        if ($params["?returnUrl"]) {
+        notifications.success("Logged in successfully")
-          window.location = decodeURIComponent($params["?returnUrl"])
+        $goto("../portal")
-        } else {
-          notifications.success("Logged in successfully")
-          $goto("../portal")
-        }
       }
     } catch (err) {
       console.error(err)

packages/builder/src/pages/builder/portal/apps/index.svelte

@@ -11,6 +11,7 @@
     notifications,
     Body,
     Search,
+    Icon,
   } from "@budibase/bbui"
   import Spinner from "components/common/Spinner.svelte"
   import CreateAppModal from "components/start/CreateAppModal.svelte"
@@ -27,6 +28,7 @@
   import AppRow from "components/start/AppRow.svelte"
   import { AppStatus } from "constants"
   import analytics, { Events } from "analytics"
+  import Logo from "assets/bb-space-man.svg"
 
   let sortBy = "name"
   let template
@@ -47,7 +49,6 @@
   $: filteredApps = enrichedApps.filter(app =>
     app?.name?.toLowerCase().includes(searchTerm.toLowerCase())
   )
-  $: isCloud = $admin.cloud
 
   const enrichApps = (apps, user, sortBy) => {
     const enrichedApps = apps.map(app => ({
@@ -78,6 +79,7 @@
   }
 
   const initiateAppCreation = () => {
+    template = null
     creationModal.show()
     creatingApp = true
   }
@@ -159,12 +161,10 @@
   }
 
   const viewApp = app => {
-    if (!isCloud && app.deployed) {
+    if (app.url) {
-      // special case to use the short form name if self hosted
+      window.open(`/app${app.url}`)
-      window.open(`/app/${encodeURIComponent(app.name)}`)
     } else {
-      const id = app.deployed ? app.prodId : app.devId
+      window.open(`/${app.prodId}`)
-      window.open(`/${id}`, "_blank")
     }
   }
 
@@ -300,14 +300,26 @@
 
       <div class="buttons">
         {#if cloud}
-          <Button icon="Export" quiet secondary on:click={initiateAppsExport}>
+          <Button
+            size="L"
+            icon="Export"
+            quiet
+            secondary
+            on:click={initiateAppsExport}
+          >
             Export apps
           </Button>
         {/if}
-        <Button icon="Import" quiet secondary on:click={initiateAppImport}>
+        <Button
+          icon="Import"
+          size="L"
+          quiet
+          secondary
+          on:click={initiateAppImport}
+        >
           Import app
         </Button>
-        <Button icon="Add" cta on:click={initiateAppCreation}>
+        <Button size="L" icon="Add" cta on:click={initiateAppCreation}>
           Create app
         </Button>
       </div>
@@ -389,9 +401,24 @@
 
     {#if !enrichedApps.length && !creatingApp && loaded}
       <div class="empty-wrapper">
-        <Modal inline>
+        <div class="centered">
-          <CreateAppModal {template} inline={true} />
+          <div class="main">
-        </Modal>
+            <Layout gap="S" justifyItems="center">
+              <img class="img-size" alt="logo" src={Logo} />
+              <div class="new-screen-text">
+                <Detail size="M">Create a business app in minutes!</Detail>
+              </div>
+              <Button on:click={() => initiateAppCreation()} size="M" cta>
+                <div class="new-screen-button">
+                  <div class="background-icon" style="color: white;">
+                    <Icon name="Add" />
+                  </div>
+                  Create App
+                </div></Button
+              >
+            </Layout>
+          </div>
+        </div>
       </div>
     {/if}
 
@@ -412,6 +439,11 @@
 >
   <CreateAppModal {template} />
 </Modal>
+
+<Modal bind:this={updatingModal} padding={false} width="600px">
+  <UpdateAppModal app={selectedApp} />
+</Modal>
+
 <ConfirmDialog
   bind:this={deletionModal}
   title="Confirm deletion"
@@ -438,7 +470,6 @@
   Are you sure you want to unpublish the app <b>{selectedApp?.name}</b>?
 </ConfirmDialog>
 
-<UpdateAppModal app={selectedApp} bind:this={updatingModal} />
 <ChooseIconModal app={selectedApp} bind:this={iconModal} />
 
 <style>
@@ -474,12 +505,15 @@
   }
 
   .grid {
+    height: 200px;
     display: grid;
+    overflow: hidden;
     grid-gap: var(--spacing-xl);
     grid-template-columns: repeat(auto-fill, minmax(270px, 1fr));
+    grid-template-rows: minmax(70px, 1fr) minmax(100px, 1fr) minmax(0px, 0);
   }
   .template-card {
-    height: 80px;
+    height: 70px;
     border-radius: var(--border-radius-s);
     border: 1px solid var(--spectrum-global-color-gray-300);
     cursor: pointer;
@@ -533,4 +567,42 @@
     justify-content: center;
     align-items: center;
   }
+
+  .centered {
+    width: calc(100% - 350px);
+    height: calc(100% - 100px);
+    display: flex;
+    flex-direction: column;
+    justify-content: center;
+    align-items: center;
+  }
+
+  .main {
+    width: 300px;
+  }
+
+  .new-screen-text {
+    width: 160px;
+    text-align: center;
+    color: #2c2c2c;
+    font-weight: 600;
+  }
+
+  .new-screen-button {
+    margin-left: 5px;
+    height: 20px;
+    width: 100px;
+    display: flex;
+    align-items: center;
+  }
+
+  .img-size {
+    width: 160px;
+    height: 160px;
+  }
+
+  .background-icon {
+    margin-top: 4px;
+    margin-right: 4px;
+  }
 </style>

packages/builder/src/stores/portal/auth.js

@@ -9,6 +9,7 @@ export function createAuthStore() {
     tenantId: "default",
     tenantSet: false,
     loaded: false,
+    postLogout: false,
   })
   const store = derived(auth, $store => {
     let initials = null
@@ -34,6 +35,7 @@ export function createAuthStore() {
       tenantId: $store.tenantId,
       tenantSet: $store.tenantSet,
       loaded: $store.loaded,
+      postLogout: $store.postLogout,
       initials,
       isAdmin,
       isBuilder,
@@ -89,6 +91,13 @@ export function createAuthStore() {
     return info
   }
 
+  async function setPostLogout() {
+    auth.update(store => {
+      store.postLogout = true
+      return store
+    })
+  }
+
   async function getInitInfo() {
     const response = await api.get(`/api/global/auth/init`)
     const json = response.json()
@@ -145,6 +154,7 @@ export function createAuthStore() {
       await response.json()
       await setInitInfo({})
       setUser(null)
+      setPostLogout()
     },
     updateSelf: async fields => {
       const newUser = { ...get(auth).user, ...fields }

packages/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/cli",
-  "version": "1.0.47",
+  "version": "1.0.46-alpha.5",
   "description": "Budibase CLI, for developers, self hosting and migrations.",
   "main": "src/index.js",
   "bin": {

packages/client/manifest.json

@@ -2065,6 +2065,26 @@
           }
         ]
       },
+      {
+        "type": "select",
+        "label": "Direction",
+        "key": "direction",
+        "defaultValue": "vertical",
+        "options": [
+          {
+            "label": "Horizontal",
+            "value": "horizontal"
+          },
+          {
+            "label": "Vertical",
+            "value": "vertical"
+          }
+        ],
+        "dependsOn": {
+          "setting": "optionsType",
+          "value": "radio"
+        }
+      },
       {
         "type": "text",
         "label": "Default value",
@@ -2419,6 +2439,11 @@
         "label": "Label",
         "key": "label"
       },
+      {
+        "type": "text",
+        "label": "Extensions",
+        "key": "extensions"
+      },
       {
         "type": "boolean",
         "label": "Disabled",
@@ -2811,7 +2836,7 @@
         "key": "dataSource"
       },
       {
-        "type": "multifield",
+        "type": "searchfield",
         "label": "Search Columns",
         "key": "searchColumns",
         "placeholder": "Choose search columns"
@@ -2958,7 +2983,7 @@
         "key": "dataSource"
       },
       {
-        "type": "multifield",
+        "type": "searchfield",
         "label": "Search Columns",
         "key": "searchColumns",
         "placeholder": "Choose search columns"
@@ -3315,5 +3340,50 @@
         "suffix": "repeater"
       }
     ]
+  },
+  "s3upload": {
+    "name": "S3 File Upload",
+    "info": "This component can't be used with S3 datasources that use custom endpoints.",
+    "icon": "UploadToCloud",
+    "styles": ["size"],
+    "editable": true,
+    "settings": [
+      {
+        "type": "field/attachment",
+        "label": "Field",
+        "key": "field"
+      },
+      {
+        "type": "text",
+        "label": "Label",
+        "key": "label"
+      },
+      {
+        "type": "dataSource/s3",
+        "label": "S3 Datasource",
+        "key": "datasourceId"
+      },
+      {
+        "type": "text",
+        "label": "Bucket",
+        "key": "bucket"
+      },
+      {
+        "type": "text",
+        "label": "File Name",
+        "key": "key"
+      },
+      {
+        "type": "boolean",
+        "label": "Disabled",
+        "key": "disabled",
+        "defaultValue": false
+      },
+      {
+        "type": "validation/attachment",
+        "label": "Validation",
+        "key": "validation"
+      }
+    ]
   }
 }

packages/client/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/client",
-  "version": "1.0.47",
+  "version": "1.0.46-alpha.5",
   "license": "MPL-2.0",
   "module": "dist/budibase-client.js",
   "main": "dist/budibase-client.js",
@@ -19,10 +19,11 @@
     "dev:builder": "rollup -cw"
   },
   "dependencies": {
-    "@budibase/bbui": "^1.0.47",
+    "@budibase/bbui": "^1.0.46-alpha.5",
     "@budibase/standard-components": "^0.9.139",
-    "@budibase/string-templates": "^1.0.47",
+    "@budibase/string-templates": "^1.0.46-alpha.5",
     "regexparam": "^1.3.0",
+    "rollup-plugin-polyfill-node": "^0.8.0",
     "shortid": "^2.2.15",
     "svelte-spa-router": "^3.0.5"
   },
@@ -45,8 +46,6 @@
     "postcss": "^8.2.10",
     "rollup": "^2.44.0",
     "rollup-plugin-json": "^4.0.0",
-    "rollup-plugin-node-builtins": "^2.1.2",
-    "rollup-plugin-node-globals": "^1.4.0",
     "rollup-plugin-postcss": "^4.0.0",
     "rollup-plugin-svelte": "^7.1.0",
     "rollup-plugin-svg": "^2.0.0",

packages/client/rollup.config.js

@@ -6,8 +6,7 @@ import { terser } from "rollup-plugin-terser"
 import postcss from "rollup-plugin-postcss"
 import svg from "rollup-plugin-svg"
 import json from "rollup-plugin-json"
-import builtins from "rollup-plugin-node-builtins"
+import nodePolyfills from "rollup-plugin-polyfill-node"
-import globals from "rollup-plugin-node-globals"
 import path from "path"
 
 const production = !process.env.ROLLUP_WATCH
@@ -75,8 +74,7 @@ export default {
     }),
     postcss(),
     commonjs(),
-    globals(),
+    nodePolyfills(),
-    builtins(),
     resolve({
       preferBuiltins: true,
       browser: true,

packages/client/src/api/api.js

@@ -36,7 +36,11 @@ const makeApiCall = async ({ method, url, body, json = true }) => {
     })
     switch (response.status) {
       case 200:
-        return response.json()
+        try {
+          return await response.json()
+        } catch (error) {
+          return null
+        }
       case 401:
         notificationStore.actions.error("Invalid credentials")
         return handleError(`Invalid credentials`)
@@ -82,14 +86,15 @@ const makeCachedApiCall = async params => {
  * Constructs an API call function for a particular HTTP method.
  */
 const requestApiCall = method => async params => {
-  const { url, cache = false } = params
+  const { external = false, url, cache = false } = params
-  const fixedUrl = `/${url}`.replace("//", "/")
+  const fixedUrl = external ? url : `/${url}`.replace("//", "/")
   const enrichedParams = { ...params, method, url: fixedUrl }
   return await (cache ? makeCachedApiCall : makeApiCall)(enrichedParams)
 }
 
 export default {
   post: requestApiCall("POST"),
+  put: requestApiCall("PUT"),
   get: requestApiCall("GET"),
   patch: requestApiCall("PATCH"),
   del: requestApiCall("DELETE"),

packages/client/src/api/attachments.js

@@ -10,3 +10,41 @@ export const uploadAttachment = async (data, tableId = "") => {
     json: false,
   })
 }
+
+/**
+ * Generates a signed URL to upload a file to an external datasource.
+ */
+export const getSignedDatasourceURL = async (datasourceId, bucket, key) => {
+  if (!datasourceId) {
+    return null
+  }
+  const res = await API.post({
+    url: `/api/attachments/${datasourceId}/url`,
+    body: { bucket, key },
+  })
+  if (res.error) {
+    throw "Could not generate signed upload URL"
+  }
+  return res
+}
+
+/**
+ * Uploads a file to an external datasource.
+ */
+export const externalUpload = async (datasourceId, bucket, key, data) => {
+  const { signedUrl, publicUrl } = await getSignedDatasourceURL(
+    datasourceId,
+    bucket,
+    key
+  )
+  const res = await API.put({
+    url: signedUrl,
+    body: data,
+    json: false,
+    external: true,
+  })
+  if (res?.error) {
+    throw "Could not upload file to signed URL"
+  }
+  return { publicUrl }
+}

packages/client/src/api/auth.js

@@ -18,6 +18,15 @@ export const logIn = async ({ email, password }) => {
   })
 }
 
+/**
+ * Logs the user out and invaidates their session.
+ */
+export const logOut = async () => {
+  return await API.post({
+    url: "/api/global/auth/logout",
+  })
+}
+
 /**
  * Fetches the currently logged in user object
  */

packages/client/src/components/ClientApp.svelte

@@ -63,8 +63,9 @@
       } else {
         // The user is not logged in, redirect them to login
         const returnUrl = `${window.location.pathname}${window.location.hash}`
-        const encodedUrl = encodeURIComponent(returnUrl)
+        // TODO: reuse `Cookies` from builder when frontend-core is added
-        window.location = `/builder/auth/login?returnUrl=${encodedUrl}`
+        window.document.cookie = `budibase:returnurl=${returnUrl}; Path=/`
+        window.location = `/builder/auth/login`
       }
     }
   }

packages/client/src/components/app/DataProvider.svelte

@@ -67,6 +67,7 @@
   $: dataContext = {
     rows: $fetch.rows,
     info: $fetch.info,
+    datasource: dataSource || {},
     schema: $fetch.schema,
     rowsLength: $fetch.rows.length,
 

packages/client/src/components/app/blocks/CardsBlock.svelte

@@ -71,12 +71,13 @@
   const enrichFilter = (filter, columns, formId) => {
     let enrichedFilter = [...(filter || [])]
     columns?.forEach(column => {
+      const safePath = column.name.split(".").map(safe).join(".")
       enrichedFilter.push({
         field: column.name,
         operator: column.type === "string" ? "string" : "equal",
         type: column.type === "string" ? "string" : "number",
         valueType: "Binding",
-        value: `{{ [${formId}].[${column.name}] }}`,
+        value: `{{ ${safe(formId)}.${safePath} }}`,
       })
     })
     return enrichedFilter
@@ -112,7 +113,9 @@
   // Load the datasource schema so we can determine column types
   const fetchSchema = async dataSource => {
     if (dataSource) {
-      schema = await fetchDatasourceSchema(dataSource)
+      schema = await fetchDatasourceSchema(dataSource, {
+        enrichRelationships: true,
+      })
     }
     schemaLoaded = true
   }

packages/client/src/components/app/blocks/TableBlock.svelte

@@ -59,12 +59,13 @@
   const enrichFilter = (filter, columns, formId) => {
     let enrichedFilter = [...(filter || [])]
     columns?.forEach(column => {
+      const safePath = column.name.split(".").map(safe).join(".")
       enrichedFilter.push({
         field: column.name,
         operator: column.type === "string" ? "string" : "equal",
         type: column.type === "string" ? "string" : "number",
         valueType: "Binding",
-        value: `{{ ${safe(formId)}.${safe(column.name)} }}`,
+        value: `{{ ${safe(formId)}.${safePath} }}`,
       })
     })
     return enrichedFilter
@@ -90,7 +91,9 @@
   // Load the datasource schema so we can determine column types
   const fetchSchema = async dataSource => {
     if (dataSource) {
-      schema = await fetchDatasourceSchema(dataSource)
+      schema = await fetchDatasourceSchema(dataSource, {
+        enrichRelationships: true,
+      })
     }
     schemaLoaded = true
   }

packages/client/src/components/app/dynamic-filter/DynamicFilter.svelte

@@ -11,11 +11,14 @@
   export let size = "M"
 
   const component = getContext("component")
-  const { builderStore, ActionTypes, getAction } = getContext("sdk")
+  const { builderStore, ActionTypes, getAction, fetchDatasourceSchema } =
+    getContext("sdk")
 
   let modal
   let tmpFilters = []
   let filters = []
+  let schemaLoaded = false,
+    schema
 
   $: dataProviderId = dataProvider?.id
   $: addExtension = getAction(
@@ -26,7 +29,7 @@
     dataProviderId,
     ActionTypes.RemoveDataProviderQueryExtension
   )
-  $: schema = dataProvider?.schema
+  $: fetchSchema(dataProvider || {})
   $: schemaFields = getSchemaFields(schema, allowedFields)
 
   // Add query extension to data provider
@@ -39,7 +42,20 @@
     }
   }
 
-  const getSchemaFields = (schema, allowedFields) => {
+  async function fetchSchema(dataProvider) {
+    const datasource = dataProvider?.datasource
+    if (datasource) {
+      schema = await fetchDatasourceSchema(datasource, {
+        enrichRelationships: true,
+      })
+    }
+    schemaLoaded = true
+  }
+
+  function getSchemaFields(schema, allowedFields) {
+    if (!schemaLoaded) {
+      return {}
+    }
     let clonedSchema = {}
     if (!allowedFields?.length) {
       clonedSchema = schema
@@ -68,18 +84,20 @@
   })
 </script>
 
-<Button
+{#if schemaLoaded}
-  onClick={openEditor}
+  <Button
-  icon="Properties"
+    onClick={openEditor}
-  text="Filter"
+    icon="Properties"
-  {size}
+    text="Filter"
-  type="secondary"
+    {size}
-  quiet
+    type="secondary"
-  active={filters?.length > 0}
+    quiet
-/>
+    active={filters?.length > 0}
+  />
 
-<Modal bind:this={modal}>
+  <Modal bind:this={modal}>
-  <ModalContent title="Edit filters" size="XL" onConfirm={updateQuery}>
+    <ModalContent title="Edit filters" size="XL" onConfirm={updateQuery}>
-    <FilterModal bind:filters={tmpFilters} {schemaFields} />
+      <FilterModal bind:filters={tmpFilters} {schemaFields} />
-  </ModalContent>
+    </ModalContent>
-</Modal>
+  </Modal>
+{/if}

packages/client/src/components/app/forms/AttachmentField.svelte

@@ -7,6 +7,7 @@
   export let label
   export let disabled = false
   export let validation
+  export let extensions
 
   let fieldState
   let fieldApi
@@ -52,6 +53,7 @@
       }}
       {processFiles}
       {handleFileTooLarge}
+      {extensions}
     />
   {/if}
 </Field>

packages/client/src/components/app/forms/InnerForm.svelte

@@ -141,8 +141,18 @@
         return
       }
 
+      // Create validation function based on field schema
+      const schemaConstraints = schema?.[field]?.constraints
+      const validator = createValidatorFromConstraints(
+        schemaConstraints,
+        validationRules,
+        field,
+        table
+      )
+
       // If we've already registered this field then keep some existing state
       let initialValue = deepGet(initialValues, field) ?? defaultValue
+      let initialError = null
       let fieldId = `id-${generateID()}`
       const existingField = getField(field)
       if (existingField) {
@@ -156,20 +166,17 @@
         } else {
           initialValue = fieldState.value ?? initialValue
         }
+
+        // If this field has already been registered and we previously had an
+        // error set, then re-run the validator to see if we can unset it
+        if (fieldState.error) {
+          initialError = validator(initialValue)
+        }
       }
 
       // Auto columns are always disabled
       const isAutoColumn = !!schema?.[field]?.autocolumn
 
-      // Create validation function based on field schema
-      const schemaConstraints = schema?.[field]?.constraints
-      const validator = createValidatorFromConstraints(
-        schemaConstraints,
-        validationRules,
-        field,
-        table
-      )
-
       // Construct field info
       const fieldInfo = writable({
         name: field,
@@ -178,7 +185,7 @@
         fieldState: {
           fieldId,
           value: initialValue,
-          error: null,
+          error: initialError,
           disabled: disabled || fieldDisabled || isAutoColumn,
           defaultValue,
           validator,

packages/client/src/components/app/forms/MultiFieldSelect.svelte

@@ -20,6 +20,7 @@
   let fieldSchema
 
   $: flatOptions = optionsSource == null || optionsSource === "schema"
+  $: expandedDefaultValue = expand(defaultValue)
   $: options = getOptions(
     optionsSource,
     fieldSchema,
@@ -28,6 +29,18 @@
     valueColumn,
     customOptions
   )
+
+  const expand = values => {
+    if (!values) {
+      return []
+    }
+
+    if (Array.isArray(values)) {
+      return values
+    }
+
+    return values.split(",").map(value => value.trim())
+  }
 </script>
 
 <Field
@@ -35,7 +48,7 @@
   {label}
   {disabled}
   {validation}
-  {defaultValue}
+  defaultValue={expandedDefaultValue}
   type="array"
   bind:fieldState
   bind:fieldApi

packages/client/src/components/app/forms/OptionsField.svelte

@@ -15,6 +15,7 @@
   export let valueColumn
   export let customOptions
   export let autocomplete = false
+  export let direction = "vertical"
 
   let fieldState
   let fieldApi
@@ -64,6 +65,7 @@
         disabled={fieldState.disabled}
         error={fieldState.error}
         {options}
+        {direction}
         on:change={e => fieldApi.setValue(e.detail)}
         getOptionLabel={flatOptions ? x => x : x => x.label}
         getOptionValue={flatOptions ? x => x : x => x.value}

packages/client/src/components/app/forms/S3Upload.svelte

@@ -0,0 +1,143 @@
+<script>
+  import Field from "./Field.svelte"
+  import { CoreDropzone, ProgressCircle } from "@budibase/bbui"
+  import { getContext, onMount, onDestroy } from "svelte"
+
+  export let datasourceId
+  export let bucket
+  export let key
+  export let field
+  export let label
+  export let disabled = false
+  export let validation
+
+  let fieldState
+  let fieldApi
+
+  const { API, notificationStore, uploadStore } = getContext("sdk")
+  const component = getContext("component")
+
+  // 5GB cap per item sent via S3 REST API
+  const MaxFileSize = 1000000000 * 5
+
+  // Actual file data to upload
+  let data
+  let loading = false
+
+  const handleFileTooLarge = () => {
+    notificationStore.actions.warning(
+      "Files cannot exceed 5GB. Please try again with a smaller file."
+    )
+  }
+
+  // Process the file input and return a serializable structure expected by
+  // the dropzone component to display the file
+  const processFiles = async fileList => {
+    return await new Promise(resolve => {
+      if (!fileList?.length) {
+        return []
+      }
+
+      // Don't read in non-image files
+      data = fileList[0]
+      if (!data.type?.startsWith("image")) {
+        resolve([
+          {
+            name: data.name,
+            type: data.type,
+          },
+        ])
+      }
+
+      // Read image files and display as preview
+      const reader = new FileReader()
+      reader.addEventListener(
+        "load",
+        () => {
+          resolve([
+            {
+              url: reader.result,
+              name: data.name,
+              type: data.type,
+            },
+          ])
+        },
+        false
+      )
+      reader.readAsDataURL(fileList[0])
+    })
+  }
+
+  const upload = async () => {
+    loading = true
+    try {
+      const res = await API.externalUpload(datasourceId, bucket, key, data)
+      notificationStore.actions.success("File uploaded successfully")
+      loading = false
+      return res
+    } catch (error) {
+      notificationStore.actions.error(`Error uploading file: ${error}`)
+    }
+  }
+
+  onMount(() => {
+    uploadStore.actions.registerFileUpload($component.id, upload)
+  })
+
+  onDestroy(() => {
+    uploadStore.actions.unregisterFileUpload($component.id)
+  })
+</script>
+
+<Field
+  {label}
+  {field}
+  {disabled}
+  {validation}
+  type="s3upload"
+  bind:fieldState
+  bind:fieldApi
+  defaultValue={[]}
+>
+  <div class="content">
+    {#if fieldState}
+      <CoreDropzone
+        value={fieldState.value}
+        disabled={loading || fieldState.disabled}
+        error={fieldState.error}
+        on:change={e => {
+          fieldApi.setValue(e.detail)
+        }}
+        {processFiles}
+        {handleFileTooLarge}
+        maximum={1}
+        fileSizeLimit={MaxFileSize}
+      />
+    {/if}
+    {#if loading}
+      <div class="overlay" />
+      <div class="loading">
+        <ProgressCircle />
+      </div>
+    {/if}
+  </div>
+</Field>
+
+<style>
+  .content {
+    position: relative;
+  }
+  .overlay,
+  .loading {
+    position: absolute;
+    top: 0;
+    height: 100%;
+    width: 100%;
+    display: grid;
+    place-items: center;
+  }
+  .overlay {
+    background-color: var(--spectrum-global-color-gray-50);
+    opacity: 0.5;
+  }
+</style>

packages/client/src/components/app/forms/index.js

@@ -12,3 +12,4 @@ export { default as relationshipfield } from "./RelationshipField.svelte"
 export { default as passwordfield } from "./PasswordField.svelte"
 export { default as formstep } from "./FormStep.svelte"
 export { default as jsonfield } from "./JSONField.svelte"
+export { default as s3upload } from "./S3Upload.svelte"

packages/client/src/components/app/forms/validation.js

@@ -37,7 +37,7 @@ export const createValidatorFromConstraints = (
       const length = schemaConstraints.length.maximum
       rules.push({
         type: "string",
-        constraint: "length",
+        constraint: "maxLength",
         value: length,
         error: `Maximum length is ${length}`,
       })

packages/client/src/sdk.js

@@ -5,6 +5,7 @@ import {
   routeStore,
   screenStore,
   builderStore,
+  uploadStore,
 } from "stores"
 import { styleable } from "utils/styleable"
 import { linkable } from "utils/linkable"
@@ -20,6 +21,7 @@ export default {
   routeStore,
   screenStore,
   builderStore,
+  uploadStore,
   styleable,
   linkable,
   getAction,

packages/client/src/stores/auth.js

@@ -11,8 +11,14 @@ const createAuthStore = () => {
   }
 
   const logOut = async () => {
+    try {
+      await API.logOut()
+    } catch (error) {
+      // Do nothing
+    }
+
+    // Manually destroy cookie to be sure
     window.document.cookie = `budibase:auth=; budibase:currentapp=; Path=/; Expires=Thu, 01 Jan 1970 00:00:01 GMT;`
-    window.location = "/builder/auth/login"
   }
 
   return {

packages/client/src/stores/index.js

@@ -9,6 +9,7 @@ export { confirmationStore } from "./confirmation"
 export { peekStore } from "./peek"
 export { stateStore } from "./state"
 export { themeStore } from "./theme"
+export { uploadStore } from "./uploads.js"
 
 // Context stores are layered and duplicated, so it is not a singleton
 export { createContextStore } from "./context"

packages/client/src/stores/routes.js

@@ -18,8 +18,8 @@ const createRouteStore = () => {
   const fetchRoutes = async () => {
     const routeConfig = await API.fetchRoutes()
     let routes = []
-    Object.values(routeConfig.routes).forEach(route => {
+    Object.values(routeConfig.routes || {}).forEach(route => {
-      Object.entries(route.subpaths).forEach(([path, config]) => {
+      Object.entries(route.subpaths || {}).forEach(([path, config]) => {
         routes.push({
           path,
           screenId: config.screenId,
@@ -83,12 +83,23 @@ const createRouteStore = () => {
   const setRouterLoaded = () => {
     store.update(state => ({ ...state, routerLoaded: true }))
   }
+  const createFullURL = relativeURL => {
+    if (!relativeURL?.startsWith("/")) {
+      return relativeURL
+    }
+    if (!window.location.href.includes("#")) {
+      return `${window.location.href}#${relativeURL}`
+    }
+    const base = window.location.href.split("#")[0]
+    return `${base}#${relativeURL}`
+  }
 
   return {
     subscribe: store.subscribe,
     actions: {
       fetchRoutes,
       navigate,
+      createFullURL,
       setRouteParams,
       setQueryParams,
       setActiveRoute,

packages/client/src/stores/uploads.js

@@ -0,0 +1,42 @@
+import { writable, get } from "svelte/store"
+
+export const createUploadStore = () => {
+  const store = writable([])
+
+  // Registers a new file upload component
+  const registerFileUpload = (componentId, callback) => {
+    if (!componentId || !callback) {
+      return
+    }
+
+    store.update(state => {
+      state.push({
+        componentId,
+        callback,
+      })
+      return state
+    })
+  }
+
+  // Unregisters a file upload component
+  const unregisterFileUpload = componentId => {
+    store.update(state => state.filter(c => c.componentId !== componentId))
+  }
+
+  // Processes a file upload for a given component ID
+  const processFileUpload = async componentId => {
+    if (!componentId) {
+      return
+    }
+
+    const component = get(store).find(c => c.componentId === componentId)
+    return await component?.callback()
+  }
+
+  return {
+    subscribe: store.subscribe,
+    actions: { registerFileUpload, unregisterFileUpload, processFileUpload },
+  }
+}
+
+export const uploadStore = createUploadStore()

packages/client/src/utils/buttonActions.js

@@ -5,6 +5,7 @@ import {
   confirmationStore,
   authStore,
   stateStore,
+  uploadStore,
 } from "stores"
 import { saveRow, deleteRow, executeQuery, triggerAutomation } from "api"
 import { ActionTypes } from "constants"
@@ -112,8 +113,20 @@ const refreshDataProviderHandler = async (action, context) => {
   )
 }
 
-const logoutHandler = async () => {
+const logoutHandler = async action => {
   await authStore.actions.logOut()
+  let redirectUrl = "/builder/auth/login"
+  let internal = false
+  if (action.parameters.redirectUrl) {
+    internal = action.parameters.redirectUrl?.startsWith("/")
+    redirectUrl = routeStore.actions.createFullURL(
+      action.parameters.redirectUrl
+    )
+  }
+  window.location.href = redirectUrl
+  if (internal) {
+    window.location.reload()
+  }
 }
 
 const clearFormHandler = async (action, context) => {
@@ -157,6 +170,17 @@ const updateStateHandler = action => {
   }
 }
 
+const s3UploadHandler = async action => {
+  const { componentId } = action.parameters
+  if (!componentId) {
+    return
+  }
+  const res = await uploadStore.actions.processFileUpload(componentId)
+  return {
+    publicUrl: res?.publicUrl,
+  }
+}
+
 const handlerMap = {
   ["Save Row"]: saveRowHandler,
   ["Duplicate Row"]: duplicateRowHandler,
@@ -171,6 +195,7 @@ const handlerMap = {
   ["Close Screen Modal"]: closeScreenModalHandler,
   ["Change Form Step"]: changeFormStepHandler,
   ["Update State"]: updateStateHandler,
+  ["Upload File to S3"]: s3UploadHandler,
 }
 
 const confirmTextMap = {

packages/client/src/utils/fetch/DataFetch.js

@@ -67,7 +67,6 @@ export default class DataFetch {
     this.getPage = this.getPage.bind(this)
     this.getInitialData = this.getInitialData.bind(this)
     this.determineFeatureFlags = this.determineFeatureFlags.bind(this)
-    this.enrichSchema = this.enrichSchema.bind(this)
     this.refresh = this.refresh.bind(this)
     this.update = this.update.bind(this)
     this.hasNextPage = this.hasNextPage.bind(this)
@@ -123,7 +122,7 @@ export default class DataFetch {
 
     // Fetch and enrich schema
     let schema = this.constructor.getSchema(datasource, definition)
-    schema = this.enrichSchema(schema)
+    schema = DataFetch.enrichSchema(schema)
     if (!schema) {
       return
     }
@@ -242,7 +241,7 @@ export default class DataFetch {
    * @param schema the datasource schema
    * @return {object} the enriched datasource schema
    */
-  enrichSchema(schema) {
+  static enrichSchema(schema) {
     if (schema == null) {
       return null
     }

packages/client/src/utils/schema.js

@@ -6,13 +6,19 @@ import RelationshipFetch from "./fetch/RelationshipFetch.js"
 import NestedProviderFetch from "./fetch/NestedProviderFetch.js"
 import FieldFetch from "./fetch/FieldFetch.js"
 import JSONArrayFetch from "./fetch/JSONArrayFetch.js"
+import DataFetch from "./fetch/DataFetch.js"
 
 /**
  * Fetches the schema of any kind of datasource.
  * All datasource fetch classes implement their own functionality to get the
  * schema of a datasource of their respective types.
+ * @param datasource the datasource to fetch the schema for
+ * @param options options for enriching the schema
  */
-export const fetchDatasourceSchema = async datasource => {
+export const fetchDatasourceSchema = async (
+  datasource,
+  options = { enrichRelationships: false }
+) => {
   const handler = {
     table: TableFetch,
     view: ViewFetch,
@@ -28,7 +34,7 @@ export const fetchDatasourceSchema = async datasource => {
 
   // Get the datasource definition and then schema
   const definition = await handler.getDefinition(datasource)
-  const schema = handler.getSchema(datasource, definition)
+  let schema = handler.getSchema(datasource, definition)
   if (!schema) {
     return null
   }
@@ -49,5 +55,28 @@ export const fetchDatasourceSchema = async datasource => {
       })
     }
   })
-  return { ...schema, ...jsonAdditions }
+  schema = { ...schema, ...jsonAdditions }
+
+  // Check for any relationship fields if required
+  if (options?.enrichRelationships && definition.sql) {
+    let relationshipAdditions = {}
+    for (let fieldKey of Object.keys(schema)) {
+      const fieldSchema = schema[fieldKey]
+      if (fieldSchema?.type === "link") {
+        const linkSchema = await fetchDatasourceSchema({
+          type: "table",
+          tableId: fieldSchema?.tableId,
+        })
+        Object.keys(linkSchema || {}).forEach(linkKey => {
+          relationshipAdditions[`${fieldKey}.${linkKey}`] = {
+            type: linkSchema[linkKey].type,
+          }
+        })
+      }
+    }
+    schema = { ...schema, ...relationshipAdditions }
+  }
+
+  // Ensure schema structure is correct
+  return DataFetch.enrichSchema(schema)
 }

packages/server/nodemon.json

@@ -1,5 +1,5 @@
 {
-  "watch": ["src", "../auth"],
+  "watch": ["src", "../backend-core"],
   "ext": "js,ts,json",
   "ignore": ["src/**/*.spec.ts", "src/**/*.spec.js"],
   "exec": "ts-node src/index.ts"

packages/server/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "1.0.47",
+  "version": "1.0.46-alpha.5",
   "description": "Budibase Web Server",
   "main": "src/index.ts",
   "repository": {
@@ -70,9 +70,9 @@
   "license": "GPL-3.0",
   "dependencies": {
     "@apidevtools/swagger-parser": "^10.0.3",
-    "@budibase/backend-core": "^1.0.47",
+    "@budibase/backend-core": "^1.0.46-alpha.5",
-    "@budibase/client": "^1.0.47",
+    "@budibase/client": "^1.0.46-alpha.5",
-    "@budibase/string-templates": "^1.0.47",
+    "@budibase/string-templates": "^1.0.46-alpha.5",
     "@bull-board/api": "^3.7.0",
     "@bull-board/koa": "^3.7.0",
     "@elastic/elasticsearch": "7.10.0",
@@ -112,7 +112,7 @@
     "mongodb": "3.6.3",
     "mssql": "6.2.3",
     "mysql2": "^2.3.1",
-    "node-fetch": "2.6.0",
+    "node-fetch": "2.6.7",
     "open": "^8.4.0",
     "pg": "8.5.1",
     "pino-pretty": "4.0.0",

packages/server/src/api/controllers/application.js

@@ -33,10 +33,7 @@ const {
   Replication,
 } = require("@budibase/backend-core/db")
 const { USERS_TABLE_SCHEMA } = require("../../constants")
-const {
+const { removeAppFromUserRoles } = require("../../utilities/workerRequests")
-  getDeployedApps,
-  removeAppFromUserRoles,
-} = require("../../utilities/workerRequests")
 const { clientLibraryPath, stringToReadStream } = require("../../utilities")
 const { getAllLocks } = require("../../utilities/redis")
 const {
@@ -78,31 +75,43 @@ function getUserRoleId(ctx) {
     : ctx.user.role._id
 }
 
-async function getAppUrlIfNotInUse(ctx) {
+async function getAppUrl(ctx) {
+  // construct the url
   let url
   if (ctx.request.body.url) {
+    // if the url is provided, use that
     url = encodeURI(ctx.request.body.url)
-  } else if (ctx.request.body.name) {
+  } else {
+    // otherwise use the name
     url = encodeURI(`${ctx.request.body.name}`)
   }
-  if (url) {
+  url = `/${url.replace(URL_REGEX_SLASH, "")}`.toLowerCase()
-    url = `/${url.replace(URL_REGEX_SLASH, "")}`.toLowerCase()
+
-  }
-  if (!env.SELF_HOSTED) {
-    return url
-  }
-  const deployedApps = await getDeployedApps()
-  if (
-    url &&
-    deployedApps[url] != null &&
-    ctx.params != null &&
-    deployedApps[url].appId !== ctx.params.appId
-  ) {
-    ctx.throw(400, "App name/URL is already in use.")
-  }
   return url
 }
 
+const checkAppUrl = (ctx, apps, url, currentAppId) => {
+  if (currentAppId) {
+    apps = apps.filter(app => app.appId !== currentAppId)
+  }
+  if (apps.some(app => app.url === url)) {
+    ctx.throw(400, "App URL is already in use.")
+  }
+}
+
+const checkAppName = (ctx, apps, name, currentAppId) => {
+  // TODO: Replace with Joi
+  if (!name) {
+    ctx.throw(400, "Name is required")
+  }
+  if (currentAppId) {
+    apps = apps.filter(app => app.appId !== currentAppId)
+  }
+  if (apps.some(app => app.name === name)) {
+    ctx.throw(400, "App name is already in use.")
+  }
+}
+
 async function createInstance(template) {
   const tenantId = isMultiTenant() ? getTenantId() : null
   const baseAppId = generateAppID(tenantId)
@@ -206,6 +215,12 @@ exports.fetchAppPackage = async ctx => {
 }
 
 exports.create = async ctx => {
+  const apps = await getAllApps(CouchDB, { dev: true })
+  const name = ctx.request.body.name
+  checkAppName(ctx, apps, name)
+  const url = await getAppUrl(ctx)
+  checkAppUrl(ctx, apps, url)
+
   const { useTemplate, templateKey, templateString } = ctx.request.body
   const instanceConfig = {
     useTemplate,
@@ -218,7 +233,6 @@ exports.create = async ctx => {
   const instance = await createInstance(instanceConfig)
   const appId = instance._id
 
-  const url = await getAppUrlIfNotInUse(ctx)
   const db = new CouchDB(appId)
   let _rev
   try {
@@ -235,7 +249,7 @@ exports.create = async ctx => {
     type: "app",
     version: packageJson.version,
     componentLibraries: ["@budibase/standard-components"],
-    name: ctx.request.body.name,
+    name: name,
     url: url,
     template: ctx.request.body.template,
     instance: instance,
@@ -263,7 +277,15 @@ exports.create = async ctx => {
 }
 
 exports.update = async ctx => {
-  const data = await updateAppPackage(ctx, ctx.request.body, ctx.params.appId)
+  const apps = await getAllApps(CouchDB, { dev: true })
+  // validation
+  const name = ctx.request.body.name
+  checkAppName(ctx, apps, name, ctx.params.appId)
+  const url = await getAppUrl(ctx)
+  checkAppUrl(ctx, apps, url, ctx.params.appId)
+
+  const appPackageUpdates = { name, url }
+  const data = await updateAppPackage(appPackageUpdates, ctx.params.appId)
   ctx.status = 200
   ctx.body = data
 }
@@ -285,7 +307,7 @@ exports.updateClient = async ctx => {
     version: packageJson.version,
     revertableVersion: currentVersion,
   }
-  const data = await updateAppPackage(ctx, appPackageUpdates, ctx.params.appId)
+  const data = await updateAppPackage(appPackageUpdates, ctx.params.appId)
   ctx.status = 200
   ctx.body = data
 }
@@ -308,7 +330,7 @@ exports.revertClient = async ctx => {
     version: application.revertableVersion,
     revertableVersion: null,
   }
-  const data = await updateAppPackage(ctx, appPackageUpdates, ctx.params.appId)
+  const data = await updateAppPackage(appPackageUpdates, ctx.params.appId)
   ctx.status = 200
   ctx.body = data
 }
@@ -381,12 +403,11 @@ exports.sync = async (ctx, next) => {
   }
 }
 
-const updateAppPackage = async (ctx, appPackage, appId) => {
+const updateAppPackage = async (appPackage, appId) => {
-  const url = await getAppUrlIfNotInUse(ctx)
   const db = new CouchDB(appId)
   const application = await db.get(DocumentTypes.APP_METADATA)
 
-  const newAppPackage = { ...application, ...appPackage, url }
+  const newAppPackage = { ...application, ...appPackage }
   if (appPackage._rev !== application._rev) {
     newAppPackage._rev = application._rev
   }

packages/server/src/api/controllers/hosting.js

@@ -1,22 +0,0 @@
-const CouchDB = require("../../db")
-const { getDeployedApps } = require("../../utilities/workerRequests")
-const { getScopedConfig } = require("@budibase/backend-core/db")
-const { Configs } = require("@budibase/backend-core/constants")
-const { checkSlashesInUrl } = require("../../utilities")
-
-exports.fetchUrls = async ctx => {
-  const appId = ctx.appId
-  const db = new CouchDB(appId)
-  const settings = await getScopedConfig(db, { type: Configs.SETTINGS })
-  let appUrl = "http://localhost:10000/app"
-  if (settings && settings["platformUrl"]) {
-    appUrl = checkSlashesInUrl(`${settings["platformUrl"]}/app`)
-  }
-  ctx.body = {
-    app: appUrl,
-  }
-}
-
-exports.getDeployedApps = async ctx => {
-  ctx.body = await getDeployedApps()
-}

packages/server/src/api/controllers/row/utils.js

@@ -58,22 +58,16 @@ exports.validate = async ({ appId, tableId, row, table }) => {
     let res
 
     // Validate.js doesn't seem to handle array
-    if (type === FieldTypes.ARRAY) {
+    if (type === FieldTypes.ARRAY && row[fieldName]) {
-      const hasValues =
+      if (row[fieldName].length) {
-        Array.isArray(row[fieldName]) && row[fieldName].length > 0
-
-      // Check values are valid if values are specified
-      if (hasValues) {
         row[fieldName].map(val => {
           if (!constraints.inclusion.includes(val)) {
-            errors[fieldName] = "Value not in list"
+            errors[fieldName] = "Field not in list"
           }
         })
-      }
+      } else if (constraints.presence && row[fieldName].length === 0) {
-
+        // non required MultiSelect creates an empty array, which should not throw errors
-      // Check for required constraint
+        errors[fieldName] = [`${fieldName} is required`]
-      if (constraints.presence === true && !hasValues) {
-        errors[fieldName] = "Required field"
       }
     } else if (type === FieldTypes.JSON && typeof row[fieldName] === "string") {
       // this should only happen if there is an error

packages/server/src/api/controllers/static/index.js

@@ -5,7 +5,7 @@ const { resolve, join } = require("../../../utilities/centralPath")
 const uuid = require("uuid")
 const { ObjectStoreBuckets } = require("../../../constants")
 const { processString } = require("@budibase/string-templates")
-const { getDeployedApps } = require("../../../utilities/workerRequests")
+const { getAllApps } = require("@budibase/backend-core/db")
 const CouchDB = require("../../../db")
 const {
   loadHandlebarsFile,
@@ -17,6 +17,8 @@ const { clientLibraryPath } = require("../../../utilities")
 const { upload } = require("../../../utilities/fileSystem")
 const { attachmentsRelativeURL } = require("../../../utilities")
 const { DocumentTypes } = require("../../../db/utils")
+const AWS = require("aws-sdk")
+const AWS_REGION = env.AWS_REGION ? env.AWS_REGION : "eu-west-1"
 
 async function prepareUpload({ s3Key, bucket, metadata, file }) {
   const response = await upload({
@@ -37,12 +39,18 @@ async function prepareUpload({ s3Key, bucket, metadata, file }) {
   }
 }
 
-async function checkForSelfHostedURL(ctx) {
+async function getAppIdFromUrl(ctx) {
-  // the "appId" component of the URL may actually be a specific self hosted URL
+  // the "appId" component of the URL can be the id or the custom url
   let possibleAppUrl = `/${encodeURI(ctx.params.appId).toLowerCase()}`
-  const apps = await getDeployedApps()
+
-  if (apps[possibleAppUrl] && apps[possibleAppUrl].appId) {
+  // search prod apps for a url that matches, exclude dev where id is always used
-    return apps[possibleAppUrl].appId
+  const apps = await getAllApps(CouchDB, { dev: false })
+  const app = apps.filter(
+    a => a.url && a.url.toLowerCase() === possibleAppUrl
+  )[0]
+
+  if (app && app.appId) {
+    return app.appId
   } else {
     return ctx.params.appId
   }
@@ -75,10 +83,7 @@ exports.uploadFile = async function (ctx) {
 }
 
 exports.serveApp = async function (ctx) {
-  let appId = ctx.params.appId
+  let appId = await getAppIdFromUrl(ctx)
-  if (env.SELF_HOSTED) {
-    appId = await checkForSelfHostedURL(ctx)
-  }
   const App = require("./templates/BudibaseApp.svelte").default
   const db = new CouchDB(appId, { skip_setup: true })
   const appInfo = await db.get(DocumentTypes.APP_METADATA)
@@ -104,3 +109,51 @@ exports.serveClientLibrary = async function (ctx) {
     root: join(NODE_MODULES_PATH, "@budibase", "client", "dist"),
   })
 }
+
+exports.getSignedUploadURL = async function (ctx) {
+  const database = new CouchDB(ctx.appId)
+
+  // Ensure datasource is valid
+  let datasource
+  try {
+    const { datasourceId } = ctx.params
+    datasource = await database.get(datasourceId)
+    if (!datasource) {
+      ctx.throw(400, "The specified datasource could not be found")
+    }
+  } catch (error) {
+    ctx.throw(400, "The specified datasource could not be found")
+  }
+
+  // Ensure we aren't using a custom endpoint
+  if (datasource?.config?.endpoint) {
+    ctx.throw(400, "S3 datasources with custom endpoints are not supported")
+  }
+
+  // Determine type of datasource and generate signed URL
+  let signedUrl
+  let publicUrl
+  if (datasource.source === "S3") {
+    const { bucket, key } = ctx.request.body || {}
+    if (!bucket || !key) {
+      ctx.throw(400, "bucket and key values are required")
+      return
+    }
+    try {
+      const s3 = new AWS.S3({
+        region: AWS_REGION,
+        accessKeyId: datasource?.config?.accessKeyId,
+        secretAccessKey: datasource?.config?.secretAccessKey,
+        apiVersion: "2006-03-01",
+        signatureVersion: "v4",
+      })
+      const params = { Bucket: bucket, Key: key }
+      signedUrl = s3.getSignedUrl("putObject", params)
+      publicUrl = `https://${bucket}.s3.${AWS_REGION}.amazonaws.com/${key}`
+    } catch (error) {
+      ctx.throw(400, error)
+    }
+  }
+
+  ctx.body = { signedUrl, publicUrl }
+}

packages/server/src/api/controllers/table/index.js

@@ -2,7 +2,7 @@ const CouchDB = require("../../../db")
 const internal = require("./internal")
 const external = require("./external")
 const csvParser = require("../../../utilities/csvParser")
-const { isExternalTable } = require("../../../integrations/utils")
+const { isExternalTable, isSQL } = require("../../../integrations/utils")
 const {
   getTableParams,
   getDatasourceParams,
@@ -32,8 +32,8 @@ exports.fetch = async function (ctx) {
     })
   )
 
-  const internal = internalTables.rows.map(row => ({
+  const internal = internalTables.rows.map(tableDoc => ({
-    ...row.doc,
+    ...tableDoc.doc,
     type: "internal",
     sourceId: BudibaseInternalDB._id,
   }))
@@ -44,12 +44,18 @@ exports.fetch = async function (ctx) {
     })
   )
 
-  const external = externalTables.rows.flatMap(row => {
+  const external = externalTables.rows.flatMap(tableDoc => {
-    return Object.values(row.doc.entities || {}).map(entity => ({
+    let entities = tableDoc.doc.entities
-      ...entity,
+    if (entities) {
-      type: "external",
+      return Object.values(entities).map(entity => ({
-      sourceId: row.doc._id,
+        ...entity,
-    }))
+        type: "external",
+        sourceId: tableDoc.doc._id,
+        sql: isSQL(tableDoc.doc),
+      }))
+    } else {
+      return []
+    }
   })
 
   ctx.body = [...internal, ...external]

packages/server/src/api/controllers/table/internal.js

@@ -8,6 +8,7 @@ const {
   getTable,
   handleDataImport,
 } = require("./utils")
+const usageQuota = require("../../../utilities/usageQuota")
 
 exports.save = async function (ctx) {
   const appId = ctx.appId
@@ -119,6 +120,7 @@ exports.destroy = async function (ctx) {
     })
   )
   await db.bulkDocs(rows.rows.map(row => ({ ...row.doc, _deleted: true })))
+  await usageQuota.update(usageQuota.Properties.ROW, -rows.rows.length)
 
   // update linked rows
   await linkRows.updateLinks({

packages/server/src/api/controllers/table/utils.js

@@ -12,9 +12,11 @@ const { USERS_TABLE_SCHEMA, SwitchableTypes } = require("../../../constants")
 const {
   isExternalTable,
   breakExternalTableId,
+  isSQL,
 } = require("../../../integrations/utils")
 const { getViews, saveView } = require("../view/utils")
 const viewTemplate = require("../view/viewBuilder")
+const usageQuota = require("../../../utilities/usageQuota")
 
 exports.checkForColumnUpdates = async (db, oldTable, updatedTable) => {
   let updatedRows = []
@@ -111,7 +113,11 @@ exports.handleDataImport = async (appId, user, table, dataImport) => {
     finalData.push(row)
   }
 
+  await usageQuota.update(usageQuota.Properties.ROW, finalData.length, {
+    dryRun: true,
+  })
   await db.bulkDocs(finalData)
+  await usageQuota.update(usageQuota.Properties.ROW, finalData.length)
   let response = await db.put(table)
   table._rev = response._rev
   return table
@@ -242,7 +248,9 @@ exports.getTable = async (appId, tableId) => {
   const db = new CouchDB(appId)
   if (isExternalTable(tableId)) {
     let { datasourceId, tableName } = breakExternalTableId(tableId)
-    return exports.getExternalTable(appId, datasourceId, tableName)
+    const datasource = await db.get(datasourceId)
+    const table = await exports.getExternalTable(appId, datasourceId, tableName)
+    return { ...table, sql: isSQL(datasource) }
   } else {
     return db.get(tableId)
   }

packages/server/src/api/routes/hosting.js

@@ -1,13 +0,0 @@
-const Router = require("@koa/router")
-const controller = require("../controllers/hosting")
-const authorized = require("../../middleware/authorized")
-const { BUILDER } = require("@budibase/backend-core/permissions")
-
-const router = Router()
-
-router
-  .get("/api/hosting/urls", authorized(BUILDER), controller.fetchUrls)
-  // this isn't risky, doesn't return anything about apps other than names and URLs
-  .get("/api/hosting/apps", controller.getDeployedApps)
-
-module.exports = router

packages/server/src/api/routes/index.js

@@ -20,7 +20,6 @@ const integrationRoutes = require("./integration")
 const permissionRoutes = require("./permission")
 const datasourceRoutes = require("./datasource")
 const queryRoutes = require("./query")
-const hostingRoutes = require("./hosting")
 const backupRoutes = require("./backup")
 const metadataRoutes = require("./metadata")
 const devRoutes = require("./dev")
@@ -46,7 +45,6 @@ exports.mainRoutes = [
   permissionRoutes,
   datasourceRoutes,
   queryRoutes,
-  hostingRoutes,
   backupRoutes,
   metadataRoutes,
   devRoutes,

packages/server/src/api/routes/static.js

@@ -46,5 +46,10 @@ router
   )
   // TODO: this likely needs to be secured in some way
   .get("/:appId/:path*", controller.serveApp)
+  .post(
+    "/api/attachments/:datasourceId/url",
+    authorized(PermissionTypes.TABLE, PermissionLevels.READ),
+    controller.getSignedUploadURL
+  )
 
 module.exports = router

packages/server/src/api/routes/tests/application.spec.js

@@ -53,8 +53,8 @@ describe("/applications", () => {
 
   describe("fetch", () => {
     it("lists all applications", async () => {
-      await config.createApp(request, "app1")
+      await config.createApp("app1")
-      await config.createApp(request, "app2")
+      await config.createApp("app2")
 
       const res = await request
         .get(`/api/applications?status=${AppStatus.DEV}`)

packages/server/src/api/routes/tests/hosting.spec.js

@@ -1,36 +0,0 @@
-// mock out node fetch for this
-jest.mock("node-fetch")
-
-const { checkBuilderEndpoint } = require("./utilities/TestFunctions")
-const setup = require("./utilities")
-
-describe("/hosting", () => {
-  let request = setup.getRequest()
-  let config = setup.getConfig()
-  let app
-
-  afterAll(setup.afterAll)
-
-  beforeEach(async () => {
-    app = await config.init()
-  })
-
-  describe("fetchUrls", () => {
-    it("should be able to fetch current app URLs", async () => {
-      const res = await request
-        .get(`/api/hosting/urls`)
-        .set(config.defaultHeaders())
-        .expect("Content-Type", /json/)
-        .expect(200)
-      expect(res.body.app).toEqual(`http://localhost:10000/app`)
-    })
-
-    it("should apply authorization to endpoint", async () => {
-      await checkBuilderEndpoint({
-        config,
-        method: "GET",
-        url: `/api/hosting/urls`,
-      })
-    })
-  })
-})

packages/server/src/api/routes/tests/static.spec.js

@@ -0,0 +1,98 @@
+jest.mock("node-fetch")
+jest.mock("aws-sdk", () => ({
+  config: {
+    update: jest.fn(),
+  },
+  DynamoDB: {
+    DocumentClient: jest.fn(),
+  },
+  S3: jest.fn(() => ({
+    getSignedUrl: jest.fn(() => {
+      return "my-url"
+    }),
+  })),
+}))
+
+const setup = require("./utilities")
+
+describe("/attachments", () => {
+  let request = setup.getRequest()
+  let config = setup.getConfig()
+  let app
+
+  afterAll(setup.afterAll)
+
+  beforeEach(async () => {
+    app = await config.init()
+  })
+
+  describe("generateSignedUrls", () => {
+    let datasource
+
+    beforeEach(async () => {
+      datasource = await config.createDatasource({
+        datasource: {
+          type: "datasource",
+          name: "Test",
+          source: "S3",
+          config: {},
+        },
+      })
+    })
+
+    it("should be able to generate a signed upload URL", async () => {
+      const bucket = "foo"
+      const key = "bar"
+      const res = await request
+        .post(`/api/attachments/${datasource._id}/url`)
+        .send({ bucket, key })
+        .set(config.defaultHeaders())
+        .expect("Content-Type", /json/)
+        .expect(200)
+      expect(res.body.signedUrl).toEqual("my-url")
+      expect(res.body.publicUrl).toEqual(
+        `https://${bucket}.s3.eu-west-1.amazonaws.com/${key}`
+      )
+    })
+
+    it("should handle an invalid datasource ID", async () => {
+      const res = await request
+        .post(`/api/attachments/foo/url`)
+        .send({
+          bucket: "foo",
+          key: "bar",
+        })
+        .set(config.defaultHeaders())
+        .expect("Content-Type", /json/)
+        .expect(400)
+      expect(res.body.message).toEqual(
+        "The specified datasource could not be found"
+      )
+    })
+
+    it("should require a bucket parameter", async () => {
+      const res = await request
+        .post(`/api/attachments/${datasource._id}/url`)
+        .send({
+          bucket: undefined,
+          key: "bar",
+        })
+        .set(config.defaultHeaders())
+        .expect("Content-Type", /json/)
+        .expect(400)
+      expect(res.body.message).toEqual("bucket and key values are required")
+    })
+
+    it("should require a key parameter", async () => {
+      const res = await request
+        .post(`/api/attachments/${datasource._id}/url`)
+        .send({
+          bucket: "foo",
+        })
+        .set(config.defaultHeaders())
+        .expect("Content-Type", /json/)
+        .expect(400)
+      expect(res.body.message).toEqual("bucket and key values are required")
+    })
+  })
+})

packages/server/src/automations/steps/createRow.js

@@ -1,6 +1,5 @@
 const rowController = require("../../api/controllers/row")
 const automationUtils = require("../automationUtils")
-const env = require("../../environment")
 const usage = require("../../utilities/usageQuota")
 const { buildCtx } = require("./utils")
 
@@ -83,10 +82,9 @@ exports.run = async function ({ inputs, appId, emitter }) {
       inputs.row.tableId,
       inputs.row
     )
-    if (env.USE_QUOTAS) {
+    await usage.update(usage.Properties.ROW, 1, { dryRun: true })
-      await usage.update(usage.Properties.ROW, 1)
-    }
     await rowController.save(ctx)
+    await usage.update(usage.Properties.ROW, 1)
     return {
       row: inputs.row,
       response: ctx.body,

packages/server/src/automations/steps/deleteRow.js

@@ -1,5 +1,4 @@
 const rowController = require("../../api/controllers/row")
-const env = require("../../environment")
 const usage = require("../../utilities/usageQuota")
 const { buildCtx } = require("./utils")
 const automationUtils = require("../automationUtils")
@@ -74,9 +73,7 @@ exports.run = async function ({ inputs, appId, emitter }) {
   })
 
   try {
-    if (env.isProd()) {
+    await usage.update(usage.Properties.ROW, -1)
-      await usage.update(usage.Properties.ROW, -1)
-    }
     await rowController.destroy(ctx)
     return {
       response: ctx.body,