Merge pull request #3156 from Budibase/develop

Develop -> Master

.github/workflows/release-develop.yml

@@ -9,7 +9,6 @@ env:
   POSTHOG_TOKEN: ${{ secrets.POSTHOG_TOKEN }}
   INTERCOM_TOKEN: ${{ secrets.INTERCOM_TOKEN }}
   POSTHOG_URL: ${{ secrets.POSTHOG_URL }}
-  SENTRY_DSN: ${{ secrets.SENTRY_DSN }}
       
 jobs:
   release:

.github/workflows/release-selfhost.yml

@@ -7,7 +7,6 @@ env:
   POSTHOG_TOKEN: ${{ secrets.POSTHOG_TOKEN }}
   INTERCOM_TOKEN: ${{ secrets.INTERCOM_TOKEN }}
   POSTHOG_URL: ${{ secrets.POSTHOG_URL }}
-  SENTRY_DSN: ${{ secrets.SENTRY_DSN }}
 
 jobs:
   release:

.vscode/launch.json

@@ -4,39 +4,27 @@
     // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
     "version": "0.2.0",
     "configurations": [
-        {
-            "type": "node",
-            "request": "launch",
-            "name": "Launch Program",
-            "program": "${workspaceFolder}/app.js",
-            "skipFiles": [
-                "<node_internals>/**"
-            ]
-        },
-        {
-            "type": "node",
-            "request": "launch",
-            "name": "Debug External",
-            "program": "${workspaceFolder}/packages/cli/bin/budi",
-            "args": [],
-            "cwd":"C:/code/my-apps",
-            "console": "externalTerminal" 
-        },
         {
             "name": "Budibase Server",
             "type": "node",
             "request": "launch",
-            "runtimeArgs": ["--nolazy", "-r", "ts-node/register/transpile-only"],
+            "runtimeArgs": [
-            "args": ["${workspaceFolder}/packages/server/src/index.ts"],
+                "--nolazy",
+                "-r",
+                "ts-node/register/transpile-only"
+            ],
+            "args": [
+                "${workspaceFolder}/packages/server/src/index.ts"
+            ],
             "cwd": "${workspaceFolder}/packages/server"
-          },
+        },
-          {
+        {
             "name": "Budibase Worker",
             "type": "node",
             "request": "launch",
             "program": "${workspaceFolder}/packages/worker/src/index.js",
             "cwd": "${workspaceFolder}/packages/worker"
-          }
+        }
     ],
     "compounds": [
         {

README.md

@@ -8,18 +8,19 @@
 </h1>
 
 <h3 align="center">
-  Build, automate and self-host internal tools in minutes
+  The low code platform you'll enjoy using
 </h3>
 <p align="center">
-  Budibase is an open-source low-code platform, helping developers and IT professionals build, automate, and ship internal tools on their own infrastructure in minutes.
+  Budibase is an open source low-code platform, and the easiest way to build internal tools that improve productivity.
 </p>
 
 <h3 align="center">
  🤖 🎨 🚀
 </h3>
+<br>
 
 <p align="center">
-  <img alt="Budibase design ui" src="https://i.imgur.com/5BnXPsN.png">
+  <img alt="Budibase design ui" src="https://res.cloudinary.com/daog6scxm/image/upload/v1633524049/ui/design-ui-wide-mobile_gdaveq.jpg">
 </p>
 
 <p align="center">
@@ -65,68 +66,25 @@
 
 - **Admin paradise.** Budibase is made to scale. With Budibase, you can self-host on your own infrastructure and globally manage users, onboarding, SMTP, apps, groups, theming and more. You can also provide users/groups with an app portal and disseminate user-management to the group manager.
 
-<br />
+<br /><br /><br />
-
----
-
-<br />
 
 ## 🏁 Get started
-Currently there are two ways to get started with Budibase; Digital Ocean, and Docker.
+
+<img src="https://res.cloudinary.com/daog6scxm/image/upload/v1634808888/logo/deploy_npl9za.png" />
+
+Deploy Budibase self-Hosted in your existing infrastructure, using Docker, Kubernetes, and Digital Ocean.
+Or use Budibase Cloud if you don't need to self-host, and would like to get started quickly.
+
+### [Get started with Budibase](https://budibase.com)
+
+
 <br /><br />
 
-### Get started with Digital Ocean
-The easiest and quickest way to get started, is to use Digital Ocean:
-<a href="https://marketplace.digitalocean.com/apps/budibase">1-click Digital Ocean deploy</a>
-
-<a href="https://marketplace.digitalocean.com/apps/budibase">
-  <img src="https://user-images.githubusercontent.com/552074/87779219-5c3b7600-c824-11ea-9898-981a8ba94f6c.png" alt="digital ocean badge">
-</a>  
-<br /><br />
-
-### Get started with Docker
-To get started, you must have docker and docker compose installed on your machine. 
-Once you have Docker installed, the process takes 5 minutes, with these four steps:
-
-1. Install the Budibase CLI.
-
-```
-$ npm i -g @budibase/cli 
-```
-
-
-2. Setup Budibase (select where to store Budibase, and the port to run it on)
-
-```
-budi hosting --init 
-```
-
-
-3. Run Budibase
-
-```
-budi hosting --start 
-```
-
-
-4. Create your admin user
-
-Enter the email and password for the new admin user.
-
-Done! You are now ready to build powerful internal tools in minutes. For additional information on how to get started and learn Budibase, visit our [docs](https://docs.budibase.com/getting-started).
-
-<br />
-
----
-
-<br />
-
 ## 🎓 Learning Budibase
 
 The Budibase documentation [lives here](https://docs.budibase.com).
 <br />
 
----
 
 <br /><br />
 
@@ -134,22 +92,17 @@ The Budibase documentation [lives here](https://docs.budibase.com).
 
 If you have a question or would like to talk with other Budibase users and join our community, please hop over to [Github discussions](https://github.com/Budibase/budibase/discussions)
 
-<img src="https://d33wubrfki0l68.cloudfront.net/e9241201fd89f9abbbdaac4fe44bb16312752abe/84013/img/hero-images/community.webp" />
+<br /><br /><br />
 
-<br /><br />
-
----
-
-<br />
 
 ## ❗ Code of conduct
 
 Budibase is dedicated to providing a welcoming, diverse, and harrassment-free experience for everyone. We expect everyone in the Budibase community to abide by our [**Code of Conduct**](https://github.com/Budibase/budibase/blob/HEAD/.github/CODE_OF_CONDUCT.md). Please read it.
 <br />
 
----
 
-<br />
+<br /><br />
+
 
 ## 🙌 Contributing to Budibase
 
@@ -168,32 +121,22 @@ Budibase is a monorepo managed by lerna. Lerna manages the building and publishi
 - [packages/server](https://github.com/Budibase/budibase/tree/HEAD/packages/server) - The budibase server. This Koa app is responsible for serving the JS for the builder and budibase apps, as well as providing the API for interaction with the database and file system.
 
 For more information, see [CONTRIBUTING.md](https://github.com/Budibase/budibase/blob/HEAD/.github/CONTRIBUTING.md)
-<br /><br />
-
----
 
 <br /><br />
 
+
 ## 📝 License
 
 Budibase is open-source, licensed as [GPL v3](https://www.gnu.org/licenses/gpl-3.0.en.html). The client and component libraries are licensed as [MPL](https://directory.fsf.org/wiki/License:MPL-2.0) - so the apps that you build can be licensed however you like.
+
 <br /><br />
 
----
-
-
-<br />
-
 ## ⭐ Stargazers over time
 
 [![Stargazers over time](https://starchart.cc/Budibase/budibase.svg)](https://starchart.cc/Budibase/budibase)
 
 If you are having issues between updates of the builder, please use the guide [here](https://github.com/Budibase/budibase/blob/HEAD/.github/CONTRIBUTING.md#troubleshooting) to clear down your environment.
 
-<br />
-
----
-
 <br /><br />
 
 ## Contributors ✨

hosting/docker-compose.yaml

@@ -48,10 +48,10 @@ services:
       COUCH_DB_USERNAME: ${COUCH_DB_USER}
       COUCH_DB_PASSWORD: ${COUCH_DB_PASSWORD}
       COUCH_DB_URL: http://${COUCH_DB_USER}:${COUCH_DB_PASSWORD}@couchdb-service:5984
+      SENTRY_DSN: https://a34ae347621946bf8acded18e5b7d4b8@o420233.ingest.sentry.io/5338131
       INTERNAL_API_KEY: ${INTERNAL_API_KEY}
       REDIS_URL: redis-service:6379
       REDIS_PASSWORD: ${REDIS_PASSWORD}
-      ACCOUNT_PORTAL_URL: https://portal.budi.live
     volumes:
       - ./logs:/logs
     depends_on:

hosting/envoy.dev.yaml.hbs

@@ -41,6 +41,7 @@ static_resources:
               - match: { prefix: "/api/" }
                 route:
                   cluster: server-dev
+                  timeout: 120s
 
               - match: { prefix: "/app_" }
                 route:

hosting/envoy.yaml

@@ -58,6 +58,7 @@ static_resources:
               - match: { prefix: "/api/" }
                 route:
                   cluster: app-service
+                  timeout: 120s
 
               - match: { prefix: "/worker/" }
                 route: 

hosting/kubernetes/budibase/templates/worker-service-deployment.yaml

@@ -91,6 +91,8 @@ spec:
           {{ end }}
         - name: SELF_HOSTED
           value: {{ .Values.globals.selfHosted | quote }}
+        - name: SENTRY_DSN
+          value: {{ .Values.globals.sentryDSN }}
         - name: ACCOUNT_PORTAL_URL
           value: {{ .Values.globals.accountPortalUrl | quote }}
         - name: ACCOUNT_PORTAL_API_KEY

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "0.9.172",
+  "version": "0.9.173-alpha.3",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

packages/auth/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/auth",
-  "version": "0.9.172",
+  "version": "0.9.173-alpha.3",
   "description": "Authentication middlewares for budibase builder and apps",
   "main": "src/index.js",
   "author": "Budibase",
@@ -18,6 +18,7 @@
     "jsonwebtoken": "^8.5.1",
     "koa-passport": "^4.1.4",
     "lodash": "^4.17.21",
+    "lodash.isarguments": "^3.1.0",
     "node-fetch": "^2.6.1",
     "passport-google-auth": "^1.0.2",
     "passport-google-oauth": "^2.0.0",

packages/auth/src/db/Replication.js

@@ -45,22 +45,6 @@ class Replication {
     return this.replication
   }
 
-  /**
-   * Set up an ongoing live sync between 2 CouchDB databases.
-   * @param {Object} opts - PouchDB replication options
-   */
-  subscribe(opts = {}) {
-    this.replication = this.source.replicate
-      .to(this.target, {
-        live: true,
-        retry: true,
-        ...opts,
-      })
-      .on("error", function (err) {
-        throw new Error(`Replication Error: ${err}`)
-      })
-  }
-
   /**
    * Rollback the target DB back to the state of the source DB
    */

packages/auth/src/db/constants.js

@@ -13,6 +13,7 @@ exports.DocumentTypes = {
   APP_DEV: `${PRE_APP}${exports.SEPARATOR}${PRE_DEV}`,
   APP_METADATA: `${PRE_APP}${exports.SEPARATOR}metadata`,
   ROLE: "role",
+  MIGRATIONS: "migrations",
 }
 
 exports.StaticDatabases = {

packages/auth/src/db/views.js

@@ -21,7 +21,7 @@ exports.createUserEmailView = async db => {
     // if using variables in a map function need to inject them before use
     map: `function(doc) {
       if (doc._id.startsWith("${DocumentTypes.USER}")) {
-        emit(doc.email, doc._id)
+        emit(doc.email.toLowerCase(), doc._id)
       }
     }`,
   }

packages/auth/src/middleware/passport/tests/third-party-common.spec.js

@@ -1,6 +1,6 @@
 // Mock data
 
-require("./utilities/test-config")
+require("../../../tests/utilities/dbConfig")
 
 const database = require("../../../db")
 const { authenticateThirdParty } = require("../third-party-common")
@@ -72,7 +72,6 @@ describe("third party common", () => {
 
     const expectUserIsSynced = (user, thirdPartyUser) => {
       expect(user.provider).toBe(thirdPartyUser.provider)
-      expect(user.email).toBe(thirdPartyUser.email)
       expect(user.firstName).toBe(thirdPartyUser.profile.name.givenName)
       expect(user.lastName).toBe(thirdPartyUser.profile.name.familyName)
       expect(user.thirdPartyProfile).toStrictEqual(thirdPartyUser.profile._json)
@@ -135,6 +134,24 @@ describe("third party common", () => {
         })
       })
 
+      describe("exists by email with different casing", () => {
+        beforeEach(async () => {
+          id = generateGlobalUserID(newid()) // random id
+          email = thirdPartyUser.email.toUpperCase() //  matching email except for casing
+          await createUser()
+        })
+
+        it("syncs and authenticates the user", async () => {
+          await authenticateThirdParty(thirdPartyUser, true, done, saveUser)
+        
+          const user = expectUserIsAuthenticated()
+          expectUserIsSynced(user, thirdPartyUser)
+          expectUserIsUpdated(user)
+          expect(user.email).toBe(thirdPartyUser.email.toUpperCase())
+        })
+      })
+
+
       describe("exists by id", () => {
         beforeEach(async () => {
           id = generateGlobalUserID(thirdPartyUser.userId) // matching id

packages/auth/src/middleware/passport/third-party-common.js

@@ -66,12 +66,16 @@ exports.authenticateThirdParty = async function (
     // setup a blank user using the third party id
     dbUser = {
       _id: userId,
+      email: thirdPartyUser.email,
       roles: {},
     }
   }
 
   dbUser = await syncUser(dbUser, thirdPartyUser)
 
+  // never prompt for password reset
+  dbUser.forceResetPassword = false
+
   // create or sync the user
   let response
   try {
@@ -122,9 +126,6 @@ async function syncUser(user, thirdPartyUser) {
   user.provider = thirdPartyUser.provider
   user.providerType = thirdPartyUser.providerType
 
-  // email
-  user.email = thirdPartyUser.email
-
   if (thirdPartyUser.profile) {
     const profile = thirdPartyUser.profile
 

packages/auth/src/migrations/index.js

@@ -0,0 +1,61 @@
+const { DocumentTypes } = require("../db/constants")
+const { getGlobalDB } = require("../tenancy")
+
+exports.MIGRATION_DBS = {
+  GLOBAL_DB: "GLOBAL_DB",
+}
+
+exports.MIGRATIONS = {
+  USER_EMAIL_VIEW_CASING: "user_email_view_casing",
+}
+
+const DB_LOOKUP = {
+  [exports.MIGRATION_DBS.GLOBAL_DB]: [
+    exports.MIGRATIONS.USER_EMAIL_VIEW_CASING,
+  ],
+}
+
+exports.getMigrationsDoc = async db => {
+  // get the migrations doc
+  try {
+    return await db.get(DocumentTypes.MIGRATIONS)
+  } catch (err) {
+    if (err.status && err.status === 404) {
+      return { _id: DocumentTypes.MIGRATIONS }
+    }
+  }
+}
+
+exports.migrateIfRequired = async (migrationDb, migrationName, migrateFn) => {
+  try {
+    let db
+    if (migrationDb === exports.MIGRATION_DBS.GLOBAL_DB) {
+      db = getGlobalDB()
+    } else {
+      throw new Error(`Unrecognised migration db [${migrationDb}]`)
+    }
+
+    if (!DB_LOOKUP[migrationDb].includes(migrationName)) {
+      throw new Error(
+        `Unrecognised migration name [${migrationName}] for db [${migrationDb}]`
+      )
+    }
+
+    const doc = await exports.getMigrationsDoc(db)
+    // exit if the migration has been performed
+    if (doc[migrationName]) {
+      return
+    }
+
+    console.log(`Performing migration: ${migrationName}`)
+    await migrateFn()
+    console.log(`Migration complete: ${migrationName}`)
+
+    // mark as complete
+    doc[migrationName] = Date.now()
+    await db.put(doc)
+  } catch (err) {
+    console.error(`Error performing migration: ${migrationName}: `, err)
+    throw err
+  }
+}

packages/auth/src/migrations/tests/__snapshots__/index.spec.js.snap

@@ -0,0 +1,9 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`migrations should match snapshot 1`] = `
+Object {
+  "_id": "migrations",
+  "_rev": "1-af6c272fe081efafecd2ea49a8fcbb40",
+  "user_email_view_casing": 1487076708000,
+}
+`;

packages/auth/src/migrations/tests/index.spec.js

@@ -0,0 +1,60 @@
+require("../../tests/utilities/dbConfig")
+
+const { migrateIfRequired, MIGRATION_DBS, MIGRATIONS, getMigrationsDoc } = require("../index")
+const database = require("../../db")
+const {
+  StaticDatabases,
+} = require("../../db/utils")
+
+Date.now = jest.fn(() => 1487076708000)
+let db
+
+describe("migrations", () => {
+
+  const migrationFunction = jest.fn()
+
+  beforeEach(() => {
+    db = database.getDB(StaticDatabases.GLOBAL.name)
+  })
+
+  afterEach(async () => {
+    jest.clearAllMocks()
+    await db.destroy()
+  })
+
+  const validMigration = () => {
+    return migrateIfRequired(MIGRATION_DBS.GLOBAL_DB, MIGRATIONS.USER_EMAIL_VIEW_CASING, migrationFunction)
+  }
+
+  it("should run a new migration", async () => {
+    await validMigration()
+    expect(migrationFunction).toHaveBeenCalled()
+  })
+
+  it("should match snapshot", async () => {
+    await validMigration()
+    const doc = await getMigrationsDoc(db)
+    expect(doc).toMatchSnapshot()
+  })
+
+  it("should skip a previously run migration", async () => {
+    await validMigration()
+    await validMigration()
+    expect(migrationFunction).toHaveBeenCalledTimes(1)
+  })
+
+  it("should reject an unknown migration name", async () => {
+    expect(async () => { 
+      await migrateIfRequired(MIGRATION_DBS.GLOBAL_DB, "bogus_name", migrationFunction)
+    }).rejects.toThrow()
+    expect(migrationFunction).not.toHaveBeenCalled()
+  })
+
+  it("should reject an unknown database name", async () => {
+    expect(async () => { 
+      await migrateIfRequired("bogus_db", MIGRATIONS.USER_EMAIL_VIEW_CASING, migrationFunction)
+    }).rejects.toThrow()
+    expect(migrationFunction).not.toHaveBeenCalled()
+  })
+
+})

packages/auth/src/tests/utilities/db.js

@@ -1,5 +1,5 @@
 const PouchDB = require("pouchdb")
-const env = require("../../../../environment")
+const env = require("../../environment")
 
 let POUCH_DB_DEFAULTS
 

packages/auth/src/tests/utilities/dbConfig.js

@@ -1,3 +1,3 @@
-const packageConfiguration = require("../../../../index")
+const packageConfiguration = require("../../index")
 const CouchDB = require("./db")
 packageConfiguration.init(CouchDB)

packages/auth/src/utils.js

@@ -20,6 +20,9 @@ const { hash } = require("./hashing")
 const userCache = require("./cache/user")
 const env = require("./environment")
 const { getUserSessions, invalidateSessions } = require("./security/sessions")
+const { migrateIfRequired } = require("./migrations")
+const { USER_EMAIL_VIEW_CASING } = require("./migrations").MIGRATIONS
+const { GLOBAL_DB } = require("./migrations").MIGRATION_DBS
 
 const APP_PREFIX = DocumentTypes.APP + SEPARATOR
 
@@ -128,10 +131,16 @@ exports.getGlobalUserByEmail = async email => {
     throw "Must supply an email address to view"
   }
   const db = getGlobalDB()
+
+  await migrateIfRequired(GLOBAL_DB, USER_EMAIL_VIEW_CASING, async () => {
+    // re-create the view with latest changes
+    await createUserEmailView(db)
+  })
+
   try {
     let users = (
       await db.query(`database/${ViewNames.USER_BY_EMAIL}`, {
-        key: email,
+        key: email.toLowerCase(),
         include_docs: true,
       })
     ).rows

packages/auth/yarn.lock

@@ -3038,6 +3038,11 @@ lodash.includes@^4.3.0:
   resolved "https://registry.yarnpkg.com/lodash.includes/-/lodash.includes-4.3.0.tgz#60bb98a87cb923c68ca1e51325483314849f553f"
   integrity sha1-YLuYqHy5I8aMoeUTJUgzFISfVT8=
 
+lodash.isarguments@^3.1.0:
+  version "3.1.0"
+  resolved "https://registry.yarnpkg.com/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz#2f573d85c6a24289ff00663b491c1d338ff3458a"
+  integrity sha1-L1c9hcaiQon/AGY7SRwdM4/zRYo=
+
 lodash.isboolean@^3.0.3:
   version "3.0.3"
   resolved "https://registry.yarnpkg.com/lodash.isboolean/-/lodash.isboolean-3.0.3.tgz#6c2e171db2a257cd96802fd43b01b20d5f5870f6"

packages/bbui/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/bbui",
   "description": "A UI solution used in the different Budibase projects.",
-  "version": "0.9.172",
+  "version": "0.9.173-alpha.3",
   "license": "AGPL-3.0",
   "svelte": "src/index.js",
   "module": "dist/bbui.es.js",

packages/bbui/src/Form/Core/DatePicker.svelte

@@ -31,7 +31,11 @@
 
   const handleChange = event => {
     const [dates] = event.detail
-    dispatch("change", dates[0])
+    let newValue = dates[0]
+    if (newValue) {
+      newValue = newValue.toISOString()
+    }
+    dispatch("change", newValue)
   }
 
   const clearDateOnBackspace = event => {
@@ -57,11 +61,36 @@
     const els = document.querySelectorAll(`#${flatpickrId} input`)
     els.forEach(el => el.blur())
   }
+
+  const parseDate = val => {
+    if (!val) {
+      return null
+    }
+    let date
+    if (val instanceof Date) {
+      // Use real date obj if already parsed
+      date = val
+    } else if (isNaN(val)) {
+      // Treat as date string of some sort
+      date = new Date(val)
+    } else {
+      // Treat as numerical timestamp
+      date = new Date(parseInt(val))
+    }
+    const time = date.getTime()
+    if (isNaN(time)) {
+      return null
+    }
+    // By rounding to the nearest second we avoid locking up in an endless
+    // loop in the builder, caused by potentially enriching {{ now }} to every
+    // millisecond.
+    return new Date(Math.floor(time / 1000) * 1000)
+  }
 </script>
 
 <Flatpickr
   bind:flatpickr
-  {value}
+  value={parseDate(value)}
   on:open={onOpen}
   on:close={onClose}
   options={flatpickrOptions}

packages/bbui/src/Form/Core/Picker.svelte

@@ -115,7 +115,7 @@
 {#if open}
   <div
     use:clickOutside={() => (open = false)}
-    transition:fly={{ y: -20, duration: 200 }}
+    transition:fly|local={{ y: -20, duration: 200 }}
     class="spectrum-Popover spectrum-Popover--bottom spectrum-Picker-popover is-open"
     class:auto-width={autoWidth}
   >

packages/bbui/src/Form/DatePicker.svelte

@@ -13,10 +13,10 @@
   export let appendTo = undefined
 
   const dispatch = createEventDispatcher()
+
   const onChange = e => {
-    const isoString = e.detail.toISOString()
+    value = e.detail
-    value = isoString
+    dispatch("change", e.detail)
-    dispatch("change", isoString)
   }
 </script>
 

packages/bbui/src/Table/CellRenderer.svelte

@@ -5,6 +5,7 @@
   import RelationshipRenderer from "./RelationshipRenderer.svelte"
   import AttachmentRenderer from "./AttachmentRenderer.svelte"
   import ArrayRenderer from "./ArrayRenderer.svelte"
+  import InternalRenderer from "./InternalRenderer.svelte"
 
   export let row
   export let schema
@@ -22,8 +23,8 @@
     number: StringRenderer,
     longform: StringRenderer,
     array: ArrayRenderer,
+    internal: InternalRenderer,
   }
-
   $: type = schema?.type ?? "string"
   $: customRenderer = customRenderers?.find(x => x.column === schema?.name)
   $: renderer = customRenderer?.component ?? typeMap[type] ?? StringRenderer

packages/bbui/src/Tabs/Tab.svelte

@@ -8,11 +8,19 @@
   const selected = getContext("tab")
   let tab
   let tabInfo
+
   const setTabInfo = () => {
-    tabInfo = tab.getBoundingClientRect()
+    // If the tabs are being rendered inside a component which uses
-    if ($selected.title === title) {
+    // a svelte transition to enter, then this initial getBoundingClientRect
-      $selected.info = tabInfo
+    // will return an incorrect position.
-    }
+    // We just need to get this off the main thread to fix this, by using
+    // a 0ms timeout.
+    setTimeout(() => {
+      tabInfo = tab.getBoundingClientRect()
+      if ($selected.title === title) {
+        $selected.info = tabInfo
+      }
+    }, 0)
   }
 
   onMount(() => {

packages/builder/cypress/integration/createTable.spec.js

@@ -31,7 +31,7 @@ context("Create a Table", () => {
     cy.contains("nameupdated ").should("contain", "nameupdated")
   })
 
-  /*
+  
   it("edits a row", () => {
     cy.contains("button", "Edit").click({ force: true })
     cy.wait(1000)
@@ -40,7 +40,7 @@ context("Create a Table", () => {
     cy.contains("Save").click()
     cy.contains("Updated").should("have.text", "Updated")
   })
-  */
+  
   it("deletes a row", () => {
     cy.get(".spectrum-Checkbox-input").check({ force: true })
     cy.contains("Delete 1 row(s)").click()

packages/builder/cypress/integration/customThemingProperties.spec.js

@@ -1,16 +1,16 @@
-context("Custom Theming Properties", () => {
+xcontext("Custom Theming Properties", () => {
     before(() => {
       cy.login()
       cy.createTestApp()
       cy.navigateToFrontend()
     })
 
-    // Default Values
+    /* Default Values:
-    // Button roundness = Large
+    Button roundness = Large
-    // Accent colour = Blue 600
+    Accent colour = Blue 600
-    // Accent colour (hover) = Blue 500
+    Accent colour (hover) = Blue 500
-    // Navigation bar background colour = Gray 100
+    Navigation bar background colour = Gray 100
-    // Navigation bar text colour = Gray 800
+    Navigation bar text colour = Gray 800 */
     it("should reset the color property values", () => {
       // Open Theme modal and change colours  
       cy.get(".spectrum-ActionButton-label").contains("Theme").click()
@@ -24,6 +24,29 @@ context("Custom Theming Properties", () => {
         checkThemeColorDefaults()
     })
     
+    /* Button Roundness Values:
+    None = 0
+    Small = 4px
+    Medium = 8px
+    Large = 16px */
+    it("should test button roundness", () => {
+      const buttonRoundnessValues = ["0", "4px", "8px", "16px"]
+      cy.wait(1000)
+      // Add button, change roundness and confirm value
+      cy.addComponent("Button", null).then((componentId) => {
+        buttonRoundnessValues.forEach(function (item, index){
+          cy.get(".spectrum-ActionButton-label").contains("Theme").click()
+          cy.get(".setting").contains("Button roundness").parent()
+          .get(".select-wrapper").click()
+          cy.get(".spectrum-Popover").find('li').eq(index).click()
+          cy.get(".spectrum-Button").contains("View changes").click({force: true})
+          cy.reload()
+          cy.getComponent(componentId)
+        .parents(".svelte-xiqd1c").eq(0).should('have.attr', 'style').and('contains', `--buttonBorderRadius:${item}`)
+      })
+    })
+  })
+    
     const changeThemeColors = () => {
       // Changes the theme colours
       cy.get(".spectrum-FieldLabel").contains("Accent color")

packages/builder/cypress/integration/renameAnApplication.spec.js

@@ -0,0 +1,102 @@
+context("Rename an App", () => {
+    beforeEach(() => {
+      cy.login()
+      cy.createTestApp()
+    })
+
+it("should rename an unpublished application", () => {
+    const appRename = "Cypress Renamed"
+    // Rename app, Search for app, Confirm name was changed
+    cy.get(".home-logo").click()
+    renameApp(appRename)
+    cy.searchForApplication(appRename)
+    cy.get(".appGrid").find(".wrapper").should("have.length", 1)
+    })
+    
+xit("Should rename a published application", () => {
+    // It is not possible to rename a published application
+    const appRename = "Cypress Renamed"
+    // Publish the app
+    cy.get(".toprightnav")
+    cy.get(".spectrum-Button").contains("Publish").click({force: true})
+    cy.get(".spectrum-Dialog-grid")
+        .within(() => {
+            // Click publish again within the modal
+            cy.get(".spectrum-Button").contains("Publish").click({force: true})
+        })
+    // Rename app, Search for app, Confirm name was changed
+    cy.get(".home-logo").click()
+    renameApp(appRename, true)
+    cy.searchForApplication(appRename)
+    cy.get(".appGrid").find(".wrapper").should("have.length", 1)
+})
+
+it("Should try to rename an application to have no name", () => {
+    cy.get(".home-logo").click()
+    renameApp(" ", false, true)
+    // Close modal and confirm name has not been changed
+    cy.get(".spectrum-Dialog-grid").contains("Cancel").click()
+    cy.searchForApplication("Cypress Tests")
+    cy.get(".appGrid").find(".wrapper").should("have.length", 1)
+})
+
+xit("Should create two applications with the same name", () => {
+    // It is not possible to have applications with the same name
+    const appName = "Cypress Tests"
+    cy.visit(`localhost:${Cypress.env("PORT")}/builder`)
+    cy.wait(500)
+    cy.get(".spectrum-Button").contains("Create app").click({force: true})
+    cy.contains(/Start from scratch/).click()
+    cy.get(".spectrum-Modal")
+    .within(() => {
+      cy.get("input").eq(0).type(appName)
+      cy.get(".spectrum-ButtonGroup").contains("Create app").click({force: true})
+      cy.get(".error").should("have.text", "Another app with the same name already exists")      
+    })
+})
+
+it("should validate application names", () => {
+    // App name must be letters, numbers and spaces only
+    // This test checks numbers and special characters specifically
+    const numberName = 12345
+    const specialCharName = "£$%^"
+    cy.get(".home-logo").click()
+    renameApp(numberName)
+    cy.searchForApplication(numberName)
+    cy.get(".appGrid").find(".wrapper").should("have.length", 1)
+    renameApp(specialCharName)
+    cy.get(".error").should("have.text", "App name must be letters, numbers and spaces only")
+})
+    
+    const renameApp = (appName, published, noName) => {
+    cy.request(`localhost:${Cypress.env("PORT")}/api/applications?status=all`)
+    .its("body")
+    .then(val => {
+        if (val.length > 0) {
+            cy.get(".title > :nth-child(3) > .spectrum-Icon").click()
+            // Check for when an app is published
+            if (published == true){
+                // Should not have Edit as option, will unpublish app
+                cy.should("not.have.value", "Edit")
+                cy.get(".spectrum-Menu").contains("Unpublish").click()
+                cy.get(".spectrum-Dialog-grid").contains("Unpublish app").click()
+                cy.get(".title > :nth-child(3) > .spectrum-Icon").click()
+            }
+            cy.contains("Edit").click()
+            cy.get(".spectrum-Modal")
+            .within(() => {
+                if (noName == true){
+                    cy.get("input").clear()
+                    cy.get(".spectrum-Dialog-grid").click()
+                    .contains("App name must be letters, numbers and spaces only")
+                    return cy
+                }
+                cy.get("input").clear()
+                cy.get("input").eq(0).type(appName).should("have.value", appName).blur()
+                cy.get(".spectrum-ButtonGroup").contains("Save").click({force: true})
+                cy.wait(500)
+      })
+    }
+    })
+}
+})

packages/builder/cypress/support/commands.js

@@ -35,19 +35,12 @@ Cypress.Commands.add("login", () => {
 Cypress.Commands.add("createApp", name => {
   cy.visit(`localhost:${Cypress.env("PORT")}/builder`)
   cy.wait(500)
-  cy.contains(/Start from scratch/).click()
+  cy.contains(/Start from scratch/).dblclick()
-  cy.get(".spectrum-Modal")
+  cy.get(".spectrum-Modal").within(() => {
-    .within(() => {
+    cy.get("input").eq(0).type(name).should("have.value", name).blur()
-      cy.get("input").eq(0).type(name).should("have.value", name).blur()
+    cy.get(".spectrum-ButtonGroup").contains("Create app").click()
-      cy.get(".spectrum-ButtonGroup").contains("Create app").click()
+    cy.wait(7000)
-      cy.wait(7000)
+  })
-    })
-    .then(() => {
-      // Because we show the datasource modal on entry, we need to create a table to get rid of the modal in the future
-      cy.createInitialDatasource("initialTable")
-      cy.expandBudibaseConnection()
-      cy.get(".nav-item.selected > .content").should("be.visible")
-    })
 })
 
 Cypress.Commands.add("deleteApp", () => {
@@ -77,22 +70,6 @@ Cypress.Commands.add("createTestTableWithData", () => {
   cy.addColumn("dog", "age", "Number")
 })
 
-Cypress.Commands.add("createInitialDatasource", tableName => {
-  // Enter table name
-  cy.get(".spectrum-Modal").within(() => {
-    cy.contains("Budibase DB").trigger("mouseover").click().click()
-    cy.wait(1000)
-    cy.contains("Continue").click()
-  })
-
-  cy.get(".spectrum-Modal").within(() => {
-    cy.wait(1000)
-    cy.get("input").first().type(tableName).blur()
-    cy.get(".spectrum-ButtonGroup").contains("Create").click()
-  })
-  cy.contains(tableName).should("be.visible")
-})
-
 Cypress.Commands.add("createTable", tableName => {
   cy.contains("Budibase DB").click()
   cy.contains("Create new table").click()
@@ -247,3 +224,9 @@ Cypress.Commands.add("addCustomSourceOptions", totalOptions => {
       cy.get(".spectrum-Button").contains("Save").click({ force: true })
     })
 })
+
+Cypress.Commands.add("searchForApplication", appName => {
+  cy.get(".spectrum-Textfield").within(() => {
+    cy.get("input").eq(0).type(appName)
+  })
+})

packages/builder/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/builder",
-  "version": "0.9.172",
+  "version": "0.9.173-alpha.3",
   "license": "AGPL-3.0",
   "private": true,
   "scripts": {
@@ -65,10 +65,10 @@
     }
   },
   "dependencies": {
-    "@budibase/bbui": "^0.9.172",
+    "@budibase/bbui": "^0.9.173-alpha.3",
-    "@budibase/client": "^0.9.172",
+    "@budibase/client": "^0.9.173-alpha.3",
     "@budibase/colorpicker": "1.1.2",
-    "@budibase/string-templates": "^0.9.172",
+    "@budibase/string-templates": "^0.9.173-alpha.3",
     "@sentry/browser": "5.19.1",
     "@spectrum-css/page": "^3.0.1",
     "@spectrum-css/vars": "^3.0.1",

packages/builder/src/builderStore/api.js

@@ -15,7 +15,7 @@ const apiCall =
     if (resp.status === 403) {
       removeCookie(Cookies.Auth)
       // reload after removing cookie, go to login
-      if (!url.includes("self")) {
+      if (!url.includes("self") && !url.includes("login")) {
         location.reload()
       }
     }

packages/builder/src/builderStore/dataBinding.js

@@ -7,11 +7,17 @@ import {
 } from "./storeUtils"
 import { store } from "builderStore"
 import { queries as queriesStores, tables as tablesStore } from "stores/backend"
-import { makePropSafe } from "@budibase/string-templates"
+import {
+  makePropSafe,
+  isJSBinding,
+  decodeJSBinding,
+  encodeJSBinding,
+} from "@budibase/string-templates"
 import { TableNames } from "../constants"
 
 // Regex to match all instances of template strings
 const CAPTURE_VAR_INSIDE_TEMPLATE = /{{([^}]+)}}/g
+const CAPTURE_VAR_INSIDE_JS = /\$\("([^")]+)"\)/g
 const CAPTURE_HBS_TEMPLATE = /{{[\S\s]*?}}/g
 
 /**
@@ -430,6 +436,15 @@ function replaceBetween(string, start, end, replacement) {
  * utility function for the readableToRuntimeBinding and runtimeToReadableBinding.
  */
 function bindingReplacement(bindableProperties, textWithBindings, convertTo) {
+  // Decide from base64 if using JS
+  const isJS = isJSBinding(textWithBindings)
+  if (isJS) {
+    textWithBindings = decodeJSBinding(textWithBindings)
+  }
+
+  // Determine correct regex to find bindings to replace
+  const regex = isJS ? CAPTURE_VAR_INSIDE_JS : CAPTURE_VAR_INSIDE_TEMPLATE
+
   const convertFrom =
     convertTo === "runtimeBinding" ? "readableBinding" : "runtimeBinding"
   if (typeof textWithBindings !== "string") {
@@ -441,7 +456,7 @@ function bindingReplacement(bindableProperties, textWithBindings, convertTo) {
     .sort((a, b) => {
       return b.length - a.length
     })
-  const boundValues = textWithBindings.match(CAPTURE_VAR_INSIDE_TEMPLATE) || []
+  const boundValues = textWithBindings.match(regex) || []
   let result = textWithBindings
   for (let boundValue of boundValues) {
     let newBoundValue = boundValue
@@ -449,7 +464,7 @@ function bindingReplacement(bindableProperties, textWithBindings, convertTo) {
     // in the search, working from longest to shortest so always use best match first
     let searchString = newBoundValue
     for (let from of convertFromProps) {
-      if (shouldReplaceBinding(newBoundValue, from, convertTo)) {
+      if (isJS || shouldReplaceBinding(newBoundValue, from, convertTo)) {
         const binding = bindableProperties.find(el => el[convertFrom] === from)
         let idx
         do {
@@ -457,7 +472,7 @@ function bindingReplacement(bindableProperties, textWithBindings, convertTo) {
           idx = searchString.indexOf(from)
           if (idx !== -1) {
             let end = idx + from.length,
-              searchReplace = Array(binding[convertTo].length).join("*")
+              searchReplace = Array(binding[convertTo].length + 1).join("*")
             // blank out parts of the search string
             searchString = replaceBetween(searchString, idx, end, searchReplace)
             newBoundValue = replaceBetween(
@@ -472,6 +487,12 @@ function bindingReplacement(bindableProperties, textWithBindings, convertTo) {
     }
     result = result.replace(boundValue, newBoundValue)
   }
+
+  // Re-encode to base64 if using JS
+  if (isJS) {
+    result = encodeJSBinding(result)
+  }
+
   return result
 }
 

packages/builder/src/components/automation/AutomationBuilder/FlowChart/ActionModal.svelte

@@ -1,11 +1,26 @@
 <script>
-  import { ModalContent, Layout, Detail, Body, Icon } from "@budibase/bbui"
+  import {
+    ModalContent,
+    Layout,
+    Detail,
+    Body,
+    Icon,
+    Tooltip,
+  } from "@budibase/bbui"
   import { automationStore } from "builderStore"
+  import { admin } from "stores/portal"
   import { externalActions } from "./ExternalActions"
 
   export let blockIdx
   export let blockComplete
 
+  const disabled = {
+    SEND_EMAIL_SMTP: {
+      disabled: !$admin.checklist.smtp.checked,
+      message: "Please configure SMTP",
+    },
+  }
+
   let selectedAction
   let actionVal
   let actions = Object.entries($automationStore.blockDefinitions.ACTION)
@@ -55,6 +70,7 @@
   }}
 >
   <Body size="XS">Select an app or event.</Body>
+
   <Layout noPadding>
     <Body size="S">Apps</Body>
 
@@ -85,24 +101,46 @@
 
     <div class="item-list">
       {#each Object.entries(internal) as [idx, action]}
-        <div
+        {#if disabled[idx] && disabled[idx].disabled}
-          class="item"
+          <Tooltip text={disabled[idx].message} direction="bottom">
-          class:selected={selectedAction === action.name}
+            <div
-          on:click={() => selectAction(action)}
+              class="item"
-        >
+              class:selected={selectedAction === action.name}
-          <div class="item-body">
+              class:disabled={true}
-            <Icon name={action.icon} />
+              on:click={() => selectAction(action)}
-            <span class="icon-spacing">
-              <Body size="XS">{action.name}</Body></span
             >
+              <div class="item-body">
+                <Icon name={action.icon} />
+                <span class="icon-spacing">
+                  <Body size="XS">{action.name}</Body></span
+                >
+              </div>
+            </div>
+          </Tooltip>
+        {:else}
+          <div
+            class="item"
+            class:selected={selectedAction === action.name}
+            on:click={() => selectAction(action)}
+          >
+            <div class="item-body">
+              <Icon name={action.icon} />
+              <span class="icon-spacing">
+                <Body size="XS">{action.name}</Body></span
+              >
+            </div>
           </div>
-        </div>
+        {/if}
       {/each}
     </div>
   </Layout>
 </ModalContent>
 
 <style>
+  .disabled {
+    opacity: 0.3;
+    pointer-events: none;
+  }
   .icon-spacing {
     margin-left: var(--spacing-m);
   }
@@ -118,7 +156,6 @@
 
   .item {
     cursor: pointer;
-    display: grid;
     grid-gap: var(--spectrum-alias-grid-margin-xsmall);
     padding: var(--spectrum-alias-item-padding-s);
     background: var(--spectrum-alias-background-color-secondary);

packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowItem.svelte

@@ -103,7 +103,7 @@
           <Detail size="S">{block?.name?.toUpperCase() || ""}</Detail>
         </div>
       </div>
-      {#if testResult}
+      {#if testResult && testResult[0]}
         <span on:click={() => resultsModal.show()}>
           <StatusLight
             positive={isTrigger || testResult[0].outputs?.success}

packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte

@@ -194,6 +194,7 @@
             value={inputData[key]}
             on:change={e => onChange(e, key)}
             {bindings}
+            allowJS={false}
           />
         {/if}
       {:else if value.customType === "query"}
@@ -259,6 +260,7 @@
               value={inputData[key]}
               on:change={e => onChange(e, key)}
               {bindings}
+              allowJS={false}
             />
           </div>
         {/if}

packages/builder/src/components/automation/SetupPanel/QueryParamSelector.svelte

@@ -1,15 +1,27 @@
 <script>
+  import { createEventDispatcher } from "svelte"
   import { queries } from "stores/backend"
   import { Select } from "@budibase/bbui"
   import DrawerBindableInput from "../../common/bindings/DrawerBindableInput.svelte"
   import AutomationBindingPanel from "../../common/bindings/ServerBindingPanel.svelte"
 
+  const dispatch = createEventDispatcher()
+
   export let value
   export let bindings
 
+  const onChangeQuery = e => {
+    value.queryId = e.detail
+    dispatch("change", value)
+  }
+
+  const onChange = (e, field) => {
+    value[field.name] = e.detail
+    dispatch("change", value)
+  }
+
   $: query = $queries.list.find(query => query._id === value?.queryId)
   $: parameters = query?.parameters ?? []
-
   // Ensure any nullish queryId values get set to empty string so
   // that the select works
   $: if (value?.queryId == null) value = { queryId: "" }
@@ -18,7 +30,8 @@
 <div class="block-field">
   <Select
     label="Query"
-    bind:value={value.queryId}
+    on:change={onChangeQuery}
+    value={value.queryId}
     options={$queries.list}
     getOptionValue={query => query._id}
     getOptionLabel={query => query.name}
@@ -32,13 +45,12 @@
         panel={AutomationBindingPanel}
         extraThin
         value={value[field.name]}
-        on:change={e => {
+        on:change={e => onChange(e, field)}
-          value[field.name] = e.detail
-        }}
         label={field.name}
         type="string"
         {bindings}
         fillWidth={true}
+        allowJS={false}
       />
     {/each}
   </div>

packages/builder/src/components/automation/SetupPanel/QuerySelector.svelte

@@ -6,7 +6,7 @@
 </script>
 
 <div class="block-field">
-  <Select bind:value secondary extraThin>
+  <Select on:change bind:value secondary extraThin>
     <option value="">Choose an option</option>
     {#each $queries.list as query}
       <option value={query._id}>{query.name}</option>

packages/builder/src/components/automation/SetupPanel/RowSelector.svelte

@@ -1,6 +1,12 @@
 <script>
   import { tables } from "stores/backend"
-  import { Select, Toggle, DatePicker, Multiselect } from "@budibase/bbui"
+  import {
+    Select,
+    Toggle,
+    DatePicker,
+    Multiselect,
+    TextArea,
+  } from "@budibase/bbui"
   import DrawerBindableInput from "../../common/bindings/DrawerBindableInput.svelte"
   import AutomationBindingPanel from "../../common/bindings/ServerBindingPanel.svelte"
   import { createEventDispatcher } from "svelte"
@@ -52,7 +58,6 @@
   getOptionLabel={table => table.name}
   getOptionValue={table => table._id}
 />
-
 {#if schemaFields.length}
   <div class="schema-fields">
     {#each schemaFields as [field, schema]}
@@ -82,6 +87,8 @@
             label={field}
             options={schema.constraints.inclusion}
           />
+        {:else if schema.type === "longform"}
+          <TextArea label={field} bind:value={value[field]} />
         {:else if schema.type === "link"}
           <LinkedRowSelector bind:linkedRows={value[field]} {schema} />
         {:else if schema.type === "string" || schema.type === "number"}
@@ -103,6 +110,7 @@
               type="string"
               {bindings}
               fillWidth={true}
+              allowJS={false}
             />
           {/if}
         {/if}

packages/builder/src/components/backend/DataTable/DataTable.svelte

@@ -20,10 +20,30 @@
   $: type = $tables.selected?.type
   $: isInternal = type !== "external"
   $: schema = $tables.selected?.schema
+  $: enrichedSchema = enrichSchema($tables.selected?.schema)
   $: id = $tables.selected?._id
   $: search = searchTable(id)
   $: columnOptions = Object.keys($search.schema || {})
 
+  const enrichSchema = schema => {
+    let tempSchema = { ...schema }
+    tempSchema._id = {
+      type: "internal",
+      editable: false,
+      displayName: "ID",
+      autocolumn: true,
+    }
+    if (isInternal) {
+      tempSchema._rev = {
+        type: "internal",
+        editable: false,
+        displayName: "Revision",
+        autocolumn: true,
+      }
+    }
+
+    return tempSchema
+  }
   // Fetches new data whenever the table changes
   const searchTable = tableId => {
     return fetchTableData({
@@ -66,7 +86,7 @@
 <div>
   <Table
     title={$tables.selected?.name}
-    {schema}
+    schema={enrichedSchema}
     {type}
     tableId={id}
     data={$search.rows}

packages/builder/src/components/backend/DataTable/formula.js

@@ -4,10 +4,15 @@ import { get as svelteGet } from "svelte/store"
 
 // currently supported level of relationship depth (server side)
 const MAX_DEPTH = 1
+
+//https://github.com/Budibase/budibase/issues/3030
+const internalType = "internal"
+
 const TYPES_TO_SKIP = [
   FIELDS.FORMULA.type,
   FIELDS.LONGFORM.type,
   FIELDS.ATTACHMENT.type,
+  internalType,
 ]
 
 export function getBindings({
@@ -53,6 +58,7 @@ export function getBindings({
     const field = Object.values(FIELDS).find(
       field => field.type === schema.type
     )
+
     const label = path == null ? column : `${path}.0.${column}`
     // only supply a description for relationship paths
     const description =

packages/builder/src/components/backend/DataTable/modals/CreateEditRow.svelte

@@ -38,7 +38,7 @@
         message: `${field} ${rowResponse.validationErrors[field][0]}`,
       }))
       return false
-    } else if (rowResponse.status === 500) {
+    } else if (rowResponse.status >= 400) {
       errors = [{ message: rowResponse.message }]
       return false
     }

packages/builder/src/components/backend/DataTable/modals/FilterModal.svelte

@@ -42,6 +42,14 @@
       name: "Contains",
       key: "CONTAINS",
     },
+    {
+      name: "Is Not Empty",
+      key: "NOT_EMPTY",
+    },
+    {
+      name: "Is Empty",
+      key: "EMPTY",
+    },
   ]
 
   const CONJUNCTIONS = [
@@ -82,40 +90,40 @@
 
   function isMultipleChoice(field) {
     return (
-      (viewTable.schema[field].constraints &&
+      viewTable.schema[field]?.constraints?.inclusion?.length ||
-        viewTable.schema[field].constraints.inclusion &&
+      viewTable.schema[field]?.type === "boolean"
-        viewTable.schema[field].constraints.inclusion.length) ||
-      viewTable.schema[field].type === "boolean"
     )
   }
 
   function fieldOptions(field) {
-    return viewTable.schema[field].type === "options"
+    return viewTable.schema[field]?.type === "options"
-      ? viewTable.schema[field].constraints.inclusion
+      ? viewTable.schema[field]?.constraints.inclusion
       : [true, false]
   }
 
   function isDate(field) {
-    return viewTable.schema[field].type === "datetime"
+    return viewTable.schema[field]?.type === "datetime"
   }
 
   function isNumber(field) {
-    return viewTable.schema[field].type === "number"
+    return viewTable.schema[field]?.type === "number"
   }
 
   const fieldChanged = filter => ev => {
-    // reset if type changed
+    // Reset if type changed
-    if (
+    const oldType = viewTable.schema[filter.key]?.type
-      filter.key &&
+    const newType = viewTable.schema[ev.detail]?.type
-      ev.detail &&
+    if (filter.key && ev.detail && oldType !== newType) {
-      viewTable.schema[filter.key].type !== viewTable.schema[ev.detail].type
-    ) {
       filter.value = ""
     }
   }
 
   const getOptionLabel = x => x.name
   const getOptionValue = x => x.key
+
+  const showValue = filter => {
+    return !(filter.condition === "EMPTY" || filter.condition === "NOT_EMPTY")
+  }
 </script>
 
 <ModalContent title="Filter" confirmText="Save" onConfirm={saveView} size="L">
@@ -144,30 +152,36 @@
           {getOptionLabel}
           {getOptionValue}
         />
-        {#if filter.key && isMultipleChoice(filter.key)}
+        {#if showValue(filter)}
-          <Select
+          {#if filter.key && isMultipleChoice(filter.key)}
-            bind:value={filter.value}
+            <Select
-            options={fieldOptions(filter.key)}
+              bind:value={filter.value}
-            getOptionLabel={x => x.toString()}
+              options={fieldOptions(filter.key)}
-          />
+              getOptionLabel={x => x.toString()}
-        {:else if filter.key && isDate(filter.key)}
+            />
-          <DatePicker
+          {:else if filter.key && isDate(filter.key)}
-            bind:value={filter.value}
+            <DatePicker
-            placeholder={filter.key || fields[0]}
+              bind:value={filter.value}
-          />
+              placeholder={filter.key || fields[0]}
-        {:else if filter.key && isNumber(filter.key)}
+            />
-          <Input
+          {:else if filter.key && isNumber(filter.key)}
-            bind:value={filter.value}
+            <Input
-            placeholder={filter.key || fields[0]}
+              bind:value={filter.value}
-            type="number"
+              placeholder={filter.key || fields[0]}
-          />
+              type="number"
+            />
+          {:else}
+            <Input
+              placeholder={filter.key || fields[0]}
+              bind:value={filter.value}
+            />
+          {/if}
+          <Icon hoverable name="Close" on:click={() => removeFilter(idx)} />
         {:else}
-          <Input
+          <Icon hoverable name="Close" on:click={() => removeFilter(idx)} />
-            placeholder={filter.key || fields[0]}
+          <!-- empty div to preserve spacing -->
-            bind:value={filter.value}
+          <div />
-          />
         {/if}
-        <Icon hoverable name="Close" on:click={() => removeFilter(idx)} />
       {/each}
     </div>
   {:else}

packages/builder/src/components/common/CodeMirrorEditor.svelte

@@ -0,0 +1,159 @@
+<script context="module">
+  import { Label } from "@budibase/bbui"
+
+  export const EditorModes = {
+    JS: {
+      name: "javascript",
+      json: false,
+    },
+    JSON: {
+      name: "javascript",
+      json: true,
+    },
+    SQL: {
+      name: "sql",
+    },
+    Handlebars: {
+      name: "handlebars",
+      base: "text/html",
+    },
+  }
+</script>
+
+<script>
+  import CodeMirror from "components/integration/codemirror"
+  import { themeStore } from "builderStore"
+  import { createEventDispatcher, onMount } from "svelte"
+
+  export let mode = EditorModes.JS
+  export let value = ""
+  export let height = 300
+  export let resize = "none"
+  export let readonly = false
+  export let hints = []
+  export let label
+
+  const dispatch = createEventDispatcher()
+  let textarea
+  let editor
+
+  // Keep editor up to date with value
+  $: editor?.setValue(value || "")
+
+  // Creates an instance of a code mirror editor
+  async function createEditor(mode, value) {
+    if (!CodeMirror || !textarea || editor) {
+      return
+    }
+
+    // Configure CM options
+    const lightTheme = $themeStore.theme.includes("light")
+    const options = {
+      mode,
+      value: value || "",
+      readOnly: readonly,
+      theme: lightTheme ? "default" : "tomorrow-night-eighties",
+
+      // Style
+      lineNumbers: true,
+      lineWrapping: true,
+      indentWithTabs: true,
+      indentUnit: 2,
+      tabSize: 2,
+
+      // QOL addons
+      extraKeys: { "Ctrl-Space": "autocomplete" },
+      styleActiveLine: { nonEmpty: true },
+      autoCloseBrackets: true,
+      matchBrackets: true,
+    }
+
+    // Register hints plugin if desired
+    if (hints?.length) {
+      CodeMirror.registerHelper("hint", "dictionaryHint", function (editor) {
+        const cursor = editor.getCursor()
+        return {
+          list: hints,
+          from: CodeMirror.Pos(cursor.line, cursor.ch),
+          to: CodeMirror.Pos(cursor.line, cursor.ch),
+        }
+      })
+      CodeMirror.commands.autocomplete = function (cm) {
+        CodeMirror.showHint(cm, CodeMirror.hint.dictionaryHint)
+      }
+    }
+
+    // Construct CM instance
+    editor = CodeMirror.fromTextArea(textarea, options)
+
+    // Use a blur handler to update the value
+    editor.on("blur", instance => {
+      dispatch("change", instance.getValue())
+    })
+  }
+
+  // Export a function to expose caret position
+  export const getCaretPosition = () => {
+    const cursor = editor.getCursor()
+    return {
+      start: cursor.ch,
+      end: cursor.ch,
+    }
+  }
+
+  onMount(() => {
+    // Create the editor with initial value
+    createEditor(mode, value)
+
+    // Clean up editor on unmount
+    return () => {
+      if (editor) {
+        editor.toTextArea()
+      }
+    }
+  })
+</script>
+
+{#if label}
+  <div style="margin-bottom: var(--spacing-s)">
+    <Label small>{label}</Label>
+  </div>
+{/if}
+<div
+  style={`--code-mirror-height: ${height}px; --code-mirror-resize: ${resize}`}
+>
+  <textarea tabindex="0" bind:this={textarea} readonly {value} />
+</div>
+
+<style>
+  div :global(.CodeMirror) {
+    height: var(--code-mirror-height);
+    min-height: var(--code-mirror-height);
+    font-family: monospace;
+    line-height: 1.3;
+    border: var(--spectrum-alias-border-size-thin) solid;
+    border-color: var(--spectrum-alias-border-color);
+    border-radius: var(--border-radius-s);
+    resize: var(--code-mirror-resize);
+    overflow: hidden;
+  }
+
+  /* Override default active line highlight colour in dark theme */
+  div
+    :global(.CodeMirror-focused.cm-s-tomorrow-night-eighties
+      .CodeMirror-activeline-background) {
+    background: rgba(255, 255, 255, 0.075);
+  }
+
+  /* Remove active line styling when not focused */
+  div
+    :global(.CodeMirror:not(.CodeMirror-focused)
+      .CodeMirror-activeline-background) {
+    background: unset;
+  }
+
+  /* Add a spectrum themed border when focused */
+  div :global(.CodeMirror-focused) {
+    border-color: var(--spectrum-alias-border-color-mouse-focus);
+  }
+</style>

packages/builder/src/components/common/bindings/BindingPanel.svelte

@@ -1,32 +1,98 @@
 <script>
   import groupBy from "lodash/fp/groupBy"
-  import { Search, TextArea, DrawerContent } from "@budibase/bbui"
+  import {
-  import { createEventDispatcher } from "svelte"
+    Search,
-  import { isValid } from "@budibase/string-templates"
+    TextArea,
+    DrawerContent,
+    Tabs,
+    Tab,
+    Body,
+    Layout,
+  } from "@budibase/bbui"
+  import { createEventDispatcher, onMount } from "svelte"
+  import {
+    isValid,
+    decodeJSBinding,
+    encodeJSBinding,
+  } from "@budibase/string-templates"
   import { readableToRuntimeBinding } from "builderStore/dataBinding"
   import { handlebarsCompletions } from "constants/completions"
-  import { addToText } from "./utils"
+  import { addHBSBinding, addJSBinding } from "./utils"
+  import CodeMirrorEditor from "components/common/CodeMirrorEditor.svelte"
 
   const dispatch = createEventDispatcher()
 
   export let bindableProperties
   export let value = ""
   export let valid
+  export let allowJS = false
 
   let helpers = handlebarsCompletions()
   let getCaretPosition
   let search = ""
+  let initialValueJS = value?.startsWith("{{ js ")
+  let mode = initialValueJS ? "JavaScript" : "Handlebars"
+  let jsValue = initialValueJS ? value : null
+  let hbsValue = initialValueJS ? null : value
 
-  $: valid = isValid(readableToRuntimeBinding(bindableProperties, value))
+  $: usingJS = mode === "JavaScript"
-  $: dispatch("change", value)
   $: ({ context } = groupBy("type", bindableProperties))
   $: searchRgx = new RegExp(search, "ig")
-  $: filteredColumns = context?.filter(context => {
+  $: filteredBindings = context?.filter(context => {
     return context.readableBinding.match(searchRgx)
   })
   $: filteredHelpers = helpers?.filter(helper => {
     return helper.label.match(searchRgx) || helper.description.match(searchRgx)
   })
+
+  const updateValue = value => {
+    valid = isValid(readableToRuntimeBinding(bindableProperties, value))
+    if (valid) {
+      dispatch("change", value)
+    }
+  }
+
+  // Adds a HBS helper to the expression
+  const addHelper = helper => {
+    hbsValue = addHBSBinding(value, getCaretPosition(), helper.text)
+    updateValue(hbsValue)
+  }
+
+  // Adds a data binding to the expression
+  const addBinding = binding => {
+    if (usingJS) {
+      let js = decodeJSBinding(jsValue)
+      js = addJSBinding(js, getCaretPosition(), binding.readableBinding)
+      jsValue = encodeJSBinding(js)
+      updateValue(jsValue)
+    } else {
+      hbsValue = addHBSBinding(
+        hbsValue,
+        getCaretPosition(),
+        binding.readableBinding
+      )
+      updateValue(hbsValue)
+    }
+  }
+
+  const onChangeMode = e => {
+    mode = e.detail
+    updateValue(mode === "JavaScript" ? jsValue : hbsValue)
+  }
+
+  const onChangeHBSValue = e => {
+    hbsValue = e.detail
+    updateValue(hbsValue)
+  }
+
+  const onChangeJSValue = e => {
+    jsValue = encodeJSBinding(e.detail)
+    updateValue(jsValue)
+  }
+
+  onMount(() => {
+    valid = isValid(readableToRuntimeBinding(bindableProperties, value))
+  })
 </script>
 
 <DrawerContent>
@@ -36,32 +102,24 @@
         <div class="heading">Search</div>
         <Search placeholder="Search" bind:value={search} />
       </section>
-      {#if filteredColumns?.length}
+      {#if filteredBindings?.length}
         <section>
           <div class="heading">Bindable Values</div>
           <ul>
-            {#each filteredColumns as { readableBinding }}
+            {#each filteredBindings as binding}
-              <li
+              <li on:click={() => addBinding(binding)}>
-                on:click={() => {
+                {binding.readableBinding}
-                  value = addToText(value, getCaretPosition(), readableBinding)
-                }}
-              >
-                {readableBinding}
               </li>
             {/each}
           </ul>
         </section>
       {/if}
-      {#if filteredHelpers?.length}
+      {#if filteredHelpers?.length && !usingJS}
         <section>
           <div class="heading">Helpers</div>
           <ul>
             {#each filteredHelpers as helper}
-              <li
+              <li on:click={() => addHelper(helper)}>
-                on:click={() => {
-                  value = addToText(value, getCaretPosition(), helper.text)
-                }}
-              >
                 <div class="helper">
                   <div class="helper__name">{helper.displayText}</div>
                   <div class="helper__description">
@@ -77,24 +135,56 @@
     </div>
   </svelte:fragment>
   <div class="main">
-    <TextArea
+    <Tabs selected={mode} on:select={onChangeMode}>
-      bind:getCaretPosition
+      <Tab title="Handlebars">
-      bind:value
+        <div class="main-content">
-      placeholder="Add text, or click the objects on the left to add them to the textbox."
+          <TextArea
-    />
+            bind:getCaretPosition
-    {#if !valid}
+            value={hbsValue}
-      <p class="syntax-error">
+            on:change={onChangeHBSValue}
-        Current Handlebars syntax is invalid, please check the guide
+            placeholder="Add text, or click the objects on the left to add them to the textbox."
-        <a href="https://handlebarsjs.com/guide/">here</a>
+          />
-        for more details.
+          {#if !valid}
-      </p>
+            <p class="syntax-error">
-    {/if}
+              Current Handlebars syntax is invalid, please check the guide
+              <a href="https://handlebarsjs.com/guide/">here</a>
+              for more details.
+            </p>
+          {/if}
+        </div>
+      </Tab>
+      {#if allowJS}
+        <Tab title="JavaScript">
+          <div class="main-content">
+            <Layout noPadding gap="XS">
+              <CodeMirrorEditor
+                bind:getCaretPosition
+                height={200}
+                value={decodeJSBinding(jsValue)}
+                on:change={onChangeJSValue}
+                hints={context?.map(x => `$("${x.readableBinding}")`)}
+              />
+              <Body size="S">
+                JavaScript expressions are executed as functions, so ensure that
+                your expression returns a value.
+              </Body>
+            </Layout>
+          </div>
+        </Tab>
+      {/if}
+    </Tabs>
   </div>
 </DrawerContent>
 
 <style>
   .main :global(textarea) {
-    min-height: 150px !important;
+    min-height: 202px !important;
+  }
+  .main {
+    margin: calc(-1 * var(--spacing-xl));
+  }
+  .main-content {
+    padding: var(--spacing-s) var(--spacing-xl);
   }
 
   .container {

packages/builder/src/components/common/bindings/DrawerBindableCombobox.svelte

@@ -6,6 +6,7 @@
   } from "builderStore/dataBinding"
   import BindingPanel from "components/common/bindings/BindingPanel.svelte"
   import { createEventDispatcher } from "svelte"
+  import { isJSBinding } from "@budibase/string-templates"
 
   export let panel = BindingPanel
   export let value = ""
@@ -15,11 +16,14 @@
   export let label
   export let disabled = false
   export let options
+  export let allowJS = true
 
   const dispatch = createEventDispatcher()
   let bindingDrawer
+
   $: readableValue = runtimeToReadableBinding(bindings, value)
   $: tempValue = readableValue
+  $: isJS = isJSBinding(value)
 
   const handleClose = () => {
     onChange(tempValue)
@@ -35,7 +39,7 @@
   <Combobox
     {label}
     {disabled}
-    value={readableValue}
+    value={isJS ? "(JavaScript function)" : readableValue}
     on:change={event => onChange(event.detail)}
     {placeholder}
     {options}
@@ -58,6 +62,7 @@
     close={handleClose}
     on:change={event => (tempValue = event.detail)}
     bindableProperties={bindings}
+    {allowJS}
   />
 </Drawer>
 

packages/builder/src/components/common/bindings/DrawerBindableInput.svelte

@@ -6,6 +6,7 @@
   } from "builderStore/dataBinding"
   import BindingPanel from "components/common/bindings/BindingPanel.svelte"
   import { createEventDispatcher } from "svelte"
+  import { isJSBinding } from "@budibase/string-templates"
 
   export let panel = BindingPanel
   export let value = ""
@@ -15,12 +16,15 @@
   export let label
   export let disabled = false
   export let fillWidth
+  export let allowJS = true
 
   const dispatch = createEventDispatcher()
   let bindingDrawer
   let valid = true
+
   $: readableValue = runtimeToReadableBinding(bindings, value)
   $: tempValue = readableValue
+  $: isJS = isJSBinding(value)
 
   const saveBinding = () => {
     onChange(tempValue)
@@ -36,7 +40,7 @@
   <Input
     {label}
     {disabled}
-    value={readableValue}
+    value={isJS ? "(JavaScript function)" : readableValue}
     on:change={event => onChange(event.detail)}
     {placeholder}
   />
@@ -60,6 +64,7 @@
     value={readableValue}
     on:change={event => (tempValue = event.detail)}
     bindableProperties={bindings}
+    {allowJS}
   />
 </Drawer>
 

packages/builder/src/components/common/bindings/ServerBindingPanel.svelte

@@ -5,7 +5,7 @@
   import { isValid } from "@budibase/string-templates"
   import { handlebarsCompletions } from "constants/completions"
   import { readableToRuntimeBinding } from "builderStore/dataBinding"
-  import { addToText } from "./utils"
+  import { addHBSBinding } from "./utils"
 
   const dispatch = createEventDispatcher()
 
@@ -47,7 +47,7 @@
               {#each bindings as binding}
                 <li
                   on:click={() => {
-                    value = addToText(value, getCaretPosition(), binding)
+                    value = addHBSBinding(value, getCaretPosition(), binding)
                   }}
                 >
                   <span class="binding__label">{binding.label}</span>
@@ -71,7 +71,7 @@
             {#each filteredHelpers as helper}
               <li
                 on:click={() => {
-                  value = addToText(value, getCaretPosition(), helper.text)
+                  value = addHBSBinding(value, getCaretPosition(), helper.text)
                 }}
               >
                 <div class="helper">

packages/builder/src/components/common/bindings/utils.js

@@ -1,4 +1,4 @@
-export function addToText(value, caretPos, binding) {
+export function addHBSBinding(value, caretPos, binding) {
   binding = typeof binding === "string" ? binding : binding.path
   value = value == null ? "" : value
   if (!value.includes("{{") && !value.includes("}}")) {
@@ -14,3 +14,18 @@ export function addToText(value, caretPos, binding) {
   }
   return value
 }
+
+export function addJSBinding(value, caretPos, binding) {
+  binding = typeof binding === "string" ? binding : binding.path
+  value = value == null ? "" : value
+  binding = `$("${binding}")`
+  if (caretPos.start) {
+    value =
+      value.substring(0, caretPos.start) +
+      binding +
+      value.substring(caretPos.end, value.length)
+  } else {
+    value += binding
+  }
+  return value
+}

packages/builder/src/components/design/NavigationPanel/ComponentNavigationTree/ComponentTree.svelte

@@ -11,6 +11,8 @@
   export let level = 0
   export let dragDropStore
 
+  let closedNodes = {}
+
   const selectComponent = component => {
     store.actions.components.select(component)
   }
@@ -51,6 +53,15 @@
       "component"
     return capitalise(type)
   }
+
+  function toggleNodeOpen(componentId) {
+    if (closedNodes[componentId]) {
+      delete closedNodes[componentId]
+    } else {
+      closedNodes[componentId] = true
+    }
+    closedNodes = closedNodes
+  }
 </script>
 
 <ul>
@@ -71,16 +82,18 @@
         on:dragend={dragDropStore.actions.reset}
         on:dragstart={dragstart(component)}
         on:dragover={dragover(component, index)}
+        on:iconClick={() => toggleNodeOpen(component._id)}
         on:drop={dragDropStore.actions.drop}
         text={getComponentText(component)}
         withArrow
         indentLevel={level + 1}
         selected={$store.selectedComponentId === component._id}
+        opened={!closedNodes[component._id] && component?._children?.length}
       >
         <ComponentDropdownMenu {component} />
       </NavItem>
 
-      {#if component._children}
+      {#if component._children && !closedNodes[component._id]}
         <svelte:self
           components={component._children}
           {currentComponent}

packages/builder/src/components/design/PropertiesPanel/PropertyControls/FormFieldSelect.svelte

@@ -20,12 +20,20 @@
   $: schema = getSchemaForDatasource($currentAsset, datasource, true).schema
   $: options = getOptions(schema, type)
 
-  const getOptions = (schema, fieldType) => {
+  const getOptions = (schema, type) => {
     let entries = Object.entries(schema ?? {})
-    if (fieldType) {
+
-      fieldType = fieldType.split("/")[1]
+    let types = []
-      entries = entries.filter(entry => entry[1].type === fieldType)
+    if (type === "field/options") {
+      // allow options to be used on both options and string fields
+      types = [type, "field/string"]
+    } else {
+      types = [type]
     }
+
+    types = types.map(type => type.split("/")[1])
+    entries = entries.filter(entry => types.includes(entry[1].type))
+
     return entries.map(entry => entry[0])
   }
 </script>

packages/builder/src/components/design/PropertiesPanel/PropertyControls/Input.svelte

@@ -0,0 +1,15 @@
+<script>
+  import { Input } from "@budibase/bbui"
+  import { isJSBinding } from "@budibase/string-templates"
+
+  export let value
+
+  $: isJS = isJSBinding(value)
+</script>
+
+<Input
+  {...$$props}
+  value={isJS ? "(JavaScript function)" : value}
+  readonly={isJS}
+  on:change
+/>

packages/builder/src/components/design/PropertiesPanel/PropertyControls/PropertyControl.svelte

@@ -105,6 +105,7 @@
           value={safeValue}
           on:change={e => (tempValue = e.detail)}
           bindableProperties={bindings}
+          allowJS
         />
       </Drawer>
     {/if}

packages/builder/src/components/design/PropertiesPanel/PropertyControls/componentSettings.js

@@ -1,4 +1,4 @@
-import { Checkbox, Input, Select, Stepper } from "@budibase/bbui"
+import { Checkbox, Select, Stepper } from "@budibase/bbui"
 import DataSourceSelect from "./DataSourceSelect.svelte"
 import DataProviderSelect from "./DataProviderSelect.svelte"
 import EventsEditor from "./EventsEditor"
@@ -15,6 +15,7 @@ import URLSelect from "./URLSelect.svelte"
 import OptionsEditor from "./OptionsEditor/OptionsEditor.svelte"
 import FormFieldSelect from "./FormFieldSelect.svelte"
 import ValidationEditor from "./ValidationEditor/ValidationEditor.svelte"
+import Input from "./Input.svelte"
 
 const componentMap = {
   text: Input,

packages/builder/src/components/integration/QueryViewer.svelte

@@ -21,12 +21,15 @@
   import ParameterBuilder from "components/integration/QueryParameterBuilder.svelte"
   import { datasources, integrations, queries } from "stores/backend"
   import { capitalise } from "../../helpers"
+  import CodeMirrorEditor from "components/common/CodeMirrorEditor.svelte"
 
   export let query
   export let fields = []
 
   let parameters
   let data = []
+  const transformerDocs =
+    "https://docs.budibase.com/building-apps/data/transformers"
   const typeOptions = [
     { label: "Text", value: "STRING" },
     { label: "Number", value: "NUMBER" },
@@ -52,6 +55,11 @@
   $: readQuery = query.queryVerb === "read" || query.readable
   $: queryInvalid = !query.name || (readQuery && data.length === 0)
 
+  // seed the transformer
+  if (query && !query.transformer) {
+    query.transformer = "return data"
+  }
+
   function newField() {
     fields = [...fields, {}]
   }
@@ -74,6 +82,7 @@
       const response = await api.post(`/api/queries/preview`, {
         fields: query.fields,
         queryVerb: query.queryVerb,
+        transformer: query.transformer,
         parameters: query.parameters.reduce(
           (acc, next) => ({
             ...acc,
@@ -160,12 +169,34 @@
       <IntegrationQueryEditor
         {datasource}
         {query}
-        height={300}
+        height={200}
         schema={queryConfig[query.queryVerb]}
         bind:parameters
       />
       <Divider />
     </div>
+    <div class="config">
+      <div class="help-heading">
+        <Heading size="S">Transformer</Heading>
+        <Icon
+          on:click={() => window.open(transformerDocs)}
+          hoverable
+          name="Help"
+          size="L"
+        />
+      </div>
+      <Body size="S"
+        >Add a JavaScript function to transform the query result.</Body
+      >
+      <CodeMirrorEditor
+        height={200}
+        label="Transformer"
+        value={query.transformer}
+        resize="vertical"
+        on:change={e => (query.transformer = e.detail)}
+      />
+      <Divider />
+    </div>
     <div class="viewer-controls">
       <Heading size="S">Results</Heading>
       <ButtonGroup>
@@ -220,6 +251,7 @@
     display: grid;
     grid-gap: var(--spacing-s);
   }
+
   .config-field {
     display: grid;
     grid-template-columns: 20% 1fr;
@@ -227,6 +259,11 @@
     align-items: center;
   }
 
+  .help-heading {
+    display: flex;
+    justify-content: space-between;
+  }
+
   .field {
     display: grid;
     grid-template-columns: 1fr 1fr 5%;

packages/builder/src/components/integration/codemirror.js

@@ -1,12 +1,22 @@
 import CodeMirror from "codemirror"
 import "codemirror/lib/codemirror.css"
-import "codemirror/theme/tomorrow-night-eighties.css"
+
-import "codemirror/addon/hint/show-hint.css"
+// Modes
-import "codemirror/theme/neo.css"
+import "codemirror/mode/javascript/javascript"
 import "codemirror/mode/sql/sql"
 import "codemirror/mode/css/css"
 import "codemirror/mode/handlebars/handlebars"
-import "codemirror/mode/javascript/javascript"
+
+// Hints
 import "codemirror/addon/hint/show-hint"
+import "codemirror/addon/hint/show-hint.css"
+
+// Theming
+import "codemirror/theme/tomorrow-night-eighties.css"
+
+// Functional addons
+import "codemirror/addon/selection/active-line"
+import "codemirror/addon/edit/closebrackets"
+import "codemirror/addon/edit/matchbrackets"
 
 export default CodeMirror

packages/builder/src/components/settings/UpdateUserInfoModal.svelte

@@ -26,6 +26,7 @@
   <Body size="S">
     Personalise the platform by adding your first name and last name.
   </Body>
+  <Input disabled bind:value={$auth.user.email} label="Email" />
   <Input bind:value={$values.firstName} label="First name" />
   <Input bind:value={$values.lastName} label="Last name" />
 </ModalContent>

packages/builder/src/components/start/CreateAppModal.svelte

@@ -20,6 +20,7 @@
   import TemplateList from "./TemplateList.svelte"
 
   export let template
+  export let inline
 
   const values = writable({ name: null })
   const errors = writable({})
@@ -39,9 +40,10 @@
 
   let submitting = false
   let valid = false
+  let initialTemplateInfo = template?.fromFile || template?.key
 
   $: checkValidity($values, validator)
-  $: showTemplateSelection = !template?.fromFile && !template?.key
+  $: showTemplateSelection = !template && !initialTemplateInfo
 
   onMount(async () => {
     await hostingStore.actions.fetchDeployedApps()
@@ -64,6 +66,11 @@
   const checkValidity = async (values, validator) => {
     const obj = object().shape(validator)
     Object.keys(validator).forEach(key => ($errors[key] = null))
+    if (template?.fromFile && values.file == null) {
+      valid = false
+      return
+    }
+
     try {
       await obj.validate(values, { abortEarly: false })
     } catch (validationErrors) {
@@ -71,14 +78,17 @@
         $errors[error.path] = capitalise(error.message)
       })
     }
+
     valid = await obj.isValid(values)
   }
 
   async function createNewApp() {
+    const letTemplateToUse =
+      Object.keys(template).length === 0 ? null : template
     submitting = true
 
     // Check a template exists if we are important
-    if (template?.fromFile && !$values.file) {
+    if (letTemplateToUse?.fromFile && !$values.file) {
       $errors.file = "Please choose a file to import"
       valid = false
       submitting = false
@@ -89,10 +99,10 @@
       // Create form data to create app
       let data = new FormData()
       data.append("name", $values.name.trim())
-      data.append("useTemplate", template != null)
+      data.append("useTemplate", letTemplateToUse != null)
-      if (template) {
+      if (letTemplateToUse) {
-        data.append("templateName", template.name)
+        data.append("templateName", letTemplateToUse.name)
-        data.append("templateKey", template.key)
+        data.append("templateKey", letTemplateToUse.key)
         data.append("templateFile", $values.file)
       }
 
@@ -106,7 +116,7 @@
       analytics.captureEvent(Events.APP.CREATED, {
         name: $values.name,
         appId: appJson.instance._id,
-        template,
+        letTemplateToUse,
       })
 
       // Select Correct Application/DB in prep for creating user
@@ -144,20 +154,18 @@
     showConfirmButton={false}
     size="L"
     onConfirm={() => {
-      showTemplateSelection = false
+      template = {}
       return false
     }}
-    showCancelButton={false}
+    showCancelButton={!inline}
-    showCloseIcon={false}
+    showCloseIcon={!inline}
   >
-    <Body size="M">Select a template below, or start from scratch.</Body>
     <TemplateList
-      onSelect={selected => {
+      onSelect={(selected, { useImport } = {}) => {
         if (!selected) {
-          showTemplateSelection = false
+          template = useImport ? { fromFile: true } : {}
           return
         }
-
         template = selected
       }}
     />
@@ -167,6 +175,9 @@
     title={template?.fromFile ? "Import app" : "Create app"}
     confirmText={template?.fromFile ? "Import app" : "Create app"}
     onConfirm={createNewApp}
+    onCancel={inline ? () => (template = null) : null}
+    cancelText={inline ? "Back" : undefined}
+    showCloseIcon={!inline}
     disabled={!valid}
   >
     {#if template?.fromFile}

packages/builder/src/components/start/TemplateList.svelte

@@ -1,5 +1,5 @@
 <script>
-  import { Heading, Layout, Icon } from "@budibase/bbui"
+  import { Heading, Layout, Icon, Body } from "@budibase/bbui"
   import Spinner from "components/common/Spinner.svelte"
   import api from "builderStore/api"
 
@@ -19,6 +19,11 @@
       <Spinner size="30" />
     </div>
   {:then templates}
+    {#if templates?.length > 0}
+      <Body size="M">Select a template below, or start from scratch.</Body>
+    {:else}
+      <Body size="M">Start your app from scratch below.</Body>
+    {/if}
     <div class="templates">
       {#each templates as template}
         <div class="template" on:click={() => onSelect(template)}>
@@ -42,6 +47,19 @@
         <Heading size="XS">Start from scratch</Heading>
         <p class="detail">BLANK</p>
       </div>
+      <div
+        class="template import"
+        on:click={() => onSelect(null, { useImport: true })}
+      >
+        <div
+          class="background-icon"
+          style={`background: rgb(50, 50, 50); color: white;`}
+        >
+          <Icon name="Add" />
+        </div>
+        <Heading size="XS">Import an app</Heading>
+        <p class="detail">BLANK</p>
+      </div>
     </div>
   {:catch err}
     <h1 style="color:red">{err}</h1>
@@ -69,10 +87,10 @@
   }
 
   .template {
-    height: 60px;
+    min-height: 60px;
     display: grid;
-    grid-gap: var(--layout-m);
+    grid-gap: var(--layout-s);
-    grid-template-columns: 5% 1fr 15%;
+    grid-template-columns: auto 1fr auto;
     border: 1px solid #494949;
     align-items: center;
     cursor: pointer;
@@ -89,4 +107,8 @@
     background: var(--spectrum-global-color-gray-50);
     margin-top: 20px;
   }
+
+  .import {
+    background: var(--spectrum-global-color-gray-50);
+  }
 </style>

packages/builder/src/helpers/fetchTableData.js

@@ -48,7 +48,6 @@ export const fetchTableData = opts => {
   const fetchPage = async bookmark => {
     lastBookmark = bookmark
     const { tableId, limit, sortColumn, sortOrder, paginate } = options
-    store.update($store => ({ ...$store, loading: true }))
     const res = await API.post(`/api/${options.tableId}/search`, {
       tableId,
       query,
@@ -59,7 +58,6 @@ export const fetchTableData = opts => {
       paginate,
       bookmark,
     })
-    store.update($store => ({ ...$store, loading: false, loaded: true }))
     return await res.json()
   }
 
@@ -103,7 +101,7 @@ export const fetchTableData = opts => {
     if (!schema) {
       return
     }
-    store.update($store => ({ ...$store, schema }))
+    store.update($store => ({ ...$store, schema, loading: true }))
 
     // Work out what sort type to use
     if (!sortColumn || !schema[sortColumn]) {
@@ -135,6 +133,7 @@ export const fetchTableData = opts => {
     }
 
     // Fetch next page
+    store.update($store => ({ ...$store, loading: true }))
     const page = await fetchPage(state.bookmarks[state.pageNumber + 1])
 
     // Update state
@@ -148,6 +147,7 @@ export const fetchTableData = opts => {
         pageNumber: pageNumber + 1,
         rows: page.rows,
         bookmarks,
+        loading: false,
       }
     })
   }
@@ -160,6 +160,7 @@ export const fetchTableData = opts => {
     }
 
     // Fetch previous page
+    store.update($store => ({ ...$store, loading: true }))
     const page = await fetchPage(state.bookmarks[state.pageNumber - 1])
 
     // Update state
@@ -168,6 +169,7 @@ export const fetchTableData = opts => {
         ...$store,
         pageNumber: $store.pageNumber - 1,
         rows: page.rows,
+        loading: false,
       }
     })
   }

packages/builder/src/pages/builder/admin/index.svelte

@@ -39,7 +39,7 @@
       await admin.init()
       $goto("../portal")
     } catch (err) {
-      notifications.error(`Failed to create admin user`)
+      notifications.error(`Failed to create admin user: ${err}`)
     }
   }
 

packages/builder/src/pages/builder/app/[application]/_layout.svelte

@@ -1,21 +1,24 @@
 <script>
   import { store, automationStore } from "builderStore"
   import { roles } from "stores/backend"
-  import { Icon, ActionGroup, Tabs, Tab } from "@budibase/bbui"
+  import { Icon, ActionGroup, Tabs, Tab, notifications } from "@budibase/bbui"
   import DeployModal from "components/deploy/DeployModal.svelte"
   import RevertModal from "components/deploy/RevertModal.svelte"
   import VersionModal from "components/deploy/VersionModal.svelte"
   import NPSFeedbackForm from "components/feedback/NPSFeedbackForm.svelte"
-  import { get } from "builderStore/api"
+  import { get, post } from "builderStore/api"
   import { auth, admin } from "stores/portal"
   import { isActive, goto, layout, redirect } from "@roxi/routify"
   import Logo from "assets/bb-emblem.svg"
   import { capitalise } from "helpers"
   import UpgradeModal from "../../../../components/upgrade/UpgradeModal.svelte"
+  import { onMount } from "svelte"
 
   // Get Package and set store
   export let application
   let promise = getPackage()
+  // sync once when you load the app
+  let hasSynced = false
   $: selected = capitalise(
     $layout.children.find(layout => $isActive(layout.path))?.title ?? "data"
   )
@@ -67,6 +70,16 @@
       return state
     })
   }
+
+  onMount(async () => {
+    if (!hasSynced && application) {
+      const res = await post(`/api/applications/${application}/sync`)
+      if (res.status !== 200) {
+        notifications.error("Failed to sync with production database")
+      }
+      hasSynced = true
+    }
+  })
 </script>
 
 {#await promise}

packages/builder/src/pages/builder/app/[application]/data/index.svelte

@@ -1,6 +1,7 @@
 <script>
   import { goto } from "@roxi/routify"
   import { onMount } from "svelte"
+  import { admin } from "stores/portal"
   import CreateDatasourceModal from "components/backend/DatasourceNavigator/modals/CreateDatasourceModal.svelte"
   import { datasources } from "stores/backend"
 
@@ -10,7 +11,7 @@
     $datasources.list.length > 1
 
   onMount(() => {
-    if (!setupComplete) {
+    if (!setupComplete && !$admin.isDev) {
       modal.show()
     } else {
       $goto("./table")

packages/builder/src/pages/builder/apps/index.svelte

@@ -34,6 +34,12 @@
   const publishedAppsOnly = app => app.status === AppStatus.DEPLOYED
 
   $: publishedApps = $apps.filter(publishedAppsOnly)
+
+  $: userApps = $auth.user?.builder?.global
+    ? publishedApps
+    : publishedApps.filter(app =>
+        Object.keys($auth.user?.roles).includes(app.prodId)
+      )
 </script>
 
 {#if $auth.user && loaded}
@@ -82,11 +88,11 @@
             </Body>
           </Layout>
           <Divider />
-          {#if publishedApps.length}
+          {#if userApps.length}
             <Heading>Apps</Heading>
             <div class="group">
               <Layout gap="S" noPadding>
-                {#each publishedApps as app, idx (app.appId)}
+                {#each userApps as app, idx (app.appId)}
                   <a class="app" target="_blank" href={`/${app.prodId}`}>
                     <div class="preview" use:gradient={{ seed: app.name }} />
                     <div class="app-info">

packages/builder/src/pages/builder/portal/apps/index.svelte

@@ -112,16 +112,8 @@
 
   const exportApp = app => {
     const id = app.deployed ? app.prodId : app.devId
-    try {
+    const appName = encodeURIComponent(app.name)
-      download(
+    window.location = `/api/backups/export?appId=${id}&appname=${appName}`
-        `/api/backups/export?appId=${id}&appname=${encodeURIComponent(
-          app.name
-        )}`
-      )
-      notifications.success("App exported successfully")
-    } catch (err) {
-      notifications.error(`Error exporting app: ${err}`)
-    }
   }
 
   const unpublishApp = app => {
@@ -268,7 +260,7 @@
   {#if !enrichedApps.length && !creatingApp && loaded}
     <div class="empty-wrapper">
       <Modal inline>
-        <CreateAppModal {template} />
+        <CreateAppModal {template} inline={true} />
       </Modal>
     </div>
   {/if}

packages/builder/src/pages/builder/portal/manage/users/[userId].svelte

@@ -34,9 +34,13 @@
     role: {},
   }
 
-  $: defaultRoleId = $userFetch?.data?.builder?.global ? "ADMIN" : "BASIC"
+  const noRoleSchema = {
+    name: { displayName: "App" },
+  }
+
+  $: defaultRoleId = $userFetch?.data?.builder?.global ? "ADMIN" : ""
   // Merge the Apps list and the roles response to get something that makes sense for the table
-  $: appList = Object.keys($apps?.data).map(id => {
+  $: allAppList = Object.keys($apps?.data).map(id => {
     const roleId = $userFetch?.data?.roles?.[id] || defaultRoleId
     const role = $apps?.data?.[id].roles.find(role => role._id === roleId)
     return {
@@ -45,6 +49,15 @@
       role: [role],
     }
   })
+
+  $: appList = allAppList.filter(app => !!app.role[0])
+  $: noRoleAppList = allAppList
+    .filter(app => !app.role[0])
+    .map(app => {
+      delete app.role
+      return app
+    })
+
   let selectedApp
 
   const userFetch = fetchData(`/api/global/users/${userId}`)
@@ -173,6 +186,7 @@
   <Divider size="S" />
   <Layout gap="S" noPadding>
     <Heading size="S">Configure roles</Heading>
+    <Body>Specify a role to grant access to an app.</Body>
     <Table
       on:click={openUpdateRolesModal}
       schema={roleSchema}
@@ -183,6 +197,21 @@
       customRenderers={[{ column: "role", component: TagsRenderer }]}
     />
   </Layout>
+  <Layout gap="S" noPadding>
+    <Heading size="XS">No Access</Heading>
+    <Body
+      >Apps do not appear in the users portal. Public pages may still be viewed
+      if visited directly.</Body
+    >
+    <Table
+      on:click={openUpdateRolesModal}
+      schema={noRoleSchema}
+      data={noRoleAppList}
+      allowEditColumns={false}
+      allowEditRows={false}
+      allowSelectRows={false}
+    />
+  </Layout>
   <Divider size="S" />
   <Layout gap="XS" noPadding>
     <Heading size="S">Delete user</Heading>

packages/builder/src/pages/builder/portal/manage/users/_components/BasicOnboardingModal.svelte

@@ -16,7 +16,13 @@
     admin = false
 
   async function createUser() {
-    const res = await users.create({ email: $email, password, builder, admin })
+    const res = await users.create({
+      email: $email,
+      password,
+      builder,
+      admin,
+      forceResetPassword: true,
+    })
     if (res.status) {
       notifications.error(res.message)
     } else {

packages/builder/src/pages/builder/portal/manage/users/_components/UpdateRolesModal.svelte

@@ -6,22 +6,40 @@
   export let app
   export let user
 
+  const NO_ACCESS = "NO_ACCESS"
+
   const dispatch = createEventDispatcher()
 
   const roles = app.roles
   let options = roles
     .filter(role => role._id !== "PUBLIC")
     .map(role => ({ value: role._id, label: role.name }))
+  options.push({ value: NO_ACCESS, label: "No Access" })
   let selectedRole = user?.roles?.[app?._id]
 
   async function updateUserRoles() {
-    const res = await users.save({
+    let res
-      ...user,
+    if (selectedRole === NO_ACCESS) {
-      roles: {
+      // remove the user role
-        ...user.roles,
+      const filteredRoles = { ...user.roles }
-        [app._id]: selectedRole,
+      delete filteredRoles[app?._id]
-      },
+      res = await users.save({
-    })
+        ...user,
+        roles: {
+          ...filteredRoles,
+        },
+      })
+    } else {
+      // add the user role
+      res = await users.save({
+        ...user,
+        roles: {
+          ...user.roles,
+          [app._id]: selectedRole,
+        },
+      })
+    }
+
     if (res.status === 400) {
       notifications.error("Failed to update role")
     } else {

packages/builder/src/stores/portal/admin.js

@@ -7,6 +7,7 @@ export function createAdminStore() {
     loaded: false,
     multiTenancy: false,
     cloud: false,
+    isDev: false,
     disableAccountPortal: false,
     accountPortalUrl: "",
     importComplete: false,
@@ -62,6 +63,7 @@ export function createAdminStore() {
     let cloud = false
     let disableAccountPortal = false
     let accountPortalUrl = ""
+    let isDev = false
     try {
       const response = await api.get(`/api/system/environment`)
       const json = await response.json()
@@ -69,6 +71,7 @@ export function createAdminStore() {
       cloud = json.cloud
       disableAccountPortal = json.disableAccountPortal
       accountPortalUrl = json.accountPortalUrl
+      isDev = json.isDev
     } catch (err) {
       // just let it stay disabled
     }
@@ -77,6 +80,7 @@ export function createAdminStore() {
       store.cloud = cloud
       store.disableAccountPortal = disableAccountPortal
       store.accountPortalUrl = accountPortalUrl
+      store.isDev = isDev
       return store
     })
   }

packages/builder/src/stores/portal/users.js

@@ -35,12 +35,21 @@ export function createUsersStore() {
     return await response.json()
   }
 
-  async function create({ email, password, admin, builder }) {
+  async function create({
+    email,
+    password,
+    admin,
+    builder,
+    forceResetPassword,
+  }) {
     const body = {
       email,
       password,
       roles: {},
     }
+    if (forceResetPassword) {
+      body.forceResetPassword = forceResetPassword
+    }
     if (builder) {
       body.builder = { global: true }
     }

packages/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/cli",
-  "version": "0.9.172",
+  "version": "0.9.173-alpha.3",
   "description": "Budibase CLI, for developers, self hosting and migrations.",
   "main": "src/index.js",
   "bin": {

packages/client/manifest.json

@@ -2385,7 +2385,7 @@
   },
   "dataprovider": {
     "name": "Data Provider",
-    "info": "Pagination is only available for data stored in internal tables.",
+    "info": "Pagination is only available for data stored in tables.",
     "icon": "Data",
     "illegalChildren": ["section"],
     "hasChildren": true,

packages/client/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/client",
-  "version": "0.9.172",
+  "version": "0.9.173-alpha.3",
   "license": "MPL-2.0",
   "module": "dist/budibase-client.js",
   "main": "dist/budibase-client.js",
@@ -19,9 +19,9 @@
     "dev:builder": "rollup -cw"
   },
   "dependencies": {
-    "@budibase/bbui": "^0.9.172",
+    "@budibase/bbui": "^0.9.173-alpha.3",
     "@budibase/standard-components": "^0.9.139",
-    "@budibase/string-templates": "^0.9.172",
+    "@budibase/string-templates": "^0.9.173-alpha.3",
     "regexparam": "^1.3.0",
     "shortid": "^2.2.15",
     "svelte-spa-router": "^3.0.5"

packages/client/src/api/queries.js

@@ -20,7 +20,7 @@ export const executeQuery = async ({ queryId, parameters }) => {
     notificationStore.actions.error("An error has occurred")
   } else if (!query.readable) {
     notificationStore.actions.success("Query executed successfully")
-    dataSourceStore.actions.invalidateDataSource(query.datasourceId)
+    await dataSourceStore.actions.invalidateDataSource(query.datasourceId)
   }
   return res
 }

packages/client/src/api/rows.js

@@ -31,7 +31,7 @@ export const saveRow = async row => {
     : notificationStore.actions.success("Row saved")
 
   // Refresh related datasources
-  dataSourceStore.actions.invalidateDataSource(row.tableId)
+  await dataSourceStore.actions.invalidateDataSource(row.tableId)
 
   return res
 }
@@ -52,7 +52,7 @@ export const updateRow = async row => {
     : notificationStore.actions.success("Row updated")
 
   // Refresh related datasources
-  dataSourceStore.actions.invalidateDataSource(row.tableId)
+  await dataSourceStore.actions.invalidateDataSource(row.tableId)
 
   return res
 }
@@ -76,7 +76,7 @@ export const deleteRow = async ({ tableId, rowId, revId }) => {
     : notificationStore.actions.success("Row deleted")
 
   // Refresh related datasources
-  dataSourceStore.actions.invalidateDataSource(tableId)
+  await dataSourceStore.actions.invalidateDataSource(tableId)
 
   return res
 }
@@ -99,7 +99,7 @@ export const deleteRows = async ({ tableId, rows }) => {
     : notificationStore.actions.success(`${rows.length} row(s) deleted`)
 
   // Refresh related datasources
-  dataSourceStore.actions.invalidateDataSource(tableId)
+  await dataSourceStore.actions.invalidateDataSource(tableId)
 
   return res
 }

packages/client/src/components/ClientApp.svelte

@@ -113,6 +113,13 @@
                     />
                   {/key}
 
+                  <!--
+                    Flatpickr needs to be inside the theme wrapper.
+                    It also needs its own container because otherwise it hijacks
+                    key events on the whole page. It is painful to work with.
+                  -->
+                  <div id="flatpickr-root" />
+
                   <!-- Layers on top of app -->
                   <NotificationDisplay />
                   <ConfirmationDisplay />

packages/client/src/components/CustomThemeWrapper.svelte

@@ -16,7 +16,10 @@
     /*  Buttons */
     --spectrum-semantic-cta-color-background-default: var(--primaryColor);
     --spectrum-semantic-cta-color-background-hover: var(--primaryColorHover);
+    --spectrum-button-primary-s-border-radius: var(--buttonBorderRadius);
     --spectrum-button-primary-m-border-radius: var(--buttonBorderRadius);
+    --spectrum-button-primary-l-border-radius: var(--buttonBorderRadius);
+    --spectrum-button-primary-xl-border-radius: var(--buttonBorderRadius);
 
     /* Loading spinners */
     --spectrum-progresscircle-medium-track-fill-color: var(--primaryColor);

packages/client/src/components/app/forms/DateTimeField.svelte

@@ -12,31 +12,6 @@
 
   let fieldState
   let fieldApi
-
-  const parseDate = val => {
-    if (!val) {
-      return null
-    }
-    let date
-    if (val instanceof Date) {
-      // Use real date obj if already parsed
-      date = val
-    } else if (isNaN(val)) {
-      // Treat as date string of some sort
-      date = new Date(val)
-    } else {
-      // Treat as numerical timestamp
-      date = new Date(parseInt(val))
-    }
-    const time = date.getTime()
-    if (isNaN(time)) {
-      return null
-    }
-    // By rounding to the nearest second we avoid locking up in an endless
-    // loop in the builder, caused by potentially enriching {{ now }} to every
-    // millisecond.
-    return new Date(Math.floor(time / 1000) * 1000)
-  }
 </script>
 
 <Field
@@ -44,7 +19,7 @@
   {field}
   {disabled}
   {validation}
-  defaultValue={parseDate(defaultValue)}
+  {defaultValue}
   type="datetime"
   bind:fieldState
   bind:fieldApi
@@ -56,7 +31,7 @@
       disabled={fieldState.disabled}
       error={fieldState.error}
       id={fieldState.fieldId}
-      appendTo={document.getElementById("theme-root")}
+      appendTo={document.getElementById("flatpickr-root")}
       {enableTime}
       {placeholder}
     />

packages/client/src/components/app/forms/Field.svelte

@@ -22,7 +22,7 @@
 
   // Register field with form
   const formApi = formContext?.formApi
-  const labelPosition = fieldGroupContext?.labelPosition || "above"
+  const labelPos = fieldGroupContext?.labelPosition || "above"
   const formField = formApi?.registerField(
     field,
     type,
@@ -38,17 +38,23 @@
     fieldApi = value?.fieldApi
     fieldSchema = value?.fieldSchema
   })
-  onDestroy(() => unsubscribe && unsubscribe())
+  onDestroy(() => unsubscribe?.())
 
-  // Keep validation rules up to date
+  // Keep field state up to date with props which might change due to
+  // conditional UI
   $: updateValidation(validation)
+  $: updateDisabled(disabled)
+
+  // Determine label class from position
+  $: labelClass = labelPos === "above" ? "" : `spectrum-FieldLabel--${labelPos}`
+
   const updateValidation = validation => {
     fieldApi?.updateValidation(validation)
   }
 
-  // Extract label position from field group context
+  const updateDisabled = disabled => {
-  $: labelPositionClass =
+    fieldApi?.setDisabled(disabled)
-    labelPosition === "above" ? "" : `spectrum-FieldLabel--${labelPosition}`
+  }
 </script>
 
 <FieldGroupFallback>
@@ -56,7 +62,7 @@
     <label
       class:hidden={!label}
       for={fieldState?.fieldId}
-      class={`spectrum-FieldLabel spectrum-FieldLabel--sizeM spectrum-Form-itemLabel ${labelPositionClass}`}
+      class={`spectrum-FieldLabel spectrum-FieldLabel--sizeM spectrum-Form-itemLabel ${labelClass}`}
     >
       {label || ""}
     </label>
@@ -67,7 +73,7 @@
         <Placeholder
           text="Add the Field setting to start using your component"
         />
-      {:else if fieldSchema?.type && fieldSchema?.type !== type}
+      {:else if fieldSchema?.type && fieldSchema?.type !== type && type !== "options"}
         <Placeholder
           text="This Field setting is the wrong data type for this component"
         />

packages/client/src/components/app/forms/InnerForm.svelte

@@ -248,10 +248,25 @@
       }
     }
 
+    // Updates the disabled state of a certain field
+    const setDisabled = fieldDisabled => {
+      const fieldInfo = getField(field)
+
+      // Auto columns are always disabled
+      const isAutoColumn = !!schema?.[field]?.autocolumn
+
+      // Update disabled state
+      fieldInfo.update(state => {
+        state.fieldState.disabled = disabled || fieldDisabled || isAutoColumn
+        return state
+      })
+    }
+
     return {
       setValue,
       clearValue,
       updateValidation,
+      setDisabled,
       validate: () => {
         // Validate the field by force setting the same value again
         const { fieldState } = get(getField(field))

packages/client/src/components/context/DeviceBindingsProvider.svelte

@@ -1,6 +1,6 @@
 <script>
   import Provider from "./Provider.svelte"
-  import { onMount } from "svelte"
+  import { onMount, onDestroy } from "svelte"
 
   let width = window.innerWidth
   let height = window.innerHeight
@@ -21,12 +21,11 @@
   }
 
   onMount(() => {
-    const doc = document.getElementById("app-root")
+    resizeObserver.observe(document.getElementById("app-root"))
-    resizeObserver.observe(doc)
+  })
 
-    return () => {
+  onDestroy(() => {
-      resizeObserver.unobserve(doc)
+    resizeObserver.unobserve(document.getElementById("app-root"))
-    }
   })
 </script>
 

packages/client/src/components/context/Provider.svelte

@@ -1,5 +1,5 @@
 <script>
-  import { getContext, setContext, onMount } from "svelte"
+  import { getContext, setContext, onDestroy } from "svelte"
   import { dataSourceStore, createContextStore } from "stores"
   import { ActionTypes } from "constants"
   import { generate } from "shortid"
@@ -56,9 +56,9 @@
     }
   }
 
-  onMount(() => {
+  onDestroy(() => {
     // Unregister all datasource instances when unmounting this provider
-    return () => dataSourceStore.actions.unregisterInstance(instanceId)
+    dataSourceStore.actions.unregisterInstance(instanceId)
   })
 </script>
 

packages/client/src/components/preview/DNDHandler.svelte

@@ -8,7 +8,7 @@
 </script>
 
 <script>
-  import { onMount } from "svelte"
+  import { onMount, onDestroy } from "svelte"
   import { get } from "svelte/store"
   import IndicatorSet from "./IndicatorSet.svelte"
   import DNDPositionIndicator from "./DNDPositionIndicator.svelte"
@@ -209,18 +209,18 @@
     document.addEventListener("dragenter", onDragEnter, false)
     document.addEventListener("dragleave", onDragLeave, false)
     document.addEventListener("drop", onDrop, false)
+  })
 
-    return () => {
+  onDestroy(() => {
-      // Events fired on the draggable target
+    // Events fired on the draggable target
-      document.removeEventListener("dragstart", onDragStart, false)
+    document.removeEventListener("dragstart", onDragStart, false)
-      document.removeEventListener("dragend", onDragEnd, false)
+    document.removeEventListener("dragend", onDragEnd, false)
 
-      // Events fired on the drop targets
+    // Events fired on the drop targets
-      document.removeEventListener("dragover", onDragOver, false)
+    document.removeEventListener("dragover", onDragOver, false)
-      document.removeEventListener("dragenter", onDragEnter, false)
+    document.removeEventListener("dragenter", onDragEnter, false)
-      document.removeEventListener("dragleave", onDragLeave, false)
+    document.removeEventListener("dragleave", onDragLeave, false)
-      document.removeEventListener("drop", onDrop, false)
+    document.removeEventListener("drop", onDrop, false)
-    }
   })
 </script>
 

packages/client/src/stores/dataSource.js

@@ -1,4 +1,5 @@
 import { writable, get } from "svelte/store"
+import { fetchTableDefinition } from "../api"
 
 export const createDataSourceStore = () => {
   const store = writable([])
@@ -9,43 +10,32 @@ export const createDataSourceStore = () => {
       return
     }
 
-    // Create a list of all relevant dataSource IDs which would require that
+    // Extract the relevant datasource ID for this datasource
-    // this dataSource is refreshed
+    let dataSourceId = null
-    let dataSourceIds = []
 
     // Extract table ID
     if (dataSource.type === "table" || dataSource.type === "view") {
-      if (dataSource.tableId) {
+      dataSourceId = dataSource.tableId
-        dataSourceIds.push(dataSource.tableId)
-      }
     }
 
-    // Extract both table IDs from both sides of the relationship
+    // Only one side of the relationship is required as a trigger, as it will
+    // automatically invalidate related table IDs
     else if (dataSource.type === "link") {
-      if (dataSource.rowTableId) {
+      dataSourceId = dataSource.tableId || dataSource.rowTableId
-        dataSourceIds.push(dataSource.rowTableId)
-      }
-      if (dataSource.tableId) {
-        dataSourceIds.push(dataSource.tableId)
-      }
     }
 
     // Extract the dataSource ID (not the query ID) for queries
     else if (dataSource.type === "query") {
-      if (dataSource.dataSourceId) {
+      dataSourceId = dataSource.dataSourceId
-        dataSourceIds.push(dataSource.dataSourceId)
-      }
     }
 
     // Store configs for each relevant dataSource ID
-    if (dataSourceIds.length) {
+    if (dataSourceId) {
       store.update(state => {
-        dataSourceIds.forEach(id => {
+        state.push({
-          state.push({
+          dataSourceId,
-            dataSourceId: id,
+          instanceId,
-            instanceId,
+          refresh,
-            refresh,
-          })
         })
         return state
       })
@@ -62,13 +52,10 @@ export const createDataSourceStore = () => {
 
   // Invalidates a specific dataSource ID by refreshing all instances
   // which depend on data from that dataSource
-  const invalidateDataSource = dataSourceId => {
+  const invalidateDataSource = async dataSourceId => {
-    const relatedInstances = get(store).filter(instance => {
+    if (!dataSourceId) {
-      return instance.dataSourceId === dataSourceId
+      return
-    })
+    }
-    relatedInstances?.forEach(instance => {
-      instance.refresh()
-    })
 
     // Emit this as a window event, so parent screens which are iframing us in
     // can also invalidate the same datasource
@@ -77,6 +64,36 @@ export const createDataSourceStore = () => {
         detail: { dataSourceId },
       })
     )
+
+    let invalidations = [dataSourceId]
+
+    // Fetch related table IDs from table schema
+    const definition = await fetchTableDefinition(dataSourceId)
+    const schema = definition?.schema
+    if (schema) {
+      Object.values(schema).forEach(fieldSchema => {
+        if (
+          fieldSchema.type === "link" &&
+          fieldSchema.tableId &&
+          !fieldSchema.autocolumn
+        ) {
+          invalidations.push(fieldSchema.tableId)
+        }
+      })
+    }
+
+    // Remove any dupes
+    invalidations = [...new Set(invalidations)]
+
+    // Invalidate all sources
+    invalidations.forEach(id => {
+      const relatedInstances = get(store).filter(instance => {
+        return instance.dataSourceId === id
+      })
+      relatedInstances?.forEach(instance => {
+        instance.refresh()
+      })
+    })
   }
 
   return {

packages/server/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "0.9.172",
+  "version": "0.9.173-alpha.3",
   "description": "Budibase Web Server",
   "main": "src/index.js",
   "repository": {
@@ -68,13 +68,13 @@
   "author": "Budibase",
   "license": "AGPL-3.0-or-later",
   "dependencies": {
-    "@budibase/auth": "^0.9.172",
+    "@budibase/auth": "^0.9.173-alpha.3",
-    "@budibase/client": "^0.9.172",
+    "@budibase/client": "^0.9.173-alpha.3",
-    "@budibase/string-templates": "^0.9.172",
+    "@budibase/string-templates": "^0.9.173-alpha.3",
     "@elastic/elasticsearch": "7.10.0",
     "@koa/router": "8.0.0",
     "@sendgrid/mail": "7.1.1",
-    "@sentry/node": "5.19.2",
+    "@sentry/node": "^6.0.0",
     "airtable": "0.10.1",
     "arangojs": "7.2.0",
     "aws-sdk": "^2.767.0",
@@ -103,7 +103,7 @@
     "memorystream": "^0.3.1",
     "mongodb": "3.6.3",
     "mssql": "6.2.3",
-    "mysql": "2.18.1",
+    "mysql2": "^2.3.1",
     "node-fetch": "2.6.0",
     "open": "7.3.0",
     "pg": "8.5.1",
@@ -119,6 +119,7 @@
     "to-json-schema": "0.2.5",
     "uuid": "3.3.2",
     "validate.js": "0.13.1",
+    "vm2": "^3.9.3",
     "yargs": "13.2.4",
     "zlib": "1.0.5"
   },

packages/server/scripts/jestSetup.js

@@ -8,3 +8,5 @@ env._set("CLIENT_ID", "test-client-id")
 env._set("BUDIBASE_DIR", tmpdir("budibase-unittests"))
 env._set("LOG_LEVEL", "silent")
 env._set("PORT", 0)
+
+global.console.log = jest.fn() // console.log are ignored in tests

packages/server/src/api/controllers/analytics.js

@@ -21,6 +21,10 @@ exports.endUserPing = async ctx => {
     return
   }
 
+  posthogClient.identify({
+    distinctId: ctx.user && ctx.user._id,
+    properties: {},
+  })
   posthogClient.capture({
     event: "budibase:end_user_ping",
     distinctId: ctx.user && ctx.user._id,

packages/server/src/api/controllers/application.js

@@ -25,7 +25,12 @@ const { BASE_LAYOUTS } = require("../../constants/layouts")
 const { createHomeScreen } = require("../../constants/screens")
 const { cloneDeep } = require("lodash/fp")
 const { processObject } = require("@budibase/string-templates")
-const { getAllApps } = require("@budibase/auth/db")
+const {
+  getAllApps,
+  isDevAppID,
+  getDeployedAppID,
+  Replication,
+} = require("@budibase/auth/db")
 const { USERS_TABLE_SCHEMA } = require("../../constants")
 const {
   getDeployedApps,
@@ -134,7 +139,7 @@ async function createInstance(template) {
   return { _id: appId }
 }
 
-exports.fetch = async function (ctx) {
+exports.fetch = async ctx => {
   const dev = ctx.query && ctx.query.status === AppStatus.DEV
   const all = ctx.query && ctx.query.status === AppStatus.ALL
   const apps = await getAllApps(CouchDB, { dev, all })
@@ -159,7 +164,7 @@ exports.fetch = async function (ctx) {
   ctx.body = apps
 }
 
-exports.fetchAppDefinition = async function (ctx) {
+exports.fetchAppDefinition = async ctx => {
   const db = new CouchDB(ctx.params.appId)
   const layouts = await getLayouts(db)
   const userRoleId = getUserRoleId(ctx)
@@ -175,7 +180,7 @@ exports.fetchAppDefinition = async function (ctx) {
   }
 }
 
-exports.fetchAppPackage = async function (ctx) {
+exports.fetchAppPackage = async ctx => {
   const db = new CouchDB(ctx.params.appId)
   const application = await db.get(DocumentTypes.APP_METADATA)
   const layouts = await getLayouts(db)
@@ -196,7 +201,7 @@ exports.fetchAppPackage = async function (ctx) {
   }
 }
 
-exports.create = async function (ctx) {
+exports.create = async ctx => {
   const { useTemplate, templateKey, templateString } = ctx.request.body
   const instanceConfig = {
     useTemplate,
@@ -252,13 +257,13 @@ exports.create = async function (ctx) {
   ctx.body = newApplication
 }
 
-exports.update = async function (ctx) {
+exports.update = async ctx => {
   const data = await updateAppPackage(ctx, ctx.request.body, ctx.params.appId)
   ctx.status = 200
   ctx.body = data
 }
 
-exports.updateClient = async function (ctx) {
+exports.updateClient = async ctx => {
   // Get current app version
   const db = new CouchDB(ctx.params.appId)
   const application = await db.get(DocumentTypes.APP_METADATA)
@@ -280,7 +285,7 @@ exports.updateClient = async function (ctx) {
   ctx.body = data
 }
 
-exports.revertClient = async function (ctx) {
+exports.revertClient = async ctx => {
   // Check app can be reverted
   const db = new CouchDB(ctx.params.appId)
   const application = await db.get(DocumentTypes.APP_METADATA)
@@ -303,7 +308,7 @@ exports.revertClient = async function (ctx) {
   ctx.body = data
 }
 
-exports.delete = async function (ctx) {
+exports.delete = async ctx => {
   const db = new CouchDB(ctx.params.appId)
 
   const result = await db.destroy()
@@ -318,6 +323,35 @@ exports.delete = async function (ctx) {
   ctx.body = result
 }
 
+exports.sync = async ctx => {
+  const appId = ctx.params.appId
+  if (!isDevAppID(appId)) {
+    ctx.throw(400, "This action cannot be performed for production apps")
+  }
+  const prodAppId = getDeployedAppID(appId)
+  const replication = new Replication({
+    source: prodAppId,
+    target: appId,
+  })
+  let error
+  try {
+    await replication.replicate({
+      filter: function (doc) {
+        return doc._id !== DocumentTypes.APP_METADATA
+      },
+    })
+  } catch (err) {
+    error = err
+  }
+  if (error) {
+    ctx.throw(400, error)
+  } else {
+    ctx.body = {
+      message: "App sync completed successfully.",
+    }
+  }
+}
+
 const updateAppPackage = async (ctx, appPackage, appId) => {
   const url = await getAppUrlIfNotInUse(ctx)
   const db = new CouchDB(appId)

packages/server/src/api/controllers/auth.js

@@ -28,14 +28,23 @@ exports.fetchSelf = async ctx => {
         ...metadata,
       })
     } catch (err) {
+      let response
       // user didn't exist in app, don't pretend they do
       if (user.roleId === BUILTIN_ROLE_IDS.PUBLIC) {
-        ctx.body = {}
+        response = {}
       }
       // user has a role of some sort, return them
-      else {
+      else if (err.status === 404) {
-        ctx.body = user
+        const metadata = {
+          _id: userId,
+        }
+        const dbResp = await db.put(metadata)
+        user._rev = dbResp.rev
+        response = user
+      } else {
+        response = user
       }
+      ctx.body = response
     }
   } else {
     ctx.body = user

packages/server/src/api/controllers/backup.js

@@ -1,10 +1,9 @@
-const { performBackup } = require("../../utilities/fileSystem")
+const { streamBackup } = require("../../utilities/fileSystem")
 
 exports.exportAppDump = async function (ctx) {
   const { appId } = ctx.query
-  const appname = decodeURI(ctx.query.appname)
+  const appName = decodeURI(ctx.query.appname)
-  const backupIdentifier = `${appname}Backup${new Date().getTime()}.txt`
+  const backupIdentifier = `${appName}-export-${new Date().getTime()}.txt`
-
   ctx.attachment(backupIdentifier)
-  ctx.body = await performBackup(appId, backupIdentifier)
+  ctx.body = await streamBackup(appId)
 }

packages/server/src/api/controllers/deploy/index.js

@@ -1,6 +1,6 @@
 const CouchDB = require("../../../db")
 const Deployment = require("./Deployment")
-const { Replication } = require("@budibase/auth/db")
+const { Replication, getDeployedAppID } = require("@budibase/auth/db")
 const { DocumentTypes, getAutomationParams } = require("../../../db/utils")
 const {
   disableAllCrons,
@@ -87,7 +87,7 @@ async function initDeployedApp(prodAppId) {
 
 async function deployApp(deployment) {
   try {
-    const productionAppId = deployment.appId.replace("_dev", "")
+    const productionAppId = getDeployedAppID(deployment.appId)
 
     const replication = new Replication({
       source: deployment.appId,
@@ -104,23 +104,8 @@ async function deployApp(deployment) {
     appDoc.instance._id = productionAppId
     await db.put(appDoc)
     console.log("New app doc written successfully.")
-
-    console.log("Setting up live repl between dev and prod")
-    // Set up live sync between the live and dev instances
-    const liveReplication = new Replication({
-      source: productionAppId,
-      target: deployment.appId,
-    })
-    liveReplication.subscribe({
-      filter: function (doc) {
-        return doc._id !== DocumentTypes.APP_METADATA
-      },
-    })
-    console.log("Set up live repl between dev and prod")
-
-    console.log("Initialising deployed app")
     await initDeployedApp(productionAppId)
-    console.log("Init complete, setting deployment to successful")
+    console.log("Deployed app initialised, setting deployment to successful")
     deployment.setStatus(DeploymentStatus.SUCCESS)
     await storeDeploymentHistory(deployment)
   } catch (err) {

packages/server/src/api/controllers/dev.js

@@ -7,11 +7,13 @@ const { clearLock } = require("../../utilities/redis")
 const { Replication } = require("@budibase/auth").db
 const { DocumentTypes } = require("../../db/utils")
 
-async function redirect(ctx, method) {
+async function redirect(ctx, method, path = "global") {
   const { devPath } = ctx.params
   const queryString = ctx.originalUrl.split("?")[1] || ""
   const response = await fetch(
-    checkSlashesInUrl(`${env.WORKER_URL}/api/global/${devPath}?${queryString}`),
+    checkSlashesInUrl(
+      `${env.WORKER_URL}/api/${path}/${devPath}?${queryString}`
+    ),
     request(
       ctx,
       {
@@ -41,16 +43,22 @@ async function redirect(ctx, method) {
   ctx.cookies
 }
 
-exports.redirectGet = async ctx => {
+exports.buildRedirectGet = path => {
-  await redirect(ctx, "GET")
+  return async ctx => {
+    await redirect(ctx, "GET", path)
+  }
 }
 
-exports.redirectPost = async ctx => {
+exports.buildRedirectPost = path => {
-  await redirect(ctx, "POST")
+  return async ctx => {
+    await redirect(ctx, "POST", path)
+  }
 }
 
-exports.redirectDelete = async ctx => {
+exports.buildRedirectDelete = path => {
-  await redirect(ctx, "DELETE")
+  return async ctx => {
+    await redirect(ctx, "DELETE", path)
+  }
 }
 
 exports.clearLock = async ctx => {

packages/server/src/api/controllers/permission.js

@@ -147,6 +147,7 @@ exports.getResourcePerms = async function (ctx) {
       const rolePerms = role.permissions
       if (
         rolePerms &&
+        rolePerms[resourceId] &&
         (rolePerms[resourceId] === level ||
           rolePerms[resourceId].indexOf(level) !== -1)
       ) {

packages/server/src/api/controllers/query.js

@@ -4,6 +4,7 @@ const { generateQueryID, getQueryParams } = require("../../db/utils")
 const { integrations } = require("../../integrations")
 const { BaseQueryVerbs } = require("../../constants")
 const env = require("../../environment")
+const ScriptRunner = require("../../utilities/scriptRunner")
 
 // simple function to append "readable" to all read queries
 function enrichQueries(input) {
@@ -28,12 +29,39 @@ function formatResponse(resp) {
       resp = { response: resp }
     }
   }
-  if (!Array.isArray(resp)) {
-    resp = [resp]
-  }
   return resp
 }
 
+async function runAndTransform(
+  integration,
+  queryVerb,
+  enrichedQuery,
+  transformer
+) {
+  let rows = formatResponse(await integration[queryVerb](enrichedQuery))
+
+  // transform as required
+  if (transformer) {
+    const runner = new ScriptRunner(transformer, { data: rows })
+    rows = runner.execute()
+  }
+
+  // needs to an array for next step
+  if (!Array.isArray(rows)) {
+    rows = [rows]
+  }
+
+  // map into JSON if just raw primitive here
+  if (rows.find(row => typeof row !== "object")) {
+    rows = rows.map(value => ({ value }))
+  }
+
+  // get all the potential fields in the schema
+  let keys = rows.flatMap(Object.keys)
+
+  return { rows, keys }
+}
+
 exports.fetch = async function (ctx) {
   const db = new CouchDB(ctx.appId)
 
@@ -122,15 +150,16 @@ exports.preview = async function (ctx) {
     ctx.throw(400, "Integration type does not exist.")
   }
 
-  const { fields, parameters, queryVerb } = ctx.request.body
+  const { fields, parameters, queryVerb, transformer } = ctx.request.body
-
   const enrichedQuery = await enrichQueryFields(fields, parameters)
-
   const integration = new Integration(datasource.config)
-  const rows = formatResponse(await integration[queryVerb](enrichedQuery))
 
-  // get all the potential fields in the schema
+  const { rows, keys } = await runAndTransform(
-  const keys = rows.flatMap(Object.keys)
+    integration,
+    queryVerb,
+    enrichedQuery,
+    transformer
+  )
 
   ctx.body = {
     rows,
@@ -158,10 +187,16 @@ exports.execute = async function (ctx) {
     query.fields,
     ctx.request.body.parameters
   )
-
   const integration = new Integration(datasource.config)
+
   // call the relevant CRUD method on the integration class
-  ctx.body = formatResponse(await integration[query.queryVerb](enrichedQuery))
+  const { rows } = await runAndTransform(
+    integration,
+    query.queryVerb,
+    enrichedQuery,
+    query.transformer
+  )
+  ctx.body = rows
   // cleanup
   if (integration.end) {
     integration.end()

packages/server/src/api/controllers/row/ExternalRequest.ts

@@ -15,8 +15,9 @@ import {
 import {
   breakRowIdField,
   generateRowIdField,
+  isRowId,
+  convertRowId,
 } from "../../../integrations/utils"
-import { RelationshipTypes } from "../../../constants"
 
 interface ManyRelationship {
   tableId?: string
@@ -36,7 +37,11 @@ interface RunConfig {
 
 module External {
   const { makeExternalQuery } = require("./utils")
-  const { DataSourceOperation, FieldTypes } = require("../../../constants")
+  const {
+    DataSourceOperation,
+    FieldTypes,
+    RelationshipTypes,
+  } = require("../../../constants")
   const { breakExternalTableId, isSQL } = require("../../../integrations/utils")
   const { processObjectSync } = require("@budibase/string-templates")
   const { cloneDeep } = require("lodash/fp")
@@ -83,6 +88,48 @@ module External {
     }
   }
 
+  /**
+   * This function checks the incoming parameters to make sure all the inputs are
+   * valid based on on the table schema. The main thing this is looking for is when a
+   * user has made use of the _id field of a row for a foreign key or a search parameter.
+   * In these cases the key will be sent up as [1], rather than 1. In these cases we will
+   * simplify it down to the requirements. This function is quite complex as we try to be
+   * relatively restrictive over what types of columns we will perform this action for.
+   */
+  function cleanupConfig(config: RunConfig, table: Table): RunConfig {
+    const primaryOptions = [
+      FieldTypes.STRING,
+      FieldTypes.LONGFORM,
+      FieldTypes.OPTIONS,
+      FieldTypes.NUMBER,
+    ]
+    // filter out fields which cannot be keys
+    const fieldNames = Object.entries(table.schema)
+      .filter(schema => primaryOptions.find(val => val === schema[1].type))
+      .map(([fieldName]) => fieldName)
+    const iterateObject = (obj: { [key: string]: any }) => {
+      for (let [field, value] of Object.entries(obj)) {
+        if (fieldNames.find(name => name === field) && isRowId(value)) {
+          obj[field] = convertRowId(value)
+        }
+      }
+    }
+    // check the row and filters to make sure they aren't a key of some sort
+    if (config.filters) {
+      for (let filter of Object.values(config.filters)) {
+        if (typeof filter !== "object" || Object.keys(filter).length === 0) {
+          continue
+        }
+        iterateObject(filter)
+      }
+    }
+    if (config.row) {
+      iterateObject(config.row)
+    }
+
+    return config
+  }
+
   function generateIdForRow(row: Row | undefined, table: Table): string {
     const primary = table.primary
     if (!row || !primary) {
@@ -509,7 +556,7 @@ module External {
       return fields
     }
 
-    async run({ id, row, filters, sort, paginate }: RunConfig) {
+    async run(config: RunConfig) {
       const { appId, operation, tableId } = this
       let { datasourceId, tableName } = breakExternalTableId(tableId)
       if (!this.datasource) {
@@ -525,9 +572,11 @@ module External {
       if (!table) {
         throw `Unable to process query, table "${tableName}" not defined.`
       }
-      // clean up row on ingress using schema
+      // look for specific components of config which may not be considered acceptable
+      let { id, row, filters, sort, paginate } = cleanupConfig(config, table)
       filters = buildFilters(id, filters || {}, table)
       const relationships = this.buildRelationships(table)
+      // clean up row on ingress using schema
       const processed = this.inputProcessing(row, table)
       row = processed.row
       if (

packages/server/src/api/controllers/script.js

@@ -1,24 +1,9 @@
-const fetch = require("node-fetch")
+const ScriptRunner = require("../../utilities/scriptRunner")
-const vm = require("vm")
-
-class ScriptExecutor {
-  constructor(body) {
-    const code = `let fn = () => {\n${body.script}\n}; out = fn();`
-    this.script = new vm.Script(code)
-    this.context = vm.createContext(body.context)
-    this.context.fetch = fetch
-  }
-
-  execute() {
-    this.script.runInContext(this.context)
-    return this.context.out
-  }
-}
 
 exports.execute = async function (ctx) {
-  const executor = new ScriptExecutor(ctx.request.body)
+  const { script, context } = ctx.request.body
-
+  const runner = new ScriptRunner(script, context)
-  ctx.body = executor.execute()
+  ctx.body = runner.execute()
 }
 
 exports.save = async function (ctx) {