Merge branch 'master' of https://github.com/Budibase/budibase into plus-datasources

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "0.9.25",
+  "version": "0.9.27",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

packages/auth/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/auth",
-  "version": "0.9.25",
+  "version": "0.9.27",
   "description": "Authentication middlewares for budibase builder and apps",
   "main": "src/index.js",
   "author": "Budibase",

packages/auth/src/security/roles.js

@@ -13,7 +13,6 @@ const BUILTIN_IDS = {
   POWER: "POWER",
   BASIC: "BASIC",
   PUBLIC: "PUBLIC",
-  BUILDER: "BUILDER",
 }
 
 // exclude internal roles like builder

packages/bbui/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/bbui",
   "description": "A UI solution used in the different Budibase projects.",
-  "version": "0.9.25",
+  "version": "0.9.27",
   "license": "AGPL-3.0",
   "svelte": "src/index.js",
   "module": "dist/bbui.es.js",

packages/bbui/src/Form/Core/RadioGroup.svelte

@@ -37,3 +37,9 @@
     {/each}
   {/if}
 </div>
+
+<style>
+  .spectrum-Radio-input {
+    opacity: 0;
+  }
+</style>

packages/builder/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/builder",
-  "version": "0.9.25",
+  "version": "0.9.27",
   "license": "AGPL-3.0",
   "private": true,
   "scripts": {
@@ -65,10 +65,10 @@
     }
   },
   "dependencies": {
-    "@budibase/bbui": "^0.9.25",
-    "@budibase/client": "^0.9.25",
+    "@budibase/bbui": "^0.9.27",
+    "@budibase/client": "^0.9.27",
     "@budibase/colorpicker": "1.1.2",
-    "@budibase/string-templates": "^0.9.25",
+    "@budibase/string-templates": "^0.9.27",
     "@sentry/browser": "5.19.1",
     "@spectrum-css/page": "^3.0.1",
     "@spectrum-css/vars": "^3.0.1",

packages/builder/src/builderStore/dataBinding.js

@@ -136,7 +136,7 @@ const getContextBindings = (asset, componentId) => {
       if (!datasource) {
         return
       }
-      const info = getSchemaForDatasource(datasource)
+      const info = getSchemaForDatasource(asset, datasource)
       schema = info.schema
       readablePrefix = info.table?.name
     }
@@ -191,7 +191,7 @@ const getContextBindings = (asset, componentId) => {
  */
 const getUserBindings = () => {
   let bindings = []
-  const { schema } = getSchemaForDatasource({
+  const { schema } = getSchemaForDatasource(null, {
     type: "table",
     tableId: TableNames.USERS,
   })
@@ -244,11 +244,15 @@ const getUrlBindings = asset => {
 /**
  * Gets a schema for a datasource object.
  */
-export const getSchemaForDatasource = (datasource, isForm = false) => {
+export const getSchemaForDatasource = (asset, datasource, isForm = false) => {
   let schema, table
   if (datasource) {
     const { type } = datasource
-    if (type === "query") {
+    if (type === "provider") {
+      const component = findComponent(asset.props, datasource.providerId)
+      const source = getDatasourceForProvider(asset, component)
+      return getSchemaForDatasource(asset, source, isForm)
+    } else if (type === "query") {
       const queries = get(queriesStores).list
       table = queries.find(query => query._id === datasource._id)
     } else {

packages/builder/src/builderStore/store/screenTemplates/utils/commonComponents.js

@@ -174,7 +174,7 @@ const fieldTypeToComponentMap = {
 }
 
 export function makeDatasourceFormComponents(datasource) {
-  const { schema } = getSchemaForDatasource(datasource, true)
+  const { schema } = getSchemaForDatasource(null, datasource, true)
   let components = []
   let fields = Object.keys(schema || {})
   fields.forEach(field => {

packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte

@@ -43,6 +43,7 @@
   }
 
   let originalName = field.name
+  const linkEditDisabled = originalName != null
   let primaryDisplay =
     $tables.selected.primaryDisplay == null ||
     $tables.selected.primaryDisplay === field.name
@@ -197,7 +198,7 @@
   <Input
     label="Name"
     bind:value={field.name}
-    disabled={uneditable || field.type === LINK_TYPE}
+    disabled={uneditable || (linkEditDisabled && field.type === LINK_TYPE)}
   />
 
   <Select
@@ -284,6 +285,7 @@
   {:else if field.type === "link"}
     <Select
       label="Table"
+      disabled={linkEditDisabled}
       bind:value={field.tableId}
       options={tableOptions}
       getOptionLabel={table => table.name}
@@ -291,7 +293,7 @@
     />
     {#if relationshipOptions && relationshipOptions.length > 0}
       <RadioGroup
-        disabled={originalName != null}
+        disabled={linkEditDisabled}
         label="Define the relationship"
         bind:value={field.relationshipType}
         options={relationshipOptions}
@@ -299,7 +301,11 @@
         getOptionValue={option => option.value}
       />
     {/if}
-    <Input label={`Column name in other table`} bind:value={field.fieldName} />
+    <Input
+      disabled={linkEditDisabled}
+      label={`Column name in other table`}
+      bind:value={field.fieldName}
+    />
   {:else if field.type === FORMULA_TYPE}
     <ModalBindableInput
       title="Handlebars Formula"

packages/builder/src/components/design/AppPreview/componentStructure.json

@@ -18,7 +18,8 @@
       "longformfield",
       "datetimefield",
       "attachmentfield",
-      "relationshipfield"
+      "relationshipfield",
+      "daterangepicker"
     ]
   },
   {

packages/builder/src/components/design/PropertiesPanel/PropertyControls/DataSourceSelect.svelte

@@ -1,5 +1,8 @@
 <script>
-  import { getBindableProperties } from "builderStore/dataBinding"
+  import {
+    getBindableProperties,
+    getDataProviderComponents,
+  } from "builderStore/dataBinding"
   import {
     Button,
     Popover,
@@ -61,6 +64,17 @@
     $currentAsset,
     $store.selectedComponentId
   )
+  $: dataProviders = getDataProviderComponents(
+    $currentAsset,
+    $store.selectedComponentId
+  ).map(provider => ({
+    label: provider._instanceName,
+    name: provider._instanceName,
+    providerId: provider._id,
+    value: `{{ literal [${provider._id}] }}`,
+    type: "provider",
+    schema: provider.schema,
+  }))
   $: queryBindableProperties = bindableProperties.map(property => ({
     ...property,
     category: property.type === "instance" ? "Component" : "Table",
@@ -182,7 +196,20 @@
         </li>
       {/each}
     </ul>
-
+    <Divider size="S" />
+    <div class="title">
+      <Heading size="XS">Data Providers</Heading>
+    </div>
+    <ul>
+      {#each dataProviders as provider}
+        <li
+          class:selected={value === provider}
+          on:click={() => handleSelected(provider)}
+        >
+          {provider.label}
+        </li>
+      {/each}
+    </ul>
     {#if otherSources?.length}
       <Divider size="S" />
       <div class="title">

packages/builder/src/components/design/PropertiesPanel/PropertyControls/EventsEditor/actions/SaveRow.svelte

@@ -14,11 +14,11 @@
     $currentAsset,
     $store.selectedComponentId
   )
-  $: schemaFields = getSchemaFields(parameters?.tableId)
+  $: schemaFields = getSchemaFields($currentAsset, parameters?.tableId)
   $: tableOptions = $tables.list || []
 
-  const getSchemaFields = tableId => {
-    const { schema } = getSchemaForDatasource({ type: "table", tableId })
+  const getSchemaFields = (asset, tableId) => {
+    const { schema } = getSchemaForDatasource(asset, { type: "table", tableId })
     return Object.values(schema || {})
   }
 

packages/builder/src/components/design/PropertiesPanel/PropertyControls/EventsEditor/actions/index.js

@@ -5,11 +5,13 @@ import ExecuteQuery from "./ExecuteQuery.svelte"
 import TriggerAutomation from "./TriggerAutomation.svelte"
 import ValidateForm from "./ValidateForm.svelte"
 
-// defines what actions are available, when adding a new one
-// the component is the setup panel for the action
-// NOTE that the "name" is used by the client library,
-// so if you want to change it, you must change it client lib too
-
+// Defines which actions are available to configure in the front end.
+// Unfortunately the "name" property is used as the identifier so please don't
+// change them.
+// The client library removes any spaces when processing actions, so they can
+// be considered as camel case too.
+// There is technical debt here to sanitize all these and standardise them
+// across the packages but it's a breaking change to existing apps.
 export default [
   {
     name: "Save Row",

packages/builder/src/components/design/PropertiesPanel/PropertyControls/FieldSelect.svelte

@@ -14,7 +14,7 @@
 
   const dispatch = createEventDispatcher()
   $: datasource = getDatasourceForProvider($currentAsset, componentInstance)
-  $: schema = getSchemaForDatasource(datasource).schema
+  $: schema = getSchemaForDatasource($currentAsset, datasource).schema
   $: options = Object.keys(schema || {})
   $: boundValue = getValidValue(value, options)
 

packages/builder/src/components/design/PropertiesPanel/PropertyControls/FilterEditor/FilterEditor.svelte

@@ -27,19 +27,16 @@
     ? tempValue.length
     : Object.keys(tempValue || {}).length
   $: dataSource = getDatasourceForProvider($currentAsset, componentInstance)
-  $: schema = getSchemaForDatasource(dataSource)?.schema
+  $: schema = getSchemaForDatasource($currentAsset, dataSource)?.schema
   $: schemaFields = Object.values(schema || {})
   $: internalTable = dataSource?.type === "table"
 
   // Reset value if value is wrong type for the datasource.
   // Lucene editor needs an array, and simple editor needs an object.
   $: {
-    if (internalTable && !Array.isArray(value)) {
+    if (!Array.isArray(value)) {
       tempValue = []
       dispatch("change", [])
-    } else if (!internalTable && Array.isArray(value)) {
-      tempValue = {}
-      dispatch("change", {})
     }
   }
 
@@ -63,28 +60,7 @@
           constaints.
         {/if}
       </Body>
-      {#if internalTable}
-        <LuceneFilterBuilder bind:value={tempValue} {schemaFields} />
-      {:else}
-        <div class="fields">
-          <SaveFields
-            parameterFields={Array.isArray(value) ? {} : value}
-            {schemaFields}
-            valueLabel="Equals"
-            on:change={e => (tempValue = e.detail)}
-          />
-        </div>
-      {/if}
+      <LuceneFilterBuilder bind:value={tempValue} {schemaFields} />
     </Layout>
   </DrawerContent>
 </Drawer>
-
-<style>
-  .fields {
-    display: grid;
-    column-gap: var(--spacing-l);
-    row-gap: var(--spacing-s);
-    align-items: center;
-    grid-template-columns: auto 1fr auto 1fr auto;
-  }
-</style>

packages/builder/src/components/design/PropertiesPanel/PropertyControls/FormFieldSelect.svelte

@@ -18,7 +18,7 @@
     component => component._component === "@budibase/standard-components/form"
   )
   $: datasource = getDatasourceForProvider($currentAsset, form)
-  $: schema = getSchemaForDatasource(datasource, true).schema
+  $: schema = getSchemaForDatasource($currentAsset, datasource, true).schema
   $: options = getOptions(schema, type)
 
   const getOptions = (schema, fieldType) => {

packages/builder/src/components/design/PropertiesPanel/PropertyControls/MultiFieldSelect.svelte

@@ -14,7 +14,7 @@
 
   const dispatch = createEventDispatcher()
   $: datasource = getDatasourceForProvider($currentAsset, componentInstance)
-  $: schema = getSchemaForDatasource(datasource).schema
+  $: schema = getSchemaForDatasource($currentAsset, datasource).schema
   $: options = Object.keys(schema || {})
   $: boundValue = getValidOptions(value, options)
 

packages/builder/src/stores/backend/permissions.js

@@ -6,6 +6,13 @@ export function createPermissionStore() {
 
   return {
     subscribe,
+    save: async ({ level, role, resource }) => {
+      const response = await api.post(
+        `/api/permission/${role}/${resource}/${level}`
+      )
+      const json = await response.json()
+      return json
+    },
     forResource: async resourceId => {
       const response = await api.get(`/api/permission/${resourceId}`)
       const json = await response.json()

packages/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/cli",
-  "version": "0.9.25",
+  "version": "0.9.27",
   "description": "Budibase CLI, for developers, self hosting and migrations.",
   "main": "src/index.js",
   "bin": {

packages/client/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/client",
-  "version": "0.9.25",
+  "version": "0.9.27",
   "license": "MPL-2.0",
   "module": "dist/budibase-client.js",
   "main": "dist/budibase-client.js",
@@ -18,13 +18,13 @@
     "dev:builder": "rollup -cw"
   },
   "dependencies": {
-    "@budibase/string-templates": "^0.9.25",
+    "@budibase/string-templates": "^0.9.27",
     "regexparam": "^1.3.0",
     "shortid": "^2.2.15",
     "svelte-spa-router": "^3.0.5"
   },
   "devDependencies": {
-    "@budibase/standard-components": "^0.9.25",
+    "@budibase/standard-components": "^0.9.27",
     "@rollup/plugin-commonjs": "^18.0.0",
     "@rollup/plugin-node-resolve": "^11.2.1",
     "fs-extra": "^8.1.0",

packages/client/src/components/Component.svelte

@@ -55,13 +55,13 @@
   }
 
   // Enriches any string component props using handlebars
-  const updateComponentProps = async (definition, context) => {
+  const updateComponentProps = (definition, context) => {
     // Record the timestamp so we can reference it after enrichment
     latestUpdateTime = Date.now()
     const enrichmentTime = latestUpdateTime
 
     // Enrich props with context
-    const enrichedProps = await enrichProps(definition, context)
+    const enrichedProps = enrichProps(definition, context)
 
     // Abandon this update if a newer update has started
     if (enrichmentTime !== latestUpdateTime) {

packages/client/src/constants.js

@@ -5,4 +5,5 @@ export const TableNames = {
 export const ActionTypes = {
   ValidateForm: "ValidateForm",
   RefreshDatasource: "RefreshDatasource",
+  SetDataProviderQuery: "SetDataProviderQuery",
 }

packages/client/src/sdk.js

@@ -9,6 +9,7 @@ import {
 import { styleable } from "./utils/styleable"
 import transition from "./utils/transition"
 import { linkable } from "./utils/linkable"
+import { getAction } from "./utils/getAction"
 import Provider from "./components/Provider.svelte"
 import { ActionTypes } from "./constants"
 
@@ -22,6 +23,7 @@ export default {
   styleable,
   transition,
   linkable,
+  getAction,
   Provider,
   ActionTypes,
 }

packages/client/src/utils/componentProps.js

@@ -21,7 +21,7 @@ export const propsAreSame = (a, b) => {
  * Enriches component props.
  * Data bindings are enriched, and button actions are enriched.
  */
-export const enrichProps = async (props, context) => {
+export const enrichProps = (props, context) => {
   // Exclude all private props that start with an underscore
   let validProps = {}
   Object.entries(props)
@@ -41,7 +41,7 @@ export const enrichProps = async (props, context) => {
   }
 
   // Enrich all data bindings in top level props
-  let enrichedProps = await enrichDataBindings(validProps, totalContext)
+  let enrichedProps = enrichDataBindings(validProps, totalContext)
 
   // Enrich click actions if they exist
   if (enrichedProps.onClick) {

packages/client/src/utils/enrichDataBinding.js

@@ -1,5 +1,5 @@
 import { cloneDeep } from "lodash/fp"
-import { processString, processObject } from "@budibase/string-templates"
+import { processString, processObjectSync } from "@budibase/string-templates"
 
 // Regex to test inputs with to see if they are likely candidates for template strings
 const looksLikeTemplate = /{{.*}}/
@@ -23,6 +23,6 @@ export const enrichDataBinding = async (input, context) => {
  * Recursively enriches all props in a props object and returns the new props.
  * Props are deeply cloned so that no mutation is done to the source object.
  */
-export const enrichDataBindings = async (props, context) => {
-  return await processObject(cloneDeep(props), context)
+export const enrichDataBindings = (props, context) => {
+  return processObjectSync(cloneDeep(props), context)
 }

packages/client/src/utils/getAction.js

@@ -0,0 +1,19 @@
+import { get } from "svelte/store"
+import { getContext } from "svelte"
+
+/**
+ * Gets a component action.
+ * @param id The component ID that provides the action
+ * @param type The action type to get
+ * @returns {null|*} The action function
+ */
+export const getAction = (id, type) => {
+  if (!id || !type) {
+    return null
+  }
+  const context = getContext("context")
+  if (!context) {
+    return null
+  }
+  return get(context)?.[`${id}_${type}`]
+}

packages/server/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "0.9.25",
+  "version": "0.9.27",
   "description": "Budibase Web Server",
   "main": "src/electron.js",
   "repository": {
@@ -55,9 +55,9 @@
   "author": "Budibase",
   "license": "AGPL-3.0-or-later",
   "dependencies": {
-    "@budibase/auth": "^0.9.25",
-    "@budibase/client": "^0.9.25",
-    "@budibase/string-templates": "^0.9.25",
+    "@budibase/auth": "^0.9.27",
+    "@budibase/client": "^0.9.27",
+    "@budibase/string-templates": "^0.9.27",
     "@elastic/elasticsearch": "7.10.0",
     "@koa/router": "8.0.0",
     "@sendgrid/mail": "7.1.1",
@@ -109,7 +109,7 @@
   "devDependencies": {
     "@babel/core": "^7.14.3",
     "@babel/preset-env": "^7.14.4",
-    "@budibase/standard-components": "^0.9.25",
+    "@budibase/standard-components": "^0.9.27",
     "@jest/test-sequencer": "^24.8.0",
     "babel-jest": "^27.0.2",
     "docker-compose": "^0.23.6",

packages/server/src/api/controllers/application.js

@@ -99,12 +99,18 @@ async function createInstance(template) {
   // replicate the template data to the instance DB
   // this is currently very hard to test, downloading and importing template files
   /* istanbul ignore next */
+  let _rev
   if (template && template.useTemplate === "true") {
     const { ok } = await db.load(await getTemplateStream(template))
     if (!ok) {
       throw "Error loading database dump from template."
     }
-    var { _rev } = await db.get(DocumentTypes.APP_METADATA)
+    try {
+      const response = await db.get(DocumentTypes.APP_METADATA)
+      _rev = response._rev
+    } catch (err) {
+      _rev = null
+    }
   } else {
     // create the users table
     await db.put(USERS_TABLE_SCHEMA)

packages/server/src/api/controllers/auth.js

@@ -5,7 +5,7 @@ const { getFullUser } = require("../../utilities/users")
 
 exports.fetchSelf = async ctx => {
   const appId = ctx.appId
-  const { userId } = ctx.user
+  let userId = ctx.user.userId || ctx.user._id
   /* istanbul ignore next */
   if (!userId) {
     ctx.body = {}

packages/server/src/api/controllers/routing.js

@@ -63,10 +63,6 @@ exports.fetch = async ctx => {
 exports.clientFetch = async ctx => {
   const routing = await getRoutingStructure(ctx.appId)
   let roleId = ctx.user.role._id
-  // builder is a special case, always return the full routing structure
-  if (roleId === BUILTIN_ROLE_IDS.BUILDER) {
-    roleId = BUILTIN_ROLE_IDS.ADMIN
-  }
   const roleIds = await getUserRoleHierarchy(ctx.appId, roleId)
   for (let topLevel of Object.values(routing.routes)) {
     for (let subpathKey of Object.keys(topLevel.subpaths)) {

packages/server/src/api/controllers/user.js

@@ -4,7 +4,6 @@ const {
   getUserMetadataParams,
 } = require("../../db/utils")
 const { InternalTables } = require("../../db/utils")
-const { BUILTIN_ROLE_IDS } = require("@budibase/auth/roles")
 const {
   getGlobalUsers,
   addAppRoleToUser,
@@ -47,10 +46,6 @@ exports.fetchMetadata = async function (ctx) {
 exports.updateSelfMetadata = async function (ctx) {
   // overwrite the ID with current users
   ctx.request.body._id = ctx.user._id
-  if (ctx.user.builder && ctx.user.builder.global) {
-    // specific case, update self role in global user
-    await addAppRoleToUser(ctx, ctx.appId, BUILTIN_ROLE_IDS.ADMIN)
-  }
   // make sure no stale rev
   delete ctx.request.body._rev
   await exports.updateMetadata(ctx)

packages/server/src/api/routes/tests/routing.spec.js

@@ -28,9 +28,7 @@ describe("/routing", () => {
     it("returns the correct routing for basic user", async () => {
       workerRequests.getGlobalUsers.mockImplementationOnce((ctx, appId) => {
         return {
-          roles: {
-            [appId]: BUILTIN_ROLE_IDS.BASIC,
-          }
+          roleId: BUILTIN_ROLE_IDS.BASIC,
         }
       })
       const res = await request

packages/server/src/api/routes/tests/utilities/TestFunctions.js

@@ -2,6 +2,7 @@ const rowController = require("../../../controllers/row")
 const appController = require("../../../controllers/application")
 const CouchDB = require("../../../../db")
 const { AppStatus } = require("../../../../db/utils")
+const { BUILTIN_ROLE_IDS } = require("@budibase/auth/roles")
 
 function Request(appId, params) {
   this.appId = appId
@@ -77,11 +78,17 @@ exports.checkPermissionsEndpoint = async ({
     .set(passHeader)
     .expect(200)
 
-  user = await config.createUser("fail@budibase.com", password, failRole)
-  const failHeader = await config.login("fail@budibase.com", password, {
-    roleId: failRole,
-    userId: user.globalId,
-  })
+  let failHeader
+  if (failRole === BUILTIN_ROLE_IDS.PUBLIC) {
+    failHeader = config.publicHeaders()
+  } else {
+    user = await config.createUser("fail@budibase.com", password, failRole)
+    failHeader = await config.login("fail@budibase.com", password, {
+      roleId: failRole,
+      userId: user.globalId,
+      builder: false,
+    })
+  }
 
   await exports
     .createRequest(config.request, method, url, body)

packages/server/src/middleware/currentapp.js

@@ -33,7 +33,7 @@ module.exports = async (ctx, next) => {
     updateCookie = true
     appId = requestAppId
     // retrieving global user gets the right role
-    roleId = globalUser.roleId
+    roleId = globalUser.roleId || BUILTIN_ROLE_IDS.PUBLIC
   } else if (appCookie != null) {
     appId = appCookie.appId
     roleId = appCookie.roleId || BUILTIN_ROLE_IDS.PUBLIC

packages/server/src/tests/utilities/TestConfiguration.js

@@ -101,7 +101,7 @@ class TestConfiguration {
       userId: GLOBAL_USER_ID,
     }
     const app = {
-      roleId: BUILTIN_ROLE_IDS.BUILDER,
+      roleId: BUILTIN_ROLE_IDS.ADMIN,
       appId: this.appId,
     }
     const authToken = jwt.sign(auth, env.JWT_SECRET)
@@ -306,12 +306,9 @@ class TestConfiguration {
     return await this._req(config, null, controllers.layout.save)
   }
 
-  async createUser(roleId = BUILTIN_ROLE_IDS.POWER) {
+  async createUser() {
     const globalId = `us_${Math.random()}`
-    const resp = await this.globalUser(
-      globalId,
-      roleId === BUILTIN_ROLE_IDS.BUILDER
-    )
+    const resp = await this.globalUser(globalId)
     return {
       ...resp,
       globalId,
@@ -319,7 +316,6 @@ class TestConfiguration {
   }
 
   async login(email, password, { roleId, userId, builder } = {}) {
-    roleId = !roleId ? BUILTIN_ROLE_IDS.BUILDER : roleId
     userId = !userId ? `us_uuid1` : userId
     if (!this.request) {
       throw "Server has not been opened, cannot login."

packages/server/src/utilities/rowProcessor.js

@@ -169,9 +169,14 @@ exports.inputProcessing = (user = {}, table, row) => {
   let clonedRow = cloneDeep(row)
   // need to copy the table so it can be differenced on way out
   const copiedTable = cloneDeep(table)
+  const dontCleanseKeys = ["type", "_id", "_rev", "tableId"]
   for (let [key, value] of Object.entries(clonedRow)) {
     const field = table.schema[key]
+    // cleanse fields that aren't in the schema
     if (!field) {
+      if (dontCleanseKeys.indexOf(key) === -1) {
+        delete clonedRow[key]
+      }
       continue
     }
     clonedRow[key] = exports.coerce(value, field.type)

packages/server/src/utilities/workerRequests.js

@@ -9,19 +9,26 @@ function getAppRole(appId, user) {
   if (!user.roles) {
     return user
   }
-  // always use the deployed app
-  user.roleId = user.roles[getDeployedAppID(appId)]
-  if (!user.roleId) {
-    user.roleId = BUILTIN_ROLE_IDS.PUBLIC
+  if (user.builder && user.builder.global) {
+    user.roleId = BUILTIN_ROLE_IDS.ADMIN
+  } else {
+    // always use the deployed app
+    user.roleId = user.roles[getDeployedAppID(appId)]
+    if (!user.roleId) {
+      user.roleId = BUILTIN_ROLE_IDS.PUBLIC
+    }
   }
   delete user.roles
   return user
 }
 
-function request(ctx, request) {
+function request(ctx, request, noApiKey) {
   if (!request.headers) {
     request.headers = {}
   }
+  if (!noApiKey) {
+    request.headers["x-budibase-api-key"] = env.INTERNAL_API_KEY
+  }
   if (request.body && Object.keys(request.body).length > 0) {
     request.headers["Content-Type"] = "application/json"
     request.body =
@@ -44,9 +51,6 @@ exports.sendSmtpEmail = async (to, from, subject, contents) => {
     checkSlashesInUrl(env.WORKER_URL + `/api/admin/email/send`),
     request(null, {
       method: "POST",
-      headers: {
-        "x-budibase-api-key": env.INTERNAL_API_KEY,
-      },
       body: {
         email: to,
         from,
@@ -86,16 +90,6 @@ exports.getDeployedApps = async ctx => {
   }
 }
 
-exports.deleteGlobalUser = async (ctx, globalId) => {
-  const endpoint = `/api/admin/users/${globalId}`
-  const reqCfg = { method: "DELETE" }
-  const response = await fetch(
-    checkSlashesInUrl(env.WORKER_URL + endpoint),
-    request(ctx, reqCfg)
-  )
-  return response.json()
-}
-
 exports.getGlobalUsers = async (ctx, appId = null, globalId = null) => {
   const endpoint = globalId
     ? `/api/admin/users/${globalId}`
@@ -121,7 +115,8 @@ exports.getGlobalSelf = async (ctx, appId = null) => {
   const endpoint = `/api/admin/users/self`
   const response = await fetch(
     checkSlashesInUrl(env.WORKER_URL + endpoint),
-    request(ctx, { method: "GET" })
+    // we don't want to use API key when getting self
+    request(ctx, { method: "GET" }, true)
   )
   if (response.status !== 200) {
     ctx.throw(400, "Unable to get self globally.")
@@ -172,9 +167,6 @@ exports.removeAppFromUserRoles = async appId => {
     checkSlashesInUrl(env.WORKER_URL + `/api/admin/roles/${deployedAppId}`),
     request(null, {
       method: "DELETE",
-      headers: {
-        "x-budibase-api-key": env.INTERNAL_API_KEY,
-      },
     })
   )
   if (response.status !== 200) {

packages/standard-components/manifest.json

@@ -1505,5 +1505,38 @@
     "context": {
       "type": "schema"
     }
+  },
+  "daterangepicker": {
+    "name": "Date Range",
+    "icon": "Date",
+    "styleable": true,
+    "hasChildren": false,
+    "info": "Your data provider will be automatically filtered to the given date range.",
+    "settings": [
+      {
+        "type": "dataProvider",
+        "label": "Provider",
+        "key": "dataProvider"
+      },
+      {
+        "type": "field",
+        "label": "Date field",
+        "key": "field"
+      },
+      {
+        "type": "select",
+        "label": "Default range",
+        "key": "defaultValue",
+        "options": [
+          "Last 1 day",
+          "Last 7 days",
+          "Last 30 days",
+          "Last 3 months",
+          "Last 6 months",
+          "Last 1 year"
+        ],
+        "defaultValue": "Last 30 days"
+      }
+    ]
   }
 }

packages/standard-components/package.json

@@ -29,14 +29,15 @@
   "keywords": [
     "svelte"
   ],
-  "version": "0.9.25",
+  "version": "0.9.27",
   "license": "MIT",
   "gitHead": "d1836a898cab3f8ab80ee6d8f42be1a9eed7dcdc",
   "dependencies": {
-    "@budibase/bbui": "^0.9.25",
+    "@budibase/bbui": "^0.9.27",
     "@spectrum-css/page": "^3.0.1",
     "@spectrum-css/vars": "^3.0.1",
     "apexcharts": "^3.22.1",
+    "dayjs": "^1.10.5",
     "svelte-apexcharts": "^1.0.2",
     "svelte-flatpickr": "^3.1.0"
   }

packages/standard-components/src/DataProvider.svelte

@@ -1,6 +1,12 @@
 <script>
   import { getContext } from "svelte"
   import { ProgressCircle, Pagination } from "@budibase/bbui"
+  import {
+    buildLuceneQuery,
+    luceneQuery,
+    luceneSort,
+    luceneLimit,
+  } from "./lucene"
 
   export let dataSource
   export let filter
@@ -25,9 +31,11 @@
   let schema = {}
   let bookmarks = [null]
   let pageNumber = 0
+  let query = null
 
+  $: query = buildLuceneQuery(filter)
   $: internalTable = dataSource?.type === "table"
-  $: query = internalTable ? buildLuceneQuery(filter) : null
+  $: nestedProvider = dataSource?.type === "provider"
   $: hasNextPage = bookmarks[pageNumber + 1] != null
   $: hasPrevPage = pageNumber > 0
   $: getSchema(dataSource)
@@ -48,22 +56,38 @@
     }
   }
   $: {
-    // Sort and limit rows in memory when we aren't searching internal tables
     if (internalTable) {
+      // Internal tables are already processed server-side
       rows = allRows
     } else {
-      const sortedRows = sortRows(allRows, sortColumn, sortOrder)
-      rows = limitRows(sortedRows, limit)
+      // For anything else we use client-side implementations to filter, sort
+      // and limit
+      const filtered = luceneQuery(allRows, query)
+      const sorted = luceneSort(filtered, sortColumn, sortOrder, sortType)
+      rows = luceneLimit(sorted, limit)
     }
   }
   $: actions = [
     {
       type: ActionTypes.RefreshDatasource,
-      callback: () => fetchData(dataSource),
+      callback: () => refresh(),
       metadata: { dataSource },
     },
+    {
+      type: ActionTypes.SetDataProviderQuery,
+      callback: newQuery => (query = newQuery),
+    },
   ]
-  $: dataContext = { rows, schema, rowsLength: rows.length }
+  $: dataContext = {
+    rows,
+    schema,
+    rowsLength: rows.length,
+
+    // Undocumented properties. These aren't supposed to be used in builder
+    // bindings, but are used internally by other components
+    id: $component?.id,
+    state: { query },
+  }
 
   const getSortType = (schema, sortColumn) => {
     if (!schema || !sortColumn || !schema[sortColumn]) {
@@ -73,36 +97,18 @@
     return type === "number" ? "number" : "string"
   }
 
-  const buildLuceneQuery = filter => {
-    let query = {
-      string: {},
-      fuzzy: {},
-      range: {},
-      equal: {},
-      notEqual: {},
-      empty: {},
-      notEmpty: {},
+  const refresh = async () => {
+    if (schemaLoaded && !nestedProvider) {
+      fetchData(
+        dataSource,
+        query,
+        limit,
+        sortColumn,
+        sortOrder,
+        sortType,
+        paginate
+      )
     }
-    if (Array.isArray(filter)) {
-      filter.forEach(({ operator, field, type, value }) => {
-        if (operator.startsWith("range")) {
-          if (!query.range[field]) {
-            query.range[field] = {
-              low: type === "number" ? Number.MIN_SAFE_INTEGER : "0000",
-              high: type === "number" ? Number.MAX_SAFE_INTEGER : "9999",
-            }
-          }
-          if (operator === "rangeLow") {
-            query.range[field].low = value
-          } else if (operator === "rangeHigh") {
-            query.range[field].high = value
-          }
-        } else if (query[operator]) {
-          query[operator][field] = value
-        }
-      })
-    }
-    return query
   }
 
   const fetchData = async (
@@ -116,6 +122,7 @@
   ) => {
     loading = true
     if (dataSource?.type === "table") {
+      // For internal tables we use server-side processing
       const res = await API.searchTable({
         tableId: dataSource.tableId,
         query,
@@ -132,55 +139,27 @@
       } else {
         bookmarks = [null]
       }
+    } else if (dataSource?.type === "provider") {
+      // For providers referencing another provider, just use the rows it
+      // provides
+      allRows = dataSource?.value?.rows ?? []
     } else {
-      const rows = await API.fetchDatasource(dataSource)
-      allRows = inMemoryFilterRows(rows, filter)
+      // For other data sources like queries or views, fetch all rows from the
+      // server
+      allRows = await API.fetchDatasource(dataSource)
     }
     loading = false
     loaded = true
   }
 
-  const inMemoryFilterRows = (rows, filter) => {
-    let filteredData = [...rows]
-    Object.entries(filter || {}).forEach(([field, value]) => {
-      if (value != null && value !== "") {
-        filteredData = filteredData.filter(row => {
-          return row[field] === value
-        })
-      }
-    })
-    return filteredData
-  }
-
-  const sortRows = (rows, sortColumn, sortOrder) => {
-    if (!sortColumn || !sortOrder) {
-      return rows
-    }
-    return rows.slice().sort((a, b) => {
-      const colA = a[sortColumn]
-      const colB = b[sortColumn]
-      if (sortOrder === "Descending") {
-        return colA > colB ? -1 : 1
-      } else {
-        return colA > colB ? 1 : -1
-      }
-    })
-  }
-
-  const limitRows = (rows, limit) => {
-    const numLimit = parseFloat(limit)
-    if (isNaN(numLimit)) {
-      return rows
-    }
-    return rows.slice(0, numLimit)
-  }
-
   const getSchema = async dataSource => {
     if (dataSource?.schema) {
       schema = dataSource.schema
     } else if (dataSource?.tableId) {
       const definition = await API.fetchTableDefinition(dataSource.tableId)
       schema = definition?.schema ?? {}
+    } else if (dataSource?.type === "provider") {
+      schema = dataSource.value?.schema ?? {}
     } else {
       schema = {}
     }

packages/standard-components/src/DateRangePicker.svelte

@@ -0,0 +1,77 @@
+<script>
+  import { Select } from "@budibase/bbui"
+  import { getContext } from "svelte"
+  import dayjs from "dayjs"
+  import utc from "dayjs/plugin/utc"
+  import { onMount } from "svelte"
+
+  dayjs.extend(utc)
+
+  export let dataProvider
+  export let field
+  export let defaultValue
+
+  const dataContext = getContext("context")
+  const component = getContext("component")
+  const { styleable, builderStore, ActionTypes, getAction } = getContext("sdk")
+
+  const setQuery = getAction(dataProvider?.id, ActionTypes.SetDataProviderQuery)
+  const options = [
+    "Last 1 day",
+    "Last 7 days",
+    "Last 30 days",
+    "Last 3 months",
+    "Last 6 months",
+    "Last 1 year",
+  ]
+  let value = options.includes(defaultValue) ? defaultValue : "Last 30 days"
+
+  const updateDateRange = option => {
+    const query = dataProvider?.state?.query
+    if (!query || !setQuery) {
+      return
+    }
+
+    value = option
+    let low = dayjs.utc().subtract(1, "year")
+    let high = dayjs.utc().add(1, "day")
+
+    if (option === "Last 1 day") {
+      low = dayjs.utc().subtract(1, "day")
+    } else if (option === "Last 7 days") {
+      low = dayjs.utc().subtract(7, "days")
+    } else if (option === "Last 30 days") {
+      low = dayjs.utc().subtract(30, "days")
+    } else if (option === "Last 3 months") {
+      low = dayjs.utc().subtract(3, "months")
+    } else if (option === "Last 6 months") {
+      low = dayjs.utc().subtract(6, "months")
+    }
+
+    // Update data provider query with the new filter
+    setQuery({
+      ...query,
+      range: {
+        ...query.range,
+        [field]: {
+          high: high.format(),
+          low: low.format(),
+        },
+      },
+    })
+  }
+
+  // Update the range on mount to the initial value
+  onMount(() => {
+    updateDateRange(value)
+  })
+</script>
+
+<div use:styleable={$component.styles}>
+  <Select
+    placeholder={null}
+    {options}
+    {value}
+    on:change={e => updateDateRange(e.detail)}
+  />
+</div>

packages/standard-components/src/forms/StringField.svelte

@@ -22,6 +22,7 @@
 >
   {#if fieldState}
     <CoreTextField
+      updateOnChange={false}
       value={$fieldState.value}
       on:change={e => fieldApi.setValue(e.detail)}
       disabled={$fieldState.disabled}

packages/standard-components/src/index.js

@@ -27,6 +27,7 @@ export { default as cardhorizontal } from "./CardHorizontal.svelte"
 export { default as cardstat } from "./CardStat.svelte"
 export { default as icon } from "./Icon.svelte"
 export { default as backgroundimage } from "./BackgroundImage.svelte"
+export { default as daterangepicker } from "./DateRangePicker.svelte"
 export * from "./charts"
 export * from "./forms"
 export * from "./table"

packages/standard-components/src/lucene.js

@@ -0,0 +1,146 @@
+/**
+ * Builds a lucene JSON query from the filter structure generated in the builder
+ * @param filter the builder filter structure
+ */
+export const buildLuceneQuery = filter => {
+  let query = {
+    string: {},
+    fuzzy: {},
+    range: {},
+    equal: {},
+    notEqual: {},
+    empty: {},
+    notEmpty: {},
+  }
+  if (Array.isArray(filter)) {
+    filter.forEach(({ operator, field, type, value }) => {
+      if (operator.startsWith("range")) {
+        if (!query.range[field]) {
+          query.range[field] = {
+            low:
+              type === "number"
+                ? Number.MIN_SAFE_INTEGER
+                : "0000-00-00T00:00:00.000Z",
+            high:
+              type === "number"
+                ? Number.MAX_SAFE_INTEGER
+                : "9999-00-00T00:00:00.000Z",
+          }
+        }
+        if (operator === "rangeLow" && value != null && value !== "") {
+          query.range[field].low = value
+        } else if (operator === "rangeHigh" && value != null && value !== "") {
+          query.range[field].high = value
+        }
+      } else if (query[operator]) {
+        query[operator][field] = value
+      }
+    })
+  }
+  return query
+}
+
+/**
+ * Performs a client-side lucene search on an array of data
+ * @param docs the data
+ * @param query the JSON lucene query
+ */
+export const luceneQuery = (docs, query) => {
+  if (!query) {
+    return docs
+  }
+
+  // Iterates over a set of filters and evaluates a fail function against a doc
+  const match = (type, failFn) => doc => {
+    const filters = Object.entries(query[type] || {})
+    for (let i = 0; i < filters.length; i++) {
+      if (failFn(filters[i][0], filters[i][1], doc)) {
+        return false
+      }
+    }
+    return true
+  }
+
+  // Process a string match (fails if the value does not start with the string)
+  const stringMatch = match("string", (key, value, doc) => {
+    return !doc[key] || !doc[key].startsWith(value)
+  })
+
+  // Process a range match
+  const rangeMatch = match("range", (key, value, doc) => {
+    return !doc[key] || doc[key] < value.low || doc[key] > value.high
+  })
+
+  // Process an equal match (fails if the value is different)
+  const equalMatch = match("equal", (key, value, doc) => {
+    return doc[key] !== value
+  })
+
+  // Process a not-equal match (fails if the value is the same)
+  const notEqualMatch = match("notEqual", (key, value, doc) => {
+    return doc[key] === value
+  })
+
+  // Process an empty match (fails if the value is not empty)
+  const emptyMatch = match("empty", (key, value, doc) => {
+    return doc[key] != null && doc[key] !== ""
+  })
+
+  // Process a not-empty match (fails is the value is empty)
+  const notEmptyMatch = match("notEmpty", (key, value, doc) => {
+    return doc[key] == null || doc[key] === ""
+  })
+
+  // Match a document against all criteria
+  const docMatch = doc => {
+    return (
+      stringMatch(doc) &&
+      rangeMatch(doc) &&
+      equalMatch(doc) &&
+      notEqualMatch(doc) &&
+      emptyMatch(doc) &&
+      notEmptyMatch(doc)
+    )
+  }
+
+  // Process all docs
+  return docs.filter(docMatch)
+}
+
+/**
+ * Performs a client-side sort from the equivalent server-side lucene sort
+ * parameters.
+ * @param docs the data
+ * @param sort the sort column
+ * @param sortOrder the sort order ("ascending" or "descending")
+ * @param sortType the type of sort ("string" or "number")
+ */
+export const luceneSort = (docs, sort, sortOrder, sortType = "string") => {
+  if (!sort || !sortOrder || !sortType) {
+    return docs
+  }
+  const parse = sortType === "string" ? x => `${x}` : x => parseFloat(x)
+  return docs.slice().sort((a, b) => {
+    const colA = parse(a[sort])
+    const colB = parse(b[sort])
+    if (sortOrder === "Descending") {
+      return colA > colB ? -1 : 1
+    } else {
+      return colA > colB ? 1 : -1
+    }
+  })
+}
+
+/**
+ * Limits the specified docs to the specified number of rows from the equivalent
+ * server-side lucene limit parameters.
+ * @param docs the data
+ * @param limit the number of docs to limit to
+ */
+export const luceneLimit = (docs, limit) => {
+  const numLimit = parseFloat(limit)
+  if (isNaN(numLimit)) {
+    return docs
+  }
+  return docs.slice(0, numLimit)
+}

packages/standard-components/yarn.lock

@@ -80,6 +80,11 @@ colorette@^1.2.2:
   resolved "https://registry.yarnpkg.com/colorette/-/colorette-1.2.2.tgz#cbcc79d5e99caea2dbf10eb3a26fd8b3e6acfa94"
   integrity sha512-MKGMzyfeuutC/ZJ1cba9NqcNpfeqMUcYmyF1ZFY6/Cn7CNSAKx6a+s48sqLqyAiZuaP2TcqMhoo+dlwFnVxT9w==
 
+dayjs@^1.10.5:
+  version "1.10.5"
+  resolved "https://registry.yarnpkg.com/dayjs/-/dayjs-1.10.5.tgz#5600df4548fc2453b3f163ebb2abbe965ccfb986"
+  integrity sha512-BUFis41ikLz+65iH6LHQCDm4YPMj5r1YFLdupPIyM4SGcXMmtiLQ7U37i+hGS8urIuqe7I/ou3IS1jVc4nbN4g==
+
 debug@^4.3.2:
   version "4.3.2"
   resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.2.tgz#f0a49c18ac8779e31d4a0c6029dfb76873c7428b"

packages/string-templates/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/string-templates",
-  "version": "0.9.25",
+  "version": "0.9.27",
   "description": "Handlebars wrapper for Budibase templating.",
   "main": "src/index.cjs",
   "module": "dist/bundle.mjs",

packages/string-templates/src/index.cjs

@@ -33,7 +33,7 @@ function testObject(object) {
  */
 module.exports.processObject = async (object, context) => {
   testObject(object)
-  for (let key of Object.keys(object)) {
+  for (let key of Object.keys(object || {})) {
     if (object[key] != null) {
       let val = object[key]
       if (typeof val === "string") {
@@ -68,7 +68,7 @@ module.exports.processString = async (string, context) => {
  */
 module.exports.processObjectSync = (object, context) => {
   testObject(object)
-  for (let key of Object.keys(object)) {
+  for (let key of Object.keys(object || {})) {
     let val = object[key]
     if (typeof val === "string") {
       object[key] = module.exports.processStringSync(object[key], context)

packages/worker/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/worker",
   "email": "hi@budibase.com",
-  "version": "0.9.25",
+  "version": "0.9.27",
   "description": "Budibase background service",
   "main": "src/index.js",
   "repository": {
@@ -21,8 +21,8 @@
   "author": "Budibase",
   "license": "AGPL-3.0-or-later",
   "dependencies": {
-    "@budibase/auth": "^0.9.25",
-    "@budibase/string-templates": "^0.9.25",
+    "@budibase/auth": "^0.9.27",
+    "@budibase/string-templates": "^0.9.27",
     "@koa/router": "^8.0.0",
     "aws-sdk": "^2.811.0",
     "bcryptjs": "^2.4.3",