Merge pull request #13549 from Budibase/feature/filter-bindings

Filter bindings

packages/backend-core/src/context/mainContext.ts

@@ -281,7 +281,7 @@ export function doInScimContext(task: any) {
   return newContext(updates, task)
 }
 
-export async function ensureSnippetContext() {
+export async function ensureSnippetContext(enabled = !env.isTest()) {
   const ctx = getCurrentContext()
 
   // If we've already added snippets to context, continue
@@ -292,7 +292,7 @@ export async function ensureSnippetContext() {
   // Otherwise get snippets for this app and update context
   let snippets: Snippet[] | undefined
   const db = getAppDB()
-  if (db && !env.isTest()) {
+  if (db && enabled) {
     const app = await db.get<App>(DocumentType.APP_METADATA)
     snippets = app.snippets
   }

packages/builder/src/components/backend/DataTable/buttons/TableFilterButton.svelte

@@ -1,7 +1,9 @@
 <script>
   import { createEventDispatcher } from "svelte"
-  import { ActionButton, Modal, ModalContent } from "@budibase/bbui"
+  import { ActionButton, Drawer, DrawerContent, Button } from "@budibase/bbui"
   import FilterBuilder from "components/design/settings/controls/FilterEditor/FilterBuilder.svelte"
+  import { getUserBindings } from "dataBinding"
+  import { makePropSafe } from "@budibase/string-templates"
 
   export let schema
   export let filters
@@ -10,7 +12,7 @@
 
   const dispatch = createEventDispatcher()
 
-  let modal
+  let drawer
 
   $: tempValue = filters || []
   $: schemaFields = Object.entries(schema || {}).map(
@@ -22,37 +24,53 @@
 
   $: text = getText(filters)
   $: selected = tempValue.filter(x => !x.onEmptyFilter)?.length > 0
-
+  $: bindings = [
+    {
+      type: "context",
+      runtimeBinding: `${makePropSafe("now")}`,
+      readableBinding: `Date`,
+      category: "Date",
+      icon: "Date",
+      display: {
+        name: "Server date",
+      },
+    },
+    ...getUserBindings(),
+  ]
   const getText = filters => {
     const count = filters?.filter(filter => filter.field)?.length
     return count ? `Filter (${count})` : "Filter"
   }
 </script>
 
-<ActionButton icon="Filter" quiet {disabled} on:click={modal.show} {selected}>
+<ActionButton icon="Filter" quiet {disabled} on:click={drawer.show} {selected}>
   {text}
 </ActionButton>
-<Modal bind:this={modal}>
-  <ModalContent
-    title="Filter"
-    confirmText="Save"
-    size="XL"
-    onConfirm={() => dispatch("change", tempValue)}
-  >
-    <div class="wrapper">
-      <FilterBuilder
-        allowBindings={false}
-        {filters}
-        {schemaFields}
-        datasource={{ type: "table", tableId }}
-        on:change={e => (tempValue = e.detail)}
-      />
-    </div>
-  </ModalContent>
-</Modal>
 
-<style>
-  .wrapper :global(.main) {
-    padding: 0;
-  }
-</style>
+<Drawer
+  bind:this={drawer}
+  title="Filtering"
+  on:drawerHide
+  on:drawerShow
+  forceModal
+>
+  <Button
+    cta
+    slot="buttons"
+    on:click={() => {
+      dispatch("change", tempValue)
+      drawer.hide()
+    }}
+  >
+    Save
+  </Button>
+  <DrawerContent slot="body">
+    <FilterBuilder
+      {filters}
+      {schemaFields}
+      datasource={{ type: "table", tableId }}
+      on:change={e => (tempValue = e.detail)}
+      {bindings}
+    />
+  </DrawerContent>
+</Drawer>

packages/frontend-core/src/components/FilterBuilder.svelte

@@ -289,6 +289,7 @@
                     OperatorOptions.ContainsAny.value,
                   ].includes(filter.operator)}
                   disabled={filter.noValue}
+                  type={filter.valueType}
                 />
               {:else}
                 <Input disabled />
@@ -325,8 +326,6 @@
 <style>
   .container {
     width: 100%;
-    max-width: 1000px;
-    margin: 0 auto;
   }
   .fields {
     display: grid;

packages/frontend-core/src/components/FilterUsers.svelte

@@ -4,6 +4,7 @@
   import { createAPIClient } from "../api"
 
   export let API = createAPIClient()
+
   export let value = null
   export let disabled
   export let multiselect = false
@@ -23,12 +24,14 @@
   $: component = multiselect ? Multiselect : Select
 </script>
 
-<svelte:component
-  this={component}
-  bind:value
-  autocomplete
-  {options}
-  getOptionLabel={option => option.email}
-  getOptionValue={option => option._id}
-  {disabled}
-/>
+<div class="user-control">
+  <svelte:component
+    this={component}
+    bind:value
+    autocomplete
+    {options}
+    getOptionLabel={option => option.email}
+    getOptionValue={option => option._id}
+    {disabled}
+  />
+</div>

packages/server/src/api/controllers/row/index.ts

@@ -2,7 +2,7 @@ import stream from "stream"
 import archiver from "archiver"
 
 import { quotas } from "@budibase/pro"
-import { objectStore } from "@budibase/backend-core"
+import { objectStore, context } from "@budibase/backend-core"
 import * as internal from "./internal"
 import * as external from "./external"
 import { isExternalTableID } from "../../../integrations/utils"
@@ -198,8 +198,18 @@ export async function destroy(ctx: UserCtx<DeleteRowRequest>) {
 export async function search(ctx: Ctx<SearchRowRequest, SearchRowResponse>) {
   const tableId = utils.getTableId(ctx)
 
+  await context.ensureSnippetContext(true)
+
+  const enrichedQuery = await utils.enrichSearchContext(
+    { ...ctx.request.body.query },
+    {
+      user: sdk.users.getUserContextBindings(ctx.user),
+    }
+  )
+
   const searchParams: RowSearchParams = {
     ...ctx.request.body,
+    query: enrichedQuery,
     tableId,
   }
 

packages/server/src/api/controllers/row/utils/utils.ts

@@ -22,7 +22,7 @@ import {
   getInternalRowId,
 } from "./basic"
 import sdk from "../../../../sdk"
-
+import { processStringSync } from "@budibase/string-templates"
 import validateJs from "validate.js"
 
 validateJs.extend(validateJs.validators.datetime, {
@@ -204,3 +204,63 @@ export async function sqlOutputProcessing(
 export function isUserMetadataTable(tableId: string) {
   return tableId === InternalTables.USER_METADATA
 }
+
+export async function enrichArrayContext(
+  fields: any[],
+  inputs = {},
+  helpers = true
+): Promise<any[]> {
+  const map: Record<string, any> = {}
+  for (let index in fields) {
+    map[index] = fields[index]
+  }
+  const output = await enrichSearchContext(map, inputs, helpers)
+  const outputArray: any[] = []
+  for (let [key, value] of Object.entries(output)) {
+    outputArray[parseInt(key)] = value
+  }
+  return outputArray
+}
+
+export async function enrichSearchContext(
+  fields: Record<string, any>,
+  inputs = {},
+  helpers = true
+): Promise<Record<string, any>> {
+  const enrichedQuery: Record<string, any> = {}
+  if (!fields || !inputs) {
+    return enrichedQuery
+  }
+  const parameters = { ...inputs }
+
+  if (Array.isArray(fields)) {
+    return enrichArrayContext(fields, inputs, helpers)
+  }
+
+  // enrich the fields with dynamic parameters
+  for (let key of Object.keys(fields)) {
+    if (fields[key] == null) {
+      enrichedQuery[key] = null
+      continue
+    }
+    if (typeof fields[key] === "object") {
+      // enrich nested fields object
+      enrichedQuery[key] = await enrichSearchContext(
+        fields[key],
+        parameters,
+        helpers
+      )
+    } else if (typeof fields[key] === "string") {
+      // enrich string value as normal
+      enrichedQuery[key] = processStringSync(fields[key], parameters, {
+        noEscaping: true,
+        noHelpers: !helpers,
+        escapeNewlines: true,
+      })
+    } else {
+      enrichedQuery[key] = fields[key]
+    }
+  }
+
+  return enrichedQuery
+}

packages/server/src/api/controllers/row/views.ts

@@ -9,7 +9,8 @@ import {
 } from "@budibase/types"
 import { dataFilters } from "@budibase/shared-core"
 import sdk from "../../../sdk"
-import { db } from "@budibase/backend-core"
+import { db, context } from "@budibase/backend-core"
+import { enrichSearchContext } from "./utils"
 
 export async function searchView(
   ctx: UserCtx<SearchViewRowRequest, SearchRowResponse>
@@ -56,10 +57,16 @@ export async function searchView(
     })
   }
 
+  await context.ensureSnippetContext(true)
+
+  const enrichedQuery = await enrichSearchContext(query, {
+    user: sdk.users.getUserContextBindings(ctx.user),
+  })
+
   const searchOptions: RequiredKeys<SearchViewRowRequest> &
     RequiredKeys<Pick<RowSearchParams, "tableId" | "query" | "fields">> = {
     tableId: view.tableId,
-    query,
+    query: enrichedQuery,
     fields: viewFields,
     ...getSortOptions(body, view),
     limit: body.limit,

packages/server/src/api/routes/tests/search.spec.ts

@@ -1,11 +1,13 @@
 import { tableForDatasource } from "../../../tests/utilities/structures"
 import { DatabaseName, getDatasource } from "../../../integrations/tests/utils"
+import { db as dbCore } from "@budibase/backend-core"
 
 import * as setup from "./utilities"
 import {
   AutoFieldSubType,
   Datasource,
   EmptyFilterOption,
+  BBReferenceFieldSubType,
   FieldType,
   RowSearchParams,
   SearchFilters,
@@ -13,8 +15,14 @@ import {
   SortType,
   Table,
   TableSchema,
+  User,
 } from "@budibase/types"
 import _ from "lodash"
+import tk from "timekeeper"
+import { encodeJSBinding } from "@budibase/string-templates"
+
+const serverTime = new Date("2024-05-06T00:00:00.000Z")
+tk.freeze(serverTime)
 
 describe.each([
   ["lucene", undefined],
@@ -33,11 +41,25 @@ describe.each([
   let datasource: Datasource | undefined
   let table: Table
 
+  const snippets = [
+    {
+      name: "WeeksAgo",
+      code: "return function (weeks) {\n  const currentTime = new Date();\n  currentTime.setDate(currentTime.getDate()-(7 * (weeks || 1)));\n  return currentTime.toISOString();\n}",
+    },
+  ]
+
   beforeAll(async () => {
     if (isSqs) {
       envCleanup = config.setEnv({ SQS_SEARCH_ENABLE: "true" })
     }
     await config.init()
+
+    if (config.app?.appId) {
+      config.app = await config.api.application.update(config.app?.appId, {
+        snippets,
+      })
+    }
+
     if (dsProvider) {
       datasource = await config.createDatasource({
         datasource: await dsProvider,
@@ -225,6 +247,230 @@ describe.each([
     })
   })
 
+  // Ensure all bindings resolve and perform as expected
+  describe("bindings", () => {
+    let globalUsers: any = []
+
+    const future = new Date(serverTime.getTime())
+    future.setDate(future.getDate() + 30)
+
+    const rows = (currentUser: User) => {
+      return [
+        { name: "foo", appointment: "1982-01-05T00:00:00.000Z" },
+        { name: "bar", appointment: "1995-05-06T00:00:00.000Z" },
+        { name: currentUser.firstName, appointment: future.toISOString() },
+        { name: "serverDate", appointment: serverTime.toISOString() },
+        {
+          name: "single user, session user",
+          single_user: JSON.stringify([currentUser]),
+        },
+        {
+          name: "single user",
+          single_user: JSON.stringify([globalUsers[0]]),
+        },
+        {
+          name: "multi user",
+          multi_user: JSON.stringify(globalUsers),
+        },
+        {
+          name: "multi user with session user",
+          multi_user: JSON.stringify([...globalUsers, currentUser]),
+        },
+      ]
+    }
+
+    beforeAll(async () => {
+      // Set up some global users
+      globalUsers = await Promise.all(
+        Array(2)
+          .fill(0)
+          .map(async () => {
+            const globalUser = await config.globalUser()
+            const userMedataId = globalUser._id
+              ? dbCore.generateUserMetadataID(globalUser._id)
+              : null
+            return {
+              _id: globalUser._id,
+              _meta: userMedataId,
+            }
+          })
+      )
+
+      await createTable({
+        name: { name: "name", type: FieldType.STRING },
+        appointment: { name: "appointment", type: FieldType.DATETIME },
+        single_user: {
+          name: "single_user",
+          type: FieldType.BB_REFERENCE,
+          subtype: BBReferenceFieldSubType.USER,
+        },
+        multi_user: {
+          name: "multi_user",
+          type: FieldType.BB_REFERENCE,
+          subtype: BBReferenceFieldSubType.USERS,
+        },
+      })
+      await createRows(rows(config.getUser()))
+    })
+
+    // !! Current User is auto generated per run
+    it("should return all rows matching the session user firstname", async () => {
+      await expectQuery({
+        equal: { name: "{{ [user].firstName }}" },
+      }).toContainExactly([
+        {
+          name: config.getUser().firstName,
+          appointment: future.toISOString(),
+        },
+      ])
+    })
+
+    it("should parse the date binding and return all rows after the resolved value", async () => {
+      await expectQuery({
+        range: {
+          appointment: {
+            low: "{{ [now] }}",
+            high: "9999-00-00T00:00:00.000Z",
+          },
+        },
+      }).toContainExactly([
+        {
+          name: config.getUser().firstName,
+          appointment: future.toISOString(),
+        },
+        { name: "serverDate", appointment: serverTime.toISOString() },
+      ])
+    })
+
+    it("should parse the date binding and return all rows before the resolved value", async () => {
+      await expectQuery({
+        range: {
+          appointment: {
+            low: "0000-00-00T00:00:00.000Z",
+            high: "{{ [now] }}",
+          },
+        },
+      }).toContainExactly([
+        { name: "foo", appointment: "1982-01-05T00:00:00.000Z" },
+        { name: "bar", appointment: "1995-05-06T00:00:00.000Z" },
+        { name: "serverDate", appointment: serverTime.toISOString() },
+      ])
+    })
+
+    it("should parse the encoded js snippet. Return rows with appointments up to 1 week in the past", async () => {
+      const jsBinding = "return snippets.WeeksAgo();"
+      const encodedBinding = encodeJSBinding(jsBinding)
+
+      await expectQuery({
+        range: {
+          appointment: {
+            low: "0000-00-00T00:00:00.000Z",
+            high: encodedBinding,
+          },
+        },
+      }).toContainExactly([
+        { name: "foo", appointment: "1982-01-05T00:00:00.000Z" },
+        { name: "bar", appointment: "1995-05-06T00:00:00.000Z" },
+      ])
+    })
+
+    it("should parse the encoded js binding. Return rows with appointments 2 weeks in the past", async () => {
+      const jsBinding =
+        "const currentTime = new Date()\ncurrentTime.setDate(currentTime.getDate()-14);\nreturn currentTime.toISOString();"
+      const encodedBinding = encodeJSBinding(jsBinding)
+
+      await expectQuery({
+        range: {
+          appointment: {
+            low: "0000-00-00T00:00:00.000Z",
+            high: encodedBinding,
+          },
+        },
+      }).toContainExactly([
+        { name: "foo", appointment: "1982-01-05T00:00:00.000Z" },
+        { name: "bar", appointment: "1995-05-06T00:00:00.000Z" },
+      ])
+    })
+
+    it("should match a single user row by the session user id", async () => {
+      await expectQuery({
+        equal: { single_user: "{{ [user]._id }}" },
+      }).toContainExactly([
+        {
+          name: "single user, session user",
+          single_user: [{ _id: config.getUser()._id }],
+        },
+      ])
+    })
+    // TODO(samwho): fix for SQS
+    !isSqs &&
+      it("should match the session user id in a multi user field", async () => {
+        const allUsers = [...globalUsers, config.getUser()].map((user: any) => {
+          return { _id: user._id }
+        })
+
+        await expectQuery({
+          contains: { multi_user: ["{{ [user]._id }}"] },
+        }).toContainExactly([
+          {
+            name: "multi user with session user",
+            multi_user: allUsers,
+          },
+        ])
+      })
+
+    !isSqs &&
+      it("should not match the session user id in a multi user field", async () => {
+        await expectQuery({
+          notContains: { multi_user: ["{{ [user]._id }}"] },
+          notEmpty: { multi_user: true },
+        }).toContainExactly([
+          {
+            name: "multi user",
+            multi_user: globalUsers.map((user: any) => {
+              return { _id: user._id }
+            }),
+          },
+        ])
+      })
+
+    it("should match the session user id and a user table row id using helpers, user binding and a static user id.", async () => {
+      await expectQuery({
+        oneOf: {
+          single_user: [
+            "{{ default [user]._id '_empty_' }}",
+            globalUsers[0]._id,
+          ],
+        },
+      }).toContainExactly([
+        {
+          name: "single user, session user",
+          single_user: [{ _id: config.getUser()._id }],
+        },
+        {
+          name: "single user",
+          single_user: [{ _id: globalUsers[0]._id }],
+        },
+      ])
+    })
+
+    it("should resolve 'default' helper to '_empty_' when binding resolves to nothing", async () => {
+      await expectQuery({
+        oneOf: {
+          single_user: [
+            "{{ default [user]._idx '_empty_' }}",
+            globalUsers[0]._id,
+          ],
+        },
+      }).toContainExactly([
+        {
+          name: "single user",
+          single_user: [{ _id: globalUsers[0]._id }],
+        },
+      ])
+    })
+  })
+
   describe.each([FieldType.STRING, FieldType.LONGFORM])("%s", () => {
     beforeAll(async () => {
       await createTable({

packages/server/src/sdk/users/utils.ts

@@ -124,3 +124,12 @@ export async function syncGlobalUsers() {
     await db.bulkDocs(toWrite)
   }
 }
+
+export function getUserContextBindings(user: ContextUser) {
+  if (!user) {
+    return {}
+  }
+  // Current user context for bindable search
+  const { _id, _rev, firstName, lastName, email, status, roleId } = user
+  return { _id, _rev, firstName, lastName, email, status, roleId }
+}

packages/server/src/tests/utilities/TestConfiguration.ts

@@ -81,6 +81,12 @@ mocks.licenses.useUnlimited()
 
 dbInit()
 
+export interface CreateAppRequest {
+  appName: string
+  url?: string
+  snippets?: any[]
+}
+
 export interface TableToBuild extends Omit<Table, "sourceId" | "sourceType"> {
   sourceId?: string
   sourceType?: TableSourceType
@@ -580,8 +586,6 @@ export default class TestConfiguration {
 
   // APP
   async createApp(appName: string, url?: string): Promise<App> {
-    // create dev app
-    // clear any old app
     this.appId = undefined
     this.app = await context.doInTenant(
       this.tenantId!,
@@ -592,6 +596,7 @@ export default class TestConfiguration {
         })) as App
     )
     this.appId = this.app.appId
+
     return await context.doInAppContext(this.app.appId!, async () => {
       // create production app
       this.prodApp = await this.publish()