Merge branch 'master' into budi-8166/search-filter-operators-dont-have-options-for-some-types

.github/workflows/budibase_ci.yml

@@ -64,10 +64,11 @@ jobs:
       - run: yarn --frozen-lockfile
 
       # Run build all the projects
-      - name: Build
-        run: |
-          yarn build:oss
-          yarn build:account-portal
+      - name: Build OSS
+        run: yarn build:oss
+      - name: Build account portal
+        run: yarn build:account-portal
+        if: ${{ env.IS_OSS_CONTRIBUTOR == 'false' }}
       # Check the types of the projects built via esbuild
       - name: Check types
         run: |

hosting/proxy/nginx.prod.conf

@@ -51,7 +51,7 @@ http {
     proxy_buffering off;
 
     set $csp_default "default-src 'self'";
-    set $csp_script "script-src 'self' 'unsafe-inline' 'unsafe-eval' https://*.budibase.net https://cdn.budi.live https://js.intercomcdn.com https://widget.intercom.io https://d2l5prqdbvm3op.cloudfront.net";
+    set $csp_script "script-src 'self' 'unsafe-inline' 'unsafe-eval' https://*.budibase.net https://cdn.budi.live https://js.intercomcdn.com https://widget.intercom.io https://d2l5prqdbvm3op.cloudfront.net https://us-assets.i.posthog.com";
     set $csp_style "style-src 'self' 'unsafe-inline' https://cdn.jsdelivr.net https://fonts.googleapis.com https://rsms.me https://maxcdn.bootstrapcdn.com";
     set $csp_object "object-src 'none'";
     set $csp_base_uri "base-uri 'self'";

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "2.23.1",
+  "version": "2.23.4",
   "npmClient": "yarn",
   "packages": [
     "packages/*",

packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/ComponentList/ComponentKeyHandler.svelte

@@ -137,8 +137,12 @@
     const activeTag = document.activeElement?.tagName.toLowerCase()
     const inCodeEditor =
       document.activeElement?.classList?.contains("cm-content")
+    const inPosthogSurvey =
+      document.activeElement?.classList?.[0]?.startsWith("PostHogSurvey")
     if (
-      (inCodeEditor || ["input", "textarea"].indexOf(activeTag) !== -1) &&
+      (inCodeEditor ||
+        inPosthogSurvey ||
+        ["input", "textarea"].indexOf(activeTag) !== -1) &&
       e.key !== "Escape"
     ) {
       return

packages/server/src/api/controllers/row/external.ts

@@ -17,11 +17,9 @@ import {
   Row,
   Table,
   UserCtx,
-  EmptyFilterOption,
 } from "@budibase/types"
 import sdk from "../../../sdk"
 import * as utils from "./utils"
-import { dataFilters } from "@budibase/shared-core"
 import {
   inputProcessing,
   outputProcessing,
@@ -33,17 +31,6 @@ export async function handleRequest<T extends Operation>(
   tableId: string,
   opts?: RunConfig
 ): Promise<ExternalRequestReturnType<T>> {
-  // make sure the filters are cleaned up, no empty strings for equals, fuzzy or string
-  if (opts && opts.filters) {
-    opts.filters = sdk.rows.removeEmptyFilters(opts.filters)
-  }
-  if (
-    !dataFilters.hasFilters(opts?.filters) &&
-    opts?.filters?.onEmptyFilter === EmptyFilterOption.RETURN_NONE
-  ) {
-    return [] as any
-  }
-
   return new ExternalRequest<T>(operation, tableId, opts?.datasource).run(
     opts || {}
   )

packages/server/src/api/routes/tests/search.spec.ts

@@ -6,9 +6,12 @@ import {
   Datasource,
   EmptyFilterOption,
   FieldType,
+  RowSearchParams,
   SearchFilters,
   Table,
+  TableSchema,
 } from "@budibase/types"
+import _ from "lodash"
 
 jest.unmock("mssql")
 
@@ -24,8 +27,8 @@ describe.each([
   const config = setup.getConfig()
 
   let envCleanup: (() => void) | undefined
-  let table: Table
   let datasource: Datasource | undefined
+  let table: Table
 
   beforeAll(async () => {
     if (isSqs) {
@@ -46,231 +49,217 @@ describe.each([
     }
   })
 
+  async function createTable(schema: TableSchema) {
+    table = await config.api.table.save(
+      tableForDatasource(datasource, { schema })
+    )
+  }
+
+  async function createRows(rows: Record<string, any>[]) {
+    await Promise.all(rows.map(r => config.api.row.save(table._id!, r)))
+  }
+
+  class SearchAssertion {
+    constructor(private readonly query: RowSearchParams) {}
+
+    async toFind(expectedRows: any[]) {
+      const { rows: foundRows } = await config.api.row.search(table._id!, {
+        ...this.query,
+        tableId: table._id!,
+      })
+
+      // eslint-disable-next-line jest/no-standalone-expect
+      expect(foundRows).toHaveLength(expectedRows.length)
+      // eslint-disable-next-line jest/no-standalone-expect
+      expect(foundRows).toEqual(
+        expect.arrayContaining(
+          expectedRows.map((expectedRow: any) =>
+            expect.objectContaining(
+              foundRows.find(foundRow => _.isMatch(foundRow, expectedRow))
+            )
+          )
+        )
+      )
+    }
+
+    async toFindNothing() {
+      await this.toFind([])
+    }
+  }
+
+  function expectSearch(query: Omit<RowSearchParams, "tableId">) {
+    return new SearchAssertion({ ...query, tableId: table._id! })
+  }
+
+  function expectQuery(query: SearchFilters) {
+    return expectSearch({ query })
+  }
+
   describe("strings", () => {
-    beforeEach(async () => {
-      table = await config.api.table.save(
-        tableForDatasource(datasource, {
-          schema: {
-            name: {
-              name: "name",
-              type: FieldType.STRING,
-            },
-          },
+    beforeAll(async () => {
+      await createTable({
+        name: { name: "name", type: FieldType.STRING },
       })
-      )
+      await createRows([{ name: "foo" }, { name: "bar" }])
     })
 
-    const rows = [{ name: "foo" }, { name: "bar" }]
+    describe("misc", () => {
+      it("should return all if no query is passed", () =>
+        expectSearch({} as RowSearchParams).toFind([
+          { name: "foo" },
+          { name: "bar" },
+        ]))
 
-    interface StringSearchTest {
-      query: SearchFilters
-      expected: (typeof rows)[number][]
-    }
+      it("should return all if empty query is passed", () =>
+        expectQuery({}).toFind([{ name: "foo" }, { name: "bar" }]))
 
-    const stringSearchTests: StringSearchTest[] = [
-      { query: {}, expected: rows },
-      {
-        query: { onEmptyFilter: EmptyFilterOption.RETURN_ALL },
-        expected: rows,
-      },
-      {
-        query: { onEmptyFilter: EmptyFilterOption.RETURN_NONE },
-        expected: [],
-      },
-      { query: { string: { name: "foo" } }, expected: [rows[0]] },
-      { query: { string: { name: "none" } }, expected: [] },
-      { query: { fuzzy: { name: "oo" } }, expected: [rows[0]] },
-      { query: { equal: { name: "foo" } }, expected: [rows[0]] },
-      { query: { notEqual: { name: "foo" } }, expected: [rows[1]] },
-      { query: { oneOf: { name: ["foo"] } }, expected: [rows[0]] },
-    ]
+      it("should return all if onEmptyFilter is RETURN_ALL", () =>
+        expectQuery({
+          onEmptyFilter: EmptyFilterOption.RETURN_ALL,
+        }).toFind([{ name: "foo" }, { name: "bar" }]))
 
-    it.each(stringSearchTests)(
-      `should be able to run query: $query`,
-      async ({ query, expected }) => {
-        const savedRows = await Promise.all(
-          rows.map(r => config.api.row.save(table._id!, r))
-        )
-        const { rows: foundRows } = await config.api.row.search(table._id!, {
-          tableId: table._id!,
-          query,
-        })
-        expect(foundRows).toEqual(
-          expect.arrayContaining(
-            expected.map(r =>
-              expect.objectContaining(savedRows.find(sr => sr.name === r.name)!)
-            )
-          )
-        )
-      }
-    )
+      it("should return nothing if onEmptyFilter is RETURN_NONE", () =>
+        expectQuery({
+          onEmptyFilter: EmptyFilterOption.RETURN_NONE,
+        }).toFindNothing())
     })
 
-  describe("number", () => {
-    beforeEach(async () => {
-      table = await config.api.table.save(
-        tableForDatasource(datasource, {
-          schema: {
-            age: {
-              name: "age",
-              type: FieldType.NUMBER,
-            },
-          },
-        })
-      )
+    describe("equal", () => {
+      it("successfully finds a row", () =>
+        expectQuery({ equal: { name: "foo" } }).toFind([{ name: "foo" }]))
+
+      it("fails to find nonexistent row", () =>
+        expectQuery({ equal: { name: "none" } }).toFindNothing())
     })
 
-    const rows = [{ age: 1 }, { age: 10 }]
+    describe("notEqual", () => {
+      it("successfully finds a row", () =>
+        expectQuery({ notEqual: { name: "foo" } }).toFind([{ name: "bar" }]))
 
-    interface NumberSearchTest {
-      query: SearchFilters
-      expected: (typeof rows)[number][]
-    }
-
-    const numberSearchTests: NumberSearchTest[] = [
-      { query: {}, expected: rows },
-      {
-        query: { onEmptyFilter: EmptyFilterOption.RETURN_ALL },
-        expected: rows,
-      },
-      {
-        query: { onEmptyFilter: EmptyFilterOption.RETURN_NONE },
-        expected: [],
-      },
-      { query: { equal: { age: 1 } }, expected: [rows[0]] },
-      { query: { equal: { age: 2 } }, expected: [] },
-      { query: { notEqual: { age: 1 } }, expected: [rows[1]] },
-      { query: { oneOf: { age: [1] } }, expected: [rows[0]] },
-      { query: { range: { age: { low: 1, high: 5 } } }, expected: [rows[0]] },
-      { query: { range: { age: { low: 0, high: 1 } } }, expected: [rows[0]] },
-      { query: { range: { age: { low: 3, high: 4 } } }, expected: [] },
-      { query: { range: { age: { low: 0, high: 11 } } }, expected: rows },
-    ]
-
-    it.each(numberSearchTests)(
-      `should be able to run query: $query`,
-      async ({ query, expected }) => {
-        const savedRows = await Promise.all(
-          rows.map(r => config.api.row.save(table._id!, r))
-        )
-        const { rows: foundRows } = await config.api.row.search(table._id!, {
-          tableId: table._id!,
-          query,
+      it("fails to find nonexistent row", () =>
+        expectQuery({ notEqual: { name: "bar" } }).toFind([{ name: "foo" }]))
+    })
+
+    describe("oneOf", () => {
+      it("successfully finds a row", () =>
+        expectQuery({ oneOf: { name: ["foo"] } }).toFind([{ name: "foo" }]))
+
+      it("fails to find nonexistent row", () =>
+        expectQuery({ oneOf: { name: ["none"] } }).toFindNothing())
+    })
+
+    describe("fuzzy", () => {
+      it("successfully finds a row", () =>
+        expectQuery({ fuzzy: { name: "oo" } }).toFind([{ name: "foo" }]))
+
+      it("fails to find nonexistent row", () =>
+        expectQuery({ fuzzy: { name: "none" } }).toFindNothing())
+    })
+  })
+
+  describe("numbers", () => {
+    beforeAll(async () => {
+      await createTable({
+        age: { name: "age", type: FieldType.NUMBER },
+      })
+      await createRows([{ age: 1 }, { age: 10 }])
+    })
+
+    describe("equal", () => {
+      it("successfully finds a row", () =>
+        expectQuery({ equal: { age: 1 } }).toFind([{ age: 1 }]))
+
+      it("fails to find nonexistent row", () =>
+        expectQuery({ equal: { age: 2 } }).toFindNothing())
+    })
+
+    describe("notEqual", () => {
+      it("successfully finds a row", () =>
+        expectQuery({ notEqual: { age: 1 } }).toFind([{ age: 10 }]))
+
+      it("fails to find nonexistent row", () =>
+        expectQuery({ notEqual: { age: 10 } }).toFind([{ age: 1 }]))
+    })
+
+    describe("oneOf", () => {
+      it("successfully finds a row", () =>
+        expectQuery({ oneOf: { age: [1] } }).toFind([{ age: 1 }]))
+
+      it("fails to find nonexistent row", () =>
+        expectQuery({ oneOf: { age: [2] } }).toFindNothing())
+    })
+
+    describe("range", () => {
+      it("successfully finds a row", () =>
+        expectQuery({
+          range: { age: { low: 1, high: 5 } },
+        }).toFind([{ age: 1 }]))
+
+      it("successfully finds multiple rows", () =>
+        expectQuery({
+          range: { age: { low: 1, high: 10 } },
+        }).toFind([{ age: 1 }, { age: 10 }]))
+
+      it("successfully finds a row with a high bound", () =>
+        expectQuery({
+          range: { age: { low: 5, high: 10 } },
+        }).toFind([{ age: 10 }]))
     })
-        expect(foundRows).toEqual(
-          expect.arrayContaining(
-            expected.map(r =>
-              expect.objectContaining(savedRows.find(sr => sr.age === r.age)!)
-            )
-          )
-        )
-      }
-    )
   })
 
   describe("dates", () => {
-    beforeEach(async () => {
-      table = await config.api.table.save(
-        tableForDatasource(datasource, {
-          schema: {
-            dob: {
-              name: "dob",
-              type: FieldType.DATETIME,
-            },
-          },
-        })
-      )
+    const JAN_1ST = "2020-01-01T00:00:00.000Z"
+    const JAN_2ND = "2020-01-02T00:00:00.000Z"
+    const JAN_5TH = "2020-01-05T00:00:00.000Z"
+    const JAN_10TH = "2020-01-10T00:00:00.000Z"
+
+    beforeAll(async () => {
+      await createTable({
+        dob: { name: "dob", type: FieldType.DATETIME },
       })
 
-    const rows = [
-      { dob: new Date("2020-01-01").toISOString() },
-      { dob: new Date("2020-01-10").toISOString() },
-    ]
-
-    interface DateSearchTest {
-      query: SearchFilters
-      expected: (typeof rows)[number][]
-    }
-
-    const dateSearchTests: DateSearchTest[] = [
-      { query: {}, expected: rows },
-      {
-        query: { onEmptyFilter: EmptyFilterOption.RETURN_ALL },
-        expected: rows,
-      },
-      {
-        query: { onEmptyFilter: EmptyFilterOption.RETURN_NONE },
-        expected: [],
-      },
-      {
-        query: { equal: { dob: new Date("2020-01-01").toISOString() } },
-        expected: [rows[0]],
-      },
-      {
-        query: { equal: { dob: new Date("2020-01-02").toISOString() } },
-        expected: [],
-      },
-      {
-        query: { notEqual: { dob: new Date("2020-01-01").toISOString() } },
-        expected: [rows[1]],
-      },
-      {
-        query: { oneOf: { dob: [new Date("2020-01-01").toISOString()] } },
-        expected: [rows[0]],
-      },
-      {
-        query: {
-          range: {
-            dob: {
-              low: new Date("2020-01-01").toISOString(),
-              high: new Date("2020-01-05").toISOString(),
-            },
-          },
-        },
-        expected: [rows[0]],
-      },
-      {
-        query: {
-          range: {
-            dob: {
-              low: new Date("2020-01-01").toISOString(),
-              high: new Date("2020-01-10").toISOString(),
-            },
-          },
-        },
-        expected: rows,
-      },
-      {
-        query: {
-          range: {
-            dob: {
-              low: new Date("2020-01-05").toISOString(),
-              high: new Date("2020-01-10").toISOString(),
-            },
-          },
-        },
-        expected: [rows[1]],
-      },
-    ]
-
-    it.each(dateSearchTests)(
-      `should be able to run query: $query`,
-      async ({ query, expected }) => {
-        const savedRows = await Promise.all(
-          rows.map(r => config.api.row.save(table._id!, r))
-        )
-        const { rows: foundRows } = await config.api.row.search(table._id!, {
-          tableId: table._id!,
-          query,
+      await createRows([{ dob: JAN_1ST }, { dob: JAN_10TH }])
+    })
+
+    describe("equal", () => {
+      it("successfully finds a row", () =>
+        expectQuery({ equal: { dob: JAN_1ST } }).toFind([{ dob: JAN_1ST }]))
+
+      it("fails to find nonexistent row", () =>
+        expectQuery({ equal: { dob: JAN_2ND } }).toFindNothing())
+    })
+
+    describe("notEqual", () => {
+      it("successfully finds a row", () =>
+        expectQuery({ notEqual: { dob: JAN_1ST } }).toFind([{ dob: JAN_10TH }]))
+
+      it("fails to find nonexistent row", () =>
+        expectQuery({ notEqual: { dob: JAN_10TH } }).toFind([{ dob: JAN_1ST }]))
+    })
+
+    describe("oneOf", () => {
+      it("successfully finds a row", () =>
+        expectQuery({ oneOf: { dob: [JAN_1ST] } }).toFind([{ dob: JAN_1ST }]))
+
+      it("fails to find nonexistent row", () =>
+        expectQuery({ oneOf: { dob: [JAN_2ND] } }).toFindNothing())
+    })
+
+    describe("range", () => {
+      it("successfully finds a row", () =>
+        expectQuery({
+          range: { dob: { low: JAN_1ST, high: JAN_5TH } },
+        }).toFind([{ dob: JAN_1ST }]))
+
+      it("successfully finds multiple rows", () =>
+        expectQuery({
+          range: { dob: { low: JAN_1ST, high: JAN_10TH } },
+        }).toFind([{ dob: JAN_1ST }, { dob: JAN_10TH }]))
+
+      it("successfully finds a row with a high bound", () =>
+        expectQuery({
+          range: { dob: { low: JAN_5TH, high: JAN_10TH } },
+        }).toFind([{ dob: JAN_10TH }]))
     })
-        expect(foundRows).toEqual(
-          expect.arrayContaining(
-            expected.map(r =>
-              expect.objectContaining(savedRows.find(sr => sr.dob === r.dob)!)
-            )
-          )
-        )
-      }
-    )
   })
 })

packages/server/src/integrations/base/sql.ts

@@ -334,6 +334,7 @@ class InternalBuilder {
     if (filters.containsAny) {
       contains(filters.containsAny, true)
     }
+
     return query
   }
 

packages/server/src/sdk/app/rows/search.ts

@@ -1,4 +1,5 @@
 import {
+  EmptyFilterOption,
   Row,
   RowSearchParams,
   SearchFilters,
@@ -11,6 +12,7 @@ import { NoEmptyFilterStrings } from "../../../constants"
 import * as sqs from "./search/sqs"
 import env from "../../../environment"
 import { ExportRowsParams, ExportRowsResult } from "./search/types"
+import { dataFilters } from "@budibase/shared-core"
 
 export { isValidFilter } from "../../../integrations/utils"
 
@@ -60,6 +62,16 @@ export async function search(
   options: RowSearchParams
 ): Promise<SearchResponse<Row>> {
   const isExternalTable = isExternalTableID(options.tableId)
+  options.query = removeEmptyFilters(options.query || {})
+  if (
+    !dataFilters.hasFilters(options.query) &&
+    options.query.onEmptyFilter === EmptyFilterOption.RETURN_NONE
+  ) {
+    return {
+      rows: [],
+    }
+  }
+
   if (isExternalTable) {
     return external.search(options)
   } else if (env.SQS_SEARCH_ENABLE) {