Merge branch 'master' into fix/conditions-on-views

packages/backend-core/src/sql/sql.ts

@@ -36,6 +36,7 @@ import {
 } from "@budibase/types"
 import environment from "../environment"
 import { dataFilters, helpers } from "@budibase/shared-core"
+import { cloneDeep } from "lodash"
 
 type QueryFunction = (query: SqlQuery | SqlQuery[], operation: Operation) => any
 
@@ -268,6 +269,7 @@ class InternalBuilder {
   }
 
   private parseFilters(filters: SearchFilters): SearchFilters {
+    filters = cloneDeep(filters)
     for (const op of Object.values(BasicOperator)) {
       const filter = filters[op]
       if (!filter) {
@@ -371,10 +373,11 @@ class InternalBuilder {
             ),
             castedTypeValue.values
           )
-        } else if (!opts?.relationship && !isRelationshipField) {
+        } else if (!isRelationshipField) {
           const alias = getTableAlias(tableName)
           fn(alias ? `${alias}.${updatedKey}` : updatedKey, value)
-        } else if (opts?.relationship && isRelationshipField) {
+        }
+        if (opts?.relationship && isRelationshipField) {
           const [filterTableName, property] = updatedKey.split(".")
           const alias = getTableAlias(filterTableName)
           fn(alias ? `${alias}.${property}` : property, value)

packages/server/src/api/controllers/row/ExternalRequest.ts

@@ -45,6 +45,7 @@ import { db as dbCore } from "@budibase/backend-core"
 import sdk from "../../../sdk"
 import env from "../../../environment"
 import { makeExternalQuery } from "../../../integrations/base/query"
+import { dataFilters } from "@budibase/shared-core"
 
 export interface ManyRelationship {
   tableId?: string
@@ -195,29 +196,33 @@ export class ExternalRequest<T extends Operation> {
     if (filters) {
       // need to map over the filters and make sure the _id field isn't present
       let prefix = 1
-      for (const [operatorType, operator] of Object.entries(filters)) {
+      const checkFilters = (innerFilters: SearchFilters): SearchFilters => {
-        const isArrayOp = sdk.rows.utils.isArrayFilter(operatorType)
+        for (const [operatorType, operator] of Object.entries(innerFilters)) {
-        for (const field of Object.keys(operator || {})) {
+          const isArrayOp = sdk.rows.utils.isArrayFilter(operatorType)
-          if (dbCore.removeKeyNumbering(field) === "_id") {
+          for (const field of Object.keys(operator || {})) {
-            if (primary) {
+            if (dbCore.removeKeyNumbering(field) === "_id") {
-              const parts = breakRowIdField(operator[field])
+              if (primary) {
-              if (primary.length > 1 && isArrayOp) {
+                const parts = breakRowIdField(operator[field])
-                operator[InternalSearchFilterOperator.COMPLEX_ID_OPERATOR] = {
+                if (primary.length > 1 && isArrayOp) {
-                  id: primary,
+                  operator[InternalSearchFilterOperator.COMPLEX_ID_OPERATOR] = {
-                  values: parts[0],
+                    id: primary,
+                    values: parts[0],
+                  }
+                } else {
+                  for (let field of primary) {
+                    operator[`${prefix}:${field}`] = parts.shift()
+                  }
+                  prefix++
                 }
-              } else {
-                for (let field of primary) {
-                  operator[`${prefix}:${field}`] = parts.shift()
-                }
-                prefix++
               }
+              // make sure this field doesn't exist on any filter
+              delete operator[field]
             }
-            // make sure this field doesn't exist on any filter
-            delete operator[field]
           }
         }
+        return dataFilters.recurseLogicalOperators(innerFilters, checkFilters)
       }
+      checkFilters(filters)
     }
     // there is no id, just use the user provided filters
     if (!idCopy || !table) {

packages/server/src/api/controllers/row/views.ts

@@ -56,13 +56,12 @@ export async function searchView(
           }
         })
       })
-    } else {
+    } else
       query = {
         $and: {
           conditions: [query, body.query],
         },
       }
-    }
   }
 
   await context.ensureSnippetContext(true)

packages/server/src/integrations/tests/sql.spec.ts

@@ -194,8 +194,8 @@ describe("SQL query builder", () => {
       })
     )
     expect(query).toEqual({
-      bindings: ["john%", limit, 5000],
+      bindings: ["john%", limit, "john%", 5000],
-      sql: `select * from (select * from (select * from (select * from "test" where LOWER("test"."name") LIKE :1 order by "test"."id" asc) where rownum <= :2) "test" order by "test"."id" asc) where rownum <= :3`,
+      sql: `select * from (select * from (select * from (select * from "test" where LOWER("test"."name") LIKE :1 order by "test"."id" asc) where rownum <= :2) "test" where LOWER("test"."name") LIKE :3 order by "test"."id" asc) where rownum <= :4`,
     })
 
     query = new Sql(SqlClient.ORACLE, limit)._query(
@@ -208,9 +208,10 @@ describe("SQL query builder", () => {
         },
       })
     )
+    const filterSet = [`%20%`, `%25%`, `%"john"%`, `%"mary"%`]
     expect(query).toEqual({
-      bindings: ["%20%", "%25%", `%"john"%`, `%"mary"%`, limit, 5000],
+      bindings: [...filterSet, limit, ...filterSet, 5000],
-      sql: `select * from (select * from (select * from (select * from "test" where COALESCE(LOWER("test"."age"), '') LIKE :1 AND COALESCE(LOWER("test"."age"), '') LIKE :2 and COALESCE(LOWER("test"."name"), '') LIKE :3 AND COALESCE(LOWER("test"."name"), '') LIKE :4 order by "test"."id" asc) where rownum <= :5) "test" order by "test"."id" asc) where rownum <= :6`,
+      sql: `select * from (select * from (select * from (select * from "test" where COALESCE(LOWER("test"."age"), '') LIKE :1 AND COALESCE(LOWER("test"."age"), '') LIKE :2 and COALESCE(LOWER("test"."name"), '') LIKE :3 AND COALESCE(LOWER("test"."name"), '') LIKE :4 order by "test"."id" asc) where rownum <= :5) "test" where COALESCE(LOWER("test"."age"), '') LIKE :6 AND COALESCE(LOWER("test"."age"), '') LIKE :7 and COALESCE(LOWER("test"."name"), '') LIKE :8 AND COALESCE(LOWER("test"."name"), '') LIKE :9 order by "test"."id" asc) where rownum <= :10`,
     })
 
     query = new Sql(SqlClient.ORACLE, limit)._query(
@@ -223,8 +224,8 @@ describe("SQL query builder", () => {
       })
     )
     expect(query).toEqual({
-      bindings: [`%jo%`, limit, 5000],
+      bindings: [`%jo%`, limit, `%jo%`, 5000],
-      sql: `select * from (select * from (select * from (select * from "test" where LOWER("test"."name") LIKE :1 order by "test"."id" asc) where rownum <= :2) "test" order by "test"."id" asc) where rownum <= :3`,
+      sql: `select * from (select * from (select * from (select * from "test" where LOWER("test"."name") LIKE :1 order by "test"."id" asc) where rownum <= :2) "test" where LOWER("test"."name") LIKE :3 order by "test"."id" asc) where rownum <= :4`,
     })
   })
 
@@ -241,8 +242,8 @@ describe("SQL query builder", () => {
     )
 
     expect(query).toEqual({
-      bindings: ["John", limit, 5000],
+      bindings: ["John", limit, "John", 5000],
-      sql: `select * from (select * from (select * from (select * from "test" where (to_char("test"."name") IS NOT NULL AND to_char("test"."name") = :1) order by "test"."id" asc) where rownum <= :2) "test" order by "test"."id" asc) where rownum <= :3`,
+      sql: `select * from (select * from (select * from (select * from "test" where (to_char("test"."name") IS NOT NULL AND to_char("test"."name") = :1) order by "test"."id" asc) where rownum <= :2) "test" where (to_char("test"."name") IS NOT NULL AND to_char("test"."name") = :3) order by "test"."id" asc) where rownum <= :4`,
     })
   })
 
@@ -259,8 +260,8 @@ describe("SQL query builder", () => {
     )
 
     expect(query).toEqual({
-      bindings: ["John", limit, 5000],
+      bindings: ["John", limit, "John", 5000],
-      sql: `select * from (select * from (select * from (select * from "test" where (to_char("test"."name") IS NOT NULL AND to_char("test"."name") != :1) OR to_char("test"."name") IS NULL order by "test"."id" asc) where rownum <= :2) "test" order by "test"."id" asc) where rownum <= :3`,
+      sql: `select * from (select * from (select * from (select * from "test" where (to_char("test"."name") IS NOT NULL AND to_char("test"."name") != :1) OR to_char("test"."name") IS NULL order by "test"."id" asc) where rownum <= :2) "test" where (to_char("test"."name") IS NOT NULL AND to_char("test"."name") != :3) OR to_char("test"."name") IS NULL order by "test"."id" asc) where rownum <= :4`,
     })
   })
 })

packages/server/src/integrations/tests/sqlAlias.spec.ts

@@ -97,13 +97,14 @@ describe("Captures of real examples", () => {
       const filters = queryJson.filters?.oneOf?.taskid as number[]
       let query = new Sql(SqlClient.POSTGRES, limit)._query(queryJson)
       expect(query).toEqual({
-        bindings: [...filters, limit, limit],
+        bindings: [...filters, limit, ...filters, limit],
-        sql: multiline(`select "a"."executorid" as "a.executorid", "a"."taskname" as "a.taskname", 
+        sql: multiline(
-              "a"."taskid" as "a.taskid", "a"."completed" as "a.completed", "a"."qaid" as "a.qaid", 
+          `select "a"."executorid" as "a.executorid", "a"."taskname" as "a.taskname", "a"."taskid" as "a.taskid",
-              "b"."productname" as "b.productname", "b"."productid" as "b.productid" 
+          "a"."completed" as "a.completed", "a"."qaid" as "a.qaid", "b"."productname" as "b.productname", "b"."productid" as "b.productid"
-              from (select * from "tasks" as "a" where "a"."taskid" in ($1, $2) order by "a"."taskid" asc limit $3) as "a" 
+          from (select * from "tasks" as "a" where "a"."taskid" in ($1, $2) order by "a"."taskid" asc limit $3) as "a"
-              left join "products_tasks" as "c" on "a"."taskid" = "c"."taskid" 
+          left join "products_tasks" as "c" on "a"."taskid" = "c"."taskid" left join "products" as "b" on "b"."productid" = "c"."productid"
-              left join "products" as "b" on "b"."productid" = "c"."productid" order by "a"."taskid" asc limit $4`),
+          where "a"."taskid" in ($4, $5) order by "a"."taskid" asc limit $6`
+        ),
       })
     })
 
@@ -123,6 +124,7 @@ describe("Captures of real examples", () => {
           rangeValue.low,
           rangeValue.high,
           equalValue,
+          true,
           limit,
         ],
         sql: expect.stringContaining(
@@ -186,8 +188,9 @@ describe("Captures of real examples", () => {
       }, queryJson)
       expect(returningQuery).toEqual({
         sql: multiline(`select top (@p0) * from (select top (@p1) * from [people] where CASE WHEN [people].[name] = @p2 
-          THEN 1 ELSE 0 END = 1 and CASE WHEN [people].[age] = @p3 THEN 1 ELSE 0 END = 1 order by [people].[name] asc) as [people]`),
+             THEN 1 ELSE 0 END = 1 and CASE WHEN [people].[age] = @p3 THEN 1 ELSE 0 END = 1 order by [people].[name] asc) as [people] 
-        bindings: [5000, 1, "Test", 22],
+             where CASE WHEN [people].[name] = @p4 THEN 1 ELSE 0 END = 1 and CASE WHEN [people].[age] = @p5 THEN 1 ELSE 0 END = 1`),
+        bindings: [5000, 1, "Test", 22, "Test", 22],
       })
     })
   })

packages/server/src/sdk/app/rows/search/filters.ts

@@ -5,6 +5,7 @@ import {
   Table,
 } from "@budibase/types"
 import { isPlainObject } from "lodash"
+import { dataFilters } from "@budibase/shared-core"
 
 export function getRelationshipColumns(table: Table): {
   name: string
@@ -58,5 +59,7 @@ export function updateFilterKeys(
       }
     }
   }
-  return filters
+  return dataFilters.recurseLogicalOperators(filters, (f: SearchFilters) => {
+    return updateFilterKeys(f, updates)
+  })
 }

packages/server/src/sdk/app/rows/sqlAlias.ts

@@ -12,6 +12,7 @@ import { getSQLClient } from "./utils"
 import { cloneDeep } from "lodash"
 import datasources from "../datasources"
 import { BudibaseInternalDB } from "../../../db/utils"
+import { dataFilters } from "@budibase/shared-core"
 
 type PerformQueryFunction = (
   datasource: Datasource,
@@ -199,16 +200,20 @@ export default class AliasTables {
         )
       }
       if (json.filters) {
-        for (let [filterKey, filter] of Object.entries(json.filters)) {
+        const aliasFilters = (filters: SearchFilters): SearchFilters => {
-          if (typeof filter !== "object") {
+          for (let [filterKey, filter] of Object.entries(filters)) {
-            continue
+            if (typeof filter !== "object") {
+              continue
+            }
+            const aliasedFilters: typeof filter = {}
+            for (let key of Object.keys(filter)) {
+              aliasedFilters[this.aliasField(key)] = filter[key]
+            }
+            filters[filterKey as keyof SearchFilters] = aliasedFilters
           }
-          const aliasedFilters: typeof filter = {}
+          return dataFilters.recurseLogicalOperators(filters, aliasFilters)
-          for (let key of Object.keys(filter)) {
-            aliasedFilters[this.aliasField(key)] = filter[key]
-          }
-          json.filters[filterKey as keyof SearchFilters] = aliasedFilters
         }
+        json.filters = aliasFilters(json.filters)
       }
       if (json.meta?.table) {
         this.getAlias(json.meta.table.name)

packages/shared-core/src/filters.ts

@@ -113,6 +113,20 @@ export const NoEmptyFilterStrings = [
   OperatorOptions.In.value,
 ] as (keyof SearchQueryFields)[]
 
+export function recurseLogicalOperators(
+  filters: SearchFilters,
+  fn: (f: SearchFilters) => SearchFilters
+) {
+  for (const logical of Object.values(LogicalOperator)) {
+    if (filters[logical]) {
+      filters[logical]!.conditions = filters[logical]!.conditions.map(
+        condition => fn(condition)
+      )
+    }
+  }
+  return filters
+}
+
 /**
  * Removes any fields that contain empty strings that would cause inconsistent
  * behaviour with how backend tables are filtered (no value means no filter).
@@ -145,6 +159,7 @@ export const cleanupQuery = (query: SearchFilters) => {
       }
     }
   }
+  query = recurseLogicalOperators(query, cleanupQuery)
   return query
 }
 
@@ -410,6 +425,7 @@ export function fixupFilterArrays(filters: SearchFilters) {
       }
     }
   }
+  recurseLogicalOperators(filters, fixupFilterArrays)
   return filters
 }