Getting audit log searching with sqs working - a bit more work than expected but fixed quite a few SQS bugs along the way.

packages/backend-core/src/constants/db.ts

@@ -65,6 +65,7 @@ export const StaticDatabases = {
 export const APP_PREFIX = prefixed(DocumentType.APP)
 export const APP_DEV = prefixed(DocumentType.APP_DEV)
 export const APP_DEV_PREFIX = APP_DEV
+export const SQS_DATASOURCE_INTERNAL = "internal"
 export const BUDIBASE_DATASOURCE_TYPE = "budibase"
 export const SQLITE_DESIGN_DOC_ID = "_design/sqlite"
 export const DEFAULT_JOBS_TABLE_ID = "ta_bb_jobs"

packages/backend-core/src/sql/sql.ts

@@ -50,7 +50,7 @@ function likeKey(client: string, key: string): string {
       end = "]"
       break
     case SqlClient.SQL_LITE:
-      start = end = "'"
+      start = end = ""
       break
     default:
       throw new Error("Unknown client generating like key")
@@ -198,17 +198,20 @@ class InternalBuilder {
         const updatedKey = dbCore.removeKeyNumbering(key)
         const isRelationshipField = updatedKey.includes(".")
         if (!opts.relationship && !isRelationshipField) {
-          fn(`${getTableAlias(tableName)}.${updatedKey}`, value)
+          const alias = getTableAlias(tableName)
+          fn(alias ? `${alias}.${updatedKey}` : updatedKey, value)
         }
         if (opts.relationship && isRelationshipField) {
           const [filterTableName, property] = updatedKey.split(".")
-          fn(`${getTableAlias(filterTableName)}.${property}`, value)
+          const alias = getTableAlias(filterTableName)
+          fn(alias ? `${alias}.${property}` : property, value)
         }
       }
     }
 
     const like = (key: string, value: any) => {
-      const fnc = allOr ? "orWhere" : "where"
+      const fuzzyOr = filters?.fuzzyOr
+      const fnc = fuzzyOr || allOr ? "orWhere" : "where"
       // postgres supports ilike, nothing else does
       if (this.client === SqlClient.POSTGRES) {
         query = query[fnc](key, "ilike", `%${value}%`)

packages/pro

@@ -1 +1 @@
-Subproject commit d3c3077011a8e20ed3c48dcd6301caca4120b6ac
+Subproject commit a6492c51ea691c3ff969a7b92d4c66f919c06417

packages/server/src/db/utils.ts

@@ -40,7 +40,6 @@ export const USER_METDATA_PREFIX = `${DocumentType.ROW}${SEPARATOR}${dbCore.Inte
 export const LINK_USER_METADATA_PREFIX = `${DocumentType.LINK}${SEPARATOR}${dbCore.InternalTable.USER_METADATA}${SEPARATOR}`
 export const TABLE_ROW_PREFIX = `${DocumentType.ROW}${SEPARATOR}${DocumentType.TABLE}`
 export const AUTOMATION_LOG_PREFIX = `${DocumentType.AUTOMATION_LOG}${SEPARATOR}`
-export const SQS_DATASOURCE_INTERNAL = "internal"
 export const ViewName = dbCore.ViewName
 export const InternalTables = dbCore.InternalTable
 export const UNICODE_MAX = dbCore.UNICODE_MAX

packages/server/src/sdk/app/rows/search/sqs.ts

@@ -18,11 +18,13 @@ import {
   sqlOutputProcessing,
 } from "../../../../api/controllers/row/utils"
 import sdk from "../../../index"
-import { context, sql, SQLITE_DESIGN_DOC_ID } from "@budibase/backend-core"
 import {
-  CONSTANT_INTERNAL_ROW_COLS,
+  context,
+  sql,
+  SQLITE_DESIGN_DOC_ID,
   SQS_DATASOURCE_INTERNAL,
-} from "../../../../db/utils"
+} from "@budibase/backend-core"
+import { CONSTANT_INTERNAL_ROW_COLS } from "../../../../db/utils"
 import AliasTables from "../sqlAlias"
 import { outputProcessing } from "../../../../utilities/rowProcessor"
 
@@ -146,10 +148,16 @@ export async function search(
       },
     }
   }
+
+  if (typeof params.bookmark !== "number") {
+    throw new Error("Unable to paginate with string based bookmarks")
+  }
+  const bookmark: number = params.bookmark || 1
+  const limit = params.limit
   if (paginate && params.limit) {
     request.paginate = {
       limit: params.limit,
-      page: params.bookmark,
+      page: bookmark,
     }
   }
   try {
@@ -185,12 +193,22 @@ export async function search(
       }
     )
 
-    return {
+    const finalRows = await outputProcessing<Row[]>(table, processed, {
-      // final row processing for response
+      preserveLinks: true,
-      rows: await outputProcessing<Row[]>(table, processed, {
+      squash: true,
-        preserveLinks: true,
+    })
-        squash: true,
+    if (paginate && limit) {
-      }),
+      return {
+        // final row processing for response
+        rows: finalRows,
+        bookmark: bookmark + 1,
+        // TODO: need to work out if next page available
+        hasNextPage: false,
+      }
+    } else {
+      return {
+        rows: finalRows,
+      }
     }
   } catch (err: any) {
     const msg = typeof err === "string" ? err : err.message

packages/server/src/sdk/app/rows/sqlAlias.ts

@@ -7,11 +7,11 @@ import {
   SearchFilters,
   SqlClient,
 } from "@budibase/types"
+import { SQS_DATASOURCE_INTERNAL } from "@budibase/backend-core"
 import { getSQLClient } from "./utils"
 import { cloneDeep } from "lodash"
 import datasources from "../datasources"
 import { makeExternalQuery } from "../../../integrations/base/query"
-import { SQS_DATASOURCE_INTERNAL } from "../../../db/utils"
 
 const WRITE_OPERATIONS: Operation[] = [
   Operation.CREATE,

packages/types/src/documents/global/auditLogs.ts

@@ -2,6 +2,7 @@ import { Document } from "../document"
 import { Event } from "../../sdk"
 
 export const AuditLogSystemUser = "SYSTEM"
+export const AUDIT_LOG_TYPE = "auditLog"
 
 export type FallbackInfo = {
   appName?: string
@@ -15,5 +16,6 @@ export interface AuditLogDoc extends Document {
   timestamp: string
   metadata: any
   name: string
+  type?: "auditLog"
   fallback?: FallbackInfo
 }

packages/types/src/sdk/search.ts

@@ -19,6 +19,9 @@ export enum SearchFilterOperator {
 
 export interface SearchFilters {
   allOr?: boolean
+  // TODO: this is just around for now - we need a better way to do or/and
+  // allows just fuzzy to be or - all the fuzzy/like parameters
+  fuzzyOr?: boolean
   onEmptyFilter?: EmptyFilterOption
   [SearchFilterOperator.STRING]?: {
     [key: string]: string