Refactoring search SDK to make it obvious the different search methods, exports was using lucene always when doing internal export rows which shouldn't be the case, should go through the complete search SDK.

packages/server/src/sdk/app/rows/search.ts

@@ -8,7 +8,6 @@ import {
 import { isExternalTableID } from "../../../integrations/utils"
 import * as internal from "./search/internal"
 import * as external from "./search/external"
-import * as sqs from "./search/sqs"
 import { ExportRowsParams, ExportRowsResult } from "./search/types"
 import { dataFilters } from "@budibase/shared-core"
 import sdk from "../../index"
@@ -55,9 +54,9 @@ export async function search(
   if (isExternalTable) {
     return external.search(options, table)
   } else if (dbCore.isSqsEnabledForTenant()) {
-    return sqs.search(options, table)
+    return internal.sqs.search(options, table)
   } else {
-    return internal.search(options, table)
+    return internal.lucene.search(options, table)
   }
 }
 

packages/server/src/sdk/app/rows/search/internal/index.ts

@@ -0,0 +1,3 @@
+export * as sqs from "./sqs"
+export * as lucene from "./lucene"
+export * from "./internal"

packages/server/src/sdk/app/rows/search/internal/internal.ts

@@ -1,90 +1,30 @@
 import { context, HTTPError } from "@budibase/backend-core"
-import { PROTECTED_INTERNAL_COLUMNS } from "@budibase/shared-core"
-import env from "../../../../environment"
-import { fullSearch, paginatedSearch } from "./utils"
-import { getRowParams, InternalTables } from "../../../../db/utils"
+import env from "../../../../../environment"
+import { getRowParams, InternalTables } from "../../../../../db/utils"
 import {
   Database,
   DocumentType,
   Row,
-  RowSearchParams,
-  SearchResponse,
-  SortType,
   Table,
   TableSchema,
-  User,
 } from "@budibase/types"
-import { getGlobalUsersFromMetadata } from "../../../../utilities/global"
-import { outputProcessing } from "../../../../utilities/rowProcessor"
+import { outputProcessing } from "../../../../../utilities/rowProcessor"
 import {
   csv,
   Format,
   json,
   jsonWithSchema,
-} from "../../../../api/controllers/view/exporters"
-import * as inMemoryViews from "../../../../db/inMemoryView"
+} from "../../../../../api/controllers/view/exporters"
+import * as inMemoryViews from "../../../../../db/inMemoryView"
 import {
   getFromDesignDoc,
   getFromMemoryDoc,
   migrateToDesignView,
   migrateToInMemoryView,
-} from "../../../../api/controllers/view/utils"
-import sdk from "../../../../sdk"
-import { ExportRowsParams, ExportRowsResult } from "./types"
-import pick from "lodash/pick"
-import { breakRowIdField } from "../../../../integrations/utils"
-
-export async function search(
-  options: RowSearchParams,
-  table: Table
-): Promise<SearchResponse<Row>> {
-  const { tableId } = options
-
-  const { paginate, query } = options
-
-  const params: RowSearchParams = {
-    tableId: options.tableId,
-    sort: options.sort,
-    sortOrder: options.sortOrder,
-    sortType: options.sortType,
-    limit: options.limit,
-    bookmark: options.bookmark,
-    version: options.version,
-    disableEscaping: options.disableEscaping,
-    query: {},
-  }
-
-  if (params.sort && !params.sortType) {
-    const schema = table.schema
-    const sortField = schema[params.sort]
-    params.sortType =
-      sortField.type === "number" ? SortType.NUMBER : SortType.STRING
-  }
-
-  let response
-  if (paginate) {
-    response = await paginatedSearch(query, params)
-  } else {
-    response = await fullSearch(query, params)
-  }
-
-  // Enrich search results with relationships
-  if (response.rows && response.rows.length) {
-    // enrich with global users if from users table
-    if (tableId === InternalTables.USER_METADATA) {
-      response.rows = await getGlobalUsersFromMetadata(response.rows as User[])
-    }
-
-    if (options.fields) {
-      const fields = [...options.fields, ...PROTECTED_INTERNAL_COLUMNS]
-      response.rows = response.rows.map((r: any) => pick(r, fields))
-    }
-
-    response.rows = await outputProcessing(table, response.rows)
-  }
-
-  return response
-}
+} from "../../../../../api/controllers/view/utils"
+import sdk from "../../../../../sdk"
+import { ExportRowsParams, ExportRowsResult } from "../types"
+import { breakRowIdField } from "../../../../../integrations/utils"
 
 export async function exportRows(
   options: ExportRowsParams
@@ -123,15 +63,12 @@ export async function exportRows(
 
     result = await outputProcessing<Row[]>(table, response)
   } else if (query) {
-    let searchResponse = await search(
-      {
-        tableId,
-        query,
-        sort,
-        sortOrder,
-      },
-      table
-    )
+    let searchResponse = await sdk.rows.search({
+      tableId,
+      query,
+      sort,
+      sortOrder,
+    })
     result = searchResponse.rows
   }
 

packages/server/src/sdk/app/rows/search/internal/lucene.ts

@@ -0,0 +1,66 @@
+import { PROTECTED_INTERNAL_COLUMNS } from "@budibase/shared-core"
+import { fullSearch, paginatedSearch } from "../utils"
+import { InternalTables } from "../../../../../db/utils"
+import {
+  Row,
+  RowSearchParams,
+  SearchResponse,
+  SortType,
+  Table,
+  User,
+} from "@budibase/types"
+import { getGlobalUsersFromMetadata } from "../../../../../utilities/global"
+import { outputProcessing } from "../../../../../utilities/rowProcessor"
+import pick from "lodash/pick"
+
+export async function search(
+  options: RowSearchParams,
+  table: Table
+): Promise<SearchResponse<Row>> {
+  const { tableId } = options
+
+  const { paginate, query } = options
+
+  const params: RowSearchParams = {
+    tableId: options.tableId,
+    sort: options.sort,
+    sortOrder: options.sortOrder,
+    sortType: options.sortType,
+    limit: options.limit,
+    bookmark: options.bookmark,
+    version: options.version,
+    disableEscaping: options.disableEscaping,
+    query: {},
+  }
+
+  if (params.sort && !params.sortType) {
+    const schema = table.schema
+    const sortField = schema[params.sort]
+    params.sortType =
+      sortField.type === "number" ? SortType.NUMBER : SortType.STRING
+  }
+
+  let response
+  if (paginate) {
+    response = await paginatedSearch(query, params)
+  } else {
+    response = await fullSearch(query, params)
+  }
+
+  // Enrich search results with relationships
+  if (response.rows && response.rows.length) {
+    // enrich with global users if from users table
+    if (tableId === InternalTables.USER_METADATA) {
+      response.rows = await getGlobalUsersFromMetadata(response.rows as User[])
+    }
+
+    if (options.fields) {
+      const fields = [...options.fields, ...PROTECTED_INTERNAL_COLUMNS]
+      response.rows = response.rows.map((r: any) => pick(r, fields))
+    }
+
+    response.rows = await outputProcessing(table, response.rows)
+  }
+
+  return response
+}

packages/server/src/sdk/app/rows/search/internal/sqs.ts

@@ -18,31 +18,31 @@ import {
 import {
   buildInternalRelationships,
   sqlOutputProcessing,
-} from "../../../../api/controllers/row/utils"
+} from "../../../../../api/controllers/row/utils"
 import {
   decodeNonAscii,
   mapToUserColumn,
   USER_COLUMN_PREFIX,
-} from "../../tables/internal/sqs"
-import sdk from "../../../index"
+} from "../../../tables/internal/sqs"
+import sdk from "../../../../index"
 import {
   context,
   sql,
   SQLITE_DESIGN_DOC_ID,
   SQS_DATASOURCE_INTERNAL,
 } from "@budibase/backend-core"
-import { generateJunctionTableID } from "../../../../db/utils"
-import AliasTables from "../sqlAlias"
-import { outputProcessing } from "../../../../utilities/rowProcessor"
+import { generateJunctionTableID } from "../../../../../db/utils"
+import AliasTables from "../../sqlAlias"
+import { outputProcessing } from "../../../../../utilities/rowProcessor"
 import pick from "lodash/pick"
-import { processRowCountResponse } from "../utils"
+import { processRowCountResponse } from "../../utils"
 import {
   updateFilterKeys,
   getRelationshipColumns,
   getTableIDList,
-} from "./filters"
+} from "../filters"
 import { dataFilters, PROTECTED_INTERNAL_COLUMNS } from "@budibase/shared-core"
-import { isSearchingByRowID } from "./utils"
+import { isSearchingByRowID } from "../utils"
 import tracer from "dd-trace"
 
 const builder = new sql.Sql(SqlClient.SQL_LITE)