Merge pull request #11239 from Budibase/BUDI-7189/extract_search_functionality

Moving files and functions

.tool-versions

@@ -1,2 +1,2 @@
-nodejs 14.20.1
+nodejs 14.21.3
 python 3.10.0

packages/server/src/api/controllers/datasource.ts

@@ -9,7 +9,6 @@ import {
 import { destroy as tableDestroy } from "./table/internal"
 import { BuildSchemaErrors, InvalidColumns } from "../../constants"
 import { getIntegration } from "../../integrations"
-import { getDatasourceAndQuery } from "./row/utils"
 import { invalidateDynamicVariables } from "../../threads/utils"
 import { db as dbCore, context, events } from "@budibase/backend-core"
 import {
@@ -442,7 +441,7 @@ export async function find(ctx: UserCtx) {
 export async function query(ctx: UserCtx) {
   const queryJson = ctx.request.body
   try {
-    ctx.body = await getDatasourceAndQuery(queryJson)
+    ctx.body = await sdk.rows.utils.getDatasourceAndQuery(queryJson)
   } catch (err: any) {
     ctx.throw(400, err)
   }

packages/server/src/api/controllers/row/ExternalRequest.ts

@@ -23,14 +23,13 @@ import {
   isRowId,
   isSQL,
 } from "../../../integrations/utils"
-import { getDatasourceAndQuery } from "./utils"
+import { getDatasourceAndQuery } from "../../../sdk/app/rows/utils"
 import { FieldTypes } from "../../../constants"
 import { processObjectSync } from "@budibase/string-templates"
 import { cloneDeep } from "lodash/fp"
 import { processDates, processFormulas } from "../../../utilities/rowProcessor"
 import { db as dbCore } from "@budibase/backend-core"
 import sdk from "../../../sdk"
-import { isEditableColumn } from "../../../sdk/app/tables/validation"
 
 export interface ManyRelationship {
   tableId?: string

packages/server/src/api/controllers/row/external.ts

@@ -1,30 +1,20 @@
-import {
+import { FieldTypes, NoEmptyFilterStrings } from "../../../constants"
-  FieldTypes,
-  NoEmptyFilterStrings,
-  SortDirection,
-} from "../../../constants"
 import {
   breakExternalTableId,
   breakRowIdField,
 } from "../../../integrations/utils"
 import { ExternalRequest, RunConfig } from "./ExternalRequest"
-import * as exporters from "../view/exporters"
-import { apiFileReturn } from "../../../utilities/fileSystem"
 import {
   Datasource,
   IncludeRelationship,
   Operation,
-  PaginationJson,
   Row,
-  SortJson,
   Table,
   UserCtx,
 } from "@budibase/types"
 import sdk from "../../../sdk"
 import * as utils from "./utils"
 
-const { cleanExportRows } = require("./utils")
-
 async function getRow(
   tableId: string,
   rowId: string,
@@ -114,21 +104,6 @@ export async function save(ctx: UserCtx) {
   }
 }
 
-export async function fetchView(ctx: UserCtx) {
-  // there are no views in external datasources, shouldn't ever be called
-  // for now just fetch
-  const split = ctx.params.viewName.split("all_")
-  ctx.params.tableId = split[1] ? split[1] : split[0]
-  return fetch(ctx)
-}
-
-export async function fetch(ctx: UserCtx) {
-  const tableId = ctx.params.tableId
-  return handleRequest(Operation.READ, tableId, {
-    includeSqlRelationships: IncludeRelationship.INCLUDE,
-  })
-}
-
 export async function find(ctx: UserCtx) {
   const id = ctx.params.rowId
   const tableId = ctx.params.tableId
@@ -161,129 +136,6 @@ export async function bulkDestroy(ctx: UserCtx) {
   return { response: { ok: true }, rows: responses.map(resp => resp.row) }
 }
 
-export async function search(ctx: UserCtx) {
-  const tableId = ctx.params.tableId
-  const { paginate, query, ...params } = ctx.request.body
-  let { bookmark, limit } = params
-  if (!bookmark && paginate) {
-    bookmark = 1
-  }
-  let paginateObj = {}
-
-  if (paginate) {
-    paginateObj = {
-      // add one so we can track if there is another page
-      limit: limit,
-      page: bookmark,
-    }
-  } else if (params && limit) {
-    paginateObj = {
-      limit: limit,
-    }
-  }
-  let sort: SortJson | undefined
-  if (params.sort) {
-    const direction =
-      params.sortOrder === "descending"
-        ? SortDirection.DESCENDING
-        : SortDirection.ASCENDING
-    sort = {
-      [params.sort]: { direction },
-    }
-  }
-  try {
-    const rows = (await handleRequest(Operation.READ, tableId, {
-      filters: query,
-      sort,
-      paginate: paginateObj as PaginationJson,
-      includeSqlRelationships: IncludeRelationship.INCLUDE,
-    })) as Row[]
-    let hasNextPage = false
-    if (paginate && rows.length === limit) {
-      const nextRows = (await handleRequest(Operation.READ, tableId, {
-        filters: query,
-        sort,
-        paginate: {
-          limit: 1,
-          page: bookmark * limit + 1,
-        },
-        includeSqlRelationships: IncludeRelationship.INCLUDE,
-      })) as Row[]
-      hasNextPage = nextRows.length > 0
-    }
-    // need wrapper object for bookmarks etc when paginating
-    return { rows, hasNextPage, bookmark: bookmark + 1 }
-  } catch (err: any) {
-    if (err.message && err.message.includes("does not exist")) {
-      throw new Error(
-        `Table updated externally, please re-fetch - ${err.message}`
-      )
-    } else {
-      throw err
-    }
-  }
-}
-
-export async function exportRows(ctx: UserCtx) {
-  const { datasourceId, tableName } = breakExternalTableId(ctx.params.tableId)
-  const format = ctx.query.format
-  const { columns } = ctx.request.body
-  const datasource = await sdk.datasources.get(datasourceId!)
-  if (!datasource || !datasource.entities) {
-    ctx.throw(400, "Datasource has not been configured for plus API.")
-  }
-
-  if (ctx.request.body.rows) {
-    ctx.request.body = {
-      query: {
-        oneOf: {
-          _id: ctx.request.body.rows.map((row: string) => {
-            const ids = JSON.parse(
-              decodeURI(row).replace(/'/g, `"`).replace(/%2C/g, ",")
-            )
-            if (ids.length > 1) {
-              ctx.throw(400, "Export data does not support composite keys.")
-            }
-            return ids[0]
-          }),
-        },
-      },
-    }
-  }
-
-  let result = await search(ctx)
-  let rows: Row[] = []
-
-  // Filter data to only specified columns if required
-
-  if (columns && columns.length) {
-    for (let i = 0; i < result.rows.length; i++) {
-      rows[i] = {}
-      for (let column of columns) {
-        rows[i][column] = result.rows[i][column]
-      }
-    }
-  } else {
-    rows = result.rows
-  }
-
-  if (!tableName) {
-    ctx.throw(400, "Could not find table name.")
-  }
-  let schema = datasource.entities[tableName].schema
-  let exportRows = cleanExportRows(rows, schema, format, columns)
-
-  let headers = Object.keys(schema)
-
-  // @ts-ignore
-  const exporter = exporters[format]
-  const filename = `export.${format}`
-
-  // send down the file
-  ctx.attachment(filename)
-  return apiFileReturn(exporter(headers, exportRows))
-}
-
 export async function fetchEnrichedRow(ctx: UserCtx) {
   const id = ctx.params.rowId
   const tableId = ctx.params.tableId

packages/server/src/api/controllers/row/index.ts

@@ -5,6 +5,7 @@ import { isExternalTable } from "../../../integrations/utils"
 import { Ctx } from "@budibase/types"
 import * as utils from "./utils"
 import { gridSocket } from "../../../websockets"
+import sdk from "../../../sdk"
 
 function pickApi(tableId: any) {
   if (isExternalTable(tableId)) {
@@ -64,14 +65,14 @@ export const save = async (ctx: any) => {
 }
 export async function fetchView(ctx: any) {
   const tableId = utils.getTableId(ctx)
-  ctx.body = await quotas.addQuery(() => pickApi(tableId).fetchView(ctx), {
+  ctx.body = await quotas.addQuery(() => sdk.rows.fetchView(tableId, ctx), {
     datasourceId: tableId,
   })
 }
 
 export async function fetch(ctx: any) {
   const tableId = utils.getTableId(ctx)
-  ctx.body = await quotas.addQuery(() => pickApi(tableId).fetch(ctx), {
+  ctx.body = await quotas.addQuery(() => sdk.rows.fetch(tableId, ctx), {
     datasourceId: tableId,
   })
 }
@@ -120,7 +121,8 @@ export async function destroy(ctx: any) {
 export async function search(ctx: any) {
   const tableId = utils.getTableId(ctx)
   ctx.status = 200
-  ctx.body = await quotas.addQuery(() => pickApi(tableId).search(ctx), {
+
+  ctx.body = await quotas.addQuery(() => sdk.rows.search(tableId, ctx), {
     datasourceId: tableId,
   })
 }
@@ -150,7 +152,7 @@ export async function fetchEnrichedRow(ctx: any) {
 
 export const exportRows = async (ctx: any) => {
   const tableId = utils.getTableId(ctx)
-  ctx.body = await quotas.addQuery(() => pickApi(tableId).exportRows(ctx), {
+  ctx.body = await quotas.addQuery(() => sdk.rows.exportRows(tableId, ctx), {
     datasourceId: tableId,
   })
 }

packages/server/src/api/controllers/row/internal.ts

@@ -1,7 +1,6 @@
 import * as linkRows from "../../../db/linkedRows"
 import {
   generateRowID,
-  getRowParams,
   getTableIDFromRowID,
   DocumentType,
   InternalTables,
@@ -14,8 +13,6 @@ import {
 } from "../../../utilities/rowProcessor"
 import { FieldTypes } from "../../../constants"
 import * as utils from "./utils"
-import { fullSearch, paginatedSearch } from "./internalSearch"
-import { getGlobalUsersFromMetadata } from "../../../utilities/global"
 import * as inMemoryViews from "../../../db/inMemoryView"
 import env from "../../../environment"
 import {
@@ -27,65 +24,7 @@ import {
 import { cloneDeep } from "lodash/fp"
 import { context, db as dbCore } from "@budibase/backend-core"
 import { finaliseRow, updateRelatedFormula } from "./staticFormula"
-import { csv, json, jsonWithSchema, Format } from "../view/exporters"
+import { UserCtx, LinkDocumentValue, Row, Table } from "@budibase/types"
-import { apiFileReturn } from "../../../utilities/fileSystem"
-import {
-  UserCtx,
-  Database,
-  LinkDocumentValue,
-  Row,
-  Table,
-} from "@budibase/types"
-
-import { cleanExportRows } from "./utils"
-
-const CALCULATION_TYPES = {
-  SUM: "sum",
-  COUNT: "count",
-  STATS: "stats",
-}
-
-async function getView(db: Database, viewName: string) {
-  let mainGetter = env.SELF_HOSTED ? getFromDesignDoc : getFromMemoryDoc
-  let secondaryGetter = env.SELF_HOSTED ? getFromMemoryDoc : getFromDesignDoc
-  let migration = env.SELF_HOSTED ? migrateToDesignView : migrateToInMemoryView
-  let viewInfo,
-    migrate = false
-  try {
-    viewInfo = await mainGetter(db, viewName)
-  } catch (err: any) {
-    // check if it can be retrieved from design doc (needs migrated)
-    if (err.status !== 404) {
-      viewInfo = null
-    } else {
-      viewInfo = await secondaryGetter(db, viewName)
-      migrate = !!viewInfo
-    }
-  }
-  if (migrate) {
-    await migration(db, viewName)
-  }
-  if (!viewInfo) {
-    throw "View does not exist."
-  }
-  return viewInfo
-}
-
-async function getRawTableData(ctx: UserCtx, db: Database, tableId: string) {
-  let rows
-  if (tableId === InternalTables.USER_METADATA) {
-    await userController.fetchMetadata(ctx)
-    rows = ctx.body
-  } else {
-    const response = await db.allDocs(
-      getRowParams(tableId, null, {
-        include_docs: true,
-      })
-    )
-    rows = response.rows.map(row => row.doc)
-  }
-  return rows as Row[]
-}
 
 export async function patch(ctx: UserCtx) {
   const db = context.getAppDB()
@@ -195,82 +134,6 @@ export async function save(ctx: UserCtx) {
   })
 }
 
-export async function fetchView(ctx: UserCtx) {
-  const viewName = decodeURIComponent(ctx.params.viewName)
-
-  // if this is a table view being looked for just transfer to that
-  if (viewName.startsWith(DocumentType.TABLE)) {
-    ctx.params.tableId = viewName
-    return fetch(ctx)
-  }
-
-  const db = context.getAppDB()
-  const { calculation, group, field } = ctx.query
-  const viewInfo = await getView(db, viewName)
-  let response
-  if (env.SELF_HOSTED) {
-    response = await db.query(`database/${viewName}`, {
-      include_docs: !calculation,
-      group: !!group,
-    })
-  } else {
-    const tableId = viewInfo.meta.tableId
-    const data = await getRawTableData(ctx, db, tableId)
-    response = await inMemoryViews.runView(
-      viewInfo,
-      calculation as string,
-      !!group,
-      data
-    )
-  }
-
-  let rows
-  if (!calculation) {
-    response.rows = response.rows.map(row => row.doc)
-    let table
-    try {
-      table = await db.get(viewInfo.meta.tableId)
-    } catch (err) {
-      /* istanbul ignore next */
-      table = {
-        schema: {},
-      }
-    }
-    rows = await outputProcessing(table, response.rows)
-  }
-
-  if (calculation === CALCULATION_TYPES.STATS) {
-    response.rows = response.rows.map(row => ({
-      group: row.key,
-      field,
-      ...row.value,
-      avg: row.value.sum / row.value.count,
-    }))
-    rows = response.rows
-  }
-
-  if (
-    calculation === CALCULATION_TYPES.COUNT ||
-    calculation === CALCULATION_TYPES.SUM
-  ) {
-    rows = response.rows.map(row => ({
-      group: row.key,
-      field,
-      value: row.value,
-    }))
-  }
-  return rows
-}
-
-export async function fetch(ctx: UserCtx) {
-  const db = context.getAppDB()
-
-  const tableId = ctx.params.tableId
-  let table = await db.get(tableId)
-  let rows = await getRawTableData(ctx, db, tableId)
-  return outputProcessing(table, rows)
-}
-
 export async function find(ctx: UserCtx) {
   const db = dbCore.getDB(ctx.appId)
   const table = await db.get(ctx.params.tableId)
@@ -354,101 +217,6 @@ export async function bulkDestroy(ctx: UserCtx) {
   return { response: { ok: true }, rows: processedRows }
 }
 
-export async function search(ctx: UserCtx) {
-  // Fetch the whole table when running in cypress, as search doesn't work
-  if (!env.COUCH_DB_URL && env.isCypress()) {
-    return { rows: await fetch(ctx) }
-  }
-
-  const { tableId } = ctx.params
-  const db = context.getAppDB()
-  const { paginate, query, ...params } = ctx.request.body
-  params.version = ctx.version
-  params.tableId = tableId
-
-  let table
-  if (params.sort && !params.sortType) {
-    table = await db.get(tableId)
-    const schema = table.schema
-    const sortField = schema[params.sort]
-    params.sortType = sortField.type == "number" ? "number" : "string"
-  }
-
-  let response
-  if (paginate) {
-    response = await paginatedSearch(query, params)
-  } else {
-    response = await fullSearch(query, params)
-  }
-
-  // Enrich search results with relationships
-  if (response.rows && response.rows.length) {
-    // enrich with global users if from users table
-    if (tableId === InternalTables.USER_METADATA) {
-      response.rows = await getGlobalUsersFromMetadata(response.rows)
-    }
-    table = table || (await db.get(tableId))
-    response.rows = await outputProcessing(table, response.rows)
-  }
-
-  return response
-}
-
-export async function exportRows(ctx: UserCtx) {
-  const db = context.getAppDB()
-  const table = await db.get(ctx.params.tableId)
-  const rowIds = ctx.request.body.rows
-  let format = ctx.query.format
-  if (typeof format !== "string") {
-    ctx.throw(400, "Format parameter is not valid")
-  }
-  const { columns, query } = ctx.request.body
-
-  let result
-  if (rowIds) {
-    let response = (
-      await db.allDocs({
-        include_docs: true,
-        keys: rowIds,
-      })
-    ).rows.map(row => row.doc)
-
-    result = await outputProcessing(table, response)
-  } else if (query) {
-    let searchResponse = await search(ctx)
-    result = searchResponse.rows
-  }
-
-  let rows: Row[] = []
-  let schema = table.schema
-
-  // Filter data to only specified columns if required
-  if (columns && columns.length) {
-    for (let i = 0; i < result.length; i++) {
-      rows[i] = {}
-      for (let column of columns) {
-        rows[i][column] = result[i][column]
-      }
-    }
-  } else {
-    rows = result
-  }
-
-  let exportRows = cleanExportRows(rows, schema, format, columns)
-  if (format === Format.CSV) {
-    ctx.attachment("export.csv")
-    return apiFileReturn(csv(Object.keys(rows[0]), exportRows))
-  } else if (format === Format.JSON) {
-    ctx.attachment("export.json")
-    return apiFileReturn(json(exportRows))
-  } else if (format === Format.JSON_WITH_SCHEMA) {
-    ctx.attachment("export.json")
-    return apiFileReturn(jsonWithSchema(schema, exportRows))
-  } else {
-    throw "Format not recognised"
-  }
-}
-
 export async function fetchEnrichedRow(ctx: UserCtx) {
   const db = context.getAppDB()
   const tableId = ctx.params.tableId

packages/server/src/api/controllers/row/utils.ts

@@ -1,14 +1,19 @@
 import { InternalTables } from "../../../db/utils"
 import * as userController from "../user"
-import { FieldTypes } from "../../../constants"
 import { context } from "@budibase/backend-core"
-import { makeExternalQuery } from "../../../integrations/base/query"
+import { Ctx, FieldType, Row, Table, UserCtx } from "@budibase/types"
-import { FieldType, Row, Table, UserCtx } from "@budibase/types"
+import { FieldTypes } from "../../../constants"
-import { Format } from "../view/exporters"
 import sdk from "../../../sdk"
 
-const validateJs = require("validate.js")
+import validateJs from "validate.js"
-const { cloneDeep } = require("lodash/fp")
+import { cloneDeep } from "lodash/fp"
+
+function isForeignKey(key: string, table: Table) {
+  const relationships = Object.values(table.schema).filter(
+    column => column.type === FieldType.LINK
+  )
+  return relationships.some(relationship => relationship.foreignKey === key)
+}
 
 validateJs.extend(validateJs.validators.datetime, {
   parse: function (value: string) {
@@ -20,19 +25,6 @@ validateJs.extend(validateJs.validators.datetime, {
   },
 })
 
-function isForeignKey(key: string, table: Table) {
-  const relationships = Object.values(table.schema).filter(
-    column => column.type === FieldType.LINK
-  )
-  return relationships.some(relationship => relationship.foreignKey === key)
-}
-
-export async function getDatasourceAndQuery(json: any) {
-  const datasourceId = json.endpoint.datasourceId
-  const datasource = await sdk.datasources.get(datasourceId)
-  return makeExternalQuery(datasource, json)
-}
-
 export async function findRow(ctx: UserCtx, tableId: string, rowId: string) {
   const db = context.getAppDB()
   let row
@@ -52,6 +44,18 @@ export async function findRow(ctx: UserCtx, tableId: string, rowId: string) {
   return row
 }
 
+export function getTableId(ctx: Ctx) {
+  if (ctx.request.body && ctx.request.body.tableId) {
+    return ctx.request.body.tableId
+  }
+  if (ctx.params && ctx.params.tableId) {
+    return ctx.params.tableId
+  }
+  if (ctx.params && ctx.params.viewName) {
+    return ctx.params.viewName
+  }
+}
+
 export async function validate({
   tableId,
   row,
@@ -81,8 +85,8 @@ export async function validate({
       continue
     }
     // special case for options, need to always allow unselected (empty)
-    if (type === FieldTypes.OPTIONS && constraints.inclusion) {
+    if (type === FieldTypes.OPTIONS && constraints?.inclusion) {
-      constraints.inclusion.push(null, "")
+      constraints.inclusion.push(null as any, "")
     }
     let res
 
@@ -94,13 +98,13 @@ export async function validate({
         }
         row[fieldName].map((val: any) => {
           if (
-            !constraints.inclusion.includes(val) &&
+            !constraints?.inclusion?.includes(val) &&
-            constraints.inclusion.length !== 0
+            constraints?.inclusion?.length !== 0
           ) {
             errors[fieldName] = "Field not in list"
           }
         })
-      } else if (constraints.presence && row[fieldName].length === 0) {
+      } else if (constraints?.presence && row[fieldName].length === 0) {
         // non required MultiSelect creates an empty array, which should not throw errors
         errors[fieldName] = [`${fieldName} is required`]
       }
@@ -128,52 +132,3 @@ export async function validate({
   }
   return { valid: Object.keys(errors).length === 0, errors }
 }
-
-export function cleanExportRows(
-  rows: any[],
-  schema: any,
-  format: string,
-  columns: string[]
-) {
-  let cleanRows = [...rows]
-
-  const relationships = Object.entries(schema)
-    .filter((entry: any[]) => entry[1].type === FieldTypes.LINK)
-    .map(entry => entry[0])
-
-  relationships.forEach(column => {
-    cleanRows.forEach(row => {
-      delete row[column]
-    })
-    delete schema[column]
-  })
-
-  if (format === Format.CSV) {
-    // Intended to append empty values in export
-    const schemaKeys = Object.keys(schema)
-    for (let key of schemaKeys) {
-      if (columns?.length && columns.indexOf(key) > 0) {
-        continue
-      }
-      for (let row of cleanRows) {
-        if (row[key] == null) {
-          row[key] = undefined
-        }
-      }
-    }
-  }
-
-  return cleanRows
-}
-
-export function getTableId(ctx: any) {
-  if (ctx.request.body && ctx.request.body.tableId) {
-    return ctx.request.body.tableId
-  }
-  if (ctx.params && ctx.params.tableId) {
-    return ctx.params.tableId
-  }
-  if (ctx.params && ctx.params.viewName) {
-    return ctx.params.viewName
-  }
-}

packages/server/src/api/controllers/user.ts

@@ -3,10 +3,10 @@ import { InternalTables } from "../../db/utils"
 import { getGlobalUsers } from "../../utilities/global"
 import { getFullUser } from "../../utilities/users"
 import { context } from "@budibase/backend-core"
-import { UserCtx } from "@budibase/types"
+import { Ctx, UserCtx } from "@budibase/types"
 import sdk from "../../sdk"
 
-export async function fetchMetadata(ctx: UserCtx) {
+export async function fetchMetadata(ctx: Ctx) {
   const global = await getGlobalUsers()
   const metadata = await sdk.users.rawUserMetadata()
   const users = []

packages/server/src/api/controllers/view/index.ts

@@ -15,7 +15,6 @@ import {
   TableSchema,
   View,
 } from "@budibase/types"
-import { cleanExportRows } from "../row/utils"
 import { builderSocket } from "../../../websockets"
 
 const { cloneDeep, isEqual } = require("lodash")
@@ -169,7 +168,7 @@ export async function exportView(ctx: Ctx) {
     schema = table.schema
   }
 
-  let exportRows = cleanExportRows(rows, schema, format, [])
+  let exportRows = sdk.rows.utils.cleanExportRows(rows, schema, format, [])
 
   if (format === Format.CSV) {
     ctx.attachment(`${viewName}.csv`)

packages/server/src/api/routes/tests/internalSearch.spec.js

@@ -1,197 +0,0 @@
-const fetch = require("node-fetch")
-fetch.mockSearch()
-const search = require("../../controllers/row/internalSearch")
-// this will be mocked out for _search endpoint
-const PARAMS = {
-  tableId: "ta_12345679abcdef",
-  version: "1",
-  bookmark: null,
-  sort: null,
-  sortOrder: "ascending",
-  sortType: "string",
-}
-
-function checkLucene(resp, expected, params = PARAMS) {
-  const query = resp.rows[0].query
-  const json = JSON.parse(query)
-  if (PARAMS.sort) {
-    expect(json.sort).toBe(`${PARAMS.sort}<${PARAMS.sortType}>`)
-  }
-  if (PARAMS.bookmark) {
-    expect(json.bookmark).toBe(PARAMS.bookmark)
-  }
-  expect(json.include_docs).toBe(true)
-  expect(json.q).toBe(`${expected} AND tableId:"${params.tableId}"`)
-  expect(json.limit).toBe(params.limit || 50)
-}
-
-describe("internal search", () => {
-  it("default query", async () => {
-    const response = await search.paginatedSearch({
-    }, PARAMS)
-    checkLucene(response, `*:*`)
-  })
-
-  it("test equal query", async () => {
-    const response = await search.paginatedSearch({
-      equal: {
-        "column": "1",
-      }
-    }, PARAMS)
-    checkLucene(response, `*:* AND column:"1"`)
-  })
-
-  it("test notEqual query", async () => {
-    const response = await search.paginatedSearch({
-      notEqual: {
-        "column": "1",
-      }
-    }, PARAMS)
-    checkLucene(response, `*:* AND !column:"1"`)
-  })
-
-  it("test OR query", async () => {
-    const response = await search.paginatedSearch({
-      allOr: true,
-      equal: {
-        "column": "2",
-      },
-      notEqual: {
-        "column": "1",
-      }
-    }, PARAMS)
-    checkLucene(response, `(column:"2" OR !column:"1")`)
-  })
-
-  it("test AND query", async () => {
-    const response = await search.paginatedSearch({
-      equal: {
-        "column": "2",
-      },
-      notEqual: {
-        "column": "1",
-      }
-    }, PARAMS)
-    checkLucene(response, `(*:* AND column:"2" AND !column:"1")`)
-  })
-
-  it("test pagination query", async () => {
-    const updatedParams = {
-      ...PARAMS,
-      limit: 100,
-      bookmark: "awd",
-      sort: "column",
-    }
-    const response = await search.paginatedSearch({
-      string: {
-        "column": "2",
-      },
-    }, updatedParams)
-    checkLucene(response, `*:* AND column:2*`, updatedParams)
-  })
-
-  it("test range query", async () => {
-    const response = await search.paginatedSearch({
-      range: {
-        "column": { low: 1, high: 2 },
-      },
-    }, PARAMS)
-    checkLucene(response, `*:* AND column:[1 TO 2]`, PARAMS)
-  })
-
-  it("test empty query", async () => {
-    const response = await search.paginatedSearch({
-      empty: {
-        "column": "",
-      },
-    }, PARAMS)
-    checkLucene(response, `*:* AND (*:* -column:["" TO *])`, PARAMS)
-  })
-
-  it("test notEmpty query", async () => {
-    const response = await search.paginatedSearch({
-      notEmpty: {
-        "column": "",
-      },
-    }, PARAMS)
-    checkLucene(response, `*:* AND column:["" TO *]`, PARAMS)
-  })
-
-  it("test oneOf query", async () => {
-    const response = await search.paginatedSearch({
-      oneOf: {
-        "column": ["a", "b"],
-      },
-    }, PARAMS)
-    checkLucene(response, `*:* AND column:("a" OR "b")`, PARAMS)
-  })
-
-  it("test contains query", async () => {
-    const response = await search.paginatedSearch({
-      contains: {
-        "column": "a",
-        "colArr": [1, 2, 3],
-      },
-    }, PARAMS)
-    checkLucene(response, `(*:* AND column:a AND colArr:(1 AND 2 AND 3))`, PARAMS)
-  })
-
-  it("test multiple of same column", async () => {
-    const response = await search.paginatedSearch({
-      allOr: true,
-      equal: {
-        "1:column": "a",
-        "2:column": "b",
-        "3:column": "c",
-      },
-    }, PARAMS)
-    checkLucene(response, `(column:"a" OR column:"b" OR column:"c")`, PARAMS)
-  })
-
-  it("check a weird case for lucene building", async () => {
-    const response = await search.paginatedSearch({
-      equal: {
-        "1:1:column": "a",
-      },
-    }, PARAMS)
-    checkLucene(response, `*:* AND 1\\:column:"a"`, PARAMS)
-  })
-
-  it("test containsAny query", async () => {
-    const response = await search.paginatedSearch({
-      containsAny: {
-        "column": ["a", "b", "c"]
-      },
-    }, PARAMS)
-    checkLucene(response, `*:* AND column:(a OR b OR c)`, PARAMS)
-  })
-
-  it("test notContains query", async () => {
-    const response = await search.paginatedSearch({
-      notContains: {
-        "column": ["a", "b", "c"]
-      },
-    }, PARAMS)
-    checkLucene(response, `*:* AND NOT column:(a AND b AND c)`, PARAMS)
-  })
-
-  it("test equal without version query", async () => {
-    PARAMS.version = null
-    const response = await search.paginatedSearch({
-      equal: {
-        "column": "1",
-      }
-    }, PARAMS)
-    
-    const query = response.rows[0].query
-    const json = JSON.parse(query)
-    if (PARAMS.sort) {
-      expect(json.sort).toBe(`${PARAMS.sort}<${PARAMS.sortType}>`)
-    }
-    if (PARAMS.bookmark) {
-      expect(json.bookmark).toBe(PARAMS.bookmark)
-    }
-    expect(json.include_docs).toBe(true)
-    expect(json.q).toBe(`*:* AND column:"1" AND tableId:${PARAMS.tableId}`)
-  })
-})

packages/server/src/api/routes/tests/internalSearch.spec.ts

@@ -0,0 +1,248 @@
+const nodeFetch = require("node-fetch")
+nodeFetch.mockSearch()
+import { SearchParams } from "@budibase/backend-core"
+import * as search from "../../../sdk/app/rows/search/internalSearch"
+import { Row } from "@budibase/types"
+
+// this will be mocked out for _search endpoint
+const PARAMS: SearchParams<Row> = {
+  tableId: "ta_12345679abcdef",
+  version: "1",
+  bookmark: undefined,
+  sort: undefined,
+  sortOrder: "ascending",
+  sortType: "string",
+}
+
+function checkLucene(resp: any, expected: any, params = PARAMS) {
+  const query = resp.rows[0].query
+  const json = JSON.parse(query)
+  if (PARAMS.sort) {
+    expect(json.sort).toBe(`${PARAMS.sort}<${PARAMS.sortType}>`)
+  }
+  if (PARAMS.bookmark) {
+    expect(json.bookmark).toBe(PARAMS.bookmark)
+  }
+  expect(json.include_docs).toBe(true)
+  expect(json.q).toBe(`${expected} AND tableId:"${params.tableId}"`)
+  expect(json.limit).toBe(params.limit || 50)
+}
+
+describe("internal search", () => {
+  it("default query", async () => {
+    const response = await search.paginatedSearch({}, PARAMS)
+    checkLucene(response, `*:*`)
+  })
+
+  it("test equal query", async () => {
+    const response = await search.paginatedSearch(
+      {
+        equal: {
+          column: "1",
+        },
+      },
+      PARAMS
+    )
+    checkLucene(response, `*:* AND column:"1"`)
+  })
+
+  it("test notEqual query", async () => {
+    const response = await search.paginatedSearch(
+      {
+        notEqual: {
+          column: "1",
+        },
+      },
+      PARAMS
+    )
+    checkLucene(response, `*:* AND !column:"1"`)
+  })
+
+  it("test OR query", async () => {
+    const response = await search.paginatedSearch(
+      {
+        allOr: true,
+        equal: {
+          column: "2",
+        },
+        notEqual: {
+          column: "1",
+        },
+      },
+      PARAMS
+    )
+    checkLucene(response, `(column:"2" OR !column:"1")`)
+  })
+
+  it("test AND query", async () => {
+    const response = await search.paginatedSearch(
+      {
+        equal: {
+          column: "2",
+        },
+        notEqual: {
+          column: "1",
+        },
+      },
+      PARAMS
+    )
+    checkLucene(response, `(*:* AND column:"2" AND !column:"1")`)
+  })
+
+  it("test pagination query", async () => {
+    const updatedParams = {
+      ...PARAMS,
+      limit: 100,
+      bookmark: "awd",
+      sort: "column",
+    }
+    const response = await search.paginatedSearch(
+      {
+        string: {
+          column: "2",
+        },
+      },
+      updatedParams
+    )
+    checkLucene(response, `*:* AND column:2*`, updatedParams)
+  })
+
+  it("test range query", async () => {
+    const response = await search.paginatedSearch(
+      {
+        range: {
+          column: { low: 1, high: 2 },
+        },
+      },
+      PARAMS
+    )
+    checkLucene(response, `*:* AND column:[1 TO 2]`, PARAMS)
+  })
+
+  it("test empty query", async () => {
+    const response = await search.paginatedSearch(
+      {
+        empty: {
+          column: "",
+        },
+      },
+      PARAMS
+    )
+    checkLucene(response, `*:* AND (*:* -column:["" TO *])`, PARAMS)
+  })
+
+  it("test notEmpty query", async () => {
+    const response = await search.paginatedSearch(
+      {
+        notEmpty: {
+          column: "",
+        },
+      },
+      PARAMS
+    )
+    checkLucene(response, `*:* AND column:["" TO *]`, PARAMS)
+  })
+
+  it("test oneOf query", async () => {
+    const response = await search.paginatedSearch(
+      {
+        oneOf: {
+          column: ["a", "b"],
+        },
+      },
+      PARAMS
+    )
+    checkLucene(response, `*:* AND column:("a" OR "b")`, PARAMS)
+  })
+
+  it("test contains query", async () => {
+    const response = await search.paginatedSearch(
+      {
+        contains: {
+          column: "a",
+          colArr: [1, 2, 3],
+        },
+      },
+      PARAMS
+    )
+    checkLucene(
+      response,
+      `(*:* AND column:a AND colArr:(1 AND 2 AND 3))`,
+      PARAMS
+    )
+  })
+
+  it("test multiple of same column", async () => {
+    const response = await search.paginatedSearch(
+      {
+        allOr: true,
+        equal: {
+          "1:column": "a",
+          "2:column": "b",
+          "3:column": "c",
+        },
+      },
+      PARAMS
+    )
+    checkLucene(response, `(column:"a" OR column:"b" OR column:"c")`, PARAMS)
+  })
+
+  it("check a weird case for lucene building", async () => {
+    const response = await search.paginatedSearch(
+      {
+        equal: {
+          "1:1:column": "a",
+        },
+      },
+      PARAMS
+    )
+    checkLucene(response, `*:* AND 1\\:column:"a"`, PARAMS)
+  })
+
+  it("test containsAny query", async () => {
+    const response = await search.paginatedSearch(
+      {
+        containsAny: {
+          column: ["a", "b", "c"],
+        },
+      },
+      PARAMS
+    )
+    checkLucene(response, `*:* AND column:(a OR b OR c)`, PARAMS)
+  })
+
+  it("test notContains query", async () => {
+    const response = await search.paginatedSearch(
+      {
+        notContains: {
+          column: ["a", "b", "c"],
+        },
+      },
+      PARAMS
+    )
+    checkLucene(response, `*:* AND NOT column:(a AND b AND c)`, PARAMS)
+  })
+
+  it("test equal without version query", async () => {
+    PARAMS.version = undefined
+    const response = await search.paginatedSearch(
+      {
+        equal: {
+          column: "1",
+        },
+      },
+      PARAMS
+    )
+
+    const query = response.rows[0].query
+    const json = JSON.parse(query)
+    if (PARAMS.sort) {
+      expect(json.sort).toBe(`${PARAMS.sort}<${PARAMS.sortType}>`)
+    }
+    if (PARAMS.bookmark) {
+      expect(json.bookmark).toBe(PARAMS.bookmark)
+    }
+    expect(json.include_docs).toBe(true)
+    expect(json.q).toBe(`*:* AND column:"1" AND tableId:${PARAMS.tableId}`)
+  })
+})

packages/server/src/automations/tests/queryRows.spec.ts

@@ -1,6 +1,6 @@
 // lucene searching not supported in test due to use of PouchDB
-let rows = []
+let rows: Row[] = []
-jest.mock("../../api/controllers/row/internalSearch", () => ({
+jest.mock("../../sdk/app/rows/search/internalSearch", () => ({
   fullSearch: jest.fn(() => {
     return {
       rows,
@@ -8,12 +8,13 @@ jest.mock("../../api/controllers/row/internalSearch", () => ({
   }),
   paginatedSearch: jest.fn(),
 }))
-const setup = require("./utilities")
+import { Row, Table } from "@budibase/types"
+import * as setup from "./utilities"
 
 const NAME = "Test"
 
 describe("Test a query step automation", () => {
-  let table
+  let table: Table
   let config = setup.getConfig()
 
   beforeAll(async () => {
@@ -87,8 +88,8 @@ describe("Test a query step automation", () => {
       filters: {},
       "filters-def": [
         {
-          value: null
+          value: null,
-        }
+        },
       ],
       sortColumn: "name",
       sortOrder: "ascending",

packages/server/src/sdk/app/rows/index.ts

@@ -1,7 +1,11 @@
 import * as attachments from "./attachments"
 import * as rows from "./rows"
+import * as search from "./search"
+import * as utils from "./utils"
 
 export default {
   ...attachments,
   ...rows,
+  ...search,
+  utils: utils,
 }

packages/server/src/sdk/app/rows/search.ts

@@ -0,0 +1,38 @@
+import { Ctx, SearchFilters } from "@budibase/types"
+import { isExternalTable } from "../../../integrations/utils"
+import * as internal from "./search/internal"
+import * as external from "./search/external"
+
+export interface SearchParams {
+  tableId: string
+  paginate: boolean
+  query?: SearchFilters
+  bookmark?: number
+  limit: number
+  sort?: string
+  sortOrder?: string
+  sortType?: string
+}
+
+function pickApi(tableId: any) {
+  if (isExternalTable(tableId)) {
+    return external
+  }
+  return internal
+}
+
+export async function search(tableId: string, ctx: Ctx) {
+  return pickApi(tableId).search(ctx)
+}
+
+export async function exportRows(tableId: string, ctx: Ctx) {
+  return pickApi(tableId).exportRows(ctx)
+}
+
+export async function fetch(tableId: string, ctx: Ctx) {
+  return pickApi(tableId).fetch(ctx)
+}
+
+export async function fetchView(tableId: string, ctx: Ctx) {
+  return pickApi(tableId).fetchView(ctx)
+}

packages/server/src/sdk/app/rows/search/external.ts

@@ -0,0 +1,177 @@
+import {
+  SortJson,
+  SortDirection,
+  Operation,
+  PaginationJson,
+  IncludeRelationship,
+  Row,
+  Ctx,
+} from "@budibase/types"
+import * as exporters from "../../../../api/controllers/view/exporters"
+import sdk from "../../../../sdk"
+import { handleRequest } from "../../../../api/controllers/row/external"
+import { breakExternalTableId } from "../../../../integrations/utils"
+import { cleanExportRows } from "../utils"
+import { apiFileReturn } from "../../../../utilities/fileSystem"
+import { utils } from "@budibase/shared-core"
+
+export async function search(ctx: Ctx) {
+  const tableId = ctx.params.tableId
+  const { paginate, query, ...params } = ctx.request.body
+  let { bookmark, limit } = params
+  if (!bookmark && paginate) {
+    bookmark = 1
+  }
+  let paginateObj = {}
+
+  if (paginate) {
+    paginateObj = {
+      // add one so we can track if there is another page
+      limit: limit,
+      page: bookmark,
+    }
+  } else if (params && limit) {
+    paginateObj = {
+      limit: limit,
+    }
+  }
+  let sort: SortJson | undefined
+  if (params.sort) {
+    const direction =
+      params.sortOrder === "descending"
+        ? SortDirection.DESCENDING
+        : SortDirection.ASCENDING
+    sort = {
+      [params.sort]: { direction },
+    }
+  }
+  try {
+    const rows = (await handleRequest(Operation.READ, tableId, {
+      filters: query,
+      sort,
+      paginate: paginateObj as PaginationJson,
+      includeSqlRelationships: IncludeRelationship.INCLUDE,
+    })) as Row[]
+    let hasNextPage = false
+    if (paginate && rows.length === limit) {
+      const nextRows = (await handleRequest(Operation.READ, tableId, {
+        filters: query,
+        sort,
+        paginate: {
+          limit: 1,
+          page: bookmark * limit + 1,
+        },
+        includeSqlRelationships: IncludeRelationship.INCLUDE,
+      })) as Row[]
+      hasNextPage = nextRows.length > 0
+    }
+    // need wrapper object for bookmarks etc when paginating
+    return { rows, hasNextPage, bookmark: bookmark + 1 }
+  } catch (err: any) {
+    if (err.message && err.message.includes("does not exist")) {
+      throw new Error(
+        `Table updated externally, please re-fetch - ${err.message}`
+      )
+    } else {
+      throw err
+    }
+  }
+}
+
+export async function exportRows(ctx: Ctx) {
+  const { datasourceId, tableName } = breakExternalTableId(ctx.params.tableId)
+  const format = ctx.query.format as string
+  const { columns } = ctx.request.body
+  const datasource = await sdk.datasources.get(datasourceId!)
+  if (!datasource || !datasource.entities) {
+    ctx.throw(400, "Datasource has not been configured for plus API.")
+  }
+
+  if (!exporters.isFormat(format)) {
+    ctx.throw(
+      400,
+      `Format ${format} not valid. Valid values: ${Object.values(
+        exporters.Format
+      )}`
+    )
+  }
+
+  if (ctx.request.body.rows) {
+    ctx.request.body = {
+      query: {
+        oneOf: {
+          _id: ctx.request.body.rows.map((row: string) => {
+            const ids = JSON.parse(
+              decodeURI(row).replace(/'/g, `"`).replace(/%2C/g, ",")
+            )
+            if (ids.length > 1) {
+              ctx.throw(400, "Export data does not support composite keys.")
+            }
+            return ids[0]
+          }),
+        },
+      },
+    }
+  }
+
+  let result = await search(ctx)
+  let rows: Row[] = []
+
+  // Filter data to only specified columns if required
+
+  if (columns && columns.length) {
+    for (let i = 0; i < result.rows.length; i++) {
+      rows[i] = {}
+      for (let column of columns) {
+        rows[i][column] = result.rows[i][column]
+      }
+    }
+  } else {
+    rows = result.rows
+  }
+
+  if (!tableName) {
+    ctx.throw(400, "Could not find table name.")
+  }
+  let schema = datasource.entities[tableName].schema
+  let exportRows = cleanExportRows(rows, schema, format, columns)
+
+  let headers = Object.keys(schema)
+
+  let content
+  switch (format) {
+    case exporters.Format.CSV:
+      content = exporters.csv(headers, exportRows)
+      break
+    case exporters.Format.JSON:
+      content = exporters.json(exportRows)
+      break
+    case exporters.Format.JSON_WITH_SCHEMA:
+      content = exporters.jsonWithSchema(schema, exportRows)
+      break
+    default:
+      utils.unreachable(format)
+      break
+  }
+
+  const filename = `export.${format}`
+
+  // send down the file
+  ctx.attachment(filename)
+  return apiFileReturn(content)
+}
+
+export async function fetch(ctx: Ctx) {
+  const tableId = ctx.params.tableId
+  return handleRequest(Operation.READ, tableId, {
+    includeSqlRelationships: IncludeRelationship.INCLUDE,
+  })
+}
+
+export async function fetchView(ctx: Ctx) {
+  // there are no views in external datasources, shouldn't ever be called
+  // for now just fetch
+  const split = ctx.params.viewName.split("all_")
+  ctx.params.tableId = split[1] ? split[1] : split[0]
+  return fetch(ctx)
+}

packages/server/src/sdk/app/rows/search/internal.ts

@@ -0,0 +1,246 @@
+import { context } from "@budibase/backend-core"
+import env from "../../../../environment"
+import { fullSearch, paginatedSearch } from "./internalSearch"
+import {
+  InternalTables,
+  getRowParams,
+  DocumentType,
+} from "../../../../db/utils"
+import { getGlobalUsersFromMetadata } from "../../../../utilities/global"
+import { outputProcessing } from "../../../../utilities/rowProcessor"
+import { Ctx, Database, Row, UserCtx } from "@budibase/types"
+import { cleanExportRows } from "../utils"
+import {
+  Format,
+  csv,
+  json,
+  jsonWithSchema,
+} from "../../../../api/controllers/view/exporters"
+import { apiFileReturn } from "../../../../utilities/fileSystem"
+import * as userController from "../../../../api/controllers/user"
+import * as inMemoryViews from "../../../../db/inMemoryView"
+import {
+  migrateToInMemoryView,
+  migrateToDesignView,
+  getFromDesignDoc,
+  getFromMemoryDoc,
+} from "../../../../api/controllers/view/utils"
+
+export async function search(ctx: Ctx) {
+  // Fetch the whole table when running in cypress, as search doesn't work
+  if (!env.COUCH_DB_URL && env.isCypress()) {
+    return { rows: await fetch(ctx) }
+  }
+
+  const { tableId } = ctx.params
+  const db = context.getAppDB()
+  const { paginate, query, ...params } = ctx.request.body
+  params.version = ctx.version
+  params.tableId = tableId
+
+  let table
+  if (params.sort && !params.sortType) {
+    table = await db.get(tableId)
+    const schema = table.schema
+    const sortField = schema[params.sort]
+    params.sortType = sortField.type == "number" ? "number" : "string"
+  }
+
+  let response
+  if (paginate) {
+    response = await paginatedSearch(query, params)
+  } else {
+    response = await fullSearch(query, params)
+  }
+
+  // Enrich search results with relationships
+  if (response.rows && response.rows.length) {
+    // enrich with global users if from users table
+    if (tableId === InternalTables.USER_METADATA) {
+      response.rows = await getGlobalUsersFromMetadata(response.rows)
+    }
+    table = table || (await db.get(tableId))
+    response.rows = await outputProcessing(table, response.rows)
+  }
+
+  return response
+}
+
+export async function exportRows(ctx: Ctx) {
+  const db = context.getAppDB()
+  const table = await db.get(ctx.params.tableId)
+  const rowIds = ctx.request.body.rows
+  let format = ctx.query.format
+  if (typeof format !== "string") {
+    ctx.throw(400, "Format parameter is not valid")
+  }
+  const { columns, query } = ctx.request.body
+
+  let result
+  if (rowIds) {
+    let response = (
+      await db.allDocs({
+        include_docs: true,
+        keys: rowIds,
+      })
+    ).rows.map(row => row.doc)
+
+    result = await outputProcessing(table, response)
+  } else if (query) {
+    let searchResponse = await search(ctx)
+    result = searchResponse.rows
+  }
+
+  let rows: Row[] = []
+  let schema = table.schema
+
+  // Filter data to only specified columns if required
+  if (columns && columns.length) {
+    for (let i = 0; i < result.length; i++) {
+      rows[i] = {}
+      for (let column of columns) {
+        rows[i][column] = result[i][column]
+      }
+    }
+  } else {
+    rows = result
+  }
+
+  let exportRows = cleanExportRows(rows, schema, format, columns)
+  if (format === Format.CSV) {
+    ctx.attachment("export.csv")
+    return apiFileReturn(csv(Object.keys(rows[0]), exportRows))
+  } else if (format === Format.JSON) {
+    ctx.attachment("export.json")
+    return apiFileReturn(json(exportRows))
+  } else if (format === Format.JSON_WITH_SCHEMA) {
+    ctx.attachment("export.json")
+    return apiFileReturn(jsonWithSchema(schema, exportRows))
+  } else {
+    throw "Format not recognised"
+  }
+}
+
+export async function fetch(ctx: Ctx) {
+  const db = context.getAppDB()
+
+  const tableId = ctx.params.tableId
+  let table = await db.get(tableId)
+  let rows = await getRawTableData(ctx, db, tableId)
+  return outputProcessing(table, rows)
+}
+
+async function getRawTableData(ctx: Ctx, db: Database, tableId: string) {
+  let rows
+  if (tableId === InternalTables.USER_METADATA) {
+    await userController.fetchMetadata(ctx)
+    rows = ctx.body
+  } else {
+    const response = await db.allDocs(
+      getRowParams(tableId, null, {
+        include_docs: true,
+      })
+    )
+    rows = response.rows.map(row => row.doc)
+  }
+  return rows as Row[]
+}
+
+export async function fetchView(ctx: Ctx) {
+  const viewName = decodeURIComponent(ctx.params.viewName)
+
+  // if this is a table view being looked for just transfer to that
+  if (viewName.startsWith(DocumentType.TABLE)) {
+    ctx.params.tableId = viewName
+    return fetch(ctx)
+  }
+
+  const db = context.getAppDB()
+  const { calculation, group, field } = ctx.query
+  const viewInfo = await getView(db, viewName)
+  let response
+  if (env.SELF_HOSTED) {
+    response = await db.query(`database/${viewName}`, {
+      include_docs: !calculation,
+      group: !!group,
+    })
+  } else {
+    const tableId = viewInfo.meta.tableId
+    const data = await getRawTableData(ctx, db, tableId)
+    response = await inMemoryViews.runView(
+      viewInfo,
+      calculation as string,
+      !!group,
+      data
+    )
+  }
+
+  let rows
+  if (!calculation) {
+    response.rows = response.rows.map(row => row.doc)
+    let table
+    try {
+      table = await db.get(viewInfo.meta.tableId)
+    } catch (err) {
+      /* istanbul ignore next */
+      table = {
+        schema: {},
+      }
+    }
+    rows = await outputProcessing(table, response.rows)
+  }
+
+  if (calculation === CALCULATION_TYPES.STATS) {
+    response.rows = response.rows.map(row => ({
+      group: row.key,
+      field,
+      ...row.value,
+      avg: row.value.sum / row.value.count,
+    }))
+    rows = response.rows
+  }
+
+  if (
+    calculation === CALCULATION_TYPES.COUNT ||
+    calculation === CALCULATION_TYPES.SUM
+  ) {
+    rows = response.rows.map(row => ({
+      group: row.key,
+      field,
+      value: row.value,
+    }))
+  }
+  return rows
+}
+
+const CALCULATION_TYPES = {
+  SUM: "sum",
+  COUNT: "count",
+  STATS: "stats",
+}
+
+async function getView(db: Database, viewName: string) {
+  let mainGetter = env.SELF_HOSTED ? getFromDesignDoc : getFromMemoryDoc
+  let secondaryGetter = env.SELF_HOSTED ? getFromMemoryDoc : getFromDesignDoc
+  let migration = env.SELF_HOSTED ? migrateToDesignView : migrateToInMemoryView
+  let viewInfo,
+    migrate = false
+  try {
+    viewInfo = await mainGetter(db, viewName)
+  } catch (err: any) {
+    // check if it can be retrieved from design doc (needs migrated)
+    if (err.status !== 404) {
+      viewInfo = null
+    } else {
+      viewInfo = await secondaryGetter(db, viewName)
+      migrate = !!viewInfo
+    }
+  }
+  if (migrate) {
+    await migration(db, viewName)
+  }
+  if (!viewInfo) {
+    throw "View does not exist."
+  }
+  return viewInfo
+}

packages/server/src/sdk/app/rows/utils.ts

@@ -0,0 +1,47 @@
+import { FieldTypes } from "../../../constants"
+import { makeExternalQuery } from "../../../integrations/base/query"
+import { Format } from "../../../api/controllers/view/exporters"
+import sdk from "../.."
+
+export async function getDatasourceAndQuery(json: any) {
+  const datasourceId = json.endpoint.datasourceId
+  const datasource = await sdk.datasources.get(datasourceId)
+  return makeExternalQuery(datasource, json)
+}
+
+export function cleanExportRows(
+  rows: any[],
+  schema: any,
+  format: string,
+  columns: string[]
+) {
+  let cleanRows = [...rows]
+
+  const relationships = Object.entries(schema)
+    .filter((entry: any[]) => entry[1].type === FieldTypes.LINK)
+    .map(entry => entry[0])
+
+  relationships.forEach(column => {
+    cleanRows.forEach(row => {
+      delete row[column]
+    })
+    delete schema[column]
+  })
+
+  if (format === Format.CSV) {
+    // Intended to append empty values in export
+    const schemaKeys = Object.keys(schema)
+    for (let key of schemaKeys) {
+      if (columns?.length && columns.indexOf(key) > 0) {
+        continue
+      }
+      for (let row of cleanRows) {
+        if (row[key] == null) {
+          row[key] = undefined
+        }
+      }
+    }
+  }
+
+  return cleanRows
+}

packages/server/src/sdk/tests/rows/row.spec.ts

@@ -1,14 +1,14 @@
-import { exportRows } from "../row/external"
+import { exportRows } from "../../app/rows/search/external"
-import sdk from "../../../sdk"
+import sdk from "../.."
-import { ExternalRequest } from "../row/ExternalRequest"
+import { ExternalRequest } from "../../../api/controllers/row/ExternalRequest"
 
-// @ts-ignore
+const mockDatasourcesGet = jest.fn()
-sdk.datasources = {
+sdk.datasources.get = mockDatasourcesGet
-  get: jest.fn(),
-}
 
-jest.mock("../row/ExternalRequest")
+jest.mock("../../../api/controllers/row/ExternalRequest")
-jest.mock("../view/exporters", () => ({
+
+jest.mock("../../../api/controllers/view/exporters", () => ({
+  ...jest.requireActual("../../../api/controllers/view/exporters"),
   csv: jest.fn(),
   Format: {
     CSV: "csv",
@@ -31,14 +31,15 @@ function getUserCtx() {
       throw "Err"
     }),
     attachment: jest.fn(),
-  }
+  } as any
 }
 
 describe("external row controller", () => {
   describe("exportRows", () => {
     beforeAll(() => {
-      //@ts-ignore
+      jest
-      jest.spyOn(ExternalRequest.prototype, "run").mockImplementation(() => [])
+        .spyOn(ExternalRequest.prototype, "run")
+        .mockImplementation(() => Promise.resolve([]))
     })
 
     afterEach(() => {
@@ -48,7 +49,6 @@ describe("external row controller", () => {
     it("should throw a 400 if no datasource entities are present", async () => {
       let userCtx = getUserCtx()
       try {
-        //@ts-ignore
         await exportRows(userCtx)
       } catch (e) {
         expect(userCtx.throw).toHaveBeenCalledWith(
@@ -59,8 +59,7 @@ describe("external row controller", () => {
     })
 
     it("should handle single quotes from a row ID", async () => {
-      //@ts-ignore
+      mockDatasourcesGet.mockImplementation(async () => ({
-      sdk.datasources.get.mockImplementation(() => ({
         entities: {
           tablename: {
             schema: {},
@@ -72,7 +71,6 @@ describe("external row controller", () => {
         rows: ["['d001']"],
       }
 
-      //@ts-ignore
       await exportRows(userCtx)
 
       expect(userCtx.request.body).toEqual({
@@ -90,7 +88,6 @@ describe("external row controller", () => {
         rows: ["[123]", "['d001'%2C'10111']"],
       }
       try {
-        //@ts-ignore
         await exportRows(userCtx)
       } catch (e) {
         expect(userCtx.throw).toHaveBeenCalledWith(
@@ -107,7 +104,6 @@ describe("external row controller", () => {
         rows: ["[123]"],
       }
       try {
-        //@ts-ignore
         await exportRows(userCtx)
       } catch (e) {
         expect(userCtx.throw).toHaveBeenCalledWith(

packages/types/src/sdk/search.ts

@@ -32,7 +32,7 @@ export interface SearchFilters {
     [key: string]: any[]
   }
   contains?: {
-    [key: string]: any[]
+    [key: string]: any[] | any
   }
   notContains?: {
     [key: string]: any[]