Bit more work towards row counting, as well as moving external SQL to use row + 1 for working out pagination.

packages/backend-core/src/sql/sql.ts

@@ -571,15 +571,10 @@ class InternalBuilder {
     return query.insert(parsedBody)
   }
 
-  read(
+  read(knex: Knex, json: QueryJson, limit: number): Knex.QueryBuilder {
-    knex: Knex,
-    json: QueryJson,
-    limit: number,
-    opts?: { counting?: boolean }
-  ): Knex.QueryBuilder {
     let { endpoint, resource, filters, paginate, relationships, tableAliases } =
       json
-    const counting = opts?.counting
+    const counting = endpoint.operation === Operation.COUNT
 
     const tableName = endpoint.entityId
     // select all if not specified
@@ -730,6 +725,7 @@ class SqlQueryBuilder extends SqlTableQueryBuilder {
         query = builder.create(client, json, opts)
         break
       case Operation.READ:
+      case Operation.COUNT:
         query = builder.read(client, json, this.limit)
         break
       case Operation.UPDATE:
@@ -752,20 +748,6 @@ class SqlQueryBuilder extends SqlTableQueryBuilder {
     return this.convertToNative(query, opts)
   }
 
-  _count(json: QueryJson, opts: QueryOptions = {}) {
-    const sqlClient = this.getSqlClient()
-    const config: Knex.Config = {
-      client: sqlClient,
-    }
-    if (sqlClient === SqlClient.SQL_LITE) {
-      config.useNullAsDefault = true
-    }
-    const client = knex(config)
-    const builder = new InternalBuilder(sqlClient)
-    const query = builder.read(client, json, this.limit, { counting: true })
-    return this.convertToNative(query, opts)
-  }
-
   async getReturningRow(queryFn: QueryFunction, json: QueryJson) {
     if (!json.extra || !json.extra.idFilter) {
       return {}

packages/server/src/api/controllers/row/ExternalRequest.ts

@@ -39,6 +39,7 @@ import { cloneDeep } from "lodash/fp"
 import { db as dbCore } from "@budibase/backend-core"
 import sdk from "../../../sdk"
 import env from "../../../environment"
+import { makeExternalQuery } from "../../../integrations/base/query"
 
 export interface ManyRelationship {
   tableId?: string
@@ -517,7 +518,7 @@ export class ExternalRequest<T extends Operation> {
     // finally cleanup anything that needs to be removed
     for (let [colName, { isMany, rows, tableId }] of Object.entries(related)) {
       const table: Table | undefined = this.getTable(tableId)
-      // if its not the foreign key skip it, nothing to do
+      // if it's not the foreign key skip it, nothing to do
       if (
         !table ||
         (!isMany && table.primary && table.primary.indexOf(colName) !== -1)
@@ -667,7 +668,7 @@ export class ExternalRequest<T extends Operation> {
       response = await getDatasourceAndQuery(json)
     } else {
       const aliasing = new sdk.rows.AliasTables(Object.keys(this.tables))
-      response = await aliasing.queryWithAliasing(json)
+      response = await aliasing.queryWithAliasing(json, makeExternalQuery)
     }
 
     const responseRows = Array.isArray(response) ? response : []

packages/server/src/integrations/base/query.ts

@@ -8,8 +8,8 @@ import { getIntegration } from "../index"
 import sdk from "../../sdk"
 
 export async function makeExternalQuery(
-  datasource: Datasource,
+  json: QueryJson,
-  json: QueryJson
+  datasource?: Datasource
 ): Promise<DatasourcePlusQueryResponse> {
   const entityId = json.endpoint.entityId,
     tableName = json.meta.table.name,
@@ -22,6 +22,9 @@ export async function makeExternalQuery(
   ) {
     throw new Error("Entity ID and table metadata do not align")
   }
+  if (!datasource) {
+    throw new Error("No datasource provided for external query")
+  }
   datasource = await sdk.datasources.enrich(datasource)
   const Integration = await getIntegration(datasource.source)
   // query is the opinionated function

packages/server/src/sdk/app/rows/search/external.ts

@@ -28,7 +28,7 @@ export async function search(
   table: Table
 ): Promise<SearchResponse<Row>> {
   const { tableId } = options
-  const { paginate, query, ...params } = options
+  const { countRows, paginate, query, ...params } = options
   const { limit } = params
   let bookmark =
     (params.bookmark && parseInt(params.bookmark as string)) || undefined
@@ -37,10 +37,14 @@ export async function search(
   }
   let paginateObj = {}
 
-  if (paginate) {
+  if (paginate && !limit) {
+    throw new Error("Cannot paginate query without a limit")
+  }
+
+  if (paginate && limit) {
     paginateObj = {
       // add one so we can track if there is another page
-      limit: limit,
+      limit: limit + 1,
       page: bookmark,
     }
   } else if (params && limit) {
@@ -76,17 +80,10 @@ export async function search(
       includeSqlRelationships: IncludeRelationship.INCLUDE,
     })
     let hasNextPage = false
-    if (paginate && rows.length === limit) {
+    // remove the extra row if it's there
-      const nextRows = await handleRequest(Operation.READ, tableId, {
+    if (paginate && limit && rows.length > limit) {
-        filters: query,
+      rows.pop()
-        sort,
+      hasNextPage = true
-        paginate: {
-          limit: 1,
-          page: bookmark! * limit + 1,
-        },
-        includeSqlRelationships: IncludeRelationship.INCLUDE,
-      })
-      hasNextPage = nextRows.length > 0
     }
 
     if (options.fields) {

packages/server/src/sdk/app/rows/search/sqs.ts

@@ -12,7 +12,6 @@ import {
   SortOrder,
   SortType,
   SqlClient,
-  SqlQuery,
   Table,
 } from "@budibase/types"
 import {
@@ -114,17 +113,13 @@ async function runSqlQuery(
   opts?: { countTotalRows?: boolean }
 ) {
   const alias = new AliasTables(tables.map(table => table.name))
+  if (opts?.countTotalRows) {
+    json.endpoint.operation = Operation.COUNT
+  }
   const processSQLQuery = async (json: QueryJson) => {
-    let query: SqlQuery | SqlQuery[]
+    const query = builder._query(json, {
-    if (opts?.countTotalRows) {
+      disableReturning: true,
-      query = builder._count(json, {
+    })
-        disableReturning: true,
-      })
-    } else {
-      query = builder._query(json, {
-        disableReturning: true,
-      })
-    }
 
     if (Array.isArray(query)) {
       throw new Error("SQS cannot currently handle multiple queries")
@@ -227,14 +222,18 @@ export async function search(
     )
 
     // check for pagination final row
-    let nextRow: Row | undefined, rowCount: number | undefined
+    let nextRow: Row | undefined
     if (paginate && params.limit && processed.length > params.limit) {
-      // get the total count of rows
-      rowCount = await runSqlQuery(request, allTables, { countTotalRows: true })
       // remove the extra row that confirmed if there is another row to move to
       nextRow = processed.pop()
     }
 
+    let rowCount: number | undefined
+    if (options.countRows) {
+      // get the total count of rows
+      rowCount = await runSqlQuery(request, allTables, { countTotalRows: true })
+    }
+
     // get the rows
     let finalRows = await outputProcessing<Row[]>(table, processed, {
       preserveLinks: true,
@@ -255,9 +254,11 @@ export async function search(
       const hasNextPage = !!nextRow
       response.hasNextPage = hasNextPage
       if (hasNextPage) {
-        response.totalRows = rowCount
         response.bookmark = bookmark + 1
       }
+      if (rowCount != null) {
+        response.totalRows = rowCount
+      }
       return response
     } else {
       return {

packages/server/src/sdk/app/rows/sqlAlias.ts

@@ -11,7 +11,11 @@ import { SQS_DATASOURCE_INTERNAL } from "@budibase/backend-core"
 import { getSQLClient } from "./utils"
 import { cloneDeep } from "lodash"
 import datasources from "../datasources"
-import { makeExternalQuery } from "../../../integrations/base/query"
+
+type PerformQueryFunction = (
+  json: QueryJson,
+  datasource?: Datasource
+) => Promise<DatasourcePlusQueryResponse>
 
 const WRITE_OPERATIONS: Operation[] = [
   Operation.CREATE,
@@ -171,7 +175,7 @@ export default class AliasTables {
 
   async queryWithAliasing(
     json: QueryJson,
-    queryFn?: (json: QueryJson) => Promise<DatasourcePlusQueryResponse>
+    queryFn: PerformQueryFunction
   ): Promise<DatasourcePlusQueryResponse> {
     const datasourceId = json.endpoint.datasourceId
     const isSqs = datasourceId === SQS_DATASOURCE_INTERNAL
@@ -229,14 +233,7 @@ export default class AliasTables {
       json.tableAliases = invertedTableAliases
     }
 
-    let response: DatasourcePlusQueryResponse
+    let response: DatasourcePlusQueryResponse = await queryFn(json, datasource)
-    if (datasource && !isSqs) {
-      response = await makeExternalQuery(datasource, json)
-    } else if (queryFn) {
-      response = await queryFn(json)
-    } else {
-      throw new Error("No supplied method to perform aliased query")
-    }
     if (Array.isArray(response) && aliasingEnabled) {
       return this.reverse(response)
     } else {
@@ -247,8 +244,9 @@ export default class AliasTables {
   // handles getting the count out of the query
   async countWithAliasing(
     json: QueryJson,
-    queryFn?: (json: QueryJson) => Promise<DatasourcePlusQueryResponse>
+    queryFn: PerformQueryFunction
   ): Promise<number> {
+    json.endpoint.operation = Operation.COUNT
     let response = await this.queryWithAliasing(json, queryFn)
     if (response && response.length === 1 && "total" in response[0]) {
       return response[0].total

packages/types/src/api/web/app/rows.ts

@@ -25,6 +25,7 @@ export interface SearchViewRowRequest
     | "bookmark"
     | "paginate"
     | "query"
+    | "countRows"
   > {}
 
 export interface SearchRowResponse {

packages/types/src/sdk/row.ts

@@ -17,6 +17,7 @@ export interface SearchParams {
   fields?: string[]
   indexer?: () => Promise<any>
   rows?: Row[]
+  countRows?: boolean
 }
 
 // when searching for rows we want a more extensive search type that requires certain properties