Base implementation of counting (plumbing).

packages/server/src/sdk/app/rows/search/sqs.ts

@@ -12,6 +12,7 @@ import {
   SortOrder,
   SortType,
   SqlClient,
+  SqlQuery,
   Table,
 } from "@budibase/types"
 import {
@@ -101,12 +102,29 @@ function buildTableMap(tables: Table[]) {
   return tableMap
 }
 
-async function runSqlQuery(json: QueryJson, tables: Table[]) {
+function runSqlQuery(json: QueryJson, tables: Table[]): Promise<Row[]>
+function runSqlQuery(
+  json: QueryJson,
+  tables: Table[],
+  opts: { countTotalRows: boolean }
+): Promise<number>
+async function runSqlQuery(
+  json: QueryJson,
+  tables: Table[],
+  opts?: { countTotalRows?: boolean }
+) {
   const alias = new AliasTables(tables.map(table => table.name))
-  return await alias.queryWithAliasing(json, async json => {
-    const query = builder._query(json, {
-      disableReturning: true,
-    })
+  const processSQLQuery = async (json: QueryJson) => {
+    let query: SqlQuery | SqlQuery[]
+    if (opts?.countTotalRows) {
+      query = builder._count(json, {
+        disableReturning: true,
+      })
+    } else {
+      query = builder._query(json, {
+        disableReturning: true,
+      })
+    }
 
     if (Array.isArray(query)) {
       throw new Error("SQS cannot currently handle multiple queries")
@@ -125,7 +143,12 @@ async function runSqlQuery(json: QueryJson, tables: Table[]) {
 
     const db = context.getAppDB()
     return await db.sql<Row>(sql, bindings)
-  })
+  }
+  if (opts?.countTotalRows) {
+    return await alias.countWithAliasing(json, processSQLQuery)
+  } else {
+    return await alias.queryWithAliasing(json, processSQLQuery)
+  }
 }
 
 export async function search(
@@ -204,8 +227,11 @@ export async function search(
     )
 
     // check for pagination final row
-    let nextRow: Row | undefined
+    let nextRow: Row | undefined, rowCount: number | undefined
     if (paginate && params.limit && processed.length > params.limit) {
+      // get the total count of rows
+      rowCount = await runSqlQuery(request, allTables, { countTotalRows: true })
+      // remove the extra row that confirmed if there is another row to move to
       nextRow = processed.pop()
     }
 
@@ -226,14 +252,10 @@ export async function search(
       const response: SearchResponse<Row> = {
         rows: finalRows,
       }
-      const prevLimit = request.paginate!.limit
-      request.paginate = {
-        limit: 1,
-        page: bookmark * prevLimit + 1,
-      }
       const hasNextPage = !!nextRow
       response.hasNextPage = hasNextPage
       if (hasNextPage) {
+        response.totalRows = rowCount
         response.bookmark = bookmark + 1
       }
       return response

packages/server/src/sdk/app/rows/sqlAlias.ts

@@ -65,7 +65,7 @@ export default class AliasTables {
     this.charSeq = new CharSequence()
   }
 
-  isAliasingEnabled(json: QueryJson, datasource: Datasource) {
+  isAliasingEnabled(json: QueryJson, datasource?: Datasource) {
     const operation = json.endpoint.operation
     const fieldLength = json.resource?.fields?.length
     if (
@@ -75,6 +75,10 @@ export default class AliasTables {
     ) {
       return false
     }
+    // SQS - doesn't have a datasource
+    if (!datasource) {
+      return true
+    }
     try {
       const sqlClient = getSQLClient(datasource)
       const isWrite = WRITE_OPERATIONS.includes(operation)
@@ -173,7 +177,7 @@ export default class AliasTables {
     const isSqs = datasourceId === SQS_DATASOURCE_INTERNAL
     let aliasingEnabled: boolean, datasource: Datasource | undefined
     if (isSqs) {
-      aliasingEnabled = true
+      aliasingEnabled = this.isAliasingEnabled(json)
     } else {
       datasource = await datasources.get(datasourceId)
       aliasingEnabled = this.isAliasingEnabled(json, datasource)
@@ -239,4 +243,17 @@ export default class AliasTables {
       return response
     }
   }
+
+  // handles getting the count out of the query
+  async countWithAliasing(
+    json: QueryJson,
+    queryFn?: (json: QueryJson) => Promise<DatasourcePlusQueryResponse>
+  ): Promise<number> {
+    let response = await this.queryWithAliasing(json, queryFn)
+    if (response && response.length === 1 && "total" in response[0]) {
+      return response[0].total
+    } else {
+      throw new Error("Unable to count rows in query - no count response")
+    }
+  }
 }

packages/types/src/sdk/datasources.ts

@@ -8,6 +8,7 @@ export enum Operation {
   READ = "READ",
   UPDATE = "UPDATE",
   DELETE = "DELETE",
+  COUNT = "COUNT",
   BULK_CREATE = "BULK_CREATE",
   CREATE_TABLE = "CREATE_TABLE",
   UPDATE_TABLE = "UPDATE_TABLE",

yarn.lock

@@ -3495,10 +3495,10 @@
   dependencies:
     lodash "^4.17.21"
 
-"@koa/cors@^3.1.0":
-  version "3.4.3"
-  resolved "https://registry.yarnpkg.com/@koa/cors/-/cors-3.4.3.tgz#d669ee6e8d6e4f0ec4a7a7b0a17e7a3ed3752ebb"
-  integrity sha512-WPXQUaAeAMVaLTEFpoq3T2O1C+FstkjJnDQqy95Ck1UdILajsRhu6mhJ8H2f4NFPRBoCNN+qywTJfq/gGki5mw==
+"@koa/cors@^5.0.0":
+  version "5.0.0"
+  resolved "https://registry.yarnpkg.com/@koa/cors/-/cors-5.0.0.tgz#0029b5f057fa0d0ae0e37dd2c89ece315a0daffd"
+  integrity sha512-x/iUDjcS90W69PryLDIMgFyV21YLTnG9zOpPXS7Bkt2b8AsY3zZsIpOLBkYr9fBcF3HbkKaER5hOBZLfpLgYNw==
   dependencies:
     vary "^1.1.2"
 
@@ -5817,10 +5817,10 @@
     "@types/koa-compose" "*"
     "@types/node" "*"
 
-"@types/koa__cors@^3.1.1":
-  version "3.3.1"
-  resolved "https://registry.yarnpkg.com/@types/koa__cors/-/koa__cors-3.3.1.tgz#0ec7543c4c620fd23451bfdd3e21b9a6aadedccd"
-  integrity sha512-aFGYhTFW7651KhmZZ05VG0QZJre7QxBxDj2LF1lf6GA/wSXEfKVAJxiQQWzRV4ZoMzQIO8vJBXKsUcRuvYK9qw==
+"@types/koa__cors@^5.0.0":
+  version "5.0.0"
+  resolved "https://registry.yarnpkg.com/@types/koa__cors/-/koa__cors-5.0.0.tgz#74567a045b599266e2cd3940cef96cedecc2ef1f"
+  integrity sha512-LCk/n25Obq5qlernGOK/2LUwa/2YJb2lxHUkkvYFDOpLXlVI6tKcdfCHRBQnOY4LwH6el5WOLs6PD/a8Uzau6g==
   dependencies:
     "@types/koa" "*"
 
@@ -16343,10 +16343,10 @@ node-source-walk@^5.0.0:
   dependencies:
     "@babel/parser" "^7.0.0"
 
-nodemailer@6.7.2:
-  version "6.7.2"
-  resolved "https://registry.yarnpkg.com/nodemailer/-/nodemailer-6.7.2.tgz#44b2ad5f7ed71b7067f7a21c4fedabaec62b85e0"
-  integrity sha512-Dz7zVwlef4k5R71fdmxwR8Q39fiboGbu3xgswkzGwczUfjp873rVxt1O46+Fh0j1ORnAC6L9+heI8uUpO6DT7Q==
+nodemailer@6.9.13:
+  version "6.9.13"
+  resolved "https://registry.yarnpkg.com/nodemailer/-/nodemailer-6.9.13.tgz#5b292bf1e92645f4852ca872c56a6ba6c4a3d3d6"
+  integrity sha512-7o38Yogx6krdoBf3jCAqnIN4oSQFx+fMa0I7dK1D+me9kBxx12D+/33wSb+fhOCtIxvYJ+4x4IMEhmhCKfAiOA==
 
 nodemailer@6.9.9:
   version "6.9.9"