From b7d7923904583b74e356c9b95fb7f3c47ace5392 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <aptkingston@ssl-mail.com>
Date: Wed, 5 Jan 2022 17:28:57 +0000
Subject: [PATCH] Add support for cursor based pagination in query params

---
 packages/client/src/utils/fetch/QueryFetch.js |  7 ++++---
 packages/server/src/integrations/rest.ts      | 14 ++++++++++++--
 packages/server/src/threads/query.js          |  6 ++++--
 3 files changed, 20 insertions(+), 7 deletions(-)

diff --git a/packages/client/src/utils/fetch/QueryFetch.js b/packages/client/src/utils/fetch/QueryFetch.js
index 08155bb94a..5176b11c25 100644
--- a/packages/client/src/utils/fetch/QueryFetch.js
+++ b/packages/client/src/utils/fetch/QueryFetch.js
@@ -35,7 +35,7 @@ export default class QueryFetch extends DataFetch {
     // Add pagination to query if supported
     let queryPayload = { queryId: datasource?._id, parameters }
     if (supportsPagination) {
-      const requestCursor = type === "page" ? parseInt(cursor || 0) : cursor
+      const requestCursor = type === "page" ? parseInt(cursor || 1) : cursor
       queryPayload.pagination = { page: requestCursor, limit }
     }
 
@@ -49,11 +49,12 @@ export default class QueryFetch extends DataFetch {
       if (type === "page") {
         // For "page number" pagination, increment the existing page number
         nextCursor = queryPayload.pagination.page + 1
+        hasNextPage = data?.length === limit && limit > 0
       } else {
         // For "cursor" pagination, the cursor should be in the response
-        nextCursor = pagination.cursor
+        nextCursor = pagination?.cursor
+        hasNextPage = nextCursor != null
       }
-      hasNextPage = data?.length === limit && limit > 0
     }
 
     return {
diff --git a/packages/server/src/integrations/rest.ts b/packages/server/src/integrations/rest.ts
index ad8f46e5f5..024eee14fe 100644
--- a/packages/server/src/integrations/rest.ts
+++ b/packages/server/src/integrations/rest.ts
@@ -120,7 +120,7 @@ module RestModule {
       this.config = config
     }
 
-    async parseResponse(response: any) {
+    async parseResponse(response: any, pagination: PaginationConfig | null) {
       let data, raw, headers
       const contentType = response.headers.get("content-type") || ""
       try {
@@ -159,6 +159,13 @@ module RestModule {
       for (let [key, value] of Object.entries(headers)) {
         headers[key] = Array.isArray(value) ? value[0] : value
       }
+
+      // Check if a pagination cursor exists in the response
+      let nextCursor = null
+      if (pagination?.responseParam) {
+        nextCursor = data?.[pagination.responseParam]
+      }
+
       return {
         data,
         info: {
@@ -170,6 +177,9 @@ module RestModule {
           raw,
           headers,
         },
+        pagination: {
+          cursor: nextCursor
+        }
       }
     }
 
@@ -325,7 +335,7 @@ module RestModule {
       this.startTimeMs = performance.now()
       const url = this.getUrl(path, queryString, pagination, paginationValues)
       const response = await fetch(url, input)
-      return await this.parseResponse(response)
+      return await this.parseResponse(response, pagination)
     }
 
     async create(opts: RestQuery) {
diff --git a/packages/server/src/threads/query.js b/packages/server/src/threads/query.js
index 8232321084..ad461008df 100644
--- a/packages/server/src/threads/query.js
+++ b/packages/server/src/threads/query.js
@@ -47,11 +47,13 @@ class QueryRunner {
     let output = threadUtils.formatResponse(await integration[queryVerb](query))
     let rows = output,
       info = undefined,
-      extra = undefined
+      extra = undefined,
+      pagination = undefined
     if (threadUtils.hasExtraData(output)) {
       rows = output.data
       info = output.info
       extra = output.extra
+      pagination = output.pagination
     }
 
     // transform as required
@@ -90,7 +92,7 @@ class QueryRunner {
       integration.end()
     }
 
-    return { rows, keys, info, extra }
+    return { rows, keys, info, extra, pagination }
   }
 
   async runAnotherQuery(queryId, parameters) {