Add support for query param based pagination in REST queries

packages/client/src/components/app/DataProvider.svelte

@@ -134,7 +134,7 @@
       {:else}
         <slot />
       {/if}
-      {#if paginate && dataSource?.type === "table"}
+      {#if paginate && $fetch.supportsPagination}
         <div class="pagination">
           <Pagination
             page={$fetch.pageNumber + 1}

packages/client/src/utils/fetch/DataFetch.js

@@ -14,9 +14,11 @@ import { fetchTableDefinition } from "api"
  */
 export default class DataFetch {
   // Feature flags
-  supportsSearch = false
+  featureStore = writable({
-  supportsSort = false
+    supportsSearch: false,
-  supportsPagination = false
+    supportsSort: false,
+    supportsPagination: false,
+  })
 
   // Config
   options = {
@@ -74,13 +76,17 @@ export default class DataFetch {
     this.prevPage = this.prevPage.bind(this)
 
     // Derive certain properties to return
-    this.derivedStore = derived(this.store, $store => {
+    this.derivedStore = derived(
+      [this.store, this.featureStore],
+      ([$store, $featureStore]) => {
         return {
           ...$store,
+          ...$featureStore,
           hasNextPage: this.hasNextPage($store),
           hasPrevPage: this.hasPrevPage($store),
         }
-    })
+      }
+    )
 
     // Mark as loaded if we have no datasource
     if (!this.options.datasource) {
@@ -114,7 +120,12 @@ export default class DataFetch {
 
     // Fetch datasource definition and determine feature flags
     const definition = await this.constructor.getDefinition(datasource)
-    this.determineFeatureFlags(definition)
+    const features = this.determineFeatureFlags(definition)
+    this.featureStore.set({
+      supportsSearch: !!features?.supportsSearch,
+      supportsSort: !!features?.supportsSort,
+      supportsPagination: !!features?.supportsPagination,
+    })
 
     // Fetch and enrich schema
     let schema = this.constructor.getSchema(datasource, definition)
@@ -167,22 +178,23 @@ export default class DataFetch {
   async getPage() {
     const { sortColumn, sortOrder, sortType, limit } = this.options
     const { query } = get(this.store)
+    const features = get(this.featureStore)
 
     // Get the actual data
     let { rows, info, hasNextPage, cursor } = await this.getData()
 
     // If we don't support searching, do a client search
-    if (!this.supportsSearch) {
+    if (!features.supportsSearch) {
       rows = luceneQuery(rows, query)
     }
 
     // If we don't support sorting, do a client-side sort
-    if (!this.supportsSort) {
+    if (!features.supportsSort) {
       rows = luceneSort(rows, sortColumn, sortOrder, sortType)
     }
 
     // If we don't support pagination, do a client-side limit
-    if (!this.supportsPagination) {
+    if (!features.supportsPagination) {
       rows = luceneLimit(rows, limit)
     }
 
@@ -263,9 +275,11 @@ export default class DataFetch {
    */
   // eslint-disable-next-line no-unused-vars
   determineFeatureFlags(definition) {
-    this.supportsSearch = false
+    return {
-    this.supportsSort = false
+      supportsSearch: false,
-    this.supportsPagination = false
+      supportsSort: false,
+      supportsPagination: false,
+    }
   }
 
   /**

packages/client/src/utils/fetch/QueryFetch.js

@@ -5,10 +5,10 @@ import { get } from "svelte/store"
 
 export default class QueryFetch extends DataFetch {
   determineFeatureFlags(definition) {
-    console.log("pagination config", definition?.fields?.pagination)
+    const supportsPagination =
-    this.supportsPagination =
       definition?.fields?.pagination?.type != null &&
       definition?.fields?.pagination?.pageParam != null
+    return { supportsPagination }
   }
 
   static async getDefinition(datasource) {
@@ -20,6 +20,9 @@ export default class QueryFetch extends DataFetch {
 
   async getData() {
     const { datasource, limit } = this.options
+    const { supportsPagination } = get(this.featureStore)
+    const { cursor, definition } = get(this.store)
+    const { type } = definition.fields.pagination
 
     // Set the default query params
     let parameters = cloneDeep(datasource?.queryParams || {})
@@ -31,19 +34,33 @@ export default class QueryFetch extends DataFetch {
 
     // Add pagination to query if supported
     let queryPayload = { queryId: datasource?._id, parameters }
-    if (this.supportsPagination) {
+    if (supportsPagination) {
-      const { cursor, definition, pageNumber } = get(this.store)
+      const requestCursor = type === "page" ? parseInt(cursor || 0) : cursor
-      const { type } = definition.fields.pagination
+      queryPayload.pagination = { page: requestCursor, limit }
-      const page = type === "page" ? pageNumber : cursor
-      queryPayload.pagination = { page, limit }
     }
 
+    // Execute query
     const { data, pagination, ...rest } = await executeQuery(queryPayload)
+
+    // Derive pagination info from response
+    let nextCursor = null
+    let hasNextPage = false
+    if (supportsPagination) {
+      if (type === "page") {
+        // For "page number" pagination, increment the existing page number
+        nextCursor = queryPayload.pagination.page + 1
+      } else {
+        // For "cursor" pagination, the cursor should be in the response
+        nextCursor = pagination.cursor
+      }
+      hasNextPage = data?.length === limit && limit > 0
+    }
+
     return {
       rows: data || [],
       info: rest,
-      cursor: pagination?.page,
+      cursor: nextCursor,
-      hasNextPage: data?.length === limit && limit > 0,
+      hasNextPage,
     }
   }
 }

packages/client/src/utils/fetch/TableFetch.js

@@ -4,9 +4,11 @@ import { searchTable } from "api"
 
 export default class TableFetch extends DataFetch {
   determineFeatureFlags() {
-    this.supportsSearch = true
+    return {
-    this.supportsSort = true
+      supportsSearch: true,
-    this.supportsPagination = true
+      supportsSort: true,
+      supportsPagination: true,
+    }
   }
 
   async getData() {

packages/server/src/api/controllers/query/index.js

@@ -145,6 +145,7 @@ async function execute(ctx, opts = { rowsOnly: false }) {
       queryVerb: query.queryVerb,
       fields: query.fields,
       parameters: ctx.request.body.parameter,
+      pagination: ctx.request.body.pagination,
       transformer: query.transformer,
       queryId: ctx.params.queryId,
     })

packages/server/src/definitions/datasource.ts

@@ -233,6 +233,7 @@ export interface RestQueryFields {
   method: string
   authConfigId: string
   pagination: PaginationConfig | null
+  paginationValues: PaginationValues | null
 }
 
 export interface RestConfig {
@@ -245,11 +246,17 @@ export interface RestConfig {
 
 export interface PaginationConfig {
   type: string
+  location: string
   pageParam: string
   sizeParam: string | null
   responseParam: string | null
 }
 
+export interface PaginationValues {
+  page: string | number | null
+  limit: number | null
+}
+
 export interface Query {
   _id?: string
   datasourceId: string

packages/server/src/integrations/rest.ts

@@ -4,9 +4,11 @@ import {
   QueryTypes,
   RestConfig,
   RestQueryFields as RestQuery,
+  PaginationConfig,
   AuthType,
   BasicAuthConfig,
   BearerAuthConfig,
+  PaginationValues,
 } from "../definitions/datasource"
 import { IntegrationBase } from "./base/IntegrationBase"
 
@@ -40,6 +42,9 @@ const coreFields = {
     type: DatasourceFieldTypes.STRING,
     enum: Object.values(BodyTypes),
   },
+  pagination: {
+    type: DatasourceFieldTypes.OBJECT
+  }
 }
 
 module RestModule {
@@ -165,7 +170,29 @@ module RestModule {
       }
     }
 
-    getUrl(path: string, queryString: string): string {
+    getUrl(path: string, queryString: string, pagination: PaginationConfig | null, paginationValues: PaginationValues | null): string {
+      // Add pagination params to query string if required
+      if (pagination?.location === "query" && paginationValues) {
+        const { pageParam, sizeParam } = pagination
+        const params = new URLSearchParams()
+
+        // Append page number or cursor param if configured
+        if (pageParam && paginationValues.page != null) {
+          params.append(pageParam, paginationValues.page)
+        }
+
+        // Append page size param if configured
+        if (sizeParam && paginationValues.limit != null) {
+          params.append(sizeParam, paginationValues.limit)
+        }
+
+        // Prepend query string with pagination params
+        let paginationString = params.toString()
+        if (paginationString) {
+          queryString = `${paginationString}&${queryString}`
+        }
+      }
+
       const main = `${path}?${queryString}`
       let complete = main
       if (this.config.url && !main.startsWith(this.config.url)) {
@@ -268,6 +295,8 @@ module RestModule {
         bodyType,
         requestBody,
         authConfigId,
+        pagination,
+        paginationValues
       } = query
       const authHeaders = this.getAuthHeaders(authConfigId)
 
@@ -291,7 +320,7 @@ module RestModule {
       }
 
       this.startTimeMs = performance.now()
-      const url = this.getUrl(path, queryString)
+      const url = this.getUrl(path, queryString, pagination, paginationValues)
       const response = await fetch(url, input)
       return await this.parseResponse(response)
     }

packages/server/src/threads/query.js

@@ -12,6 +12,7 @@ class QueryRunner {
     this.queryVerb = input.queryVerb
     this.fields = input.fields
     this.parameters = input.parameters
+    this.pagination = input.pagination
     this.transformer = input.transformer
     this.queryId = input.queryId
     this.noRecursiveQuery = flags.noRecursiveQuery
@@ -27,7 +28,7 @@ class QueryRunner {
     let { datasource, fields, queryVerb, transformer } = this
     // pre-query, make sure datasource variables are added to parameters
     const parameters = await this.addDatasourceVariables()
-    const query = threadUtils.enrichQueryFields(fields, parameters)
+    let query = this.enrichQueryFields(fields, parameters)
     const Integration = integrations[datasource.source]
     if (!Integration) {
       throw "Integration type does not exist."
@@ -156,6 +157,52 @@ class QueryRunner {
     }
     return parameters
   }
+
+  enrichQueryFields(fields, parameters = {}) {
+    const enrichedQuery = {}
+
+    // enrich the fields with dynamic parameters
+    for (let key of Object.keys(fields)) {
+      if (fields[key] == null) {
+        continue
+      }
+      if (typeof fields[key] === "object") {
+        // enrich nested fields object
+        enrichedQuery[key] = this.enrichQueryFields(fields[key], parameters)
+      } else if (typeof fields[key] === "string") {
+        // enrich string value as normal
+        enrichedQuery[key] = processStringSync(fields[key], parameters, {
+          noHelpers: true,
+        })
+      } else {
+        enrichedQuery[key] = fields[key]
+      }
+    }
+
+    if (
+      enrichedQuery.json ||
+      enrichedQuery.customData ||
+      enrichedQuery.requestBody
+    ) {
+      try {
+        enrichedQuery.json = JSON.parse(
+          enrichedQuery.json ||
+            enrichedQuery.customData ||
+            enrichedQuery.requestBody
+        )
+      } catch (err) {
+        // no json found, ignore
+      }
+      delete enrichedQuery.customData
+    }
+
+    // Just for REST queries
+    if (this.pagination) {
+      enrichedQuery.paginationValues = this.pagination
+    }
+
+    return enrichedQuery
+  }
 }
 
 module.exports = (input, callback) => {

packages/server/src/threads/utils.js

@@ -76,44 +76,3 @@ exports.hasExtraData = response => {
     response.info != null
   )
 }
-
-exports.enrichQueryFields = (fields, parameters = {}) => {
-  const enrichedQuery = {}
-
-  // enrich the fields with dynamic parameters
-  for (let key of Object.keys(fields)) {
-    if (fields[key] == null) {
-      continue
-    }
-    if (typeof fields[key] === "object") {
-      // enrich nested fields object
-      enrichedQuery[key] = this.enrichQueryFields(fields[key], parameters)
-    } else if (typeof fields[key] === "string") {
-      // enrich string value as normal
-      enrichedQuery[key] = processStringSync(fields[key], parameters, {
-        noHelpers: true,
-      })
-    } else {
-      enrichedQuery[key] = fields[key]
-    }
-  }
-
-  if (
-    enrichedQuery.json ||
-    enrichedQuery.customData ||
-    enrichedQuery.requestBody
-  ) {
-    try {
-      enrichedQuery.json = JSON.parse(
-        enrichedQuery.json ||
-          enrichedQuery.customData ||
-          enrichedQuery.requestBody
-      )
-    } catch (err) {
-      // no json found, ignore
-    }
-    delete enrichedQuery.customData
-  }
-
-  return enrichedQuery
-}