Merge pull request #79 from superglue-ai/feature/cursor-based-pagination

cursor based pagination

api.graphql

@@ -84,6 +84,7 @@ type RunResult {
 type Pagination {
   type: PaginationType!
   pageSize: String
+  cursorPath: String
 }
 
 type RunList {
@@ -141,6 +142,7 @@ enum CacheMode {
 enum PaginationType {
   OFFSET_BASED
   PAGE_BASED
+  CURSOR_BASED
   DISABLED
 }
 
@@ -154,6 +156,7 @@ enum FileType {
 input PaginationInput {
   type: PaginationType!
   pageSize: Int
+  cursorPath: String
 }
 
 input ApiInput {

packages/core/utils/api.test.ts

@@ -195,6 +195,126 @@ describe('API Utilities', () => {
       );
     });
 
+    it('should handle cursor-based pagination', async () => {
+      const config = {
+        ...testConfig,
+        dataPath: 'data',
+        pagination: {
+          type: PaginationType.CURSOR_BASED,
+          pageSize: 2,
+          cursorPath: 'meta.next_cursor'
+        }
+      } as ApiConfig;
+
+      const mockResponses = [
+        { 
+          status: 200, 
+          data: { 
+            data: [{ id: 1 }, { id: 2 }],
+            meta: { next_cursor: 'cursor123' }
+          },
+          statusText: 'OK',
+          headers: {},
+          config: {} as any 
+        },
+        { 
+          status: 200, 
+          data: { 
+            data: [{ id: 3 }],
+            meta: { next_cursor: null }
+          },
+          statusText: 'OK',
+          headers: {},
+          config: {} as any 
+        }
+      ];
+
+      mockedTools.callAxios
+        .mockResolvedValueOnce(mockResponses[0])
+        .mockResolvedValueOnce(mockResponses[1]);
+
+      const result = await callEndpoint(config, {}, {}, {});
+
+      expect(result.data).toHaveLength(3);
+    });
+
+    it('should stop pagination when receiving duplicate data', async () => {
+      const config = {
+        ...testConfig,
+        pagination: {
+          type: PaginationType.PAGE_BASED,
+          pageSize: 2
+        }
+      } as ApiConfig;
+
+      const sameResponse = { 
+        status: 200, 
+        data: [{ id: 1 }, { id: 2 }],
+        statusText: 'OK',
+        headers: {},
+        config: {} as any 
+      };
+
+      mockedTools.callAxios
+        .mockResolvedValueOnce(sameResponse)
+        .mockResolvedValueOnce(sameResponse); // Same data returned
+
+      const result = await callEndpoint(config, {}, {}, {});
+
+      expect(result.data).toHaveLength(2); // Should only include unique data
+      expect(mockedTools.callAxios).toHaveBeenCalledTimes(2);
+    });
+
+    it('should stop after 500 iterations', async () => {
+      const config = {
+        ...testConfig,
+        pagination: {
+          type: PaginationType.OFFSET_BASED,
+          pageSize: 1
+        }
+      } as ApiConfig;
+
+      // Mock 501 responses to test the loop limit
+      const mockResponse = { 
+        status: 200, 
+        statusText: 'OK',
+        headers: {},
+        config: {} as any 
+      };
+      for(let i = 0; i < 505; i++) {
+        mockedTools.callAxios.mockResolvedValueOnce({...mockResponse, data: [{ id: i }]});
+      }
+      const result = await callEndpoint(config, {}, {}, {});
+      // Should stop at 500 iterations (as defined in the code)
+      expect(mockedTools.callAxios).toHaveBeenCalledTimes(500);
+    });
+
+    it('if 2 responses are the same, stop pagination', async () => {
+      const config = {
+        ...testConfig,
+        pagination: {
+          type: PaginationType.OFFSET_BASED,
+          pageSize: 1
+        }
+      } as ApiConfig;
+
+      // Mock 501 responses to test the loop limit
+      const mockResponse = { 
+        status: 200, 
+        data: [{ id: 1 }],
+        statusText: 'OK',
+        headers: {},
+        config: {} as any 
+      };
+
+      mockedTools.callAxios.mockResolvedValue(mockResponse);
+
+      const result = await callEndpoint(config, {}, {}, {});
+
+      // Should stop at 500 iterations (as defined in the code)
+      expect(mockedTools.callAxios).toHaveBeenCalledTimes(2);
+    });
+
     it('should handle error responses', async () => {
       const errorResponse = { 
         status: 400, 

packages/core/utils/api.ts

@@ -63,21 +63,26 @@ export async function callEndpoint(endpoint: ApiConfig, payload: Record<string,
   let allResults = [];
   let page = 1;
   let offset = 0;
+  let cursor = null;
   let hasMore = true;
   let loopCounter = 0;
 
-  while (hasMore && loopCounter <= 500) {
+  while (hasMore && loopCounter < 500) {
     // Generate pagination variables if enabled
     let paginationVars = {};
-    if (endpoint.pagination?.type === PaginationType.PAGE_BASED) {
-      paginationVars = { page, limit: endpoint.pagination?.pageSize || 50 };
-      page++;
-    } else if (endpoint.pagination?.type === PaginationType.OFFSET_BASED) {
-      paginationVars = { offset, limit: endpoint.pagination?.pageSize || 50 };
-      offset += endpoint.pagination?.pageSize || 50;
-    }
-    else {
-      hasMore = false;
+    switch (endpoint.pagination?.type) {
+      case PaginationType.PAGE_BASED:
+        paginationVars = { page, limit: endpoint.pagination?.pageSize || 50 };
+        break;
+      case PaginationType.OFFSET_BASED:
+        paginationVars = { offset, limit: endpoint.pagination?.pageSize || 50 };
+        break;
+      case PaginationType.CURSOR_BASED:
+        paginationVars = { cursor: cursor, limit: endpoint.pagination?.pageSize || 50 };
+        break;
+      default:
+        hasMore = false;
+        break;
     }
 
     // Combine all variables
@@ -157,10 +162,12 @@ export async function callEndpoint(endpoint: ApiConfig, payload: Record<string,
         This usually indicates an error page or invalid endpoint.\nResponse: ${response.data.slice(0, 2000)}`);
     }
 
-    let responseData = response.data;
     let dataPathSuccess = true;
-    if (endpoint.dataPath) {
 
+    // TODO: we need to remove the data path and just join the data with the next page of data, otherwise we will have to do a lot of gymnastics to get the data path right
+
+    let responseData = response.data;
+    if (endpoint.dataPath) {
       // Navigate to the specified data path
       const pathParts = endpoint.dataPath.split('.');
 
@@ -179,7 +186,7 @@ export async function callEndpoint(endpoint: ApiConfig, payload: Record<string,
       if(responseData.length < endpoint.pagination?.pageSize) {
         hasMore = false;
       }
-      
+
       if(JSON.stringify(responseData) !== JSON.stringify(allResults)) {
         allResults = allResults.concat(responseData);
       }
@@ -194,6 +201,25 @@ export async function callEndpoint(endpoint: ApiConfig, payload: Record<string,
     else {
       hasMore = false;
     }
+
+    // update pagination
+    if(endpoint.pagination?.type === PaginationType.PAGE_BASED) {
+      page++;
+    }
+    else if(endpoint.pagination?.type === PaginationType.OFFSET_BASED) {
+      offset += endpoint.pagination?.pageSize || 50;
+    }
+    else if (endpoint.pagination?.type === PaginationType.CURSOR_BASED) {
+        const cursorParts = (endpoint.pagination?.cursorPath || 'next_cursor').split('.');
+        let nextCursor = response.data;
+        for (const part of cursorParts) {
+          nextCursor = nextCursor?.[part] ?? nextCursor;
+        }
+        cursor = nextCursor;
+        if(!cursor) {
+          hasMore = false;
+        }
+    }
     loopCounter++;
   }
 
@@ -221,6 +247,7 @@ async function generateApiConfig(
     pagination: z.object({
       type: z.enum(Object.values(PaginationType) as [string, ...string[]]),
       pageSize: z.string().describe("Number of items per page. Set this to a number. In headers or query params, you can access it as {limit}."),
+      cursorPath: z.string().optional().describe("If cursor_based: The path to the cursor in the response. E.g. cursor.current or next_cursor")
     }).optional()
   }));
   const openai = new OpenAI({
@@ -317,8 +344,8 @@ Documentation: ${String(documentation)}`
       headers: generatedConfig.headers,
       body: generatedConfig.body,
       authentication: generatedConfig.authentication,
-      pagination: apiConfig.pagination || generatedConfig.pagination,
-      dataPath: apiConfig.dataPath || generatedConfig.dataPath,
+      pagination: generatedConfig.pagination,
+      dataPath: generatedConfig.dataPath,
       documentationUrl: apiConfig.documentationUrl,
       responseSchema: apiConfig.responseSchema,
       responseMapping: apiConfig.responseMapping,
@@ -335,7 +362,8 @@ function validateVariables(generatedConfig: any, vars: string[]) {
     ...vars,
     "page",
     "limit",
-    "offset"
+    "offset",
+    "cursor"
   ]
 
   // Helper function to find only template variables in a string

packages/shared/types.ts

@@ -49,6 +49,7 @@ export enum DecompressionMethod {
 export enum PaginationType {
   OFFSET_BASED = "OFFSET_BASED",
   PAGE_BASED = "PAGE_BASED",
+  CURSOR_BASED = "CURSOR_BASED",
   DISABLED = "DISABLED"
 }
 
@@ -110,6 +111,7 @@ export interface TransformConfig extends BaseConfig {
 export type Pagination = {
   type: PaginationType;
   pageSize?: number;
+  cursorPath?: string;
 };
 
 export type RunResult = BaseResult & {

packages/web/src/components/ApiConfigForm.tsx

@@ -60,7 +60,7 @@ import React from 'react';
 
 const HTTP_METHODS = ['GET', 'POST', 'PUT', 'DELETE', 'PATCH', 'HEAD', 'OPTIONS'];
 const AUTH_TYPES = ['NONE', 'HEADER', 'QUERY_PARAM', 'OAUTH2'];
-const PAGINATION_TYPES = ['OFFSET_BASED', 'PAGE_BASED', 'DISABLED'];
+const PAGINATION_TYPES = ['OFFSET_BASED', 'PAGE_BASED', 'CURSOR_BASED', 'DISABLED'];
 
 const InfoTooltip = ({ text }: { text: string }) => (
   <TooltipProvider delayDuration={100}>