Merge pull request #81 from superglue-ai/bugfix/cursor-based-pagination

add the cursor to the response
superglue-ai · Mar 6, 2025 · 526fe17 · 526fe17
2 parents 6466e36 + 037a3c8
commit 526fe17
Show file tree

Hide file tree

Showing 2 changed files with 12 additions and 2 deletions.
diff --git a/packages/core/utils/api.test.ts b/packages/core/utils/api.test.ts
@@ -235,7 +235,8 @@ describe('API Utilities', () => {
 
       const result = await callEndpoint(config, {}, {}, {});
 
-      expect(result.data).toHaveLength(3);
+      expect(result.data.results).toHaveLength(3);
+      expect(result.data.next_cursor).toBeNull();
     });
 
     it('should stop pagination when receiving duplicate data', async () => {

diff --git a/packages/core/utils/api.ts b/packages/core/utils/api.ts
@@ -213,7 +213,7 @@ export async function callEndpoint(endpoint: ApiConfig, payload: Record<string,
         const cursorParts = (endpoint.pagination?.cursorPath || 'next_cursor').split('.');
         let nextCursor = response.data;
         for (const part of cursorParts) {
-          nextCursor = nextCursor?.[part] ?? nextCursor;
+          nextCursor = nextCursor?.[part];
         }
         cursor = nextCursor;
         if(!cursor) {
@@ -223,6 +223,15 @@ export async function callEndpoint(endpoint: ApiConfig, payload: Record<string,
     loopCounter++;
   }
 
+  if(endpoint.pagination?.type === PaginationType.CURSOR_BASED) {
+    return {
+      data: {
+        results: allResults,
+        next_cursor: cursor
+      }
+    };
+  }
+
   return {
     data: allResults?.length == 1 ? allResults[0] : allResults
   };