From 85b3e3de683f06393e5c493f1dfd82ddbe8b030d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sven=20H=C3=B6ffler?= <sven@flowmate.io>
Date: Tue, 18 Mar 2025 13:54:48 +0100
Subject: [PATCH] Added safeguard against emitting identical data to paginators

---
 templates/lib/utils/paginator.js      |  25 +++++++
 templates/lib/utils/paginator.test.js | 101 +++++++++++++++++++++++++-
 2 files changed, 125 insertions(+), 1 deletion(-)

diff --git a/templates/lib/utils/paginator.js b/templates/lib/utils/paginator.js
index 4752bd2..42257ca 100644
--- a/templates/lib/utils/paginator.js
+++ b/templates/lib/utils/paginator.js
@@ -1,4 +1,5 @@
 const lodashGet = require("lodash.get");
+const crypto = require('crypto');
 
 module.exports.createPaginator = function createPaginator(config) {
   if (config.strategy.type === "cursor") {
@@ -14,9 +15,17 @@ module.exports.createPaginator = function createPaginator(config) {
 class CursorPaginator {
   constructor(config) {
     this.config = config;
+    this.lastPageHash = "";
   }
 
   hasNextPage({ headers, body }) {
+    const hashedBody = crypto.createHash('sha256').update(JSON.stringify(body), 'utf8').digest('hex');
+    if (hashedBody === this.lastPageHash) {
+      console.warn("Received same data twice in a row, aborting pagination...")
+      return false;
+    }
+    this.lastPageHash = hashedBody;
+
     return !!this.getNextPageToken({ headers, body });
   }
 
@@ -31,9 +40,17 @@ class PageIncrementPaginator {
 
   constructor(config) {
     this.config = config;
+    this.lastPageHash = "";
   }
 
   hasNextPage({ body }) {
+    const hashedBody = crypto.createHash('sha256').update(JSON.stringify(body), 'utf8').digest('hex');
+    if (hashedBody === this.lastPageHash) {
+      console.warn("Received same data twice in a row, aborting pagination...")
+      return false;
+    }
+    this.lastPageHash = hashedBody;
+
     const resultsPath = [];
     if (this.config.strategy.resultsPath) {
       resultsPath.push(...this.config.strategy.resultsPath.split('.'));
@@ -53,9 +70,17 @@ class OffsetIncrementPaginator {
 
   constructor(config) {
     this.config = config;
+    this.lastPageHash = "";
   }
 
   hasNextPage({ body }) {
+    const hashedBody = crypto.createHash('sha256').update(JSON.stringify(body), 'utf8').digest('hex');
+    if (hashedBody === this.lastPageHash) {
+      console.warn("Received same data twice in a row, aborting pagination...")
+      return false;
+    }
+    this.lastPageHash = hashedBody;
+
     const resultsPath = [];
     if (this.config.strategy.resultsPath) {
       resultsPath.push(...this.config.strategy.resultsPath.split('.'));
diff --git a/templates/lib/utils/paginator.test.js b/templates/lib/utils/paginator.test.js
index 5a1a34c..eac4c74 100644
--- a/templates/lib/utils/paginator.test.js
+++ b/templates/lib/utils/paginator.test.js
@@ -103,6 +103,36 @@ describe("Paginators", () => {
       expect(paginator.hasNextPage(response)).toBeTruthy();
       expect(paginator.getNextPageToken(response)).toBe("next-page-token");
     });
+
+    it("should abort when receiving the same data twice in a row", () => {
+      const paginator = new CursorPaginator({
+        pageTokenOption: {
+          fieldName: "after",
+        },
+        strategy: {
+          type: "cursor",
+          tokenIn: "body",
+          nextCursorPath: "meta.next.token",
+        },
+      });
+
+      const response = {
+        headers: {},
+        body: {
+          data: ['abc', 'def'],
+          meta: {
+            next: {
+              token: "next-page-token",
+            },
+          },
+        },
+      };
+
+      expect(paginator.hasNextPage(response)).toBeTruthy();
+      expect(paginator.getNextPageToken(response)).toBe("next-page-token");
+
+      expect(paginator.hasNextPage(response)).toBeFalsy();
+    });
   });
 
   describe("PageIncrementPaginator", () => {
@@ -181,6 +211,40 @@ describe("Paginators", () => {
       expect(paginator.getNextPageToken(response)).toBe(2);
       expect(paginator.getNextPageToken(response)).toBe(3);
     });
+
+    it("should abort when receiving the same data twice in a row", () => {
+      const paginator = createPaginator({
+        pageSizeOption: {
+          fieldName: "per_page",
+        },
+        pageTokenOption: {
+          fieldName: "page",
+        },
+        strategy: {
+          type: "page_increment",
+          pageSize: 2,
+          resultsPath: "results",
+        },
+      });
+
+      const response1 = {
+        body: {
+          results: [{ id: 1 }, { id: 2 }],
+        },
+        headers: {},
+      };
+
+      const response2 = {
+        body: {
+          results: [{ id: 3 }, { id: 4 }],
+        },
+        headers: {},
+      };
+
+      expect(paginator.hasNextPage(response1)).toBeTruthy();
+      expect(paginator.hasNextPage(response2)).toBeTruthy();
+      expect(paginator.hasNextPage(response2)).toBeFalsy();
+    });
   });
 
   describe("OffsetIncrementPaginator", () => {
@@ -229,7 +293,7 @@ describe("Paginators", () => {
       const response = {
         body: {
           d: {
-            results: [{id: 1}, {id: 2}]
+            results: [{ id: 1 }, { id: 2 }]
           },
         },
         headers: {},
@@ -288,6 +352,41 @@ describe("Paginators", () => {
       expect(paginator.getNextPageToken(response)).toBe(2);
       expect(paginator.getNextPageToken(response)).toBe(4);
     });
+
+    it("should when receiving the same data twice in a row", () => {
+      const paginator = createPaginator({
+        pageSizeOption: {
+          fieldName: "per_page",
+        },
+        pageTokenOption: {
+          fieldName: "page",
+        },
+        strategy: {
+          type: "offset_increment",
+          pageSize: 2,
+          resultsPath: "results",
+        },
+      });
+
+      const response1 = {
+        body: {
+          results: [{ id: 1 }, { id: 2 }],
+        },
+        headers: {},
+      };
+
+      const response2 = {
+        body: {
+          results: [{ id: 3 }, { id: 4 }],
+        },
+        headers: {},
+      };
+
+      expect(paginator.hasNextPage(response1)).toBeTruthy();
+      expect(paginator.hasNextPage(response2)).toBeTruthy();
+      expect(paginator.hasNextPage(response2)).toBeFalsy();
+    });
+
   });
 
   describe("NoPagingPaginator", () => {