Merging tag specific checks

apps/database-abstractor/src/main/java/com/akto/merging/MergingLogic.java

@@ -9,6 +9,7 @@
 import com.akto.dao.SampleDataDao;
 import com.akto.dao.SingleTypeInfoDao;
 import com.akto.dao.context.Context;
+import com.akto.util.Constants;
 import com.akto.dto.ApiCollection;
 import com.akto.dto.ApiInfo;
 import com.akto.dto.traffic.SampleData;
@@ -239,6 +240,12 @@ public static ApiMergerResult tryMergeURLsInCollection(int apiCollectionId, bool
                     LogDb.DB_ABS);
                 return new ApiMergerResult(new HashMap<>());
             }
+            if (ApiCollectionsDao.hasRoutingTags(apiCollection)) {
+                loggerMaker.infoAndAddToDb(
+                    "Skipping merging for API collection " + apiCollectionId + " in account " + Constants.ROUTING_SKIP_ACCOUNT_ID + " as it has a tag ending with one of: " + Constants.ROUTING_TAG_SUFFIXES,
+                    LogDb.DB_ABS);
+                return new ApiMergerResult(new HashMap<>());
+            }
         }
 
         Bson filterQ = null;

libs/dao/src/main/java/com/akto/dao/ApiCollectionsDao.java

@@ -265,5 +265,25 @@ public static List<BasicDBObject> fetchEndpointsInCollectionUsingHost(int apiCol
 
             return endpoints;
         }
-    }    
+    }
+
+    /**
+     * Checks if an API collection has tags ending with routing suffixes for a specific account.
+     * @param apiCollection The API collection to check
+     * @return true if collection has routing tags and account matches, false otherwise
+     */
+    public static boolean hasRoutingTags(ApiCollection apiCollection) {
+        if (apiCollection == null || apiCollection.getTagsList() == null) {
+            return false;
+        }
+
+        // Only check for specific account
+        if (Context.accountId.get() != Constants.ROUTING_SKIP_ACCOUNT_ID) {
+            return false;
+        }
+
+        return apiCollection.getTagsList().stream()
+            .anyMatch(t -> t.getValue() != null &&
+                Constants.ROUTING_TAG_SUFFIXES.stream().anyMatch(suffix -> t.getValue().endsWith(suffix)));
+    }
 }

libs/dao/src/main/java/com/akto/util/Constants.java

@@ -1,5 +1,8 @@
 package com.akto.util;
 
+import java.util.Arrays;
+import java.util.List;
+
 public class Constants {
     private Constants() {}
 
@@ -29,4 +32,8 @@ private Constants() {}
     public static final String STATUS_IN_PROGRESS = "In Progress";
     public static final String STATUS_COMPLETED = "Completed";
     public static final String STATUS_FAILED = "Failed";
+
+    // Constants for routing tag filtering
+    public static final int ROUTING_SKIP_ACCOUNT_ID = 1736798101;
+    public static final List<String> ROUTING_TAG_SUFFIXES = Arrays.asList("booking-creation", "booking-query", "cashout-api", "ebe-creditcard-service-netcore", "partner-credit-card-api");
 }

libs/utils/src/main/java/com/akto/data_actor/DbLayer.java

@@ -100,6 +100,7 @@
 import com.akto.dto.traffic.TrafficInfo;
 import com.akto.dto.traffic_metrics.RuntimeMetrics;
 import com.akto.dto.traffic_metrics.TrafficMetrics;
+import com.akto.dto.type.APICatalog;
 import com.akto.dto.type.SingleTypeInfo;
 import com.akto.dto.type.URLMethods;
 import com.akto.dto.type.URLMethods.Method;
@@ -123,6 +124,11 @@ public class DbLayer {
 
     private static final ConcurrentHashMap<Integer, Integer> lastUpdatedTsMap = new ConcurrentHashMap<>();
 
+    // Cache for routing collection IDs (with routing tags)
+    private static volatile Set<Integer> routingCollectionIdsCache = null;
+    private static volatile long routingCollectionIdsCacheTimestamp = 0;
+    private static final long CACHE_TTL_MS = 60 * 60 * 1000; // 60 minutes
+
     private static int getLastUpdatedTsForAccount(int accountId) {
         return lastUpdatedTsMap.computeIfAbsent(accountId, k -> 0);
     }
@@ -375,11 +381,88 @@ public static List<SingleTypeInfo> fetchSti(Bson filterQ) {
 //    }
 
     public static List<SingleTypeInfo> fetchStiBasedOnHostHeaders(ObjectId objectId) {
-        Bson filterForHostHeader = SingleTypeInfoDao.filterForHostHeader(-1,false);
-        Bson filterForSkip = Filters.gt("_id", objectId);
-        Bson finalFilter = objectId == null ? filterForHostHeader : Filters.and(filterForHostHeader, filterForSkip);
-        int limit = 1000;
-        return SingleTypeInfoDao.instance.findAll(finalFilter, 0, limit, Sorts.ascending("_id"), Projections.exclude(SingleTypeInfo._VALUES));
+        List<SingleTypeInfo> results = new ArrayList<>();
+        ObjectId currentObjectId = objectId;
+        Set<Integer> routingCollectionIds = null;
+
+        if (Context.accountId.get() == Constants.ROUTING_SKIP_ACCOUNT_ID) {
+            routingCollectionIds = getRoutingCollectionIds();
+        }
+
+        // Loop and discard template URLs from collections belonging to routing tags
+        // Keep fetching until we get a non-empty list or exhaust documents
+        while (results.isEmpty()) {
+            Bson filterForHostHeader = SingleTypeInfoDao.filterForHostHeader(-1, false);
+            Bson filterForSkip = currentObjectId == null ? null : Filters.gt("_id", currentObjectId);
+            Bson finalFilter = filterForSkip == null ? filterForHostHeader : Filters.and(filterForHostHeader, filterForSkip);
+
+            int limit = 1000;
+            List<SingleTypeInfo> batch = SingleTypeInfoDao.instance.findAll(finalFilter, 0, limit, Sorts.ascending("_id"), Projections.exclude(SingleTypeInfo._VALUES));
+
+            if (batch.isEmpty()) {
+                break; // Exhausted all documents
+            }
+
+            // Always update cursor to last item in batch
+            currentObjectId = batch.get(batch.size() - 1).getId();
+
+            // Filter out template URLs from routing collections
+            if (routingCollectionIds != null) {
+                for (SingleTypeInfo sti : batch) {
+                    if (!routingCollectionIds.contains(sti.getApiCollectionId())) {
+                        // Not from routing collection - keep everything
+                        results.add(sti);
+                    } else if (!APICatalog.isTemplateUrl(sti.getUrl())) {
+                        // From routing collection but static URL - keep it
+                        results.add(sti);
+                    }
+                    // From routing collection AND template URL - discard
+                }
+            } else {
+                results.addAll(batch);
+            }
+
+            if (batch.size() < limit) {
+                break; // Last page reached
+            }
+        }
+
+        return results;
+    }
+
+    private static Set<Integer> getRoutingCollectionIds() {
+        long currentTime = System.currentTimeMillis();
+
+        // Check if cache is valid
+        if (routingCollectionIdsCache != null &&
+            (currentTime - routingCollectionIdsCacheTimestamp) < CACHE_TTL_MS) {
+            return routingCollectionIdsCache;
+        }
+
+        // Cache miss or expired - rebuild cache
+        synchronized (DbLayer.class) {
+            // Double-check after acquiring lock
+            if (routingCollectionIdsCache != null &&
+                (currentTime - routingCollectionIdsCacheTimestamp) < CACHE_TTL_MS) {
+                return routingCollectionIdsCache;
+            }
+
+            List<Integer> apiCollectionIds = fetchApiCollectionIds();
+            Set<Integer> routingCollectionIds = new HashSet<>();
+
+            for (Integer collectionId : apiCollectionIds) {
+                ApiCollection collection = ApiCollectionsDao.instance.getMeta(collectionId);
+                if (ApiCollectionsDao.hasRoutingTags(collection)) {
+                    routingCollectionIds.add(collectionId);
+                }
+            }
+
+            // Update cache
+            routingCollectionIdsCache = routingCollectionIds;
+            routingCollectionIdsCacheTimestamp = currentTime;
+
+            return routingCollectionIds;
+        }
     }
 
     public static List<Integer> fetchApiCollectionIds() {