From 4c97534ab06518671de7450fca1121f232805b58 Mon Sep 17 00:00:00 2001 From: Andres Galindo Date: Mon, 11 Aug 2025 08:41:58 -0400 Subject: [PATCH 1/3] Switch log line to metric for detecting hash collisions When experiencing a high rate of hash collissions we're observing bad performance in the agent and we suspect the deluge of logs contributes to the problem. --- .../network/push/ConsistentHashingRouter.java | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/mantis-network/src/main/java/io/reactivex/mantis/network/push/ConsistentHashingRouter.java b/mantis-network/src/main/java/io/reactivex/mantis/network/push/ConsistentHashingRouter.java index 4a90ab476..027f50557 100644 --- a/mantis-network/src/main/java/io/reactivex/mantis/network/push/ConsistentHashingRouter.java +++ b/mantis-network/src/main/java/io/reactivex/mantis/network/push/ConsistentHashingRouter.java @@ -16,6 +16,9 @@ package io.reactivex.mantis.network.push; +import io.mantisrx.common.metrics.Counter; +import io.mantisrx.common.metrics.Metrics; +import io.mantisrx.common.metrics.spectator.MetricGroupId; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -37,12 +40,21 @@ public class ConsistentHashingRouter extends Router> { private static long validCacheAgeMSec = 5000; private HashFunction hashFunction; private AtomicReference>>>> cachedRingRef = new AtomicReference<>(); + private final MetricGroupId metricGroup = new MetricGroupId("ConsistentHashingRouter"); + private final Metrics metrics; + private final Counter collisionsCounter; public ConsistentHashingRouter(String name, Func1, byte[]> dataEncoder, HashFunction hashFunction) { super("ConsistentHashingRouter_" + name, dataEncoder); this.hashFunction = hashFunction; + this.metrics = new Metrics.Builder() + .id(metricGroup) + .addCounter("numHashCollisions") + .build(); + + this.collisionsCounter = this.metrics.getCounter("numHashCollisions"); } @Override @@ -108,7 +120,7 @@ private void computeRing(Set>> connections) { byte[] connectionBytes = (connectionId + "-" + i).getBytes(); long hash = hashFunction.computeHash(connectionBytes); if (ring.containsKey(hash)) { - logger.error("Hash collision when computing ring. {} hashed to a value already in the ring.", connectionId + "-" + i); + this.collisionsCounter.increment(); } ring.put(hash, connection); } From c5c21ae02388073e356e349a28160588eec069b4 Mon Sep 17 00:00:00 2001 From: Andres Galindo Date: Tue, 12 Aug 2025 08:18:40 -0400 Subject: [PATCH 2/3] Fixed up registration --- .../mantis/network/push/ConsistentHashingRouter.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mantis-network/src/main/java/io/reactivex/mantis/network/push/ConsistentHashingRouter.java b/mantis-network/src/main/java/io/reactivex/mantis/network/push/ConsistentHashingRouter.java index 027f50557..cf60e6636 100644 --- a/mantis-network/src/main/java/io/reactivex/mantis/network/push/ConsistentHashingRouter.java +++ b/mantis-network/src/main/java/io/reactivex/mantis/network/push/ConsistentHashingRouter.java @@ -18,6 +18,7 @@ import io.mantisrx.common.metrics.Counter; import io.mantisrx.common.metrics.Metrics; +import io.mantisrx.common.metrics.MetricsRegistry; import io.mantisrx.common.metrics.spectator.MetricGroupId; import java.util.ArrayList; import java.util.HashMap; @@ -49,11 +50,13 @@ public ConsistentHashingRouter(String name, HashFunction hashFunction) { super("ConsistentHashingRouter_" + name, dataEncoder); this.hashFunction = hashFunction; - this.metrics = new Metrics.Builder() + + Metrics metrics = new Metrics.Builder() .id(metricGroup) .addCounter("numHashCollisions") .build(); + this.metrics = MetricsRegistry.getInstance().registerAndGet(metrics); this.collisionsCounter = this.metrics.getCounter("numHashCollisions"); } From 2241869f8d1e930aae76afa61528c583c28e0ef0 Mon Sep 17 00:00:00 2001 From: Andres Galindo Date: Tue, 12 Aug 2025 08:30:15 -0400 Subject: [PATCH 3/3] Added debug log --- .../mantis/network/push/ConsistentHashingRouter.java | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/mantis-network/src/main/java/io/reactivex/mantis/network/push/ConsistentHashingRouter.java b/mantis-network/src/main/java/io/reactivex/mantis/network/push/ConsistentHashingRouter.java index cf60e6636..f622cb993 100644 --- a/mantis-network/src/main/java/io/reactivex/mantis/network/push/ConsistentHashingRouter.java +++ b/mantis-network/src/main/java/io/reactivex/mantis/network/push/ConsistentHashingRouter.java @@ -113,6 +113,7 @@ private AsyncConnection> lookupConnection(long hash, SortedMa private void computeRing(Set>> connections) { SortedMap>> ring = new TreeMap>>(); + Map> collisions = new HashMap<>(); for (AsyncConnection> connection : connections) { for (int i = 0; i < connectionRepetitionOnRing; i++) { // hash node on ring @@ -124,11 +125,20 @@ private void computeRing(Set>> connections) { long hash = hashFunction.computeHash(connectionBytes); if (ring.containsKey(hash)) { this.collisionsCounter.increment(); + if(!collisions.containsKey(hash)) { + collisions.put(hash, new ArrayList<>()); + collisions.get(hash).add(ring.get(hash).getSlotId()); + } + + collisions.get(hash).add(connection.toString()); } ring.put(hash, connection); } } cachedRingRef.set(new SnapshotCache>>>(ring)); + if(!collisions.isEmpty()) { + logger.debug("hash collisions were found while recomputing ring: {}", collisions); + } } private SortedMap>> hashConnections(Set>> connections) {