A possible concurrent access to "idToReplicaset" map

[nurzhan-saktaganov]

Case 1
We expose an original map idToReplicaset via the RouterRouteAll method. Nothing prevents a caller from modifying this map.

Case 2
An external topology provider has indirect access to this map via TopologyController. No sync primitive is used to restrict concurrent access (rw).

As a possible solution, we should not modify idToReplicaset at all (make maps immutable by convention). Just create an entirely new map whenever we need to change the current topology. Additionally, we have to guard the idToReplicaset variable using some RWMutex.

@KaymeKaydex I will come up with a Merge Request later if you don't mind.

The basic idea:

type Router struct {
        // ...
    	idToReplicasetMutex sync.RWMutex // guards not the map itself, but the variable idToReplicaset
	idToReplicaset      map[uuid.UUID]*Replicaset
	// ...
}

func (c *controller) AddReplicaset(ctx context.Context, rsInfo ReplicasetInfo, instances []InstanceInfo) error {
	var newIdToReplicaset  map[uuid.UUID]*Replicaset
	var oldIDtoReplicaset map[uuid.UUID]*Replicaset
	
	c.r.idToReplicasetMutex.RLock()
	oldIDtoReplicaset = c.r.idToReplicaset // We guarantee atomic assignment using RW mutex
	c.r.idToReplicasetMutex.RUnlock()
	
	// Here we make a deep copy of oldIDtoReplicaset into newIdToReplicaset
	// TODO
	
	// Here we can modify newIdToReplicaset safely because it was not exposed yet
	// TODO
	
	// Expose new topology, the only place where we use write-lock.
	c.r.idToReplicasetMutex.Lock()
	c.r.idToReplicaset  = newIdToReplicaset
	c.r.idToReplicasetMutex.Unlock()
	
	// Don't do anything with oldIDtoReplicaset (since it was exposed before), the garbage collector will handle it
	
	return nil
}

// RouterRouteAll return map of all replicasets.
func (r *Router) RouterRouteAll() map[uuid.UUID]*Replicaset {
        var idToReplicasetDeepCopy map[uuid.UUID]*Replicaset
        var idToReplicsetRef map[uuid.UUID]*Replicaset
        
        c.r.idToReplicasetMutex.RLock()
	idToReplicsetRef = r.idToReplicaset // We guarantee atomic assignment using RW mutex
	c.r.idToReplicasetMutex.RUnlock()
        
        // We can safely make a deep copy of idToReplicsetRef into idToReplicasetDeepCopy
        // TODO
        
	return idToReplicasetDeepCopy
}

func (r *Router) RouteMapClean() {
 	var idToReplicsetRef map[uuid.UUID]*Replicaset
 	
 	c.r.idToReplicasetMutex.RLock()
	idToReplicsetRef = r.idToReplicaset // We guarantee atomic assignment using RW mutex
	c.r.idToReplicasetMutex.RUnlock()

	r.routeMap = make([]*Replicaset, r.cfg.TotalBucketCount+1)
	r.knownBucketCount.Store(0)

	// We can safely iterate over idToReplicsetRef because it's immutable regarding our convention
	for _, rs := range idToReplicsetRef {
		rs.bucketCount.Store(0)
	}
}