From a1ffb9aca1c80c13b92315cc016d100ea66423bc Mon Sep 17 00:00:00 2001 From: diwanshu Date: Tue, 8 Aug 2023 13:14:42 -0400 Subject: [PATCH] Add metrics for tracking blacklist rules usage --- carbonj.service/Dockerfile | 5 +- .../carbonj/service/engine/MetricList.java | 60 +++++++++++++------ 2 files changed, 46 insertions(+), 19 deletions(-) diff --git a/carbonj.service/Dockerfile b/carbonj.service/Dockerfile index 93a8d711..214af0f1 100644 --- a/carbonj.service/Dockerfile +++ b/carbonj.service/Dockerfile @@ -26,7 +26,7 @@ RUN yum update -y && \ RUN yum install -y gcc-c++ gcc make libtool automake autoconf make python3-devel RUN rpm --import http://repos.azulsystems.com/RPM-GPG-KEY-azulsystems && \ - curl -o /etc/yum.repos.d/zulu.repo http://repos.azulsystems.com/rhel/zulu.repo && \ + curl -L -o /etc/yum.repos.d/zulu.repo http://repos.azulsystems.com/rhel/zulu.repo && \ yum update -y && \ yum install -y zulu-11 \ python3 \ @@ -35,7 +35,8 @@ RUN rpm --import http://repos.azulsystems.com/RPM-GPG-KEY-azulsystems && \ yum clean all # Install aiohttp required by the script -RUN pip3 install aiohttp +RUN pip3 install --upgrade pip setuptools wheel && \ + pip3 install aiohttp # Ensure crontab starts RUN /sbin/chkconfig crond on && \ diff --git a/carbonj.service/src/main/java/com/demandware/carbonj/service/engine/MetricList.java b/carbonj.service/src/main/java/com/demandware/carbonj/service/engine/MetricList.java index a7998638..601e8aef 100644 --- a/carbonj.service/src/main/java/com/demandware/carbonj/service/engine/MetricList.java +++ b/carbonj.service/src/main/java/com/demandware/carbonj/service/engine/MetricList.java @@ -6,6 +6,7 @@ */ package com.demandware.carbonj.service.engine; +import org.apache.commons.lang3.tuple.Pair; import com.codahale.metrics.Counter; import com.codahale.metrics.MetricRegistry; import com.demandware.carbonj.service.db.util.StatsAware; @@ -36,20 +37,23 @@ public class MetricList implements StatsAware volatile private List configLines = new ArrayList<>( ); - volatile private List patterns = new ArrayList<>( ); + volatile private List> patternsAndCounters = new ArrayList<>( ); - volatile private boolean empty = patterns.isEmpty(); + volatile private boolean empty = patternsAndCounters.isEmpty(); private final String confSrc; private final ConfigServerUtil configServerUtil; + private final MetricRegistry metricRegistry; + public MetricList(MetricRegistry metricRegistry, String name, File confFile, String confSrc, - ConfigServerUtil configServerUtil ) + ConfigServerUtil configServerUtil ) { this.name = Preconditions.checkNotNull(name); this.confFile = Preconditions.checkNotNull( confFile ); log.info( String.format("Creating metric list [%s] with config file [%s]", name, confFile) ); + this.metricRegistry = metricRegistry; this.droppedMetrics = metricRegistry.counter( MetricRegistry.name( name, "drop" ) ); this.confSrc = confSrc; this.configServerUtil = configServerUtil; @@ -59,25 +63,36 @@ public MetricList(MetricRegistry metricRegistry, String name, File confFile, St public boolean match(String name) { - if ( patterns.isEmpty() ) + if ( patternsAndCounters.isEmpty() ) { return false; } - List currentPatterns = patterns; // copy so we don't keep hitting the volatile barrier - for ( Pattern p : currentPatterns ) + List> currentPatternsAndCounters = patternsAndCounters; // copy so we don't keep hitting the volatile barrier + for ( int i = 0; i < currentPatternsAndCounters.size(); i++ ) { - if( ".*".equals( p.pattern() ) ) + Pair p = currentPatternsAndCounters.get(i); + + if( ".*".equals( p.getLeft().pattern() ) ) { + patternsAndCounters.get(i).getRight().inc(); droppedMetrics.inc(); return true; } - if ( p.matcher( name ).find() ) + long startTime = System.nanoTime(); // Record start time + if ( p.getLeft().matcher( name ).find() ) { + patternsAndCounters.get(i).getRight().inc(); droppedMetrics.inc(); + long endTime = System.nanoTime(); // Record end time + long duration = endTime - startTime; // Calculate duration in nanoseconds + if (log.isDebugEnabled()) { + log.debug("Pattern match runtime for {}: {} nanoseconds", p.getLeft().pattern(), duration); + } return true; } + } return false; } @@ -122,9 +137,8 @@ public void reload() log.info(String.format("Metric list [%s] configuration file has changed. File: [%s]", name, confFile)); List oldLines = this.configLines; - List newPatterns = parseConfig( lines ); - - this.patterns = newPatterns; + List> newPatternsAndCounters = parseConfig( lines ); + this.patternsAndCounters = newPatternsAndCounters; this.configLines = lines; log.info(String.format("Metric list [%s] updated.", name)); if( log.isDebugEnabled() ) @@ -138,12 +152,24 @@ public void reload() } } - private List parseConfig(List lines) + private List> parseConfig(List lines) { - return lines.stream() - .map( String::trim ) - .filter( line -> line.length() > 0 && !line.startsWith( "#" ) ) - .map( Pattern::compile ).collect( Collectors.toList() ); + // Create an empty list to hold pairs of Pattern and Counter + List> patternCounterPairs = lines.stream() + .map(String::trim) + .filter(line -> line.length() > 0 && !line.startsWith("#")) + .map(line -> { + Pattern pattern = Pattern.compile(line); + Counter counter = metricRegistry.counter( MetricRegistry.name( name, "blacklist" ) ); + return Pair.of(pattern, counter); // Create and return the Pair + }) + .collect(Collectors.toList()); + + // Reset the counters here if needed + for (Pair pair : patternCounterPairs) { + pair.getValue().dec(pair.getValue().getCount()); // Reset the counter to zero + } + return patternCounterPairs; } @Override @@ -154,6 +180,6 @@ public void dumpStats() public boolean isEmpty() { - return patterns.isEmpty(); + return patternsAndCounters.isEmpty(); } }