From acbe33da364e16339f08b5d457c1e5bda18727aa Mon Sep 17 00:00:00 2001 From: Marius Oehler Date: Wed, 4 Dec 2019 11:06:31 +0100 Subject: [PATCH] Optimized imports and reformatted code (#1) --- build.gradle | 21 ++-- .../de/novatec/baselining/Application.java | 6 -- .../novatec/baselining/BaseliningService.java | 20 ++-- .../de/novatec/baselining/InfluxAccess.java | 36 +++---- .../de/novatec/baselining/InfluxUtils.java | 18 ++-- .../baselines/BaselineGenerator.java | 96 ++++++++++--------- .../config/BaselineServiceSettings.java | 6 +- .../baselining/config/InfluxSettings.java | 2 - .../baselines/AbstractBaselineDefinition.java | 8 +- ...ctTaggedAggregatingBaselineDefinition.java | 8 +- .../baselines/CounterBaselineDefinition.java | 4 +- .../CounterRatioBaselineDefinition.java | 3 +- .../baselines/GaugeBaselineDefinition.java | 3 +- .../baselines/RateBaselineDefinition.java | 2 +- .../MeasurementFieldNameConverter.java | 5 +- .../converters/MeasurementNameConverter.java | 4 +- .../measurement/MeasurementFieldName.java | 1 - .../baselining/data/AbstractTimedPoint.java | 1 - .../baselining/data/AggregatePoint.java | 23 +++-- .../de/novatec/baselining/data/DataPoint.java | 1 - .../de/novatec/baselining/data/TagValues.java | 10 +- .../data/transformations/Aggregations.java | 58 +++++------ .../transformations/LinearInterpolator.java | 18 ++-- .../data/transformations/Transformations.java | 34 +++---- .../datasources/BaselineDataSource.java | 8 +- .../datasources/CounterDataSource.java | 16 ++-- .../datasources/CounterRatioDataSource.java | 32 ++++--- .../datasources/GaugeDataSource.java | 19 ++-- .../datasources/RateBaselineSource.java | 33 +++---- 29 files changed, 246 insertions(+), 250 deletions(-) diff --git a/build.gradle b/build.gradle index 35eadb8..345fe2a 100755 --- a/build.gradle +++ b/build.gradle @@ -5,7 +5,7 @@ plugins { apply plugin: 'io.spring.dependency-management' group 'de.novatec' -version '1.0-SNAPSHOT' +version '0.2-SNAPSHOT' sourceCompatibility = '1.8' repositories { @@ -17,16 +17,20 @@ test { } dependencies { - implementation( - "org.springframework.boot:spring-boot-starter", - "org.hibernate:hibernate-validator:6.1.0.Final" + annotationProcessor( + "org.projectlombok:lombok:1.18.4" ) - compileOnly "org.projectlombok:lombok:1.18.4" - annotationProcessor "org.projectlombok:lombok:1.18.4" - implementation group: 'org.apache.commons', name: 'commons-math3', version: '3.0' + compileOnly( + "org.projectlombok:lombok:1.18.4" + ) - implementation group: 'org.influxdb', name: 'influxdb-java', version: '2.15' + implementation( + "org.springframework.boot:spring-boot-starter", + "org.hibernate:hibernate-validator:6.1.0.Final", + "org.influxdb:influxdb-java:2.15", + "org.apache.commons:commons-math3:3.0" + ) testImplementation( 'org.springframework.boot:spring-boot-starter-test:2.1.6.RELEASE', @@ -34,5 +38,4 @@ dependencies { 'org.mockito:mockito-junit-jupiter:2.23.0' ) - } diff --git a/src/main/java/de/novatec/baselining/Application.java b/src/main/java/de/novatec/baselining/Application.java index 4637361..47ff861 100755 --- a/src/main/java/de/novatec/baselining/Application.java +++ b/src/main/java/de/novatec/baselining/Application.java @@ -1,15 +1,9 @@ package de.novatec.baselining; -import de.novatec.baselining.config.BaselineServiceSettings; -import de.novatec.baselining.datasources.GaugeDataSource; import lombok.extern.slf4j.Slf4j; -import lombok.val; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.CommandLineRunner; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; - @SpringBootApplication @Slf4j public class Application { diff --git a/src/main/java/de/novatec/baselining/BaseliningService.java b/src/main/java/de/novatec/baselining/BaseliningService.java index f61a4d2..dcdbc64 100755 --- a/src/main/java/de/novatec/baselining/BaseliningService.java +++ b/src/main/java/de/novatec/baselining/BaseliningService.java @@ -38,10 +38,10 @@ void start() { baselines.addAll(buildRateBaselines()); baselines.addAll(buildCounterBaselines()); baselines.addAll(buildCounterRatioBaselines()); - baselines.forEach(blg -> lastUpdatedTimestamp.put(blg,start)); + baselines.forEach(blg -> lastUpdatedTimestamp.put(blg, start)); new Thread(() -> { - while(true) { + while (true) { updateAll(); try { Thread.sleep(5000); @@ -53,18 +53,18 @@ void start() { } private void updateAll() { - for(BaselineGenerator generator : baselines) { + for (BaselineGenerator generator : baselines) { long now = System.currentTimeMillis() - config.getUpdateDelay().toMillis() - generator.getMinimumDelayMillis(); long updateTimestamp = lastUpdatedTimestamp.get(generator); try { - while(generator.getIntervalIndex(now) != generator.getIntervalIndex(updateTimestamp)) { + while (generator.getIntervalIndex(now) != generator.getIntervalIndex(updateTimestamp)) { long updateTo = Math.min(now, updateTimestamp + generator.getMaxUpdateIntervalSizeMillis()); generator.updateBaselines(updateTimestamp, updateTo); lastUpdatedTimestamp.put(generator, updateTo); updateTimestamp = updateTo; } - } catch(Throwable t) { - log.error("An error occurred updating the baseline",t); + } catch (Throwable t) { + log.error("An error occurred updating the baseline", t); } } } @@ -82,7 +82,7 @@ private List buildGaugeBaselines() { private List buildCounterBaselines() { return config.getCounters().stream() .map(definition -> { - CounterDataSource src = new CounterDataSource(influx,definition); + CounterDataSource src = new CounterDataSource(influx, definition); return buildBaselineGenerator(definition, src); }) .collect(Collectors.toList()); @@ -91,7 +91,7 @@ private List buildCounterBaselines() { private List buildCounterRatioBaselines() { return config.getCounterRatios().stream() .map(definition -> { - CounterRatioDataSource src = new CounterRatioDataSource(influx,definition); + CounterRatioDataSource src = new CounterRatioDataSource(influx, definition); return buildBaselineGenerator(definition, src); }) .collect(Collectors.toList()); @@ -100,13 +100,13 @@ private List buildCounterRatioBaselines() { private List buildRateBaselines() { return config.getRates().stream() .map(definition -> { - RateBaselineSource src = new RateBaselineSource(influx,definition); + RateBaselineSource src = new RateBaselineSource(influx, definition); return buildBaselineGenerator(definition, src); }) .collect(Collectors.toList()); } private BaselineGenerator buildBaselineGenerator(AbstractBaselineDefinition definition, BaselineDataSource source) { - return new BaselineGenerator(influx,source,definition); + return new BaselineGenerator(influx, source, definition); } } diff --git a/src/main/java/de/novatec/baselining/InfluxAccess.java b/src/main/java/de/novatec/baselining/InfluxAccess.java index 02350bb..53f368c 100755 --- a/src/main/java/de/novatec/baselining/InfluxAccess.java +++ b/src/main/java/de/novatec/baselining/InfluxAccess.java @@ -29,7 +29,7 @@ public class InfluxAccess { private InfluxDB influx; void connect() { - if(influx== null) { + if (influx == null) { OkHttpClient.Builder okHttpClientBuilder = new OkHttpClient().newBuilder() .connectTimeout(config.getConnectTimeout().getSeconds(), TimeUnit.SECONDS) .readTimeout(config.getReadTimeout().getSeconds(), TimeUnit.SECONDS) @@ -37,7 +37,7 @@ void connect() { boolean userEmpty = StringUtils.isEmpty(config.getUser()); boolean passwordEmpty = StringUtils.isEmpty(config.getPassword()); - if(userEmpty && passwordEmpty) { + if (userEmpty && passwordEmpty) { influx = InfluxDBFactory.connect(config.getUrl().toString(), okHttpClientBuilder); } else { influx = InfluxDBFactory.connect(config.getUrl().toString(), config.getUser(), config.getPassword(), okHttpClientBuilder); @@ -56,17 +56,17 @@ public QueryResult query(String selectFrom, long startMillis, long endMillis) { return query(selectFrom, null, "*", startMillis, endMillis); } - public QueryResult query(String selectFrom,String filter, String groupBy, long startMillis, long endMillis) { + public QueryResult query(String selectFrom, String filter, String groupBy, long startMillis, long endMillis) { StringBuilder query = new StringBuilder(selectFrom); query.append(" WHERE ").append(buildTimeFilter(startMillis, endMillis)); - if(!StringUtils.isEmpty(filter)) { + if (!StringUtils.isEmpty(filter)) { query.append(" AND ").append(filter); } query.append(" GROUP BY ").append(groupBy); try { connect(); return influx.query(new Query(query.toString()), TimeUnit.MILLISECONDS); - } catch(Throwable t) { + } catch (Throwable t) { try { influx.close(); } catch (Exception e) { @@ -83,22 +83,22 @@ public Map> querySingleField(String selectFrom, long } public Map> queryAggregate(String selectFrom, long startMillis, long endMillis, long intervalMillis) { - String groupBy = "*, time("+intervalMillis+"ms) fill(none)"; - QueryResult queryResult = query(selectFrom,null,groupBy, startMillis, endMillis); + String groupBy = "*, time(" + intervalMillis + "ms) fill(none)"; + QueryResult queryResult = query(selectFrom, null, groupBy, startMillis, endMillis); return extractSeriesResults(queryResult); } private Map> extractSeriesResults(QueryResult queryResult) { - if(queryResult.getError() != null) { + if (queryResult.getError() != null) { throw new RuntimeException("Influx Returned Error: " + queryResult.getError()); } - if(queryResult.getResults() != null) { + if (queryResult.getResults() != null) { return queryResult.getResults().stream() .filter(Objects::nonNull) .map(result -> result.getSeries()) .filter(Objects::nonNull) .flatMap(List::stream) - .filter(series -> series.getValues()!= null && !series.getValues().isEmpty()) + .filter(series -> series.getValues() != null && !series.getValues().isEmpty()) .collect(Collectors.toMap( series -> TagValues.from(series.getTags()), series -> seriesToPoints(series) @@ -109,7 +109,7 @@ private Map> extractSeriesResults(QueryResult queryRe } private List seriesToPoints(QueryResult.Series series) { - if(series.getColumns().size() != 2) { + if (series.getColumns().size() != 2) { throw new IllegalArgumentException("Query returned more than one non-time field: " + series.getColumns()); } int timeIndex = series.getColumns().indexOf("time"); @@ -117,9 +117,9 @@ private List seriesToPoints(QueryResult.Series series) { return series.getValues().stream() .filter(vals -> vals.get(fieldIndex) != null) .map(values -> DataPoint.builder() - .time(((Number)values.get(timeIndex)).longValue()) - .value(((Number)values.get(fieldIndex)).doubleValue()) - .build()) + .time(((Number) values.get(timeIndex)).longValue()) + .value(((Number) values.get(fieldIndex)).doubleValue()) + .build()) .collect(Collectors.toList()); } @@ -132,13 +132,13 @@ public void writePoints(String database, String retention, String measurement, M .addField("value", pt.getValue()) .build()) .collect(Collectors.toList()); - if(!converted.isEmpty()) { - writePoints(database,retention,tags.getTags(),converted); + if (!converted.isEmpty()) { + writePoints(database, retention, tags.getTags(), converted); } }); } - public void writePoints(String database,String retention, Map tags, Collection points) { + public void writePoints(String database, String retention, Map tags, Collection points) { BatchPoints.Builder builder = BatchPoints.database(database) .retentionPolicy(retention) .precision(TimeUnit.MILLISECONDS); @@ -152,7 +152,7 @@ public void writePoints(String database,String retention, Map tag try { connect(); influx.write(builder.build()); - } catch(Throwable t) { + } catch (Throwable t) { try { influx.close(); } catch (Exception e) { diff --git a/src/main/java/de/novatec/baselining/InfluxUtils.java b/src/main/java/de/novatec/baselining/InfluxUtils.java index fc4dfbf..2e39065 100755 --- a/src/main/java/de/novatec/baselining/InfluxUtils.java +++ b/src/main/java/de/novatec/baselining/InfluxUtils.java @@ -26,38 +26,38 @@ public static String getMeasurement(String dbRpMeasurement) { private static String[] splitToSegments(String dbRpMeasurement) { String[] segments = dbRpMeasurement.split("\\."); - if(segments.length != 3) { - throw new IllegalArgumentException("'"+dbRpMeasurement+"' is not int the form .."); + if (segments.length != 3) { + throw new IllegalArgumentException("'" + dbRpMeasurement + "' is not int the form .."); } return segments; } private static String unquote(String segment) { return segment - .replace("\"","") - .replace("'",""); + .replace("\"", "") + .replace("'", ""); } public static String prettyPrintDuration(long millis) { long timeLeft = millis; StringBuilder result = new StringBuilder(); - if(timeLeft / MILLIS_PER_DAY > 0) { + if (timeLeft / MILLIS_PER_DAY > 0) { result.append(timeLeft / MILLIS_PER_DAY).append("d"); timeLeft = timeLeft % MILLIS_PER_DAY; } - if(timeLeft / MILLIS_PER_HOUR > 0) { + if (timeLeft / MILLIS_PER_HOUR > 0) { result.append(timeLeft / MILLIS_PER_HOUR).append("h"); timeLeft = timeLeft % MILLIS_PER_HOUR; } - if(timeLeft / MILLIS_PER_MINUTE > 0) { + if (timeLeft / MILLIS_PER_MINUTE > 0) { result.append(timeLeft / MILLIS_PER_MINUTE).append("m"); timeLeft = timeLeft % MILLIS_PER_MINUTE; } - if(timeLeft / MILLIS_PER_SECOND > 0) { + if (timeLeft / MILLIS_PER_SECOND > 0) { result.append(timeLeft / MILLIS_PER_SECOND).append("s"); timeLeft = timeLeft % MILLIS_PER_SECOND; } - if(timeLeft > 0 ||result.length() == 0) { + if (timeLeft > 0 || result.length() == 0) { result.append(timeLeft).append("ms"); } return result.toString(); diff --git a/src/main/java/de/novatec/baselining/baselines/BaselineGenerator.java b/src/main/java/de/novatec/baselining/baselines/BaselineGenerator.java index 02a46a4..4333c9d 100755 --- a/src/main/java/de/novatec/baselining/baselines/BaselineGenerator.java +++ b/src/main/java/de/novatec/baselining/baselines/BaselineGenerator.java @@ -49,7 +49,7 @@ public BaselineGenerator(InfluxAccess influx, BaselineDataSource src, AbstractBa /** * Some BaselineSources need to know data in the future for computing baselines at a given point in time. * For example, in order to perform outlier filtering. - * + *

* Therefore it is required, that at least the amount of time returned by this method has passed * before invoking {@link #updateBaselines(long, long)}. * @@ -62,8 +62,9 @@ public long getMinimumDelayMillis() { /** * A suggestion on the maximum number of milliseconds between the start and the end timestamp * when invoking {@link #updateBaselines(long, long)}. - * + *

* This minimized therisk of read or write timeouts when generating baselines. + * * @return the number of milliseconds */ public long getMaxUpdateIntervalSizeMillis() { @@ -75,8 +76,9 @@ public long getMaxUpdateIntervalSizeMillis() { * Because baselines actually do operate on intervals, * the baseline computation will start with the interval in which the start timestamp lies (inclusive) * and end with the interval within which the end timestamp lies (exclusive). + * * @param startMillis the start timestamp since the epoche - * @param endMillis he start timestamp since the epoche + * @param endMillis he start timestamp since the epoche */ public void updateBaselines(long startMillis, long endMillis) { long startInterval = getIntervalIndex(startMillis); @@ -86,11 +88,11 @@ public void updateBaselines(long startMillis, long endMillis) { Date startDate = new Date(startInterval * precisionMillis); Date endDate = new Date(endInterval * precisionMillis); - log.info("Updating Baselines '{}' from {} to {}", outputPrefix.getFullMeasurementName(),startDate,endDate); + log.info("Updating Baselines '{}' from {} to {}", outputPrefix.getFullMeasurementName(), startDate, endDate); updateInfinityBaseline(startInterval, endInterval); - for(long windowSize : windowMillis) { - updateWindowedBaseline(startInterval + seasonIntervalCount, endInterval +seasonIntervalCount, windowSize); + for (long windowSize : windowMillis) { + updateWindowedBaseline(startInterval + seasonIntervalCount, endInterval + seasonIntervalCount, windowSize); } log.info("Update finished"); } @@ -105,13 +107,13 @@ private void updateInfinityBaseline(long startInterval, long endInterval) { long previousRelevant = Math.min(endInterval, startInterval + seasonIntervalCount); Map> previousBaselines = fetchInfinityBaselines(startInterval, previousRelevant); - Map> newData = src.fetch(precisionMillis, startInterval, endInterval); + Map> newData = src.fetch(precisionMillis, startInterval, endInterval); Set allTags = new HashSet<>(); allTags.addAll(previousBaselines.keySet()); allTags.addAll(newData.keySet()); - for(TagValues tags : allTags) { + for (TagValues tags : allTags) { List oldBaseline = previousBaselines.get(tags); List newPoints = newData.get(tags); @@ -120,30 +122,30 @@ private void updateInfinityBaseline(long startInterval, long endInterval) { } private void updateInfinityBaselineSeriesWithNewData(long startIntervall, long endIntervall, TagValues tags, List oldBaseline, List newPoints) { - Map intervallToBaselineMap = indexPointsByInterval(oldBaseline); - Map intervallToDataMap = indexPointsByInterval(newPoints); + Map intervallToBaselineMap = indexPointsByInterval(oldBaseline); + Map intervallToDataMap = indexPointsByInterval(newPoints); List outputPoints = new ArrayList<>(); - for(long intervall = startIntervall; intervall < endIntervall; intervall++) { + for (long intervall = startIntervall; intervall < endIntervall; intervall++) { AggregatePoint previousBaseline = intervallToBaselineMap.get(intervall); AggregatePoint newValue = intervallToDataMap.get(intervall); AggregatePoint resultBaseline = incrementBaseline(previousBaseline, newValue); - if(resultBaseline != null) { + if (resultBaseline != null) { intervallToBaselineMap.put(getIntervalIndex(resultBaseline.getTime()), resultBaseline); outputPoints.add(resultBaseline); } } - writeBaselinePoints(outputPrefix.getMeasurement()+"_inf",true,tags, outputPoints); + writeBaselinePoints(outputPrefix.getMeasurement() + "_inf", true, tags, outputPoints); } private void updateWindowedBaseline(long startInterval, long endInterval, long windowDuration) { String durationSuffix = "_" + InfluxUtils.prettyPrintDuration(windowDuration); - long windowIntervallCount = windowDuration/precisionMillis; + long windowIntervallCount = windowDuration / precisionMillis; Map> now = fetchInfinityBaselines(startInterval, endInterval); Map> past = fetchInfinityBaselines(startInterval - windowIntervallCount, endInterval - windowIntervallCount); @@ -152,34 +154,34 @@ private void updateWindowedBaseline(long startInterval, long endInterval, long w allTags.addAll(now.keySet()); allTags.addAll(past.keySet()); - for(TagValues tags : allTags) { - Map nowValues = indexPointsByInterval(now.get(tags)); - Map pastValues = indexPointsByInterval(past.get(tags)); + for (TagValues tags : allTags) { + Map nowValues = indexPointsByInterval(now.get(tags)); + Map pastValues = indexPointsByInterval(past.get(tags)); List outputPoints = new ArrayList<>(); - for(long intervall = startInterval; intervall < endInterval; intervall++) { + for (long intervall = startInterval; intervall < endInterval; intervall++) { - AggregatePoint previousPoint = pastValues.get(intervall- windowIntervallCount); + AggregatePoint previousPoint = pastValues.get(intervall - windowIntervallCount); AggregatePoint nowPoint = nowValues.get(intervall); AggregatePoint resultBaseline = computeDelta(previousPoint, nowPoint); - if(resultBaseline != null) { + if (resultBaseline != null) { outputPoints.add(resultBaseline); } } - writeBaselinePoints(outputPrefix.getMeasurement()+durationSuffix,false,tags, outputPoints); + writeBaselinePoints(outputPrefix.getMeasurement() + durationSuffix, false, tags, outputPoints); } } private AggregatePoint computeDelta(AggregatePoint firstPoint, AggregatePoint secondPoint) { - if(firstPoint == null && secondPoint != null) { + if (firstPoint == null && secondPoint != null) { return secondPoint.toBuilder().build(); - } else if(firstPoint != null && secondPoint != null) { + } else if (firstPoint != null && secondPoint != null) { AggregatePoint diff = secondPoint.minus(firstPoint); - if(diff.getCount() > 0) { + if (diff.getCount() > 0) { return diff; } } @@ -189,32 +191,32 @@ private AggregatePoint computeDelta(AggregatePoint firstPoint, AggregatePoint se private void writeBaselinePoints(String measurementName, boolean includeAggregates, TagValues tags, List outputPoints) { List points = outputPoints.stream() - .map(pt -> toInfluxPoint(pt,measurementName,includeAggregates)) + .map(pt -> toInfluxPoint(pt, measurementName, includeAggregates)) .filter(Optional::isPresent) .map(Optional::get) .collect(Collectors.toList()); - influx.writePoints(outputPrefix.getDatabase(),outputPrefix.getRetention(),tags.getTags(), points); + influx.writePoints(outputPrefix.getDatabase(), outputPrefix.getRetention(), tags.getTags(), points); } private AggregatePoint incrementBaseline(AggregatePoint previousBaseline, AggregatePoint newSeasonValue) { - if(newSeasonValue != null) { - if(previousBaseline == null || previousBaseline.getCount() == 0) { + if (newSeasonValue != null) { + if (previousBaseline == null || previousBaseline.getCount() == 0) { return newSeasonValue.shift(seasonalityMillis); } else { return previousBaseline.add(newSeasonValue).shift(seasonalityMillis); } - } else if(previousBaseline != null) { + } else if (previousBaseline != null) { return previousBaseline.shift(seasonalityMillis); } return null; } private Map indexPointsByInterval(Collection points) { - Map result = new HashMap<>(); - if(points != null) { - for(PT pt : points) { + Map result = new HashMap<>(); + if (points != null) { + for (PT pt : points) { long intervall = getIntervalIndex(pt.getTime()); - if(result.containsKey(intervall)) { + if (result.containsKey(intervall)) { throw new IllegalArgumentException("Input point set contains multiple points falling into the same intervall!"); } result.put(intervall, pt); @@ -224,7 +226,7 @@ private Map indexPointsByInterval(Coll } private Optional toInfluxPoint(AggregatePoint pt, String measurementName, boolean includeAggregates) { - if(pt.getCount() == 0) { + if (pt.getCount() == 0) { return Optional.empty(); } double value = pt.getAvgValue(); @@ -235,24 +237,24 @@ private Optional toInfluxPoint(AggregatePoint pt, String measurementName, .addField("value", value) .addField("stddev", stddev) .addField("seasons", pt.getCount()); - if(includeAggregates) { + if (includeAggregates) { builder - .addField("sum", pt.getValuesSum()) - .addField("sumSq", pt.getSquaredValuesSum()); + .addField("sum", pt.getValuesSum()) + .addField("sumSq", pt.getSquaredValuesSum()); } return Optional.of(builder.build()); } private Map> fetchInfinityBaselines(long startIntervall, long endIntervall) { - long start = startIntervall*precisionMillis; - long end = endIntervall *precisionMillis; + long start = startIntervall * precisionMillis; + long end = endIntervall * precisionMillis; - String query = "SELECT sum, sumSq, seasons FROM " + outputPrefix.getFullMeasurementName()+"_inf"; + String query = "SELECT sum, sumSq, seasons FROM " + outputPrefix.getFullMeasurementName() + "_inf"; QueryResult result = influx.query(query, start, end); Map> baselines = new HashMap<>(); - if(result.getResults() != null) { + if (result.getResults() != null) { return result.getResults().stream() .filter(Objects::nonNull) .map(QueryResult.Result::getSeries) @@ -276,19 +278,19 @@ private List decodeBaselinePoints(QueryResult.Series series) { List baselinePoints = new ArrayList<>(); - for(List values : series.getValues()) { + for (List values : series.getValues()) { Object time = values.get(timeIdx); Object seasons = values.get(seasonsIdx); Object sum = values.get(sumIdx); Object sumSq = values.get(sumSqIdx); - if(time instanceof Number && seasons instanceof Number && sum instanceof Number && sumSq instanceof Number) { + if (time instanceof Number && seasons instanceof Number && sum instanceof Number && sumSq instanceof Number) { AggregatePoint pt = AggregatePoint.builder() - .time(((Number)time).longValue()) - .count(((Number)seasons).longValue()) - .valuesSum(((Number)sum).doubleValue()) - .squaredValuesSum(((Number)sumSq).doubleValue()) + .time(((Number) time).longValue()) + .count(((Number) seasons).longValue()) + .valuesSum(((Number) sum).doubleValue()) + .squaredValuesSum(((Number) sumSq).doubleValue()) .build(); baselinePoints.add(pt); } diff --git a/src/main/java/de/novatec/baselining/config/BaselineServiceSettings.java b/src/main/java/de/novatec/baselining/config/BaselineServiceSettings.java index d664307..0bc7f05 100755 --- a/src/main/java/de/novatec/baselining/config/BaselineServiceSettings.java +++ b/src/main/java/de/novatec/baselining/config/BaselineServiceSettings.java @@ -1,9 +1,11 @@ package de.novatec.baselining.config; -import de.novatec.baselining.config.baselines.*; +import de.novatec.baselining.config.baselines.CounterBaselineDefinition; +import de.novatec.baselining.config.baselines.CounterRatioBaselineDefinition; +import de.novatec.baselining.config.baselines.GaugeBaselineDefinition; +import de.novatec.baselining.config.baselines.RateBaselineDefinition; import lombok.Data; import lombok.NoArgsConstructor; -import org.hibernate.validator.constraints.time.DurationMin; import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.context.annotation.Configuration; import org.springframework.validation.annotation.Validated; diff --git a/src/main/java/de/novatec/baselining/config/InfluxSettings.java b/src/main/java/de/novatec/baselining/config/InfluxSettings.java index efebd24..045cd0a 100755 --- a/src/main/java/de/novatec/baselining/config/InfluxSettings.java +++ b/src/main/java/de/novatec/baselining/config/InfluxSettings.java @@ -1,13 +1,11 @@ package de.novatec.baselining.config; -import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.context.annotation.Configuration; import org.springframework.validation.annotation.Validated; -import javax.validation.constraints.NotBlank; import javax.validation.constraints.NotNull; import java.net.URL; import java.time.Duration; diff --git a/src/main/java/de/novatec/baselining/config/baselines/AbstractBaselineDefinition.java b/src/main/java/de/novatec/baselining/config/baselines/AbstractBaselineDefinition.java index bf58e08..2572b10 100755 --- a/src/main/java/de/novatec/baselining/config/baselines/AbstractBaselineDefinition.java +++ b/src/main/java/de/novatec/baselining/config/baselines/AbstractBaselineDefinition.java @@ -35,9 +35,9 @@ public boolean isSeasonalityMultipleOfPrecision() { @AssertTrue public boolean isWindowsMultiplesOfSeasonality() { - if(windows != null) { - for(Duration window : windows) { - if(window.toMillis() % seasonality.toMillis() != 0) { + if (windows != null) { + for (Duration window : windows) { + if (window.toMillis() % seasonality.toMillis() != 0) { return false; } } @@ -46,7 +46,7 @@ public boolean isWindowsMultiplesOfSeasonality() { } public List getWindowsWithDefault() { - if(windows == null) { + if (windows == null) { return Collections.singletonList(seasonality.multipliedBy(10)); } else { return windows; diff --git a/src/main/java/de/novatec/baselining/config/baselines/AbstractTaggedAggregatingBaselineDefinition.java b/src/main/java/de/novatec/baselining/config/baselines/AbstractTaggedAggregatingBaselineDefinition.java index aa4287b..ef13b51 100755 --- a/src/main/java/de/novatec/baselining/config/baselines/AbstractTaggedAggregatingBaselineDefinition.java +++ b/src/main/java/de/novatec/baselining/config/baselines/AbstractTaggedAggregatingBaselineDefinition.java @@ -1,12 +1,10 @@ package de.novatec.baselining.config.baselines; -import de.novatec.baselining.config.measurement.MeasurementFieldName; import de.novatec.baselining.config.measurement.MeasurementName; import lombok.Data; import lombok.EqualsAndHashCode; import lombok.NoArgsConstructor; -import javax.validation.Valid; import javax.validation.constraints.AssertTrue; import javax.validation.constraints.NotBlank; import javax.validation.constraints.NotNull; @@ -16,7 +14,7 @@ @Data @NoArgsConstructor @EqualsAndHashCode(callSuper = true) -public class AbstractTaggedAggregatingBaselineDefinition extends AbstractBaselineDefinition{ +public class AbstractTaggedAggregatingBaselineDefinition extends AbstractBaselineDefinition { private List<@NotBlank String> tags; @@ -31,9 +29,9 @@ public boolean isPrecisionMultipleOfSamplePrecision() { } public MeasurementName getLoopBackMetric() { - if(loopBackSrc) { + if (loopBackSrc) { return getOutput().toBuilder() - .measurement(getOutput().getMeasurement()+"_src") + .measurement(getOutput().getMeasurement() + "_src") .build(); } else { return null; diff --git a/src/main/java/de/novatec/baselining/config/baselines/CounterBaselineDefinition.java b/src/main/java/de/novatec/baselining/config/baselines/CounterBaselineDefinition.java index a2ca15f..7490b7d 100755 --- a/src/main/java/de/novatec/baselining/config/baselines/CounterBaselineDefinition.java +++ b/src/main/java/de/novatec/baselining/config/baselines/CounterBaselineDefinition.java @@ -6,15 +6,13 @@ import lombok.NoArgsConstructor; import javax.validation.Valid; -import javax.validation.constraints.NotBlank; import javax.validation.constraints.NotNull; import java.time.Duration; -import java.util.List; @Data @NoArgsConstructor @EqualsAndHashCode(callSuper = true) -public class CounterBaselineDefinition extends AbstractTaggedAggregatingBaselineDefinition{ +public class CounterBaselineDefinition extends AbstractTaggedAggregatingBaselineDefinition { @Valid private MeasurementFieldName input; diff --git a/src/main/java/de/novatec/baselining/config/baselines/CounterRatioBaselineDefinition.java b/src/main/java/de/novatec/baselining/config/baselines/CounterRatioBaselineDefinition.java index 8ce7820..1d6ef96 100755 --- a/src/main/java/de/novatec/baselining/config/baselines/CounterRatioBaselineDefinition.java +++ b/src/main/java/de/novatec/baselining/config/baselines/CounterRatioBaselineDefinition.java @@ -8,12 +8,11 @@ import javax.validation.Valid; import javax.validation.constraints.NotNull; import java.time.Duration; -import java.util.List; @Data @NoArgsConstructor @EqualsAndHashCode(callSuper = true) -public class CounterRatioBaselineDefinition extends AbstractTaggedAggregatingBaselineDefinition{ +public class CounterRatioBaselineDefinition extends AbstractTaggedAggregatingBaselineDefinition { @NotNull @Valid diff --git a/src/main/java/de/novatec/baselining/config/baselines/GaugeBaselineDefinition.java b/src/main/java/de/novatec/baselining/config/baselines/GaugeBaselineDefinition.java index d092ca7..821c637 100755 --- a/src/main/java/de/novatec/baselining/config/baselines/GaugeBaselineDefinition.java +++ b/src/main/java/de/novatec/baselining/config/baselines/GaugeBaselineDefinition.java @@ -7,12 +7,11 @@ import javax.validation.Valid; import javax.validation.constraints.NotNull; -import java.util.List; @Data() @NoArgsConstructor @EqualsAndHashCode(callSuper = true) -public class GaugeBaselineDefinition extends AbstractTaggedAggregatingBaselineDefinition{ +public class GaugeBaselineDefinition extends AbstractTaggedAggregatingBaselineDefinition { @Valid @NotNull diff --git a/src/main/java/de/novatec/baselining/config/baselines/RateBaselineDefinition.java b/src/main/java/de/novatec/baselining/config/baselines/RateBaselineDefinition.java index ed9b713..9d9f6ff 100755 --- a/src/main/java/de/novatec/baselining/config/baselines/RateBaselineDefinition.java +++ b/src/main/java/de/novatec/baselining/config/baselines/RateBaselineDefinition.java @@ -9,7 +9,7 @@ @Data @NoArgsConstructor @EqualsAndHashCode(callSuper = true) -public class RateBaselineDefinition extends AbstractBaselineDefinition{ +public class RateBaselineDefinition extends AbstractBaselineDefinition { private String input; diff --git a/src/main/java/de/novatec/baselining/config/converters/MeasurementFieldNameConverter.java b/src/main/java/de/novatec/baselining/config/converters/MeasurementFieldNameConverter.java index fb19790..3203ade 100755 --- a/src/main/java/de/novatec/baselining/config/converters/MeasurementFieldNameConverter.java +++ b/src/main/java/de/novatec/baselining/config/converters/MeasurementFieldNameConverter.java @@ -1,7 +1,6 @@ package de.novatec.baselining.config.converters; import de.novatec.baselining.config.measurement.MeasurementFieldName; -import de.novatec.baselining.config.measurement.MeasurementName; import org.springframework.boot.context.properties.ConfigurationPropertiesBinding; import org.springframework.core.convert.converter.Converter; import org.springframework.stereotype.Component; @@ -13,8 +12,8 @@ public class MeasurementFieldNameConverter implements Converter...'"); + if (segments.length != 4) { + throw new IllegalArgumentException("'" + source + "' must have the form '...'"); } return new MeasurementFieldName(segments[0].trim(), segments[1].trim(), segments[2].trim(), segments[3].trim()); } diff --git a/src/main/java/de/novatec/baselining/config/converters/MeasurementNameConverter.java b/src/main/java/de/novatec/baselining/config/converters/MeasurementNameConverter.java index f75b626..c26c41a 100755 --- a/src/main/java/de/novatec/baselining/config/converters/MeasurementNameConverter.java +++ b/src/main/java/de/novatec/baselining/config/converters/MeasurementNameConverter.java @@ -12,8 +12,8 @@ public class MeasurementNameConverter implements Converter..'"); + if (segements.length != 3) { + throw new IllegalArgumentException("'" + source + "' must have the form '..'"); } return new MeasurementName(segements[0].trim(), segements[1].trim(), segements[2].trim()); } diff --git a/src/main/java/de/novatec/baselining/config/measurement/MeasurementFieldName.java b/src/main/java/de/novatec/baselining/config/measurement/MeasurementFieldName.java index e591341..ba7a623 100755 --- a/src/main/java/de/novatec/baselining/config/measurement/MeasurementFieldName.java +++ b/src/main/java/de/novatec/baselining/config/measurement/MeasurementFieldName.java @@ -1,6 +1,5 @@ package de.novatec.baselining.config.measurement; -import lombok.AllArgsConstructor; import lombok.Data; import lombok.EqualsAndHashCode; import lombok.NoArgsConstructor; diff --git a/src/main/java/de/novatec/baselining/data/AbstractTimedPoint.java b/src/main/java/de/novatec/baselining/data/AbstractTimedPoint.java index b8a87dc..26fd014 100755 --- a/src/main/java/de/novatec/baselining/data/AbstractTimedPoint.java +++ b/src/main/java/de/novatec/baselining/data/AbstractTimedPoint.java @@ -3,7 +3,6 @@ import lombok.AllArgsConstructor; import lombok.EqualsAndHashCode; import lombok.Getter; -import lombok.Value; import java.util.Comparator; diff --git a/src/main/java/de/novatec/baselining/data/AggregatePoint.java b/src/main/java/de/novatec/baselining/data/AggregatePoint.java index e3ee829..0a58162 100755 --- a/src/main/java/de/novatec/baselining/data/AggregatePoint.java +++ b/src/main/java/de/novatec/baselining/data/AggregatePoint.java @@ -1,13 +1,16 @@ package de.novatec.baselining.data; -import lombok.*; +import lombok.Builder; +import lombok.EqualsAndHashCode; +import lombok.Value; import java.text.SimpleDateFormat; -import java.util.*; +import java.util.Collection; +import java.util.Date; @Value @EqualsAndHashCode(callSuper = true) -public class AggregatePoint extends AbstractTimedPoint{ +public class AggregatePoint extends AbstractTimedPoint { double valuesSum; @@ -39,9 +42,9 @@ public AggregatePoint shift(long millis) { public AggregatePoint add(AggregatePoint other) { return new AggregatePoint( this.time, - this.valuesSum + other.valuesSum, - this.squaredValuesSum + other.squaredValuesSum, - this.count + other.count); + this.valuesSum + other.valuesSum, + this.squaredValuesSum + other.squaredValuesSum, + this.count + other.count); } public AggregatePoint add(DataPoint point) { @@ -49,7 +52,7 @@ public AggregatePoint add(DataPoint point) { return new AggregatePoint( this.time, this.valuesSum + value, - this.squaredValuesSum + value*value, + this.squaredValuesSum + value * value, this.count + 1); } @@ -63,7 +66,7 @@ public AggregatePoint minus(AggregatePoint other) { public static AggregatePoint from(DataPoint point) { double value = point.getValue(); - return new AggregatePoint(point.getTime(), value, value*value, 1); + return new AggregatePoint(point.getTime(), value, value * value, 1); } public static AggregatePoint from(long timestamp, Collection points) { @@ -74,7 +77,7 @@ public static AggregatePoint from(long timestamp, Collection value*value) + .map(value -> value * value) .sum(); return new AggregatePoint(timestamp, sum, sumOfSquares, count); } @@ -82,6 +85,6 @@ public static AggregatePoint from(long timestamp, Collection tags; + private Map tags; - public static TagValues from(Map tags) { - if(tags == null) { + public static TagValues from(Map tags) { + if (tags == null) { return TagValues.builder().build(); } else { return TagValues.builder().tags(tags).build(); @@ -24,8 +24,8 @@ public static TagValues from(Map tags) { public TagValues keepOnly(Collection tagKeys) { TagValues.TagValuesBuilder result = TagValues.builder(); - for(String tag : tagKeys) { - if(tags.containsKey(tag)) { + for (String tag : tagKeys) { + if (tags.containsKey(tag)) { result.tag(tag, tags.get(tag)); } } diff --git a/src/main/java/de/novatec/baselining/data/transformations/Aggregations.java b/src/main/java/de/novatec/baselining/data/transformations/Aggregations.java index e2fdb48..7f1362c 100755 --- a/src/main/java/de/novatec/baselining/data/transformations/Aggregations.java +++ b/src/main/java/de/novatec/baselining/data/transformations/Aggregations.java @@ -12,19 +12,19 @@ public class Aggregations { public static Map> aggregateByTags(Collection tagsToKeep, Map> data, BiFunction, List, List> aggregation) { Map> result = new HashMap<>(); - data.forEach((tags,values) -> { + data.forEach((tags, values) -> { TagValues newTags = tags.keepOnly(tagsToKeep); List previousValues = result.get(newTags); - if(previousValues == null) { + if (previousValues == null) { result.put(newTags, values); } else { - result.put(newTags, aggregation.apply(previousValues,values)); + result.put(newTags, aggregation.apply(previousValues, values)); } }); return result; } - public static List byIntervall(Collection points, long intervallMillis, BiFunction,? extends R> aggregation) { + public static List byIntervall(Collection points, long intervallMillis, BiFunction, ? extends R> aggregation) { PriorityQueue pq = new PriorityQueue<>(AbstractTimedPoint.TIME_COMPARATOR); pq.addAll(points); @@ -34,22 +34,22 @@ public static List byIntervall(Collection results = new ArrayList<>(); - while(!pq.isEmpty()) { + while (!pq.isEmpty()) { PT point = pq.poll(); long pointIntervall = point.getTime() / intervallMillis; - if(pointIntervall != currentIntervall) { - if(pointsInIntervall.size() > 0) { + if (pointIntervall != currentIntervall) { + if (pointsInIntervall.size() > 0) { long time = currentIntervall * intervallMillis; - results.add(aggregation.apply(time,pointsInIntervall)); - pointsInIntervall= new ArrayList<>(); + results.add(aggregation.apply(time, pointsInIntervall)); + pointsInIntervall = new ArrayList<>(); } currentIntervall = pointIntervall; } pointsInIntervall.add(point); } - if(pointsInIntervall.size() > 0) { + if (pointsInIntervall.size() > 0) { long time = currentIntervall * intervallMillis; - results.add(aggregation.apply(time,pointsInIntervall)); + results.add(aggregation.apply(time, pointsInIntervall)); } return results; } @@ -57,13 +57,13 @@ public static List byIntervall(Collection joinInterpolating( Collection left, Collection right, - BiFunction aggregation) { + BiFunction aggregation) { LinearInterpolator leftInterpolation = new LinearInterpolator(left); LinearInterpolator rightInterpolation = new LinearInterpolator(right); - return joinByTime(left,right, - (ptL,ptR) -> Optional.ofNullable(aggregation.apply(ptL.getValue(), ptR.getValue())) + return joinByTime(left, right, + (ptL, ptR) -> Optional.ofNullable(aggregation.apply(ptL.getValue(), ptR.getValue())) .map(value -> new DataPoint(ptL.getTime(), value)) .orElse(null), (ptL) -> Optional.ofNullable(aggregation.apply(ptL.getValue(), rightInterpolation.interpolate(ptL.getTime()))) @@ -72,16 +72,16 @@ public static List joinInterpolating( (ptR) -> Optional.ofNullable(aggregation.apply(leftInterpolation.interpolate(ptR.getTime()), ptR.getValue())) .map(value -> new DataPoint(ptR.getTime(), value)) .orElse(null) - ); + ); } - public static List joinByTime( + public static List joinByTime( Collection left, Collection right, - BiFunction joinFunction, - Function leftOuterFunction, - Function rightOuterFunction) { + BiFunction joinFunction, + Function leftOuterFunction, + Function rightOuterFunction) { PriorityQueue leftPq = new PriorityQueue<>(AbstractTimedPoint.TIME_COMPARATOR); leftPq.addAll(left); @@ -91,34 +91,34 @@ public static List results = new ArrayList<>(); - while(!leftPq.isEmpty() && ! rightPq.isEmpty()) { + while (!leftPq.isEmpty() && !rightPq.isEmpty()) { long timeLeft = leftPq.peek().getTime(); long timeRight = rightPq.peek().getTime(); R result = null; - if(timeLeft == timeRight) { + if (timeLeft == timeRight) { result = joinFunction.apply(leftPq.poll(), rightPq.poll()); } else if (timeLeft < timeRight) { PTL point = leftPq.poll(); - if(leftOuterFunction != null) { + if (leftOuterFunction != null) { result = leftOuterFunction.apply(point); } } else { PTR point = rightPq.poll(); - if(rightOuterFunction != null) { + if (rightOuterFunction != null) { result = rightOuterFunction.apply(point); } } - if(result != null) { + if (result != null) { results.add(result); } } - if(leftOuterFunction != null) { + if (leftOuterFunction != null) { leftPq.stream() .map(leftOuterFunction) .filter(Objects::nonNull) .forEach(results::add); } - if(rightOuterFunction != null) { + if (rightOuterFunction != null) { rightPq.stream() .map(rightOuterFunction) .filter(Objects::nonNull) @@ -127,10 +127,10 @@ public static return results; } - public static List outerJoinByTime( + public static List outerJoinByTime( Collection left, Collection right, - BiFunction joinFunction) { - return joinByTime(left,right,joinFunction, Function.identity(), Function.identity()); + BiFunction joinFunction) { + return joinByTime(left, right, joinFunction, Function.identity(), Function.identity()); } } diff --git a/src/main/java/de/novatec/baselining/data/transformations/LinearInterpolator.java b/src/main/java/de/novatec/baselining/data/transformations/LinearInterpolator.java index b8dc092..7739d5c 100755 --- a/src/main/java/de/novatec/baselining/data/transformations/LinearInterpolator.java +++ b/src/main/java/de/novatec/baselining/data/transformations/LinearInterpolator.java @@ -3,39 +3,37 @@ import de.novatec.baselining.data.AbstractTimedPoint; import de.novatec.baselining.data.DataPoint; -import java.util.ArrayList; import java.util.Collection; -import java.util.Collections; import java.util.TreeSet; public class LinearInterpolator { private TreeSet pointsSorted; - public LinearInterpolator(Collection points){ + public LinearInterpolator(Collection points) { pointsSorted = new TreeSet<>(AbstractTimedPoint.TIME_COMPARATOR); pointsSorted.addAll(points); } public double interpolate(long time) { - DataPoint dummy = new DataPoint(time,0); + DataPoint dummy = new DataPoint(time, 0); DataPoint lower = pointsSorted.floor(dummy); DataPoint upper = pointsSorted.ceiling(dummy); - if(lower == upper) { - if(lower == null) { + if (lower == upper) { + if (lower == null) { return 0; } else { return lower.getValue(); } } else { - if(lower == null) { + if (lower == null) { return upper.getValue(); } - if(upper == null) { + if (upper == null) { return lower.getValue(); } - double upperWeight = (time - lower.getTime()) / (double)(upper.getTime() - lower.getTime()); - return (1- upperWeight) * lower.getValue() + upperWeight * upper.getValue(); + double upperWeight = (time - lower.getTime()) / (double) (upper.getTime() - lower.getTime()); + return (1 - upperWeight) * lower.getValue() + upperWeight * upper.getValue(); } } } diff --git a/src/main/java/de/novatec/baselining/data/transformations/Transformations.java b/src/main/java/de/novatec/baselining/data/transformations/Transformations.java index d568d8b..69eee0f 100755 --- a/src/main/java/de/novatec/baselining/data/transformations/Transformations.java +++ b/src/main/java/de/novatec/baselining/data/transformations/Transformations.java @@ -5,7 +5,6 @@ import de.novatec.baselining.data.DataPoint; import de.novatec.baselining.data.TagValues; -import java.lang.reflect.Array; import java.time.Duration; import java.util.*; import java.util.function.BiFunction; @@ -16,16 +15,17 @@ public class Transformations { /** * Returns teh difference betweens successive points + * * @param points */ public static List rate(Collection points, Duration unit) { double unitMillis = unit.toMillis(); - return combineNeighbors(points, (first,second) -> { + return combineNeighbors(points, (first, second) -> { double diff = second.getValue() - first.getValue(); double timeDiff = second.getTime() - first.getTime(); - if(timeDiff > 0) { + if (timeDiff > 0) { return new DataPoint(second.getTime(), diff / timeDiff * unitMillis); - }else { + } else { return null; } }); @@ -38,25 +38,25 @@ public static List rateSince(Collection points, } - public static Map> rateSince(Map> data, long sinceTimestamp, Duration unit) { - return mapValues(data, pts -> rateSince(pts,sinceTimestamp,unit)); + public static Map> rateSince(Map> data, long sinceTimestamp, Duration unit) { + return mapValues(data, pts -> rateSince(pts, sinceTimestamp, unit)); } - public static Map mapValues(Map input, Function transformation) { + public static Map mapValues(Map input, Function transformation) { Map result = new HashMap<>(); - input.forEach((key,value) -> result.put(key,transformation.apply(value))); + input.forEach((key, value) -> result.put(key, transformation.apply(value))); return result; } - public static List combineNeighbors(Collection points, BiFunction combiner) { + public static List combineNeighbors(Collection points, BiFunction combiner) { ArrayList sorted = new ArrayList<>(points); sorted.sort(AbstractTimedPoint.TIME_COMPARATOR); ArrayList result = new ArrayList<>(); - for(int i=1; i List combineNeighbors(Collec public static Map> meanByInterval(Map> data, long intervalMillis) { return mapValues(data, points -> - Aggregations - .byIntervall(points, intervalMillis, AggregatePoint::from) - .stream() - .map(pt -> new AggregatePoint(pt.getTime(), pt.getAvgValue(), pt.getAvgSquaredValue(), 1)) - .collect(Collectors.toList()) - ); + Aggregations + .byIntervall(points, intervalMillis, AggregatePoint::from) + .stream() + .map(pt -> new AggregatePoint(pt.getTime(), pt.getAvgValue(), pt.getAvgSquaredValue(), 1)) + .collect(Collectors.toList()) + ); } } diff --git a/src/main/java/de/novatec/baselining/datasources/BaselineDataSource.java b/src/main/java/de/novatec/baselining/datasources/BaselineDataSource.java index ed5a6dd..aac773b 100755 --- a/src/main/java/de/novatec/baselining/datasources/BaselineDataSource.java +++ b/src/main/java/de/novatec/baselining/datasources/BaselineDataSource.java @@ -1,7 +1,7 @@ package de.novatec.baselining.datasources; -import de.novatec.baselining.data.TagValues; import de.novatec.baselining.data.AggregatePoint; +import de.novatec.baselining.data.TagValues; import java.util.List; import java.util.Map; @@ -13,13 +13,13 @@ public interface BaselineDataSource { /** * Provides the data for buildign a baseline over a specified time range. - * + *

* Baselines operator on intervals, which are specified by the precision setting. * It is required, that this method returns at most one point per interval. * * @param intervalMillis the number of milliseconds within one interval - * @param startInterval the index of the interval to start with (inclusive), meaning the query should start at intervalMillis*startInterval - * @param endInterval the index of the interval to end with (exclusive), meaning the query should end at (endInterval-1)*startInterval + * @param startInterval the index of the interval to start with (inclusive), meaning the query should start at intervalMillis*startInterval + * @param endInterval the index of the interval to end with (exclusive), meaning the query should end at (endInterval-1)*startInterval * @return the aggregates which are used to update the baseline */ Map> fetch(long intervalMillis, long startInterval, long endInterval); diff --git a/src/main/java/de/novatec/baselining/datasources/CounterDataSource.java b/src/main/java/de/novatec/baselining/datasources/CounterDataSource.java index 518d911..d640aee 100755 --- a/src/main/java/de/novatec/baselining/datasources/CounterDataSource.java +++ b/src/main/java/de/novatec/baselining/datasources/CounterDataSource.java @@ -3,7 +3,9 @@ import de.novatec.baselining.InfluxAccess; import de.novatec.baselining.config.baselines.CounterBaselineDefinition; import de.novatec.baselining.config.measurement.MeasurementName; -import de.novatec.baselining.data.*; +import de.novatec.baselining.data.AggregatePoint; +import de.novatec.baselining.data.DataPoint; +import de.novatec.baselining.data.TagValues; import de.novatec.baselining.data.transformations.Aggregations; import de.novatec.baselining.data.transformations.Transformations; import lombok.extern.slf4j.Slf4j; @@ -30,7 +32,7 @@ public class CounterDataSource implements BaselineDataSource { public CounterDataSource(InfluxAccess influx, CounterBaselineDefinition settings) { this.influx = influx; - this.query = "SELECT LAST("+settings.getInput().getField()+") FROM "+settings.getInput().getFullMeasurementName(); + this.query = "SELECT LAST(" + settings.getInput().getField() + ") FROM " + settings.getInput().getFullMeasurementName(); this.tags = settings.getTags(); this.lookBackMillis = settings.getLookBack().toMillis(); this.samplePrecisionMillis = settings.getSamplePrecision().toMillis(); @@ -42,17 +44,17 @@ public Map> fetch(long intervalMillis, long star long start = startInterval * intervalMillis; long end = endInterval * intervalMillis; - Map> data = influx.queryAggregate(query,start - lookBackMillis,end,samplePrecisionMillis); + Map> data = influx.queryAggregate(query, start - lookBackMillis, end, samplePrecisionMillis); data = Transformations.rateSince(data, start, Duration.ofSeconds(1)); - if(tags != null) { - data = Aggregations.aggregateByTags(tags,data,(a, b) -> - Aggregations.joinInterpolating(a,b, (v1,v2) -> v1 + v2) + if (tags != null) { + data = Aggregations.aggregateByTags(tags, data, (a, b) -> + Aggregations.joinInterpolating(a, b, (v1, v2) -> v1 + v2) ); } - if(rawOuput != null) { + if (rawOuput != null) { influx.writePoints(rawOuput.getDatabase(), rawOuput.getRetention(), rawOuput.getMeasurement(), data); } return Transformations.meanByInterval(data, intervalMillis); diff --git a/src/main/java/de/novatec/baselining/datasources/CounterRatioDataSource.java b/src/main/java/de/novatec/baselining/datasources/CounterRatioDataSource.java index d77bb88..f26cde3 100755 --- a/src/main/java/de/novatec/baselining/datasources/CounterRatioDataSource.java +++ b/src/main/java/de/novatec/baselining/datasources/CounterRatioDataSource.java @@ -3,7 +3,9 @@ import de.novatec.baselining.InfluxAccess; import de.novatec.baselining.config.baselines.CounterRatioBaselineDefinition; import de.novatec.baselining.config.measurement.MeasurementName; -import de.novatec.baselining.data.*; +import de.novatec.baselining.data.AggregatePoint; +import de.novatec.baselining.data.DataPoint; +import de.novatec.baselining.data.TagValues; import de.novatec.baselining.data.transformations.Aggregations; import de.novatec.baselining.data.transformations.Transformations; import lombok.extern.slf4j.Slf4j; @@ -33,8 +35,8 @@ public class CounterRatioDataSource implements BaselineDataSource { public CounterRatioDataSource(InfluxAccess influx, CounterRatioBaselineDefinition settings) { this.influx = influx; - this.numeratorQuery = "SELECT LAST("+settings.getInput().getField()+") FROM "+settings.getInput().getFullMeasurementName(); - this.denominatorQuery = "SELECT LAST("+settings.getDivideBy().getField()+") FROM "+settings.getDivideBy().getFullMeasurementName(); + this.numeratorQuery = "SELECT LAST(" + settings.getInput().getField() + ") FROM " + settings.getInput().getFullMeasurementName(); + this.denominatorQuery = "SELECT LAST(" + settings.getDivideBy().getField() + ") FROM " + settings.getDivideBy().getFullMeasurementName(); this.tags = settings.getTags(); this.lookBackMillis = settings.getLookBack().toMillis(); this.samplePrecisionMillis = settings.getSamplePrecision().toMillis(); @@ -47,22 +49,22 @@ public Map> fetch(long intervalMillis, long star long start = startInterval * intervalMillis; long end = endInterval * intervalMillis; - Map> numerators = influx.queryAggregate(numeratorQuery,start - lookBackMillis,end, samplePrecisionMillis); - Map> denominators = influx.queryAggregate(denominatorQuery,start - lookBackMillis,end, samplePrecisionMillis); + Map> numerators = influx.queryAggregate(numeratorQuery, start - lookBackMillis, end, samplePrecisionMillis); + Map> denominators = influx.queryAggregate(denominatorQuery, start - lookBackMillis, end, samplePrecisionMillis); - if(tags != null) { - numerators = Aggregations.aggregateByTags(tags,numerators,(a, b) -> - Aggregations.joinInterpolating(a,b, (v1,v2) -> v1 + v2) + if (tags != null) { + numerators = Aggregations.aggregateByTags(tags, numerators, (a, b) -> + Aggregations.joinInterpolating(a, b, (v1, v2) -> v1 + v2) ); - denominators = Aggregations.aggregateByTags(tags,denominators,(a, b) -> - Aggregations.joinInterpolating(a,b, (v1,v2) -> v1 + v2) + denominators = Aggregations.aggregateByTags(tags, denominators, (a, b) -> + Aggregations.joinInterpolating(a, b, (v1, v2) -> v1 + v2) ); } Map> averages = divideCounters(start, numerators, denominators); - if(rawOuput != null) { + if (rawOuput != null) { influx.writePoints(rawOuput.getDatabase(), rawOuput.getRetention(), rawOuput.getMeasurement(), averages); } @@ -71,13 +73,13 @@ public Map> fetch(long intervalMillis, long star private Map> divideCounters(long start, Map> numerators, Map> denominators) { Map> averages = new HashMap<>(); - for(TagValues tags : numerators.keySet()) { - if(denominators.containsKey(tags)) { + for (TagValues tags : numerators.keySet()) { + if (denominators.containsKey(tags)) { List joined = Aggregations.joinInterpolating( Transformations.rate(numerators.get(tags), Duration.ofHours(1)), Transformations.rate(denominators.get(tags), Duration.ofHours(1)), - (num,denom) -> { - if(denom > 0) { + (num, denom) -> { + if (denom > 0) { return num / denom; } else { return null; diff --git a/src/main/java/de/novatec/baselining/datasources/GaugeDataSource.java b/src/main/java/de/novatec/baselining/datasources/GaugeDataSource.java index fc201e5..a978c56 100755 --- a/src/main/java/de/novatec/baselining/datasources/GaugeDataSource.java +++ b/src/main/java/de/novatec/baselining/datasources/GaugeDataSource.java @@ -1,17 +1,18 @@ package de.novatec.baselining.datasources; -import de.novatec.baselining.*; +import de.novatec.baselining.InfluxAccess; import de.novatec.baselining.config.baselines.GaugeBaselineDefinition; import de.novatec.baselining.config.measurement.MeasurementName; import de.novatec.baselining.data.AggregatePoint; -import de.novatec.baselining.data.transformations.Aggregations; import de.novatec.baselining.data.DataPoint; import de.novatec.baselining.data.TagValues; +import de.novatec.baselining.data.transformations.Aggregations; import de.novatec.baselining.data.transformations.Transformations; import lombok.extern.slf4j.Slf4j; -import java.util.*; -import java.util.stream.Collectors; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; @Slf4j public class GaugeDataSource implements BaselineDataSource { @@ -29,7 +30,7 @@ public class GaugeDataSource implements BaselineDataSource { public GaugeDataSource(InfluxAccess influx, GaugeBaselineDefinition settings) { this.influx = influx; - this.query = "SELECT MEAN("+settings.getInput().getField()+") FROM "+settings.getInput().getFullMeasurementName(); + this.query = "SELECT MEAN(" + settings.getInput().getField() + ") FROM " + settings.getInput().getFullMeasurementName(); this.tags = settings.getTags(); this.samplePrecisionMillis = settings.getSamplePrecision().toMillis(); this.rawOuput = settings.getLoopBackMetric(); @@ -40,10 +41,10 @@ public Map> fetch(long intervalMillis, long star long start = startInterval * intervalMillis; long end = endInterval * intervalMillis; - Map> rawPoints = influx.queryAggregate(query,start,end,samplePrecisionMillis); + Map> rawPoints = influx.queryAggregate(query, start, end, samplePrecisionMillis); - if(tags != null) { - rawPoints = Aggregations.aggregateByTags(tags,rawPoints,(a, b) -> { + if (tags != null) { + rawPoints = Aggregations.aggregateByTags(tags, rawPoints, (a, b) -> { ArrayList combined = new ArrayList<>(a); combined.addAll(b); return combined; @@ -51,7 +52,7 @@ public Map> fetch(long intervalMillis, long star } - if(rawOuput != null) { + if (rawOuput != null) { influx.writePoints(rawOuput.getDatabase(), rawOuput.getRetention(), rawOuput.getMeasurement(), rawPoints); } diff --git a/src/main/java/de/novatec/baselining/datasources/RateBaselineSource.java b/src/main/java/de/novatec/baselining/datasources/RateBaselineSource.java index 1f88a29..e30296c 100755 --- a/src/main/java/de/novatec/baselining/datasources/RateBaselineSource.java +++ b/src/main/java/de/novatec/baselining/datasources/RateBaselineSource.java @@ -3,7 +3,9 @@ import de.novatec.baselining.InfluxAccess; import de.novatec.baselining.config.baselines.OutlierRemovalSettings; import de.novatec.baselining.config.baselines.RateBaselineDefinition; -import de.novatec.baselining.data.*; +import de.novatec.baselining.data.AggregatePoint; +import de.novatec.baselining.data.DataPoint; +import de.novatec.baselining.data.TagValues; import de.novatec.baselining.data.transformations.Aggregations; import lombok.extern.slf4j.Slf4j; import org.apache.commons.math3.stat.descriptive.rank.Percentile; @@ -25,7 +27,7 @@ public class RateBaselineSource implements BaselineDataSource { public RateBaselineSource(InfluxAccess influx, RateBaselineDefinition settings) { this.influx = influx; - this.query = "SELECT "+settings.getField()+" FROM "+settings.getInput(); + this.query = "SELECT " + settings.getField() + " FROM " + settings.getInput(); this.tags = settings.getTags(); this.outlierRemovalConfig = settings.getOutliers(); } @@ -42,7 +44,7 @@ public Map> fetch(long intervalMillis, long star double intervallToHours = 60 * 60 * 1000.0 / intervalMillis; Map> result = new HashMap<>(); - filteredPoints.forEach((tagValues,values) -> { + filteredPoints.forEach((tagValues, values) -> { List aggregated = Aggregations .byIntervall(values, intervalMillis, AggregatePoint::from).stream() .map(pt -> new AggregatePoint(pt.getTime(), @@ -51,11 +53,10 @@ public Map> fetch(long intervalMillis, long star 1)) .map(pt -> pt.shift(-intervalMillis / 2)) .collect(Collectors.toList()); - result.put(tagValues,aggregated); + result.put(tagValues, aggregated); }); - return result; } @@ -64,12 +65,12 @@ private Map> fetchFilteredData(long intervallMillis, long end = endIntervall * intervallMillis; long outlierWindowSize = outlierRemovalConfig.getWindow().toMillis(); - long startWithHalo = start - outlierWindowSize/2; - long endWithHalo = end + outlierWindowSize/2; + long startWithHalo = start - outlierWindowSize / 2; + long endWithHalo = end + outlierWindowSize / 2; Map> rawPoints = influx.querySingleField(query, startWithHalo, endWithHalo); - if(tags != null) { - rawPoints = Aggregations.aggregateByTags(tags,rawPoints,(a, b) -> { + if (tags != null) { + rawPoints = Aggregations.aggregateByTags(tags, rawPoints, (a, b) -> { ArrayList combined = new ArrayList<>(a); combined.addAll(b); return combined; @@ -79,20 +80,20 @@ private Map> fetchFilteredData(long intervallMillis, return rawPoints.entrySet().stream() .collect(Collectors.toMap(Map.Entry::getKey, - points -> filterOutliers(points.getValue(),intervallMillis,startIntervall, endIntervall))); + points -> filterOutliers(points.getValue(), intervallMillis, startIntervall, endIntervall))); } - private List filterOutliers( Collection data, long intervallMillis, long startIntervall, long endIntervall) { + private List filterOutliers(Collection data, long intervallMillis, long startIntervall, long endIntervall) { List result = new ArrayList<>(); TreeMap pointsSorted = new TreeMap<>(); data.forEach(pt -> pointsSorted.put(pt.getTime(), pt)); - for(long intervall = startIntervall; intervall filterOutliers( Collection data, long interva double percentileValue = Double.MAX_VALUE; Collection windowPoints = pointsSorted.subMap(windowStart, true, windowEnd, false).values(); - if(windowPoints.size() >= outlierRemovalConfig.getMinPointCount()) { + if (windowPoints.size() >= outlierRemovalConfig.getMinPointCount()) { double[] valuesArray = windowPoints.stream().mapToDouble(DataPoint::getValue).toArray(); Percentile p = new Percentile(outlierRemovalConfig.getPercentile() * 100); p.setData(valuesArray);