Skip to content

Commit

Permalink
Merge pull request #425 from NOAA-OWP/issue404
Browse files Browse the repository at this point in the history 
Filter declared values from the time-series used in event detection, …
  • Loading branch information
@james-d-brown
james-d-brown authored Feb 19, 2025
2 parents b5612ec + c80ff77 commit 5dd62f5
Show file tree
Hide file tree
Showing 4 changed files with 138 additions and 84 deletions.
29 changes: 25 additions & 4 deletions src/wres/pipeline/pooling/EventsGenerator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
import java.util.Set;
import java.util.TreeSet;
import java.util.concurrent.atomic.AtomicLong;
import java.util.function.DoubleUnaryOperator;
import java.util.function.Function;
import java.util.function.Predicate;
import java.util.function.UnaryOperator;
Expand All @@ -28,11 +29,13 @@
import wres.config.yaml.components.LeadTimeInterval;
import wres.config.yaml.components.TimeInterval;
import wres.config.yaml.components.TimeWindowAggregation;
import wres.datamodel.Slicer;
import wres.datamodel.scale.TimeScaleOuter;
import wres.datamodel.space.Feature;
import wres.datamodel.space.FeatureGroup;
import wres.datamodel.space.FeatureTuple;
import wres.datamodel.time.RescaledTimeSeriesPlusValidation;
import wres.datamodel.time.TimeSeriesSlicer;
import wres.datamodel.time.TimeSeriesUpscaler;
import wres.datamodel.time.TimeWindowOuter;
import wres.datamodel.time.TimeSeries;
Expand Down Expand Up @@ -640,12 +643,30 @@ private Set<TimeWindowOuter> adjustTimeSeriesAndDetectEvents( TimeSeries<Double>
// Rescale the time-series if needed
TimeSeries<Double> adjusted = this.getRescaledTimeSeries( timeSeries, details, upscaler );

LOGGER.debug( "Performing event detection of a time-series dataset containing {} events and the following "
+ "metadata: {}.", adjusted.getEvents()
.size(), adjusted.getMetadata() );
// Filter event values, if necessary
DoubleUnaryOperator valueTransformer = Slicer.getValueTransformer( details.declaration()
.values() );
TimeSeries<Double> transformed = TimeSeriesSlicer.transform( adjusted,
valueTransformer::applyAsDouble,
m -> m );

if ( LOGGER.isDebugEnabled() )
{
LOGGER.debug( "Following rescaling and filtering by value, discovered {} time-series values for event "
+ "detection. The original time-series contained {} values. Value filtering removed {} "
+ "values from the rescaled time-series. The time-series metadata is: {}.",
transformed.getEvents()
.size(),
timeSeries.getEvents()
.size(),
adjusted.getEvents()
.size() - transformed.getEvents()
.size(),
transformed.getMetadata() );
}

return this.eventDetector()
.detect( adjusted );
.detect( transformed );
}

/**
Expand Down
74 changes: 3 additions & 71 deletions src/wres/pipeline/pooling/PoolFactory.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,6 @@
import wres.config.yaml.components.Offset;
import wres.config.yaml.components.Season;
import wres.config.yaml.components.Source;
import wres.config.yaml.components.Values;
import wres.datamodel.time.TimeWindowSlicer;
import wres.datamodel.types.Ensemble;
import wres.datamodel.types.Ensemble.Labels;
Expand Down Expand Up @@ -627,7 +626,7 @@ else if ( method == GeneratedBaselines.CLIMATOLOGY )
Map.Entry::getValue ) );
Function<Geometry, DoubleUnaryOperator> leftOffsetGenerator =
this.getOffsetTransformer( leftOffsets, DatasetOrientation.LEFT );
DoubleUnaryOperator valueTransformer = this.getValueTransformer( declaration.values() );
DoubleUnaryOperator valueTransformer = Slicer.getValueTransformer( declaration.values() );
UnaryOperator<TimeSeries<Double>> leftValueTransformer =
this.getValueTransformer( leftOffsetGenerator, valueTransformer );

Expand Down Expand Up @@ -790,7 +789,7 @@ private List<SupplierWithPoolRequest<Pool<TimeSeries<Pair<Double, Ensemble>>>>>
this.getOffsetTransformer( leftOffsets, DatasetOrientation.LEFT );


DoubleUnaryOperator leftValueTransformer = this.getValueTransformer( declaration.values() );
DoubleUnaryOperator leftValueTransformer = Slicer.getValueTransformer( declaration.values() );
UnaryOperator<TimeSeries<Double>> leftValueAndUnitTransformer =
this.getValueTransformer( leftOffsetGenerator, leftValueTransformer );

Expand Down Expand Up @@ -972,7 +971,7 @@ private List<SupplierWithPoolRequest<Pool<TimeSeries<Pair<Double, Ensemble>>>>>
Function<Geometry, DoubleUnaryOperator> leftOffsetGenerator =
this.getOffsetTransformer( leftOffsets, DatasetOrientation.LEFT );

DoubleUnaryOperator leftValueTransformer = this.getValueTransformer( declaration.values() );
DoubleUnaryOperator leftValueTransformer = Slicer.getValueTransformer( declaration.values() );
UnaryOperator<TimeSeries<Double>> leftValueAndUnitTransformer =
this.getValueTransformer( leftOffsetGenerator, leftValueTransformer );

Expand Down Expand Up @@ -1585,73 +1584,6 @@ private UnaryOperator<Event<Ensemble>> getEnsembleValueTransformer( DoubleUnaryO
};
}

/**
* Returns a transformer for single-valued data if required.
*
* @param values the value declaration
* @return a transformer or null
*/

private DoubleUnaryOperator getValueTransformer( Values values )
{
if ( Objects.isNull( values ) )
{
return value -> value;
}

double assignToLowMiss = MissingValues.DOUBLE;
double assignToHighMiss = MissingValues.DOUBLE;

double minimum = Double.NEGATIVE_INFINITY;
double maximum = Double.POSITIVE_INFINITY;

if ( Objects.nonNull( values.belowMinimum() ) )
{
assignToLowMiss = values.belowMinimum();
}

if ( Objects.nonNull( values.aboveMaximum() ) )
{
assignToHighMiss = values.aboveMaximum();
}

if ( Objects.nonNull( values.minimum() ) )
{
minimum = values.minimum();
}

if ( Objects.nonNull( values.maximum() ) )
{
maximum = values.maximum();
}

// Effectively final constants for use
// within enclosing scope
double assignLow = assignToLowMiss;
double assignHigh = assignToHighMiss;

double low = minimum;
double high = maximum;

return toTransform -> {

// Low miss
if ( toTransform < low )
{
return assignLow;
}

// High miss
if ( toTransform > high )
{
return assignHigh;
}

// Within bounds
return toTransform;
};
}

/**
* Returns a transformer for ensemble data if required.
*
Expand Down
68 changes: 68 additions & 0 deletions wres-datamodel/src/wres/datamodel/Slicer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
import org.slf4j.LoggerFactory;

import wres.config.yaml.components.DatasetOrientation;
import wres.config.yaml.components.Values;
import wres.datamodel.types.Climatology;
import wres.datamodel.types.Ensemble;
import wres.datamodel.types.Ensemble.Labels;
Expand Down Expand Up @@ -492,6 +493,73 @@ public static Map<Integer, List<Pair<Double, Ensemble>>> filterByRightSize( List
.size() ) );
}

/**
* Returns a transformer for single-valued data.
*
* @param values the value declaration, possibly null (for identity operator)
* @return a transformer
*/

public static DoubleUnaryOperator getValueTransformer( Values values )
{
if ( Objects.isNull( values ) )
{
return value -> value;
}

double assignToLowMiss = MissingValues.DOUBLE;
double assignToHighMiss = MissingValues.DOUBLE;

double minimum = Double.NEGATIVE_INFINITY;
double maximum = Double.POSITIVE_INFINITY;

if ( Objects.nonNull( values.belowMinimum() ) )
{
assignToLowMiss = values.belowMinimum();
}

if ( Objects.nonNull( values.aboveMaximum() ) )
{
assignToHighMiss = values.aboveMaximum();
}

if ( Objects.nonNull( values.minimum() ) )
{
minimum = values.minimum();
}

if ( Objects.nonNull( values.maximum() ) )
{
maximum = values.maximum();
}

// Effectively final constants for use
// within enclosing scope
double assignLow = assignToLowMiss;
double assignHigh = assignToHighMiss;

double low = minimum;
double high = maximum;

return toTransform -> {

// Low miss
if ( toTransform < low )
{
return assignLow;
}

// High miss
if ( toTransform > high )
{
return assignHigh;
}

// Within bounds
return toTransform;
};
}

/**
* <p>Discovers the unique instances of a given type of statistic. The mapper function identifies the type to
* discover. For example, to discover the unique thresholds contained in the list of outputs:</p>
Expand Down
51 changes: 42 additions & 9 deletions wres-datamodel/test/wres/datamodel/SlicerTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,13 +18,16 @@
import org.apache.commons.lang3.tuple.Pair;
import org.junit.jupiter.api.Test;

import static org.junit.jupiter.api.Assertions.assertAll;
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;

import wres.config.yaml.components.ThresholdOperator;
import wres.config.yaml.components.Values;
import wres.config.yaml.components.ValuesBuilder;
import wres.datamodel.types.Climatology;
import wres.datamodel.types.Ensemble;
import wres.datamodel.types.Ensemble.Labels;
Expand Down Expand Up @@ -424,17 +427,17 @@ void testFilterListOfMetricOutputs()
TimeWindowOuter windowOne = TimeWindowOuter.of( MessageUtilities.getTimeWindow( Instant.MIN,
Instant.MAX,
Duration.ofHours(
1 ) ) );
1 ) ) );

TimeWindowOuter windowTwo = TimeWindowOuter.of( MessageUtilities.getTimeWindow( Instant.MIN,
Instant.MAX,
Duration.ofHours(
2 ) ) );
2 ) ) );

TimeWindowOuter windowThree = TimeWindowOuter.of( MessageUtilities.getTimeWindow( Instant.MIN,
Instant.MAX,
Duration.ofHours(
3 ) ) );
3 ) ) );

OneOrTwoThresholds thresholdOne =
OneOrTwoThresholds.of( ThresholdOuter.of( OneOrTwoDoubles.of( 1.0 ),
Expand Down Expand Up @@ -527,17 +530,17 @@ void testDiscoverListOfMetricOutputs()
TimeWindowOuter windowOne = TimeWindowOuter.of( MessageUtilities.getTimeWindow( Instant.MIN,
Instant.MAX,
Duration.ofHours(
1 ) ) );
1 ) ) );

TimeWindowOuter windowTwo = TimeWindowOuter.of( MessageUtilities.getTimeWindow( Instant.MIN,
Instant.MAX,
Duration.ofHours(
2 ) ) );
2 ) ) );

TimeWindowOuter windowThree = TimeWindowOuter.of( MessageUtilities.getTimeWindow( Instant.MIN,
Instant.MAX,
Duration.ofHours(
2 ) ) );
2 ) ) );

OneOrTwoThresholds thresholdOne =
OneOrTwoThresholds.of( ThresholdOuter.of( OneOrTwoDoubles.of( 1.0 ),
Expand Down Expand Up @@ -691,12 +694,12 @@ void testSortStatisticsByTimeWindowAndThreshold()
TimeWindowOuter windowOne = TimeWindowOuter.of( MessageUtilities.getTimeWindow( Instant.MIN,
Instant.MAX,
Duration.ofHours(
1 ) ) );
1 ) ) );

TimeWindowOuter windowTwo = TimeWindowOuter.of( MessageUtilities.getTimeWindow( Instant.MIN,
Instant.MAX,
Duration.ofHours(
2 ) ) );
2 ) ) );

TimeWindowOuter windowThree = TimeWindowOuter.of( MessageUtilities.getTimeWindow( Instant.MIN,
Instant.MAX,
Expand Down Expand Up @@ -843,4 +846,34 @@ void testRounderProducesInputValueWhenInputIsNotFinite()

assertEquals( Double.NaN, slicer.applyAsDouble( Double.NaN ) );
}
}

@Test
void testValueTransformer()
{
Values valuesOne = ValuesBuilder.builder()
.maximum( 23.0 )
.aboveMaximum( 99.0 )
.build();
DoubleUnaryOperator transformerOne = Slicer.getValueTransformer( valuesOne );

Values valuesTwo = ValuesBuilder.builder()
.minimum( 17.0 )
.belowMinimum( 12.0 )
.build();
DoubleUnaryOperator transformerTwo = Slicer.getValueTransformer( valuesTwo );

Values valuesThree = ValuesBuilder.builder()
.minimum( 17.0 )
.belowMinimum( 12.0 )
.maximum( 23.0 )
.aboveMaximum( 99.0 )
.build();
DoubleUnaryOperator transformerThree = Slicer.getValueTransformer( valuesThree );

assertAll( () -> assertEquals( 99.0, transformerOne.applyAsDouble( 24.0 ), 1 ),
() -> assertEquals( 12.0, transformerTwo.applyAsDouble( 16.0 ), 1 ),
() -> assertEquals( 12.0, transformerThree.applyAsDouble( 16.0 ), 1 ),
() -> assertEquals( 99.0, transformerThree.applyAsDouble( 24.0 ), 1 ) );
}

}

0 comments on commit 5dd62f5

Please sign in to comment.