Support feature-specific offsets to allow for manual datum adjustment…

…s or similar constant adjustments, #36.
NOAA-OWP · Aug 16, 2024 · 335b811 · 335b811
1 parent b1a52a7
commit 335b811
Show file tree

Hide file tree

Showing 34 changed files with 741 additions and 154 deletions.
diff --git a/src/wres/pipeline/pooling/PoolFactory.java b/src/wres/pipeline/pooling/PoolFactory.java
diff --git a/test/wres/pipeline/EvaluationUtilitiesTest.java b/test/wres/pipeline/EvaluationUtilitiesTest.java
@@ -15,6 +15,7 @@
 import wres.config.yaml.components.EvaluationDeclaration;
 import wres.config.yaml.components.EvaluationDeclarationBuilder;
 import wres.config.yaml.components.FeatureGroups;
+import wres.config.yaml.components.FeatureGroupsBuilder;
 import wres.config.yaml.components.LeadTimeInterval;
 import wres.config.yaml.components.ThresholdBuilder;
 import wres.config.yaml.components.ThresholdType;
@@ -261,7 +262,9 @@ void testGetSummaryStatisticsCalculatorsWithTwoTimeWindowsAcrossFeatureGroups()
 
         Set<GeometryGroup> geometryGroups = Set.of( firstGroup, secondGroup );
 
-        FeatureGroups featureGroups = new FeatureGroups( geometryGroups );
+        FeatureGroups featureGroups = FeatureGroupsBuilder.builder()
+                                                          .geometryGroups( geometryGroups )
+                                                          .build();
 
         EvaluationDeclaration evaluation = EvaluationDeclarationBuilder.builder()
                                                                        .leadTimes( leadTimeInterval )

diff --git a/wres-config/nonsrc/schema.yml b/wres-config/nonsrc/schema.yml
@@ -566,9 +566,9 @@ definitions:
     cross-pairing chosen."
     anyOf:
       - "$ref": "#/definitions/CrossPairEnum"
-      - "$ref": "#/definitions/CrossPairFull"
+      - "$ref": "#/definitions/CrossPairWithParameters"
 
-  CrossPairFull:
+  CrossPairWithParameters:
     type: object
     additionalProperties: false
     properties:
@@ -665,25 +665,25 @@ definitions:
     properties:
       observed:
         anyOf:
-          - "$ref": "#/definitions/FeatureTupleFull"
+          - "$ref": "#/definitions/FeatureWithParameters"
           - type: string
             minLength: 1
             maxLength: 32
       predicted:
         anyOf:
-          - "$ref": "#/definitions/FeatureTupleFull"
+          - "$ref": "#/definitions/FeatureWithParameters"
           - type: string
             minLength: 1
             maxLength: 32
       baseline:
         anyOf:
-          - "$ref": "#/definitions/FeatureTupleFull"
+          - "$ref": "#/definitions/FeatureWithParameters"
           - type: string
             minLength: 1
             maxLength: 32
 
-  FeatureTupleFull:
-    title: A feature tuple with additional properties
+  FeatureWithParameters:
+    title: A feature with additional properties
     type: object
     additionalProperties: false
     properties:
@@ -695,6 +695,8 @@ definitions:
         type: string
         minLength: 1
         maxLength: 2083
+      offset:
+        type: number
     required:
       - name
 

diff --git a/wres-config/src/wres/config/yaml/DeclarationFactory.java b/wres-config/src/wres/config/yaml/DeclarationFactory.java
@@ -165,11 +165,11 @@ public class DeclarationFactory
                                      .enable( DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY )
                                      .enable( DeserializationFeature.FAIL_ON_READING_DUP_TREE_KEY )
                                      .enable( JsonParser.Feature.STRICT_DUPLICATE_DETECTION )
+                                     .disable( DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES )
                                      .build()
                                      .registerModule( new ProtobufModule() )
                                      .registerModule( new JavaTimeModule() )
-                                     .configure( DeserializationFeature.FAIL_ON_NULL_FOR_PRIMITIVES,
-                                                 true );
+                                     .enable( DeserializationFeature.FAIL_ON_NULL_FOR_PRIMITIVES );
 
     /** Mapper for serialization. */
     private static final ObjectMapper SERIALIZER =

diff --git a/wres-config/src/wres/config/yaml/DeclarationInterpolator.java b/wres-config/src/wres/config/yaml/DeclarationInterpolator.java
@@ -34,6 +34,7 @@
 import wres.config.yaml.components.EvaluationDeclarationBuilder;
 import wres.config.yaml.components.FeatureAuthority;
 import wres.config.yaml.components.FeatureGroups;
+import wres.config.yaml.components.FeatureGroupsBuilder;
 import wres.config.yaml.components.Features;
 import wres.config.yaml.components.FeaturesBuilder;
 import wres.config.yaml.components.Formats;
@@ -486,7 +487,8 @@ private static void interpolateSparseFeatures( EvaluationDeclarationBuilder buil
                                                      .geometries();
                 Set<GeometryTuple> denseFeatures = DeclarationInterpolator.interpolateSparseFeatures( features,
                                                                                                       hasBaseline );
-                Features adjustedFeatures = new Features( denseFeatures );
+                Features adjustedFeatures = new Features( denseFeatures, builder.features()
+                                                                                .offsets() );
                 builder.features( adjustedFeatures );
             }
 
@@ -512,7 +514,9 @@ private static void interpolateSparseFeatures( EvaluationDeclarationBuilder buil
                                      .build();
                     adjustedGeoGroups.add( nextAdjustedGroup );
                 }
-                FeatureGroups adjustedFeatureGroups = new FeatureGroups( adjustedGeoGroups );
+                FeatureGroups adjustedFeatureGroups = FeatureGroupsBuilder.builder()
+                                                                          .geometryGroups( adjustedGeoGroups )
+                                                                          .build();
                 builder.featureGroups( adjustedFeatureGroups );
             }
         }

diff --git a/wres-config/src/wres/config/yaml/DeclarationMigrator.java b/wres-config/src/wres/config/yaml/DeclarationMigrator.java
@@ -82,10 +82,12 @@
 import wres.config.yaml.components.EvaluationDeclarationBuilder;
 import wres.config.yaml.components.FeatureAuthority;
 import wres.config.yaml.components.FeatureGroups;
+import wres.config.yaml.components.FeatureGroupsBuilder;
 import wres.config.yaml.components.FeatureService;
 import wres.config.yaml.components.FeatureServiceBuilder;
 import wres.config.yaml.components.FeatureServiceGroup;
 import wres.config.yaml.components.Features;
+import wres.config.yaml.components.FeaturesBuilder;
 import wres.config.yaml.components.Format;
 import wres.config.yaml.components.Formats;
 import wres.config.yaml.components.GeneratedBaseline;
@@ -293,7 +295,9 @@ private static void migrateFeatures( List<NamedFeature> features, EvaluationDecl
         if ( !features.isEmpty() )
         {
             Set<GeometryTuple> geometries = DeclarationMigrator.migrateFeatures( features );
-            Features wrappedFeatures = new Features( geometries );
+            Features wrappedFeatures = FeaturesBuilder.builder()
+                                                      .geometries( geometries )
+                                                      .build();
             builder.features( wrappedFeatures );
         }
     }
@@ -312,7 +316,9 @@ private static void migrateFeatureGroups( List<FeaturePool> featureGroups, Evalu
                     featureGroups.stream()
                                  .map( DeclarationMigrator::migrateFeatureGroup )
                                  .collect( Collectors.toSet() );
-            FeatureGroups wrappedGroups = new FeatureGroups( geometryGroups );
+            FeatureGroups wrappedGroups = FeatureGroupsBuilder.builder()
+                                                              .geometryGroups( geometryGroups )
+                                                              .build();
             builder.featureGroups( wrappedGroups );
         }
     }

diff --git a/wres-config/src/wres/config/yaml/DeclarationUtilities.java b/wres-config/src/wres/config/yaml/DeclarationUtilities.java
@@ -49,6 +49,7 @@
 import wres.config.yaml.components.EvaluationDeclarationBuilder;
 import wres.config.yaml.components.FeatureAuthority;
 import wres.config.yaml.components.FeatureGroups;
+import wres.config.yaml.components.FeatureGroupsBuilder;
 import wres.config.yaml.components.FeatureServiceGroup;
 import wres.config.yaml.components.Features;
 import wres.config.yaml.components.LeadTimeInterval;
@@ -2243,7 +2244,8 @@ private static EvaluationDeclaration removeFeaturesWithoutThresholds( Evaluation
                                                   .collect( Collectors.toSet() );
 
             // Set the new features
-            Features filteredFeatures = new Features( filtered );
+            Features filteredFeatures = new Features( filtered, declaration.features()
+                                                                           .offsets() );
             builder.features( filteredFeatures );
 
             if ( LOGGER.isWarnEnabled()
@@ -2305,7 +2307,9 @@ private static EvaluationDeclaration removeFeaturesWithoutThresholds( Evaluation
                                                   .toList() );
             }
 
-            FeatureGroups finalFeatureGroups = new FeatureGroups( adjustedGroups );
+            FeatureGroups finalFeatureGroups = FeatureGroupsBuilder.builder()
+                                                                   .geometryGroups( adjustedGroups )
+                                                                   .build();
             builder.featureGroups( finalFeatureGroups );
         }
 

diff --git a/wres-config/src/wres/config/yaml/DeclarationValidator.java b/wres-config/src/wres/config/yaml/DeclarationValidator.java
@@ -510,7 +510,6 @@ private static List<EvaluationStatusEvent> validateDatasets( EvaluationDeclarati
         // Data types are valid
         List<EvaluationStatusEvent> typesValid = DeclarationValidator.typesAreValid( declaration );
         events.addAll( typesValid );
-
         // Ensembles cannot be present on both left and right sides
         List<EvaluationStatusEvent> ensembles = DeclarationValidator.ensembleOnOneSideOnly( declaration );
         events.addAll( ensembles );

diff --git a/wres-config/src/wres/config/yaml/components/FeatureGroups.java b/wres-config/src/wres/config/yaml/components/FeatureGroups.java
@@ -1,7 +1,9 @@
 package wres.config.yaml.components;
 
 import java.util.Collections;
+import java.util.LinkedHashMap;
 import java.util.LinkedHashSet;
+import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
 import java.util.StringJoiner;
@@ -14,15 +16,18 @@
 import wres.config.yaml.deserializers.FeatureGroupsDeserializer;
 import wres.config.yaml.serializers.FeatureGroupsSerializer;
 import wres.statistics.generated.GeometryGroup;
+import wres.statistics.generated.GeometryTuple;
 
 /**
- * Geographic feature groups.
+ * Geographic feature groups and associated offset values (e.g., datum offsets). Absence of an offset for a given
+ * feature with a group implies no/zero offset.
  * @param geometryGroups the feature groups
+ * @param offsets the offset values associated with features in the group, such as a datum offset, if any
  */
 @RecordBuilder
 @JsonSerialize( using = FeatureGroupsSerializer.class )
 @JsonDeserialize( using = FeatureGroupsDeserializer.class )
-public record FeatureGroups( Set<GeometryGroup> geometryGroups )
+public record FeatureGroups( Set<GeometryGroup> geometryGroups, Map<GeometryTuple, Offset> offsets )
 {
     /**
      * Sets the default values.
@@ -40,6 +45,16 @@ public record FeatureGroups( Set<GeometryGroup> geometryGroups )
             // Immutable copy, preserving insertion order
             geometryGroups = Collections.unmodifiableSet( new LinkedHashSet<>( geometryGroups ) );
         }
+
+        if ( Objects.isNull( offsets ) )
+        {
+            offsets = Collections.emptyMap();
+        }
+        else
+        {
+            // Immutable copy, preserving insertion order
+            offsets = Collections.unmodifiableMap( new LinkedHashMap<>( offsets ) );
+        }
     }
 
     @Override

diff --git a/wres-config/src/wres/config/yaml/components/Features.java b/wres-config/src/wres/config/yaml/components/Features.java
@@ -1,7 +1,9 @@
 package wres.config.yaml.components;
 
 import java.util.Collections;
+import java.util.LinkedHashMap;
 import java.util.LinkedHashSet;
+import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
 import java.util.StringJoiner;
@@ -16,13 +18,15 @@
 import wres.statistics.generated.GeometryTuple;
 
 /**
- * Geographic features.
+ * Geographic features and associated offset values (e.g., datum offsets). Absence of an offset for a given feature
+ * implies no/zero offset.
  * @param geometries the features
+ * @param offsets the offset values, such as a datum offset, if any
  */
 @RecordBuilder
 @JsonSerialize( using = FeaturesSerializer.class )
 @JsonDeserialize( using = FeaturesDeserializer.class )
-public record Features( Set<GeometryTuple> geometries )
+public record Features( Set<GeometryTuple> geometries, Map<GeometryTuple,Offset> offsets )
 {
     /**
      * Sets the default values.
@@ -39,6 +43,16 @@ public record Features( Set<GeometryTuple> geometries )
             // Immutable copy, preserving insertion order
             geometries = Collections.unmodifiableSet( new LinkedHashSet<>( geometries ) );
         }
+
+        if ( Objects.isNull( offsets ) )
+        {
+            offsets = Collections.emptyMap();
+        }
+        else
+        {
+            // Immutable copy, preserving insertion order
+            offsets = Collections.unmodifiableMap( new LinkedHashMap<>( offsets ) );
+        }
     }
 
     @Override

diff --git a/wres-config/src/wres/config/yaml/components/Offset.java b/wres-config/src/wres/config/yaml/components/Offset.java
@@ -0,0 +1,11 @@
+package wres.config.yaml.components;
+
+/**
+ * The offset value to apply to each dataset orientation. For example, a datum offset.
+ * @param left the left-oriented offset
+ * @param right the right-oriented offset
+ * @param baseline the baseline-oriented offset
+ */
+public record Offset( double left, double right, double baseline )
+{
+}
diff --git a/wres-config/src/wres/config/yaml/deserializers/FeatureGroupsDeserializer.java b/wres-config/src/wres/config/yaml/deserializers/FeatureGroupsDeserializer.java
@@ -1,8 +1,9 @@
 package wres.config.yaml.deserializers;
 
 import java.io.IOException;
-import java.util.Collections;
+import java.util.LinkedHashMap;
 import java.util.LinkedHashSet;
+import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
 
@@ -16,7 +17,9 @@
 import org.slf4j.LoggerFactory;
 
 import wres.config.yaml.components.FeatureGroups;
+import wres.config.yaml.components.FeatureGroupsBuilder;
 import wres.config.yaml.components.Features;
+import wres.config.yaml.components.Offset;
 import wres.statistics.generated.GeometryGroup;
 import wres.statistics.generated.GeometryTuple;
 
@@ -71,14 +74,15 @@ private FeatureGroups getFeatureGroupsFromArray( ObjectReader reader,
     {
         // Preserve insertion order
         Set<GeometryGroup> featureGroups = new LinkedHashSet<>();
+        Map<GeometryTuple, Offset> featureOffsets = new LinkedHashMap<>();
+
         int nodeCount = featureGroupsNode.size();
 
         for ( int i = 0; i < nodeCount; i++ )
         {
             JsonNode nextNode = featureGroupsNode.get( i );
             String groupName = "";
             Set<GeometryTuple> geometries = null;
-
             // Group name
             if ( nextNode.has( "name" ) )
             {
@@ -95,8 +99,9 @@ private FeatureGroups getFeatureGroupsFromArray( ObjectReader reader,
                 parser.setCodec( reader );
                 Features features = FEATURES_DESERIALIZER.deserialize( parser, context );
                 geometries = features.geometries();
+                featureOffsets.putAll( features.offsets() );
                 LOGGER.debug( "Discovered the following collection of geometries associated with a feature group "
-                              + "named '{}': {}.", groupName, new Features( geometries ) );
+                              + "named '{}': {}.", groupName, features );
             }
 
             // Create the group
@@ -110,6 +115,9 @@ private FeatureGroups getFeatureGroupsFromArray( ObjectReader reader,
             }
         }
 
-        return new FeatureGroups( Collections.unmodifiableSet( featureGroups ) );
+        return FeatureGroupsBuilder.builder()
+                                   .geometryGroups( featureGroups )
+                                   .offsets( featureOffsets )
+                                   .build();
     }
 }