Allow URLs to shape files on command line (matsim-org#2987)

* allow using urls in ShpOptions

* more efficient geometry joining, update tests

contribs/application/src/main/java/org/matsim/application/options/ShpOptions.java

@@ -7,9 +7,7 @@
 import org.geotools.data.shapefile.ShapefileDataStore;
 import org.geotools.data.shapefile.ShapefileDataStoreFactory;
 import org.geotools.referencing.CRS;
-import org.locationtech.jts.geom.Coordinate;
-import org.locationtech.jts.geom.Envelope;
-import org.locationtech.jts.geom.Geometry;
+import org.locationtech.jts.geom.*;
 import org.locationtech.jts.index.strtree.AbstractNode;
 import org.locationtech.jts.index.strtree.Boundable;
 import org.locationtech.jts.index.strtree.ItemBoundable;
@@ -20,20 +18,27 @@
 import org.matsim.core.utils.geometry.geotools.MGC;
 import org.matsim.core.utils.geometry.transformations.TransformationFactory;
 import org.matsim.core.utils.gis.ShapeFileReader;
+import org.matsim.core.utils.io.IOUtils;
 import org.opengis.feature.simple.SimpleFeature;
 import org.opengis.feature.simple.SimpleFeatureType;
 import org.opengis.referencing.FactoryException;
 import org.opengis.referencing.crs.CoordinateReferenceSystem;
-import javax.annotation.Nullable;
 import picocli.CommandLine;
+
+import javax.annotation.Nullable;
 import java.io.IOException;
 import java.io.UncheckedIOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.net.URL;
 import java.nio.charset.Charset;
-import java.nio.file.FileSystem;
-import java.nio.file.FileSystems;
-import java.nio.file.Files;
 import java.nio.file.Path;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Set;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipInputStream;
 
 /**
  * Reusable class for shape file options.
@@ -45,7 +50,7 @@ public final class ShpOptions {
 	private static final Logger log = LogManager.getLogger(ShpOptions.class);
 
 	@CommandLine.Option(names = "--shp", description = "Optional path to shape file used for filtering", required = false)
-	private Path shp;
+	private String shp;
 
 	@CommandLine.Option(names = "--shp-crs", description = "Overwrite coordinate system of the shape file")
 	private String shpCrs;
@@ -60,22 +65,72 @@ public ShpOptions() {
 	 * Constructor to use shape options manually.
 	 */
 	public ShpOptions(Path shp, @Nullable String shpCrs, @Nullable Charset shpCharset) {
+		this.shp = shp == null ? null : shp.toString();
+		this.shpCrs = shpCrs;
+		this.shpCharset = shpCharset;
+	}
+
+	/**
+	 * Constructor to use shape options manually.
+	 */
+	public ShpOptions(String shp, @Nullable String shpCrs, @Nullable Charset shpCharset) {
 		this.shp = shp;
 		this.shpCrs = shpCrs;
 		this.shpCharset = shpCharset;
 	}
 
+	/**
+	 * Opens datastore to a shape-file.
+	 */
+	public static ShapefileDataStore openDataStore(String shp) throws IOException {
+
+		FileDataStoreFactorySpi factory = new ShapefileDataStoreFactory();
+
+		URL url = IOUtils.resolveFileOrResource(shp);
+
+		ShapefileDataStore ds;
+		if (shp.endsWith(".shp"))
+			ds = (ShapefileDataStore) factory.createDataStore(url);
+		else if (shp.endsWith(".zip")) {
+
+			// Zip files will only work with local files
+			URI uri;
+			try (ZipInputStream zip = new ZipInputStream(IOUtils.getInputStream(url))) {
+
+				ZipEntry entry;
+				while ((entry = zip.getNextEntry()) != null) {
+					if (entry.getName().endsWith(".shp"))
+						break;
+				}
+
+				if (entry == null)
+					throw new IllegalArgumentException("No .shp file found in the zip.");
+
+				log.info("Using {} from {}", entry.getName(), shp);
+				uri = new URI("jar:" + url + "!/" + entry.getName());
+			} catch (URISyntaxException e) {
+				throw new IllegalArgumentException("Could not create URI for zip file: " + url, e);
+			}
+
+			ds = (ShapefileDataStore) factory.createDataStore(uri.toURL());
+		} else {
+			throw new IllegalArgumentException("Shape file must either be .zip or .shp, but was: " + shp);
+		}
+
+		return ds;
+	}
+
 	/**
 	 * Return whether a shape was set.
 	 */
 	public boolean isDefined() {
-		return shp != null && !shp.toString().isBlank() && !shp.toString().equals("none");
+		return shp != null && !shp.isBlank() && !shp.equals("none");
 	}
 
 	/**
 	 * Get the provided input crs.
 	 */
-	public Path getShapeFile() {
+	public String getShapeFile() {
 		return shp;
 	}
 
@@ -94,8 +149,6 @@ public String getShapeCrs() {
 	public List<SimpleFeature> readFeatures() {
 		if (shp == null)
 			throw new IllegalStateException("Shape file path not specified");
-		if (!Files.exists(shp))
-			throw new IllegalStateException(String.format("Shape file %s does not exists", shp));
 
 		try {
 			ShapefileDataStore ds = openDataStore(shp);
@@ -108,57 +161,33 @@ public List<SimpleFeature> readFeatures() {
 		}
 	}
 
-	/**
-	 * Opens datastore to a shape-file.
-	 */
-	public static ShapefileDataStore openDataStore(Path shp) throws IOException {
-
-		FileDataStoreFactorySpi factory = new ShapefileDataStoreFactory();
-
-		ShapefileDataStore ds;
-		if (shp.toString().endsWith(".shp"))
-			ds = (ShapefileDataStore) factory.createDataStore(shp.toUri().toURL());
-		else if (shp.toString().endsWith(".zip")) {
-
-			FileSystem fs = FileSystems.newFileSystem(shp, ClassLoader.getSystemClassLoader());
-			Optional<Path> match = Files.walk(fs.getPath("/"))
-					.filter(p -> p.toString().endsWith(".shp"))
-					.findFirst();
-
-			if (match.isEmpty())
-				throw new IllegalArgumentException("No .shp file found in the zip.");
-
-			log.info("Using {} from {}", match.get(), shp);
-			ds = (ShapefileDataStore) factory.createDataStore(match.get().toUri().toURL());
-		} else {
-			throw new IllegalArgumentException("Shape file must either be .zip or .shp, but was: " + shp);
-		}
-
-		return ds;
-	}
-
 	/**
 	 * Return the union of all geometries in the shape file.
 	 */
 	public Geometry getGeometry() {
 
 		Collection<SimpleFeature> features = readFeatures();
-		if (features.size() < 1) {
+		if (features.isEmpty()) {
 			throw new IllegalStateException("There is no feature in the shape file. Aborting...");
 		}
-		Geometry geometry = (Geometry) features.iterator().next().getDefaultGeometry();
-		if (features.size() > 1) {
-			for (SimpleFeature simpleFeature : features) {
-				if (simpleFeature.getDefaultGeometry() == null) {
-					log.warn("Features {} has no geometry", simpleFeature);
-					continue;
-				}
 
-				Geometry subArea = (Geometry) simpleFeature.getDefaultGeometry();
-				geometry = geometry.union(subArea);
-			}
+		if (features.size() == 1) {
+            return (Geometry) features.iterator().next().getDefaultGeometry();
+		}
+
+		GeometryFactory factory = ((Geometry) features.iterator().next().getDefaultGeometry()).getFactory();
+
+		GeometryCollection geometryCollection = (GeometryCollection) factory.buildGeometry(
+			features.stream()
+				.filter(f -> f.getDefaultGeometry() != null)
+				.map(f -> (Geometry) f.getDefaultGeometry()).toList()
+		);
+
+		if (geometryCollection.isEmpty()) {
+			throw new IllegalStateException("There are noe geometries in the shape file.");
 		}
-		return geometry;
+
+		return geometryCollection.union();
 	}
 
 	/**
@@ -172,8 +201,6 @@ public Index createIndex(String queryCRS, String attr, Set<String> filter) {
 
 		if (!isDefined())
 			throw new IllegalStateException("Shape file path not specified");
-		if (!Files.exists(shp))
-			throw new IllegalStateException(String.format("Shape file %s does not exists", shp));
 		if (queryCRS == null)
 			throw new IllegalArgumentException("Query crs must not be null!");
 
@@ -242,8 +269,7 @@ public final class Index {
 		 * @param ct   coordinate transform from query to target crs
 		 * @param attr attribute for the result of {@link #query(Coord)}
 		 */
-		Index(CoordinateTransformation ct, String attr, @Nullable Set<String> filter)
-				throws IOException {
+		Index(CoordinateTransformation ct, String attr, @Nullable Set<String> filter) throws IOException {
 			ShapefileDataStore ds = openDataStore(shp);
 
 			if (shpCharset != null)

contribs/application/src/main/java/org/matsim/application/prepare/ShapeFileTextLookup.java

@@ -59,15 +59,10 @@ public Integer call() throws Exception {
             return 1;
         }
 
-        if (shp.getShapeFile() == null) {
+        if (!shp.isDefined()) {
             throw new IllegalArgumentException("Shape file must be given!");
         }
 
-        if (!Files.exists(shp.getShapeFile())) {
-            log.error("Shape file {} does not exists", shp.getShapeFile());
-            return 1;
-        }
-
         List<SimpleFeature> features = shp.readFeatures();
 
         Map<String, SimpleFeature> map = new HashMap<>();

contribs/application/src/main/java/org/matsim/application/prepare/network/CreateNetworkFromSumo.java

@@ -74,7 +74,7 @@ public static void main(String[] args) {
 	@Override
 	public Integer call() throws Exception {
 
-		SumoNetworkConverter converter = SumoNetworkConverter.newInstance(input, output, shp.getShapeFile(), crs.getInputCRS(), crs.getTargetCRS(), freeSpeedFactor);
+		SumoNetworkConverter converter = SumoNetworkConverter.newInstance(input, output, Path.of(shp.getShapeFile()), crs.getInputCRS(), crs.getTargetCRS(), freeSpeedFactor);
 
 		Network network = NetworkUtils.createNetwork();
         Lanes lanes = LanesUtils.createLanesContainer();

contribs/application/src/main/java/org/matsim/freightDemandGeneration/FreightDemandGeneration.java

@@ -196,7 +196,7 @@ public Integer call() throws IOException, InvalidAttributeValueException, Execut
 
 		Collection<SimpleFeature> polygonsInShape = null;
 		shp = new ShpOptions(shapeFilePath, shapeCRS, null);
-		if (shp.getShapeFile() != null && Files.exists(shp.getShapeFile())) {
+		if (shp.isDefined()) {
 			log.warn("Use of shpFile. Locations for the carriers and the demand only in shp: " + shp.getShapeFile());
 			polygonsInShape = shp.readFeatures();
 			crsTransformationFromNetworkToShape = shp.createTransformation(networkCRS);

contribs/application/src/test/java/org/matsim/application/options/ShpOptionsTest.java

@@ -1,6 +1,6 @@
 package org.matsim.application.options;
 
-import org.junit.Assert;
+import org.assertj.core.data.Offset;
 import org.junit.Assume;
 import org.junit.Rule;
 import org.junit.Test;
@@ -60,7 +60,7 @@ public void all() {
 		List<SimpleFeature> ft = index.getAll();
 
 		assertThat(ft)
-				.hasSize(578)
+				.hasSize(4906)
 				.hasSize(Set.copyOf(ft).size());
 
 	}
@@ -77,15 +77,9 @@ public void testGetGeometry() {
 
 		ShpOptions shp = new ShpOptions(input, null, null);
 		Geometry geometry = shp.getGeometry() ;
-		Geometry expectedGeometry = new GeometryFactory().createEmpty(2);
 
-		List<SimpleFeature> features = shp.readFeatures();
+		assertThat(geometry.getArea())
+			.isCloseTo(1.9847543618489646E-4, Offset.offset(1e-8));
 
-		for(SimpleFeature feature : features) {
-			Geometry geometryToJoin = (Geometry) feature.getDefaultGeometry();
-			expectedGeometry = expectedGeometry.union(geometryToJoin);
-		}
-
-		Assert.assertTrue(geometry.equals(expectedGeometry));
 	}
-}
+}

matsim/src/main/java/org/matsim/core/utils/io/IOUtils.java

@@ -207,7 +207,7 @@ public static void setZstdCompressionLevel(int level) {
 	public static URL resolveFileOrResource(String filename) throws UncheckedIOException {
 		try {
 			// I) do not handle URLs
-			if (filename.startsWith("jar:file:") || filename.startsWith("file:") || filename.startsWith( "https:" )) {
+			if (filename.startsWith("jar:file:") || filename.startsWith("file:") || filename.startsWith( "https:" ) || filename.startsWith( "http:" )) {
 				// looks like an URI
 				return new URL(filename);
 			}