From f76b247e863721f365d02b7afb897cb3f46958f2 Mon Sep 17 00:00:00 2001 From: Seif Eddine GHAZOUANI Date: Mon, 13 May 2024 10:41:44 +0200 Subject: [PATCH 1/4] create new transform operation RemoveStopsOutsidePolygone: remove stops outside polygone/multiPolygone --- onebusaway-gtfs-transformer/pom.xml | 5 + .../impl/RemoveStopsOutsidePolygone.java | 99 +++++++++++++++++++ 2 files changed, 104 insertions(+) create mode 100644 onebusaway-gtfs-transformer/src/main/java/org/onebusaway/gtfs_transformer/impl/RemoveStopsOutsidePolygone.java diff --git a/onebusaway-gtfs-transformer/pom.xml b/onebusaway-gtfs-transformer/pom.xml index 08c0c818..204f61a7 100644 --- a/onebusaway-gtfs-transformer/pom.xml +++ b/onebusaway-gtfs-transformer/pom.xml @@ -77,6 +77,11 @@ com.sun.xml.bind jaxb-impl + + org.locationtech.jts + jts-core + 1.19.0 + diff --git a/onebusaway-gtfs-transformer/src/main/java/org/onebusaway/gtfs_transformer/impl/RemoveStopsOutsidePolygone.java b/onebusaway-gtfs-transformer/src/main/java/org/onebusaway/gtfs_transformer/impl/RemoveStopsOutsidePolygone.java new file mode 100644 index 00000000..ecbf39a6 --- /dev/null +++ b/onebusaway-gtfs-transformer/src/main/java/org/onebusaway/gtfs_transformer/impl/RemoveStopsOutsidePolygone.java @@ -0,0 +1,99 @@ +package org.onebusaway.gtfs_transformer.impl; + +import java.util.ArrayList; +import java.util.List; + +import java.io.Serializable; + +import org.locationtech.jts.io.ParseException; +import org.locationtech.jts.io.WKTReader; +import org.locationtech.jts.geom.*; + +import org.onebusaway.csv_entities.schema.annotations.CsvField; +import org.onebusaway.gtfs.model.IdentityBean; +import org.onebusaway.gtfs.model.Stop; +import org.onebusaway.gtfs.serialization.GtfsEntitySchemaFactory; +import org.onebusaway.gtfs.services.GtfsMutableRelationalDao; +import org.onebusaway.gtfs_transformer.factory.EntityRetentionGraph; +import org.onebusaway.gtfs_transformer.services.GtfsTransformStrategy; +import org.onebusaway.gtfs_transformer.services.TransformContext; +import org.slf4j.Logger; + +import org.slf4j.LoggerFactory; + +public class RemoveStopsOutsidePolygone implements GtfsTransformStrategy { + private final Logger log = LoggerFactory.getLogger(RemoveStopsOutsidePolygone.class); + + @CsvField(optional = true) + private String polygone; + + public void setPolygone(String polygone) { + this.polygone = polygone; + } + + @Override + public String getName() { + return this.getClass().getSimpleName(); + } + + /* + * example: + * {"op":"transform","class":"org.onebusaway.gtfs_transformer.impl.RemoveStopsOutsidePolygone","polygone":wkt_polygone ..."} + */ + + @Override + public void run(TransformContext transformContext, GtfsMutableRelationalDao gtfsMutableRelationalDao) { + Geometry geometry = buildPolygone(polygone); + EntityRetentionGraph graph = new EntityRetentionGraph(gtfsMutableRelationalDao); + graph.setRetainBlocks(false); + // browse all stops and retain only those inside polygone/multiPolygone + if (geometry.isValid() && !geometry.isEmpty()){ + for (Stop stop : gtfsMutableRelationalDao.getAllStops()) { + if (insidePolygon(geometry,stop.getLon(),stop.getLat())){ + graph.retain(stop, true); + } + } + } + + // remove non retained objects + for (Class entityClass : GtfsEntitySchemaFactory.getEntityClasses()) { + List objectsToRemove = new ArrayList(); + for (Object entity : gtfsMutableRelationalDao.getAllEntitiesForType(entityClass)) { + if (!graph.isRetained(entity)){ + objectsToRemove.add(entity); + } + } + for (Object toRemove : objectsToRemove){ + gtfsMutableRelationalDao.removeEntity((IdentityBean) toRemove); + } + } + } + + /* + * create polygone/multiPolygone from 'polygone' variable in json file + * return Geometry variable + * return null if an exception is encountered when parsing the wkt string + */ + private Geometry buildPolygone(String wktPolygone) { + WKTReader reader = new WKTReader(); + try{ + return reader.read(wktPolygone); + } catch (ParseException e){ + String message = String.format("Error parsing WKT string : %s", e.getMessage()); + log.error(message); + return null; + } + + } + /* + * insidePolygone returns boolean variable + * true: if polygone contains point + * false if point is outside polygone + */ + private boolean insidePolygon(Geometry geometry, double lon, double lat) { + GeometryFactory geometryFactory = new GeometryFactory(); + Point point = geometryFactory.createPoint(new Coordinate(lon, lat)); + return geometry.contains(point); + } + +} From 333f677903428919ab9560a920a948048aa54f99 Mon Sep 17 00:00:00 2001 From: Seif Eddine GHAZOUANI Date: Tue, 10 Dec 2024 13:35:13 +0100 Subject: [PATCH 2/4] Retain up from polygon, add test file and document strategy --- docs/onebusaway-gtfs-transformer-cli.md | 21 +++++ ...Polygone.java => RetainUpFromPolygon.java} | 35 ++++----- .../impl/RetainUpFromPolygonTest.java | 76 +++++++++++++++++++ 3 files changed, 112 insertions(+), 20 deletions(-) rename onebusaway-gtfs-transformer/src/main/java/org/onebusaway/gtfs_transformer/impl/{RemoveStopsOutsidePolygone.java => RetainUpFromPolygon.java} (76%) create mode 100644 onebusaway-gtfs-transformer/src/test/java/org/onebusaway/gtfs_transformer/impl/RetainUpFromPolygonTest.java diff --git a/docs/onebusaway-gtfs-transformer-cli.md b/docs/onebusaway-gtfs-transformer-cli.md index ec386d41..c35164f5 100644 --- a/docs/onebusaway-gtfs-transformer-cli.md +++ b/docs/onebusaway-gtfs-transformer-cli.md @@ -20,6 +20,7 @@ * [Path Expressions](#path-expressions-) * [Retain an Entity](#retain-an-entity) * [Remove an Entity](#remove-an-entity) + * [Retain Up From Polygon](#retain-up-from-polygon) * [Trim a Trip](#trim-a-trip) * [Generate Stop Times](#generate-stop-times) * [Extend Service Calendars](#extend-service-calendars) @@ -270,6 +271,26 @@ You can remove a specific entity from a feed. Note that removing an entity has a cascading effect. If you remove a trip, all the stop times that depend on that trip will also be removed. If you remove a route, all the trips and stop times for that route will be removed. + +#### Retain Up From Polygon + +Retain Up From Polygon is an operation that filters GTFS input data based on a specified geographic area, using a polygon defined in WKT (Well-Known Text) format, which is configurable in the JSON transformer snippet. + +This strategy applies two main functions: + + * **Retain Function**: retains **up** all stops, trips, and routes that are located inside the defined polygon, then the algorithm automatically applies a retain **down** to these entities. + + * **Remove Function**: any entities not retained within the polygon are removed. + +This strategy ensures that the GTFS output only contains data relevant to the geographical area concerned. + +**Parameters**: + + * **polygon**: a required argument, which accepts the polygon in WKT format using the WGS84 coordinate system (SRID: 4326). This polygon defines the area of interest for filtering. + +``` +{"op":"transform","class":"org.onebusaway.gtfs_transformer.impl.RetainUpFromPolygon","polygon":"POLYGON ((-123.0 37.0, -123.0 38.0, -122.0 38.0, -122.0 37.0, -123.0 37.0))"} +``` #### Trim a Trip diff --git a/onebusaway-gtfs-transformer/src/main/java/org/onebusaway/gtfs_transformer/impl/RemoveStopsOutsidePolygone.java b/onebusaway-gtfs-transformer/src/main/java/org/onebusaway/gtfs_transformer/impl/RetainUpFromPolygon.java similarity index 76% rename from onebusaway-gtfs-transformer/src/main/java/org/onebusaway/gtfs_transformer/impl/RemoveStopsOutsidePolygone.java rename to onebusaway-gtfs-transformer/src/main/java/org/onebusaway/gtfs_transformer/impl/RetainUpFromPolygon.java index ecbf39a6..0ab4b62a 100644 --- a/onebusaway-gtfs-transformer/src/main/java/org/onebusaway/gtfs_transformer/impl/RemoveStopsOutsidePolygone.java +++ b/onebusaway-gtfs-transformer/src/main/java/org/onebusaway/gtfs_transformer/impl/RetainUpFromPolygon.java @@ -21,14 +21,14 @@ import org.slf4j.LoggerFactory; -public class RemoveStopsOutsidePolygone implements GtfsTransformStrategy { - private final Logger log = LoggerFactory.getLogger(RemoveStopsOutsidePolygone.class); +public class RetainUpFromPolygon implements GtfsTransformStrategy { + private final Logger log = LoggerFactory.getLogger(RetainUpFromPolygon.class); - @CsvField(optional = true) - private String polygone; + @CsvField(optional = false) + private String polygon; - public void setPolygone(String polygone) { - this.polygone = polygone; + public void setPolygon(String polygon) { + this.polygon = polygon; } @Override @@ -36,17 +36,12 @@ public String getName() { return this.getClass().getSimpleName(); } - /* - * example: - * {"op":"transform","class":"org.onebusaway.gtfs_transformer.impl.RemoveStopsOutsidePolygone","polygone":wkt_polygone ..."} - */ - - @Override + @Override public void run(TransformContext transformContext, GtfsMutableRelationalDao gtfsMutableRelationalDao) { - Geometry geometry = buildPolygone(polygone); + Geometry geometry = buildPolygon(polygon); EntityRetentionGraph graph = new EntityRetentionGraph(gtfsMutableRelationalDao); graph.setRetainBlocks(false); - // browse all stops and retain only those inside polygone/multiPolygone + // browse all stops and retain only those inside polygon/multipolygon if (geometry.isValid() && !geometry.isEmpty()){ for (Stop stop : gtfsMutableRelationalDao.getAllStops()) { if (insidePolygon(geometry,stop.getLon(),stop.getLat())){ @@ -70,14 +65,14 @@ public void run(TransformContext transformContext, GtfsMutableRelationalDao gtfs } /* - * create polygone/multiPolygone from 'polygone' variable in json file + * create polygon/multiPolygon from 'polygon' variable in json file * return Geometry variable * return null if an exception is encountered when parsing the wkt string */ - private Geometry buildPolygone(String wktPolygone) { + private Geometry buildPolygon(String polygonWKT) { WKTReader reader = new WKTReader(); try{ - return reader.read(wktPolygone); + return reader.read(polygonWKT); } catch (ParseException e){ String message = String.format("Error parsing WKT string : %s", e.getMessage()); log.error(message); @@ -86,9 +81,9 @@ private Geometry buildPolygone(String wktPolygone) { } /* - * insidePolygone returns boolean variable - * true: if polygone contains point - * false if point is outside polygone + * insidePolygon returns boolean variable + * true: if polygon contains point + * false if point is outside polygon */ private boolean insidePolygon(Geometry geometry, double lon, double lat) { GeometryFactory geometryFactory = new GeometryFactory(); diff --git a/onebusaway-gtfs-transformer/src/test/java/org/onebusaway/gtfs_transformer/impl/RetainUpFromPolygonTest.java b/onebusaway-gtfs-transformer/src/test/java/org/onebusaway/gtfs_transformer/impl/RetainUpFromPolygonTest.java new file mode 100644 index 00000000..a00febd0 --- /dev/null +++ b/onebusaway-gtfs-transformer/src/test/java/org/onebusaway/gtfs_transformer/impl/RetainUpFromPolygonTest.java @@ -0,0 +1,76 @@ +package org.onebusaway.gtfs_transformer.impl; +import static org.junit.jupiter.api.Assertions.assertEquals; +import java.io.IOException; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.onebusaway.gtfs.services.GtfsMutableRelationalDao; +import org.onebusaway.gtfs.services.MockGtfs; +import org.onebusaway.gtfs_transformer.services.TransformContext; + +public class RetainUpFromPolygonTest { + + private RetainUpFromPolygon retainUpFromPolygon = new RetainUpFromPolygon(); + private TransformContext _context = new TransformContext(); + private MockGtfs _gtfs; + + @BeforeEach + public void setup() throws IOException{ + + _gtfs = MockGtfs.create(); + // Insert mock data into the GTFS for testing: + // 1 agency + _gtfs.putAgencies(1); + // 4 routes + _gtfs.putRoutes(4); + // 4 trips + _gtfs.putTrips(4, "r$0","sid$0"); + // 8 stops + _gtfs.putStops(8); + // 13 stop times + _gtfs.putLines("stop_times.txt", + "trip_id,arrival_time,departure_time,stop_id,stop_sequence,stop_headsign,pickup_type,drop_off_type,shape_dist_traveled", + // Trip t0: sequence of stops s0,s1,s2,s3 + "t0,08:00:00,08:25:00,s0,0,,,,", + "t0,08:30:00,08:55:00,s1,1,,,,", + "t0,09:00:00,09:55:00,s2,2,,,,", + "t0,10:00:00,10:30:00,s3,3,,,,", + // Trip t1: reverse sequence of stops s3,s2,s1,s0 + "t1,08:00:00,08:25:00,s3,0,,,,", + "t1,08:30:00,08:55:00,s2,1,,,,", + "t1,09:00:00,09:55:00,s1,2,,,,", + "t1,10:00:00,10:00:00,s0,3,,,,", + // Trip t2: sequence of stops s3,s4,s5 + "t2,10:00:00,10:55:00,s3,0,,,,", + "t2,11:00:00,11:25:00,s4,1,,,,", + "t2,11:30:00,11:55:00,s5,2,,,,", + // Trip t3: Additional stops + "t3,12:00:00,12:25:00,s6,0,,,,", + "t3,12:30:00,12:55:00,s7,1,,,,"); + } + + @Test + public void testRetainUpFromPolygonTest() throws IOException { + GtfsMutableRelationalDao dao = _gtfs.read(); + + // Define a polygon in WKT (Well-Known Text) format + // This polygon is designed to include only the first 4 stops (S0 to S4) + String polygonWKT = "POLYGON ((-122.308 47.653, -122.308 47.666, -122.307 47.666, -122.307 47.665, -122.307 47.661, -122.307 47.657, -122.307 47.653, -122.308 47.653))"; + retainUpFromPolygon.setPolygon(polygonWKT); + + // Execute the retainUpFromPolygon strategy based on the polygon + retainUpFromPolygon.run(_context, dao); + + // Verify that the number of routes is reduced to 3 + assertEquals(3,dao.getAllRoutes().size()); + + // Verify that the number of trips is reduced to 3 + assertEquals(3,dao.getAllTrips().size()); + + // Verify that the number of stops is reduced to 6 + assertEquals(6,dao.getAllStops().size()); + + // Verify that the number of stop times is reduced to 11 + assertEquals(11,dao.getAllStopTimes().size()); + } +} \ No newline at end of file From af9cb26a7db0278ec92965e2f9609462917766a8 Mon Sep 17 00:00:00 2001 From: Seif Eddine GHAZOUANI Date: Thu, 12 Dec 2024 11:06:03 +0100 Subject: [PATCH 3/4] Change buildPolygon and the check function into the setter --- .../impl/RetainUpFromPolygon.java | 47 ++++++++++--------- 1 file changed, 26 insertions(+), 21 deletions(-) diff --git a/onebusaway-gtfs-transformer/src/main/java/org/onebusaway/gtfs_transformer/impl/RetainUpFromPolygon.java b/onebusaway-gtfs-transformer/src/main/java/org/onebusaway/gtfs_transformer/impl/RetainUpFromPolygon.java index 0ab4b62a..d7adfb52 100644 --- a/onebusaway-gtfs-transformer/src/main/java/org/onebusaway/gtfs_transformer/impl/RetainUpFromPolygon.java +++ b/onebusaway-gtfs-transformer/src/main/java/org/onebusaway/gtfs_transformer/impl/RetainUpFromPolygon.java @@ -17,18 +17,22 @@ import org.onebusaway.gtfs_transformer.factory.EntityRetentionGraph; import org.onebusaway.gtfs_transformer.services.GtfsTransformStrategy; import org.onebusaway.gtfs_transformer.services.TransformContext; -import org.slf4j.Logger; - -import org.slf4j.LoggerFactory; public class RetainUpFromPolygon implements GtfsTransformStrategy { - private final Logger log = LoggerFactory.getLogger(RetainUpFromPolygon.class); @CsvField(optional = false) private String polygon; + @CsvField(ignore = true) + private Geometry polygonGeometry; + public void setPolygon(String polygon) { this.polygon = polygon; + this.polygonGeometry = buildPolygon(polygon); + + if (this.polygonGeometry == null || !this.polygonGeometry.isValid() || this.polygonGeometry.isEmpty()) { + throw new IllegalArgumentException("The provided polygon is invalid or empty."); + } } @Override @@ -38,15 +42,12 @@ public String getName() { @Override public void run(TransformContext transformContext, GtfsMutableRelationalDao gtfsMutableRelationalDao) { - Geometry geometry = buildPolygon(polygon); EntityRetentionGraph graph = new EntityRetentionGraph(gtfsMutableRelationalDao); graph.setRetainBlocks(false); // browse all stops and retain only those inside polygon/multipolygon - if (geometry.isValid() && !geometry.isEmpty()){ - for (Stop stop : gtfsMutableRelationalDao.getAllStops()) { - if (insidePolygon(geometry,stop.getLon(),stop.getLat())){ - graph.retain(stop, true); - } + for (Stop stop : gtfsMutableRelationalDao.getAllStops()) { + if (insidePolygon(polygonGeometry,stop.getLon(),stop.getLat())){ + graph.retain(stop, true); } } @@ -65,25 +66,29 @@ public void run(TransformContext transformContext, GtfsMutableRelationalDao gtfs } /* - * create polygon/multiPolygon from 'polygon' variable in json file - * return Geometry variable - * return null if an exception is encountered when parsing the wkt string + * Creates a Geometry object (polygon or multi-polygon) from the provided WKT string. + * + * @param polygonWKT The WKT representation of the polygon. + * @return The Geometry object. + * @throws IllegalArgumentException if the WKT string is invalid or cannot be parsed. */ private Geometry buildPolygon(String polygonWKT) { WKTReader reader = new WKTReader(); try{ return reader.read(polygonWKT); } catch (ParseException e){ - String message = String.format("Error parsing WKT string : %s", e.getMessage()); - log.error(message); - return null; + throw new IllegalArgumentException( + String.format("Error parsing WKT string: %s", e.getMessage()), e + ); } - } /* - * insidePolygon returns boolean variable - * true: if polygon contains point - * false if point is outside polygon + * insidePolygon Checks whether a given point (specified by its longitude and latitude) is inside a given polygon or multipolygon. + * + * @param geometry The Geometry object representing the polygon or multipolygon. + * @param lon the longitude of the point to check. + * @param lat the latitude of the point to check. + * @return true if the point is within the boundaries of the geometry; false otherwise. */ private boolean insidePolygon(Geometry geometry, double lon, double lat) { GeometryFactory geometryFactory = new GeometryFactory(); @@ -91,4 +96,4 @@ private boolean insidePolygon(Geometry geometry, double lon, double lat) { return geometry.contains(point); } -} +} \ No newline at end of file From c573cfa454e643e788141059116e9e8bf6bb9632 Mon Sep 17 00:00:00 2001 From: Seif Eddine GHAZOUANI Date: Thu, 12 Dec 2024 13:44:12 +0100 Subject: [PATCH 4/4] Update documentation for RetainUpFromPolygon strategy with detailed retainUp and retainDown process --- docs/onebusaway-gtfs-transformer-cli.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/onebusaway-gtfs-transformer-cli.md b/docs/onebusaway-gtfs-transformer-cli.md index c35164f5..efe4c6cc 100644 --- a/docs/onebusaway-gtfs-transformer-cli.md +++ b/docs/onebusaway-gtfs-transformer-cli.md @@ -278,11 +278,15 @@ Retain Up From Polygon is an operation that filters GTFS input data based on a s This strategy applies two main functions: - * **Retain Function**: retains **up** all stops, trips, and routes that are located inside the defined polygon, then the algorithm automatically applies a retain **down** to these entities. + * **Retain Function**: retains **up** all stops, trips, and routes that are located inside the defined polygon. + + The algorithm starts by applying retain up to each entity, traversing the entity dependency tree. Starting from the stop, retain up is applied to the stop_times referencing this stop, then to the trips, and so on. + + Once the base of the entity tree is reached, it automatically applies retain **down** to all the traversed entities. Therefore, all the trips of the route and then all the stop_times of each trip will be tagged as **retain**. * **Remove Function**: any entities not retained within the polygon are removed. -This strategy ensures that the GTFS output only contains data relevant to the geographical area concerned. +This strategy ensures that the GTFS output retains only the entities directly or indirectly linked to the geographical area concerned. **Parameters**: