From d60a1141cda5fc6be008a00b8b0503e4463f94ba Mon Sep 17 00:00:00 2001 From: sbonnet Date: Fri, 3 May 2024 10:50:05 +0200 Subject: [PATCH 1/5] Add "remove_today" attribute (cherry picked from commit 05c71727aed0d3afd5fb672dc89f12c27b665e1e) --- .../impl/RemoveOldCalendarStatements.java | 21 ++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/onebusaway-gtfs-transformer/src/main/java/org/onebusaway/gtfs_transformer/impl/RemoveOldCalendarStatements.java b/onebusaway-gtfs-transformer/src/main/java/org/onebusaway/gtfs_transformer/impl/RemoveOldCalendarStatements.java index 434672c77..864ad9c96 100644 --- a/onebusaway-gtfs-transformer/src/main/java/org/onebusaway/gtfs_transformer/impl/RemoveOldCalendarStatements.java +++ b/onebusaway-gtfs-transformer/src/main/java/org/onebusaway/gtfs_transformer/impl/RemoveOldCalendarStatements.java @@ -15,10 +15,13 @@ */ package org.onebusaway.gtfs_transformer.impl; +import org.onebusaway.csv_entities.schema.annotations.CsvField; import org.onebusaway.gtfs.model.ServiceCalendar; +import org.onebusaway.gtfs.model.ServiceCalendarDate; import org.onebusaway.gtfs.services.GtfsMutableRelationalDao; import org.onebusaway.gtfs_transformer.services.GtfsTransformStrategy; import org.onebusaway.gtfs_transformer.services.TransformContext; +import org.onebusaway.gtfs_transformer.util.CalendarFunctions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -28,6 +31,17 @@ public class RemoveOldCalendarStatements implements GtfsTransformStrategy { private final Logger _log = LoggerFactory.getLogger(RemoveOldCalendarStatements.class); + + @CsvField(optional = true) + private boolean removeToday = true; + + @CsvField(ignore = true) + private CalendarFunctions helper = new CalendarFunctions(); + + public void setRemoveToday(boolean removeToday) { + this.removeToday = removeToday; + } + @Override public String getName() { return this.getClass().getSimpleName(); @@ -37,8 +51,13 @@ public String getName() { public void run(TransformContext transformContext, GtfsMutableRelationalDao gtfsMutableRelationalDao) { RemoveEntityLibrary removeEntityLibrary = new RemoveEntityLibrary(); Set serviceCalendarsToRemove = new HashSet(); + java.util.Date today = new java.util.Date(); + + if(!removeToday) { + today = helper.removeTime(today); + } + for (ServiceCalendar calendar: gtfsMutableRelationalDao.getAllCalendars()) { - java.util.Date today = new java.util.Date(); if (calendar.getEndDate().getAsDate().before(today)){ serviceCalendarsToRemove.add(calendar); } From bbb72a5d62d9e43d6df4a73c6a5958edef08e273 Mon Sep 17 00:00:00 2001 From: sbonnet Date: Fri, 3 May 2024 14:04:43 +0200 Subject: [PATCH 2/5] Clean, comment and format --- .../impl/RemoveOldCalendarStatements.java | 23 +++++++++---------- 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/onebusaway-gtfs-transformer/src/main/java/org/onebusaway/gtfs_transformer/impl/RemoveOldCalendarStatements.java b/onebusaway-gtfs-transformer/src/main/java/org/onebusaway/gtfs_transformer/impl/RemoveOldCalendarStatements.java index 864ad9c96..5a9e0b982 100644 --- a/onebusaway-gtfs-transformer/src/main/java/org/onebusaway/gtfs_transformer/impl/RemoveOldCalendarStatements.java +++ b/onebusaway-gtfs-transformer/src/main/java/org/onebusaway/gtfs_transformer/impl/RemoveOldCalendarStatements.java @@ -15,23 +15,22 @@ */ package org.onebusaway.gtfs_transformer.impl; +import java.util.HashSet; +import java.util.Set; + import org.onebusaway.csv_entities.schema.annotations.CsvField; import org.onebusaway.gtfs.model.ServiceCalendar; -import org.onebusaway.gtfs.model.ServiceCalendarDate; import org.onebusaway.gtfs.services.GtfsMutableRelationalDao; import org.onebusaway.gtfs_transformer.services.GtfsTransformStrategy; import org.onebusaway.gtfs_transformer.services.TransformContext; import org.onebusaway.gtfs_transformer.util.CalendarFunctions; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.HashSet; -import java.util.Set; +/** + * Remove calendar dates that are past. + * + * remove_today: delete today's calendar dates if true. Default value is true + */ public class RemoveOldCalendarStatements implements GtfsTransformStrategy { - - private final Logger _log = LoggerFactory.getLogger(RemoveOldCalendarStatements.class); - @CsvField(optional = true) private boolean removeToday = true; @@ -53,12 +52,12 @@ public void run(TransformContext transformContext, GtfsMutableRelationalDao gtfs Set serviceCalendarsToRemove = new HashSet(); java.util.Date today = new java.util.Date(); - if(!removeToday) { + if (!removeToday) { today = helper.removeTime(today); } - for (ServiceCalendar calendar: gtfsMutableRelationalDao.getAllCalendars()) { - if (calendar.getEndDate().getAsDate().before(today)){ + for (ServiceCalendar calendar : gtfsMutableRelationalDao.getAllCalendars()) { + if (calendar.getEndDate().getAsDate().before(today)) { serviceCalendarsToRemove.add(calendar); } } From c17f67b0a9fe7ff01bd7ebedb8afb931f695cb39 Mon Sep 17 00:00:00 2001 From: Seif Eddine GHAZOUANI Date: Tue, 3 Dec 2024 17:12:11 +0100 Subject: [PATCH 3/5] Add test case to validate the removal of calendars with today's date --- .../impl/RemoveOldCalendarStatementsTest.java | 71 +++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 onebusaway-gtfs-transformer/src/test/java/org/onebusaway/gtfs_transformer/impl/RemoveOldCalendarStatementsTest.java diff --git a/onebusaway-gtfs-transformer/src/test/java/org/onebusaway/gtfs_transformer/impl/RemoveOldCalendarStatementsTest.java b/onebusaway-gtfs-transformer/src/test/java/org/onebusaway/gtfs_transformer/impl/RemoveOldCalendarStatementsTest.java new file mode 100644 index 000000000..e246045d1 --- /dev/null +++ b/onebusaway-gtfs-transformer/src/test/java/org/onebusaway/gtfs_transformer/impl/RemoveOldCalendarStatementsTest.java @@ -0,0 +1,71 @@ +package org.onebusaway.gtfs_transformer.impl; + +import static org.junit.Assert.assertEquals; + +import java.io.IOException; +import java.time.LocalDate; +import java.time.format.DateTimeFormatter; + +import org.junit.Before; +import org.junit.Test; +import org.onebusaway.gtfs.services.GtfsMutableRelationalDao; +import org.onebusaway.gtfs.services.MockGtfs; +import org.onebusaway.gtfs_transformer.services.TransformContext; + +public class RemoveOldCalendarStatementsTest { + + private RemoveOldCalendarStatements removeOldCalendarStatements = new RemoveOldCalendarStatements(); + private TransformContext _context = new TransformContext(); + private MockGtfs _gtfs; + + @Before + public void setup() throws IOException{ + + _gtfs = MockGtfs.create(); + _gtfs.putAgencies(1); + _gtfs.putStops(1); + _gtfs.putRoutes(1); + _gtfs.putTrips(1, "r0", "sid0"); + _gtfs.putStopTimes("t0", "s0"); + + // Insert a calendar entry wtih start and end dates set to today's date + String startDate = getCurrentDateFormatted(-3); + String endDate = getCurrentDateFormatted(null); + + _gtfs.putCalendars( + 1, + "start_date="+startDate, + "end_date="+endDate + ); + } + + @Test + public void testRemoveCalendarForToday() throws IOException { + GtfsMutableRelationalDao dao = _gtfs.read(); + // Set removeToday to true to allow the removal of the calendar for today's date + removeOldCalendarStatements.setRemoveToday(true); + removeOldCalendarStatements.run(_context, dao); + // Verify that GtfsMutableRelationalDao object no longer contains any calendar entries after removing the calendar for today's date + assertEquals(0,dao.getAllCalendars().size()); + } + + @Test + public void testRemoveCalendar() throws IOException { + GtfsMutableRelationalDao dao = _gtfs.read(); + removeOldCalendarStatements.setRemoveToday(false); + removeOldCalendarStatements.run(_context, dao); + // Verify that GtfsMutableRelationalDao object still contain the initially added calendar entry + assertEquals(1,dao.getAllCalendars().size()); + } + + // Helper function to get today's date in the required format + public static String getCurrentDateFormatted(Integer daysOffset) { + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyyMMdd"); + LocalDate date = LocalDate.now(); + if (daysOffset != null){ + date = date.plusDays(daysOffset); + } + // Format date as "yyyyMMdd" + return date.format(formatter); + } +} From 907653aa43c9820f92aa903b7156b720b90153bb Mon Sep 17 00:00:00 2001 From: Seif Eddine GHAZOUANI Date: Thu, 5 Dec 2024 09:50:54 +0100 Subject: [PATCH 4/5] Fix junit issue + add documentation --- docs/onebusaway-gtfs-transformer-cli.md | 23 ++++++++++++++++++- .../impl/RemoveOldCalendarStatementsTest.java | 11 ++++----- 2 files changed, 27 insertions(+), 7 deletions(-) diff --git a/docs/onebusaway-gtfs-transformer-cli.md b/docs/onebusaway-gtfs-transformer-cli.md index 8bd625bf2..069fe6a16 100644 --- a/docs/onebusaway-gtfs-transformer-cli.md +++ b/docs/onebusaway-gtfs-transformer-cli.md @@ -23,6 +23,7 @@ * [Trim a Trip](#trim-a-trip) * [Generate Stop Times](#generate-stop-times) * [Extend Service Calendars](#extend-service-calendars) + * [Remove Old Calendar Statements](#remove-old-calendar-statements) * [Deduplicate Calendar Entries](#deduplicate-calendar-entries) * [Merge Trips and Simplify Calendar Entries](#merge-trips-and-simplify-calendar-entries) * [Shift Negative Stop Times](#shift-negative-stop-times) @@ -337,7 +338,27 @@ Calendars that have expired before the specified date will be considered inactiv _Note_: We don't make any effort to extend canceled service dates, as specified in `calendar_dates.txt` for holidays and other special events. It's too tricky to automatically determine how they should be handled. You may need to still handle -those manually. +those manually. + +#### Remove Old Calendar Statements + +RemoveOldCalendarStatements is an operation designed to remove calendar entries that are no longer valid on today's date. + +By default, it deletes entries from the calendar.txt file whose end_date field has passed, including those valid until today. + +With the remove_today attribute added to the JSON transformer snippet, users can control whether calendar entries valid for today are included or excluded in the output GTFS. + + * If remove_today is set to true or not specified, the transformer will remove the calendar entries for the current date. + +``` + {"op":"transform", "class":"org.onebusaway.gtfs_transformer.impl.RemoveOldCalendarStatements", "remove_today":true} +``` + + * If remove_today is set to false, the transformer will retain the calendar entries for the current date. + +``` +{"op":"transform", "class":"org.onebusaway.gtfs_transformer.impl.RemoveOldCalendarStatements", "remove_today":false} +``` #### Deduplicate Calendar Entries diff --git a/onebusaway-gtfs-transformer/src/test/java/org/onebusaway/gtfs_transformer/impl/RemoveOldCalendarStatementsTest.java b/onebusaway-gtfs-transformer/src/test/java/org/onebusaway/gtfs_transformer/impl/RemoveOldCalendarStatementsTest.java index e246045d1..7706d48dc 100644 --- a/onebusaway-gtfs-transformer/src/test/java/org/onebusaway/gtfs_transformer/impl/RemoveOldCalendarStatementsTest.java +++ b/onebusaway-gtfs-transformer/src/test/java/org/onebusaway/gtfs_transformer/impl/RemoveOldCalendarStatementsTest.java @@ -1,13 +1,12 @@ package org.onebusaway.gtfs_transformer.impl; -import static org.junit.Assert.assertEquals; - +import static org.junit.jupiter.api.Assertions.assertEquals; import java.io.IOException; import java.time.LocalDate; import java.time.format.DateTimeFormatter; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import org.onebusaway.gtfs.services.GtfsMutableRelationalDao; import org.onebusaway.gtfs.services.MockGtfs; import org.onebusaway.gtfs_transformer.services.TransformContext; @@ -18,7 +17,7 @@ public class RemoveOldCalendarStatementsTest { private TransformContext _context = new TransformContext(); private MockGtfs _gtfs; - @Before + @BeforeEach public void setup() throws IOException{ _gtfs = MockGtfs.create(); @@ -68,4 +67,4 @@ public static String getCurrentDateFormatted(Integer daysOffset) { // Format date as "yyyyMMdd" return date.format(formatter); } -} +} \ No newline at end of file From 3b0bf35c11cbb0fcf104f42f0da3768571866905 Mon Sep 17 00:00:00 2001 From: Seif Eddine GHAZOUANI Date: Fri, 6 Dec 2024 11:04:43 +0100 Subject: [PATCH 5/5] Change default behaviour: set the default value of removeToday to false in the remove old calendar statements operation --- docs/onebusaway-gtfs-transformer-cli.md | 6 +++--- .../gtfs_transformer/impl/RemoveOldCalendarStatements.java | 4 ++-- .../impl/RemoveOldCalendarStatementsTest.java | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/onebusaway-gtfs-transformer-cli.md b/docs/onebusaway-gtfs-transformer-cli.md index 069fe6a16..f590f7b85 100644 --- a/docs/onebusaway-gtfs-transformer-cli.md +++ b/docs/onebusaway-gtfs-transformer-cli.md @@ -344,17 +344,17 @@ those manually. RemoveOldCalendarStatements is an operation designed to remove calendar entries that are no longer valid on today's date. -By default, it deletes entries from the calendar.txt file whose end_date field has passed, including those valid until today. +By default, it deletes entries from the calendar.txt file whose end_date field has passed. With the remove_today attribute added to the JSON transformer snippet, users can control whether calendar entries valid for today are included or excluded in the output GTFS. - * If remove_today is set to true or not specified, the transformer will remove the calendar entries for the current date. + * If remove_today is set to true, the transformer will remove the calendar entries for the current date. ``` {"op":"transform", "class":"org.onebusaway.gtfs_transformer.impl.RemoveOldCalendarStatements", "remove_today":true} ``` - * If remove_today is set to false, the transformer will retain the calendar entries for the current date. + * If remove_today is set to false or not specified, the transformer will retain the calendar entries for the current date. ``` {"op":"transform", "class":"org.onebusaway.gtfs_transformer.impl.RemoveOldCalendarStatements", "remove_today":false} diff --git a/onebusaway-gtfs-transformer/src/main/java/org/onebusaway/gtfs_transformer/impl/RemoveOldCalendarStatements.java b/onebusaway-gtfs-transformer/src/main/java/org/onebusaway/gtfs_transformer/impl/RemoveOldCalendarStatements.java index 5a9e0b982..5fa491838 100644 --- a/onebusaway-gtfs-transformer/src/main/java/org/onebusaway/gtfs_transformer/impl/RemoveOldCalendarStatements.java +++ b/onebusaway-gtfs-transformer/src/main/java/org/onebusaway/gtfs_transformer/impl/RemoveOldCalendarStatements.java @@ -28,11 +28,11 @@ /** * Remove calendar dates that are past. * - * remove_today: delete today's calendar dates if true. Default value is true + * remove_today: delete today's calendar dates if true. Default value is false */ public class RemoveOldCalendarStatements implements GtfsTransformStrategy { @CsvField(optional = true) - private boolean removeToday = true; + private boolean removeToday = false; @CsvField(ignore = true) private CalendarFunctions helper = new CalendarFunctions(); diff --git a/onebusaway-gtfs-transformer/src/test/java/org/onebusaway/gtfs_transformer/impl/RemoveOldCalendarStatementsTest.java b/onebusaway-gtfs-transformer/src/test/java/org/onebusaway/gtfs_transformer/impl/RemoveOldCalendarStatementsTest.java index 7706d48dc..3519bfc26 100644 --- a/onebusaway-gtfs-transformer/src/test/java/org/onebusaway/gtfs_transformer/impl/RemoveOldCalendarStatementsTest.java +++ b/onebusaway-gtfs-transformer/src/test/java/org/onebusaway/gtfs_transformer/impl/RemoveOldCalendarStatementsTest.java @@ -51,7 +51,7 @@ public void testRemoveCalendarForToday() throws IOException { @Test public void testRemoveCalendar() throws IOException { GtfsMutableRelationalDao dao = _gtfs.read(); - removeOldCalendarStatements.setRemoveToday(false); + // Keep the default value as false and do not change it removeOldCalendarStatements.run(_context, dao); // Verify that GtfsMutableRelationalDao object still contain the initially added calendar entry assertEquals(1,dao.getAllCalendars().size());