From 686119b1e8dee9ffa6311d762eefe4ada7d57f2b Mon Sep 17 00:00:00 2001 From: Daniel Heppner Date: Tue, 22 Mar 2022 14:29:04 -0700 Subject: [PATCH 001/248] feature(Docker): Adds Dockerfile --- Dockerfile | 12 ++++++++++++ docker-compose.yml | 25 +++++++++++++++++++++++++ 2 files changed, 37 insertions(+) create mode 100644 Dockerfile create mode 100644 docker-compose.yml diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..0400fc7b9 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,12 @@ +# syntax=docker/dockerfile:1 +FROM openjdk:11 +WORKDIR /datatools + +# Grab latest dev build +COPY target/dt*.jar ./datatools-server-3.8.1-SNAPSHOT.jar + +RUN mkdir -p /var/datatools_gtfs +# Launch server (relies on env.yml being placed in volume!) +# Try: docker run --publish 4000:4000 -v ~/config/:/config datatools-latest +CMD ["java", "-jar", "datatools-server-3.8.1-SNAPSHOT.jar", "/config/env.yml", "/config/server.yml"] +EXPOSE 4000 \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 000000000..baf5605fd --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,25 @@ +version: '3.8' +services: + datatools-server: + # Temporary! move back to "latest" + build: ./ + ports: + - "4000:4000" + volumes: + - type: bind + source: ./configurations/docker/ + target: /config + - type: bind + source: ~/.aws + target: /root/.aws + depends_on: + - mongo + - postgres + mongo: + image: mongo + restart: always + postgres: + environment: + POSTGRES_HOST_AUTH_METHOD: trust + image: postgres + restart: always \ No newline at end of file From ad842f07ecd0a12b0864b15e7342381647162d84 Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Wed, 23 Mar 2022 14:14:20 -0700 Subject: [PATCH 002/248] refactor(docker): include working UI --- configurations/docker/env.yml.tmp | 19 ++++++++ configurations/docker/server.yml.tmp | 65 ++++++++++++++++++++++++++++ docker-compose.yml | 13 +++++- 3 files changed, 95 insertions(+), 2 deletions(-) create mode 100644 configurations/docker/env.yml.tmp create mode 100644 configurations/docker/server.yml.tmp diff --git a/configurations/docker/env.yml.tmp b/configurations/docker/env.yml.tmp new file mode 100644 index 000000000..1081f48d7 --- /dev/null +++ b/configurations/docker/env.yml.tmp @@ -0,0 +1,19 @@ +# This client ID refers to the UI client in Auth0. +AUTH0_CLIENT_ID: your-auth0-client-id +AUTH0_DOMAIN: your-auth0-domain +# Note: One of AUTH0_SECRET or AUTH0_PUBLIC_KEY should be used depending on the signing algorithm set on the client. +# It seems that newer Auth0 accounts (2017 and later) might default to RS256 (public key). +AUTH0_SECRET: your-auth0-secret # uses HS256 signing algorithm +# AUTH0_PUBLIC_KEY: /path/to/auth0.pem # uses RS256 signing algorithm +# This client/secret pair refer to a machine-to-machine Auth0 application used to access the Management API. +AUTH0_API_CLIENT: your-api-client-id +AUTH0_API_SECRET: your-api-secret-id +DISABLE_AUTH: false +OSM_VEX: http://localhost:1000 +SPARKPOST_KEY: your-sparkpost-key +SPARKPOST_EMAIL: email@example.com +GTFS_DATABASE_URL: jdbc:postgresql://localhost/catalogue +# GTFS_DATABASE_USER: +# GTFS_DATABASE_PASSWORD: +MONGO_DB_NAME: data_manager +MONGO_HOST: mongo:27017 diff --git a/configurations/docker/server.yml.tmp b/configurations/docker/server.yml.tmp new file mode 100644 index 000000000..2e7c136e6 --- /dev/null +++ b/configurations/docker/server.yml.tmp @@ -0,0 +1,65 @@ +application: + title: Data Tools + logo: https://d2tyb7byn1fef9.cloudfront.net/ibi_group-128x128.png + logo_large: https://d2tyb7byn1fef9.cloudfront.net/ibi_group_black-512x512.png + client_assets_url: http://localhost:8888/ + shortcut_icon_url: https://d2tyb7byn1fef9.cloudfront.net/ibi-logo-original%402x.png + public_url: http://localhost:9966 + notifications_enabled: false + docs_url: http://conveyal-data-tools.readthedocs.org + support_email: support@ibigroup.com + port: 4000 + data: + gtfs: /tmp + use_s3_storage: false + s3_region: us-east-1 + gtfs_s3_bucket: bucket-name +modules: + enterprise: + enabled: false + editor: + enabled: true + deployment: + enabled: true + ec2: + enabled: false + default_ami: ami-your-ami-id + # Note: using a cloudfront URL for these download URLs will greatly + # increase download/deploy speed. + otp_download_url: https://optional-otp-repo.com + user_admin: + enabled: true + gtfsapi: + enabled: true + load_on_fetch: false + # use_extension: mtc + # update_frequency: 30 # in seconds + manager: + normalizeFieldTransformation: + # Enter capitalization exceptions (e.g. acronyms), in the desired case, and separated by commas. + defaultCapitalizationExceptions: + - ACE + - BART + # Enter substitutions (e.g. substitute '@' with 'at'), one dashed entry for each substitution, with: + # - pattern: the regex string pattern that will be replaced, + # - replacement: the replacement string for that pattern, + # - normalizeSpace: if true, the resulting field value will include one space before and after the replacement string. + # Note: if the replacement must be blank, then normalizeSpace should be set to false + # and whitespace management should be handled in pattern instead. + # Substitutions are executed in order they appear in the list. + defaultSubstitutions: + - description: "Replace '@' with 'at', and normalize space." + pattern: "@" + replacement: at + normalizeSpace: true + - description: "Replace '+' (\\+ in regex) and '&' with 'and', and normalize space." + pattern: "[\\+&]" + replacement: and + normalizeSpace: true +extensions: + transitland: + enabled: true + api: https://transit.land/api/v1/feeds + transitfeeds: + enabled: true + api: http://api.transitfeeds.com/v1/getFeeds diff --git a/docker-compose.yml b/docker-compose.yml index baf5605fd..76c856e84 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,10 +1,16 @@ version: '3.8' services: + datatools-ui: + # Temporary! move to image (?) -- includes config variables + build: ../datatools-ui + restart: always + ports: + - "8888:80" datatools-server: - # Temporary! move back to "latest" + # Temporary! move to image build: ./ ports: - - "4000:4000" + - "9966:4000" volumes: - type: bind source: ./configurations/docker/ @@ -15,11 +21,14 @@ services: depends_on: - mongo - postgres + - datatools-ui mongo: image: mongo restart: always postgres: environment: POSTGRES_HOST_AUTH_METHOD: trust + POSTGRES_USER: root + POSTGRES_DB: dmtest image: postgres restart: always \ No newline at end of file From f424f69aad2db9158c77795b011e30663d0fb985 Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Thu, 24 Mar 2022 09:10:33 -0700 Subject: [PATCH 003/248] chore(gitignore): update to save docker configs --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index b04d422d8..ff81a64e7 100644 --- a/.gitignore +++ b/.gitignore @@ -14,9 +14,10 @@ tmp/ deploy/ # Configurations -configurations/* +configurations/*.yml !configurations/default !configurations/test +!configurations/docker # Secret config files .env From 37c0b8660a2d6edc83ab48d7e279623212805a1a Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Fri, 25 Mar 2022 12:01:07 -0700 Subject: [PATCH 004/248] refactor: new way to run e2e tests --- Dockerfile e2e | 11 +++++++++++ docker-compose.e2e.yml | 26 ++++++++++++++++++++++++++ 2 files changed, 37 insertions(+) create mode 100644 Dockerfile e2e create mode 100644 docker-compose.e2e.yml diff --git a/Dockerfile e2e b/Dockerfile e2e new file mode 100644 index 000000000..1f69e5927 --- /dev/null +++ b/Dockerfile e2e @@ -0,0 +1,11 @@ +# syntax=docker/dockerfile:1 +FROM maven:3-jdk-8-alpine +WORKDIR /datatools + +# Get code into container +COPY ./ . +COPY ./configurations/docker/env.yml ./configurations/test/env.yml.tmp +COPY ./configurations/docker/server.yml ./configurations/test/server.yml.tmp + +ENV SHOULD_RUN_E2E=true +CMD ["mvn", "test"] \ No newline at end of file diff --git a/docker-compose.e2e.yml b/docker-compose.e2e.yml new file mode 100644 index 000000000..2a3627a64 --- /dev/null +++ b/docker-compose.e2e.yml @@ -0,0 +1,26 @@ +version: '3.8' +services: + datatools-server: + build: + context: ./ + dockerfile: ./Dockerfile e2e + volumes: + - type: bind + source: ./configurations/docker/ + target: /config + - type: bind + source: ~/.aws + target: /root/.aws + depends_on: + - mongo + - postgres + mongo: + image: mongo + restart: always + postgres: + environment: + POSTGRES_HOST_AUTH_METHOD: trust + POSTGRES_USER: root + POSTGRES_DB: dmtest + image: postgres + restart: always \ No newline at end of file From 09bd6a2ee775fc5b7bebc964ecfd7eadd21f8199 Mon Sep 17 00:00:00 2001 From: binh-dam-ibigroup <56846598+binh-dam-ibigroup@users.noreply.github.com> Date: Tue, 28 Jun 2022 18:14:59 -0400 Subject: [PATCH 005/248] fix(SparkUtils): Remove conflicting content encoding header data. --- .../java/com/conveyal/datatools/common/utils/SparkUtils.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/common/utils/SparkUtils.java b/src/main/java/com/conveyal/datatools/common/utils/SparkUtils.java index 522beca71..f6f08167e 100644 --- a/src/main/java/com/conveyal/datatools/common/utils/SparkUtils.java +++ b/src/main/java/com/conveyal/datatools/common/utils/SparkUtils.java @@ -53,9 +53,9 @@ public static HttpServletResponse downloadFile(File file, String filename, Reque if (file == null) logMessageAndHalt(req, 404, "File is null"); HttpServletResponse raw = res.raw(); raw.setContentType("application/octet-stream"); - raw.setHeader("Content-Disposition", "attachment; filename=" + filename); // Override the gzip content encoding applied to standard API responses. - res.header("Content-Encoding", "identity"); + raw.setHeader("Content-Encoding", "identity"); + raw.setHeader("Content-Disposition", "attachment; filename=" + filename); try ( FileInputStream fileInputStream = new FileInputStream(file); ServletOutputStream outputStream = raw.getOutputStream() From 6026e52314b345efd550c6b3663003432b60bb64 Mon Sep 17 00:00:00 2001 From: binh-dam-ibigroup <56846598+binh-dam-ibigroup@users.noreply.github.com> Date: Tue, 5 Jul 2022 17:34:09 -0400 Subject: [PATCH 006/248] fix(EditorLockController): Support deleting editor locks via navigator.sendBeacon() --- .../controllers/EditorLockController.java | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/src/main/java/com/conveyal/datatools/editor/controllers/EditorLockController.java b/src/main/java/com/conveyal/datatools/editor/controllers/EditorLockController.java index 61db70a55..625f8d703 100644 --- a/src/main/java/com/conveyal/datatools/editor/controllers/EditorLockController.java +++ b/src/main/java/com/conveyal/datatools/editor/controllers/EditorLockController.java @@ -137,6 +137,34 @@ private static String maintainLock(Request req, Response res) { } } + /** + * Simple handler that just deletes the specified session id if it is the active one for a feed, + * without checking for a user token. + * Intended for use from a browser's Navigator.sendBeacon() method. + */ + private static String deleteFeedLockBeacon(Request req, Response res) { + // TODO: Refactor + res.type("application/json"); + String feedId = req.queryParams("feedId"); + String sessionId = req.params("id"); + EditorSession currentSession = sessionsForFeedIds.get(feedId); + if (currentSession == null) { + // If there is no current session to delete/overwrite, request that user reloads browser. + LOG.warn("No active session to overwrite/delete."); + return SparkUtils.formatJSON("No active session to take over. Please refresh your browser and try editing later.", 202); + } else if (!currentSession.sessionId.equals(sessionId)) { + // If there is a different active session for some user, don't remove the session. + LOG.warn("Not overwriting session {} for user {}.", currentSession.sessionId, currentSession.userEmail); + return SparkUtils.formatJSON("Not processing request to delete lock. There is already an active session for user " + currentSession.userEmail, 202); + } else { + // Otherwise, the current session matches the session from which the delete request came. This indicates that + // the user's editing session has been closed (by either exiting the editor or closing the browser tab). + LOG.info("Closed session {} for feed {} successfully.", currentSession.sessionId, currentSession.feedId); + sessionsForFeedIds.remove(feedId); + return formatJSON("Session has been closed successfully.", 200, feedId, sessionId); + } + } + private static String deleteFeedLock(Request req, Response res) { // FIXME: why is content type not being set in before()/after()? res.type("application/json"); @@ -178,6 +206,9 @@ public static void register(String apiPrefix) { post(apiPrefix + "secure/lock", EditorLockController::lockFeed, json::write); delete(apiPrefix + "secure/lock/:id", EditorLockController::deleteFeedLock, json::write); put(apiPrefix + "secure/lock/:id", EditorLockController::maintainLock, json::write); + // Extra, unsecure POST method for removing lock via a browser's Navigator.sendBeacon() method. + // (Navigator.sendBeacon() sends a POST and does not support authorization headers.) + post(apiPrefix + "deletelock/:id", EditorLockController::deleteFeedLockBeacon, json::write); } private static String formatJSON(String message, int code, String feedId, String sessionId) { From 724ed722dbd70e00386514f038fe3f2f6b37861a Mon Sep 17 00:00:00 2001 From: binh-dam-ibigroup <56846598+binh-dam-ibigroup@users.noreply.github.com> Date: Wed, 6 Jul 2022 12:00:55 -0400 Subject: [PATCH 007/248] refactor(EditorLockController): Refactor duplicate code. --- .../controllers/EditorLockController.java | 46 ++++++++----------- 1 file changed, 19 insertions(+), 27 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/editor/controllers/EditorLockController.java b/src/main/java/com/conveyal/datatools/editor/controllers/EditorLockController.java index 625f8d703..5f0a56e29 100644 --- a/src/main/java/com/conveyal/datatools/editor/controllers/EditorLockController.java +++ b/src/main/java/com/conveyal/datatools/editor/controllers/EditorLockController.java @@ -138,37 +138,23 @@ private static String maintainLock(Request req, Response res) { } /** - * Simple handler that just deletes the specified session id if it is the active one for a feed, - * without checking for a user token. - * Intended for use from a browser's Navigator.sendBeacon() method. + * Normal path for deleting a feed lock. + */ + private static String deleteFeedLock(Request req, Response res) { + return deleteFeedLock(req, res, req.attribute("user")); + } + + /** + * Remove a feed lock when a browser calls sendBeacon() when closing/refreshing/navigating away from editor. */ private static String deleteFeedLockBeacon(Request req, Response res) { - // TODO: Refactor - res.type("application/json"); - String feedId = req.queryParams("feedId"); - String sessionId = req.params("id"); - EditorSession currentSession = sessionsForFeedIds.get(feedId); - if (currentSession == null) { - // If there is no current session to delete/overwrite, request that user reloads browser. - LOG.warn("No active session to overwrite/delete."); - return SparkUtils.formatJSON("No active session to take over. Please refresh your browser and try editing later.", 202); - } else if (!currentSession.sessionId.equals(sessionId)) { - // If there is a different active session for some user, don't remove the session. - LOG.warn("Not overwriting session {} for user {}.", currentSession.sessionId, currentSession.userEmail); - return SparkUtils.formatJSON("Not processing request to delete lock. There is already an active session for user " + currentSession.userEmail, 202); - } else { - // Otherwise, the current session matches the session from which the delete request came. This indicates that - // the user's editing session has been closed (by either exiting the editor or closing the browser tab). - LOG.info("Closed session {} for feed {} successfully.", currentSession.sessionId, currentSession.feedId); - sessionsForFeedIds.remove(feedId); - return formatJSON("Session has been closed successfully.", 200, feedId, sessionId); - } + // The sendBeacon call does not contain any Authorization headers, so we just pass a null userProfile. + return deleteFeedLock(req, res, null); } - private static String deleteFeedLock(Request req, Response res) { + private static String deleteFeedLock(Request req, Response res, Auth0UserProfile userProfile) { // FIXME: why is content type not being set in before()/after()? res.type("application/json"); - Auth0UserProfile userProfile = req.attribute("user"); String feedId = req.queryParams("feedId"); String sessionId = req.params("id"); EditorSession currentSession = sessionsForFeedIds.get(feedId); @@ -182,7 +168,7 @@ private static String deleteFeedLock(Request req, Response res) { // session; however, this has been removed because in practice it became a nuisance. Respectful users with // shared access to a feed can generally be trusted not to boot one another out in a combative manner. boolean overwrite = Boolean.valueOf(req.queryParams("overwrite")); - if (overwrite) { + if (userProfile != null && overwrite) { sessionId = invalidateAndCreateNewSession(req); EditorSession newEditorSession = new EditorSession(feedId, sessionId, userProfile); sessionsForFeedIds.put(feedId, newEditorSession); @@ -193,7 +179,13 @@ private static String deleteFeedLock(Request req, Response res) { return SparkUtils.formatJSON("Not processing request to delete lock. There is already an active session for user " + currentSession.userEmail, 202); } } else { - LOG.info("Current session: {} {}; User session: {} {}", currentSession.userEmail, currentSession.sessionId, userProfile.getEmail(), sessionId); + LOG.info( + "Current session: {} {}; User session: {} {}", + currentSession.userEmail, + currentSession.sessionId, + userProfile != null ? userProfile.getEmail() : "(email unavailable)", + sessionId + ); // Otherwise, the current session matches the session from which the delete request came. This indicates that // the user's editing session has been closed (by either exiting the editor or closing the browser tab). LOG.info("Closed session {} for feed {} successfully.", currentSession.sessionId, currentSession.feedId); From 5d3b428d376834821789a3a68c431b0c34356b2e Mon Sep 17 00:00:00 2001 From: binh-dam-ibigroup <56846598+binh-dam-ibigroup@users.noreply.github.com> Date: Wed, 6 Jul 2022 12:32:58 -0400 Subject: [PATCH 008/248] refactor(EditorLockController): Remove obsolete FIXMEs and related code. --- .../controllers/EditorLockController.java | 20 ++++++------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/editor/controllers/EditorLockController.java b/src/main/java/com/conveyal/datatools/editor/controllers/EditorLockController.java index 5f0a56e29..91011ac6e 100644 --- a/src/main/java/com/conveyal/datatools/editor/controllers/EditorLockController.java +++ b/src/main/java/com/conveyal/datatools/editor/controllers/EditorLockController.java @@ -35,8 +35,6 @@ public class EditorLockController { private static String lockFeed (Request req, Response res) { - // FIXME: why is content type not being set in before()/after()? - res.type("application/json"); Auth0UserProfile userProfile = req.attribute("user"); String feedId = req.queryParams("feedId"); EditorSession currentSession = sessionsForFeedIds.get(feedId); @@ -95,13 +93,10 @@ private static String getLockedFeedMessage(EditorSession session, long minutesUn private static String invalidateAndCreateNewSession(Request req) { req.session().invalidate(); Session session = req.session(true); - String newSessionId = session.id(); - return newSessionId; + return session.id(); } private static String maintainLock(Request req, Response res) { - // FIXME: why is content type not being set in before()/after()? - res.type("application/json"); String sessionId = req.params("id"); String feedId = req.queryParams("feedId"); Auth0UserProfile userProfile = req.attribute("user"); @@ -121,7 +116,7 @@ private static String maintainLock(Request req, Response res) { if (currentSession.userEmail.equals(userProfile.getEmail())) { // If the new current session is held by this user, give them the option to evict the current session / // unlock the feed. - LOG.warn("User {} already has an active editor session () for feed {}.", userProfile.getEmail(), currentSession.sessionId, currentSession.feedId); + LOG.warn("User {} already has an active editor session {} for feed {}.", userProfile.getEmail(), currentSession.sessionId, currentSession.feedId); logMessageAndHalt(req, 400, "Warning! You have an active editing session for this feed underway in a different browser tab."); } else { LOG.warn("User {} attempted editor session for feed {} while active session underway for user {}.", userProfile.getEmail(), currentSession.feedId, currentSession.userEmail); @@ -132,7 +127,6 @@ private static String maintainLock(Request req, Response res) { // Otherwise, the current session matches the session the user is attempting to maintain. Update the // lastEdited time. currentSession.lastCheckIn = System.currentTimeMillis(); -// LOG.info("Updating session {} check-in time to {} for user {}", currentSession.sessionId, currentSession.lastCheckIn, currentSession.userEmail); return formatJSON("Updating time for user " + currentSession.userEmail, 200, feedId, null); } } @@ -141,7 +135,7 @@ private static String maintainLock(Request req, Response res) { * Normal path for deleting a feed lock. */ private static String deleteFeedLock(Request req, Response res) { - return deleteFeedLock(req, res, req.attribute("user")); + return deleteFeedLockCore(req, req.attribute("user")); } /** @@ -149,12 +143,10 @@ private static String deleteFeedLock(Request req, Response res) { */ private static String deleteFeedLockBeacon(Request req, Response res) { // The sendBeacon call does not contain any Authorization headers, so we just pass a null userProfile. - return deleteFeedLock(req, res, null); + return deleteFeedLockCore(req, null); } - private static String deleteFeedLock(Request req, Response res, Auth0UserProfile userProfile) { - // FIXME: why is content type not being set in before()/after()? - res.type("application/json"); + private static String deleteFeedLockCore(Request req, Auth0UserProfile userProfile) { String feedId = req.queryParams("feedId"); String sessionId = req.params("id"); EditorSession currentSession = sessionsForFeedIds.get(feedId); @@ -167,7 +159,7 @@ private static String deleteFeedLock(Request req, Response res, Auth0UserProfile // Note: There used to be a check here that the requesting user was the same as the user with an open // session; however, this has been removed because in practice it became a nuisance. Respectful users with // shared access to a feed can generally be trusted not to boot one another out in a combative manner. - boolean overwrite = Boolean.valueOf(req.queryParams("overwrite")); + boolean overwrite = Boolean.parseBoolean(req.queryParams("overwrite")); if (userProfile != null && overwrite) { sessionId = invalidateAndCreateNewSession(req); EditorSession newEditorSession = new EditorSession(feedId, sessionId, userProfile); From cf82be8730ad2462988d96889a1a9cbd42d46068 Mon Sep 17 00:00:00 2001 From: Jonathan Leitschuh Date: Mon, 8 Aug 2022 20:34:30 +0000 Subject: [PATCH 009/248] vuln-fix: Zip Slip Vulnerability This fixes a Zip-Slip vulnerability. This change does one of two things. This change either 1. Inserts a guard to protect against Zip Slip. OR 2. Replaces `dir.getCanonicalPath().startsWith(parent.getCanonicalPath())`, which is vulnerable to partial path traversal attacks, with the more secure `dir.getCanonicalFile().toPath().startsWith(parent.getCanonicalFile().toPath())`. For number 2, consider `"/usr/outnot".startsWith("/usr/out")`. The check is bypassed although `/outnot` is not under the `/out` directory. It's important to understand that the terminating slash may be removed when using various `String` representations of the `File` object. For example, on Linux, `println(new File("/var"))` will print `/var`, but `println(new File("/var", "/")` will print `/var/`; however, `println(new File("/var", "/").getCanonicalPath())` will print `/var`. Weakness: CWE-22: Improper Limitation of a Pathname to a Restricted Directory ('Path Traversal') Severity: High CVSSS: 7.4 Detection: CodeQL (https://codeql.github.com/codeql-query-help/java/java-zipslip/) & OpenRewrite (https://public.moderne.io/recipes/org.openrewrite.java.security.ZipSlip) Reported-by: Jonathan Leitschuh Signed-off-by: Jonathan Leitschuh Bug-tracker: https://github.com/JLLeitschuh/security-research/issues/16 Co-authored-by: Moderne --- .../datatools/manager/jobs/DeploymentGisExportJobTest.java | 5 ++++- .../conveyal/datatools/manager/jobs/GisExportJobTest.java | 3 +++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/test/java/com/conveyal/datatools/manager/jobs/DeploymentGisExportJobTest.java b/src/test/java/com/conveyal/datatools/manager/jobs/DeploymentGisExportJobTest.java index ff4a4d2ca..aea3a6f16 100644 --- a/src/test/java/com/conveyal/datatools/manager/jobs/DeploymentGisExportJobTest.java +++ b/src/test/java/com/conveyal/datatools/manager/jobs/DeploymentGisExportJobTest.java @@ -33,6 +33,9 @@ private File[] getFoldersFromZippedShapefile(File zipFile) throws IOException { while (e.hasMoreElements()) { ZipEntry entry = e.nextElement(); File destinationPath = new File(destDir, entry.getName()); + if (!destinationPath.toPath().normalize().startsWith(destDir.toPath().normalize())) { + throw new RuntimeException("Bad zip entry"); + } // Create parent directories destinationPath.getParentFile().mkdirs(); @@ -120,4 +123,4 @@ public void canExportRoutes() throws IOException, SQLException { @Test public void canExportRoutesFromPatternStops() {} -} \ No newline at end of file +} diff --git a/src/test/java/com/conveyal/datatools/manager/jobs/GisExportJobTest.java b/src/test/java/com/conveyal/datatools/manager/jobs/GisExportJobTest.java index 7aac0062f..9b41e2d65 100644 --- a/src/test/java/com/conveyal/datatools/manager/jobs/GisExportJobTest.java +++ b/src/test/java/com/conveyal/datatools/manager/jobs/GisExportJobTest.java @@ -265,6 +265,9 @@ public File[] getFilesFromZippedShapefile(File zipFile) throws IOException { ZipEntry zipEntry = zis.getNextEntry(); while (zipEntry != null) { File newFile = new File(destDir, zipEntry.getName()); + if (!newFile.toPath().normalize().startsWith(destDir.toPath().normalize())) { + throw new RuntimeException("Bad zip entry"); + } FileOutputStream fos = new FileOutputStream(newFile); int len; while ((len = zis.read(buffer)) > 0) { From 4b016f01f977c6b700d8b44ef3d06584d6c3a3fe Mon Sep 17 00:00:00 2001 From: binh-dam-ibigroup <56846598+binh-dam-ibigroup@users.noreply.github.com> Date: Thu, 18 Aug 2022 16:31:46 -0400 Subject: [PATCH 010/248] refactor(EditorLockController): Support `item` query param in lock endpoint and session keys. --- .../controllers/EditorLockController.java | 137 +++++++++++------- 1 file changed, 86 insertions(+), 51 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/editor/controllers/EditorLockController.java b/src/main/java/com/conveyal/datatools/editor/controllers/EditorLockController.java index 91011ac6e..d10d1bf0f 100644 --- a/src/main/java/com/conveyal/datatools/editor/controllers/EditorLockController.java +++ b/src/main/java/com/conveyal/datatools/editor/controllers/EditorLockController.java @@ -33,51 +33,78 @@ public class EditorLockController { public static final Map sessionsForFeedIds = new HashMap<>(); private static final long SESSION_LENGTH_IN_SECONDS = 10 * 60; // Ten minutes - private static String lockFeed (Request req, Response res) { Auth0UserProfile userProfile = req.attribute("user"); + String email = userProfile.getEmail(); String feedId = req.queryParams("feedId"); - EditorSession currentSession = sessionsForFeedIds.get(feedId); + String itemToLock = req.queryParamOrDefault("item", ""); + EditorSession currentSession = sessionsForFeedIds.get(getSessionKey(feedId, itemToLock)); if (currentSession == null) { // If there is no active session for the feed ID, create a new one, which allows only the current user + // session to edit. - // Create new session - String newSessionId = invalidateAndCreateNewSession(req); - EditorSession newEditorSession = new EditorSession(feedId, newSessionId, userProfile); - sessionsForFeedIds.put(feedId, newEditorSession); - LOG.info("Locking feed {} for editing session {} by user {}", feedId, newSessionId, userProfile.getEmail()); - return formatJSON("Locking editor feed for user " + newEditorSession.userEmail, - 200, - feedId, - newSessionId); + return invalidateAndCreateNewSession( + req, + userProfile, + feedId, + itemToLock, + String.format("Locking feed for user %s on %s", email, itemToLock), + String.format("Locking feed %s for user %s on %s", feedId, email, itemToLock) + ); } - long secondsSinceLastCheckIn = TimeUnit.MILLISECONDS.toSeconds (System.currentTimeMillis() - currentSession.lastCheckIn); - long minutesSinceLastCheckIn = TimeUnit.SECONDS.toMinutes(secondsSinceLastCheckIn); - long minutesUntilExpiration = TimeUnit.SECONDS.toMinutes(SESSION_LENGTH_IN_SECONDS - secondsSinceLastCheckIn); + long secondsSinceLastCheckIn = currentSession.secondsSinceLastCheckIn(); if (secondsSinceLastCheckIn > SESSION_LENGTH_IN_SECONDS) { // There is an active session, but the user with active session has not checked in for some time. Booting // the current session in favor of new session. - // Create new session - String newSessionId = invalidateAndCreateNewSession(req); - LOG.info("User {} (session ID: {}) has not maintained lock for {} minutes. Booting.", currentSession.userEmail, currentSession.sessionId, minutesSinceLastCheckIn); - EditorSession newEditorSession = new EditorSession(feedId, newSessionId, userProfile); - sessionsForFeedIds.put(feedId, newEditorSession); - return formatJSON("Locking editor feed for user " + newEditorSession.userEmail, 200, feedId, newSessionId); + long minutesSinceLastCheckIn = TimeUnit.SECONDS.toMinutes(secondsSinceLastCheckIn); + return invalidateAndCreateNewSession( + req, + userProfile, + feedId, + itemToLock, + String.format("Locking feed for user %s on %s", email, itemToLock), + String.format("User %s has not maintained lock for %d minutes. Booting", email, minutesSinceLastCheckIn) + ); } else if (!currentSession.userId.equals(userProfile.getUser_id())) { // If the session has not expired, and another user has the active session. - LOG.warn("Edit session {} for user {} in progress for feed {}. User {} not permitted to lock feed for {} minutes.", currentSession.sessionId, currentSession.userEmail, currentSession.feedId, userProfile.getEmail(), minutesUntilExpiration); - logMessageAndHalt(req, 400, getLockedFeedMessage(currentSession, minutesUntilExpiration)); + LOG.warn( + "Edit session {} for user {} in progress for feed {}. User {} not permitted to lock feed for {} minutes.", + currentSession.sessionId, + currentSession.userEmail, + currentSession.feedId, + email, + currentSession.minutesUntilExpiration()); + logMessageAndHalt(req, 400, getLockedFeedMessage(currentSession)); return null; } else { String sessionId = req.session().id(); - LOG.warn("User {} is editing feed {} in another session {}. Cannot create lock for session {}", userProfile.getEmail(), feedId, currentSession.sessionId, sessionId); + LOG.warn("User {} is editing feed {} in another session {}. Cannot create lock for session {}", email, feedId, currentSession.sessionId, sessionId); logMessageAndHalt(req, 400, "Warning! You are editing this feed in another session/browser tab!"); return null; } } - private static String getLockedFeedMessage(EditorSession session, long minutesUntilExpiration) { + /** + * Returns a composite key made of: + * - a feed id that the lock affects, + * - an id of the item being locked by that user in a session. + */ + private static String getSessionKey(String feedId, String lockedItem) { + return String.format("%s-%s", feedId, lockedItem); + } + + private static String invalidateAndCreateNewSession(Request req, Auth0UserProfile userProfile, String feedId, String itemToLock, String message, String logMessage) { + req.session().invalidate(); + Session session = req.session(true); + String newSessionId = session.id(); + + EditorSession newEditorSession = new EditorSession(feedId, newSessionId, userProfile, itemToLock); + sessionsForFeedIds.put(getSessionKey(feedId, itemToLock), newEditorSession); + LOG.info("{} (Session ID: {})", logMessage, newSessionId); + return formatJSON(message, feedId, newSessionId); + } + + private static String getLockedFeedMessage(EditorSession session) { String timestamp = session.lastEdit > 0 ? SimpleDateFormat.getInstance().format(new Date(session.lastEdit)) : null; @@ -86,48 +113,42 @@ private static String getLockedFeedMessage(EditorSession session, long minutesUn "Warning! There is an editor session already in progress for user %s. " + "Their session will expire after %d minutes of inactivity (%s).", session.userEmail, - minutesUntilExpiration, + session.minutesUntilExpiration(), lastEditMessage); } - private static String invalidateAndCreateNewSession(Request req) { - req.session().invalidate(); - Session session = req.session(true); - return session.id(); - } - private static String maintainLock(Request req, Response res) { String sessionId = req.params("id"); String feedId = req.queryParams("feedId"); + String itemToLock = req.queryParamOrDefault("item", ""); Auth0UserProfile userProfile = req.attribute("user"); - EditorSession currentSession = sessionsForFeedIds.get(feedId); + EditorSession currentSession = sessionsForFeedIds.get(getSessionKey(feedId, itemToLock)); if (currentSession == null) { // If there is no current session to maintain, request that user reloads browser. LOG.warn("No active editor session to maintain {}.", sessionId); logMessageAndHalt(req, 400, "No active session for feedId. Please refresh your browser and try editing later."); return null; } else if (!currentSession.sessionId.equals(sessionId)) { - long secondsSinceLastCheckIn = TimeUnit.MILLISECONDS.toSeconds (System.currentTimeMillis() - currentSession.lastCheckIn); - long minutesUntilExpiration = TimeUnit.SECONDS.toMinutes(SESSION_LENGTH_IN_SECONDS - secondsSinceLastCheckIn); // If there is an active session but it doesn't match the session, someone else (or the current user) is // editing elsewhere. A user should only be trying to maintain a lock if it had an active session at one // point. If we get to this point, it is because the user's session has expired and some other session took // its place. - if (currentSession.userEmail.equals(userProfile.getEmail())) { + String email = userProfile.getEmail(); + if (currentSession.userEmail.equals(email)) { // If the new current session is held by this user, give them the option to evict the current session / // unlock the feed. - LOG.warn("User {} already has an active editor session {} for feed {}.", userProfile.getEmail(), currentSession.sessionId, currentSession.feedId); + LOG.warn("User {} already has an active editor session {} for feed {}.", email, currentSession.sessionId, currentSession.feedId); logMessageAndHalt(req, 400, "Warning! You have an active editing session for this feed underway in a different browser tab."); } else { - LOG.warn("User {} attempted editor session for feed {} while active session underway for user {}.", userProfile.getEmail(), currentSession.feedId, currentSession.userEmail); - logMessageAndHalt(req, 400, getLockedFeedMessage(currentSession, minutesUntilExpiration)); + LOG.warn("User {} attempted editor session for feed {} while active session underway for user {}.", email, currentSession.feedId, currentSession.userEmail); + logMessageAndHalt(req, 400, getLockedFeedMessage(currentSession)); } return null; } else { // Otherwise, the current session matches the session the user is attempting to maintain. Update the // lastEdited time. currentSession.lastCheckIn = System.currentTimeMillis(); - return formatJSON("Updating time for user " + currentSession.userEmail, 200, feedId, null); + return formatJSON("Updating time for user " + currentSession.userEmail, feedId, null); } } @@ -149,7 +170,8 @@ private static String deleteFeedLockBeacon(Request req, Response res) { private static String deleteFeedLockCore(Request req, Auth0UserProfile userProfile) { String feedId = req.queryParams("feedId"); String sessionId = req.params("id"); - EditorSession currentSession = sessionsForFeedIds.get(feedId); + String itemToLock = req.queryParamOrDefault("item", ""); + EditorSession currentSession = sessionsForFeedIds.get(getSessionKey(feedId, itemToLock)); if (currentSession == null) { // If there is no current session to delete/overwrite, request that user reloads browser. LOG.warn("No active session to overwrite/delete."); @@ -161,11 +183,14 @@ private static String deleteFeedLockCore(Request req, Auth0UserProfile userProfi // shared access to a feed can generally be trusted not to boot one another out in a combative manner. boolean overwrite = Boolean.parseBoolean(req.queryParams("overwrite")); if (userProfile != null && overwrite) { - sessionId = invalidateAndCreateNewSession(req); - EditorSession newEditorSession = new EditorSession(feedId, sessionId, userProfile); - sessionsForFeedIds.put(feedId, newEditorSession); - LOG.warn("Previously active session {} has been overwritten with new session {}.", currentSession.sessionId, newEditorSession.sessionId); - return formatJSON("Previous session lock has been overwritten with new session.", 200, feedId, sessionId); + return invalidateAndCreateNewSession( + req, + userProfile, + feedId, + itemToLock, + "Previous session lock has been overwritten with new session.", + String.format("Previously active session %s has been overwritten with new session.", currentSession.sessionId) + ); } else { LOG.warn("Not overwriting session {} for user {}.", currentSession.sessionId, currentSession.userEmail); return SparkUtils.formatJSON("Not processing request to delete lock. There is already an active session for user " + currentSession.userEmail, 202); @@ -182,7 +207,7 @@ private static String deleteFeedLockCore(Request req, Auth0UserProfile userProfi // the user's editing session has been closed (by either exiting the editor or closing the browser tab). LOG.info("Closed session {} for feed {} successfully.", currentSession.sessionId, currentSession.feedId); sessionsForFeedIds.remove(feedId); - return formatJSON("Session has been closed successfully.", 200, feedId, sessionId); + return formatJSON("Session has been closed successfully.", feedId, sessionId); } } @@ -195,11 +220,11 @@ public static void register(String apiPrefix) { post(apiPrefix + "deletelock/:id", EditorLockController::deleteFeedLockBeacon, json::write); } - private static String formatJSON(String message, int code, String feedId, String sessionId) { + private static String formatJSON(String message, String feedId, String sessionId) { JsonObject object = new JsonObject(); - object.addProperty("result", code >= 400 ? "ERR" : "OK"); + object.addProperty("result", "OK"); object.addProperty("message", message); - object.addProperty("code", code); + object.addProperty("code", 200); if (sessionId != null) { object.addProperty("sessionId", sessionId); } @@ -214,13 +239,23 @@ public static class EditorSession { public final String userEmail; public long lastCheckIn; public long lastEdit; + public final String lockedItem; - EditorSession (String feedId, String sessionId, Auth0UserProfile userProfile) { + EditorSession (String feedId, String sessionId, Auth0UserProfile userProfile, String itemToLock) { this.feedId = feedId; this.sessionId = sessionId; this.userId = userProfile != null ? userProfile.getUser_id() : "no_user_id"; this.userEmail = userProfile != null ? userProfile.getEmail() : "no_user_email"; - lastCheckIn = System.currentTimeMillis(); + this.lastCheckIn = System.currentTimeMillis(); + this.lockedItem = itemToLock; + } + + public long secondsSinceLastCheckIn() { + return TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis() - lastCheckIn); + } + + public long minutesUntilExpiration() { + return TimeUnit.SECONDS.toMinutes(SESSION_LENGTH_IN_SECONDS - secondsSinceLastCheckIn()); } } } From ae0c3c37a24c6d0c7a3fb904a002c86643e63698 Mon Sep 17 00:00:00 2001 From: binh-dam-ibigroup <56846598+binh-dam-ibigroup@users.noreply.github.com> Date: Fri, 19 Aug 2022 07:23:34 -0400 Subject: [PATCH 011/248] chore(pom.xml): Remove retired boundlessgeo repo. --- pom.xml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/pom.xml b/pom.xml index 144cc41c2..abb695c6f 100644 --- a/pom.xml +++ b/pom.xml @@ -197,15 +197,6 @@ always - - boundless - Boundless Maven Repository - https://repo.boundlessgeo.com/main - - true - always - - jitpack.io From 44ba003896cba46ac9be708fa755a6dddc9207f4 Mon Sep 17 00:00:00 2001 From: binh-dam-ibigroup <56846598+binh-dam-ibigroup@users.noreply.github.com> Date: Fri, 19 Aug 2022 07:37:39 -0400 Subject: [PATCH 012/248] chore(pom.xml): Update gtfs-lib version for jitpack --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index abb695c6f..e57748b52 100644 --- a/pom.xml +++ b/pom.xml @@ -261,8 +261,8 @@ com.github.conveyal gtfs-lib - - 3826eeda92f488f9d9ab4022d08be3b1281980de + + ca419a8149 From 596c1ddcac6e05cf18abab830fb3510ffa168249 Mon Sep 17 00:00:00 2001 From: binh-dam-ibigroup <56846598+binh-dam-ibigroup@users.noreply.github.com> Date: Fri, 19 Aug 2022 12:36:21 -0400 Subject: [PATCH 013/248] refactor(EditorLockController): Extract code to get current session. EditorController also passes "gtfs-lib" as locked item. --- .../controllers/EditorLockController.java | 126 ++++++++++-------- .../controllers/api/EditorController.java | 4 +- 2 files changed, 75 insertions(+), 55 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/editor/controllers/EditorLockController.java b/src/main/java/com/conveyal/datatools/editor/controllers/EditorLockController.java index d10d1bf0f..2ec10682f 100644 --- a/src/main/java/com/conveyal/datatools/editor/controllers/EditorLockController.java +++ b/src/main/java/com/conveyal/datatools/editor/controllers/EditorLockController.java @@ -33,22 +33,54 @@ public class EditorLockController { public static final Map sessionsForFeedIds = new HashMap<>(); private static final long SESSION_LENGTH_IN_SECONDS = 10 * 60; // Ten minutes + /** + * Holds useful data from an editor lock request. + */ + public static class ParsedRequest { + public final Request request; + public final Auth0UserProfile userProfile; + public final String feedId; + public final String itemToLock; + public final String sessionId; + + public ParsedRequest(Request req, String sessionId, String itemToLock) { + this.request = req; + this.userProfile = req.attribute("user"); + this.feedId = req.queryParams("feedId"); + this.itemToLock = itemToLock; + this.sessionId = sessionId; + System.out.println("sessionId: " + sessionId); + } + + public ParsedRequest(Request req) { + this(req, req.params("id"), req.queryParamOrDefault("item", "")); + } + + /** + * Returns a composite key made of: + * - a feed id that the lock affects, + * - an id of the item being locked by that user in a session. + */ + public String getSessionKey() { + return String.format("%s-%s", feedId, itemToLock); + } + } + + public static EditorSession getCurrentSession(ParsedRequest req) { + return sessionsForFeedIds.get(req.getSessionKey()); + } + private static String lockFeed (Request req, Response res) { - Auth0UserProfile userProfile = req.attribute("user"); - String email = userProfile.getEmail(); - String feedId = req.queryParams("feedId"); - String itemToLock = req.queryParamOrDefault("item", ""); - EditorSession currentSession = sessionsForFeedIds.get(getSessionKey(feedId, itemToLock)); + ParsedRequest parsedReq = new ParsedRequest(req); + EditorSession currentSession = getCurrentSession(parsedReq); + String email = parsedReq.userProfile.getEmail(); if (currentSession == null) { // If there is no active session for the feed ID, create a new one, which allows only the current user + // session to edit. return invalidateAndCreateNewSession( - req, - userProfile, - feedId, - itemToLock, - String.format("Locking feed for user %s on %s", email, itemToLock), - String.format("Locking feed %s for user %s on %s", feedId, email, itemToLock) + parsedReq, + String.format("Locking feed for user %s on %s", email, parsedReq.itemToLock), + String.format("Locking feed %s for user %s on %s", parsedReq.feedId, email, parsedReq.itemToLock) ); } @@ -58,14 +90,11 @@ private static String lockFeed (Request req, Response res) { // the current session in favor of new session. long minutesSinceLastCheckIn = TimeUnit.SECONDS.toMinutes(secondsSinceLastCheckIn); return invalidateAndCreateNewSession( - req, - userProfile, - feedId, - itemToLock, - String.format("Locking feed for user %s on %s", email, itemToLock), + parsedReq, + String.format("Locking feed for user %s on %s", email, parsedReq.itemToLock), String.format("User %s has not maintained lock for %d minutes. Booting", email, minutesSinceLastCheckIn) ); - } else if (!currentSession.userId.equals(userProfile.getUser_id())) { + } else if (!currentSession.userId.equals(parsedReq.userProfile.getUser_id())) { // If the session has not expired, and another user has the active session. LOG.warn( "Edit session {} for user {} in progress for feed {}. User {} not permitted to lock feed for {} minutes.", @@ -78,30 +107,27 @@ private static String lockFeed (Request req, Response res) { return null; } else { String sessionId = req.session().id(); - LOG.warn("User {} is editing feed {} in another session {}. Cannot create lock for session {}", email, feedId, currentSession.sessionId, sessionId); + LOG.warn( + "User {} is editing feed {} in another session {}. Cannot create lock for session {}", + email, + parsedReq.feedId, + currentSession.sessionId, + sessionId + ); logMessageAndHalt(req, 400, "Warning! You are editing this feed in another session/browser tab!"); return null; } } - /** - * Returns a composite key made of: - * - a feed id that the lock affects, - * - an id of the item being locked by that user in a session. - */ - private static String getSessionKey(String feedId, String lockedItem) { - return String.format("%s-%s", feedId, lockedItem); - } - - private static String invalidateAndCreateNewSession(Request req, Auth0UserProfile userProfile, String feedId, String itemToLock, String message, String logMessage) { - req.session().invalidate(); - Session session = req.session(true); + private static String invalidateAndCreateNewSession(ParsedRequest req, String message, String logMessage) { + req.request.session().invalidate(); + Session session = req.request.session(true); String newSessionId = session.id(); - EditorSession newEditorSession = new EditorSession(feedId, newSessionId, userProfile, itemToLock); - sessionsForFeedIds.put(getSessionKey(feedId, itemToLock), newEditorSession); + EditorSession newEditorSession = new EditorSession(req.feedId, newSessionId, req.userProfile, req.itemToLock); + sessionsForFeedIds.put(req.getSessionKey(), newEditorSession); LOG.info("{} (Session ID: {})", logMessage, newSessionId); - return formatJSON(message, feedId, newSessionId); + return formatJSON(message, req.feedId, newSessionId); } private static String getLockedFeedMessage(EditorSession session) { @@ -118,22 +144,19 @@ private static String getLockedFeedMessage(EditorSession session) { } private static String maintainLock(Request req, Response res) { - String sessionId = req.params("id"); - String feedId = req.queryParams("feedId"); - String itemToLock = req.queryParamOrDefault("item", ""); - Auth0UserProfile userProfile = req.attribute("user"); - EditorSession currentSession = sessionsForFeedIds.get(getSessionKey(feedId, itemToLock)); + ParsedRequest parsedReq = new ParsedRequest(req); + EditorSession currentSession = getCurrentSession(parsedReq); if (currentSession == null) { // If there is no current session to maintain, request that user reloads browser. - LOG.warn("No active editor session to maintain {}.", sessionId); + LOG.warn("No active editor session to maintain {}.", parsedReq.sessionId); logMessageAndHalt(req, 400, "No active session for feedId. Please refresh your browser and try editing later."); return null; - } else if (!currentSession.sessionId.equals(sessionId)) { + } else if (!currentSession.sessionId.equals(parsedReq.sessionId)) { // If there is an active session but it doesn't match the session, someone else (or the current user) is // editing elsewhere. A user should only be trying to maintain a lock if it had an active session at one // point. If we get to this point, it is because the user's session has expired and some other session took // its place. - String email = userProfile.getEmail(); + String email = parsedReq.userProfile.getEmail(); if (currentSession.userEmail.equals(email)) { // If the new current session is held by this user, give them the option to evict the current session / // unlock the feed. @@ -148,7 +171,7 @@ private static String maintainLock(Request req, Response res) { // Otherwise, the current session matches the session the user is attempting to maintain. Update the // lastEdited time. currentSession.lastCheckIn = System.currentTimeMillis(); - return formatJSON("Updating time for user " + currentSession.userEmail, feedId, null); + return formatJSON("Updating time for user " + currentSession.userEmail, parsedReq.feedId, null); } } @@ -168,15 +191,13 @@ private static String deleteFeedLockBeacon(Request req, Response res) { } private static String deleteFeedLockCore(Request req, Auth0UserProfile userProfile) { - String feedId = req.queryParams("feedId"); - String sessionId = req.params("id"); - String itemToLock = req.queryParamOrDefault("item", ""); - EditorSession currentSession = sessionsForFeedIds.get(getSessionKey(feedId, itemToLock)); + ParsedRequest parsedReq = new ParsedRequest(req); + EditorSession currentSession = getCurrentSession(parsedReq); if (currentSession == null) { // If there is no current session to delete/overwrite, request that user reloads browser. LOG.warn("No active session to overwrite/delete."); return SparkUtils.formatJSON("No active session to take over. Please refresh your browser and try editing later.", 202); - } else if (!currentSession.sessionId.equals(sessionId)) { + } else if (!currentSession.sessionId.equals(parsedReq.sessionId)) { // If there is a different active session for some user, allow deletion / overwrite. // Note: There used to be a check here that the requesting user was the same as the user with an open // session; however, this has been removed because in practice it became a nuisance. Respectful users with @@ -184,10 +205,7 @@ private static String deleteFeedLockCore(Request req, Auth0UserProfile userProfi boolean overwrite = Boolean.parseBoolean(req.queryParams("overwrite")); if (userProfile != null && overwrite) { return invalidateAndCreateNewSession( - req, - userProfile, - feedId, - itemToLock, + parsedReq, "Previous session lock has been overwritten with new session.", String.format("Previously active session %s has been overwritten with new session.", currentSession.sessionId) ); @@ -201,13 +219,13 @@ private static String deleteFeedLockCore(Request req, Auth0UserProfile userProfi currentSession.userEmail, currentSession.sessionId, userProfile != null ? userProfile.getEmail() : "(email unavailable)", - sessionId + parsedReq.sessionId ); // Otherwise, the current session matches the session from which the delete request came. This indicates that // the user's editing session has been closed (by either exiting the editor or closing the browser tab). LOG.info("Closed session {} for feed {} successfully.", currentSession.sessionId, currentSession.feedId); - sessionsForFeedIds.remove(feedId); - return formatJSON("Session has been closed successfully.", feedId, sessionId); + sessionsForFeedIds.remove(parsedReq.getSessionKey()); + return formatJSON("Session has been closed successfully.", parsedReq.feedId, parsedReq.sessionId); } } diff --git a/src/main/java/com/conveyal/datatools/editor/controllers/api/EditorController.java b/src/main/java/com/conveyal/datatools/editor/controllers/api/EditorController.java index ea73c0cdf..44b4e7464 100644 --- a/src/main/java/com/conveyal/datatools/editor/controllers/api/EditorController.java +++ b/src/main/java/com/conveyal/datatools/editor/controllers/api/EditorController.java @@ -425,7 +425,9 @@ private static String getNamespaceAndValidateSession(Request req) { // Only check for editing session if not in testing environment. // TODO: Add way to mock session. if (!inTestingEnvironment()) { - EditorLockController.EditorSession currentSession = sessionsForFeedIds.get(feedId); + // TODO: Refactor, looks like duplicate code + EditorLockController.ParsedRequest parsedReq = new EditorLockController.ParsedRequest(req, sessionId, "gtfs-editor"); + EditorLockController.EditorSession currentSession = EditorLockController.getCurrentSession(parsedReq); if (currentSession == null) { logMessageAndHalt(req, 400, "There is no active editing session for user."); } From bbce27d784c5df6f41dfb1a3eccd8ba1e19e921e Mon Sep 17 00:00:00 2001 From: binh-dam-ibigroup <56846598+binh-dam-ibigroup@users.noreply.github.com> Date: Fri, 19 Aug 2022 13:34:23 -0400 Subject: [PATCH 014/248] refactor(EditorController): Obtain session based on session id. --- .../controllers/EditorLockController.java | 28 ++++++++++++------- .../controllers/api/EditorController.java | 4 +-- 2 files changed, 19 insertions(+), 13 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/editor/controllers/EditorLockController.java b/src/main/java/com/conveyal/datatools/editor/controllers/EditorLockController.java index 2ec10682f..e0c3ca7fe 100644 --- a/src/main/java/com/conveyal/datatools/editor/controllers/EditorLockController.java +++ b/src/main/java/com/conveyal/datatools/editor/controllers/EditorLockController.java @@ -30,8 +30,8 @@ public class EditorLockController { private static final JsonManager json = new JsonManager<>(EditorLockController.class, JsonViews.UserInterface.class); private static final Logger LOG = LoggerFactory.getLogger(EditorLockController.class); - public static final Map sessionsForFeedIds = new HashMap<>(); - private static final long SESSION_LENGTH_IN_SECONDS = 10 * 60; // Ten minutes + private static final Map sessionsForFeedIds = new HashMap<>(); + private static final long SESSION_LENGTH_IN_SECONDS = 10 * 60L; // Ten minutes /** * Holds useful data from an editor lock request. @@ -43,17 +43,12 @@ public static class ParsedRequest { public final String itemToLock; public final String sessionId; - public ParsedRequest(Request req, String sessionId, String itemToLock) { + public ParsedRequest(Request req) { this.request = req; this.userProfile = req.attribute("user"); this.feedId = req.queryParams("feedId"); - this.itemToLock = itemToLock; - this.sessionId = sessionId; - System.out.println("sessionId: " + sessionId); - } - - public ParsedRequest(Request req) { - this(req, req.params("id"), req.queryParamOrDefault("item", "")); + this.itemToLock = req.queryParamOrDefault("item", ""); + this.sessionId = req.params("id"); } /** @@ -66,10 +61,23 @@ public String getSessionKey() { } } + /** + * Returns the current session based on the info provided. + */ public static EditorSession getCurrentSession(ParsedRequest req) { return sessionsForFeedIds.get(req.getSessionKey()); } + /** + * Returns the session based on its id. + */ + public static EditorSession getSession(String sessionId) { + return sessionsForFeedIds.values().stream() + .filter(s -> s.sessionId.equals(sessionId)) + .findFirst() + .orElse(null); + } + private static String lockFeed (Request req, Response res) { ParsedRequest parsedReq = new ParsedRequest(req); EditorSession currentSession = getCurrentSession(parsedReq); diff --git a/src/main/java/com/conveyal/datatools/editor/controllers/api/EditorController.java b/src/main/java/com/conveyal/datatools/editor/controllers/api/EditorController.java index 44b4e7464..114a3b6a8 100644 --- a/src/main/java/com/conveyal/datatools/editor/controllers/api/EditorController.java +++ b/src/main/java/com/conveyal/datatools/editor/controllers/api/EditorController.java @@ -40,7 +40,6 @@ import static com.conveyal.datatools.common.utils.SparkUtils.formatJSON; import static com.conveyal.datatools.common.utils.SparkUtils.getObjectNode; import static com.conveyal.datatools.common.utils.SparkUtils.logMessageAndHalt; -import static com.conveyal.datatools.editor.controllers.EditorLockController.sessionsForFeedIds; import static com.conveyal.datatools.manager.controllers.api.UserController.inTestingEnvironment; import static spark.Spark.delete; import static spark.Spark.options; @@ -426,8 +425,7 @@ private static String getNamespaceAndValidateSession(Request req) { // TODO: Add way to mock session. if (!inTestingEnvironment()) { // TODO: Refactor, looks like duplicate code - EditorLockController.ParsedRequest parsedReq = new EditorLockController.ParsedRequest(req, sessionId, "gtfs-editor"); - EditorLockController.EditorSession currentSession = EditorLockController.getCurrentSession(parsedReq); + EditorLockController.EditorSession currentSession = EditorLockController.getSession(sessionId); if (currentSession == null) { logMessageAndHalt(req, 400, "There is no active editing session for user."); } From 27ce86a519ce77f36ab29881e8bbe1d15ff34073 Mon Sep 17 00:00:00 2001 From: binh-dam-ibigroup <56846598+binh-dam-ibigroup@users.noreply.github.com> Date: Fri, 19 Aug 2022 15:08:26 -0400 Subject: [PATCH 015/248] refactor(EditorController): Reuse check for active user session from EditorLockController. --- .../controllers/EditorLockController.java | 88 ++++++++++--------- .../controllers/api/EditorController.java | 19 +--- 2 files changed, 50 insertions(+), 57 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/editor/controllers/EditorLockController.java b/src/main/java/com/conveyal/datatools/editor/controllers/EditorLockController.java index e0c3ca7fe..6f3ab2b5f 100644 --- a/src/main/java/com/conveyal/datatools/editor/controllers/EditorLockController.java +++ b/src/main/java/com/conveyal/datatools/editor/controllers/EditorLockController.java @@ -33,34 +33,6 @@ public class EditorLockController { private static final Map sessionsForFeedIds = new HashMap<>(); private static final long SESSION_LENGTH_IN_SECONDS = 10 * 60L; // Ten minutes - /** - * Holds useful data from an editor lock request. - */ - public static class ParsedRequest { - public final Request request; - public final Auth0UserProfile userProfile; - public final String feedId; - public final String itemToLock; - public final String sessionId; - - public ParsedRequest(Request req) { - this.request = req; - this.userProfile = req.attribute("user"); - this.feedId = req.queryParams("feedId"); - this.itemToLock = req.queryParamOrDefault("item", ""); - this.sessionId = req.params("id"); - } - - /** - * Returns a composite key made of: - * - a feed id that the lock affects, - * - an id of the item being locked by that user in a session. - */ - public String getSessionKey() { - return String.format("%s-%s", feedId, itemToLock); - } - } - /** * Returns the current session based on the info provided. */ @@ -69,11 +41,11 @@ public static EditorSession getCurrentSession(ParsedRequest req) { } /** - * Returns the session based on its id. + * Returns a session based on a feed id. */ - public static EditorSession getSession(String sessionId) { + public static EditorSession getSession(String feedId) { return sessionsForFeedIds.values().stream() - .filter(s -> s.sessionId.equals(sessionId)) + .filter(s -> s.feedId.equals(feedId)) .findFirst() .orElse(null); } @@ -151,20 +123,17 @@ private static String getLockedFeedMessage(EditorSession session) { lastEditMessage); } - private static String maintainLock(Request req, Response res) { - ParsedRequest parsedReq = new ParsedRequest(req); - EditorSession currentSession = getCurrentSession(parsedReq); + public static boolean checkUserHasActiveSession(Request req, String sessionId, String email, EditorSession currentSession) { if (currentSession == null) { // If there is no current session to maintain, request that user reloads browser. - LOG.warn("No active editor session to maintain {}.", parsedReq.sessionId); + LOG.warn("No active editor session to maintain {}.", sessionId); logMessageAndHalt(req, 400, "No active session for feedId. Please refresh your browser and try editing later."); - return null; - } else if (!currentSession.sessionId.equals(parsedReq.sessionId)) { + return false; + } else if (!currentSession.sessionId.equals(sessionId)) { // If there is an active session but it doesn't match the session, someone else (or the current user) is // editing elsewhere. A user should only be trying to maintain a lock if it had an active session at one // point. If we get to this point, it is because the user's session has expired and some other session took // its place. - String email = parsedReq.userProfile.getEmail(); if (currentSession.userEmail.equals(email)) { // If the new current session is held by this user, give them the option to evict the current session / // unlock the feed. @@ -174,12 +143,21 @@ private static String maintainLock(Request req, Response res) { LOG.warn("User {} attempted editor session for feed {} while active session underway for user {}.", email, currentSession.feedId, currentSession.userEmail); logMessageAndHalt(req, 400, getLockedFeedMessage(currentSession)); } - return null; - } else { - // Otherwise, the current session matches the session the user is attempting to maintain. Update the + return false; + } + return true; + } + + private static String maintainLock(Request req, Response res) { + ParsedRequest parsedReq = new ParsedRequest(req); + EditorSession currentSession = getCurrentSession(parsedReq); + if (checkUserHasActiveSession(req, parsedReq.sessionId, parsedReq.userProfile.getEmail(), currentSession)) { + // If the current session matches the session the user is attempting to maintain. Update the // lastEdited time. currentSession.lastCheckIn = System.currentTimeMillis(); return formatJSON("Updating time for user " + currentSession.userEmail, parsedReq.feedId, null); + } else { + return null; } } @@ -284,4 +262,32 @@ public long minutesUntilExpiration() { return TimeUnit.SECONDS.toMinutes(SESSION_LENGTH_IN_SECONDS - secondsSinceLastCheckIn()); } } + + /** + * Holds useful data from an editor lock request. + */ + public static class ParsedRequest { + public final Request request; + public final Auth0UserProfile userProfile; + public final String feedId; + public final String itemToLock; + public final String sessionId; + + public ParsedRequest(Request req) { + this.request = req; + this.userProfile = req.attribute("user"); + this.feedId = req.queryParams("feedId"); + this.itemToLock = req.queryParamOrDefault("item", ""); + this.sessionId = req.params("id"); + } + + /** + * Returns a composite key made of: + * - a feed id that the lock affects, + * - an id of the item being locked by that user in a session. + */ + public String getSessionKey() { + return String.format("%s-%s", feedId, itemToLock); + } + } } diff --git a/src/main/java/com/conveyal/datatools/editor/controllers/api/EditorController.java b/src/main/java/com/conveyal/datatools/editor/controllers/api/EditorController.java index 114a3b6a8..371499300 100644 --- a/src/main/java/com/conveyal/datatools/editor/controllers/api/EditorController.java +++ b/src/main/java/com/conveyal/datatools/editor/controllers/api/EditorController.java @@ -424,22 +424,9 @@ private static String getNamespaceAndValidateSession(Request req) { // Only check for editing session if not in testing environment. // TODO: Add way to mock session. if (!inTestingEnvironment()) { - // TODO: Refactor, looks like duplicate code - EditorLockController.EditorSession currentSession = EditorLockController.getSession(sessionId); - if (currentSession == null) { - logMessageAndHalt(req, 400, "There is no active editing session for user."); - } - if (!currentSession.sessionId.equals(sessionId)) { - // This session does not match the current active session for the feed. - Auth0UserProfile userProfile = req.attribute("user"); - if (currentSession.userEmail.equals(userProfile.getEmail())) { - LOG.warn("User {} already has editor session {} for feed {}. Same user cannot make edits on session {}.", currentSession.userEmail, currentSession.sessionId, feedId, req.session().id()); - logMessageAndHalt(req, 400, "You have another editing session open for " + feedSource.name); - } else { - LOG.warn("User {} already has editor session {} for feed {}. User {} cannot make edits on session {}.", currentSession.userEmail, currentSession.sessionId, feedId, userProfile.getEmail(), req.session().id()); - logMessageAndHalt(req, 400, "Somebody else is editing the " + feedSource.name + " feed."); - } - } else { + Auth0UserProfile userProfile = req.attribute("user"); + EditorLockController.EditorSession currentSession = EditorLockController.getSession(feedId); + if (EditorLockController.checkUserHasActiveSession(req, sessionId, userProfile.getEmail(), currentSession)) { currentSession.lastEdit = System.currentTimeMillis(); LOG.info("Updating session {} last edit time to {}", sessionId, currentSession.lastEdit); } From 954d80b166c42e3b8ea0b66962a00666ca77ecf1 Mon Sep 17 00:00:00 2001 From: "philip.cline" Date: Tue, 23 Aug 2022 09:47:43 -0400 Subject: [PATCH 016/248] fix(FeedVersionSummary): check if validationResult exists for old feeds --- .../datatools/manager/models/FeedVersionSummary.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedVersionSummary.java b/src/main/java/com/conveyal/datatools/manager/models/FeedVersionSummary.java index 54fe453a8..be2413a9a 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedVersionSummary.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedVersionSummary.java @@ -56,8 +56,11 @@ public class PartialValidationSummary { public LocalDate endDate; PartialValidationSummary() { - this.startDate = validationResult.firstCalendarDate; - this.endDate = validationResult.lastCalendarDate; + // Older feeds created in datatools may not have validationResult + if (validationResult != null) { + this.startDate = validationResult.firstCalendarDate; + this.endDate = validationResult.lastCalendarDate; + } } } } From 0f428ba0bee38bd435ff20e70e836faac6ea4b7e Mon Sep 17 00:00:00 2001 From: "philip.cline" Date: Tue, 23 Aug 2022 10:28:05 -0400 Subject: [PATCH 017/248] refactor(dependencies): apply boundless geo fix so CI tests pass --- pom.xml | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/pom.xml b/pom.xml index 144cc41c2..e57748b52 100644 --- a/pom.xml +++ b/pom.xml @@ -197,15 +197,6 @@ always - - boundless - Boundless Maven Repository - https://repo.boundlessgeo.com/main - - true - always - - jitpack.io @@ -270,8 +261,8 @@ com.github.conveyal gtfs-lib - - 3826eeda92f488f9d9ab4022d08be3b1281980de + + ca419a8149 From f6d73d31981229dce0dedcc8020f89a459acc354 Mon Sep 17 00:00:00 2001 From: binh-dam-ibigroup <56846598+binh-dam-ibigroup@users.noreply.github.com> Date: Tue, 6 Sep 2022 11:17:29 -0400 Subject: [PATCH 018/248] refactor(EditorLockController): Rename successful JSON output method. --- .../editor/controllers/EditorLockController.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/editor/controllers/EditorLockController.java b/src/main/java/com/conveyal/datatools/editor/controllers/EditorLockController.java index 6f3ab2b5f..507f59395 100644 --- a/src/main/java/com/conveyal/datatools/editor/controllers/EditorLockController.java +++ b/src/main/java/com/conveyal/datatools/editor/controllers/EditorLockController.java @@ -107,7 +107,7 @@ private static String invalidateAndCreateNewSession(ParsedRequest req, String me EditorSession newEditorSession = new EditorSession(req.feedId, newSessionId, req.userProfile, req.itemToLock); sessionsForFeedIds.put(req.getSessionKey(), newEditorSession); LOG.info("{} (Session ID: {})", logMessage, newSessionId); - return formatJSON(message, req.feedId, newSessionId); + return formatSuccessJSON(message, req.feedId, newSessionId); } private static String getLockedFeedMessage(EditorSession session) { @@ -155,7 +155,7 @@ private static String maintainLock(Request req, Response res) { // If the current session matches the session the user is attempting to maintain. Update the // lastEdited time. currentSession.lastCheckIn = System.currentTimeMillis(); - return formatJSON("Updating time for user " + currentSession.userEmail, parsedReq.feedId, null); + return formatSuccessJSON("Updating time for user " + currentSession.userEmail, parsedReq.feedId, null); } else { return null; } @@ -211,7 +211,7 @@ private static String deleteFeedLockCore(Request req, Auth0UserProfile userProfi // the user's editing session has been closed (by either exiting the editor or closing the browser tab). LOG.info("Closed session {} for feed {} successfully.", currentSession.sessionId, currentSession.feedId); sessionsForFeedIds.remove(parsedReq.getSessionKey()); - return formatJSON("Session has been closed successfully.", parsedReq.feedId, parsedReq.sessionId); + return formatSuccessJSON("Session has been closed successfully.", parsedReq.feedId, parsedReq.sessionId); } } @@ -224,7 +224,7 @@ public static void register(String apiPrefix) { post(apiPrefix + "deletelock/:id", EditorLockController::deleteFeedLockBeacon, json::write); } - private static String formatJSON(String message, String feedId, String sessionId) { + private static String formatSuccessJSON(String message, String feedId, String sessionId) { JsonObject object = new JsonObject(); object.addProperty("result", "OK"); object.addProperty("message", message); From cf1912fbf46c05a78c90bed839dc710556bcef5b Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Fri, 9 Sep 2022 10:18:10 -0400 Subject: [PATCH 019/248] refactor: remove remains of old docker approach --- .gitignore | 1 - Dockerfile e2e | 11 ----- configurations/docker/env.yml.tmp | 19 -------- configurations/docker/server.yml.tmp | 65 ---------------------------- docker-compose.e2e.yml | 26 ----------- docker-compose.yml | 10 +---- 6 files changed, 1 insertion(+), 131 deletions(-) delete mode 100644 Dockerfile e2e delete mode 100644 configurations/docker/env.yml.tmp delete mode 100644 configurations/docker/server.yml.tmp delete mode 100644 docker-compose.e2e.yml diff --git a/.gitignore b/.gitignore index ff81a64e7..c4fe5d5ac 100644 --- a/.gitignore +++ b/.gitignore @@ -17,7 +17,6 @@ deploy/ configurations/*.yml !configurations/default !configurations/test -!configurations/docker # Secret config files .env diff --git a/Dockerfile e2e b/Dockerfile e2e deleted file mode 100644 index 1f69e5927..000000000 --- a/Dockerfile e2e +++ /dev/null @@ -1,11 +0,0 @@ -# syntax=docker/dockerfile:1 -FROM maven:3-jdk-8-alpine -WORKDIR /datatools - -# Get code into container -COPY ./ . -COPY ./configurations/docker/env.yml ./configurations/test/env.yml.tmp -COPY ./configurations/docker/server.yml ./configurations/test/server.yml.tmp - -ENV SHOULD_RUN_E2E=true -CMD ["mvn", "test"] \ No newline at end of file diff --git a/configurations/docker/env.yml.tmp b/configurations/docker/env.yml.tmp deleted file mode 100644 index 1081f48d7..000000000 --- a/configurations/docker/env.yml.tmp +++ /dev/null @@ -1,19 +0,0 @@ -# This client ID refers to the UI client in Auth0. -AUTH0_CLIENT_ID: your-auth0-client-id -AUTH0_DOMAIN: your-auth0-domain -# Note: One of AUTH0_SECRET or AUTH0_PUBLIC_KEY should be used depending on the signing algorithm set on the client. -# It seems that newer Auth0 accounts (2017 and later) might default to RS256 (public key). -AUTH0_SECRET: your-auth0-secret # uses HS256 signing algorithm -# AUTH0_PUBLIC_KEY: /path/to/auth0.pem # uses RS256 signing algorithm -# This client/secret pair refer to a machine-to-machine Auth0 application used to access the Management API. -AUTH0_API_CLIENT: your-api-client-id -AUTH0_API_SECRET: your-api-secret-id -DISABLE_AUTH: false -OSM_VEX: http://localhost:1000 -SPARKPOST_KEY: your-sparkpost-key -SPARKPOST_EMAIL: email@example.com -GTFS_DATABASE_URL: jdbc:postgresql://localhost/catalogue -# GTFS_DATABASE_USER: -# GTFS_DATABASE_PASSWORD: -MONGO_DB_NAME: data_manager -MONGO_HOST: mongo:27017 diff --git a/configurations/docker/server.yml.tmp b/configurations/docker/server.yml.tmp deleted file mode 100644 index 2e7c136e6..000000000 --- a/configurations/docker/server.yml.tmp +++ /dev/null @@ -1,65 +0,0 @@ -application: - title: Data Tools - logo: https://d2tyb7byn1fef9.cloudfront.net/ibi_group-128x128.png - logo_large: https://d2tyb7byn1fef9.cloudfront.net/ibi_group_black-512x512.png - client_assets_url: http://localhost:8888/ - shortcut_icon_url: https://d2tyb7byn1fef9.cloudfront.net/ibi-logo-original%402x.png - public_url: http://localhost:9966 - notifications_enabled: false - docs_url: http://conveyal-data-tools.readthedocs.org - support_email: support@ibigroup.com - port: 4000 - data: - gtfs: /tmp - use_s3_storage: false - s3_region: us-east-1 - gtfs_s3_bucket: bucket-name -modules: - enterprise: - enabled: false - editor: - enabled: true - deployment: - enabled: true - ec2: - enabled: false - default_ami: ami-your-ami-id - # Note: using a cloudfront URL for these download URLs will greatly - # increase download/deploy speed. - otp_download_url: https://optional-otp-repo.com - user_admin: - enabled: true - gtfsapi: - enabled: true - load_on_fetch: false - # use_extension: mtc - # update_frequency: 30 # in seconds - manager: - normalizeFieldTransformation: - # Enter capitalization exceptions (e.g. acronyms), in the desired case, and separated by commas. - defaultCapitalizationExceptions: - - ACE - - BART - # Enter substitutions (e.g. substitute '@' with 'at'), one dashed entry for each substitution, with: - # - pattern: the regex string pattern that will be replaced, - # - replacement: the replacement string for that pattern, - # - normalizeSpace: if true, the resulting field value will include one space before and after the replacement string. - # Note: if the replacement must be blank, then normalizeSpace should be set to false - # and whitespace management should be handled in pattern instead. - # Substitutions are executed in order they appear in the list. - defaultSubstitutions: - - description: "Replace '@' with 'at', and normalize space." - pattern: "@" - replacement: at - normalizeSpace: true - - description: "Replace '+' (\\+ in regex) and '&' with 'and', and normalize space." - pattern: "[\\+&]" - replacement: and - normalizeSpace: true -extensions: - transitland: - enabled: true - api: https://transit.land/api/v1/feeds - transitfeeds: - enabled: true - api: http://api.transitfeeds.com/v1/getFeeds diff --git a/docker-compose.e2e.yml b/docker-compose.e2e.yml deleted file mode 100644 index 2a3627a64..000000000 --- a/docker-compose.e2e.yml +++ /dev/null @@ -1,26 +0,0 @@ -version: '3.8' -services: - datatools-server: - build: - context: ./ - dockerfile: ./Dockerfile e2e - volumes: - - type: bind - source: ./configurations/docker/ - target: /config - - type: bind - source: ~/.aws - target: /root/.aws - depends_on: - - mongo - - postgres - mongo: - image: mongo - restart: always - postgres: - environment: - POSTGRES_HOST_AUTH_METHOD: trust - POSTGRES_USER: root - POSTGRES_DB: dmtest - image: postgres - restart: always \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 76c856e84..d9fa79e26 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,16 +1,9 @@ version: '3.8' services: - datatools-ui: - # Temporary! move to image (?) -- includes config variables - build: ../datatools-ui - restart: always - ports: - - "8888:80" datatools-server: - # Temporary! move to image build: ./ ports: - - "9966:4000" + - "4000:4000" volumes: - type: bind source: ./configurations/docker/ @@ -21,7 +14,6 @@ services: depends_on: - mongo - postgres - - datatools-ui mongo: image: mongo restart: always From b28699c721dbe78395e9dcfed02fac9d1c16d620 Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Fri, 9 Sep 2022 10:53:30 -0400 Subject: [PATCH 020/248] refactor: add docker image and compose file --- .github/workflows/build-docker.yml | 43 ++++++++++++++++++++++++++++++ Dockerfile | 17 +++++++----- configurations/default/env.yml.tmp | 4 +-- docker-compose.yml | 2 +- 4 files changed, 57 insertions(+), 9 deletions(-) create mode 100644 .github/workflows/build-docker.yml diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml new file mode 100644 index 000000000..73f9ba629 --- /dev/null +++ b/.github/workflows/build-docker.yml @@ -0,0 +1,43 @@ +name: Create and publish a Docker image +# https://docs.github.com/en/actions/publishing-packages/publishing-docker-images + +on: [push] + # push: + # branches: ['dev', 'master'] + + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +jobs: + build-and-push-image: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Log in to the Container registry + uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + + - name: Build and push Docker image + uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc + with: + context: . + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 0400fc7b9..0b92958a8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,12 +1,17 @@ # syntax=docker/dockerfile:1 -FROM openjdk:11 +FROM maven:3.8.6-openjdk-11 + +COPY . /datatools + +# Build jar WORKDIR /datatools +RUN mvn package -DskipTests +RUN cp target/dt*.jar ./datatools-server.jar -# Grab latest dev build -COPY target/dt*.jar ./datatools-server-3.8.1-SNAPSHOT.jar +RUN mkdir -p /var/datatools_gtfs/gtfsplus -RUN mkdir -p /var/datatools_gtfs -# Launch server (relies on env.yml being placed in volume!) +# Launch server +# This relies on a configuration volume and aws volume being present. See `docker-compose.yml`, or the example below # Try: docker run --publish 4000:4000 -v ~/config/:/config datatools-latest -CMD ["java", "-jar", "datatools-server-3.8.1-SNAPSHOT.jar", "/config/env.yml", "/config/server.yml"] +CMD ["java", "-jar", "datatools-server.jar", "/config/env.yml", "/config/server.yml"] EXPOSE 4000 \ No newline at end of file diff --git a/configurations/default/env.yml.tmp b/configurations/default/env.yml.tmp index 119b399a6..91e53f92b 100644 --- a/configurations/default/env.yml.tmp +++ b/configurations/default/env.yml.tmp @@ -12,8 +12,8 @@ DISABLE_AUTH: false OSM_VEX: http://localhost:1000 SPARKPOST_KEY: your-sparkpost-key SPARKPOST_EMAIL: email@example.com -GTFS_DATABASE_URL: jdbc:postgresql://localhost/catalogue +GTFS_DATABASE_URL: jdbc:postgresql://localhost/catalogue # If running via docker, this is jdbc:postgresql://postgres/dmtest # GTFS_DATABASE_USER: # GTFS_DATABASE_PASSWORD: -#MONGO_HOST: mongo-host:27017 +#MONGO_HOST: mongo-host:27017 # If running via docker, this is mongo:27017 MONGO_DB_NAME: catalogue diff --git a/docker-compose.yml b/docker-compose.yml index d9fa79e26..cc847068d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,7 +6,7 @@ services: - "4000:4000" volumes: - type: bind - source: ./configurations/docker/ + source: ./configurations/default/ target: /config - type: bind source: ~/.aws From 705d2673a933ebff95e0c61a4f226e8661db6681 Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Fri, 9 Sep 2022 10:58:54 -0400 Subject: [PATCH 021/248] =?UTF-8?q?refactor(build-docker.yml):=20don?= =?UTF-8?q?=E2=80=99t=20build=20docker=20image=20on=20every=20commit?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/build-docker.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml index 73f9ba629..e0702e703 100644 --- a/.github/workflows/build-docker.yml +++ b/.github/workflows/build-docker.yml @@ -1,10 +1,9 @@ name: Create and publish a Docker image # https://docs.github.com/en/actions/publishing-packages/publishing-docker-images -on: [push] - # push: - # branches: ['dev', 'master'] - +on: + push: + branches: ['dev', 'master', 'dev-flex'] env: REGISTRY: ghcr.io From 868e786e6c4cad7b32ae6e284b45833b508886ea Mon Sep 17 00:00:00 2001 From: miles-grant-ibigroup <86619099+miles-grant-ibigroup@users.noreply.github.com> Date: Fri, 9 Sep 2022 11:03:07 -0400 Subject: [PATCH 022/248] ci: add codeql analysis --- .github/workflows/codeql-analysis.yml | 72 +++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 .github/workflows/codeql-analysis.yml diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml new file mode 100644 index 000000000..4da74de30 --- /dev/null +++ b/.github/workflows/codeql-analysis.yml @@ -0,0 +1,72 @@ +# For most projects, this workflow file will not need changing; you simply need +# to commit it to your repository. +# +# You may wish to alter this file to override the set of languages analyzed, +# or to provide custom queries or build logic. +# +# ******** NOTE ******** +# We have attempted to detect the languages in your repository. Please check +# the `language` matrix defined below to confirm you have the correct set of +# supported CodeQL languages. +# +name: "CodeQL" + +on: + push: + branches: [ "dev", master ] + pull_request: + # The branches below must be a subset of the branches above + branches: [ "dev" ] + schedule: + - cron: '22 22 * * 2' + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + security-events: write + + strategy: + fail-fast: false + matrix: + language: [ 'java', 'javascript', 'python' ] + # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] + # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v2 + with: + languages: ${{ matrix.language }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + + # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs + # queries: security-extended,security-and-quality + + + # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). + # If this step fails, then you should remove it and run the build manually (see below) + - name: Autobuild + uses: github/codeql-action/autobuild@v2 + + # ℹ️ Command-line programs to run using the OS shell. + # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun + + # If the Autobuild fails above, remove it and uncomment the following three lines. + # modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance. + + # - run: | + # echo "Run, Build Application using script" + # ./location_of_script_within_repo/buildscript.sh + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v2 From 9864554ca2fa8405ed2818b2c71ff8841883b1d8 Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Mon, 19 Sep 2022 08:43:12 -0400 Subject: [PATCH 023/248] chore(deps): upgrade gtfs-lib --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index e57748b52..c61d6bf89 100644 --- a/pom.xml +++ b/pom.xml @@ -262,7 +262,7 @@ com.github.conveyal gtfs-lib - ca419a8149 + 3aac816 From 2f6485a968ef4cbd4d6cc10914d735e73e5324c1 Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Thu, 22 Sep 2022 10:43:22 -0400 Subject: [PATCH 024/248] refactor(docker): clean up and add instructions --- Dockerfile | 2 +- README.md | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 0b92958a8..46a4bb0db 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,7 +6,7 @@ COPY . /datatools # Build jar WORKDIR /datatools RUN mvn package -DskipTests -RUN cp target/dt*.jar ./datatools-server.jar +RUN cp target/dt*.jar /datatools/ RUN mkdir -p /var/datatools_gtfs/gtfsplus diff --git a/README.md b/README.md index 53620da17..9df4b1ba9 100644 --- a/README.md +++ b/README.md @@ -6,4 +6,7 @@ The core application for IBI Group's transit data tools suite. View the [latest documentation](http://conveyal-data-tools.readthedocs.org/en/latest/) at ReadTheDocs. -Note: `dev` branch docs can be found [here](http://conveyal-data-tools.readthedocs.org/en/dev/). \ No newline at end of file +Note: `dev` branch docs can be found [here](http://conveyal-data-tools.readthedocs.org/en/dev/). + +## Docker Image +The easiest way to get `datatools-server` running is to use the provided `Dockerfile` and `docker-compose.yml`. The `docker-compose.yml` includes both database servers that are needed. Edit the supplied configurations in the `configurations` directory to ensure the server starts correctly. Once this is done running `docker-compose up` will start Datatools and all required database servers. \ No newline at end of file From 573f358a7ce8e919153d657366edbbb3118ea28a Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Thu, 22 Sep 2022 12:33:26 -0400 Subject: [PATCH 025/248] refactor(Dockerfile): more resilient jar building --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 46a4bb0db..b086610d9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,6 +7,7 @@ COPY . /datatools WORKDIR /datatools RUN mvn package -DskipTests RUN cp target/dt*.jar /datatools/ +RUN mv dt*.jar datatools-server.jar RUN mkdir -p /var/datatools_gtfs/gtfsplus From 8581ff916efe09f2aeb6cce7bc9482b99aefdbbe Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Fri, 30 Sep 2022 08:39:50 +0100 Subject: [PATCH 026/248] refactor(Added deployment summary): Added new api end point to provide deployment summaries for a pr --- .../controllers/api/DeploymentController.java | 17 ++- .../manager/models/DeploymentSummary.java | 84 +++++++++++++ .../datatools/manager/models/Project.java | 11 ++ .../api/DeploymentControllerTest.java | 110 ++++++++++++++++++ 4 files changed, 221 insertions(+), 1 deletion(-) create mode 100644 src/main/java/com/conveyal/datatools/manager/models/DeploymentSummary.java create mode 100644 src/test/java/com/conveyal/datatools/manager/controllers/api/DeploymentControllerTest.java diff --git a/src/main/java/com/conveyal/datatools/manager/controllers/api/DeploymentController.java b/src/main/java/com/conveyal/datatools/manager/controllers/api/DeploymentController.java index f5a1162f1..d796710ff 100644 --- a/src/main/java/com/conveyal/datatools/manager/controllers/api/DeploymentController.java +++ b/src/main/java/com/conveyal/datatools/manager/controllers/api/DeploymentController.java @@ -1,6 +1,5 @@ package com.conveyal.datatools.manager.controllers.api; -import com.amazonaws.AmazonServiceException; import com.amazonaws.services.s3.AmazonS3URI; import com.amazonaws.services.s3.model.DeleteObjectRequest; import com.conveyal.datatools.common.status.MonitorableJob; @@ -14,6 +13,7 @@ import com.conveyal.datatools.manager.jobs.DeploymentGisExportJob; import com.conveyal.datatools.manager.jobs.PeliasUpdateJob; import com.conveyal.datatools.manager.models.Deployment; +import com.conveyal.datatools.manager.models.DeploymentSummary; import com.conveyal.datatools.manager.models.EC2InstanceSummary; import com.conveyal.datatools.manager.models.FeedDownloadToken; import com.conveyal.datatools.manager.models.FeedSource; @@ -233,6 +233,20 @@ private static Collection getAllDeployments (Request req, Response r } } + private static Collection getAllDeploymentSummaries(Request req, Response res) { + Auth0UserProfile userProfile = req.attribute("user"); + String projectId = req.queryParams("projectId"); + Project project = Persistence.projects.getById(projectId); + if (project == null) { + logMessageAndHalt(req, 400, "Must provide valid projectId value."); + return new ArrayList<>(); + } + if (!userProfile.canAdministerProject(project)) { + logMessageAndHalt(req, 401, "User not authorized to view project deployments."); + } + return project.retrieveDeploymentSummaries(); + } + /** * Create a new deployment for the project. All feed sources with a valid latest version are added to the new * deployment. @@ -594,6 +608,7 @@ public static void register (String apiPrefix) { get(apiPrefix + "secure/deployments/:id", DeploymentController::getDeployment, fullJson::write); delete(apiPrefix + "secure/deployments/:id", DeploymentController::deleteDeployment, fullJson::write); get(apiPrefix + "secure/deployments", DeploymentController::getAllDeployments, slimJson::write); + get(apiPrefix + "secure/deploymentSummaries", DeploymentController::getAllDeploymentSummaries, slimJson::write); post(apiPrefix + "secure/deployments", DeploymentController::createDeployment, fullJson::write); put(apiPrefix + "secure/deployments/:id", DeploymentController::updateDeployment, fullJson::write); post(apiPrefix + "secure/deployments/fromfeedsource/:id", DeploymentController::createDeploymentFromFeedSource, fullJson::write); diff --git a/src/main/java/com/conveyal/datatools/manager/models/DeploymentSummary.java b/src/main/java/com/conveyal/datatools/manager/models/DeploymentSummary.java new file mode 100644 index 000000000..05874e287 --- /dev/null +++ b/src/main/java/com/conveyal/datatools/manager/models/DeploymentSummary.java @@ -0,0 +1,84 @@ +package com.conveyal.datatools.manager.models; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; +import java.util.Date; + +/** + * A summary of deployment items used to list available deployments without the need to return all deployment information. + */ +@JsonInclude() +@JsonIgnoreProperties(ignoreUnknown = true) +public class DeploymentSummary { + + /** + * Deployment Id. + */ + public String id; + + /** + * Name given to this deployment. + */ + public String name; + + /** + * Date when the deployment was created. + */ + public Date created; + + /** + * Date when the deployment was last deployed to a server. + */ + public Date lastDeployed; + + /** + * What server is this currently deployed to, if any? + */ + public String deployedTo; + + /** + * Number of feed versions associated with this project. + */ + public int numberOfFeedVersions; + + /** + * If not using the 'default' router, this value will be set to true. + */ + public boolean test; + + /** + * Deployment is pinned. + */ + public boolean isPinned; + + /** + * The name of the server this is currently deployed to, if any. + */ + public String serverName; + + /** + * Create an empty deployment summary, for use with dump/restore and testing. + */ + public DeploymentSummary() { + } + + /** + * Extract from a deployment and parent project, deployment summary values. + */ + public DeploymentSummary(Deployment deployment, Project project) { + this.id = deployment.id; + this.name = deployment.name; + this.created = deployment.dateCreated; + this.lastDeployed = deployment.retrieveLastDeployed(); + this.deployedTo = deployment.deployedTo; + this.numberOfFeedVersions = deployment.feedVersionIds.size(); + this.test = deployment.routerId != null; + this.isPinned = project.pinnedDeploymentId != null && project.pinnedDeploymentId.equals(deployment.id); + OtpServer server = project.availableOtpServers() + .stream() + .filter(s -> s.id.equals(deployedTo)) + .findFirst() + .orElse(null); + this.serverName = (server != null) ? server.name : null; + } +} diff --git a/src/main/java/com/conveyal/datatools/manager/models/Project.java b/src/main/java/com/conveyal/datatools/manager/models/Project.java index 28b3d677e..d6546b382 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/Project.java +++ b/src/main/java/com/conveyal/datatools/manager/models/Project.java @@ -8,6 +8,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.ArrayList; import java.util.Collection; import java.util.Date; import java.util.HashSet; @@ -135,6 +136,16 @@ public Collection retrieveDeployments() { return Persistence.deployments.getFiltered(eq("projectId", this.id)); } + /** + * Get all deployment summaries for this project. + */ + public Collection retrieveDeploymentSummaries() { + List deploymentSummaries = new ArrayList<>(); + Collection deployments = retrieveDeployments(); + deployments.forEach(deployment -> deploymentSummaries.add(new DeploymentSummary(deployment, this))); + return deploymentSummaries; + } + // TODO: Does this need to be returned with JSON API response public Organization retrieveOrganization() { if (organizationId != null) { diff --git a/src/test/java/com/conveyal/datatools/manager/controllers/api/DeploymentControllerTest.java b/src/test/java/com/conveyal/datatools/manager/controllers/api/DeploymentControllerTest.java new file mode 100644 index 000000000..13734533b --- /dev/null +++ b/src/test/java/com/conveyal/datatools/manager/controllers/api/DeploymentControllerTest.java @@ -0,0 +1,110 @@ +package com.conveyal.datatools.manager.controllers.api; + +import com.conveyal.datatools.DatatoolsTest; +import com.conveyal.datatools.TestUtils; +import com.conveyal.datatools.manager.auth.Auth0Connection; +import com.conveyal.datatools.manager.jobs.DeployJob; +import com.conveyal.datatools.manager.models.Deployment; +import com.conveyal.datatools.manager.models.DeploymentSummary; +import com.conveyal.datatools.manager.models.OtpServer; +import com.conveyal.datatools.manager.models.Project; +import com.conveyal.datatools.manager.persistence.Persistence; +import com.conveyal.datatools.manager.utils.HttpUtils; +import com.conveyal.datatools.manager.utils.SimpleHttpResponse; +import com.conveyal.datatools.manager.utils.json.JsonUtil; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.util.Collections; +import java.util.Date; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class DeploymentControllerTest extends DatatoolsTest { + private static Project project; + private static OtpServer server; + private static Deployment deployment; + private static DeployJob.DeploySummary deploySummary; + + /** + * Add project, server, deploy summary and deployment to prepare for test. + */ + @BeforeAll + public static void setUp() throws IOException { + // start server if it isn't already running + DatatoolsTest.setUp(); + Auth0Connection.setAuthDisabled(true); + project = new Project(); + project.name = String.format("Test %s", new Date()); + Persistence.projects.create(project); + + server = new OtpServer(); + server.name = "Test Server"; + server.projectId = project.id; + server.s3Bucket = "datatools-dev"; + Persistence.servers.create(server); + + deploySummary = new DeployJob.DeploySummary(); + deploySummary.serverId = server.id; + + deployment = new Deployment(); + deployment.feedVersionIds = Collections.singletonList("feedVersionId"); + deployment.projectId = project.id; + deployment.name = "Test deployment"; + deployment.deployJobSummaries.add(deploySummary); + deployment.deployedTo = server.id; + deployment.routerId = "Test router id"; + Persistence.deployments.create(deployment); + + // Update project with pinned deployment. + project.pinnedDeploymentId = deployment.id; + Persistence.projects.replace(project.id, project); + } + + /** + * Remove objects from the database after test and reset auth. + */ + @AfterAll + public static void tearDown() { + Auth0Connection.setAuthDisabled(Auth0Connection.getDefaultAuthDisabled()); + if (project != null) { + Persistence.projects.removeById(project.id); + } + if (server != null) { + Persistence.servers.removeById(server.id); + } + if (deployment != null) { + Persistence.deployments.removeById(deployment.id); + } + } + + /** + * Retrieve a deployment summary for the previously created project, and it's related objects. + */ + @Test + void canRetrieveDeploymentSummaries() throws IOException { + SimpleHttpResponse response = TestUtils.makeRequest( + "/api/manager/secure/deploymentSummaries" + String.format("?projectId=%s", project.id), + null, + HttpUtils.REQUEST_METHOD.GET + ); + List deploymentSummaries = + JsonUtil.getPOJOFromJSONAsList( + JsonUtil.getJsonNodeFromResponse(response), + DeploymentSummary.class + ); + assertEquals(deployment.id, deploymentSummaries.get(0).id); + assertEquals(deployment.name, deploymentSummaries.get(0).name); + assertEquals(deployment.dateCreated, deploymentSummaries.get(0).created); + assertEquals(new Date(deploySummary.finishTime), deploymentSummaries.get(0).lastDeployed); + assertEquals(server.id, deploymentSummaries.get(0).deployedTo); + assertEquals(deployment.feedVersionIds.size(), deploymentSummaries.get(0).numberOfFeedVersions); + assertTrue(deploymentSummaries.get(0).test); + assertTrue(deploymentSummaries.get(0).isPinned); + assertEquals(server.name, deploymentSummaries.get(0).serverName); + } +} From 1d915779b99959cb329ce7da5336df0c78fe0838 Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Wed, 5 Oct 2022 08:29:50 +0100 Subject: [PATCH 027/248] refactor(Addressed PR feedback): Addressed PR feedback --- .../manager/controllers/api/DeploymentController.java | 1 - .../datatools/manager/models/DeploymentSummary.java | 9 +++++---- .../com/conveyal/datatools/manager/models/Project.java | 5 +++-- .../controllers/api/DeploymentControllerTest.java | 4 ++-- 4 files changed, 10 insertions(+), 9 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/controllers/api/DeploymentController.java b/src/main/java/com/conveyal/datatools/manager/controllers/api/DeploymentController.java index d796710ff..9d43a6616 100644 --- a/src/main/java/com/conveyal/datatools/manager/controllers/api/DeploymentController.java +++ b/src/main/java/com/conveyal/datatools/manager/controllers/api/DeploymentController.java @@ -239,7 +239,6 @@ private static Collection getAllDeploymentSummaries(Request r Project project = Persistence.projects.getById(projectId); if (project == null) { logMessageAndHalt(req, 400, "Must provide valid projectId value."); - return new ArrayList<>(); } if (!userProfile.canAdministerProject(project)) { logMessageAndHalt(req, 401, "User not authorized to view project deployments."); diff --git a/src/main/java/com/conveyal/datatools/manager/models/DeploymentSummary.java b/src/main/java/com/conveyal/datatools/manager/models/DeploymentSummary.java index 05874e287..d3959ac27 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/DeploymentSummary.java +++ b/src/main/java/com/conveyal/datatools/manager/models/DeploymentSummary.java @@ -3,6 +3,7 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonInclude; import java.util.Date; +import java.util.List; /** * A summary of deployment items used to list available deployments without the need to return all deployment information. @@ -24,7 +25,7 @@ public class DeploymentSummary { /** * Date when the deployment was created. */ - public Date created; + public Date dateCreated; /** * Date when the deployment was last deployed to a server. @@ -65,16 +66,16 @@ public DeploymentSummary() { /** * Extract from a deployment and parent project, deployment summary values. */ - public DeploymentSummary(Deployment deployment, Project project) { + public DeploymentSummary(Deployment deployment, Project project, List otpServers) { this.id = deployment.id; this.name = deployment.name; - this.created = deployment.dateCreated; + this.dateCreated = deployment.dateCreated; this.lastDeployed = deployment.retrieveLastDeployed(); this.deployedTo = deployment.deployedTo; this.numberOfFeedVersions = deployment.feedVersionIds.size(); this.test = deployment.routerId != null; this.isPinned = project.pinnedDeploymentId != null && project.pinnedDeploymentId.equals(deployment.id); - OtpServer server = project.availableOtpServers() + OtpServer server = otpServers .stream() .filter(s -> s.id.equals(deployedTo)) .findFirst() diff --git a/src/main/java/com/conveyal/datatools/manager/models/Project.java b/src/main/java/com/conveyal/datatools/manager/models/Project.java index d6546b382..4c27f8e40 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/Project.java +++ b/src/main/java/com/conveyal/datatools/manager/models/Project.java @@ -140,9 +140,10 @@ public Collection retrieveDeployments() { * Get all deployment summaries for this project. */ public Collection retrieveDeploymentSummaries() { - List deploymentSummaries = new ArrayList<>(); Collection deployments = retrieveDeployments(); - deployments.forEach(deployment -> deploymentSummaries.add(new DeploymentSummary(deployment, this))); + List otpServers = availableOtpServers(); + List deploymentSummaries = new ArrayList<>(); + deployments.forEach(deployment -> deploymentSummaries.add(new DeploymentSummary(deployment, this, otpServers))); return deploymentSummaries; } diff --git a/src/test/java/com/conveyal/datatools/manager/controllers/api/DeploymentControllerTest.java b/src/test/java/com/conveyal/datatools/manager/controllers/api/DeploymentControllerTest.java index 13734533b..671afd127 100644 --- a/src/test/java/com/conveyal/datatools/manager/controllers/api/DeploymentControllerTest.java +++ b/src/test/java/com/conveyal/datatools/manager/controllers/api/DeploymentControllerTest.java @@ -88,7 +88,7 @@ public static void tearDown() { @Test void canRetrieveDeploymentSummaries() throws IOException { SimpleHttpResponse response = TestUtils.makeRequest( - "/api/manager/secure/deploymentSummaries" + String.format("?projectId=%s", project.id), + String.format("/api/manager/secure/deploymentSummaries?projectId=%s", project.id), null, HttpUtils.REQUEST_METHOD.GET ); @@ -99,7 +99,7 @@ void canRetrieveDeploymentSummaries() throws IOException { ); assertEquals(deployment.id, deploymentSummaries.get(0).id); assertEquals(deployment.name, deploymentSummaries.get(0).name); - assertEquals(deployment.dateCreated, deploymentSummaries.get(0).created); + assertEquals(deployment.dateCreated, deploymentSummaries.get(0).dateCreated); assertEquals(new Date(deploySummary.finishTime), deploymentSummaries.get(0).lastDeployed); assertEquals(server.id, deploymentSummaries.get(0).deployedTo); assertEquals(deployment.feedVersionIds.size(), deploymentSummaries.get(0).numberOfFeedVersions); From 0c625788846afb9c9073f62317d859cf685b5a1d Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Wed, 5 Oct 2022 13:57:38 +0100 Subject: [PATCH 028/248] refactor(Project.java): Addressed PR feedback --- .../com/conveyal/datatools/manager/models/Project.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/models/Project.java b/src/main/java/com/conveyal/datatools/manager/models/Project.java index 4c27f8e40..2087a95e7 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/Project.java +++ b/src/main/java/com/conveyal/datatools/manager/models/Project.java @@ -14,6 +14,7 @@ import java.util.HashSet; import java.util.List; import java.util.Set; +import java.util.stream.Collectors; import static com.mongodb.client.model.Filters.and; import static com.mongodb.client.model.Filters.eq; @@ -142,9 +143,10 @@ public Collection retrieveDeployments() { public Collection retrieveDeploymentSummaries() { Collection deployments = retrieveDeployments(); List otpServers = availableOtpServers(); - List deploymentSummaries = new ArrayList<>(); - deployments.forEach(deployment -> deploymentSummaries.add(new DeploymentSummary(deployment, this, otpServers))); - return deploymentSummaries; + return deployments + .stream() + .map(deployment -> new DeploymentSummary(deployment, this, otpServers)) + .collect(Collectors.toList()); } // TODO: Does this need to be returned with JSON API response From 0528a2c273a4f95853290c7758bb74c1e38a1cdb Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Wed, 5 Oct 2022 12:24:04 -0400 Subject: [PATCH 029/248] refactor(Dockerfile): use all available container ram --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index b086610d9..f7607df81 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,5 +14,5 @@ RUN mkdir -p /var/datatools_gtfs/gtfsplus # Launch server # This relies on a configuration volume and aws volume being present. See `docker-compose.yml`, or the example below # Try: docker run --publish 4000:4000 -v ~/config/:/config datatools-latest -CMD ["java", "-jar", "datatools-server.jar", "/config/env.yml", "/config/server.yml"] +CMD ["java", "-XX:MaxRAMPercentage=95", "-jar", "datatools-server.jar", "/config/env.yml", "/config/server.yml"] EXPOSE 4000 \ No newline at end of file From cd2542694e22f905a396bcce143773c8cc87e1d1 Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Fri, 7 Oct 2022 10:27:43 +0100 Subject: [PATCH 030/248] refactor(Handle specific entity update): Provide the trip id on conflict update --- .../controllers/api/EditorController.java | 26 ++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/conveyal/datatools/editor/controllers/api/EditorController.java b/src/main/java/com/conveyal/datatools/editor/controllers/api/EditorController.java index 371499300..7b6b94270 100644 --- a/src/main/java/com/conveyal/datatools/editor/controllers/api/EditorController.java +++ b/src/main/java/com/conveyal/datatools/editor/controllers/api/EditorController.java @@ -393,7 +393,7 @@ private String createOrUpdate(Request req, Response res) { if (isCreating) { return tableWriter.create(jsonBody, true); } else { - return tableWriter.update(id, jsonBody, true); + return update(tableWriter, id, jsonBody); } } catch (InvalidNamespaceException e) { logMessageAndHalt(req, 400, "Invalid namespace"); @@ -408,6 +408,30 @@ private String createOrUpdate(Request req, Response res) { return null; } + /** + * Handle specific entity updates. + */ + private String update(JdbcTableWriter tableWriter, Integer id, String jsonBody) throws IOException, SQLException { + try { + return tableWriter.update(id, jsonBody, true); + } catch (SQLException e) { + if (table.name.equals(Table.TRIPS.name)) { + // If an exception is thrown updating a trip, provide additional information to help rectify the + // issue. + JsonNode trip = mapper.readTree(jsonBody); + throw new SQLException( + String.format( + "Trip id %s conflicts with an existing trip id.", + trip.get("trip_id").asText() + ) + ); + } else { + // Continue to pass the exception to the frontend. + throw e; + } + } + } + /** * Get the namespace for the feed ID found in the request. Also, check that the user has an active editing session * for the provided feed ID. From 8bca1fa86559acf1f48a9df8d6989d0b17a68b86 Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Mon, 31 Oct 2022 14:25:50 +0000 Subject: [PATCH 031/248] refactor(FeedSource.java): Exposed validation summary end date --- .../conveyal/datatools/manager/models/FeedSource.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java b/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java index 3dcd7f4e8..bc84f939f 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java @@ -38,6 +38,7 @@ import java.net.HttpURLConnection; import java.net.MalformedURLException; import java.net.URL; +import java.time.LocalDate; import java.util.ArrayList; import java.util.Collection; import java.util.Date; @@ -461,6 +462,14 @@ public String latestVersionId() { return latest != null ? latest.id : null; } + @JsonInclude(JsonInclude.Include.NON_NULL) + @JsonView(JsonViews.UserInterface.class) + @JsonProperty("latestVersionEndDate") + public LocalDate latestVersionEndDate() { + FeedVersion latest = retrieveLatest(); + return latest != null ? latest.validationSummary().endDate : null; + } + /** * Number of {@link FeedVersion}s that exist for the feed source. */ From 899bf15ce966bc10b32f2c3db6b388316b32c6c9 Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Mon, 31 Oct 2022 14:47:50 +0000 Subject: [PATCH 032/248] refactor(FeedSource.java): Exposed published version id --- .../com/conveyal/datatools/manager/models/FeedSource.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java b/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java index bc84f939f..700b62003 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java @@ -462,6 +462,13 @@ public String latestVersionId() { return latest != null ? latest.id : null; } + @JsonInclude(JsonInclude.Include.NON_NULL) + @JsonView(JsonViews.UserInterface.class) + @JsonProperty("latestPublishedVersionId") + public String latestPublishedVersionId() { + return this.publishedVersionId != null ? this.publishedVersionId : null; + } + @JsonInclude(JsonInclude.Include.NON_NULL) @JsonView(JsonViews.UserInterface.class) @JsonProperty("latestVersionEndDate") From b9c533594ed2ba5e9ef8a4678c3e5d1291d0c151 Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Mon, 31 Oct 2022 14:55:57 +0000 Subject: [PATCH 033/248] refactor(FeedSource.java): Exposed latest published end date --- .../datatools/manager/models/FeedSource.java | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java b/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java index 700b62003..b6737671a 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java @@ -462,6 +462,14 @@ public String latestVersionId() { return latest != null ? latest.id : null; } + @JsonInclude(JsonInclude.Include.NON_NULL) + @JsonView(JsonViews.UserInterface.class) + @JsonProperty("latestVersionEndDate") + public LocalDate latestVersionEndDate() { + FeedVersion latest = retrieveLatest(); + return latest != null ? latest.validationSummary().endDate : null; + } + @JsonInclude(JsonInclude.Include.NON_NULL) @JsonView(JsonViews.UserInterface.class) @JsonProperty("latestPublishedVersionId") @@ -471,10 +479,10 @@ public String latestPublishedVersionId() { @JsonInclude(JsonInclude.Include.NON_NULL) @JsonView(JsonViews.UserInterface.class) - @JsonProperty("latestVersionEndDate") - public LocalDate latestVersionEndDate() { - FeedVersion latest = retrieveLatest(); - return latest != null ? latest.validationSummary().endDate : null; + @JsonProperty("latestPublishedEndDate") + public LocalDate latestPublishedEndDate() { + FeedVersion publishedVersion = retrievePublishedVersion(); + return publishedVersion != null ? publishedVersion.validationSummary().endDate : null; } /** From 2d4c104ae5c4687492d6905bd1db8c9f72fc8b58 Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Wed, 2 Nov 2022 12:05:09 +0000 Subject: [PATCH 034/248] refactor(Update to provide deployed feed version): Update to feed source to provide deployed feed ve --- .../datatools/manager/models/FeedSource.java | 58 +++++++--- .../manager/persistence/TypedPersistence.java | 13 ++- .../api/FeedSourceControllerTest.java | 101 +++++++++++++++++- 3 files changed, 156 insertions(+), 16 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java b/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java index b6737671a..3132dc83d 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java @@ -462,27 +462,59 @@ public String latestVersionId() { return latest != null ? latest.id : null; } + /** + * The feed version used in the latest deployment. + * This cannot be returned because of a circular reference between feed source and feed version. Instead, individual + * parameters (version id and end date) are returned. + */ + @JsonIgnore + @BsonIgnore + private FeedVersion deployedFeedVersion = null; + @JsonInclude(JsonInclude.Include.NON_NULL) @JsonView(JsonViews.UserInterface.class) - @JsonProperty("latestVersionEndDate") - public LocalDate latestVersionEndDate() { - FeedVersion latest = retrieveLatest(); - return latest != null ? latest.validationSummary().endDate : null; + @JsonProperty("deployedFeedVersionId") + public String getDeployedFeedVersionId() { + FeedVersion feedVersion = getDeployedFeedVersion(); + return feedVersion != null ? feedVersion.id : null; } @JsonInclude(JsonInclude.Include.NON_NULL) @JsonView(JsonViews.UserInterface.class) - @JsonProperty("latestPublishedVersionId") - public String latestPublishedVersionId() { - return this.publishedVersionId != null ? this.publishedVersionId : null; + @JsonProperty("deployedFeedVersionEndDate") + public LocalDate getDeployedFeedVersionEndDate() { + FeedVersion feedVersion = getDeployedFeedVersion(); + return feedVersion != null ? feedVersion.validationSummary().endDate : null; } - @JsonInclude(JsonInclude.Include.NON_NULL) - @JsonView(JsonViews.UserInterface.class) - @JsonProperty("latestPublishedEndDate") - public LocalDate latestPublishedEndDate() { - FeedVersion publishedVersion = retrievePublishedVersion(); - return publishedVersion != null ? publishedVersion.validationSummary().endDate : null; + /** + * Find the latest deployment containing a feed version for a feed source. + */ + private FeedVersion getDeployedFeedVersion() { + if (deployedFeedVersion != null) { + // If the deployed feed version is already defined use this instead of recreating. + return deployedFeedVersion; + } + Collection deployments = Persistence.deployments.getFiltered( + eq("projectId", this.projectId), + Sorts.descending("dateCreated") + ); + Collection feedVersions = Persistence.feedVersions.getFiltered(eq("feedSourceId", this.id)); + if (deployments.isEmpty() || feedVersions.isEmpty()) { + return null; + } + for (Deployment deployment : deployments) { + // Iterate through deployments newest to oldest. + deployedFeedVersion = feedVersions.stream() + .filter(feedVersion -> deployment.feedVersionIds.contains(feedVersion.id)) + .findAny() + .orElse(null); + if (deployedFeedVersion != null) { + // Found deployment containing feed version. + break; + } + } + return deployedFeedVersion; } /** diff --git a/src/main/java/com/conveyal/datatools/manager/persistence/TypedPersistence.java b/src/main/java/com/conveyal/datatools/manager/persistence/TypedPersistence.java index 28288394f..8174647e3 100644 --- a/src/main/java/com/conveyal/datatools/manager/persistence/TypedPersistence.java +++ b/src/main/java/com/conveyal/datatools/manager/persistence/TypedPersistence.java @@ -169,7 +169,18 @@ public List getByIds (List ids) { * We should really have a bit more abstraction here. */ public List getFiltered (Bson filter) { - return mongoCollection.find(filter).into(new ArrayList<>()); + return getFiltered(filter, null); + } + + /** + * Get all objects satisfying the supplied Mongo filter and sort by. + */ + public List getFiltered (Bson filter, Bson sortBy) { + if (sortBy != null) { + return mongoCollection.find(filter).sort(sortBy).into(new ArrayList<>()); + } else { + return mongoCollection.find(filter).into(new ArrayList<>()); + } } /** diff --git a/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java b/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java index bb12a29da..2eaeccec1 100644 --- a/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java +++ b/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java @@ -4,8 +4,10 @@ import com.conveyal.datatools.TestUtils; import com.conveyal.datatools.common.utils.Scheduler; import com.conveyal.datatools.manager.auth.Auth0Connection; +import com.conveyal.datatools.manager.models.Deployment; import com.conveyal.datatools.manager.models.FeedRetrievalMethod; import com.conveyal.datatools.manager.models.FeedSource; +import com.conveyal.datatools.manager.models.FeedVersion; import com.conveyal.datatools.manager.models.FetchFrequency; import com.conveyal.datatools.manager.models.Label; import com.conveyal.datatools.manager.models.Project; @@ -13,12 +15,18 @@ import com.conveyal.datatools.manager.utils.HttpUtils; import com.conveyal.datatools.manager.utils.SimpleHttpResponse; import com.conveyal.datatools.manager.utils.json.JsonUtil; +import com.conveyal.gtfs.validator.ValidationResult; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import java.io.IOException; import java.net.URL; +import java.time.LocalDate; +import java.time.Month; +import java.time.ZoneId; +import java.util.Collections; +import java.util.Date; import java.util.List; import static com.mongodb.client.model.Filters.eq; @@ -27,7 +35,6 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNull; - public class FeedSourceControllerTest extends DatatoolsTest { private static Project project = null; private static Project projectToBeDeleted = null; @@ -35,8 +42,14 @@ public class FeedSourceControllerTest extends DatatoolsTest { private static FeedSource feedSourceWithNoUrl = null; private static FeedSource feedSourceWithLabels = null; private static FeedSource feedSourceWithInvalidLabels = null; + private static FeedSource feedSourceWithDeployedFeedVersion = null; private static Label publicLabel = null; private static Label adminOnlyLabel = null; + private static FeedVersion feedVersionSuperseded = null; + private static FeedVersion feedVersionDeployed = null; + private static FeedVersion feedVersionLatest = null; + private static Deployment deploymentSuperseded = null; + private static Deployment deploymentDeployed = null; @BeforeAll public static void setUp() throws IOException { @@ -60,6 +73,18 @@ public static void setUp() throws IOException { adminOnlyLabel = createLabel("Admin Only Label"); adminOnlyLabel.adminOnly = true; publicLabel = createLabel("Public Label"); + + feedSourceWithDeployedFeedVersion = createFeedSource("FeedSource", null, project, true); + + LocalDate supersededDate = LocalDate.of(2020, Month.DECEMBER, 25); + LocalDate deployedDate = LocalDate.of(2021, Month.MARCH, 12); + feedVersionSuperseded = createFeedVersion("superseded", feedSourceWithDeployedFeedVersion.id, supersededDate); + feedVersionDeployed = createFeedVersion("deployed", feedSourceWithDeployedFeedVersion.id, deployedDate); + feedVersionLatest = createFeedVersion("latest", feedSourceWithDeployedFeedVersion.id, LocalDate.of(2022, Month.NOVEMBER, 2)); + + deploymentSuperseded = createDeployment("superseded", project, feedVersionSuperseded.id, supersededDate); + deploymentDeployed = createDeployment("deployed", project, feedVersionDeployed.id, deployedDate); + } @AfterAll @@ -83,6 +108,24 @@ public static void tearDown() { if (adminOnlyLabel != null) { Persistence.labels.removeById(adminOnlyLabel.id); } + if (feedSourceWithDeployedFeedVersion != null) { + Persistence.feedSources.removeById(feedSourceWithDeployedFeedVersion.id); + } + if (feedVersionSuperseded != null) { + Persistence.feedVersions.removeById(feedVersionSuperseded.id); + } + if (feedVersionDeployed != null) { + Persistence.feedVersions.removeById(feedVersionDeployed.id); + } + if (feedVersionLatest != null) { + Persistence.feedVersions.removeById(feedVersionLatest.id); + } + if (deploymentSuperseded != null) { + Persistence.deployments.removeById(deploymentSuperseded.id); + } + if (deploymentDeployed != null) { + Persistence.deployments.removeById(deploymentDeployed.id); + } } /** @@ -206,9 +249,35 @@ public void createFeedSourceWithLabels() throws IOException { } /** - * Helper method to create feed source. + * Retrieve a feed source containing the deployed feed version. */ + @Test + void canRetrieveFeedSourceWithDeployedFeedVersion() throws IOException { + SimpleHttpResponse response = TestUtils.makeRequest( + String.format("/api/manager/secure/feedsource?projectId=%s", project.id), + null, + HttpUtils.REQUEST_METHOD.GET + ); + assertEquals(OK_200, response.status); + List feedSources = + JsonUtil.getPOJOFromJSONAsList( + JsonUtil.getJsonNodeFromResponse(response), + FeedSource.class + ); + assertEquals(1, feedSources.size()); + assertEquals(feedVersionDeployed.id, feedSources.get(0).getDeployedFeedVersionId()); + assertEquals(feedVersionDeployed.validationSummary().endDate, feedSources.get(0).getDeployedFeedVersionEndDate()); + } + + private static FeedSource createFeedSource(String name, URL url, Project project) { + return createFeedSource(name, url, project, false); + } + + /** + * Helper method to create feed source. + */ + private static FeedSource createFeedSource(String name, URL url, Project project, boolean persist) { FeedSource feedSource = new FeedSource(); feedSource.fetchFrequency = FetchFrequency.MINUTES; feedSource.fetchInterval = 1; @@ -217,9 +286,37 @@ private static FeedSource createFeedSource(String name, URL url, Project project feedSource.projectId = project.id; feedSource.retrievalMethod = FeedRetrievalMethod.FETCHED_AUTOMATICALLY; feedSource.url = url; + if (persist) Persistence.feedSources.create(feedSource); return feedSource; } + /** + * Helper method to create a deployment. + */ + private static Deployment createDeployment(String name, Project project, String feedVersionId, LocalDate dateCreated) { + Deployment deployment = new Deployment(); + deployment.dateCreated = Date.from(dateCreated.atStartOfDay(ZoneId.systemDefault()).toInstant()); + deployment.feedVersionIds = Collections.singletonList(feedVersionId); + deployment.projectId = project.id; + deployment.name = name; + Persistence.deployments.create(deployment); + return deployment; + } + + /** + * Helper method to create a feed version. + */ + private static FeedVersion createFeedVersion(String name, String feedSourceId, LocalDate endDate) { + FeedVersion feedVersion = new FeedVersion(); + feedVersion.name = name; + feedVersion.feedSourceId = feedSourceId; + ValidationResult validationResult = new ValidationResult(); + validationResult.lastCalendarDate = endDate; + feedVersion.validationResult = validationResult; + Persistence.feedVersions.create(feedVersion); + return feedVersion; + } + /** * Helper method to create label */ From 1b99b6edfbc6604cf61ff5f958aad48666311c4f Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Thu, 3 Nov 2022 15:58:06 +0000 Subject: [PATCH 035/248] refactor(Added deployed feed version start date): Updated feed source to provide deployed feed versi --- .../datatools/manager/models/FeedSource.java | 48 +++++++++++-------- .../api/FeedSourceControllerTest.java | 13 +++-- 2 files changed, 37 insertions(+), 24 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java b/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java index 3132dc83d..5b88abd88 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java @@ -479,6 +479,14 @@ public String getDeployedFeedVersionId() { return feedVersion != null ? feedVersion.id : null; } + @JsonInclude(JsonInclude.Include.NON_NULL) + @JsonView(JsonViews.UserInterface.class) + @JsonProperty("deployedFeedVersionStartDate") + public LocalDate getDeployedFeedVersionStartDate() { + FeedVersion feedVersion = getDeployedFeedVersion(); + return feedVersion != null ? feedVersion.validationSummary().startDate : null; + } + @JsonInclude(JsonInclude.Include.NON_NULL) @JsonView(JsonViews.UserInterface.class) @JsonProperty("deployedFeedVersionEndDate") @@ -491,27 +499,25 @@ public LocalDate getDeployedFeedVersionEndDate() { * Find the latest deployment containing a feed version for a feed source. */ private FeedVersion getDeployedFeedVersion() { - if (deployedFeedVersion != null) { - // If the deployed feed version is already defined use this instead of recreating. - return deployedFeedVersion; - } - Collection deployments = Persistence.deployments.getFiltered( - eq("projectId", this.projectId), - Sorts.descending("dateCreated") - ); - Collection feedVersions = Persistence.feedVersions.getFiltered(eq("feedSourceId", this.id)); - if (deployments.isEmpty() || feedVersions.isEmpty()) { - return null; - } - for (Deployment deployment : deployments) { - // Iterate through deployments newest to oldest. - deployedFeedVersion = feedVersions.stream() - .filter(feedVersion -> deployment.feedVersionIds.contains(feedVersion.id)) - .findAny() - .orElse(null); - if (deployedFeedVersion != null) { - // Found deployment containing feed version. - break; + if (deployedFeedVersion == null) { + Collection deployments = Persistence.deployments.getFiltered( + eq("projectId", this.projectId), + Sorts.descending("dateCreated") + ); + Collection feedVersions = Persistence.feedVersions.getFiltered(eq("feedSourceId", this.id)); + if (deployments.isEmpty() || feedVersions.isEmpty()) { + return null; + } + for (Deployment deployment : deployments) { + // Iterate through deployments newest to oldest. + deployedFeedVersion = feedVersions.stream() + .filter(feedVersion -> deployment.feedVersionIds.contains(feedVersion.id)) + .findAny() + .orElse(null); + if (deployedFeedVersion != null) { + // Found deployment containing feed version. + break; + } } } return deployedFeedVersion; diff --git a/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java b/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java index 2eaeccec1..564d76777 100644 --- a/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java +++ b/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java @@ -77,13 +77,14 @@ public static void setUp() throws IOException { feedSourceWithDeployedFeedVersion = createFeedSource("FeedSource", null, project, true); LocalDate supersededDate = LocalDate.of(2020, Month.DECEMBER, 25); - LocalDate deployedDate = LocalDate.of(2021, Month.MARCH, 12); + LocalDate deployedEndDate = LocalDate.of(2021, Month.MARCH, 12); + LocalDate deployedStartDate = LocalDate.of(2021, Month.MARCH, 1); feedVersionSuperseded = createFeedVersion("superseded", feedSourceWithDeployedFeedVersion.id, supersededDate); - feedVersionDeployed = createFeedVersion("deployed", feedSourceWithDeployedFeedVersion.id, deployedDate); + feedVersionDeployed = createFeedVersion("deployed", feedSourceWithDeployedFeedVersion.id, deployedStartDate, deployedEndDate); feedVersionLatest = createFeedVersion("latest", feedSourceWithDeployedFeedVersion.id, LocalDate.of(2022, Month.NOVEMBER, 2)); deploymentSuperseded = createDeployment("superseded", project, feedVersionSuperseded.id, supersededDate); - deploymentDeployed = createDeployment("deployed", project, feedVersionDeployed.id, deployedDate); + deploymentDeployed = createDeployment("deployed", project, feedVersionDeployed.id, deployedEndDate); } @@ -267,6 +268,7 @@ void canRetrieveFeedSourceWithDeployedFeedVersion() throws IOException { assertEquals(1, feedSources.size()); assertEquals(feedVersionDeployed.id, feedSources.get(0).getDeployedFeedVersionId()); assertEquals(feedVersionDeployed.validationSummary().endDate, feedSources.get(0).getDeployedFeedVersionEndDate()); + assertEquals(feedVersionDeployed.validationSummary().startDate, feedSources.get(0).getDeployedFeedVersionStartDate()); } @@ -307,10 +309,15 @@ private static Deployment createDeployment(String name, Project project, String * Helper method to create a feed version. */ private static FeedVersion createFeedVersion(String name, String feedSourceId, LocalDate endDate) { + return createFeedVersion(name, feedSourceId, null, endDate); + } + + private static FeedVersion createFeedVersion(String name, String feedSourceId, LocalDate startDate, LocalDate endDate) { FeedVersion feedVersion = new FeedVersion(); feedVersion.name = name; feedVersion.feedSourceId = feedSourceId; ValidationResult validationResult = new ValidationResult(); + validationResult.firstCalendarDate = startDate; validationResult.lastCalendarDate = endDate; feedVersion.validationResult = validationResult; Persistence.feedVersions.create(feedVersion); From 94487abbf468965a0863aaa1a130120b697e8dc4 Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Mon, 21 Nov 2022 12:32:18 -0500 Subject: [PATCH 036/248] chore(deps): add jaxb --- pom.xml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pom.xml b/pom.xml index c61d6bf89..3aaa9f11b 100644 --- a/pom.xml +++ b/pom.xml @@ -287,6 +287,12 @@ guava 30.0-jre + + javax.xml.bind + jaxb-api + 2.3.1 + + - 3aac816 + 4fcf36e From 21e37020521b16418aeac2e8b239042569468a47 Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Fri, 27 Jan 2023 11:12:39 +0000 Subject: [PATCH 074/248] refactor(Streamlined DB calls): Reduced feed versions returned and deployment fields returned --- .../datatools/manager/models/FeedSource.java | 35 ++++++++++--------- .../manager/persistence/TypedPersistence.java | 13 +++++++ 2 files changed, 31 insertions(+), 17 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java b/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java index 195874a0e..ad8a975ba 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java @@ -505,15 +505,15 @@ private FeedVersion getDeployedFeedVersion() { Project project = Persistence.projects.getById(this.projectId); if (project.pinnedDeploymentId != null) { Deployment deployment = Persistence.deployments.getById(project.pinnedDeploymentId); - Collection feedVersions = getDeploymentFeedVersionsForFeedSource(deployment.pinnedfeedVersionIds); - if (!feedVersions.isEmpty()) { - // The first feed version will be the latest pinned version for this feed source, if available. - deployedFeedVersion = feedVersions.iterator().next(); + FeedVersion feedVersion = getLatestDeployedFeedVersionForFeedSource(deployment.pinnedfeedVersionIds); + if (feedVersion != null) { + // This feed version will be the latest pinned version for this feed source, if available. + deployedFeedVersion = feedVersion; } else { - feedVersions = getDeploymentFeedVersionsForFeedSource(deployment.feedVersionIds); - if (!feedVersions.isEmpty()) { - // The first feed version will be the latest version for this feed source, if available. - deployedFeedVersion = feedVersions.iterator().next(); + feedVersion = getLatestDeployedFeedVersionForFeedSource(deployment.feedVersionIds); + if (feedVersion != null) { + // This feed version will be the latest version for this feed source, if available. + deployedFeedVersion = feedVersion; } } } @@ -522,17 +522,18 @@ private FeedVersion getDeployedFeedVersion() { // find the latest feed version for this feed source. if (deployedFeedVersion == null) { // Get all deployments for this project. - Collection deployments = Persistence.deployments.getFiltered( + List deployments = Persistence.deployments.getFilteredLimitedFields( eq("projectId", this.projectId), - Sorts.descending("lastUpdated") + Sorts.descending("lastUpdated"), + "feedVersionIds" ); // Iterate through deployments newest to oldest. for (Deployment deployment : deployments) { - Collection feedVersions = getDeploymentFeedVersionsForFeedSource(deployment.feedVersionIds); - if (!feedVersions.isEmpty()) { - // The first feed version will be the latest feed version for this feed source from the newest + FeedVersion feedVersion = getLatestDeployedFeedVersionForFeedSource(deployment.feedVersionIds); + if (feedVersion != null) { + // This feed version will be the latest feed version for this feed source from the newest // deployment. - deployedFeedVersion = feedVersions.iterator().next(); + deployedFeedVersion = feedVersion; break; } } @@ -542,10 +543,10 @@ private FeedVersion getDeployedFeedVersion() { } /** - * Get a deployment's feed versions for this feed source (if any) and order by last updated. + * Get the latest deployed feed version for this feed source, if available. */ - private Collection getDeploymentFeedVersionsForFeedSource(Collection feedVersionIds) { - return Persistence.feedVersions.getFiltered( + private FeedVersion getLatestDeployedFeedVersionForFeedSource(Collection feedVersionIds) { + return Persistence.feedVersions.getOneFiltered( and( eq("feedSourceId", this.id), in("_id", feedVersionIds) diff --git a/src/main/java/com/conveyal/datatools/manager/persistence/TypedPersistence.java b/src/main/java/com/conveyal/datatools/manager/persistence/TypedPersistence.java index 50e7ebc43..d63fd4d55 100644 --- a/src/main/java/com/conveyal/datatools/manager/persistence/TypedPersistence.java +++ b/src/main/java/com/conveyal/datatools/manager/persistence/TypedPersistence.java @@ -20,6 +20,8 @@ import static com.mongodb.client.model.Filters.eq; import static com.mongodb.client.model.Filters.in; +import static com.mongodb.client.model.Projections.fields; +import static com.mongodb.client.model.Projections.include; import static com.mongodb.client.model.Updates.pull; /** @@ -181,6 +183,17 @@ public List getFiltered (Bson filter, Bson sortBy) { : mongoCollection.find(filter).into(new ArrayList<>()); } + /** + * Get all objects populating only the included fields satisfying the supplied Mongo filter and sort by. + */ + public List getFilteredLimitedFields(Bson filter, Bson sortBy, String... includedFields) { + return mongoCollection + .find(filter) + .projection(fields(include(includedFields))) + .sort(sortBy) + .into(new ArrayList<>()); + } + /** * Expose the internal MongoCollection to the caller. * This ties our persistence directly to Mongo for now but is expedient. From 487df013c541e4b4813b31372a81d19321d3451d Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Fri, 27 Jan 2023 11:51:32 +0000 Subject: [PATCH 075/248] refactor(Update to limite fields returned searching by id): --- .../conveyal/datatools/manager/models/FeedSource.java | 5 ++++- .../manager/persistence/TypedPersistence.java | 10 ++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java b/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java index ad8a975ba..9d0e24ce0 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java @@ -504,7 +504,10 @@ private FeedVersion getDeployedFeedVersion() { if (deployedFeedVersion == null) { Project project = Persistence.projects.getById(this.projectId); if (project.pinnedDeploymentId != null) { - Deployment deployment = Persistence.deployments.getById(project.pinnedDeploymentId); + Deployment deployment = Persistence.deployments.getByIdLimitedFields( + project.pinnedDeploymentId, + "pinnedfeedVersionIds", "feedVersionIds" + ); FeedVersion feedVersion = getLatestDeployedFeedVersionForFeedSource(deployment.pinnedfeedVersionIds); if (feedVersion != null) { // This feed version will be the latest pinned version for this feed source, if available. diff --git a/src/main/java/com/conveyal/datatools/manager/persistence/TypedPersistence.java b/src/main/java/com/conveyal/datatools/manager/persistence/TypedPersistence.java index d63fd4d55..5fe847228 100644 --- a/src/main/java/com/conveyal/datatools/manager/persistence/TypedPersistence.java +++ b/src/main/java/com/conveyal/datatools/manager/persistence/TypedPersistence.java @@ -148,6 +148,16 @@ public T getById (String id) { return mongoCollection.find(eq(id)).first(); } + /** + * Get first matching object populating only the included fields. + */ + public T getByIdLimitedFields (String id, String... includedFields) { + return mongoCollection + .find(eq(id)) + .projection(fields(include(includedFields))) + .first(); + } + /** * This is not memory efficient. * TODO: Always use iterators / streams, always perform selection of subsets on the Mongo server side ("where clause"). From 4cad7cbbabd0f872a0460004fbb016511b3e469a Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Mon, 30 Jan 2023 16:35:26 +0000 Subject: [PATCH 076/248] refactor(FeedSource.java): Addded BsonIgnore to deployed feed version params --- .../java/com/conveyal/datatools/manager/models/FeedSource.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java b/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java index 9d0e24ce0..3e27ec5d6 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java @@ -462,6 +462,7 @@ public String latestVersionId() { @JsonInclude(JsonInclude.Include.NON_NULL) @JsonView(JsonViews.UserInterface.class) @JsonProperty("deployedFeedVersionId") + @BsonIgnore public String getDeployedFeedVersionId() { FeedVersion feedVersion = getDeployedFeedVersion(); return feedVersion != null ? feedVersion.id : null; @@ -470,6 +471,7 @@ public String getDeployedFeedVersionId() { @JsonInclude(JsonInclude.Include.NON_NULL) @JsonView(JsonViews.UserInterface.class) @JsonProperty("deployedFeedVersionStartDate") + @BsonIgnore public LocalDate getDeployedFeedVersionStartDate() { FeedVersion feedVersion = getDeployedFeedVersion(); return feedVersion != null ? feedVersion.validationSummary().startDate : null; @@ -478,6 +480,7 @@ public LocalDate getDeployedFeedVersionStartDate() { @JsonInclude(JsonInclude.Include.NON_NULL) @JsonView(JsonViews.UserInterface.class) @JsonProperty("deployedFeedVersionEndDate") + @BsonIgnore public LocalDate getDeployedFeedVersionEndDate() { FeedVersion feedVersion = getDeployedFeedVersion(); return feedVersion != null ? feedVersion.validationSummary().endDate : null; From 68d430766ca6c89ff96c4661cd7a8e3042ef4eab Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Thu, 16 Feb 2023 10:05:38 -0500 Subject: [PATCH 077/248] feat: Add MobilityData GTFS Validator to validation step --- pom.xml | 14 ++++++-- .../datatools/manager/models/FeedVersion.java | 36 +++++++++++++++++++ 2 files changed, 48 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 4c79d94da..a44caf512 100644 --- a/pom.xml +++ b/pom.xml @@ -96,8 +96,8 @@ maven-compiler-plugin 3.7.0 - 1.8 - 1.8 + 11 + 11 @@ -440,6 +440,16 @@ aws-java-sdk-sts ${awsjavasdk.version} + + com.github.MobilityData.gtfs-validator + gtfs-validator-main + 4.0.0 + + + com.google.code.gson + gson + 2.8.6 + diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java b/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java index b3615939f..33be5d7cb 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java @@ -20,14 +20,21 @@ import com.fasterxml.jackson.annotation.JsonInclude.Include; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonView; +import org.bson.Document; import org.bson.codecs.pojo.annotations.BsonProperty; +import org.mobilitydata.gtfsvalidator.runner.ValidationRunner; +import org.mobilitydata.gtfsvalidator.util.VersionResolver; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.mobilitydata.gtfsvalidator.runner.ValidationRunnerConfig; +import java.io.BufferedReader; import java.io.File; +import java.io.FileReader; import java.io.IOException; import java.io.InputStream; import java.io.Serializable; +import java.nio.file.Path; import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.ResultSet; @@ -253,6 +260,8 @@ public FeedValidationResultSummary validationSummary() { * */ public Date processedByExternalPublisher; + public Document mobilityDataResult; + public String formattedTimestamp() { SimpleDateFormat format = new SimpleDateFormat(HUMAN_READABLE_TIMESTAMP_FORMAT); return format.format(this.updated); @@ -345,8 +354,35 @@ public void validate(MonitorableJob.Status status) { if (status == null) status = new MonitorableJob.Status(); // VALIDATE GTFS feed + FileReader fr = null; try { LOG.info("Beginning validation..."); + status.update("MobilityData Analysis...", 11); + + File gtfsZip = this.retrieveGtfsFile(); + // Namespace based folders avoid clash for validation being run on multiple versions of a feed + // TODO: do we know that there will always be a namespace? + String validatorOutputDirectory = "/tmp/datatools_gtfs/" + this.namespace + "/"; + + // Set up MobilityData validator + ValidationRunnerConfig.Builder builder = ValidationRunnerConfig.builder(); + builder.setGtfsSource(gtfsZip.toURI()); + builder.setOutputDirectory(Path.of(validatorOutputDirectory)); + ValidationRunnerConfig mbValidatorConfig = builder.build(); + + // Run MobilityData validator + ValidationRunner runner = new ValidationRunner(new VersionResolver()); + runner.run(mbValidatorConfig); + + // Read generated report and save to Mongo + fr = new FileReader(validatorOutputDirectory + "report.json"); + BufferedReader in = new BufferedReader(fr); + String json = in.lines().collect(Collectors.joining(System.lineSeparator())); + fr.close(); + + // This will persist the document to Mongo + this.mobilityDataResult = Document.parse(json); + // FIXME: pass status to validate? Or somehow listen to events? status.update("Validating feed...", 33); From ba24a7e8bf7dc87b16eb38dad1062947102050af Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Thu, 16 Feb 2023 10:10:32 -0500 Subject: [PATCH 078/248] chore: update maven java version --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index a44caf512..a64c54cf3 100644 --- a/pom.xml +++ b/pom.xml @@ -96,8 +96,8 @@ maven-compiler-plugin 3.7.0 - 11 - 11 + 18 + 18 From 923e047ec900ae32a576b4f2e1f72391df1022d5 Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Thu, 16 Feb 2023 10:19:16 -0500 Subject: [PATCH 079/248] refactor: support Java language version 11 --- pom.xml | 4 ++-- .../models/transform/NormalizeFieldTransformation.java | 8 ++++---- .../transform/ReplaceFileFromStringTransformation.java | 2 +- .../transform/ReplaceFileFromVersionTransformation.java | 4 ++-- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/pom.xml b/pom.xml index a64c54cf3..a44caf512 100644 --- a/pom.xml +++ b/pom.xml @@ -96,8 +96,8 @@ maven-compiler-plugin 3.7.0 - 18 - 18 + 11 + 11 diff --git a/src/main/java/com/conveyal/datatools/manager/models/transform/NormalizeFieldTransformation.java b/src/main/java/com/conveyal/datatools/manager/models/transform/NormalizeFieldTransformation.java index 7e9cd390d..d9a4c146f 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/transform/NormalizeFieldTransformation.java +++ b/src/main/java/com/conveyal/datatools/manager/models/transform/NormalizeFieldTransformation.java @@ -234,10 +234,10 @@ public void transform(FeedTransformZipTarget zipTarget, MonitorableJob.Status st // Copy csv input stream into the zip file, replacing the existing file. try ( - // Modify target zip file that we just read. - FileSystem targetZipFs = FileSystems.newFileSystem(tempZipPath, null); - // Stream for file copy operation. - InputStream inputStream = new ByteArrayInputStream(stringWriter.toString().getBytes(StandardCharsets.UTF_8)) + // Modify target zip file that we just read. + FileSystem targetZipFs = FileSystems.newFileSystem(tempZipPath, (ClassLoader) null); + // Stream for file copy operation. + InputStream inputStream = new ByteArrayInputStream(stringWriter.toString().getBytes(StandardCharsets.UTF_8)) ) { Path targetTxtFilePath = getTablePathInZip(tableName, targetZipFs); Files.copy(inputStream, targetTxtFilePath, StandardCopyOption.REPLACE_EXISTING); diff --git a/src/main/java/com/conveyal/datatools/manager/models/transform/ReplaceFileFromStringTransformation.java b/src/main/java/com/conveyal/datatools/manager/models/transform/ReplaceFileFromStringTransformation.java index 5673d0c05..2ac87e569 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/transform/ReplaceFileFromStringTransformation.java +++ b/src/main/java/com/conveyal/datatools/manager/models/transform/ReplaceFileFromStringTransformation.java @@ -42,7 +42,7 @@ public void transform(FeedTransformZipTarget zipTarget, MonitorableJob.Status st String tableName = table + ".txt"; // Run the replace transformation Path targetZipPath = Paths.get(zipTarget.gtfsFile.getAbsolutePath()); - try( FileSystem targetZipFs = FileSystems.newFileSystem(targetZipPath, null) ){ + try( FileSystem targetZipFs = FileSystems.newFileSystem(targetZipPath, (ClassLoader) null) ){ // Convert csv data to input stream. InputStream inputStream = new ByteArrayInputStream(csvData.getBytes(StandardCharsets.UTF_8)); Path targetTxtFilePath = getTablePathInZip(tableName, targetZipFs); diff --git a/src/main/java/com/conveyal/datatools/manager/models/transform/ReplaceFileFromVersionTransformation.java b/src/main/java/com/conveyal/datatools/manager/models/transform/ReplaceFileFromVersionTransformation.java index e78d21f72..53963fb16 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/transform/ReplaceFileFromVersionTransformation.java +++ b/src/main/java/com/conveyal/datatools/manager/models/transform/ReplaceFileFromVersionTransformation.java @@ -45,12 +45,12 @@ public void transform(FeedTransformZipTarget zipTarget, MonitorableJob.Status st String tableName = table + ".txt"; // Run the replace transformation Path sourceZipPath = Paths.get(sourceVersion.retrieveGtfsFile().getAbsolutePath()); - try (FileSystem sourceZipFs = FileSystems.newFileSystem(sourceZipPath, null)) { + try (FileSystem sourceZipFs = FileSystems.newFileSystem(sourceZipPath, (ClassLoader) null)) { // If the source txt file does not exist, NoSuchFileException will be thrown and caught below. Path sourceTxtFilePath = getTablePathInZip(tableName, sourceZipFs); Path targetZipPath = Paths.get(zipTarget.gtfsFile.getAbsolutePath()); LOG.info("Replacing file {} in zip file {} with source {}", tableName, targetZipPath.getFileName(), sourceVersion.id); - try (FileSystem targetZipFs = FileSystems.newFileSystem(targetZipPath, null)) { + try (FileSystem targetZipFs = FileSystems.newFileSystem(targetZipPath, (ClassLoader) null)) { Path targetTxtFilePath = getTablePathInZip(tableName, targetZipFs); // Set transform type according to whether target file exists. TransformType type = Files.exists(targetTxtFilePath) From 955b74c0d299b3cbdafe35217c343bc64e8d9285 Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Thu, 16 Feb 2023 10:22:41 -0500 Subject: [PATCH 080/248] ci: use JDK 19 --- .github/workflows/maven.yml | 9 +++++---- pom.xml | 5 +++++ .../conveyal/datatools/HandleCorruptGTFSFileTest.java | 2 +- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index d410ed9a4..0ae8ee807 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -27,10 +27,11 @@ jobs: --health-retries 5 steps: - uses: actions/checkout@v2 - - name: Set up JDK 1.8 - uses: actions/setup-java@v1 + - name: Set up JDK 19 + uses: actions/setup-java@v3 with: - java-version: 1.8 + java-version: 19 + distribution: 'oracle' # Install node 14 for running e2e tests (and for maven-semantic-release). - name: Use Node.js 18.x uses: actions/setup-node@v1 @@ -67,7 +68,7 @@ jobs: - name: Setup GTFS+ directory (used during testing) run: mkdir /tmp/gtfsplus - name: Build with Maven (run unit tests) - run: mvn --no-transfer-progress package + run: mvn --no-transfer-progress -X package - name: Run e2e tests if: env.SHOULD_RUN_E2E == 'true' run: mvn test diff --git a/pom.xml b/pom.xml index a44caf512..9ab411b8d 100644 --- a/pom.xml +++ b/pom.xml @@ -163,6 +163,11 @@ maven-surefire-plugin 2.22.2 + + + --illegal-access=permit + + diff --git a/src/test/java/com/conveyal/datatools/HandleCorruptGTFSFileTest.java b/src/test/java/com/conveyal/datatools/HandleCorruptGTFSFileTest.java index 34ac94f39..31aefd4fa 100644 --- a/src/test/java/com/conveyal/datatools/HandleCorruptGTFSFileTest.java +++ b/src/test/java/com/conveyal/datatools/HandleCorruptGTFSFileTest.java @@ -56,7 +56,7 @@ public void canHandleCorruptGTFSFile() { assertTrue(subJob.status.error); if (subJob instanceof LoadFeedJob) { assertEquals( - "Could not load feed due to java.util.zip.ZipException: error in opening zip file", + "Could not load feed due to java.util.zip.ZipException: zip END header not found", subJob.status.message ); } From 2180c98f2796c2833e44a538a0825aff15ca92f4 Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Fri, 17 Feb 2023 16:32:07 +0000 Subject: [PATCH 081/248] refactor(threaded valiators): Original and mobility data validation executed in separate threads --- .../manager/controllers/DumpController.java | 8 +-- .../manager/jobs/ProcessSingleFeedJob.java | 1 + .../jobs/ValidateMobilityDataFeedJob.java | 52 +++++++++++++++++++ .../datatools/manager/models/FeedVersion.java | 50 ++++++++++++------ 4 files changed, 90 insertions(+), 21 deletions(-) create mode 100644 src/main/java/com/conveyal/datatools/manager/jobs/ValidateMobilityDataFeedJob.java diff --git a/src/main/java/com/conveyal/datatools/manager/controllers/DumpController.java b/src/main/java/com/conveyal/datatools/manager/controllers/DumpController.java index b26dbcde0..ffe0d6e53 100644 --- a/src/main/java/com/conveyal/datatools/manager/controllers/DumpController.java +++ b/src/main/java/com/conveyal/datatools/manager/controllers/DumpController.java @@ -4,6 +4,7 @@ import com.conveyal.datatools.manager.auth.Auth0UserProfile; import com.conveyal.datatools.manager.jobs.ProcessSingleFeedJob; import com.conveyal.datatools.manager.jobs.ValidateFeedJob; +import com.conveyal.datatools.manager.jobs.ValidateMobilityDataFeedJob; import com.conveyal.datatools.manager.models.Deployment; import com.conveyal.datatools.manager.models.ExternalFeedSourceProperty; import com.conveyal.datatools.manager.models.FeedRetrievalMethod; @@ -354,18 +355,17 @@ public static boolean validateAll (boolean load, boolean force, String filterFee // If the force option is not true and the validation result did not fail, re-validate. continue; } - MonitorableJob job; if (filterFeedId != null && !version.feedSourceId.equals(filterFeedId)) { // Skip all feeds except Cortland for now. continue; } Auth0UserProfile systemUser = Auth0UserProfile.createSystemUser(); if (load) { - job = new ProcessSingleFeedJob(version, systemUser, false); + JobUtils.heavyExecutor.execute(new ProcessSingleFeedJob(version, systemUser, false)); } else { - job = new ValidateFeedJob(version, systemUser, false); + JobUtils.heavyExecutor.execute(new ValidateFeedJob(version, systemUser, false)); + JobUtils.heavyExecutor.execute(new ValidateMobilityDataFeedJob(version, systemUser)); } - JobUtils.heavyExecutor.execute(job); } // ValidateAllFeedsJob validateAllFeedsJob = new ValidateAllFeedsJob("system", force, load); return true; diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/ProcessSingleFeedJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/ProcessSingleFeedJob.java index 978e50162..11dcb5251 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/ProcessSingleFeedJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/ProcessSingleFeedJob.java @@ -108,6 +108,7 @@ public void jobLogic() { // Next, validate the feed. addNextJob(new ValidateFeedJob(feedVersion, owner, isNewVersion)); + addNextJob(new ValidateMobilityDataFeedJob(feedVersion, owner)); // We only need to snapshot the feed if there are transformations at the database level. In the case that there // are, the snapshot namespace will be the target of these modifications. If we were to apply the modifications diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/ValidateMobilityDataFeedJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/ValidateMobilityDataFeedJob.java new file mode 100644 index 000000000..1c3e916de --- /dev/null +++ b/src/main/java/com/conveyal/datatools/manager/jobs/ValidateMobilityDataFeedJob.java @@ -0,0 +1,52 @@ +package com.conveyal.datatools.manager.jobs; + +import com.conveyal.datatools.common.status.FeedVersionJob; +import com.conveyal.datatools.manager.auth.Auth0UserProfile; +import com.conveyal.datatools.manager.models.FeedVersion; +import com.fasterxml.jackson.annotation.JsonProperty; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * This job handles the MobilityData validation of a given feed version. If the version is not new, it will simply + * replace the existing version with the version object that has updated validation info. + */ +public class ValidateMobilityDataFeedJob extends FeedVersionJob { + public static final Logger LOG = LoggerFactory.getLogger(ValidateMobilityDataFeedJob.class); + + private FeedVersion feedVersion; + + public ValidateMobilityDataFeedJob(FeedVersion version, Auth0UserProfile owner) { + super(owner, "Validating Feed using MobilityData", JobType.VALIDATE_FEED); + feedVersion = version; + status.update("Waiting to begin MobilityData validation...", 0); + } + + @Override + public void jobLogic () { + LOG.info("Running ValidateMobilityDataFeedJob for {}", feedVersion.id); + feedVersion.validateMobility(status); + } + + @Override + public void jobFinished () { + status.completeSuccessfully("MobilityData validation finished!"); + } + + /** + * Getter that allows a client to know the ID of the feed version that will be created as soon as the upload is + * initiated; however, we will not store the FeedVersion in the mongo application database until the upload and + * processing is completed. This prevents clients from manipulating GTFS data before it is entirely imported. + */ + @JsonProperty + public String getFeedVersionId () { + return feedVersion.id; + } + + @JsonProperty + public String getFeedSourceId () { + return feedVersion.parentFeedSource().id; + } + + +} diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java b/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java index 33be5d7cb..36f678d08 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java @@ -354,9 +354,41 @@ public void validate(MonitorableJob.Status status) { if (status == null) status = new MonitorableJob.Status(); // VALIDATE GTFS feed - FileReader fr = null; try { LOG.info("Beginning validation..."); + + // FIXME: pass status to validate? Or somehow listen to events? + status.update("Validating feed...", 33); + + // Validate the feed version. + // Certain extensions, if enabled, have extra validators + if (isExtensionEnabled("mtc")) { + validationResult = GTFS.validate(feedLoadResult.uniqueIdentifier, DataManager.GTFS_DATA_SOURCE, + RouteTypeValidatorBuilder::buildRouteValidator, + MTCValidator::new + ); + } else { + validationResult = GTFS.validate(feedLoadResult.uniqueIdentifier, DataManager.GTFS_DATA_SOURCE, + RouteTypeValidatorBuilder::buildRouteValidator + ); + } + } catch (Exception e) { + status.fail(String.format("Unable to validate feed %s", this.id), e); + // FIXME create validation result with new constructor? + validationResult = new ValidationResult(); + validationResult.fatalException = "failure!"; + } + } + + public void validateMobility(MonitorableJob.Status status) { + + // Sometimes this method is called when no status object is available. + if (status == null) status = new MonitorableJob.Status(); + + // VALIDATE GTFS feed + FileReader fr = null; + try { + LOG.info("Beginning MobilityData validation..."); status.update("MobilityData Analysis...", 11); File gtfsZip = this.retrieveGtfsFile(); @@ -382,22 +414,6 @@ public void validate(MonitorableJob.Status status) { // This will persist the document to Mongo this.mobilityDataResult = Document.parse(json); - - // FIXME: pass status to validate? Or somehow listen to events? - status.update("Validating feed...", 33); - - // Validate the feed version. - // Certain extensions, if enabled, have extra validators - if (isExtensionEnabled("mtc")) { - validationResult = GTFS.validate(feedLoadResult.uniqueIdentifier, DataManager.GTFS_DATA_SOURCE, - RouteTypeValidatorBuilder::buildRouteValidator, - MTCValidator::new - ); - } else { - validationResult = GTFS.validate(feedLoadResult.uniqueIdentifier, DataManager.GTFS_DATA_SOURCE, - RouteTypeValidatorBuilder::buildRouteValidator - ); - } } catch (Exception e) { status.fail(String.format("Unable to validate feed %s", this.id), e); // FIXME create validation result with new constructor? From 9709713d8dacf92eaf677f7e5d809482ef5aa4d4 Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Mon, 20 Feb 2023 11:20:19 +0000 Subject: [PATCH 082/248] refactor(Updated geo tools): Changed class imports to reference new package names --- pom.xml | 2 +- .../com/conveyal/datatools/editor/utils/GeoUtils.java | 10 +++++----- .../datatools/editor/utils/PolylineEncoder.java | 10 ++++++---- .../conveyal/datatools/manager/jobs/GisExportJob.java | 8 ++++---- .../datatools/manager/jobs/GisExportJobTest.java | 6 +++--- 5 files changed, 19 insertions(+), 17 deletions(-) diff --git a/pom.xml b/pom.xml index 9ab411b8d..0b5bb2b40 100644 --- a/pom.xml +++ b/pom.xml @@ -43,7 +43,7 @@ UTF-8 - 17.5 + 20.1 1.11.625 diff --git a/src/main/java/com/conveyal/datatools/editor/utils/GeoUtils.java b/src/main/java/com/conveyal/datatools/editor/utils/GeoUtils.java index 658564275..a7b3e0e48 100644 --- a/src/main/java/com/conveyal/datatools/editor/utils/GeoUtils.java +++ b/src/main/java/com/conveyal/datatools/editor/utils/GeoUtils.java @@ -1,11 +1,11 @@ package com.conveyal.datatools.editor.utils; -import com.vividsolutions.jts.geom.Coordinate; -import com.vividsolutions.jts.geom.GeometryFactory; -import com.vividsolutions.jts.geom.LineString; -import com.vividsolutions.jts.geom.LinearRing; -import com.vividsolutions.jts.geom.Polygon; import org.geotools.referencing.GeodeticCalculator; +import org.locationtech.jts.geom.Coordinate; +import org.locationtech.jts.geom.GeometryFactory; +import org.locationtech.jts.geom.LineString; +import org.locationtech.jts.geom.LinearRing; +import org.locationtech.jts.geom.Polygon; import java.awt.geom.Point2D; diff --git a/src/main/java/com/conveyal/datatools/editor/utils/PolylineEncoder.java b/src/main/java/com/conveyal/datatools/editor/utils/PolylineEncoder.java index 9072fd382..ed1f135a2 100755 --- a/src/main/java/com/conveyal/datatools/editor/utils/PolylineEncoder.java +++ b/src/main/java/com/conveyal/datatools/editor/utils/PolylineEncoder.java @@ -13,10 +13,12 @@ the License, or (at your option) any later version. package com.conveyal.datatools.editor.utils; -import com.vividsolutions.jts.geom.Coordinate; -import com.vividsolutions.jts.geom.Geometry; -import com.vividsolutions.jts.geom.LineString; -import com.vividsolutions.jts.geom.MultiLineString; + + +import org.locationtech.jts.geom.Coordinate; +import org.locationtech.jts.geom.Geometry; +import org.locationtech.jts.geom.LineString; +import org.locationtech.jts.geom.MultiLineString; import java.util.AbstractList; import java.util.ArrayList; diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/GisExportJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/GisExportJob.java index 2fb589368..849c173bb 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/GisExportJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/GisExportJob.java @@ -12,10 +12,6 @@ import com.conveyal.gtfs.model.Agency; import com.conveyal.gtfs.model.Route; import com.conveyal.gtfs.model.Stop; -import com.vividsolutions.jts.geom.Coordinate; -import com.vividsolutions.jts.geom.GeometryFactory; -import com.vividsolutions.jts.geom.LineString; -import com.vividsolutions.jts.geom.Point; import org.apache.commons.dbutils.DbUtils; import org.apache.commons.io.FileUtils; import org.geotools.data.DataUtilities; @@ -28,6 +24,10 @@ import org.geotools.feature.DefaultFeatureCollection; import org.geotools.feature.simple.SimpleFeatureBuilder; import org.geotools.referencing.crs.DefaultGeographicCRS; +import org.locationtech.jts.geom.Coordinate; +import org.locationtech.jts.geom.GeometryFactory; +import org.locationtech.jts.geom.LineString; +import org.locationtech.jts.geom.Point; import org.opengis.feature.simple.SimpleFeature; import org.opengis.feature.simple.SimpleFeatureType; import org.slf4j.Logger; diff --git a/src/test/java/com/conveyal/datatools/manager/jobs/GisExportJobTest.java b/src/test/java/com/conveyal/datatools/manager/jobs/GisExportJobTest.java index 9b41e2d65..e42976802 100644 --- a/src/test/java/com/conveyal/datatools/manager/jobs/GisExportJobTest.java +++ b/src/test/java/com/conveyal/datatools/manager/jobs/GisExportJobTest.java @@ -11,9 +11,9 @@ import com.conveyal.datatools.manager.utils.SqlAssert; import com.google.common.base.Strings; import com.google.common.io.Files; -import com.vividsolutions.jts.geom.Coordinate; -import com.vividsolutions.jts.geom.MultiLineString; -import com.vividsolutions.jts.geom.Point; +import org.locationtech.jts.geom.Coordinate; +import org.locationtech.jts.geom.MultiLineString; +import org.locationtech.jts.geom.Point; import org.apache.commons.io.FileUtils; import org.geotools.data.DataStore; import org.geotools.data.DataStoreFinder; From 25e9704cfd2624dcb83652f863fae5531417820d Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Tue, 21 Feb 2023 14:52:58 +0000 Subject: [PATCH 083/248] refactor(Create end point to get deployed feed version): Provide the deployed feed version for a giv --- .../controllers/api/FeedSourceController.java | 22 ++++ .../datatools/manager/models/FeedSource.java | 117 +++++++----------- .../manager/models/FeedVersionDeployed.java | 28 +++++ .../api/FeedSourceControllerTest.java | 44 ++++--- 4 files changed, 121 insertions(+), 90 deletions(-) create mode 100644 src/main/java/com/conveyal/datatools/manager/models/FeedVersionDeployed.java diff --git a/src/main/java/com/conveyal/datatools/manager/controllers/api/FeedSourceController.java b/src/main/java/com/conveyal/datatools/manager/controllers/api/FeedSourceController.java index cc8807c87..a57733f9e 100644 --- a/src/main/java/com/conveyal/datatools/manager/controllers/api/FeedSourceController.java +++ b/src/main/java/com/conveyal/datatools/manager/controllers/api/FeedSourceController.java @@ -12,6 +12,8 @@ import com.conveyal.datatools.manager.models.ExternalFeedSourceProperty; import com.conveyal.datatools.manager.models.FeedRetrievalMethod; import com.conveyal.datatools.manager.models.FeedSource; +import com.conveyal.datatools.manager.models.FeedVersion; +import com.conveyal.datatools.manager.models.FeedVersionDeployed; import com.conveyal.datatools.manager.models.JsonViews; import com.conveyal.datatools.manager.models.Project; import com.conveyal.datatools.manager.models.transform.NormalizeFieldTransformation; @@ -395,10 +397,30 @@ protected static FeedSource cleanFeedSourceForNonAdmins(FeedSource feedSource, b return feedSource; } + private static FeedVersionDeployed getDeployedFeedVersion(Request req, Response res) { + Auth0UserProfile userProfile = req.attribute("user"); + String projectId = req.queryParams("projectId"); + Project project = Persistence.projects.getById(projectId); + String feedSourceId = req.queryParams("feedSourceId"); + FeedSource feedSource = Persistence.feedSources.getById(feedSourceId); + if (project == null) { + logMessageAndHalt(req, 400, "Must provide valid projectId value."); + } + if (feedSource == null) { + logMessageAndHalt(req, 400, "Must provide valid feedSourceId value."); + } + if (!userProfile.canAdministerProject(project)) { + logMessageAndHalt(req, 401, "User not authorized to view deployed feed version."); + } + return feedSource.retrieveDeployedFeedVersion(project); + } + + // FIXME: use generic API controller and return JSON documents via BSON/Mongo public static void register (String apiPrefix) { get(apiPrefix + "secure/feedsource/:id", FeedSourceController::getFeedSource, json::write); get(apiPrefix + "secure/feedsource", FeedSourceController::getProjectFeedSources, json::write); + get(apiPrefix + "secure/feedsourcedeployedfeedversion", FeedSourceController::getDeployedFeedVersion, json::write); post(apiPrefix + "secure/feedsource", FeedSourceController::createFeedSource, json::write); put(apiPrefix + "secure/feedsource/:id", FeedSourceController::updateFeedSource, json::write); put(apiPrefix + "secure/feedsource/:id/updateExternal", FeedSourceController::updateExternalFeedResource, json::write); diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java b/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java index 3e27ec5d6..df7da4abb 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java @@ -459,43 +459,9 @@ public String latestVersionId() { return latest != null ? latest.id : null; } - @JsonInclude(JsonInclude.Include.NON_NULL) - @JsonView(JsonViews.UserInterface.class) - @JsonProperty("deployedFeedVersionId") - @BsonIgnore - public String getDeployedFeedVersionId() { - FeedVersion feedVersion = getDeployedFeedVersion(); - return feedVersion != null ? feedVersion.id : null; - } - - @JsonInclude(JsonInclude.Include.NON_NULL) - @JsonView(JsonViews.UserInterface.class) - @JsonProperty("deployedFeedVersionStartDate") - @BsonIgnore - public LocalDate getDeployedFeedVersionStartDate() { - FeedVersion feedVersion = getDeployedFeedVersion(); - return feedVersion != null ? feedVersion.validationSummary().startDate : null; - } - - @JsonInclude(JsonInclude.Include.NON_NULL) - @JsonView(JsonViews.UserInterface.class) - @JsonProperty("deployedFeedVersionEndDate") - @BsonIgnore - public LocalDate getDeployedFeedVersionEndDate() { - FeedVersion feedVersion = getDeployedFeedVersion(); - return feedVersion != null ? feedVersion.validationSummary().endDate : null; - } - - /** - * The deployed feed version. - * This cannot be returned because of a circular reference between feed source and feed version. Instead, individual - * parameters (version id, start date and end date) are returned. - */ - @JsonIgnore - @BsonIgnore - private FeedVersion deployedFeedVersion = null; - /** + * Get deployed feed version for this feed source. + * * If a project has a "pinned" deployment it's feed versions take precedence over the latest feed version for this * feed source. In this case, return either the latest "pinned" feed version or just the feed version, which ever * is available. @@ -503,51 +469,60 @@ public LocalDate getDeployedFeedVersionEndDate() { * If a project does not have a "pinned" deployment (or the above is not available), return the latest feed version * for this feed source from the newest deployment. */ - private FeedVersion getDeployedFeedVersion() { - if (deployedFeedVersion == null) { - Project project = Persistence.projects.getById(this.projectId); - if (project.pinnedDeploymentId != null) { - Deployment deployment = Persistence.deployments.getByIdLimitedFields( - project.pinnedDeploymentId, - "pinnedfeedVersionIds", "feedVersionIds" - ); - FeedVersion feedVersion = getLatestDeployedFeedVersionForFeedSource(deployment.pinnedfeedVersionIds); + public FeedVersionDeployed retrieveDeployedFeedVersion(Project project) { + FeedVersionDeployed deployedFeedVersion = null; + if (project.pinnedDeploymentId != null) { + Deployment deployment = Persistence.deployments.getByIdLimitedFields( + project.pinnedDeploymentId, + "pinnedfeedVersionIds", "feedVersionIds" + ); + FeedVersion feedVersion = getLatestDeployedFeedVersionForFeedSource(deployment.pinnedfeedVersionIds); + if (feedVersion != null) { + // This feed version will be the latest pinned version for this feed source, if available. + deployedFeedVersion = getDeployedFeedVersion(feedVersion); + } else { + feedVersion = getLatestDeployedFeedVersionForFeedSource(deployment.feedVersionIds); if (feedVersion != null) { - // This feed version will be the latest pinned version for this feed source, if available. - deployedFeedVersion = feedVersion; - } else { - feedVersion = getLatestDeployedFeedVersionForFeedSource(deployment.feedVersionIds); - if (feedVersion != null) { - // This feed version will be the latest version for this feed source, if available. - deployedFeedVersion = feedVersion; - } + // This feed version will be the latest version for this feed source, if available. + deployedFeedVersion = getDeployedFeedVersion(feedVersion); } } + } - // If there is no pinned deployment or none of the deployment's feed versions are related to this feed source, - // find the latest feed version for this feed source. - if (deployedFeedVersion == null) { - // Get all deployments for this project. - List deployments = Persistence.deployments.getFilteredLimitedFields( - eq("projectId", this.projectId), - Sorts.descending("lastUpdated"), - "feedVersionIds" - ); - // Iterate through deployments newest to oldest. - for (Deployment deployment : deployments) { - FeedVersion feedVersion = getLatestDeployedFeedVersionForFeedSource(deployment.feedVersionIds); - if (feedVersion != null) { - // This feed version will be the latest feed version for this feed source from the newest - // deployment. - deployedFeedVersion = feedVersion; - break; - } + // If there is no pinned deployment or none of the deployment's feed versions are related to this feed source, + // find the latest feed version for this feed source. + if (deployedFeedVersion == null) { + // Get all deployments for this project. + List deployments = Persistence.deployments.getFilteredLimitedFields( + eq("projectId", project.id), + Sorts.descending("lastUpdated"), + "feedVersionIds" + ); + // Iterate through deployments newest to oldest. + for (Deployment deployment : deployments) { + FeedVersion feedVersion = getLatestDeployedFeedVersionForFeedSource(deployment.feedVersionIds); + if (feedVersion != null) { + // This feed version will be the latest feed version for this feed source from the newest + // deployment. + deployedFeedVersion = getDeployedFeedVersion(feedVersion); + break; } } } return deployedFeedVersion; } + /** + * Extract deployed feed version information from a complete feed version. + */ + private FeedVersionDeployed getDeployedFeedVersion(FeedVersion feedVersion) { + return new FeedVersionDeployed( + feedVersion.id, + feedVersion.validationSummary().startDate, + feedVersion.validationSummary().endDate + ); + } + /** * Get the latest deployed feed version for this feed source, if available. */ diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedVersionDeployed.java b/src/main/java/com/conveyal/datatools/manager/models/FeedVersionDeployed.java new file mode 100644 index 000000000..dc0fabdfa --- /dev/null +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedVersionDeployed.java @@ -0,0 +1,28 @@ +package com.conveyal.datatools.manager.models; + +import com.conveyal.datatools.editor.utils.JacksonSerializers; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; + +import java.time.LocalDate; + +public class FeedVersionDeployed { + public String id; + + @JsonSerialize(using = JacksonSerializers.LocalDateIsoSerializer.class) + @JsonDeserialize(using = JacksonSerializers.LocalDateIsoDeserializer.class) + public LocalDate startDate; + + @JsonSerialize(using = JacksonSerializers.LocalDateIsoSerializer.class) + @JsonDeserialize(using = JacksonSerializers.LocalDateIsoDeserializer.class) + public LocalDate endDate; + + public FeedVersionDeployed() { + } + + public FeedVersionDeployed(String id, LocalDate startDate, LocalDate endDate) { + this.id = id; + this.startDate = startDate; + this.endDate = endDate; + } +} diff --git a/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java b/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java index 8a2ccb872..a152ad007 100644 --- a/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java +++ b/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java @@ -8,6 +8,7 @@ import com.conveyal.datatools.manager.models.FeedRetrievalMethod; import com.conveyal.datatools.manager.models.FeedSource; import com.conveyal.datatools.manager.models.FeedVersion; +import com.conveyal.datatools.manager.models.FeedVersionDeployed; import com.conveyal.datatools.manager.models.FetchFrequency; import com.conveyal.datatools.manager.models.Label; import com.conveyal.datatools.manager.models.Project; @@ -33,6 +34,7 @@ import static org.eclipse.jetty.http.HttpStatus.BAD_REQUEST_400; import static org.eclipse.jetty.http.HttpStatus.OK_200; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; public class FeedSourceControllerTest extends DatatoolsTest { @@ -281,20 +283,24 @@ public void createFeedSourceWithLabels() { @Test void canRetrieveFeedSourceWithDeployedFeedVersion() throws IOException { SimpleHttpResponse response = TestUtils.makeRequest( - String.format("/api/manager/secure/feedsource?projectId=%s", project.id), + String.format( + "/api/manager/secure/feedsourcedeployedfeedversion?projectId=%s&feedSourceId=%s", + project.id, + feedSourceWithDeployedFeedVersion.id + ), null, HttpUtils.REQUEST_METHOD.GET ); assertEquals(OK_200, response.status); - List feedSources = - JsonUtil.getPOJOFromJSONAsList( - JsonUtil.getJsonNodeFromResponse(response), - FeedSource.class + FeedVersionDeployed feedVersion = + JsonUtil.getPOJOFromResponse( + response, + FeedVersionDeployed.class ); - assertEquals(1, feedSources.size()); - assertEquals(feedVersionDeployed.id, feedSources.get(0).getDeployedFeedVersionId()); - assertEquals(feedVersionDeployed.validationSummary().endDate, feedSources.get(0).getDeployedFeedVersionEndDate()); - assertEquals(feedVersionDeployed.validationSummary().startDate, feedSources.get(0).getDeployedFeedVersionStartDate()); + assertNotNull(feedVersion); + assertEquals(feedVersionDeployed.id, feedVersion.id); + assertEquals(feedVersionDeployed.validationSummary().endDate, feedVersion.endDate); + assertEquals(feedVersionDeployed.validationSummary().startDate, feedVersion.startDate); } /** @@ -303,21 +309,21 @@ void canRetrieveFeedSourceWithDeployedFeedVersion() throws IOException { @Test void canRetrieveFeedSourceWithPinnedFeedVersion() throws IOException { SimpleHttpResponse response = TestUtils.makeRequest( - String.format("/api/manager/secure/feedsource?projectId=%s", projectWithPinnedDeployment.id), + String.format("/api/manager/secure/feedsourcedeployedfeedversion?projectId=%s&feedSourceId=%s", projectWithPinnedDeployment.id, feedSourceWithPinnedFeedVersion.id), null, HttpUtils.REQUEST_METHOD.GET ); assertEquals(OK_200, response.status); - List feedSources = - JsonUtil.getPOJOFromJSONAsList( - JsonUtil.getJsonNodeFromResponse(response), - FeedSource.class + FeedVersionDeployed feedVersion = + JsonUtil.getPOJOFromResponse( + response, + FeedVersionDeployed.class ); - assertEquals(1, feedSources.size()); - assertEquals(feedVersionPinned.id, feedSources.get(0).getDeployedFeedVersionId()); - assertEquals(feedVersionPinned.validationSummary().endDate, feedSources.get(0).getDeployedFeedVersionEndDate()); - assertEquals(feedVersionPinned.validationSummary().startDate, feedSources.get(0).getDeployedFeedVersionStartDate()); - } + assertNotNull(feedVersion); + assertEquals(feedVersionPinned.id, feedVersion.id); + assertEquals(feedVersionPinned.validationSummary().endDate, feedVersion.endDate); + assertEquals(feedVersionPinned.validationSummary().startDate, feedVersion.startDate); + } private static FeedSource createFeedSource(String name, URL url, Project project) { From 49206300068a9a77e1206b4b068b645e478eca45 Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Wed, 22 Feb 2023 16:17:11 +0000 Subject: [PATCH 084/248] refactor(Handling of feed version saving): Control saving and update of feed version in relation to --- .../manager/controllers/DumpController.java | 2 +- .../manager/jobs/ProcessSingleFeedJob.java | 2 +- .../manager/jobs/ValidateFeedJob.java | 8 +------- .../jobs/ValidateMobilityDataFeedJob.java | 19 ++++++++++++++++--- .../datatools/manager/models/FeedVersion.java | 17 +++++++++++++++++ 5 files changed, 36 insertions(+), 12 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/controllers/DumpController.java b/src/main/java/com/conveyal/datatools/manager/controllers/DumpController.java index ffe0d6e53..28f4fec28 100644 --- a/src/main/java/com/conveyal/datatools/manager/controllers/DumpController.java +++ b/src/main/java/com/conveyal/datatools/manager/controllers/DumpController.java @@ -364,7 +364,7 @@ public static boolean validateAll (boolean load, boolean force, String filterFee JobUtils.heavyExecutor.execute(new ProcessSingleFeedJob(version, systemUser, false)); } else { JobUtils.heavyExecutor.execute(new ValidateFeedJob(version, systemUser, false)); - JobUtils.heavyExecutor.execute(new ValidateMobilityDataFeedJob(version, systemUser)); + JobUtils.heavyExecutor.execute(new ValidateMobilityDataFeedJob(version, systemUser, false)); } } // ValidateAllFeedsJob validateAllFeedsJob = new ValidateAllFeedsJob("system", force, load); diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/ProcessSingleFeedJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/ProcessSingleFeedJob.java index 11dcb5251..f26eba655 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/ProcessSingleFeedJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/ProcessSingleFeedJob.java @@ -108,7 +108,7 @@ public void jobLogic() { // Next, validate the feed. addNextJob(new ValidateFeedJob(feedVersion, owner, isNewVersion)); - addNextJob(new ValidateMobilityDataFeedJob(feedVersion, owner)); + addNextJob(new ValidateMobilityDataFeedJob(feedVersion, owner, isNewVersion)); // We only need to snapshot the feed if there are transformations at the database level. In the case that there // are, the snapshot namespace will be the target of these modifications. If we were to apply the modifications diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/ValidateFeedJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/ValidateFeedJob.java index 7c8a06541..6f4c6feb6 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/ValidateFeedJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/ValidateFeedJob.java @@ -44,13 +44,7 @@ public void jobFinished () { // such as BuildTransportNetwork, to finish. If those subsequent jobs fail, // the version won't get loaded into MongoDB (even though it exists in postgres). feedVersion.storeUser(owner); - if (isNewVersion) { - int count = feedVersion.parentFeedSource().feedVersionCount(); - feedVersion.version = count + 1; - Persistence.feedVersions.create(feedVersion); - } else { - Persistence.feedVersions.replace(feedVersion.id, feedVersion); - } + feedVersion.persistFeedVersionAfterValidation(isNewVersion); // Schedule expiration notification jobs. Scheduler.scheduleExpirationNotifications(feedVersion.parentFeedSource()); } diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/ValidateMobilityDataFeedJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/ValidateMobilityDataFeedJob.java index 1c3e916de..1c3702082 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/ValidateMobilityDataFeedJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/ValidateMobilityDataFeedJob.java @@ -14,11 +14,13 @@ public class ValidateMobilityDataFeedJob extends FeedVersionJob { public static final Logger LOG = LoggerFactory.getLogger(ValidateMobilityDataFeedJob.class); - private FeedVersion feedVersion; + private final FeedVersion feedVersion; + private final boolean isNewVersion; - public ValidateMobilityDataFeedJob(FeedVersion version, Auth0UserProfile owner) { + public ValidateMobilityDataFeedJob(FeedVersion version, Auth0UserProfile owner, boolean isNewVersion) { super(owner, "Validating Feed using MobilityData", JobType.VALIDATE_FEED); feedVersion = version; + this.isNewVersion = isNewVersion; status.update("Waiting to begin MobilityData validation...", 0); } @@ -30,7 +32,18 @@ public void jobLogic () { @Override public void jobFinished () { - status.completeSuccessfully("MobilityData validation finished!"); + if (!status.error) { + if (parentJobId != null && JobType.PROCESS_FEED.equals(parentJobType)) { + // Validate stage is happening as part of an overall process feed job. + // At this point all GTFS data has been loaded and validated, so we record + // the FeedVersion into mongo. + // This happens here because otherwise we would have to wait for other jobs, + // such as BuildTransportNetwork, to finish. If those subsequent jobs fail, + // the version won't get loaded into MongoDB (even though it exists in postgres). + feedVersion.persistFeedVersionAfterValidation(isNewVersion); + } + status.completeSuccessfully("MobilityData validation finished!"); + } } /** diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java b/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java index 36f678d08..c2faa1e76 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java @@ -3,6 +3,8 @@ import com.conveyal.datatools.common.status.MonitorableJob; import com.conveyal.datatools.common.utils.Scheduler; import com.conveyal.datatools.manager.DataManager; +import com.conveyal.datatools.manager.jobs.ValidateFeedJob; +import com.conveyal.datatools.manager.jobs.ValidateMobilityDataFeedJob; import com.conveyal.datatools.manager.jobs.validation.RouteTypeValidatorBuilder; import com.conveyal.datatools.manager.persistence.FeedStore; import com.conveyal.datatools.manager.persistence.Persistence; @@ -598,4 +600,19 @@ public void assignGtfsFileAttributes(File newGtfsFile) { public boolean isSameAs(FeedVersion otherVersion) { return otherVersion != null && this.hash.equals(otherVersion.hash); } + + /** + * {@link ValidateFeedJob} and {@link ValidateMobilityDataFeedJob} both require to save a feed version after their + * subsequent validation checks have completed. Either could finish first, therefore this method makes sure that + * only one instance is saved (the last to finish updates). + */ + public void persistFeedVersionAfterValidation(boolean isNewVersion) { + if (isNewVersion && Persistence.feedVersions.getById(id) == null) { + int count = parentFeedSource().feedVersionCount(); + version = count + 1; + Persistence.feedVersions.create(this); + } else { + Persistence.feedVersions.replace(id, this); + } + } } From 2932f39c3b29b842e6abcdb7d4e58c0e36db8596 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 22 Feb 2023 19:07:54 +0000 Subject: [PATCH 085/248] chore(deps): bump mongodb-driver-sync from 4.0.5 to 4.0.6 Bumps [mongodb-driver-sync](https://github.com/mongodb/mongo-java-driver) from 4.0.5 to 4.0.6. - [Release notes](https://github.com/mongodb/mongo-java-driver/releases) - [Commits](https://github.com/mongodb/mongo-java-driver/compare/r4.0.5...r4.0.6) --- updated-dependencies: - dependency-name: org.mongodb:mongodb-driver-sync dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 4c79d94da..55e03a79f 100644 --- a/pom.xml +++ b/pom.xml @@ -278,7 +278,7 @@ org.mongodb mongodb-driver-sync - 4.0.5 + 4.0.6 From 7b23e23bf0511809009aac8e93a4c2c064b69bd3 Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Thu, 23 Feb 2023 09:06:59 +0000 Subject: [PATCH 086/248] refactor(Added feed version delete): Delete the feed version if the ValidateMobilityDataFeedJob fail --- .../datatools/manager/jobs/ValidateMobilityDataFeedJob.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/ValidateMobilityDataFeedJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/ValidateMobilityDataFeedJob.java index 1c3702082..c811c10d0 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/ValidateMobilityDataFeedJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/ValidateMobilityDataFeedJob.java @@ -43,6 +43,11 @@ public void jobFinished () { feedVersion.persistFeedVersionAfterValidation(isNewVersion); } status.completeSuccessfully("MobilityData validation finished!"); + } else { + // If the version was not stored successfully, call FeedVersion#delete to reset things to before the version + // was uploaded/fetched. Note: delete calls made to MongoDB on the version ID will not succeed, but that is + // expected. + feedVersion.delete(); } } From 346b97c951eab129f5c15dc1dc2ae4eb5dece7fc Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Fri, 24 Feb 2023 13:45:31 -0500 Subject: [PATCH 087/248] chore(Dockerfile): update java version --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index f7607df81..a64eddcd4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # syntax=docker/dockerfile:1 -FROM maven:3.8.6-openjdk-11 +FROM maven:3.8.7-openjdk-18-slim COPY . /datatools From eaff169e07dc3dbfacea3a84063630996a5de4d6 Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Tue, 28 Feb 2023 13:40:21 +0000 Subject: [PATCH 088/248] refactor(Added new feed source end point to get deployed feed version): --- .../controllers/api/FeedSourceController.java | 14 +- .../datatools/manager/models/FeedSource.java | 173 ++++++++++++------ .../manager/models/FeedVersionDeployed.java | 24 ++- .../manager/persistence/Persistence.java | 7 + .../api/FeedSourceControllerTest.java | 81 ++++---- 5 files changed, 176 insertions(+), 123 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/controllers/api/FeedSourceController.java b/src/main/java/com/conveyal/datatools/manager/controllers/api/FeedSourceController.java index a57733f9e..ac2e6114f 100644 --- a/src/main/java/com/conveyal/datatools/manager/controllers/api/FeedSourceController.java +++ b/src/main/java/com/conveyal/datatools/manager/controllers/api/FeedSourceController.java @@ -12,7 +12,6 @@ import com.conveyal.datatools.manager.models.ExternalFeedSourceProperty; import com.conveyal.datatools.manager.models.FeedRetrievalMethod; import com.conveyal.datatools.manager.models.FeedSource; -import com.conveyal.datatools.manager.models.FeedVersion; import com.conveyal.datatools.manager.models.FeedVersionDeployed; import com.conveyal.datatools.manager.models.JsonViews; import com.conveyal.datatools.manager.models.Project; @@ -398,21 +397,16 @@ protected static FeedSource cleanFeedSourceForNonAdmins(FeedSource feedSource, b } private static FeedVersionDeployed getDeployedFeedVersion(Request req, Response res) { + FeedSource feedSource = requestFeedSourceById(req, Actions.MANAGE); Auth0UserProfile userProfile = req.attribute("user"); - String projectId = req.queryParams("projectId"); - Project project = Persistence.projects.getById(projectId); - String feedSourceId = req.queryParams("feedSourceId"); - FeedSource feedSource = Persistence.feedSources.getById(feedSourceId); + Project project = Persistence.projects.getById(feedSource.projectId); if (project == null) { logMessageAndHalt(req, 400, "Must provide valid projectId value."); } - if (feedSource == null) { - logMessageAndHalt(req, 400, "Must provide valid feedSourceId value."); - } if (!userProfile.canAdministerProject(project)) { logMessageAndHalt(req, 401, "User not authorized to view deployed feed version."); } - return feedSource.retrieveDeployedFeedVersion(project); + return feedSource.retrieveDeployedFeedVersion(project.id); } @@ -420,7 +414,7 @@ private static FeedVersionDeployed getDeployedFeedVersion(Request req, Response public static void register (String apiPrefix) { get(apiPrefix + "secure/feedsource/:id", FeedSourceController::getFeedSource, json::write); get(apiPrefix + "secure/feedsource", FeedSourceController::getProjectFeedSources, json::write); - get(apiPrefix + "secure/feedsourcedeployedfeedversion", FeedSourceController::getDeployedFeedVersion, json::write); + get(apiPrefix + "secure/feedsource/:id/deployedfeedversion", FeedSourceController::getDeployedFeedVersion, json::write); post(apiPrefix + "secure/feedsource", FeedSourceController::createFeedSource, json::write); put(apiPrefix + "secure/feedsource/:id", FeedSourceController::updateFeedSource, json::write); put(apiPrefix + "secure/feedsource/:id/updateExternal", FeedSourceController::updateExternalFeedResource, json::write); diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java b/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java index df7da4abb..aea320c46 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java @@ -26,8 +26,10 @@ import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonView; +import com.google.common.collect.Lists; import com.mongodb.client.FindIterable; import com.mongodb.client.model.Sorts; +import org.bson.Document; import org.bson.codecs.pojo.annotations.BsonIgnore; import org.bson.conversions.Bson; import org.slf4j.Logger; @@ -38,7 +40,6 @@ import java.net.HttpURLConnection; import java.net.MalformedURLException; import java.net.URL; -import java.time.LocalDate; import java.util.ArrayList; import java.util.Collection; import java.util.Date; @@ -50,6 +51,12 @@ import static com.conveyal.datatools.manager.models.FeedRetrievalMethod.FETCHED_AUTOMATICALLY; import static com.conveyal.datatools.manager.utils.StringUtils.getCleanName; +import static com.mongodb.client.model.Aggregates.limit; +import static com.mongodb.client.model.Aggregates.lookup; +import static com.mongodb.client.model.Aggregates.match; +import static com.mongodb.client.model.Aggregates.replaceRoot; +import static com.mongodb.client.model.Aggregates.sort; +import static com.mongodb.client.model.Aggregates.unwind; import static com.mongodb.client.model.Filters.and; import static com.mongodb.client.model.Filters.eq; import static com.mongodb.client.model.Filters.in; @@ -462,78 +469,124 @@ public String latestVersionId() { /** * Get deployed feed version for this feed source. * - * If a project has a "pinned" deployment it's feed versions take precedence over the latest feed version for this - * feed source. In this case, return either the latest "pinned" feed version or just the feed version, which ever - * is available. + * If a project has a "pinned" deployment, the deployment's feed versions take precedence over the latest feed + * version for this feed source. In this case, return the feed version from the pinned deployment. * * If a project does not have a "pinned" deployment (or the above is not available), return the latest feed version - * for this feed source from the newest deployment. - */ - public FeedVersionDeployed retrieveDeployedFeedVersion(Project project) { - FeedVersionDeployed deployedFeedVersion = null; - if (project.pinnedDeploymentId != null) { - Deployment deployment = Persistence.deployments.getByIdLimitedFields( - project.pinnedDeploymentId, - "pinnedfeedVersionIds", "feedVersionIds" - ); - FeedVersion feedVersion = getLatestDeployedFeedVersionForFeedSource(deployment.pinnedfeedVersionIds); - if (feedVersion != null) { - // This feed version will be the latest pinned version for this feed source, if available. - deployedFeedVersion = getDeployedFeedVersion(feedVersion); - } else { - feedVersion = getLatestDeployedFeedVersionForFeedSource(deployment.feedVersionIds); - if (feedVersion != null) { - // This feed version will be the latest version for this feed source, if available. - deployedFeedVersion = getDeployedFeedVersion(feedVersion); - } - } - } - - // If there is no pinned deployment or none of the deployment's feed versions are related to this feed source, - // find the latest feed version for this feed source. - if (deployedFeedVersion == null) { - // Get all deployments for this project. - List deployments = Persistence.deployments.getFilteredLimitedFields( - eq("projectId", project.id), - Sorts.descending("lastUpdated"), - "feedVersionIds" - ); - // Iterate through deployments newest to oldest. - for (Deployment deployment : deployments) { - FeedVersion feedVersion = getLatestDeployedFeedVersionForFeedSource(deployment.feedVersionIds); - if (feedVersion != null) { - // This feed version will be the latest feed version for this feed source from the newest - // deployment. - deployedFeedVersion = getDeployedFeedVersion(feedVersion); - break; - } - } - } - return deployedFeedVersion; + * for this feed source. + */ + public FeedVersionDeployed retrieveDeployedFeedVersion(String projectId) { + FeedVersionDeployed deployedFeedVersion = getFeedVersionFromPinnedDeployment(projectId); + return (deployedFeedVersion != null) ? deployedFeedVersion : getLatestFeedVersionForFeedSource(); } /** - * Extract deployed feed version information from a complete feed version. + * Get the deployed feed version from the pinned deployment for this feed source. + */ + private FeedVersionDeployed getFeedVersionFromPinnedDeployment(String projectId) { + /* + db.getCollection('Project').aggregate([ + { + $match: { + _id: "6f5a3785-68ee-4820-8828-fc8f7f0f4d33" + } + }, + { + $lookup:{ + from:"Deployment", + localField:"pinnedDeploymentId", + foreignField:"_id", + as:"deployment" + } + }, + { + $lookup:{ + from:"FeedVersion", + localField:"deployment.feedVersionIds", + foreignField:"_id", + as:"feedVersions" + } + }, + { + $unwind: "$feedVersions" + }, + { + "$replaceRoot": { + "newRoot": "$feedVersions" + } + }, + { + $match: { + feedSourceId: "78adb78e-7273-41a8-9267-9a9796279894" + } + }, + { + $sort: { + lastUpdated : -1 + } + }, + { + $limit: 1 + } + ]) */ - private FeedVersionDeployed getDeployedFeedVersion(FeedVersion feedVersion) { - return new FeedVersionDeployed( - feedVersion.id, - feedVersion.validationSummary().startDate, - feedVersion.validationSummary().endDate + List stages = Lists.newArrayList( + match( + in("_id", projectId) + ), + lookup("Deployment", "pinnedDeploymentId", "_id", "deployment"), + lookup("FeedVersion", "deployment.feedVersionIds", "_id", "feedVersions"), + unwind("$feedVersions"), + replaceRoot("$feedVersions"), + match( + in("feedSourceId", this.id) + ), + // If more than one feed version for a feed source is held against a deployment the latest is used. + sort(Sorts.descending("lastUpdated")), + limit(1) ); + Document feedVersionDocument = Persistence + .getMongoDatabase() + .getCollection("Project") + .aggregate(stages) + .first(); + return (feedVersionDocument == null) ? null : new FeedVersionDeployed(feedVersionDocument); } /** - * Get the latest deployed feed version for this feed source, if available. + * Get the latest feed version for this feed source as the deployed feed version. */ - private FeedVersion getLatestDeployedFeedVersionForFeedSource(Collection feedVersionIds) { - return Persistence.feedVersions.getOneFiltered( - and( - eq("feedSourceId", this.id), - in("_id", feedVersionIds) + private FeedVersionDeployed getLatestFeedVersionForFeedSource() { + /* + db.getCollection('FeedVersion').aggregate([ + { + $match: { + feedSourceId: "78adb78e-7273-41a8-9267-9a9796279894" + } + }, + { + $sort: { + lastUpdated : -1 + } + }, + { + $limit: 1 + } + ]) + */ + List stages = Lists.newArrayList( + match( + in("feedSourceId", this.id) ), - Sorts.descending("updated") + sort(Sorts.descending("lastUpdated")), + limit(1) ); + FeedVersion feedVersion = Persistence + .feedVersions + .getMongoCollection() + .aggregate(stages) + .first(); + return (feedVersion == null) ? null : new FeedVersionDeployed(feedVersion); } /** diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedVersionDeployed.java b/src/main/java/com/conveyal/datatools/manager/models/FeedVersionDeployed.java index dc0fabdfa..223f83858 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedVersionDeployed.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedVersionDeployed.java @@ -1,10 +1,16 @@ package com.conveyal.datatools.manager.models; import com.conveyal.datatools.editor.utils.JacksonSerializers; +import com.conveyal.gtfs.validator.ValidationResult; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import org.bson.Document; +import javax.print.Doc; import java.time.LocalDate; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; +import java.util.Date; public class FeedVersionDeployed { public String id; @@ -20,9 +26,19 @@ public class FeedVersionDeployed { public FeedVersionDeployed() { } - public FeedVersionDeployed(String id, LocalDate startDate, LocalDate endDate) { - this.id = id; - this.startDate = startDate; - this.endDate = endDate; + public FeedVersionDeployed(Document feedVersionDocument) { + this.id = feedVersionDocument.getString("_id"); + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyyMMdd"); + Document validationResult = (Document) feedVersionDocument.get("validationResult"); + String first = validationResult.getString("firstCalendarDate"); + String last = validationResult.getString("lastCalendarDate"); + this.startDate = (first == null) ? null : LocalDate.parse(first, formatter); + this.endDate = (last == null) ? null : LocalDate.parse(last, formatter); + } + + public FeedVersionDeployed(FeedVersion feedVersion) { + this.id = feedVersion.id; + this.startDate = feedVersion.validationSummary().startDate; + this.endDate = feedVersion.validationSummary().endDate; } } diff --git a/src/main/java/com/conveyal/datatools/manager/persistence/Persistence.java b/src/main/java/com/conveyal/datatools/manager/persistence/Persistence.java index 91f03bf85..2bd79ef6f 100644 --- a/src/main/java/com/conveyal/datatools/manager/persistence/Persistence.java +++ b/src/main/java/com/conveyal/datatools/manager/persistence/Persistence.java @@ -128,5 +128,12 @@ public static void initialize () { // feedVersions.getMongoCollection().createIndex(Indexes.descending("feedSourceId", "version")); // snapshots.getMongoCollection().createIndex(Indexes.descending("feedSourceId", "version")); } + + /** + * Provide a direct link to the Mongo database which is not tied to a specific entity type. + */ + public static MongoDatabase getMongoDatabase() { + return mongoDatabase; + } } diff --git a/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java b/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java index a152ad007..d28a49768 100644 --- a/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java +++ b/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java @@ -45,16 +45,14 @@ public class FeedSourceControllerTest extends DatatoolsTest { private static FeedSource feedSourceWithNoUrl = null; private static FeedSource feedSourceWithLabels = null; private static FeedSource feedSourceWithInvalidLabels = null; - private static FeedSource feedSourceWithDeployedFeedVersion = null; + private static FeedSource feedSourceWithLatestFeedVersion = null; private static FeedSource feedSourceWithPinnedFeedVersion = null; private static Label publicLabel = null; private static Label adminOnlyLabel = null; private static FeedVersion feedVersionSuperseded = null; private static FeedVersion feedVersionDeployed = null; private static FeedVersion feedVersionLatest = null; - private static FeedVersion feedVersionPinned = null; - private static Deployment deploymentSuperseded = null; - private static Deployment deploymentDeployed = null; + private static FeedVersion feedVersionFromPinnedDeployment = null; private static Deployment deploymentPinned = null; @BeforeAll @@ -71,9 +69,6 @@ public static void setUp() throws IOException { projectToBeDeleted.autoFetchFeeds = false; Persistence.projects.create(projectToBeDeleted); - projectWithPinnedDeployment = new Project(); - projectWithPinnedDeployment.name = "ProjectThree"; - Persistence.projects.create(projectWithPinnedDeployment); feedSourceWithUrl = createFeedSource("FeedSourceOne", new URL("http://www.feedsource.com"), project); feedSourceWithNoUrl = createFeedSource("FeedSourceTwo", null, project); @@ -84,24 +79,24 @@ public static void setUp() throws IOException { adminOnlyLabel.adminOnly = true; publicLabel = createLabel("Public Label"); - feedSourceWithDeployedFeedVersion = createFeedSource("FeedSource", null, project, true); - feedSourceWithPinnedFeedVersion = createFeedSource("FeedSourceWithPinnedFeedVersion", null, projectWithPinnedDeployment, true); - + // Latest feed version of feed source. + feedSourceWithLatestFeedVersion = createFeedSource("FeedSource", null, project, true); LocalDate supersededDate = LocalDate.of(2020, Month.DECEMBER, 25); LocalDate deployedEndDate = LocalDate.of(2021, Month.MARCH, 12); LocalDate deployedStartDate = LocalDate.of(2021, Month.MARCH, 1); - feedVersionSuperseded = createFeedVersion("superseded", feedSourceWithDeployedFeedVersion.id, supersededDate); - feedVersionDeployed = createFeedVersion("deployed", feedSourceWithDeployedFeedVersion.id, deployedStartDate, deployedEndDate); - feedVersionLatest = createFeedVersion("latest", feedSourceWithDeployedFeedVersion.id, LocalDate.of(2022, Month.NOVEMBER, 2)); - feedVersionPinned = createFeedVersion("pinned", feedSourceWithPinnedFeedVersion.id, LocalDate.of(2022, Month.NOVEMBER, 2)); - - deploymentSuperseded = createDeployment("superseded", project, feedVersionSuperseded.id, null, supersededDate); - deploymentDeployed = createDeployment("deployed", project, feedVersionDeployed.id, null, deployedEndDate); - deploymentPinned = createDeployment("pinned", projectWithPinnedDeployment, null, feedVersionPinned.id, deployedEndDate); + feedVersionSuperseded = createFeedVersion("superseded", feedSourceWithLatestFeedVersion.id, supersededDate); + feedVersionDeployed = createFeedVersion("deployed", feedSourceWithLatestFeedVersion.id, deployedStartDate, deployedEndDate); + feedVersionLatest = createFeedVersion("latest", feedSourceWithLatestFeedVersion.id, LocalDate.of(2022, Month.NOVEMBER, 2)); + // Feed version from pinned deployment. + projectWithPinnedDeployment = new Project(); + projectWithPinnedDeployment.name = "ProjectThree"; + Persistence.projects.create(projectWithPinnedDeployment); + feedSourceWithPinnedFeedVersion = createFeedSource("FeedSourceWithPinnedFeedVersion", null, projectWithPinnedDeployment, true); + feedVersionFromPinnedDeployment = createFeedVersion("pinned", feedSourceWithPinnedFeedVersion.id, LocalDate.of(2022, Month.NOVEMBER, 2)); + deploymentPinned = createDeployment("pinned", projectWithPinnedDeployment, feedVersionFromPinnedDeployment.id, deployedEndDate); projectWithPinnedDeployment.pinnedDeploymentId = deploymentPinned.id; Persistence.projects.replace(projectWithPinnedDeployment.id, projectWithPinnedDeployment); - } @AfterAll @@ -128,8 +123,8 @@ public static void tearDown() { if (adminOnlyLabel != null) { Persistence.labels.removeById(adminOnlyLabel.id); } - if (feedSourceWithDeployedFeedVersion != null) { - Persistence.feedSources.removeById(feedSourceWithDeployedFeedVersion.id); + if (feedSourceWithLatestFeedVersion != null) { + Persistence.feedSources.removeById(feedSourceWithLatestFeedVersion.id); } if (feedSourceWithPinnedFeedVersion != null) { Persistence.feedSources.removeById(feedSourceWithPinnedFeedVersion.id); @@ -143,14 +138,8 @@ public static void tearDown() { if (feedVersionLatest != null) { Persistence.feedVersions.removeById(feedVersionLatest.id); } - if (feedVersionPinned != null) { - Persistence.feedVersions.removeById(feedVersionPinned.id); - } - if (deploymentSuperseded != null) { - Persistence.deployments.removeById(deploymentSuperseded.id); - } - if (deploymentDeployed != null) { - Persistence.deployments.removeById(deploymentDeployed.id); + if (feedVersionFromPinnedDeployment != null) { + Persistence.feedVersions.removeById(feedVersionFromPinnedDeployment.id); } if (deploymentPinned != null) { Persistence.deployments.removeById(deploymentPinned.id); @@ -277,16 +266,12 @@ public void createFeedSourceWithLabels() { assertEquals(0, Persistence.labels.getFiltered(eq("projectId", projectToBeDeleted.id)).size()); } - /** - * Retrieve the latest feed version for a feed source. - */ @Test - void canRetrieveFeedSourceWithDeployedFeedVersion() throws IOException { + void canRetrieveLatestFeedVersionForFeedSource() throws IOException { SimpleHttpResponse response = TestUtils.makeRequest( String.format( - "/api/manager/secure/feedsourcedeployedfeedversion?projectId=%s&feedSourceId=%s", - project.id, - feedSourceWithDeployedFeedVersion.id + "/api/manager/secure/feedsource/%s/deployedfeedversion", + feedSourceWithLatestFeedVersion.id ), null, HttpUtils.REQUEST_METHOD.GET @@ -298,18 +283,18 @@ void canRetrieveFeedSourceWithDeployedFeedVersion() throws IOException { FeedVersionDeployed.class ); assertNotNull(feedVersion); - assertEquals(feedVersionDeployed.id, feedVersion.id); - assertEquals(feedVersionDeployed.validationSummary().endDate, feedVersion.endDate); - assertEquals(feedVersionDeployed.validationSummary().startDate, feedVersion.startDate); + assertEquals(feedVersionLatest.id, feedVersion.id); + assertEquals(feedVersionLatest.validationSummary().endDate, feedVersion.endDate); + assertEquals(feedVersionLatest.validationSummary().startDate, feedVersion.startDate); } - /** - * Retrieve the latest pinned feed version for a feed source. - */ @Test - void canRetrieveFeedSourceWithPinnedFeedVersion() throws IOException { + void canRetrieveDeployedFeedVersionFromPinnedDeployment() throws IOException { SimpleHttpResponse response = TestUtils.makeRequest( - String.format("/api/manager/secure/feedsourcedeployedfeedversion?projectId=%s&feedSourceId=%s", projectWithPinnedDeployment.id, feedSourceWithPinnedFeedVersion.id), + String.format( + "/api/manager/secure/feedsource/%s/deployedfeedversion", + feedSourceWithPinnedFeedVersion.id + ), null, HttpUtils.REQUEST_METHOD.GET ); @@ -320,9 +305,9 @@ void canRetrieveFeedSourceWithPinnedFeedVersion() throws IOException { FeedVersionDeployed.class ); assertNotNull(feedVersion); - assertEquals(feedVersionPinned.id, feedVersion.id); - assertEquals(feedVersionPinned.validationSummary().endDate, feedVersion.endDate); - assertEquals(feedVersionPinned.validationSummary().startDate, feedVersion.startDate); + assertEquals(feedVersionFromPinnedDeployment.id, feedVersion.id); + assertEquals(feedVersionFromPinnedDeployment.validationSummary().endDate, feedVersion.endDate); + assertEquals(feedVersionFromPinnedDeployment.validationSummary().startDate, feedVersion.startDate); } @@ -353,13 +338,11 @@ private static Deployment createDeployment( String name, Project project, String feedVersionId, - String pinnedFeedVersionId, LocalDate dateCreated ) { Deployment deployment = new Deployment(); deployment.dateCreated = Date.from(dateCreated.atStartOfDay(ZoneId.systemDefault()).toInstant()); deployment.feedVersionIds = Collections.singletonList(feedVersionId); - deployment.pinnedfeedVersionIds = Collections.singletonList(pinnedFeedVersionId); deployment.projectId = project.id; deployment.name = name; Persistence.deployments.create(deployment); From 842df9de62b2ec5d9eb820bfc2c1ea45b4e1a134 Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Mon, 6 Mar 2023 16:36:41 -0500 Subject: [PATCH 089/248] refactor: begin MobilityData validator cleanup --- .../datatools/manager/models/FeedVersion.java | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java b/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java index c2faa1e76..78c85b4c5 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java @@ -388,31 +388,36 @@ public void validateMobility(MonitorableJob.Status status) { if (status == null) status = new MonitorableJob.Status(); // VALIDATE GTFS feed - FileReader fr = null; try { LOG.info("Beginning MobilityData validation..."); status.update("MobilityData Analysis...", 11); + // Wait for the file to be entirely copied into the directory. + Thread.sleep(1000); File gtfsZip = this.retrieveGtfsFile(); // Namespace based folders avoid clash for validation being run on multiple versions of a feed // TODO: do we know that there will always be a namespace? String validatorOutputDirectory = "/tmp/datatools_gtfs/" + this.namespace + "/"; + status.update("MobilityData Analysis...", 20); // Set up MobilityData validator ValidationRunnerConfig.Builder builder = ValidationRunnerConfig.builder(); builder.setGtfsSource(gtfsZip.toURI()); builder.setOutputDirectory(Path.of(validatorOutputDirectory)); ValidationRunnerConfig mbValidatorConfig = builder.build(); + status.update("MobilityData Analysis...", 40); // Run MobilityData validator ValidationRunner runner = new ValidationRunner(new VersionResolver()); runner.run(mbValidatorConfig); + status.update("MobilityData Analysis...", 80); // Read generated report and save to Mongo - fr = new FileReader(validatorOutputDirectory + "report.json"); - BufferedReader in = new BufferedReader(fr); - String json = in.lines().collect(Collectors.joining(System.lineSeparator())); - fr.close(); + String json; + try (FileReader fr = new FileReader(validatorOutputDirectory + "report.json")) { + BufferedReader in = new BufferedReader(fr); + json = in.lines().collect(Collectors.joining(System.lineSeparator())); + } // This will persist the document to Mongo this.mobilityDataResult = Document.parse(json); From 3d0727541de67a968c9522e26fed8f793716e6d2 Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Mon, 6 Mar 2023 16:45:05 -0500 Subject: [PATCH 090/248] improve mobility data thread wait --- .../com/conveyal/datatools/manager/models/FeedVersion.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java b/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java index 78c85b4c5..6966a63c0 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java @@ -393,7 +393,8 @@ public void validateMobility(MonitorableJob.Status status) { status.update("MobilityData Analysis...", 11); // Wait for the file to be entirely copied into the directory. - Thread.sleep(1000); + // TODO: base this on the file being completely saved rather than a fixed amount of time. + Thread.sleep(5000); File gtfsZip = this.retrieveGtfsFile(); // Namespace based folders avoid clash for validation being run on multiple versions of a feed // TODO: do we know that there will always be a namespace? From a7830b9c320248a6a13a65f9d78c4c37c19388aa Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Wed, 8 Mar 2023 13:16:23 -0500 Subject: [PATCH 091/248] refactor: address pr feedback --- pom.xml | 1 + .../datatools/manager/models/FeedVersion.java | 12 ++++++------ .../transform/NormalizeFieldTransformation.java | 8 ++++---- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/pom.xml b/pom.xml index 200ed631b..5acc39ef2 100644 --- a/pom.xml +++ b/pom.xml @@ -164,6 +164,7 @@ maven-surefire-plugin 2.22.2 + --illegal-access=permit diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java b/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java index 6966a63c0..ab81f4c31 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java @@ -25,10 +25,10 @@ import org.bson.Document; import org.bson.codecs.pojo.annotations.BsonProperty; import org.mobilitydata.gtfsvalidator.runner.ValidationRunner; +import org.mobilitydata.gtfsvalidator.runner.ValidationRunnerConfig; import org.mobilitydata.gtfsvalidator.util.VersionResolver; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.mobilitydata.gtfsvalidator.runner.ValidationRunnerConfig; import java.io.BufferedReader; import java.io.File; @@ -355,7 +355,7 @@ public void validate(MonitorableJob.Status status) { // Sometimes this method is called when no status object is available. if (status == null) status = new MonitorableJob.Status(); - // VALIDATE GTFS feed + // VALIDATE GTFS feed. try { LOG.info("Beginning validation..."); @@ -396,12 +396,12 @@ public void validateMobility(MonitorableJob.Status status) { // TODO: base this on the file being completely saved rather than a fixed amount of time. Thread.sleep(5000); File gtfsZip = this.retrieveGtfsFile(); - // Namespace based folders avoid clash for validation being run on multiple versions of a feed + // Namespace based folders avoid clash for validation being run on multiple versions of a feed. // TODO: do we know that there will always be a namespace? String validatorOutputDirectory = "/tmp/datatools_gtfs/" + this.namespace + "/"; status.update("MobilityData Analysis...", 20); - // Set up MobilityData validator + // Set up MobilityData validator. ValidationRunnerConfig.Builder builder = ValidationRunnerConfig.builder(); builder.setGtfsSource(gtfsZip.toURI()); builder.setOutputDirectory(Path.of(validatorOutputDirectory)); @@ -413,14 +413,14 @@ public void validateMobility(MonitorableJob.Status status) { runner.run(mbValidatorConfig); status.update("MobilityData Analysis...", 80); - // Read generated report and save to Mongo + // Read generated report and save to Mongo. String json; try (FileReader fr = new FileReader(validatorOutputDirectory + "report.json")) { BufferedReader in = new BufferedReader(fr); json = in.lines().collect(Collectors.joining(System.lineSeparator())); } - // This will persist the document to Mongo + // This will persist the document to Mongo. this.mobilityDataResult = Document.parse(json); } catch (Exception e) { status.fail(String.format("Unable to validate feed %s", this.id), e); diff --git a/src/main/java/com/conveyal/datatools/manager/models/transform/NormalizeFieldTransformation.java b/src/main/java/com/conveyal/datatools/manager/models/transform/NormalizeFieldTransformation.java index d9a4c146f..783828c6c 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/transform/NormalizeFieldTransformation.java +++ b/src/main/java/com/conveyal/datatools/manager/models/transform/NormalizeFieldTransformation.java @@ -234,10 +234,10 @@ public void transform(FeedTransformZipTarget zipTarget, MonitorableJob.Status st // Copy csv input stream into the zip file, replacing the existing file. try ( - // Modify target zip file that we just read. - FileSystem targetZipFs = FileSystems.newFileSystem(tempZipPath, (ClassLoader) null); - // Stream for file copy operation. - InputStream inputStream = new ByteArrayInputStream(stringWriter.toString().getBytes(StandardCharsets.UTF_8)) + // Modify target zip file that we just read. + FileSystem targetZipFs = FileSystems.newFileSystem(tempZipPath, (ClassLoader) null); + // Stream for file copy operation. + InputStream inputStream = new ByteArrayInputStream(stringWriter.toString().getBytes(StandardCharsets.UTF_8)) ) { Path targetTxtFilePath = getTablePathInZip(tableName, targetZipFs); Files.copy(inputStream, targetTxtFilePath, StandardCopyOption.REPLACE_EXISTING); From 5e822fbadbff3020c9557224e01b657e0620f5f6 Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Thu, 9 Mar 2023 15:26:36 +0000 Subject: [PATCH 092/248] refactor(Update to include latest deployment feed version. Matching tests and restructuring of mongo --- .../controllers/api/FeedSourceController.java | 16 -- .../datatools/manager/models/FeedSource.java | 166 ++++--------- .../manager/models/FeedVersionDeployed.java | 219 +++++++++++++++++- .../api/FeedSourceControllerTest.java | 169 +++++++++----- 4 files changed, 362 insertions(+), 208 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/controllers/api/FeedSourceController.java b/src/main/java/com/conveyal/datatools/manager/controllers/api/FeedSourceController.java index ac2e6114f..cc8807c87 100644 --- a/src/main/java/com/conveyal/datatools/manager/controllers/api/FeedSourceController.java +++ b/src/main/java/com/conveyal/datatools/manager/controllers/api/FeedSourceController.java @@ -12,7 +12,6 @@ import com.conveyal.datatools.manager.models.ExternalFeedSourceProperty; import com.conveyal.datatools.manager.models.FeedRetrievalMethod; import com.conveyal.datatools.manager.models.FeedSource; -import com.conveyal.datatools.manager.models.FeedVersionDeployed; import com.conveyal.datatools.manager.models.JsonViews; import com.conveyal.datatools.manager.models.Project; import com.conveyal.datatools.manager.models.transform.NormalizeFieldTransformation; @@ -396,25 +395,10 @@ protected static FeedSource cleanFeedSourceForNonAdmins(FeedSource feedSource, b return feedSource; } - private static FeedVersionDeployed getDeployedFeedVersion(Request req, Response res) { - FeedSource feedSource = requestFeedSourceById(req, Actions.MANAGE); - Auth0UserProfile userProfile = req.attribute("user"); - Project project = Persistence.projects.getById(feedSource.projectId); - if (project == null) { - logMessageAndHalt(req, 400, "Must provide valid projectId value."); - } - if (!userProfile.canAdministerProject(project)) { - logMessageAndHalt(req, 401, "User not authorized to view deployed feed version."); - } - return feedSource.retrieveDeployedFeedVersion(project.id); - } - - // FIXME: use generic API controller and return JSON documents via BSON/Mongo public static void register (String apiPrefix) { get(apiPrefix + "secure/feedsource/:id", FeedSourceController::getFeedSource, json::write); get(apiPrefix + "secure/feedsource", FeedSourceController::getProjectFeedSources, json::write); - get(apiPrefix + "secure/feedsource/:id/deployedfeedversion", FeedSourceController::getDeployedFeedVersion, json::write); post(apiPrefix + "secure/feedsource", FeedSourceController::createFeedSource, json::write); put(apiPrefix + "secure/feedsource/:id", FeedSourceController::updateFeedSource, json::write); put(apiPrefix + "secure/feedsource/:id/updateExternal", FeedSourceController::updateExternalFeedResource, json::write); diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java b/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java index aea320c46..5e004e71c 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java @@ -26,10 +26,8 @@ import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonView; -import com.google.common.collect.Lists; import com.mongodb.client.FindIterable; import com.mongodb.client.model.Sorts; -import org.bson.Document; import org.bson.codecs.pojo.annotations.BsonIgnore; import org.bson.conversions.Bson; import org.slf4j.Logger; @@ -40,6 +38,7 @@ import java.net.HttpURLConnection; import java.net.MalformedURLException; import java.net.URL; +import java.time.LocalDate; import java.util.ArrayList; import java.util.Collection; import java.util.Date; @@ -51,12 +50,6 @@ import static com.conveyal.datatools.manager.models.FeedRetrievalMethod.FETCHED_AUTOMATICALLY; import static com.conveyal.datatools.manager.utils.StringUtils.getCleanName; -import static com.mongodb.client.model.Aggregates.limit; -import static com.mongodb.client.model.Aggregates.lookup; -import static com.mongodb.client.model.Aggregates.match; -import static com.mongodb.client.model.Aggregates.replaceRoot; -import static com.mongodb.client.model.Aggregates.sort; -import static com.mongodb.client.model.Aggregates.unwind; import static com.mongodb.client.model.Filters.and; import static com.mongodb.client.model.Filters.eq; import static com.mongodb.client.model.Filters.in; @@ -466,127 +459,54 @@ public String latestVersionId() { return latest != null ? latest.id : null; } - /** - * Get deployed feed version for this feed source. - * - * If a project has a "pinned" deployment, the deployment's feed versions take precedence over the latest feed - * version for this feed source. In this case, return the feed version from the pinned deployment. - * - * If a project does not have a "pinned" deployment (or the above is not available), return the latest feed version - * for this feed source. - */ - public FeedVersionDeployed retrieveDeployedFeedVersion(String projectId) { - FeedVersionDeployed deployedFeedVersion = getFeedVersionFromPinnedDeployment(projectId); - return (deployedFeedVersion != null) ? deployedFeedVersion : getLatestFeedVersionForFeedSource(); + @JsonIgnore + @BsonIgnore + private FeedVersionDeployed deployedFeedVersion; + + @JsonInclude(JsonInclude.Include.NON_NULL) + @JsonView(JsonViews.UserInterface.class) + @JsonProperty("deployedFeedVersionId") + @BsonIgnore + public String getDeployedFeedVersionId() { + deployedFeedVersion = retrieveDeployedFeedVersion(); + return deployedFeedVersion != null ? deployedFeedVersion.id : null; } - /** - * Get the deployed feed version from the pinned deployment for this feed source. - */ - private FeedVersionDeployed getFeedVersionFromPinnedDeployment(String projectId) { - /* - db.getCollection('Project').aggregate([ - { - $match: { - _id: "6f5a3785-68ee-4820-8828-fc8f7f0f4d33" - } - }, - { - $lookup:{ - from:"Deployment", - localField:"pinnedDeploymentId", - foreignField:"_id", - as:"deployment" - } - }, - { - $lookup:{ - from:"FeedVersion", - localField:"deployment.feedVersionIds", - foreignField:"_id", - as:"feedVersions" - } - }, - { - $unwind: "$feedVersions" - }, - { - "$replaceRoot": { - "newRoot": "$feedVersions" - } - }, - { - $match: { - feedSourceId: "78adb78e-7273-41a8-9267-9a9796279894" - } - }, - { - $sort: { - lastUpdated : -1 - } - }, - { - $limit: 1 - } - ]) - */ - List stages = Lists.newArrayList( - match( - in("_id", projectId) - ), - lookup("Deployment", "pinnedDeploymentId", "_id", "deployment"), - lookup("FeedVersion", "deployment.feedVersionIds", "_id", "feedVersions"), - unwind("$feedVersions"), - replaceRoot("$feedVersions"), - match( - in("feedSourceId", this.id) - ), - // If more than one feed version for a feed source is held against a deployment the latest is used. - sort(Sorts.descending("lastUpdated")), - limit(1) - ); - Document feedVersionDocument = Persistence - .getMongoDatabase() - .getCollection("Project") - .aggregate(stages) - .first(); - return (feedVersionDocument == null) ? null : new FeedVersionDeployed(feedVersionDocument); + @JsonInclude(JsonInclude.Include.NON_NULL) + @JsonView(JsonViews.UserInterface.class) + @JsonProperty("deployedFeedVersionStartDate") + @BsonIgnore + public LocalDate getDeployedFeedVersionStartDate() { + deployedFeedVersion = retrieveDeployedFeedVersion(); + return deployedFeedVersion != null ? deployedFeedVersion.startDate : null; + } + + @JsonInclude(JsonInclude.Include.NON_NULL) + @JsonView(JsonViews.UserInterface.class) + @JsonProperty("deployedFeedVersionEndDate") + @BsonIgnore + public LocalDate getDeployedFeedVersionEndDate() { + deployedFeedVersion = retrieveDeployedFeedVersion(); + return deployedFeedVersion != null ? deployedFeedVersion.endDate : null; } /** - * Get the latest feed version for this feed source as the deployed feed version. + * Get deployed feed version for this feed source. + * + * If a project has a "pinned" deployment, that deployment's feed versions take precedence over the latest + * deployment's feed versions for this feed source. In this case, return the feed version from the pinned deployment. + * + * If a project does not have a "pinned" deployment, return the latest deployment's feed versions for this feed + * source. */ - private FeedVersionDeployed getLatestFeedVersionForFeedSource() { - /* - db.getCollection('FeedVersion').aggregate([ - { - $match: { - feedSourceId: "78adb78e-7273-41a8-9267-9a9796279894" - } - }, - { - $sort: { - lastUpdated : -1 - } - }, - { - $limit: 1 - } - ]) - */ - List stages = Lists.newArrayList( - match( - in("feedSourceId", this.id) - ), - sort(Sorts.descending("lastUpdated")), - limit(1) - ); - FeedVersion feedVersion = Persistence - .feedVersions - .getMongoCollection() - .aggregate(stages) - .first(); - return (feedVersion == null) ? null : new FeedVersionDeployed(feedVersion); + public FeedVersionDeployed retrieveDeployedFeedVersion() { + if (deployedFeedVersion != null) { + return deployedFeedVersion; + } + deployedFeedVersion = FeedVersionDeployed.getFeedVersionFromPinnedDeployment(projectId, id); + return (deployedFeedVersion != null) + ? deployedFeedVersion + : FeedVersionDeployed.getFeedVersionFromLatestDeployment(projectId, id); } /** diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedVersionDeployed.java b/src/main/java/com/conveyal/datatools/manager/models/FeedVersionDeployed.java index 223f83858..4fafc888d 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedVersionDeployed.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedVersionDeployed.java @@ -1,16 +1,25 @@ package com.conveyal.datatools.manager.models; import com.conveyal.datatools.editor.utils.JacksonSerializers; -import com.conveyal.gtfs.validator.ValidationResult; +import com.conveyal.datatools.manager.persistence.Persistence; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import com.google.common.collect.Lists; +import com.mongodb.client.model.Sorts; import org.bson.Document; +import org.bson.conversions.Bson; -import javax.print.Doc; import java.time.LocalDate; -import java.time.ZoneId; import java.time.format.DateTimeFormatter; -import java.util.Date; +import java.util.List; + +import static com.mongodb.client.model.Aggregates.limit; +import static com.mongodb.client.model.Aggregates.lookup; +import static com.mongodb.client.model.Aggregates.match; +import static com.mongodb.client.model.Aggregates.replaceRoot; +import static com.mongodb.client.model.Aggregates.sort; +import static com.mongodb.client.model.Aggregates.unwind; +import static com.mongodb.client.model.Filters.in; public class FeedVersionDeployed { public String id; @@ -30,15 +39,201 @@ public FeedVersionDeployed(Document feedVersionDocument) { this.id = feedVersionDocument.getString("_id"); DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyyMMdd"); Document validationResult = (Document) feedVersionDocument.get("validationResult"); - String first = validationResult.getString("firstCalendarDate"); - String last = validationResult.getString("lastCalendarDate"); - this.startDate = (first == null) ? null : LocalDate.parse(first, formatter); - this.endDate = (last == null) ? null : LocalDate.parse(last, formatter); + if (validationResult != null) { + String first = validationResult.getString("firstCalendarDate"); + String last = validationResult.getString("lastCalendarDate"); + this.startDate = (first == null) ? null : LocalDate.parse(first, formatter); + this.endDate = (last == null) ? null : LocalDate.parse(last, formatter); + } + } + + /** + * Get the deployed feed version from the pinned deployment for this feed source. + */ + public static FeedVersionDeployed getFeedVersionFromPinnedDeployment(String projectId, String feedSourceId) { + /* + Note: To test this script: + 1) Comment out the call to tearDownDeployedFeedVersion() in FeedSourceControllerTest -> tearDown(). + 2) Run FeedSourceControllerTest to created required objects referenced here. + 3) Once complete, delete documents via MongoDB. + 4) Uncomment the call to tearDownDeployedFeedVersion() in FeedSourceControllerTest -> tearDown(). + 5) Re-run FeedSourceControllerTest to confirm deletion of objects. + + db.getCollection('Project').aggregate([ + { + $match: { + _id: "project-with-pinned-deployment" + } + }, + { + $lookup:{ + from:"Deployment", + localField:"pinnedDeploymentId", + foreignField:"_id", + as:"deployment" + } + }, + { + $lookup:{ + from:"FeedVersion", + localField:"deployment.feedVersionIds", + foreignField:"_id", + as:"feedVersions" + } + }, + { + $unwind: "$feedVersions" + }, + { + "$replaceRoot": { + "newRoot": "$feedVersions" + } + }, + { + $match: { + feedSourceId: "feed-source-with-pinned-deployment-feed-version" + } + }, + { + $sort: { + lastUpdated : -1 + } + }, + { + $limit: 1 + } + ]) + */ + List stages = Lists.newArrayList( + match( + in("_id", projectId) + ), + lookup("Deployment", "pinnedDeploymentId", "_id", "deployment"), + lookup("FeedVersion", "deployment.feedVersionIds", "_id", "feedVersions"), + unwind("$feedVersions"), + replaceRoot("$feedVersions"), + match( + in("feedSourceId", feedSourceId) + ), + // If more than one feed version for a feed source is held against a deployment the latest is used. + sort(Sorts.descending("lastUpdated")), + limit(1) + ); + return getFeedVersionDeployed(stages); + } + + /** + * Get the deployed feed version from the latest deployment for this feed source. + */ + public static FeedVersionDeployed getFeedVersionFromLatestDeployment(String projectId, String feedSourceId) { + /* + Note: To test this script: + 1) Comment out the call to tearDownDeployedFeedVersion() in FeedSourceControllerTest -> tearDown(). + 2) Run FeedSourceControllerTest to created required objects referenced here. + 3) Once complete, delete documents via MongoDB. + 4) Uncomment the call to tearDownDeployedFeedVersion() in FeedSourceControllerTest -> tearDown(). + 5) Re-run FeedSourceControllerTest to confirm deletion of objects. + + db.getCollection('Project').aggregate([ + { + // Match provided project id. + $match: { + _id: "project-with-latest-deployment" + } + }, + { + // Get all deployments for this project. + $lookup:{ + from:"Deployment", + localField:"_id", + foreignField:"projectId", + as:"deployments" + } + }, + { + // Deconstruct deployments array to a document for each element. + $unwind: "$deployments" + }, + { + // Make the deployment documents the input/root document. + "$replaceRoot": { + "newRoot": "$deployments" + } + }, + { + // Sort descending. + $sort: { + lastUpdated : -1 + } + }, + { + // At this point we will have the latest deployment for a project. + $limit: 1 + }, + { + // Get all feed versions that have been deployed as part of the latest deployment. + $lookup:{ + from:"FeedVersion", + localField:"feedVersionIds", + foreignField:"_id", + as:"feedVersions" + } + }, + { + // Deconstruct feedVersions array to a document for each element. + $unwind: "$feedVersions" + }, + { + // Make the feed version documents the input/root document. + "$replaceRoot": { + "newRoot": "$feedVersions" + } + }, + { + // Match the required feed source. + $match: { + feedSourceId: "feed-source-with-latest-deployment-feed-version" + } + }, + { + $sort: { + lastUpdated : -1 + } + }, + { + // At this point we will have the latest feed version from the latest deployment for a feed source. + $limit: 1 + } + ]) + */ + List stages = Lists.newArrayList( + match( + in("_id", projectId) + ), + lookup("Deployment", "_id", "projectId", "deployments"), + unwind("$deployments"), + replaceRoot("$deployments"), + sort(Sorts.descending("lastUpdated")), + limit(1), + lookup("FeedVersion", "feedVersionIds", "_id", "feedVersions"), + unwind("$feedVersions"), + replaceRoot("$feedVersions"), + match( + in("feedSourceId", feedSourceId) + ), + // If more than one feed version for a feed source is held against a deployment the latest is used. + sort(Sorts.descending("lastUpdated")), + limit(1) + ); + return getFeedVersionDeployed(stages); } - public FeedVersionDeployed(FeedVersion feedVersion) { - this.id = feedVersion.id; - this.startDate = feedVersion.validationSummary().startDate; - this.endDate = feedVersion.validationSummary().endDate; + private static FeedVersionDeployed getFeedVersionDeployed(List stages) { + Document feedVersionDocument = Persistence + .getMongoDatabase() + .getCollection("Project") + .aggregate(stages) + .first(); + return (feedVersionDocument == null) ? null : new FeedVersionDeployed(feedVersionDocument); } } diff --git a/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java b/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java index d28a49768..e93b9baaf 100644 --- a/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java +++ b/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java @@ -8,7 +8,6 @@ import com.conveyal.datatools.manager.models.FeedRetrievalMethod; import com.conveyal.datatools.manager.models.FeedSource; import com.conveyal.datatools.manager.models.FeedVersion; -import com.conveyal.datatools.manager.models.FeedVersionDeployed; import com.conveyal.datatools.manager.models.FetchFrequency; import com.conveyal.datatools.manager.models.Label; import com.conveyal.datatools.manager.models.Project; @@ -40,18 +39,21 @@ public class FeedSourceControllerTest extends DatatoolsTest { private static Project project = null; private static Project projectToBeDeleted = null; - private static Project projectWithPinnedDeployment = null; private static FeedSource feedSourceWithUrl = null; private static FeedSource feedSourceWithNoUrl = null; private static FeedSource feedSourceWithLabels = null; private static FeedSource feedSourceWithInvalidLabels = null; - private static FeedSource feedSourceWithLatestFeedVersion = null; - private static FeedSource feedSourceWithPinnedFeedVersion = null; private static Label publicLabel = null; private static Label adminOnlyLabel = null; - private static FeedVersion feedVersionSuperseded = null; - private static FeedVersion feedVersionDeployed = null; - private static FeedVersion feedVersionLatest = null; + + private static Project projectWithLatestDeployment = null; + private static FeedSource feedSourceWithLatestDeploymentFeedVersion = null; + private static FeedVersion feedVersionFromLatestDeployment = null; + private static Deployment deploymentLatest = null; + private static Deployment deploymentSuperseded = null; + + private static Project projectWithPinnedDeployment = null; + private static FeedSource feedSourceWithPinnedDeploymentFeedVersion = null; private static FeedVersion feedVersionFromPinnedDeployment = null; private static Deployment deploymentPinned = null; @@ -79,22 +81,61 @@ public static void setUp() throws IOException { adminOnlyLabel.adminOnly = true; publicLabel = createLabel("Public Label"); - // Latest feed version of feed source. - feedSourceWithLatestFeedVersion = createFeedSource("FeedSource", null, project, true); - LocalDate supersededDate = LocalDate.of(2020, Month.DECEMBER, 25); + // Feed version from latest deployment. + projectWithLatestDeployment = new Project(); + projectWithLatestDeployment.id = "project-with-latest-deployment"; + Persistence.projects.create(projectWithLatestDeployment); + feedSourceWithLatestDeploymentFeedVersion = createFeedSource( + "feed-source-with-latest-deployment-feed-version", + "FeedSource", + null, + projectWithLatestDeployment, + true + ); + LocalDate deployedSuperseded = LocalDate.of(2020, Month.MARCH, 12); LocalDate deployedEndDate = LocalDate.of(2021, Month.MARCH, 12); LocalDate deployedStartDate = LocalDate.of(2021, Month.MARCH, 1); - feedVersionSuperseded = createFeedVersion("superseded", feedSourceWithLatestFeedVersion.id, supersededDate); - feedVersionDeployed = createFeedVersion("deployed", feedSourceWithLatestFeedVersion.id, deployedStartDate, deployedEndDate); - feedVersionLatest = createFeedVersion("latest", feedSourceWithLatestFeedVersion.id, LocalDate.of(2022, Month.NOVEMBER, 2)); + feedVersionFromLatestDeployment = createFeedVersion( + "feed-version-from-latest-deployment", + feedSourceWithLatestDeploymentFeedVersion.id, + deployedStartDate, + deployedEndDate + ); + deploymentSuperseded = createDeployment( + "deployment-superseded", + projectWithLatestDeployment, + feedVersionFromLatestDeployment.id, + deployedSuperseded + ); + deploymentLatest = createDeployment( + "deployment-latest", + projectWithLatestDeployment, + feedVersionFromLatestDeployment.id, + deployedEndDate + ); // Feed version from pinned deployment. projectWithPinnedDeployment = new Project(); - projectWithPinnedDeployment.name = "ProjectThree"; + projectWithPinnedDeployment.id = "project-with-pinned-deployment"; Persistence.projects.create(projectWithPinnedDeployment); - feedSourceWithPinnedFeedVersion = createFeedSource("FeedSourceWithPinnedFeedVersion", null, projectWithPinnedDeployment, true); - feedVersionFromPinnedDeployment = createFeedVersion("pinned", feedSourceWithPinnedFeedVersion.id, LocalDate.of(2022, Month.NOVEMBER, 2)); - deploymentPinned = createDeployment("pinned", projectWithPinnedDeployment, feedVersionFromPinnedDeployment.id, deployedEndDate); + feedSourceWithPinnedDeploymentFeedVersion = createFeedSource( + "feed-source-with-pinned-deployment-feed-version", + "FeedSourceWithPinnedFeedVersion", + null, + projectWithPinnedDeployment, + true + ); + feedVersionFromPinnedDeployment = createFeedVersion( + "feed-version-from-pinned-deployment", + feedSourceWithPinnedDeploymentFeedVersion.id, + LocalDate.of(2022, Month.NOVEMBER, 2) + ); + deploymentPinned = createDeployment( + "deployment-pinned", + projectWithPinnedDeployment, + feedVersionFromPinnedDeployment.id, + deployedEndDate + ); projectWithPinnedDeployment.pinnedDeploymentId = deploymentPinned.id; Persistence.projects.replace(projectWithPinnedDeployment.id, projectWithPinnedDeployment); } @@ -108,9 +149,6 @@ public static void tearDown() { if (projectToBeDeleted != null) { Persistence.projects.removeById(projectToBeDeleted.id); } - if (projectWithPinnedDeployment != null) { - Persistence.projects.removeById(projectWithPinnedDeployment.id); - } if (feedSourceWithUrl != null) { Persistence.feedSources.removeById(feedSourceWithUrl.id); } @@ -123,20 +161,29 @@ public static void tearDown() { if (adminOnlyLabel != null) { Persistence.labels.removeById(adminOnlyLabel.id); } - if (feedSourceWithLatestFeedVersion != null) { - Persistence.feedSources.removeById(feedSourceWithLatestFeedVersion.id); + tearDownDeployedFeedVersion(); + } + + /** + * These entities are removed separately so that if the need arises they can be kept. + * This would then allow the Mongo queries defined in FeedSource#getFeedVersionFromLatestDeployment and + * FeedSource#getFeedVersionFromPinnedDeployment to be tested. + */ + private static void tearDownDeployedFeedVersion() { + if (projectWithPinnedDeployment != null) { + Persistence.projects.removeById(projectWithPinnedDeployment.id); } - if (feedSourceWithPinnedFeedVersion != null) { - Persistence.feedSources.removeById(feedSourceWithPinnedFeedVersion.id); + if (projectWithLatestDeployment != null) { + Persistence.projects.removeById(projectWithLatestDeployment.id); } - if (feedVersionSuperseded != null) { - Persistence.feedVersions.removeById(feedVersionSuperseded.id); + if (feedSourceWithLatestDeploymentFeedVersion != null) { + Persistence.feedSources.removeById(feedSourceWithLatestDeploymentFeedVersion.id); } - if (feedVersionDeployed != null) { - Persistence.feedVersions.removeById(feedVersionDeployed.id); + if (feedSourceWithPinnedDeploymentFeedVersion != null) { + Persistence.feedSources.removeById(feedSourceWithPinnedDeploymentFeedVersion.id); } - if (feedVersionLatest != null) { - Persistence.feedVersions.removeById(feedVersionLatest.id); + if (feedVersionFromLatestDeployment != null) { + Persistence.feedVersions.removeById(feedVersionFromLatestDeployment.id); } if (feedVersionFromPinnedDeployment != null) { Persistence.feedVersions.removeById(feedVersionFromPinnedDeployment.id); @@ -144,6 +191,12 @@ public static void tearDown() { if (deploymentPinned != null) { Persistence.deployments.removeById(deploymentPinned.id); } + if (deploymentLatest != null) { + Persistence.deployments.removeById(deploymentLatest.id); + } + if (deploymentSuperseded != null) { + Persistence.deployments.removeById(deploymentSuperseded.id); + } } /** @@ -267,59 +320,61 @@ public void createFeedSourceWithLabels() { } @Test - void canRetrieveLatestFeedVersionForFeedSource() throws IOException { + void canRetrieveDeployedFeedVersionFromLatestDeployment() throws IOException { SimpleHttpResponse response = TestUtils.makeRequest( String.format( - "/api/manager/secure/feedsource/%s/deployedfeedversion", - feedSourceWithLatestFeedVersion.id + "/api/manager/secure/feedsource/%s", + feedSourceWithLatestDeploymentFeedVersion.id ), null, HttpUtils.REQUEST_METHOD.GET ); assertEquals(OK_200, response.status); - FeedVersionDeployed feedVersion = + FeedSource feedSource = JsonUtil.getPOJOFromResponse( response, - FeedVersionDeployed.class + FeedSource.class ); - assertNotNull(feedVersion); - assertEquals(feedVersionLatest.id, feedVersion.id); - assertEquals(feedVersionLatest.validationSummary().endDate, feedVersion.endDate); - assertEquals(feedVersionLatest.validationSummary().startDate, feedVersion.startDate); + assertNotNull(feedSource); + assertEquals(feedSourceWithLatestDeploymentFeedVersion.id, feedSource.id); + assertEquals(feedVersionFromLatestDeployment.id, feedSource.getDeployedFeedVersionId()); + assertEquals(feedVersionFromLatestDeployment.validationSummary().startDate, feedSource.getDeployedFeedVersionStartDate()); + assertEquals(feedVersionFromLatestDeployment.validationSummary().endDate, feedSource.getDeployedFeedVersionEndDate()); } @Test void canRetrieveDeployedFeedVersionFromPinnedDeployment() throws IOException { SimpleHttpResponse response = TestUtils.makeRequest( String.format( - "/api/manager/secure/feedsource/%s/deployedfeedversion", - feedSourceWithPinnedFeedVersion.id + "/api/manager/secure/feedsource/%s", + feedSourceWithPinnedDeploymentFeedVersion.id ), null, HttpUtils.REQUEST_METHOD.GET ); assertEquals(OK_200, response.status); - FeedVersionDeployed feedVersion = + FeedSource feedSource = JsonUtil.getPOJOFromResponse( response, - FeedVersionDeployed.class + FeedSource.class ); - assertNotNull(feedVersion); - assertEquals(feedVersionFromPinnedDeployment.id, feedVersion.id); - assertEquals(feedVersionFromPinnedDeployment.validationSummary().endDate, feedVersion.endDate); - assertEquals(feedVersionFromPinnedDeployment.validationSummary().startDate, feedVersion.startDate); - } - + assertNotNull(feedSource); + assertEquals(feedSourceWithPinnedDeploymentFeedVersion.id, feedSource.id); + assertEquals(feedVersionFromPinnedDeployment.id, feedSource.getDeployedFeedVersionId()); + assertEquals(feedVersionFromPinnedDeployment.validationSummary().endDate, feedSource.getDeployedFeedVersionEndDate()); + assertEquals(feedVersionFromPinnedDeployment.validationSummary().startDate, feedSource.getDeployedFeedVersionStartDate()); + } private static FeedSource createFeedSource(String name, URL url, Project project) { - return createFeedSource(name, url, project, false); + return createFeedSource(null, name, url, project, false); } /** * Helper method to create feed source. */ - private static FeedSource createFeedSource(String name, URL url, Project project, boolean persist) { + private static FeedSource createFeedSource(String id, String name, URL url, Project project, boolean persist) { FeedSource feedSource = new FeedSource(); + if (id != null) feedSource.id = id; feedSource.fetchFrequency = FetchFrequency.MINUTES; feedSource.fetchInterval = 1; feedSource.deployable = false; @@ -335,7 +390,7 @@ private static FeedSource createFeedSource(String name, URL url, Project project * Helper method to create a deployment. */ private static Deployment createDeployment( - String name, + String id, Project project, String feedVersionId, LocalDate dateCreated @@ -344,7 +399,7 @@ private static Deployment createDeployment( deployment.dateCreated = Date.from(dateCreated.atStartOfDay(ZoneId.systemDefault()).toInstant()); deployment.feedVersionIds = Collections.singletonList(feedVersionId); deployment.projectId = project.id; - deployment.name = name; + deployment.id = id; Persistence.deployments.create(deployment); return deployment; } @@ -352,16 +407,16 @@ private static Deployment createDeployment( /** * Helper method to create a feed version with no start date. */ - private static FeedVersion createFeedVersion(String name, String feedSourceId, LocalDate endDate) { - return createFeedVersion(name, feedSourceId, null, endDate); + private static FeedVersion createFeedVersion(String id, String feedSourceId, LocalDate endDate) { + return createFeedVersion(id, feedSourceId, null, endDate); } /** * Helper method to create a feed version. */ - private static FeedVersion createFeedVersion(String name, String feedSourceId, LocalDate startDate, LocalDate endDate) { + private static FeedVersion createFeedVersion(String id, String feedSourceId, LocalDate startDate, LocalDate endDate) { FeedVersion feedVersion = new FeedVersion(); - feedVersion.name = name; + feedVersion.id = id; feedVersion.feedSourceId = feedSourceId; ValidationResult validationResult = new ValidationResult(); validationResult.firstCalendarDate = startDate; From 9ea575b3fcfa8030c7b6176b4ce7d4909c7aabe9 Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Thu, 9 Mar 2023 15:29:42 +0000 Subject: [PATCH 093/248] refactor(FeedSource.java): Added missing comment --- .../com/conveyal/datatools/manager/models/FeedSource.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java b/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java index 5e004e71c..76283dea3 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java @@ -459,6 +459,11 @@ public String latestVersionId() { return latest != null ? latest.id : null; } + /** + * The deployed feed version. + * This cannot be returned because of a circular reference between feed source and feed version. Instead, individual + * parameters (version id, start date and end date) are returned. + */ @JsonIgnore @BsonIgnore private FeedVersionDeployed deployedFeedVersion; From f4479adac8a08157632d6b04d62c1e1ec50192c9 Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Fri, 10 Mar 2023 11:32:16 +0000 Subject: [PATCH 094/248] refactor(FeedSource.java): deployed feed version is now mutually exclusive between pinned and latest --- .../datatools/manager/models/FeedSource.java | 28 +++++++++++++------ 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java b/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java index 76283dea3..a36cbf03c 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java @@ -468,6 +468,14 @@ public String latestVersionId() { @BsonIgnore private FeedVersionDeployed deployedFeedVersion; + /** + * This value is set to true once an attempt has been made to get the deployed feed version. This prevents subsequent + * attempts where the deployed feed version is not available. + */ + @JsonIgnore + @BsonIgnore + private boolean deployedFeedVersionDefined; + @JsonInclude(JsonInclude.Include.NON_NULL) @JsonView(JsonViews.UserInterface.class) @JsonProperty("deployedFeedVersionId") @@ -498,20 +506,24 @@ public LocalDate getDeployedFeedVersionEndDate() { /** * Get deployed feed version for this feed source. * - * If a project has a "pinned" deployment, that deployment's feed versions take precedence over the latest - * deployment's feed versions for this feed source. In this case, return the feed version from the pinned deployment. + * If a project has a "pinned" deployment, return the feed version from this pinned deployment. If it is not + * available return null and don't attempt to get the feed version from the latest deployment. * * If a project does not have a "pinned" deployment, return the latest deployment's feed versions for this feed - * source. + * source, if available. */ public FeedVersionDeployed retrieveDeployedFeedVersion() { - if (deployedFeedVersion != null) { + if (deployedFeedVersionDefined) { return deployedFeedVersion; } - deployedFeedVersion = FeedVersionDeployed.getFeedVersionFromPinnedDeployment(projectId, id); - return (deployedFeedVersion != null) - ? deployedFeedVersion - : FeedVersionDeployed.getFeedVersionFromLatestDeployment(projectId, id); + Project project = Persistence.projects.getById(projectId); + if (project.pinnedDeploymentId != null && !project.pinnedDeploymentId.isEmpty()) { + deployedFeedVersion = FeedVersionDeployed.getFeedVersionFromPinnedDeployment(projectId, id); + } else { + deployedFeedVersion = FeedVersionDeployed.getFeedVersionFromLatestDeployment(projectId, id); + } + deployedFeedVersionDefined = true; + return deployedFeedVersion; } /** From 5873e727e10ad49825388bb22f915db29ea896d1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 10 Mar 2023 13:31:43 +0000 Subject: [PATCH 095/248] chore(deps): bump gson from 2.8.6 to 2.8.9 Bumps [gson](https://github.com/google/gson) from 2.8.6 to 2.8.9. - [Release notes](https://github.com/google/gson/releases) - [Changelog](https://github.com/google/gson/blob/master/CHANGELOG.md) - [Commits](https://github.com/google/gson/compare/gson-parent-2.8.6...gson-parent-2.8.9) --- updated-dependencies: - dependency-name: com.google.code.gson:gson dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 5acc39ef2..6e2f4533e 100644 --- a/pom.xml +++ b/pom.xml @@ -454,7 +454,7 @@ com.google.code.gson gson - 2.8.6 + 2.8.9 From 1815299ea13570bf0b609bef4c74d6705968f084 Mon Sep 17 00:00:00 2001 From: binh-dam-ibigroup <56846598+binh-dam-ibigroup@users.noreply.github.com> Date: Fri, 10 Feb 2023 16:11:42 -0500 Subject: [PATCH 096/248] chore(deps): Upgrade sparkjava to 2.9.4. --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 5acc39ef2..bb5068196 100644 --- a/pom.xml +++ b/pom.xml @@ -215,7 +215,7 @@ com.sparkjava spark-core - 2.7.2 + 2.9.4 From b55c40bef6f3619c877e5fd966cbdc9132772310 Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Thu, 16 Mar 2023 15:15:21 +0000 Subject: [PATCH 097/248] refactor(FeedSource.java): Updated comment and refactored if/else statement --- .../conveyal/datatools/manager/models/FeedSource.java | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java b/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java index a36cbf03c..555414f71 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java @@ -470,7 +470,7 @@ public String latestVersionId() { /** * This value is set to true once an attempt has been made to get the deployed feed version. This prevents subsequent - * attempts where the deployed feed version is not available. + * attempts by Json annotated properties to get a deployed feed version that is not available. */ @JsonIgnore @BsonIgnore @@ -517,11 +517,9 @@ public FeedVersionDeployed retrieveDeployedFeedVersion() { return deployedFeedVersion; } Project project = Persistence.projects.getById(projectId); - if (project.pinnedDeploymentId != null && !project.pinnedDeploymentId.isEmpty()) { - deployedFeedVersion = FeedVersionDeployed.getFeedVersionFromPinnedDeployment(projectId, id); - } else { - deployedFeedVersion = FeedVersionDeployed.getFeedVersionFromLatestDeployment(projectId, id); - } + deployedFeedVersion = (project.pinnedDeploymentId != null && !project.pinnedDeploymentId.isEmpty()) + ? FeedVersionDeployed.getFeedVersionFromPinnedDeployment(projectId, id) + : FeedVersionDeployed.getFeedVersionFromLatestDeployment(projectId, id); deployedFeedVersionDefined = true; return deployedFeedVersion; } From e95896011b7ddd831b7bab239fe0a812025b79f2 Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Fri, 24 Mar 2023 15:24:48 -0400 Subject: [PATCH 098/248] fix(MobilityDataValidation): slow on large gtfs files --- .../com/conveyal/datatools/manager/models/FeedVersion.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java b/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java index ab81f4c31..ec53c251a 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java @@ -393,8 +393,8 @@ public void validateMobility(MonitorableJob.Status status) { status.update("MobilityData Analysis...", 11); // Wait for the file to be entirely copied into the directory. - // TODO: base this on the file being completely saved rather than a fixed amount of time. - Thread.sleep(5000); + // 5 seconds + ~1 second per 10mb + Thread.sleep(5000 + (this.fileSize / 10000)); File gtfsZip = this.retrieveGtfsFile(); // Namespace based folders avoid clash for validation being run on multiple versions of a feed. // TODO: do we know that there will always be a namespace? From cd0d0f85de9d12bce3ef06ce826f9b2615ff628b Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Fri, 14 Apr 2023 17:12:48 +0100 Subject: [PATCH 099/248] refactor(Check for pinned deployment in mongo db): Check if a project has a pinned deployment and re --- .../datatools/manager/models/FeedSource.java | 4 +- .../datatools/manager/models/Project.java | 43 ++++++++++++++++++- 2 files changed, 44 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java b/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java index 555414f71..f90ad15a7 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java @@ -49,6 +49,7 @@ import java.util.stream.Collectors; import static com.conveyal.datatools.manager.models.FeedRetrievalMethod.FETCHED_AUTOMATICALLY; +import static com.conveyal.datatools.manager.models.Project.hasPinnedDeployment; import static com.conveyal.datatools.manager.utils.StringUtils.getCleanName; import static com.mongodb.client.model.Filters.and; import static com.mongodb.client.model.Filters.eq; @@ -516,8 +517,7 @@ public FeedVersionDeployed retrieveDeployedFeedVersion() { if (deployedFeedVersionDefined) { return deployedFeedVersion; } - Project project = Persistence.projects.getById(projectId); - deployedFeedVersion = (project.pinnedDeploymentId != null && !project.pinnedDeploymentId.isEmpty()) + deployedFeedVersion = hasPinnedDeployment(projectId) ? FeedVersionDeployed.getFeedVersionFromPinnedDeployment(projectId, id) : FeedVersionDeployed.getFeedVersionFromLatestDeployment(projectId, id); deployedFeedVersionDefined = true; diff --git a/src/main/java/com/conveyal/datatools/manager/models/Project.java b/src/main/java/com/conveyal/datatools/manager/models/Project.java index 110618cb5..ddd2298a2 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/Project.java +++ b/src/main/java/com/conveyal/datatools/manager/models/Project.java @@ -2,13 +2,17 @@ import com.conveyal.datatools.manager.jobs.AutoDeployType; import com.conveyal.datatools.manager.persistence.Persistence; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.collect.Lists; +import com.mongodb.client.model.Projections; +import org.bson.Document; import org.bson.codecs.pojo.annotations.BsonIgnore; +import org.bson.conversions.Bson; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.ArrayList; import java.util.Collection; import java.util.Date; import java.util.HashSet; @@ -16,8 +20,11 @@ import java.util.Set; import java.util.stream.Collectors; +import static com.mongodb.client.model.Aggregates.match; +import static com.mongodb.client.model.Aggregates.project; import static com.mongodb.client.model.Filters.and; import static com.mongodb.client.model.Filters.eq; +import static com.mongodb.client.model.Filters.in; import static com.mongodb.client.model.Filters.or; /** @@ -191,4 +198,38 @@ public abstract static class ProjectWithOtpServers { @JsonProperty("otpServers") public abstract Collection availableOtpServers (); } + + @BsonIgnore + @JsonIgnore + public static boolean hasPinnedDeployment(String projectId) { + /* + db.getCollection('Project').aggregate([ + { + // Match provided project id. + $match: { + _id: "project-with-latest-deployment" + } + }, + { + $project: { + _id: 0, + pinnedDeploymentId: 1 + } + } + ]) + */ + List stages = Lists.newArrayList( + match( + in("_id", projectId) + ), + project(Projections.excludeId()), + project(Projections.include("pinnedDeploymentId")) + ); + Document project = Persistence + .getMongoDatabase() + .getCollection("Project") + .aggregate(stages) + .first(); + return !project.isEmpty(); + } } From b16eee7674c804e5d46ed090c6e6046c78e67f4c Mon Sep 17 00:00:00 2001 From: "philip.cline" Date: Tue, 25 Apr 2023 09:08:13 -0400 Subject: [PATCH 100/248] chore(gtfs lib): bump GTFS lib version --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index bb5068196..0e6a637b1 100644 --- a/pom.xml +++ b/pom.xml @@ -268,7 +268,7 @@ com.github.conveyal gtfs-lib - 4fcf36e + bdb76ee From 90e20d1334cf7d6184f9196bc110f95006fdf92e Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Tue, 25 Apr 2023 13:05:11 -0400 Subject: [PATCH 101/248] ci: attempt adjusting java version --- .github/workflows/maven.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index 0ae8ee807..7fe151840 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -31,7 +31,7 @@ jobs: uses: actions/setup-java@v3 with: java-version: 19 - distribution: 'oracle' + distribution: 'temurin' # Install node 14 for running e2e tests (and for maven-semantic-release). - name: Use Node.js 18.x uses: actions/setup-node@v1 From cafd7f9ea9bf9bc52dc4ccd3e891a4639cb8af26 Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Mon, 1 May 2023 16:35:36 -0400 Subject: [PATCH 102/248] adjust OtpRouterConfig to be otp2-compatible --- .../com/conveyal/datatools/manager/models/OtpRouterConfig.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/conveyal/datatools/manager/models/OtpRouterConfig.java b/src/main/java/com/conveyal/datatools/manager/models/OtpRouterConfig.java index 1a5b342a5..80dd431da 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/OtpRouterConfig.java +++ b/src/main/java/com/conveyal/datatools/manager/models/OtpRouterConfig.java @@ -31,7 +31,7 @@ public static class Updater implements Serializable { public String url; - public String defaultAgencyId; + public String feedId; } public String requestLogFile; From 128a180fa687026356952a5a5bfbab50892f86b2 Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Wed, 3 May 2023 07:58:41 -0400 Subject: [PATCH 103/248] add more otp2 fields --- .../datatools/manager/models/OtpRouterConfig.java | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/models/OtpRouterConfig.java b/src/main/java/com/conveyal/datatools/manager/models/OtpRouterConfig.java index 80dd431da..09771fb76 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/OtpRouterConfig.java +++ b/src/main/java/com/conveyal/datatools/manager/models/OtpRouterConfig.java @@ -11,16 +11,19 @@ @JsonIgnoreProperties(ignoreUnknown = true) public class OtpRouterConfig implements Serializable { private static final long serialVersionUID = 1L; - public Integer numItineraries; - - public Double walkSpeed; + public Double driveDistanceReluctance; public Double stairsReluctance; - public Double carDropoffTime; - public Collection updaters; + public ItineraryFilter itineraryFilters; + + public static class ItineraryFilter implements Serializable { + private static final long serialVersionUID = 1L; + public String nonTransitGeneralizedCostLimit; + } + public static class Updater implements Serializable { private static final long serialVersionUID = 1L; public String type; From 438a3c04addf017cedb32039afa26a288dbc7875 Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Wed, 3 May 2023 15:55:53 +0100 Subject: [PATCH 104/248] refactor(Added feed source summary): All feed source summaries are available via a new end point --- .../controllers/api/FeedSourceController.java | 17 + .../datatools/manager/models/FeedSource.java | 66 --- .../manager/models/FeedSourceSummary.java | 527 ++++++++++++++++++ .../manager/models/FeedVersionDeployed.java | 239 -------- .../datatools/manager/models/Project.java | 19 + .../manager/persistence/Persistence.java | 14 +- .../api/FeedSourceControllerTest.java | 51 +- 7 files changed, 605 insertions(+), 328 deletions(-) create mode 100644 src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java delete mode 100644 src/main/java/com/conveyal/datatools/manager/models/FeedVersionDeployed.java diff --git a/src/main/java/com/conveyal/datatools/manager/controllers/api/FeedSourceController.java b/src/main/java/com/conveyal/datatools/manager/controllers/api/FeedSourceController.java index cc8807c87..badedda76 100644 --- a/src/main/java/com/conveyal/datatools/manager/controllers/api/FeedSourceController.java +++ b/src/main/java/com/conveyal/datatools/manager/controllers/api/FeedSourceController.java @@ -9,9 +9,11 @@ import com.conveyal.datatools.manager.extensions.ExternalFeedResource; import com.conveyal.datatools.manager.jobs.FetchSingleFeedJob; import com.conveyal.datatools.manager.jobs.NotifyUsersForSubscriptionJob; +import com.conveyal.datatools.manager.models.DeploymentSummary; import com.conveyal.datatools.manager.models.ExternalFeedSourceProperty; import com.conveyal.datatools.manager.models.FeedRetrievalMethod; import com.conveyal.datatools.manager.models.FeedSource; +import com.conveyal.datatools.manager.models.FeedSourceSummary; import com.conveyal.datatools.manager.models.JsonViews; import com.conveyal.datatools.manager.models.Project; import com.conveyal.datatools.manager.models.transform.NormalizeFieldTransformation; @@ -395,6 +397,20 @@ protected static FeedSource cleanFeedSourceForNonAdmins(FeedSource feedSource, b return feedSource; } + private static Collection getAllFeedSourceSummaries(Request req, Response res) { + Auth0UserProfile userProfile = req.attribute("user"); + String projectId = req.queryParams("projectId"); + Project project = Persistence.projects.getById(projectId); + if (project == null) { + logMessageAndHalt(req, 400, "Must provide valid projectId value."); + } + if (!userProfile.canAdministerProject(project)) { + logMessageAndHalt(req, 401, "User not authorized to view project feed sources."); + } + return project.retrieveFeedSourceSummaries(); + } + + // FIXME: use generic API controller and return JSON documents via BSON/Mongo public static void register (String apiPrefix) { get(apiPrefix + "secure/feedsource/:id", FeedSourceController::getFeedSource, json::write); @@ -404,5 +420,6 @@ public static void register (String apiPrefix) { put(apiPrefix + "secure/feedsource/:id/updateExternal", FeedSourceController::updateExternalFeedResource, json::write); delete(apiPrefix + "secure/feedsource/:id", FeedSourceController::deleteFeedSource, json::write); post(apiPrefix + "secure/feedsource/:id/fetch", FeedSourceController::fetch, json::write); + get(apiPrefix + "secure/feedsourceSummaries", FeedSourceController::getAllFeedSourceSummaries, json::write); } } diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java b/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java index f90ad15a7..db7eea7a9 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java @@ -38,7 +38,6 @@ import java.net.HttpURLConnection; import java.net.MalformedURLException; import java.net.URL; -import java.time.LocalDate; import java.util.ArrayList; import java.util.Collection; import java.util.Date; @@ -49,7 +48,6 @@ import java.util.stream.Collectors; import static com.conveyal.datatools.manager.models.FeedRetrievalMethod.FETCHED_AUTOMATICALLY; -import static com.conveyal.datatools.manager.models.Project.hasPinnedDeployment; import static com.conveyal.datatools.manager.utils.StringUtils.getCleanName; import static com.mongodb.client.model.Filters.and; import static com.mongodb.client.model.Filters.eq; @@ -460,70 +458,6 @@ public String latestVersionId() { return latest != null ? latest.id : null; } - /** - * The deployed feed version. - * This cannot be returned because of a circular reference between feed source and feed version. Instead, individual - * parameters (version id, start date and end date) are returned. - */ - @JsonIgnore - @BsonIgnore - private FeedVersionDeployed deployedFeedVersion; - - /** - * This value is set to true once an attempt has been made to get the deployed feed version. This prevents subsequent - * attempts by Json annotated properties to get a deployed feed version that is not available. - */ - @JsonIgnore - @BsonIgnore - private boolean deployedFeedVersionDefined; - - @JsonInclude(JsonInclude.Include.NON_NULL) - @JsonView(JsonViews.UserInterface.class) - @JsonProperty("deployedFeedVersionId") - @BsonIgnore - public String getDeployedFeedVersionId() { - deployedFeedVersion = retrieveDeployedFeedVersion(); - return deployedFeedVersion != null ? deployedFeedVersion.id : null; - } - - @JsonInclude(JsonInclude.Include.NON_NULL) - @JsonView(JsonViews.UserInterface.class) - @JsonProperty("deployedFeedVersionStartDate") - @BsonIgnore - public LocalDate getDeployedFeedVersionStartDate() { - deployedFeedVersion = retrieveDeployedFeedVersion(); - return deployedFeedVersion != null ? deployedFeedVersion.startDate : null; - } - - @JsonInclude(JsonInclude.Include.NON_NULL) - @JsonView(JsonViews.UserInterface.class) - @JsonProperty("deployedFeedVersionEndDate") - @BsonIgnore - public LocalDate getDeployedFeedVersionEndDate() { - deployedFeedVersion = retrieveDeployedFeedVersion(); - return deployedFeedVersion != null ? deployedFeedVersion.endDate : null; - } - - /** - * Get deployed feed version for this feed source. - * - * If a project has a "pinned" deployment, return the feed version from this pinned deployment. If it is not - * available return null and don't attempt to get the feed version from the latest deployment. - * - * If a project does not have a "pinned" deployment, return the latest deployment's feed versions for this feed - * source, if available. - */ - public FeedVersionDeployed retrieveDeployedFeedVersion() { - if (deployedFeedVersionDefined) { - return deployedFeedVersion; - } - deployedFeedVersion = hasPinnedDeployment(projectId) - ? FeedVersionDeployed.getFeedVersionFromPinnedDeployment(projectId, id) - : FeedVersionDeployed.getFeedVersionFromLatestDeployment(projectId, id); - deployedFeedVersionDefined = true; - return deployedFeedVersion; - } - /** * Number of {@link FeedVersion}s that exist for the feed source. */ diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java b/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java new file mode 100644 index 000000000..484760c32 --- /dev/null +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java @@ -0,0 +1,527 @@ +package com.conveyal.datatools.manager.models; + +import com.conveyal.datatools.editor.utils.JacksonSerializers; +import com.conveyal.datatools.manager.persistence.Persistence; +import com.conveyal.gtfs.validator.ValidationResult; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import com.google.common.collect.Lists; +import com.mongodb.client.model.Accumulators; +import com.mongodb.client.model.Projections; +import com.mongodb.client.model.Sorts; +import org.bson.Document; +import org.bson.conversions.Bson; + +import java.time.LocalDate; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static com.mongodb.client.model.Aggregates.group; +import static com.mongodb.client.model.Aggregates.limit; +import static com.mongodb.client.model.Aggregates.lookup; +import static com.mongodb.client.model.Aggregates.match; +import static com.mongodb.client.model.Aggregates.project; +import static com.mongodb.client.model.Aggregates.replaceRoot; +import static com.mongodb.client.model.Aggregates.sort; +import static com.mongodb.client.model.Aggregates.unwind; +import static com.mongodb.client.model.Filters.in; + +public class FeedSourceSummary { + private static final DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyyMMdd"); + + public String id; + public String name; + public boolean deployable; + public boolean isPublic; + + @JsonSerialize(using = JacksonSerializers.LocalDateIsoSerializer.class) + @JsonDeserialize(using = JacksonSerializers.LocalDateIsoDeserializer.class) + public LocalDate lastUpdated; + + public String deployedFeedVersionId; + + @JsonSerialize(using = JacksonSerializers.LocalDateIsoSerializer.class) + @JsonDeserialize(using = JacksonSerializers.LocalDateIsoDeserializer.class) + public LocalDate deployedFeedVersionStartDate; + + @JsonSerialize(using = JacksonSerializers.LocalDateIsoSerializer.class) + @JsonDeserialize(using = JacksonSerializers.LocalDateIsoDeserializer.class) + public LocalDate deployedFeedVersionEndDate; + + public Integer deployedFeedVersionIssues; + + public String latestFeedVersionId; + + @JsonSerialize(using = JacksonSerializers.LocalDateIsoSerializer.class) + @JsonDeserialize(using = JacksonSerializers.LocalDateIsoDeserializer.class) + public LocalDate latestFeedVersionStartDate; + + @JsonSerialize(using = JacksonSerializers.LocalDateIsoSerializer.class) + @JsonDeserialize(using = JacksonSerializers.LocalDateIsoDeserializer.class) + public LocalDate latestFeedVersionEndDate; + + public Integer latestFeedVersionIssues; + + public FeedSourceSummary() { + } + + public FeedSourceSummary(Document feedSourceDocument) { + this.id = feedSourceDocument.getString("_id"); + this.name = feedSourceDocument.getString("name"); + this.deployable = feedSourceDocument.getBoolean("deployable"); + this.isPublic = feedSourceDocument.getBoolean("isPublic"); + // Convert to local date type for consistency. + this.lastUpdated = getLocalDateFromDate(feedSourceDocument.getDate("lastUpdated")); + } + + /** + * Set the appropriate feed version. For consistency, if no error count is available set the related number of + * issues to null. + */ + public void setFeedVersion(FeedVersionSummary feedVersionSummary, boolean isDeployed) { + if (feedVersionSummary != null) { + if (isDeployed) { + this.deployedFeedVersionId = feedVersionSummary.id; + this.deployedFeedVersionStartDate = feedVersionSummary.validationResult.firstCalendarDate; + this.deployedFeedVersionEndDate = feedVersionSummary.validationResult.lastCalendarDate; + this.deployedFeedVersionIssues = (feedVersionSummary.validationResult.errorCount == -1) + ? null + : feedVersionSummary.validationResult.errorCount; + } else { + this.latestFeedVersionId = feedVersionSummary.id; + this.latestFeedVersionStartDate = feedVersionSummary.validationResult.firstCalendarDate; + this.latestFeedVersionEndDate = feedVersionSummary.validationResult.lastCalendarDate; + this.latestFeedVersionIssues = (feedVersionSummary.validationResult.errorCount == -1) + ? null + : feedVersionSummary.validationResult.errorCount; + } + } + } + + /** + * Get all feed source summaries matching the project id. + */ + public static List getFeedSourceSummaries(String projectId) { + /* + db.getCollection('FeedSource').aggregate([ + { + // Match provided project id. + $match: { + projectId: "" + } + }, + { + $project: { + "_id": 1, + "name": 1, + "deployable": 1, + "isPublic": 1, + "lastUpdated": 1 + } + }, + { + $sort: { + "name": 1 + } + } + ]) + */ + List stages = Lists.newArrayList( + match( + in("projectId", projectId) + ), + project( + Projections.fields(Projections.include( + "_id", + "name", + "deployable", + "isPublic", + "lastUpdated") + ) + ), + sort(Sorts.ascending("name")) + ); + return extractFeedSourceSummaries(stages); + } + + /** + * Get the latest feed version from all feed sources for this project. + */ + public static Map getLatestFeedVersionForFeedSources(String projectId) { + /* + Note: To test this script: + 1) Comment out the call to tearDownDeployedFeedVersion() in FeedSourceControllerTest -> tearDown(). + 2) Run FeedSourceControllerTest to created required objects referenced here. + 3) Once complete, delete documents via MongoDB. + 4) Uncomment the call to tearDownDeployedFeedVersion() in FeedSourceControllerTest -> tearDown(). + 5) Re-run FeedSourceControllerTest to confirm deletion of objects. + + db.getCollection('FeedSource').aggregate([ + { + // Match provided project id. + $match: { + projectId: "project-with-latest-deployment" + } + }, + { + $lookup: { + from: "FeedVersion", + localField: "_id", + foreignField: "feedSourceId", + as: "feedVersions" + } + }, + { + $unwind: "$feedVersions" + }, + { + $group: { + _id: "$_id", + doc: { + $max: { + version: "$feedVersions.version", + feedVersionId: "$feedVersions._id", + firstCalendarDate: "$feedVersions.validationResult.firstCalendarDate", + lastCalendarDate: "$feedVersions.validationResult.lastCalendarDate", + issues: "$feedVersions.validationResult.errorCount" + } + } + } + } + ]) + */ + List stages = Lists.newArrayList( + match( + in("projectId", projectId) + ), + lookup("FeedVersion", "_id", "feedSourceId", "feedVersions"), + unwind("$feedVersions"), + group( + "$_id", + Accumulators.first("feedVersionId", "$feedVersions._id"), + Accumulators.first("firstCalendarDate", "$feedVersions.validationResult.firstCalendarDate"), + Accumulators.first("lastCalendarDate", "$feedVersions.validationResult.lastCalendarDate"), + Accumulators.first("errorCount", "$feedVersions.validationResult.errorCount") + ) + ); + return extractFeedVersionSummaries( + "FeedSource", + "feedVersionId", + "_id", + false, + stages); + } + + /** + * Get the deployed feed versions from the latest deployment for this project. + */ + public static Map getFeedVersionsFromLatestDeployment(String projectId) { + /* + Note: To test this script: + 1) Comment out the call to tearDownDeployedFeedVersion() in FeedSourceControllerTest -> tearDown(). + 2) Run FeedSourceControllerTest to created required objects referenced here. + 3) Once complete, delete documents via MongoDB. + 4) Uncomment the call to tearDownDeployedFeedVersion() in FeedSourceControllerTest -> tearDown(). + 5) Re-run FeedSourceControllerTest to confirm deletion of objects. + + db.getCollection('Project').aggregate([ + { + // Match provided project id. + $match: { + _id: "project-with-latest-deployment" + } + }, + { + // Get all deployments for this project. + $lookup:{ + from:"Deployment", + localField:"_id", + foreignField:"projectId", + as:"deployment" + } + }, + { + // Deconstruct deployments array to a document for each element. + $unwind: "$deployment" + }, + { + // Make the deployment documents the input/root document. + "$replaceRoot": { + "newRoot": "$deployment" + } + }, + { + // Sort descending. + $sort: { + lastUpdated : -1 + } + }, + { + // At this point we will have the latest deployment for a project. + $limit: 1 + }, + { + $lookup:{ + from:"FeedVersion", + localField:"feedVersionIds", + foreignField:"_id", + as:"feedVersions" + } + }, + { + // Deconstruct feedVersions array to a document for each element. + $unwind: "$feedVersions" + }, + { + // Make the feed version documents the input/root document. + "$replaceRoot": { + "newRoot": "$feedVersions" + } + }, + { + $project: { + "_id": 1, + "feedSourceId": 1, + "validationResult.firstCalendarDate": 1, + "validationResult.lastCalendarDate": 1, + "validationResult.errorCount": 1 + } + } + ]) + */ + List stages = Lists.newArrayList( + match( + in("_id", projectId) + ), + lookup("Deployment", "_id", "projectId", "deployments"), + unwind("$deployments"), + replaceRoot("$deployments"), + sort(Sorts.descending("lastUpdated")), + limit(1), + lookup("FeedVersion", "feedVersionIds", "_id", "feedVersions"), + unwind("$feedVersions"), + replaceRoot("$feedVersions"), + project( + Projections.fields(Projections.include( + "feedSourceId", + "validationResult.firstCalendarDate", + "validationResult.lastCalendarDate", + "validationResult.errorCount") + ) + ) + ); + return extractFeedVersionSummaries( + "Project", + "_id", + "feedSourceId", + true, + stages); + } + + /** + * Get the deployed feed version from the pinned deployment for this feed source. + */ + public static Map getFeedVersionsFromPinnedDeployment(String projectId) { + /* + Note: To test this script: + 1) Comment out the call to tearDownDeployedFeedVersion() in FeedSourceControllerTest -> tearDown(). + 2) Run FeedSourceControllerTest to created required objects referenced here. + 3) Once complete, delete documents via MongoDB. + 4) Uncomment the call to tearDownDeployedFeedVersion() in FeedSourceControllerTest -> tearDown(). + 5) Re-run FeedSourceControllerTest to confirm deletion of objects. + + db.getCollection('Project').aggregate([ + { + // Match provided project id. + $match: { + _id: "project-with-pinned-deployment" + } + }, + { + $project: { + pinnedDeploymentId: 1 + } + }, + { + $lookup:{ + from:"Deployment", + localField:"pinnedDeploymentId", + foreignField:"_id", + as:"deployment" + } + }, + { + $unwind: "$deployment" + }, + { + $lookup:{ + from:"FeedVersion", + localField:"deployment.feedVersionIds", + foreignField:"_id", + as:"feedVersions" + } + }, + { + // Deconstruct feedVersions array to a document for each element. + $unwind: "$feedVersions" + }, + { + // Make the feed version documents the input/root document. + "$replaceRoot": { + "newRoot": "$feedVersions" + } + }, + { + $project: { + "_id": 1, + "feedSourceId": 1, + "validationResult.firstCalendarDate": 1, + "validationResult.lastCalendarDate": 1, + "validationResult.errorCount": 1 + } + } + ]) + */ + + List stages = Lists.newArrayList( + match( + in("_id", projectId) + ), + project( + Projections.fields(Projections.include("pinnedDeploymentId")) + ), + lookup("Deployment", "pinnedDeploymentId", "_id", "deployment"), + unwind("$deployment"), + lookup("FeedVersion", "deployment.feedVersionIds", "_id", "feedVersions"), + unwind("$feedVersions"), + replaceRoot("$feedVersions"), + project( + Projections.fields(Projections.include( + "feedSourceId", + "validationResult.firstCalendarDate", + "validationResult.lastCalendarDate", + "validationResult.errorCount") + ) + ) + ); + return extractFeedVersionSummaries( + "Project", + "_id", + "feedSourceId", + true, + stages); + } + + + /** + * Produce a list of all feed source summaries for a project. + */ + private static List extractFeedSourceSummaries(List stages) { + List feedSourceSummaries = new ArrayList<>(); + for (Document feedSourceDocument : Persistence.getDocuments("FeedSource", stages)) { + feedSourceSummaries.add(new FeedSourceSummary(feedSourceDocument)); + } + return feedSourceSummaries; + } + + /** + * Extract feed version summaries from feed version documents. Each feed version is held against the matching feed + * source. + */ + private static Map extractFeedVersionSummaries( + String collection, + String feedVersionKey, + String feedSourceKey, + boolean hasChildValidationResultDocument, + List stages + ) { + Map feedVersionSummaries = new HashMap<>(); + for (Document feedVersionDocument : Persistence.getDocuments(collection, stages)) { + FeedVersionSummary feedVersionSummary = new FeedVersionSummary(); + feedVersionSummary.id = feedVersionDocument.getString(feedVersionKey); + feedVersionSummary.validationResult = getValidationResult(hasChildValidationResultDocument, feedVersionDocument); + feedVersionSummaries.put(feedVersionDocument.getString(feedSourceKey), feedVersionSummary); + } + return feedVersionSummaries; + } + + /** + * Build validation result from feed version document. + */ + private static ValidationResult getValidationResult(boolean hasChildValidationResultDocument, Document feedVersionDocument) { + ValidationResult validationResult = new ValidationResult(); + validationResult.errorCount = getValidationResultErrorCount(hasChildValidationResultDocument, feedVersionDocument); + validationResult.firstCalendarDate = getValidationResultDate(hasChildValidationResultDocument, feedVersionDocument, "firstCalendarDate"); + validationResult.lastCalendarDate = getValidationResultDate(hasChildValidationResultDocument, feedVersionDocument, "lastCalendarDate"); + return validationResult; + } + + private static LocalDate getValidationResultDate( + boolean hasChildValidationResultDocument, + Document feedVersionDocument, + String key + ) { + return (hasChildValidationResultDocument) + ? getDateFieldFromDocument(feedVersionDocument, key) + : getDateFromString(feedVersionDocument.getString(key)); + } + + /** + * Extract date value from validation result document. + */ + private static LocalDate getDateFieldFromDocument(Document document, String dateKey) { + Document validationResult = getDocumentChild(document, "validationResult"); + return (validationResult != null) + ? getDateFromString(validationResult.getString(dateKey)) + : null; + } + + /** + * Extract the error count from the parent document or child validation result document. If the error count is not + * available, return -1. + */ + private static int getValidationResultErrorCount(boolean hasChildValidationResultDocument, Document feedVersionDocument) { + int errorCount; + try { + errorCount = (hasChildValidationResultDocument) + ? getErrorCount(feedVersionDocument) + : feedVersionDocument.getInteger("errorCount"); + } catch (NullPointerException e) { + errorCount = -1; + } + return errorCount; + } + + /** + * Get the child validation result document and extract the error count from this. + */ + private static int getErrorCount(Document document) { + return getDocumentChild(document, "validationResult").getInteger("errorCount"); + } + + /** + * Extract child document matching provided name. + */ + private static Document getDocumentChild(Document document, String name) { + return (Document) document.get(name); + } + + /** + * Convert String date (if not null) into LocalDate. + */ + private static LocalDate getDateFromString(String date) { + return (date == null) ? null : LocalDate.parse(date, formatter); + } + + /** + * Convert Date object into LocalDate object. + */ + private static LocalDate getLocalDateFromDate(Date date) { + return date.toInstant().atZone(ZoneId.systemDefault()).toLocalDate(); + } +} \ No newline at end of file diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedVersionDeployed.java b/src/main/java/com/conveyal/datatools/manager/models/FeedVersionDeployed.java deleted file mode 100644 index 4fafc888d..000000000 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedVersionDeployed.java +++ /dev/null @@ -1,239 +0,0 @@ -package com.conveyal.datatools.manager.models; - -import com.conveyal.datatools.editor.utils.JacksonSerializers; -import com.conveyal.datatools.manager.persistence.Persistence; -import com.fasterxml.jackson.databind.annotation.JsonDeserialize; -import com.fasterxml.jackson.databind.annotation.JsonSerialize; -import com.google.common.collect.Lists; -import com.mongodb.client.model.Sorts; -import org.bson.Document; -import org.bson.conversions.Bson; - -import java.time.LocalDate; -import java.time.format.DateTimeFormatter; -import java.util.List; - -import static com.mongodb.client.model.Aggregates.limit; -import static com.mongodb.client.model.Aggregates.lookup; -import static com.mongodb.client.model.Aggregates.match; -import static com.mongodb.client.model.Aggregates.replaceRoot; -import static com.mongodb.client.model.Aggregates.sort; -import static com.mongodb.client.model.Aggregates.unwind; -import static com.mongodb.client.model.Filters.in; - -public class FeedVersionDeployed { - public String id; - - @JsonSerialize(using = JacksonSerializers.LocalDateIsoSerializer.class) - @JsonDeserialize(using = JacksonSerializers.LocalDateIsoDeserializer.class) - public LocalDate startDate; - - @JsonSerialize(using = JacksonSerializers.LocalDateIsoSerializer.class) - @JsonDeserialize(using = JacksonSerializers.LocalDateIsoDeserializer.class) - public LocalDate endDate; - - public FeedVersionDeployed() { - } - - public FeedVersionDeployed(Document feedVersionDocument) { - this.id = feedVersionDocument.getString("_id"); - DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyyMMdd"); - Document validationResult = (Document) feedVersionDocument.get("validationResult"); - if (validationResult != null) { - String first = validationResult.getString("firstCalendarDate"); - String last = validationResult.getString("lastCalendarDate"); - this.startDate = (first == null) ? null : LocalDate.parse(first, formatter); - this.endDate = (last == null) ? null : LocalDate.parse(last, formatter); - } - } - - /** - * Get the deployed feed version from the pinned deployment for this feed source. - */ - public static FeedVersionDeployed getFeedVersionFromPinnedDeployment(String projectId, String feedSourceId) { - /* - Note: To test this script: - 1) Comment out the call to tearDownDeployedFeedVersion() in FeedSourceControllerTest -> tearDown(). - 2) Run FeedSourceControllerTest to created required objects referenced here. - 3) Once complete, delete documents via MongoDB. - 4) Uncomment the call to tearDownDeployedFeedVersion() in FeedSourceControllerTest -> tearDown(). - 5) Re-run FeedSourceControllerTest to confirm deletion of objects. - - db.getCollection('Project').aggregate([ - { - $match: { - _id: "project-with-pinned-deployment" - } - }, - { - $lookup:{ - from:"Deployment", - localField:"pinnedDeploymentId", - foreignField:"_id", - as:"deployment" - } - }, - { - $lookup:{ - from:"FeedVersion", - localField:"deployment.feedVersionIds", - foreignField:"_id", - as:"feedVersions" - } - }, - { - $unwind: "$feedVersions" - }, - { - "$replaceRoot": { - "newRoot": "$feedVersions" - } - }, - { - $match: { - feedSourceId: "feed-source-with-pinned-deployment-feed-version" - } - }, - { - $sort: { - lastUpdated : -1 - } - }, - { - $limit: 1 - } - ]) - */ - List stages = Lists.newArrayList( - match( - in("_id", projectId) - ), - lookup("Deployment", "pinnedDeploymentId", "_id", "deployment"), - lookup("FeedVersion", "deployment.feedVersionIds", "_id", "feedVersions"), - unwind("$feedVersions"), - replaceRoot("$feedVersions"), - match( - in("feedSourceId", feedSourceId) - ), - // If more than one feed version for a feed source is held against a deployment the latest is used. - sort(Sorts.descending("lastUpdated")), - limit(1) - ); - return getFeedVersionDeployed(stages); - } - - /** - * Get the deployed feed version from the latest deployment for this feed source. - */ - public static FeedVersionDeployed getFeedVersionFromLatestDeployment(String projectId, String feedSourceId) { - /* - Note: To test this script: - 1) Comment out the call to tearDownDeployedFeedVersion() in FeedSourceControllerTest -> tearDown(). - 2) Run FeedSourceControllerTest to created required objects referenced here. - 3) Once complete, delete documents via MongoDB. - 4) Uncomment the call to tearDownDeployedFeedVersion() in FeedSourceControllerTest -> tearDown(). - 5) Re-run FeedSourceControllerTest to confirm deletion of objects. - - db.getCollection('Project').aggregate([ - { - // Match provided project id. - $match: { - _id: "project-with-latest-deployment" - } - }, - { - // Get all deployments for this project. - $lookup:{ - from:"Deployment", - localField:"_id", - foreignField:"projectId", - as:"deployments" - } - }, - { - // Deconstruct deployments array to a document for each element. - $unwind: "$deployments" - }, - { - // Make the deployment documents the input/root document. - "$replaceRoot": { - "newRoot": "$deployments" - } - }, - { - // Sort descending. - $sort: { - lastUpdated : -1 - } - }, - { - // At this point we will have the latest deployment for a project. - $limit: 1 - }, - { - // Get all feed versions that have been deployed as part of the latest deployment. - $lookup:{ - from:"FeedVersion", - localField:"feedVersionIds", - foreignField:"_id", - as:"feedVersions" - } - }, - { - // Deconstruct feedVersions array to a document for each element. - $unwind: "$feedVersions" - }, - { - // Make the feed version documents the input/root document. - "$replaceRoot": { - "newRoot": "$feedVersions" - } - }, - { - // Match the required feed source. - $match: { - feedSourceId: "feed-source-with-latest-deployment-feed-version" - } - }, - { - $sort: { - lastUpdated : -1 - } - }, - { - // At this point we will have the latest feed version from the latest deployment for a feed source. - $limit: 1 - } - ]) - */ - List stages = Lists.newArrayList( - match( - in("_id", projectId) - ), - lookup("Deployment", "_id", "projectId", "deployments"), - unwind("$deployments"), - replaceRoot("$deployments"), - sort(Sorts.descending("lastUpdated")), - limit(1), - lookup("FeedVersion", "feedVersionIds", "_id", "feedVersions"), - unwind("$feedVersions"), - replaceRoot("$feedVersions"), - match( - in("feedSourceId", feedSourceId) - ), - // If more than one feed version for a feed source is held against a deployment the latest is used. - sort(Sorts.descending("lastUpdated")), - limit(1) - ); - return getFeedVersionDeployed(stages); - } - - private static FeedVersionDeployed getFeedVersionDeployed(List stages) { - Document feedVersionDocument = Persistence - .getMongoDatabase() - .getCollection("Project") - .aggregate(stages) - .first(); - return (feedVersionDocument == null) ? null : new FeedVersionDeployed(feedVersionDocument); - } -} diff --git a/src/main/java/com/conveyal/datatools/manager/models/Project.java b/src/main/java/com/conveyal/datatools/manager/models/Project.java index ddd2298a2..b868c6b45 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/Project.java +++ b/src/main/java/com/conveyal/datatools/manager/models/Project.java @@ -17,6 +17,7 @@ import java.util.Date; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.stream.Collectors; @@ -164,6 +165,24 @@ public Collection retrieveDeploymentSummaries() { .collect(Collectors.toList()); } + /** + * Get all feed source summaries for this project. + */ + public Collection retrieveFeedSourceSummaries() { + List feedSourceSummaries = FeedSourceSummary.getFeedSourceSummaries(id); + Map latestFeedVersionForFeedSources = FeedSourceSummary.getLatestFeedVersionForFeedSources(id); + Map deployedFeedVersions = FeedSourceSummary.getFeedVersionsFromPinnedDeployment(id); + if (deployedFeedVersions.isEmpty()) { + // No pinned deployments, instead, get the deployed feed versions from the latest deployment. + deployedFeedVersions = FeedSourceSummary.getFeedVersionsFromLatestDeployment(id); + } + for (FeedSourceSummary feedSourceSummary : feedSourceSummaries) { + feedSourceSummary.setFeedVersion(latestFeedVersionForFeedSources.get(feedSourceSummary.id), false); + feedSourceSummary.setFeedVersion(deployedFeedVersions.get(feedSourceSummary.id), true); + } + return feedSourceSummaries; + } + // TODO: Does this need to be returned with JSON API response public Organization retrieveOrganization() { if (organizationId != null) { diff --git a/src/main/java/com/conveyal/datatools/manager/persistence/Persistence.java b/src/main/java/com/conveyal/datatools/manager/persistence/Persistence.java index 2bd79ef6f..e215c6742 100644 --- a/src/main/java/com/conveyal/datatools/manager/persistence/Persistence.java +++ b/src/main/java/com/conveyal/datatools/manager/persistence/Persistence.java @@ -18,15 +18,20 @@ import com.conveyal.datatools.manager.models.Snapshot; import com.mongodb.ConnectionString; import com.mongodb.MongoClientSettings; +import com.mongodb.client.AggregateIterable; import com.mongodb.client.MongoClient; import com.mongodb.client.MongoClients; import com.mongodb.client.MongoDatabase; +import org.bson.Document; import org.bson.codecs.configuration.CodecRegistries; import org.bson.codecs.configuration.CodecRegistry; import org.bson.codecs.pojo.PojoCodecProvider; +import org.bson.conversions.Bson; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.List; + import static com.conveyal.datatools.manager.DataManager.getConfigPropertyAsText; /** @@ -135,5 +140,12 @@ public static void initialize () { public static MongoDatabase getMongoDatabase() { return mongoDatabase; } - + + /** + * Get all bespoke documents matching query. These documents are tailored to the query response and are not tied + * directly to a persistence type. + */ + public static AggregateIterable getDocuments(String collection, List stages) { + return getMongoDatabase().getCollection(collection).aggregate(stages); + } } diff --git a/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java b/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java index e93b9baaf..a32474337 100644 --- a/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java +++ b/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java @@ -7,6 +7,7 @@ import com.conveyal.datatools.manager.models.Deployment; import com.conveyal.datatools.manager.models.FeedRetrievalMethod; import com.conveyal.datatools.manager.models.FeedSource; +import com.conveyal.datatools.manager.models.FeedSourceSummary; import com.conveyal.datatools.manager.models.FeedVersion; import com.conveyal.datatools.manager.models.FetchFrequency; import com.conveyal.datatools.manager.models.Label; @@ -323,46 +324,51 @@ public void createFeedSourceWithLabels() { void canRetrieveDeployedFeedVersionFromLatestDeployment() throws IOException { SimpleHttpResponse response = TestUtils.makeRequest( String.format( - "/api/manager/secure/feedsource/%s", - feedSourceWithLatestDeploymentFeedVersion.id + "/api/manager/secure/feedsourceSummaries?projectId=%s", + feedSourceWithLatestDeploymentFeedVersion.projectId ), null, HttpUtils.REQUEST_METHOD.GET ); assertEquals(OK_200, response.status); - FeedSource feedSource = - JsonUtil.getPOJOFromResponse( - response, - FeedSource.class + + List feedSourceSummaries = + JsonUtil.getPOJOFromJSONAsList( + JsonUtil.getJsonNodeFromResponse(response), + FeedSourceSummary.class ); - assertNotNull(feedSource); - assertEquals(feedSourceWithLatestDeploymentFeedVersion.id, feedSource.id); - assertEquals(feedVersionFromLatestDeployment.id, feedSource.getDeployedFeedVersionId()); - assertEquals(feedVersionFromLatestDeployment.validationSummary().startDate, feedSource.getDeployedFeedVersionStartDate()); - assertEquals(feedVersionFromLatestDeployment.validationSummary().endDate, feedSource.getDeployedFeedVersionEndDate()); + + assertNotNull(feedSourceSummaries); + assertEquals(feedSourceWithLatestDeploymentFeedVersion.id, feedSourceSummaries.get(0).id); + assertEquals(feedVersionFromLatestDeployment.id, feedSourceSummaries.get(0).deployedFeedVersionId); + assertEquals(feedVersionFromLatestDeployment.validationSummary().startDate, feedSourceSummaries.get(0).deployedFeedVersionStartDate); + assertEquals(feedVersionFromLatestDeployment.validationSummary().endDate, feedSourceSummaries.get(0).deployedFeedVersionEndDate); + assertEquals(feedVersionFromLatestDeployment.validationSummary().errorCount, feedSourceSummaries.get(0).deployedFeedVersionIssues); } @Test void canRetrieveDeployedFeedVersionFromPinnedDeployment() throws IOException { SimpleHttpResponse response = TestUtils.makeRequest( String.format( - "/api/manager/secure/feedsource/%s", - feedSourceWithPinnedDeploymentFeedVersion.id + "/api/manager/secure/feedsourceSummaries?projectId=%s", + feedSourceWithPinnedDeploymentFeedVersion.projectId ), null, HttpUtils.REQUEST_METHOD.GET ); assertEquals(OK_200, response.status); - FeedSource feedSource = - JsonUtil.getPOJOFromResponse( - response, - FeedSource.class + + List feedSourceSummaries = + JsonUtil.getPOJOFromJSONAsList( + JsonUtil.getJsonNodeFromResponse(response), + FeedSourceSummary.class ); - assertNotNull(feedSource); - assertEquals(feedSourceWithPinnedDeploymentFeedVersion.id, feedSource.id); - assertEquals(feedVersionFromPinnedDeployment.id, feedSource.getDeployedFeedVersionId()); - assertEquals(feedVersionFromPinnedDeployment.validationSummary().endDate, feedSource.getDeployedFeedVersionEndDate()); - assertEquals(feedVersionFromPinnedDeployment.validationSummary().startDate, feedSource.getDeployedFeedVersionStartDate()); + assertNotNull(feedSourceSummaries); + assertEquals(feedSourceWithPinnedDeploymentFeedVersion.id, feedSourceSummaries.get(0).id); + assertEquals(feedVersionFromPinnedDeployment.id, feedSourceSummaries.get(0).deployedFeedVersionId); + assertEquals(feedVersionFromPinnedDeployment.validationSummary().startDate, feedSourceSummaries.get(0).deployedFeedVersionStartDate); + assertEquals(feedVersionFromPinnedDeployment.validationSummary().endDate, feedSourceSummaries.get(0).deployedFeedVersionEndDate); + assertEquals(feedVersionFromPinnedDeployment.validationSummary().errorCount, feedSourceSummaries.get(0).deployedFeedVersionIssues); } private static FeedSource createFeedSource(String name, URL url, Project project) { @@ -421,6 +427,7 @@ private static FeedVersion createFeedVersion(String id, String feedSourceId, Loc ValidationResult validationResult = new ValidationResult(); validationResult.firstCalendarDate = startDate; validationResult.lastCalendarDate = endDate; + validationResult.errorCount = 5 + (int)(Math.random() * ((1000 - 5) + 1)); feedVersion.validationResult = validationResult; Persistence.feedVersions.create(feedVersion); return feedVersion; From ea9e41b4eb36d869a7352c54db39cc7d4a77d458 Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Wed, 3 May 2023 14:48:49 -0400 Subject: [PATCH 105/248] support merging gui and textbox router configs --- .../datatools/manager/models/Deployment.java | 40 +++++++++++++++---- 1 file changed, 33 insertions(+), 7 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/models/Deployment.java b/src/main/java/com/conveyal/datatools/manager/models/Deployment.java index 39b0d668e..9720b25a3 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/Deployment.java +++ b/src/main/java/com/conveyal/datatools/manager/models/Deployment.java @@ -4,7 +4,6 @@ import com.conveyal.datatools.common.utils.aws.CheckedAWSException; import com.conveyal.datatools.common.utils.aws.EC2Utils; import com.conveyal.datatools.manager.DataManager; -import static com.conveyal.datatools.manager.DataManager.getConfigPropertyAsText; import com.conveyal.datatools.manager.jobs.DeployJob; import com.conveyal.datatools.manager.persistence.Persistence; import com.conveyal.datatools.manager.utils.StringUtils; @@ -42,11 +41,14 @@ import java.util.Collection; import java.util.Collections; import java.util.Date; +import java.util.HashMap; import java.util.List; import java.util.Locale; +import java.util.Map; import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; +import static com.conveyal.datatools.manager.DataManager.getConfigPropertyAsText; import static com.mongodb.client.model.Filters.and; import static com.mongodb.client.model.Filters.eq; @@ -436,19 +438,43 @@ private String writeToString(O object) { } /** Generate router config for deployment as string. */ - public byte[] generateRouterConfig() { + public byte[] generateRouterConfig() throws IOException { Project project = this.parentProject(); - return customRouterConfig != null - ? customRouterConfig.getBytes(StandardCharsets.UTF_8) - : project.routerConfig != null + + byte[] customRouterConfigString = customRouterConfig != null + ? customRouterConfig.getBytes(StandardCharsets.UTF_8) : null; + + byte[] routerConfigString = project.routerConfig != null ? writeToBytes(project.routerConfig) : null; + + // If both router configs are present, merge the JSON before returning + // Merger code from: https://stackoverflow.com/questions/35747813/how-to-merge-two-json-strings-into-one-in-java + if (customRouterConfigString != null && routerConfigString != null) { + ObjectMapper mapper = new ObjectMapper(); + Map map1 = mapper.readValue(customRouterConfigString, Map.class); + Map map2 = mapper.readValue(routerConfigString, Map.class); + Map merged = new HashMap(map2); + merged.putAll(map1); + return mapper.writeValueAsString(merged).getBytes(); + } + + return customRouterConfigString != null + ? customRouterConfigString + : routerConfigString != null + ? routerConfigString + : null; } /** Generate router config for deployment as byte array (for writing to file output stream). */ public String generateRouterConfigAsString() { - if (customRouterConfig != null) return customRouterConfig; - return writeToString(this.parentProject().routerConfig); + try { + return new String(this.generateRouterConfig(), "UTF-8"); + } catch (IOException e) { + LOG.error("Failed to generate router config:"); + LOG.error(e.toString()); + return ""; + } } /** From 27aed32e8817d298c7420b83af4d5bd809d0b800 Mon Sep 17 00:00:00 2001 From: "philip.cline" Date: Fri, 5 May 2023 14:30:12 -0400 Subject: [PATCH 106/248] chore(GTFS-lib): update gtfs-lib --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 0e6a637b1..632b0d7da 100644 --- a/pom.xml +++ b/pom.xml @@ -268,7 +268,7 @@ com.github.conveyal gtfs-lib - bdb76ee + 3d79493 From ae40dddafc93724f5f45a7214cc15eb0137a016b Mon Sep 17 00:00:00 2001 From: "philip.cline" Date: Wed, 10 May 2023 14:59:00 -0400 Subject: [PATCH 107/248] feat(transformations): add preserve fields transformation --- .../models/transform/FeedTransformation.java | 9 +- .../PreserveCustomFieldsTransformation.java | 184 ++++++++++++++++++ 2 files changed, 189 insertions(+), 4 deletions(-) create mode 100644 src/main/java/com/conveyal/datatools/manager/models/transform/PreserveCustomFieldsTransformation.java diff --git a/src/main/java/com/conveyal/datatools/manager/models/transform/FeedTransformation.java b/src/main/java/com/conveyal/datatools/manager/models/transform/FeedTransformation.java index 0433fa457..221b26c67 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/transform/FeedTransformation.java +++ b/src/main/java/com/conveyal/datatools/manager/models/transform/FeedTransformation.java @@ -27,10 +27,11 @@ @JsonIgnoreProperties(ignoreUnknown = true) @JsonTypeInfo(use = JsonTypeInfo.Id.NAME) @JsonSubTypes({ - @JsonSubTypes.Type(value = DeleteRecordsTransformation.class, name = "DeleteRecordsTransformation"), - @JsonSubTypes.Type(value = NormalizeFieldTransformation.class, name = "NormalizeFieldTransformation"), - @JsonSubTypes.Type(value = ReplaceFileFromVersionTransformation.class, name = "ReplaceFileFromVersionTransformation"), - @JsonSubTypes.Type(value = ReplaceFileFromStringTransformation.class, name = "ReplaceFileFromStringTransformation") + @JsonSubTypes.Type(value = DeleteRecordsTransformation.class, name = "DeleteRecordsTransformation"), + @JsonSubTypes.Type(value = NormalizeFieldTransformation.class, name = "NormalizeFieldTransformation"), + @JsonSubTypes.Type(value = ReplaceFileFromVersionTransformation.class, name = "ReplaceFileFromVersionTransformation"), + @JsonSubTypes.Type(value = ReplaceFileFromStringTransformation.class, name = "ReplaceFileFromStringTransformation"), + @JsonSubTypes.Type(value = PreserveCustomFieldsTransformation.class, name = "PreserveCustomFieldsTransformation") }) public abstract class FeedTransformation implements Serializable { private static final long serialVersionUID = 1L; diff --git a/src/main/java/com/conveyal/datatools/manager/models/transform/PreserveCustomFieldsTransformation.java b/src/main/java/com/conveyal/datatools/manager/models/transform/PreserveCustomFieldsTransformation.java new file mode 100644 index 000000000..439837718 --- /dev/null +++ b/src/main/java/com/conveyal/datatools/manager/models/transform/PreserveCustomFieldsTransformation.java @@ -0,0 +1,184 @@ +package com.conveyal.datatools.manager.models.transform; + +import com.conveyal.datatools.common.status.MonitorableJob; +import com.conveyal.datatools.manager.models.TableTransformResult; +import com.conveyal.datatools.manager.models.TransformType; +import com.opencsv.CSVReader; +import com.opencsv.CSVWriter; +import com.opencsv.exceptions.CsvValidationException; +import org.apache.commons.io.input.BOMInputStream; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import com.conveyal.gtfs.loader.Table; + +import java.io.*; +import java.nio.charset.StandardCharsets; +import java.nio.file.*; +import java.util.*; +import java.util.stream.Collector; +import java.util.stream.Collectors; +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.text.WordUtils; + + + +/** + * This feed transformation will attempt to preserve any custom fields from an entered csv in the final GTFS output. + */ +public class PreserveCustomFieldsTransformation extends ZipTransformation { + private static final Logger LOG = LoggerFactory.getLogger(PreserveCustomFieldsTransformation.class); + private static List tablePrimaryKeys = new ArrayList<>(); + private static Table specTable; + /** no-arg constructor for de/serialization */ + public PreserveCustomFieldsTransformation() {} + + public static PreserveCustomFieldsTransformation create(String sourceVersionId, String table) { + PreserveCustomFieldsTransformation transformation = new PreserveCustomFieldsTransformation(); + transformation.sourceVersionId = sourceVersionId; + transformation.table = table; + return transformation; + } + + @Override + public void validateParameters(MonitorableJob.Status status) { + if (csvData == null) { + status.fail("CSV data must not be null (delete table not yet supported)"); + } + } + + public static Collector toSingleton() { + return Collectors.collectingAndThen( + Collectors.toList(), + list -> { + if (list.size() != 1) { + throw new IllegalStateException(); + } + return list.get(0); + } + ); + } + + private static HashMap createCsvHashMap(CSVReader reader, List primaryKeys) throws CsvValidationException, IOException { + HashMap lookup = new HashMap<>(); + + String[] nextLine; + while ((nextLine = reader.readNext()) != null) { + final String[] finalNextLine = nextLine; + List customCsvKeyValues = primaryKeys.stream().map(column -> finalNextLine[column]).collect(Collectors.toList()); + + // Concatenate keys to make a lookup hash and add to the hash map + String hashKey = StringUtils.join(customCsvKeyValues, "_"); + lookup.put(hashKey, finalNextLine); + } + + return lookup; + } + + private static void writeLine(CSVWriter writer, String[] row, List customFields, Map customHeaders, String[] customValues) { + // Add new custom fields to the editor csv rows + String[] newRow = Arrays.copyOf(row, row.length + customFields.size()); + if (customValues != null) { + // Write the custom values, if we have a match + for (int columnDiff = 0; columnDiff < customFields.size(); columnDiff++) { + String customField = customFields.get(columnDiff); + int customFieldIndex = customHeaders.get(customField); + newRow[row.length + columnDiff] = customValues[customFieldIndex]; + } + } + writer.writeNext(newRow); + } + + private static Map mapHeaderColumns(String[] headers) { + Map headersMapping = new HashMap<>(); + for (int i = 0; i < headers.length; i++) headersMapping.put(headers[i], i); + return headersMapping; + } + + private static String getClassNameFromTable (String table) { + String underscoreRemoved = table.replace("_", " "); + String capitalized = WordUtils.capitalize(underscoreRemoved); + String oneWordName = capitalized.replace(" ", ""); + if (oneWordName.substring(oneWordName.length() - 1).equals("s")) { + oneWordName = oneWordName.substring(0, oneWordName.length() - 1); + } + return "com.conveyal.model." + oneWordName + "DTO"; + } + + @Override + public void transform(FeedTransformZipTarget zipTarget, MonitorableJob.Status status) { + String tableName = table + ".txt"; + Path targetZipPath = Paths.get(zipTarget.gtfsFile.getAbsolutePath()); + // Try to dynamically load the class for CSVBean + // String csvDataClassName = getClassNameFromTable(table); +// Class csvDataClass = Class.forName(csvDataClassName); + + // TODO: is there a better way to do this than using a Singleton collector? + specTable = Arrays.stream(Table.tablesInOrder) + .filter(t -> t.name.equals(table)) + .collect(toSingleton()); + tablePrimaryKeys = specTable.getPrimaryKeyNames(); + + try( FileSystem targetZipFs = FileSystems.newFileSystem(targetZipPath, (ClassLoader) null) ){ + List specTableFields = specTable.specFields().stream().map(f -> f.name).collect(Collectors.toList()); + Path targetTxtFilePath = getTablePathInZip(tableName, targetZipFs); + + // TODO: There must be a better way to do this. + InputStream is = Files.newInputStream(targetTxtFilePath); + final File tempFile = File.createTempFile(tableName + "-temp", ".txt"); + File output = File.createTempFile(tableName + "-output-temp", ".txt"); + FileOutputStream out = new FileOutputStream(tempFile); + IOUtils.copy(is, out); + + FileInputStream fileInputStream = new FileInputStream(tempFile); + // BOMInputStream to avoid any Byte Order Marks at the start of files. + CSVReader editorFileReader = new CSVReader(new InputStreamReader(new BOMInputStream(fileInputStream), StandardCharsets.UTF_8)); + CSVReader customFileReader = new CSVReader(new StringReader(csvData)); + + // Store the headers with their indices in CSV for later lookups + String[] customHeaders = customFileReader.readNext(); + String[] editorHeaders = editorFileReader.readNext(); + Map customCsvHeaders = mapHeaderColumns(customHeaders); + Map editorCsvHeaders = mapHeaderColumns(editorHeaders); + + // Find the customFields in the input csv + List customFields = Arrays.stream(customHeaders).filter(h -> !specTableFields.contains(h)).collect(Collectors.toList()); + if (customFields.size() == 0) return; + + // Find the key columns in the custom CSV + List customCsvKeyColumns = tablePrimaryKeys.stream() + .map(customCsvHeaders::get) + .collect(Collectors.toList()); + + HashMap lookup = createCsvHashMap(customFileReader, customCsvKeyColumns); + CSVWriter writer = new CSVWriter(new FileWriter(output)); + writeLine(writer, editorHeaders, customFields, customCsvHeaders, customHeaders); // Write headers before starting lookups + + String[] nextLine; + while((nextLine = editorFileReader.readNext()) != null) { + String[] finalNextLine = nextLine; // TODO: there must be some way around this. + List editorCsvPrimaryKeyValues = tablePrimaryKeys.stream() + .map(key -> finalNextLine[editorCsvHeaders.get(key)]) + .collect(Collectors.toList()); // Map the keys to the values for the row + + String hashKey = StringUtils.join(editorCsvPrimaryKeyValues, "_"); + String[] customCsvLine = lookup.get(hashKey); + writeLine(writer, nextLine, customFields, customCsvHeaders, customCsvLine); + } + writer.close(); + + Files.copy(output.toPath(), targetTxtFilePath, StandardCopyOption.REPLACE_EXISTING); + tempFile.deleteOnExit(); + output.deleteOnExit(); + zipTarget.feedTransformResult.tableTransformResults.add(new TableTransformResult(tableName, TransformType.TABLE_MODIFIED)); + } catch (NoSuchFileException e) { + status.fail("Source version does not contain table: " + tableName, e); + } catch(IOException e) { + status.fail("An exception occurred when writing output with custom fields", e); + } catch (CsvValidationException ex) { + ex.printStackTrace(); + } catch (Exception e) { + status.fail("Unknown error encountered while transforming zip file", e); + } + } +} From e831764dfb5cf55be2bd50b9eea99e44f9da5296 Mon Sep 17 00:00:00 2001 From: "philip.cline" Date: Wed, 10 May 2023 16:22:23 -0400 Subject: [PATCH 108/248] refactor(transformations): update gtfs-lib to include Table class changes. --- pom.xml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 632b0d7da..dd972bbe4 100644 --- a/pom.xml +++ b/pom.xml @@ -268,7 +268,7 @@ com.github.conveyal gtfs-lib - 3d79493 + 41a6503 @@ -414,6 +414,14 @@ snakeyaml 1.26 + + + + com.opencsv + opencsv + 5.7.1 + + - 3d79493 + a3e5707 From 7084e51eec7eab514b20abb79400c30c4f6b2782 Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Wed, 17 May 2023 16:01:38 +0100 Subject: [PATCH 114/248] refactor(Updated scope): Now deleted stop and related stop times --- .../controllers/api/EditorController.java | 48 ++++++++++++------- .../controllers/api/EditorControllerTest.java | 46 +++++++++++------- 2 files changed, 60 insertions(+), 34 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/editor/controllers/api/EditorController.java b/src/main/java/com/conveyal/datatools/editor/controllers/api/EditorController.java index 8629607d7..a253fbb6b 100644 --- a/src/main/java/com/conveyal/datatools/editor/controllers/api/EditorController.java +++ b/src/main/java/com/conveyal/datatools/editor/controllers/api/EditorController.java @@ -134,7 +134,7 @@ private void registerRoutes() { } if ("stop".equals(classToLowercase)) { - patch(ROOT_ROUTE + "/deletefrompatternstops", this::deleteStopFromPatternStops, json::write); + patch(ROOT_ROUTE + "/cascadeDeleteStop", this::cascadeDeleteStop, json::write); } } @@ -266,12 +266,16 @@ private String deleteTripsForPattern(Request req, Response res) { } /** - * HTTP endpoint to delete a stop from all pattern stops given a string stopId (i.e. not the integer ID field). - * Then normalize the stop times for all updated patterns (i.e. the ones where the stop has been deleted). + * HTTP endpoint to delete a stop and all references in stop times and pattern stops given a string stopId (i.e. not + * the integer ID field). Then normalize the stop times for all updated patterns (i.e. the ones where the stop has + * been deleted). */ - private String deleteStopFromPatternStops(Request req, Response res) { + private String cascadeDeleteStop(Request req, Response res) { + // Table writer closes the database connection after use, so a new one is required for each task. + JdbcTableWriter tableWriter; long startTime = System.currentTimeMillis(); String namespace = getNamespaceAndValidateSession(req); + String stopIdColumnName = "stop_id"; // NOTE: This is a string stop ID, not the integer ID that all other HTTP endpoints use. String stopId = req.queryParams("stopId"); @@ -282,7 +286,7 @@ private String deleteStopFromPatternStops(Request req, Response res) { try ( Connection connection = datasource.getConnection(); PreparedStatement statement = connection.prepareStatement( - String.format("select pattern_id, stop_sequence from %s.pattern_stops where stop_id = ?", namespace) + String.format("select id, stop_sequence from %s.pattern_stops where %s = ?", namespace, stopIdColumnName) ) ) { // Get the patterns to be normalized before the related stop is deleted. @@ -291,28 +295,40 @@ private String deleteStopFromPatternStops(Request req, Response res) { Map patternsToBeNormalized = new HashMap<>(); while (resultSet.next()) { patternsToBeNormalized.put( - resultSet.getInt("pattern_id"), + resultSet.getInt("id"), resultSet.getInt("stop_sequence") ); } - int deletedCount = 0; - // Table writer closes the database connection after use, so a new one is required for each task. - JdbcTableWriter tableWriter; + tableWriter = new JdbcTableWriter(Table.STOP_TIMES, datasource, namespace); + int deletedCountStopTimes = tableWriter.deleteWhere(stopIdColumnName, stopId, true); + + int deletedCountPatternStops = 0; if (!patternsToBeNormalized.isEmpty()) { tableWriter = new JdbcTableWriter(Table.PATTERN_STOP, datasource, namespace); - deletedCount = tableWriter.deleteWhere("stop_id", stopId, true); - if (deletedCount > 0) { - for (Map.Entry patternId : patternsToBeNormalized.entrySet()) { + deletedCountPatternStops = tableWriter.deleteWhere(stopIdColumnName, stopId, true); + if (deletedCountPatternStops > 0) { + for (Map.Entry patternStop : patternsToBeNormalized.entrySet()) { tableWriter = new JdbcTableWriter(Table.PATTERN_STOP, datasource, namespace); - int stopSequence = patternId.getValue(); + int stopSequence = patternStop.getValue(); // Begin with the stop prior to the one deleted, unless at the beginning. int beginWithSequence = (stopSequence != 0) ? stopSequence - 1 : stopSequence; - tableWriter.normalizeStopTimesForPattern(patternId.getKey(), beginWithSequence); + tableWriter.normalizeStopTimesForPattern(patternStop.getKey(), beginWithSequence); } } } - return formatJSON(String.format("Deleted %d.", deletedCount), OK_200); + + tableWriter = new JdbcTableWriter(Table.STOPS, datasource, namespace); + int deletedCountStop = tableWriter.deleteWhere(stopIdColumnName, stopId, true); + + return formatJSON( + String.format( + "Deleted %d stop, %d pattern stops and %d stop times.", + deletedCountStop, + deletedCountPatternStops, + deletedCountStopTimes), + OK_200 + ); } catch (InvalidNamespaceException e) { logMessageAndHalt(req, 400, "Invalid namespace.", e); return null; @@ -320,7 +336,7 @@ private String deleteStopFromPatternStops(Request req, Response res) { logMessageAndHalt(req, 500, "Error deleting entity.", e); return null; } finally { - LOG.info("Delete stop from pattern stops operation took {} msec.", System.currentTimeMillis() - startTime); + LOG.info("Cascade delete of stop operation took {} msec.", System.currentTimeMillis() - startTime); } } diff --git a/src/test/java/com/conveyal/datatools/editor/controllers/api/EditorControllerTest.java b/src/test/java/com/conveyal/datatools/editor/controllers/api/EditorControllerTest.java index 73a945a79..59c0a2d47 100644 --- a/src/test/java/com/conveyal/datatools/editor/controllers/api/EditorControllerTest.java +++ b/src/test/java/com/conveyal/datatools/editor/controllers/api/EditorControllerTest.java @@ -151,22 +151,35 @@ public void canPatchStopsConditionally() throws IOException { * Test the removal of a stop from stop patterns. */ @Test - void canRemoveStopFromPatternStops() throws IOException, SQLException { + void canRemoveStopFromStopTimesAndPatternStops() throws IOException, SQLException { // Get a fresh feed source so that the editor namespace was updated after snapshot. FeedSource freshFeedSource = Persistence.feedSources.getById(feedVersion.feedSourceId); String stopId = "WARM"; - // Check for presence of stopId in pattern stops. - assertThatSqlCountQueryYieldsExpectedCount( - String.format( - "SELECT count(*) FROM %s.pattern_stops WHERE stop_id = '%s'", - freshFeedSource.editorNamespace, - stopId - ), - 4 + String stopCountSql = String.format( + "SELECT count(*) FROM %s.stops WHERE stop_id = '%s'", + freshFeedSource.editorNamespace, + stopId + ); + String stopTimesCountSql = String.format( + "SELECT count(*) FROM %s.stop_times WHERE stop_id = '%s'", + freshFeedSource.editorNamespace, + stopId + ); + String patternStopsCountSql = String.format( + "SELECT count(*) FROM %s.pattern_stops WHERE stop_id = '%s'", + freshFeedSource.editorNamespace, + stopId ); + // Check for presence of stopId in stops. + assertThatSqlCountQueryYieldsExpectedCount(stopCountSql ,1); + // Check for presence of stopId in stop times. + assertThatSqlCountQueryYieldsExpectedCount(stopTimesCountSql ,522); + // Check for presence of stopId in pattern stops. + assertThatSqlCountQueryYieldsExpectedCount(patternStopsCountSql, 4); + String path = String.format( - "/api/editor/secure/stop/deletefrompatternstops?stopId=%s&feedId=%s&sessionId=test", + "/api/editor/secure/stop/cascadeDeleteStop?stopId=%s&feedId=%s&sessionId=test", stopId, feedVersion.feedSourceId ); @@ -180,15 +193,12 @@ void canRemoveStopFromPatternStops() throws IOException, SQLException { JsonNode json = mapper.readTree(response); assertEquals(OK_200, json.get("code").asInt()); + // Check for removal of stopId in stops. + assertThatSqlCountQueryYieldsExpectedCount(stopCountSql ,0); + // Check for removal of stopId in stop times. + assertThatSqlCountQueryYieldsExpectedCount(stopTimesCountSql ,0); // Check for removal of stopId in pattern stops. - assertThatSqlCountQueryYieldsExpectedCount( - String.format( - "SELECT count(*) FROM %s.pattern_stops WHERE stop_id = '%s'", - freshFeedSource.editorNamespace, - stopId - ), - 0 - ); + assertThatSqlCountQueryYieldsExpectedCount(patternStopsCountSql, 0); } /** From e8349d071a398c2980aa83a4fed7dbd2e476e859 Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Thu, 18 May 2023 14:41:51 -0400 Subject: [PATCH 115/248] fix: support transforming non-existent fields --- .../NormalizeFieldTransformation.java | 27 +++++++++++++++---- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/models/transform/NormalizeFieldTransformation.java b/src/main/java/com/conveyal/datatools/manager/models/transform/NormalizeFieldTransformation.java index 783828c6c..661013ab3 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/transform/NormalizeFieldTransformation.java +++ b/src/main/java/com/conveyal/datatools/manager/models/transform/NormalizeFieldTransformation.java @@ -198,10 +198,18 @@ public void transform(FeedTransformZipTarget zipTarget, MonitorableJob.Status st Field[] fieldsFoundInZip = gtfsTable.getFieldsFromFieldHeaders(headers, null); int transformFieldIndex = getFieldIndex(fieldsFoundInZip, fieldName); + // If the index is -1, this is a new column, and we need to add it accordingly + if (transformFieldIndex == -1) { + String[] expandedHeaders = new String[headers.length + 1]; + System.arraycopy(headers, 0, expandedHeaders, 0, headers.length); + expandedHeaders[headers.length] = fieldName; + writer.write(expandedHeaders); + } else { + writer.write(headers); + } + int modifiedRowCount = 0; - // Write headers and processed CSV rows. - writer.write(headers); while (csvReader.readRecord()) { String originalValue = csvReader.get(transformFieldIndex); String transformedValue = originalValue; @@ -219,10 +227,19 @@ public void transform(FeedTransformZipTarget zipTarget, MonitorableJob.Status st // Re-assemble the CSV line and place in buffer. String[] csvValues = csvReader.getValues(); - csvValues[transformFieldIndex] = transformedValue; - // Write line to table (plus new line char). - writer.write(csvValues); + // If the index is -1, this is a new column, and we need to add it accordingly + if (transformFieldIndex == -1) { + String[] expandedCsvValues = new String[headers.length + 1]; + System.arraycopy(csvValues, 0, expandedCsvValues, 0, csvValues.length); + expandedCsvValues[csvValues.length] = transformedValue; + writer.write(expandedCsvValues); + } else { + csvValues[transformFieldIndex] = transformedValue; + + // Write line to table (plus new line char). + writer.write(csvValues); + } // Count number of CSV rows changed. if (!originalValue.equals(transformedValue)) { From af5b89c219fb2a09c1792b075444518b1acf5600 Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Fri, 19 May 2023 10:28:27 +0100 Subject: [PATCH 116/248] refactor(Addressed PR feedback): End point type is now delete --- .../controllers/api/EditorController.java | 8 ++-- .../controllers/api/EditorControllerTest.java | 46 +++++++++---------- 2 files changed, 25 insertions(+), 29 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/editor/controllers/api/EditorController.java b/src/main/java/com/conveyal/datatools/editor/controllers/api/EditorController.java index a253fbb6b..bd50dff8f 100644 --- a/src/main/java/com/conveyal/datatools/editor/controllers/api/EditorController.java +++ b/src/main/java/com/conveyal/datatools/editor/controllers/api/EditorController.java @@ -134,7 +134,7 @@ private void registerRoutes() { } if ("stop".equals(classToLowercase)) { - patch(ROOT_ROUTE + "/cascadeDeleteStop", this::cascadeDeleteStop, json::write); + delete(ROOT_ROUTE + ID_PARAM + "/cascadeDeleteStop", this::cascadeDeleteStop, json::write); } } @@ -266,7 +266,7 @@ private String deleteTripsForPattern(Request req, Response res) { } /** - * HTTP endpoint to delete a stop and all references in stop times and pattern stops given a string stopId (i.e. not + * HTTP endpoint to delete a stop and all references in stop times and pattern stops given a string stop_id (i.e. not * the integer ID field). Then normalize the stop times for all updated patterns (i.e. the ones where the stop has * been deleted). */ @@ -277,8 +277,8 @@ private String cascadeDeleteStop(Request req, Response res) { String namespace = getNamespaceAndValidateSession(req); String stopIdColumnName = "stop_id"; - // NOTE: This is a string stop ID, not the integer ID that all other HTTP endpoints use. - String stopId = req.queryParams("stopId"); + // NOTE: This is a string stop ID, not the integer ID that other HTTP endpoints use. + String stopId = req.params("id"); if (stopId == null) { logMessageAndHalt(req, 400, "Must provide a valid stopId."); } diff --git a/src/test/java/com/conveyal/datatools/editor/controllers/api/EditorControllerTest.java b/src/test/java/com/conveyal/datatools/editor/controllers/api/EditorControllerTest.java index 59c0a2d47..8f7bf9199 100644 --- a/src/test/java/com/conveyal/datatools/editor/controllers/api/EditorControllerTest.java +++ b/src/test/java/com/conveyal/datatools/editor/controllers/api/EditorControllerTest.java @@ -148,44 +148,30 @@ public void canPatchStopsConditionally() throws IOException { } /** - * Test the removal of a stop from stop patterns. + * Test the removal of a stop and all references in stop times and pattern stops. */ @Test - void canRemoveStopFromStopTimesAndPatternStops() throws IOException, SQLException { + void canCascadeDeleteStop() throws IOException, SQLException { // Get a fresh feed source so that the editor namespace was updated after snapshot. FeedSource freshFeedSource = Persistence.feedSources.getById(feedVersion.feedSourceId); String stopId = "WARM"; - String stopCountSql = String.format( - "SELECT count(*) FROM %s.stops WHERE stop_id = '%s'", - freshFeedSource.editorNamespace, - stopId - ); - String stopTimesCountSql = String.format( - "SELECT count(*) FROM %s.stop_times WHERE stop_id = '%s'", - freshFeedSource.editorNamespace, - stopId - ); - String patternStopsCountSql = String.format( - "SELECT count(*) FROM %s.pattern_stops WHERE stop_id = '%s'", - freshFeedSource.editorNamespace, - stopId - ); + String stopCountSql = getCountSql(freshFeedSource.editorNamespace, "stops", stopId); + String stopTimesCountSql = getCountSql(freshFeedSource.editorNamespace, "stop_times", stopId); + String patternStopsCountSql = getCountSql(freshFeedSource.editorNamespace, "pattern_stops", stopId); - // Check for presence of stopId in stops. + // Check for presence of stopId in stops, stop times and pattern stops. assertThatSqlCountQueryYieldsExpectedCount(stopCountSql ,1); - // Check for presence of stopId in stop times. assertThatSqlCountQueryYieldsExpectedCount(stopTimesCountSql ,522); - // Check for presence of stopId in pattern stops. assertThatSqlCountQueryYieldsExpectedCount(patternStopsCountSql, 4); String path = String.format( - "/api/editor/secure/stop/cascadeDeleteStop?stopId=%s&feedId=%s&sessionId=test", + "/api/editor/secure/stop/%s/cascadeDeleteStop?feedId=%s&sessionId=test", stopId, feedVersion.feedSourceId ); String response = given() .port(DataManager.PORT) - .patch(path) + .delete(path) .then() .extract() .response() @@ -193,11 +179,9 @@ void canRemoveStopFromStopTimesAndPatternStops() throws IOException, SQLExceptio JsonNode json = mapper.readTree(response); assertEquals(OK_200, json.get("code").asInt()); - // Check for removal of stopId in stops. + // Check for removal of stopId in stops, stop times and pattern stops. assertThatSqlCountQueryYieldsExpectedCount(stopCountSql ,0); - // Check for removal of stopId in stop times. assertThatSqlCountQueryYieldsExpectedCount(stopTimesCountSql ,0); - // Check for removal of stopId in pattern stops. assertThatSqlCountQueryYieldsExpectedCount(patternStopsCountSql, 0); } @@ -240,4 +224,16 @@ private static JsonNode graphqlQuery (String namespace, String graphQLQueryFile) .asString(); return mapper.readTree(graphQLString); } + + /** + * Build a sql statement to provide a count on the number of rows matching the stop id. + */ + private static String getCountSql(String namespace, String tableName, String stopId) { + return String.format( + "SELECT count(*) FROM %s.%s WHERE stop_id = '%s'", + namespace, + tableName, + stopId + ); + } } From 59e1cd80ee161dd6da3d669f4449002dc953a1ab Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Fri, 19 May 2023 10:38:32 +0100 Subject: [PATCH 117/248] refactor(EditorControllerTest.java): Corrected formatting. --- .../editor/controllers/api/EditorControllerTest.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/test/java/com/conveyal/datatools/editor/controllers/api/EditorControllerTest.java b/src/test/java/com/conveyal/datatools/editor/controllers/api/EditorControllerTest.java index 8f7bf9199..8512cf941 100644 --- a/src/test/java/com/conveyal/datatools/editor/controllers/api/EditorControllerTest.java +++ b/src/test/java/com/conveyal/datatools/editor/controllers/api/EditorControllerTest.java @@ -160,8 +160,8 @@ void canCascadeDeleteStop() throws IOException, SQLException { String patternStopsCountSql = getCountSql(freshFeedSource.editorNamespace, "pattern_stops", stopId); // Check for presence of stopId in stops, stop times and pattern stops. - assertThatSqlCountQueryYieldsExpectedCount(stopCountSql ,1); - assertThatSqlCountQueryYieldsExpectedCount(stopTimesCountSql ,522); + assertThatSqlCountQueryYieldsExpectedCount(stopCountSql, 1); + assertThatSqlCountQueryYieldsExpectedCount(stopTimesCountSql, 522); assertThatSqlCountQueryYieldsExpectedCount(patternStopsCountSql, 4); String path = String.format( @@ -180,8 +180,8 @@ void canCascadeDeleteStop() throws IOException, SQLException { assertEquals(OK_200, json.get("code").asInt()); // Check for removal of stopId in stops, stop times and pattern stops. - assertThatSqlCountQueryYieldsExpectedCount(stopCountSql ,0); - assertThatSqlCountQueryYieldsExpectedCount(stopTimesCountSql ,0); + assertThatSqlCountQueryYieldsExpectedCount(stopCountSql, 0); + assertThatSqlCountQueryYieldsExpectedCount(stopTimesCountSql, 0); assertThatSqlCountQueryYieldsExpectedCount(patternStopsCountSql, 0); } From 0751bdf37da809e1f9ade79dafbe1042d3e0ee2f Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Fri, 19 May 2023 11:54:47 +0100 Subject: [PATCH 118/248] refactor(EditorControllerTest.java): Created a distinct feed version for the cascade delete stop tes Deleting a stop from the previously shared feed version was causing another test to fail. --- .../controllers/api/EditorControllerTest.java | 36 ++++++++++++++----- 1 file changed, 27 insertions(+), 9 deletions(-) diff --git a/src/test/java/com/conveyal/datatools/editor/controllers/api/EditorControllerTest.java b/src/test/java/com/conveyal/datatools/editor/controllers/api/EditorControllerTest.java index 8512cf941..45a658b3f 100644 --- a/src/test/java/com/conveyal/datatools/editor/controllers/api/EditorControllerTest.java +++ b/src/test/java/com/conveyal/datatools/editor/controllers/api/EditorControllerTest.java @@ -44,7 +44,9 @@ public class EditorControllerTest extends UnitTest { private static final Logger LOG = LoggerFactory.getLogger(EditorControllerTest.class); private static Project project; private static FeedSource feedSource; + private static FeedSource feedSourceCascadeDelete; private static FeedVersion feedVersion; + private static FeedVersion feedVersionCascadeDelete; private static final ObjectMapper mapper = new ObjectMapper(); /** @@ -59,18 +61,23 @@ public static void setUp() throws Exception { UserController.setBaseUsersUrl("http://" + TEST_AUTH0_DOMAIN + USERS_API_PATH); // Create a project, feed sources, and feed versions to merge. project = new Project(); - project.name = String.format("Test %s", new Date().toString()); + project.name = String.format("Test %s", new Date()); Persistence.projects.create(project); + feedSource = new FeedSource("BART"); feedSource.projectId = project.id; Persistence.feedSources.create(feedSource); + + feedSourceCascadeDelete = new FeedSource("CASCADE_DELETE"); + feedSourceCascadeDelete.projectId = project.id; + Persistence.feedSources.create(feedSourceCascadeDelete); + feedVersion = createFeedVersionFromGtfsZip(feedSource, "bart_old.zip"); - // Create and run snapshot job - Snapshot snapshot = new Snapshot("Snapshot of " + feedVersion.name, feedSource.id, feedVersion.namespace); - CreateSnapshotJob createSnapshotJob = - new CreateSnapshotJob(Auth0UserProfile.createTestAdminUser(), snapshot, true, false, false); - // Run in current thread so tests do not run until this is complete. - createSnapshotJob.run(); + feedVersionCascadeDelete = createFeedVersionFromGtfsZip(feedSourceCascadeDelete, "bart_old.zip"); + + // Create and run snapshot jobs + crateAndRunSnapshotJob(feedVersion.name, feedSource.id, feedVersion.namespace); + crateAndRunSnapshotJob(feedVersionCascadeDelete.name, feedSourceCascadeDelete.id, feedVersionCascadeDelete.namespace); LOG.info("{} setup completed in {} ms", EditorControllerTest.class.getSimpleName(), System.currentTimeMillis() - startTime); } @@ -78,6 +85,17 @@ public static void setUp() throws Exception { public static void tearDown() { project.delete(); feedSource.delete(); + feedSourceCascadeDelete.delete(); + } + + /** + * Create and run a snapshot job in the current thread (so tests do not run until this is complete). + */ + private static void crateAndRunSnapshotJob(String feedVersionName, String feedSourceId, String namespace) { + Snapshot snapshot = new Snapshot("Snapshot of " + feedVersionName, feedSourceId, namespace); + CreateSnapshotJob createSnapshotJob = + new CreateSnapshotJob(Auth0UserProfile.createTestAdminUser(), snapshot, true, false, false); + createSnapshotJob.run(); } private static Stream createPatchTableTests() { @@ -153,7 +171,7 @@ public void canPatchStopsConditionally() throws IOException { @Test void canCascadeDeleteStop() throws IOException, SQLException { // Get a fresh feed source so that the editor namespace was updated after snapshot. - FeedSource freshFeedSource = Persistence.feedSources.getById(feedVersion.feedSourceId); + FeedSource freshFeedSource = Persistence.feedSources.getById(feedVersionCascadeDelete.feedSourceId); String stopId = "WARM"; String stopCountSql = getCountSql(freshFeedSource.editorNamespace, "stops", stopId); String stopTimesCountSql = getCountSql(freshFeedSource.editorNamespace, "stop_times", stopId); @@ -167,7 +185,7 @@ void canCascadeDeleteStop() throws IOException, SQLException { String path = String.format( "/api/editor/secure/stop/%s/cascadeDeleteStop?feedId=%s&sessionId=test", stopId, - feedVersion.feedSourceId + feedVersionCascadeDelete.feedSourceId ); String response = given() .port(DataManager.PORT) From daa7eacbb350d80b31a9fc45275f3b69ec770eb0 Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Mon, 22 May 2023 10:10:47 -0400 Subject: [PATCH 119/248] refactor: address pr feedback --- .../NormalizeFieldTransformation.java | 4 ++-- .../NormalizeFieldTransformationTest.java | 18 ++++++++++++++++++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/models/transform/NormalizeFieldTransformation.java b/src/main/java/com/conveyal/datatools/manager/models/transform/NormalizeFieldTransformation.java index 661013ab3..bc24a7e1f 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/transform/NormalizeFieldTransformation.java +++ b/src/main/java/com/conveyal/datatools/manager/models/transform/NormalizeFieldTransformation.java @@ -198,7 +198,7 @@ public void transform(FeedTransformZipTarget zipTarget, MonitorableJob.Status st Field[] fieldsFoundInZip = gtfsTable.getFieldsFromFieldHeaders(headers, null); int transformFieldIndex = getFieldIndex(fieldsFoundInZip, fieldName); - // If the index is -1, this is a new column, and we need to add it accordingly + // If the index is -1, this is a new column, and we need to add it accordingly. if (transformFieldIndex == -1) { String[] expandedHeaders = new String[headers.length + 1]; System.arraycopy(headers, 0, expandedHeaders, 0, headers.length); @@ -228,7 +228,7 @@ public void transform(FeedTransformZipTarget zipTarget, MonitorableJob.Status st // Re-assemble the CSV line and place in buffer. String[] csvValues = csvReader.getValues(); - // If the index is -1, this is a new column, and we need to add it accordingly + // If the index is -1, this is a new column, and we need to add it accordingly. if (transformFieldIndex == -1) { String[] expandedCsvValues = new String[headers.length + 1]; System.arraycopy(csvValues, 0, expandedCsvValues, 0, csvValues.length); diff --git a/src/test/java/com/conveyal/datatools/manager/models/transform/NormalizeFieldTransformationTest.java b/src/test/java/com/conveyal/datatools/manager/models/transform/NormalizeFieldTransformationTest.java index 0ab417ddb..aad3bc3c6 100644 --- a/src/test/java/com/conveyal/datatools/manager/models/transform/NormalizeFieldTransformationTest.java +++ b/src/test/java/com/conveyal/datatools/manager/models/transform/NormalizeFieldTransformationTest.java @@ -106,6 +106,24 @@ private static Stream createSubstitutionCasesWithOwnExceptions() { ); } + + @ParameterizedTest + @MethodSource("createCreateNewField") + public void testCreateNewField(String input, String expected) { + NormalizeFieldTransformation transform = createTransformation( + "table", "new_field", null, Lists.newArrayList( + new Substitution("", "new_value") + )); + assertEquals(expected, transform.performSubstitutions(input)); + } + + private static Stream createCreateNewField() { + return Stream.of( + Arguments.of("", "new_value") + ); + } + + /** * Proxy to create a transformation * (called by {@link com.conveyal.datatools.manager.jobs.NormalizeFieldTransformJobTest}). From 0ebf2227078653ede1d54a77676b22bb4a89a726 Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Wed, 24 May 2023 09:33:48 -0400 Subject: [PATCH 120/248] refactor: correct comment --- .../java/com/conveyal/datatools/manager/models/FeedVersion.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java b/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java index ec53c251a..a2c358167 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java @@ -393,7 +393,7 @@ public void validateMobility(MonitorableJob.Status status) { status.update("MobilityData Analysis...", 11); // Wait for the file to be entirely copied into the directory. - // 5 seconds + ~1 second per 10mb + // 5 seconds + ~1 second per 100kb Thread.sleep(5000 + (this.fileSize / 10000)); File gtfsZip = this.retrieveGtfsFile(); // Namespace based folders avoid clash for validation being run on multiple versions of a feed. From 638c7a80d73a976d0ad00a5842ecd70504d2146e Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Wed, 24 May 2023 10:22:33 -0400 Subject: [PATCH 121/248] Revert "refactor: correct comment" This reverts commit 0ebf2227078653ede1d54a77676b22bb4a89a726. --- .../java/com/conveyal/datatools/manager/models/FeedVersion.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java b/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java index 6fa211f1f..523ee181a 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java @@ -393,7 +393,7 @@ public void validateMobility(MonitorableJob.Status status) { status.update("MobilityData Analysis...", 11); // Wait for the file to be entirely copied into the directory. - // 5 seconds + ~1 second per 100kb + // 5 seconds + ~1 second per 10mb Thread.sleep(5000 + (this.fileSize / 10000)); File gtfsZip = this.retrieveGtfsFile(); // Namespace based folders avoid clash for validation being run on multiple versions of a feed. From 433ddc8dd8b169b05db63a154bd412e1b6275966 Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Tue, 30 May 2023 13:57:33 -0400 Subject: [PATCH 122/248] refactor: address pr feedback --- .../NormalizeFieldTransformation.java | 20 +++++++++++-------- .../NormalizeFieldTransformationTest.java | 2 +- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/models/transform/NormalizeFieldTransformation.java b/src/main/java/com/conveyal/datatools/manager/models/transform/NormalizeFieldTransformation.java index bc24a7e1f..727bc0e3e 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/transform/NormalizeFieldTransformation.java +++ b/src/main/java/com/conveyal/datatools/manager/models/transform/NormalizeFieldTransformation.java @@ -200,10 +200,7 @@ public void transform(FeedTransformZipTarget zipTarget, MonitorableJob.Status st // If the index is -1, this is a new column, and we need to add it accordingly. if (transformFieldIndex == -1) { - String[] expandedHeaders = new String[headers.length + 1]; - System.arraycopy(headers, 0, expandedHeaders, 0, headers.length); - expandedHeaders[headers.length] = fieldName; - writer.write(expandedHeaders); + writer.write(expandArray(headers, fieldName)); } else { writer.write(headers); } @@ -230,10 +227,7 @@ public void transform(FeedTransformZipTarget zipTarget, MonitorableJob.Status st // If the index is -1, this is a new column, and we need to add it accordingly. if (transformFieldIndex == -1) { - String[] expandedCsvValues = new String[headers.length + 1]; - System.arraycopy(csvValues, 0, expandedCsvValues, 0, csvValues.length); - expandedCsvValues[csvValues.length] = transformedValue; - writer.write(expandedCsvValues); + writer.write(expandArray(csvValues, transformedValue)); } else { csvValues[transformFieldIndex] = transformedValue; @@ -304,4 +298,14 @@ public String performSubstitutions(String inputString) { } return result; } + + /** + * Copies a fixed length array, and appends a new element at the end + */ + private String[] expandArray(String[] array, String value) { + String[] expanded = new String[array.length + 1]; + System.arraycopy(array, 0, expanded, 0, array.length); + expanded[array.length] = value; + return expanded; + } } diff --git a/src/test/java/com/conveyal/datatools/manager/models/transform/NormalizeFieldTransformationTest.java b/src/test/java/com/conveyal/datatools/manager/models/transform/NormalizeFieldTransformationTest.java index aad3bc3c6..f06b8919e 100644 --- a/src/test/java/com/conveyal/datatools/manager/models/transform/NormalizeFieldTransformationTest.java +++ b/src/test/java/com/conveyal/datatools/manager/models/transform/NormalizeFieldTransformationTest.java @@ -109,7 +109,7 @@ private static Stream createSubstitutionCasesWithOwnExceptions() { @ParameterizedTest @MethodSource("createCreateNewField") - public void testCreateNewField(String input, String expected) { + void testCreateNewField(String input, String expected) { NormalizeFieldTransformation transform = createTransformation( "table", "new_field", null, Lists.newArrayList( new Substitution("", "new_value") From 87b27b73482cf3464c4de462aa14cbd0ca4faada Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Tue, 30 May 2023 15:25:51 -0400 Subject: [PATCH 123/248] refactor: revert broken changes --- .../java/com/conveyal/datatools/manager/models/Deployment.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/conveyal/datatools/manager/models/Deployment.java b/src/main/java/com/conveyal/datatools/manager/models/Deployment.java index 5b783122a..a910c31b4 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/Deployment.java +++ b/src/main/java/com/conveyal/datatools/manager/models/Deployment.java @@ -456,7 +456,7 @@ public byte[] generateRouterConfig() throws IOException { Map map1 = mapper.readValue(customRouterConfigString, Map.class); Map map2 = mapper.readValue(routerConfigString, Map.class); Map merged = new HashMap(map2); - new HashMap(map2).putAll(map1); + merged.putAll(map1); return mapper.writeValueAsString(merged).getBytes(); } From 0c3f371ccdf0d536cfd460714641ef7f2d606d45 Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Wed, 31 May 2023 09:46:28 -0400 Subject: [PATCH 124/248] refactor: address pr feedback --- .../manager/models/transform/NormalizeFieldTransformation.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/conveyal/datatools/manager/models/transform/NormalizeFieldTransformation.java b/src/main/java/com/conveyal/datatools/manager/models/transform/NormalizeFieldTransformation.java index 727bc0e3e..e9c4d6ce7 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/transform/NormalizeFieldTransformation.java +++ b/src/main/java/com/conveyal/datatools/manager/models/transform/NormalizeFieldTransformation.java @@ -300,7 +300,7 @@ public String performSubstitutions(String inputString) { } /** - * Copies a fixed length array, and appends a new element at the end + * Copies a fixed length array, and appends a new element at the end. */ private String[] expandArray(String[] array, String value) { String[] expanded = new String[array.length + 1]; From efa0acc67e33eca5a572ae96ee028ea3db28b2f2 Mon Sep 17 00:00:00 2001 From: "philip.cline" Date: Thu, 8 Jun 2023 12:46:58 -0400 Subject: [PATCH 125/248] refactor(custom transformation): add separate class for custom file transformation --- .../AddCustomFileFromStringTransformation.java | 13 +++++++++++++ .../models/transform/FeedTransformation.java | 3 ++- 2 files changed, 15 insertions(+), 1 deletion(-) create mode 100644 src/main/java/com/conveyal/datatools/manager/models/transform/AddCustomFileFromStringTransformation.java diff --git a/src/main/java/com/conveyal/datatools/manager/models/transform/AddCustomFileFromStringTransformation.java b/src/main/java/com/conveyal/datatools/manager/models/transform/AddCustomFileFromStringTransformation.java new file mode 100644 index 000000000..f6b14096e --- /dev/null +++ b/src/main/java/com/conveyal/datatools/manager/models/transform/AddCustomFileFromStringTransformation.java @@ -0,0 +1,13 @@ +package com.conveyal.datatools.manager.models.transform; +import com.conveyal.datatools.common.status.MonitorableJob; + +public class AddCustomFileFromStringTransformation extends ReplaceFileFromStringTransformation { + + @Override + public void validateTableName(MonitorableJob.Status status) { + if (table.contains(".txt")) { + status.fail("CSV Table name should not contain .txt"); + } + } + +} diff --git a/src/main/java/com/conveyal/datatools/manager/models/transform/FeedTransformation.java b/src/main/java/com/conveyal/datatools/manager/models/transform/FeedTransformation.java index 221b26c67..0520e83f6 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/transform/FeedTransformation.java +++ b/src/main/java/com/conveyal/datatools/manager/models/transform/FeedTransformation.java @@ -31,7 +31,8 @@ @JsonSubTypes.Type(value = NormalizeFieldTransformation.class, name = "NormalizeFieldTransformation"), @JsonSubTypes.Type(value = ReplaceFileFromVersionTransformation.class, name = "ReplaceFileFromVersionTransformation"), @JsonSubTypes.Type(value = ReplaceFileFromStringTransformation.class, name = "ReplaceFileFromStringTransformation"), - @JsonSubTypes.Type(value = PreserveCustomFieldsTransformation.class, name = "PreserveCustomFieldsTransformation") + @JsonSubTypes.Type(value = PreserveCustomFieldsTransformation.class, name = "PreserveCustomFieldsTransformation"), + @JsonSubTypes.Type(value = AddCustomFileFromStringTransformation.class, name = "AddCustomFileTransformation") }) public abstract class FeedTransformation implements Serializable { private static final long serialVersionUID = 1L; From 77bd3472d7180e75906356c2a79d86b449ac08c7 Mon Sep 17 00:00:00 2001 From: "philip.cline" Date: Thu, 8 Jun 2023 22:45:13 -0400 Subject: [PATCH 126/248] refactor(custom transformation): rewrite preserve custom field transformation to use existing CSV libraries --- pom.xml | 7 -- .../PreserveCustomFieldsTransformation.java | 110 +++++------------- 2 files changed, 29 insertions(+), 88 deletions(-) diff --git a/pom.xml b/pom.xml index dd972bbe4..95e722c26 100644 --- a/pom.xml +++ b/pom.xml @@ -415,13 +415,6 @@ 1.26 - - - com.opencsv - opencsv - 5.7.1 - - - a3e5707 + 9ea7d34 diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java index bbcdcbb9c..6793575ea 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java @@ -240,7 +240,9 @@ public void startNewRow() throws IOException { public boolean checkForeignReferences(FieldContext fieldContext) throws IOException { Field field = fieldContext.getField(); if (field.isForeignReference()) { - String key = getTableScopedValue(field.referenceTable, fieldContext.getValue()); + // FIXME: GTFS-lib is now able to define multiple table references! This is only expecting one, so this update + // will definitely break this part of the code. + String key = getTableScopedValue(field.referenceTables.iterator().next(), fieldContext.getValue()); // Check if we're performing a service period merge, this ref field is a service_id, and it // is not found in the list of service_ids (e.g., it was removed). boolean isValidServiceId = mergeFeedsResult.serviceIds.contains(fieldContext.getValueToWrite()); From 07a455aeedd637cec3be7768001e5a9a157b0c8c Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Fri, 23 Jun 2023 15:33:42 +0100 Subject: [PATCH 138/248] refactor(MergeLineContext): Updated to use multi table refs --- .../jobs/feedmerge/MergeLineContext.java | 61 ++++++++++--------- 1 file changed, 31 insertions(+), 30 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java index 6793575ea..eefe7981e 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java @@ -240,38 +240,39 @@ public void startNewRow() throws IOException { public boolean checkForeignReferences(FieldContext fieldContext) throws IOException { Field field = fieldContext.getField(); if (field.isForeignReference()) { - // FIXME: GTFS-lib is now able to define multiple table references! This is only expecting one, so this update - // will definitely break this part of the code. - String key = getTableScopedValue(field.referenceTables.iterator().next(), fieldContext.getValue()); - // Check if we're performing a service period merge, this ref field is a service_id, and it - // is not found in the list of service_ids (e.g., it was removed). - boolean isValidServiceId = mergeFeedsResult.serviceIds.contains(fieldContext.getValueToWrite()); - - // If the current foreign ref points to another record that has - // been skipped or is a ref to a non-existent service_id during a service period merge, skip - // this record and add its primary key to the list of skipped IDs (so that other references - // can be properly omitted). - if (serviceIdHasKeyOrShouldBeSkipped(fieldContext, key, isValidServiceId)) { - // If a calendar#service_id has been skipped (it's listed in skippedIds), but there were - // valid service_ids found in calendar_dates, do not skip that record for both the - // calendar_date and any related trips. - if (fieldContext.nameEquals(SERVICE_ID) && isValidServiceId) { - LOG.warn("Not skipping valid service_id {} for {} {}", fieldContext.getValueToWrite(), table.name, keyValue); - } else { - String skippedKey = getTableScopedValue(keyValue); - if (orderField != null) { - skippedKey = String.join(":", skippedKey, getCsvValue(orderField)); + // FIXME: GTFS-lib is now able to define multiple table references! This update will most likely break this part of the code. + for (Table referenceTable : field.referenceTables) { + String key = getTableScopedValue(referenceTable, fieldContext.getValue()); + // Check if we're performing a service period merge, this ref field is a service_id, and it + // is not found in the list of service_ids (e.g., it was removed). + boolean isValidServiceId = mergeFeedsResult.serviceIds.contains(fieldContext.getValueToWrite()); + + // If the current foreign ref points to another record that has + // been skipped or is a ref to a non-existent service_id during a service period merge, skip + // this record and add its primary key to the list of skipped IDs (so that other references + // can be properly omitted). + if (serviceIdHasKeyOrShouldBeSkipped(fieldContext, key, isValidServiceId)) { + // If a calendar#service_id has been skipped (it's listed in skippedIds), but there were + // valid service_ids found in calendar_dates, do not skip that record for both the + // calendar_date and any related trips. + if (fieldContext.nameEquals(SERVICE_ID) && isValidServiceId) { + LOG.warn("Not skipping valid service_id {} for {} {}", fieldContext.getValueToWrite(), table.name, keyValue); + } else { + String skippedKey = getTableScopedValue(keyValue); + if (orderField != null) { + skippedKey = String.join(":", skippedKey, getCsvValue(orderField)); + } + mergeFeedsResult.skippedIds.add(skippedKey); + return false; } - mergeFeedsResult.skippedIds.add(skippedKey); - return false; } - } - // If the field is a foreign reference, check to see whether the reference has been - // remapped due to a conflicting ID from another feed (e.g., calendar#service_id). - if (mergeFeedsResult.remappedIds.containsKey(key)) { - mergeFeedsResult.remappedReferences++; - // If the value has been remapped update the value to write. - fieldContext.setValueToWrite(mergeFeedsResult.remappedIds.get(key)); + // If the field is a foreign reference, check to see whether the reference has been + // remapped due to a conflicting ID from another feed (e.g., calendar#service_id). + if (mergeFeedsResult.remappedIds.containsKey(key)) { + mergeFeedsResult.remappedReferences++; + // If the value has been remapped update the value to write. + fieldContext.setValueToWrite(mergeFeedsResult.remappedIds.get(key)); + } } } return true; From 8e5df795c0f67ab50076bc8de7c35a8813dc08ec Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Fri, 23 Jun 2023 14:32:05 -0400 Subject: [PATCH 139/248] fix: Support custom tags --- .../datatools/manager/jobs/DeployJob.java | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java index 0ffc1fc9e..b3bc61606 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java @@ -10,8 +10,10 @@ import com.amazonaws.services.ec2.model.InstanceNetworkInterfaceSpecification; import com.amazonaws.services.ec2.model.InstanceStateChange; import com.amazonaws.services.ec2.model.InstanceType; +import com.amazonaws.services.ec2.model.ResourceType; import com.amazonaws.services.ec2.model.RunInstancesRequest; import com.amazonaws.services.ec2.model.Tag; +import com.amazonaws.services.ec2.model.TagSpecification; import com.amazonaws.services.ec2.model.TerminateInstancesResult; import com.amazonaws.services.elasticloadbalancingv2.AmazonElasticLoadBalancing; import com.amazonaws.services.s3.AmazonS3; @@ -69,6 +71,7 @@ import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.Date; import java.util.HashSet; @@ -993,6 +996,29 @@ private List startEC2Instances(int count, boolean graphAlreadyBuilt) { return Collections.EMPTY_LIST; } status.message = String.format("Starting up %d new instance(s) to run OTP", count); + + // If a tag is set in the server.yml, add the tag to the instance + TagSpecification ts = new TagSpecification(); + ts.setResourceType(ResourceType.Instance); + Collection tags = new ArrayList(); + + String tagKey = DataManager.getConfigPropertyAsText("modules.deployment.ec2.tag_key"); + String tagValue = DataManager.getConfigPropertyAsText("modules.deployment.ec2.tag_value"); + + Tag t = new Tag(); + t.setKey("datatools-job-id"); + t.setValue(this.jobId); + tags.add(t); + + if (tagValue != null && tagKey != null) { + Tag customTag = new Tag(); + customTag.setKey(tagKey); + customTag.setValue(tagValue); + tags.add(customTag); + } + ts.withTags(tags); + + RunInstancesRequest runInstancesRequest = new RunInstancesRequest() .withNetworkInterfaces(interfaceSpecification) .withInstanceType(instanceType) @@ -1006,6 +1032,7 @@ private List startEC2Instances(int count, boolean graphAlreadyBuilt) { .withIamInstanceProfile(new IamInstanceProfileSpecification().withArn(otpServer.ec2Info.iamInstanceProfileArn)) .withImageId(amiId) .withKeyName(otpServer.ec2Info.keyName) + .withTagSpecifications(ts) // This will have the instance terminate when it is shut down. .withInstanceInitiatedShutdownBehavior("terminate") .withUserData(Base64.encodeBase64String(userData.getBytes())); @@ -1046,6 +1073,7 @@ private List startEC2Instances(int count, boolean graphAlreadyBuilt) { .withTags(new Tag("serverId", otpServer.id)) .withTags(new Tag("routerId", getRouterId())) .withTags(new Tag("user", retrieveEmail())) + .withResources(instance.getInstanceId()) ); } catch (Exception e) { From 9c1db8f0ead18bbf1b30448347c8abafbb173ff4 Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Fri, 23 Jun 2023 14:39:22 -0400 Subject: [PATCH 140/248] refactor: make use of existing tag infrastructure --- configurations/default/server.yml.tmp | 2 + .../datatools/manager/jobs/DeployJob.java | 40 ++++++------------- 2 files changed, 15 insertions(+), 27 deletions(-) diff --git a/configurations/default/server.yml.tmp b/configurations/default/server.yml.tmp index 20edb3008..06bdaa3f7 100644 --- a/configurations/default/server.yml.tmp +++ b/configurations/default/server.yml.tmp @@ -24,6 +24,8 @@ modules: ec2: enabled: false default_ami: ami-your-ami-id + tag_key: a-tag-key-to-add-to-all-instances + tag_value: a-tag-value-to-add-to-all-instances # Note: using a cloudfront URL for these download URLs will greatly # increase download/deploy speed. otp_download_url: https://optional-otp-repo.com diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java index b3bc61606..2ad288e18 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java @@ -16,6 +16,7 @@ import com.amazonaws.services.ec2.model.TagSpecification; import com.amazonaws.services.ec2.model.TerminateInstancesResult; import com.amazonaws.services.elasticloadbalancingv2.AmazonElasticLoadBalancing; +import com.amazonaws.services.identitymanagement.model.TagUserRequest; import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.AmazonS3URI; import com.amazonaws.services.s3.model.CopyObjectRequest; @@ -997,28 +998,6 @@ private List startEC2Instances(int count, boolean graphAlreadyBuilt) { } status.message = String.format("Starting up %d new instance(s) to run OTP", count); - // If a tag is set in the server.yml, add the tag to the instance - TagSpecification ts = new TagSpecification(); - ts.setResourceType(ResourceType.Instance); - Collection tags = new ArrayList(); - - String tagKey = DataManager.getConfigPropertyAsText("modules.deployment.ec2.tag_key"); - String tagValue = DataManager.getConfigPropertyAsText("modules.deployment.ec2.tag_value"); - - Tag t = new Tag(); - t.setKey("datatools-job-id"); - t.setValue(this.jobId); - tags.add(t); - - if (tagValue != null && tagKey != null) { - Tag customTag = new Tag(); - customTag.setKey(tagKey); - customTag.setValue(tagValue); - tags.add(customTag); - } - ts.withTags(tags); - - RunInstancesRequest runInstancesRequest = new RunInstancesRequest() .withNetworkInterfaces(interfaceSpecification) .withInstanceType(instanceType) @@ -1032,7 +1011,6 @@ private List startEC2Instances(int count, boolean graphAlreadyBuilt) { .withIamInstanceProfile(new IamInstanceProfileSpecification().withArn(otpServer.ec2Info.iamInstanceProfileArn)) .withImageId(amiId) .withKeyName(otpServer.ec2Info.keyName) - .withTagSpecifications(ts) // This will have the instance terminate when it is shut down. .withInstanceInitiatedShutdownBehavior("terminate") .withUserData(Base64.encodeBase64String(userData.getBytes())); @@ -1065,7 +1043,7 @@ private List startEC2Instances(int count, boolean graphAlreadyBuilt) { String serverName = String.format("%s %s (%s) %d %s", deployment.tripPlannerVersion, deployment.name, dateString, serverCounter++, graphAlreadyBuilt ? "clone" : "builder"); LOG.info("Creating tags for new EC2 instance {}", serverName); try { - getEC2ClientForDeployJob().createTags(new CreateTagsRequest() + CreateTagsRequest createTagsRequest = new CreateTagsRequest() .withTags(new Tag("Name", serverName)) .withTags(new Tag("projectId", deployment.projectId)) .withTags(new Tag("deploymentId", deployment.id)) @@ -1073,9 +1051,17 @@ private List startEC2Instances(int count, boolean graphAlreadyBuilt) { .withTags(new Tag("serverId", otpServer.id)) .withTags(new Tag("routerId", getRouterId())) .withTags(new Tag("user", retrieveEmail())) - - .withResources(instance.getInstanceId()) - ); + .withResources(instance.getInstanceId()); + + String tagKey = DataManager.getConfigPropertyAsText("modules.deployment.ec2.tag_key"); + String tagValue = DataManager.getConfigPropertyAsText("modules.deployment.ec2.tag_value"); + + Tag customTag = new Tag(); + customTag.setKey(tagKey); + customTag.setValue(tagValue); + + createTagsRequest = createTagsRequest.withTags(customTag); + getEC2ClientForDeployJob().createTags(createTagsRequest); } catch (Exception e) { status.fail("Failed to create tags for instances.", e); return instances; From 6eb1105d71fc6e4ed11ce5201fb71f40b6d0adce Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Fri, 23 Jun 2023 14:43:53 -0400 Subject: [PATCH 141/248] refactor: clean up --- .../java/com/conveyal/datatools/manager/jobs/DeployJob.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java index 2ad288e18..4083dfc72 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java @@ -10,13 +10,10 @@ import com.amazonaws.services.ec2.model.InstanceNetworkInterfaceSpecification; import com.amazonaws.services.ec2.model.InstanceStateChange; import com.amazonaws.services.ec2.model.InstanceType; -import com.amazonaws.services.ec2.model.ResourceType; import com.amazonaws.services.ec2.model.RunInstancesRequest; import com.amazonaws.services.ec2.model.Tag; -import com.amazonaws.services.ec2.model.TagSpecification; import com.amazonaws.services.ec2.model.TerminateInstancesResult; import com.amazonaws.services.elasticloadbalancingv2.AmazonElasticLoadBalancing; -import com.amazonaws.services.identitymanagement.model.TagUserRequest; import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.AmazonS3URI; import com.amazonaws.services.s3.model.CopyObjectRequest; @@ -72,7 +69,6 @@ import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collection; import java.util.Collections; import java.util.Date; import java.util.HashSet; From ace391ed382ebe5313466c12ce2604d0e19e405b Mon Sep 17 00:00:00 2001 From: "philip.cline" Date: Mon, 26 Jun 2023 18:46:10 -0400 Subject: [PATCH 142/248] refactor(transformation): address PR feedback --- pom.xml | 1 - .../models/transform/PreserveCustomFieldsTransformation.java | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 95e722c26..e6ea2cf95 100644 --- a/pom.xml +++ b/pom.xml @@ -414,7 +414,6 @@ snakeyaml 1.26 - - 9ea7d34 + ba747df diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarDatesMergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarDatesMergeLineContext.java index 649e0c088..d65548e23 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarDatesMergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarDatesMergeLineContext.java @@ -87,6 +87,7 @@ private boolean checkCalendarDatesIds(FieldContext fieldContext) throws IOExcept // the valid date range, i.e., before the future feed's first date. if (!shouldSkipRecord && fieldContext.nameEquals(SERVICE_ID)) { mergeFeedsResult.serviceIds.add(fieldContext.getValueToWrite()); + mergeFeedsResult.calendarDatesServiceIds.add(fieldContext.getValueToWrite()); } return !shouldSkipRecord; diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarMergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarMergeLineContext.java index ab8902531..1f6cd27aa 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarMergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarMergeLineContext.java @@ -111,6 +111,7 @@ private boolean checkCalendarIds(Set idErrors, FieldContext fieldC // If service is going to be cloned, add to the output service ids. if (!shouldSkipRecord && fieldContext.nameEquals(SERVICE_ID)) { mergeFeedsResult.serviceIds.add(fieldContext.getValueToWrite()); + mergeFeedsResult.calendarServiceIds.add(fieldContext.getValueToWrite()); } return !shouldSkipRecord; diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeFeedsResult.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeFeedsResult.java index 1adadbb91..5fa9b25b3 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeFeedsResult.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeFeedsResult.java @@ -21,10 +21,20 @@ public class MergeFeedsResult implements Serializable { /** Contains the set of IDs for records that were excluded in the merged feed */ public Set skippedIds = new HashSet<>(); /** - * Track the set of service IDs to end up in the merged feed in order to determine which calendar_dates and trips - * records should be retained in the merged result. + * Track the set of service IDs to end up in the merged feed in order to determine which calendar, calendar_dates and + * trip records should be retained in the merged result. */ public Set serviceIds = new HashSet<>(); + + /** + * Track the set of service IDs obtained from calendar records. + */ + public Set calendarServiceIds = new HashSet<>(); + /** + * Track the set of service IDs obtained from calendar date records. + */ + public Set calendarDatesServiceIds = new HashSet<>(); + /** * Track the set of route IDs to end up in the merged feed in order to determine which route_attributes * records should be retained in the merged result. diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java index eefe7981e..491b4f740 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java @@ -237,42 +237,102 @@ public void startNewRow() throws IOException { .collect(Collectors.toList()); } + public enum ReferenceTableLookup { + TRIP_SERVICE_ID_KEY( + String.join(":", Table.TRIPS.name, SERVICE_ID, Table.CALENDAR.name, Table.CALENDAR_DATES.name) + ); + private final String value; + ReferenceTableLookup(String value) { + this.value = value; + } + public String getValue() { + return value; + } + + public static ReferenceTableLookup fromValue(String key) { + for (ReferenceTableLookup ref: ReferenceTableLookup.values()) { + if (ref.getValue().equals(key)) { + return ref; + } + } + throw new UnsupportedOperationException(String.format("Unsupported reference table combination: %s.", key)); + } + } + + /** + * Determine which reference table to use. If there is only one reference use this. If there are multiple references + * determine the context and then the correct reference table to use. + */ + private Table getReferenceTable(FieldContext fieldContext, Field field) { + if (field.referenceTables.size() == 1) { + return field.referenceTables.iterator().next(); + } + Table defaultTable = null; + switch (ReferenceTableLookup.fromValue(createKey(field))) { + case TRIP_SERVICE_ID_KEY: + boolean isCalendarServiceId = mergeFeedsResult.calendarServiceIds.contains(fieldContext.getValueToWrite()); + boolean isCalendarDatesServiceId = mergeFeedsResult.calendarDatesServiceIds.contains(fieldContext.getValueToWrite()); + if (isCalendarServiceId && !isCalendarDatesServiceId) { + return Table.CALENDAR; + } else if (!isCalendarServiceId && isCalendarDatesServiceId) { + return Table.CALENDAR_DATES; + } else { + // A table is still required if the service id is not present in either look-up. For this case it + // doesn't seem to matter which is returned so going with calendar. + defaultTable = Table.CALENDAR; + } + // Add other cases as multiple references are added e.g. flex!. + } + return defaultTable; + } + + /** + * Create a unique key for this table, field and reference tables. + */ + private String createKey(Field field) { + return String.format( + "%s:%s:%s", + table.name, + field.name, + field.referenceTables.stream().map(r -> r.name).collect(Collectors.joining(":")) + ); + } + + + public boolean checkForeignReferences(FieldContext fieldContext) throws IOException { Field field = fieldContext.getField(); if (field.isForeignReference()) { - // FIXME: GTFS-lib is now able to define multiple table references! This update will most likely break this part of the code. - for (Table referenceTable : field.referenceTables) { - String key = getTableScopedValue(referenceTable, fieldContext.getValue()); - // Check if we're performing a service period merge, this ref field is a service_id, and it - // is not found in the list of service_ids (e.g., it was removed). - boolean isValidServiceId = mergeFeedsResult.serviceIds.contains(fieldContext.getValueToWrite()); - - // If the current foreign ref points to another record that has - // been skipped or is a ref to a non-existent service_id during a service period merge, skip - // this record and add its primary key to the list of skipped IDs (so that other references - // can be properly omitted). - if (serviceIdHasKeyOrShouldBeSkipped(fieldContext, key, isValidServiceId)) { - // If a calendar#service_id has been skipped (it's listed in skippedIds), but there were - // valid service_ids found in calendar_dates, do not skip that record for both the - // calendar_date and any related trips. - if (fieldContext.nameEquals(SERVICE_ID) && isValidServiceId) { - LOG.warn("Not skipping valid service_id {} for {} {}", fieldContext.getValueToWrite(), table.name, keyValue); - } else { - String skippedKey = getTableScopedValue(keyValue); - if (orderField != null) { - skippedKey = String.join(":", skippedKey, getCsvValue(orderField)); - } - mergeFeedsResult.skippedIds.add(skippedKey); - return false; + String key = getTableScopedValue(getReferenceTable(fieldContext, field), fieldContext.getValue()); + // Check if we're performing a service period merge, this ref field is a service_id, and it + // is not found in the list of service_ids (e.g., it was removed). + boolean isValidServiceId = mergeFeedsResult.serviceIds.contains(fieldContext.getValueToWrite()); + + // If the current foreign ref points to another record that has + // been skipped or is a ref to a non-existent service_id during a service period merge, skip + // this record and add its primary key to the list of skipped IDs (so that other references + // can be properly omitted). + if (serviceIdHasKeyOrShouldBeSkipped(fieldContext, key, isValidServiceId)) { + // If a calendar#service_id has been skipped (it's listed in skippedIds), but there were + // valid service_ids found in calendar_dates, do not skip that record for both the + // calendar_date and any related trips. + if (fieldContext.nameEquals(SERVICE_ID) && isValidServiceId) { + LOG.warn("Not skipping valid service_id {} for {} {}", fieldContext.getValueToWrite(), table.name, keyValue); + } else { + String skippedKey = getTableScopedValue(keyValue); + if (orderField != null) { + skippedKey = String.join(":", skippedKey, getCsvValue(orderField)); } + mergeFeedsResult.skippedIds.add(skippedKey); + return false; } - // If the field is a foreign reference, check to see whether the reference has been - // remapped due to a conflicting ID from another feed (e.g., calendar#service_id). - if (mergeFeedsResult.remappedIds.containsKey(key)) { - mergeFeedsResult.remappedReferences++; - // If the value has been remapped update the value to write. - fieldContext.setValueToWrite(mergeFeedsResult.remappedIds.get(key)); - } + } + // If the field is a foreign reference, check to see whether the reference has been + // remapped due to a conflicting ID from another feed (e.g., calendar#service_id). + if (mergeFeedsResult.remappedIds.containsKey(key)) { + mergeFeedsResult.remappedReferences++; + // If the value has been remapped update the value to write. + fieldContext.setValueToWrite(mergeFeedsResult.remappedIds.get(key)); } } return true; From 38fa3a21e0ba4f270f9ed240d20b0e3bc0a3c556 Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Fri, 30 Jun 2023 13:00:32 +0100 Subject: [PATCH 146/248] refactor(Addressed line spacing and updated comments): --- .../manager/jobs/feedmerge/MergeFeedsResult.java | 1 + .../manager/jobs/feedmerge/MergeLineContext.java | 10 ++++++---- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeFeedsResult.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeFeedsResult.java index 5fa9b25b3..881c78051 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeFeedsResult.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeFeedsResult.java @@ -30,6 +30,7 @@ public class MergeFeedsResult implements Serializable { * Track the set of service IDs obtained from calendar records. */ public Set calendarServiceIds = new HashSet<>(); + /** * Track the set of service IDs obtained from calendar date records. */ diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java index 491b4f740..95f6f5420 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java @@ -241,10 +241,13 @@ public enum ReferenceTableLookup { TRIP_SERVICE_ID_KEY( String.join(":", Table.TRIPS.name, SERVICE_ID, Table.CALENDAR.name, Table.CALENDAR_DATES.name) ); + private final String value; + ReferenceTableLookup(String value) { this.value = value; } + public String getValue() { return value; } @@ -267,6 +270,8 @@ private Table getReferenceTable(FieldContext fieldContext, Field field) { if (field.referenceTables.size() == 1) { return field.referenceTables.iterator().next(); } + + // A table reference is still required even if no match is found. Table defaultTable = null; switch (ReferenceTableLookup.fromValue(createKey(field))) { case TRIP_SERVICE_ID_KEY: @@ -277,8 +282,7 @@ private Table getReferenceTable(FieldContext fieldContext, Field field) { } else if (!isCalendarServiceId && isCalendarDatesServiceId) { return Table.CALENDAR_DATES; } else { - // A table is still required if the service id is not present in either look-up. For this case it - // doesn't seem to matter which is returned so going with calendar. + // For this case it doesn't seem to matter which is returned so going with calendar. defaultTable = Table.CALENDAR; } // Add other cases as multiple references are added e.g. flex!. @@ -298,8 +302,6 @@ private String createKey(Field field) { ); } - - public boolean checkForeignReferences(FieldContext fieldContext) throws IOException { Field field = fieldContext.getField(); if (field.isForeignReference()) { From 73ffc88304dd672662ef0922a67ed5aace2b1e62 Mon Sep 17 00:00:00 2001 From: "philip.cline" Date: Fri, 30 Jun 2023 11:37:10 -0400 Subject: [PATCH 147/248] refactor(transformations): respond to PR comments --- .../models/transform/FeedTransformation.java | 5 +- .../PreserveCustomFieldsTransformation.java | 16 ++-- .../jobs/ArbitraryTransformJobTest.java | 78 ++++++++++--------- 3 files changed, 51 insertions(+), 48 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/models/transform/FeedTransformation.java b/src/main/java/com/conveyal/datatools/manager/models/transform/FeedTransformation.java index ee4d86f51..e63001b37 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/transform/FeedTransformation.java +++ b/src/main/java/com/conveyal/datatools/manager/models/transform/FeedTransformation.java @@ -70,6 +70,8 @@ public void doTransform(FeedTransformTarget target, MonitorableJob.Status status status.fail( String.format("Transformation must be of type '%s'.", getTransformationTypeName()) ); + } catch (Exception e) { + status.fail(e.toString()); } } @@ -80,7 +82,7 @@ public void doTransform(FeedTransformTarget target, MonitorableJob.Status status * @param target The database-bound or ZIP-file-bound target the transformation will operate on. * @param status Used to report success or failure status and details. */ - public abstract void transform(Target target, MonitorableJob.Status status); + public abstract void transform(Target target, MonitorableJob.Status status) throws Exception; /** * At the moment, used by DbTransformation to validate field names. @@ -100,7 +102,6 @@ protected void validateTableName(MonitorableJob.Status status) { // Validate fields before running transform. if (GtfsUtils.getGtfsTable(table) == null) { status.fail("Table must be valid GTFS spec table name (without .txt)."); - return; } } } diff --git a/src/main/java/com/conveyal/datatools/manager/models/transform/PreserveCustomFieldsTransformation.java b/src/main/java/com/conveyal/datatools/manager/models/transform/PreserveCustomFieldsTransformation.java index a2b5d7587..0598fa944 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/transform/PreserveCustomFieldsTransformation.java +++ b/src/main/java/com/conveyal/datatools/manager/models/transform/PreserveCustomFieldsTransformation.java @@ -59,17 +59,17 @@ private static HashMap> createCsvHashMap(CsvMapReade } @Override - public void transform(FeedTransformZipTarget zipTarget, MonitorableJob.Status status) { + public void transform(FeedTransformZipTarget zipTarget, MonitorableJob.Status status) throws Exception{ String tableName = table + ".txt"; Path targetZipPath = Paths.get(zipTarget.gtfsFile.getAbsolutePath()); - Table specTable = Arrays.stream(Table.tablesInOrder) + Optional streamResult = Arrays.stream(Table.tablesInOrder) .filter(t -> t.name.equals(table)) - .findFirst() - .get(); + .findFirst(); - try (FileSystem targetZipFs = FileSystems.newFileSystem(targetZipPath, (ClassLoader) null)) { - if (specTable == null) throw new Exception(String.format("could not find specTable for table %s", table)); + if (!streamResult.isPresent()) {throw new Exception(String.format("could not find specTable for table %s", table));} + Table specTable = streamResult.get(); + try (FileSystem targetZipFs = FileSystems.newFileSystem(targetZipPath, (ClassLoader) null)) { List specTableFields = specTable.specFields().stream().map(f -> f.name).collect(Collectors.toList()); List tablePrimaryKeys = specTable.getPrimaryKeyNames(); @@ -77,7 +77,6 @@ public void transform(FeedTransformZipTarget zipTarget, MonitorableJob.Status st final File tempFile = File.createTempFile(tableName + "-temp", ".txt"); File output = File.createTempFile(tableName + "-output-temp", ".txt"); - int rowsModified = 0; List customFields; try ( @@ -106,11 +105,10 @@ public void transform(FeedTransformZipTarget zipTarget, MonitorableJob.Status st String hashKey = StringUtils.join(editorCsvPrimaryKeyValues, "_"); Map customCsvValues = customFieldsLookup.get(hashKey); Map finalRow = row; - customFields.stream().forEach(customField -> { + customFields.forEach(customField -> { String value = customCsvValues == null ? null : customCsvValues.get(customField); finalRow.put(customField, value); }); - if (customCsvValues != null) rowsModified++; writer.write(finalRow, fullHeaders); } } diff --git a/src/test/java/com/conveyal/datatools/manager/jobs/ArbitraryTransformJobTest.java b/src/test/java/com/conveyal/datatools/manager/jobs/ArbitraryTransformJobTest.java index 81683bd8c..4499051b0 100644 --- a/src/test/java/com/conveyal/datatools/manager/jobs/ArbitraryTransformJobTest.java +++ b/src/test/java/com/conveyal/datatools/manager/jobs/ArbitraryTransformJobTest.java @@ -239,31 +239,35 @@ void canPreserveCustomFieldsInStops() throws IOException { zipFolderFiles("fake-agency-with-only-calendar-dates") ); LOG.info("Checking assertions."); - CsvMapReader finalStopsReader = getCsvMapReaderFromZip(targetVersion.retrieveGtfsFile(), "stops.txt"); - CsvMapReader originalStopsReader = getCsvMapReaderFromZip(sourceVersion.retrieveGtfsFile(), "stops.txt"); - String[] finalHeaders = finalStopsReader.getHeader(true); - String[] originalHeaders = originalStopsReader.getHeader(true); - int columnsAdded = finalHeaders.length - originalHeaders.length; + try ( + CsvMapReader finalStopsReader = getCsvMapReaderFromZip(targetVersion.retrieveGtfsFile(), "stops.txt"); + CsvMapReader originalStopsReader = getCsvMapReaderFromZip(sourceVersion.retrieveGtfsFile(), "stops.txt") + ) { + String[] finalHeaders = finalStopsReader.getHeader(true); + String[] originalHeaders = originalStopsReader.getHeader(true); + int columnsAdded = finalHeaders.length - originalHeaders.length; - // Get the number of rows that have been modified with custom fields - Map row; - int updatedRowCount = 0; - while ((row = finalStopsReader.read(finalHeaders)) != null) { - // Count the number of cases where the custom column is not null, that's the number modified. - // *** Assumes that our sample data below includes values for every customColumn for every row. - if (row.get("custom_column1") != null) updatedRowCount++; - } - assertEquals( - 2, - columnsAdded, - "stops.txt custom column count should equal input csv data # of custom columns" - ); + // Get the number of rows that have been modified with custom fields + Map row; + int updatedRowCount = 0; + while ((row = finalStopsReader.read(finalHeaders)) != null) { + // Count the number of cases where the custom column is not null, that's the number modified. + // *** Assumes that our sample data below includes values for every customColumn for every row. + if (row.get("custom_column1") != null) updatedRowCount++; - assertEquals( - 2, - updatedRowCount, - "stops.txt row count modified with custom content should equal input csv data # of custom columns" - ); + assertEquals( + 2, + columnsAdded, + "stops.txt custom column count should equal input csv data # of custom columns" + ); + + assertEquals( + 2, + updatedRowCount, + "stops.txt row count modified with custom content should equal input csv data # of custom columns" + ); + } + } } @Test @@ -277,25 +281,25 @@ void canAddCustomFile() throws IOException { feedSource, zipFolderFiles("fake-agency-with-only-calendar-dates") ); + try (CsvMapReader customCsvReader = getCsvMapReaderFromZip(targetVersion.retrieveGtfsFile(), "custom-file.txt")) { + String[] customHeaders = customCsvReader.getHeader(true); + int rowCount = 0; + while(customCsvReader.read(customHeaders) != null) rowCount++; - CsvMapReader customCsvReader = getCsvMapReaderFromZip(targetVersion.retrieveGtfsFile(), "custom-file.txt"); - String[] customHeaders = customCsvReader.getHeader(true); - int rowCount = 0; - while(customCsvReader.read(customHeaders) != null) rowCount++; - - LOG.info("Checking assertions."); - assertEquals( - 2, - rowCount, - "custom-file.txt custom row count should equal input csv data # of rows" - ); - + LOG.info("Checking assertions."); + assertEquals( + 2, + rowCount, + "custom-file.txt custom row count should equal input csv data # of rows" + ); + } } private static CsvMapReader getCsvMapReaderFromZip(File gtfsFile, String table) throws IOException { ZipFile zipFile = new ZipFile(gtfsFile); ZipEntry entry = zipFile.getEntry(table); - InputStream is = zipFile.getInputStream(entry); - return new CsvMapReader(new InputStreamReader(is), CsvPreference.STANDARD_PREFERENCE); + try (InputStream is = zipFile.getInputStream(entry)) { + return new CsvMapReader(new InputStreamReader(is), CsvPreference.STANDARD_PREFERENCE); + } } private static String generateFeedInfo(String feedId) { From afed13cc963ee6a89d7affb4966b846aefa774da Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Fri, 30 Jun 2023 13:57:03 -0400 Subject: [PATCH 148/248] feat: Support pelias synonyms file --- .../com/conveyal/datatools/manager/jobs/PeliasUpdateJob.java | 3 +++ .../java/com/conveyal/datatools/manager/models/Deployment.java | 2 ++ 2 files changed, 5 insertions(+) diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/PeliasUpdateJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/PeliasUpdateJob.java index 964f75fed..375a051a3 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/PeliasUpdateJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/PeliasUpdateJob.java @@ -142,6 +142,7 @@ private String makeWebhookRequest() { PeliasWebhookRequestBody peliasWebhookRequestBody = new PeliasWebhookRequestBody(); peliasWebhookRequestBody.gtfsFeeds = gtfsFeeds; peliasWebhookRequestBody.csvFiles = deployment.peliasCsvFiles; + peliasWebhookRequestBody.peliasSynonymsBase64 = deployment.peliasSynonymsBase64; peliasWebhookRequestBody.logUploadUrl = logUploadS3URI.toString(); peliasWebhookRequestBody.deploymentId = deployment.id; peliasWebhookRequestBody.resetDb = deployment.peliasResetDb; @@ -218,6 +219,8 @@ private static class PeliasWebhookRequestBody { public String logUploadUrl; public String deploymentId; public boolean resetDb; + + public String peliasSynonymsBase64; } /** diff --git a/src/main/java/com/conveyal/datatools/manager/models/Deployment.java b/src/main/java/com/conveyal/datatools/manager/models/Deployment.java index a910c31b4..b43092f87 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/Deployment.java +++ b/src/main/java/com/conveyal/datatools/manager/models/Deployment.java @@ -81,6 +81,8 @@ public class Deployment extends Model implements Serializable { public boolean peliasResetDb; public List peliasCsvFiles = new ArrayList<>(); + public String peliasSynonymsBase64; + /** * Get parent project for deployment. Note: at one point this was a JSON property of this class, but severe * performance issues prevent this field from scaling to be fetched/assigned to a large collection of deployments. From ec87de24d086d86115fc5f95b7cd539c74eb3d3b Mon Sep 17 00:00:00 2001 From: "philip.cline" Date: Fri, 30 Jun 2023 15:37:44 -0400 Subject: [PATCH 149/248] refactor(transformations): add results to TableTransformResult again --- .../manager/models/TableTransformResult.java | 17 +++++ .../PreserveCustomFieldsTransformation.java | 11 +++- .../transform/StringTransformation.java | 6 +- .../jobs/ArbitraryTransformJobTest.java | 65 +++++-------------- 4 files changed, 48 insertions(+), 51 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/models/TableTransformResult.java b/src/main/java/com/conveyal/datatools/manager/models/TableTransformResult.java index ad8d2cf58..0ff2b0622 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/TableTransformResult.java +++ b/src/main/java/com/conveyal/datatools/manager/models/TableTransformResult.java @@ -11,6 +11,7 @@ public class TableTransformResult implements Serializable { public int deletedCount; public int updatedCount; public int addedCount; + public int customColumnsAdded; public TransformType transformType; public String tableName; @@ -21,6 +22,22 @@ public TableTransformResult(String tableName, TransformType transformType) { this.transformType = transformType; } + public TableTransformResult( + String tableName, + TransformType transformType, + int deletedCount, + int updatedCount, + int addedCount, + int customColumnsAdded + ) { + this.tableName = tableName; + this.transformType = transformType; + this.deletedCount = deletedCount; + this.updatedCount = updatedCount; + this.addedCount = addedCount; + this.customColumnsAdded = customColumnsAdded; + } + public TableTransformResult(String tableName, int deletedCount, int updatedCount, int addedCount) { this.tableName = tableName; this.transformType = TransformType.TABLE_MODIFIED; diff --git a/src/main/java/com/conveyal/datatools/manager/models/transform/PreserveCustomFieldsTransformation.java b/src/main/java/com/conveyal/datatools/manager/models/transform/PreserveCustomFieldsTransformation.java index 0598fa944..bd97218c9 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/transform/PreserveCustomFieldsTransformation.java +++ b/src/main/java/com/conveyal/datatools/manager/models/transform/PreserveCustomFieldsTransformation.java @@ -77,6 +77,7 @@ public void transform(FeedTransformZipTarget zipTarget, MonitorableJob.Status st final File tempFile = File.createTempFile(tableName + "-temp", ".txt"); File output = File.createTempFile(tableName + "-output-temp", ".txt"); + int rowsModified = 0; List customFields; try ( @@ -109,13 +110,21 @@ public void transform(FeedTransformZipTarget zipTarget, MonitorableJob.Status st String value = customCsvValues == null ? null : customCsvValues.get(customField); finalRow.put(customField, value); }); + if (customCsvValues != null) rowsModified++; writer.write(finalRow, fullHeaders); } } Files.copy(output.toPath(), targetTxtFilePath, StandardCopyOption.REPLACE_EXISTING); tempFile.deleteOnExit(); output.deleteOnExit(); - zipTarget.feedTransformResult.tableTransformResults.add(new TableTransformResult(tableName, TransformType.TABLE_MODIFIED)); + zipTarget.feedTransformResult.tableTransformResults.add(new TableTransformResult( + tableName, + TransformType.TABLE_MODIFIED, + 0, + rowsModified, + 0, + customFields.size() + )); } catch (NoSuchFileException e) { status.fail("Source version does not contain table: " + tableName, e); } catch (IOException e) { diff --git a/src/main/java/com/conveyal/datatools/manager/models/transform/StringTransformation.java b/src/main/java/com/conveyal/datatools/manager/models/transform/StringTransformation.java index f0b8242bb..aa6731a3a 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/transform/StringTransformation.java +++ b/src/main/java/com/conveyal/datatools/manager/models/transform/StringTransformation.java @@ -44,7 +44,11 @@ public void transform(FeedTransformZipTarget zipTarget, MonitorableJob.Status st : TransformType.TABLE_ADDED; // Copy csv input stream into the zip file, replacing it if it already exists. Files.copy(inputStream, targetTxtFilePath, StandardCopyOption.REPLACE_EXISTING); - zipTarget.feedTransformResult.tableTransformResults.add(new TableTransformResult(tableName, type)); + final int NEW_LINE_CHARACTER_CODE = 10; + int lineCount = (int) csvData.chars().filter(c -> c == NEW_LINE_CHARACTER_CODE).count(); + int addedCount = type == TransformType.TABLE_ADDED ? lineCount : 0; + int updatedCount = type == TransformType.TABLE_MODIFIED ? lineCount : 0; + zipTarget.feedTransformResult.tableTransformResults.add(new TableTransformResult(tableName, type, 0, updatedCount, addedCount, 0)); } catch (Exception e) { status.fail("Unknown error encountered while transforming zip file", e); } diff --git a/src/test/java/com/conveyal/datatools/manager/jobs/ArbitraryTransformJobTest.java b/src/test/java/com/conveyal/datatools/manager/jobs/ArbitraryTransformJobTest.java index 4499051b0..541503322 100644 --- a/src/test/java/com/conveyal/datatools/manager/jobs/ArbitraryTransformJobTest.java +++ b/src/test/java/com/conveyal/datatools/manager/jobs/ArbitraryTransformJobTest.java @@ -226,10 +226,6 @@ void canReplaceFeedInfo() throws SQLException, IOException { @Test void canPreserveCustomFieldsInStops() throws IOException { String stops = generateStopsWithCustomFields(); - sourceVersion = createFeedVersion( - feedSource, - zipFolderFiles("fake-agency-with-only-calendar") - ); FeedTransformation transformation = PreserveCustomFieldsTransformation.create(stops, "stops"); FeedTransformRules transformRules = new FeedTransformRules(transformation); feedSource.transformRules.add(transformRules); @@ -239,35 +235,18 @@ void canPreserveCustomFieldsInStops() throws IOException { zipFolderFiles("fake-agency-with-only-calendar-dates") ); LOG.info("Checking assertions."); - try ( - CsvMapReader finalStopsReader = getCsvMapReaderFromZip(targetVersion.retrieveGtfsFile(), "stops.txt"); - CsvMapReader originalStopsReader = getCsvMapReaderFromZip(sourceVersion.retrieveGtfsFile(), "stops.txt") - ) { - String[] finalHeaders = finalStopsReader.getHeader(true); - String[] originalHeaders = originalStopsReader.getHeader(true); - int columnsAdded = finalHeaders.length - originalHeaders.length; - - // Get the number of rows that have been modified with custom fields - Map row; - int updatedRowCount = 0; - while ((row = finalStopsReader.read(finalHeaders)) != null) { - // Count the number of cases where the custom column is not null, that's the number modified. - // *** Assumes that our sample data below includes values for every customColumn for every row. - if (row.get("custom_column1") != null) updatedRowCount++; - assertEquals( - 2, - columnsAdded, - "stops.txt custom column count should equal input csv data # of custom columns" - ); + assertEquals( + 2, + targetVersion.feedTransformResult.tableTransformResults.get(0).customColumnsAdded, + "stops.txt custom column count should equal input csv data # of custom columns" + ); - assertEquals( - 2, - updatedRowCount, - "stops.txt row count modified with custom content should equal input csv data # of custom columns" - ); - } - } + assertEquals( + 2, + targetVersion.feedTransformResult.tableTransformResults.get(0).updatedCount, + "stops.txt row count modified with custom content should equal input csv data # of custom columns" + ); } @Test @@ -281,25 +260,13 @@ void canAddCustomFile() throws IOException { feedSource, zipFolderFiles("fake-agency-with-only-calendar-dates") ); - try (CsvMapReader customCsvReader = getCsvMapReaderFromZip(targetVersion.retrieveGtfsFile(), "custom-file.txt")) { - String[] customHeaders = customCsvReader.getHeader(true); - int rowCount = 0; - while(customCsvReader.read(customHeaders) != null) rowCount++; - LOG.info("Checking assertions."); - assertEquals( - 2, - rowCount, - "custom-file.txt custom row count should equal input csv data # of rows" - ); - } - } - private static CsvMapReader getCsvMapReaderFromZip(File gtfsFile, String table) throws IOException { - ZipFile zipFile = new ZipFile(gtfsFile); - ZipEntry entry = zipFile.getEntry(table); - try (InputStream is = zipFile.getInputStream(entry)) { - return new CsvMapReader(new InputStreamReader(is), CsvPreference.STANDARD_PREFERENCE); - } + LOG.info("Checking assertions."); + assertEquals( + 2, + targetVersion.feedTransformResult.tableTransformResults.get(0).addedCount, + "custom-file.txt custom row count should equal input csv data # of rows" + ); } private static String generateFeedInfo(String feedId) { From 8fecae1d4f2ffc96a05a3fb7d88e5de1b7b6475a Mon Sep 17 00:00:00 2001 From: "philip.cline" Date: Thu, 6 Jul 2023 11:50:08 -0400 Subject: [PATCH 150/248] refactor(transformations): clean up, update pom to match dev --- pom.xml | 2 +- .../transform/PreserveCustomFieldsTransformation.java | 4 +++- .../models/transform/StringTransformation.java | 9 ++++++++- .../manager/jobs/ArbitraryTransformJobTest.java | 11 ++++------- 4 files changed, 16 insertions(+), 10 deletions(-) diff --git a/pom.xml b/pom.xml index e6ea2cf95..33a52e498 100644 --- a/pom.xml +++ b/pom.xml @@ -268,7 +268,7 @@ com.github.conveyal gtfs-lib - 41a6503 + a3e5707 diff --git a/src/main/java/com/conveyal/datatools/manager/models/transform/PreserveCustomFieldsTransformation.java b/src/main/java/com/conveyal/datatools/manager/models/transform/PreserveCustomFieldsTransformation.java index bd97218c9..e247d2db3 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/transform/PreserveCustomFieldsTransformation.java +++ b/src/main/java/com/conveyal/datatools/manager/models/transform/PreserveCustomFieldsTransformation.java @@ -66,7 +66,9 @@ public void transform(FeedTransformZipTarget zipTarget, MonitorableJob.Status st .filter(t -> t.name.equals(table)) .findFirst(); - if (!streamResult.isPresent()) {throw new Exception(String.format("could not find specTable for table %s", table));} + if (!streamResult.isPresent()) { + throw new Exception(String.format("could not find specTable for table %s", table)); + } Table specTable = streamResult.get(); try (FileSystem targetZipFs = FileSystems.newFileSystem(targetZipPath, (ClassLoader) null)) { diff --git a/src/main/java/com/conveyal/datatools/manager/models/transform/StringTransformation.java b/src/main/java/com/conveyal/datatools/manager/models/transform/StringTransformation.java index aa6731a3a..c845d4f37 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/transform/StringTransformation.java +++ b/src/main/java/com/conveyal/datatools/manager/models/transform/StringTransformation.java @@ -48,7 +48,14 @@ public void transform(FeedTransformZipTarget zipTarget, MonitorableJob.Status st int lineCount = (int) csvData.chars().filter(c -> c == NEW_LINE_CHARACTER_CODE).count(); int addedCount = type == TransformType.TABLE_ADDED ? lineCount : 0; int updatedCount = type == TransformType.TABLE_MODIFIED ? lineCount : 0; - zipTarget.feedTransformResult.tableTransformResults.add(new TableTransformResult(tableName, type, 0, updatedCount, addedCount, 0)); + zipTarget.feedTransformResult.tableTransformResults.add(new TableTransformResult( + tableName, + type, + 0, + updatedCount, + addedCount, + 0 + )); } catch (Exception e) { status.fail("Unknown error encountered while transforming zip file", e); } diff --git a/src/test/java/com/conveyal/datatools/manager/jobs/ArbitraryTransformJobTest.java b/src/test/java/com/conveyal/datatools/manager/jobs/ArbitraryTransformJobTest.java index 541503322..3bb40f228 100644 --- a/src/test/java/com/conveyal/datatools/manager/jobs/ArbitraryTransformJobTest.java +++ b/src/test/java/com/conveyal/datatools/manager/jobs/ArbitraryTransformJobTest.java @@ -8,6 +8,7 @@ import com.conveyal.datatools.manager.models.FeedVersion; import com.conveyal.datatools.manager.models.Project; import com.conveyal.datatools.manager.models.Snapshot; +import com.conveyal.datatools.manager.models.TableTransformResult; import com.conveyal.datatools.manager.models.transform.AddCustomFileFromStringTransformation; import com.conveyal.datatools.manager.models.transform.DeleteRecordsTransformation; import com.conveyal.datatools.manager.models.transform.FeedTransformRules; @@ -234,17 +235,15 @@ void canPreserveCustomFieldsInStops() throws IOException { feedSource, zipFolderFiles("fake-agency-with-only-calendar-dates") ); - LOG.info("Checking assertions."); - + TableTransformResult transformResult = targetVersion.feedTransformResult.tableTransformResults.get(0); assertEquals( 2, - targetVersion.feedTransformResult.tableTransformResults.get(0).customColumnsAdded, + transformResult.customColumnsAdded, "stops.txt custom column count should equal input csv data # of custom columns" ); - assertEquals( 2, - targetVersion.feedTransformResult.tableTransformResults.get(0).updatedCount, + transformResult.updatedCount, "stops.txt row count modified with custom content should equal input csv data # of custom columns" ); } @@ -260,8 +259,6 @@ void canAddCustomFile() throws IOException { feedSource, zipFolderFiles("fake-agency-with-only-calendar-dates") ); - - LOG.info("Checking assertions."); assertEquals( 2, targetVersion.feedTransformResult.tableTransformResults.get(0).addedCount, From 988fd5153d248b1d746aaa2285c5eadf77149347 Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Fri, 7 Jul 2023 15:41:30 +0100 Subject: [PATCH 151/248] refactor(Multiple reference tables lookup): Feed merge updated to handle multi ref tables --- pom.xml | 2 +- .../jobs/feedmerge/MergeLineContext.java | 67 ++++----------- .../feedmerge/ReferenceTableDiscovery.java | 83 +++++++++++++++++++ .../manager/jobs/MergeFeedsJobTest.java | 8 +- 4 files changed, 105 insertions(+), 55 deletions(-) create mode 100644 src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/ReferenceTableDiscovery.java diff --git a/pom.xml b/pom.xml index 55c59adc8..34eb973c8 100644 --- a/pom.xml +++ b/pom.xml @@ -268,7 +268,7 @@ com.github.conveyal gtfs-lib - ba747df + 59b568a diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java index 95f6f5420..3f6701079 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java @@ -237,31 +237,6 @@ public void startNewRow() throws IOException { .collect(Collectors.toList()); } - public enum ReferenceTableLookup { - TRIP_SERVICE_ID_KEY( - String.join(":", Table.TRIPS.name, SERVICE_ID, Table.CALENDAR.name, Table.CALENDAR_DATES.name) - ); - - private final String value; - - ReferenceTableLookup(String value) { - this.value = value; - } - - public String getValue() { - return value; - } - - public static ReferenceTableLookup fromValue(String key) { - for (ReferenceTableLookup ref: ReferenceTableLookup.values()) { - if (ref.getValue().equals(key)) { - return ref; - } - } - throw new UnsupportedOperationException(String.format("Unsupported reference table combination: %s.", key)); - } - } - /** * Determine which reference table to use. If there is only one reference use this. If there are multiple references * determine the context and then the correct reference table to use. @@ -271,41 +246,27 @@ private Table getReferenceTable(FieldContext fieldContext, Field field) { return field.referenceTables.iterator().next(); } - // A table reference is still required even if no match is found. - Table defaultTable = null; - switch (ReferenceTableLookup.fromValue(createKey(field))) { + switch (ReferenceTableDiscovery.getReferenceTableKey(field, table)) { case TRIP_SERVICE_ID_KEY: - boolean isCalendarServiceId = mergeFeedsResult.calendarServiceIds.contains(fieldContext.getValueToWrite()); - boolean isCalendarDatesServiceId = mergeFeedsResult.calendarDatesServiceIds.contains(fieldContext.getValueToWrite()); - if (isCalendarServiceId && !isCalendarDatesServiceId) { - return Table.CALENDAR; - } else if (!isCalendarServiceId && isCalendarDatesServiceId) { - return Table.CALENDAR_DATES; - } else { - // For this case it doesn't seem to matter which is returned so going with calendar. - defaultTable = Table.CALENDAR; - } - // Add other cases as multiple references are added e.g. flex!. + return ReferenceTableDiscovery.getTripServiceIdReferenceTable( + fieldContext, + mergeFeedsResult, + getTableScopedValue(Table.CALENDAR, fieldContext.getValue()), + getTableScopedValue(Table.CALENDAR_DATES, fieldContext.getValue()) + ); + // Include other cases as multiple references are added e.g. flex!. + default: + return null; } - return defaultTable; - } - - /** - * Create a unique key for this table, field and reference tables. - */ - private String createKey(Field field) { - return String.format( - "%s:%s:%s", - table.name, - field.name, - field.referenceTables.stream().map(r -> r.name).collect(Collectors.joining(":")) - ); } public boolean checkForeignReferences(FieldContext fieldContext) throws IOException { Field field = fieldContext.getField(); if (field.isForeignReference()) { - String key = getTableScopedValue(getReferenceTable(fieldContext, field), fieldContext.getValue()); + Table refTable = getReferenceTable(fieldContext, field); + String key = (refTable != null) + ? getTableScopedValue(refTable, fieldContext.getValue()) + : "unknown"; // Check if we're performing a service period merge, this ref field is a service_id, and it // is not found in the list of service_ids (e.g., it was removed). boolean isValidServiceId = mergeFeedsResult.serviceIds.contains(fieldContext.getValueToWrite()); diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/ReferenceTableDiscovery.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/ReferenceTableDiscovery.java new file mode 100644 index 000000000..9f08d18a3 --- /dev/null +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/ReferenceTableDiscovery.java @@ -0,0 +1,83 @@ +package com.conveyal.datatools.manager.jobs.feedmerge; + +import com.conveyal.gtfs.loader.Field; +import com.conveyal.gtfs.loader.Table; + +import java.util.stream.Collectors; + +import static com.conveyal.datatools.manager.jobs.feedmerge.MergeLineContext.SERVICE_ID; + +public class ReferenceTableDiscovery { + + public static final String REF_TABLE_SEPARATOR = "#~#"; + + public enum ReferenceTableKey { + + TRIP_SERVICE_ID_KEY( + String.join(REF_TABLE_SEPARATOR, Table.TRIPS.name, SERVICE_ID, Table.CALENDAR.name, Table.CALENDAR_DATES.name) + ); + + private final String value; + + ReferenceTableKey(String value) { + this.value = value; + } + + public String getValue() { + return value; + } + + public static ReferenceTableKey fromValue(String key) { + for (ReferenceTableKey ref: ReferenceTableKey.values()) { + if (ref.getValue().equals(key)) { + return ref; + } + } + throw new UnsupportedOperationException(String.format("Unsupported reference table key: %s.", key)); + } + } + + /** + * Get reference table key by matching the provided values to predefined reference table keys. + */ + public static ReferenceTableKey getReferenceTableKey(Field field, Table table) { + return ReferenceTableKey.fromValue(createKey(field, table)); + } + + /** + * Create a unique key for this table, field and reference tables. + */ + public static String createKey(Field field, Table table) { + return String.format( + "%s%s%s%s%s", + table.name, + REF_TABLE_SEPARATOR, + field.name, + REF_TABLE_SEPARATOR, + field.referenceTables.stream().map(r -> r.name).collect(Collectors.joining(REF_TABLE_SEPARATOR)) + ); + } + + /** + * Define the reference table for a trip service id. + */ + public static Table getTripServiceIdReferenceTable( + FieldContext fieldContext, + MergeFeedsResult mergeFeedsResult, + String calendarKey, + String calendarDatesKey + ) { + if ( + mergeFeedsResult.calendarServiceIds.contains(fieldContext.getValueToWrite()) || + mergeFeedsResult.skippedIds.contains(calendarKey) + ) { + return Table.CALENDAR; + } else if ( + mergeFeedsResult.calendarDatesServiceIds.contains(fieldContext.getValueToWrite()) || + mergeFeedsResult.skippedIds.contains(calendarDatesKey) + ) { + return Table.CALENDAR_DATES; + } + return null; + } +} diff --git a/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java b/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java index 430adbfcc..1670f5c3d 100644 --- a/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java +++ b/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java @@ -765,7 +765,7 @@ void canMergeFeedsWithMTCForServiceIds4 () throws SQLException { mergeFeedsJob.run(); assertFeedMergeSucceeded(mergeFeedsJob); SqlAssert sqlAssert = new SqlAssert(mergeFeedsJob.mergedVersion); - // FIXME: "version3" contains ref integrity errors... was hat intentional? + // FIXME: "version3" contains ref integrity errors... was that intentional? // sqlAssert.assertNoRefIntegrityErrors(); // - calendar table should have 3 records. @@ -775,12 +775,18 @@ void canMergeFeedsWithMTCForServiceIds4 () throws SQLException { // all records from future feed and keep_one from the active feed. sqlAssert.calendarDates.assertCount(3); + // Calendar dates service exception should still be available. + sqlAssert.calendarDates.assertCount(1, "service_id='Fake_Agency5:keep_one'"); + // - trips table should have 3 records. sqlAssert.trips.assertCount(3); // common_id service_id should be scoped for earlier feed version. sqlAssert.trips.assertCount(1, "service_id='Fake_Agency5:common_id'"); + // service_id should still reference calendar dates service exception. + sqlAssert.trips.assertCount(1, "service_id='Fake_Agency5:keep_one'"); + // Amended calendar record from earlier feed version should also have a modified end date (one day before the // earliest start_date from the future feed). sqlAssert.calendar.assertCount(1, "service_id='Fake_Agency5:common_id' AND end_date='20170914'"); From bf52aae8c9233943c5216d43f360e73487090b74 Mon Sep 17 00:00:00 2001 From: "philip.cline" Date: Fri, 7 Jul 2023 14:26:33 -0400 Subject: [PATCH 152/248] chore(deps): update GTFS-lib ref --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 33a52e498..628aa09fe 100644 --- a/pom.xml +++ b/pom.xml @@ -268,7 +268,7 @@ com.github.conveyal gtfs-lib - a3e5707 + f2ceb59 From 9b71c0cf8a1383d2ef2c0b99097887052f3e0d15 Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Thu, 13 Jul 2023 09:37:05 -0400 Subject: [PATCH 153/248] chores(deps): upgrade MobilityData GTFS Validator to version 4.1.0 --- pom.xml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/pom.xml b/pom.xml index a265b5442..718d327aa 100644 --- a/pom.xml +++ b/pom.xml @@ -259,7 +259,11 @@ 5.5.2 test - + + com.github.mobilitydata + gtfs-validator + 4.1.0 + - f2ceb59 + 9837b6499796a0eeb5de76314e6a1f3125d695fb From f6deff34def8901ddb4e544ece9a53d122025c5b Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Fri, 11 Aug 2023 12:25:55 -0400 Subject: [PATCH 159/248] refactor: address pr feedback --- .../jobs/validation/SharedStopsValidator.java | 30 ++++++++++++------- .../datatools/manager/models/FeedVersion.java | 7 +++-- 2 files changed, 25 insertions(+), 12 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/validation/SharedStopsValidator.java b/src/main/java/com/conveyal/datatools/manager/jobs/validation/SharedStopsValidator.java index 2795aa726..41084d0e5 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/validation/SharedStopsValidator.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/validation/SharedStopsValidator.java @@ -1,5 +1,6 @@ package com.conveyal.datatools.manager.jobs.validation; +import com.conveyal.datatools.common.utils.aws.S3Utils; import com.conveyal.datatools.manager.models.Project; import com.conveyal.gtfs.error.NewGTFSError; import com.conveyal.gtfs.error.NewGTFSErrorType; @@ -8,6 +9,8 @@ import com.conveyal.gtfs.model.Stop; import com.conveyal.gtfs.validator.FeedValidator; import com.csvreader.CsvReader; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.IOException; import java.util.ArrayList; @@ -18,6 +21,8 @@ public class SharedStopsValidator extends FeedValidator { + private static final Logger LOG = LoggerFactory.getLogger(S3Utils.class); + Feed feed; Project project; @@ -29,7 +34,7 @@ public SharedStopsValidator(Project project) { // This method can only be run on a SharedStopsValidator that has been set up with a project only public SharedStopsValidator buildSharedStopsValidator(Feed feed, SQLErrorStorage errorStorage) { if (this.project == null) { - throw new RuntimeException("buildSharedStopsValidator can only be called with a project already set!"); + throw new RuntimeException("Shared stops validator can not be called because no project has been set!"); } return new SharedStopsValidator(feed, errorStorage, this.project); } @@ -48,9 +53,9 @@ public void validate() { CsvReader configReader = CsvReader.parse(config); - int STOP_GROUP_ID_INDEX = 0; - int STOP_ID_INDEX = 2; - int IS_PRIMARY_INDEX = 3; + final int STOP_GROUP_ID_INDEX = 0; + final int STOP_ID_INDEX = 2; + final int IS_PRIMARY_INDEX = 3; // Build list of stop Ids. List stopIds = new ArrayList<>(); @@ -75,14 +80,17 @@ public void validate() { continue; } - if (!stopGroupStops.containsKey(stopGroupId)) { - stopGroupStops.put(stopGroupId, new HashSet<>()); - } + stopGroupStops.putIfAbsent(stopGroupId, new HashSet<>()); Set seenStopIds = stopGroupStops.get(stopGroupId); // Check for SS_01 (stop id appearing in multiple stop groups) if (seenStopIds.contains(stopId)) { - registerError(stops.stream().filter(stop -> stop.stop_id.equals(stopId)).findFirst().orElse(stops.get(0)), NewGTFSErrorType.MULTIPLE_SHARED_STOPS_GROUPS); + registerError(stops + .stream() + .filter(stop -> stop.stop_id.equals(stopId)) + .findFirst() + .orElse(stops.get(0)), NewGTFSErrorType.MULTIPLE_SHARED_STOPS_GROUPS + ); } else { seenStopIds.add(stopId); } @@ -101,8 +109,10 @@ public void validate() { } } } catch (IOException e) { - // TODO Fail gracefully - throw new RuntimeException(e); + LOG.error(e.toString()); + } + finally { + configReader.close(); } diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java b/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java index e160b1a9c..69edac06f 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java @@ -374,8 +374,11 @@ public void validate(MonitorableJob.Status status) { FeedSource fs = Persistence.feedSources.getById(this.feedSourceId); SharedStopsValidator ssv = new SharedStopsValidator(fs.retrieveProject()); - validationResult = GTFS.validate(feedLoadResult.uniqueIdentifier, DataManager.GTFS_DATA_SOURCE, - RouteTypeValidatorBuilder::buildRouteValidator, ssv::buildSharedStopsValidator + validationResult = GTFS.validate( + feedLoadResult.uniqueIdentifier, + DataManager.GTFS_DATA_SOURCE, + RouteTypeValidatorBuilder::buildRouteValidator, + ssv::buildSharedStopsValidator ); } } catch (Exception e) { From 268b949ba5a8a22b43dc9d59795848696ec9f898 Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Mon, 14 Aug 2023 12:08:54 +0100 Subject: [PATCH 160/248] refactor(Added project id to feed source summary): --- .../datatools/manager/models/FeedSourceSummary.java | 12 ++++++++---- .../controllers/api/FeedSourceControllerTest.java | 2 ++ 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java b/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java index ccce4a0da..1f380d541 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java @@ -34,7 +34,10 @@ public class FeedSourceSummary { private static final DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyyMMdd"); + public String projectId; + public String id; + public String name; public boolean deployable; public boolean isPublic; @@ -72,7 +75,8 @@ public class FeedSourceSummary { public FeedSourceSummary() { } - public FeedSourceSummary(Document feedSourceDocument) { + public FeedSourceSummary(String projectId, Document feedSourceDocument) { + this.projectId = projectId; this.id = feedSourceDocument.getString("_id"); this.name = feedSourceDocument.getString("name"); this.deployable = feedSourceDocument.getBoolean("deployable"); @@ -151,7 +155,7 @@ public static List getFeedSourceSummaries(String projectId) { ), sort(Sorts.ascending("name")) ); - return extractFeedSourceSummaries(stages); + return extractFeedSourceSummaries(projectId, stages); } /** @@ -426,10 +430,10 @@ public static Map getFeedVersionsFromPinnedDeploymen /** * Produce a list of all feed source summaries for a project. */ - private static List extractFeedSourceSummaries(List stages) { + private static List extractFeedSourceSummaries(String projectId, List stages) { List feedSourceSummaries = new ArrayList<>(); for (Document feedSourceDocument : Persistence.getDocuments("FeedSource", stages)) { - feedSourceSummaries.add(new FeedSourceSummary(feedSourceDocument)); + feedSourceSummaries.add(new FeedSourceSummary(projectId, feedSourceDocument)); } return feedSourceSummaries; } diff --git a/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java b/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java index 217b05586..1f58ea1ce 100644 --- a/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java +++ b/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java @@ -342,6 +342,7 @@ void canRetrieveDeployedFeedVersionFromLatestDeployment() throws IOException { assertNotNull(feedSourceSummaries); assertEquals(feedSourceWithLatestDeploymentFeedVersion.id, feedSourceSummaries.get(0).id); + assertEquals(feedSourceWithLatestDeploymentFeedVersion.projectId, feedSourceSummaries.get(0).projectId); assertEquals(feedSourceWithLatestDeploymentFeedVersion.labelIds, feedSourceSummaries.get(0).labelIds); assertEquals(feedVersionFromLatestDeployment.id, feedSourceSummaries.get(0).deployedFeedVersionId); assertEquals(feedVersionFromLatestDeployment.validationSummary().startDate, feedSourceSummaries.get(0).deployedFeedVersionStartDate); @@ -372,6 +373,7 @@ void canRetrieveDeployedFeedVersionFromPinnedDeployment() throws IOException { ); assertNotNull(feedSourceSummaries); assertEquals(feedSourceWithPinnedDeploymentFeedVersion.id, feedSourceSummaries.get(0).id); + assertEquals(feedSourceWithPinnedDeploymentFeedVersion.projectId, feedSourceSummaries.get(0).projectId); assertEquals(feedSourceWithPinnedDeploymentFeedVersion.labelIds, feedSourceSummaries.get(0).labelIds); assertEquals(feedVersionFromPinnedDeployment.id, feedSourceSummaries.get(0).deployedFeedVersionId); assertEquals(feedVersionFromPinnedDeployment.validationSummary().startDate, feedSourceSummaries.get(0).deployedFeedVersionStartDate); From 1f580545d80a57f456ee337492f5b5fb6a3d05d5 Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Mon, 14 Aug 2023 12:19:27 +0100 Subject: [PATCH 161/248] refactor(pom.xml): Updated gtfs-lib version --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index a265b5442..1566ccefa 100644 --- a/pom.xml +++ b/pom.xml @@ -268,7 +268,7 @@ com.github.conveyal gtfs-lib - f2ceb59 + f2ceb59027 From 2cc5aab3ea2126d67d5c650920b4e76a11ef732d Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Wed, 16 Aug 2023 10:39:30 +0100 Subject: [PATCH 162/248] refactor(pom.xml): Addressed short hash for gtfs lib Jitpack now requires 10 instead of 7 characters --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index a265b5442..1566ccefa 100644 --- a/pom.xml +++ b/pom.xml @@ -268,7 +268,7 @@ com.github.conveyal gtfs-lib - f2ceb59 + f2ceb59027 From d43f40afa415fc4a112395f1ee0fea23ee6da06e Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Wed, 16 Aug 2023 11:28:22 +0100 Subject: [PATCH 163/248] refactor(Disable auth for tests): --- pom.xml | 2 +- .../manager/gtfsplus/GtfsPlusValidationTest.java | 3 +++ .../datatools/manager/jobs/ArbitraryTransformJobTest.java | 3 +++ .../datatools/manager/jobs/AutoPublishJobTest.java | 3 +++ .../manager/jobs/DeploymentGisExportJobTest.java | 8 ++++++++ .../manager/jobs/FetchLoadFeedCombinationTest.java | 3 +++ .../conveyal/datatools/manager/jobs/GisExportJobTest.java | 8 ++++++++ .../datatools/manager/jobs/MergeFeedsJobTest.java | 3 +++ .../datatools/manager/models/FeedVersionTest.java | 3 +++ 9 files changed, 35 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index a265b5442..1566ccefa 100644 --- a/pom.xml +++ b/pom.xml @@ -268,7 +268,7 @@ com.github.conveyal gtfs-lib - f2ceb59 + f2ceb59027 diff --git a/src/test/java/com/conveyal/datatools/manager/gtfsplus/GtfsPlusValidationTest.java b/src/test/java/com/conveyal/datatools/manager/gtfsplus/GtfsPlusValidationTest.java index ac24c29bb..d1fe8924b 100644 --- a/src/test/java/com/conveyal/datatools/manager/gtfsplus/GtfsPlusValidationTest.java +++ b/src/test/java/com/conveyal/datatools/manager/gtfsplus/GtfsPlusValidationTest.java @@ -3,6 +3,7 @@ import com.conveyal.datatools.DatatoolsTest; import com.conveyal.datatools.UnitTest; import com.conveyal.datatools.manager.DataManager; +import com.conveyal.datatools.manager.auth.Auth0Connection; import com.conveyal.datatools.manager.jobs.MergeFeedsJobTest; import com.conveyal.datatools.manager.models.FeedSource; import com.conveyal.datatools.manager.models.FeedVersion; @@ -44,6 +45,7 @@ public class GtfsPlusValidationTest extends UnitTest { public static void setUp() throws IOException { // Start server if it isn't already running. DatatoolsTest.setUp(); + Auth0Connection.setAuthDisabled(true); // Create a project, feed sources, and feed versions to merge. project = new Project(); project.name = String.format("Test %s", new Date()); @@ -65,6 +67,7 @@ public static void setUp() throws IOException { @AfterAll static void tearDown() { + Auth0Connection.setAuthDisabled(Auth0Connection.getDefaultAuthDisabled()); project.delete(); } diff --git a/src/test/java/com/conveyal/datatools/manager/jobs/ArbitraryTransformJobTest.java b/src/test/java/com/conveyal/datatools/manager/jobs/ArbitraryTransformJobTest.java index 3bb40f228..a8924973e 100644 --- a/src/test/java/com/conveyal/datatools/manager/jobs/ArbitraryTransformJobTest.java +++ b/src/test/java/com/conveyal/datatools/manager/jobs/ArbitraryTransformJobTest.java @@ -2,6 +2,7 @@ import com.conveyal.datatools.DatatoolsTest; import com.conveyal.datatools.UnitTest; +import com.conveyal.datatools.manager.auth.Auth0Connection; import com.conveyal.datatools.manager.auth.Auth0UserProfile; import com.conveyal.datatools.manager.models.FeedRetrievalMethod; import com.conveyal.datatools.manager.models.FeedSource; @@ -65,6 +66,7 @@ public class ArbitraryTransformJobTest extends UnitTest { public static void setUp() throws IOException { // start server if it isn't already running DatatoolsTest.setUp(); + Auth0Connection.setAuthDisabled(true); // Create a project, feed sources, and feed versions to merge. project = new Project(); @@ -81,6 +83,7 @@ public static void setUp() throws IOException { */ @AfterAll public static void tearDown() { + Auth0Connection.setAuthDisabled(Auth0Connection.getDefaultAuthDisabled()); // Project delete cascades to feed sources. project.delete(); } diff --git a/src/test/java/com/conveyal/datatools/manager/jobs/AutoPublishJobTest.java b/src/test/java/com/conveyal/datatools/manager/jobs/AutoPublishJobTest.java index 3c21a1599..4b15cbd56 100644 --- a/src/test/java/com/conveyal/datatools/manager/jobs/AutoPublishJobTest.java +++ b/src/test/java/com/conveyal/datatools/manager/jobs/AutoPublishJobTest.java @@ -3,6 +3,7 @@ import com.amazonaws.services.s3.model.S3ObjectSummary; import com.conveyal.datatools.DatatoolsTest; import com.conveyal.datatools.UnitTest; +import com.conveyal.datatools.manager.auth.Auth0Connection; import com.conveyal.datatools.manager.auth.Auth0UserProfile; import com.conveyal.datatools.manager.models.ExternalFeedSourceProperty; import com.conveyal.datatools.manager.models.FeedSource; @@ -52,6 +53,7 @@ public class AutoPublishJobTest extends UnitTest { public static void setUp() throws IOException { // start server if it isn't already running DatatoolsTest.setUp(); + Auth0Connection.setAuthDisabled(true); // Create a project, feed sources, and feed versions to merge. project = new Project(); @@ -76,6 +78,7 @@ public static void setUp() throws IOException { @AfterAll public static void tearDown() { + Auth0Connection.setAuthDisabled(Auth0Connection.getDefaultAuthDisabled()); if (project != null) { project.delete(); } diff --git a/src/test/java/com/conveyal/datatools/manager/jobs/DeploymentGisExportJobTest.java b/src/test/java/com/conveyal/datatools/manager/jobs/DeploymentGisExportJobTest.java index aea3a6f16..133864c31 100644 --- a/src/test/java/com/conveyal/datatools/manager/jobs/DeploymentGisExportJobTest.java +++ b/src/test/java/com/conveyal/datatools/manager/jobs/DeploymentGisExportJobTest.java @@ -1,11 +1,13 @@ package com.conveyal.datatools.manager.jobs; import com.conveyal.datatools.DatatoolsTest; +import com.conveyal.datatools.manager.auth.Auth0Connection; import com.conveyal.datatools.manager.auth.Auth0UserProfile; import com.conveyal.datatools.manager.models.*; import com.conveyal.datatools.manager.persistence.Persistence; import com.google.common.io.Files; +import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.slf4j.Logger; @@ -63,6 +65,7 @@ private File[] getFoldersFromZippedShapefile(File zipFile) throws IOException { @BeforeAll public static void setUp() throws IOException { DatatoolsTest.setUp(); + Auth0Connection.setAuthDisabled(true); // Calling GisExportJobTest.setUp() creates project and feeds GisExportJobTest.setUp(); user = Auth0UserProfile.createTestAdminUser(); @@ -72,6 +75,11 @@ public static void setUp() throws IOException { Persistence.deployments.create(deployment); } + @AfterAll + public static void tearDown() { + Auth0Connection.setAuthDisabled(Auth0Connection.getDefaultAuthDisabled()); + } + /** * Ensures that shapefiles containing stop features for a deployment can be exported and * contain geometry for each stop. diff --git a/src/test/java/com/conveyal/datatools/manager/jobs/FetchLoadFeedCombinationTest.java b/src/test/java/com/conveyal/datatools/manager/jobs/FetchLoadFeedCombinationTest.java index 56c99d32c..f7dc2a33b 100644 --- a/src/test/java/com/conveyal/datatools/manager/jobs/FetchLoadFeedCombinationTest.java +++ b/src/test/java/com/conveyal/datatools/manager/jobs/FetchLoadFeedCombinationTest.java @@ -3,6 +3,7 @@ import com.conveyal.datatools.DatatoolsTest; import com.conveyal.datatools.UnitTest; import com.conveyal.datatools.common.status.MonitorableJob; +import com.conveyal.datatools.manager.auth.Auth0Connection; import com.conveyal.datatools.manager.auth.Auth0UserProfile; import com.conveyal.datatools.manager.models.FeedSource; import com.conveyal.datatools.manager.models.FeedVersion; @@ -53,6 +54,7 @@ public class FetchLoadFeedCombinationTest extends UnitTest { public static void setUp() throws IOException { // start server if it isn't already running DatatoolsTest.setUp(); + Auth0Connection.setAuthDisabled(true); // Create a project and feed sources. project = new Project(); @@ -70,6 +72,7 @@ public static void setUp() throws IOException { @AfterAll public static void tearDown() { + Auth0Connection.setAuthDisabled(Auth0Connection.getDefaultAuthDisabled()); wireMockServer.stop(); if (project != null) { project.delete(); diff --git a/src/test/java/com/conveyal/datatools/manager/jobs/GisExportJobTest.java b/src/test/java/com/conveyal/datatools/manager/jobs/GisExportJobTest.java index e42976802..6d5fd163d 100644 --- a/src/test/java/com/conveyal/datatools/manager/jobs/GisExportJobTest.java +++ b/src/test/java/com/conveyal/datatools/manager/jobs/GisExportJobTest.java @@ -2,6 +2,7 @@ import com.conveyal.datatools.DatatoolsTest; import com.conveyal.datatools.UnitTest; +import com.conveyal.datatools.manager.auth.Auth0Connection; import com.conveyal.datatools.manager.auth.Auth0UserProfile; import com.conveyal.datatools.manager.models.FeedRetrievalMethod; import com.conveyal.datatools.manager.models.FeedSource; @@ -11,6 +12,7 @@ import com.conveyal.datatools.manager.utils.SqlAssert; import com.google.common.base.Strings; import com.google.common.io.Files; +import org.junit.jupiter.api.AfterAll; import org.locationtech.jts.geom.Coordinate; import org.locationtech.jts.geom.MultiLineString; import org.locationtech.jts.geom.Point; @@ -149,6 +151,7 @@ public void validateShapefiles(File[] files, String agencyName, GisExportJob.Exp @BeforeAll public static void setUp() throws IOException { DatatoolsTest.setUp(); + Auth0Connection.setAuthDisabled(true); LOG.info("{} setup", GisExportJobTest.class.getSimpleName()); // Create a project, feed sources, and feed versions to merge. @@ -165,6 +168,11 @@ public static void setUp() throws IOException { hawaiiVersion = createFeedVersionFromGtfsZip(hawaii, "hawaii_fake_no_shapes.zip"); } + @AfterAll + public static void tearDown() { + Auth0Connection.setAuthDisabled(Auth0Connection.getDefaultAuthDisabled()); + } + /** * Ensures that a shapefile containing stop features for a feed version can be exported and * contains geometry for each stop. diff --git a/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java b/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java index 430adbfcc..50ae60586 100644 --- a/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java +++ b/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java @@ -2,6 +2,7 @@ import com.conveyal.datatools.DatatoolsTest; import com.conveyal.datatools.UnitTest; +import com.conveyal.datatools.manager.auth.Auth0Connection; import com.conveyal.datatools.manager.auth.Auth0UserProfile; import com.conveyal.datatools.manager.gtfsplus.GtfsPlusValidation; import com.conveyal.datatools.manager.jobs.feedmerge.MergeFeedsType; @@ -86,6 +87,7 @@ public class MergeFeedsJobTest extends UnitTest { public static void setUp() throws IOException { // start server if it isn't already running DatatoolsTest.setUp(); + Auth0Connection.setAuthDisabled(true); // Create a project, feed sources, and feed versions to merge. project = new Project(); @@ -157,6 +159,7 @@ public static void setUp() throws IOException { */ @AfterAll public static void tearDown() { + Auth0Connection.setAuthDisabled(Auth0Connection.getDefaultAuthDisabled()); if (project != null) { project.delete(); } diff --git a/src/test/java/com/conveyal/datatools/manager/models/FeedVersionTest.java b/src/test/java/com/conveyal/datatools/manager/models/FeedVersionTest.java index 9bbca2755..13a90b0da 100644 --- a/src/test/java/com/conveyal/datatools/manager/models/FeedVersionTest.java +++ b/src/test/java/com/conveyal/datatools/manager/models/FeedVersionTest.java @@ -2,6 +2,7 @@ import com.conveyal.datatools.DatatoolsTest; import com.conveyal.datatools.UnitTest; +import com.conveyal.datatools.manager.auth.Auth0Connection; import com.conveyal.datatools.manager.persistence.Persistence; import com.conveyal.gtfs.error.NewGTFSError; import com.conveyal.gtfs.error.NewGTFSErrorType; @@ -35,6 +36,7 @@ public class FeedVersionTest extends UnitTest { public static void setUp() throws Exception { // start server if it isn't already running DatatoolsTest.setUp(); + Auth0Connection.setAuthDisabled(true); // set up project project = new Project(); @@ -48,6 +50,7 @@ public static void setUp() throws Exception { @AfterAll public static void tearDown() { + Auth0Connection.setAuthDisabled(Auth0Connection.getDefaultAuthDisabled()); if (project != null) { project.delete(); } From ca60d4bfedb370ad0e7fc6b726c7527002c16499 Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Wed, 16 Aug 2023 11:44:37 +0100 Subject: [PATCH 164/248] refactor(NormalizeFieldTransformJobTest): Disabled auth --- .../datatools/manager/jobs/NormalizeFieldTransformJobTest.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/test/java/com/conveyal/datatools/manager/jobs/NormalizeFieldTransformJobTest.java b/src/test/java/com/conveyal/datatools/manager/jobs/NormalizeFieldTransformJobTest.java index 8eb99259e..6a35edb56 100644 --- a/src/test/java/com/conveyal/datatools/manager/jobs/NormalizeFieldTransformJobTest.java +++ b/src/test/java/com/conveyal/datatools/manager/jobs/NormalizeFieldTransformJobTest.java @@ -1,6 +1,7 @@ package com.conveyal.datatools.manager.jobs; import com.conveyal.datatools.DatatoolsTest; +import com.conveyal.datatools.manager.auth.Auth0Connection; import com.conveyal.datatools.manager.models.FeedRetrievalMethod; import com.conveyal.datatools.manager.models.FeedSource; import com.conveyal.datatools.manager.models.FeedVersion; @@ -48,6 +49,7 @@ public class NormalizeFieldTransformJobTest extends DatatoolsTest { public static void setUp() throws IOException { // start server if it isn't already running DatatoolsTest.setUp(); + Auth0Connection.setAuthDisabled(true); // Create a project. project = createProject(); @@ -58,6 +60,7 @@ public static void setUp() throws IOException { */ @AfterAll public static void tearDown() { + Auth0Connection.setAuthDisabled(Auth0Connection.getDefaultAuthDisabled()); // Project delete cascades to feed sources. project.delete(); } From 89f6f11c0ac44da433ab4689797da71afbb2419c Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Wed, 16 Aug 2023 12:13:07 +0100 Subject: [PATCH 165/248] refactor(Disabled auth): --- .../java/com/conveyal/datatools/HandleCorruptGTFSFileTest.java | 3 +++ .../datatools/manager/extensions/mtc/MtcFeedResourceTest.java | 3 +++ 2 files changed, 6 insertions(+) diff --git a/src/test/java/com/conveyal/datatools/HandleCorruptGTFSFileTest.java b/src/test/java/com/conveyal/datatools/HandleCorruptGTFSFileTest.java index 31aefd4fa..72e536dd4 100644 --- a/src/test/java/com/conveyal/datatools/HandleCorruptGTFSFileTest.java +++ b/src/test/java/com/conveyal/datatools/HandleCorruptGTFSFileTest.java @@ -1,6 +1,7 @@ package com.conveyal.datatools; import com.conveyal.datatools.common.status.MonitorableJob; +import com.conveyal.datatools.manager.auth.Auth0Connection; import com.conveyal.datatools.manager.jobs.LoadFeedJob; import com.conveyal.datatools.manager.jobs.ProcessSingleFeedJob; import com.conveyal.datatools.manager.jobs.ValidateFeedJob; @@ -30,10 +31,12 @@ public class HandleCorruptGTFSFileTest { public static void setUp() throws IOException { // start server if it isn't already running DatatoolsTest.setUp(); + Auth0Connection.setAuthDisabled(true); } @AfterAll public static void tearDown() { + Auth0Connection.setAuthDisabled(Auth0Connection.getDefaultAuthDisabled()); mockProject.delete(); } diff --git a/src/test/java/com/conveyal/datatools/manager/extensions/mtc/MtcFeedResourceTest.java b/src/test/java/com/conveyal/datatools/manager/extensions/mtc/MtcFeedResourceTest.java index 741627985..3a6cb6a31 100644 --- a/src/test/java/com/conveyal/datatools/manager/extensions/mtc/MtcFeedResourceTest.java +++ b/src/test/java/com/conveyal/datatools/manager/extensions/mtc/MtcFeedResourceTest.java @@ -2,6 +2,7 @@ import com.conveyal.datatools.DatatoolsTest; import com.conveyal.datatools.UnitTest; +import com.conveyal.datatools.manager.auth.Auth0Connection; import com.conveyal.datatools.manager.models.ExternalFeedSourceProperty; import com.conveyal.datatools.manager.models.FeedSource; import com.conveyal.datatools.manager.models.FeedVersion; @@ -50,6 +51,7 @@ class MtcFeedResourceTest extends UnitTest { static void setUp() throws IOException { // start server if it isn't already running DatatoolsTest.setUp(); + Auth0Connection.setAuthDisabled(true); // Create a project, feed sources. project = new Project(); project.name = String.format("Test %s", new Date()); @@ -69,6 +71,7 @@ static void setUp() throws IOException { @AfterAll static void tearDown() { + Auth0Connection.setAuthDisabled(Auth0Connection.getDefaultAuthDisabled()); wireMockServer.stop(); if (project != null) { project.delete(); From 026fa2f70d66d6a42650236cad41dda7718beb8f Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Wed, 16 Aug 2023 13:52:49 +0100 Subject: [PATCH 166/248] refactor(SqlSchemaUpdaterTest): Disabled auth --- .../datatools/manager/utils/sql/SqlSchemaUpdaterTest.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/test/java/com/conveyal/datatools/manager/utils/sql/SqlSchemaUpdaterTest.java b/src/test/java/com/conveyal/datatools/manager/utils/sql/SqlSchemaUpdaterTest.java index d3050493f..c374cc010 100644 --- a/src/test/java/com/conveyal/datatools/manager/utils/sql/SqlSchemaUpdaterTest.java +++ b/src/test/java/com/conveyal/datatools/manager/utils/sql/SqlSchemaUpdaterTest.java @@ -3,6 +3,7 @@ import com.conveyal.datatools.DatatoolsTest; import com.conveyal.datatools.UnitTest; import com.conveyal.datatools.manager.DataManager; +import com.conveyal.datatools.manager.auth.Auth0Connection; import com.conveyal.datatools.manager.models.FeedSource; import com.conveyal.datatools.manager.models.FeedVersion; import com.conveyal.datatools.manager.models.Project; @@ -40,6 +41,7 @@ class SqlSchemaUpdaterTest extends UnitTest { public static void setUp() throws IOException { // start server if it isn't already running. DatatoolsTest.setUp(); + Auth0Connection.setAuthDisabled(true); // Create a project and feed sources. project = new Project(); @@ -60,6 +62,7 @@ public static void setUp() throws IOException { */ @AfterAll public static void tearDown() { + Auth0Connection.setAuthDisabled(Auth0Connection.getDefaultAuthDisabled()); // Project delete cascades to feed sources. project.delete(); } From b76e31831e1ad70961dee799682892fab2498c3a Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Wed, 16 Aug 2023 13:41:55 -0400 Subject: [PATCH 167/248] refactor: address pr feedback --- .../manager/jobs/validation/SharedStopsValidator.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/validation/SharedStopsValidator.java b/src/main/java/com/conveyal/datatools/manager/jobs/validation/SharedStopsValidator.java index 41084d0e5..4e791b608 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/validation/SharedStopsValidator.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/validation/SharedStopsValidator.java @@ -108,9 +108,7 @@ public void validate() { registerError(NewGTFSError.forFeed(NewGTFSErrorType.SHARED_STOP_GROUP_ENTITY_DOES_NOT_EXIST, stopId)); } } - } catch (IOException e) { - LOG.error(e.toString()); - } + } catch (IOException e) { LOG.error(e.toString()); } finally { configReader.close(); } From 36391dc84c6034b612acb1171b1138c8334f3b05 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 17 Aug 2023 15:07:17 +0000 Subject: [PATCH 168/248] chore(deps): bump com.google.guava:guava from 30.0-jre to 32.0.0-jre Bumps [com.google.guava:guava](https://github.com/google/guava) from 30.0-jre to 32.0.0-jre. - [Release notes](https://github.com/google/guava/releases) - [Commits](https://github.com/google/guava/commits) --- updated-dependencies: - dependency-name: com.google.guava:guava dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 1566ccefa..658ae2d4a 100644 --- a/pom.xml +++ b/pom.xml @@ -291,7 +291,7 @@ com.google.guava guava - 30.0-jre + 32.0.0-jre javax.xml.bind From be6f44030750a5734372436e775f174dd0dcee52 Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Fri, 18 Aug 2023 09:59:15 +0100 Subject: [PATCH 169/248] refactor(Update to include a new private class to hold latest validation result): This new class ali --- .../manager/models/FeedSourceSummary.java | 46 ++++++++++++------- .../api/FeedSourceControllerTest.java | 16 +++---- 2 files changed, 37 insertions(+), 25 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java b/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java index 1f380d541..931fb49f3 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java @@ -60,17 +60,7 @@ public class FeedSourceSummary { public Integer deployedFeedVersionIssues; - public String latestFeedVersionId; - - @JsonSerialize(using = JacksonSerializers.LocalDateIsoSerializer.class) - @JsonDeserialize(using = JacksonSerializers.LocalDateIsoDeserializer.class) - public LocalDate latestFeedVersionStartDate; - - @JsonSerialize(using = JacksonSerializers.LocalDateIsoSerializer.class) - @JsonDeserialize(using = JacksonSerializers.LocalDateIsoDeserializer.class) - public LocalDate latestFeedVersionEndDate; - - public Integer latestFeedVersionIssues; + public LatestValidationResult latestValidation; public FeedSourceSummary() { } @@ -100,12 +90,7 @@ public void setFeedVersion(FeedVersionSummary feedVersionSummary, boolean isDepl ? null : feedVersionSummary.validationResult.errorCount; } else { - this.latestFeedVersionId = feedVersionSummary.id; - this.latestFeedVersionStartDate = feedVersionSummary.validationResult.firstCalendarDate; - this.latestFeedVersionEndDate = feedVersionSummary.validationResult.lastCalendarDate; - this.latestFeedVersionIssues = (feedVersionSummary.validationResult.errorCount == -1) - ? null - : feedVersionSummary.validationResult.errorCount; + this.latestValidation = new LatestValidationResult(feedVersionSummary); } } } @@ -533,4 +518,31 @@ private static LocalDate getDateFromString(String date) { private static LocalDate getLocalDateFromDate(Date date) { return date.toInstant().atZone(ZoneId.systemDefault()).toLocalDate(); } + + public static class LatestValidationResult { + + public String feedVersionId; + @JsonSerialize(using = JacksonSerializers.LocalDateIsoSerializer.class) + @JsonDeserialize(using = JacksonSerializers.LocalDateIsoDeserializer.class) + public LocalDate feedVersionStartDate; + + @JsonSerialize(using = JacksonSerializers.LocalDateIsoSerializer.class) + @JsonDeserialize(using = JacksonSerializers.LocalDateIsoDeserializer.class) + public LocalDate feedVersionEndDate; + + public Integer feedVersionIssues; + + /** Required for JSON de/serializing. **/ + public LatestValidationResult() {} + + LatestValidationResult(FeedVersionSummary feedVersionSummary) { + this.feedVersionId = feedVersionSummary.id; + this.feedVersionStartDate = feedVersionSummary.validationResult.firstCalendarDate; + this.feedVersionEndDate = feedVersionSummary.validationResult.lastCalendarDate; + this.feedVersionIssues = (feedVersionSummary.validationResult.errorCount == -1) + ? null + : feedVersionSummary.validationResult.errorCount; + } + } + } \ No newline at end of file diff --git a/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java b/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java index 1f58ea1ce..0026e15fc 100644 --- a/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java +++ b/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java @@ -348,10 +348,10 @@ void canRetrieveDeployedFeedVersionFromLatestDeployment() throws IOException { assertEquals(feedVersionFromLatestDeployment.validationSummary().startDate, feedSourceSummaries.get(0).deployedFeedVersionStartDate); assertEquals(feedVersionFromLatestDeployment.validationSummary().endDate, feedSourceSummaries.get(0).deployedFeedVersionEndDate); assertEquals(feedVersionFromLatestDeployment.validationSummary().errorCount, feedSourceSummaries.get(0).deployedFeedVersionIssues); - assertEquals(feedVersionFromLatestDeployment.id, feedSourceSummaries.get(0).latestFeedVersionId); - assertEquals(feedVersionFromLatestDeployment.validationSummary().startDate, feedSourceSummaries.get(0).latestFeedVersionStartDate); - assertEquals(feedVersionFromLatestDeployment.validationSummary().endDate, feedSourceSummaries.get(0).latestFeedVersionEndDate); - assertEquals(feedVersionFromLatestDeployment.validationSummary().errorCount, feedSourceSummaries.get(0).latestFeedVersionIssues); + assertEquals(feedVersionFromLatestDeployment.id, feedSourceSummaries.get(0).latestValidation.feedVersionId); + assertEquals(feedVersionFromLatestDeployment.validationSummary().startDate, feedSourceSummaries.get(0).latestValidation.feedVersionStartDate); + assertEquals(feedVersionFromLatestDeployment.validationSummary().endDate, feedSourceSummaries.get(0).latestValidation.feedVersionEndDate); + assertEquals(feedVersionFromLatestDeployment.validationSummary().errorCount, feedSourceSummaries.get(0).latestValidation.feedVersionIssues); } @Test @@ -379,10 +379,10 @@ void canRetrieveDeployedFeedVersionFromPinnedDeployment() throws IOException { assertEquals(feedVersionFromPinnedDeployment.validationSummary().startDate, feedSourceSummaries.get(0).deployedFeedVersionStartDate); assertEquals(feedVersionFromPinnedDeployment.validationSummary().endDate, feedSourceSummaries.get(0).deployedFeedVersionEndDate); assertEquals(feedVersionFromPinnedDeployment.validationSummary().errorCount, feedSourceSummaries.get(0).deployedFeedVersionIssues); - assertEquals(feedVersionFromPinnedDeployment.id, feedSourceSummaries.get(0).latestFeedVersionId); - assertEquals(feedVersionFromPinnedDeployment.validationSummary().startDate, feedSourceSummaries.get(0).latestFeedVersionStartDate); - assertEquals(feedVersionFromPinnedDeployment.validationSummary().endDate, feedSourceSummaries.get(0).latestFeedVersionEndDate); - assertEquals(feedVersionFromPinnedDeployment.validationSummary().errorCount, feedSourceSummaries.get(0).latestFeedVersionIssues); + assertEquals(feedVersionFromPinnedDeployment.id, feedSourceSummaries.get(0).latestValidation.feedVersionId); + assertEquals(feedVersionFromPinnedDeployment.validationSummary().startDate, feedSourceSummaries.get(0).latestValidation.feedVersionStartDate); + assertEquals(feedVersionFromPinnedDeployment.validationSummary().endDate, feedSourceSummaries.get(0).latestValidation.feedVersionEndDate); + assertEquals(feedVersionFromPinnedDeployment.validationSummary().errorCount, feedSourceSummaries.get(0).latestValidation.feedVersionIssues); } private static FeedSource createFeedSource(String name, URL url, Project project) { From 15d43e7cc8aac9cec6b831a8ac507332b7813200 Mon Sep 17 00:00:00 2001 From: "philip.cline" Date: Fri, 18 Aug 2023 15:10:22 -0400 Subject: [PATCH 170/248] refactor(feedsourceSummary): rename fields to agree with old format --- .../datatools/manager/models/FeedSourceSummary.java | 12 ++++++------ .../controllers/api/FeedSourceControllerTest.java | 12 ++++++------ 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java b/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java index 931fb49f3..d472b05c4 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java @@ -524,22 +524,22 @@ public static class LatestValidationResult { public String feedVersionId; @JsonSerialize(using = JacksonSerializers.LocalDateIsoSerializer.class) @JsonDeserialize(using = JacksonSerializers.LocalDateIsoDeserializer.class) - public LocalDate feedVersionStartDate; + public LocalDate startDate; @JsonSerialize(using = JacksonSerializers.LocalDateIsoSerializer.class) @JsonDeserialize(using = JacksonSerializers.LocalDateIsoDeserializer.class) - public LocalDate feedVersionEndDate; + public LocalDate endDate; - public Integer feedVersionIssues; + public Integer errorCount; /** Required for JSON de/serializing. **/ public LatestValidationResult() {} LatestValidationResult(FeedVersionSummary feedVersionSummary) { this.feedVersionId = feedVersionSummary.id; - this.feedVersionStartDate = feedVersionSummary.validationResult.firstCalendarDate; - this.feedVersionEndDate = feedVersionSummary.validationResult.lastCalendarDate; - this.feedVersionIssues = (feedVersionSummary.validationResult.errorCount == -1) + this.startDate = feedVersionSummary.validationResult.firstCalendarDate; + this.endDate = feedVersionSummary.validationResult.lastCalendarDate; + this.errorCount = (feedVersionSummary.validationResult.errorCount == -1) ? null : feedVersionSummary.validationResult.errorCount; } diff --git a/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java b/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java index 0026e15fc..21faa0dd9 100644 --- a/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java +++ b/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java @@ -349,9 +349,9 @@ void canRetrieveDeployedFeedVersionFromLatestDeployment() throws IOException { assertEquals(feedVersionFromLatestDeployment.validationSummary().endDate, feedSourceSummaries.get(0).deployedFeedVersionEndDate); assertEquals(feedVersionFromLatestDeployment.validationSummary().errorCount, feedSourceSummaries.get(0).deployedFeedVersionIssues); assertEquals(feedVersionFromLatestDeployment.id, feedSourceSummaries.get(0).latestValidation.feedVersionId); - assertEquals(feedVersionFromLatestDeployment.validationSummary().startDate, feedSourceSummaries.get(0).latestValidation.feedVersionStartDate); - assertEquals(feedVersionFromLatestDeployment.validationSummary().endDate, feedSourceSummaries.get(0).latestValidation.feedVersionEndDate); - assertEquals(feedVersionFromLatestDeployment.validationSummary().errorCount, feedSourceSummaries.get(0).latestValidation.feedVersionIssues); + assertEquals(feedVersionFromLatestDeployment.validationSummary().startDate, feedSourceSummaries.get(0).latestValidation.startDate); + assertEquals(feedVersionFromLatestDeployment.validationSummary().endDate, feedSourceSummaries.get(0).latestValidation.endDate); + assertEquals(feedVersionFromLatestDeployment.validationSummary().errorCount, feedSourceSummaries.get(0).latestValidation.errorCount); } @Test @@ -380,9 +380,9 @@ void canRetrieveDeployedFeedVersionFromPinnedDeployment() throws IOException { assertEquals(feedVersionFromPinnedDeployment.validationSummary().endDate, feedSourceSummaries.get(0).deployedFeedVersionEndDate); assertEquals(feedVersionFromPinnedDeployment.validationSummary().errorCount, feedSourceSummaries.get(0).deployedFeedVersionIssues); assertEquals(feedVersionFromPinnedDeployment.id, feedSourceSummaries.get(0).latestValidation.feedVersionId); - assertEquals(feedVersionFromPinnedDeployment.validationSummary().startDate, feedSourceSummaries.get(0).latestValidation.feedVersionStartDate); - assertEquals(feedVersionFromPinnedDeployment.validationSummary().endDate, feedSourceSummaries.get(0).latestValidation.feedVersionEndDate); - assertEquals(feedVersionFromPinnedDeployment.validationSummary().errorCount, feedSourceSummaries.get(0).latestValidation.feedVersionIssues); + assertEquals(feedVersionFromPinnedDeployment.validationSummary().startDate, feedSourceSummaries.get(0).latestValidation.startDate); + assertEquals(feedVersionFromPinnedDeployment.validationSummary().endDate, feedSourceSummaries.get(0).latestValidation.endDate); + assertEquals(feedVersionFromPinnedDeployment.validationSummary().errorCount, feedSourceSummaries.get(0).latestValidation.errorCount); } private static FeedSource createFeedSource(String name, URL url, Project project) { From f7f904d46790de4b4cc4fbd2dfe08466404be4ed Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Mon, 21 Aug 2023 09:44:51 +0100 Subject: [PATCH 171/248] refactor(pom.xml): Bumped gtfs lib to latest commit for exception based scheduling --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index b4b2c27ad..1cd5aba21 100644 --- a/pom.xml +++ b/pom.xml @@ -268,7 +268,7 @@ com.github.conveyal gtfs-lib - 59b568a + 49a722e161a52cae98fba9cb23995dae82fd01e4 From ab782d57e8e8178fcfd6e3262d4141a80c8c7944 Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Thu, 24 Aug 2023 09:09:59 +0100 Subject: [PATCH 172/248] refactor(FeedSourceSummary.java): Addressed PR feedback --- .../manager/models/FeedSourceSummary.java | 21 +++++++++++-------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java b/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java index d472b05c4..b9917e94a 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java @@ -71,7 +71,10 @@ public FeedSourceSummary(String projectId, Document feedSourceDocument) { this.name = feedSourceDocument.getString("name"); this.deployable = feedSourceDocument.getBoolean("deployable"); this.isPublic = feedSourceDocument.getBoolean("isPublic"); - this.labelIds = feedSourceDocument.getList("labelIds", String.class); + List documentLabelIds = feedSourceDocument.getList("labelIds", String.class); + if (documentLabelIds != null) { + this.labelIds = documentLabelIds; + } // Convert to local date type for consistency. this.lastUpdated = getLocalDateFromDate(feedSourceDocument.getDate("lastUpdated")); } @@ -87,7 +90,7 @@ public void setFeedVersion(FeedVersionSummary feedVersionSummary, boolean isDepl this.deployedFeedVersionStartDate = feedVersionSummary.validationResult.firstCalendarDate; this.deployedFeedVersionEndDate = feedVersionSummary.validationResult.lastCalendarDate; this.deployedFeedVersionIssues = (feedVersionSummary.validationResult.errorCount == -1) - ? null + ? 0 : feedVersionSummary.validationResult.errorCount; } else { this.latestValidation = new LatestValidationResult(feedVersionSummary); @@ -341,25 +344,25 @@ public static Map getFeedVersionsFromPinnedDeploymen pinnedDeploymentId: 1 } }, - { + { $lookup:{ from:"Deployment", localField:"pinnedDeploymentId", foreignField:"_id", as:"deployment" } - }, - { - $unwind: "$deployment" - }, - { + }, + { + $unwind: "$deployment" + }, + { $lookup:{ from:"FeedVersion", localField:"deployment.feedVersionIds", foreignField:"_id", as:"feedVersions" } - }, + }, { // Deconstruct feedVersions array to a document for each element. $unwind: "$feedVersions" From c860bce849316eff0cc78a3fd171ffdb98182048 Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Thu, 24 Aug 2023 11:28:31 -0400 Subject: [PATCH 173/248] shared stops validator: validate feed_id --- pom.xml | 2 +- .../jobs/validation/SharedStopsValidator.java | 19 +++++++++++++------ .../datatools/manager/models/FeedVersion.java | 10 +++++++++- 3 files changed, 23 insertions(+), 8 deletions(-) diff --git a/pom.xml b/pom.xml index ff0612eb5..3a3708edd 100644 --- a/pom.xml +++ b/pom.xml @@ -268,7 +268,7 @@ com.github.conveyal gtfs-lib - 9837b6499796a0eeb5de76314e6a1f3125d695fb + cb8d03c1ee4b3137377986b795de406ab8e1a825 diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/validation/SharedStopsValidator.java b/src/main/java/com/conveyal/datatools/manager/jobs/validation/SharedStopsValidator.java index b4d4db0a2..a69175532 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/validation/SharedStopsValidator.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/validation/SharedStopsValidator.java @@ -1,6 +1,5 @@ package com.conveyal.datatools.manager.jobs.validation; -import com.conveyal.datatools.common.utils.aws.S3Utils; import com.conveyal.datatools.manager.models.Project; import com.conveyal.gtfs.error.NewGTFSError; import com.conveyal.gtfs.error.NewGTFSErrorType; @@ -24,11 +23,13 @@ public class SharedStopsValidator extends FeedValidator { private static final Logger LOG = LoggerFactory.getLogger(SharedStopsValidator.class); Feed feed; + String feedId; Project project; - public SharedStopsValidator(Project project) { + public SharedStopsValidator(Project project, String feedId) { super(null, null); this.project = project; + this.feedId = feedId; } // This method can only be run on a SharedStopsValidator that has been set up with a project only @@ -36,12 +37,16 @@ public SharedStopsValidator buildSharedStopsValidator(Feed feed, SQLErrorStorage if (this.project == null) { throw new RuntimeException("Shared stops validator can not be called because no project has been set!"); } - return new SharedStopsValidator(feed, errorStorage, this.project); + if (this.feedId == null) { + throw new RuntimeException("Shared stops validator can not be called because no feed ID has been set!"); + } + return new SharedStopsValidator(feed, errorStorage, this.project, this.feedId); } - public SharedStopsValidator(Feed feed, SQLErrorStorage errorStorage, Project project) { + public SharedStopsValidator(Feed feed, SQLErrorStorage errorStorage, Project project, String feedId) { super(feed, errorStorage); this.feed = feed; this.project = project; + this.feedId = feedId; } @Override @@ -56,6 +61,7 @@ public void validate() { final int STOP_GROUP_ID_INDEX = 0; final int STOP_ID_INDEX = 2; final int IS_PRIMARY_INDEX = 3; + final int FEED_ID_INDEX = 1; // Build list of stop Ids. List stopIds = new ArrayList<>(); @@ -74,6 +80,7 @@ public void validate() { while (configReader.readRecord()) { String stopGroupId = configReader.get(STOP_GROUP_ID_INDEX); String stopId = configReader.get(STOP_ID_INDEX); + String sharedStopFeedId = configReader.get(FEED_ID_INDEX); if (stopId.equals("stop_id")) { // Swallow header row @@ -103,8 +110,8 @@ public void validate() { } // Check for SS_03 (stop_id referenced doesn't exist) - // TODO: CHECK FEED ID (adjust the pre-build constructor to include feed_id) - if (!stopIds.contains(stopId)) { + // Make sure this error is only returned if we are inside the feed that is being checked + if (feedId.equals(sharedStopFeedId) && !stopIds.contains(stopId)) { registerError(NewGTFSError.forFeed(NewGTFSErrorType.SHARED_STOP_GROUP_ENTITY_DOES_NOT_EXIST, stopId)); } } diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java b/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java index 69edac06f..ce1a3e442 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java @@ -13,6 +13,7 @@ import com.conveyal.gtfs.BaseGTFSCache; import com.conveyal.gtfs.GTFS; import com.conveyal.gtfs.error.NewGTFSErrorType; +import com.conveyal.gtfs.graphql.fetchers.JDBCFetcher; import com.conveyal.gtfs.loader.Feed; import com.conveyal.gtfs.loader.FeedLoadResult; import com.conveyal.gtfs.validator.MTCValidator; @@ -372,7 +373,14 @@ public void validate(MonitorableJob.Status status) { ); } else { FeedSource fs = Persistence.feedSources.getById(this.feedSourceId); - SharedStopsValidator ssv = new SharedStopsValidator(fs.retrieveProject()); + + // Get feed_id from feed version... Really awful hack! + JDBCFetcher feedFetcher = new JDBCFetcher("feed_info", null); + Object gtfsFeedId = feedFetcher.getResults(this.namespace, null, null).get(0).get("feed_id"); + + + String feedId = gtfsFeedId == null ? "" : gtfsFeedId.toString(); + SharedStopsValidator ssv = new SharedStopsValidator(fs.retrieveProject(), feedId); validationResult = GTFS.validate( feedLoadResult.uniqueIdentifier, From 04c99d708d7ee7860c56c3b2769e0db6f9c1184b Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Fri, 8 Sep 2023 11:38:40 -0400 Subject: [PATCH 174/248] address pr feedback --- .../java/com/conveyal/datatools/manager/models/FeedVersion.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java b/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java index ce1a3e442..a15c486b2 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java @@ -375,7 +375,7 @@ public void validate(MonitorableJob.Status status) { FeedSource fs = Persistence.feedSources.getById(this.feedSourceId); // Get feed_id from feed version... Really awful hack! - JDBCFetcher feedFetcher = new JDBCFetcher("feed_info", null); + JDBCFetcher feedFetcher = new JDBCFetcher("feed_info"); Object gtfsFeedId = feedFetcher.getResults(this.namespace, null, null).get(0).get("feed_id"); From 4c9cfc08cc487a1881520620ed3e8384c77ffafc Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Mon, 11 Sep 2023 12:39:46 -0400 Subject: [PATCH 175/248] clarify feed_id retrieval comment --- .../conveyal/datatools/manager/models/FeedVersion.java | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java b/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java index a15c486b2..44cf2dc04 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java @@ -374,7 +374,15 @@ public void validate(MonitorableJob.Status status) { } else { FeedSource fs = Persistence.feedSources.getById(this.feedSourceId); - // Get feed_id from feed version... Really awful hack! + /* + Get feed_id from feed version + + This could potentially happen inside gtfs-lib, however + because this functionality is specific to datatools and the + shared stops feature, it lives only here instead. Changes to + gtfs-lib have been avoided, so that gtfs-lib isn't being modified + to support proprietary features. + */ JDBCFetcher feedFetcher = new JDBCFetcher("feed_info"); Object gtfsFeedId = feedFetcher.getResults(this.namespace, null, null).get(0).get("feed_id"); From 5ea408b9b5bfbb542445ec4878c661fb497e7ddb Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Thu, 14 Sep 2023 11:38:30 -0400 Subject: [PATCH 176/248] address pr feedback --- .../datatools/manager/jobs/validation/SharedStopsValidator.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/validation/SharedStopsValidator.java b/src/main/java/com/conveyal/datatools/manager/jobs/validation/SharedStopsValidator.java index a69175532..d608fe249 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/validation/SharedStopsValidator.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/validation/SharedStopsValidator.java @@ -58,6 +58,7 @@ public void validate() { CsvReader configReader = CsvReader.parse(config); + // TODO: pull indicies from the CSV header final int STOP_GROUP_ID_INDEX = 0; final int STOP_ID_INDEX = 2; final int IS_PRIMARY_INDEX = 3; From 76198d94986014ddfd3e79c3c0ecf1aa56f75f95 Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Thu, 21 Sep 2023 18:35:39 +0100 Subject: [PATCH 177/248] try to correct mobilitydata validator origin --- pom.xml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/pom.xml b/pom.xml index f635d2823..406486e76 100644 --- a/pom.xml +++ b/pom.xml @@ -173,6 +173,11 @@ + + + jitpack.io + https://jitpack.io + - - jitpack.io - https://jitpack.io - + @@ -262,7 +263,7 @@ com.github.mobilitydata gtfs-validator - 4.1.0 + v4.1.0 - cb8d03c1ee4b3137377986b795de406ab8e1a825 + d2469631f9fddfeecdd22c81d218479ca32a2bae From 91afbca3cd1f2ddf7c4658f42ed6ce132db174c6 Mon Sep 17 00:00:00 2001 From: "philip.cline" Date: Wed, 27 Sep 2023 11:20:57 -0400 Subject: [PATCH 182/248] feat(deps): update gtfs lib to merge commit --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 816f0f47a..7e384a78c 100644 --- a/pom.xml +++ b/pom.xml @@ -268,7 +268,7 @@ com.github.conveyal gtfs-lib - d2469631f9fddfeecdd22c81d218479ca32a2bae + 91b05180e68955ba41355cc6dc43e67c17eec8ca From cf83716376c894677cddf3752eb439962f1f4f62 Mon Sep 17 00:00:00 2001 From: "philip.cline" Date: Wed, 27 Sep 2023 14:19:40 -0400 Subject: [PATCH 183/248] refactor(JdbcTableWriter): respond to PR comments" --- .../manager/gtfsplus/GtfsPlusValidation.java | 47 ++++++------------- 1 file changed, 14 insertions(+), 33 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/gtfsplus/GtfsPlusValidation.java b/src/main/java/com/conveyal/datatools/manager/gtfsplus/GtfsPlusValidation.java index d120c839d..eb434923f 100644 --- a/src/main/java/com/conveyal/datatools/manager/gtfsplus/GtfsPlusValidation.java +++ b/src/main/java/com/conveyal/datatools/manager/gtfsplus/GtfsPlusValidation.java @@ -5,7 +5,6 @@ import com.conveyal.datatools.manager.persistence.FeedStore; import com.conveyal.datatools.manager.persistence.Persistence; import com.conveyal.gtfs.GTFSFeed; -import com.conveyal.gtfs.model.Route; import com.csvreader.CsvReader; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ArrayNode; @@ -22,10 +21,11 @@ import java.util.Arrays; import java.util.Collection; import java.util.Enumeration; -import java.util.HashMap; +import java.util.HashSet; import java.util.LinkedList; import java.util.List; -import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; import java.util.zip.ZipEntry; import java.util.zip.ZipFile; @@ -125,12 +125,12 @@ private static void validateTable( GTFSFeed gtfsFeed ) throws IOException { String tableId = specTable.get("id").asText(); - Boolean tableIsDirections = tableId.equals("directions"); + boolean tableIsDirections = tableId.equals("directions"); - Map gtfsRoutes = new HashMap<>(); + Set gtfsRoutes = new HashSet<>(); if (tableIsDirections) { - // Copy the gtfs routes into a map we can "check them off" in (remove them) - gtfsRoutes.putAll(gtfsFeed.routes); + // Copy the gtfs routes into a map we can "check them off" in (remove them). Stream is required in order to copy keys. + gtfsRoutes.addAll(gtfsFeed.routes.keySet()); } // Read in table data from input stream. @@ -180,19 +180,15 @@ private static void validateTable( JsonNode specField = fieldsFound[f]; // If value exists for index, use that. Otherwise, default to null to avoid out of bounds exception. String val = f < recordColumnCount ? rowValues[f] : null; - if (tableIsDirections && specField.get("name").asText().equals("route_id")) { - validateTableValue(issues, tableId, rowIndex, rowValues, val, fieldsFound, specField, gtfsFeed, gtfsRoutes); - } else { - validateTableValue(issues, tableId, rowIndex, rowValues, val, fieldsFound, specField, gtfsFeed); - } + validateTableValue(issues, tableId, rowIndex, rowValues, val, fieldsFound, specField, gtfsFeed, gtfsRoutes, tableIsDirections); } - // After we're done validating all the table values, check if every route was checked off in directions.txt } rowIndex++; } csvReader.close(); if (tableIsDirections && !gtfsRoutes.isEmpty()) { + // After we're done validating all the table values, check if every route was checked off in directions.txt issues.add(new ValidationIssue(tableId, "route_id", -1, "Directions table does not define direction names for all routes.")); } // Add issues for wrong number of columns and for empty rows after processing all rows. @@ -224,7 +220,9 @@ private static void validateTableValue( String value, JsonNode[] specFieldsFound, JsonNode specField, - GTFSFeed gtfsFeed + GTFSFeed gtfsFeed, + Set gtfsRoutes, + boolean tableIsDirections ) { if (specField == null) return; String fieldName = specField.get("name").asText(); @@ -318,26 +316,9 @@ private static void validateTableValue( } break; } - } - - /** Validate a single route_id value for the directions.txt GTFS+ table. */ - private static void validateTableValue( - Collection issues, - String tableId, - int rowIndex, - String[] allValues, - String value, - JsonNode[] specFieldsFound, - JsonNode specField, - GTFSFeed gtfsFeed, - Map gtfsRoutes - ) { - if (specField == null) return; - validateTableValue(issues, tableId, rowIndex, allValues, value, specFieldsFound, specField, gtfsFeed); - if (!gtfsRoutes.containsKey(value)) return; - // "Check off" the route_id from the list to verify every route id has a direction - gtfsRoutes.remove(value); + // "Check off" the route_id in directions.txt from the list to verify every route id has a direction + if (tableIsDirections && fieldName.equals("route_id")) gtfsRoutes.remove(value); } /** Construct missing ID text for validation issue description. */ From 5529162bc1e832f356b5110cb2f7e5b9f2f971a9 Mon Sep 17 00:00:00 2001 From: "philip.cline" Date: Thu, 28 Sep 2023 10:29:46 -0400 Subject: [PATCH 184/248] refactor(GtfsPlusValidation): Update wording --- .../datatools/manager/gtfsplus/GtfsPlusValidation.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/gtfsplus/GtfsPlusValidation.java b/src/main/java/com/conveyal/datatools/manager/gtfsplus/GtfsPlusValidation.java index eb434923f..c71f5d2f2 100644 --- a/src/main/java/com/conveyal/datatools/manager/gtfsplus/GtfsPlusValidation.java +++ b/src/main/java/com/conveyal/datatools/manager/gtfsplus/GtfsPlusValidation.java @@ -25,7 +25,6 @@ import java.util.LinkedList; import java.util.List; import java.util.Set; -import java.util.stream.Collectors; import java.util.zip.ZipEntry; import java.util.zip.ZipFile; @@ -129,7 +128,7 @@ private static void validateTable( Set gtfsRoutes = new HashSet<>(); if (tableIsDirections) { - // Copy the gtfs routes into a map we can "check them off" in (remove them). Stream is required in order to copy keys. + // Copy the gtfs routes into a set so that we can "check them off" (remove them). gtfsRoutes.addAll(gtfsFeed.routes.keySet()); } From e9cd115e533211873ab85826c82df95f07a232d9 Mon Sep 17 00:00:00 2001 From: "philip.cline" Date: Thu, 28 Sep 2023 10:30:43 -0400 Subject: [PATCH 185/248] refactor(GtfsPlusValidationTest): Update test wording --- .../datatools/manager/gtfsplus/GtfsPlusValidationTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/com/conveyal/datatools/manager/gtfsplus/GtfsPlusValidationTest.java b/src/test/java/com/conveyal/datatools/manager/gtfsplus/GtfsPlusValidationTest.java index 45eb504aa..2ed4d6381 100644 --- a/src/test/java/com/conveyal/datatools/manager/gtfsplus/GtfsPlusValidationTest.java +++ b/src/test/java/com/conveyal/datatools/manager/gtfsplus/GtfsPlusValidationTest.java @@ -76,7 +76,7 @@ void canValidateCleanGtfsPlus() throws Exception { LOG.info("Validation BART GTFS+"); GtfsPlusValidation validation = GtfsPlusValidation.validate(bartVersion1.id); // Expect issues to be only one with directions.txt file - assertThat("Issues count for clean BART feed is zero", validation.issues.size(), equalTo(1)); + assertThat("Clean BART feed and incomplete directions.txt results in one issue", validation.issues.size(), equalTo(1)); } @Test From b3b32997919795eb2272aeb450424399fc7dcf9a Mon Sep 17 00:00:00 2001 From: "philip.cline" Date: Thu, 28 Sep 2023 15:49:48 -0400 Subject: [PATCH 186/248] feat(deps): update gtfs lib again --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 7e384a78c..5f5a80fe2 100644 --- a/pom.xml +++ b/pom.xml @@ -268,7 +268,7 @@ com.github.conveyal gtfs-lib - 91b05180e68955ba41355cc6dc43e67c17eec8ca + 80a968cd3a071aa46a994ce2632535303a6e4384 From 2cc547dcfffd1519669a3f47251ddbeb5ba86491 Mon Sep 17 00:00:00 2001 From: "philip.cline" Date: Fri, 29 Sep 2023 14:52:35 -0400 Subject: [PATCH 187/248] fix(shared stops): fix everything --- .../datatools/manager/models/FeedSourceSummary.java | 8 ++++---- .../datatools/manager/models/FeedVersion.java | 11 +++++++---- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java b/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java index b9917e94a..9cf031df7 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java @@ -200,10 +200,10 @@ public static Map getLatestFeedVersionForFeedSources unwind("$feedVersions"), group( "$_id", - Accumulators.first("feedVersionId", "$feedVersions._id"), - Accumulators.first("firstCalendarDate", "$feedVersions.validationResult.firstCalendarDate"), - Accumulators.first("lastCalendarDate", "$feedVersions.validationResult.lastCalendarDate"), - Accumulators.first("errorCount", "$feedVersions.validationResult.errorCount") + Accumulators.last("feedVersionId", "$feedVersions._id"), + Accumulators.last("firstCalendarDate", "$feedVersions.validationResult.firstCalendarDate"), + Accumulators.last("lastCalendarDate", "$feedVersions.validationResult.lastCalendarDate"), + Accumulators.last("errorCount", "$feedVersions.validationResult.errorCount") ) ); return extractFeedVersionSummaries( diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java b/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java index 44cf2dc04..060f8589b 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java @@ -365,7 +365,7 @@ public void validate(MonitorableJob.Status status) { status.update("Validating feed...", 33); // Validate the feed version. - // Certain extensions, if enabled, have extra validators + // Certain extensions, if enabled, have extra validators. if (isExtensionEnabled("mtc")) { validationResult = GTFS.validate(feedLoadResult.uniqueIdentifier, DataManager.GTFS_DATA_SOURCE, RouteTypeValidatorBuilder::buildRouteValidator, @@ -384,9 +384,12 @@ public void validate(MonitorableJob.Status status) { to support proprietary features. */ JDBCFetcher feedFetcher = new JDBCFetcher("feed_info"); - Object gtfsFeedId = feedFetcher.getResults(this.namespace, null, null).get(0).get("feed_id"); - - + Object gtfsFeedId = new Object(); + try { + gtfsFeedId = feedFetcher.getResults(this.namespace, null, null).get(0).get("feed_id"); + } catch (RuntimeException e) { + LOG.warn("RuntimeException occurred while fetching feedId"); + } String feedId = gtfsFeedId == null ? "" : gtfsFeedId.toString(); SharedStopsValidator ssv = new SharedStopsValidator(fs.retrieveProject(), feedId); From 8a5c857e2578102c01884a74e26765e6a8d81f14 Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Fri, 29 Sep 2023 14:57:18 -0400 Subject: [PATCH 188/248] correct version --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 406486e76..d4e240b40 100644 --- a/pom.xml +++ b/pom.xml @@ -263,7 +263,7 @@ com.github.mobilitydata gtfs-validator - v4.1.0 + 4.1.0 - com.github.conveyal + com.conveyal gtfs-lib - 80a968cd3a071aa46a994ce2632535303a6e4384 + 7.0.2 diff --git a/src/main/java/com/conveyal/datatools/editor/controllers/api/SnapshotController.java b/src/main/java/com/conveyal/datatools/editor/controllers/api/SnapshotController.java index bce416e93..6ed68bcc7 100644 --- a/src/main/java/com/conveyal/datatools/editor/controllers/api/SnapshotController.java +++ b/src/main/java/com/conveyal/datatools/editor/controllers/api/SnapshotController.java @@ -84,6 +84,9 @@ private static String createSnapshot (Request req, Response res) throws IOExcept boolean publishNewVersion = Boolean.parseBoolean( req.queryParamOrDefault("publishNewVersion", Boolean.FALSE.toString()) ); + boolean publishProprietaryFiles = Boolean.parseBoolean( + req.queryParamOrDefault("publishProprietaryFiles", Boolean.FALSE.toString()) + ); FeedSource feedSource = FeedVersionController.requestFeedSourceById(req, Actions.EDIT, "feedId"); // Take fields from request body for creating snapshot (i.e., feedId/feedSourceId, name, comment). Snapshot snapshot = json.read(req.body()); @@ -99,7 +102,7 @@ private static String createSnapshot (Request req, Response res) throws IOExcept new CreateSnapshotJob(userProfile, snapshot, bufferIsEmpty, !bufferIsEmpty, false); // Add publish feed version job if specified by request. if (publishNewVersion) { - createSnapshotJob.addNextJob(new CreateFeedVersionFromSnapshotJob(feedSource, snapshot, userProfile)); + createSnapshotJob.addNextJob(new CreateFeedVersionFromSnapshotJob(feedSource, snapshot, userProfile, publishProprietaryFiles)); } // Begin asynchronous execution. JobUtils.heavyExecutor.execute(createSnapshotJob); diff --git a/src/main/java/com/conveyal/datatools/editor/jobs/ExportSnapshotToGTFSJob.java b/src/main/java/com/conveyal/datatools/editor/jobs/ExportSnapshotToGTFSJob.java index 3798c2ee8..53c56b746 100644 --- a/src/main/java/com/conveyal/datatools/editor/jobs/ExportSnapshotToGTFSJob.java +++ b/src/main/java/com/conveyal/datatools/editor/jobs/ExportSnapshotToGTFSJob.java @@ -28,11 +28,21 @@ public class ExportSnapshotToGTFSJob extends MonitorableJob { private final Snapshot snapshot; private final FeedVersion feedVersion; private File tempFile; + private final boolean publishProprietaryFiles; public ExportSnapshotToGTFSJob(Auth0UserProfile owner, Snapshot snapshot, FeedVersion feedVersion) { super(owner, "Exporting snapshot " + snapshot.name, JobType.EXPORT_SNAPSHOT_TO_GTFS); this.snapshot = snapshot; this.feedVersion = feedVersion; + this.publishProprietaryFiles = false; // DEFAULT. TODO: don't hide this here. + status.update("Starting database snapshot...", 10); + } + + public ExportSnapshotToGTFSJob(Auth0UserProfile owner, Snapshot snapshot, FeedVersion feedVersion, boolean publishProprietaryFiles) { + super(owner, "Exporting snapshot " + snapshot.name, JobType.EXPORT_SNAPSHOT_TO_GTFS); + this.snapshot = snapshot; + this.feedVersion = feedVersion; + this.publishProprietaryFiles = publishProprietaryFiles; status.update("Starting database snapshot...", 10); } @@ -57,7 +67,7 @@ public void jobLogic() { status.fail("Error creating local file for snapshot.", e); return; } - JdbcGtfsExporter exporter = new JdbcGtfsExporter(snapshot.namespace, tempFile.getAbsolutePath(), DataManager.GTFS_DATA_SOURCE, true); + JdbcGtfsExporter exporter = new JdbcGtfsExporter(snapshot.namespace, tempFile.getAbsolutePath(), DataManager.GTFS_DATA_SOURCE, true, publishProprietaryFiles); FeedLoadResult result = exporter.exportTables(); if (result.fatalException != null) { status.fail(String.format("Error (%s) encountered while exporting database tables.", result.fatalException)); diff --git a/src/main/java/com/conveyal/datatools/manager/controllers/api/FeedVersionController.java b/src/main/java/com/conveyal/datatools/manager/controllers/api/FeedVersionController.java index 035269e66..a33a64881 100644 --- a/src/main/java/com/conveyal/datatools/manager/controllers/api/FeedVersionController.java +++ b/src/main/java/com/conveyal/datatools/manager/controllers/api/FeedVersionController.java @@ -179,7 +179,7 @@ private static boolean createFeedVersionFromSnapshot (Request req, Response res) logMessageAndHalt(req, 400, "Must provide valid snapshot ID"); } CreateFeedVersionFromSnapshotJob createFromSnapshotJob = - new CreateFeedVersionFromSnapshotJob(feedSource, snapshot, userProfile); + new CreateFeedVersionFromSnapshotJob(feedSource, snapshot, userProfile, false); JobUtils.heavyExecutor.execute(createFromSnapshotJob); return true; diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/CreateFeedVersionFromSnapshotJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/CreateFeedVersionFromSnapshotJob.java index 6c711c74e..c6929a93b 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/CreateFeedVersionFromSnapshotJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/CreateFeedVersionFromSnapshotJob.java @@ -22,11 +22,13 @@ public class CreateFeedVersionFromSnapshotJob extends FeedSourceJob { private final FeedVersion feedVersion; private final Snapshot snapshot; + private final boolean publishProprietaryFiles; - public CreateFeedVersionFromSnapshotJob(FeedSource feedSource, Snapshot snapshot, Auth0UserProfile owner) { + public CreateFeedVersionFromSnapshotJob(FeedSource feedSource, Snapshot snapshot, Auth0UserProfile owner, boolean publishProprietaryFiles) { super(owner, "Creating Feed Version from Snapshot for " + feedSource.name, JobType.CREATE_FEEDVERSION_FROM_SNAPSHOT); this.feedVersion = new FeedVersion(feedSource, snapshot); this.snapshot = snapshot; + this.publishProprietaryFiles = publishProprietaryFiles; } @Override @@ -35,7 +37,7 @@ public void jobLogic() { // Add the jobs to handle this operation in order. addNextJob( // First export the snapshot to GTFS. - new ExportSnapshotToGTFSJob(owner, snapshot, feedVersion), + new ExportSnapshotToGTFSJob(owner, snapshot, feedVersion, publishProprietaryFiles), // Then, process feed version once GTFS file written. new ProcessSingleFeedJob(feedVersion, owner, true) ); diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/ProcessSingleFeedJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/ProcessSingleFeedJob.java index f26eba655..27585d946 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/ProcessSingleFeedJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/ProcessSingleFeedJob.java @@ -130,7 +130,7 @@ public void jobLogic() { snapshot.feedTransformResult = dbTarget.feedTransformResult; // If the user has selected to create a new version from the resulting snapshot, do so here. if (rules.createNewVersion) { - addNextJob(new CreateFeedVersionFromSnapshotJob(feedSource, snapshot, owner)); + addNextJob(new CreateFeedVersionFromSnapshotJob(feedSource, snapshot, owner, false)); } } diff --git a/src/test/java/com/conveyal/datatools/manager/jobs/ArbitraryTransformJobTest.java b/src/test/java/com/conveyal/datatools/manager/jobs/ArbitraryTransformJobTest.java index a8924973e..9303bf821 100644 --- a/src/test/java/com/conveyal/datatools/manager/jobs/ArbitraryTransformJobTest.java +++ b/src/test/java/com/conveyal/datatools/manager/jobs/ArbitraryTransformJobTest.java @@ -160,7 +160,7 @@ void canDeleteTrips() throws IOException { ); // Fetch snapshot where modifications were made and create new version from it. Snapshot snapshotWithModifications = feedSource.retrieveSnapshots().iterator().next(); - CreateFeedVersionFromSnapshotJob newVersionJob = new CreateFeedVersionFromSnapshotJob(feedSource, snapshotWithModifications, user); + CreateFeedVersionFromSnapshotJob newVersionJob = new CreateFeedVersionFromSnapshotJob(feedSource, snapshotWithModifications, user, false); newVersionJob.run(); // Grab the modified version and check that the trips count matches expectation. FeedVersion newVersion = feedSource.retrieveLatest(); From d4e780430a0969d08bd484a32e251db5f2b0d446 Mon Sep 17 00:00:00 2001 From: "philip.cline" Date: Fri, 6 Oct 2023 15:53:10 -0400 Subject: [PATCH 191/248] refactor(SnapshotExport): update gtfs-lib, add comment --- pom.xml | 4 ++-- .../manager/controllers/api/FeedVersionController.java | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 60e1a74b8..5f5a80fe2 100644 --- a/pom.xml +++ b/pom.xml @@ -265,10 +265,10 @@ - AWS S3 SDK - putting/getting objects into/out of S3. --> - com.conveyal + com.github.conveyal gtfs-lib - 7.0.2 + 80a968cd3a071aa46a994ce2632535303a6e4384 diff --git a/src/main/java/com/conveyal/datatools/manager/controllers/api/FeedVersionController.java b/src/main/java/com/conveyal/datatools/manager/controllers/api/FeedVersionController.java index a33a64881..f4fe5c4d6 100644 --- a/src/main/java/com/conveyal/datatools/manager/controllers/api/FeedVersionController.java +++ b/src/main/java/com/conveyal/datatools/manager/controllers/api/FeedVersionController.java @@ -178,6 +178,7 @@ private static boolean createFeedVersionFromSnapshot (Request req, Response res) if (snapshot == null) { logMessageAndHalt(req, 400, "Must provide valid snapshot ID"); } + // TODO: Allow publishing with proprietary files from this endpoint? CreateFeedVersionFromSnapshotJob createFromSnapshotJob = new CreateFeedVersionFromSnapshotJob(feedSource, snapshot, userProfile, false); JobUtils.heavyExecutor.execute(createFromSnapshotJob); From 1a92792ce9fa09737b94d87e507add8e5acfcd3b Mon Sep 17 00:00:00 2001 From: "philip.cline" Date: Fri, 6 Oct 2023 15:56:30 -0400 Subject: [PATCH 192/248] refactor(SnapshotExport): update gtfs-lib again --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 5f5a80fe2..0c3d529d1 100644 --- a/pom.xml +++ b/pom.xml @@ -268,7 +268,7 @@ com.github.conveyal gtfs-lib - 80a968cd3a071aa46a994ce2632535303a6e4384 + a5f60417e38043d35148aa6873c93056e69e9c87 From 87e54a46fbe4bc60cecf12c297de19c57b28dab5 Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Tue, 10 Oct 2023 09:55:18 +0100 Subject: [PATCH 193/248] refactor(Upstream changes to export call): Updated to export proprietary files with comments --- pom.xml | 2 +- .../editor/controllers/api/SnapshotController.java | 13 +++++++++---- .../editor/jobs/ExportSnapshotToGTFSJob.java | 12 ++---------- .../controllers/api/FeedVersionController.java | 4 ++-- .../manager/jobs/ProcessSingleFeedJob.java | 3 ++- 5 files changed, 16 insertions(+), 18 deletions(-) diff --git a/pom.xml b/pom.xml index 0c3d529d1..8affc0ccb 100644 --- a/pom.xml +++ b/pom.xml @@ -268,7 +268,7 @@ com.github.conveyal gtfs-lib - a5f60417e38043d35148aa6873c93056e69e9c87 + 026599ba4907f4c42b24e883d154e63b0777dd31 diff --git a/src/main/java/com/conveyal/datatools/editor/controllers/api/SnapshotController.java b/src/main/java/com/conveyal/datatools/editor/controllers/api/SnapshotController.java index 6ed68bcc7..ebe45b692 100644 --- a/src/main/java/com/conveyal/datatools/editor/controllers/api/SnapshotController.java +++ b/src/main/java/com/conveyal/datatools/editor/controllers/api/SnapshotController.java @@ -63,6 +63,12 @@ private static Snapshot getSnapshotFromRequest(Request req) { return Persistence.snapshots.getById(id); } + private static boolean getPublishProprietaryFiles(Request req) { + return Boolean.parseBoolean( + req.queryParamOrDefault("publishProprietaryFiles",Boolean.FALSE.toString()) + ); + } + /** * HTTP endpoint that returns the list of snapshots for a given feed source. */ @@ -84,9 +90,7 @@ private static String createSnapshot (Request req, Response res) throws IOExcept boolean publishNewVersion = Boolean.parseBoolean( req.queryParamOrDefault("publishNewVersion", Boolean.FALSE.toString()) ); - boolean publishProprietaryFiles = Boolean.parseBoolean( - req.queryParamOrDefault("publishProprietaryFiles", Boolean.FALSE.toString()) - ); + boolean publishProprietaryFiles = getPublishProprietaryFiles(req); FeedSource feedSource = FeedVersionController.requestFeedSourceById(req, Actions.EDIT, "feedId"); // Take fields from request body for creating snapshot (i.e., feedId/feedSourceId, name, comment). Snapshot snapshot = json.read(req.body()); @@ -176,9 +180,10 @@ private static String restoreSnapshot (Request req, Response res) { private static String downloadSnapshotAsGTFS(Request req, Response res) { Auth0UserProfile userProfile = req.attribute("user"); Snapshot snapshot = getSnapshotFromRequest(req); + boolean publishProprietaryFiles = getPublishProprietaryFiles(req); // Create and kick off export job. // FIXME: what if a snapshot is already written to S3? - ExportSnapshotToGTFSJob exportSnapshotToGTFSJob = new ExportSnapshotToGTFSJob(userProfile, snapshot); + ExportSnapshotToGTFSJob exportSnapshotToGTFSJob = new ExportSnapshotToGTFSJob(userProfile, snapshot, publishProprietaryFiles); JobUtils.heavyExecutor.execute(exportSnapshotToGTFSJob); return formatJobMessage(exportSnapshotToGTFSJob.jobId, "Exporting snapshot to GTFS."); } diff --git a/src/main/java/com/conveyal/datatools/editor/jobs/ExportSnapshotToGTFSJob.java b/src/main/java/com/conveyal/datatools/editor/jobs/ExportSnapshotToGTFSJob.java index 53c56b746..fc76890b5 100644 --- a/src/main/java/com/conveyal/datatools/editor/jobs/ExportSnapshotToGTFSJob.java +++ b/src/main/java/com/conveyal/datatools/editor/jobs/ExportSnapshotToGTFSJob.java @@ -30,14 +30,6 @@ public class ExportSnapshotToGTFSJob extends MonitorableJob { private File tempFile; private final boolean publishProprietaryFiles; - public ExportSnapshotToGTFSJob(Auth0UserProfile owner, Snapshot snapshot, FeedVersion feedVersion) { - super(owner, "Exporting snapshot " + snapshot.name, JobType.EXPORT_SNAPSHOT_TO_GTFS); - this.snapshot = snapshot; - this.feedVersion = feedVersion; - this.publishProprietaryFiles = false; // DEFAULT. TODO: don't hide this here. - status.update("Starting database snapshot...", 10); - } - public ExportSnapshotToGTFSJob(Auth0UserProfile owner, Snapshot snapshot, FeedVersion feedVersion, boolean publishProprietaryFiles) { super(owner, "Exporting snapshot " + snapshot.name, JobType.EXPORT_SNAPSHOT_TO_GTFS); this.snapshot = snapshot; @@ -46,8 +38,8 @@ public ExportSnapshotToGTFSJob(Auth0UserProfile owner, Snapshot snapshot, FeedVe status.update("Starting database snapshot...", 10); } - public ExportSnapshotToGTFSJob(Auth0UserProfile owner, Snapshot snapshot) { - this(owner, snapshot, null); + public ExportSnapshotToGTFSJob(Auth0UserProfile owner, Snapshot snapshot, boolean publishProprietaryFiles) { + this(owner, snapshot, null, publishProprietaryFiles); } @JsonProperty diff --git a/src/main/java/com/conveyal/datatools/manager/controllers/api/FeedVersionController.java b/src/main/java/com/conveyal/datatools/manager/controllers/api/FeedVersionController.java index f4fe5c4d6..8f855efea 100644 --- a/src/main/java/com/conveyal/datatools/manager/controllers/api/FeedVersionController.java +++ b/src/main/java/com/conveyal/datatools/manager/controllers/api/FeedVersionController.java @@ -178,9 +178,9 @@ private static boolean createFeedVersionFromSnapshot (Request req, Response res) if (snapshot == null) { logMessageAndHalt(req, 400, "Must provide valid snapshot ID"); } - // TODO: Allow publishing with proprietary files from this endpoint? + // Publishing the proprietary files will preserve the pattern names in the newly published feed version. CreateFeedVersionFromSnapshotJob createFromSnapshotJob = - new CreateFeedVersionFromSnapshotJob(feedSource, snapshot, userProfile, false); + new CreateFeedVersionFromSnapshotJob(feedSource, snapshot, userProfile, true); JobUtils.heavyExecutor.execute(createFromSnapshotJob); return true; diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/ProcessSingleFeedJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/ProcessSingleFeedJob.java index 27585d946..c267fdd9a 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/ProcessSingleFeedJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/ProcessSingleFeedJob.java @@ -130,7 +130,8 @@ public void jobLogic() { snapshot.feedTransformResult = dbTarget.feedTransformResult; // If the user has selected to create a new version from the resulting snapshot, do so here. if (rules.createNewVersion) { - addNextJob(new CreateFeedVersionFromSnapshotJob(feedSource, snapshot, owner, false)); + // Publishing the proprietary files will preserve the pattern names in the newly created feed version. + addNextJob(new CreateFeedVersionFromSnapshotJob(feedSource, snapshot, owner, true)); } } From 7f64621960eb162b7fe162db790d45e93c03ed04 Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Tue, 10 Oct 2023 12:33:43 +0100 Subject: [PATCH 194/248] refactor(pom.xml): Bumped GTFS-lib version --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 8affc0ccb..5cb786373 100644 --- a/pom.xml +++ b/pom.xml @@ -268,7 +268,7 @@ com.github.conveyal gtfs-lib - 026599ba4907f4c42b24e883d154e63b0777dd31 + 10307d43c6610d5dd1cc07d30091596816772727 From 69a9b653158849e89708a239db4a7bb7b426926c Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Fri, 13 Oct 2023 11:08:58 +0100 Subject: [PATCH 195/248] refactor(Added auth checks): Auth checks and added notes and org id to feed source summary --- .../controllers/api/FeedSourceController.java | 40 +++++--- .../manager/models/FeedSourceSummary.java | 47 +++++++-- .../datatools/manager/models/Label.java | 13 ++- .../datatools/manager/models/Note.java | 15 +++ .../datatools/manager/models/Project.java | 2 +- .../api/FeedSourceControllerTest.java | 97 ++++++++++++++++--- 6 files changed, 177 insertions(+), 37 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/controllers/api/FeedSourceController.java b/src/main/java/com/conveyal/datatools/manager/controllers/api/FeedSourceController.java index badedda76..080c79915 100644 --- a/src/main/java/com/conveyal/datatools/manager/controllers/api/FeedSourceController.java +++ b/src/main/java/com/conveyal/datatools/manager/controllers/api/FeedSourceController.java @@ -45,6 +45,7 @@ import static com.conveyal.datatools.common.utils.SparkUtils.getPOJOFromRequestBody; import static com.conveyal.datatools.common.utils.SparkUtils.logMessageAndHalt; import static com.conveyal.datatools.manager.models.ExternalFeedSourceProperty.constructId; +import static com.conveyal.datatools.manager.models.FeedSourceSummary.cleanFeedSourceSummaryForNonAdmins; import static com.conveyal.datatools.manager.models.transform.NormalizeFieldTransformation.getInvalidSubstitutionMessage; import static com.conveyal.datatools.manager.models.transform.NormalizeFieldTransformation.getInvalidSubstitutionPatterns; import static com.mongodb.client.model.Filters.in; @@ -103,6 +104,31 @@ private static Collection getProjectFeedSources(Request req, Respons return feedSourcesToReturn; } + private static Collection getAllFeedSourceSummaries(Request req, Response res) { + Collection feedSourcesToReturn = new ArrayList<>(); + Auth0UserProfile user = req.attribute("user"); + String projectId = req.queryParams("projectId"); + + Project project = Persistence.projects.getById(projectId); + + if (project == null) { + logMessageAndHalt(req, 400, "Must provide valid projectId value."); + } else { + boolean isAdmin = user.canAdministerProject(project); + Collection feedSourceSummaries = project.retrieveFeedSourceSummaries(); + for (FeedSourceSummary feedSourceSummary : feedSourceSummaries) { + // If user can view or manage feed, add to list of feeds to return. NOTE: By default most users with access + // to a project should be able to view all feed sources. Custom privileges would need to be provided to + // override this behavior. + if (user.canManageOrViewFeed(project.organizationId, feedSourceSummary.projectId, feedSourceSummary.id)) { + // Remove labels user can't view, then add to list of feeds to return. + feedSourcesToReturn.add(cleanFeedSourceSummaryForNonAdmins(feedSourceSummary, isAdmin)); + } + } + } + return feedSourcesToReturn; + } + /** * HTTP endpoint to create a new feed source. */ @@ -397,20 +423,6 @@ protected static FeedSource cleanFeedSourceForNonAdmins(FeedSource feedSource, b return feedSource; } - private static Collection getAllFeedSourceSummaries(Request req, Response res) { - Auth0UserProfile userProfile = req.attribute("user"); - String projectId = req.queryParams("projectId"); - Project project = Persistence.projects.getById(projectId); - if (project == null) { - logMessageAndHalt(req, 400, "Must provide valid projectId value."); - } - if (!userProfile.canAdministerProject(project)) { - logMessageAndHalt(req, 401, "User not authorized to view project feed sources."); - } - return project.retrieveFeedSourceSummaries(); - } - - // FIXME: use generic API controller and return JSON documents via BSON/Mongo public static void register (String apiPrefix) { get(apiPrefix + "secure/feedsource/:id", FeedSourceController::getFeedSource, json::write); diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java b/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java index 2c5d6197f..6116bac3f 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java @@ -2,6 +2,7 @@ import com.conveyal.datatools.editor.utils.JacksonSerializers; import com.conveyal.datatools.manager.persistence.Persistence; +import com.conveyal.datatools.manager.utils.PersistenceUtils; import com.conveyal.gtfs.validator.ValidationResult; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import com.fasterxml.jackson.databind.annotation.JsonSerialize; @@ -20,6 +21,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; import static com.mongodb.client.model.Aggregates.group; import static com.mongodb.client.model.Aggregates.limit; @@ -64,11 +66,16 @@ public class FeedSourceSummary { public String url; + public List noteIds = new ArrayList<>(); + + public String organizationId; + public FeedSourceSummary() { } - public FeedSourceSummary(String projectId, Document feedSourceDocument) { + public FeedSourceSummary(String projectId, String organizationId, Document feedSourceDocument) { this.projectId = projectId; + this.organizationId = organizationId; this.id = feedSourceDocument.getString("_id"); this.name = feedSourceDocument.getString("name"); this.deployable = feedSourceDocument.getBoolean("deployable"); @@ -77,11 +84,35 @@ public FeedSourceSummary(String projectId, Document feedSourceDocument) { if (documentLabelIds != null) { this.labelIds = documentLabelIds; } + List documentNoteIds = feedSourceDocument.getList("noteIds", String.class); + if (documentNoteIds != null) { + this.noteIds = documentNoteIds; + } // Convert to local date type for consistency. this.lastUpdated = getLocalDateFromDate(feedSourceDocument.getDate("lastUpdated")); this.url = feedSourceDocument.getString("url"); } + /** + * Removes labels and notes from a feed that a user is not allowed to view. Returns cleaned feed source. + * @param feedSourceSummary The feed source to clean + * @param isAdmin Is the user an admin? Changes what is returned. + * @return A feed source containing only labels/notes the user is allowed to see + */ + public static FeedSourceSummary cleanFeedSourceSummaryForNonAdmins(FeedSourceSummary feedSourceSummary, boolean isAdmin) { + // Admin can view all feed labels, but a non-admin should only see those with adminOnly=false + feedSourceSummary.labelIds = Persistence.labels + .getFiltered(PersistenceUtils.applyAdminFilter(in("_id", feedSourceSummary.labelIds), isAdmin)).stream() + .map(label -> label.id) + .collect(Collectors.toList()); + feedSourceSummary.noteIds = Persistence.notes + .getFiltered(PersistenceUtils.applyAdminFilter(in("_id", feedSourceSummary.noteIds), isAdmin)).stream() + .map(note -> note.id) + .collect(Collectors.toList()); + return feedSourceSummary; + } + + /** * Set the appropriate feed version. For consistency, if no error count is available set the related number of * issues to null. @@ -104,7 +135,7 @@ public void setFeedVersion(FeedVersionSummary feedVersionSummary, boolean isDepl /** * Get all feed source summaries matching the project id. */ - public static List getFeedSourceSummaries(String projectId) { + public static List getFeedSourceSummaries(String projectId, String organizationId) { /* db.getCollection('FeedSource').aggregate([ { @@ -121,7 +152,8 @@ public static List getFeedSourceSummaries(String projectId) { "isPublic": 1, "lastUpdated": 1, "labelIds": 1, - "url": 1 + "url": 1, + "noteIds": 1 } }, { @@ -143,12 +175,13 @@ public static List getFeedSourceSummaries(String projectId) { "isPublic", "lastUpdated", "labelIds", - "url") + "url", + "noteIds") ) ), sort(Sorts.ascending("name")) ); - return extractFeedSourceSummaries(projectId, stages); + return extractFeedSourceSummaries(projectId, organizationId, stages); } /** @@ -423,10 +456,10 @@ public static Map getFeedVersionsFromPinnedDeploymen /** * Produce a list of all feed source summaries for a project. */ - private static List extractFeedSourceSummaries(String projectId, List stages) { + private static List extractFeedSourceSummaries(String projectId, String organizationId, List stages) { List feedSourceSummaries = new ArrayList<>(); for (Document feedSourceDocument : Persistence.getDocuments("FeedSource", stages)) { - feedSourceSummaries.add(new FeedSourceSummary(projectId, feedSourceDocument)); + feedSourceSummaries.add(new FeedSourceSummary(projectId, organizationId, feedSourceDocument)); } return feedSourceSummaries; } diff --git a/src/main/java/com/conveyal/datatools/manager/models/Label.java b/src/main/java/com/conveyal/datatools/manager/models/Label.java index 8bf239798..0afe2563e 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/Label.java +++ b/src/main/java/com/conveyal/datatools/manager/models/Label.java @@ -39,15 +39,24 @@ public String organizationId () { public Auth0UserProfile user; /** - * Create a new label + * Create a new label with auto-gen id. */ public Label (String name, String description, String color, boolean adminOnly, String projectId) { + this(null, name, description, color, adminOnly, projectId); + } + + /** + * Create a new label with provided id. + */ + public Label (String id, String name, String description, String color, boolean adminOnly, String projectId) { super(); + if (id != null) { + this.id = id; + } this.name = name; this.description = description != null ? description : ""; this.color = color != null ? color : "#000000"; this.adminOnly = adminOnly; - this.projectId = projectId; } diff --git a/src/main/java/com/conveyal/datatools/manager/models/Note.java b/src/main/java/com/conveyal/datatools/manager/models/Note.java index 87b5f452f..438a784b5 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/Note.java +++ b/src/main/java/com/conveyal/datatools/manager/models/Note.java @@ -32,6 +32,21 @@ public class Note extends Model implements Serializable { /** Whether the note should be visible to project admins only */ public boolean adminOnly; + /** + * Create a new note with provided id. + */ + public Note(String id, String body, boolean adminOnly) { + super(); + if (id != null) { + this.id = id; + } + this.body = body; + this.adminOnly = adminOnly; + } + + public Note() { + } + /** * The types of object that can have notes recorded on them. */ diff --git a/src/main/java/com/conveyal/datatools/manager/models/Project.java b/src/main/java/com/conveyal/datatools/manager/models/Project.java index 4ccad306f..eccb7d33a 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/Project.java +++ b/src/main/java/com/conveyal/datatools/manager/models/Project.java @@ -173,7 +173,7 @@ public Collection retrieveDeploymentSummaries() { * Get all feed source summaries for this project. */ public Collection retrieveFeedSourceSummaries() { - List feedSourceSummaries = FeedSourceSummary.getFeedSourceSummaries(id); + List feedSourceSummaries = FeedSourceSummary.getFeedSourceSummaries(id, organizationId); Map latestFeedVersionForFeedSources = FeedSourceSummary.getLatestFeedVersionForFeedSources(id); Map deployedFeedVersions = FeedSourceSummary.getFeedVersionsFromPinnedDeployment(id); if (deployedFeedVersions.isEmpty()) { diff --git a/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java b/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java index a93f14630..a4c6d8d0a 100644 --- a/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java +++ b/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java @@ -11,6 +11,7 @@ import com.conveyal.datatools.manager.models.FeedVersion; import com.conveyal.datatools.manager.models.FetchFrequency; import com.conveyal.datatools.manager.models.Label; +import com.conveyal.datatools.manager.models.Note; import com.conveyal.datatools.manager.models.Project; import com.conveyal.datatools.manager.persistence.Persistence; import com.conveyal.datatools.manager.utils.HttpUtils; @@ -22,6 +23,7 @@ import org.junit.jupiter.api.Test; import java.io.IOException; +import java.net.MalformedURLException; import java.net.URL; import java.time.LocalDate; import java.time.Month; @@ -46,7 +48,10 @@ public class FeedSourceControllerTest extends DatatoolsTest { private static FeedSource feedSourceWithInvalidLabels = null; private static Label publicLabel = null; private static Label adminOnlyLabel = null; - + private static Label feedSourceWithLatestDeploymentAdminOnlyLabel = null; + private static Label feedSourceWithPinnedDeploymentAdminOnlyLabel = null; + private static Note feedSourceWithLatestDeploymentAdminOnlyNote = null; + private static Note feedSourceWithPinnedDeploymentAdminOnlyNote = null; private static Project projectWithLatestDeployment = null; private static FeedSource feedSourceWithLatestDeploymentFeedVersion = null; private static FeedVersion feedVersionFromLatestDeployment = null; @@ -78,22 +83,37 @@ public static void setUp() throws IOException { feedSourceWithLabels = createFeedSource("FeedSourceThree", new URL("http://www.feedsource.com"), projectToBeDeleted); feedSourceWithInvalidLabels = createFeedSource("FeedSourceFour", new URL("http://www.feedsource.com"), project); - adminOnlyLabel = createLabel("Admin Only Label"); + adminOnlyLabel = createLabel("Admin Only Label", projectToBeDeleted.id); adminOnlyLabel.adminOnly = true; - publicLabel = createLabel("Public Label"); + publicLabel = createLabel("Public Label", projectToBeDeleted.id); + + setUpFeedVersionFromLatestDeployment(); + setUpFeedVersionFromPinnedDeployment(); + + } - // Feed version from latest deployment. + /** + * Create all the required objects to test a feed version from the latest deployment. + */ + private static void setUpFeedVersionFromLatestDeployment() throws MalformedURLException { projectWithLatestDeployment = new Project(); projectWithLatestDeployment.id = "project-with-latest-deployment"; + projectWithLatestDeployment.organizationId = "project-with-latest-deployment-org-id"; Persistence.projects.create(projectWithLatestDeployment); + + feedSourceWithLatestDeploymentAdminOnlyLabel = createLabel("label-id-latest-deployment", "Admin Only Label", projectWithLatestDeployment.id); + feedSourceWithLatestDeploymentAdminOnlyNote = createNote("note-id-latest-deployment", "A test note"); + feedSourceWithLatestDeploymentFeedVersion = createFeedSource( "feed-source-with-latest-deployment-feed-version", "FeedSource", new URL("http://www.feedsource.com"), projectWithLatestDeployment, true, - List.of("labelOne", "labelTwo") + List.of(feedSourceWithLatestDeploymentAdminOnlyLabel.id), + List.of(feedSourceWithLatestDeploymentAdminOnlyNote.id) ); + LocalDate deployedSuperseded = LocalDate.of(2020, Month.MARCH, 12); LocalDate deployedEndDate = LocalDate.of(2021, Month.MARCH, 12); LocalDate deployedStartDate = LocalDate.of(2021, Month.MARCH, 1); @@ -115,18 +135,28 @@ public static void setUp() throws IOException { feedVersionFromLatestDeployment.id, deployedEndDate ); + } - // Feed version from pinned deployment. + /** + * Create all the required objects to test a feed version from a pinned deployment. + */ + private static void setUpFeedVersionFromPinnedDeployment() throws MalformedURLException { projectWithPinnedDeployment = new Project(); projectWithPinnedDeployment.id = "project-with-pinned-deployment"; + projectWithPinnedDeployment.organizationId = "project-with-pinned-deployment-org-id"; Persistence.projects.create(projectWithPinnedDeployment); + + feedSourceWithPinnedDeploymentAdminOnlyLabel = createLabel("label-id-pinned-deployment", "Admin Only Label", projectWithPinnedDeployment.id); + feedSourceWithPinnedDeploymentAdminOnlyNote = createNote("note-id-pinned-deployment", "A test note"); + feedSourceWithPinnedDeploymentFeedVersion = createFeedSource( "feed-source-with-pinned-deployment-feed-version", "FeedSourceWithPinnedFeedVersion", new URL("http://www.feedsource.com"), projectWithPinnedDeployment, true, - List.of("labelOne", "labelTwo") + List.of(feedSourceWithPinnedDeploymentAdminOnlyLabel.id), + List.of(feedSourceWithPinnedDeploymentAdminOnlyNote.id) ); feedVersionFromPinnedDeployment = createFeedVersion( "feed-version-from-pinned-deployment", @@ -137,7 +167,7 @@ public static void setUp() throws IOException { "deployment-pinned", projectWithPinnedDeployment, feedVersionFromPinnedDeployment.id, - deployedEndDate + LocalDate.of(2021, Month.MARCH, 12) ); projectWithPinnedDeployment.pinnedDeploymentId = deploymentPinned.id; Persistence.projects.replace(projectWithPinnedDeployment.id, projectWithPinnedDeployment); @@ -200,6 +230,18 @@ private static void tearDownDeployedFeedVersion() { if (deploymentSuperseded != null) { Persistence.deployments.removeById(deploymentSuperseded.id); } + if (feedSourceWithPinnedDeploymentAdminOnlyLabel != null) { + Persistence.labels.removeById(feedSourceWithPinnedDeploymentAdminOnlyLabel.id); + } + if (feedSourceWithLatestDeploymentAdminOnlyLabel != null) { + Persistence.labels.removeById(feedSourceWithLatestDeploymentAdminOnlyLabel.id); + } + if (feedSourceWithPinnedDeploymentAdminOnlyNote != null) { + Persistence.notes.removeById(feedSourceWithPinnedDeploymentAdminOnlyNote.id); + } + if (feedSourceWithLatestDeploymentAdminOnlyNote != null) { + Persistence.notes.removeById(feedSourceWithLatestDeploymentAdminOnlyNote.id); + } } /** @@ -345,6 +387,8 @@ void canRetrieveDeployedFeedVersionFromLatestDeployment() throws IOException { assertEquals(feedSourceWithLatestDeploymentFeedVersion.projectId, feedSourceSummaries.get(0).projectId); assertEquals(feedSourceWithLatestDeploymentFeedVersion.labelIds, feedSourceSummaries.get(0).labelIds); assertEquals(feedSourceWithLatestDeploymentFeedVersion.url.toString(), feedSourceSummaries.get(0).url); + assertEquals(feedSourceWithLatestDeploymentFeedVersion.noteIds, feedSourceSummaries.get(0).noteIds); + assertEquals(feedSourceWithLatestDeploymentFeedVersion.organizationId(), feedSourceSummaries.get(0).organizationId); assertEquals(feedVersionFromLatestDeployment.id, feedSourceSummaries.get(0).deployedFeedVersionId); assertEquals(feedVersionFromLatestDeployment.validationSummary().startDate, feedSourceSummaries.get(0).deployedFeedVersionStartDate); assertEquals(feedVersionFromLatestDeployment.validationSummary().endDate, feedSourceSummaries.get(0).deployedFeedVersionEndDate); @@ -377,6 +421,8 @@ void canRetrieveDeployedFeedVersionFromPinnedDeployment() throws IOException { assertEquals(feedSourceWithPinnedDeploymentFeedVersion.projectId, feedSourceSummaries.get(0).projectId); assertEquals(feedSourceWithPinnedDeploymentFeedVersion.labelIds, feedSourceSummaries.get(0).labelIds); assertEquals(feedSourceWithPinnedDeploymentFeedVersion.url.toString(), feedSourceSummaries.get(0).url); + assertEquals(feedSourceWithPinnedDeploymentFeedVersion.noteIds, feedSourceSummaries.get(0).noteIds); + assertEquals(feedSourceWithPinnedDeploymentFeedVersion.organizationId(), feedSourceSummaries.get(0).organizationId); assertEquals(feedVersionFromPinnedDeployment.id, feedSourceSummaries.get(0).deployedFeedVersionId); assertEquals(feedVersionFromPinnedDeployment.validationSummary().startDate, feedSourceSummaries.get(0).deployedFeedVersionStartDate); assertEquals(feedVersionFromPinnedDeployment.validationSummary().endDate, feedSourceSummaries.get(0).deployedFeedVersionEndDate); @@ -395,7 +441,7 @@ private static FeedSource createFeedSource(String name, URL url, Project project * Helper method to create feed source. */ private static FeedSource createFeedSource(String id, String name, URL url, Project project, boolean persist) { - return createFeedSource(id, name, url, project, persist, null); + return createFeedSource(id, name, url, project, persist, null, null); } private static FeedSource createFeedSource( String id, @@ -403,7 +449,8 @@ private static FeedSource createFeedSource( URL url, Project project, boolean persist, - List labels + List labels, + List notes ) { FeedSource feedSource = new FeedSource(); if (id != null) feedSource.id = id; @@ -415,6 +462,7 @@ private static FeedSource createFeedSource( feedSource.retrievalMethod = FeedRetrievalMethod.FETCHED_AUTOMATICALLY; feedSource.url = url; if (labels != null) feedSource.labelIds = labels; + if (notes != null) feedSource.noteIds = notes; if (persist) Persistence.feedSources.create(feedSource); return feedSource; } @@ -461,10 +509,33 @@ private static FeedVersion createFeedVersion(String id, String feedSourceId, Loc } /** - * Helper method to create label + * Helper method to create note. + */ + private static Note createNote(String id, String body) { + Note note = new Note(id, body, false); + Persistence.notes.create(note); + return note; + } + + /** + * Helper method to create label. If the id is provided save the label now if not deffer to test to save. + */ + private static Label createLabel(String id, String name, String projectId) { + Label label; + if (id != null) { + label = new Label(id, name, "A label used during testing", "#123", false, projectId); + Persistence.labels.create(label); + } else { + label = new Label(name, "A label used during testing", "#123", false, projectId); + } + return label; + } + + /** + * Helper method to create label. */ - private static Label createLabel(String name) { - return new Label(name, "A label used during testing", "#123", false, projectToBeDeleted.id); + private static Label createLabel(String name, String projectId) { + return createLabel(null, name, projectId); } /** From 6695674e23dda07cd1f3a9eb32e347338286b179 Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Fri, 13 Oct 2023 12:47:13 +0100 Subject: [PATCH 196/248] refactor(Refactored label and note auth checks): --- .../controllers/api/FeedSourceController.java | 77 ++++++++++++++----- .../manager/models/FeedSourceSummary.java | 29 ++----- 2 files changed, 63 insertions(+), 43 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/controllers/api/FeedSourceController.java b/src/main/java/com/conveyal/datatools/manager/controllers/api/FeedSourceController.java index 080c79915..628d7815c 100644 --- a/src/main/java/com/conveyal/datatools/manager/controllers/api/FeedSourceController.java +++ b/src/main/java/com/conveyal/datatools/manager/controllers/api/FeedSourceController.java @@ -9,7 +9,6 @@ import com.conveyal.datatools.manager.extensions.ExternalFeedResource; import com.conveyal.datatools.manager.jobs.FetchSingleFeedJob; import com.conveyal.datatools.manager.jobs.NotifyUsersForSubscriptionJob; -import com.conveyal.datatools.manager.models.DeploymentSummary; import com.conveyal.datatools.manager.models.ExternalFeedSourceProperty; import com.conveyal.datatools.manager.models.FeedRetrievalMethod; import com.conveyal.datatools.manager.models.FeedSource; @@ -45,7 +44,6 @@ import static com.conveyal.datatools.common.utils.SparkUtils.getPOJOFromRequestBody; import static com.conveyal.datatools.common.utils.SparkUtils.logMessageAndHalt; import static com.conveyal.datatools.manager.models.ExternalFeedSourceProperty.constructId; -import static com.conveyal.datatools.manager.models.FeedSourceSummary.cleanFeedSourceSummaryForNonAdmins; import static com.conveyal.datatools.manager.models.transform.NormalizeFieldTransformation.getInvalidSubstitutionMessage; import static com.conveyal.datatools.manager.models.transform.NormalizeFieldTransformation.getInvalidSubstitutionPatterns; import static com.mongodb.client.model.Filters.in; @@ -88,7 +86,7 @@ private static Collection getProjectFeedSources(Request req, Respons boolean isAdmin = user.canAdministerProject(project); Collection projectFeedSources = project.retrieveProjectFeedSources(); - for (FeedSource source: projectFeedSources) { + for (FeedSource source : projectFeedSources) { String orgId = source.organizationId(); // If user can view or manage feed, add to list of feeds to return. NOTE: By default most users with access // to a project should be able to view all feed sources. Custom privileges would need to be provided to @@ -323,7 +321,7 @@ private static FeedSource deleteFeedSource(Request req, Response res) { /** * Re-fetch this feed from the feed source URL. */ - private static String fetch (Request req, Response res) { + private static String fetch(Request req, Response res) { FeedSource s = requestFeedSourceById(req, Actions.MANAGE); if (s.url == null) { logMessageAndHalt(req, HttpStatus.BAD_REQUEST_400, "Cannot fetch feed source with null URL."); @@ -341,7 +339,8 @@ private static String fetch (Request req, Response res) { /** * Helper function returns feed source if user has permission for specified action. - * @param req spark Request object from API request + * + * @param req spark Request object from API request * @param action action type (either "view" or Permission.MANAGE) * @return feedsource object for ID */ @@ -392,39 +391,79 @@ public static FeedSource checkFeedSourcePermissions(Request req, FeedSource feed return cleanFeedSourceForNonAdmins(feedSource, isProjectAdmin); } - /** Determines whether a change to a feed source is significant enough that it warrants sending a notification + /** + * Determines whether a change to a feed source is significant enough that it warrants sending a notification * * @param formerFeedSource A feed source object, without new changes * @param updatedFeedSource A feed source object, with new changes - * @return A boolean value indicating if the updated feed source is changed enough to warrant sending a notification. + * @return A boolean value indicating if the updated feed source is changed enough to warrant sending a notification. */ private static boolean shouldNotifyUsersOnFeedUpdated(FeedSource formerFeedSource, FeedSource updatedFeedSource) { - return - // If only labels have changed, don't send out an email - formerFeedSource.equalsExceptLabels(updatedFeedSource); + // If only labels have changed, don't send out an email. + return formerFeedSource.equalsExceptLabels(updatedFeedSource); } /** * Removes labels and notes from a feed that a user is not allowed to view. Returns cleaned feed source. - * @param feedSource The feed source to clean - * @param isAdmin Is the user an admin? Changes what is returned. - * @return A feed source containing only labels/notes the user is allowed to see + * + * @param feedSource The feed source to clean. + * @param isAdmin Is the user an admin? Changes what is returned. + * @return A feed source containing only labels/notes the user is allowed to see. */ protected static FeedSource cleanFeedSourceForNonAdmins(FeedSource feedSource, boolean isAdmin) { // Admin can view all feed labels, but a non-admin should only see those with adminOnly=false - feedSource.labelIds = Persistence.labels - .getFiltered(PersistenceUtils.applyAdminFilter(in("_id", feedSource.labelIds), isAdmin)).stream() + feedSource.labelIds = cleanFeedSourceLabelIdsForNonAdmins(feedSource.labelIds, isAdmin); + feedSource.noteIds = cleanFeedSourceNotesForNonAdmins(feedSource.noteIds, isAdmin); + return feedSource; + } + + /** + * Removes labels and notes from a feed that a user is not allowed to view. Returns cleaned feed source summary. + * + * @param feedSourceSummary The feed source to clean. + * @param isAdmin Is the user an admin? Changes what is returned. + * @return A feed source summary containing only labels/notes the user is allowed to see. + */ + protected static FeedSourceSummary cleanFeedSourceSummaryForNonAdmins(FeedSourceSummary feedSourceSummary, boolean isAdmin) { + // Admin can view all feed labels, but a non-admin should only see those with adminOnly=false + feedSourceSummary.labelIds = cleanFeedSourceLabelIdsForNonAdmins(feedSourceSummary.labelIds, isAdmin); + feedSourceSummary.noteIds = cleanFeedSourceNotesForNonAdmins(feedSourceSummary.noteIds, isAdmin); + return feedSourceSummary; + } + + /** + * Removes labels from a feed that a user is not allowed to view. Returns cleaned notes. + * + * @param labelIds The labels to clean. + * @param isAdmin Is the user an admin? Changes what is returned. + * @return Labels the user is allowed to see. + */ + protected static List cleanFeedSourceLabelIdsForNonAdmins(List labelIds, boolean isAdmin) { + // Admin can view all feed labels, but a non-admin should only see those with adminOnly=false. + return Persistence.labels + .getFiltered(PersistenceUtils.applyAdminFilter(in("_id", labelIds), isAdmin)).stream() .map(label -> label.id) .collect(Collectors.toList()); - feedSource.noteIds = Persistence.notes - .getFiltered(PersistenceUtils.applyAdminFilter(in("_id", feedSource.noteIds), isAdmin)).stream() + } + + /** + * Removes notes from a feed that a user is not allowed to view. Returns cleaned notes. + * + * @param noteIds The notes to clean. + * @param isAdmin Is the user an admin? Changes what is returned. + * @return Notes the user is allowed to see. + */ + protected static List cleanFeedSourceNotesForNonAdmins(List noteIds, boolean isAdmin) { + // Admin can view all feed notes, but a non-admin should only see those with adminOnly=false. + return Persistence.notes + .getFiltered(PersistenceUtils.applyAdminFilter(in("_id", noteIds), isAdmin)).stream() .map(note -> note.id) .collect(Collectors.toList()); - return feedSource; } + // FIXME: use generic API controller and return JSON documents via BSON/Mongo - public static void register (String apiPrefix) { + public static void register(String apiPrefix) { get(apiPrefix + "secure/feedsource/:id", FeedSourceController::getFeedSource, json::write); get(apiPrefix + "secure/feedsource", FeedSourceController::getProjectFeedSources, json::write); post(apiPrefix + "secure/feedsource", FeedSourceController::createFeedSource, json::write); diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java b/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java index 6116bac3f..15f93e57b 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java @@ -2,7 +2,6 @@ import com.conveyal.datatools.editor.utils.JacksonSerializers; import com.conveyal.datatools.manager.persistence.Persistence; -import com.conveyal.datatools.manager.utils.PersistenceUtils; import com.conveyal.gtfs.validator.ValidationResult; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import com.fasterxml.jackson.databind.annotation.JsonSerialize; @@ -21,7 +20,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.stream.Collectors; import static com.mongodb.client.model.Aggregates.group; import static com.mongodb.client.model.Aggregates.limit; @@ -93,26 +91,6 @@ public FeedSourceSummary(String projectId, String organizationId, Document feedS this.url = feedSourceDocument.getString("url"); } - /** - * Removes labels and notes from a feed that a user is not allowed to view. Returns cleaned feed source. - * @param feedSourceSummary The feed source to clean - * @param isAdmin Is the user an admin? Changes what is returned. - * @return A feed source containing only labels/notes the user is allowed to see - */ - public static FeedSourceSummary cleanFeedSourceSummaryForNonAdmins(FeedSourceSummary feedSourceSummary, boolean isAdmin) { - // Admin can view all feed labels, but a non-admin should only see those with adminOnly=false - feedSourceSummary.labelIds = Persistence.labels - .getFiltered(PersistenceUtils.applyAdminFilter(in("_id", feedSourceSummary.labelIds), isAdmin)).stream() - .map(label -> label.id) - .collect(Collectors.toList()); - feedSourceSummary.noteIds = Persistence.notes - .getFiltered(PersistenceUtils.applyAdminFilter(in("_id", feedSourceSummary.noteIds), isAdmin)).stream() - .map(note -> note.id) - .collect(Collectors.toList()); - return feedSourceSummary; - } - - /** * Set the appropriate feed version. For consistency, if no error count is available set the related number of * issues to null. @@ -573,8 +551,11 @@ public static class LatestValidationResult { public Integer errorCount; - /** Required for JSON de/serializing. **/ - public LatestValidationResult() {} + /** + * Required for JSON de/serializing. + **/ + public LatestValidationResult() { + } LatestValidationResult(FeedVersionSummary feedVersionSummary) { this.feedVersionId = feedVersionSummary.id; From 1601ef88c4c41dd27ccc8b9454f04048d2da5e05 Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Fri, 20 Oct 2023 14:42:18 -0400 Subject: [PATCH 197/248] use new mobilitydata repo --- pom.xml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index d4e240b40..cf430554d 100644 --- a/pom.xml +++ b/pom.xml @@ -208,6 +208,10 @@ always + + sonatype-staging + https://s01.oss.sonatype.org/content/repositories/orgmobilitydata-1029/ + @@ -261,9 +265,9 @@ test - com.github.mobilitydata - gtfs-validator - 4.1.0 + org.mobilitydata.gtfs-validator + gtfs-validator-main + 4.0.0 - 80a968cd3a071aa46a994ce2632535303a6e4384 + 5f34e048ab92936b98e344e69d8a41c9cf978dc4 From 4b641c05be2f1188a2a7ebcd0d5d89b8f43eef4d Mon Sep 17 00:00:00 2001 From: "philip.cline" Date: Mon, 6 Nov 2023 17:00:35 -0500 Subject: [PATCH 203/248] refactor(FeedVersionController): parse publishProprietaryFiles for snapshot req --- .../manager/controllers/api/FeedVersionController.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/conveyal/datatools/manager/controllers/api/FeedVersionController.java b/src/main/java/com/conveyal/datatools/manager/controllers/api/FeedVersionController.java index 8f855efea..9dffd6adc 100644 --- a/src/main/java/com/conveyal/datatools/manager/controllers/api/FeedVersionController.java +++ b/src/main/java/com/conveyal/datatools/manager/controllers/api/FeedVersionController.java @@ -172,6 +172,7 @@ protected static FeedVersion cleanFeedVersionForNonAdmins(FeedVersion feedVersio private static boolean createFeedVersionFromSnapshot (Request req, Response res) { Auth0UserProfile userProfile = req.attribute("user"); + Boolean publishProprietaryFiles = Boolean.parseBoolean(req.queryParams("publishProprietaryFiles")); // TODO: Should the ability to create a feedVersion from snapshot be controlled by the 'edit-gtfs' privilege? FeedSource feedSource = requestFeedSourceById(req, Actions.MANAGE); Snapshot snapshot = Persistence.snapshots.getById(req.queryParams("snapshotId")); @@ -180,7 +181,7 @@ private static boolean createFeedVersionFromSnapshot (Request req, Response res) } // Publishing the proprietary files will preserve the pattern names in the newly published feed version. CreateFeedVersionFromSnapshotJob createFromSnapshotJob = - new CreateFeedVersionFromSnapshotJob(feedSource, snapshot, userProfile, true); + new CreateFeedVersionFromSnapshotJob(feedSource, snapshot, userProfile, publishProprietaryFiles); JobUtils.heavyExecutor.execute(createFromSnapshotJob); return true; From ffff51a9b38e5a0053a0ea44187c696643b1d440 Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Tue, 14 Nov 2023 15:32:09 +0000 Subject: [PATCH 204/248] refactor(pom.xml): Bumped GTFS-Lib version --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 4de555203..84807c320 100644 --- a/pom.xml +++ b/pom.xml @@ -268,7 +268,7 @@ com.github.conveyal gtfs-lib - 49a722e161a52cae98fba9cb23995dae82fd01e4 + 9bc752d3ef28e0ea2eda439ebd915fe02a37b0f4 From 1c03a7571ba64169656ea51ac1e897e858106753 Mon Sep 17 00:00:00 2001 From: "philip.cline" Date: Wed, 15 Nov 2023 13:50:16 -0500 Subject: [PATCH 205/248] refactor(SnapshotController): update formatting --- .../datatools/editor/controllers/api/SnapshotController.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/conveyal/datatools/editor/controllers/api/SnapshotController.java b/src/main/java/com/conveyal/datatools/editor/controllers/api/SnapshotController.java index ebe45b692..58d93fee9 100644 --- a/src/main/java/com/conveyal/datatools/editor/controllers/api/SnapshotController.java +++ b/src/main/java/com/conveyal/datatools/editor/controllers/api/SnapshotController.java @@ -65,7 +65,7 @@ private static Snapshot getSnapshotFromRequest(Request req) { private static boolean getPublishProprietaryFiles(Request req) { return Boolean.parseBoolean( - req.queryParamOrDefault("publishProprietaryFiles",Boolean.FALSE.toString()) + req.queryParamOrDefault("publishProprietaryFiles", Boolean.FALSE.toString()) ); } From b2a3116e10b2f7bd95554ad220a2568344b39d66 Mon Sep 17 00:00:00 2001 From: "philip.cline" Date: Wed, 15 Nov 2023 13:57:31 -0500 Subject: [PATCH 206/248] refactor(deps): update gtfs lib commit --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 5cb786373..3d44b0dd6 100644 --- a/pom.xml +++ b/pom.xml @@ -268,7 +268,7 @@ com.github.conveyal gtfs-lib - 10307d43c6610d5dd1cc07d30091596816772727 + 10307d43c6 From 0a7763fc28d01459e00dfe0053fb93e79baf437b Mon Sep 17 00:00:00 2001 From: "philip.cline" Date: Wed, 15 Nov 2023 15:08:33 -0500 Subject: [PATCH 207/248] refactor(deps): update gtfs-lib commit --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index b2f2cd2fe..bec80e21d 100644 --- a/pom.xml +++ b/pom.xml @@ -272,7 +272,7 @@ com.github.conveyal gtfs-lib - 9bc752d3ef28e0ea2eda439ebd915fe02a37b0f4 + 9bc752d3ef From d24605431a6570456bbdb4738bfe7f2a598b99bf Mon Sep 17 00:00:00 2001 From: "philip.cline" Date: Wed, 15 Nov 2023 20:22:16 -0500 Subject: [PATCH 208/248] feat(PublicFeedsPage): add config option for support email --- configurations/default/server.yml.tmp | 1 + .../datatools/manager/jobs/PublishProjectFeedsJob.java | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/configurations/default/server.yml.tmp b/configurations/default/server.yml.tmp index 06bdaa3f7..2a94ab43f 100644 --- a/configurations/default/server.yml.tmp +++ b/configurations/default/server.yml.tmp @@ -8,6 +8,7 @@ application: notifications_enabled: false docs_url: http://conveyal-data-tools.readthedocs.org support_email: support@ibigroup.com + public_gtfs_contact_email: public-gtfs-support-email@yourdomain.com port: 4000 data: gtfs: /tmp diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/PublishProjectFeedsJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/PublishProjectFeedsJob.java index 01ef645a5..fc881e014 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/PublishProjectFeedsJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/PublishProjectFeedsJob.java @@ -16,6 +16,8 @@ import java.io.IOException; import java.text.SimpleDateFormat; +import static com.conveyal.datatools.manager.DataManager.getConfigPropertyAsText; + /** * Publish the latest GTFS files for all public feeds in a project. */ @@ -23,6 +25,7 @@ public class PublishProjectFeedsJob extends MonitorableJob { public static final Logger LOG = LoggerFactory.getLogger(PublishProjectFeedsJob.class); private Project project; + private static final String dataSupportEmail = getConfigPropertyAsText("application.public_gtfs_contact_email"); public PublishProjectFeedsJob(Project project, Auth0UserProfile owner) { super(owner, "Generating public html for " + project.name, JobType.MAKE_PROJECT_PUBLIC); @@ -54,6 +57,9 @@ public void jobLogic () { r.append("\n"); r.append("

" + title + "

\n"); r.append("The following feeds, in GTFS format, are available for download and use.\n"); + if (dataSupportEmail != null) { + r.append(String.format("If you have inquiries, please contact us at: %1$s", dataSupportEmail)); + } r.append("
    \n"); status.update("Ensuring public GTFS files are up-to-date.", 50); project.retrieveProjectFeedSources().stream() From 811453704cd40370fb6d99321ee83b923424574a Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Thu, 16 Nov 2023 10:50:30 +0000 Subject: [PATCH 209/248] refactor(ReferenceTableDiscovery.java): Updated trip service id key key now includes reference to schedule exceptions matching trip service_id references --- .../manager/jobs/feedmerge/ReferenceTableDiscovery.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/ReferenceTableDiscovery.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/ReferenceTableDiscovery.java index df16c2d6f..a972dbc25 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/ReferenceTableDiscovery.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/ReferenceTableDiscovery.java @@ -14,7 +14,14 @@ public class ReferenceTableDiscovery { public enum ReferenceTableKey { TRIP_SERVICE_ID_KEY( - String.join(REF_TABLE_SEPARATOR, Table.TRIPS.name, SERVICE_ID, Table.CALENDAR.name, Table.CALENDAR_DATES.name) + String.join( + REF_TABLE_SEPARATOR, + Table.TRIPS.name, + SERVICE_ID, + Table.CALENDAR.name, + Table.CALENDAR_DATES.name, + Table.SCHEDULE_EXCEPTIONS.name + ) ); private final String value; From b72cff37ff7fa287f813f621cefb804bffc0a6cf Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Mon, 20 Nov 2023 17:13:08 +0100 Subject: [PATCH 210/248] support new Url config fields --- .../datatools/manager/models/Deployment.java | 49 +++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/src/main/java/com/conveyal/datatools/manager/models/Deployment.java b/src/main/java/com/conveyal/datatools/manager/models/Deployment.java index 04731841a..ab54ccc7f 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/Deployment.java +++ b/src/main/java/com/conveyal/datatools/manager/models/Deployment.java @@ -24,6 +24,8 @@ import java.awt.geom.Point2D; import java.awt.geom.Rectangle2D; +import java.io.BufferedInputStream; +import java.io.ByteArrayOutputStream; import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; @@ -209,7 +211,9 @@ public DeployJob.DeploySummary latest () { public String routerId; public String customBuildConfig; + public String customBuildConfigUrl; public String customRouterConfig; + public String customRouterConfigUrl; public List customFiles = new ArrayList<>(); @@ -414,9 +418,44 @@ public void dump (File output, boolean includeManifest, boolean includeOsm, bool out.close(); } + private byte[] downloadFileFromURL(URL url) throws IOException { + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + + InputStream stream = null; + try { + byte[] chunk = new byte[4096]; + int bytesRead; + stream = url.openStream(); + + while ((bytesRead = stream.read(chunk)) > 0) { + outputStream.write(chunk, 0, bytesRead); + } + + } catch (IOException e) { + e.printStackTrace(); + return new byte[0]; + } finally { + outputStream.close(); + if (stream != null) stream.close(); + } + + return outputStream.toByteArray(); + } + /** Generate build config for deployment as byte array (for writing to file output stream). */ public byte[] generateBuildConfig() { Project project = this.parentProject(); + + // If there is a custombuildconfigUrl, set the customBuildConfig based on the URL + if (this.customBuildConfigUrl != null) { + try { + this.customBuildConfig = new String(downloadFileFromURL(new URL(customBuildConfigUrl)), StandardCharsets.UTF_8); + } catch (IOException e) { + LOG.error("Could not download file from {}", customBuildConfigUrl); + throw new RuntimeException(customBuildConfigUrl); + } + } + return customBuildConfig != null ? customBuildConfig.getBytes(StandardCharsets.UTF_8) : project.buildConfig != null @@ -453,6 +492,16 @@ private String writeToString(O object) { public byte[] generateRouterConfig() throws IOException { Project project = this.parentProject(); + if (this.customRouterConfigUrl != null) { + try { + // TODO: should Mongo be updated here? + this.customRouterConfig = new String(downloadFileFromURL(new URL(customRouterConfigUrl)), StandardCharsets.UTF_8); + } catch (IOException e) { + LOG.error("Could not download file from {}", customRouterConfigUrl); + throw new RuntimeException(e); + } + } + byte[] customRouterConfigString = customRouterConfig != null ? customRouterConfig.getBytes(StandardCharsets.UTF_8) : null; From b3210a8e049472f8cc7f88f4e31ba165543ec5b7 Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Tue, 21 Nov 2023 12:02:57 +0000 Subject: [PATCH 211/248] refactor(Refactored the config download aspect of Deployment and added test): --- .../datatools/manager/models/Deployment.java | 82 +++++++------------ .../datatools/manager/utils/HttpUtils.java | 20 +++++ .../datatools/manager/jobs/DeployJobTest.java | 11 +++ 3 files changed, 61 insertions(+), 52 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/models/Deployment.java b/src/main/java/com/conveyal/datatools/manager/models/Deployment.java index ab54ccc7f..111b6c2b8 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/Deployment.java +++ b/src/main/java/com/conveyal/datatools/manager/models/Deployment.java @@ -51,6 +51,7 @@ import java.util.zip.ZipOutputStream; import static com.conveyal.datatools.manager.DataManager.getConfigPropertyAsText; +import static com.conveyal.datatools.manager.utils.HttpUtils.downloadFileFromURL; import static com.mongodb.client.model.Filters.and; import static com.mongodb.client.model.Filters.eq; @@ -418,49 +419,36 @@ public void dump (File output, boolean includeManifest, boolean includeOsm, bool out.close(); } - private byte[] downloadFileFromURL(URL url) throws IOException { - ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); - - InputStream stream = null; - try { - byte[] chunk = new byte[4096]; - int bytesRead; - stream = url.openStream(); - - while ((bytesRead = stream.read(chunk)) > 0) { - outputStream.write(chunk, 0, bytesRead); - } - - } catch (IOException e) { - e.printStackTrace(); - return new byte[0]; - } finally { - outputStream.close(); - if (stream != null) stream.close(); - } - - return outputStream.toByteArray(); - } - - /** Generate build config for deployment as byte array (for writing to file output stream). */ - public byte[] generateBuildConfig() { - Project project = this.parentProject(); - - // If there is a custombuildconfigUrl, set the customBuildConfig based on the URL - if (this.customBuildConfigUrl != null) { + /** Download config from provided URL. */ + public String downloadConfig(String configUrl) throws IOException { + if (configUrl != null) { try { - this.customBuildConfig = new String(downloadFileFromURL(new URL(customBuildConfigUrl)), StandardCharsets.UTF_8); + return new String(downloadFileFromURL(new URL(configUrl)), StandardCharsets.UTF_8); } catch (IOException e) { - LOG.error("Could not download file from {}", customBuildConfigUrl); - throw new RuntimeException(customBuildConfigUrl); + String message = String.format("Could not download file from %s.", configUrl); + LOG.error(message); + throw new IOException(message, e); } } + return null; + } + /** Generate build config for deployment as byte array (for writing to file output stream). */ + public byte[] generateBuildConfig() throws IOException { + customBuildConfig = downloadConfig(customBuildConfigUrl); return customBuildConfig != null ? customBuildConfig.getBytes(StandardCharsets.UTF_8) - : project.buildConfig != null - ? writeToBytes(project.buildConfig) - : null; + : getProjectBuildConfig(); + } + + /** + * If a project build config exists, return this as a byte array, or null if not available. + */ + private byte[] getProjectBuildConfig() { + Project project = parentProject(); + return project.buildConfig != null + ? writeToBytes(project.buildConfig) + : null; } public String generateBuildConfigAsString() { @@ -490,25 +478,15 @@ private String writeToString(O object) { /** Generate router config for deployment as string. */ public byte[] generateRouterConfig() throws IOException { - Project project = this.parentProject(); - - if (this.customRouterConfigUrl != null) { - try { - // TODO: should Mongo be updated here? - this.customRouterConfig = new String(downloadFileFromURL(new URL(customRouterConfigUrl)), StandardCharsets.UTF_8); - } catch (IOException e) { - LOG.error("Could not download file from {}", customRouterConfigUrl); - throw new RuntimeException(e); - } - } + customRouterConfig = downloadConfig(customRouterConfigUrl); byte[] customRouterConfigString = customRouterConfig != null - ? customRouterConfig.getBytes(StandardCharsets.UTF_8) - : null; + ? customRouterConfig.getBytes(StandardCharsets.UTF_8) + : null; - byte[] routerConfigString = project.routerConfig != null - ? writeToBytes(project.routerConfig) - : null; + byte[] routerConfigString = parentProject().routerConfig != null + ? writeToBytes(parentProject().routerConfig) + : null; // If both router configs are present, merge the JSON before returning // Merger code from: https://stackoverflow.com/questions/35747813/how-to-merge-two-json-strings-into-one-in-java diff --git a/src/main/java/com/conveyal/datatools/manager/utils/HttpUtils.java b/src/main/java/com/conveyal/datatools/manager/utils/HttpUtils.java index fdd6ed0d5..2f3461a9a 100644 --- a/src/main/java/com/conveyal/datatools/manager/utils/HttpUtils.java +++ b/src/main/java/com/conveyal/datatools/manager/utils/HttpUtils.java @@ -14,9 +14,12 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.io.InputStream; import java.io.UnsupportedEncodingException; import java.net.URI; +import java.net.URL; import java.util.ArrayList; import java.util.List; @@ -113,4 +116,21 @@ public static SimpleHttpResponse httpRequestRawResponse( return null; } } + + /** + * Download a file from a URL and return as a byte array. + */ + public static byte[] downloadFileFromURL(URL url) throws IOException { + try ( + InputStream inputStream = url.openStream(); + ByteArrayOutputStream outputStream = new ByteArrayOutputStream() + ) { + byte[] chunk = new byte[4096]; + int bytesRead; + while ((bytesRead = inputStream.read(chunk)) >= 0) { + outputStream.write(chunk, 0, bytesRead); + } + return outputStream.toByteArray(); + } + } } diff --git a/src/test/java/com/conveyal/datatools/manager/jobs/DeployJobTest.java b/src/test/java/com/conveyal/datatools/manager/jobs/DeployJobTest.java index d3812276a..87b6db729 100644 --- a/src/test/java/com/conveyal/datatools/manager/jobs/DeployJobTest.java +++ b/src/test/java/com/conveyal/datatools/manager/jobs/DeployJobTest.java @@ -27,6 +27,8 @@ import static com.zenika.snapshotmatcher.SnapshotMatcher.matchesSnapshot; import static org.hamcrest.MatcherAssert.assertThat; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assumptions.assumeTrue; /** @@ -80,6 +82,15 @@ public static void setUp() throws IOException { Persistence.deployments.create(deployment); } + @Test + void canDownloadConfigs() throws IOException { + deployment.customBuildConfigUrl = "http://www.google.com"; + assertNotNull(deployment.downloadConfig(deployment.customBuildConfigUrl)); + + deployment.customRouterConfigUrl = "http://www.google.com"; + assertNotNull(deployment.downloadConfig(deployment.customRouterConfigUrl)); + } + /** * Tests that the otp-runner manifest and user data for a graph build + run server instance can be generated * properly From a2cbede21f100beaf0d8a3c0ea89d25eee8aa6db Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Tue, 21 Nov 2023 14:14:20 +0000 Subject: [PATCH 212/248] refactor(Deployment.java): Refactor downloaded config assignment. --- .../datatools/manager/models/Deployment.java | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/models/Deployment.java b/src/main/java/com/conveyal/datatools/manager/models/Deployment.java index 111b6c2b8..97737fe1f 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/Deployment.java +++ b/src/main/java/com/conveyal/datatools/manager/models/Deployment.java @@ -433,9 +433,14 @@ public String downloadConfig(String configUrl) throws IOException { return null; } - /** Generate build config for deployment as byte array (for writing to file output stream). */ + /** Generate build config for deployment as byte array (for writing to file output stream). If an external build + * config is available and is successfully downloaded, use this instead of the deployment build config. If there is + * no deployment build config, use the project build config. */ public byte[] generateBuildConfig() throws IOException { - customBuildConfig = downloadConfig(customBuildConfigUrl); + String downloadedConfig = downloadConfig(customBuildConfigUrl); + if (downloadedConfig != null) { + customBuildConfig = downloadedConfig; + } return customBuildConfig != null ? customBuildConfig.getBytes(StandardCharsets.UTF_8) : getProjectBuildConfig(); @@ -478,7 +483,10 @@ private String writeToString(O object) { /** Generate router config for deployment as string. */ public byte[] generateRouterConfig() throws IOException { - customRouterConfig = downloadConfig(customRouterConfigUrl); + String downloadedConfig = downloadConfig(customRouterConfigUrl); + if (downloadedConfig != null) { + customRouterConfig = downloadedConfig; + } byte[] customRouterConfigString = customRouterConfig != null ? customRouterConfig.getBytes(StandardCharsets.UTF_8) From f142def9eb5996c12711f09d47d6c516799679b1 Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Tue, 21 Nov 2023 14:18:36 +0000 Subject: [PATCH 213/248] refactor(Removed unused imports.): --- .../java/com/conveyal/datatools/manager/models/Deployment.java | 2 -- .../java/com/conveyal/datatools/manager/jobs/DeployJobTest.java | 1 - 2 files changed, 3 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/models/Deployment.java b/src/main/java/com/conveyal/datatools/manager/models/Deployment.java index 97737fe1f..5357d0732 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/Deployment.java +++ b/src/main/java/com/conveyal/datatools/manager/models/Deployment.java @@ -24,8 +24,6 @@ import java.awt.geom.Point2D; import java.awt.geom.Rectangle2D; -import java.io.BufferedInputStream; -import java.io.ByteArrayOutputStream; import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; diff --git a/src/test/java/com/conveyal/datatools/manager/jobs/DeployJobTest.java b/src/test/java/com/conveyal/datatools/manager/jobs/DeployJobTest.java index 87b6db729..e0504d589 100644 --- a/src/test/java/com/conveyal/datatools/manager/jobs/DeployJobTest.java +++ b/src/test/java/com/conveyal/datatools/manager/jobs/DeployJobTest.java @@ -28,7 +28,6 @@ import static org.hamcrest.MatcherAssert.assertThat; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assumptions.assumeTrue; /** From 9d0ca4f8b091090696461f602756b9ff459524e3 Mon Sep 17 00:00:00 2001 From: "philip.cline" Date: Tue, 21 Nov 2023 15:03:04 -0500 Subject: [PATCH 214/248] chore(deps): update gtfs-lib commit after merge --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 43c5a8182..3806912d1 100644 --- a/pom.xml +++ b/pom.xml @@ -268,7 +268,7 @@ com.github.conveyal gtfs-lib - 5f34e048ab92936b98e344e69d8a41c9cf978dc4 + b657dfe7d7c75eacfbc165cf37c0c0ff3f8cd23e From 8828781bdadd255122b91dbee04aa27407e9cc28 Mon Sep 17 00:00:00 2001 From: "philip.cline" Date: Tue, 21 Nov 2023 15:07:10 -0500 Subject: [PATCH 215/248] chore(deps): fix gtfs-lib commit after merge --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 3806912d1..bf2c081f0 100644 --- a/pom.xml +++ b/pom.xml @@ -268,7 +268,7 @@ com.github.conveyal gtfs-lib - b657dfe7d7c75eacfbc165cf37c0c0ff3f8cd23e + b657dfe7d7 From 8357af0ee6d2f20827e0a32772e00f7a88bf3eb7 Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Mon, 27 Nov 2023 09:38:16 -0500 Subject: [PATCH 216/248] address pr feedback --- .../java/com/conveyal/datatools/manager/models/Deployment.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/conveyal/datatools/manager/models/Deployment.java b/src/main/java/com/conveyal/datatools/manager/models/Deployment.java index 5357d0732..cb06e25fd 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/Deployment.java +++ b/src/main/java/com/conveyal/datatools/manager/models/Deployment.java @@ -421,9 +421,10 @@ public void dump (File output, boolean includeManifest, boolean includeOsm, bool public String downloadConfig(String configUrl) throws IOException { if (configUrl != null) { try { + // TODO: validate JSON? return new String(downloadFileFromURL(new URL(configUrl)), StandardCharsets.UTF_8); } catch (IOException e) { - String message = String.format("Could not download file from %s.", configUrl); + String message = String.format("Could not download config file from %s.", configUrl); LOG.error(message); throw new IOException(message, e); } From 1ad4236d31e7017f8065ae1a7a5fa816975f3f4b Mon Sep 17 00:00:00 2001 From: "philip.cline" Date: Thu, 30 Nov 2023 11:21:15 -0500 Subject: [PATCH 217/248] chore(gtfs-lib): add ibi fork, update gtfs-lib --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index fff59653c..9abe57646 100644 --- a/pom.xml +++ b/pom.xml @@ -269,10 +269,10 @@ - AWS S3 SDK - putting/getting objects into/out of S3. --> - com.github.conveyal + com.github.ibi-group gtfs-lib - b657dfe7d7 + e12dcf8 From 64c7632f231c3f761a9f5865866af1845e4c5f4f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 5 Dec 2023 22:17:30 +0000 Subject: [PATCH 218/248] chore(deps): bump ch.qos.logback:logback-classic from 1.2.3 to 1.2.13 Bumps [ch.qos.logback:logback-classic](https://github.com/qos-ch/logback) from 1.2.3 to 1.2.13. - [Commits](https://github.com/qos-ch/logback/compare/v_1.2.3...v_1.2.13) --- updated-dependencies: - dependency-name: ch.qos.logback:logback-classic dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 9abe57646..cea10e000 100644 --- a/pom.xml +++ b/pom.xml @@ -228,7 +228,7 @@ ch.qos.logback logback-classic - 1.2.3 + 1.2.13 From 3c2d8bca7b9325e9a978799df6be74c167128f76 Mon Sep 17 00:00:00 2001 From: "philip.cline" Date: Wed, 20 Dec 2023 11:49:16 -0500 Subject: [PATCH 219/248] feat(Md Validator): upgrade md validator --- pom.xml | 2 +- .../com/conveyal/datatools/manager/models/FeedVersion.java | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 9abe57646..03f5c710d 100644 --- a/pom.xml +++ b/pom.xml @@ -262,7 +262,7 @@ org.mobilitydata.gtfs-validator gtfs-validator-main - 4.1.0 + 4.2.0 - e12dcf8 + 3fa0da9 From b2e67553196f5383ed60c327fa911b513aab67fc Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Mon, 25 Dec 2023 17:53:33 -0500 Subject: [PATCH 221/248] correct shared stops validator behaviors --- .../jobs/validation/SharedStopsValidator.java | 71 +++++++++++++------ 1 file changed, 50 insertions(+), 21 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/validation/SharedStopsValidator.java b/src/main/java/com/conveyal/datatools/manager/jobs/validation/SharedStopsValidator.java index d608fe249..209aefd4b 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/validation/SharedStopsValidator.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/validation/SharedStopsValidator.java @@ -13,10 +13,8 @@ import java.io.IOException; import java.util.ArrayList; -import java.util.HashMap; import java.util.HashSet; import java.util.List; -import java.util.Set; public class SharedStopsValidator extends FeedValidator { @@ -58,11 +56,40 @@ public void validate() { CsvReader configReader = CsvReader.parse(config); - // TODO: pull indicies from the CSV header - final int STOP_GROUP_ID_INDEX = 0; - final int STOP_ID_INDEX = 2; - final int IS_PRIMARY_INDEX = 3; - final int FEED_ID_INDEX = 1; + int STOP_GROUP_ID_INDEX = -1; + int STOP_ID_INDEX = -1; + int IS_PRIMARY_INDEX = -1; + int FEED_ID_INDEX = -1; + + try { + configReader.setHeaders(new String[]{"stop_group_id", "feed_id", "stop_id", "is_primary"}); + String[] headers = configReader.getHeaders(); + for (int i = 0; i < headers.length; i++) { + String header = headers[i]; + switch(header) { + case "stop_group_id": + STOP_GROUP_ID_INDEX = i; + break; + case "feed_id": + FEED_ID_INDEX = i; + break; + case "stop_id": + STOP_ID_INDEX = i; + break; + case "is_primary": + IS_PRIMARY_INDEX = i; + break; + default: + throw new RuntimeException("shared_stops.csv contained invalid headers!"); + } + } + + if (STOP_GROUP_ID_INDEX==-1 || FEED_ID_INDEX==-1 || STOP_ID_INDEX==-1 || IS_PRIMARY_INDEX==-1) { + throw new RuntimeException("shared_stops.csv is missing headers!"); + } + } catch (IOException e) { + throw new RuntimeException("shared_stops.csv was invalid! " + e.toString()); + } // Build list of stop Ids. List stopIds = new ArrayList<>(); @@ -74,7 +101,7 @@ public void validate() { } // Initialize hashmaps to hold duplication info - HashMap> stopGroupStops = new HashMap<>(); + HashSet seenStopIds = new HashSet<>(); HashSet stopGroupsWithPrimaryStops = new HashSet<>(); try { @@ -88,26 +115,28 @@ public void validate() { continue; } - stopGroupStops.putIfAbsent(stopGroupId, new HashSet<>()); - Set seenStopIds = stopGroupStops.get(stopGroupId); - // Check for SS_01 (stop id appearing in multiple stop groups) + // Make sure this error is only returned if we are inside the feed that is being checked if (seenStopIds.contains(stopId)) { - registerError(stops - .stream() - .filter(stop -> stop.stop_id.equals(stopId)) - .findFirst() - .orElse(stops.get(0)), NewGTFSErrorType.MULTIPLE_SHARED_STOPS_GROUPS - ); + if (this.feedId.equals(sharedStopFeedId)) { + registerError(stops + .stream() + .filter(stop -> stop.stop_id.equals(stopId)) + .findFirst() + .orElse(new Stop()), NewGTFSErrorType.MULTIPLE_SHARED_STOPS_GROUPS + ); + } } else { seenStopIds.add(stopId); } // Check for SS_02 (multiple primary stops per stop group) - if (stopGroupsWithPrimaryStops.contains(stopGroupId)) { - registerError(NewGTFSError.forFeed(NewGTFSErrorType.SHARED_STOP_GROUP_MUTLIPLE_PRIMARY_STOPS, stopGroupId)); - } else if (configReader.get(IS_PRIMARY_INDEX).equals("true")) { - stopGroupsWithPrimaryStops.add(stopGroupId); + if (configReader.get(IS_PRIMARY_INDEX).equals("1") || configReader.get(IS_PRIMARY_INDEX).equals("true")) { + if (stopGroupsWithPrimaryStops.contains(stopGroupId)) { + registerError(NewGTFSError.forFeed(NewGTFSErrorType.SHARED_STOP_GROUP_MUTLIPLE_PRIMARY_STOPS, stopGroupId)); + } else { + stopGroupsWithPrimaryStops.add(stopGroupId); + } } // Check for SS_03 (stop_id referenced doesn't exist) From 6ad9abfce4f3fd21df1f0dbf74e2e2876dbb1e1b Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Mon, 25 Dec 2023 17:56:49 -0500 Subject: [PATCH 222/248] fix formatting --- .../datatools/manager/jobs/validation/SharedStopsValidator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/validation/SharedStopsValidator.java b/src/main/java/com/conveyal/datatools/manager/jobs/validation/SharedStopsValidator.java index 209aefd4b..d06dd0729 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/validation/SharedStopsValidator.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/validation/SharedStopsValidator.java @@ -84,7 +84,7 @@ public void validate() { } } - if (STOP_GROUP_ID_INDEX==-1 || FEED_ID_INDEX==-1 || STOP_ID_INDEX==-1 || IS_PRIMARY_INDEX==-1) { + if (STOP_GROUP_ID_INDEX == -1 || FEED_ID_INDEX == -1 || STOP_ID_INDEX == -1 || IS_PRIMARY_INDEX == -1) { throw new RuntimeException("shared_stops.csv is missing headers!"); } } catch (IOException e) { From 8eabd138e54ce55e4ba6c4d1cfc42450bf4b02bf Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Fri, 5 Jan 2024 20:44:40 +0100 Subject: [PATCH 223/248] refactor: address pr feedback --- .../manager/jobs/validation/SharedStopsValidator.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/validation/SharedStopsValidator.java b/src/main/java/com/conveyal/datatools/manager/jobs/validation/SharedStopsValidator.java index d06dd0729..83d4dd0d7 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/validation/SharedStopsValidator.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/validation/SharedStopsValidator.java @@ -62,11 +62,13 @@ public void validate() { int FEED_ID_INDEX = -1; try { - configReader.setHeaders(new String[]{"stop_group_id", "feed_id", "stop_id", "is_primary"}); + configReader.readRecord(); + + configReader.setHeaders(configReader.getRawRecord().split(",")); String[] headers = configReader.getHeaders(); for (int i = 0; i < headers.length; i++) { String header = headers[i]; - switch(header) { + switch (header) { case "stop_group_id": STOP_GROUP_ID_INDEX = i; break; @@ -88,7 +90,7 @@ public void validate() { throw new RuntimeException("shared_stops.csv is missing headers!"); } } catch (IOException e) { - throw new RuntimeException("shared_stops.csv was invalid! " + e.toString()); + throw new RuntimeException("shared_stops.csv was invalid!", e); } // Build list of stop Ids. @@ -118,7 +120,7 @@ public void validate() { // Check for SS_01 (stop id appearing in multiple stop groups) // Make sure this error is only returned if we are inside the feed that is being checked if (seenStopIds.contains(stopId)) { - if (this.feedId.equals(sharedStopFeedId)) { + if (feedId.equals(sharedStopFeedId)) { registerError(stops .stream() .filter(stop -> stop.stop_id.equals(stopId)) From a55e37e864e61e08cef8c97f1cee55ae59b41acb Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Fri, 5 Jan 2024 21:00:07 +0100 Subject: [PATCH 224/248] deps: update gtfs-lib --- pom.xml | 2 +- .../datatools/manager/jobs/validation/SharedStopsValidator.java | 2 +- .../java/com/conveyal/datatools/manager/models/FeedVersion.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index 31fbf057d..3e478f354 100644 --- a/pom.xml +++ b/pom.xml @@ -272,7 +272,7 @@ com.github.ibi-group gtfs-lib - 3fa0da9 + 761456fdc66bc1b72fc860b5e5853b0f2904e009 diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/validation/SharedStopsValidator.java b/src/main/java/com/conveyal/datatools/manager/jobs/validation/SharedStopsValidator.java index 83d4dd0d7..b185d0f87 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/validation/SharedStopsValidator.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/validation/SharedStopsValidator.java @@ -135,7 +135,7 @@ public void validate() { // Check for SS_02 (multiple primary stops per stop group) if (configReader.get(IS_PRIMARY_INDEX).equals("1") || configReader.get(IS_PRIMARY_INDEX).equals("true")) { if (stopGroupsWithPrimaryStops.contains(stopGroupId)) { - registerError(NewGTFSError.forFeed(NewGTFSErrorType.SHARED_STOP_GROUP_MUTLIPLE_PRIMARY_STOPS, stopGroupId)); + registerError(NewGTFSError.forFeed(NewGTFSErrorType.SHARED_STOP_GROUP_MULTIPLE_PRIMARY_STOPS, stopGroupId)); } else { stopGroupsWithPrimaryStops.add(stopGroupId); } diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java b/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java index 060f8589b..071ff9556 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java @@ -517,7 +517,7 @@ public boolean hasBlockingIssuesForPublishing() { NewGTFSErrorType.TABLE_IN_SUBDIRECTORY, NewGTFSErrorType.WRONG_NUMBER_OF_FIELDS, NewGTFSErrorType.MULTIPLE_SHARED_STOPS_GROUPS, - NewGTFSErrorType.SHARED_STOP_GROUP_MUTLIPLE_PRIMARY_STOPS + NewGTFSErrorType.SHARED_STOP_GROUP_MULTIPLE_PRIMARY_STOPS )); } From 7521cc7b5a40d52717e60dce4f9766b8d2358063 Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Thu, 11 Jan 2024 21:09:47 +0200 Subject: [PATCH 225/248] extract testable functionality to unit tests --- .../jobs/validation/SharedStopsValidator.java | 67 ++++++++++++------- .../validation/SharedStopsValidatorTest.java | 51 ++++++++++++++ .../canHandleVariousCorrectCSV-0.json | 6 ++ .../canHandleVariousCorrectCSV-1.json | 6 ++ .../canHandleVariousCorrectCSV-2.json | 6 ++ 5 files changed, 113 insertions(+), 23 deletions(-) create mode 100644 src/test/java/com/conveyal/datatools/manager/jobs/validation/SharedStopsValidatorTest.java create mode 100644 src/test/resources/snapshots/com/conveyal/datatools/manager/jobs/validation/SharedStopsValidatorTest/canHandleVariousCorrectCSV-0.json create mode 100644 src/test/resources/snapshots/com/conveyal/datatools/manager/jobs/validation/SharedStopsValidatorTest/canHandleVariousCorrectCSV-1.json create mode 100644 src/test/resources/snapshots/com/conveyal/datatools/manager/jobs/validation/SharedStopsValidatorTest/canHandleVariousCorrectCSV-2.json diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/validation/SharedStopsValidator.java b/src/main/java/com/conveyal/datatools/manager/jobs/validation/SharedStopsValidator.java index b185d0f87..dfc795879 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/validation/SharedStopsValidator.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/validation/SharedStopsValidator.java @@ -13,9 +13,17 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Map; +enum SharedStopsHeader { + STOP_GROUP_ID_INDEX, + STOP_ID_INDEX, + IS_PRIMARY_INDEX, + FEED_ID_INDEX +} public class SharedStopsValidator extends FeedValidator { private static final Logger LOG = LoggerFactory.getLogger(SharedStopsValidator.class); @@ -47,52 +55,62 @@ public SharedStopsValidator(Feed feed, SQLErrorStorage errorStorage, Project pro this.feedId = feedId; } - @Override - public void validate() { - String config = project.sharedStopsConfig; - if (config == null) { - return; - } - - CsvReader configReader = CsvReader.parse(config); + public static String[] getHeaders(CsvReader configReader) throws IOException { + configReader.setHeaders(configReader.getRawRecord().split(",")); + return configReader.getHeaders(); + } - int STOP_GROUP_ID_INDEX = -1; - int STOP_ID_INDEX = -1; - int IS_PRIMARY_INDEX = -1; - int FEED_ID_INDEX = -1; + public static Map getHeaderIncedes(CsvReader configReader) { + HashMap map = new HashMap(); try { configReader.readRecord(); + String[] headers = getHeaders(configReader); - configReader.setHeaders(configReader.getRawRecord().split(",")); - String[] headers = configReader.getHeaders(); for (int i = 0; i < headers.length; i++) { String header = headers[i]; switch (header) { case "stop_group_id": - STOP_GROUP_ID_INDEX = i; + map.put(SharedStopsHeader.STOP_GROUP_ID_INDEX, i); break; case "feed_id": - FEED_ID_INDEX = i; + map.put(SharedStopsHeader.FEED_ID_INDEX, i); break; case "stop_id": - STOP_ID_INDEX = i; + map.put(SharedStopsHeader.STOP_ID_INDEX, i); break; case "is_primary": - IS_PRIMARY_INDEX = i; + map.put(SharedStopsHeader.IS_PRIMARY_INDEX, i); break; default: throw new RuntimeException("shared_stops.csv contained invalid headers!"); } } - if (STOP_GROUP_ID_INDEX == -1 || FEED_ID_INDEX == -1 || STOP_ID_INDEX == -1 || IS_PRIMARY_INDEX == -1) { + // TODO: some way to generate this from the enum? + if (!map.containsKey(SharedStopsHeader.STOP_GROUP_ID_INDEX) || + !map.containsKey(SharedStopsHeader.FEED_ID_INDEX) || + !map.containsKey(SharedStopsHeader.STOP_ID_INDEX) || + !map.containsKey(SharedStopsHeader.IS_PRIMARY_INDEX)) { throw new RuntimeException("shared_stops.csv is missing headers!"); } } catch (IOException e) { throw new RuntimeException("shared_stops.csv was invalid!", e); } + return map; + } + + @Override + public void validate() { + String config = project.sharedStopsConfig; + if (config == null) { + return; + } + + CsvReader configReader = CsvReader.parse(config); + + // Build list of stop Ids. List stopIds = new ArrayList<>(); List stops = new ArrayList<>(); @@ -102,15 +120,18 @@ public void validate() { stopIds.add(stop.stop_id); } + Map indeces = getHeaderIncedes(configReader); + // Initialize hashmaps to hold duplication info HashSet seenStopIds = new HashSet<>(); HashSet stopGroupsWithPrimaryStops = new HashSet<>(); try { while (configReader.readRecord()) { - String stopGroupId = configReader.get(STOP_GROUP_ID_INDEX); - String stopId = configReader.get(STOP_ID_INDEX); - String sharedStopFeedId = configReader.get(FEED_ID_INDEX); + // TODO: Avoid casting here? It must be possible with the enums + String stopGroupId = configReader.get((Integer) indeces.get(SharedStopsHeader.STOP_GROUP_ID_INDEX)); + String stopId = configReader.get((Integer) indeces.get(SharedStopsHeader.STOP_ID_INDEX)); + String sharedStopFeedId = configReader.get((Integer) indeces.get(SharedStopsHeader.FEED_ID_INDEX)); if (stopId.equals("stop_id")) { // Swallow header row @@ -133,7 +154,7 @@ public void validate() { } // Check for SS_02 (multiple primary stops per stop group) - if (configReader.get(IS_PRIMARY_INDEX).equals("1") || configReader.get(IS_PRIMARY_INDEX).equals("true")) { + if (configReader.get((Integer) indeces.get(SharedStopsHeader.IS_PRIMARY_INDEX)).equals("1") || configReader.get((Integer) indeces.get(SharedStopsHeader.IS_PRIMARY_INDEX)).equals("true")) { if (stopGroupsWithPrimaryStops.contains(stopGroupId)) { registerError(NewGTFSError.forFeed(NewGTFSErrorType.SHARED_STOP_GROUP_MULTIPLE_PRIMARY_STOPS, stopGroupId)); } else { diff --git a/src/test/java/com/conveyal/datatools/manager/jobs/validation/SharedStopsValidatorTest.java b/src/test/java/com/conveyal/datatools/manager/jobs/validation/SharedStopsValidatorTest.java new file mode 100644 index 000000000..9ef157393 --- /dev/null +++ b/src/test/java/com/conveyal/datatools/manager/jobs/validation/SharedStopsValidatorTest.java @@ -0,0 +1,51 @@ +package com.conveyal.datatools.manager.jobs.validation; + +import com.conveyal.datatools.UnitTest; +import com.csvreader.CsvReader; +import org.junit.jupiter.api.Test; + +import java.util.Map; + +import static com.zenika.snapshotmatcher.SnapshotMatcher.matchesSnapshot; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +public class SharedStopsValidatorTest extends UnitTest { + private void attemptToParseInvalidCSV(String csv, String expectedError) { + CsvReader configReader = CsvReader.parse(csv); + Exception exception = assertThrows(RuntimeException.class, () -> { + SharedStopsValidator.getHeaderIncedes(configReader); + }); + + String error = exception.getMessage(); + assertEquals(expectedError, error); + } + + private Map parseValidCSV(String csv) { + CsvReader configReader = CsvReader.parse(csv); + return SharedStopsValidator.getHeaderIncedes(configReader); + } + + @Test + void canHandleIncorrectCsv() { + String invalid = "shared_stops.csv contained invalid headers!"; + String missing = "shared_stops.csv is missing headers!"; + + attemptToParseInvalidCSV("this is a great string, but it's not a shared_stops csv!", invalid); + attemptToParseInvalidCSV("", invalid); + attemptToParseInvalidCSV("is_primary,stop_id", missing); + attemptToParseInvalidCSV("is_primary,feed_id,,stop_group_id", invalid); + attemptToParseInvalidCSV("is_primary,feed_id,stop_group_id", missing); + attemptToParseInvalidCSV("feed_id, is_primary, stop_group_id,stop_id", invalid); + } + + @Test + void canHandleVariousCorrectCSV() { + assertThat(parseValidCSV("is_primary,feed_id,stop_id,stop_group_id"), matchesSnapshot()); + assertThat(parseValidCSV("feed_id,stop_id,stop_group_id,is_primary"), matchesSnapshot()); + + // Only last is handled + assertThat(parseValidCSV("feed_id,is_primary,stop_group_id,stop_id,feed_id"), matchesSnapshot()); + } +} diff --git a/src/test/resources/snapshots/com/conveyal/datatools/manager/jobs/validation/SharedStopsValidatorTest/canHandleVariousCorrectCSV-0.json b/src/test/resources/snapshots/com/conveyal/datatools/manager/jobs/validation/SharedStopsValidatorTest/canHandleVariousCorrectCSV-0.json new file mode 100644 index 000000000..efafa6fbc --- /dev/null +++ b/src/test/resources/snapshots/com/conveyal/datatools/manager/jobs/validation/SharedStopsValidatorTest/canHandleVariousCorrectCSV-0.json @@ -0,0 +1,6 @@ +{ + "STOP_GROUP_ID_INDEX" : 3, + "STOP_ID_INDEX" : 2, + "IS_PRIMARY_INDEX" : 0, + "FEED_ID_INDEX" : 1 +} \ No newline at end of file diff --git a/src/test/resources/snapshots/com/conveyal/datatools/manager/jobs/validation/SharedStopsValidatorTest/canHandleVariousCorrectCSV-1.json b/src/test/resources/snapshots/com/conveyal/datatools/manager/jobs/validation/SharedStopsValidatorTest/canHandleVariousCorrectCSV-1.json new file mode 100644 index 000000000..0a5c3f7a5 --- /dev/null +++ b/src/test/resources/snapshots/com/conveyal/datatools/manager/jobs/validation/SharedStopsValidatorTest/canHandleVariousCorrectCSV-1.json @@ -0,0 +1,6 @@ +{ + "STOP_GROUP_ID_INDEX" : 2, + "STOP_ID_INDEX" : 1, + "IS_PRIMARY_INDEX" : 3, + "FEED_ID_INDEX" : 0 +} \ No newline at end of file diff --git a/src/test/resources/snapshots/com/conveyal/datatools/manager/jobs/validation/SharedStopsValidatorTest/canHandleVariousCorrectCSV-2.json b/src/test/resources/snapshots/com/conveyal/datatools/manager/jobs/validation/SharedStopsValidatorTest/canHandleVariousCorrectCSV-2.json new file mode 100644 index 000000000..351f7a0d6 --- /dev/null +++ b/src/test/resources/snapshots/com/conveyal/datatools/manager/jobs/validation/SharedStopsValidatorTest/canHandleVariousCorrectCSV-2.json @@ -0,0 +1,6 @@ +{ + "STOP_GROUP_ID_INDEX" : 2, + "STOP_ID_INDEX" : 3, + "IS_PRIMARY_INDEX" : 1, + "FEED_ID_INDEX" : 4 +} \ No newline at end of file From a2bdbff0f2a0127f532b6b1220227f8b49b8ba31 Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Fri, 12 Jan 2024 10:53:33 +0000 Subject: [PATCH 226/248] refactor(Updates to enum use and parameterized unit tests): --- .../jobs/validation/SharedStopsHeader.java | 37 ++++++++ .../jobs/validation/SharedStopsValidator.java | 91 ++++++++----------- .../validation/SharedStopsValidatorTest.java | 68 ++++++++------ 3 files changed, 116 insertions(+), 80 deletions(-) create mode 100644 src/main/java/com/conveyal/datatools/manager/jobs/validation/SharedStopsHeader.java diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/validation/SharedStopsHeader.java b/src/main/java/com/conveyal/datatools/manager/jobs/validation/SharedStopsHeader.java new file mode 100644 index 000000000..9d63ae7ce --- /dev/null +++ b/src/main/java/com/conveyal/datatools/manager/jobs/validation/SharedStopsHeader.java @@ -0,0 +1,37 @@ +package com.conveyal.datatools.manager.jobs.validation; + +import java.util.Map; + +public enum SharedStopsHeader { + + STOP_GROUP_ID_INDEX("stop_group_id"), + + STOP_ID_INDEX("stop_id"), + + IS_PRIMARY_INDEX("is_primary"), + + FEED_ID_INDEX("feed_id"); + + public final String headerName; + + SharedStopsHeader(String name) { + this.headerName = name; + } + + public String getHeaderName() { + return headerName; + } + + public static SharedStopsHeader fromValue(String headerName) { + for (SharedStopsHeader sharedStopsHeader : SharedStopsHeader.values()) { + if (sharedStopsHeader.getHeaderName().equalsIgnoreCase(headerName)) { + return sharedStopsHeader; + } + } + throw new UnsupportedOperationException(String.format("Unknown shared stops header: %s", headerName)); + } + + public static boolean hasRequiredHeaders(Map headerIndices) { + return headerIndices.size() == SharedStopsHeader.values().length; + } +} diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/validation/SharedStopsValidator.java b/src/main/java/com/conveyal/datatools/manager/jobs/validation/SharedStopsValidator.java index dfc795879..a99e686f2 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/validation/SharedStopsValidator.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/validation/SharedStopsValidator.java @@ -13,17 +13,13 @@ import java.io.IOException; import java.util.ArrayList; -import java.util.HashMap; +import java.util.EnumMap; import java.util.HashSet; import java.util.List; import java.util.Map; -enum SharedStopsHeader { - STOP_GROUP_ID_INDEX, - STOP_ID_INDEX, - IS_PRIMARY_INDEX, - FEED_ID_INDEX -} +import static com.conveyal.datatools.manager.jobs.validation.SharedStopsHeader.STOP_GROUP_ID_INDEX; +import static com.conveyal.datatools.manager.jobs.validation.SharedStopsHeader.STOP_ID_INDEX; public class SharedStopsValidator extends FeedValidator { private static final Logger LOG = LoggerFactory.getLogger(SharedStopsValidator.class); @@ -48,6 +44,7 @@ public SharedStopsValidator buildSharedStopsValidator(Feed feed, SQLErrorStorage } return new SharedStopsValidator(feed, errorStorage, this.project, this.feedId); } + public SharedStopsValidator(Feed feed, SQLErrorStorage errorStorage, Project project, String feedId) { super(feed, errorStorage); this.feed = feed; @@ -60,8 +57,8 @@ public static String[] getHeaders(CsvReader configReader) throws IOException { return configReader.getHeaders(); } - public static Map getHeaderIncedes(CsvReader configReader) { - HashMap map = new HashMap(); + public static Map getHeaderIndices(CsvReader configReader) { + Map headerIndices = new EnumMap<>(SharedStopsHeader.class); try { configReader.readRecord(); @@ -69,36 +66,29 @@ public static Map getHeaderIncedes(CsvReader configR for (int i = 0; i < headers.length; i++) { String header = headers[i]; - switch (header) { - case "stop_group_id": - map.put(SharedStopsHeader.STOP_GROUP_ID_INDEX, i); + switch (SharedStopsHeader.fromValue(header)) { + case STOP_GROUP_ID_INDEX: + headerIndices.put(STOP_GROUP_ID_INDEX, i); break; - case "feed_id": - map.put(SharedStopsHeader.FEED_ID_INDEX, i); + case FEED_ID_INDEX: + headerIndices.put(SharedStopsHeader.FEED_ID_INDEX, i); break; - case "stop_id": - map.put(SharedStopsHeader.STOP_ID_INDEX, i); + case STOP_ID_INDEX: + headerIndices.put(STOP_ID_INDEX, i); break; - case "is_primary": - map.put(SharedStopsHeader.IS_PRIMARY_INDEX, i); + case IS_PRIMARY_INDEX: + headerIndices.put(SharedStopsHeader.IS_PRIMARY_INDEX, i); break; - default: - throw new RuntimeException("shared_stops.csv contained invalid headers!"); } } - // TODO: some way to generate this from the enum? - if (!map.containsKey(SharedStopsHeader.STOP_GROUP_ID_INDEX) || - !map.containsKey(SharedStopsHeader.FEED_ID_INDEX) || - !map.containsKey(SharedStopsHeader.STOP_ID_INDEX) || - !map.containsKey(SharedStopsHeader.IS_PRIMARY_INDEX)) { + if (!SharedStopsHeader.hasRequiredHeaders(headerIndices)) { throw new RuntimeException("shared_stops.csv is missing headers!"); } } catch (IOException e) { throw new RuntimeException("shared_stops.csv was invalid!", e); } - - return map; + return headerIndices; } @Override @@ -110,7 +100,6 @@ public void validate() { CsvReader configReader = CsvReader.parse(config); - // Build list of stop Ids. List stopIds = new ArrayList<>(); List stops = new ArrayList<>(); @@ -120,41 +109,42 @@ public void validate() { stopIds.add(stop.stop_id); } - Map indeces = getHeaderIncedes(configReader); - - // Initialize hashmaps to hold duplication info + // Initialize hashmaps to hold duplication info. HashSet seenStopIds = new HashSet<>(); HashSet stopGroupsWithPrimaryStops = new HashSet<>(); try { + Map headerIndices = getHeaderIndices(configReader); while (configReader.readRecord()) { - // TODO: Avoid casting here? It must be possible with the enums - String stopGroupId = configReader.get((Integer) indeces.get(SharedStopsHeader.STOP_GROUP_ID_INDEX)); - String stopId = configReader.get((Integer) indeces.get(SharedStopsHeader.STOP_ID_INDEX)); - String sharedStopFeedId = configReader.get((Integer) indeces.get(SharedStopsHeader.FEED_ID_INDEX)); + String stopGroupId = configReader.get(headerIndices.get(STOP_GROUP_ID_INDEX)); + String stopId = configReader.get(headerIndices.get(STOP_ID_INDEX)); + String sharedStopFeedId = configReader.get(headerIndices.get(SharedStopsHeader.FEED_ID_INDEX)); - if (stopId.equals("stop_id")) { - // Swallow header row + if (stopId.equals(STOP_ID_INDEX.headerName)) { + // Swallow header row. continue; } - // Check for SS_01 (stop id appearing in multiple stop groups) - // Make sure this error is only returned if we are inside the feed that is being checked + // Check for SS_01 (stop id appearing in multiple stop groups). + // Make sure this error is only returned if we are inside the feed that is being checked. if (seenStopIds.contains(stopId)) { if (feedId.equals(sharedStopFeedId)) { registerError(stops - .stream() - .filter(stop -> stop.stop_id.equals(stopId)) - .findFirst() - .orElse(new Stop()), NewGTFSErrorType.MULTIPLE_SHARED_STOPS_GROUPS + .stream() + .filter(stop -> stop.stop_id.equals(stopId)) + .findFirst() + .orElse(new Stop()), NewGTFSErrorType.MULTIPLE_SHARED_STOPS_GROUPS ); } } else { seenStopIds.add(stopId); } - // Check for SS_02 (multiple primary stops per stop group) - if (configReader.get((Integer) indeces.get(SharedStopsHeader.IS_PRIMARY_INDEX)).equals("1") || configReader.get((Integer) indeces.get(SharedStopsHeader.IS_PRIMARY_INDEX)).equals("true")) { + // Check for SS_02 (multiple primary stops per stop group). + if ( + configReader.get(headerIndices.get(SharedStopsHeader.IS_PRIMARY_INDEX)).equals("1") || + configReader.get(headerIndices.get(SharedStopsHeader.IS_PRIMARY_INDEX)).equals("true") + ) { if (stopGroupsWithPrimaryStops.contains(stopGroupId)) { registerError(NewGTFSError.forFeed(NewGTFSErrorType.SHARED_STOP_GROUP_MULTIPLE_PRIMARY_STOPS, stopGroupId)); } else { @@ -162,17 +152,16 @@ public void validate() { } } - // Check for SS_03 (stop_id referenced doesn't exist) - // Make sure this error is only returned if we are inside the feed that is being checked + // Check for SS_03 (stop_id referenced doesn't exist). + // Make sure this error is only returned if we are inside the feed that is being checked. if (feedId.equals(sharedStopFeedId) && !stopIds.contains(stopId)) { registerError(NewGTFSError.forFeed(NewGTFSErrorType.SHARED_STOP_GROUP_ENTITY_DOES_NOT_EXIST, stopId)); } } - } catch (IOException e) { LOG.error(e.toString()); } - finally { + } catch (IOException e) { + LOG.error("Unable to validate shared stops.", e); + } finally { configReader.close(); } - - } } diff --git a/src/test/java/com/conveyal/datatools/manager/jobs/validation/SharedStopsValidatorTest.java b/src/test/java/com/conveyal/datatools/manager/jobs/validation/SharedStopsValidatorTest.java index 9ef157393..b8a944f71 100644 --- a/src/test/java/com/conveyal/datatools/manager/jobs/validation/SharedStopsValidatorTest.java +++ b/src/test/java/com/conveyal/datatools/manager/jobs/validation/SharedStopsValidatorTest.java @@ -2,50 +2,60 @@ import com.conveyal.datatools.UnitTest; import com.csvreader.CsvReader; -import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; -import java.util.Map; +import java.util.stream.Stream; import static com.zenika.snapshotmatcher.SnapshotMatcher.matchesSnapshot; import static org.hamcrest.MatcherAssert.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; -public class SharedStopsValidatorTest extends UnitTest { - private void attemptToParseInvalidCSV(String csv, String expectedError) { - CsvReader configReader = CsvReader.parse(csv); - Exception exception = assertThrows(RuntimeException.class, () -> { - SharedStopsValidator.getHeaderIncedes(configReader); - }); +class SharedStopsValidatorTest extends UnitTest { - String error = exception.getMessage(); - assertEquals(expectedError, error); + @ParameterizedTest + @MethodSource("createCSVHeaders") + void canHandleVariousCorrectCSV(String headers) { + assertThat(SharedStopsValidator.getHeaderIndices(CsvReader.parse(headers)), matchesSnapshot()); } - private Map parseValidCSV(String csv) { - CsvReader configReader = CsvReader.parse(csv); - return SharedStopsValidator.getHeaderIncedes(configReader); + private static Stream createCSVHeaders() { + return Stream.of( + "is_primary,feed_id,stop_id,stop_group_id", + "feed_id,stop_id,stop_group_id,is_primary", + "feed_id,is_primary,stop_group_id,stop_id,feed_id" + ); } - @Test - void canHandleIncorrectCsv() { - String invalid = "shared_stops.csv contained invalid headers!"; - String missing = "shared_stops.csv is missing headers!"; + @ParameterizedTest + @MethodSource("createInvalidCSVArguments") + void attemptToParseInvalidCSV(InvalidCSVArgument invalidCSVArgument) { + CsvReader configReader = CsvReader.parse(invalidCSVArgument.csv); + Exception exception = assertThrows(RuntimeException.class, () -> SharedStopsValidator.getHeaderIndices(configReader)); + assertEquals(invalidCSVArgument.expectedError, exception.getMessage()); + } - attemptToParseInvalidCSV("this is a great string, but it's not a shared_stops csv!", invalid); - attemptToParseInvalidCSV("", invalid); - attemptToParseInvalidCSV("is_primary,stop_id", missing); - attemptToParseInvalidCSV("is_primary,feed_id,,stop_group_id", invalid); - attemptToParseInvalidCSV("is_primary,feed_id,stop_group_id", missing); - attemptToParseInvalidCSV("feed_id, is_primary, stop_group_id,stop_id", invalid); + private static Stream createInvalidCSVArguments() { + String invalid = "Unknown shared stops header: "; + String missing = "shared_stops.csv is missing headers!"; + return Stream.of( + new InvalidCSVArgument("this is a great string, but it's not a shared_stops csv!", invalid + "this is a great string"), + new InvalidCSVArgument("", invalid), + new InvalidCSVArgument("is_primary,stop_id", missing), + new InvalidCSVArgument("is_primary,feed_id,,stop_group_id", invalid), + new InvalidCSVArgument("is_primary,feed_id,stop_group_id", missing), + new InvalidCSVArgument("feed_id, is_primary, stop_group_id,stop_id", invalid + " is_primary") + ); } - @Test - void canHandleVariousCorrectCSV() { - assertThat(parseValidCSV("is_primary,feed_id,stop_id,stop_group_id"), matchesSnapshot()); - assertThat(parseValidCSV("feed_id,stop_id,stop_group_id,is_primary"), matchesSnapshot()); + private static class InvalidCSVArgument { + public String csv; + public String expectedError; - // Only last is handled - assertThat(parseValidCSV("feed_id,is_primary,stop_group_id,stop_id,feed_id"), matchesSnapshot()); + public InvalidCSVArgument(String csv, String expectedError) { + this.csv = csv; + this.expectedError = expectedError; + } } } From 1c817d8ff1b0e974ed16d662d33977898f421206 Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Tue, 16 Jan 2024 12:14:44 +0000 Subject: [PATCH 227/248] refactor(Updated to include gtfs-lib with pattern name update): --- .github/workflows/maven.yml | 10 +++++----- pom.xml | 2 +- .../manager/jobs/validation/SharedStopsValidator.java | 2 +- .../conveyal/datatools/manager/models/FeedVersion.java | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index 7fe151840..f596c83be 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -32,11 +32,11 @@ jobs: with: java-version: 19 distribution: 'temurin' - # Install node 14 for running e2e tests (and for maven-semantic-release). - - name: Use Node.js 18.x + # Install Node 20.11.0 (LTS) for running e2e tests (and for maven-semantic-release). + - name: Use Node.js 20.x uses: actions/setup-node@v1 with: - node-version: 18.x + node-version: 20.11.0 - name: Start MongoDB uses: supercharge/mongodb-github-action@1.3.0 with: @@ -98,10 +98,10 @@ jobs: # Run maven-semantic-release to potentially create a new release of datatools-server. The flag --skip-maven-deploy is # used to avoid deploying to maven central. So essentially, this just creates a release with a changelog on github. - - name: Use Node.js 18.x + - name: Use Node.js 20.x uses: actions/setup-node@v1 with: - node-version: 18.x + node-version: 20.11.0 - name: Run maven-semantic-release if: env.SAVE_JAR_TO_S3 == 'true' env: diff --git a/pom.xml b/pom.xml index cc7e70c2d..9f0da896a 100644 --- a/pom.xml +++ b/pom.xml @@ -272,7 +272,7 @@ com.github.ibi-group gtfs-lib - 3fa0da9 + 12a990f2158d23681788419f832678895d345046 diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/validation/SharedStopsValidator.java b/src/main/java/com/conveyal/datatools/manager/jobs/validation/SharedStopsValidator.java index d608fe249..0377051cb 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/validation/SharedStopsValidator.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/validation/SharedStopsValidator.java @@ -105,7 +105,7 @@ public void validate() { // Check for SS_02 (multiple primary stops per stop group) if (stopGroupsWithPrimaryStops.contains(stopGroupId)) { - registerError(NewGTFSError.forFeed(NewGTFSErrorType.SHARED_STOP_GROUP_MUTLIPLE_PRIMARY_STOPS, stopGroupId)); + registerError(NewGTFSError.forFeed(NewGTFSErrorType.SHARED_STOP_GROUP_MULTIPLE_PRIMARY_STOPS, stopGroupId)); } else if (configReader.get(IS_PRIMARY_INDEX).equals("true")) { stopGroupsWithPrimaryStops.add(stopGroupId); } diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java b/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java index d05af58dd..bb87c6187 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java @@ -518,7 +518,7 @@ public boolean hasBlockingIssuesForPublishing() { NewGTFSErrorType.TABLE_IN_SUBDIRECTORY, NewGTFSErrorType.WRONG_NUMBER_OF_FIELDS, NewGTFSErrorType.MULTIPLE_SHARED_STOPS_GROUPS, - NewGTFSErrorType.SHARED_STOP_GROUP_MUTLIPLE_PRIMARY_STOPS + NewGTFSErrorType.SHARED_STOP_GROUP_MULTIPLE_PRIMARY_STOPS )); } From ea9d595f00b9af3ff92307cd7cc8404631de416e Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Tue, 16 Jan 2024 14:24:34 +0000 Subject: [PATCH 228/248] refactor(pom.xml): Bumped GTFS-Lib version --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 9f0da896a..9f9c6ca2f 100644 --- a/pom.xml +++ b/pom.xml @@ -272,7 +272,7 @@ com.github.ibi-group gtfs-lib - 12a990f2158d23681788419f832678895d345046 + f9500a0d9012e22ede8386d64b918ad2d9f4f8d5 From d14b775ce7efa11ead579f36a3604d4e274b58f1 Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Tue, 16 Jan 2024 14:41:22 +0000 Subject: [PATCH 229/248] refactor(maven.yml): Corrected indentation --- .github/workflows/maven.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index f596c83be..0193ac499 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -36,7 +36,7 @@ jobs: - name: Use Node.js 20.x uses: actions/setup-node@v1 with: - node-version: 20.11.0 + node-version: 20.11.0 - name: Start MongoDB uses: supercharge/mongodb-github-action@1.3.0 with: From 509cec48d558adcca7a97d8a963f7b051c42d7e1 Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Tue, 16 Jan 2024 10:12:54 -0500 Subject: [PATCH 230/248] update ci node --- .github/workflows/maven.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index 7fe151840..2616dcec9 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -33,10 +33,10 @@ jobs: java-version: 19 distribution: 'temurin' # Install node 14 for running e2e tests (and for maven-semantic-release). - - name: Use Node.js 18.x + - name: Use Node.js 20.x uses: actions/setup-node@v1 with: - node-version: 18.x + node-version: 20.x - name: Start MongoDB uses: supercharge/mongodb-github-action@1.3.0 with: From 0f768ed52e16c3b8227e5a9117fefc992e837cc0 Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Tue, 16 Jan 2024 11:08:42 -0500 Subject: [PATCH 231/248] correctly update node version --- .github/workflows/maven.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index 2616dcec9..bb9a73ab1 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -98,10 +98,10 @@ jobs: # Run maven-semantic-release to potentially create a new release of datatools-server. The flag --skip-maven-deploy is # used to avoid deploying to maven central. So essentially, this just creates a release with a changelog on github. - - name: Use Node.js 18.x + - name: Use Node.js 20.x uses: actions/setup-node@v1 with: - node-version: 18.x + node-version: 20.x - name: Run maven-semantic-release if: env.SAVE_JAR_TO_S3 == 'true' env: From 99f73d795e1c877513ca0e7c6fb8399e11bd600e Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Wed, 31 Jan 2024 14:14:45 -0500 Subject: [PATCH 232/248] shared stops validator: only consider stops from feed being checked --- .../jobs/validation/SharedStopsValidator.java | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/validation/SharedStopsValidator.java b/src/main/java/com/conveyal/datatools/manager/jobs/validation/SharedStopsValidator.java index a99e686f2..6da6bcefa 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/validation/SharedStopsValidator.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/validation/SharedStopsValidator.java @@ -127,17 +127,19 @@ public void validate() { // Check for SS_01 (stop id appearing in multiple stop groups). // Make sure this error is only returned if we are inside the feed that is being checked. - if (seenStopIds.contains(stopId)) { - if (feedId.equals(sharedStopFeedId)) { + Stop syntheticStop = new Stop(); + syntheticStop.stop_id = stopId; + if (feedId.equals(sharedStopFeedId)) { + if (seenStopIds.contains(stopId)) { registerError(stops .stream() .filter(stop -> stop.stop_id.equals(stopId)) .findFirst() - .orElse(new Stop()), NewGTFSErrorType.MULTIPLE_SHARED_STOPS_GROUPS + .orElse(syntheticStop), NewGTFSErrorType.MULTIPLE_SHARED_STOPS_GROUPS ); + }else { + seenStopIds.add(stopId); } - } else { - seenStopIds.add(stopId); } // Check for SS_02 (multiple primary stops per stop group). From eed8d91b70bebf501c401a3fddb440a76b1cfa83 Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Wed, 31 Jan 2024 14:16:08 -0500 Subject: [PATCH 233/248] lint --- .../datatools/manager/jobs/validation/SharedStopsValidator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/validation/SharedStopsValidator.java b/src/main/java/com/conveyal/datatools/manager/jobs/validation/SharedStopsValidator.java index 6da6bcefa..ed8d1a58e 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/validation/SharedStopsValidator.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/validation/SharedStopsValidator.java @@ -137,7 +137,7 @@ public void validate() { .findFirst() .orElse(syntheticStop), NewGTFSErrorType.MULTIPLE_SHARED_STOPS_GROUPS ); - }else { + } else { seenStopIds.add(stopId); } } From 21506608ed2944961274e017fd0e4b8304ffce67 Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Wed, 13 Mar 2024 16:59:31 -0400 Subject: [PATCH 234/248] support new `prefer_s3_links` config property --- configurations/default/server.yml.tmp | 2 ++ .../datatools/manager/jobs/PublishProjectFeedsJob.java | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/configurations/default/server.yml.tmp b/configurations/default/server.yml.tmp index 2a94ab43f..74b0b2338 100644 --- a/configurations/default/server.yml.tmp +++ b/configurations/default/server.yml.tmp @@ -18,6 +18,8 @@ application: modules: enterprise: enabled: false + # Setting this to true will upload all feeds to S3 instead of linking to their URL + prefer_s3_links: false editor: enabled: true deployment: diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/PublishProjectFeedsJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/PublishProjectFeedsJob.java index fc881e014..214790baa 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/PublishProjectFeedsJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/PublishProjectFeedsJob.java @@ -17,6 +17,7 @@ import java.text.SimpleDateFormat; import static com.conveyal.datatools.manager.DataManager.getConfigPropertyAsText; +import static com.conveyal.datatools.manager.DataManager.hasConfigProperty; /** * Publish the latest GTFS files for all public feeds in a project. @@ -67,7 +68,7 @@ public void jobLogic () { .forEach(fs -> { // generate list item for feed source String url; - if (fs.url != null) { + if (fs.url != null && !hasConfigProperty("modules.enterprise.prefer_s3_links")) { url = fs.url.toString(); } else { From 1b4043fe4decee8652beaf0dc1b11d386ce9d0bf Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Tue, 26 Mar 2024 16:36:17 -0400 Subject: [PATCH 235/248] add append transformation --- .../transform/AppendToFileTransformation.java | 76 +++++++++++++++++++ .../models/transform/FeedTransformation.java | 3 +- .../jobs/ArbitraryTransformJobTest.java | 36 +++++++++ 3 files changed, 114 insertions(+), 1 deletion(-) create mode 100644 src/main/java/com/conveyal/datatools/manager/models/transform/AppendToFileTransformation.java diff --git a/src/main/java/com/conveyal/datatools/manager/models/transform/AppendToFileTransformation.java b/src/main/java/com/conveyal/datatools/manager/models/transform/AppendToFileTransformation.java new file mode 100644 index 000000000..535ec6eed --- /dev/null +++ b/src/main/java/com/conveyal/datatools/manager/models/transform/AppendToFileTransformation.java @@ -0,0 +1,76 @@ + +package com.conveyal.datatools.manager.models.transform; + +import com.conveyal.datatools.common.status.MonitorableJob; +import com.conveyal.datatools.manager.models.TableTransformResult; +import com.conveyal.datatools.manager.models.TransformType; + +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.FileOutputStream; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.FileSystem; +import java.nio.file.FileSystems; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; + +public class AppendToFileTransformation extends ZipTransformation { + + public static AppendToFileTransformation create(String csvData, String table) { + AppendToFileTransformation transformation = new AppendToFileTransformation(); + transformation.csvData = csvData; + transformation.table = table; + return transformation; + } + + @Override + public void validateParameters(MonitorableJob.Status status) { + if (csvData == null) { + status.fail("CSV data must not be null"); + } + } + + @Override + public void transform(FeedTransformZipTarget zipTarget, MonitorableJob.Status status) { + String tableName = table + ".txt"; + Path targetZipPath = Paths.get(zipTarget.gtfsFile.getAbsolutePath()); + try ( + FileSystem targetZipFs = FileSystems.newFileSystem(targetZipPath, (ClassLoader) null); + InputStream inputStream = new ByteArrayInputStream(csvData.getBytes(StandardCharsets.UTF_8)); + ) { + TransformType type = TransformType.TABLE_MODIFIED; + + Path targetTxtFilePath = getTablePathInZip(tableName, targetZipFs); + + final File tempFile = File.createTempFile(tableName + "-temp", ".txt"); + Files.copy(targetTxtFilePath, tempFile.toPath(), StandardCopyOption.REPLACE_EXISTING); + + // Append CSV data into the target file in the temporary copy of file + try (OutputStream os = new FileOutputStream(tempFile, true)) { + os.write(inputStream.readAllBytes()); + } catch (Exception e) { + status.fail("Failed to write to target file", e); + } + + // Copy modified file into zip + Files.copy(tempFile.toPath(), targetTxtFilePath, StandardCopyOption.REPLACE_EXISTING); + + final int NEW_LINE_CHARACTER_CODE = 10; + int lineCount = (int) csvData.chars().filter(c -> c == NEW_LINE_CHARACTER_CODE).count(); + zipTarget.feedTransformResult.tableTransformResults.add(new TableTransformResult( + tableName, + type, + 0, + 0, + lineCount + 1, + 0 + )); + } catch (Exception e) { + status.fail("Unknown error encountered while transforming zip file", e); + } + } +} diff --git a/src/main/java/com/conveyal/datatools/manager/models/transform/FeedTransformation.java b/src/main/java/com/conveyal/datatools/manager/models/transform/FeedTransformation.java index e63001b37..bed22f506 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/transform/FeedTransformation.java +++ b/src/main/java/com/conveyal/datatools/manager/models/transform/FeedTransformation.java @@ -32,7 +32,8 @@ @JsonSubTypes.Type(value = ReplaceFileFromVersionTransformation.class, name = "ReplaceFileFromVersionTransformation"), @JsonSubTypes.Type(value = ReplaceFileFromStringTransformation.class, name = "ReplaceFileFromStringTransformation"), @JsonSubTypes.Type(value = PreserveCustomFieldsTransformation.class, name = "PreserveCustomFieldsTransformation"), - @JsonSubTypes.Type(value = AddCustomFileFromStringTransformation.class, name = "AddCustomFileTransformation") + @JsonSubTypes.Type(value = AddCustomFileFromStringTransformation.class, name = "AddCustomFileTransformation"), + @JsonSubTypes.Type(value = AppendToFileTransformation.class, name = "AppendToFileTransformation") }) public abstract class FeedTransformation implements Serializable { private static final long serialVersionUID = 1L; diff --git a/src/test/java/com/conveyal/datatools/manager/jobs/ArbitraryTransformJobTest.java b/src/test/java/com/conveyal/datatools/manager/jobs/ArbitraryTransformJobTest.java index 9303bf821..8541aa9ef 100644 --- a/src/test/java/com/conveyal/datatools/manager/jobs/ArbitraryTransformJobTest.java +++ b/src/test/java/com/conveyal/datatools/manager/jobs/ArbitraryTransformJobTest.java @@ -11,6 +11,7 @@ import com.conveyal.datatools.manager.models.Snapshot; import com.conveyal.datatools.manager.models.TableTransformResult; import com.conveyal.datatools.manager.models.transform.AddCustomFileFromStringTransformation; +import com.conveyal.datatools.manager.models.transform.AppendToFileTransformation; import com.conveyal.datatools.manager.models.transform.DeleteRecordsTransformation; import com.conveyal.datatools.manager.models.transform.FeedTransformRules; import com.conveyal.datatools.manager.models.transform.FeedTransformation; @@ -192,6 +193,37 @@ void replaceGtfsPlusFileFailsIfSourceIsMissing() throws IOException { assertThat(targetVersion.validationResult, Matchers.nullValue()); } + @Test + void canAppendToStops() throws SQLException, IOException { + sourceVersion = createFeedVersion( + feedSource, + zipFolderFiles("fake-agency-with-only-calendar") + ); + FeedTransformation transformation = AppendToFileTransformation.create(generateStopRow(), "stops"); + FeedTransformRules transformRules = new FeedTransformRules(transformation); + feedSource.transformRules.add(transformRules); + Persistence.feedSources.replace(feedSource.id, feedSource); + // Create new target version (note: the folder has no stop_attributes.txt file) + targetVersion = createFeedVersion( + feedSource, + zipFolderFiles("fake-agency-with-only-calendar-dates") + ); + LOG.info("Checking assertions."); + assertEquals( + 6, // Magic number should match row count in string produced by generateFeedInfo + targetVersion.feedLoadResult.stops.rowCount, + "stops.txt row count should equal input csv data # of rows + 1 extra row" + ); + // Check for presence of new stop id in database (one record). + assertThatSqlCountQueryYieldsExpectedCount( + String.format( + "SELECT count(*) FROM %s.stops WHERE stop_id = '%s'", + targetVersion.namespace, + "new" + ), + 1 + ); + } @Test void canReplaceFeedInfo() throws SQLException, IOException { // Generate random UUID for feedId, which gets placed into the csv data. @@ -282,6 +314,10 @@ private static String generateStopsWithCustomFields() { + "\n1234567,customValue3,customValue4"; } + private static String generateStopRow() { + return "\nnew,new,appended stop,,37.06668,-122.07781,,,0,123,,"; + } + private static String generateCustomCsvData() { return "custom_column1,custom_column2,custom_column3" + "\ncustomValue1,customValue2,customValue3" From ac604859233e9c41257faf66dc825f753d939321 Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Tue, 26 Mar 2024 16:39:33 -0400 Subject: [PATCH 236/248] correct comment --- .../datatools/manager/jobs/ArbitraryTransformJobTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/com/conveyal/datatools/manager/jobs/ArbitraryTransformJobTest.java b/src/test/java/com/conveyal/datatools/manager/jobs/ArbitraryTransformJobTest.java index 8541aa9ef..01d57be88 100644 --- a/src/test/java/com/conveyal/datatools/manager/jobs/ArbitraryTransformJobTest.java +++ b/src/test/java/com/conveyal/datatools/manager/jobs/ArbitraryTransformJobTest.java @@ -210,7 +210,7 @@ void canAppendToStops() throws SQLException, IOException { ); LOG.info("Checking assertions."); assertEquals( - 6, // Magic number should match row count in string produced by generateFeedInfo + 6, // Magic number should match row count of stops.txt with one extra targetVersion.feedLoadResult.stops.rowCount, "stops.txt row count should equal input csv data # of rows + 1 extra row" ); From 8082faceae934234d3be91b1adb7eec868cc80a8 Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Wed, 27 Mar 2024 16:07:11 -0400 Subject: [PATCH 237/248] refactor: address pr feedback --- .../transform/AppendToFileTransformation.java | 9 +++++++-- .../manager/jobs/ArbitraryTransformJobTest.java | 16 +++++----------- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/models/transform/AppendToFileTransformation.java b/src/main/java/com/conveyal/datatools/manager/models/transform/AppendToFileTransformation.java index 535ec6eed..d290afd66 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/transform/AppendToFileTransformation.java +++ b/src/main/java/com/conveyal/datatools/manager/models/transform/AppendToFileTransformation.java @@ -38,9 +38,11 @@ public void validateParameters(MonitorableJob.Status status) { public void transform(FeedTransformZipTarget zipTarget, MonitorableJob.Status status) { String tableName = table + ".txt"; Path targetZipPath = Paths.get(zipTarget.gtfsFile.getAbsolutePath()); + try ( - FileSystem targetZipFs = FileSystems.newFileSystem(targetZipPath, (ClassLoader) null); - InputStream inputStream = new ByteArrayInputStream(csvData.getBytes(StandardCharsets.UTF_8)); + FileSystem targetZipFs = FileSystems.newFileSystem(targetZipPath, (ClassLoader) null); + InputStream newLineStream = new ByteArrayInputStream("\n".getBytes(StandardCharsets.UTF_8)); + InputStream inputStream = new ByteArrayInputStream(csvData.getBytes(StandardCharsets.UTF_8)); ) { TransformType type = TransformType.TABLE_MODIFIED; @@ -51,6 +53,9 @@ public void transform(FeedTransformZipTarget zipTarget, MonitorableJob.Status st // Append CSV data into the target file in the temporary copy of file try (OutputStream os = new FileOutputStream(tempFile, true)) { + // Append a newline in case our data doesn't include one + // Having an extra newline is not a problem! + os.write(newLineStream.readAllBytes()); os.write(inputStream.readAllBytes()); } catch (Exception e) { status.fail("Failed to write to target file", e); diff --git a/src/test/java/com/conveyal/datatools/manager/jobs/ArbitraryTransformJobTest.java b/src/test/java/com/conveyal/datatools/manager/jobs/ArbitraryTransformJobTest.java index 01d57be88..eb25b619c 100644 --- a/src/test/java/com/conveyal/datatools/manager/jobs/ArbitraryTransformJobTest.java +++ b/src/test/java/com/conveyal/datatools/manager/jobs/ArbitraryTransformJobTest.java @@ -4,7 +4,6 @@ import com.conveyal.datatools.UnitTest; import com.conveyal.datatools.manager.auth.Auth0Connection; import com.conveyal.datatools.manager.auth.Auth0UserProfile; -import com.conveyal.datatools.manager.models.FeedRetrievalMethod; import com.conveyal.datatools.manager.models.FeedSource; import com.conveyal.datatools.manager.models.FeedVersion; import com.conveyal.datatools.manager.models.Project; @@ -27,17 +26,11 @@ import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.supercsv.io.CsvMapReader; -import org.supercsv.prefs.CsvPreference; -import java.io.File; -import java.io.InputStream; import java.io.IOException; -import java.io.InputStreamReader; import java.sql.SQLException; import java.util.ArrayList; import java.util.List; -import java.util.Map; import java.util.UUID; import java.util.zip.ZipEntry; import java.util.zip.ZipFile; @@ -47,7 +40,6 @@ import static com.conveyal.datatools.TestUtils.createFeedVersion; import static com.conveyal.datatools.TestUtils.zipFolderFiles; import static com.conveyal.datatools.manager.models.FeedRetrievalMethod.MANUALLY_UPLOADED; -import static com.conveyal.datatools.manager.models.FeedRetrievalMethod.VERSION_CLONE; import static org.hamcrest.MatcherAssert.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; @@ -210,9 +202,9 @@ void canAppendToStops() throws SQLException, IOException { ); LOG.info("Checking assertions."); assertEquals( - 6, // Magic number should match row count of stops.txt with one extra + 5 + 3, // Magic number should match row count of stops.txt with three extra targetVersion.feedLoadResult.stops.rowCount, - "stops.txt row count should equal input csv data # of rows + 1 extra row" + "stops.txt row count should equal input csv data # of rows + 3 extra rows" ); // Check for presence of new stop id in database (one record). assertThatSqlCountQueryYieldsExpectedCount( @@ -315,7 +307,9 @@ private static String generateStopsWithCustomFields() { } private static String generateStopRow() { - return "\nnew,new,appended stop,,37.06668,-122.07781,,,0,123,,"; + return "new3,new3,appended stop,,37,-122,,,0,123,," + + "\nnew2,new2,appended stop,,37,-122,,,0,123,," + + "\nnew,new,appended stop,,37.06668,-122.07781,,,0,123,,"; } private static String generateCustomCsvData() { From 3582d2d2a8daf415c5e88414caa275135a54e7fd Mon Sep 17 00:00:00 2001 From: Daniel Heppner Date: Mon, 13 May 2024 14:42:16 -0700 Subject: [PATCH 238/248] deps: update AWS sdk to latest version The AWS SDK is used to validate the instance type, updating it will allow us to use newer EC2 instances --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index c2206d7ba..9c412d876 100644 --- a/pom.xml +++ b/pom.xml @@ -44,7 +44,7 @@ 20.1 - 1.11.625 + 1.12.720 From 63c945a27ec9a36452cfc926ebd114f4fd2e44eb Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Tue, 14 May 2024 11:03:19 -0400 Subject: [PATCH 239/248] update tests to cover newline-based edge cases --- .../jobs/ArbitraryTransformJobTest.java | 75 +++++++++++++++++++ .../agency.txt | 2 + .../calendar.txt | 3 + .../feed_info.txt | 2 + .../routes.txt | 3 + .../stop_attributes.txt | 3 + .../stop_times.txt | 5 ++ .../stops.txt | 7 ++ .../trips.txt | 3 + 9 files changed, 103 insertions(+) create mode 100755 src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-only-calendar-and-trailing-newlines/agency.txt create mode 100755 src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-only-calendar-and-trailing-newlines/calendar.txt create mode 100644 src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-only-calendar-and-trailing-newlines/feed_info.txt create mode 100755 src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-only-calendar-and-trailing-newlines/routes.txt create mode 100644 src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-only-calendar-and-trailing-newlines/stop_attributes.txt create mode 100755 src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-only-calendar-and-trailing-newlines/stop_times.txt create mode 100755 src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-only-calendar-and-trailing-newlines/stops.txt create mode 100755 src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-only-calendar-and-trailing-newlines/trips.txt diff --git a/src/test/java/com/conveyal/datatools/manager/jobs/ArbitraryTransformJobTest.java b/src/test/java/com/conveyal/datatools/manager/jobs/ArbitraryTransformJobTest.java index eb25b619c..40fa2cdba 100644 --- a/src/test/java/com/conveyal/datatools/manager/jobs/ArbitraryTransformJobTest.java +++ b/src/test/java/com/conveyal/datatools/manager/jobs/ArbitraryTransformJobTest.java @@ -216,6 +216,71 @@ void canAppendToStops() throws SQLException, IOException { 1 ); } + + + @Test + void canAppendToStopsWithLeadingNewlineInData() throws SQLException, IOException { + sourceVersion = createFeedVersion( + feedSource, + zipFolderFiles("fake-agency-with-only-calendar-and-trailing-newlines") + ); + FeedTransformation transformation = AppendToFileTransformation.create(generateStopRowWithLeadingNewline(), "stops"); + FeedTransformRules transformRules = new FeedTransformRules(transformation); + feedSource.transformRules.add(transformRules); + Persistence.feedSources.replace(feedSource.id, feedSource); + // Create new target version (note: the folder has no stop_attributes.txt file) + targetVersion = createFeedVersion( + feedSource, + zipFolderFiles("fake-agency-with-only-calendar-dates") + ); + LOG.info("Checking assertions."); + assertEquals( + 5 + 3, // Magic number should match row count of stops.txt with three extra + targetVersion.feedLoadResult.stops.rowCount, + "stops.txt row count should equal input csv data # of rows + 3 extra rows" + ); + // Check for presence of new stop id in database (one record). + assertThatSqlCountQueryYieldsExpectedCount( + String.format( + "SELECT count(*) FROM %s.stops WHERE stop_id = '%s'", + targetVersion.namespace, + "new" + ), + 1 + ); + } + @Test + void canAppendToStopsWithTrailingNewlineInData() throws SQLException, IOException { + sourceVersion = createFeedVersion( + feedSource, + zipFolderFiles("fake-agency-with-only-calendar-and-trailing-newlines") + ); + FeedTransformation transformation = AppendToFileTransformation.create(generateStopRowWithTrailingNewline(), "stops"); + FeedTransformRules transformRules = new FeedTransformRules(transformation); + feedSource.transformRules.add(transformRules); + Persistence.feedSources.replace(feedSource.id, feedSource); + // Create new target version (note: the folder has no stop_attributes.txt file) + targetVersion = createFeedVersion( + feedSource, + zipFolderFiles("fake-agency-with-only-calendar-dates") + ); + LOG.info("Checking assertions."); + assertEquals( + 5 + 3, // Magic number should match row count of stops.txt with three extra + targetVersion.feedLoadResult.stops.rowCount, + "stops.txt row count should equal input csv data # of rows + 3 extra rows" + ); + // Check for presence of new stop id in database (one record). + assertThatSqlCountQueryYieldsExpectedCount( + String.format( + "SELECT count(*) FROM %s.stops WHERE stop_id = '%s'", + targetVersion.namespace, + "new" + ), + 1 + ); + } + @Test void canReplaceFeedInfo() throws SQLException, IOException { // Generate random UUID for feedId, which gets placed into the csv data. @@ -311,6 +376,16 @@ private static String generateStopRow() { "\nnew2,new2,appended stop,,37,-122,,,0,123,," + "\nnew,new,appended stop,,37.06668,-122.07781,,,0,123,,"; } + private static String generateStopRowWithLeadingNewline() { + return "\nnew3,new3,appended stop,,37,-122,,,0,123,," + + "\nnew2,new2,appended stop,,37,-122,,,0,123,," + + "\nnew,new,appended stop,,37.06668,-122.07781,,,0,123,,"; + } + private static String generateStopRowWithTrailingNewline() { + return "new3,new3,appended stop,,37,-122,,,0,123,," + + "\nnew2,new2,appended stop,,37,-122,,,0,123,," + + "\nnew,new,appended stop,,37.06668,-122.07781,,,0,123,,\n"; + } private static String generateCustomCsvData() { return "custom_column1,custom_column2,custom_column3" diff --git a/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-only-calendar-and-trailing-newlines/agency.txt b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-only-calendar-and-trailing-newlines/agency.txt new file mode 100755 index 000000000..a916ce91b --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-only-calendar-and-trailing-newlines/agency.txt @@ -0,0 +1,2 @@ +agency_id,agency_name,agency_url,agency_lang,agency_phone,agency_email,agency_timezone,agency_fare_url,agency_branding_url +1,Fake Transit,,,,,America/Los_Angeles,, diff --git a/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-only-calendar-and-trailing-newlines/calendar.txt b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-only-calendar-and-trailing-newlines/calendar.txt new file mode 100755 index 000000000..0e75afb73 --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-only-calendar-and-trailing-newlines/calendar.txt @@ -0,0 +1,3 @@ +service_id,monday,tuesday,wednesday,thursday,friday,saturday,sunday,start_date,end_date +common_id,1,1,1,1,1,1,1,20170918,20170920 +only_calendar_id,1,1,1,1,1,1,1,20170921,20170922 diff --git a/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-only-calendar-and-trailing-newlines/feed_info.txt b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-only-calendar-and-trailing-newlines/feed_info.txt new file mode 100644 index 000000000..ceac60810 --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-only-calendar-and-trailing-newlines/feed_info.txt @@ -0,0 +1,2 @@ +feed_id,feed_publisher_name,feed_publisher_url,feed_lang,feed_version +fake_transit,Conveyal,http://www.conveyal.com,en,1.0 \ No newline at end of file diff --git a/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-only-calendar-and-trailing-newlines/routes.txt b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-only-calendar-and-trailing-newlines/routes.txt new file mode 100755 index 000000000..b13480efa --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-only-calendar-and-trailing-newlines/routes.txt @@ -0,0 +1,3 @@ +agency_id,route_id,route_short_name,route_long_name,route_desc,route_type,route_url,route_color,route_text_color,route_branding_url +1,1,1,Route 1,,3,,7CE6E7,FFFFFF, +1,2,2,Route 2,,3,,7CE6E7,FFFFFF, diff --git a/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-only-calendar-and-trailing-newlines/stop_attributes.txt b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-only-calendar-and-trailing-newlines/stop_attributes.txt new file mode 100644 index 000000000..b77c473e0 --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-only-calendar-and-trailing-newlines/stop_attributes.txt @@ -0,0 +1,3 @@ +stop_id,accessibility_id,cardinal_direction,relative_position,stop_city +4u6g,0,SE,FS,Scotts Valley +johv,0,SE,FS,Scotts Valley diff --git a/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-only-calendar-and-trailing-newlines/stop_times.txt b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-only-calendar-and-trailing-newlines/stop_times.txt new file mode 100755 index 000000000..646706c5c --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-only-calendar-and-trailing-newlines/stop_times.txt @@ -0,0 +1,5 @@ +trip_id,arrival_time,departure_time,stop_id,stop_sequence,stop_headsign,pickup_type,drop_off_type,shape_dist_traveled,timepoint +only-calendar-trip1,07:00:00,07:00:00,4u6g,1,,0,0,0.0000000, +only-calendar-trip1,07:01:00,07:01:00,johv,2,,0,0,341.4491961, +only-calendar-trip2,07:00:00,07:00:00,johv,1,,0,0,0.0000000, +only-calendar-trip2,07:01:00,07:01:00,4u6g,2,,0,0,341.4491961, \ No newline at end of file diff --git a/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-only-calendar-and-trailing-newlines/stops.txt b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-only-calendar-and-trailing-newlines/stops.txt new file mode 100755 index 000000000..b3f2e8d9f --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-only-calendar-and-trailing-newlines/stops.txt @@ -0,0 +1,7 @@ +stop_id,stop_code,stop_name,stop_desc,stop_lat,stop_lon,zone_id,stop_url,location_type,parent_station,stop_timezone,wheelchair_boarding +4u6g,4u6g,Butler Ln,,37.0612132,-122.0074332,,,0,,, +johv,johv,Scotts Valley Dr & Victor Sq,,37.0590172,-122.0096058,,,0,,, +123,,Parent Station,,37.0666,-122.0777,,,1,,, + +1234,1234,Child Stop,,37.06662,-122.07772,,,0,123,, +1234567,1234567,Unused stop,,37.06668,-122.07781,,,0,123,, diff --git a/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-only-calendar-and-trailing-newlines/trips.txt b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-only-calendar-and-trailing-newlines/trips.txt new file mode 100755 index 000000000..387b076cd --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-only-calendar-and-trailing-newlines/trips.txt @@ -0,0 +1,3 @@ +route_id,trip_id,trip_headsign,trip_short_name,direction_id,block_id,shape_id,bikes_allowed,wheelchair_accessible,service_id +1,only-calendar-trip1,,,0,,,0,0,common_id +2,only-calendar-trip2,,,0,,,0,0,common_id \ No newline at end of file From a4deb2c2cf4f1dd1abf72308d96338d4abb26286 Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Tue, 14 May 2024 11:18:11 -0400 Subject: [PATCH 240/248] correctly handle newlines while appending --- .../transform/AppendToFileTransformation.java | 28 +++++++++++++++++-- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/models/transform/AppendToFileTransformation.java b/src/main/java/com/conveyal/datatools/manager/models/transform/AppendToFileTransformation.java index d290afd66..65994582b 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/transform/AppendToFileTransformation.java +++ b/src/main/java/com/conveyal/datatools/manager/models/transform/AppendToFileTransformation.java @@ -5,9 +5,11 @@ import com.conveyal.datatools.manager.models.TableTransformResult; import com.conveyal.datatools.manager.models.TransformType; +import java.io.BufferedReader; import java.io.ByteArrayInputStream; import java.io.File; import java.io.FileOutputStream; +import java.io.FileReader; import java.io.InputStream; import java.io.OutputStream; import java.nio.charset.StandardCharsets; @@ -49,20 +51,40 @@ public void transform(FeedTransformZipTarget zipTarget, MonitorableJob.Status st Path targetTxtFilePath = getTablePathInZip(tableName, targetZipFs); final File tempFile = File.createTempFile(tableName + "-temp", ".txt"); + final File tempFileWithStrippedNewlines = File.createTempFile(tableName + "-temp-no-newlines", ".txt"); Files.copy(targetTxtFilePath, tempFile.toPath(), StandardCopyOption.REPLACE_EXISTING); // Append CSV data into the target file in the temporary copy of file try (OutputStream os = new FileOutputStream(tempFile, true)) { - // Append a newline in case our data doesn't include one - // Having an extra newline is not a problem! os.write(newLineStream.readAllBytes()); os.write(inputStream.readAllBytes()); + os.flush(); + } catch (Exception e) { status.fail("Failed to write to target file", e); } + + // Re-write file without extra line breaks + try ( + OutputStream noNewlineOs = new FileOutputStream(tempFileWithStrippedNewlines, false); + FileReader fr = new FileReader(tempFile); + BufferedReader br = new BufferedReader(fr); + ) { + String line = null; + while ((line = br.readLine()) != null) { + if (line.matches("\n") || line.equals("")) { + continue; + } + + noNewlineOs.write(line.getBytes()); + noNewlineOs.write("\n".getBytes()); + } + noNewlineOs.flush(); + } + // Copy modified file into zip - Files.copy(tempFile.toPath(), targetTxtFilePath, StandardCopyOption.REPLACE_EXISTING); + Files.copy(tempFileWithStrippedNewlines.toPath(), targetTxtFilePath, StandardCopyOption.REPLACE_EXISTING); final int NEW_LINE_CHARACTER_CODE = 10; int lineCount = (int) csvData.chars().filter(c -> c == NEW_LINE_CHARACTER_CODE).count(); From bb5b2daa747dc101e2b24b29647cc082cf2317d9 Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Wed, 15 May 2024 08:39:10 -0400 Subject: [PATCH 241/248] refactor: address pr feedback --- .../models/transform/AppendToFileTransformation.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/models/transform/AppendToFileTransformation.java b/src/main/java/com/conveyal/datatools/manager/models/transform/AppendToFileTransformation.java index 65994582b..68f7cb9d4 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/transform/AppendToFileTransformation.java +++ b/src/main/java/com/conveyal/datatools/manager/models/transform/AppendToFileTransformation.java @@ -67,13 +67,13 @@ public void transform(FeedTransformZipTarget zipTarget, MonitorableJob.Status st // Re-write file without extra line breaks try ( - OutputStream noNewlineOs = new FileOutputStream(tempFileWithStrippedNewlines, false); - FileReader fr = new FileReader(tempFile); - BufferedReader br = new BufferedReader(fr); + OutputStream noNewlineOs = new FileOutputStream(tempFileWithStrippedNewlines, false); + FileReader fr = new FileReader(tempFile); + BufferedReader br = new BufferedReader(fr); ) { - String line = null; + String line; while ((line = br.readLine()) != null) { - if (line.matches("\n") || line.equals("")) { + if (line.matches("\n") || line.isEmpty()) { continue; } From 6a11f0e4e66d0ddda3130c486f4e335f331d31c6 Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Tue, 21 May 2024 13:38:58 -0400 Subject: [PATCH 242/248] disable tokens requirement --- .../java/com/conveyal/datatools/manager/jobs/DeployJob.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java index 162f28525..ffa8bd9a8 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java @@ -7,6 +7,7 @@ import com.amazonaws.services.ec2.model.Filter; import com.amazonaws.services.ec2.model.IamInstanceProfileSpecification; import com.amazonaws.services.ec2.model.Instance; +import com.amazonaws.services.ec2.model.InstanceMetadataOptionsRequest; import com.amazonaws.services.ec2.model.InstanceNetworkInterfaceSpecification; import com.amazonaws.services.ec2.model.InstanceStateChange; import com.amazonaws.services.ec2.model.InstanceType; @@ -1009,6 +1010,7 @@ private List startEC2Instances(int count, boolean graphAlreadyBuilt) { .withKeyName(otpServer.ec2Info.keyName) // This will have the instance terminate when it is shut down. .withInstanceInitiatedShutdownBehavior("terminate") + .withMetadataOptions(new InstanceMetadataOptionsRequest().withHttpTokens("optional").withHttpEndpoint("enabled")) .withUserData(Base64.encodeBase64String(userData.getBytes())); List instances; try { @@ -1383,6 +1385,7 @@ public String constructUserData(boolean graphAlreadyBuilt) { List lines = new ArrayList<>(); lines.add("#!/bin/bash"); + lines.add("mkdir /opt/otp/"); // NOTE: user data output is logged to `/var/log/cloud-init-output.log` automatically with ec2 instances // Add some items to the $PATH as the $PATH with user-data scripts differs from the ssh $PATH. lines.add("export PATH=\"$PATH:/home/ubuntu/.yarn/bin\""); From e9753de0610080e6b7212b1f162468ed3faa5cc3 Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Wed, 22 May 2024 13:48:19 -0400 Subject: [PATCH 243/248] update snapshots --- .../canMakeGraphBuildAndServeManifestAndUserData-1.json | 2 +- .../canMakeOtp2GraphBuildAndServeManifestAndUserData-1.json | 2 +- .../DeployJobTest/canMakeServerOnlyManifestAndUserData-1.json | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/test/resources/snapshots/com/conveyal/datatools/manager/jobs/DeployJobTest/canMakeGraphBuildAndServeManifestAndUserData-1.json b/src/test/resources/snapshots/com/conveyal/datatools/manager/jobs/DeployJobTest/canMakeGraphBuildAndServeManifestAndUserData-1.json index fd57ffc19..a38842fc8 100644 --- a/src/test/resources/snapshots/com/conveyal/datatools/manager/jobs/DeployJobTest/canMakeGraphBuildAndServeManifestAndUserData-1.json +++ b/src/test/resources/snapshots/com/conveyal/datatools/manager/jobs/DeployJobTest/canMakeGraphBuildAndServeManifestAndUserData-1.json @@ -1 +1 @@ -"#!/bin/bash\nexport PATH=\"$PATH:/home/ubuntu/.yarn/bin\"\nexport PATH=\"$PATH:/home/ubuntu/.nvm/versions/node/v12.16.3/bin\"\nrm /usr/share/nginx/client/status.json || echo '' > /dev/null\nrm /var/otp/otp-runner-manifest.json || echo '' > /dev/null\nrm /opt/otp/otp-latest-trimet-dev || echo '' > /dev/null\nrm /var/log/otp-runner.log || echo '' > /dev/null\nrm /var/log/otp-build.log || echo '' > /dev/null\nrm /var/log/otp-server.log || echo '' > /dev/null\naws s3 cp s3://datatools-dev/test-deploy/otp-runner-graph-build-manifest.json /var/otp/otp-runner-manifest.json\nyarn global add https://github.com/ibi-group/otp-runner.git#master\notp-runner /var/otp/otp-runner-manifest.json" \ No newline at end of file +"#!/bin/bash\nmkdir /opt/otp/\nexport PATH=\"$PATH:/home/ubuntu/.yarn/bin\"\nexport PATH=\"$PATH:/home/ubuntu/.nvm/versions/node/v12.16.3/bin\"\nrm /usr/share/nginx/client/status.json || echo '' > /dev/null\nrm /var/otp/otp-runner-manifest.json || echo '' > /dev/null\nrm /opt/otp/otp-latest-trimet-dev || echo '' > /dev/null\nrm /var/log/otp-runner.log || echo '' > /dev/null\nrm /var/log/otp-build.log || echo '' > /dev/null\nrm /var/log/otp-server.log || echo '' > /dev/null\naws s3 cp s3://datatools-dev/test-deploy/otp-runner-graph-build-manifest.json /var/otp/otp-runner-manifest.json\nyarn global add https://github.com/ibi-group/otp-runner.git#master\notp-runner /var/otp/otp-runner-manifest.json" \ No newline at end of file diff --git a/src/test/resources/snapshots/com/conveyal/datatools/manager/jobs/DeployJobTest/canMakeOtp2GraphBuildAndServeManifestAndUserData-1.json b/src/test/resources/snapshots/com/conveyal/datatools/manager/jobs/DeployJobTest/canMakeOtp2GraphBuildAndServeManifestAndUserData-1.json index fd57ffc19..a38842fc8 100644 --- a/src/test/resources/snapshots/com/conveyal/datatools/manager/jobs/DeployJobTest/canMakeOtp2GraphBuildAndServeManifestAndUserData-1.json +++ b/src/test/resources/snapshots/com/conveyal/datatools/manager/jobs/DeployJobTest/canMakeOtp2GraphBuildAndServeManifestAndUserData-1.json @@ -1 +1 @@ -"#!/bin/bash\nexport PATH=\"$PATH:/home/ubuntu/.yarn/bin\"\nexport PATH=\"$PATH:/home/ubuntu/.nvm/versions/node/v12.16.3/bin\"\nrm /usr/share/nginx/client/status.json || echo '' > /dev/null\nrm /var/otp/otp-runner-manifest.json || echo '' > /dev/null\nrm /opt/otp/otp-latest-trimet-dev || echo '' > /dev/null\nrm /var/log/otp-runner.log || echo '' > /dev/null\nrm /var/log/otp-build.log || echo '' > /dev/null\nrm /var/log/otp-server.log || echo '' > /dev/null\naws s3 cp s3://datatools-dev/test-deploy/otp-runner-graph-build-manifest.json /var/otp/otp-runner-manifest.json\nyarn global add https://github.com/ibi-group/otp-runner.git#master\notp-runner /var/otp/otp-runner-manifest.json" \ No newline at end of file +"#!/bin/bash\nmkdir /opt/otp/\nexport PATH=\"$PATH:/home/ubuntu/.yarn/bin\"\nexport PATH=\"$PATH:/home/ubuntu/.nvm/versions/node/v12.16.3/bin\"\nrm /usr/share/nginx/client/status.json || echo '' > /dev/null\nrm /var/otp/otp-runner-manifest.json || echo '' > /dev/null\nrm /opt/otp/otp-latest-trimet-dev || echo '' > /dev/null\nrm /var/log/otp-runner.log || echo '' > /dev/null\nrm /var/log/otp-build.log || echo '' > /dev/null\nrm /var/log/otp-server.log || echo '' > /dev/null\naws s3 cp s3://datatools-dev/test-deploy/otp-runner-graph-build-manifest.json /var/otp/otp-runner-manifest.json\nyarn global add https://github.com/ibi-group/otp-runner.git#master\notp-runner /var/otp/otp-runner-manifest.json" \ No newline at end of file diff --git a/src/test/resources/snapshots/com/conveyal/datatools/manager/jobs/DeployJobTest/canMakeServerOnlyManifestAndUserData-1.json b/src/test/resources/snapshots/com/conveyal/datatools/manager/jobs/DeployJobTest/canMakeServerOnlyManifestAndUserData-1.json index 64c1ee425..26213dbbb 100644 --- a/src/test/resources/snapshots/com/conveyal/datatools/manager/jobs/DeployJobTest/canMakeServerOnlyManifestAndUserData-1.json +++ b/src/test/resources/snapshots/com/conveyal/datatools/manager/jobs/DeployJobTest/canMakeServerOnlyManifestAndUserData-1.json @@ -1 +1 @@ -"#!/bin/bash\nexport PATH=\"$PATH:/home/ubuntu/.yarn/bin\"\nexport PATH=\"$PATH:/home/ubuntu/.nvm/versions/node/v12.16.3/bin\"\nrm /usr/share/nginx/client/status.json || echo '' > /dev/null\nrm /var/otp/otp-runner-manifest.json || echo '' > /dev/null\nrm /opt/otp/otp-latest-trimet-dev || echo '' > /dev/null\nrm /var/log/otp-runner.log || echo '' > /dev/null\nrm /var/log/otp-build.log || echo '' > /dev/null\nrm /var/log/otp-server.log || echo '' > /dev/null\naws s3 cp s3://datatools-dev/test-deploy/otp-runner-server-only-manifest.json /var/otp/otp-runner-manifest.json\nyarn global add https://github.com/ibi-group/otp-runner.git#master\notp-runner /var/otp/otp-runner-manifest.json" \ No newline at end of file +"#!/bin/bash\nmkdir /opt/otp/\nexport PATH=\"$PATH:/home/ubuntu/.yarn/bin\"\nexport PATH=\"$PATH:/home/ubuntu/.nvm/versions/node/v12.16.3/bin\"\nrm /usr/share/nginx/client/status.json || echo '' > /dev/null\nrm /var/otp/otp-runner-manifest.json || echo '' > /dev/null\nrm /opt/otp/otp-latest-trimet-dev || echo '' > /dev/null\nrm /var/log/otp-runner.log || echo '' > /dev/null\nrm /var/log/otp-build.log || echo '' > /dev/null\nrm /var/log/otp-server.log || echo '' > /dev/null\naws s3 cp s3://datatools-dev/test-deploy/otp-runner-server-only-manifest.json /var/otp/otp-runner-manifest.json\nyarn global add https://github.com/ibi-group/otp-runner.git#master\notp-runner /var/otp/otp-runner-manifest.json" \ No newline at end of file From d09319aab602c4b701bbfe1348b5382e7d8bb9e1 Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Wed, 27 Nov 2024 17:03:35 -0500 Subject: [PATCH 244/248] chore(deps): update mobilitydata validator --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 9c412d876..a5ef5d82d 100644 --- a/pom.xml +++ b/pom.xml @@ -262,7 +262,7 @@ org.mobilitydata.gtfs-validator gtfs-validator-main - 4.2.0 + 6.0.0 - a8a376c82b + e9de250