Merge branch 'dev' into revise-simpson-desert

.github/workflows/codeql-analysis.yml

@@ -43,6 +43,15 @@ jobs:
     - name: Checkout repository
       uses: actions/checkout@v3
 
+    # Java setup step completes very fast, no need to run in a preconfigured docker container.
+    # CodeQL is intended to detect any Java toolchains added to the execution environment.
+    - name: Set up JDK 21
+      uses: actions/setup-java@v3
+      with:
+        distribution: corretto
+        java-version: 21
+        cache: gradle
+
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
       uses: github/codeql-action/init@v2

.github/workflows/gradle.yml

@@ -28,24 +28,25 @@ jobs:
     #     BUILD_TARGET:staging
     steps:
     # Starting in v2.2 checkout action fetches all tags when fetch-depth=0, for auto-versioning.
-    - uses: actions/checkout@v2.3.2
+    - uses: actions/checkout@v3
       with:
         fetch-depth: 0
     # Java setup step completes very fast, no need to run in a preconfigured docker container.
-    - name: Set up JDK 11
+    - name: Set up JDK 21
       uses: actions/setup-java@v3
       with:
-        java-version: 11
-        distribution: temurin
-        cache: 'gradle'
+        distribution: corretto
+        java-version: 21
+        cache: gradle
     - name: Show version string
       run: gradle -q printVersion | head -n1
-    - name: Build and Test
+    - name: Build and test
       run: gradle build
-    - name: Ensure shadow JAR is runnable as local backend
+    # Check that build product is able to start up as a backend server before handing it to end-to-end testing
+    - name: Ensure backend runnable
       run: |
         cp analysis.properties.template analysis.properties
-        gradle testShadowJarRunnable
+        gradle testRunnable -x test
     - name: Publish to GH Packages
       # Supply access token to build.gradle (used in publishing.repositories.maven.credentials)
       env:

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2020 Conveyal
+Copyright (c) 2020-2023 Conveyal
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

build.gradle

@@ -1,6 +1,6 @@
 plugins {
     id 'java'
-    id 'com.github.johnrengelman.shadow' version '8.1.0'
+    id 'application'
     id 'maven-publish'
     id 'com.palantir.git-version' version '2.0.0'
 }
@@ -10,18 +10,19 @@ group = 'com.conveyal'
 version gitVersion()
 
 java {
-    sourceCompatibility = JavaVersion.VERSION_11
-    targetCompatibility = JavaVersion.VERSION_11
+    toolchain {
+        languageVersion.set(JavaLanguageVersion.of(21))
+    }
 }
 
 jar {
-    // For Java 11 Modules, specify a module name.
+    // For Java 11+ Modules, specify a module name.
     // Do not create module-info.java until all our dependencies specify a module name.
     // Main-Class BackendMain will start a local backend.
     // Build-Jdk-Spec mimics a Maven manifest entry that helps us automatically install the right JVM.
     // Implementation-X attributes are needed for ImageIO (used by Geotools) to initialize in some environments.
     manifest {
-        attributes 'Automatic-Module-Name': 'com.conveyal.analysis',
+        attributes 'Automatic-Module-Name': 'com.conveyal.r5',
             'Main-Class': 'com.conveyal.analysis.BackendMain',
             'Build-Jdk-Spec': targetCompatibility.getMajorVersion(),
             'Implementation-Title': 'Conveyal Analysis Backend',
@@ -30,18 +31,15 @@ jar {
     }
 }
 
-shadowJar {
-    mergeServiceFiles()
-}
-
-// Allow reflective access by Kryo to normally closed Java internals.
-// This is used for testing equality, but also for building automatic Kryo (de)serializers.
+// Allow reflective access by ObjectDiffer to normally closed Java internals. Used for round-trip testing serialization.
+// IntelliJ seems not to pass these JVM arguments when running tests from within the IDE, so the Kryo serialization
+// tests may only succeed under command line Gradle.
 test {
+    useJUnitPlatform()
     jvmArgs = ['--add-opens=java.base/java.io=ALL-UNNAMED',
                '--add-opens=java.base/java.time=ALL-UNNAMED',
                '--add-opens=java.base/java.time.zone=ALL-UNNAMED',
                '--add-opens=java.base/java.lang=ALL-UNNAMED']
-    useJUnitPlatform()
 }
 
 // `gradle publish` will upload both shadow and simple JAR to Github Packages
@@ -80,19 +78,25 @@ task copyDependencies(type: Copy) {
     into 'dependencies'
 }
 
+application {
+    applicationDefaultJvmArgs = ['-Xmx6G']
+    mainClass = 'com.conveyal.analysis.BackendMain'
+}
+
 // Run R5 as a local analysis backend with all dependencies on the classpath, without building a shadowJar.
 task runBackend (type: JavaExec) {
-   dependsOn(build)
-   classpath(sourceSets.main.runtimeClasspath)
-   mainClass = 'com.conveyal.analysis.BackendMain'
+    dependsOn(build)
+    maxHeapSize('7G')
+    classpath(sourceSets.main.runtimeClasspath)
+    mainClass = 'com.conveyal.analysis.BackendMain'
 }
 
-// Start up an analysis local backend from a shaded JAR and ask it to shut down immediately.
-// This is used to check in the automated build that the JAR is usable before we keep it.
+// Start up an analysis local backend and ask it to shut down immediately.
+// This is used to check in the automated build that the JAR is usable in end-to-end tests before we keep it.
 // Create a configuration properties file (by copying the template) before running this task.
-task testShadowJarRunnable(type: JavaExec) {
-    dependsOn(shadowJar)
-    classpath(shadowJar.archiveFile.get())
+task testRunnable(type: JavaExec) {
+    dependsOn(build)
+    classpath(sourceSets.main.runtimeClasspath)
     mainClass = 'com.conveyal.analysis.BackendMain'
     jvmArgs("-Dconveyal.immediate.shutdown=true")
 }
@@ -116,6 +120,12 @@ task createVersionProperties(dependsOn: processResources) {
     }
 }
 
+// Fix inconsistent Gradle behavior (see https://github.com/gradle/gradle/issues/16791)
+// By default JavaExec tasks use the JVM Gradle was launched with, ignoring the project-level toolchain.
+tasks.withType(JavaExec).configureEach {
+    javaLauncher.set(javaToolchains.launcherFor(java.toolchain))
+}
+
 classes {
     dependsOn createVersionProperties
 }
@@ -136,10 +146,10 @@ configurations.all {
 
 dependencies {
     // Provides our logging API
-    implementation 'org.slf4j:slf4j-api:1.7.30'
+    implementation 'org.slf4j:slf4j-api:2.0.7'
 
     // Implementation of the logging API
-    implementation 'ch.qos.logback:logback-classic:1.2.3'
+    implementation 'ch.qos.logback:logback-classic:1.4.11'
 
     // Spark is an HTTP framework built on Jetty. Its name is the same as several other projects.
     implementation (group: 'com.sparkjava', name: 'spark-core', version: '2.7.2') {
@@ -195,9 +205,6 @@ dependencies {
     // Commons IO gives us BOMInputStream for handling UTF-8 Byte Order Marks.
     implementation 'commons-io:commons-io:2.6'
 
-    // Guava provides a lot of functionality, collections, and tools "missing" from the Java standard library.
-    implementation 'com.google.guava:guava:28.2-jre'
-
     // Java 8 rewrite of the Guava cache with asynchronous LoadingCaches. We don't currently use the async
     // capabilities, but Caffeine's LoadingCache syntax is more modern idiomatic Java than Guava's.
     implementation 'com.github.ben-manes.caffeine:caffeine:2.8.1'
@@ -214,15 +221,19 @@ dependencies {
     // Commons Math gives us FastMath, MersenneTwister, and low-discrepancy vector generators.
     implementation 'org.apache.commons:commons-math3:3.0'
 
-    // Provides some shared serializers for Kryo. Introduces transitive dependencies on Guava, Trove, and Kryo.
+    // Provides some Kryo serializers for Guava and Trove collecitons.
     // Also provides classes for testing that a round trip through serialization reproduces the same network.
     // This is an external dependency (not merged into backend) because it's also used by OTP2.
-    // TODO arguably we should declare non-transitive dependencies on Guava, Trove, and Kryo since we use them directly
-    implementation 'com.conveyal:kryo-tools:1.3.0'
+    implementation 'com.conveyal:kryo-tools:1.6.0'
 
+    // Ensure the versions of the next three dependencies match the transitive dependencies of kryo-tools.
+    implementation 'com.esotericsoftware:kryo:5.5.0'
+    // Guava provides a lot of functionality, collections, and tools "missing" from the Java standard library.
+    implementation 'com.google.guava:guava:32.1.2-jre'
     // Trove supplies very efficient collections of primitive data types for Java.
     implementation 'net.sf.trove4j:trove4j:3.0.3'
 
+
     // TODO eliminate custom Conveyal geojson library, use Geotools?
     implementation 'com.conveyal:jackson2-geojson:0.9'
 
@@ -245,8 +256,7 @@ dependencies {
     ////// Test-only dependencies //////
 
     // Java unit testing framework.
-    testImplementation(platform('org.junit:junit-bom:5.7.0'))
-    testImplementation('org.junit.jupiter:junit-jupiter')
+    testImplementation('org.junit.jupiter:junit-jupiter:5.9.2')
 
     // Chart drawing library for examining travel time distributions when crafting tests.
     // Although rarely used it should be low-impact: it is a test-only dependency with no transitive dependenices.

src/main/java/com/conveyal/analysis/models/AnalysisRequest.java

@@ -216,9 +216,7 @@ public void populateTask (AnalysisWorkerTask task, UserPermissions userPermissio
         task.maxFare = maxFare;
         task.inRoutingFareCalculator = inRoutingFareCalculator;
 
-        // TODO define class with static factory function WebMercatorGridBounds.fromLatLonBounds().
-        //      Also include getIndex(x, y), getX(index), getY(index), totalTasks()
-        WebMercatorExtents extents = WebMercatorExtents.forWgsEnvelope(bounds.envelope(), zoom);
+        WebMercatorExtents extents = WebMercatorExtents.forTrimmedWgsEnvelope(bounds.envelope(), zoom);
         task.height = extents.height;
         task.north = extents.north;
         task.west = extents.west;

src/main/java/com/conveyal/r5/analyst/FreeFormPointSet.java

@@ -251,8 +251,7 @@ public Envelope getWgsEnvelope () {
     public WebMercatorExtents getWebMercatorExtents () {
         final int DEFAULT_ZOOM = 9;
         Envelope wgsEnvelope = this.getWgsEnvelope();
-        WebMercatorExtents webMercatorExtents = WebMercatorExtents.forWgsEnvelope(wgsEnvelope, DEFAULT_ZOOM);
-        return webMercatorExtents;
+        return WebMercatorExtents.forBufferedWgsEnvelope(wgsEnvelope, DEFAULT_ZOOM);
     }
 
     /** Construct a freeform point set containing one opportunity at each specified geographic coordinate. */