diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index b3db3cf8c..b53d54936 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -43,6 +43,15 @@ jobs: - name: Checkout repository uses: actions/checkout@v3 + # Java setup step completes very fast, no need to run in a preconfigured docker container. + # CodeQL is intended to detect any Java toolchains added to the execution environment. + - name: Set up JDK 21 + uses: actions/setup-java@v3 + with: + distribution: corretto + java-version: 21 + cache: gradle + # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL uses: github/codeql-action/init@v2 diff --git a/.github/workflows/cypress-integration.yml b/.github/workflows/cypress-integration.yml deleted file mode 100644 index 0fd16c257..000000000 --- a/.github/workflows/cypress-integration.yml +++ /dev/null @@ -1,54 +0,0 @@ -name: 'Cypress Integration Tests' - -on: [push] - -jobs: - cypressIntegration: - services: - mongo: - image: mongo - ports: - - 27017:27017 - - runs-on: ubuntu-latest - steps: - # Checkout each repo into sub-directories - - uses: actions/checkout@v2 - with: - repository: conveyal/analysis-ui - ref: 1701bd9aea859a4714bc3f35f5bcf767a3256a64 - path: ui - - uses: actions/checkout@v2 - with: - fetch-depth: 0 - path: r5 - # Build .jar and copy to ./ui directory (along with config file) - # Cache Gradle dependencies to speed up the shadow jar build - - uses: actions/cache@v1 - id: cache - with: - path: ~/.gradle/caches - key: gradle-caches - - uses: actions/setup-java@v1 - with: - java-version: 11 - - run: gradle shadowJar -x test - working-directory: r5 - - run: cp $(ls ./r5/build/libs/*-all.jar | head -n1) ./ui/latest.jar - - run: cp ./r5/analysis.properties.template ./ui/analysis.properties - - # Install / cache dependencies with Cypress to handle caching Cypress binary. - - uses: actions/setup-node@v2 - with: - node-version: '12' - - uses: cypress-io/github-action@v2 - env: - NEXT_PUBLIC_BASEMAP_DISABLED: true - NEXT_PUBLIC_CYPRESS: true - NEXT_PUBLIC_MAPBOX_ACCESS_TOKEN: ${{ secrets.MAPBOX_ACCESS_TOKEN }} - with: - build: yarn build - start: yarn start, yarn start-backend # runs frontend and java server together - wait-on: 'http://localhost:3000, http://localhost:7070/version' - wait-on-timeout: 60 - working-directory: ui diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index 24f08b055..7224c579c 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -28,24 +28,25 @@ jobs: # BUILD_TARGET:staging steps: # Starting in v2.2 checkout action fetches all tags when fetch-depth=0, for auto-versioning. - - uses: actions/checkout@v2.3.2 + - uses: actions/checkout@v3 with: fetch-depth: 0 # Java setup step completes very fast, no need to run in a preconfigured docker container. - - name: Set up JDK 11 + - name: Set up JDK 21 uses: actions/setup-java@v3 with: - java-version: 11 - distribution: temurin - cache: 'gradle' + distribution: corretto + java-version: 21 + cache: gradle - name: Show version string run: gradle -q printVersion | head -n1 - - name: Build and Test + - name: Build and test run: gradle build - - name: Ensure shadow JAR is runnable as local backend + # Check that build product is able to start up as a backend server before handing it to end-to-end testing + - name: Ensure backend runnable run: | cp analysis.properties.template analysis.properties - gradle testShadowJarRunnable + gradle testRunnable -x test - name: Publish to GH Packages # Supply access token to build.gradle (used in publishing.repositories.maven.credentials) env: diff --git a/LICENSE b/LICENSE index dbe6da4ec..f845c96e3 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2020 Conveyal +Copyright (c) 2020-2023 Conveyal Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/build.gradle b/build.gradle index 618a7bb88..7bc3a4fff 100644 --- a/build.gradle +++ b/build.gradle @@ -1,6 +1,6 @@ plugins { id 'java' - id 'com.github.johnrengelman.shadow' version '8.1.0' + id 'application' id 'maven-publish' id 'com.palantir.git-version' version '2.0.0' } @@ -10,18 +10,19 @@ group = 'com.conveyal' version gitVersion() java { - sourceCompatibility = JavaVersion.VERSION_11 - targetCompatibility = JavaVersion.VERSION_11 + toolchain { + languageVersion.set(JavaLanguageVersion.of(21)) + } } jar { - // For Java 11 Modules, specify a module name. + // For Java 11+ Modules, specify a module name. // Do not create module-info.java until all our dependencies specify a module name. // Main-Class BackendMain will start a local backend. // Build-Jdk-Spec mimics a Maven manifest entry that helps us automatically install the right JVM. // Implementation-X attributes are needed for ImageIO (used by Geotools) to initialize in some environments. manifest { - attributes 'Automatic-Module-Name': 'com.conveyal.analysis', + attributes 'Automatic-Module-Name': 'com.conveyal.r5', 'Main-Class': 'com.conveyal.analysis.BackendMain', 'Build-Jdk-Spec': targetCompatibility.getMajorVersion(), 'Implementation-Title': 'Conveyal Analysis Backend', @@ -30,18 +31,15 @@ jar { } } -shadowJar { - mergeServiceFiles() -} - -// Allow reflective access by Kryo to normally closed Java internals. -// This is used for testing equality, but also for building automatic Kryo (de)serializers. +// Allow reflective access by ObjectDiffer to normally closed Java internals. Used for round-trip testing serialization. +// IntelliJ seems not to pass these JVM arguments when running tests from within the IDE, so the Kryo serialization +// tests may only succeed under command line Gradle. test { + useJUnitPlatform() jvmArgs = ['--add-opens=java.base/java.io=ALL-UNNAMED', '--add-opens=java.base/java.time=ALL-UNNAMED', '--add-opens=java.base/java.time.zone=ALL-UNNAMED', '--add-opens=java.base/java.lang=ALL-UNNAMED'] - useJUnitPlatform() } // `gradle publish` will upload both shadow and simple JAR to Github Packages @@ -80,19 +78,25 @@ task copyDependencies(type: Copy) { into 'dependencies' } +application { + applicationDefaultJvmArgs = ['-Xmx6G'] + mainClass = 'com.conveyal.analysis.BackendMain' +} + // Run R5 as a local analysis backend with all dependencies on the classpath, without building a shadowJar. task runBackend (type: JavaExec) { - dependsOn(build) - classpath(sourceSets.main.runtimeClasspath) - mainClass = 'com.conveyal.analysis.BackendMain' + dependsOn(build) + maxHeapSize('7G') + classpath(sourceSets.main.runtimeClasspath) + mainClass = 'com.conveyal.analysis.BackendMain' } -// Start up an analysis local backend from a shaded JAR and ask it to shut down immediately. -// This is used to check in the automated build that the JAR is usable before we keep it. +// Start up an analysis local backend and ask it to shut down immediately. +// This is used to check in the automated build that the JAR is usable in end-to-end tests before we keep it. // Create a configuration properties file (by copying the template) before running this task. -task testShadowJarRunnable(type: JavaExec) { - dependsOn(shadowJar) - classpath(shadowJar.archiveFile.get()) +task testRunnable(type: JavaExec) { + dependsOn(build) + classpath(sourceSets.main.runtimeClasspath) mainClass = 'com.conveyal.analysis.BackendMain' jvmArgs("-Dconveyal.immediate.shutdown=true") } @@ -116,6 +120,12 @@ task createVersionProperties(dependsOn: processResources) { } } +// Fix inconsistent Gradle behavior (see https://github.com/gradle/gradle/issues/16791) +// By default JavaExec tasks use the JVM Gradle was launched with, ignoring the project-level toolchain. +tasks.withType(JavaExec).configureEach { + javaLauncher.set(javaToolchains.launcherFor(java.toolchain)) +} + classes { dependsOn createVersionProperties } @@ -136,10 +146,10 @@ configurations.all { dependencies { // Provides our logging API - implementation 'org.slf4j:slf4j-api:1.7.30' + implementation 'org.slf4j:slf4j-api:2.0.7' // Implementation of the logging API - implementation 'ch.qos.logback:logback-classic:1.2.3' + implementation 'ch.qos.logback:logback-classic:1.4.11' // Spark is an HTTP framework built on Jetty. Its name is the same as several other projects. implementation (group: 'com.sparkjava', name: 'spark-core', version: '2.7.2') { @@ -195,9 +205,6 @@ dependencies { // Commons IO gives us BOMInputStream for handling UTF-8 Byte Order Marks. implementation 'commons-io:commons-io:2.6' - // Guava provides a lot of functionality, collections, and tools "missing" from the Java standard library. - implementation 'com.google.guava:guava:28.2-jre' - // Java 8 rewrite of the Guava cache with asynchronous LoadingCaches. We don't currently use the async // capabilities, but Caffeine's LoadingCache syntax is more modern idiomatic Java than Guava's. implementation 'com.github.ben-manes.caffeine:caffeine:2.8.1' @@ -214,15 +221,19 @@ dependencies { // Commons Math gives us FastMath, MersenneTwister, and low-discrepancy vector generators. implementation 'org.apache.commons:commons-math3:3.0' - // Provides some shared serializers for Kryo. Introduces transitive dependencies on Guava, Trove, and Kryo. + // Provides some Kryo serializers for Guava and Trove collecitons. // Also provides classes for testing that a round trip through serialization reproduces the same network. // This is an external dependency (not merged into backend) because it's also used by OTP2. - // TODO arguably we should declare non-transitive dependencies on Guava, Trove, and Kryo since we use them directly - implementation 'com.conveyal:kryo-tools:1.3.0' + implementation 'com.conveyal:kryo-tools:1.6.0' + // Ensure the versions of the next three dependencies match the transitive dependencies of kryo-tools. + implementation 'com.esotericsoftware:kryo:5.5.0' + // Guava provides a lot of functionality, collections, and tools "missing" from the Java standard library. + implementation 'com.google.guava:guava:32.1.2-jre' // Trove supplies very efficient collections of primitive data types for Java. implementation 'net.sf.trove4j:trove4j:3.0.3' + // TODO eliminate custom Conveyal geojson library, use Geotools? implementation 'com.conveyal:jackson2-geojson:0.9' @@ -245,8 +256,7 @@ dependencies { ////// Test-only dependencies ////// // Java unit testing framework. - testImplementation(platform('org.junit:junit-bom:5.7.0')) - testImplementation('org.junit.jupiter:junit-jupiter') + testImplementation('org.junit.jupiter:junit-jupiter:5.9.2') // Chart drawing library for examining travel time distributions when crafting tests. // Although rarely used it should be low-impact: it is a test-only dependency with no transitive dependenices. diff --git a/src/main/java/com/conveyal/r5/kryo/KryoNetworkSerializer.java b/src/main/java/com/conveyal/r5/kryo/KryoNetworkSerializer.java index 78fd0ca05..67298b1bc 100644 --- a/src/main/java/com/conveyal/r5/kryo/KryoNetworkSerializer.java +++ b/src/main/java/com/conveyal/r5/kryo/KryoNetworkSerializer.java @@ -9,8 +9,9 @@ import com.esotericsoftware.kryo.io.Input; import com.esotericsoftware.kryo.io.Output; import com.esotericsoftware.kryo.serializers.ExternalizableSerializer; +import com.esotericsoftware.kryo.serializers.ImmutableCollectionsSerializers; import com.esotericsoftware.kryo.serializers.JavaSerializer; -import com.esotericsoftware.kryo.util.DefaultStreamFactory; +import com.esotericsoftware.kryo.util.DefaultInstantiatorStrategy; import com.esotericsoftware.kryo.util.MapReferenceResolver; import gnu.trove.impl.hash.TPrimitiveHash; import gnu.trove.list.array.TIntArrayList; @@ -43,8 +44,13 @@ public abstract class KryoNetworkSerializer { * It should also be changed when the semantic content changes from that produced by earlier versions, even when * the serialization format itself does not change. This will ensure newer workers will not load cached older files. * We considered using an ISO date string as the version but that could get confusing when seen in filenames. + * + * History of Network Version (NV) changes: + * nv3 use Kryo 5 serialization format + * nv2 2022-04-05 + * nv1 2021-04-30 stopped using r5 version string (which caused networks to be rebuilt for every new r5 version) */ - public static final String NETWORK_FORMAT_VERSION = "nv2"; + public static final String NETWORK_FORMAT_VERSION = "nv3"; public static final byte[] HEADER = "R5NETWORK".getBytes(); @@ -61,7 +67,7 @@ public abstract class KryoNetworkSerializer { private static Kryo makeKryo () { Kryo kryo; if (COUNT_CLASS_INSTANCES) { - kryo = new Kryo(new InstanceCountingClassResolver(), new MapReferenceResolver(), new DefaultStreamFactory()); + kryo = new Kryo(new InstanceCountingClassResolver(), null); } else { kryo = new Kryo(); } @@ -88,7 +94,7 @@ private static Kryo makeKryo () { // The default strategy requires every class you serialize, even in your dependencies, to have a zero-arg // constructor (which can be private). The setInstantiatorStrategy method completely replaces that default // strategy. The nesting below specifies the Java approach as a fallback strategy to the default strategy. - kryo.setInstantiatorStrategy(new Kryo.DefaultInstantiatorStrategy(new SerializingInstantiatorStrategy())); + kryo.setInstantiatorStrategy(new DefaultInstantiatorStrategy(new SerializingInstantiatorStrategy())); return kryo; } diff --git a/src/main/java/com/conveyal/r5/util/Histogram.java b/src/main/java/com/conveyal/r5/util/Histogram.java index cad306e5b..a585e8b0e 100644 --- a/src/main/java/com/conveyal/r5/util/Histogram.java +++ b/src/main/java/com/conveyal/r5/util/Histogram.java @@ -111,9 +111,9 @@ public void displayHorizontal () { row.append(' '); } - String start = new Integer(minBin).toString(); + String start = Integer.toString(minBin); row.replace(0, start.length(), start); - String end = new Integer(maxBin).toString(); + String end = Integer.toString(maxBin); row.replace(row.length() - end.length(), row.length(), end); System.out.println(row); }