Skip to content

Commit 919fa2b

Browse files
paulheinrpaulheinr
authored
Merge pull request #141 from matsim-vsp/library-updates
Library updates
2 parents a48dc40 + b68e29d commit 919fa2b

File tree

19 files changed

+741
-334
lines changed

19 files changed

+741
-334
lines changed

‎Cargo.lock

+495-244
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Cargo.toml

+11-10
Original file line numberDiff line numberDiff line change
@@ -8,42 +8,43 @@ edition = "2021"
88
[dependencies]
99
serde = { version = "1.0.137", features = ["derive"] }
1010
serde_json = "1.0.93"
11+
# sticking with quick-xml 0.23.0 because the serialization API changed: https://github.com/tafia/quick-xml/issues/499
12+
# since the new API would need an adapter from std::fmt::Write to std::io::Write, we stick with the old API
1113
quick-xml = { version = "0.23.0", features = ["serialize"] }
1214
flate2 = { version = "1.0.24" }
1315
rand = "0.8.5"
14-
metis = "0.1.2"
15-
flexi_logger = { version = "0.22", features = ["async"] }
16+
metis = "0.2.2"
17+
flexi_logger = { version = "0.29.6", features = ["async"] }
1618
clap = { version = "4.0.29", features = ["derive"] }
17-
# point to the github repo directly because we need a more recent version of bindgen. Otherwise the build fails
18-
# due to some incompatibility starting with macOs 13.6
19-
mpi = { git = "https://github.com/rsmpi/rsmpi", rev = "1622101c9fc3a78620e9649f7f895616289d3558" }
19+
mpi = "0.8.0"
2020
# dependencies for serialization with protobuf
21-
prost = "0.11.5"
21+
prost = "0.13.3"
2222
# bytes replaces the signature of std::io::Cursor to implemnt Buf somehow...
2323
bytes = "1.3.0"
24-
serial_test = "2.0.0"
24+
serial_test = "3.2.0"
2525
wait-timeout = "0.2.0"
2626
tracing = "0.1"
2727
tracing-subscriber = { version = "0.3.0", features = ["json", "fmt", "std", "registry"] }
2828
tracing-appender = "0.2"
29-
itertools = "0.10.5"
29+
itertools = "0.13.0"
3030
assert_approx_eq = "1.1.0"
3131
nohash-hasher = "0.2.0"
3232
rand_distr = "0.4.3"
3333
serde_path_to_error = "0.1.14"
34-
geo = "0.26.0"
34+
geo = "0.29.1"
3535
ahash = "0.8.6"
3636
keyed_priority_queue = "0.4.1"
3737
xml = "0.8.10"
3838
lz4_flex = "0.11.1"
39-
typetag = "0.2.13"
39+
typetag = "0.2.18"
4040
serde_yaml = "0.9.27"
4141

4242
[build-dependencies]
4343
# generates types based on .proto files
4444
prost-build = "0.11"
4545
# provides the protoc compiler - this makes the build slow but one doesn't need to
4646
# install the protoc compiler as prerequisite.
47+
# updating to 2.1.0 causes huge compile times (https://github.com/MaterializeInc/rust-protobuf-native/issues/24). Maybe we should switch to another crate.
4748
protobuf-src = "1.1.0"
4849

4950
[profile.bench]

‎README.md

+66-21
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ The most recent release can be cited with the following reference
88
[![DOI](https://zenodo.org/badge/498376436.svg)](https://zenodo.org/doi/10.5281/zenodo.13928119)
99

1010
The project is described in two conference papers, which were presented at ISPDC 24 in Chur, Switzerland, July 2024:
11+
1112
- [High-Performance Simulations for Urban Planning: Implementing Parallel Distributed Multi-Agent Systems in MATSim](https://doi.org/10.1109/ISPDC62236.2024.10705395)
1213
- [Real-Time Routing in Traffic Simulations: A Distributed Event Processing Approach](https://doi.org/10.1109/ISPDC62236.2024.10705399)
1314

@@ -16,8 +17,8 @@ The project is described in two conference papers, which were presented at ISPDC
1617
Install Rust for your operating system as described [here](https://www.rust-lang.org/tools/install). For WSL this would
1718
be
1819

19-
```
20-
$ curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh
20+
```shell
21+
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh
2122
```
2223

2324
## Set Up Prerequisites
@@ -28,8 +29,8 @@ compiled with the project, but need to be present on the operating system.
2829
### METIS
2930

3031
The project uses the [metis](https://crates.io/crates/metis) crate as a dependency which
31-
is a wrapper for the [METIS C Library](https://github.com/KarypisLab/METIS). The C-Library is
32-
expected to be present on the machine. Also, the `metis` crate requires `libclang` on the machine
32+
is a wrapper for the [METIS C Library](https://github.com/KarypisLab/METIS). The C-Library is
33+
expected to be present on the machine. Also, the `metis` crate requires `libclang` on the machine
3334
this project is built on.
3435

3536
### MPI
@@ -51,61 +52,105 @@ The dependencies named above need to be installed before the project can be buit
5152

5253
Install dev versions of required packages because dev stuff is required during compilation
5354

54-
```
55-
$ sudo apt -y install libclang-dev llvm-dev libmetis-dev libopenmpi-dev
55+
```shell
56+
sudo apt -y install libclang-dev llvm-dev libmetis-dev libopenmpi-dev
5657
```
5758

5859
#### MacOs
5960

6061
The dependencies are available via [homebrew](https://brew.sh/) on macOS.
6162

62-
```
63-
$ brew install metis open-mpi
63+
```shell
64+
brew install metis open-mpi
6465
```
6566

6667
The project contains a `config.toml` which tries to set the `CPATH` and the `RUSTFLAGS` environment variable. In case
6768
this doesn't work, they need to be set like the following:
68-
```
69-
$ export CPATH=$HOMEBREW_PREFIX/include
70-
$ export RUSTFLAGS="-L$HOMEBREW_PREFIX/lib"
69+
70+
```shell
71+
export CPATH=$HOMEBREW_PREFIX/include
72+
export RUSTFLAGS="-L$HOMEBREW_PREFIX/lib"
7173
```
7274

7375
Both variables are necessary to compile the METIS and MPI wrapper libraries.
7476

7577
#### Math Cluster
7678

7779
The math cluster has all dependencies installed. They need to be enabled via the module system:
80+
81+
```shell
82+
module load metis-5.1 ompi/gcc/4.1.2
7883
```
79-
$ module load metis-5.1 ompi/gcc/4.1.2
80-
```
84+
8185
#### HLRN
8286

83-
The HLRN cluster has all dependencies installed. They need to be enabled via the module system:
87+
##### Setup conda
88+
89+
Unfortunately, there is no `libclang` dependency installed. You need to install it yourself via `conda`. If you use it
90+
for the first time, load the conda module and initialize it, such that it is available in your shell whenever you login:
91+
92+
```shell
93+
module load anaconda3/2023.09
94+
conda init bash
95+
```
96+
97+
Then create your own environment and install `libclang`:
98+
99+
```shell
100+
conda create -n your_env_name
101+
conda activate your_env_name
102+
conda install libclang
103+
```
104+
105+
##### Load dependencies
106+
107+
The HLRN cluster has **some** dependencies installed. They need to be enabled via the module system:
108+
109+
```shell
110+
module load intel/2024.2 openmpi/gcc/5.0.3
84111
```
85-
$ module load gcc/9.3.0 llvm/9.0.0 openmpi/gcc.9/4.1.4 metis/5.1.0
112+
113+
So, before you run the project, you need to activate the environment:
114+
115+
```shell
116+
conda activate your_env_name
86117
```
87118

119+
The activation automatically updates the environment variables such that `libclang` files can be found by the compiler.
120+
121+
Source: https://nhr-zib.atlassian.net/wiki/spaces/PUB/pages/430343/Anaconda+conda+and+Mamba
122+
88123
## Run the project
89124

90125
The project is built using cargo.
91126

92-
```
93-
$ cargo build --release
127+
```shell
128+
cargo build --release
94129
```
95130

96131
Then a simulation can be started like the following:
97-
```
98-
$ mpirun -np 2 ./target/release/mpi_qsim --config-path /path/to/config.yml
132+
133+
```shell
134+
mpirun -np 2 ./target/release/mpi_qsim --config-path /path/to/config.yml
99135
```
100136

101137
It is also possible to execute a build before running by executing the following. This is way, one doesn't
102138
forget to re-compile before running.
103-
```
104-
$ cargo mpirun --np 2 --release --bin mpi_qsim -- --config-path /path/to/config.yaml
139+
140+
```shell
141+
cargo mpirun --np 2 --release --bin mpi_qsim -- --config-path /path/to/config.yaml
105142
```
106143

107144
We also have a
108145

109146
### Test
110147

111148
Run `$ cargo test` to execute all tests. To have immediate output use `$ cargo test -- --nocapture`
149+
150+
## Create input files
151+
152+
You need to create protobuf files from the xml files. This can be done with the following command:
153+
154+
```shell
155+
cargo run --bin convert_to_binary --release -- --network network.xml --population population.xml --vehicles vehicles.xml --output-dir output --run-id run
156+
```

‎assets/adhoc_routing/with_updates/network.xml

+77-14
Original file line numberDiff line numberDiff line change
@@ -2,23 +2,86 @@
22
<!DOCTYPE network SYSTEM "http://www.matsim.org/files/dtd/network_v2.dtd">
33
<network>
44
<nodes>
5-
<node id="node0" x="-100.0" y="0.0"/>
6-
<node id="node1" x="0.0" y="0.0"/>
7-
<node id="node2" x="100.0" y="0.0"/>
8-
<node id="node3" x="1100.0" y="0.0"/>
9-
<node id="node4" x="1200.0" y="0.0" />
10-
<node id="node5" x="1300.0" y="0.0" />
11-
<node id="node6" x="600.0" y="300.0" />
5+
<node id="node0" x="-100.0" y="0.0">
6+
<attributes>
7+
<attribute name="partition" class="java.lang.Integer">1</attribute>
8+
</attributes>
9+
</node>
10+
<node id="node1" x="0.0" y="0.0">
11+
<attributes>
12+
<attribute name="partition" class="java.lang.Integer">1</attribute>
13+
</attributes>
14+
</node>
15+
<node id="node2" x="100.0" y="0.0">
16+
<attributes>
17+
<attribute name="partition" class="java.lang.Integer">1</attribute>
18+
</attributes>
19+
</node>
20+
<node id="node3" x="1100.0" y="0.0">
21+
<attributes>
22+
<attribute name="partition" class="java.lang.Integer">0</attribute>
23+
</attributes>
24+
</node>
25+
<node id="node4" x="1200.0" y="0.0">
26+
<attributes>
27+
<attribute name="partition" class="java.lang.Integer">0</attribute>
28+
</attributes>
29+
</node>
30+
<node id="node5" x="1300.0" y="0.0">
31+
<attributes>
32+
<attribute name="partition" class="java.lang.Integer">0</attribute>
33+
</attributes>
34+
</node>
35+
<node id="node6" x="600.0" y="300.0">
36+
<attributes>
37+
<attribute name="partition" class="java.lang.Integer">1</attribute>
38+
</attributes>
39+
</node>
1240
</nodes>
1341
<links capperiod="01:00:00" effectivecellsize="7.5" effectivelanewidth="3.75">
14-
<link id="0" from="node0" to="node1" length="100" freespeed="10" capacity="100.0" permlanes="1.0" oneway="1" modes="bike,car" />
15-
<link id="1" from="node1" to="node2" length="100" freespeed="10" capacity="100.0" permlanes="1.0" oneway="1" modes="bike,car" />
16-
<link id="2" from="node2" to="node3" length="1000" freespeed="10" capacity="3.0" permlanes="1.0" oneway="1" modes="bike,car" />
17-
<link id="3" from="node3" to="node4" length="100" freespeed="10" capacity="100.0" permlanes="1.0" oneway="1" modes="bike,car" />
18-
<link id="4" from="node4" to="node5" length="100" freespeed="10" capacity="100.0" permlanes="1.0" oneway="1" modes="bike,car"/>
42+
<link id="0" from="node0" to="node1" length="100" freespeed="10" capacity="100.0" permlanes="1.0" oneway="1"
43+
modes="bike,car">
44+
<attributes>
45+
<attribute name="partition" class="java.lang.Integer">1</attribute>
46+
</attributes>
47+
</link>
48+
<link id="1" from="node1" to="node2" length="100" freespeed="10" capacity="100.0" permlanes="1.0" oneway="1"
49+
modes="bike,car">
50+
<attributes>
51+
<attribute name="partition" class="java.lang.Integer">1</attribute>
52+
</attributes>
53+
</link>
54+
<link id="2" from="node2" to="node3" length="1000" freespeed="10" capacity="3.0" permlanes="1.0" oneway="1"
55+
modes="bike,car">
56+
<attributes>
57+
<attribute name="partition" class="java.lang.Integer">0</attribute>
58+
</attributes>
59+
</link>
60+
<link id="3" from="node3" to="node4" length="100" freespeed="10" capacity="100.0" permlanes="1.0" oneway="1"
61+
modes="bike,car">
62+
<attributes>
63+
<attribute name="partition" class="java.lang.Integer">0</attribute>
64+
</attributes>
65+
</link>
66+
<link id="4" from="node4" to="node5" length="100" freespeed="10" capacity="100.0" permlanes="1.0" oneway="1"
67+
modes="bike,car">
68+
<attributes>
69+
<attribute name="partition" class="java.lang.Integer">0</attribute>
70+
</attributes>
71+
</link>
1972

20-
<link id="5" from="node2" to="node6" length="1000" freespeed="10" capacity="100.0" permlanes="1.0" oneway="1" modes="bike,car"/>
21-
<link id="6" from="node6" to="node3" length="1000" freespeed="10" capacity="100.0" permlanes="1.0" oneway="1" modes="bike,car"/>
73+
<link id="5" from="node2" to="node6" length="1000" freespeed="10" capacity="100.0" permlanes="1.0" oneway="1"
74+
modes="bike,car">
75+
<attributes>
76+
<attribute name="partition" class="java.lang.Integer">1</attribute>
77+
</attributes>
78+
</link>
79+
<link id="6" from="node6" to="node3" length="1000" freespeed="10" capacity="100.0" permlanes="1.0" oneway="1"
80+
modes="bike,car">
81+
<attributes>
82+
<attribute name="partition" class="java.lang.Integer">0</attribute>
83+
</attributes>
84+
</link>
2285

2386
</links>
2487
</network>

‎assets/rvr-1pct.config.yml

+27
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
modules:
2+
protofiles:
3+
type: ProtoFiles
4+
network: /Users/paulheinrich/Nextcloud/Shared/RustQSim/scenarios/rvr/input/rvr-1pct/rvr-1pct.network.binpb
5+
population: /Users/paulheinrich/Nextcloud/Shared/RustQSim/scenarios/rvr/input/rvr-1pct/rvr-1pct.plans.binpb
6+
vehicles: /Users/paulheinrich/Nextcloud/Shared/RustQSim/scenarios/rvr/input/rvr-1pct/rvr-1pct.vehicles.binpb
7+
ids: /Users/paulheinrich/Nextcloud/Shared/RustQSim/scenarios/rvr/input/rvr-1pct/rvr-1pct.ids.binpb
8+
partitioning:
9+
type: Partitioning
10+
num_parts: 0
11+
method: !Metis
12+
vertex_weight:
13+
- PreComputed
14+
output:
15+
type: Output
16+
output_dir: ./assets/rvr-1pct/size
17+
profiling: !CSV
18+
level: INFO
19+
routing:
20+
type: Routing
21+
mode: UsePlans
22+
simulation:
23+
type: Simulation
24+
start_time: 0
25+
end_time: 129600
26+
sample_size: 0.01
27+
stuck_threshold: 30

‎rust-toolchain.toml

-3
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,2 @@
1-
# use nighly build because rust road router need experimental features.
21
[toolchain]
3-
# due to https://github.com/intellij-rust/intellij-rust/issues/10427
4-
#channel = "nightly-2023-05-01"
52
channel = "stable"

‎src/simulation/controller.rs

+7-4
Original file line numberDiff line numberDiff line change
@@ -60,13 +60,16 @@ pub fn run_channel() {
6060
pub fn run_mpi() {
6161
let universe = mpi::initialize().unwrap();
6262
let world = universe.world();
63+
let size = world.size();
64+
let rank = world.rank();
65+
6366
let comm = MpiSimCommunicator {
6467
mpi_communicator: world,
6568
};
6669

6770
let mut args = CommandLineArgs::parse();
6871
// override the num part argument, with the number of processes mpi has started.
69-
args.num_parts = Some(world.size() as u32);
72+
args.num_parts = Some(size as u32);
7073
let config = Config::from_file(&args);
7174

7275
let _guards = logging::init_logging(&config, &args.config_path, comm.rank());
@@ -77,9 +80,9 @@ pub fn run_mpi() {
7780
);
7881
execute_partition(comm, &args);
7982

80-
info!("#{} at barrier.", world.rank());
83+
info!("#{} at barrier.", rank);
8184
universe.world().barrier();
82-
info!("Process #{} finishing.", world.rank());
85+
info!("Process #{} finishing.", rank);
8386
}
8487

8588
fn execute_partition<C: SimCommunicator + 'static>(comm: C, args: &CommandLineArgs) {
@@ -230,7 +233,7 @@ pub fn get_numbered_output_filename(output_dir: &Path, input_file: &Path, part:
230233
insert_number_in_proto_filename(&out, part)
231234
}
232235

233-
fn create_output_filename(output_dir: &Path, input_file: &Path) -> PathBuf {
236+
pub fn create_output_filename(output_dir: &Path, input_file: &Path) -> PathBuf {
234237
let filename = input_file.file_name().unwrap();
235238
output_dir.join(filename)
236239
}

0 commit comments

Comments
 (0)