diff --git a/Cargo.lock b/Cargo.lock index 5594987..b6e1b57 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -37,6 +37,21 @@ dependencies = [ "memchr", ] +[[package]] +name = "android-tzdata" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + [[package]] name = "anstream" version = "0.6.4" @@ -188,6 +203,12 @@ version = "3.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec" +[[package]] +name = "bytemuck" +version = "1.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "374d28ec25809ee0e23827c2ab573d729e293f281dfe393500e7ad618baa61c6" + [[package]] name = "byteorder" version = "1.5.0" @@ -243,6 +264,20 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "chrono" +version = "0.4.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f2c685bad3eb3d45a01354cedb7d5faa66194d1d58ba6e267a8de788f79db38" +dependencies = [ + "android-tzdata", + "iana-time-zone", + "js-sys", + "num-traits", + "wasm-bindgen", + "windows-targets 0.48.5", +] + [[package]] name = "cipher" version = "0.4.4" @@ -314,6 +349,21 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "702fc72eb24e5a1e48ce58027a675bc24edd52096d5397d4aea7c6dd9eca0bd1" +[[package]] +name = "cmake" +version = "0.1.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a31c789563b815f77f4250caee12365734369f942439b7defd71e18a48197130" +dependencies = [ + "cc", +] + +[[package]] +name = "color_quant" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b" + [[package]] name = "colorchoice" version = "1.0.0" @@ -333,6 +383,12 @@ dependencies = [ "windows-sys 0.45.0", ] +[[package]] +name = "const-cstr" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed3d0b5ff30645a68f35ece8cea4556ca14ef8a1651455f789a099a0513532a6" + [[package]] name = "constant_time_eq" version = "0.1.5" @@ -355,6 +411,42 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" +[[package]] +name = "core-graphics" +version = "0.22.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2581bbab3b8ffc6fcbd550bf46c355135d16e9ff2a6ea032ad6b9bf1d7efe4fb" +dependencies = [ + "bitflags 1.3.2", + "core-foundation", + "core-graphics-types", + "foreign-types", + "libc", +] + +[[package]] +name = "core-graphics-types" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bb142d41022986c1d8ff29103a1411c8a3dfad3552f87a4f8dc50d61d4f4e33" +dependencies = [ + "bitflags 1.3.2", + "core-foundation", + "libc", +] + +[[package]] +name = "core-text" +version = "19.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99d74ada66e07c1cefa18f8abfba765b486f250de2e4a999e5727fc0dd4b4a25" +dependencies = [ + "core-foundation", + "core-graphics", + "foreign-types", + "libc", +] + [[package]] name = "cpufeatures" version = "0.2.11" @@ -514,6 +606,48 @@ dependencies = [ "subtle", ] +[[package]] +name = "dirs-next" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1" +dependencies = [ + "cfg-if", + "dirs-sys-next", +] + +[[package]] +name = "dirs-sys-next" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d" +dependencies = [ + "libc", + "redox_users", + "winapi", +] + +[[package]] +name = "dlib" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330c60081dcc4c72131f8eb70510f1ac07223e5d4163db481a04a0befcffa412" +dependencies = [ + "libloading", +] + +[[package]] +name = "dwrote" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439a1c2ba5611ad3ed731280541d36d2e9c4ac5e7fb818a27b604bdc5a6aa65b" +dependencies = [ + "lazy_static", + "libc", + "winapi", + "wio", +] + [[package]] name = "either" version = "1.9.0" @@ -564,6 +698,15 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5" +[[package]] +name = "fdeflate" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d329bdeac514ee06249dabc27877490f17f5d371ec693360768b838e19f3ae10" +dependencies = [ + "simd-adler32", +] + [[package]] name = "flate2" version = "1.0.28" @@ -574,12 +717,43 @@ dependencies = [ "miniz_oxide", ] +[[package]] +name = "float-ord" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7bad48618fdb549078c333a7a8528acb57af271d0433bdecd523eb620628364e" + [[package]] name = "fnv" version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "font-kit" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21fe28504d371085fae9ac7a3450f0b289ab71e07c8e57baa3fb68b9e57d6ce5" +dependencies = [ + "bitflags 1.3.2", + "byteorder", + "core-foundation", + "core-graphics", + "core-text", + "dirs-next", + "dwrote", + "float-ord", + "freetype", + "lazy_static", + "libc", + "log", + "pathfinder_geometry", + "pathfinder_simd", + "walkdir", + "winapi", + "yeslogic-fontconfig-sys", +] + [[package]] name = "foreign-types" version = "0.3.2" @@ -604,6 +778,27 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "freetype" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bee38378a9e3db1cc693b4f88d166ae375338a0ff75cb8263e1c601d51f35dc6" +dependencies = [ + "freetype-sys", + "libc", +] + +[[package]] +name = "freetype-sys" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a37d4011c0cc628dfa766fcc195454f4b068d7afdc2adfd28861191d866e731a" +dependencies = [ + "cmake", + "libc", + "pkg-config", +] + [[package]] name = "futures" version = "0.3.29" @@ -702,6 +897,16 @@ dependencies = [ "wasi", ] +[[package]] +name = "gif" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80792593675e051cf94a4b111980da2ba60d4a83e43e0048c5693baab3977045" +dependencies = [ + "color_quant", + "weezl", +] + [[package]] name = "gimli" version = "0.28.0" @@ -878,6 +1083,29 @@ dependencies = [ "tokio-native-tls", ] +[[package]] +name = "iana-time-zone" +version = "0.1.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fad5b825842d2b38bd206f3e81d6957625fd7f0a361e345c30e01a0ae2dd613" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "wasm-bindgen", + "windows", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + [[package]] name = "idna" version = "0.4.0" @@ -888,6 +1116,21 @@ dependencies = [ "unicode-normalization", ] +[[package]] +name = "image" +version = "0.24.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f3dfdbdd72063086ff443e297b61695500514b1e41095b6fb9a5ab48a70a711" +dependencies = [ + "bytemuck", + "byteorder", + "color_quant", + "jpeg-decoder", + "num-rational", + "num-traits", + "png", +] + [[package]] name = "indexmap" version = "1.9.3" @@ -976,6 +1219,12 @@ dependencies = [ "libc", ] +[[package]] +name = "jpeg-decoder" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc0000e42512c92e31c2252315bda326620a4e034105e900c98ec492fa077b3e" + [[package]] name = "js-sys" version = "0.3.64" @@ -997,6 +1246,16 @@ version = "0.2.149" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a08173bc88b7955d1b3145aa561539096c421ac8debde8cbc3612ec635fee29b" +[[package]] +name = "libloading" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c571b676ddfc9a8c12f1f3d3085a7b163966a8fd8098a90640953ce5f6170161" +dependencies = [ + "cfg-if", + "windows-sys 0.48.0", +] + [[package]] name = "linux-raw-sys" version = "0.4.10" @@ -1047,6 +1306,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7" dependencies = [ "adler", + "simd-adler32", ] [[package]] @@ -1084,6 +1344,27 @@ dependencies = [ "tempfile", ] +[[package]] +name = "num-integer" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" +dependencies = [ + "autocfg", + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0638a1c9d0a3c0914158145bc76cff373a75a627e6ecbfb71cbe6f453a5a19b0" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.17" @@ -1185,6 +1466,25 @@ dependencies = [ "subtle", ] +[[package]] +name = "pathfinder_geometry" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b7b7e7b4ea703700ce73ebf128e1450eb69c3a8329199ffbfb9b2a0418e5ad3" +dependencies = [ + "log", + "pathfinder_simd", +] + +[[package]] +name = "pathfinder_simd" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0444332826c70dc47be74a7c6a5fc44e23a7905ad6858d4162b658320455ef93" +dependencies = [ + "rustc_version", +] + [[package]] name = "pbkdf2" version = "0.11.0" @@ -1227,9 +1527,16 @@ version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2c224ba00d7cadd4d5c660deaf2098e5e80e07846537c51f9cfa4be50c1fd45" dependencies = [ + "chrono", + "font-kit", + "image", + "lazy_static", "num-traits", + "pathfinder_geometry", "plotters-backend", + "plotters-bitmap", "plotters-svg", + "ttf-parser", "wasm-bindgen", "web-sys", ] @@ -1240,6 +1547,17 @@ version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e76628b4d3a7581389a35d5b6e2139607ad7c75b17aed325f210aa91f4a9609" +[[package]] +name = "plotters-bitmap" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cebbe1f70205299abc69e8b295035bb52a6a70ee35474ad10011f0a4efb8543" +dependencies = [ + "gif", + "image", + "plotters-backend", +] + [[package]] name = "plotters-svg" version = "0.3.5" @@ -1249,6 +1567,19 @@ dependencies = [ "plotters-backend", ] +[[package]] +name = "png" +version = "0.17.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd75bf2d8dd3702b9707cdbc56a5b9ef42cec752eb8b3bafc01234558442aa64" +dependencies = [ + "bitflags 1.3.2", + "crc32fast", + "fdeflate", + "flate2", + "miniz_oxide", +] + [[package]] name = "portable-atomic" version = "1.5.0" @@ -1347,6 +1678,15 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "redox_syscall" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" +dependencies = [ + "bitflags 1.3.2", +] + [[package]] name = "redox_syscall" version = "0.4.1" @@ -1356,6 +1696,17 @@ dependencies = [ "bitflags 1.3.2", ] +[[package]] +name = "redox_users" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b" +dependencies = [ + "getrandom", + "redox_syscall 0.2.16", + "thiserror", +] + [[package]] name = "regex" version = "1.10.2" @@ -1446,6 +1797,7 @@ dependencies = [ "itertools 0.10.5", "log", "monitor", + "plotters", "progress-streams", "ranges", "rayon", @@ -1480,6 +1832,15 @@ version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" +[[package]] +name = "rustc_version" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" +dependencies = [ + "semver", +] + [[package]] name = "rustix" version = "0.38.21" @@ -1552,6 +1913,12 @@ dependencies = [ "libc", ] +[[package]] +name = "semver" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "836fa6a3e1e547f9a2c4040802ec865b5d85f4014efe00555d7090a3dcaa1090" + [[package]] name = "serde" version = "1.0.190" @@ -1627,6 +1994,12 @@ dependencies = [ "digest", ] +[[package]] +name = "simd-adler32" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe" + [[package]] name = "slab" version = "0.4.9" @@ -1741,7 +2114,7 @@ checksum = "7ef1adac450ad7f4b3c28589471ade84f25f731a7a0fe30d71dfa9f60fd808e5" dependencies = [ "cfg-if", "fastrand", - "redox_syscall", + "redox_syscall 0.4.1", "rustix", "windows-sys 0.48.0", ] @@ -1909,6 +2282,12 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" +[[package]] +name = "ttf-parser" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "375812fa44dab6df41c195cd2f7fecb488f6c09fbaafb62807488cefab642bff" + [[package]] name = "typenum" version = "1.17.0" @@ -2072,6 +2451,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "weezl" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9193164d4de03a926d909d3bc7c30543cecb35400c02114792c2cae20d5e2dbb" + [[package]] name = "winapi" version = "0.3.9" @@ -2103,6 +2488,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e686886bc078bc1b0b600cac0147aadb815089b6e4da64016cbd754b6342700f" +dependencies = [ + "windows-targets 0.48.5", +] + [[package]] name = "windows-sys" version = "0.45.0" @@ -2245,6 +2639,27 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "wio" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d129932f4644ac2396cb456385cbf9e63b5b30c6e8dc4820bdca4eb082037a5" +dependencies = [ + "winapi", +] + +[[package]] +name = "yeslogic-fontconfig-sys" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2bbd69036d397ebbff671b1b8e4d918610c181c5a16073b96f984a38d08c386" +dependencies = [ + "const-cstr", + "dlib", + "once_cell", + "pkg-config", +] + [[package]] name = "zip" version = "0.6.6" diff --git a/Cargo.toml b/Cargo.toml index c80f3dd..f251e32 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,6 +18,7 @@ repository = "https://github.com/google/ripunzip" [features] real_world_benchmark = [] +plot_reads = [ "plotters" ] [dependencies] anyhow = "1.0.66" @@ -28,6 +29,7 @@ indicatif = "0.17.2" itertools = "0.10.5" log = "0.4.17" monitor = "0.1.0" +plotters = { version="0.3.5", optional=true } progress-streams = "1.1.0" ranges = "0.3.3" rayon = "1.6.0" diff --git a/src/unzip/charting_read_plotter.rs b/src/unzip/charting_read_plotter.rs new file mode 100644 index 0000000..c8f65bc --- /dev/null +++ b/src/unzip/charting_read_plotter.rs @@ -0,0 +1,82 @@ +// Copyright 2023 Google LLC + +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +use crate::unzip::seekable_http_reader::ReadPlotter; +use crate::unzip::seekable_http_reader::ReadType; +use plotters::prelude::*; + +const TRACE_PNG: &str = "read_pattern.png"; + +#[derive(Default)] +pub struct ChartingReadPlotter { + total_length: u64, + data: Vec<(u64, u64, ReadType)>, +} + +impl ReadType { + fn style(&self) -> ShapeStyle { + match self { + ReadType::FromCache => GREEN, + ReadType::FromCacheDeferred => CYAN, + ReadType::FromCacheAfterDirectAfterRewind => RED, + ReadType::FromCacheAfterDirectAfterSkipOver => BLUE, + ReadType::FromCacheAfterDirect => MAGENTA, + ReadType::IntoCacheDuringFastForward => BLACK, + } + .filled() + } +} + +impl ReadPlotter for ChartingReadPlotter { + fn set_len(&mut self, total_length: u64) { + self.total_length = total_length; + } + fn plot_read(&mut self, start: u64, len: usize, read_type: ReadType) { + if !matches!(read_type, ReadType::FromCache) { + // too noisy + let end = start + len as u64; + self.data.push((start, end, read_type)); + } + } +} + +impl Drop for ChartingReadPlotter { + fn drop(&mut self) { + self.draw() + .unwrap_or_else(|error| log::error!("Unable to save trace plot because {:?}", error)); + } +} + +impl ChartingReadPlotter { + fn draw(&self) -> Result<(), Box> { + let drawing_area = BitMapBackend::new(TRACE_PNG, (4096, 3072)).into_drawing_area(); + drawing_area.fill(&WHITE)?; + + let num_reads = self.data.len() as u64; + + let mut chart = ChartBuilder::on(&drawing_area) + .x_label_area_size(40) + .y_label_area_size(40) + .caption("Read pattern", ("sans-serif", 30.0).into_font()) + .build_cartesian_2d(0u64..self.total_length, 0u64..num_reads)?; + + chart.configure_mesh().light_line_style(WHITE).draw()?; + + chart.draw_series(self.data.iter().enumerate().map( + |(counter, (start, end, read_type))| { + Rectangle::new( + [(*start, counter as u64), (*end, counter as u64 + 1)], + read_type.style(), + ) + }, + ))?; + + log::info!("Trace has been saved to {}", TRACE_PNG); + Ok(()) + } +} diff --git a/src/unzip/mod.rs b/src/unzip/mod.rs index ebfd9fc..6e6c168 100644 --- a/src/unzip/mod.rs +++ b/src/unzip/mod.rs @@ -6,6 +6,8 @@ // option. This file may not be copied, modified, or distributed // except according to those terms. +#[cfg(feature = "plot_reads")] +mod charting_read_plotter; mod cloneable_seekable_reader; mod http_range_reader; mod progress_updater; diff --git a/src/unzip/seekable_http_reader.rs b/src/unzip/seekable_http_reader.rs index 4912ac0..f128d99 100644 --- a/src/unzip/seekable_http_reader.rs +++ b/src/unzip/seekable_http_reader.rs @@ -23,6 +23,12 @@ use super::{ http_range_reader::{self, RangeFetcher}, }; +#[cfg(feature = "plot_reads")] +type ReadPlotterImpl = super::charting_read_plotter::ChartingReadPlotter; + +#[cfg(not(feature = "plot_reads"))] +type ReadPlotterImpl = NullReadPlotter; + /// This is how much we read from the underlying HTTP stream in a given thread, /// before signalling other threads that they may wish to continue with their /// CPU-bound unzipping. Empirically determined. @@ -134,6 +140,8 @@ struct State { max_block: usize, /// Some statistics about how we're doing. stats: SeekableHttpReaderStatistics, + /// Traces the read requests we get and how we service them + read_plotter: ReadPlotterImpl, } impl State { @@ -315,18 +323,20 @@ impl SeekableHttpReaderEngine { return Err(Error::AcceptRangesNotSupported); } let len = range_fetcher.len(); + let mut state = State::new( + readahead_limit, + access_pattern, + skip_ahead_threshold, + max_block, + ); + state.read_plotter.set_len(len); Ok(Arc::new(Self { len, reader: Mutex::new(ReadingMaterials { range_fetcher, reader: None, }), - state: Mutex::new(State::new( - readahead_limit, - access_pattern, - skip_ahead_threshold, - max_block, - )), + state: Mutex::new(state), read_completed: Condvar::new(), })) } @@ -393,6 +403,9 @@ impl SeekableHttpReaderEngine { // - If yes, release CACHE mutex, and return if let Some(bytes_read_from_cache) = state.read_from_cache(pos, buf) { log::debug!("Immediate cache success"); + state + .read_plotter + .plot_read(pos, buf.len(), ReadType::FromCache); return Ok(bytes_read_from_cache); } // - If no, check if read in progress @@ -404,6 +417,9 @@ impl SeekableHttpReaderEngine { // check cache again if let Some(bytes_read_from_cache) = state.read_from_cache(pos, buf) { log::debug!("Deferred cache success"); + state + .read_plotter + .plot_read(pos, buf.len(), ReadType::FromCacheDeferred); return Ok(bytes_read_from_cache); } read_in_progress = state.read_in_progress; @@ -424,6 +440,7 @@ impl SeekableHttpReaderEngine { // perform read // First check if we need to rewind, OR if we need to fast forward // and are expecting to skip over some significant data. + let mut read_type = ReadType::FromCacheAfterDirect; if let Some((_, readerpos)) = reading_stuff.reader.as_ref() { if pos < *readerpos { log::debug!( @@ -432,6 +449,7 @@ impl SeekableHttpReaderEngine { *readerpos ); reading_stuff.reader = None; + read_type = ReadType::FromCacheAfterDirectAfterRewind; } else if pos > *readerpos { let delta = pos - *readerpos; // Discard the existing stream and create a new one if we're skipping ahead a lot, @@ -444,6 +462,7 @@ impl SeekableHttpReaderEngine { *readerpos ); reading_stuff.reader = None; + read_type = ReadType::FromCacheAfterDirectAfterSkipOver; } } } @@ -479,6 +498,11 @@ impl SeekableHttpReaderEngine { // claim STATE mutex let mut state = self.state.lock().unwrap(); state.insert(*reader_pos, new_block); + state.read_plotter.plot_read( + *reader_pos, + to_read, + ReadType::IntoCacheDuringFastForward, + ); // Tell any waiting threads they should re-check the cache self.read_completed.notify_all(); *reader_pos += to_read as u64; @@ -492,6 +516,7 @@ impl SeekableHttpReaderEngine { .read_from_cache(pos, buf) .expect("Cache still couldn't satisfy request event after reading beyond read pos"); log::debug!("Cache success after read"); + state.read_plotter.plot_read(pos, buf.len(), read_type); if reader_created { state.stats.num_http_streams += 1; } @@ -605,6 +630,34 @@ impl HasLength for SeekableHttpReader { } } +#[derive(Debug, Clone, Copy)] +pub enum ReadType { + FromCache, + FromCacheDeferred, + FromCacheAfterDirectAfterRewind, + FromCacheAfterDirectAfterSkipOver, + FromCacheAfterDirect, + IntoCacheDuringFastForward, +} + +/// To be implemented by code which can record a trace of all the +/// various types of read we perform, for diagnostic purposes. +pub trait ReadPlotter { + fn set_len(&mut self, total_length: u64); + fn plot_read(&mut self, start: u64, len: usize, read_type: ReadType); +} + +#[cfg(not(feature = "plot_reads"))] +#[derive(Default)] +struct NullReadPlotter; + +#[cfg(not(feature = "plot_reads"))] +impl ReadPlotter for NullReadPlotter { + fn set_len(&mut self, _total_length: u64) {} + + fn plot_read(&mut self, _start: u64, _len: usize, _read_type: ReadType) {} +} + #[cfg(test)] mod tests { use regex::Regex;