Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Chain stream instead of merge #76

Merged
merged 5 commits into from
Aug 21, 2024
Merged

Chain stream instead of merge #76

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
b3cbcd5
Add a newline to avoid printing jobstats at same level as previous stats
RDruon Aug 20, 2024
137339f
Bump version
RDruon Aug 20, 2024
8e7b786
Fix test
RDruon Aug 20, 2024
bc281a3
Add unit test
RDruon Aug 20, 2024
fd007df
Chain stream instead of merge
RDruon Aug 20, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions Cargo.lock
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ resolver = "2"
authors = ["EMF Team <emf@whamcloud.com>"]
edition = "2021"
license = "MIT"
version = "0.9.1"
version = "0.9.2"

[workspace.lints.rust]
unreachable_pub = "deny"
Expand Down
2 changes: 1 addition & 1 deletion lustrefs-exporter/debian/changelog
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
prometheus-lustrefs-exporter (0.9.1-0.1) unstable; urgency=medium
prometheus-lustrefs-exporter (0.9.2-0.1) unstable; urgency=medium

* New release.

Expand Down
3 changes: 3 additions & 0 deletions lustrefs-exporter/fixtures/jobstats_only/2.14.0_164.txt
View file
Open in desktop
Git LFS file not shown
2 changes: 1 addition & 1 deletion lustrefs-exporter/lustrefs_exporter.spec
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
Name: lustrefs_exporter
Version: 0.9.1
Version: 0.9.2
Release: 1%{?dist}
Summary: prometheus exporter for lustre
License: MIT
Expand Down
31 changes: 27 additions & 4 deletions lustrefs-exporter/src/jobstats.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -134,6 +134,9 @@ pub fn jobstats_stream<R: BufRead + std::marker::Send + 'static>(
let x = tokio::task::spawn_blocking(move || {
let mut state = State::Empty;

// Send a new line to make sure we are printing stats with a separating empty line
_ = tx.blocking_send("\n".to_compact_string());
jgrund marked this conversation as resolved.
Show resolved Hide resolved

for line in f.lines() {
let r = handle_line(&tx, line.map_err(Error::Io), state);

Expand Down Expand Up @@ -335,7 +338,7 @@ pub mod tests {

fut.await.unwrap();

assert_eq!(cnt, 21_147_876);
assert_eq!(cnt, 21_147_876 + 1);
}

#[tokio::test(flavor = "multi_thread")]
Expand All @@ -354,7 +357,7 @@ pub mod tests {

fut.await.unwrap();

assert_eq!(cnt, 5_310_036);
assert_eq!(cnt, 5_310_036 + 1);
}

#[tokio::test(flavor = "multi_thread")]
Expand All @@ -373,7 +376,7 @@ pub mod tests {

fut.await.unwrap();

assert_eq!(cnt, 1_728);
assert_eq!(cnt, 1_728 + 1);
}

const JOBSTAT_JOB: &str = r#"
Expand Down Expand Up @@ -421,6 +424,7 @@ job_stats:{}"#,
4 + // 4 metrics per write_bytes
10) // 10 metrics for "getattr" | "setattr" | "punch" | "sync" | "destroy" | "create" | "statfs" | "get_info" | "set_info" | "quotactl"
* 10
+ 1
);
}

Expand All @@ -440,6 +444,25 @@ job_stats:{}"#,

fut.await.unwrap();

assert_eq!(cnt, 108);
assert_eq!(cnt, 108 + 1);
}

#[tokio::test(flavor = "multi_thread")]
async fn parse_2_14_0_164_jobstats() {
let f = File::open("fixtures/jobstats_only/2.14.0_164.txt").unwrap();

let f = BufReader::with_capacity(128 * 1_024, f);

let (fut, mut rx) = jobstats_stream(f);

let mut output = r#"previous_stat{foo="bar"} 0"#.to_string();

while let Some(x) = rx.recv().await {
output.push_str(x.as_str());
}

fut.await.unwrap();

insta::assert_snapshot!(output);
}
}
2 changes: 1 addition & 1 deletion lustrefs-exporter/src/main.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -187,7 +187,7 @@ async fn scrape(Query(params): Query<Params>) -> Result<Response<Body>, Error> {

let body = if let Some(stream) = jobstats {
let merged =
tokio_stream::StreamExt::merge(stream, tokio_stream::once(Ok(lustre_stats.into())));
tokio_stream::StreamExt::chain(tokio_stream::once(Ok(lustre_stats.into())), stream);

Body::from_stream(merged)
} else {
Expand Down
293 changes: 293 additions & 0 deletions ...exporter/src/snapshots/lustrefs_exporter__jobstats__tests__parse_2_14_0_164_jobstats.snap
View file
Open in desktop

Large diffs are not rendered by default.

Loading