From 71e75df3bcdff0bd16db6b841789c7c731c65c6c Mon Sep 17 00:00:00 2001 From: Raphael Druon Date: Mon, 10 Jul 2023 09:18:28 -0600 Subject: [PATCH] Implement osd quota stats --- src/fixtures/valid/valid.txt | 26 +++ src/lib.rs | 2 +- src/osd_parser.rs | 43 +++++ src/parser.rs | 6 +- src/{qmt => quota}/mod.rs | 6 +- .../qmt_parser.rs => quota/quota_parser.rs} | 21 ++- ...collector__parser__tests__node_output.snap | 157 ++++++++++++++++++ src/stats_parser.rs | 2 +- src/types.rs | 19 +++ 9 files changed, 272 insertions(+), 10 deletions(-) rename src/{qmt => quota}/mod.rs (81%) rename src/{qmt/qmt_parser.rs => quota/quota_parser.rs} (87%) diff --git a/src/fixtures/valid/valid.txt b/src/fixtures/valid/valid.txt index 317c57d..e9b68eb 100644 --- a/src/fixtures/valid/valid.txt +++ b/src/fixtures/valid/valid.txt @@ -332,4 +332,30 @@ qmt.testfs-QMT0000.md-0x0.glb-usr= global_pool0_md_usr - id: 0 limits: { hard: 0, soft: 0, granted: 0, time: 604800 } +osd-ldiskfs.testfs-MDT0000.quota_slave.acct_group= +grp_accounting: +- id: 0 + usage: { inodes: 396, kbytes: 529400 } +osd-ldiskfs.testfs-MDT0000.quota_slave.acct_project= +prj_accounting: +- id: 0 + usage: { inodes: 393, kbytes: 529396 } +- id: 1337 + usage: { inodes: 3, kbytes: 4 } +osd-ldiskfs.testfs-MDT0000.quota_slave.acct_user= +usr_accounting: +- id: 0 + usage: { inodes: 396, kbytes: 529400 } +osd-ldiskfs.testfs-MDT0001.quota_slave.acct_group= +grp_accounting: +- id: 0 + usage: { inodes: 265, kbytes: 356672 } +osd-ldiskfs.testfs-MDT0001.quota_slave.acct_project= +prj_accounting: +- id: 0 + usage: { inodes: 265, kbytes: 356672 } +osd-ldiskfs.testfs-MDT0001.quota_slave.acct_user= +usr_accounting: +- id: 0 + usage: { inodes: 265, kbytes: 356672 } ldlm.namespaces.filter-ai400-OST0001_UUID.resource_count=0 diff --git a/src/lib.rs b/src/lib.rs index 2594998..8390dfb 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -13,7 +13,7 @@ mod node_stats_parsers; mod osd_parser; mod oss; pub mod parser; -pub(crate) mod qmt; +pub(crate) mod quota; pub mod recovery_status_parser; mod stats_parser; mod time; diff --git a/src/osd_parser.rs b/src/osd_parser.rs index 88833cf..da46e6d 100644 --- a/src/osd_parser.rs +++ b/src/osd_parser.rs @@ -5,7 +5,9 @@ use crate::{ base_parsers::{digits, param, period, target, till_newline, till_period}, brw_stats_parser::brw_stats, + quota::quota_parser::quota_stats_osd, types::{BrwStats, Param, Record, Target, TargetStat, TargetStats, TargetVariant}, + QuotaKind, QuotaStatsOsd, }; use combine::{ attempt, choice, @@ -24,6 +26,10 @@ pub(crate) const FS_TYPE: &str = "fstype"; pub(crate) const BRW_STATS: &str = "brw_stats"; +pub(crate) const QUOTA_ACCT_GRP: &str = "quota_slave.acct_group"; +pub(crate) const QUOTA_ACCT_USR: &str = "quota_slave.acct_user"; +pub(crate) const QUOTA_ACCT_PRJ: &str = "quota_slave.acct_project"; + pub(crate) fn params() -> Vec { vec![ format!("osd-*.*.{FILES_FREE}"), @@ -33,6 +39,9 @@ pub(crate) fn params() -> Vec { format!("osd-*.*.{KBYTES_FREE}"), format!("osd-*.*.{KBYTES_TOTAL}"), format!("osd-*.*.{BRW_STATS}"), + format!("osd-*.{QUOTA_ACCT_GRP}"), + format!("osd-*.{QUOTA_ACCT_USR}"), + format!("osd-*.{QUOTA_ACCT_PRJ}"), ] } @@ -51,6 +60,7 @@ enum OsdStat { /// Total disk space BytesTotal(u64), BrwStats(Vec), + QuotaStats(QuotaStatsOsd), } fn target_and_variant() -> impl Parser @@ -116,6 +126,33 @@ where .map(|x| x * 1024) .map(OsdStat::BytesTotal), ), + ( + param(QUOTA_ACCT_GRP), + quota_stats_osd().map(|stats| { + OsdStat::QuotaStats(QuotaStatsOsd { + kind: QuotaKind::Grp, + stats, + }) + }), + ), + ( + param(QUOTA_ACCT_PRJ), + quota_stats_osd().map(|stats| { + OsdStat::QuotaStats(QuotaStatsOsd { + kind: QuotaKind::Prj, + stats, + }) + }), + ), + ( + param(QUOTA_ACCT_USR), + quota_stats_osd().map(|stats| { + OsdStat::QuotaStats(QuotaStatsOsd { + kind: QuotaKind::Usr, + stats, + }) + }), + ), )) } @@ -168,6 +205,12 @@ where param, value, }), + OsdStat::QuotaStats(value) => TargetStats::QuotaStatsOsd(TargetStat { + kind, + target, + param, + value, + }), }) .map(Record::Target) .message("while parsing osd") diff --git a/src/parser.rs b/src/parser.rs index 929638e..d314896 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -8,7 +8,7 @@ use crate::{ mgs::mgs_parser, osd_parser, oss::oss_parser, - qmt, top_level_parser, + quota, top_level_parser, types::Record, }; use combine::{choice, error::ParseError, many, Parser, Stream}; @@ -23,7 +23,7 @@ pub fn params() -> Vec { .chain(oss_parser::params()) .chain(mds_parser::params()) .chain(ldlm::params()) - .chain(qmt::params()) + .chain(quota::params()) .collect() } @@ -40,7 +40,7 @@ where mds_parser::parse().map(|x| vec![x]), oss_parser::parse().map(|x| vec![x]), ldlm::parse().map(|x| vec![x]), - qmt::parse().map(|x| vec![x]), + quota::parse().map(|x| vec![x]), ))) .map(|xs: Vec<_>| xs.into_iter().flatten().collect()) } diff --git a/src/qmt/mod.rs b/src/quota/mod.rs similarity index 81% rename from src/qmt/mod.rs rename to src/quota/mod.rs index 04ba45c..ea652df 100644 --- a/src/qmt/mod.rs +++ b/src/quota/mod.rs @@ -5,12 +5,12 @@ use crate::{base_parsers::period, Record}; use combine::{parser::char::string, ParseError, Parser, Stream}; -mod qmt_parser; +pub(crate) mod quota_parser; pub(crate) const QMT: &str = "qmt"; pub(crate) fn params() -> Vec { - qmt_parser::params() + quota_parser::params() } pub(crate) fn parse() -> impl Parser @@ -18,5 +18,5 @@ where I: Stream, I::Error: ParseError, { - (string(QMT), period()).with(qmt_parser::parse()) + (string(QMT), period()).with(quota_parser::parse()) } diff --git a/src/qmt/qmt_parser.rs b/src/quota/quota_parser.rs similarity index 87% rename from src/qmt/qmt_parser.rs rename to src/quota/quota_parser.rs index e41b856..cd846ab 100644 --- a/src/qmt/qmt_parser.rs +++ b/src/quota/quota_parser.rs @@ -4,9 +4,9 @@ use crate::{ base_parsers::{param, period, target}, - qmt::QMT, + quota::QMT, types::{Param, Record, Target, TargetStats}, - QuotaKind, QuotaStat, QuotaStats, TargetQuotaStat, + QuotaKind, QuotaStat, QuotaStatOsd, QuotaStats, TargetQuotaStat, }; use combine::{ attempt, choice, @@ -59,6 +59,23 @@ where }) } +pub(crate) fn quota_stats_osd() -> impl Parser> +where + I: Stream, + I::Error: ParseError, +{ + ( + optional(newline()), // If quota stats are present, the whole yaml blob will be on a newline + many::, _, _>(alpha_num().or(one_of("_-:".chars()))), // But yaml header might not be properly formatted, ignore it + newline(), + take_until(attempt((newline(), alpha_num()))), + ) + .skip(newline()) + .and_then(|(_, _, _, x): (_, _, _, String)| { + serde_yaml::from_str::>(&x).map_err(StreamErrorFor::::other) + }) +} + #[derive(Debug)] pub enum QMTStat { Usr(Vec), diff --git a/src/snapshots/lustre_collector__parser__tests__node_output.snap b/src/snapshots/lustre_collector__parser__tests__node_output.snap index 7008ce5..0a7b1b8 100644 --- a/src/snapshots/lustre_collector__parser__tests__node_output.snap +++ b/src/snapshots/lustre_collector__parser__tests__node_output.snap @@ -2217,6 +2217,163 @@ expression: result }, ), ), + Target( + QuotaStatsOsd( + TargetStat { + kind: Mdt, + param: Param( + "quota_slave.acct_group", + ), + target: Target( + "testfs-MDT0000", + ), + value: QuotaStatsOsd { + kind: Grp, + stats: [ + QuotaStatOsd { + id: 0, + usage: QuotaStatUsage { + inodes: 396, + kbytes: 529400, + }, + }, + ], + }, + }, + ), + ), + Target( + QuotaStatsOsd( + TargetStat { + kind: Mdt, + param: Param( + "quota_slave.acct_project", + ), + target: Target( + "testfs-MDT0000", + ), + value: QuotaStatsOsd { + kind: Prj, + stats: [ + QuotaStatOsd { + id: 0, + usage: QuotaStatUsage { + inodes: 393, + kbytes: 529396, + }, + }, + QuotaStatOsd { + id: 1337, + usage: QuotaStatUsage { + inodes: 3, + kbytes: 4, + }, + }, + ], + }, + }, + ), + ), + Target( + QuotaStatsOsd( + TargetStat { + kind: Mdt, + param: Param( + "quota_slave.acct_user", + ), + target: Target( + "testfs-MDT0000", + ), + value: QuotaStatsOsd { + kind: Usr, + stats: [ + QuotaStatOsd { + id: 0, + usage: QuotaStatUsage { + inodes: 396, + kbytes: 529400, + }, + }, + ], + }, + }, + ), + ), + Target( + QuotaStatsOsd( + TargetStat { + kind: Mdt, + param: Param( + "quota_slave.acct_group", + ), + target: Target( + "testfs-MDT0001", + ), + value: QuotaStatsOsd { + kind: Grp, + stats: [ + QuotaStatOsd { + id: 0, + usage: QuotaStatUsage { + inodes: 265, + kbytes: 356672, + }, + }, + ], + }, + }, + ), + ), + Target( + QuotaStatsOsd( + TargetStat { + kind: Mdt, + param: Param( + "quota_slave.acct_project", + ), + target: Target( + "testfs-MDT0001", + ), + value: QuotaStatsOsd { + kind: Prj, + stats: [ + QuotaStatOsd { + id: 0, + usage: QuotaStatUsage { + inodes: 265, + kbytes: 356672, + }, + }, + ], + }, + }, + ), + ), + Target( + QuotaStatsOsd( + TargetStat { + kind: Mdt, + param: Param( + "quota_slave.acct_user", + ), + target: Target( + "testfs-MDT0001", + ), + value: QuotaStatsOsd { + kind: Usr, + stats: [ + QuotaStatOsd { + id: 0, + usage: QuotaStatUsage { + inodes: 265, + kbytes: 356672, + }, + }, + ], + }, + }, + ), + ), Target( ResourceCount( TargetStat { diff --git a/src/stats_parser.rs b/src/stats_parser.rs index e359d16..e9409c0 100644 --- a/src/stats_parser.rs +++ b/src/stats_parser.rs @@ -5,7 +5,7 @@ use crate::{ base_parsers::{digits, not_words, word}, ldlm::LDLM, - qmt::QMT, + quota::QMT, time::time_triple, types::Stat, }; diff --git a/src/types.rs b/src/types.rs index 01606cb..0f47b67 100644 --- a/src/types.rs +++ b/src/types.rs @@ -506,6 +506,7 @@ pub enum TargetStats { ThreadsStarted(TargetStat), RecoveryStatus(TargetStat), QuotaStats(TargetQuotaStat), + QuotaStatsOsd(TargetStat), } #[derive(PartialEq, Eq, Debug, serde::Serialize, serde::Deserialize)] @@ -541,18 +542,36 @@ pub struct QuotaStatLimits { pub time: u64, } +#[derive(PartialEq, Eq, Debug, serde::Serialize, serde::Deserialize)] +pub struct QuotaStatUsage { + pub inodes: u64, + pub kbytes: u64, +} + #[derive(PartialEq, Eq, Debug, serde::Serialize, serde::Deserialize)] pub struct QuotaStat { pub id: u64, pub limits: QuotaStatLimits, } +#[derive(PartialEq, Eq, Debug, serde::Serialize, serde::Deserialize)] +pub struct QuotaStatOsd { + pub id: u64, + pub usage: QuotaStatUsage, +} + #[derive(PartialEq, Eq, Debug, serde::Serialize, serde::Deserialize)] pub struct QuotaStats { pub kind: QuotaKind, pub stats: Vec, } +#[derive(PartialEq, Eq, Debug, serde::Serialize, serde::Deserialize)] +pub struct QuotaStatsOsd { + pub kind: QuotaKind, + pub stats: Vec, +} + #[derive(PartialEq, Eq, Debug, serde::Serialize, serde::Deserialize)] pub enum QuotaKind { Usr,