From e179a25bcfb2fa175b068f3c96a587a6970b4466 Mon Sep 17 00:00:00 2001 From: Brian Balser Date: Wed, 26 Jun 2024 16:46:22 -0400 Subject: [PATCH 01/23] Update mobile ingest to support sp_boosted_rewards_ban_radio messages --- Cargo.lock | 52 +++++++++++++++++----------- Cargo.toml | 4 +-- file_store/src/file_info.rs | 11 ++++++ file_store/src/traits/msg_verify.rs | 2 ++ ingest/src/server_mobile.rs | 53 +++++++++++++++++++++++++++-- 5 files changed, 97 insertions(+), 25 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ab8eefdee..cd5a87395 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1617,17 +1617,17 @@ checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" [[package]] name = "beacon" version = "0.1.0" -source = "git+https://github.com/helium/proto?branch=master#fdb83a38fbe8aead4ff6cb39d1e996f0ad0646b7" +source = "git+https://github.com/helium/proto?branch=bbalser/hip-125#aa27cf79e0ed886bb138253263297a36741f3975" dependencies = [ "base64 0.21.7", "byteorder", - "helium-proto", + "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=bbalser/hip-125)", "prost", "rand 0.8.5", "rand_chacha 0.3.0", "rust_decimal", "serde", - "sha2 0.10.8", + "sha2 0.9.9", "thiserror", ] @@ -1776,7 +1776,7 @@ dependencies = [ "futures", "futures-util", "helium-crypto", - "helium-proto", + "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=bbalser/hip-125)", "http 0.2.11", "http-serde", "humantime-serde", @@ -2618,7 +2618,7 @@ dependencies = [ "axum 0.7.4", "bs58 0.4.0", "helium-crypto", - "helium-proto", + "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=bbalser/hip-125)", "http 0.2.11", "notify", "serde", @@ -3200,7 +3200,7 @@ dependencies = [ "futures-util", "h3o", "helium-crypto", - "helium-proto", + "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=bbalser/hip-125)", "hex-literal", "http 0.2.11", "lazy_static", @@ -3776,7 +3776,7 @@ dependencies = [ "h3o", "helium-anchor-gen", "helium-crypto", - "helium-proto", + "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=master)", "hex", "itertools", "jsonrpc_client", @@ -3801,7 +3801,7 @@ dependencies = [ [[package]] name = "helium-proto" version = "0.1.0" -source = "git+https://github.com/helium/proto?branch=master#fdb83a38fbe8aead4ff6cb39d1e996f0ad0646b7" +source = "git+https://github.com/helium/proto?branch=bbalser/hip-125#aa27cf79e0ed886bb138253263297a36741f3975" dependencies = [ "bytes", "prost", @@ -3812,6 +3812,18 @@ dependencies = [ "tonic-build", ] +[[package]] +name = "helium-proto" +version = "0.1.0" +source = "git+https://github.com/helium/proto?branch=master#fdb83a38fbe8aead4ff6cb39d1e996f0ad0646b7" +dependencies = [ + "bytes", + "prost", + "prost-build", + "serde", + "serde_json", +] + [[package]] name = "helium-sub-daos" version = "0.1.5" @@ -3853,7 +3865,7 @@ dependencies = [ "async-trait", "chrono", "derive_builder", - "helium-proto", + "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=bbalser/hip-125)", "hextree", "rust_decimal", "rust_decimal_macros", @@ -4269,7 +4281,7 @@ dependencies = [ "futures", "futures-util", "helium-crypto", - "helium-proto", + "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=bbalser/hip-125)", "http 0.2.11", "humantime-serde", "metrics", @@ -4338,7 +4350,7 @@ dependencies = [ "futures", "futures-util", "helium-crypto", - "helium-proto", + "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=bbalser/hip-125)", "hextree", "http 0.2.11", "http-serde", @@ -4380,7 +4392,7 @@ dependencies = [ "futures", "futures-util", "helium-crypto", - "helium-proto", + "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=bbalser/hip-125)", "http 0.2.11", "http-serde", "humantime-serde", @@ -4422,7 +4434,7 @@ dependencies = [ "futures-util", "h3o", "helium-crypto", - "helium-proto", + "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=bbalser/hip-125)", "http-serde", "humantime-serde", "iot-config", @@ -5010,7 +5022,7 @@ dependencies = [ "futures", "futures-util", "helium-crypto", - "helium-proto", + "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=bbalser/hip-125)", "hextree", "http 0.2.11", "http-serde", @@ -5050,7 +5062,7 @@ dependencies = [ "futures", "h3o", "helium-crypto", - "helium-proto", + "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=bbalser/hip-125)", "mobile-config", "prost", "rand 0.8.5", @@ -5086,7 +5098,7 @@ dependencies = [ "futures", "futures-util", "helium-crypto", - "helium-proto", + "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=bbalser/hip-125)", "http 0.2.11", "http-serde", "humantime-serde", @@ -5130,7 +5142,7 @@ dependencies = [ "futures-util", "h3o", "helium-crypto", - "helium-proto", + "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=bbalser/hip-125)", "hex-assignments", "hextree", "http-serde", @@ -5813,7 +5825,7 @@ dependencies = [ "futures", "futures-util", "helium-crypto", - "helium-proto", + "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=bbalser/hip-125)", "http 0.2.11", "hyper 0.14.28", "jsonrpsee", @@ -5896,7 +5908,7 @@ dependencies = [ "futures-util", "helium-anchor-gen", "helium-lib", - "helium-proto", + "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=bbalser/hip-125)", "humantime-serde", "metrics", "metrics-exporter-prometheus", @@ -6503,7 +6515,7 @@ dependencies = [ "futures", "futures-util", "helium-crypto", - "helium-proto", + "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=bbalser/hip-125)", "humantime-serde", "lazy_static", "metrics", diff --git a/Cargo.toml b/Cargo.toml index 870fd79e0..87977c4f7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -70,7 +70,7 @@ helium-lib = { git = "https://github.com/helium/helium-wallet-rs.git", branch = hextree = { git = "https://github.com/jaykickliter/HexTree", branch = "main", features = [ "disktree", ] } -helium-proto = { git = "https://github.com/helium/proto", branch = "master", features = [ +helium-proto = { git = "https://github.com/helium/proto", branch = "bbalser/hip-125", features = [ "services", ] } solana-client = "1.18" @@ -82,7 +82,7 @@ reqwest = { version = "0", default-features = false, features = [ "json", "rustls-tls", ] } -beacon = { git = "https://github.com/helium/proto", branch = "master" } +beacon = { git = "https://github.com/helium/proto", branch = "bbalser/hip-125" } humantime = "2" humantime-serde = "1" metrics = ">=0.22" diff --git a/file_store/src/file_info.rs b/file_store/src/file_info.rs index 1494af783..7addfae61 100644 --- a/file_store/src/file_info.rs +++ b/file_store/src/file_info.rs @@ -157,6 +157,7 @@ pub const ORACLE_BOOSTING_REPORT: &str = "oracle_boosting_report"; pub const URBANIZATION_DATA_SET: &str = "urbanization"; pub const FOOTFALL_DATA_SET: &str = "footfall"; pub const LANDTYPE_DATA_SET: &str = "landtype"; +pub const SP_BOOSTED_REWARDS_BANNED_RADIO: &str = "service_provider_boosted_rewards_banned_radio"; #[derive(Debug, PartialEq, Eq, Clone, Serialize, Copy, strum::EnumCount)] #[serde(rename_all = "snake_case")] @@ -209,6 +210,7 @@ pub enum FileType { InvalidatedRadioThresholdReq, InvalidatedRadioThresholdIngestReport, VerifiedInvalidatedRadioThresholdIngestReport, + ServiceProviderBoostedRewardsBannedRadioIngestReport, } impl fmt::Display for FileType { @@ -270,6 +272,9 @@ impl fmt::Display for FileType { Self::UrbanizationDataSet => URBANIZATION_DATA_SET, Self::FootfallDataSet => FOOTFALL_DATA_SET, Self::LandtypeDataSet => LANDTYPE_DATA_SET, + Self::ServiceProviderBoostedRewardsBannedRadioIngestReport => { + SP_BOOSTED_REWARDS_BANNED_RADIO + } }; f.write_str(s) } @@ -334,6 +339,9 @@ impl FileType { Self::UrbanizationDataSet => URBANIZATION_DATA_SET, Self::FootfallDataSet => FOOTFALL_DATA_SET, Self::LandtypeDataSet => LANDTYPE_DATA_SET, + Self::ServiceProviderBoostedRewardsBannedRadioIngestReport => { + SP_BOOSTED_REWARDS_BANNED_RADIO + } } } } @@ -398,6 +406,9 @@ impl FromStr for FileType { URBANIZATION_DATA_SET => Self::UrbanizationDataSet, FOOTFALL_DATA_SET => Self::FootfallDataSet, LANDTYPE_DATA_SET => Self::LandtypeDataSet, + SP_BOOSTED_REWARDS_BANNED_RADIO => { + Self::ServiceProviderBoostedRewardsBannedRadioIngestReport + } _ => return Err(Error::from(io::Error::from(io::ErrorKind::InvalidInput))), }; Ok(result) diff --git a/file_store/src/traits/msg_verify.rs b/file_store/src/traits/msg_verify.rs index 72398ff84..617de377b 100644 --- a/file_store/src/traits/msg_verify.rs +++ b/file_store/src/traits/msg_verify.rs @@ -3,6 +3,7 @@ use helium_crypto::{PublicKey, Verify}; use helium_proto::services::{ iot_config, mobile_config, poc_lora::{LoraBeaconReportReqV1, LoraStreamSessionInitV1, LoraWitnessReportReqV1}, + poc_mobile::ServiceProviderBoostedRewardsBannedRadioReqV1, }; use helium_proto::{ services::poc_mobile::{ @@ -41,6 +42,7 @@ impl_msg_verify!(LoraWitnessReportReqV1, signature); impl_msg_verify!(LoraStreamSessionInitV1, signature); impl_msg_verify!(DataTransferSessionReqV1, signature); impl_msg_verify!(CoverageObjectReqV1, signature); +impl_msg_verify!(ServiceProviderBoostedRewardsBannedRadioReqV1, signature); impl_msg_verify!(iot_config::OrgCreateHeliumReqV1, signature); impl_msg_verify!(iot_config::OrgCreateRoamerReqV1, signature); impl_msg_verify!(iot_config::OrgUpdateReqV1, signature); diff --git a/ingest/src/server_mobile.rs b/ingest/src/server_mobile.rs index 06d57d5d0..13033754e 100644 --- a/ingest/src/server_mobile.rs +++ b/ingest/src/server_mobile.rs @@ -16,9 +16,11 @@ use helium_proto::services::poc_mobile::{ DataTransferSessionIngestReportV1, DataTransferSessionReqV1, DataTransferSessionRespV1, InvalidatedRadioThresholdIngestReportV1, InvalidatedRadioThresholdReportReqV1, InvalidatedRadioThresholdReportRespV1, RadioThresholdIngestReportV1, RadioThresholdReportReqV1, - RadioThresholdReportRespV1, SpeedtestIngestReportV1, SpeedtestReqV1, SpeedtestRespV1, - SubscriberLocationIngestReportV1, SubscriberLocationReqV1, SubscriberLocationRespV1, - WifiHeartbeatIngestReportV1, WifiHeartbeatReqV1, WifiHeartbeatRespV1, + RadioThresholdReportRespV1, ServiceProviderBoostedRewardsBannedRadioIngestReportV1, + ServiceProviderBoostedRewardsBannedRadioReqV1, ServiceProviderBoostedRewardsBannedRadioRespV1, + SpeedtestIngestReportV1, SpeedtestReqV1, SpeedtestRespV1, SubscriberLocationIngestReportV1, + SubscriberLocationReqV1, SubscriberLocationRespV1, WifiHeartbeatIngestReportV1, + WifiHeartbeatReqV1, WifiHeartbeatRespV1, }; use std::{net::SocketAddr, path::Path}; use task_manager::{ManagedTask, TaskManager}; @@ -39,6 +41,7 @@ pub struct GrpcServer { radio_threshold_report_sink: FileSinkClient, invalidated_radio_threshold_report_sink: FileSinkClient, coverage_object_report_sink: FileSinkClient, + sp_boosted_rewards_ban_sink: FileSinkClient, required_network: Network, address: SocketAddr, api_token: MetadataValue, @@ -337,6 +340,34 @@ impl poc_mobile::PocMobile for GrpcServer { let id = timestamp.to_string(); Ok(Response::new(CoverageObjectRespV1 { id })) } + + async fn submit_sp_boosted_rewards_banned_radio( + &self, + request: Request, + ) -> GrpcResult { + let timestamp = Utc::now().timestamp_millis() as u64; + let event = request.into_inner(); + + custom_tracing::record_b58("pub_key", &event.pub_key); + + let report = self + .verify_public_key(event.pub_key.as_ref()) + .and_then(|public_key| self.verify_network(public_key)) + .and_then(|public_key| self.verify_signature(public_key, event)) + .map( + |(_, event)| ServiceProviderBoostedRewardsBannedRadioIngestReportV1 { + received_timestamp: timestamp, + report: Some(event), + }, + )?; + + _ = self.sp_boosted_rewards_ban_sink.write(report, []).await; + + let id = timestamp.to_string(); + Ok(Response::new( + ServiceProviderBoostedRewardsBannedRadioRespV1 { id }, + )) + } } pub async fn grpc_server(settings: &Settings) -> Result<()> { @@ -439,6 +470,20 @@ pub async fn grpc_server(settings: &Settings) -> Result<()> { .create() .await?; + let (sp_boosted_rewards_ban_sink, sp_boosted_rewards_ban_sink_server) = + file_sink::FileSinkBuilder::new( + FileType::ServiceProviderBoostedRewardsBannedRadioIngestReport, + store_base_path, + file_upload.clone(), + concat!( + env!("CARGO_PKG_NAME"), + "_service_provider_boosted_rewards_banned_radio" + ), + ) + .roll_time(settings.roll_time) + .create() + .await?; + let Some(api_token) = settings .token .as_ref() @@ -456,6 +501,7 @@ pub async fn grpc_server(settings: &Settings) -> Result<()> { radio_threshold_report_sink, invalidated_radio_threshold_report_sink, coverage_object_report_sink, + sp_boosted_rewards_ban_sink, required_network: settings.network, address: settings.listen_addr, api_token, @@ -477,6 +523,7 @@ pub async fn grpc_server(settings: &Settings) -> Result<()> { .add_task(radio_threshold_report_sink_server) .add_task(invalidated_radio_threshold_report_sink_server) .add_task(coverage_object_report_sink_server) + .add_task(sp_boosted_rewards_ban_sink_server) .add_task(grpc_server) .build() .start() From de6b64cfc91002619acfa6b979280d6aebd69d56 Mon Sep 17 00:00:00 2001 From: Brian Balser Date: Thu, 27 Jun 2024 10:54:32 -0400 Subject: [PATCH 02/23] Start to sp boosted rewards ban ingestor --- file_store/src/error.rs | 2 +- file_store/src/lib.rs | 1 + file_store/src/sp_boosted_rewards_bans.rs | 78 ++++++++++ .../migrations/34_sp_boosted_rewards_bans.sql | 7 + mobile_verifier/src/cli/server.rs | 12 +- mobile_verifier/src/lib.rs | 1 + .../src/sp_boosted_rewards_bans.rs | 143 ++++++++++++++++++ 7 files changed, 242 insertions(+), 2 deletions(-) create mode 100644 file_store/src/sp_boosted_rewards_bans.rs create mode 100644 mobile_verifier/migrations/34_sp_boosted_rewards_bans.sql create mode 100644 mobile_verifier/src/sp_boosted_rewards_bans.rs diff --git a/file_store/src/error.rs b/file_store/src/error.rs index d2c8a2d55..2725d5bc3 100644 --- a/file_store/src/error.rs +++ b/file_store/src/error.rs @@ -8,7 +8,7 @@ pub enum Error { Io(#[from] std::io::Error), #[error("encode error")] Encode(#[from] EncodeError), - #[error("dencode error")] + #[error("decode error")] Decode(#[from] DecodeError), #[error("not found")] NotFound(String), diff --git a/file_store/src/lib.rs b/file_store/src/lib.rs index 66ab76973..d7577e67e 100644 --- a/file_store/src/lib.rs +++ b/file_store/src/lib.rs @@ -22,6 +22,7 @@ pub mod mobile_subscriber; pub mod mobile_transfer; pub mod reward_manifest; mod settings; +pub mod sp_boosted_rewards_bans; pub mod speedtest; pub mod traits; pub mod wifi_heartbeat; diff --git a/file_store/src/sp_boosted_rewards_bans.rs b/file_store/src/sp_boosted_rewards_bans.rs new file mode 100644 index 000000000..06eb9e3bb --- /dev/null +++ b/file_store/src/sp_boosted_rewards_bans.rs @@ -0,0 +1,78 @@ +use chrono::{DateTime, TimeZone, Utc}; +use helium_crypto::PublicKeyBinary; +use helium_proto::services::poc_mobile as proto; +use serde::{Deserialize, Serialize}; + +use crate::{error::DecodeError, traits::MsgDecode, Error}; + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct ServiceProviderBoostedRewardBannedRadio { + pub pubkey: PublicKeyBinary, + pub key_type: KeyType, + pub reason: proto::service_provider_boosted_rewards_banned_radio_req_v1::SpBoostedRewardsBannedRadioReason, + pub until: DateTime, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub enum KeyType { + CbsdId(String), + HotspotKey(PublicKeyBinary), +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct ServiceProviderBoostedRewardBannedRadioIngest { + pub received_timestamp: DateTime, + pub report: ServiceProviderBoostedRewardBannedRadio, +} + +impl MsgDecode for ServiceProviderBoostedRewardBannedRadio { + type Msg = proto::ServiceProviderBoostedRewardsBannedRadioReqV1; +} + +impl MsgDecode for ServiceProviderBoostedRewardBannedRadioIngest { + type Msg = proto::ServiceProviderBoostedRewardsBannedRadioIngestReportV1; +} + +impl TryFrom + for ServiceProviderBoostedRewardBannedRadio +{ + type Error = Error; + + fn try_from( + value: proto::ServiceProviderBoostedRewardsBannedRadioReqV1, + ) -> Result { + let reason = value.reason(); + + Ok(Self { + pubkey: value.pub_key.into(), + key_type: match value.key_type { + Some(proto::service_provider_boosted_rewards_banned_radio_req_v1::KeyType::CbsdId(cbsd_id)) => KeyType::CbsdId(cbsd_id), + Some(proto::service_provider_boosted_rewards_banned_radio_req_v1::KeyType::HotspotKey(bytes)) => KeyType::HotspotKey(bytes.into()), + None => return Err(Error::NotFound("key_type".to_string())), + }, + reason, + until: Utc.timestamp_opt(value.until as i64, 0).single().ok_or_else(|| DecodeError::invalid_timestamp(value.until))?, + }) + } +} + +impl TryFrom + for ServiceProviderBoostedRewardBannedRadioIngest +{ + type Error = Error; + + fn try_from( + value: proto::ServiceProviderBoostedRewardsBannedRadioIngestReportV1, + ) -> Result { + Ok(Self { + received_timestamp: Utc + .timestamp_millis_opt(value.received_timestamp as i64) + .single() + .ok_or_else(|| DecodeError::invalid_timestamp(value.received_timestamp))?, + report: value + .report + .ok_or_else(|| Error::not_found("report not found"))? + .try_into()?, + }) + } +} diff --git a/mobile_verifier/migrations/34_sp_boosted_rewards_bans.sql b/mobile_verifier/migrations/34_sp_boosted_rewards_bans.sql new file mode 100644 index 000000000..e9d86f2d0 --- /dev/null +++ b/mobile_verifier/migrations/34_sp_boosted_rewards_bans.sql @@ -0,0 +1,7 @@ +CREATE TABLE IF NOT EXISTS sp_boosted_rewards_bans ( + radio_type radio_type NOT NULL, + radio_key TEXT NOT NULL, + received_timestamp TIMESTAMPTZ NOT NULL, + until TIMESTAMPTZ NOT NULL, + PRIMARY KEY (radio_type, radio_key) +); diff --git a/mobile_verifier/src/cli/server.rs b/mobile_verifier/src/cli/server.rs index b6bb453af..7a90a8c21 100644 --- a/mobile_verifier/src/cli/server.rs +++ b/mobile_verifier/src/cli/server.rs @@ -8,6 +8,7 @@ use crate::{ heartbeats::{cbrs::CbrsHeartbeatDaemon, wifi::WifiHeartbeatDaemon}, radio_threshold::RadioThresholdIngestor, rewarder::Rewarder, + sp_boosted_rewards_bans::ServiceProviderBoostedRewardsBanIngestor, speedtests::SpeedtestDaemon, subscriber_location::SubscriberLocationIngestor, telemetry, Settings, @@ -176,12 +177,21 @@ impl Cmd { pool.clone(), settings, file_upload.clone(), + report_ingest.clone(), + auth_client.clone(), + ) + .await?, + ) + .add_task(DataSessionIngestor::create_managed_task(pool.clone(), settings).await?) + .add_task( + ServiceProviderBoostedRewardsBanIngestor::create_managed_task( + pool.clone(), report_ingest, auth_client, + settings.start_after, ) .await?, ) - .add_task(DataSessionIngestor::create_managed_task(pool.clone(), settings).await?) .add_task( Rewarder::create_managed_task( pool, diff --git a/mobile_verifier/src/lib.rs b/mobile_verifier/src/lib.rs index 82353ef52..9a95bccb1 100644 --- a/mobile_verifier/src/lib.rs +++ b/mobile_verifier/src/lib.rs @@ -9,6 +9,7 @@ pub mod radio_threshold; pub mod reward_shares; pub mod rewarder; mod settings; +pub mod sp_boosted_rewards_bans; pub mod speedtests; pub mod speedtests_average; pub mod subscriber_location; diff --git a/mobile_verifier/src/sp_boosted_rewards_bans.rs b/mobile_verifier/src/sp_boosted_rewards_bans.rs new file mode 100644 index 000000000..21bba4fdd --- /dev/null +++ b/mobile_verifier/src/sp_boosted_rewards_bans.rs @@ -0,0 +1,143 @@ +use chrono::{DateTime, Utc}; +use file_store::{ + file_info_poller::{FileInfoStream, LookbackBehavior}, + file_source, + sp_boosted_rewards_bans::{KeyType, ServiceProviderBoostedRewardBannedRadioIngest}, + FileStore, FileType, +}; +use futures::{prelude::future::LocalBoxFuture, StreamExt, TryFutureExt, TryStreamExt}; +use helium_crypto::PublicKeyBinary; +use helium_proto::services::mobile_config::NetworkKeyRole; +use mobile_config::client::authorization_client::AuthorizationVerifier; +use sqlx::{PgPool, Postgres, Transaction}; +use task_manager::{ManagedTask, TaskManager}; +use tokio::sync::mpsc::Receiver; + +pub struct ServiceProviderBoostedRewardsBanIngestor { + pool: PgPool, + authorization_verifier: AV, + receiver: Receiver>, +} + +impl ManagedTask for ServiceProviderBoostedRewardsBanIngestor +where + AV: AuthorizationVerifier + Send + Sync + 'static, + AV::Error: std::error::Error + Send + Sync + 'static, +{ + fn start_task( + self: Box, + shutdown: triggered::Listener, + ) -> LocalBoxFuture<'static, anyhow::Result<()>> { + let handle = tokio::spawn(self.run(shutdown)); + Box::pin( + handle + .map_err(anyhow::Error::from) + .and_then(|result| async move { result.map_err(anyhow::Error::from) }), + ) + } +} + +impl ServiceProviderBoostedRewardsBanIngestor +where + AV: AuthorizationVerifier + Send + Sync + 'static, + AV::Error: std::error::Error + Send + Sync + 'static, +{ + pub async fn create_managed_task( + pool: PgPool, + file_store: FileStore, + authorization_verifier: AV, + start_after: DateTime, + ) -> anyhow::Result { + let (receiver, ingest_server) = + file_source::continuous_source::() + .state(pool.clone()) + .store(file_store) + .lookback(LookbackBehavior::StartAfter(start_after)) + .prefix(FileType::ServiceProviderBoostedRewardsBannedRadioIngestReport.to_string()) + .create() + .await?; + + let ingestor = Self { + pool, + authorization_verifier, + receiver, + }; + + Ok(TaskManager::builder() + .add_task(ingest_server) + .add_task(ingestor) + .build()) + } + + async fn run(mut self, mut shutdown: triggered::Listener) -> anyhow::Result<()> { + tracing::info!("service provider boosted rewards ban ingestor starting"); + loop { + tokio::select! { + biased; + _ = &mut shutdown => break, + Some(file) = self.receiver.recv() => { + self.process_file(file).await?; + } + } + } + tracing::info!("stopping service provider boosted rewards ban ingestor"); + + Ok(()) + } + + async fn process_file( + &self, + file_info_stream: FileInfoStream, + ) -> anyhow::Result<()> { + tracing::info!(file = %file_info_stream.file_info.key, "processing sp boosted rewards ban file"); + let mut transaction = self.pool.begin().await?; + file_info_stream + .into_stream(&mut transaction) + .await? + .map(anyhow::Ok) + .try_fold(transaction, |mut tx, ingest| async move { + if self.is_authorized(&ingest.report.pubkey).await? { + save(&mut tx, ingest).await?; + } + + Ok(tx) + }) + .await? + .commit() + .await?; + + Ok(()) + } + + async fn is_authorized(&self, pubkey: &PublicKeyBinary) -> anyhow::Result { + self.authorization_verifier + .verify_authorized_key(pubkey, NetworkKeyRole::MobileCarrier) + .await + .map_err(anyhow::Error::from) + } +} + +async fn save( + transaction: &mut Transaction<'_, Postgres>, + ingest: ServiceProviderBoostedRewardBannedRadioIngest, +) -> anyhow::Result<()> { + let (radio_type, radio_key) = match ingest.report.key_type { + KeyType::CbsdId(cbsd_id) => ("cbrs", cbsd_id), + KeyType::HotspotKey(pubkey) => ("wifi", pubkey.to_string()), + }; + + sqlx::query( + r#" + INSERT INTO sp_boosted_rewards_bans(radio_type, radio_key, received_timestamp, until) + VALUES($1,$2,$3,$4) + "#, + ) + .bind(radio_type) + .bind(radio_key) + .bind(ingest.received_timestamp) + .bind(ingest.report.until) + .execute(transaction) + .await + .map(|_| ()) + .map_err(anyhow::Error::from) +} From 3645a3737229659b52ea8cd57d89e3a147117bd1 Mon Sep 17 00:00:00 2001 From: Brian Balser Date: Thu, 27 Jun 2024 10:54:38 -0400 Subject: [PATCH 03/23] make clippy happy --- custom_tracing/src/http_layer.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/custom_tracing/src/http_layer.rs b/custom_tracing/src/http_layer.rs index aed429d43..50924c7e8 100644 --- a/custom_tracing/src/http_layer.rs +++ b/custom_tracing/src/http_layer.rs @@ -6,6 +6,7 @@ use tower_http::{ }; use tracing::{Level, Span}; +#[allow(clippy::type_complexity)] pub fn new_with_span( make_span: fn(&Request) -> Span, ) -> TraceLayer< From 00b56e9880201cb9797c114d5dc9b5c07992019f Mon Sep 17 00:00:00 2001 From: Brian Balser Date: Thu, 27 Jun 2024 12:25:34 -0400 Subject: [PATCH 04/23] Simplify structs and add verified file sink --- Cargo.lock | 6 +- file_store/src/file_info.rs | 12 ++ file_store/src/file_info_poller.rs | 39 ++++- file_store/src/file_source.rs | 8 +- file_store/src/lib.rs | 1 - file_store/src/sp_boosted_rewards_bans.rs | 78 ---------- mobile_verifier/src/cli/server.rs | 3 +- .../src/sp_boosted_rewards_bans.rs | 142 ++++++++++++++---- 8 files changed, 172 insertions(+), 117 deletions(-) delete mode 100644 file_store/src/sp_boosted_rewards_bans.rs diff --git a/Cargo.lock b/Cargo.lock index cd5a87395..b5d09e78e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1617,7 +1617,7 @@ checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" [[package]] name = "beacon" version = "0.1.0" -source = "git+https://github.com/helium/proto?branch=bbalser/hip-125#aa27cf79e0ed886bb138253263297a36741f3975" +source = "git+https://github.com/helium/proto?branch=bbalser/hip-125#64813bee906724e9ac3abccb2ffcce4bd677f6a6" dependencies = [ "base64 0.21.7", "byteorder", @@ -3801,7 +3801,7 @@ dependencies = [ [[package]] name = "helium-proto" version = "0.1.0" -source = "git+https://github.com/helium/proto?branch=bbalser/hip-125#aa27cf79e0ed886bb138253263297a36741f3975" +source = "git+https://github.com/helium/proto?branch=bbalser/hip-125#64813bee906724e9ac3abccb2ffcce4bd677f6a6" dependencies = [ "bytes", "prost", @@ -9863,7 +9863,7 @@ dependencies = [ "rand 0.8.5", "serde", "serde_json", - "sha2 0.10.8", + "sha2 0.9.9", "thiserror", "twox-hash", "xorf", diff --git a/file_store/src/file_info.rs b/file_store/src/file_info.rs index 7addfae61..4eaab2d0e 100644 --- a/file_store/src/file_info.rs +++ b/file_store/src/file_info.rs @@ -158,6 +158,8 @@ pub const URBANIZATION_DATA_SET: &str = "urbanization"; pub const FOOTFALL_DATA_SET: &str = "footfall"; pub const LANDTYPE_DATA_SET: &str = "landtype"; pub const SP_BOOSTED_REWARDS_BANNED_RADIO: &str = "service_provider_boosted_rewards_banned_radio"; +pub const VERIFIED_SP_BOOSTED_REWARDS_BANNED_RADIO: &str = + "verified_service_provider_boosted_rewards_banned_radio"; #[derive(Debug, PartialEq, Eq, Clone, Serialize, Copy, strum::EnumCount)] #[serde(rename_all = "snake_case")] @@ -211,6 +213,7 @@ pub enum FileType { InvalidatedRadioThresholdIngestReport, VerifiedInvalidatedRadioThresholdIngestReport, ServiceProviderBoostedRewardsBannedRadioIngestReport, + VerifiedServiceProviderBoostedRewardsBannedRadioIngestReport, } impl fmt::Display for FileType { @@ -275,6 +278,9 @@ impl fmt::Display for FileType { Self::ServiceProviderBoostedRewardsBannedRadioIngestReport => { SP_BOOSTED_REWARDS_BANNED_RADIO } + Self::VerifiedServiceProviderBoostedRewardsBannedRadioIngestReport => { + VERIFIED_SP_BOOSTED_REWARDS_BANNED_RADIO + } }; f.write_str(s) } @@ -342,6 +348,9 @@ impl FileType { Self::ServiceProviderBoostedRewardsBannedRadioIngestReport => { SP_BOOSTED_REWARDS_BANNED_RADIO } + Self::VerifiedServiceProviderBoostedRewardsBannedRadioIngestReport => { + VERIFIED_SP_BOOSTED_REWARDS_BANNED_RADIO + } } } } @@ -409,6 +418,9 @@ impl FromStr for FileType { SP_BOOSTED_REWARDS_BANNED_RADIO => { Self::ServiceProviderBoostedRewardsBannedRadioIngestReport } + VERIFIED_SP_BOOSTED_REWARDS_BANNED_RADIO => { + Self::VerifiedServiceProviderBoostedRewardsBannedRadioIngestReport + } _ => return Err(Error::from(io::Error::from(io::ErrorKind::InvalidInput))), }; Ok(result) diff --git a/file_store/src/file_info_poller.rs b/file_store/src/file_info_poller.rs index a7bf6950a..7d80183bf 100644 --- a/file_store/src/file_info_poller.rs +++ b/file_store/src/file_info_poller.rs @@ -277,10 +277,10 @@ where } } -pub struct ProtoFileInfoPollerParser; +pub struct MsgDecodeFileInfoPollerParser; #[async_trait::async_trait] -impl FileInfoPollerParser for ProtoFileInfoPollerParser +impl FileInfoPollerParser for MsgDecodeFileInfoPollerParser where T: MsgDecode + TryFrom + Send + Sync + 'static, { @@ -312,6 +312,41 @@ where } } +pub struct ProstFileInfoPollerParser; + +#[async_trait::async_trait] +impl FileInfoPollerParser for ProstFileInfoPollerParser +where + T: helium_proto::Message + Default, +{ + async fn parse(&self, byte_stream: ByteStream) -> Result> { + Ok(file_store::stream_source(byte_stream) + .filter_map(|msg| async { + msg.map_err(|err| { + tracing::error!( + "Error streaming entry in file of type {}: {err:?}", + std::any::type_name::() + ); + err + }) + .ok() + }) + .filter_map(|msg| async { + ::decode(msg) + .map_err(|err| { + tracing::error!( + "Error in decoding message of type {}: {err:?}", + std::any::type_name::() + ); + err + }) + .ok() + }) + .collect() + .await) + } +} + fn create_cache() -> MemoryFileCache { Arc::new(Cache::new()) } diff --git a/file_store/src/file_source.rs b/file_store/src/file_source.rs index 8290d75ab..ff77f1162 100644 --- a/file_store/src/file_source.rs +++ b/file_store/src/file_source.rs @@ -1,5 +1,5 @@ use crate::{ - file_info_poller::{FileInfoPollerConfigBuilder, ProtoFileInfoPollerParser}, + file_info_poller::{FileInfoPollerConfigBuilder, MsgDecodeFileInfoPollerParser}, file_sink, BytesMutStream, Error, }; use async_compression::tokio::bufread::GzipDecoder; @@ -11,12 +11,12 @@ use std::path::{Path, PathBuf}; use tokio::{fs::File, io::BufReader}; use tokio_util::codec::{length_delimited::LengthDelimitedCodec, FramedRead}; -pub fn continuous_source() -> FileInfoPollerConfigBuilder +pub fn continuous_source() -> FileInfoPollerConfigBuilder where T: Clone, { - FileInfoPollerConfigBuilder::::default() - .parser(ProtoFileInfoPollerParser) + FileInfoPollerConfigBuilder::::default() + .parser(MsgDecodeFileInfoPollerParser) } pub fn source(paths: I) -> BytesMutStream diff --git a/file_store/src/lib.rs b/file_store/src/lib.rs index d7577e67e..66ab76973 100644 --- a/file_store/src/lib.rs +++ b/file_store/src/lib.rs @@ -22,7 +22,6 @@ pub mod mobile_subscriber; pub mod mobile_transfer; pub mod reward_manifest; mod settings; -pub mod sp_boosted_rewards_bans; pub mod speedtest; pub mod traits; pub mod wifi_heartbeat; diff --git a/file_store/src/sp_boosted_rewards_bans.rs b/file_store/src/sp_boosted_rewards_bans.rs deleted file mode 100644 index 06eb9e3bb..000000000 --- a/file_store/src/sp_boosted_rewards_bans.rs +++ /dev/null @@ -1,78 +0,0 @@ -use chrono::{DateTime, TimeZone, Utc}; -use helium_crypto::PublicKeyBinary; -use helium_proto::services::poc_mobile as proto; -use serde::{Deserialize, Serialize}; - -use crate::{error::DecodeError, traits::MsgDecode, Error}; - -#[derive(Serialize, Deserialize, Debug, Clone)] -pub struct ServiceProviderBoostedRewardBannedRadio { - pub pubkey: PublicKeyBinary, - pub key_type: KeyType, - pub reason: proto::service_provider_boosted_rewards_banned_radio_req_v1::SpBoostedRewardsBannedRadioReason, - pub until: DateTime, -} - -#[derive(Serialize, Deserialize, Debug, Clone)] -pub enum KeyType { - CbsdId(String), - HotspotKey(PublicKeyBinary), -} - -#[derive(Serialize, Deserialize, Debug, Clone)] -pub struct ServiceProviderBoostedRewardBannedRadioIngest { - pub received_timestamp: DateTime, - pub report: ServiceProviderBoostedRewardBannedRadio, -} - -impl MsgDecode for ServiceProviderBoostedRewardBannedRadio { - type Msg = proto::ServiceProviderBoostedRewardsBannedRadioReqV1; -} - -impl MsgDecode for ServiceProviderBoostedRewardBannedRadioIngest { - type Msg = proto::ServiceProviderBoostedRewardsBannedRadioIngestReportV1; -} - -impl TryFrom - for ServiceProviderBoostedRewardBannedRadio -{ - type Error = Error; - - fn try_from( - value: proto::ServiceProviderBoostedRewardsBannedRadioReqV1, - ) -> Result { - let reason = value.reason(); - - Ok(Self { - pubkey: value.pub_key.into(), - key_type: match value.key_type { - Some(proto::service_provider_boosted_rewards_banned_radio_req_v1::KeyType::CbsdId(cbsd_id)) => KeyType::CbsdId(cbsd_id), - Some(proto::service_provider_boosted_rewards_banned_radio_req_v1::KeyType::HotspotKey(bytes)) => KeyType::HotspotKey(bytes.into()), - None => return Err(Error::NotFound("key_type".to_string())), - }, - reason, - until: Utc.timestamp_opt(value.until as i64, 0).single().ok_or_else(|| DecodeError::invalid_timestamp(value.until))?, - }) - } -} - -impl TryFrom - for ServiceProviderBoostedRewardBannedRadioIngest -{ - type Error = Error; - - fn try_from( - value: proto::ServiceProviderBoostedRewardsBannedRadioIngestReportV1, - ) -> Result { - Ok(Self { - received_timestamp: Utc - .timestamp_millis_opt(value.received_timestamp as i64) - .single() - .ok_or_else(|| DecodeError::invalid_timestamp(value.received_timestamp))?, - report: value - .report - .ok_or_else(|| Error::not_found("report not found"))? - .try_into()?, - }) - } -} diff --git a/mobile_verifier/src/cli/server.rs b/mobile_verifier/src/cli/server.rs index 7a90a8c21..dd32b1a77 100644 --- a/mobile_verifier/src/cli/server.rs +++ b/mobile_verifier/src/cli/server.rs @@ -186,9 +186,10 @@ impl Cmd { .add_task( ServiceProviderBoostedRewardsBanIngestor::create_managed_task( pool.clone(), + file_upload.clone(), report_ingest, auth_client, - settings.start_after, + settings, ) .await?, ) diff --git a/mobile_verifier/src/sp_boosted_rewards_bans.rs b/mobile_verifier/src/sp_boosted_rewards_bans.rs index 21bba4fdd..ea2fa854d 100644 --- a/mobile_verifier/src/sp_boosted_rewards_bans.rs +++ b/mobile_verifier/src/sp_boosted_rewards_bans.rs @@ -1,22 +1,77 @@ -use chrono::{DateTime, Utc}; +use chrono::{DateTime, TimeZone, Utc}; use file_store::{ - file_info_poller::{FileInfoStream, LookbackBehavior}, - file_source, - sp_boosted_rewards_bans::{KeyType, ServiceProviderBoostedRewardBannedRadioIngest}, + file_info_poller::{ + FileInfoPollerConfigBuilder, FileInfoStream, LookbackBehavior, ProstFileInfoPollerParser, + }, + file_sink::{self, FileSinkClient}, + file_upload::FileUpload, FileStore, FileType, }; use futures::{prelude::future::LocalBoxFuture, StreamExt, TryFutureExt, TryStreamExt}; use helium_crypto::PublicKeyBinary; -use helium_proto::services::mobile_config::NetworkKeyRole; +use helium_proto::services::{ + mobile_config::NetworkKeyRole, + poc_mobile::{ + service_provider_boosted_rewards_banned_radio_req_v1::KeyType, + ServiceProviderBoostedRewardsBannedRadioIngestReportV1, + ServiceProviderBoostedRewardsBannedRadioVerificationStatus, + VerifiedServiceProviderBoostedRewardsBannedRadioIngestReportV1, + }, +}; use mobile_config::client::authorization_client::AuthorizationVerifier; use sqlx::{PgPool, Postgres, Transaction}; use task_manager::{ManagedTask, TaskManager}; use tokio::sync::mpsc::Receiver; +use crate::Settings; + +struct BannedRadioReport { + received_timestamp: DateTime, + pubkey: PublicKeyBinary, + radio_type: String, + radio_key: String, + until: DateTime, +} + +impl TryFrom for BannedRadioReport { + type Error = anyhow::Error; + + fn try_from( + value: ServiceProviderBoostedRewardsBannedRadioIngestReportV1, + ) -> Result { + let report = value + .report + .ok_or_else(|| anyhow::anyhow!("invalid ingest report"))?; + + let (radio_type, radio_key) = match report.key_type { + Some(KeyType::CbsdId(cbsd_id)) => ("cbrs", cbsd_id), + Some(KeyType::HotspotKey(bytes)) => ("wifi", PublicKeyBinary::from(bytes).to_string()), + None => anyhow::bail!("Invalid keytype"), + }; + + Ok(Self { + received_timestamp: Utc + .timestamp_millis_opt(value.received_timestamp as i64) + .single() + .ok_or_else(|| { + anyhow::anyhow!("invalid received timestamp, {}", value.received_timestamp) + })?, + pubkey: report.pub_key.into(), + radio_type: radio_type.to_string(), + radio_key, + until: Utc + .timestamp_opt(report.until as i64, 0) + .single() + .ok_or_else(|| anyhow::anyhow!("invalid until: {}", report.until))?, + }) + } +} + pub struct ServiceProviderBoostedRewardsBanIngestor { pool: PgPool, authorization_verifier: AV, - receiver: Receiver>, + receiver: Receiver>, + verified_sink: FileSinkClient, } impl ManagedTask for ServiceProviderBoostedRewardsBanIngestor @@ -44,26 +99,43 @@ where { pub async fn create_managed_task( pool: PgPool, + file_upload: FileUpload, file_store: FileStore, authorization_verifier: AV, - start_after: DateTime, + settings: &Settings, ) -> anyhow::Result { - let (receiver, ingest_server) = - file_source::continuous_source::() - .state(pool.clone()) - .store(file_store) - .lookback(LookbackBehavior::StartAfter(start_after)) - .prefix(FileType::ServiceProviderBoostedRewardsBannedRadioIngestReport.to_string()) - .create() - .await?; + let (verified_sink, verified_sink_server) = file_sink::FileSinkBuilder::new( + FileType::VerifiedServiceProviderBoostedRewardsBannedRadioIngestReport, + settings.store_base_path(), + file_upload, + concat!(env!("CARGO_PKG_NAME"), "_verified_sp_boosted_rewards_ban"), + ) + .auto_commit(false) + .create() + .await?; + + let (receiver, ingest_server) = FileInfoPollerConfigBuilder::< + ServiceProviderBoostedRewardsBannedRadioIngestReportV1, + _, + _, + >::default() + .parser(ProstFileInfoPollerParser) + .state(pool.clone()) + .store(file_store) + .lookback(LookbackBehavior::StartAfter(settings.start_after)) + .prefix(FileType::ServiceProviderBoostedRewardsBannedRadioIngestReport.to_string()) + .create() + .await?; let ingestor = Self { pool, authorization_verifier, receiver, + verified_sink, }; Ok(TaskManager::builder() + .add_task(verified_sink_server) .add_task(ingest_server) .add_task(ingestor) .build()) @@ -87,7 +159,7 @@ where async fn process_file( &self, - file_info_stream: FileInfoStream, + file_info_stream: FileInfoStream, ) -> anyhow::Result<()> { tracing::info!(file = %file_info_stream.file_info.key, "processing sp boosted rewards ban file"); let mut transaction = self.pool.begin().await?; @@ -96,16 +168,35 @@ where .await? .map(anyhow::Ok) .try_fold(transaction, |mut tx, ingest| async move { - if self.is_authorized(&ingest.report.pubkey).await? { - save(&mut tx, ingest).await?; + let report = BannedRadioReport::try_from(ingest.clone())?; + let is_authorized = self.is_authorized(&report.pubkey).await?; + + if is_authorized { + save(&mut tx, report).await?; } + let status = match is_authorized { + true => ServiceProviderBoostedRewardsBannedRadioVerificationStatus::SpBoostedRewardsBanValid, + false => ServiceProviderBoostedRewardsBannedRadioVerificationStatus::SpBoostedRewardsBanInvalidCarrierKey, + }; + + let verified_report = + VerifiedServiceProviderBoostedRewardsBannedRadioIngestReportV1 { + report: Some(ingest), + status: status.into(), + timestamp: Utc::now().timestamp_millis() as u64, + }; + + self.verified_sink.write(verified_report, &[("status", status.as_str_name())]).await?; + Ok(tx) }) .await? .commit() .await?; + self.verified_sink.commit().await?; + Ok(()) } @@ -119,23 +210,18 @@ where async fn save( transaction: &mut Transaction<'_, Postgres>, - ingest: ServiceProviderBoostedRewardBannedRadioIngest, + report: BannedRadioReport, ) -> anyhow::Result<()> { - let (radio_type, radio_key) = match ingest.report.key_type { - KeyType::CbsdId(cbsd_id) => ("cbrs", cbsd_id), - KeyType::HotspotKey(pubkey) => ("wifi", pubkey.to_string()), - }; - sqlx::query( r#" INSERT INTO sp_boosted_rewards_bans(radio_type, radio_key, received_timestamp, until) VALUES($1,$2,$3,$4) "#, ) - .bind(radio_type) - .bind(radio_key) - .bind(ingest.received_timestamp) - .bind(ingest.report.until) + .bind(report.radio_type) + .bind(report.radio_key) + .bind(report.received_timestamp) + .bind(report.until) .execute(transaction) .await .map(|_| ()) From a45e673f3e44791686c92750e8d9f8c51b44c719 Mon Sep 17 00:00:00 2001 From: Brian Balser Date: Thu, 27 Jun 2024 14:06:49 -0400 Subject: [PATCH 05/23] Add cleanup task to remove old data from db table --- .../migrations/34_sp_boosted_rewards_bans.sql | 1 + .../src/sp_boosted_rewards_bans.rs | 152 +++++++++++++----- 2 files changed, 117 insertions(+), 36 deletions(-) diff --git a/mobile_verifier/migrations/34_sp_boosted_rewards_bans.sql b/mobile_verifier/migrations/34_sp_boosted_rewards_bans.sql index e9d86f2d0..c8b768af2 100644 --- a/mobile_verifier/migrations/34_sp_boosted_rewards_bans.sql +++ b/mobile_verifier/migrations/34_sp_boosted_rewards_bans.sql @@ -3,5 +3,6 @@ CREATE TABLE IF NOT EXISTS sp_boosted_rewards_bans ( radio_key TEXT NOT NULL, received_timestamp TIMESTAMPTZ NOT NULL, until TIMESTAMPTZ NOT NULL, + invalidated_at TIMESTAMPTZ, PRIMARY KEY (radio_type, radio_key) ); diff --git a/mobile_verifier/src/sp_boosted_rewards_bans.rs b/mobile_verifier/src/sp_boosted_rewards_bans.rs index ea2fa854d..4355a6d00 100644 --- a/mobile_verifier/src/sp_boosted_rewards_bans.rs +++ b/mobile_verifier/src/sp_boosted_rewards_bans.rs @@ -12,7 +12,9 @@ use helium_crypto::PublicKeyBinary; use helium_proto::services::{ mobile_config::NetworkKeyRole, poc_mobile::{ - service_provider_boosted_rewards_banned_radio_req_v1::KeyType, + service_provider_boosted_rewards_banned_radio_req_v1::{ + KeyType, SpBoostedRewardsBannedRadioReason, + }, ServiceProviderBoostedRewardsBannedRadioIngestReportV1, ServiceProviderBoostedRewardsBannedRadioVerificationStatus, VerifiedServiceProviderBoostedRewardsBannedRadioIngestReportV1, @@ -25,12 +27,16 @@ use tokio::sync::mpsc::Receiver; use crate::Settings; +const CLEANUP_DAYS: i64 = 7; +const CLEANUP_INTERVAL_SECS: u64 = 12 * 60 * 60; + struct BannedRadioReport { received_timestamp: DateTime, pubkey: PublicKeyBinary, radio_type: String, radio_key: String, until: DateTime, + reason: SpBoostedRewardsBannedRadioReason, } impl TryFrom for BannedRadioReport { @@ -43,6 +49,8 @@ impl TryFrom for BannedR .report .ok_or_else(|| anyhow::anyhow!("invalid ingest report"))?; + let reason = report.reason(); + let (radio_type, radio_key) = match report.key_type { Some(KeyType::CbsdId(cbsd_id)) => ("cbrs", cbsd_id), Some(KeyType::HotspotKey(bytes)) => ("wifi", PublicKeyBinary::from(bytes).to_string()), @@ -63,6 +71,7 @@ impl TryFrom for BannedR .timestamp_opt(report.until as i64, 0) .single() .ok_or_else(|| anyhow::anyhow!("invalid until: {}", report.until))?, + reason, }) } } @@ -143,10 +152,17 @@ where async fn run(mut self, mut shutdown: triggered::Listener) -> anyhow::Result<()> { tracing::info!("service provider boosted rewards ban ingestor starting"); + + let mut cleanup_interval = + tokio::time::interval(std::time::Duration::from_secs(CLEANUP_INTERVAL_SECS)); + loop { tokio::select! { biased; _ = &mut shutdown => break, + _ = cleanup_interval.tick() => { + db::cleanup(&self.pool).await?; + } Some(file) = self.receiver.recv() => { self.process_file(file).await?; } @@ -168,35 +184,45 @@ where .await? .map(anyhow::Ok) .try_fold(transaction, |mut tx, ingest| async move { - let report = BannedRadioReport::try_from(ingest.clone())?; - let is_authorized = self.is_authorized(&report.pubkey).await?; + self.process_ingest_report(&mut tx, ingest).await?; + Ok(tx) + }) + .await? + .commit() + .await?; - if is_authorized { - save(&mut tx, report).await?; - } + self.verified_sink.commit().await?; - let status = match is_authorized { + Ok(()) + } + + async fn process_ingest_report( + &self, + transaction: &mut Transaction<'_, Postgres>, + ingest: ServiceProviderBoostedRewardsBannedRadioIngestReportV1, + ) -> anyhow::Result<()> { + let report = BannedRadioReport::try_from(ingest.clone())?; + let is_authorized = self.is_authorized(&report.pubkey).await?; + + if is_authorized { + db::update_report(transaction, report).await?; + } + + let status = match is_authorized { true => ServiceProviderBoostedRewardsBannedRadioVerificationStatus::SpBoostedRewardsBanValid, false => ServiceProviderBoostedRewardsBannedRadioVerificationStatus::SpBoostedRewardsBanInvalidCarrierKey, }; - let verified_report = - VerifiedServiceProviderBoostedRewardsBannedRadioIngestReportV1 { - report: Some(ingest), - status: status.into(), - timestamp: Utc::now().timestamp_millis() as u64, - }; - - self.verified_sink.write(verified_report, &[("status", status.as_str_name())]).await?; + let verified_report = VerifiedServiceProviderBoostedRewardsBannedRadioIngestReportV1 { + report: Some(ingest), + status: status.into(), + timestamp: Utc::now().timestamp_millis() as u64, + }; - Ok(tx) - }) - .await? - .commit() + self.verified_sink + .write(verified_report, &[("status", status.as_str_name())]) .await?; - self.verified_sink.commit().await?; - Ok(()) } @@ -208,22 +234,76 @@ where } } -async fn save( - transaction: &mut Transaction<'_, Postgres>, - report: BannedRadioReport, -) -> anyhow::Result<()> { - sqlx::query( - r#" +mod db { + use chrono::Duration; + + use super::*; + + pub(super) async fn cleanup(pool: &PgPool) -> anyhow::Result<()> { + sqlx::query( + r#" + DELETE FROM sp_boosted_rewards_bans + WHERE until < $1 or invalidated_at < $1 + "#, + ) + .bind(Utc::now() - Duration::days(CLEANUP_DAYS)) + .execute(pool) + .await + .map(|_| ()) + .map_err(anyhow::Error::from) + } + + pub(super) async fn update_report( + transaction: &mut Transaction<'_, Postgres>, + report: BannedRadioReport, + ) -> anyhow::Result<()> { + match report.reason { + SpBoostedRewardsBannedRadioReason::Unbanned => { + invalidate_all_before(transaction, report).await + } + _ => save(transaction, report).await, + } + } + + async fn save( + transaction: &mut Transaction<'_, Postgres>, + report: BannedRadioReport, + ) -> anyhow::Result<()> { + sqlx::query( + r#" INSERT INTO sp_boosted_rewards_bans(radio_type, radio_key, received_timestamp, until) VALUES($1,$2,$3,$4) "#, - ) - .bind(report.radio_type) - .bind(report.radio_key) - .bind(report.received_timestamp) - .bind(report.until) - .execute(transaction) - .await - .map(|_| ()) - .map_err(anyhow::Error::from) + ) + .bind(report.radio_type) + .bind(report.radio_key) + .bind(report.received_timestamp) + .bind(report.until) + .execute(transaction) + .await + .map(|_| ()) + .map_err(anyhow::Error::from) + } + + async fn invalidate_all_before( + transaction: &mut Transaction<'_, Postgres>, + report: BannedRadioReport, + ) -> anyhow::Result<()> { + sqlx::query( + r#" + UPDATE sp_boosted_rewards_bans + SET invalidated_at = now() + WHERE radio_type = $1 + AND radio_key = $2 + AND received_timestamp <= $3 + "#, + ) + .bind(report.radio_type) + .bind(report.radio_key) + .bind(report.received_timestamp) + .execute(transaction) + .await + .map(|_| ()) + .map_err(anyhow::Error::from) + } } From a44a23f1de9e384d58803d78bd97ea4e79d5d61e Mon Sep 17 00:00:00 2001 From: Brian Balser Date: Mon, 1 Jul 2024 10:03:20 -0400 Subject: [PATCH 06/23] Update mobile-verifier rewarder to incorporate sp bans --- coverage_point_calculator/src/lib.rs | 85 +++++++++++-------- .../tests/coverage_point_calculator.rs | 14 +-- mobile_verifier/src/cli/reward_from_db.rs | 4 +- mobile_verifier/src/reward_shares.rs | 31 +++---- mobile_verifier/src/rewarder.rs | 20 +++-- .../src/rewarder/boosted_hex_eligibility.rs | 43 ++++++++++ .../src/sp_boosted_rewards_bans.rs | 70 +++++++++++---- .../tests/integrations/boosting_oracles.rs | 4 +- .../tests/integrations/modeled_coverage.rs | 14 +-- 9 files changed, 190 insertions(+), 95 deletions(-) create mode 100644 mobile_verifier/src/rewarder/boosted_hex_eligibility.rs diff --git a/coverage_point_calculator/src/lib.rs b/coverage_point_calculator/src/lib.rs index 856dece52..f3141bf42 100644 --- a/coverage_point_calculator/src/lib.rs +++ b/coverage_point_calculator/src/lib.rs @@ -122,7 +122,7 @@ pub struct CoveragePoints { /// Input Radio Type pub radio_type: RadioType, /// Input RadioThreshold - pub radio_threshold: RadioThreshold, + pub service_provider_boosted_reward_eligibility: ServiceProviderBoostedRewardEligibility, /// Derived Eligibility for Boosted Hex Rewards pub boosted_hex_eligibility: BoostedHexStatus, /// Speedtests used in calculcation @@ -136,7 +136,7 @@ pub struct CoveragePoints { impl CoveragePoints { pub fn new( radio_type: RadioType, - radio_threshold: RadioThreshold, + service_provider_boosted_reward_eligibility: ServiceProviderBoostedRewardEligibility, speedtests: Vec, trust_scores: Vec, ranked_coverage: Vec, @@ -144,8 +144,11 @@ impl CoveragePoints { let location_trust_scores = location::clean_trust_scores(trust_scores, &ranked_coverage); let location_trust_multiplier = location::multiplier(radio_type, &location_trust_scores); - let boost_eligibility = - BoostedHexStatus::new(&radio_type, location_trust_multiplier, &radio_threshold); + let boost_eligibility = BoostedHexStatus::new( + &radio_type, + location_trust_multiplier, + &service_provider_boosted_reward_eligibility, + ); let covered_hexes = hexes::clean_covered_hexes(radio_type, boost_eligibility, ranked_coverage)?; @@ -159,7 +162,7 @@ impl CoveragePoints { location_trust_multiplier, speedtest_multiplier, radio_type, - radio_threshold, + service_provider_boosted_reward_eligibility, boosted_hex_eligibility: boost_eligibility, speedtests, location_trust_scores, @@ -207,6 +210,7 @@ impl CoveragePoints { BoostedHexStatus::Eligible => self.coverage_points.boosted, BoostedHexStatus::WifiLocationScoreBelowThreshold(_) => dec!(0), BoostedHexStatus::RadioThresholdNotMet => dec!(0), + BoostedHexStatus::ServiceProviderBanned => dec!(0), } } } @@ -216,13 +220,14 @@ pub enum BoostedHexStatus { Eligible, WifiLocationScoreBelowThreshold(Decimal), RadioThresholdNotMet, + ServiceProviderBanned, } impl BoostedHexStatus { fn new( radio_type: &RadioType, location_trust_score: Decimal, - radio_threshold: &RadioThreshold, + service_provider_boosted_reward_eligibility: &ServiceProviderBoostedRewardEligibility, ) -> Self { // hip-93: if radio is wifi & location_trust score multiplier < 0.75, no boosting if radio_type.is_wifi() && location_trust_score < dec!(0.75) { @@ -230,11 +235,15 @@ impl BoostedHexStatus { } // hip-84: if radio has not met minimum data and subscriber thresholds, no boosting - if !radio_threshold.is_met() { - return Self::RadioThresholdNotMet; + match service_provider_boosted_reward_eligibility { + ServiceProviderBoostedRewardEligibility::Eligible => Self::Eligible, + ServiceProviderBoostedRewardEligibility::ServiceProviderBanned => { + Self::ServiceProviderBanned + } + ServiceProviderBoostedRewardEligibility::RadioThresholdNotMet => { + Self::RadioThresholdNotMet + } } - - Self::Eligible } fn is_eligible(&self) -> bool { @@ -307,16 +316,23 @@ impl RadioType { } #[derive(Debug, Clone, Copy, PartialEq)] -pub enum RadioThreshold { - Verified, - Unverified, +pub enum ServiceProviderBoostedRewardEligibility { + Eligible, + ServiceProviderBanned, + RadioThresholdNotMet, } -impl RadioThreshold { - fn is_met(&self) -> bool { - matches!(self, Self::Verified) - } -} +// #[derive(Debug, Clone, Copy, PartialEq)] +// pub enum RadioThreshold { +// Verified, +// Unverified, +// } + +// impl RadioThreshold { +// fn is_met(&self) -> bool { +// matches!(self, Self::Verified) +// } +// } #[cfg(test)] mod tests { @@ -342,7 +358,7 @@ mod tests { ) { let wifi = CoveragePoints::new( RadioType::IndoorWifi, - RadioThreshold::Verified, + ServiceProviderBoostedRewardEligibility::Eligible, speedtest_maximum(), location_trust_maximum(), vec![RankedCoverage { @@ -364,10 +380,10 @@ mod tests { #[test] fn hip_84_radio_meets_minimum_subscriber_threshold_for_boosted_hexes() { - let calculate_wifi = |radio_verified: RadioThreshold| { + let calculate_wifi = |eligibility: ServiceProviderBoostedRewardEligibility| { CoveragePoints::new( RadioType::IndoorWifi, - radio_verified, + eligibility, speedtest_maximum(), location_trust_maximum(), vec![RankedCoverage { @@ -389,12 +405,13 @@ mod tests { // Radio meeting the threshold is eligible for boosted hexes. // Boosted hex provides radio with more than base_points. - let verified_wifi = calculate_wifi(RadioThreshold::Verified); + let verified_wifi = calculate_wifi(ServiceProviderBoostedRewardEligibility::Eligible); assert_eq!(base_points * dec!(5), verified_wifi.coverage_points_v1()); // Radio not meeting the threshold is not eligible for boosted hexes. // Boost from hex is not applied, radio receives base points. - let unverified_wifi = calculate_wifi(RadioThreshold::Unverified); + let unverified_wifi = + calculate_wifi(ServiceProviderBoostedRewardEligibility::RadioThresholdNotMet); assert_eq!(base_points, unverified_wifi.coverage_points_v1()); } @@ -403,7 +420,7 @@ mod tests { let calculate_wifi = |location_trust_scores: Vec| { CoveragePoints::new( RadioType::IndoorWifi, - RadioThreshold::Verified, + ServiceProviderBoostedRewardEligibility::Eligible, speedtest_maximum(), location_trust_scores, vec![RankedCoverage { @@ -441,7 +458,7 @@ mod tests { let calculate_indoor_cbrs = |speedtests: Vec| { CoveragePoints::new( RadioType::IndoorCbrs, - RadioThreshold::Verified, + ServiceProviderBoostedRewardEligibility::Eligible, speedtests, location_trust_maximum(), vec![RankedCoverage { @@ -529,7 +546,7 @@ mod tests { use Assignment::*; let indoor_cbrs = CoveragePoints::new( RadioType::IndoorCbrs, - RadioThreshold::Verified, + ServiceProviderBoostedRewardEligibility::Eligible, speedtest_maximum(), location_trust_maximum(), vec![ @@ -586,7 +603,7 @@ mod tests { ) { let outdoor_wifi = CoveragePoints::new( radio_type, - RadioThreshold::Verified, + ServiceProviderBoostedRewardEligibility::Eligible, speedtest_maximum(), location_trust_maximum(), vec![RankedCoverage { @@ -615,7 +632,7 @@ mod tests { ) { let indoor_wifi = CoveragePoints::new( radio_type, - RadioThreshold::Verified, + ServiceProviderBoostedRewardEligibility::Eligible, speedtest_maximum(), location_trust_maximum(), vec![ @@ -658,7 +675,7 @@ mod tests { // Location scores are averaged together let indoor_wifi = CoveragePoints::new( RadioType::IndoorWifi, - RadioThreshold::Verified, + ServiceProviderBoostedRewardEligibility::Eligible, speedtest_maximum(), location_trust_with_scores(&[dec!(0.1), dec!(0.2), dec!(0.3), dec!(0.4)]), vec![RankedCoverage { @@ -702,7 +719,7 @@ mod tests { ]; let indoor_wifi = CoveragePoints::new( RadioType::IndoorWifi, - RadioThreshold::Verified, + ServiceProviderBoostedRewardEligibility::Eligible, speedtest_maximum(), location_trust_maximum(), covered_hexes.clone(), @@ -725,7 +742,7 @@ mod tests { ) { let outdoor_cbrs = CoveragePoints::new( RadioType::OutdoorCbrs, - RadioThreshold::Verified, + ServiceProviderBoostedRewardEligibility::Eligible, speedtest_maximum(), location_trust_maximum(), vec![RankedCoverage { @@ -752,7 +769,7 @@ mod tests { ) { let indoor_cbrs = CoveragePoints::new( RadioType::IndoorCbrs, - RadioThreshold::Verified, + ServiceProviderBoostedRewardEligibility::Eligible, speedtest_maximum(), location_trust_maximum(), vec![RankedCoverage { @@ -781,7 +798,7 @@ mod tests { ) { let outdoor_wifi = CoveragePoints::new( RadioType::OutdoorWifi, - RadioThreshold::Verified, + ServiceProviderBoostedRewardEligibility::Eligible, speedtest_maximum(), location_trust_maximum(), vec![RankedCoverage { @@ -808,7 +825,7 @@ mod tests { ) { let indoor_wifi = CoveragePoints::new( RadioType::IndoorWifi, - RadioThreshold::Verified, + ServiceProviderBoostedRewardEligibility::Eligible, speedtest_maximum(), location_trust_maximum(), vec![RankedCoverage { diff --git a/coverage_point_calculator/tests/coverage_point_calculator.rs b/coverage_point_calculator/tests/coverage_point_calculator.rs index 8f3cfb462..aba3323ed 100644 --- a/coverage_point_calculator/tests/coverage_point_calculator.rs +++ b/coverage_point_calculator/tests/coverage_point_calculator.rs @@ -3,8 +3,8 @@ use std::num::NonZeroU32; use chrono::Utc; use coverage_map::{BoostedHexMap, RankedCoverage, SignalLevel, UnrankedCoverage}; use coverage_point_calculator::{ - BytesPs, CoveragePoints, LocationTrust, RadioThreshold, RadioType, Result, Speedtest, - SpeedtestTier, + BytesPs, CoveragePoints, LocationTrust, RadioType, Result, + ServiceProviderBoostedRewardEligibility, Speedtest, SpeedtestTier, }; use hex_assignments::{assignment::HexAssignments, Assignment}; use rust_decimal_macros::dec; @@ -52,7 +52,7 @@ fn base_radio_coverage_points() { ] { let coverage_points = CoveragePoints::new( radio_type, - RadioThreshold::Verified, + ServiceProviderBoostedRewardEligibility::Eligible, speedtests.clone(), location_trust_scores.clone(), hexes.clone(), @@ -113,7 +113,7 @@ fn radios_with_coverage() { ] { let coverage_points = CoveragePoints::new( radio_type, - RadioThreshold::Verified, + ServiceProviderBoostedRewardEligibility::Eligible, default_speedtests.clone(), default_location_trust_scores.clone(), base_hex_iter.clone().take(num_hexes).collect(), @@ -240,7 +240,7 @@ fn cbrs_outdoor_with_mixed_signal_level_coverage() -> Result { let radio = CoveragePoints::new( RadioType::OutdoorCbrs, - RadioThreshold::Verified, + ServiceProviderBoostedRewardEligibility::Eligible, Speedtest::mock(SpeedtestTier::Good), vec![], // Location Trust is ignored for Cbrs vec![ @@ -372,7 +372,7 @@ fn indoor_cbrs_radio( ) -> Result { CoveragePoints::new( RadioType::IndoorCbrs, - RadioThreshold::Verified, + ServiceProviderBoostedRewardEligibility::Eligible, Speedtest::mock(speedtest_tier), vec![], coverage.to_owned(), @@ -385,7 +385,7 @@ fn outdoor_cbrs_radio( ) -> Result { CoveragePoints::new( RadioType::OutdoorCbrs, - RadioThreshold::Verified, + ServiceProviderBoostedRewardEligibility::Eligible, Speedtest::mock(speedtest_tier), vec![], coverage.to_owned(), diff --git a/mobile_verifier/src/cli/reward_from_db.rs b/mobile_verifier/src/cli/reward_from_db.rs index e11b618a6..283bb37ac 100644 --- a/mobile_verifier/src/cli/reward_from_db.rs +++ b/mobile_verifier/src/cli/reward_from_db.rs @@ -1,9 +1,9 @@ use crate::{ heartbeats::HeartbeatReward, - radio_threshold::VerifiedRadioThresholds, reward_shares::{ get_scheduled_tokens_for_poc, CoverageShares, DataTransferAndPocAllocatedRewardBuckets, }, + rewarder::boosted_hex_eligibility::BoostedHexEligibility, speedtests_average::SpeedtestAverages, Settings, }; @@ -47,7 +47,7 @@ impl Cmd { heartbeats, &speedtest_averages, &BoostedHexes::default(), - &VerifiedRadioThresholds::default(), + &BoostedHexEligibility::default(), &epoch, ) .await?; diff --git a/mobile_verifier/src/reward_shares.rs b/mobile_verifier/src/reward_shares.rs index 81913e28d..b0efe9f4e 100644 --- a/mobile_verifier/src/reward_shares.rs +++ b/mobile_verifier/src/reward_shares.rs @@ -2,11 +2,12 @@ use crate::{ coverage::{CoveredHexStream, Seniority}, data_session::{HotspotMap, ServiceProviderDataSession}, heartbeats::HeartbeatReward, - radio_threshold::VerifiedRadioThresholds, + rewarder::boosted_hex_eligibility::BoostedHexEligibility, speedtests_average::SpeedtestAverages, subscriber_location::SubscriberValidatedLocations, }; use chrono::{DateTime, Duration, Utc}; +use coverage_point_calculator::ServiceProviderBoostedRewardEligibility; use file_store::traits::TimestampEncode; use futures::{Stream, StreamExt}; use helium_crypto::PublicKeyBinary; @@ -425,7 +426,7 @@ struct RadioInfo { coverage_obj_uuid: Uuid, seniority: Seniority, trust_scores: Vec, - verified_radio_threshold: coverage_point_calculator::RadioThreshold, + sp_boosted_reward_eligibility: ServiceProviderBoostedRewardEligibility, speedtests: Vec, } @@ -441,7 +442,7 @@ impl CoverageShares { heartbeats: impl Stream>, speedtest_averages: &SpeedtestAverages, boosted_hexes: &BoostedHexes, - verified_radio_thresholds: &VerifiedRadioThresholds, + boosted_hex_eligibility: &BoostedHexEligibility, reward_period: &Range>, ) -> anyhow::Result { let mut radio_infos: HashMap = HashMap::new(); @@ -512,12 +513,8 @@ impl CoverageShares { }) .collect(); - use coverage_point_calculator::RadioThreshold; - let verified_radio_threshold = - match verified_radio_thresholds.is_verified(pubkey, cbsd_id) { - true => RadioThreshold::Verified, - false => RadioThreshold::Unverified, - }; + let sp_boosted_reward_eligibility = + boosted_hex_eligibility.eligibility(pubkey, cbsd_id); use coverage_point_calculator::LocationTrust; let trust_scores = heartbeat @@ -535,7 +532,7 @@ impl CoverageShares { coverage_obj_uuid: heartbeat.coverage_object, seniority, trust_scores, - verified_radio_threshold, + sp_boosted_reward_eligibility, speedtests, }, ); @@ -566,7 +563,7 @@ impl CoverageShares { let coverage_points = coverage_point_calculator::CoveragePoints::new( radio_info.radio_type, - radio_info.verified_radio_threshold, + radio_info.sp_boosted_reward_eligibility, radio_info.speedtests.clone(), radio_info.trust_scores.clone(), hexes, @@ -1494,7 +1491,7 @@ mod test { stream::iter(heartbeat_rewards), &speedtest_avgs, &BoostedHexes::default(), - &VerifiedRadioThresholds::default(), + &BoostedHexEligibility::default(), &epoch, ) .await @@ -1668,7 +1665,7 @@ mod test { stream::iter(heartbeat_rewards), &speedtest_avgs, &BoostedHexes::default(), - &VerifiedRadioThresholds::default(), + &BoostedHexEligibility::default(), &epoch, ) .await @@ -1799,7 +1796,7 @@ mod test { stream::iter(heartbeat_rewards), &speedtest_avgs, &BoostedHexes::default(), - &VerifiedRadioThresholds::default(), + &BoostedHexEligibility::default(), &epoch, ) .await @@ -1930,7 +1927,7 @@ mod test { stream::iter(heartbeat_rewards), &speedtest_avgs, &BoostedHexes::default(), - &VerifiedRadioThresholds::default(), + &BoostedHexEligibility::default(), &epoch, ) .await @@ -2027,7 +2024,7 @@ mod test { inserted_at: now, update_reason: 0, }, - verified_radio_threshold: coverage_point_calculator::RadioThreshold::Verified, + sp_boosted_reward_eligibility: ServiceProviderBoostedRewardEligibility::Eligible, speedtests: vec![ coverage_point_calculator::Speedtest { upload_speed: coverage_point_calculator::BytesPs::new(100_000_000), @@ -2060,7 +2057,7 @@ mod test { inserted_at: now, update_reason: 0, }, - verified_radio_threshold: coverage_point_calculator::RadioThreshold::Verified, + sp_boosted_reward_eligibility: ServiceProviderBoostedRewardEligibility::Eligible, speedtests: vec![], }, ); diff --git a/mobile_verifier/src/rewarder.rs b/mobile_verifier/src/rewarder.rs index 6a6a23868..164969c76 100644 --- a/mobile_verifier/src/rewarder.rs +++ b/mobile_verifier/src/rewarder.rs @@ -3,11 +3,8 @@ use crate::{ coverage, data_session, heartbeats::{self, HeartbeatReward}, radio_threshold, - reward_shares::{ - self, CoverageShares, DataTransferAndPocAllocatedRewardBuckets, MapperShares, - ServiceProviderShares, TransferRewards, - }, - speedtests, + reward_shares::{self, CoverageShares, MapperShares, ServiceProviderShares, TransferRewards}, + sp_boosted_rewards_bans, speedtests, speedtests_average::SpeedtestAverages, subscriber_location, telemetry, Settings, }; @@ -42,6 +39,10 @@ use std::{ops::Range, time::Duration}; use task_manager::{ManagedTask, TaskManager}; use tokio::time::sleep; +use self::boosted_hex_eligibility::BoostedHexEligibility; + +pub mod boosted_hex_eligibility; + const REWARDS_NOT_CURRENT_DELAY_PERIOD: i64 = 5; pub struct Rewarder { @@ -288,6 +289,7 @@ where speedtests::clear_speedtests(&mut transaction, &reward_period.start).await?; data_session::clear_hotspot_data_sessions(&mut transaction, &reward_period.start).await?; coverage::clear_coverage_objects(&mut transaction, &reward_period.start).await?; + sp_boosted_rewards_bans::clear_bans(&mut transaction, reward_period.start).await?; // subscriber_location::clear_location_shares(&mut transaction, &reward_period.end).await?; let next_reward_period = scheduler.next_reward_period(); @@ -407,15 +409,17 @@ async fn reward_poc( let boosted_hexes = BoostedHexes::get_all(hex_service_client).await?; - let verified_radio_thresholds = - radio_threshold::verified_radio_thresholds(pool, reward_period).await?; + let boosted_hex_eligibility = BoostedHexEligibility::new( + radio_threshold::verified_radio_thresholds(pool, reward_period).await?, + sp_boosted_rewards_bans::db::get_banned_radios(pool, reward_period.end).await?, + ); let coverage_shares = CoverageShares::new( pool, heartbeats, &speedtest_averages, &boosted_hexes, - &verified_radio_thresholds, + &boosted_hex_eligibility, reward_period, ) .await?; diff --git a/mobile_verifier/src/rewarder/boosted_hex_eligibility.rs b/mobile_verifier/src/rewarder/boosted_hex_eligibility.rs new file mode 100644 index 000000000..583165543 --- /dev/null +++ b/mobile_verifier/src/rewarder/boosted_hex_eligibility.rs @@ -0,0 +1,43 @@ +use std::collections::HashSet; + +use coverage_point_calculator::ServiceProviderBoostedRewardEligibility; +use helium_crypto::PublicKeyBinary; + +use crate::radio_threshold::VerifiedRadioThresholds; + +#[derive(Debug, Default)] +pub struct BoostedHexEligibility { + radio_thresholds: VerifiedRadioThresholds, + sp_bans: HashSet<(Option, Option)>, +} + +impl BoostedHexEligibility { + pub fn new( + radio_thresholds: VerifiedRadioThresholds, + sp_bans: HashSet<(Option, Option)>, + ) -> Self { + Self { + radio_thresholds, + sp_bans, + } + } + + pub fn eligibility( + &self, + key: PublicKeyBinary, + cbsd_id_opt: Option, + ) -> ServiceProviderBoostedRewardEligibility { + let ban_key = match cbsd_id_opt.clone() { + Some(cbsd_id) => (None, Some(cbsd_id)), + None => (Some(key.clone()), None), + }; + + if self.sp_bans.contains(&ban_key) { + ServiceProviderBoostedRewardEligibility::ServiceProviderBanned + } else if self.radio_thresholds.is_verified(key, cbsd_id_opt) { + ServiceProviderBoostedRewardEligibility::Eligible + } else { + ServiceProviderBoostedRewardEligibility::RadioThresholdNotMet + } + } +} diff --git a/mobile_verifier/src/sp_boosted_rewards_bans.rs b/mobile_verifier/src/sp_boosted_rewards_bans.rs index 4355a6d00..9d6f447d3 100644 --- a/mobile_verifier/src/sp_boosted_rewards_bans.rs +++ b/mobile_verifier/src/sp_boosted_rewards_bans.rs @@ -28,7 +28,6 @@ use tokio::sync::mpsc::Receiver; use crate::Settings; const CLEANUP_DAYS: i64 = 7; -const CLEANUP_INTERVAL_SECS: u64 = 12 * 60 * 60; struct BannedRadioReport { received_timestamp: DateTime, @@ -153,16 +152,10 @@ where async fn run(mut self, mut shutdown: triggered::Listener) -> anyhow::Result<()> { tracing::info!("service provider boosted rewards ban ingestor starting"); - let mut cleanup_interval = - tokio::time::interval(std::time::Duration::from_secs(CLEANUP_INTERVAL_SECS)); - loop { tokio::select! { biased; _ = &mut shutdown => break, - _ = cleanup_interval.tick() => { - db::cleanup(&self.pool).await?; - } Some(file) = self.receiver.recv() => { self.process_file(file).await?; } @@ -234,20 +227,61 @@ where } } -mod db { +pub async fn clear_bans( + transaction: &mut Transaction<'_, Postgres>, + before: DateTime, +) -> anyhow::Result<()> { + db::cleanup(transaction, before).await +} + +pub mod db { + use std::{collections::HashSet, str::FromStr}; + use chrono::Duration; + use sqlx::Row; use super::*; - pub(super) async fn cleanup(pool: &PgPool) -> anyhow::Result<()> { + pub async fn get_banned_radios( + pool: &PgPool, + date_time: DateTime, + ) -> anyhow::Result, Option)>> { sqlx::query( r#" - DELETE FROM sp_boosted_rewards_bans - WHERE until < $1 or invalidated_at < $1 - "#, + SELECT radio_type, radio_key + FROM sp_boosted_rewards_bans + WHERE until < $1 or invalidated_at < $1 + "#, ) - .bind(Utc::now() - Duration::days(CLEANUP_DAYS)) - .execute(pool) + .bind(date_time) + .fetch(pool) + .map_err(anyhow::Error::from) + .try_fold(HashSet::new(), |mut set, row| async move { + let radio_type = row.get::<&str, &str>("radio_type"); + let radio_key = row.get::("radio_key"); + match radio_type { + "wifi" => set.insert((Some(PublicKeyBinary::from_str(&radio_key)?), None)), + "cbrs" => set.insert((None, Some(radio_key))), + _ => anyhow::bail!("Inavlid radio type: {}", radio_type), + }; + + Ok(set) + }) + .await + } + + pub(super) async fn cleanup( + transaction: &mut Transaction<'_, Postgres>, + before: DateTime, + ) -> anyhow::Result<()> { + sqlx::query( + r#" + DELETE FROM sp_boosted_rewards_bans + WHERE until < $1 or invalidated_at < $1 + "#, + ) + .bind(before - Duration::days(CLEANUP_DAYS)) + .execute(transaction) .await .map(|_| ()) .map_err(anyhow::Error::from) @@ -271,9 +305,9 @@ mod db { ) -> anyhow::Result<()> { sqlx::query( r#" - INSERT INTO sp_boosted_rewards_bans(radio_type, radio_key, received_timestamp, until) - VALUES($1,$2,$3,$4) - "#, + INSERT INTO sp_boosted_rewards_bans(radio_type, radio_key, received_timestamp, until) + VALUES($1,$2,$3,$4) + "#, ) .bind(report.radio_type) .bind(report.radio_key) @@ -295,7 +329,7 @@ mod db { SET invalidated_at = now() WHERE radio_type = $1 AND radio_key = $2 - AND received_timestamp <= $3 + AND received_timestamp <= $3 "#, ) .bind(report.radio_type) diff --git a/mobile_verifier/tests/integrations/boosting_oracles.rs b/mobile_verifier/tests/integrations/boosting_oracles.rs index 1436437b5..1a6c8f4d4 100644 --- a/mobile_verifier/tests/integrations/boosting_oracles.rs +++ b/mobile_verifier/tests/integrations/boosting_oracles.rs @@ -21,8 +21,8 @@ use mobile_verifier::{ last_location::LocationCache, Heartbeat, HeartbeatReward, SeniorityUpdate, ValidatedHeartbeat, }, - radio_threshold::VerifiedRadioThresholds, reward_shares::CoverageShares, + rewarder::boosted_hex_eligibility::BoostedHexEligibility, speedtests::Speedtest, speedtests_average::{SpeedtestAverage, SpeedtestAverages}, GatewayResolution, GatewayResolver, @@ -405,7 +405,7 @@ async fn test_footfall_and_urbanization_and_landtype(pool: PgPool) -> anyhow::Re heartbeats, &speedtest_avgs, &BoostedHexes::default(), - &VerifiedRadioThresholds::default(), + &BoostedHexEligibility::default(), &epoch, ) .await diff --git a/mobile_verifier/tests/integrations/modeled_coverage.rs b/mobile_verifier/tests/integrations/modeled_coverage.rs index e8ab799ba..2253cbe04 100644 --- a/mobile_verifier/tests/integrations/modeled_coverage.rs +++ b/mobile_verifier/tests/integrations/modeled_coverage.rs @@ -21,8 +21,8 @@ use mobile_verifier::{ last_location::LocationCache, Heartbeat, HeartbeatReward, KeyType, SeniorityUpdate, ValidatedHeartbeat, }, - radio_threshold::VerifiedRadioThresholds, reward_shares::CoverageShares, + rewarder::boosted_hex_eligibility::BoostedHexEligibility, speedtests::Speedtest, speedtests_average::{SpeedtestAverage, SpeedtestAverages}, GatewayResolution, GatewayResolver, IsAuthorized, @@ -497,7 +497,7 @@ async fn scenario_one(pool: PgPool) -> anyhow::Result<()> { heartbeats, &speedtest_avgs, &BoostedHexes::default(), - &VerifiedRadioThresholds::default(), + &BoostedHexEligibility::default(), &reward_period, ) .await?; @@ -599,7 +599,7 @@ async fn scenario_two(pool: PgPool) -> anyhow::Result<()> { heartbeats, &speedtest_avgs, &BoostedHexes::default(), - &VerifiedRadioThresholds::default(), + &BoostedHexEligibility::default(), &reward_period, ) .await?; @@ -887,7 +887,7 @@ async fn scenario_three(pool: PgPool) -> anyhow::Result<()> { heartbeats, &speedtest_avgs, &boosted_hexes, - &VerifiedRadioThresholds::default(), + &BoostedHexEligibility::default(), &reward_period, ) .await?; @@ -976,7 +976,7 @@ async fn scenario_four(pool: PgPool) -> anyhow::Result<()> { heartbeats, &speedtest_avgs, &BoostedHexes::default(), - &VerifiedRadioThresholds::default(), + &BoostedHexEligibility::default(), &reward_period, ) .await?; @@ -1077,7 +1077,7 @@ async fn scenario_five(pool: PgPool) -> anyhow::Result<()> { heartbeats, &speedtest_avgs, &BoostedHexes::default(), - &VerifiedRadioThresholds::default(), + &BoostedHexEligibility::default(), &reward_period, ) .await?; @@ -1326,7 +1326,7 @@ async fn scenario_six(pool: PgPool) -> anyhow::Result<()> { heartbeats, &speedtest_avgs, &BoostedHexes::default(), - &VerifiedRadioThresholds::default(), + &BoostedHexEligibility::default(), &reward_period, ) .await?; From 7df62853b78abbef95e82614992e2bbfa9c39112 Mon Sep 17 00:00:00 2001 From: Brian Balser Date: Mon, 1 Jul 2024 11:29:59 -0400 Subject: [PATCH 07/23] Refactoring --- .../src/rewarder/boosted_hex_eligibility.rs | 115 +++++++++++++++--- .../src/sp_boosted_rewards_bans.rs | 35 +++++- 2 files changed, 130 insertions(+), 20 deletions(-) diff --git a/mobile_verifier/src/rewarder/boosted_hex_eligibility.rs b/mobile_verifier/src/rewarder/boosted_hex_eligibility.rs index 583165543..21f327fea 100644 --- a/mobile_verifier/src/rewarder/boosted_hex_eligibility.rs +++ b/mobile_verifier/src/rewarder/boosted_hex_eligibility.rs @@ -1,24 +1,19 @@ -use std::collections::HashSet; - use coverage_point_calculator::ServiceProviderBoostedRewardEligibility; use helium_crypto::PublicKeyBinary; -use crate::radio_threshold::VerifiedRadioThresholds; +use crate::{radio_threshold::VerifiedRadioThresholds, sp_boosted_rewards_bans::BannedRadios}; #[derive(Debug, Default)] pub struct BoostedHexEligibility { radio_thresholds: VerifiedRadioThresholds, - sp_bans: HashSet<(Option, Option)>, + banned_radios: BannedRadios, } impl BoostedHexEligibility { - pub fn new( - radio_thresholds: VerifiedRadioThresholds, - sp_bans: HashSet<(Option, Option)>, - ) -> Self { + pub fn new(radio_thresholds: VerifiedRadioThresholds, banned_radios: BannedRadios) -> Self { Self { radio_thresholds, - sp_bans, + banned_radios, } } @@ -27,12 +22,7 @@ impl BoostedHexEligibility { key: PublicKeyBinary, cbsd_id_opt: Option, ) -> ServiceProviderBoostedRewardEligibility { - let ban_key = match cbsd_id_opt.clone() { - Some(cbsd_id) => (None, Some(cbsd_id)), - None => (Some(key.clone()), None), - }; - - if self.sp_bans.contains(&ban_key) { + if self.banned_radios.contains(&key, cbsd_id_opt.as_deref()) { ServiceProviderBoostedRewardEligibility::ServiceProviderBanned } else if self.radio_thresholds.is_verified(key, cbsd_id_opt) { ServiceProviderBoostedRewardEligibility::Eligible @@ -41,3 +31,98 @@ impl BoostedHexEligibility { } } } + +#[cfg(test)] +mod tests { + use helium_crypto::{KeyTag, Keypair}; + use rand::rngs::OsRng; + + use super::*; + + #[test] + fn banned() { + let keypair = generate_keypair(); + + let pub_key: PublicKeyBinary = keypair.public_key().to_vec().into(); + let cbsd_id = "cbsd-id-1".to_string(); + + let mut banned_radios = BannedRadios::default(); + banned_radios.insert_wifi(pub_key.clone()); + banned_radios.insert_cbrs(cbsd_id.clone()); + + let boosted_hex_eligibility = + BoostedHexEligibility::new(VerifiedRadioThresholds::default(), banned_radios); + + let eligibility = boosted_hex_eligibility.eligibility(pub_key.clone(), None); + + assert_eq!( + ServiceProviderBoostedRewardEligibility::ServiceProviderBanned, + eligibility + ); + + let eligibility = boosted_hex_eligibility.eligibility(pub_key, Some(cbsd_id)); + + assert_eq!( + ServiceProviderBoostedRewardEligibility::ServiceProviderBanned, + eligibility + ); + } + + #[test] + fn radio_threshold_not_met() { + let keypair = generate_keypair(); + + let pub_key: PublicKeyBinary = keypair.public_key().to_vec().into(); + let cbsd_id = "cbsd-id-1".to_string(); + + let boosted_hex_eligibility = + BoostedHexEligibility::new(VerifiedRadioThresholds::default(), BannedRadios::default()); + + let eligibility = boosted_hex_eligibility.eligibility(pub_key.clone(), None); + + assert_eq!( + ServiceProviderBoostedRewardEligibility::RadioThresholdNotMet, + eligibility + ); + + let eligibility = boosted_hex_eligibility.eligibility(pub_key, Some(cbsd_id)); + + assert_eq!( + ServiceProviderBoostedRewardEligibility::RadioThresholdNotMet, + eligibility + ); + } + + #[test] + fn eligible() { + let keypair = generate_keypair(); + + let pub_key: PublicKeyBinary = keypair.public_key().to_vec().into(); + let cbsd_id = "cbsd-id-1".to_string(); + + let mut verified_radio_thresholds = VerifiedRadioThresholds::default(); + verified_radio_thresholds.insert(pub_key.clone(), None); + verified_radio_thresholds.insert(pub_key.clone(), Some(cbsd_id.clone())); + + let boosted_hex_eligibility = + BoostedHexEligibility::new(verified_radio_thresholds, BannedRadios::default()); + + let eligibility = boosted_hex_eligibility.eligibility(pub_key.clone(), None); + + assert_eq!( + ServiceProviderBoostedRewardEligibility::Eligible, + eligibility + ); + + let eligibility = boosted_hex_eligibility.eligibility(pub_key, Some(cbsd_id)); + + assert_eq!( + ServiceProviderBoostedRewardEligibility::Eligible, + eligibility + ); + } + + fn generate_keypair() -> Keypair { + Keypair::generate(KeyTag::default(), &mut OsRng) + } +} diff --git a/mobile_verifier/src/sp_boosted_rewards_bans.rs b/mobile_verifier/src/sp_boosted_rewards_bans.rs index 9d6f447d3..99a8739eb 100644 --- a/mobile_verifier/src/sp_boosted_rewards_bans.rs +++ b/mobile_verifier/src/sp_boosted_rewards_bans.rs @@ -1,3 +1,5 @@ +use std::collections::HashSet; + use chrono::{DateTime, TimeZone, Utc}; use file_store::{ file_info_poller::{ @@ -75,6 +77,29 @@ impl TryFrom for BannedR } } +#[derive(Debug, Default)] +pub struct BannedRadios { + wifi: HashSet, + cbrs: HashSet, +} + +impl BannedRadios { + pub fn insert_wifi(&mut self, pubkey: PublicKeyBinary) { + self.wifi.insert(pubkey); + } + + pub fn insert_cbrs(&mut self, cbsd_id: String) { + self.cbrs.insert(cbsd_id); + } + + pub fn contains(&self, pubkey: &PublicKeyBinary, cbsd_id_opt: Option<&str>) -> bool { + match cbsd_id_opt { + Some(cbsd_id) => self.cbrs.contains(cbsd_id), + None => self.wifi.contains(pubkey), + } + } +} + pub struct ServiceProviderBoostedRewardsBanIngestor { pool: PgPool, authorization_verifier: AV, @@ -235,7 +260,7 @@ pub async fn clear_bans( } pub mod db { - use std::{collections::HashSet, str::FromStr}; + use std::str::FromStr; use chrono::Duration; use sqlx::Row; @@ -245,7 +270,7 @@ pub mod db { pub async fn get_banned_radios( pool: &PgPool, date_time: DateTime, - ) -> anyhow::Result, Option)>> { + ) -> anyhow::Result { sqlx::query( r#" SELECT radio_type, radio_key @@ -256,12 +281,12 @@ pub mod db { .bind(date_time) .fetch(pool) .map_err(anyhow::Error::from) - .try_fold(HashSet::new(), |mut set, row| async move { + .try_fold(BannedRadios::default(), |mut set, row| async move { let radio_type = row.get::<&str, &str>("radio_type"); let radio_key = row.get::("radio_key"); match radio_type { - "wifi" => set.insert((Some(PublicKeyBinary::from_str(&radio_key)?), None)), - "cbrs" => set.insert((None, Some(radio_key))), + "wifi" => set.insert_wifi(PublicKeyBinary::from_str(&radio_key)?), + "cbrs" => set.insert_cbrs(radio_key), _ => anyhow::bail!("Inavlid radio type: {}", radio_type), }; From e62fd4405d9f19af58d81d5a52290725f05a3ec3 Mon Sep 17 00:00:00 2001 From: Brian Balser Date: Mon, 1 Jul 2024 16:00:13 -0400 Subject: [PATCH 08/23] move seniority logic to own module --- Cargo.lock | 8 +- mobile_verifier/src/coverage.rs | 24 +-- mobile_verifier/src/heartbeats/mod.rs | 163 +-------------- mobile_verifier/src/lib.rs | 1 + mobile_verifier/src/reward_shares.rs | 5 +- mobile_verifier/src/seniority.rs | 189 ++++++++++++++++++ .../tests/integrations/boosting_oracles.rs | 8 +- .../tests/integrations/modeled_coverage.rs | 6 +- .../tests/integrations/seniority.rs | 6 +- 9 files changed, 210 insertions(+), 200 deletions(-) create mode 100644 mobile_verifier/src/seniority.rs diff --git a/Cargo.lock b/Cargo.lock index b5d09e78e..baf21cbf9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1617,7 +1617,7 @@ checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" [[package]] name = "beacon" version = "0.1.0" -source = "git+https://github.com/helium/proto?branch=bbalser/hip-125#64813bee906724e9ac3abccb2ffcce4bd677f6a6" +source = "git+https://github.com/helium/proto?branch=bbalser/hip-125#6c059f9dac3d2617a4884861f4e973162d7f18c3" dependencies = [ "base64 0.21.7", "byteorder", @@ -1627,7 +1627,7 @@ dependencies = [ "rand_chacha 0.3.0", "rust_decimal", "serde", - "sha2 0.9.9", + "sha2 0.10.8", "thiserror", ] @@ -3801,7 +3801,7 @@ dependencies = [ [[package]] name = "helium-proto" version = "0.1.0" -source = "git+https://github.com/helium/proto?branch=bbalser/hip-125#64813bee906724e9ac3abccb2ffcce4bd677f6a6" +source = "git+https://github.com/helium/proto?branch=bbalser/hip-125#6c059f9dac3d2617a4884861f4e973162d7f18c3" dependencies = [ "bytes", "prost", @@ -9863,7 +9863,7 @@ dependencies = [ "rand 0.8.5", "serde", "serde_json", - "sha2 0.9.9", + "sha2 0.10.8", "thiserror", "twox-hash", "xorf", diff --git a/mobile_verifier/src/coverage.rs b/mobile_verifier/src/coverage.rs index 1b47810ea..055cd3149 100644 --- a/mobile_verifier/src/coverage.rs +++ b/mobile_verifier/src/coverage.rs @@ -1,5 +1,6 @@ use crate::{ heartbeats::{HbType, KeyType, OwnedKeyType}, + seniority::Seniority, IsAuthorized, Settings, }; use chrono::{DateTime, Utc}; @@ -392,29 +393,6 @@ pub trait CoveredHexStream { ) -> Result; } -#[derive(Clone, Debug, PartialEq, sqlx::FromRow)] -pub struct Seniority { - pub uuid: Uuid, - pub seniority_ts: DateTime, - pub last_heartbeat: DateTime, - pub inserted_at: DateTime, - pub update_reason: i32, -} - -impl Seniority { - pub async fn fetch_latest( - key: KeyType<'_>, - exec: &mut Transaction<'_, Postgres>, - ) -> Result, sqlx::Error> { - sqlx::query_as( - "SELECT uuid, seniority_ts, last_heartbeat, inserted_at, update_reason FROM seniority WHERE radio_key = $1 ORDER BY last_heartbeat DESC LIMIT 1", - ) - .bind(key) - .fetch_optional(&mut *exec) - .await - } -} - #[async_trait::async_trait] impl CoveredHexStream for Pool { async fn covered_hex_stream<'a>( diff --git a/mobile_verifier/src/heartbeats/mod.rs b/mobile_verifier/src/heartbeats/mod.rs index ea9cddc79..df820e720 100644 --- a/mobile_verifier/src/heartbeats/mod.rs +++ b/mobile_verifier/src/heartbeats/mod.rs @@ -4,8 +4,9 @@ pub mod wifi; use crate::{ cell_type::{CellType, CellTypeLabel}, - coverage::{CoverageClaimTimeCache, CoverageObjectCache, CoverageObjectMeta, Seniority}, + coverage::{CoverageClaimTimeCache, CoverageObjectCache, CoverageObjectMeta}, geofence::GeofenceValidator, + seniority::{Seniority, SeniorityUpdate}, GatewayResolution, GatewayResolver, }; use anyhow::anyhow; @@ -761,166 +762,10 @@ pub async fn clear_heartbeats( Ok(()) } -pub struct SeniorityUpdate<'a> { - heartbeat: &'a ValidatedHeartbeat, - action: SeniorityUpdateAction, -} - -#[derive(Debug, PartialEq)] -pub enum SeniorityUpdateAction { - NoAction, - Insert { - new_seniority: DateTime, - update_reason: proto::SeniorityUpdateReason, - }, - Update { - curr_seniority: DateTime, - }, -} - -impl<'a> SeniorityUpdate<'a> { - pub fn new(heartbeat: &'a ValidatedHeartbeat, action: SeniorityUpdateAction) -> Self { - Self { heartbeat, action } - } - - pub fn determine_update_action( - heartbeat: &'a ValidatedHeartbeat, - coverage_claim_time: DateTime, - modeled_coverage_start: DateTime, - latest_seniority: Option, - ) -> Self { - use proto::SeniorityUpdateReason::*; - - if let Some(prev_seniority) = latest_seniority { - if heartbeat.heartbeat.coverage_object != Some(prev_seniority.uuid) { - if prev_seniority.update_reason == HeartbeatNotSeen as i32 - && coverage_claim_time < prev_seniority.seniority_ts - { - Self::new(heartbeat, SeniorityUpdateAction::NoAction) - } else { - Self::new( - heartbeat, - SeniorityUpdateAction::Insert { - new_seniority: coverage_claim_time, - update_reason: NewCoverageClaimTime, - }, - ) - } - } else if heartbeat.heartbeat.timestamp - prev_seniority.last_heartbeat - > Duration::days(3) - && coverage_claim_time < heartbeat.heartbeat.timestamp - { - Self::new( - heartbeat, - SeniorityUpdateAction::Insert { - new_seniority: heartbeat.heartbeat.timestamp, - update_reason: HeartbeatNotSeen, - }, - ) - } else { - Self::new( - heartbeat, - SeniorityUpdateAction::Update { - curr_seniority: prev_seniority.seniority_ts, - }, - ) - } - } else if heartbeat.heartbeat.timestamp - modeled_coverage_start > Duration::days(3) { - // This will become the default case 72 hours after we launch modeled coverage - Self::new( - heartbeat, - SeniorityUpdateAction::Insert { - new_seniority: heartbeat.heartbeat.timestamp, - update_reason: HeartbeatNotSeen, - }, - ) - } else { - Self::new( - heartbeat, - SeniorityUpdateAction::Insert { - new_seniority: coverage_claim_time, - update_reason: NewCoverageClaimTime, - }, - ) - } - } -} - -impl SeniorityUpdate<'_> { - #[allow(deprecated)] - pub async fn write(&self, seniorities: &FileSinkClient) -> anyhow::Result<()> { - if let SeniorityUpdateAction::Insert { - new_seniority, - update_reason, - } = self.action - { - seniorities - .write( - proto::SeniorityUpdate { - key_type: Some(self.heartbeat.heartbeat.key().into()), - new_seniority_timestamp: new_seniority.timestamp() as u64, - reason: update_reason as i32, - new_seniority_timestamp_ms: new_seniority.timestamp_millis() as u64, - }, - [], - ) - .await?; - } - Ok(()) - } - - pub async fn execute(self, exec: &mut Transaction<'_, Postgres>) -> anyhow::Result<()> { - match self.action { - SeniorityUpdateAction::NoAction => (), - SeniorityUpdateAction::Insert { - new_seniority, - update_reason, - } => { - sqlx::query( - r#" - INSERT INTO seniority - (radio_key, last_heartbeat, uuid, seniority_ts, inserted_at, update_reason, radio_type) - VALUES - ($1, $2, $3, $4, $5, $6, $7) - ON CONFLICT (radio_key, radio_type, seniority_ts) DO UPDATE SET - uuid = EXCLUDED.uuid, - last_heartbeat = EXCLUDED.last_heartbeat, - update_reason = EXCLUDED.update_reason - "#, - ) - .bind(self.heartbeat.heartbeat.key()) - .bind(self.heartbeat.heartbeat.timestamp) - .bind(self.heartbeat.heartbeat.coverage_object) - .bind(new_seniority) - .bind(self.heartbeat.heartbeat.timestamp) - .bind(update_reason as i32) - .bind(self.heartbeat.heartbeat.hb_type) - .execute(&mut *exec) - .await?; - } - SeniorityUpdateAction::Update { curr_seniority } => { - sqlx::query( - r#" - UPDATE seniority - SET last_heartbeat = $1 - WHERE - radio_key = $2 AND - seniority_ts = $3 - "#, - ) - .bind(self.heartbeat.heartbeat.timestamp) - .bind(self.heartbeat.heartbeat.key()) - .bind(curr_seniority) - .execute(&mut *exec) - .await?; - } - } - Ok(()) - } -} - #[cfg(test)] mod test { + use crate::seniority::SeniorityUpdateAction; + use super::*; use proto::SeniorityUpdateReason::*; diff --git a/mobile_verifier/src/lib.rs b/mobile_verifier/src/lib.rs index 9a95bccb1..697ce9d5a 100644 --- a/mobile_verifier/src/lib.rs +++ b/mobile_verifier/src/lib.rs @@ -8,6 +8,7 @@ pub mod heartbeats; pub mod radio_threshold; pub mod reward_shares; pub mod rewarder; +pub mod seniority; mod settings; pub mod sp_boosted_rewards_bans; pub mod speedtests; diff --git a/mobile_verifier/src/reward_shares.rs b/mobile_verifier/src/reward_shares.rs index b0efe9f4e..d412147a9 100644 --- a/mobile_verifier/src/reward_shares.rs +++ b/mobile_verifier/src/reward_shares.rs @@ -1,8 +1,9 @@ use crate::{ - coverage::{CoveredHexStream, Seniority}, + coverage::CoveredHexStream, data_session::{HotspotMap, ServiceProviderDataSession}, heartbeats::HeartbeatReward, rewarder::boosted_hex_eligibility::BoostedHexEligibility, + seniority::Seniority, speedtests_average::SpeedtestAverages, subscriber_location::SubscriberValidatedLocations, }; @@ -786,7 +787,7 @@ mod test { use crate::{ cell_type::CellType, - coverage::{CoveredHexStream, HexCoverage, Seniority}, + coverage::{CoveredHexStream, HexCoverage}, data_session::{self, HotspotDataSession, HotspotReward}, heartbeats::{HeartbeatReward, KeyType, OwnedKeyType}, reward_shares, diff --git a/mobile_verifier/src/seniority.rs b/mobile_verifier/src/seniority.rs new file mode 100644 index 000000000..3b252fa1e --- /dev/null +++ b/mobile_verifier/src/seniority.rs @@ -0,0 +1,189 @@ +use chrono::{DateTime, Duration, Utc}; +use file_store::file_sink::FileSinkClient; +use sqlx::{Postgres, Transaction}; +use uuid::Uuid; + +use helium_proto::services::poc_mobile as proto; + +use crate::heartbeats::{KeyType, ValidatedHeartbeat}; + +#[derive(Clone, Debug, PartialEq, sqlx::FromRow)] +pub struct Seniority { + pub uuid: Uuid, + pub seniority_ts: DateTime, + pub last_heartbeat: DateTime, + pub inserted_at: DateTime, + pub update_reason: i32, +} + +impl Seniority { + pub async fn fetch_latest( + key: KeyType<'_>, + exec: &mut Transaction<'_, Postgres>, + ) -> Result, sqlx::Error> { + sqlx::query_as( + "SELECT uuid, seniority_ts, last_heartbeat, inserted_at, update_reason FROM seniority WHERE radio_key = $1 ORDER BY last_heartbeat DESC LIMIT 1", + ) + .bind(key) + .fetch_optional(&mut *exec) + .await + } +} + +pub struct SeniorityUpdate<'a> { + heartbeat: &'a ValidatedHeartbeat, + pub action: SeniorityUpdateAction, +} + +#[derive(Debug, PartialEq)] +pub enum SeniorityUpdateAction { + NoAction, + Insert { + new_seniority: DateTime, + update_reason: proto::SeniorityUpdateReason, + }, + Update { + curr_seniority: DateTime, + }, +} + +impl<'a> SeniorityUpdate<'a> { + pub fn new(heartbeat: &'a ValidatedHeartbeat, action: SeniorityUpdateAction) -> Self { + Self { heartbeat, action } + } + + pub fn determine_update_action( + heartbeat: &'a ValidatedHeartbeat, + coverage_claim_time: DateTime, + modeled_coverage_start: DateTime, + latest_seniority: Option, + ) -> Self { + use proto::SeniorityUpdateReason::*; + + if let Some(prev_seniority) = latest_seniority { + if heartbeat.heartbeat.coverage_object != Some(prev_seniority.uuid) { + if prev_seniority.update_reason == HeartbeatNotSeen as i32 + && coverage_claim_time < prev_seniority.seniority_ts + { + Self::new(heartbeat, SeniorityUpdateAction::NoAction) + } else { + Self::new( + heartbeat, + SeniorityUpdateAction::Insert { + new_seniority: coverage_claim_time, + update_reason: NewCoverageClaimTime, + }, + ) + } + } else if heartbeat.heartbeat.timestamp - prev_seniority.last_heartbeat + > Duration::days(3) + && coverage_claim_time < heartbeat.heartbeat.timestamp + { + Self::new( + heartbeat, + SeniorityUpdateAction::Insert { + new_seniority: heartbeat.heartbeat.timestamp, + update_reason: HeartbeatNotSeen, + }, + ) + } else { + Self::new( + heartbeat, + SeniorityUpdateAction::Update { + curr_seniority: prev_seniority.seniority_ts, + }, + ) + } + } else if heartbeat.heartbeat.timestamp - modeled_coverage_start > Duration::days(3) { + // This will become the default case 72 hours after we launch modeled coverage + Self::new( + heartbeat, + SeniorityUpdateAction::Insert { + new_seniority: heartbeat.heartbeat.timestamp, + update_reason: HeartbeatNotSeen, + }, + ) + } else { + Self::new( + heartbeat, + SeniorityUpdateAction::Insert { + new_seniority: coverage_claim_time, + update_reason: NewCoverageClaimTime, + }, + ) + } + } +} + +impl SeniorityUpdate<'_> { + #[allow(deprecated)] + pub async fn write(&self, seniorities: &FileSinkClient) -> anyhow::Result<()> { + if let SeniorityUpdateAction::Insert { + new_seniority, + update_reason, + } = self.action + { + seniorities + .write( + proto::SeniorityUpdate { + key_type: Some(self.heartbeat.heartbeat.key().into()), + new_seniority_timestamp: new_seniority.timestamp() as u64, + reason: update_reason as i32, + new_seniority_timestamp_ms: new_seniority.timestamp_millis() as u64, + }, + [], + ) + .await?; + } + Ok(()) + } + + pub async fn execute(self, exec: &mut Transaction<'_, Postgres>) -> anyhow::Result<()> { + match self.action { + SeniorityUpdateAction::NoAction => (), + SeniorityUpdateAction::Insert { + new_seniority, + update_reason, + } => { + sqlx::query( + r#" + INSERT INTO seniority + (radio_key, last_heartbeat, uuid, seniority_ts, inserted_at, update_reason, radio_type) + VALUES + ($1, $2, $3, $4, $5, $6, $7) + ON CONFLICT (radio_key, radio_type, seniority_ts) DO UPDATE SET + uuid = EXCLUDED.uuid, + last_heartbeat = EXCLUDED.last_heartbeat, + update_reason = EXCLUDED.update_reason + "#, + ) + .bind(self.heartbeat.heartbeat.key()) + .bind(self.heartbeat.heartbeat.timestamp) + .bind(self.heartbeat.heartbeat.coverage_object) + .bind(new_seniority) + .bind(self.heartbeat.heartbeat.timestamp) + .bind(update_reason as i32) + .bind(self.heartbeat.heartbeat.hb_type) + .execute(&mut *exec) + .await?; + } + SeniorityUpdateAction::Update { curr_seniority } => { + sqlx::query( + r#" + UPDATE seniority + SET last_heartbeat = $1 + WHERE + radio_key = $2 AND + seniority_ts = $3 + "#, + ) + .bind(self.heartbeat.heartbeat.timestamp) + .bind(self.heartbeat.heartbeat.key()) + .bind(curr_seniority) + .execute(&mut *exec) + .await?; + } + } + Ok(()) + } +} diff --git a/mobile_verifier/tests/integrations/boosting_oracles.rs b/mobile_verifier/tests/integrations/boosting_oracles.rs index 1a6c8f4d4..b7de2ddf6 100644 --- a/mobile_verifier/tests/integrations/boosting_oracles.rs +++ b/mobile_verifier/tests/integrations/boosting_oracles.rs @@ -15,14 +15,12 @@ use hex_assignments::{Assignment, HexBoostData}; use mobile_config::boosted_hex_info::BoostedHexes; use mobile_verifier::{ - coverage::{CoverageClaimTimeCache, CoverageObject, CoverageObjectCache, Seniority}, + coverage::{CoverageClaimTimeCache, CoverageObject, CoverageObjectCache}, geofence::GeofenceValidator, - heartbeats::{ - last_location::LocationCache, Heartbeat, HeartbeatReward, SeniorityUpdate, - ValidatedHeartbeat, - }, + heartbeats::{last_location::LocationCache, Heartbeat, HeartbeatReward, ValidatedHeartbeat}, reward_shares::CoverageShares, rewarder::boosted_hex_eligibility::BoostedHexEligibility, + seniority::{Seniority, SeniorityUpdate}, speedtests::Speedtest, speedtests_average::{SpeedtestAverage, SpeedtestAverages}, GatewayResolution, GatewayResolver, diff --git a/mobile_verifier/tests/integrations/modeled_coverage.rs b/mobile_verifier/tests/integrations/modeled_coverage.rs index 2253cbe04..ca372e9da 100644 --- a/mobile_verifier/tests/integrations/modeled_coverage.rs +++ b/mobile_verifier/tests/integrations/modeled_coverage.rs @@ -15,14 +15,14 @@ use hextree::Cell; use mobile_config::boosted_hex_info::{BoostedHexInfo, BoostedHexes}; use mobile_verifier::{ - coverage::{CoverageClaimTimeCache, CoverageObject, CoverageObjectCache, Seniority}, + coverage::{CoverageClaimTimeCache, CoverageObject, CoverageObjectCache}, geofence::GeofenceValidator, heartbeats::{ - last_location::LocationCache, Heartbeat, HeartbeatReward, KeyType, SeniorityUpdate, - ValidatedHeartbeat, + last_location::LocationCache, Heartbeat, HeartbeatReward, KeyType, ValidatedHeartbeat, }, reward_shares::CoverageShares, rewarder::boosted_hex_eligibility::BoostedHexEligibility, + seniority::{Seniority, SeniorityUpdate}, speedtests::Speedtest, speedtests_average::{SpeedtestAverage, SpeedtestAverages}, GatewayResolution, GatewayResolver, IsAuthorized, diff --git a/mobile_verifier/tests/integrations/seniority.rs b/mobile_verifier/tests/integrations/seniority.rs index 54bb59605..906b2c14c 100644 --- a/mobile_verifier/tests/integrations/seniority.rs +++ b/mobile_verifier/tests/integrations/seniority.rs @@ -1,10 +1,8 @@ use chrono::{DateTime, Utc}; use helium_proto::services::poc_mobile::{HeartbeatValidity, SeniorityUpdateReason}; use mobile_verifier::cell_type::CellType; -use mobile_verifier::coverage::Seniority; -use mobile_verifier::heartbeats::{ - HbType, Heartbeat, SeniorityUpdate, SeniorityUpdateAction, ValidatedHeartbeat, -}; +use mobile_verifier::heartbeats::{HbType, Heartbeat, ValidatedHeartbeat}; +use mobile_verifier::seniority::{Seniority, SeniorityUpdate, SeniorityUpdateAction}; use rust_decimal_macros::dec; use sqlx::PgPool; use uuid::Uuid; From 68ba9425a3e3b35f6e08e904c1cb30d87e6aa66b Mon Sep 17 00:00:00 2001 From: Brian Balser Date: Mon, 1 Jul 2024 16:48:41 -0400 Subject: [PATCH 09/23] refactoring seniority update --- mobile_verifier/src/heartbeats/mod.rs | 35 +++++++------ mobile_verifier/src/seniority.rs | 50 ++++++++++++------- .../tests/integrations/boosting_oracles.rs | 2 +- .../tests/integrations/modeled_coverage.rs | 2 +- .../tests/integrations/seniority.rs | 8 +-- 5 files changed, 59 insertions(+), 38 deletions(-) diff --git a/mobile_verifier/src/heartbeats/mod.rs b/mobile_verifier/src/heartbeats/mod.rs index df820e720..c86d114b2 100644 --- a/mobile_verifier/src/heartbeats/mod.rs +++ b/mobile_verifier/src/heartbeats/mod.rs @@ -728,7 +728,7 @@ pub(crate) async fn process_validated_heartbeats( coverage_claim_time, modeled_coverage_start, latest_seniority, - ); + )?; seniority_update.write(seniority_sink).await?; seniority_update.execute(&mut *transaction).await?; } @@ -791,7 +791,7 @@ mod test { } #[test] - fn ensure_first_seniority_causes_update() { + fn ensure_first_seniority_causes_update() -> anyhow::Result<()> { let modeled_coverage_start = "2023-08-20 00:00:00.000000000 UTC".parse().unwrap(); let coverage_claim_time: DateTime = "2023-08-22 00:00:00.000000000 UTC".parse().unwrap(); @@ -805,7 +805,7 @@ mod test { coverage_claim_time, modeled_coverage_start, None, - ); + )?; assert_eq!( seniority_action.action, @@ -814,10 +814,12 @@ mod test { update_reason: NewCoverageClaimTime, } ); + Ok(()) } #[test] - fn ensure_first_seniority_72_hours_after_start_resets_coverage_claim_time() { + fn ensure_first_seniority_72_hours_after_start_resets_coverage_claim_time() -> anyhow::Result<()> + { let modeled_coverage_start = "2023-08-20 00:00:00.000000000 UTC".parse().unwrap(); let coverage_claim_time: DateTime = "2023-08-22 00:00:00.000000000 UTC".parse().unwrap(); @@ -831,7 +833,7 @@ mod test { coverage_claim_time, modeled_coverage_start, None, - ); + )?; assert_eq!( seniority_action.action, @@ -840,10 +842,11 @@ mod test { update_reason: HeartbeatNotSeen, } ); + Ok(()) } #[test] - fn ensure_seniority_updates_on_new_coverage_object() { + fn ensure_seniority_updates_on_new_coverage_object() -> anyhow::Result<()> { let modeled_coverage_start = "2023-08-20 00:00:00.000000000 UTC".parse().unwrap(); let coverage_claim_time: DateTime = "2023-08-22 00:00:00.000000000 UTC".parse().unwrap(); @@ -866,7 +869,7 @@ mod test { new_coverage_claim_time, modeled_coverage_start, Some(latest_seniority.clone()), - ); + )?; assert_eq!( seniority_action.action, @@ -875,10 +878,11 @@ mod test { update_reason: NewCoverageClaimTime, } ); + Ok(()) } #[test] - fn ensure_last_heartbeat_updates_on_same_coverage_object() { + fn ensure_last_heartbeat_updates_on_same_coverage_object() -> anyhow::Result<()> { let modeled_coverage_start = "2023-08-20 00:00:00.000000000 UTC".parse().unwrap(); let coverage_claim_time: DateTime = "2023-08-22 00:00:00.000000000 UTC".parse().unwrap(); @@ -899,7 +903,7 @@ mod test { coverage_claim_time, modeled_coverage_start, Some(latest_seniority.clone()), - ); + )?; assert_eq!( seniority_action.action, @@ -907,10 +911,11 @@ mod test { curr_seniority: coverage_claim_time, } ); + Ok(()) } #[test] - fn ensure_seniority_updates_after_72_hours() { + fn ensure_seniority_updates_after_72_hours() -> anyhow::Result<()> { let modeled_coverage_start = "2023-08-20 00:00:00.000000000 UTC".parse().unwrap(); let coverage_claim_time: DateTime = "2023-08-22 00:00:00.000000000 UTC".parse().unwrap(); @@ -930,7 +935,7 @@ mod test { coverage_claim_time, modeled_coverage_start, Some(latest_seniority), - ); + )?; assert_eq!( seniority_action.action, SeniorityUpdateAction::Insert { @@ -938,10 +943,11 @@ mod test { update_reason: HeartbeatNotSeen, } ); + Ok(()) } #[test] - fn ensure_seniority_updates_after_not_seen_if_in_future() { + fn ensure_seniority_updates_after_not_seen_if_in_future() -> anyhow::Result<()> { let modeled_coverage_start = "2023-08-20 00:00:00.000000000 UTC".parse().unwrap(); let coverage_claim_time: DateTime = "2023-08-22 00:00:00.000000000 UTC".parse().unwrap(); @@ -964,7 +970,7 @@ mod test { new_coverage_claim_time, modeled_coverage_start, Some(latest_seniority.clone()), - ); + )?; assert_eq!( seniority_action.action, SeniorityUpdateAction::Insert { @@ -983,7 +989,8 @@ mod test { new_coverage_claim_time, modeled_coverage_start, Some(latest_seniority), - ); + )?; assert_eq!(seniority_action.action, SeniorityUpdateAction::NoAction); + Ok(()) } } diff --git a/mobile_verifier/src/seniority.rs b/mobile_verifier/src/seniority.rs index 3b252fa1e..5a8dd0f85 100644 --- a/mobile_verifier/src/seniority.rs +++ b/mobile_verifier/src/seniority.rs @@ -31,7 +31,9 @@ impl Seniority { } pub struct SeniorityUpdate<'a> { - heartbeat: &'a ValidatedHeartbeat, + key: KeyType<'a>, + heartbeat_ts: DateTime, + uuid: Uuid, pub action: SeniorityUpdateAction, } @@ -48,8 +50,19 @@ pub enum SeniorityUpdateAction { } impl<'a> SeniorityUpdate<'a> { - pub fn new(heartbeat: &'a ValidatedHeartbeat, action: SeniorityUpdateAction) -> Self { - Self { heartbeat, action } + pub fn from( + heartbeat: &'a ValidatedHeartbeat, + action: SeniorityUpdateAction, + ) -> anyhow::Result { + Ok(Self { + key: heartbeat.heartbeat.key(), + heartbeat_ts: heartbeat.heartbeat.timestamp, + uuid: heartbeat + .heartbeat + .coverage_object + .ok_or_else(|| anyhow::anyhow!("invalid heartbeat, no coverage object found"))?, + action, + }) } pub fn determine_update_action( @@ -57,17 +70,18 @@ impl<'a> SeniorityUpdate<'a> { coverage_claim_time: DateTime, modeled_coverage_start: DateTime, latest_seniority: Option, - ) -> Self { + ) -> anyhow::Result { use proto::SeniorityUpdateReason::*; if let Some(prev_seniority) = latest_seniority { if heartbeat.heartbeat.coverage_object != Some(prev_seniority.uuid) { + // TODO need to think about this if prev_seniority.update_reason == HeartbeatNotSeen as i32 && coverage_claim_time < prev_seniority.seniority_ts { - Self::new(heartbeat, SeniorityUpdateAction::NoAction) + Self::from(heartbeat, SeniorityUpdateAction::NoAction) } else { - Self::new( + Self::from( heartbeat, SeniorityUpdateAction::Insert { new_seniority: coverage_claim_time, @@ -79,7 +93,7 @@ impl<'a> SeniorityUpdate<'a> { > Duration::days(3) && coverage_claim_time < heartbeat.heartbeat.timestamp { - Self::new( + Self::from( heartbeat, SeniorityUpdateAction::Insert { new_seniority: heartbeat.heartbeat.timestamp, @@ -87,7 +101,7 @@ impl<'a> SeniorityUpdate<'a> { }, ) } else { - Self::new( + Self::from( heartbeat, SeniorityUpdateAction::Update { curr_seniority: prev_seniority.seniority_ts, @@ -96,7 +110,7 @@ impl<'a> SeniorityUpdate<'a> { } } else if heartbeat.heartbeat.timestamp - modeled_coverage_start > Duration::days(3) { // This will become the default case 72 hours after we launch modeled coverage - Self::new( + Self::from( heartbeat, SeniorityUpdateAction::Insert { new_seniority: heartbeat.heartbeat.timestamp, @@ -104,7 +118,7 @@ impl<'a> SeniorityUpdate<'a> { }, ) } else { - Self::new( + Self::from( heartbeat, SeniorityUpdateAction::Insert { new_seniority: coverage_claim_time, @@ -126,7 +140,7 @@ impl SeniorityUpdate<'_> { seniorities .write( proto::SeniorityUpdate { - key_type: Some(self.heartbeat.heartbeat.key().into()), + key_type: Some(self.key.into()), new_seniority_timestamp: new_seniority.timestamp() as u64, reason: update_reason as i32, new_seniority_timestamp_ms: new_seniority.timestamp_millis() as u64, @@ -157,13 +171,13 @@ impl SeniorityUpdate<'_> { update_reason = EXCLUDED.update_reason "#, ) - .bind(self.heartbeat.heartbeat.key()) - .bind(self.heartbeat.heartbeat.timestamp) - .bind(self.heartbeat.heartbeat.coverage_object) + .bind(self.key) + .bind(self.heartbeat_ts) + .bind(self.uuid) .bind(new_seniority) - .bind(self.heartbeat.heartbeat.timestamp) + .bind(self.heartbeat_ts) .bind(update_reason as i32) - .bind(self.heartbeat.heartbeat.hb_type) + .bind(self.key.hb_type()) .execute(&mut *exec) .await?; } @@ -177,8 +191,8 @@ impl SeniorityUpdate<'_> { seniority_ts = $3 "#, ) - .bind(self.heartbeat.heartbeat.timestamp) - .bind(self.heartbeat.heartbeat.key()) + .bind(self.heartbeat_ts) + .bind(self.key) .bind(curr_seniority) .execute(&mut *exec) .await?; diff --git a/mobile_verifier/tests/integrations/boosting_oracles.rs b/mobile_verifier/tests/integrations/boosting_oracles.rs index b7de2ddf6..7cd41772a 100644 --- a/mobile_verifier/tests/integrations/boosting_oracles.rs +++ b/mobile_verifier/tests/integrations/boosting_oracles.rs @@ -382,7 +382,7 @@ async fn test_footfall_and_urbanization_and_landtype(pool: PgPool) -> anyhow::Re coverage_claim_time.unwrap(), epoch.start, latest_seniority, - ); + )?; seniority_update.execute(&mut transaction).await?; heartbeat.save(&mut transaction).await?; } diff --git a/mobile_verifier/tests/integrations/modeled_coverage.rs b/mobile_verifier/tests/integrations/modeled_coverage.rs index ca372e9da..6ae06706a 100644 --- a/mobile_verifier/tests/integrations/modeled_coverage.rs +++ b/mobile_verifier/tests/integrations/modeled_coverage.rs @@ -435,7 +435,7 @@ async fn process_input( coverage_claim_time.unwrap(), epoch.start, latest_seniority, - ); + )?; seniority_update.execute(&mut transaction).await?; heartbeat.save(&mut transaction).await?; } diff --git a/mobile_verifier/tests/integrations/seniority.rs b/mobile_verifier/tests/integrations/seniority.rs index 906b2c14c..85cb95dec 100644 --- a/mobile_verifier/tests/integrations/seniority.rs +++ b/mobile_verifier/tests/integrations/seniority.rs @@ -36,13 +36,13 @@ async fn test_seniority_updates(pool: PgPool) -> anyhow::Result<()> { assert_eq!(latest_seniority, None); - let action1 = SeniorityUpdate::new( + let action1 = SeniorityUpdate::from( &heartbeat, SeniorityUpdateAction::Insert { new_seniority: "2023-08-25 00:00:00.000000000 UTC".parse().unwrap(), update_reason: SeniorityUpdateReason::HeartbeatNotSeen, }, - ); + )?; action1.execute(&mut transaction).await?; @@ -60,12 +60,12 @@ async fn test_seniority_updates(pool: PgPool) -> anyhow::Result<()> { heartbeat.heartbeat.timestamp = "2023-08-24 00:00:00.000000000 UTC".parse().unwrap(); - let action2 = SeniorityUpdate::new( + let action2 = SeniorityUpdate::from( &heartbeat, SeniorityUpdateAction::Update { curr_seniority: "2023-08-25 00:00:00.000000000 UTC".parse().unwrap(), }, - ); + )?; action2.execute(&mut transaction).await?; From 43053a623bccc4a62b42090242fab30c8a99982e Mon Sep 17 00:00:00 2001 From: Brian Balser Date: Tue, 2 Jul 2024 08:32:39 -0400 Subject: [PATCH 10/23] reset radio seniority when receiving a service provider ban --- mobile_verifier/src/cli/server.rs | 3 +- mobile_verifier/src/heartbeats/mod.rs | 7 ++ mobile_verifier/src/seniority.rs | 14 +++ .../src/sp_boosted_rewards_bans.rs | 85 ++++++++++++++----- 4 files changed, 87 insertions(+), 22 deletions(-) diff --git a/mobile_verifier/src/cli/server.rs b/mobile_verifier/src/cli/server.rs index dd32b1a77..6f4effad2 100644 --- a/mobile_verifier/src/cli/server.rs +++ b/mobile_verifier/src/cli/server.rs @@ -125,7 +125,7 @@ impl Cmd { report_ingest.clone(), gateway_client.clone(), valid_heartbeats, - seniority_updates, + seniority_updates.clone(), usa_and_mexico_geofence, ) .await?, @@ -190,6 +190,7 @@ impl Cmd { report_ingest, auth_client, settings, + seniority_updates, ) .await?, ) diff --git a/mobile_verifier/src/heartbeats/mod.rs b/mobile_verifier/src/heartbeats/mod.rs index c86d114b2..747a671e0 100644 --- a/mobile_verifier/src/heartbeats/mod.rs +++ b/mobile_verifier/src/heartbeats/mod.rs @@ -128,6 +128,13 @@ impl OwnedKeyType { } } + pub fn to_ref(&self) -> KeyType { + match self { + OwnedKeyType::Cbrs(cbsd_id) => KeyType::Cbrs(cbsd_id), + OwnedKeyType::Wifi(pubkey) => KeyType::Wifi(pubkey), + } + } + pub fn is_cbrs(&self) -> bool { matches!(self, Self::Cbrs(_)) } diff --git a/mobile_verifier/src/seniority.rs b/mobile_verifier/src/seniority.rs index 5a8dd0f85..bb2b5fbce 100644 --- a/mobile_verifier/src/seniority.rs +++ b/mobile_verifier/src/seniority.rs @@ -50,6 +50,20 @@ pub enum SeniorityUpdateAction { } impl<'a> SeniorityUpdate<'a> { + pub fn new( + key: KeyType<'a>, + heartbeat_ts: DateTime, + uuid: Uuid, + action: SeniorityUpdateAction, + ) -> Self { + Self { + key, + heartbeat_ts, + uuid, + action, + } + } + pub fn from( heartbeat: &'a ValidatedHeartbeat, action: SeniorityUpdateAction, diff --git a/mobile_verifier/src/sp_boosted_rewards_bans.rs b/mobile_verifier/src/sp_boosted_rewards_bans.rs index 99a8739eb..5eea361fd 100644 --- a/mobile_verifier/src/sp_boosted_rewards_bans.rs +++ b/mobile_verifier/src/sp_boosted_rewards_bans.rs @@ -15,9 +15,9 @@ use helium_proto::services::{ mobile_config::NetworkKeyRole, poc_mobile::{ service_provider_boosted_rewards_banned_radio_req_v1::{ - KeyType, SpBoostedRewardsBannedRadioReason, + KeyType as ProtoKeyType, SpBoostedRewardsBannedRadioReason, }, - ServiceProviderBoostedRewardsBannedRadioIngestReportV1, + SeniorityUpdateReason, ServiceProviderBoostedRewardsBannedRadioIngestReportV1, ServiceProviderBoostedRewardsBannedRadioVerificationStatus, VerifiedServiceProviderBoostedRewardsBannedRadioIngestReportV1, }, @@ -27,19 +27,31 @@ use sqlx::{PgPool, Postgres, Transaction}; use task_manager::{ManagedTask, TaskManager}; use tokio::sync::mpsc::Receiver; -use crate::Settings; +use crate::{ + heartbeats::OwnedKeyType, + seniority::{Seniority, SeniorityUpdate, SeniorityUpdateAction}, + Settings, +}; const CLEANUP_DAYS: i64 = 7; struct BannedRadioReport { received_timestamp: DateTime, pubkey: PublicKeyBinary, - radio_type: String, - radio_key: String, + key: OwnedKeyType, until: DateTime, reason: SpBoostedRewardsBannedRadioReason, } +impl BannedRadioReport { + fn radio_type(&self) -> &'static str { + match self.key { + OwnedKeyType::Cbrs(_) => "cbrs", + OwnedKeyType::Wifi(_) => "wifi", + } + } +} + impl TryFrom for BannedRadioReport { type Error = anyhow::Error; @@ -52,9 +64,11 @@ impl TryFrom for BannedR let reason = report.reason(); - let (radio_type, radio_key) = match report.key_type { - Some(KeyType::CbsdId(cbsd_id)) => ("cbrs", cbsd_id), - Some(KeyType::HotspotKey(bytes)) => ("wifi", PublicKeyBinary::from(bytes).to_string()), + let key = match report.key_type { + Some(ProtoKeyType::CbsdId(cbsd_id)) => OwnedKeyType::Cbrs(cbsd_id), + Some(ProtoKeyType::HotspotKey(bytes)) => { + OwnedKeyType::Wifi(PublicKeyBinary::from(bytes)) + } None => anyhow::bail!("Invalid keytype"), }; @@ -66,8 +80,7 @@ impl TryFrom for BannedR anyhow::anyhow!("invalid received timestamp, {}", value.received_timestamp) })?, pubkey: report.pub_key.into(), - radio_type: radio_type.to_string(), - radio_key, + key, until: Utc .timestamp_opt(report.until as i64, 0) .single() @@ -105,6 +118,7 @@ pub struct ServiceProviderBoostedRewardsBanIngestor { authorization_verifier: AV, receiver: Receiver>, verified_sink: FileSinkClient, + seniority_update_sink: FileSinkClient, } impl ManagedTask for ServiceProviderBoostedRewardsBanIngestor @@ -136,6 +150,7 @@ where file_store: FileStore, authorization_verifier: AV, settings: &Settings, + seniority_update_sink: FileSinkClient, ) -> anyhow::Result { let (verified_sink, verified_sink_server) = file_sink::FileSinkBuilder::new( FileType::VerifiedServiceProviderBoostedRewardsBannedRadioIngestReport, @@ -165,6 +180,7 @@ where authorization_verifier, receiver, verified_sink, + seniority_update_sink, }; Ok(TaskManager::builder() @@ -210,6 +226,7 @@ where .await?; self.verified_sink.commit().await?; + self.seniority_update_sink.commit().await?; Ok(()) } @@ -223,13 +240,14 @@ where let is_authorized = self.is_authorized(&report.pubkey).await?; if is_authorized { - db::update_report(transaction, report).await?; + db::update_report(transaction, &report).await?; + self.update_seniority(transaction, &report).await?; } let status = match is_authorized { - true => ServiceProviderBoostedRewardsBannedRadioVerificationStatus::SpBoostedRewardsBanValid, - false => ServiceProviderBoostedRewardsBannedRadioVerificationStatus::SpBoostedRewardsBanInvalidCarrierKey, - }; + true => ServiceProviderBoostedRewardsBannedRadioVerificationStatus::SpBoostedRewardsBanValid, + false => ServiceProviderBoostedRewardsBannedRadioVerificationStatus::SpBoostedRewardsBanInvalidCarrierKey, + }; let verified_report = VerifiedServiceProviderBoostedRewardsBannedRadioIngestReportV1 { report: Some(ingest), @@ -250,6 +268,31 @@ where .await .map_err(anyhow::Error::from) } + + async fn update_seniority( + &self, + transaction: &mut Transaction<'_, Postgres>, + report: &BannedRadioReport, + ) -> anyhow::Result<()> { + if let Some(current_seniority) = + Seniority::fetch_latest(report.key.to_ref(), transaction).await? + { + let seniority_update = SeniorityUpdate::new( + report.key.to_ref(), + current_seniority.last_heartbeat, + current_seniority.uuid, + SeniorityUpdateAction::Insert { + new_seniority: Utc::now(), + update_reason: SeniorityUpdateReason::ServiceProviderBan, + }, + ); + + seniority_update.write(&self.seniority_update_sink).await?; + seniority_update.execute(transaction).await?; + } + + Ok(()) + } } pub async fn clear_bans( @@ -314,7 +357,7 @@ pub mod db { pub(super) async fn update_report( transaction: &mut Transaction<'_, Postgres>, - report: BannedRadioReport, + report: &BannedRadioReport, ) -> anyhow::Result<()> { match report.reason { SpBoostedRewardsBannedRadioReason::Unbanned => { @@ -326,7 +369,7 @@ pub mod db { async fn save( transaction: &mut Transaction<'_, Postgres>, - report: BannedRadioReport, + report: &BannedRadioReport, ) -> anyhow::Result<()> { sqlx::query( r#" @@ -334,8 +377,8 @@ pub mod db { VALUES($1,$2,$3,$4) "#, ) - .bind(report.radio_type) - .bind(report.radio_key) + .bind(report.radio_type()) + .bind(&report.key) .bind(report.received_timestamp) .bind(report.until) .execute(transaction) @@ -346,7 +389,7 @@ pub mod db { async fn invalidate_all_before( transaction: &mut Transaction<'_, Postgres>, - report: BannedRadioReport, + report: &BannedRadioReport, ) -> anyhow::Result<()> { sqlx::query( r#" @@ -357,8 +400,8 @@ pub mod db { AND received_timestamp <= $3 "#, ) - .bind(report.radio_type) - .bind(report.radio_key) + .bind(report.radio_type()) + .bind(&report.key) .bind(report.received_timestamp) .execute(transaction) .await From 8b395f44b1131b2b9848757df9624b2a293c709c Mon Sep 17 00:00:00 2001 From: Brian Balser Date: Tue, 2 Jul 2024 09:05:54 -0400 Subject: [PATCH 11/23] remove deprecated modeled coverage start date --- mobile_verifier/src/heartbeats/cbrs.rs | 5 -- mobile_verifier/src/heartbeats/mod.rs | 48 +------------------ mobile_verifier/src/heartbeats/wifi.rs | 5 -- mobile_verifier/src/seniority.rs | 11 ----- mobile_verifier/src/settings.rs | 1 - .../tests/integrations/boosting_oracles.rs | 1 - .../tests/integrations/modeled_coverage.rs | 1 - 7 files changed, 2 insertions(+), 70 deletions(-) diff --git a/mobile_verifier/src/heartbeats/cbrs.rs b/mobile_verifier/src/heartbeats/cbrs.rs index a95ef1a2a..fa3cdf3a6 100644 --- a/mobile_verifier/src/heartbeats/cbrs.rs +++ b/mobile_verifier/src/heartbeats/cbrs.rs @@ -28,7 +28,6 @@ pub struct CbrsHeartbeatDaemon { pool: sqlx::Pool, gateway_info_resolver: GIR, heartbeats: Receiver>, - modeled_coverage_start: DateTime, max_distance_to_asserted: u32, max_distance_to_coverage: u32, heartbeat_sink: FileSinkClient, @@ -66,7 +65,6 @@ where pool, gateway_resolver, cbrs_heartbeats, - settings.modeled_coverage_start, settings.max_asserted_distance_deviation, settings.max_distance_from_coverage, valid_heartbeats, @@ -85,7 +83,6 @@ where pool: sqlx::Pool, gateway_info_resolver: GIR, heartbeats: Receiver>, - modeled_coverage_start: DateTime, max_distance_to_asserted: u32, max_distance_to_coverage: u32, heartbeat_sink: FileSinkClient, @@ -96,7 +93,6 @@ where pool, gateway_info_resolver, heartbeats, - modeled_coverage_start, max_distance_to_asserted, max_distance_to_coverage, heartbeat_sink, @@ -181,7 +177,6 @@ where ), heartbeat_cache, coverage_claim_time_cache, - self.modeled_coverage_start, &self.heartbeat_sink, &self.seniority_sink, &mut transaction, diff --git a/mobile_verifier/src/heartbeats/mod.rs b/mobile_verifier/src/heartbeats/mod.rs index 747a671e0..2e21a68a9 100644 --- a/mobile_verifier/src/heartbeats/mod.rs +++ b/mobile_verifier/src/heartbeats/mod.rs @@ -707,7 +707,6 @@ pub(crate) async fn process_validated_heartbeats( validated_heartbeats: impl Stream>, heartbeat_cache: &Cache<(String, DateTime), ()>, coverage_claim_time_cache: &CoverageClaimTimeCache, - modeled_coverage_start: DateTime, heartbeat_sink: &FileSinkClient, seniority_sink: &FileSinkClient, transaction: &mut Transaction<'_, Postgres>, @@ -733,7 +732,6 @@ pub(crate) async fn process_validated_heartbeats( let seniority_update = SeniorityUpdate::determine_update_action( &validated_heartbeat, coverage_claim_time, - modeled_coverage_start, latest_seniority, )?; seniority_update.write(seniority_sink).await?; @@ -799,7 +797,6 @@ mod test { #[test] fn ensure_first_seniority_causes_update() -> anyhow::Result<()> { - let modeled_coverage_start = "2023-08-20 00:00:00.000000000 UTC".parse().unwrap(); let coverage_claim_time: DateTime = "2023-08-22 00:00:00.000000000 UTC".parse().unwrap(); let coverage_object = Uuid::new_v4(); @@ -807,12 +804,8 @@ mod test { let received_timestamp: DateTime = "2023-08-23 00:00:00.000000000 UTC".parse().unwrap(); let new_heartbeat = heartbeat(received_timestamp, coverage_object); - let seniority_action = SeniorityUpdate::determine_update_action( - &new_heartbeat, - coverage_claim_time, - modeled_coverage_start, - None, - )?; + let seniority_action = + SeniorityUpdate::determine_update_action(&new_heartbeat, coverage_claim_time, None)?; assert_eq!( seniority_action.action, @@ -824,37 +817,8 @@ mod test { Ok(()) } - #[test] - fn ensure_first_seniority_72_hours_after_start_resets_coverage_claim_time() -> anyhow::Result<()> - { - let modeled_coverage_start = "2023-08-20 00:00:00.000000000 UTC".parse().unwrap(); - let coverage_claim_time: DateTime = - "2023-08-22 00:00:00.000000000 UTC".parse().unwrap(); - let coverage_object = Uuid::new_v4(); - - let received_timestamp: DateTime = - "2023-08-23 00:00:01.000000000 UTC".parse().unwrap(); - let new_heartbeat = heartbeat(received_timestamp, coverage_object); - let seniority_action = SeniorityUpdate::determine_update_action( - &new_heartbeat, - coverage_claim_time, - modeled_coverage_start, - None, - )?; - - assert_eq!( - seniority_action.action, - SeniorityUpdateAction::Insert { - new_seniority: received_timestamp, - update_reason: HeartbeatNotSeen, - } - ); - Ok(()) - } - #[test] fn ensure_seniority_updates_on_new_coverage_object() -> anyhow::Result<()> { - let modeled_coverage_start = "2023-08-20 00:00:00.000000000 UTC".parse().unwrap(); let coverage_claim_time: DateTime = "2023-08-22 00:00:00.000000000 UTC".parse().unwrap(); let coverage_object = Uuid::new_v4(); @@ -874,7 +838,6 @@ mod test { let seniority_action = SeniorityUpdate::determine_update_action( &new_heartbeat, new_coverage_claim_time, - modeled_coverage_start, Some(latest_seniority.clone()), )?; @@ -890,7 +853,6 @@ mod test { #[test] fn ensure_last_heartbeat_updates_on_same_coverage_object() -> anyhow::Result<()> { - let modeled_coverage_start = "2023-08-20 00:00:00.000000000 UTC".parse().unwrap(); let coverage_claim_time: DateTime = "2023-08-22 00:00:00.000000000 UTC".parse().unwrap(); let coverage_object = Uuid::new_v4(); @@ -908,7 +870,6 @@ mod test { let seniority_action = SeniorityUpdate::determine_update_action( &new_heartbeat, coverage_claim_time, - modeled_coverage_start, Some(latest_seniority.clone()), )?; @@ -923,7 +884,6 @@ mod test { #[test] fn ensure_seniority_updates_after_72_hours() -> anyhow::Result<()> { - let modeled_coverage_start = "2023-08-20 00:00:00.000000000 UTC".parse().unwrap(); let coverage_claim_time: DateTime = "2023-08-22 00:00:00.000000000 UTC".parse().unwrap(); let last_heartbeat: DateTime = "2023-08-23 00:00:00.000000000 UTC".parse().unwrap(); @@ -940,7 +900,6 @@ mod test { let seniority_action = SeniorityUpdate::determine_update_action( &new_heartbeat, coverage_claim_time, - modeled_coverage_start, Some(latest_seniority), )?; assert_eq!( @@ -955,7 +914,6 @@ mod test { #[test] fn ensure_seniority_updates_after_not_seen_if_in_future() -> anyhow::Result<()> { - let modeled_coverage_start = "2023-08-20 00:00:00.000000000 UTC".parse().unwrap(); let coverage_claim_time: DateTime = "2023-08-22 00:00:00.000000000 UTC".parse().unwrap(); let coverage_object = Uuid::new_v4(); @@ -975,7 +933,6 @@ mod test { let seniority_action = SeniorityUpdate::determine_update_action( &new_heartbeat, new_coverage_claim_time, - modeled_coverage_start, Some(latest_seniority.clone()), )?; assert_eq!( @@ -994,7 +951,6 @@ mod test { let seniority_action = SeniorityUpdate::determine_update_action( &new_heartbeat, new_coverage_claim_time, - modeled_coverage_start, Some(latest_seniority), )?; assert_eq!(seniority_action.action, SeniorityUpdateAction::NoAction); diff --git a/mobile_verifier/src/heartbeats/wifi.rs b/mobile_verifier/src/heartbeats/wifi.rs index 295a1c8df..7456af788 100644 --- a/mobile_verifier/src/heartbeats/wifi.rs +++ b/mobile_verifier/src/heartbeats/wifi.rs @@ -27,7 +27,6 @@ pub struct WifiHeartbeatDaemon { pool: sqlx::Pool, gateway_info_resolver: GIR, heartbeats: Receiver>, - modeled_coverage_start: DateTime, max_distance_to_asserted: u32, max_distance_to_coverage: u32, heartbeat_sink: FileSinkClient, @@ -64,7 +63,6 @@ where pool, gateway_resolver, wifi_heartbeats, - settings.modeled_coverage_start, settings.max_asserted_distance_deviation, settings.max_distance_from_coverage, valid_heartbeats, @@ -83,7 +81,6 @@ where pool: sqlx::Pool, gateway_info_resolver: GIR, heartbeats: Receiver>, - modeled_coverage_start: DateTime, max_distance_to_asserted: u32, max_distance_to_coverage: u32, heartbeat_sink: FileSinkClient, @@ -94,7 +91,6 @@ where pool, gateway_info_resolver, heartbeats, - modeled_coverage_start, max_distance_to_asserted, max_distance_to_coverage, heartbeat_sink, @@ -172,7 +168,6 @@ where ), heartbeat_cache, coverage_claim_time_cache, - self.modeled_coverage_start, &self.heartbeat_sink, &self.seniority_sink, &mut transaction, diff --git a/mobile_verifier/src/seniority.rs b/mobile_verifier/src/seniority.rs index bb2b5fbce..60b45e794 100644 --- a/mobile_verifier/src/seniority.rs +++ b/mobile_verifier/src/seniority.rs @@ -82,14 +82,12 @@ impl<'a> SeniorityUpdate<'a> { pub fn determine_update_action( heartbeat: &'a ValidatedHeartbeat, coverage_claim_time: DateTime, - modeled_coverage_start: DateTime, latest_seniority: Option, ) -> anyhow::Result { use proto::SeniorityUpdateReason::*; if let Some(prev_seniority) = latest_seniority { if heartbeat.heartbeat.coverage_object != Some(prev_seniority.uuid) { - // TODO need to think about this if prev_seniority.update_reason == HeartbeatNotSeen as i32 && coverage_claim_time < prev_seniority.seniority_ts { @@ -122,15 +120,6 @@ impl<'a> SeniorityUpdate<'a> { }, ) } - } else if heartbeat.heartbeat.timestamp - modeled_coverage_start > Duration::days(3) { - // This will become the default case 72 hours after we launch modeled coverage - Self::from( - heartbeat, - SeniorityUpdateAction::Insert { - new_seniority: heartbeat.heartbeat.timestamp, - update_reason: HeartbeatNotSeen, - }, - ) } else { Self::from( heartbeat, diff --git a/mobile_verifier/src/settings.rs b/mobile_verifier/src/settings.rs index 4a5c681b5..1a3949ebd 100644 --- a/mobile_verifier/src/settings.rs +++ b/mobile_verifier/src/settings.rs @@ -34,7 +34,6 @@ pub struct Settings { pub config_client: mobile_config::ClientSettings, #[serde(default = "default_start_after")] pub start_after: DateTime, - pub modeled_coverage_start: DateTime, /// Max distance in meters between the heartbeat and all of the hexes in /// its respective coverage object #[serde(default = "default_max_distance_from_coverage")] diff --git a/mobile_verifier/tests/integrations/boosting_oracles.rs b/mobile_verifier/tests/integrations/boosting_oracles.rs index 7cd41772a..0cf60efeb 100644 --- a/mobile_verifier/tests/integrations/boosting_oracles.rs +++ b/mobile_verifier/tests/integrations/boosting_oracles.rs @@ -380,7 +380,6 @@ async fn test_footfall_and_urbanization_and_landtype(pool: PgPool) -> anyhow::Re let seniority_update = SeniorityUpdate::determine_update_action( &heartbeat, coverage_claim_time.unwrap(), - epoch.start, latest_seniority, )?; seniority_update.execute(&mut transaction).await?; diff --git a/mobile_verifier/tests/integrations/modeled_coverage.rs b/mobile_verifier/tests/integrations/modeled_coverage.rs index 6ae06706a..4d5c64432 100644 --- a/mobile_verifier/tests/integrations/modeled_coverage.rs +++ b/mobile_verifier/tests/integrations/modeled_coverage.rs @@ -433,7 +433,6 @@ async fn process_input( let seniority_update = SeniorityUpdate::determine_update_action( &heartbeat, coverage_claim_time.unwrap(), - epoch.start, latest_seniority, )?; seniority_update.execute(&mut transaction).await?; From 6949a11654086885632e307a04ae10082ed10e37 Mon Sep 17 00:00:00 2001 From: Brian Balser Date: Tue, 2 Jul 2024 13:12:40 -0400 Subject: [PATCH 12/23] Fixing bugs adding tests --- .../src/sp_boosted_rewards_bans.rs | 211 +++++++++++++++++- 1 file changed, 202 insertions(+), 9 deletions(-) diff --git a/mobile_verifier/src/sp_boosted_rewards_bans.rs b/mobile_verifier/src/sp_boosted_rewards_bans.rs index 5eea361fd..888803201 100644 --- a/mobile_verifier/src/sp_boosted_rewards_bans.rs +++ b/mobile_verifier/src/sp_boosted_rewards_bans.rs @@ -28,7 +28,7 @@ use task_manager::{ManagedTask, TaskManager}; use tokio::sync::mpsc::Receiver; use crate::{ - heartbeats::OwnedKeyType, + heartbeats::{HbType, OwnedKeyType}, seniority::{Seniority, SeniorityUpdate, SeniorityUpdateAction}, Settings, }; @@ -44,10 +44,10 @@ struct BannedRadioReport { } impl BannedRadioReport { - fn radio_type(&self) -> &'static str { + fn radio_type(&self) -> HbType { match self.key { - OwnedKeyType::Cbrs(_) => "cbrs", - OwnedKeyType::Wifi(_) => "wifi", + OwnedKeyType::Cbrs(_) => HbType::Cbrs, + OwnedKeyType::Wifi(_) => HbType::Wifi, } } } @@ -318,19 +318,19 @@ pub mod db { r#" SELECT radio_type, radio_key FROM sp_boosted_rewards_bans - WHERE until < $1 or invalidated_at < $1 + WHERE until > $1 + AND (invalidated_at > $1 OR invalidated_at IS NULL) "#, ) .bind(date_time) .fetch(pool) .map_err(anyhow::Error::from) .try_fold(BannedRadios::default(), |mut set, row| async move { - let radio_type = row.get::<&str, &str>("radio_type"); + let radio_type = row.get::("radio_type"); let radio_key = row.get::("radio_key"); match radio_type { - "wifi" => set.insert_wifi(PublicKeyBinary::from_str(&radio_key)?), - "cbrs" => set.insert_cbrs(radio_key), - _ => anyhow::bail!("Inavlid radio type: {}", radio_type), + HbType::Wifi => set.insert_wifi(PublicKeyBinary::from_str(&radio_key)?), + HbType::Cbrs => set.insert_cbrs(radio_key), }; Ok(set) @@ -409,3 +409,196 @@ pub mod db { .map_err(anyhow::Error::from) } } + +#[cfg(test)] +mod tests { + use chrono::Duration; + use file_store::file_sink::Message; + use helium_crypto::{KeyTag, Keypair, PublicKey}; + use helium_proto::services::poc_mobile::ServiceProviderBoostedRewardsBannedRadioReqV1; + use rand::rngs::OsRng; + use tokio::sync::mpsc; + + use super::*; + + #[derive(thiserror::Error, Debug)] + enum TestError {} + struct AllVerified; + + #[async_trait::async_trait] + impl AuthorizationVerifier for AllVerified { + type Error = TestError; + + async fn verify_authorized_key( + &self, + _pubkey: &PublicKeyBinary, + _role: helium_proto::services::mobile_config::NetworkKeyRole, + ) -> Result { + Ok(true) + } + } + + struct TestSetup { + ingestor: ServiceProviderBoostedRewardsBanIngestor, + _verified_receiver: Receiver, + _seniority_receiver: Receiver, + } + + impl TestSetup { + fn create(pool: PgPool, verifier: AV) -> Self { + let (_fip_sender, fip_receiver) = mpsc::channel(1); + let (verified_sender, verified_receiver) = mpsc::channel(5); + let (seniority_sender, seniority_receiver) = mpsc::channel(5); + + let verified_sink = FileSinkClient::new(verified_sender, "verified"); + let seniority_sink = FileSinkClient::new(seniority_sender, "seniority"); + + let ingestor = ServiceProviderBoostedRewardsBanIngestor { + pool, + authorization_verifier: verifier, + receiver: fip_receiver, + verified_sink, + seniority_update_sink: seniority_sink, + }; + + Self { + ingestor, + _verified_receiver: verified_receiver, + _seniority_receiver: seniority_receiver, + } + } + } + + fn wifi_ban_report( + key: &PublicKey, + until: DateTime, + reason: SpBoostedRewardsBannedRadioReason, + ) -> ServiceProviderBoostedRewardsBannedRadioIngestReportV1 { + let signer_keypair = generate_keypair(); + + ServiceProviderBoostedRewardsBannedRadioIngestReportV1 { + received_timestamp: Utc::now().timestamp_millis() as u64, + report: Some(ServiceProviderBoostedRewardsBannedRadioReqV1 { + pub_key: signer_keypair.public_key().into(), + reason: reason as i32, + until: until.timestamp() as u64, + signature: vec![], + key_type: Some(ProtoKeyType::HotspotKey(key.into())), + }), + } + } + + fn cbrs_ban_report( + cbsd_id: String, + until: DateTime, + reason: SpBoostedRewardsBannedRadioReason, + ) -> ServiceProviderBoostedRewardsBannedRadioIngestReportV1 { + let signer_keypair = generate_keypair(); + + ServiceProviderBoostedRewardsBannedRadioIngestReportV1 { + received_timestamp: Utc::now().timestamp_millis() as u64, + report: Some(ServiceProviderBoostedRewardsBannedRadioReqV1 { + pub_key: signer_keypair.public_key().into(), + reason: reason as i32, + until: until.timestamp() as u64, + signature: vec![], + key_type: Some(ProtoKeyType::CbsdId(cbsd_id)), + }), + } + } + + #[sqlx::test] + async fn wifi_radio_can_get_banned_and_unbanned(pool: PgPool) -> anyhow::Result<()> { + let setup = TestSetup::create(pool.clone(), AllVerified); + let keypair = generate_keypair(); + let cbsd_id = "cbsd-id-1".to_string(); + + let report = cbrs_ban_report( + cbsd_id.clone(), + Utc::now() + Duration::days(7), + SpBoostedRewardsBannedRadioReason::NoNetworkCorrelation, + ); + + let mut transaction = pool.begin().await?; + setup + .ingestor + .process_ingest_report(&mut transaction, report) + .await?; + transaction.commit().await?; + + let banned_radios = db::get_banned_radios(&pool, Utc::now()).await?; + let result = + banned_radios.contains(&keypair.public_key().to_owned().into(), Some(&cbsd_id)); + + assert!(result); + + let report = cbrs_ban_report( + cbsd_id.clone(), + Utc::now() - Duration::days(7), + SpBoostedRewardsBannedRadioReason::Unbanned, + ); + + let mut transaction = pool.begin().await?; + setup + .ingestor + .process_ingest_report(&mut transaction, report) + .await?; + transaction.commit().await?; + + let banned_radios = db::get_banned_radios(&pool, Utc::now()).await?; + let result = + banned_radios.contains(&keypair.public_key().to_owned().into(), Some(&cbsd_id)); + + assert!(!result); + + Ok(()) + } + + #[sqlx::test] + async fn cbrs_radio_can_get_banned_and_unbanned(pool: PgPool) -> anyhow::Result<()> { + let setup = TestSetup::create(pool.clone(), AllVerified); + let keypair = generate_keypair(); + + let report = wifi_ban_report( + keypair.public_key(), + Utc::now() + Duration::days(7), + SpBoostedRewardsBannedRadioReason::NoNetworkCorrelation, + ); + + let mut transaction = pool.begin().await?; + setup + .ingestor + .process_ingest_report(&mut transaction, report) + .await?; + transaction.commit().await?; + + let banned_radios = db::get_banned_radios(&pool, Utc::now()).await?; + let result = banned_radios.contains(&keypair.public_key().to_owned().into(), None); + + assert!(result); + + let report = wifi_ban_report( + keypair.public_key(), + Utc::now() - Duration::days(7), + SpBoostedRewardsBannedRadioReason::Unbanned, + ); + + let mut transaction = pool.begin().await?; + setup + .ingestor + .process_ingest_report(&mut transaction, report) + .await?; + transaction.commit().await?; + + let banned_radios = db::get_banned_radios(&pool, Utc::now()).await?; + let result = banned_radios.contains(&keypair.public_key().to_owned().into(), None); + + assert!(!result); + + Ok(()) + } + + fn generate_keypair() -> Keypair { + Keypair::generate(KeyTag::default(), &mut OsRng) + } +} From 2c19e9b707603d855bc7e53dc4e46de6f902ef68 Mon Sep 17 00:00:00 2001 From: Brian Balser Date: Wed, 3 Jul 2024 08:33:57 -0400 Subject: [PATCH 13/23] Add test for updating seniority --- mobile_verifier/src/seniority.rs | 2 +- .../src/sp_boosted_rewards_bans.rs | 50 +++++++++++++++++++ 2 files changed, 51 insertions(+), 1 deletion(-) diff --git a/mobile_verifier/src/seniority.rs b/mobile_verifier/src/seniority.rs index 60b45e794..a8154129c 100644 --- a/mobile_verifier/src/seniority.rs +++ b/mobile_verifier/src/seniority.rs @@ -22,7 +22,7 @@ impl Seniority { exec: &mut Transaction<'_, Postgres>, ) -> Result, sqlx::Error> { sqlx::query_as( - "SELECT uuid, seniority_ts, last_heartbeat, inserted_at, update_reason FROM seniority WHERE radio_key = $1 ORDER BY last_heartbeat DESC LIMIT 1", + "SELECT uuid, seniority_ts, last_heartbeat, inserted_at, update_reason FROM seniority WHERE radio_key = $1 ORDER BY last_heartbeat DESC, seniority_ts DESC LIMIT 1", ) .bind(key) .fetch_optional(&mut *exec) diff --git a/mobile_verifier/src/sp_boosted_rewards_bans.rs b/mobile_verifier/src/sp_boosted_rewards_bans.rs index 888803201..44ca13219 100644 --- a/mobile_verifier/src/sp_boosted_rewards_bans.rs +++ b/mobile_verifier/src/sp_boosted_rewards_bans.rs @@ -419,6 +419,8 @@ mod tests { use rand::rngs::OsRng; use tokio::sync::mpsc; + use crate::heartbeats::KeyType; + use super::*; #[derive(thiserror::Error, Debug)] @@ -598,6 +600,54 @@ mod tests { Ok(()) } + #[sqlx::test] + async fn getting_banned_reset_seniority(pool: PgPool) -> anyhow::Result<()> { + let setup = TestSetup::create(pool.clone(), AllVerified); + let keypair = generate_keypair(); + let pubkey = PublicKeyBinary::from(keypair.public_key().to_owned()); + + let last_heartbeat_ts = Utc::now() - Duration::hours(5); + let uuid = uuid::Uuid::new_v4(); + let key_type = KeyType::Wifi(&pubkey); + + let seniority_update = SeniorityUpdate::new( + key_type, + last_heartbeat_ts, + uuid, + SeniorityUpdateAction::Insert { + new_seniority: last_heartbeat_ts, + update_reason: SeniorityUpdateReason::NewCoverageClaimTime, + }, + ); + + let report = wifi_ban_report( + keypair.public_key(), + Utc::now() + Duration::days(7), + SpBoostedRewardsBannedRadioReason::NoNetworkCorrelation, + ); + + let mut transaction = pool.begin().await?; + seniority_update.execute(&mut transaction).await?; + + setup + .ingestor + .process_ingest_report(&mut transaction, report) + .await?; + + let seniority = Seniority::fetch_latest(key_type, &mut transaction) + .await? + .unwrap(); + transaction.commit().await?; + + assert!(seniority.seniority_ts > last_heartbeat_ts); + assert_eq!( + seniority.update_reason, + SeniorityUpdateReason::ServiceProviderBan as i32 + ); + + Ok(()) + } + fn generate_keypair() -> Keypair { Keypair::generate(KeyTag::default(), &mut OsRng) } From 691b5e78f21ad0f5c89d57bf7c941f0bd0924c08 Mon Sep 17 00:00:00 2001 From: Brian Balser Date: Wed, 3 Jul 2024 09:07:28 -0400 Subject: [PATCH 14/23] remove commented out code --- coverage_point_calculator/src/lib.rs | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/coverage_point_calculator/src/lib.rs b/coverage_point_calculator/src/lib.rs index f3141bf42..5e4eea3e0 100644 --- a/coverage_point_calculator/src/lib.rs +++ b/coverage_point_calculator/src/lib.rs @@ -322,18 +322,6 @@ pub enum ServiceProviderBoostedRewardEligibility { RadioThresholdNotMet, } -// #[derive(Debug, Clone, Copy, PartialEq)] -// pub enum RadioThreshold { -// Verified, -// Unverified, -// } - -// impl RadioThreshold { -// fn is_met(&self) -> bool { -// matches!(self, Self::Verified) -// } -// } - #[cfg(test)] mod tests { From a21ed78c5c7a68adadbac40201a3d940e9863e29 Mon Sep 17 00:00:00 2001 From: Brian Balser Date: Wed, 3 Jul 2024 10:54:02 -0400 Subject: [PATCH 15/23] Addressing feedback --- mobile_verifier/src/seniority.rs | 12 ++++++------ mobile_verifier/src/sp_boosted_rewards_bans.rs | 12 +++++++----- mobile_verifier/tests/integrations/seniority.rs | 4 ++-- 3 files changed, 15 insertions(+), 13 deletions(-) diff --git a/mobile_verifier/src/seniority.rs b/mobile_verifier/src/seniority.rs index a8154129c..8b4a07c59 100644 --- a/mobile_verifier/src/seniority.rs +++ b/mobile_verifier/src/seniority.rs @@ -64,7 +64,7 @@ impl<'a> SeniorityUpdate<'a> { } } - pub fn from( + pub fn from_heartbeat( heartbeat: &'a ValidatedHeartbeat, action: SeniorityUpdateAction, ) -> anyhow::Result { @@ -91,9 +91,9 @@ impl<'a> SeniorityUpdate<'a> { if prev_seniority.update_reason == HeartbeatNotSeen as i32 && coverage_claim_time < prev_seniority.seniority_ts { - Self::from(heartbeat, SeniorityUpdateAction::NoAction) + Self::from_heartbeat(heartbeat, SeniorityUpdateAction::NoAction) } else { - Self::from( + Self::from_heartbeat( heartbeat, SeniorityUpdateAction::Insert { new_seniority: coverage_claim_time, @@ -105,7 +105,7 @@ impl<'a> SeniorityUpdate<'a> { > Duration::days(3) && coverage_claim_time < heartbeat.heartbeat.timestamp { - Self::from( + Self::from_heartbeat( heartbeat, SeniorityUpdateAction::Insert { new_seniority: heartbeat.heartbeat.timestamp, @@ -113,7 +113,7 @@ impl<'a> SeniorityUpdate<'a> { }, ) } else { - Self::from( + Self::from_heartbeat( heartbeat, SeniorityUpdateAction::Update { curr_seniority: prev_seniority.seniority_ts, @@ -121,7 +121,7 @@ impl<'a> SeniorityUpdate<'a> { ) } } else { - Self::from( + Self::from_heartbeat( heartbeat, SeniorityUpdateAction::Insert { new_seniority: coverage_claim_time, diff --git a/mobile_verifier/src/sp_boosted_rewards_bans.rs b/mobile_verifier/src/sp_boosted_rewards_bans.rs index 44ca13219..621987529 100644 --- a/mobile_verifier/src/sp_boosted_rewards_bans.rs +++ b/mobile_verifier/src/sp_boosted_rewards_bans.rs @@ -244,9 +244,10 @@ where self.update_seniority(transaction, &report).await?; } - let status = match is_authorized { - true => ServiceProviderBoostedRewardsBannedRadioVerificationStatus::SpBoostedRewardsBanValid, - false => ServiceProviderBoostedRewardsBannedRadioVerificationStatus::SpBoostedRewardsBanInvalidCarrierKey, + let status = if is_authorized { + ServiceProviderBoostedRewardsBannedRadioVerificationStatus::SpBoostedRewardsBanValid + } else { + ServiceProviderBoostedRewardsBannedRadioVerificationStatus::SpBoostedRewardsBanInvalidCarrierKey }; let verified_report = VerifiedServiceProviderBoostedRewardsBannedRadioIngestReportV1 { @@ -318,8 +319,9 @@ pub mod db { r#" SELECT radio_type, radio_key FROM sp_boosted_rewards_bans - WHERE until > $1 - AND (invalidated_at > $1 OR invalidated_at IS NULL) + WHERE received_timestamp <= $1 + AND until > $1 + AND COALESCE(invalidated_at > $1, TRUE) "#, ) .bind(date_time) diff --git a/mobile_verifier/tests/integrations/seniority.rs b/mobile_verifier/tests/integrations/seniority.rs index 85cb95dec..6b1fae38d 100644 --- a/mobile_verifier/tests/integrations/seniority.rs +++ b/mobile_verifier/tests/integrations/seniority.rs @@ -36,7 +36,7 @@ async fn test_seniority_updates(pool: PgPool) -> anyhow::Result<()> { assert_eq!(latest_seniority, None); - let action1 = SeniorityUpdate::from( + let action1 = SeniorityUpdate::from_heartbeat( &heartbeat, SeniorityUpdateAction::Insert { new_seniority: "2023-08-25 00:00:00.000000000 UTC".parse().unwrap(), @@ -60,7 +60,7 @@ async fn test_seniority_updates(pool: PgPool) -> anyhow::Result<()> { heartbeat.heartbeat.timestamp = "2023-08-24 00:00:00.000000000 UTC".parse().unwrap(); - let action2 = SeniorityUpdate::from( + let action2 = SeniorityUpdate::from_heartbeat( &heartbeat, SeniorityUpdateAction::Update { curr_seniority: "2023-08-25 00:00:00.000000000 UTC".parse().unwrap(), From 0b0fc0de1b70ed75e8f5e4444491953712012320 Mon Sep 17 00:00:00 2001 From: Brian Balser Date: Tue, 9 Jul 2024 09:42:03 -0400 Subject: [PATCH 16/23] Fix comment --- coverage_point_calculator/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/coverage_point_calculator/src/lib.rs b/coverage_point_calculator/src/lib.rs index 5e4eea3e0..2e23ee181 100644 --- a/coverage_point_calculator/src/lib.rs +++ b/coverage_point_calculator/src/lib.rs @@ -121,7 +121,7 @@ pub struct CoveragePoints { pub speedtest_multiplier: Decimal, /// Input Radio Type pub radio_type: RadioType, - /// Input RadioThreshold + /// Input ServiceProviderBoostedRewardEligibility pub service_provider_boosted_reward_eligibility: ServiceProviderBoostedRewardEligibility, /// Derived Eligibility for Boosted Hex Rewards pub boosted_hex_eligibility: BoostedHexStatus, From 1a0138d29e234792c79915c9e51ecb917ca5233a Mon Sep 17 00:00:00 2001 From: Brian Balser Date: Tue, 9 Jul 2024 13:54:49 -0400 Subject: [PATCH 17/23] invalidate old bans if processing new ban --- Cargo.lock | 8 ++++---- mobile_verifier/src/sp_boosted_rewards_bans.rs | 7 +++++-- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index baf21cbf9..0ed638db6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1617,7 +1617,7 @@ checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" [[package]] name = "beacon" version = "0.1.0" -source = "git+https://github.com/helium/proto?branch=bbalser/hip-125#6c059f9dac3d2617a4884861f4e973162d7f18c3" +source = "git+https://github.com/helium/proto?branch=bbalser/hip-125#36fae78eb628479b84684b2c842fd465ba030cc7" dependencies = [ "base64 0.21.7", "byteorder", @@ -1627,7 +1627,7 @@ dependencies = [ "rand_chacha 0.3.0", "rust_decimal", "serde", - "sha2 0.10.8", + "sha2 0.9.9", "thiserror", ] @@ -3801,7 +3801,7 @@ dependencies = [ [[package]] name = "helium-proto" version = "0.1.0" -source = "git+https://github.com/helium/proto?branch=bbalser/hip-125#6c059f9dac3d2617a4884861f4e973162d7f18c3" +source = "git+https://github.com/helium/proto?branch=bbalser/hip-125#36fae78eb628479b84684b2c842fd465ba030cc7" dependencies = [ "bytes", "prost", @@ -9863,7 +9863,7 @@ dependencies = [ "rand 0.8.5", "serde", "serde_json", - "sha2 0.10.8", + "sha2 0.9.9", "thiserror", "twox-hash", "xorf", diff --git a/mobile_verifier/src/sp_boosted_rewards_bans.rs b/mobile_verifier/src/sp_boosted_rewards_bans.rs index 621987529..d20c6db65 100644 --- a/mobile_verifier/src/sp_boosted_rewards_bans.rs +++ b/mobile_verifier/src/sp_boosted_rewards_bans.rs @@ -317,7 +317,7 @@ pub mod db { ) -> anyhow::Result { sqlx::query( r#" - SELECT radio_type, radio_key + SELECT distinct radio_type, radio_key FROM sp_boosted_rewards_bans WHERE received_timestamp <= $1 AND until > $1 @@ -365,7 +365,10 @@ pub mod db { SpBoostedRewardsBannedRadioReason::Unbanned => { invalidate_all_before(transaction, report).await } - _ => save(transaction, report).await, + _ => { + invalidate_all_before(transaction, report).await?; + save(transaction, report).await + } } } From bdca463a6300d337cd7e58fddb11e2caee9f170c Mon Sep 17 00:00:00 2001 From: Brian Balser Date: Tue, 9 Jul 2024 14:25:45 -0400 Subject: [PATCH 18/23] fix migration --- mobile_verifier/migrations/34_sp_boosted_rewards_bans.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mobile_verifier/migrations/34_sp_boosted_rewards_bans.sql b/mobile_verifier/migrations/34_sp_boosted_rewards_bans.sql index c8b768af2..e8f4eac58 100644 --- a/mobile_verifier/migrations/34_sp_boosted_rewards_bans.sql +++ b/mobile_verifier/migrations/34_sp_boosted_rewards_bans.sql @@ -4,5 +4,5 @@ CREATE TABLE IF NOT EXISTS sp_boosted_rewards_bans ( received_timestamp TIMESTAMPTZ NOT NULL, until TIMESTAMPTZ NOT NULL, invalidated_at TIMESTAMPTZ, - PRIMARY KEY (radio_type, radio_key) + PRIMARY KEY (radio_type, radio_key, received_timestamp) ); From a772859126e111a7055757736675056f818529c3 Mon Sep 17 00:00:00 2001 From: Brian Balser Date: Tue, 9 Jul 2024 14:43:23 -0400 Subject: [PATCH 19/23] Fix naming after proto change --- Cargo.lock | 8 ++++---- ingest/src/server_mobile.rs | 4 ++-- mobile_verifier/src/sp_boosted_rewards_bans.rs | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0ed638db6..720c61285 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1617,7 +1617,7 @@ checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" [[package]] name = "beacon" version = "0.1.0" -source = "git+https://github.com/helium/proto?branch=bbalser/hip-125#36fae78eb628479b84684b2c842fd465ba030cc7" +source = "git+https://github.com/helium/proto?branch=bbalser/hip-125#8f9a42803fef0efd0e2fe5d14687d88c48cbbea3" dependencies = [ "base64 0.21.7", "byteorder", @@ -1627,7 +1627,7 @@ dependencies = [ "rand_chacha 0.3.0", "rust_decimal", "serde", - "sha2 0.9.9", + "sha2 0.10.8", "thiserror", ] @@ -3801,7 +3801,7 @@ dependencies = [ [[package]] name = "helium-proto" version = "0.1.0" -source = "git+https://github.com/helium/proto?branch=bbalser/hip-125#36fae78eb628479b84684b2c842fd465ba030cc7" +source = "git+https://github.com/helium/proto?branch=bbalser/hip-125#8f9a42803fef0efd0e2fe5d14687d88c48cbbea3" dependencies = [ "bytes", "prost", @@ -9863,7 +9863,7 @@ dependencies = [ "rand 0.8.5", "serde", "serde_json", - "sha2 0.9.9", + "sha2 0.10.8", "thiserror", "twox-hash", "xorf", diff --git a/ingest/src/server_mobile.rs b/ingest/src/server_mobile.rs index 13033754e..e1188898a 100644 --- a/ingest/src/server_mobile.rs +++ b/ingest/src/server_mobile.rs @@ -348,10 +348,10 @@ impl poc_mobile::PocMobile for GrpcServer { let timestamp = Utc::now().timestamp_millis() as u64; let event = request.into_inner(); - custom_tracing::record_b58("pub_key", &event.pub_key); + custom_tracing::record_b58("pub_key", &event.pubkey); let report = self - .verify_public_key(event.pub_key.as_ref()) + .verify_public_key(event.pubkey.as_ref()) .and_then(|public_key| self.verify_network(public_key)) .and_then(|public_key| self.verify_signature(public_key, event)) .map( diff --git a/mobile_verifier/src/sp_boosted_rewards_bans.rs b/mobile_verifier/src/sp_boosted_rewards_bans.rs index d20c6db65..5129f05bf 100644 --- a/mobile_verifier/src/sp_boosted_rewards_bans.rs +++ b/mobile_verifier/src/sp_boosted_rewards_bans.rs @@ -79,7 +79,7 @@ impl TryFrom for BannedR .ok_or_else(|| { anyhow::anyhow!("invalid received timestamp, {}", value.received_timestamp) })?, - pubkey: report.pub_key.into(), + pubkey: report.pubkey.into(), key, until: Utc .timestamp_opt(report.until as i64, 0) @@ -486,7 +486,7 @@ mod tests { ServiceProviderBoostedRewardsBannedRadioIngestReportV1 { received_timestamp: Utc::now().timestamp_millis() as u64, report: Some(ServiceProviderBoostedRewardsBannedRadioReqV1 { - pub_key: signer_keypair.public_key().into(), + pubkey: signer_keypair.public_key().into(), reason: reason as i32, until: until.timestamp() as u64, signature: vec![], @@ -505,7 +505,7 @@ mod tests { ServiceProviderBoostedRewardsBannedRadioIngestReportV1 { received_timestamp: Utc::now().timestamp_millis() as u64, report: Some(ServiceProviderBoostedRewardsBannedRadioReqV1 { - pub_key: signer_keypair.public_key().into(), + pubkey: signer_keypair.public_key().into(), reason: reason as i32, until: until.timestamp() as u64, signature: vec![], From 0bdcab1217eea373a85c9e919f843d01ab0b9b0b Mon Sep 17 00:00:00 2001 From: Brian Balser Date: Tue, 9 Jul 2024 14:57:22 -0400 Subject: [PATCH 20/23] fix imports --- mobile_verifier/src/rewarder.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mobile_verifier/src/rewarder.rs b/mobile_verifier/src/rewarder.rs index 164969c76..838c12f21 100644 --- a/mobile_verifier/src/rewarder.rs +++ b/mobile_verifier/src/rewarder.rs @@ -3,7 +3,10 @@ use crate::{ coverage, data_session, heartbeats::{self, HeartbeatReward}, radio_threshold, - reward_shares::{self, CoverageShares, MapperShares, ServiceProviderShares, TransferRewards}, + reward_shares::{ + self, CoverageShares, DataTransferAndPocAllocatedRewardBuckets, MapperShares, + ServiceProviderShares, TransferRewards, + }, sp_boosted_rewards_bans, speedtests, speedtests_average::SpeedtestAverages, subscriber_location, telemetry, Settings, From 0011683fcdec7a0cb7a067696962ef5b8f8fc008 Mon Sep 17 00:00:00 2001 From: Brian Balser Date: Tue, 9 Jul 2024 15:10:46 -0400 Subject: [PATCH 21/23] Rename file_info --- file_store/src/file_info.rs | 22 +++++++------------ ingest/src/server_mobile.rs | 2 +- .../src/sp_boosted_rewards_bans.rs | 4 ++-- 3 files changed, 11 insertions(+), 17 deletions(-) diff --git a/file_store/src/file_info.rs b/file_store/src/file_info.rs index 4eaab2d0e..ece994502 100644 --- a/file_store/src/file_info.rs +++ b/file_store/src/file_info.rs @@ -212,8 +212,8 @@ pub enum FileType { InvalidatedRadioThresholdReq, InvalidatedRadioThresholdIngestReport, VerifiedInvalidatedRadioThresholdIngestReport, - ServiceProviderBoostedRewardsBannedRadioIngestReport, - VerifiedServiceProviderBoostedRewardsBannedRadioIngestReport, + SPBoostedRewardsBannedRadioIngestReport, + VerifiedSPBoostedRewardsBannedRadioIngestReport, } impl fmt::Display for FileType { @@ -275,10 +275,8 @@ impl fmt::Display for FileType { Self::UrbanizationDataSet => URBANIZATION_DATA_SET, Self::FootfallDataSet => FOOTFALL_DATA_SET, Self::LandtypeDataSet => LANDTYPE_DATA_SET, - Self::ServiceProviderBoostedRewardsBannedRadioIngestReport => { - SP_BOOSTED_REWARDS_BANNED_RADIO - } - Self::VerifiedServiceProviderBoostedRewardsBannedRadioIngestReport => { + Self::SPBoostedRewardsBannedRadioIngestReport => SP_BOOSTED_REWARDS_BANNED_RADIO, + Self::VerifiedSPBoostedRewardsBannedRadioIngestReport => { VERIFIED_SP_BOOSTED_REWARDS_BANNED_RADIO } }; @@ -345,10 +343,8 @@ impl FileType { Self::UrbanizationDataSet => URBANIZATION_DATA_SET, Self::FootfallDataSet => FOOTFALL_DATA_SET, Self::LandtypeDataSet => LANDTYPE_DATA_SET, - Self::ServiceProviderBoostedRewardsBannedRadioIngestReport => { - SP_BOOSTED_REWARDS_BANNED_RADIO - } - Self::VerifiedServiceProviderBoostedRewardsBannedRadioIngestReport => { + Self::SPBoostedRewardsBannedRadioIngestReport => SP_BOOSTED_REWARDS_BANNED_RADIO, + Self::VerifiedSPBoostedRewardsBannedRadioIngestReport => { VERIFIED_SP_BOOSTED_REWARDS_BANNED_RADIO } } @@ -415,11 +411,9 @@ impl FromStr for FileType { URBANIZATION_DATA_SET => Self::UrbanizationDataSet, FOOTFALL_DATA_SET => Self::FootfallDataSet, LANDTYPE_DATA_SET => Self::LandtypeDataSet, - SP_BOOSTED_REWARDS_BANNED_RADIO => { - Self::ServiceProviderBoostedRewardsBannedRadioIngestReport - } + SP_BOOSTED_REWARDS_BANNED_RADIO => Self::SPBoostedRewardsBannedRadioIngestReport, VERIFIED_SP_BOOSTED_REWARDS_BANNED_RADIO => { - Self::VerifiedServiceProviderBoostedRewardsBannedRadioIngestReport + Self::VerifiedSPBoostedRewardsBannedRadioIngestReport } _ => return Err(Error::from(io::Error::from(io::ErrorKind::InvalidInput))), }; diff --git a/ingest/src/server_mobile.rs b/ingest/src/server_mobile.rs index e1188898a..2172b7da6 100644 --- a/ingest/src/server_mobile.rs +++ b/ingest/src/server_mobile.rs @@ -472,7 +472,7 @@ pub async fn grpc_server(settings: &Settings) -> Result<()> { let (sp_boosted_rewards_ban_sink, sp_boosted_rewards_ban_sink_server) = file_sink::FileSinkBuilder::new( - FileType::ServiceProviderBoostedRewardsBannedRadioIngestReport, + FileType::SPBoostedRewardsBannedRadioIngestReport, store_base_path, file_upload.clone(), concat!( diff --git a/mobile_verifier/src/sp_boosted_rewards_bans.rs b/mobile_verifier/src/sp_boosted_rewards_bans.rs index 5129f05bf..697b12960 100644 --- a/mobile_verifier/src/sp_boosted_rewards_bans.rs +++ b/mobile_verifier/src/sp_boosted_rewards_bans.rs @@ -153,7 +153,7 @@ where seniority_update_sink: FileSinkClient, ) -> anyhow::Result { let (verified_sink, verified_sink_server) = file_sink::FileSinkBuilder::new( - FileType::VerifiedServiceProviderBoostedRewardsBannedRadioIngestReport, + FileType::VerifiedSPBoostedRewardsBannedRadioIngestReport, settings.store_base_path(), file_upload, concat!(env!("CARGO_PKG_NAME"), "_verified_sp_boosted_rewards_ban"), @@ -171,7 +171,7 @@ where .state(pool.clone()) .store(file_store) .lookback(LookbackBehavior::StartAfter(settings.start_after)) - .prefix(FileType::ServiceProviderBoostedRewardsBannedRadioIngestReport.to_string()) + .prefix(FileType::SPBoostedRewardsBannedRadioIngestReport.to_string()) .create() .await?; From f22eecd7a3f5c90da2d7711f6a2af3bfd1e393be Mon Sep 17 00:00:00 2001 From: Brian Balser Date: Tue, 9 Jul 2024 15:39:55 -0400 Subject: [PATCH 22/23] move proto back to master --- Cargo.lock | 52 ++++++++++++++++++++-------------------------------- Cargo.toml | 4 ++-- 2 files changed, 22 insertions(+), 34 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 720c61285..dda7cc030 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1617,17 +1617,17 @@ checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" [[package]] name = "beacon" version = "0.1.0" -source = "git+https://github.com/helium/proto?branch=bbalser/hip-125#8f9a42803fef0efd0e2fe5d14687d88c48cbbea3" +source = "git+https://github.com/helium/proto?branch=master#ff0d2babb0ddb58f1590d932dd53c95186b64287" dependencies = [ "base64 0.21.7", "byteorder", - "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=bbalser/hip-125)", + "helium-proto", "prost", "rand 0.8.5", "rand_chacha 0.3.0", "rust_decimal", "serde", - "sha2 0.10.8", + "sha2 0.9.9", "thiserror", ] @@ -1776,7 +1776,7 @@ dependencies = [ "futures", "futures-util", "helium-crypto", - "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=bbalser/hip-125)", + "helium-proto", "http 0.2.11", "http-serde", "humantime-serde", @@ -2618,7 +2618,7 @@ dependencies = [ "axum 0.7.4", "bs58 0.4.0", "helium-crypto", - "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=bbalser/hip-125)", + "helium-proto", "http 0.2.11", "notify", "serde", @@ -3200,7 +3200,7 @@ dependencies = [ "futures-util", "h3o", "helium-crypto", - "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=bbalser/hip-125)", + "helium-proto", "hex-literal", "http 0.2.11", "lazy_static", @@ -3776,7 +3776,7 @@ dependencies = [ "h3o", "helium-anchor-gen", "helium-crypto", - "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=master)", + "helium-proto", "hex", "itertools", "jsonrpc_client", @@ -3801,7 +3801,7 @@ dependencies = [ [[package]] name = "helium-proto" version = "0.1.0" -source = "git+https://github.com/helium/proto?branch=bbalser/hip-125#8f9a42803fef0efd0e2fe5d14687d88c48cbbea3" +source = "git+https://github.com/helium/proto?branch=master#fdb83a38fbe8aead4ff6cb39d1e996f0ad0646b7" dependencies = [ "bytes", "prost", @@ -3812,18 +3812,6 @@ dependencies = [ "tonic-build", ] -[[package]] -name = "helium-proto" -version = "0.1.0" -source = "git+https://github.com/helium/proto?branch=master#fdb83a38fbe8aead4ff6cb39d1e996f0ad0646b7" -dependencies = [ - "bytes", - "prost", - "prost-build", - "serde", - "serde_json", -] - [[package]] name = "helium-sub-daos" version = "0.1.5" @@ -3865,7 +3853,7 @@ dependencies = [ "async-trait", "chrono", "derive_builder", - "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=bbalser/hip-125)", + "helium-proto", "hextree", "rust_decimal", "rust_decimal_macros", @@ -4281,7 +4269,7 @@ dependencies = [ "futures", "futures-util", "helium-crypto", - "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=bbalser/hip-125)", + "helium-proto", "http 0.2.11", "humantime-serde", "metrics", @@ -4350,7 +4338,7 @@ dependencies = [ "futures", "futures-util", "helium-crypto", - "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=bbalser/hip-125)", + "helium-proto", "hextree", "http 0.2.11", "http-serde", @@ -4392,7 +4380,7 @@ dependencies = [ "futures", "futures-util", "helium-crypto", - "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=bbalser/hip-125)", + "helium-proto", "http 0.2.11", "http-serde", "humantime-serde", @@ -4434,7 +4422,7 @@ dependencies = [ "futures-util", "h3o", "helium-crypto", - "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=bbalser/hip-125)", + "helium-proto", "http-serde", "humantime-serde", "iot-config", @@ -5022,7 +5010,7 @@ dependencies = [ "futures", "futures-util", "helium-crypto", - "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=bbalser/hip-125)", + "helium-proto", "hextree", "http 0.2.11", "http-serde", @@ -5062,7 +5050,7 @@ dependencies = [ "futures", "h3o", "helium-crypto", - "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=bbalser/hip-125)", + "helium-proto", "mobile-config", "prost", "rand 0.8.5", @@ -5098,7 +5086,7 @@ dependencies = [ "futures", "futures-util", "helium-crypto", - "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=bbalser/hip-125)", + "helium-proto", "http 0.2.11", "http-serde", "humantime-serde", @@ -5142,7 +5130,7 @@ dependencies = [ "futures-util", "h3o", "helium-crypto", - "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=bbalser/hip-125)", + "helium-proto", "hex-assignments", "hextree", "http-serde", @@ -5825,7 +5813,7 @@ dependencies = [ "futures", "futures-util", "helium-crypto", - "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=bbalser/hip-125)", + "helium-proto", "http 0.2.11", "hyper 0.14.28", "jsonrpsee", @@ -5908,7 +5896,7 @@ dependencies = [ "futures-util", "helium-anchor-gen", "helium-lib", - "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=bbalser/hip-125)", + "helium-proto", "humantime-serde", "metrics", "metrics-exporter-prometheus", @@ -6515,7 +6503,7 @@ dependencies = [ "futures", "futures-util", "helium-crypto", - "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=bbalser/hip-125)", + "helium-proto", "humantime-serde", "lazy_static", "metrics", diff --git a/Cargo.toml b/Cargo.toml index 87977c4f7..870fd79e0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -70,7 +70,7 @@ helium-lib = { git = "https://github.com/helium/helium-wallet-rs.git", branch = hextree = { git = "https://github.com/jaykickliter/HexTree", branch = "main", features = [ "disktree", ] } -helium-proto = { git = "https://github.com/helium/proto", branch = "bbalser/hip-125", features = [ +helium-proto = { git = "https://github.com/helium/proto", branch = "master", features = [ "services", ] } solana-client = "1.18" @@ -82,7 +82,7 @@ reqwest = { version = "0", default-features = false, features = [ "json", "rustls-tls", ] } -beacon = { git = "https://github.com/helium/proto", branch = "bbalser/hip-125" } +beacon = { git = "https://github.com/helium/proto", branch = "master" } humantime = "2" humantime-serde = "1" metrics = ">=0.22" From 66e26d96d4329f401acce56279b48b30616c796e Mon Sep 17 00:00:00 2001 From: Brian Balser Date: Tue, 9 Jul 2024 15:46:05 -0400 Subject: [PATCH 23/23] fix proto --- Cargo.lock | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index dda7cc030..3a2923089 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1627,7 +1627,7 @@ dependencies = [ "rand_chacha 0.3.0", "rust_decimal", "serde", - "sha2 0.9.9", + "sha2 0.10.8", "thiserror", ] @@ -3736,7 +3736,7 @@ dependencies = [ "bs58 0.5.0", "byteorder", "ed25519-compact", - "getrandom 0.2.10", + "getrandom 0.1.16", "k256", "lazy_static", "multihash", @@ -3801,7 +3801,7 @@ dependencies = [ [[package]] name = "helium-proto" version = "0.1.0" -source = "git+https://github.com/helium/proto?branch=master#fdb83a38fbe8aead4ff6cb39d1e996f0ad0646b7" +source = "git+https://github.com/helium/proto?branch=master#ff0d2babb0ddb58f1590d932dd53c95186b64287" dependencies = [ "bytes", "prost", @@ -5497,7 +5497,7 @@ version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "681030a937600a36906c185595136d26abfebb4aa9c65701cefcaf8578bb982b" dependencies = [ - "proc-macro-crate 3.1.0", + "proc-macro-crate 1.1.3", "proc-macro2", "quote", "syn 2.0.58", @@ -9196,7 +9196,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97fee6b57c6a41524a810daee9286c02d7752c4253064d0b05472833a438f675" dependencies = [ "cfg-if", - "rand 0.8.5", + "rand 0.7.3", "static_assertions", ]