From bf03e021885615401079af7217a5452e42284ae4 Mon Sep 17 00:00:00 2001 From: Macpie Date: Mon, 30 Sep 2024 11:59:19 -0700 Subject: [PATCH 01/31] Add radio location estimates ingest --- Cargo.lock | 1 - Cargo.toml | 4 +- file_store/src/file_info.rs | 5 + file_store/src/lib.rs | 1 + file_store/src/radio_location_estimates.rs | 108 +++++++++++++++++++++ file_store/src/traits/file_sink_write.rs | 5 + file_store/src/traits/msg_verify.rs | 1 + ingest/src/server_mobile.rs | 47 ++++++++- ingest/tests/common/mod.rs | 3 + 9 files changed, 170 insertions(+), 5 deletions(-) create mode 100644 file_store/src/radio_location_estimates.rs diff --git a/Cargo.lock b/Cargo.lock index b8c2ad9c7..61aa1b268 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3882,7 +3882,6 @@ dependencies = [ [[package]] name = "helium-proto" version = "0.1.0" -source = "git+https://github.com/helium/proto?branch=master#47598512b7d282c34b619adf140403feeb138e71" dependencies = [ "bytes", "prost", diff --git a/Cargo.toml b/Cargo.toml index a45355a99..08068ea68 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -130,6 +130,6 @@ sqlx = { git = "https://github.com/helium/sqlx.git", rev = "92a2268f02e0cac6fccb # Patching for beacon must point directly to the crate, it will not look in the # repo for sibling crates. # -# [patch.'https://github.com/helium/proto'] -# helium-proto = { path = "../proto" } +[patch.'https://github.com/helium/proto'] +helium-proto = { path = "../proto" } # beacon = { path = "../proto/beacon" } diff --git a/file_store/src/file_info.rs b/file_store/src/file_info.rs index d0f824432..aac8ecf33 100644 --- a/file_store/src/file_info.rs +++ b/file_store/src/file_info.rs @@ -168,6 +168,7 @@ pub const VERIFIED_SUBSCRIBER_VERIFIED_MAPPING_INGEST_REPORT: &str = pub const PROMOTION_REWARD_INGEST_REPORT: &str = "promotion_reward_ingest_report"; pub const VERIFIED_PROMOTION_REWARD: &str = "verified_promotion_reward"; pub const SERVICE_PROVIDER_PROMOTION_FUND: &str = "service_provider_promotion_fund"; +pub const RADIO_LOCATION_ESTIMATES_INGEST_REPORT: &str = "radio_location_estimates_ingest_report"; #[derive(Debug, PartialEq, Eq, Clone, Serialize, Copy, strum::EnumCount)] #[serde(rename_all = "snake_case")] @@ -228,6 +229,7 @@ pub enum FileType { PromotionRewardIngestReport, VerifiedPromotionReward, ServiceProviderPromotionFund, + RadioLocationEstimatesIngestReport, } impl fmt::Display for FileType { @@ -303,6 +305,7 @@ impl fmt::Display for FileType { Self::PromotionRewardIngestReport => PROMOTION_REWARD_INGEST_REPORT, Self::VerifiedPromotionReward => VERIFIED_PROMOTION_REWARD, Self::ServiceProviderPromotionFund => SERVICE_PROVIDER_PROMOTION_FUND, + Self::RadioLocationEstimatesIngestReport => RADIO_LOCATION_ESTIMATES_INGEST_REPORT, }; f.write_str(s) } @@ -381,6 +384,7 @@ impl FileType { Self::PromotionRewardIngestReport => PROMOTION_REWARD_INGEST_REPORT, Self::VerifiedPromotionReward => VERIFIED_PROMOTION_REWARD, Self::ServiceProviderPromotionFund => SERVICE_PROVIDER_PROMOTION_FUND, + Self::RadioLocationEstimatesIngestReport => RADIO_LOCATION_ESTIMATES_INGEST_REPORT, } } } @@ -458,6 +462,7 @@ impl FromStr for FileType { PROMOTION_REWARD_INGEST_REPORT => Self::PromotionRewardIngestReport, VERIFIED_PROMOTION_REWARD => Self::VerifiedPromotionReward, SERVICE_PROVIDER_PROMOTION_FUND => Self::ServiceProviderPromotionFund, + RADIO_LOCATION_ESTIMATES_INGEST_REPORT => Self::RadioLocationEstimatesIngestReport, _ => return Err(Error::from(io::Error::from(io::ErrorKind::InvalidInput))), }; Ok(result) diff --git a/file_store/src/lib.rs b/file_store/src/lib.rs index 9c5a132cd..c6fc5d6e8 100644 --- a/file_store/src/lib.rs +++ b/file_store/src/lib.rs @@ -21,6 +21,7 @@ pub mod mobile_session; pub mod mobile_subscriber; pub mod mobile_transfer; pub mod promotion_reward; +pub mod radio_location_estimates; pub mod reward_manifest; mod settings; pub mod speedtest; diff --git a/file_store/src/radio_location_estimates.rs b/file_store/src/radio_location_estimates.rs new file mode 100644 index 000000000..219a52ae0 --- /dev/null +++ b/file_store/src/radio_location_estimates.rs @@ -0,0 +1,108 @@ +use crate::{ + traits::{MsgDecode, MsgTimestamp, TimestampDecode, TimestampEncode}, + Error, Result, +}; +use chrono::{DateTime, Utc}; +use helium_crypto::PublicKeyBinary; +use helium_proto::services::poc_mobile::{ + RadioLocationEstimateV1, RadioLocationEstimatesReqV1, RleEventV1, +}; +use rust_decimal::Decimal; +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Deserialize, Serialize, Debug, PartialEq)] +pub struct RadioLocationEstimates { + pub radio_id: String, + pub estimates: Vec, + pub timestamp: DateTime, + pub signer: PublicKeyBinary, +} + +impl MsgDecode for RadioLocationEstimates { + type Msg = RadioLocationEstimatesReqV1; +} + +impl MsgTimestamp>> for RadioLocationEstimatesReqV1 { + fn timestamp(&self) -> Result> { + self.timestamp.to_timestamp() + } +} + +impl MsgTimestamp for RadioLocationEstimates { + fn timestamp(&self) -> u64 { + self.timestamp.encode_timestamp() + } +} + +impl TryFrom for RadioLocationEstimates { + type Error = Error; + fn try_from(req: RadioLocationEstimatesReqV1) -> Result { + let timestamp = req.timestamp()?; + Ok(Self { + radio_id: req.radio_id, + estimates: req + .estimates + .into_iter() + .map(|e| e.try_into().unwrap()) + .collect(), + timestamp, + signer: req.signer.into(), + }) + } +} + +#[derive(Clone, Deserialize, Serialize, Debug, PartialEq)] +pub struct RadioLocationEstimate { + pub radius: Decimal, + pub confidence: Decimal, + pub events: Vec, +} + +impl TryFrom for RadioLocationEstimate { + type Error = Error; + fn try_from(estimate: RadioLocationEstimateV1) -> Result { + Ok(Self { + radius: to_rust_decimal(estimate.radius.unwrap()), + confidence: to_rust_decimal(estimate.confidence.unwrap()), + events: estimate + .events + .into_iter() + .map(|e| e.try_into().unwrap()) + .collect(), + }) + } +} + +#[derive(Clone, Deserialize, Serialize, Debug, PartialEq)] +pub struct RadioLocationEstimateEvent { + pub id: String, + pub timestamp: DateTime, +} + +impl MsgTimestamp>> for RleEventV1 { + fn timestamp(&self) -> Result> { + self.timestamp.to_timestamp() + } +} + +impl MsgTimestamp for RadioLocationEstimateEvent { + fn timestamp(&self) -> u64 { + self.timestamp.encode_timestamp() + } +} + +impl TryFrom for RadioLocationEstimateEvent { + type Error = Error; + fn try_from(event: RleEventV1) -> Result { + let timestamp = event.timestamp()?; + Ok(Self { + id: event.id, + timestamp, + }) + } +} + +fn to_rust_decimal(x: helium_proto::Decimal) -> rust_decimal::Decimal { + let str = x.value.as_str(); + rust_decimal::Decimal::from_str_exact(str).unwrap() +} diff --git a/file_store/src/traits/file_sink_write.rs b/file_store/src/traits/file_sink_write.rs index 54ea53100..19f77bd72 100644 --- a/file_store/src/traits/file_sink_write.rs +++ b/file_store/src/traits/file_sink_write.rs @@ -288,3 +288,8 @@ impl_file_sink!( FileType::VerifiedPromotionReward.to_str(), "verified_promotion_reward" ); +impl_file_sink!( + poc_mobile::RadioLocationEstimatesIngestReportV1, + FileType::RadioLocationEstimatesIngestReport.to_str(), + "radio_location_estimates_ingest_report" +); diff --git a/file_store/src/traits/msg_verify.rs b/file_store/src/traits/msg_verify.rs index 017a1e315..b9f7ceb9a 100644 --- a/file_store/src/traits/msg_verify.rs +++ b/file_store/src/traits/msg_verify.rs @@ -96,6 +96,7 @@ impl_msg_verify!(mobile_config::BoostedHexModifiedInfoStreamReqV1, signature); impl_msg_verify!(mobile_config::BoostedHexInfoStreamResV1, signature); impl_msg_verify!(poc_mobile::SubscriberVerifiedMappingEventReqV1, signature); impl_msg_verify!(poc_mobile::PromotionRewardReqV1, signature); +impl_msg_verify!(poc_mobile::RadioLocationEstimatesReqV1, signature); #[cfg(test)] mod test { diff --git a/ingest/src/server_mobile.rs b/ingest/src/server_mobile.rs index 71bf2bf07..10ce1befe 100644 --- a/ingest/src/server_mobile.rs +++ b/ingest/src/server_mobile.rs @@ -14,9 +14,13 @@ use helium_proto::services::poc_mobile::{ CoverageObjectIngestReportV1, CoverageObjectReqV1, CoverageObjectRespV1, DataTransferSessionIngestReportV1, DataTransferSessionReqV1, DataTransferSessionRespV1, InvalidatedRadioThresholdIngestReportV1, InvalidatedRadioThresholdReportReqV1, - InvalidatedRadioThresholdReportRespV1, PromotionRewardIngestReportV1, PromotionRewardReqV1, - PromotionRewardRespV1, RadioThresholdIngestReportV1, RadioThresholdReportReqV1, + InvalidatedRadioThresholdReportRespV1, InvalidatedRadioThresholdReportRespV1, + PromotionRewardIngestReportV1, PromotionRewardReqV1, PromotionRewardRespV1, + RadioLocationEstimatesIngestReportV1, RadioLocationEstimatesReqV1, + RadioLocationEstimatesRespV1, RadioThresholdIngestReportV1, RadioThresholdIngestReportV1, + RadioThresholdReportReqV1, RadioThresholdReportReqV1, RadioThresholdReportRespV1, RadioThresholdReportRespV1, ServiceProviderBoostedRewardsBannedRadioIngestReportV1, + ServiceProviderBoostedRewardsBannedRadioIngestReportV1, ServiceProviderBoostedRewardsBannedRadioReqV1, ServiceProviderBoostedRewardsBannedRadioRespV1, SpeedtestIngestReportV1, SpeedtestReqV1, SpeedtestRespV1, SubscriberLocationIngestReportV1, SubscriberLocationReqV1, SubscriberLocationRespV1, @@ -48,6 +52,7 @@ pub struct GrpcServer { FileSinkClient, subscriber_mapping_event_sink: FileSinkClient, promotion_reward_sink: FileSinkClient, + radio_location_estimate_sink: FileSinkClient, required_network: Network, address: SocketAddr, api_token: MetadataValue, @@ -88,6 +93,7 @@ impl GrpcServer { >, subscriber_mapping_event_sink: FileSinkClient, promotion_reward_sink: FileSinkClient, + radio_location_estimate_sink: FileSinkClient, required_network: Network, address: SocketAddr, api_token: MetadataValue, @@ -104,6 +110,7 @@ impl GrpcServer { sp_boosted_rewards_ban_sink, subscriber_mapping_event_sink, promotion_reward_sink, + radio_location_estimate_sink, required_network, address, api_token, @@ -465,6 +472,30 @@ impl poc_mobile::PocMobile for GrpcServer { let id = received_timestamp.to_string(); Ok(Response::new(PromotionRewardRespV1 { id })) } + + async fn submit_radio_location_estimates( + &self, + request: Request, + ) -> GrpcResult { + let timestamp: u64 = Utc::now().timestamp_millis() as u64; + let req: RadioLocationEstimatesReqV1 = request.into_inner(); + + custom_tracing::record_b58("pub_key", &req.signer); + + let report = self + .verify_public_key(req.signer.as_ref()) + .and_then(|public_key| self.verify_network(public_key)) + .and_then(|public_key| self.verify_signature(public_key, req)) + .map(|(_, req)| RadioLocationEstimatesIngestReportV1 { + received_timestamp: timestamp, + report: Some(req), + })?; + + _ = self.radio_location_estimate_sink.write(report, []).await; + + let id = timestamp.to_string(); + Ok(Response::new(RadioLocationEstimatesRespV1 { id })) + } } pub async fn grpc_server(settings: &Settings) -> Result<()> { @@ -584,6 +615,16 @@ pub async fn grpc_server(settings: &Settings) -> Result<()> { ) .await?; + let (radio_location_estimates_sink, radio_location_estimates_server) = + RadioLocationEstimatesIngestReportV1::file_sink( + store_base_path, + file_upload.clone(), + FileSinkCommitStrategy::Automatic, + FileSinkRollTime::Duration(settings.roll_time), + env!("CARGO_PKG_NAME"), + ) + .await?; + let Some(api_token) = settings .token .as_ref() @@ -604,6 +645,7 @@ pub async fn grpc_server(settings: &Settings) -> Result<()> { sp_boosted_rewards_ban_sink, subscriber_mapping_event_sink, subscriber_referral_eligibility_sink, + radio_location_estimates_sink, settings.network, settings.listen_addr, api_token, @@ -628,6 +670,7 @@ pub async fn grpc_server(settings: &Settings) -> Result<()> { .add_task(sp_boosted_rewards_ban_sink_server) .add_task(subscriber_mapping_event_server) .add_task(subscriber_referral_eligibility_server) + .add_task(radio_location_estimates_server) .add_task(grpc_server) .build() .start() diff --git a/ingest/tests/common/mod.rs b/ingest/tests/common/mod.rs index 5584416a0..e927a083f 100644 --- a/ingest/tests/common/mod.rs +++ b/ingest/tests/common/mod.rs @@ -45,6 +45,8 @@ pub async fn setup_mobile() -> anyhow::Result<(TestClient, Trigger)> { let (sp_boosted_tx, _rx) = tokio::sync::mpsc::channel(10); let (subscriber_mapping_tx, subscriber_mapping_rx) = tokio::sync::mpsc::channel(10); let (promotion_rewards_tx, _rx) = tokio::sync::mpsc::channel(10); + let (radio_location_estimates_tx, _radio_location_estimates_rx) = + tokio::sync::mpsc::channel(10); tokio::spawn(async move { let grpc_server = GrpcServer::new( @@ -59,6 +61,7 @@ pub async fn setup_mobile() -> anyhow::Result<(TestClient, Trigger)> { FileSinkClient::new(sp_boosted_tx, "noop"), FileSinkClient::new(subscriber_mapping_tx, "test_file_sink"), FileSinkClient::new(promotion_rewards_tx, "noop"), + FileSinkClient::new(radio_location_estimates_tx, "noop"), Network::MainNet, socket_addr, api_token, From 22f9ffc4a9fe975412f82eee5304f549b7766610 Mon Sep 17 00:00:00 2001 From: Macpie Date: Mon, 30 Sep 2024 12:38:10 -0700 Subject: [PATCH 02/31] Add ingest test --- Cargo.lock | 2 + ingest/Cargo.toml | 44 +++++++++++----------- ingest/tests/common/mod.rs | 69 +++++++++++++++++++++++++++++++---- ingest/tests/mobile_ingest.rs | 53 ++++++++++++++++++++++++++- 4 files changed, 139 insertions(+), 29 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 61aa1b268..4a033c247 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4377,6 +4377,8 @@ dependencies = [ "poc-metrics", "prost", "rand 0.8.5", + "rust_decimal", + "rust_decimal_macros", "serde", "serde_json", "sha2 0.10.8", diff --git a/ingest/Cargo.toml b/ingest/Cargo.toml index b8df84ebe..17b497c42 100644 --- a/ingest/Cargo.toml +++ b/ingest/Cargo.toml @@ -8,36 +8,38 @@ license.workspace = true [dependencies] anyhow = { workspace = true } -config = { workspace = true } -clap = { workspace = true } -thiserror = { workspace = true } -serde = { workspace = true } -serde_json = { workspace = true } base64 = { workspace = true } bs58 = { workspace = true } -sha2 = { workspace = true } -http = { workspace = true } -tonic = { workspace = true } -triggered = { workspace = true } +chrono = { workspace = true } +clap = { workspace = true } +config = { workspace = true } +custom-tracing = { path = "../custom_tracing", features = ["grpc"] } +file-store = { path = "../file_store" } futures = { workspace = true } futures-util = { workspace = true } +helium-crypto = { workspace = true } +helium-proto = { workspace = true } +http = { workspace = true } +humantime-serde = { workspace = true } +metrics = { workspace = true } +metrics-exporter-prometheus = { workspace = true } +poc-metrics = { path = "../metrics" } prost = { workspace = true } +rand = { workspace = true } +rust_decimal = { workspace = true } +rust_decimal_macros = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +sha2 = { workspace = true } +task-manager = { path = "../task_manager" } +thiserror = { workspace = true } tokio = { workspace = true } -tokio-util = { workspace = true } tokio-stream = { workspace = true } +tokio-util = { workspace = true } +tonic = { workspace = true } tracing = { workspace = true } tracing-subscriber = { workspace = true } -chrono = { workspace = true } -helium-proto = { workspace = true } -helium-crypto = { workspace = true } -file-store = { path = "../file_store" } -poc-metrics = { path = "../metrics" } -metrics = { workspace = true } -metrics-exporter-prometheus = { workspace = true } -task-manager = { path = "../task_manager" } -rand = { workspace = true } -custom-tracing = { path = "../custom_tracing", features = ["grpc"] } -humantime-serde = { workspace = true } +triggered = { workspace = true } [dev-dependencies] backon = "0" diff --git a/ingest/tests/common/mod.rs b/ingest/tests/common/mod.rs index e927a083f..b06eedb70 100644 --- a/ingest/tests/common/mod.rs +++ b/ingest/tests/common/mod.rs @@ -3,8 +3,10 @@ use backon::{ExponentialBuilder, Retryable}; use file_store::file_sink::FileSinkClient; use helium_crypto::{KeyTag, Keypair, Network, Sign}; use helium_proto::services::poc_mobile::{ - Client as PocMobileClient, SubscriberVerifiedMappingEventIngestReportV1, - SubscriberVerifiedMappingEventReqV1, SubscriberVerifiedMappingEventResV1, + Client as PocMobileClient, RadioLocationEstimateV1, RadioLocationEstimatesIngestReportV1, + RadioLocationEstimatesReqV1, RadioLocationEstimatesRespV1, + SubscriberVerifiedMappingEventIngestReportV1, SubscriberVerifiedMappingEventReqV1, + SubscriberVerifiedMappingEventResV1, }; use ingest::server_mobile::GrpcServer; use prost::Message; @@ -75,6 +77,7 @@ pub async fn setup_mobile() -> anyhow::Result<(TestClient, Trigger)> { key_pair, token.to_string(), subscriber_mapping_rx, + radio_location_estimates_rx, ) .await; @@ -85,8 +88,10 @@ pub struct TestClient { client: PocMobileClient, key_pair: Arc, authorization: MetadataValue, - file_sink_rx: + subscriber_mapping_rx: Receiver>, + radio_location_estimates_rx: + Receiver>, } impl TestClient { @@ -94,9 +99,12 @@ impl TestClient { socket_addr: SocketAddr, key_pair: Keypair, api_token: String, - file_sink_rx: Receiver< + subscriber_mapping_rx: Receiver< file_store::file_sink::Message, >, + radio_location_estimates_rx: Receiver< + file_store::file_sink::Message, + >, ) -> TestClient { let client = (|| PocMobileClient::connect(format!("http://{socket_addr}"))) .retry(&ExponentialBuilder::default()) @@ -107,12 +115,34 @@ impl TestClient { client, key_pair: Arc::new(key_pair), authorization: format!("Bearer {}", api_token).try_into().unwrap(), - file_sink_rx, + subscriber_mapping_rx, + radio_location_estimates_rx, + } + } + + pub async fn recv_subscriber_mapping( + mut self, + ) -> anyhow::Result { + match timeout(Duration::from_secs(2), self.subscriber_mapping_rx.recv()).await { + Ok(Some(msg)) => match msg { + file_store::file_sink::Message::Commit(_) => bail!("got Commit"), + file_store::file_sink::Message::Rollback(_) => bail!("got Rollback"), + file_store::file_sink::Message::Data(_, data) => Ok(data), + }, + Ok(None) => bail!("got none"), + Err(reason) => bail!("got error {reason}"), } } - pub async fn recv(mut self) -> anyhow::Result { - match timeout(Duration::from_secs(2), self.file_sink_rx.recv()).await { + pub async fn recv_radio_location_estimates( + mut self, + ) -> anyhow::Result { + match timeout( + Duration::from_secs(2), + self.radio_location_estimates_rx.recv(), + ) + .await + { Ok(Some(msg)) => match msg { file_store::file_sink::Message::Commit(_) => bail!("got Commit"), file_store::file_sink::Message::Rollback(_) => bail!("got Rollback"), @@ -150,6 +180,31 @@ impl TestClient { Ok(res.into_inner()) } + + pub async fn submit_radio_location_estimates( + &mut self, + radio_id: String, + estimates: Vec, + ) -> anyhow::Result { + let mut req = RadioLocationEstimatesReqV1 { + radio_id, + estimates, + timestamp: 0, + signer: self.key_pair.public_key().to_vec(), + signature: vec![], + }; + + req.signature = self.key_pair.sign(&req.encode_to_vec()).expect("sign"); + + let mut request = Request::new(req); + let metadata = request.metadata_mut(); + + metadata.insert("authorization", self.authorization.clone()); + + let res = self.client.submit_radio_location_estimates(request).await?; + + Ok(res.into_inner()) + } } pub fn generate_keypair() -> Keypair { diff --git a/ingest/tests/mobile_ingest.rs b/ingest/tests/mobile_ingest.rs index 477e2ede2..c555ae375 100644 --- a/ingest/tests/mobile_ingest.rs +++ b/ingest/tests/mobile_ingest.rs @@ -1,3 +1,6 @@ +use helium_proto::services::poc_mobile::{RadioLocationEstimateV1, RleEventV1}; +use rust_decimal::prelude::*; + mod common; #[tokio::test] @@ -15,7 +18,7 @@ async fn submit_verified_subscriber_mapping_event() -> anyhow::Result<()> { let timestamp: String = res.unwrap().id; - match client.recv().await { + match client.recv_subscriber_mapping().await { Ok(report) => { assert_eq!(timestamp, report.received_timestamp.to_string()); @@ -33,3 +36,51 @@ async fn submit_verified_subscriber_mapping_event() -> anyhow::Result<()> { trigger.trigger(); Ok(()) } + +#[tokio::test] +async fn submit_radio_location_estimates() -> anyhow::Result<()> { + let (mut client, trigger) = common::setup_mobile().await?; + + let radio_id = "radio_id".to_string(); + let estimates = vec![RadioLocationEstimateV1 { + radius: to_proto_decimal(2.0), + confidence: to_proto_decimal(0.75), + events: vec![RleEventV1 { + id: "event_1".to_string(), + timestamp: 0, + }], + }]; + + let res = client + .submit_radio_location_estimates(radio_id.clone(), estimates.clone()) + .await; + + assert!(res.is_ok()); + + let timestamp: String = res.unwrap().id; + + match client.recv_radio_location_estimates().await { + Ok(report) => { + assert_eq!(timestamp, report.received_timestamp.to_string()); + + match report.report { + None => panic!("No report found"), + Some(req) => { + assert_eq!(radio_id, req.radio_id); + assert_eq!(estimates, req.estimates); + } + } + } + Err(e) => panic!("got error {e}"), + } + + trigger.trigger(); + Ok(()) +} + +fn to_proto_decimal(x: f64) -> Option { + let d = Decimal::from_f64(x).unwrap(); + Some(helium_proto::Decimal { + value: d.to_string(), + }) +} From 4ae1e0dc83fbbe5fbe2086afbc11faacb885cd1f Mon Sep 17 00:00:00 2001 From: Macpie Date: Tue, 1 Oct 2024 11:40:09 -0700 Subject: [PATCH 03/31] basic verifier daemon --- file_store/src/file_info.rs | 6 + file_store/src/lib.rs | 2 + file_store/src/radio_location_estimates.rs | 59 ++++++++- .../radio_location_estimates_ingest_report.rs | 58 +++++++++ file_store/src/traits/file_sink_write.rs | 5 + .../src/verified_radio_location_estimates.rs | 63 ++++++++++ ingest/src/server_mobile.rs | 4 +- ingest/tests/common/mod.rs | 2 +- mobile_verifier/src/lib.rs | 1 + .../src/radio_location_estimates.rs | 112 ++++++++++++++++++ 10 files changed, 303 insertions(+), 9 deletions(-) create mode 100644 file_store/src/radio_location_estimates_ingest_report.rs create mode 100644 file_store/src/verified_radio_location_estimates.rs create mode 100644 mobile_verifier/src/radio_location_estimates.rs diff --git a/file_store/src/file_info.rs b/file_store/src/file_info.rs index aac8ecf33..a7b18fcde 100644 --- a/file_store/src/file_info.rs +++ b/file_store/src/file_info.rs @@ -169,6 +169,8 @@ pub const PROMOTION_REWARD_INGEST_REPORT: &str = "promotion_reward_ingest_report pub const VERIFIED_PROMOTION_REWARD: &str = "verified_promotion_reward"; pub const SERVICE_PROVIDER_PROMOTION_FUND: &str = "service_provider_promotion_fund"; pub const RADIO_LOCATION_ESTIMATES_INGEST_REPORT: &str = "radio_location_estimates_ingest_report"; +pub const VERIFIED_RADIO_LOCATION_ESTIMATES_REPORT: &str = + "verified_radio_location_estimates_report"; #[derive(Debug, PartialEq, Eq, Clone, Serialize, Copy, strum::EnumCount)] #[serde(rename_all = "snake_case")] @@ -230,6 +232,7 @@ pub enum FileType { VerifiedPromotionReward, ServiceProviderPromotionFund, RadioLocationEstimatesIngestReport, + VerifiedRadioLocationEstimatesReport, } impl fmt::Display for FileType { @@ -306,6 +309,7 @@ impl fmt::Display for FileType { Self::VerifiedPromotionReward => VERIFIED_PROMOTION_REWARD, Self::ServiceProviderPromotionFund => SERVICE_PROVIDER_PROMOTION_FUND, Self::RadioLocationEstimatesIngestReport => RADIO_LOCATION_ESTIMATES_INGEST_REPORT, + Self::VerifiedRadioLocationEstimatesReport => VERIFIED_RADIO_LOCATION_ESTIMATES_REPORT, }; f.write_str(s) } @@ -385,6 +389,7 @@ impl FileType { Self::VerifiedPromotionReward => VERIFIED_PROMOTION_REWARD, Self::ServiceProviderPromotionFund => SERVICE_PROVIDER_PROMOTION_FUND, Self::RadioLocationEstimatesIngestReport => RADIO_LOCATION_ESTIMATES_INGEST_REPORT, + Self::VerifiedRadioLocationEstimatesReport => VERIFIED_RADIO_LOCATION_ESTIMATES_REPORT, } } } @@ -463,6 +468,7 @@ impl FromStr for FileType { VERIFIED_PROMOTION_REWARD => Self::VerifiedPromotionReward, SERVICE_PROVIDER_PROMOTION_FUND => Self::ServiceProviderPromotionFund, RADIO_LOCATION_ESTIMATES_INGEST_REPORT => Self::RadioLocationEstimatesIngestReport, + VERIFIED_RADIO_LOCATION_ESTIMATES_REPORT => Self::VerifiedRadioLocationEstimatesReport, _ => return Err(Error::from(io::Error::from(io::ErrorKind::InvalidInput))), }; Ok(result) diff --git a/file_store/src/lib.rs b/file_store/src/lib.rs index c6fc5d6e8..d8737ac8c 100644 --- a/file_store/src/lib.rs +++ b/file_store/src/lib.rs @@ -22,12 +22,14 @@ pub mod mobile_subscriber; pub mod mobile_transfer; pub mod promotion_reward; pub mod radio_location_estimates; +pub mod radio_location_estimates_ingest_report; pub mod reward_manifest; mod settings; pub mod speedtest; pub mod subscriber_verified_mapping_event; pub mod subscriber_verified_mapping_event_ingest_report; pub mod traits; +pub mod verified_radio_location_estimates; pub mod verified_subscriber_verified_mapping_event_ingest_report; pub mod wifi_heartbeat; diff --git a/file_store/src/radio_location_estimates.rs b/file_store/src/radio_location_estimates.rs index 219a52ae0..81aa93ac0 100644 --- a/file_store/src/radio_location_estimates.rs +++ b/file_store/src/radio_location_estimates.rs @@ -11,14 +11,14 @@ use rust_decimal::Decimal; use serde::{Deserialize, Serialize}; #[derive(Clone, Deserialize, Serialize, Debug, PartialEq)] -pub struct RadioLocationEstimates { +pub struct RadioLocationEstimatesReq { pub radio_id: String, pub estimates: Vec, pub timestamp: DateTime, - pub signer: PublicKeyBinary, + pub carrier_key: PublicKeyBinary, } -impl MsgDecode for RadioLocationEstimates { +impl MsgDecode for RadioLocationEstimatesReq { type Msg = RadioLocationEstimatesReqV1; } @@ -28,13 +28,30 @@ impl MsgTimestamp>> for RadioLocationEstimatesReqV1 { } } -impl MsgTimestamp for RadioLocationEstimates { +impl MsgTimestamp for RadioLocationEstimatesReq { fn timestamp(&self) -> u64 { self.timestamp.encode_timestamp() } } -impl TryFrom for RadioLocationEstimates { +impl From for RadioLocationEstimatesReqV1 { + fn from(rle: RadioLocationEstimatesReq) -> Self { + let timestamp = rle.timestamp(); + RadioLocationEstimatesReqV1 { + radio_id: rle.radio_id, + estimates: rle + .estimates + .into_iter() + .map(|e| e.try_into().unwrap()) + .collect(), + timestamp, + carrier_key: rle.carrier_key.into(), + signature: vec![], + } + } +} + +impl TryFrom for RadioLocationEstimatesReq { type Error = Error; fn try_from(req: RadioLocationEstimatesReqV1) -> Result { let timestamp = req.timestamp()?; @@ -46,7 +63,7 @@ impl TryFrom for RadioLocationEstimates { .map(|e| e.try_into().unwrap()) .collect(), timestamp, - signer: req.signer.into(), + carrier_key: req.carrier_key.into(), }) } } @@ -58,6 +75,20 @@ pub struct RadioLocationEstimate { pub events: Vec, } +impl From for RadioLocationEstimateV1 { + fn from(rle: RadioLocationEstimate) -> Self { + RadioLocationEstimateV1 { + radius: Some(to_proto_decimal(rle.radius)), + confidence: Some(to_proto_decimal(rle.confidence)), + events: rle + .events + .into_iter() + .map(|e| e.try_into().unwrap()) + .collect(), + } + } +} + impl TryFrom for RadioLocationEstimate { type Error = Error; fn try_from(estimate: RadioLocationEstimateV1) -> Result { @@ -91,6 +122,16 @@ impl MsgTimestamp for RadioLocationEstimateEvent { } } +impl From for RleEventV1 { + fn from(event: RadioLocationEstimateEvent) -> Self { + let timestamp = event.timestamp(); + RleEventV1 { + id: event.id, + timestamp, + } + } +} + impl TryFrom for RadioLocationEstimateEvent { type Error = Error; fn try_from(event: RleEventV1) -> Result { @@ -106,3 +147,9 @@ fn to_rust_decimal(x: helium_proto::Decimal) -> rust_decimal::Decimal { let str = x.value.as_str(); rust_decimal::Decimal::from_str_exact(str).unwrap() } + +fn to_proto_decimal(x: rust_decimal::Decimal) -> helium_proto::Decimal { + helium_proto::Decimal { + value: x.to_string(), + } +} diff --git a/file_store/src/radio_location_estimates_ingest_report.rs b/file_store/src/radio_location_estimates_ingest_report.rs new file mode 100644 index 000000000..26d8e32e2 --- /dev/null +++ b/file_store/src/radio_location_estimates_ingest_report.rs @@ -0,0 +1,58 @@ +use crate::{ + radio_location_estimates::RadioLocationEstimatesReq, + traits::{MsgDecode, MsgTimestamp, TimestampDecode, TimestampEncode}, + Error, Result, +}; +use chrono::{DateTime, Utc}; +use helium_proto::services::poc_mobile::{ + RadioLocationEstimatesIngestReportV1, RadioLocationEstimatesReqV1, +}; +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Deserialize, Serialize, Debug, PartialEq)] +pub struct RadioLocationEstimatesIngestReport { + pub received_timestamp: DateTime, + pub report: RadioLocationEstimatesReq, +} + +impl MsgDecode for RadioLocationEstimatesIngestReport { + type Msg = RadioLocationEstimatesIngestReportV1; +} + +impl MsgTimestamp>> for RadioLocationEstimatesIngestReportV1 { + fn timestamp(&self) -> Result> { + self.received_timestamp.to_timestamp() + } +} + +impl MsgTimestamp for RadioLocationEstimatesIngestReport { + fn timestamp(&self) -> u64 { + self.received_timestamp.encode_timestamp() + } +} + +impl From for RadioLocationEstimatesIngestReportV1 { + fn from(v: RadioLocationEstimatesIngestReport) -> Self { + let received_timestamp = v.timestamp(); + let report: RadioLocationEstimatesReqV1 = v.report.into(); + Self { + received_timestamp, + report: Some(report), + } + } +} + +impl TryFrom for RadioLocationEstimatesIngestReport { + type Error = Error; + fn try_from(v: RadioLocationEstimatesIngestReportV1) -> Result { + Ok(Self { + received_timestamp: v.timestamp()?, + report: v + .report + .ok_or_else(|| { + Error::not_found("ingest RadioLocationEstimatesIngestReport report") + })? + .try_into()?, + }) + } +} diff --git a/file_store/src/traits/file_sink_write.rs b/file_store/src/traits/file_sink_write.rs index 19f77bd72..1922ca228 100644 --- a/file_store/src/traits/file_sink_write.rs +++ b/file_store/src/traits/file_sink_write.rs @@ -293,3 +293,8 @@ impl_file_sink!( FileType::RadioLocationEstimatesIngestReport.to_str(), "radio_location_estimates_ingest_report" ); +impl_file_sink!( + poc_mobile::VerifiedRadioLocationEstimatesReportV1, + FileType::VerifiedRadioLocationEstimatesReport.to_str(), + "verified_radio_location_estimates_report" +); diff --git a/file_store/src/verified_radio_location_estimates.rs b/file_store/src/verified_radio_location_estimates.rs new file mode 100644 index 000000000..8602a10d4 --- /dev/null +++ b/file_store/src/verified_radio_location_estimates.rs @@ -0,0 +1,63 @@ +use crate::{ + radio_location_estimates_ingest_report::RadioLocationEstimatesIngestReport, + traits::{MsgDecode, MsgTimestamp, TimestampDecode, TimestampEncode}, + Error, Result, +}; +use chrono::{DateTime, Utc}; +use helium_proto::services::poc_mobile::{ + RadioLocationEstimatesIngestReportV1, RadioLocationEstimatesVerificationStatus, + VerifiedRadioLocationEstimatesReportV1, +}; +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Deserialize, Serialize, Debug, PartialEq)] +pub struct VerifiedRadioLocationEstimatesReport { + pub report: RadioLocationEstimatesIngestReport, + pub status: RadioLocationEstimatesVerificationStatus, + pub timestamp: DateTime, +} + +impl MsgDecode for VerifiedRadioLocationEstimatesReport { + type Msg = VerifiedRadioLocationEstimatesReportV1; +} + +impl MsgTimestamp>> for VerifiedRadioLocationEstimatesReportV1 { + fn timestamp(&self) -> Result> { + self.timestamp.to_timestamp() + } +} + +impl MsgTimestamp for VerifiedRadioLocationEstimatesReport { + fn timestamp(&self) -> u64 { + self.timestamp.encode_timestamp() + } +} + +impl From for VerifiedRadioLocationEstimatesReportV1 { + fn from(v: VerifiedRadioLocationEstimatesReport) -> Self { + let timestamp = v.timestamp(); + let report: RadioLocationEstimatesIngestReportV1 = v.report.into(); + Self { + report: Some(report), + status: v.status as i32, + timestamp, + } + } +} + +impl TryFrom for VerifiedRadioLocationEstimatesReport { + type Error = Error; + fn try_from(v: VerifiedRadioLocationEstimatesReportV1) -> Result { + Ok(Self { + report: v + .clone() + .report + .ok_or_else(|| { + Error::not_found("ingest VerifiedRadioLocationEstimatesReport report") + })? + .try_into()?, + status: v.status.try_into()?, + timestamp: v.timestamp()?, + }) + } +} diff --git a/ingest/src/server_mobile.rs b/ingest/src/server_mobile.rs index 10ce1befe..036946715 100644 --- a/ingest/src/server_mobile.rs +++ b/ingest/src/server_mobile.rs @@ -480,10 +480,10 @@ impl poc_mobile::PocMobile for GrpcServer { let timestamp: u64 = Utc::now().timestamp_millis() as u64; let req: RadioLocationEstimatesReqV1 = request.into_inner(); - custom_tracing::record_b58("pub_key", &req.signer); + custom_tracing::record_b58("pub_key", &req.carrier_key); let report = self - .verify_public_key(req.signer.as_ref()) + .verify_public_key(req.carrier_key.as_ref()) .and_then(|public_key| self.verify_network(public_key)) .and_then(|public_key| self.verify_signature(public_key, req)) .map(|(_, req)| RadioLocationEstimatesIngestReportV1 { diff --git a/ingest/tests/common/mod.rs b/ingest/tests/common/mod.rs index b06eedb70..a25fe9fac 100644 --- a/ingest/tests/common/mod.rs +++ b/ingest/tests/common/mod.rs @@ -190,7 +190,7 @@ impl TestClient { radio_id, estimates, timestamp: 0, - signer: self.key_pair.public_key().to_vec(), + carrier_key: self.key_pair.public_key().to_vec(), signature: vec![], }; diff --git a/mobile_verifier/src/lib.rs b/mobile_verifier/src/lib.rs index 9fc3757c0..5a7645b43 100644 --- a/mobile_verifier/src/lib.rs +++ b/mobile_verifier/src/lib.rs @@ -5,6 +5,7 @@ pub mod coverage; pub mod data_session; pub mod geofence; pub mod heartbeats; +pub mod radio_location_estimates; pub mod radio_threshold; pub mod reward_shares; pub mod rewarder; diff --git a/mobile_verifier/src/radio_location_estimates.rs b/mobile_verifier/src/radio_location_estimates.rs new file mode 100644 index 000000000..d02cafb72 --- /dev/null +++ b/mobile_verifier/src/radio_location_estimates.rs @@ -0,0 +1,112 @@ +use crate::Settings; +use file_store::{ + file_info_poller::{FileInfoStream, LookbackBehavior}, + file_sink::FileSinkClient, + file_source, + file_upload::FileUpload, + radio_location_estimates_ingest_report::RadioLocationEstimatesIngestReport, + traits::{FileSinkCommitStrategy, FileSinkRollTime, FileSinkWriteExt}, + FileStore, FileType, +}; +use helium_proto::services::poc_mobile::{ + RadioLocationEstimatesIngestReportV1, VerifiedRadioLocationEstimatesReportV1, +}; +use mobile_config::client::authorization_client::AuthorizationVerifier; +use sqlx::{Pool, Postgres}; +use task_manager::{ManagedTask, TaskManager}; +use tokio::sync::mpsc::Receiver; + +pub struct RadioLocationEstimatesDaemon { + pool: Pool, + authorization_verifier: AV, + reports_receiver: Receiver>, + verified_report_sink: FileSinkClient, +} + +impl RadioLocationEstimatesDaemon +where + AV: AuthorizationVerifier + Send + Sync + 'static, +{ + pub fn new( + pool: Pool, + authorization_verifier: AV, + reports_receiver: Receiver>, + verified_report_sink: FileSinkClient, + ) -> Self { + Self { + pool, + authorization_verifier, + reports_receiver, + verified_report_sink, + } + } + + pub async fn create_managed_task( + pool: Pool, + settings: &Settings, + authorization_verifier: AV, + file_store: FileStore, + file_upload: FileUpload, + ) -> anyhow::Result { + let (reports_receiver, reports_receiver_server) = + file_source::continuous_source::() + .state(pool.clone()) + .store(file_store) + .lookback(LookbackBehavior::StartAfter(settings.start_after)) + .prefix(FileType::RadioLocationEstimatesIngestReport.to_string()) + .create() + .await?; + + let (verified_report_sink, verified_report_sink_server) = + VerifiedRadioLocationEstimatesReportV1::file_sink( + settings.store_base_path(), + file_upload.clone(), + FileSinkCommitStrategy::Manual, + FileSinkRollTime::Default, + env!("CARGO_PKG_NAME"), + ) + .await?; + + let task = Self::new( + pool, + authorization_verifier, + reports_receiver, + verified_report_sink, + ); + + Ok(TaskManager::builder() + .add_task(reports_receiver_server) + .add_task(verified_report_sink_server) + .add_task(task) + .build()) + } + + pub async fn run(mut self, shutdown: triggered::Listener) -> anyhow::Result<()> { + tracing::info!("Starting sme deamon"); + loop { + tokio::select! { + biased; + _ = shutdown.clone() => { + tracing::info!("sme deamon shutting down"); + break; + } + Some(_file) = self.reports_receiver.recv() => { + // self.process_file(file).await?; + } + } + } + Ok(()) + } +} + +impl ManagedTask for RadioLocationEstimatesDaemon +where + AV: AuthorizationVerifier + Send + Sync + 'static, +{ + fn start_task( + self: Box, + shutdown: triggered::Listener, + ) -> futures::future::LocalBoxFuture<'static, anyhow::Result<()>> { + Box::pin(self.run(shutdown)) + } +} From d9fc8e9e9b838da9f3dccbff40f932b4c43e28da Mon Sep 17 00:00:00 2001 From: Macpie Date: Tue, 1 Oct 2024 14:36:09 -0700 Subject: [PATCH 04/31] Process reports --- Cargo.lock | 1 + mobile_verifier/Cargo.toml | 1 + .../37_radio_location_estimates.sql | 12 ++ .../src/radio_location_estimates.rs | 147 +++++++++++++++++- 4 files changed, 156 insertions(+), 5 deletions(-) create mode 100644 mobile_verifier/migrations/37_radio_location_estimates.sql diff --git a/Cargo.lock b/Cargo.lock index 4a033c247..d2341be5f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5260,6 +5260,7 @@ dependencies = [ "h3o", "helium-crypto", "helium-proto", + "hex", "hex-assignments", "hextree", "http-serde", diff --git a/mobile_verifier/Cargo.toml b/mobile_verifier/Cargo.toml index 84e6c61b3..62a60e666 100644 --- a/mobile_verifier/Cargo.toml +++ b/mobile_verifier/Cargo.toml @@ -60,6 +60,7 @@ custom-tracing = { path = "../custom_tracing" } hex-assignments = { path = "../hex_assignments" } coverage-point-calculator = { path = "../coverage_point_calculator" } coverage-map = { path = "../coverage_map" } +hex = "0.4" [dev-dependencies] backon = "0" diff --git a/mobile_verifier/migrations/37_radio_location_estimates.sql b/mobile_verifier/migrations/37_radio_location_estimates.sql new file mode 100644 index 000000000..96b3e7dd3 --- /dev/null +++ b/mobile_verifier/migrations/37_radio_location_estimates.sql @@ -0,0 +1,12 @@ +CREATE TABLE IF NOT EXISTS radio_location_estimates ( + hashed_key TEXT NOT NULL, + radio_id TEXT NOT NULL, + received_timestamp TIMESTAMPTZ NOT NULL, + radius DECIMAL NOT NULL, + lat DECIMAL NOT NULL, + long DECIMAL NOT NULL, + confidence DECIMAL NOT NULL, + is_valid BOOLEAN NOT NULL, + inserted_at TIMESTAMPTZ DEFAULT now(), + PRIMARY KEY (hashed_key) +); \ No newline at end of file diff --git a/mobile_verifier/src/radio_location_estimates.rs b/mobile_verifier/src/radio_location_estimates.rs index d02cafb72..67fe25a43 100644 --- a/mobile_verifier/src/radio_location_estimates.rs +++ b/mobile_verifier/src/radio_location_estimates.rs @@ -1,18 +1,28 @@ use crate::Settings; +use chrono::{DateTime, Utc}; use file_store::{ file_info_poller::{FileInfoStream, LookbackBehavior}, file_sink::FileSinkClient, file_source, file_upload::FileUpload, + radio_location_estimates::{RadioLocationEstimate, RadioLocationEstimatesReq}, radio_location_estimates_ingest_report::RadioLocationEstimatesIngestReport, traits::{FileSinkCommitStrategy, FileSinkRollTime, FileSinkWriteExt}, + verified_radio_location_estimates::VerifiedRadioLocationEstimatesReport, FileStore, FileType, }; -use helium_proto::services::poc_mobile::{ - RadioLocationEstimatesIngestReportV1, VerifiedRadioLocationEstimatesReportV1, +use futures::{StreamExt, TryStreamExt}; +use helium_crypto::PublicKeyBinary; +use helium_proto::services::{ + mobile_config::NetworkKeyRole, + poc_mobile::{ + RadioLocationEstimatesIngestReportV1, RadioLocationEstimatesVerificationStatus, + VerifiedRadioLocationEstimatesReportV1, + }, }; use mobile_config::client::authorization_client::AuthorizationVerifier; -use sqlx::{Pool, Postgres}; +use sha2::{Digest, Sha256}; +use sqlx::{Pool, Postgres, Transaction}; use task_manager::{ManagedTask, TaskManager}; use tokio::sync::mpsc::Receiver; @@ -90,13 +100,82 @@ where tracing::info!("sme deamon shutting down"); break; } - Some(_file) = self.reports_receiver.recv() => { - // self.process_file(file).await?; + Some(file) = self.reports_receiver.recv() => { + self.process_file(file).await?; } } } Ok(()) } + + async fn process_file( + &self, + file_info_stream: FileInfoStream, + ) -> anyhow::Result<()> { + tracing::info!( + "Processing Radio Location Estimates file {}", + file_info_stream.file_info.key + ); + + let mut transaction = self.pool.begin().await?; + + file_info_stream + .into_stream(&mut transaction) + .await? + .map(anyhow::Ok) + .try_fold( + transaction, + |mut transaction, report: RadioLocationEstimatesIngestReport| async move { + let verified_report_status = self.verify_report(&report.report).await; + + if verified_report_status == RadioLocationEstimatesVerificationStatus::Valid { + save_to_db(&report, &mut transaction).await?; + } + + let verified_report_proto: VerifiedRadioLocationEstimatesReportV1 = + VerifiedRadioLocationEstimatesReport { + report, + status: verified_report_status, + timestamp: Utc::now(), + } + .into(); + + self.verified_report_sink + .write( + verified_report_proto, + &[("report_status", verified_report_status.as_str_name())], + ) + .await?; + + Ok(transaction) + }, + ) + .await? + .commit() + .await?; + + self.verified_report_sink.commit().await?; + + Ok(()) + } + + async fn verify_report( + &self, + req: &RadioLocationEstimatesReq, + ) -> RadioLocationEstimatesVerificationStatus { + if !self.verify_known_carrier_key(&req.carrier_key).await { + return RadioLocationEstimatesVerificationStatus::InvalidKey; + } + + RadioLocationEstimatesVerificationStatus::Valid + } + + async fn verify_known_carrier_key(&self, public_key: &PublicKeyBinary) -> bool { + self.authorization_verifier + .verify_authorized_key(public_key, NetworkKeyRole::MobileCarrier) + .await + .unwrap_or_default() + } } impl ManagedTask for RadioLocationEstimatesDaemon @@ -110,3 +189,61 @@ where Box::pin(self.run(shutdown)) } } + +async fn save_to_db( + report: &RadioLocationEstimatesIngestReport, + exec: &mut Transaction<'_, Postgres>, +) -> Result<(), sqlx::Error> { + let estimates = &report.report.estimates; + for estimate in estimates { + insert_estimate( + report.report.radio_id.clone(), + report.received_timestamp, + estimate, + exec, + ) + .await?; + } + + Ok(()) +} + +async fn insert_estimate( + radio_id: String, + received_timestamp: DateTime, + estimate: &RadioLocationEstimate, + exec: &mut Transaction<'_, Postgres>, +) -> Result<(), sqlx::Error> { + let radius = estimate.radius; + let lat = 0.0; + let long = 0.0; + + let key = format!( + "{}{}{}{}{}", + radio_id, received_timestamp, radius, lat, long + ); + + let mut hasher = Sha256::new(); + hasher.update(key); + let hashed_key = hasher.finalize(); + + sqlx::query( + r#" + INSERT INTO radio_location_estimates (hashed_key, radio_id, received_timestamp, radius, lat, long, confidence, is_valid) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8) + ON CONFLICT (hashed_key) DO NOTHING + "#, + ) + .bind(hex::encode(hashed_key)) + .bind(radio_id) + .bind(received_timestamp) + .bind(estimate.radius) + .bind(lat) + .bind(long) + .bind(estimate.confidence) + .bind(true) + .execute(exec) + .await?; + + Ok(()) +} From da4d5e82e2291ec71a261fc5f2502e004c30904e Mon Sep 17 00:00:00 2001 From: Macpie Date: Tue, 1 Oct 2024 14:42:16 -0700 Subject: [PATCH 05/31] Clippy --- file_store/src/radio_location_estimates.rs | 12 ++---------- mobile_verifier/src/radio_location_estimates.rs | 3 +-- 2 files changed, 3 insertions(+), 12 deletions(-) diff --git a/file_store/src/radio_location_estimates.rs b/file_store/src/radio_location_estimates.rs index 81aa93ac0..42838ec9a 100644 --- a/file_store/src/radio_location_estimates.rs +++ b/file_store/src/radio_location_estimates.rs @@ -39,11 +39,7 @@ impl From for RadioLocationEstimatesReqV1 { let timestamp = rle.timestamp(); RadioLocationEstimatesReqV1 { radio_id: rle.radio_id, - estimates: rle - .estimates - .into_iter() - .map(|e| e.try_into().unwrap()) - .collect(), + estimates: rle.estimates.into_iter().map(|e| e.into()).collect(), timestamp, carrier_key: rle.carrier_key.into(), signature: vec![], @@ -80,11 +76,7 @@ impl From for RadioLocationEstimateV1 { RadioLocationEstimateV1 { radius: Some(to_proto_decimal(rle.radius)), confidence: Some(to_proto_decimal(rle.confidence)), - events: rle - .events - .into_iter() - .map(|e| e.try_into().unwrap()) - .collect(), + events: rle.events.into_iter().map(|e| e.into()).collect(), } } } diff --git a/mobile_verifier/src/radio_location_estimates.rs b/mobile_verifier/src/radio_location_estimates.rs index 67fe25a43..928a4ed66 100644 --- a/mobile_verifier/src/radio_location_estimates.rs +++ b/mobile_verifier/src/radio_location_estimates.rs @@ -16,8 +16,7 @@ use helium_crypto::PublicKeyBinary; use helium_proto::services::{ mobile_config::NetworkKeyRole, poc_mobile::{ - RadioLocationEstimatesIngestReportV1, RadioLocationEstimatesVerificationStatus, - VerifiedRadioLocationEstimatesReportV1, + RadioLocationEstimatesVerificationStatus, VerifiedRadioLocationEstimatesReportV1, }, }; use mobile_config::client::authorization_client::AuthorizationVerifier; From cfbcc1a36fe90e1bc96c783b36ec8f00cb3d50b5 Mon Sep 17 00:00:00 2001 From: Macpie Date: Tue, 1 Oct 2024 14:42:23 -0700 Subject: [PATCH 06/31] Order dependencies --- mobile_verifier/Cargo.toml | 74 +++++++++++++++++++------------------- 1 file changed, 37 insertions(+), 37 deletions(-) diff --git a/mobile_verifier/Cargo.toml b/mobile_verifier/Cargo.toml index 62a60e666..dad9bf2d2 100644 --- a/mobile_verifier/Cargo.toml +++ b/mobile_verifier/Cargo.toml @@ -9,58 +9,58 @@ authors.workspace = true [dependencies] anyhow = { workspace = true } async-compression = { version = "0", features = ["tokio", "gzip"] } -config = { workspace = true } -thiserror = { workspace = true } -serde = { workspace = true } -serde_json = { workspace = true } -h3o = { workspace = true, features = ["geo"] } -hextree = { workspace = true } -http-serde = { workspace = true } -clap = { workspace = true } -sqlx = { workspace = true } -tokio = { workspace = true } -tracing = { workspace = true } -tracing-subscriber = { workspace = true } +async-trait = { workspace = true } base64 = { workspace = true } -sha2 = { workspace = true } -lazy_static = { workspace = true } chrono = { workspace = true } -triggered = { workspace = true } +clap = { workspace = true } +config = { workspace = true } +coverage-map = { path = "../coverage_map" } +coverage-point-calculator = { path = "../coverage_point_calculator" } +custom-tracing = { path = "../custom_tracing" } +db-store = { path = "../db_store" } +derive_builder = { workspace = true } +file-store = { path = "../file_store" } flate2 = "1" futures = { workspace = true } futures-util = { workspace = true } -prost = { workspace = true } -once_cell = { workspace = true } -helium-proto = { workspace = true } +h3o = { workspace = true, features = ["geo"] } helium-crypto = { workspace = true, features = ["sqlx-postgres"] } +helium-proto = { workspace = true } +hex = "0.4" +hex-assignments = { path = "../hex_assignments" } +hextree = { workspace = true } +http-serde = { workspace = true } humantime = { workspace = true } -rust_decimal = { workspace = true } -rust_decimal_macros = { workspace = true } -tonic = { workspace = true } -tokio-stream = { workspace = true } -tokio-util = { workspace = true } +humantime-serde = { workspace = true } +lazy_static = { workspace = true } metrics = { workspace = true } metrics-exporter-prometheus = { workspace = true } mobile-config = { path = "../mobile_config" } -file-store = { path = "../file_store" } -db-store = { path = "../db_store" } +once_cell = { workspace = true } poc-metrics = { path = "../metrics" } -reward-scheduler = { path = "../reward_scheduler" } price = { path = "../price" } +prost = { workspace = true } rand = { workspace = true } -async-trait = { workspace = true } +regex = "1" retainer = { workspace = true } -uuid = { workspace = true } -task-manager = { path = "../task_manager" } +reward-scheduler = { path = "../reward_scheduler" } +rust_decimal = { workspace = true } +rust_decimal_macros = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +sha2 = { workspace = true } solana-sdk = { workspace = true } -derive_builder = { workspace = true } -regex = "1" -humantime-serde = { workspace = true } -custom-tracing = { path = "../custom_tracing" } -hex-assignments = { path = "../hex_assignments" } -coverage-point-calculator = { path = "../coverage_point_calculator" } -coverage-map = { path = "../coverage_map" } -hex = "0.4" +sqlx = { workspace = true } +task-manager = { path = "../task_manager" } +thiserror = { workspace = true } +tokio = { workspace = true } +tokio-stream = { workspace = true } +tokio-util = { workspace = true } +tonic = { workspace = true } +tracing = { workspace = true } +tracing-subscriber = { workspace = true } +triggered = { workspace = true } +uuid = { workspace = true } [dev-dependencies] backon = "0" From 619085727f0f024d6ad0c13fb16944f663133b50 Mon Sep 17 00:00:00 2001 From: Macpie Date: Wed, 2 Oct 2024 15:15:40 -0700 Subject: [PATCH 07/31] Add lat/long to estimates --- file_store/src/radio_location_estimates.rs | 6 ++++++ ingest/tests/mobile_ingest.rs | 2 ++ mobile_verifier/src/radio_location_estimates.rs | 4 ++-- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/file_store/src/radio_location_estimates.rs b/file_store/src/radio_location_estimates.rs index 42838ec9a..3b9eaf547 100644 --- a/file_store/src/radio_location_estimates.rs +++ b/file_store/src/radio_location_estimates.rs @@ -67,6 +67,8 @@ impl TryFrom for RadioLocationEstimatesReq { #[derive(Clone, Deserialize, Serialize, Debug, PartialEq)] pub struct RadioLocationEstimate { pub radius: Decimal, + pub lat: Decimal, + pub long: Decimal, pub confidence: Decimal, pub events: Vec, } @@ -75,6 +77,8 @@ impl From for RadioLocationEstimateV1 { fn from(rle: RadioLocationEstimate) -> Self { RadioLocationEstimateV1 { radius: Some(to_proto_decimal(rle.radius)), + lat: Some(to_proto_decimal(rle.lat)), + long: Some(to_proto_decimal(rle.long)), confidence: Some(to_proto_decimal(rle.confidence)), events: rle.events.into_iter().map(|e| e.into()).collect(), } @@ -86,6 +90,8 @@ impl TryFrom for RadioLocationEstimate { fn try_from(estimate: RadioLocationEstimateV1) -> Result { Ok(Self { radius: to_rust_decimal(estimate.radius.unwrap()), + lat: to_rust_decimal(estimate.lat.unwrap()), + long: to_rust_decimal(estimate.long.unwrap()), confidence: to_rust_decimal(estimate.confidence.unwrap()), events: estimate .events diff --git a/ingest/tests/mobile_ingest.rs b/ingest/tests/mobile_ingest.rs index c555ae375..f50e6acbd 100644 --- a/ingest/tests/mobile_ingest.rs +++ b/ingest/tests/mobile_ingest.rs @@ -44,6 +44,8 @@ async fn submit_radio_location_estimates() -> anyhow::Result<()> { let radio_id = "radio_id".to_string(); let estimates = vec![RadioLocationEstimateV1 { radius: to_proto_decimal(2.0), + lat: to_proto_decimal(41.41208), + long: to_proto_decimal(-122.19288), confidence: to_proto_decimal(0.75), events: vec![RleEventV1 { id: "event_1".to_string(), diff --git a/mobile_verifier/src/radio_location_estimates.rs b/mobile_verifier/src/radio_location_estimates.rs index 928a4ed66..f6594de84 100644 --- a/mobile_verifier/src/radio_location_estimates.rs +++ b/mobile_verifier/src/radio_location_estimates.rs @@ -214,8 +214,8 @@ async fn insert_estimate( exec: &mut Transaction<'_, Postgres>, ) -> Result<(), sqlx::Error> { let radius = estimate.radius; - let lat = 0.0; - let long = 0.0; + let lat = estimate.lat; + let long = estimate.long; let key = format!( "{}{}{}{}{}", From e6859f87da0cc174709198734cf33044c4d45cbf Mon Sep 17 00:00:00 2001 From: Macpie Date: Wed, 2 Oct 2024 16:09:10 -0700 Subject: [PATCH 08/31] Update proto to branch and not local --- Cargo.lock | 81 +++++++++++++++++++++++++++++++++++++++--------------- Cargo.toml | 9 +++--- 2 files changed, 63 insertions(+), 27 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d2341be5f..a80b07327 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1617,11 +1617,11 @@ checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" [[package]] name = "beacon" version = "0.1.0" -source = "git+https://github.com/helium/proto?branch=master#47598512b7d282c34b619adf140403feeb138e71" +source = "git+https://github.com/helium/proto?branch=macpie/radio_location_estimates#68faaac22bca564240affb3cacc0a9a2257d1cd1" dependencies = [ "base64 0.21.7", "byteorder", - "helium-proto", + "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=macpie/radio_location_estimates)", "prost", "rand 0.8.5", "rand_chacha 0.3.0", @@ -1791,7 +1791,7 @@ dependencies = [ "futures", "futures-util", "helium-crypto", - "helium-proto", + "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=macpie/radio_location_estimates)", "http 0.2.11", "http-serde", "humantime-serde", @@ -2642,7 +2642,7 @@ dependencies = [ "axum 0.7.4", "bs58 0.4.0", "helium-crypto", - "helium-proto", + "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=macpie/radio_location_estimates)", "http 0.2.11", "notify", "serde", @@ -3242,7 +3242,7 @@ dependencies = [ "futures-util", "h3o", "helium-crypto", - "helium-proto", + "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=macpie/radio_location_estimates)", "hex-literal", "http 0.2.11", "lazy_static", @@ -3257,8 +3257,8 @@ dependencies = [ "serde_json", "sha2 0.10.8", "sqlx", - "strum", - "strum_macros", + "strum 0.24.1", + "strum_macros 0.24.3", "task-manager", "tempfile", "thiserror", @@ -3855,7 +3855,7 @@ dependencies = [ "h3o", "helium-anchor-gen 0.1.0 (git+https://github.com/helium/helium-anchor-gen.git?branch=main)", "helium-crypto", - "helium-proto", + "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=master)", "hex", "hex-literal", "itertools", @@ -3882,6 +3882,7 @@ dependencies = [ [[package]] name = "helium-proto" version = "0.1.0" +source = "git+https://github.com/helium/proto?branch=macpie/radio_location_estimates#68faaac22bca564240affb3cacc0a9a2257d1cd1" dependencies = [ "bytes", "prost", @@ -3894,6 +3895,20 @@ dependencies = [ "tonic-build", ] +[[package]] +name = "helium-proto" +version = "0.1.0" +source = "git+https://github.com/helium/proto?branch=master#197ff9c6cde7dc0d8334d6b4e27c58779e6a7ce0" +dependencies = [ + "bytes", + "prost", + "prost-build", + "serde", + "serde_json", + "strum 0.26.3", + "strum_macros 0.26.4", +] + [[package]] name = "helium-sub-daos" version = "0.1.8" @@ -3944,7 +3959,7 @@ dependencies = [ "async-trait", "chrono", "derive_builder", - "helium-proto", + "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=macpie/radio_location_estimates)", "hextree", "rust_decimal", "rust_decimal_macros", @@ -4369,7 +4384,7 @@ dependencies = [ "futures", "futures-util", "helium-crypto", - "helium-proto", + "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=macpie/radio_location_estimates)", "http 0.2.11", "humantime-serde", "metrics", @@ -4440,7 +4455,7 @@ dependencies = [ "futures", "futures-util", "helium-crypto", - "helium-proto", + "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=macpie/radio_location_estimates)", "hextree", "http 0.2.11", "http-serde", @@ -4482,7 +4497,7 @@ dependencies = [ "futures", "futures-util", "helium-crypto", - "helium-proto", + "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=macpie/radio_location_estimates)", "http 0.2.11", "http-serde", "humantime-serde", @@ -4524,7 +4539,7 @@ dependencies = [ "futures-util", "h3o", "helium-crypto", - "helium-proto", + "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=macpie/radio_location_estimates)", "http-serde", "humantime-serde", "iot-config", @@ -5130,7 +5145,7 @@ dependencies = [ "futures", "futures-util", "helium-crypto", - "helium-proto", + "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=macpie/radio_location_estimates)", "hextree", "http 0.2.11", "http-serde", @@ -5170,7 +5185,7 @@ dependencies = [ "futures", "h3o", "helium-crypto", - "helium-proto", + "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=macpie/radio_location_estimates)", "mobile-config", "prost", "rand 0.8.5", @@ -5215,7 +5230,7 @@ dependencies = [ "futures", "futures-util", "helium-crypto", - "helium-proto", + "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=macpie/radio_location_estimates)", "http 0.2.11", "http-serde", "humantime-serde", @@ -5259,7 +5274,7 @@ dependencies = [ "futures-util", "h3o", "helium-crypto", - "helium-proto", + "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=macpie/radio_location_estimates)", "hex", "hex-assignments", "hextree", @@ -5944,7 +5959,7 @@ dependencies = [ "futures", "futures-util", "helium-crypto", - "helium-proto", + "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=macpie/radio_location_estimates)", "http 0.2.11", "hyper 0.14.28", "jsonrpsee", @@ -6027,7 +6042,7 @@ dependencies = [ "futures-util", "helium-anchor-gen 0.1.0 (git+https://github.com/helium/helium-anchor-gen.git)", "helium-lib", - "helium-proto", + "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=macpie/radio_location_estimates)", "humantime-serde", "metrics", "metrics-exporter-prometheus", @@ -6177,7 +6192,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "80b776a1b2dc779f5ee0641f8ade0125bc1298dd41a9a0c16d8bd57b42d222b1" dependencies = [ "bytes", - "heck 0.4.0", + "heck 0.5.0", "itertools", "log", "multimap", @@ -6678,7 +6693,7 @@ dependencies = [ "futures", "futures-util", "helium-crypto", - "helium-proto", + "helium-proto 0.1.0 (git+https://github.com/helium/proto?branch=macpie/radio_location_estimates)", "humantime-serde", "lazy_static", "metrics", @@ -8799,7 +8814,29 @@ version = "0.26.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" dependencies = [ - "strum_macros", + "strum_macros 0.24.3", +] + +[[package]] +name = "strum" +version = "0.26.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" +dependencies = [ + "strum_macros 0.26.4", +] + +[[package]] +name = "strum_macros" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" +dependencies = [ + "heck 0.5.0", + "proc-macro2", + "quote", + "rustversion", + "syn 2.0.58", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 08068ea68..933e461c0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -70,10 +70,10 @@ helium-lib = { git = "https://github.com/helium/helium-wallet-rs.git", branch = hextree = { git = "https://github.com/jaykickliter/HexTree", branch = "main", features = [ "disktree", ] } -helium-proto = { git = "https://github.com/helium/proto", branch = "master", features = [ +helium-proto = { git = "https://github.com/helium/proto", branch = "macpie/radio_location_estimates", features = [ "services", ] } -beacon = { git = "https://github.com/helium/proto", branch = "master" } +beacon = { git = "https://github.com/helium/proto", branch = "macpie/radio_location_estimates" } solana-client = "1.18" solana-sdk = "1.18" solana-program = "1.18" @@ -130,6 +130,5 @@ sqlx = { git = "https://github.com/helium/sqlx.git", rev = "92a2268f02e0cac6fccb # Patching for beacon must point directly to the crate, it will not look in the # repo for sibling crates. # -[patch.'https://github.com/helium/proto'] -helium-proto = { path = "../proto" } -# beacon = { path = "../proto/beacon" } +# [patch.'https://github.com/helium/proto'] +# helium-proto = { path = "../proto" } From a9ed267e7a1449ff8d38dafe51ba54918121cb9f Mon Sep 17 00:00:00 2001 From: Macpie Date: Thu, 10 Oct 2024 13:47:54 -0700 Subject: [PATCH 09/31] Change is_valid to invalided_at and invalidate old estimate when new one comes in --- .../37_radio_location_estimates.sql | 2 +- .../src/radio_location_estimates.rs | 35 ++++++++++++++----- 2 files changed, 27 insertions(+), 10 deletions(-) diff --git a/mobile_verifier/migrations/37_radio_location_estimates.sql b/mobile_verifier/migrations/37_radio_location_estimates.sql index 96b3e7dd3..04fef8a86 100644 --- a/mobile_verifier/migrations/37_radio_location_estimates.sql +++ b/mobile_verifier/migrations/37_radio_location_estimates.sql @@ -6,7 +6,7 @@ CREATE TABLE IF NOT EXISTS radio_location_estimates ( lat DECIMAL NOT NULL, long DECIMAL NOT NULL, confidence DECIMAL NOT NULL, - is_valid BOOLEAN NOT NULL, + invalided_at TIMESTAMPTZ DEFAULT NULL, inserted_at TIMESTAMPTZ DEFAULT now(), PRIMARY KEY (hashed_key) ); \ No newline at end of file diff --git a/mobile_verifier/src/radio_location_estimates.rs b/mobile_verifier/src/radio_location_estimates.rs index f6594de84..3658af940 100644 --- a/mobile_verifier/src/radio_location_estimates.rs +++ b/mobile_verifier/src/radio_location_estimates.rs @@ -194,15 +194,33 @@ async fn save_to_db( exec: &mut Transaction<'_, Postgres>, ) -> Result<(), sqlx::Error> { let estimates = &report.report.estimates; + let radio_id = &report.report.radio_id; + let received_timestamp = report.received_timestamp; for estimate in estimates { - insert_estimate( - report.report.radio_id.clone(), - report.received_timestamp, - estimate, - exec, - ) - .await?; + insert_estimate(radio_id.clone(), received_timestamp, estimate, exec).await?; } + invalidate_old_estimates(radio_id.clone(), received_timestamp, exec).await?; + + Ok(()) +} + +async fn invalidate_old_estimates( + radio_id: String, + timestamp: DateTime, + exec: &mut Transaction<'_, Postgres>, +) -> Result<(), sqlx::Error> { + sqlx::query( + r#" + UPDATE radio_location_estimates + SET invalided_at = now() + WHERE radio_id = $1 + AND received_timestamp < $2; + "#, + ) + .bind(radio_id) + .bind(timestamp) + .execute(exec) + .await?; Ok(()) } @@ -228,7 +246,7 @@ async fn insert_estimate( sqlx::query( r#" - INSERT INTO radio_location_estimates (hashed_key, radio_id, received_timestamp, radius, lat, long, confidence, is_valid) + INSERT INTO radio_location_estimates (hashed_key, radio_id, received_timestamp, radius, lat, long, confidence) VALUES ($1, $2, $3, $4, $5, $6, $7, $8) ON CONFLICT (hashed_key) DO NOTHING "#, @@ -240,7 +258,6 @@ async fn insert_estimate( .bind(lat) .bind(long) .bind(estimate.confidence) - .bind(true) .execute(exec) .await?; From 7ec39de0dce53ad5191fda423ee0f53a6fc7ab39 Mon Sep 17 00:00:00 2001 From: Macpie Date: Thu, 10 Oct 2024 14:07:25 -0700 Subject: [PATCH 10/31] Fix lock after rebase --- Cargo.lock | 44 ++++++++++++-------------------------------- 1 file changed, 12 insertions(+), 32 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a80b07327..725db7d14 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1617,7 +1617,7 @@ checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" [[package]] name = "beacon" version = "0.1.0" -source = "git+https://github.com/helium/proto?branch=macpie/radio_location_estimates#68faaac22bca564240affb3cacc0a9a2257d1cd1" +source = "git+https://github.com/helium/proto?branch=macpie/radio_location_estimates#571c0781bf2c06db375a3ff9f837cc94fd11053d" dependencies = [ "base64 0.21.7", "byteorder", @@ -1627,7 +1627,7 @@ dependencies = [ "rand_chacha 0.3.0", "rust_decimal", "serde", - "sha2 0.10.8", + "sha2 0.9.9", "thiserror", ] @@ -3257,8 +3257,8 @@ dependencies = [ "serde_json", "sha2 0.10.8", "sqlx", - "strum 0.24.1", - "strum_macros 0.24.3", + "strum", + "strum_macros", "task-manager", "tempfile", "thiserror", @@ -3882,7 +3882,7 @@ dependencies = [ [[package]] name = "helium-proto" version = "0.1.0" -source = "git+https://github.com/helium/proto?branch=macpie/radio_location_estimates#68faaac22bca564240affb3cacc0a9a2257d1cd1" +source = "git+https://github.com/helium/proto?branch=macpie/radio_location_estimates#571c0781bf2c06db375a3ff9f837cc94fd11053d" dependencies = [ "bytes", "prost", @@ -3905,8 +3905,10 @@ dependencies = [ "prost-build", "serde", "serde_json", - "strum 0.26.3", - "strum_macros 0.26.4", + "strum", + "strum_macros", + "tonic", + "tonic-build", ] [[package]] @@ -6192,7 +6194,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "80b776a1b2dc779f5ee0641f8ade0125bc1298dd41a9a0c16d8bd57b42d222b1" dependencies = [ "bytes", - "heck 0.5.0", + "heck 0.4.0", "itertools", "log", "multimap", @@ -8814,29 +8816,7 @@ version = "0.26.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" dependencies = [ - "strum_macros 0.24.3", -] - -[[package]] -name = "strum" -version = "0.26.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" -dependencies = [ - "strum_macros 0.26.4", -] - -[[package]] -name = "strum_macros" -version = "0.26.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" -dependencies = [ - "heck 0.5.0", - "proc-macro2", - "quote", - "rustversion", - "syn 2.0.58", + "strum_macros", ] [[package]] @@ -10117,7 +10097,7 @@ dependencies = [ "rand 0.8.5", "serde", "serde_json", - "sha2 0.10.8", + "sha2 0.9.9", "thiserror", "twox-hash", "xorf", From 85942f2122424ff5b6555565f2adcdcdf3a7cdd3 Mon Sep 17 00:00:00 2001 From: Macpie Date: Thu, 10 Oct 2024 14:11:20 -0700 Subject: [PATCH 11/31] More rebase fix --- ingest/src/server_mobile.rs | 9 +++------ ingest/tests/common/mod.rs | 3 +-- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/ingest/src/server_mobile.rs b/ingest/src/server_mobile.rs index 036946715..6081ac63d 100644 --- a/ingest/src/server_mobile.rs +++ b/ingest/src/server_mobile.rs @@ -14,13 +14,10 @@ use helium_proto::services::poc_mobile::{ CoverageObjectIngestReportV1, CoverageObjectReqV1, CoverageObjectRespV1, DataTransferSessionIngestReportV1, DataTransferSessionReqV1, DataTransferSessionRespV1, InvalidatedRadioThresholdIngestReportV1, InvalidatedRadioThresholdReportReqV1, - InvalidatedRadioThresholdReportRespV1, InvalidatedRadioThresholdReportRespV1, - PromotionRewardIngestReportV1, PromotionRewardReqV1, PromotionRewardRespV1, - RadioLocationEstimatesIngestReportV1, RadioLocationEstimatesReqV1, - RadioLocationEstimatesRespV1, RadioThresholdIngestReportV1, RadioThresholdIngestReportV1, - RadioThresholdReportReqV1, RadioThresholdReportReqV1, RadioThresholdReportRespV1, + InvalidatedRadioThresholdReportRespV1, PromotionRewardIngestReportV1, PromotionRewardReqV1, + PromotionRewardRespV1, RadioLocationEstimatesIngestReportV1, RadioLocationEstimatesReqV1, + RadioLocationEstimatesRespV1, RadioThresholdIngestReportV1, RadioThresholdReportReqV1, RadioThresholdReportRespV1, ServiceProviderBoostedRewardsBannedRadioIngestReportV1, - ServiceProviderBoostedRewardsBannedRadioIngestReportV1, ServiceProviderBoostedRewardsBannedRadioReqV1, ServiceProviderBoostedRewardsBannedRadioRespV1, SpeedtestIngestReportV1, SpeedtestReqV1, SpeedtestRespV1, SubscriberLocationIngestReportV1, SubscriberLocationReqV1, SubscriberLocationRespV1, diff --git a/ingest/tests/common/mod.rs b/ingest/tests/common/mod.rs index a25fe9fac..f36e0bb38 100644 --- a/ingest/tests/common/mod.rs +++ b/ingest/tests/common/mod.rs @@ -47,8 +47,7 @@ pub async fn setup_mobile() -> anyhow::Result<(TestClient, Trigger)> { let (sp_boosted_tx, _rx) = tokio::sync::mpsc::channel(10); let (subscriber_mapping_tx, subscriber_mapping_rx) = tokio::sync::mpsc::channel(10); let (promotion_rewards_tx, _rx) = tokio::sync::mpsc::channel(10); - let (radio_location_estimates_tx, _radio_location_estimates_rx) = - tokio::sync::mpsc::channel(10); + let (radio_location_estimates_tx, radio_location_estimates_rx) = tokio::sync::mpsc::channel(10); tokio::spawn(async move { let grpc_server = GrpcServer::new( From 1e42e4c711934f97dce3320bc55528cbf963540f Mon Sep 17 00:00:00 2001 From: Macpie Date: Thu, 10 Oct 2024 17:57:37 -0700 Subject: [PATCH 12/31] Add test for verifier --- ...es.sql => 38_radio_location_estimates.sql} | 0 .../src/radio_location_estimates.rs | 30 ++- mobile_verifier/tests/integrations/main.rs | 1 + .../integrations/radio_location_estimates.rs | 236 ++++++++++++++++++ 4 files changed, 256 insertions(+), 11 deletions(-) rename mobile_verifier/migrations/{37_radio_location_estimates.sql => 38_radio_location_estimates.sql} (100%) create mode 100644 mobile_verifier/tests/integrations/radio_location_estimates.rs diff --git a/mobile_verifier/migrations/37_radio_location_estimates.sql b/mobile_verifier/migrations/38_radio_location_estimates.sql similarity index 100% rename from mobile_verifier/migrations/37_radio_location_estimates.sql rename to mobile_verifier/migrations/38_radio_location_estimates.sql diff --git a/mobile_verifier/src/radio_location_estimates.rs b/mobile_verifier/src/radio_location_estimates.rs index 3658af940..1024deb5a 100644 --- a/mobile_verifier/src/radio_location_estimates.rs +++ b/mobile_verifier/src/radio_location_estimates.rs @@ -20,6 +20,7 @@ use helium_proto::services::{ }, }; use mobile_config::client::authorization_client::AuthorizationVerifier; +use rust_decimal::Decimal; use sha2::{Digest, Sha256}; use sqlx::{Pool, Postgres, Transaction}; use task_manager::{ManagedTask, TaskManager}; @@ -234,24 +235,16 @@ async fn insert_estimate( let radius = estimate.radius; let lat = estimate.lat; let long = estimate.long; - - let key = format!( - "{}{}{}{}{}", - radio_id, received_timestamp, radius, lat, long - ); - - let mut hasher = Sha256::new(); - hasher.update(key); - let hashed_key = hasher.finalize(); + let hashed_key = hash_key(radio_id.clone(), received_timestamp, radius, lat, long); sqlx::query( r#" INSERT INTO radio_location_estimates (hashed_key, radio_id, received_timestamp, radius, lat, long, confidence) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8) + VALUES ($1, $2, $3, $4, $5, $6, $7) ON CONFLICT (hashed_key) DO NOTHING "#, ) - .bind(hex::encode(hashed_key)) + .bind(hashed_key) .bind(radio_id) .bind(received_timestamp) .bind(estimate.radius) @@ -263,3 +256,18 @@ async fn insert_estimate( Ok(()) } + +pub fn hash_key( + radio_id: String, + timestamp: DateTime, + radius: Decimal, + lat: Decimal, + long: Decimal, +) -> String { + let key = format!("{}{}{}{}{}", radio_id, timestamp, radius, lat, long); + + let mut hasher = Sha256::new(); + hasher.update(key); + let hashed_key = hasher.finalize(); + hex::encode(hashed_key) +} diff --git a/mobile_verifier/tests/integrations/main.rs b/mobile_verifier/tests/integrations/main.rs index 0980b0065..b346016f6 100644 --- a/mobile_verifier/tests/integrations/main.rs +++ b/mobile_verifier/tests/integrations/main.rs @@ -5,6 +5,7 @@ mod heartbeats; mod hex_boosting; mod last_location; mod modeled_coverage; +mod radio_location_estimates; mod rewarder_mappers; mod rewarder_oracles; mod rewarder_poc_dc; diff --git a/mobile_verifier/tests/integrations/radio_location_estimates.rs b/mobile_verifier/tests/integrations/radio_location_estimates.rs new file mode 100644 index 000000000..2478427ee --- /dev/null +++ b/mobile_verifier/tests/integrations/radio_location_estimates.rs @@ -0,0 +1,236 @@ +use crate::common::MockAuthorizationClient; +use chrono::{DateTime, Duration, Utc}; +use file_store::{ + file_info_poller::FileInfoStream, + file_sink::FileSinkClient, + radio_location_estimates::{ + RadioLocationEstimate, RadioLocationEstimateEvent, RadioLocationEstimatesReq, + }, + radio_location_estimates_ingest_report::RadioLocationEstimatesIngestReport, + FileInfo, +}; +use helium_crypto::{KeyTag, Keypair, PublicKeyBinary}; +use mobile_verifier::radio_location_estimates::{hash_key, RadioLocationEstimatesDaemon}; +use rand::rngs::OsRng; +use rust_decimal::prelude::FromPrimitive; +use sqlx::{PgPool, Pool, Postgres, Row}; + +#[sqlx::test] +async fn main_test(pool: PgPool) -> anyhow::Result<()> { + let task_pool = pool.clone(); + let (reports_tx, reports_rx) = tokio::sync::mpsc::channel(10); + let (sink_tx, _sink_rx) = tokio::sync::mpsc::channel(10); + let (trigger, listener) = triggered::trigger(); + + tokio::spawn(async move { + let deamon = RadioLocationEstimatesDaemon::new( + task_pool, + MockAuthorizationClient::new(), + reports_rx, + FileSinkClient::new(sink_tx, "metric"), + ); + + deamon.run(listener).await.expect("failed to complete task"); + }); + + // Sending reports as if they are coming from ingestor + let (fis, reports, _public_key_binary) = file_info_stream(); + reports_tx.send(fis).await?; + + let mut retry = 0; + const MAX_RETRIES: u32 = 3; + const RETRY_WAIT: std::time::Duration = std::time::Duration::from_secs(1); + + let mut expected_n = 0; + for report in &reports { + expected_n += report.report.estimates.len(); + } + + while retry <= MAX_RETRIES { + let saved_estimates = select_radio_location_estimates(&pool).await?; + + if expected_n == saved_estimates.len() { + let expected1 = &reports[0]; + let invalid_estimate = &saved_estimates[0]; + assert_eq!( + hash_key( + expected1.report.radio_id.clone(), + invalid_estimate.received_timestamp, + expected1.report.estimates[0].radius, + expected1.report.estimates[0].lat, + expected1.report.estimates[0].long + ), + invalid_estimate.hashed_key + ); + + assert_eq!(expected1.report.radio_id, invalid_estimate.radio_id); + assert!(timestamp_match( + expected1.received_timestamp, + invalid_estimate.received_timestamp + )); + assert_eq!( + expected1.report.estimates[0].radius, + invalid_estimate.radius + ); + assert_eq!(expected1.report.estimates[0].lat, invalid_estimate.lat); + assert_eq!(expected1.report.estimates[0].long, invalid_estimate.long); + assert_eq!( + expected1.report.estimates[0].confidence, + invalid_estimate.confidence + ); + assert!(invalid_estimate.invalided_at.is_some()); + + let expected2 = &reports[1]; + let valid_estimate = &saved_estimates[1]; + assert_eq!( + hash_key( + expected2.report.radio_id.clone(), + valid_estimate.received_timestamp, + expected2.report.estimates[0].radius, + expected2.report.estimates[0].lat, + expected2.report.estimates[0].long + ), + valid_estimate.hashed_key + ); + assert_eq!(expected2.report.radio_id, valid_estimate.radio_id); + assert!(timestamp_match( + expected2.received_timestamp, + valid_estimate.received_timestamp + )); + assert_eq!(expected2.report.estimates[0].radius, valid_estimate.radius); + assert_eq!(expected2.report.estimates[0].lat, valid_estimate.lat); + assert_eq!(expected2.report.estimates[0].long, valid_estimate.long); + assert_eq!( + expected2.report.estimates[0].confidence, + valid_estimate.confidence + ); + assert_eq!(None, valid_estimate.invalided_at); + + break; + } else { + retry += 1; + tokio::time::sleep(RETRY_WAIT).await; + } + } + + assert!( + retry <= MAX_RETRIES, + "Exceeded maximum retries: {}", + MAX_RETRIES + ); + + trigger.trigger(); + + Ok(()) +} + +fn file_info_stream() -> ( + FileInfoStream, + Vec, + PublicKeyBinary, +) { + let file_info = FileInfo { + key: "test_file_info".to_string(), + prefix: "verified_mapping_event".to_string(), + timestamp: Utc::now(), + size: 0, + }; + + let key_pair = generate_keypair(); + let public_key_binary: PublicKeyBinary = key_pair.public_key().to_owned().into(); + + let reports = vec![ + RadioLocationEstimatesIngestReport { + received_timestamp: Utc::now() - Duration::hours(1), + report: RadioLocationEstimatesReq { + radio_id: "radio_1".to_string(), + estimates: vec![RadioLocationEstimate { + radius: rust_decimal::Decimal::from_f32(0.1).unwrap(), + lat: rust_decimal::Decimal::from_f32(0.1).unwrap(), + long: rust_decimal::Decimal::from_f32(-0.1).unwrap(), + confidence: rust_decimal::Decimal::from_f32(0.1).unwrap(), + events: vec![RadioLocationEstimateEvent { + id: "event_1".to_string(), + timestamp: Utc::now() - Duration::hours(1), + }], + }], + timestamp: Utc::now() - Duration::hours(1), + carrier_key: public_key_binary.clone(), + }, + }, + RadioLocationEstimatesIngestReport { + received_timestamp: Utc::now(), + report: RadioLocationEstimatesReq { + radio_id: "radio_1".to_string(), + estimates: vec![RadioLocationEstimate { + radius: rust_decimal::Decimal::from_f32(0.2).unwrap(), + lat: rust_decimal::Decimal::from_f32(0.2).unwrap(), + long: rust_decimal::Decimal::from_f32(-0.2).unwrap(), + confidence: rust_decimal::Decimal::from_f32(0.2).unwrap(), + events: vec![RadioLocationEstimateEvent { + id: "event_1".to_string(), + timestamp: Utc::now(), + }], + }], + timestamp: Utc::now(), + carrier_key: public_key_binary.clone(), + }, + }, + ]; + ( + FileInfoStream::new("default".to_string(), file_info, reports.clone()), + reports, + public_key_binary, + ) +} + +fn generate_keypair() -> Keypair { + Keypair::generate(KeyTag::default(), &mut OsRng) +} + +fn timestamp_match(dt1: DateTime, dt2: DateTime) -> bool { + let difference = dt1.signed_duration_since(dt2); + difference.num_seconds().abs() < 1 +} + +#[derive(Debug)] +pub struct RadioLocationEstimateDB { + pub hashed_key: String, + pub radio_id: String, + pub received_timestamp: DateTime, + pub radius: rust_decimal::Decimal, + pub lat: rust_decimal::Decimal, + pub long: rust_decimal::Decimal, + pub confidence: rust_decimal::Decimal, + pub invalided_at: Option>, +} + +pub async fn select_radio_location_estimates( + pool: &Pool, +) -> anyhow::Result> { + let rows = sqlx::query( + r#" + SELECT hashed_key, radio_id, received_timestamp, radius, lat, long, confidence, invalided_at + FROM radio_location_estimates + ORDER BY received_timestamp ASC + "#, + ) + .fetch_all(pool) + .await?; + + let estimates = rows + .into_iter() + .map(|row| RadioLocationEstimateDB { + hashed_key: row.get("hashed_key"), + radio_id: row.get("radio_id"), + received_timestamp: row.get("received_timestamp"), + radius: row.get("radius"), + lat: row.get("lat"), + long: row.get("long"), + confidence: row.get("confidence"), + invalided_at: row.try_get("invalided_at").ok(), + }) + .collect(); + + Ok(estimates) +} From f8d2bd35fa12d92ad92692fb52ea2dc52e02fa83 Mon Sep 17 00:00:00 2001 From: Macpie Date: Thu, 10 Oct 2024 22:28:54 -0700 Subject: [PATCH 13/31] Fix hash key timestamp --- .../tests/integrations/radio_location_estimates.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mobile_verifier/tests/integrations/radio_location_estimates.rs b/mobile_verifier/tests/integrations/radio_location_estimates.rs index 2478427ee..df6302577 100644 --- a/mobile_verifier/tests/integrations/radio_location_estimates.rs +++ b/mobile_verifier/tests/integrations/radio_location_estimates.rs @@ -55,7 +55,7 @@ async fn main_test(pool: PgPool) -> anyhow::Result<()> { assert_eq!( hash_key( expected1.report.radio_id.clone(), - invalid_estimate.received_timestamp, + expected1.received_timestamp, expected1.report.estimates[0].radius, expected1.report.estimates[0].lat, expected1.report.estimates[0].long @@ -85,7 +85,7 @@ async fn main_test(pool: PgPool) -> anyhow::Result<()> { assert_eq!( hash_key( expected2.report.radio_id.clone(), - valid_estimate.received_timestamp, + expected2.received_timestamp, expected2.report.estimates[0].radius, expected2.report.estimates[0].lat, expected2.report.estimates[0].long From 6cf97795ed2517f6a97c70f7d14b185de319bc59 Mon Sep 17 00:00:00 2001 From: Macpie Date: Fri, 11 Oct 2024 09:55:50 -0700 Subject: [PATCH 14/31] Add clear_invalided and improve test --- .../src/radio_location_estimates.rs | 17 +++ mobile_verifier/src/rewarder.rs | 3 +- .../integrations/radio_location_estimates.rs | 110 +++++++++--------- 3 files changed, 71 insertions(+), 59 deletions(-) diff --git a/mobile_verifier/src/radio_location_estimates.rs b/mobile_verifier/src/radio_location_estimates.rs index 1024deb5a..96b873dec 100644 --- a/mobile_verifier/src/radio_location_estimates.rs +++ b/mobile_verifier/src/radio_location_estimates.rs @@ -271,3 +271,20 @@ pub fn hash_key( let hashed_key = hasher.finalize(); hex::encode(hashed_key) } + +pub async fn clear_invalided( + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + timestamp: &DateTime, +) -> Result<(), sqlx::Error> { + sqlx::query( + r#" + DELETE FROM radio_location_estimates + WHERE invalided_at IS NOT NULL + AND invalided_at < $1 + "#, + ) + .bind(timestamp) + .execute(&mut *tx) + .await?; + Ok(()) +} diff --git a/mobile_verifier/src/rewarder.rs b/mobile_verifier/src/rewarder.rs index fb6fa2b3d..580777859 100644 --- a/mobile_verifier/src/rewarder.rs +++ b/mobile_verifier/src/rewarder.rs @@ -2,7 +2,7 @@ use crate::{ boosting_oracles::db::check_for_unprocessed_data_sets, coverage, data_session, heartbeats::{self, HeartbeatReward}, - radio_threshold, + radio_location_estimates, radio_threshold, reward_shares::{ self, CalculatedPocRewardShares, CoverageShares, DataTransferAndPocAllocatedRewardBuckets, MapperShares, TransferRewards, @@ -298,6 +298,7 @@ where subscriber_verified_mapping_event::clear(&mut transaction, &reward_period.start).await?; service_provider::db::clear_promotion_rewards(&mut transaction, &reward_period.start) .await?; + radio_location_estimates::clear_invalided(&mut transaction, &reward_period.start).await?; // subscriber_location::clear_location_shares(&mut transaction, &reward_period.end).await?; let next_reward_period = scheduler.next_reward_period(); diff --git a/mobile_verifier/tests/integrations/radio_location_estimates.rs b/mobile_verifier/tests/integrations/radio_location_estimates.rs index df6302577..f8465c2a6 100644 --- a/mobile_verifier/tests/integrations/radio_location_estimates.rs +++ b/mobile_verifier/tests/integrations/radio_location_estimates.rs @@ -10,13 +10,15 @@ use file_store::{ FileInfo, }; use helium_crypto::{KeyTag, Keypair, PublicKeyBinary}; -use mobile_verifier::radio_location_estimates::{hash_key, RadioLocationEstimatesDaemon}; +use mobile_verifier::radio_location_estimates::{ + clear_invalided, hash_key, RadioLocationEstimatesDaemon, +}; use rand::rngs::OsRng; use rust_decimal::prelude::FromPrimitive; use sqlx::{PgPool, Pool, Postgres, Row}; #[sqlx::test] -async fn main_test(pool: PgPool) -> anyhow::Result<()> { +async fn verifier_test(pool: PgPool) -> anyhow::Result<()> { let task_pool = pool.clone(); let (reports_tx, reports_rx) = tokio::sync::mpsc::channel(10); let (sink_tx, _sink_rx) = tokio::sync::mpsc::channel(10); @@ -49,63 +51,12 @@ async fn main_test(pool: PgPool) -> anyhow::Result<()> { while retry <= MAX_RETRIES { let saved_estimates = select_radio_location_estimates(&pool).await?; + // Check that we have expected (2) number of estimates saved in DB + // 1 should be invalidated and other should be valid + // We know the order (invalid first becase we order by in select_radio_location_estimates) if expected_n == saved_estimates.len() { - let expected1 = &reports[0]; - let invalid_estimate = &saved_estimates[0]; - assert_eq!( - hash_key( - expected1.report.radio_id.clone(), - expected1.received_timestamp, - expected1.report.estimates[0].radius, - expected1.report.estimates[0].lat, - expected1.report.estimates[0].long - ), - invalid_estimate.hashed_key - ); - - assert_eq!(expected1.report.radio_id, invalid_estimate.radio_id); - assert!(timestamp_match( - expected1.received_timestamp, - invalid_estimate.received_timestamp - )); - assert_eq!( - expected1.report.estimates[0].radius, - invalid_estimate.radius - ); - assert_eq!(expected1.report.estimates[0].lat, invalid_estimate.lat); - assert_eq!(expected1.report.estimates[0].long, invalid_estimate.long); - assert_eq!( - expected1.report.estimates[0].confidence, - invalid_estimate.confidence - ); - assert!(invalid_estimate.invalided_at.is_some()); - - let expected2 = &reports[1]; - let valid_estimate = &saved_estimates[1]; - assert_eq!( - hash_key( - expected2.report.radio_id.clone(), - expected2.received_timestamp, - expected2.report.estimates[0].radius, - expected2.report.estimates[0].lat, - expected2.report.estimates[0].long - ), - valid_estimate.hashed_key - ); - assert_eq!(expected2.report.radio_id, valid_estimate.radio_id); - assert!(timestamp_match( - expected2.received_timestamp, - valid_estimate.received_timestamp - )); - assert_eq!(expected2.report.estimates[0].radius, valid_estimate.radius); - assert_eq!(expected2.report.estimates[0].lat, valid_estimate.lat); - assert_eq!(expected2.report.estimates[0].long, valid_estimate.long); - assert_eq!( - expected2.report.estimates[0].confidence, - valid_estimate.confidence - ); - assert_eq!(None, valid_estimate.invalided_at); - + compare_report_and_estimate(&reports[0], &saved_estimates[0], false); + compare_report_and_estimate(&reports[1], &saved_estimates[1], true); break; } else { retry += 1; @@ -119,6 +70,16 @@ async fn main_test(pool: PgPool) -> anyhow::Result<()> { MAX_RETRIES ); + // Now clear invalidated estimates there should be only 1 left in DB + let mut tx = pool.begin().await?; + clear_invalided(&mut tx, &Utc::now()).await?; + tx.commit().await?; + + let leftover_estimates = select_radio_location_estimates(&pool).await?; + assert_eq!(1, leftover_estimates.len()); + // Check that we have the right estimate left over + compare_report_and_estimate(&reports[1], &leftover_estimates[0], true); + trigger.trigger(); Ok(()) @@ -193,6 +154,39 @@ fn timestamp_match(dt1: DateTime, dt2: DateTime) -> bool { difference.num_seconds().abs() < 1 } +fn compare_report_and_estimate( + report: &RadioLocationEstimatesIngestReport, + estimate: &RadioLocationEstimateDB, + should_be_valid: bool, +) { + assert_eq!( + hash_key( + report.report.radio_id.clone(), + report.received_timestamp, + report.report.estimates[0].radius, + report.report.estimates[0].lat, + report.report.estimates[0].long + ), + estimate.hashed_key + ); + + assert_eq!(report.report.radio_id, estimate.radio_id); + assert!(timestamp_match( + report.received_timestamp, + estimate.received_timestamp + )); + assert_eq!(report.report.estimates[0].radius, estimate.radius); + assert_eq!(report.report.estimates[0].lat, estimate.lat); + assert_eq!(report.report.estimates[0].long, estimate.long); + assert_eq!(report.report.estimates[0].confidence, estimate.confidence); + + if should_be_valid { + assert!(estimate.invalided_at.is_none()); + } else { + assert!(estimate.invalided_at.is_some()); + } +} + #[derive(Debug)] pub struct RadioLocationEstimateDB { pub hashed_key: String, From 1e47db4a5fc8f682e2b26aade6e616b0a02f2932 Mon Sep 17 00:00:00 2001 From: Macpie Date: Fri, 11 Oct 2024 10:15:20 -0700 Subject: [PATCH 15/31] Maybe ban --- .../src/radio_location_estimates.rs | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/mobile_verifier/src/radio_location_estimates.rs b/mobile_verifier/src/radio_location_estimates.rs index 96b873dec..cb253c381 100644 --- a/mobile_verifier/src/radio_location_estimates.rs +++ b/mobile_verifier/src/radio_location_estimates.rs @@ -21,11 +21,14 @@ use helium_proto::services::{ }; use mobile_config::client::authorization_client::AuthorizationVerifier; use rust_decimal::Decimal; +use rust_decimal_macros::dec; use sha2::{Digest, Sha256}; use sqlx::{Pool, Postgres, Transaction}; use task_manager::{ManagedTask, TaskManager}; use tokio::sync::mpsc::Receiver; +const CONFIDENCE_THRESHOLD: Decimal = dec!(0.75); + pub struct RadioLocationEstimatesDaemon { pool: Pool, authorization_verifier: AV, @@ -130,6 +133,9 @@ where if verified_report_status == RadioLocationEstimatesVerificationStatus::Valid { save_to_db(&report, &mut transaction).await?; + + // Once they are saved to DB should we directly write to ban table? + // maybe_ban_radios(&report, &mut transaction).await?; } let verified_report_proto: VerifiedRadioLocationEstimatesReportV1 = @@ -205,6 +211,26 @@ async fn save_to_db( Ok(()) } +// TODO: knowing that radio could also be a PublicKey for wifi hotspot. +// Should we make radio_id in report a binary or some kind of proto enum to have one or other? + +// async fn maybe_ban_radios( +// report: &RadioLocationEstimatesIngestReport, +// exec: &mut Transaction<'_, Postgres>, +// ) -> Result<(), sqlx::Error> { +// let estimates = &report.report.estimates; +// let radio_id = &report.report.radio_id; +// let mut will_ban = true; + +// for estimate in estimates { +// if estimate.confidence > CONFIDENCE_THRESHOLD { +// will_ban = false; +// } +// } + +// Ok(()) +// } + async fn invalidate_old_estimates( radio_id: String, timestamp: DateTime, From 72526b949369481469eb59f3698783030e212610 Mon Sep 17 00:00:00 2001 From: Macpie Date: Fri, 11 Oct 2024 12:31:45 -0700 Subject: [PATCH 16/31] Comment unused code for now --- mobile_verifier/src/radio_location_estimates.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/mobile_verifier/src/radio_location_estimates.rs b/mobile_verifier/src/radio_location_estimates.rs index cb253c381..998263070 100644 --- a/mobile_verifier/src/radio_location_estimates.rs +++ b/mobile_verifier/src/radio_location_estimates.rs @@ -21,13 +21,12 @@ use helium_proto::services::{ }; use mobile_config::client::authorization_client::AuthorizationVerifier; use rust_decimal::Decimal; -use rust_decimal_macros::dec; use sha2::{Digest, Sha256}; use sqlx::{Pool, Postgres, Transaction}; use task_manager::{ManagedTask, TaskManager}; use tokio::sync::mpsc::Receiver; -const CONFIDENCE_THRESHOLD: Decimal = dec!(0.75); +// const CONFIDENCE_THRESHOLD: Decimal = dec!(0.75); pub struct RadioLocationEstimatesDaemon { pool: Pool, From e3f4c9f6c2cc60dc0dceba4989c0855edda0abd9 Mon Sep 17 00:00:00 2001 From: Macpie Date: Mon, 14 Oct 2024 12:44:37 -0700 Subject: [PATCH 17/31] Update proto and add Entity --- Cargo.lock | 4 +- file_store/src/radio_location_estimates.rs | 50 +++++++++++++++++-- ingest/tests/common/mod.rs | 16 +++--- ingest/tests/mobile_ingest.rs | 25 ++++++++-- .../38_radio_location_estimates.sql | 3 +- .../src/radio_location_estimates.rs | 38 ++++++++------ .../integrations/radio_location_estimates.rs | 35 +++++++------ 7 files changed, 123 insertions(+), 48 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 725db7d14..be41306c0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1617,7 +1617,7 @@ checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" [[package]] name = "beacon" version = "0.1.0" -source = "git+https://github.com/helium/proto?branch=macpie/radio_location_estimates#571c0781bf2c06db375a3ff9f837cc94fd11053d" +source = "git+https://github.com/helium/proto?branch=macpie/radio_location_estimates#4cefc5d337a4dddebd3fa3896b3c29f12e86f1ef" dependencies = [ "base64 0.21.7", "byteorder", @@ -3882,7 +3882,7 @@ dependencies = [ [[package]] name = "helium-proto" version = "0.1.0" -source = "git+https://github.com/helium/proto?branch=macpie/radio_location_estimates#571c0781bf2c06db375a3ff9f837cc94fd11053d" +source = "git+https://github.com/helium/proto?branch=macpie/radio_location_estimates#4cefc5d337a4dddebd3fa3896b3c29f12e86f1ef" dependencies = [ "bytes", "prost", diff --git a/file_store/src/radio_location_estimates.rs b/file_store/src/radio_location_estimates.rs index 3b9eaf547..25fc6d518 100644 --- a/file_store/src/radio_location_estimates.rs +++ b/file_store/src/radio_location_estimates.rs @@ -5,14 +5,52 @@ use crate::{ use chrono::{DateTime, Utc}; use helium_crypto::PublicKeyBinary; use helium_proto::services::poc_mobile::{ - RadioLocationEstimateV1, RadioLocationEstimatesReqV1, RleEventV1, + self as proto, RadioLocationEstimateV1, RadioLocationEstimatesReqV1, RleEventV1, }; use rust_decimal::Decimal; use serde::{Deserialize, Serialize}; +use std::fmt; + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub enum Entity { + CbrsId(String), + WifiPubKey(PublicKeyBinary), +} + +impl fmt::Display for Entity { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Entity::CbrsId(id) => write!(f, "{}", id), + Entity::WifiPubKey(pub_key) => write!(f, "{}", pub_key), + } + } +} + +impl From for Entity { + fn from(entity: proto::radio_location_estimates_req_v1::Entity) -> Self { + match entity { + proto::radio_location_estimates_req_v1::Entity::CbrsId(v) => Entity::CbrsId(v), + proto::radio_location_estimates_req_v1::Entity::WifiPubKey(k) => { + Entity::WifiPubKey(k.into()) + } + } + } +} + +impl From for proto::radio_location_estimates_req_v1::Entity { + fn from(entity: Entity) -> Self { + match entity { + Entity::CbrsId(v) => proto::radio_location_estimates_req_v1::Entity::CbrsId(v), + Entity::WifiPubKey(k) => { + proto::radio_location_estimates_req_v1::Entity::WifiPubKey(k.into()) + } + } + } +} #[derive(Clone, Deserialize, Serialize, Debug, PartialEq)] pub struct RadioLocationEstimatesReq { - pub radio_id: String, + pub entity: Entity, pub estimates: Vec, pub timestamp: DateTime, pub carrier_key: PublicKeyBinary, @@ -38,7 +76,7 @@ impl From for RadioLocationEstimatesReqV1 { fn from(rle: RadioLocationEstimatesReq) -> Self { let timestamp = rle.timestamp(); RadioLocationEstimatesReqV1 { - radio_id: rle.radio_id, + entity: Some(rle.entity.into()), estimates: rle.estimates.into_iter().map(|e| e.into()).collect(), timestamp, carrier_key: rle.carrier_key.into(), @@ -52,7 +90,11 @@ impl TryFrom for RadioLocationEstimatesReq { fn try_from(req: RadioLocationEstimatesReqV1) -> Result { let timestamp = req.timestamp()?; Ok(Self { - radio_id: req.radio_id, + entity: if let Some(entity) = req.entity { + entity.into() + } else { + return Err(Error::NotFound("entity".to_string())); + }, estimates: req .estimates .into_iter() diff --git a/ingest/tests/common/mod.rs b/ingest/tests/common/mod.rs index f36e0bb38..bd5e0b51b 100644 --- a/ingest/tests/common/mod.rs +++ b/ingest/tests/common/mod.rs @@ -1,12 +1,12 @@ use anyhow::bail; use backon::{ExponentialBuilder, Retryable}; use file_store::file_sink::FileSinkClient; -use helium_crypto::{KeyTag, Keypair, Network, Sign}; +use helium_crypto::{KeyTag, Keypair, Network, PublicKey, Sign}; use helium_proto::services::poc_mobile::{ - Client as PocMobileClient, RadioLocationEstimateV1, RadioLocationEstimatesIngestReportV1, - RadioLocationEstimatesReqV1, RadioLocationEstimatesRespV1, - SubscriberVerifiedMappingEventIngestReportV1, SubscriberVerifiedMappingEventReqV1, - SubscriberVerifiedMappingEventResV1, + radio_location_estimates_req_v1, Client as PocMobileClient, RadioLocationEstimateV1, + RadioLocationEstimatesIngestReportV1, RadioLocationEstimatesReqV1, + RadioLocationEstimatesRespV1, SubscriberVerifiedMappingEventIngestReportV1, + SubscriberVerifiedMappingEventReqV1, SubscriberVerifiedMappingEventResV1, }; use ingest::server_mobile::GrpcServer; use prost::Message; @@ -182,11 +182,13 @@ impl TestClient { pub async fn submit_radio_location_estimates( &mut self, - radio_id: String, + pub_key: &PublicKey, estimates: Vec, ) -> anyhow::Result { let mut req = RadioLocationEstimatesReqV1 { - radio_id, + entity: Some(radio_location_estimates_req_v1::Entity::WifiPubKey( + pub_key.into(), + )), estimates, timestamp: 0, carrier_key: self.key_pair.public_key().to_vec(), diff --git a/ingest/tests/mobile_ingest.rs b/ingest/tests/mobile_ingest.rs index f50e6acbd..f0597b75e 100644 --- a/ingest/tests/mobile_ingest.rs +++ b/ingest/tests/mobile_ingest.rs @@ -1,4 +1,9 @@ -use helium_proto::services::poc_mobile::{RadioLocationEstimateV1, RleEventV1}; +use helium_crypto::{KeyTag, Keypair, PublicKey}; +use helium_proto::services::poc_mobile::{ + radio_location_estimates_req_v1::Entity, RadioLocationEstimateV1, RadioLocationEstimatesReqV1, + RleEventV1, +}; +use rand::rngs::OsRng; use rust_decimal::prelude::*; mod common; @@ -41,7 +46,8 @@ async fn submit_verified_subscriber_mapping_event() -> anyhow::Result<()> { async fn submit_radio_location_estimates() -> anyhow::Result<()> { let (mut client, trigger) = common::setup_mobile().await?; - let radio_id = "radio_id".to_string(); + let key_pair = Keypair::generate(KeyTag::default(), &mut OsRng); + let public_key = key_pair.public_key(); let estimates = vec![RadioLocationEstimateV1 { radius: to_proto_decimal(2.0), lat: to_proto_decimal(41.41208), @@ -54,7 +60,7 @@ async fn submit_radio_location_estimates() -> anyhow::Result<()> { }]; let res = client - .submit_radio_location_estimates(radio_id.clone(), estimates.clone()) + .submit_radio_location_estimates(public_key, estimates.clone()) .await; assert!(res.is_ok()); @@ -68,7 +74,8 @@ async fn submit_radio_location_estimates() -> anyhow::Result<()> { match report.report { None => panic!("No report found"), Some(req) => { - assert_eq!(radio_id, req.radio_id); + let req_public_key = wifi_public_key(req.clone())?; + assert_eq!(public_key.to_string(), req_public_key.to_string()); assert_eq!(estimates, req.estimates); } } @@ -86,3 +93,13 @@ fn to_proto_decimal(x: f64) -> Option { value: d.to_string(), }) } + +fn wifi_public_key(req: RadioLocationEstimatesReqV1) -> anyhow::Result { + let entity: Entity = req.entity.unwrap(); + let Entity::WifiPubKey(public_key_bytes) = entity.clone() else { + anyhow::bail!("not WifiPubKey") + }; + let public_key = PublicKey::from_bytes(&public_key_bytes)?; + + Ok(public_key) +} diff --git a/mobile_verifier/migrations/38_radio_location_estimates.sql b/mobile_verifier/migrations/38_radio_location_estimates.sql index 04fef8a86..e9ddbd176 100644 --- a/mobile_verifier/migrations/38_radio_location_estimates.sql +++ b/mobile_verifier/migrations/38_radio_location_estimates.sql @@ -1,6 +1,7 @@ CREATE TABLE IF NOT EXISTS radio_location_estimates ( hashed_key TEXT NOT NULL, - radio_id TEXT NOT NULL, + radio_type radio_type NOT NULL, + radio_key TEXT NOT NULL, received_timestamp TIMESTAMPTZ NOT NULL, radius DECIMAL NOT NULL, lat DECIMAL NOT NULL, diff --git a/mobile_verifier/src/radio_location_estimates.rs b/mobile_verifier/src/radio_location_estimates.rs index 998263070..b72b39dcc 100644 --- a/mobile_verifier/src/radio_location_estimates.rs +++ b/mobile_verifier/src/radio_location_estimates.rs @@ -1,11 +1,11 @@ -use crate::Settings; +use crate::{heartbeats::HbType, Settings}; use chrono::{DateTime, Utc}; use file_store::{ file_info_poller::{FileInfoStream, LookbackBehavior}, file_sink::FileSinkClient, file_source, file_upload::FileUpload, - radio_location_estimates::{RadioLocationEstimate, RadioLocationEstimatesReq}, + radio_location_estimates::{Entity, RadioLocationEstimate, RadioLocationEstimatesReq}, radio_location_estimates_ingest_report::RadioLocationEstimatesIngestReport, traits::{FileSinkCommitStrategy, FileSinkRollTime, FileSinkWriteExt}, verified_radio_location_estimates::VerifiedRadioLocationEstimatesReport, @@ -200,12 +200,12 @@ async fn save_to_db( exec: &mut Transaction<'_, Postgres>, ) -> Result<(), sqlx::Error> { let estimates = &report.report.estimates; - let radio_id = &report.report.radio_id; + let entity = &report.report.entity; let received_timestamp = report.received_timestamp; for estimate in estimates { - insert_estimate(radio_id.clone(), received_timestamp, estimate, exec).await?; + insert_estimate(entity, received_timestamp, estimate, exec).await?; } - invalidate_old_estimates(radio_id.clone(), received_timestamp, exec).await?; + invalidate_old_estimates(entity, received_timestamp, exec).await?; Ok(()) } @@ -231,7 +231,7 @@ async fn save_to_db( // } async fn invalidate_old_estimates( - radio_id: String, + entity: &Entity, timestamp: DateTime, exec: &mut Transaction<'_, Postgres>, ) -> Result<(), sqlx::Error> { @@ -239,11 +239,11 @@ async fn invalidate_old_estimates( r#" UPDATE radio_location_estimates SET invalided_at = now() - WHERE radio_id = $1 + WHERE radio_key = $1 AND received_timestamp < $2; "#, ) - .bind(radio_id) + .bind(entity.to_string()) .bind(timestamp) .execute(exec) .await?; @@ -252,7 +252,7 @@ async fn invalidate_old_estimates( } async fn insert_estimate( - radio_id: String, + entity: &Entity, received_timestamp: DateTime, estimate: &RadioLocationEstimate, exec: &mut Transaction<'_, Postgres>, @@ -260,17 +260,18 @@ async fn insert_estimate( let radius = estimate.radius; let lat = estimate.lat; let long = estimate.long; - let hashed_key = hash_key(radio_id.clone(), received_timestamp, radius, lat, long); + let hashed_key = hash_key(entity, received_timestamp, radius, lat, long); sqlx::query( r#" - INSERT INTO radio_location_estimates (hashed_key, radio_id, received_timestamp, radius, lat, long, confidence) - VALUES ($1, $2, $3, $4, $5, $6, $7) + INSERT INTO radio_location_estimates (hashed_key, radio_type, radio_key, received_timestamp, radius, lat, long, confidence) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8) ON CONFLICT (hashed_key) DO NOTHING "#, ) .bind(hashed_key) - .bind(radio_id) + .bind(entity_to_radio_type(entity)) + .bind(entity.to_string()) .bind(received_timestamp) .bind(estimate.radius) .bind(lat) @@ -283,13 +284,13 @@ async fn insert_estimate( } pub fn hash_key( - radio_id: String, + entity: &Entity, timestamp: DateTime, radius: Decimal, lat: Decimal, long: Decimal, ) -> String { - let key = format!("{}{}{}{}{}", radio_id, timestamp, radius, lat, long); + let key = format!("{}{}{}{}{}", entity, timestamp, radius, lat, long); let mut hasher = Sha256::new(); hasher.update(key); @@ -313,3 +314,10 @@ pub async fn clear_invalided( .await?; Ok(()) } + +fn entity_to_radio_type(entity: &Entity) -> HbType { + match entity { + Entity::CbrsId(_) => HbType::Cbrs, + Entity::WifiPubKey(_) => HbType::Wifi, + } +} diff --git a/mobile_verifier/tests/integrations/radio_location_estimates.rs b/mobile_verifier/tests/integrations/radio_location_estimates.rs index f8465c2a6..c0d64bb4c 100644 --- a/mobile_verifier/tests/integrations/radio_location_estimates.rs +++ b/mobile_verifier/tests/integrations/radio_location_estimates.rs @@ -4,7 +4,7 @@ use file_store::{ file_info_poller::FileInfoStream, file_sink::FileSinkClient, radio_location_estimates::{ - RadioLocationEstimate, RadioLocationEstimateEvent, RadioLocationEstimatesReq, + Entity, RadioLocationEstimate, RadioLocationEstimateEvent, RadioLocationEstimatesReq, }, radio_location_estimates_ingest_report::RadioLocationEstimatesIngestReport, FileInfo, @@ -36,7 +36,7 @@ async fn verifier_test(pool: PgPool) -> anyhow::Result<()> { }); // Sending reports as if they are coming from ingestor - let (fis, reports, _public_key_binary) = file_info_stream(); + let (fis, reports) = file_info_stream(); reports_tx.send(fis).await?; let mut retry = 0; @@ -88,7 +88,6 @@ async fn verifier_test(pool: PgPool) -> anyhow::Result<()> { fn file_info_stream() -> ( FileInfoStream, Vec, - PublicKeyBinary, ) { let file_info = FileInfo { key: "test_file_info".to_string(), @@ -97,14 +96,21 @@ fn file_info_stream() -> ( size: 0, }; - let key_pair = generate_keypair(); - let public_key_binary: PublicKeyBinary = key_pair.public_key().to_owned().into(); + let carrier_key_pair = generate_keypair(); + let carrier_public_key_binary: PublicKeyBinary = + carrier_key_pair.public_key().to_owned().into(); + + let hotspot_key_pair = generate_keypair(); + let hotspot_public_key_binary: PublicKeyBinary = + hotspot_key_pair.public_key().to_owned().into(); + + let entity = Entity::WifiPubKey(hotspot_public_key_binary); let reports = vec![ RadioLocationEstimatesIngestReport { received_timestamp: Utc::now() - Duration::hours(1), report: RadioLocationEstimatesReq { - radio_id: "radio_1".to_string(), + entity: entity.clone(), estimates: vec![RadioLocationEstimate { radius: rust_decimal::Decimal::from_f32(0.1).unwrap(), lat: rust_decimal::Decimal::from_f32(0.1).unwrap(), @@ -116,13 +122,13 @@ fn file_info_stream() -> ( }], }], timestamp: Utc::now() - Duration::hours(1), - carrier_key: public_key_binary.clone(), + carrier_key: carrier_public_key_binary.clone(), }, }, RadioLocationEstimatesIngestReport { received_timestamp: Utc::now(), report: RadioLocationEstimatesReq { - radio_id: "radio_1".to_string(), + entity: entity.clone(), estimates: vec![RadioLocationEstimate { radius: rust_decimal::Decimal::from_f32(0.2).unwrap(), lat: rust_decimal::Decimal::from_f32(0.2).unwrap(), @@ -134,14 +140,13 @@ fn file_info_stream() -> ( }], }], timestamp: Utc::now(), - carrier_key: public_key_binary.clone(), + carrier_key: carrier_public_key_binary.clone(), }, }, ]; ( FileInfoStream::new("default".to_string(), file_info, reports.clone()), reports, - public_key_binary, ) } @@ -161,7 +166,7 @@ fn compare_report_and_estimate( ) { assert_eq!( hash_key( - report.report.radio_id.clone(), + &report.report.entity, report.received_timestamp, report.report.estimates[0].radius, report.report.estimates[0].lat, @@ -170,7 +175,7 @@ fn compare_report_and_estimate( estimate.hashed_key ); - assert_eq!(report.report.radio_id, estimate.radio_id); + assert_eq!(report.report.entity.to_string(), estimate.radio_key); assert!(timestamp_match( report.received_timestamp, estimate.received_timestamp @@ -190,7 +195,7 @@ fn compare_report_and_estimate( #[derive(Debug)] pub struct RadioLocationEstimateDB { pub hashed_key: String, - pub radio_id: String, + pub radio_key: String, pub received_timestamp: DateTime, pub radius: rust_decimal::Decimal, pub lat: rust_decimal::Decimal, @@ -204,7 +209,7 @@ pub async fn select_radio_location_estimates( ) -> anyhow::Result> { let rows = sqlx::query( r#" - SELECT hashed_key, radio_id, received_timestamp, radius, lat, long, confidence, invalided_at + SELECT hashed_key, radio_key, hashed_key, received_timestamp, radius, lat, long, confidence, invalided_at FROM radio_location_estimates ORDER BY received_timestamp ASC "#, @@ -216,7 +221,7 @@ pub async fn select_radio_location_estimates( .into_iter() .map(|row| RadioLocationEstimateDB { hashed_key: row.get("hashed_key"), - radio_id: row.get("radio_id"), + radio_key: row.get("radio_key"), received_timestamp: row.get("received_timestamp"), radius: row.get("radius"), lat: row.get("lat"), From 865b6d159af9d740709ce4ad82c89d365a2a24c2 Mon Sep 17 00:00:00 2001 From: Macpie Date: Tue, 15 Oct 2024 17:03:04 -0700 Subject: [PATCH 18/31] Add location cache as a top level thingy --- mobile_verifier/src/cli/server.rs | 27 ++++++-- mobile_verifier/src/heartbeats/cbrs.rs | 11 ++-- mobile_verifier/src/heartbeats/mod.rs | 2 + mobile_verifier/src/heartbeats/wifi.rs | 12 ++-- .../src/radio_location_estimates.rs | 62 +++++++++++-------- mobile_verifier/src/rewarder.rs | 11 ++-- 6 files changed, 81 insertions(+), 44 deletions(-) diff --git a/mobile_verifier/src/cli/server.rs b/mobile_verifier/src/cli/server.rs index 50d4fb61e..99db34828 100644 --- a/mobile_verifier/src/cli/server.rs +++ b/mobile_verifier/src/cli/server.rs @@ -1,11 +1,12 @@ -use std::time::Duration; - use crate::{ boosting_oracles::DataSetDownloaderDaemon, coverage::{new_coverage_object_notification_channel, CoverageDaemon}, data_session::DataSessionIngestor, geofence::Geofence, - heartbeats::{cbrs::CbrsHeartbeatDaemon, wifi::WifiHeartbeatDaemon}, + heartbeats::{ + cbrs::CbrsHeartbeatDaemon, last_location::LocationCache, wifi::WifiHeartbeatDaemon, + }, + radio_location_estimates::RadioLocationEstimatesDaemon, radio_threshold::RadioThresholdIngestor, rewarder::Rewarder, service_provider, @@ -26,6 +27,7 @@ use mobile_config::client::{ entity_client::EntityClient, hex_boosting_client::HexBoostingClient, AuthorizationClient, CarrierServiceClient, GatewayClient, }; +use std::time::Duration; use task_manager::TaskManager; #[derive(Debug, clap::Args)] @@ -103,6 +105,8 @@ impl Cmd { let (new_coverage_obj_notifier, new_coverage_obj_notification) = new_coverage_object_notification_channel(); + let location_cache = LocationCache::new(&pool); + TaskManager::builder() .add_task(file_upload_server) .add_task(valid_heartbeats_server) @@ -117,6 +121,7 @@ impl Cmd { valid_heartbeats.clone(), seniority_updates.clone(), usa_geofence, + location_cache.clone(), ) .await?, ) @@ -129,6 +134,7 @@ impl Cmd { valid_heartbeats, seniority_updates.clone(), usa_and_mexico_geofence, + location_cache.clone(), ) .await?, ) @@ -213,13 +219,23 @@ impl Cmd { ServiceProviderBoostedRewardsBanIngestor::create_managed_task( pool.clone(), file_upload.clone(), - report_ingest, - auth_client, + report_ingest.clone(), + auth_client.clone(), settings, seniority_updates, ) .await?, ) + .add_task( + RadioLocationEstimatesDaemon::create_managed_task( + pool.clone(), + settings, + file_upload.clone(), + report_ingest.clone(), + auth_client.clone(), + ) + .await?, + ) .add_task( Rewarder::create_managed_task( pool, @@ -228,6 +244,7 @@ impl Cmd { carrier_client, hex_boosting_client, speedtests_avg, + location_cache, ) .await?, ) diff --git a/mobile_verifier/src/heartbeats/cbrs.rs b/mobile_verifier/src/heartbeats/cbrs.rs index e02010c81..9e17332f9 100644 --- a/mobile_verifier/src/heartbeats/cbrs.rs +++ b/mobile_verifier/src/heartbeats/cbrs.rs @@ -33,6 +33,7 @@ pub struct CbrsHeartbeatDaemon { heartbeat_sink: FileSinkClient, seniority_sink: FileSinkClient, geofence: GFV, + location_cache: LocationCache, } impl CbrsHeartbeatDaemon @@ -49,6 +50,7 @@ where valid_heartbeats: FileSinkClient, seniority_updates: FileSinkClient, geofence: GFV, + location_cache: LocationCache, ) -> anyhow::Result { // CBRS Heartbeats let (cbrs_heartbeats, cbrs_heartbeats_server) = @@ -69,6 +71,7 @@ where valid_heartbeats, seniority_updates, geofence, + location_cache, ); Ok(TaskManager::builder() @@ -86,6 +89,7 @@ where heartbeat_sink: FileSinkClient, seniority_sink: FileSinkClient, geofence: GFV, + location_cache: LocationCache, ) -> Self { Self { pool, @@ -95,6 +99,7 @@ where heartbeat_sink, seniority_sink, geofence, + location_cache, } } @@ -111,8 +116,6 @@ where let coverage_claim_time_cache = CoverageClaimTimeCache::new(); let coverage_object_cache = CoverageObjectCache::new(&self.pool); - // Unused: - let location_cache = LocationCache::new(&self.pool); loop { tokio::select! { @@ -128,7 +131,6 @@ where &heartbeat_cache, &coverage_claim_time_cache, &coverage_object_cache, - &location_cache, ).await?; metrics::histogram!("cbrs_heartbeat_processing_time") .record(start.elapsed()); @@ -145,7 +147,6 @@ where heartbeat_cache: &Arc), ()>>, coverage_claim_time_cache: &CoverageClaimTimeCache, coverage_object_cache: &CoverageObjectCache, - location_cache: &LocationCache, ) -> anyhow::Result<()> { tracing::info!("Processing CBRS heartbeat file {}", file.file_info.key); let mut transaction = self.pool.begin().await?; @@ -166,7 +167,7 @@ where heartbeats, &self.gateway_info_resolver, coverage_object_cache, - location_cache, + &self.location_cache, self.max_distance_to_coverage, &epoch, &self.geofence, diff --git a/mobile_verifier/src/heartbeats/mod.rs b/mobile_verifier/src/heartbeats/mod.rs index b72b47526..932207f3d 100644 --- a/mobile_verifier/src/heartbeats/mod.rs +++ b/mobile_verifier/src/heartbeats/mod.rs @@ -496,6 +496,8 @@ impl ValidatedHeartbeat { Some(coverage_object.meta), proto::HeartbeatValidity::InvalidDeviceType, )), + // TODO do we get there when CBRS? + // Should I then update location form here then? GatewayResolution::GatewayNotFound => Ok(Self::new( heartbeat, cell_type, diff --git a/mobile_verifier/src/heartbeats/wifi.rs b/mobile_verifier/src/heartbeats/wifi.rs index 9d2ed0d5f..52336a584 100644 --- a/mobile_verifier/src/heartbeats/wifi.rs +++ b/mobile_verifier/src/heartbeats/wifi.rs @@ -32,6 +32,7 @@ pub struct WifiHeartbeatDaemon { heartbeat_sink: FileSinkClient, seniority_sink: FileSinkClient, geofence: GFV, + location_cache: LocationCache, } impl WifiHeartbeatDaemon @@ -48,6 +49,7 @@ where valid_heartbeats: FileSinkClient, seniority_updates: FileSinkClient, geofence: GFV, + location_cache: LocationCache, ) -> anyhow::Result { // Wifi Heartbeats let (wifi_heartbeats, wifi_heartbeats_server) = @@ -67,6 +69,7 @@ where valid_heartbeats, seniority_updates, geofence, + location_cache, ); Ok(TaskManager::builder() @@ -84,6 +87,7 @@ where heartbeat_sink: FileSinkClient, seniority_sink: FileSinkClient, geofence: GFV, + location_cache: LocationCache, ) -> Self { Self { pool, @@ -93,6 +97,7 @@ where heartbeat_sink, seniority_sink, geofence, + location_cache, } } @@ -109,7 +114,6 @@ where let coverage_claim_time_cache = CoverageClaimTimeCache::new(); let coverage_object_cache = CoverageObjectCache::new(&self.pool); - let location_cache = LocationCache::new(&self.pool); loop { tokio::select! { @@ -124,8 +128,7 @@ where file, &heartbeat_cache, &coverage_claim_time_cache, - &coverage_object_cache, - &location_cache + &coverage_object_cache ).await?; metrics::histogram!("wifi_heartbeat_processing_time") .record(start.elapsed()); @@ -142,7 +145,6 @@ where heartbeat_cache: &Cache<(String, DateTime), ()>, coverage_claim_time_cache: &CoverageClaimTimeCache, coverage_object_cache: &CoverageObjectCache, - location_cache: &LocationCache, ) -> anyhow::Result<()> { tracing::info!("Processing WIFI heartbeat file {}", file.file_info.key); let mut transaction = self.pool.begin().await?; @@ -157,7 +159,7 @@ where heartbeats, &self.gateway_info_resolver, coverage_object_cache, - location_cache, + &self.location_cache, self.max_distance_to_coverage, &epoch, &self.geofence, diff --git a/mobile_verifier/src/radio_location_estimates.rs b/mobile_verifier/src/radio_location_estimates.rs index b72b39dcc..6a70f7e6d 100644 --- a/mobile_verifier/src/radio_location_estimates.rs +++ b/mobile_verifier/src/radio_location_estimates.rs @@ -1,4 +1,6 @@ -use crate::{heartbeats::HbType, Settings}; +use std::str::FromStr; + +use crate::{heartbeats::HbType, sp_boosted_rewards_bans::BannedRadios, Settings}; use chrono::{DateTime, Utc}; use file_store::{ file_info_poller::{FileInfoStream, LookbackBehavior}, @@ -21,12 +23,13 @@ use helium_proto::services::{ }; use mobile_config::client::authorization_client::AuthorizationVerifier; use rust_decimal::Decimal; +use rust_decimal_macros::dec; use sha2::{Digest, Sha256}; -use sqlx::{Pool, Postgres, Transaction}; +use sqlx::{PgPool, Pool, Postgres, Row, Transaction}; use task_manager::{ManagedTask, TaskManager}; use tokio::sync::mpsc::Receiver; -// const CONFIDENCE_THRESHOLD: Decimal = dec!(0.75); +const CONFIDENCE_THRESHOLD: Decimal = dec!(0.75); pub struct RadioLocationEstimatesDaemon { pool: Pool, @@ -56,9 +59,9 @@ where pub async fn create_managed_task( pool: Pool, settings: &Settings, - authorization_verifier: AV, - file_store: FileStore, file_upload: FileUpload, + file_store: FileStore, + authorization_verifier: AV, ) -> anyhow::Result { let (reports_receiver, reports_receiver_server) = file_source::continuous_source::() @@ -210,26 +213,6 @@ async fn save_to_db( Ok(()) } -// TODO: knowing that radio could also be a PublicKey for wifi hotspot. -// Should we make radio_id in report a binary or some kind of proto enum to have one or other? - -// async fn maybe_ban_radios( -// report: &RadioLocationEstimatesIngestReport, -// exec: &mut Transaction<'_, Postgres>, -// ) -> Result<(), sqlx::Error> { -// let estimates = &report.report.estimates; -// let radio_id = &report.report.radio_id; -// let mut will_ban = true; - -// for estimate in estimates { -// if estimate.confidence > CONFIDENCE_THRESHOLD { -// will_ban = false; -// } -// } - -// Ok(()) -// } - async fn invalidate_old_estimates( entity: &Entity, timestamp: DateTime, @@ -315,6 +298,35 @@ pub async fn clear_invalided( Ok(()) } +// This is wrong should be a get estimates but will fix later +pub async fn get_banned_radios(pool: &PgPool) -> anyhow::Result { + // TODO: Do we still want to ban any radio that is NOT in this table? + // Might be multiple per radio + // check assertion in circle as well + sqlx::query( + r#" + SELECT radio_type, radio_key + FROM radio_location_estimates + WHERE confidence < $1 + AND invalided_at IS NULL + "#, + ) + .bind(CONFIDENCE_THRESHOLD) + .fetch(pool) + .map_err(anyhow::Error::from) + .try_fold(BannedRadios::default(), |mut set, row| async move { + let radio_type = row.get::("radio_type"); + let radio_key = row.get::("radio_key"); + match radio_type { + HbType::Wifi => set.insert_wifi(PublicKeyBinary::from_str(&radio_key)?), + HbType::Cbrs => set.insert_cbrs(radio_key), + }; + + Ok(set) + }) + .await +} + fn entity_to_radio_type(entity: &Entity) -> HbType { match entity { Entity::CbrsId(_) => HbType::Cbrs, diff --git a/mobile_verifier/src/rewarder.rs b/mobile_verifier/src/rewarder.rs index 580777859..6d07428ee 100644 --- a/mobile_verifier/src/rewarder.rs +++ b/mobile_verifier/src/rewarder.rs @@ -1,7 +1,8 @@ +use self::boosted_hex_eligibility::BoostedHexEligibility; use crate::{ boosting_oracles::db::check_for_unprocessed_data_sets, coverage, data_session, - heartbeats::{self, HeartbeatReward}, + heartbeats::{self, last_location::LocationCache, HeartbeatReward}, radio_location_estimates, radio_threshold, reward_shares::{ self, CalculatedPocRewardShares, CoverageShares, DataTransferAndPocAllocatedRewardBuckets, @@ -20,7 +21,6 @@ use file_store::{ traits::{FileSinkCommitStrategy, FileSinkRollTime, FileSinkWriteExt, TimestampEncode}, }; use futures_util::TryFutureExt; - use helium_proto::{ reward_manifest::RewardData::MobileRewardData, services::poc_mobile::{ @@ -46,8 +46,6 @@ use std::{ops::Range, time::Duration}; use task_manager::{ManagedTask, TaskManager}; use tokio::time::sleep; -use self::boosted_hex_eligibility::BoostedHexEligibility; - pub mod boosted_hex_eligibility; const REWARDS_NOT_CURRENT_DELAY_PERIOD: i64 = 5; @@ -62,6 +60,7 @@ pub struct Rewarder { reward_manifests: FileSinkClient, price_tracker: PriceTracker, speedtest_averages: FileSinkClient, + location_cache: LocationCache, } impl Rewarder @@ -76,6 +75,7 @@ where carrier_service_verifier: A, hex_boosting_info_resolver: B, speedtests_avg: FileSinkClient, + location_cache: LocationCache, ) -> anyhow::Result { let (price_tracker, price_daemon) = PriceTracker::new_tm(&settings.price_tracker).await?; @@ -107,6 +107,7 @@ where reward_manifests, price_tracker, speedtests_avg, + location_cache, ); Ok(TaskManager::builder() @@ -128,6 +129,7 @@ where reward_manifests: FileSinkClient, price_tracker: PriceTracker, speedtest_averages: FileSinkClient, + location_cache: LocationCache, ) -> Self { Self { pool, @@ -139,6 +141,7 @@ where reward_manifests, price_tracker, speedtest_averages, + location_cache, } } From 1167c2d6402f2ea6e33167bead5d611b3d3e53ff Mon Sep 17 00:00:00 2001 From: Macpie Date: Wed, 16 Oct 2024 12:31:39 -0700 Subject: [PATCH 19/31] Update location cache to handle cbrs and wifi --- .../src/heartbeats/last_location.rs | 83 ++++++++++--------- mobile_verifier/src/heartbeats/mod.rs | 5 +- 2 files changed, 45 insertions(+), 43 deletions(-) diff --git a/mobile_verifier/src/heartbeats/last_location.rs b/mobile_verifier/src/heartbeats/last_location.rs index cb6e45c76..794926251 100644 --- a/mobile_verifier/src/heartbeats/last_location.rs +++ b/mobile_verifier/src/heartbeats/last_location.rs @@ -1,9 +1,14 @@ -use std::sync::Arc; - use chrono::{DateTime, Duration, Utc}; use helium_crypto::PublicKeyBinary; use retainer::Cache; use sqlx::PgPool; +use std::sync::Arc; + +#[derive(Debug, Clone, Eq, Ord, PartialEq, PartialOrd)] +pub enum Key { + CbrsId(String), + WifiPubKey(PublicKeyBinary), +} #[derive(sqlx::FromRow, Copy, Clone)] pub struct LastLocation { @@ -38,7 +43,7 @@ impl LastLocation { #[derive(Clone)] pub struct LocationCache { pool: PgPool, - locations: Arc>>, + locations: Arc>>, } impl LocationCache { @@ -56,9 +61,39 @@ impl LocationCache { } } - async fn fetch_from_db_and_set( + pub async fn fetch_last_location(&self, key: Key) -> anyhow::Result> { + Ok( + if let Some(last_location) = self.locations.get(&key).await { + *last_location + } else { + match key { + Key::WifiPubKey(pub_key_bin) => self.fetch_wifi_and_set(pub_key_bin).await?, + Key::CbrsId(_) => None, + } + }, + ) + } + + pub async fn set_last_location( + &self, + key: Key, + last_location: LastLocation, + ) -> anyhow::Result<()> { + let duration_to_expiration = last_location.duration_to_expiration(); + self.locations + .insert(key, Some(last_location), duration_to_expiration.to_std()?) + .await; + Ok(()) + } + + /// Only used for testing. + pub async fn delete_last_location(&self, key: Key) { + self.locations.remove(&key).await; + } + + async fn fetch_wifi_and_set( &self, - hotspot: &PublicKeyBinary, + pub_key_bin: PublicKeyBinary, ) -> anyhow::Result> { let last_location: Option = sqlx::query_as( r#" @@ -72,12 +107,12 @@ impl LocationCache { "#, ) .bind(Utc::now() - Duration::hours(12)) - .bind(hotspot) + .bind(pub_key_bin.clone()) .fetch_optional(&self.pool) .await?; self.locations .insert( - hotspot.clone(), + Key::WifiPubKey(pub_key_bin), last_location, last_location .map(|x| x.duration_to_expiration()) @@ -87,38 +122,4 @@ impl LocationCache { .await; Ok(last_location) } - - pub async fn fetch_last_location( - &self, - hotspot: &PublicKeyBinary, - ) -> anyhow::Result> { - Ok( - if let Some(last_location) = self.locations.get(hotspot).await { - *last_location - } else { - self.fetch_from_db_and_set(hotspot).await? - }, - ) - } - - pub async fn set_last_location( - &self, - hotspot: &PublicKeyBinary, - last_location: LastLocation, - ) -> anyhow::Result<()> { - let duration_to_expiration = last_location.duration_to_expiration(); - self.locations - .insert( - hotspot.clone(), - Some(last_location), - duration_to_expiration.to_std()?, - ) - .await; - Ok(()) - } - - /// Only used for testing. - pub async fn delete_last_location(&self, hotspot: &PublicKeyBinary) { - self.locations.remove(hotspot).await; - } } diff --git a/mobile_verifier/src/heartbeats/mod.rs b/mobile_verifier/src/heartbeats/mod.rs index 932207f3d..afc107dbd 100644 --- a/mobile_verifier/src/heartbeats/mod.rs +++ b/mobile_verifier/src/heartbeats/mod.rs @@ -19,6 +19,7 @@ use futures::stream::{Stream, StreamExt}; use h3o::{CellIndex, LatLng}; use helium_crypto::PublicKeyBinary; use helium_proto::services::poc_mobile::{self as proto, LocationSource}; +use last_location::Key; use retainer::Cache; use rust_decimal::{prelude::ToPrimitive, Decimal}; use rust_decimal_macros::dec; @@ -521,7 +522,7 @@ impl ValidatedHeartbeat { let is_valid = match heartbeat.location_validation_timestamp { None => { if let Some(last_location) = last_location_cache - .fetch_last_location(&heartbeat.hotspot_key) + .fetch_last_location(Key::WifiPubKey(heartbeat.hotspot_key.clone())) .await? { heartbeat.lat = last_location.lat; @@ -538,7 +539,7 @@ impl ValidatedHeartbeat { Some(location_validation_timestamp) => { last_location_cache .set_last_location( - &heartbeat.hotspot_key, + Key::WifiPubKey(heartbeat.hotspot_key.clone()), LastLocation::new( location_validation_timestamp, heartbeat.timestamp, From 925e64eb83917aabd9c564f4751a92d75c07b87c Mon Sep 17 00:00:00 2001 From: Macpie Date: Wed, 16 Oct 2024 12:47:21 -0700 Subject: [PATCH 20/31] Populate cache with cbrs as well --- .../migrations/39_update_cbrs_hearbeats.sql | 4 +++ .../src/heartbeats/last_location.rs | 31 ++++++++++++++++++- mobile_verifier/src/heartbeats/mod.rs | 7 +++-- 3 files changed, 39 insertions(+), 3 deletions(-) create mode 100644 mobile_verifier/migrations/39_update_cbrs_hearbeats.sql diff --git a/mobile_verifier/migrations/39_update_cbrs_hearbeats.sql b/mobile_verifier/migrations/39_update_cbrs_hearbeats.sql new file mode 100644 index 000000000..6a1056a3d --- /dev/null +++ b/mobile_verifier/migrations/39_update_cbrs_hearbeats.sql @@ -0,0 +1,4 @@ +ALTER TABLE cbrs_heartbeats +ADD COLUMN location_validation_timestamp TIMESTAMPTZ, +ADD COLUMN lat DOUBLE PRECISION NOT NULL DEFAULT 0.0, +ADD COLUMN lon DOUBLE PRECISION NOT NULL DEFAULT 0.0; \ No newline at end of file diff --git a/mobile_verifier/src/heartbeats/last_location.rs b/mobile_verifier/src/heartbeats/last_location.rs index 794926251..a422ca815 100644 --- a/mobile_verifier/src/heartbeats/last_location.rs +++ b/mobile_verifier/src/heartbeats/last_location.rs @@ -68,7 +68,7 @@ impl LocationCache { } else { match key { Key::WifiPubKey(pub_key_bin) => self.fetch_wifi_and_set(pub_key_bin).await?, - Key::CbrsId(_) => None, + Key::CbrsId(id) => self.fetch_cbrs_and_set(id).await?, } }, ) @@ -122,4 +122,33 @@ impl LocationCache { .await; Ok(last_location) } + + async fn fetch_cbrs_and_set(&self, cbsd_id: String) -> anyhow::Result> { + let last_location: Option = sqlx::query_as( + r#" + SELECT location_validation_timestamp, latest_timestamp, lat, lon + FROM cbrs_heartbeats + WHERE location_validation_timestamp IS NOT NULL + AND latest_timestamp >= $1 + AND hotspot_key = $2 + ORDER BY latest_timestamp DESC + LIMIT 1 + "#, + ) + .bind(Utc::now() - Duration::hours(12)) + .bind(cbsd_id.clone()) + .fetch_optional(&self.pool) + .await?; + self.locations + .insert( + Key::CbrsId(cbsd_id), + last_location, + last_location + .map(|x| x.duration_to_expiration()) + .unwrap_or_else(|| Duration::days(365)) + .to_std()?, + ) + .await; + Ok(last_location) + } } diff --git a/mobile_verifier/src/heartbeats/mod.rs b/mobile_verifier/src/heartbeats/mod.rs index afc107dbd..5f35609b9 100644 --- a/mobile_verifier/src/heartbeats/mod.rs +++ b/mobile_verifier/src/heartbeats/mod.rs @@ -680,8 +680,8 @@ impl ValidatedHeartbeat { let truncated_timestamp = self.truncated_timestamp()?; sqlx::query( r#" - INSERT INTO cbrs_heartbeats (cbsd_id, hotspot_key, cell_type, latest_timestamp, truncated_timestamp, coverage_object, location_trust_score_multiplier) - VALUES ($1, $2, $3, $4, $5, $6, $7) + INSERT INTO cbrs_heartbeats (cbsd_id, hotspot_key, cell_type, latest_timestamp, truncated_timestamp, coverage_object, location_trust_score_multiplier, location_validation_timestamp, lat, lon) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) ON CONFLICT (cbsd_id, truncated_timestamp) DO UPDATE SET latest_timestamp = EXCLUDED.latest_timestamp, coverage_object = EXCLUDED.coverage_object @@ -694,6 +694,9 @@ impl ValidatedHeartbeat { .bind(truncated_timestamp) .bind(self.heartbeat.coverage_object) .bind(self.location_trust_score_multiplier) + .bind(self.heartbeat.location_validation_timestamp) + .bind(self.heartbeat.lat) + .bind(self.heartbeat.lon) .execute(&mut *exec) .await?; Ok(()) From 2e57553113093239bd0e764cdbe05a9877313301 Mon Sep 17 00:00:00 2001 From: Macpie Date: Wed, 16 Oct 2024 12:52:33 -0700 Subject: [PATCH 21/31] Fix test --- mobile_verifier/tests/integrations/last_location.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/mobile_verifier/tests/integrations/last_location.rs b/mobile_verifier/tests/integrations/last_location.rs index c04d0678d..723a1bbef 100644 --- a/mobile_verifier/tests/integrations/last_location.rs +++ b/mobile_verifier/tests/integrations/last_location.rs @@ -8,7 +8,10 @@ use helium_proto::services::poc_mobile::{self as proto, LocationSource}; use mobile_verifier::{ coverage::{CoverageObject, CoverageObjectCache}, geofence::GeofenceValidator, - heartbeats::{last_location::LocationCache, HbType, Heartbeat, ValidatedHeartbeat}, + heartbeats::{ + last_location::{Key, LocationCache}, + HbType, Heartbeat, ValidatedHeartbeat, + }, }; use rust_decimal_macros::dec; use sqlx::{PgPool, Postgres, Transaction}; @@ -122,7 +125,9 @@ async fn heartbeat_will_use_last_good_location_from_db(pool: PgPool) -> anyhow:: dec!(1.0) ); - location_cache.delete_last_location(&hotspot).await; + location_cache + .delete_last_location(Key::WifiPubKey(hotspot.clone())) + .await; transaction = pool.begin().await?; validated_heartbeat_1.clone().save(&mut transaction).await?; transaction.commit().await?; From 04ccd71504988918ea15187a5ab7f2eece10a1f7 Mon Sep 17 00:00:00 2001 From: Macpie Date: Wed, 16 Oct 2024 15:29:38 -0700 Subject: [PATCH 22/31] Update Location Cache --- mobile_verifier/src/cli/server.rs | 4 +- mobile_verifier/src/heartbeats/cbrs.rs | 10 +- .../src/heartbeats/last_location.rs | 154 ----------------- .../src/heartbeats/location_cache.rs | 160 ++++++++++++++++++ mobile_verifier/src/heartbeats/mod.rs | 26 ++- mobile_verifier/src/heartbeats/wifi.rs | 5 +- mobile_verifier/src/rewarder.rs | 3 +- .../tests/integrations/boosting_oracles.rs | 2 +- .../tests/integrations/last_location.rs | 7 +- .../tests/integrations/modeled_coverage.rs | 2 +- 10 files changed, 188 insertions(+), 185 deletions(-) delete mode 100644 mobile_verifier/src/heartbeats/last_location.rs create mode 100644 mobile_verifier/src/heartbeats/location_cache.rs diff --git a/mobile_verifier/src/cli/server.rs b/mobile_verifier/src/cli/server.rs index 99db34828..9b6dfa3fe 100644 --- a/mobile_verifier/src/cli/server.rs +++ b/mobile_verifier/src/cli/server.rs @@ -3,9 +3,7 @@ use crate::{ coverage::{new_coverage_object_notification_channel, CoverageDaemon}, data_session::DataSessionIngestor, geofence::Geofence, - heartbeats::{ - cbrs::CbrsHeartbeatDaemon, last_location::LocationCache, wifi::WifiHeartbeatDaemon, - }, + heartbeats::{cbrs::CbrsHeartbeatDaemon, location_cache::LocationCache, wifi::WifiHeartbeatDaemon}, radio_location_estimates::RadioLocationEstimatesDaemon, radio_threshold::RadioThresholdIngestor, rewarder::Rewarder, diff --git a/mobile_verifier/src/heartbeats/cbrs.rs b/mobile_verifier/src/heartbeats/cbrs.rs index 9e17332f9..143405ae5 100644 --- a/mobile_verifier/src/heartbeats/cbrs.rs +++ b/mobile_verifier/src/heartbeats/cbrs.rs @@ -1,8 +1,9 @@ -use super::{process_validated_heartbeats, Heartbeat, ValidatedHeartbeat}; +use super::{ + location_cache::LocationCache, process_validated_heartbeats, Heartbeat, ValidatedHeartbeat, +}; use crate::{ coverage::{CoverageClaimTimeCache, CoverageObjectCache}, geofence::GeofenceValidator, - heartbeats::LocationCache, GatewayResolver, Settings, }; @@ -16,7 +17,6 @@ use file_store::{ }; use futures::{stream::StreamExt, TryFutureExt}; use helium_proto::services::poc_mobile as proto; -use retainer::Cache; use sqlx::{Pool, Postgres}; use std::{ sync::Arc, @@ -105,7 +105,7 @@ where pub async fn run(mut self, shutdown: triggered::Listener) -> anyhow::Result<()> { tracing::info!("Starting CBRS HeartbeatDaemon"); - let heartbeat_cache = Arc::new(Cache::<(String, DateTime), ()>::new()); + let heartbeat_cache = Arc::new(retainer::Cache::<(String, DateTime), ()>::new()); let heartbeat_cache_clone = heartbeat_cache.clone(); tokio::spawn(async move { @@ -144,7 +144,7 @@ where async fn process_file( &self, file: FileInfoStream, - heartbeat_cache: &Arc), ()>>, + heartbeat_cache: &Arc), ()>>, coverage_claim_time_cache: &CoverageClaimTimeCache, coverage_object_cache: &CoverageObjectCache, ) -> anyhow::Result<()> { diff --git a/mobile_verifier/src/heartbeats/last_location.rs b/mobile_verifier/src/heartbeats/last_location.rs deleted file mode 100644 index a422ca815..000000000 --- a/mobile_verifier/src/heartbeats/last_location.rs +++ /dev/null @@ -1,154 +0,0 @@ -use chrono::{DateTime, Duration, Utc}; -use helium_crypto::PublicKeyBinary; -use retainer::Cache; -use sqlx::PgPool; -use std::sync::Arc; - -#[derive(Debug, Clone, Eq, Ord, PartialEq, PartialOrd)] -pub enum Key { - CbrsId(String), - WifiPubKey(PublicKeyBinary), -} - -#[derive(sqlx::FromRow, Copy, Clone)] -pub struct LastLocation { - pub location_validation_timestamp: DateTime, - pub latest_timestamp: DateTime, - pub lat: f64, - pub lon: f64, -} - -impl LastLocation { - pub fn new( - location_validation_timestamp: DateTime, - latest_timestamp: DateTime, - lat: f64, - lon: f64, - ) -> Self { - Self { - location_validation_timestamp, - latest_timestamp, - lat, - lon, - } - } - - /// Calculates the duration from now in which last_valid_timestamp is 12 hours old - pub fn duration_to_expiration(&self) -> Duration { - ((self.latest_timestamp + Duration::hours(12)) - Utc::now()).max(Duration::zero()) - } -} - -/// A cache for previous valid (or invalid) WiFi heartbeat locations -#[derive(Clone)] -pub struct LocationCache { - pool: PgPool, - locations: Arc>>, -} - -impl LocationCache { - pub fn new(pool: &PgPool) -> Self { - let locations = Arc::new(Cache::new()); - let locations_clone = locations.clone(); - tokio::spawn(async move { - locations_clone - .monitor(4, 0.25, std::time::Duration::from_secs(60 * 60 * 24)) - .await - }); - Self { - pool: pool.clone(), - locations, - } - } - - pub async fn fetch_last_location(&self, key: Key) -> anyhow::Result> { - Ok( - if let Some(last_location) = self.locations.get(&key).await { - *last_location - } else { - match key { - Key::WifiPubKey(pub_key_bin) => self.fetch_wifi_and_set(pub_key_bin).await?, - Key::CbrsId(id) => self.fetch_cbrs_and_set(id).await?, - } - }, - ) - } - - pub async fn set_last_location( - &self, - key: Key, - last_location: LastLocation, - ) -> anyhow::Result<()> { - let duration_to_expiration = last_location.duration_to_expiration(); - self.locations - .insert(key, Some(last_location), duration_to_expiration.to_std()?) - .await; - Ok(()) - } - - /// Only used for testing. - pub async fn delete_last_location(&self, key: Key) { - self.locations.remove(&key).await; - } - - async fn fetch_wifi_and_set( - &self, - pub_key_bin: PublicKeyBinary, - ) -> anyhow::Result> { - let last_location: Option = sqlx::query_as( - r#" - SELECT location_validation_timestamp, latest_timestamp, lat, lon - FROM wifi_heartbeats - WHERE location_validation_timestamp IS NOT NULL - AND latest_timestamp >= $1 - AND hotspot_key = $2 - ORDER BY latest_timestamp DESC - LIMIT 1 - "#, - ) - .bind(Utc::now() - Duration::hours(12)) - .bind(pub_key_bin.clone()) - .fetch_optional(&self.pool) - .await?; - self.locations - .insert( - Key::WifiPubKey(pub_key_bin), - last_location, - last_location - .map(|x| x.duration_to_expiration()) - .unwrap_or_else(|| Duration::days(365)) - .to_std()?, - ) - .await; - Ok(last_location) - } - - async fn fetch_cbrs_and_set(&self, cbsd_id: String) -> anyhow::Result> { - let last_location: Option = sqlx::query_as( - r#" - SELECT location_validation_timestamp, latest_timestamp, lat, lon - FROM cbrs_heartbeats - WHERE location_validation_timestamp IS NOT NULL - AND latest_timestamp >= $1 - AND hotspot_key = $2 - ORDER BY latest_timestamp DESC - LIMIT 1 - "#, - ) - .bind(Utc::now() - Duration::hours(12)) - .bind(cbsd_id.clone()) - .fetch_optional(&self.pool) - .await?; - self.locations - .insert( - Key::CbrsId(cbsd_id), - last_location, - last_location - .map(|x| x.duration_to_expiration()) - .unwrap_or_else(|| Duration::days(365)) - .to_std()?, - ) - .await; - Ok(last_location) - } -} diff --git a/mobile_verifier/src/heartbeats/location_cache.rs b/mobile_verifier/src/heartbeats/location_cache.rs new file mode 100644 index 000000000..cb9ea2422 --- /dev/null +++ b/mobile_verifier/src/heartbeats/location_cache.rs @@ -0,0 +1,160 @@ +use chrono::{DateTime, Duration, Utc}; +use helium_crypto::PublicKeyBinary; +use sqlx::PgPool; +use std::{collections::HashMap, sync::Arc}; +use tokio::sync::Mutex; +use tracing::info; + +#[derive(Debug, Clone, Hash, Eq, PartialEq)] +pub enum LocationCacheKey { + CbrsId(String), + WifiPubKey(PublicKeyBinary), +} + +#[derive(sqlx::FromRow, Copy, Clone, Debug)] +pub struct LocationCacheValue { + pub lat: f64, + pub lon: f64, + pub timestamp: DateTime, +} + +impl LocationCacheValue { + pub fn new(lat: f64, lon: f64, timestamp: DateTime) -> Self { + Self { + lat, + lon, + timestamp, + } + } +} + +/// A cache WiFi/Cbrs heartbeat locations +#[derive(Clone)] +pub struct LocationCache { + pool: PgPool, + data: Arc>>, +} + +impl LocationCache { + pub fn new(pool: &PgPool) -> Self { + let data = Arc::new(Mutex::new( + HashMap::::new(), + )); + let data_clone = data.clone(); + tokio::spawn(async move { + loop { + // Sleep 1 hour + let duration = core::time::Duration::from_secs(60 * 60); + tokio::time::sleep(duration).await; + + let now = Utc::now(); + // Set the 12-hour threshold + let twelve_hours_ago = now - Duration::hours(12); + + let mut data = data_clone.lock().await; + let size_before = data.len() as f64; + + // Retain only values that are within the last 12 hours + data.retain(|_, v| v.timestamp > twelve_hours_ago); + + let size_after = data.len() as f64; + info!("cleaned {}", size_before - size_after); + } + }); + Self { + pool: pool.clone(), + data, + } + } + + pub async fn get(&self, key: LocationCacheKey) -> anyhow::Result> { + { + let data = self.data.lock().await; + if let Some(&value) = data.get(&key) { + return Ok(Some(value)); + } + } + match key { + LocationCacheKey::WifiPubKey(pub_key_bin) => { + self.fetch_wifi_and_insert(pub_key_bin).await + } + LocationCacheKey::CbrsId(id) => self.fetch_cbrs_and_insert(id).await, + } + } + + pub async fn insert( + &self, + key: LocationCacheKey, + value: LocationCacheValue, + ) -> anyhow::Result<()> { + let mut data = self.data.lock().await; + data.insert(key, value); + Ok(()) + } + + /// Only used for testing. + pub async fn remove(&self, key: LocationCacheKey) -> anyhow::Result<()> { + let mut data = self.data.lock().await; + data.remove(&key); + Ok(()) + } + + async fn fetch_wifi_and_insert( + &self, + pub_key_bin: PublicKeyBinary, + ) -> anyhow::Result> { + let sqlx_return: Option = sqlx::query_as( + r#" + SELECT lat, lon, location_validation_timestamp AS timestamp + FROM wifi_heartbeats + WHERE location_validation_timestamp IS NOT NULL + AND location_validation_timestamp >= $1 + AND hotspot_key = $2 + ORDER BY location_validation_timestamp DESC + LIMIT 1 + "#, + ) + .bind(Utc::now() - Duration::hours(12)) + .bind(pub_key_bin.clone()) + .fetch_optional(&self.pool) + .await?; + match sqlx_return { + None => Ok(None), + Some(value) => { + let mut data = self.data.lock().await; + data.insert(LocationCacheKey::WifiPubKey(pub_key_bin), value); + Ok(Some(value)) + } + } + } + + async fn fetch_cbrs_and_insert( + &self, + cbsd_id: String, + ) -> anyhow::Result> { + let sqlx_return: Option = sqlx::query_as( + r#" + SELECT lat, lon, location_validation_timestamp AS timestamp + FROM cbrs_heartbeats + WHERE location_validation_timestamp IS NOT NULL + AND location_validation_timestamp >= $1 + AND hotspot_key = $2 + ORDER BY location_validation_timestamp DESC + LIMIT 1 + "#, + ) + .bind(Utc::now() - Duration::hours(12)) + .bind(cbsd_id.clone()) + .fetch_optional(&self.pool) + .await?; + + match sqlx_return { + None => Ok(None), + Some(value) => { + let mut data = self.data.lock().await; + data.insert(LocationCacheKey::CbrsId(cbsd_id), value); + Ok(Some(value)) + } + } + } +} diff --git a/mobile_verifier/src/heartbeats/mod.rs b/mobile_verifier/src/heartbeats/mod.rs index 5f35609b9..6938314fc 100644 --- a/mobile_verifier/src/heartbeats/mod.rs +++ b/mobile_verifier/src/heartbeats/mod.rs @@ -1,7 +1,8 @@ pub mod cbrs; -pub mod last_location; +pub mod location_cache; pub mod wifi; +use self::location_cache::{LocationCache, LocationCacheKey, LocationCacheValue}; use crate::{ cell_type::{CellType, CellTypeLabel}, coverage::{CoverageClaimTimeCache, CoverageObjectCache, CoverageObjectMeta}, @@ -19,7 +20,6 @@ use futures::stream::{Stream, StreamExt}; use h3o::{CellIndex, LatLng}; use helium_crypto::PublicKeyBinary; use helium_proto::services::poc_mobile::{self as proto, LocationSource}; -use last_location::Key; use retainer::Cache; use rust_decimal::{prelude::ToPrimitive, Decimal}; use rust_decimal_macros::dec; @@ -27,8 +27,6 @@ use sqlx::{postgres::PgTypeInfo, Decode, Encode, Postgres, Transaction, Type}; use std::{ops::Range, pin::pin, time}; use uuid::Uuid; -use self::last_location::{LastLocation, LocationCache}; - /// Minimum number of heartbeats required to give a reward to the hotspot. const MINIMUM_HEARTBEAT_COUNT: i64 = 12; @@ -376,7 +374,7 @@ impl ValidatedHeartbeat { mut heartbeat: Heartbeat, gateway_info_resolver: &impl GatewayResolver, coverage_object_cache: &CoverageObjectCache, - last_location_cache: &LocationCache, + location_cache: &LocationCache, max_distance_to_coverage: u32, epoch: &Range>, geofence: &impl GeofenceValidator, @@ -521,14 +519,13 @@ impl ValidatedHeartbeat { let asserted_latlng: LatLng = CellIndex::try_from(location)?.into(); let is_valid = match heartbeat.location_validation_timestamp { None => { - if let Some(last_location) = last_location_cache - .fetch_last_location(Key::WifiPubKey(heartbeat.hotspot_key.clone())) + if let Some(last_location) = location_cache + .get(LocationCacheKey::WifiPubKey(heartbeat.hotspot_key.clone())) .await? { heartbeat.lat = last_location.lat; heartbeat.lon = last_location.lon; - heartbeat.location_validation_timestamp = - Some(last_location.location_validation_timestamp); + heartbeat.location_validation_timestamp = Some(last_location.timestamp); // Can't panic, previous lat and lon must be valid. hb_latlng = heartbeat.centered_latlng().unwrap(); true @@ -537,14 +534,13 @@ impl ValidatedHeartbeat { } } Some(location_validation_timestamp) => { - last_location_cache - .set_last_location( - Key::WifiPubKey(heartbeat.hotspot_key.clone()), - LastLocation::new( - location_validation_timestamp, - heartbeat.timestamp, + location_cache + .insert( + LocationCacheKey::WifiPubKey(heartbeat.hotspot_key.clone()), + LocationCacheValue::new( heartbeat.lat, heartbeat.lon, + location_validation_timestamp, ), ) .await?; diff --git a/mobile_verifier/src/heartbeats/wifi.rs b/mobile_verifier/src/heartbeats/wifi.rs index 52336a584..e6dfd8322 100644 --- a/mobile_verifier/src/heartbeats/wifi.rs +++ b/mobile_verifier/src/heartbeats/wifi.rs @@ -1,8 +1,9 @@ -use super::{process_validated_heartbeats, Heartbeat, ValidatedHeartbeat}; +use super::{ + location_cache::LocationCache, process_validated_heartbeats, Heartbeat, ValidatedHeartbeat, +}; use crate::{ coverage::{CoverageClaimTimeCache, CoverageObjectCache}, geofence::GeofenceValidator, - heartbeats::LocationCache, GatewayResolver, Settings, }; use chrono::{DateTime, Duration, Utc}; diff --git a/mobile_verifier/src/rewarder.rs b/mobile_verifier/src/rewarder.rs index 6d07428ee..b03be005c 100644 --- a/mobile_verifier/src/rewarder.rs +++ b/mobile_verifier/src/rewarder.rs @@ -2,7 +2,7 @@ use self::boosted_hex_eligibility::BoostedHexEligibility; use crate::{ boosting_oracles::db::check_for_unprocessed_data_sets, coverage, data_session, - heartbeats::{self, last_location::LocationCache, HeartbeatReward}, + heartbeats::{self, location_cache::LocationCache, HeartbeatReward}, radio_location_estimates, radio_threshold, reward_shares::{ self, CalculatedPocRewardShares, CoverageShares, DataTransferAndPocAllocatedRewardBuckets, @@ -60,6 +60,7 @@ pub struct Rewarder { reward_manifests: FileSinkClient, price_tracker: PriceTracker, speedtest_averages: FileSinkClient, + #[allow(dead_code)] location_cache: LocationCache, } diff --git a/mobile_verifier/tests/integrations/boosting_oracles.rs b/mobile_verifier/tests/integrations/boosting_oracles.rs index 55f1e58eb..b67451908 100644 --- a/mobile_verifier/tests/integrations/boosting_oracles.rs +++ b/mobile_verifier/tests/integrations/boosting_oracles.rs @@ -17,7 +17,7 @@ use mobile_config::boosted_hex_info::BoostedHexes; use mobile_verifier::{ coverage::{CoverageClaimTimeCache, CoverageObject, CoverageObjectCache}, geofence::GeofenceValidator, - heartbeats::{last_location::LocationCache, Heartbeat, HeartbeatReward, ValidatedHeartbeat}, + heartbeats::{location_cache::LocationCache, Heartbeat, HeartbeatReward, ValidatedHeartbeat}, reward_shares::CoverageShares, rewarder::boosted_hex_eligibility::BoostedHexEligibility, seniority::{Seniority, SeniorityUpdate}, diff --git a/mobile_verifier/tests/integrations/last_location.rs b/mobile_verifier/tests/integrations/last_location.rs index 723a1bbef..842a722fd 100644 --- a/mobile_verifier/tests/integrations/last_location.rs +++ b/mobile_verifier/tests/integrations/last_location.rs @@ -9,7 +9,7 @@ use mobile_verifier::{ coverage::{CoverageObject, CoverageObjectCache}, geofence::GeofenceValidator, heartbeats::{ - last_location::{Key, LocationCache}, + location_cache::{LocationCache, LocationCacheKey}, HbType, Heartbeat, ValidatedHeartbeat, }, }; @@ -126,8 +126,9 @@ async fn heartbeat_will_use_last_good_location_from_db(pool: PgPool) -> anyhow:: ); location_cache - .delete_last_location(Key::WifiPubKey(hotspot.clone())) - .await; + .remove(LocationCacheKey::WifiPubKey(hotspot.clone())) + .await?; + transaction = pool.begin().await?; validated_heartbeat_1.clone().save(&mut transaction).await?; transaction.commit().await?; diff --git a/mobile_verifier/tests/integrations/modeled_coverage.rs b/mobile_verifier/tests/integrations/modeled_coverage.rs index a4e7d8224..3a5010cef 100644 --- a/mobile_verifier/tests/integrations/modeled_coverage.rs +++ b/mobile_verifier/tests/integrations/modeled_coverage.rs @@ -19,7 +19,7 @@ use mobile_verifier::{ coverage::{CoverageClaimTimeCache, CoverageObject, CoverageObjectCache}, geofence::GeofenceValidator, heartbeats::{ - last_location::LocationCache, Heartbeat, HeartbeatReward, KeyType, ValidatedHeartbeat, + location_cache::LocationCache, Heartbeat, HeartbeatReward, KeyType, ValidatedHeartbeat, }, reward_shares::CoverageShares, rewarder::boosted_hex_eligibility::BoostedHexEligibility, From 45fda23eb300abeb0768b69a67be92032f906f63 Mon Sep 17 00:00:00 2001 From: Macpie Date: Wed, 16 Oct 2024 16:45:43 -0700 Subject: [PATCH 23/31] Make rewarder calculate distance --- .../src/heartbeats/location_cache.rs | 15 ++++ .../src/radio_location_estimates.rs | 58 ++++++++------- mobile_verifier/src/rewarder.rs | 72 ++++++++++++++++++- .../tests/integrations/hex_boosting.rs | 22 ++++-- .../tests/integrations/rewarder_poc_dc.rs | 5 +- 5 files changed, 135 insertions(+), 37 deletions(-) diff --git a/mobile_verifier/src/heartbeats/location_cache.rs b/mobile_verifier/src/heartbeats/location_cache.rs index cb9ea2422..76150bdb3 100644 --- a/mobile_verifier/src/heartbeats/location_cache.rs +++ b/mobile_verifier/src/heartbeats/location_cache.rs @@ -1,4 +1,5 @@ use chrono::{DateTime, Duration, Utc}; +use file_store::radio_location_estimates::Entity; use helium_crypto::PublicKeyBinary; use sqlx::PgPool; use std::{collections::HashMap, sync::Arc}; @@ -61,6 +62,7 @@ impl LocationCache { info!("cleaned {}", size_before - size_after); } }); + // TODO: We could spawn an hydrate from DB here? Self { pool: pool.clone(), data, @@ -71,6 +73,7 @@ impl LocationCache { { let data = self.data.lock().await; if let Some(&value) = data.get(&key) { + // TODO: When we get it timestamp more than 12h old should we remove an try to fetch new one? return Ok(Some(value)); } } @@ -82,6 +85,11 @@ impl LocationCache { } } + pub async fn get_all(&self) -> HashMap { + let data = self.data.lock().await; + data.clone() + } + pub async fn insert( &self, key: LocationCacheKey, @@ -158,3 +166,10 @@ impl LocationCache { } } } + +pub fn key_to_entity(entity: LocationCacheKey) -> Entity { + match entity { + LocationCacheKey::CbrsId(id) => Entity::CbrsId(id), + LocationCacheKey::WifiPubKey(pub_key) => Entity::WifiPubKey(pub_key), + } +} diff --git a/mobile_verifier/src/radio_location_estimates.rs b/mobile_verifier/src/radio_location_estimates.rs index 6a70f7e6d..8c8792c3e 100644 --- a/mobile_verifier/src/radio_location_estimates.rs +++ b/mobile_verifier/src/radio_location_estimates.rs @@ -1,6 +1,4 @@ -use std::str::FromStr; - -use crate::{heartbeats::HbType, sp_boosted_rewards_bans::BannedRadios, Settings}; +use crate::{heartbeats::HbType, Settings}; use chrono::{DateTime, Utc}; use file_store::{ file_info_poller::{FileInfoStream, LookbackBehavior}, @@ -23,14 +21,11 @@ use helium_proto::services::{ }; use mobile_config::client::authorization_client::AuthorizationVerifier; use rust_decimal::Decimal; -use rust_decimal_macros::dec; use sha2::{Digest, Sha256}; use sqlx::{PgPool, Pool, Postgres, Row, Transaction}; use task_manager::{ManagedTask, TaskManager}; use tokio::sync::mpsc::Receiver; -const CONFIDENCE_THRESHOLD: Decimal = dec!(0.75); - pub struct RadioLocationEstimatesDaemon { pool: Pool, authorization_verifier: AV, @@ -298,33 +293,36 @@ pub async fn clear_invalided( Ok(()) } -// This is wrong should be a get estimates but will fix later -pub async fn get_banned_radios(pool: &PgPool) -> anyhow::Result { - // TODO: Do we still want to ban any radio that is NOT in this table? - // Might be multiple per radio - // check assertion in circle as well - sqlx::query( +pub async fn get_valid_estimates( + pool: &PgPool, + radio_key: &Entity, + threshold: Decimal, +) -> anyhow::Result> { + let rows = sqlx::query( r#" - SELECT radio_type, radio_key - FROM radio_location_estimates - WHERE confidence < $1 - AND invalided_at IS NULL + SELECT radius, lat, long + FROM radio_location_estimates + WHERE radio_key = $1 + AND confidence >= $2 + AND invalided_at IS NULL "#, ) - .bind(CONFIDENCE_THRESHOLD) - .fetch(pool) - .map_err(anyhow::Error::from) - .try_fold(BannedRadios::default(), |mut set, row| async move { - let radio_type = row.get::("radio_type"); - let radio_key = row.get::("radio_key"); - match radio_type { - HbType::Wifi => set.insert_wifi(PublicKeyBinary::from_str(&radio_key)?), - HbType::Cbrs => set.insert_cbrs(radio_key), - }; - - Ok(set) - }) - .await + .bind(radio_key.to_string()) + .bind(threshold) + .fetch_all(pool) + .await?; + + let results = rows + .into_iter() + .map(|row| { + let radius: Decimal = row.get("radius"); + let lat: Decimal = row.get("lat"); + let lon: Decimal = row.get("long"); + (radius, lat, lon) + }) + .collect(); + + Ok(results) } fn entity_to_radio_type(entity: &Entity) -> HbType { diff --git a/mobile_verifier/src/rewarder.rs b/mobile_verifier/src/rewarder.rs index b03be005c..ad95476f5 100644 --- a/mobile_verifier/src/rewarder.rs +++ b/mobile_verifier/src/rewarder.rs @@ -2,7 +2,11 @@ use self::boosted_hex_eligibility::BoostedHexEligibility; use crate::{ boosting_oracles::db::check_for_unprocessed_data_sets, coverage, data_session, - heartbeats::{self, location_cache::LocationCache, HeartbeatReward}, + heartbeats::{ + self, + location_cache::{self, LocationCache}, + HeartbeatReward, + }, radio_location_estimates, radio_threshold, reward_shares::{ self, CalculatedPocRewardShares, CoverageShares, DataTransferAndPocAllocatedRewardBuckets, @@ -21,6 +25,7 @@ use file_store::{ traits::{FileSinkCommitStrategy, FileSinkRollTime, FileSinkWriteExt, TimestampEncode}, }; use futures_util::TryFutureExt; +use h3o::{LatLng, Resolution}; use helium_proto::{ reward_manifest::RewardData::MobileRewardData, services::poc_mobile::{ @@ -268,6 +273,7 @@ where &self.hex_service_client, &self.mobile_rewards, &self.speedtest_averages, + &self.location_cache, reward_period, mobile_bone_price, ) @@ -362,6 +368,7 @@ pub async fn reward_poc_and_dc( hex_service_client: &impl HexBoostingInfoResolver, mobile_rewards: &FileSinkClient, speedtest_avg_sink: &FileSinkClient, + location_cache: &LocationCache, reward_period: &Range>, mobile_bone_price: Decimal, ) -> anyhow::Result { @@ -399,6 +406,7 @@ pub async fn reward_poc_and_dc( speedtest_avg_sink, reward_period, reward_shares, + location_cache, ) .await?; @@ -425,6 +433,7 @@ async fn reward_poc( speedtest_avg_sink: &FileSinkClient, reward_period: &Range>, reward_shares: DataTransferAndPocAllocatedRewardBuckets, + location_cache: &LocationCache, ) -> anyhow::Result<(Decimal, CalculatedPocRewardShares)> { let heartbeats = HeartbeatReward::validated(pool, reward_period); let speedtest_averages = @@ -451,6 +460,27 @@ async fn reward_poc( ) .await?; + { + let locations = location_cache.get_all().await; + for (key, value) in locations.iter() { + let entity = location_cache::key_to_entity(key.clone()); + let estimates = + radio_location_estimates::get_valid_estimates(pool, &entity, dec!(0.75)).await?; + if estimates.is_empty() { + // TODO we ban that key + todo!() + } else { + match is_within_radius(value.lat, value.lon, estimates) { + Ok(true) => todo!(), + // TODO we ban that key + Ok(false) => todo!(), + // TODO we ban that key + Err(_) => todo!(), + } + } + } + } + let coverage_shares = CoverageShares::new( pool, heartbeats, @@ -497,6 +527,46 @@ async fn reward_poc( Ok((unallocated_poc_amount, calculated_poc_rewards_per_share)) } +fn is_within_radius( + lat_a: f64, + lon_a: f64, + comparators: Vec<(Decimal, Decimal, Decimal)>, +) -> anyhow::Result { + let resolution = Resolution::Twelve; + + let point_a = + LatLng::new(lat_a, lon_a).map_err(|e| anyhow::anyhow!("Invalid LatLng for A: {}", e))?; + let h3_index_a = point_a.to_cell(resolution); + + for (radius_meters, lat_b, lon_b) in comparators { + let lat_b_f64 = lat_b + .to_f64() + .ok_or_else(|| anyhow::anyhow!("Failed to convert lat_b to f64"))?; + let lon_b_f64 = lon_b + .to_f64() + .ok_or_else(|| anyhow::anyhow!("Failed to convert lon_b to f64"))?; + let radius_meters_f64 = radius_meters + .to_f64() + .ok_or_else(|| anyhow::anyhow!("Failed to convert radius_meters to f64"))?; + + let point_b = LatLng::new(lat_b_f64, lon_b_f64) + .map_err(|e| anyhow::anyhow!("Invalid LatLng for B: {}", e))?; + let h3_index_b = point_b.to_cell(resolution); + + let grid_distance = h3_index_a + .grid_distance(h3_index_b) + .map_err(|e| anyhow::anyhow!("Failed to calculate grid distance: {}", e))?; + + let max_grid_distance = (radius_meters_f64 / 9.0).round() as i32; + + if grid_distance <= max_grid_distance { + return Ok(true); + } + } + + Ok(false) +} + pub async fn reward_dc( mobile_rewards: &FileSinkClient, reward_period: &Range>, diff --git a/mobile_verifier/tests/integrations/hex_boosting.rs b/mobile_verifier/tests/integrations/hex_boosting.rs index ae11f4363..c9a9081c1 100644 --- a/mobile_verifier/tests/integrations/hex_boosting.rs +++ b/mobile_verifier/tests/integrations/hex_boosting.rs @@ -18,7 +18,7 @@ use mobile_config::boosted_hex_info::BoostedHexInfo; use mobile_verifier::{ cell_type::CellType, coverage::CoverageObject, - heartbeats::{HbType, Heartbeat, ValidatedHeartbeat}, + heartbeats::{location_cache::LocationCache, HbType, Heartbeat, ValidatedHeartbeat}, radio_threshold, reward_shares, rewarder, speedtests, }; use rust_decimal::prelude::*; @@ -137,6 +137,8 @@ async fn test_poc_with_boosted_hexes(pool: PgPool) -> anyhow::Result<()> { .to_u64() .unwrap(); + let location_cache = LocationCache::new(&pool); + let (_, rewards) = tokio::join!( // run rewards for poc and dc rewarder::reward_poc_and_dc( @@ -144,6 +146,7 @@ async fn test_poc_with_boosted_hexes(pool: PgPool) -> anyhow::Result<()> { &hex_boosting_client, &mobile_rewards_client, &speedtest_avg_client, + &location_cache, &epoch, dec!(0.0001) ), @@ -320,7 +323,7 @@ async fn test_poc_boosted_hexes_thresholds_not_met(pool: PgPool) -> anyhow::Resu ]; let hex_boosting_client = MockHexBoostingClient::new(boosted_hexes); - + let location_cache = LocationCache::new(&pool); let (_, rewards) = tokio::join!( // run rewards for poc and dc rewarder::reward_poc_and_dc( @@ -328,6 +331,7 @@ async fn test_poc_boosted_hexes_thresholds_not_met(pool: PgPool) -> anyhow::Resu &hex_boosting_client, &mobile_rewards_client, &speedtest_avg_client, + &location_cache, &epoch, dec!(0.0001) ), @@ -483,6 +487,9 @@ async fn test_poc_with_multi_coverage_boosted_hexes(pool: PgPool) -> anyhow::Res .unwrap(); let hex_boosting_client = MockHexBoostingClient::new(boosted_hexes); + + let location_cache = LocationCache::new(&pool); + let (_, rewards) = tokio::join!( // run rewards for poc and dc rewarder::reward_poc_and_dc( @@ -490,6 +497,7 @@ async fn test_poc_with_multi_coverage_boosted_hexes(pool: PgPool) -> anyhow::Res &hex_boosting_client, &mobile_rewards_client, &speedtest_avg_client, + &location_cache, &epoch, dec!(0.0001) ), @@ -657,7 +665,7 @@ async fn test_expired_boosted_hex(pool: PgPool) -> anyhow::Result<()> { ]; let hex_boosting_client = MockHexBoostingClient::new(boosted_hexes); - + let location_cache = LocationCache::new(&pool); let (_, rewards) = tokio::join!( // run rewards for poc and dc rewarder::reward_poc_and_dc( @@ -665,6 +673,7 @@ async fn test_expired_boosted_hex(pool: PgPool) -> anyhow::Result<()> { &hex_boosting_client, &mobile_rewards_client, &speedtest_avg_client, + &location_cache, &epoch, dec!(0.0001) ), @@ -790,6 +799,7 @@ async fn test_reduced_location_score_with_boosted_hexes(pool: PgPool) -> anyhow: .to_u64() .unwrap(); + let location_cache = LocationCache::new(&pool); let (_, rewards) = tokio::join!( // run rewards for poc and dc rewarder::reward_poc_and_dc( @@ -797,6 +807,7 @@ async fn test_reduced_location_score_with_boosted_hexes(pool: PgPool) -> anyhow: &hex_boosting_client, &mobile_rewards_client, &speedtest_avg_client, + &location_cache, &epoch, dec!(0.0001) ), @@ -969,7 +980,7 @@ async fn test_distance_from_asserted_removes_boosting_but_not_location_trust( let total_poc_emissions = reward_shares::get_scheduled_tokens_for_poc(epoch_duration) .to_u64() .unwrap(); - + let location_cache = LocationCache::new(&pool); let (_, rewards) = tokio::join!( // run rewards for poc and dc rewarder::reward_poc_and_dc( @@ -977,6 +988,7 @@ async fn test_distance_from_asserted_removes_boosting_but_not_location_trust( &hex_boosting_client, &mobile_rewards_client, &speedtest_avg_client, + &location_cache, &epoch, dec!(0.0001) ), @@ -1175,6 +1187,7 @@ async fn test_poc_with_cbrs_and_multi_coverage_boosted_hexes(pool: PgPool) -> an .to_u64() .unwrap(); + let location_cache = LocationCache::new(&pool); let (_, rewards) = tokio::join!( // run rewards for poc and dc rewarder::reward_poc_and_dc( @@ -1182,6 +1195,7 @@ async fn test_poc_with_cbrs_and_multi_coverage_boosted_hexes(pool: PgPool) -> an &hex_boosting_client, &mobile_rewards_client, &speedtest_avg_client, + &location_cache, &epoch, dec!(0.0001) ), diff --git a/mobile_verifier/tests/integrations/rewarder_poc_dc.rs b/mobile_verifier/tests/integrations/rewarder_poc_dc.rs index e26af88b8..ed9cc3c4f 100644 --- a/mobile_verifier/tests/integrations/rewarder_poc_dc.rs +++ b/mobile_verifier/tests/integrations/rewarder_poc_dc.rs @@ -13,7 +13,7 @@ use mobile_verifier::{ cell_type::CellType, coverage::CoverageObject, data_session, - heartbeats::{HbType, Heartbeat, ValidatedHeartbeat}, + heartbeats::{location_cache::LocationCache, HbType, Heartbeat, ValidatedHeartbeat}, reward_shares, rewarder, speedtests, }; use rust_decimal::prelude::*; @@ -44,7 +44,7 @@ async fn test_poc_and_dc_rewards(pool: PgPool) -> anyhow::Result<()> { let boosted_hexes = vec![]; let hex_boosting_client = MockHexBoostingClient::new(boosted_hexes); - + let location_cache = LocationCache::new(&pool); let (_, rewards) = tokio::join!( // run rewards for poc and dc rewarder::reward_poc_and_dc( @@ -52,6 +52,7 @@ async fn test_poc_and_dc_rewards(pool: PgPool) -> anyhow::Result<()> { &hex_boosting_client, &mobile_rewards_client, &speedtest_avg_client, + &location_cache, &epoch, dec!(0.0001) ), From 42a3bd0ef9598921dfac8fb0c91d2bb33457eb66 Mon Sep 17 00:00:00 2001 From: Macpie Date: Wed, 16 Oct 2024 17:07:38 -0700 Subject: [PATCH 24/31] Update long to lon --- Cargo.lock | 4 ++-- file_store/src/radio_location_estimates.rs | 6 +++--- ingest/tests/mobile_ingest.rs | 2 +- .../migrations/38_radio_location_estimates.sql | 2 +- mobile_verifier/src/radio_location_estimates.rs | 8 ++++---- mobile_verifier/src/rewarder.rs | 16 ++++++++-------- .../integrations/radio_location_estimates.rs | 8 ++++---- 7 files changed, 23 insertions(+), 23 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index be41306c0..937ca3ed8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1617,7 +1617,7 @@ checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" [[package]] name = "beacon" version = "0.1.0" -source = "git+https://github.com/helium/proto?branch=macpie/radio_location_estimates#4cefc5d337a4dddebd3fa3896b3c29f12e86f1ef" +source = "git+https://github.com/helium/proto?branch=macpie/radio_location_estimates#289b2095198b63ed5fb7e69dea1c3b8e1a1cb991" dependencies = [ "base64 0.21.7", "byteorder", @@ -3882,7 +3882,7 @@ dependencies = [ [[package]] name = "helium-proto" version = "0.1.0" -source = "git+https://github.com/helium/proto?branch=macpie/radio_location_estimates#4cefc5d337a4dddebd3fa3896b3c29f12e86f1ef" +source = "git+https://github.com/helium/proto?branch=macpie/radio_location_estimates#289b2095198b63ed5fb7e69dea1c3b8e1a1cb991" dependencies = [ "bytes", "prost", diff --git a/file_store/src/radio_location_estimates.rs b/file_store/src/radio_location_estimates.rs index 25fc6d518..77e75c8f5 100644 --- a/file_store/src/radio_location_estimates.rs +++ b/file_store/src/radio_location_estimates.rs @@ -110,7 +110,7 @@ impl TryFrom for RadioLocationEstimatesReq { pub struct RadioLocationEstimate { pub radius: Decimal, pub lat: Decimal, - pub long: Decimal, + pub lon: Decimal, pub confidence: Decimal, pub events: Vec, } @@ -120,7 +120,7 @@ impl From for RadioLocationEstimateV1 { RadioLocationEstimateV1 { radius: Some(to_proto_decimal(rle.radius)), lat: Some(to_proto_decimal(rle.lat)), - long: Some(to_proto_decimal(rle.long)), + lon: Some(to_proto_decimal(rle.lon)), confidence: Some(to_proto_decimal(rle.confidence)), events: rle.events.into_iter().map(|e| e.into()).collect(), } @@ -133,7 +133,7 @@ impl TryFrom for RadioLocationEstimate { Ok(Self { radius: to_rust_decimal(estimate.radius.unwrap()), lat: to_rust_decimal(estimate.lat.unwrap()), - long: to_rust_decimal(estimate.long.unwrap()), + lon: to_rust_decimal(estimate.lon.unwrap()), confidence: to_rust_decimal(estimate.confidence.unwrap()), events: estimate .events diff --git a/ingest/tests/mobile_ingest.rs b/ingest/tests/mobile_ingest.rs index f0597b75e..25c3ba7e5 100644 --- a/ingest/tests/mobile_ingest.rs +++ b/ingest/tests/mobile_ingest.rs @@ -51,7 +51,7 @@ async fn submit_radio_location_estimates() -> anyhow::Result<()> { let estimates = vec![RadioLocationEstimateV1 { radius: to_proto_decimal(2.0), lat: to_proto_decimal(41.41208), - long: to_proto_decimal(-122.19288), + lon: to_proto_decimal(-122.19288), confidence: to_proto_decimal(0.75), events: vec![RleEventV1 { id: "event_1".to_string(), diff --git a/mobile_verifier/migrations/38_radio_location_estimates.sql b/mobile_verifier/migrations/38_radio_location_estimates.sql index e9ddbd176..5b90186d1 100644 --- a/mobile_verifier/migrations/38_radio_location_estimates.sql +++ b/mobile_verifier/migrations/38_radio_location_estimates.sql @@ -5,7 +5,7 @@ CREATE TABLE IF NOT EXISTS radio_location_estimates ( received_timestamp TIMESTAMPTZ NOT NULL, radius DECIMAL NOT NULL, lat DECIMAL NOT NULL, - long DECIMAL NOT NULL, + lon DECIMAL NOT NULL, confidence DECIMAL NOT NULL, invalided_at TIMESTAMPTZ DEFAULT NULL, inserted_at TIMESTAMPTZ DEFAULT now(), diff --git a/mobile_verifier/src/radio_location_estimates.rs b/mobile_verifier/src/radio_location_estimates.rs index 8c8792c3e..f1e7c335d 100644 --- a/mobile_verifier/src/radio_location_estimates.rs +++ b/mobile_verifier/src/radio_location_estimates.rs @@ -237,12 +237,12 @@ async fn insert_estimate( ) -> Result<(), sqlx::Error> { let radius = estimate.radius; let lat = estimate.lat; - let long = estimate.long; + let long = estimate.lon; let hashed_key = hash_key(entity, received_timestamp, radius, lat, long); sqlx::query( r#" - INSERT INTO radio_location_estimates (hashed_key, radio_type, radio_key, received_timestamp, radius, lat, long, confidence) + INSERT INTO radio_location_estimates (hashed_key, radio_type, radio_key, received_timestamp, radius, lat, lon, confidence) VALUES ($1, $2, $3, $4, $5, $6, $7, $8) ON CONFLICT (hashed_key) DO NOTHING "#, @@ -300,7 +300,7 @@ pub async fn get_valid_estimates( ) -> anyhow::Result> { let rows = sqlx::query( r#" - SELECT radius, lat, long + SELECT radius, lat, lon FROM radio_location_estimates WHERE radio_key = $1 AND confidence >= $2 @@ -317,7 +317,7 @@ pub async fn get_valid_estimates( .map(|row| { let radius: Decimal = row.get("radius"); let lat: Decimal = row.get("lat"); - let lon: Decimal = row.get("long"); + let lon: Decimal = row.get("lon"); (radius, lat, lon) }) .collect(); diff --git a/mobile_verifier/src/rewarder.rs b/mobile_verifier/src/rewarder.rs index ad95476f5..d571eaf96 100644 --- a/mobile_verifier/src/rewarder.rs +++ b/mobile_verifier/src/rewarder.rs @@ -528,28 +528,28 @@ async fn reward_poc( } fn is_within_radius( - lat_a: f64, - lon_a: f64, + loc_lat: f64, + loc_lon: f64, comparators: Vec<(Decimal, Decimal, Decimal)>, ) -> anyhow::Result { let resolution = Resolution::Twelve; - let point_a = - LatLng::new(lat_a, lon_a).map_err(|e| anyhow::anyhow!("Invalid LatLng for A: {}", e))?; + let point_a = LatLng::new(loc_lat, loc_lon) + .map_err(|e| anyhow::anyhow!("Invalid LatLng for A: {}", e))?; let h3_index_a = point_a.to_cell(resolution); - for (radius_meters, lat_b, lon_b) in comparators { - let lat_b_f64 = lat_b + for (radius_meters, lat, lon) in comparators { + let lat_f64 = lat .to_f64() .ok_or_else(|| anyhow::anyhow!("Failed to convert lat_b to f64"))?; - let lon_b_f64 = lon_b + let lon_f64 = lon .to_f64() .ok_or_else(|| anyhow::anyhow!("Failed to convert lon_b to f64"))?; let radius_meters_f64 = radius_meters .to_f64() .ok_or_else(|| anyhow::anyhow!("Failed to convert radius_meters to f64"))?; - let point_b = LatLng::new(lat_b_f64, lon_b_f64) + let point_b = LatLng::new(lat_f64, lon_f64) .map_err(|e| anyhow::anyhow!("Invalid LatLng for B: {}", e))?; let h3_index_b = point_b.to_cell(resolution); diff --git a/mobile_verifier/tests/integrations/radio_location_estimates.rs b/mobile_verifier/tests/integrations/radio_location_estimates.rs index c0d64bb4c..9c0c43cfa 100644 --- a/mobile_verifier/tests/integrations/radio_location_estimates.rs +++ b/mobile_verifier/tests/integrations/radio_location_estimates.rs @@ -114,7 +114,7 @@ fn file_info_stream() -> ( estimates: vec![RadioLocationEstimate { radius: rust_decimal::Decimal::from_f32(0.1).unwrap(), lat: rust_decimal::Decimal::from_f32(0.1).unwrap(), - long: rust_decimal::Decimal::from_f32(-0.1).unwrap(), + lon: rust_decimal::Decimal::from_f32(-0.1).unwrap(), confidence: rust_decimal::Decimal::from_f32(0.1).unwrap(), events: vec![RadioLocationEstimateEvent { id: "event_1".to_string(), @@ -132,7 +132,7 @@ fn file_info_stream() -> ( estimates: vec![RadioLocationEstimate { radius: rust_decimal::Decimal::from_f32(0.2).unwrap(), lat: rust_decimal::Decimal::from_f32(0.2).unwrap(), - long: rust_decimal::Decimal::from_f32(-0.2).unwrap(), + lon: rust_decimal::Decimal::from_f32(-0.2).unwrap(), confidence: rust_decimal::Decimal::from_f32(0.2).unwrap(), events: vec![RadioLocationEstimateEvent { id: "event_1".to_string(), @@ -170,7 +170,7 @@ fn compare_report_and_estimate( report.received_timestamp, report.report.estimates[0].radius, report.report.estimates[0].lat, - report.report.estimates[0].long + report.report.estimates[0].lon ), estimate.hashed_key ); @@ -182,7 +182,7 @@ fn compare_report_and_estimate( )); assert_eq!(report.report.estimates[0].radius, estimate.radius); assert_eq!(report.report.estimates[0].lat, estimate.lat); - assert_eq!(report.report.estimates[0].long, estimate.long); + assert_eq!(report.report.estimates[0].lon, estimate.long); assert_eq!(report.report.estimates[0].confidence, estimate.confidence); if should_be_valid { From e3ad81f146d0cb6dce490d149e2e53538af4cf72 Mon Sep 17 00:00:00 2001 From: Macpie Date: Wed, 16 Oct 2024 17:11:33 -0700 Subject: [PATCH 25/31] Improve query --- mobile_verifier/src/radio_location_estimates.rs | 1 + mobile_verifier/src/rewarder.rs | 2 ++ 2 files changed, 3 insertions(+) diff --git a/mobile_verifier/src/radio_location_estimates.rs b/mobile_verifier/src/radio_location_estimates.rs index f1e7c335d..472a4e8d3 100644 --- a/mobile_verifier/src/radio_location_estimates.rs +++ b/mobile_verifier/src/radio_location_estimates.rs @@ -305,6 +305,7 @@ pub async fn get_valid_estimates( WHERE radio_key = $1 AND confidence >= $2 AND invalided_at IS NULL + ORDER BY radius DESC, confidence DESC "#, ) .bind(radio_key.to_string()) diff --git a/mobile_verifier/src/rewarder.rs b/mobile_verifier/src/rewarder.rs index d571eaf96..994ddef4b 100644 --- a/mobile_verifier/src/rewarder.rs +++ b/mobile_verifier/src/rewarder.rs @@ -464,6 +464,8 @@ async fn reward_poc( let locations = location_cache.get_all().await; for (key, value) in locations.iter() { let entity = location_cache::key_to_entity(key.clone()); + // Estimates are ordered by bigger radius first it should allow us to do less calculation + // and find a match faster let estimates = radio_location_estimates::get_valid_estimates(pool, &entity, dec!(0.75)).await?; if estimates.is_empty() { From 0b6352272db903067a48947d9a866af8b2454f24 Mon Sep 17 00:00:00 2001 From: Macpie Date: Wed, 16 Oct 2024 17:13:49 -0700 Subject: [PATCH 26/31] Comments --- mobile_verifier/src/rewarder.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/mobile_verifier/src/rewarder.rs b/mobile_verifier/src/rewarder.rs index 994ddef4b..15e3d959d 100644 --- a/mobile_verifier/src/rewarder.rs +++ b/mobile_verifier/src/rewarder.rs @@ -461,6 +461,7 @@ async fn reward_poc( .await?; { + // TODO: Maybe we should hydrate cache on start to avoid banning too many hotspot let locations = location_cache.get_all().await; for (key, value) in locations.iter() { let entity = location_cache::key_to_entity(key.clone()); From d17932af2adab71d8e22b12770022e8f876cb3ad Mon Sep 17 00:00:00 2001 From: Macpie Date: Wed, 16 Oct 2024 17:19:46 -0700 Subject: [PATCH 27/31] Update cbrs id --- mobile_verifier/src/heartbeats/mod.rs | 29 +++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/mobile_verifier/src/heartbeats/mod.rs b/mobile_verifier/src/heartbeats/mod.rs index 6938314fc..1a0bd5019 100644 --- a/mobile_verifier/src/heartbeats/mod.rs +++ b/mobile_verifier/src/heartbeats/mod.rs @@ -497,6 +497,35 @@ impl ValidatedHeartbeat { )), // TODO do we get there when CBRS? // Should I then update location form here then? + GatewayResolution::GatewayNotFound if heartbeat.hb_type == HbType::Cbrs => { + match ( + heartbeat.location_validation_timestamp, + heartbeat.cbsd_id.clone(), + ) { + (Some(location_validation_timestamp), Some(cbsd_id)) => { + location_cache + .insert( + LocationCacheKey::CbrsId(cbsd_id), + LocationCacheValue::new( + heartbeat.lat, + heartbeat.lon, + location_validation_timestamp, + ), + ) + .await?; + } + (_, _) => (), + }; + + Ok(Self::new( + heartbeat, + cell_type, + dec!(0), + None, + Some(coverage_object.meta), + proto::HeartbeatValidity::GatewayNotFound, + )) + } GatewayResolution::GatewayNotFound => Ok(Self::new( heartbeat, cell_type, From 28df64055a19a11741709b11ab2d7d7ef34448e4 Mon Sep 17 00:00:00 2001 From: Macpie Date: Wed, 16 Oct 2024 19:04:57 -0700 Subject: [PATCH 28/31] fmt clippy test --- mobile_verifier/src/cli/server.rs | 4 ++- mobile_verifier/src/heartbeats/mod.rs | 25 ++++++++----------- .../integrations/radio_location_estimates.rs | 8 +++--- 3 files changed, 18 insertions(+), 19 deletions(-) diff --git a/mobile_verifier/src/cli/server.rs b/mobile_verifier/src/cli/server.rs index 9b6dfa3fe..9e1624039 100644 --- a/mobile_verifier/src/cli/server.rs +++ b/mobile_verifier/src/cli/server.rs @@ -3,7 +3,9 @@ use crate::{ coverage::{new_coverage_object_notification_channel, CoverageDaemon}, data_session::DataSessionIngestor, geofence::Geofence, - heartbeats::{cbrs::CbrsHeartbeatDaemon, location_cache::LocationCache, wifi::WifiHeartbeatDaemon}, + heartbeats::{ + cbrs::CbrsHeartbeatDaemon, location_cache::LocationCache, wifi::WifiHeartbeatDaemon, + }, radio_location_estimates::RadioLocationEstimatesDaemon, radio_threshold::RadioThresholdIngestor, rewarder::Rewarder, diff --git a/mobile_verifier/src/heartbeats/mod.rs b/mobile_verifier/src/heartbeats/mod.rs index 1a0bd5019..f90a9912e 100644 --- a/mobile_verifier/src/heartbeats/mod.rs +++ b/mobile_verifier/src/heartbeats/mod.rs @@ -498,23 +498,20 @@ impl ValidatedHeartbeat { // TODO do we get there when CBRS? // Should I then update location form here then? GatewayResolution::GatewayNotFound if heartbeat.hb_type == HbType::Cbrs => { - match ( + if let (Some(location_validation_timestamp), Some(cbsd_id)) = ( heartbeat.location_validation_timestamp, heartbeat.cbsd_id.clone(), ) { - (Some(location_validation_timestamp), Some(cbsd_id)) => { - location_cache - .insert( - LocationCacheKey::CbrsId(cbsd_id), - LocationCacheValue::new( - heartbeat.lat, - heartbeat.lon, - location_validation_timestamp, - ), - ) - .await?; - } - (_, _) => (), + location_cache + .insert( + LocationCacheKey::CbrsId(cbsd_id), + LocationCacheValue::new( + heartbeat.lat, + heartbeat.lon, + location_validation_timestamp, + ), + ) + .await?; }; Ok(Self::new( diff --git a/mobile_verifier/tests/integrations/radio_location_estimates.rs b/mobile_verifier/tests/integrations/radio_location_estimates.rs index 9c0c43cfa..7f5a8a22c 100644 --- a/mobile_verifier/tests/integrations/radio_location_estimates.rs +++ b/mobile_verifier/tests/integrations/radio_location_estimates.rs @@ -182,7 +182,7 @@ fn compare_report_and_estimate( )); assert_eq!(report.report.estimates[0].radius, estimate.radius); assert_eq!(report.report.estimates[0].lat, estimate.lat); - assert_eq!(report.report.estimates[0].lon, estimate.long); + assert_eq!(report.report.estimates[0].lon, estimate.lon); assert_eq!(report.report.estimates[0].confidence, estimate.confidence); if should_be_valid { @@ -199,7 +199,7 @@ pub struct RadioLocationEstimateDB { pub received_timestamp: DateTime, pub radius: rust_decimal::Decimal, pub lat: rust_decimal::Decimal, - pub long: rust_decimal::Decimal, + pub lon: rust_decimal::Decimal, pub confidence: rust_decimal::Decimal, pub invalided_at: Option>, } @@ -209,7 +209,7 @@ pub async fn select_radio_location_estimates( ) -> anyhow::Result> { let rows = sqlx::query( r#" - SELECT hashed_key, radio_key, hashed_key, received_timestamp, radius, lat, long, confidence, invalided_at + SELECT hashed_key, radio_key, hashed_key, received_timestamp, radius, lat, lon, confidence, invalided_at FROM radio_location_estimates ORDER BY received_timestamp ASC "#, @@ -225,7 +225,7 @@ pub async fn select_radio_location_estimates( received_timestamp: row.get("received_timestamp"), radius: row.get("radius"), lat: row.get("lat"), - long: row.get("long"), + lon: row.get("lon"), confidence: row.get("confidence"), invalided_at: row.try_get("invalided_at").ok(), }) From eeea8e9e8e0ce283fc6864305ceb3682b9de2c41 Mon Sep 17 00:00:00 2001 From: Macpie Date: Wed, 16 Oct 2024 19:08:19 -0700 Subject: [PATCH 29/31] Fix heartbeat_does_not_use_last_good_location_when_more_than_12_hours --- mobile_verifier/src/heartbeats/location_cache.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/mobile_verifier/src/heartbeats/location_cache.rs b/mobile_verifier/src/heartbeats/location_cache.rs index 76150bdb3..1b8d1b0b3 100644 --- a/mobile_verifier/src/heartbeats/location_cache.rs +++ b/mobile_verifier/src/heartbeats/location_cache.rs @@ -73,8 +73,13 @@ impl LocationCache { { let data = self.data.lock().await; if let Some(&value) = data.get(&key) { - // TODO: When we get it timestamp more than 12h old should we remove an try to fetch new one? - return Ok(Some(value)); + let now = Utc::now(); + let twelve_hours_ago = now - Duration::hours(12); + if value.timestamp > twelve_hours_ago { + return Ok(None); + } else { + return Ok(Some(value)); + } } } match key { From 1004ffb971aec42162561ac7650c461b50977930 Mon Sep 17 00:00:00 2001 From: Macpie Date: Thu, 17 Oct 2024 15:02:02 -0700 Subject: [PATCH 30/31] - Rework location cache to have 2 hashmap and not cleanup anymore - Remove location_validation_timestamp from cbrs_heartbeats - Handle GatewayResolution::GatewayNotAsserted for cbrs and insert in cache --- .../migrations/39_update_cbrs_hearbeats.sql | 1 - .../src/heartbeats/location_cache.rs | 107 ++++++++++-------- mobile_verifier/src/heartbeats/mod.rs | 69 ++++++----- mobile_verifier/src/rewarder.rs | 4 +- .../tests/integrations/last_location.rs | 8 +- 5 files changed, 96 insertions(+), 93 deletions(-) diff --git a/mobile_verifier/migrations/39_update_cbrs_hearbeats.sql b/mobile_verifier/migrations/39_update_cbrs_hearbeats.sql index 6a1056a3d..e5f55707f 100644 --- a/mobile_verifier/migrations/39_update_cbrs_hearbeats.sql +++ b/mobile_verifier/migrations/39_update_cbrs_hearbeats.sql @@ -1,4 +1,3 @@ ALTER TABLE cbrs_heartbeats -ADD COLUMN location_validation_timestamp TIMESTAMPTZ, ADD COLUMN lat DOUBLE PRECISION NOT NULL DEFAULT 0.0, ADD COLUMN lon DOUBLE PRECISION NOT NULL DEFAULT 0.0; \ No newline at end of file diff --git a/mobile_verifier/src/heartbeats/location_cache.rs b/mobile_verifier/src/heartbeats/location_cache.rs index 1b8d1b0b3..6d62beacd 100644 --- a/mobile_verifier/src/heartbeats/location_cache.rs +++ b/mobile_verifier/src/heartbeats/location_cache.rs @@ -3,8 +3,7 @@ use file_store::radio_location_estimates::Entity; use helium_crypto::PublicKeyBinary; use sqlx::PgPool; use std::{collections::HashMap, sync::Arc}; -use tokio::sync::Mutex; -use tracing::info; +use tokio::sync::{Mutex, MutexGuard}; #[derive(Debug, Clone, Hash, Eq, PartialEq)] pub enum LocationCacheKey { @@ -29,56 +28,57 @@ impl LocationCacheValue { } } +type LocationCacheData = HashMap; + /// A cache WiFi/Cbrs heartbeat locations #[derive(Clone)] pub struct LocationCache { pool: PgPool, - data: Arc>>, + wifi: Arc>, + cbrs: Arc>, } impl LocationCache { pub fn new(pool: &PgPool) -> Self { - let data = Arc::new(Mutex::new( - HashMap::::new(), - )); - let data_clone = data.clone(); - tokio::spawn(async move { - loop { - // Sleep 1 hour - let duration = core::time::Duration::from_secs(60 * 60); - tokio::time::sleep(duration).await; - - let now = Utc::now(); - // Set the 12-hour threshold - let twelve_hours_ago = now - Duration::hours(12); - - let mut data = data_clone.lock().await; - let size_before = data.len() as f64; - - // Retain only values that are within the last 12 hours - data.retain(|_, v| v.timestamp > twelve_hours_ago); - - let size_after = data.len() as f64; - info!("cleaned {}", size_before - size_after); - } - }); + let wifi = Arc::new(Mutex::new(HashMap::new())); + let cbrs = Arc::new(Mutex::new(HashMap::new())); // TODO: We could spawn an hydrate from DB here? Self { pool: pool.clone(), - data, + wifi, + cbrs, } } pub async fn get(&self, key: LocationCacheKey) -> anyhow::Result> { { - let data = self.data.lock().await; + let data = self.key_to_lock(&key).await; + if let Some(&value) = data.get(&key) { + return Ok(Some(value)); + } + } + match key { + LocationCacheKey::WifiPubKey(pub_key_bin) => { + self.fetch_wifi_and_insert(pub_key_bin).await + } + LocationCacheKey::CbrsId(id) => self.fetch_cbrs_and_insert(id).await, + } + } + + pub async fn get_recent( + &self, + key: LocationCacheKey, + when: Duration, + ) -> anyhow::Result> { + { + let data = self.key_to_lock(&key).await; if let Some(&value) = data.get(&key) { let now = Utc::now(); - let twelve_hours_ago = now - Duration::hours(12); - if value.timestamp > twelve_hours_ago { - return Ok(None); - } else { + let before = now - when; + if value.timestamp > before { return Ok(Some(value)); + } else { + return Ok(None); } } } @@ -90,9 +90,15 @@ impl LocationCache { } } - pub async fn get_all(&self) -> HashMap { - let data = self.data.lock().await; - data.clone() + pub async fn get_all(&self) -> LocationCacheData { + let wifi_data = self.wifi.lock().await; + let mut wifi_data_cloned = wifi_data.clone(); + + let cbrs_data = self.cbrs.lock().await; + let cbrs_data_cloned = cbrs_data.clone(); + + wifi_data_cloned.extend(cbrs_data_cloned); + wifi_data_cloned } pub async fn insert( @@ -100,18 +106,25 @@ impl LocationCache { key: LocationCacheKey, value: LocationCacheValue, ) -> anyhow::Result<()> { - let mut data = self.data.lock().await; + let mut data = self.key_to_lock(&key).await; data.insert(key, value); Ok(()) } /// Only used for testing. pub async fn remove(&self, key: LocationCacheKey) -> anyhow::Result<()> { - let mut data = self.data.lock().await; + let mut data = self.key_to_lock(&key).await; data.remove(&key); Ok(()) } + async fn key_to_lock(&self, key: &LocationCacheKey) -> MutexGuard<'_, LocationCacheData> { + match key { + LocationCacheKey::WifiPubKey(_) => self.wifi.lock().await, + LocationCacheKey::CbrsId(_) => self.cbrs.lock().await, + } + } + async fn fetch_wifi_and_insert( &self, pub_key_bin: PublicKeyBinary, @@ -134,8 +147,9 @@ impl LocationCache { match sqlx_return { None => Ok(None), Some(value) => { - let mut data = self.data.lock().await; - data.insert(LocationCacheKey::WifiPubKey(pub_key_bin), value); + let key = LocationCacheKey::WifiPubKey(pub_key_bin); + let mut data = self.key_to_lock(&key).await; + data.insert(key, value); Ok(Some(value)) } } @@ -147,12 +161,12 @@ impl LocationCache { ) -> anyhow::Result> { let sqlx_return: Option = sqlx::query_as( r#" - SELECT lat, lon, location_validation_timestamp AS timestamp + SELECT lat, lon, latest_timestamp AS timestamp FROM cbrs_heartbeats - WHERE location_validation_timestamp IS NOT NULL - AND location_validation_timestamp >= $1 + WHERE latest_timestamp IS NOT NULL + AND latest_timestamp >= $1 AND hotspot_key = $2 - ORDER BY location_validation_timestamp DESC + ORDER BY latest_timestamp DESC LIMIT 1 "#, ) @@ -164,8 +178,9 @@ impl LocationCache { match sqlx_return { None => Ok(None), Some(value) => { - let mut data = self.data.lock().await; - data.insert(LocationCacheKey::CbrsId(cbsd_id), value); + let key = LocationCacheKey::CbrsId(cbsd_id); + let mut data = self.key_to_lock(&key).await; + data.insert(key, value); Ok(Some(value)) } } diff --git a/mobile_verifier/src/heartbeats/mod.rs b/mobile_verifier/src/heartbeats/mod.rs index f90a9912e..3ba6829e0 100644 --- a/mobile_verifier/src/heartbeats/mod.rs +++ b/mobile_verifier/src/heartbeats/mod.rs @@ -482,7 +482,6 @@ impl ValidatedHeartbeat { proto::HeartbeatValidity::UnsupportedLocation, )); } - match gateway_info_resolver .resolve_gateway(&heartbeat.hotspot_key) .await? @@ -495,34 +494,6 @@ impl ValidatedHeartbeat { Some(coverage_object.meta), proto::HeartbeatValidity::InvalidDeviceType, )), - // TODO do we get there when CBRS? - // Should I then update location form here then? - GatewayResolution::GatewayNotFound if heartbeat.hb_type == HbType::Cbrs => { - if let (Some(location_validation_timestamp), Some(cbsd_id)) = ( - heartbeat.location_validation_timestamp, - heartbeat.cbsd_id.clone(), - ) { - location_cache - .insert( - LocationCacheKey::CbrsId(cbsd_id), - LocationCacheValue::new( - heartbeat.lat, - heartbeat.lon, - location_validation_timestamp, - ), - ) - .await?; - }; - - Ok(Self::new( - heartbeat, - cell_type, - dec!(0), - None, - Some(coverage_object.meta), - proto::HeartbeatValidity::GatewayNotFound, - )) - } GatewayResolution::GatewayNotFound => Ok(Self::new( heartbeat, cell_type, @@ -531,22 +502,47 @@ impl ValidatedHeartbeat { Some(coverage_object.meta), proto::HeartbeatValidity::GatewayNotFound, )), - GatewayResolution::GatewayNotAsserted if heartbeat.hb_type == HbType::Wifi => { - Ok(Self::new( + GatewayResolution::GatewayNotAsserted => match heartbeat.hb_type { + HbType::Wifi => Ok(Self::new( heartbeat, cell_type, dec!(0), None, Some(coverage_object.meta), proto::HeartbeatValidity::GatewayNotAsserted, - )) - } + )), + HbType::Cbrs => { + if let Some(cbsd_id) = heartbeat.cbsd_id.clone() { + location_cache + .insert( + LocationCacheKey::CbrsId(cbsd_id), + LocationCacheValue::new( + heartbeat.lat, + heartbeat.lon, + heartbeat.timestamp, + ), + ) + .await?; + }; + Ok(Self::new( + heartbeat, + cell_type, + dec!(1.0), + None, + Some(coverage_object.meta), + proto::HeartbeatValidity::Valid, + )) + } + }, GatewayResolution::AssertedLocation(location) if heartbeat.hb_type == HbType::Wifi => { let asserted_latlng: LatLng = CellIndex::try_from(location)?.into(); let is_valid = match heartbeat.location_validation_timestamp { None => { if let Some(last_location) = location_cache - .get(LocationCacheKey::WifiPubKey(heartbeat.hotspot_key.clone())) + .get_recent( + LocationCacheKey::WifiPubKey(heartbeat.hotspot_key.clone()), + Duration::hours(12), + ) .await? { heartbeat.lat = last_location.lat; @@ -702,8 +698,8 @@ impl ValidatedHeartbeat { let truncated_timestamp = self.truncated_timestamp()?; sqlx::query( r#" - INSERT INTO cbrs_heartbeats (cbsd_id, hotspot_key, cell_type, latest_timestamp, truncated_timestamp, coverage_object, location_trust_score_multiplier, location_validation_timestamp, lat, lon) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) + INSERT INTO cbrs_heartbeats (cbsd_id, hotspot_key, cell_type, latest_timestamp, truncated_timestamp, coverage_object, location_trust_score_multiplier, lat, lon) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) ON CONFLICT (cbsd_id, truncated_timestamp) DO UPDATE SET latest_timestamp = EXCLUDED.latest_timestamp, coverage_object = EXCLUDED.coverage_object @@ -716,7 +712,6 @@ impl ValidatedHeartbeat { .bind(truncated_timestamp) .bind(self.heartbeat.coverage_object) .bind(self.location_trust_score_multiplier) - .bind(self.heartbeat.location_validation_timestamp) .bind(self.heartbeat.lat) .bind(self.heartbeat.lon) .execute(&mut *exec) diff --git a/mobile_verifier/src/rewarder.rs b/mobile_verifier/src/rewarder.rs index 15e3d959d..b494e32ee 100644 --- a/mobile_verifier/src/rewarder.rs +++ b/mobile_verifier/src/rewarder.rs @@ -533,7 +533,7 @@ async fn reward_poc( fn is_within_radius( loc_lat: f64, loc_lon: f64, - comparators: Vec<(Decimal, Decimal, Decimal)>, + estimates: Vec<(Decimal, Decimal, Decimal)>, ) -> anyhow::Result { let resolution = Resolution::Twelve; @@ -541,7 +541,7 @@ fn is_within_radius( .map_err(|e| anyhow::anyhow!("Invalid LatLng for A: {}", e))?; let h3_index_a = point_a.to_cell(resolution); - for (radius_meters, lat, lon) in comparators { + for (radius_meters, lat, lon) in estimates { let lat_f64 = lat .to_f64() .ok_or_else(|| anyhow::anyhow!("Failed to convert lat_b to f64"))?; diff --git a/mobile_verifier/tests/integrations/last_location.rs b/mobile_verifier/tests/integrations/last_location.rs index 842a722fd..f51650565 100644 --- a/mobile_verifier/tests/integrations/last_location.rs +++ b/mobile_verifier/tests/integrations/last_location.rs @@ -181,8 +181,7 @@ async fn heartbeat_does_not_use_last_good_location_when_more_than_12_hours( let validated_heartbeat_1 = ValidatedHeartbeat::validate( heartbeat(&hotspot, &coverage_object) - .location_validation_timestamp(Utc::now()) - .timestamp(Utc::now() - Duration::hours(12) - Duration::seconds(1)) + .location_validation_timestamp(Utc::now() - Duration::hours(12) - Duration::seconds(1)) .build(), &GatewayClientAllOwnersValid, &coverage_objects, @@ -248,11 +247,6 @@ impl HeartbeatBuilder { self } - fn timestamp(mut self, ts: DateTime) -> Self { - self.timestamp = Some(ts); - self - } - fn build(self) -> Heartbeat { let (lat, lon) = self.latlng.unwrap_or_else(|| { let lat_lng: LatLng = self From 62bde0f0a1ce2aefaffc143312e7701c9c2216e6 Mon Sep 17 00:00:00 2001 From: Macpie Date: Thu, 17 Oct 2024 15:37:50 -0700 Subject: [PATCH 31/31] Update cache to hydrate on new Remove rewarder part to put in diff PR --- mobile_verifier/src/cli/server.rs | 2 +- .../src/heartbeats/location_cache.rs | 89 +++++++++++++++++-- mobile_verifier/src/rewarder.rs | 73 +-------------- .../tests/integrations/boosting_oracles.rs | 2 +- .../tests/integrations/hex_boosting.rs | 14 +-- .../tests/integrations/last_location.rs | 6 +- .../tests/integrations/modeled_coverage.rs | 4 +- .../tests/integrations/rewarder_poc_dc.rs | 2 +- 8 files changed, 99 insertions(+), 93 deletions(-) diff --git a/mobile_verifier/src/cli/server.rs b/mobile_verifier/src/cli/server.rs index 9e1624039..bc16c483c 100644 --- a/mobile_verifier/src/cli/server.rs +++ b/mobile_verifier/src/cli/server.rs @@ -105,7 +105,7 @@ impl Cmd { let (new_coverage_obj_notifier, new_coverage_obj_notification) = new_coverage_object_notification_channel(); - let location_cache = LocationCache::new(&pool); + let location_cache = LocationCache::new(&pool).await?; TaskManager::builder() .add_task(file_upload_server) diff --git a/mobile_verifier/src/heartbeats/location_cache.rs b/mobile_verifier/src/heartbeats/location_cache.rs index 6d62beacd..9a4bb5122 100644 --- a/mobile_verifier/src/heartbeats/location_cache.rs +++ b/mobile_verifier/src/heartbeats/location_cache.rs @@ -1,8 +1,9 @@ use chrono::{DateTime, Duration, Utc}; use file_store::radio_location_estimates::Entity; +use futures::StreamExt; use helium_crypto::PublicKeyBinary; -use sqlx::PgPool; -use std::{collections::HashMap, sync::Arc}; +use sqlx::{PgPool, Row}; +use std::{collections::HashMap, str::FromStr, sync::Arc}; use tokio::sync::{Mutex, MutexGuard}; #[derive(Debug, Clone, Hash, Eq, PartialEq)] @@ -39,15 +40,18 @@ pub struct LocationCache { } impl LocationCache { - pub fn new(pool: &PgPool) -> Self { + pub async fn new(pool: &PgPool) -> anyhow::Result { let wifi = Arc::new(Mutex::new(HashMap::new())); let cbrs = Arc::new(Mutex::new(HashMap::new())); - // TODO: We could spawn an hydrate from DB here? - Self { + + hydrate_wifi(pool, &wifi).await?; + hydrate_cbrs(pool, &cbrs).await?; + + Ok(Self { pool: pool.clone(), wifi, cbrs, - } + }) } pub async fn get(&self, key: LocationCacheKey) -> anyhow::Result> { @@ -165,7 +169,7 @@ impl LocationCache { FROM cbrs_heartbeats WHERE latest_timestamp IS NOT NULL AND latest_timestamp >= $1 - AND hotspot_key = $2 + AND cbsd_id = $2 ORDER BY latest_timestamp DESC LIMIT 1 "#, @@ -187,6 +191,77 @@ impl LocationCache { } } +async fn hydrate_wifi(pool: &PgPool, mutex: &Arc>) -> anyhow::Result<()> { + let mut rows = sqlx::query( + r#" + SELECT wh.lat, wh.lon, wh.location_validation_timestamp AS timestamp, wh.hotspot_key + FROM wifi_heartbeats wh + JOIN ( + SELECT hotspot_key, MAX(location_validation_timestamp) AS max_timestamp + FROM wifi_heartbeats + WHERE location_validation_timestamp IS NOT NULL + GROUP BY hotspot_key + ) latest ON wh.hotspot_key = latest.hotspot_key + AND wh.location_validation_timestamp = latest.max_timestamp + "#, + ) + .fetch(pool); + + while let Some(row_result) = rows.next().await { + let row = row_result?; + + let hotspot_key: String = row.get("hotspot_key"); + let pub_key_bin = PublicKeyBinary::from_str(&hotspot_key)?; + let key = LocationCacheKey::WifiPubKey(pub_key_bin); + + let value = LocationCacheValue { + lat: row.get("lat"), + lon: row.get("lon"), + timestamp: row.get("timestamp"), + }; + + let mut data = mutex.lock().await; + data.insert(key.clone(), value); + } + + Ok(()) +} + +async fn hydrate_cbrs(pool: &PgPool, mutex: &Arc>) -> anyhow::Result<()> { + let mut rows = sqlx::query( + r#" + SELECT ch.lat, ch.lon, ch.latest_timestamp AS timestamp, ch.cbsd_id + FROM cbrs_heartbeats ch + JOIN ( + SELECT cbsd_id, MAX(latest_timestamp) AS max_timestamp + FROM cbrs_heartbeats + WHERE latest_timestamp IS NOT NULL + GROUP BY cbsd_id + ) latest ON ch.cbsd_id = latest.cbsd_id + AND ch.latest_timestamp = latest.max_timestamp + "#, + ) + .fetch(pool); + + while let Some(row_result) = rows.next().await { + let row = row_result?; + + let id: String = row.get("cbsd_id"); + let key = LocationCacheKey::CbrsId(id); + + let value = LocationCacheValue { + lat: row.get("lat"), + lon: row.get("lon"), + timestamp: row.get("timestamp"), + }; + + let mut data = mutex.lock().await; + data.insert(key.clone(), value); + } + + Ok(()) +} + pub fn key_to_entity(entity: LocationCacheKey) -> Entity { match entity { LocationCacheKey::CbrsId(id) => Entity::CbrsId(id), diff --git a/mobile_verifier/src/rewarder.rs b/mobile_verifier/src/rewarder.rs index b494e32ee..5f5a512d1 100644 --- a/mobile_verifier/src/rewarder.rs +++ b/mobile_verifier/src/rewarder.rs @@ -2,11 +2,7 @@ use self::boosted_hex_eligibility::BoostedHexEligibility; use crate::{ boosting_oracles::db::check_for_unprocessed_data_sets, coverage, data_session, - heartbeats::{ - self, - location_cache::{self, LocationCache}, - HeartbeatReward, - }, + heartbeats::{self, location_cache::LocationCache, HeartbeatReward}, radio_location_estimates, radio_threshold, reward_shares::{ self, CalculatedPocRewardShares, CoverageShares, DataTransferAndPocAllocatedRewardBuckets, @@ -25,7 +21,6 @@ use file_store::{ traits::{FileSinkCommitStrategy, FileSinkRollTime, FileSinkWriteExt, TimestampEncode}, }; use futures_util::TryFutureExt; -use h3o::{LatLng, Resolution}; use helium_proto::{ reward_manifest::RewardData::MobileRewardData, services::poc_mobile::{ @@ -433,7 +428,7 @@ async fn reward_poc( speedtest_avg_sink: &FileSinkClient, reward_period: &Range>, reward_shares: DataTransferAndPocAllocatedRewardBuckets, - location_cache: &LocationCache, + _location_cache: &LocationCache, ) -> anyhow::Result<(Decimal, CalculatedPocRewardShares)> { let heartbeats = HeartbeatReward::validated(pool, reward_period); let speedtest_averages = @@ -460,30 +455,6 @@ async fn reward_poc( ) .await?; - { - // TODO: Maybe we should hydrate cache on start to avoid banning too many hotspot - let locations = location_cache.get_all().await; - for (key, value) in locations.iter() { - let entity = location_cache::key_to_entity(key.clone()); - // Estimates are ordered by bigger radius first it should allow us to do less calculation - // and find a match faster - let estimates = - radio_location_estimates::get_valid_estimates(pool, &entity, dec!(0.75)).await?; - if estimates.is_empty() { - // TODO we ban that key - todo!() - } else { - match is_within_radius(value.lat, value.lon, estimates) { - Ok(true) => todo!(), - // TODO we ban that key - Ok(false) => todo!(), - // TODO we ban that key - Err(_) => todo!(), - } - } - } - } - let coverage_shares = CoverageShares::new( pool, heartbeats, @@ -530,46 +501,6 @@ async fn reward_poc( Ok((unallocated_poc_amount, calculated_poc_rewards_per_share)) } -fn is_within_radius( - loc_lat: f64, - loc_lon: f64, - estimates: Vec<(Decimal, Decimal, Decimal)>, -) -> anyhow::Result { - let resolution = Resolution::Twelve; - - let point_a = LatLng::new(loc_lat, loc_lon) - .map_err(|e| anyhow::anyhow!("Invalid LatLng for A: {}", e))?; - let h3_index_a = point_a.to_cell(resolution); - - for (radius_meters, lat, lon) in estimates { - let lat_f64 = lat - .to_f64() - .ok_or_else(|| anyhow::anyhow!("Failed to convert lat_b to f64"))?; - let lon_f64 = lon - .to_f64() - .ok_or_else(|| anyhow::anyhow!("Failed to convert lon_b to f64"))?; - let radius_meters_f64 = radius_meters - .to_f64() - .ok_or_else(|| anyhow::anyhow!("Failed to convert radius_meters to f64"))?; - - let point_b = LatLng::new(lat_f64, lon_f64) - .map_err(|e| anyhow::anyhow!("Invalid LatLng for B: {}", e))?; - let h3_index_b = point_b.to_cell(resolution); - - let grid_distance = h3_index_a - .grid_distance(h3_index_b) - .map_err(|e| anyhow::anyhow!("Failed to calculate grid distance: {}", e))?; - - let max_grid_distance = (radius_meters_f64 / 9.0).round() as i32; - - if grid_distance <= max_grid_distance { - return Ok(true); - } - } - - Ok(false) -} - pub async fn reward_dc( mobile_rewards: &FileSinkClient, reward_period: &Range>, diff --git a/mobile_verifier/tests/integrations/boosting_oracles.rs b/mobile_verifier/tests/integrations/boosting_oracles.rs index b67451908..329018dea 100644 --- a/mobile_verifier/tests/integrations/boosting_oracles.rs +++ b/mobile_verifier/tests/integrations/boosting_oracles.rs @@ -338,7 +338,7 @@ async fn test_footfall_and_urbanization_and_landtype(pool: PgPool) -> anyhow::Re let coverage_objects = CoverageObjectCache::new(&pool); let coverage_claim_time_cache = CoverageClaimTimeCache::new(); - let location_cache = LocationCache::new(&pool); + let location_cache = LocationCache::new(&pool).await?; let epoch = start..end; let mut heartbeats = pin!(ValidatedHeartbeat::validate_heartbeats( diff --git a/mobile_verifier/tests/integrations/hex_boosting.rs b/mobile_verifier/tests/integrations/hex_boosting.rs index c9a9081c1..3981969f8 100644 --- a/mobile_verifier/tests/integrations/hex_boosting.rs +++ b/mobile_verifier/tests/integrations/hex_boosting.rs @@ -137,7 +137,7 @@ async fn test_poc_with_boosted_hexes(pool: PgPool) -> anyhow::Result<()> { .to_u64() .unwrap(); - let location_cache = LocationCache::new(&pool); + let location_cache = LocationCache::new(&pool).await?; let (_, rewards) = tokio::join!( // run rewards for poc and dc @@ -323,7 +323,7 @@ async fn test_poc_boosted_hexes_thresholds_not_met(pool: PgPool) -> anyhow::Resu ]; let hex_boosting_client = MockHexBoostingClient::new(boosted_hexes); - let location_cache = LocationCache::new(&pool); + let location_cache = LocationCache::new(&pool).await?; let (_, rewards) = tokio::join!( // run rewards for poc and dc rewarder::reward_poc_and_dc( @@ -488,7 +488,7 @@ async fn test_poc_with_multi_coverage_boosted_hexes(pool: PgPool) -> anyhow::Res let hex_boosting_client = MockHexBoostingClient::new(boosted_hexes); - let location_cache = LocationCache::new(&pool); + let location_cache = LocationCache::new(&pool).await?; let (_, rewards) = tokio::join!( // run rewards for poc and dc @@ -665,7 +665,7 @@ async fn test_expired_boosted_hex(pool: PgPool) -> anyhow::Result<()> { ]; let hex_boosting_client = MockHexBoostingClient::new(boosted_hexes); - let location_cache = LocationCache::new(&pool); + let location_cache = LocationCache::new(&pool).await?; let (_, rewards) = tokio::join!( // run rewards for poc and dc rewarder::reward_poc_and_dc( @@ -799,7 +799,7 @@ async fn test_reduced_location_score_with_boosted_hexes(pool: PgPool) -> anyhow: .to_u64() .unwrap(); - let location_cache = LocationCache::new(&pool); + let location_cache = LocationCache::new(&pool).await?; let (_, rewards) = tokio::join!( // run rewards for poc and dc rewarder::reward_poc_and_dc( @@ -980,7 +980,7 @@ async fn test_distance_from_asserted_removes_boosting_but_not_location_trust( let total_poc_emissions = reward_shares::get_scheduled_tokens_for_poc(epoch_duration) .to_u64() .unwrap(); - let location_cache = LocationCache::new(&pool); + let location_cache = LocationCache::new(&pool).await?; let (_, rewards) = tokio::join!( // run rewards for poc and dc rewarder::reward_poc_and_dc( @@ -1187,7 +1187,7 @@ async fn test_poc_with_cbrs_and_multi_coverage_boosted_hexes(pool: PgPool) -> an .to_u64() .unwrap(); - let location_cache = LocationCache::new(&pool); + let location_cache = LocationCache::new(&pool).await?; let (_, rewards) = tokio::join!( // run rewards for poc and dc rewarder::reward_poc_and_dc( diff --git a/mobile_verifier/tests/integrations/last_location.rs b/mobile_verifier/tests/integrations/last_location.rs index f51650565..dad91c936 100644 --- a/mobile_verifier/tests/integrations/last_location.rs +++ b/mobile_verifier/tests/integrations/last_location.rs @@ -39,7 +39,7 @@ async fn heartbeat_uses_last_good_location_when_invalid_location( let epoch_end = epoch_start + Duration::days(2); let coverage_objects = CoverageObjectCache::new(&pool); - let location_cache = LocationCache::new(&pool); + let location_cache = LocationCache::new(&pool).await?; let mut transaction = pool.begin().await?; let coverage_object = coverage_object(&hotspot, &mut transaction).await?; @@ -101,7 +101,7 @@ async fn heartbeat_will_use_last_good_location_from_db(pool: PgPool) -> anyhow:: let epoch_end = epoch_start + Duration::days(2); let coverage_objects = CoverageObjectCache::new(&pool); - let location_cache = LocationCache::new(&pool); + let location_cache = LocationCache::new(&pool).await?; let mut transaction = pool.begin().await?; let coverage_object = coverage_object(&hotspot, &mut transaction).await?; @@ -173,7 +173,7 @@ async fn heartbeat_does_not_use_last_good_location_when_more_than_12_hours( let epoch_end = epoch_start + Duration::days(2); let coverage_objects = CoverageObjectCache::new(&pool); - let location_cache = LocationCache::new(&pool); + let location_cache = LocationCache::new(&pool).await?; let mut transaction = pool.begin().await?; let coverage_object = coverage_object(&hotspot, &mut transaction).await?; diff --git a/mobile_verifier/tests/integrations/modeled_coverage.rs b/mobile_verifier/tests/integrations/modeled_coverage.rs index 3a5010cef..b22a2449a 100644 --- a/mobile_verifier/tests/integrations/modeled_coverage.rs +++ b/mobile_verifier/tests/integrations/modeled_coverage.rs @@ -378,7 +378,7 @@ async fn process_input( ) -> anyhow::Result<()> { let coverage_objects = CoverageObjectCache::new(pool); let coverage_claim_time_cache = CoverageClaimTimeCache::new(); - let location_cache = LocationCache::new(pool); + let location_cache = LocationCache::new(pool).await?; let mut transaction = pool.begin().await?; let mut coverage_objs = pin!(CoverageObject::validate_coverage_objects( @@ -1376,7 +1376,7 @@ async fn ensure_lower_trust_score_for_distant_heartbeats(pool: PgPool) -> anyhow let max_covered_distance = 5_000; let coverage_object_cache = CoverageObjectCache::new(&pool); - let location_cache = LocationCache::new(&pool); + let location_cache = LocationCache::new(&pool).await?; let mk_heartbeat = |latlng: LatLng| WifiHeartbeatIngestReport { report: WifiHeartbeat { diff --git a/mobile_verifier/tests/integrations/rewarder_poc_dc.rs b/mobile_verifier/tests/integrations/rewarder_poc_dc.rs index ed9cc3c4f..c889cbd60 100644 --- a/mobile_verifier/tests/integrations/rewarder_poc_dc.rs +++ b/mobile_verifier/tests/integrations/rewarder_poc_dc.rs @@ -44,7 +44,7 @@ async fn test_poc_and_dc_rewards(pool: PgPool) -> anyhow::Result<()> { let boosted_hexes = vec![]; let hex_boosting_client = MockHexBoostingClient::new(boosted_hexes); - let location_cache = LocationCache::new(&pool); + let location_cache = LocationCache::new(&pool).await?; let (_, rewards) = tokio::join!( // run rewards for poc and dc rewarder::reward_poc_and_dc(